From b6bf915cec8074f6fc244af23bfe11a58d3aab11 Mon Sep 17 00:00:00 2001 From: ggman12 Date: Tue, 17 Feb 2026 16:59:09 -0500 Subject: [PATCH] Filter rows by date in compress_parquet_part function --- src/adsb/compress_adsb_to_aircraft_data.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/adsb/compress_adsb_to_aircraft_data.py b/src/adsb/compress_adsb_to_aircraft_data.py index 4fde19b..61ac8e8 100644 --- a/src/adsb/compress_adsb_to_aircraft_data.py +++ b/src/adsb/compress_adsb_to_aircraft_data.py @@ -180,6 +180,10 @@ def compress_parquet_part(part_id: int, date: str) -> pl.DataFrame: if df.height == 0: return df + + # Filter to rows within the given date (UTC-naive). This is because sometimes adsb.lol export can have rows at 00:00:00 of next day or similar. + date_lit = pl.lit(date).str.strptime(pl.Date, "%Y-%m-%d") + df = df.filter(pl.col("time").dt.date() == date_lit) print(f"Loaded {df.height} raw records for part {part_id}, date {date}")