diff --git a/src/create_daily_faa_release.py b/src/create_daily_faa_release.py index 4e7adfd..b668518 100644 --- a/src/create_daily_faa_release.py +++ b/src/create_daily_faa_release.py @@ -37,13 +37,20 @@ from derive_from_faa_master_txt import convert_faa_master_txt_to_df, concat_faa_ from get_latest_release import get_latest_aircraft_faa_csv_df df_new = convert_faa_master_txt_to_df(zip_path, date_str) +# Only a genuine first run may rebuild from a single day. A rate limit, a parse error or a +# non-monotonic download_date must stop the run: this file becomes tomorrow's base, so +# silently republishing one day erases the accumulated history. try: df_base, start_date_str = get_latest_aircraft_faa_csv_df() - df_base = concat_faa_historical_df(df_base, df_new) - assert df_base['download_date'].is_monotonic_increasing, "download_date is not monotonic increasing" -except Exception as e: - print(f"No existing FAA release found, using only new data: {e}") - df_base = df_new +except FileNotFoundError as e: + print(f"No existing FAA release found, bootstrapping from today only: {e}") + df_base = None start_date_str = date_str +if df_base is not None: + df_base = concat_faa_historical_df(df_base, df_new) + assert df_base['download_date'].is_monotonic_increasing, "download_date is not monotonic increasing" +else: + df_base = df_new + df_base.to_csv(OUT_ROOT / f"openairframes_faa_{start_date_str}_{date_str}.csv", index=False) \ No newline at end of file diff --git a/src/get_latest_release.py b/src/get_latest_release.py index 0161d2e..e2738ff 100644 --- a/src/get_latest_release.py +++ b/src/get_latest_release.py @@ -139,15 +139,29 @@ def download_latest_aircraft_csv( Path to the downloaded file """ output_dir = Path(output_dir) - assets = get_latest_release_assets(repo, github_token=github_token) - try: - asset = pick_asset(assets, name_regex=r"^openairframes_faa_.*\.csv$") - except FileNotFoundError: - # Fallback to old naming pattern - asset = pick_asset(assets, name_regex=r"^openairframes_\d{4}-\d{2}-\d{2}_.*\.csv$") - saved_to = download_asset(asset, output_dir / asset.name, github_token=github_token) - print(f"Downloaded: {asset.name} ({asset.size} bytes) -> {saved_to}") - return saved_to + github_token = github_token or os.environ.get("GITHUB_TOKEN") + + for release in get_releases(repo, github_token=github_token, per_page=30): + assets = get_release_assets_from_release_data(release) + try: + asset = pick_asset(assets, name_regex=r"^openairframes_faa_.*\.csv$") + except FileNotFoundError: + try: + # Fallback to old naming pattern + asset = pick_asset(assets, name_regex=r"^openairframes_\d{4}-\d{2}-\d{2}_.*\.csv$") + except FileNotFoundError: + continue + saved_to = download_asset(asset, output_dir / asset.name, github_token=github_token) + if asset.size and saved_to.stat().st_size != asset.size: + raise RuntimeError( + f"{asset.name}: downloaded {saved_to.stat().st_size} bytes, expected {asset.size}" + ) + print(f"Downloaded: {asset.name} ({asset.size} bytes) -> {saved_to}") + return saved_to + + raise FileNotFoundError( + "No release in the last 30 releases has an asset matching 'openairframes_faa_.*\\.csv$'" + ) def get_latest_aircraft_faa_csv_df(): csv_path = download_latest_aircraft_csv()