Merge pull request #70 from PlaneQuery/develop

Develop to Main anchildress1 changes

.github/workflows/openairframes-daily-release.yaml

@@ -2,7 +2,7 @@ name: openairframes-daily-release
 
 on:
   schedule:
-    # 6:00pm UTC every day - runs on default branch, triggers both
+    # 06:00 UTC every day - runs on default branch, triggers both
     - cron: "0 06 * * *"
   workflow_dispatch:
     inputs:

.github/workflows/update-community-prs.yaml

@@ -5,7 +5,7 @@ on:
     branches: [main]
     paths:
       - 'community/**'
-      - 'schemas/community_submission.v1.schema.json'
+      - 'schemas/**'
 
 permissions:
   contents: write

.github/workflows/validate-community-submission.yaml

@@ -23,24 +23,17 @@ jobs:
       - name: Install dependencies
         run: pip install jsonschema
 
-      - name: Debug issue body
-        run: |
-          echo "=== Issue Body ==="
-          cat << 'ISSUE_BODY_EOF'
-          ${{ github.event.issue.body }}
-          ISSUE_BODY_EOF
-
       - name: Save issue body to file
-        run: |
-          cat << 'ISSUE_BODY_EOF' > /tmp/issue_body.txt
-          ${{ github.event.issue.body }}
-          ISSUE_BODY_EOF
+        env:
+          ISSUE_BODY: ${{ github.event.issue.body }}
+        run: printf '%s' "$ISSUE_BODY" > "$RUNNER_TEMP/issue_body.txt"
 
       - name: Validate submission
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GITHUB_REPOSITORY: ${{ github.repository }}
+          ISSUE_NUMBER: ${{ github.event.issue.number }}
         run: |
           python -m src.contributions.validate_submission \
-            --issue-body-file /tmp/issue_body.txt \
-            --issue-number ${{ github.event.issue.number }}
+            --issue-body-file "$RUNNER_TEMP/issue_body.txt" \
+            --issue-number "$ISSUE_NUMBER"

AGENTS.md

@@ -0,0 +1,121 @@
+## Never interpolate `${{ github.event.* }}` into a `run:` block
+
+Actions substitutes `${{ }}` as raw text before the shell parses it, and issue bodies here are
+public and unauthenticated. Pass untrusted values through `env:` and quote them.
+
+A quoted heredoc delimiter does not save you: the body can *contain* the delimiter, close the
+heredoc early, and execute every line after it. `validate-community-submission.yaml` had that shape.
+Never fix this class of bug by escaping, sanitizing, or renaming the delimiter — move it to `env:`.
+
+## Run everything from the repo root
+
+`src/create_daily_faa_release.py` must be invoked as a **script** (`python src/create_daily_faa_release.py`).
+It uses bare sibling imports, so `-m src.create_daily_faa_release` raises `ModuleNotFoundError`.
+Everything under `src/adsb/` and `src/contributions/` is the opposite — `python -m`, package-relative.
+
+Output paths are CWD-relative.
+
+## Verification
+
+There is no test framework, linter, or packaging config. **Do not add one unprompted**, and do not
+treat "nothing broke" as verification.
+
+The ADS-B path has no cheap end-to-end check — one day of input is tens of GB. Exercise
+`compress_multi_icao_df` / `compress_df_polars` directly against a small hand-built Polars frame.
+
+**Never `gh workflow run` to test a change.** Every dispatch pulls tens of GB and fans out over date
+matrices. Reason about the YAML statically.
+
+**Never commit generated data.** The product is a GitHub Release; jobs pass state as artifacts.
+
+## ADS-B invariants
+
+- `FINAL_COLUMN_ORDER` (`compress_adsb_to_aircraft_data.py`) is the only definition of the released
+  column contract. `pl.concat` matches by **position** after `.select()`, and
+  `get_latest_release.get_latest_aircraft_adsb_csv_df` parses released CSVs against the same order —
+  so a forked copy corrupts the release with no error anywhere.
+- `load_parquet_part()` deleting its source parquet is **deliberate**: the raw part is many GB and
+  the runner would otherwise exhaust disk. Do not defer the delete to make reruns easier; that raises
+  peak disk by the size of the part.
+- A released row means "most informative observation for this ICAO on this UTC day" — non-empty
+  fields not a subset of another row's, tie-broken by signature frequency. It is not a registry record.
+- HTTP 404 is terminal in the release fetch. Restoring the retry makes the Dec-31 next-year-repo
+  probe stall ~45 minutes on a repo that does not exist yet.
+
+## Fork and upstream
+
+`src/get_latest_release.py` pins `REPO = "PlaneQuery/openairframes"` on purpose: this fork reads
+**upstream's** releases wherever it runs. Do not repoint it at `github.repository` without being asked.
+
+Upstream develops on `develop` and PRs into `main`. The daily release **deletes the existing release
+and tag** before recreating them.
+
+## Community submissions are automation-owned
+
+Merging to `community/**` or `schemas/**` force-pushes every open `community`-labeled PR branch back
+onto main. Anything you hand-edit on such a branch is destroyed on the next merge.
+
+- Never hand-author files in `community/` — the filename encodes `sha256(content)[:8]`, so an edit
+  orphans the hash and duplicates on re-approval.
+- Never invent or copy a `contributor_uuid`; it is derived from the GitHub user id.
+- Do not reintroduce a hardcoded `"main"` or `v1` filename. Both are resolved at runtime now.
+
+**A tag's JSON type is fixed by its first-ever submission and enforced forever** — emergent from
+`build_tag_type_registry` + `validate_submission`, and written nowhere in the schema. Retyping or
+renaming an existing tag breaks every future contributor, not just the current one.
+
+Dropping a `community_submission.v2.schema.json` into `schemas/` promotes it atomically across every
+reader and writer. That is a one-way door for contributors — only on explicit request.
+
+## Conventions
+
+- **Empty string, not null**, everywhere in released frames.
+- Reuse `derive_from_faa_master_txt.normalize()` for `openairframes_id`; never re-derive the format.
+- Python `3.14` for FAA/community/vendor jobs, `3.12` for ADS-B jobs (pyarrow pin + multiprocessing).
+  Deliberate. Match the surrounding job; do not unify.
+
+## References with no target — do not chase as regressions
+
+| Reference | Missing |
+|---|---|
+| `process-historical-faa.yaml` | `src/get_historical_faa.py`, `scripts/concat_csvs.py` |
+| `af-klm-fleet/package.json` → `npm run validate` | `af-klm-fleet/scripts/validate.js` |
+
+`process-historical-faa.yaml` is dead, not stale — it also uses the disabled `::set-output`.
+Repair-vs-delete is the owner's call; leave it alone unprompted.
+
+## `af-klm-fleet/` and `community-routes/` are unwired
+
+Nothing in CI touches either, and nothing consumes `community-routes/`. `af-klm-fleet/` is a vendored
+project by a different author with its own license — its aircraft model is unrelated to
+`schemas/community_submission.*`, so do not merge the two. Its `README.md` is generated by
+`generate-readme.js`; hand edits are overwritten.
+
+## Warts left standing — flag, do not silently fix
+
+- `NUMBER_PARTS` is restated by the matrix and four hand-written upload steps in
+  `adsb-to-aircraft-for-day.yaml`; changing the constant alone silently drops data. YAML cannot loop
+  upload steps and one merged artifact would force every map job to download all parts, so any real
+  fix is a restructure.
+- `MAX_WORKERS = OS_CPU_COUNT if OS_CPU_COUNT > 4 else 1` collapses to a single worker on a ≤4-core
+  runner, shrinking `files_per_batch` with it. Possibly intentional memory control — do not raise it
+  without measuring peak RSS on the target runner.
+- `update-community-prs.yaml` runs `regenerate_pr_schema || true` then force-pushes, so a
+  regeneration failure ships anyway. Making it fatal leaves PRs un-rebased instead — a judgment call.
+- `approve_submission.py` wraps its schema update in a bare `except Exception`, so a submission can
+  merge without its new tags reaching the schema.
+
+## Workflow authoring
+
+The user's global GitHub Actions rules apply. Existing workflows violate most of them.
+**Do not bulk-remediate** — bring only the file you were asked to touch up to standard, and surface
+the rest in chat.
+
+## External sources
+
+`registry.faa.gov` and ADS-B Exchange are **required** — the release fails without them. Mictronics is
+**tolerated**: it retries, then the job continues without it. adsb.lol may simply not have published a
+given day, in which case the previous CSV is re-released rather than failing.
+
+FAA refreshes at 05:30 UTC; the release cron fires at 06:00 UTC. That 30-minute margin is the reason
+for the schedule.

CLAUDE.md

@@ -0,0 +1,6 @@
+# CLAUDE.md
+
+Read [AGENTS.md](./AGENTS.md) before touching anything in this repo. It is the single source of
+repo-specific rules; this file adds nothing of its own and is never the place to record new ones.
+
+Record new repo-specific guidance in `AGENTS.md`.

README.md

@@ -16,11 +16,19 @@ A daily release is created at **06:00 UTC** and includes:
 - **openairframes_community.csv**  
   All community submissions
 
+- **openairframes_adsb.csv**  
+  Airframes dataset derived from ADSB.lol network data. For each UTC day, a row is created for every icao observed in that day’s ADS-B messages, using registration data from [tar1090-db](https://github.com/wiedehopf/tar1090-db) (ADSBExchange & Mictronics).
+Example Usage:
+```python
+import pandas as pd
+url = "https://github.com/PlaneQuery/OpenAirframes/releases/download/openairframes-2026-03-18-main/openairframes_adsb_2024-01-01_2026-03-17.csv.gz" # 1GB
+df = pd.read_csv(url)
+df
+```
+![](docs/images/df_adsb_example_0.png)
 - **openairframes_faa.csv**  
   All [FAA registration data](https://www.faa.gov/licenses_certificates/aircraft_certification/aircraft_registry/releasable_aircraft_download) from 2023-08-16 to present (~260 MB)
 
-- **openairframes_adsb.csv**  
-  Airframe information derived from ADS-B messages on the [ADSB.lol](https://www.adsb.lol/) network, from 2026-02-12 to present (will be from 2024-01-01 soon). The airframe information originates from [mictronics aircraft database](https://www.mictronics.de/aircraft-database/) (~5 MB).
 
 - **ReleasableAircraft_{date}.zip**  
   A daily snapshot of the FAA database, which updates at **05:30 UTC**

community-routes/psimpson/os1 (1).txt

@@ -0,0 +1,36 @@
+TAP50Y  lis lhr
+EXS96WT man ibz
+baw837  dbv lhr
+exs6yr  nce lba
+tom1lx  ncl ibz
+exs62vc edi pmi
+tom35j  boj lgw
+tom509  dlm lgw
+afr902  cdg ndj nsi cdg
+tom71a  spc man
+tom8ke  man her
+nsz3868 bll opo
+exs95wl mah ncl
+exs18rk stn reu
+tom9db  mah bhx
+tom2bw  reu bhx
+kac113  kwi man
+tom18e  ibz gla
+ocn8k   snn fra
+tfl365  ams cur bon ams
+exs29y  zth bhx
+exs79cf olb man
+asl508  beg yyz
+tom4nw  pmi man
+exs3uq  zth ema
+exs23ml her man
+gfa003  bah lhr
+baw703  bjv lhr
+tom2fb  mme pmi
+tom7el  ibz lgw
+tom7bd  lba pmi
+ual967  nap ewr
+ein4ec  dub cfu
+tom78v  lgw lca
+eva067  tpe bkk lhr
+ezy85xv nce lpl

community-routes/psimpson/routes.2.txt

@@ -0,0 +1,31 @@
+efw979y klx lgw
+ezy74wg ayt lgw
+ezy95yg ibz sen
+exs65lg kgs bhx
+tom5ky  her lgw
+tom213  dlm man
+jbu1990 sju ewr
+exs68pv pmi stn
+ice48p  kef cdg
+exs45ra man spu
+klm741  ams bog ctg ams
+exs42nu man olb
+ein55g  lys dub
+baw538  lhr bds
+uae74w  lgw dxb
+ely312  ltn tlv
+tfl757  ams puj cur ams
+wja41   lgw yhx
+tom7pj  reu man
+ryr817l bzr stn
+ein429  psa dub
+exs3lf  olb bhx
+ezy38en lrh lgw
+ezy85wd rmu man
+apo7579 lgw los
+tom13a  mah man
+baw2279 lgw yvr
+exs406p gro edi
+tom5jl  pmi stn
+ein42m  dub vce
+

community-routes/psimpson/routes.3.txt

@@ -0,0 +1,30 @@
+klm1045 ams bhx
+dhk591  hkg del ema
+sht22a  lhr gci
+etd75f  auh lhr
+tom92g  pmi ema
+tom767  nbe brs
+qtr28u  doh lhr
+tom56m  pmi ncl
+aca883  nap yul
+tsc691  ath yul
+srr902  hgh nvi bhx bll
+kac109  kwi lhr
+cfe4ed  ibz lcy
+exs628  dbv ema
+tom581  nbe ema
+exs86j  ema puy
+exs67am skg lgw
+tom37d  kva bhx
+tom9dy  pmi bhx
+qtr72b  doh stn
+exs52cj efl brs
+ezy2816 pvk brs
+tom2bk  mah ncl
+exs86pf jsi bhx
+exs39yr jsi brs
+exs17j  mah lpl
+qtr2c   doh dub
+cfe979  pmi lcy
+sht21b  gci lhr
+exs12lf spu bhx

community-routes/psimpson/routes.4.txt

@@ -0,0 +1,32 @@
+tom8ax  kva man
+tom6nk  cfu man
+gfa003  bah lhr
+sxs7by  adb dub
+tom5gk  ext kgs
+tom62w  efl brs
+exs77j  stn zth
+tom4lw  boh her
+exs916  spu man
+tom54y  zth man
+tom34g  brs pfo
+exs3th  nap gla
+exs9dw  nap man
+tom24m  kgs lgw
+tom748  ema sid
+exs718d nte edi
+exs53ru brs kgs
+exs9eh  pvk brs
+etd71m  lhr auh
+exs29wk zth bhx
+exs46qw kgs stn
+wuk369  ltn pmi
+tom5dc  her brs
+wuk9768 ltn jmk
+tom5gl  lba pmi
+exs21dw bhx klx
+tom5ka  cwl lca
+ein46p  dub cta
+tom73e  efl stn
+ely316  lhr tlv
+efw26pp lgw mah
+qtr47y  lhr doh

community-routes/psimpson/routes.5.txt

@@ -0,0 +1,30 @@
+kmm3118 mla lgw
+baw539  bds lhr
+tom29k  zth brs
+tom32x  rho bhx
+ezy71zj lgw pvk
+baw536  lhr bds
+ent429  lgw pvk
+tom7cl  brs cfu
+qtr1f   doh lhr
+sxs5mq  man ayt
+klm767  ams aua bon ams
+vlg5ml  lcg lhr
+exs92se puy ema
+tom850  man nbe
+exs5sq  ncl pmi
+apo7576 abv lhr
+tom1an  stn her
+exs1kp  gla pmi
+ryr1794 ibz stn
+kmm3119 lgw mla
+isr116  ltn tlv
+sht9f   edi lhr
+baw9cj  lhr bru
+ezy93wm brs pmo
+ezy42eu ltn bsl
+bbc201  dac zyl lhr
+bbc202  lhr zyl dac
+tom3lw  rho lgw
+sxs7fz  ayt stn
+exs71mf stn pmi

community-routes/psimpson/routes.6.txt

@@ -0,0 +1,33 @@
+qtr33w  doh lhr
+tom86d  cwl her
+ezy49zc lgw bjv
+cpa008  lhr cdg hkg
+ely317  tlv lhr
+tom4ej  bhx cfu
+tom93j  gla ibz
+sva117  jed lhr
+ely313  tlv ltn
+qtr67h  lhr doh
+box442  fra yyz ord
+baw710c lhr lca
+wuk784  pmi ltn
+tom23m  efl man
+baw455  ibz lhr
+baw595  olb lhr
+baw621  peg lhr
+azg394  bhx gyd
+tom47x  zth bhx
+ezy45rl bsl ltn
+tom10y  ibz man
+baw663  zth lhr
+tom6en  bhx pmi
+efw74v  kgs lgw
+sva118  lhr jed
+ezy38xg bod bhx
+ein463  cta dub
+baw537  bds lhr
+exs689l pmi ncl
+tom43j  kgs ext
+tom9gx  pmi lgw
+apo7577 lhr abv
+ely318  lhr tlv

community-routes/psimpson/routes.7.txt

@@ -0,0 +1,33 @@
+exs42m  pmi stn
+exs51nw ibz man
+tom68h  pfo brs
+qtr61c  doh lgg ord
+wja51   lgw yyt
+exs93pk gro ema
+uae34y  dxb man
+cfe38z  lcy fao
+tom33j  pmi ema
+tom82k  mah stn
+tom8ya  pmi bhx
+exs1386 puy bhx
+sva119  jed lhr
+tom25a  mah man
+etd75f  auh lhr
+tom6ev  mah bhx
+efw16yk cag lgw
+ajt8620 bru mia
+uae9j   dxb stn
+tom429  nbe cwl
+sxs9gg  ayt bhx
+baw947l spu lhr
+tom7dm  cfu ema
+tom3nh  skg brs
+tom5hy  ibz man
+tom7hk  pmi gla
+tom9jw  boj cwl
+tom8be  bud bhx
+exs32y  brs zth
+tom7an  spu man
+tom84y  pmi ncl
+exs5qd  efl lba
+tom58h  bhx zth

community-routes/psimpson/routes.8.txt

@@ -0,0 +1,38 @@
+exs3uq  zth ema
+efw67a  lgw ayt
+tom657  nbe gla
+tom7cd  cfu gla
+exs79ue pmi man
+tom3lk  nap ema
+exs732  zth edi
+cfe12g  olb lcy
+tom2xj  jsi lgw
+gfa003  bah lhr
+gfa006  lhr bah
+tfl4mh  ams lpa
+exs9dw  nap man
+tom6ym  cwl cfu
+cfe316  ibz lcy
+qtr2c   doh dub
+exs48rz jsi man
+afr018  cdg lax ppt
+ewg8gj  str lgw str
+exs6yr  nce lba
+ely313  tlv ltn
+ely317  tlv lhr
+wuk13gw ltn tia
+baw58xp mxp lhr
+noz38w  aes lgw
+exs98dm ema fao
+eju15uv lpl mxp
+tom15x  cfu man
+ezy81qh ltn ibz
+wuk784  pmi ltn
+exs1898 zth brs
+baw841  dbv lhr
+sht6d   lhr gla
+tom2bg  cfu cwl
+exs45yk stn pmi
+ezy36ep pmi lgw
+tom9yg  zth bhx
+tom30w  spu lgw

community-routes/psimpson/routes.txt

@@ -0,0 +1,31 @@
+baw693  jtr lhr
+baw58xp mxp lhr
+uae9393 dwc lgg ord
+exs91au ncl pmi
+baw699w her lhr
+cfe91g  mah gla
+vlg49uc lhr lcg
+tom2nh  pmi man
+ezy56rd spu ltn
+tom3fa  reu bhx
+eag8sb  bhd sou
+cfe31y  pmi gla
+cfe92y  pmi edi
+ibs18my lgw mad
+exs1418 spu stn
+exs41m  vrn stn
+tom2wt  ibz nwi
+baw661  efl lhr
+wuk2818 zth ltn
+eag9st  sou bhd
+tom2ga  kgs brs
+exs25db pmi edi
+dhk812  bah lej ema
+baw15   lhr sin syd
+baw16   syd sin lhr
+tom59a  jtr man
+exs45yk stn pmi
+apo7577 lhr abv
+tom6aw  man pmi
+baw675  pvk lhr
+

src/adsb/compress_adsb_to_aircraft_data.py

@@ -4,6 +4,10 @@ import polars as pl
 
 COLUMNS = ['dbFlags', 'ownOp', 'year', 'desc', 'aircraft_category', 'r', 't']
 
+# Positional contract for every released ADS-B artifact. polars concatenates by
+# position after .select(), so a divergent copy corrupts output without erroring.
+FINAL_COLUMN_ORDER = ['time', 'icao', 'r', 't', 'dbFlags', 'ownOp', 'year', 'desc', 'aircraft_category']
+
 
 def compress_df_polars(df: pl.DataFrame, icao: str) -> pl.DataFrame:
     """Compress a single ICAO group to its most informative row using Polars."""
@@ -137,7 +141,7 @@ def load_parquet_part(part_id: int, date: str) -> pl.DataFrame:
     """Load a single parquet part file for a date.
     
     Args:
-        part_id: Part ID (e.g., 1, 2, 3)
+        part_id: Part ID (0-indexed, e.g. 0, 1, 2, 3)
         date: Date string in YYYY-MM-DD format
     
     Returns:
@@ -164,7 +168,7 @@ def load_parquet_part(part_id: int, date: str) -> pl.DataFrame:
     print(f"Loading from parquet: {parquet_file}")
     df = pl.read_parquet(
         parquet_file,
-        columns=['time', 'icao', 'r', 't', 'dbFlags', 'ownOp', 'year', 'desc', 'aircraft_category']
+        columns=FINAL_COLUMN_ORDER
     )
     
     # Convert to timezone-naive datetime

src/adsb/concat_parquet_to_final.py

@@ -2,8 +2,11 @@ from pathlib import Path
 import polars as pl
 import argparse
 import os
+import sys
+
+from src.adsb.compress_adsb_to_aircraft_data import FINAL_COLUMN_ORDER
+
 OUTPUT_DIR = Path("./data/output")
-CORRECT_ORDER_OF_COLUMNS = ["time", "icao", "r", "t", "dbFlags", "ownOp", "year", "desc", "aircraft_category"]
 
 def main():
     parser = argparse.ArgumentParser(description="Concatenate compressed parquet files for a single day")
@@ -17,13 +20,13 @@ def main():
     parquet_files = sorted(date_dir.glob("*.parquet"))
     df = None
     if parquet_files: # TODO: This logic could be updated slightly.
-        print(f"No parquet files found in {date_dir}")
+        print(f"Found {len(parquet_files)} parquet part(s) in {date_dir}")
 
         frames = [pl.read_parquet(p) for p in parquet_files]
         df = pl.concat(frames, how="vertical", rechunk=True)
 
         df = df.sort(["time", "icao"])
-        df = df.select(CORRECT_ORDER_OF_COLUMNS)
+        df = df.select(FINAL_COLUMN_ORDER)
         
         output_path = OUTPUT_DIR / f"openairframes_adsb_{args.date}.parquet"
         print(f"Writing combined parquet to {output_path} with {df.height} rows")
@@ -32,6 +35,13 @@ def main():
         csv_output_path = OUTPUT_DIR / f"openairframes_adsb_{args.date}.csv.gz"
         print(f"Writing combined csv.gz to {csv_output_path} with {df.height} rows")
         df.write_csv(csv_output_path, compression="gzip")
+    elif not args.concat_with_latest_csv:
+        # Nothing to merge and no release to fall back on: exiting 0 here would let the
+        # caller mistake "produced nothing" for "succeeded".
+        print(f"ERROR: No parquet files found in {date_dir} and --concat_with_latest_csv not set")
+        sys.exit(1)
+    else:
+        print(f"No parquet files found in {date_dir}; falling back to the latest released CSV")
 
     if args.concat_with_latest_csv:
         print("Loading latest CSV from GitHub releases to concatenate with...")
@@ -50,15 +60,15 @@ def main():
             print("Writing latest CSV directly without concatenation to avoid duplicates")
             os.makedirs(OUTPUT_DIR, exist_ok=True)
             final_csv_output_path = OUTPUT_DIR / f"openairframes_adsb_{csv_start_date}_{csv_end_date}.csv.gz"
-            df_latest_csv = df_latest_csv.select(CORRECT_ORDER_OF_COLUMNS)
+            df_latest_csv = df_latest_csv.select(FINAL_COLUMN_ORDER)
             df_latest_csv.write_csv(final_csv_output_path, compression="gzip")
         else:
             print(f"Concatenating latest CSV (through {csv_end_date}) with new data ({args.date})")
             # Ensure column order matches before concatenating
-            df_latest_csv = df_latest_csv.select(CORRECT_ORDER_OF_COLUMNS)
+            df_latest_csv = df_latest_csv.select(FINAL_COLUMN_ORDER)
             from src.adsb.compress_adsb_to_aircraft_data import concat_compressed_dfs
             df_final = concat_compressed_dfs(df_latest_csv, df)
-            df_final = df_final.select(CORRECT_ORDER_OF_COLUMNS)
+            df_final = df_final.select(FINAL_COLUMN_ORDER)
             final_csv_output_path = OUTPUT_DIR / f"openairframes_adsb_{csv_start_date}_{args.date}.csv.gz"
             df_final.write_csv(final_csv_output_path, compression="gzip")
         print(f"Final CSV written to {final_csv_output_path}")

src/adsb/download_adsb_data_to_parquet.py

@@ -93,6 +93,19 @@ def _fetch_releases_from_repo(year: str, version_date: str) -> list:
                         else:
                             print(f"Giving up after {max_retries} attempts")
                             return releases
+            except urllib.error.HTTPError as e:
+                # 404 means the repo/page does not exist. Retrying cannot change that,
+                # and 10 attempts x 5 min burns ~45 min of runner time to learn nothing.
+                if e.code == 404:
+                    print(f"Not found (HTTP 404): {BASE_URL}?page={page} - not retrying")
+                    return releases
+                print(f"Request exception (attempt {attempt}/{max_retries}): {e}")
+                if attempt < max_retries:
+                    print(f"Waiting {retry_delay} seconds before retry")
+                    time.sleep(retry_delay)
+                else:
+                    print(f"Giving up after {max_retries} attempts")
+                    return releases
             except Exception as e:
                 print(f"Request exception (attempt {attempt}/{max_retries}): {e}")
                 if attempt < max_retries:

src/adsb/process_icao_chunk.py

@@ -116,7 +116,7 @@ from pathlib import Path
 
 def main():
     parser = argparse.ArgumentParser(description="Process a single archive part for a day")
-    parser.add_argument("--part-id", type=int, required=True, help="Part ID (1-indexed)")
+    parser.add_argument("--part-id", type=int, required=True, help="Part ID (0-indexed)")
     parser.add_argument("--date", type=str, required=True, help="Date in YYYY-MM-DD format")
     args = parser.parse_args()
     

src/combine_historical_faa.py

@@ -1,14 +0,0 @@
-#unique_regulatory_id
-# 1. read historoical and output
-# 2. read sequentially
-
-# Instead of reading all csvs I can read just the latest release csv to get everything.
-
-from pathlib import Path
-
-base = Path("data/faa_releasable_historical")
-for day_dir in sorted(base.glob("2024-02-*")):
-    master = day_dir / "Master.txt"
-    if master.exists():
-        out_csv = master_txt_to_releasable_csv(master, out_dir="data/faa_releasable_historical_csv")
-        print(day_dir.name, "->", out_csv)

src/contributions/approve_submission.py

@@ -21,7 +21,7 @@ import urllib.request
 import urllib.error
 from datetime import datetime, timezone
 
-from .schema import extract_json_from_issue_body, extract_contributor_name_from_issue_body, parse_and_validate, load_schema, SCHEMAS_DIR
+from .schema import extract_json_from_issue_body, extract_contributor_name_from_issue_body, parse_and_validate, load_schema, get_schema_path
 from .contributor import (
     generate_contributor_uuid,
     generate_submission_filename,
@@ -72,9 +72,14 @@ def add_issue_comment(issue_number: int, body: str) -> None:
     github_api_request("POST", f"/issues/{issue_number}/comments", {"body": body})
 
 
-def get_default_branch_sha() -> str:
-    """Get the SHA of the default branch (main)."""
-    ref = github_api_request("GET", "/git/ref/heads/main")
+def get_default_branch() -> str:
+    """Get the repository's default branch name."""
+    return github_api_request("GET", "")["default_branch"]
+
+
+def get_branch_sha(branch: str) -> str:
+    """Get the head SHA of a branch."""
+    ref = github_api_request("GET", f"/git/ref/heads/{branch}")
     return ref["object"]["sha"]
 
 
@@ -199,14 +204,14 @@ def process_submission(
     
     # Create branch
     branch_name = f"community-submission-{issue_number}"
-    default_sha = get_default_branch_sha()
-    create_branch(branch_name, default_sha)
+    base_branch = get_default_branch()
+    create_branch(branch_name, get_branch_sha(base_branch))
     
     # Create file
     commit_message = f"Add community submission from @{author_username} (closes #{issue_number})"
     create_or_update_file(file_path, content_json, commit_message, branch_name)
     
-    # Update schema with any new tags (modifies v1 in place)
+    # Update schema with any new tags (rewrites the resolved schema version in place)
     schema_updated = False
     new_tags = []
     try:
@@ -232,7 +237,7 @@ def process_submission(
             schema_json = json.dumps(updated_schema, indent=2) + "\n"
             
             create_or_update_file(
-                "schemas/community_submission.v1.schema.json",
+                f"schemas/{get_schema_path().name}",
                 schema_json,
                 f"Update schema with new tags: {', '.join(new_tags)}",
                 branch_name
@@ -276,7 +281,7 @@ Closes #{issue_number}
     pr = create_pull_request(
         title=f"Community submission: {filename}",
         head=branch_name,
-        base="main",
+        base=base_branch,
         body=pr_body,
     )
     

src/contributions/regenerate_pr_schema.py

@@ -20,7 +20,7 @@ from src.contributions.update_schema import (
     check_for_new_tags,
     generate_updated_schema,
 )
-from src.contributions.schema import load_schema, SCHEMAS_DIR
+from src.contributions.schema import load_schema, get_schema_path
 
 
 def main():
@@ -51,8 +51,8 @@ def main():
         # Generate updated schema
         updated_schema = generate_updated_schema(current_schema, tag_registry)
         
-        # Write updated schema (in place)
-        schema_path = SCHEMAS_DIR / "community_submission.v1.schema.json"
+        # Write back to whichever version load_schema() resolved to
+        schema_path = get_schema_path()
         with open(schema_path, 'w') as f:
             json.dump(updated_schema, f, indent=2)
             f.write("\n")

src/contributions/schema.py

@@ -12,9 +12,6 @@ except ImportError:
 
 SCHEMAS_DIR = Path(__file__).parent.parent.parent / "schemas"
 
-# For backwards compatibility
-SCHEMA_PATH = SCHEMAS_DIR / "community_submission.v1.schema.json"
-
 
 def get_latest_schema_version() -> int:
     """