Community submission (rebased on main)

2026-09-19 04:12:15 +02:00 · 2026-02-18 22:19:18 +00:00
40 changed files with 187 additions and 1919 deletions
@@ -49,38 +49,11 @@ jobs:
          python -m src.adsb.download_and_list_icaos --date "$DATE"
          ls -lah data/output/adsb_archives/"$DATE" || true

-      - name: Upload archive part 0
+      - name: Upload archives
        uses: actions/upload-artifact@v4
        with:
-          name: adsb-archive-${{ inputs.date }}-part-0
-          path: data/output/adsb_archives/${{ inputs.date }}/${{ inputs.date }}_part_0.tar.gz
-          retention-days: 1
-          compression-level: 0
-          if-no-files-found: error
-
-      - name: Upload archive part 1
-        uses: actions/upload-artifact@v4
-        with:
-          name: adsb-archive-${{ inputs.date }}-part-1
-          path: data/output/adsb_archives/${{ inputs.date }}/${{ inputs.date }}_part_1.tar.gz
-          retention-days: 1
-          compression-level: 0
-          if-no-files-found: error
-
-      - name: Upload archive part 2
-        uses: actions/upload-artifact@v4
-        with:
-          name: adsb-archive-${{ inputs.date }}-part-2
-          path: data/output/adsb_archives/${{ inputs.date }}/${{ inputs.date }}_part_2.tar.gz
-          retention-days: 1
-          compression-level: 0
-          if-no-files-found: error
-
-      - name: Upload archive part 3
-        uses: actions/upload-artifact@v4
-        with:
-          name: adsb-archive-${{ inputs.date }}-part-3
-          path: data/output/adsb_archives/${{ inputs.date }}/${{ inputs.date }}_part_3.tar.gz
+          name: adsb-archives-${{ inputs.date }}
+          path: data/output/adsb_archives/${{ inputs.date }}
          retention-days: 1
          compression-level: 0
          if-no-files-found: error
@@ -106,22 +79,12 @@ jobs:
          python -m pip install --upgrade pip
          pip install -r requirements.txt

-      - name: Download archive part
+      - name: Download archives
        uses: actions/download-artifact@v4
        with:
-          name: adsb-archive-${{ inputs.date }}-part-${{ matrix.part_id }}
+          name: adsb-archives-${{ inputs.date }}
          path: data/output/adsb_archives/${{ inputs.date }}

-      - name: Verify archive
-        run: |
-          FILE="data/output/adsb_archives/${{ inputs.date }}/${{ inputs.date }}_part_${{ matrix.part_id }}.tar.gz"
-          ls -lah data/output/adsb_archives/${{ inputs.date }}/
-          if [ ! -f "$FILE" ]; then
-            echo "::error::Archive not found: $FILE"
-            exit 1
-          fi
-          echo "Verified: $(du -h "$FILE")"
-
      - name: Process part
        env:
          DATE: ${{ inputs.date }}
@@ -177,6 +140,6 @@ jobs:
        uses: actions/upload-artifact@v4
        with:
          name: openairframes_adsb-${{ inputs.date }}
-          path: data/output/openairframes_adsb_*
+          path: data/output/openairframes_adsb_${{ inputs.date }}*
          retention-days: 30
          if-no-files-found: error
@@ -2,7 +2,7 @@ name: openairframes-daily-release

 on:
  schedule:
-    # 06:00 UTC every day - runs on default branch, triggers both
+    # 6:00pm UTC every day - runs on default branch, triggers both
    - cron: "0 06 * * *"
  workflow_dispatch:
    inputs:
@@ -10,19 +10,11 @@ on:
        description: 'Date to process (YYYY-MM-DD format, default: yesterday)'
        required: false
        type: string
-      bootstrap_source:
-        description: 'Onboarding only: the one source id permitted to rebuild from a single day'
-        required: false
-        type: string

 permissions:
  contents: write
  actions: write

-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: false
-
 jobs:
  trigger-releases:
    runs-on: ubuntu-latest
@@ -50,81 +42,39 @@ jobs:
              ref: 'develop'
            });

-  # One thread per registry. Adding a source is one matrix entry plus
-  # src/create_daily_<source>_release.py; join-registry picks it up by artifact pattern.
-  build-registry-source:
-    if: github.event_name != 'schedule'
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - source: faa
-            required: true
-          - source: tc
-            required: false
-    uses: ./.github/workflows/registry-source.yaml
-    with:
-      source: ${{ matrix.source }}
-      date: ${{ inputs.date }}
-      required: ${{ matrix.required }}
-      # Standing permission would also fire on an outage. A source bootstraps only when a
-      # human dispatches the run naming it.
-      allow_bootstrap: ${{ inputs.bootstrap_source == matrix.source }}
-
-  join-registry:
-    needs: build-registry-source
-    # No always(): a tolerated source fails its step without failing its leg, so this only
-    # blocks when a required source could not be built.
-    if: github.event_name != 'schedule'
+  build-faa:
    runs-on: ubuntu-24.04-arm
-    timeout-minutes: 20
-    permissions:
-      contents: read
+    if: github.event_name != 'schedule'
    steps:
      - name: Checkout
-        uses: actions/checkout@v7
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0

      - name: Setup Python
-        uses: actions/setup-python@v7
+        uses: actions/setup-python@v6
        with:
          python-version: "3.14"
-          cache: 'pip'
-          cache-dependency-path: requirements.txt

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt

-      - name: Download every source thread
-        uses: actions/download-artifact@v8
-        with:
-          pattern: registry-*
-          path: artifacts/registry
-          merge-multiple: true
-
-      - name: Join sources into one registry
-        env:
-          RUN_DATE: ${{ inputs.date }}
+      - name: Run FAA release script
        run: |
-          python src/build_registry.py --input-dir artifacts/registry ${RUN_DATE:+--date "$RUN_DATE"}
+          python src/create_daily_faa_release.py ${{ inputs.date && format('--date {0}', inputs.date) || '' }}
+          ls -lah data/faa_releasable
+          ls -lah data/openairframes

-      - name: Stage release assets
-        run: |
-          mkdir -p data/release-out
-          # Every per-source CSV must ship: each source reads its OWN previous asset to
-          # accumulate, so one that is never published can never be anything but day one.
-          cp artifacts/registry/* data/release-out/
-          cp data/openairframes/openairframes_registry_*.csv data/release-out/
-          ls -lah data/release-out
-
-      - name: Upload registry
-        uses: actions/upload-artifact@v7
+      - name: Upload FAA artifacts
+        uses: actions/upload-artifact@v4
        with:
-          name: union-registry
-          path: data/release-out
+          name: faa-release
+          path: |
+            data/openairframes/openairframes_faa_*.csv
+            data/faa_releasable/ReleasableAircraft_*.zip
          retention-days: 1
-          if-no-files-found: error

  resolve-dates:
    runs-on: ubuntu-latest
@@ -151,51 +101,6 @@ jobs:
      date: ${{ needs.resolve-dates.outputs.adsb_date }}
      concat_with_latest_csv: true

-  adsb-reduce:
-    needs: [resolve-dates, adsb-to-aircraft]
-    if: always() && github.event_name != 'schedule' && needs.adsb-to-aircraft.result == 'failure'
-    runs-on: ubuntu-24.04-arm
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-
-      - name: Setup Python
-        uses: actions/setup-python@v6
-        with:
-          python-version: '3.12'
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
-
-      - name: Download compressed outputs
-        uses: actions/download-artifact@v4
-        with:
-          pattern: adsb-compressed-${{ needs.resolve-dates.outputs.adsb_date }}-part-*
-          path: data/output/compressed/${{ needs.resolve-dates.outputs.adsb_date }}
-          merge-multiple: true
-
-      - name: Concatenate final outputs
-        env:
-          DATE: ${{ needs.resolve-dates.outputs.adsb_date }}
-          CONCAT_WITH_LATEST_CSV: true
-        run: |
-          EXTRA=""
-          if [ "$CONCAT_WITH_LATEST_CSV" = "true" ]; then
-            EXTRA="--concat_with_latest_csv"
-          fi
-          python -m src.adsb.concat_parquet_to_final --date "$DATE" $EXTRA
-          ls -lah data/output/ || true
-
-      - name: Upload final artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: openairframes_adsb-${{ needs.resolve-dates.outputs.adsb_date }}
-          path: data/output/openairframes_adsb_*
-          retention-days: 30
-          if-no-files-found: error
-
  build-community:
    runs-on: ubuntu-latest
    if: github.event_name != 'schedule'
@@ -283,31 +188,30 @@ jobs:

  create-release:
    runs-on: ubuntu-latest
-    needs: [resolve-dates, join-registry, adsb-to-aircraft, adsb-reduce, build-community, build-adsbexchange-json, build-mictronics-db]
+    needs: [resolve-dates, build-faa, adsb-to-aircraft, build-community, build-adsbexchange-json, build-mictronics-db]
    if: github.event_name != 'schedule' && !cancelled()
    steps:
-      - name: Check ADS-B workflow status
-        if: needs.adsb-to-aircraft.result != 'success' && needs.adsb-reduce.result != 'success'
+      - name: Check adsb-to-aircraft status
+        if: needs.adsb-to-aircraft.result != 'success'
        run: |
-          echo "WARNING: ADS-B workflow failed (adsb-to-aircraft='${{ needs.adsb-to-aircraft.result }}', adsb-reduce='${{ needs.adsb-reduce.result }}'), will continue without ADS-B artifacts"
+          echo "WARNING: adsb-to-aircraft result was '${{ needs.adsb-to-aircraft.result }}', will continue without ADS-B artifacts"

      - name: Checkout for gh CLI
        uses: actions/checkout@v4
        with:
          sparse-checkout: |
            .github
-            NOTICE
          sparse-checkout-cone-mode: false

-      - name: Download joined registry
+      - name: Download FAA artifacts
        uses: actions/download-artifact@v5
        with:
-          name: union-registry
+          name: faa-release
          path: artifacts/faa

      - name: Download ADS-B artifacts
        uses: actions/download-artifact@v5
-        if: needs.adsb-to-aircraft.result == 'success' || needs.adsb-reduce.result == 'success'
+        if: needs.adsb-to-aircraft.result == 'success'
        continue-on-error: true
        with:
          name: openairframes_adsb-${{ needs.resolve-dates.outputs.adsb_date }}
@@ -362,17 +266,7 @@ jobs:
          
          # Find files from artifacts using find (handles nested structures)
          CSV_FILE_FAA=$(find artifacts/faa -name "openairframes_faa_*.csv" -type f 2>/dev/null | head -1)
-          CSV_FILE_REGISTRY=$(find artifacts/faa -name "openairframes_registry_*.csv" -type f 2>/dev/null | head -1)
-          # Every per-source registry CSV, whatever sources the matrix ran.
-          SOURCE_CSVS=$(find artifacts/faa -name "openairframes_*.csv" -type f 2>/dev/null \
-            | grep -vE '/openairframes_(registry|community|adsb)_[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{4}-[0-9]{2}-[0-9]{2}\.csv$' | sort)
-          echo "Per-source registry CSVs found:"
-          echo "$SOURCE_CSVS"
-          # Prefer concatenated file (with date range) over single-day file
-          CSV_FILE_ADSB=$(find artifacts/adsb -name "openairframes_adsb_*_*.csv.gz" -type f 2>/dev/null | head -1)
-          if [ -z "$CSV_FILE_ADSB" ]; then
-            CSV_FILE_ADSB=$(find artifacts/adsb -name "openairframes_adsb_*.csv.gz" -type f 2>/dev/null | head -1)
-          fi
+          CSV_FILE_ADSB=$(find artifacts/adsb -name "openairframes_adsb_*.csv.gz" -type f 2>/dev/null | head -1)
          CSV_FILE_COMMUNITY=$(find artifacts/community -name "openairframes_community_*.csv" -type f 2>/dev/null | head -1)
          ZIP_FILE=$(find artifacts/faa -name "ReleasableAircraft_*.zip" -type f 2>/dev/null | head -1)
          JSON_FILE_ADSBX=$(find artifacts/adsbexchange -name "basic-ac-db_*.json.gz" -type f 2>/dev/null | head -1)
@@ -389,14 +283,6 @@ jobs:
          if [ -z "$JSON_FILE_ADSBX" ] || [ ! -f "$JSON_FILE_ADSBX" ]; then
            MISSING_FILES="$MISSING_FILES ADSBX_JSON"
          fi
-          if [ -z "$CSV_FILE_REGISTRY" ] || [ ! -f "$CSV_FILE_REGISTRY" ]; then
-            MISSING_FILES="$MISSING_FILES REGISTRY_CSV"
-          fi
-          # NOTICE carries the terms that make each asset redistributable. Shipping data
-          # without it removes the permission, so it is required rather than optional.
-          if [ ! -f NOTICE ]; then
-            MISSING_FILES="$MISSING_FILES NOTICE"
-          fi
          
          # Optional files - warn but don't fail
          OPTIONAL_MISSING=""
@@ -434,24 +320,12 @@ jobs:
          fi
          
          if [ -n "$OPTIONAL_MISSING" ]; then
-            echo "::warning title=Missing optional release assets::$OPTIONAL_MISSING"
+            echo "WARNING: Optional files missing:$OPTIONAL_MISSING (will continue without them)"
          fi
          
          echo "date=$DATE" >> "$GITHUB_OUTPUT"
          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
          echo "csv_file_faa=$CSV_FILE_FAA" >> "$GITHUB_OUTPUT"
-          echo "csv_file_registry=$CSV_FILE_REGISTRY" >> "$GITHUB_OUTPUT"
-          {
-            echo "source_csvs<<SOURCE_CSVS_EOF"
-            echo "$SOURCE_CSVS"
-            echo "SOURCE_CSVS_EOF"
-            echo "source_basenames<<SOURCE_BASENAMES_EOF"
-            echo "$SOURCE_CSVS" | while read -r f; do
-              [ -n "$f" ] && echo "- $(basename "$f")"
-            done
-            echo "SOURCE_BASENAMES_EOF"
-          } >> "$GITHUB_OUTPUT"
-          echo "csv_basename_registry=$(basename "$CSV_FILE_REGISTRY")" >> "$GITHUB_OUTPUT"
          echo "csv_basename_faa=$CSV_BASENAME_FAA" >> "$GITHUB_OUTPUT"
          echo "csv_file_adsb=$CSV_FILE_ADSB" >> "$GITHUB_OUTPUT"
          echo "csv_basename_adsb=$CSV_BASENAME_ADSB" >> "$GITHUB_OUTPUT"
@@ -476,14 +350,7 @@ jobs:
      - name: Delete existing release if exists
        run: |
          echo "Attempting to delete release: ${{ steps.meta.outputs.tag }}"
-          # `|| echo` here would swallow a 403 or a partial delete, leaving yesterday's
-          # asset attached alongside today's; the next run then matches two and rebuilds
-          # the dataset from a single day.
-          if gh release view "${{ steps.meta.outputs.tag }}" >/dev/null 2>&1; then
-            gh release delete "${{ steps.meta.outputs.tag }}" --yes --cleanup-tag
-          else
-            echo "No existing release to delete"
-          fi
+          gh release delete "${{ steps.meta.outputs.tag }}" --yes --cleanup-tag || echo "No existing release to delete"
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

@@ -497,18 +364,14 @@ jobs:
            Automated daily snapshot generated at 06:00 UTC for ${{ steps.meta.outputs.date }}.

            Assets:
-            - NOTICE (source terms; required for redistribution)
-            - ${{ steps.meta.outputs.csv_basename_registry }}
-            ${{ steps.meta.outputs.source_basenames }}
+            - ${{ steps.meta.outputs.csv_basename_faa }}
            ${{ steps.meta.outputs.csv_basename_adsb && format('- {0}', steps.meta.outputs.csv_basename_adsb) || '' }}
            - ${{ steps.meta.outputs.csv_basename_community }}
            - ${{ steps.meta.outputs.zip_basename }}
            - ${{ steps.meta.outputs.json_basename_adsbx }}
            ${{ steps.meta.outputs.zip_basename_mictronics && format('- {0}', steps.meta.outputs.zip_basename_mictronics) || '' }}
          files: |
-            ${{ steps.meta.outputs.csv_file_registry }}
-            ${{ steps.meta.outputs.source_csvs }}
-            NOTICE
+            ${{ steps.meta.outputs.csv_file_faa }}
            ${{ steps.meta.outputs.csv_file_adsb }}
            ${{ steps.meta.outputs.csv_file_community }}
            ${{ steps.meta.outputs.zip_file }}
@@ -1,104 +0,0 @@
-name: registry-source
-
-# One registry source, on its own thread. Called once per entry in the caller's matrix, so
-# adding a registry is a matrix entry plus src/create_daily_<source>_release.py — no new job.
-
-on:
-  workflow_call:
-    inputs:
-      source:
-        description: 'Source id; must match src/create_daily_<source>_release.py'
-        required: true
-        type: string
-      date:
-        description: 'Date to process (YYYY-MM-DD, default: today UTC)'
-        required: false
-        type: string
-      python-version:
-        required: false
-        type: string
-        default: '3.14'
-      required:
-        description: 'Fail the thread when this source cannot be built'
-        required: false
-        type: boolean
-        default: false
-      allow_bootstrap:
-        description: 'Onboarding only: permit a single-day rebuild when no asset has ever been published'
-        required: false
-        type: boolean
-        default: false
-
-# Keyed on the source: without it every matrix leg shares one group and the legs cancel
-# each other, which is the opposite of running them in parallel.
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.source }}
-  cancel-in-progress: false
-
-jobs:
-  build:
-    runs-on: ubuntu-24.04-arm
-    timeout-minutes: 30
-    permissions:
-      contents: read
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v7
-
-      - name: Setup Python
-        uses: actions/setup-python@v7
-        with:
-          python-version: ${{ inputs.python-version }}
-          cache: 'pip'
-          cache-dependency-path: requirements.txt
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
-
-      # Deliberately outside the tolerated step below: a typo in the matrix is a config
-      # error, and must go red even for an optional source.
-      - name: Check the source has a build script
-        env:
-          SOURCE: ${{ inputs.source }}
-        run: |
-          script="src/create_daily_${SOURCE}_release.py"
-          if [ ! -f "$script" ]; then
-            echo "::error title=Unknown registry source::$script does not exist"
-            exit 1
-          fi
-
-      - name: Build ${{ inputs.source }} registry
-        continue-on-error: ${{ inputs.required == false }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          SOURCE: ${{ inputs.source }}
-          RUN_DATE: ${{ inputs.date }}
-          BOOTSTRAP: ${{ inputs.allow_bootstrap && '1' || '' }}
-        run: |
-          python "src/create_daily_${SOURCE}_release.py" ${RUN_DATE:+--date "$RUN_DATE"} ${BOOTSTRAP:+--allow-bootstrap}
-          # Stage into one directory so every leg's artifact has the same root; a second
-          # search path would move the root to the common ancestor for some legs only.
-          shopt -s nullglob
-          built=(data/openairframes/openairframes_"${SOURCE}"_*.csv)
-          if [ ${#built[@]} -ne 1 ]; then
-            echo "::error title=${SOURCE} produced no registry CSV::expected one openairframes_${SOURCE}_*.csv, found ${#built[@]}"
-            exit 1
-          fi
-          # Stage into one directory so every leg's artifact has the same root; a second
-          # search path would move the root to the common ancestor for some legs only.
-          mkdir -p data/registry-out
-          cp "${built[@]}" data/registry-out/
-          cp data/faa_releasable/ReleasableAircraft_*.zip data/registry-out/ 2>/dev/null || true
-          ls -lah data/registry-out
-
-      - name: Upload ${{ inputs.source }} registry
-        uses: actions/upload-artifact@v7
-        with:
-          name: registry-${{ inputs.source }}
-          path: data/registry-out
-          retention-days: 1
-          # A tolerated source that failed has nothing to upload; only a required
-          # source missing its artifact is an error.
-          if-no-files-found: ${{ inputs.required && 'error' || 'ignore' }}
@@ -5,7 +5,7 @@ on:
    branches: [main]
    paths:
      - 'community/**'
-      - 'schemas/**'
+      - 'schemas/community_submission.v1.schema.json'

 permissions:
  contents: write
@@ -23,17 +23,24 @@ jobs:
      - name: Install dependencies
        run: pip install jsonschema

+      - name: Debug issue body
+        run: |
+          echo "=== Issue Body ==="
+          cat << 'ISSUE_BODY_EOF'
+          ${{ github.event.issue.body }}
+          ISSUE_BODY_EOF
+
      - name: Save issue body to file
-        env:
-          ISSUE_BODY: ${{ github.event.issue.body }}
-        run: printf '%s' "$ISSUE_BODY" > "$RUNNER_TEMP/issue_body.txt"
+        run: |
+          cat << 'ISSUE_BODY_EOF' > /tmp/issue_body.txt
+          ${{ github.event.issue.body }}
+          ISSUE_BODY_EOF

      - name: Validate submission
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          GITHUB_REPOSITORY: ${{ github.repository }}
-          ISSUE_NUMBER: ${{ github.event.issue.number }}
        run: |
          python -m src.contributions.validate_submission \
-            --issue-body-file "$RUNNER_TEMP/issue_body.txt" \
-            --issue-number "$ISSUE_NUMBER"
+            --issue-body-file /tmp/issue_body.txt \
+            --issue-number ${{ github.event.issue.number }}
@@ -1,82 +0,0 @@
-## Never interpolate `${{ github.event.* }}` or `${{ inputs.* }}` into a `run:` block
-
-Pass them via `env:` and quote the shell variable. A quoted heredoc does not help — the body can
-contain the delimiter and close it early. Never fix this by escaping or renaming the delimiter.
-
-## Invocation
-
- `src/*.py` at the root of `src/` are scripts: `python src/create_daily_faa_release.py`. Bare
-  sibling imports, so `-m` raises `ModuleNotFoundError`.
- `src/adsb/*`, `src/contributions/*` are packages: `python -m`.
- Run from the repo root; output paths are CWD-relative.
-
-## Verification
-
- No test framework, linter, or packaging config. Do not add one unprompted.
- Never `gh workflow run` to test a change — every dispatch pulls tens of GB.
- Never commit generated data. The product is a GitHub Release; jobs pass state as artifacts.
- ADS-B has no cheap end-to-end check. Exercise `compress_multi_icao_df` on a hand-built frame.
-
-## Release invariants
-
- `FINAL_COLUMN_ORDER` (`compress_adsb_to_aircraft_data.py`) is the only definition of the ADS-B
-  column contract. `pl.concat` matches by position after `.select()`; a forked copy corrupts the
-  release with no error.
- Empty string, never null, in every released frame.
- `openairframes_id` = `normalize(manufacturer)|normalize(model)|normalize(serial)`. Reuse
-  `derive_from_faa_master_txt.normalize()`.
- Each source's daily build reads its own previous release asset and appends. Falling back to a
-  single-day rebuild on anything but `FileNotFoundError` republishes one day as the whole dataset,
-  which the next run then reads back as its base. Keep the fallback narrow.
- Python `3.14` for FAA/community/vendor jobs, `3.12` for ADS-B. Match the surrounding job.
-
-## Registry sources
-
- Judge **redistribution**, not access. A public licence travels to this project; a bilateral
-  permission granted to another project does not — that alone disqualifies Taiwan, Estonia, Chile.
-  Non-commercial-only terms are a separate, independent bar.
- `NOTICE` carries the terms that make each asset redistributable and is a required release file.
-  Never edit or drop an entry. Transport Canada requires both its notices together.
- `LICENSE` is MIT and covers code only. Claim nothing about released data.
- Owner/registrant mailing addresses are published for every registry. FAA and TC must not diverge.
- CCARCS `ACTIVE_FLAG` is not "current owner": 1,932 Registered marks carry only `I` parties, and
-  those rows are the `MAIL_RECIPIENT`. Prefer `A`, fall back to all.
- CCARCS addresses come from the single `MAIL_RECIPIENT == "Y"` row, never merged across co-owners.
-
-## ADS-B
-
- `load_parquet_part()` deleting its source parquet is deliberate — disk pressure. Do not defer it.
- A released row is the most informative observation for that ICAO on that UTC day, not a registry
-  record.
- HTTP 404 is terminal in the release fetch; restoring the retry stalls the Dec-31 probe ~45 min.
-
-## Community submissions are automation-owned
-
- Merging `community/**` or `schemas/**` force-pushes every open `community` PR branch onto main.
-  Hand edits there are destroyed.
- Never hand-author `community/` files — the filename encodes `sha256(content)[:8]`.
- A tag's JSON type is fixed by its first submission and enforced forever. Emergent from
-  `build_tag_type_registry` + `validate_submission`; written nowhere in the schema.
- Adding `community_submission.v2.schema.json` promotes it atomically across every reader and
-  writer. One-way door — only on request.
-
-## Fork
-
-`get_latest_release.REPO` pins upstream `PlaneQuery/openairframes` on purpose. Do not repoint it.
-Upstream develops on `develop`. The daily release deletes the existing release and tag first.
-
-## Do not chase
-
- `process-historical-faa.yaml` is dead: missing `src/get_historical_faa.py`,
-  `scripts/concat_csvs.py`, and uses the disabled `::set-output`.
- `af-klm-fleet/package.json` → `npm run validate` has no `scripts/validate.js`.
- `af-klm-fleet/` and `community-routes/` are unwired; nothing in CI touches them.
-  `af-klm-fleet/README.md` is generated.
-
-## Flag, do not silently fix
-
- `NUMBER_PARTS` is restated by the matrix and four upload steps in `adsb-to-aircraft-for-day.yaml`.
- `MAX_WORKERS = ... if OS_CPU_COUNT > 4 else 1` collapses to one worker on a ≤4-core runner.
- `update-community-prs.yaml` runs `regenerate_pr_schema || true`, then force-pushes.
- `approve_submission.py` wraps its schema update in a bare `except Exception`.
- Existing workflows violate the global GHA rules. Fix only the file you were asked to touch.
@@ -1,6 +0,0 @@
-# CLAUDE.md
-
-Read [AGENTS.md](./AGENTS.md) before touching anything in this repo. It is the single source of
-repo-specific rules; this file adds nothing of its own and is never the place to record new ones.
-
-Record new repo-specific guidance in `AGENTS.md`.
@@ -1,66 +0,0 @@
-OpenAirframes — Data Source Notices
-===================================
-
-The LICENSE file covers the *code* in this repository. It does not cover the
-*data* published in releases. Several upstream registries permit redistribution
-only on condition that specific notices travel with the data. Those conditions
-are reproduced below. This file is published as an asset on every release; anyone
-redistributing a release asset further must carry the corresponding notice with it.
-
-Removing or altering a notice in this file removes the permission that makes the
-corresponding asset redistributable.
-
-
-Transport Canada — Canadian Civil Aircraft Register (CCARCS)
------------------------------------------------------------
-Asset: openairframes_tc_*.csv
-Source: https://wwwapps.tc.gc.ca/saf-sec-sur/2/ccarcs-riacc/download/ccarcsdb.zip
-
-Redistribution is permitted under the Government of Canada terms recorded for this
-dataset. They require both of the following notices, verbatim, and require that the
-two reach the consumer together. The governing instrument is not published on the
-CCARCS download page, so no licence name or URL is asserted here.
-
-    Reproduced and distributed with the permission of the Government of Canada.
-
-    This product has been produced by or for the OpenAirframes project and
-    includes data provided by the Government of Canada. The incorporation of
-    data sourced from the Government of Canada within this product shall not be
-    construed as constituting an endorsement by the Government of Canada of our
-    product.
-
-Registered-owner mailing addresses are redistributed, matching the registrant
-addresses the FAA asset already carries. Because CCARCS lists one row per party,
-the published address is that of the single designated mail recipient rather than
-a merge across co-owners.
-
-
-FAA — Releasable Aircraft Database
-----------------------------------
-Source: https://registry.faa.gov/database/ReleasableAircraft.zip
-
-ReleasableAircraft_*.zip is redistributed unmodified. It is a work of the United
-States federal government, not subject to copyright protection in the United
-States (17 U.S.C. § 105). No notice is required; it is credited for provenance.
-
-openairframes_faa_*.csv is a derived product built by this repository —
-normalized, joined, deduplicated, and extended with an identifier this project
-defines. Section 105 disclaims copyright in the government's own work and says
-nothing about a derivative, so no claim is made here about the CSV's status.
-
-
-Other redistributed assets
---------------------------
-The following assets are republished from third parties whose terms have not
-been assessed in this repository. They are listed for provenance only; nothing
-here asserts a licence over them.
-
-    openairframes_adsb_*.csv.gz   derived from adsb.lol daily globe history
-                                  (github.com/adsblol), with registration data
-                                  from tar1090-db
-    basic-ac-db_*.json.gz         ADS-B Exchange, downloads.adsbexchange.com
-    mictronics-db_*.zip           Mictronics, www.mictronics.de
-
-Community submissions under community/ are contributed by their authors through
-the repository's submission workflow and are published with the attribution each
-contributor selected.
@@ -16,44 +16,15 @@ A daily release is created at **06:00 UTC** and includes:
 - **openairframes_community.csv**  
  All community submissions

- **openairframes_adsb.csv**  
-  Airframes dataset derived from ADSB.lol network data. For each UTC day, a row is created for every icao observed in that day’s ADS-B messages, using registration data from [tar1090-db](https://github.com/wiedehopf/tar1090-db) (ADSBExchange & Mictronics).
-Example Usage:
-```python
-import pandas as pd
-url = "https://github.com/PlaneQuery/OpenAirframes/releases/download/openairframes-2026-03-18-main/openairframes_adsb_2024-01-01_2026-03-17.csv.gz" # 1GB
-df = pd.read_csv(url)
-df
-```
-![](docs/images/df_adsb_example_0.png)
- **openairframes_registry.csv**  
-  Every national registry in one table, one row per registration record, with a `source`
-  column naming the registry it came from. Currently the FAA (United States) and Transport
-  Canada. Identifier columns (`transponder_code_hex`, `registration_number`,
-  `openairframes_id`) lead the table and are populated for every source.
-
 - **openairframes_faa.csv**  
-  All [FAA registration data](https://www.faa.gov/licenses_certificates/aircraft_certification/aircraft_registry/releasable_aircraft_download) from 2023-08-16 to present (~275 MB).
-  Superseded by `openairframes_registry.csv`; still published so existing consumers keep working.
+  All [FAA registration data](https://www.faa.gov/licenses_certificates/aircraft_certification/aircraft_registry/releasable_aircraft_download) from 2023-08-16 to present (~260 MB)

- **openairframes_tc.csv**  
-  The [Transport Canada Civil Aircraft Register](https://wwwapps.tc.gc.ca/saf-sec-sur/2/ccarcs-riacc/RchSimp.aspx),
-  ~35k aircraft with full ICAO 24-bit hex coverage. Also folded into
-  `openairframes_registry.csv`; published separately for the same reason as the FAA CSV.
+- **openairframes_adsb.csv**  
+  Airframe information derived from ADS-B messages on the [ADSB.lol](https://www.adsb.lol/) network, from 2026-02-12 to present (will be from 2024-01-01 soon). The airframe information originates from [mictronics aircraft database](https://www.mictronics.de/aircraft-database/) (~5 MB).

 - **ReleasableAircraft_{date}.zip**  
  A daily snapshot of the FAA database, which updates at **05:30 UTC**

- **basic-ac-db.json.gz**  
-  [ADS-B Exchange](https://www.adsbexchange.com/) basic aircraft database, republished unmodified.
-
- **mictronics-db.zip**  
-  [Mictronics](https://www.mictronics.de/aircraft-database/) aircraft database, republished
-  unmodified. Best effort — the release ships without it when the source is unavailable.
-
-Redistribution terms for the underlying sources travel with the release in **NOTICE**. Some
-registries permit redistribution only on condition that specific notices reach you with the data.
-
 ---

 ## For Contributors
@@ -1,36 +0,0 @@
-TAP50Y  lis lhr
-EXS96WT man ibz
-baw837  dbv lhr
-exs6yr  nce lba
-tom1lx  ncl ibz
-exs62vc edi pmi
-tom35j  boj lgw
-tom509  dlm lgw
-afr902  cdg ndj nsi cdg
-tom71a  spc man
-tom8ke  man her
-nsz3868 bll opo
-exs95wl mah ncl
-exs18rk stn reu
-tom9db  mah bhx
-tom2bw  reu bhx
-kac113  kwi man
-tom18e  ibz gla
-ocn8k   snn fra
-tfl365  ams cur bon ams
-exs29y  zth bhx
-exs79cf olb man
-asl508  beg yyz
-tom4nw  pmi man
-exs3uq  zth ema
-exs23ml her man
-gfa003  bah lhr
-baw703  bjv lhr
-tom2fb  mme pmi
-tom7el  ibz lgw
-tom7bd  lba pmi
-ual967  nap ewr
-ein4ec  dub cfu
-tom78v  lgw lca
-eva067  tpe bkk lhr
-ezy85xv nce lpl
@@ -1,31 +0,0 @@
-efw979y klx lgw
-ezy74wg ayt lgw
-ezy95yg ibz sen
-exs65lg kgs bhx
-tom5ky  her lgw
-tom213  dlm man
-jbu1990 sju ewr
-exs68pv pmi stn
-ice48p  kef cdg
-exs45ra man spu
-klm741  ams bog ctg ams
-exs42nu man olb
-ein55g  lys dub
-baw538  lhr bds
-uae74w  lgw dxb
-ely312  ltn tlv
-tfl757  ams puj cur ams
-wja41   lgw yhx
-tom7pj  reu man
-ryr817l bzr stn
-ein429  psa dub
-exs3lf  olb bhx
-ezy38en lrh lgw
-ezy85wd rmu man
-apo7579 lgw los
-tom13a  mah man
-baw2279 lgw yvr
-exs406p gro edi
-tom5jl  pmi stn
-ein42m  dub vce
-
@@ -1,30 +0,0 @@
-klm1045 ams bhx
-dhk591  hkg del ema
-sht22a  lhr gci
-etd75f  auh lhr
-tom92g  pmi ema
-tom767  nbe brs
-qtr28u  doh lhr
-tom56m  pmi ncl
-aca883  nap yul
-tsc691  ath yul
-srr902  hgh nvi bhx bll
-kac109  kwi lhr
-cfe4ed  ibz lcy
-exs628  dbv ema
-tom581  nbe ema
-exs86j  ema puy
-exs67am skg lgw
-tom37d  kva bhx
-tom9dy  pmi bhx
-qtr72b  doh stn
-exs52cj efl brs
-ezy2816 pvk brs
-tom2bk  mah ncl
-exs86pf jsi bhx
-exs39yr jsi brs
-exs17j  mah lpl
-qtr2c   doh dub
-cfe979  pmi lcy
-sht21b  gci lhr
-exs12lf spu bhx
@@ -1,32 +0,0 @@
-tom8ax  kva man
-tom6nk  cfu man
-gfa003  bah lhr
-sxs7by  adb dub
-tom5gk  ext kgs
-tom62w  efl brs
-exs77j  stn zth
-tom4lw  boh her
-exs916  spu man
-tom54y  zth man
-tom34g  brs pfo
-exs3th  nap gla
-exs9dw  nap man
-tom24m  kgs lgw
-tom748  ema sid
-exs718d nte edi
-exs53ru brs kgs
-exs9eh  pvk brs
-etd71m  lhr auh
-exs29wk zth bhx
-exs46qw kgs stn
-wuk369  ltn pmi
-tom5dc  her brs
-wuk9768 ltn jmk
-tom5gl  lba pmi
-exs21dw bhx klx
-tom5ka  cwl lca
-ein46p  dub cta
-tom73e  efl stn
-ely316  lhr tlv
-efw26pp lgw mah
-qtr47y  lhr doh
@@ -1,30 +0,0 @@
-kmm3118 mla lgw
-baw539  bds lhr
-tom29k  zth brs
-tom32x  rho bhx
-ezy71zj lgw pvk
-baw536  lhr bds
-ent429  lgw pvk
-tom7cl  brs cfu
-qtr1f   doh lhr
-sxs5mq  man ayt
-klm767  ams aua bon ams
-vlg5ml  lcg lhr
-exs92se puy ema
-tom850  man nbe
-exs5sq  ncl pmi
-apo7576 abv lhr
-tom1an  stn her
-exs1kp  gla pmi
-ryr1794 ibz stn
-kmm3119 lgw mla
-isr116  ltn tlv
-sht9f   edi lhr
-baw9cj  lhr bru
-ezy93wm brs pmo
-ezy42eu ltn bsl
-bbc201  dac zyl lhr
-bbc202  lhr zyl dac
-tom3lw  rho lgw
-sxs7fz  ayt stn
-exs71mf stn pmi
@@ -1,33 +0,0 @@
-qtr33w  doh lhr
-tom86d  cwl her
-ezy49zc lgw bjv
-cpa008  lhr cdg hkg
-ely317  tlv lhr
-tom4ej  bhx cfu
-tom93j  gla ibz
-sva117  jed lhr
-ely313  tlv ltn
-qtr67h  lhr doh
-box442  fra yyz ord
-baw710c lhr lca
-wuk784  pmi ltn
-tom23m  efl man
-baw455  ibz lhr
-baw595  olb lhr
-baw621  peg lhr
-azg394  bhx gyd
-tom47x  zth bhx
-ezy45rl bsl ltn
-tom10y  ibz man
-baw663  zth lhr
-tom6en  bhx pmi
-efw74v  kgs lgw
-sva118  lhr jed
-ezy38xg bod bhx
-ein463  cta dub
-baw537  bds lhr
-exs689l pmi ncl
-tom43j  kgs ext
-tom9gx  pmi lgw
-apo7577 lhr abv
-ely318  lhr tlv
@@ -1,33 +0,0 @@
-exs42m  pmi stn
-exs51nw ibz man
-tom68h  pfo brs
-qtr61c  doh lgg ord
-wja51   lgw yyt
-exs93pk gro ema
-uae34y  dxb man
-cfe38z  lcy fao
-tom33j  pmi ema
-tom82k  mah stn
-tom8ya  pmi bhx
-exs1386 puy bhx
-sva119  jed lhr
-tom25a  mah man
-etd75f  auh lhr
-tom6ev  mah bhx
-efw16yk cag lgw
-ajt8620 bru mia
-uae9j   dxb stn
-tom429  nbe cwl
-sxs9gg  ayt bhx
-baw947l spu lhr
-tom7dm  cfu ema
-tom3nh  skg brs
-tom5hy  ibz man
-tom7hk  pmi gla
-tom9jw  boj cwl
-tom8be  bud bhx
-exs32y  brs zth
-tom7an  spu man
-tom84y  pmi ncl
-exs5qd  efl lba
-tom58h  bhx zth
@@ -1,38 +0,0 @@
-exs3uq  zth ema
-efw67a  lgw ayt
-tom657  nbe gla
-tom7cd  cfu gla
-exs79ue pmi man
-tom3lk  nap ema
-exs732  zth edi
-cfe12g  olb lcy
-tom2xj  jsi lgw
-gfa003  bah lhr
-gfa006  lhr bah
-tfl4mh  ams lpa
-exs9dw  nap man
-tom6ym  cwl cfu
-cfe316  ibz lcy
-qtr2c   doh dub
-exs48rz jsi man
-afr018  cdg lax ppt
-ewg8gj  str lgw str
-exs6yr  nce lba
-ely313  tlv ltn
-ely317  tlv lhr
-wuk13gw ltn tia
-baw58xp mxp lhr
-noz38w  aes lgw
-exs98dm ema fao
-eju15uv lpl mxp
-tom15x  cfu man
-ezy81qh ltn ibz
-wuk784  pmi ltn
-exs1898 zth brs
-baw841  dbv lhr
-sht6d   lhr gla
-tom2bg  cfu cwl
-exs45yk stn pmi
-ezy36ep pmi lgw
-tom9yg  zth bhx
-tom30w  spu lgw
@@ -1,31 +0,0 @@
-baw693  jtr lhr
-baw58xp mxp lhr
-uae9393 dwc lgg ord
-exs91au ncl pmi
-baw699w her lhr
-cfe91g  mah gla
-vlg49uc lhr lcg
-tom2nh  pmi man
-ezy56rd spu ltn
-tom3fa  reu bhx
-eag8sb  bhd sou
-cfe31y  pmi gla
-cfe92y  pmi edi
-ibs18my lgw mad
-exs1418 spu stn
-exs41m  vrn stn
-tom2wt  ibz nwi
-baw661  efl lhr
-wuk2818 zth ltn
-eag9st  sou bhd
-tom2ga  kgs brs
-exs25db pmi edi
-dhk812  bah lej ema
-baw15   lhr sin syd
-baw16   syd sin lhr
-tom59a  jtr man
-exs45yk stn pmi
-apo7577 lhr abv
-tom6aw  man pmi
-baw675  pvk lhr
-
@@ -0,0 +1,21 @@
+[
+  {
+    "contributor_name": "applesauce123",
+    "contributor_uuid": "2981c3ee-8712-5f96-84bf-732eda515a3f",
+    "creation_timestamp": "2026-02-13T16:58:21.863525+00:00",
+    "registration_number": "N12345",
+    "tags": {
+      "internet": "starlink"
+    }
+  },
+  {
+    "contributor_name": "applesauce123",
+    "contributor_uuid": "2981c3ee-8712-5f96-84bf-732eda515a3f",
+    "creation_timestamp": "2026-02-13T16:58:21.863525+00:00",
+    "tags": {
+      "internet": "viasat",
+      "owner": "John Doe"
+    },
+    "transponder_code_hex": "ABC123"
+  }
+]
@@ -61,6 +61,9 @@
        "icao_aircraft_type": {
          "type": "string"
        },
+        "internet": {
+          "type": "string"
+        },
        "manufacturer_icao": {
          "type": "string"
        },
@@ -79,6 +82,9 @@
        "operator_icao": {
          "type": "string"
        },
+        "owner": {
+          "type": "string"
+        },
        "serial_number": {
          "type": "string"
        },
@@ -23,12 +23,6 @@ gh run list \
    "repos/$REPO/actions/runs/$run_id/artifacts" \
    --jq '.artifacts[] | select(.name | test("^openairframes_adsb-[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{4}-[0-9]{2}-[0-9]{2}$")) | .name' | while read -r artifact_name; do
    
-    # Check if artifact directory already exists and has files
-    if [ -d "downloads/adsb_artifacts/$artifact_name" ] && [ -n "$(ls -A "downloads/adsb_artifacts/$artifact_name" 2>/dev/null)" ]; then
-      echo "  Skipping (already exists): $artifact_name"
-      continue
-    fi
-    
    echo "  Downloading: $artifact_name"
    gh run download "$run_id" \
      --repo "$REPO" \
@@ -194,7 +194,7 @@ def main():
    if triggered_runs and not args.dry_run:
        import json
        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
-        runs_file = f"./output/triggered_runs_{timestamp}.json"
+        runs_file = f"./triggered_runs_{timestamp}.json"
        with open(runs_file, 'w') as f:
            json.dump({
                'start_date': args.start_date,
@@ -1,242 +0,0 @@
-#!/usr/bin/env python3
-"""
-Parse TheAirTraffic Database CSV and produce community_submission.v1 JSON.
-
-Source: "TheAirTraffic Database - Aircraft 2.csv"
-Output: community/YYYY-MM-DD/theairtraffic_<date>_<hash>.json
-
-Categories in the spreadsheet columns (paired: name, registrations, separator):
-  Col  1-3:  Business
-  Col  4-6:  Government
-  Col  7-9:  People
-  Col 10-12: Sports
-  Col 13-15: Celebrity
-  Col 16-18: State Govt./Law
-  Col 19-21: Other
-  Col 22-24: Test Aircraft
-  Col 25-27: YouTubers
-  Col 28-30: Formula 1 VIP's
-  Col 31-33: Active GII's and GIII's  (test/demo aircraft)
-  Col 34-37: Russia & Ukraine          (extra col for old/new)
-  Col 38-40: Helicopters & Blimps
-  Col 41-43: Unique Reg's
-  Col 44-46: Saudi & UAE
-  Col 47-49: Schools
-  Col 50-52: Special Charter
-  Col 53-55: Unknown Owners
-  Col 56-59: Frequent Flyers           (extra cols: name, aircraft, logged, hours)
-"""
-
-import csv
-import json
-import hashlib
-import re
-import sys
-import uuid
-from datetime import datetime, timezone
-from pathlib import Path
-
-# ── Category mapping ────────────────────────────────────────────────────────
-# Each entry: (name_col, reg_col, owner_category_tags)
-# owner_category_tags is a dict of tag keys to add beyond "owner"
-CATEGORY_COLUMNS = [
-    # (name_col, reg_col, {tag_key: tag_value, ...})
-    (1,  2,  {"owner_category_0": "business"}),
-    (4,  5,  {"owner_category_0": "government"}),
-    (7,  8,  {"owner_category_0": "celebrity"}),
-    (10, 11, {"owner_category_0": "sports"}),
-    (13, 14, {"owner_category_0": "celebrity"}),
-    (16, 17, {"owner_category_0": "government", "owner_category_1": "law_enforcement"}),
-    (19, 20, {"owner_category_0": "other"}),
-    (22, 23, {"owner_category_0": "test_aircraft"}),
-    (25, 26, {"owner_category_0": "youtuber", "owner_category_1": "celebrity"}),
-    (28, 29, {"owner_category_0": "celebrity", "owner_category_1": "motorsport"}),
-    (31, 32, {"owner_category_0": "test_aircraft"}),
-    # Russia & Ukraine: col 34=name, col 35 or 36 may have reg
-    (34, 35, {"owner_category_0": "russia_ukraine"}),
-    (38, 39, {"owner_category_0": "celebrity", "category": "helicopter_or_blimp"}),
-    (41, 42, {"owner_category_0": "other"}),
-    (44, 45, {"owner_category_0": "government", "owner_category_1": "royal_family"}),
-    (47, 48, {"owner_category_0": "education"}),
-    (50, 51, {"owner_category_0": "charter"}),
-    (53, 54, {"owner_category_0": "unknown"}),
-    (56, 57, {"owner_category_0": "celebrity"}),   # Frequent Flyers name col, aircraft col
-]
-
-# First data row index (0-based) in the CSV
-DATA_START_ROW = 4
-
-# ── Contributor info ────────────────────────────────────────────────────────
-CONTRIBUTOR_NAME = "TheAirTraffic"
-# Deterministic UUID v5 from contributor name
-CONTRIBUTOR_UUID = str(uuid.uuid5(uuid.NAMESPACE_URL, "https://theairtraffic.com"))
-
-# Citation
-CITATION = "https://docs.google.com/spreadsheets/d/1JHhfJBnJPNBA6TgiSHjkXFkHBdVTTz_nXxaUDRWcHpk"
-
-
-def looks_like_military_serial(reg: str) -> bool:
-    """
-    Detect military-style serials like 92-9000, 82-8000, 98-0001
-    or pure numeric IDs like 929000, 828000, 980001.
-    These aren't standard civil registrations; use openairframes_id.
-    """
-    # Pattern: NN-NNNN
-    if re.match(r'^\d{2}-\d{4}$', reg):
-        return True
-    # Pure 6-digit numbers (likely ICAO hex or military mode-S)
-    if re.match(r'^\d{6}$', reg):
-        return True
-    # Short numeric-only (1-5 digits) like "01", "02", "676"
-    if re.match(r'^\d{1,5}$', reg):
-        return True
-    return False
-
-
-def normalize_reg(raw: str) -> str:
-    """Clean up a registration string."""
-    reg = raw.strip().rstrip(',').strip()
-    # Remove carriage returns and other whitespace
-    reg = reg.replace('\r', '').replace('\n', '').strip()
-    return reg
-
-
-def parse_regs(cell_value: str) -> list[str]:
-    """
-    Parse a cell that may contain one or many registrations,
-    separated by commas, possibly wrapped in quotes.
-    """
-    if not cell_value or not cell_value.strip():
-        return []
-
-    # Some cells have ADS-B exchange URLs – skip those
-    if 'globe.adsbexchange.com' in cell_value:
-        return []
-    if cell_value.strip() in ('.', ',', ''):
-        return []
-
-    results = []
-    # Split on comma
-    parts = cell_value.split(',')
-    for part in parts:
-        reg = normalize_reg(part)
-        if not reg:
-            continue
-        # Skip URLs, section labels, etc.
-        if reg.startswith('http') or reg.startswith('Link') or reg == 'Section 1':
-            continue
-        # Skip if it's just whitespace or dots
-        if reg in ('.', '..', '...'):
-            continue
-        results.append(reg)
-    return results
-
-
-def make_submission(
-    reg: str,
-    owner: str,
-    category_tags: dict[str, str],
-) -> dict:
-    """Build a single community_submission.v1 object."""
-
-    entry: dict = {}
-
-    # Decide identifier field
-    if looks_like_military_serial(reg):
-        entry["openairframes_id"] = reg
-    else:
-        entry["registration_number"] = reg
-
-    # Tags
-    tags: dict = {
-        "citation_0": CITATION,
-    }
-    if owner:
-        tags["owner"] = owner.strip()
-    tags.update(category_tags)
-    entry["tags"] = tags
-
-    return entry
-
-
-def main():
-    csv_path = Path(sys.argv[1]) if len(sys.argv) > 1 else Path(
-        "/Users/jonahgoode/Downloads/TheAirTraffic Database - Aircraft 2.csv"
-    )
-
-    if not csv_path.exists():
-        print(f"ERROR: CSV not found at {csv_path}", file=sys.stderr)
-        sys.exit(1)
-
-    # Read CSV
-    with open(csv_path, 'r', encoding='utf-8-sig') as f:
-        reader = csv.reader(f)
-        rows = list(reader)
-
-    print(f"Read {len(rows)} rows from {csv_path.name}")
-
-    date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d")
-
-    submissions: list[dict] = []
-    seen: set[tuple] = set()  # (reg, owner) dedup
-
-    for row_idx in range(DATA_START_ROW, len(rows)):
-        row = rows[row_idx]
-        if len(row) < 3:
-            continue
-
-        for name_col, reg_col, cat_tags in CATEGORY_COLUMNS:
-            if reg_col >= len(row) or name_col >= len(row):
-                continue
-
-            owner_raw = row[name_col].strip().rstrip(',').strip()
-            reg_raw = row[reg_col]
-
-            # Clean owner name
-            owner = owner_raw.replace('\r', '').replace('\n', '').strip()
-            if not owner or owner in ('.', ',', 'Section 1'):
-                continue
-            # Skip header-like values
-            if owner.startswith('http') or owner.startswith('Link '):
-                continue
-
-            regs = parse_regs(reg_raw)
-            if not regs:
-                # For Russia & Ukraine, try the next column too (col 35 might have old reg, col 36 new)
-                if name_col == 34 and reg_col + 1 < len(row):
-                    regs = parse_regs(row[reg_col + 1])
-
-            for reg in regs:
-                key = (reg, owner)
-                if key in seen:
-                    continue
-                seen.add(key)
-                submissions.append(make_submission(reg, owner, cat_tags))
-
-    print(f"Generated {len(submissions)} submissions")
-
-    # Write output
-    proj_root = Path(__file__).resolve().parent.parent
-    out_dir = proj_root / "community" / date_str
-    out_dir.mkdir(parents=True, exist_ok=True)
-
-    out_file = out_dir / f"theairtraffic_{date_str}.json"
-
-    with open(out_file, 'w', encoding='utf-8') as f:
-        json.dump(submissions, f, indent=2, ensure_ascii=False)
-
-    print(f"Written to {out_file}")
-    print(f"Sample entry:\n{json.dumps(submissions[0], indent=2)}")
-
-    # Quick stats
-    cats = {}
-    for s in submissions:
-        c = s['tags'].get('owner_category_0', 'NONE')
-        cats[c] = cats.get(c, 0) + 1
-    print("\nCategory breakdown:")
-    for c, n in sorted(cats.items(), key=lambda x: -x[1]):
-        print(f"  {c}: {n}")
-
-
-if __name__ == "__main__":
-    main()
@@ -1,69 +0,0 @@
-#!/usr/bin/env python3
-"""Validate the generated theairtraffic JSON output."""
-import json
-import glob
-import sys
-
-# Find the latest output
-files = sorted(glob.glob("community/2026-02-*/theairtraffic_*.json"))
-if not files:
-    print("No output files found!")
-    sys.exit(1)
-
-path = files[-1]
-print(f"Validating: {path}")
-
-with open(path) as f:
-    data = json.load(f)
-
-print(f"Total entries: {len(data)}")
-
-# Check military serial handling
-mil = [d for d in data if "openairframes_id" in d]
-print(f"\nEntries using openairframes_id: {len(mil)}")
-for m in mil[:10]:
-    print(f"  {m['openairframes_id']} -> owner: {m['tags'].get('owner','?')}")
-
-# Check youtuber entries
-yt = [d for d in data if d["tags"].get("owner_category_0") == "youtuber"]
-print(f"\nYouTuber entries: {len(yt)}")
-for y in yt[:5]:
-    reg = y.get("registration_number", y.get("openairframes_id"))
-    c0 = y["tags"].get("owner_category_0")
-    c1 = y["tags"].get("owner_category_1")
-    print(f"  {reg} -> owner: {y['tags']['owner']}, cat0: {c0}, cat1: {c1}")
-
-# Check US Govt / military
-gov = [d for d in data if d["tags"].get("owner") == "United States of America 747/757"]
-print(f"\nUSA 747/757 entries: {len(gov)}")
-for g in gov:
-    oid = g.get("openairframes_id", g.get("registration_number"))
-    print(f"  {oid}")
-
-# Schema validation
-issues = 0
-for i, d in enumerate(data):
-    has_id = any(k in d for k in ["registration_number", "transponder_code_hex", "openairframes_id"])
-    if not has_id:
-        print(f"  Entry {i}: no identifier!")
-        issues += 1
-    if "tags" not in d:
-        print(f"  Entry {i}: no tags!")
-        issues += 1
-    # Check tag key format
-    for k in d.get("tags", {}):
-        import re
-        if not re.match(r"^[a-z][a-z0-9_]{0,63}$", k):
-            print(f"  Entry {i}: invalid tag key '{k}'")
-            issues += 1
-
-print(f"\nSchema issues: {issues}")
-
-# Category breakdown
-cats = {}
-for s in data:
-    c = s["tags"].get("owner_category_0", "NONE")
-    cats[c] = cats.get(c, 0) + 1
-print("\nCategory breakdown:")
-for c, n in sorted(cats.items(), key=lambda x: -x[1]):
-    print(f"  {c}: {n}")
@@ -4,10 +4,6 @@ import polars as pl

 COLUMNS = ['dbFlags', 'ownOp', 'year', 'desc', 'aircraft_category', 'r', 't']

-# Positional contract for every released ADS-B artifact. polars concatenates by
-# position after .select(), so a divergent copy corrupts output without erroring.
-FINAL_COLUMN_ORDER = ['time', 'icao', 'r', 't', 'dbFlags', 'ownOp', 'year', 'desc', 'aircraft_category']
-

 def compress_df_polars(df: pl.DataFrame, icao: str) -> pl.DataFrame:
    """Compress a single ICAO group to its most informative row using Polars."""
@@ -141,7 +137,7 @@ def load_parquet_part(part_id: int, date: str) -> pl.DataFrame:
    """Load a single parquet part file for a date.
    
    Args:
-        part_id: Part ID (0-indexed, e.g. 0, 1, 2, 3)
+        part_id: Part ID (e.g., 1, 2, 3)
        date: Date string in YYYY-MM-DD format
    
    Returns:
@@ -168,7 +164,7 @@ def load_parquet_part(part_id: int, date: str) -> pl.DataFrame:
    print(f"Loading from parquet: {parquet_file}")
    df = pl.read_parquet(
        parquet_file,
-        columns=FINAL_COLUMN_ORDER
+        columns=['time', 'icao', 'r', 't', 'dbFlags', 'ownOp', 'year', 'desc', 'aircraft_category']
    )
    
    # Convert to timezone-naive datetime
@@ -1,12 +1,9 @@
 from pathlib import Path
 import polars as pl
 import argparse
-import os
-import sys
-
-from src.adsb.compress_adsb_to_aircraft_data import FINAL_COLUMN_ORDER

 OUTPUT_DIR = Path("./data/output")
+CORRECT_ORDER_OF_COLUMNS = ["time", "icao", "r", "t", "dbFlags", "ownOp", "year", "desc", "aircraft_category"]

 def main():
    parser = argparse.ArgumentParser(description="Concatenate compressed parquet files for a single day")
@@ -16,62 +13,38 @@ def main():

    compressed_dir = OUTPUT_DIR / "compressed"
    date_dir = compressed_dir / args.date
+    if not date_dir.is_dir():
+        raise FileNotFoundError(f"No date folder found: {date_dir}")

    parquet_files = sorted(date_dir.glob("*.parquet"))
-    df = None
-    if parquet_files: # TODO: This logic could be updated slightly.
-        print(f"Found {len(parquet_files)} parquet part(s) in {date_dir}")
+    if not parquet_files:
+        raise FileNotFoundError(f"No parquet files found in {date_dir}")

-        frames = [pl.read_parquet(p) for p in parquet_files]
-        df = pl.concat(frames, how="vertical", rechunk=True)
+    frames = [pl.read_parquet(p) for p in parquet_files]
+    df = pl.concat(frames, how="vertical", rechunk=True)

-        df = df.sort(["time", "icao"])
-        df = df.select(FINAL_COLUMN_ORDER)
-        
-        output_path = OUTPUT_DIR / f"openairframes_adsb_{args.date}.parquet"
-        print(f"Writing combined parquet to {output_path} with {df.height} rows")
-        df.write_parquet(output_path)
+    df = df.sort(["time", "icao"])
+    df = df.select(CORRECT_ORDER_OF_COLUMNS)
+    
+    output_path = OUTPUT_DIR / f"openairframes_adsb_{args.date}.parquet"
+    print(f"Writing combined parquet to {output_path} with {df.height} rows")
+    df.write_parquet(output_path)

-        csv_output_path = OUTPUT_DIR / f"openairframes_adsb_{args.date}.csv.gz"
-        print(f"Writing combined csv.gz to {csv_output_path} with {df.height} rows")
-        df.write_csv(csv_output_path, compression="gzip")
-    elif not args.concat_with_latest_csv:
-        # Nothing to merge and no release to fall back on: exiting 0 here would let the
-        # caller mistake "produced nothing" for "succeeded".
-        print(f"ERROR: No parquet files found in {date_dir} and --concat_with_latest_csv not set")
-        sys.exit(1)
-    else:
-        print(f"No parquet files found in {date_dir}; falling back to the latest released CSV")
+    csv_output_path = OUTPUT_DIR / f"openairframes_adsb_{args.date}.csv.gz"
+    print(f"Writing combined csv.gz to {csv_output_path} with {df.height} rows")
+    df.write_csv(csv_output_path, compression="gzip")

    if args.concat_with_latest_csv:
        print("Loading latest CSV from GitHub releases to concatenate with...")
        from src.get_latest_release import get_latest_aircraft_adsb_csv_df
-        from datetime import datetime
-        
-        df_latest_csv, csv_start_date, csv_end_date = get_latest_aircraft_adsb_csv_df()
-        
-        # Compare dates: end_date is exclusive, so if csv_end_date > args.date, 
-        # the latest CSV already includes this day's data
-        csv_end_dt = datetime.strptime(csv_end_date, "%Y-%m-%d")
-        args_dt = datetime.strptime(args.date, "%Y-%m-%d")
-        
-        if df is None or csv_end_dt >= args_dt:
-            print(f"Latest CSV already includes data through {args.date} (end_date={csv_end_date} is exclusive)")
-            print("Writing latest CSV directly without concatenation to avoid duplicates")
-            os.makedirs(OUTPUT_DIR, exist_ok=True)
-            final_csv_output_path = OUTPUT_DIR / f"openairframes_adsb_{csv_start_date}_{csv_end_date}.csv.gz"
-            df_latest_csv = df_latest_csv.select(FINAL_COLUMN_ORDER)
-            df_latest_csv.write_csv(final_csv_output_path, compression="gzip")
-        else:
-            print(f"Concatenating latest CSV (through {csv_end_date}) with new data ({args.date})")
-            # Ensure column order matches before concatenating
-            df_latest_csv = df_latest_csv.select(FINAL_COLUMN_ORDER)
-            from src.adsb.compress_adsb_to_aircraft_data import concat_compressed_dfs
-            df_final = concat_compressed_dfs(df_latest_csv, df)
-            df_final = df_final.select(FINAL_COLUMN_ORDER)
-            final_csv_output_path = OUTPUT_DIR / f"openairframes_adsb_{csv_start_date}_{args.date}.csv.gz"
-            df_final.write_csv(final_csv_output_path, compression="gzip")
-        print(f"Final CSV written to {final_csv_output_path}")
+        df_latest_csv, csv_date = get_latest_aircraft_adsb_csv_df()
+        # Ensure column order matches before concatenating
+        df_latest_csv = df_latest_csv.select(CORRECT_ORDER_OF_COLUMNS)
+        from src.adsb.compress_adsb_to_aircraft_data import concat_compressed_dfs
+        df_final = concat_compressed_dfs(df_latest_csv, df)
+        df_final = df_final.select(CORRECT_ORDER_OF_COLUMNS)
+        final_csv_output_path = OUTPUT_DIR / f"openairframes_adsb_{csv_date}_{args.date}.csv.gz"
+        df_final.write_csv(final_csv_output_path, compression="gzip")

 if __name__ == "__main__":
    main()
@@ -93,19 +93,6 @@ def _fetch_releases_from_repo(year: str, version_date: str) -> list:
                        else:
                            print(f"Giving up after {max_retries} attempts")
                            return releases
-            except urllib.error.HTTPError as e:
-                # 404 means the repo/page does not exist. Retrying cannot change that,
-                # and 10 attempts x 5 min burns ~45 min of runner time to learn nothing.
-                if e.code == 404:
-                    print(f"Not found (HTTP 404): {BASE_URL}?page={page} - not retrying")
-                    return releases
-                print(f"Request exception (attempt {attempt}/{max_retries}): {e}")
-                if attempt < max_retries:
-                    print(f"Waiting {retry_delay} seconds before retry")
-                    time.sleep(retry_delay)
-                else:
-                    print(f"Giving up after {max_retries} attempts")
-                    return releases
            except Exception as e:
                print(f"Request exception (attempt {attempt}/{max_retries}): {e}")
                if attempt < max_retries:
@@ -142,32 +129,13 @@ def fetch_releases(version_date: str) -> list:
    return releases


-def download_asset(asset_url: str, file_path: str, expected_size: int | None = None) -> bool:
-    """Download a single release asset with size verification.
-    
-    Args:
-        asset_url: URL to download from
-        file_path: Local path to save to
-        expected_size: Expected file size in bytes (for verification)
-    
-    Returns:
-        True if download succeeded and size matches (if provided), False otherwise
-    """
+def download_asset(asset_url: str, file_path: str) -> bool:
+    """Download a single release asset."""
    os.makedirs(os.path.dirname(file_path) or OUTPUT_DIR, exist_ok=True)
    
-    # Check if file exists and has correct size
    if os.path.exists(file_path):
-        if expected_size is not None:
-            actual_size = os.path.getsize(file_path)
-            if actual_size == expected_size:
-                print(f"[SKIP] {file_path} already downloaded and verified ({actual_size} bytes).")
-                return True
-            else:
-                print(f"[WARN] {file_path} exists but size mismatch (expected {expected_size}, got {actual_size}). Re-downloading.")
-                os.remove(file_path)
-        else:
-            print(f"[SKIP] {file_path} already downloaded.")
-            return True
+        print(f"[SKIP] {file_path} already downloaded.")
+        return True
    
    max_retries = 2
    retry_delay = 30
@@ -185,21 +153,7 @@ def download_asset(asset_url: str, file_path: str, expected_size: int | None = N
                            if not chunk:
                                break
                            file.write(chunk)
-                    
-                    # Verify file size if expected_size was provided
-                    if expected_size is not None:
-                        actual_size = os.path.getsize(file_path)
-                        if actual_size != expected_size:
-                            print(f"[ERROR] Size mismatch for {file_path}: expected {expected_size} bytes, got {actual_size} bytes")
-                            os.remove(file_path)
-                            if attempt < max_retries:
-                                print(f"Waiting {retry_delay} seconds before retry")
-                                time.sleep(retry_delay)
-                                continue
-                            return False
-                        print(f"Saved {file_path} ({actual_size} bytes, verified)")
-                    else:
-                        print(f"Saved {file_path}")
+                    print(f"Saved {file_path}")
                    return True
                else:
                    print(f"Failed to download {asset_url}: {response.status} {response.msg}")
@@ -273,6 +227,7 @@ def extract_split_archive(file_paths: list, extract_dir: str) -> bool:
            stdin=cat_proc.stdout,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
+            check=True
        )
        cat_proc.stdout.close()
        cat_stderr = cat_proc.stderr.read().decode() if cat_proc.stderr else ""
@@ -281,24 +236,6 @@ def extract_split_archive(file_paths: list, extract_dir: str) -> bool:
        if cat_stderr:
            print(f"cat stderr: {cat_stderr}")
        
-        tar_stderr = result.stderr.decode() if result.stderr else ""
-        if result.returncode != 0:
-            # GNU tar exits non-zero for format issues that BSD tar silently
-            # tolerates (e.g. trailing junk after the last valid entry).
-            # Check whether files were actually extracted before giving up.
-            extracted_items = os.listdir(extract_dir)
-            if extracted_items:
-                print(f"[WARN] tar exited {result.returncode} but extracted "
-                      f"{len(extracted_items)} items — treating as success")
-                if tar_stderr:
-                    print(f"tar stderr: {tar_stderr}")
-            else:
-                print(f"Failed to extract split archive (tar exit {result.returncode})")
-                if tar_stderr:
-                    print(f"tar stderr: {tar_stderr}")
-                shutil.rmtree(extract_dir, ignore_errors=True)
-                return False
-        
        print(f"Successfully extracted archive to {extract_dir}")
        
        # Delete tar files immediately after extraction
@@ -315,9 +252,11 @@ def extract_split_archive(file_paths: list, extract_dir: str) -> bool:
        print(f"Disk space after tar deletion: {free_gb:.1f}GB free")
        
        return True
-    except Exception as e:
+    except subprocess.CalledProcessError as e:
+        stderr_output = e.stderr.decode() if e.stderr else ""
        print(f"Failed to extract split archive: {e}")
-        shutil.rmtree(extract_dir, ignore_errors=True)
+        if stderr_output:
+            print(f"tar stderr: {stderr_output}")
        return False


@@ -77,9 +77,8 @@ def download_and_extract(version_date: str) -> str | None:
            for asset in use_assets:
                asset_name = asset["name"]
                asset_url = asset["browser_download_url"]
-                asset_size = asset.get("size")  # Get expected file size
                file_path = os.path.join(OUTPUT_DIR, asset_name)
-                if download_asset(asset_url, file_path, expected_size=asset_size):
+                if download_asset(asset_url, file_path):
                    downloaded_files.append(file_path)
    
    if not downloaded_files:
@@ -116,23 +116,14 @@ from pathlib import Path

 def main():
    parser = argparse.ArgumentParser(description="Process a single archive part for a day")
-    parser.add_argument("--part-id", type=int, required=True, help="Part ID (0-indexed)")
+    parser.add_argument("--part-id", type=int, required=True, help="Part ID (1-indexed)")
    parser.add_argument("--date", type=str, required=True, help="Date in YYYY-MM-DD format")
    args = parser.parse_args()
    
    print(f"Processing part {args.part_id} for {args.date}")
    
    # Get specific archive file for this part
-    archive_dir = os.path.join(OUTPUT_DIR, "adsb_archives", args.date)
-    archive_path = os.path.join(archive_dir, f"{args.date}_part_{args.part_id}.tar.gz")
-    
-    if not os.path.isfile(archive_path):
-        print(f"ERROR: Archive not found: {archive_path}")
-        if os.path.isdir(archive_dir):
-            print(f"Files in {archive_dir}: {os.listdir(archive_dir)}")
-        else:
-            print(f"Directory does not exist: {archive_dir}")
-        sys.exit(1)
+    archive_path = os.path.join(OUTPUT_DIR, "adsb_archives", args.date, f"{args.date}_part_{args.part_id}.tar.gz")
    
    # Extract and collect trace files
    trace_map = build_trace_file_map(archive_path)
@@ -1,128 +0,0 @@
-"""Join the per-source registry CSVs into one union table.
-
-Every source publishes its own `openairframes_<source>_{start}_{end}.csv` on its own thread.
-This reads whatever landed, aligns them on the union of columns, and writes a single
-`openairframes_registry_{start}_{end}.csv` discriminated by the `source` column.
-
-Adding a registry means adding a source to the workflow matrix; nothing here changes.
-
-Usage:
-    python src/build_registry.py --input-dir artifacts/registry --date 2026-08-31
-"""
-from datetime import datetime, timezone
-from pathlib import Path
-import argparse
-import re
-import sys
-
-import pandas as pd
-
-# Sources that are registries. Community and ADS-B are published separately: they are
-# observations and contributions, not registration records, and do not share this schema.
-FILENAME_RE = re.compile(
-    r"\Aopenairframes_(?P<source>[a-z0-9_]+?)_"
-    r"(?P<start>\d{4}-\d{2}-\d{2})_(?P<end>\d{4}-\d{2}-\d{2})\.csv\Z"
-)
-EXCLUDED_SOURCES = {"community", "adsb", "registry"}
-
-# Identifier columns lead the union so the table is usable without reading 70 headers.
-LEADING_COLUMNS = [
-    "download_date",
-    "source",
-    "transponder_code_hex",
-    "registration_number",
-    "openairframes_id",
-]
-
-
-def discover(input_dir: Path) -> list[tuple[str, str, str, Path]]:
-    """Return (source, start, end, path) for each per-source registry CSV found."""
-    found = []
-    for path in sorted(input_dir.rglob("openairframes_*.csv")):
-        match = FILENAME_RE.match(path.name)
-        if not match:
-            print(f"  SKIP {path.name}: does not match {FILENAME_RE.pattern}")
-            continue
-        source = match.group("source")
-        if source in EXCLUDED_SOURCES:
-            print(f"  SKIP {path.name}: {source!r} is published as its own asset")
-            continue
-        found.append((source, match.group("start"), match.group("end"), path))
-    return found
-
-
-def build(input_dir: Path, date_str: str) -> tuple[pd.DataFrame, str, str]:
-    parts = discover(input_dir)
-    if not parts:
-        raise SystemExit(f"No per-source registry CSVs found under {input_dir}")
-
-    seen = [p[0] for p in parts]
-    duplicated = {s for s in seen if seen.count(s) > 1}
-    if duplicated:
-        raise SystemExit(f"More than one file claims source {sorted(duplicated)}")
-
-    frames = []
-    for source, _, _, path in parts:
-        # keep_default_na=False so a literal "NA" survives the round trip unchanged.
-        df = pd.read_csv(path, dtype=str, keep_default_na=False)
-        if "source" not in df.columns:
-            raise SystemExit(f"{path.name}: no source column; cannot discriminate rows")
-        if df.empty:
-            print(f"  {source}: empty, skipping")
-            continue
-        actual = {v.strip().lower() for v in df["source"].unique()}
-        if actual != {source.lower()}:
-            # A file whose rows disagree with its name would duplicate another source into
-            # the union under the wrong label.
-            raise SystemExit(
-                f"{path.name}: filename says {source!r}, rows say {sorted(actual)}"
-            )
-        print(f"  {source}: {len(df)} rows, {len(df.columns)} columns from {path.name}")
-        frames.append(df)
-
-    if not frames:
-        raise SystemExit("Every discovered source was empty; refusing to publish an empty registry")
-
-    if len(frames) > 1:
-        shared = set.intersection(*(set(f.columns) for f in frames)) - set(LEADING_COLUMNS)
-        print(f"  columns shared across sources ({len(shared)}): {sorted(shared)}")
-
-    columns = list(dict.fromkeys(c for df in frames for c in df.columns))
-    ordered = [c for c in LEADING_COLUMNS if c in columns]
-    ordered += [c for c in columns if c not in ordered]
-
-    # reindex rather than concat directly: a source missing a column must yield an empty
-    # cell, never a shifted row.
-    df_union = pd.concat([df.reindex(columns=ordered) for df in frames], ignore_index=True)
-    df_union = df_union.fillna("")
-
-    # Earliest start across sources, not per-source coverage: a source added today still
-    # carries the oldest source's start date in the filename.
-    start = min(p[1] for p in parts)
-    end = max(p[2] for p in parts + [("", "", date_str, Path())])
-    return df_union, start, end
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(description="Join per-source registry CSVs into one table")
-    parser.add_argument("--input-dir", default="artifacts/registry", help="Directory to search")
-    parser.add_argument("--output-dir", default="data/openairframes", help="Where to write")
-    parser.add_argument("--date", help="Run date (YYYY-MM-DD, default: today UTC)")
-    args = parser.parse_args()
-
-    date_str = args.date or datetime.now(timezone.utc).strftime("%Y-%m-%d")
-    df, start, end = build(Path(args.input_dir), date_str)
-
-    out_dir = Path(args.output_dir)
-    out_dir.mkdir(parents=True, exist_ok=True)
-    out_path = out_dir / f"openairframes_registry_{start}_{end}.csv"
-    df.to_csv(out_path, index=False)
-
-    print(f"Wrote {out_path}: {len(df)} rows, {len(df.columns)} columns")
-    print("  rows per source:")
-    for source, count in df["source"].value_counts().items():
-        print(f"    {source}: {count}")
-
-
-if __name__ == "__main__":
-    main()
@@ -0,0 +1,14 @@
+#unique_regulatory_id
+# 1. read historoical and output
+# 2. read sequentially
+
+# Instead of reading all csvs I can read just the latest release csv to get everything.
+
+from pathlib import Path
+
+base = Path("data/faa_releasable_historical")
+for day_dir in sorted(base.glob("2024-02-*")):
+    master = day_dir / "Master.txt"
+    if master.exists():
+        out_csv = master_txt_to_releasable_csv(master, out_dir="data/faa_releasable_historical_csv")
+        print(day_dir.name, "->", out_csv)
@@ -21,7 +21,7 @@ import urllib.request
 import urllib.error
 from datetime import datetime, timezone

-from .schema import extract_json_from_issue_body, extract_contributor_name_from_issue_body, parse_and_validate, load_schema, get_schema_path
+from .schema import extract_json_from_issue_body, extract_contributor_name_from_issue_body, parse_and_validate, load_schema, SCHEMAS_DIR
 from .contributor import (
    generate_contributor_uuid,
    generate_submission_filename,
@@ -72,14 +72,9 @@ def add_issue_comment(issue_number: int, body: str) -> None:
    github_api_request("POST", f"/issues/{issue_number}/comments", {"body": body})


-def get_default_branch() -> str:
-    """Get the repository's default branch name."""
-    return github_api_request("GET", "")["default_branch"]
-
-
-def get_branch_sha(branch: str) -> str:
-    """Get the head SHA of a branch."""
-    ref = github_api_request("GET", f"/git/ref/heads/{branch}")
+def get_default_branch_sha() -> str:
+    """Get the SHA of the default branch (main)."""
+    ref = github_api_request("GET", "/git/ref/heads/main")
    return ref["object"]["sha"]


@@ -204,14 +199,14 @@ def process_submission(
    
    # Create branch
    branch_name = f"community-submission-{issue_number}"
-    base_branch = get_default_branch()
-    create_branch(branch_name, get_branch_sha(base_branch))
+    default_sha = get_default_branch_sha()
+    create_branch(branch_name, default_sha)
    
    # Create file
    commit_message = f"Add community submission from @{author_username} (closes #{issue_number})"
    create_or_update_file(file_path, content_json, commit_message, branch_name)
    
-    # Update schema with any new tags (rewrites the resolved schema version in place)
+    # Update schema with any new tags (modifies v1 in place)
    schema_updated = False
    new_tags = []
    try:
@@ -237,7 +232,7 @@ def process_submission(
            schema_json = json.dumps(updated_schema, indent=2) + "\n"
            
            create_or_update_file(
-                f"schemas/{get_schema_path().name}",
+                "schemas/community_submission.v1.schema.json",
                schema_json,
                f"Update schema with new tags: {', '.join(new_tags)}",
                branch_name
@@ -251,20 +246,6 @@ def process_submission(
    if schema_updated:
        schema_note = f"\n**Schema Updated:** Added new tags: `{', '.join(new_tags)}`\n"
    
-    # Truncate JSON preview to stay under GitHub's 65536 char body limit
-    max_json_preview = 50000
-    if len(content_json) > max_json_preview:
-        # Show first few entries as a preview
-        preview_entries = submissions[:10]
-        preview_json = json.dumps(preview_entries, indent=2, sort_keys=True)
-        json_section = (
-            f"### Submissions (showing 10 of {len(submissions)})\n"
-            f"```json\n{preview_json}\n```\n\n"
-            f"*Full submission ({len(submissions)} entries, {len(content_json):,} chars) is in the committed file.*"
-        )
-    else:
-        json_section = f"### Submissions\n```json\n{content_json}\n```"
-
    pr_body = f"""## Community Submission

 Adds {len(submissions)} submission(s) from @{author_username}.
@@ -276,12 +257,15 @@ Closes #{issue_number}

 ---

-{json_section}"""
+### Submissions
+```json
+{content_json}
+```"""
    
    pr = create_pull_request(
        title=f"Community submission: {filename}",
        head=branch_name,
-        base=base_branch,
+        base="main",
        body=pr_body,
    )
    
@@ -24,7 +24,7 @@ def read_all_submissions(community_dir: Path) -> list[dict]:
    """Read all JSON submissions from the community directory."""
    all_submissions = []
    
-    for json_file in sorted(community_dir.glob("**/*.json")):
+    for json_file in sorted(community_dir.glob("*.json")):
        try:
            with open(json_file) as f:
                data = json.load(f)
@@ -20,7 +20,7 @@ from src.contributions.update_schema import (
    check_for_new_tags,
    generate_updated_schema,
 )
-from src.contributions.schema import load_schema, get_schema_path
+from src.contributions.schema import load_schema, SCHEMAS_DIR


 def main():
@@ -51,8 +51,8 @@ def main():
        # Generate updated schema
        updated_schema = generate_updated_schema(current_schema, tag_registry)
        
-        # Write back to whichever version load_schema() resolved to
-        schema_path = get_schema_path()
+        # Write updated schema (in place)
+        schema_path = SCHEMAS_DIR / "community_submission.v1.schema.json"
        with open(schema_path, 'w') as f:
            json.dump(updated_schema, f, indent=2)
            f.write("\n")
@@ -12,6 +12,9 @@ except ImportError:

 SCHEMAS_DIR = Path(__file__).parent.parent.parent / "schemas"

+# For backwards compatibility
+SCHEMA_PATH = SCHEMAS_DIR / "community_submission.v1.schema.json"
+

 def get_latest_schema_version() -> int:
    """
@@ -4,10 +4,6 @@ import argparse

 parser = argparse.ArgumentParser(description="Create daily FAA release")
 parser.add_argument("--date", type=str, help="Date to process (YYYY-MM-DD format, default: today)")
-parser.add_argument("--allow-bootstrap", action="store_true",
-                    help="Permit rebuilding from a single day when no published asset is found. "
-                         "Onboarding only: a missing asset is otherwise indistinguishable from a "
-                         "transient outage, and rebuilding would erase the accumulated history.")
 args = parser.parse_args()

 if args.date:
@@ -41,26 +37,13 @@ from derive_from_faa_master_txt import convert_faa_master_txt_to_df, concat_faa_
 from get_latest_release import get_latest_aircraft_faa_csv_df
 df_new = convert_faa_master_txt_to_df(zip_path, date_str)

-# Only a genuine first run may rebuild from a single day. A rate limit, a parse error or a
-# non-monotonic download_date must stop the run: this file becomes tomorrow's base, so
-# silently republishing one day erases the accumulated history.
 try:
    df_base, start_date_str = get_latest_aircraft_faa_csv_df()
-except FileNotFoundError as e:
-    if not args.allow_bootstrap:
-        raise SystemExit(
-            f"No published FAA asset found: {e}\n"
-            "This is indistinguishable from a transient outage, and rebuilding from one day "
-            "would erase the accumulated history. Pass --allow-bootstrap when onboarding."
-        ) from None
-    print(f"Bootstrapping FAA from today only (--allow-bootstrap): {e}")
-    df_base = None
-    start_date_str = date_str
-
-if df_base is not None:
    df_base = concat_faa_historical_df(df_base, df_new)
    assert df_base['download_date'].is_monotonic_increasing, "download_date is not monotonic increasing"
-else:
+except Exception as e:
+    print(f"No existing FAA release found, using only new data: {e}")
    df_base = df_new
+    start_date_str = date_str

 df_base.to_csv(OUT_ROOT / f"openairframes_faa_{start_date_str}_{date_str}.csv", index=False)
@@ -1,83 +0,0 @@
-from pathlib import Path
-from datetime import datetime, timezone
-import argparse
-
-parser = argparse.ArgumentParser(description="Create daily Transport Canada release")
-parser.add_argument("--date", type=str, help="Date to process (YYYY-MM-DD format, default: today)")
-parser.add_argument("--allow-bootstrap", action="store_true",
-                    help="Permit rebuilding from a single day when no published asset is found. "
-                         "Onboarding only: a missing asset is otherwise indistinguishable from a "
-                         "transient outage, and rebuilding would erase the accumulated history.")
-args = parser.parse_args()
-
-if args.date:
-    date_str = args.date
-else:
-    date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d")
-
-out_dir = Path("data/tc_ccarcs")
-out_dir.mkdir(parents=True, exist_ok=True)
-zip_name = f"ccarcsdb_{date_str}.zip"
-
-zip_path = out_dir / zip_name
-if not zip_path.exists():
-    url = "https://wwwapps.tc.gc.ca/saf-sec-sur/2/ccarcs-riacc/download/ccarcsdb.zip"
-    from urllib.request import Request, urlopen
-
-    # CCARCS 403s a default urllib agent. Any browser-like UA works; the exact
-    # version string is not load-bearing.
-    req = Request(
-        url,
-        headers={
-            "User-Agent": (
-                "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
-                "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
-            )
-        },
-        method="GET",
-    )
-
-    with urlopen(req, timeout=120) as r:
-        body = r.read()
-    # TC serves an HTML maintenance page with a 200, which would otherwise be cached
-    # under a .zip name and re-read on every later run.
-    if body[:2] != b"PK":
-        raise RuntimeError(f"{url} did not return a zip (got {body[:40]!r})")
-    tmp_path = zip_path.with_suffix(".part")
-    tmp_path.write_bytes(body)
-    tmp_path.replace(zip_path)
-
-OUT_ROOT = Path("data/openairframes")
-OUT_ROOT.mkdir(parents=True, exist_ok=True)
-from derive_from_tc_ccarcs import convert_tc_ccarcs_to_df
-# Named for FAA but column-agnostic: fingerprints every column except download_date.
-from derive_from_faa_master_txt import concat_faa_historical_df
-from get_latest_release import get_latest_aircraft_tc_csv_df
-df_new = convert_tc_ccarcs_to_df(zip_path, date_str)
-
-# Only a genuine first run may rebuild from a single day. Every other failure -- a rate
-# limit, a schema change, a truncated download -- must stop the run, because this file
-# becomes tomorrow's base and silently republishing one day erases the whole history.
-try:
-    df_base, start_date_str = get_latest_aircraft_tc_csv_df()
-except FileNotFoundError as e:
-    if not args.allow_bootstrap:
-        raise SystemExit(
-            f"No published Transport Canada asset found: {e}\n"
-            "This is indistinguishable from a transient outage, and rebuilding from one day "
-            "would erase the accumulated history. Pass --allow-bootstrap when onboarding."
-        ) from None
-    print(f"Bootstrapping Transport Canada from today only (--allow-bootstrap): {e}")
-    df_base = None
-    start_date_str = date_str
-
-if df_base is not None:
-    missing = set(df_base.columns) ^ set(df_new.columns)
-    if missing:
-        raise SystemExit(f"Column set changed since the last release: {sorted(missing)}")
-    df_base = concat_faa_historical_df(df_base, df_new)
-    assert df_base['download_date'].is_monotonic_increasing, "download_date is not monotonic increasing"
-else:
-    df_base = df_new
-
-df_base.to_csv(OUT_ROOT / f"openairframes_tc_{start_date_str}_{date_str}.csv", index=False)
@@ -1,248 +0,0 @@
-from pathlib import Path
-import csv
-import io
-import re
-import zipfile
-
-import pandas as pd
-
-from derive_from_faa_master_txt import normalize
-
-# CCARCS ships headerless, latin1, comma-delimited exports. Column names come from
-# carslayout.txt in the same archive and must stay in file order.
-CARSCURR_COLUMNS = [
-    "MARK", "REGISTRATION_SUB_TYPE_E", "REGISTRATION_SUB_TYPE_F", "COMMON_NAME",
-    "MODEL_NAME", "MANUFACTURERS_SERIAL_NUMBER", "MANUFACTURER_SERIAL_COMPRESSED",
-    "ID_PLATE_MANUFACTURERS_NAME", "BASIS_FOR_REGISTRATION", "BASIS_FOR_REGISTRATION_F",
-    "AIRCRAFT_CATEGORY_E", "AIRCRAFT_CATEGORY_F", "DATE_OF_IMPORT", "ENGINE_MANUF",
-    "POWERGLIDER_FLAG", "ENGINE_CATEGORY_E", "ENGINE_CATEGORY_F", "NUMBER_OF_ENGINES",
-    "NUMBER_OF_SEATS", "AIR_WEIGHT_KILOS", "SALE_REPORTED", "ISSUE_DATE",
-    "EFFECTIVE_DATE", "INEFFECTIVE_DATE", "REGISTERED_PURPOSE_E", "REGISTERED_PURPOSE_F",
-    "FLIGHT_AUTHORITY_E", "FLIGHT_AUTHORITY_F", "MANUFACTURE_OR_ASSEMBLY",
-    "COUNTRY_MANUFACTURE_ASS_E", "COUNTRY_MANUFACTURE_ASS_F", "DATE_MANUFACTURE_ASSEMBLY",
-    "BASE_OF_OPERATIONS_CTRY_E", "BASE_OF_OPERATIONS_CTRY_F", "BASE_PROVINCE_OR_STATE_E",
-    "BASE_PROVINCE_OR_STATE_F", "CITY_AIRPORT", "TYPE_CERTIFICATE_NUMBER",
-    "REGISTRATION_AUTH_STATUS_E", "REGISTRATION_AUTH_STATUS_F", "MULTIPLE_OWNER_FLAG",
-    "MODIFIED_DATE", "MODE_S_TRANSPONDER_BINARY", "PHYSICAL_FILE_REGION_E",
-    "PHYSICAL_FILE_REGION_F", "EX_MILITARY_MARK", "TRIMMED_MARK",
-]
-
-CARSOWNR_COLUMNS = [
-    "MARK_LINK", "FULL_NAME", "TRADE_NAME", "STREET_NAME", "STREET_NAME2", "CITY",
-    "PROVINCE_OR_STATE_E", "PROVINCE_OR_STATE_F", "POSTAL_CODE", "COUNTRY_E", "COUNTRY_F",
-    "TYPE_OF_OWNER_E", "TYPE_OF_OWNER_F", "ACTIVE_FLAG", "CARE_OF", "REGION_E", "REGION_F",
-    "OWNER_NAME_OLD_FORMAT", "MAIL_RECIPIENT", "TRIMMED_MARK",
-]
-
-# Mailing address of the single designated recipient, matching the registrant_* address
-# the FAA build already publishes. Addresses are per-party, so they are taken from the one
-# MAIL_RECIPIENT row rather than merged across co-owners.
-# registrant_zip_code holds the Canadian postal code: the name is the FAA's, and a union
-# table needs one column per concept, not one per country's vocabulary.
-OWNER_ADDRESS_COLUMNS = {
-    "STREET_NAME": "registrant_street_1",
-    "STREET_NAME2": "registrant_street_2",
-    "CITY": "registrant_city",
-    "POSTAL_CODE": "registrant_zip_code",
-    "CARE_OF": "registrant_care_of",
-}
-
-
-FOOTER_RE = re.compile(r"\s*(\d+) rows selected\.\s*")
-
-# Floor, not an expectation: Canada's register is ~35k aircraft and carsownr is larger
-# still, so 1000 only catches a grossly truncated export. The footer row-count check above
-# is what actually validates the parse; this guards the case where the footer agrees with a
-# near-empty body.
-MIN_EXPECTED_ROWS = 1000
-
-
-def _read_ccarcs_entry(zip_path: Path, entry: str, columns: list[str]) -> pd.DataFrame:
-    """Read one headerless CCARCS export into a DataFrame.
-
-    Raises:
-        ValueError: on any row whose width is neither the declared column count nor a
-            blank/footer line, on a missing or disagreeing "N rows selected." footer, or
-            on a row count below MIN_EXPECTED_ROWS.
-    """
-    with zipfile.ZipFile(zip_path) as z:
-        text = z.read(entry).decode("latin1")
-
-    rows = []
-    declared = None
-    # newline="" so a CRLF export does not leave \r on the final field of every row.
-    for row in csv.reader(io.StringIO(text, newline="")):
-        if len(row) == len(columns):
-            rows.append([cell.strip() for cell in row])
-            continue
-        if not row or not any(cell.strip() for cell in row):
-            continue  # trailing blank line
-        match = FOOTER_RE.fullmatch(row[0]) if len(row) == 1 else None
-        if match:
-            declared = int(match.group(1))
-            continue
-        raise ValueError(
-            f"{entry}: row with {len(row)} fields, expected {len(columns)}: {row[:3]!r}"
-        )
-
-    # The spool footer is a free checksum from the source; a short export is otherwise
-    # indistinguishable from a genuinely smaller register.
-    if declared is None:
-        raise ValueError(f"{entry}: no 'N rows selected.' footer; export is truncated")
-    if declared != len(rows):
-        raise ValueError(f"{entry}: footer declares {declared} rows, parsed {len(rows)}")
-    if len(rows) < MIN_EXPECTED_ROWS:
-        raise ValueError(f"{entry}: only {len(rows)} rows, expected >= {MIN_EXPECTED_ROWS}")
-
-    return pd.DataFrame(rows, columns=columns)
-
-
-def tc_full_registration(mark: str) -> str:
-    """Expand a trimmed CCARCS mark into the full Canadian registration.
-
-    CCARCS stores the bare mark in both MARK and TRIMMED_MARK, so the prefix has to be
-    reconstructed: three-character marks are vintage CF- registrations, everything else
-    takes the modern C- prefix. Returns "" for a blank mark.
-    """
-    mark = (mark or "").strip().upper()
-    if not mark:
-        return ""
-    return f"CF-{mark}" if len(mark) == 3 else f"C-{mark}"
-
-
-def binary_to_hex(binary: str) -> str:
-    """Convert a 24-bit Mode S binary string to a 6-digit uppercase hex address.
-
-    Returns "" for empty, non-binary, or non-24-bit input. Width is checked because
-    this column is the join key against ADS-B data: a short field would otherwise
-    zero-pad into a plausible address belonging to a different aircraft.
-    """
-    binary = (binary or "").strip()
-    if len(binary) != 24 or any(c not in "01" for c in binary):
-        return ""
-    return f"{int(binary, 2):06X}"
-
-
-def _merge_owners(df_ownr: pd.DataFrame) -> pd.DataFrame:
-    """Collapse the active registered parties for each mark into a single row.
-
-    A co-owned mark repeats with a different party each time; keeping only the mail
-    recipient would silently drop the rest.
-
-    Each field is deduplicated and blank-skipped independently, so the values are NOT
-    index-parallel: a mark with three owners can emit three names but one province.
-    Consumers must not split on ", " and zip the columns together.
-    """
-    # ACTIVE_FLAG is "A"/"I", but "I" does not mean "former owner": 1,932 currently
-    # Registered marks carry only "I" parties, and those rows are the MAIL_RECIPIENT.
-    # So prefer active parties where a mark has any, and fall back to all of them
-    # rather than publishing a registered aircraft with no owner at all.
-    all_parties = df_ownr
-    active = df_ownr[df_ownr["ACTIVE_FLAG"].str.upper() == "A"]
-    marks_with_active = set(active["TRIMMED_MARK"])
-    df_ownr = pd.concat([
-        active,
-        df_ownr[~df_ownr["TRIMMED_MARK"].isin(marks_with_active)],
-    ])
-    def join_unique(series: pd.Series) -> str:
-        seen = []
-        for value in series:
-            value = (value or "").strip()
-            if value and value not in seen:
-                seen.append(value)
-        return ", ".join(seen)
-
-    def count_distinct(series: pd.Series) -> int:
-        return len({v.strip() for v in series if v and v.strip()})
-
-    grouped = df_ownr.groupby("TRIMMED_MARK", sort=False).agg(
-        registrant_name=("FULL_NAME", join_unique),
-        registrant_state=("PROVINCE_OR_STATE_E", join_unique),
-        registrant_country=("COUNTRY_E", join_unique),
-        registrant_type=("TYPE_OF_OWNER_E", join_unique),
-        registrant_party_count=("FULL_NAME", count_distinct),
-    ).reset_index()
-
-    # A party row states its own type ("Individual"); that stops being true of the mark
-    # once several parties share it. Counting distinct names rather than rows keeps this
-    # consistent with owner_name, which is also deduplicated.
-    grouped.loc[grouped["registrant_party_count"] > 1, "registrant_type"] = "Co-owner"
-
-    # Taken from the unfiltered frame: the designated recipient is the designated
-    # recipient even when its own party row is flagged inactive.
-    recipient = (
-        all_parties[all_parties["MAIL_RECIPIENT"].str.upper() == "Y"]
-        .drop_duplicates(subset="TRIMMED_MARK", keep="first")
-        .rename(columns=OWNER_ADDRESS_COLUMNS)
-    )
-    return grouped.merge(
-        recipient[["TRIMMED_MARK", *OWNER_ADDRESS_COLUMNS.values()]],
-        on="TRIMMED_MARK",
-        how="left",
-    )
-
-
-def convert_tc_ccarcs_to_df(zip_path: Path, date: str) -> pd.DataFrame:
-    """Build the OpenAirframes Transport Canada frame from a CCARCS zip."""
-    df = _read_ccarcs_entry(zip_path, "carscurr.txt", CARSCURR_COLUMNS)
-    df_ownr = _read_ccarcs_entry(zip_path, "carsownr.txt", CARSOWNR_COLUMNS)
-
-    df = df.merge(_merge_owners(df_ownr), on="TRIMMED_MARK", how="left")
-
-    out = pd.DataFrame({
-        "download_date": date,
-        # The FAA frame already carries `source`; it is the union discriminator.
-        "source": "TC",
-        "transponder_code_hex": df["MODE_S_TRANSPONDER_BINARY"].map(binary_to_hex),
-        "registration_number": df["TRIMMED_MARK"].map(tc_full_registration),
-        "mark": df["TRIMMED_MARK"],
-        "aircraft_manufacturer": df["COMMON_NAME"],
-        "aircraft_model": df["MODEL_NAME"],
-        "serial_number": df["MANUFACTURERS_SERIAL_NUMBER"],
-        "aircraft_category": df["AIRCRAFT_CATEGORY_E"],
-        "engine_manufacturer": df["ENGINE_MANUF"],
-        "engine_category": df["ENGINE_CATEGORY_E"],
-        "aircraft_number_of_engines": df["NUMBER_OF_ENGINES"],
-        "aircraft_number_of_seats": df["NUMBER_OF_SEATS"],
-        "max_weight_kilos": df["AIR_WEIGHT_KILOS"],
-        "status": df["REGISTRATION_AUTH_STATUS_E"],
-        "registration_sub_type": df["REGISTRATION_SUB_TYPE_E"],
-        "basis_for_registration": df["BASIS_FOR_REGISTRATION"],
-        "registered_purpose": df["REGISTERED_PURPOSE_E"],
-        "flight_authority": df["FLIGHT_AUTHORITY_E"],
-        "type_certificate_number": df["TYPE_CERTIFICATE_NUMBER"],
-        "country_manufacture": df["COUNTRY_MANUFACTURE_ASS_E"],
-        "date_manufacture_assembly": df["DATE_MANUFACTURE_ASSEMBLY"],
-        "base_country": df["BASE_OF_OPERATIONS_CTRY_E"],
-        "base_province_or_state": df["BASE_PROVINCE_OR_STATE_E"],
-        "city_airport": df["CITY_AIRPORT"],
-        "ex_military_mark": df["EX_MILITARY_MARK"],
-        "multiple_owner_flag": df["MULTIPLE_OWNER_FLAG"],
-        "registrant_name": df["registrant_name"],
-        "registrant_type": df["registrant_type"],
-        "registrant_state": df["registrant_state"],
-        "registrant_country": df["registrant_country"],
-        "registrant_care_of": df["registrant_care_of"],
-        "registrant_street_1": df["registrant_street_1"],
-        "registrant_street_2": df["registrant_street_2"],
-        "registrant_city": df["registrant_city"],
-        "registrant_zip_code": df["registrant_zip_code"],
-        "issue_date": df["ISSUE_DATE"],
-        "effective_date": df["EFFECTIVE_DATE"],
-        "ineffective_date": df["INEFFECTIVE_DATE"],
-        "modified_date": df["MODIFIED_DATE"],
-    })
-
-    # Position matches the FAA frame (after registration_number). Ordering is cosmetic:
-    # concat_faa_historical_df reindexes df_new to the base's columns before merging.
-    out.insert(3, "openairframes_id", (
-        normalize(out["aircraft_manufacturer"])
-        + "|"
-        + normalize(out["aircraft_model"])
-        + "|"
-        + normalize(out["serial_number"])
-    ))
-
-    out = out.fillna("")
-    out = out.replace("None", "")
-    return out
@@ -3,7 +3,6 @@ from __future__ import annotations
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Iterable, Optional
-import os
 import re
 import urllib.request
 import urllib.error
@@ -28,33 +27,6 @@ def _http_get_json(url: str, headers: dict[str, str]) -> dict:
    return json.loads(data.decode("utf-8"))


-def get_releases(repo: str = REPO, github_token: Optional[str] = None, per_page: int = 30) -> list[dict]:
-    """Get a list of releases from the repository."""
-    url = f"https://api.github.com/repos/{repo}/releases?per_page={per_page}"
-    headers = {
-        "Accept": "application/vnd.github+json",
-        "User-Agent": "openairframes-downloader/1.0",
-    }
-    if github_token:
-        headers["Authorization"] = f"Bearer {github_token}"
-
-    return _http_get_json(url, headers=headers)
-
-
-def get_release_assets_from_release_data(release_data: dict) -> list[ReleaseAsset]:
-    """Extract assets from a release data dictionary."""
-    assets = []
-    for a in release_data.get("assets", []):
-        assets.append(
-            ReleaseAsset(
-                name=a["name"],
-                download_url=a["browser_download_url"],
-                size=int(a.get("size", 0)),
-            )
-        )
-    return assets
-
-
 def get_latest_release_assets(repo: str = REPO, github_token: Optional[str] = None) -> list[ReleaseAsset]:
    url = f"https://api.github.com/repos/{repo}/releases/latest"
    headers = {
@@ -65,7 +37,16 @@ def get_latest_release_assets(repo: str = REPO, github_token: Optional[str] = No
        headers["Authorization"] = f"Bearer {github_token}"

    payload = _http_get_json(url, headers=headers)
-    return get_release_assets_from_release_data(payload)
+    assets = []
+    for a in payload.get("assets", []):
+        assets.append(
+            ReleaseAsset(
+                name=a["name"],
+                download_url=a["browser_download_url"],
+                size=int(a.get("size", 0)),
+            )
+        )
+    return assets


 def pick_asset(
@@ -139,29 +120,15 @@ def download_latest_aircraft_csv(
        Path to the downloaded file
    """
    output_dir = Path(output_dir)
-    github_token = github_token or os.environ.get("GITHUB_TOKEN")
-
-    for release in get_releases(repo, github_token=github_token, per_page=30):
-        assets = get_release_assets_from_release_data(release)
-        try:
-            asset = pick_asset(assets, name_regex=r"^openairframes_faa_.*\.csv$")
-        except FileNotFoundError:
-            try:
-                # Fallback to old naming pattern
-                asset = pick_asset(assets, name_regex=r"^openairframes_\d{4}-\d{2}-\d{2}_.*\.csv$")
-            except FileNotFoundError:
-                continue
-        saved_to = download_asset(asset, output_dir / asset.name, github_token=github_token)
-        if asset.size and saved_to.stat().st_size != asset.size:
-            raise RuntimeError(
-                f"{asset.name}: downloaded {saved_to.stat().st_size} bytes, expected {asset.size}"
-            )
-        print(f"Downloaded: {asset.name} ({asset.size} bytes) -> {saved_to}")
-        return saved_to
-
-    raise FileNotFoundError(
-        "No release in the last 30 releases has an asset matching 'openairframes_faa_.*\\.csv$'"
-    )
+    assets = get_latest_release_assets(repo, github_token=github_token)
+    try:
+        asset = pick_asset(assets, name_regex=r"^openairframes_faa_.*\.csv$")
+    except FileNotFoundError:
+        # Fallback to old naming pattern
+        asset = pick_asset(assets, name_regex=r"^openairframes_\d{4}-\d{2}-\d{2}_.*\.csv$")
+    saved_to = download_asset(asset, output_dir / asset.name, github_token=github_token)
+    print(f"Downloaded: {asset.name} ({asset.size} bytes) -> {saved_to}")
+    return saved_to

 def get_latest_aircraft_faa_csv_df():
    csv_path = download_latest_aircraft_csv()
@@ -182,77 +149,13 @@ def get_latest_aircraft_faa_csv_df():
    return df, date_str


-def download_latest_aircraft_tc_csv(
-    output_dir: Path = Path("downloads"),
-    github_token: Optional[str] = None,
-    repo: str = REPO,
-) -> Path:
-    """
-    Download the latest openairframes_tc_*.csv file from the latest GitHub release.
-
-    Args:
-        output_dir: Directory to save the downloaded file (default: "downloads")
-        github_token: Optional GitHub token for authentication
-        repo: GitHub repository in format "owner/repo" (default: REPO)
-
-    Returns:
-        Path to the downloaded file
-    """
-    output_dir = Path(output_dir)
-    github_token = github_token or os.environ.get("GITHUB_TOKEN")
-
-    # Walk back through releases rather than reading only `latest`. The TC asset is
-    # optional, so a single failed build publishes a release without it; anchoring on
-    # `latest` would then make the caller rebuild history from one day and republish
-    # that as the whole dataset.
-    for release in get_releases(repo, github_token=github_token, per_page=30):
-        assets = get_release_assets_from_release_data(release)
-        try:
-            asset = pick_asset(assets, name_regex=r"^openairframes_tc_.*\.csv$")
-        except FileNotFoundError:
-            continue
-        saved_to = download_asset(asset, output_dir / asset.name, github_token=github_token)
-        if asset.size and saved_to.stat().st_size != asset.size:
-            raise RuntimeError(
-                f"{asset.name}: downloaded {saved_to.stat().st_size} bytes, expected {asset.size}"
-            )
-        print(f"Downloaded: {asset.name} ({asset.size} bytes) -> {saved_to}")
-        return saved_to
-
-    raise FileNotFoundError(
-        "No release in the last 30 releases has an asset matching 'openairframes_tc_.*\\.csv$'"
-    )
-
-
-def get_latest_aircraft_tc_csv_df():
-    """Return (DataFrame, start_date_str) for the most recent published TC release.
-
-    Raises FileNotFoundError when no recent release carries a TC asset, and ValueError
-    when the asset filename has no parseable start date.
-    """
-    csv_path = download_latest_aircraft_tc_csv()
-    import pandas as pd
-    # keep_default_na=False: a literal "NA"/"N/A" in the source would otherwise read back
-    # as NaN -> "" while the fresh parse keeps the string, so the row fingerprints would
-    # never match and every affected record would re-append on every run.
-    df = pd.read_csv(csv_path, dtype=str, keep_default_na=False)
-    df = df.fillna("")
-    # Only the start date is taken; the end date is always the run's own date.
-    match = re.search(r"openairframes_tc_(\d{4}-\d{2}-\d{2})_", str(csv_path))
-    if not match:
-        raise ValueError(f"Could not extract date from filename: {csv_path.name}")
-
-    return df, match.group(1)
-
-
 def download_latest_aircraft_adsb_csv(
    output_dir: Path = Path("downloads"),
    github_token: Optional[str] = None,
    repo: str = REPO,
 ) -> Path:
    """
-    Download the latest openairframes_adsb_*.csv file from GitHub releases.
-    If the latest release doesn't have the file, searches previous releases.
+    Download the latest openairframes_adsb_*.csv file from the latest GitHub release.

    Args:
        output_dir: Directory to save the downloaded file (default: "downloads")
@@ -263,33 +166,15 @@ def download_latest_aircraft_adsb_csv(
        Path to the downloaded file
    """
    output_dir = Path(output_dir)
-    
-    # Get multiple releases
-    releases = get_releases(repo, github_token=github_token, per_page=30)
-    
-    # Try each release until we find one with the matching asset
-    for release in releases:
-        assets = get_release_assets_from_release_data(release)
-        try:
-            asset = pick_asset(assets, name_regex=r"^openairframes_adsb_.*\.csv(\.gz)?$")
-            saved_to = download_asset(asset, output_dir / asset.name, github_token=github_token)
-            print(f"Downloaded: {asset.name} ({asset.size} bytes) -> {saved_to}")
-            return saved_to
-        except FileNotFoundError:
-            # This release doesn't have the matching asset, try the next one
-            continue
-    
-    raise FileNotFoundError(
-        f"No release in the last 30 releases has an asset matching 'openairframes_adsb_.*\\.csv(\\.gz)?$'"
-    )
+    assets = get_latest_release_assets(repo, github_token=github_token)
+    asset = pick_asset(assets, name_regex=r"^openairframes_adsb_.*\.csv(\.gz)?$")
+    saved_to = download_asset(asset, output_dir / asset.name, github_token=github_token)
+    print(f"Downloaded: {asset.name} ({asset.size} bytes) -> {saved_to}")
+    return saved_to

 import polars as pl
 def get_latest_aircraft_adsb_csv_df():
-    """Download and load the latest ADS-B CSV from GitHub releases.
-    
-    Returns:
-        tuple: (df, start_date, end_date) where dates are in YYYY-MM-DD format
-    """
+    """Download and load the latest ADS-B CSV from GitHub releases."""
    import re
    
    csv_path = download_latest_aircraft_adsb_csv()
@@ -313,19 +198,17 @@ def get_latest_aircraft_adsb_csv_df():
        if df[col].dtype == pl.Utf8:
            df = df.with_columns(pl.col(col).fill_null(""))
    
-    # Extract start and end dates from filename pattern: openairframes_adsb_{start_date}_{end_date}.csv[.gz]
-    match = re.search(r"openairframes_adsb_(\d{4}-\d{2}-\d{2})_(\d{4}-\d{2}-\d{2})\.csv", str(csv_path))
+    # Extract start date from filename pattern: openairframes_adsb_{start_date}_{end_date}.csv[.gz]
+    match = re.search(r"openairframes_adsb_(\d{4}-\d{2}-\d{2})_", str(csv_path))
    if not match:
-        raise ValueError(f"Could not extract dates from filename: {csv_path.name}")
+        raise ValueError(f"Could not extract date from filename: {csv_path.name}")
    
-    start_date = match.group(1)
-    end_date = match.group(2)
+    date_str = match.group(1)
    print(df.columns)
    print(df.dtypes)
-    return df, start_date, end_date
+    return df, date_str



 if __name__ == "__main__":
    download_latest_aircraft_csv()
-    download_latest_aircraft_adsb_csv()