Daily ADSB and Histoircal updates. Update readme.md

2026-04-29 06:26:07 +02:00 · 2026-02-13 11:49:18 -05:00
parent 4015a5fcf1
commit d216ea9329
32 changed files with 1489 additions and 1744 deletions
@@ -1,4 +1,4 @@
-name: OpenAirframes Daily Release
+name: openairframes-daily-release

 on:
  schedule:
@@ -76,159 +76,30 @@ jobs:
            data/faa_releasable/ReleasableAircraft_*.zip
          retention-days: 1

-  adsb-extract:
-    runs-on: ubuntu-24.04-arm
+  resolve-dates:
+    runs-on: ubuntu-latest
    if: github.event_name != 'schedule'
    outputs:
-      manifest-exists: ${{ steps.check.outputs.exists }}
+      date: ${{ steps.out.outputs.date }}
+      adsb_date: ${{ steps.out.outputs.adsb_date }}
    steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python
-        uses: actions/setup-python@v6
-        with:
-          python-version: "3.14"
-
-      - name: Install dependencies
+      - id: out
        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
-
-      - name: Download and extract ADS-B data
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          python -m src.adsb.download_and_list_icaos ${{ inputs.date && format('--date {0}', inputs.date) || '' }}
-          ls -lah data/output/
-
-      - name: Check manifest exists
-        id: check
-        run: |
-          if ls data/output/icao_manifest_*.txt 1>/dev/null 2>&1; then
-            echo "exists=true" >> "$GITHUB_OUTPUT"
+          if [ -n "${{ inputs.date }}" ]; then
+            echo "date=${{ inputs.date }}" >> "$GITHUB_OUTPUT"
+            echo "adsb_date=${{ inputs.date }}" >> "$GITHUB_OUTPUT"
          else
-            echo "exists=false" >> "$GITHUB_OUTPUT"
+            echo "date=$(date -u -d 'yesterday' +%Y-%m-%d)" >> "$GITHUB_OUTPUT"
+            echo "adsb_date=$(date -u -d 'yesterday' +%Y-%m-%d)" >> "$GITHUB_OUTPUT"
          fi

-      - name: Create tar of extracted data
-        run: |
-          cd data/output
-          tar -cf extracted_data.tar *-planes-readsb-prod-0.tar_0 icao_manifest_*.txt
-          ls -lah extracted_data.tar
-
-      - name: Upload extracted data
-        uses: actions/upload-artifact@v4
-        with:
-          name: adsb-extracted
-          path: data/output/extracted_data.tar
-          retention-days: 1
-          compression-level: 0  # Already compressed trace files
-
-  adsb-map:
-    runs-on: ubuntu-24.04-arm
-    needs: adsb-extract
-    if: github.event_name != 'schedule' && needs.adsb-extract.outputs.manifest-exists == 'true'
-    strategy:
-      fail-fast: false
-      matrix:
-        chunk: [0, 1, 2, 3]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python
-        uses: actions/setup-python@v6
-        with:
-          python-version: "3.14"
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
-
-      - name: Download extracted data
-        uses: actions/download-artifact@v4
-        with:
-          name: adsb-extracted
-          path: data/output/
-
-      - name: Extract tar
-        run: |
-          cd data/output
-          tar -xf extracted_data.tar
-          rm extracted_data.tar
-          echo "=== Contents of data/output ==="
-          ls -lah
-          echo "=== Looking for manifest ==="
-          cat icao_manifest_*.txt | head -20 || echo "No manifest found"
-          echo "=== Looking for extracted dirs ==="
-          ls -d *-planes-readsb-prod-0* 2>/dev/null || echo "No extracted dirs"
-
-      - name: Process chunk ${{ matrix.chunk }}
-        run: |
-          python -m src.adsb.process_icao_chunk --chunk-id ${{ matrix.chunk }} --total-chunks 4 ${{ inputs.date && format('--date {0}', inputs.date) || '' }}
-          mkdir -p data/output/adsb_chunks
-          ls -lah data/output/adsb_chunks/ || echo "No chunks created"
-
-      - name: Upload chunk artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: adsb-chunk-${{ matrix.chunk }}
-          path: data/output/adsb_chunks/
-          retention-days: 1
-
-  adsb-reduce:
-    runs-on: ubuntu-24.04-arm
-    needs: adsb-map
+  adsb-to-aircraft:
+    needs: resolve-dates
    if: github.event_name != 'schedule'
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-        with:
-          fetch-depth: 0
-
-      - name: Setup Python
-        uses: actions/setup-python@v6
-        with:
-          python-version: "3.14"
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
-
-      - name: Download all chunk artifacts
-        uses: actions/download-artifact@v4
-        with:
-          pattern: adsb-chunk-*
-          path: data/output/adsb_chunks/
-          merge-multiple: true
-
-      - name: Debug downloaded files
-        run: |
-          echo "=== Listing data/ ==="
-          find data/ -type f 2>/dev/null | head -50 || echo "No files in data/"
-          echo "=== Looking for parquet files ==="
-          find . -name "*.parquet" 2>/dev/null | head -20 || echo "No parquet files found"
-
-      - name: Combine chunks to CSV
-        run: |
-          mkdir -p data/output/adsb_chunks
-          ls -lah data/output/adsb_chunks/ || echo "Directory empty or does not exist"
-          python -m src.adsb.combine_chunks_to_csv --chunks-dir data/output/adsb_chunks ${{ inputs.date && format('--date {0}', inputs.date) || '' }}
-          ls -lah data/openairframes/
-
-      - name: Upload ADS-B artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: adsb-release
-          path: data/openairframes/openairframes_adsb_*.csv
-          retention-days: 1
+    uses: ./.github/workflows/adsb-to-aircraft-for-day.yaml
+    with:
+      date: ${{ needs.resolve-dates.outputs.adsb_date }}
+      concat_with_latest_csv: true

  build-community:
    runs-on: ubuntu-latest
@@ -261,11 +132,71 @@ jobs:
          path: data/openairframes/openairframes_community_*.csv
          retention-days: 1

-  create-release:
+  build-adsbexchange-json:
    runs-on: ubuntu-latest
-    needs: [build-faa, adsb-reduce, build-community]
    if: github.event_name != 'schedule'
    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.14"
+
+      - name: Run ADS-B Exchange JSON release script
+        run: |
+          python -m src.contributions.create_daily_adsbexchange_release ${{ inputs.date && format('--date {0}', inputs.date) || '' }}
+          ls -lah data/openairframes
+
+      - name: Upload ADS-B Exchange JSON artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: adsbexchange-json
+          path: data/openairframes/basic-ac-db_*.json.gz
+          retention-days: 1
+
+  build-mictronics-db:
+    runs-on: ubuntu-latest
+    if: github.event_name != 'schedule'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.14"
+
+      - name: Run Mictronics DB release script
+        continue-on-error: true
+        run: |
+          python -m src.contributions.create_daily_microtonics_release ${{ inputs.date && format('--date {0}', inputs.date) || '' }}
+          ls -lah data/openairframes
+
+      - name: Upload Mictronics DB artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: mictronics-db
+          path: data/openairframes/mictronics-db_*.zip
+          retention-days: 1
+          if-no-files-found: ignore
+
+  create-release:
+    runs-on: ubuntu-latest
+    needs: [resolve-dates, build-faa, adsb-to-aircraft, build-community, build-adsbexchange-json, build-mictronics-db]
+    if: github.event_name != 'schedule' && !cancelled()
+    steps:
+      - name: Require adsb-to-aircraft success
+        if: needs.adsb-to-aircraft.result != 'success'
+        run: |
+          echo "adsb-to-aircraft result was '${{ needs.adsb-to-aircraft.result }}', expected 'success'"
+          exit 1
+
      - name: Checkout for gh CLI
        uses: actions/checkout@v4
        with:
@@ -274,23 +205,36 @@ jobs:
          sparse-checkout-cone-mode: false

      - name: Download FAA artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v5
        with:
          name: faa-release
          path: artifacts/faa

      - name: Download ADS-B artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v5
        with:
-          name: adsb-release
+          name: openairframes_adsb-${{ needs.resolve-dates.outputs.adsb_date }}
          path: artifacts/adsb

      - name: Download Community artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v5
        with:
          name: community-release
          path: artifacts/community

+      - name: Download ADS-B Exchange JSON artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: adsbexchange-json
+          path: artifacts/adsbexchange
+
+      - name: Download Mictronics DB artifact
+        uses: actions/download-artifact@v5
+        continue-on-error: true
+        with:
+          name: mictronics-db
+          path: artifacts/mictronics
+
      - name: Debug artifact structure
        run: |
          echo "=== Full artifacts tree ==="
@@ -301,6 +245,10 @@ jobs:
          find artifacts/adsb -type f 2>/dev/null || echo "No files found in artifacts/adsb"
          echo "=== Community artifacts ==="
          find artifacts/community -type f 2>/dev/null || echo "No files found in artifacts/community"
+          echo "=== ADS-B Exchange JSON artifacts ==="
+          find artifacts/adsbexchange -type f 2>/dev/null || echo "No files found in artifacts/adsbexchange"
+          echo "=== Mictronics DB artifacts ==="
+          find artifacts/mictronics -type f 2>/dev/null || echo "No files found in artifacts/mictronics"

      - name: Prepare release metadata
        id: meta
@@ -317,9 +265,11 @@ jobs:
          
          # Find files from artifacts using find (handles nested structures)
          CSV_FILE_FAA=$(find artifacts/faa -name "openairframes_faa_*.csv" -type f 2>/dev/null | head -1)
-          CSV_FILE_ADSB=$(find artifacts/adsb -name "openairframes_adsb_*.csv" -type f 2>/dev/null | head -1)
+          CSV_FILE_ADSB=$(find artifacts/adsb -name "openairframes_adsb_*.csv.gz" -type f 2>/dev/null | head -1)
          CSV_FILE_COMMUNITY=$(find artifacts/community -name "openairframes_community_*.csv" -type f 2>/dev/null | head -1)
          ZIP_FILE=$(find artifacts/faa -name "ReleasableAircraft_*.zip" -type f 2>/dev/null | head -1)
+          JSON_FILE_ADSBX=$(find artifacts/adsbexchange -name "basic-ac-db_*.json.gz" -type f 2>/dev/null | head -1)
+          ZIP_FILE_MICTRONICS=$(find artifacts/mictronics -name "mictronics-db_*.zip" -type f 2>/dev/null | head -1)
          
          # Validate required files exist
          MISSING_FILES=""
@@ -332,12 +282,24 @@ jobs:
          if [ -z "$ZIP_FILE" ] || [ ! -f "$ZIP_FILE" ]; then
            MISSING_FILES="$MISSING_FILES FAA_ZIP"
          fi
+          if [ -z "$JSON_FILE_ADSBX" ] || [ ! -f "$JSON_FILE_ADSBX" ]; then
+            MISSING_FILES="$MISSING_FILES ADSBX_JSON"
+          fi
+          
+          # Optional files - warn but don't fail
+          OPTIONAL_MISSING=""
+          if [ -z "$ZIP_FILE_MICTRONICS" ] || [ ! -f "$ZIP_FILE_MICTRONICS" ]; then
+            OPTIONAL_MISSING="$OPTIONAL_MISSING MICTRONICS_ZIP"
+            ZIP_FILE_MICTRONICS=""
+          fi
          
          if [ -n "$MISSING_FILES" ]; then
            echo "ERROR: Missing required release files:$MISSING_FILES"
            echo "FAA CSV: $CSV_FILE_FAA"
            echo "ADSB CSV: $CSV_FILE_ADSB"
            echo "ZIP: $ZIP_FILE"
+            echo "ADSBX JSON: $JSON_FILE_ADSBX"
+            echo "MICTRONICS ZIP: $ZIP_FILE_MICTRONICS"
            exit 1
          fi
          
@@ -346,6 +308,15 @@ jobs:
          CSV_BASENAME_ADSB=$(basename "$CSV_FILE_ADSB")
          CSV_BASENAME_COMMUNITY=$(basename "$CSV_FILE_COMMUNITY" 2>/dev/null || echo "")
          ZIP_BASENAME=$(basename "$ZIP_FILE")
+          JSON_BASENAME_ADSBX=$(basename "$JSON_FILE_ADSBX")
+          ZIP_BASENAME_MICTRONICS=""
+          if [ -n "$ZIP_FILE_MICTRONICS" ]; then
+            ZIP_BASENAME_MICTRONICS=$(basename "$ZIP_FILE_MICTRONICS")
+          fi
+          
+          if [ -n "$OPTIONAL_MISSING" ]; then
+            echo "WARNING: Optional files missing:$OPTIONAL_MISSING (will continue without them)"
+          fi
          
          echo "date=$DATE" >> "$GITHUB_OUTPUT"
          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
@@ -357,6 +328,10 @@ jobs:
          echo "csv_basename_community=$CSV_BASENAME_COMMUNITY" >> "$GITHUB_OUTPUT"
          echo "zip_file=$ZIP_FILE" >> "$GITHUB_OUTPUT"
          echo "zip_basename=$ZIP_BASENAME" >> "$GITHUB_OUTPUT"
+          echo "json_file_adsbx=$JSON_FILE_ADSBX" >> "$GITHUB_OUTPUT"
+          echo "json_basename_adsbx=$JSON_BASENAME_ADSBX" >> "$GITHUB_OUTPUT"
+          echo "zip_file_mictronics=$ZIP_FILE_MICTRONICS" >> "$GITHUB_OUTPUT"
+          echo "zip_basename_mictronics=$ZIP_BASENAME_MICTRONICS" >> "$GITHUB_OUTPUT"
          echo "name=OpenAirframes snapshot ($DATE)${BRANCH_SUFFIX}" >> "$GITHUB_OUTPUT"
          
          echo "Found files:"
@@ -364,6 +339,8 @@ jobs:
          echo "  ADSB CSV: $CSV_FILE_ADSB"
          echo "  Community CSV: $CSV_FILE_COMMUNITY"
          echo "  ZIP: $ZIP_FILE"
+          echo "  ADSBX JSON: $JSON_FILE_ADSBX"
+          echo "  MICTRONICS ZIP: $ZIP_FILE_MICTRONICS"

      - name: Delete existing release if exists
        run: |
@@ -377,7 +354,7 @@ jobs:
        with:
          tag_name: ${{ steps.meta.outputs.tag }}
          name: ${{ steps.meta.outputs.name }}
-          fail_on_unmatched_files: true
+          fail_on_unmatched_files: false
          body: |
            Automated daily snapshot generated at 06:00 UTC for ${{ steps.meta.outputs.date }}.

@@ -386,10 +363,14 @@ jobs:
            - ${{ steps.meta.outputs.csv_basename_adsb }}
            - ${{ steps.meta.outputs.csv_basename_community }}
            - ${{ steps.meta.outputs.zip_basename }}
+            - ${{ steps.meta.outputs.json_basename_adsbx }}
+            ${{ steps.meta.outputs.zip_basename_mictronics && format('- {0}', steps.meta.outputs.zip_basename_mictronics) || '' }}
          files: |
            ${{ steps.meta.outputs.csv_file_faa }}
            ${{ steps.meta.outputs.csv_file_adsb }}
            ${{ steps.meta.outputs.csv_file_community }}
            ${{ steps.meta.outputs.zip_file }}
+            ${{ steps.meta.outputs.json_file_adsbx }}
+            ${{ steps.meta.outputs.zip_file_mictronics }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}