OpenAirframes/.github/workflows/historical-adsb.yaml

name: Historical ADS-B Processing

on:
  workflow_dispatch:
    inputs:
      start_date:
        description: 'Start date (YYYY-MM-DD, inclusive)'
        required: true
        type: string
      end_date:
        description: 'End date (YYYY-MM-DD, exclusive)'
        required: true
        type: string
      chunk_days:
        description: 'Days per job chunk (default: 7)'
        required: false
        type: number
        default: 7

jobs:
  generate-matrix:
    runs-on: ubuntu-latest
    outputs:
      chunks: ${{ steps.generate.outputs.chunks }}
      global_start: ${{ inputs.start_date }}
      global_end: ${{ inputs.end_date }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Generate date chunks
        id: generate
        env:
          INPUT_START_DATE: ${{ inputs.start_date }}
          INPUT_END_DATE: ${{ inputs.end_date }}
          INPUT_CHUNK_DAYS: ${{ inputs.chunk_days }}
        run: python src/adsb/historical_generate_matrix.py

  adsb-extract:
    needs: generate-matrix
    runs-on: ubuntu-24.04-arm
    strategy:
      matrix:
        chunk: ${{ fromJson(needs.generate-matrix.outputs.chunks) }}
      max-parallel: 3
      fail-fast: false
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt

      - name: Free disk space
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf /usr/local/share/boost
          df -h

      - name: Download and extract ADS-B data
        env:
          START_DATE: ${{ matrix.chunk.start_date }}
          END_DATE: ${{ matrix.chunk.end_date }}
        run: |
          python -m src.adsb.download_and_list_icaos --start-date "$START_DATE" --end-date "$END_DATE"
          ls -lah data/output/

      - name: Create tar of extracted data
        run: |
          cd data/output
          echo "=== Disk space before tar ==="
          df -h .
          echo "=== Files to tar ==="
          ls -lah *-planes-readsb-prod-0.tar_0 icao_manifest_*.txt 2>/dev/null || echo "No files found"
          
          # Create tar with explicit error checking
          if ls *-planes-readsb-prod-0.tar_0 1>/dev/null 2>&1; then
            tar -cvf extracted_data.tar *-planes-readsb-prod-0.tar_0 icao_manifest_*.txt
            echo "=== Tar file created ==="
            ls -lah extracted_data.tar
            # Verify tar integrity
            tar -tf extracted_data.tar > /dev/null && echo "Tar integrity check passed" || { echo "Tar integrity check FAILED"; exit 1; }
          else
            echo "ERROR: No extracted directories found, cannot create tar"
            exit 1
          fi

      - name: Upload extracted data
        uses: actions/upload-artifact@v4
        with:
          name: adsb-extracted-${{ matrix.chunk.start_date }}-${{ matrix.chunk.end_date }}
          path: data/output/extracted_data.tar
          retention-days: 1
          compression-level: 0
          if-no-files-found: warn

  adsb-map:
    needs: [generate-matrix, adsb-extract]
    runs-on: ubuntu-24.04-arm
    strategy:
      fail-fast: true
      matrix:
        chunk: ${{ fromJson(needs.generate-matrix.outputs.chunks) }}
        icao_chunk: [0, 1, 2, 3]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt

      - name: Free disk space
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf /usr/local/share/boost
          df -h

      - name: Download extracted data
        uses: actions/download-artifact@v4
        with:
          name: adsb-extracted-${{ matrix.chunk.start_date }}-${{ matrix.chunk.end_date }}
          path: data/output/
        continue-on-error: true

      - name: Extract tar
        id: extract
        run: |
          cd data/output
          if [ -f extracted_data.tar ]; then
            echo "=== Tar file info ==="
            ls -lah extracted_data.tar
            echo "=== Verifying tar integrity ==="
            tar -tf extracted_data.tar > /dev/null || { echo "ERROR: Tar file is corrupted"; exit 1; }
            echo "=== Extracting ==="
            tar -xvf extracted_data.tar
            rm extracted_data.tar
            echo "has_data=true" >> "$GITHUB_OUTPUT"
            echo "=== Contents of data/output ==="
            ls -lah
          else
            echo "No extracted_data.tar found"
            echo "has_data=false" >> "$GITHUB_OUTPUT"
          fi

      - name: Process ICAO chunk
        if: steps.extract.outputs.has_data == 'true'
        env:
          START_DATE: ${{ matrix.chunk.start_date }}
          END_DATE: ${{ matrix.chunk.end_date }}
        run: |
          python -m src.adsb.process_icao_chunk --chunk-id ${{ matrix.icao_chunk }} --total-chunks 4 --start-date "$START_DATE" --end-date "$END_DATE"
          ls -lah data/output/adsb_chunks/ || echo "No chunks created"

      - name: Upload chunk artifacts
        if: steps.extract.outputs.has_data == 'true'
        uses: actions/upload-artifact@v4
        with:
          name: adsb-map-${{ matrix.chunk.start_date }}-${{ matrix.chunk.end_date }}-chunk-${{ matrix.icao_chunk }}
          path: data/output/adsb_chunks/
          retention-days: 1
          if-no-files-found: ignore

  adsb-reduce:
    needs: [generate-matrix, adsb-map]
    runs-on: ubuntu-24.04-arm
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt

      - name: Download all chunk artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: adsb-map-*
          path: data/output/adsb_chunks/
          merge-multiple: true

      - name: Debug downloaded files
        run: |
          echo "=== Disk space before processing ==="
          df -h
          echo "=== Listing data/output/adsb_chunks/ ==="
          find data/output/adsb_chunks/ -type f 2>/dev/null | wc -l
          echo "=== Total parquet size ==="
          du -sh data/output/adsb_chunks/ || echo "No chunks dir"

      - name: Combine chunks to CSV
        env:
          START_DATE: ${{ needs.generate-matrix.outputs.global_start }}
          END_DATE: ${{ needs.generate-matrix.outputs.global_end }}
        run: |
          python -m src.adsb.combine_chunks_to_csv --chunks-dir data/output/adsb_chunks --start-date "$START_DATE" --end-date "$END_DATE" --skip-base --stream
          ls -lah data/planequery_aircraft/

      - name: Upload final artifact
        uses: actions/upload-artifact@v4
        with:
          name: planequery_aircraft_adsb-${{ needs.generate-matrix.outputs.global_start }}-${{ needs.generate-matrix.outputs.global_end }}
          path: data/planequery_aircraft/*.csv
          retention-days: 30