FIX: trigger for planequery-aircraft daily release workflow. Update contributions issue template.

.github/workflows/historical-adsb.yaml

@@ -0,0 +1,209 @@
+name: Historical ADS-B Processing
+
+on:
+  workflow_dispatch:
+    inputs:
+      start_date:
+        description: 'Start date (YYYY-MM-DD, inclusive)'
+        required: true
+        type: string
+      end_date:
+        description: 'End date (YYYY-MM-DD, exclusive)'
+        required: true
+        type: string
+      chunk_days:
+        description: 'Days per job chunk (default: 7)'
+        required: false
+        type: number
+        default: 7
+
+jobs:
+  generate-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      chunks: ${{ steps.generate.outputs.chunks }}
+      global_start: ${{ inputs.start_date }}
+      global_end: ${{ inputs.end_date }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Generate date chunks
+        id: generate
+        env:
+          INPUT_START_DATE: ${{ inputs.start_date }}
+          INPUT_END_DATE: ${{ inputs.end_date }}
+          INPUT_CHUNK_DAYS: ${{ inputs.chunk_days }}
+        run: python src/adsb/historical_generate_matrix.py
+
+  adsb-extract:
+    needs: generate-matrix
+    runs-on: ubuntu-24.04-arm
+    strategy:
+      matrix:
+        chunk: ${{ fromJson(needs.generate-matrix.outputs.chunks) }}
+      max-parallel: 3
+      fail-fast: false
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Free disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          df -h
+
+      - name: Download and extract ADS-B data
+        env:
+          START_DATE: ${{ matrix.chunk.start_date }}
+          END_DATE: ${{ matrix.chunk.end_date }}
+        run: |
+          python -m src.adsb.download_and_list_icaos --start-date "$START_DATE" --end-date "$END_DATE"
+          ls -lah data/output/
+
+      - name: Create tar of extracted data
+        run: |
+          cd data/output
+          tar -cf extracted_data.tar *-planes-readsb-prod-0.tar_0 icao_manifest_*.txt 2>/dev/null || echo "Some files may not exist"
+          ls -lah extracted_data.tar || echo "No tar created"
+
+      - name: Upload extracted data
+        uses: actions/upload-artifact@v4
+        with:
+          name: adsb-extracted-${{ matrix.chunk.start_date }}-${{ matrix.chunk.end_date }}
+          path: data/output/extracted_data.tar
+          retention-days: 1
+          compression-level: 0
+          if-no-files-found: warn
+
+  adsb-map:
+    needs: [generate-matrix, adsb-extract]
+    runs-on: ubuntu-24.04-arm
+    strategy:
+      fail-fast: false
+      matrix:
+        chunk: ${{ fromJson(needs.generate-matrix.outputs.chunks) }}
+        icao_chunk: [0, 1, 2, 3]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Free disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          df -h
+
+      - name: Download extracted data
+        uses: actions/download-artifact@v4
+        with:
+          name: adsb-extracted-${{ matrix.chunk.start_date }}-${{ matrix.chunk.end_date }}
+          path: data/output/
+        continue-on-error: true
+
+      - name: Extract tar
+        id: extract
+        run: |
+          cd data/output
+          if [ -f extracted_data.tar ]; then
+            tar -xf extracted_data.tar
+            rm extracted_data.tar
+            echo "has_data=true" >> "$GITHUB_OUTPUT"
+            echo "=== Contents of data/output ==="
+            ls -lah
+          else
+            echo "No extracted_data.tar found"
+            echo "has_data=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Process ICAO chunk
+        if: steps.extract.outputs.has_data == 'true'
+        env:
+          START_DATE: ${{ matrix.chunk.start_date }}
+          END_DATE: ${{ matrix.chunk.end_date }}
+        run: |
+          python -m src.adsb.process_icao_chunk --chunk-id ${{ matrix.icao_chunk }} --total-chunks 4 --start-date "$START_DATE" --end-date "$END_DATE"
+          ls -lah data/output/adsb_chunks/ || echo "No chunks created"
+
+      - name: Upload chunk artifacts
+        if: steps.extract.outputs.has_data == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: adsb-map-${{ matrix.chunk.start_date }}-${{ matrix.chunk.end_date }}-chunk-${{ matrix.icao_chunk }}
+          path: data/output/adsb_chunks/
+          retention-days: 1
+          if-no-files-found: ignore
+
+  adsb-reduce:
+    needs: [generate-matrix, adsb-map]
+    runs-on: ubuntu-24.04-arm
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Download all chunk artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: adsb-map-*
+          path: data/output/adsb_chunks/
+          merge-multiple: true
+
+      - name: Debug downloaded files
+        run: |
+          echo "=== Listing data/output/adsb_chunks/ ==="
+          find data/output/adsb_chunks/ -type f 2>/dev/null | head -50 || echo "No files found"
+          echo "=== Looking for parquet files ==="
+          find . -name "*.parquet" 2>/dev/null | head -20 || echo "No parquet files found"
+
+      - name: Combine chunks to CSV
+        env:
+          START_DATE: ${{ needs.generate-matrix.outputs.global_start }}
+          END_DATE: ${{ needs.generate-matrix.outputs.global_end }}
+        run: |
+          python -m src.adsb.combine_chunks_to_csv --chunks-dir data/output/adsb_chunks --start-date "$START_DATE" --end-date "$END_DATE" --skip-base
+          ls -lah data/planequery_aircraft/
+
+      - name: Upload final artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: planequery_aircraft_adsb-${{ needs.generate-matrix.outputs.global_start }}-${{ needs.generate-matrix.outputs.global_end }}
+          path: data/planequery_aircraft/*.csv
+          retention-days: 30

.github/workflows/planequery-aircraft-daily-release.yaml

@@ -8,6 +8,7 @@ on:
 
 permissions:
   contents: write
+  actions: write
 
 jobs:
   trigger-releases:

.github/workflows/process-historical-faa.yaml

@@ -1,171 +0,0 @@
-name: Process Historical FAA Data
-
-on:
-  workflow_dispatch:  # Manual trigger
-
-jobs:
-  generate-matrix:
-    runs-on: ubuntu-latest
-    outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
-    steps:
-      - name: Generate date ranges
-        id: set-matrix
-        run: |
-          python3 << 'EOF'
-          import json
-          from datetime import datetime, timedelta
-          
-          start = datetime(2023, 8, 16)
-          end = datetime(2026, 1, 1)
-          
-          ranges = []
-          current = start
-          
-          # Process in 4-day chunks
-          while current < end:
-            chunk_end = current + timedelta(days=4)
-            # Don't go past the end date
-            if chunk_end > end:
-              chunk_end = end
-            
-            ranges.append({
-              "since": current.strftime("%Y-%m-%d"),
-              "until": chunk_end.strftime("%Y-%m-%d")
-            })
-            
-            current = chunk_end
-          
-          print(f"::set-output name=matrix::{json.dumps(ranges)}")
-          EOF
-
-  clone-faa-repo:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Cache FAA repository
-        id: cache-faa-repo
-        uses: actions/cache@v4
-        with:
-          path: data/scrape-faa-releasable-aircraft
-          key: faa-repo-v1
-          
-      - name: Clone FAA repository
-        if: steps.cache-faa-repo.outputs.cache-hit != 'true'
-        run: |
-          mkdir -p data
-          git clone https://github.com/simonw/scrape-faa-releasable-aircraft data/scrape-faa-releasable-aircraft
-          echo "Repository cloned successfully"
-
-  process-chunk:
-    needs: [generate-matrix, clone-faa-repo]
-    runs-on: ubuntu-latest
-    strategy:
-      max-parallel: 5  # Process 5 chunks at a time
-      matrix:
-        range: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.12'
-      
-      - name: Restore FAA repository cache
-        uses: actions/cache/restore@v4
-        with:
-          path: data/scrape-faa-releasable-aircraft
-          key: faa-repo-v1
-          fail-on-cache-miss: true
-      
-      - name: Install dependencies
-        run: |
-          pip install -r requirements.txt
-      
-      - name: Process chunk ${{ matrix.range.since }} to ${{ matrix.range.until }}
-        run: |
-          python src/get_historical_faa.py "${{ matrix.range.since }}" "${{ matrix.range.until }}"
-      
-      - name: Upload CSV artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: csv-${{ matrix.range.since }}-to-${{ matrix.range.until }}
-          path: data/faa_releasable_historical/*.csv
-          retention-days: 1
-
-  create-release:
-    needs: process-chunk
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - name: Download all artifacts
-        uses: actions/download-artifact@v4
-        with:
-          path: artifacts
-      
-      - name: Prepare release files
-        run: |
-          mkdir -p release-files
-          find artifacts -name "*.csv" -exec cp {} release-files/ \;
-          ls -lh release-files/
-      
-      - name: Create Release
-        uses: softprops/action-gh-release@v1
-        with:
-          tag_name: historical-faa-${{ github.run_number }}
-          name: Historical FAA Data Release ${{ github.run_number }}
-          body: |
-            Automated release of historical FAA aircraft data
-            Processing period: 2023-08-16 to 2026-01-01
-            Generated: ${{ github.event.repository.updated_at }}
-          files: release-files/*.csv
-          draft: false
-          prerelease: false
-
-  concatenate-and-release:
-    needs: process-chunk
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.12'
-      
-      - name: Install dependencies
-        run: |
-          pip install -r requirements.txt
-      
-      - name: Download all artifacts
-        uses: actions/download-artifact@v4
-        with:
-          path: artifacts
-      
-      - name: Prepare CSVs for concatenation
-        run: |
-          mkdir -p data/faa_releasable_historical
-          find artifacts -name "*.csv" -exec cp {} data/faa_releasable_historical/ \;
-          ls -lh data/faa_releasable_historical/
-      
-      - name: Concatenate all CSVs
-        run: |
-          python scripts/concat_csvs.py
-      
-      - name: Create Combined Release
-        uses: softprops/action-gh-release@v1
-        with:
-          tag_name: historical-faa-combined-${{ github.run_number }}
-          name: Historical FAA Data Combined Release ${{ github.run_number }}
-          body: |
-            Combined historical FAA aircraft data (all chunks concatenated)
-            Processing period: 2023-08-16 to 2026-01-01
-            Generated: ${{ github.event.repository.updated_at }}
-          files: data/planequery_aircraft/*.csv
-          draft: false
-          prerelease: false