mirror of
https://github.com/PlaneQuery/OpenAirframes.git
synced 2026-04-24 03:56:24 +02:00
copilot-swe-agent[bot]
7ea815732c
Co-authored-by: ggman12 <17393221+ggman12@users.noreply.github.com>
300 lines
10 KiB
YAML
300 lines
10 KiB
YAML
name: Historical ADS-B Processing
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
start_date:
|
|
description: 'Start date (YYYY-MM-DD, inclusive)'
|
|
required: true
|
|
type: string
|
|
end_date:
|
|
description: 'End date (YYYY-MM-DD, exclusive)'
|
|
required: true
|
|
type: string
|
|
chunk_days:
|
|
description: 'Days per job chunk (default: 7)'
|
|
required: false
|
|
type: number
|
|
default: 7
|
|
|
|
jobs:
|
|
generate-matrix:
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
chunks: ${{ steps.generate.outputs.chunks }}
|
|
global_start: ${{ inputs.start_date }}
|
|
global_end: ${{ inputs.end_date }}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.12'
|
|
|
|
- name: Generate date chunks
|
|
id: generate
|
|
env:
|
|
INPUT_START_DATE: ${{ inputs.start_date }}
|
|
INPUT_END_DATE: ${{ inputs.end_date }}
|
|
INPUT_CHUNK_DAYS: ${{ inputs.chunk_days }}
|
|
run: python src/adsb/historical_generate_matrix.py
|
|
|
|
adsb-extract:
|
|
needs: generate-matrix
|
|
runs-on: ubuntu-24.04-arm
|
|
strategy:
|
|
matrix:
|
|
chunk: ${{ fromJson(needs.generate-matrix.outputs.chunks) }}
|
|
max-parallel: 3
|
|
fail-fast: true
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.12'
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install -r requirements.txt
|
|
|
|
- name: Free disk space
|
|
run: |
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo rm -rf /opt/ghc
|
|
sudo rm -rf /usr/local/share/boost
|
|
df -h
|
|
|
|
- name: Download and extract ADS-B data
|
|
env:
|
|
START_DATE: ${{ matrix.chunk.start_date }}
|
|
END_DATE: ${{ matrix.chunk.end_date }}
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: |
|
|
python -m src.adsb.download_and_list_icaos --start-date "$START_DATE" --end-date "$END_DATE"
|
|
ls -lah data/output/
|
|
|
|
- name: Create tar of extracted data and split into chunks
|
|
run: |
|
|
cd data/output
|
|
echo "=== Disk space before tar ==="
|
|
df -h .
|
|
echo "=== Files to tar ==="
|
|
ls -lah *-planes-readsb-prod-0.tar_0 icao_manifest_*.txt 2>/dev/null || echo "No files found"
|
|
|
|
# Create tar with explicit error checking
|
|
if ls *-planes-readsb-prod-0.tar_0 1>/dev/null 2>&1; then
|
|
tar -cvf extracted_data.tar *-planes-readsb-prod-0.tar_0 icao_manifest_*.txt
|
|
echo "=== Tar file created ==="
|
|
ls -lah extracted_data.tar
|
|
# Verify tar integrity
|
|
tar -tf extracted_data.tar > /dev/null && echo "Tar integrity check passed" || { echo "Tar integrity check FAILED"; exit 1; }
|
|
|
|
# Record tar size and checksum for verification after reassembly
|
|
echo "=== Recording tar metadata ==="
|
|
ORIGINAL_SIZE=$(stat --format=%s extracted_data.tar)
|
|
ORIGINAL_SHA=$(sha256sum extracted_data.tar | awk '{print $1}')
|
|
echo "Size: $ORIGINAL_SIZE"
|
|
echo "SHA256: $ORIGINAL_SHA"
|
|
|
|
# Split into 500MB chunks to avoid artifact upload issues
|
|
echo "=== Splitting tar into 500MB chunks ==="
|
|
mkdir -p tar_chunks
|
|
split -b 500M extracted_data.tar tar_chunks/extracted_data.tar.part_
|
|
rm extracted_data.tar
|
|
|
|
# Write metadata file (plain text so artifact upload won't skip it)
|
|
echo "$ORIGINAL_SHA extracted_data.tar" > tar_chunks/checksum.txt
|
|
echo "$ORIGINAL_SIZE" >> tar_chunks/checksum.txt
|
|
|
|
echo "=== Chunks created ==="
|
|
ls -lah tar_chunks/
|
|
echo "=== Checksum file ==="
|
|
cat tar_chunks/checksum.txt
|
|
else
|
|
echo "ERROR: No extracted directories found, cannot create tar"
|
|
exit 1
|
|
fi
|
|
|
|
- name: Upload extracted data chunks
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: adsb-extracted-${{ matrix.chunk.start_date }}-${{ matrix.chunk.end_date }}
|
|
path: data/output/tar_chunks/
|
|
retention-days: 1
|
|
compression-level: 0
|
|
if-no-files-found: warn
|
|
|
|
adsb-map:
|
|
needs: [generate-matrix, adsb-extract]
|
|
runs-on: ubuntu-24.04-arm
|
|
strategy:
|
|
fail-fast: true
|
|
matrix:
|
|
chunk: ${{ fromJson(needs.generate-matrix.outputs.chunks) }}
|
|
icao_chunk: [0, 1, 2, 3]
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.12'
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install -r requirements.txt
|
|
|
|
- name: Free disk space
|
|
run: |
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo rm -rf /opt/ghc
|
|
sudo rm -rf /usr/local/share/boost
|
|
df -h
|
|
|
|
- name: Download extracted data
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: adsb-extracted-${{ matrix.chunk.start_date }}-${{ matrix.chunk.end_date }}
|
|
path: data/output/tar_chunks/
|
|
|
|
- name: Reassemble and extract tar
|
|
id: extract
|
|
run: |
|
|
cd data/output
|
|
if [ -d tar_chunks ] && ls tar_chunks/extracted_data.tar.part_* 1>/dev/null 2>&1; then
|
|
echo "=== Chunk files info ==="
|
|
ls -lah tar_chunks/
|
|
|
|
cd tar_chunks
|
|
|
|
# Reassemble tar with explicit sorting
|
|
echo "=== Reassembling tar file ==="
|
|
ls -1 extracted_data.tar.part_?? | sort | while read part; do
|
|
echo "Appending $part..."
|
|
cat "$part" >> ../extracted_data.tar
|
|
done
|
|
cd ..
|
|
|
|
echo "=== Reassembled tar file info ==="
|
|
ls -lah extracted_data.tar
|
|
|
|
# Verify integrity
|
|
echo "=== Verifying reassembled tar ==="
|
|
if [ -f tar_chunks/checksum.txt ]; then
|
|
EXPECTED_SHA=$(head -1 tar_chunks/checksum.txt | awk '{print $1}')
|
|
EXPECTED_SIZE=$(sed -n '2p' tar_chunks/checksum.txt)
|
|
ACTUAL_SHA=$(sha256sum extracted_data.tar | awk '{print $1}')
|
|
ACTUAL_SIZE=$(stat --format=%s extracted_data.tar)
|
|
echo "Expected: SHA=$EXPECTED_SHA Size=$EXPECTED_SIZE"
|
|
echo "Actual: SHA=$ACTUAL_SHA Size=$ACTUAL_SIZE"
|
|
if [ "$EXPECTED_SHA" != "$ACTUAL_SHA" ] || [ "$EXPECTED_SIZE" != "$ACTUAL_SIZE" ]; then
|
|
echo "ERROR: Reassembled tar does not match original - data corrupted during transfer"
|
|
exit 1
|
|
fi
|
|
echo "Checksum and size verified"
|
|
else
|
|
echo "WARNING: No checksum file found, falling back to tar integrity check"
|
|
tar -tf extracted_data.tar > /dev/null || { echo "ERROR: Tar file is corrupted"; exit 1; }
|
|
echo "Tar integrity check passed"
|
|
fi
|
|
|
|
rm -rf tar_chunks
|
|
|
|
echo "=== Extracting ==="
|
|
tar -xf extracted_data.tar
|
|
rm extracted_data.tar
|
|
echo "has_data=true" >> "$GITHUB_OUTPUT"
|
|
echo "=== Contents of data/output ==="
|
|
ls -lah
|
|
else
|
|
echo "No tar chunks found"
|
|
echo "has_data=false" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
- name: Process ICAO chunk
|
|
if: steps.extract.outputs.has_data == 'true'
|
|
env:
|
|
START_DATE: ${{ matrix.chunk.start_date }}
|
|
END_DATE: ${{ matrix.chunk.end_date }}
|
|
run: |
|
|
python -m src.adsb.process_icao_chunk --chunk-id ${{ matrix.icao_chunk }} --total-chunks 4 --start-date "$START_DATE" --end-date "$END_DATE"
|
|
ls -lah data/output/adsb_chunks/ || echo "No chunks created"
|
|
|
|
- name: Upload chunk artifacts
|
|
if: steps.extract.outputs.has_data == 'true'
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: adsb-map-${{ matrix.chunk.start_date }}-${{ matrix.chunk.end_date }}-chunk-${{ matrix.icao_chunk }}
|
|
path: data/output/adsb_chunks/
|
|
retention-days: 1
|
|
if-no-files-found: ignore
|
|
|
|
adsb-reduce:
|
|
needs: [generate-matrix, adsb-map]
|
|
runs-on: ubuntu-24.04-arm
|
|
timeout-minutes: 120
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.12'
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install -r requirements.txt
|
|
|
|
- name: Download all chunk artifacts
|
|
id: download
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
pattern: adsb-map-*
|
|
path: data/output/adsb_chunks/
|
|
merge-multiple: true
|
|
timeout-minutes: 60
|
|
continue-on-error: true
|
|
|
|
- name: Retry artifact download on failure
|
|
if: steps.download.outcome == 'failure'
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
pattern: adsb-map-*
|
|
path: data/output/adsb_chunks/
|
|
merge-multiple: true
|
|
timeout-minutes: 60
|
|
|
|
- name: Debug downloaded files
|
|
run: |
|
|
echo "=== Disk space before processing ==="
|
|
df -h
|
|
echo "=== Listing data/output/adsb_chunks/ ==="
|
|
find data/output/adsb_chunks/ -type f 2>/dev/null | wc -l
|
|
echo "=== Total parquet size ==="
|
|
du -sh data/output/adsb_chunks/ || echo "No chunks dir"
|
|
|
|
- name: Combine chunks to CSV
|
|
env:
|
|
START_DATE: ${{ needs.generate-matrix.outputs.global_start }}
|
|
END_DATE: ${{ needs.generate-matrix.outputs.global_end }}
|
|
run: |
|
|
python -m src.adsb.combine_chunks_to_csv --chunks-dir data/output/adsb_chunks --start-date "$START_DATE" --end-date "$END_DATE" --skip-base --stream
|
|
ls -lah data/openairframes/
|
|
|
|
- name: Upload final artifact
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: openairframes_adsb-${{ needs.generate-matrix.outputs.global_start }}-${{ needs.generate-matrix.outputs.global_end }}
|
|
path: data/openairframes/*.csv.gz
|
|
retention-days: 30
|