diff --git a/.github/workflows/adsb-to-aircraft-for-day.yaml b/.github/workflows/adsb-to-aircraft-for-day.yaml index d10676e..a41494d 100644 --- a/.github/workflows/adsb-to-aircraft-for-day.yaml +++ b/.github/workflows/adsb-to-aircraft-for-day.yaml @@ -49,11 +49,38 @@ jobs: python -m src.adsb.download_and_list_icaos --date "$DATE" ls -lah data/output/adsb_archives/"$DATE" || true - - name: Upload archives + - name: Upload archive part 0 uses: actions/upload-artifact@v4 with: - name: adsb-archives-${{ inputs.date }} - path: data/output/adsb_archives/${{ inputs.date }} + name: adsb-archive-${{ inputs.date }}-part-0 + path: data/output/adsb_archives/${{ inputs.date }}/${{ inputs.date }}_part_0.tar.gz + retention-days: 1 + compression-level: 0 + if-no-files-found: error + + - name: Upload archive part 1 + uses: actions/upload-artifact@v4 + with: + name: adsb-archive-${{ inputs.date }}-part-1 + path: data/output/adsb_archives/${{ inputs.date }}/${{ inputs.date }}_part_1.tar.gz + retention-days: 1 + compression-level: 0 + if-no-files-found: error + + - name: Upload archive part 2 + uses: actions/upload-artifact@v4 + with: + name: adsb-archive-${{ inputs.date }}-part-2 + path: data/output/adsb_archives/${{ inputs.date }}/${{ inputs.date }}_part_2.tar.gz + retention-days: 1 + compression-level: 0 + if-no-files-found: error + + - name: Upload archive part 3 + uses: actions/upload-artifact@v4 + with: + name: adsb-archive-${{ inputs.date }}-part-3 + path: data/output/adsb_archives/${{ inputs.date }}/${{ inputs.date }}_part_3.tar.gz retention-days: 1 compression-level: 0 if-no-files-found: error @@ -79,12 +106,22 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt - - name: Download archives + - name: Download archive part uses: actions/download-artifact@v4 with: - name: adsb-archives-${{ inputs.date }} + name: adsb-archive-${{ inputs.date }}-part-${{ matrix.part_id }} path: data/output/adsb_archives/${{ inputs.date }} + - name: Verify archive + run: | + FILE="data/output/adsb_archives/${{ inputs.date }}/${{ inputs.date }}_part_${{ matrix.part_id }}.tar.gz" + ls -lah data/output/adsb_archives/${{ inputs.date }}/ + if [ ! -f "$FILE" ]; then + echo "::error::Archive not found: $FILE" + exit 1 + fi + echo "Verified: $(du -h "$FILE")" + - name: Process part env: DATE: ${{ inputs.date }} diff --git a/src/adsb/process_icao_chunk.py b/src/adsb/process_icao_chunk.py index 0687652..7d01feb 100644 --- a/src/adsb/process_icao_chunk.py +++ b/src/adsb/process_icao_chunk.py @@ -123,7 +123,16 @@ def main(): print(f"Processing part {args.part_id} for {args.date}") # Get specific archive file for this part - archive_path = os.path.join(OUTPUT_DIR, "adsb_archives", args.date, f"{args.date}_part_{args.part_id}.tar.gz") + archive_dir = os.path.join(OUTPUT_DIR, "adsb_archives", args.date) + archive_path = os.path.join(archive_dir, f"{args.date}_part_{args.part_id}.tar.gz") + + if not os.path.isfile(archive_path): + print(f"ERROR: Archive not found: {archive_path}") + if os.path.isdir(archive_dir): + print(f"Files in {archive_dir}: {os.listdir(archive_dir)}") + else: + print(f"Directory does not exist: {archive_dir}") + sys.exit(1) # Extract and collect trace files trace_map = build_trace_file_map(archive_path)