CalvinBackup/OpenAirframes
1
0
Fork 0
mirror of https://github.com/PlaneQuery/OpenAirframes.git synced 2026-09-14 01:48:55 +02:00
Code Issues Packages Projects Releases Wiki Activity
Files
9ec3c2ca480b6248e13c9b4662c524f5ffd0039e
OpenAirframes/.github/workflows/openairframes-daily-release.yaml
T
Ashley Childress 9ec3c2ca48 fix: publish every per-source registry CSV, not just the joined one 
Each source accumulates by reading its own previous release asset, but only the joined
registry and the FAA CSV were published. Transport Canada could therefore never find a
prior asset and would have rebuilt from a single day on every run, permanently.

- stage the joined CSV and every per-source asset into one flat directory before upload,
  so the artifact has a single root and nothing is lost to the least-common-ancestor rule
- carry the per-source CSVs into the release by glob, so a new source needs no edit here

Generated-by: Claude Opus 5 <noreply@anthropic.com>
2026-09-11 11:06:51 -04:00

511 lines
19 KiB
YAML
Raw Blame History

name: openairframes-daily-release
on:
schedule:
# 06:00 UTC every day - runs on default branch, triggers both
- cron: "0 06 * * *"
workflow_dispatch:
inputs:
date:
description: 'Date to process (YYYY-MM-DD format, default: yesterday)'
required: false
type: string
permissions:
contents: write
actions: write
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: false
jobs:
trigger-releases:
runs-on: ubuntu-latest
if: github.event_name == 'schedule'
steps:
- name: Trigger main branch release
uses: actions/github-script@v7
with:
script: |
await github.rest.actions.createWorkflowDispatch({
owner: context.repo.owner,
repo: context.repo.repo,
workflow_id: 'openairframes-daily-release.yaml',
ref: 'main'
});
- name: Trigger develop branch release
uses: actions/github-script@v7
with:
script: |
await github.rest.actions.createWorkflowDispatch({
owner: context.repo.owner,
repo: context.repo.repo,
workflow_id: 'openairframes-daily-release.yaml',
ref: 'develop'
});
# One thread per registry. Adding a source is one matrix entry plus
# src/create_daily_<source>_release.py; join-registry picks it up by artifact pattern.
build-registry-source:
if: github.event_name != 'schedule'
strategy:
fail-fast: false
matrix:
include:
- source: faa
required: true
- source: tc
required: false
# Transport Canada has never published an asset. Remove this once the first
# run has succeeded: it exists to permit day one, not to paper over an outage.
allow_bootstrap: true
uses: ./.github/workflows/registry-source.yaml
with:
source: ${{ matrix.source }}
date: ${{ inputs.date }}
required: ${{ matrix.required }}
allow_bootstrap: ${{ matrix.allow_bootstrap == true }}
join-registry:
needs: build-registry-source
# No always(): a tolerated source fails its step without failing its leg, so this only
# blocks when a required source could not be built.
if: github.event_name != 'schedule'
runs-on: ubuntu-24.04-arm
timeout-minutes: 20
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@v7
- name: Setup Python
uses: actions/setup-python@v7
with:
python-version: "3.14"
cache: 'pip'
cache-dependency-path: requirements.txt
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Download every source thread
uses: actions/download-artifact@v8
with:
pattern: registry-*
path: artifacts/registry
merge-multiple: true
- name: Join sources into one registry
env:
RUN_DATE: ${{ inputs.date }}
run: |
python src/build_registry.py --input-dir artifacts/registry ${RUN_DATE:+--date "$RUN_DATE"}
- name: Stage release assets
run: |
mkdir -p data/release-out
# Every per-source CSV must ship: each source reads its OWN previous asset to
# accumulate, so one that is never published can never be anything but day one.
cp artifacts/registry/* data/release-out/
cp data/openairframes/openairframes_registry_*.csv data/release-out/
ls -lah data/release-out
- name: Upload registry
uses: actions/upload-artifact@v7
with:
name: union-registry
path: data/release-out
retention-days: 1
if-no-files-found: error
resolve-dates:
runs-on: ubuntu-latest
if: github.event_name != 'schedule'
outputs:
date: ${{ steps.out.outputs.date }}
adsb_date: ${{ steps.out.outputs.adsb_date }}
steps:
- id: out
run: |
if [ -n "${{ inputs.date }}" ]; then
echo "date=${{ inputs.date }}" >> "$GITHUB_OUTPUT"
echo "adsb_date=${{ inputs.date }}" >> "$GITHUB_OUTPUT"
else
echo "date=$(date -u -d 'yesterday' +%Y-%m-%d)" >> "$GITHUB_OUTPUT"
echo "adsb_date=$(date -u -d 'yesterday' +%Y-%m-%d)" >> "$GITHUB_OUTPUT"
fi
adsb-to-aircraft:
needs: resolve-dates
if: github.event_name != 'schedule'
uses: ./.github/workflows/adsb-to-aircraft-for-day.yaml
with:
date: ${{ needs.resolve-dates.outputs.adsb_date }}
concat_with_latest_csv: true
adsb-reduce:
needs: [resolve-dates, adsb-to-aircraft]
if: always() && github.event_name != 'schedule' && needs.adsb-to-aircraft.result == 'failure'
runs-on: ubuntu-24.04-arm
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Setup Python
uses: actions/setup-python@v6
with:
python-version: '3.12'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Download compressed outputs
uses: actions/download-artifact@v4
with:
pattern: adsb-compressed-${{ needs.resolve-dates.outputs.adsb_date }}-part-*
path: data/output/compressed/${{ needs.resolve-dates.outputs.adsb_date }}
merge-multiple: true
- name: Concatenate final outputs
env:
DATE: ${{ needs.resolve-dates.outputs.adsb_date }}
CONCAT_WITH_LATEST_CSV: true
run: |
EXTRA=""
if [ "$CONCAT_WITH_LATEST_CSV" = "true" ]; then
EXTRA="--concat_with_latest_csv"
fi
python -m src.adsb.concat_parquet_to_final --date "$DATE" $EXTRA
ls -lah data/output/ || true
- name: Upload final artifacts
uses: actions/upload-artifact@v4
with:
name: openairframes_adsb-${{ needs.resolve-dates.outputs.adsb_date }}
path: data/output/openairframes_adsb_*
retention-days: 30
if-no-files-found: error
build-community:
runs-on: ubuntu-latest
if: github.event_name != 'schedule'
steps:
- name: Checkout
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Setup Python
uses: actions/setup-python@v6
with:
python-version: "3.14"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install pandas
- name: Run Community release script
run: |
python -m src.contributions.create_daily_community_release
ls -lah data/openairframes
- name: Upload Community artifacts
uses: actions/upload-artifact@v4
with:
name: community-release
path: data/openairframes/openairframes_community_*.csv
retention-days: 1
build-adsbexchange-json:
runs-on: ubuntu-latest
if: github.event_name != 'schedule'
steps:
- name: Checkout
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Setup Python
uses: actions/setup-python@v6
with:
python-version: "3.14"
- name: Run ADS-B Exchange JSON release script
run: |
python -m src.contributions.create_daily_adsbexchange_release ${{ inputs.date && format('--date {0}', inputs.date) || '' }}
ls -lah data/openairframes
- name: Upload ADS-B Exchange JSON artifact
uses: actions/upload-artifact@v4
with:
name: adsbexchange-json
path: data/openairframes/basic-ac-db_*.json.gz
retention-days: 1
build-mictronics-db:
runs-on: ubuntu-latest
if: github.event_name != 'schedule'
steps:
- name: Checkout
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Setup Python
uses: actions/setup-python@v6
with:
python-version: "3.14"
- name: Run Mictronics DB release script
continue-on-error: true
run: |
python -m src.contributions.create_daily_microtonics_release ${{ inputs.date && format('--date {0}', inputs.date) || '' }}
ls -lah data/openairframes
- name: Upload Mictronics DB artifact
uses: actions/upload-artifact@v4
with:
name: mictronics-db
path: data/openairframes/mictronics-db_*.zip
retention-days: 1
if-no-files-found: ignore
create-release:
runs-on: ubuntu-latest
needs: [resolve-dates, join-registry, adsb-to-aircraft, adsb-reduce, build-community, build-adsbexchange-json, build-mictronics-db]
if: github.event_name != 'schedule' && !cancelled()
steps:
- name: Check ADS-B workflow status
if: needs.adsb-to-aircraft.result != 'success' && needs.adsb-reduce.result != 'success'
run: |
echo "WARNING: ADS-B workflow failed (adsb-to-aircraft='${{ needs.adsb-to-aircraft.result }}', adsb-reduce='${{ needs.adsb-reduce.result }}'), will continue without ADS-B artifacts"
- name: Checkout for gh CLI
uses: actions/checkout@v4
with:
sparse-checkout: |
.github
NOTICE
sparse-checkout-cone-mode: false
- name: Download joined registry
uses: actions/download-artifact@v5
with:
name: union-registry
path: artifacts/faa
- name: Download ADS-B artifacts
uses: actions/download-artifact@v5
if: needs.adsb-to-aircraft.result == 'success' || needs.adsb-reduce.result == 'success'
continue-on-error: true
with:
name: openairframes_adsb-${{ needs.resolve-dates.outputs.adsb_date }}
path: artifacts/adsb
- name: Download Community artifacts
uses: actions/download-artifact@v5
with:
name: community-release
path: artifacts/community
- name: Download ADS-B Exchange JSON artifact
uses: actions/download-artifact@v5
with:
name: adsbexchange-json
path: artifacts/adsbexchange
- name: Download Mictronics DB artifact
uses: actions/download-artifact@v5
continue-on-error: true
with:
name: mictronics-db
path: artifacts/mictronics
- name: Debug artifact structure
run: |
echo "=== Full artifacts tree ==="
find artifacts -type f 2>/dev/null || echo "No files found in artifacts"
echo "=== FAA artifacts ==="
find artifacts/faa -type f 2>/dev/null || echo "No files found in artifacts/faa"
echo "=== ADS-B artifacts ==="
find artifacts/adsb -type f 2>/dev/null || echo "No files found in artifacts/adsb"
echo "=== Community artifacts ==="
find artifacts/community -type f 2>/dev/null || echo "No files found in artifacts/community"
echo "=== ADS-B Exchange JSON artifacts ==="
find artifacts/adsbexchange -type f 2>/dev/null || echo "No files found in artifacts/adsbexchange"
echo "=== Mictronics DB artifacts ==="
find artifacts/mictronics -type f 2>/dev/null || echo "No files found in artifacts/mictronics"
- name: Prepare release metadata
id: meta
run: |
DATE=$(date -u +"%Y-%m-%d")
BRANCH_NAME="${GITHUB_REF#refs/heads/}"
BRANCH_SUFFIX=""
if [ "$BRANCH_NAME" = "main" ]; then
BRANCH_SUFFIX="-main"
elif [ "$BRANCH_NAME" = "develop" ]; then
BRANCH_SUFFIX="-develop"
fi
TAG="openairframes-${DATE}${BRANCH_SUFFIX}"
# Find files from artifacts using find (handles nested structures)
CSV_FILE_FAA=$(find artifacts/faa -name "openairframes_faa_*.csv" -type f 2>/dev/null | head -1)
CSV_FILE_REGISTRY=$(find artifacts/faa -name "openairframes_registry_*.csv" -type f 2>/dev/null | head -1)
# Every per-source registry CSV, whatever sources the matrix ran.
SOURCE_CSVS=$(find artifacts/faa -name "openairframes_*.csv" -type f 2>/dev/null \
| grep -v "openairframes_registry_" | grep -v "openairframes_community_" | sort)
echo "Per-source registry CSVs found:"
echo "$SOURCE_CSVS"
# Prefer concatenated file (with date range) over single-day file
CSV_FILE_ADSB=$(find artifacts/adsb -name "openairframes_adsb_*_*.csv.gz" -type f 2>/dev/null | head -1)
if [ -z "$CSV_FILE_ADSB" ]; then
CSV_FILE_ADSB=$(find artifacts/adsb -name "openairframes_adsb_*.csv.gz" -type f 2>/dev/null | head -1)
fi
CSV_FILE_COMMUNITY=$(find artifacts/community -name "openairframes_community_*.csv" -type f 2>/dev/null | head -1)
ZIP_FILE=$(find artifacts/faa -name "ReleasableAircraft_*.zip" -type f 2>/dev/null | head -1)
JSON_FILE_ADSBX=$(find artifacts/adsbexchange -name "basic-ac-db_*.json.gz" -type f 2>/dev/null | head -1)
ZIP_FILE_MICTRONICS=$(find artifacts/mictronics -name "mictronics-db_*.zip" -type f 2>/dev/null | head -1)
# Validate required files exist
MISSING_FILES=""
if [ -z "$CSV_FILE_FAA" ] || [ ! -f "$CSV_FILE_FAA" ]; then
MISSING_FILES="$MISSING_FILES FAA_CSV"
fi
if [ -z "$ZIP_FILE" ] || [ ! -f "$ZIP_FILE" ]; then
MISSING_FILES="$MISSING_FILES FAA_ZIP"
fi
if [ -z "$JSON_FILE_ADSBX" ] || [ ! -f "$JSON_FILE_ADSBX" ]; then
MISSING_FILES="$MISSING_FILES ADSBX_JSON"
fi
if [ -z "$CSV_FILE_REGISTRY" ] || [ ! -f "$CSV_FILE_REGISTRY" ]; then
MISSING_FILES="$MISSING_FILES REGISTRY_CSV"
fi
# NOTICE carries the terms that make each asset redistributable. Shipping data
# without it removes the permission, so it is required rather than optional.
if [ ! -f NOTICE ]; then
MISSING_FILES="$MISSING_FILES NOTICE"
fi
# Optional files - warn but don't fail
OPTIONAL_MISSING=""
if [ -z "$CSV_FILE_ADSB" ] || [ ! -f "$CSV_FILE_ADSB" ]; then
OPTIONAL_MISSING="$OPTIONAL_MISSING ADSB_CSV"
CSV_FILE_ADSB=""
CSV_BASENAME_ADSB=""
fi
if [ -z "$ZIP_FILE_MICTRONICS" ] || [ ! -f "$ZIP_FILE_MICTRONICS" ]; then
OPTIONAL_MISSING="$OPTIONAL_MISSING MICTRONICS_ZIP"
ZIP_FILE_MICTRONICS=""
fi
if [ -n "$MISSING_FILES" ]; then
echo "ERROR: Missing required release files:$MISSING_FILES"
echo "FAA CSV: $CSV_FILE_FAA"
echo "ADSB CSV: $CSV_FILE_ADSB"
echo "ZIP: $ZIP_FILE"
echo "ADSBX JSON: $JSON_FILE_ADSBX"
echo "MICTRONICS ZIP: $ZIP_FILE_MICTRONICS"
exit 1
fi
# Get basenames for display
CSV_BASENAME_FAA=$(basename "$CSV_FILE_FAA")
if [ -n "$CSV_FILE_ADSB" ]; then
CSV_BASENAME_ADSB=$(basename "$CSV_FILE_ADSB")
fi
CSV_BASENAME_COMMUNITY=$(basename "$CSV_FILE_COMMUNITY" 2>/dev/null || echo "")
ZIP_BASENAME=$(basename "$ZIP_FILE")
JSON_BASENAME_ADSBX=$(basename "$JSON_FILE_ADSBX")
ZIP_BASENAME_MICTRONICS=""
if [ -n "$ZIP_FILE_MICTRONICS" ]; then
ZIP_BASENAME_MICTRONICS=$(basename "$ZIP_FILE_MICTRONICS")
fi
if [ -n "$OPTIONAL_MISSING" ]; then
echo "::warning title=Missing optional release assets::$OPTIONAL_MISSING"
fi
echo "date=$DATE" >> "$GITHUB_OUTPUT"
echo "tag=$TAG" >> "$GITHUB_OUTPUT"
echo "csv_file_faa=$CSV_FILE_FAA" >> "$GITHUB_OUTPUT"
echo "csv_file_registry=$CSV_FILE_REGISTRY" >> "$GITHUB_OUTPUT"
{
echo "source_csvs<<SOURCE_CSVS_EOF"
echo "$SOURCE_CSVS"
echo "SOURCE_CSVS_EOF"
} >> "$GITHUB_OUTPUT"
echo "csv_basename_registry=$(basename "$CSV_FILE_REGISTRY")" >> "$GITHUB_OUTPUT"
echo "csv_basename_faa=$CSV_BASENAME_FAA" >> "$GITHUB_OUTPUT"
echo "csv_file_adsb=$CSV_FILE_ADSB" >> "$GITHUB_OUTPUT"
echo "csv_basename_adsb=$CSV_BASENAME_ADSB" >> "$GITHUB_OUTPUT"
echo "csv_file_community=$CSV_FILE_COMMUNITY" >> "$GITHUB_OUTPUT"
echo "csv_basename_community=$CSV_BASENAME_COMMUNITY" >> "$GITHUB_OUTPUT"
echo "zip_file=$ZIP_FILE" >> "$GITHUB_OUTPUT"
echo "zip_basename=$ZIP_BASENAME" >> "$GITHUB_OUTPUT"
echo "json_file_adsbx=$JSON_FILE_ADSBX" >> "$GITHUB_OUTPUT"
echo "json_basename_adsbx=$JSON_BASENAME_ADSBX" >> "$GITHUB_OUTPUT"
echo "zip_file_mictronics=$ZIP_FILE_MICTRONICS" >> "$GITHUB_OUTPUT"
echo "zip_basename_mictronics=$ZIP_BASENAME_MICTRONICS" >> "$GITHUB_OUTPUT"
echo "name=OpenAirframes snapshot ($DATE)${BRANCH_SUFFIX}" >> "$GITHUB_OUTPUT"
echo "Found files:"
echo " FAA CSV: $CSV_FILE_FAA"
echo " ADSB CSV: $CSV_FILE_ADSB"
echo " Community CSV: $CSV_FILE_COMMUNITY"
echo " ZIP: $ZIP_FILE"
echo " ADSBX JSON: $JSON_FILE_ADSBX"
echo " MICTRONICS ZIP: $ZIP_FILE_MICTRONICS"
- name: Delete existing release if exists
run: |
echo "Attempting to delete release: ${{ steps.meta.outputs.tag }}"
# `|| echo` here would swallow a 403 or a partial delete, leaving yesterday's
# asset attached alongside today's; the next run then matches two and rebuilds
# the dataset from a single day.
if gh release view "${{ steps.meta.outputs.tag }}" >/dev/null 2>&1; then
gh release delete "${{ steps.meta.outputs.tag }}" --yes --cleanup-tag
else
echo "No existing release to delete"
fi
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Create GitHub Release and upload assets
uses: softprops/action-gh-release@v2
with:
tag_name: ${{ steps.meta.outputs.tag }}
name: ${{ steps.meta.outputs.name }}
fail_on_unmatched_files: false
body: |
Automated daily snapshot generated at 06:00 UTC for ${{ steps.meta.outputs.date }}.
Assets:
- NOTICE (source terms; required for redistribution)
- ${{ steps.meta.outputs.csv_basename_registry }}
- ${{ steps.meta.outputs.csv_basename_faa }}
${{ steps.meta.outputs.csv_basename_adsb && format('- {0}', steps.meta.outputs.csv_basename_adsb) || '' }}
- ${{ steps.meta.outputs.csv_basename_community }}
- ${{ steps.meta.outputs.zip_basename }}
- ${{ steps.meta.outputs.json_basename_adsbx }}
${{ steps.meta.outputs.zip_basename_mictronics && format('- {0}', steps.meta.outputs.zip_basename_mictronics) || '' }}
files: |
${{ steps.meta.outputs.csv_file_registry }}
${{ steps.meta.outputs.source_csvs }}
NOTICE
${{ steps.meta.outputs.csv_file_adsb }}
${{ steps.meta.outputs.csv_file_community }}
${{ steps.meta.outputs.zip_file }}
${{ steps.meta.outputs.json_file_adsbx }}
${{ steps.meta.outputs.zip_file_mictronics }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
Reference in New Issue View Git Blame Copy Permalink