From 10062956a43c78ea5bc56ca6ab9a3b731476dd66 Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Thu, 10 Sep 2026 20:53:53 -0400 Subject: [PATCH] ci: close the gaps that turned source failures into green runs - key the reusable workflow's concurrency on the source: one shared group made the matrix legs cancel each other, defeating the parallelism - resolve action versions at write time on the jobs this change adds (checkout v7, setup-python v7, upload-artifact v7, download-artifact v8) and add the missing workflow concurrency block and job-level permissions - stage each leg's outputs into one directory so every artifact has the same root; two search paths moved the root to the common ancestor for some legs only, and the FAA CSV and zip then matched nothing downstream - fail a leg that produced no CSV, and split the unknown-source guard out of the tolerated step so a matrix typo goes red - write continue-on-error as an explicit comparison rather than relying on ! coercion - gate the join on success rather than always, and rename its artifact so the download pattern cannot re-ingest it on a re-run - delete the previous release only when it exists, so a 403 stops the run instead of leaving two assets that the next run reads as an ambiguous base Generated-by: Claude Opus 5 --- .../openairframes-daily-release.yaml | 34 ++++++++--- .github/workflows/registry-source.yaml | 56 ++++++++++++++----- 2 files changed, 67 insertions(+), 23 deletions(-) diff --git a/.github/workflows/openairframes-daily-release.yaml b/.github/workflows/openairframes-daily-release.yaml index a8c2f68..bb40281 100644 --- a/.github/workflows/openairframes-daily-release.yaml +++ b/.github/workflows/openairframes-daily-release.yaml @@ -15,6 +15,10 @@ permissions: contents: write actions: write +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + jobs: trigger-releases: runs-on: ubuntu-latest @@ -62,17 +66,23 @@ jobs: join-registry: needs: build-registry-source - if: always() && github.event_name != 'schedule' + # No always(): a tolerated source fails its step without failing its leg, so this only + # blocks when a required source could not be built. + if: github.event_name != 'schedule' runs-on: ubuntu-24.04-arm timeout-minutes: 20 + permissions: + contents: read steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Setup Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: "3.14" + cache: 'pip' + cache-dependency-path: requirements.txt - name: Install dependencies run: | @@ -80,7 +90,7 @@ jobs: pip install -r requirements.txt - name: Download every source thread - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: pattern: registry-* path: artifacts/registry @@ -93,9 +103,9 @@ jobs: python src/build_registry.py --input-dir artifacts/registry ${RUN_DATE:+--date "$RUN_DATE"} - name: Upload registry - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: registry-union + name: union-registry path: | data/openairframes/openairframes_registry_*.csv artifacts/registry/openairframes_faa_*.csv @@ -279,7 +289,7 @@ jobs: - name: Download joined registry uses: actions/download-artifact@v5 with: - name: registry-union + name: union-registry path: artifacts/faa - name: Download ADS-B artifacts @@ -438,7 +448,14 @@ jobs: - name: Delete existing release if exists run: | echo "Attempting to delete release: ${{ steps.meta.outputs.tag }}" - gh release delete "${{ steps.meta.outputs.tag }}" --yes --cleanup-tag || echo "No existing release to delete" + # `|| echo` here would swallow a 403 or a partial delete, leaving yesterday's + # asset attached alongside today's; the next run then matches two and rebuilds + # the dataset from a single day. + if gh release view "${{ steps.meta.outputs.tag }}" >/dev/null 2>&1; then + gh release delete "${{ steps.meta.outputs.tag }}" --yes --cleanup-tag + else + echo "No existing release to delete" + fi env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -452,6 +469,7 @@ jobs: Automated daily snapshot generated at 06:00 UTC for ${{ steps.meta.outputs.date }}. Assets: + - NOTICE (source terms; required for redistribution) - ${{ steps.meta.outputs.csv_basename_registry }} - ${{ steps.meta.outputs.csv_basename_faa }} ${{ steps.meta.outputs.csv_basename_adsb && format('- {0}', steps.meta.outputs.csv_basename_adsb) || '' }} diff --git a/.github/workflows/registry-source.yaml b/.github/workflows/registry-source.yaml index 027f8cf..0fdd3db 100644 --- a/.github/workflows/registry-source.yaml +++ b/.github/workflows/registry-source.yaml @@ -24,47 +24,73 @@ on: type: boolean default: false +# Keyed on the source: without it every matrix leg shares one group and the legs cancel +# each other, which is the opposite of running them in parallel. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.source }} + cancel-in-progress: false + jobs: build: runs-on: ubuntu-24.04-arm timeout-minutes: 30 + permissions: + contents: read steps: - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 + uses: actions/checkout@v7 - name: Setup Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ inputs.python-version }} + cache: 'pip' + cache-dependency-path: requirements.txt - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt - - name: Build ${{ inputs.source }} registry - continue-on-error: ${{ !inputs.required }} + # Deliberately outside the tolerated step below: a typo in the matrix is a config + # error, and must go red even for an optional source. + - name: Check the source has a build script env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SOURCE: ${{ inputs.source }} - RUN_DATE: ${{ inputs.date }} run: | script="src/create_daily_${SOURCE}_release.py" if [ ! -f "$script" ]; then echo "::error title=Unknown registry source::$script does not exist" exit 1 fi - python "$script" ${RUN_DATE:+--date "$RUN_DATE"} - ls -lah data/openairframes + + - name: Build ${{ inputs.source }} registry + continue-on-error: ${{ inputs.required == false }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SOURCE: ${{ inputs.source }} + RUN_DATE: ${{ inputs.date }} + run: | + python "src/create_daily_${SOURCE}_release.py" ${RUN_DATE:+--date "$RUN_DATE"} + # Stage into one directory so every leg's artifact has the same root; a second + # search path would move the root to the common ancestor for some legs only. + shopt -s nullglob + built=(data/openairframes/openairframes_"${SOURCE}"_*.csv) + if [ ${#built[@]} -ne 1 ]; then + echo "::error title=${SOURCE} produced no registry CSV::expected one openairframes_${SOURCE}_*.csv, found ${#built[@]}" + exit 1 + fi + # Stage into one directory so every leg's artifact has the same root; a second + # search path would move the root to the common ancestor for some legs only. + mkdir -p data/registry-out + cp "${built[@]}" data/registry-out/ + cp data/faa_releasable/ReleasableAircraft_*.zip data/registry-out/ 2>/dev/null || true + ls -lah data/registry-out - name: Upload ${{ inputs.source }} registry - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: registry-${{ inputs.source }} - path: | - data/openairframes/openairframes_${{ inputs.source }}_*.csv - data/faa_releasable/ReleasableAircraft_*.zip + path: data/registry-out retention-days: 1 - if-no-files-found: ignore + if-no-files-found: error