name: openairframes-daily-release

on:
  schedule:
    # 06:00 UTC every day - runs on default branch, triggers both
    - cron: "0 06 * * *"
  workflow_dispatch:
    inputs:
      date:
        description: 'Date to process (YYYY-MM-DD format, default: yesterday)'
        required: false
        type: string
      bootstrap_source:
        description: 'Onboarding only: the one source id permitted to rebuild from a single day'
        required: false
        type: string

permissions:
  contents: write
  actions: write

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: false

jobs:
  trigger-releases:
    runs-on: ubuntu-latest
    if: github.event_name == 'schedule'
    steps:
      - name: Trigger main branch release
        uses: actions/github-script@v7
        with:
          script: |
            await github.rest.actions.createWorkflowDispatch({
              owner: context.repo.owner,
              repo: context.repo.repo,
              workflow_id: 'openairframes-daily-release.yaml',
              ref: 'main'
            });
      
      - name: Trigger develop branch release
        uses: actions/github-script@v7
        with:
          script: |
            await github.rest.actions.createWorkflowDispatch({
              owner: context.repo.owner,
              repo: context.repo.repo,
              workflow_id: 'openairframes-daily-release.yaml',
              ref: 'develop'
            });

  # One thread per registry. Adding a source is one matrix entry plus
  # src/create_daily_<source>_release.py; join-registry picks it up by artifact pattern.
  build-registry-source:
    if: github.event_name != 'schedule'
    strategy:
      fail-fast: false
      matrix:
        include:
          - source: faa
            required: true
          - source: tc
            required: false
    uses: ./.github/workflows/registry-source.yaml
    with:
      source: ${{ matrix.source }}
      date: ${{ inputs.date }}
      required: ${{ matrix.required }}
      # Standing permission would also fire on an outage. A source bootstraps only when a
      # human dispatches the run naming it.
      allow_bootstrap: ${{ inputs.bootstrap_source == matrix.source }}

  join-registry:
    needs: build-registry-source
    # No always(): a tolerated source fails its step without failing its leg, so this only
    # blocks when a required source could not be built.
    if: github.event_name != 'schedule'
    runs-on: ubuntu-24.04-arm
    timeout-minutes: 20
    permissions:
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@v7

      - name: Setup Python
        uses: actions/setup-python@v7
        with:
          python-version: "3.14"
          cache: 'pip'
          cache-dependency-path: requirements.txt

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt

      - name: Download every source thread
        uses: actions/download-artifact@v8
        with:
          pattern: registry-*
          path: artifacts/registry
          merge-multiple: true

      - name: Join sources into one registry
        env:
          RUN_DATE: ${{ inputs.date }}
        run: |
          python src/build_registry.py --input-dir artifacts/registry ${RUN_DATE:+--date "$RUN_DATE"}

      - name: Stage release assets
        run: |
          mkdir -p data/release-out
          # Every per-source CSV must ship: each source reads its OWN previous asset to
          # accumulate, so one that is never published can never be anything but day one.
          cp artifacts/registry/* data/release-out/
          cp data/openairframes/openairframes_registry_*.csv data/release-out/
          ls -lah data/release-out

      - name: Upload registry
        uses: actions/upload-artifact@v7
        with:
          name: union-registry
          path: data/release-out
          retention-days: 1
          if-no-files-found: error

  resolve-dates:
    runs-on: ubuntu-latest
    if: github.event_name != 'schedule'
    outputs:
      date: ${{ steps.out.outputs.date }}
      adsb_date: ${{ steps.out.outputs.adsb_date }}
    steps:
      - id: out
        run: |
          if [ -n "${{ inputs.date }}" ]; then
            echo "date=${{ inputs.date }}" >> "$GITHUB_OUTPUT"
            echo "adsb_date=${{ inputs.date }}" >> "$GITHUB_OUTPUT"
          else
            echo "date=$(date -u -d 'yesterday' +%Y-%m-%d)" >> "$GITHUB_OUTPUT"
            echo "adsb_date=$(date -u -d 'yesterday' +%Y-%m-%d)" >> "$GITHUB_OUTPUT"
          fi

  adsb-to-aircraft:
    needs: resolve-dates
    if: github.event_name != 'schedule'
    uses: ./.github/workflows/adsb-to-aircraft-for-day.yaml
    with:
      date: ${{ needs.resolve-dates.outputs.adsb_date }}
      concat_with_latest_csv: true

  adsb-reduce:
    needs: [resolve-dates, adsb-to-aircraft]
    if: always() && github.event_name != 'schedule' && needs.adsb-to-aircraft.result == 'failure'
    runs-on: ubuntu-24.04-arm
    steps:
      - name: Checkout
        uses: actions/checkout@v6

      - name: Setup Python
        uses: actions/setup-python@v6
        with:
          python-version: '3.12'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt

      - name: Download compressed outputs
        uses: actions/download-artifact@v4
        with:
          pattern: adsb-compressed-${{ needs.resolve-dates.outputs.adsb_date }}-part-*
          path: data/output/compressed/${{ needs.resolve-dates.outputs.adsb_date }}
          merge-multiple: true

      - name: Concatenate final outputs
        env:
          DATE: ${{ needs.resolve-dates.outputs.adsb_date }}
          CONCAT_WITH_LATEST_CSV: true
        run: |
          EXTRA=""
          if [ "$CONCAT_WITH_LATEST_CSV" = "true" ]; then
            EXTRA="--concat_with_latest_csv"
          fi
          python -m src.adsb.concat_parquet_to_final --date "$DATE" $EXTRA
          ls -lah data/output/ || true

      - name: Upload final artifacts
        uses: actions/upload-artifact@v4
        with:
          name: openairframes_adsb-${{ needs.resolve-dates.outputs.adsb_date }}
          path: data/output/openairframes_adsb_*
          retention-days: 30
          if-no-files-found: error

  build-community:
    runs-on: ubuntu-latest
    if: github.event_name != 'schedule'
    steps:
      - name: Checkout
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Setup Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.14"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install pandas

      - name: Run Community release script
        run: |
          python -m src.contributions.create_daily_community_release
          ls -lah data/openairframes

      - name: Upload Community artifacts
        uses: actions/upload-artifact@v4
        with:
          name: community-release
          path: data/openairframes/openairframes_community_*.csv
          retention-days: 1

  build-adsbexchange-json:
    runs-on: ubuntu-latest
    if: github.event_name != 'schedule'
    steps:
      - name: Checkout
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Setup Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.14"

      - name: Run ADS-B Exchange JSON release script
        run: |
          python -m src.contributions.create_daily_adsbexchange_release ${{ inputs.date && format('--date {0}', inputs.date) || '' }}
          ls -lah data/openairframes

      - name: Upload ADS-B Exchange JSON artifact
        uses: actions/upload-artifact@v4
        with:
          name: adsbexchange-json
          path: data/openairframes/basic-ac-db_*.json.gz
          retention-days: 1

  build-mictronics-db:
    runs-on: ubuntu-latest
    if: github.event_name != 'schedule'
    steps:
      - name: Checkout
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Setup Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.14"

      - name: Run Mictronics DB release script
        continue-on-error: true
        run: |
          python -m src.contributions.create_daily_microtonics_release ${{ inputs.date && format('--date {0}', inputs.date) || '' }}
          ls -lah data/openairframes

      - name: Upload Mictronics DB artifact
        uses: actions/upload-artifact@v4
        with:
          name: mictronics-db
          path: data/openairframes/mictronics-db_*.zip
          retention-days: 1
          if-no-files-found: ignore

  create-release:
    runs-on: ubuntu-latest
    needs: [resolve-dates, join-registry, adsb-to-aircraft, adsb-reduce, build-community, build-adsbexchange-json, build-mictronics-db]
    if: github.event_name != 'schedule' && !cancelled()
    steps:
      - name: Check ADS-B workflow status
        if: needs.adsb-to-aircraft.result != 'success' && needs.adsb-reduce.result != 'success'
        run: |
          echo "WARNING: ADS-B workflow failed (adsb-to-aircraft='${{ needs.adsb-to-aircraft.result }}', adsb-reduce='${{ needs.adsb-reduce.result }}'), will continue without ADS-B artifacts"

      - name: Checkout for gh CLI
        uses: actions/checkout@v4
        with:
          sparse-checkout: |
            .github
            NOTICE
          sparse-checkout-cone-mode: false

      - name: Download joined registry
        uses: actions/download-artifact@v5
        with:
          name: union-registry
          path: artifacts/faa

      - name: Download ADS-B artifacts
        uses: actions/download-artifact@v5
        if: needs.adsb-to-aircraft.result == 'success' || needs.adsb-reduce.result == 'success'
        continue-on-error: true
        with:
          name: openairframes_adsb-${{ needs.resolve-dates.outputs.adsb_date }}
          path: artifacts/adsb

      - name: Download Community artifacts
        uses: actions/download-artifact@v5
        with:
          name: community-release
          path: artifacts/community

      - name: Download ADS-B Exchange JSON artifact
        uses: actions/download-artifact@v5
        with:
          name: adsbexchange-json
          path: artifacts/adsbexchange

      - name: Download Mictronics DB artifact
        uses: actions/download-artifact@v5
        continue-on-error: true
        with:
          name: mictronics-db
          path: artifacts/mictronics

      - name: Debug artifact structure
        run: |
          echo "=== Full artifacts tree ==="
          find artifacts -type f 2>/dev/null || echo "No files found in artifacts"
          echo "=== FAA artifacts ==="
          find artifacts/faa -type f 2>/dev/null || echo "No files found in artifacts/faa"
          echo "=== ADS-B artifacts ==="
          find artifacts/adsb -type f 2>/dev/null || echo "No files found in artifacts/adsb"
          echo "=== Community artifacts ==="
          find artifacts/community -type f 2>/dev/null || echo "No files found in artifacts/community"
          echo "=== ADS-B Exchange JSON artifacts ==="
          find artifacts/adsbexchange -type f 2>/dev/null || echo "No files found in artifacts/adsbexchange"
          echo "=== Mictronics DB artifacts ==="
          find artifacts/mictronics -type f 2>/dev/null || echo "No files found in artifacts/mictronics"

      - name: Prepare release metadata
        id: meta
        run: |
          DATE=$(date -u +"%Y-%m-%d")
          BRANCH_NAME="${GITHUB_REF#refs/heads/}"
          BRANCH_SUFFIX=""
          if [ "$BRANCH_NAME" = "main" ]; then
            BRANCH_SUFFIX="-main"
          elif [ "$BRANCH_NAME" = "develop" ]; then
            BRANCH_SUFFIX="-develop"
          fi
          TAG="openairframes-${DATE}${BRANCH_SUFFIX}"
          
          # Find files from artifacts using find (handles nested structures)
          CSV_FILE_FAA=$(find artifacts/faa -name "openairframes_faa_*.csv" -type f 2>/dev/null | head -1)
          CSV_FILE_REGISTRY=$(find artifacts/faa -name "openairframes_registry_*.csv" -type f 2>/dev/null | head -1)
          # Every per-source registry CSV, whatever sources the matrix ran.
          SOURCE_CSVS=$(find artifacts/faa -name "openairframes_*.csv" -type f 2>/dev/null \
            | grep -vE '/openairframes_(registry|community|adsb)_[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{4}-[0-9]{2}-[0-9]{2}\.csv$' | sort)
          echo "Per-source registry CSVs found:"
          echo "$SOURCE_CSVS"
          # Prefer concatenated file (with date range) over single-day file
          CSV_FILE_ADSB=$(find artifacts/adsb -name "openairframes_adsb_*_*.csv.gz" -type f 2>/dev/null | head -1)
          if [ -z "$CSV_FILE_ADSB" ]; then
            CSV_FILE_ADSB=$(find artifacts/adsb -name "openairframes_adsb_*.csv.gz" -type f 2>/dev/null | head -1)
          fi
          CSV_FILE_COMMUNITY=$(find artifacts/community -name "openairframes_community_*.csv" -type f 2>/dev/null | head -1)
          ZIP_FILE=$(find artifacts/faa -name "ReleasableAircraft_*.zip" -type f 2>/dev/null | head -1)
          JSON_FILE_ADSBX=$(find artifacts/adsbexchange -name "basic-ac-db_*.json.gz" -type f 2>/dev/null | head -1)
          ZIP_FILE_MICTRONICS=$(find artifacts/mictronics -name "mictronics-db_*.zip" -type f 2>/dev/null | head -1)
          
          # Validate required files exist
          MISSING_FILES=""
          if [ -z "$CSV_FILE_FAA" ] || [ ! -f "$CSV_FILE_FAA" ]; then
            MISSING_FILES="$MISSING_FILES FAA_CSV"
          fi
          if [ -z "$ZIP_FILE" ] || [ ! -f "$ZIP_FILE" ]; then
            MISSING_FILES="$MISSING_FILES FAA_ZIP"
          fi
          if [ -z "$JSON_FILE_ADSBX" ] || [ ! -f "$JSON_FILE_ADSBX" ]; then
            MISSING_FILES="$MISSING_FILES ADSBX_JSON"
          fi
          if [ -z "$CSV_FILE_REGISTRY" ] || [ ! -f "$CSV_FILE_REGISTRY" ]; then
            MISSING_FILES="$MISSING_FILES REGISTRY_CSV"
          fi
          # NOTICE carries the terms that make each asset redistributable. Shipping data
          # without it removes the permission, so it is required rather than optional.
          if [ ! -f NOTICE ]; then
            MISSING_FILES="$MISSING_FILES NOTICE"
          fi
          
          # Optional files - warn but don't fail
          OPTIONAL_MISSING=""
          if [ -z "$CSV_FILE_ADSB" ] || [ ! -f "$CSV_FILE_ADSB" ]; then
            OPTIONAL_MISSING="$OPTIONAL_MISSING ADSB_CSV"
            CSV_FILE_ADSB=""
            CSV_BASENAME_ADSB=""
          fi
          if [ -z "$ZIP_FILE_MICTRONICS" ] || [ ! -f "$ZIP_FILE_MICTRONICS" ]; then
            OPTIONAL_MISSING="$OPTIONAL_MISSING MICTRONICS_ZIP"
            ZIP_FILE_MICTRONICS=""
          fi
          
          if [ -n "$MISSING_FILES" ]; then
            echo "ERROR: Missing required release files:$MISSING_FILES"
            echo "FAA CSV: $CSV_FILE_FAA"
            echo "ADSB CSV: $CSV_FILE_ADSB"
            echo "ZIP: $ZIP_FILE"
            echo "ADSBX JSON: $JSON_FILE_ADSBX"
            echo "MICTRONICS ZIP: $ZIP_FILE_MICTRONICS"
            exit 1
          fi
          
          # Get basenames for display
          CSV_BASENAME_FAA=$(basename "$CSV_FILE_FAA")
          if [ -n "$CSV_FILE_ADSB" ]; then
            CSV_BASENAME_ADSB=$(basename "$CSV_FILE_ADSB")
          fi
          CSV_BASENAME_COMMUNITY=$(basename "$CSV_FILE_COMMUNITY" 2>/dev/null || echo "")
          ZIP_BASENAME=$(basename "$ZIP_FILE")
          JSON_BASENAME_ADSBX=$(basename "$JSON_FILE_ADSBX")
          ZIP_BASENAME_MICTRONICS=""
          if [ -n "$ZIP_FILE_MICTRONICS" ]; then
            ZIP_BASENAME_MICTRONICS=$(basename "$ZIP_FILE_MICTRONICS")
          fi
          
          if [ -n "$OPTIONAL_MISSING" ]; then
            echo "::warning title=Missing optional release assets::$OPTIONAL_MISSING"
          fi
          
          echo "date=$DATE" >> "$GITHUB_OUTPUT"
          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
          echo "csv_file_faa=$CSV_FILE_FAA" >> "$GITHUB_OUTPUT"
          echo "csv_file_registry=$CSV_FILE_REGISTRY" >> "$GITHUB_OUTPUT"
          {
            echo "source_csvs<<SOURCE_CSVS_EOF"
            echo "$SOURCE_CSVS"
            echo "SOURCE_CSVS_EOF"
            echo "source_basenames<<SOURCE_BASENAMES_EOF"
            echo "$SOURCE_CSVS" | while read -r f; do
              [ -n "$f" ] && echo "- $(basename "$f")"
            done
            echo "SOURCE_BASENAMES_EOF"
          } >> "$GITHUB_OUTPUT"
          echo "csv_basename_registry=$(basename "$CSV_FILE_REGISTRY")" >> "$GITHUB_OUTPUT"
          echo "csv_basename_faa=$CSV_BASENAME_FAA" >> "$GITHUB_OUTPUT"
          echo "csv_file_adsb=$CSV_FILE_ADSB" >> "$GITHUB_OUTPUT"
          echo "csv_basename_adsb=$CSV_BASENAME_ADSB" >> "$GITHUB_OUTPUT"
          echo "csv_file_community=$CSV_FILE_COMMUNITY" >> "$GITHUB_OUTPUT"
          echo "csv_basename_community=$CSV_BASENAME_COMMUNITY" >> "$GITHUB_OUTPUT"
          echo "zip_file=$ZIP_FILE" >> "$GITHUB_OUTPUT"
          echo "zip_basename=$ZIP_BASENAME" >> "$GITHUB_OUTPUT"
          echo "json_file_adsbx=$JSON_FILE_ADSBX" >> "$GITHUB_OUTPUT"
          echo "json_basename_adsbx=$JSON_BASENAME_ADSBX" >> "$GITHUB_OUTPUT"
          echo "zip_file_mictronics=$ZIP_FILE_MICTRONICS" >> "$GITHUB_OUTPUT"
          echo "zip_basename_mictronics=$ZIP_BASENAME_MICTRONICS" >> "$GITHUB_OUTPUT"
          echo "name=OpenAirframes snapshot ($DATE)${BRANCH_SUFFIX}" >> "$GITHUB_OUTPUT"
          
          echo "Found files:"
          echo "  FAA CSV: $CSV_FILE_FAA"
          echo "  ADSB CSV: $CSV_FILE_ADSB"
          echo "  Community CSV: $CSV_FILE_COMMUNITY"
          echo "  ZIP: $ZIP_FILE"
          echo "  ADSBX JSON: $JSON_FILE_ADSBX"
          echo "  MICTRONICS ZIP: $ZIP_FILE_MICTRONICS"

      - name: Delete existing release if exists
        run: |
          echo "Attempting to delete release: ${{ steps.meta.outputs.tag }}"
          # `|| echo` here would swallow a 403 or a partial delete, leaving yesterday's
          # asset attached alongside today's; the next run then matches two and rebuilds
          # the dataset from a single day.
          if gh release view "${{ steps.meta.outputs.tag }}" >/dev/null 2>&1; then
            gh release delete "${{ steps.meta.outputs.tag }}" --yes --cleanup-tag
          else
            echo "No existing release to delete"
          fi
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Create GitHub Release and upload assets
        uses: softprops/action-gh-release@v2
        with:
          tag_name: ${{ steps.meta.outputs.tag }}
          name: ${{ steps.meta.outputs.name }}
          fail_on_unmatched_files: false
          body: |
            Automated daily snapshot generated at 06:00 UTC for ${{ steps.meta.outputs.date }}.

            Assets:
            - NOTICE (source terms; required for redistribution)
            - ${{ steps.meta.outputs.csv_basename_registry }}
            ${{ steps.meta.outputs.source_basenames }}
            ${{ steps.meta.outputs.csv_basename_adsb && format('- {0}', steps.meta.outputs.csv_basename_adsb) || '' }}
            - ${{ steps.meta.outputs.csv_basename_community }}
            - ${{ steps.meta.outputs.zip_basename }}
            - ${{ steps.meta.outputs.json_basename_adsbx }}
            ${{ steps.meta.outputs.zip_basename_mictronics && format('- {0}', steps.meta.outputs.zip_basename_mictronics) || '' }}
          files: |
            ${{ steps.meta.outputs.csv_file_registry }}
            ${{ steps.meta.outputs.source_csvs }}
            NOTICE
            ${{ steps.meta.outputs.csv_file_adsb }}
            ${{ steps.meta.outputs.csv_file_community }}
            ${{ steps.meta.outputs.zip_file }}
            ${{ steps.meta.outputs.json_file_adsbx }}
            ${{ steps.meta.outputs.zip_file_mictronics }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}