mirror of
https://github.com/PlaneQuery/OpenAirframes.git
synced 2026-04-24 12:06:31 +02:00
remove process historical-faa github workflow
This commit is contained in:
ggman12
@@ -1,171 +0,0 @@
|
||||
name: Process Historical FAA Data
|
||||
|
||||
on:
|
||||
workflow_dispatch: # Manual trigger
|
||||
|
||||
jobs:
|
||||
generate-matrix:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
matrix: ${{ steps.set-matrix.outputs.matrix }}
|
||||
steps:
|
||||
- name: Generate date ranges
|
||||
id: set-matrix
|
||||
run: |
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
start = datetime(2023, 8, 16)
|
||||
end = datetime(2026, 1, 1)
|
||||
|
||||
ranges = []
|
||||
current = start
|
||||
|
||||
# Process in 4-day chunks
|
||||
while current < end:
|
||||
chunk_end = current + timedelta(days=4)
|
||||
# Don't go past the end date
|
||||
if chunk_end > end:
|
||||
chunk_end = end
|
||||
|
||||
ranges.append({
|
||||
"since": current.strftime("%Y-%m-%d"),
|
||||
"until": chunk_end.strftime("%Y-%m-%d")
|
||||
})
|
||||
|
||||
current = chunk_end
|
||||
|
||||
print(f"::set-output name=matrix::{json.dumps(ranges)}")
|
||||
EOF
|
||||
|
||||
clone-faa-repo:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Cache FAA repository
|
||||
id: cache-faa-repo
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: data/scrape-faa-releasable-aircraft
|
||||
key: faa-repo-v1
|
||||
|
||||
- name: Clone FAA repository
|
||||
if: steps.cache-faa-repo.outputs.cache-hit != 'true'
|
||||
run: |
|
||||
mkdir -p data
|
||||
git clone https://github.com/simonw/scrape-faa-releasable-aircraft data/scrape-faa-releasable-aircraft
|
||||
echo "Repository cloned successfully"
|
||||
|
||||
process-chunk:
|
||||
needs: [generate-matrix, clone-faa-repo]
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
max-parallel: 5 # Process 5 chunks at a time
|
||||
matrix:
|
||||
range: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Restore FAA repository cache
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: data/scrape-faa-releasable-aircraft
|
||||
key: faa-repo-v1
|
||||
fail-on-cache-miss: true
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
|
||||
- name: Process chunk ${{ matrix.range.since }} to ${{ matrix.range.until }}
|
||||
run: |
|
||||
python src/get_historical_faa.py "${{ matrix.range.since }}" "${{ matrix.range.until }}"
|
||||
|
||||
- name: Upload CSV artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: csv-${{ matrix.range.since }}-to-${{ matrix.range.until }}
|
||||
path: data/faa_releasable_historical/*.csv
|
||||
retention-days: 1
|
||||
|
||||
create-release:
|
||||
needs: process-chunk
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: artifacts
|
||||
|
||||
- name: Prepare release files
|
||||
run: |
|
||||
mkdir -p release-files
|
||||
find artifacts -name "*.csv" -exec cp {} release-files/ \;
|
||||
ls -lh release-files/
|
||||
|
||||
- name: Create Release
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
tag_name: historical-faa-${{ github.run_number }}
|
||||
name: Historical FAA Data Release ${{ github.run_number }}
|
||||
body: |
|
||||
Automated release of historical FAA aircraft data
|
||||
Processing period: 2023-08-16 to 2026-01-01
|
||||
Generated: ${{ github.event.repository.updated_at }}
|
||||
files: release-files/*.csv
|
||||
draft: false
|
||||
prerelease: false
|
||||
|
||||
concatenate-and-release:
|
||||
needs: process-chunk
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: artifacts
|
||||
|
||||
- name: Prepare CSVs for concatenation
|
||||
run: |
|
||||
mkdir -p data/faa_releasable_historical
|
||||
find artifacts -name "*.csv" -exec cp {} data/faa_releasable_historical/ \;
|
||||
ls -lh data/faa_releasable_historical/
|
||||
|
||||
- name: Concatenate all CSVs
|
||||
run: |
|
||||
python scripts/concat_csvs.py
|
||||
|
||||
- name: Create Combined Release
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
tag_name: historical-faa-combined-${{ github.run_number }}
|
||||
name: Historical FAA Data Combined Release ${{ github.run_number }}
|
||||
body: |
|
||||
Combined historical FAA aircraft data (all chunks concatenated)
|
||||
Processing period: 2023-08-16 to 2026-01-01
|
||||
Generated: ${{ github.event.repository.updated_at }}
|
||||
files: data/planequery_aircraft/*.csv
|
||||
draft: false
|
||||
prerelease: false
|
||||
Reference in New Issue
Block a user