mirror of
https://github.com/PlaneQuery/OpenAirframes.git
synced 2026-04-23 19:46:09 +02:00
remove process historical-faa github workflow
This commit is contained in:
ggman12
@@ -1,171 +0,0 @@
|
||||
name: Process Historical FAA Data
|
||||
|
||||
on:
|
||||
workflow_dispatch: # Manual trigger
|
||||
|
||||
jobs:
|
||||
generate-matrix:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
matrix: ${{ steps.set-matrix.outputs.matrix }}
|
||||
steps:
|
||||
- name: Generate date ranges
|
||||
id: set-matrix
|
||||
run: |
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
start = datetime(2023, 8, 16)
|
||||
end = datetime(2026, 1, 1)
|
||||
|
||||
ranges = []
|
||||
current = start
|
||||
|
||||
# Process in 4-day chunks
|
||||
while current < end:
|
||||
chunk_end = current + timedelta(days=4)
|
||||
# Don't go past the end date
|
||||
if chunk_end > end:
|
||||
chunk_end = end
|
||||
|
||||
ranges.append({
|
||||
"since": current.strftime("%Y-%m-%d"),
|
||||
"until": chunk_end.strftime("%Y-%m-%d")
|
||||
})
|
||||
|
||||
current = chunk_end
|
||||
|
||||
print(f"::set-output name=matrix::{json.dumps(ranges)}")
|
||||
EOF
|
||||
|
||||
clone-faa-repo:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Cache FAA repository
|
||||
id: cache-faa-repo
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: data/scrape-faa-releasable-aircraft
|
||||
key: faa-repo-v1
|
||||
|
||||
- name: Clone FAA repository
|
||||
if: steps.cache-faa-repo.outputs.cache-hit != 'true'
|
||||
run: |
|
||||
mkdir -p data
|
||||
git clone https://github.com/simonw/scrape-faa-releasable-aircraft data/scrape-faa-releasable-aircraft
|
||||
echo "Repository cloned successfully"
|
||||
|
||||
process-chunk:
|
||||
needs: [generate-matrix, clone-faa-repo]
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
max-parallel: 5 # Process 5 chunks at a time
|
||||
matrix:
|
||||
range: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Restore FAA repository cache
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: data/scrape-faa-releasable-aircraft
|
||||
key: faa-repo-v1
|
||||
fail-on-cache-miss: true
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
|
||||
- name: Process chunk ${{ matrix.range.since }} to ${{ matrix.range.until }}
|
||||
run: |
|
||||
python src/get_historical_faa.py "${{ matrix.range.since }}" "${{ matrix.range.until }}"
|
||||
|
||||
- name: Upload CSV artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: csv-${{ matrix.range.since }}-to-${{ matrix.range.until }}
|
||||
path: data/faa_releasable_historical/*.csv
|
||||
retention-days: 1
|
||||
|
||||
create-release:
|
||||
needs: process-chunk
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: artifacts
|
||||
|
||||
- name: Prepare release files
|
||||
run: |
|
||||
mkdir -p release-files
|
||||
find artifacts -name "*.csv" -exec cp {} release-files/ \;
|
||||
ls -lh release-files/
|
||||
|
||||
- name: Create Release
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
tag_name: historical-faa-${{ github.run_number }}
|
||||
name: Historical FAA Data Release ${{ github.run_number }}
|
||||
body: |
|
||||
Automated release of historical FAA aircraft data
|
||||
Processing period: 2023-08-16 to 2026-01-01
|
||||
Generated: ${{ github.event.repository.updated_at }}
|
||||
files: release-files/*.csv
|
||||
draft: false
|
||||
prerelease: false
|
||||
|
||||
concatenate-and-release:
|
||||
needs: process-chunk
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: artifacts
|
||||
|
||||
- name: Prepare CSVs for concatenation
|
||||
run: |
|
||||
mkdir -p data/faa_releasable_historical
|
||||
find artifacts -name "*.csv" -exec cp {} data/faa_releasable_historical/ \;
|
||||
ls -lh data/faa_releasable_historical/
|
||||
|
||||
- name: Concatenate all CSVs
|
||||
run: |
|
||||
python scripts/concat_csvs.py
|
||||
|
||||
- name: Create Combined Release
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
tag_name: historical-faa-combined-${{ github.run_number }}
|
||||
name: Historical FAA Data Combined Release ${{ github.run_number }}
|
||||
body: |
|
||||
Combined historical FAA aircraft data (all chunks concatenated)
|
||||
Processing period: 2023-08-16 to 2026-01-01
|
||||
Generated: ${{ github.event.repository.updated_at }}
|
||||
files: data/planequery_aircraft/*.csv
|
||||
draft: false
|
||||
prerelease: false
|
||||
@@ -1,116 +0,0 @@
|
||||
"""
|
||||
For each commit-day in Feb 2024 (last commit per day):
|
||||
- Write ALL FAA text files from that commit into: data/faa_releasable_historical/YYYY-MM-DD/
|
||||
ACFTREF.txt, DEALER.txt, DOCINDEX.txt, ENGINE.txt, RESERVED.txt
|
||||
- Recombine MASTER-*.txt into Master.txt
|
||||
- Produce Master.csv via convert_faa_master_txt_to_csv
|
||||
|
||||
Assumes the non-master files are present in every commit.
|
||||
"""
|
||||
import subprocess, re
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
from collections import OrderedDict
|
||||
from derive_from_faa_master_txt import convert_faa_master_txt_to_df, concat_faa_historical_df
|
||||
import zipfile
|
||||
import pandas as pd
|
||||
import argparse
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
# Parse command line arguments
|
||||
parser = argparse.ArgumentParser(description="Process historical FAA data from git commits")
|
||||
parser.add_argument("since", help="Start date (YYYY-MM-DD)")
|
||||
parser.add_argument("until", help="End date (YYYY-MM-DD)")
|
||||
args = parser.parse_args()
|
||||
|
||||
# Clone repository if it doesn't exist
|
||||
REPO = Path("data/scrape-faa-releasable-aircraft")
|
||||
OUT_ROOT = Path("data/faa_releasable_historical")
|
||||
OUT_ROOT.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def run_git_text(*args: str) -> str:
|
||||
return subprocess.check_output(["git", "-C", str(REPO), *args], text=True).strip()
|
||||
|
||||
def run_git_bytes(*args: str) -> bytes:
|
||||
return subprocess.check_output(["git", "-C", str(REPO), *args])
|
||||
|
||||
# Parse dates and adjust --since to the day before
|
||||
since_date = datetime.strptime(args.since, "%Y-%m-%d")
|
||||
adjusted_since = (since_date - timedelta(days=1)).strftime("%Y-%m-%d")
|
||||
|
||||
# All commits in specified date range (oldest -> newest)
|
||||
log = run_git_text(
|
||||
"log",
|
||||
"--reverse",
|
||||
"--format=%H %cs",
|
||||
f"--since={adjusted_since}",
|
||||
f"--until={args.until}",
|
||||
)
|
||||
lines = [ln for ln in log.splitlines() if ln.strip()]
|
||||
if not lines:
|
||||
raise SystemExit(f"No commits found between {args.since} and {args.until}.")
|
||||
|
||||
# date -> last SHA that day
|
||||
date_to_sha = OrderedDict()
|
||||
for ln in lines:
|
||||
sha, date = ln.split()
|
||||
date_to_sha[date] = sha
|
||||
|
||||
OTHER_FILES = ["ACFTREF.txt", "DEALER.txt", "DOCINDEX.txt", "ENGINE.txt", "RESERVED.txt"]
|
||||
master_re = re.compile(r"^MASTER-(\d+)\.txt$")
|
||||
df_base = pd.DataFrame()
|
||||
start_date = None
|
||||
end_date = None
|
||||
for date, sha in date_to_sha.items():
|
||||
if start_date is None:
|
||||
start_date = date
|
||||
end_date = date
|
||||
day_dir = OUT_ROOT / date
|
||||
day_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Write auxiliary files (assumed present)
|
||||
for fname in OTHER_FILES:
|
||||
(day_dir / fname).write_bytes(run_git_bytes("show", f"{sha}:{fname}"))
|
||||
|
||||
# Recombine MASTER parts
|
||||
names = run_git_text("ls-tree", "--name-only", sha).splitlines()
|
||||
parts = []
|
||||
for n in names:
|
||||
m = master_re.match(n)
|
||||
if m:
|
||||
parts.append((int(m.group(1)), n))
|
||||
parts.sort()
|
||||
if not parts:
|
||||
raise RuntimeError(f"{date} {sha[:7]}: no MASTER-*.txt parts found")
|
||||
|
||||
master_path = day_dir / "MASTER.txt"
|
||||
with master_path.open("wb") as w:
|
||||
for _, fname in parts:
|
||||
data = run_git_bytes("show", f"{sha}:{fname}")
|
||||
w.write(data)
|
||||
if data and not data.endswith(b"\n"):
|
||||
w.write(b"\n")
|
||||
|
||||
# 3) Zip the day's files
|
||||
zip_path = day_dir / f"ReleasableAircraft.zip"
|
||||
with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as z:
|
||||
for p in day_dir.iterdir():
|
||||
z.write(p, arcname=p.name)
|
||||
|
||||
print(f"{date} {sha[:7]} -> {day_dir} (master parts: {len(parts)})")
|
||||
# 4) Convert ZIP -> CSV
|
||||
df_new = convert_faa_master_txt_to_df(zip_path, date)
|
||||
if df_base.empty:
|
||||
df_base = df_new
|
||||
print(len(df_base), "total entries so far")
|
||||
# Delete all files in the day directory
|
||||
shutil.rmtree(day_dir)
|
||||
continue
|
||||
|
||||
df_base = concat_faa_historical_df(df_base, df_new)
|
||||
shutil.rmtree(day_dir)
|
||||
print(len(df_base), "total entries so far")
|
||||
|
||||
assert df_base['download_date'].is_monotonic_increasing, "download_date is not monotonic increasing"
|
||||
df_base.to_csv(OUT_ROOT / f"planequery_aircraft_faa_{start_date}_{end_date}.csv", index=False)
|
||||
# TODO: get average number of new rows per day.
|
||||
Reference in New Issue
Block a user