From 953a3647dfca14721eb3c9e358d528299f356ef6 Mon Sep 17 00:00:00 2001 From: ggman12 Date: Wed, 11 Feb 2026 23:40:46 -0500 Subject: [PATCH] remove process historical-faa github workflow --- .github/workflows/process-historical-faa.yaml | 171 ------------------ src/get_historical_faa.py | 116 ------------ 2 files changed, 287 deletions(-) delete mode 100644 .github/workflows/process-historical-faa.yaml delete mode 100644 src/get_historical_faa.py diff --git a/.github/workflows/process-historical-faa.yaml b/.github/workflows/process-historical-faa.yaml deleted file mode 100644 index d015499..0000000 --- a/.github/workflows/process-historical-faa.yaml +++ /dev/null @@ -1,171 +0,0 @@ -name: Process Historical FAA Data - -on: - workflow_dispatch: # Manual trigger - -jobs: - generate-matrix: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - name: Generate date ranges - id: set-matrix - run: | - python3 << 'EOF' - import json - from datetime import datetime, timedelta - - start = datetime(2023, 8, 16) - end = datetime(2026, 1, 1) - - ranges = [] - current = start - - # Process in 4-day chunks - while current < end: - chunk_end = current + timedelta(days=4) - # Don't go past the end date - if chunk_end > end: - chunk_end = end - - ranges.append({ - "since": current.strftime("%Y-%m-%d"), - "until": chunk_end.strftime("%Y-%m-%d") - }) - - current = chunk_end - - print(f"::set-output name=matrix::{json.dumps(ranges)}") - EOF - - clone-faa-repo: - runs-on: ubuntu-latest - steps: - - name: Cache FAA repository - id: cache-faa-repo - uses: actions/cache@v4 - with: - path: data/scrape-faa-releasable-aircraft - key: faa-repo-v1 - - - name: Clone FAA repository - if: steps.cache-faa-repo.outputs.cache-hit != 'true' - run: | - mkdir -p data - git clone https://github.com/simonw/scrape-faa-releasable-aircraft data/scrape-faa-releasable-aircraft - echo "Repository cloned successfully" - - process-chunk: - needs: [generate-matrix, clone-faa-repo] - runs-on: ubuntu-latest - strategy: - max-parallel: 5 # Process 5 chunks at a time - matrix: - range: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Restore FAA repository cache - uses: actions/cache/restore@v4 - with: - path: data/scrape-faa-releasable-aircraft - key: faa-repo-v1 - fail-on-cache-miss: true - - - name: Install dependencies - run: | - pip install -r requirements.txt - - - name: Process chunk ${{ matrix.range.since }} to ${{ matrix.range.until }} - run: | - python src/get_historical_faa.py "${{ matrix.range.since }}" "${{ matrix.range.until }}" - - - name: Upload CSV artifact - uses: actions/upload-artifact@v4 - with: - name: csv-${{ matrix.range.since }}-to-${{ matrix.range.until }} - path: data/faa_releasable_historical/*.csv - retention-days: 1 - - create-release: - needs: process-chunk - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts - - - name: Prepare release files - run: | - mkdir -p release-files - find artifacts -name "*.csv" -exec cp {} release-files/ \; - ls -lh release-files/ - - - name: Create Release - uses: softprops/action-gh-release@v1 - with: - tag_name: historical-faa-${{ github.run_number }} - name: Historical FAA Data Release ${{ github.run_number }} - body: | - Automated release of historical FAA aircraft data - Processing period: 2023-08-16 to 2026-01-01 - Generated: ${{ github.event.repository.updated_at }} - files: release-files/*.csv - draft: false - prerelease: false - - concatenate-and-release: - needs: process-chunk - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install dependencies - run: | - pip install -r requirements.txt - - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts - - - name: Prepare CSVs for concatenation - run: | - mkdir -p data/faa_releasable_historical - find artifacts -name "*.csv" -exec cp {} data/faa_releasable_historical/ \; - ls -lh data/faa_releasable_historical/ - - - name: Concatenate all CSVs - run: | - python scripts/concat_csvs.py - - - name: Create Combined Release - uses: softprops/action-gh-release@v1 - with: - tag_name: historical-faa-combined-${{ github.run_number }} - name: Historical FAA Data Combined Release ${{ github.run_number }} - body: | - Combined historical FAA aircraft data (all chunks concatenated) - Processing period: 2023-08-16 to 2026-01-01 - Generated: ${{ github.event.repository.updated_at }} - files: data/planequery_aircraft/*.csv - draft: false - prerelease: false \ No newline at end of file diff --git a/src/get_historical_faa.py b/src/get_historical_faa.py deleted file mode 100644 index 656345e..0000000 --- a/src/get_historical_faa.py +++ /dev/null @@ -1,116 +0,0 @@ -""" -For each commit-day in Feb 2024 (last commit per day): -- Write ALL FAA text files from that commit into: data/faa_releasable_historical/YYYY-MM-DD/ - ACFTREF.txt, DEALER.txt, DOCINDEX.txt, ENGINE.txt, RESERVED.txt -- Recombine MASTER-*.txt into Master.txt -- Produce Master.csv via convert_faa_master_txt_to_csv - -Assumes the non-master files are present in every commit. -""" -import subprocess, re -from pathlib import Path -import shutil -from collections import OrderedDict -from derive_from_faa_master_txt import convert_faa_master_txt_to_df, concat_faa_historical_df -import zipfile -import pandas as pd -import argparse -from datetime import datetime, timedelta - -# Parse command line arguments -parser = argparse.ArgumentParser(description="Process historical FAA data from git commits") -parser.add_argument("since", help="Start date (YYYY-MM-DD)") -parser.add_argument("until", help="End date (YYYY-MM-DD)") -args = parser.parse_args() - -# Clone repository if it doesn't exist -REPO = Path("data/scrape-faa-releasable-aircraft") -OUT_ROOT = Path("data/faa_releasable_historical") -OUT_ROOT.mkdir(parents=True, exist_ok=True) - -def run_git_text(*args: str) -> str: - return subprocess.check_output(["git", "-C", str(REPO), *args], text=True).strip() - -def run_git_bytes(*args: str) -> bytes: - return subprocess.check_output(["git", "-C", str(REPO), *args]) - -# Parse dates and adjust --since to the day before -since_date = datetime.strptime(args.since, "%Y-%m-%d") -adjusted_since = (since_date - timedelta(days=1)).strftime("%Y-%m-%d") - -# All commits in specified date range (oldest -> newest) -log = run_git_text( - "log", - "--reverse", - "--format=%H %cs", - f"--since={adjusted_since}", - f"--until={args.until}", -) -lines = [ln for ln in log.splitlines() if ln.strip()] -if not lines: - raise SystemExit(f"No commits found between {args.since} and {args.until}.") - -# date -> last SHA that day -date_to_sha = OrderedDict() -for ln in lines: - sha, date = ln.split() - date_to_sha[date] = sha - -OTHER_FILES = ["ACFTREF.txt", "DEALER.txt", "DOCINDEX.txt", "ENGINE.txt", "RESERVED.txt"] -master_re = re.compile(r"^MASTER-(\d+)\.txt$") -df_base = pd.DataFrame() -start_date = None -end_date = None -for date, sha in date_to_sha.items(): - if start_date is None: - start_date = date - end_date = date - day_dir = OUT_ROOT / date - day_dir.mkdir(parents=True, exist_ok=True) - - # Write auxiliary files (assumed present) - for fname in OTHER_FILES: - (day_dir / fname).write_bytes(run_git_bytes("show", f"{sha}:{fname}")) - - # Recombine MASTER parts - names = run_git_text("ls-tree", "--name-only", sha).splitlines() - parts = [] - for n in names: - m = master_re.match(n) - if m: - parts.append((int(m.group(1)), n)) - parts.sort() - if not parts: - raise RuntimeError(f"{date} {sha[:7]}: no MASTER-*.txt parts found") - - master_path = day_dir / "MASTER.txt" - with master_path.open("wb") as w: - for _, fname in parts: - data = run_git_bytes("show", f"{sha}:{fname}") - w.write(data) - if data and not data.endswith(b"\n"): - w.write(b"\n") - - # 3) Zip the day's files - zip_path = day_dir / f"ReleasableAircraft.zip" - with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as z: - for p in day_dir.iterdir(): - z.write(p, arcname=p.name) - - print(f"{date} {sha[:7]} -> {day_dir} (master parts: {len(parts)})") - # 4) Convert ZIP -> CSV - df_new = convert_faa_master_txt_to_df(zip_path, date) - if df_base.empty: - df_base = df_new - print(len(df_base), "total entries so far") - # Delete all files in the day directory - shutil.rmtree(day_dir) - continue - - df_base = concat_faa_historical_df(df_base, df_new) - shutil.rmtree(day_dir) - print(len(df_base), "total entries so far") - -assert df_base['download_date'].is_monotonic_increasing, "download_date is not monotonic increasing" -df_base.to_csv(OUT_ROOT / f"planequery_aircraft_faa_{start_date}_{end_date}.csv", index=False) -# TODO: get average number of new rows per day.