diff --git a/.github/workflows/openairframes-daily-release.yaml b/.github/workflows/openairframes-daily-release.yaml index c0787bb..ca9003d 100644 --- a/.github/workflows/openairframes-daily-release.yaml +++ b/.github/workflows/openairframes-daily-release.yaml @@ -2,7 +2,7 @@ name: openairframes-daily-release on: schedule: - # 6:00pm UTC every day - runs on default branch, triggers both + # 06:00 UTC every day - runs on default branch, triggers both - cron: "0 06 * * *" workflow_dispatch: inputs: diff --git a/.github/workflows/update-community-prs.yaml b/.github/workflows/update-community-prs.yaml index cf7e826..f51eaba 100644 --- a/.github/workflows/update-community-prs.yaml +++ b/.github/workflows/update-community-prs.yaml @@ -5,7 +5,7 @@ on: branches: [main] paths: - 'community/**' - - 'schemas/community_submission.v1.schema.json' + - 'schemas/**' permissions: contents: write diff --git a/.github/workflows/validate-community-submission.yaml b/.github/workflows/validate-community-submission.yaml index ed2973d..956ba35 100644 --- a/.github/workflows/validate-community-submission.yaml +++ b/.github/workflows/validate-community-submission.yaml @@ -23,24 +23,17 @@ jobs: - name: Install dependencies run: pip install jsonschema - - name: Debug issue body - run: | - echo "=== Issue Body ===" - cat << 'ISSUE_BODY_EOF' - ${{ github.event.issue.body }} - ISSUE_BODY_EOF - - name: Save issue body to file - run: | - cat << 'ISSUE_BODY_EOF' > /tmp/issue_body.txt - ${{ github.event.issue.body }} - ISSUE_BODY_EOF + env: + ISSUE_BODY: ${{ github.event.issue.body }} + run: printf '%s' "$ISSUE_BODY" > "$RUNNER_TEMP/issue_body.txt" - name: Validate submission env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_REPOSITORY: ${{ github.repository }} + ISSUE_NUMBER: ${{ github.event.issue.number }} run: | python -m src.contributions.validate_submission \ - --issue-body-file /tmp/issue_body.txt \ - --issue-number ${{ github.event.issue.number }} + --issue-body-file "$RUNNER_TEMP/issue_body.txt" \ + --issue-number "$ISSUE_NUMBER" diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..b523458 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,121 @@ +## Never interpolate `${{ github.event.* }}` into a `run:` block + +Actions substitutes `${{ }}` as raw text before the shell parses it, and issue bodies here are +public and unauthenticated. Pass untrusted values through `env:` and quote them. + +A quoted heredoc delimiter does not save you: the body can *contain* the delimiter, close the +heredoc early, and execute every line after it. `validate-community-submission.yaml` had that shape. +Never fix this class of bug by escaping, sanitizing, or renaming the delimiter — move it to `env:`. + +## Run everything from the repo root + +`src/create_daily_faa_release.py` must be invoked as a **script** (`python src/create_daily_faa_release.py`). +It uses bare sibling imports, so `-m src.create_daily_faa_release` raises `ModuleNotFoundError`. +Everything under `src/adsb/` and `src/contributions/` is the opposite — `python -m`, package-relative. + +Output paths are CWD-relative. + +## Verification + +There is no test framework, linter, or packaging config. **Do not add one unprompted**, and do not +treat "nothing broke" as verification. + +The ADS-B path has no cheap end-to-end check — one day of input is tens of GB. Exercise +`compress_multi_icao_df` / `compress_df_polars` directly against a small hand-built Polars frame. + +**Never `gh workflow run` to test a change.** Every dispatch pulls tens of GB and fans out over date +matrices. Reason about the YAML statically. + +**Never commit generated data.** The product is a GitHub Release; jobs pass state as artifacts. + +## ADS-B invariants + +- `FINAL_COLUMN_ORDER` (`compress_adsb_to_aircraft_data.py`) is the only definition of the released + column contract. `pl.concat` matches by **position** after `.select()`, and + `get_latest_release.get_latest_aircraft_adsb_csv_df` parses released CSVs against the same order — + so a forked copy corrupts the release with no error anywhere. +- `load_parquet_part()` deleting its source parquet is **deliberate**: the raw part is many GB and + the runner would otherwise exhaust disk. Do not defer the delete to make reruns easier; that raises + peak disk by the size of the part. +- A released row means "most informative observation for this ICAO on this UTC day" — non-empty + fields not a subset of another row's, tie-broken by signature frequency. It is not a registry record. +- HTTP 404 is terminal in the release fetch. Restoring the retry makes the Dec-31 next-year-repo + probe stall ~45 minutes on a repo that does not exist yet. + +## Fork and upstream + +`src/get_latest_release.py` pins `REPO = "PlaneQuery/openairframes"` on purpose: this fork reads +**upstream's** releases wherever it runs. Do not repoint it at `github.repository` without being asked. + +Upstream develops on `develop` and PRs into `main`. The daily release **deletes the existing release +and tag** before recreating them. + +## Community submissions are automation-owned + +Merging to `community/**` or `schemas/**` force-pushes every open `community`-labeled PR branch back +onto main. Anything you hand-edit on such a branch is destroyed on the next merge. + +- Never hand-author files in `community/` — the filename encodes `sha256(content)[:8]`, so an edit + orphans the hash and duplicates on re-approval. +- Never invent or copy a `contributor_uuid`; it is derived from the GitHub user id. +- Do not reintroduce a hardcoded `"main"` or `v1` filename. Both are resolved at runtime now. + +**A tag's JSON type is fixed by its first-ever submission and enforced forever** — emergent from +`build_tag_type_registry` + `validate_submission`, and written nowhere in the schema. Retyping or +renaming an existing tag breaks every future contributor, not just the current one. + +Dropping a `community_submission.v2.schema.json` into `schemas/` promotes it atomically across every +reader and writer. That is a one-way door for contributors — only on explicit request. + +## Conventions + +- **Empty string, not null**, everywhere in released frames. +- Reuse `derive_from_faa_master_txt.normalize()` for `openairframes_id`; never re-derive the format. +- Python `3.14` for FAA/community/vendor jobs, `3.12` for ADS-B jobs (pyarrow pin + multiprocessing). + Deliberate. Match the surrounding job; do not unify. + +## References with no target — do not chase as regressions + +| Reference | Missing | +|---|---| +| `process-historical-faa.yaml` | `src/get_historical_faa.py`, `scripts/concat_csvs.py` | +| `af-klm-fleet/package.json` → `npm run validate` | `af-klm-fleet/scripts/validate.js` | + +`process-historical-faa.yaml` is dead, not stale — it also uses the disabled `::set-output`. +Repair-vs-delete is the owner's call; leave it alone unprompted. + +## `af-klm-fleet/` and `community-routes/` are unwired + +Nothing in CI touches either, and nothing consumes `community-routes/`. `af-klm-fleet/` is a vendored +project by a different author with its own license — its aircraft model is unrelated to +`schemas/community_submission.*`, so do not merge the two. Its `README.md` is generated by +`generate-readme.js`; hand edits are overwritten. + +## Warts left standing — flag, do not silently fix + +- `NUMBER_PARTS` is restated by the matrix and four hand-written upload steps in + `adsb-to-aircraft-for-day.yaml`; changing the constant alone silently drops data. YAML cannot loop + upload steps and one merged artifact would force every map job to download all parts, so any real + fix is a restructure. +- `MAX_WORKERS = OS_CPU_COUNT if OS_CPU_COUNT > 4 else 1` collapses to a single worker on a ≤4-core + runner, shrinking `files_per_batch` with it. Possibly intentional memory control — do not raise it + without measuring peak RSS on the target runner. +- `update-community-prs.yaml` runs `regenerate_pr_schema || true` then force-pushes, so a + regeneration failure ships anyway. Making it fatal leaves PRs un-rebased instead — a judgment call. +- `approve_submission.py` wraps its schema update in a bare `except Exception`, so a submission can + merge without its new tags reaching the schema. + +## Workflow authoring + +The user's global GitHub Actions rules apply. Existing workflows violate most of them. +**Do not bulk-remediate** — bring only the file you were asked to touch up to standard, and surface +the rest in chat. + +## External sources + +`registry.faa.gov` and ADS-B Exchange are **required** — the release fails without them. Mictronics is +**tolerated**: it retries, then the job continues without it. adsb.lol may simply not have published a +given day, in which case the previous CSV is re-released rather than failing. + +FAA refreshes at 05:30 UTC; the release cron fires at 06:00 UTC. That 30-minute margin is the reason +for the schedule. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..0be359e --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,6 @@ +# CLAUDE.md + +Read [AGENTS.md](./AGENTS.md) before touching anything in this repo. It is the single source of +repo-specific rules; this file adds nothing of its own and is never the place to record new ones. + +Record new repo-specific guidance in `AGENTS.md`. diff --git a/src/adsb/compress_adsb_to_aircraft_data.py b/src/adsb/compress_adsb_to_aircraft_data.py index cf995b7..989dd2b 100644 --- a/src/adsb/compress_adsb_to_aircraft_data.py +++ b/src/adsb/compress_adsb_to_aircraft_data.py @@ -4,6 +4,10 @@ import polars as pl COLUMNS = ['dbFlags', 'ownOp', 'year', 'desc', 'aircraft_category', 'r', 't'] +# Positional contract for every released ADS-B artifact. polars concatenates by +# position after .select(), so a divergent copy corrupts output without erroring. +FINAL_COLUMN_ORDER = ['time', 'icao', 'r', 't', 'dbFlags', 'ownOp', 'year', 'desc', 'aircraft_category'] + def compress_df_polars(df: pl.DataFrame, icao: str) -> pl.DataFrame: """Compress a single ICAO group to its most informative row using Polars.""" @@ -137,7 +141,7 @@ def load_parquet_part(part_id: int, date: str) -> pl.DataFrame: """Load a single parquet part file for a date. Args: - part_id: Part ID (e.g., 1, 2, 3) + part_id: Part ID (0-indexed, e.g. 0, 1, 2, 3) date: Date string in YYYY-MM-DD format Returns: @@ -164,7 +168,7 @@ def load_parquet_part(part_id: int, date: str) -> pl.DataFrame: print(f"Loading from parquet: {parquet_file}") df = pl.read_parquet( parquet_file, - columns=['time', 'icao', 'r', 't', 'dbFlags', 'ownOp', 'year', 'desc', 'aircraft_category'] + columns=FINAL_COLUMN_ORDER ) # Convert to timezone-naive datetime diff --git a/src/adsb/concat_parquet_to_final.py b/src/adsb/concat_parquet_to_final.py index 21b86e4..0016fdb 100644 --- a/src/adsb/concat_parquet_to_final.py +++ b/src/adsb/concat_parquet_to_final.py @@ -2,8 +2,11 @@ from pathlib import Path import polars as pl import argparse import os +import sys + +from src.adsb.compress_adsb_to_aircraft_data import FINAL_COLUMN_ORDER + OUTPUT_DIR = Path("./data/output") -CORRECT_ORDER_OF_COLUMNS = ["time", "icao", "r", "t", "dbFlags", "ownOp", "year", "desc", "aircraft_category"] def main(): parser = argparse.ArgumentParser(description="Concatenate compressed parquet files for a single day") @@ -17,13 +20,13 @@ def main(): parquet_files = sorted(date_dir.glob("*.parquet")) df = None if parquet_files: # TODO: This logic could be updated slightly. - print(f"No parquet files found in {date_dir}") + print(f"Found {len(parquet_files)} parquet part(s) in {date_dir}") frames = [pl.read_parquet(p) for p in parquet_files] df = pl.concat(frames, how="vertical", rechunk=True) df = df.sort(["time", "icao"]) - df = df.select(CORRECT_ORDER_OF_COLUMNS) + df = df.select(FINAL_COLUMN_ORDER) output_path = OUTPUT_DIR / f"openairframes_adsb_{args.date}.parquet" print(f"Writing combined parquet to {output_path} with {df.height} rows") @@ -32,6 +35,13 @@ def main(): csv_output_path = OUTPUT_DIR / f"openairframes_adsb_{args.date}.csv.gz" print(f"Writing combined csv.gz to {csv_output_path} with {df.height} rows") df.write_csv(csv_output_path, compression="gzip") + elif not args.concat_with_latest_csv: + # Nothing to merge and no release to fall back on: exiting 0 here would let the + # caller mistake "produced nothing" for "succeeded". + print(f"ERROR: No parquet files found in {date_dir} and --concat_with_latest_csv not set") + sys.exit(1) + else: + print(f"No parquet files found in {date_dir}; falling back to the latest released CSV") if args.concat_with_latest_csv: print("Loading latest CSV from GitHub releases to concatenate with...") @@ -50,15 +60,15 @@ def main(): print("Writing latest CSV directly without concatenation to avoid duplicates") os.makedirs(OUTPUT_DIR, exist_ok=True) final_csv_output_path = OUTPUT_DIR / f"openairframes_adsb_{csv_start_date}_{csv_end_date}.csv.gz" - df_latest_csv = df_latest_csv.select(CORRECT_ORDER_OF_COLUMNS) + df_latest_csv = df_latest_csv.select(FINAL_COLUMN_ORDER) df_latest_csv.write_csv(final_csv_output_path, compression="gzip") else: print(f"Concatenating latest CSV (through {csv_end_date}) with new data ({args.date})") # Ensure column order matches before concatenating - df_latest_csv = df_latest_csv.select(CORRECT_ORDER_OF_COLUMNS) + df_latest_csv = df_latest_csv.select(FINAL_COLUMN_ORDER) from src.adsb.compress_adsb_to_aircraft_data import concat_compressed_dfs df_final = concat_compressed_dfs(df_latest_csv, df) - df_final = df_final.select(CORRECT_ORDER_OF_COLUMNS) + df_final = df_final.select(FINAL_COLUMN_ORDER) final_csv_output_path = OUTPUT_DIR / f"openairframes_adsb_{csv_start_date}_{args.date}.csv.gz" df_final.write_csv(final_csv_output_path, compression="gzip") print(f"Final CSV written to {final_csv_output_path}") diff --git a/src/adsb/download_adsb_data_to_parquet.py b/src/adsb/download_adsb_data_to_parquet.py index d69f12f..5a4bb87 100644 --- a/src/adsb/download_adsb_data_to_parquet.py +++ b/src/adsb/download_adsb_data_to_parquet.py @@ -93,6 +93,19 @@ def _fetch_releases_from_repo(year: str, version_date: str) -> list: else: print(f"Giving up after {max_retries} attempts") return releases + except urllib.error.HTTPError as e: + # 404 means the repo/page does not exist. Retrying cannot change that, + # and 10 attempts x 5 min burns ~45 min of runner time to learn nothing. + if e.code == 404: + print(f"Not found (HTTP 404): {BASE_URL}?page={page} - not retrying") + return releases + print(f"Request exception (attempt {attempt}/{max_retries}): {e}") + if attempt < max_retries: + print(f"Waiting {retry_delay} seconds before retry") + time.sleep(retry_delay) + else: + print(f"Giving up after {max_retries} attempts") + return releases except Exception as e: print(f"Request exception (attempt {attempt}/{max_retries}): {e}") if attempt < max_retries: diff --git a/src/adsb/process_icao_chunk.py b/src/adsb/process_icao_chunk.py index 7d01feb..938858b 100644 --- a/src/adsb/process_icao_chunk.py +++ b/src/adsb/process_icao_chunk.py @@ -116,7 +116,7 @@ from pathlib import Path def main(): parser = argparse.ArgumentParser(description="Process a single archive part for a day") - parser.add_argument("--part-id", type=int, required=True, help="Part ID (1-indexed)") + parser.add_argument("--part-id", type=int, required=True, help="Part ID (0-indexed)") parser.add_argument("--date", type=str, required=True, help="Date in YYYY-MM-DD format") args = parser.parse_args() diff --git a/src/combine_historical_faa.py b/src/combine_historical_faa.py deleted file mode 100644 index c66fec3..0000000 --- a/src/combine_historical_faa.py +++ /dev/null @@ -1,14 +0,0 @@ -#unique_regulatory_id -# 1. read historoical and output -# 2. read sequentially - -# Instead of reading all csvs I can read just the latest release csv to get everything. - -from pathlib import Path - -base = Path("data/faa_releasable_historical") -for day_dir in sorted(base.glob("2024-02-*")): - master = day_dir / "Master.txt" - if master.exists(): - out_csv = master_txt_to_releasable_csv(master, out_dir="data/faa_releasable_historical_csv") - print(day_dir.name, "->", out_csv) \ No newline at end of file diff --git a/src/contributions/approve_submission.py b/src/contributions/approve_submission.py index bb421af..d5ae507 100644 --- a/src/contributions/approve_submission.py +++ b/src/contributions/approve_submission.py @@ -21,7 +21,7 @@ import urllib.request import urllib.error from datetime import datetime, timezone -from .schema import extract_json_from_issue_body, extract_contributor_name_from_issue_body, parse_and_validate, load_schema, SCHEMAS_DIR +from .schema import extract_json_from_issue_body, extract_contributor_name_from_issue_body, parse_and_validate, load_schema, get_schema_path from .contributor import ( generate_contributor_uuid, generate_submission_filename, @@ -72,9 +72,14 @@ def add_issue_comment(issue_number: int, body: str) -> None: github_api_request("POST", f"/issues/{issue_number}/comments", {"body": body}) -def get_default_branch_sha() -> str: - """Get the SHA of the default branch (main).""" - ref = github_api_request("GET", "/git/ref/heads/main") +def get_default_branch() -> str: + """Get the repository's default branch name.""" + return github_api_request("GET", "")["default_branch"] + + +def get_branch_sha(branch: str) -> str: + """Get the head SHA of a branch.""" + ref = github_api_request("GET", f"/git/ref/heads/{branch}") return ref["object"]["sha"] @@ -199,14 +204,14 @@ def process_submission( # Create branch branch_name = f"community-submission-{issue_number}" - default_sha = get_default_branch_sha() - create_branch(branch_name, default_sha) + base_branch = get_default_branch() + create_branch(branch_name, get_branch_sha(base_branch)) # Create file commit_message = f"Add community submission from @{author_username} (closes #{issue_number})" create_or_update_file(file_path, content_json, commit_message, branch_name) - # Update schema with any new tags (modifies v1 in place) + # Update schema with any new tags (rewrites the resolved schema version in place) schema_updated = False new_tags = [] try: @@ -232,7 +237,7 @@ def process_submission( schema_json = json.dumps(updated_schema, indent=2) + "\n" create_or_update_file( - "schemas/community_submission.v1.schema.json", + f"schemas/{get_schema_path().name}", schema_json, f"Update schema with new tags: {', '.join(new_tags)}", branch_name @@ -276,7 +281,7 @@ Closes #{issue_number} pr = create_pull_request( title=f"Community submission: {filename}", head=branch_name, - base="main", + base=base_branch, body=pr_body, ) diff --git a/src/contributions/regenerate_pr_schema.py b/src/contributions/regenerate_pr_schema.py index 5209e13..94eff97 100644 --- a/src/contributions/regenerate_pr_schema.py +++ b/src/contributions/regenerate_pr_schema.py @@ -20,7 +20,7 @@ from src.contributions.update_schema import ( check_for_new_tags, generate_updated_schema, ) -from src.contributions.schema import load_schema, SCHEMAS_DIR +from src.contributions.schema import load_schema, get_schema_path def main(): @@ -51,8 +51,8 @@ def main(): # Generate updated schema updated_schema = generate_updated_schema(current_schema, tag_registry) - # Write updated schema (in place) - schema_path = SCHEMAS_DIR / "community_submission.v1.schema.json" + # Write back to whichever version load_schema() resolved to + schema_path = get_schema_path() with open(schema_path, 'w') as f: json.dump(updated_schema, f, indent=2) f.write("\n") diff --git a/src/contributions/schema.py b/src/contributions/schema.py index bb29dcd..42fdd99 100644 --- a/src/contributions/schema.py +++ b/src/contributions/schema.py @@ -12,9 +12,6 @@ except ImportError: SCHEMAS_DIR = Path(__file__).parent.parent.parent / "schemas" -# For backwards compatibility -SCHEMA_PATH = SCHEMAS_DIR / "community_submission.v1.schema.json" - def get_latest_schema_version() -> int: """