From a7356162015b663cde25c3609ad3bed4ec69bbd8 Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Fri, 28 Aug 2026 23:57:27 -0400 Subject: [PATCH 01/18] docs: add NOTICE for redistributed data source terms - carry the two Government of Canada notices the open licence requires verbatim - record FAA public-domain provenance and unassessed third-party republished assets - LICENSE covers code only; release data carries its own conditions Generated-by: Claude Opus 5 --- NOTICE | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 NOTICE diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..22726d7 --- /dev/null +++ b/NOTICE @@ -0,0 +1,57 @@ +OpenAirframes — Data Source Notices +=================================== + +The LICENSE file covers the *code* in this repository. It does not cover the +*data* published in releases. Several upstream registries permit redistribution +only on condition that specific notices travel with the data. Those conditions +are reproduced below and must ship alongside any redistributed release asset. + +Removing or altering a notice in this file removes the permission that makes the +corresponding asset redistributable. + + +Transport Canada — Canadian Civil Aircraft Register (CCARCS) +------------------------------------------------------------ +Asset: openairframes_tc_*.csv +Source: https://wwwapps.tc.gc.ca/saf-sec-sur/2/ccarcs-riacc/download/ccarcsdb.zip + +The Government of Canada open licence requires both of the following notices, +verbatim, and requires that they reach the consumer together: + + Reproduced and distributed with the permission of the Government of Canada. + + This product has been produced by or for the OpenAirframes project and + includes data provided by the Government of Canada. The incorporation of + data sourced from the Government of Canada within this product shall not be + construed as constituting an endorsement by the Government of Canada of our + product. + +Owner mailing addresses published in the CCARCS export (street, city, postal +code, care-of) are dropped during ingestion and are not redistributed. + + +FAA — Releasable Aircraft Database +----------------------------------- +Asset: openairframes_faa_*.csv, ReleasableAircraft_*.zip +Source: https://registry.faa.gov/database/ReleasableAircraft.zip + +A work of the United States federal government, not subject to copyright +protection in the United States (17 U.S.C. § 105). No notice is required; the +source is credited here for provenance. + + +Other redistributed assets +--------------------------- +The following assets are republished from third parties whose terms have not +been assessed in this repository. They are listed for provenance only; nothing +here asserts a licence over them. + + openairframes_adsb_*.csv.gz derived from adsb.lol daily globe history + (github.com/adsblol), with registration data + from tar1090-db + basic-ac-db_*.json.gz ADS-B Exchange, downloads.adsbexchange.com + mictronics-db_*.zip Mictronics, www.mictronics.de + +Community submissions under community/ are contributed by their authors through +the repository's submission workflow and are published with the attribution each +contributor selected. From 2711b2d0f82292ed29c36feb76b074bed37d451b Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Fri, 28 Aug 2026 23:57:27 -0400 Subject: [PATCH 02/18] feat: build a daily Transport Canada aircraft register release - parse the headerless latin1 CCARCS export against its declared column layout - derive transponder_code_hex from the 24-bit Mode S binary, populated for all 34,913 rows - expand marks to C- and vintage CF- registrations and drop owner mailing addresses - mirror the FAA build: same concat-with-latest-release dedup and output conventions Generated-by: Claude Opus 5 --- src/create_daily_tc_release.py | 56 +++++++++++ src/derive_from_tc_ccarcs.py | 172 +++++++++++++++++++++++++++++++++ src/get_latest_release.py | 37 +++++++ 3 files changed, 265 insertions(+) create mode 100644 src/create_daily_tc_release.py create mode 100644 src/derive_from_tc_ccarcs.py diff --git a/src/create_daily_tc_release.py b/src/create_daily_tc_release.py new file mode 100644 index 0000000..912d83a --- /dev/null +++ b/src/create_daily_tc_release.py @@ -0,0 +1,56 @@ +from pathlib import Path +from datetime import datetime, timezone +import argparse + +parser = argparse.ArgumentParser(description="Create daily Transport Canada release") +parser.add_argument("--date", type=str, help="Date to process (YYYY-MM-DD format, default: today)") +args = parser.parse_args() + +if args.date: + date_str = args.date +else: + date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d") + +out_dir = Path("data/tc_ccarcs") +out_dir.mkdir(parents=True, exist_ok=True) +zip_name = f"ccarcsdb_{date_str}.zip" + +zip_path = out_dir / zip_name +if not zip_path.exists(): + url = "https://wwwapps.tc.gc.ca/saf-sec-sur/2/ccarcs-riacc/download/ccarcsdb.zip" + from urllib.request import Request, urlopen + + # CCARCS rejects default urllib agents. + req = Request( + url, + headers={ + "User-Agent": ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" + ) + }, + method="GET", + ) + + with urlopen(req, timeout=120) as r: + body = r.read() + zip_path.write_bytes(body) + +OUT_ROOT = Path("data/openairframes") +OUT_ROOT.mkdir(parents=True, exist_ok=True) +from derive_from_tc_ccarcs import convert_tc_ccarcs_to_df +# Row-fingerprint dedup is source-agnostic; reused rather than forked. +from derive_from_faa_master_txt import concat_faa_historical_df +from get_latest_release import get_latest_aircraft_tc_csv_df +df_new = convert_tc_ccarcs_to_df(zip_path, date_str) + +try: + df_base, start_date_str = get_latest_aircraft_tc_csv_df() + df_base = concat_faa_historical_df(df_base, df_new) + assert df_base['download_date'].is_monotonic_increasing, "download_date is not monotonic increasing" +except Exception as e: + print(f"No existing Transport Canada release found, using only new data: {e}") + df_base = df_new + start_date_str = date_str + +df_base.to_csv(OUT_ROOT / f"openairframes_tc_{start_date_str}_{date_str}.csv", index=False) diff --git a/src/derive_from_tc_ccarcs.py b/src/derive_from_tc_ccarcs.py new file mode 100644 index 0000000..31f68be --- /dev/null +++ b/src/derive_from_tc_ccarcs.py @@ -0,0 +1,172 @@ +from pathlib import Path +import csv +import io +import zipfile + +import pandas as pd + +from derive_from_faa_master_txt import normalize + +# CCARCS ships headerless, latin1, comma-delimited exports. Column names come from +# carslayout.txt in the same archive and must stay in file order. +CARSCURR_COLUMNS = [ + "MARK", "REGISTRATION_SUB_TYPE_E", "REGISTRATION_SUB_TYPE_F", "COMMON_NAME", + "MODEL_NAME", "MANUFACTURERS_SERIAL_NUMBER", "MANUFACTURER_SERIAL_COMPRESSED", + "ID_PLATE_MANUFACTURERS_NAME", "BASIS_FOR_REGISTRATION", "BASIS_FOR_REGISTRATION_F", + "AIRCRAFT_CATEGORY_E", "AIRCRAFT_CATEGORY_F", "DATE_OF_IMPORT", "ENGINE_MANUF", + "POWERGLIDER_FLAG", "ENGINE_CATEGORY_E", "ENGINE_CATEGORY_F", "NUMBER_OF_ENGINES", + "NUMBER_OF_SEATS", "AIR_WEIGHT_KILOS", "SALE_REPORTED", "ISSUE_DATE", + "EFFECTIVE_DATE", "INEFFECTIVE_DATE", "REGISTERED_PURPOSE_E", "REGISTERED_PURPOSE_F", + "FLIGHT_AUTHORITY_E", "FLIGHT_AUTHORITY_F", "MANUFACTURE_OR_ASSEMBLY", + "COUNTRY_MANUFACTURE_ASS_E", "COUNTRY_MANUFACTURE_ASS_F", "DATE_MANUFACTURE_ASSEMBLY", + "BASE_OF_OPERATIONS_CTRY_E", "BASE_OF_OPERATIONS_CTRY_F", "BASE_PROVINCE_OR_STATE_E", + "BASE_PROVINCE_OR_STATE_F", "CITY_AIRPORT", "TYPE_CERTIFICATE_NUMBER", + "REGISTRATION_AUTH_STATUS_E", "REGISTRATION_AUTH_STATUS_F", "MULTIPLE_OWNER_FLAG", + "MODIFIED_DATE", "MODE_S_TRANSPONDER_BINARY", "PHYSICAL_FILE_REGION_E", + "PHYSICAL_FILE_REGION_F", "EX_MILITARY_MARK", "TRIMMED_MARK", +] + +CARSOWNR_COLUMNS = [ + "MARK_LINK", "FULL_NAME", "TRADE_NAME", "STREET_NAME", "STREET_NAME2", "CITY", + "PROVINCE_OR_STATE_E", "PROVINCE_OR_STATE_F", "POSTAL_CODE", "COUNTRY_E", "COUNTRY_F", + "TYPE_OF_OWNER_E", "TYPE_OF_OWNER_F", "ACTIVE_FLAG", "CARE_OF", "REGION_E", "REGION_F", + "OWNER_NAME_OLD_FORMAT", "MAIL_RECIPIENT", "TRIMMED_MARK", +] + +# Owner mailing addresses are dropped rather than republished; see NOTICE. +OWNER_PII_COLUMNS = ["STREET_NAME", "STREET_NAME2", "CITY", "POSTAL_CODE", "CARE_OF"] + + +def _read_ccarcs_entry(zip_path: Path, entry: str, columns: list[str]) -> pd.DataFrame: + """Read one headerless CCARCS export into a DataFrame, dropping the Oracle footer. + + The export ends with a bare "N rows selected." line and a blank line; both are + narrower than the declared column count. Any *other* width mismatch is silent + field loss, so it raises instead. + """ + with zipfile.ZipFile(zip_path) as z: + text = z.read(entry).decode("latin1") + + rows = [] + ragged = 0 + for row in csv.reader(io.StringIO(text)): + if len(row) == len(columns): + rows.append([cell.strip() for cell in row]) + elif len(row) <= 1: + ragged += 1 # footer or trailing blank + else: + raise ValueError( + f"{entry}: row with {len(row)} fields, expected {len(columns)}" + ) + + if ragged > 2: + raise ValueError(f"{entry}: {ragged} ragged rows, expected at most 2") + + return pd.DataFrame(rows, columns=columns) + + +def tc_full_registration(mark: str) -> str: + """Expand a trimmed CCARCS mark into the full Canadian registration. + + Three-character marks are vintage CF- registrations; everything else takes the + modern C- prefix. + """ + mark = (mark or "").strip().upper() + if not mark: + return "" + return f"CF-{mark}" if len(mark) == 3 else f"C-{mark}" + + +def binary_to_hex(binary: str) -> str: + """Convert a 24-bit Mode S binary string to uppercase hex.""" + binary = (binary or "").strip() + if not binary or any(c not in "01" for c in binary): + return "" + return f"{int(binary, 2):06X}" + + +def _merge_owners(df_ownr: pd.DataFrame) -> pd.DataFrame: + """Collapse one row per registered party into one row per mark. + + A co-owned mark repeats with a different party each time; keeping only the mail + recipient would silently drop the rest. Each field is deduplicated independently + and blanks are skipped, so values are not index-parallel across columns. + """ + def join_unique(series: pd.Series) -> str: + seen = [] + for value in series: + value = (value or "").strip() + if value and value not in seen: + seen.append(value) + return ", ".join(seen) + + grouped = df_ownr.groupby("TRIMMED_MARK", sort=False).agg( + owner_name=("FULL_NAME", join_unique), + owner_province_or_state=("PROVINCE_OR_STATE_E", join_unique), + owner_country=("COUNTRY_E", join_unique), + owner_type=("TYPE_OF_OWNER_E", join_unique), + owner_party_count=("FULL_NAME", "size"), + ).reset_index() + + # A party row states its own type ("Individual"); that stops being true of the + # mark once several parties share it. + grouped.loc[grouped["owner_party_count"] > 1, "owner_type"] = "Co-owner" + return grouped + + +def convert_tc_ccarcs_to_df(zip_path: Path, date: str) -> pd.DataFrame: + """Build the OpenAirframes Transport Canada frame from a CCARCS zip.""" + df = _read_ccarcs_entry(zip_path, "carscurr.txt", CARSCURR_COLUMNS) + df_ownr = _read_ccarcs_entry(zip_path, "carsownr.txt", CARSOWNR_COLUMNS) + df_ownr = df_ownr.drop(columns=OWNER_PII_COLUMNS) + + df = df.merge(_merge_owners(df_ownr), on="TRIMMED_MARK", how="left") + + out = pd.DataFrame({ + "download_date": date, + "transponder_code_hex": df["MODE_S_TRANSPONDER_BINARY"].map(binary_to_hex), + "registration_number": df["TRIMMED_MARK"].map(tc_full_registration), + "mark": df["TRIMMED_MARK"], + "aircraft_manufacturer": df["COMMON_NAME"], + "aircraft_model": df["MODEL_NAME"], + "serial_number": df["MANUFACTURERS_SERIAL_NUMBER"], + "aircraft_category": df["AIRCRAFT_CATEGORY_E"], + "engine_manufacturer": df["ENGINE_MANUF"], + "engine_category": df["ENGINE_CATEGORY_E"], + "number_of_engines": df["NUMBER_OF_ENGINES"], + "number_of_seats": df["NUMBER_OF_SEATS"], + "max_weight_kilos": df["AIR_WEIGHT_KILOS"], + "registration_status": df["REGISTRATION_AUTH_STATUS_E"], + "registration_sub_type": df["REGISTRATION_SUB_TYPE_E"], + "basis_for_registration": df["BASIS_FOR_REGISTRATION"], + "registered_purpose": df["REGISTERED_PURPOSE_E"], + "flight_authority": df["FLIGHT_AUTHORITY_E"], + "type_certificate_number": df["TYPE_CERTIFICATE_NUMBER"], + "country_manufacture": df["COUNTRY_MANUFACTURE_ASS_E"], + "date_manufacture_assembly": df["DATE_MANUFACTURE_ASSEMBLY"], + "base_country": df["BASE_OF_OPERATIONS_CTRY_E"], + "base_province_or_state": df["BASE_PROVINCE_OR_STATE_E"], + "city_airport": df["CITY_AIRPORT"], + "ex_military_mark": df["EX_MILITARY_MARK"], + "multiple_owner_flag": df["MULTIPLE_OWNER_FLAG"], + "owner_name": df["owner_name"], + "owner_type": df["owner_type"], + "owner_province_or_state": df["owner_province_or_state"], + "owner_country": df["owner_country"], + "issue_date": df["ISSUE_DATE"], + "effective_date": df["EFFECTIVE_DATE"], + "ineffective_date": df["INEFFECTIVE_DATE"], + "modified_date": df["MODIFIED_DATE"], + }) + + out.insert(3, "openairframes_id", ( + normalize(out["aircraft_manufacturer"]) + + "|" + + normalize(out["aircraft_model"]) + + "|" + + normalize(out["serial_number"]) + )) + + out = out.fillna("") + out = out.replace("None", "") + return out diff --git a/src/get_latest_release.py b/src/get_latest_release.py index 27a2eca..d201f74 100644 --- a/src/get_latest_release.py +++ b/src/get_latest_release.py @@ -167,6 +167,43 @@ def get_latest_aircraft_faa_csv_df(): return df, date_str +def download_latest_aircraft_tc_csv( + output_dir: Path = Path("downloads"), + github_token: Optional[str] = None, + repo: str = REPO, +) -> Path: + """ + Download the latest openairframes_tc_*.csv file from the latest GitHub release. + + Args: + output_dir: Directory to save the downloaded file (default: "downloads") + github_token: Optional GitHub token for authentication + repo: GitHub repository in format "owner/repo" (default: REPO) + + Returns: + Path to the downloaded file + """ + output_dir = Path(output_dir) + assets = get_latest_release_assets(repo, github_token=github_token) + asset = pick_asset(assets, name_regex=r"^openairframes_tc_.*\.csv$") + saved_to = download_asset(asset, output_dir / asset.name, github_token=github_token) + print(f"Downloaded: {asset.name} ({asset.size} bytes) -> {saved_to}") + return saved_to + + +def get_latest_aircraft_tc_csv_df(): + csv_path = download_latest_aircraft_tc_csv() + import pandas as pd + df = pd.read_csv(csv_path, dtype=str) + df = df.fillna("") + # Filename pattern: openairframes_tc_{start_date}_{end_date}.csv + match = re.search(r"openairframes_tc_(\d{4}-\d{2}-\d{2})_", str(csv_path)) + if not match: + raise ValueError(f"Could not extract date from filename: {csv_path.name}") + + return df, match.group(1) + + def download_latest_aircraft_adsb_csv( output_dir: Path = Path("downloads"), github_token: Optional[str] = None, From 705a97690f4f65678e59a5c5cacb4c18f95fb545 Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Fri, 28 Aug 2026 23:57:27 -0400 Subject: [PATCH 03/18] docs: record the attribution and redistribution rules in AGENTS.md - attribution is a licence condition and NOTICE must travel with release assets - a public licence travels to this project; a bilateral permission does not - non-commercial-only registries are incompatible with the MIT-licensed releases Generated-by: Claude Opus 5 --- AGENTS.md | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b523458..5534525 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,8 +9,9 @@ Never fix this class of bug by escaping, sanitizing, or renaming the delimiter ## Run everything from the repo root -`src/create_daily_faa_release.py` must be invoked as a **script** (`python src/create_daily_faa_release.py`). -It uses bare sibling imports, so `-m src.create_daily_faa_release` raises `ModuleNotFoundError`. +`src/create_daily_faa_release.py` and `src/create_daily_tc_release.py` must be invoked as **scripts** +(`python src/create_daily_faa_release.py`). They use bare sibling imports, so `-m` raises +`ModuleNotFoundError`. Everything under `src/adsb/` and `src/contributions/` is the opposite — `python -m`, package-relative. Output paths are CWD-relative. @@ -42,6 +43,24 @@ matrices. Reason about the YAML statically. - HTTP 404 is terminal in the release fetch. Restoring the retry makes the Dec-31 next-year-repo probe stall ~45 minutes on a repo that does not exist yet. +## Attribution is a licence condition, not a courtesy + +`NOTICE` carries the terms that make redistributable sources redistributable, and it is uploaded as +a release asset so it travels with the data. Deleting or editing an entry removes the permission for +the corresponding asset. + +Transport Canada requires **both** its notices — reproduction and value-added — to reach the +consumer together. `NOTICE` must also survive the `create-release` sparse checkout; it is listed +there explicitly. + +Before adding any registry, check redistribution, not just access. A public licence (CC BY, an +open-government licence) travels to this project; a bilateral permission granted to a different +project does not. Non-commercial-only sources are incompatible with the MIT-licensed releases — +that rules out Taiwan, Estonia and Chile even though they are cleared for private use elsewhere. + +Owner mailing addresses in the CCARCS export are dropped during ingestion; only name, province and +country are published. + ## Fork and upstream `src/get_latest_release.py` pins `REPO = "PlaneQuery/openairframes"` on purpose: this fork reads From f5423724bc5f00585f497bf101246718c72c5fae Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Mon, 31 Aug 2026 20:40:04 -0400 Subject: [PATCH 04/18] fix: validate the CCARCS parse and correct owner aggregation - reject Mode S fields that are not 24 binary digits; a short field zero-padded into a plausible address belonging to a different aircraft - check the "N rows selected." footer against the parsed row count and enforce a row floor, so an upstream short export fails instead of publishing as a smaller register - prefer ACTIVE_FLAG "A" parties but fall back to all: 1,932 Registered marks carry only "I" rows, and those are the MAIL_RECIPIENT, so filtering on "A" alone drops real owners - count distinct owner names rather than rows, fixing 154 marks labelled Co-owner in error - match the spool footer by pattern instead of a brittle ragged-row count Generated-by: Claude Opus 5 --- src/derive_from_tc_ccarcs.py | 95 ++++++++++++++++++++++++++---------- 1 file changed, 70 insertions(+), 25 deletions(-) diff --git a/src/derive_from_tc_ccarcs.py b/src/derive_from_tc_ccarcs.py index 31f68be..d89d4c6 100644 --- a/src/derive_from_tc_ccarcs.py +++ b/src/derive_from_tc_ccarcs.py @@ -1,6 +1,7 @@ from pathlib import Path import csv import io +import re import zipfile import pandas as pd @@ -37,30 +38,49 @@ CARSOWNR_COLUMNS = [ OWNER_PII_COLUMNS = ["STREET_NAME", "STREET_NAME2", "CITY", "POSTAL_CODE", "CARE_OF"] -def _read_ccarcs_entry(zip_path: Path, entry: str, columns: list[str]) -> pd.DataFrame: - """Read one headerless CCARCS export into a DataFrame, dropping the Oracle footer. +FOOTER_RE = re.compile(r"\s*(\d+) rows selected\.\s*") - The export ends with a bare "N rows selected." line and a blank line; both are - narrower than the declared column count. Any *other* width mismatch is silent - field loss, so it raises instead. +# Canada's register is ~35k aircraft. Any parse yielding less than this means the +# export was truncated upstream, which must not be published as a real snapshot. +MIN_EXPECTED_ROWS = 1000 + + +def _read_ccarcs_entry(zip_path: Path, entry: str, columns: list[str]) -> pd.DataFrame: + """Read one headerless CCARCS export into a DataFrame. + + Raises: + ValueError: on any row whose width is neither the declared column count nor a + blank/footer line, on a missing or disagreeing "N rows selected." footer, or + on a row count below MIN_EXPECTED_ROWS. """ with zipfile.ZipFile(zip_path) as z: text = z.read(entry).decode("latin1") rows = [] - ragged = 0 - for row in csv.reader(io.StringIO(text)): + declared = None + # newline="" so a CRLF export does not leave \r on the final field of every row. + for row in csv.reader(io.StringIO(text, newline="")): if len(row) == len(columns): rows.append([cell.strip() for cell in row]) - elif len(row) <= 1: - ragged += 1 # footer or trailing blank - else: - raise ValueError( - f"{entry}: row with {len(row)} fields, expected {len(columns)}" - ) + continue + if not row or not any(cell.strip() for cell in row): + continue # trailing blank line + match = FOOTER_RE.fullmatch(row[0]) if len(row) == 1 else None + if match: + declared = int(match.group(1)) + continue + raise ValueError( + f"{entry}: row with {len(row)} fields, expected {len(columns)}: {row[:3]!r}" + ) - if ragged > 2: - raise ValueError(f"{entry}: {ragged} ragged rows, expected at most 2") + # The spool footer is a free checksum from the source; a short export is otherwise + # indistinguishable from a genuinely smaller register. + if declared is None: + raise ValueError(f"{entry}: no 'N rows selected.' footer; export is truncated") + if declared != len(rows): + raise ValueError(f"{entry}: footer declares {declared} rows, parsed {len(rows)}") + if len(rows) < MIN_EXPECTED_ROWS: + raise ValueError(f"{entry}: only {len(rows)} rows, expected >= {MIN_EXPECTED_ROWS}") return pd.DataFrame(rows, columns=columns) @@ -68,8 +88,9 @@ def _read_ccarcs_entry(zip_path: Path, entry: str, columns: list[str]) -> pd.Dat def tc_full_registration(mark: str) -> str: """Expand a trimmed CCARCS mark into the full Canadian registration. - Three-character marks are vintage CF- registrations; everything else takes the - modern C- prefix. + CCARCS stores the bare mark in both MARK and TRIMMED_MARK, so the prefix has to be + reconstructed: three-character marks are vintage CF- registrations, everything else + takes the modern C- prefix. Returns "" for a blank mark. """ mark = (mark or "").strip().upper() if not mark: @@ -78,20 +99,38 @@ def tc_full_registration(mark: str) -> str: def binary_to_hex(binary: str) -> str: - """Convert a 24-bit Mode S binary string to uppercase hex.""" + """Convert a 24-bit Mode S binary string to a 6-digit uppercase hex address. + + Returns "" for empty, non-binary, or non-24-bit input. Width is checked because + this column is the join key against ADS-B data: a short field would otherwise + zero-pad into a plausible address belonging to a different aircraft. + """ binary = (binary or "").strip() - if not binary or any(c not in "01" for c in binary): + if len(binary) != 24 or any(c not in "01" for c in binary): return "" return f"{int(binary, 2):06X}" def _merge_owners(df_ownr: pd.DataFrame) -> pd.DataFrame: - """Collapse one row per registered party into one row per mark. + """Collapse the active registered parties for each mark into a single row. A co-owned mark repeats with a different party each time; keeping only the mail - recipient would silently drop the rest. Each field is deduplicated independently - and blanks are skipped, so values are not index-parallel across columns. + recipient would silently drop the rest. + + Each field is deduplicated and blank-skipped independently, so the values are NOT + index-parallel: a mark with three owners can emit three names but one province. + Consumers must not split on ", " and zip the columns together. """ + # ACTIVE_FLAG is "A"/"I", but "I" does not mean "former owner": 1,932 currently + # Registered marks carry only "I" parties, and those rows are the MAIL_RECIPIENT. + # So prefer active parties where a mark has any, and fall back to all of them + # rather than publishing a registered aircraft with no owner at all. + active = df_ownr[df_ownr["ACTIVE_FLAG"].str.upper() == "A"] + marks_with_active = set(active["TRIMMED_MARK"]) + df_ownr = pd.concat([ + active, + df_ownr[~df_ownr["TRIMMED_MARK"].isin(marks_with_active)], + ]) def join_unique(series: pd.Series) -> str: seen = [] for value in series: @@ -100,16 +139,20 @@ def _merge_owners(df_ownr: pd.DataFrame) -> pd.DataFrame: seen.append(value) return ", ".join(seen) + def count_distinct(series: pd.Series) -> int: + return len({v.strip() for v in series if v and v.strip()}) + grouped = df_ownr.groupby("TRIMMED_MARK", sort=False).agg( owner_name=("FULL_NAME", join_unique), owner_province_or_state=("PROVINCE_OR_STATE_E", join_unique), owner_country=("COUNTRY_E", join_unique), owner_type=("TYPE_OF_OWNER_E", join_unique), - owner_party_count=("FULL_NAME", "size"), + owner_party_count=("FULL_NAME", count_distinct), ).reset_index() - # A party row states its own type ("Individual"); that stops being true of the - # mark once several parties share it. + # A party row states its own type ("Individual"); that stops being true of the mark + # once several parties share it. Counting distinct names rather than rows keeps this + # consistent with owner_name, which is also deduplicated. grouped.loc[grouped["owner_party_count"] > 1, "owner_type"] = "Co-owner" return grouped @@ -159,6 +202,8 @@ def convert_tc_ccarcs_to_df(zip_path: Path, date: str) -> pd.DataFrame: "modified_date": df["MODIFIED_DATE"], }) + # Position matches the FAA frame (after registration_number). Ordering is cosmetic: + # concat_faa_historical_df reindexes df_new to the base's columns before merging. out.insert(3, "openairframes_id", ( normalize(out["aircraft_manufacturer"]) + "|" From 9a1b828d3c5177e4dbcc1bd0995e5f0570c5853c Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Mon, 31 Aug 2026 20:40:04 -0400 Subject: [PATCH 05/18] fix: stop a failed Transport Canada build from erasing release history - fall back to a single-day rebuild only on FileNotFoundError; a rate limit, schema change or truncated download previously took the same path and republished one day as the whole dataset, which the next run then read back as its base - move the monotonic download_date assert out of the try so corruption cannot select the destructive branch - walk back through releases like the ADS-B reader, so one missing optional asset does not strand the accumulation - authenticate release reads and verify downloaded asset size - read the previous CSV with keep_default_na=False so literal NA values round-trip - write the download atomically and reject non-zip responses Generated-by: Claude Opus 5 --- src/create_daily_tc_release.py | 33 ++++++++++++++++++++------- src/get_latest_release.py | 41 ++++++++++++++++++++++++++++------ 2 files changed, 59 insertions(+), 15 deletions(-) diff --git a/src/create_daily_tc_release.py b/src/create_daily_tc_release.py index 912d83a..ecfc508 100644 --- a/src/create_daily_tc_release.py +++ b/src/create_daily_tc_release.py @@ -20,7 +20,8 @@ if not zip_path.exists(): url = "https://wwwapps.tc.gc.ca/saf-sec-sur/2/ccarcs-riacc/download/ccarcsdb.zip" from urllib.request import Request, urlopen - # CCARCS rejects default urllib agents. + # CCARCS 403s a default urllib agent. Any browser-like UA works; the exact + # version string is not load-bearing. req = Request( url, headers={ @@ -34,23 +35,39 @@ if not zip_path.exists(): with urlopen(req, timeout=120) as r: body = r.read() - zip_path.write_bytes(body) + # TC serves an HTML maintenance page with a 200, which would otherwise be cached + # under a .zip name and re-read on every later run. + if body[:2] != b"PK": + raise RuntimeError(f"{url} did not return a zip (got {body[:40]!r})") + tmp_path = zip_path.with_suffix(".part") + tmp_path.write_bytes(body) + tmp_path.replace(zip_path) OUT_ROOT = Path("data/openairframes") OUT_ROOT.mkdir(parents=True, exist_ok=True) from derive_from_tc_ccarcs import convert_tc_ccarcs_to_df -# Row-fingerprint dedup is source-agnostic; reused rather than forked. +# Named for FAA but column-agnostic: fingerprints every column except download_date. from derive_from_faa_master_txt import concat_faa_historical_df from get_latest_release import get_latest_aircraft_tc_csv_df df_new = convert_tc_ccarcs_to_df(zip_path, date_str) +# Only a genuine first run may rebuild from a single day. Every other failure -- a rate +# limit, a schema change, a truncated download -- must stop the run, because this file +# becomes tomorrow's base and silently republishing one day erases the whole history. try: df_base, start_date_str = get_latest_aircraft_tc_csv_df() - df_base = concat_faa_historical_df(df_base, df_new) - assert df_base['download_date'].is_monotonic_increasing, "download_date is not monotonic increasing" -except Exception as e: - print(f"No existing Transport Canada release found, using only new data: {e}") - df_base = df_new +except FileNotFoundError as e: + print(f"No existing Transport Canada release found, bootstrapping from today only: {e}") + df_base = None start_date_str = date_str +if df_base is not None: + missing = set(df_base.columns) ^ set(df_new.columns) + if missing: + raise SystemExit(f"Column set changed since the last release: {sorted(missing)}") + df_base = concat_faa_historical_df(df_base, df_new) + assert df_base['download_date'].is_monotonic_increasing, "download_date is not monotonic increasing" +else: + df_base = df_new + df_base.to_csv(OUT_ROOT / f"openairframes_tc_{start_date_str}_{date_str}.csv", index=False) diff --git a/src/get_latest_release.py b/src/get_latest_release.py index d201f74..0161d2e 100644 --- a/src/get_latest_release.py +++ b/src/get_latest_release.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import dataclass from pathlib import Path from typing import Iterable, Optional +import os import re import urllib.request import urllib.error @@ -184,19 +185,45 @@ def download_latest_aircraft_tc_csv( Path to the downloaded file """ output_dir = Path(output_dir) - assets = get_latest_release_assets(repo, github_token=github_token) - asset = pick_asset(assets, name_regex=r"^openairframes_tc_.*\.csv$") - saved_to = download_asset(asset, output_dir / asset.name, github_token=github_token) - print(f"Downloaded: {asset.name} ({asset.size} bytes) -> {saved_to}") - return saved_to + github_token = github_token or os.environ.get("GITHUB_TOKEN") + + # Walk back through releases rather than reading only `latest`. The TC asset is + # optional, so a single failed build publishes a release without it; anchoring on + # `latest` would then make the caller rebuild history from one day and republish + # that as the whole dataset. + for release in get_releases(repo, github_token=github_token, per_page=30): + assets = get_release_assets_from_release_data(release) + try: + asset = pick_asset(assets, name_regex=r"^openairframes_tc_.*\.csv$") + except FileNotFoundError: + continue + saved_to = download_asset(asset, output_dir / asset.name, github_token=github_token) + if asset.size and saved_to.stat().st_size != asset.size: + raise RuntimeError( + f"{asset.name}: downloaded {saved_to.stat().st_size} bytes, expected {asset.size}" + ) + print(f"Downloaded: {asset.name} ({asset.size} bytes) -> {saved_to}") + return saved_to + + raise FileNotFoundError( + "No release in the last 30 releases has an asset matching 'openairframes_tc_.*\\.csv$'" + ) def get_latest_aircraft_tc_csv_df(): + """Return (DataFrame, start_date_str) for the most recent published TC release. + + Raises FileNotFoundError when no recent release carries a TC asset, and ValueError + when the asset filename has no parseable start date. + """ csv_path = download_latest_aircraft_tc_csv() import pandas as pd - df = pd.read_csv(csv_path, dtype=str) + # keep_default_na=False: a literal "NA"/"N/A" in the source would otherwise read back + # as NaN -> "" while the fresh parse keeps the string, so the row fingerprints would + # never match and every affected record would re-append on every run. + df = pd.read_csv(csv_path, dtype=str, keep_default_na=False) df = df.fillna("") - # Filename pattern: openairframes_tc_{start_date}_{end_date}.csv + # Only the start date is taken; the end date is always the run's own date. match = re.search(r"openairframes_tc_(\d{4}-\d{2}-\d{2})_", str(csv_path)) if not match: raise ValueError(f"Could not extract date from filename: {csv_path.name}") From bd78cc5b2ce62feb14a19ce1f4c9905a6c2866e4 Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Mon, 31 Aug 2026 20:40:04 -0400 Subject: [PATCH 06/18] docs: correct licence and ownership claims in NOTICE and AGENTS.md - stop asserting US public-domain status for the derived FAA CSV; section 105 covers the government's own work, not this repository's derivative - name no licence instrument for Transport Canada, which does not publish one on the download page, rather than citing one that cannot be verified - state that MIT covers code only, resolving a contradiction with AGENTS.md - lead the source-eligibility rule with the bilateral-permission bar; Taiwan is OGDL licensed and excluded for that reason, not for commercial terms Generated-by: Claude Opus 5 --- AGENTS.md | 32 ++++++++++++++++++++------------ NOTICE | 21 ++++++++++++++------- 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 5534525..9745ffd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,22 +43,30 @@ matrices. Reason about the YAML statically. - HTTP 404 is terminal in the release fetch. Restoring the retry makes the Dec-31 next-year-repo probe stall ~45 minutes on a repo that does not exist yet. -## Attribution is a licence condition, not a courtesy +## Never edit or drop a `NOTICE` entry -`NOTICE` carries the terms that make redistributable sources redistributable, and it is uploaded as -a release asset so it travels with the data. Deleting or editing an entry removes the permission for -the corresponding asset. +Each entry is the permission that makes its asset redistributable; removing one removes the +permission. `NOTICE` is validated as a **required** release file and is listed in the +`create-release` sparse checkout — keep both. Transport Canada requires its two notices, +reproduction and value-added, to reach the consumer together. -Transport Canada requires **both** its notices — reproduction and value-added — to reach the -consumer together. `NOTICE` must also survive the `create-release` sparse checkout; it is listed -there explicitly. +`LICENSE` is MIT and covers **code only**. It makes no claim over released data, and neither may +you — an asset derived from a public-domain source is not itself public domain. -Before adding any registry, check redistribution, not just access. A public licence (CC BY, an -open-government licence) travels to this project; a bilateral permission granted to a different -project does not. Non-commercial-only sources are incompatible with the MIT-licensed releases — -that rules out Taiwan, Estonia and Chile even though they are cleared for private use elsewhere. +Before adding any registry, judge redistribution, not access, in this order: -Owner mailing addresses in the CCARCS export are dropped during ingestion; only name, province and +1. **A public licence travels; a bilateral permission does not.** Written permission granted to + another project or person is not a licence to this one. That alone disqualifies Taiwan, Estonia + and Chile, whatever their commercial terms say. +2. **Non-commercial-only conditions are a second, independent bar** — they conflict with how these + releases are consumed. Do not treat a source clearing this bar as cleared overall; rule 1 still + applies. Taiwan is licensed OGDL v1.0, an open licence: its restriction is bilateral, not licence-borne. + +CCARCS `ACTIVE_FLAG` does **not** mean "current owner": 1,932 currently-Registered marks carry only +`I` parties, and those rows are the `MAIL_RECIPIENT`. Prefer `A` parties where a mark has any, fall +back to all of them otherwise. Filtering on `A` alone publishes registered aircraft with no owner. + +Owner mailing addresses (street, city, postal code, care-of) are dropped; name, type, province and country are published. ## Fork and upstream diff --git a/NOTICE b/NOTICE index 22726d7..d038cd1 100644 --- a/NOTICE +++ b/NOTICE @@ -4,7 +4,8 @@ OpenAirframes — Data Source Notices The LICENSE file covers the *code* in this repository. It does not cover the *data* published in releases. Several upstream registries permit redistribution only on condition that specific notices travel with the data. Those conditions -are reproduced below and must ship alongside any redistributed release asset. +are reproduced below. This file is published as an asset on every release; anyone +redistributing a release asset further must carry the corresponding notice with it. Removing or altering a notice in this file removes the permission that makes the corresponding asset redistributable. @@ -15,8 +16,10 @@ Transport Canada — Canadian Civil Aircraft Register (CCARCS) Asset: openairframes_tc_*.csv Source: https://wwwapps.tc.gc.ca/saf-sec-sur/2/ccarcs-riacc/download/ccarcsdb.zip -The Government of Canada open licence requires both of the following notices, -verbatim, and requires that they reach the consumer together: +Redistribution is permitted under the Government of Canada terms recorded for this +dataset. They require both of the following notices, verbatim, and require that the +two reach the consumer together. The governing instrument is not published on the +CCARCS download page, so no licence name or URL is asserted here. Reproduced and distributed with the permission of the Government of Canada. @@ -32,12 +35,16 @@ code, care-of) are dropped during ingestion and are not redistributed. FAA — Releasable Aircraft Database ----------------------------------- -Asset: openairframes_faa_*.csv, ReleasableAircraft_*.zip Source: https://registry.faa.gov/database/ReleasableAircraft.zip -A work of the United States federal government, not subject to copyright -protection in the United States (17 U.S.C. § 105). No notice is required; the -source is credited here for provenance. +ReleasableAircraft_*.zip is redistributed unmodified. It is a work of the United +States federal government, not subject to copyright protection in the United +States (17 U.S.C. § 105). No notice is required; it is credited for provenance. + +openairframes_faa_*.csv is a derived product built by this repository — +normalized, joined, deduplicated, and extended with an identifier this project +defines. Section 105 disclaims copyright in the government's own work and says +nothing about a derivative, so no claim is made here about the CSV's status. Other redistributed assets From fce0b8d18cc7f3b0cfc53779b2007837e5f022c1 Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Mon, 31 Aug 2026 20:55:29 -0400 Subject: [PATCH 07/18] feat: redistribute Canadian owner addresses to match the FAA asset - publish street, city, postal code and care-of, which the FAA asset already carries as registrant_* for 99.7% of US registrants; dropping them here left one repository with two different postures on the same class of data - take the address from the single MAIL_RECIPIENT row rather than merging across parties, since a co-owned mark lists several people in different cities Generated-by: Claude Opus 5 --- AGENTS.md | 6 ++++-- NOTICE | 6 ++++-- src/derive_from_tc_ccarcs.py | 33 +++++++++++++++++++++++++++++---- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9745ffd..6ff20e8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,8 +66,10 @@ CCARCS `ACTIVE_FLAG` does **not** mean "current owner": 1,932 currently-Register `I` parties, and those rows are the `MAIL_RECIPIENT`. Prefer `A` parties where a mark has any, fall back to all of them otherwise. Filtering on `A` alone publishes registered aircraft with no owner. -Owner mailing addresses (street, city, postal code, care-of) are dropped; name, type, province and -country are published. +Owner mailing addresses **are** published, matching the `registrant_*` address the FAA asset already +carries — the two sources must not diverge on this. Addresses are per-party, so they come from the +single `MAIL_RECIPIENT == "Y"` row (exactly one per mark) and are never merged across co-owners; only +name, type, province and country are merged lists. ## Fork and upstream diff --git a/NOTICE b/NOTICE index d038cd1..fcfffde 100644 --- a/NOTICE +++ b/NOTICE @@ -29,8 +29,10 @@ CCARCS download page, so no licence name or URL is asserted here. construed as constituting an endorsement by the Government of Canada of our product. -Owner mailing addresses published in the CCARCS export (street, city, postal -code, care-of) are dropped during ingestion and are not redistributed. +Registered-owner mailing addresses are redistributed, matching the registrant +addresses the FAA asset already carries. Because CCARCS lists one row per party, +the published address is that of the single designated mail recipient rather than +a merge across co-owners. FAA — Releasable Aircraft Database diff --git a/src/derive_from_tc_ccarcs.py b/src/derive_from_tc_ccarcs.py index d89d4c6..1cc9f7d 100644 --- a/src/derive_from_tc_ccarcs.py +++ b/src/derive_from_tc_ccarcs.py @@ -34,8 +34,16 @@ CARSOWNR_COLUMNS = [ "OWNER_NAME_OLD_FORMAT", "MAIL_RECIPIENT", "TRIMMED_MARK", ] -# Owner mailing addresses are dropped rather than republished; see NOTICE. -OWNER_PII_COLUMNS = ["STREET_NAME", "STREET_NAME2", "CITY", "POSTAL_CODE", "CARE_OF"] +# Mailing address of the single designated recipient, matching the registrant_* address +# the FAA build already publishes. Addresses are per-party, so they are taken from the one +# MAIL_RECIPIENT row rather than merged across co-owners. +OWNER_ADDRESS_COLUMNS = { + "STREET_NAME": "owner_street_1", + "STREET_NAME2": "owner_street_2", + "CITY": "owner_city", + "POSTAL_CODE": "owner_postal_code", + "CARE_OF": "owner_care_of", +} FOOTER_RE = re.compile(r"\s*(\d+) rows selected\.\s*") @@ -125,6 +133,7 @@ def _merge_owners(df_ownr: pd.DataFrame) -> pd.DataFrame: # Registered marks carry only "I" parties, and those rows are the MAIL_RECIPIENT. # So prefer active parties where a mark has any, and fall back to all of them # rather than publishing a registered aircraft with no owner at all. + all_parties = df_ownr active = df_ownr[df_ownr["ACTIVE_FLAG"].str.upper() == "A"] marks_with_active = set(active["TRIMMED_MARK"]) df_ownr = pd.concat([ @@ -154,14 +163,25 @@ def _merge_owners(df_ownr: pd.DataFrame) -> pd.DataFrame: # once several parties share it. Counting distinct names rather than rows keeps this # consistent with owner_name, which is also deduplicated. grouped.loc[grouped["owner_party_count"] > 1, "owner_type"] = "Co-owner" - return grouped + + # Taken from the unfiltered frame: the designated recipient is the designated + # recipient even when its own party row is flagged inactive. + recipient = ( + all_parties[all_parties["MAIL_RECIPIENT"].str.upper() == "Y"] + .drop_duplicates(subset="TRIMMED_MARK", keep="first") + .rename(columns=OWNER_ADDRESS_COLUMNS) + ) + return grouped.merge( + recipient[["TRIMMED_MARK", *OWNER_ADDRESS_COLUMNS.values()]], + on="TRIMMED_MARK", + how="left", + ) def convert_tc_ccarcs_to_df(zip_path: Path, date: str) -> pd.DataFrame: """Build the OpenAirframes Transport Canada frame from a CCARCS zip.""" df = _read_ccarcs_entry(zip_path, "carscurr.txt", CARSCURR_COLUMNS) df_ownr = _read_ccarcs_entry(zip_path, "carsownr.txt", CARSOWNR_COLUMNS) - df_ownr = df_ownr.drop(columns=OWNER_PII_COLUMNS) df = df.merge(_merge_owners(df_ownr), on="TRIMMED_MARK", how="left") @@ -196,6 +216,11 @@ def convert_tc_ccarcs_to_df(zip_path: Path, date: str) -> pd.DataFrame: "owner_type": df["owner_type"], "owner_province_or_state": df["owner_province_or_state"], "owner_country": df["owner_country"], + "owner_care_of": df["owner_care_of"], + "owner_street_1": df["owner_street_1"], + "owner_street_2": df["owner_street_2"], + "owner_city": df["owner_city"], + "owner_postal_code": df["owner_postal_code"], "issue_date": df["ISSUE_DATE"], "effective_date": df["EFFECTIVE_DATE"], "ineffective_date": df["INEFFECTIVE_DATE"], From 1b6de19afb63d277c3da3f483a284acb30b182e2 Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Mon, 31 Aug 2026 21:12:22 -0400 Subject: [PATCH 08/18] refactor: name Transport Canada columns in the FAA vocabulary - registrant_* rather than owner_*, status rather than registration_status, so both registries describe the same concept with the same column name - registrant_zip_code carries the Canadian postal code: a union table needs one column per concept, not one per country's vocabulary - set source="TC", matching the discriminator the FAA frame already carries - raises the column names shared with the FAA frame from 9 to 21 Generated-by: Claude Opus 5 --- src/derive_from_tc_ccarcs.py | 50 +++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/src/derive_from_tc_ccarcs.py b/src/derive_from_tc_ccarcs.py index 1cc9f7d..510f2c5 100644 --- a/src/derive_from_tc_ccarcs.py +++ b/src/derive_from_tc_ccarcs.py @@ -37,12 +37,14 @@ CARSOWNR_COLUMNS = [ # Mailing address of the single designated recipient, matching the registrant_* address # the FAA build already publishes. Addresses are per-party, so they are taken from the one # MAIL_RECIPIENT row rather than merged across co-owners. +# registrant_zip_code holds the Canadian postal code: the name is the FAA's, and a union +# table needs one column per concept, not one per country's vocabulary. OWNER_ADDRESS_COLUMNS = { - "STREET_NAME": "owner_street_1", - "STREET_NAME2": "owner_street_2", - "CITY": "owner_city", - "POSTAL_CODE": "owner_postal_code", - "CARE_OF": "owner_care_of", + "STREET_NAME": "registrant_street_1", + "STREET_NAME2": "registrant_street_2", + "CITY": "registrant_city", + "POSTAL_CODE": "registrant_zip_code", + "CARE_OF": "registrant_care_of", } @@ -152,17 +154,17 @@ def _merge_owners(df_ownr: pd.DataFrame) -> pd.DataFrame: return len({v.strip() for v in series if v and v.strip()}) grouped = df_ownr.groupby("TRIMMED_MARK", sort=False).agg( - owner_name=("FULL_NAME", join_unique), - owner_province_or_state=("PROVINCE_OR_STATE_E", join_unique), - owner_country=("COUNTRY_E", join_unique), - owner_type=("TYPE_OF_OWNER_E", join_unique), - owner_party_count=("FULL_NAME", count_distinct), + registrant_name=("FULL_NAME", join_unique), + registrant_state=("PROVINCE_OR_STATE_E", join_unique), + registrant_country=("COUNTRY_E", join_unique), + registrant_type=("TYPE_OF_OWNER_E", join_unique), + registrant_party_count=("FULL_NAME", count_distinct), ).reset_index() # A party row states its own type ("Individual"); that stops being true of the mark # once several parties share it. Counting distinct names rather than rows keeps this # consistent with owner_name, which is also deduplicated. - grouped.loc[grouped["owner_party_count"] > 1, "owner_type"] = "Co-owner" + grouped.loc[grouped["registrant_party_count"] > 1, "registrant_type"] = "Co-owner" # Taken from the unfiltered frame: the designated recipient is the designated # recipient even when its own party row is flagged inactive. @@ -187,6 +189,8 @@ def convert_tc_ccarcs_to_df(zip_path: Path, date: str) -> pd.DataFrame: out = pd.DataFrame({ "download_date": date, + # The FAA frame already carries `source`; it is the union discriminator. + "source": "TC", "transponder_code_hex": df["MODE_S_TRANSPONDER_BINARY"].map(binary_to_hex), "registration_number": df["TRIMMED_MARK"].map(tc_full_registration), "mark": df["TRIMMED_MARK"], @@ -196,10 +200,10 @@ def convert_tc_ccarcs_to_df(zip_path: Path, date: str) -> pd.DataFrame: "aircraft_category": df["AIRCRAFT_CATEGORY_E"], "engine_manufacturer": df["ENGINE_MANUF"], "engine_category": df["ENGINE_CATEGORY_E"], - "number_of_engines": df["NUMBER_OF_ENGINES"], - "number_of_seats": df["NUMBER_OF_SEATS"], + "aircraft_number_of_engines": df["NUMBER_OF_ENGINES"], + "aircraft_number_of_seats": df["NUMBER_OF_SEATS"], "max_weight_kilos": df["AIR_WEIGHT_KILOS"], - "registration_status": df["REGISTRATION_AUTH_STATUS_E"], + "status": df["REGISTRATION_AUTH_STATUS_E"], "registration_sub_type": df["REGISTRATION_SUB_TYPE_E"], "basis_for_registration": df["BASIS_FOR_REGISTRATION"], "registered_purpose": df["REGISTERED_PURPOSE_E"], @@ -212,15 +216,15 @@ def convert_tc_ccarcs_to_df(zip_path: Path, date: str) -> pd.DataFrame: "city_airport": df["CITY_AIRPORT"], "ex_military_mark": df["EX_MILITARY_MARK"], "multiple_owner_flag": df["MULTIPLE_OWNER_FLAG"], - "owner_name": df["owner_name"], - "owner_type": df["owner_type"], - "owner_province_or_state": df["owner_province_or_state"], - "owner_country": df["owner_country"], - "owner_care_of": df["owner_care_of"], - "owner_street_1": df["owner_street_1"], - "owner_street_2": df["owner_street_2"], - "owner_city": df["owner_city"], - "owner_postal_code": df["owner_postal_code"], + "registrant_name": df["registrant_name"], + "registrant_type": df["registrant_type"], + "registrant_state": df["registrant_state"], + "registrant_country": df["registrant_country"], + "registrant_care_of": df["registrant_care_of"], + "registrant_street_1": df["registrant_street_1"], + "registrant_street_2": df["registrant_street_2"], + "registrant_city": df["registrant_city"], + "registrant_zip_code": df["registrant_zip_code"], "issue_date": df["ISSUE_DATE"], "effective_date": df["EFFECTIVE_DATE"], "ineffective_date": df["INEFFECTIVE_DATE"], From 2c5e8b962129858358565e1c9f6efb39b095a62b Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Mon, 31 Aug 2026 21:12:22 -0400 Subject: [PATCH 09/18] docs: cut AGENTS.md back to operative constraints - drop narrative rationale, restated rules and background a reader can grep for - one rule per line; the file is read by a model about to act, not by a person catching up - 151 lines to 85 Generated-by: Claude Opus 5 --- AGENTS.md | 190 ++++++++++++++++++------------------------------------ 1 file changed, 61 insertions(+), 129 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6ff20e8..1dc9c76 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,150 +1,82 @@ -## Never interpolate `${{ github.event.* }}` into a `run:` block +## Never interpolate `${{ github.event.* }}` or `${{ inputs.* }}` into a `run:` block -Actions substitutes `${{ }}` as raw text before the shell parses it, and issue bodies here are -public and unauthenticated. Pass untrusted values through `env:` and quote them. +Pass them via `env:` and quote the shell variable. A quoted heredoc does not help — the body can +contain the delimiter and close it early. Never fix this by escaping or renaming the delimiter. -A quoted heredoc delimiter does not save you: the body can *contain* the delimiter, close the -heredoc early, and execute every line after it. `validate-community-submission.yaml` had that shape. -Never fix this class of bug by escaping, sanitizing, or renaming the delimiter — move it to `env:`. +## Invocation -## Run everything from the repo root - -`src/create_daily_faa_release.py` and `src/create_daily_tc_release.py` must be invoked as **scripts** -(`python src/create_daily_faa_release.py`). They use bare sibling imports, so `-m` raises -`ModuleNotFoundError`. -Everything under `src/adsb/` and `src/contributions/` is the opposite — `python -m`, package-relative. - -Output paths are CWD-relative. +- `src/*.py` at the root of `src/` are scripts: `python src/create_daily_faa_release.py`. Bare + sibling imports, so `-m` raises `ModuleNotFoundError`. +- `src/adsb/*`, `src/contributions/*` are packages: `python -m`. +- Run from the repo root; output paths are CWD-relative. ## Verification -There is no test framework, linter, or packaging config. **Do not add one unprompted**, and do not -treat "nothing broke" as verification. +- No test framework, linter, or packaging config. Do not add one unprompted. +- Never `gh workflow run` to test a change — every dispatch pulls tens of GB. +- Never commit generated data. The product is a GitHub Release; jobs pass state as artifacts. +- ADS-B has no cheap end-to-end check. Exercise `compress_multi_icao_df` on a hand-built frame. -The ADS-B path has no cheap end-to-end check — one day of input is tens of GB. Exercise -`compress_multi_icao_df` / `compress_df_polars` directly against a small hand-built Polars frame. +## Release invariants -**Never `gh workflow run` to test a change.** Every dispatch pulls tens of GB and fans out over date -matrices. Reason about the YAML statically. +- `FINAL_COLUMN_ORDER` (`compress_adsb_to_aircraft_data.py`) is the only definition of the ADS-B + column contract. `pl.concat` matches by position after `.select()`; a forked copy corrupts the + release with no error. +- Empty string, never null, in every released frame. +- `openairframes_id` = `normalize(manufacturer)|normalize(model)|normalize(serial)`. Reuse + `derive_from_faa_master_txt.normalize()`. +- Each source's daily build reads its own previous release asset and appends. Falling back to a + single-day rebuild on anything but `FileNotFoundError` republishes one day as the whole dataset, + which the next run then reads back as its base. Keep the fallback narrow. +- Python `3.14` for FAA/community/vendor jobs, `3.12` for ADS-B. Match the surrounding job. -**Never commit generated data.** The product is a GitHub Release; jobs pass state as artifacts. +## Registry sources -## ADS-B invariants +- Judge **redistribution**, not access. A public licence travels to this project; a bilateral + permission granted to another project does not — that alone disqualifies Taiwan, Estonia, Chile. + Non-commercial-only terms are a separate, independent bar. +- `NOTICE` carries the terms that make each asset redistributable and is a required release file. + Never edit or drop an entry. Transport Canada requires both its notices together. +- `LICENSE` is MIT and covers code only. Claim nothing about released data. +- Owner/registrant mailing addresses are published for every registry. FAA and TC must not diverge. +- CCARCS `ACTIVE_FLAG` is not "current owner": 1,932 Registered marks carry only `I` parties, and + those rows are the `MAIL_RECIPIENT`. Prefer `A`, fall back to all. +- CCARCS addresses come from the single `MAIL_RECIPIENT == "Y"` row, never merged across co-owners. -- `FINAL_COLUMN_ORDER` (`compress_adsb_to_aircraft_data.py`) is the only definition of the released - column contract. `pl.concat` matches by **position** after `.select()`, and - `get_latest_release.get_latest_aircraft_adsb_csv_df` parses released CSVs against the same order — - so a forked copy corrupts the release with no error anywhere. -- `load_parquet_part()` deleting its source parquet is **deliberate**: the raw part is many GB and - the runner would otherwise exhaust disk. Do not defer the delete to make reruns easier; that raises - peak disk by the size of the part. -- A released row means "most informative observation for this ICAO on this UTC day" — non-empty - fields not a subset of another row's, tie-broken by signature frequency. It is not a registry record. -- HTTP 404 is terminal in the release fetch. Restoring the retry makes the Dec-31 next-year-repo - probe stall ~45 minutes on a repo that does not exist yet. +## ADS-B -## Never edit or drop a `NOTICE` entry - -Each entry is the permission that makes its asset redistributable; removing one removes the -permission. `NOTICE` is validated as a **required** release file and is listed in the -`create-release` sparse checkout — keep both. Transport Canada requires its two notices, -reproduction and value-added, to reach the consumer together. - -`LICENSE` is MIT and covers **code only**. It makes no claim over released data, and neither may -you — an asset derived from a public-domain source is not itself public domain. - -Before adding any registry, judge redistribution, not access, in this order: - -1. **A public licence travels; a bilateral permission does not.** Written permission granted to - another project or person is not a licence to this one. That alone disqualifies Taiwan, Estonia - and Chile, whatever their commercial terms say. -2. **Non-commercial-only conditions are a second, independent bar** — they conflict with how these - releases are consumed. Do not treat a source clearing this bar as cleared overall; rule 1 still - applies. Taiwan is licensed OGDL v1.0, an open licence: its restriction is bilateral, not licence-borne. - -CCARCS `ACTIVE_FLAG` does **not** mean "current owner": 1,932 currently-Registered marks carry only -`I` parties, and those rows are the `MAIL_RECIPIENT`. Prefer `A` parties where a mark has any, fall -back to all of them otherwise. Filtering on `A` alone publishes registered aircraft with no owner. - -Owner mailing addresses **are** published, matching the `registrant_*` address the FAA asset already -carries — the two sources must not diverge on this. Addresses are per-party, so they come from the -single `MAIL_RECIPIENT == "Y"` row (exactly one per mark) and are never merged across co-owners; only -name, type, province and country are merged lists. - -## Fork and upstream - -`src/get_latest_release.py` pins `REPO = "PlaneQuery/openairframes"` on purpose: this fork reads -**upstream's** releases wherever it runs. Do not repoint it at `github.repository` without being asked. - -Upstream develops on `develop` and PRs into `main`. The daily release **deletes the existing release -and tag** before recreating them. +- `load_parquet_part()` deleting its source parquet is deliberate — disk pressure. Do not defer it. +- A released row is the most informative observation for that ICAO on that UTC day, not a registry + record. +- HTTP 404 is terminal in the release fetch; restoring the retry stalls the Dec-31 probe ~45 min. ## Community submissions are automation-owned -Merging to `community/**` or `schemas/**` force-pushes every open `community`-labeled PR branch back -onto main. Anything you hand-edit on such a branch is destroyed on the next merge. +- Merging `community/**` or `schemas/**` force-pushes every open `community` PR branch onto main. + Hand edits there are destroyed. +- Never hand-author `community/` files — the filename encodes `sha256(content)[:8]`. +- A tag's JSON type is fixed by its first submission and enforced forever. Emergent from + `build_tag_type_registry` + `validate_submission`; written nowhere in the schema. +- Adding `community_submission.v2.schema.json` promotes it atomically across every reader and + writer. One-way door — only on request. -- Never hand-author files in `community/` — the filename encodes `sha256(content)[:8]`, so an edit - orphans the hash and duplicates on re-approval. -- Never invent or copy a `contributor_uuid`; it is derived from the GitHub user id. -- Do not reintroduce a hardcoded `"main"` or `v1` filename. Both are resolved at runtime now. +## Fork -**A tag's JSON type is fixed by its first-ever submission and enforced forever** — emergent from -`build_tag_type_registry` + `validate_submission`, and written nowhere in the schema. Retyping or -renaming an existing tag breaks every future contributor, not just the current one. +`get_latest_release.REPO` pins upstream `PlaneQuery/openairframes` on purpose. Do not repoint it. +Upstream develops on `develop`. The daily release deletes the existing release and tag first. -Dropping a `community_submission.v2.schema.json` into `schemas/` promotes it atomically across every -reader and writer. That is a one-way door for contributors — only on explicit request. +## Do not chase -## Conventions +- `process-historical-faa.yaml` is dead: missing `src/get_historical_faa.py`, + `scripts/concat_csvs.py`, and uses the disabled `::set-output`. +- `af-klm-fleet/package.json` → `npm run validate` has no `scripts/validate.js`. +- `af-klm-fleet/` and `community-routes/` are unwired; nothing in CI touches them. + `af-klm-fleet/README.md` is generated. -- **Empty string, not null**, everywhere in released frames. -- Reuse `derive_from_faa_master_txt.normalize()` for `openairframes_id`; never re-derive the format. -- Python `3.14` for FAA/community/vendor jobs, `3.12` for ADS-B jobs (pyarrow pin + multiprocessing). - Deliberate. Match the surrounding job; do not unify. +## Flag, do not silently fix -## References with no target — do not chase as regressions - -| Reference | Missing | -|---|---| -| `process-historical-faa.yaml` | `src/get_historical_faa.py`, `scripts/concat_csvs.py` | -| `af-klm-fleet/package.json` → `npm run validate` | `af-klm-fleet/scripts/validate.js` | - -`process-historical-faa.yaml` is dead, not stale — it also uses the disabled `::set-output`. -Repair-vs-delete is the owner's call; leave it alone unprompted. - -## `af-klm-fleet/` and `community-routes/` are unwired - -Nothing in CI touches either, and nothing consumes `community-routes/`. `af-klm-fleet/` is a vendored -project by a different author with its own license — its aircraft model is unrelated to -`schemas/community_submission.*`, so do not merge the two. Its `README.md` is generated by -`generate-readme.js`; hand edits are overwritten. - -## Warts left standing — flag, do not silently fix - -- `NUMBER_PARTS` is restated by the matrix and four hand-written upload steps in - `adsb-to-aircraft-for-day.yaml`; changing the constant alone silently drops data. YAML cannot loop - upload steps and one merged artifact would force every map job to download all parts, so any real - fix is a restructure. -- `MAX_WORKERS = OS_CPU_COUNT if OS_CPU_COUNT > 4 else 1` collapses to a single worker on a ≤4-core - runner, shrinking `files_per_batch` with it. Possibly intentional memory control — do not raise it - without measuring peak RSS on the target runner. -- `update-community-prs.yaml` runs `regenerate_pr_schema || true` then force-pushes, so a - regeneration failure ships anyway. Making it fatal leaves PRs un-rebased instead — a judgment call. -- `approve_submission.py` wraps its schema update in a bare `except Exception`, so a submission can - merge without its new tags reaching the schema. - -## Workflow authoring - -The user's global GitHub Actions rules apply. Existing workflows violate most of them. -**Do not bulk-remediate** — bring only the file you were asked to touch up to standard, and surface -the rest in chat. - -## External sources - -`registry.faa.gov` and ADS-B Exchange are **required** — the release fails without them. Mictronics is -**tolerated**: it retries, then the job continues without it. adsb.lol may simply not have published a -given day, in which case the previous CSV is re-released rather than failing. - -FAA refreshes at 05:30 UTC; the release cron fires at 06:00 UTC. That 30-minute margin is the reason -for the schedule. +- `NUMBER_PARTS` is restated by the matrix and four upload steps in `adsb-to-aircraft-for-day.yaml`. +- `MAX_WORKERS = ... if OS_CPU_COUNT > 4 else 1` collapses to one worker on a ≤4-core runner. +- `update-community-prs.yaml` runs `regenerate_pr_schema || true`, then force-pushes. +- `approve_submission.py` wraps its schema update in a bare `except Exception`. +- Existing workflows violate the global GHA rules. Fix only the file you were asked to touch. From 1bbaba909eff29b8732b2938a5da5cf80c80d708 Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Mon, 31 Aug 2026 21:15:11 -0400 Subject: [PATCH 10/18] feat: build each registry source on its own thread and join them - add a reusable registry-source workflow so every source runs in parallel rather than as a hand-written job; adding a registry becomes one matrix entry plus one script - add build_registry.py to align the per-source CSVs on the union of columns and emit a single table discriminated by the source column - reindex each frame to the union before concatenating, so a source missing a column yields an empty cell rather than a shifted row Generated-by: Claude Opus 5 --- .github/workflows/registry-source.yaml | 70 +++++++++++++++++ src/build_registry.py | 102 +++++++++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 .github/workflows/registry-source.yaml create mode 100644 src/build_registry.py diff --git a/.github/workflows/registry-source.yaml b/.github/workflows/registry-source.yaml new file mode 100644 index 0000000..027f8cf --- /dev/null +++ b/.github/workflows/registry-source.yaml @@ -0,0 +1,70 @@ +name: registry-source + +# One registry source, on its own thread. Called once per entry in the caller's matrix, so +# adding a registry is a matrix entry plus src/create_daily__release.py — no new job. + +on: + workflow_call: + inputs: + source: + description: 'Source id; must match src/create_daily__release.py' + required: true + type: string + date: + description: 'Date to process (YYYY-MM-DD, default: today UTC)' + required: false + type: string + python-version: + required: false + type: string + default: '3.14' + required: + description: 'Fail the thread when this source cannot be built' + required: false + type: boolean + default: false + +jobs: + build: + runs-on: ubuntu-24.04-arm + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: ${{ inputs.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Build ${{ inputs.source }} registry + continue-on-error: ${{ !inputs.required }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SOURCE: ${{ inputs.source }} + RUN_DATE: ${{ inputs.date }} + run: | + script="src/create_daily_${SOURCE}_release.py" + if [ ! -f "$script" ]; then + echo "::error title=Unknown registry source::$script does not exist" + exit 1 + fi + python "$script" ${RUN_DATE:+--date "$RUN_DATE"} + ls -lah data/openairframes + + - name: Upload ${{ inputs.source }} registry + uses: actions/upload-artifact@v4 + with: + name: registry-${{ inputs.source }} + path: | + data/openairframes/openairframes_${{ inputs.source }}_*.csv + data/faa_releasable/ReleasableAircraft_*.zip + retention-days: 1 + if-no-files-found: ignore diff --git a/src/build_registry.py b/src/build_registry.py new file mode 100644 index 0000000..6d2be58 --- /dev/null +++ b/src/build_registry.py @@ -0,0 +1,102 @@ +"""Join the per-source registry CSVs into one union table. + +Every source publishes its own `openairframes__{start}_{end}.csv` on its own thread. +This reads whatever landed, aligns them on the union of columns, and writes a single +`openairframes_registry_{start}_{end}.csv` discriminated by the `source` column. + +Adding a registry means adding a source to the workflow matrix; nothing here changes. + +Usage: + python src/build_registry.py --input-dir artifacts/registry --date 2026-08-31 +""" +from datetime import datetime, timezone +from pathlib import Path +import argparse +import re +import sys + +import pandas as pd + +# Sources that are registries. Community and ADS-B are published separately: they are +# observations and contributions, not registration records, and do not share this schema. +FILENAME_RE = re.compile(r"openairframes_(?P[a-z]+)_(?P\d{4}-\d{2}-\d{2})_(?P\d{4}-\d{2}-\d{2})\.csv$") +EXCLUDED_SOURCES = {"community", "adsb", "registry"} + +# Identifier columns lead the union so the table is usable without reading 70 headers. +LEADING_COLUMNS = [ + "download_date", + "source", + "transponder_code_hex", + "registration_number", + "openairframes_id", +] + + +def discover(input_dir: Path) -> list[tuple[str, str, str, Path]]: + """Return (source, start, end, path) for each per-source registry CSV found.""" + found = [] + for path in sorted(input_dir.rglob("openairframes_*.csv")): + match = FILENAME_RE.search(path.name) + if not match: + continue + source = match.group("source") + if source in EXCLUDED_SOURCES: + continue + found.append((source, match.group("start"), match.group("end"), path)) + return found + + +def build(input_dir: Path, date_str: str) -> tuple[pd.DataFrame, str, str]: + parts = discover(input_dir) + if not parts: + raise SystemExit(f"No per-source registry CSVs found under {input_dir}") + + frames = [] + for source, _, _, path in parts: + # keep_default_na=False so a literal "NA" survives the round trip unchanged. + df = pd.read_csv(path, dtype=str, keep_default_na=False) + if "source" not in df.columns: + raise SystemExit(f"{path.name}: no source column; cannot discriminate rows") + actual = set(df["source"].unique()) + if len(actual) != 1: + raise SystemExit(f"{path.name}: expected one source value, found {sorted(actual)}") + print(f" {source}: {len(df)} rows, {len(df.columns)} columns from {path.name}") + frames.append(df) + + columns = list(dict.fromkeys(c for df in frames for c in df.columns)) + ordered = [c for c in LEADING_COLUMNS if c in columns] + ordered += [c for c in columns if c not in ordered] + + # reindex rather than concat directly: a source missing a column must yield an empty + # cell, never a shifted row. + df_union = pd.concat([df.reindex(columns=ordered) for df in frames], ignore_index=True) + df_union = df_union.fillna("") + + start = min(p[1] for p in parts) + end = max(p[2] for p in parts + [("", "", date_str, Path())]) + return df_union, start, end + + +def main() -> None: + parser = argparse.ArgumentParser(description="Join per-source registry CSVs into one table") + parser.add_argument("--input-dir", default="artifacts/registry", help="Directory to search") + parser.add_argument("--output-dir", default="data/openairframes", help="Where to write") + parser.add_argument("--date", help="Run date (YYYY-MM-DD, default: today UTC)") + args = parser.parse_args() + + date_str = args.date or datetime.now(timezone.utc).strftime("%Y-%m-%d") + df, start, end = build(Path(args.input_dir), date_str) + + out_dir = Path(args.output_dir) + out_dir.mkdir(parents=True, exist_ok=True) + out_path = out_dir / f"openairframes_registry_{start}_{end}.csv" + df.to_csv(out_path, index=False) + + print(f"Wrote {out_path}: {len(df)} rows, {len(df.columns)} columns") + print(" rows per source:") + for source, count in df["source"].value_counts().items(): + print(f" {source}: {count}") + + +if __name__ == "__main__": + main() From b025de6f3320b99031c6427834ef90b7f93e24b5 Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Mon, 31 Aug 2026 21:15:11 -0400 Subject: [PATCH 11/18] ci: publish a single joined registry asset - replace the hand-written build-faa job with a matrixed call per source, faa required and tc tolerated, joined once every thread has finished - publish openairframes_registry_*.csv and keep openairframes_faa_*.csv during transition - validate the joined registry and NOTICE as required release files - report missing optional assets as a workflow annotation rather than a plain echo Generated-by: Claude Opus 5 --- .../openairframes-daily-release.yaml | 76 ++++++++++++++----- 1 file changed, 59 insertions(+), 17 deletions(-) diff --git a/.github/workflows/openairframes-daily-release.yaml b/.github/workflows/openairframes-daily-release.yaml index ca9003d..a8c2f68 100644 --- a/.github/workflows/openairframes-daily-release.yaml +++ b/.github/workflows/openairframes-daily-release.yaml @@ -42,14 +42,32 @@ jobs: ref: 'develop' }); - build-faa: - runs-on: ubuntu-24.04-arm + # One thread per registry. Adding a source is one matrix entry plus + # src/create_daily__release.py; join-registry picks it up by artifact pattern. + build-registry-source: if: github.event_name != 'schedule' + strategy: + fail-fast: false + matrix: + include: + - source: faa + required: true + - source: tc + required: false + uses: ./.github/workflows/registry-source.yaml + with: + source: ${{ matrix.source }} + date: ${{ inputs.date }} + required: ${{ matrix.required }} + + join-registry: + needs: build-registry-source + if: always() && github.event_name != 'schedule' + runs-on: ubuntu-24.04-arm + timeout-minutes: 20 steps: - name: Checkout uses: actions/checkout@v6 - with: - fetch-depth: 0 - name: Setup Python uses: actions/setup-python@v6 @@ -61,20 +79,29 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt - - name: Run FAA release script - run: | - python src/create_daily_faa_release.py ${{ inputs.date && format('--date {0}', inputs.date) || '' }} - ls -lah data/faa_releasable - ls -lah data/openairframes + - name: Download every source thread + uses: actions/download-artifact@v4 + with: + pattern: registry-* + path: artifacts/registry + merge-multiple: true - - name: Upload FAA artifacts + - name: Join sources into one registry + env: + RUN_DATE: ${{ inputs.date }} + run: | + python src/build_registry.py --input-dir artifacts/registry ${RUN_DATE:+--date "$RUN_DATE"} + + - name: Upload registry uses: actions/upload-artifact@v4 with: - name: faa-release + name: registry-union path: | - data/openairframes/openairframes_faa_*.csv - data/faa_releasable/ReleasableAircraft_*.zip + data/openairframes/openairframes_registry_*.csv + artifacts/registry/openairframes_faa_*.csv + artifacts/registry/ReleasableAircraft_*.zip retention-days: 1 + if-no-files-found: error resolve-dates: runs-on: ubuntu-latest @@ -233,7 +260,7 @@ jobs: create-release: runs-on: ubuntu-latest - needs: [resolve-dates, build-faa, adsb-to-aircraft, adsb-reduce, build-community, build-adsbexchange-json, build-mictronics-db] + needs: [resolve-dates, join-registry, adsb-to-aircraft, adsb-reduce, build-community, build-adsbexchange-json, build-mictronics-db] if: github.event_name != 'schedule' && !cancelled() steps: - name: Check ADS-B workflow status @@ -246,12 +273,13 @@ jobs: with: sparse-checkout: | .github + NOTICE sparse-checkout-cone-mode: false - - name: Download FAA artifacts + - name: Download joined registry uses: actions/download-artifact@v5 with: - name: faa-release + name: registry-union path: artifacts/faa - name: Download ADS-B artifacts @@ -311,6 +339,7 @@ jobs: # Find files from artifacts using find (handles nested structures) CSV_FILE_FAA=$(find artifacts/faa -name "openairframes_faa_*.csv" -type f 2>/dev/null | head -1) + CSV_FILE_REGISTRY=$(find artifacts/faa -name "openairframes_registry_*.csv" -type f 2>/dev/null | head -1) # Prefer concatenated file (with date range) over single-day file CSV_FILE_ADSB=$(find artifacts/adsb -name "openairframes_adsb_*_*.csv.gz" -type f 2>/dev/null | head -1) if [ -z "$CSV_FILE_ADSB" ]; then @@ -332,6 +361,14 @@ jobs: if [ -z "$JSON_FILE_ADSBX" ] || [ ! -f "$JSON_FILE_ADSBX" ]; then MISSING_FILES="$MISSING_FILES ADSBX_JSON" fi + if [ -z "$CSV_FILE_REGISTRY" ] || [ ! -f "$CSV_FILE_REGISTRY" ]; then + MISSING_FILES="$MISSING_FILES REGISTRY_CSV" + fi + # NOTICE carries the terms that make each asset redistributable. Shipping data + # without it removes the permission, so it is required rather than optional. + if [ ! -f NOTICE ]; then + MISSING_FILES="$MISSING_FILES NOTICE" + fi # Optional files - warn but don't fail OPTIONAL_MISSING="" @@ -369,12 +406,14 @@ jobs: fi if [ -n "$OPTIONAL_MISSING" ]; then - echo "WARNING: Optional files missing:$OPTIONAL_MISSING (will continue without them)" + echo "::warning title=Missing optional release assets::$OPTIONAL_MISSING" fi echo "date=$DATE" >> "$GITHUB_OUTPUT" echo "tag=$TAG" >> "$GITHUB_OUTPUT" echo "csv_file_faa=$CSV_FILE_FAA" >> "$GITHUB_OUTPUT" + echo "csv_file_registry=$CSV_FILE_REGISTRY" >> "$GITHUB_OUTPUT" + echo "csv_basename_registry=$(basename "$CSV_FILE_REGISTRY")" >> "$GITHUB_OUTPUT" echo "csv_basename_faa=$CSV_BASENAME_FAA" >> "$GITHUB_OUTPUT" echo "csv_file_adsb=$CSV_FILE_ADSB" >> "$GITHUB_OUTPUT" echo "csv_basename_adsb=$CSV_BASENAME_ADSB" >> "$GITHUB_OUTPUT" @@ -413,6 +452,7 @@ jobs: Automated daily snapshot generated at 06:00 UTC for ${{ steps.meta.outputs.date }}. Assets: + - ${{ steps.meta.outputs.csv_basename_registry }} - ${{ steps.meta.outputs.csv_basename_faa }} ${{ steps.meta.outputs.csv_basename_adsb && format('- {0}', steps.meta.outputs.csv_basename_adsb) || '' }} - ${{ steps.meta.outputs.csv_basename_community }} @@ -420,7 +460,9 @@ jobs: - ${{ steps.meta.outputs.json_basename_adsbx }} ${{ steps.meta.outputs.zip_basename_mictronics && format('- {0}', steps.meta.outputs.zip_basename_mictronics) || '' }} files: | + ${{ steps.meta.outputs.csv_file_registry }} ${{ steps.meta.outputs.csv_file_faa }} + NOTICE ${{ steps.meta.outputs.csv_file_adsb }} ${{ steps.meta.outputs.csv_file_community }} ${{ steps.meta.outputs.zip_file }} From 10062956a43c78ea5bc56ca6ab9a3b731476dd66 Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Thu, 10 Sep 2026 20:53:53 -0400 Subject: [PATCH 12/18] ci: close the gaps that turned source failures into green runs - key the reusable workflow's concurrency on the source: one shared group made the matrix legs cancel each other, defeating the parallelism - resolve action versions at write time on the jobs this change adds (checkout v7, setup-python v7, upload-artifact v7, download-artifact v8) and add the missing workflow concurrency block and job-level permissions - stage each leg's outputs into one directory so every artifact has the same root; two search paths moved the root to the common ancestor for some legs only, and the FAA CSV and zip then matched nothing downstream - fail a leg that produced no CSV, and split the unknown-source guard out of the tolerated step so a matrix typo goes red - write continue-on-error as an explicit comparison rather than relying on ! coercion - gate the join on success rather than always, and rename its artifact so the download pattern cannot re-ingest it on a re-run - delete the previous release only when it exists, so a 403 stops the run instead of leaving two assets that the next run reads as an ambiguous base Generated-by: Claude Opus 5 --- .../openairframes-daily-release.yaml | 34 ++++++++--- .github/workflows/registry-source.yaml | 56 ++++++++++++++----- 2 files changed, 67 insertions(+), 23 deletions(-) diff --git a/.github/workflows/openairframes-daily-release.yaml b/.github/workflows/openairframes-daily-release.yaml index a8c2f68..bb40281 100644 --- a/.github/workflows/openairframes-daily-release.yaml +++ b/.github/workflows/openairframes-daily-release.yaml @@ -15,6 +15,10 @@ permissions: contents: write actions: write +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + jobs: trigger-releases: runs-on: ubuntu-latest @@ -62,17 +66,23 @@ jobs: join-registry: needs: build-registry-source - if: always() && github.event_name != 'schedule' + # No always(): a tolerated source fails its step without failing its leg, so this only + # blocks when a required source could not be built. + if: github.event_name != 'schedule' runs-on: ubuntu-24.04-arm timeout-minutes: 20 + permissions: + contents: read steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Setup Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: "3.14" + cache: 'pip' + cache-dependency-path: requirements.txt - name: Install dependencies run: | @@ -80,7 +90,7 @@ jobs: pip install -r requirements.txt - name: Download every source thread - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: pattern: registry-* path: artifacts/registry @@ -93,9 +103,9 @@ jobs: python src/build_registry.py --input-dir artifacts/registry ${RUN_DATE:+--date "$RUN_DATE"} - name: Upload registry - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: - name: registry-union + name: union-registry path: | data/openairframes/openairframes_registry_*.csv artifacts/registry/openairframes_faa_*.csv @@ -279,7 +289,7 @@ jobs: - name: Download joined registry uses: actions/download-artifact@v5 with: - name: registry-union + name: union-registry path: artifacts/faa - name: Download ADS-B artifacts @@ -438,7 +448,14 @@ jobs: - name: Delete existing release if exists run: | echo "Attempting to delete release: ${{ steps.meta.outputs.tag }}" - gh release delete "${{ steps.meta.outputs.tag }}" --yes --cleanup-tag || echo "No existing release to delete" + # `|| echo` here would swallow a 403 or a partial delete, leaving yesterday's + # asset attached alongside today's; the next run then matches two and rebuilds + # the dataset from a single day. + if gh release view "${{ steps.meta.outputs.tag }}" >/dev/null 2>&1; then + gh release delete "${{ steps.meta.outputs.tag }}" --yes --cleanup-tag + else + echo "No existing release to delete" + fi env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -452,6 +469,7 @@ jobs: Automated daily snapshot generated at 06:00 UTC for ${{ steps.meta.outputs.date }}. Assets: + - NOTICE (source terms; required for redistribution) - ${{ steps.meta.outputs.csv_basename_registry }} - ${{ steps.meta.outputs.csv_basename_faa }} ${{ steps.meta.outputs.csv_basename_adsb && format('- {0}', steps.meta.outputs.csv_basename_adsb) || '' }} diff --git a/.github/workflows/registry-source.yaml b/.github/workflows/registry-source.yaml index 027f8cf..0fdd3db 100644 --- a/.github/workflows/registry-source.yaml +++ b/.github/workflows/registry-source.yaml @@ -24,47 +24,73 @@ on: type: boolean default: false +# Keyed on the source: without it every matrix leg shares one group and the legs cancel +# each other, which is the opposite of running them in parallel. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.source }} + cancel-in-progress: false + jobs: build: runs-on: ubuntu-24.04-arm timeout-minutes: 30 + permissions: + contents: read steps: - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 + uses: actions/checkout@v7 - name: Setup Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ inputs.python-version }} + cache: 'pip' + cache-dependency-path: requirements.txt - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt - - name: Build ${{ inputs.source }} registry - continue-on-error: ${{ !inputs.required }} + # Deliberately outside the tolerated step below: a typo in the matrix is a config + # error, and must go red even for an optional source. + - name: Check the source has a build script env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SOURCE: ${{ inputs.source }} - RUN_DATE: ${{ inputs.date }} run: | script="src/create_daily_${SOURCE}_release.py" if [ ! -f "$script" ]; then echo "::error title=Unknown registry source::$script does not exist" exit 1 fi - python "$script" ${RUN_DATE:+--date "$RUN_DATE"} - ls -lah data/openairframes + + - name: Build ${{ inputs.source }} registry + continue-on-error: ${{ inputs.required == false }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SOURCE: ${{ inputs.source }} + RUN_DATE: ${{ inputs.date }} + run: | + python "src/create_daily_${SOURCE}_release.py" ${RUN_DATE:+--date "$RUN_DATE"} + # Stage into one directory so every leg's artifact has the same root; a second + # search path would move the root to the common ancestor for some legs only. + shopt -s nullglob + built=(data/openairframes/openairframes_"${SOURCE}"_*.csv) + if [ ${#built[@]} -ne 1 ]; then + echo "::error title=${SOURCE} produced no registry CSV::expected one openairframes_${SOURCE}_*.csv, found ${#built[@]}" + exit 1 + fi + # Stage into one directory so every leg's artifact has the same root; a second + # search path would move the root to the common ancestor for some legs only. + mkdir -p data/registry-out + cp "${built[@]}" data/registry-out/ + cp data/faa_releasable/ReleasableAircraft_*.zip data/registry-out/ 2>/dev/null || true + ls -lah data/registry-out - name: Upload ${{ inputs.source }} registry - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: registry-${{ inputs.source }} - path: | - data/openairframes/openairframes_${{ inputs.source }}_*.csv - data/faa_releasable/ReleasableAircraft_*.zip + path: data/registry-out retention-days: 1 - if-no-files-found: ignore + if-no-files-found: error From f78fa1c3ee43e01ad34179f024aa324f718a3bb6 Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Thu, 10 Sep 2026 20:53:54 -0400 Subject: [PATCH 13/18] fix: make registry join failures visible instead of silent - accept source ids containing digits and underscores; the previous pattern dropped openairframes_uk_caa_*.csv with no output, contradicting the documented promise that adding a source needs no change here - reject a file whose source column disagrees with its filename, which would otherwise merge one registry into the union under another's label - refuse duplicate source files, skip an empty optional source rather than aborting, and log every skipped file and the column names shared across sources - correct the MIN_EXPECTED_ROWS comment, which described a 35k floor for a 1000 value Generated-by: Claude Opus 5 --- src/build_registry.py | 36 +++++++++++++++++++++++++++++++----- src/derive_from_tc_ccarcs.py | 6 ++++-- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/build_registry.py b/src/build_registry.py index 6d2be58..f3e582c 100644 --- a/src/build_registry.py +++ b/src/build_registry.py @@ -19,7 +19,10 @@ import pandas as pd # Sources that are registries. Community and ADS-B are published separately: they are # observations and contributions, not registration records, and do not share this schema. -FILENAME_RE = re.compile(r"openairframes_(?P[a-z]+)_(?P\d{4}-\d{2}-\d{2})_(?P\d{4}-\d{2}-\d{2})\.csv$") +FILENAME_RE = re.compile( + r"\Aopenairframes_(?P[a-z0-9_]+?)_" + r"(?P\d{4}-\d{2}-\d{2})_(?P\d{4}-\d{2}-\d{2})\.csv\Z" +) EXCLUDED_SOURCES = {"community", "adsb", "registry"} # Identifier columns lead the union so the table is usable without reading 70 headers. @@ -36,11 +39,13 @@ def discover(input_dir: Path) -> list[tuple[str, str, str, Path]]: """Return (source, start, end, path) for each per-source registry CSV found.""" found = [] for path in sorted(input_dir.rglob("openairframes_*.csv")): - match = FILENAME_RE.search(path.name) + match = FILENAME_RE.match(path.name) if not match: + print(f" SKIP {path.name}: does not match {FILENAME_RE.pattern}") continue source = match.group("source") if source in EXCLUDED_SOURCES: + print(f" SKIP {path.name}: {source!r} is published as its own asset") continue found.append((source, match.group("start"), match.group("end"), path)) return found @@ -51,18 +56,37 @@ def build(input_dir: Path, date_str: str) -> tuple[pd.DataFrame, str, str]: if not parts: raise SystemExit(f"No per-source registry CSVs found under {input_dir}") + seen = [p[0] for p in parts] + duplicated = {s for s in seen if seen.count(s) > 1} + if duplicated: + raise SystemExit(f"More than one file claims source {sorted(duplicated)}") + frames = [] for source, _, _, path in parts: # keep_default_na=False so a literal "NA" survives the round trip unchanged. df = pd.read_csv(path, dtype=str, keep_default_na=False) if "source" not in df.columns: raise SystemExit(f"{path.name}: no source column; cannot discriminate rows") - actual = set(df["source"].unique()) - if len(actual) != 1: - raise SystemExit(f"{path.name}: expected one source value, found {sorted(actual)}") + if df.empty: + print(f" {source}: empty, skipping") + continue + actual = {v.strip().lower() for v in df["source"].unique()} + if actual != {source.lower()}: + # A file whose rows disagree with its name would duplicate another source into + # the union under the wrong label. + raise SystemExit( + f"{path.name}: filename says {source!r}, rows say {sorted(actual)}" + ) print(f" {source}: {len(df)} rows, {len(df.columns)} columns from {path.name}") frames.append(df) + if not frames: + raise SystemExit("Every discovered source was empty; refusing to publish an empty registry") + + if len(frames) > 1: + shared = set.intersection(*(set(f.columns) for f in frames)) - set(LEADING_COLUMNS) + print(f" columns shared across sources ({len(shared)}): {sorted(shared)}") + columns = list(dict.fromkeys(c for df in frames for c in df.columns)) ordered = [c for c in LEADING_COLUMNS if c in columns] ordered += [c for c in columns if c not in ordered] @@ -72,6 +96,8 @@ def build(input_dir: Path, date_str: str) -> tuple[pd.DataFrame, str, str]: df_union = pd.concat([df.reindex(columns=ordered) for df in frames], ignore_index=True) df_union = df_union.fillna("") + # Earliest start across sources, not per-source coverage: a source added today still + # carries the oldest source's start date in the filename. start = min(p[1] for p in parts) end = max(p[2] for p in parts + [("", "", date_str, Path())]) return df_union, start, end diff --git a/src/derive_from_tc_ccarcs.py b/src/derive_from_tc_ccarcs.py index 510f2c5..5bf3c45 100644 --- a/src/derive_from_tc_ccarcs.py +++ b/src/derive_from_tc_ccarcs.py @@ -50,8 +50,10 @@ OWNER_ADDRESS_COLUMNS = { FOOTER_RE = re.compile(r"\s*(\d+) rows selected\.\s*") -# Canada's register is ~35k aircraft. Any parse yielding less than this means the -# export was truncated upstream, which must not be published as a real snapshot. +# Floor, not an expectation: Canada's register is ~35k aircraft and carsownr is larger +# still, so 1000 only catches a grossly truncated export. The footer row-count check above +# is what actually validates the parse; this guards the case where the footer agrees with a +# near-empty body. MIN_EXPECTED_ROWS = 1000 From e3c8a0e242a2dffbb898b5e9ede05b7dcaf6a2a3 Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Thu, 10 Sep 2026 20:53:54 -0400 Subject: [PATCH 14/18] fix: stop a transient error erasing the FAA release history - narrow the fallback to FileNotFoundError and move the monotonic assert out of the try: a rate limit, parse error or corrupt download previously rebuilt three years of registry from a single day and republished it as the whole dataset - authenticate release reads and walk back through releases, matching the Transport Canada reader; the FAA path was unauthenticated at 60 requests an hour on shared runner IPs - verify downloaded asset size This is what the matrix's required: true flag on faa claims to protect, so the flag was advertising a guarantee the script could not keep. Generated-by: Claude Opus 5 --- src/create_daily_faa_release.py | 17 ++++++++++++----- src/get_latest_release.py | 32 +++++++++++++++++++++++--------- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/src/create_daily_faa_release.py b/src/create_daily_faa_release.py index 4e7adfd..b668518 100644 --- a/src/create_daily_faa_release.py +++ b/src/create_daily_faa_release.py @@ -37,13 +37,20 @@ from derive_from_faa_master_txt import convert_faa_master_txt_to_df, concat_faa_ from get_latest_release import get_latest_aircraft_faa_csv_df df_new = convert_faa_master_txt_to_df(zip_path, date_str) +# Only a genuine first run may rebuild from a single day. A rate limit, a parse error or a +# non-monotonic download_date must stop the run: this file becomes tomorrow's base, so +# silently republishing one day erases the accumulated history. try: df_base, start_date_str = get_latest_aircraft_faa_csv_df() - df_base = concat_faa_historical_df(df_base, df_new) - assert df_base['download_date'].is_monotonic_increasing, "download_date is not monotonic increasing" -except Exception as e: - print(f"No existing FAA release found, using only new data: {e}") - df_base = df_new +except FileNotFoundError as e: + print(f"No existing FAA release found, bootstrapping from today only: {e}") + df_base = None start_date_str = date_str +if df_base is not None: + df_base = concat_faa_historical_df(df_base, df_new) + assert df_base['download_date'].is_monotonic_increasing, "download_date is not monotonic increasing" +else: + df_base = df_new + df_base.to_csv(OUT_ROOT / f"openairframes_faa_{start_date_str}_{date_str}.csv", index=False) \ No newline at end of file diff --git a/src/get_latest_release.py b/src/get_latest_release.py index 0161d2e..e2738ff 100644 --- a/src/get_latest_release.py +++ b/src/get_latest_release.py @@ -139,15 +139,29 @@ def download_latest_aircraft_csv( Path to the downloaded file """ output_dir = Path(output_dir) - assets = get_latest_release_assets(repo, github_token=github_token) - try: - asset = pick_asset(assets, name_regex=r"^openairframes_faa_.*\.csv$") - except FileNotFoundError: - # Fallback to old naming pattern - asset = pick_asset(assets, name_regex=r"^openairframes_\d{4}-\d{2}-\d{2}_.*\.csv$") - saved_to = download_asset(asset, output_dir / asset.name, github_token=github_token) - print(f"Downloaded: {asset.name} ({asset.size} bytes) -> {saved_to}") - return saved_to + github_token = github_token or os.environ.get("GITHUB_TOKEN") + + for release in get_releases(repo, github_token=github_token, per_page=30): + assets = get_release_assets_from_release_data(release) + try: + asset = pick_asset(assets, name_regex=r"^openairframes_faa_.*\.csv$") + except FileNotFoundError: + try: + # Fallback to old naming pattern + asset = pick_asset(assets, name_regex=r"^openairframes_\d{4}-\d{2}-\d{2}_.*\.csv$") + except FileNotFoundError: + continue + saved_to = download_asset(asset, output_dir / asset.name, github_token=github_token) + if asset.size and saved_to.stat().st_size != asset.size: + raise RuntimeError( + f"{asset.name}: downloaded {saved_to.stat().st_size} bytes, expected {asset.size}" + ) + print(f"Downloaded: {asset.name} ({asset.size} bytes) -> {saved_to}") + return saved_to + + raise FileNotFoundError( + "No release in the last 30 releases has an asset matching 'openairframes_faa_.*\\.csv$'" + ) def get_latest_aircraft_faa_csv_df(): csv_path = download_latest_aircraft_csv() From 4c4e7052ba9aabd1ee5e1f7cf352aeab5ae9c04c Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Thu, 10 Sep 2026 20:53:54 -0400 Subject: [PATCH 15/18] docs: describe the release assets the daily build actually publishes - add the joined registry, and note that the FAA CSV it supersedes still ships - document basic-ac-db.json.gz and mictronics-db.zip, which have been published for months without appearing in the README - point at NOTICE for the redistribution terms that travel with the data Generated-by: Claude Opus 5 --- README.md | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8c8eb94..b98a250 100644 --- a/README.md +++ b/README.md @@ -26,13 +26,29 @@ df = pd.read_csv(url) df ``` ![](docs/images/df_adsb_example_0.png) -- **openairframes_faa.csv** - All [FAA registration data](https://www.faa.gov/licenses_certificates/aircraft_certification/aircraft_registry/releasable_aircraft_download) from 2023-08-16 to present (~260 MB) +- **openairframes_registry.csv** + Every national registry in one table, one row per registration record, with a `source` + column naming the registry it came from. Currently the FAA (United States) and Transport + Canada. Identifier columns (`transponder_code_hex`, `registration_number`, + `openairframes_id`) lead the table and are populated for every source. +- **openairframes_faa.csv** + All [FAA registration data](https://www.faa.gov/licenses_certificates/aircraft_certification/aircraft_registry/releasable_aircraft_download) from 2023-08-16 to present (~275 MB). + Superseded by `openairframes_registry.csv`; still published so existing consumers keep working. - **ReleasableAircraft_{date}.zip** A daily snapshot of the FAA database, which updates at **05:30 UTC** +- **basic-ac-db.json.gz** + [ADS-B Exchange](https://www.adsbexchange.com/) basic aircraft database, republished unmodified. + +- **mictronics-db.zip** + [Mictronics](https://www.mictronics.de/aircraft-database/) aircraft database, republished + unmodified. Best effort — the release ships without it when the source is unavailable. + +Redistribution terms for the underlying sources travel with the release in **NOTICE**. Some +registries permit redistribution only on condition that specific notices reach you with the data. + --- ## For Contributors From 9ec3c2ca480b6248e13c9b4662c524f5ffd0039e Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Fri, 11 Sep 2026 11:06:51 -0400 Subject: [PATCH 16/18] fix: publish every per-source registry CSV, not just the joined one Each source accumulates by reading its own previous release asset, but only the joined registry and the FAA CSV were published. Transport Canada could therefore never find a prior asset and would have rebuilt from a single day on every run, permanently. - stage the joined CSV and every per-source asset into one flat directory before upload, so the artifact has a single root and nothing is lost to the least-common-ancestor rule - carry the per-source CSVs into the release by glob, so a new source needs no edit here Generated-by: Claude Opus 5 --- .../openairframes-daily-release.yaml | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/.github/workflows/openairframes-daily-release.yaml b/.github/workflows/openairframes-daily-release.yaml index bb40281..ea09758 100644 --- a/.github/workflows/openairframes-daily-release.yaml +++ b/.github/workflows/openairframes-daily-release.yaml @@ -58,11 +58,15 @@ jobs: required: true - source: tc required: false + # Transport Canada has never published an asset. Remove this once the first + # run has succeeded: it exists to permit day one, not to paper over an outage. + allow_bootstrap: true uses: ./.github/workflows/registry-source.yaml with: source: ${{ matrix.source }} date: ${{ inputs.date }} required: ${{ matrix.required }} + allow_bootstrap: ${{ matrix.allow_bootstrap == true }} join-registry: needs: build-registry-source @@ -102,14 +106,20 @@ jobs: run: | python src/build_registry.py --input-dir artifacts/registry ${RUN_DATE:+--date "$RUN_DATE"} + - name: Stage release assets + run: | + mkdir -p data/release-out + # Every per-source CSV must ship: each source reads its OWN previous asset to + # accumulate, so one that is never published can never be anything but day one. + cp artifacts/registry/* data/release-out/ + cp data/openairframes/openairframes_registry_*.csv data/release-out/ + ls -lah data/release-out + - name: Upload registry uses: actions/upload-artifact@v7 with: name: union-registry - path: | - data/openairframes/openairframes_registry_*.csv - artifacts/registry/openairframes_faa_*.csv - artifacts/registry/ReleasableAircraft_*.zip + path: data/release-out retention-days: 1 if-no-files-found: error @@ -350,6 +360,11 @@ jobs: # Find files from artifacts using find (handles nested structures) CSV_FILE_FAA=$(find artifacts/faa -name "openairframes_faa_*.csv" -type f 2>/dev/null | head -1) CSV_FILE_REGISTRY=$(find artifacts/faa -name "openairframes_registry_*.csv" -type f 2>/dev/null | head -1) + # Every per-source registry CSV, whatever sources the matrix ran. + SOURCE_CSVS=$(find artifacts/faa -name "openairframes_*.csv" -type f 2>/dev/null \ + | grep -v "openairframes_registry_" | grep -v "openairframes_community_" | sort) + echo "Per-source registry CSVs found:" + echo "$SOURCE_CSVS" # Prefer concatenated file (with date range) over single-day file CSV_FILE_ADSB=$(find artifacts/adsb -name "openairframes_adsb_*_*.csv.gz" -type f 2>/dev/null | head -1) if [ -z "$CSV_FILE_ADSB" ]; then @@ -423,6 +438,11 @@ jobs: echo "tag=$TAG" >> "$GITHUB_OUTPUT" echo "csv_file_faa=$CSV_FILE_FAA" >> "$GITHUB_OUTPUT" echo "csv_file_registry=$CSV_FILE_REGISTRY" >> "$GITHUB_OUTPUT" + { + echo "source_csvs<> "$GITHUB_OUTPUT" echo "csv_basename_registry=$(basename "$CSV_FILE_REGISTRY")" >> "$GITHUB_OUTPUT" echo "csv_basename_faa=$CSV_BASENAME_FAA" >> "$GITHUB_OUTPUT" echo "csv_file_adsb=$CSV_FILE_ADSB" >> "$GITHUB_OUTPUT" @@ -479,7 +499,7 @@ jobs: ${{ steps.meta.outputs.zip_basename_mictronics && format('- {0}', steps.meta.outputs.zip_basename_mictronics) || '' }} files: | ${{ steps.meta.outputs.csv_file_registry }} - ${{ steps.meta.outputs.csv_file_faa }} + ${{ steps.meta.outputs.source_csvs }} NOTICE ${{ steps.meta.outputs.csv_file_adsb }} ${{ steps.meta.outputs.csv_file_community }} From 0a0a2855f3d20c5f9eb7b49d86fe158a2f2c102f Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Fri, 11 Sep 2026 11:06:51 -0400 Subject: [PATCH 17/18] fix: require an explicit opt-in before rebuilding a source from one day FileNotFoundError means no recent release carries the asset, which is not the same as the source never having published: a rate limit or a run of releases missing the asset reaches the same branch and would erase the accumulated history. - add --allow-bootstrap; without it a missing asset is now a hard failure - plumb it through the reusable workflow, and set it for Transport Canada only, which has genuinely never published - let a tolerated source that produced nothing upload nothing, rather than failing its leg and blocking the join a required source depends on Generated-by: Claude Opus 5 --- .github/workflows/registry-source.yaml | 12 ++++++++++-- src/create_daily_faa_release.py | 12 +++++++++++- src/create_daily_tc_release.py | 12 +++++++++++- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/.github/workflows/registry-source.yaml b/.github/workflows/registry-source.yaml index 0fdd3db..d39fe46 100644 --- a/.github/workflows/registry-source.yaml +++ b/.github/workflows/registry-source.yaml @@ -23,6 +23,11 @@ on: required: false type: boolean default: false + allow_bootstrap: + description: 'Onboarding only: permit a single-day rebuild when no asset has ever been published' + required: false + type: boolean + default: false # Keyed on the source: without it every matrix leg shares one group and the legs cancel # each other, which is the opposite of running them in parallel. @@ -70,8 +75,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SOURCE: ${{ inputs.source }} RUN_DATE: ${{ inputs.date }} + BOOTSTRAP: ${{ inputs.allow_bootstrap && '1' || '' }} run: | - python "src/create_daily_${SOURCE}_release.py" ${RUN_DATE:+--date "$RUN_DATE"} + python "src/create_daily_${SOURCE}_release.py" ${RUN_DATE:+--date "$RUN_DATE"} ${BOOTSTRAP:+--allow-bootstrap} # Stage into one directory so every leg's artifact has the same root; a second # search path would move the root to the common ancestor for some legs only. shopt -s nullglob @@ -93,4 +99,6 @@ jobs: name: registry-${{ inputs.source }} path: data/registry-out retention-days: 1 - if-no-files-found: error + # A tolerated source that failed has nothing to upload; only a required + # source missing its artifact is an error. + if-no-files-found: ${{ inputs.required && 'error' || 'ignore' }} diff --git a/src/create_daily_faa_release.py b/src/create_daily_faa_release.py index b668518..8e5e49f 100644 --- a/src/create_daily_faa_release.py +++ b/src/create_daily_faa_release.py @@ -4,6 +4,10 @@ import argparse parser = argparse.ArgumentParser(description="Create daily FAA release") parser.add_argument("--date", type=str, help="Date to process (YYYY-MM-DD format, default: today)") +parser.add_argument("--allow-bootstrap", action="store_true", + help="Permit rebuilding from a single day when no published asset is found. " + "Onboarding only: a missing asset is otherwise indistinguishable from a " + "transient outage, and rebuilding would erase the accumulated history.") args = parser.parse_args() if args.date: @@ -43,7 +47,13 @@ df_new = convert_faa_master_txt_to_df(zip_path, date_str) try: df_base, start_date_str = get_latest_aircraft_faa_csv_df() except FileNotFoundError as e: - print(f"No existing FAA release found, bootstrapping from today only: {e}") + if not args.allow_bootstrap: + raise SystemExit( + f"No published FAA asset found: {e}\n" + "This is indistinguishable from a transient outage, and rebuilding from one day " + "would erase the accumulated history. Pass --allow-bootstrap when onboarding." + ) from None + print(f"Bootstrapping FAA from today only (--allow-bootstrap): {e}") df_base = None start_date_str = date_str diff --git a/src/create_daily_tc_release.py b/src/create_daily_tc_release.py index ecfc508..ba7ae3b 100644 --- a/src/create_daily_tc_release.py +++ b/src/create_daily_tc_release.py @@ -4,6 +4,10 @@ import argparse parser = argparse.ArgumentParser(description="Create daily Transport Canada release") parser.add_argument("--date", type=str, help="Date to process (YYYY-MM-DD format, default: today)") +parser.add_argument("--allow-bootstrap", action="store_true", + help="Permit rebuilding from a single day when no published asset is found. " + "Onboarding only: a missing asset is otherwise indistinguishable from a " + "transient outage, and rebuilding would erase the accumulated history.") args = parser.parse_args() if args.date: @@ -57,7 +61,13 @@ df_new = convert_tc_ccarcs_to_df(zip_path, date_str) try: df_base, start_date_str = get_latest_aircraft_tc_csv_df() except FileNotFoundError as e: - print(f"No existing Transport Canada release found, bootstrapping from today only: {e}") + if not args.allow_bootstrap: + raise SystemExit( + f"No published Transport Canada asset found: {e}\n" + "This is indistinguishable from a transient outage, and rebuilding from one day " + "would erase the accumulated history. Pass --allow-bootstrap when onboarding." + ) from None + print(f"Bootstrapping Transport Canada from today only (--allow-bootstrap): {e}") df_base = None start_date_str = date_str From 766b575ff86a4bbf67ec5ed9857ad82c2f39dfa5 Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Fri, 11 Sep 2026 11:15:47 -0400 Subject: [PATCH 18/18] fix: make bootstrapping a deliberate dispatch and list every asset released Self-review of the two preceding commits. - the release body hardcoded one FAA basename while files: uploads every source by glob, so each release attached an asset it did not list; basenames are now derived the same way - allow_bootstrap was a standing matrix key on tc that nothing forced anyone to remove, so an outage longer than the release walk-back would have rebuilt from one day with a green run; it is now a workflow_dispatch input naming the one source permitted to bootstrap - the per-source glob excluded prefixes by substring while build_registry.py matches the source token exactly, so a future source named registry_* would have been dropped here and kept there; both now anchor on the same filename shape - README lists the Transport Canada asset, which the previous commit began publishing Generated-by: Claude Opus 5 --- .../openairframes-daily-release.yaml | 20 +++++++++++++------ README.md | 5 +++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/openairframes-daily-release.yaml b/.github/workflows/openairframes-daily-release.yaml index ea09758..7dcb5ce 100644 --- a/.github/workflows/openairframes-daily-release.yaml +++ b/.github/workflows/openairframes-daily-release.yaml @@ -10,6 +10,10 @@ on: description: 'Date to process (YYYY-MM-DD format, default: yesterday)' required: false type: string + bootstrap_source: + description: 'Onboarding only: the one source id permitted to rebuild from a single day' + required: false + type: string permissions: contents: write @@ -58,15 +62,14 @@ jobs: required: true - source: tc required: false - # Transport Canada has never published an asset. Remove this once the first - # run has succeeded: it exists to permit day one, not to paper over an outage. - allow_bootstrap: true uses: ./.github/workflows/registry-source.yaml with: source: ${{ matrix.source }} date: ${{ inputs.date }} required: ${{ matrix.required }} - allow_bootstrap: ${{ matrix.allow_bootstrap == true }} + # Standing permission would also fire on an outage. A source bootstraps only when a + # human dispatches the run naming it. + allow_bootstrap: ${{ inputs.bootstrap_source == matrix.source }} join-registry: needs: build-registry-source @@ -362,7 +365,7 @@ jobs: CSV_FILE_REGISTRY=$(find artifacts/faa -name "openairframes_registry_*.csv" -type f 2>/dev/null | head -1) # Every per-source registry CSV, whatever sources the matrix ran. SOURCE_CSVS=$(find artifacts/faa -name "openairframes_*.csv" -type f 2>/dev/null \ - | grep -v "openairframes_registry_" | grep -v "openairframes_community_" | sort) + | grep -vE '/openairframes_(registry|community|adsb)_[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{4}-[0-9]{2}-[0-9]{2}\.csv$' | sort) echo "Per-source registry CSVs found:" echo "$SOURCE_CSVS" # Prefer concatenated file (with date range) over single-day file @@ -442,6 +445,11 @@ jobs: echo "source_csvs<> "$GITHUB_OUTPUT" echo "csv_basename_registry=$(basename "$CSV_FILE_REGISTRY")" >> "$GITHUB_OUTPUT" echo "csv_basename_faa=$CSV_BASENAME_FAA" >> "$GITHUB_OUTPUT" @@ -491,7 +499,7 @@ jobs: Assets: - NOTICE (source terms; required for redistribution) - ${{ steps.meta.outputs.csv_basename_registry }} - - ${{ steps.meta.outputs.csv_basename_faa }} + ${{ steps.meta.outputs.source_basenames }} ${{ steps.meta.outputs.csv_basename_adsb && format('- {0}', steps.meta.outputs.csv_basename_adsb) || '' }} - ${{ steps.meta.outputs.csv_basename_community }} - ${{ steps.meta.outputs.zip_basename }} diff --git a/README.md b/README.md index b98a250..b3b2e30 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,11 @@ df All [FAA registration data](https://www.faa.gov/licenses_certificates/aircraft_certification/aircraft_registry/releasable_aircraft_download) from 2023-08-16 to present (~275 MB). Superseded by `openairframes_registry.csv`; still published so existing consumers keep working. +- **openairframes_tc.csv** + The [Transport Canada Civil Aircraft Register](https://wwwapps.tc.gc.ca/saf-sec-sur/2/ccarcs-riacc/RchSimp.aspx), + ~35k aircraft with full ICAO 24-bit hex coverage. Also folded into + `openairframes_registry.csv`; published separately for the same reason as the FAA CSV. + - **ReleasableAircraft_{date}.zip** A daily snapshot of the FAA database, which updates at **05:30 UTC**