From f78fa1c3ee43e01ad34179f024aa324f718a3bb6 Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Thu, 10 Sep 2026 20:53:54 -0400 Subject: [PATCH] fix: make registry join failures visible instead of silent - accept source ids containing digits and underscores; the previous pattern dropped openairframes_uk_caa_*.csv with no output, contradicting the documented promise that adding a source needs no change here - reject a file whose source column disagrees with its filename, which would otherwise merge one registry into the union under another's label - refuse duplicate source files, skip an empty optional source rather than aborting, and log every skipped file and the column names shared across sources - correct the MIN_EXPECTED_ROWS comment, which described a 35k floor for a 1000 value Generated-by: Claude Opus 5 --- src/build_registry.py | 36 +++++++++++++++++++++++++++++++----- src/derive_from_tc_ccarcs.py | 6 ++++-- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/build_registry.py b/src/build_registry.py index 6d2be58..f3e582c 100644 --- a/src/build_registry.py +++ b/src/build_registry.py @@ -19,7 +19,10 @@ import pandas as pd # Sources that are registries. Community and ADS-B are published separately: they are # observations and contributions, not registration records, and do not share this schema. -FILENAME_RE = re.compile(r"openairframes_(?P[a-z]+)_(?P\d{4}-\d{2}-\d{2})_(?P\d{4}-\d{2}-\d{2})\.csv$") +FILENAME_RE = re.compile( + r"\Aopenairframes_(?P[a-z0-9_]+?)_" + r"(?P\d{4}-\d{2}-\d{2})_(?P\d{4}-\d{2}-\d{2})\.csv\Z" +) EXCLUDED_SOURCES = {"community", "adsb", "registry"} # Identifier columns lead the union so the table is usable without reading 70 headers. @@ -36,11 +39,13 @@ def discover(input_dir: Path) -> list[tuple[str, str, str, Path]]: """Return (source, start, end, path) for each per-source registry CSV found.""" found = [] for path in sorted(input_dir.rglob("openairframes_*.csv")): - match = FILENAME_RE.search(path.name) + match = FILENAME_RE.match(path.name) if not match: + print(f" SKIP {path.name}: does not match {FILENAME_RE.pattern}") continue source = match.group("source") if source in EXCLUDED_SOURCES: + print(f" SKIP {path.name}: {source!r} is published as its own asset") continue found.append((source, match.group("start"), match.group("end"), path)) return found @@ -51,18 +56,37 @@ def build(input_dir: Path, date_str: str) -> tuple[pd.DataFrame, str, str]: if not parts: raise SystemExit(f"No per-source registry CSVs found under {input_dir}") + seen = [p[0] for p in parts] + duplicated = {s for s in seen if seen.count(s) > 1} + if duplicated: + raise SystemExit(f"More than one file claims source {sorted(duplicated)}") + frames = [] for source, _, _, path in parts: # keep_default_na=False so a literal "NA" survives the round trip unchanged. df = pd.read_csv(path, dtype=str, keep_default_na=False) if "source" not in df.columns: raise SystemExit(f"{path.name}: no source column; cannot discriminate rows") - actual = set(df["source"].unique()) - if len(actual) != 1: - raise SystemExit(f"{path.name}: expected one source value, found {sorted(actual)}") + if df.empty: + print(f" {source}: empty, skipping") + continue + actual = {v.strip().lower() for v in df["source"].unique()} + if actual != {source.lower()}: + # A file whose rows disagree with its name would duplicate another source into + # the union under the wrong label. + raise SystemExit( + f"{path.name}: filename says {source!r}, rows say {sorted(actual)}" + ) print(f" {source}: {len(df)} rows, {len(df.columns)} columns from {path.name}") frames.append(df) + if not frames: + raise SystemExit("Every discovered source was empty; refusing to publish an empty registry") + + if len(frames) > 1: + shared = set.intersection(*(set(f.columns) for f in frames)) - set(LEADING_COLUMNS) + print(f" columns shared across sources ({len(shared)}): {sorted(shared)}") + columns = list(dict.fromkeys(c for df in frames for c in df.columns)) ordered = [c for c in LEADING_COLUMNS if c in columns] ordered += [c for c in columns if c not in ordered] @@ -72,6 +96,8 @@ def build(input_dir: Path, date_str: str) -> tuple[pd.DataFrame, str, str]: df_union = pd.concat([df.reindex(columns=ordered) for df in frames], ignore_index=True) df_union = df_union.fillna("") + # Earliest start across sources, not per-source coverage: a source added today still + # carries the oldest source's start date in the filename. start = min(p[1] for p in parts) end = max(p[2] for p in parts + [("", "", date_str, Path())]) return df_union, start, end diff --git a/src/derive_from_tc_ccarcs.py b/src/derive_from_tc_ccarcs.py index 510f2c5..5bf3c45 100644 --- a/src/derive_from_tc_ccarcs.py +++ b/src/derive_from_tc_ccarcs.py @@ -50,8 +50,10 @@ OWNER_ADDRESS_COLUMNS = { FOOTER_RE = re.compile(r"\s*(\d+) rows selected\.\s*") -# Canada's register is ~35k aircraft. Any parse yielding less than this means the -# export was truncated upstream, which must not be published as a real snapshot. +# Floor, not an expectation: Canada's register is ~35k aircraft and carsownr is larger +# still, so 1000 only catches a grossly truncated export. The footer row-count check above +# is what actually validates the parse; this guards the case where the footer agrees with a +# near-empty body. MIN_EXPECTED_ROWS = 1000