feat: build a daily Transport Canada aircraft register release

- parse the headerless latin1 CCARCS export against its declared column layout
- derive transponder_code_hex from the 24-bit Mode S binary, populated for all 34,913 rows
- expand marks to C- and vintage CF- registrations and drop owner mailing addresses
- mirror the FAA build: same concat-with-latest-release dedup and output conventions

Generated-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Ashley Childress
2026-08-31 21:11:59 -04:00
parent a735616201
commit 2711b2d0f8
3 changed files with 265 additions and 0 deletions
+56
View File
@@ -0,0 +1,56 @@
from pathlib import Path
from datetime import datetime, timezone
import argparse
parser = argparse.ArgumentParser(description="Create daily Transport Canada release")
parser.add_argument("--date", type=str, help="Date to process (YYYY-MM-DD format, default: today)")
args = parser.parse_args()
if args.date:
date_str = args.date
else:
date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d")
out_dir = Path("data/tc_ccarcs")
out_dir.mkdir(parents=True, exist_ok=True)
zip_name = f"ccarcsdb_{date_str}.zip"
zip_path = out_dir / zip_name
if not zip_path.exists():
url = "https://wwwapps.tc.gc.ca/saf-sec-sur/2/ccarcs-riacc/download/ccarcsdb.zip"
from urllib.request import Request, urlopen
# CCARCS rejects default urllib agents.
req = Request(
url,
headers={
"User-Agent": (
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
)
},
method="GET",
)
with urlopen(req, timeout=120) as r:
body = r.read()
zip_path.write_bytes(body)
OUT_ROOT = Path("data/openairframes")
OUT_ROOT.mkdir(parents=True, exist_ok=True)
from derive_from_tc_ccarcs import convert_tc_ccarcs_to_df
# Row-fingerprint dedup is source-agnostic; reused rather than forked.
from derive_from_faa_master_txt import concat_faa_historical_df
from get_latest_release import get_latest_aircraft_tc_csv_df
df_new = convert_tc_ccarcs_to_df(zip_path, date_str)
try:
df_base, start_date_str = get_latest_aircraft_tc_csv_df()
df_base = concat_faa_historical_df(df_base, df_new)
assert df_base['download_date'].is_monotonic_increasing, "download_date is not monotonic increasing"
except Exception as e:
print(f"No existing Transport Canada release found, using only new data: {e}")
df_base = df_new
start_date_str = date_str
df_base.to_csv(OUT_ROOT / f"openairframes_tc_{start_date_str}_{date_str}.csv", index=False)