CalvinBackup/Shadowbroker
1
0
Fork 0
mirror of https://github.com/BigBodyCobain/Shadowbroker.git synced 2026-05-28 01:52:28 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat(flights): stamp source attribution on every flight record

Browse Source 
Pre-fix, adsb.lol records (the primary source for most flights) carried
no source marker. OpenSky records got is_opensky: True and supplementals
got supplemental_source, so any UI inspecting source labels saw
OpenSky/airplanes.live records as explicitly tagged and adsb.lol records
as "unlabeled" — making it look like adsb.lol wasn't being used at all
even though it's the primary source.

Changes:

* _fetch_adsb_lol_regions stamps source="adsb.lol" on each aircraft
  before returning, so the tag survives the OpenSky dedupe-by-hex merge.
* OpenSky records get source="OpenSky" (alongside is_opensky=True for
  back-compat).
* military fetcher tags source on both adsb.lol and airplanes.live
  records before they're merged, and propagates source into the
  military_flights and uavs output dicts.
* _classify_and_publish promotes the explicit source field into the
  published flight dict. Falls back to legacy supplemental_source if
  source is absent. Final fallback "adsb.lol" preserves prior behavior
  for any caller synthesizing records without going through a fetcher.

8 new tests cover the published-dict propagation, OpenSky tagging,
supplemental fallback, explicit-wins precedence, default behavior, the
adsb.lol regional fetcher tagging, and the military output dict.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
BigBodyCobain
2026-05-23 06:14:39 -06:00
parent febcce9125
commit f14d4feb6d
3 changed files with 382 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/services/fetchers/flights.py
+23 -1
View File
@@ -459,6 +459,18 @@ def _classify_and_publish(all_adsb_flights):
ac_category = "heli" if model_upper in _HELI_TYPES_BACKEND else "plane"
# Source attribution: prefer the explicit ``source`` tag stamped
# at fetch time (adsb.lol, OpenSky). If absent, fall back to the
# legacy ``supplemental_source`` (airplanes.live, adsb.fi) so
# supplementals are still attributed without changing their
# tagger. Final fallback "adsb.lol" preserves prior behavior for
# any caller that synthesizes records without going through one
# of our fetchers (e.g. tests).
source = (
f.get("source")
or f.get("supplemental_source")
or "adsb.lol"
)
flights.append(
{
"callsign": flight_str,
@@ -480,6 +492,7 @@ def _classify_and_publish(all_adsb_flights):
"airline_code": airline_code,
"aircraft_category": ac_category,
"nac_p": f.get("nac_p"),
"source": source,
}
)
except (ValueError, TypeError, KeyError, AttributeError) as loop_e:
@@ -849,7 +862,15 @@ def _fetch_adsb_lol_regions():
res = fetch_with_curl(url, timeout=10)
if res.status_code == 200:
data = res.json()
return data.get("ac", [])
aircraft = data.get("ac", [])
# Stamp the source at the fetch site so attribution survives
# the OpenSky/supplemental dedupe-by-hex merge downstream.
# Previously adsb.lol records carried no marker while OpenSky
# records got ``is_opensky: True`` — which made flight tooltips
# look like everything came from OpenSky.
for a in aircraft:
a["source"] = "adsb.lol"
return aircraft
except (
requests.RequestException,
ConnectionError,
@@ -932,6 +953,7 @@ def _enrich_with_opensky_and_supplemental(adsb_flights):
"gs": (s[9] * 1.94384) if s[9] else 0,
"t": "Unknown",
"is_opensky": True,
"source": "OpenSky",
}
)
elif os_res.status_code == 429:
backend/services/fetchers/military.py
+5 -1
View File
@@ -171,6 +171,7 @@ def fetch_military_flights():
h = a.get("hex", "").lower()
if h and h not in seen_hex:
seen_hex.add(h)
a["source"] = "adsb.lol"
all_mil_ac.append(a)
except Exception as e:
logger.warning(f"adsb.lol mil fetch failed: {e}")
@@ -182,6 +183,7 @@ def fetch_military_flights():
h = a.get("hex", "").lower()
if h and h not in seen_hex:
seen_hex.add(h)
a["source"] = "airplanes.live"
all_mil_ac.append(a)
logger.info(f"airplanes.live mil: +{len(resp2.json().get('ac', []))} raw, {len(all_mil_ac)} total unique")
except Exception as e:
@@ -234,6 +236,7 @@ def fetch_military_flights():
"registration": f.get("r", "N/A"),
"icao24": icao_hex,
"squawk": f.get("squawk", ""),
"source": f.get("source") or "adsb.lol",
})
continue
@@ -258,7 +261,8 @@ def fetch_military_flights():
"model": f.get("t", "Unknown"),
"icao24": icao_hex,
"speed_knots": speed_knots,
"squawk": f.get("squawk", "")
"squawk": f.get("squawk", ""),
"source": f.get("source") or "adsb.lol",
})
except Exception as loop_e:
logger.error(f"Mil flight interpolation error: {loop_e}")
backend/tests/test_flight_source_attribution.py
+354
View File
@@ -0,0 +1,354 @@
"""Per-flight source attribution.
Background
----------
Pre-fix, adsb.lol records (the primary source for most flights) carried
no source marker. OpenSky records got ``is_opensky: True`` and
supplementals got ``supplemental_source``, so any UI that wanted to show
which provider a flight came from saw OpenSky/airplanes.live records as
explicitly tagged and adsb.lol records as "unlabeled" — making it look
like adsb.lol wasn't even being used.
This caused user confusion ("only military planes have adsb.lol
telemetry") that was diagnostic noise, not a real bug. The actual fix:
stamp ``source`` at every fetch site so the downstream consumer can
attribute the provider with no guesswork.
These tests pin:
* adsb.lol regional records get ``source: "adsb.lol"`` at fetch time
(synthesized via the published flight dict).
* OpenSky records get ``source: "OpenSky"`` (alongside the existing
``is_opensky: True`` for backwards compat).
* Supplementals (airplanes.live, adsb.fi) flow through with their
``supplemental_source`` honored.
* The military fetcher tags ``source`` on military_flights and uavs.
* The published flight dict carries ``source`` so downstream code
can render attribution.
"""
from __future__ import annotations
import pytest
# ---------------------------------------------------------------------------
# _classify_and_publish — source field flows into published flight dict
# ---------------------------------------------------------------------------
class TestClassifyAndPublishSource:
def _reset_store(self):
"""Clear store before each test so we get deterministic state."""
from services.fetchers._store import latest_data, _data_lock
with _data_lock:
for key in (
"flights", "commercial_flights", "private_flights",
"private_jets", "military_flights", "tracked_flights",
):
latest_data[key] = []
return latest_data
def test_adsb_lol_record_tagged_in_published_flight(self, monkeypatch):
"""A raw adsb.lol record (carrying ``source: 'adsb.lol'`` from the
fetch site) flows through ``_classify_and_publish`` and the
published flight dict carries the same ``source`` field."""
from services.fetchers import flights as flights_module
from services.fetchers._store import latest_data, _data_lock
self._reset_store()
# Patch route + type lookups so they don't try to hit the network.
monkeypatch.setattr(flights_module, "lookup_route", lambda _: None)
monkeypatch.setattr(flights_module, "lookup_aircraft_type", lambda _: "")
flights_module._classify_and_publish(
[
{
"hex": "ad7701",
"flight": "JBU711",
"r": "N967JT",
"t": "A321",
"lat": 40.0,
"lon": -100.0,
"alt_baro": 36000,
"gs": 401.6,
"nac_p": 9,
"source": "adsb.lol", # stamped at fetch site
}
]
)
with _data_lock:
published = list(latest_data.get("flights", []))
assert len(published) == 1
assert published[0]["source"] == "adsb.lol"
# nac_p still flows through too — sanity check that adding source
# didn't break the existing GPS jamming signal.
assert published[0]["nac_p"] == 9
def test_opensky_record_tagged_in_published_flight(self, monkeypatch):
"""OpenSky-sourced records carry ``source: 'OpenSky'`` (plus the
existing ``is_opensky: True`` for back-compat)."""
from services.fetchers import flights as flights_module
from services.fetchers._store import latest_data, _data_lock
self._reset_store()
monkeypatch.setattr(flights_module, "lookup_route", lambda _: None)
monkeypatch.setattr(flights_module, "lookup_aircraft_type", lambda _: "")
flights_module._classify_and_publish(
[
{
"hex": "a12345",
"flight": "UAL100",
"r": "N100UA",
"t": "Unknown",
"lat": 41.0,
"lon": -87.0,
"alt_baro": 35000,
"gs": 450,
# No nac_p — OpenSky doesn't carry it.
"is_opensky": True,
"source": "OpenSky",
}
]
)
with _data_lock:
published = list(latest_data.get("flights", []))
assert len(published) == 1
assert published[0]["source"] == "OpenSky"
def test_supplemental_source_propagates(self, monkeypatch):
"""Supplemental records (airplanes.live, adsb.fi) have their
legacy ``supplemental_source`` field promoted to the unified
``source`` field in the published dict — so consumers don't have
to inspect two different keys."""
from services.fetchers import flights as flights_module
from services.fetchers._store import latest_data, _data_lock
self._reset_store()
monkeypatch.setattr(flights_module, "lookup_route", lambda _: None)
monkeypatch.setattr(flights_module, "lookup_aircraft_type", lambda _: "")
flights_module._classify_and_publish(
[
{
"hex": "b22222",
"flight": "DAL200",
"r": "N200DL",
"t": "B738",
"lat": 42.0,
"lon": -90.0,
"alt_baro": 32000,
"gs": 420,
"supplemental_source": "airplanes.live",
# No explicit "source" — should fall through to
# supplemental_source.
}
]
)
with _data_lock:
published = list(latest_data.get("flights", []))
assert len(published) == 1
assert published[0]["source"] == "airplanes.live"
def test_explicit_source_wins_over_supplemental_source(self, monkeypatch):
"""If both fields are present, explicit ``source`` wins (it's the
newer canonical tag)."""
from services.fetchers import flights as flights_module
from services.fetchers._store import latest_data, _data_lock
self._reset_store()
monkeypatch.setattr(flights_module, "lookup_route", lambda _: None)
monkeypatch.setattr(flights_module, "lookup_aircraft_type", lambda _: "")
flights_module._classify_and_publish(
[
{
"hex": "c33333",
"flight": "AAL300",
"r": "N300AA",
"t": "A321",
"lat": 33.0,
"lon": -97.0,
"alt_baro": 34000,
"gs": 430,
"source": "adsb.lol",
"supplemental_source": "adsb.fi",
}
]
)
with _data_lock:
published = list(latest_data.get("flights", []))
assert published[0]["source"] == "adsb.lol"
def test_untagged_record_defaults_to_adsb_lol(self, monkeypatch):
"""A record with neither ``source`` nor ``supplemental_source``
(e.g. synthesized by a test, or a fetcher that hasn't been
migrated yet) defaults to ``"adsb.lol"`` since that's been the
primary source historically. Defensive default — better than
empty string."""
from services.fetchers import flights as flights_module
from services.fetchers._store import latest_data, _data_lock
self._reset_store()
monkeypatch.setattr(flights_module, "lookup_route", lambda _: None)
monkeypatch.setattr(flights_module, "lookup_aircraft_type", lambda _: "")
flights_module._classify_and_publish(
[
{
"hex": "d44444",
"flight": "SWA400",
"r": "N400SW",
"t": "B737",
"lat": 32.0,
"lon": -110.0,
"alt_baro": 30000,
"gs": 410,
}
]
)
with _data_lock:
published = list(latest_data.get("flights", []))
assert published[0]["source"] == "adsb.lol"
# ---------------------------------------------------------------------------
# adsb.lol regional fetcher tags at fetch time
# ---------------------------------------------------------------------------
class TestAdsbLolRegionalTagging:
def test_fetch_region_stamps_source_on_each_aircraft(self, monkeypatch):
"""The wrapper around the adsb.lol regional endpoint stamps
``source: 'adsb.lol'`` on every record before returning, so the
downstream merge step sees attribution survive even when the
record gets reshuffled (e.g. dedupe-by-hex during OpenSky merge)."""
from services.fetchers import flights as flights_module
# Fake response — 3 aircraft, none have a source field originally.
class FakeResp:
status_code = 200
def json(self):
return {
"ac": [
{"hex": "a1", "lat": 40.0, "lon": -100.0, "nac_p": 8},
{"hex": "a2", "lat": 40.1, "lon": -100.1, "nac_p": 9},
{"hex": "a3", "lat": 40.2, "lon": -100.2, "nac_p": 10},
]
}
monkeypatch.setattr(
flights_module, "fetch_with_curl", lambda *a, **kw: FakeResp()
)
results = flights_module._fetch_adsb_lol_regions()
assert len(results) >= 3
# Every aircraft we got back must be tagged.
sources = {a.get("source") for a in results}
assert sources == {"adsb.lol"}, (
f"adsb.lol regional fetcher must stamp source on every record; "
f"got: {sources}"
)
def test_fetch_region_failure_returns_empty_without_crashing(self, monkeypatch):
"""If adsb.lol returns non-200, the fetcher returns [] gracefully —
downstream code already handles this. Sanity check that the source
tagging doesn't introduce a new failure mode."""
from services.fetchers import flights as flights_module
class FakeResp:
status_code = 500
def json(self): return {}
monkeypatch.setattr(
flights_module, "fetch_with_curl", lambda *a, **kw: FakeResp()
)
results = flights_module._fetch_adsb_lol_regions()
assert results == []
# ---------------------------------------------------------------------------
# Military fetcher tags source on output dicts
# ---------------------------------------------------------------------------
class TestMilitarySourceTagging:
def test_military_output_carries_source_field(self, monkeypatch):
"""Each entry in ``military_flights`` should carry a ``source``
field. Pre-fix the only military attribution was inferring from
which endpoint we hit; now it's explicit."""
from services.fetchers import military as mil_module
from services.fetchers._store import latest_data, _data_lock
# Reset relevant store state.
with _data_lock:
latest_data["military_flights"] = []
latest_data["uavs"] = []
latest_data["tracked_flights"] = []
# Stub _store.is_any_active so the fetch doesn't early-return.
# The military module imports the function inline at call time,
# so we have to patch it on the _store module itself rather than
# on the military module.
from services.fetchers import _store as store_module
monkeypatch.setattr(store_module, "is_any_active", lambda *_: True)
# Stub fetch_with_curl to return one synthetic military aircraft
# from adsb.lol, none from airplanes.live.
class _RespMil:
status_code = 200
def json(self):
return {
"ac": [
{
"hex": "ae6c1d",
"flight": "CRUSH52",
"r": "170281",
"t": "C30J",
"lat": 47.594,
"lon": -124.879,
"alt_baro": 9025,
"gs": 162.8,
"track": 334.5,
"nac_p": 10,
}
]
}
class _RespEmpty:
status_code = 200
def json(self):
return {"ac": []}
def _fake_fetch(url, *a, **kw):
if "adsb.lol" in url:
return _RespMil()
return _RespEmpty()
monkeypatch.setattr(mil_module, "fetch_with_curl", _fake_fetch)
# Stubs for downstream enrichments that try to hit external state.
monkeypatch.setattr(mil_module, "enrich_with_plane_alert", lambda mf: None)
monkeypatch.setattr(mil_module, "_enrich_country", lambda hex_, flag: ("US", "USAF"))
monkeypatch.setattr(mil_module, "_classify_military_type", lambda t: "transport")
monkeypatch.setattr(mil_module, "_classify_uav", lambda m, c: (False, "", ""))
monkeypatch.setattr(mil_module, "get_emissions_info", lambda model: None)
monkeypatch.setattr(mil_module, "_mark_fresh", lambda *keys: None)
mil_module.fetch_military_flights()
with _data_lock:
mil_published = list(latest_data.get("military_flights", []))
assert len(mil_published) == 1
assert mil_published[0]["source"] == "adsb.lol"