mirror of
https://github.com/BigBodyCobain/Shadowbroker.git
synced 2026-07-08 13:27:50 +02:00
TheYellowBeanieGuy
71a2ef4ce7
get_latest_data_deepcopy_snapshot deep-copies layers outside the data lock; a writer mutating a nested object in place races it and raises "dictionary changed size during iteration" (500 on /api/health, /api/live-data). Two changes: (1) _merge_sigint_snapshot now shallow-copies each entry so latest_data["sigint"] no longer aliases the SIGINT bridge dicts or the meshtastic_map_nodes layer (the concrete offender); (2) the snapshot retries a few times as defense-in-depth for any other in-place mutator. Plus regression tests.
112 lines
4.1 KiB
Python
112 lines
4.1 KiB
Python
"""SIGINT fetcher — pulls latest signals from the SIGINT Grid into latest_data.
|
|
|
|
Merges live MQTT signals with cached Meshtastic map API nodes.
|
|
Live MQTT signals always take priority (fresher) — API nodes fill in the gaps
|
|
for the thousands of nodes our MQTT listener hasn't heard yet.
|
|
"""
|
|
|
|
import logging
|
|
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
|
|
|
logger = logging.getLogger("services.data_fetcher")
|
|
|
|
|
|
def _merge_sigint_snapshot(
|
|
live_signals: list[dict],
|
|
api_nodes: list[dict],
|
|
) -> list[dict]:
|
|
"""Merge live bridge signals with cached Meshtastic map nodes.
|
|
|
|
Live Meshtastic observations always win over map/API nodes for the same callsign
|
|
because they include fresher region/channel metadata.
|
|
"""
|
|
|
|
# Shallow-copy every entry so the published list owns its own dicts. The
|
|
# inputs alias objects that other threads keep mutating in place: live
|
|
# signals are the SIGINT bridge's own dicts (updated as packets arrive),
|
|
# and api_nodes are the same objects published under latest_data
|
|
# ["meshtastic_map_nodes"]. Publishing those references into
|
|
# latest_data["sigint"] lets a concurrent mutation race the lock-free
|
|
# deepcopy in get_latest_data_deepcopy_snapshot() (/api/health, /api/live-
|
|
# data) and raise "dictionary changed size during iteration". Copying
|
|
# honors the replace-don't-mutate contract in fetchers/_store.py.
|
|
merged = [dict(s) for s in live_signals]
|
|
live_callsigns = {s["callsign"] for s in merged if s.get("source") == "meshtastic"}
|
|
for node in api_nodes:
|
|
if node.get("callsign") in live_callsigns:
|
|
continue
|
|
merged.append(dict(node))
|
|
merged.sort(key=lambda item: str(item.get("timestamp", "") or ""), reverse=True)
|
|
return merged
|
|
|
|
|
|
def _sigint_totals(signals: list[dict]) -> dict[str, int]:
|
|
totals = {
|
|
"total": len(signals),
|
|
"meshtastic": 0,
|
|
"meshtastic_live": 0,
|
|
"meshtastic_map": 0,
|
|
"aprs": 0,
|
|
"js8call": 0,
|
|
}
|
|
for sig in signals:
|
|
source = str(sig.get("source", "") or "").lower()
|
|
if source == "meshtastic":
|
|
totals["meshtastic"] += 1
|
|
if bool(sig.get("from_api")):
|
|
totals["meshtastic_map"] += 1
|
|
else:
|
|
totals["meshtastic_live"] += 1
|
|
elif source == "aprs":
|
|
totals["aprs"] += 1
|
|
elif source == "js8call":
|
|
totals["js8call"] += 1
|
|
return totals
|
|
|
|
|
|
def build_sigint_snapshot() -> tuple[list[dict], dict[str, object], dict[str, int]]:
|
|
"""Build the current merged SIGINT snapshot without hitting the network."""
|
|
|
|
from services.sigint_bridge import sigint_grid
|
|
|
|
live_signals = sigint_grid.get_all_signals()
|
|
with _data_lock:
|
|
api_nodes = list(latest_data.get("meshtastic_map_nodes", []))
|
|
merged = _merge_sigint_snapshot(live_signals, api_nodes)
|
|
channel_stats = sigint_grid.get_mesh_channel_stats(api_nodes or None)
|
|
totals = _sigint_totals(merged)
|
|
return merged, channel_stats, totals
|
|
|
|
|
|
def refresh_sigint_snapshot() -> tuple[list[dict], dict[str, object], dict[str, int]]:
|
|
"""Refresh latest_data SIGINT state from current bridge + cache state."""
|
|
|
|
signals, channel_stats, totals = build_sigint_snapshot()
|
|
with _data_lock:
|
|
latest_data["sigint"] = signals
|
|
latest_data["mesh_channel_stats"] = channel_stats
|
|
latest_data["sigint_totals"] = totals
|
|
_mark_fresh("sigint")
|
|
return signals, channel_stats, totals
|
|
|
|
|
|
def fetch_sigint():
|
|
"""Fetch all signals from the SIGINT Grid, merge with Meshtastic map nodes."""
|
|
from services.fetchers._store import is_any_active
|
|
|
|
if not is_any_active("sigint_meshtastic", "sigint_aprs"):
|
|
return
|
|
from services.sigint_bridge import sigint_grid
|
|
|
|
# Start bridges on first call (idempotent)
|
|
sigint_grid.start()
|
|
|
|
signals, channel_stats, totals = refresh_sigint_snapshot()
|
|
|
|
status = sigint_grid.status
|
|
logger.info(
|
|
f"SIGINT: {len(signals)} signals "
|
|
f"(APRS:{status['aprs']} MESH:{status['meshtastic']} "
|
|
f"JS8:{status['js8call']} MAP:{totals['meshtastic_map']})"
|
|
)
|