Release v0.9.75 runtime and onboarding update

Ship the 0.9.75 source update with improved startup/runtime hardening, operator API key onboarding, Meshtastic MQTT controls, Infonet/MeshChat separation, desktop package versioning, and aircraft telemetry refinements.

Also updates focused backend/frontend tests for node settings, Meshtastic MQTT settings, and desktop runtime behavior.

backend/services/ais_stream.py

@@ -339,16 +339,61 @@ def get_country_from_mmsi(mmsi: int) -> str:
 
 # Global vessel store: MMSI → vessel dict
 _vessels: dict[int, dict] = {}
+_vessel_trails: dict[int, dict] = {}
 _vessels_lock = threading.Lock()
 _ws_thread: threading.Thread | None = None
 _ws_running = False
 _proxy_process = None
+_VESSEL_TRAIL_INTERVAL_S = 120
+_VESSEL_TRAIL_MAX_POINTS = 240
 
 import os
 
 CACHE_FILE = os.path.join(os.path.dirname(__file__), "ais_cache.json")
 
 
+def _record_vessel_trail_locked(mmsi: int, lat, lng, sog=0, now_ts: float | None = None) -> None:
+    """Append a sampled AIS trail point. Caller must hold _vessels_lock."""
+    if lat is None or lng is None:
+        return
+    try:
+        lat_f = float(lat)
+        lng_f = float(lng)
+    except (TypeError, ValueError):
+        return
+    if abs(lat_f) > 90 or abs(lng_f) > 180 or (lat_f == 0 and lng_f == 0):
+        return
+    now = now_ts or time.time()
+    trail_data = _vessel_trails.setdefault(int(mmsi), {"points": [], "last_seen": now})
+    point = [round(lat_f, 5), round(lng_f, 5), round(float(sog or 0), 1), round(now)]
+    last_point_ts = trail_data["points"][-1][3] if trail_data["points"] else 0
+    if now - last_point_ts < _VESSEL_TRAIL_INTERVAL_S:
+        trail_data["last_seen"] = now
+        return
+    if (
+        trail_data["points"]
+        and trail_data["points"][-1][0] == point[0]
+        and trail_data["points"][-1][1] == point[1]
+    ):
+        trail_data["last_seen"] = now
+        return
+    trail_data["points"].append(point)
+    trail_data["last_seen"] = now
+    if len(trail_data["points"]) > _VESSEL_TRAIL_MAX_POINTS:
+        trail_data["points"] = trail_data["points"][-_VESSEL_TRAIL_MAX_POINTS:]
+
+
+def get_vessel_trail(mmsi: int) -> list:
+    """Return the accumulated trail for a single vessel without expanding live payloads."""
+    try:
+        key = int(mmsi)
+    except (TypeError, ValueError):
+        return []
+    with _vessels_lock:
+        points = _vessel_trails.get(key, {}).get("points", [])
+        return [list(point) for point in points]
+
+
 def _save_cache():
     """Save vessel data to disk for persistence across restarts."""
     try:
@@ -391,6 +436,7 @@ def prune_stale_vessels():
         stale_keys = [k for k, v in _vessels.items() if v.get("_updated", 0) < stale_cutoff]
         for k in stale_keys:
             del _vessels[k]
+            _vessel_trails.pop(k, None)
     if stale_keys:
         logger.info(f"AIS pruned {len(stale_keys)} stale vessels")
 
@@ -459,6 +505,7 @@ def ingest_ais_catcher(msgs: list[dict]) -> int:
                     heading = msg.get("heading", 511)
                     vessel["heading"] = heading if heading != 511 else vessel.get("cog", 0)
                     vessel["_updated"] = now
+                    _record_vessel_trail_locked(mmsi, lat, lon, vessel["sog"], now)
                     if msg.get("shipname"):
                         vessel["name"] = msg["shipname"].strip()
                     count += 1
@@ -595,7 +642,9 @@ def _ais_stream_loop():
                         vessel["cog"] = report.get("Cog", 0)
                         heading = report.get("TrueHeading", 511)
                         vessel["heading"] = heading if heading != 511 else report.get("Cog", 0)
-                        vessel["_updated"] = time.time()
+                        now_ts = time.time()
+                        vessel["_updated"] = now_ts
+                        _record_vessel_trail_locked(mmsi, lat, lng, vessel["sog"], now_ts)
                         # Use metadata name if we don't have one yet
                         if not vessel.get("name") or vessel["name"] == "UNKNOWN":
                             vessel["name"] = (

backend/services/fetchers/aircraft_database.py

@@ -32,7 +32,7 @@ _REFRESH_INTERVAL_S = 5 * 24 * 3600
 _LIST_TIMEOUT_S = 30
 _DOWNLOAD_TIMEOUT_S = 600
 _USER_AGENT = (
-    "ShadowBroker-OSINT/0.9.7 "
+    "ShadowBroker-OSINT/0.9.75 "
     "(+https://github.com/BigBodyCobain/Shadowbroker; "
     "contact: bigbodycobain@gmail.com)"
 )

backend/services/fetchers/flights.py

@@ -258,6 +258,16 @@ flight_trails = {}  # {icao_hex: {points: [[lat, lng, alt, ts], ...], last_seen:
 _trails_lock = threading.Lock()
 _MAX_TRACKED_TRAILS = 2000
 
+
+def get_flight_trail(icao24: str) -> list:
+    """Return the accumulated trail for a single aircraft without expanding live payloads."""
+    hex_id = str(icao24 or "").strip().lower()
+    if not hex_id:
+        return []
+    with _trails_lock:
+        points = flight_trails.get(hex_id, {}).get("points", [])
+        return [list(point) for point in points]
+
 # Route enrichment is now served from services.fetchers.route_database, which
 # bulk-loads vrs-standing-data.adsb.lol/routes.csv.gz once per day and looks up
 # callsigns from an in-memory index. Replaces the legacy /api/0/routeset POST,
@@ -612,13 +622,22 @@ def _classify_and_publish(all_adsb_flights):
     )
 
     # --- Trail Accumulation ---
-    _TRAIL_INTERVAL_S = 600  # only record a new trail point every 10 minutes
+    _TRAIL_INTERVAL_S = 60  # selected trails need enough resolution to show where unknown-route traffic came from
 
     def _accumulate_trail(f, now_ts, check_route=True):
         hex_id = f.get("icao24", "").lower()
         if not hex_id:
             return 0, None
-        if check_route and f.get("origin_name", "UNKNOWN") != "UNKNOWN":
+
+        def _known_route_name(value):
+            normalized = str(value or "").strip().upper()
+            return bool(normalized and normalized != "UNKNOWN")
+
+        has_known_route = bool(
+            (f.get("origin_loc") and f.get("dest_loc"))
+            or (_known_route_name(f.get("origin_name")) and _known_route_name(f.get("dest_name")))
+        )
+        if check_route and has_known_route:
             f["trail"] = []
             return 0, hex_id
         lat, lng, alt = f.get("lat"), f.get("lng"), f.get("alt", 0)
@@ -629,7 +648,7 @@ def _classify_and_publish(all_adsb_flights):
         if hex_id not in flight_trails:
             flight_trails[hex_id] = {"points": [], "last_seen": now_ts}
         trail_data = flight_trails[hex_id]
-        # Only append a new point if 10 minutes have passed since the last one
+        # Only append a new point if enough time has passed since the last one
         last_point_ts = trail_data["points"][-1][3] if trail_data["points"] else 0
         if now_ts - last_point_ts < _TRAIL_INTERVAL_S:
             trail_data["last_seen"] = now_ts
@@ -649,14 +668,16 @@ def _classify_and_publish(all_adsb_flights):
 
     now_ts = datetime.utcnow().timestamp()
     with _data_lock:
+        commercial_snapshot = copy.deepcopy(latest_data.get("commercial_flights", []))
+        private_jets_snapshot = copy.deepcopy(latest_data.get("private_jets", []))
+        private_ga_snapshot = copy.deepcopy(latest_data.get("private_flights", []))
         military_snapshot = copy.deepcopy(latest_data.get("military_flights", []))
         tracked_snapshot = copy.deepcopy(latest_data.get("tracked_flights", []))
         raw_flights_snapshot = list(latest_data.get("flights", []))
 
-    # Commercial/private: skip trail if route is known (route line replaces trail)
-    route_check_lists = [commercial, private_jets, private_ga]
-    # Tracked + military: ALWAYS accumulate trails (high-interest flights)
-    always_trail_lists = [existing_tracked, military_snapshot]
+    # Skip trails for any flight with a known route; the route line replaces historical trail.
+    route_check_lists = [commercial_snapshot, private_jets_snapshot, private_ga_snapshot]
+    always_trail_lists = [tracked_snapshot, military_snapshot]
     seen_hexes = set()
     trail_count = 0
     with _trails_lock:
@@ -669,7 +690,7 @@ def _classify_and_publish(all_adsb_flights):
 
         for flist in always_trail_lists:
             for f in flist:
-                count, hex_id = _accumulate_trail(f, now_ts, check_route=False)
+                count, hex_id = _accumulate_trail(f, now_ts, check_route=True)
                 trail_count += count
                 if hex_id:
                     seen_hexes.add(hex_id)
@@ -693,6 +714,13 @@ def _classify_and_publish(all_adsb_flights):
         f"Trail accumulation: {trail_count} active trails, {len(stale_keys)} pruned, {len(flight_trails)} total"
     )
 
+    with _data_lock:
+        latest_data["commercial_flights"] = commercial_snapshot
+        latest_data["private_jets"] = private_jets_snapshot
+        latest_data["private_flights"] = private_ga_snapshot
+        latest_data["tracked_flights"] = tracked_snapshot
+        latest_data["military_flights"] = military_snapshot
+
     # --- GPS Jamming Detection ---
     # Uses NACp (Navigation Accuracy Category – Position) from ADS-B to infer
     # GPS interference zones, similar to GPSJam.org / Flightradar24.

backend/services/fetchers/meshtastic_map.py

@@ -182,7 +182,7 @@ def fetch_meshtastic_nodes():
         callsign = str(getattr(get_settings(), "MESHTASTIC_OPERATOR_CALLSIGN", "") or "").strip()
     except Exception:
         callsign = ""
-    ua_base = "ShadowBroker-OSINT/0.9.7 (+https://github.com/BigBodyCobain/Shadowbroker; contact: bigbodycobain@gmail.com; 24h polling)"
+    ua_base = "ShadowBroker-OSINT/0.9.75 (+https://github.com/BigBodyCobain/Shadowbroker; contact: bigbodycobain@gmail.com; 24h polling)"
     user_agent = f"{ua_base}; node={callsign}" if callsign else ua_base
 
     try:

backend/services/fetchers/route_database.py

@@ -25,7 +25,7 @@ _REFRESH_INTERVAL_S = 5 * 24 * 3600
 _HTTP_TIMEOUT_S = 60
 
 _USER_AGENT = (
-    "ShadowBroker-OSINT/0.9.7 "
+    "ShadowBroker-OSINT/0.9.75 "
     "(+https://github.com/BigBodyCobain/Shadowbroker; "
     "contact: bigbodycobain@gmail.com)"
 )

backend/services/mesh/mesh_router.py

@@ -390,15 +390,9 @@ class MeshtasticTransport:
     def _mqtt_config() -> tuple[str, int, str, str]:
         """Return (broker, port, user, password) from settings."""
         try:
-            from services.config import get_settings
+            from services.meshtastic_mqtt_settings import mqtt_connection_config
 
-            s = get_settings()
-            return (
-                str(s.MESH_MQTT_BROKER or "mqtt.meshtastic.org"),
-                int(s.MESH_MQTT_PORT or 1883),
-                str(s.MESH_MQTT_USER or "meshdev"),
-                str(s.MESH_MQTT_PASS or "large4cats"),
-            )
+            return mqtt_connection_config()
         except Exception:
             return ("mqtt.meshtastic.org", 1883, "meshdev", "large4cats")
 
@@ -433,8 +427,9 @@ class MeshtasticTransport:
     def _resolve_psk(cls) -> bytes:
         """Return the PSK from config, or the default LongFast key if empty."""
         try:
-            from services.config import get_settings
-            raw = str(getattr(get_settings(), "MESH_MQTT_PSK", "") or "").strip()
+            from services.meshtastic_mqtt_settings import mqtt_psk_hex
+
+            raw = mqtt_psk_hex()
         except Exception:
             raw = ""
         if not raw:
@@ -449,7 +444,10 @@ class MeshtasticTransport:
 
     @staticmethod
     def mesh_address_for_sender(sender_id: str) -> str:
-        """Return the synthetic public mesh address used for MQTT-originated sends."""
+        """Return the public mesh address used for MQTT-originated sends."""
+        parsed = MeshtasticTransport._parse_node_id(sender_id)
+        if parsed is not None:
+            return f"!{parsed:08x}"
         return f"!{MeshtasticTransport._stable_node_id(sender_id):08x}"
 
     @staticmethod
@@ -489,7 +487,8 @@ class MeshtasticTransport:
 
             # Generate IDs
             packet_id = random.randint(1, 0xFFFFFFFF)
-            from_node = self._stable_node_id(envelope.sender_id)
+            parsed_sender = self._parse_node_id(envelope.sender_id)
+            from_node = parsed_sender if parsed_sender is not None else self._stable_node_id(envelope.sender_id)
             direct_node = self._parse_node_id(envelope.destination)
             to_node = direct_node if direct_node is not None else 0xFFFFFFFF
 
@@ -529,9 +528,7 @@ class MeshtasticTransport:
                     error_msg[0] = f"MQTT connect refused: rc={rc}"
                     client.disconnect()
 
-            client = mqtt.Client(
-                client_id=f"shadowbroker-tx-{envelope.message_id[:8]}", protocol=mqtt.MQTTv311
-            )
+            client = mqtt.Client(client_id=f"meshchat-tx-{envelope.message_id[:8]}", protocol=mqtt.MQTTv311)
             broker, port, user, pw = self._mqtt_config()
             client.username_pw_set(user, pw)
             client.on_connect = _on_connect

backend/services/mesh/meshtastic_topics.py

@@ -8,6 +8,7 @@ from typing import Iterable
 # Default subscription roots — US-only to avoid flooding the public broker.
 # Users can opt into additional regions via MESH_MQTT_EXTRA_ROOTS.
 DEFAULT_ROOTS: tuple[str, ...] = ("US",)
+DEFAULT_CHANNEL = "LongFast"
 
 # Every known official region root (for UI dropdowns / manual opt-in).
 ALL_OFFICIAL_ROOTS: tuple[str, ...] = (
@@ -107,6 +108,10 @@ def normalize_topic_filter(value: str) -> str | None:
     return "/".join(parts)
 
 
+def _default_topic_for_root(root: str) -> str:
+    return f"msh/{root}/2/e/{DEFAULT_CHANNEL}/#"
+
+
 def build_subscription_topics(
     extra_roots: str = "",
     extra_topics: str = "",
@@ -119,7 +124,7 @@ def build_subscription_topics(
         # via MESH_MQTT_EXTRA_ROOTS to avoid flooding the public broker.
     roots.extend(root for root in (normalize_root(item) for item in _split_config_values(extra_roots)) if root)
 
-    topics = [f"msh/{root}/#" for root in _dedupe(roots)]
+    topics = [_default_topic_for_root(root) for root in _dedupe(roots)]
     topics.extend(
         topic
         for topic in (
@@ -137,7 +142,7 @@ def known_roots(extra_roots: str = "", include_defaults: bool = True) -> list[st
     for topic in topics:
         if not topic.startswith("msh/") or not topic.endswith("/#"):
             continue
-        root = normalize_root(topic[4:-2])
+        root = normalize_root(parse_topic_metadata(topic)["root"])
         if root:
             roots.append(root)
     return _dedupe(roots)

backend/services/meshtastic_mqtt_settings.py

@@ -0,0 +1,172 @@
+from __future__ import annotations
+
+import json
+import os
+import time
+from pathlib import Path
+from typing import Any
+
+from services.config import get_settings
+
+
+PUBLIC_DEFAULT_USER = "meshdev"
+PUBLIC_DEFAULT_PASS = "large4cats"
+DATA_DIR = Path(os.environ.get("SB_DATA_DIR", str(Path(__file__).parent.parent / "data")))
+if not DATA_DIR.is_absolute():
+    DATA_DIR = Path(__file__).parent.parent / DATA_DIR
+
+SETTINGS_FILE = DATA_DIR / "meshtastic_mqtt.json"
+_cache: dict[str, Any] | None = None
+_cache_ts: float = 0.0
+_CACHE_TTL = 2.0
+
+
+def _settings_defaults() -> dict[str, Any]:
+    try:
+        s = get_settings()
+        return {
+            "enabled": bool(getattr(s, "MESH_MQTT_ENABLED", False)),
+            "broker": str(getattr(s, "MESH_MQTT_BROKER", "") or "mqtt.meshtastic.org"),
+            "port": int(getattr(s, "MESH_MQTT_PORT", 1883) or 1883),
+            "username": str(getattr(s, "MESH_MQTT_USER", "") or PUBLIC_DEFAULT_USER),
+            "password": str(getattr(s, "MESH_MQTT_PASS", "") or PUBLIC_DEFAULT_PASS),
+            "psk": str(getattr(s, "MESH_MQTT_PSK", "") or ""),
+            "include_default_roots": bool(getattr(s, "MESH_MQTT_INCLUDE_DEFAULT_ROOTS", True)),
+            "extra_roots": str(getattr(s, "MESH_MQTT_EXTRA_ROOTS", "") or ""),
+            "extra_topics": str(getattr(s, "MESH_MQTT_EXTRA_TOPICS", "") or ""),
+        }
+    except Exception:
+        return {
+            "enabled": False,
+            "broker": "mqtt.meshtastic.org",
+            "port": 1883,
+            "username": PUBLIC_DEFAULT_USER,
+            "password": PUBLIC_DEFAULT_PASS,
+            "psk": "",
+            "include_default_roots": True,
+            "extra_roots": "",
+            "extra_topics": "",
+        }
+
+
+def _safe_int(value: Any, default: int) -> int:
+    try:
+        parsed = int(value)
+    except (TypeError, ValueError):
+        return default
+    if parsed < 1 or parsed > 65535:
+        return default
+    return parsed
+
+
+def _normalize(data: dict[str, Any]) -> dict[str, Any]:
+    defaults = _settings_defaults()
+    return {
+        "enabled": bool(data.get("enabled", defaults["enabled"])),
+        "broker": str(data.get("broker", defaults["broker"]) or defaults["broker"]).strip(),
+        "port": _safe_int(data.get("port", defaults["port"]), defaults["port"]),
+        "username": str(data.get("username", defaults["username"]) or "").strip(),
+        "password": str(data.get("password", defaults["password"]) or ""),
+        "psk": str(data.get("psk", defaults["psk"]) or "").strip(),
+        "include_default_roots": bool(data.get("include_default_roots", defaults["include_default_roots"])),
+        "extra_roots": str(data.get("extra_roots", defaults["extra_roots"]) or "").strip(),
+        "extra_topics": str(data.get("extra_topics", defaults["extra_topics"]) or "").strip(),
+        "updated_at": _safe_int(data.get("updated_at", 0), 0),
+    }
+
+
+def read_meshtastic_mqtt_settings() -> dict[str, Any]:
+    global _cache, _cache_ts
+    now = time.monotonic()
+    if _cache is not None and (now - _cache_ts) < _CACHE_TTL:
+        return dict(_cache)
+    if not SETTINGS_FILE.exists():
+        result = {**_settings_defaults(), "updated_at": 0}
+    else:
+        try:
+            loaded = json.loads(SETTINGS_FILE.read_text(encoding="utf-8"))
+        except Exception:
+            loaded = {}
+        result = _normalize(loaded if isinstance(loaded, dict) else {})
+    _cache = result
+    _cache_ts = now
+    return dict(result)
+
+
+def write_meshtastic_mqtt_settings(**updates: Any) -> dict[str, Any]:
+    DATA_DIR.mkdir(parents=True, exist_ok=True)
+    existing = read_meshtastic_mqtt_settings()
+    next_data = dict(existing)
+    for key in (
+        "enabled",
+        "broker",
+        "port",
+        "username",
+        "password",
+        "psk",
+        "include_default_roots",
+        "extra_roots",
+        "extra_topics",
+    ):
+        if key in updates and updates[key] is not None:
+            next_data[key] = updates[key]
+    if "username" in updates and not str(updates.get("username") or "").strip() and "password" not in updates:
+        next_data["password"] = PUBLIC_DEFAULT_PASS
+    next_data["updated_at"] = int(time.time())
+    normalized = _normalize(next_data)
+    SETTINGS_FILE.write_text(json.dumps(normalized, indent=2), encoding="utf-8")
+    if os.name != "nt":
+        os.chmod(SETTINGS_FILE, 0o600)
+    global _cache, _cache_ts
+    _cache = normalized
+    _cache_ts = time.monotonic()
+    return dict(normalized)
+
+
+def redacted_meshtastic_mqtt_settings(data: dict[str, Any] | None = None) -> dict[str, Any]:
+    source = read_meshtastic_mqtt_settings() if data is None else dict(data)
+    username = str(source.get("username", "") or "")
+    uses_default_credentials = username in ("", PUBLIC_DEFAULT_USER) and str(source.get("password", "") or "") in (
+        "",
+        PUBLIC_DEFAULT_PASS,
+    )
+    return {
+        "enabled": bool(source.get("enabled")),
+        "broker": str(source.get("broker", "")),
+        "port": int(source.get("port", 1883) or 1883),
+        "username": "" if uses_default_credentials else username,
+        "uses_default_credentials": uses_default_credentials,
+        "has_password": bool(str(source.get("password", "") or "")),
+        "has_psk": bool(str(source.get("psk", "") or "")),
+        "include_default_roots": bool(source.get("include_default_roots", True)),
+        "extra_roots": str(source.get("extra_roots", "") or ""),
+        "extra_topics": str(source.get("extra_topics", "") or ""),
+        "updated_at": int(source.get("updated_at", 0) or 0),
+    }
+
+
+def mqtt_connection_config() -> tuple[str, int, str, str]:
+    data = read_meshtastic_mqtt_settings()
+    return (
+        str(data.get("broker") or "mqtt.meshtastic.org"),
+        int(data.get("port") or 1883),
+        str(data.get("username") or PUBLIC_DEFAULT_USER),
+        str(data.get("password") or PUBLIC_DEFAULT_PASS),
+    )
+
+
+def mqtt_bridge_enabled() -> bool:
+    return bool(read_meshtastic_mqtt_settings().get("enabled"))
+
+
+def mqtt_psk_hex() -> str:
+    return str(read_meshtastic_mqtt_settings().get("psk", "") or "").strip()
+
+
+def mqtt_subscription_settings() -> tuple[str, str, bool]:
+    data = read_meshtastic_mqtt_settings()
+    return (
+        str(data.get("extra_roots", "") or ""),
+        str(data.get("extra_topics", "") or ""),
+        bool(data.get("include_default_roots", True)),
+    )

backend/services/network_utils.py

@@ -73,7 +73,7 @@ def fetch_with_curl(url, method="GET", json_data=None, timeout=15, headers=None,
     both Python requests and the barebones Windows system curl.
     """
     default_headers = {
-        "User-Agent": "ShadowBroker-OSINT/0.9.7 (+https://github.com/BigBodyCobain/Shadowbroker; contact: bigbodycobain@gmail.com)",
+        "User-Agent": "ShadowBroker-OSINT/0.9.75 (+https://github.com/BigBodyCobain/Shadowbroker; contact: bigbodycobain@gmail.com)",
     }
     if headers:
         default_headers.update(headers)

backend/services/node_settings.py

@@ -10,7 +10,8 @@ _cache: dict | None = None
 _cache_ts: float = 0.0
 _CACHE_TTL = 5.0
 _DEFAULTS = {
-    "enabled": False,
+    "enabled": True,
+    "operator_disabled": False,
     "timemachine_enabled": False,
 }
 
@@ -35,8 +36,16 @@ def read_node_settings() -> dict:
         except Exception:
             result = {**_DEFAULTS, "updated_at": 0}
         else:
+            operator_disabled = bool(data.get("operator_disabled", False))
+            raw_enabled = data.get("enabled", _DEFAULTS["enabled"])
+            # v0.9.7 initially wrote enabled:false as a default/offline state,
+            # which accidentally blocked InfoNet participation. Treat legacy
+            # false-without-marker as auto-enabled; only an explicit operator
+            # disable should keep the participant sync loop off.
+            enabled = False if operator_disabled else bool(raw_enabled or "operator_disabled" not in data)
             result = {
-                "enabled": bool(data.get("enabled", _DEFAULTS["enabled"])),
+                "enabled": enabled,
+                "operator_disabled": operator_disabled,
                 "timemachine_enabled": bool(data.get("timemachine_enabled", _DEFAULTS["timemachine_enabled"])),
                 "updated_at": _safe_int(data.get("updated_at", 0) or 0),
             }
@@ -48,8 +57,10 @@ def read_node_settings() -> dict:
 def write_node_settings(*, enabled: bool | None = None, timemachine_enabled: bool | None = None) -> dict:
     DATA_DIR.mkdir(parents=True, exist_ok=True)
     existing = read_node_settings()
+    next_enabled = bool(existing.get("enabled", _DEFAULTS["enabled"])) if enabled is None else bool(enabled)
     payload = {
-        "enabled": bool(existing.get("enabled", _DEFAULTS["enabled"])) if enabled is None else bool(enabled),
+        "enabled": next_enabled,
+        "operator_disabled": bool(existing.get("operator_disabled", _DEFAULTS["operator_disabled"])) if enabled is None else not next_enabled,
         "timemachine_enabled": bool(existing.get("timemachine_enabled", _DEFAULTS["timemachine_enabled"])) if timemachine_enabled is None else bool(timemachine_enabled),
         "updated_at": int(time.time()),
     }

backend/services/shodan_connector.py

@@ -20,7 +20,7 @@ from cachetools import TTLCache
 logger = logging.getLogger(__name__)
 
 _SHODAN_BASE = "https://api.shodan.io"
-_USER_AGENT = "ShadowBroker/0.9.7 local Shodan connector"
+_USER_AGENT = "ShadowBroker/0.9.75 local Shodan connector"
 _REQUEST_TIMEOUT = 15
 _MIN_INTERVAL_SECONDS = 1.05  # Shodan docs say API plans are rate limited to ~1 req/sec.
 _DEFAULT_SEARCH_PAGES = 1

backend/services/sigint_bridge.py

@@ -22,6 +22,12 @@ from collections import deque
 from datetime import datetime, timezone
 
 from services.config import get_settings
+from services.meshtastic_mqtt_settings import (
+    mqtt_bridge_enabled,
+    mqtt_connection_config,
+    mqtt_psk_hex,
+    mqtt_subscription_settings,
+)
 from services.mesh.meshtastic_topics import all_available_roots, build_subscription_topics, known_roots, parse_topic_metadata
 
 logger = logging.getLogger("services.sigint")
@@ -477,22 +483,13 @@ class MeshtasticBridge:
     @staticmethod
     def _mqtt_config() -> tuple[str, int, str, str]:
         """Return (broker, port, user, password) from settings."""
-        try:
-            s = get_settings()
-            return (
-                str(s.MESH_MQTT_BROKER or "mqtt.meshtastic.org"),
-                int(s.MESH_MQTT_PORT or 1883),
-                str(s.MESH_MQTT_USER or "meshdev"),
-                str(s.MESH_MQTT_PASS or "large4cats"),
-            )
-        except Exception:
-            return ("mqtt.meshtastic.org", 1883, "meshdev", "large4cats")
+        return mqtt_connection_config()
 
     @classmethod
     def _resolve_psk(cls) -> bytes:
         """Return the PSK from config, or the default LongFast key if empty."""
         try:
-            raw = str(getattr(get_settings(), "MESH_MQTT_PSK", "") or "").strip()
+            raw = mqtt_psk_hex()
         except Exception:
             raw = ""
         if not raw:
@@ -506,6 +503,11 @@ class MeshtasticBridge:
         self._thread: threading.Thread | None = None
         self._stop = threading.Event()
         self._client_id = self._build_client_id()
+        self._connected = False
+        self._last_error = ""
+        self._last_connected_at = 0.0
+        self._last_disconnected_at = 0.0
+        self._last_broker = ""
         # Rate-limiter: sliding window of receive timestamps
         self._rx_timestamps: deque[float] = deque()
         self._rx_dropped = 0
@@ -518,10 +520,11 @@ class MeshtasticBridge:
         second client connects with the same id. Using a fixed id made separate
         ShadowBroker instances kick each other off the broker.
 
-        Includes the app version so the Meshtastic team can track our footprint.
+        This is deliberately not tied to the user's public mesh address or
+        ShadowBroker node identity; it is only an MQTT session handle.
         """
         suffix = uuid.uuid4().hex[:8]
-        return f"sb096-{suffix}"
+        return f"meshchat-{suffix}"
 
     def _dedupe_message(
         self,
@@ -544,7 +547,12 @@ class MeshtasticBridge:
 
     def start(self):
         if self._thread and self._thread.is_alive():
-            return
+            if not self._stop.is_set():
+                return
+            self._thread.join(timeout=2.0)
+            if self._thread.is_alive():
+                logger.warning("Meshtastic MQTT bridge is still stopping; start deferred")
+                return
         self._stop.clear()
         self._thread = threading.Thread(target=self._run, daemon=True, name="mesh-bridge")
         self._thread.start()
@@ -552,13 +560,37 @@ class MeshtasticBridge:
 
     def stop(self):
         self._stop.set()
+        self._connected = False
+
+    def is_running(self) -> bool:
+        return bool(self._thread and self._thread.is_alive() and not self._stop.is_set())
+
+    def status(self) -> dict:
+        broker, port, user, _pw = self._mqtt_config()
+        display_user = "" if user == "meshdev" else user
+        return {
+            "enabled": mqtt_bridge_enabled(),
+            "running": self.is_running(),
+            "connected": bool(self._connected),
+            "broker": broker,
+            "port": port,
+            "username": display_user,
+            "client_id": self._client_id,
+            "message_log_size": len(self.messages),
+            "signal_log_size": len(self.signals),
+            "last_error": self._last_error,
+            "last_broker": self._last_broker,
+            "last_connected_at": self._last_connected_at,
+            "last_disconnected_at": self._last_disconnected_at,
+            "rx_dropped": self._rx_dropped,
+        }
 
     def _subscription_topics(self) -> list[str]:
-        settings = get_settings()
+        extra_roots, extra_topics, include_defaults = mqtt_subscription_settings()
         return build_subscription_topics(
-            extra_roots=str(getattr(settings, "MESH_MQTT_EXTRA_ROOTS", "") or ""),
-            extra_topics=str(getattr(settings, "MESH_MQTT_EXTRA_TOPICS", "") or ""),
-            include_defaults=bool(getattr(settings, "MESH_MQTT_INCLUDE_DEFAULT_ROOTS", True)),
+            extra_roots=extra_roots,
+            extra_topics=extra_topics,
+            include_defaults=include_defaults,
         )
 
     def _run(self):
@@ -582,6 +614,9 @@ class MeshtasticBridge:
 
         def _on_connect(client, userdata, flags, rc):
             if rc == 0:
+                self._connected = True
+                self._last_error = ""
+                self._last_connected_at = time.time()
                 logger.info(
                     "Meshtastic MQTT connected (%s), subscribing to %s",
                     self._client_id,
@@ -590,6 +625,8 @@ class MeshtasticBridge:
                 for topic in topics:
                     client.subscribe(topic, qos=0)
             else:
+                self._connected = False
+                self._last_error = f"connect_refused:{rc}"
                 logger.error(
                     "Meshtastic MQTT connection refused (%s): rc=%s",
                     self._client_id,
@@ -597,7 +634,10 @@ class MeshtasticBridge:
                 )
 
         def _on_disconnect(client, userdata, rc):
+            self._connected = False
+            self._last_disconnected_at = time.time()
             if rc != 0:
+                self._last_error = f"disconnect:{rc}"
                 logger.warning(
                     "Meshtastic MQTT disconnected unexpectedly (%s, rc=%s), will auto-reconnect",
                     self._client_id,
@@ -607,6 +647,7 @@ class MeshtasticBridge:
                 logger.info("Meshtastic MQTT disconnected cleanly (%s)", self._client_id)
 
         broker, port, user, pw = self._mqtt_config()
+        self._last_broker = f"{broker}:{port}"
         client = mqtt.Client(client_id=self._client_id, protocol=mqtt.MQTTv311)
         client.username_pw_set(user, pw)
         client.on_connect = _on_connect
@@ -645,9 +686,6 @@ class MeshtasticBridge:
     def _on_message(self, client, userdata, msg):
         """Parse Meshtastic MQTT messages — protobuf + AES decryption."""
         try:
-            if self._rate_limited():
-                return
-
             payload = msg.payload
             topic = msg.topic
 
@@ -655,6 +693,8 @@ class MeshtasticBridge:
             if "/json/" in topic:
                 try:
                     data = json.loads(payload)
+                    if self._rate_limited():
+                        return
                     self._ingest_data(data, topic)
                     return
                 except (json.JSONDecodeError, UnicodeDecodeError):
@@ -687,6 +727,8 @@ class MeshtasticBridge:
                         }
                     )
                 else:
+                    if self._rate_limited():
+                        return
                     self._ingest_data(data, topic)
 
         except Exception as e:
@@ -1011,7 +1053,7 @@ class SIGINTGrid:
         self._started = True
         self.aprs.start()
         try:
-            mqtt_enabled = bool(getattr(get_settings(), "MESH_MQTT_ENABLED", False))
+            mqtt_enabled = mqtt_bridge_enabled()
         except Exception:
             mqtt_enabled = False
         if mqtt_enabled:
@@ -1123,13 +1165,12 @@ class SIGINTGrid:
             ch = msg.get("channel", "LongFast")
             channel_msgs[ch] = channel_msgs.get(ch, 0) + 1
 
+        extra_roots, _extra_topics, include_defaults = mqtt_subscription_settings()
+
         return {
             "regions": regions,
             "roots": roots,
-            "known_roots": known_roots(
-                str(getattr(get_settings(), "MESH_MQTT_EXTRA_ROOTS", "") or ""),
-                include_defaults=bool(getattr(get_settings(), "MESH_MQTT_INCLUDE_DEFAULT_ROOTS", True)),
-            ),
+            "known_roots": known_roots(extra_roots, include_defaults=include_defaults),
             "all_roots": all_available_roots(),
             "channel_messages": channel_msgs,
             "total_nodes": len(seen_callsigns),

backend/services/tor_hidden_service.py

@@ -1,13 +1,9 @@
-"""Tor Hidden Service auto-provisioner.
+"""Tor hidden-service auto-provisioner.
 
 Manages a Tor hidden service that points to the local ShadowBroker backend.
-Tor is started as a subprocess with a generated torrc — no manual config needed.
-Auto-installs the Tor Expert Bundle on Windows if not present.
-
-Usage:
-    from services.tor_hidden_service import tor_service
-    status = tor_service.start()   # -> {"ok": True, "onion_address": "http://xxxx.onion:8000"}
-    tor_service.stop()
+Tor is started as a subprocess with a generated torrc. Windows source installs
+can download the Tor Expert Bundle into backend/data without admin rights.
+Docker images should already include the `tor` package.
 """
 
 from __future__ import annotations
@@ -31,31 +27,33 @@ HOSTNAME_PATH = TOR_DIR / "hidden_service" / "hostname"
 TOR_DATA_DIR = TOR_DIR / "data"
 PIDFILE_PATH = TOR_DIR / "tor.pid"
 
-# Bundled Tor install location (inside our data dir so no admin rights needed)
+# Bundled Tor install location (inside data dir so no admin rights are needed).
 TOR_INSTALL_DIR = TOR_DIR / "tor_bin"
 
-# How long to wait for Tor to generate the hostname file
 _STARTUP_TIMEOUT_S = 90
 _POLL_INTERVAL_S = 1.0
 
-# Tor Expert Bundle download URL (Windows x86_64)
-_TOR_EXPERT_BUNDLE_URL = "https://dist.torproject.org/torbrowser/15.0.8/tor-expert-bundle-windows-x86_64-15.0.8.tar.gz"
+# Windows x86_64 Tor Expert Bundle URLs. Keep a fallback so first-run
+# onboarding does not break when Tor rotates point releases.
+_TOR_EXPERT_BUNDLE_URLS = [
+    "https://dist.torproject.org/torbrowser/15.0.11/tor-expert-bundle-windows-x86_64-15.0.11.tar.gz",
+    "https://dist.torproject.org/torbrowser/15.0.8/tor-expert-bundle-windows-x86_64-15.0.8.tar.gz",
+]
 
 
 def _find_tor_binary() -> str | None:
     """Locate the tor binary on the system, including our bundled install."""
-    # Check our bundled install first
     bundled = TOR_INSTALL_DIR / "tor" / "tor.exe"
     if bundled.exists():
         return str(bundled)
-    # Also check for extracted layout variants
+
     for sub in TOR_INSTALL_DIR.rglob("tor.exe"):
         return str(sub)
 
     tor = shutil.which("tor")
     if tor:
         return tor
-    # Common locations on Windows
+
     for candidate in [
         r"C:\Program Files\Tor Browser\Browser\TorBrowser\Tor\tor.exe",
         r"C:\Program Files (x86)\Tor Browser\Browser\TorBrowser\Tor\tor.exe",
@@ -67,77 +65,65 @@ def _find_tor_binary() -> str | None:
 
 
 def _auto_install_tor() -> str | None:
-    """Download and extract the Tor Expert Bundle. Returns path to tor binary or None."""
+    """Install or download Tor when it is safe to do so."""
     if os.name != "nt":
-        # On Linux/Mac, try package manager
-        try:
-            if shutil.which("apt-get"):
-                subprocess.run(["sudo", "apt-get", "install", "-y", "tor"], check=True, capture_output=True, timeout=120)
-            elif shutil.which("brew"):
-                subprocess.run(["brew", "install", "tor"], check=True, capture_output=True, timeout=120)
-            elif shutil.which("pacman"):
-                subprocess.run(["sudo", "pacman", "-S", "--noconfirm", "tor"], check=True, capture_output=True, timeout=120)
-            else:
-                logger.warning("No supported package manager found for auto-install")
-                return None
-            return shutil.which("tor")
-        except Exception as exc:
-            logger.error("Failed to auto-install Tor via package manager: %s", exc)
-            return None
+        # In Docker this should already be baked into the image. For source
+        # installs we avoid unattended sudo prompts from a web request path.
+        logger.warning("Tor is not installed. Install the tor package or use the Docker image with Tor baked in.")
+        return None
 
-    # Windows: download Tor Expert Bundle (no admin needed)
     TOR_INSTALL_DIR.mkdir(parents=True, exist_ok=True)
-    archive_path = TOR_INSTALL_DIR / "tor-expert-bundle.tar.gz"
-
-    try:
-        logger.info("Downloading Tor Expert Bundle over HTTPS from dist.torproject.org...")
-        urlretrieve(_TOR_EXPERT_BUNDLE_URL, str(archive_path))
-
-        # Verify SHA-256 of the downloaded archive
-        sha256_url = _TOR_EXPERT_BUNDLE_URL + ".sha256sum"
-        sha256_file = TOR_INSTALL_DIR / "sha256sum.txt"
+    for bundle_url in _TOR_EXPERT_BUNDLE_URLS:
+        archive_path = TOR_INSTALL_DIR / "tor-expert-bundle.tar.gz"
         try:
-            urlretrieve(sha256_url, str(sha256_file))
-            expected_hash = sha256_file.read_text().strip().split()[0].lower()
-            import hashlib
-            actual_hash = hashlib.sha256(archive_path.read_bytes()).hexdigest().lower()
-            sha256_file.unlink(missing_ok=True)
-            if actual_hash != expected_hash:
-                logger.error("SHA-256 MISMATCH — download may be compromised! Expected %s, got %s", expected_hash, actual_hash)
-                archive_path.unlink(missing_ok=True)
-                return None
-            logger.info("SHA-256 verified: %s", actual_hash[:16] + "...")
-        except Exception as hash_err:
-            # If we can't fetch the hash file, warn but proceed (HTTPS provides baseline integrity)
-            logger.warning("Could not verify SHA-256 (hash file unavailable): %s — proceeding with HTTPS-only verification", hash_err)
+            logger.info("Downloading Tor Expert Bundle over HTTPS from %s...", bundle_url)
+            urlretrieve(bundle_url, str(archive_path))
 
-        logger.info("Download complete, extracting...")
+            sha256_url = bundle_url + ".sha256sum"
+            sha256_file = TOR_INSTALL_DIR / "sha256sum.txt"
+            try:
+                urlretrieve(sha256_url, str(sha256_file))
+                expected_hash = sha256_file.read_text().strip().split()[0].lower()
+                import hashlib
 
-        # Extract .tar.gz with path traversal protection
-        import tarfile
-        with tarfile.open(str(archive_path), "r:gz") as tar:
-            for member in tar.getmembers():
-                member_path = (TOR_INSTALL_DIR / member.name).resolve()
-                if not str(member_path).startswith(str(TOR_INSTALL_DIR.resolve())):
-                    logger.error("Tar path traversal blocked: %s", member.name)
+                actual_hash = hashlib.sha256(archive_path.read_bytes()).hexdigest().lower()
+                sha256_file.unlink(missing_ok=True)
+                if actual_hash != expected_hash:
+                    logger.error("SHA-256 mismatch for Tor download. Expected %s, got %s", expected_hash, actual_hash)
                     archive_path.unlink(missing_ok=True)
-                    return None
-            tar.extractall(path=str(TOR_INSTALL_DIR))
+                    continue
+                logger.info("SHA-256 verified: %s", actual_hash[:16] + "...")
+            except Exception as hash_err:
+                logger.warning(
+                    "Could not verify SHA-256 (hash file unavailable): %s; proceeding with HTTPS-only verification",
+                    hash_err,
+                )
 
-        # Clean up archive
-        archive_path.unlink(missing_ok=True)
+            logger.info("Download complete, extracting...")
+            import tarfile
 
-        # Find the tor.exe in extracted files
-        for p in TOR_INSTALL_DIR.rglob("tor.exe"):
-            logger.info("Tor installed at: %s", p)
-            return str(p)
+            with tarfile.open(str(archive_path), "r:gz") as tar:
+                for member in tar.getmembers():
+                    member_path = (TOR_INSTALL_DIR / member.name).resolve()
+                    if not str(member_path).startswith(str(TOR_INSTALL_DIR.resolve())):
+                        logger.error("Tar path traversal blocked: %s", member.name)
+                        archive_path.unlink(missing_ok=True)
+                        return None
+                tar.extractall(path=str(TOR_INSTALL_DIR))
 
-        logger.error("tor.exe not found after extraction")
-        return None
-    except Exception as exc:
-        logger.error("Failed to download/extract Tor: %s", exc)
-        archive_path.unlink(missing_ok=True)
-        return None
+            archive_path.unlink(missing_ok=True)
+
+            for p in TOR_INSTALL_DIR.rglob("tor.exe"):
+                logger.info("Tor installed at: %s", p)
+                return str(p)
+
+            logger.error("tor.exe not found after extracting %s", bundle_url)
+        except Exception as exc:
+            logger.error("Failed to download/extract Tor from %s: %s", bundle_url, exc)
+        finally:
+            archive_path.unlink(missing_ok=True)
+
+    return None
 
 
 class TorHiddenService:
@@ -150,7 +136,6 @@ class TorHiddenService:
         self._running = False
         self._error: str = ""
 
-        # Check if we already have a hostname from a previous run
         if HOSTNAME_PATH.exists():
             try:
                 hostname = HOSTNAME_PATH.read_text().strip()
@@ -198,19 +183,20 @@ class TorHiddenService:
             self._error = ""
             tor_bin = _find_tor_binary()
             if not tor_bin:
-                logger.info("Tor not found, attempting auto-install...")
+                logger.info("Tor not found, attempting bootstrap...")
                 tor_bin = _auto_install_tor()
             if not tor_bin:
-                self._error = "Failed to auto-install Tor. Please install it manually."
+                self._error = (
+                    "Could not prepare Tor automatically. Check network access to dist.torproject.org "
+                    "or install Tor, then try again."
+                )
                 return {"ok": False, "detail": self._error}
 
-            # Create directories
             TOR_DIR.mkdir(parents=True, exist_ok=True)
             TOR_DATA_DIR.mkdir(parents=True, exist_ok=True)
             hidden_service_dir = TOR_DIR / "hidden_service"
             hidden_service_dir.mkdir(parents=True, exist_ok=True)
 
-            # On non-Windows, Tor requires strict permissions on HiddenServiceDir
             if os.name != "nt":
                 try:
                     os.chmod(str(hidden_service_dir), 0o700)
@@ -218,19 +204,15 @@ class TorHiddenService:
                 except OSError:
                     pass
 
-            # Write torrc — enables both hidden service (inbound) and SOCKS proxy
-            # (outbound) so the mesh/wormhole system can route node-to-node
-            # traffic through Tor as well.
             torrc_content = (
                 f"DataDirectory {TOR_DATA_DIR.as_posix()}\n"
                 f"HiddenServiceDir {hidden_service_dir.as_posix()}\n"
                 f"HiddenServicePort {target_port} 127.0.0.1:{target_port}\n"
-                f"SocksPort 9050\n"
-                f"Log notice stderr\n"
+                "SocksPort 9050\n"
+                "Log notice stderr\n"
             )
             TORRC_PATH.write_text(torrc_content, encoding="utf-8")
 
-            # Start Tor
             try:
                 self._process = subprocess.Popen(
                     [tor_bin, "-f", str(TORRC_PATH)],
@@ -245,15 +227,12 @@ class TorHiddenService:
                 logger.error(self._error)
                 return {"ok": False, "detail": self._error}
 
-            # Wait for hostname file to appear
             deadline = time.monotonic() + _STARTUP_TIMEOUT_S
             while time.monotonic() < deadline:
                 if self._process.poll() is not None:
-                    # Tor exited prematurely
                     stdout = self._process.stdout.read() if self._process.stdout else ""
                     self._error = f"Tor exited with code {self._process.returncode}"
                     if stdout:
-                        # Get last few lines for error context
                         lines = stdout.strip().split("\n")
                         self._error += ": " + " | ".join(lines[-3:])
                     self._running = False
@@ -273,7 +252,6 @@ class TorHiddenService:
 
                 time.sleep(_POLL_INTERVAL_S)
 
-            # Timeout
             self._error = f"Tor did not generate hostname within {_STARTUP_TIMEOUT_S}s"
             self.stop()
             return {"ok": False, "detail": self._error}
@@ -292,10 +270,7 @@ class TorHiddenService:
                         pass
                 self._process = None
             self._running = False
-            # Keep the onion_address — it persists across restarts
-            # since the key is stored in hidden_service_dir
             return {"ok": True, "detail": "stopped"}
 
 
-# Singleton
 tor_service = TorHiddenService()

backend/services/unusual_whales_connector.py

@@ -24,7 +24,7 @@ from cachetools import TTLCache
 logger = logging.getLogger(__name__)
 
 _FINNHUB_BASE = "https://finnhub.io/api/v1"
-_USER_AGENT = "ShadowBroker/0.9.7 Finnhub connector"
+_USER_AGENT = "ShadowBroker/0.9.75 Finnhub connector"
 _REQUEST_TIMEOUT = 12
 _MIN_INTERVAL_SECONDS = 0.35  # Stay well under 60 calls/min