Close tg12 outbound audit (#348-#366): operator UA, opt-ins, docs

- User-Agent is per-install handle only (no Shadowbroker product token)
- LiveUAMap: Windows UI consent when enabling Global Incidents; env override
- Meshtastic callsign upstream header off by default (opt-in true)
- Expanded docs/OUTBOUND_DATA.md and README link for CCTV, basemap, Broadcastify

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/services/fetchers/aircraft_database.py

@@ -38,8 +38,6 @@ _S3_NS = "{http://s3.amazonaws.com/doc/2006-03-01/}"
 _REFRESH_INTERVAL_S = 5 * 24 * 3600
 _LIST_TIMEOUT_S = 30
 _DOWNLOAD_TIMEOUT_S = 600
-from services.network_utils import DEFAULT_USER_AGENT as _USER_AGENT
-
 _lock = threading.RLock()
 _aircraft_by_hex: dict[str, dict[str, str]] = {}
 _last_refresh = 0.0

backend/services/fetchers/geo.py

@@ -20,17 +20,9 @@ def _env_flag(name: str) -> str:
 
 
 def liveuamap_scraper_enabled() -> bool:
-    """Return whether the Playwright-based LiveUAMap scraper should run.
+    from services.liveuamap_settings import liveuamap_scraper_enabled as _enabled
 
-    It is useful enrichment, but it starts a browser/Node driver and must not be
-    allowed to destabilize Windows local startup.
-    """
-    setting = _env_flag("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER")
-    if setting in {"1", "true", "yes", "on"}:
-        return True
-    if setting in {"0", "false", "no", "off"}:
-        return False
-    return os.name != "nt"
+    return _enabled()
 
 
 # ---------------------------------------------------------------------------
@@ -210,10 +202,17 @@ def update_liveuamap():
     if not is_any_active("global_incidents"):
         return
     if not liveuamap_scraper_enabled():
-        logger.info(
-            "Liveuamap scraper disabled for this runtime; set "
-            "SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=1 to opt in."
-        )
+        from services.liveuamap_settings import liveuamap_requires_ui_opt_in
+
+        if liveuamap_requires_ui_opt_in():
+            logger.info(
+                "Liveuamap scraper disabled: enable Global Incidents in the UI to "
+                "consent, or set SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=1."
+            )
+        else:
+            logger.info(
+                "Liveuamap scraper disabled; set SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=1 to opt in."
+            )
         return
     logger.info("Running scheduled Liveuamap scraper...")
     try:

backend/services/fetchers/meshtastic_map.py

@@ -188,8 +188,8 @@ def fetch_meshtastic_nodes():
         callsign = ""
 
     send_callsign_header = str(
-        _os.environ.get("MESHTASTIC_SEND_CALLSIGN_HEADER", "true")
-    ).strip().lower() not in {"0", "false", "no", "off", ""}
+        _os.environ.get("MESHTASTIC_SEND_CALLSIGN_HEADER", "false")
+    ).strip().lower() in {"1", "true", "yes", "on"}
 
     # Round 7a: outbound_user_agent already includes the per-install handle.
     # The optional Meshtastic callsign is appended as additional context so

backend/services/fetchers/route_database.py

@@ -30,8 +30,6 @@ _AIRPORTS_URL = "https://vrs-standing-data.adsb.lol/airports.csv.gz"
 _REFRESH_INTERVAL_S = 5 * 24 * 3600
 _HTTP_TIMEOUT_S = 60
 
-from services.network_utils import DEFAULT_USER_AGENT as _USER_AGENT
-
 _lock = threading.RLock()
 _routes_by_callsign: dict[str, dict[str, Any]] = {}
 _airports_by_icao: dict[str, dict[str, Any]] = {}

backend/services/liveuamap_scraper.py

@@ -27,8 +27,15 @@ def fetch_liveuamap():
         browser = p.chromium.launch(
             headless=True, args=["--disable-blink-features=AutomationControlled"]
         )
+        from services.network_utils import outbound_user_agent
+
+        # Per-install handle (no shared Shadowbroker product token). Stealth remains
+        # for Turnstile; see docs/OUTBOUND_DATA.md #348.
+        playwright_ua = (
+            f"Mozilla/5.0 (compatible; {outbound_user_agent('liveuamap')})"
+        )
         context = browser.new_context(
-            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+            user_agent=playwright_ua,
             viewport={"width": 1920, "height": 1080},
             color_scheme="dark",
         )

backend/services/liveuamap_settings.py

@@ -0,0 +1,73 @@
+"""LiveUAMap Playwright scraper opt-in (#348) — UI consent on Windows."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import threading
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+_OPT_IN_FILE = Path(__file__).resolve().parent.parent / "data" / "liveuamap_scraper_opt_in.json"
+_OPT_IN_LOCK = threading.Lock()
+
+
+def _env_flag(name: str) -> str:
+    return str(os.getenv(name, "")).strip().lower()
+
+
+def liveuamap_requires_ui_opt_in() -> bool:
+    """Windows local installs need explicit consent before Playwright contacts LiveUAMap."""
+    return os.name == "nt"
+
+
+def get_liveuamap_ui_opt_in() -> bool:
+    if not _OPT_IN_FILE.exists():
+        return False
+    try:
+        payload = json.loads(_OPT_IN_FILE.read_text(encoding="utf-8"))
+        return bool(payload.get("opted_in"))
+    except (OSError, json.JSONDecodeError, TypeError) as e:
+        logger.warning("LiveUAMap opt-in file unreadable: %s", e)
+        return False
+
+
+def set_liveuamap_ui_opt_in(opted_in: bool) -> None:
+    _OPT_IN_FILE.parent.mkdir(parents=True, exist_ok=True)
+    with _OPT_IN_LOCK:
+        _OPT_IN_FILE.write_text(
+            json.dumps({"opted_in": bool(opted_in)}, indent=2),
+            encoding="utf-8",
+        )
+
+
+def liveuamap_scraper_enabled() -> bool:
+    """Whether the Playwright LiveUAMap scraper may run on this backend."""
+    setting = _env_flag("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER")
+    if setting in {"1", "true", "yes", "on"}:
+        return True
+    if setting in {"0", "false", "no", "off"}:
+        return False
+    if not liveuamap_requires_ui_opt_in():
+        return True
+    return get_liveuamap_ui_opt_in()
+
+
+def liveuamap_scraper_status() -> dict[str, Any]:
+    setting = _env_flag("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER")
+    env_override = None
+    if setting in {"1", "true", "yes", "on"}:
+        env_override = "on"
+    elif setting in {"0", "false", "no", "off"}:
+        env_override = "off"
+    ui_opted_in = get_liveuamap_ui_opt_in()
+    requires = liveuamap_requires_ui_opt_in()
+    return {
+        "platform_requires_opt_in": requires,
+        "ui_opted_in": ui_opted_in,
+        "scraper_enabled": liveuamap_scraper_enabled(),
+        "env_override": env_override,
+    }

backend/services/mesh/mesh_wormhole_prekey.py

@@ -234,12 +234,12 @@ def _fetch_dm_prekey_bundle_from_public_lookup(lookup_token: str) -> dict[str, A
         # Generic UA: any peer-facing crypto request should not carry a
         # fork-specific identifier — that turns prekey lookups into a
         # software-fingerprinting beacon.
-        from services.network_utils import DEFAULT_USER_AGENT
+        from services.network_utils import default_user_agent
         request = urllib.request.Request(
             f"{normalized_peer_url}/api/mesh/dm/prekey-bundle?{encoded}",
             headers={
                 "Accept": "application/json",
-                "User-Agent": DEFAULT_USER_AGENT,
+                "User-Agent": default_user_agent(),
             },
             method="GET",
         )

backend/services/network_utils.py

@@ -34,9 +34,9 @@ _session.mount("http://", HTTPAdapter(max_retries=_retry, pool_maxsize=10))
 # upstream's only recourse was to block "Shadowbroker" as a whole — which
 # would take out every other install too.
 #
-# Fix: give each install a stable pseudonymous handle and include it in
-# the User-Agent. Now an upstream can rate-limit or block the offending
-# operator without affecting anyone else.
+# Fix: give each install a stable pseudonymous handle used as the entire
+# User-Agent product token (no shared "Shadowbroker" label). Upstreams see
+# ``operator-7f3a92`` (or ``OPERATOR_HANDLE``), not one monolithic app name.
 #
 # The handle:
 #
@@ -51,7 +51,6 @@ _session.mount("http://", HTTPAdapter(max_retries=_retry, pool_maxsize=10))
 # - Is NEVER mixed into mesh / Wormhole / Infonet identity. This layer is
 #   strictly for public third-party API attribution.
 
-_SHADOWBROKER_VERSION = "0.9"
 _OPERATOR_HANDLE_FILE = (
     Path(__file__).parent.parent / "data" / "operator_handle.json"
 )
@@ -175,41 +174,21 @@ def _normalize_handle(raw: str) -> str:
     return safe[:48] if safe else "anonymous"
 
 
-_CONTACT_URL = "https://github.com/BigBodyCobain/Shadowbroker/issues"
-
-
 def outbound_user_agent(purpose: str = "") -> str:
     """Build a User-Agent for an outbound third-party HTTP request.
 
-    Returns something like::
+    Returns the per-install handle only, e.g. ``operator-7f3a92`` or
+    ``operator-7f3a92 (purpose: wikipedia)``. No shared project name — so
+    upstream abuse teams cannot block every install with one ``Shadowbroker``
+    rule.
 
-        Shadowbroker/0.9 (operator: operator-7f3a92; purpose: wikipedia;
-         +https://github.com/BigBodyCobain/Shadowbroker/issues)
-
-    The ``purpose`` is optional but recommended — it tells the upstream
-    what feature of ours is making the call (``wikipedia``, ``openmhz``,
-    ``nominatim``, etc.), which makes their logs and our complaints
-    actionable.
-
-    Every outbound call in the backend that previously sent a custom
-    User-Agent should call this helper instead. Centralizing here means:
-      - one place to change the contact URL,
-      - one place to bump the version on release,
-      - one place a Wikimedia / OpenMHz operator can reach to ask for
-        the project to back off, with a per-install handle so they can
-        target the specific install instead of the project as a whole.
+    Set ``SHADOWBROKER_USER_AGENT`` to override the entire string if needed.
     """
     handle = get_operator_handle()
     if purpose:
         purpose_clean = _normalize_handle(purpose)
-        return (
-            f"Shadowbroker/{_SHADOWBROKER_VERSION} "
-            f"(operator: {handle}; purpose: {purpose_clean}; +{_CONTACT_URL})"
-        )
-    return (
-        f"Shadowbroker/{_SHADOWBROKER_VERSION} "
-        f"(operator: {handle}; +{_CONTACT_URL})"
-    )
+        return f"{handle} (purpose: {purpose_clean})"
+    return handle
 
 
 def _reset_operator_handle_cache_for_tests() -> None:
@@ -220,19 +199,13 @@ def _reset_operator_handle_cache_for_tests() -> None:
         _OPERATOR_HANDLE_CACHE = ""
 
 
-# Default outbound User-Agent. Retained for backwards compatibility with
-# call sites that haven't been migrated to ``outbound_user_agent()`` yet.
-# Operators who want full per-install attribution should set the
-# ``OPERATOR_HANDLE`` setting and migrate call sites incrementally.
-#
-# Operators who run a public-facing relay can also override the whole UA
-# string via the ``SHADOWBROKER_USER_AGENT`` env var. That override
-# completely bypasses the per-operator helper; only use it if you know
-# what you're doing.
-DEFAULT_USER_AGENT = os.environ.get(
-    "SHADOWBROKER_USER_AGENT",
-    f"Shadowbroker/{_SHADOWBROKER_VERSION}",
-)
+def default_user_agent() -> str:
+    """Default User-Agent for ``fetch_with_curl`` and legacy call sites."""
+    custom = (os.environ.get("SHADOWBROKER_USER_AGENT") or "").strip()
+    if custom:
+        return custom
+    return outbound_user_agent()
+
 
 # Find bash for curl fallback — Git bash's curl has the TLS features
 # needed to pass CDN fingerprint checks (brotli, zstd, libpsl)
@@ -288,7 +261,7 @@ def fetch_with_curl(url, method="GET", json_data=None, timeout=15, headers=None,
     both Python requests and the barebones Windows system curl.
     """
     default_headers = {
-        "User-Agent": DEFAULT_USER_AGENT,
+        "User-Agent": default_user_agent(),
     }
     if headers:
         default_headers.update(headers)