Merge pull request #518 from BigBodyCobain/fix/liveuamap-resilience-516-517

fix(liveuamap): restore resilient optional incident enrichment

.env.example

@@ -77,10 +77,20 @@ ADMIN_KEY=
 # NUFORC_MAPBOX_TOKEN=
 
 # Optional startup-risk controls.
-# On Windows, external curl fallback is off by default. LiveUAMap uses UI consent
-# when you enable Global Incidents (or set SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=true).
+# On Windows, external curl fallback is off by default. Global Incidents always
+# has the GDELT baseline; the LiveUAMap browser provider is optional enrichment.
+# Windows asks once when you enable Global Incidents. Declining keeps GDELT on.
+# Linux/macOS/Docker preserve the existing browser-enrichment default unless
+# SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER is explicitly false.
 # SHADOWBROKER_ENABLE_WINDOWS_CURL_FALLBACK=false
 # SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=false
+# Optional paid/contracted LiveUAMap API provider. Shadowbroker does not require
+# this. Configure the final HTTPS JSON/GeoJSON endpoint supplied to you.
+# LIVEUAMAP_API_URL=
+# LIVEUAMAP_API_KEY=
+# LIVEUAMAP_API_AUTH_HEADER=Authorization
+# LIVEUAMAP_API_AUTH_SCHEME=Bearer
+# LIVEUAMAP_API_TIMEOUT_S=30
 # AIS starts by default when AIS_API_KEY is set. Set to 0/false to force-disable.
 # SHADOWBROKER_ENABLE_AIS_STREAM_PROXY=true
 # Minimum visible satellite catalog before forcing a CelesTrak refresh.

.github/workflows/ci.yml

@@ -79,4 +79,8 @@ jobs:
             tests/mesh/test_mesh_canonical.py \
             tests/mesh/test_mesh_merkle.py \
             tests/test_release_helper.py \
+            tests/test_liveuamap_scraper_opt_in.py \
+            tests/test_liveuamap_parser.py \
+            tests/test_liveuamap_provider.py \
+            tests/test_liveuamap_docker_contract.py \
             -v --tb=short

backend/Dockerfile

@@ -40,14 +40,24 @@ ENV PATH="/root/.local/bin:$PATH"
 # Install into system Python (no venv needed inside container)
 ENV UV_PROJECT_ENVIRONMENT=/usr/local
 
+# Playwright installs browsers under the current user's cache by default. The
+# dependency install below runs as root while the backend runs as backenduser,
+# which caused runtime Playwright to look in /app/.cache for a browser that had
+# actually been baked under /root/.cache (#516). Use one image-wide location
+# that is readable by the non-root runtime user.
+ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
+
 # Copy workspace root files for UV resolution (build context is repo root)
 COPY pyproject.toml /workspace/pyproject.toml
 COPY uv.lock /workspace/uv.lock
 COPY backend/pyproject.toml /workspace/backend/pyproject.toml
 
-# Install Python dependencies using the lockfile
+# Install Python dependencies using the lockfile. Playwright's Chromium install
+# also includes the matching headless-shell bundle used by default headless
+# launches; the runtime-user assertion below verifies both are actually present.
 RUN cd /workspace/backend && uv sync --frozen --no-dev --extra road-corridor \
-    && playwright install --with-deps chromium
+    && playwright install --with-deps chromium \
+    && chmod -R a+rX "$PLAYWRIGHT_BROWSERS_PATH"
 
 # Copy backend source code
 COPY backend/ .
@@ -81,6 +91,14 @@ RUN adduser --system --uid 1001 --home /app backenduser \
 # Switch to the non-root user
 USER backenduser
 
+# Build-time packaging assertion for #516. Do not launch Chromium here because
+# downstream users may cross-build under emulation; instead verify that the exact
+# runtime user resolves an executable Chromium and can see the architecture-
+# specific headless-shell binary Playwright's default headless launch expects.
+# Playwright names that binary chrome-headless-shell on Linux x64 and
+# headless_shell on Linux arm64.
+RUN python -c "import os; from pathlib import Path; from playwright.sync_api import sync_playwright; p=sync_playwright().start(); path=p.chromium.executable_path; assert os.path.isfile(path), path; assert os.access(path, os.X_OK), path; root=Path(os.environ['PLAYWRIGHT_BROWSERS_PATH']); shells=[s for s in root.glob('chromium_headless_shell-*/*/*') if s.name in {'chrome-headless-shell','headless_shell'} and s.is_file() and os.access(s, os.X_OK)]; assert shells, list(root.glob('chromium_headless_shell-*/*/*')); print('Playwright runtime browser:', path, 'headless-shell:', shells[0]); p.stop()"
+
 # Expose port
 EXPOSE 8000
 

backend/services/liveuamap_parser.py

@@ -0,0 +1,250 @@
+"""Defensive parsing helpers for LiveUAMap provider payloads.
+
+LiveUAMap's browser page exposes an undocumented ``ovens`` value whose shape
+has changed over time. The optional supported API may also return JSON or
+GeoJSON. Keep representation decoding and schema normalization isolated here so
+upstream drift degrades one provider instead of crashing the fetch scheduler.
+"""
+
+from __future__ import annotations
+
+import ast
+import base64
+import binascii
+import json
+import math
+import re
+from collections.abc import Iterable
+from typing import Any
+from urllib.parse import unquote
+
+_MAX_DECODE_DEPTH = 5
+_MAX_CANDIDATES = 10_000
+_MAX_STRING_BYTES = 8 * 1024 * 1024
+_WRAPPER_KEYS = ("ovens", "markers", "items", "events", "data", "results", "features")
+
+
+def extract_ovens_expression(html: str) -> str | None:
+    """Extract a legacy ``var/let/const ovens = ...;`` expression from HTML.
+
+    Evaluating ``window.ovens`` in the browser is preferred; this exists only
+    as a fallback for pages that still embed the value in source text.
+    """
+    if not html:
+        return None
+    match = re.search(
+        r"(?:var|let|const)\s+ovens\s*=\s*(.+?);(?=\s*(?:</script>|(?:var|let|const|function)\b|$))",
+        html,
+        re.DOTALL | re.IGNORECASE,
+    )
+    if match:
+        return match.group(1).strip()
+
+    # Compatibility with older pages where the next token after the semicolon
+    # is arbitrary markup rather than another JavaScript declaration.
+    match = re.search(r"(?:var|let|const)\s+ovens\s*=\s*(.*?);", html, re.DOTALL | re.IGNORECASE)
+    return match.group(1).strip() if match else None
+
+
+def payload_shape(value: Any) -> str:
+    """Return a non-sensitive structural description for drift diagnostics."""
+    if isinstance(value, dict):
+        keys = sorted(str(key) for key in value.keys())[:8]
+        return f"dict(keys={keys}, size={len(value)})"
+    if isinstance(value, list):
+        item_types = sorted({type(item).__name__ for item in value[:20]})
+        return f"list(size={len(value)}, item_types={item_types})"
+    if isinstance(value, str):
+        return f"str(len={len(value)})"
+    return type(value).__name__
+
+
+def normalize_liveuamap_payload(value: Any) -> list[dict[str, Any]]:
+    """Normalize JSON/GeoJSON/legacy payload shapes into marker dictionaries.
+
+    Unknown or malformed values are ignored. This function intentionally never
+    assumes iterable items are mappings; issue #517 was caused by calling
+    ``.get`` on strings after an upstream representation change.
+    """
+    out: list[dict[str, Any]] = []
+    _collect(value, out, depth=0, inherited_id=None)
+    return out[:_MAX_CANDIDATES]
+
+
+def _collect(
+    value: Any,
+    out: list[dict[str, Any]],
+    *,
+    depth: int,
+    inherited_id: str | None,
+) -> None:
+    if depth > _MAX_DECODE_DEPTH or len(out) >= _MAX_CANDIDATES:
+        return
+
+    if value is None:
+        return
+
+    if isinstance(value, str):
+        decoded = _decode_string(value)
+        if decoded is value or decoded == value:
+            return
+        _collect(decoded, out, depth=depth + 1, inherited_id=inherited_id)
+        return
+
+    if isinstance(value, list):
+        for item in value[:_MAX_CANDIDATES - len(out)]:
+            _collect(item, out, depth=depth + 1, inherited_id=None)
+            if len(out) >= _MAX_CANDIDATES:
+                break
+        return
+
+    if not isinstance(value, dict):
+        return
+
+    # GeoJSON FeatureCollection / Feature.
+    if value.get("type") == "FeatureCollection" and isinstance(value.get("features"), list):
+        _collect(value["features"], out, depth=depth + 1, inherited_id=None)
+        return
+    if value.get("type") == "Feature":
+        marker = _marker_from_geojson_feature(value)
+        if marker is not None:
+            if inherited_id and not marker.get("id"):
+                marker["id"] = inherited_id
+            out.append(marker)
+        return
+
+    # Coordinate-bearing objects are markers even if they also contain a field
+    # named `data`/`events`. Check them before generic wrapper traversal so a
+    # legitimate marker cannot be swallowed by wrapper heuristics.
+    if _looks_like_marker(value):
+        marker = dict(value)
+        if inherited_id and not marker.get("id"):
+            marker["id"] = inherited_id
+        out.append(marker)
+        return
+
+    # Common wrapper shapes returned by APIs or page-side serialization.
+    for key in _WRAPPER_KEYS:
+        if key in value and isinstance(value[key], (dict, list, str)):
+            _collect(value[key], out, depth=depth + 1, inherited_id=None)
+            return
+
+    # Some versions expose a dictionary keyed by marker ID. Traverse mapping
+    # values while preserving the key as a fallback identifier.
+    traversable = [
+        (str(key), item)
+        for key, item in value.items()
+        if isinstance(item, (dict, list, str))
+    ]
+    if traversable:
+        for key, item in traversable[:_MAX_CANDIDATES - len(out)]:
+            _collect(item, out, depth=depth + 1, inherited_id=key)
+            if len(out) >= _MAX_CANDIDATES:
+                break
+
+
+def _decode_string(raw: str) -> Any:
+    text = raw.strip()
+    if not text or len(text.encode("utf-8", errors="ignore")) > _MAX_STRING_BYTES:
+        return raw
+
+    # A JavaScript string literal can include escaping that plain strip("'")
+    # corrupts. literal_eval safely handles quoted string syntax only.
+    if len(text) >= 2 and text[0] == text[-1] and text[0] in {"'", '"'}:
+        try:
+            literal = ast.literal_eval(text)
+        except (SyntaxError, ValueError):
+            literal = None
+        if isinstance(literal, str) and literal != text:
+            return literal
+
+    decoded = _try_json(text)
+    if decoded is not None:
+        return decoded
+
+    # Legacy LiveUAMap payloads have appeared URL-encoded before decoding.
+    url_decoded = unquote(text)
+    if url_decoded != text:
+        decoded = _try_json(url_decoded)
+        if decoded is not None:
+            return decoded
+        text = url_decoded
+
+    # Older scraper versions expected a base64-wrapped JSON blob. Decode only
+    # when the result itself is valid JSON, so arbitrary titles/IDs are never
+    # interpreted as base64 data.
+    compact = "".join(text.split())
+    if compact and len(compact) % 4 == 0:
+        try:
+            raw_bytes = base64.b64decode(compact, validate=True)
+            decoded_text = raw_bytes.decode("utf-8")
+        except (binascii.Error, UnicodeDecodeError, ValueError):
+            decoded_text = ""
+        if decoded_text:
+            decoded = _try_json(decoded_text)
+            if decoded is not None:
+                return decoded
+
+    return raw
+
+
+def _try_json(text: str) -> Any | None:
+    try:
+        return json.loads(text)
+    except (json.JSONDecodeError, TypeError, ValueError):
+        return None
+
+
+def _looks_like_marker(value: dict[str, Any]) -> bool:
+    keys = set(value)
+    if {"lat", "lng"}.issubset(keys) or {"lat", "lon"}.issubset(keys):
+        return True
+    if "latitude" in keys and ("longitude" in keys or "lon" in keys or "lng" in keys):
+        return True
+    marker_metadata = {"id", "s", "title", "d", "desc", "description", "link", "url", "time", "t"}
+    return bool(keys.intersection(marker_metadata)) and not any(key in value for key in _WRAPPER_KEYS)
+
+
+def _marker_from_geojson_feature(feature: dict[str, Any]) -> dict[str, Any] | None:
+    geometry = feature.get("geometry")
+    if not isinstance(geometry, dict) or geometry.get("type") != "Point":
+        return None
+    coordinates = geometry.get("coordinates")
+    if not isinstance(coordinates, (list, tuple)) or len(coordinates) < 2:
+        return None
+    lng = _finite_coordinate(coordinates[0], minimum=-180.0, maximum=180.0)
+    lat = _finite_coordinate(coordinates[1], minimum=-90.0, maximum=90.0)
+    if lat is None or lng is None:
+        return None
+    properties = feature.get("properties")
+    marker = dict(properties) if isinstance(properties, dict) else {}
+    marker.setdefault("lat", lat)
+    marker.setdefault("lng", lng)
+    feature_id = feature.get("id")
+    if feature_id is not None:
+        marker.setdefault("id", feature_id)
+    return marker
+
+
+def _finite_coordinate(value: Any, *, minimum: float, maximum: float) -> float | None:
+    try:
+        number = float(value)
+    except (TypeError, ValueError):
+        return None
+    if not math.isfinite(number) or not minimum <= number <= maximum:
+        return None
+    return number
+
+
+def iter_valid_coordinates(markers: Iterable[dict[str, Any]]) -> Iterable[tuple[dict[str, Any], float, float]]:
+    """Yield markers with finite in-range latitude/longitude values."""
+    for marker in markers:
+        if not isinstance(marker, dict):
+            continue
+        lat = marker.get("lat", marker.get("latitude"))
+        lng = marker.get("lng", marker.get("lon", marker.get("longitude")))
+        lat_value = _finite_coordinate(lat, minimum=-90.0, maximum=90.0)
+        lng_value = _finite_coordinate(lng, minimum=-180.0, maximum=180.0)
+        if lat_value is None or lng_value is None:
+            continue
+        yield marker, lat_value, lng_value

backend/services/liveuamap_scraper.py

@@ -1,145 +1,490 @@
-import json
+"""Resilient LiveUAMap enrichment providers.
+
+Global Incidents itself is backed independently by GDELT. This module adds
+LiveUAMap pins when either an operator-configured supported API is available or
+the existing Playwright provider is allowed. Provider failures are isolated and
+return an empty enrichment set instead of breaking the fetch scheduler.
+
+The browser provider intentionally does not add any new anti-bot behavior. It
+retains the repository's pre-existing Playwright/stealth profile for backward
+compatibility while making parsing, packaging failures, and upstream drift
+fail-soft.
+"""
+
+from __future__ import annotations
+
+import hashlib
 import logging
-import base64
-import urllib.parse
-import re
-from playwright.sync_api import sync_playwright
-from playwright_stealth import stealth_sync
+import math
+import os
+import threading
+import time
+from datetime import datetime, timezone
+from typing import Any
+from urllib.parse import urljoin, urlparse
+
+import requests
+
+from services.liveuamap_parser import (
+    extract_ovens_expression,
+    iter_valid_coordinates,
+    normalize_liveuamap_payload,
+    payload_shape,
+)
 
 logger = logging.getLogger(__name__)
 
+_REGIONS = (
+    {"name": "Ukraine", "url": "https://liveuamap.com"},
+    {"name": "Middle East", "url": "https://mideast.liveuamap.com"},
+    {"name": "Israel-Palestine", "url": "https://israelpalestine.liveuamap.com"},
+    {"name": "Syria", "url": "https://syria.liveuamap.com"},
+)
 
-def fetch_liveuamap():
-    logger.info("Starting Liveuamap scraper with Playwright Stealth...")
+_BROWSER_FAILURE_THRESHOLD = 3
+_BROWSER_BACKOFF_BASE_S = 15 * 60
+_BROWSER_BACKOFF_MAX_S = 6 * 60 * 60
+_browser_failures = 0
+_browser_blocked_until = 0.0
+_browser_health_lock = threading.Lock()
 
-    regions = [
-        {"name": "Ukraine", "url": "https://liveuamap.com"},
-        {"name": "Middle East", "url": "https://mideast.liveuamap.com"},
-        {"name": "Israel-Palestine", "url": "https://israelpalestine.liveuamap.com"},
-        {"name": "Syria", "url": "https://syria.liveuamap.com"},
-    ]
+_CHALLENGE_MARKERS = (
+    "cf-turnstile",
+    "challenge-platform",
+    "just a moment",
+    "checking your browser",
+    "verify you are human",
+)
 
-    all_markers = []
-    seen_ids = set()
 
-    with sync_playwright() as p:
-        # Launching with a real user agent to bypass Turnstile
-        browser = p.chromium.launch(
-            headless=True, args=["--disable-blink-features=AutomationControlled"]
+def _bounded_int_env(name: str, default: int, *, minimum: int, maximum: int) -> int:
+    try:
+        value = int(str(os.getenv(name, default)).strip())
+    except (TypeError, ValueError):
+        value = default
+    return max(minimum, min(maximum, value))
+
+
+def _safe_header_name(raw: str, default: str) -> str:
+    value = (raw or "").strip()
+    if not value or any(ch in value for ch in "\r\n:"):
+        return default
+    if not all(ch.isalnum() or ch in "-_" for ch in value):
+        return default
+    return value
+
+
+def _api_url() -> str:
+    raw = str(os.getenv("LIVEUAMAP_API_URL", "") or "").strip()
+    if not raw:
+        return ""
+    try:
+        parsed = urlparse(raw)
+    except ValueError:
+        return ""
+    if (
+        parsed.scheme.lower() != "https"
+        or not parsed.hostname
+        or parsed.username is not None
+        or parsed.password is not None
+    ):
+        return ""
+    return raw
+
+
+def _public_api_base_url(raw: str) -> str:
+    """Return a URL safe to use for relative links without exposing query auth."""
+    parsed = urlparse(raw)
+    return parsed._replace(query="", fragment="").geturl()
+
+
+def _api_headers() -> dict[str, str]:
+    from services.network_utils import outbound_user_agent
+
+    headers = {
+        "Accept": "application/geo+json, application/json;q=0.9",
+        "User-Agent": outbound_user_agent("liveuamap-api"),
+    }
+    api_key = str(os.getenv("LIVEUAMAP_API_KEY", "") or "").strip()
+    if api_key:
+        header = _safe_header_name(
+            str(os.getenv("LIVEUAMAP_API_AUTH_HEADER", "Authorization") or ""),
+            "Authorization",
         )
-        from services.network_utils import outbound_user_agent
+        scheme = str(os.getenv("LIVEUAMAP_API_AUTH_SCHEME", "Bearer") or "").strip()
+        if any(ch in scheme for ch in "\r\n"):
+            scheme = "Bearer"
+        headers[header] = f"{scheme} {api_key}".strip() if scheme else api_key
+    return headers
 
-        # Per-install handle (no shared Shadowbroker product token). Stealth remains
-        # for Turnstile; see docs/OUTBOUND_DATA.md #348.
-        playwright_ua = (
-            f"Mozilla/5.0 (compatible; {outbound_user_agent('liveuamap')})"
-        )
-        context = browser.new_context(
-            user_agent=playwright_ua,
-            viewport={"width": 1920, "height": 1080},
-            color_scheme="dark",
-        )
-        # Bound navigation and script evaluation so a stuck region cannot hang the slow pool.
-        context.set_default_navigation_timeout(60_000)
-        context.set_default_timeout(30_000)
-        page = context.new_page()
-        stealth_sync(page)
 
-        for region in regions:
+def _fetch_liveuamap_api() -> list[dict[str, Any]]:
+    """Fetch an operator-configured supported LiveUAMap JSON/GeoJSON endpoint."""
+    url = _api_url()
+    if not url:
+        return []
+
+    timeout_s = _bounded_int_env("LIVEUAMAP_API_TIMEOUT_S", 30, minimum=5, maximum=120)
+    hostname = urlparse(url).hostname or "configured endpoint"
+    logger.info("Fetching LiveUAMap supported API from %s", hostname)
+    # Refuse redirects while sending operator credentials. A custom auth header
+    # is not guaranteed to be stripped by requests on a redirect, so the safest
+    # contract is that LIVEUAMAP_API_URL names the final HTTPS endpoint.
+    response = requests.get(
+        url,
+        headers=_api_headers(),
+        timeout=(5, timeout_s),
+        allow_redirects=False,
+    )
+    if 300 <= response.status_code < 400:
+        raise requests.HTTPError("LiveUAMap API redirected; configure the final HTTPS endpoint")
+    response.raise_for_status()
+    try:
+        payload = response.json()
+    except ValueError as exc:
+        raise ValueError("LiveUAMap API did not return JSON/GeoJSON") from exc
+
+    candidates = normalize_liveuamap_payload(payload)
+    markers = _format_markers(
+        candidates,
+        region="LiveUAMap",
+        base_url=_public_api_base_url(url),
+        fallback_link="https://liveuamap.com",
+        provider="api",
+    )
+    if not markers:
+        raise ValueError(
+            f"LiveUAMap API returned no recognizable point markers ({payload_shape(payload)})"
+        )
+    logger.info("LiveUAMap API returned %s normalized markers", len(markers))
+    return markers
+
+
+def _browser_circuit_open() -> tuple[bool, int]:
+    now = time.monotonic()
+    with _browser_health_lock:
+        remaining = max(0, int(_browser_blocked_until - now))
+        return remaining > 0, remaining
+
+
+def _record_browser_success() -> None:
+    global _browser_failures, _browser_blocked_until
+    with _browser_health_lock:
+        _browser_failures = 0
+        _browser_blocked_until = 0.0
+
+
+def _record_browser_failure(reason: str) -> None:
+    global _browser_failures, _browser_blocked_until
+    now = time.monotonic()
+    with _browser_health_lock:
+        _browser_failures += 1
+        failures = _browser_failures
+        if failures < _BROWSER_FAILURE_THRESHOLD:
+            logger.warning(
+                "LiveUAMap browser provider failure %s/%s: %s",
+                failures,
+                _BROWSER_FAILURE_THRESHOLD,
+                reason,
+            )
+            return
+        exponent = failures - _BROWSER_FAILURE_THRESHOLD
+        delay_s = min(_BROWSER_BACKOFF_BASE_S * (2**exponent), _BROWSER_BACKOFF_MAX_S)
+        _browser_blocked_until = max(_browser_blocked_until, now + delay_s)
+    logger.warning(
+        "LiveUAMap browser provider paused for %ss after repeated failures: %s",
+        delay_s,
+        reason,
+    )
+
+
+def _looks_like_challenge(html: str) -> bool:
+    lowered = (html or "").lower()
+    return any(marker in lowered for marker in _CHALLENGE_MARKERS)
+
+
+def _read_page_payload(page: Any, html: str) -> Any:
+    """Prefer evaluated page state, then fall back to the legacy source variable."""
+    try:
+        serialized = page.evaluate(
+            "() => typeof ovens !== 'undefined' ? JSON.stringify(ovens) : null"
+        )
+        if serialized:
+            return serialized
+    except Exception as exc:  # Playwright exception types differ across releases.
+        logger.debug("LiveUAMap ovens JS evaluation unavailable: %s", exc)
+
+    expression = extract_ovens_expression(html)
+    return expression if expression is not None else None
+
+
+def _fetch_liveuamap_browser() -> list[dict[str, Any]]:
+    open_now, remaining_s = _browser_circuit_open()
+    if open_now:
+        logger.info(
+            "LiveUAMap browser provider circuit open; skipping Chromium for another %ss",
+            remaining_s,
+        )
+        return []
+
+    # Import browser-only dependencies lazily so API-only deployments do not
+    # require Chromium just to import this module.
+    from playwright.sync_api import sync_playwright
+    from playwright_stealth import stealth_sync
+    from services.network_utils import outbound_user_agent
+
+    all_markers: list[dict[str, Any]] = []
+    seen_ids: set[str] = set()
+    successful_regions = 0
+    failed_regions = 0
+
+    try:
+        with sync_playwright() as playwright:
+            # Existing repository behavior retained for compatibility. This PR
+            # deliberately adds no further anti-detection/evasion measures.
+            browser = playwright.chromium.launch(
+                headless=True,
+                args=["--disable-blink-features=AutomationControlled"],
+            )
             try:
-                logger.info(f"Scraping Liveuamap region: {region['name']}")
-                page.goto(region["url"], timeout=60000, wait_until="domcontentloaded")
+                context = browser.new_context(
+                    user_agent=f"Mozilla/5.0 (compatible; {outbound_user_agent('liveuamap')})",
+                    viewport={"width": 1920, "height": 1080},
+                    color_scheme="dark",
+                )
+                context.set_default_navigation_timeout(60_000)
+                context.set_default_timeout(30_000)
+                page = context.new_page()
+                stealth_sync(page)
 
-                # Wait for the map canvas or markers script to load, max 10s wait
-                try:
-                    page.wait_for_timeout(5000)
-                except (TimeoutError, OSError):  # non-critical: page load delay
-                    pass
-
-                html = page.content()
-
-                m = re.search(r"var\s+ovens\s*=\s*(.*?);(?!function)", html, re.DOTALL)
-                if not m:
-                    logger.warning(f"Could not find 'ovens' data for {region['name']} in raw HTML")
-                    # Let's try grabbing the evaluated JavaScript variable if it's there
+                for region in _REGIONS:
                     try:
-                        ovens_json = page.evaluate(
-                            "() => typeof ovens !== 'undefined' ? JSON.stringify(ovens) : null"
-                        )
-                        if ovens_json:
-                            markers = json.loads(ovens_json)
-                            # process below
-                            html = f"var ovens={ovens_json};"
-                            m = re.search(r"var\s+ovens=(.*?);", html, re.DOTALL)
-                    except (ValueError, KeyError, OSError) as e:  # non-critical: JS eval fallback
-                        logger.debug(
-                            f"Could not evaluate ovens JS variable for {region['name']}: {e}"
+                        logger.info("Fetching LiveUAMap browser region: %s", region["name"])
+                        response = page.goto(
+                            region["url"],
+                            timeout=60_000,
+                            wait_until="domcontentloaded",
                         )
+                        if response is not None and response.status >= 400:
+                            logger.warning(
+                                "LiveUAMap %s returned HTTP %s",
+                                region["name"],
+                                response.status,
+                            )
+                        page.wait_for_timeout(5_000)
+                        html = page.content()
 
-                if m:
-                    json_str = m.group(1).strip()
-                    if json_str.startswith("'") or json_str.startswith('"'):
-                        json_str = json_str.strip("\"'")
-                        json_str = base64.b64decode(urllib.parse.unquote(json_str)).decode("utf-8")
-
-                    try:
-                        markers = json.loads(json_str)
-                        for marker in markers:
-                            mid = marker.get("id")
-                            if mid and mid not in seen_ids:
-                                seen_ids.add(mid)
-                                title = (marker.get("s") or marker.get("title") or "Unknown Event").strip()
-                                # Extract all available fields from the marker
-                                description = (marker.get("d") or marker.get("desc") or marker.get("description") or "").strip()
-                                category = (marker.get("c") or marker.get("cat") or marker.get("category") or "").strip()
-                                img = marker.get("img") or marker.get("image") or marker.get("photo") or ""
-                                source = (marker.get("source") or marker.get("src") or "").strip()
-                                event_time = marker.get("time") or marker.get("t") or ""
-                                link = marker.get("link") or marker.get("url") or ""
-                                # Format date from unix timestamp if available
-                                date_str = ""
-                                if event_time:
-                                    try:
-                                        from datetime import datetime, timezone
-                                        ts = int(event_time) if not isinstance(event_time, int) else event_time
-                                        dt = datetime.fromtimestamp(ts, tz=timezone.utc)
-                                        date_str = dt.strftime("%Y-%m-%d %H:%M UTC")
-                                    except (ValueError, TypeError, OSError):
-                                        date_str = str(event_time)
-                                # Build full link URL
-                                if link and not link.startswith("http"):
-                                    base = region["url"].rstrip("/")
-                                    link = f"{base}/{link.lstrip('/')}"
-                                all_markers.append(
-                                    {
-                                        "id": mid,
-                                        "type": "liveuamap",
-                                        "title": title,
-                                        "description": description[:500] if description else "",
-                                        "lat": marker.get("lat"),
-                                        "lng": marker.get("lng"),
-                                        "timestamp": event_time,
-                                        "date": date_str,
-                                        "link": link or region["url"],
-                                        "region": region["name"],
-                                        "category": category,
-                                        "image": img,
-                                        "source": source,
-                                    }
+                        # Try useful marker state before classifying the page as
+                        # a challenge. Normal pages may load Turnstile assets;
+                        # valid marker data should win over that heuristic.
+                        payload = _read_page_payload(page, html)
+                        if payload is None:
+                            if _looks_like_challenge(html):
+                                logger.warning(
+                                    "LiveUAMap %s appears to be serving an access challenge; "
+                                    "leaving this region empty",
+                                    region["name"],
                                 )
-                    except (json.JSONDecodeError, ValueError, KeyError) as e:
-                        logger.error(f"Error parsing JSON for {region['name']}: {e}")
+                            else:
+                                logger.warning(
+                                    "LiveUAMap %s did not expose an ovens payload",
+                                    region["name"],
+                                )
+                            failed_regions += 1
+                            continue
 
-            except Exception as e:
-                logger.error(f"Error scraping Liveuamap {region['name']}: {e}")
+                        candidates = normalize_liveuamap_payload(payload)
+                        region_markers = _format_markers(
+                            candidates,
+                            region=region["name"],
+                            base_url=region["url"],
+                            provider="browser",
+                            seen_ids=seen_ids,
+                        )
+                        if not region_markers:
+                            if _looks_like_challenge(html):
+                                logger.warning(
+                                    "LiveUAMap %s returned no markers and appears challenge-gated",
+                                    region["name"],
+                                )
+                            else:
+                                logger.warning(
+                                    "LiveUAMap %s payload contained no recognizable point markers (%s)",
+                                    region["name"],
+                                    payload_shape(payload),
+                                )
+                            failed_regions += 1
+                            continue
 
-        browser.close()
+                        all_markers.extend(region_markers)
+                        successful_regions += 1
+                    except Exception as exc:  # Keep one region from killing the other three.
+                        failed_regions += 1
+                        logger.warning("LiveUAMap %s fetch failed: %s", region["name"], exc)
+            finally:
+                browser.close()
+    except Exception as exc:
+        _record_browser_failure(f"Chromium/provider launch failed: {exc}")
+        return []
 
-    logger.info(f"Liveuamap scraper finished, extracted {len(all_markers)} unique markers.")
-    return all_markers
+    if successful_regions:
+        _record_browser_success()
+        logger.info(
+            "LiveUAMap browser provider normalized %s markers from %s/%s regions",
+            len(all_markers),
+            successful_regions,
+            len(_REGIONS),
+        )
+        return all_markers
+
+    _record_browser_failure(f"all {failed_regions or len(_REGIONS)} regions failed or drifted")
+    return []
+
+
+def _normalize_marker_link(raw: Any, base_url: str) -> str:
+    """Resolve only HTTP(S) marker links; reject active/non-web URL schemes."""
+    text = _as_text(raw).strip()
+    if not text:
+        return ""
+    try:
+        parsed = urlparse(text)
+    except ValueError:
+        return ""
+    if parsed.scheme:
+        return text if parsed.scheme.lower() in {"http", "https"} else ""
+
+    try:
+        resolved = urljoin(base_url.rstrip("/") + "/", text)
+        resolved_scheme = urlparse(resolved).scheme.lower()
+    except ValueError:
+        return ""
+    return resolved if resolved_scheme in {"http", "https"} else ""
+
+
+def _format_markers(
+    candidates: list[dict[str, Any]],
+    *,
+    region: str,
+    base_url: str,
+    provider: str,
+    seen_ids: set[str] | None = None,
+    fallback_link: str | None = None,
+) -> list[dict[str, Any]]:
+    output: list[dict[str, Any]] = []
+    dedupe = seen_ids if seen_ids is not None else set()
+
+    for marker, lat, lng in iter_valid_coordinates(candidates):
+        title = _as_text(
+            marker.get("s")
+            or marker.get("title")
+            or marker.get("name")
+            or marker.get("event")
+            or "Unknown Event"
+        ).strip()
+        description = _as_text(
+            marker.get("d")
+            or marker.get("desc")
+            or marker.get("description")
+            or marker.get("summary")
+            or ""
+        ).strip()
+        category = _as_text(
+            marker.get("c") or marker.get("cat") or marker.get("category") or ""
+        ).strip()
+        image = _as_text(marker.get("img") or marker.get("image") or marker.get("photo") or "").strip()
+        source = _as_text(marker.get("source") or marker.get("src") or "").strip()
+        event_time = marker.get("time", marker.get("t", marker.get("timestamp", "")))
+        link = _normalize_marker_link(marker.get("link") or marker.get("url") or "", base_url)
+
+        raw_id = marker.get("id", marker.get("event_id"))
+        marker_id = _as_text(raw_id).strip() if raw_id is not None else ""
+        if not marker_id:
+            marker_id = _stable_marker_id(lat, lng, title, event_time, link)
+        if marker_id in dedupe:
+            continue
+        dedupe.add(marker_id)
+
+        date_str = _format_event_time(event_time)
+        output.append(
+            {
+                "id": marker_id,
+                "type": "liveuamap",
+                "title": title or "Unknown Event",
+                "description": description[:500],
+                "lat": lat,
+                "lng": lng,
+                "timestamp": event_time if event_time is not None else "",
+                "date": date_str,
+                "link": link or fallback_link or base_url,
+                "region": _as_text(marker.get("region") or region).strip() or region,
+                "category": category,
+                "image": image,
+                "source": source,
+                "provider": provider,
+            }
+        )
+    return output
+
+
+def _stable_marker_id(lat: float, lng: float, title: str, event_time: Any, link: str) -> str:
+    fingerprint = f"{lat:.6f}|{lng:.6f}|{title}|{event_time}|{link}".encode(
+        "utf-8", errors="replace"
+    )
+    return f"liveuamap-{hashlib.sha256(fingerprint).hexdigest()[:20]}"
+
+
+def _format_event_time(value: Any) -> str:
+    if value in (None, ""):
+        return ""
+    try:
+        numeric = float(value)
+        if not math.isfinite(numeric):
+            raise ValueError("non-finite timestamp")
+        if abs(numeric) > 100_000_000_000:  # milliseconds since epoch
+            numeric /= 1000.0
+        dt = datetime.fromtimestamp(numeric, tz=timezone.utc)
+        return dt.strftime("%Y-%m-%d %H:%M UTC")
+    except (TypeError, ValueError, OSError, OverflowError):
+        return _as_text(value)
+
+
+def _as_text(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    if isinstance(value, (int, float, bool)):
+        return str(value)
+    return ""
+
+
+def fetch_liveuamap() -> list[dict[str, Any]]:
+    """Return LiveUAMap enrichment without making Global Incidents depend on it."""
+    from services.liveuamap_settings import (
+        liveuamap_api_configured,
+        liveuamap_browser_scraper_enabled,
+    )
+
+    if liveuamap_api_configured():
+        try:
+            return _fetch_liveuamap_api()
+        except (requests.RequestException, ValueError, OSError) as exc:
+            logger.warning(
+                "LiveUAMap supported API failed (%s); considering browser fallback",
+                type(exc).__name__,
+            )
+            # POSIX installs preserve their historical browser fallback; on
+            # Windows it remains available only after the operator opted in.
+
+    if liveuamap_browser_scraper_enabled():
+        return _fetch_liveuamap_browser()
+
+    logger.info("LiveUAMap enrichment disabled/unavailable; Global Incidents continues with GDELT")
+    return []
 
 
 if __name__ == "__main__":
+    import json
+
     logging.basicConfig(level=logging.INFO)
-    res = fetch_liveuamap()
-    print(json.dumps(res[:3], indent=2))
+    print(json.dumps(fetch_liveuamap()[:3], indent=2))

backend/services/liveuamap_settings.py

@@ -1,4 +1,10 @@
-"""LiveUAMap Playwright scraper opt-in (#348) — UI consent on Windows."""
+"""LiveUAMap provider settings and operator-consent state.
+
+Global Incidents is a broader Shadowbroker feature backed by GDELT regardless
+of whether LiveUAMap enrichment is available. The browser provider keeps the
+historical platform behavior (automatic on POSIX, opt-in on Windows) while a
+configured supported API can satisfy LiveUAMap enrichment without Chromium.
+"""
 
 from __future__ import annotations
 
@@ -8,6 +14,7 @@ import os
 import threading
 from pathlib import Path
 from typing import Any
+from urllib.parse import urlparse
 
 logger = logging.getLogger(__name__)
 
@@ -19,33 +26,64 @@ def _env_flag(name: str) -> str:
     return str(os.getenv(name, "")).strip().lower()
 
 
+def _valid_https_url(raw: str) -> bool:
+    try:
+        parsed = urlparse(raw)
+    except ValueError:
+        return False
+    return (
+        parsed.scheme.lower() == "https"
+        and bool(parsed.hostname)
+        and parsed.username is None
+        and parsed.password is None
+    )
+
+
 def liveuamap_requires_ui_opt_in() -> bool:
-    """Windows local installs need explicit consent before Playwright contacts LiveUAMap."""
+    """Windows local installs need an explicit choice before browser scraping."""
     return os.name == "nt"
 
 
+def liveuamap_ui_choice_recorded() -> bool:
+    """Whether the operator has already accepted or declined browser contact."""
+    return _OPT_IN_FILE.exists()
+
+
 def get_liveuamap_ui_opt_in() -> bool:
     if not _OPT_IN_FILE.exists():
         return False
     try:
         payload = json.loads(_OPT_IN_FILE.read_text(encoding="utf-8"))
         return bool(payload.get("opted_in"))
-    except (OSError, json.JSONDecodeError, TypeError) as e:
-        logger.warning("LiveUAMap opt-in file unreadable: %s", e)
+    except (OSError, json.JSONDecodeError, TypeError) as exc:
+        logger.warning("LiveUAMap opt-in file unreadable: %s", exc)
         return False
 
 
 def set_liveuamap_ui_opt_in(opted_in: bool) -> None:
+    """Persist an explicit browser-provider choice, including a decline."""
     _OPT_IN_FILE.parent.mkdir(parents=True, exist_ok=True)
+    payload = json.dumps({"opted_in": bool(opted_in)}, indent=2)
     with _OPT_IN_LOCK:
-        _OPT_IN_FILE.write_text(
-            json.dumps({"opted_in": bool(opted_in)}, indent=2),
-            encoding="utf-8",
-        )
+        temp_path = _OPT_IN_FILE.with_suffix(_OPT_IN_FILE.suffix + ".tmp")
+        temp_path.write_text(payload, encoding="utf-8")
+        os.replace(temp_path, _OPT_IN_FILE)
 
 
-def liveuamap_scraper_enabled() -> bool:
-    """Whether the Playwright LiveUAMap scraper may run on this backend."""
+def liveuamap_api_configured() -> bool:
+    """Whether an operator supplied a syntactically valid HTTPS API endpoint."""
+    url = str(os.getenv("LIVEUAMAP_API_URL", "") or "").strip()
+    return bool(url and _valid_https_url(url))
+
+
+def liveuamap_browser_scraper_enabled() -> bool:
+    """Whether the existing Playwright provider may contact LiveUAMap.
+
+    Preserve the established UX on Linux/macOS/Docker: browser enrichment is
+    available when Global Incidents is active unless explicitly disabled.
+    Windows keeps the existing opt-in boundary. An environment override always
+    wins for the browser provider only; it does not disable a configured API.
+    """
     setting = _env_flag("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER")
     if setting in {"1", "true", "yes", "on"}:
         return True
@@ -56,6 +94,15 @@ def liveuamap_scraper_enabled() -> bool:
     return get_liveuamap_ui_opt_in()
 
 
+def liveuamap_scraper_enabled() -> bool:
+    """Historical scheduler gate: whether *any* LiveUAMap provider can run.
+
+    The name is retained for call-site compatibility. Supported API access is
+    preferred when configured; otherwise the optional browser provider may run.
+    """
+    return liveuamap_api_configured() or liveuamap_browser_scraper_enabled()
+
+
 def liveuamap_scraper_status() -> dict[str, Any]:
     setting = _env_flag("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER")
     env_override = None
@@ -63,11 +110,29 @@ def liveuamap_scraper_status() -> dict[str, Any]:
         env_override = "on"
     elif setting in {"0", "false", "no", "off"}:
         env_override = "off"
+
     ui_opted_in = get_liveuamap_ui_opt_in()
     requires = liveuamap_requires_ui_opt_in()
+    api_configured = liveuamap_api_configured()
+    browser_enabled = liveuamap_browser_scraper_enabled()
+    enrichment_enabled = api_configured or browser_enabled
+
+    if api_configured:
+        provider_mode = "api"
+    elif browser_enabled:
+        provider_mode = "scraper"
+    else:
+        provider_mode = "gdelt-only"
+
     return {
+        # Existing fields remain stable for current frontends.
         "platform_requires_opt_in": requires,
         "ui_opted_in": ui_opted_in,
-        "scraper_enabled": liveuamap_scraper_enabled(),
+        "scraper_enabled": browser_enabled,
         "env_override": env_override,
+        # Additive provider/UX diagnostics.
+        "ui_choice_recorded": liveuamap_ui_choice_recorded(),
+        "api_configured": api_configured,
+        "enrichment_enabled": enrichment_enabled,
+        "provider_mode": provider_mode,
     }

backend/tests/test_liveuamap_docker_contract.py

@@ -0,0 +1,16 @@
+"""Static contract for the Playwright runtime-user packaging fix in #516."""
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def test_dockerfile_uses_shared_playwright_browser_path_and_runtime_user_check() -> None:
+    dockerfile = Path(__file__).resolve().parents[1] / "Dockerfile"
+    text = dockerfile.read_text(encoding="utf-8")
+
+    assert "ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright" in text
+    assert "playwright install --with-deps chromium" in text
+    assert "USER backenduser" in text
+    assert "p.chromium.executable_path" in text
+    assert "assert os.path.isfile(path)" in text
+    assert "chromium_headless_shell-*" in text

backend/tests/test_liveuamap_parser.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+import base64
+import json
+from urllib.parse import quote
+
+from services.liveuamap_parser import (
+    extract_ovens_expression,
+    iter_valid_coordinates,
+    normalize_liveuamap_payload,
+)
+
+
+def _ids(value):
+    return [str(item.get("id")) for item in normalize_liveuamap_payload(value)]
+
+
+def test_plain_marker_list():
+    payload = [{"id": 1, "lat": 1, "lng": 2, "title": "a"}]
+    assert _ids(payload) == ["1"]
+
+
+def test_double_encoded_json():
+    payload = json.dumps(json.dumps([{"id": "double", "lat": 1, "lng": 2}]))
+    assert _ids(payload) == ["double"]
+
+
+def test_list_of_json_strings_regression_517():
+    payload = [
+        json.dumps({"id": "a", "lat": 10, "lng": 20}),
+        json.dumps({"id": "b", "lat": 30, "lng": 40}),
+    ]
+    assert _ids(payload) == ["a", "b"]
+
+
+def test_mapping_key_becomes_fallback_marker_id():
+    payload = {"123": {"lat": 1, "lng": 2, "title": "keyed"}}
+    markers = normalize_liveuamap_payload(payload)
+    assert markers[0]["id"] == "123"
+
+
+def test_common_wrapper_shape():
+    payload = {"data": {"markers": [{"id": "wrapped", "lat": 1, "lng": 2}]}}
+    assert _ids(payload) == ["wrapped"]
+
+
+def test_coordinate_marker_wins_over_wrapper_named_field():
+    payload = {
+        "id": "direct",
+        "lat": 10,
+        "lng": 20,
+        "data": {"diagnostic": "metadata, not a marker wrapper"},
+    }
+    markers = normalize_liveuamap_payload(payload)
+    assert len(markers) == 1
+    assert markers[0]["id"] == "direct"
+    assert markers[0]["lat"] == 10
+
+
+def test_legacy_urlencoded_base64_json():
+    raw = json.dumps([{"id": "legacy", "lat": 1, "lng": 2}]).encode()
+    payload = quote(base64.b64encode(raw).decode())
+    assert _ids(payload) == ["legacy"]
+
+
+def test_geojson_feature_collection():
+    payload = {
+        "type": "FeatureCollection",
+        "features": [
+            {
+                "type": "Feature",
+                "id": "geo",
+                "geometry": {"type": "Point", "coordinates": [20, 10]},
+                "properties": {"title": "Geo event"},
+            }
+        ],
+    }
+    markers = normalize_liveuamap_payload(payload)
+    assert markers == [{"title": "Geo event", "lat": 10.0, "lng": 20.0, "id": "geo"}]
+
+
+def test_malformed_scalars_are_ignored_instead_of_crashing():
+    payload = ["not-json", 42, None, True, {"nested": object()}]
+    assert normalize_liveuamap_payload(payload) == []
+
+
+def test_coordinate_iterator_rejects_out_of_range_and_nonfinite():
+    markers = [
+        {"id": "good", "lat": "10", "lng": "20"},
+        {"id": "bad-lat", "lat": 100, "lng": 20},
+        {"id": "bad-lng", "lat": 10, "lng": 200},
+        {"id": "nan", "lat": float("nan"), "lng": 20},
+    ]
+    valid = list(iter_valid_coordinates(markers))
+    assert [(item[0]["id"], item[1], item[2]) for item in valid] == [("good", 10.0, 20.0)]
+
+
+def test_extracts_var_let_and_const_ovens():
+    assert extract_ovens_expression('<script>var ovens = [{"id":1}];</script>') == '[{"id":1}]'
+    assert extract_ovens_expression('<script>let ovens = "abc";</script>') == '"abc"'
+    assert extract_ovens_expression('<script>const ovens = {"data":[]};</script>') == '{"data":[]}'

backend/tests/test_liveuamap_provider.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+import requests
+
+from services import liveuamap_scraper as scraper
+from services import liveuamap_settings as settings
+
+
+class _Response:
+    def __init__(self, payload, status_code=200):
+        self._payload = payload
+        self.status_code = status_code
+
+    def raise_for_status(self):
+        if self.status_code >= 400:
+            raise requests.HTTPError(f"HTTP {self.status_code}")
+
+    def json(self):
+        return self._payload
+
+
+def test_api_geojson_is_normalized_and_auth_header_is_sent(monkeypatch):
+    monkeypatch.setenv("LIVEUAMAP_API_URL", "https://api.example.test/events")
+    monkeypatch.setenv("LIVEUAMAP_API_KEY", "secret-key")
+    monkeypatch.setenv("LIVEUAMAP_API_TIMEOUT_S", "12")
+    monkeypatch.setattr(
+        "services.network_utils.outbound_user_agent",
+        lambda purpose="": f"operator-test ({purpose})",
+    )
+    seen = {}
+
+    def fake_get(url, *, headers, timeout, allow_redirects):
+        seen.update(url=url, headers=headers, timeout=timeout, allow_redirects=allow_redirects)
+        return _Response(
+            {
+                "type": "FeatureCollection",
+                "features": [
+                    {
+                        "type": "Feature",
+                        "id": "evt-1",
+                        "geometry": {"type": "Point", "coordinates": [30.5, 50.5]},
+                        "properties": {"title": "Event", "url": "https://example.test/e/1"},
+                    }
+                ],
+            }
+        )
+
+    monkeypatch.setattr(scraper.requests, "get", fake_get)
+    markers = scraper._fetch_liveuamap_api()
+    assert markers[0]["id"] == "evt-1"
+    assert markers[0]["lat"] == 50.5
+    assert markers[0]["lng"] == 30.5
+    assert markers[0]["provider"] == "api"
+    assert seen["headers"]["Authorization"] == "Bearer secret-key"
+    assert seen["timeout"] == (5, 12)
+    assert seen["allow_redirects"] is False
+
+
+def test_api_query_token_is_not_exposed_as_marker_fallback(monkeypatch):
+    monkeypatch.setenv(
+        "LIVEUAMAP_API_URL",
+        "https://api.example.test/events?token=super-secret",
+    )
+    monkeypatch.setattr(
+        scraper.requests,
+        "get",
+        lambda *args, **kwargs: _Response(
+            [{"id": "evt", "lat": 10, "lng": 20, "title": "No link"}]
+        ),
+    )
+    markers = scraper._fetch_liveuamap_api()
+    assert markers[0]["link"] == "https://liveuamap.com"
+    assert "super-secret" not in repr(markers)
+
+
+def test_non_http_marker_link_is_rejected():
+    markers = scraper._format_markers(
+        [{"id": "evt", "lat": 10, "lng": 20, "link": "javascript:alert(1)"}],
+        region="Ukraine",
+        base_url="https://liveuamap.com",
+        provider="browser",
+    )
+    assert markers[0]["link"] == "https://liveuamap.com"
+    assert "javascript:" not in repr(markers)
+
+
+def test_api_redirect_is_refused_before_following_credentials(monkeypatch):
+    monkeypatch.setenv("LIVEUAMAP_API_URL", "https://api.example.test/events")
+    monkeypatch.setattr(
+        scraper.requests,
+        "get",
+        lambda *args, **kwargs: _Response({}, status_code=302),
+    )
+    try:
+        scraper._fetch_liveuamap_api()
+    except requests.HTTPError as exc:
+        assert "redirected" in str(exc)
+    else:
+        raise AssertionError("redirect should have been rejected")
+
+
+def test_api_failure_falls_back_to_browser_when_browser_is_allowed(monkeypatch):
+    monkeypatch.setattr(settings, "liveuamap_api_configured", lambda: True)
+    monkeypatch.setattr(settings, "liveuamap_browser_scraper_enabled", lambda: True)
+    monkeypatch.setattr(
+        scraper,
+        "_fetch_liveuamap_api",
+        lambda: (_ for _ in ()).throw(requests.Timeout("boom")),
+    )
+    monkeypatch.setattr(scraper, "_fetch_liveuamap_browser", lambda: [{"id": "browser"}])
+    assert scraper.fetch_liveuamap() == [{"id": "browser"}]
+
+
+def test_api_failure_does_not_force_browser_when_browser_is_disabled(monkeypatch):
+    monkeypatch.setattr(settings, "liveuamap_api_configured", lambda: True)
+    monkeypatch.setattr(settings, "liveuamap_browser_scraper_enabled", lambda: False)
+    monkeypatch.setattr(
+        scraper,
+        "_fetch_liveuamap_api",
+        lambda: (_ for _ in ()).throw(requests.Timeout("boom")),
+    )
+    called = False
+
+    def browser():
+        nonlocal called
+        called = True
+        return []
+
+    monkeypatch.setattr(scraper, "_fetch_liveuamap_browser", browser)
+    assert scraper.fetch_liveuamap() == []
+    assert called is False
+
+
+def test_browser_disable_does_not_disable_configured_api_scheduler_gate(monkeypatch, tmp_path):
+    monkeypatch.setattr(settings, "_OPT_IN_FILE", tmp_path / "choice.json")
+    monkeypatch.setattr(settings.os, "name", "nt")
+    monkeypatch.setenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", "false")
+    monkeypatch.setenv("LIVEUAMAP_API_URL", "https://api.example.test/events")
+    assert settings.liveuamap_browser_scraper_enabled() is False
+    assert settings.liveuamap_scraper_enabled() is True
+
+
+def test_api_url_with_embedded_credentials_is_rejected(monkeypatch, tmp_path):
+    monkeypatch.setattr(settings, "_OPT_IN_FILE", tmp_path / "choice.json")
+    monkeypatch.setattr(settings.os, "name", "nt")
+    monkeypatch.setenv("LIVEUAMAP_API_URL", "https://user:password@api.example.test/events")
+    assert settings.liveuamap_api_configured() is False
+    assert scraper._api_url() == ""
+
+
+def test_http_api_endpoint_is_not_used(monkeypatch):
+    monkeypatch.setenv("LIVEUAMAP_API_URL", "http://api.example.test/events")
+    monkeypatch.setattr(scraper, "_fetch_liveuamap_browser", lambda: [])
+    assert scraper._api_url() == ""

backend/tests/test_liveuamap_scraper_opt_in.py

@@ -1,8 +1,8 @@
-"""LiveUAMap scraper UI opt-in on Windows (#348)."""
+"""LiveUAMap provider opt-in and compatibility behavior."""
+
 from __future__ import annotations
 
 import json
-from pathlib import Path
 
 import pytest
 
@@ -13,33 +13,63 @@ from services import liveuamap_settings as settings
 def opt_in_file(tmp_path, monkeypatch):
     path = tmp_path / "liveuamap_scraper_opt_in.json"
     monkeypatch.setattr(settings, "_OPT_IN_FILE", path)
+    monkeypatch.delenv("LIVEUAMAP_API_URL", raising=False)
+    monkeypatch.delenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", raising=False)
     return path
 
 
-def test_windows_defaults_off_without_opt_in(monkeypatch, opt_in_file):
+def test_windows_defaults_browser_off_without_choice(monkeypatch, opt_in_file):
     monkeypatch.setattr(settings.os, "name", "nt")
-    monkeypatch.delenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", raising=False)
-    assert settings.liveuamap_scraper_enabled() is False
     assert settings.liveuamap_requires_ui_opt_in() is True
+    assert settings.liveuamap_ui_choice_recorded() is False
+    assert settings.liveuamap_browser_scraper_enabled() is False
+    assert settings.liveuamap_scraper_enabled() is False
 
 
-def test_windows_opt_in_enables_scraper(monkeypatch, opt_in_file):
+def test_windows_opt_in_enables_browser(monkeypatch, opt_in_file):
     monkeypatch.setattr(settings.os, "name", "nt")
-    monkeypatch.delenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", raising=False)
     settings.set_liveuamap_ui_opt_in(True)
-    assert settings.liveuamap_scraper_enabled() is True
+    assert settings.liveuamap_ui_choice_recorded() is True
+    assert settings.liveuamap_browser_scraper_enabled() is True
     assert json.loads(opt_in_file.read_text())["opted_in"] is True
 
 
-def test_linux_enabled_without_opt_in(monkeypatch, opt_in_file):
+def test_windows_decline_is_recorded_without_enabling_browser(monkeypatch, opt_in_file):
+    monkeypatch.setattr(settings.os, "name", "nt")
+    settings.set_liveuamap_ui_opt_in(False)
+    assert settings.liveuamap_ui_choice_recorded() is True
+    assert settings.get_liveuamap_ui_opt_in() is False
+    assert settings.liveuamap_browser_scraper_enabled() is False
+
+
+def test_linux_preserves_existing_auto_enrichment_default(monkeypatch, opt_in_file):
     monkeypatch.setattr(settings.os, "name", "posix")
-    monkeypatch.delenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", raising=False)
     assert settings.liveuamap_requires_ui_opt_in() is False
+    assert settings.liveuamap_browser_scraper_enabled() is True
     assert settings.liveuamap_scraper_enabled() is True
 
 
-def test_env_force_off_overrides_ui_opt_in(monkeypatch, opt_in_file):
+def test_env_force_off_disables_browser_even_after_opt_in(monkeypatch, opt_in_file):
     monkeypatch.setattr(settings.os, "name", "nt")
-    monkeypatch.setenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", "false")
     settings.set_liveuamap_ui_opt_in(True)
+    monkeypatch.setenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", "false")
+    assert settings.liveuamap_browser_scraper_enabled() is False
+    assert settings.liveuamap_scraper_enabled() is False
+
+
+def test_api_provider_does_not_require_browser_consent(monkeypatch, opt_in_file):
+    monkeypatch.setattr(settings.os, "name", "nt")
+    monkeypatch.setenv("LIVEUAMAP_API_URL", "https://api.example.test/liveuamap")
+    status = settings.liveuamap_scraper_status()
+    assert status["api_configured"] is True
+    assert status["scraper_enabled"] is False
+    assert status["enrichment_enabled"] is True
+    assert status["provider_mode"] == "api"
+    assert settings.liveuamap_scraper_enabled() is True
+
+
+def test_invalid_http_api_url_does_not_count_as_configured(monkeypatch, opt_in_file):
+    monkeypatch.setattr(settings.os, "name", "nt")
+    monkeypatch.setenv("LIVEUAMAP_API_URL", "http://api.example.test/liveuamap")
+    assert settings.liveuamap_api_configured() is False
     assert settings.liveuamap_scraper_enabled() is False

docker-compose.yml

@@ -26,6 +26,15 @@ services:
       - ADMIN_KEY=${ADMIN_KEY:-}
       - FINNHUB_API_KEY=${FINNHUB_API_KEY:-}
       - AIRFRAMES_API_KEY=${AIRFRAMES_API_KEY:-}
+      # LiveUAMap is optional enrichment for Global Incidents. GDELT remains
+      # available regardless. The browser provider preserves existing Linux/
+      # Docker behavior unless explicitly disabled; paid API access is optional.
+      - SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=${SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER:-}
+      - LIVEUAMAP_API_URL=${LIVEUAMAP_API_URL:-}
+      - LIVEUAMAP_API_KEY=${LIVEUAMAP_API_KEY:-}
+      - LIVEUAMAP_API_AUTH_HEADER=${LIVEUAMAP_API_AUTH_HEADER:-Authorization}
+      - LIVEUAMAP_API_AUTH_SCHEME=${LIVEUAMAP_API_AUTH_SCHEME:-Bearer}
+      - LIVEUAMAP_API_TIMEOUT_S=${LIVEUAMAP_API_TIMEOUT_S:-30}
       # Override allowed CORS origins (comma-separated). Auto-detects LAN IPs if empty.
       - CORS_ORIGINS=${CORS_ORIGINS:-}
       # Private Infonet bootstrap seeds. Seeds are discovery hints, not fixed roots.

docs/LIVEUAMAP.md

@@ -0,0 +1,92 @@
+# LiveUAMap enrichment
+
+Shadowbroker's **Global Incidents** layer does not depend on LiveUAMap. GDELT
+remains the baseline incident source; LiveUAMap adds optional map-pin enrichment
+when one of the providers below is available.
+
+## Provider order
+
+1. **Supported LiveUAMap API (optional)** — preferred when an operator has a
+   paid/contracted API endpoint. Shadowbroker does not require this service.
+2. **Browser provider (best effort)** — the existing Playwright integration.
+   This remains enabled by default on Linux/macOS/Docker when Global Incidents
+   is active, preserving existing behavior. Windows asks once before allowing
+   the backend to contact LiveUAMap through the browser provider.
+3. **GDELT-only** — if neither LiveUAMap provider is usable, Global Incidents
+   still turns on and continues to receive GDELT data.
+
+A LiveUAMap provider failure must never disable the broader Global Incidents
+feature.
+
+## Supported API configuration
+
+Because LiveUAMap API endpoint/auth details are supplied under the operator's
+service agreement, Shadowbroker does not hard-code a vendor account endpoint.
+Configure the HTTPS JSON/GeoJSON URL you were given:
+
+```env
+LIVEUAMAP_API_URL=https://your-liveuamap-api-endpoint.example/events
+LIVEUAMAP_API_KEY=your-key
+LIVEUAMAP_API_AUTH_HEADER=Authorization
+LIVEUAMAP_API_AUTH_SCHEME=Bearer
+LIVEUAMAP_API_TIMEOUT_S=30
+```
+
+`LIVEUAMAP_API_KEY` is optional at the code level so deployments whose endpoint
+already contains/handles authentication can still use the provider. The API URL
+must use HTTPS. Keys are sent only in the configured request header and are not
+included in provider-status responses or logs.
+
+If the API request fails, Linux/macOS/Docker may fall back to the browser
+provider under the existing browser-provider policy. Windows falls back only if
+the operator separately opted into browser contact.
+
+## Browser-provider behavior
+
+The browser provider is **best effort** because it consumes an undocumented web
+page representation rather than a stable public schema. Shadowbroker therefore:
+
+- treats strings, wrapped objects, keyed objects, double-encoded JSON, legacy
+  base64 payloads, and GeoJSON as bounded parser inputs;
+- validates point coordinates before emitting markers;
+- skips malformed entries instead of failing an entire region;
+- detects obvious access/challenge pages and fails soft;
+- pauses repeated browser attempts after consecutive complete failures;
+- logs only structural payload diagnostics, not raw upstream payloads; and
+- retains the existing browser/stealth profile without adding new anti-bot
+  bypass techniques.
+
+### Docker browser location (#516)
+
+Published backend images install Playwright browsers into the shared
+`/ms-playwright` directory through `PLAYWRIGHT_BROWSERS_PATH`. The image build
+then verifies, as the non-root runtime user, that both Chromium and the matching
+headless-shell bundle are present and executable. This prevents the previous
+root-cache/runtime-user mismatch where the browser existed under `/root` while
+Playwright searched under `/app/.cache`.
+
+## Operator controls
+
+```env
+# Explicitly enable or disable only the browser provider.
+SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=true
+SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=false
+```
+
+On Windows, the first Global Incidents enable offers LiveUAMap browser
+enrichment. **Accepting or declining never blocks Global Incidents itself.** A
+decline is remembered so the UI does not nag on every toggle. The environment
+flag remains the explicit override.
+
+On Linux/macOS/Docker, leaving the flag unset preserves the historical behavior:
+the browser provider may run while Global Incidents is active. Set it to
+`false` if the operator wants GDELT-only operation unless a supported API is
+configured.
+
+## Failure semantics
+
+LiveUAMap data is enrichment. If Chromium is missing, the upstream schema drifts,
+the site presents an access challenge, the paid API is unavailable, or every
+region returns malformed data, the provider returns no new pins and the error is
+contained. GDELT fetching and the rest of the Shadowbroker data pipeline continue
+independently.

frontend/src/__tests__/hooks/useLiveUamapScraperOptIn.test.ts

@@ -0,0 +1,92 @@
+import { act, cleanup, renderHook, waitFor } from '@testing-library/react';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+import { useLiveUamapScraperOptIn } from '@/hooks/useLiveUamapScraperOptIn';
+
+afterEach(() => {
+  cleanup();
+  vi.restoreAllMocks();
+});
+
+describe('useLiveUamapScraperOptIn', () => {
+  it('never blocks Global Incidents when the operator declines LiveUAMap', async () => {
+    const fetchMock = vi
+      .spyOn(globalThis, 'fetch')
+      .mockResolvedValueOnce(
+        new Response(
+          JSON.stringify({
+            platform_requires_opt_in: true,
+            ui_opted_in: false,
+            ui_choice_recorded: false,
+            scraper_enabled: false,
+            env_override: null,
+            api_configured: false,
+            enrichment_enabled: false,
+            provider_mode: 'gdelt-only',
+          }),
+          { status: 200 },
+        ),
+      )
+      .mockResolvedValueOnce(
+        new Response(
+          JSON.stringify({
+            platform_requires_opt_in: true,
+            ui_opted_in: false,
+            ui_choice_recorded: true,
+            scraper_enabled: false,
+            env_override: null,
+            api_configured: false,
+            enrichment_enabled: false,
+            provider_mode: 'gdelt-only',
+          }),
+          { status: 200 },
+        ),
+      );
+    const confirmMock = vi.spyOn(window, 'confirm').mockReturnValue(false);
+
+    const { result } = renderHook(() => useLiveUamapScraperOptIn());
+    await waitFor(() => expect(result.current.status).not.toBeNull());
+
+    let blocked = true;
+    act(() => {
+      blocked = result.current.needsConsentBeforeEnable('global_incidents', true);
+    });
+
+    expect(blocked).toBe(false);
+    expect(confirmMock).toHaveBeenCalledOnce();
+    await waitFor(
+      () => {
+        expect(fetchMock).toHaveBeenCalledTimes(2);
+      },
+      { timeout: 1000 },
+    );
+    const [, options] = fetchMock.mock.calls[1];
+    expect(options?.method).toBe('POST');
+    expect(options?.body).toBe(JSON.stringify({ opted_in: false }));
+  });
+
+  it('does not prompt when a supported API provider is configured', async () => {
+    vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(
+      new Response(
+        JSON.stringify({
+          platform_requires_opt_in: true,
+          ui_opted_in: false,
+          ui_choice_recorded: false,
+          scraper_enabled: false,
+          env_override: null,
+          api_configured: true,
+          enrichment_enabled: true,
+          provider_mode: 'api',
+        }),
+        { status: 200 },
+      ),
+    );
+    const confirmMock = vi.spyOn(window, 'confirm').mockReturnValue(true);
+
+    const { result } = renderHook(() => useLiveUamapScraperOptIn());
+    await waitFor(() => expect(result.current.status?.api_configured).toBe(true));
+
+    expect(result.current.needsConsentBeforeEnable('global_incidents', true)).toBe(false);
+    expect(confirmMock).not.toHaveBeenCalled();
+  });
+});

frontend/src/__tests__/liveUamapProxyBoundary.test.ts

@@ -0,0 +1,52 @@
+import { describe, expect, it } from 'vitest';
+import type { NextRequest } from 'next/server';
+
+import { proxy } from '@/proxy';
+
+function request(path: string, headers: Record<string, string>): NextRequest {
+  return {
+    nextUrl: { pathname: path },
+    headers: new Headers(headers),
+  } as unknown as NextRequest;
+}
+
+describe('LiveUAMap privileged proxy boundary', () => {
+  it('rejects a cross-site browser opt-in request', async () => {
+    const response = proxy(
+      request('/api/liveuamap/scraper-opt-in', {
+        host: 'localhost:3000',
+        origin: 'https://evil.example',
+        'sec-fetch-site': 'cross-site',
+      }),
+    );
+
+    expect(response.status).toBe(403);
+    await expect(response.json()).resolves.toEqual({
+      detail: 'Cross-origin privileged request denied',
+    });
+  });
+
+  it('allows the same-origin dashboard request without adding page CSP', () => {
+    const response = proxy(
+      request('/api/liveuamap/scraper-opt-in', {
+        host: 'localhost:3000',
+        origin: 'http://localhost:3000',
+        'sec-fetch-site': 'same-origin',
+      }),
+    );
+
+    expect(response.status).toBe(200);
+    expect(response.headers.get('content-security-policy')).toBeNull();
+  });
+
+  it('also protects the provider status endpoint from cross-site browser reads', () => {
+    const response = proxy(
+      request('/api/liveuamap/scraper-status', {
+        host: 'localhost:3000',
+        'sec-fetch-site': 'cross-site',
+      }),
+    );
+
+    expect(response.status).toBe(403);
+  });
+});

frontend/src/hooks/useLiveUamapScraperOptIn.ts

@@ -1,6 +1,6 @@
 'use client';
 
-import { useCallback, useEffect, useState } from 'react';
+import { useCallback, useEffect, useRef, useState } from 'react';
 import { API_BASE } from '@/lib/api';
 
 export type LiveUamapScraperStatus = {
@@ -8,10 +8,15 @@ export type LiveUamapScraperStatus = {
   ui_opted_in: boolean;
   scraper_enabled: boolean;
   env_override: 'on' | 'off' | null;
+  ui_choice_recorded?: boolean;
+  api_configured?: boolean;
+  enrichment_enabled?: boolean;
+  provider_mode?: 'api' | 'scraper' | 'gdelt-only';
 };
 
 export function useLiveUamapScraperOptIn(enabled = true) {
   const [status, setStatus] = useState<LiveUamapScraperStatus | null>(null);
+  const choicePromptedRef = useRef(false);
 
   const refreshStatus = useCallback(async () => {
     try {
@@ -29,20 +34,11 @@ export function useLiveUamapScraperOptIn(enabled = true) {
     void refreshStatus();
   }, [enabled, refreshStatus]);
 
-  const needsConsentBeforeEnable = useCallback(
-    (layerId: string, turningOn: boolean) =>
-      layerId === 'global_incidents' &&
-      turningOn &&
-      Boolean(status?.platform_requires_opt_in) &&
-      !status?.ui_opted_in,
-    [status],
-  );
-
-  const confirmOptIn = useCallback(async () => {
+  const setOptIn = useCallback(async (optedIn: boolean) => {
     const res = await fetch(`${API_BASE}/api/liveuamap/scraper-opt-in`, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({ opted_in: true }),
+      body: JSON.stringify({ opted_in: optedIn }),
     });
     if (!res.ok) {
       throw new Error(`LiveUAMap opt-in failed (${res.status})`);
@@ -52,6 +48,45 @@ export function useLiveUamapScraperOptIn(enabled = true) {
     return body;
   }, []);
 
+  const needsConsentBeforeEnable = useCallback(
+    (layerId: string, turningOn: boolean) => {
+      if (layerId !== 'global_incidents' || !turningOn) return false;
+
+      const choiceRecorded = status?.ui_choice_recorded ?? status?.ui_opted_in ?? false;
+      const shouldOfferBrowserEnrichment =
+        Boolean(status?.platform_requires_opt_in) &&
+        !choiceRecorded &&
+        !status?.api_configured &&
+        status?.env_override === null;
+
+      if (
+        shouldOfferBrowserEnrichment &&
+        !choicePromptedRef.current &&
+        typeof window !== 'undefined'
+      ) {
+        choicePromptedRef.current = true;
+        const optedIn = window.confirm(
+          "Global Incidents will turn on with GDELT either way. Add optional LiveUAMap pins too? LiveUAMap will see this server's IP. OK enables LiveUAMap; Cancel keeps GDELT-only incidents.",
+        );
+
+        // Do not make the Global Incidents toggle wait on an optional provider.
+        // Give the layer-state update a moment to reach the backend before the
+        // opt-in endpoint opportunistically starts an immediate refresh.
+        window.setTimeout(() => {
+          void setOptIn(optedIn).catch((error) => {
+            console.warn('LiveUAMap preference update failed:', error);
+          });
+        }, 250);
+      }
+
+      // LiveUAMap is enrichment, never a prerequisite for Global Incidents.
+      return false;
+    },
+    [setOptIn, status],
+  );
+
+  const confirmOptIn = useCallback(() => setOptIn(true), [setOptIn]);
+
   return {
     status,
     refreshStatus,

frontend/src/proxy.ts

@@ -43,6 +43,8 @@ function isPrivilegedApiPath(pathname: string): boolean {
     path === '/api/debug-latest' ||
     path === '/api/system/update' ||
     path === '/api/layers' ||
+    path === '/api/liveuamap' ||
+    path.startsWith('/api/liveuamap/') ||
     path === '/api/ais/feed' ||
     path === '/api/mesh/infonet/ingest' ||
     path === '/api/mesh/meshtastic/send' ||