From ed97e9e4bcdcb7544b071a2a23ddc7e219a692c4 Mon Sep 17 00:00:00 2001
From: Shadowbroker <43977454+BigBodyCobain@users.noreply.github.com>
Date: Tue, 18 Aug 2026 15:52:46 -0600
Subject: [PATCH] fix(liveuamap): make enrichment resilient and non-blocking
---
.github/workflows/ci.yml | 3 +
backend/Dockerfile | 17 +-
backend/services/liveuamap_parser.py | 249 ++++++++
backend/services/liveuamap_scraper.py | 534 ++++++++++++++----
backend/services/liveuamap_settings.py | 82 ++-
backend/tests/test_liveuamap_parser.py | 88 +++
backend/tests/test_liveuamap_provider.py | 94 +++
.../tests/test_liveuamap_scraper_opt_in.py | 54 +-
docker-compose.yml | 9 +
docs/LIVEUAMAP.md | 92 +++
.../hooks/useLiveUamapScraperOptIn.test.ts | 92 +++
.../src/hooks/useLiveUamapScraperOptIn.ts | 59 +-
12 files changed, 1217 insertions(+), 156 deletions(-)
create mode 100644 backend/services/liveuamap_parser.py
create mode 100644 backend/tests/test_liveuamap_parser.py
create mode 100644 backend/tests/test_liveuamap_provider.py
create mode 100644 docs/LIVEUAMAP.md
create mode 100644 frontend/src/__tests__/hooks/useLiveUamapScraperOptIn.test.ts
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index fd5ed6e..b08d37e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -79,4 +79,7 @@ jobs:
tests/mesh/test_mesh_canonical.py \
tests/mesh/test_mesh_merkle.py \
tests/test_release_helper.py \
+ tests/test_liveuamap_scraper_opt_in.py \
+ tests/test_liveuamap_parser.py \
+ tests/test_liveuamap_provider.py \
-v --tb=short
diff --git a/backend/Dockerfile b/backend/Dockerfile
index 5c47cf9..71cac5c 100644
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -40,6 +40,13 @@ ENV PATH="/root/.local/bin:$PATH"
# Install into system Python (no venv needed inside container)
ENV UV_PROJECT_ENVIRONMENT=/usr/local
+# Playwright installs browsers under the current user's cache by default. The
+# dependency install below runs as root while the backend runs as backenduser,
+# which caused runtime Playwright to look in /app/.cache for a browser that had
+# actually been baked under /root/.cache (#516). Use one image-wide location
+# that is readable by the non-root runtime user.
+ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
+
# Copy workspace root files for UV resolution (build context is repo root)
COPY pyproject.toml /workspace/pyproject.toml
COPY uv.lock /workspace/uv.lock
@@ -47,7 +54,9 @@ COPY backend/pyproject.toml /workspace/backend/pyproject.toml
# Install Python dependencies using the lockfile
RUN cd /workspace/backend && uv sync --frozen --no-dev --extra road-corridor \
- && playwright install --with-deps chromium
+ && playwright install --with-deps chromium \
+ && playwright install chromium-headless-shell \
+ && chmod -R a+rX "$PLAYWRIGHT_BROWSERS_PATH"
# Copy backend source code
COPY backend/ .
@@ -81,6 +90,12 @@ RUN adduser --system --uid 1001 --home /app backenduser \
# Switch to the non-root user
USER backenduser
+# Build-time packaging assertion for #516. Do not launch Chromium here because
+# multi-arch image builds may run under emulation; instead verify that the exact
+# runtime user resolves an executable Chromium and that the headless-shell
+# bundle used by headless launches is present and executable.
+RUN python -c "import os; from pathlib import Path; from playwright.sync_api import sync_playwright; p=sync_playwright().start(); path=p.chromium.executable_path; assert os.path.isfile(path), path; assert os.access(path, os.X_OK), path; shells=list(Path(os.environ['PLAYWRIGHT_BROWSERS_PATH']).glob('chromium_headless_shell-*/**/headless_shell')); assert any(s.is_file() and os.access(s, os.X_OK) for s in shells), shells; print('Playwright runtime browser:', path, 'headless-shell:', shells[0]); p.stop()"
+
# Expose port
EXPOSE 8000
diff --git a/backend/services/liveuamap_parser.py b/backend/services/liveuamap_parser.py
new file mode 100644
index 0000000..fc4f97a
--- /dev/null
+++ b/backend/services/liveuamap_parser.py
@@ -0,0 +1,249 @@
+"""Defensive parsing helpers for LiveUAMap provider payloads.
+
+LiveUAMap's browser page exposes an undocumented ``ovens`` value whose shape
+has changed over time. The optional supported API may also return JSON or
+GeoJSON. Keep representation decoding and schema normalization isolated here so
+upstream drift degrades one provider instead of crashing the fetch scheduler.
+"""
+
+from __future__ import annotations
+
+import ast
+import base64
+import binascii
+import json
+import math
+import re
+from collections.abc import Iterable
+from typing import Any
+from urllib.parse import unquote
+
+_MAX_DECODE_DEPTH = 5
+_MAX_CANDIDATES = 10_000
+_MAX_STRING_BYTES = 8 * 1024 * 1024
+_WRAPPER_KEYS = ("ovens", "markers", "items", "events", "data", "results", "features")
+
+
+def extract_ovens_expression(html: str) -> str | None:
+ """Extract a legacy ``var/let/const ovens = ...;`` expression from HTML.
+
+ Evaluating ``window.ovens`` in the browser is preferred; this exists only
+ as a fallback for pages that still embed the value in source text.
+ """
+ if not html:
+ return None
+ match = re.search(
+ r"(?:var|let|const)\s+ovens\s*=\s*(.+?);(?=\s*(?:|(?:var|let|const|function)\b|$))",
+ html,
+ re.DOTALL | re.IGNORECASE,
+ )
+ if match:
+ return match.group(1).strip()
+
+ # Compatibility with older pages where the next token after the semicolon
+ # is arbitrary markup rather than another JavaScript declaration.
+ match = re.search(r"(?:var|let|const)\s+ovens\s*=\s*(.*?);", html, re.DOTALL | re.IGNORECASE)
+ return match.group(1).strip() if match else None
+
+
+def payload_shape(value: Any) -> str:
+ """Return a non-sensitive structural description for drift diagnostics."""
+ if isinstance(value, dict):
+ keys = sorted(str(key) for key in value.keys())[:8]
+ return f"dict(keys={keys}, size={len(value)})"
+ if isinstance(value, list):
+ item_types = sorted({type(item).__name__ for item in value[:20]})
+ return f"list(size={len(value)}, item_types={item_types})"
+ if isinstance(value, str):
+ return f"str(len={len(value)})"
+ return type(value).__name__
+
+
+def normalize_liveuamap_payload(value: Any) -> list[dict[str, Any]]:
+ """Normalize JSON/GeoJSON/legacy payload shapes into marker dictionaries.
+
+ Unknown or malformed values are ignored. This function intentionally never
+ assumes iterable items are mappings; issue #517 was caused by calling
+ ``.get`` on strings after an upstream representation change.
+ """
+ out: list[dict[str, Any]] = []
+ _collect(value, out, depth=0, inherited_id=None)
+ return out[:_MAX_CANDIDATES]
+
+
+def _collect(
+ value: Any,
+ out: list[dict[str, Any]],
+ *,
+ depth: int,
+ inherited_id: str | None,
+) -> None:
+ if depth > _MAX_DECODE_DEPTH or len(out) >= _MAX_CANDIDATES:
+ return
+
+ if value is None:
+ return
+
+ if isinstance(value, str):
+ decoded = _decode_string(value)
+ if decoded is value or decoded == value:
+ return
+ _collect(decoded, out, depth=depth + 1, inherited_id=inherited_id)
+ return
+
+ if isinstance(value, list):
+ for item in value[:_MAX_CANDIDATES - len(out)]:
+ _collect(item, out, depth=depth + 1, inherited_id=None)
+ if len(out) >= _MAX_CANDIDATES:
+ break
+ return
+
+ if not isinstance(value, dict):
+ return
+
+ # GeoJSON FeatureCollection / Feature.
+ if value.get("type") == "FeatureCollection" and isinstance(value.get("features"), list):
+ _collect(value["features"], out, depth=depth + 1, inherited_id=None)
+ return
+ if value.get("type") == "Feature":
+ marker = _marker_from_geojson_feature(value)
+ if marker is not None:
+ if inherited_id and not marker.get("id"):
+ marker["id"] = inherited_id
+ out.append(marker)
+ return
+
+ # Common wrapper shapes returned by APIs or page-side serialization.
+ for key in _WRAPPER_KEYS:
+ if key in value and isinstance(value[key], (dict, list, str)):
+ _collect(value[key], out, depth=depth + 1, inherited_id=None)
+ return
+
+ # A direct marker is accepted even if coordinates are absent here; the
+ # provider formatter performs the final coordinate/range validation.
+ if _looks_like_marker(value):
+ marker = dict(value)
+ if inherited_id and not marker.get("id"):
+ marker["id"] = inherited_id
+ out.append(marker)
+ return
+
+ # Some versions expose a dictionary keyed by marker ID. Traverse mapping
+ # values while preserving the key as a fallback identifier.
+ traversable = [
+ (str(key), item)
+ for key, item in value.items()
+ if isinstance(item, (dict, list, str))
+ ]
+ if traversable:
+ for key, item in traversable[:_MAX_CANDIDATES - len(out)]:
+ _collect(item, out, depth=depth + 1, inherited_id=key)
+ if len(out) >= _MAX_CANDIDATES:
+ break
+
+
+def _decode_string(raw: str) -> Any:
+ text = raw.strip()
+ if not text or len(text.encode("utf-8", errors="ignore")) > _MAX_STRING_BYTES:
+ return raw
+
+ # A JavaScript string literal can include escaping that plain strip("'")
+ # corrupts. literal_eval safely handles quoted string syntax only.
+ if len(text) >= 2 and text[0] == text[-1] and text[0] in {"'", '"'}:
+ try:
+ literal = ast.literal_eval(text)
+ except (SyntaxError, ValueError):
+ literal = None
+ if isinstance(literal, str) and literal != text:
+ return literal
+
+ decoded = _try_json(text)
+ if decoded is not None:
+ return decoded
+
+ # Legacy LiveUAMap payloads have appeared URL-encoded before decoding.
+ url_decoded = unquote(text)
+ if url_decoded != text:
+ decoded = _try_json(url_decoded)
+ if decoded is not None:
+ return decoded
+ text = url_decoded
+
+ # Older scraper versions expected a base64-wrapped JSON blob. Decode only
+ # when the result itself is valid JSON, so arbitrary titles/IDs are never
+ # interpreted as base64 data.
+ compact = "".join(text.split())
+ if compact and len(compact) % 4 == 0:
+ try:
+ raw_bytes = base64.b64decode(compact, validate=True)
+ decoded_text = raw_bytes.decode("utf-8")
+ except (binascii.Error, UnicodeDecodeError, ValueError):
+ decoded_text = ""
+ if decoded_text:
+ decoded = _try_json(decoded_text)
+ if decoded is not None:
+ return decoded
+
+ return raw
+
+
+def _try_json(text: str) -> Any | None:
+ try:
+ return json.loads(text)
+ except (json.JSONDecodeError, TypeError, ValueError):
+ return None
+
+
+def _looks_like_marker(value: dict[str, Any]) -> bool:
+ keys = set(value)
+ if {"lat", "lng"}.issubset(keys) or {"lat", "lon"}.issubset(keys):
+ return True
+ if "latitude" in keys and ("longitude" in keys or "lon" in keys or "lng" in keys):
+ return True
+ marker_metadata = {"id", "s", "title", "d", "desc", "description", "link", "url", "time", "t"}
+ return bool(keys.intersection(marker_metadata)) and not any(key in value for key in _WRAPPER_KEYS)
+
+
+def _marker_from_geojson_feature(feature: dict[str, Any]) -> dict[str, Any] | None:
+ geometry = feature.get("geometry")
+ if not isinstance(geometry, dict) or geometry.get("type") != "Point":
+ return None
+ coordinates = geometry.get("coordinates")
+ if not isinstance(coordinates, (list, tuple)) or len(coordinates) < 2:
+ return None
+ lng = _finite_coordinate(coordinates[0], minimum=-180.0, maximum=180.0)
+ lat = _finite_coordinate(coordinates[1], minimum=-90.0, maximum=90.0)
+ if lat is None or lng is None:
+ return None
+ properties = feature.get("properties")
+ marker = dict(properties) if isinstance(properties, dict) else {}
+ marker.setdefault("lat", lat)
+ marker.setdefault("lng", lng)
+ feature_id = feature.get("id")
+ if feature_id is not None:
+ marker.setdefault("id", feature_id)
+ return marker
+
+
+def _finite_coordinate(value: Any, *, minimum: float, maximum: float) -> float | None:
+ try:
+ number = float(value)
+ except (TypeError, ValueError):
+ return None
+ if not math.isfinite(number) or not minimum <= number <= maximum:
+ return None
+ return number
+
+
+def iter_valid_coordinates(markers: Iterable[dict[str, Any]]) -> Iterable[tuple[dict[str, Any], float, float]]:
+ """Yield markers with finite in-range latitude/longitude values."""
+ for marker in markers:
+ if not isinstance(marker, dict):
+ continue
+ lat = marker.get("lat", marker.get("latitude"))
+ lng = marker.get("lng", marker.get("lon", marker.get("longitude")))
+ lat_value = _finite_coordinate(lat, minimum=-90.0, maximum=90.0)
+ lng_value = _finite_coordinate(lng, minimum=-180.0, maximum=180.0)
+ if lat_value is None or lng_value is None:
+ continue
+ yield marker, lat_value, lng_value
diff --git a/backend/services/liveuamap_scraper.py b/backend/services/liveuamap_scraper.py
index 2d408bb..9426655 100644
--- a/backend/services/liveuamap_scraper.py
+++ b/backend/services/liveuamap_scraper.py
@@ -1,145 +1,439 @@
-import json
+"""Resilient LiveUAMap enrichment providers.
+
+Global Incidents itself is backed independently by GDELT. This module adds
+LiveUAMap pins when either an operator-configured supported API is available or
+the existing Playwright provider is allowed. Provider failures are isolated and
+return an empty enrichment set instead of breaking the scheduler.
+
+The browser provider intentionally does not add any new anti-bot behavior. It
+retains the repository's pre-existing Playwright/stealth profile for backward
+compatibility while making parsing, packaging failures, and upstream drift
+fail-soft.
+"""
+
+from __future__ import annotations
+
+import hashlib
import logging
-import base64
-import urllib.parse
-import re
-from playwright.sync_api import sync_playwright
-from playwright_stealth import stealth_sync
+import math
+import os
+import threading
+import time
+from datetime import datetime, timezone
+from typing import Any
+from urllib.parse import urljoin, urlparse
+
+import requests
+
+from services.liveuamap_parser import (
+ extract_ovens_expression,
+ iter_valid_coordinates,
+ normalize_liveuamap_payload,
+ payload_shape,
+)
logger = logging.getLogger(__name__)
+_REGIONS = (
+ {"name": "Ukraine", "url": "https://liveuamap.com"},
+ {"name": "Middle East", "url": "https://mideast.liveuamap.com"},
+ {"name": "Israel-Palestine", "url": "https://israelpalestine.liveuamap.com"},
+ {"name": "Syria", "url": "https://syria.liveuamap.com"},
+)
-def fetch_liveuamap():
- logger.info("Starting Liveuamap scraper with Playwright Stealth...")
+_BROWSER_FAILURE_THRESHOLD = 3
+_BROWSER_BACKOFF_BASE_S = 15 * 60
+_BROWSER_BACKOFF_MAX_S = 6 * 60 * 60
+_browser_failures = 0
+_browser_blocked_until = 0.0
+_browser_health_lock = threading.Lock()
- regions = [
- {"name": "Ukraine", "url": "https://liveuamap.com"},
- {"name": "Middle East", "url": "https://mideast.liveuamap.com"},
- {"name": "Israel-Palestine", "url": "https://israelpalestine.liveuamap.com"},
- {"name": "Syria", "url": "https://syria.liveuamap.com"},
- ]
+_CHALLENGE_MARKERS = (
+ "cf-turnstile",
+ "challenge-platform",
+ "just a moment",
+ "checking your browser",
+ "verify you are human",
+)
- all_markers = []
- seen_ids = set()
- with sync_playwright() as p:
- # Launching with a real user agent to bypass Turnstile
- browser = p.chromium.launch(
- headless=True, args=["--disable-blink-features=AutomationControlled"]
+def _bounded_int_env(name: str, default: int, *, minimum: int, maximum: int) -> int:
+ try:
+ value = int(str(os.getenv(name, default)).strip())
+ except (TypeError, ValueError):
+ value = default
+ return max(minimum, min(maximum, value))
+
+
+def _safe_header_name(raw: str, default: str) -> str:
+ value = (raw or "").strip()
+ if not value or any(ch in value for ch in "\r\n:"):
+ return default
+ if not all(ch.isalnum() or ch in "-_" for ch in value):
+ return default
+ return value
+
+
+def _api_url() -> str:
+ raw = str(os.getenv("LIVEUAMAP_API_URL", "") or "").strip()
+ if not raw:
+ return ""
+ try:
+ parsed = urlparse(raw)
+ except ValueError:
+ return ""
+ if parsed.scheme.lower() != "https" or not parsed.netloc:
+ return ""
+ return raw
+
+
+def _api_headers() -> dict[str, str]:
+ from services.network_utils import outbound_user_agent
+
+ headers = {
+ "Accept": "application/geo+json, application/json;q=0.9",
+ "User-Agent": outbound_user_agent("liveuamap-api"),
+ }
+ api_key = str(os.getenv("LIVEUAMAP_API_KEY", "") or "").strip()
+ if api_key:
+ header = _safe_header_name(
+ str(os.getenv("LIVEUAMAP_API_AUTH_HEADER", "Authorization") or ""),
+ "Authorization",
)
- from services.network_utils import outbound_user_agent
+ scheme = str(os.getenv("LIVEUAMAP_API_AUTH_SCHEME", "Bearer") or "").strip()
+ if any(ch in scheme for ch in "\r\n"):
+ scheme = "Bearer"
+ headers[header] = f"{scheme} {api_key}".strip() if scheme else api_key
+ return headers
- # Per-install handle (no shared Shadowbroker product token). Stealth remains
- # for Turnstile; see docs/OUTBOUND_DATA.md #348.
- playwright_ua = (
- f"Mozilla/5.0 (compatible; {outbound_user_agent('liveuamap')})"
- )
- context = browser.new_context(
- user_agent=playwright_ua,
- viewport={"width": 1920, "height": 1080},
- color_scheme="dark",
- )
- # Bound navigation and script evaluation so a stuck region cannot hang the slow pool.
- context.set_default_navigation_timeout(60_000)
- context.set_default_timeout(30_000)
- page = context.new_page()
- stealth_sync(page)
- for region in regions:
+def _fetch_liveuamap_api() -> list[dict[str, Any]]:
+ """Fetch an operator-configured supported LiveUAMap JSON/GeoJSON endpoint."""
+ url = _api_url()
+ if not url:
+ return []
+
+ timeout_s = _bounded_int_env("LIVEUAMAP_API_TIMEOUT_S", 30, minimum=5, maximum=120)
+ hostname = urlparse(url).hostname or "configured endpoint"
+ logger.info("Fetching LiveUAMap supported API from %s", hostname)
+ response = requests.get(url, headers=_api_headers(), timeout=(5, timeout_s))
+ response.raise_for_status()
+ try:
+ payload = response.json()
+ except (requests.JSONDecodeError, ValueError) as exc:
+ raise ValueError("LiveUAMap API did not return JSON/GeoJSON") from exc
+
+ candidates = normalize_liveuamap_payload(payload)
+ markers = _format_markers(
+ candidates,
+ region="LiveUAMap",
+ base_url=url,
+ provider="api",
+ )
+ if not markers:
+ raise ValueError(f"LiveUAMap API returned no recognizable point markers ({payload_shape(payload)})")
+ logger.info("LiveUAMap API returned %s normalized markers", len(markers))
+ return markers
+
+
+def _browser_circuit_open() -> tuple[bool, int]:
+ now = time.monotonic()
+ with _browser_health_lock:
+ remaining = max(0, int(_browser_blocked_until - now))
+ return remaining > 0, remaining
+
+
+def _record_browser_success() -> None:
+ global _browser_failures, _browser_blocked_until
+ with _browser_health_lock:
+ _browser_failures = 0
+ _browser_blocked_until = 0.0
+
+
+def _record_browser_failure(reason: str) -> None:
+ global _browser_failures, _browser_blocked_until
+ now = time.monotonic()
+ with _browser_health_lock:
+ _browser_failures += 1
+ failures = _browser_failures
+ if failures < _BROWSER_FAILURE_THRESHOLD:
+ logger.warning(
+ "LiveUAMap browser provider failure %s/%s: %s",
+ failures,
+ _BROWSER_FAILURE_THRESHOLD,
+ reason,
+ )
+ return
+ exponent = failures - _BROWSER_FAILURE_THRESHOLD
+ delay_s = min(_BROWSER_BACKOFF_BASE_S * (2**exponent), _BROWSER_BACKOFF_MAX_S)
+ _browser_blocked_until = max(_browser_blocked_until, now + delay_s)
+ logger.warning(
+ "LiveUAMap browser provider paused for %ss after repeated failures: %s",
+ delay_s,
+ reason,
+ )
+
+
+def _looks_like_challenge(html: str) -> bool:
+ lowered = (html or "").lower()
+ return any(marker in lowered for marker in _CHALLENGE_MARKERS)
+
+
+def _read_page_payload(page: Any, html: str) -> Any:
+ """Prefer evaluated page state, then fall back to the legacy source variable."""
+ try:
+ serialized = page.evaluate(
+ "() => typeof ovens !== 'undefined' ? JSON.stringify(ovens) : null"
+ )
+ if serialized:
+ return serialized
+ except Exception as exc: # Playwright exception types differ across releases.
+ logger.debug("LiveUAMap ovens JS evaluation unavailable: %s", exc)
+
+ expression = extract_ovens_expression(html)
+ return expression if expression is not None else None
+
+
+def _fetch_liveuamap_browser() -> list[dict[str, Any]]:
+ open_now, remaining_s = _browser_circuit_open()
+ if open_now:
+ logger.info(
+ "LiveUAMap browser provider circuit open; skipping Chromium for another %ss",
+ remaining_s,
+ )
+ return []
+
+ # Import browser-only dependencies lazily so API-only deployments do not
+ # require Chromium just to import this module.
+ from playwright.sync_api import sync_playwright
+ from playwright_stealth import stealth_sync
+ from services.network_utils import outbound_user_agent
+
+ all_markers: list[dict[str, Any]] = []
+ seen_ids: set[str] = set()
+ successful_regions = 0
+ failed_regions = 0
+
+ try:
+ with sync_playwright() as playwright:
+ # Existing repository behavior retained for compatibility. This PR
+ # deliberately adds no further anti-detection/evasion measures.
+ browser = playwright.chromium.launch(
+ headless=True,
+ args=["--disable-blink-features=AutomationControlled"],
+ )
try:
- logger.info(f"Scraping Liveuamap region: {region['name']}")
- page.goto(region["url"], timeout=60000, wait_until="domcontentloaded")
+ context = browser.new_context(
+ user_agent=f"Mozilla/5.0 (compatible; {outbound_user_agent('liveuamap')})",
+ viewport={"width": 1920, "height": 1080},
+ color_scheme="dark",
+ )
+ context.set_default_navigation_timeout(60_000)
+ context.set_default_timeout(30_000)
+ page = context.new_page()
+ stealth_sync(page)
- # Wait for the map canvas or markers script to load, max 10s wait
- try:
- page.wait_for_timeout(5000)
- except (TimeoutError, OSError): # non-critical: page load delay
- pass
-
- html = page.content()
-
- m = re.search(r"var\s+ovens\s*=\s*(.*?);(?!function)", html, re.DOTALL)
- if not m:
- logger.warning(f"Could not find 'ovens' data for {region['name']} in raw HTML")
- # Let's try grabbing the evaluated JavaScript variable if it's there
+ for region in _REGIONS:
try:
- ovens_json = page.evaluate(
- "() => typeof ovens !== 'undefined' ? JSON.stringify(ovens) : null"
+ logger.info("Fetching LiveUAMap browser region: %s", region["name"])
+ response = page.goto(
+ region["url"],
+ timeout=60_000,
+ wait_until="domcontentloaded",
)
- if ovens_json:
- markers = json.loads(ovens_json)
- # process below
- html = f"var ovens={ovens_json};"
- m = re.search(r"var\s+ovens=(.*?);", html, re.DOTALL)
- except (ValueError, KeyError, OSError) as e: # non-critical: JS eval fallback
- logger.debug(
- f"Could not evaluate ovens JS variable for {region['name']}: {e}"
+ if response is not None and response.status >= 400:
+ logger.warning(
+ "LiveUAMap %s returned HTTP %s",
+ region["name"],
+ response.status,
+ )
+ page.wait_for_timeout(5_000)
+ html = page.content()
+ if _looks_like_challenge(html):
+ logger.warning(
+ "LiveUAMap %s appears to be serving an access challenge; "
+ "leaving this region empty",
+ region["name"],
+ )
+ failed_regions += 1
+ continue
+
+ payload = _read_page_payload(page, html)
+ if payload is None:
+ logger.warning(
+ "LiveUAMap %s did not expose an ovens payload",
+ region["name"],
+ )
+ failed_regions += 1
+ continue
+
+ candidates = normalize_liveuamap_payload(payload)
+ region_markers = _format_markers(
+ candidates,
+ region=region["name"],
+ base_url=region["url"],
+ provider="browser",
+ seen_ids=seen_ids,
)
+ if not region_markers:
+ logger.warning(
+ "LiveUAMap %s payload contained no recognizable point markers (%s)",
+ region["name"],
+ payload_shape(payload),
+ )
+ failed_regions += 1
+ continue
- if m:
- json_str = m.group(1).strip()
- if json_str.startswith("'") or json_str.startswith('"'):
- json_str = json_str.strip("\"'")
- json_str = base64.b64decode(urllib.parse.unquote(json_str)).decode("utf-8")
+ all_markers.extend(region_markers)
+ successful_regions += 1
+ except Exception as exc: # Keep one region from killing the other three.
+ failed_regions += 1
+ logger.warning("LiveUAMap %s fetch failed: %s", region["name"], exc)
+ finally:
+ browser.close()
+ except Exception as exc:
+ _record_browser_failure(f"Chromium/provider launch failed: {exc}")
+ return []
- try:
- markers = json.loads(json_str)
- for marker in markers:
- mid = marker.get("id")
- if mid and mid not in seen_ids:
- seen_ids.add(mid)
- title = (marker.get("s") or marker.get("title") or "Unknown Event").strip()
- # Extract all available fields from the marker
- description = (marker.get("d") or marker.get("desc") or marker.get("description") or "").strip()
- category = (marker.get("c") or marker.get("cat") or marker.get("category") or "").strip()
- img = marker.get("img") or marker.get("image") or marker.get("photo") or ""
- source = (marker.get("source") or marker.get("src") or "").strip()
- event_time = marker.get("time") or marker.get("t") or ""
- link = marker.get("link") or marker.get("url") or ""
- # Format date from unix timestamp if available
- date_str = ""
- if event_time:
- try:
- from datetime import datetime, timezone
- ts = int(event_time) if not isinstance(event_time, int) else event_time
- dt = datetime.fromtimestamp(ts, tz=timezone.utc)
- date_str = dt.strftime("%Y-%m-%d %H:%M UTC")
- except (ValueError, TypeError, OSError):
- date_str = str(event_time)
- # Build full link URL
- if link and not link.startswith("http"):
- base = region["url"].rstrip("/")
- link = f"{base}/{link.lstrip('/')}"
- all_markers.append(
- {
- "id": mid,
- "type": "liveuamap",
- "title": title,
- "description": description[:500] if description else "",
- "lat": marker.get("lat"),
- "lng": marker.get("lng"),
- "timestamp": event_time,
- "date": date_str,
- "link": link or region["url"],
- "region": region["name"],
- "category": category,
- "image": img,
- "source": source,
- }
- )
- except (json.JSONDecodeError, ValueError, KeyError) as e:
- logger.error(f"Error parsing JSON for {region['name']}: {e}")
+ if successful_regions:
+ _record_browser_success()
+ logger.info(
+ "LiveUAMap browser provider normalized %s markers from %s/%s regions",
+ len(all_markers),
+ successful_regions,
+ len(_REGIONS),
+ )
+ return all_markers
- except Exception as e:
- logger.error(f"Error scraping Liveuamap {region['name']}: {e}")
+ _record_browser_failure(f"all {failed_regions or len(_REGIONS)} regions failed or drifted")
+ return []
- browser.close()
- logger.info(f"Liveuamap scraper finished, extracted {len(all_markers)} unique markers.")
- return all_markers
+def _format_markers(
+ candidates: list[dict[str, Any]],
+ *,
+ region: str,
+ base_url: str,
+ provider: str,
+ seen_ids: set[str] | None = None,
+) -> list[dict[str, Any]]:
+ output: list[dict[str, Any]] = []
+ dedupe = seen_ids if seen_ids is not None else set()
+
+ for marker, lat, lng in iter_valid_coordinates(candidates):
+ title = _as_text(
+ marker.get("s")
+ or marker.get("title")
+ or marker.get("name")
+ or marker.get("event")
+ or "Unknown Event"
+ ).strip()
+ description = _as_text(
+ marker.get("d")
+ or marker.get("desc")
+ or marker.get("description")
+ or marker.get("summary")
+ or ""
+ ).strip()
+ category = _as_text(
+ marker.get("c") or marker.get("cat") or marker.get("category") or ""
+ ).strip()
+ image = _as_text(marker.get("img") or marker.get("image") or marker.get("photo") or "").strip()
+ source = _as_text(marker.get("source") or marker.get("src") or "").strip()
+ event_time = marker.get("time", marker.get("t", marker.get("timestamp", "")))
+ link = _as_text(marker.get("link") or marker.get("url") or "").strip()
+ if link and not urlparse(link).scheme:
+ link = urljoin(base_url.rstrip("/") + "/", link.lstrip("/"))
+
+ raw_id = marker.get("id", marker.get("event_id"))
+ marker_id = _as_text(raw_id).strip() if raw_id is not None else ""
+ if not marker_id:
+ marker_id = _stable_marker_id(lat, lng, title, event_time, link)
+ if marker_id in dedupe:
+ continue
+ dedupe.add(marker_id)
+
+ date_str = _format_event_time(event_time)
+ output.append(
+ {
+ "id": marker_id,
+ "type": "liveuamap",
+ "title": title or "Unknown Event",
+ "description": description[:500],
+ "lat": lat,
+ "lng": lng,
+ "timestamp": event_time if event_time is not None else "",
+ "date": date_str,
+ "link": link or base_url,
+ "region": _as_text(marker.get("region") or region).strip() or region,
+ "category": category,
+ "image": image,
+ "source": source,
+ "provider": provider,
+ }
+ )
+ return output
+
+
+def _stable_marker_id(lat: float, lng: float, title: str, event_time: Any, link: str) -> str:
+ fingerprint = f"{lat:.6f}|{lng:.6f}|{title}|{event_time}|{link}".encode(
+ "utf-8", errors="replace"
+ )
+ return f"liveuamap-{hashlib.sha256(fingerprint).hexdigest()[:20]}"
+
+
+def _format_event_time(value: Any) -> str:
+ if value in (None, ""):
+ return ""
+ try:
+ numeric = float(value)
+ if not math.isfinite(numeric):
+ raise ValueError("non-finite timestamp")
+ if abs(numeric) > 100_000_000_000: # milliseconds since epoch
+ numeric /= 1000.0
+ dt = datetime.fromtimestamp(numeric, tz=timezone.utc)
+ return dt.strftime("%Y-%m-%d %H:%M UTC")
+ except (TypeError, ValueError, OSError, OverflowError):
+ return _as_text(value)
+
+
+def _as_text(value: Any) -> str:
+ if value is None:
+ return ""
+ if isinstance(value, str):
+ return value
+ if isinstance(value, (int, float, bool)):
+ return str(value)
+ return ""
+
+
+def fetch_liveuamap() -> list[dict[str, Any]]:
+ """Return LiveUAMap enrichment without making Global Incidents depend on it."""
+ from services.liveuamap_settings import (
+ liveuamap_api_configured,
+ liveuamap_browser_scraper_enabled,
+ )
+
+ if liveuamap_api_configured():
+ try:
+ return _fetch_liveuamap_api()
+ except (requests.RequestException, ValueError, OSError) as exc:
+ logger.warning(
+ "LiveUAMap supported API failed (%s); considering browser fallback",
+ type(exc).__name__,
+ )
+ # POSIX installs preserve their historical browser fallback; on
+ # Windows it remains available only after the operator opted in.
+
+ if liveuamap_browser_scraper_enabled():
+ return _fetch_liveuamap_browser()
+
+ logger.info("LiveUAMap enrichment disabled/unavailable; Global Incidents continues with GDELT")
+ return []
if __name__ == "__main__":
+ import json
+
logging.basicConfig(level=logging.INFO)
- res = fetch_liveuamap()
- print(json.dumps(res[:3], indent=2))
+ print(json.dumps(fetch_liveuamap()[:3], indent=2))
diff --git a/backend/services/liveuamap_settings.py b/backend/services/liveuamap_settings.py
index 5b3d9ec..5e3d36a 100644
--- a/backend/services/liveuamap_settings.py
+++ b/backend/services/liveuamap_settings.py
@@ -1,4 +1,10 @@
-"""LiveUAMap Playwright scraper opt-in (#348) — UI consent on Windows."""
+"""LiveUAMap provider settings and operator-consent state.
+
+Global Incidents is a broader Shadowbroker feature backed by GDELT regardless
+of whether LiveUAMap enrichment is available. The browser provider keeps the
+historical platform behavior (automatic on POSIX, opt-in on Windows) while a
+configured supported API can satisfy LiveUAMap enrichment without Chromium.
+"""
from __future__ import annotations
@@ -8,6 +14,7 @@ import os
import threading
from pathlib import Path
from typing import Any
+from urllib.parse import urlparse
logger = logging.getLogger(__name__)
@@ -19,33 +26,59 @@ def _env_flag(name: str) -> str:
return str(os.getenv(name, "")).strip().lower()
+def _valid_https_url(raw: str) -> bool:
+ try:
+ parsed = urlparse(raw)
+ except ValueError:
+ return False
+ return parsed.scheme.lower() == "https" and bool(parsed.netloc)
+
+
def liveuamap_requires_ui_opt_in() -> bool:
- """Windows local installs need explicit consent before Playwright contacts LiveUAMap."""
+ """Windows local installs need an explicit choice before browser scraping."""
return os.name == "nt"
+def liveuamap_ui_choice_recorded() -> bool:
+ """Whether the operator has already accepted or declined browser contact."""
+ return _OPT_IN_FILE.exists()
+
+
def get_liveuamap_ui_opt_in() -> bool:
if not _OPT_IN_FILE.exists():
return False
try:
payload = json.loads(_OPT_IN_FILE.read_text(encoding="utf-8"))
return bool(payload.get("opted_in"))
- except (OSError, json.JSONDecodeError, TypeError) as e:
- logger.warning("LiveUAMap opt-in file unreadable: %s", e)
+ except (OSError, json.JSONDecodeError, TypeError) as exc:
+ logger.warning("LiveUAMap opt-in file unreadable: %s", exc)
return False
def set_liveuamap_ui_opt_in(opted_in: bool) -> None:
+ """Persist an explicit browser-provider choice, including a decline."""
_OPT_IN_FILE.parent.mkdir(parents=True, exist_ok=True)
+ payload = json.dumps({"opted_in": bool(opted_in)}, indent=2)
with _OPT_IN_LOCK:
- _OPT_IN_FILE.write_text(
- json.dumps({"opted_in": bool(opted_in)}, indent=2),
- encoding="utf-8",
- )
+ temp_path = _OPT_IN_FILE.with_suffix(_OPT_IN_FILE.suffix + ".tmp")
+ temp_path.write_text(payload, encoding="utf-8")
+ os.replace(temp_path, _OPT_IN_FILE)
-def liveuamap_scraper_enabled() -> bool:
- """Whether the Playwright LiveUAMap scraper may run on this backend."""
+def liveuamap_api_configured() -> bool:
+ """Whether an operator supplied a syntactically valid HTTPS API endpoint."""
+ url = str(os.getenv("LIVEUAMAP_API_URL", "") or "").strip()
+ return bool(url and _valid_https_url(url))
+
+
+def liveuamap_browser_scraper_enabled() -> bool:
+ """Whether the existing Playwright provider may contact LiveUAMap.
+
+ Preserve the established UX on Linux/macOS/Docker: browser enrichment is
+ available when Global Incidents is active unless explicitly disabled.
+ Windows keeps the existing opt-in boundary. An environment override always
+ wins for the browser provider only; it does not disable a configured API.
+ """
setting = _env_flag("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER")
if setting in {"1", "true", "yes", "on"}:
return True
@@ -56,6 +89,15 @@ def liveuamap_scraper_enabled() -> bool:
return get_liveuamap_ui_opt_in()
+def liveuamap_scraper_enabled() -> bool:
+ """Historical scheduler gate: whether *any* LiveUAMap provider can run.
+
+ The name is retained for call-site compatibility. Supported API access is
+ preferred when configured; otherwise the optional browser provider may run.
+ """
+ return liveuamap_api_configured() or liveuamap_browser_scraper_enabled()
+
+
def liveuamap_scraper_status() -> dict[str, Any]:
setting = _env_flag("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER")
env_override = None
@@ -63,11 +105,29 @@ def liveuamap_scraper_status() -> dict[str, Any]:
env_override = "on"
elif setting in {"0", "false", "no", "off"}:
env_override = "off"
+
ui_opted_in = get_liveuamap_ui_opt_in()
requires = liveuamap_requires_ui_opt_in()
+ api_configured = liveuamap_api_configured()
+ browser_enabled = liveuamap_browser_scraper_enabled()
+ enrichment_enabled = api_configured or browser_enabled
+
+ if api_configured:
+ provider_mode = "api"
+ elif browser_enabled:
+ provider_mode = "scraper"
+ else:
+ provider_mode = "gdelt-only"
+
return {
+ # Existing fields remain stable for current frontends.
"platform_requires_opt_in": requires,
"ui_opted_in": ui_opted_in,
- "scraper_enabled": liveuamap_scraper_enabled(),
+ "scraper_enabled": browser_enabled,
"env_override": env_override,
+ # Additive provider/UX diagnostics.
+ "ui_choice_recorded": liveuamap_ui_choice_recorded(),
+ "api_configured": api_configured,
+ "enrichment_enabled": enrichment_enabled,
+ "provider_mode": provider_mode,
}
diff --git a/backend/tests/test_liveuamap_parser.py b/backend/tests/test_liveuamap_parser.py
new file mode 100644
index 0000000..7474e0c
--- /dev/null
+++ b/backend/tests/test_liveuamap_parser.py
@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+import base64
+import json
+from urllib.parse import quote
+
+from services.liveuamap_parser import (
+ extract_ovens_expression,
+ iter_valid_coordinates,
+ normalize_liveuamap_payload,
+)
+
+
+def _ids(value):
+ return [str(item.get("id")) for item in normalize_liveuamap_payload(value)]
+
+
+def test_plain_marker_list():
+ payload = [{"id": 1, "lat": 1, "lng": 2, "title": "a"}]
+ assert _ids(payload) == ["1"]
+
+
+def test_double_encoded_json():
+ payload = json.dumps(json.dumps([{"id": "double", "lat": 1, "lng": 2}]))
+ assert _ids(payload) == ["double"]
+
+
+def test_list_of_json_strings_regression_517():
+ payload = [
+ json.dumps({"id": "a", "lat": 10, "lng": 20}),
+ json.dumps({"id": "b", "lat": 30, "lng": 40}),
+ ]
+ assert _ids(payload) == ["a", "b"]
+
+
+def test_mapping_key_becomes_fallback_marker_id():
+ payload = {"123": {"lat": 1, "lng": 2, "title": "keyed"}}
+ markers = normalize_liveuamap_payload(payload)
+ assert markers[0]["id"] == "123"
+
+
+def test_common_wrapper_shape():
+ payload = {"data": {"markers": [{"id": "wrapped", "lat": 1, "lng": 2}]}}
+ assert _ids(payload) == ["wrapped"]
+
+
+def test_legacy_urlencoded_base64_json():
+ raw = json.dumps([{"id": "legacy", "lat": 1, "lng": 2}]).encode()
+ payload = quote(base64.b64encode(raw).decode())
+ assert _ids(payload) == ["legacy"]
+
+
+def test_geojson_feature_collection():
+ payload = {
+ "type": "FeatureCollection",
+ "features": [
+ {
+ "type": "Feature",
+ "id": "geo",
+ "geometry": {"type": "Point", "coordinates": [20, 10]},
+ "properties": {"title": "Geo event"},
+ }
+ ],
+ }
+ markers = normalize_liveuamap_payload(payload)
+ assert markers == [{"title": "Geo event", "lat": 10.0, "lng": 20.0, "id": "geo"}]
+
+
+def test_malformed_scalars_are_ignored_instead_of_crashing():
+ payload = ["not-json", 42, None, True, {"nested": object()}]
+ assert normalize_liveuamap_payload(payload) == []
+
+
+def test_coordinate_iterator_rejects_out_of_range_and_nonfinite():
+ markers = [
+ {"id": "good", "lat": "10", "lng": "20"},
+ {"id": "bad-lat", "lat": 100, "lng": 20},
+ {"id": "bad-lng", "lat": 10, "lng": 200},
+ {"id": "nan", "lat": float("nan"), "lng": 20},
+ ]
+ valid = list(iter_valid_coordinates(markers))
+ assert [(item[0]["id"], item[1], item[2]) for item in valid] == [("good", 10.0, 20.0)]
+
+
+def test_extracts_var_let_and_const_ovens():
+ assert extract_ovens_expression('') == '[{"id":1}]'
+ assert extract_ovens_expression('') == '"abc"'
+ assert extract_ovens_expression('') == '{"data":[]}'
diff --git a/backend/tests/test_liveuamap_provider.py b/backend/tests/test_liveuamap_provider.py
new file mode 100644
index 0000000..8455e62
--- /dev/null
+++ b/backend/tests/test_liveuamap_provider.py
@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import requests
+
+from services import liveuamap_scraper as scraper
+from services import liveuamap_settings as settings
+
+
+class _Response:
+ def __init__(self, payload, status_code=200):
+ self._payload = payload
+ self.status_code = status_code
+
+ def raise_for_status(self):
+ if self.status_code >= 400:
+ raise requests.HTTPError(f"HTTP {self.status_code}")
+
+ def json(self):
+ return self._payload
+
+
+def test_api_geojson_is_normalized_and_auth_header_is_sent(monkeypatch):
+ monkeypatch.setenv("LIVEUAMAP_API_URL", "https://api.example.test/events")
+ monkeypatch.setenv("LIVEUAMAP_API_KEY", "secret-key")
+ monkeypatch.setenv("LIVEUAMAP_API_TIMEOUT_S", "12")
+ monkeypatch.setattr(
+ "services.network_utils.outbound_user_agent",
+ lambda purpose="": f"operator-test ({purpose})",
+ )
+ seen = {}
+
+ def fake_get(url, *, headers, timeout):
+ seen.update(url=url, headers=headers, timeout=timeout)
+ return _Response(
+ {
+ "type": "FeatureCollection",
+ "features": [
+ {
+ "type": "Feature",
+ "id": "evt-1",
+ "geometry": {"type": "Point", "coordinates": [30.5, 50.5]},
+ "properties": {"title": "Event", "url": "https://example.test/e/1"},
+ }
+ ],
+ }
+ )
+
+ monkeypatch.setattr(scraper.requests, "get", fake_get)
+ markers = scraper._fetch_liveuamap_api()
+ assert markers[0]["id"] == "evt-1"
+ assert markers[0]["lat"] == 50.5
+ assert markers[0]["lng"] == 30.5
+ assert markers[0]["provider"] == "api"
+ assert seen["headers"]["Authorization"] == "Bearer secret-key"
+ assert seen["timeout"] == (5, 12)
+
+
+def test_api_failure_falls_back_to_browser_when_browser_is_allowed(monkeypatch):
+ monkeypatch.setattr(settings, "liveuamap_api_configured", lambda: True)
+ monkeypatch.setattr(settings, "liveuamap_browser_scraper_enabled", lambda: True)
+ monkeypatch.setattr(scraper, "_fetch_liveuamap_api", lambda: (_ for _ in ()).throw(requests.Timeout("boom")))
+ monkeypatch.setattr(scraper, "_fetch_liveuamap_browser", lambda: [{"id": "browser"}])
+ assert scraper.fetch_liveuamap() == [{"id": "browser"}]
+
+
+def test_api_failure_does_not_force_browser_when_browser_is_disabled(monkeypatch):
+ monkeypatch.setattr(settings, "liveuamap_api_configured", lambda: True)
+ monkeypatch.setattr(settings, "liveuamap_browser_scraper_enabled", lambda: False)
+ monkeypatch.setattr(scraper, "_fetch_liveuamap_api", lambda: (_ for _ in ()).throw(requests.Timeout("boom")))
+ called = False
+
+ def browser():
+ nonlocal called
+ called = True
+ return []
+
+ monkeypatch.setattr(scraper, "_fetch_liveuamap_browser", browser)
+ assert scraper.fetch_liveuamap() == []
+ assert called is False
+
+
+def test_browser_disable_does_not_disable_configured_api_scheduler_gate(monkeypatch, tmp_path):
+ monkeypatch.setattr(settings, "_OPT_IN_FILE", tmp_path / "choice.json")
+ monkeypatch.setattr(settings.os, "name", "nt")
+ monkeypatch.setenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", "false")
+ monkeypatch.setenv("LIVEUAMAP_API_URL", "https://api.example.test/events")
+ assert settings.liveuamap_browser_scraper_enabled() is False
+ assert settings.liveuamap_scraper_enabled() is True
+
+
+def test_http_api_endpoint_is_not_used(monkeypatch):
+ monkeypatch.setenv("LIVEUAMAP_API_URL", "http://api.example.test/events")
+ monkeypatch.setattr(scraper, "_fetch_liveuamap_browser", lambda: [])
+ assert scraper._api_url() == ""
diff --git a/backend/tests/test_liveuamap_scraper_opt_in.py b/backend/tests/test_liveuamap_scraper_opt_in.py
index 762a346..c82b186 100644
--- a/backend/tests/test_liveuamap_scraper_opt_in.py
+++ b/backend/tests/test_liveuamap_scraper_opt_in.py
@@ -1,8 +1,8 @@
-"""LiveUAMap scraper UI opt-in on Windows (#348)."""
+"""LiveUAMap provider opt-in and compatibility behavior."""
+
from __future__ import annotations
import json
-from pathlib import Path
import pytest
@@ -13,33 +13,63 @@ from services import liveuamap_settings as settings
def opt_in_file(tmp_path, monkeypatch):
path = tmp_path / "liveuamap_scraper_opt_in.json"
monkeypatch.setattr(settings, "_OPT_IN_FILE", path)
+ monkeypatch.delenv("LIVEUAMAP_API_URL", raising=False)
+ monkeypatch.delenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", raising=False)
return path
-def test_windows_defaults_off_without_opt_in(monkeypatch, opt_in_file):
+def test_windows_defaults_browser_off_without_choice(monkeypatch, opt_in_file):
monkeypatch.setattr(settings.os, "name", "nt")
- monkeypatch.delenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", raising=False)
- assert settings.liveuamap_scraper_enabled() is False
assert settings.liveuamap_requires_ui_opt_in() is True
+ assert settings.liveuamap_ui_choice_recorded() is False
+ assert settings.liveuamap_browser_scraper_enabled() is False
+ assert settings.liveuamap_scraper_enabled() is False
-def test_windows_opt_in_enables_scraper(monkeypatch, opt_in_file):
+def test_windows_opt_in_enables_browser(monkeypatch, opt_in_file):
monkeypatch.setattr(settings.os, "name", "nt")
- monkeypatch.delenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", raising=False)
settings.set_liveuamap_ui_opt_in(True)
- assert settings.liveuamap_scraper_enabled() is True
+ assert settings.liveuamap_ui_choice_recorded() is True
+ assert settings.liveuamap_browser_scraper_enabled() is True
assert json.loads(opt_in_file.read_text())["opted_in"] is True
-def test_linux_enabled_without_opt_in(monkeypatch, opt_in_file):
+def test_windows_decline_is_recorded_without_enabling_browser(monkeypatch, opt_in_file):
+ monkeypatch.setattr(settings.os, "name", "nt")
+ settings.set_liveuamap_ui_opt_in(False)
+ assert settings.liveuamap_ui_choice_recorded() is True
+ assert settings.get_liveuamap_ui_opt_in() is False
+ assert settings.liveuamap_browser_scraper_enabled() is False
+
+
+def test_linux_preserves_existing_auto_enrichment_default(monkeypatch, opt_in_file):
monkeypatch.setattr(settings.os, "name", "posix")
- monkeypatch.delenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", raising=False)
assert settings.liveuamap_requires_ui_opt_in() is False
+ assert settings.liveuamap_browser_scraper_enabled() is True
assert settings.liveuamap_scraper_enabled() is True
-def test_env_force_off_overrides_ui_opt_in(monkeypatch, opt_in_file):
+def test_env_force_off_disables_browser_even_after_opt_in(monkeypatch, opt_in_file):
monkeypatch.setattr(settings.os, "name", "nt")
- monkeypatch.setenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", "false")
settings.set_liveuamap_ui_opt_in(True)
+ monkeypatch.setenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", "false")
+ assert settings.liveuamap_browser_scraper_enabled() is False
+ assert settings.liveuamap_scraper_enabled() is False
+
+
+def test_api_provider_does_not_require_browser_consent(monkeypatch, opt_in_file):
+ monkeypatch.setattr(settings.os, "name", "nt")
+ monkeypatch.setenv("LIVEUAMAP_API_URL", "https://api.example.test/liveuamap")
+ status = settings.liveuamap_scraper_status()
+ assert status["api_configured"] is True
+ assert status["scraper_enabled"] is False
+ assert status["enrichment_enabled"] is True
+ assert status["provider_mode"] == "api"
+ assert settings.liveuamap_scraper_enabled() is True
+
+
+def test_invalid_http_api_url_does_not_count_as_configured(monkeypatch, opt_in_file):
+ monkeypatch.setattr(settings.os, "name", "nt")
+ monkeypatch.setenv("LIVEUAMAP_API_URL", "http://api.example.test/liveuamap")
+ assert settings.liveuamap_api_configured() is False
assert settings.liveuamap_scraper_enabled() is False
diff --git a/docker-compose.yml b/docker-compose.yml
index 707a25d..d6806b3 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -26,6 +26,15 @@ services:
- ADMIN_KEY=${ADMIN_KEY:-}
- FINNHUB_API_KEY=${FINNHUB_API_KEY:-}
- AIRFRAMES_API_KEY=${AIRFRAMES_API_KEY:-}
+ # LiveUAMap is optional enrichment for Global Incidents. GDELT remains
+ # available regardless. The browser provider preserves existing Linux/
+ # Docker behavior unless explicitly disabled; paid API access is optional.
+ - SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=${SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER:-}
+ - LIVEUAMAP_API_URL=${LIVEUAMAP_API_URL:-}
+ - LIVEUAMAP_API_KEY=${LIVEUAMAP_API_KEY:-}
+ - LIVEUAMAP_API_AUTH_HEADER=${LIVEUAMAP_API_AUTH_HEADER:-Authorization}
+ - LIVEUAMAP_API_AUTH_SCHEME=${LIVEUAMAP_API_AUTH_SCHEME:-Bearer}
+ - LIVEUAMAP_API_TIMEOUT_S=${LIVEUAMAP_API_TIMEOUT_S:-30}
# Override allowed CORS origins (comma-separated). Auto-detects LAN IPs if empty.
- CORS_ORIGINS=${CORS_ORIGINS:-}
# Private Infonet bootstrap seeds. Seeds are discovery hints, not fixed roots.
diff --git a/docs/LIVEUAMAP.md b/docs/LIVEUAMAP.md
new file mode 100644
index 0000000..3d5252c
--- /dev/null
+++ b/docs/LIVEUAMAP.md
@@ -0,0 +1,92 @@
+# LiveUAMap enrichment
+
+Shadowbroker's **Global Incidents** layer does not depend on LiveUAMap. GDELT
+remains the baseline incident source; LiveUAMap adds optional map-pin enrichment
+when one of the providers below is available.
+
+## Provider order
+
+1. **Supported LiveUAMap API (optional)** — preferred when an operator has a
+ paid/contracted API endpoint. Shadowbroker does not require this service.
+2. **Browser provider (best effort)** — the existing Playwright integration.
+ This remains enabled by default on Linux/macOS/Docker when Global Incidents
+ is active, preserving existing behavior. Windows asks once before allowing
+ the backend to contact LiveUAMap through the browser provider.
+3. **GDELT-only** — if neither LiveUAMap provider is usable, Global Incidents
+ still turns on and continues to receive GDELT data.
+
+A LiveUAMap provider failure must never disable the broader Global Incidents
+feature.
+
+## Supported API configuration
+
+Because LiveUAMap API endpoint/auth details are supplied under the operator's
+service agreement, Shadowbroker does not hard-code a vendor account endpoint.
+Configure the HTTPS JSON/GeoJSON URL you were given:
+
+```env
+LIVEUAMAP_API_URL=https://your-liveuamap-api-endpoint.example/events
+LIVEUAMAP_API_KEY=your-key
+LIVEUAMAP_API_AUTH_HEADER=Authorization
+LIVEUAMAP_API_AUTH_SCHEME=Bearer
+LIVEUAMAP_API_TIMEOUT_S=30
+```
+
+`LIVEUAMAP_API_KEY` is optional at the code level so deployments whose endpoint
+already contains/handles authentication can still use the provider. The API URL
+must use HTTPS. Keys are sent only in the configured request header and are not
+included in provider-status responses or logs.
+
+If the API request fails, Linux/macOS/Docker may fall back to the browser
+provider under the existing browser-provider policy. Windows falls back only if
+the operator separately opted into browser contact.
+
+## Browser-provider behavior
+
+The browser provider is **best effort** because it consumes an undocumented web
+page representation rather than a stable public schema. Shadowbroker therefore:
+
+- treats strings, wrapped objects, keyed objects, double-encoded JSON, legacy
+ base64 payloads, and GeoJSON as bounded parser inputs;
+- validates point coordinates before emitting markers;
+- skips malformed entries instead of failing an entire region;
+- detects obvious access/challenge pages and fails soft;
+- pauses repeated browser attempts after consecutive complete failures;
+- logs only structural payload diagnostics, not raw upstream payloads; and
+- retains the existing browser/stealth profile without adding new anti-bot
+ bypass techniques.
+
+### Docker browser location (#516)
+
+Published backend images install Playwright browsers into the shared
+`/ms-playwright` directory through `PLAYWRIGHT_BROWSERS_PATH`. The image build
+then verifies, as the non-root runtime user, that both Chromium and the matching
+headless-shell bundle are present and executable. This prevents the previous
+root-cache/runtime-user mismatch where the browser existed under `/root` while
+Playwright searched under `/app/.cache`.
+
+## Operator controls
+
+```env
+# Explicitly enable or disable only the browser provider.
+SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=true
+SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER=false
+```
+
+On Windows, the first Global Incidents enable offers LiveUAMap browser
+enrichment. **Accepting or declining never blocks Global Incidents itself.** A
+decline is remembered so the UI does not nag on every toggle. The environment
+flag remains the explicit override.
+
+On Linux/macOS/Docker, leaving the flag unset preserves the historical behavior:
+the browser provider may run while Global Incidents is active. Set it to
+`false` if the operator wants GDELT-only operation unless a supported API is
+configured.
+
+## Failure semantics
+
+LiveUAMap data is enrichment. If Chromium is missing, the upstream schema drifts,
+the site presents an access challenge, the paid API is unavailable, or every
+region returns malformed data, the provider returns no new pins and the error is
+contained. GDELT fetching and the rest of the Shadowbroker data pipeline continue
+independently.
diff --git a/frontend/src/__tests__/hooks/useLiveUamapScraperOptIn.test.ts b/frontend/src/__tests__/hooks/useLiveUamapScraperOptIn.test.ts
new file mode 100644
index 0000000..52859b4
--- /dev/null
+++ b/frontend/src/__tests__/hooks/useLiveUamapScraperOptIn.test.ts
@@ -0,0 +1,92 @@
+import { act, cleanup, renderHook, waitFor } from '@testing-library/react';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+import { useLiveUamapScraperOptIn } from '@/hooks/useLiveUamapScraperOptIn';
+
+afterEach(() => {
+ cleanup();
+ vi.restoreAllMocks();
+});
+
+describe('useLiveUamapScraperOptIn', () => {
+ it('never blocks Global Incidents when the operator declines LiveUAMap', async () => {
+ const fetchMock = vi
+ .spyOn(globalThis, 'fetch')
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ platform_requires_opt_in: true,
+ ui_opted_in: false,
+ ui_choice_recorded: false,
+ scraper_enabled: false,
+ env_override: null,
+ api_configured: false,
+ enrichment_enabled: false,
+ provider_mode: 'gdelt-only',
+ }),
+ { status: 200 },
+ ),
+ )
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ platform_requires_opt_in: true,
+ ui_opted_in: false,
+ ui_choice_recorded: true,
+ scraper_enabled: false,
+ env_override: null,
+ api_configured: false,
+ enrichment_enabled: false,
+ provider_mode: 'gdelt-only',
+ }),
+ { status: 200 },
+ ),
+ );
+ const confirmMock = vi.spyOn(window, 'confirm').mockReturnValue(false);
+
+ const { result } = renderHook(() => useLiveUamapScraperOptIn());
+ await waitFor(() => expect(result.current.status).not.toBeNull());
+
+ let blocked = true;
+ act(() => {
+ blocked = result.current.needsConsentBeforeEnable('global_incidents', true);
+ });
+
+ expect(blocked).toBe(false);
+ expect(confirmMock).toHaveBeenCalledOnce();
+ await waitFor(
+ () => {
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ },
+ { timeout: 1000 },
+ );
+ const [, options] = fetchMock.mock.calls[1];
+ expect(options?.method).toBe('POST');
+ expect(options?.body).toBe(JSON.stringify({ opted_in: false }));
+ });
+
+ it('does not prompt when a supported API provider is configured', async () => {
+ vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ platform_requires_opt_in: true,
+ ui_opted_in: false,
+ ui_choice_recorded: false,
+ scraper_enabled: false,
+ env_override: null,
+ api_configured: true,
+ enrichment_enabled: true,
+ provider_mode: 'api',
+ }),
+ { status: 200 },
+ ),
+ );
+ const confirmMock = vi.spyOn(window, 'confirm').mockReturnValue(true);
+
+ const { result } = renderHook(() => useLiveUamapScraperOptIn());
+ await waitFor(() => expect(result.current.status?.api_configured).toBe(true));
+
+ expect(result.current.needsConsentBeforeEnable('global_incidents', true)).toBe(false);
+ expect(confirmMock).not.toHaveBeenCalled();
+ });
+});
diff --git a/frontend/src/hooks/useLiveUamapScraperOptIn.ts b/frontend/src/hooks/useLiveUamapScraperOptIn.ts
index 257f582..0090c5d 100644
--- a/frontend/src/hooks/useLiveUamapScraperOptIn.ts
+++ b/frontend/src/hooks/useLiveUamapScraperOptIn.ts
@@ -1,6 +1,6 @@
'use client';
-import { useCallback, useEffect, useState } from 'react';
+import { useCallback, useEffect, useRef, useState } from 'react';
import { API_BASE } from '@/lib/api';
export type LiveUamapScraperStatus = {
@@ -8,10 +8,15 @@ export type LiveUamapScraperStatus = {
ui_opted_in: boolean;
scraper_enabled: boolean;
env_override: 'on' | 'off' | null;
+ ui_choice_recorded?: boolean;
+ api_configured?: boolean;
+ enrichment_enabled?: boolean;
+ provider_mode?: 'api' | 'scraper' | 'gdelt-only';
};
export function useLiveUamapScraperOptIn(enabled = true) {
const [status, setStatus] = useState(null);
+ const choicePromptedRef = useRef(false);
const refreshStatus = useCallback(async () => {
try {
@@ -29,20 +34,11 @@ export function useLiveUamapScraperOptIn(enabled = true) {
void refreshStatus();
}, [enabled, refreshStatus]);
- const needsConsentBeforeEnable = useCallback(
- (layerId: string, turningOn: boolean) =>
- layerId === 'global_incidents' &&
- turningOn &&
- Boolean(status?.platform_requires_opt_in) &&
- !status?.ui_opted_in,
- [status],
- );
-
- const confirmOptIn = useCallback(async () => {
+ const setOptIn = useCallback(async (optedIn: boolean) => {
const res = await fetch(`${API_BASE}/api/liveuamap/scraper-opt-in`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
- body: JSON.stringify({ opted_in: true }),
+ body: JSON.stringify({ opted_in: optedIn }),
});
if (!res.ok) {
throw new Error(`LiveUAMap opt-in failed (${res.status})`);
@@ -52,6 +48,45 @@ export function useLiveUamapScraperOptIn(enabled = true) {
return body;
}, []);
+ const needsConsentBeforeEnable = useCallback(
+ (layerId: string, turningOn: boolean) => {
+ if (layerId !== 'global_incidents' || !turningOn) return false;
+
+ const choiceRecorded = status?.ui_choice_recorded ?? status?.ui_opted_in ?? false;
+ const shouldOfferBrowserEnrichment =
+ Boolean(status?.platform_requires_opt_in) &&
+ !choiceRecorded &&
+ !status?.api_configured &&
+ status?.env_override === null;
+
+ if (
+ shouldOfferBrowserEnrichment &&
+ !choicePromptedRef.current &&
+ typeof window !== 'undefined'
+ ) {
+ choicePromptedRef.current = true;
+ const optedIn = window.confirm(
+ "Global Incidents will turn on with GDELT either way. Add optional LiveUAMap pins too? LiveUAMap will see this server's IP. OK enables LiveUAMap; Cancel keeps GDELT-only incidents.",
+ );
+
+ // Do not make the Global Incidents toggle wait on an optional provider.
+ // Give the layer-state update a moment to reach the backend before the
+ // opt-in endpoint opportunistically starts an immediate refresh.
+ window.setTimeout(() => {
+ void setOptIn(optedIn).catch((error) => {
+ console.warn('LiveUAMap preference update failed:', error);
+ });
+ }, 250);
+ }
+
+ // LiveUAMap is enrichment, never a prerequisite for Global Incidents.
+ return false;
+ },
+ [setOptIn, status],
+ );
+
+ const confirmOptIn = useCallback(() => setOptIn(true), [setOptIn]);
+
return {
status,
refreshStatus,