fix(liveuamap): make enrichment resilient and non-blocking

backend/services/liveuamap_parser.py

@@ -0,0 +1,249 @@
+"""Defensive parsing helpers for LiveUAMap provider payloads.
+
+LiveUAMap's browser page exposes an undocumented ``ovens`` value whose shape
+has changed over time. The optional supported API may also return JSON or
+GeoJSON. Keep representation decoding and schema normalization isolated here so
+upstream drift degrades one provider instead of crashing the fetch scheduler.
+"""
+
+from __future__ import annotations
+
+import ast
+import base64
+import binascii
+import json
+import math
+import re
+from collections.abc import Iterable
+from typing import Any
+from urllib.parse import unquote
+
+_MAX_DECODE_DEPTH = 5
+_MAX_CANDIDATES = 10_000
+_MAX_STRING_BYTES = 8 * 1024 * 1024
+_WRAPPER_KEYS = ("ovens", "markers", "items", "events", "data", "results", "features")
+
+
+def extract_ovens_expression(html: str) -> str | None:
+    """Extract a legacy ``var/let/const ovens = ...;`` expression from HTML.
+
+    Evaluating ``window.ovens`` in the browser is preferred; this exists only
+    as a fallback for pages that still embed the value in source text.
+    """
+    if not html:
+        return None
+    match = re.search(
+        r"(?:var|let|const)\s+ovens\s*=\s*(.+?);(?=\s*(?:</script>|(?:var|let|const|function)\b|$))",
+        html,
+        re.DOTALL | re.IGNORECASE,
+    )
+    if match:
+        return match.group(1).strip()
+
+    # Compatibility with older pages where the next token after the semicolon
+    # is arbitrary markup rather than another JavaScript declaration.
+    match = re.search(r"(?:var|let|const)\s+ovens\s*=\s*(.*?);", html, re.DOTALL | re.IGNORECASE)
+    return match.group(1).strip() if match else None
+
+
+def payload_shape(value: Any) -> str:
+    """Return a non-sensitive structural description for drift diagnostics."""
+    if isinstance(value, dict):
+        keys = sorted(str(key) for key in value.keys())[:8]
+        return f"dict(keys={keys}, size={len(value)})"
+    if isinstance(value, list):
+        item_types = sorted({type(item).__name__ for item in value[:20]})
+        return f"list(size={len(value)}, item_types={item_types})"
+    if isinstance(value, str):
+        return f"str(len={len(value)})"
+    return type(value).__name__
+
+
+def normalize_liveuamap_payload(value: Any) -> list[dict[str, Any]]:
+    """Normalize JSON/GeoJSON/legacy payload shapes into marker dictionaries.
+
+    Unknown or malformed values are ignored. This function intentionally never
+    assumes iterable items are mappings; issue #517 was caused by calling
+    ``.get`` on strings after an upstream representation change.
+    """
+    out: list[dict[str, Any]] = []
+    _collect(value, out, depth=0, inherited_id=None)
+    return out[:_MAX_CANDIDATES]
+
+
+def _collect(
+    value: Any,
+    out: list[dict[str, Any]],
+    *,
+    depth: int,
+    inherited_id: str | None,
+) -> None:
+    if depth > _MAX_DECODE_DEPTH or len(out) >= _MAX_CANDIDATES:
+        return
+
+    if value is None:
+        return
+
+    if isinstance(value, str):
+        decoded = _decode_string(value)
+        if decoded is value or decoded == value:
+            return
+        _collect(decoded, out, depth=depth + 1, inherited_id=inherited_id)
+        return
+
+    if isinstance(value, list):
+        for item in value[:_MAX_CANDIDATES - len(out)]:
+            _collect(item, out, depth=depth + 1, inherited_id=None)
+            if len(out) >= _MAX_CANDIDATES:
+                break
+        return
+
+    if not isinstance(value, dict):
+        return
+
+    # GeoJSON FeatureCollection / Feature.
+    if value.get("type") == "FeatureCollection" and isinstance(value.get("features"), list):
+        _collect(value["features"], out, depth=depth + 1, inherited_id=None)
+        return
+    if value.get("type") == "Feature":
+        marker = _marker_from_geojson_feature(value)
+        if marker is not None:
+            if inherited_id and not marker.get("id"):
+                marker["id"] = inherited_id
+            out.append(marker)
+        return
+
+    # Common wrapper shapes returned by APIs or page-side serialization.
+    for key in _WRAPPER_KEYS:
+        if key in value and isinstance(value[key], (dict, list, str)):
+            _collect(value[key], out, depth=depth + 1, inherited_id=None)
+            return
+
+    # A direct marker is accepted even if coordinates are absent here; the
+    # provider formatter performs the final coordinate/range validation.
+    if _looks_like_marker(value):
+        marker = dict(value)
+        if inherited_id and not marker.get("id"):
+            marker["id"] = inherited_id
+        out.append(marker)
+        return
+
+    # Some versions expose a dictionary keyed by marker ID. Traverse mapping
+    # values while preserving the key as a fallback identifier.
+    traversable = [
+        (str(key), item)
+        for key, item in value.items()
+        if isinstance(item, (dict, list, str))
+    ]
+    if traversable:
+        for key, item in traversable[:_MAX_CANDIDATES - len(out)]:
+            _collect(item, out, depth=depth + 1, inherited_id=key)
+            if len(out) >= _MAX_CANDIDATES:
+                break
+
+
+def _decode_string(raw: str) -> Any:
+    text = raw.strip()
+    if not text or len(text.encode("utf-8", errors="ignore")) > _MAX_STRING_BYTES:
+        return raw
+
+    # A JavaScript string literal can include escaping that plain strip("'")
+    # corrupts. literal_eval safely handles quoted string syntax only.
+    if len(text) >= 2 and text[0] == text[-1] and text[0] in {"'", '"'}:
+        try:
+            literal = ast.literal_eval(text)
+        except (SyntaxError, ValueError):
+            literal = None
+        if isinstance(literal, str) and literal != text:
+            return literal
+
+    decoded = _try_json(text)
+    if decoded is not None:
+        return decoded
+
+    # Legacy LiveUAMap payloads have appeared URL-encoded before decoding.
+    url_decoded = unquote(text)
+    if url_decoded != text:
+        decoded = _try_json(url_decoded)
+        if decoded is not None:
+            return decoded
+        text = url_decoded
+
+    # Older scraper versions expected a base64-wrapped JSON blob. Decode only
+    # when the result itself is valid JSON, so arbitrary titles/IDs are never
+    # interpreted as base64 data.
+    compact = "".join(text.split())
+    if compact and len(compact) % 4 == 0:
+        try:
+            raw_bytes = base64.b64decode(compact, validate=True)
+            decoded_text = raw_bytes.decode("utf-8")
+        except (binascii.Error, UnicodeDecodeError, ValueError):
+            decoded_text = ""
+        if decoded_text:
+            decoded = _try_json(decoded_text)
+            if decoded is not None:
+                return decoded
+
+    return raw
+
+
+def _try_json(text: str) -> Any | None:
+    try:
+        return json.loads(text)
+    except (json.JSONDecodeError, TypeError, ValueError):
+        return None
+
+
+def _looks_like_marker(value: dict[str, Any]) -> bool:
+    keys = set(value)
+    if {"lat", "lng"}.issubset(keys) or {"lat", "lon"}.issubset(keys):
+        return True
+    if "latitude" in keys and ("longitude" in keys or "lon" in keys or "lng" in keys):
+        return True
+    marker_metadata = {"id", "s", "title", "d", "desc", "description", "link", "url", "time", "t"}
+    return bool(keys.intersection(marker_metadata)) and not any(key in value for key in _WRAPPER_KEYS)
+
+
+def _marker_from_geojson_feature(feature: dict[str, Any]) -> dict[str, Any] | None:
+    geometry = feature.get("geometry")
+    if not isinstance(geometry, dict) or geometry.get("type") != "Point":
+        return None
+    coordinates = geometry.get("coordinates")
+    if not isinstance(coordinates, (list, tuple)) or len(coordinates) < 2:
+        return None
+    lng = _finite_coordinate(coordinates[0], minimum=-180.0, maximum=180.0)
+    lat = _finite_coordinate(coordinates[1], minimum=-90.0, maximum=90.0)
+    if lat is None or lng is None:
+        return None
+    properties = feature.get("properties")
+    marker = dict(properties) if isinstance(properties, dict) else {}
+    marker.setdefault("lat", lat)
+    marker.setdefault("lng", lng)
+    feature_id = feature.get("id")
+    if feature_id is not None:
+        marker.setdefault("id", feature_id)
+    return marker
+
+
+def _finite_coordinate(value: Any, *, minimum: float, maximum: float) -> float | None:
+    try:
+        number = float(value)
+    except (TypeError, ValueError):
+        return None
+    if not math.isfinite(number) or not minimum <= number <= maximum:
+        return None
+    return number
+
+
+def iter_valid_coordinates(markers: Iterable[dict[str, Any]]) -> Iterable[tuple[dict[str, Any], float, float]]:
+    """Yield markers with finite in-range latitude/longitude values."""
+    for marker in markers:
+        if not isinstance(marker, dict):
+            continue
+        lat = marker.get("lat", marker.get("latitude"))
+        lng = marker.get("lng", marker.get("lon", marker.get("longitude")))
+        lat_value = _finite_coordinate(lat, minimum=-90.0, maximum=90.0)
+        lng_value = _finite_coordinate(lng, minimum=-180.0, maximum=180.0)
+        if lat_value is None or lng_value is None:
+            continue
+        yield marker, lat_value, lng_value

backend/services/liveuamap_scraper.py

@@ -1,145 +1,439 @@
-import json
+"""Resilient LiveUAMap enrichment providers.
+
+Global Incidents itself is backed independently by GDELT. This module adds
+LiveUAMap pins when either an operator-configured supported API is available or
+the existing Playwright provider is allowed. Provider failures are isolated and
+return an empty enrichment set instead of breaking the scheduler.
+
+The browser provider intentionally does not add any new anti-bot behavior. It
+retains the repository's pre-existing Playwright/stealth profile for backward
+compatibility while making parsing, packaging failures, and upstream drift
+fail-soft.
+"""
+
+from __future__ import annotations
+
+import hashlib
 import logging
-import base64
-import urllib.parse
-import re
-from playwright.sync_api import sync_playwright
-from playwright_stealth import stealth_sync
+import math
+import os
+import threading
+import time
+from datetime import datetime, timezone
+from typing import Any
+from urllib.parse import urljoin, urlparse
+
+import requests
+
+from services.liveuamap_parser import (
+    extract_ovens_expression,
+    iter_valid_coordinates,
+    normalize_liveuamap_payload,
+    payload_shape,
+)
 
 logger = logging.getLogger(__name__)
 
+_REGIONS = (
+    {"name": "Ukraine", "url": "https://liveuamap.com"},
+    {"name": "Middle East", "url": "https://mideast.liveuamap.com"},
+    {"name": "Israel-Palestine", "url": "https://israelpalestine.liveuamap.com"},
+    {"name": "Syria", "url": "https://syria.liveuamap.com"},
+)
 
-def fetch_liveuamap():
-    logger.info("Starting Liveuamap scraper with Playwright Stealth...")
+_BROWSER_FAILURE_THRESHOLD = 3
+_BROWSER_BACKOFF_BASE_S = 15 * 60
+_BROWSER_BACKOFF_MAX_S = 6 * 60 * 60
+_browser_failures = 0
+_browser_blocked_until = 0.0
+_browser_health_lock = threading.Lock()
 
-    regions = [
-        {"name": "Ukraine", "url": "https://liveuamap.com"},
-        {"name": "Middle East", "url": "https://mideast.liveuamap.com"},
-        {"name": "Israel-Palestine", "url": "https://israelpalestine.liveuamap.com"},
-        {"name": "Syria", "url": "https://syria.liveuamap.com"},
-    ]
+_CHALLENGE_MARKERS = (
+    "cf-turnstile",
+    "challenge-platform",
+    "just a moment",
+    "checking your browser",
+    "verify you are human",
+)
 
-    all_markers = []
-    seen_ids = set()
 
-    with sync_playwright() as p:
-        # Launching with a real user agent to bypass Turnstile
-        browser = p.chromium.launch(
-            headless=True, args=["--disable-blink-features=AutomationControlled"]
+def _bounded_int_env(name: str, default: int, *, minimum: int, maximum: int) -> int:
+    try:
+        value = int(str(os.getenv(name, default)).strip())
+    except (TypeError, ValueError):
+        value = default
+    return max(minimum, min(maximum, value))
+
+
+def _safe_header_name(raw: str, default: str) -> str:
+    value = (raw or "").strip()
+    if not value or any(ch in value for ch in "\r\n:"):
+        return default
+    if not all(ch.isalnum() or ch in "-_" for ch in value):
+        return default
+    return value
+
+
+def _api_url() -> str:
+    raw = str(os.getenv("LIVEUAMAP_API_URL", "") or "").strip()
+    if not raw:
+        return ""
+    try:
+        parsed = urlparse(raw)
+    except ValueError:
+        return ""
+    if parsed.scheme.lower() != "https" or not parsed.netloc:
+        return ""
+    return raw
+
+
+def _api_headers() -> dict[str, str]:
+    from services.network_utils import outbound_user_agent
+
+    headers = {
+        "Accept": "application/geo+json, application/json;q=0.9",
+        "User-Agent": outbound_user_agent("liveuamap-api"),
+    }
+    api_key = str(os.getenv("LIVEUAMAP_API_KEY", "") or "").strip()
+    if api_key:
+        header = _safe_header_name(
+            str(os.getenv("LIVEUAMAP_API_AUTH_HEADER", "Authorization") or ""),
+            "Authorization",
         )
-        from services.network_utils import outbound_user_agent
+        scheme = str(os.getenv("LIVEUAMAP_API_AUTH_SCHEME", "Bearer") or "").strip()
+        if any(ch in scheme for ch in "\r\n"):
+            scheme = "Bearer"
+        headers[header] = f"{scheme} {api_key}".strip() if scheme else api_key
+    return headers
 
-        # Per-install handle (no shared Shadowbroker product token). Stealth remains
-        # for Turnstile; see docs/OUTBOUND_DATA.md #348.
-        playwright_ua = (
-            f"Mozilla/5.0 (compatible; {outbound_user_agent('liveuamap')})"
-        )
-        context = browser.new_context(
-            user_agent=playwright_ua,
-            viewport={"width": 1920, "height": 1080},
-            color_scheme="dark",
-        )
-        # Bound navigation and script evaluation so a stuck region cannot hang the slow pool.
-        context.set_default_navigation_timeout(60_000)
-        context.set_default_timeout(30_000)
-        page = context.new_page()
-        stealth_sync(page)
 
-        for region in regions:
+def _fetch_liveuamap_api() -> list[dict[str, Any]]:
+    """Fetch an operator-configured supported LiveUAMap JSON/GeoJSON endpoint."""
+    url = _api_url()
+    if not url:
+        return []
+
+    timeout_s = _bounded_int_env("LIVEUAMAP_API_TIMEOUT_S", 30, minimum=5, maximum=120)
+    hostname = urlparse(url).hostname or "configured endpoint"
+    logger.info("Fetching LiveUAMap supported API from %s", hostname)
+    response = requests.get(url, headers=_api_headers(), timeout=(5, timeout_s))
+    response.raise_for_status()
+    try:
+        payload = response.json()
+    except (requests.JSONDecodeError, ValueError) as exc:
+        raise ValueError("LiveUAMap API did not return JSON/GeoJSON") from exc
+
+    candidates = normalize_liveuamap_payload(payload)
+    markers = _format_markers(
+        candidates,
+        region="LiveUAMap",
+        base_url=url,
+        provider="api",
+    )
+    if not markers:
+        raise ValueError(f"LiveUAMap API returned no recognizable point markers ({payload_shape(payload)})")
+    logger.info("LiveUAMap API returned %s normalized markers", len(markers))
+    return markers
+
+
+def _browser_circuit_open() -> tuple[bool, int]:
+    now = time.monotonic()
+    with _browser_health_lock:
+        remaining = max(0, int(_browser_blocked_until - now))
+        return remaining > 0, remaining
+
+
+def _record_browser_success() -> None:
+    global _browser_failures, _browser_blocked_until
+    with _browser_health_lock:
+        _browser_failures = 0
+        _browser_blocked_until = 0.0
+
+
+def _record_browser_failure(reason: str) -> None:
+    global _browser_failures, _browser_blocked_until
+    now = time.monotonic()
+    with _browser_health_lock:
+        _browser_failures += 1
+        failures = _browser_failures
+        if failures < _BROWSER_FAILURE_THRESHOLD:
+            logger.warning(
+                "LiveUAMap browser provider failure %s/%s: %s",
+                failures,
+                _BROWSER_FAILURE_THRESHOLD,
+                reason,
+            )
+            return
+        exponent = failures - _BROWSER_FAILURE_THRESHOLD
+        delay_s = min(_BROWSER_BACKOFF_BASE_S * (2**exponent), _BROWSER_BACKOFF_MAX_S)
+        _browser_blocked_until = max(_browser_blocked_until, now + delay_s)
+    logger.warning(
+        "LiveUAMap browser provider paused for %ss after repeated failures: %s",
+        delay_s,
+        reason,
+    )
+
+
+def _looks_like_challenge(html: str) -> bool:
+    lowered = (html or "").lower()
+    return any(marker in lowered for marker in _CHALLENGE_MARKERS)
+
+
+def _read_page_payload(page: Any, html: str) -> Any:
+    """Prefer evaluated page state, then fall back to the legacy source variable."""
+    try:
+        serialized = page.evaluate(
+            "() => typeof ovens !== 'undefined' ? JSON.stringify(ovens) : null"
+        )
+        if serialized:
+            return serialized
+    except Exception as exc:  # Playwright exception types differ across releases.
+        logger.debug("LiveUAMap ovens JS evaluation unavailable: %s", exc)
+
+    expression = extract_ovens_expression(html)
+    return expression if expression is not None else None
+
+
+def _fetch_liveuamap_browser() -> list[dict[str, Any]]:
+    open_now, remaining_s = _browser_circuit_open()
+    if open_now:
+        logger.info(
+            "LiveUAMap browser provider circuit open; skipping Chromium for another %ss",
+            remaining_s,
+        )
+        return []
+
+    # Import browser-only dependencies lazily so API-only deployments do not
+    # require Chromium just to import this module.
+    from playwright.sync_api import sync_playwright
+    from playwright_stealth import stealth_sync
+    from services.network_utils import outbound_user_agent
+
+    all_markers: list[dict[str, Any]] = []
+    seen_ids: set[str] = set()
+    successful_regions = 0
+    failed_regions = 0
+
+    try:
+        with sync_playwright() as playwright:
+            # Existing repository behavior retained for compatibility. This PR
+            # deliberately adds no further anti-detection/evasion measures.
+            browser = playwright.chromium.launch(
+                headless=True,
+                args=["--disable-blink-features=AutomationControlled"],
+            )
             try:
-                logger.info(f"Scraping Liveuamap region: {region['name']}")
-                page.goto(region["url"], timeout=60000, wait_until="domcontentloaded")
+                context = browser.new_context(
+                    user_agent=f"Mozilla/5.0 (compatible; {outbound_user_agent('liveuamap')})",
+                    viewport={"width": 1920, "height": 1080},
+                    color_scheme="dark",
+                )
+                context.set_default_navigation_timeout(60_000)
+                context.set_default_timeout(30_000)
+                page = context.new_page()
+                stealth_sync(page)
 
-                # Wait for the map canvas or markers script to load, max 10s wait
-                try:
-                    page.wait_for_timeout(5000)
-                except (TimeoutError, OSError):  # non-critical: page load delay
-                    pass
-
-                html = page.content()
-
-                m = re.search(r"var\s+ovens\s*=\s*(.*?);(?!function)", html, re.DOTALL)
-                if not m:
-                    logger.warning(f"Could not find 'ovens' data for {region['name']} in raw HTML")
-                    # Let's try grabbing the evaluated JavaScript variable if it's there
+                for region in _REGIONS:
                     try:
-                        ovens_json = page.evaluate(
-                            "() => typeof ovens !== 'undefined' ? JSON.stringify(ovens) : null"
+                        logger.info("Fetching LiveUAMap browser region: %s", region["name"])
+                        response = page.goto(
+                            region["url"],
+                            timeout=60_000,
+                            wait_until="domcontentloaded",
                         )
-                        if ovens_json:
-                            markers = json.loads(ovens_json)
-                            # process below
-                            html = f"var ovens={ovens_json};"
-                            m = re.search(r"var\s+ovens=(.*?);", html, re.DOTALL)
-                    except (ValueError, KeyError, OSError) as e:  # non-critical: JS eval fallback
-                        logger.debug(
-                            f"Could not evaluate ovens JS variable for {region['name']}: {e}"
+                        if response is not None and response.status >= 400:
+                            logger.warning(
+                                "LiveUAMap %s returned HTTP %s",
+                                region["name"],
+                                response.status,
+                            )
+                        page.wait_for_timeout(5_000)
+                        html = page.content()
+                        if _looks_like_challenge(html):
+                            logger.warning(
+                                "LiveUAMap %s appears to be serving an access challenge; "
+                                "leaving this region empty",
+                                region["name"],
+                            )
+                            failed_regions += 1
+                            continue
+
+                        payload = _read_page_payload(page, html)
+                        if payload is None:
+                            logger.warning(
+                                "LiveUAMap %s did not expose an ovens payload",
+                                region["name"],
+                            )
+                            failed_regions += 1
+                            continue
+
+                        candidates = normalize_liveuamap_payload(payload)
+                        region_markers = _format_markers(
+                            candidates,
+                            region=region["name"],
+                            base_url=region["url"],
+                            provider="browser",
+                            seen_ids=seen_ids,
                         )
+                        if not region_markers:
+                            logger.warning(
+                                "LiveUAMap %s payload contained no recognizable point markers (%s)",
+                                region["name"],
+                                payload_shape(payload),
+                            )
+                            failed_regions += 1
+                            continue
 
-                if m:
-                    json_str = m.group(1).strip()
-                    if json_str.startswith("'") or json_str.startswith('"'):
-                        json_str = json_str.strip("\"'")
-                        json_str = base64.b64decode(urllib.parse.unquote(json_str)).decode("utf-8")
+                        all_markers.extend(region_markers)
+                        successful_regions += 1
+                    except Exception as exc:  # Keep one region from killing the other three.
+                        failed_regions += 1
+                        logger.warning("LiveUAMap %s fetch failed: %s", region["name"], exc)
+            finally:
+                browser.close()
+    except Exception as exc:
+        _record_browser_failure(f"Chromium/provider launch failed: {exc}")
+        return []
 
-                    try:
-                        markers = json.loads(json_str)
-                        for marker in markers:
-                            mid = marker.get("id")
-                            if mid and mid not in seen_ids:
-                                seen_ids.add(mid)
-                                title = (marker.get("s") or marker.get("title") or "Unknown Event").strip()
-                                # Extract all available fields from the marker
-                                description = (marker.get("d") or marker.get("desc") or marker.get("description") or "").strip()
-                                category = (marker.get("c") or marker.get("cat") or marker.get("category") or "").strip()
-                                img = marker.get("img") or marker.get("image") or marker.get("photo") or ""
-                                source = (marker.get("source") or marker.get("src") or "").strip()
-                                event_time = marker.get("time") or marker.get("t") or ""
-                                link = marker.get("link") or marker.get("url") or ""
-                                # Format date from unix timestamp if available
-                                date_str = ""
-                                if event_time:
-                                    try:
-                                        from datetime import datetime, timezone
-                                        ts = int(event_time) if not isinstance(event_time, int) else event_time
-                                        dt = datetime.fromtimestamp(ts, tz=timezone.utc)
-                                        date_str = dt.strftime("%Y-%m-%d %H:%M UTC")
-                                    except (ValueError, TypeError, OSError):
-                                        date_str = str(event_time)
-                                # Build full link URL
-                                if link and not link.startswith("http"):
-                                    base = region["url"].rstrip("/")
-                                    link = f"{base}/{link.lstrip('/')}"
-                                all_markers.append(
-                                    {
-                                        "id": mid,
-                                        "type": "liveuamap",
-                                        "title": title,
-                                        "description": description[:500] if description else "",
-                                        "lat": marker.get("lat"),
-                                        "lng": marker.get("lng"),
-                                        "timestamp": event_time,
-                                        "date": date_str,
-                                        "link": link or region["url"],
-                                        "region": region["name"],
-                                        "category": category,
-                                        "image": img,
-                                        "source": source,
-                                    }
-                                )
-                    except (json.JSONDecodeError, ValueError, KeyError) as e:
-                        logger.error(f"Error parsing JSON for {region['name']}: {e}")
+    if successful_regions:
+        _record_browser_success()
+        logger.info(
+            "LiveUAMap browser provider normalized %s markers from %s/%s regions",
+            len(all_markers),
+            successful_regions,
+            len(_REGIONS),
+        )
+        return all_markers
 
-            except Exception as e:
-                logger.error(f"Error scraping Liveuamap {region['name']}: {e}")
+    _record_browser_failure(f"all {failed_regions or len(_REGIONS)} regions failed or drifted")
+    return []
 
-        browser.close()
 
-    logger.info(f"Liveuamap scraper finished, extracted {len(all_markers)} unique markers.")
-    return all_markers
+def _format_markers(
+    candidates: list[dict[str, Any]],
+    *,
+    region: str,
+    base_url: str,
+    provider: str,
+    seen_ids: set[str] | None = None,
+) -> list[dict[str, Any]]:
+    output: list[dict[str, Any]] = []
+    dedupe = seen_ids if seen_ids is not None else set()
+
+    for marker, lat, lng in iter_valid_coordinates(candidates):
+        title = _as_text(
+            marker.get("s")
+            or marker.get("title")
+            or marker.get("name")
+            or marker.get("event")
+            or "Unknown Event"
+        ).strip()
+        description = _as_text(
+            marker.get("d")
+            or marker.get("desc")
+            or marker.get("description")
+            or marker.get("summary")
+            or ""
+        ).strip()
+        category = _as_text(
+            marker.get("c") or marker.get("cat") or marker.get("category") or ""
+        ).strip()
+        image = _as_text(marker.get("img") or marker.get("image") or marker.get("photo") or "").strip()
+        source = _as_text(marker.get("source") or marker.get("src") or "").strip()
+        event_time = marker.get("time", marker.get("t", marker.get("timestamp", "")))
+        link = _as_text(marker.get("link") or marker.get("url") or "").strip()
+        if link and not urlparse(link).scheme:
+            link = urljoin(base_url.rstrip("/") + "/", link.lstrip("/"))
+
+        raw_id = marker.get("id", marker.get("event_id"))
+        marker_id = _as_text(raw_id).strip() if raw_id is not None else ""
+        if not marker_id:
+            marker_id = _stable_marker_id(lat, lng, title, event_time, link)
+        if marker_id in dedupe:
+            continue
+        dedupe.add(marker_id)
+
+        date_str = _format_event_time(event_time)
+        output.append(
+            {
+                "id": marker_id,
+                "type": "liveuamap",
+                "title": title or "Unknown Event",
+                "description": description[:500],
+                "lat": lat,
+                "lng": lng,
+                "timestamp": event_time if event_time is not None else "",
+                "date": date_str,
+                "link": link or base_url,
+                "region": _as_text(marker.get("region") or region).strip() or region,
+                "category": category,
+                "image": image,
+                "source": source,
+                "provider": provider,
+            }
+        )
+    return output
+
+
+def _stable_marker_id(lat: float, lng: float, title: str, event_time: Any, link: str) -> str:
+    fingerprint = f"{lat:.6f}|{lng:.6f}|{title}|{event_time}|{link}".encode(
+        "utf-8", errors="replace"
+    )
+    return f"liveuamap-{hashlib.sha256(fingerprint).hexdigest()[:20]}"
+
+
+def _format_event_time(value: Any) -> str:
+    if value in (None, ""):
+        return ""
+    try:
+        numeric = float(value)
+        if not math.isfinite(numeric):
+            raise ValueError("non-finite timestamp")
+        if abs(numeric) > 100_000_000_000:  # milliseconds since epoch
+            numeric /= 1000.0
+        dt = datetime.fromtimestamp(numeric, tz=timezone.utc)
+        return dt.strftime("%Y-%m-%d %H:%M UTC")
+    except (TypeError, ValueError, OSError, OverflowError):
+        return _as_text(value)
+
+
+def _as_text(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    if isinstance(value, (int, float, bool)):
+        return str(value)
+    return ""
+
+
+def fetch_liveuamap() -> list[dict[str, Any]]:
+    """Return LiveUAMap enrichment without making Global Incidents depend on it."""
+    from services.liveuamap_settings import (
+        liveuamap_api_configured,
+        liveuamap_browser_scraper_enabled,
+    )
+
+    if liveuamap_api_configured():
+        try:
+            return _fetch_liveuamap_api()
+        except (requests.RequestException, ValueError, OSError) as exc:
+            logger.warning(
+                "LiveUAMap supported API failed (%s); considering browser fallback",
+                type(exc).__name__,
+            )
+            # POSIX installs preserve their historical browser fallback; on
+            # Windows it remains available only after the operator opted in.
+
+    if liveuamap_browser_scraper_enabled():
+        return _fetch_liveuamap_browser()
+
+    logger.info("LiveUAMap enrichment disabled/unavailable; Global Incidents continues with GDELT")
+    return []
 
 
 if __name__ == "__main__":
+    import json
+
     logging.basicConfig(level=logging.INFO)
-    res = fetch_liveuamap()
-    print(json.dumps(res[:3], indent=2))
+    print(json.dumps(fetch_liveuamap()[:3], indent=2))

backend/services/liveuamap_settings.py

@@ -1,4 +1,10 @@
-"""LiveUAMap Playwright scraper opt-in (#348) — UI consent on Windows."""
+"""LiveUAMap provider settings and operator-consent state.
+
+Global Incidents is a broader Shadowbroker feature backed by GDELT regardless
+of whether LiveUAMap enrichment is available. The browser provider keeps the
+historical platform behavior (automatic on POSIX, opt-in on Windows) while a
+configured supported API can satisfy LiveUAMap enrichment without Chromium.
+"""
 
 from __future__ import annotations
 
@@ -8,6 +14,7 @@ import os
 import threading
 from pathlib import Path
 from typing import Any
+from urllib.parse import urlparse
 
 logger = logging.getLogger(__name__)
 
@@ -19,33 +26,59 @@ def _env_flag(name: str) -> str:
     return str(os.getenv(name, "")).strip().lower()
 
 
+def _valid_https_url(raw: str) -> bool:
+    try:
+        parsed = urlparse(raw)
+    except ValueError:
+        return False
+    return parsed.scheme.lower() == "https" and bool(parsed.netloc)
+
+
 def liveuamap_requires_ui_opt_in() -> bool:
-    """Windows local installs need explicit consent before Playwright contacts LiveUAMap."""
+    """Windows local installs need an explicit choice before browser scraping."""
     return os.name == "nt"
 
 
+def liveuamap_ui_choice_recorded() -> bool:
+    """Whether the operator has already accepted or declined browser contact."""
+    return _OPT_IN_FILE.exists()
+
+
 def get_liveuamap_ui_opt_in() -> bool:
     if not _OPT_IN_FILE.exists():
         return False
     try:
         payload = json.loads(_OPT_IN_FILE.read_text(encoding="utf-8"))
         return bool(payload.get("opted_in"))
-    except (OSError, json.JSONDecodeError, TypeError) as e:
-        logger.warning("LiveUAMap opt-in file unreadable: %s", e)
+    except (OSError, json.JSONDecodeError, TypeError) as exc:
+        logger.warning("LiveUAMap opt-in file unreadable: %s", exc)
         return False
 
 
 def set_liveuamap_ui_opt_in(opted_in: bool) -> None:
+    """Persist an explicit browser-provider choice, including a decline."""
     _OPT_IN_FILE.parent.mkdir(parents=True, exist_ok=True)
+    payload = json.dumps({"opted_in": bool(opted_in)}, indent=2)
     with _OPT_IN_LOCK:
-        _OPT_IN_FILE.write_text(
-            json.dumps({"opted_in": bool(opted_in)}, indent=2),
-            encoding="utf-8",
-        )
+        temp_path = _OPT_IN_FILE.with_suffix(_OPT_IN_FILE.suffix + ".tmp")
+        temp_path.write_text(payload, encoding="utf-8")
+        os.replace(temp_path, _OPT_IN_FILE)
 
 
-def liveuamap_scraper_enabled() -> bool:
-    """Whether the Playwright LiveUAMap scraper may run on this backend."""
+def liveuamap_api_configured() -> bool:
+    """Whether an operator supplied a syntactically valid HTTPS API endpoint."""
+    url = str(os.getenv("LIVEUAMAP_API_URL", "") or "").strip()
+    return bool(url and _valid_https_url(url))
+
+
+def liveuamap_browser_scraper_enabled() -> bool:
+    """Whether the existing Playwright provider may contact LiveUAMap.
+
+    Preserve the established UX on Linux/macOS/Docker: browser enrichment is
+    available when Global Incidents is active unless explicitly disabled.
+    Windows keeps the existing opt-in boundary. An environment override always
+    wins for the browser provider only; it does not disable a configured API.
+    """
     setting = _env_flag("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER")
     if setting in {"1", "true", "yes", "on"}:
         return True
@@ -56,6 +89,15 @@ def liveuamap_scraper_enabled() -> bool:
     return get_liveuamap_ui_opt_in()
 
 
+def liveuamap_scraper_enabled() -> bool:
+    """Historical scheduler gate: whether *any* LiveUAMap provider can run.
+
+    The name is retained for call-site compatibility. Supported API access is
+    preferred when configured; otherwise the optional browser provider may run.
+    """
+    return liveuamap_api_configured() or liveuamap_browser_scraper_enabled()
+
+
 def liveuamap_scraper_status() -> dict[str, Any]:
     setting = _env_flag("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER")
     env_override = None
@@ -63,11 +105,29 @@ def liveuamap_scraper_status() -> dict[str, Any]:
         env_override = "on"
     elif setting in {"0", "false", "no", "off"}:
         env_override = "off"
+
     ui_opted_in = get_liveuamap_ui_opt_in()
     requires = liveuamap_requires_ui_opt_in()
+    api_configured = liveuamap_api_configured()
+    browser_enabled = liveuamap_browser_scraper_enabled()
+    enrichment_enabled = api_configured or browser_enabled
+
+    if api_configured:
+        provider_mode = "api"
+    elif browser_enabled:
+        provider_mode = "scraper"
+    else:
+        provider_mode = "gdelt-only"
+
     return {
+        # Existing fields remain stable for current frontends.
         "platform_requires_opt_in": requires,
         "ui_opted_in": ui_opted_in,
-        "scraper_enabled": liveuamap_scraper_enabled(),
+        "scraper_enabled": browser_enabled,
         "env_override": env_override,
+        # Additive provider/UX diagnostics.
+        "ui_choice_recorded": liveuamap_ui_choice_recorded(),
+        "api_configured": api_configured,
+        "enrichment_enabled": enrichment_enabled,
+        "provider_mode": provider_mode,
     }