fix(liveuamap): make enrichment resilient and non-blocking

This commit is contained in:
Shadowbroker
2026-08-18 15:52:46 -06:00
parent 2e7a0038b5
commit ed97e9e4bc
12 changed files with 1217 additions and 156 deletions
+16 -1
View File
@@ -40,6 +40,13 @@ ENV PATH="/root/.local/bin:$PATH"
# Install into system Python (no venv needed inside container)
ENV UV_PROJECT_ENVIRONMENT=/usr/local
# Playwright installs browsers under the current user's cache by default. The
# dependency install below runs as root while the backend runs as backenduser,
# which caused runtime Playwright to look in /app/.cache for a browser that had
# actually been baked under /root/.cache (#516). Use one image-wide location
# that is readable by the non-root runtime user.
ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
# Copy workspace root files for UV resolution (build context is repo root)
COPY pyproject.toml /workspace/pyproject.toml
COPY uv.lock /workspace/uv.lock
@@ -47,7 +54,9 @@ COPY backend/pyproject.toml /workspace/backend/pyproject.toml
# Install Python dependencies using the lockfile
RUN cd /workspace/backend && uv sync --frozen --no-dev --extra road-corridor \
&& playwright install --with-deps chromium
&& playwright install --with-deps chromium \
&& playwright install chromium-headless-shell \
&& chmod -R a+rX "$PLAYWRIGHT_BROWSERS_PATH"
# Copy backend source code
COPY backend/ .
@@ -81,6 +90,12 @@ RUN adduser --system --uid 1001 --home /app backenduser \
# Switch to the non-root user
USER backenduser
# Build-time packaging assertion for #516. Do not launch Chromium here because
# multi-arch image builds may run under emulation; instead verify that the exact
# runtime user resolves an executable Chromium and that the headless-shell
# bundle used by headless launches is present and executable.
RUN python -c "import os; from pathlib import Path; from playwright.sync_api import sync_playwright; p=sync_playwright().start(); path=p.chromium.executable_path; assert os.path.isfile(path), path; assert os.access(path, os.X_OK), path; shells=list(Path(os.environ['PLAYWRIGHT_BROWSERS_PATH']).glob('chromium_headless_shell-*/**/headless_shell')); assert any(s.is_file() and os.access(s, os.X_OK) for s in shells), shells; print('Playwright runtime browser:', path, 'headless-shell:', shells[0]); p.stop()"
# Expose port
EXPOSE 8000
+249
View File
@@ -0,0 +1,249 @@
"""Defensive parsing helpers for LiveUAMap provider payloads.
LiveUAMap's browser page exposes an undocumented ``ovens`` value whose shape
has changed over time. The optional supported API may also return JSON or
GeoJSON. Keep representation decoding and schema normalization isolated here so
upstream drift degrades one provider instead of crashing the fetch scheduler.
"""
from __future__ import annotations
import ast
import base64
import binascii
import json
import math
import re
from collections.abc import Iterable
from typing import Any
from urllib.parse import unquote
_MAX_DECODE_DEPTH = 5
_MAX_CANDIDATES = 10_000
_MAX_STRING_BYTES = 8 * 1024 * 1024
_WRAPPER_KEYS = ("ovens", "markers", "items", "events", "data", "results", "features")
def extract_ovens_expression(html: str) -> str | None:
"""Extract a legacy ``var/let/const ovens = ...;`` expression from HTML.
Evaluating ``window.ovens`` in the browser is preferred; this exists only
as a fallback for pages that still embed the value in source text.
"""
if not html:
return None
match = re.search(
r"(?:var|let|const)\s+ovens\s*=\s*(.+?);(?=\s*(?:</script>|(?:var|let|const|function)\b|$))",
html,
re.DOTALL | re.IGNORECASE,
)
if match:
return match.group(1).strip()
# Compatibility with older pages where the next token after the semicolon
# is arbitrary markup rather than another JavaScript declaration.
match = re.search(r"(?:var|let|const)\s+ovens\s*=\s*(.*?);", html, re.DOTALL | re.IGNORECASE)
return match.group(1).strip() if match else None
def payload_shape(value: Any) -> str:
"""Return a non-sensitive structural description for drift diagnostics."""
if isinstance(value, dict):
keys = sorted(str(key) for key in value.keys())[:8]
return f"dict(keys={keys}, size={len(value)})"
if isinstance(value, list):
item_types = sorted({type(item).__name__ for item in value[:20]})
return f"list(size={len(value)}, item_types={item_types})"
if isinstance(value, str):
return f"str(len={len(value)})"
return type(value).__name__
def normalize_liveuamap_payload(value: Any) -> list[dict[str, Any]]:
"""Normalize JSON/GeoJSON/legacy payload shapes into marker dictionaries.
Unknown or malformed values are ignored. This function intentionally never
assumes iterable items are mappings; issue #517 was caused by calling
``.get`` on strings after an upstream representation change.
"""
out: list[dict[str, Any]] = []
_collect(value, out, depth=0, inherited_id=None)
return out[:_MAX_CANDIDATES]
def _collect(
value: Any,
out: list[dict[str, Any]],
*,
depth: int,
inherited_id: str | None,
) -> None:
if depth > _MAX_DECODE_DEPTH or len(out) >= _MAX_CANDIDATES:
return
if value is None:
return
if isinstance(value, str):
decoded = _decode_string(value)
if decoded is value or decoded == value:
return
_collect(decoded, out, depth=depth + 1, inherited_id=inherited_id)
return
if isinstance(value, list):
for item in value[:_MAX_CANDIDATES - len(out)]:
_collect(item, out, depth=depth + 1, inherited_id=None)
if len(out) >= _MAX_CANDIDATES:
break
return
if not isinstance(value, dict):
return
# GeoJSON FeatureCollection / Feature.
if value.get("type") == "FeatureCollection" and isinstance(value.get("features"), list):
_collect(value["features"], out, depth=depth + 1, inherited_id=None)
return
if value.get("type") == "Feature":
marker = _marker_from_geojson_feature(value)
if marker is not None:
if inherited_id and not marker.get("id"):
marker["id"] = inherited_id
out.append(marker)
return
# Common wrapper shapes returned by APIs or page-side serialization.
for key in _WRAPPER_KEYS:
if key in value and isinstance(value[key], (dict, list, str)):
_collect(value[key], out, depth=depth + 1, inherited_id=None)
return
# A direct marker is accepted even if coordinates are absent here; the
# provider formatter performs the final coordinate/range validation.
if _looks_like_marker(value):
marker = dict(value)
if inherited_id and not marker.get("id"):
marker["id"] = inherited_id
out.append(marker)
return
# Some versions expose a dictionary keyed by marker ID. Traverse mapping
# values while preserving the key as a fallback identifier.
traversable = [
(str(key), item)
for key, item in value.items()
if isinstance(item, (dict, list, str))
]
if traversable:
for key, item in traversable[:_MAX_CANDIDATES - len(out)]:
_collect(item, out, depth=depth + 1, inherited_id=key)
if len(out) >= _MAX_CANDIDATES:
break
def _decode_string(raw: str) -> Any:
text = raw.strip()
if not text or len(text.encode("utf-8", errors="ignore")) > _MAX_STRING_BYTES:
return raw
# A JavaScript string literal can include escaping that plain strip("'")
# corrupts. literal_eval safely handles quoted string syntax only.
if len(text) >= 2 and text[0] == text[-1] and text[0] in {"'", '"'}:
try:
literal = ast.literal_eval(text)
except (SyntaxError, ValueError):
literal = None
if isinstance(literal, str) and literal != text:
return literal
decoded = _try_json(text)
if decoded is not None:
return decoded
# Legacy LiveUAMap payloads have appeared URL-encoded before decoding.
url_decoded = unquote(text)
if url_decoded != text:
decoded = _try_json(url_decoded)
if decoded is not None:
return decoded
text = url_decoded
# Older scraper versions expected a base64-wrapped JSON blob. Decode only
# when the result itself is valid JSON, so arbitrary titles/IDs are never
# interpreted as base64 data.
compact = "".join(text.split())
if compact and len(compact) % 4 == 0:
try:
raw_bytes = base64.b64decode(compact, validate=True)
decoded_text = raw_bytes.decode("utf-8")
except (binascii.Error, UnicodeDecodeError, ValueError):
decoded_text = ""
if decoded_text:
decoded = _try_json(decoded_text)
if decoded is not None:
return decoded
return raw
def _try_json(text: str) -> Any | None:
try:
return json.loads(text)
except (json.JSONDecodeError, TypeError, ValueError):
return None
def _looks_like_marker(value: dict[str, Any]) -> bool:
keys = set(value)
if {"lat", "lng"}.issubset(keys) or {"lat", "lon"}.issubset(keys):
return True
if "latitude" in keys and ("longitude" in keys or "lon" in keys or "lng" in keys):
return True
marker_metadata = {"id", "s", "title", "d", "desc", "description", "link", "url", "time", "t"}
return bool(keys.intersection(marker_metadata)) and not any(key in value for key in _WRAPPER_KEYS)
def _marker_from_geojson_feature(feature: dict[str, Any]) -> dict[str, Any] | None:
geometry = feature.get("geometry")
if not isinstance(geometry, dict) or geometry.get("type") != "Point":
return None
coordinates = geometry.get("coordinates")
if not isinstance(coordinates, (list, tuple)) or len(coordinates) < 2:
return None
lng = _finite_coordinate(coordinates[0], minimum=-180.0, maximum=180.0)
lat = _finite_coordinate(coordinates[1], minimum=-90.0, maximum=90.0)
if lat is None or lng is None:
return None
properties = feature.get("properties")
marker = dict(properties) if isinstance(properties, dict) else {}
marker.setdefault("lat", lat)
marker.setdefault("lng", lng)
feature_id = feature.get("id")
if feature_id is not None:
marker.setdefault("id", feature_id)
return marker
def _finite_coordinate(value: Any, *, minimum: float, maximum: float) -> float | None:
try:
number = float(value)
except (TypeError, ValueError):
return None
if not math.isfinite(number) or not minimum <= number <= maximum:
return None
return number
def iter_valid_coordinates(markers: Iterable[dict[str, Any]]) -> Iterable[tuple[dict[str, Any], float, float]]:
"""Yield markers with finite in-range latitude/longitude values."""
for marker in markers:
if not isinstance(marker, dict):
continue
lat = marker.get("lat", marker.get("latitude"))
lng = marker.get("lng", marker.get("lon", marker.get("longitude")))
lat_value = _finite_coordinate(lat, minimum=-90.0, maximum=90.0)
lng_value = _finite_coordinate(lng, minimum=-180.0, maximum=180.0)
if lat_value is None or lng_value is None:
continue
yield marker, lat_value, lng_value
+414 -120
View File
@@ -1,145 +1,439 @@
import json
"""Resilient LiveUAMap enrichment providers.
Global Incidents itself is backed independently by GDELT. This module adds
LiveUAMap pins when either an operator-configured supported API is available or
the existing Playwright provider is allowed. Provider failures are isolated and
return an empty enrichment set instead of breaking the scheduler.
The browser provider intentionally does not add any new anti-bot behavior. It
retains the repository's pre-existing Playwright/stealth profile for backward
compatibility while making parsing, packaging failures, and upstream drift
fail-soft.
"""
from __future__ import annotations
import hashlib
import logging
import base64
import urllib.parse
import re
from playwright.sync_api import sync_playwright
from playwright_stealth import stealth_sync
import math
import os
import threading
import time
from datetime import datetime, timezone
from typing import Any
from urllib.parse import urljoin, urlparse
import requests
from services.liveuamap_parser import (
extract_ovens_expression,
iter_valid_coordinates,
normalize_liveuamap_payload,
payload_shape,
)
logger = logging.getLogger(__name__)
_REGIONS = (
{"name": "Ukraine", "url": "https://liveuamap.com"},
{"name": "Middle East", "url": "https://mideast.liveuamap.com"},
{"name": "Israel-Palestine", "url": "https://israelpalestine.liveuamap.com"},
{"name": "Syria", "url": "https://syria.liveuamap.com"},
)
def fetch_liveuamap():
logger.info("Starting Liveuamap scraper with Playwright Stealth...")
_BROWSER_FAILURE_THRESHOLD = 3
_BROWSER_BACKOFF_BASE_S = 15 * 60
_BROWSER_BACKOFF_MAX_S = 6 * 60 * 60
_browser_failures = 0
_browser_blocked_until = 0.0
_browser_health_lock = threading.Lock()
regions = [
{"name": "Ukraine", "url": "https://liveuamap.com"},
{"name": "Middle East", "url": "https://mideast.liveuamap.com"},
{"name": "Israel-Palestine", "url": "https://israelpalestine.liveuamap.com"},
{"name": "Syria", "url": "https://syria.liveuamap.com"},
]
_CHALLENGE_MARKERS = (
"cf-turnstile",
"challenge-platform",
"just a moment",
"checking your browser",
"verify you are human",
)
all_markers = []
seen_ids = set()
with sync_playwright() as p:
# Launching with a real user agent to bypass Turnstile
browser = p.chromium.launch(
headless=True, args=["--disable-blink-features=AutomationControlled"]
def _bounded_int_env(name: str, default: int, *, minimum: int, maximum: int) -> int:
try:
value = int(str(os.getenv(name, default)).strip())
except (TypeError, ValueError):
value = default
return max(minimum, min(maximum, value))
def _safe_header_name(raw: str, default: str) -> str:
value = (raw or "").strip()
if not value or any(ch in value for ch in "\r\n:"):
return default
if not all(ch.isalnum() or ch in "-_" for ch in value):
return default
return value
def _api_url() -> str:
raw = str(os.getenv("LIVEUAMAP_API_URL", "") or "").strip()
if not raw:
return ""
try:
parsed = urlparse(raw)
except ValueError:
return ""
if parsed.scheme.lower() != "https" or not parsed.netloc:
return ""
return raw
def _api_headers() -> dict[str, str]:
from services.network_utils import outbound_user_agent
headers = {
"Accept": "application/geo+json, application/json;q=0.9",
"User-Agent": outbound_user_agent("liveuamap-api"),
}
api_key = str(os.getenv("LIVEUAMAP_API_KEY", "") or "").strip()
if api_key:
header = _safe_header_name(
str(os.getenv("LIVEUAMAP_API_AUTH_HEADER", "Authorization") or ""),
"Authorization",
)
from services.network_utils import outbound_user_agent
scheme = str(os.getenv("LIVEUAMAP_API_AUTH_SCHEME", "Bearer") or "").strip()
if any(ch in scheme for ch in "\r\n"):
scheme = "Bearer"
headers[header] = f"{scheme} {api_key}".strip() if scheme else api_key
return headers
# Per-install handle (no shared Shadowbroker product token). Stealth remains
# for Turnstile; see docs/OUTBOUND_DATA.md #348.
playwright_ua = (
f"Mozilla/5.0 (compatible; {outbound_user_agent('liveuamap')})"
)
context = browser.new_context(
user_agent=playwright_ua,
viewport={"width": 1920, "height": 1080},
color_scheme="dark",
)
# Bound navigation and script evaluation so a stuck region cannot hang the slow pool.
context.set_default_navigation_timeout(60_000)
context.set_default_timeout(30_000)
page = context.new_page()
stealth_sync(page)
for region in regions:
def _fetch_liveuamap_api() -> list[dict[str, Any]]:
"""Fetch an operator-configured supported LiveUAMap JSON/GeoJSON endpoint."""
url = _api_url()
if not url:
return []
timeout_s = _bounded_int_env("LIVEUAMAP_API_TIMEOUT_S", 30, minimum=5, maximum=120)
hostname = urlparse(url).hostname or "configured endpoint"
logger.info("Fetching LiveUAMap supported API from %s", hostname)
response = requests.get(url, headers=_api_headers(), timeout=(5, timeout_s))
response.raise_for_status()
try:
payload = response.json()
except (requests.JSONDecodeError, ValueError) as exc:
raise ValueError("LiveUAMap API did not return JSON/GeoJSON") from exc
candidates = normalize_liveuamap_payload(payload)
markers = _format_markers(
candidates,
region="LiveUAMap",
base_url=url,
provider="api",
)
if not markers:
raise ValueError(f"LiveUAMap API returned no recognizable point markers ({payload_shape(payload)})")
logger.info("LiveUAMap API returned %s normalized markers", len(markers))
return markers
def _browser_circuit_open() -> tuple[bool, int]:
now = time.monotonic()
with _browser_health_lock:
remaining = max(0, int(_browser_blocked_until - now))
return remaining > 0, remaining
def _record_browser_success() -> None:
global _browser_failures, _browser_blocked_until
with _browser_health_lock:
_browser_failures = 0
_browser_blocked_until = 0.0
def _record_browser_failure(reason: str) -> None:
global _browser_failures, _browser_blocked_until
now = time.monotonic()
with _browser_health_lock:
_browser_failures += 1
failures = _browser_failures
if failures < _BROWSER_FAILURE_THRESHOLD:
logger.warning(
"LiveUAMap browser provider failure %s/%s: %s",
failures,
_BROWSER_FAILURE_THRESHOLD,
reason,
)
return
exponent = failures - _BROWSER_FAILURE_THRESHOLD
delay_s = min(_BROWSER_BACKOFF_BASE_S * (2**exponent), _BROWSER_BACKOFF_MAX_S)
_browser_blocked_until = max(_browser_blocked_until, now + delay_s)
logger.warning(
"LiveUAMap browser provider paused for %ss after repeated failures: %s",
delay_s,
reason,
)
def _looks_like_challenge(html: str) -> bool:
lowered = (html or "").lower()
return any(marker in lowered for marker in _CHALLENGE_MARKERS)
def _read_page_payload(page: Any, html: str) -> Any:
"""Prefer evaluated page state, then fall back to the legacy source variable."""
try:
serialized = page.evaluate(
"() => typeof ovens !== 'undefined' ? JSON.stringify(ovens) : null"
)
if serialized:
return serialized
except Exception as exc: # Playwright exception types differ across releases.
logger.debug("LiveUAMap ovens JS evaluation unavailable: %s", exc)
expression = extract_ovens_expression(html)
return expression if expression is not None else None
def _fetch_liveuamap_browser() -> list[dict[str, Any]]:
open_now, remaining_s = _browser_circuit_open()
if open_now:
logger.info(
"LiveUAMap browser provider circuit open; skipping Chromium for another %ss",
remaining_s,
)
return []
# Import browser-only dependencies lazily so API-only deployments do not
# require Chromium just to import this module.
from playwright.sync_api import sync_playwright
from playwright_stealth import stealth_sync
from services.network_utils import outbound_user_agent
all_markers: list[dict[str, Any]] = []
seen_ids: set[str] = set()
successful_regions = 0
failed_regions = 0
try:
with sync_playwright() as playwright:
# Existing repository behavior retained for compatibility. This PR
# deliberately adds no further anti-detection/evasion measures.
browser = playwright.chromium.launch(
headless=True,
args=["--disable-blink-features=AutomationControlled"],
)
try:
logger.info(f"Scraping Liveuamap region: {region['name']}")
page.goto(region["url"], timeout=60000, wait_until="domcontentloaded")
context = browser.new_context(
user_agent=f"Mozilla/5.0 (compatible; {outbound_user_agent('liveuamap')})",
viewport={"width": 1920, "height": 1080},
color_scheme="dark",
)
context.set_default_navigation_timeout(60_000)
context.set_default_timeout(30_000)
page = context.new_page()
stealth_sync(page)
# Wait for the map canvas or markers script to load, max 10s wait
try:
page.wait_for_timeout(5000)
except (TimeoutError, OSError): # non-critical: page load delay
pass
html = page.content()
m = re.search(r"var\s+ovens\s*=\s*(.*?);(?!function)", html, re.DOTALL)
if not m:
logger.warning(f"Could not find 'ovens' data for {region['name']} in raw HTML")
# Let's try grabbing the evaluated JavaScript variable if it's there
for region in _REGIONS:
try:
ovens_json = page.evaluate(
"() => typeof ovens !== 'undefined' ? JSON.stringify(ovens) : null"
logger.info("Fetching LiveUAMap browser region: %s", region["name"])
response = page.goto(
region["url"],
timeout=60_000,
wait_until="domcontentloaded",
)
if ovens_json:
markers = json.loads(ovens_json)
# process below
html = f"var ovens={ovens_json};"
m = re.search(r"var\s+ovens=(.*?);", html, re.DOTALL)
except (ValueError, KeyError, OSError) as e: # non-critical: JS eval fallback
logger.debug(
f"Could not evaluate ovens JS variable for {region['name']}: {e}"
if response is not None and response.status >= 400:
logger.warning(
"LiveUAMap %s returned HTTP %s",
region["name"],
response.status,
)
page.wait_for_timeout(5_000)
html = page.content()
if _looks_like_challenge(html):
logger.warning(
"LiveUAMap %s appears to be serving an access challenge; "
"leaving this region empty",
region["name"],
)
failed_regions += 1
continue
payload = _read_page_payload(page, html)
if payload is None:
logger.warning(
"LiveUAMap %s did not expose an ovens payload",
region["name"],
)
failed_regions += 1
continue
candidates = normalize_liveuamap_payload(payload)
region_markers = _format_markers(
candidates,
region=region["name"],
base_url=region["url"],
provider="browser",
seen_ids=seen_ids,
)
if not region_markers:
logger.warning(
"LiveUAMap %s payload contained no recognizable point markers (%s)",
region["name"],
payload_shape(payload),
)
failed_regions += 1
continue
if m:
json_str = m.group(1).strip()
if json_str.startswith("'") or json_str.startswith('"'):
json_str = json_str.strip("\"'")
json_str = base64.b64decode(urllib.parse.unquote(json_str)).decode("utf-8")
all_markers.extend(region_markers)
successful_regions += 1
except Exception as exc: # Keep one region from killing the other three.
failed_regions += 1
logger.warning("LiveUAMap %s fetch failed: %s", region["name"], exc)
finally:
browser.close()
except Exception as exc:
_record_browser_failure(f"Chromium/provider launch failed: {exc}")
return []
try:
markers = json.loads(json_str)
for marker in markers:
mid = marker.get("id")
if mid and mid not in seen_ids:
seen_ids.add(mid)
title = (marker.get("s") or marker.get("title") or "Unknown Event").strip()
# Extract all available fields from the marker
description = (marker.get("d") or marker.get("desc") or marker.get("description") or "").strip()
category = (marker.get("c") or marker.get("cat") or marker.get("category") or "").strip()
img = marker.get("img") or marker.get("image") or marker.get("photo") or ""
source = (marker.get("source") or marker.get("src") or "").strip()
event_time = marker.get("time") or marker.get("t") or ""
link = marker.get("link") or marker.get("url") or ""
# Format date from unix timestamp if available
date_str = ""
if event_time:
try:
from datetime import datetime, timezone
ts = int(event_time) if not isinstance(event_time, int) else event_time
dt = datetime.fromtimestamp(ts, tz=timezone.utc)
date_str = dt.strftime("%Y-%m-%d %H:%M UTC")
except (ValueError, TypeError, OSError):
date_str = str(event_time)
# Build full link URL
if link and not link.startswith("http"):
base = region["url"].rstrip("/")
link = f"{base}/{link.lstrip('/')}"
all_markers.append(
{
"id": mid,
"type": "liveuamap",
"title": title,
"description": description[:500] if description else "",
"lat": marker.get("lat"),
"lng": marker.get("lng"),
"timestamp": event_time,
"date": date_str,
"link": link or region["url"],
"region": region["name"],
"category": category,
"image": img,
"source": source,
}
)
except (json.JSONDecodeError, ValueError, KeyError) as e:
logger.error(f"Error parsing JSON for {region['name']}: {e}")
if successful_regions:
_record_browser_success()
logger.info(
"LiveUAMap browser provider normalized %s markers from %s/%s regions",
len(all_markers),
successful_regions,
len(_REGIONS),
)
return all_markers
except Exception as e:
logger.error(f"Error scraping Liveuamap {region['name']}: {e}")
_record_browser_failure(f"all {failed_regions or len(_REGIONS)} regions failed or drifted")
return []
browser.close()
logger.info(f"Liveuamap scraper finished, extracted {len(all_markers)} unique markers.")
return all_markers
def _format_markers(
candidates: list[dict[str, Any]],
*,
region: str,
base_url: str,
provider: str,
seen_ids: set[str] | None = None,
) -> list[dict[str, Any]]:
output: list[dict[str, Any]] = []
dedupe = seen_ids if seen_ids is not None else set()
for marker, lat, lng in iter_valid_coordinates(candidates):
title = _as_text(
marker.get("s")
or marker.get("title")
or marker.get("name")
or marker.get("event")
or "Unknown Event"
).strip()
description = _as_text(
marker.get("d")
or marker.get("desc")
or marker.get("description")
or marker.get("summary")
or ""
).strip()
category = _as_text(
marker.get("c") or marker.get("cat") or marker.get("category") or ""
).strip()
image = _as_text(marker.get("img") or marker.get("image") or marker.get("photo") or "").strip()
source = _as_text(marker.get("source") or marker.get("src") or "").strip()
event_time = marker.get("time", marker.get("t", marker.get("timestamp", "")))
link = _as_text(marker.get("link") or marker.get("url") or "").strip()
if link and not urlparse(link).scheme:
link = urljoin(base_url.rstrip("/") + "/", link.lstrip("/"))
raw_id = marker.get("id", marker.get("event_id"))
marker_id = _as_text(raw_id).strip() if raw_id is not None else ""
if not marker_id:
marker_id = _stable_marker_id(lat, lng, title, event_time, link)
if marker_id in dedupe:
continue
dedupe.add(marker_id)
date_str = _format_event_time(event_time)
output.append(
{
"id": marker_id,
"type": "liveuamap",
"title": title or "Unknown Event",
"description": description[:500],
"lat": lat,
"lng": lng,
"timestamp": event_time if event_time is not None else "",
"date": date_str,
"link": link or base_url,
"region": _as_text(marker.get("region") or region).strip() or region,
"category": category,
"image": image,
"source": source,
"provider": provider,
}
)
return output
def _stable_marker_id(lat: float, lng: float, title: str, event_time: Any, link: str) -> str:
fingerprint = f"{lat:.6f}|{lng:.6f}|{title}|{event_time}|{link}".encode(
"utf-8", errors="replace"
)
return f"liveuamap-{hashlib.sha256(fingerprint).hexdigest()[:20]}"
def _format_event_time(value: Any) -> str:
if value in (None, ""):
return ""
try:
numeric = float(value)
if not math.isfinite(numeric):
raise ValueError("non-finite timestamp")
if abs(numeric) > 100_000_000_000: # milliseconds since epoch
numeric /= 1000.0
dt = datetime.fromtimestamp(numeric, tz=timezone.utc)
return dt.strftime("%Y-%m-%d %H:%M UTC")
except (TypeError, ValueError, OSError, OverflowError):
return _as_text(value)
def _as_text(value: Any) -> str:
if value is None:
return ""
if isinstance(value, str):
return value
if isinstance(value, (int, float, bool)):
return str(value)
return ""
def fetch_liveuamap() -> list[dict[str, Any]]:
"""Return LiveUAMap enrichment without making Global Incidents depend on it."""
from services.liveuamap_settings import (
liveuamap_api_configured,
liveuamap_browser_scraper_enabled,
)
if liveuamap_api_configured():
try:
return _fetch_liveuamap_api()
except (requests.RequestException, ValueError, OSError) as exc:
logger.warning(
"LiveUAMap supported API failed (%s); considering browser fallback",
type(exc).__name__,
)
# POSIX installs preserve their historical browser fallback; on
# Windows it remains available only after the operator opted in.
if liveuamap_browser_scraper_enabled():
return _fetch_liveuamap_browser()
logger.info("LiveUAMap enrichment disabled/unavailable; Global Incidents continues with GDELT")
return []
if __name__ == "__main__":
import json
logging.basicConfig(level=logging.INFO)
res = fetch_liveuamap()
print(json.dumps(res[:3], indent=2))
print(json.dumps(fetch_liveuamap()[:3], indent=2))
+71 -11
View File
@@ -1,4 +1,10 @@
"""LiveUAMap Playwright scraper opt-in (#348) — UI consent on Windows."""
"""LiveUAMap provider settings and operator-consent state.
Global Incidents is a broader Shadowbroker feature backed by GDELT regardless
of whether LiveUAMap enrichment is available. The browser provider keeps the
historical platform behavior (automatic on POSIX, opt-in on Windows) while a
configured supported API can satisfy LiveUAMap enrichment without Chromium.
"""
from __future__ import annotations
@@ -8,6 +14,7 @@ import os
import threading
from pathlib import Path
from typing import Any
from urllib.parse import urlparse
logger = logging.getLogger(__name__)
@@ -19,33 +26,59 @@ def _env_flag(name: str) -> str:
return str(os.getenv(name, "")).strip().lower()
def _valid_https_url(raw: str) -> bool:
try:
parsed = urlparse(raw)
except ValueError:
return False
return parsed.scheme.lower() == "https" and bool(parsed.netloc)
def liveuamap_requires_ui_opt_in() -> bool:
"""Windows local installs need explicit consent before Playwright contacts LiveUAMap."""
"""Windows local installs need an explicit choice before browser scraping."""
return os.name == "nt"
def liveuamap_ui_choice_recorded() -> bool:
"""Whether the operator has already accepted or declined browser contact."""
return _OPT_IN_FILE.exists()
def get_liveuamap_ui_opt_in() -> bool:
if not _OPT_IN_FILE.exists():
return False
try:
payload = json.loads(_OPT_IN_FILE.read_text(encoding="utf-8"))
return bool(payload.get("opted_in"))
except (OSError, json.JSONDecodeError, TypeError) as e:
logger.warning("LiveUAMap opt-in file unreadable: %s", e)
except (OSError, json.JSONDecodeError, TypeError) as exc:
logger.warning("LiveUAMap opt-in file unreadable: %s", exc)
return False
def set_liveuamap_ui_opt_in(opted_in: bool) -> None:
"""Persist an explicit browser-provider choice, including a decline."""
_OPT_IN_FILE.parent.mkdir(parents=True, exist_ok=True)
payload = json.dumps({"opted_in": bool(opted_in)}, indent=2)
with _OPT_IN_LOCK:
_OPT_IN_FILE.write_text(
json.dumps({"opted_in": bool(opted_in)}, indent=2),
encoding="utf-8",
)
temp_path = _OPT_IN_FILE.with_suffix(_OPT_IN_FILE.suffix + ".tmp")
temp_path.write_text(payload, encoding="utf-8")
os.replace(temp_path, _OPT_IN_FILE)
def liveuamap_scraper_enabled() -> bool:
"""Whether the Playwright LiveUAMap scraper may run on this backend."""
def liveuamap_api_configured() -> bool:
"""Whether an operator supplied a syntactically valid HTTPS API endpoint."""
url = str(os.getenv("LIVEUAMAP_API_URL", "") or "").strip()
return bool(url and _valid_https_url(url))
def liveuamap_browser_scraper_enabled() -> bool:
"""Whether the existing Playwright provider may contact LiveUAMap.
Preserve the established UX on Linux/macOS/Docker: browser enrichment is
available when Global Incidents is active unless explicitly disabled.
Windows keeps the existing opt-in boundary. An environment override always
wins for the browser provider only; it does not disable a configured API.
"""
setting = _env_flag("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER")
if setting in {"1", "true", "yes", "on"}:
return True
@@ -56,6 +89,15 @@ def liveuamap_scraper_enabled() -> bool:
return get_liveuamap_ui_opt_in()
def liveuamap_scraper_enabled() -> bool:
"""Historical scheduler gate: whether *any* LiveUAMap provider can run.
The name is retained for call-site compatibility. Supported API access is
preferred when configured; otherwise the optional browser provider may run.
"""
return liveuamap_api_configured() or liveuamap_browser_scraper_enabled()
def liveuamap_scraper_status() -> dict[str, Any]:
setting = _env_flag("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER")
env_override = None
@@ -63,11 +105,29 @@ def liveuamap_scraper_status() -> dict[str, Any]:
env_override = "on"
elif setting in {"0", "false", "no", "off"}:
env_override = "off"
ui_opted_in = get_liveuamap_ui_opt_in()
requires = liveuamap_requires_ui_opt_in()
api_configured = liveuamap_api_configured()
browser_enabled = liveuamap_browser_scraper_enabled()
enrichment_enabled = api_configured or browser_enabled
if api_configured:
provider_mode = "api"
elif browser_enabled:
provider_mode = "scraper"
else:
provider_mode = "gdelt-only"
return {
# Existing fields remain stable for current frontends.
"platform_requires_opt_in": requires,
"ui_opted_in": ui_opted_in,
"scraper_enabled": liveuamap_scraper_enabled(),
"scraper_enabled": browser_enabled,
"env_override": env_override,
# Additive provider/UX diagnostics.
"ui_choice_recorded": liveuamap_ui_choice_recorded(),
"api_configured": api_configured,
"enrichment_enabled": enrichment_enabled,
"provider_mode": provider_mode,
}
+88
View File
@@ -0,0 +1,88 @@
from __future__ import annotations
import base64
import json
from urllib.parse import quote
from services.liveuamap_parser import (
extract_ovens_expression,
iter_valid_coordinates,
normalize_liveuamap_payload,
)
def _ids(value):
return [str(item.get("id")) for item in normalize_liveuamap_payload(value)]
def test_plain_marker_list():
payload = [{"id": 1, "lat": 1, "lng": 2, "title": "a"}]
assert _ids(payload) == ["1"]
def test_double_encoded_json():
payload = json.dumps(json.dumps([{"id": "double", "lat": 1, "lng": 2}]))
assert _ids(payload) == ["double"]
def test_list_of_json_strings_regression_517():
payload = [
json.dumps({"id": "a", "lat": 10, "lng": 20}),
json.dumps({"id": "b", "lat": 30, "lng": 40}),
]
assert _ids(payload) == ["a", "b"]
def test_mapping_key_becomes_fallback_marker_id():
payload = {"123": {"lat": 1, "lng": 2, "title": "keyed"}}
markers = normalize_liveuamap_payload(payload)
assert markers[0]["id"] == "123"
def test_common_wrapper_shape():
payload = {"data": {"markers": [{"id": "wrapped", "lat": 1, "lng": 2}]}}
assert _ids(payload) == ["wrapped"]
def test_legacy_urlencoded_base64_json():
raw = json.dumps([{"id": "legacy", "lat": 1, "lng": 2}]).encode()
payload = quote(base64.b64encode(raw).decode())
assert _ids(payload) == ["legacy"]
def test_geojson_feature_collection():
payload = {
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"id": "geo",
"geometry": {"type": "Point", "coordinates": [20, 10]},
"properties": {"title": "Geo event"},
}
],
}
markers = normalize_liveuamap_payload(payload)
assert markers == [{"title": "Geo event", "lat": 10.0, "lng": 20.0, "id": "geo"}]
def test_malformed_scalars_are_ignored_instead_of_crashing():
payload = ["not-json", 42, None, True, {"nested": object()}]
assert normalize_liveuamap_payload(payload) == []
def test_coordinate_iterator_rejects_out_of_range_and_nonfinite():
markers = [
{"id": "good", "lat": "10", "lng": "20"},
{"id": "bad-lat", "lat": 100, "lng": 20},
{"id": "bad-lng", "lat": 10, "lng": 200},
{"id": "nan", "lat": float("nan"), "lng": 20},
]
valid = list(iter_valid_coordinates(markers))
assert [(item[0]["id"], item[1], item[2]) for item in valid] == [("good", 10.0, 20.0)]
def test_extracts_var_let_and_const_ovens():
assert extract_ovens_expression('<script>var ovens = [{"id":1}];</script>') == '[{"id":1}]'
assert extract_ovens_expression('<script>let ovens = "abc";</script>') == '"abc"'
assert extract_ovens_expression('<script>const ovens = {"data":[]};</script>') == '{"data":[]}'
+94
View File
@@ -0,0 +1,94 @@
from __future__ import annotations
import requests
from services import liveuamap_scraper as scraper
from services import liveuamap_settings as settings
class _Response:
def __init__(self, payload, status_code=200):
self._payload = payload
self.status_code = status_code
def raise_for_status(self):
if self.status_code >= 400:
raise requests.HTTPError(f"HTTP {self.status_code}")
def json(self):
return self._payload
def test_api_geojson_is_normalized_and_auth_header_is_sent(monkeypatch):
monkeypatch.setenv("LIVEUAMAP_API_URL", "https://api.example.test/events")
monkeypatch.setenv("LIVEUAMAP_API_KEY", "secret-key")
monkeypatch.setenv("LIVEUAMAP_API_TIMEOUT_S", "12")
monkeypatch.setattr(
"services.network_utils.outbound_user_agent",
lambda purpose="": f"operator-test ({purpose})",
)
seen = {}
def fake_get(url, *, headers, timeout):
seen.update(url=url, headers=headers, timeout=timeout)
return _Response(
{
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"id": "evt-1",
"geometry": {"type": "Point", "coordinates": [30.5, 50.5]},
"properties": {"title": "Event", "url": "https://example.test/e/1"},
}
],
}
)
monkeypatch.setattr(scraper.requests, "get", fake_get)
markers = scraper._fetch_liveuamap_api()
assert markers[0]["id"] == "evt-1"
assert markers[0]["lat"] == 50.5
assert markers[0]["lng"] == 30.5
assert markers[0]["provider"] == "api"
assert seen["headers"]["Authorization"] == "Bearer secret-key"
assert seen["timeout"] == (5, 12)
def test_api_failure_falls_back_to_browser_when_browser_is_allowed(monkeypatch):
monkeypatch.setattr(settings, "liveuamap_api_configured", lambda: True)
monkeypatch.setattr(settings, "liveuamap_browser_scraper_enabled", lambda: True)
monkeypatch.setattr(scraper, "_fetch_liveuamap_api", lambda: (_ for _ in ()).throw(requests.Timeout("boom")))
monkeypatch.setattr(scraper, "_fetch_liveuamap_browser", lambda: [{"id": "browser"}])
assert scraper.fetch_liveuamap() == [{"id": "browser"}]
def test_api_failure_does_not_force_browser_when_browser_is_disabled(monkeypatch):
monkeypatch.setattr(settings, "liveuamap_api_configured", lambda: True)
monkeypatch.setattr(settings, "liveuamap_browser_scraper_enabled", lambda: False)
monkeypatch.setattr(scraper, "_fetch_liveuamap_api", lambda: (_ for _ in ()).throw(requests.Timeout("boom")))
called = False
def browser():
nonlocal called
called = True
return []
monkeypatch.setattr(scraper, "_fetch_liveuamap_browser", browser)
assert scraper.fetch_liveuamap() == []
assert called is False
def test_browser_disable_does_not_disable_configured_api_scheduler_gate(monkeypatch, tmp_path):
monkeypatch.setattr(settings, "_OPT_IN_FILE", tmp_path / "choice.json")
monkeypatch.setattr(settings.os, "name", "nt")
monkeypatch.setenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", "false")
monkeypatch.setenv("LIVEUAMAP_API_URL", "https://api.example.test/events")
assert settings.liveuamap_browser_scraper_enabled() is False
assert settings.liveuamap_scraper_enabled() is True
def test_http_api_endpoint_is_not_used(monkeypatch):
monkeypatch.setenv("LIVEUAMAP_API_URL", "http://api.example.test/events")
monkeypatch.setattr(scraper, "_fetch_liveuamap_browser", lambda: [])
assert scraper._api_url() == ""
+42 -12
View File
@@ -1,8 +1,8 @@
"""LiveUAMap scraper UI opt-in on Windows (#348)."""
"""LiveUAMap provider opt-in and compatibility behavior."""
from __future__ import annotations
import json
from pathlib import Path
import pytest
@@ -13,33 +13,63 @@ from services import liveuamap_settings as settings
def opt_in_file(tmp_path, monkeypatch):
path = tmp_path / "liveuamap_scraper_opt_in.json"
monkeypatch.setattr(settings, "_OPT_IN_FILE", path)
monkeypatch.delenv("LIVEUAMAP_API_URL", raising=False)
monkeypatch.delenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", raising=False)
return path
def test_windows_defaults_off_without_opt_in(monkeypatch, opt_in_file):
def test_windows_defaults_browser_off_without_choice(monkeypatch, opt_in_file):
monkeypatch.setattr(settings.os, "name", "nt")
monkeypatch.delenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", raising=False)
assert settings.liveuamap_scraper_enabled() is False
assert settings.liveuamap_requires_ui_opt_in() is True
assert settings.liveuamap_ui_choice_recorded() is False
assert settings.liveuamap_browser_scraper_enabled() is False
assert settings.liveuamap_scraper_enabled() is False
def test_windows_opt_in_enables_scraper(monkeypatch, opt_in_file):
def test_windows_opt_in_enables_browser(monkeypatch, opt_in_file):
monkeypatch.setattr(settings.os, "name", "nt")
monkeypatch.delenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", raising=False)
settings.set_liveuamap_ui_opt_in(True)
assert settings.liveuamap_scraper_enabled() is True
assert settings.liveuamap_ui_choice_recorded() is True
assert settings.liveuamap_browser_scraper_enabled() is True
assert json.loads(opt_in_file.read_text())["opted_in"] is True
def test_linux_enabled_without_opt_in(monkeypatch, opt_in_file):
def test_windows_decline_is_recorded_without_enabling_browser(monkeypatch, opt_in_file):
monkeypatch.setattr(settings.os, "name", "nt")
settings.set_liveuamap_ui_opt_in(False)
assert settings.liveuamap_ui_choice_recorded() is True
assert settings.get_liveuamap_ui_opt_in() is False
assert settings.liveuamap_browser_scraper_enabled() is False
def test_linux_preserves_existing_auto_enrichment_default(monkeypatch, opt_in_file):
monkeypatch.setattr(settings.os, "name", "posix")
monkeypatch.delenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", raising=False)
assert settings.liveuamap_requires_ui_opt_in() is False
assert settings.liveuamap_browser_scraper_enabled() is True
assert settings.liveuamap_scraper_enabled() is True
def test_env_force_off_overrides_ui_opt_in(monkeypatch, opt_in_file):
def test_env_force_off_disables_browser_even_after_opt_in(monkeypatch, opt_in_file):
monkeypatch.setattr(settings.os, "name", "nt")
monkeypatch.setenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", "false")
settings.set_liveuamap_ui_opt_in(True)
monkeypatch.setenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", "false")
assert settings.liveuamap_browser_scraper_enabled() is False
assert settings.liveuamap_scraper_enabled() is False
def test_api_provider_does_not_require_browser_consent(monkeypatch, opt_in_file):
monkeypatch.setattr(settings.os, "name", "nt")
monkeypatch.setenv("LIVEUAMAP_API_URL", "https://api.example.test/liveuamap")
status = settings.liveuamap_scraper_status()
assert status["api_configured"] is True
assert status["scraper_enabled"] is False
assert status["enrichment_enabled"] is True
assert status["provider_mode"] == "api"
assert settings.liveuamap_scraper_enabled() is True
def test_invalid_http_api_url_does_not_count_as_configured(monkeypatch, opt_in_file):
monkeypatch.setattr(settings.os, "name", "nt")
monkeypatch.setenv("LIVEUAMAP_API_URL", "http://api.example.test/liveuamap")
assert settings.liveuamap_api_configured() is False
assert settings.liveuamap_scraper_enabled() is False