fix(liveuamap): make enrichment resilient and non-blocking

backend/tests/test_liveuamap_parser.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+import base64
+import json
+from urllib.parse import quote
+
+from services.liveuamap_parser import (
+    extract_ovens_expression,
+    iter_valid_coordinates,
+    normalize_liveuamap_payload,
+)
+
+
+def _ids(value):
+    return [str(item.get("id")) for item in normalize_liveuamap_payload(value)]
+
+
+def test_plain_marker_list():
+    payload = [{"id": 1, "lat": 1, "lng": 2, "title": "a"}]
+    assert _ids(payload) == ["1"]
+
+
+def test_double_encoded_json():
+    payload = json.dumps(json.dumps([{"id": "double", "lat": 1, "lng": 2}]))
+    assert _ids(payload) == ["double"]
+
+
+def test_list_of_json_strings_regression_517():
+    payload = [
+        json.dumps({"id": "a", "lat": 10, "lng": 20}),
+        json.dumps({"id": "b", "lat": 30, "lng": 40}),
+    ]
+    assert _ids(payload) == ["a", "b"]
+
+
+def test_mapping_key_becomes_fallback_marker_id():
+    payload = {"123": {"lat": 1, "lng": 2, "title": "keyed"}}
+    markers = normalize_liveuamap_payload(payload)
+    assert markers[0]["id"] == "123"
+
+
+def test_common_wrapper_shape():
+    payload = {"data": {"markers": [{"id": "wrapped", "lat": 1, "lng": 2}]}}
+    assert _ids(payload) == ["wrapped"]
+
+
+def test_legacy_urlencoded_base64_json():
+    raw = json.dumps([{"id": "legacy", "lat": 1, "lng": 2}]).encode()
+    payload = quote(base64.b64encode(raw).decode())
+    assert _ids(payload) == ["legacy"]
+
+
+def test_geojson_feature_collection():
+    payload = {
+        "type": "FeatureCollection",
+        "features": [
+            {
+                "type": "Feature",
+                "id": "geo",
+                "geometry": {"type": "Point", "coordinates": [20, 10]},
+                "properties": {"title": "Geo event"},
+            }
+        ],
+    }
+    markers = normalize_liveuamap_payload(payload)
+    assert markers == [{"title": "Geo event", "lat": 10.0, "lng": 20.0, "id": "geo"}]
+
+
+def test_malformed_scalars_are_ignored_instead_of_crashing():
+    payload = ["not-json", 42, None, True, {"nested": object()}]
+    assert normalize_liveuamap_payload(payload) == []
+
+
+def test_coordinate_iterator_rejects_out_of_range_and_nonfinite():
+    markers = [
+        {"id": "good", "lat": "10", "lng": "20"},
+        {"id": "bad-lat", "lat": 100, "lng": 20},
+        {"id": "bad-lng", "lat": 10, "lng": 200},
+        {"id": "nan", "lat": float("nan"), "lng": 20},
+    ]
+    valid = list(iter_valid_coordinates(markers))
+    assert [(item[0]["id"], item[1], item[2]) for item in valid] == [("good", 10.0, 20.0)]
+
+
+def test_extracts_var_let_and_const_ovens():
+    assert extract_ovens_expression('<script>var ovens = [{"id":1}];</script>') == '[{"id":1}]'
+    assert extract_ovens_expression('<script>let ovens = "abc";</script>') == '"abc"'
+    assert extract_ovens_expression('<script>const ovens = {"data":[]};</script>') == '{"data":[]}'

backend/tests/test_liveuamap_provider.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import requests
+
+from services import liveuamap_scraper as scraper
+from services import liveuamap_settings as settings
+
+
+class _Response:
+    def __init__(self, payload, status_code=200):
+        self._payload = payload
+        self.status_code = status_code
+
+    def raise_for_status(self):
+        if self.status_code >= 400:
+            raise requests.HTTPError(f"HTTP {self.status_code}")
+
+    def json(self):
+        return self._payload
+
+
+def test_api_geojson_is_normalized_and_auth_header_is_sent(monkeypatch):
+    monkeypatch.setenv("LIVEUAMAP_API_URL", "https://api.example.test/events")
+    monkeypatch.setenv("LIVEUAMAP_API_KEY", "secret-key")
+    monkeypatch.setenv("LIVEUAMAP_API_TIMEOUT_S", "12")
+    monkeypatch.setattr(
+        "services.network_utils.outbound_user_agent",
+        lambda purpose="": f"operator-test ({purpose})",
+    )
+    seen = {}
+
+    def fake_get(url, *, headers, timeout):
+        seen.update(url=url, headers=headers, timeout=timeout)
+        return _Response(
+            {
+                "type": "FeatureCollection",
+                "features": [
+                    {
+                        "type": "Feature",
+                        "id": "evt-1",
+                        "geometry": {"type": "Point", "coordinates": [30.5, 50.5]},
+                        "properties": {"title": "Event", "url": "https://example.test/e/1"},
+                    }
+                ],
+            }
+        )
+
+    monkeypatch.setattr(scraper.requests, "get", fake_get)
+    markers = scraper._fetch_liveuamap_api()
+    assert markers[0]["id"] == "evt-1"
+    assert markers[0]["lat"] == 50.5
+    assert markers[0]["lng"] == 30.5
+    assert markers[0]["provider"] == "api"
+    assert seen["headers"]["Authorization"] == "Bearer secret-key"
+    assert seen["timeout"] == (5, 12)
+
+
+def test_api_failure_falls_back_to_browser_when_browser_is_allowed(monkeypatch):
+    monkeypatch.setattr(settings, "liveuamap_api_configured", lambda: True)
+    monkeypatch.setattr(settings, "liveuamap_browser_scraper_enabled", lambda: True)
+    monkeypatch.setattr(scraper, "_fetch_liveuamap_api", lambda: (_ for _ in ()).throw(requests.Timeout("boom")))
+    monkeypatch.setattr(scraper, "_fetch_liveuamap_browser", lambda: [{"id": "browser"}])
+    assert scraper.fetch_liveuamap() == [{"id": "browser"}]
+
+
+def test_api_failure_does_not_force_browser_when_browser_is_disabled(monkeypatch):
+    monkeypatch.setattr(settings, "liveuamap_api_configured", lambda: True)
+    monkeypatch.setattr(settings, "liveuamap_browser_scraper_enabled", lambda: False)
+    monkeypatch.setattr(scraper, "_fetch_liveuamap_api", lambda: (_ for _ in ()).throw(requests.Timeout("boom")))
+    called = False
+
+    def browser():
+        nonlocal called
+        called = True
+        return []
+
+    monkeypatch.setattr(scraper, "_fetch_liveuamap_browser", browser)
+    assert scraper.fetch_liveuamap() == []
+    assert called is False
+
+
+def test_browser_disable_does_not_disable_configured_api_scheduler_gate(monkeypatch, tmp_path):
+    monkeypatch.setattr(settings, "_OPT_IN_FILE", tmp_path / "choice.json")
+    monkeypatch.setattr(settings.os, "name", "nt")
+    monkeypatch.setenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", "false")
+    monkeypatch.setenv("LIVEUAMAP_API_URL", "https://api.example.test/events")
+    assert settings.liveuamap_browser_scraper_enabled() is False
+    assert settings.liveuamap_scraper_enabled() is True
+
+
+def test_http_api_endpoint_is_not_used(monkeypatch):
+    monkeypatch.setenv("LIVEUAMAP_API_URL", "http://api.example.test/events")
+    monkeypatch.setattr(scraper, "_fetch_liveuamap_browser", lambda: [])
+    assert scraper._api_url() == ""

backend/tests/test_liveuamap_scraper_opt_in.py

@@ -1,8 +1,8 @@
-"""LiveUAMap scraper UI opt-in on Windows (#348)."""
+"""LiveUAMap provider opt-in and compatibility behavior."""
+
 from __future__ import annotations
 
 import json
-from pathlib import Path
 
 import pytest
 
@@ -13,33 +13,63 @@ from services import liveuamap_settings as settings
 def opt_in_file(tmp_path, monkeypatch):
     path = tmp_path / "liveuamap_scraper_opt_in.json"
     monkeypatch.setattr(settings, "_OPT_IN_FILE", path)
+    monkeypatch.delenv("LIVEUAMAP_API_URL", raising=False)
+    monkeypatch.delenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", raising=False)
     return path
 
 
-def test_windows_defaults_off_without_opt_in(monkeypatch, opt_in_file):
+def test_windows_defaults_browser_off_without_choice(monkeypatch, opt_in_file):
     monkeypatch.setattr(settings.os, "name", "nt")
-    monkeypatch.delenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", raising=False)
-    assert settings.liveuamap_scraper_enabled() is False
     assert settings.liveuamap_requires_ui_opt_in() is True
+    assert settings.liveuamap_ui_choice_recorded() is False
+    assert settings.liveuamap_browser_scraper_enabled() is False
+    assert settings.liveuamap_scraper_enabled() is False
 
 
-def test_windows_opt_in_enables_scraper(monkeypatch, opt_in_file):
+def test_windows_opt_in_enables_browser(monkeypatch, opt_in_file):
     monkeypatch.setattr(settings.os, "name", "nt")
-    monkeypatch.delenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", raising=False)
     settings.set_liveuamap_ui_opt_in(True)
-    assert settings.liveuamap_scraper_enabled() is True
+    assert settings.liveuamap_ui_choice_recorded() is True
+    assert settings.liveuamap_browser_scraper_enabled() is True
     assert json.loads(opt_in_file.read_text())["opted_in"] is True
 
 
-def test_linux_enabled_without_opt_in(monkeypatch, opt_in_file):
+def test_windows_decline_is_recorded_without_enabling_browser(monkeypatch, opt_in_file):
+    monkeypatch.setattr(settings.os, "name", "nt")
+    settings.set_liveuamap_ui_opt_in(False)
+    assert settings.liveuamap_ui_choice_recorded() is True
+    assert settings.get_liveuamap_ui_opt_in() is False
+    assert settings.liveuamap_browser_scraper_enabled() is False
+
+
+def test_linux_preserves_existing_auto_enrichment_default(monkeypatch, opt_in_file):
     monkeypatch.setattr(settings.os, "name", "posix")
-    monkeypatch.delenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", raising=False)
     assert settings.liveuamap_requires_ui_opt_in() is False
+    assert settings.liveuamap_browser_scraper_enabled() is True
     assert settings.liveuamap_scraper_enabled() is True
 
 
-def test_env_force_off_overrides_ui_opt_in(monkeypatch, opt_in_file):
+def test_env_force_off_disables_browser_even_after_opt_in(monkeypatch, opt_in_file):
     monkeypatch.setattr(settings.os, "name", "nt")
-    monkeypatch.setenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", "false")
     settings.set_liveuamap_ui_opt_in(True)
+    monkeypatch.setenv("SHADOWBROKER_ENABLE_LIVEUAMAP_SCRAPER", "false")
+    assert settings.liveuamap_browser_scraper_enabled() is False
+    assert settings.liveuamap_scraper_enabled() is False
+
+
+def test_api_provider_does_not_require_browser_consent(monkeypatch, opt_in_file):
+    monkeypatch.setattr(settings.os, "name", "nt")
+    monkeypatch.setenv("LIVEUAMAP_API_URL", "https://api.example.test/liveuamap")
+    status = settings.liveuamap_scraper_status()
+    assert status["api_configured"] is True
+    assert status["scraper_enabled"] is False
+    assert status["enrichment_enabled"] is True
+    assert status["provider_mode"] == "api"
+    assert settings.liveuamap_scraper_enabled() is True
+
+
+def test_invalid_http_api_url_does_not_count_as_configured(monkeypatch, opt_in_file):
+    monkeypatch.setattr(settings.os, "name", "nt")
+    monkeypatch.setenv("LIVEUAMAP_API_URL", "http://api.example.test/liveuamap")
+    assert settings.liveuamap_api_configured() is False
     assert settings.liveuamap_scraper_enabled() is False