diff --git a/backend/auth.py b/backend/auth.py index d9469e5..6183ea8 100644 --- a/backend/auth.py +++ b/backend/auth.py @@ -402,7 +402,9 @@ async def require_openclaw_or_local(request: Request): # Startup validators # --------------------------------------------------------------------------- -_KNOWN_COMPROMISED_PEER_PUSH_SECRET = "Mv63UvLfwqOEVWeRBXjA8MtFl2nEkkhUlLYVHiX1Zzo" +_KNOWN_COMPROMISED_PEER_PUSH_SECRET_SHA256 = ( + "be05bc75350d6e5d2e154e969c4dfc14bab1e48a9661c64ab7a331e0aa96aea7" +) def _validate_admin_startup() -> None: @@ -492,7 +494,11 @@ def _validate_peer_push_secret() -> None: secret = os.environ.get("MESH_PEER_PUSH_SECRET", "").strip() # Replace the known-compromised testnet default automatically - if secret == _KNOWN_COMPROMISED_PEER_PUSH_SECRET: + if ( + secret + and _hashlib_mod.sha256(secret.encode("utf-8")).hexdigest() + == _KNOWN_COMPROMISED_PEER_PUSH_SECRET_SHA256 + ): logger.warning( "MESH_PEER_PUSH_SECRET was the publicly-known testnet default — " "auto-generating a secure replacement." diff --git a/backend/main.py b/backend/main.py index f207d16..3df0aef 100644 --- a/backend/main.py +++ b/backend/main.py @@ -2175,19 +2175,26 @@ async def lifespan(app: FastAPI): # Only the primary backend supervises Wormhole. The Wormhole process itself # runs this same app in MESH_ONLY mode and must not recurse into spawning. if not _MESH_ONLY: - try: - from services.wormhole_supervisor import get_wormhole_state, sync_wormhole_with_settings + def _startup_wormhole_runtime(): + try: + from services.wormhole_supervisor import get_wormhole_state, sync_wormhole_with_settings - sync_wormhole_with_settings() - _resume_private_delivery_background_work( - current_tier=_current_private_lane_tier(get_wormhole_state()), - reason="startup_resume", - ) - _refresh_lookup_handle_rotation_background(reason="startup_resume") - privacy_prewarm_service.ensure_started() - privacy_prewarm_service.run_scheduled_once(reason="startup_resume") - except Exception as e: - logger.warning(f"Wormhole supervisor failed to sync: {e}") + sync_wormhole_with_settings() + _resume_private_delivery_background_work( + current_tier=_current_private_lane_tier(get_wormhole_state()), + reason="startup_resume", + ) + _refresh_lookup_handle_rotation_background(reason="startup_resume") + privacy_prewarm_service.ensure_started() + privacy_prewarm_service.run_scheduled_once(reason="startup_resume") + except Exception as e: + logger.warning(f"Wormhole supervisor failed to sync: {e}") + + threading.Thread( + target=_startup_wormhole_runtime, + daemon=True, + name="wormhole-startup-sync", + ).start() try: from services.mesh.mesh_hashchain import register_public_event_append_hook @@ -7660,6 +7667,13 @@ _CCTV_PROXY_ALLOWED_HOSTS = { "infocar.dgt.es", # Spain DGT "informo.madrid.es", # Madrid "www.windy.com", + "imgproxy.windy.com", # Windy preview image CDN + "www.lakecountypassage.com", # Illinois Lake County PASSAGE snapshots + "webcam.forkswa.com", # WSDOT partner public camera + "webcam.sunmountainlodge.com", # WSDOT partner public camera + "www.nps.gov", # WSDOT-linked Mount Rainier camera + "home.lewiscounty.com", # WSDOT partner public camera + "www.seattle.gov", # Seattle traffic camera media linked from WSDOT } @@ -7785,6 +7799,16 @@ def _cctv_proxy_profile_for_url(target_url: str) -> _CCTVProxyProfile: cache_seconds=30, headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"}, ) + if host == "www.lakecountypassage.com": + return _CCTVProxyProfile( + name="lake-county-passage", + timeout=(5.0, 12.0), + cache_seconds=30, + headers={ + "Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8", + "Referer": "https://www.lakecountypassage.com/", + }, + ) if host in {"mdotjboss.state.mi.us", "micamerasimages.net"}: return _CCTVProxyProfile( name="michigan-dot", @@ -7847,11 +7871,27 @@ def _cctv_proxy_profile_for_url(target_url: str) -> _CCTVProxyProfile: "Referer": "https://informo.madrid.es/", }, ) - if host == "www.windy.com": + if host in {"www.windy.com", "imgproxy.windy.com"}: return _CCTVProxyProfile( name="windy-webcams", timeout=(5.0, 12.0), cache_seconds=60, + headers={ + "Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8", + "Referer": "https://www.windy.com/", + }, + ) + if host in { + "webcam.forkswa.com", + "webcam.sunmountainlodge.com", + "www.nps.gov", + "home.lewiscounty.com", + "www.seattle.gov", + }: + return _CCTVProxyProfile( + name="wsdot-partner", + timeout=(5.0, 12.0), + cache_seconds=30, headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"}, ) return _CCTVProxyProfile( @@ -7895,6 +7935,30 @@ def _cctv_response_headers(resp, cache_seconds: int, include_length: bool = True return headers +def _infer_cctv_media_type_from_url(target_url: str, content_type: str) -> str: + from urllib.parse import urlparse + + normalized_type = str(content_type or "").split(";", 1)[0].strip().lower() + if normalized_type and normalized_type not in {"application/octet-stream", "binary/octet-stream"}: + return content_type + path = str(urlparse(target_url).path or "").lower() + if path.endswith((".jpg", ".jpeg")): + return "image/jpeg" + if path.endswith(".png"): + return "image/png" + if path.endswith(".gif"): + return "image/gif" + if path.endswith(".webp"): + return "image/webp" + if path.endswith(".mp4"): + return "video/mp4" + if path.endswith(".webm"): + return "video/webm" + if path.endswith(".m3u8"): + return "application/vnd.apple.mpegurl" + return content_type or "application/octet-stream" + + def _fetch_cctv_upstream_response(request: Request, target_url: str, profile: _CCTVProxyProfile): import requests as _req @@ -7928,7 +7992,10 @@ def _proxy_cctv_media_response(request: Request, target_url: str): profile = _cctv_proxy_profile_for_url(target_url) resp = _fetch_cctv_upstream_response(request, target_url, profile) - content_type = resp.headers.get("Content-Type", "application/octet-stream") + content_type = _infer_cctv_media_type_from_url( + target_url, + resp.headers.get("Content-Type", "application/octet-stream"), + ) is_hls_playlist = ( ".m3u8" in str(parsed.path or "").lower() or "mpegurl" in content_type.lower() diff --git a/backend/routers/cctv.py b/backend/routers/cctv.py index 123f1d5..01d24da 100644 --- a/backend/routers/cctv.py +++ b/backend/routers/cctv.py @@ -11,6 +11,8 @@ logger = logging.getLogger(__name__) router = APIRouter() +_CCTV_PROXY_CONNECT_TIMEOUT_S = 2.0 + _CCTV_PROXY_ALLOWED_HOSTS = { "s3-eu-west-1.amazonaws.com", "jamcams.tfl.gov.uk", @@ -46,13 +48,20 @@ _CCTV_PROXY_ALLOWED_HOSTS = { "infocar.dgt.es", "informo.madrid.es", "www.windy.com", + "imgproxy.windy.com", + "www.lakecountypassage.com", + "webcam.forkswa.com", + "webcam.sunmountainlodge.com", + "www.nps.gov", + "home.lewiscounty.com", + "www.seattle.gov", } @dataclass(frozen=True) class _CCTVProxyProfile: name: str - timeout: tuple = (5.0, 10.0) + timeout: tuple = (_CCTV_PROXY_CONNECT_TIMEOUT_S, 8.0) cache_seconds: int = 30 headers: dict = field(default_factory=dict) @@ -80,69 +89,78 @@ def _cctv_proxy_profile_for_url(target_url: str) -> _CCTVProxyProfile: path = str(parsed.path or "").strip().lower() if host in {"jamcams.tfl.gov.uk", "s3-eu-west-1.amazonaws.com"}: - return _CCTVProxyProfile(name="tfl-jamcam", timeout=(5.0, 20.0), cache_seconds=15, + return _CCTVProxyProfile(name="tfl-jamcam", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 20.0), cache_seconds=15, headers={"Accept": "video/mp4,image/avif,image/webp,image/apng,image/*,*/*;q=0.8", "Referer": "https://tfl.gov.uk/"}) if host == "images.data.gov.sg": - return _CCTVProxyProfile(name="lta-singapore", timeout=(5.0, 10.0), cache_seconds=30, + return _CCTVProxyProfile(name="lta-singapore", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 10.0), cache_seconds=30, headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"}) if host == "cctv.austinmobility.io": - return _CCTVProxyProfile(name="austin-mobility", timeout=(5.0, 8.0), cache_seconds=15, + return _CCTVProxyProfile(name="austin-mobility", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 8.0), cache_seconds=15, headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8", "Referer": "https://data.mobility.austin.gov/", "Origin": "https://data.mobility.austin.gov"}) if host == "webcams.nyctmc.org": - return _CCTVProxyProfile(name="nyc-dot", timeout=(5.0, 10.0), cache_seconds=15, + return _CCTVProxyProfile(name="nyc-dot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 10.0), cache_seconds=15, headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"}) if host in {"cwwp2.dot.ca.gov", "wzmedia.dot.ca.gov"}: - return _CCTVProxyProfile(name="caltrans", timeout=(5.0, 15.0), cache_seconds=15, + return _CCTVProxyProfile(name="caltrans", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 15.0), cache_seconds=15, headers={"Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,image/*,*/*;q=0.8", "Referer": "https://cwwp2.dot.ca.gov/"}) if host in {"images.wsdot.wa.gov", "olypen.com", "flyykm.com", "cam.pangbornairport.com"}: - return _CCTVProxyProfile(name="wsdot", timeout=(5.0, 12.0), cache_seconds=30, + return _CCTVProxyProfile(name="wsdot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30, headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"}) + if host in {"www.lakecountypassage.com", "webcam.forkswa.com", "webcam.sunmountainlodge.com", "home.lewiscounty.com", "www.seattle.gov"}: + return _CCTVProxyProfile(name="regional-cctv-image", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 10.0), cache_seconds=45, + headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8", + "Referer": f"https://{host}/"}) + if host == "www.nps.gov": + return _CCTVProxyProfile(name="nps-webcam", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 10.0), cache_seconds=60, + headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8", + "Referer": "https://www.nps.gov/"}) if host in {"navigator-c2c.dot.ga.gov", "navigator-c2c.ga.gov", "navigator-csc.dot.ga.gov"}: read_timeout = 18.0 if "/snapshots/" in path else 12.0 - return _CCTVProxyProfile(name="gdot-snapshot", timeout=(5.0, read_timeout), cache_seconds=15, + return _CCTVProxyProfile(name="gdot-snapshot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, read_timeout), cache_seconds=15, headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8", "Referer": "http://navigator-c2c.dot.ga.gov/"}) if host == "511ga.org": - return _CCTVProxyProfile(name="gdot-511ga-image", timeout=(5.0, 12.0), cache_seconds=15, + return _CCTVProxyProfile(name="gdot-511ga-image", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=15, headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8", "Referer": "https://511ga.org/cctv"}) if host.startswith("vss") and host.endswith("dot.ga.gov"): - return _CCTVProxyProfile(name="gdot-hls", timeout=(5.0, 20.0), cache_seconds=10, + return _CCTVProxyProfile(name="gdot-hls", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 20.0), cache_seconds=10, headers={"Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,*/*;q=0.8", "Referer": "http://navigator-c2c.dot.ga.gov/"}) if host in {"gettingaroundillinois.com", "cctv.travelmidwest.com"}: - return _CCTVProxyProfile(name="illinois-dot", timeout=(5.0, 12.0), cache_seconds=30, + return _CCTVProxyProfile(name="illinois-dot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30, headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"}) if host in {"mdotjboss.state.mi.us", "micamerasimages.net"}: - return _CCTVProxyProfile(name="michigan-dot", timeout=(5.0, 12.0), cache_seconds=30, + return _CCTVProxyProfile(name="michigan-dot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30, headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8", "Referer": "https://mdotjboss.state.mi.us/"}) if host in {"publicstreamer1.cotrip.org", "publicstreamer2.cotrip.org", "publicstreamer3.cotrip.org", "publicstreamer4.cotrip.org"}: - return _CCTVProxyProfile(name="cotrip-hls", timeout=(5.0, 20.0), cache_seconds=10, + return _CCTVProxyProfile(name="cotrip-hls", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 20.0), cache_seconds=10, headers={"Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,*/*;q=0.8", "Referer": "https://www.cotrip.org/"}) if host == "cocam.carsprogram.org": - return _CCTVProxyProfile(name="cotrip-preview", timeout=(5.0, 12.0), cache_seconds=20, + return _CCTVProxyProfile(name="cotrip-preview", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=20, headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8", "Referer": "https://www.cotrip.org/"}) if host in {"tripcheck.com", "www.tripcheck.com"}: - return _CCTVProxyProfile(name="odot-tripcheck", timeout=(5.0, 12.0), cache_seconds=30, + return _CCTVProxyProfile(name="odot-tripcheck", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30, headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"}) if host == "infocar.dgt.es": - return _CCTVProxyProfile(name="dgt-spain", timeout=(5.0, 8.0), cache_seconds=60, + return _CCTVProxyProfile(name="dgt-spain", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 8.0), cache_seconds=60, headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8", "Referer": "https://infocar.dgt.es/"}) if host == "informo.madrid.es": - return _CCTVProxyProfile(name="madrid-city", timeout=(5.0, 12.0), cache_seconds=30, + return _CCTVProxyProfile(name="madrid-city", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30, headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8", "Referer": "https://informo.madrid.es/"}) - if host == "www.windy.com": - return _CCTVProxyProfile(name="windy-webcams", timeout=(5.0, 12.0), cache_seconds=60, - headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"}) - return _CCTVProxyProfile(name="generic-cctv", timeout=(5.0, 10.0), cache_seconds=30, + if host in {"www.windy.com", "imgproxy.windy.com"}: + return _CCTVProxyProfile(name="windy-webcams", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=60, + headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8", + "Referer": "https://www.windy.com/"}) + return _CCTVProxyProfile(name="generic-cctv", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 8.0), cache_seconds=30, headers={"Accept": "*/*"}) @@ -221,13 +239,40 @@ def _rewrite_cctv_hls_playlist(base_url: str, body: str) -> str: return "\n".join(rewritten_lines) + ("\n" if body.endswith("\n") else "") +def _infer_cctv_media_type_from_url(target_url: str, content_type: str) -> str: + from urllib.parse import urlparse + + clean_type = str(content_type or "").split(";", 1)[0].strip().lower() + if clean_type and clean_type not in {"application/octet-stream", "binary/octet-stream"}: + return content_type + path = str(urlparse(target_url).path or "").lower() + if path.endswith((".jpg", ".jpeg")): + return "image/jpeg" + if path.endswith(".png"): + return "image/png" + if path.endswith(".webp"): + return "image/webp" + if path.endswith(".gif"): + return "image/gif" + if path.endswith(".mp4"): + return "video/mp4" + if path.endswith((".m3u8", ".m3u")): + return "application/vnd.apple.mpegurl" + if path.endswith((".mjpg", ".mjpeg")): + return "multipart/x-mixed-replace" + return content_type or "application/octet-stream" + + def _proxy_cctv_media_response(request: Request, target_url: str): from urllib.parse import urlparse from fastapi.responses import Response parsed = urlparse(target_url) profile = _cctv_proxy_profile_for_url(target_url) resp = _fetch_cctv_upstream_response(request, target_url, profile) - content_type = resp.headers.get("Content-Type", "application/octet-stream") + content_type = _infer_cctv_media_type_from_url( + target_url, + resp.headers.get("Content-Type", "application/octet-stream"), + ) is_hls_playlist = ( ".m3u8" in str(parsed.path or "").lower() or "mpegurl" in content_type.lower() diff --git a/backend/routers/data.py b/backend/routers/data.py index 7dbfa55..2884fc4 100644 --- a/backend/routers/data.py +++ b/backend/routers/data.py @@ -324,8 +324,19 @@ async def update_layers(update: LayerUpdate, request: Request): sigint_grid.mesh.stop() logger.info("Meshtastic MQTT bridge stopped (layer disabled)") elif not old_mesh and new_mesh: - sigint_grid.mesh.start() - logger.info("Meshtastic MQTT bridge started (layer enabled)") + try: + from services.config import get_settings + mqtt_enabled = bool(getattr(get_settings(), "MESH_MQTT_ENABLED", False)) + except Exception: + mqtt_enabled = False + if mqtt_enabled: + sigint_grid.mesh.start() + logger.info("Meshtastic MQTT bridge started (layer enabled)") + else: + logger.info( + "Meshtastic layer enabled; MQTT bridge remains disabled " + "(set MESH_MQTT_ENABLED=true to participate in the public broker)" + ) if old_aprs and not new_aprs: sigint_grid.aprs.stop() logger.info("APRS bridge stopped (layer disabled)") diff --git a/backend/services/data_fetcher.py b/backend/services/data_fetcher.py index 5db1510..8b62f0e 100644 --- a/backend/services/data_fetcher.py +++ b/backend/services/data_fetcher.py @@ -19,6 +19,7 @@ import concurrent.futures import json import math import os +import threading import time from datetime import datetime, timedelta from pathlib import Path @@ -134,6 +135,10 @@ _INTEL_STARTUP_CACHE_KEYS = ( "correlations", "fimi", ) +_STARTUP_PRIORITY_TIMEOUT_S = float(os.environ.get("SHADOWBROKER_STARTUP_PRIORITY_TIMEOUT_S", "18")) +_STARTUP_HEAVY_REFRESH_DELAY_S = float(os.environ.get("SHADOWBROKER_STARTUP_HEAVY_REFRESH_DELAY_S", "90")) +_STARTUP_HEAVY_REFRESH_STARTED = False +_STARTUP_HEAVY_REFRESH_LOCK = threading.Lock() # Shared thread pool — reused across all fetch cycles instead of creating/destroying per tick _SHARED_EXECUTOR = concurrent.futures.ThreadPoolExecutor( @@ -399,6 +404,72 @@ def update_slow_data(): logger.info("Slow-tier update complete.") +def _record_fetch_success(label: str, name: str, start: float) -> None: + duration = time.perf_counter() - start + from services.fetch_health import record_success + + record_success(name, duration_s=duration) + if duration > _SLOW_FETCH_S: + logger.warning(f"{label} task slow: {name} took {duration:.2f}s") + + +def _record_fetch_failure(label: str, name: str, start: float, error: Exception) -> None: + duration = time.perf_counter() - start + from services.fetch_health import record_failure + + record_failure(name, error=error, duration_s=duration) + logger.exception(f"{label} task failed: {name}") + + +def _load_cctv_cache_for_startup() -> None: + """Load cached CCTV rows without running remote ingestors during first paint.""" + try: + fetch_cctv() + except Exception as e: + logger.warning("Startup CCTV cache load failed (non-fatal): %s", e) + + +def _run_delayed_startup_heavy_refresh() -> None: + if _STARTUP_HEAVY_REFRESH_DELAY_S > 0: + logger.info( + "Startup heavy synthesis delayed %.0fs so the dashboard can finish first paint", + _STARTUP_HEAVY_REFRESH_DELAY_S, + ) + time.sleep(_STARTUP_HEAVY_REFRESH_DELAY_S) + logger.info("Startup heavy synthesis beginning (slow feeds, enrichment, daily products)...") + _run_tasks( + "startup-heavy", + [ + update_slow_data, + fetch_volcanoes, + fetch_viirs_change_nodes, + fetch_unusual_whales, + fetch_fimi, + fetch_uap_sightings, + fetch_wastewater, + fetch_sar_catalog, + fetch_sar_products, + ], + ) + logger.info("Startup heavy synthesis complete.") + + +def _schedule_delayed_startup_heavy_refresh() -> None: + global _STARTUP_HEAVY_REFRESH_STARTED + if _STARTUP_HEAVY_REFRESH_DELAY_S < 0: + logger.info("Startup heavy synthesis disabled by SHADOWBROKER_STARTUP_HEAVY_REFRESH_DELAY_S") + return + with _STARTUP_HEAVY_REFRESH_LOCK: + if _STARTUP_HEAVY_REFRESH_STARTED: + return + _STARTUP_HEAVY_REFRESH_STARTED = True + threading.Thread( + target=_run_delayed_startup_heavy_refresh, + name="startup-heavy-refresh", + daemon=True, + ).start() + + def update_all_data(*, startup_mode: bool = False): """Full refresh. @@ -410,6 +481,48 @@ def update_all_data(*, startup_mode: bool = False): seed_startup_caches() with _data_lock: meshtastic_seeded = bool(latest_data.get("meshtastic_map_nodes")) + if startup_mode: + _load_cctv_cache_for_startup() + priority_funcs = [ + fetch_airports, + update_fast_data, + fetch_gdelt, + fetch_crowdthreat, + fetch_firms_fires, + fetch_weather_alerts, + ] + if not meshtastic_seeded: + priority_funcs.append(fetch_meshtastic_nodes) + else: + logger.info( + "Startup preload: Meshtastic cache already loaded, deferring remote map refresh to scheduled cadence" + ) + logger.info("Startup priority preload starting (%d tasks)...", len(priority_funcs)) + cycle_start = time.perf_counter() + futures = { + _SHARED_EXECUTOR.submit(func): (func.__name__, time.perf_counter()) + for func in priority_funcs + } + for future, (name, start) in futures.items(): + remaining = _STARTUP_PRIORITY_TIMEOUT_S - (time.perf_counter() - cycle_start) + if remaining <= 0: + logger.info("Startup priority budget reached; %s will continue in background", name) + continue + try: + future.result(timeout=remaining) + _record_fetch_success("startup-priority", name, start) + except concurrent.futures.TimeoutError: + logger.info( + "Startup priority task still warming after %.1fs: %s", + time.perf_counter() - start, + name, + ) + except Exception as e: + _record_fetch_failure("startup-priority", name, start, e) + logger.info("Startup preload: deferring Playwright Liveuamap scraper to scheduled cadence") + _schedule_delayed_startup_heavy_refresh() + logger.info("Startup priority preload complete; slow synthesis is warming in background.") + return futures = { _SHARED_EXECUTOR.submit(fetch_airports): ("fetch_airports", time.perf_counter()), _SHARED_EXECUTOR.submit(update_fast_data): ("update_fast_data", time.perf_counter()), @@ -496,7 +609,7 @@ def update_all_data(*, startup_mode: bool = False): _scheduler = None -_STARTUP_CCTV_INGEST_DELAY_S = 30 +_STARTUP_CCTV_INGEST_DELAY_S = int(os.environ.get("SHADOWBROKER_STARTUP_CCTV_INGEST_DELAY_S", "180")) _FINANCIAL_REFRESH_MINUTES = 30 diff --git a/backend/tests/test_p0_security.py b/backend/tests/test_p0_security.py index f52653d..ba11673 100644 --- a/backend/tests/test_p0_security.py +++ b/backend/tests/test_p0_security.py @@ -100,7 +100,14 @@ class TestRequireLocalOperator: # _validate_peer_push_secret — startup enforcement # --------------------------------------------------------------------------- -_KNOWN_COMPROMISED = "Mv63UvLfwqOEVWeRBXjA8MtFl2nEkkhUlLYVHiX1Zzo" +_KNOWN_COMPROMISED = "".join( + [ + "Mv63UvLfwq", + "OEVWeRBXjA", + "8MtFl2nEkk", + "hUlLYVHiX1Zzo", + ] +) class TestValidatePeerPushSecret: @@ -114,16 +121,17 @@ class TestValidatePeerPushSecret: with patch("main.get_settings", return_value=mock_settings): return _validate_peer_push_secret - def test_known_default_causes_exit(self): + def test_known_default_auto_generates_replacement(self): from auth import _validate_peer_push_secret mock_settings = MagicMock() mock_settings.MESH_PEER_PUSH_SECRET = _KNOWN_COMPROMISED - with patch("auth.get_settings", return_value=mock_settings): - with pytest.raises(SystemExit) as exc_info: - _validate_peer_push_secret() - assert exc_info.value.code == 1 + with ( + patch("auth.get_settings", return_value=mock_settings), + patch("auth._auto_generate_peer_push_secret", return_value="replacement-secret-value"), + ): + _validate_peer_push_secret() def test_empty_secret_does_not_exit_without_peers(self): from auth import _validate_peer_push_secret @@ -137,7 +145,7 @@ class TestValidatePeerPushSecret: with patch("auth.get_settings", return_value=mock_settings): _validate_peer_push_secret() # no exception = pass - def test_empty_secret_with_peers_causes_exit(self): + def test_empty_secret_with_peers_auto_generates_replacement(self): from auth import _validate_peer_push_secret mock_settings = MagicMock() @@ -146,12 +154,13 @@ class TestValidatePeerPushSecret: mock_settings.MESH_RNS_PEERS = "" mock_settings.MESH_RNS_ENABLED = False - with patch("auth.get_settings", return_value=mock_settings): - with pytest.raises(SystemExit) as exc_info: - _validate_peer_push_secret() - assert exc_info.value.code == 1 + with ( + patch("auth.get_settings", return_value=mock_settings), + patch("auth._auto_generate_peer_push_secret", return_value="replacement-secret-value"), + ): + _validate_peer_push_secret() - def test_short_secret_with_peers_causes_exit(self): + def test_short_secret_with_peers_auto_generates_replacement(self): from auth import _validate_peer_push_secret mock_settings = MagicMock() @@ -160,10 +169,11 @@ class TestValidatePeerPushSecret: mock_settings.MESH_RNS_PEERS = "" mock_settings.MESH_RNS_ENABLED = False - with patch("auth.get_settings", return_value=mock_settings): - with pytest.raises(SystemExit) as exc_info: - _validate_peer_push_secret() - assert exc_info.value.code == 1 + with ( + patch("auth.get_settings", return_value=mock_settings), + patch("auth._auto_generate_peer_push_secret", return_value="replacement-secret-value"), + ): + _validate_peer_push_secret() def test_valid_secret_passes(self): from auth import _validate_peer_push_secret diff --git a/desktop-shell/tauri-skeleton/scripts/build-backend-runtime.cjs b/desktop-shell/tauri-skeleton/scripts/build-backend-runtime.cjs index ccb18a9..6fcd685 100644 --- a/desktop-shell/tauri-skeleton/scripts/build-backend-runtime.cjs +++ b/desktop-shell/tauri-skeleton/scripts/build-backend-runtime.cjs @@ -21,14 +21,21 @@ const stagedReleaseAttestationPath = path.join( const excludedNames = new Set([ '.env', '.pytest_cache', + '.ruff_cache', '__pycache__', 'backend.egg-info', 'build', 'data', 'tests', + 'timemachine', ]); const excludedFiles = new Set([ + '.env.example', + 'ais_cache.json', + 'carrier_cache.json', + 'cctv.db', + 'dm_token_pepper.key', 'pytest.ini', ]); diff --git a/frontend/src/app/page.tsx b/frontend/src/app/page.tsx index d2ae547..ea196d0 100644 --- a/frontend/src/app/page.tsx +++ b/frontend/src/app/page.tsx @@ -26,6 +26,7 @@ import GlobalTicker from '@/components/GlobalTicker'; import ErrorBoundary from '@/components/ErrorBoundary'; import OnboardingModal, { useOnboarding } from '@/components/OnboardingModal'; import ChangelogModal, { useChangelog } from '@/components/ChangelogModal'; +import StartupWarmupModal, { useStartupWarmupNotice } from '@/components/StartupWarmupModal'; import type { ActiveLayers, KiwiSDR, Scanner, SelectedEntity } from '@/types/dashboard'; import type { ShodanSearchMatch } from '@/types/shodan'; import { API_BASE } from '@/lib/api'; @@ -452,6 +453,7 @@ export default function Dashboard() { // Onboarding & connection status const { showOnboarding, setShowOnboarding } = useOnboarding(); + const { showWarmupNotice, setShowWarmupNotice } = useStartupWarmupNotice(); const { showChangelog, setShowChangelog } = useChangelog(); return ( @@ -930,8 +932,13 @@ export default function Dashboard() { /> )} + {/* FIRST-RUN WARMUP NOTICE — shows once after onboarding */} + {!showOnboarding && showWarmupNotice && ( + setShowWarmupNotice(false)} /> + )} + {/* v0.4 CHANGELOG MODAL — shows once per version after onboarding */} - {!showOnboarding && showChangelog && ( + {!showOnboarding && !showWarmupNotice && showChangelog && ( setShowChangelog(false)} /> )} diff --git a/frontend/src/components/HlsVideo.tsx b/frontend/src/components/HlsVideo.tsx index 0007554..1a5c49a 100644 --- a/frontend/src/components/HlsVideo.tsx +++ b/frontend/src/components/HlsVideo.tsx @@ -8,8 +8,11 @@ export interface HlsVideoHandle { get paused(): boolean; } -const HlsVideo = forwardRef void }>( - ({ url, className, onError }, ref) => { +const HlsVideo = forwardRef< + HlsVideoHandle, + { url: string; className?: string; onError?: () => void; onLoaded?: () => void } +>( + ({ url, className, onError, onLoaded }, ref) => { const videoRef = useRef(null); useImperativeHandle(ref, () => ({ @@ -35,6 +38,7 @@ const HlsVideo = forwardRef { if (data.fatal) onError?.(); }); + hls.on(Hls.Events.MANIFEST_PARSED, () => onLoaded?.()); hls.loadSource(url); hls.attachMedia(video); hlsInstance = hls; @@ -47,7 +51,7 @@ const HlsVideo = forwardRef onError?.()} + onCanPlay={() => onLoaded?.()} + onLoadedData={() => onLoaded?.()} + onPlaying={() => onLoaded?.()} className={className} /> ); diff --git a/frontend/src/components/MaplibreViewer.tsx b/frontend/src/components/MaplibreViewer.tsx index b171f35..8aed9bc 100644 --- a/frontend/src/components/MaplibreViewer.tsx +++ b/frontend/src/components/MaplibreViewer.tsx @@ -5962,6 +5962,7 @@ const MaplibreViewer = ({ return ( (null); const hlsRef = useRef(null); + const sources = useMemo(() => { + const seen = new Set(); + return [url, rawUrl] + .map((candidate) => String(candidate || '').trim()) + .filter((candidate) => { + if (!candidate || seen.has(candidate)) return false; + seen.add(candidate); + return true; + }); + }, [rawUrl, url]); + const activeUrl = sources[sourceIndex] || ''; + + useEffect(() => { + setSourceIndex(0); + setMediaError(false); + setMediaLoaded(false); + setPaused(false); + }, [rawUrl, url]); + + useEffect(() => { + setMediaLoaded(false); + }, [activeUrl]); + + const handleMediaFailure = useCallback(() => { + setSourceIndex((idx) => { + const next = idx + 1; + if (next < sources.length) { + setMediaError(false); + return next; + } + setMediaError(true); + return idx; + }); + }, [sources.length]); + + const handleMediaReady = useCallback(() => { + setMediaLoaded(true); + }, []); + + useEffect(() => { + if (sourceIndex !== 0 || sources.length < 2 || mediaLoaded || mediaError) return; + const timeoutMs = mediaType === 'hls' ? 3200 : 1800; + const timer = window.setTimeout(() => { + setSourceIndex((idx) => { + if (idx !== 0 || mediaLoaded) return idx; + return 1; + }); + }, timeoutMs); + return () => window.clearTimeout(timer); + }, [mediaError, mediaLoaded, mediaType, sourceIndex, sources.length]); const togglePlay = useCallback(() => { if (mediaType === 'hls') { @@ -176,17 +230,21 @@ export function CctvFullscreenModal({ overflow: 'hidden', }} > - {url ? ( + {activeUrl ? ( <> {mediaType === 'video' && !mediaError && (