feat: Telegram OSINT map layer, Osiris intel ports, and maritime settings

Add Telegram OSINT with hourly incremental t.me scraping, metro geocoding
separate from news centroids, threat-intercept popup UI with inline media,
and HTML markers above alert boxes so pins stay clickable. Expose GFW_API_TOKEN
in onboarding and Settings Maritime; harden GFW/CCTV/geo fetchers. Port Osiris-
derived recon, SCM, entity graph, malware/cyber feeds, sanctions, and submarine
cable layers with tests and documentation.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/routers/cctv.py

@@ -47,6 +47,8 @@ _CCTV_PROXY_ALLOWED_HOSTS = {
     "www.tripcheck.com",
     "infocar.dgt.es",
     "informo.madrid.es",
+    "webcams2.asfinag.at",
+    "odo.asfinag.at",
     "www.windy.com",
     "imgproxy.windy.com",
     "www.lakecountypassage.com",
@@ -55,6 +57,14 @@ _CCTV_PROXY_ALLOWED_HOSTS = {
     "www.nps.gov",
     "home.lewiscounty.com",
     "www.seattle.gov",
+    "511on.ca",
+    "511.alberta.ca",
+    "fl511.com",
+    "www.fl511.com",
+    "webcams.transport.nsw.gov.au",
+    "www.livetraffic.com",
+    "livetraffic.com",
+    "opendata.ndw.nu",
 }
 
 
@@ -120,7 +130,7 @@ def _cctv_proxy_profile_for_url(target_url: str) -> _CCTVProxyProfile:
         read_timeout = 18.0 if "/snapshots/" in path else 12.0
         return _CCTVProxyProfile(name="gdot-snapshot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, read_timeout), cache_seconds=15,
             headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
-                     "Referer": "http://navigator-c2c.dot.ga.gov/"})
+                     "Referer": "https://navigator-c2c.dot.ga.gov/"})
     if host == "511ga.org":
         return _CCTVProxyProfile(name="gdot-511ga-image", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=15,
             headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
@@ -128,7 +138,7 @@ def _cctv_proxy_profile_for_url(target_url: str) -> _CCTVProxyProfile:
     if host.startswith("vss") and host.endswith("dot.ga.gov"):
         return _CCTVProxyProfile(name="gdot-hls", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 20.0), cache_seconds=10,
             headers={"Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,*/*;q=0.8",
-                     "Referer": "http://navigator-c2c.dot.ga.gov/"})
+                     "Referer": "https://navigator-c2c.dot.ga.gov/"})
     if host in {"gettingaroundillinois.com", "cctv.travelmidwest.com"}:
         return _CCTVProxyProfile(name="illinois-dot", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
             headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"})
@@ -156,10 +166,34 @@ def _cctv_proxy_profile_for_url(target_url: str) -> _CCTVProxyProfile:
         return _CCTVProxyProfile(name="madrid-city", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=30,
             headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
                      "Referer": "https://informo.madrid.es/"})
+    if host in {"webcams2.asfinag.at", "odo.asfinag.at"}:
+        return _CCTVProxyProfile(name="asfinag-austria", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 15.0), cache_seconds=60,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.asfinag.at/"})
     if host in {"www.windy.com", "imgproxy.windy.com"}:
         return _CCTVProxyProfile(name="windy-webcams", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=60,
             headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
                      "Referer": "https://www.windy.com/"})
+    if host == "511on.ca":
+        return _CCTVProxyProfile(name="ontario-511", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 15.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://511on.ca/"})
+    if host == "511.alberta.ca":
+        return _CCTVProxyProfile(name="alberta-511", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 15.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://511.alberta.ca/"})
+    if host in {"fl511.com", "www.fl511.com"}:
+        return _CCTVProxyProfile(name="florida-511", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 15.0), cache_seconds=30,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://fl511.com/"})
+    if host == "webcams.transport.nsw.gov.au":
+        return _CCTVProxyProfile(name="nsw-live-traffic", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=60,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.livetraffic.com/"})
+    if host in {"opendata.ndw.nu", "www.ndw.nu"}:
+        return _CCTVProxyProfile(name="ndw-netherlands", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 12.0), cache_seconds=120,
+            headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
+                     "Referer": "https://www.ndw.nu/"})
     return _CCTVProxyProfile(name="generic-cctv", timeout=(_CCTV_PROXY_CONNECT_TIMEOUT_S, 8.0), cache_seconds=30,
         headers={"Accept": "*/*"})
 

backend/routers/data.py

@@ -502,6 +502,8 @@ async def update_layers(update: LayerUpdate, request: Request):
     old_mesh = is_any_active("sigint_meshtastic")
     old_aprs = is_any_active("sigint_aprs")
     old_viirs = is_any_active("viirs_nightlights")
+    old_datacenters = is_any_active("datacenters")
+    old_fishing = is_any_active("fishing_activity")
     changed = False
     for key, value in update.layers.items():
         if key in active_layers:
@@ -514,6 +516,8 @@ async def update_layers(update: LayerUpdate, request: Request):
     new_mesh = is_any_active("sigint_meshtastic")
     new_aprs = is_any_active("sigint_aprs")
     new_viirs = is_any_active("viirs_nightlights")
+    new_datacenters = is_any_active("datacenters")
+    new_fishing = is_any_active("fishing_activity")
     if old_ships and not new_ships:
         from services.ais_stream import stop_ais_stream
         stop_ais_stream()
@@ -557,6 +561,16 @@ async def update_layers(update: LayerUpdate, request: Request):
     if not old_viirs and new_viirs:
         _queue_viirs_change_refresh()
         logger.info("VIIRS change refresh queued (layer enabled)")
+    if not old_datacenters and new_datacenters:
+        from services.fetchers.infrastructure import fetch_datacenters
+
+        fetch_datacenters()
+        logger.info("Datacenters loaded (layer enabled)")
+    if not old_fishing and new_fishing:
+        from services.fetchers.geo import fetch_fishing_activity
+
+        fetch_fishing_activity()
+        logger.info("Fishing activity refresh queued (layer enabled)")
     return {"status": "ok"}
 
 
@@ -759,6 +773,7 @@ async def live_data_slow(
         "scanners", "weather_alerts", "ukraine_alerts", "air_quality", "volcanoes",
         "fishing_activity", "psk_reporter", "correlations", "uap_sightings", "wastewater",
         "crowdthreat", "threat_level", "trending_markets", "road_corridor_trends",
+        "malware_threats", "cyber_threats", "scm_suppliers", "telegram_osint",
     )
     freshness = get_source_timestamps_snapshot()
     payload = {
@@ -804,6 +819,26 @@ async def live_data_slow(
         )
         if active_layers.get("road_corridor_trends", False)
         else {"updated_at": None, "corridors": []},
+        "malware_threats": (
+            d.get("malware_threats") or {"threats": [], "total": 0}
+        )
+        if active_layers.get("malware_c2", False)
+        else {"threats": [], "total": 0},
+        "cyber_threats": (
+            d.get("cyber_threats") or {"threats": [], "stats": {}}
+        )
+        if active_layers.get("cyber_threats", False)
+        else {"threats": [], "stats": {}},
+        "scm_suppliers": (
+            d.get("scm_suppliers") or {"suppliers": [], "total": 0, "critical_count": 0}
+        )
+        if active_layers.get("scm_suppliers", False)
+        else {"suppliers": [], "total": 0, "critical_count": 0},
+        "telegram_osint": (
+            d.get("telegram_osint") or {"posts": [], "total": 0, "geolocated": 0}
+        )
+        if active_layers.get("telegram_osint", True)
+        else {"posts": [], "total": 0, "geolocated": 0},
         "freshness": freshness,
     }
     # Issue #288: bbox filter heavy/dense layers only when all four bounds

backend/routers/entity_graph.py

@@ -0,0 +1,30 @@
+"""Entity graph expansion (intel layer)."""
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
+
+from auth import require_local_operator
+from limiter import limiter
+from services.osint_intel.resolve import resolve_entity
+
+router = APIRouter()
+
+
+@router.get("/api/entity/expand")
+@limiter.limit("30/minute")
+async def entity_expand(
+    request: Request,
+    _: None = Depends(require_local_operator),
+    type: str = Query(..., min_length=3, max_length=32),
+    id: str = Query(..., min_length=2, max_length=200),
+    registration: str | None = Query(default=None, max_length=32),
+    model: str | None = Query(default=None, max_length=64),
+    icao24: str | None = Query(default=None, max_length=16),
+) -> dict:
+    props = {"label": id, "registration": registration, "model": model, "icao24": icao24}
+    try:
+        return resolve_entity(type, id, props)
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+    except Exception as exc:
+        raise HTTPException(status_code=502, detail="Intelligence layer unavailable") from exc

backend/routers/intel_feeds.py

@@ -0,0 +1,122 @@
+"""Malware, cyber threats, and country risk feeds."""
+from __future__ import annotations
+
+import logging
+from urllib.parse import urlparse
+
+import requests
+from fastapi import APIRouter, HTTPException, Query, Request
+from fastapi.responses import StreamingResponse
+from starlette.background import BackgroundTask
+
+from limiter import limiter
+from services.fetchers._store import get_latest_data_subset_refs
+from services.fetchers.telegram_osint import telegram_media_host_allowed
+from services.intel_feeds.country_risk import build_country_risk_payload
+from services.network_utils import outbound_user_agent
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+@router.get("/api/malware")
+@limiter.limit("60/minute")
+async def malware_feed(request: Request) -> dict:
+    snap = get_latest_data_subset_refs("malware_threats")
+    payload = snap.get("malware_threats")
+    if isinstance(payload, dict) and payload.get("threats") is not None:
+        return payload
+    return {"threats": [], "total": 0, "timestamp": None, "source": "abuse.ch"}
+
+
+@router.get("/api/cyber-threats")
+@limiter.limit("60/minute")
+async def cyber_threats(request: Request) -> dict:
+    snap = get_latest_data_subset_refs("cyber_threats")
+    return snap.get("cyber_threats") or {"threats": [], "stats": {}}
+
+
+@router.get("/api/country-risk")
+@limiter.limit("30/minute")
+async def country_risk(request: Request) -> dict:
+    return build_country_risk_payload()
+
+
+@router.get("/api/telegram-feed")
+@limiter.limit("30/minute")
+async def telegram_feed(request: Request) -> dict:
+    snap = get_latest_data_subset_refs("telegram_osint")
+    payload = snap.get("telegram_osint")
+    if isinstance(payload, dict) and payload.get("posts") is not None:
+        return payload
+    return {"posts": [], "total": 0, "geolocated": 0, "timestamp": None}
+
+
+def _infer_telegram_media_type(target_url: str, content_type: str) -> str:
+    clean_type = str(content_type or "").split(";", 1)[0].strip().lower()
+    if clean_type and clean_type not in {"application/octet-stream", "binary/octet-stream"}:
+        return content_type
+    path = str(urlparse(target_url).path or "").lower()
+    if path.endswith((".jpg", ".jpeg")):
+        return "image/jpeg"
+    if path.endswith(".png"):
+        return "image/png"
+    if path.endswith(".webp"):
+        return "image/webp"
+    if path.endswith(".gif"):
+        return "image/gif"
+    if path.endswith(".mp4"):
+        return "video/mp4"
+    if path.endswith(".webm"):
+        return "video/webm"
+    return content_type or "application/octet-stream"
+
+
+@router.get("/api/telegram/media")
+@limiter.limit("60/minute")
+async def telegram_media_proxy(request: Request, url: str = Query(...)) -> StreamingResponse:
+    """Stream Telegram CDN media for in-app playback (host allowlist only)."""
+    parsed = urlparse(url)
+    if parsed.scheme not in ("http", "https"):
+        raise HTTPException(status_code=400, detail="Invalid scheme")
+    if not telegram_media_host_allowed(parsed.hostname):
+        raise HTTPException(status_code=403, detail="Host not allowed")
+
+    headers = {
+        "User-Agent": (
+            f"Mozilla/5.0 (compatible; {outbound_user_agent('telegram-media')}) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+        ),
+        "Accept": "*/*",
+    }
+    if range_header := request.headers.get("range"):
+        headers["Range"] = range_header
+
+    try:
+        resp = requests.get(url, stream=True, timeout=(3, 45), headers=headers)
+    except requests.RequestException as exc:
+        logger.warning("Telegram media upstream failure %s: %s", url, exc)
+        raise HTTPException(status_code=502, detail="Upstream fetch failed") from exc
+
+    if resp.status_code >= 400:
+        resp.close()
+        raise HTTPException(status_code=int(resp.status_code), detail=f"Upstream returned {resp.status_code}")
+
+    media_type = _infer_telegram_media_type(url, resp.headers.get("Content-Type", "application/octet-stream"))
+    response_headers = {
+        "Cache-Control": "private, max-age=300",
+        "Accept-Ranges": resp.headers.get("Accept-Ranges", "bytes"),
+    }
+    if content_length := resp.headers.get("Content-Length"):
+        response_headers["Content-Length"] = content_length
+    if content_range := resp.headers.get("Content-Range"):
+        response_headers["Content-Range"] = content_range
+
+    return StreamingResponse(
+        resp.iter_content(chunk_size=65536),
+        status_code=resp.status_code,
+        media_type=media_type,
+        headers=response_headers,
+        background=BackgroundTask(resp.close),
+    )

backend/routers/osint.py

@@ -0,0 +1,151 @@
+"""Operator OSINT recon routes (server-side proxies, SSRF guarded)."""
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
+from pydantic import BaseModel, Field
+
+from auth import require_local_operator
+from limiter import limiter
+from services.osint import lookups
+
+router = APIRouter(dependencies=[Depends(require_local_operator)])
+
+_ALLOWED_SCHEMAS = {
+    "Person",
+    "Organization",
+    "Company",
+    "Vessel",
+    "Airplane",
+    "LegalEntity",
+}
+
+
+class SweepScanRequest(BaseModel):
+    ip: str = Field(min_length=7, max_length=45)
+    cidr: int = Field(default=24, ge=24, le=32)
+
+
+def _bad_request(exc: ValueError) -> HTTPException:
+    return HTTPException(status_code=400, detail=str(exc))
+
+
+@router.get("/api/osint/ip")
+@limiter.limit("20/minute")
+async def osint_ip(request: Request, ip: str = Query(..., min_length=7, max_length=45)) -> dict:
+    try:
+        return lookups.lookup_ip(ip)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/dns")
+@limiter.limit("20/minute")
+async def osint_dns(request: Request, domain: str = Query(..., min_length=4, max_length=253)) -> dict:
+    try:
+        return lookups.lookup_dns(domain)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/whois")
+@limiter.limit("20/minute")
+async def osint_whois(request: Request, domain: str = Query(..., min_length=4, max_length=253)) -> dict:
+    try:
+        return lookups.lookup_whois(domain)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/certs")
+@limiter.limit("20/minute")
+async def osint_certs(request: Request, domain: str = Query(..., min_length=4, max_length=253)) -> dict:
+    try:
+        return lookups.lookup_certs(domain)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/threats")
+@limiter.limit("20/minute")
+async def osint_threats(request: Request, query: str | None = Query(default=None, max_length=253)) -> dict:
+    return lookups.lookup_threats(query)
+
+
+@router.get("/api/osint/bgp")
+@limiter.limit("20/minute")
+async def osint_bgp(request: Request, query: str = Query(..., min_length=2, max_length=64)) -> dict:
+    try:
+        return lookups.lookup_bgp(query)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/sanctions")
+@limiter.limit("20/minute")
+async def osint_sanctions(
+    request: Request,
+    query: str = Query(..., min_length=4, max_length=200),
+    schema: str | None = Query(default=None),
+    limit: int = Query(default=25, ge=1, le=100),
+) -> dict:
+    if schema and schema not in _ALLOWED_SCHEMAS:
+        raise HTTPException(status_code=400, detail=f"Invalid schema. Allowed: {', '.join(sorted(_ALLOWED_SCHEMAS))}")
+    return lookups.lookup_sanctions(query, schema=schema, limit=limit)
+
+
+@router.get("/api/osint/cve")
+@limiter.limit("30/minute")
+async def osint_cve(request: Request, cve: str = Query(..., min_length=10, max_length=32)) -> dict:
+    try:
+        return lookups.lookup_cve(cve)
+    except ValueError as exc:
+        raise HTTPException(status_code=404 if "not found" in str(exc).lower() else 400, detail=str(exc)) from exc
+
+
+@router.get("/api/osint/mac")
+@limiter.limit("20/minute")
+async def osint_mac(request: Request, mac: str = Query(..., min_length=5, max_length=32)) -> dict:
+    return lookups.lookup_mac(mac)
+
+
+@router.get("/api/osint/github")
+@limiter.limit("20/minute")
+async def osint_github(request: Request, username: str = Query(..., min_length=1, max_length=64)) -> dict:
+    try:
+        return lookups.lookup_github(username)
+    except ValueError as exc:
+        raise HTTPException(status_code=404, detail=str(exc)) from exc
+
+
+@router.get("/api/osint/leaks")
+@limiter.limit("10/minute")
+async def osint_leaks(request: Request, email: str = Query(..., min_length=5, max_length=254)) -> dict:
+    try:
+        return lookups.lookup_leaks(email)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.get("/api/osint/sweep")
+@limiter.limit("5/minute")
+async def osint_sweep_init(
+    request: Request,
+    ip: str = Query(..., min_length=7, max_length=45),
+    cidr: int = Query(default=24, ge=24, le=32),
+) -> dict:
+    try:
+        return lookups.sweep_init(ip, cidr)
+    except ValueError as exc:
+        raise _bad_request(exc) from exc
+
+
+@router.post("/api/osint/sweep/scan")
+@limiter.limit("3/minute")
+async def osint_sweep_scan(request: Request, payload: SweepScanRequest) -> dict:
+    try:
+        subnet = lookups.subnet_start_for(payload.ip, payload.cidr)
+        scan = lookups.sweep_scan(subnet, payload.cidr)
+        init = lookups.sweep_init(payload.ip, payload.cidr)
+        return {**init, **scan, "subnet": f"{subnet}/{payload.cidr}"}
+    except ValueError as exc:
+        raise _bad_request(exc) from exc

backend/routers/scm.py

@@ -0,0 +1,16 @@
+"""Supply-chain risk overlay."""
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, Request
+
+from auth import require_local_operator
+from limiter import limiter
+from services.scm.suppliers import build_scm_payload
+
+router = APIRouter()
+
+
+@router.get("/api/scm-suppliers")
+@limiter.limit("30/minute")
+async def scm_suppliers(request: Request, _: None = Depends(require_local_operator)) -> dict:
+    return build_scm_payload()