Add Airframes ACARS datalink on plane dossiers and Meshtastic planet scan.

Bulk-ingest Airframes messages on a rate-limited staggered queue with instant cache lookups and priority per-aircraft refresh when opening a dossier; add Meshtastic manual SCAN PLANET control in the SIGINT panel.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/services/api_settings.py

@@ -6,6 +6,7 @@ Keys are stored in the backend .env file and loaded via python-dotenv.
 import os
 import re
 import tempfile
+import threading
 from pathlib import Path
 
 # Path to the backend .env file
@@ -87,6 +88,15 @@ API_REGISTRY = [
         "url": "https://firms.modaps.eosdis.nasa.gov/api/area/",
         "required": False,
     },
+    {
+        "id": "airframes_api_key",
+        "env_key": "AIRFRAMES_API_KEY",
+        "name": "Airframes.io — API Key",
+        "description": "ACARS/VDL datalink for plane dossiers. ShadowBroker bulk-ingests the global Airframes firehose (up to 100 messages per API call, one call every 2s, refill every 15 minutes) and indexes by tail/ICAO. Opening a dossier with no cache queues a single-plane lookup. Get a key at app.airframes.io → Dashboard → API Key.",
+        "category": "Aviation",
+        "url": "https://app.airframes.io/user/dashboard",
+        "required": False,
+    },
     {
         "id": "celestrak",
         "env_key": None,
@@ -366,6 +376,17 @@ def save_api_keys(updates: dict[str, str]) -> dict:
             flights.opensky_client.expires_at = 0
         except Exception:
             pass
+    if "AIRFRAMES_API_KEY" in clean:
+        try:
+            from services.fetchers.airframes import sync_airframes_messages
+
+            threading.Thread(
+                target=lambda: sync_airframes_messages(force=True),
+                daemon=True,
+                name="airframes-initial-sync",
+            ).start()
+        except Exception:
+            pass
 
     try:
         from services.config import get_settings

backend/services/data_fetcher.py

@@ -77,6 +77,7 @@ from services.fetchers.infrastructure import (  # noqa: F401
     fetch_psk_reporter,
 )
 from services.fetchers.road_corridor_sat import fetch_road_corridor_trends  # noqa: F401
+from services.fetchers.airframes import sync_airframes_messages  # noqa: F401
 from services.fetchers.geo import (  # noqa: F401
     fetch_ships,
     fetch_airports,
@@ -1263,6 +1264,17 @@ def start_scheduler():
         next_run_time=datetime.utcnow() + timedelta(minutes=5),  # first snapshot 5m after startup
     )
 
+    _airframes_interval_m = max(5, int(os.environ.get("AIRFRAMES_SYNC_INTERVAL_MINUTES", "15")))
+    _scheduler.add_job(
+        lambda: _run_task_with_health(sync_airframes_messages, "sync_airframes_messages"),
+        "interval",
+        minutes=_airframes_interval_m,
+        id="airframes_datalink",
+        max_instances=1,
+        misfire_grace_time=120,
+        next_run_time=datetime.utcnow() + timedelta(seconds=90),
+    )
+
     _scheduler.start()
     logger.info("Scheduler started.")
 

backend/services/fetchers/airframes.py

@@ -0,0 +1,605 @@
+"""Airframes.io ACARS/VDL datalink ingest — staggered queue cache for plane dossiers."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import re
+import threading
+import time
+from collections import deque
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any
+
+import requests
+
+logger = logging.getLogger("services.airframes")
+
+API_BASE = os.environ.get("AIRFRAMES_API_BASE", "https://api.airframes.io/v1").rstrip("/")
+SYNC_INTERVAL_MINUTES = max(5, int(os.environ.get("AIRFRAMES_SYNC_INTERVAL_MINUTES", "15")))
+MAX_BULK_PAGES_PER_CYCLE = max(1, int(os.environ.get("AIRFRAMES_MAX_PAGES_PER_SYNC", "28")))
+MESSAGES_PER_AIRCRAFT = max(5, int(os.environ.get("AIRFRAMES_MESSAGES_PER_AIRCRAFT", "40")))
+RETENTION_HOURS = max(6, int(os.environ.get("AIRFRAMES_RETENTION_HOURS", "48")))
+# 2s between calls => 30/min, safely under Airframes 60/min cap.
+REQUEST_PAUSE_S = float(os.environ.get("AIRFRAMES_REQUEST_PAUSE_S", "2.0"))
+PRIORITY_LOOKBACK_HOURS = max(6, int(os.environ.get("AIRFRAMES_PRIORITY_LOOKBACK_HOURS", "48")))
+FETCH_TIMEOUT_S = max(5, int(os.environ.get("AIRFRAMES_FETCH_TIMEOUT_S", "20")))
+
+_DATA_DIR = Path(os.environ.get("SB_DATA_DIR", str(Path(__file__).resolve().parents[2] / "data")))
+if not _DATA_DIR.is_absolute():
+    _DATA_DIR = Path(__file__).resolve().parents[2] / _DATA_DIR
+_CACHE_PATH = _DATA_DIR / "airframes_datalink_cache.json"
+
+_lock = threading.Lock()
+_queue_lock = threading.Lock()
+_worker_guard = threading.Lock()
+_queue: deque[dict[str, Any]] = deque()
+_queued_aircraft_keys: set[str] = set()
+_bulk_cursor: dict[str, Any] = {"since_iso": "", "before_id": None, "pages": 0}
+_worker_started = False
+_cache_loaded = False
+_save_timer: threading.Timer | None = None
+_save_timer_lock = threading.Lock()
+_api_key_known_configured: bool | None = None
+_cache: dict[str, Any] = {
+    "last_sync_at": None,
+    "last_success_at": None,
+    "last_error": None,
+    "pages_fetched": 0,
+    "messages_ingested": 0,
+    "bulk_pages_this_cycle": 0,
+    "ticks_processed": 0,
+    "by_icao": {},
+    "by_tail": {},
+    "by_callsign": {},
+}
+
+
+def _utc_now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _iso(dt: datetime) -> str:
+    return dt.astimezone(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
+
+
+def _parse_ts(value: str | None) -> datetime | None:
+    if not value:
+        return None
+    try:
+        cleaned = value.replace("Z", "+00:00")
+        return datetime.fromisoformat(cleaned).astimezone(timezone.utc)
+    except ValueError:
+        return None
+
+
+def api_key_configured() -> bool:
+    global _api_key_known_configured
+    if os.environ.get("AIRFRAMES_API_KEY", "").strip():
+        _api_key_known_configured = True
+        return True
+    if _api_key_known_configured is False:
+        return False
+    from services.api_settings import load_persisted_api_keys_into_environ
+
+    load_persisted_api_keys_into_environ()
+    _api_key_known_configured = bool(os.environ.get("AIRFRAMES_API_KEY", "").strip())
+    return _api_key_known_configured
+
+
+def _norm_hex(value: str | None) -> str:
+    return (value or "").strip().lower()
+
+
+def _norm_tail(value: str | None) -> str:
+    return re.sub(r"[^A-Z0-9]", "", (value or "").strip().upper())
+
+
+def _norm_callsign(value: str | None) -> str:
+    return re.sub(r"\s+", "", (value or "").strip().upper())
+
+
+def _aircraft_queue_key(entry: dict[str, str]) -> str:
+    return f"{entry.get('icao24', '')}|{entry.get('registration', '')}|{entry.get('callsign', '')}"
+
+
+def _tail_lookup_keys(value: str | None) -> list[str]:
+    tail = _norm_tail(value)
+    if not tail:
+        return []
+    keys = [tail]
+    raw = (value or "").strip().upper()
+    if raw and raw not in keys:
+        keys.append(raw)
+    return keys
+
+
+def _load_cache_if_cold() -> None:
+    global _cache, _cache_loaded
+    if _cache_loaded:
+        return
+    loaded: dict[str, Any] | None = None
+    if _CACHE_PATH.exists():
+        try:
+            with _CACHE_PATH.open(encoding="utf-8") as handle:
+                parsed = json.load(handle)
+            if isinstance(parsed, dict):
+                loaded = parsed
+        except (OSError, json.JSONDecodeError, ValueError) as exc:
+            logger.warning("Failed to load Airframes cache: %s", exc)
+    with _lock:
+        if _cache_loaded:
+            return
+        if loaded:
+            _cache.update(loaded)
+            _cache.setdefault("by_callsign", {})
+        _cache_loaded = True
+
+
+def _persist_cache_now() -> None:
+    with _lock:
+        snapshot = json.dumps(_cache, indent=2, ensure_ascii=False) + "\n"
+    _DATA_DIR.mkdir(parents=True, exist_ok=True)
+    tmp = _CACHE_PATH.with_suffix(".tmp")
+    tmp.write_text(snapshot, encoding="utf-8")
+    tmp.replace(_CACHE_PATH)
+
+
+def _schedule_cache_persist() -> None:
+    global _save_timer
+
+    def _flush() -> None:
+        global _save_timer
+        try:
+            _persist_cache_now()
+        except OSError as exc:
+            logger.warning("Failed to save Airframes cache: %s", exc)
+        finally:
+            with _save_timer_lock:
+                _save_timer = None
+
+    with _save_timer_lock:
+        if _save_timer is not None:
+            _save_timer.cancel()
+        _save_timer = threading.Timer(0.75, _flush)
+        _save_timer.daemon = True
+        _save_timer.start()
+
+
+def _save_cache() -> None:
+    _schedule_cache_persist()
+
+
+def _compact_message(raw: dict[str, Any]) -> dict[str, Any] | None:
+    text = (raw.get("text") or raw.get("data") or "").strip()
+    if not text:
+        return None
+    msg_id = raw.get("id")
+    if msg_id is None:
+        return None
+    return {
+        "id": int(msg_id),
+        "timestamp": raw.get("timestamp") or raw.get("createdAt") or "",
+        "label": str(raw.get("label") or "").strip(),
+        "text": text[:500],
+        "source_type": str(raw.get("sourceType") or raw.get("source") or "").strip(),
+        "tail": _norm_tail(raw.get("tail")),
+        "flight_number": _norm_callsign(raw.get("flightNumber")),
+        "from_hex": _norm_hex(raw.get("fromHex")),
+        "to_hex": _norm_hex(raw.get("toHex")),
+    }
+
+
+def _bucket_key(store: dict[str, list], key: str, message: dict[str, Any]) -> None:
+    if not key:
+        return
+    bucket = store.setdefault(key, [])
+    if any(existing.get("id") == message["id"] for existing in bucket):
+        return
+    bucket.append(message)
+    bucket.sort(key=lambda item: item.get("timestamp") or "", reverse=True)
+    del bucket[MESSAGES_PER_AIRCRAFT:]
+
+
+def _index_message(compact: dict[str, Any]) -> None:
+    for hex_code in (compact.get("from_hex"), compact.get("to_hex")):
+        if hex_code:
+            _bucket_key(_cache["by_icao"], hex_code, compact)
+    for tail_key in _tail_lookup_keys(compact.get("tail")):
+        _bucket_key(_cache["by_tail"], tail_key, compact)
+    callsign = compact.get("flight_number")
+    if callsign:
+        _bucket_key(_cache["by_callsign"], callsign, compact)
+
+
+def _prune_store(store: dict[str, list]) -> None:
+    cutoff = _utc_now() - timedelta(hours=RETENTION_HOURS)
+    for key in list(store.keys()):
+        kept = []
+        for message in store.get(key, []):
+            ts = _parse_ts(message.get("timestamp"))
+            if ts is None or ts >= cutoff:
+                kept.append(message)
+        if kept:
+            store[key] = kept[:MESSAGES_PER_AIRCRAFT]
+        else:
+            del store[key]
+
+
+def _ingest_message(message: dict[str, Any]) -> bool:
+    compact = _compact_message(message)
+    if not compact:
+        return False
+    _index_message(compact)
+    return True
+
+
+def _ingest_messages_batch(raw_messages: list[dict[str, Any]]) -> int:
+    if not raw_messages:
+        return 0
+    ingested = 0
+    with _lock:
+        _cache.setdefault("by_callsign", {})
+        for raw in raw_messages:
+            if _ingest_message(raw):
+                ingested += 1
+        if ingested:
+            _cache["messages_ingested"] = int(_cache.get("messages_ingested", 0)) + ingested
+            _cache["last_success_at"] = _iso(_utc_now())
+            _save_cache()
+    return ingested
+
+
+def _fetch_messages(*, api_key: str, params: dict[str, Any]) -> list[dict[str, Any]]:
+    response = requests.get(
+        f"{API_BASE}/messages",
+        headers={"Authorization": f"Bearer {api_key}"},
+        params=params,
+        timeout=FETCH_TIMEOUT_S,
+    )
+    if response.status_code == 404:
+        logger.debug("Airframes messages 404 for params=%s", params)
+        return []
+    if response.status_code == 429:
+        retry_after = int(response.headers.get("Retry-After", "60"))
+        raise RuntimeError(f"rate_limited:{retry_after}")
+    response.raise_for_status()
+    payload = response.json()
+    if not isinstance(payload, list):
+        return []
+    return [item for item in payload if isinstance(item, dict)]
+
+
+def _refill_queue(*, since_iso: str, force: bool = False) -> int:
+    """Queue bulk global ingest only — each bulk call returns up to 100 messages
+    across many aircraft. Per-plane calls happen only on dossier cache miss."""
+    global _bulk_cursor, _queued_aircraft_keys
+
+    with _queue_lock:
+        if force:
+            _queue.clear()
+            _queued_aircraft_keys = set()
+            _bulk_cursor = {"since_iso": since_iso, "before_id": None, "pages": 0}
+
+        added = 0
+        has_bulk = any(item.get("type") == "bulk" for item in _queue)
+        if not has_bulk:
+            _bulk_cursor["since_iso"] = since_iso
+            _queue.append({"type": "bulk", "since_iso": since_iso, "before_id": None})
+            added += 1
+
+    with _lock:
+        _cache["bulk_pages_this_cycle"] = 0
+        _save_cache()
+
+    return added
+
+
+def _prioritize_aircraft_scan(entry: dict[str, str]) -> bool:
+    """Jump this aircraft to the front of the queue — next API tick (~2s)."""
+    key = _aircraft_queue_key(entry)
+    if key.replace("|", "").strip() == "":
+        return False
+
+    item = {"type": "aircraft", **entry}
+    with _queue_lock:
+        kept: deque[dict[str, Any]] = deque()
+        for queued in _queue:
+            if queued.get("type") == "aircraft" and _aircraft_queue_key(queued) == key:
+                continue
+            kept.append(queued)
+        _queue.clear()
+        _queue.extend(kept)
+        _queued_aircraft_keys.discard(key)
+        _queued_aircraft_keys.add(key)
+        _queue.appendleft(item)
+    return True
+
+
+def _enqueue_bulk_page(*, since_iso: str, before_id: int | None = None) -> None:
+    with _queue_lock:
+        _queue.append({"type": "bulk", "since_iso": since_iso, "before_id": before_id})
+
+
+def _process_aircraft_item(api_key: str, entry: dict[str, str]) -> int:
+    since_iso = _iso(_utc_now() - timedelta(hours=PRIORITY_LOOKBACK_HOURS))
+    params: dict[str, Any] = {
+        "since": since_iso,
+        "limit": 100,
+        "exclude_errors": "1",
+    }
+    if entry.get("icao24"):
+        params["icao"] = entry["icao24"]
+    elif entry.get("registration"):
+        params["text"] = entry["registration"]
+    elif entry.get("callsign"):
+        params["text"] = entry["callsign"]
+    else:
+        return 0
+
+    try:
+        batch = _fetch_messages(api_key=api_key, params=params)
+    except Exception as exc:
+        logger.debug("Airframes aircraft fetch failed for %s: %s", entry, exc)
+        with _lock:
+            _cache["last_error"] = str(exc)[:240]
+            _save_cache()
+        return 0
+
+    return _ingest_messages_batch(batch)
+
+
+def _process_bulk_item(api_key: str, item: dict[str, Any]) -> int:
+    global _bulk_cursor
+
+    params: dict[str, Any] = {
+        "since": item["since_iso"],
+        "limit": 100,
+        "exclude_errors": "1",
+    }
+    before_id = item.get("before_id")
+    if before_id is not None:
+        params["before_id"] = before_id
+
+    try:
+        batch = _fetch_messages(api_key=api_key, params=params)
+    except Exception as exc:
+        logger.debug("Airframes bulk fetch failed: %s", exc)
+        with _lock:
+            _cache["last_error"] = str(exc)[:240]
+            _save_cache()
+        return 0
+
+    ingested = _ingest_messages_batch(batch)
+
+    with _lock:
+        _bulk_cursor["pages"] = int(_bulk_cursor.get("pages", 0)) + 1
+        _cache["pages_fetched"] = int(_cache.get("pages_fetched", 0)) + 1
+        _save_cache()
+
+    if (
+        batch
+        and len(batch) >= 100
+        and _bulk_cursor.get("pages", 0) < MAX_BULK_PAGES_PER_CYCLE
+    ):
+        ids = [int(row["id"]) for row in batch if row.get("id") is not None]
+        if ids:
+            next_before = min(ids)
+            if before_id is None or next_before < before_id:
+                _enqueue_bulk_page(since_iso=item["since_iso"], before_id=next_before)
+
+    return ingested
+
+
+def _process_one_staggered_tick() -> int:
+    """Process exactly one queued Airframes API call. Used by the background worker."""
+    if not api_key_configured():
+        return 0
+
+    api_key = os.environ.get("AIRFRAMES_API_KEY", "").strip()
+    with _queue_lock:
+        if not _queue:
+            return 0
+        item = _queue.popleft()
+
+    if item.get("type") == "aircraft":
+        key = _aircraft_queue_key(item)
+        with _queue_lock:
+            _queued_aircraft_keys.discard(key)
+        ingested = _process_aircraft_item(api_key, item)
+    elif item.get("type") == "bulk":
+        ingested = _process_bulk_item(api_key, item)
+    else:
+        ingested = 0
+
+    with _lock:
+        _cache["ticks_processed"] = int(_cache.get("ticks_processed", 0)) + 1
+        if int(_cache.get("ticks_processed", 0)) % 25 == 0:
+            for store_key in ("by_icao", "by_tail", "by_callsign"):
+                _prune_store(_cache[store_key])
+            _save_cache()
+
+    return ingested
+
+
+def _stagger_worker_loop() -> None:
+    while True:
+        time.sleep(REQUEST_PAUSE_S)
+        try:
+            _process_one_staggered_tick()
+        except Exception as exc:
+            logger.error("Airframes stagger worker tick failed: %s", exc)
+
+
+def _ensure_stagger_worker() -> None:
+    global _worker_started
+    if _worker_started:
+        return
+    with _worker_guard:
+        if _worker_started:
+            return
+        _worker_started = True
+        threading.Thread(
+            target=_stagger_worker_loop,
+            daemon=True,
+            name="airframes-stagger",
+        ).start()
+        logger.info(
+            "Airframes stagger worker started (bulk ingest: 1 call / %.1fs, up to %s msgs/call, refill every %sm)",
+            REQUEST_PAUSE_S,
+            100,
+            SYNC_INTERVAL_MINUTES,
+        )
+
+
+def sync_airframes_messages(*, force: bool = False) -> dict[str, Any]:
+    """Queue staggered Airframes fetches — one API call every REQUEST_PAUSE_S."""
+    if not api_key_configured():
+        return {"ok": False, "skipped": True, "reason": "AIRFRAMES_API_KEY not configured"}
+
+    started = _utc_now()
+    _load_cache_if_cold()
+
+    with _lock:
+        _cache.setdefault("by_callsign", {})
+        last_sync_at = _parse_ts(_cache.get("last_sync_at"))
+        if (
+            not force
+            and last_sync_at is not None
+            and started - last_sync_at < timedelta(minutes=SYNC_INTERVAL_MINUTES - 1)
+        ):
+            return {"ok": True, "skipped": True, "reason": "sync_interval_not_elapsed"}
+
+        if _cache.get("last_success_at"):
+            since_dt = _parse_ts(_cache.get("last_success_at")) or (
+                started - timedelta(minutes=SYNC_INTERVAL_MINUTES)
+            )
+            since_dt -= timedelta(minutes=2)
+        else:
+            since_dt = started - timedelta(hours=PRIORITY_LOOKBACK_HOURS)
+        since_iso = _iso(since_dt)
+        _cache["last_sync_at"] = _iso(started)
+        _cache["last_error"] = None
+        _save_cache()
+
+    queued = _refill_queue(since_iso=since_iso, force=force)
+    _ensure_stagger_worker()
+
+    with _queue_lock:
+        queue_depth = len(_queue)
+
+    logger.info(
+        "Airframes cycle queued: added=%s depth=%s interval=%.1fs",
+        queued,
+        queue_depth,
+        REQUEST_PAUSE_S,
+    )
+    return {
+        "ok": True,
+        "queued": queued,
+        "queue_depth": queue_depth,
+        "request_interval_s": REQUEST_PAUSE_S,
+        "sync_interval_minutes": SYNC_INTERVAL_MINUTES,
+    }
+
+
+def _lookup_from_cache(
+    *,
+    hex_key: str,
+    tail_keys: list[str],
+    callsign_key: str,
+) -> tuple[list[dict[str, Any]], str | None]:
+    _load_cache_if_cold()
+    with _lock:
+        _cache.setdefault("by_callsign", {})
+        merged: dict[int, dict[str, Any]] = {}
+        if hex_key:
+            for message in _cache.get("by_icao", {}).get(hex_key, []):
+                merged[message["id"]] = message
+        for tail_key in tail_keys:
+            for message in _cache.get("by_tail", {}).get(tail_key, []):
+                merged[message["id"]] = message
+        if callsign_key:
+            for message in _cache.get("by_callsign", {}).get(callsign_key, []):
+                merged[message["id"]] = message
+        last_success_at = _cache.get("last_success_at")
+
+    messages = sorted(merged.values(), key=lambda item: item.get("timestamp") or "", reverse=True)
+    return messages[:MESSAGES_PER_AIRCRAFT], last_success_at
+
+
+def get_datalink_status() -> dict[str, Any]:
+    configured = api_key_configured()
+    _load_cache_if_cold()
+    with _queue_lock:
+        queue_depth = len(_queue)
+    with _lock:
+        return {
+            "configured": configured,
+            "sync_interval_minutes": SYNC_INTERVAL_MINUTES,
+            "request_interval_s": REQUEST_PAUSE_S,
+            "last_sync_at": _cache.get("last_sync_at"),
+            "last_success_at": _cache.get("last_success_at"),
+            "last_error": _cache.get("last_error"),
+            "pages_fetched": _cache.get("pages_fetched", 0),
+            "messages_ingested": _cache.get("messages_ingested", 0),
+            "bulk_pages_this_cycle": int(_bulk_cursor.get("pages", 0)),
+            "bulk_pages_per_cycle": MAX_BULK_PAGES_PER_CYCLE,
+            "messages_per_bulk_call": 100,
+            "queue_depth": queue_depth,
+            "ticks_processed": _cache.get("ticks_processed", 0),
+            "icao_keys": len(_cache.get("by_icao", {})),
+            "tail_keys": len(_cache.get("by_tail", {})),
+            "callsign_keys": len(_cache.get("by_callsign", {})),
+        }
+
+
+def lookup_datalink_messages(
+    *,
+    icao24: str = "",
+    registration: str = "",
+    callsign: str = "",
+    allow_live: bool = False,
+) -> dict[str, Any]:
+    configured = bool(os.environ.get("AIRFRAMES_API_KEY", "").strip()) or api_key_configured()
+    if not configured:
+        return {
+            "configured": False,
+            "messages": [],
+            "hint": "Add AIRFRAMES_API_KEY in Settings → API Keys to enable ACARS datalink.",
+        }
+
+    hex_key = _norm_hex(icao24)
+    tail_keys = _tail_lookup_keys(registration)
+    callsign_key = _norm_callsign(callsign)
+
+    messages, last_success_at = _lookup_from_cache(
+        hex_key=hex_key,
+        tail_keys=tail_keys,
+        callsign_key=callsign_key,
+    )
+
+    queued_refresh = False
+    if hex_key or tail_keys or callsign_key:
+        queued_refresh = _prioritize_aircraft_scan(
+            {
+                "icao24": hex_key,
+                "registration": _norm_tail(registration),
+                "callsign": callsign_key,
+            }
+        )
+        if queued_refresh:
+            _ensure_stagger_worker()
+
+    return {
+        "configured": True,
+        "messages": messages,
+        "last_success_at": last_success_at,
+        "queued_refresh": queued_refresh,
+        "priority_scan": queued_refresh,
+    }
+
+
+_load_cache_if_cold()

backend/services/fetchers/meshtastic_map.py

@@ -12,9 +12,11 @@ Polling interval deliberately kept low (4h) to be respectful to the service.
 
 import json
 import logging
+import threading
 import time
 from datetime import datetime, timezone, timedelta
 from pathlib import Path
+from typing import Any
 
 import requests
 
@@ -30,9 +32,6 @@ _MAX_AGE_HOURS = 24  # discard nodes not seen within this window
 # one-person hobby service, so we prefer stale data over hammering it.
 _CACHE_TRUST_HOURS = 20
 
-# Track when we last fetched so the frontend can show staleness
-_last_fetch_ts: float = 0.0
-
 
 def _parse_node(node: dict) -> dict | None:
     """Convert an API node into a slim signal-like dict."""
@@ -132,7 +131,43 @@ def _save_cache(nodes: list[dict], fetch_ts: float):
         logger.warning(f"Failed to save meshtastic cache: {e}")
 
 
-def fetch_meshtastic_nodes():
+# Track when we last fetched so the frontend can show staleness
+_last_fetch_ts: float = 0.0
+_scan_lock = threading.Lock()
+_scan_in_progress = False
+
+
+def get_meshtastic_map_status() -> dict[str, Any]:
+    from services.fetchers._store import get_latest_data_subset_refs
+
+    snap = get_latest_data_subset_refs("meshtastic_map_nodes", "meshtastic_map_fetched_at")
+    nodes = snap.get("meshtastic_map_nodes") or []
+    fetched_at = snap.get("meshtastic_map_fetched_at")
+    return {
+        "node_count": len(nodes) if isinstance(nodes, list) else 0,
+        "fetched_at": fetched_at,
+        "scan_in_progress": _scan_in_progress,
+    }
+
+
+def start_meshtastic_planet_scan() -> dict[str, Any]:
+    if not _scan_lock.acquire(blocking=False):
+        return {"ok": False, "status": "scan already in progress"}
+
+    def _run() -> None:
+        global _scan_in_progress
+        try:
+            _scan_in_progress = True
+            fetch_meshtastic_nodes(force=True)
+        finally:
+            _scan_in_progress = False
+            _scan_lock.release()
+
+    threading.Thread(target=_run, daemon=True, name="meshtastic-planet-scan").start()
+    return {"ok": True, "status": "scanning"}
+
+
+def fetch_meshtastic_nodes(*, force: bool = False):
     """Fetch global Meshtastic node positions from Liam Cottle's map API.
 
     Stores processed nodes in latest_data["meshtastic_map_nodes"].
@@ -140,39 +175,40 @@ def fetch_meshtastic_nodes():
     """
     from services.fetchers._store import is_any_active
 
-    if not is_any_active("sigint_meshtastic"):
+    if not force and not is_any_active("sigint_meshtastic"):
         return
     global _last_fetch_ts
 
     # Trust a recent cache on disk — avoids hammering the upstream HTTP API
     # when every install polls on roughly the same cadence.
-    try:
-        if _CACHE_FILE.exists():
-            mtime = _CACHE_FILE.stat().st_mtime
-            if time.time() - mtime < _CACHE_TRUST_HOURS * 3600:
-                # If memory is empty (cold start), hydrate from cache and skip fetch.
-                with _data_lock:
-                    has_memory = bool(latest_data.get("meshtastic_map_nodes"))
-                if not has_memory:
-                    cached = _load_cache()
-                    if cached:
-                        with _data_lock:
-                            latest_data["meshtastic_map_nodes"] = cached
-                            latest_data["meshtastic_map_fetched_at"] = mtime
-                        _mark_fresh("meshtastic_map")
+    if not force:
+        try:
+            if _CACHE_FILE.exists():
+                mtime = _CACHE_FILE.stat().st_mtime
+                if time.time() - mtime < _CACHE_TRUST_HOURS * 3600:
+                    # If memory is empty (cold start), hydrate from cache and skip fetch.
+                    with _data_lock:
+                        has_memory = bool(latest_data.get("meshtastic_map_nodes"))
+                    if not has_memory:
+                        cached = _load_cache()
+                        if cached:
+                            with _data_lock:
+                                latest_data["meshtastic_map_nodes"] = cached
+                                latest_data["meshtastic_map_fetched_at"] = mtime
+                            _mark_fresh("meshtastic_map")
+                            logger.info(
+                                "Meshtastic map: cache fresh (<%.0fh), skipping network fetch",
+                                _CACHE_TRUST_HOURS,
+                            )
+                            return
+                    else:
                         logger.info(
                             "Meshtastic map: cache fresh (<%.0fh), skipping network fetch",
                             _CACHE_TRUST_HOURS,
                         )
                         return
-                else:
-                    logger.info(
-                        "Meshtastic map: cache fresh (<%.0fh), skipping network fetch",
-                        _CACHE_TRUST_HOURS,
-                    )
-                    return
-    except Exception as e:
-        logger.debug(f"Meshtastic cache freshness check failed: {e}")
+        except Exception as e:
+            logger.debug(f"Meshtastic cache freshness check failed: {e}")
 
     # Build a polite User-Agent. Historically this included the operator
     # callsign so meshtastic.org could rate-limit per-install; that's still