perf: live-data deltas, payload caps, and map render polish

Cut fast-tier payload cost with zoom-aware sampling, row deltas, CCTV bbox columns, and MapLibre/motion polish; force viewport snapshot refetches so regional pans refill aircraft immediately. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-05 02:08:44 +02:00 · 2026-07-30 19:40:11 -06:00
parent d38c886af9
commit 5ae1e5b272
56 changed files with 2104 additions and 492 deletions
@@ -1,15 +1,86 @@
 import { useEffect, useRef } from "react";
 import { API_BASE } from "@/lib/api";
-import { mergeData, setBackendStatus as setStoreBackendStatus } from "./useDataStore";
+import {
+  applyLayerDeltas,
+  mergeData,
+  setBackendStatus as setStoreBackendStatus,
+} from "./useDataStore";
 import { appendLiveDataBoundsParams, liveDataBoundsKey } from "@/lib/liveDataViewport";
 import { VIEWPORT_COMMITTED_EVENT } from "@/components/map/hooks/useViewportBounds";

 export type BackendStatus = 'connecting' | 'connected' | 'disconnected';

+const DELTA_LAYER_KEYS = [
+  "ships",
+  "commercial_flights",
+  "military_flights",
+  "tracked_flights",
+  "private_flights",
+  "private_jets",
+  "cctv",
+  "uavs",
+  "liveuamap",
+  "gps_jamming",
+  "satellites",
+  "sigint",
+  "trains",
+] as const;
+
+function formatLayerVersions(lv: Record<string, number> | null): string | null {
+  if (!lv) return null;
+  const parts: string[] = [];
+  for (const key of DELTA_LAYER_KEYS) {
+    const ver = lv[key];
+    if (typeof ver === "number" && Number.isFinite(ver)) {
+      parts.push(`${key}:${ver}`);
+    }
+  }
+  // Need at least the primary delta layers before requesting deltas.
+  if (!parts.some((p) => p.startsWith("ships:") || p.startsWith("commercial_flights:"))) {
+    return null;
+  }
+  return parts.join(",");
+}
+
+function ingestFastPayload(
+  json: Record<string, unknown>,
+  layerVersionsRef: { current: Record<string, number> | null },
+): boolean {
+  const mode = String(json.mode || "snapshot");
+  if (json.layer_versions && typeof json.layer_versions === "object") {
+    layerVersionsRef.current = {
+      ...(layerVersionsRef.current || {}),
+      ...(json.layer_versions as Record<string, number>),
+    };
+  }
+
+  if (mode === "delta") {
+    const deltas = (json.deltas || {}) as Record<
+      string,
+      { upsert?: unknown[]; delete?: string[]; version?: number }
+    >;
+    const ok = applyLayerDeltas(deltas);
+    if (!ok) return false;
+    const layers = (json.layers || {}) as Record<string, unknown>;
+    if (layers && Object.keys(layers).length > 0) {
+      mergeData(layers);
+    }
+    if (json.freshness) mergeData({ freshness: json.freshness });
+    if (json.cctv_total != null) mergeData({ cctv_total: json.cctv_total });
+    if (json.sigint_totals) mergeData({ sigint_totals: json.sigint_totals });
+    return true;
+  }
+
+  mergeData(json);
+  return true;
+}
+
 // ---------------------------------------------------------------------------
 // Polling pause/resume — used by Time Machine snapshot playback
 // ---------------------------------------------------------------------------
 let _pollingPaused = false;
+/** True while the browser tab is hidden — timers are cleared; refresh on focus. */
+let _tabHidden = false;
 let _fastEtagRef: { current: string | null } | null = null;
 let _slowEtagRef: { current: string | null } | null = null;

@@ -100,7 +171,10 @@ const VIEWPORT_FAST_REFETCH_MIN_INTERVAL_MS = 2500;
 * the shared ETag cache exactly like the pre-#288 behaviour.
 *
 * Viewport commits trigger a debounced fast-tier refetch so regional pans
- * refill aircraft/ships without waiting for the 15s poll cadence.
+ * refill aircraft/ships without waiting for the 15s poll cadence. That refetch
+ * clears layer-version delta state — row deltas are not bbox-aware for the
+ * client's existing arrays, so an empty delta after a pan would leave the
+ * previous region's aircraft on screen (or none, after inView culls them).
 *
 * The AIS stream viewport POST (/api/viewport) is still handled separately
 * by useViewportBounds to limit upstream AIS ingestion.
@@ -113,15 +187,18 @@ export function useDataPolling() {
    // Expose refs so pausePolling/resumePolling can invalidate ETags
    _fastEtagRef = fastEtag;
    _slowEtagRef = slowEtag;
+    _tabHidden = typeof document !== 'undefined' && document.hidden;

    let hasData = false;
    let fetchedStartupFastPayload = false;
    let fastTimerId: ReturnType<typeof setTimeout> | null = null;
    let slowTimerId: ReturnType<typeof setTimeout> | null = null;
    let viewportDebounceTimer: ReturnType<typeof setTimeout> | null = null;
+    let layerToggleRetryTimer: ReturnType<typeof setTimeout> | null = null;
    const fastAbortRef = { current: null as AbortController | null };
    const slowAbortRef = { current: null as AbortController | null };
    const fastFetchGenRef = { current: 0 };
+    const layerVersionsRef = { current: null as Record<string, number> | null };
    let lastViewportFetchKey: string | null = null;
    let lastViewportFetchAt = 0;

@@ -174,9 +251,13 @@ export function useDataPolling() {
        const useStartupPayload = !fetchedStartupFastPayload && !fastEtag.current;
        const headers: Record<string, string> = {};
        if (!useStartupPayload && fastEtag.current) headers['If-None-Match'] = fastEtag.current;
-        const url = appendLiveDataBoundsParams(
+        let url = appendLiveDataBoundsParams(
          `${API_BASE}/api/live-data/fast${useStartupPayload ? '?initial=1' : ''}`,
        );
+        const lvParam = !useStartupPayload ? formatLayerVersions(layerVersionsRef.current) : null;
+        if (lvParam) {
+          url += (url.includes('?') ? '&' : '?') + `lv=${encodeURIComponent(lvParam)}`;
+        }
        const res = await fetch(url, {
          headers,
          signal: controller.signal,
@@ -195,8 +276,14 @@ export function useDataPolling() {
          if (useStartupPayload) fetchedStartupFastPayload = true;
          const json = await res.json();
          if (fetchGen !== fastFetchGenRef.current) return;
-          mergeData(json);
-          if (hasMeaningfulFastData(json)) hasData = true;
+          const applied = ingestFastPayload(json, layerVersionsRef);
+          if (!applied) {
+            // Delta apply failed — force a full snapshot next tick.
+            layerVersionsRef.current = null;
+            fastEtag.current = null;
+          } else if (hasMeaningfulFastData(json) || json.mode === 'delta') {
+            hasData = true;
+          }
        }
      } catch (e) {
        const aborted =
@@ -256,6 +343,8 @@ export function useDataPolling() {

    // Adaptive polling: retry every 3s during startup, back off to normal cadence once data arrives
    const scheduleNext = (tier: 'fast' | 'slow', fetchGen?: number) => {
+      // Pause scheduling while the tab is backgrounded; visibilitychange resumes with a refresh.
+      if (_tabHidden || document.hidden) return;
      if (tier === 'fast') {
        if (fetchGen !== undefined && fetchGen !== fastFetchGenRef.current) return;
        const delay = hasData ? 15000 : 3000; // 3s startup retry → 15s steady state
@@ -267,57 +356,101 @@ export function useDataPolling() {
      }
    };

-    const queueViewportFastRefetch = () => {
-      if (_pollingPaused) return;
+    const clearPollTimers = () => {
+      if (fastTimerId) {
+        clearTimeout(fastTimerId);
+        fastTimerId = null;
+      }
+      if (slowTimerId) {
+        clearTimeout(slowTimerId);
+        slowTimerId = null;
+      }
+    };

-      const key = liveDataBoundsKey();
-      if (!key) {
-        lastViewportFetchKey = null;
+    const onVisibilityChange = () => {
+      if (document.hidden) {
+        _tabHidden = true;
+        clearPollTimers();
        return;
      }
-      if (key === lastViewportFetchKey) return;
+      _tabHidden = false;
+      // Resume with one refresh on focus so the UI doesn't feel stuck.
+      // Keep ETags — 304 is fine when nothing changed. Still respect Time Machine pause.
+      if (_pollingPaused) return;
+      void fetchFastData();
+      void fetchSlowData();
+    };
+
+    const fireViewportFastRefetch = () => {
+      if (_pollingPaused || _tabHidden || document.hidden) return;
+
+      // null bounds → world-scale fetch; still refetch when leaving a region
+      // so the store is not stuck with the previous bbox-filtered arrays.
+      const currentFetchKey = liveDataBoundsKey() ?? '__world__';
+      if (currentFetchKey === lastViewportFetchKey) return;
+
+      const now = Date.now();
+      const waitMs = VIEWPORT_FAST_REFETCH_MIN_INTERVAL_MS - (now - lastViewportFetchAt);
+      if (waitMs > 0) {
+        // Do not drop the pan — retry when the rate window opens.
+        if (viewportDebounceTimer) clearTimeout(viewportDebounceTimer);
+        viewportDebounceTimer = setTimeout(() => {
+          viewportDebounceTimer = null;
+          fireViewportFastRefetch();
+        }, waitMs);
+        return;
+      }
+
+      lastViewportFetchKey = currentFetchKey;
+      lastViewportFetchAt = now;
+      fastEtag.current = null;
+      // Force a bbox-scoped snapshot. Delta mode only patches rows that changed
+      // in the store; it cannot replace "Europe flights" with "America flights".
+      layerVersionsRef.current = null;
+      void fetchFastData();
+    };
+
+    const queueViewportFastRefetch = () => {
+      if (_pollingPaused || _tabHidden || document.hidden) return;
+
+      const fetchKey = liveDataBoundsKey() ?? '__world__';
+      if (fetchKey === lastViewportFetchKey) return;

      if (viewportDebounceTimer) clearTimeout(viewportDebounceTimer);
      viewportDebounceTimer = setTimeout(() => {
        viewportDebounceTimer = null;
-        if (_pollingPaused) return;
-
-        const currentKey = liveDataBoundsKey();
-        if (!currentKey || currentKey === lastViewportFetchKey) return;
-
-        const now = Date.now();
-        if (now - lastViewportFetchAt < VIEWPORT_FAST_REFETCH_MIN_INTERVAL_MS) return;
-
-        lastViewportFetchKey = currentKey;
-        lastViewportFetchAt = now;
-        fastEtag.current = null;
-        void fetchFastData();
+        fireViewportFastRefetch();
      }, VIEWPORT_FAST_REFETCH_DEBOUNCE_MS);
    };

-    // When a layer toggle fires, refetch live tiers immediately and retry a few
-    // times so network-heavy on-enable fetches (FIRMS, PSK, …) can finish in the
-    // background without blocking POST /api/layers on the single API worker.
+    // When a layer toggle fires, refetch live tiers immediately and one follow-up
+    // retry so network-heavy on-enable fetches (FIRMS, PSK, …) can land without
+    // a multi-retry storm (was 1s/2.5s/5s → up to 8 requests).
    const onLayerToggle = () => {
      slowEtag.current = null;
      fastEtag.current = null;
+      // Force full snapshot after toggle — deltas may miss cold-layer fills.
+      layerVersionsRef.current = null;
      if (slowTimerId) clearTimeout(slowTimerId);
      slowTimerId = null;
+      if (layerToggleRetryTimer) {
+        clearTimeout(layerToggleRetryTimer);
+        layerToggleRetryTimer = null;
+      }
      void fetchFastData();
      void fetchSlowData();
-      const retryDelaysMs = [1000, 2500, 5000];
-      for (const delay of retryDelaysMs) {
-        setTimeout(() => {
-          if (_pollingPaused) return;
-          slowEtag.current = null;
-          fastEtag.current = null;
-          void fetchSlowData();
-          void fetchFastData();
-        }, delay);
-      }
+      layerToggleRetryTimer = setTimeout(() => {
+        layerToggleRetryTimer = null;
+        if (_pollingPaused || _tabHidden || document.hidden) return;
+        slowEtag.current = null;
+        fastEtag.current = null;
+        void fetchSlowData();
+        void fetchFastData();
+      }, 2500);
    };
    window.addEventListener(LAYER_TOGGLE_EVENT, onLayerToggle);
    window.addEventListener(VIEWPORT_COMMITTED_EVENT, queueViewportFastRefetch);
+    document.addEventListener('visibilitychange', onVisibilityChange);

    void (async () => {
      await fetchCriticalBootstrap();
@@ -329,9 +462,10 @@ export function useDataPolling() {
    return () => {
      window.removeEventListener(LAYER_TOGGLE_EVENT, onLayerToggle);
      window.removeEventListener(VIEWPORT_COMMITTED_EVENT, queueViewportFastRefetch);
-      if (fastTimerId) clearTimeout(fastTimerId);
-      if (slowTimerId) clearTimeout(slowTimerId);
+      document.removeEventListener('visibilitychange', onVisibilityChange);
+      clearPollTimers();
      if (viewportDebounceTimer) clearTimeout(viewportDebounceTimer);
+      if (layerToggleRetryTimer) clearTimeout(layerToggleRetryTimer);
      abortInFlightFastFetch();
      if (slowAbortRef.current) slowAbortRef.current.abort();
    };