CalvinBackup/Shadowbroker
1
0
Fork 0
mirror of https://github.com/BigBodyCobain/Shadowbroker.git synced 2026-09-18 15:12:20 +02:00
Code Issues Packages Projects Releases Wiki Activity

perf: live-data deltas, payload caps, and map render polish

Browse Source 
Cut fast-tier payload cost with zoom-aware sampling, row deltas, CCTV bbox columns, and MapLibre/motion polish; force viewport snapshot refetches so regional pans refill aircraft immediately.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
BigBodyCobain
2026-07-30 19:40:11 -06:00
co-authored by Cursor
parent d38c886af9
commit 5ae1e5b272
56 changed files with 2104 additions and 492 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/hooks/useDataPolling.ts
+172 -38
View File
@@ -1,15 +1,86 @@
import { useEffect, useRef } from "react";
import { API_BASE } from "@/lib/api";
import { mergeData, setBackendStatus as setStoreBackendStatus } from "./useDataStore";
import {
applyLayerDeltas,
mergeData,
setBackendStatus as setStoreBackendStatus,
} from "./useDataStore";
import { appendLiveDataBoundsParams, liveDataBoundsKey } from "@/lib/liveDataViewport";
import { VIEWPORT_COMMITTED_EVENT } from "@/components/map/hooks/useViewportBounds";
export type BackendStatus = 'connecting' | 'connected' | 'disconnected';
const DELTA_LAYER_KEYS = [
"ships",
"commercial_flights",
"military_flights",
"tracked_flights",
"private_flights",
"private_jets",
"cctv",
"uavs",
"liveuamap",
"gps_jamming",
"satellites",
"sigint",
"trains",
] as const;
function formatLayerVersions(lv: Record<string, number> | null): string | null {
if (!lv) return null;
const parts: string[] = [];
for (const key of DELTA_LAYER_KEYS) {
const ver = lv[key];
if (typeof ver === "number" && Number.isFinite(ver)) {
parts.push(`${key}:${ver}`);
}
}
// Need at least the primary delta layers before requesting deltas.
if (!parts.some((p) => p.startsWith("ships:") || p.startsWith("commercial_flights:"))) {
return null;
}
return parts.join(",");
}
function ingestFastPayload(
json: Record<string, unknown>,
layerVersionsRef: { current: Record<string, number> | null },
): boolean {
const mode = String(json.mode || "snapshot");
if (json.layer_versions && typeof json.layer_versions === "object") {
layerVersionsRef.current = {
...(layerVersionsRef.current || {}),
...(json.layer_versions as Record<string, number>),
};
}
if (mode === "delta") {
const deltas = (json.deltas || {}) as Record<
string,
{ upsert?: unknown[]; delete?: string[]; version?: number }
>;
const ok = applyLayerDeltas(deltas);
if (!ok) return false;
const layers = (json.layers || {}) as Record<string, unknown>;
if (layers && Object.keys(layers).length > 0) {
mergeData(layers);
}
if (json.freshness) mergeData({ freshness: json.freshness });
if (json.cctv_total != null) mergeData({ cctv_total: json.cctv_total });
if (json.sigint_totals) mergeData({ sigint_totals: json.sigint_totals });
return true;
}
mergeData(json);
return true;
}
// ---------------------------------------------------------------------------
// Polling pause/resume — used by Time Machine snapshot playback
// ---------------------------------------------------------------------------
let _pollingPaused = false;
/** True while the browser tab is hidden — timers are cleared; refresh on focus. */
let _tabHidden = false;
let _fastEtagRef: { current: string | null } | null = null;
let _slowEtagRef: { current: string | null } | null = null;
@@ -100,7 +171,10 @@ const VIEWPORT_FAST_REFETCH_MIN_INTERVAL_MS = 2500;
* the shared ETag cache exactly like the pre-#288 behaviour.
*
* Viewport commits trigger a debounced fast-tier refetch so regional pans
* refill aircraft/ships without waiting for the 15s poll cadence.
* refill aircraft/ships without waiting for the 15s poll cadence. That refetch
* clears layer-version delta state — row deltas are not bbox-aware for the
* client's existing arrays, so an empty delta after a pan would leave the
* previous region's aircraft on screen (or none, after inView culls them).
*
* The AIS stream viewport POST (/api/viewport) is still handled separately
* by useViewportBounds to limit upstream AIS ingestion.
@@ -113,15 +187,18 @@ export function useDataPolling() {
// Expose refs so pausePolling/resumePolling can invalidate ETags
_fastEtagRef = fastEtag;
_slowEtagRef = slowEtag;
_tabHidden = typeof document !== 'undefined' && document.hidden;
let hasData = false;
let fetchedStartupFastPayload = false;
let fastTimerId: ReturnType<typeof setTimeout> | null = null;
let slowTimerId: ReturnType<typeof setTimeout> | null = null;
let viewportDebounceTimer: ReturnType<typeof setTimeout> | null = null;
let layerToggleRetryTimer: ReturnType<typeof setTimeout> | null = null;
const fastAbortRef = { current: null as AbortController | null };
const slowAbortRef = { current: null as AbortController | null };
const fastFetchGenRef = { current: 0 };
const layerVersionsRef = { current: null as Record<string, number> | null };
let lastViewportFetchKey: string | null = null;
let lastViewportFetchAt = 0;
@@ -174,9 +251,13 @@ export function useDataPolling() {
const useStartupPayload = !fetchedStartupFastPayload && !fastEtag.current;
const headers: Record<string, string> = {};
if (!useStartupPayload && fastEtag.current) headers['If-None-Match'] = fastEtag.current;
const url = appendLiveDataBoundsParams(
let url = appendLiveDataBoundsParams(
`${API_BASE}/api/live-data/fast${useStartupPayload ? '?initial=1' : ''}`,
);
const lvParam = !useStartupPayload ? formatLayerVersions(layerVersionsRef.current) : null;
if (lvParam) {
url += (url.includes('?') ? '&' : '?') + `lv=${encodeURIComponent(lvParam)}`;
}
const res = await fetch(url, {
headers,
signal: controller.signal,
@@ -195,8 +276,14 @@ export function useDataPolling() {
if (useStartupPayload) fetchedStartupFastPayload = true;
const json = await res.json();
if (fetchGen !== fastFetchGenRef.current) return;
mergeData(json);
if (hasMeaningfulFastData(json)) hasData = true;
const applied = ingestFastPayload(json, layerVersionsRef);
if (!applied) {
// Delta apply failed — force a full snapshot next tick.
layerVersionsRef.current = null;
fastEtag.current = null;
} else if (hasMeaningfulFastData(json) || json.mode === 'delta') {
hasData = true;
}
}
} catch (e) {
const aborted =
@@ -256,6 +343,8 @@ export function useDataPolling() {
// Adaptive polling: retry every 3s during startup, back off to normal cadence once data arrives
const scheduleNext = (tier: 'fast' | 'slow', fetchGen?: number) => {
// Pause scheduling while the tab is backgrounded; visibilitychange resumes with a refresh.
if (_tabHidden || document.hidden) return;
if (tier === 'fast') {
if (fetchGen !== undefined && fetchGen !== fastFetchGenRef.current) return;
const delay = hasData ? 15000 : 3000; // 3s startup retry → 15s steady state
@@ -267,57 +356,101 @@ export function useDataPolling() {
}
};
const queueViewportFastRefetch = () => {
if (_pollingPaused) return;
const clearPollTimers = () => {
if (fastTimerId) {
clearTimeout(fastTimerId);
fastTimerId = null;
}
if (slowTimerId) {
clearTimeout(slowTimerId);
slowTimerId = null;
}
};
const key = liveDataBoundsKey();
if (!key) {
lastViewportFetchKey = null;
const onVisibilityChange = () => {
if (document.hidden) {
_tabHidden = true;
clearPollTimers();
return;
}
if (key === lastViewportFetchKey) return;
_tabHidden = false;
// Resume with one refresh on focus so the UI doesn't feel stuck.
// Keep ETags — 304 is fine when nothing changed. Still respect Time Machine pause.
if (_pollingPaused) return;
void fetchFastData();
void fetchSlowData();
};
const fireViewportFastRefetch = () => {
if (_pollingPaused || _tabHidden || document.hidden) return;
// null bounds → world-scale fetch; still refetch when leaving a region
// so the store is not stuck with the previous bbox-filtered arrays.
const currentFetchKey = liveDataBoundsKey() ?? '__world__';
if (currentFetchKey === lastViewportFetchKey) return;
const now = Date.now();
const waitMs = VIEWPORT_FAST_REFETCH_MIN_INTERVAL_MS - (now - lastViewportFetchAt);
if (waitMs > 0) {
// Do not drop the pan — retry when the rate window opens.
if (viewportDebounceTimer) clearTimeout(viewportDebounceTimer);
viewportDebounceTimer = setTimeout(() => {
viewportDebounceTimer = null;
fireViewportFastRefetch();
}, waitMs);
return;
}
lastViewportFetchKey = currentFetchKey;
lastViewportFetchAt = now;
fastEtag.current = null;
// Force a bbox-scoped snapshot. Delta mode only patches rows that changed
// in the store; it cannot replace "Europe flights" with "America flights".
layerVersionsRef.current = null;
void fetchFastData();
};
const queueViewportFastRefetch = () => {
if (_pollingPaused || _tabHidden || document.hidden) return;
const fetchKey = liveDataBoundsKey() ?? '__world__';
if (fetchKey === lastViewportFetchKey) return;
if (viewportDebounceTimer) clearTimeout(viewportDebounceTimer);
viewportDebounceTimer = setTimeout(() => {
viewportDebounceTimer = null;
if (_pollingPaused) return;
const currentKey = liveDataBoundsKey();
if (!currentKey || currentKey === lastViewportFetchKey) return;
const now = Date.now();
if (now - lastViewportFetchAt < VIEWPORT_FAST_REFETCH_MIN_INTERVAL_MS) return;
lastViewportFetchKey = currentKey;
lastViewportFetchAt = now;
fastEtag.current = null;
void fetchFastData();
fireViewportFastRefetch();
}, VIEWPORT_FAST_REFETCH_DEBOUNCE_MS);
};
// When a layer toggle fires, refetch live tiers immediately and retry a few
// times so network-heavy on-enable fetches (FIRMS, PSK, …) can finish in the
// background without blocking POST /api/layers on the single API worker.
// When a layer toggle fires, refetch live tiers immediately and one follow-up
// retry so network-heavy on-enable fetches (FIRMS, PSK, …) can land without
// a multi-retry storm (was 1s/2.5s/5s → up to 8 requests).
const onLayerToggle = () => {
slowEtag.current = null;
fastEtag.current = null;
// Force full snapshot after toggle — deltas may miss cold-layer fills.
layerVersionsRef.current = null;
if (slowTimerId) clearTimeout(slowTimerId);
slowTimerId = null;
if (layerToggleRetryTimer) {
clearTimeout(layerToggleRetryTimer);
layerToggleRetryTimer = null;
}
void fetchFastData();
void fetchSlowData();
const retryDelaysMs = [1000, 2500, 5000];
for (const delay of retryDelaysMs) {
setTimeout(() => {
if (_pollingPaused) return;
slowEtag.current = null;
fastEtag.current = null;
void fetchSlowData();
void fetchFastData();
}, delay);
}
layerToggleRetryTimer = setTimeout(() => {
layerToggleRetryTimer = null;
if (_pollingPaused || _tabHidden || document.hidden) return;
slowEtag.current = null;
fastEtag.current = null;
void fetchSlowData();
void fetchFastData();
}, 2500);
};
window.addEventListener(LAYER_TOGGLE_EVENT, onLayerToggle);
window.addEventListener(VIEWPORT_COMMITTED_EVENT, queueViewportFastRefetch);
document.addEventListener('visibilitychange', onVisibilityChange);
void (async () => {
await fetchCriticalBootstrap();
@@ -329,9 +462,10 @@ export function useDataPolling() {
return () => {
window.removeEventListener(LAYER_TOGGLE_EVENT, onLayerToggle);
window.removeEventListener(VIEWPORT_COMMITTED_EVENT, queueViewportFastRefetch);
if (fastTimerId) clearTimeout(fastTimerId);
if (slowTimerId) clearTimeout(slowTimerId);
document.removeEventListener('visibilitychange', onVisibilityChange);
clearPollTimers();
if (viewportDebounceTimer) clearTimeout(viewportDebounceTimer);
if (layerToggleRetryTimer) clearTimeout(layerToggleRetryTimer);
abortInFlightFastFetch();
if (slowAbortRef.current) slowAbortRef.current.abort();
};
frontend/src/hooks/useDataStore.ts
+134 -4
View File
@@ -25,17 +25,97 @@ const store: Record<string, unknown> = {};
let backendStatus: BackendStatus = "connecting";
const statusListeners = new Set<Listener>();
// ── Content-aware equality (stable merge) ────────────────────────────────
/** Cheap O(n) fingerprint for dashboard arrays — id + rounded lat/lng. */
export function arrayFingerprint(arr: unknown[]): string {
let h = String(arr.length);
for (let i = 0; i < arr.length; i++) {
const item = arr[i] as Record<string, unknown> | null | undefined;
if (item == null || typeof item !== "object") {
h += `|${item}`;
continue;
}
const id =
item.icao ?? item.id ?? item.mmsi ?? item.hex ?? item.callsign ?? item.name ?? i;
const lat = item.lat != null ? Math.round(Number(item.lat) * 100) : "";
const lng = item.lng != null ? Math.round(Number(item.lng) * 100) : "";
h += `|${id}:${lat}:${lng}`;
}
return h;
}
/**
* Content-aware equality for mergeData: keep the previous reference when
* values are equivalent enough that subscribers need not re-render.
*/
export function valuesEquivalent(prev: unknown, next: unknown): boolean {
if (prev === next) return true;
if (prev == null || next == null) return prev === next;
if (typeof prev !== typeof next) return false;
if (Array.isArray(prev) && Array.isArray(next)) {
if (prev.length !== next.length) return false;
if (prev.length === 0) return true;
// Geo / track layers: cheap id+position fingerprint. Other arrays
// (news, etc.) fall through as not-equivalent so new refs still notify.
const sample = prev[0];
const isGeoish =
sample != null &&
typeof sample === "object" &&
("lat" in (sample as object) ||
"lng" in (sample as object) ||
"icao" in (sample as object) ||
"icao24" in (sample as object) ||
"mmsi" in (sample as object));
if (!isGeoish) return false;
return arrayFingerprint(prev) === arrayFingerprint(next);
}
if (typeof prev === "object" && typeof next === "object") {
// Shallow: same keys and === values
const pk = Object.keys(prev as object);
const nk = Object.keys(next as object);
if (pk.length !== nk.length) return false;
for (const k of pk) {
if ((prev as Record<string, unknown>)[k] !== (next as Record<string, unknown>)[k]) {
return false;
}
}
return true;
}
return prev === next;
}
// ── Write API (called from useDataPolling) ───────────────────────────────
/** Merge a partial payload into the store, notifying only affected keys. */
export function mergeData(patch: Record<string, unknown>) {
const changedKeys: string[] = [];
for (const key of Object.keys(patch)) {
const next = patch[key];
if (store[key] !== next) {
store[key] = next;
changedKeys.push(key);
// Protocol / meta fields from live-data — never materialize as layers.
if (
key === "mode" ||
key === "deltas" ||
key === "layers" ||
key === "store_version" ||
key === "layer_versions" ||
key === "payload_scale" ||
key === "payload_sampled" ||
key === "layer_totals" ||
key === "startup_payload" ||
key === "bootstrap_payload" ||
key === "bootstrap_ready"
) {
continue;
}
const next = patch[key];
const prev = store[key];
if (valuesEquivalent(prev, next)) {
// Keep previous reference so subscribers do not fire
store[key] = prev;
continue;
}
store[key] = next;
changedKeys.push(key);
}
// Notify per-key subscribers
for (const key of changedKeys) {
@@ -48,6 +128,56 @@ export function mergeData(patch: Record<string, unknown>) {
}
}
function entityIdForLayer(layer: string, item: Record<string, unknown>): string {
if (layer === "ships") {
return String(item.mmsi ?? item.id ?? "").trim();
}
return String(item.icao24 ?? item.icao ?? item.id ?? item.hex ?? "")
.trim()
.toLowerCase();
}
/**
* Apply row-level upsert/delete patches from `/api/live-data/fast` delta mode.
* Returns false if a layer patch cannot be applied safely (caller should full-resync).
*/
export function applyLayerDeltas(
deltas: Record<string, { upsert?: unknown[]; delete?: string[]; version?: number }>,
): boolean {
const changedKeys: string[] = [];
for (const [key, patch] of Object.entries(deltas || {})) {
if (!patch || typeof patch !== "object") return false;
const prev = store[key];
const base = Array.isArray(prev) ? (prev as Record<string, unknown>[]) : [];
const byId = new Map<string, Record<string, unknown>>();
for (const item of base) {
if (!item || typeof item !== "object") continue;
const id = entityIdForLayer(key, item);
if (id) byId.set(id, item);
}
for (const id of patch.delete || []) {
byId.delete(String(id));
}
for (const raw of patch.upsert || []) {
if (!raw || typeof raw !== "object") continue;
const item = raw as Record<string, unknown>;
const id = entityIdForLayer(key, item);
if (!id) continue;
byId.set(id, item);
}
store[key] = Array.from(byId.values());
changedKeys.push(key);
}
for (const key of changedKeys) {
const set = keyListeners.get(key);
if (set) for (const fn of set) fn();
}
if (changedKeys.length > 0) {
for (const fn of globalListeners) fn();
}
return true;
}
export function setBackendStatus(next: BackendStatus) {
if (backendStatus === next) return;
backendStatus = next;