CalvinBackup/Shadowbroker
1
0
Fork 0
mirror of https://github.com/BigBodyCobain/Shadowbroker.git synced 2026-04-24 03:26:06 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: integrate AI codebase optimizations (memory safety, spatial hashing, centralized API base)

Browse Source
This commit is contained in:
anoracleofra-code
2026-03-08 15:39:33 -06:00
parent abbc51096b
commit cd03bb966f
11 changed files with 188 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/main.py
+19 -1
View File
@@ -112,7 +112,25 @@ async def debug_latest_data():
@app.get("/api/health")
async def health_check():
return {"status": "ok"}
import time
d = get_latest_data()
last = d.get("last_updated")
return {
"status": "ok",
"last_updated": last,
"sources": {
"flights": len(d.get("commercial_flights", [])),
"military": len(d.get("military_flights", [])),
"ships": len(d.get("ships", [])),
"satellites": len(d.get("satellites", [])),
"earthquakes": len(d.get("earthquakes", [])),
"cctv": len(d.get("cctv", [])),
"news": len(d.get("news", [])),
},
"uptime_seconds": round(time.time() - _start_time),
}
_start_time = __import__("time").time()
from services.radio_intercept import get_top_broadcastify_feeds, get_openmhz_systems, get_recent_openmhz_calls, find_nearest_openmhz_system
backend/services/ais_cache.json
+1 -1
View File
File diff suppressed because one or more lines are too long
backend/services/ais_stream.py
+9 -4
View File
@@ -211,9 +211,10 @@ def _ais_stream_loop():
"""Main loop: spawn node proxy and process messages from stdout."""
import subprocess
import os
proxy_script = os.path.join(os.path.dirname(os.path.dirname(__file__)), "ais_proxy.js")
backoff = 1 # Exponential backoff starting at 1 second
while _ws_running:
try:
logger.info("Starting Node.js AIS Stream Proxy...")
@@ -323,8 +324,12 @@ def _ais_stream_loop():
except Exception as e:
logger.error(f"AIS proxy connection error: {e}")
if _ws_running:
logger.info("Restarting AIS proxy in 5 seconds...")
time.sleep(5)
logger.info(f"Restarting AIS proxy in {backoff}s (exponential backoff)...")
time.sleep(backoff)
backoff = min(backoff * 2, 60) # Double up to 60s max
continue
# Reset backoff on successful connection (got at least some messages)
backoff = 1
def _run_ais_loop():
backend/services/data_fetcher.py
+40 -24
View File
@@ -72,8 +72,8 @@ class OpenSkyClient:
# User provided credentials
opensky_client = OpenSkyClient(
client_id=os.environ.get("OPENSKY_CLIENT_ID", "vancecook-api-client"),
client_secret=os.environ.get("OPENSKY_CLIENT_SECRET", "YOUR_OPENSKY_SECRET")
client_id=os.environ.get("OPENSKY_CLIENT_ID", ""),
client_secret=os.environ.get("OPENSKY_CLIENT_SECRET", "")
)
# Throttling and caching for OpenSky to observe the 400 req/day limit
@@ -885,9 +885,10 @@ def fetch_flights():
by_icao[id(f)] = f # no icao — keep as unique
return list(by_icao.values())
latest_data['commercial_flights'] = _merge_category(commercial, latest_data.get('commercial_flights', []))
latest_data['private_jets'] = _merge_category(private_jets, latest_data.get('private_jets', []))
latest_data['private_flights'] = _merge_category(private_ga, latest_data.get('private_flights', []))
with _data_lock:
latest_data['commercial_flights'] = _merge_category(commercial, latest_data.get('commercial_flights', []))
latest_data['private_jets'] = _merge_category(private_jets, latest_data.get('private_jets', []))
latest_data['private_flights'] = _merge_category(private_ga, latest_data.get('private_flights', []))
# Always write raw flights for GPS jamming analysis (nac_p field)
if flights:
@@ -964,27 +965,39 @@ def fetch_flights():
all_lists = [commercial, private_jets, private_ga, existing_tracked]
seen_hexes = set()
trail_count = 0
for flist in all_lists:
for f in flist:
count, hex_id = _accumulate_trail(f, now_ts, check_route=True)
with _trails_lock:
for flist in all_lists:
for f in flist:
count, hex_id = _accumulate_trail(f, now_ts, check_route=True)
trail_count += count
if hex_id:
seen_hexes.add(hex_id)
# Also process military flights (separate list)
for mf in latest_data.get('military_flights', []):
count, hex_id = _accumulate_trail(mf, now_ts, check_route=False)
trail_count += count
if hex_id:
seen_hexes.add(hex_id)
# Also process military flights (separate list)
for mf in latest_data.get('military_flights', []):
count, hex_id = _accumulate_trail(mf, now_ts, check_route=False)
trail_count += count
if hex_id:
seen_hexes.add(hex_id)
# Prune trails for aircraft not seen in 30 minutes
stale_cutoff = now_ts - 1800
stale_keys = [k for k, v in flight_trails.items() if v['last_seen'] < stale_cutoff]
for k in stale_keys:
del flight_trails[k]
logger.info(f"Trail accumulation: {trail_count} active trails, {len(stale_keys)} pruned")
# Prune stale trails (10 min for non-tracked, 30 min for tracked)
tracked_hexes = {t.get('icao24', '').lower() for t in latest_data.get('tracked_flights', [])}
stale_keys = []
for k, v in flight_trails.items():
cutoff = now_ts - 1800 if k in tracked_hexes else now_ts - 600
if v['last_seen'] < cutoff:
stale_keys.append(k)
for k in stale_keys:
del flight_trails[k]
# Enforce global cap — evict oldest trails first
if len(flight_trails) > _MAX_TRACKED_TRAILS:
sorted_keys = sorted(flight_trails.keys(), key=lambda k: flight_trails[k]['last_seen'])
evict_count = len(flight_trails) - _MAX_TRACKED_TRAILS
for k in sorted_keys[:evict_count]:
del flight_trails[k]
logger.info(f"Trail accumulation: {trail_count} active trails, {len(stale_keys)} pruned, {len(flight_trails)} total")
# -----------------------------------------------------------------------
# GPS / GNSS Jamming Detection — aggregate NACp from ADS-B transponders
@@ -1567,6 +1580,8 @@ def fetch_uavs():
cached_airports = []
flight_trails = {} # {icao_hex: {points: [[lat, lng, alt, ts], ...], last_seen: ts}}
_trails_lock = threading.Lock()
_MAX_TRACKED_TRAILS = 2000 # Global cap on number of aircraft trails in memory
# (math imported at module top)
@@ -1751,5 +1766,6 @@ def stop_scheduler():
scheduler.shutdown()
def get_latest_data():
return latest_data
with _data_lock:
return dict(latest_data)
backend/services/network_utils.py
+11 -3
View File
@@ -3,10 +3,19 @@ import json
import subprocess
import shutil
import time
import requests
from urllib.parse import urlparse
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
logger = logging.getLogger(__name__)
# Reusable session with connection pooling and retry logic
_session = requests.Session()
_retry = Retry(total=2, backoff_factor=0.5, status_forcelist=[502, 503, 504])
_session.mount("https://", HTTPAdapter(max_retries=_retry, pool_maxsize=20))
_session.mount("http://", HTTPAdapter(max_retries=_retry, pool_maxsize=10))
# Find bash for curl fallback — Git bash's curl has the TLS features
# needed to pass CDN fingerprint checks (brotli, zstd, libpsl)
_BASH_PATH = shutil.which("bash") or "bash"
@@ -50,11 +59,10 @@ def fetch_with_curl(url, method="GET", json_data=None, timeout=15, headers=None)
pass # Fall through to curl below
else:
try:
import requests
if method == "POST":
res = requests.post(url, json=json_data, timeout=timeout, headers=default_headers)
res = _session.post(url, json=json_data, timeout=timeout, headers=default_headers)
else:
res = requests.get(url, timeout=timeout, headers=default_headers)
res = _session.get(url, timeout=timeout, headers=default_headers)
res.raise_for_status()
# Clear failure cache on success
_domain_fail_cache.pop(domain, None)
frontend/build_error.txt
+17
View File
@@ -0,0 +1,17 @@
> frontend@0.1.0 build
> next build
Γû▓ Next.js 16.1.6 (Turbopack)
- Environments: .env.local
Creating an optimized production build ...
Γ£ô Compiled successfully in 9.9s
Running TypeScript ...
Failed to compile.
Type error: Cannot find type definition file for 'mapbox__point-geometry'.
The file is in the program because:
Entry point for implicit type library 'mapbox__point-geometry'
Next.js build worker exited with code: 1 and signal: null
frontend/src/app/page.tsx
+8 -7
View File
@@ -1,5 +1,6 @@
"use client";
import { API_BASE } from "@/lib/api";
import { useEffect, useState, useRef, useCallback } from "react";
import dynamic from 'next/dynamic';
import { motion } from "framer-motion";
@@ -146,7 +147,7 @@ export default function Dashboard() {
setRegionDossierLoading(true);
setRegionDossier(null);
try {
const res = await fetch(`http://localhost:8000/api/region-dossier?lat=${coords.lat}&lng=${coords.lng}`);
const res = await fetch(`${API_BASE}/api/region-dossier?lat=${coords.lat}&lng=${coords.lng}`);
if (res.ok) {
const data = await res.json();
setRegionDossier(data);
@@ -175,7 +176,7 @@ export default function Dashboard() {
try {
const headers: Record<string, string> = {};
if (fastEtag.current) headers['If-None-Match'] = fastEtag.current;
const res = await fetch("http://localhost:8000/api/live-data/fast", { headers });
const res = await fetch(`${API_BASE}/api/live-data/fast`, { headers });
if (res.status === 304) return; // Data unchanged, skip update
if (res.ok) {
fastEtag.current = res.headers.get('etag') || null;
@@ -192,7 +193,7 @@ export default function Dashboard() {
try {
const headers: Record<string, string> = {};
if (slowEtag.current) headers['If-None-Match'] = slowEtag.current;
const res = await fetch("http://localhost:8000/api/live-data/slow", { headers });
const res = await fetch(`${API_BASE}/api/live-data/slow`, { headers });
if (res.status === 304) return;
if (res.ok) {
slowEtag.current = res.headers.get('etag') || null;
@@ -208,10 +209,10 @@ export default function Dashboard() {
fetchFastData();
fetchSlowData();
// Fast polling: 15s (backend updates every 60s — polling more often just yields 304s)
// Slow polling: 60s (backend updates every 30min)
const fastInterval = setInterval(fetchFastData, 15000);
const slowInterval = setInterval(fetchSlowData, 60000);
// Fast polling: 60s (matches backend update cadence — was 15s, wasting 75% on 304s)
// Slow polling: 120s (backend updates every 30min)
const fastInterval = setInterval(fetchFastData, 60000);
const slowInterval = setInterval(fetchSlowData, 120000);
return () => {
clearInterval(fastInterval);
frontend/src/components/MaplibreViewer.tsx
+73 -39
View File
@@ -1,5 +1,6 @@
"use client";
import { API_BASE } from "@/lib/api";
import React, { useMemo, useState, useEffect, useCallback, useRef } from "react";
import Map, { Source, Layer, MapRef, ViewState, Popup, Marker } from "react-map-gl/maplibre";
import "maplibre-gl/dist/maplibre-gl.css";
@@ -267,7 +268,7 @@ const MaplibreViewer = ({ data, activeLayers, onEntityClick, flyToLocation, sele
if (callsign && callsign !== prevCallsign.current) {
prevCallsign.current = callsign;
fetch(`http://localhost:8000/api/route/${callsign}`)
fetch(`${API_BASE}/api/route/${callsign}`)
.then(res => res.json())
.then(routeData => {
if (isMounted) setDynamicRoute(routeData);
@@ -669,10 +670,17 @@ const MaplibreViewer = ({ data, activeLayers, onEntityClick, flyToLocation, sele
}, [activeLayers.ships_important, activeLayers.ships_civilian, activeLayers.ships_passenger, data?.ships, inView]);
// Extract ship cluster positions from the map source for HTML labels
const shipClusterHandlerRef = useRef<(() => void) | null>(null);
useEffect(() => {
const map = mapRef.current?.getMap();
if (!map || !shipsGeoJSON) { setShipClusters([]); return; }
// Remove previous handler if it exists
if (shipClusterHandlerRef.current) {
map.off('moveend', shipClusterHandlerRef.current);
map.off('sourcedata', shipClusterHandlerRef.current);
}
const update = () => {
try {
const features = map.querySourceFeatures('ships');
@@ -689,6 +697,7 @@ const MaplibreViewer = ({ data, activeLayers, onEntityClick, flyToLocation, sele
setShipClusters(unique);
} catch { setShipClusters([]); }
};
shipClusterHandlerRef.current = update;
map.on('moveend', update);
map.on('sourcedata', update);
@@ -698,10 +707,16 @@ const MaplibreViewer = ({ data, activeLayers, onEntityClick, flyToLocation, sele
}, [shipsGeoJSON]);
// Extract earthquake cluster positions from the map source for HTML labels
const eqClusterHandlerRef = useRef<(() => void) | null>(null);
useEffect(() => {
const map = mapRef.current?.getMap();
if (!map || !earthquakesGeoJSON) { setEqClusters([]); return; }
if (eqClusterHandlerRef.current) {
map.off('moveend', eqClusterHandlerRef.current);
map.off('sourcedata', eqClusterHandlerRef.current);
}
const update = () => {
try {
const features = map.querySourceFeatures('earthquakes');
@@ -718,6 +733,7 @@ const MaplibreViewer = ({ data, activeLayers, onEntityClick, flyToLocation, sele
setEqClusters(unique);
} catch { setEqClusters([]); }
};
eqClusterHandlerRef.current = update;
map.on('moveend', update);
map.on('sourcedata', update);
@@ -848,42 +864,58 @@ const MaplibreViewer = ({ data, activeLayers, onEntityClick, flyToLocation, sele
const GAP = 6; // Minimum gap between boxes
const MAX_OFFSET = 350;
// 2. Iterative Collision Resolution Loop
const maxIter = 40;
// 2. Grid-based Collision Resolution (O(n) per iteration instead of O(n²))
const CELL_W = BOX_W + GAP;
const CELL_H = 100; // Approximate max box height + gap
const maxIter = 30;
for (let iter = 0; iter < maxIter; iter++) {
let moved = false;
// Build spatial grid
const grid: Record<string, number[]> = {};
for (let i = 0; i < items.length; i++) {
for (let j = i + 1; j < items.length; j++) {
const a = items[i];
const b = items[j];
const aX = a.x + a.offsetX;
const aY = a.y + a.offsetY;
const bX = b.x + b.offsetX;
const bY = b.y + b.offsetY;
const dx = Math.abs(aX - bX);
const dy = Math.abs(aY - bY);
// Per-pair min distances using each box's actual estimated height
const minDistX = BOX_W + GAP;
const minDistY = (a.boxH + b.boxH) / 2 + GAP;
if (dx < minDistX && dy < minDistY) {
moved = true;
const overlapX = minDistX - dx;
const overlapY = minDistY - dy;
// Push each by half the overlap + 1px to guarantee separation
if (overlapY < overlapX) {
const push = (overlapY / 2) + 1;
if (aY <= bY) { a.offsetY -= push; b.offsetY += push; }
else { a.offsetY += push; b.offsetY -= push; }
} else {
const push = (overlapX / 2) + 1;
if (aX <= bX) { a.offsetX -= push; b.offsetX += push; }
else { a.offsetX += push; b.offsetX -= push; }
const cx = Math.floor((items[i].x + items[i].offsetX) / CELL_W);
const cy = Math.floor((items[i].y + items[i].offsetY) / CELL_H);
const key = `${cx},${cy}`;
(grid[key] ??= []).push(i);
}
// Check collisions only within same/adjacent cells
const checked = new Set<string>();
for (const key in grid) {
const [cx, cy] = key.split(',').map(Number);
for (let dx = -1; dx <= 1; dx++) {
for (let dy = -1; dy <= 1; dy++) {
const nk = `${cx + dx},${cy + dy}`;
if (!grid[nk]) continue;
const pairKey = cx + dx < cx || (cx + dx === cx && cy + dy < cy) ? `${nk}|${key}` : `${key}|${nk}`;
if (key !== nk && checked.has(pairKey)) continue;
checked.add(pairKey);
const cellA = grid[key];
const cellB = key === nk ? cellA : grid[nk];
for (const i of cellA) {
const startJ = key === nk ? cellA.indexOf(i) + 1 : 0;
for (let jIdx = startJ; jIdx < cellB.length; jIdx++) {
const j = cellB[jIdx];
if (i === j) continue;
const a = items[i], b = items[j];
const adx = Math.abs((a.x + a.offsetX) - (b.x + b.offsetX));
const ady = Math.abs((a.y + a.offsetY) - (b.y + b.offsetY));
const minDistX = BOX_W + GAP;
const minDistY = (a.boxH + b.boxH) / 2 + GAP;
if (adx < minDistX && ady < minDistY) {
moved = true;
const overlapX = minDistX - adx;
const overlapY = minDistY - ady;
if (overlapY < overlapX) {
const push = (overlapY / 2) + 1;
if ((a.y + a.offsetY) <= (b.y + b.offsetY)) { a.offsetY -= push; b.offsetY += push; }
else { a.offsetY += push; b.offsetY -= push; }
} else {
const push = (overlapX / 2) + 1;
if ((a.x + a.offsetX) <= (b.x + b.offsetX)) { a.offsetX -= push; b.offsetX += push; }
else { a.offsetX += push; b.offsetX -= push; }
}
}
}
}
}
}
@@ -941,7 +973,7 @@ const MaplibreViewer = ({ data, activeLayers, onEntityClick, flyToLocation, sele
return {
type: 'FeatureCollection',
features: data.uavs.map((uav: any, i: number) => {
if (uav.lat == null || uav.lng == null) return null;
if (uav.lat == null || uav.lng == null || !inView(uav.lat, uav.lng)) return null;
return {
type: 'Feature',
properties: {
@@ -962,7 +994,7 @@ const MaplibreViewer = ({ data, activeLayers, onEntityClick, flyToLocation, sele
};
}).filter(Boolean)
};
}, [activeLayers.military, data?.uavs]);
}, [activeLayers.military, data?.uavs, inView]);
// UAV operational range circle — only for the selected UAV
const uavRangeGeoJSON = useMemo(() => {
@@ -996,6 +1028,8 @@ const MaplibreViewer = ({ data, activeLayers, onEntityClick, flyToLocation, sele
type: 'FeatureCollection',
features: data.gdelt.map((g: any, i: number) => {
if (!g.geometry || !g.geometry.coordinates) return null;
const [gLng, gLat] = g.geometry.coordinates;
if (!inView(gLat, gLng)) return null;
return {
type: 'Feature',
properties: { id: i, type: 'gdelt', title: g.title },
@@ -1003,14 +1037,14 @@ const MaplibreViewer = ({ data, activeLayers, onEntityClick, flyToLocation, sele
};
}).filter(Boolean)
};
}, [activeLayers.global_incidents, data?.gdelt]);
}, [activeLayers.global_incidents, data?.gdelt, inView]);
const liveuaGeoJSON = useMemo(() => {
if (!activeLayers.global_incidents || !data?.liveuamap) return null;
return {
type: 'FeatureCollection',
features: data.liveuamap.map((incident: any, i: number) => {
if (incident.lat == null || incident.lng == null) return null;
if (incident.lat == null || incident.lng == null || !inView(incident.lat, incident.lng)) return null;
const isViolent = /bomb|missil|strike|attack|kill|destroy|fire|shoot|expl|raid/i.test(incident.title || "");
return {
type: 'Feature',
@@ -1019,7 +1053,7 @@ const MaplibreViewer = ({ data, activeLayers, onEntityClick, flyToLocation, sele
};
}).filter(Boolean)
};
}, [activeLayers.global_incidents, data?.liveuamap]);
}, [activeLayers.global_incidents, data?.liveuamap, inView]);
const frontlineGeoJSON = useMemo(() => {
if (!activeLayers.ukraine_frontline || !data?.frontlines) return null;
frontend/src/components/RadioInterceptPanel.tsx
+6 -5
View File
@@ -1,5 +1,6 @@
"use client";
import { API_BASE } from "@/lib/api";
import { useState, useEffect, useRef } from 'react';
import { motion, AnimatePresence } from 'framer-motion';
import { RadioReceiver, Activity, Play, Square, FastForward, ChevronDown, ChevronUp } from 'lucide-react';
@@ -18,7 +19,7 @@ export default function RadioInterceptPanel({ data, isEavesdropping, setIsEavesd
useEffect(() => {
const fetchFeeds = async () => {
try {
const res = await fetch("http://localhost:8000/api/radio/top");
const res = await fetch(`${API_BASE}/api/radio/top`);
if (res.ok) {
const json = await res.json();
setFeeds(json);
@@ -47,12 +48,12 @@ export default function RadioInterceptPanel({ data, isEavesdropping, setIsEavesd
category: 'SIGINT'
}, ...prev]);
const res = await fetch(`http://localhost:8000/api/radio/nearest?lat=${eavesdropLocation.lat}&lng=${eavesdropLocation.lng}`);
const res = await fetch(`${API_BASE}/api/radio/nearest?lat=${eavesdropLocation.lat}&lng=${eavesdropLocation.lng}`);
if (res.ok) {
const system = await res.json();
if (system && system.shortName) {
// Valid OpenMHZ system found! Fetch recent calls
const callRes = await fetch(`http://localhost:8000/api/radio/openmhz/calls/${system.shortName}`);
const callRes = await fetch(`${API_BASE}/api/radio/openmhz/calls/${system.shortName}`);
if (callRes.ok) {
const calls = await callRes.json();
if (calls && calls.length > 0) {
@@ -189,14 +190,14 @@ export default function RadioInterceptPanel({ data, isEavesdropping, setIsEavesd
if (scanLoc) {
try {
const res = await fetch(`http://localhost:8000/api/radio/nearest-list?lat=${scanLoc.lat}&lng=${scanLoc.lng}&limit=3`);
const res = await fetch(`${API_BASE}/api/radio/nearest-list?lat=${scanLoc.lat}&lng=${scanLoc.lng}&limit=3`);
if (res.ok) {
const systems = await res.json();
// Try to find a system with an active unplayed burst
for (const system of systems) {
if (system && system.shortName) {
const callRes = await fetch(`http://localhost:8000/api/radio/openmhz/calls/${system.shortName}`);
const callRes = await fetch(`${API_BASE}/api/radio/openmhz/calls/${system.shortName}`);
if (callRes.ok) {
const calls = await callRes.json();
if (calls && calls.length > 0) {
frontend/src/components/SettingsPanel.tsx
+3 -2
View File
@@ -1,5 +1,6 @@
"use client";
import { API_BASE } from "@/lib/api";
import React, { useState, useEffect, useCallback } from "react";
import { motion, AnimatePresence } from "framer-motion";
import { Settings, Eye, EyeOff, Copy, Check, ExternalLink, Key, Shield, X, Save, ChevronDown, ChevronUp } from "lucide-react";
@@ -41,7 +42,7 @@ const SettingsPanel = React.memo(function SettingsPanel({ isOpen, onClose }: { i
const fetchKeys = useCallback(async () => {
try {
const res = await fetch("http://localhost:8000/api/settings/api-keys");
const res = await fetch(`${API_BASE}/api/settings/api-keys`);
if (res.ok) {
const data = await res.json();
setApis(data);
@@ -83,7 +84,7 @@ const SettingsPanel = React.memo(function SettingsPanel({ isOpen, onClose }: { i
if (!api.env_key) return;
setSaving(true);
try {
const res = await fetch("http://localhost:8000/api/settings/api-keys", {
const res = await fetch(`${API_BASE}/api/settings/api-keys`, {
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ env_key: api.env_key, value: editValue }),
frontend/src/lib/api.ts
+1
View File
@@ -0,0 +1 @@
export const API_BASE = process.env.NEXT_PUBLIC_API_URL || "http://localhost:8000";