mirror of
https://github.com/BigBodyCobain/Shadowbroker.git
synced 2026-10-10 17:13:59 +02:00
feat: integrate AI codebase optimizations (memory safety, spatial hashing, centralized API base)
This commit is contained in:
anoracleofra-code committed
1 parent
abbc51096b
commit
cd03bb966f
11 files changed
+188
-86
No files matched your search
@@ -1 +1 @@
|
||||
{"367413270": {"_updated": 1772687710.5994134, "lat": 29.638846666666666, "lng": -91.09025333333334, "sog": 0, "cog": 360, "heading": 360, "name": "MIKE HOOKS", "callsign": "WDE9086", "imo": 0, "destination": "", "ais_type_code": 33, "type": "other"}, "244100829": {"_updated": 1772687706.1995316, "name": "UK371 ESTER JACOBA", "callsign": "PBKX", "imo": 8571524, "destination": "URK", "ais_type_code": 30, "type": "other", "lat": 52.659915, "lng": 5.600176666666667, "sog": 0, "cog": 4.2, "heading": 358}, "244020081": {"_updated": 1772687790.8755257, "lat": 53.17992666666667, "lng": 5.437545, "sog": 0, "cog": 360, "heading": 360, "name": "DURK", "callsign": "PD3283", "imo": 0, "destination": "HARLINGEN?", "ais_type_code": 33, "type": "other"}, "503086960": {"_updated": 1772687658.0071797, "lat": -27.212828333333334, "lng": 153.10039999999998, "sog": 0, "cog": 360, "heading": 360, "name": "KRISTINA"}, "244740824": {"_updated": 1772687780.3278255, "lat": 52.56198, "lng": 4.740971666666667, "sog": 0, "cog": 0, "heading": 0, "name": "CLAES HORN", "callsign": "PH9656", "imo": 0, "destination": "IRSD9T H", "ais_type_code": 69, "type": "passenger"}, "341797000": {"_updated": 1772687790.2410219, "lat": 36.476841666666665, "lng": 28.051465, "sog": 6.7, "cog": 83.4, "heading": 83.4, "name": "ZAMZAM I", "callsign": "V4SD2", "imo": 8415689, "destination": "LB SAI", "ais_type_code": 70, "type": "cargo"}, "237041100": {"_updated": 1772687781.818753, "lat": 37.951298333333334, "lng": 23.549086666666668, "sog": 0, "cog": 273.1, "heading": 267, "name": "PROTUG 18", "callsign": "SX 7744", "imo": 8410213, "destination": "ELEFSIS ANCHORAGE", "ais_type_code": 52, "type": "other"}, "265649360": {"_updated": 1772687789.944359, "lat": 59.40253, "lng": 18.355825, "sog": 0, "cog": 288.6, "heading": 253, "name": "NAMDO", "callsign": "SMRN", "imo": 0, "destination": "", "ais_type_code": 60, "type": "passenger"}, "352906000": {"_updated": 1772687711.1407297, "lat": 46.5827, "lng": 142.88598333333331, "sog": 1.4, "cog": 77, "heading": 91, "name": "K.JASMINE", "callsign": "3EPQ5", "imo": 9373008, "destination": "RUPGN", "ais_type_code": 83, "type": "tanker"}, "367797960": {"_updated": 1772687120.9925582, "lat": 47.658413333333336, "lng": -122.26828, "sog": 0, "cog": 308.1, "heading": 206, "name": "ROI"}, "244810432": {"_updated": 1772687779.4856718, "name": "VLB 3", "callsign": "PA6644", "imo": 0, "destination": "EVERINGENN", "ais_type_code": 52, "type": "other", "lat": 51.46107166666666, "lng": 3.6848383333333334, "sog": 0, "cog": 360, "heading": 360}, "244260009": {"_updated": 1772687799.0210416, "lat": 52.517605, "lng": 5.437293333333333, "sog": 0, "cog": 360, "heading": 360, "name": "ALLURE", "callsign": "PC2365", "imo": 0, "destination": "LELYSTAD^", "ais_type_code": 69, "type": "passenger"}, "368179250": {"_updated": 1772687811.1863346, "lat": 25.771801666666665, "lng": -80.15136666666668, "sog": 0, "cog": 360, "heading": 123, "name": "SEAHAWK", "callsign": "WLD9455", "imo": 0, "destination": "MIAMI", "ais_type_code": 60, "type": "passenger"}, "311001596": {"_updated": 1772687782.0889556, "lat": 1.2662316666666666, "lng": 103.859545, "sog": 0, "cog": 323.6, "heading": 324, "name": "CARNIVAL ENCOUNTER", "callsign": "C6HX7", "imo": 9192363, "destination": "SNSIN PBEGA", "ais_type_code": 60, "type": "passenger"}, "414031000": {"_updated": 1772687661.7838142, "lat": -32.89040166666667, "lng": 151.77405833333333, "sog": 0, "cog": 79, "heading": 192, "name": "CHANG HANG HAO HAI", "callsign": "BUWV", "imo": 9628219, "destination": "AUNTL", "ais_type_code": 70, "type": "cargo"}, "211756140": {"_updated": 1772687793.5456395, "lat": 51.84095166666667, "lng": 6.163704999999999, "sog": 5.6, "cog": 118.4, "heading": 118.4, "name": "MAINFRANKEN", "callsign": "DK9407", "imo": 0, "destination": "GENT", "ais_type_code": 80, "type": "tanker"}, "211439110": {"_updated": 1772687791.5555127, "lat": 52.38180666666666, "lng": 4.943501666666666, "sog": 0.2, "cog": 27.1, "heading": 277, "name": "EILTANK 27", "callsign": "DB5300", "imo": 0, "destination": "AMSTERDAM", "ais_type_code": 81, "type": "tanker"}, "244700908": {"_updated": 1772687810.929097, "lat": 51.53488, "lng": 6.117226666666666, "sog": 7.1, "cog": 160.1, "heading": 160.1, "name": "LUCIENNE-D"}, "477997316": {"_updated": 1772687802.3148613, "lat": 22.32745, "lng": 114.14555999999999, "sog": 0.4, "cog": 352.9, "heading": 352.9, "name": "WAH FU NO. 108", "callsign": "VRS6539", "imo": 0, "destination": "", "ais_type_code": 80, "type": "tanker"}, "244820279": {"_updated": 1772687781.5231802, "lat": 52.84115833333333, "lng": 5.706255, "sog": 0, "cog": 263.2, "heading": 263.2, "name": "GERO B", "callsign": "PA6953", "imo": 0, "destination": "", "ais_type_code": 79, "type": "cargo"}, "235065995": {"_updated": 1772687806.2401736, "lat": 50.795631666666665, "lng": -1.1088083333333332, "sog": 0, "cog": 329.6, "heading": 329.6, "name": "ALBION", "callsign": "MCXY3", "imo": 0, "destination": "", "ais_tLine truncated
|
||||
{"211257820": {"_updated": 1773005951.5752046, "lat": 53.536725000000004, "lng": 9.878601666666667, "sog": 0, "cog": 110, "heading": 137, "name": "ST.PAULI"}, "232059853": {"_updated": 1773005960.6712666, "lat": 50.56957166666667, "lng": -2.43728, "sog": 0, "cog": 311, "heading": 311, "name": "WYKE CASTLE"}, "211759860": {"_updated": 1773005912.3523576, "lat": 52.52735833333334, "lng": 13.29661, "sog": 0, "cog": 360, "heading": 360, "name": "KREUZ-AS"}, "244740408": {"_updated": 1773005873.9496202, "lat": 53.05554166666666, "lng": 8.873066666666666, "sog": 0, "cog": 222, "heading": 339, "name": "ISANDRA"}, "244700604": {"_updated": 1773005869.079398, "name": "KVB JABO", "callsign": "PF9258", "imo": 0, "destination": "FRKHLVN188HRBAR02969", "ais_type_code": 70, "type": "cargo", "lat": 52.39876666666667, "lng": 4.8605, "sog": 0, "cog": 249.5, "heading": 214}, "259046990": {"_updated": 1773005871.8398657, "lat": 60.43211833333333, "lng": 5.238841666666667, "sog": 0, "cog": 175.1, "heading": 175, "name": "NANO"}, "477937500": {"_updated": 1773005933.12507, "lat": 22.327023333333333, "lng": 114.14463166666665, "sog": 0, "cog": 229.1, "heading": 123, "name": "THE PARISIAN"}, "244740537": {"_updated": 1773005933.1848793, "lat": 51.562108333333335, "lng": 3.695911666666667, "sog": 0, "cog": 360, "heading": 360, "name": "ALBATROS"}, "730151806": {"_updated": 1773005948.7997103, "lat": 10.883998333333333, "lng": -74.73919500000001, "sog": 0, "cog": 360, "heading": 170, "name": "IMPALA PUERTO BERRIO"}, "212136000": {"_updated": 1773005872.4314587, "name": "ALSTERDIEP", "callsign": "5BKK2", "imo": 9472000, "destination": "ESVLC", "ais_type_code": 70, "type": "cargo", "lat": 39.45379833333333, "lng": -0.31999666666666665, "sog": 0, "cog": 322.6, "heading": 329}, "538008236": {"_updated": 1773005953.9659784, "name": "TRADE", "callsign": "V7A2120", "imo": 9425942, "destination": "BG VAR", "ais_type_code": 70, "type": "cargo", "lat": 43.18985833333333, "lng": 27.901678333333333, "sog": 0, "cog": 322.9, "heading": 322.9}, "261184100": {"_updated": 1773005954.8008614, "lat": 52.71782333333333, "lng": 9.192773333333333, "sog": 0, "cog": 360, "heading": 360, "name": "GABI", "callsign": "SR4100", "imo": 0, "destination": "LEESERINGEN", "ais_type_code": 99, "type": "unknown"}, "232002653": {"_updated": 1773005871.236972, "lat": 57.279095, "lng": -5.708236666666666, "sog": 0, "cog": 356.9, "heading": 77, "name": "SD WARDEN"}, "205393590": {"_updated": 1773005951.3364427, "lat": 51.23241666666667, "lng": 2.92374, "sog": 0, "cog": 360, "heading": 360, "name": "DE NIEUWE VISIE"}, "338926402": {"_updated": 1773005874.1483874, "lat": 25.770353333333333, "lng": -80.14540166666667, "sog": 0, "cog": 339.6, "heading": 327, "name": "RICHARD ETHERIDGE"}, "229365000": {"_updated": 1773005870.9735918, "lat": 52.420703333333336, "lng": 4.750003333333334, "sog": 0, "cog": 173.6, "heading": 350, "name": "SEAPRIDE", "callsign": "9HA3272", "imo": 9629562, "destination": "NLAMS", "ais_type_code": 81, "type": "tanker"}, "244660131": {"_updated": 1773005919.309042, "name": "BIESBOSCH", "callsign": "PBOO", "imo": 0, "destination": "ROTTERDAM", "ais_type_code": 33, "type": "other", "lat": 51.688586666666666, "lng": 4.59424, "sog": 0, "cog": 360, "heading": 360}, "211754860": {"_updated": 1773005918.3653634, "name": "LOESCHBOOT III", "callsign": "DC2894", "imo": 0, "destination": "", "ais_type_code": 99, "type": "unknown", "lat": 52.533946666666665, "lng": 13.218406666666668, "sog": 0, "cog": 360, "heading": 360}, "210495000": {"_updated": 1773005896.519025, "lat": 46.885795, "lng": -2.449698333333333, "sog": 0.1, "cog": 86.9, "heading": 251, "name": "IWS WINDWALKER", "callsign": "5BBL6", "imo": 9958638, "destination": "NOY OWF", "ais_type_code": 90, "type": "unknown"}, "244070453": {"_updated": 1773005912.083425, "lat": 52.38550333333333, "lng": 4.890680000000001, "sog": 0, "cog": 360, "heading": 360, "name": "JACKSON 3"}, "244710818": {"_updated": 1773005931.4982648, "lat": 52.39692, "lng": 4.859489999999999, "sog": 0, "cog": 59.9, "heading": 59.9, "name": "WATERMAN", "callsign": "PI3808", "imo": 0, "destination": "AMSTERDAM", "ais_type_code": 52, "type": "other"}, "563275700": {"_updated": 1773005933.1140635, "lat": 1.2647650000000001, "lng": 103.86371000000001, "sog": 2.6, "cog": 328, "heading": 162, "name": "MARINA AMETHYST"}, "219035687": {"_updated": 1773005941.822461, "lat": 54.7509, "lng": 10.670819999999999, "sog": 0, "cog": 189.6, "heading": 226, "name": "KINGFISH"}, "209731000": {"_updated": 1773005947.2841218, "lat": 60.44249833333333, "lng": 22.218708333333332, "sog": 0.5, "cog": 6.3, "heading": 323, "name": "DELAMAR", "callsign": "5BJJ5", "imo": 9528500, "destination": "FITKU", "ais_type_code": 70, "type": "cargo"}, "368258230": {"_updated": 1773005932.3698566, "lat": 39.37461166666667, "lng": -81.29741333333332, "sog": 0, "cog": 360, "heading": 360, "name": "NATALIE JANE"}, "367691190": {"_updated": 1773005933.3683534, "lat": 30.0095983Line truncated
|
||||
@@ -211,9 +211,10 @@ def _ais_stream_loop():
|
||||
"""Main loop: spawn node proxy and process messages from stdout."""
|
||||
import subprocess
|
||||
import os
|
||||
|
||||
|
||||
proxy_script = os.path.join(os.path.dirname(os.path.dirname(__file__)), "ais_proxy.js")
|
||||
|
||||
backoff = 1 # Exponential backoff starting at 1 second
|
||||
|
||||
while _ws_running:
|
||||
try:
|
||||
logger.info("Starting Node.js AIS Stream Proxy...")
|
||||
@@ -323,8 +324,12 @@ def _ais_stream_loop():
|
||||
except Exception as e:
|
||||
logger.error(f"AIS proxy connection error: {e}")
|
||||
if _ws_running:
|
||||
logger.info("Restarting AIS proxy in 5 seconds...")
|
||||
time.sleep(5)
|
||||
logger.info(f"Restarting AIS proxy in {backoff}s (exponential backoff)...")
|
||||
time.sleep(backoff)
|
||||
backoff = min(backoff * 2, 60) # Double up to 60s max
|
||||
continue
|
||||
# Reset backoff on successful connection (got at least some messages)
|
||||
backoff = 1
|
||||
|
||||
|
||||
def _run_ais_loop():
|
||||
|
||||
@@ -72,8 +72,8 @@ class OpenSkyClient:
|
||||
|
||||
# User provided credentials
|
||||
opensky_client = OpenSkyClient(
|
||||
client_id=os.environ.get("OPENSKY_CLIENT_ID", "vancecook-api-client"),
|
||||
client_secret=os.environ.get("OPENSKY_CLIENT_SECRET", "YOUR_OPENSKY_SECRET")
|
||||
client_id=os.environ.get("OPENSKY_CLIENT_ID", ""),
|
||||
client_secret=os.environ.get("OPENSKY_CLIENT_SECRET", "")
|
||||
)
|
||||
|
||||
# Throttling and caching for OpenSky to observe the 400 req/day limit
|
||||
@@ -885,9 +885,10 @@ def fetch_flights():
|
||||
by_icao[id(f)] = f # no icao — keep as unique
|
||||
return list(by_icao.values())
|
||||
|
||||
latest_data['commercial_flights'] = _merge_category(commercial, latest_data.get('commercial_flights', []))
|
||||
latest_data['private_jets'] = _merge_category(private_jets, latest_data.get('private_jets', []))
|
||||
latest_data['private_flights'] = _merge_category(private_ga, latest_data.get('private_flights', []))
|
||||
with _data_lock:
|
||||
latest_data['commercial_flights'] = _merge_category(commercial, latest_data.get('commercial_flights', []))
|
||||
latest_data['private_jets'] = _merge_category(private_jets, latest_data.get('private_jets', []))
|
||||
latest_data['private_flights'] = _merge_category(private_ga, latest_data.get('private_flights', []))
|
||||
|
||||
# Always write raw flights for GPS jamming analysis (nac_p field)
|
||||
if flights:
|
||||
@@ -964,27 +965,39 @@ def fetch_flights():
|
||||
all_lists = [commercial, private_jets, private_ga, existing_tracked]
|
||||
seen_hexes = set()
|
||||
trail_count = 0
|
||||
for flist in all_lists:
|
||||
for f in flist:
|
||||
count, hex_id = _accumulate_trail(f, now_ts, check_route=True)
|
||||
with _trails_lock:
|
||||
for flist in all_lists:
|
||||
for f in flist:
|
||||
count, hex_id = _accumulate_trail(f, now_ts, check_route=True)
|
||||
trail_count += count
|
||||
if hex_id:
|
||||
seen_hexes.add(hex_id)
|
||||
|
||||
# Also process military flights (separate list)
|
||||
for mf in latest_data.get('military_flights', []):
|
||||
count, hex_id = _accumulate_trail(mf, now_ts, check_route=False)
|
||||
trail_count += count
|
||||
if hex_id:
|
||||
seen_hexes.add(hex_id)
|
||||
|
||||
# Also process military flights (separate list)
|
||||
for mf in latest_data.get('military_flights', []):
|
||||
count, hex_id = _accumulate_trail(mf, now_ts, check_route=False)
|
||||
trail_count += count
|
||||
if hex_id:
|
||||
seen_hexes.add(hex_id)
|
||||
|
||||
# Prune trails for aircraft not seen in 30 minutes
|
||||
stale_cutoff = now_ts - 1800
|
||||
stale_keys = [k for k, v in flight_trails.items() if v['last_seen'] < stale_cutoff]
|
||||
for k in stale_keys:
|
||||
del flight_trails[k]
|
||||
|
||||
logger.info(f"Trail accumulation: {trail_count} active trails, {len(stale_keys)} pruned")
|
||||
|
||||
# Prune stale trails (10 min for non-tracked, 30 min for tracked)
|
||||
tracked_hexes = {t.get('icao24', '').lower() for t in latest_data.get('tracked_flights', [])}
|
||||
stale_keys = []
|
||||
for k, v in flight_trails.items():
|
||||
cutoff = now_ts - 1800 if k in tracked_hexes else now_ts - 600
|
||||
if v['last_seen'] < cutoff:
|
||||
stale_keys.append(k)
|
||||
for k in stale_keys:
|
||||
del flight_trails[k]
|
||||
|
||||
# Enforce global cap — evict oldest trails first
|
||||
if len(flight_trails) > _MAX_TRACKED_TRAILS:
|
||||
sorted_keys = sorted(flight_trails.keys(), key=lambda k: flight_trails[k]['last_seen'])
|
||||
evict_count = len(flight_trails) - _MAX_TRACKED_TRAILS
|
||||
for k in sorted_keys[:evict_count]:
|
||||
del flight_trails[k]
|
||||
|
||||
logger.info(f"Trail accumulation: {trail_count} active trails, {len(stale_keys)} pruned, {len(flight_trails)} total")
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# GPS / GNSS Jamming Detection — aggregate NACp from ADS-B transponders
|
||||
@@ -1567,6 +1580,8 @@ def fetch_uavs():
|
||||
|
||||
cached_airports = []
|
||||
flight_trails = {} # {icao_hex: {points: [[lat, lng, alt, ts], ...], last_seen: ts}}
|
||||
_trails_lock = threading.Lock()
|
||||
_MAX_TRACKED_TRAILS = 2000 # Global cap on number of aircraft trails in memory
|
||||
|
||||
# (math imported at module top)
|
||||
|
||||
@@ -1751,5 +1766,6 @@ def stop_scheduler():
|
||||
scheduler.shutdown()
|
||||
|
||||
def get_latest_data():
|
||||
return latest_data
|
||||
with _data_lock:
|
||||
return dict(latest_data)
|
||||
|
||||
@@ -3,10 +3,19 @@ import json
|
||||
import subprocess
|
||||
import shutil
|
||||
import time
|
||||
import requests
|
||||
from urllib.parse import urlparse
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Reusable session with connection pooling and retry logic
|
||||
_session = requests.Session()
|
||||
_retry = Retry(total=2, backoff_factor=0.5, status_forcelist=[502, 503, 504])
|
||||
_session.mount("https://", HTTPAdapter(max_retries=_retry, pool_maxsize=20))
|
||||
_session.mount("http://", HTTPAdapter(max_retries=_retry, pool_maxsize=10))
|
||||
|
||||
# Find bash for curl fallback — Git bash's curl has the TLS features
|
||||
# needed to pass CDN fingerprint checks (brotli, zstd, libpsl)
|
||||
_BASH_PATH = shutil.which("bash") or "bash"
|
||||
@@ -50,11 +59,10 @@ def fetch_with_curl(url, method="GET", json_data=None, timeout=15, headers=None)
|
||||
pass # Fall through to curl below
|
||||
else:
|
||||
try:
|
||||
import requests
|
||||
if method == "POST":
|
||||
res = requests.post(url, json=json_data, timeout=timeout, headers=default_headers)
|
||||
res = _session.post(url, json=json_data, timeout=timeout, headers=default_headers)
|
||||
else:
|
||||
res = requests.get(url, timeout=timeout, headers=default_headers)
|
||||
res = _session.get(url, timeout=timeout, headers=default_headers)
|
||||
res.raise_for_status()
|
||||
# Clear failure cache on success
|
||||
_domain_fail_cache.pop(domain, None)
|
||||
|
||||
Reference in new issue
Block a user