mirror of
https://github.com/BigBodyCobain/Shadowbroker.git
synced 2026-09-22 17:00:52 +02:00
v0.9.0: in-app auto-updater, ship toggle split, stable entity IDs, performance fixes
New features: - In-app auto-updater with confirmation dialog, manual download fallback, restart polling, and protected file safety net - Ship layers split into 4 independent toggles (Military/Carriers, Cargo/Tankers, Civilian, Cruise/Passenger) with per-category counts - Stable entity IDs using MMSI/callsign instead of volatile array indices - Dismissible threat alert bubbles (session-scoped, survives data refresh) Performance: - GDELT title fetching is now non-blocking (background enrichment) - Removed duplicate startup fetch jobs - Docker healthcheck start_period 15s → 90s Bug fixes: - Removed fake intelligence assessment generator (OSINT-only policy) - Fixed carrier tracker GDELT 429/TypeError crash - Fixed ETag collision (full payload hash) - Added concurrent /api/refresh guard Contributors: @imqdcr (ship split + stable IDs), @csysp (dismissible alerts, PR #48) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> Former-commit-id: a2c4c67da54345393f70a9b33b52e7e4fd6c049f
This commit is contained in:
anoracleofra-code
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
1eb2b21647
commit
fc9eff865e
+17
-1
@@ -4,13 +4,29 @@ __pycache__/
|
||||
.env
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
.git/
|
||||
node_modules/
|
||||
cctv.db
|
||||
*.sqlite
|
||||
*.db
|
||||
|
||||
# Debug/log files
|
||||
*.txt
|
||||
!requirements.txt
|
||||
# Exclude debug/cache JSON but keep package.json and tracked_names
|
||||
!requirements-dev.txt
|
||||
*.html
|
||||
*.xlsx
|
||||
|
||||
# Debug/cache JSON (keep package*.json and data files)
|
||||
ais_cache.json
|
||||
carrier_cache.json
|
||||
carrier_positions.json
|
||||
dump.json
|
||||
debug_fast.json
|
||||
nyc_full.json
|
||||
nyc_sample.json
|
||||
tmp_fast.json
|
||||
|
||||
# Test files (not needed in production image)
|
||||
test_*.py
|
||||
tests/
|
||||
|
||||
+3
-2
@@ -11,12 +11,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
|
||||
# Install Python dependencies
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
RUN pip install --no-cache-dir -r requirements.txt \
|
||||
&& playwright install --with-deps chromium
|
||||
|
||||
# Install Node.js dependencies (ws module for AIS WebSocket proxy)
|
||||
# Copy manifests first so this layer is cached unless deps change
|
||||
COPY package*.json ./
|
||||
RUN npm install --omit=dev
|
||||
RUN npm ci --omit=dev
|
||||
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
import requests
|
||||
|
||||
regions = [
|
||||
{"lat": 39.8, "lon": -98.5, "dist": 2000}, # USA
|
||||
{"lat": 50.0, "lon": 15.0, "dist": 2000}, # Europe
|
||||
{"lat": 35.0, "lon": 105.0, "dist": 2000} # Asia / China
|
||||
]
|
||||
|
||||
for r in regions:
|
||||
url = f"https://api.adsb.lol/v2/lat/{r['lat']}/lon/{r['lon']}/dist/{r['dist']}"
|
||||
res = requests.get(url, timeout=10)
|
||||
if res.status_code == 200:
|
||||
data = res.json()
|
||||
acs = data.get("ac", [])
|
||||
print(f"Region lat:{r['lat']} lon:{r['lon']} dist:{r['dist']} -> Flights: {len(acs)}")
|
||||
else:
|
||||
print(f"Error for Region lat:{r['lat']} lon:{r['lon']}: HTTP {res.status_code}")
|
||||
@@ -1,10 +0,0 @@
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
db_path = os.path.join(os.path.dirname(__file__), 'cctv.db')
|
||||
conn = sqlite3.connect(db_path)
|
||||
cur = conn.cursor()
|
||||
cur.execute("DELETE FROM cameras WHERE id LIKE 'OSM-%'")
|
||||
print(f"Deleted {cur.rowcount} OSM cameras from DB.")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
File diff suppressed because one or more lines are too long
@@ -1,25 +0,0 @@
|
||||
import re
|
||||
import json
|
||||
|
||||
try:
|
||||
with open('liveua_test.html', 'r', encoding='utf-8') as f:
|
||||
html = f.read()
|
||||
|
||||
m = re.search(r"var\s+ovens\s*=\s*(.*?);(?!function)", html, re.DOTALL)
|
||||
if m:
|
||||
json_str = m.group(1)
|
||||
# Handle if it is a string containing base64
|
||||
if json_str.startswith("'") or json_str.startswith('"'):
|
||||
json_str = json_str.strip('"\'')
|
||||
import base64
|
||||
import urllib.parse
|
||||
json_str = base64.b64decode(urllib.parse.unquote(json_str)).decode('utf-8')
|
||||
|
||||
data = json.loads(json_str)
|
||||
with open('out_liveua.json', 'w', encoding='utf-8') as f:
|
||||
json.dump(data, f, indent=2)
|
||||
print(f"Successfully extracted {len(data)} ovens items.")
|
||||
else:
|
||||
print("var ovens not found.")
|
||||
except Exception as e:
|
||||
print("Error:", e)
|
||||
@@ -1,166 +0,0 @@
|
||||
"""
|
||||
Geocode data center street addresses via Nominatim (OpenStreetMap).
|
||||
Rate limit: 1 request/second (Nominatim policy).
|
||||
Resumable: caches results in geocode_cache.json so interrupted runs can continue.
|
||||
"""
|
||||
import json
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Fix Windows console encoding + force unbuffered output
|
||||
if sys.platform == "win32":
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||||
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
|
||||
|
||||
# Force line-buffered stdout for detached processes
|
||||
class Unbuffered:
|
||||
def __init__(self, stream):
|
||||
self.stream = stream
|
||||
def write(self, data):
|
||||
self.stream.write(data)
|
||||
self.stream.flush()
|
||||
def writelines(self, datas):
|
||||
self.stream.writelines(datas)
|
||||
self.stream.flush()
|
||||
def __getattr__(self, attr):
|
||||
return getattr(self.stream, attr)
|
||||
|
||||
sys.stdout = Unbuffered(sys.stdout)
|
||||
|
||||
DATA_FILE = os.path.join(os.path.dirname(__file__), "data", "datacenters.json")
|
||||
CACHE_FILE = os.path.join(os.path.dirname(__file__), "data", "geocode_cache.json")
|
||||
OUTPUT_FILE = os.path.join(os.path.dirname(__file__), "data", "datacenters_geocoded.json")
|
||||
|
||||
NOMINATIM_URL = "https://nominatim.openstreetmap.org/search"
|
||||
USER_AGENT = "ShadowBroker-DataCenterGeocoder/1.0"
|
||||
|
||||
|
||||
def geocode_address(address: str, retries: int = 3) -> tuple[float, float] | None:
|
||||
"""Geocode a single address via Nominatim. Returns (lat, lng) or None."""
|
||||
params = urllib.parse.urlencode({"q": address, "format": "json", "limit": 1})
|
||||
url = f"{NOMINATIM_URL}?{params}"
|
||||
for attempt in range(retries):
|
||||
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=15)
|
||||
data = json.loads(resp.read())
|
||||
if data:
|
||||
return float(data[0]["lat"]), float(data[0]["lon"])
|
||||
return None # Valid response but no results
|
||||
except Exception as e:
|
||||
if attempt < retries - 1:
|
||||
wait = 2 ** (attempt + 1)
|
||||
print(f" RETRY ({attempt+1}/{retries}): {e} — waiting {wait}s")
|
||||
time.sleep(wait)
|
||||
else:
|
||||
print(f" ERROR (gave up after {retries} attempts): {e}")
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
with open(DATA_FILE, "r", encoding="utf-8") as f:
|
||||
dcs = json.load(f)
|
||||
|
||||
# Load cache
|
||||
cache = {}
|
||||
if os.path.exists(CACHE_FILE):
|
||||
with open(CACHE_FILE, "r", encoding="utf-8") as f:
|
||||
cache = json.load(f)
|
||||
print(f"Loaded {len(cache)} cached geocode results")
|
||||
|
||||
# Filter to DCs with real street addresses
|
||||
to_geocode = []
|
||||
skipped = 0
|
||||
for i, dc in enumerate(dcs):
|
||||
street = (dc.get("street") or "").strip()
|
||||
if not street or len(street) <= 3 or street.lower() in ("tbc", "n/a", "na", "-"):
|
||||
skipped += 1
|
||||
continue
|
||||
to_geocode.append((i, dc))
|
||||
|
||||
print(f"Total DCs: {len(dcs)}")
|
||||
print(f"Skipped (no real address): {skipped}")
|
||||
print(f"To geocode: {len(to_geocode)}")
|
||||
|
||||
# Count how many already cached
|
||||
already_cached = sum(1 for _, dc in to_geocode if dc.get("address", "") in cache)
|
||||
need_api = len(to_geocode) - already_cached
|
||||
print(f"Already cached: {already_cached}")
|
||||
print(f"Need API calls: {need_api}")
|
||||
if need_api > 0:
|
||||
print(f"Estimated time: {need_api // 60}m {need_api % 60}s")
|
||||
print()
|
||||
|
||||
geocoded = 0
|
||||
failed = 0
|
||||
api_calls = 0
|
||||
save_interval = 50 # Save cache every 50 API calls
|
||||
|
||||
for idx, (i, dc) in enumerate(to_geocode):
|
||||
address = dc.get("address", "").strip()
|
||||
if not address:
|
||||
# Build address from parts
|
||||
parts = [dc.get("street", ""), dc.get("zip", ""), dc.get("city", ""), dc.get("country", "")]
|
||||
address = " ".join(p.strip() for p in parts if p and p.strip())
|
||||
|
||||
if not address:
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
# Check cache first
|
||||
if address in cache:
|
||||
result = cache[address]
|
||||
if result:
|
||||
dcs[i]["lat"] = result[0]
|
||||
dcs[i]["lng"] = result[1]
|
||||
dcs[i]["geocode_source"] = "nominatim"
|
||||
geocoded += 1
|
||||
else:
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
# API call — Nominatim requires 1 req/s, use 1.5s to avoid 429s after heavy use
|
||||
time.sleep(1.5)
|
||||
coords = geocode_address(address)
|
||||
api_calls += 1
|
||||
|
||||
if coords:
|
||||
cache[address] = coords
|
||||
dcs[i]["lat"] = coords[0]
|
||||
dcs[i]["lng"] = coords[1]
|
||||
dcs[i]["geocode_source"] = "nominatim"
|
||||
geocoded += 1
|
||||
print(f"[{api_calls}/{need_api}] OK: {dc.get('name', '?')} -> ({coords[0]:.4f}, {coords[1]:.4f})")
|
||||
else:
|
||||
cache[address] = None
|
||||
failed += 1
|
||||
print(f"[{api_calls}/{need_api}] FAIL: {dc.get('name', '?')} | {address}")
|
||||
|
||||
# Periodic cache save
|
||||
if api_calls % save_interval == 0:
|
||||
with open(CACHE_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump(cache, f)
|
||||
print(f" -- Cache saved ({len(cache)} entries) --")
|
||||
|
||||
# Final save
|
||||
with open(CACHE_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump(cache, f)
|
||||
|
||||
# Write output - only DCs with real coordinates
|
||||
output = [dc for dc in dcs if dc.get("lat") is not None and dc.get("lng") is not None]
|
||||
|
||||
with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
|
||||
json.dump(output, f, indent=2)
|
||||
|
||||
print(f"\nDone!")
|
||||
print(f"Geocoded: {geocoded}")
|
||||
print(f"Failed: {failed}")
|
||||
print(f"API calls made: {api_calls}")
|
||||
print(f"Output: {len(output)} DCs with coordinates -> {OUTPUT_FILE}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+94
-13
@@ -35,17 +35,22 @@ for _var in _SECRET_VARS:
|
||||
except Exception as _e:
|
||||
logger.error(f"Failed to read secret file {_file_path} for {_var}: {_e}")
|
||||
|
||||
from fastapi import FastAPI, Request, Response
|
||||
from fastapi import FastAPI, Request, Response, Query
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from contextlib import asynccontextmanager
|
||||
from services.data_fetcher import start_scheduler, stop_scheduler, get_latest_data, source_timestamps
|
||||
from services.ais_stream import start_ais_stream, stop_ais_stream
|
||||
from services.carrier_tracker import start_carrier_tracker, stop_carrier_tracker
|
||||
from slowapi import Limiter, _rate_limit_exceeded_handler
|
||||
from slowapi.util import get_remote_address
|
||||
from slowapi.errors import RateLimitExceeded
|
||||
import uvicorn
|
||||
import hashlib
|
||||
import json as json_mod
|
||||
import socket
|
||||
|
||||
limiter = Limiter(key_func=get_remote_address)
|
||||
|
||||
|
||||
def _build_cors_origins():
|
||||
"""Build a CORS origins whitelist: localhost + LAN IPs + env overrides.
|
||||
@@ -74,10 +79,32 @@ def _build_cors_origins():
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
# Startup: Start background data fetching, AIS stream, and carrier tracker
|
||||
start_carrier_tracker()
|
||||
import threading
|
||||
|
||||
# Start AIS stream first — it loads the disk cache (instant ships) then
|
||||
# begins accumulating live vessel data via WebSocket in the background.
|
||||
start_ais_stream()
|
||||
|
||||
# Carrier tracker runs its own initial update_carrier_positions() internally
|
||||
# in _scheduler_loop, so we do NOT call it again in the preload thread.
|
||||
start_carrier_tracker()
|
||||
|
||||
# Start the recurring scheduler (fast=60s, slow=30min).
|
||||
start_scheduler()
|
||||
|
||||
# Kick off the full data preload in a background thread so the server
|
||||
# is listening on port 8000 instantly. The frontend's adaptive polling
|
||||
# (retries every 3s) will pick up data piecemeal as each fetcher finishes.
|
||||
def _background_preload():
|
||||
logger.info("=== PRELOADING DATA (background — server already accepting requests) ===")
|
||||
try:
|
||||
update_all_data()
|
||||
logger.info("=== PRELOAD COMPLETE ===")
|
||||
except Exception as e:
|
||||
logger.error(f"Data preload failed (non-fatal): {e}")
|
||||
|
||||
threading.Thread(target=_background_preload, daemon=True).start()
|
||||
|
||||
yield
|
||||
# Shutdown: Stop all background services
|
||||
stop_ais_stream()
|
||||
@@ -85,6 +112,8 @@ async def lifespan(app: FastAPI):
|
||||
stop_carrier_tracker()
|
||||
|
||||
app = FastAPI(title="Live Risk Dashboard API", lifespan=lifespan)
|
||||
app.state.limiter = limiter
|
||||
app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
|
||||
|
||||
from fastapi.middleware.gzip import GZipMiddleware
|
||||
app.add_middleware(GZipMiddleware, minimum_size=1000)
|
||||
@@ -98,11 +127,23 @@ app.add_middleware(
|
||||
|
||||
from services.data_fetcher import update_all_data
|
||||
|
||||
_refresh_in_progress = False
|
||||
|
||||
@app.get("/api/refresh")
|
||||
async def force_refresh():
|
||||
# Force an immediate synchronous update of the data payload
|
||||
@limiter.limit("2/minute")
|
||||
async def force_refresh(request: Request):
|
||||
global _refresh_in_progress
|
||||
if _refresh_in_progress:
|
||||
return {"status": "refresh already in progress"}
|
||||
import threading
|
||||
t = threading.Thread(target=update_all_data)
|
||||
def _do_refresh():
|
||||
global _refresh_in_progress
|
||||
try:
|
||||
update_all_data()
|
||||
finally:
|
||||
_refresh_in_progress = False
|
||||
_refresh_in_progress = True
|
||||
t = threading.Thread(target=_do_refresh)
|
||||
t.start()
|
||||
return {"status": "refreshing in background"}
|
||||
|
||||
@@ -113,13 +154,14 @@ async def live_data():
|
||||
def _etag_response(request: Request, payload: dict, prefix: str = "", default=None):
|
||||
"""Serialize once, hash the bytes for ETag, return 304 or full response."""
|
||||
content = json_mod.dumps(payload, default=default)
|
||||
etag = hashlib.md5(f"{prefix}{content[:256]}".encode()).hexdigest()[:16]
|
||||
etag = hashlib.md5(f"{prefix}{content}".encode()).hexdigest()[:16]
|
||||
if request.headers.get("if-none-match") == etag:
|
||||
return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
|
||||
return Response(content=content, media_type="application/json",
|
||||
headers={"ETag": etag, "Cache-Control": "no-cache"})
|
||||
|
||||
@app.get("/api/live-data/fast")
|
||||
@limiter.limit("120/minute")
|
||||
async def live_data_fast(request: Request):
|
||||
d = get_latest_data()
|
||||
payload = {
|
||||
@@ -140,6 +182,7 @@ async def live_data_fast(request: Request):
|
||||
return _etag_response(request, payload, prefix="fast|")
|
||||
|
||||
@app.get("/api/live-data/slow")
|
||||
@limiter.limit("60/minute")
|
||||
async def live_data_slow(request: Request):
|
||||
d = get_latest_data()
|
||||
payload = {
|
||||
@@ -210,13 +253,20 @@ async def api_get_openmhz_calls(sys_name: str):
|
||||
return get_recent_openmhz_calls(sys_name)
|
||||
|
||||
@app.get("/api/radio/nearest")
|
||||
async def api_get_nearest_radio(lat: float, lng: float):
|
||||
async def api_get_nearest_radio(
|
||||
lat: float = Query(..., ge=-90, le=90),
|
||||
lng: float = Query(..., ge=-180, le=180),
|
||||
):
|
||||
return find_nearest_openmhz_system(lat, lng)
|
||||
|
||||
from services.radio_intercept import find_nearest_openmhz_systems_list
|
||||
|
||||
@app.get("/api/radio/nearest-list")
|
||||
async def api_get_nearest_radios_list(lat: float, lng: float, limit: int = 5):
|
||||
async def api_get_nearest_radios_list(
|
||||
lat: float = Query(..., ge=-90, le=90),
|
||||
lng: float = Query(..., ge=-180, le=180),
|
||||
limit: int = Query(5, ge=1, le=20),
|
||||
):
|
||||
return find_nearest_openmhz_systems_list(lat, lng, limit=limit)
|
||||
|
||||
from services.network_utils import fetch_with_curl
|
||||
@@ -249,14 +299,24 @@ async def get_flight_route(callsign: str, lat: float = 0.0, lng: float = 0.0):
|
||||
from services.region_dossier import get_region_dossier
|
||||
|
||||
@app.get("/api/region-dossier")
|
||||
def api_region_dossier(lat: float, lng: float):
|
||||
@limiter.limit("30/minute")
|
||||
def api_region_dossier(
|
||||
request: Request,
|
||||
lat: float = Query(..., ge=-90, le=90),
|
||||
lng: float = Query(..., ge=-180, le=180),
|
||||
):
|
||||
"""Sync def so FastAPI runs it in a threadpool — prevents blocking the event loop."""
|
||||
return get_region_dossier(lat, lng)
|
||||
|
||||
from services.sentinel_search import search_sentinel2_scene
|
||||
|
||||
@app.get("/api/sentinel2/search")
|
||||
def api_sentinel2_search(lat: float, lng: float):
|
||||
@limiter.limit("30/minute")
|
||||
def api_sentinel2_search(
|
||||
request: Request,
|
||||
lat: float = Query(..., ge=-90, le=90),
|
||||
lng: float = Query(..., ge=-180, le=180),
|
||||
):
|
||||
"""Search for latest Sentinel-2 imagery at a point. Sync for threadpool execution."""
|
||||
return search_sentinel2_scene(lat, lng)
|
||||
|
||||
@@ -309,7 +369,28 @@ async def api_reset_news_feeds():
|
||||
return {"status": "reset", "feeds": get_feeds()}
|
||||
return {"status": "error", "message": "Failed to reset feeds"}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# System — self-update
|
||||
# ---------------------------------------------------------------------------
|
||||
from pathlib import Path
|
||||
from services.updater import perform_update, schedule_restart
|
||||
|
||||
@app.post("/api/system/update")
|
||||
@limiter.limit("1/minute")
|
||||
async def system_update(request: Request):
|
||||
"""Download latest release, backup current files, extract update, and restart."""
|
||||
project_root = str(Path(__file__).resolve().parent.parent)
|
||||
result = perform_update(project_root)
|
||||
if result.get("status") == "error":
|
||||
return Response(
|
||||
content=json_mod.dumps(result),
|
||||
status_code=500,
|
||||
media_type="application/json",
|
||||
)
|
||||
# Schedule restart AFTER response flushes (2s delay)
|
||||
import threading
|
||||
threading.Timer(2.0, schedule_restart, args=[project_root]).start()
|
||||
return result
|
||||
|
||||
if __name__ == "__main__":
|
||||
uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=True)
|
||||
|
||||
# Application successfully initialized with background scraping tasks
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
[pytest]
|
||||
testpaths = tests
|
||||
python_files = test_*.py
|
||||
python_functions = test_*
|
||||
@@ -0,0 +1,3 @@
|
||||
-r requirements.txt
|
||||
pytest==8.3.4
|
||||
httpx==0.28.1
|
||||
+19
-17
@@ -1,20 +1,22 @@
|
||||
fastapi>=0.103.1
|
||||
uvicorn>=0.23.2
|
||||
yfinance>=0.2.40
|
||||
fastapi==0.115.12
|
||||
uvicorn==0.34.0
|
||||
yfinance==0.2.54
|
||||
feedparser==6.0.10
|
||||
legacy-cgi>=2.6
|
||||
legacy-cgi==2.6.2
|
||||
requests==2.31.0
|
||||
apscheduler==3.10.3
|
||||
pydantic>=2.3.0
|
||||
pydantic-settings>=2.0.3
|
||||
playwright>=1.58.0
|
||||
beautifulsoup4>=4.12.0
|
||||
cachetools>=5.3
|
||||
cloudscraper>=1.2.71
|
||||
python-dotenv>=1.0
|
||||
lxml>=5.0
|
||||
reverse_geocoder>=1.5
|
||||
sgp4>=2.23
|
||||
geopy>=2.4.0
|
||||
pytz>=2023.3
|
||||
pystac-client>=0.7.0
|
||||
pydantic==2.11.1
|
||||
pydantic-settings==2.8.1
|
||||
playwright==1.50.0
|
||||
playwright-stealth==1.0.6
|
||||
beautifulsoup4==4.13.3
|
||||
cachetools==5.5.2
|
||||
slowapi==0.1.9
|
||||
cloudscraper==1.2.71
|
||||
python-dotenv==1.0.1
|
||||
lxml==5.3.1
|
||||
reverse_geocoder==1.5.1
|
||||
sgp4==2.23
|
||||
geopy==2.4.1
|
||||
pytz==2024.2
|
||||
pystac-client==0.8.6
|
||||
|
||||
@@ -144,7 +144,7 @@ def _save_cache():
|
||||
with open(CACHE_FILE, 'w') as f:
|
||||
json.dump(data, f)
|
||||
logger.info(f"AIS cache saved: {len(data)} vessels")
|
||||
except Exception as e:
|
||||
except (IOError, OSError) as e:
|
||||
logger.error(f"Failed to save AIS cache: {e}")
|
||||
|
||||
|
||||
@@ -165,7 +165,7 @@ def _load_cache():
|
||||
_vessels[int(k)] = v
|
||||
loaded += 1
|
||||
logger.info(f"AIS cache loaded: {loaded} vessels from disk")
|
||||
except Exception as e:
|
||||
except (IOError, OSError, json.JSONDecodeError, ValueError) as e:
|
||||
logger.error(f"Failed to load AIS cache: {e}")
|
||||
|
||||
|
||||
@@ -326,7 +326,7 @@ def _ais_stream_loop():
|
||||
_save_cache()
|
||||
last_log_time = now
|
||||
|
||||
except Exception as e:
|
||||
except (ConnectionError, TimeoutError, OSError, ValueError, KeyError) as e:
|
||||
logger.error(f"AIS proxy connection error: {e}")
|
||||
if _ws_running:
|
||||
logger.info(f"Restarting AIS proxy in {backoff}s (exponential backoff)...")
|
||||
|
||||
@@ -218,7 +218,7 @@ def _load_cache() -> Dict[str, dict]:
|
||||
data = json.loads(CACHE_FILE.read_text())
|
||||
logger.info(f"Carrier cache loaded: {len(data)} carriers from {CACHE_FILE}")
|
||||
return data
|
||||
except Exception as e:
|
||||
except (IOError, OSError, json.JSONDecodeError, ValueError) as e:
|
||||
logger.warning(f"Failed to load carrier cache: {e}")
|
||||
return {}
|
||||
|
||||
@@ -228,7 +228,7 @@ def _save_cache(positions: Dict[str, dict]):
|
||||
try:
|
||||
CACHE_FILE.write_text(json.dumps(positions, indent=2))
|
||||
logger.info(f"Carrier cache saved: {len(positions)} carriers")
|
||||
except Exception as e:
|
||||
except (IOError, OSError) as e:
|
||||
logger.warning(f"Failed to save carrier cache: {e}")
|
||||
|
||||
|
||||
@@ -275,15 +275,15 @@ def _fetch_gdelt_carrier_news() -> List[dict]:
|
||||
try:
|
||||
url = f"https://api.gdeltproject.org/api/v2/doc/doc?query={term}&mode=artlist&maxrecords=5&format=json×pan=14d"
|
||||
raw = fetch_with_curl(url, timeout=8)
|
||||
if not raw:
|
||||
if not raw or not hasattr(raw, 'text'):
|
||||
continue
|
||||
data = json.loads(raw)
|
||||
data = raw.json()
|
||||
articles = data.get("articles", [])
|
||||
for art in articles:
|
||||
title = art.get("title", "")
|
||||
url = art.get("url", "")
|
||||
results.append({"title": title, "url": url})
|
||||
except Exception as e:
|
||||
except (ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e:
|
||||
logger.debug(f"GDELT search failed for '{term}': {e}")
|
||||
continue
|
||||
|
||||
@@ -323,13 +323,8 @@ def _parse_carrier_positions_from_news(articles: List[dict]) -> Dict[str, dict]:
|
||||
return updates
|
||||
|
||||
|
||||
def update_carrier_positions():
|
||||
"""Main update function — called on startup and every 12h."""
|
||||
global _last_update
|
||||
|
||||
logger.info("Carrier tracker: updating positions from OSINT sources...")
|
||||
|
||||
# Start with fallback positions (sourced from USNI News Fleet Tracker)
|
||||
def _load_carrier_fallbacks() -> Dict[str, dict]:
|
||||
"""Build carrier positions from static fallbacks + disk cache (instant, no network)."""
|
||||
positions: Dict[str, dict] = {}
|
||||
for hull, info in CARRIER_REGISTRY.items():
|
||||
positions[hull] = {
|
||||
@@ -344,11 +339,10 @@ def update_carrier_positions():
|
||||
"updated": datetime.now(timezone.utc).isoformat()
|
||||
}
|
||||
|
||||
# Load cached positions (may have better data from previous runs)
|
||||
# Overlay cached positions from previous runs (may have GDELT data)
|
||||
cached = _load_cache()
|
||||
for hull, cached_pos in cached.items():
|
||||
if hull in positions:
|
||||
# Only use cache if it has a real OSINT source (not just static)
|
||||
if cached_pos.get("source", "").startswith("GDELT") or cached_pos.get("source", "").startswith("News"):
|
||||
positions[hull].update({
|
||||
"lat": cached_pos["lat"],
|
||||
@@ -357,8 +351,29 @@ def update_carrier_positions():
|
||||
"source": cached_pos.get("source", "Cached OSINT"),
|
||||
"updated": cached_pos.get("updated", "")
|
||||
})
|
||||
return positions
|
||||
|
||||
# Try GDELT news for fresh positions
|
||||
|
||||
def update_carrier_positions():
|
||||
"""Main update function — called on startup and every 12h.
|
||||
|
||||
Phase 1 (instant): publish fallback + cached positions so the map has carriers immediately.
|
||||
Phase 2 (slow): query GDELT for fresh OSINT positions and update in-place.
|
||||
"""
|
||||
global _last_update
|
||||
|
||||
# --- Phase 1: instant fallback + cache ---
|
||||
positions = _load_carrier_fallbacks()
|
||||
|
||||
with _positions_lock:
|
||||
# Only overwrite if positions are currently empty (first startup).
|
||||
# If we already have data from a previous cycle, keep it while GDELT runs.
|
||||
if not _carrier_positions:
|
||||
_carrier_positions.update(positions)
|
||||
_last_update = datetime.now(timezone.utc)
|
||||
logger.info(f"Carrier tracker: {len(positions)} carriers loaded from fallback/cache (GDELT enrichment starting...)")
|
||||
|
||||
# --- Phase 2: slow GDELT enrichment ---
|
||||
try:
|
||||
articles = _fetch_gdelt_carrier_news()
|
||||
news_positions = _parse_carrier_positions_from_news(articles)
|
||||
@@ -369,7 +384,7 @@ def update_carrier_positions():
|
||||
except Exception as e:
|
||||
logger.warning(f"GDELT carrier fetch failed: {e}")
|
||||
|
||||
# Save and update the global state
|
||||
# Save and update the global state with enriched positions
|
||||
with _positions_lock:
|
||||
_carrier_positions.clear()
|
||||
_carrier_positions.update(positions)
|
||||
|
||||
@@ -41,7 +41,7 @@ class BaseCCTVIngestor(ABC):
|
||||
cursor = self.conn.cursor()
|
||||
for cam in cameras:
|
||||
cursor.execute("""
|
||||
INSERT INTO cameras
|
||||
INSERT INTO cameras
|
||||
(id, source_agency, lat, lon, direction_facing, media_url, refresh_rate_seconds)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
@@ -59,6 +59,10 @@ class BaseCCTVIngestor(ABC):
|
||||
self.conn.commit()
|
||||
logger.info(f"Successfully ingested {len(cameras)} cameras from {self.__class__.__name__}")
|
||||
except Exception as e:
|
||||
try:
|
||||
self.conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
logger.error(f"Failed to ingest cameras in {self.__class__.__name__}: {e}")
|
||||
|
||||
class TFLJamCamIngestor(BaseCCTVIngestor):
|
||||
@@ -220,7 +224,7 @@ class GlobalOSMCrawlingIngestor(BaseCCTVIngestor):
|
||||
direction_str = item.get("tags", {}).get("camera:direction", "0")
|
||||
try:
|
||||
bearing = int(float(direction_str))
|
||||
except:
|
||||
except (ValueError, TypeError):
|
||||
bearing = 0
|
||||
|
||||
mapbox_key = "YOUR_MAPBOX_TOKEN_HERE"
|
||||
|
||||
+175
-1994
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,46 @@
|
||||
"""Shared in-memory data store for all fetcher modules.
|
||||
|
||||
Central location for latest_data, source_timestamps, and the data lock.
|
||||
Every fetcher imports from here instead of maintaining its own copy.
|
||||
"""
|
||||
import threading
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
logger = logging.getLogger("services.data_fetcher")
|
||||
|
||||
# In-memory store
|
||||
latest_data = {
|
||||
"last_updated": None,
|
||||
"news": [],
|
||||
"stocks": {},
|
||||
"oil": {},
|
||||
"flights": [],
|
||||
"ships": [],
|
||||
"military_flights": [],
|
||||
"tracked_flights": [],
|
||||
"cctv": [],
|
||||
"weather": None,
|
||||
"earthquakes": [],
|
||||
"uavs": [],
|
||||
"frontlines": None,
|
||||
"gdelt": [],
|
||||
"liveuamap": [],
|
||||
"kiwisdr": [],
|
||||
"space_weather": None,
|
||||
"internet_outages": [],
|
||||
"firms_fires": [],
|
||||
"datacenters": []
|
||||
}
|
||||
|
||||
# Per-source freshness timestamps
|
||||
source_timestamps = {}
|
||||
|
||||
def _mark_fresh(*keys):
|
||||
"""Record the current UTC time for one or more data source keys."""
|
||||
now = datetime.utcnow().isoformat()
|
||||
for k in keys:
|
||||
source_timestamps[k] = now
|
||||
|
||||
# Thread lock for safe reads/writes to latest_data
|
||||
_data_lock = threading.Lock()
|
||||
@@ -0,0 +1,721 @@
|
||||
"""Commercial flight fetching — ADS-B, OpenSky, supplemental sources, routes,
|
||||
trail accumulation, GPS jamming detection, and holding pattern detection."""
|
||||
import re
|
||||
import os
|
||||
import time
|
||||
import math
|
||||
import logging
|
||||
import threading
|
||||
import concurrent.futures
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from cachetools import TTLCache
|
||||
from services.network_utils import fetch_with_curl
|
||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
||||
from services.fetchers.plane_alert import enrich_with_plane_alert, enrich_with_tracked_names
|
||||
|
||||
logger = logging.getLogger("services.data_fetcher")
|
||||
|
||||
# Pre-compiled regex patterns for airline code extraction (used in hot loop)
|
||||
_RE_AIRLINE_CODE_1 = re.compile(r'^([A-Z]{3})\d')
|
||||
_RE_AIRLINE_CODE_2 = re.compile(r'^([A-Z]{3})[A-Z\d]')
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OpenSky Network API Client (OAuth2)
|
||||
# ---------------------------------------------------------------------------
|
||||
class OpenSkyClient:
|
||||
def __init__(self, client_id, client_secret):
|
||||
self.client_id = client_id
|
||||
self.client_secret = client_secret
|
||||
self.token = None
|
||||
self.expires_at = 0
|
||||
|
||||
def get_token(self):
|
||||
if self.token and time.time() < self.expires_at - 60:
|
||||
return self.token
|
||||
url = "https://auth.opensky-network.org/auth/realms/opensky-network/protocol/openid-connect/token"
|
||||
data = {
|
||||
"grant_type": "client_credentials",
|
||||
"client_id": self.client_id,
|
||||
"client_secret": self.client_secret
|
||||
}
|
||||
try:
|
||||
r = requests.post(url, data=data, timeout=10)
|
||||
if r.status_code == 200:
|
||||
res = r.json()
|
||||
self.token = res.get("access_token")
|
||||
self.expires_at = time.time() + res.get("expires_in", 1800)
|
||||
logger.info("OpenSky OAuth2 token refreshed.")
|
||||
return self.token
|
||||
else:
|
||||
logger.error(f"OpenSky Auth Failed: {r.status_code} {r.text}")
|
||||
except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e:
|
||||
logger.error(f"OpenSky Auth Exception: {e}")
|
||||
return None
|
||||
|
||||
opensky_client = OpenSkyClient(
|
||||
client_id=os.environ.get("OPENSKY_CLIENT_ID", ""),
|
||||
client_secret=os.environ.get("OPENSKY_CLIENT_SECRET", "")
|
||||
)
|
||||
|
||||
# Throttling and caching for OpenSky (400 req/day limit)
|
||||
last_opensky_fetch = 0
|
||||
cached_opensky_flights = []
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Supplemental ADS-B sources for blind-spot gap-filling
|
||||
# ---------------------------------------------------------------------------
|
||||
_BLIND_SPOT_REGIONS = [
|
||||
{"name": "Yekaterinburg", "lat": 56.8, "lon": 60.6, "radius_nm": 250},
|
||||
{"name": "Novosibirsk", "lat": 55.0, "lon": 82.9, "radius_nm": 250},
|
||||
{"name": "Krasnoyarsk", "lat": 56.0, "lon": 92.9, "radius_nm": 250},
|
||||
{"name": "Vladivostok", "lat": 43.1, "lon": 131.9, "radius_nm": 250},
|
||||
{"name": "Urumqi", "lat": 43.8, "lon": 87.6, "radius_nm": 250},
|
||||
{"name": "Chengdu", "lat": 30.6, "lon": 104.1, "radius_nm": 250},
|
||||
{"name": "Lagos-Accra", "lat": 6.5, "lon": 3.4, "radius_nm": 250},
|
||||
{"name": "Addis Ababa", "lat": 9.0, "lon": 38.7, "radius_nm": 250},
|
||||
]
|
||||
_SUPPLEMENTAL_FETCH_INTERVAL = 120
|
||||
last_supplemental_fetch = 0
|
||||
cached_supplemental_flights = []
|
||||
|
||||
# Helicopter type codes (backend classification)
|
||||
_HELI_TYPES_BACKEND = {
|
||||
"R22", "R44", "R66", "B06", "B06T", "B204", "B205", "B206", "B212", "B222", "B230",
|
||||
"B407", "B412", "B427", "B429", "B430", "B505", "B525",
|
||||
"AS32", "AS35", "AS50", "AS55", "AS65",
|
||||
"EC20", "EC25", "EC30", "EC35", "EC45", "EC55", "EC75",
|
||||
"H125", "H130", "H135", "H145", "H155", "H160", "H175", "H215", "H225",
|
||||
"S55", "S58", "S61", "S64", "S70", "S76", "S92",
|
||||
"A109", "A119", "A139", "A169", "A189", "AW09",
|
||||
"MD52", "MD60", "MDHI", "MD90", "NOTR",
|
||||
"B47G", "HUEY", "GAMA", "CABR", "EXE",
|
||||
}
|
||||
|
||||
# Private jet ICAO type designator codes
|
||||
PRIVATE_JET_TYPES = {
|
||||
"G150", "G200", "G280", "GLEX", "G500", "G550", "G600", "G650", "G700",
|
||||
"GLF2", "GLF3", "GLF4", "GLF5", "GLF6", "GL5T", "GL7T", "GV", "GIV",
|
||||
"CL30", "CL35", "CL60", "BD70", "BD10", "GL5T", "GL7T",
|
||||
"CRJ1", "CRJ2",
|
||||
"C25A", "C25B", "C25C", "C500", "C501", "C510", "C525", "C526",
|
||||
"C550", "C560", "C56X", "C680", "C68A", "C700", "C750",
|
||||
"FA10", "FA20", "FA50", "FA7X", "FA8X", "F900", "F2TH", "ASTR",
|
||||
"E35L", "E545", "E550", "E55P", "LEGA", "PH10", "PH30",
|
||||
"LJ23", "LJ24", "LJ25", "LJ28", "LJ31", "LJ35", "LJ36",
|
||||
"LJ40", "LJ45", "LJ55", "LJ60", "LJ70", "LJ75",
|
||||
"H25A", "H25B", "H25C", "HA4T", "BE40", "PRM1",
|
||||
"HDJT", "PC24", "EA50", "SF50", "GALX",
|
||||
}
|
||||
|
||||
# Flight trails state
|
||||
flight_trails = {} # {icao_hex: {points: [[lat, lng, alt, ts], ...], last_seen: ts}}
|
||||
_trails_lock = threading.Lock()
|
||||
_MAX_TRACKED_TRAILS = 2000
|
||||
|
||||
# Routes cache
|
||||
dynamic_routes_cache = TTLCache(maxsize=5000, ttl=7200)
|
||||
routes_fetch_in_progress = False
|
||||
_routes_lock = threading.Lock()
|
||||
|
||||
|
||||
def _fetch_supplemental_sources(seen_hex: set) -> list:
|
||||
"""Fetch from airplanes.live and adsb.fi to fill blind-spot gaps."""
|
||||
global last_supplemental_fetch, cached_supplemental_flights
|
||||
|
||||
now = time.time()
|
||||
if now - last_supplemental_fetch < _SUPPLEMENTAL_FETCH_INTERVAL:
|
||||
return [f for f in cached_supplemental_flights
|
||||
if f.get("hex", "").lower().strip() not in seen_hex]
|
||||
|
||||
new_supplemental = []
|
||||
supplemental_hex = set()
|
||||
|
||||
def _fetch_airplaneslive(region):
|
||||
try:
|
||||
url = (f"https://api.airplanes.live/v2/point/"
|
||||
f"{region['lat']}/{region['lon']}/{region['radius_nm']}")
|
||||
res = fetch_with_curl(url, timeout=10)
|
||||
if res.status_code == 200:
|
||||
data = res.json()
|
||||
return data.get("ac", [])
|
||||
except Exception as e:
|
||||
logger.debug(f"airplanes.live {region['name']} failed: {e}")
|
||||
return []
|
||||
|
||||
try:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
|
||||
results = list(pool.map(_fetch_airplaneslive, _BLIND_SPOT_REGIONS))
|
||||
for region_flights in results:
|
||||
for f in region_flights:
|
||||
h = f.get("hex", "").lower().strip()
|
||||
if h and h not in seen_hex and h not in supplemental_hex:
|
||||
f["supplemental_source"] = "airplanes.live"
|
||||
new_supplemental.append(f)
|
||||
supplemental_hex.add(h)
|
||||
except Exception as e:
|
||||
logger.warning(f"airplanes.live supplemental fetch failed: {e}")
|
||||
|
||||
ap_count = len(new_supplemental)
|
||||
|
||||
try:
|
||||
for region in _BLIND_SPOT_REGIONS:
|
||||
try:
|
||||
url = (f"https://opendata.adsb.fi/api/v3/lat/"
|
||||
f"{region['lat']}/lon/{region['lon']}/dist/{region['radius_nm']}")
|
||||
res = fetch_with_curl(url, timeout=10)
|
||||
if res.status_code == 200:
|
||||
data = res.json()
|
||||
for f in data.get("ac", []):
|
||||
h = f.get("hex", "").lower().strip()
|
||||
if h and h not in seen_hex and h not in supplemental_hex:
|
||||
f["supplemental_source"] = "adsb.fi"
|
||||
new_supplemental.append(f)
|
||||
supplemental_hex.add(h)
|
||||
except Exception as e:
|
||||
logger.debug(f"adsb.fi {region['name']} failed: {e}")
|
||||
time.sleep(1.1)
|
||||
except Exception as e:
|
||||
logger.warning(f"adsb.fi supplemental fetch failed: {e}")
|
||||
|
||||
fi_count = len(new_supplemental) - ap_count
|
||||
|
||||
cached_supplemental_flights = new_supplemental
|
||||
last_supplemental_fetch = now
|
||||
if new_supplemental:
|
||||
_mark_fresh("supplemental_flights")
|
||||
|
||||
logger.info(f"Supplemental: +{len(new_supplemental)} new aircraft from blind-spot "
|
||||
f"hotspots (airplanes.live: {ap_count}, adsb.fi: {fi_count})")
|
||||
return new_supplemental
|
||||
|
||||
|
||||
def fetch_routes_background(sampled):
|
||||
global routes_fetch_in_progress
|
||||
with _routes_lock:
|
||||
if routes_fetch_in_progress:
|
||||
return
|
||||
routes_fetch_in_progress = True
|
||||
|
||||
try:
|
||||
callsigns_to_query = []
|
||||
for f in sampled:
|
||||
c_sign = str(f.get("flight", "")).strip()
|
||||
if c_sign and c_sign != "UNKNOWN":
|
||||
callsigns_to_query.append({
|
||||
"callsign": c_sign,
|
||||
"lat": f.get("lat", 0),
|
||||
"lng": f.get("lon", 0)
|
||||
})
|
||||
|
||||
batch_size = 100
|
||||
batches = [callsigns_to_query[i:i+batch_size] for i in range(0, len(callsigns_to_query), batch_size)]
|
||||
|
||||
for batch in batches:
|
||||
try:
|
||||
r = fetch_with_curl("https://api.adsb.lol/api/0/routeset", method="POST", json_data={"planes": batch}, timeout=15)
|
||||
if r.status_code == 200:
|
||||
route_data = r.json()
|
||||
route_list = []
|
||||
if isinstance(route_data, dict):
|
||||
route_list = route_data.get("value", [])
|
||||
elif isinstance(route_data, list):
|
||||
route_list = route_data
|
||||
|
||||
for route in route_list:
|
||||
callsign = route.get("callsign", "")
|
||||
airports = route.get("_airports", [])
|
||||
if airports and len(airports) >= 2:
|
||||
orig_apt = airports[0]
|
||||
dest_apt = airports[-1]
|
||||
with _routes_lock:
|
||||
dynamic_routes_cache[callsign] = {
|
||||
"orig_name": f"{orig_apt.get('iata', '')}: {orig_apt.get('name', 'Unknown')}",
|
||||
"dest_name": f"{dest_apt.get('iata', '')}: {dest_apt.get('name', 'Unknown')}",
|
||||
"orig_loc": [orig_apt.get("lon", 0), orig_apt.get("lat", 0)],
|
||||
"dest_loc": [dest_apt.get("lon", 0), dest_apt.get("lat", 0)],
|
||||
}
|
||||
time.sleep(0.25)
|
||||
except Exception:
|
||||
logger.debug("Route batch request failed")
|
||||
finally:
|
||||
with _routes_lock:
|
||||
routes_fetch_in_progress = False
|
||||
|
||||
|
||||
def _classify_and_publish(all_adsb_flights):
|
||||
"""Shared pipeline: normalize raw ADS-B data → classify → merge → publish to latest_data.
|
||||
|
||||
Called once immediately after adsb.lol returns (fast path, ~3-5s),
|
||||
then again after OpenSky + supplemental gap-fill enrichment.
|
||||
"""
|
||||
flights = []
|
||||
|
||||
if not all_adsb_flights:
|
||||
return
|
||||
|
||||
with _routes_lock:
|
||||
already_running = routes_fetch_in_progress
|
||||
if not already_running:
|
||||
threading.Thread(target=fetch_routes_background, args=(all_adsb_flights,), daemon=True).start()
|
||||
|
||||
for f in all_adsb_flights:
|
||||
try:
|
||||
lat = f.get("lat")
|
||||
lng = f.get("lon")
|
||||
heading = f.get("track") or 0
|
||||
|
||||
if lat is None or lng is None:
|
||||
continue
|
||||
|
||||
flight_str = str(f.get("flight", "UNKNOWN")).strip()
|
||||
if not flight_str or flight_str == "UNKNOWN":
|
||||
flight_str = str(f.get("hex", "Unknown"))
|
||||
|
||||
origin_loc = None
|
||||
dest_loc = None
|
||||
origin_name = "UNKNOWN"
|
||||
dest_name = "UNKNOWN"
|
||||
|
||||
with _routes_lock:
|
||||
cached_route = dynamic_routes_cache.get(flight_str)
|
||||
if cached_route:
|
||||
origin_name = cached_route["orig_name"]
|
||||
dest_name = cached_route["dest_name"]
|
||||
origin_loc = cached_route["orig_loc"]
|
||||
dest_loc = cached_route["dest_loc"]
|
||||
|
||||
airline_code = ""
|
||||
match = _RE_AIRLINE_CODE_1.match(flight_str)
|
||||
if not match:
|
||||
match = _RE_AIRLINE_CODE_2.match(flight_str)
|
||||
if match:
|
||||
airline_code = match.group(1)
|
||||
|
||||
alt_raw = f.get("alt_baro")
|
||||
alt_value = 0
|
||||
if isinstance(alt_raw, (int, float)):
|
||||
alt_value = alt_raw * 0.3048
|
||||
|
||||
gs_knots = f.get("gs")
|
||||
speed_knots = round(gs_knots, 1) if isinstance(gs_knots, (int, float)) else None
|
||||
|
||||
model_upper = f.get("t", "").upper()
|
||||
if model_upper == "TWR":
|
||||
continue
|
||||
|
||||
ac_category = "heli" if model_upper in _HELI_TYPES_BACKEND else "plane"
|
||||
|
||||
flights.append({
|
||||
"callsign": flight_str,
|
||||
"country": f.get("r", "N/A"),
|
||||
"lng": float(lng),
|
||||
"lat": float(lat),
|
||||
"alt": alt_value,
|
||||
"heading": heading,
|
||||
"type": "flight",
|
||||
"origin_loc": origin_loc,
|
||||
"dest_loc": dest_loc,
|
||||
"origin_name": origin_name,
|
||||
"dest_name": dest_name,
|
||||
"registration": f.get("r", "N/A"),
|
||||
"model": f.get("t", "Unknown"),
|
||||
"icao24": f.get("hex", ""),
|
||||
"speed_knots": speed_knots,
|
||||
"squawk": f.get("squawk", ""),
|
||||
"airline_code": airline_code,
|
||||
"aircraft_category": ac_category,
|
||||
"nac_p": f.get("nac_p")
|
||||
})
|
||||
except Exception as loop_e:
|
||||
logger.error(f"Flight interpolation error: {loop_e}")
|
||||
continue
|
||||
|
||||
# --- Classification ---
|
||||
commercial = []
|
||||
private_jets = []
|
||||
private_ga = []
|
||||
tracked = []
|
||||
|
||||
for f in flights:
|
||||
enrich_with_plane_alert(f)
|
||||
enrich_with_tracked_names(f)
|
||||
|
||||
callsign = f.get('callsign', '').strip().upper()
|
||||
is_commercial_format = bool(re.match(r'^[A-Z]{3}\d{1,4}[A-Z]{0,2}$', callsign))
|
||||
|
||||
if f.get('alert_category'):
|
||||
f['type'] = 'tracked_flight'
|
||||
tracked.append(f)
|
||||
elif f.get('airline_code') or is_commercial_format:
|
||||
f['type'] = 'commercial_flight'
|
||||
commercial.append(f)
|
||||
elif f.get('model', '').upper() in PRIVATE_JET_TYPES:
|
||||
f['type'] = 'private_jet'
|
||||
private_jets.append(f)
|
||||
else:
|
||||
f['type'] = 'private_ga'
|
||||
private_ga.append(f)
|
||||
|
||||
# --- Smart merge: protect against partial API failures ---
|
||||
prev_commercial_count = len(latest_data.get('commercial_flights', []))
|
||||
prev_total = prev_commercial_count + len(latest_data.get('private_jets', [])) + len(latest_data.get('private_flights', []))
|
||||
new_total = len(commercial) + len(private_jets) + len(private_ga)
|
||||
|
||||
if new_total == 0:
|
||||
logger.warning("No civilian flights found! Skipping overwrite to prevent clearing the map.")
|
||||
elif prev_total > 100 and new_total < prev_total * 0.5:
|
||||
logger.warning(f"Flight count dropped from {prev_total} to {new_total} (>50% loss). Keeping previous data to prevent flicker.")
|
||||
else:
|
||||
_now = time.time()
|
||||
|
||||
def _merge_category(new_list, old_list, max_stale_s=120):
|
||||
by_icao = {}
|
||||
for f in old_list:
|
||||
icao = f.get('icao24', '')
|
||||
if icao:
|
||||
f.setdefault('_seen_at', _now)
|
||||
if (_now - f.get('_seen_at', _now)) < max_stale_s:
|
||||
by_icao[icao] = f
|
||||
for f in new_list:
|
||||
icao = f.get('icao24', '')
|
||||
if icao:
|
||||
f['_seen_at'] = _now
|
||||
by_icao[icao] = f
|
||||
else:
|
||||
continue
|
||||
return list(by_icao.values())
|
||||
|
||||
with _data_lock:
|
||||
latest_data['commercial_flights'] = _merge_category(commercial, latest_data.get('commercial_flights', []))
|
||||
latest_data['private_jets'] = _merge_category(private_jets, latest_data.get('private_jets', []))
|
||||
latest_data['private_flights'] = _merge_category(private_ga, latest_data.get('private_flights', []))
|
||||
|
||||
_mark_fresh("commercial_flights", "private_jets", "private_flights")
|
||||
|
||||
with _data_lock:
|
||||
if flights:
|
||||
latest_data['flights'] = flights
|
||||
|
||||
# Merge tracked civilian flights with tracked military flights
|
||||
with _data_lock:
|
||||
existing_tracked = list(latest_data.get('tracked_flights', []))
|
||||
|
||||
fresh_tracked_map = {}
|
||||
for t in tracked:
|
||||
icao = t.get('icao24', '').upper()
|
||||
if icao:
|
||||
fresh_tracked_map[icao] = t
|
||||
|
||||
merged_tracked = []
|
||||
seen_icaos = set()
|
||||
for old_t in existing_tracked:
|
||||
icao = old_t.get('icao24', '').upper()
|
||||
if icao in fresh_tracked_map:
|
||||
fresh = fresh_tracked_map[icao]
|
||||
for key in ('alert_category', 'alert_operator', 'alert_special', 'alert_flag'):
|
||||
if key in old_t and key not in fresh:
|
||||
fresh[key] = old_t[key]
|
||||
merged_tracked.append(fresh)
|
||||
seen_icaos.add(icao)
|
||||
else:
|
||||
merged_tracked.append(old_t)
|
||||
seen_icaos.add(icao)
|
||||
|
||||
for icao, t in fresh_tracked_map.items():
|
||||
if icao not in seen_icaos:
|
||||
merged_tracked.append(t)
|
||||
|
||||
with _data_lock:
|
||||
latest_data['tracked_flights'] = merged_tracked
|
||||
logger.info(f"Tracked flights: {len(merged_tracked)} total ({len(fresh_tracked_map)} fresh from civilian)")
|
||||
|
||||
# --- Trail Accumulation ---
|
||||
def _accumulate_trail(f, now_ts, check_route=True):
|
||||
hex_id = f.get('icao24', '').lower()
|
||||
if not hex_id:
|
||||
return 0, None
|
||||
if check_route and f.get('origin_name', 'UNKNOWN') != 'UNKNOWN':
|
||||
f['trail'] = []
|
||||
return 0, hex_id
|
||||
lat, lng, alt = f.get('lat'), f.get('lng'), f.get('alt', 0)
|
||||
if lat is None or lng is None:
|
||||
f['trail'] = flight_trails.get(hex_id, {}).get('points', [])
|
||||
return 0, hex_id
|
||||
point = [round(lat, 5), round(lng, 5), round(alt, 1), round(now_ts)]
|
||||
if hex_id not in flight_trails:
|
||||
flight_trails[hex_id] = {'points': [], 'last_seen': now_ts}
|
||||
trail_data = flight_trails[hex_id]
|
||||
if trail_data['points'] and trail_data['points'][-1][0] == point[0] and trail_data['points'][-1][1] == point[1]:
|
||||
trail_data['last_seen'] = now_ts
|
||||
else:
|
||||
trail_data['points'].append(point)
|
||||
trail_data['last_seen'] = now_ts
|
||||
if len(trail_data['points']) > 200:
|
||||
trail_data['points'] = trail_data['points'][-200:]
|
||||
f['trail'] = trail_data['points']
|
||||
return 1, hex_id
|
||||
|
||||
now_ts = datetime.utcnow().timestamp()
|
||||
all_lists = [commercial, private_jets, private_ga, existing_tracked]
|
||||
seen_hexes = set()
|
||||
trail_count = 0
|
||||
with _trails_lock:
|
||||
for flist in all_lists:
|
||||
for f in flist:
|
||||
count, hex_id = _accumulate_trail(f, now_ts, check_route=True)
|
||||
trail_count += count
|
||||
if hex_id:
|
||||
seen_hexes.add(hex_id)
|
||||
|
||||
for mf in latest_data.get('military_flights', []):
|
||||
count, hex_id = _accumulate_trail(mf, now_ts, check_route=False)
|
||||
trail_count += count
|
||||
if hex_id:
|
||||
seen_hexes.add(hex_id)
|
||||
|
||||
tracked_hexes = {t.get('icao24', '').lower() for t in latest_data.get('tracked_flights', [])}
|
||||
stale_keys = []
|
||||
for k, v in flight_trails.items():
|
||||
cutoff = now_ts - 1800 if k in tracked_hexes else now_ts - 300
|
||||
if v['last_seen'] < cutoff:
|
||||
stale_keys.append(k)
|
||||
for k in stale_keys:
|
||||
del flight_trails[k]
|
||||
|
||||
if len(flight_trails) > _MAX_TRACKED_TRAILS:
|
||||
sorted_keys = sorted(flight_trails.keys(), key=lambda k: flight_trails[k]['last_seen'])
|
||||
evict_count = len(flight_trails) - _MAX_TRACKED_TRAILS
|
||||
for k in sorted_keys[:evict_count]:
|
||||
del flight_trails[k]
|
||||
|
||||
logger.info(f"Trail accumulation: {trail_count} active trails, {len(stale_keys)} pruned, {len(flight_trails)} total")
|
||||
|
||||
# --- GPS Jamming Detection ---
|
||||
try:
|
||||
jamming_grid = {}
|
||||
raw_flights = latest_data.get('flights', [])
|
||||
for rf in raw_flights:
|
||||
rlat = rf.get('lat')
|
||||
rlng = rf.get('lng') or rf.get('lon')
|
||||
if rlat is None or rlng is None:
|
||||
continue
|
||||
nacp = rf.get('nac_p')
|
||||
if nacp is None:
|
||||
continue
|
||||
grid_key = f"{int(rlat)},{int(rlng)}"
|
||||
if grid_key not in jamming_grid:
|
||||
jamming_grid[grid_key] = {"degraded": 0, "total": 0}
|
||||
jamming_grid[grid_key]["total"] += 1
|
||||
if nacp < 8:
|
||||
jamming_grid[grid_key]["degraded"] += 1
|
||||
|
||||
jamming_zones = []
|
||||
for gk, counts in jamming_grid.items():
|
||||
if counts["total"] < 3:
|
||||
continue
|
||||
ratio = counts["degraded"] / counts["total"]
|
||||
if ratio > 0.25:
|
||||
lat_i, lng_i = gk.split(",")
|
||||
severity = "low" if ratio < 0.5 else "medium" if ratio < 0.75 else "high"
|
||||
jamming_zones.append({
|
||||
"lat": int(lat_i) + 0.5,
|
||||
"lng": int(lng_i) + 0.5,
|
||||
"severity": severity,
|
||||
"ratio": round(ratio, 2),
|
||||
"degraded": counts["degraded"],
|
||||
"total": counts["total"]
|
||||
})
|
||||
with _data_lock:
|
||||
latest_data['gps_jamming'] = jamming_zones
|
||||
if jamming_zones:
|
||||
logger.info(f"GPS Jamming: {len(jamming_zones)} interference zones detected")
|
||||
except Exception as e:
|
||||
logger.error(f"GPS Jamming detection error: {e}")
|
||||
with _data_lock:
|
||||
latest_data['gps_jamming'] = []
|
||||
|
||||
# --- Holding Pattern Detection ---
|
||||
try:
|
||||
holding_count = 0
|
||||
all_flight_lists = [commercial, private_jets, private_ga,
|
||||
latest_data.get('tracked_flights', []),
|
||||
latest_data.get('military_flights', [])]
|
||||
with _trails_lock:
|
||||
trails_snapshot = {k: v.get('points', [])[:] for k, v in flight_trails.items()}
|
||||
for flist in all_flight_lists:
|
||||
for f in flist:
|
||||
hex_id = f.get('icao24', '').lower()
|
||||
trail = trails_snapshot.get(hex_id, [])
|
||||
if len(trail) < 6:
|
||||
f['holding'] = False
|
||||
continue
|
||||
pts = trail[-8:]
|
||||
total_turn = 0.0
|
||||
prev_bearing = 0.0
|
||||
for i in range(1, len(pts)):
|
||||
lat1, lng1 = math.radians(pts[i-1][0]), math.radians(pts[i-1][1])
|
||||
lat2, lng2 = math.radians(pts[i][0]), math.radians(pts[i][1])
|
||||
dlng = lng2 - lng1
|
||||
x = math.sin(dlng) * math.cos(lat2)
|
||||
y = math.cos(lat1) * math.sin(lat2) - math.sin(lat1) * math.cos(lat2) * math.cos(dlng)
|
||||
bearing = math.degrees(math.atan2(x, y)) % 360
|
||||
if i > 1:
|
||||
delta = abs(bearing - prev_bearing)
|
||||
if delta > 180:
|
||||
delta = 360 - delta
|
||||
total_turn += delta
|
||||
prev_bearing = bearing
|
||||
f['holding'] = total_turn > 300
|
||||
if f['holding']:
|
||||
holding_count += 1
|
||||
if holding_count:
|
||||
logger.info(f"Holding patterns: {holding_count} aircraft circling")
|
||||
except Exception as e:
|
||||
logger.error(f"Holding pattern detection error: {e}")
|
||||
|
||||
with _data_lock:
|
||||
latest_data['last_updated'] = datetime.utcnow().isoformat()
|
||||
|
||||
|
||||
def _fetch_adsb_lol_regions():
|
||||
"""Fetch all adsb.lol regions in parallel (~3-5s). Returns raw aircraft list."""
|
||||
regions = [
|
||||
{"lat": 39.8, "lon": -98.5, "dist": 2000},
|
||||
{"lat": 50.0, "lon": 15.0, "dist": 2000},
|
||||
{"lat": 35.0, "lon": 105.0, "dist": 2000},
|
||||
{"lat": -25.0, "lon": 133.0, "dist": 2000},
|
||||
{"lat": 0.0, "lon": 20.0, "dist": 2500},
|
||||
{"lat": -15.0, "lon": -60.0, "dist": 2000}
|
||||
]
|
||||
|
||||
def _fetch_region(r):
|
||||
url = f"https://api.adsb.lol/v2/lat/{r['lat']}/lon/{r['lon']}/dist/{r['dist']}"
|
||||
try:
|
||||
res = fetch_with_curl(url, timeout=10)
|
||||
if res.status_code == 200:
|
||||
data = res.json()
|
||||
return data.get("ac", [])
|
||||
except Exception as e:
|
||||
logger.warning(f"Region fetch failed for lat={r['lat']}: {e}")
|
||||
return []
|
||||
|
||||
all_flights = []
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=6) as pool:
|
||||
results = pool.map(_fetch_region, regions)
|
||||
for region_flights in results:
|
||||
all_flights.extend(region_flights)
|
||||
return all_flights
|
||||
|
||||
|
||||
def _enrich_with_opensky_and_supplemental(adsb_flights):
|
||||
"""Slow enrichment: merge OpenSky gap-fill + supplemental sources, then re-publish.
|
||||
|
||||
Runs in a background thread so the initial adsb.lol data is already visible.
|
||||
"""
|
||||
try:
|
||||
seen_hex = set()
|
||||
for f in adsb_flights:
|
||||
h = f.get("hex")
|
||||
if h:
|
||||
seen_hex.add(h.lower().strip())
|
||||
|
||||
all_flights = list(adsb_flights) # copy to avoid mutating the original
|
||||
|
||||
# OpenSky Regional Fallback
|
||||
now = time.time()
|
||||
global last_opensky_fetch, cached_opensky_flights
|
||||
|
||||
if now - last_opensky_fetch > 300:
|
||||
token = opensky_client.get_token()
|
||||
if token:
|
||||
opensky_regions = [
|
||||
{"name": "Africa", "bbox": {"lamin": -35.0, "lomin": -20.0, "lamax": 38.0, "lomax": 55.0}},
|
||||
{"name": "Asia", "bbox": {"lamin": 0.0, "lomin": 30.0, "lamax": 75.0, "lomax": 150.0}},
|
||||
{"name": "South America", "bbox": {"lamin": -60.0, "lomin": -95.0, "lamax": 15.0, "lomax": -30.0}}
|
||||
]
|
||||
|
||||
new_opensky_flights = []
|
||||
for os_reg in opensky_regions:
|
||||
try:
|
||||
bb = os_reg["bbox"]
|
||||
os_url = f"https://opensky-network.org/api/states/all?lamin={bb['lamin']}&lomin={bb['lomin']}&lamax={bb['lamax']}&lomax={bb['lomax']}"
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
os_res = requests.get(os_url, headers=headers, timeout=15)
|
||||
|
||||
if os_res.status_code == 200:
|
||||
os_data = os_res.json()
|
||||
states = os_data.get("states") or []
|
||||
logger.info(f"OpenSky: Fetched {len(states)} states for {os_reg['name']}")
|
||||
|
||||
for s in states:
|
||||
new_opensky_flights.append({
|
||||
"hex": s[0],
|
||||
"flight": s[1].strip() if s[1] else "UNKNOWN",
|
||||
"r": s[2],
|
||||
"lon": s[5],
|
||||
"lat": s[6],
|
||||
"alt_baro": (s[7] * 3.28084) if s[7] else 0,
|
||||
"track": s[10] or 0,
|
||||
"gs": (s[9] * 1.94384) if s[9] else 0,
|
||||
"t": "Unknown",
|
||||
"is_opensky": True
|
||||
})
|
||||
else:
|
||||
logger.warning(f"OpenSky API {os_reg['name']} failed: {os_res.status_code}")
|
||||
except Exception as ex:
|
||||
logger.error(f"OpenSky fetching error for {os_reg['name']}: {ex}")
|
||||
|
||||
cached_opensky_flights = new_opensky_flights
|
||||
last_opensky_fetch = now
|
||||
|
||||
# Merge OpenSky (dedup by hex)
|
||||
for osf in cached_opensky_flights:
|
||||
h = osf.get("hex")
|
||||
if h and h.lower().strip() not in seen_hex:
|
||||
all_flights.append(osf)
|
||||
seen_hex.add(h.lower().strip())
|
||||
|
||||
# Supplemental gap-fill
|
||||
try:
|
||||
gap_fill = _fetch_supplemental_sources(seen_hex)
|
||||
for f in gap_fill:
|
||||
all_flights.append(f)
|
||||
h = f.get("hex", "").lower().strip()
|
||||
if h:
|
||||
seen_hex.add(h)
|
||||
if gap_fill:
|
||||
logger.info(f"Gap-fill: added {len(gap_fill)} aircraft to pipeline")
|
||||
except Exception as e:
|
||||
logger.warning(f"Supplemental source fetch failed (non-fatal): {e}")
|
||||
|
||||
# Re-publish with enriched data
|
||||
if len(all_flights) > len(adsb_flights):
|
||||
logger.info(f"Enrichment: {len(all_flights) - len(adsb_flights)} additional aircraft from OpenSky + supplemental")
|
||||
_classify_and_publish(all_flights)
|
||||
except Exception as e:
|
||||
logger.error(f"OpenSky/supplemental enrichment error: {e}")
|
||||
|
||||
|
||||
def fetch_flights():
|
||||
"""Two-phase flight fetching:
|
||||
Phase 1 (fast): Fetch adsb.lol → classify → publish immediately (~3-5s)
|
||||
Phase 2 (background): Merge OpenSky + supplemental → re-publish (~15-30s)
|
||||
"""
|
||||
try:
|
||||
# Phase 1: adsb.lol — fast, parallel, publish immediately
|
||||
adsb_flights = _fetch_adsb_lol_regions()
|
||||
if adsb_flights:
|
||||
logger.info(f"adsb.lol: {len(adsb_flights)} aircraft — publishing immediately")
|
||||
_classify_and_publish(adsb_flights)
|
||||
|
||||
# Phase 2: kick off slow enrichment in background
|
||||
threading.Thread(
|
||||
target=_enrich_with_opensky_and_supplemental,
|
||||
args=(adsb_flights,),
|
||||
daemon=True,
|
||||
).start()
|
||||
else:
|
||||
logger.warning("adsb.lol returned 0 aircraft")
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching flights: {e}")
|
||||
@@ -0,0 +1,218 @@
|
||||
"""Military flight tracking and UAV detection from ADS-B data."""
|
||||
import logging
|
||||
from services.network_utils import fetch_with_curl
|
||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
||||
from services.fetchers.plane_alert import enrich_with_plane_alert
|
||||
|
||||
logger = logging.getLogger("services.data_fetcher")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# UAV classification — filters military drone transponders
|
||||
# ---------------------------------------------------------------------------
|
||||
_UAV_TYPE_CODES = {"Q9", "R4", "TB2", "MALE", "HALE", "HERM", "HRON"}
|
||||
_UAV_CALLSIGN_PREFIXES = ("FORTE", "GHAWK", "REAP", "BAMS", "UAV", "UAS")
|
||||
_UAV_MODEL_KEYWORDS = ("RQ-", "MQ-", "RQ4", "MQ9", "MQ4", "MQ1", "REAPER", "GLOBALHAWK", "TRITON", "PREDATOR", "HERMES", "HERON", "BAYRAKTAR")
|
||||
_UAV_WIKI = {
|
||||
"RQ4": "https://en.wikipedia.org/wiki/Northrop_Grumman_RQ-4_Global_Hawk",
|
||||
"RQ-4": "https://en.wikipedia.org/wiki/Northrop_Grumman_RQ-4_Global_Hawk",
|
||||
"MQ4": "https://en.wikipedia.org/wiki/Northrop_Grumman_MQ-4C_Triton",
|
||||
"MQ-4": "https://en.wikipedia.org/wiki/Northrop_Grumman_MQ-4C_Triton",
|
||||
"MQ9": "https://en.wikipedia.org/wiki/General_Atomics_MQ-9_Reaper",
|
||||
"MQ-9": "https://en.wikipedia.org/wiki/General_Atomics_MQ-9_Reaper",
|
||||
"MQ1": "https://en.wikipedia.org/wiki/General_Atomics_MQ-1C_Gray_Eagle",
|
||||
"MQ-1": "https://en.wikipedia.org/wiki/General_Atomics_MQ-1C_Gray_Eagle",
|
||||
"REAPER": "https://en.wikipedia.org/wiki/General_Atomics_MQ-9_Reaper",
|
||||
"GLOBALHAWK": "https://en.wikipedia.org/wiki/Northrop_Grumman_RQ-4_Global_Hawk",
|
||||
"TRITON": "https://en.wikipedia.org/wiki/Northrop_Grumman_MQ-4C_Triton",
|
||||
"PREDATOR": "https://en.wikipedia.org/wiki/General_Atomics_MQ-1_Predator",
|
||||
"HERMES": "https://en.wikipedia.org/wiki/Elbit_Hermes_900",
|
||||
"HERON": "https://en.wikipedia.org/wiki/IAI_Heron",
|
||||
"BAYRAKTAR": "https://en.wikipedia.org/wiki/Bayraktar_TB2",
|
||||
}
|
||||
|
||||
|
||||
def _classify_uav(model: str, callsign: str):
|
||||
"""Check if an aircraft is a UAV based on type code, callsign prefix, or model keywords.
|
||||
Returns (is_uav, uav_type, wiki_url) or (False, None, None)."""
|
||||
model_up = model.upper().replace(" ", "")
|
||||
callsign_up = callsign.upper().strip()
|
||||
|
||||
if model_up in _UAV_TYPE_CODES:
|
||||
uav_type = "HALE Surveillance" if model_up in ("R4", "HALE") else "MALE ISR"
|
||||
wiki = _UAV_WIKI.get(model_up, "")
|
||||
return True, uav_type, wiki
|
||||
|
||||
for prefix in _UAV_CALLSIGN_PREFIXES:
|
||||
if callsign_up.startswith(prefix):
|
||||
uav_type = "HALE Surveillance" if prefix in ("FORTE", "GHAWK", "BAMS") else "MALE ISR"
|
||||
wiki = _UAV_WIKI.get(prefix, "")
|
||||
if prefix == "FORTE":
|
||||
wiki = _UAV_WIKI["RQ4"]
|
||||
elif prefix == "BAMS":
|
||||
wiki = _UAV_WIKI["MQ4"]
|
||||
return True, uav_type, wiki
|
||||
|
||||
for kw in _UAV_MODEL_KEYWORDS:
|
||||
if kw in model_up:
|
||||
if any(h in model_up for h in ("RQ4", "RQ-4", "GLOBALHAWK")):
|
||||
return True, "HALE Surveillance", _UAV_WIKI.get(kw, "")
|
||||
elif any(h in model_up for h in ("MQ4", "MQ-4", "TRITON")):
|
||||
return True, "HALE Maritime Surveillance", _UAV_WIKI.get(kw, "")
|
||||
elif any(h in model_up for h in ("MQ9", "MQ-9", "REAPER")):
|
||||
return True, "MALE Strike/ISR", _UAV_WIKI.get(kw, "")
|
||||
elif any(h in model_up for h in ("MQ1", "MQ-1", "PREDATOR")):
|
||||
return True, "MALE ISR/Strike", _UAV_WIKI.get(kw, "")
|
||||
elif "BAYRAKTAR" in model_up or "TB2" in model_up:
|
||||
return True, "MALE Strike", _UAV_WIKI.get("BAYRAKTAR", "")
|
||||
elif "HERMES" in model_up:
|
||||
return True, "MALE ISR", _UAV_WIKI.get("HERMES", "")
|
||||
elif "HERON" in model_up:
|
||||
return True, "MALE ISR", _UAV_WIKI.get("HERON", "")
|
||||
return True, "MALE ISR", _UAV_WIKI.get(kw, "")
|
||||
|
||||
return False, None, None
|
||||
|
||||
|
||||
def fetch_military_flights():
|
||||
military_flights = []
|
||||
detected_uavs = []
|
||||
try:
|
||||
url = "https://api.adsb.lol/v2/mil"
|
||||
response = fetch_with_curl(url, timeout=10)
|
||||
if response.status_code == 200:
|
||||
ac = response.json().get('ac', [])
|
||||
for f in ac:
|
||||
try:
|
||||
lat = f.get("lat")
|
||||
lng = f.get("lon")
|
||||
heading = f.get("track") or 0
|
||||
|
||||
if lat is None or lng is None:
|
||||
continue
|
||||
|
||||
model = str(f.get("t", "UNKNOWN")).upper()
|
||||
callsign = str(f.get("flight", "MIL-UNKN")).strip()
|
||||
|
||||
if model == "TWR":
|
||||
continue
|
||||
|
||||
alt_raw = f.get("alt_baro")
|
||||
alt_value = 0
|
||||
if isinstance(alt_raw, (int, float)):
|
||||
alt_value = alt_raw * 0.3048
|
||||
|
||||
gs_knots = f.get("gs")
|
||||
speed_knots = round(gs_knots, 1) if isinstance(gs_knots, (int, float)) else None
|
||||
|
||||
is_uav, uav_type, wiki_url = _classify_uav(model, callsign)
|
||||
if is_uav:
|
||||
detected_uavs.append({
|
||||
"id": f"uav-{f.get('hex', '')}",
|
||||
"callsign": callsign,
|
||||
"aircraft_model": f.get("t", "Unknown"),
|
||||
"lat": float(lat),
|
||||
"lng": float(lng),
|
||||
"alt": alt_value,
|
||||
"heading": heading,
|
||||
"speed_knots": speed_knots,
|
||||
"country": f.get("flag", "Unknown"),
|
||||
"uav_type": uav_type,
|
||||
"wiki": wiki_url or "",
|
||||
"type": "uav",
|
||||
"registration": f.get("r", "N/A"),
|
||||
"icao24": f.get("hex", ""),
|
||||
"squawk": f.get("squawk", ""),
|
||||
})
|
||||
continue
|
||||
|
||||
mil_cat = "default"
|
||||
if "H" in model and any(c.isdigit() for c in model):
|
||||
mil_cat = "heli"
|
||||
elif any(k in model for k in ["K35", "K46", "A33"]):
|
||||
mil_cat = "tanker"
|
||||
elif any(k in model for k in ["F16", "F35", "F22", "F15", "F18", "T38", "T6", "A10"]):
|
||||
mil_cat = "fighter"
|
||||
elif any(k in model for k in ["C17", "C5", "C130", "C30", "A400", "V22"]):
|
||||
mil_cat = "cargo"
|
||||
elif any(k in model for k in ["P8", "E3", "E8", "U2"]):
|
||||
mil_cat = "recon"
|
||||
|
||||
military_flights.append({
|
||||
"callsign": callsign,
|
||||
"country": f.get("flag", "Military Asset"),
|
||||
"lng": float(lng),
|
||||
"lat": float(lat),
|
||||
"alt": alt_value,
|
||||
"heading": heading,
|
||||
"type": "military_flight",
|
||||
"military_type": mil_cat,
|
||||
"origin_loc": None,
|
||||
"dest_loc": None,
|
||||
"origin_name": "UNKNOWN",
|
||||
"dest_name": "UNKNOWN",
|
||||
"registration": f.get("r", "N/A"),
|
||||
"model": f.get("t", "Unknown"),
|
||||
"icao24": f.get("hex", ""),
|
||||
"speed_knots": speed_knots,
|
||||
"squawk": f.get("squawk", "")
|
||||
})
|
||||
except Exception as loop_e:
|
||||
logger.error(f"Mil flight interpolation error: {loop_e}")
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching military flights: {e}")
|
||||
|
||||
if not military_flights and not detected_uavs:
|
||||
logger.warning("No military flights retrieved — keeping previous data if available")
|
||||
with _data_lock:
|
||||
if latest_data.get('military_flights'):
|
||||
return
|
||||
|
||||
with _data_lock:
|
||||
latest_data['military_flights'] = military_flights
|
||||
latest_data['uavs'] = detected_uavs
|
||||
_mark_fresh("military_flights", "uavs")
|
||||
logger.info(f"UAVs: {len(detected_uavs)} real drones detected via ADS-B")
|
||||
|
||||
# Cross-reference military flights with Plane-Alert DB
|
||||
tracked_mil = []
|
||||
remaining_mil = []
|
||||
for mf in military_flights:
|
||||
enrich_with_plane_alert(mf)
|
||||
if mf.get('alert_category'):
|
||||
mf['type'] = 'tracked_flight'
|
||||
tracked_mil.append(mf)
|
||||
else:
|
||||
remaining_mil.append(mf)
|
||||
with _data_lock:
|
||||
latest_data['military_flights'] = remaining_mil
|
||||
|
||||
# Store tracked military flights — update positions for existing entries
|
||||
with _data_lock:
|
||||
existing_tracked = list(latest_data.get('tracked_flights', []))
|
||||
fresh_mil_map = {}
|
||||
for t in tracked_mil:
|
||||
icao = t.get('icao24', '').upper()
|
||||
if icao:
|
||||
fresh_mil_map[icao] = t
|
||||
|
||||
updated_tracked = []
|
||||
seen_icaos = set()
|
||||
for old_t in existing_tracked:
|
||||
icao = old_t.get('icao24', '').upper()
|
||||
if icao in fresh_mil_map:
|
||||
fresh = fresh_mil_map[icao]
|
||||
for key in ('alert_category', 'alert_operator', 'alert_special', 'alert_flag'):
|
||||
if key in old_t and key not in fresh:
|
||||
fresh[key] = old_t[key]
|
||||
updated_tracked.append(fresh)
|
||||
seen_icaos.add(icao)
|
||||
else:
|
||||
updated_tracked.append(old_t)
|
||||
seen_icaos.add(icao)
|
||||
for icao, t in fresh_mil_map.items():
|
||||
if icao not in seen_icaos:
|
||||
updated_tracked.append(t)
|
||||
with _data_lock:
|
||||
latest_data['tracked_flights'] = updated_tracked
|
||||
logger.info(f"Tracked flights: {len(updated_tracked)} total ({len(tracked_mil)} from military)")
|
||||
@@ -0,0 +1,220 @@
|
||||
"""News fetching, geocoding, clustering, and risk assessment."""
|
||||
import re
|
||||
import logging
|
||||
import concurrent.futures
|
||||
import feedparser
|
||||
from services.network_utils import fetch_with_curl
|
||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
||||
|
||||
logger = logging.getLogger("services.data_fetcher")
|
||||
|
||||
|
||||
# Keyword -> coordinate mapping for geocoding news articles
|
||||
_KEYWORD_COORDS = {
|
||||
"venezuela": (7.119, -66.589),
|
||||
"brazil": (-14.235, -51.925),
|
||||
"argentina": (-38.416, -63.616),
|
||||
"colombia": (4.570, -74.297),
|
||||
"mexico": (23.634, -102.552),
|
||||
"united states": (38.907, -77.036),
|
||||
" usa ": (38.907, -77.036),
|
||||
" us ": (38.907, -77.036),
|
||||
"washington": (38.907, -77.036),
|
||||
"canada": (56.130, -106.346),
|
||||
"ukraine": (49.487, 31.272),
|
||||
"kyiv": (50.450, 30.523),
|
||||
"russia": (61.524, 105.318),
|
||||
"moscow": (55.755, 37.617),
|
||||
"israel": (31.046, 34.851),
|
||||
"gaza": (31.416, 34.333),
|
||||
"iran": (32.427, 53.688),
|
||||
"lebanon": (33.854, 35.862),
|
||||
"syria": (34.802, 38.996),
|
||||
"yemen": (15.552, 48.516),
|
||||
"china": (35.861, 104.195),
|
||||
"beijing": (39.904, 116.407),
|
||||
"taiwan": (23.697, 120.960),
|
||||
"north korea": (40.339, 127.510),
|
||||
"south korea": (35.907, 127.766),
|
||||
"pyongyang": (39.039, 125.762),
|
||||
"seoul": (37.566, 126.978),
|
||||
"japan": (36.204, 138.252),
|
||||
"tokyo": (35.676, 139.650),
|
||||
"afghanistan": (33.939, 67.709),
|
||||
"pakistan": (30.375, 69.345),
|
||||
"india": (20.593, 78.962),
|
||||
" uk ": (55.378, -3.435),
|
||||
"london": (51.507, -0.127),
|
||||
"france": (46.227, 2.213),
|
||||
"paris": (48.856, 2.352),
|
||||
"germany": (51.165, 10.451),
|
||||
"berlin": (52.520, 13.405),
|
||||
"sudan": (12.862, 30.217),
|
||||
"congo": (-4.038, 21.758),
|
||||
"south africa": (-30.559, 22.937),
|
||||
"nigeria": (9.082, 8.675),
|
||||
"egypt": (26.820, 30.802),
|
||||
"zimbabwe": (-19.015, 29.154),
|
||||
"kenya": (-1.292, 36.821),
|
||||
"libya": (26.335, 17.228),
|
||||
"mali": (17.570, -3.996),
|
||||
"niger": (17.607, 8.081),
|
||||
"somalia": (5.152, 46.199),
|
||||
"ethiopia": (9.145, 40.489),
|
||||
"australia": (-25.274, 133.775),
|
||||
"middle east": (31.500, 34.800),
|
||||
"europe": (48.800, 2.300),
|
||||
"africa": (0.000, 25.000),
|
||||
"america": (38.900, -77.000),
|
||||
"south america": (-14.200, -51.900),
|
||||
"asia": (34.000, 100.000),
|
||||
"california": (36.778, -119.417),
|
||||
"texas": (31.968, -99.901),
|
||||
"florida": (27.994, -81.760),
|
||||
"new york": (40.712, -74.006),
|
||||
"virginia": (37.431, -78.656),
|
||||
"british columbia": (53.726, -127.647),
|
||||
"ontario": (51.253, -85.323),
|
||||
"quebec": (52.939, -73.549),
|
||||
"delhi": (28.704, 77.102),
|
||||
"new delhi": (28.613, 77.209),
|
||||
"mumbai": (19.076, 72.877),
|
||||
"shanghai": (31.230, 121.473),
|
||||
"hong kong": (22.319, 114.169),
|
||||
"istanbul": (41.008, 28.978),
|
||||
"dubai": (25.204, 55.270),
|
||||
"singapore": (1.352, 103.819),
|
||||
"bangkok": (13.756, 100.501),
|
||||
"jakarta": (-6.208, 106.845),
|
||||
}
|
||||
|
||||
|
||||
def fetch_news():
|
||||
from services.news_feed_config import get_feeds
|
||||
feed_config = get_feeds()
|
||||
feeds = {f["name"]: f["url"] for f in feed_config}
|
||||
source_weights = {f["name"]: f["weight"] for f in feed_config}
|
||||
|
||||
clusters = {}
|
||||
_cluster_grid = {}
|
||||
|
||||
def _fetch_feed(item):
|
||||
source_name, url = item
|
||||
try:
|
||||
xml_data = fetch_with_curl(url, timeout=10).text
|
||||
return source_name, feedparser.parse(xml_data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Feed {source_name} failed: {e}")
|
||||
return source_name, None
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=len(feeds)) as pool:
|
||||
feed_results = list(pool.map(_fetch_feed, feeds.items()))
|
||||
|
||||
for source_name, feed in feed_results:
|
||||
if not feed:
|
||||
continue
|
||||
for entry in feed.entries[:5]:
|
||||
title = entry.get('title', '')
|
||||
summary = entry.get('summary', '')
|
||||
|
||||
_seismic_kw = ["earthquake", "seismic", "quake", "tremor", "magnitude", "richter"]
|
||||
_text_lower = (title + " " + summary).lower()
|
||||
if any(kw in _text_lower for kw in _seismic_kw):
|
||||
continue
|
||||
|
||||
if source_name == "GDACS":
|
||||
alert_level = entry.get("gdacs_alertlevel", "Green")
|
||||
if alert_level == "Red": risk_score = 10
|
||||
elif alert_level == "Orange": risk_score = 7
|
||||
else: risk_score = 4
|
||||
else:
|
||||
risk_keywords = ['war', 'missile', 'strike', 'attack', 'crisis', 'tension', 'military', 'conflict', 'defense', 'clash', 'nuclear']
|
||||
text = (title + " " + summary).lower()
|
||||
|
||||
risk_score = 1
|
||||
for kw in risk_keywords:
|
||||
if kw in text:
|
||||
risk_score += 2
|
||||
risk_score = min(10, risk_score)
|
||||
|
||||
keyword_coords = _KEYWORD_COORDS
|
||||
|
||||
lat, lng = None, None
|
||||
|
||||
if 'georss_point' in entry:
|
||||
geo_parts = entry['georss_point'].split()
|
||||
if len(geo_parts) == 2:
|
||||
lat, lng = float(geo_parts[0]), float(geo_parts[1])
|
||||
elif 'where' in entry and hasattr(entry['where'], 'coordinates'):
|
||||
coords = entry['where'].coordinates
|
||||
lat, lng = coords[1], coords[0]
|
||||
|
||||
if lat is None:
|
||||
# text may not be defined yet for GDACS path
|
||||
text = (title + " " + summary).lower()
|
||||
padded_text = f" {text} "
|
||||
for kw, coords in keyword_coords.items():
|
||||
if kw.startswith(" ") or kw.endswith(" "):
|
||||
if kw in padded_text:
|
||||
lat, lng = coords
|
||||
break
|
||||
else:
|
||||
if re.search(r'\b' + re.escape(kw) + r'\b', text):
|
||||
lat, lng = coords
|
||||
break
|
||||
|
||||
if lat is not None:
|
||||
key = None
|
||||
cell_x, cell_y = int(lng // 4), int(lat // 4)
|
||||
for dx in range(-1, 2):
|
||||
for dy in range(-1, 2):
|
||||
for ckey in _cluster_grid.get((cell_x + dx, cell_y + dy), []):
|
||||
parts = ckey.split(",")
|
||||
elat, elng = float(parts[0]), float(parts[1])
|
||||
if ((lat - elat)**2 + (lng - elng)**2)**0.5 < 4.0:
|
||||
key = ckey
|
||||
break
|
||||
if key:
|
||||
break
|
||||
if key:
|
||||
break
|
||||
if key is None:
|
||||
key = f"{lat},{lng}"
|
||||
_cluster_grid.setdefault((cell_x, cell_y), []).append(key)
|
||||
else:
|
||||
key = title
|
||||
|
||||
if key not in clusters:
|
||||
clusters[key] = []
|
||||
|
||||
clusters[key].append({
|
||||
"title": title,
|
||||
"link": entry.get('link', ''),
|
||||
"published": entry.get('published', ''),
|
||||
"source": source_name,
|
||||
"risk_score": risk_score,
|
||||
"coords": [lat, lng] if lat is not None else None
|
||||
})
|
||||
|
||||
news_items = []
|
||||
for key, articles in clusters.items():
|
||||
articles.sort(key=lambda x: (x['risk_score'], source_weights.get(x["source"], 0)), reverse=True)
|
||||
max_risk = articles[0]['risk_score']
|
||||
|
||||
top_article = articles[0]
|
||||
news_items.append({
|
||||
"title": top_article["title"],
|
||||
"link": top_article["link"],
|
||||
"published": top_article["published"],
|
||||
"source": top_article["source"],
|
||||
"risk_score": max_risk,
|
||||
"coords": top_article["coords"],
|
||||
"cluster_count": len(articles),
|
||||
"articles": articles,
|
||||
"machine_assessment": None
|
||||
})
|
||||
|
||||
news_items.sort(key=lambda x: x['risk_score'], reverse=True)
|
||||
with _data_lock:
|
||||
latest_data['news'] = news_items
|
||||
_mark_fresh("news")
|
||||
@@ -0,0 +1,205 @@
|
||||
"""Plane-Alert DB — load and enrich aircraft with tracked metadata."""
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger("services.data_fetcher")
|
||||
|
||||
# Exact category -> color mapping for all 53 known categories.
|
||||
# O(1) dict lookup — no keyword scanning, no false positives.
|
||||
_CATEGORY_COLOR: dict[str, str] = {
|
||||
# YELLOW — Military / Intelligence / Defense
|
||||
"USAF": "yellow",
|
||||
"Other Air Forces": "yellow",
|
||||
"Toy Soldiers": "yellow",
|
||||
"Oxcart": "yellow",
|
||||
"United States Navy": "yellow",
|
||||
"GAF": "yellow",
|
||||
"Hired Gun": "yellow",
|
||||
"United States Marine Corps": "yellow",
|
||||
"Gunship": "yellow",
|
||||
"RAF": "yellow",
|
||||
"Other Navies": "yellow",
|
||||
"Special Forces": "yellow",
|
||||
"Zoomies": "yellow",
|
||||
"Royal Navy Fleet Air Arm": "yellow",
|
||||
"Army Air Corps": "yellow",
|
||||
"Aerobatic Teams": "yellow",
|
||||
"UAV": "yellow",
|
||||
"Ukraine": "yellow",
|
||||
"Nuclear": "yellow",
|
||||
# LIME — Emergency / Medical / Rescue / Fire
|
||||
"Flying Doctors": "#32cd32",
|
||||
"Aerial Firefighter": "#32cd32",
|
||||
"Coastguard": "#32cd32",
|
||||
# BLUE — Government / Law Enforcement / Civil
|
||||
"Police Forces": "blue",
|
||||
"Governments": "blue",
|
||||
"Quango": "blue",
|
||||
"UK National Police Air Service": "blue",
|
||||
"CAP": "blue",
|
||||
# BLACK — Privacy / PIA
|
||||
"PIA": "black",
|
||||
# RED — Dictator / Oligarch
|
||||
"Dictator Alert": "red",
|
||||
"Da Comrade": "red",
|
||||
"Oligarch": "red",
|
||||
# HOT PINK — High Value Assets / VIP / Celebrity
|
||||
"Head of State": "#ff1493",
|
||||
"Royal Aircraft": "#ff1493",
|
||||
"Don't you know who I am?": "#ff1493",
|
||||
"As Seen on TV": "#ff1493",
|
||||
"Bizjets": "#ff1493",
|
||||
"Vanity Plate": "#ff1493",
|
||||
"Football": "#ff1493",
|
||||
# ORANGE — Joe Cool
|
||||
"Joe Cool": "orange",
|
||||
# WHITE — Climate Crisis
|
||||
"Climate Crisis": "white",
|
||||
# PURPLE — General Tracked / Other Notable
|
||||
"Historic": "purple",
|
||||
"Jump Johnny Jump": "purple",
|
||||
"Ptolemy would be proud": "purple",
|
||||
"Distinctive": "purple",
|
||||
"Dogs with Jobs": "purple",
|
||||
"You came here in that thing?": "purple",
|
||||
"Big Hello": "purple",
|
||||
"Watch Me Fly": "purple",
|
||||
"Perfectly Serviceable Aircraft": "purple",
|
||||
"Jesus he Knows me": "purple",
|
||||
"Gas Bags": "purple",
|
||||
"Radiohead": "purple",
|
||||
}
|
||||
|
||||
def _category_to_color(cat: str) -> str:
|
||||
"""O(1) exact lookup. Unknown categories default to purple."""
|
||||
return _CATEGORY_COLOR.get(cat, "purple")
|
||||
|
||||
_PLANE_ALERT_DB: dict = {}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# POTUS Fleet — override colors and operator names for presidential aircraft.
|
||||
# ---------------------------------------------------------------------------
|
||||
_POTUS_FLEET: dict[str, dict] = {
|
||||
"ADFDF8": {"color": "#ff1493", "operator": "Air Force One (82-8000)", "category": "Head of State", "wiki": "Air_Force_One", "fleet": "AF1"},
|
||||
"ADFDF9": {"color": "#ff1493", "operator": "Air Force One (92-9000)", "category": "Head of State", "wiki": "Air_Force_One", "fleet": "AF1"},
|
||||
"ADFEB7": {"color": "blue", "operator": "Air Force Two (98-0001)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"},
|
||||
"ADFEB8": {"color": "blue", "operator": "Air Force Two (98-0002)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"},
|
||||
"ADFEB9": {"color": "blue", "operator": "Air Force Two (99-0003)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"},
|
||||
"ADFEBA": {"color": "blue", "operator": "Air Force Two (99-0004)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"},
|
||||
"AE4AE6": {"color": "blue", "operator": "Air Force Two (09-0015)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"},
|
||||
"AE4AE8": {"color": "blue", "operator": "Air Force Two (09-0016)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"},
|
||||
"AE4AEA": {"color": "blue", "operator": "Air Force Two (09-0017)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"},
|
||||
"AE4AEC": {"color": "blue", "operator": "Air Force Two (19-0018)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"},
|
||||
"AE0865": {"color": "#ff1493", "operator": "Marine One (VH-3D)", "category": "Head of State", "wiki": "Marine_One", "fleet": "M1"},
|
||||
"AE5E76": {"color": "#ff1493", "operator": "Marine One (VH-92A)", "category": "Head of State", "wiki": "Marine_One", "fleet": "M1"},
|
||||
"AE5E77": {"color": "#ff1493", "operator": "Marine One (VH-92A)", "category": "Head of State", "wiki": "Marine_One", "fleet": "M1"},
|
||||
"AE5E79": {"color": "#ff1493", "operator": "Marine One (VH-92A)", "category": "Head of State", "wiki": "Marine_One", "fleet": "M1"},
|
||||
}
|
||||
|
||||
def _load_plane_alert_db():
|
||||
"""Load plane_alert_db.json (exported from SQLite) into memory."""
|
||||
global _PLANE_ALERT_DB
|
||||
json_path = os.path.join(
|
||||
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
|
||||
"data", "plane_alert_db.json"
|
||||
)
|
||||
if not os.path.exists(json_path):
|
||||
logger.warning(f"Plane-Alert DB not found at {json_path}")
|
||||
return
|
||||
try:
|
||||
with open(json_path, "r", encoding="utf-8") as fh:
|
||||
raw = json.load(fh)
|
||||
for icao_hex, info in raw.items():
|
||||
info["color"] = _category_to_color(info.get("category", ""))
|
||||
override = _POTUS_FLEET.get(icao_hex)
|
||||
if override:
|
||||
info["color"] = override["color"]
|
||||
info["operator"] = override["operator"]
|
||||
info["category"] = override["category"]
|
||||
info["wiki"] = override.get("wiki", "")
|
||||
info["potus_fleet"] = override.get("fleet", "")
|
||||
_PLANE_ALERT_DB[icao_hex] = info
|
||||
logger.info(f"Plane-Alert DB loaded: {len(_PLANE_ALERT_DB)} aircraft")
|
||||
except (IOError, OSError, json.JSONDecodeError, ValueError, KeyError) as e:
|
||||
logger.error(f"Failed to load Plane-Alert DB: {e}")
|
||||
|
||||
_load_plane_alert_db()
|
||||
|
||||
def enrich_with_plane_alert(flight: dict) -> dict:
|
||||
"""If flight's icao24 is in the Plane-Alert DB, add alert metadata."""
|
||||
icao = flight.get("icao24", "").strip().upper()
|
||||
if icao and icao in _PLANE_ALERT_DB:
|
||||
info = _PLANE_ALERT_DB[icao]
|
||||
flight["alert_category"] = info["category"]
|
||||
flight["alert_color"] = info["color"]
|
||||
flight["alert_operator"] = info["operator"]
|
||||
flight["alert_type"] = info["ac_type"]
|
||||
flight["alert_tags"] = info["tags"]
|
||||
flight["alert_link"] = info["link"]
|
||||
if info.get("wiki"):
|
||||
flight["alert_wiki"] = info["wiki"]
|
||||
if info.get("potus_fleet"):
|
||||
flight["potus_fleet"] = info["potus_fleet"]
|
||||
if info["registration"]:
|
||||
flight["registration"] = info["registration"]
|
||||
return flight
|
||||
|
||||
_TRACKED_NAMES_DB: dict = {}
|
||||
|
||||
def _load_tracked_names():
|
||||
global _TRACKED_NAMES_DB
|
||||
json_path = os.path.join(
|
||||
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
|
||||
"data", "tracked_names.json"
|
||||
)
|
||||
if not os.path.exists(json_path):
|
||||
return
|
||||
try:
|
||||
with open(json_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
for name, info in data.get("details", {}).items():
|
||||
cat = info.get("category", "Other")
|
||||
for reg in info.get("registrations", []):
|
||||
reg_clean = reg.strip().upper()
|
||||
if reg_clean:
|
||||
_TRACKED_NAMES_DB[reg_clean] = {"name": name, "category": cat}
|
||||
logger.info(f"Tracked Names DB loaded: {len(_TRACKED_NAMES_DB)} registrations")
|
||||
except (IOError, OSError, json.JSONDecodeError, ValueError, KeyError) as e:
|
||||
logger.error(f"Failed to load Tracked Names DB: {e}")
|
||||
|
||||
_load_tracked_names()
|
||||
|
||||
def enrich_with_tracked_names(flight: dict) -> dict:
|
||||
"""If flight's registration matches our Excel extraction, tag it as tracked."""
|
||||
icao = flight.get("icao24", "").strip().upper()
|
||||
if icao in _POTUS_FLEET:
|
||||
return flight
|
||||
|
||||
reg = flight.get("registration", "").strip().upper()
|
||||
callsign = flight.get("callsign", "").strip().upper()
|
||||
|
||||
match = None
|
||||
if reg and reg in _TRACKED_NAMES_DB:
|
||||
match = _TRACKED_NAMES_DB[reg]
|
||||
elif callsign and callsign in _TRACKED_NAMES_DB:
|
||||
match = _TRACKED_NAMES_DB[callsign]
|
||||
|
||||
if match:
|
||||
name = match["name"]
|
||||
flight["alert_operator"] = name
|
||||
flight["alert_category"] = match["category"]
|
||||
|
||||
name_lower = name.lower()
|
||||
is_gov = any(w in name_lower for w in ['state of ', 'government', 'republic', 'ministry', 'department', 'federal', 'cia'])
|
||||
is_law = any(w in name_lower for w in ['police', 'marshal', 'sheriff', 'douane', 'customs', 'patrol', 'gendarmerie', 'guardia', 'law enforcement'])
|
||||
is_med = any(w in name_lower for w in ['fire', 'bomberos', 'ambulance', 'paramedic', 'medevac', 'rescue', 'hospital', 'medical', 'lifeflight'])
|
||||
|
||||
if is_gov or is_law:
|
||||
flight["alert_color"] = "blue"
|
||||
elif is_med:
|
||||
flight["alert_color"] = "#32cd32"
|
||||
elif "alert_color" not in flight:
|
||||
flight["alert_color"] = "pink"
|
||||
|
||||
return flight
|
||||
@@ -0,0 +1,354 @@
|
||||
"""Satellite tracking — CelesTrak/TLE fetch, SGP4 propagation, intel classification."""
|
||||
import math
|
||||
import time
|
||||
import json
|
||||
import re
|
||||
import logging
|
||||
import concurrent.futures
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timedelta
|
||||
from sgp4.api import Satrec, WGS72, jday
|
||||
from services.network_utils import fetch_with_curl
|
||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
||||
|
||||
logger = logging.getLogger("services.data_fetcher")
|
||||
|
||||
|
||||
def _gmst(jd_ut1):
|
||||
"""Greenwich Mean Sidereal Time in radians from Julian Date."""
|
||||
t = (jd_ut1 - 2451545.0) / 36525.0
|
||||
gmst_sec = 67310.54841 + (876600.0 * 3600 + 8640184.812866) * t + 0.093104 * t * t - 6.2e-6 * t * t * t
|
||||
gmst_rad = (gmst_sec % 86400) / 86400.0 * 2 * math.pi
|
||||
return gmst_rad
|
||||
|
||||
|
||||
# Satellite GP data cache
|
||||
_sat_gp_cache = {"data": None, "last_fetch": 0, "source": "none"}
|
||||
_sat_classified_cache = {"data": None, "gp_fetch_ts": 0}
|
||||
_SAT_CACHE_PATH = Path(__file__).parent.parent.parent / "data" / "sat_gp_cache.json"
|
||||
|
||||
def _load_sat_cache():
|
||||
"""Load satellite GP data from local disk cache."""
|
||||
try:
|
||||
if _SAT_CACHE_PATH.exists():
|
||||
import os
|
||||
age_hours = (time.time() - os.path.getmtime(str(_SAT_CACHE_PATH))) / 3600
|
||||
if age_hours < 48:
|
||||
with open(_SAT_CACHE_PATH, "r") as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, list) and len(data) > 10:
|
||||
logger.info(f"Satellites: Loaded {len(data)} records from disk cache ({age_hours:.1f}h old)")
|
||||
return data
|
||||
else:
|
||||
logger.info(f"Satellites: Disk cache is {age_hours:.0f}h old, will try fresh fetch")
|
||||
except Exception as e:
|
||||
logger.warning(f"Satellites: Failed to load disk cache: {e}")
|
||||
return None
|
||||
|
||||
def _save_sat_cache(data):
|
||||
"""Save satellite GP data to local disk cache."""
|
||||
try:
|
||||
_SAT_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(_SAT_CACHE_PATH, "w") as f:
|
||||
json.dump(data, f)
|
||||
logger.info(f"Satellites: Saved {len(data)} records to disk cache")
|
||||
except Exception as e:
|
||||
logger.warning(f"Satellites: Failed to save disk cache: {e}")
|
||||
|
||||
|
||||
# Satellite intelligence classification database
|
||||
_SAT_INTEL_DB = [
|
||||
("USA 224", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
|
||||
("USA 245", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
|
||||
("USA 290", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
|
||||
("USA 314", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
|
||||
("USA 338", {"country": "USA", "mission": "military_recon", "sat_type": "Keyhole Successor", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
|
||||
("TOPAZ", {"country": "Russia", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)"}),
|
||||
("PERSONA", {"country": "Russia", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)"}),
|
||||
("KONDOR", {"country": "Russia", "mission": "military_sar", "sat_type": "SAR Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Kondor_(satellite)"}),
|
||||
("BARS-M", {"country": "Russia", "mission": "military_recon", "sat_type": "Mapping Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Bars-M"}),
|
||||
("YAOGAN", {"country": "China", "mission": "military_recon", "sat_type": "Remote Sensing / ELINT", "wiki": "https://en.wikipedia.org/wiki/Yaogan"}),
|
||||
("GAOFEN", {"country": "China", "mission": "military_recon", "sat_type": "High-Res Imaging", "wiki": "https://en.wikipedia.org/wiki/Gaofen"}),
|
||||
("JILIN", {"country": "China", "mission": "commercial_imaging", "sat_type": "Video / Imaging", "wiki": "https://en.wikipedia.org/wiki/Jilin-1"}),
|
||||
("OFEK", {"country": "Israel", "mission": "military_recon", "sat_type": "Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Ofeq"}),
|
||||
("CSO", {"country": "France", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/CSO_(satellite)"}),
|
||||
("IGS", {"country": "Japan", "mission": "military_recon", "sat_type": "Intelligence Gathering", "wiki": "https://en.wikipedia.org/wiki/Information_Gathering_Satellite"}),
|
||||
("CAPELLA", {"country": "USA", "mission": "sar", "sat_type": "SAR Imaging", "wiki": "https://en.wikipedia.org/wiki/Capella_Space"}),
|
||||
("ICEYE", {"country": "Finland", "mission": "sar", "sat_type": "SAR Microsatellite", "wiki": "https://en.wikipedia.org/wiki/ICEYE"}),
|
||||
("COSMO-SKYMED", {"country": "Italy", "mission": "sar", "sat_type": "SAR Constellation", "wiki": "https://en.wikipedia.org/wiki/COSMO-SkyMed"}),
|
||||
("TANDEM", {"country": "Germany", "mission": "sar", "sat_type": "SAR Interferometry", "wiki": "https://en.wikipedia.org/wiki/TanDEM-X"}),
|
||||
("PAZ", {"country": "Spain", "mission": "sar", "sat_type": "SAR Imaging", "wiki": "https://en.wikipedia.org/wiki/PAZ_(satellite)"}),
|
||||
("WORLDVIEW", {"country": "USA", "mission": "commercial_imaging", "sat_type": "Maxar High-Res", "wiki": "https://en.wikipedia.org/wiki/WorldView-3"}),
|
||||
("GEOEYE", {"country": "USA", "mission": "commercial_imaging", "sat_type": "Maxar Imaging", "wiki": "https://en.wikipedia.org/wiki/GeoEye-1"}),
|
||||
("PLEIADES", {"country": "France", "mission": "commercial_imaging", "sat_type": "Airbus Imaging", "wiki": "https://en.wikipedia.org/wiki/Pl%C3%A9iades_(satellite)"}),
|
||||
("SPOT", {"country": "France", "mission": "commercial_imaging", "sat_type": "Airbus Medium-Res", "wiki": "https://en.wikipedia.org/wiki/SPOT_(satellite)"}),
|
||||
("PLANET", {"country": "USA", "mission": "commercial_imaging", "sat_type": "PlanetScope", "wiki": "https://en.wikipedia.org/wiki/Planet_Labs"}),
|
||||
("SKYSAT", {"country": "USA", "mission": "commercial_imaging", "sat_type": "Planet Video", "wiki": "https://en.wikipedia.org/wiki/SkySat"}),
|
||||
("BLACKSKY", {"country": "USA", "mission": "commercial_imaging", "sat_type": "BlackSky Imaging", "wiki": "https://en.wikipedia.org/wiki/BlackSky"}),
|
||||
("NROL", {"country": "USA", "mission": "sigint", "sat_type": "Classified NRO", "wiki": "https://en.wikipedia.org/wiki/National_Reconnaissance_Office"}),
|
||||
("MENTOR", {"country": "USA", "mission": "sigint", "sat_type": "SIGINT / ELINT", "wiki": "https://en.wikipedia.org/wiki/Mentor_(satellite)"}),
|
||||
("LUCH", {"country": "Russia", "mission": "sigint", "sat_type": "Relay / SIGINT", "wiki": "https://en.wikipedia.org/wiki/Luch_(satellite)"}),
|
||||
("SHIJIAN", {"country": "China", "mission": "sigint", "sat_type": "ELINT / Tech Demo", "wiki": "https://en.wikipedia.org/wiki/Shijian"}),
|
||||
("NAVSTAR", {"country": "USA", "mission": "navigation", "sat_type": "GPS", "wiki": "https://en.wikipedia.org/wiki/GPS_satellite_blocks"}),
|
||||
("GLONASS", {"country": "Russia", "mission": "navigation", "sat_type": "GLONASS", "wiki": "https://en.wikipedia.org/wiki/GLONASS"}),
|
||||
("BEIDOU", {"country": "China", "mission": "navigation", "sat_type": "BeiDou", "wiki": "https://en.wikipedia.org/wiki/BeiDou"}),
|
||||
("GALILEO", {"country": "EU", "mission": "navigation", "sat_type": "Galileo", "wiki": "https://en.wikipedia.org/wiki/Galileo_(satellite_navigation)"}),
|
||||
("SBIRS", {"country": "USA", "mission": "early_warning", "sat_type": "Missile Warning", "wiki": "https://en.wikipedia.org/wiki/Space-Based_Infrared_System"}),
|
||||
("TUNDRA", {"country": "Russia", "mission": "early_warning", "sat_type": "Missile Warning", "wiki": "https://en.wikipedia.org/wiki/Tundra_(satellite)"}),
|
||||
("ISS", {"country": "Intl", "mission": "space_station", "sat_type": "Space Station", "wiki": "https://en.wikipedia.org/wiki/International_Space_Station"}),
|
||||
("TIANGONG", {"country": "China", "mission": "space_station", "sat_type": "Space Station", "wiki": "https://en.wikipedia.org/wiki/Tiangong_space_station"}),
|
||||
]
|
||||
|
||||
|
||||
def _parse_tle_to_gp(name, norad_id, line1, line2):
|
||||
"""Convert TLE two-line element to CelesTrak GP-style dict."""
|
||||
try:
|
||||
incl = float(line2[8:16].strip())
|
||||
raan = float(line2[17:25].strip())
|
||||
ecc = float("0." + line2[26:33].strip())
|
||||
argp = float(line2[34:42].strip())
|
||||
ma = float(line2[43:51].strip())
|
||||
mm = float(line2[52:63].strip())
|
||||
bstar_str = line1[53:61].strip()
|
||||
if bstar_str:
|
||||
mantissa = float(bstar_str[:-2]) / 1e5
|
||||
exponent = int(bstar_str[-2:])
|
||||
bstar = mantissa * (10 ** exponent)
|
||||
else:
|
||||
bstar = 0.0
|
||||
epoch_yr = int(line1[18:20])
|
||||
epoch_day = float(line1[20:32].strip())
|
||||
year = 2000 + epoch_yr if epoch_yr < 57 else 1900 + epoch_yr
|
||||
epoch_dt = datetime(year, 1, 1) + timedelta(days=epoch_day - 1)
|
||||
return {
|
||||
"OBJECT_NAME": name,
|
||||
"NORAD_CAT_ID": norad_id,
|
||||
"MEAN_MOTION": mm,
|
||||
"ECCENTRICITY": ecc,
|
||||
"INCLINATION": incl,
|
||||
"RA_OF_ASC_NODE": raan,
|
||||
"ARG_OF_PERICENTER": argp,
|
||||
"MEAN_ANOMALY": ma,
|
||||
"BSTAR": bstar,
|
||||
"EPOCH": epoch_dt.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
}
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _fetch_satellites_from_tle_api():
|
||||
"""Fallback: fetch satellite TLEs from tle.ivanstanojevic.me when CelesTrak is blocked."""
|
||||
search_terms = set()
|
||||
for key, _ in _SAT_INTEL_DB:
|
||||
term = key.split()[0] if len(key.split()) > 1 and key.split()[0] in ("USA", "NROL") else key
|
||||
search_terms.add(term)
|
||||
|
||||
def _fetch_term(term):
|
||||
results = []
|
||||
try:
|
||||
url = f"https://tle.ivanstanojevic.me/api/tle/?search={term}&page_size=100&format=json"
|
||||
response = fetch_with_curl(url, timeout=8)
|
||||
if response.status_code != 200:
|
||||
return results
|
||||
data = response.json()
|
||||
for member in data.get("member", []):
|
||||
gp = _parse_tle_to_gp(
|
||||
member.get("name", "UNKNOWN"),
|
||||
member.get("satelliteId"),
|
||||
member.get("line1", ""),
|
||||
member.get("line2", ""),
|
||||
)
|
||||
if gp:
|
||||
results.append(gp)
|
||||
except Exception as e:
|
||||
logger.debug(f"TLE fallback search '{term}' failed: {e}")
|
||||
return results
|
||||
|
||||
all_results = []
|
||||
seen_ids = set()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
|
||||
future_map = {executor.submit(_fetch_term, term): term for term in search_terms}
|
||||
for future in concurrent.futures.as_completed(future_map):
|
||||
for gp in future.result():
|
||||
sat_id = gp.get("NORAD_CAT_ID")
|
||||
if sat_id not in seen_ids:
|
||||
seen_ids.add(sat_id)
|
||||
all_results.append(gp)
|
||||
|
||||
return all_results
|
||||
|
||||
|
||||
def fetch_satellites():
|
||||
sats = []
|
||||
try:
|
||||
now_ts = time.time()
|
||||
if _sat_gp_cache["data"] is None or (now_ts - _sat_gp_cache["last_fetch"]) > 1800:
|
||||
gp_urls = [
|
||||
"https://celestrak.org/NORAD/elements/gp.php?GROUP=active&FORMAT=json",
|
||||
"https://celestrak.com/NORAD/elements/gp.php?GROUP=active&FORMAT=json",
|
||||
]
|
||||
for url in gp_urls:
|
||||
try:
|
||||
response = fetch_with_curl(url, timeout=5)
|
||||
if response.status_code == 200:
|
||||
gp_data = response.json()
|
||||
if isinstance(gp_data, list) and len(gp_data) > 100:
|
||||
_sat_gp_cache["data"] = gp_data
|
||||
_sat_gp_cache["last_fetch"] = now_ts
|
||||
_sat_gp_cache["source"] = "celestrak"
|
||||
_save_sat_cache(gp_data)
|
||||
logger.info(f"Satellites: Downloaded {len(gp_data)} GP records from {url}")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning(f"Satellites: Failed to fetch from {url}: {e}")
|
||||
continue
|
||||
|
||||
if _sat_gp_cache["data"] is None:
|
||||
logger.info("Satellites: CelesTrak unreachable, trying TLE fallback API...")
|
||||
try:
|
||||
fallback_data = _fetch_satellites_from_tle_api()
|
||||
if fallback_data and len(fallback_data) > 10:
|
||||
_sat_gp_cache["data"] = fallback_data
|
||||
_sat_gp_cache["last_fetch"] = now_ts
|
||||
_sat_gp_cache["source"] = "tle_api"
|
||||
_save_sat_cache(fallback_data)
|
||||
logger.info(f"Satellites: Got {len(fallback_data)} records from TLE fallback API")
|
||||
except Exception as e:
|
||||
logger.error(f"Satellites: TLE fallback also failed: {e}")
|
||||
|
||||
if _sat_gp_cache["data"] is None:
|
||||
disk_data = _load_sat_cache()
|
||||
if disk_data:
|
||||
_sat_gp_cache["data"] = disk_data
|
||||
_sat_gp_cache["last_fetch"] = now_ts - 1500
|
||||
_sat_gp_cache["source"] = "disk_cache"
|
||||
|
||||
data = _sat_gp_cache["data"]
|
||||
if not data:
|
||||
logger.warning("No satellite GP data available from any source")
|
||||
with _data_lock:
|
||||
latest_data["satellites"] = sats
|
||||
return
|
||||
|
||||
if _sat_classified_cache["gp_fetch_ts"] == _sat_gp_cache["last_fetch"] and _sat_classified_cache["data"]:
|
||||
classified = _sat_classified_cache["data"]
|
||||
logger.info(f"Satellites: Using cached classification ({len(classified)} sats, TLEs unchanged)")
|
||||
else:
|
||||
classified = []
|
||||
for sat in data:
|
||||
name = sat.get("OBJECT_NAME", "UNKNOWN").upper()
|
||||
intel = None
|
||||
for key, meta in _SAT_INTEL_DB:
|
||||
if key.upper() in name:
|
||||
intel = dict(meta)
|
||||
break
|
||||
if not intel:
|
||||
continue
|
||||
entry = {
|
||||
"id": sat.get("NORAD_CAT_ID"),
|
||||
"name": sat.get("OBJECT_NAME", "UNKNOWN"),
|
||||
"MEAN_MOTION": sat.get("MEAN_MOTION"),
|
||||
"ECCENTRICITY": sat.get("ECCENTRICITY"),
|
||||
"INCLINATION": sat.get("INCLINATION"),
|
||||
"RA_OF_ASC_NODE": sat.get("RA_OF_ASC_NODE"),
|
||||
"ARG_OF_PERICENTER": sat.get("ARG_OF_PERICENTER"),
|
||||
"MEAN_ANOMALY": sat.get("MEAN_ANOMALY"),
|
||||
"BSTAR": sat.get("BSTAR"),
|
||||
"EPOCH": sat.get("EPOCH"),
|
||||
}
|
||||
entry.update(intel)
|
||||
classified.append(entry)
|
||||
_sat_classified_cache["data"] = classified
|
||||
_sat_classified_cache["gp_fetch_ts"] = _sat_gp_cache["last_fetch"]
|
||||
logger.info(f"Satellites: {len(classified)} intel-classified out of {len(data)} total in catalog")
|
||||
|
||||
all_sats = classified
|
||||
|
||||
now = datetime.utcnow()
|
||||
jd, fr = jday(now.year, now.month, now.day, now.hour, now.minute, now.second + now.microsecond / 1e6)
|
||||
|
||||
for s in all_sats:
|
||||
try:
|
||||
mean_motion = s.get('MEAN_MOTION')
|
||||
ecc = s.get('ECCENTRICITY')
|
||||
incl = s.get('INCLINATION')
|
||||
raan = s.get('RA_OF_ASC_NODE')
|
||||
argp = s.get('ARG_OF_PERICENTER')
|
||||
ma = s.get('MEAN_ANOMALY')
|
||||
bstar = s.get('BSTAR', 0)
|
||||
epoch_str = s.get('EPOCH')
|
||||
norad_id = s.get('id', 0)
|
||||
|
||||
if mean_motion is None or ecc is None or incl is None:
|
||||
continue
|
||||
|
||||
epoch_dt = datetime.strptime(epoch_str[:19], '%Y-%m-%dT%H:%M:%S')
|
||||
epoch_jd, epoch_fr = jday(epoch_dt.year, epoch_dt.month, epoch_dt.day,
|
||||
epoch_dt.hour, epoch_dt.minute, epoch_dt.second)
|
||||
|
||||
sat_obj = Satrec()
|
||||
sat_obj.sgp4init(
|
||||
WGS72, 'i', norad_id,
|
||||
(epoch_jd + epoch_fr) - 2433281.5,
|
||||
bstar, 0.0, 0.0, ecc,
|
||||
math.radians(argp), math.radians(incl),
|
||||
math.radians(ma),
|
||||
mean_motion * 2 * math.pi / 1440.0,
|
||||
math.radians(raan)
|
||||
)
|
||||
|
||||
e, r, v = sat_obj.sgp4(jd, fr)
|
||||
if e != 0:
|
||||
continue
|
||||
|
||||
x, y, z = r
|
||||
gmst = _gmst(jd + fr)
|
||||
lng_rad = math.atan2(y, x) - gmst
|
||||
lat_rad = math.atan2(z, math.sqrt(x*x + y*y))
|
||||
alt_km = math.sqrt(x*x + y*y + z*z) - 6371.0
|
||||
|
||||
s['lat'] = round(math.degrees(lat_rad), 4)
|
||||
lng_deg = math.degrees(lng_rad) % 360
|
||||
s['lng'] = round(lng_deg - 360 if lng_deg > 180 else lng_deg, 4)
|
||||
s['alt_km'] = round(alt_km, 1)
|
||||
|
||||
vx, vy, vz = v
|
||||
omega_e = 7.2921159e-5
|
||||
vx_g = vx + omega_e * y
|
||||
vy_g = vy - omega_e * x
|
||||
vz_g = vz
|
||||
cos_lat = math.cos(lat_rad)
|
||||
sin_lat = math.sin(lat_rad)
|
||||
cos_lng = math.cos(lng_rad + gmst)
|
||||
sin_lng = math.sin(lng_rad + gmst)
|
||||
v_east = -sin_lng * vx_g + cos_lng * vy_g
|
||||
v_north = -sin_lat * cos_lng * vx_g - sin_lat * sin_lng * vy_g + cos_lat * vz_g
|
||||
ground_speed_kms = math.sqrt(v_east**2 + v_north**2)
|
||||
s['speed_knots'] = round(ground_speed_kms * 1943.84, 1)
|
||||
heading_rad = math.atan2(v_east, v_north)
|
||||
s['heading'] = round(math.degrees(heading_rad) % 360, 1)
|
||||
sat_name = s.get('name', '')
|
||||
usa_match = re.search(r'USA[\s\-]*(\d+)', sat_name)
|
||||
if usa_match:
|
||||
s['wiki'] = f"https://en.wikipedia.org/wiki/USA-{usa_match.group(1)}"
|
||||
for k in ('MEAN_MOTION', 'ECCENTRICITY', 'INCLINATION',
|
||||
'RA_OF_ASC_NODE', 'ARG_OF_PERICENTER', 'MEAN_ANOMALY',
|
||||
'BSTAR', 'EPOCH', 'tle1', 'tle2'):
|
||||
s.pop(k, None)
|
||||
sats.append(s)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
logger.info(f"Satellites: {len(classified)} classified, {len(sats)} positioned")
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching satellites: {e}")
|
||||
if sats:
|
||||
with _data_lock:
|
||||
latest_data["satellites"] = sats
|
||||
latest_data["satellite_source"] = _sat_gp_cache.get("source", "none")
|
||||
_mark_fresh("satellites")
|
||||
else:
|
||||
with _data_lock:
|
||||
if not latest_data.get("satellites"):
|
||||
latest_data["satellites"] = []
|
||||
latest_data["satellite_source"] = "none"
|
||||
@@ -1,5 +1,6 @@
|
||||
import requests
|
||||
import logging
|
||||
import zipfile
|
||||
from cachetools import cached, TTLCache
|
||||
from datetime import datetime
|
||||
from services.network_utils import fetch_with_curl
|
||||
@@ -65,7 +66,7 @@ def fetch_ukraine_frontlines():
|
||||
logger.error(f"Failed to fetch parsed Github Raw GeoJSON: {res_geo.status_code}")
|
||||
else:
|
||||
logger.error(f"Failed to fetch Github Tree for Deepstatemap: {res_tree.status_code}")
|
||||
except Exception as e:
|
||||
except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e:
|
||||
logger.error(f"Error fetching DeepStateMap: {e}")
|
||||
return None
|
||||
|
||||
@@ -81,7 +82,7 @@ def _extract_domain(url):
|
||||
if host.startswith('www.'):
|
||||
host = host[4:]
|
||||
return host
|
||||
except Exception:
|
||||
except (ValueError, AttributeError, KeyError): # non-critical
|
||||
return url[:40]
|
||||
|
||||
def _url_to_headline(url):
|
||||
@@ -137,7 +138,7 @@ def _url_to_headline(url):
|
||||
if len(headline) > 90:
|
||||
headline = headline[:87] + '...'
|
||||
return headline
|
||||
except Exception:
|
||||
except (ValueError, AttributeError, KeyError): # non-critical
|
||||
return url[:60]
|
||||
|
||||
|
||||
@@ -226,7 +227,7 @@ def _fetch_article_title(url):
|
||||
|
||||
_article_title_cache[url] = None
|
||||
return None
|
||||
except Exception:
|
||||
except (requests.RequestException, ConnectionError, TimeoutError, ValueError, AttributeError): # non-critical
|
||||
_article_title_cache[url] = None
|
||||
return None
|
||||
|
||||
@@ -242,7 +243,7 @@ def _batch_fetch_titles(urls):
|
||||
url = futures[future]
|
||||
try:
|
||||
results[url] = future.result()
|
||||
except Exception:
|
||||
except Exception: # non-critical: optional title enrichment
|
||||
results[url] = None
|
||||
return results
|
||||
|
||||
@@ -308,7 +309,7 @@ def _parse_gdelt_export_zip(zip_bytes, conflict_codes, seen_locs, features, loc_
|
||||
})
|
||||
except (ValueError, IndexError):
|
||||
continue
|
||||
except Exception as e:
|
||||
except (IOError, OSError, ValueError, KeyError, zipfile.BadZipFile) as e:
|
||||
logger.warning(f"Failed to parse GDELT export zip: {e}")
|
||||
|
||||
def _download_gdelt_export(url):
|
||||
@@ -317,16 +318,72 @@ def _download_gdelt_export(url):
|
||||
res = fetch_with_curl(url, timeout=15)
|
||||
if res.status_code == 200:
|
||||
return res.content
|
||||
except Exception:
|
||||
except (ConnectionError, TimeoutError, OSError): # non-critical
|
||||
pass
|
||||
return None
|
||||
|
||||
@cached(gdelt_cache)
|
||||
def _build_feature_html(features, fetched_titles=None):
|
||||
"""Build URL + headline arrays for frontend rendering.
|
||||
Uses fetched_titles (real article titles) when available, falls back to URL slug parsing."""
|
||||
import html as html_mod
|
||||
for f in features:
|
||||
urls = f["properties"].pop("_urls", [])
|
||||
f["properties"].pop("_domains", None)
|
||||
headlines = []
|
||||
for u in urls:
|
||||
real_title = fetched_titles.get(u) if fetched_titles else None
|
||||
headlines.append(real_title if real_title else _url_to_headline(u))
|
||||
f["properties"]["_urls_list"] = urls
|
||||
f["properties"]["_headlines_list"] = headlines
|
||||
if urls:
|
||||
links = []
|
||||
for u, h in zip(urls, headlines):
|
||||
safe_url = u if u.startswith(('http://', 'https://')) else 'about:blank'
|
||||
safe_h = html_mod.escape(h)
|
||||
links.append(f'<div style="margin-bottom:6px;"><a href="{safe_url}" target="_blank" rel="noopener noreferrer">{safe_h}</a></div>')
|
||||
f["properties"]["html"] = ''.join(links)
|
||||
else:
|
||||
f["properties"]["html"] = html_mod.escape(f["properties"]["name"])
|
||||
f.pop("_loc_key", None)
|
||||
|
||||
|
||||
def _enrich_gdelt_titles_background(features, all_article_urls):
|
||||
"""Background thread: fetch real article titles then update features in-place."""
|
||||
import html as html_mod
|
||||
try:
|
||||
logger.info(f"[BG] Fetching real article titles for {len(all_article_urls)} URLs...")
|
||||
fetched_titles = _batch_fetch_titles(all_article_urls)
|
||||
fetched_count = sum(1 for v in fetched_titles.values() if v)
|
||||
logger.info(f"[BG] Resolved {fetched_count}/{len(all_article_urls)} article titles")
|
||||
|
||||
# Update features in-place with real titles
|
||||
for f in features:
|
||||
urls = f["properties"].get("_urls_list", [])
|
||||
if not urls:
|
||||
continue
|
||||
headlines = []
|
||||
for u in urls:
|
||||
real_title = fetched_titles.get(u)
|
||||
headlines.append(real_title if real_title else _url_to_headline(u))
|
||||
f["properties"]["_headlines_list"] = headlines
|
||||
links = []
|
||||
for u, h in zip(urls, headlines):
|
||||
safe_url = u if u.startswith(('http://', 'https://')) else 'about:blank'
|
||||
safe_h = html_mod.escape(h)
|
||||
links.append(f'<div style="margin-bottom:6px;"><a href="{safe_url}" target="_blank" rel="noopener noreferrer">{safe_h}</a></div>')
|
||||
f["properties"]["html"] = ''.join(links)
|
||||
logger.info(f"[BG] GDELT title enrichment complete")
|
||||
except Exception as e:
|
||||
logger.error(f"[BG] GDELT title enrichment failed: {e}")
|
||||
|
||||
|
||||
def fetch_global_military_incidents():
|
||||
"""
|
||||
Fetches global military/conflict incidents from GDELT Events Export files.
|
||||
Aggregates the last ~8 hours of 15-minute exports to build ~1000 incidents.
|
||||
Returns immediately with URL-slug headlines; enriches with real titles in background.
|
||||
"""
|
||||
import threading
|
||||
from datetime import timedelta
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
@@ -388,45 +445,29 @@ def fetch_global_military_incidents():
|
||||
if zip_bytes:
|
||||
_parse_gdelt_export_zip(zip_bytes, CONFLICT_CODES, seen_locs, features, loc_index)
|
||||
|
||||
# Collect all unique article URLs for batch title fetching
|
||||
# Collect all unique article URLs
|
||||
all_article_urls = set()
|
||||
for f in features:
|
||||
for u in f["properties"].get("_urls", []):
|
||||
if u:
|
||||
all_article_urls.add(u)
|
||||
|
||||
logger.info(f"Fetching real article titles for {len(all_article_urls)} unique URLs...")
|
||||
fetched_titles = _batch_fetch_titles(all_article_urls)
|
||||
fetched_count = sum(1 for v in fetched_titles.values() if v)
|
||||
logger.info(f"Resolved {fetched_count}/{len(all_article_urls)} article titles from HTML")
|
||||
|
||||
# Build URL + headline arrays for frontend rendering
|
||||
for f in features:
|
||||
urls = f["properties"].pop("_urls", [])
|
||||
f["properties"].pop("_domains", None)
|
||||
headlines = []
|
||||
for u in urls:
|
||||
# Try the real fetched title first, then fall back to URL slug parsing
|
||||
real_title = fetched_titles.get(u)
|
||||
headlines.append(real_title if real_title else _url_to_headline(u))
|
||||
f["properties"]["_urls_list"] = urls
|
||||
f["properties"]["_headlines_list"] = headlines
|
||||
import html
|
||||
# Keep html as fallback
|
||||
if urls:
|
||||
links = []
|
||||
for u, h in zip(urls, headlines):
|
||||
safe_url = u if u.startswith(('http://', 'https://')) else 'about:blank'
|
||||
safe_h = html.escape(h)
|
||||
links.append(f'<div style="margin-bottom:6px;"><a href="{safe_url}" target="_blank" rel="noopener noreferrer">{safe_h}</a></div>')
|
||||
f["properties"]["html"] = ''.join(links)
|
||||
else:
|
||||
f["properties"]["html"] = html.escape(f["properties"]["name"])
|
||||
f.pop("_loc_key", None)
|
||||
# Build HTML immediately with URL-slug headlines (instant, no network)
|
||||
_build_feature_html(features)
|
||||
|
||||
logger.info(f"GDELT parsed: {len(features)} conflict locations from {successful} files (titles enriching in background)")
|
||||
|
||||
# Kick off background thread to enrich with real article titles
|
||||
# Features list is shared — background thread updates in-place
|
||||
t = threading.Thread(
|
||||
target=_enrich_gdelt_titles_background,
|
||||
args=(features, all_article_urls),
|
||||
daemon=True,
|
||||
)
|
||||
t.start()
|
||||
|
||||
logger.info(f"GDELT multi-file parsed: {len(features)} conflict locations from {successful} files")
|
||||
return features
|
||||
|
||||
except Exception as e:
|
||||
except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e:
|
||||
logger.error(f"Error fetching GDELT data: {e}")
|
||||
return []
|
||||
|
||||
@@ -6,6 +6,7 @@ Data is embedded as HTML comments inside each entry div.
|
||||
|
||||
import re
|
||||
import logging
|
||||
import requests
|
||||
from cachetools import TTLCache, cached
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -92,6 +93,6 @@ def fetch_kiwisdr_nodes() -> list[dict]:
|
||||
logger.info(f"KiwiSDR: parsed {len(nodes)} online receivers")
|
||||
return nodes
|
||||
|
||||
except Exception as e:
|
||||
except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e:
|
||||
logger.error(f"KiwiSDR fetch exception: {e}")
|
||||
return []
|
||||
|
||||
@@ -23,7 +23,7 @@ def fetch_liveuamap():
|
||||
|
||||
with sync_playwright() as p:
|
||||
# Launching with a real user agent to bypass Turnstile
|
||||
browser = p.chromium.launch(headless=False, args=["--disable-blink-features=AutomationControlled"])
|
||||
browser = p.chromium.launch(headless=True, args=["--disable-blink-features=AutomationControlled"])
|
||||
context = browser.new_context(
|
||||
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
viewport={"width": 1920, "height": 1080},
|
||||
@@ -40,7 +40,7 @@ def fetch_liveuamap():
|
||||
# Wait for the map canvas or markers script to load, max 10s wait
|
||||
try:
|
||||
page.wait_for_timeout(5000)
|
||||
except:
|
||||
except (TimeoutError, OSError): # non-critical: page load delay
|
||||
pass
|
||||
|
||||
html = page.content()
|
||||
@@ -56,8 +56,8 @@ def fetch_liveuamap():
|
||||
# process below
|
||||
html = f"var ovens={ovens_json};"
|
||||
m = re.search(r"var\s+ovens=(.*?);", html, re.DOTALL)
|
||||
except:
|
||||
pass
|
||||
except (ValueError, KeyError, OSError) as e: # non-critical: JS eval fallback
|
||||
logger.debug(f"Could not evaluate ovens JS variable for {region['name']}: {e}")
|
||||
|
||||
if m:
|
||||
json_str = m.group(1).strip()
|
||||
@@ -81,7 +81,7 @@ def fetch_liveuamap():
|
||||
"link": marker.get("link", region["url"]),
|
||||
"region": region["name"]
|
||||
})
|
||||
except Exception as e:
|
||||
except (json.JSONDecodeError, ValueError, KeyError) as e:
|
||||
logger.error(f"Error parsing JSON for {region['name']}: {e}")
|
||||
|
||||
except Exception as e:
|
||||
|
||||
@@ -10,9 +10,10 @@ from urllib3.util.retry import Retry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Reusable session with connection pooling and retry logic
|
||||
# Reusable session with connection pooling and retry logic.
|
||||
# Only retry once (total=1) to fail fast — the curl fallback is the real safety net.
|
||||
_session = requests.Session()
|
||||
_retry = Retry(total=2, backoff_factor=0.5, status_forcelist=[502, 503, 504])
|
||||
_retry = Retry(total=1, backoff_factor=0.3, status_forcelist=[502, 503, 504])
|
||||
_session.mount("https://", HTTPAdapter(max_retries=_retry, pool_maxsize=20))
|
||||
_session.mount("http://", HTTPAdapter(max_retries=_retry, pool_maxsize=10))
|
||||
|
||||
@@ -68,16 +69,19 @@ def fetch_with_curl(url, method="GET", json_data=None, timeout=15, headers=None)
|
||||
pass # Fall through to curl below
|
||||
else:
|
||||
try:
|
||||
# Use a short connect timeout (3s) so firewall blocks fail fast,
|
||||
# but allow the full timeout for reading the response body.
|
||||
req_timeout = (min(3, timeout), timeout)
|
||||
if method == "POST":
|
||||
res = _session.post(url, json=json_data, timeout=timeout, headers=default_headers)
|
||||
res = _session.post(url, json=json_data, timeout=req_timeout, headers=default_headers)
|
||||
else:
|
||||
res = _session.get(url, timeout=timeout, headers=default_headers)
|
||||
res = _session.get(url, timeout=req_timeout, headers=default_headers)
|
||||
res.raise_for_status()
|
||||
# Clear failure caches on success
|
||||
_domain_fail_cache.pop(domain, None)
|
||||
_circuit_breaker.pop(domain, None)
|
||||
return res
|
||||
except Exception as e:
|
||||
except (requests.RequestException, ConnectionError, TimeoutError, OSError) as e:
|
||||
logger.warning(f"Python requests failed for {url} ({e}), falling back to bash curl...")
|
||||
_domain_fail_cache[domain] = time.time()
|
||||
|
||||
@@ -109,7 +113,7 @@ def fetch_with_curl(url, method="GET", json_data=None, timeout=15, headers=None)
|
||||
logger.error(f"bash curl fallback failed: exit={res.returncode} stderr={res.stderr[:200]}")
|
||||
_circuit_breaker[domain] = time.time()
|
||||
return _DummyResponse(500, "")
|
||||
except Exception as curl_e:
|
||||
except (subprocess.SubprocessError, ConnectionError, TimeoutError, OSError) as curl_e:
|
||||
logger.error(f"bash curl fallback exception: {curl_e}")
|
||||
_circuit_breaker[domain] = time.time()
|
||||
return _DummyResponse(500, "")
|
||||
|
||||
@@ -31,7 +31,7 @@ def get_feeds() -> list[dict]:
|
||||
feeds = data.get("feeds", []) if isinstance(data, dict) else data
|
||||
if isinstance(feeds, list) and len(feeds) > 0:
|
||||
return feeds
|
||||
except Exception as e:
|
||||
except (IOError, OSError, json.JSONDecodeError, ValueError) as e:
|
||||
logger.warning(f"Failed to read news feed config: {e}")
|
||||
return list(DEFAULT_FEEDS)
|
||||
|
||||
@@ -64,7 +64,7 @@ def save_feeds(feeds: list[dict]) -> bool:
|
||||
encoding="utf-8",
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
except (IOError, OSError) as e:
|
||||
logger.error(f"Failed to write news feed config: {e}")
|
||||
return False
|
||||
|
||||
|
||||
@@ -72,7 +72,7 @@ def get_top_broadcastify_feeds():
|
||||
logger.info(f"Successfully scraped {len(feeds)} top feeds from Broadcastify.")
|
||||
return feeds
|
||||
|
||||
except Exception as e:
|
||||
except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e:
|
||||
logger.error(f"Broadcastify Scrape Exception: {e}")
|
||||
return []
|
||||
|
||||
@@ -92,7 +92,7 @@ def get_openmhz_systems():
|
||||
# Return list of systems
|
||||
return data.get('systems', []) if isinstance(data, dict) else []
|
||||
return []
|
||||
except Exception as e:
|
||||
except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e:
|
||||
logger.error(f"OpenMHZ Systems Scrape Exception: {e}")
|
||||
return []
|
||||
|
||||
@@ -112,7 +112,7 @@ def get_recent_openmhz_calls(sys_name: str):
|
||||
data = res.json()
|
||||
return data.get('calls', []) if isinstance(data, dict) else []
|
||||
return []
|
||||
except Exception as e:
|
||||
except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e:
|
||||
logger.error(f"OpenMHZ Calls Scrape Exception ({sys_name}): {e}")
|
||||
return []
|
||||
|
||||
|
||||
@@ -50,7 +50,7 @@ def _reverse_geocode(lat: float, lng: float) -> dict:
|
||||
continue
|
||||
else:
|
||||
logger.warning(f"Nominatim returned {res.status_code}")
|
||||
except Exception as e:
|
||||
except (_requests.RequestException, ConnectionError, TimeoutError, OSError) as e:
|
||||
logger.warning(f"Reverse geocode failed: {e}")
|
||||
return {}
|
||||
|
||||
@@ -66,7 +66,7 @@ def _fetch_country_data(country_code: str) -> dict:
|
||||
res = fetch_with_curl(url, timeout=10)
|
||||
if res.status_code == 200:
|
||||
return res.json()
|
||||
except Exception as e:
|
||||
except (ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e:
|
||||
logger.warning(f"RestCountries failed for {country_code}: {e}")
|
||||
return {}
|
||||
|
||||
@@ -96,7 +96,7 @@ def _fetch_wikidata_leader(country_name: str) -> dict:
|
||||
"leader": r.get("leaderLabel", {}).get("value", "Unknown"),
|
||||
"government_type": r.get("govTypeLabel", {}).get("value", "Unknown"),
|
||||
}
|
||||
except Exception as e:
|
||||
except (ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e:
|
||||
logger.warning(f"Wikidata SPARQL failed for {country_name}: {e}")
|
||||
return {"leader": "Unknown", "government_type": "Unknown"}
|
||||
|
||||
@@ -122,7 +122,7 @@ def _fetch_local_wiki_summary(place_name: str, country_name: str = "") -> dict:
|
||||
"extract": data.get("extract", ""),
|
||||
"thumbnail": data.get("thumbnail", {}).get("source", ""),
|
||||
}
|
||||
except Exception:
|
||||
except (ConnectionError, TimeoutError, ValueError, KeyError, OSError): # Intentional: optional enrichment
|
||||
continue
|
||||
return {}
|
||||
|
||||
@@ -158,22 +158,22 @@ def get_region_dossier(lat: float, lng: float) -> dict:
|
||||
|
||||
try:
|
||||
country_data = country_fut.result(timeout=12)
|
||||
except Exception:
|
||||
except Exception: # Intentional: optional enrichment
|
||||
logger.warning("Country data fetch timed out or failed")
|
||||
country_data = {}
|
||||
try:
|
||||
leader_data = leader_fut.result(timeout=12)
|
||||
except Exception:
|
||||
except Exception: # Intentional: optional enrichment
|
||||
logger.warning("Leader data fetch timed out or failed")
|
||||
leader_data = {"leader": "Unknown", "government_type": "Unknown"}
|
||||
try:
|
||||
local_data = local_fut.result(timeout=12)
|
||||
except Exception:
|
||||
except Exception: # Intentional: optional enrichment
|
||||
logger.warning("Local wiki fetch timed out or failed")
|
||||
local_data = {}
|
||||
try:
|
||||
country_wiki_data = country_wiki_fut.result(timeout=12)
|
||||
except Exception:
|
||||
except Exception: # Intentional: optional enrichment
|
||||
country_wiki_data = {}
|
||||
|
||||
# If no local data but we have country wiki summary, use that
|
||||
|
||||
@@ -4,6 +4,7 @@ Free, keyless search for metadata + thumbnails. Used in the right-click dossier.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import requests
|
||||
from datetime import datetime, timedelta
|
||||
from cachetools import TTLCache
|
||||
|
||||
@@ -48,7 +49,7 @@ def search_sentinel2_scene(lat: float, lng: float) -> dict:
|
||||
item = planetary_computer.sign_item(item)
|
||||
except ImportError:
|
||||
pass # planetary_computer not installed, try unsigned URLs
|
||||
except Exception as e:
|
||||
except (ConnectionError, TimeoutError, ValueError) as e:
|
||||
logger.warning(f"Sentinel-2 signing failed: {e}")
|
||||
|
||||
# Get the rendered_preview (full-res PNG) and thumbnail separately
|
||||
@@ -76,6 +77,6 @@ def search_sentinel2_scene(lat: float, lng: float) -> dict:
|
||||
except ImportError:
|
||||
logger.warning("pystac-client not installed — Sentinel-2 search unavailable")
|
||||
return {"found": False, "error": "pystac-client not installed"}
|
||||
except Exception as e:
|
||||
except (requests.RequestException, ConnectionError, TimeoutError, ValueError) as e:
|
||||
logger.error(f"Sentinel-2 search failed for ({lat}, {lng}): {e}")
|
||||
return {"found": False, "error": str(e)}
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
"""Self-update module — downloads latest GitHub release, backs up current files,
|
||||
extracts the update over the project, and restarts the app.
|
||||
|
||||
Public API:
|
||||
perform_update(project_root) -> dict (download + backup + extract)
|
||||
schedule_restart(project_root) (spawn detached start script, then exit)
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
GITHUB_RELEASES_URL = "https://api.github.com/repos/BigBodyCobain/Shadowbroker/releases/latest"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Protected patterns — files/dirs that must NEVER be overwritten during update
|
||||
# ---------------------------------------------------------------------------
|
||||
_PROTECTED_DIRS = {"venv", "node_modules", ".next", "__pycache__", ".git"}
|
||||
_PROTECTED_EXTENSIONS = {".db", ".sqlite"}
|
||||
_PROTECTED_NAMES = {
|
||||
".env",
|
||||
"ais_cache.json",
|
||||
"carrier_cache.json",
|
||||
"geocode_cache.json",
|
||||
}
|
||||
|
||||
|
||||
def _is_protected(rel_path: str) -> bool:
|
||||
"""Return True if *rel_path* (forward-slash separated) should be skipped."""
|
||||
parts = rel_path.replace("\\", "/").split("/")
|
||||
name = parts[-1]
|
||||
|
||||
# Check directory components
|
||||
for part in parts[:-1]:
|
||||
if part in _PROTECTED_DIRS:
|
||||
return True
|
||||
|
||||
# Check filename
|
||||
if name in _PROTECTED_NAMES:
|
||||
return True
|
||||
_, ext = os.path.splitext(name)
|
||||
if ext.lower() in _PROTECTED_EXTENSIONS:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Download
|
||||
# ---------------------------------------------------------------------------
|
||||
def _download_release(temp_dir: str) -> tuple:
|
||||
"""Fetch latest release info and download the zip asset.
|
||||
Returns (zip_path, version_tag, download_url).
|
||||
"""
|
||||
logger.info("Fetching latest release info from GitHub...")
|
||||
resp = requests.get(GITHUB_RELEASES_URL, timeout=15)
|
||||
resp.raise_for_status()
|
||||
release = resp.json()
|
||||
|
||||
tag = release.get("tag_name", "unknown")
|
||||
assets = release.get("assets", [])
|
||||
|
||||
# Find the .zip asset
|
||||
zip_url = None
|
||||
for asset in assets:
|
||||
url = asset.get("browser_download_url", "")
|
||||
if url.endswith(".zip"):
|
||||
zip_url = url
|
||||
break
|
||||
|
||||
if not zip_url:
|
||||
raise RuntimeError("No .zip asset found in the latest release")
|
||||
|
||||
logger.info(f"Downloading {zip_url} ...")
|
||||
zip_path = os.path.join(temp_dir, "update.zip")
|
||||
with requests.get(zip_url, stream=True, timeout=120) as dl:
|
||||
dl.raise_for_status()
|
||||
with open(zip_path, "wb") as f:
|
||||
for chunk in dl.iter_content(chunk_size=1024 * 64):
|
||||
f.write(chunk)
|
||||
|
||||
if not zipfile.is_zipfile(zip_path):
|
||||
raise RuntimeError("Downloaded file is not a valid ZIP archive")
|
||||
|
||||
size_mb = os.path.getsize(zip_path) / (1024 * 1024)
|
||||
logger.info(f"Downloaded {size_mb:.1f} MB — ZIP validated OK")
|
||||
return zip_path, tag, zip_url
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Backup
|
||||
# ---------------------------------------------------------------------------
|
||||
def _backup_current(project_root: str, temp_dir: str) -> str:
|
||||
"""Create a backup zip of backend/ and frontend/ in temp_dir."""
|
||||
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
backup_path = os.path.join(temp_dir, f"backup_{stamp}.zip")
|
||||
logger.info(f"Backing up current files to {backup_path} ...")
|
||||
|
||||
dirs_to_backup = ["backend", "frontend"]
|
||||
count = 0
|
||||
|
||||
with zipfile.ZipFile(backup_path, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||
for dir_name in dirs_to_backup:
|
||||
dir_path = os.path.join(project_root, dir_name)
|
||||
if not os.path.isdir(dir_path):
|
||||
continue
|
||||
for root, dirs, files in os.walk(dir_path):
|
||||
# Prune protected directories from walk
|
||||
dirs[:] = [d for d in dirs if d not in _PROTECTED_DIRS]
|
||||
for fname in files:
|
||||
full = os.path.join(root, fname)
|
||||
rel = os.path.relpath(full, project_root)
|
||||
if _is_protected(rel):
|
||||
continue
|
||||
try:
|
||||
zf.write(full, rel)
|
||||
count += 1
|
||||
except (PermissionError, OSError) as e:
|
||||
logger.warning(f"Backup skip (locked): {rel} — {e}")
|
||||
|
||||
logger.info(f"Backup complete: {count} files archived")
|
||||
return backup_path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extract & Copy
|
||||
# ---------------------------------------------------------------------------
|
||||
def _extract_and_copy(zip_path: str, project_root: str, temp_dir: str) -> int:
|
||||
"""Extract the update zip and copy files over the project, skipping protected files.
|
||||
Returns count of files copied.
|
||||
"""
|
||||
extract_dir = os.path.join(temp_dir, "extracted")
|
||||
logger.info("Extracting update zip...")
|
||||
with zipfile.ZipFile(zip_path, "r") as zf:
|
||||
zf.extractall(extract_dir)
|
||||
|
||||
# Detect wrapper folder: if extracted root has a single directory that
|
||||
# itself contains frontend/ or backend/, use it as the real base.
|
||||
base = extract_dir
|
||||
entries = [e for e in os.listdir(base) if not e.startswith(".")]
|
||||
if len(entries) == 1:
|
||||
candidate = os.path.join(base, entries[0])
|
||||
if os.path.isdir(candidate):
|
||||
sub = os.listdir(candidate)
|
||||
if "frontend" in sub or "backend" in sub:
|
||||
base = candidate
|
||||
logger.info(f"Detected wrapper folder: {entries[0]}")
|
||||
|
||||
copied = 0
|
||||
skipped = 0
|
||||
|
||||
for root, _dirs, files in os.walk(base):
|
||||
for fname in files:
|
||||
src = os.path.join(root, fname)
|
||||
rel = os.path.relpath(src, base).replace("\\", "/")
|
||||
|
||||
if _is_protected(rel):
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
dst = os.path.join(project_root, rel)
|
||||
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
||||
try:
|
||||
shutil.copy2(src, dst)
|
||||
copied += 1
|
||||
except (PermissionError, OSError) as e:
|
||||
logger.warning(f"Copy failed (skipping): {rel} — {e}")
|
||||
skipped += 1
|
||||
|
||||
logger.info(f"Update applied: {copied} files copied, {skipped} skipped/protected")
|
||||
return copied
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Restart
|
||||
# ---------------------------------------------------------------------------
|
||||
def schedule_restart(project_root: str):
|
||||
"""Spawn a detached process that re-runs start.bat / start.sh after a short
|
||||
delay, then forcefully exit the current Python process."""
|
||||
tmp = tempfile.mkdtemp(prefix="sb_restart_")
|
||||
|
||||
if sys.platform == "win32":
|
||||
script = os.path.join(tmp, "restart.bat")
|
||||
with open(script, "w") as f:
|
||||
f.write("@echo off\n")
|
||||
f.write("timeout /t 3 /nobreak >nul\n")
|
||||
f.write(f'cd /d "{project_root}"\n')
|
||||
f.write("call start.bat\n")
|
||||
|
||||
CREATE_NEW_PROCESS_GROUP = 0x00000200
|
||||
DETACHED_PROCESS = 0x00000008
|
||||
subprocess.Popen(
|
||||
["cmd", "/c", script],
|
||||
creationflags=DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP,
|
||||
close_fds=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
else:
|
||||
script = os.path.join(tmp, "restart.sh")
|
||||
with open(script, "w") as f:
|
||||
f.write("#!/bin/bash\n")
|
||||
f.write("sleep 3\n")
|
||||
f.write(f'cd "{project_root}"\n')
|
||||
f.write("bash start.sh\n")
|
||||
os.chmod(script, 0o755)
|
||||
subprocess.Popen(
|
||||
["bash", script],
|
||||
start_new_session=True,
|
||||
close_fds=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
logger.info("Restart script spawned — exiting current process")
|
||||
os._exit(0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
def perform_update(project_root: str) -> dict:
|
||||
"""Download the latest release, back up current files, and extract the update.
|
||||
|
||||
Returns a dict with status info on success, or {"status": "error", "message": ...}
|
||||
on failure. Does NOT trigger restart — caller should call schedule_restart()
|
||||
separately after the HTTP response has been sent.
|
||||
"""
|
||||
temp_dir = tempfile.mkdtemp(prefix="sb_update_")
|
||||
try:
|
||||
zip_path, version, url = _download_release(temp_dir)
|
||||
backup_path = _backup_current(project_root, temp_dir)
|
||||
copied = _extract_and_copy(zip_path, project_root, temp_dir)
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"version": version,
|
||||
"files_updated": copied,
|
||||
"backup_path": backup_path,
|
||||
"message": f"Updated to {version} — {copied} files replaced. Restarting...",
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Update failed: {e}", exc_info=True)
|
||||
return {
|
||||
"status": "error",
|
||||
"message": str(e),
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _suppress_background_services():
|
||||
"""Prevent real scheduler/stream/tracker from starting during tests."""
|
||||
with patch("services.data_fetcher.start_scheduler"), \
|
||||
patch("services.data_fetcher.stop_scheduler"), \
|
||||
patch("services.ais_stream.start_ais_stream"), \
|
||||
patch("services.ais_stream.stop_ais_stream"), \
|
||||
patch("services.carrier_tracker.start_carrier_tracker"), \
|
||||
patch("services.carrier_tracker.stop_carrier_tracker"):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def client(_suppress_background_services):
|
||||
"""HTTPX test client against the FastAPI app (no real network)."""
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from main import app
|
||||
import asyncio
|
||||
|
||||
transport = ASGITransport(app=app)
|
||||
|
||||
async def _make_client():
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as ac:
|
||||
return ac
|
||||
|
||||
# Return a sync-usable wrapper
|
||||
class SyncClient:
|
||||
def __init__(self):
|
||||
self._loop = asyncio.new_event_loop()
|
||||
self._transport = ASGITransport(app=app)
|
||||
|
||||
def get(self, url, **kw):
|
||||
return self._loop.run_until_complete(self._get(url, **kw))
|
||||
|
||||
async def _get(self, url, **kw):
|
||||
async with AsyncClient(transport=self._transport, base_url="http://test") as ac:
|
||||
return await ac.get(url, **kw)
|
||||
|
||||
def put(self, url, **kw):
|
||||
return self._loop.run_until_complete(self._put(url, **kw))
|
||||
|
||||
async def _put(self, url, **kw):
|
||||
async with AsyncClient(transport=self._transport, base_url="http://test") as ac:
|
||||
return await ac.put(url, **kw)
|
||||
|
||||
return SyncClient()
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Smoke tests for all API endpoints — verifies routes exist and return valid responses."""
|
||||
import pytest
|
||||
|
||||
|
||||
class TestHealthEndpoint:
|
||||
def test_health_returns_200(self, client):
|
||||
r = client.get("/api/health")
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
assert data["status"] == "ok"
|
||||
assert "sources" in data
|
||||
assert "freshness" in data
|
||||
|
||||
def test_health_has_uptime(self, client):
|
||||
r = client.get("/api/health")
|
||||
data = r.json()
|
||||
assert "uptime_seconds" in data
|
||||
assert isinstance(data["uptime_seconds"], (int, float))
|
||||
|
||||
|
||||
class TestLiveDataEndpoints:
|
||||
def test_live_data_returns_200(self, client):
|
||||
r = client.get("/api/live-data")
|
||||
assert r.status_code == 200
|
||||
|
||||
def test_live_data_fast_returns_200_or_304(self, client):
|
||||
r = client.get("/api/live-data/fast")
|
||||
assert r.status_code in (200, 304)
|
||||
if r.status_code == 200:
|
||||
data = r.json()
|
||||
assert "freshness" in data
|
||||
|
||||
def test_live_data_slow_returns_200_or_304(self, client):
|
||||
r = client.get("/api/live-data/slow")
|
||||
assert r.status_code in (200, 304)
|
||||
if r.status_code == 200:
|
||||
data = r.json()
|
||||
assert "freshness" in data
|
||||
|
||||
def test_fast_has_expected_keys(self, client):
|
||||
r = client.get("/api/live-data/fast")
|
||||
if r.status_code == 200:
|
||||
data = r.json()
|
||||
for key in ("commercial_flights", "military_flights", "ships", "satellites"):
|
||||
assert key in data, f"Missing key: {key}"
|
||||
|
||||
def test_slow_has_expected_keys(self, client):
|
||||
r = client.get("/api/live-data/slow")
|
||||
if r.status_code == 200:
|
||||
data = r.json()
|
||||
for key in ("news", "stocks", "weather", "earthquakes"):
|
||||
assert key in data, f"Missing key: {key}"
|
||||
|
||||
|
||||
class TestDebugEndpoint:
|
||||
def test_debug_latest_returns_list(self, client):
|
||||
r = client.get("/api/debug-latest")
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
assert isinstance(data, list)
|
||||
|
||||
|
||||
class TestSettingsEndpoints:
|
||||
def test_get_api_keys(self, client):
|
||||
r = client.get("/api/settings/api-keys")
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
assert isinstance(data, list)
|
||||
|
||||
def test_get_news_feeds(self, client):
|
||||
r = client.get("/api/settings/news-feeds")
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
assert isinstance(data, list)
|
||||
|
||||
|
||||
class TestRadioEndpoints:
|
||||
def test_radio_top_returns_200(self, client):
|
||||
r = client.get("/api/radio/top")
|
||||
assert r.status_code == 200
|
||||
|
||||
def test_radio_openmhz_systems(self, client):
|
||||
r = client.get("/api/radio/openmhz/systems")
|
||||
assert r.status_code == 200
|
||||
|
||||
|
||||
class TestQueryValidation:
|
||||
def test_region_dossier_rejects_invalid_lat(self, client):
|
||||
r = client.get("/api/region-dossier?lat=999&lng=0")
|
||||
assert r.status_code == 422
|
||||
|
||||
def test_region_dossier_rejects_invalid_lng(self, client):
|
||||
r = client.get("/api/region-dossier?lat=0&lng=999")
|
||||
assert r.status_code == 422
|
||||
|
||||
def test_sentinel_rejects_invalid_coords(self, client):
|
||||
r = client.get("/api/sentinel2/search?lat=-100&lng=0")
|
||||
assert r.status_code == 422
|
||||
|
||||
def test_radio_nearest_rejects_invalid_lat(self, client):
|
||||
r = client.get("/api/radio/nearest?lat=91&lng=0")
|
||||
assert r.status_code == 422
|
||||
|
||||
|
||||
class TestETagBehavior:
|
||||
def test_fast_returns_etag_header(self, client):
|
||||
r = client.get("/api/live-data/fast")
|
||||
if r.status_code == 200:
|
||||
assert "etag" in r.headers
|
||||
|
||||
def test_slow_returns_etag_header(self, client):
|
||||
r = client.get("/api/live-data/slow")
|
||||
if r.status_code == 200:
|
||||
assert "etag" in r.headers
|
||||
Reference in New Issue
Block a user