mirror of
https://github.com/BigBodyCobain/Shadowbroker.git
synced 2026-08-27 21:00:28 +02:00
release: prepare v0.9.7
This commit is contained in:
BigBodyCobain
@@ -4,6 +4,7 @@ Central location for latest_data, source_timestamps, and the data lock.
|
||||
Every fetcher imports from here instead of maintaining its own copy.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import threading
|
||||
import logging
|
||||
from datetime import datetime
|
||||
@@ -42,6 +43,7 @@ class DashboardData(TypedDict, total=False):
|
||||
gps_jamming: List[Dict[str, Any]]
|
||||
satellites: List[Dict[str, Any]]
|
||||
satellite_source: str
|
||||
satellite_analysis: Dict[str, Any]
|
||||
prediction_markets: List[Dict[str, Any]]
|
||||
sigint: List[Dict[str, Any]]
|
||||
sigint_totals: Dict[str, Any]
|
||||
@@ -61,6 +63,12 @@ class DashboardData(TypedDict, total=False):
|
||||
fimi: Dict[str, Any]
|
||||
psk_reporter: List[Dict[str, Any]]
|
||||
correlations: List[Dict[str, Any]]
|
||||
uap_sightings: List[Dict[str, Any]]
|
||||
wastewater: List[Dict[str, Any]]
|
||||
crowdthreat: List[Dict[str, Any]]
|
||||
sar_scenes: List[Dict[str, Any]]
|
||||
sar_anomalies: List[Dict[str, Any]]
|
||||
sar_aoi_coverage: List[Dict[str, Any]]
|
||||
|
||||
|
||||
# In-memory store
|
||||
@@ -105,6 +113,12 @@ latest_data: DashboardData = {
|
||||
"fimi": {},
|
||||
"psk_reporter": [],
|
||||
"correlations": [],
|
||||
"uap_sightings": [],
|
||||
"wastewater": [],
|
||||
"crowdthreat": [],
|
||||
"sar_scenes": [],
|
||||
"sar_anomalies": [],
|
||||
"sar_aoi_coverage": [],
|
||||
}
|
||||
|
||||
# Per-source freshness timestamps
|
||||
@@ -117,9 +131,21 @@ source_freshness: dict[str, dict] = {}
|
||||
def _mark_fresh(*keys):
|
||||
"""Record the current UTC time for one or more data source keys."""
|
||||
now = datetime.utcnow().isoformat()
|
||||
global _data_version
|
||||
changed: list[tuple[str, int, int]] = [] # (layer, version, count)
|
||||
with _data_lock:
|
||||
for k in keys:
|
||||
source_timestamps[k] = now
|
||||
_layer_versions[k] = _layer_versions.get(k, 0) + 1
|
||||
# Grab entity count while we hold the lock (cheap len())
|
||||
val = latest_data.get(k)
|
||||
count = len(val) if isinstance(val, list) else (1 if val is not None else 0)
|
||||
changed.append((k, _layer_versions[k], count))
|
||||
# Publish partial fetch progress immediately so the frontend can
|
||||
# observe newly available data without waiting for the entire tier.
|
||||
_data_version += 1
|
||||
# Notify SSE listeners outside the lock to avoid deadlocks
|
||||
_notify_layer_change(changed)
|
||||
|
||||
|
||||
# Thread lock for safe reads/writes to latest_data
|
||||
@@ -129,16 +155,73 @@ _data_lock = threading.Lock()
|
||||
# Used for cheap ETag generation instead of MD5-hashing the full response.
|
||||
_data_version: int = 0
|
||||
|
||||
# Per-layer version counters — incremented only when that specific layer
|
||||
# refreshes. Used by get_layer_slice for per-layer incremental updates
|
||||
# and by the SSE stream to push targeted layer_changed notifications.
|
||||
_layer_versions: dict[str, int] = {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Layer-change notification callbacks (thread → async SSE bridge)
|
||||
# ---------------------------------------------------------------------------
|
||||
_layer_change_callbacks: list = []
|
||||
_layer_change_callbacks_lock = threading.Lock()
|
||||
|
||||
|
||||
def register_layer_change_callback(callback) -> None:
|
||||
"""Register a callback invoked on every _mark_fresh().
|
||||
|
||||
Signature: callback(layer: str, version: int, count: int)
|
||||
Called from fetcher threads — must be thread-safe.
|
||||
"""
|
||||
with _layer_change_callbacks_lock:
|
||||
_layer_change_callbacks.append(callback)
|
||||
|
||||
|
||||
def unregister_layer_change_callback(callback) -> None:
|
||||
"""Remove a previously registered callback."""
|
||||
with _layer_change_callbacks_lock:
|
||||
try:
|
||||
_layer_change_callbacks.remove(callback)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
|
||||
def _notify_layer_change(changed: list[tuple[str, int, int]]) -> None:
|
||||
"""Fire all registered callbacks for each changed layer."""
|
||||
with _layer_change_callbacks_lock:
|
||||
cbs = list(_layer_change_callbacks)
|
||||
for cb in cbs:
|
||||
for layer, version, count in changed:
|
||||
try:
|
||||
cb(layer, version, count)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def get_layer_versions() -> dict[str, int]:
|
||||
"""Return a snapshot of all per-layer version counters."""
|
||||
with _data_lock:
|
||||
return dict(_layer_versions)
|
||||
|
||||
|
||||
def get_layer_version(layer: str) -> int:
|
||||
"""Return the version counter for a single layer (0 if never refreshed)."""
|
||||
with _data_lock:
|
||||
return _layer_versions.get(layer, 0)
|
||||
|
||||
|
||||
def bump_data_version() -> None:
|
||||
"""Increment the data version counter after a fetch cycle completes."""
|
||||
global _data_version
|
||||
_data_version += 1
|
||||
with _data_lock:
|
||||
_data_version += 1
|
||||
|
||||
|
||||
def get_data_version() -> int:
|
||||
"""Return the current data version (for ETag generation)."""
|
||||
return _data_version
|
||||
with _data_lock:
|
||||
return _data_version
|
||||
|
||||
|
||||
_active_layers_version: int = 0
|
||||
@@ -156,21 +239,17 @@ def get_active_layers_version() -> int:
|
||||
|
||||
|
||||
def get_latest_data_subset(*keys: str) -> DashboardData:
|
||||
"""Return a shallow snapshot of only the requested top-level keys.
|
||||
"""Return a deep snapshot of only the requested top-level keys.
|
||||
|
||||
This avoids cloning the entire dashboard store for endpoints that only need
|
||||
a small tier-specific subset.
|
||||
a small tier-specific subset. Deep copy ensures callers cannot mutate
|
||||
nested structures (e.g. individual flight dicts) and affect the live store.
|
||||
"""
|
||||
with _data_lock:
|
||||
snap: DashboardData = {}
|
||||
for key in keys:
|
||||
value = latest_data.get(key)
|
||||
if isinstance(value, list):
|
||||
snap[key] = list(value)
|
||||
elif isinstance(value, dict):
|
||||
snap[key] = dict(value)
|
||||
else:
|
||||
snap[key] = value
|
||||
snap[key] = copy.deepcopy(value)
|
||||
return snap
|
||||
|
||||
|
||||
@@ -231,10 +310,16 @@ active_layers: dict[str, bool] = {
|
||||
"satnogs": True,
|
||||
"tinygs": True,
|
||||
"ukraine_alerts": True,
|
||||
"power_plants": False,
|
||||
"power_plants": True,
|
||||
"viirs_nightlights": False,
|
||||
"psk_reporter": True,
|
||||
"correlations": True,
|
||||
"contradictions": True,
|
||||
"uap_sightings": True,
|
||||
"wastewater": True,
|
||||
"ai_intel": True,
|
||||
"crowdthreat": True,
|
||||
"sar": True,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
"""OpenSky aircraft metadata: ICAO24 hex -> ICAO type code + friendly model.
|
||||
|
||||
OpenSky's /states/all does not include aircraft type, so OpenSky-sourced
|
||||
flights arrive with ``t`` field empty. This module bulk-loads the public
|
||||
OpenSky aircraft database (one snapshot CSV per month, ~108 MB uncompressed,
|
||||
~600k aircraft) once every 5 days and exposes a fast in-memory hex lookup.
|
||||
|
||||
The data is also useful when adsb.lol's live API is degraded: even the
|
||||
adsb.lol /v2 feed sometimes returns aircraft with empty ``t`` for newly seen
|
||||
transponders, and the lookup gracefully fills those in too.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_BUCKET_LIST_URL = (
|
||||
"https://s3.opensky-network.org/data-samples?prefix=metadata/&list-type=2"
|
||||
)
|
||||
_BUCKET_BASE = "https://s3.opensky-network.org/data-samples/"
|
||||
_S3_NS = "{http://s3.amazonaws.com/doc/2006-03-01/}"
|
||||
_REFRESH_INTERVAL_S = 5 * 24 * 3600
|
||||
_LIST_TIMEOUT_S = 30
|
||||
_DOWNLOAD_TIMEOUT_S = 600
|
||||
_USER_AGENT = (
|
||||
"ShadowBroker-OSINT/0.9.7 "
|
||||
"(+https://github.com/BigBodyCobain/Shadowbroker; "
|
||||
"contact: bigbodycobain@gmail.com)"
|
||||
)
|
||||
|
||||
_lock = threading.RLock()
|
||||
_aircraft_by_hex: dict[str, dict[str, str]] = {}
|
||||
_last_refresh = 0.0
|
||||
_in_progress = False
|
||||
|
||||
|
||||
def _latest_snapshot_key() -> str:
|
||||
"""Discover the most recent aircraft-database-complete snapshot key."""
|
||||
response = requests.get(
|
||||
_BUCKET_LIST_URL,
|
||||
timeout=_LIST_TIMEOUT_S,
|
||||
headers={"User-Agent": _USER_AGENT},
|
||||
)
|
||||
response.raise_for_status()
|
||||
root = ET.fromstring(response.text)
|
||||
keys: list[str] = []
|
||||
for content in root.iter(f"{_S3_NS}Contents"):
|
||||
key_el = content.find(f"{_S3_NS}Key")
|
||||
if key_el is None or not key_el.text:
|
||||
continue
|
||||
if "aircraft-database-complete-" in key_el.text and key_el.text.endswith(".csv"):
|
||||
keys.append(key_el.text)
|
||||
if not keys:
|
||||
raise RuntimeError("no aircraft-database-complete snapshot found in bucket listing")
|
||||
return sorted(keys)[-1]
|
||||
|
||||
|
||||
def _stream_csv_index(url: str) -> dict[str, dict[str, str]]:
|
||||
"""Stream-parse the OpenSky aircraft CSV into a hex-keyed index.
|
||||
|
||||
The CSV uses single-quote quoting, so csv.DictReader is configured with
|
||||
``quotechar="'"``. Rows are processed line-by-line via iter_lines() to
|
||||
keep memory bounded even though the file is ~108 MB.
|
||||
"""
|
||||
with requests.get(
|
||||
url,
|
||||
timeout=_DOWNLOAD_TIMEOUT_S,
|
||||
stream=True,
|
||||
headers={"User-Agent": _USER_AGENT},
|
||||
) as response:
|
||||
response.raise_for_status()
|
||||
line_iter = (
|
||||
line.decode("utf-8", errors="replace")
|
||||
for line in response.iter_lines(decode_unicode=False)
|
||||
if line
|
||||
)
|
||||
reader = csv.DictReader(line_iter, quotechar="'")
|
||||
index: dict[str, dict[str, str]] = {}
|
||||
for row in reader:
|
||||
hex_code = (row.get("icao24") or "").strip().lower()
|
||||
if not hex_code or hex_code == "000000":
|
||||
continue
|
||||
typecode = (row.get("typecode") or "").strip().upper()
|
||||
model = (row.get("model") or "").strip()
|
||||
mfr = (row.get("manufacturerName") or "").strip()
|
||||
registration = (row.get("registration") or "").strip().upper()
|
||||
operator = (row.get("operator") or "").strip()
|
||||
if not (typecode or model):
|
||||
continue
|
||||
entry: dict[str, str] = {}
|
||||
if typecode:
|
||||
entry["typecode"] = typecode
|
||||
if model:
|
||||
entry["model"] = model
|
||||
if mfr:
|
||||
entry["manufacturer"] = mfr
|
||||
if registration:
|
||||
entry["registration"] = registration
|
||||
if operator:
|
||||
entry["operator"] = operator
|
||||
index[hex_code] = entry
|
||||
return index
|
||||
|
||||
|
||||
def refresh_aircraft_database(force: bool = False) -> bool:
|
||||
"""Download the latest OpenSky aircraft snapshot and rebuild the index.
|
||||
|
||||
Returns True if a refresh was performed (success or attempted), False if
|
||||
skipped because the cache is still fresh or another refresh is in flight.
|
||||
"""
|
||||
global _last_refresh, _in_progress
|
||||
|
||||
now = time.time()
|
||||
with _lock:
|
||||
if _in_progress:
|
||||
return False
|
||||
if not force and (now - _last_refresh) < _REFRESH_INTERVAL_S and _aircraft_by_hex:
|
||||
return False
|
||||
_in_progress = True
|
||||
|
||||
try:
|
||||
started = time.time()
|
||||
key = _latest_snapshot_key()
|
||||
index = _stream_csv_index(_BUCKET_BASE + key)
|
||||
with _lock:
|
||||
_aircraft_by_hex.clear()
|
||||
_aircraft_by_hex.update(index)
|
||||
_last_refresh = time.time()
|
||||
logger.info(
|
||||
"aircraft database refreshed in %.1fs from %s: %d aircraft",
|
||||
time.time() - started,
|
||||
key,
|
||||
len(index),
|
||||
)
|
||||
return True
|
||||
except (requests.RequestException, OSError, ValueError, ET.ParseError) as exc:
|
||||
logger.warning("aircraft database refresh failed: %s", exc)
|
||||
return True
|
||||
finally:
|
||||
with _lock:
|
||||
_in_progress = False
|
||||
|
||||
|
||||
def lookup_aircraft(icao24: str) -> dict[str, str] | None:
|
||||
"""Return the metadata record for an ICAO24 hex code, or None."""
|
||||
key = (icao24 or "").strip().lower()
|
||||
if not key:
|
||||
return None
|
||||
with _lock:
|
||||
entry = _aircraft_by_hex.get(key)
|
||||
return dict(entry) if entry else None
|
||||
|
||||
|
||||
def lookup_aircraft_type(icao24: str) -> str:
|
||||
"""Return the ICAO type code (e.g. 'B738', 'GLF4') or '' if unknown."""
|
||||
entry = lookup_aircraft(icao24)
|
||||
if not entry:
|
||||
return ""
|
||||
return entry.get("typecode", "")
|
||||
|
||||
|
||||
def aircraft_database_status() -> dict[str, Any]:
|
||||
with _lock:
|
||||
return {
|
||||
"last_refresh": _last_refresh,
|
||||
"aircraft": len(_aircraft_by_hex),
|
||||
"in_progress": _in_progress,
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
"""CrowdThreat fetcher — crowdsourced global threat intelligence.
|
||||
|
||||
Polls verified threat reports from CrowdThreat's public API and normalises
|
||||
them into map-ready records with category-based icon IDs.
|
||||
|
||||
No API key required — the /threats endpoint is unauthenticated.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from services.network_utils import fetch_with_curl
|
||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh, is_any_active
|
||||
from services.fetchers.retry import with_retry
|
||||
|
||||
logger = logging.getLogger("services.data_fetcher")
|
||||
|
||||
_CT_BASE = "https://backend.crowdthreat.world"
|
||||
|
||||
# CrowdThreat category_id → icon ID used on the MapLibre layer
|
||||
_CATEGORY_ICON = {
|
||||
1: "ct-security", # Security & Conflict (red)
|
||||
2: "ct-crime", # Crime & Safety (blue)
|
||||
3: "ct-aviation", # Aviation (green)
|
||||
4: "ct-maritime", # Maritime (teal)
|
||||
5: "ct-infrastructure", # Industrial & Infra (orange)
|
||||
6: "ct-special", # Special Threats (purple)
|
||||
7: "ct-social", # Social & Political (pink)
|
||||
8: "ct-other", # Other (gray)
|
||||
}
|
||||
|
||||
_CATEGORY_COLOUR = {
|
||||
1: "#ef4444", # red
|
||||
2: "#3b82f6", # blue
|
||||
3: "#22c55e", # green
|
||||
4: "#14b8a6", # teal
|
||||
5: "#f97316", # orange
|
||||
6: "#a855f7", # purple
|
||||
7: "#ec4899", # pink
|
||||
8: "#6b7280", # gray
|
||||
}
|
||||
|
||||
|
||||
@with_retry(max_retries=2, base_delay=5)
|
||||
def fetch_crowdthreat():
|
||||
"""Fetch verified threat reports from CrowdThreat public API."""
|
||||
if not is_any_active("crowdthreat"):
|
||||
return
|
||||
|
||||
try:
|
||||
resp = fetch_with_curl(f"{_CT_BASE}/threats", timeout=20)
|
||||
if not resp or resp.status_code != 200:
|
||||
logger.warning("CrowdThreat API returned %s", getattr(resp, "status_code", "None"))
|
||||
return
|
||||
|
||||
payload = resp.json()
|
||||
raw_threats = payload.get("data", {}).get("threats", [])
|
||||
if not raw_threats:
|
||||
logger.debug("CrowdThreat returned 0 threats")
|
||||
return
|
||||
|
||||
except Exception as e:
|
||||
logger.error("CrowdThreat fetch error: %s", e)
|
||||
return
|
||||
|
||||
processed = []
|
||||
for t in raw_threats:
|
||||
loc = t.get("location") or {}
|
||||
lng_lat = loc.get("lng_lat")
|
||||
if not lng_lat or len(lng_lat) < 2:
|
||||
continue
|
||||
try:
|
||||
lng = float(lng_lat[0])
|
||||
lat = float(lng_lat[1])
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
|
||||
cat = t.get("category") or {}
|
||||
cat_id = cat.get("id", 8)
|
||||
subcat = t.get("subcategory") or {}
|
||||
threat_type = t.get("type") or {}
|
||||
dates = t.get("dates") or {}
|
||||
occurred = dates.get("occurred") or {}
|
||||
reported = dates.get("reported") or {}
|
||||
|
||||
# Extract all available detail from the API response
|
||||
summary = (t.get("summary") or t.get("description") or "").strip()
|
||||
verification = (t.get("verification_status") or t.get("status") or "").strip()
|
||||
country_obj = loc.get("country") or {}
|
||||
country = country_obj.get("name", "") if isinstance(country_obj, dict) else str(country_obj or "")
|
||||
media = t.get("media") or t.get("images") or t.get("attachments") or []
|
||||
source_url = t.get("source_url") or t.get("url") or t.get("link") or ""
|
||||
severity = t.get("severity") or t.get("severity_level") or t.get("risk_level") or ""
|
||||
votes = t.get("votes") or t.get("upvotes") or 0
|
||||
reporter = t.get("user") or t.get("reporter") or {}
|
||||
reporter_name = reporter.get("name", "") if isinstance(reporter, dict) else ""
|
||||
|
||||
processed.append({
|
||||
"id": t.get("id"),
|
||||
"title": t.get("title", ""),
|
||||
"summary": summary[:500] if summary else "",
|
||||
"lat": lat,
|
||||
"lng": lng,
|
||||
"address": loc.get("name", ""),
|
||||
"city": loc.get("city", ""),
|
||||
"country": country,
|
||||
"category": cat.get("name", "Other"),
|
||||
"category_id": cat_id,
|
||||
"category_colour": _CATEGORY_COLOUR.get(cat_id, "#6b7280"),
|
||||
"subcategory": subcat.get("name", ""),
|
||||
"threat_type": threat_type.get("name", ""),
|
||||
"icon_id": _CATEGORY_ICON.get(cat_id, "ct-other"),
|
||||
"occurred": occurred.get("raw", ""),
|
||||
"occurred_iso": occurred.get("iso", ""),
|
||||
"timeago": occurred.get("timeago", ""),
|
||||
"reported": reported.get("raw", ""),
|
||||
"verification": verification,
|
||||
"severity": str(severity),
|
||||
"source_url": source_url,
|
||||
"media_urls": [m.get("url") or m for m in media[:3]] if isinstance(media, list) else [],
|
||||
"votes": int(votes) if votes else 0,
|
||||
"reporter": reporter_name,
|
||||
"source": "CrowdThreat",
|
||||
})
|
||||
|
||||
logger.info("CrowdThreat: fetched %d verified threats", len(processed))
|
||||
|
||||
with _data_lock:
|
||||
latest_data["crowdthreat"] = processed
|
||||
_mark_fresh("crowdthreat")
|
||||
@@ -1,14 +1,19 @@
|
||||
"""Earth-observation fetchers — earthquakes, FIRMS fires, space weather, weather radar,
|
||||
severe weather alerts, air quality, volcanoes."""
|
||||
|
||||
import concurrent.futures
|
||||
import csv
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
import heapq
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from services.network_utils import fetch_with_curl
|
||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
||||
@@ -596,3 +601,852 @@ def fetch_viirs_change_nodes():
|
||||
if nodes:
|
||||
_mark_fresh("viirs_change_nodes")
|
||||
logger.info(f"VIIRS change nodes: {len(nodes)} nodes from {len(_VIIRS_AOIS)} AOIs")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# UAP Sightings (NUFORC — National UAP Reporting Center)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Shape → canonical category mapping for consistent frontend filtering
|
||||
_UAP_SHAPE_MAP = {
|
||||
"light": "light", "fireball": "fireball", "orb": "orb",
|
||||
"sphere": "orb", "circle": "orb", "oval": "orb", "egg": "orb",
|
||||
"triangle": "triangle", "delta": "triangle", "chevron": "triangle",
|
||||
"boomerang": "triangle",
|
||||
"cigar": "cigar", "cylinder": "cigar", "tube": "cigar",
|
||||
"disk": "disk", "disc": "disk", "saucer": "disk",
|
||||
"diamond": "diamond", "cone": "diamond", "cross": "diamond",
|
||||
"rectangle": "rectangle", "square": "rectangle",
|
||||
"formation": "formation", "cluster": "formation",
|
||||
"changing": "changing", "flash": "flash", "star": "light",
|
||||
"tic-tac": "tic-tac", "tic tac": "tic-tac",
|
||||
}
|
||||
|
||||
# US state → approximate centroid for coarse geocoding when city lookup fails
|
||||
_US_STATE_COORDS: dict[str, tuple[float, float]] = {
|
||||
"AL": (32.8, -86.8), "AK": (64.2, -152.5), "AZ": (34.0, -111.1),
|
||||
"AR": (35.2, -91.8), "CA": (36.8, -119.4), "CO": (39.6, -105.3),
|
||||
"CT": (41.6, -72.7), "DE": (39.3, -75.5), "FL": (27.8, -81.8),
|
||||
"GA": (32.7, -83.5), "HI": (19.9, -155.6), "ID": (44.1, -114.7),
|
||||
"IL": (40.3, -89.0), "IN": (40.3, -86.1), "IA": (42.0, -93.2),
|
||||
"KS": (39.0, -98.5), "KY": (37.8, -84.3), "LA": (31.2, -92.5),
|
||||
"ME": (45.3, -69.4), "MD": (39.0, -76.6), "MA": (42.4, -71.4),
|
||||
"MI": (44.3, -85.6), "MN": (46.7, -94.7), "MS": (32.7, -89.5),
|
||||
"MO": (38.6, -91.8), "MT": (46.8, -110.4), "NE": (41.5, -99.9),
|
||||
"NV": (38.8, -116.4), "NH": (43.2, -71.6), "NJ": (40.1, -74.4),
|
||||
"NM": (34.5, -106.0), "NY": (43.0, -75.0), "NC": (35.6, -79.8),
|
||||
"ND": (47.5, -100.5), "OH": (40.4, -82.9), "OK": (35.0, -97.1),
|
||||
"OR": (43.8, -120.6), "PA": (41.2, -77.2), "RI": (41.6, -71.5),
|
||||
"SC": (33.8, -81.2), "SD": (43.9, -99.4), "TN": (35.5, -86.6),
|
||||
"TX": (31.0, -97.6), "UT": (39.3, -111.1), "VT": (44.6, -72.6),
|
||||
"VA": (37.4, -78.7), "WA": (47.4, -120.7), "WV": (38.6, -80.6),
|
||||
"WI": (43.8, -88.8), "WY": (43.1, -107.6), "DC": (38.9, -77.0),
|
||||
}
|
||||
|
||||
|
||||
def _normalize_uap_shape(raw: str) -> str:
|
||||
"""Normalize a raw NUFORC shape string to a canonical category."""
|
||||
key = raw.strip().lower()
|
||||
return _UAP_SHAPE_MAP.get(key, "unknown")
|
||||
|
||||
|
||||
def _reverse_geocode_state(lat: float, lng: float) -> tuple[str, str]:
|
||||
"""Best-effort reverse-geocode a lat/lng to (state_abbr, country).
|
||||
|
||||
Uses the _US_STATE_COORDS centroid table for fast approximate matching.
|
||||
Returns ('', 'Unknown') if no close match is found.
|
||||
"""
|
||||
best_state = ""
|
||||
best_dist = 999.0
|
||||
for st, (slat, slng) in _US_STATE_COORDS.items():
|
||||
d = ((lat - slat) ** 2 + (lng - slng) ** 2) ** 0.5
|
||||
if d < best_dist:
|
||||
best_dist = d
|
||||
best_state = st
|
||||
if best_dist < 5.0: # ~5 degrees tolerance
|
||||
return best_state, "US"
|
||||
return "", "Unknown"
|
||||
|
||||
|
||||
# ── NUFORC Mapbox Tilequery API ─────────────────────────────────────────
|
||||
# NUFORC's website switched to a JS-rendered Mapbox GL map. The old HTML
|
||||
# table scraper is defunct. We now query the Mapbox Tilequery API against
|
||||
# NUFORC's public tileset to get precise sighting coordinates.
|
||||
#
|
||||
# Tileset: nuforc.cmm18aqea06bu1mmselhpnano-0ce5v
|
||||
# Layer: Sightings Fields: Count, From, To, LinkLat, LinkLon
|
||||
#
|
||||
# We sample a grid of points across the US/world with a 100 km radius and
|
||||
# filter to sightings within the last 60 days.
|
||||
|
||||
_NUFORC_TILESET = "nuforc.cmm18aqea06bu1mmselhpnano-0ce5v"
|
||||
_NUFORC_TOKEN = os.environ.get("NUFORC_MAPBOX_TOKEN", "").strip()
|
||||
_NUFORC_RADIUS_M = 200_000 # 200 km query radius
|
||||
_NUFORC_LIMIT = 50 # max features per tilequery call
|
||||
_NUFORC_RECENT_DAYS = int(os.environ.get("NUFORC_RECENT_DAYS", "60"))
|
||||
_NUFORC_GEOCODE_WORKERS = max(1, int(os.environ.get("NUFORC_GEOCODE_WORKERS", "1")))
|
||||
# Photon (Komoot) is more lenient than Nominatim — ~200ms per query in
|
||||
# practice, so a 0.3s spacing keeps us well under any soft throttle while
|
||||
# still rebuilding a full 12-month window in ~10 minutes.
|
||||
_NUFORC_GEOCODE_SPACING_S = float(os.environ.get("NUFORC_GEOCODE_SPACING_S", "0.3"))
|
||||
_NUFORC_DATA_DIR = Path(__file__).resolve().parent.parent.parent / "data"
|
||||
_NUFORC_SIGHTINGS_CACHE_FILE = _NUFORC_DATA_DIR / "nuforc_recent_sightings.json"
|
||||
_NUFORC_LOCATION_CACHE_FILE = _NUFORC_DATA_DIR / "nuforc_location_cache.json"
|
||||
|
||||
# Live NUFORC databank scraping (wpDataTables server-side AJAX).
|
||||
# The HuggingFace mirror froze at 2023-12-20, so we pull directly from
|
||||
# nuforc.org's monthly sub-index. Each month page embeds a wdtNonce we
|
||||
# must extract, then POST to admin-ajax.php to get the DataTables JSON.
|
||||
_NUFORC_LIVE_INDEX_URL = "https://nuforc.org/subndx/?id=e{yyyymm}"
|
||||
_NUFORC_LIVE_AJAX_URL = (
|
||||
"https://nuforc.org/wp-admin/admin-ajax.php"
|
||||
"?action=get_wdtable&table_id=1&wdt_var1=YearMonth&wdt_var2={yyyymm}"
|
||||
)
|
||||
_NUFORC_LIVE_NONCE_RE = re.compile(
|
||||
r'id=["\']wdtNonceFrontendServerSide_1["\'][^>]*value=["\']([a-f0-9]+)["\']'
|
||||
)
|
||||
_NUFORC_LIVE_SIGHTING_ID_RE = re.compile(r"id=(\d+)")
|
||||
_NUFORC_LIVE_USER_AGENT = "Mozilla/5.0 (ShadowBroker-OSINT NUFORC-fetcher)"
|
||||
_NUFORC_LIVE_SESSION_COOKIES = _NUFORC_DATA_DIR / "nuforc_session.cookies"
|
||||
|
||||
# Sample grid covering continental US, Alaska, Hawaii, Canada, UK, Australia
|
||||
_TILEQUERY_GRID: list[tuple[float, float]] = [
|
||||
# Continental US — ~4° spacing (lon, lat)
|
||||
(-122.4, 37.8), (-118.2, 34.1), (-112.1, 33.4), (-104.9, 39.7),
|
||||
(-95.4, 29.8), (-96.8, 32.8), (-87.6, 41.9), (-84.4, 33.7),
|
||||
(-81.7, 41.5), (-80.2, 25.8), (-77.0, 38.9), (-74.0, 40.7),
|
||||
(-71.1, 42.4), (-90.2, 38.6), (-93.3, 44.9), (-111.9, 40.8),
|
||||
(-122.7, 45.5), (-86.2, 39.8), (-106.6, 35.1), (-73.9, 43.2),
|
||||
(-76.6, 39.3), (-97.5, 35.5), (-83.0, 42.3), (-117.2, 32.7),
|
||||
(-82.5, 28.0), (-78.6, 35.8), (-90.1, 30.0), (-71.4, 41.8),
|
||||
# Alaska, Hawaii
|
||||
(-149.9, 61.2), (-155.5, 19.9),
|
||||
# Canada
|
||||
(-79.4, 43.7), (-123.1, 49.3), (-73.6, 45.5),
|
||||
# UK & Europe
|
||||
(-0.1, 51.5), (-3.2, 55.9),
|
||||
# Australia
|
||||
(151.2, -33.9), (144.9, -37.8),
|
||||
]
|
||||
|
||||
|
||||
def _fetch_nuforc_tilequery(lng: float, lat: float) -> list[dict]:
|
||||
"""Query NUFORC Mapbox tileset around a single point, return raw features."""
|
||||
if not _NUFORC_TOKEN:
|
||||
return []
|
||||
url = (
|
||||
f"https://api.mapbox.com/v4/{_NUFORC_TILESET}/tilequery/"
|
||||
f"{lng},{lat}.json"
|
||||
f"?radius={_NUFORC_RADIUS_M}&limit={_NUFORC_LIMIT}"
|
||||
f"&access_token={_NUFORC_TOKEN}"
|
||||
)
|
||||
try:
|
||||
resp = fetch_with_curl(url, timeout=12)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
return data.get("features", [])
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
|
||||
def _parse_nuforc_tile_date(value: str) -> datetime | None:
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
raw = raw.replace("T", " ")
|
||||
raw = re.sub(r"\s+local$", "", raw, flags=re.IGNORECASE)
|
||||
raw = re.sub(r"\s+utc$", "", raw, flags=re.IGNORECASE)
|
||||
for fmt in (
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
"%Y-%m-%d %H:%M",
|
||||
"%Y-%m-%d",
|
||||
"%m/%d/%Y %H:%M",
|
||||
"%m/%d/%Y",
|
||||
):
|
||||
try:
|
||||
return datetime.strptime(raw, fmt)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
match = re.match(r"^(\d{4}-\d{2}-\d{2})", raw)
|
||||
if match:
|
||||
try:
|
||||
return datetime.strptime(match.group(1), "%Y-%m-%d")
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _load_nuforc_sightings_cache(*, force_refresh: bool = False) -> list[dict] | None:
|
||||
if force_refresh or not _NUFORC_SIGHTINGS_CACHE_FILE.exists():
|
||||
return None
|
||||
try:
|
||||
raw = json.loads(_NUFORC_SIGHTINGS_CACHE_FILE.read_text(encoding="utf-8"))
|
||||
built = raw.get("built", "")
|
||||
built_dt = datetime.fromisoformat(built) if built else None
|
||||
if built_dt is None:
|
||||
return None
|
||||
if (datetime.utcnow() - built_dt).total_seconds() > 86400:
|
||||
return None
|
||||
sightings = raw.get("sightings")
|
||||
if isinstance(sightings, list):
|
||||
if len(sightings) <= 0:
|
||||
logger.info("UAP sightings: cache is fresh but empty; rebuilding")
|
||||
return None
|
||||
logger.info(
|
||||
"UAP sightings: loaded %d cached reports from %s",
|
||||
len(sightings),
|
||||
built,
|
||||
)
|
||||
return sightings
|
||||
except Exception as e:
|
||||
logger.warning("UAP sightings: cache load error: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
def _save_nuforc_sightings_cache(sightings: list[dict]) -> None:
|
||||
if not sightings:
|
||||
logger.warning("UAP sightings: refusing to save empty daily cache")
|
||||
return
|
||||
try:
|
||||
_NUFORC_DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
payload = {
|
||||
"built": datetime.utcnow().isoformat(),
|
||||
"count": len(sightings),
|
||||
"sightings": sightings,
|
||||
}
|
||||
_NUFORC_SIGHTINGS_CACHE_FILE.write_text(
|
||||
json.dumps(payload, separators=(",", ":")),
|
||||
encoding="utf-8",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("UAP sightings: cache save error: %s", e)
|
||||
|
||||
|
||||
def _load_nuforc_location_cache() -> dict[str, list[float] | None]:
|
||||
if not _NUFORC_LOCATION_CACHE_FILE.exists():
|
||||
return {}
|
||||
try:
|
||||
raw = json.loads(_NUFORC_LOCATION_CACHE_FILE.read_text(encoding="utf-8"))
|
||||
if not isinstance(raw, dict):
|
||||
return {}
|
||||
cache: dict[str, list[float] | None] = {}
|
||||
for key, value in raw.items():
|
||||
if not isinstance(key, str):
|
||||
continue
|
||||
if (
|
||||
isinstance(value, list)
|
||||
and len(value) == 2
|
||||
and all(isinstance(v, (int, float)) for v in value)
|
||||
):
|
||||
cache[key] = [float(value[0]), float(value[1])]
|
||||
elif value is None:
|
||||
cache[key] = None
|
||||
return cache
|
||||
except Exception as e:
|
||||
logger.warning("UAP sightings: location cache load error: %s", e)
|
||||
return {}
|
||||
|
||||
|
||||
def _save_nuforc_location_cache(cache: dict[str, list[float] | None]) -> None:
|
||||
try:
|
||||
_NUFORC_DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
_NUFORC_LOCATION_CACHE_FILE.write_text(
|
||||
json.dumps(cache, separators=(",", ":")),
|
||||
encoding="utf-8",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("UAP sightings: location cache save error: %s", e)
|
||||
|
||||
|
||||
def _normalize_uap_location(raw: str) -> str:
|
||||
return re.sub(r"\s+", " ", str(raw or "").strip())
|
||||
|
||||
|
||||
def _uap_country_from_location(location: str, state: str) -> str:
|
||||
if state:
|
||||
return "US"
|
||||
upper = location.upper()
|
||||
if "USA" in upper or "UNITED STATES" in upper:
|
||||
return "US"
|
||||
parts = [part.strip() for part in location.split(",") if part.strip()]
|
||||
if not parts:
|
||||
return "Unknown"
|
||||
country = parts[-1]
|
||||
return country.upper() if len(country) == 2 else country
|
||||
|
||||
|
||||
_US_COUNTRY_ALIASES = {
|
||||
"", "USA", "US", "U.S.", "U.S.A.",
|
||||
"UNITED STATES", "UNITED STATES OF AMERICA",
|
||||
}
|
||||
|
||||
|
||||
def _uap_geocode_candidates(
|
||||
location: str, city: str, state: str, country: str = ""
|
||||
) -> list[str]:
|
||||
"""Build geocode query candidates in priority order.
|
||||
|
||||
NUFORC's live databank is international, so we must query with the
|
||||
actual country first. Only when the country is empty or explicitly US
|
||||
do we fall back to the legacy USA-assumption behavior.
|
||||
"""
|
||||
candidates: list[str] = []
|
||||
c = (country or "").strip()
|
||||
c_upper = c.upper()
|
||||
is_us = c_upper in _US_COUNTRY_ALIASES
|
||||
|
||||
if not is_us:
|
||||
# Non-US: try country-qualified queries first to prevent the
|
||||
# geocoder from fuzzy-matching to a same-named US city.
|
||||
if city and state:
|
||||
candidates.append(f"{city}, {state}, {c}")
|
||||
if city:
|
||||
candidates.append(f"{city}, {c}")
|
||||
if city and state:
|
||||
candidates.append(f"{city}, {state}")
|
||||
if city:
|
||||
candidates.append(city)
|
||||
else:
|
||||
if city and state:
|
||||
candidates.append(f"{city}, {state}, USA")
|
||||
candidates.append(f"{city}, {state}")
|
||||
if city:
|
||||
candidates.append(city)
|
||||
|
||||
normalized = _normalize_uap_location(location)
|
||||
if normalized:
|
||||
candidates.append(normalized)
|
||||
parts = [part.strip() for part in normalized.split(",") if part.strip()]
|
||||
if len(parts) >= 2:
|
||||
candidates.append(", ".join(parts[:2]))
|
||||
if parts:
|
||||
candidates.append(parts[0])
|
||||
|
||||
deduped: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for candidate in candidates:
|
||||
key = candidate.lower()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
deduped.append(candidate)
|
||||
return deduped
|
||||
|
||||
|
||||
def _photon_lookup(query: str) -> list[float] | None:
|
||||
"""Query Komoot's public Photon instance (OSM-based, no API key).
|
||||
|
||||
Returns [lat, lng] on success, None on any failure. We bypass the
|
||||
shared search_geocode() helper on purpose: it falls back to an
|
||||
airport-name token matcher on failure that confidently returns
|
||||
completely wrong coordinates, which poisoned the cache for years.
|
||||
"""
|
||||
from urllib.parse import urlencode
|
||||
|
||||
params = urlencode({"q": query, "limit": 1})
|
||||
url = f"https://photon.komoot.io/api?{params}"
|
||||
try:
|
||||
res = fetch_with_curl(
|
||||
url,
|
||||
headers={
|
||||
"User-Agent": "ShadowBroker-OSINT/1.0 (NUFORC-UAP-layer)",
|
||||
"Accept-Language": "en",
|
||||
},
|
||||
timeout=10,
|
||||
)
|
||||
except Exception:
|
||||
return None
|
||||
if not res or res.status_code != 200:
|
||||
return None
|
||||
try:
|
||||
payload = res.json()
|
||||
except Exception:
|
||||
return None
|
||||
features = (payload or {}).get("features") or []
|
||||
if not features:
|
||||
return None
|
||||
try:
|
||||
# GeoJSON order is [lng, lat] — flip to our [lat, lng] convention.
|
||||
coords = features[0]["geometry"]["coordinates"]
|
||||
return [float(coords[1]), float(coords[0])]
|
||||
except (KeyError, IndexError, TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _geocode_uap_location(
|
||||
location: str, city: str, state: str, country: str = ""
|
||||
) -> list[float] | None:
|
||||
"""Resolve a NUFORC sighting location to [lat, lng] via Photon.
|
||||
|
||||
Returns None on failure. The caller caches None alongside real hits
|
||||
so we don't retry unresolvable queries every run.
|
||||
"""
|
||||
for query in _uap_geocode_candidates(location, city, state, country):
|
||||
coords = _photon_lookup(query)
|
||||
if coords:
|
||||
return coords
|
||||
return None
|
||||
|
||||
|
||||
def _build_uap_sighting_id(row: dict, occurred: str, location: str) -> str:
|
||||
raw_id = str(row.get("Sighting", "") or row.get("sighting", "")).strip()
|
||||
if raw_id:
|
||||
return raw_id
|
||||
digest = hashlib.sha1(
|
||||
f"{occurred}|{location}|{row.get('Summary', '')}|{row.get('Text', '')}".encode("utf-8", "ignore")
|
||||
).hexdigest()[:12]
|
||||
return f"NUFORC-{digest}"
|
||||
|
||||
|
||||
def _nuforc_months_for_window(days: int) -> list[str]:
|
||||
"""Enumerate YYYYMM strings covering the rolling `days`-day window.
|
||||
|
||||
Returned newest first. Always includes the current month even if the
|
||||
window technically starts later, because new reports land there.
|
||||
"""
|
||||
today = datetime.utcnow().date()
|
||||
start = today - timedelta(days=days)
|
||||
months: list[str] = []
|
||||
cur = today.replace(day=1)
|
||||
start_floor = start.replace(day=1)
|
||||
while cur >= start_floor:
|
||||
months.append(cur.strftime("%Y%m"))
|
||||
if cur.month == 1:
|
||||
cur = cur.replace(year=cur.year - 1, month=12)
|
||||
else:
|
||||
cur = cur.replace(month=cur.month - 1)
|
||||
return months
|
||||
|
||||
|
||||
def _nuforc_fetch_month_live(yyyymm: str, cookie_jar: Path) -> list[dict]:
|
||||
"""Pull one month of NUFORC sightings via the live wpDataTables AJAX.
|
||||
|
||||
Returns a list of raw row dicts with the fields we care about:
|
||||
id, occurred (YYYY-MM-DD), posted (YYYY-MM-DD), city, state, country,
|
||||
shape_raw, summary, explanation. Empty list on any failure — caller
|
||||
decides whether a failure is fatal.
|
||||
"""
|
||||
from services.fetchers.nuforc_enrichment import _parse_date
|
||||
|
||||
curl_bin = shutil.which("curl") or "curl"
|
||||
index_url = _NUFORC_LIVE_INDEX_URL.format(yyyymm=yyyymm)
|
||||
ajax_url = _NUFORC_LIVE_AJAX_URL.format(yyyymm=yyyymm)
|
||||
|
||||
# Step 1: GET the month index to capture session cookies + fresh nonce.
|
||||
try:
|
||||
index_res = subprocess.run(
|
||||
[
|
||||
curl_bin, "-sL",
|
||||
"-A", _NUFORC_LIVE_USER_AGENT,
|
||||
"-c", str(cookie_jar),
|
||||
"-b", str(cookie_jar),
|
||||
index_url,
|
||||
],
|
||||
capture_output=True, text=True, timeout=60,
|
||||
encoding="utf-8", errors="replace",
|
||||
)
|
||||
except (subprocess.SubprocessError, OSError) as e:
|
||||
logger.warning("NUFORC live: index fetch failed for %s: %s", yyyymm, e)
|
||||
return []
|
||||
if index_res.returncode != 0 or not index_res.stdout:
|
||||
logger.warning(
|
||||
"NUFORC live: index fetch exit=%s for %s", index_res.returncode, yyyymm,
|
||||
)
|
||||
return []
|
||||
nonce_match = _NUFORC_LIVE_NONCE_RE.search(index_res.stdout)
|
||||
if not nonce_match:
|
||||
logger.warning("NUFORC live: wdtNonce not found on index page for %s", yyyymm)
|
||||
return []
|
||||
nonce = nonce_match.group(1)
|
||||
|
||||
# Step 2: POST to admin-ajax.php with length=-1 to pull the whole month.
|
||||
post_data = (
|
||||
"draw=1"
|
||||
"&columns%5B0%5D%5Bdata%5D=0&columns%5B0%5D%5Bsearchable%5D=true&columns%5B0%5D%5Borderable%5D=false"
|
||||
"&columns%5B1%5D%5Bdata%5D=1&columns%5B1%5D%5Bsearchable%5D=true&columns%5B1%5D%5Borderable%5D=true"
|
||||
"&order%5B0%5D%5Bcolumn%5D=1&order%5B0%5D%5Bdir%5D=desc"
|
||||
"&start=0&length=-1"
|
||||
"&search%5Bvalue%5D=&search%5Bregex%5D=false"
|
||||
f"&wdtNonce={nonce}"
|
||||
)
|
||||
try:
|
||||
ajax_res = subprocess.run(
|
||||
[
|
||||
curl_bin, "-sL",
|
||||
"-A", _NUFORC_LIVE_USER_AGENT,
|
||||
"-c", str(cookie_jar),
|
||||
"-b", str(cookie_jar),
|
||||
"-X", "POST",
|
||||
"-H", f"Referer: {index_url}",
|
||||
"-H", "X-Requested-With: XMLHttpRequest",
|
||||
"-H", "Content-Type: application/x-www-form-urlencoded",
|
||||
"--data", post_data,
|
||||
ajax_url,
|
||||
],
|
||||
capture_output=True, text=True, timeout=120,
|
||||
encoding="utf-8", errors="replace",
|
||||
)
|
||||
except (subprocess.SubprocessError, OSError) as e:
|
||||
logger.warning("NUFORC live: ajax fetch failed for %s: %s", yyyymm, e)
|
||||
return []
|
||||
if ajax_res.returncode != 0 or not ajax_res.stdout:
|
||||
logger.warning(
|
||||
"NUFORC live: ajax fetch exit=%s for %s", ajax_res.returncode, yyyymm,
|
||||
)
|
||||
return []
|
||||
try:
|
||||
payload = json.loads(ajax_res.stdout)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning("NUFORC live: ajax JSON decode failed for %s: %s", yyyymm, e)
|
||||
return []
|
||||
|
||||
raw_rows = payload.get("data") or []
|
||||
out: list[dict] = []
|
||||
for raw in raw_rows:
|
||||
if not isinstance(raw, list) or len(raw) < 8:
|
||||
continue
|
||||
link_html = str(raw[0] or "")
|
||||
occurred_raw = str(raw[1] or "")
|
||||
city = str(raw[2] or "").strip()
|
||||
state = str(raw[3] or "").strip()
|
||||
country = str(raw[4] or "").strip()
|
||||
shape_raw = (str(raw[5] or "").strip() or "Unknown")
|
||||
summary = str(raw[6] or "").strip()
|
||||
reported_raw = str(raw[7] or "")
|
||||
explanation = str(raw[9] or "").strip() if len(raw) > 9 and raw[9] else ""
|
||||
|
||||
occurred_ymd = _parse_date(occurred_raw)
|
||||
if not occurred_ymd:
|
||||
continue
|
||||
if not city and not state and not country:
|
||||
continue
|
||||
|
||||
id_match = _NUFORC_LIVE_SIGHTING_ID_RE.search(link_html)
|
||||
if id_match:
|
||||
sighting_id = f"NUFORC-{id_match.group(1)}"
|
||||
else:
|
||||
digest = hashlib.sha1(
|
||||
f"{occurred_ymd}|{city}|{state}|{summary}".encode("utf-8", "ignore")
|
||||
).hexdigest()[:12]
|
||||
sighting_id = f"NUFORC-{digest}"
|
||||
|
||||
if summary and len(summary) > 280:
|
||||
summary = summary[:277] + "..."
|
||||
if not summary:
|
||||
summary = "Sighting reported"
|
||||
|
||||
out.append({
|
||||
"id": sighting_id,
|
||||
"occurred": occurred_ymd,
|
||||
"posted": _parse_date(reported_raw) or occurred_ymd,
|
||||
"city": city,
|
||||
"state": state,
|
||||
"country": country,
|
||||
"shape_raw": shape_raw,
|
||||
"summary": summary,
|
||||
"explanation": explanation,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def _build_recent_uap_sightings() -> list[dict]:
|
||||
"""Build the rolling 1-year UAP sightings layer from live NUFORC data.
|
||||
|
||||
Hits nuforc.org's public sub-index once per month in the window, drops
|
||||
anything outside the exact day-precision cutoff, dedupes by sighting id,
|
||||
geocodes city+state via the existing location cache, and returns rows
|
||||
keyed to the same schema the frontend already renders.
|
||||
"""
|
||||
cutoff_dt = datetime.utcnow() - timedelta(days=_NUFORC_RECENT_DAYS)
|
||||
cutoff_str = cutoff_dt.strftime("%Y-%m-%d")
|
||||
months = _nuforc_months_for_window(_NUFORC_RECENT_DAYS)
|
||||
|
||||
try:
|
||||
_NUFORC_DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
rows: list[dict] = []
|
||||
locations: dict[str, tuple[str, str]] = {}
|
||||
seen_ids: set[str] = set()
|
||||
total_pulled = 0
|
||||
months_with_data = 0
|
||||
|
||||
for yyyymm in months:
|
||||
month_rows = _nuforc_fetch_month_live(yyyymm, _NUFORC_LIVE_SESSION_COOKIES)
|
||||
if month_rows:
|
||||
months_with_data += 1
|
||||
total_pulled += len(month_rows)
|
||||
for row in month_rows:
|
||||
if row["occurred"] < cutoff_str:
|
||||
continue
|
||||
if row["id"] in seen_ids:
|
||||
continue
|
||||
seen_ids.add(row["id"])
|
||||
|
||||
# Build the geocode key as "City, State, Country" to match the
|
||||
# existing 3,000+ entry location cache (format: "Toronto, ON, Canada").
|
||||
parts = [row["city"], row["state"], row["country"]]
|
||||
location = _normalize_uap_location(
|
||||
", ".join(p for p in parts if p) if any(parts) else ""
|
||||
)
|
||||
if not location:
|
||||
continue
|
||||
|
||||
row["location"] = location
|
||||
locations.setdefault(location, (row["city"], row["state"], row["country"]))
|
||||
row["shape"] = (
|
||||
_normalize_uap_shape(row["shape_raw"])
|
||||
if row["shape_raw"] != "Unknown"
|
||||
else "unknown"
|
||||
)
|
||||
if not row["country"]:
|
||||
row["country"] = _uap_country_from_location(location, row["state"])
|
||||
rows.append(row)
|
||||
|
||||
# Clean up the cookie jar — we don't reuse it across runs.
|
||||
try:
|
||||
if _NUFORC_LIVE_SESSION_COOKIES.exists():
|
||||
_NUFORC_LIVE_SESSION_COOKIES.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Source-integrity canary: if the upstream plugin changed its
|
||||
# DataTables schema or the wdtNonce regex is stale, total_pulled
|
||||
# collapses to ~0 without any HTTP error. assert_canary logs a loud
|
||||
# ERROR so the failure is visible in the health registry and the
|
||||
# daily refresh log, instead of silently serving a stale cache.
|
||||
from services.slo import assert_canary
|
||||
assert_canary("uap_sightings", total_pulled)
|
||||
|
||||
if not rows:
|
||||
raise RuntimeError(
|
||||
f"NUFORC live: zero rows pulled across {len(months)} months "
|
||||
f"(months_with_data={months_with_data})"
|
||||
)
|
||||
|
||||
from services.geocode_validate import coord_in_country
|
||||
|
||||
location_cache = _load_nuforc_location_cache()
|
||||
missing_locations = [location for location in locations if location not in location_cache]
|
||||
if missing_locations:
|
||||
logger.info(
|
||||
"UAP sightings: geocoding %d new locations (throttled at %.1fs spacing)",
|
||||
len(missing_locations),
|
||||
_NUFORC_GEOCODE_SPACING_S,
|
||||
)
|
||||
# Sequential with spacing — Photon is fast and lenient but we
|
||||
# stay sub-second to be polite. Incremental cache saves every 50
|
||||
# hits keep long runs resumable.
|
||||
resolved = 0
|
||||
bbox_rejected = 0
|
||||
save_every = 50
|
||||
for idx, location in enumerate(missing_locations):
|
||||
city, state, country = locations[location]
|
||||
coords = None
|
||||
try:
|
||||
coords = _geocode_uap_location(location, city, state, country)
|
||||
except Exception:
|
||||
coords = None
|
||||
|
||||
# Country-bbox post-filter: reject namesake collisions like
|
||||
# "Milan, WI" landing in Milan, Italy. Unknown countries
|
||||
# (bbox not registered) are passed through unchanged.
|
||||
if coords and country:
|
||||
inside = coord_in_country(coords[0], coords[1], country)
|
||||
if inside is False:
|
||||
logger.warning(
|
||||
"UAP sightings: bbox reject %r -> (%.3f, %.3f) not in %s",
|
||||
location, coords[0], coords[1], country,
|
||||
)
|
||||
coords = None
|
||||
bbox_rejected += 1
|
||||
|
||||
location_cache[location] = coords
|
||||
if coords:
|
||||
resolved += 1
|
||||
|
||||
if (idx + 1) % save_every == 0:
|
||||
_save_nuforc_location_cache(location_cache)
|
||||
logger.info(
|
||||
"UAP sightings: geocoded %d/%d (%d resolved, %d bbox-rejected)",
|
||||
idx + 1, len(missing_locations), resolved, bbox_rejected,
|
||||
)
|
||||
if idx + 1 < len(missing_locations):
|
||||
time.sleep(_NUFORC_GEOCODE_SPACING_S)
|
||||
_save_nuforc_location_cache(location_cache)
|
||||
logger.info(
|
||||
"UAP sightings: geocoding complete — %d/%d resolved, %d bbox-rejected",
|
||||
resolved, len(missing_locations), bbox_rejected,
|
||||
)
|
||||
|
||||
sightings: list[dict] = []
|
||||
skipped_unmapped = 0
|
||||
skipped_bbox = 0
|
||||
for row in rows:
|
||||
coords = location_cache.get(row["location"])
|
||||
if not coords:
|
||||
skipped_unmapped += 1
|
||||
continue
|
||||
# Apply bbox filter to pre-existing cache entries too — this
|
||||
# cleans up the ~1-2% of cached coords that pre-dated the bbox
|
||||
# check without requiring a full cache rebuild.
|
||||
if row.get("country"):
|
||||
inside = coord_in_country(coords[0], coords[1], row["country"])
|
||||
if inside is False:
|
||||
skipped_bbox += 1
|
||||
continue
|
||||
sightings.append(
|
||||
{
|
||||
"id": row["id"],
|
||||
"date_time": row["occurred"],
|
||||
"city": row["city"],
|
||||
"state": row["state"],
|
||||
"country": row["country"],
|
||||
"shape": row["shape"],
|
||||
"shape_raw": row["shape_raw"],
|
||||
"duration": row.get("duration", ""),
|
||||
"summary": row["summary"],
|
||||
"posted": row["posted"],
|
||||
"lat": float(coords[0]),
|
||||
"lng": float(coords[1]),
|
||||
"count": 1,
|
||||
"source": "NUFORC",
|
||||
}
|
||||
)
|
||||
if row.get("explanation"):
|
||||
sightings[-1]["explanation"] = row["explanation"]
|
||||
|
||||
sightings.sort(
|
||||
key=lambda sighting: (
|
||||
sighting.get("date_time", ""),
|
||||
sighting.get("posted", ""),
|
||||
str(sighting.get("id", "")),
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
logger.info(
|
||||
"UAP sightings: %d mapped reports from %d rows across %d months "
|
||||
"(cutoff %s, %d unmapped, %d bbox-rejected)",
|
||||
len(sightings),
|
||||
total_pulled,
|
||||
len(months),
|
||||
cutoff_str,
|
||||
skipped_unmapped,
|
||||
skipped_bbox,
|
||||
)
|
||||
return sightings
|
||||
|
||||
|
||||
@with_retry(max_retries=1, base_delay=5)
|
||||
def fetch_uap_sightings(*, force_refresh: bool = False):
|
||||
"""Fetch last-year UAP sightings from NUFORC.
|
||||
|
||||
Startup reads the cached daily snapshot when it is still fresh. The daily
|
||||
scheduler forces a rebuild so this layer updates once per day instead of
|
||||
churning continuously.
|
||||
"""
|
||||
from services.fetchers._store import is_any_active
|
||||
|
||||
if not is_any_active("uap_sightings"):
|
||||
return
|
||||
|
||||
sightings = _load_nuforc_sightings_cache(force_refresh=force_refresh)
|
||||
if sightings is None:
|
||||
sightings = _build_recent_uap_sightings()
|
||||
_save_nuforc_sightings_cache(sightings)
|
||||
|
||||
with _data_lock:
|
||||
latest_data["uap_sightings"] = sightings
|
||||
_mark_fresh("uap_sightings")
|
||||
return
|
||||
|
||||
cutoff = datetime.utcnow() - timedelta(days=_NUFORC_RECENT_DAYS)
|
||||
|
||||
# Query the grid concurrently (up to 8 threads)
|
||||
all_features: list[dict] = []
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
||||
futures = {
|
||||
pool.submit(_fetch_nuforc_tilequery, lng, lat): (lng, lat)
|
||||
for lng, lat in _TILEQUERY_GRID
|
||||
}
|
||||
for fut in concurrent.futures.as_completed(futures, timeout=60):
|
||||
try:
|
||||
all_features.extend(fut.result())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Deduplicate by (LinkLat, LinkLon) and filter to recent sightings
|
||||
seen: set[tuple[str, str]] = set()
|
||||
sightings: list[dict] = []
|
||||
enriched_count = 0
|
||||
for feat in all_features:
|
||||
props = feat.get("properties", {})
|
||||
link_lat = props.get("LinkLat", "")
|
||||
link_lon = props.get("LinkLon", "")
|
||||
if not link_lat or not link_lon:
|
||||
continue
|
||||
|
||||
key = (link_lat, link_lon)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
|
||||
# Filter by date — keep if the latest sighting date >= cutoff
|
||||
to_date = props.get("To", "")
|
||||
from_date = props.get("From", "")
|
||||
latest_date = to_date or from_date
|
||||
latest_dt = _parse_nuforc_tile_date(latest_date)
|
||||
if latest_dt is not None and latest_dt < cutoff:
|
||||
continue
|
||||
|
||||
try:
|
||||
lat = float(link_lat)
|
||||
lng = float(link_lon)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
|
||||
count = int(props.get("Count", "1") or "1")
|
||||
state_abbr, country = _reverse_geocode_state(lat, lng)
|
||||
|
||||
# Enrich with HF NUFORC dataset (shape, duration, city, summary)
|
||||
enrichment = enrich_sighting(state_abbr, from_date, to_date)
|
||||
city = enrichment.get("city", "")
|
||||
shape_raw = enrichment.get("shape_raw", "Unknown")
|
||||
shape = _normalize_uap_shape(shape_raw) if shape_raw != "Unknown" else "unknown"
|
||||
duration = enrichment.get("duration", "")
|
||||
summary = enrichment.get("summary", "")
|
||||
if enrichment:
|
||||
enriched_count += 1
|
||||
|
||||
# Build display summary: prefer enriched text, fall back to count-based
|
||||
if not summary:
|
||||
summary = f"{count} sighting(s) reported" if count > 1 else "Sighting reported"
|
||||
|
||||
sightings.append({
|
||||
"id": f"NUFORC-{hash(key) & 0xFFFFFFFF:08x}",
|
||||
"date_time": from_date if from_date == to_date else f"{from_date} to {to_date}",
|
||||
"city": city,
|
||||
"state": state_abbr,
|
||||
"country": country,
|
||||
"shape": shape,
|
||||
"shape_raw": shape_raw,
|
||||
"duration": duration,
|
||||
"summary": summary,
|
||||
"posted": to_date,
|
||||
"lat": lat,
|
||||
"lng": lng,
|
||||
"count": count,
|
||||
"source": "NUFORC",
|
||||
})
|
||||
|
||||
logger.info(
|
||||
f"UAP sightings: {len(sightings)} recent from NUFORC tilequery "
|
||||
f"({len(all_features)} raw, {enriched_count} enriched)"
|
||||
)
|
||||
|
||||
with _data_lock:
|
||||
latest_data["uap_sightings"] = sightings
|
||||
if sightings:
|
||||
_mark_fresh("uap_sightings")
|
||||
|
||||
@@ -1,20 +1,24 @@
|
||||
"""
|
||||
Fuel burn & CO2 emissions estimator for private jets.
|
||||
Fuel burn & CO2 emissions estimator.
|
||||
Based on manufacturer-published cruise fuel burn rates (GPH at long-range cruise).
|
||||
1 US gallon of Jet-A produces ~21.1 lbs (9.57 kg) of CO2.
|
||||
|
||||
Piston entries use 100LL (avgas), which is close enough to Jet-A in CO2 yield
|
||||
(~8.4 kg/gal vs 9.57 kg/gal); we keep one constant to stay simple — the result
|
||||
is a slight over-estimate for piston aircraft, which is preferable to under.
|
||||
"""
|
||||
|
||||
JET_A_CO2_KG_PER_GALLON = 9.57
|
||||
|
||||
# ICAO type code -> gallons per hour at long-range cruise
|
||||
FUEL_BURN_GPH: dict[str, int] = {
|
||||
# Gulfstream
|
||||
# ── Gulfstream ─────────────────────────────────────────────────────
|
||||
"GLF6": 430, # G650/G650ER
|
||||
"G700": 480, # G700
|
||||
"GLF5": 390, # G550
|
||||
"GVSP": 400, # GV-SP
|
||||
"GLF4": 330, # G-IV
|
||||
# Bombardier
|
||||
# ── Bombardier business ────────────────────────────────────────────
|
||||
"GL7T": 490, # Global 7500
|
||||
"GLEX": 430, # Global Express/6000/6500
|
||||
"GL5T": 420, # Global 5000/5500
|
||||
@@ -22,51 +26,208 @@ FUEL_BURN_GPH: dict[str, int] = {
|
||||
"CL60": 310, # Challenger 604/605
|
||||
"CL30": 200, # Challenger 300
|
||||
"CL65": 320, # Challenger 650
|
||||
# Dassault
|
||||
# ── Bombardier regional jets ──────────────────────────────────────
|
||||
"CRJ2": 360, # CRJ-100/200
|
||||
"CRJ7": 380, # CRJ-700
|
||||
"CRJ9": 410, # CRJ-900
|
||||
"CRJX": 440, # CRJ-1000
|
||||
# ── Dassault ───────────────────────────────────────────────────────
|
||||
"F7X": 350, # Falcon 7X
|
||||
"F8X": 370, # Falcon 8X
|
||||
"F900": 285, # Falcon 900/900EX/900LX
|
||||
"F2TH": 230, # Falcon 2000
|
||||
"FA50": 240, # Falcon 50
|
||||
# Cessna
|
||||
# ── Cessna Citation ────────────────────────────────────────────────
|
||||
"CITX": 280, # Citation X
|
||||
"C750": 280, # Citation X (alt code)
|
||||
"C68A": 195, # Citation Latitude
|
||||
"C700": 230, # Citation Longitude
|
||||
"C680": 220, # Citation Sovereign
|
||||
"C560": 190, # Citation Excel/XLS
|
||||
"C56X": 195, # Citation Excel/XLS/XLS+
|
||||
"C560": 190, # Citation Excel/XLS (legacy)
|
||||
"C550": 165, # Citation II/Bravo/V
|
||||
"C525": 80, # Citation CJ1
|
||||
"C25A": 100, # CJ1+ / 525A
|
||||
"C25B": 110, # CJ2+ / 525B
|
||||
"C25C": 130, # CJ4 (some operators)
|
||||
"C510": 75, # Citation Mustang
|
||||
"C650": 240, # Citation III/VI/VII
|
||||
"CJ3": 120, # CJ3
|
||||
"CJ4": 135, # CJ4
|
||||
# Boeing
|
||||
"B737": 850, # BBJ (737)
|
||||
"B738": 920, # BBJ2 (737-800)
|
||||
# ── Cessna piston / turboprop singles & twins ─────────────────────
|
||||
"C172": 9, # Skyhawk
|
||||
"C152": 6,
|
||||
"C150": 6,
|
||||
"C170": 8,
|
||||
"C177": 11,
|
||||
"C180": 12,
|
||||
"C182": 13, # Skylane
|
||||
"C185": 14,
|
||||
"C206": 15,
|
||||
"C208": 50, # Caravan (turboprop)
|
||||
"C210": 18,
|
||||
"C310": 32,
|
||||
"C340": 38,
|
||||
"C414": 36,
|
||||
"C421": 40,
|
||||
# ── Boeing mainline ────────────────────────────────────────────────
|
||||
"B737": 850, # 737-700 / BBJ
|
||||
"B738": 920, # 737-800
|
||||
"B739": 880, # 737-900/900ER
|
||||
"B38M": 700, # 737-8 MAX
|
||||
"B39M": 740, # 737-9 MAX
|
||||
"B752": 1100, # 757-200
|
||||
"B753": 1200, # 757-300
|
||||
"B762": 1400, # 767-200
|
||||
"B763": 1450, # 767-300/300ER
|
||||
"B764": 1500, # 767-400ER
|
||||
"B772": 1850, # 777-200
|
||||
"B77L": 1900, # 777-200LR / 777F
|
||||
"B77W": 2050, # 777-300ER
|
||||
"B788": 1200, # 787-8
|
||||
# Airbus
|
||||
"A318": 780, # ACJ318
|
||||
"A319": 850, # ACJ319
|
||||
"A320": 900, # ACJ320
|
||||
"B789": 1300, # 787-9
|
||||
"B78X": 1350, # 787-10
|
||||
"B744": 3050, # 747-400
|
||||
"B748": 2900, # 747-8
|
||||
# ── Airbus mainline ────────────────────────────────────────────────
|
||||
"A318": 780, # A318
|
||||
"A319": 850, # A319
|
||||
"A320": 900, # A320
|
||||
"A321": 990, # A321
|
||||
"A19N": 580, # A319neo
|
||||
"A20N": 580, # A320neo
|
||||
"A21N": 700, # A321neo
|
||||
"A332": 1500, # A330-200
|
||||
"A333": 1550, # A330-300
|
||||
"A338": 1300, # A330-800neo
|
||||
"A339": 1350, # A330-900neo
|
||||
"A343": 1800, # A340-300
|
||||
"A346": 2100, # A340-600
|
||||
# Pilatus
|
||||
"A359": 1450, # A350-900
|
||||
"A35K": 1600, # A350-1000
|
||||
"A388": 3200, # A380-800
|
||||
# ── Embraer regional / business ───────────────────────────────────
|
||||
"E135": 300, # Legacy 600/650 (regional ERJ-135 base)
|
||||
"E145": 320, # ERJ-145
|
||||
"E170": 460, # E170
|
||||
"E75L": 490, # E175-LR
|
||||
"E75S": 490, # E175 standard
|
||||
"E175": 490, # E175 (some)
|
||||
"E190": 580, # E190
|
||||
"E195": 600, # E195
|
||||
"E290": 510, # E190-E2
|
||||
"E295": 540, # E195-E2
|
||||
"E50P": 135, # Phenom 300 (also Phenom 100 var)
|
||||
"E55P": 185, # Praetor 500 / Legacy 500
|
||||
"E545": 170, # Praetor 500 (alt)
|
||||
"E500": 80, # Phenom 100
|
||||
# ── ATR / Bombardier / Saab turboprops ────────────────────────────
|
||||
"AT43": 230, # ATR 42-300/-320
|
||||
"AT45": 230, # ATR 42-500
|
||||
"AT46": 250, # ATR 42-600
|
||||
"AT72": 300, # ATR 72-200/-210
|
||||
"AT75": 280, # ATR 72-500
|
||||
"AT76": 280, # ATR 72-600
|
||||
"DH8A": 220, # Dash 8 -100
|
||||
"DH8B": 240, # Dash 8 -200
|
||||
"DH8C": 280, # Dash 8 -300
|
||||
"DH8D": 300, # Dash 8 Q400
|
||||
"SF34": 200, # Saab 340
|
||||
"SB20": 220, # Saab 2000
|
||||
# ── Pilatus / Daher single-engine turboprops ──────────────────────
|
||||
"PC24": 115, # PC-24
|
||||
"PC12": 60, # PC-12
|
||||
# Embraer
|
||||
"E55P": 185, # Legacy 500
|
||||
"E135": 300, # Legacy 600/650
|
||||
"E50P": 135, # Phenom 300
|
||||
"E500": 80, # Phenom 100
|
||||
# Learjet
|
||||
"TBM7": 60, # TBM 700/850
|
||||
"TBM8": 65, # TBM 850 alt
|
||||
"TBM9": 70, # TBM 900/930/940/960
|
||||
"M600": 60, # Piper M600
|
||||
"P46T": 22, # PA-46 Meridian (turboprop variant)
|
||||
# ── Learjet ────────────────────────────────────────────────────────
|
||||
"LJ60": 195, # Learjet 60
|
||||
"LJ75": 185, # Learjet 75
|
||||
"LJ45": 175, # Learjet 45
|
||||
# Hawker
|
||||
"LJ31": 165, # Learjet 31
|
||||
"LJ40": 175, # Learjet 40
|
||||
"LJ55": 195, # Learjet 55
|
||||
# ── Hawker / Beechjet ─────────────────────────────────────────────
|
||||
"H25B": 210, # Hawker 800/800XP
|
||||
"H25C": 215, # Hawker 900XP
|
||||
# Beechcraft
|
||||
"BE40": 150, # Beechjet 400 / Hawker 400XP
|
||||
"PRM1": 130, # Premier I
|
||||
# ── Beechcraft King Air ───────────────────────────────────────────
|
||||
"B350": 100, # King Air 350
|
||||
"B200": 80, # King Air 200/250
|
||||
"BE20": 80, # K-Air 200 (alt)
|
||||
"BE9L": 60, # K-Air 90
|
||||
"BE9T": 70, # K-Air F90
|
||||
"BE10": 100, # K-Air 100
|
||||
"BE30": 90, # K-Air 300
|
||||
# ── Beechcraft / Cirrus / Piper / Mooney pistons ──────────────────
|
||||
"BE23": 9, # Sundowner
|
||||
"BE33": 13, # Bonanza 33
|
||||
"BE35": 14, # Bonanza V-tail
|
||||
"BE36": 16, # A36 Bonanza
|
||||
"BE55": 24, # Baron 55
|
||||
"BE58": 28, # Baron 58
|
||||
"BE76": 17, # Duchess
|
||||
"BE95": 20, # Travel Air
|
||||
"P28A": 10, # PA-28 Warrior/Archer
|
||||
"P28B": 11, # PA-28 Cherokee
|
||||
"P28R": 12, # PA-28R Arrow
|
||||
"P32R": 14, # PA-32R Lance/Saratoga
|
||||
"PA11": 5, # Cub Special
|
||||
"PA12": 6, # Super Cruiser
|
||||
"PA18": 6, # Super Cub
|
||||
"PA22": 8, # Tri-Pacer
|
||||
"PA23": 18, # Apache / Aztec
|
||||
"PA24": 12, # Comanche
|
||||
"PA25": 12, # Pawnee
|
||||
"PA28": 10, # PA-28 generic
|
||||
"PA30": 16, # Twin Comanche
|
||||
"PA31": 30, # Navajo
|
||||
"PA32": 14, # Cherokee Six / Saratoga
|
||||
"PA34": 18, # Seneca
|
||||
"PA38": 5, # Tomahawk
|
||||
"PA44": 17, # Seminole
|
||||
"PA46": 18, # Malibu / Mirage / Matrix
|
||||
"M20P": 12, # Mooney M20 (generic)
|
||||
"SR20": 11, # Cirrus SR20
|
||||
"SR22": 16, # Cirrus SR22
|
||||
"S22T": 19, # SR22T (turbo)
|
||||
"DA40": 9, # Diamond DA40
|
||||
"DA42": 14, # Diamond DA42 TwinStar
|
||||
"DA62": 17, # Diamond DA62
|
||||
"DV20": 6, # Diamond Katana
|
||||
# ── Helicopters (civilian) ────────────────────────────────────────
|
||||
"A109": 60, # AW109
|
||||
"A119": 50, # AW119
|
||||
"A139": 130, # AW139
|
||||
"A169": 90, # AW169
|
||||
"A189": 145, # AW189
|
||||
"AS35": 55, # AS350 AStar
|
||||
"AS50": 55, # AStar (alt)
|
||||
"AS65": 110, # Dauphin
|
||||
"B06": 35, # Bell 206 JetRanger
|
||||
"B407": 50, # Bell 407
|
||||
"B412": 145, # Bell 412
|
||||
"B429": 80, # Bell 429
|
||||
"B505": 35, # Bell 505
|
||||
"EC30": 50, # H125 / EC130
|
||||
"EC35": 70, # EC135
|
||||
"EC45": 85, # EC145
|
||||
"EC75": 130, # EC175
|
||||
"H125": 55,
|
||||
"H130": 50,
|
||||
"H135": 70,
|
||||
"H145": 85,
|
||||
"H155": 110,
|
||||
"H160": 95,
|
||||
"H175": 130,
|
||||
"R22": 9, # Robinson R22 (piston)
|
||||
"R44": 16, # Robinson R44 (piston)
|
||||
"R66": 30, # Robinson R66 (turbine)
|
||||
"S76": 140, # Sikorsky S-76
|
||||
"S92": 220, # Sikorsky S-92
|
||||
}
|
||||
|
||||
# Common string names -> ICAO type code
|
||||
@@ -108,13 +269,23 @@ def get_emissions_info(model: str) -> dict | None:
|
||||
if not model:
|
||||
return None
|
||||
model_clean = model.strip()
|
||||
model_upper = model_clean.upper()
|
||||
# Try direct ICAO code match first
|
||||
gph = FUEL_BURN_GPH.get(model_clean.upper())
|
||||
gph = FUEL_BURN_GPH.get(model_upper)
|
||||
if gph is None:
|
||||
# Try alias lookup
|
||||
code = _ALIASES.get(model_clean)
|
||||
if code:
|
||||
gph = FUEL_BURN_GPH.get(code)
|
||||
if gph is None:
|
||||
# Friendly names from the Plane-Alert DB often lead with the ICAO type
|
||||
# code as the first token (e.g. "B200 Super King Air"). Probe each
|
||||
# token against FUEL_BURN_GPH directly.
|
||||
for token in model_upper.replace("-", " ").replace(",", " ").split():
|
||||
candidate = FUEL_BURN_GPH.get(token)
|
||||
if candidate is not None:
|
||||
gph = candidate
|
||||
break
|
||||
if gph is None:
|
||||
# Fuzzy: check if any alias is a substring
|
||||
model_lower = model_clean.lower()
|
||||
|
||||
@@ -13,12 +13,13 @@ import concurrent.futures
|
||||
import random
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from cachetools import TTLCache
|
||||
from services.network_utils import fetch_with_curl
|
||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
||||
from services.fetchers.plane_alert import enrich_with_plane_alert, enrich_with_tracked_names
|
||||
from services.fetchers.emissions import get_emissions_info
|
||||
from services.fetchers.retry import with_retry
|
||||
from services.fetchers.route_database import lookup_route
|
||||
from services.fetchers.aircraft_database import lookup_aircraft_type
|
||||
from services.constants import GPS_JAMMING_NACP_THRESHOLD, GPS_JAMMING_MIN_RATIO, GPS_JAMMING_MIN_AIRCRAFT
|
||||
|
||||
logger = logging.getLogger("services.data_fetcher")
|
||||
@@ -76,6 +77,7 @@ opensky_client = OpenSkyClient(
|
||||
# Throttling and caching for OpenSky (400 req/day limit)
|
||||
last_opensky_fetch = 0
|
||||
cached_opensky_flights = []
|
||||
_opensky_cache_lock = threading.Lock()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Supplemental ADS-B sources for blind-spot gap-filling
|
||||
@@ -98,6 +100,7 @@ _AIRPLANES_LIVE_DELAY_SECONDS = 1.2
|
||||
_AIRPLANES_LIVE_DELAY_JITTER_SECONDS = 0.4
|
||||
last_supplemental_fetch = 0
|
||||
cached_supplemental_flights = []
|
||||
_supplemental_cache_lock = threading.Lock()
|
||||
|
||||
# Helicopter type codes (backend classification)
|
||||
_HELI_TYPES_BACKEND = {
|
||||
@@ -255,10 +258,11 @@ flight_trails = {} # {icao_hex: {points: [[lat, lng, alt, ts], ...], last_seen:
|
||||
_trails_lock = threading.Lock()
|
||||
_MAX_TRACKED_TRAILS = 2000
|
||||
|
||||
# Routes cache
|
||||
dynamic_routes_cache = TTLCache(maxsize=5000, ttl=7200)
|
||||
routes_fetch_in_progress = False
|
||||
_routes_lock = threading.Lock()
|
||||
# Route enrichment is now served from services.fetchers.route_database, which
|
||||
# bulk-loads vrs-standing-data.adsb.lol/routes.csv.gz once per day and looks up
|
||||
# callsigns from an in-memory index. Replaces the legacy /api/0/routeset POST,
|
||||
# which was both blocked under the ShadowBroker UA (HTTP 451) and broken
|
||||
# upstream (returning 201 with empty body even for unblocked clients).
|
||||
|
||||
|
||||
def _fetch_supplemental_sources(seen_hex: set) -> list:
|
||||
@@ -266,12 +270,13 @@ def _fetch_supplemental_sources(seen_hex: set) -> list:
|
||||
global last_supplemental_fetch, cached_supplemental_flights
|
||||
|
||||
now = time.time()
|
||||
if now - last_supplemental_fetch < _SUPPLEMENTAL_FETCH_INTERVAL:
|
||||
return [
|
||||
f
|
||||
for f in cached_supplemental_flights
|
||||
if f.get("hex", "").lower().strip() not in seen_hex
|
||||
]
|
||||
with _supplemental_cache_lock:
|
||||
if now - last_supplemental_fetch < _SUPPLEMENTAL_FETCH_INTERVAL:
|
||||
return [
|
||||
f
|
||||
for f in cached_supplemental_flights
|
||||
if f.get("hex", "").lower().strip() not in seen_hex
|
||||
]
|
||||
|
||||
new_supplemental = []
|
||||
supplemental_hex = set()
|
||||
@@ -363,8 +368,9 @@ def _fetch_supplemental_sources(seen_hex: set) -> list:
|
||||
|
||||
fi_count = len(new_supplemental) - ap_count
|
||||
|
||||
cached_supplemental_flights = new_supplemental
|
||||
last_supplemental_fetch = now
|
||||
with _supplemental_cache_lock:
|
||||
cached_supplemental_flights = new_supplemental
|
||||
last_supplemental_fetch = now
|
||||
if new_supplemental:
|
||||
_mark_fresh("supplemental_flights")
|
||||
|
||||
@@ -375,73 +381,6 @@ def _fetch_supplemental_sources(seen_hex: set) -> list:
|
||||
return new_supplemental
|
||||
|
||||
|
||||
def fetch_routes_background(sampled):
|
||||
global routes_fetch_in_progress
|
||||
with _routes_lock:
|
||||
if routes_fetch_in_progress:
|
||||
return
|
||||
routes_fetch_in_progress = True
|
||||
|
||||
try:
|
||||
callsigns_to_query = []
|
||||
for f in sampled:
|
||||
c_sign = str(f.get("flight", "")).strip()
|
||||
if c_sign and c_sign != "UNKNOWN":
|
||||
callsigns_to_query.append(
|
||||
{"callsign": c_sign, "lat": f.get("lat", 0), "lng": f.get("lon", 0)}
|
||||
)
|
||||
|
||||
batch_size = 100
|
||||
batches = [
|
||||
callsigns_to_query[i : i + batch_size]
|
||||
for i in range(0, len(callsigns_to_query), batch_size)
|
||||
]
|
||||
|
||||
for batch in batches:
|
||||
try:
|
||||
r = fetch_with_curl(
|
||||
"https://api.adsb.lol/api/0/routeset",
|
||||
method="POST",
|
||||
json_data={"planes": batch},
|
||||
timeout=15,
|
||||
)
|
||||
if r.status_code == 200:
|
||||
route_data = r.json()
|
||||
route_list = []
|
||||
if isinstance(route_data, dict):
|
||||
route_list = route_data.get("value", [])
|
||||
elif isinstance(route_data, list):
|
||||
route_list = route_data
|
||||
|
||||
for route in route_list:
|
||||
callsign = route.get("callsign", "")
|
||||
airports = route.get("_airports", [])
|
||||
if airports and len(airports) >= 2:
|
||||
orig_apt = airports[0]
|
||||
dest_apt = airports[-1]
|
||||
with _routes_lock:
|
||||
dynamic_routes_cache[callsign] = {
|
||||
"orig_name": f"{orig_apt.get('iata', '')}: {orig_apt.get('name', 'Unknown')}",
|
||||
"dest_name": f"{dest_apt.get('iata', '')}: {dest_apt.get('name', 'Unknown')}",
|
||||
"orig_loc": [orig_apt.get("lon", 0), orig_apt.get("lat", 0)],
|
||||
"dest_loc": [dest_apt.get("lon", 0), dest_apt.get("lat", 0)],
|
||||
}
|
||||
time.sleep(0.25)
|
||||
except (
|
||||
requests.RequestException,
|
||||
ConnectionError,
|
||||
TimeoutError,
|
||||
ValueError,
|
||||
KeyError,
|
||||
json.JSONDecodeError,
|
||||
OSError,
|
||||
) as e:
|
||||
logger.debug(f"Route batch request failed: {e}")
|
||||
finally:
|
||||
with _routes_lock:
|
||||
routes_fetch_in_progress = False
|
||||
|
||||
|
||||
def _classify_and_publish(all_adsb_flights):
|
||||
"""Shared pipeline: normalize raw ADS-B data → classify → merge → publish to latest_data.
|
||||
|
||||
@@ -453,13 +392,6 @@ def _classify_and_publish(all_adsb_flights):
|
||||
if not all_adsb_flights:
|
||||
return
|
||||
|
||||
with _routes_lock:
|
||||
already_running = routes_fetch_in_progress
|
||||
if not already_running:
|
||||
threading.Thread(
|
||||
target=fetch_routes_background, args=(all_adsb_flights,), daemon=True
|
||||
).start()
|
||||
|
||||
for f in all_adsb_flights:
|
||||
try:
|
||||
lat = f.get("lat")
|
||||
@@ -478,8 +410,7 @@ def _classify_and_publish(all_adsb_flights):
|
||||
origin_name = "UNKNOWN"
|
||||
dest_name = "UNKNOWN"
|
||||
|
||||
with _routes_lock:
|
||||
cached_route = dynamic_routes_cache.get(flight_str)
|
||||
cached_route = lookup_route(flight_str)
|
||||
if cached_route:
|
||||
origin_name = cached_route["orig_name"]
|
||||
dest_name = cached_route["dest_name"]
|
||||
@@ -501,7 +432,18 @@ def _classify_and_publish(all_adsb_flights):
|
||||
gs_knots = f.get("gs")
|
||||
speed_knots = round(gs_knots, 1) if isinstance(gs_knots, (int, float)) else None
|
||||
|
||||
model_upper = f.get("t", "").upper()
|
||||
# OpenSky's /states/all doesn't carry the aircraft type, so its
|
||||
# records arrive with t="Unknown". Backfill from the OpenSky
|
||||
# aircraft metadata DB by ICAO24 hex so heli classification and
|
||||
# downstream emissions enrichment both see a real type code.
|
||||
raw_type = str(f.get("t") or "").strip()
|
||||
if not raw_type or raw_type.lower() == "unknown":
|
||||
looked_up_type = lookup_aircraft_type(f.get("hex", ""))
|
||||
if looked_up_type:
|
||||
f["t"] = looked_up_type
|
||||
raw_type = looked_up_type
|
||||
|
||||
model_upper = raw_type.upper()
|
||||
if model_upper == "TWR":
|
||||
continue
|
||||
|
||||
@@ -543,8 +485,14 @@ def _classify_and_publish(all_adsb_flights):
|
||||
for f in flights:
|
||||
enrich_with_plane_alert(f)
|
||||
enrich_with_tracked_names(f)
|
||||
# Attach fuel-burn / CO2 emissions estimate when model is known
|
||||
# Attach fuel-burn / CO2 emissions estimate when model is known.
|
||||
# OpenSky's /states/all doesn't carry aircraft type, so OpenSky-sourced
|
||||
# flights arrive with model="Unknown". For tracked planes, the
|
||||
# Plane-Alert DB has the friendly type name in alert_type, and the
|
||||
# emissions aliases table already maps those names to ICAO codes.
|
||||
model = f.get("model")
|
||||
if not model or model.strip().lower() in {"", "unknown"}:
|
||||
model = f.get("alert_type") or ""
|
||||
if model:
|
||||
emi = get_emissions_info(model)
|
||||
if emi:
|
||||
@@ -618,6 +566,10 @@ def _classify_and_publish(all_adsb_flights):
|
||||
latest_data["flights"] = flights
|
||||
|
||||
# Merge tracked civilian flights with tracked military flights
|
||||
# Stale tracked flights (not seen in any ADS-B source for >5 min) are dropped.
|
||||
_TRACKED_STALE_S = 300 # 5 minutes
|
||||
_merge_ts = time.time()
|
||||
|
||||
with _data_lock:
|
||||
existing_tracked = copy.deepcopy(latest_data.get("tracked_flights", []))
|
||||
|
||||
@@ -625,10 +577,12 @@ def _classify_and_publish(all_adsb_flights):
|
||||
for t in tracked:
|
||||
icao = t.get("icao24", "").upper()
|
||||
if icao:
|
||||
t["_seen_at"] = _merge_ts
|
||||
fresh_tracked_map[icao] = t
|
||||
|
||||
merged_tracked = []
|
||||
seen_icaos = set()
|
||||
stale_dropped = 0
|
||||
for old_t in existing_tracked:
|
||||
icao = old_t.get("icao24", "").upper()
|
||||
if icao in fresh_tracked_map:
|
||||
@@ -639,8 +593,13 @@ def _classify_and_publish(all_adsb_flights):
|
||||
merged_tracked.append(fresh)
|
||||
seen_icaos.add(icao)
|
||||
else:
|
||||
merged_tracked.append(old_t)
|
||||
seen_icaos.add(icao)
|
||||
# Keep stale entry only if it was seen recently
|
||||
age = _merge_ts - old_t.get("_seen_at", 0)
|
||||
if age < _TRACKED_STALE_S:
|
||||
merged_tracked.append(old_t)
|
||||
seen_icaos.add(icao)
|
||||
else:
|
||||
stale_dropped += 1
|
||||
|
||||
for icao, t in fresh_tracked_map.items():
|
||||
if icao not in seen_icaos:
|
||||
@@ -649,10 +608,12 @@ def _classify_and_publish(all_adsb_flights):
|
||||
with _data_lock:
|
||||
latest_data["tracked_flights"] = merged_tracked
|
||||
logger.info(
|
||||
f"Tracked flights: {len(merged_tracked)} total ({len(fresh_tracked_map)} fresh from civilian)"
|
||||
f"Tracked flights: {len(merged_tracked)} total ({len(fresh_tracked_map)} fresh from civilian, {stale_dropped} stale dropped)"
|
||||
)
|
||||
|
||||
# --- Trail Accumulation ---
|
||||
_TRAIL_INTERVAL_S = 600 # only record a new trail point every 10 minutes
|
||||
|
||||
def _accumulate_trail(f, now_ts, check_route=True):
|
||||
hex_id = f.get("icao24", "").lower()
|
||||
if not hex_id:
|
||||
@@ -668,7 +629,11 @@ def _classify_and_publish(all_adsb_flights):
|
||||
if hex_id not in flight_trails:
|
||||
flight_trails[hex_id] = {"points": [], "last_seen": now_ts}
|
||||
trail_data = flight_trails[hex_id]
|
||||
if (
|
||||
# Only append a new point if 10 minutes have passed since the last one
|
||||
last_point_ts = trail_data["points"][-1][3] if trail_data["points"] else 0
|
||||
if now_ts - last_point_ts < _TRAIL_INTERVAL_S:
|
||||
trail_data["last_seen"] = now_ts
|
||||
elif (
|
||||
trail_data["points"]
|
||||
and trail_data["points"][-1][0] == point[0]
|
||||
and trail_data["points"][-1][1] == point[1]
|
||||
@@ -688,22 +653,26 @@ def _classify_and_publish(all_adsb_flights):
|
||||
tracked_snapshot = copy.deepcopy(latest_data.get("tracked_flights", []))
|
||||
raw_flights_snapshot = list(latest_data.get("flights", []))
|
||||
|
||||
all_lists = [commercial, private_jets, private_ga, existing_tracked]
|
||||
# Commercial/private: skip trail if route is known (route line replaces trail)
|
||||
route_check_lists = [commercial, private_jets, private_ga]
|
||||
# Tracked + military: ALWAYS accumulate trails (high-interest flights)
|
||||
always_trail_lists = [existing_tracked, military_snapshot]
|
||||
seen_hexes = set()
|
||||
trail_count = 0
|
||||
with _trails_lock:
|
||||
for flist in all_lists:
|
||||
for flist in route_check_lists:
|
||||
for f in flist:
|
||||
count, hex_id = _accumulate_trail(f, now_ts, check_route=True)
|
||||
trail_count += count
|
||||
if hex_id:
|
||||
seen_hexes.add(hex_id)
|
||||
|
||||
for mf in military_snapshot:
|
||||
count, hex_id = _accumulate_trail(mf, now_ts, check_route=False)
|
||||
trail_count += count
|
||||
if hex_id:
|
||||
seen_hexes.add(hex_id)
|
||||
for flist in always_trail_lists:
|
||||
for f in flist:
|
||||
count, hex_id = _accumulate_trail(f, now_ts, check_route=False)
|
||||
trail_count += count
|
||||
if hex_id:
|
||||
seen_hexes.add(hex_id)
|
||||
|
||||
tracked_hexes = {t.get("icao24", "").lower() for t in tracked_snapshot}
|
||||
stale_keys = []
|
||||
@@ -889,79 +858,100 @@ def _enrich_with_opensky_and_supplemental(adsb_flights):
|
||||
now = time.time()
|
||||
global last_opensky_fetch, cached_opensky_flights
|
||||
|
||||
if now - last_opensky_fetch > 300:
|
||||
with _opensky_cache_lock:
|
||||
_need_opensky = now - last_opensky_fetch > 300
|
||||
if not _need_opensky:
|
||||
opensky_snapshot = list(cached_opensky_flights)
|
||||
|
||||
if _need_opensky:
|
||||
token = opensky_client.get_token()
|
||||
if token:
|
||||
opensky_regions = [
|
||||
{
|
||||
"name": "Africa",
|
||||
"bbox": {"lamin": -35.0, "lomin": -20.0, "lamax": 38.0, "lomax": 55.0},
|
||||
},
|
||||
{
|
||||
"name": "Asia",
|
||||
"bbox": {"lamin": 0.0, "lomin": 30.0, "lamax": 75.0, "lomax": 150.0},
|
||||
},
|
||||
{
|
||||
"name": "South America",
|
||||
"bbox": {"lamin": -60.0, "lomin": -95.0, "lamax": 15.0, "lomax": -30.0},
|
||||
},
|
||||
]
|
||||
|
||||
# One global /states/all query = 4 credits flat per OpenSky
|
||||
# docs (https://openskynetwork.github.io/opensky-api/rest.html).
|
||||
# At the current 5-minute cadence that's 4 × 288 = 1152
|
||||
# credits/day, ~29% of the 4000-credit standard daily quota,
|
||||
# and returns every aircraft worldwide in a single call.
|
||||
# The previous 3-regional-bbox approach cost 12 credits/cycle
|
||||
# AND missed North America, Europe, and Oceania entirely.
|
||||
new_opensky_flights = []
|
||||
for os_reg in opensky_regions:
|
||||
try:
|
||||
bb = os_reg["bbox"]
|
||||
os_url = f"https://opensky-network.org/api/states/all?lamin={bb['lamin']}&lomin={bb['lomin']}&lamax={bb['lamax']}&lomax={bb['lomax']}"
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
os_res = requests.get(os_url, headers=headers, timeout=15)
|
||||
try:
|
||||
os_url = "https://opensky-network.org/api/states/all"
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
os_res = requests.get(os_url, headers=headers, timeout=30)
|
||||
|
||||
if os_res.status_code == 200:
|
||||
os_data = os_res.json()
|
||||
states = os_data.get("states") or []
|
||||
logger.info(
|
||||
f"OpenSky: Fetched {len(states)} states for {os_reg['name']}"
|
||||
if os_res.status_code == 200:
|
||||
os_data = os_res.json()
|
||||
states = os_data.get("states") or []
|
||||
remaining = os_res.headers.get("X-Rate-Limit-Remaining", "?")
|
||||
logger.info(
|
||||
f"OpenSky: fetched {len(states)} global states "
|
||||
f"(credits remaining: {remaining})"
|
||||
)
|
||||
for s in states:
|
||||
if s[5] is None or s[6] is None:
|
||||
continue
|
||||
new_opensky_flights.append(
|
||||
{
|
||||
"hex": s[0],
|
||||
"flight": s[1].strip() if s[1] else "UNKNOWN",
|
||||
"r": s[2],
|
||||
"lon": s[5],
|
||||
"lat": s[6],
|
||||
"alt_baro": (s[7] * 3.28084) if s[7] else 0,
|
||||
"track": s[10] or 0,
|
||||
"gs": (s[9] * 1.94384) if s[9] else 0,
|
||||
"t": "Unknown",
|
||||
"is_opensky": True,
|
||||
}
|
||||
)
|
||||
elif os_res.status_code == 429:
|
||||
retry_after = os_res.headers.get("X-Rate-Limit-Retry-After-Seconds", "?")
|
||||
logger.warning(
|
||||
f"OpenSky daily quota exhausted (4000 credits). "
|
||||
f"Retry after {retry_after}s. Serving stale data until reset."
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
f"OpenSky /states/all failed: HTTP {os_res.status_code}"
|
||||
)
|
||||
except (
|
||||
requests.RequestException,
|
||||
ConnectionError,
|
||||
TimeoutError,
|
||||
ValueError,
|
||||
KeyError,
|
||||
json.JSONDecodeError,
|
||||
OSError,
|
||||
) as ex:
|
||||
logger.error(f"OpenSky global fetch error: {ex}")
|
||||
|
||||
for s in states:
|
||||
new_opensky_flights.append(
|
||||
{
|
||||
"hex": s[0],
|
||||
"flight": s[1].strip() if s[1] else "UNKNOWN",
|
||||
"r": s[2],
|
||||
"lon": s[5],
|
||||
"lat": s[6],
|
||||
"alt_baro": (s[7] * 3.28084) if s[7] else 0,
|
||||
"track": s[10] or 0,
|
||||
"gs": (s[9] * 1.94384) if s[9] else 0,
|
||||
"t": "Unknown",
|
||||
"is_opensky": True,
|
||||
}
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
f"OpenSky API {os_reg['name']} failed: {os_res.status_code}"
|
||||
)
|
||||
except (
|
||||
requests.RequestException,
|
||||
ConnectionError,
|
||||
TimeoutError,
|
||||
ValueError,
|
||||
KeyError,
|
||||
json.JSONDecodeError,
|
||||
OSError,
|
||||
) as ex:
|
||||
logger.error(f"OpenSky fetching error for {os_reg['name']}: {ex}")
|
||||
|
||||
cached_opensky_flights = new_opensky_flights
|
||||
last_opensky_fetch = now
|
||||
with _opensky_cache_lock:
|
||||
if new_opensky_flights:
|
||||
cached_opensky_flights = new_opensky_flights
|
||||
last_opensky_fetch = now
|
||||
opensky_snapshot = new_opensky_flights or list(cached_opensky_flights)
|
||||
else:
|
||||
# Token refresh failed — fall back to existing cached data
|
||||
with _opensky_cache_lock:
|
||||
opensky_snapshot = list(cached_opensky_flights)
|
||||
|
||||
# Merge OpenSky (dedup by hex)
|
||||
for osf in cached_opensky_flights:
|
||||
for osf in opensky_snapshot:
|
||||
h = osf.get("hex")
|
||||
if h and h.lower().strip() not in seen_hex:
|
||||
all_flights.append(osf)
|
||||
seen_hex.add(h.lower().strip())
|
||||
|
||||
# Publish OpenSky-merged data immediately so users see flights even if
|
||||
# supplemental gap-fill is slow or rate-limited (airplanes.live can take
|
||||
# 100+ seconds when its regional endpoints are throttled).
|
||||
if len(all_flights) > len(adsb_flights):
|
||||
logger.info(
|
||||
f"OpenSky merge: {len(all_flights) - len(adsb_flights)} additional aircraft, "
|
||||
"publishing before supplemental gap-fill"
|
||||
)
|
||||
_classify_and_publish(all_flights)
|
||||
|
||||
# Supplemental gap-fill
|
||||
try:
|
||||
gap_fill = _fetch_supplemental_sources(seen_hex)
|
||||
@@ -1008,14 +998,18 @@ def fetch_flights():
|
||||
if adsb_flights:
|
||||
logger.info(f"adsb.lol: {len(adsb_flights)} aircraft — publishing immediately")
|
||||
_classify_and_publish(adsb_flights)
|
||||
|
||||
# Phase 2: kick off slow enrichment in background
|
||||
threading.Thread(
|
||||
target=_enrich_with_opensky_and_supplemental,
|
||||
args=(adsb_flights,),
|
||||
daemon=True,
|
||||
).start()
|
||||
else:
|
||||
logger.warning("adsb.lol returned 0 aircraft")
|
||||
logger.warning(
|
||||
"adsb.lol returned 0 aircraft — relying on OpenSky/supplemental sources"
|
||||
)
|
||||
|
||||
# Phase 2: always run — OpenSky is the fallback when adsb.lol blocks us
|
||||
# (it has been known to 451 the bulk regional endpoint), and supplemental
|
||||
# gap-fill should always run regardless of Phase 1 success.
|
||||
threading.Thread(
|
||||
target=_enrich_with_opensky_and_supplemental,
|
||||
args=(adsb_flights,),
|
||||
daemon=True,
|
||||
).start()
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching flights: {e}")
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
"""Ship and geopolitics fetchers — AIS vessels, carriers, frontlines, GDELT, LiveUAmap, fishing."""
|
||||
|
||||
import csv
|
||||
import concurrent.futures
|
||||
import io
|
||||
import math
|
||||
import os
|
||||
import logging
|
||||
import time
|
||||
from urllib.parse import urlencode
|
||||
from services.network_utils import fetch_with_curl
|
||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
||||
from services.fetchers.retry import with_retry
|
||||
@@ -27,20 +30,24 @@ def fetch_ships():
|
||||
from services.ais_stream import get_ais_vessels
|
||||
from services.carrier_tracker import get_carrier_positions
|
||||
|
||||
ships = []
|
||||
try:
|
||||
carriers = get_carrier_positions()
|
||||
ships.extend(carriers)
|
||||
except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
|
||||
logger.error(f"Carrier tracker error (non-fatal): {e}")
|
||||
carriers = []
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=2, thread_name_prefix="ship_fetch") as executor:
|
||||
carrier_future = executor.submit(get_carrier_positions)
|
||||
ais_future = executor.submit(get_ais_vessels)
|
||||
|
||||
try:
|
||||
ais_vessels = get_ais_vessels()
|
||||
ships.extend(ais_vessels)
|
||||
except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
|
||||
logger.error(f"AIS stream error (non-fatal): {e}")
|
||||
ais_vessels = []
|
||||
try:
|
||||
carriers = carrier_future.result()
|
||||
except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
|
||||
logger.error(f"Carrier tracker error (non-fatal): {e}")
|
||||
carriers = []
|
||||
|
||||
try:
|
||||
ais_vessels = ais_future.result()
|
||||
except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
|
||||
logger.error(f"AIS stream error (non-fatal): {e}")
|
||||
ais_vessels = []
|
||||
|
||||
ships = list(carriers or [])
|
||||
ships.extend(ais_vessels or [])
|
||||
|
||||
# Enrich ships with yacht alert data (tracked superyachts)
|
||||
from services.fetchers.yacht_alert import enrich_with_yacht_alert
|
||||
@@ -200,52 +207,177 @@ def update_liveuamap():
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fishing Activity (Global Fishing Watch)
|
||||
# ---------------------------------------------------------------------------
|
||||
def _fishing_vessel_key(event: dict) -> str:
|
||||
vessel_ssvid = str(event.get("vessel_ssvid", "") or "").strip()
|
||||
if vessel_ssvid:
|
||||
return f"ssvid:{vessel_ssvid}"
|
||||
vessel_id = str(event.get("vessel_id", "") or "").strip()
|
||||
if vessel_id:
|
||||
return f"vid:{vessel_id}"
|
||||
vessel_name = str(event.get("vessel_name", "") or "").strip().upper()
|
||||
vessel_flag = str(event.get("vessel_flag", "") or "").strip().upper()
|
||||
if vessel_name:
|
||||
return f"name:{vessel_name}|flag:{vessel_flag}"
|
||||
return f"event:{event.get('id', '')}"
|
||||
|
||||
|
||||
def _fishing_event_rank(event: dict) -> tuple[str, str, float, str]:
|
||||
return (
|
||||
str(event.get("end", "") or ""),
|
||||
str(event.get("start", "") or ""),
|
||||
float(event.get("duration_hrs", 0) or 0),
|
||||
str(event.get("id", "") or ""),
|
||||
)
|
||||
|
||||
|
||||
def _dedupe_fishing_events(events: list[dict]) -> list[dict]:
|
||||
latest_by_vessel: dict[str, dict] = {}
|
||||
counts_by_vessel: dict[str, int] = {}
|
||||
|
||||
for event in events:
|
||||
vessel_key = _fishing_vessel_key(event)
|
||||
counts_by_vessel[vessel_key] = counts_by_vessel.get(vessel_key, 0) + 1
|
||||
current = latest_by_vessel.get(vessel_key)
|
||||
if current is None or _fishing_event_rank(event) > _fishing_event_rank(current):
|
||||
latest_by_vessel[vessel_key] = event
|
||||
|
||||
deduped: list[dict] = []
|
||||
for vessel_key, event in latest_by_vessel.items():
|
||||
event_copy = dict(event)
|
||||
event_copy["event_count"] = counts_by_vessel.get(vessel_key, 1)
|
||||
deduped.append(event_copy)
|
||||
|
||||
deduped.sort(key=_fishing_event_rank, reverse=True)
|
||||
return deduped
|
||||
|
||||
|
||||
_FISHING_FETCH_INTERVAL_S = 3600 # once per hour — GFW data has ~5 day lag
|
||||
_last_fishing_fetch_ts: float = 0.0
|
||||
|
||||
|
||||
@with_retry(max_retries=1, base_delay=5)
|
||||
def fetch_fishing_activity():
|
||||
"""Fetch recent fishing events from Global Fishing Watch (~5 day lag)."""
|
||||
from services.fetchers._store import is_any_active
|
||||
global _last_fishing_fetch_ts
|
||||
from services.fetchers._store import is_any_active, latest_data
|
||||
|
||||
if not is_any_active("fishing_activity"):
|
||||
return
|
||||
|
||||
# Skip if we already have data and fetched less than an hour ago
|
||||
now = time.time()
|
||||
if latest_data.get("fishing_activity") and (now - _last_fishing_fetch_ts) < _FISHING_FETCH_INTERVAL_S:
|
||||
return
|
||||
|
||||
token = os.environ.get("GFW_API_TOKEN", "")
|
||||
if not token:
|
||||
logger.debug("GFW_API_TOKEN not set, skipping fishing activity fetch")
|
||||
return
|
||||
events = []
|
||||
try:
|
||||
url = (
|
||||
"https://gateway.api.globalfishingwatch.org/v3/events"
|
||||
"?datasets[0]=public-global-fishing-events:latest"
|
||||
"&limit=500&sort=start&sort-direction=DESC"
|
||||
)
|
||||
import datetime as _dt
|
||||
|
||||
_end = _dt.date.today().isoformat()
|
||||
_start = (_dt.date.today() - _dt.timedelta(days=7)).isoformat()
|
||||
page_size = max(1, int(os.environ.get("GFW_EVENTS_PAGE_SIZE", "500") or "500"))
|
||||
offset = 0
|
||||
seen_offsets: set[int] = set()
|
||||
seen_ids: set[str] = set()
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
response = fetch_with_curl(url, timeout=30, headers=headers)
|
||||
if response.status_code == 200:
|
||||
entries = response.json().get("entries", [])
|
||||
|
||||
while True:
|
||||
if offset in seen_offsets:
|
||||
logger.warning("Fishing activity pagination repeated offset=%s; stopping fetch", offset)
|
||||
break
|
||||
seen_offsets.add(offset)
|
||||
|
||||
query = urlencode(
|
||||
{
|
||||
"datasets[0]": "public-global-fishing-events:latest",
|
||||
"start-date": _start,
|
||||
"end-date": _end,
|
||||
"limit": page_size,
|
||||
"offset": offset,
|
||||
}
|
||||
)
|
||||
url = f"https://gateway.api.globalfishingwatch.org/v3/events?{query}"
|
||||
response = fetch_with_curl(url, timeout=30, headers=headers)
|
||||
if response.status_code != 200:
|
||||
logger.warning(
|
||||
"Fishing activity fetch failed at offset=%s: HTTP %s",
|
||||
offset,
|
||||
response.status_code,
|
||||
)
|
||||
break
|
||||
|
||||
payload = response.json() or {}
|
||||
entries = payload.get("entries", [])
|
||||
if not entries:
|
||||
break
|
||||
|
||||
added_this_page = 0
|
||||
for e in entries:
|
||||
pos = e.get("position", {})
|
||||
vessel = e.get("vessel") or {}
|
||||
lat = pos.get("lat")
|
||||
lng = pos.get("lon")
|
||||
if lat is None or lng is None:
|
||||
continue
|
||||
event_id = str(e.get("id", "") or "")
|
||||
if event_id and event_id in seen_ids:
|
||||
continue
|
||||
if event_id:
|
||||
seen_ids.add(event_id)
|
||||
dur = e.get("event", {}).get("duration", 0) or 0
|
||||
events.append(
|
||||
{
|
||||
"id": e.get("id", ""),
|
||||
"id": event_id,
|
||||
"type": e.get("type", "fishing"),
|
||||
"lat": lat,
|
||||
"lng": lng,
|
||||
"start": e.get("start", ""),
|
||||
"end": e.get("end", ""),
|
||||
"vessel_name": (e.get("vessel") or {}).get("name", "Unknown"),
|
||||
"vessel_flag": (e.get("vessel") or {}).get("flag", ""),
|
||||
"vessel_id": str(vessel.get("id", "") or ""),
|
||||
"vessel_ssvid": str(vessel.get("ssvid", "") or ""),
|
||||
"vessel_name": vessel.get("name", "Unknown"),
|
||||
"vessel_flag": vessel.get("flag", ""),
|
||||
"duration_hrs": round(dur / 3600, 1),
|
||||
}
|
||||
)
|
||||
logger.info(f"Fishing activity: {len(events)} events")
|
||||
added_this_page += 1
|
||||
|
||||
if len(entries) < page_size:
|
||||
break
|
||||
|
||||
next_offset = payload.get("nextOffset")
|
||||
if next_offset is None:
|
||||
next_offset = (payload.get("pagination") or {}).get("nextOffset")
|
||||
if next_offset is None:
|
||||
next_offset = offset + page_size
|
||||
try:
|
||||
next_offset = int(next_offset)
|
||||
except (TypeError, ValueError):
|
||||
next_offset = offset + page_size
|
||||
if next_offset <= offset:
|
||||
logger.warning(
|
||||
"Fishing activity pagination produced non-increasing next offset=%s; stopping fetch",
|
||||
next_offset,
|
||||
)
|
||||
break
|
||||
if added_this_page == 0:
|
||||
logger.warning(
|
||||
"Fishing activity page at offset=%s added no new events; stopping fetch",
|
||||
offset,
|
||||
)
|
||||
break
|
||||
offset = next_offset
|
||||
raw_event_count = len(events)
|
||||
events = _dedupe_fishing_events(events)
|
||||
logger.info("Fishing activity: %s raw events -> %s deduped vessels", raw_event_count, len(events))
|
||||
except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e:
|
||||
logger.error(f"Error fetching fishing activity: {e}")
|
||||
with _data_lock:
|
||||
latest_data["fishing_activity"] = events
|
||||
if events:
|
||||
_mark_fresh("fishing_activity")
|
||||
_last_fishing_fetch_ts = time.time()
|
||||
|
||||
@@ -25,7 +25,10 @@ logger = logging.getLogger("services.data_fetcher")
|
||||
_API_URL = "https://meshtastic.liamcottle.net/api/v1/nodes"
|
||||
_CACHE_FILE = Path(__file__).resolve().parent.parent.parent / "data" / "meshtastic_nodes_cache.json"
|
||||
_FETCH_TIMEOUT = 90 # seconds — response is ~37MB, needs time on slow connections
|
||||
_MAX_AGE_HOURS = 4 # discard nodes not seen within this window (matches refresh interval)
|
||||
_MAX_AGE_HOURS = 24 # discard nodes not seen within this window
|
||||
# Skip network fetch if cached data is fresher than this — the API is a
|
||||
# one-person hobby service, so we prefer stale data over hammering it.
|
||||
_CACHE_TRUST_HOURS = 20
|
||||
|
||||
# Track when we last fetched so the frontend can show staleness
|
||||
_last_fetch_ts: float = 0.0
|
||||
@@ -141,13 +144,54 @@ def fetch_meshtastic_nodes():
|
||||
return
|
||||
global _last_fetch_ts
|
||||
|
||||
# Trust a recent cache on disk — avoids hammering the upstream HTTP API
|
||||
# when every install polls on roughly the same cadence.
|
||||
try:
|
||||
if _CACHE_FILE.exists():
|
||||
mtime = _CACHE_FILE.stat().st_mtime
|
||||
if time.time() - mtime < _CACHE_TRUST_HOURS * 3600:
|
||||
# If memory is empty (cold start), hydrate from cache and skip fetch.
|
||||
with _data_lock:
|
||||
has_memory = bool(latest_data.get("meshtastic_map_nodes"))
|
||||
if not has_memory:
|
||||
cached = _load_cache()
|
||||
if cached:
|
||||
with _data_lock:
|
||||
latest_data["meshtastic_map_nodes"] = cached
|
||||
latest_data["meshtastic_map_fetched_at"] = mtime
|
||||
_mark_fresh("meshtastic_map")
|
||||
logger.info(
|
||||
"Meshtastic map: cache fresh (<%.0fh), skipping network fetch",
|
||||
_CACHE_TRUST_HOURS,
|
||||
)
|
||||
return
|
||||
else:
|
||||
logger.info(
|
||||
"Meshtastic map: cache fresh (<%.0fh), skipping network fetch",
|
||||
_CACHE_TRUST_HOURS,
|
||||
)
|
||||
return
|
||||
except Exception as e:
|
||||
logger.debug(f"Meshtastic cache freshness check failed: {e}")
|
||||
|
||||
# Build a polite User-Agent. Include the operator callsign when set so
|
||||
# the upstream service can correlate per-install traffic if needed.
|
||||
try:
|
||||
from services.config import get_settings
|
||||
|
||||
callsign = str(getattr(get_settings(), "MESHTASTIC_OPERATOR_CALLSIGN", "") or "").strip()
|
||||
except Exception:
|
||||
callsign = ""
|
||||
ua_base = "ShadowBroker-OSINT/0.9.7 (+https://github.com/BigBodyCobain/Shadowbroker; contact: bigbodycobain@gmail.com; 24h polling)"
|
||||
user_agent = f"{ua_base}; node={callsign}" if callsign else ua_base
|
||||
|
||||
try:
|
||||
logger.info("Fetching Meshtastic map nodes from API...")
|
||||
resp = requests.get(
|
||||
_API_URL,
|
||||
timeout=_FETCH_TIMEOUT,
|
||||
headers={
|
||||
"User-Agent": "ShadowBroker/1.0 (OSINT dashboard, 4h polling)",
|
||||
"User-Agent": user_agent,
|
||||
"Accept": "application/json",
|
||||
},
|
||||
)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import requests
|
||||
from services.network_utils import fetch_with_curl
|
||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
||||
@@ -296,17 +297,23 @@ def fetch_military_flights():
|
||||
with _data_lock:
|
||||
latest_data["military_flights"] = remaining_mil
|
||||
|
||||
# Store tracked military flights — update positions for existing entries
|
||||
# Store tracked military flights — update positions for existing entries.
|
||||
# Drop stale entries not refreshed by ANY source (civilian or military) within 5 min.
|
||||
_TRACKED_STALE_S = 300 # 5 minutes
|
||||
_merge_ts = time.time()
|
||||
|
||||
with _data_lock:
|
||||
existing_tracked = list(latest_data.get("tracked_flights", []))
|
||||
fresh_mil_map = {}
|
||||
for t in tracked_mil:
|
||||
icao = t.get("icao24", "").upper()
|
||||
if icao:
|
||||
t["_seen_at"] = _merge_ts
|
||||
fresh_mil_map[icao] = t
|
||||
|
||||
updated_tracked = []
|
||||
seen_icaos = set()
|
||||
stale_dropped = 0
|
||||
for old_t in existing_tracked:
|
||||
icao = old_t.get("icao24", "").upper()
|
||||
if icao in fresh_mil_map:
|
||||
@@ -317,11 +324,16 @@ def fetch_military_flights():
|
||||
updated_tracked.append(fresh)
|
||||
seen_icaos.add(icao)
|
||||
else:
|
||||
updated_tracked.append(old_t)
|
||||
seen_icaos.add(icao)
|
||||
# Keep stale entry only if it was seen recently
|
||||
age = _merge_ts - old_t.get("_seen_at", 0)
|
||||
if age < _TRACKED_STALE_S:
|
||||
updated_tracked.append(old_t)
|
||||
seen_icaos.add(icao)
|
||||
else:
|
||||
stale_dropped += 1
|
||||
for icao, t in fresh_mil_map.items():
|
||||
if icao not in seen_icaos:
|
||||
updated_tracked.append(t)
|
||||
with _data_lock:
|
||||
latest_data["tracked_flights"] = updated_tracked
|
||||
logger.info(f"Tracked flights: {len(updated_tracked)} total ({len(tracked_mil)} from military)")
|
||||
logger.info(f"Tracked flights: {len(updated_tracked)} total ({len(tracked_mil)} from military, {stale_dropped} stale dropped)")
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
"""News fetching, geocoding, clustering, and risk assessment."""
|
||||
import re
|
||||
import time
|
||||
import logging
|
||||
import calendar
|
||||
import concurrent.futures
|
||||
import requests
|
||||
import feedparser
|
||||
@@ -11,6 +13,10 @@ from services.oracle_service import enrich_news_items, compute_global_threat_lev
|
||||
|
||||
logger = logging.getLogger("services.data_fetcher")
|
||||
|
||||
# Maximum article age in seconds. Anything older than this is dropped
|
||||
# during each fetch cycle so the threat feed stays current.
|
||||
_MAX_ARTICLE_AGE_SECS = 48 * 3600 # 48 hours
|
||||
|
||||
|
||||
# Keyword -> coordinate mapping for geocoding news articles
|
||||
_KEYWORD_COORDS = {
|
||||
@@ -178,6 +184,17 @@ def fetch_news():
|
||||
if not feed:
|
||||
continue
|
||||
for entry in feed.entries[:5]:
|
||||
# Drop articles older than the max-age threshold so the
|
||||
# threat feed doesn't show stale stories across cycles.
|
||||
pp = entry.get("published_parsed")
|
||||
if pp:
|
||||
try:
|
||||
entry_epoch = calendar.timegm(pp)
|
||||
if time.time() - entry_epoch > _MAX_ARTICLE_AGE_SECS:
|
||||
continue
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
pass # unparseable date — keep the article
|
||||
|
||||
title = entry.get('title', '')
|
||||
summary = entry.get('summary', '')
|
||||
|
||||
|
||||
@@ -0,0 +1,360 @@
|
||||
"""NUFORC Enrichment — downloads the Hugging Face NUFORC dataset and builds
|
||||
a compact spatial+temporal index for enriching tilequery hits with shape,
|
||||
duration, city, and summary text.
|
||||
|
||||
The full CSV (~170 MB) is streamed once and processed into a lightweight JSON
|
||||
cache (~1-3 MB) stored at ``backend/data/nuforc_enrichment.json``. Subsequent
|
||||
startups load from cache until it expires (30 days).
|
||||
|
||||
Index structure::
|
||||
|
||||
{
|
||||
"built": "2026-04-08T12:00:00",
|
||||
"count": 12345,
|
||||
"by_state": {
|
||||
"AZ": [
|
||||
{"d": "2024-01-15", "city": "Tucson", "shape": "triangle",
|
||||
"dur": "5 minutes", "summary": "Bright triangular object..."},
|
||||
...
|
||||
],
|
||||
...
|
||||
}
|
||||
}
|
||||
|
||||
Entries within each state are sorted by date descending (newest first).
|
||||
"""
|
||||
|
||||
import csv
|
||||
import gzip
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_DATA_DIR = Path(__file__).resolve().parent.parent.parent / "data"
|
||||
_CACHE_FILE = _DATA_DIR / "nuforc_enrichment.json"
|
||||
_CACHE_TTL_DAYS = 1 # Rebuild daily — fresh data each cycle
|
||||
|
||||
# HuggingFace dataset — use the structured string export, not the old flat blob.
|
||||
_HF_CSV_URL = (
|
||||
"https://huggingface.co/datasets/kcimc/NUFORC/resolve/main/nuforc_str.csv"
|
||||
)
|
||||
|
||||
# Only keep sightings from the last N years for the enrichment index
|
||||
_KEEP_YEARS = 5
|
||||
|
||||
# ── In-memory index ────────────────────────────────────────────────────────
|
||||
_index: dict | None = None
|
||||
_index_lock = threading.Lock()
|
||||
_building = False
|
||||
|
||||
# US state abbreviations for parsing "City, ST" locations
|
||||
_US_STATES = {
|
||||
"AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DE", "FL", "GA",
|
||||
"HI", "ID", "IL", "IN", "IA", "KS", "KY", "LA", "ME", "MD",
|
||||
"MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ",
|
||||
"NM", "NY", "NC", "ND", "OH", "OK", "OR", "PA", "RI", "SC",
|
||||
"SD", "TN", "TX", "UT", "VT", "VA", "WA", "WV", "WI", "WY",
|
||||
"DC",
|
||||
}
|
||||
|
||||
|
||||
def _parse_location(loc: str) -> tuple[str, str]:
|
||||
"""Parse 'City, ST' or 'City, ST (explanation)' → (city, state_abbr).
|
||||
|
||||
Returns ('', '') if unparseable.
|
||||
"""
|
||||
if not loc:
|
||||
return "", ""
|
||||
loc = re.sub(r"\s*\(.*\)\s*$", "", loc).strip()
|
||||
parts = [p.strip() for p in loc.split(",") if p.strip()]
|
||||
if len(parts) < 2:
|
||||
return "", ""
|
||||
for idx in range(len(parts) - 1):
|
||||
candidate = parts[idx + 1].upper().strip()
|
||||
if candidate in _US_STATES:
|
||||
city = ", ".join(parts[: idx + 1]).strip()
|
||||
return city, candidate
|
||||
candidate = parts[-1].upper().strip()
|
||||
if candidate in _US_STATES:
|
||||
return ", ".join(parts[:-1]).strip(), candidate
|
||||
return parts[0], ""
|
||||
|
||||
|
||||
def _parse_date(date_str: str) -> str:
|
||||
"""Best-effort parse NUFORC date strings → 'YYYY-MM-DD'.
|
||||
|
||||
Returns '' on failure.
|
||||
"""
|
||||
if not date_str:
|
||||
return ""
|
||||
cleaned = str(date_str).strip()
|
||||
cleaned = re.sub(r"\s+local$", "", cleaned, flags=re.IGNORECASE)
|
||||
cleaned = re.sub(r"\s+utc$", "", cleaned, flags=re.IGNORECASE)
|
||||
cleaned = cleaned.replace("T", " ")
|
||||
for fmt in (
|
||||
"%m/%d/%Y %H:%M",
|
||||
"%m/%d/%Y %I:%M:%S %p",
|
||||
"%m/%d/%Y",
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
"%Y-%m-%d %H:%M",
|
||||
"%Y-%m-%d",
|
||||
):
|
||||
try:
|
||||
return datetime.strptime(cleaned, fmt).strftime("%Y-%m-%d")
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
match = re.match(r"^(\d{4}-\d{2}-\d{2})", cleaned)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return ""
|
||||
|
||||
|
||||
def _load_cache() -> dict | None:
|
||||
"""Load the on-disk cache if it exists and is fresh enough."""
|
||||
if not _CACHE_FILE.exists():
|
||||
return None
|
||||
try:
|
||||
raw = _CACHE_FILE.read_text(encoding="utf-8")
|
||||
data = json.loads(raw)
|
||||
built = data.get("built", "")
|
||||
if built:
|
||||
built_dt = datetime.fromisoformat(built)
|
||||
if datetime.utcnow() - built_dt < timedelta(days=_CACHE_TTL_DAYS):
|
||||
if int(data.get("count", 0) or 0) <= 0:
|
||||
logger.info("NUFORC enrichment: cache is fresh but empty; rebuilding")
|
||||
return None
|
||||
logger.info(
|
||||
"NUFORC enrichment: loaded cache (%d entries, built %s)",
|
||||
data.get("count", 0), built,
|
||||
)
|
||||
return data
|
||||
else:
|
||||
logger.info("NUFORC enrichment: cache expired (built %s)", built)
|
||||
except Exception as e:
|
||||
logger.warning("NUFORC enrichment: cache load error: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
def _save_cache(data: dict):
|
||||
"""Persist the enrichment index to disk."""
|
||||
try:
|
||||
_DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
_CACHE_FILE.write_text(json.dumps(data, separators=(",", ":")), encoding="utf-8")
|
||||
logger.info("NUFORC enrichment: saved cache (%d entries)", data.get("count", 0))
|
||||
except Exception as e:
|
||||
logger.warning("NUFORC enrichment: cache save error: %s", e)
|
||||
|
||||
|
||||
def _download_and_build() -> dict | None:
|
||||
"""Stream-download the HF CSV and build the enrichment index.
|
||||
|
||||
Returns the index dict or None on failure.
|
||||
"""
|
||||
cutoff = datetime.utcnow() - timedelta(days=_KEEP_YEARS * 365)
|
||||
cutoff_str = cutoff.strftime("%Y-%m-%d")
|
||||
|
||||
logger.info("NUFORC enrichment: downloading HF dataset (this may take a minute)...")
|
||||
try:
|
||||
resp = fetch_with_curl(_HF_CSV_URL, timeout=180, follow_redirects=True)
|
||||
if not resp or resp.status_code != 200:
|
||||
logger.warning(
|
||||
"NUFORC enrichment: download failed HTTP %s",
|
||||
getattr(resp, "status_code", "None"),
|
||||
)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error("NUFORC enrichment: download error: %s", e)
|
||||
return None
|
||||
|
||||
# Parse CSV from response text
|
||||
by_state: dict[str, list[dict]] = {}
|
||||
total = 0
|
||||
kept = 0
|
||||
|
||||
try:
|
||||
reader = csv.DictReader(io.StringIO(resp.text))
|
||||
for row in reader:
|
||||
total += 1
|
||||
occurred = _parse_date(
|
||||
row.get("Occurred", "")
|
||||
or row.get("Date / Time", "")
|
||||
or row.get("Date", "")
|
||||
)
|
||||
if not occurred or occurred < cutoff_str:
|
||||
continue
|
||||
|
||||
city, state = _parse_location(
|
||||
row.get("Location", "")
|
||||
or row.get("City", "")
|
||||
or row.get("location", "")
|
||||
)
|
||||
if not state:
|
||||
continue # can't index without state
|
||||
|
||||
shape = (row.get("Shape", "") or row.get("shape", "") or "").strip()
|
||||
duration = (row.get("Duration", "") or row.get("duration", "") or "").strip()
|
||||
summary = (
|
||||
row.get("Summary", "")
|
||||
or row.get("summary", "")
|
||||
or row.get("Text", "")
|
||||
or row.get("text", "")
|
||||
or ""
|
||||
).strip()
|
||||
if summary and len(summary) > 200:
|
||||
summary = summary[:197] + "..."
|
||||
|
||||
entry = {"d": occurred, "city": city, "shape": shape}
|
||||
if duration:
|
||||
entry["dur"] = duration
|
||||
if summary:
|
||||
entry["sum"] = summary
|
||||
|
||||
by_state.setdefault(state, []).append(entry)
|
||||
kept += 1
|
||||
except Exception as e:
|
||||
logger.error("NUFORC enrichment: CSV parse error: %s", e)
|
||||
return None
|
||||
|
||||
# Sort each state's entries by date descending (newest first)
|
||||
for st in by_state:
|
||||
by_state[st].sort(key=lambda e: e["d"], reverse=True)
|
||||
|
||||
data = {
|
||||
"built": datetime.utcnow().isoformat(),
|
||||
"count": kept,
|
||||
"by_state": by_state,
|
||||
}
|
||||
logger.info(
|
||||
"NUFORC enrichment: built index — %d entries from %d total rows (%d states)",
|
||||
kept, total, len(by_state),
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
def _ensure_index():
|
||||
"""Load or build the enrichment index (thread-safe, non-blocking)."""
|
||||
global _index, _building
|
||||
|
||||
with _index_lock:
|
||||
if _index is not None:
|
||||
return
|
||||
if _building:
|
||||
return # another thread is already building
|
||||
_building = True
|
||||
|
||||
# Try loading from disk first
|
||||
cached = _load_cache()
|
||||
if cached:
|
||||
with _index_lock:
|
||||
_index = cached
|
||||
_building = False
|
||||
return
|
||||
|
||||
# Download and build in background so we don't block startup
|
||||
def _build():
|
||||
global _index, _building
|
||||
try:
|
||||
result = _download_and_build()
|
||||
if result:
|
||||
_save_cache(result)
|
||||
with _index_lock:
|
||||
_index = result
|
||||
else:
|
||||
logger.warning("NUFORC enrichment: build failed, enrichment unavailable")
|
||||
finally:
|
||||
with _index_lock:
|
||||
_building = False
|
||||
|
||||
thread = threading.Thread(target=_build, name="nuforc-enrichment", daemon=True)
|
||||
thread.start()
|
||||
|
||||
|
||||
def refresh_enrichment_index():
|
||||
"""Force-rebuild the enrichment index. Called by the daily cron job.
|
||||
|
||||
Downloads the latest HF CSV, rebuilds the in-memory + disk cache.
|
||||
Runs synchronously (meant to be called from a background thread).
|
||||
"""
|
||||
global _index
|
||||
logger.info("NUFORC enrichment: daily refresh starting...")
|
||||
result = _download_and_build()
|
||||
if result:
|
||||
_save_cache(result)
|
||||
with _index_lock:
|
||||
_index = result
|
||||
logger.info("NUFORC enrichment: daily refresh complete (%d entries)", result.get("count", 0))
|
||||
else:
|
||||
logger.warning("NUFORC enrichment: daily refresh failed, keeping stale index")
|
||||
|
||||
|
||||
def enrich_sighting(state: str, from_date: str, to_date: str) -> dict:
|
||||
"""Look up enrichment data for a tilequery hit.
|
||||
|
||||
Args:
|
||||
state: 2-letter US state code (from reverse geocode)
|
||||
from_date: earliest sighting date (YYYY-MM-DD)
|
||||
to_date: latest sighting date (YYYY-MM-DD)
|
||||
|
||||
Returns:
|
||||
Dict with optional keys: city, shape, duration, summary.
|
||||
Empty dict if no match found.
|
||||
"""
|
||||
_ensure_index()
|
||||
|
||||
with _index_lock:
|
||||
idx = _index
|
||||
|
||||
if not idx or not state:
|
||||
return {}
|
||||
|
||||
entries = idx.get("by_state", {}).get(state, [])
|
||||
if not entries:
|
||||
return {}
|
||||
|
||||
# Find the best match by date proximity
|
||||
target = to_date or from_date
|
||||
if not target:
|
||||
# No date filter — just return the most recent entry for this state
|
||||
e = entries[0]
|
||||
else:
|
||||
best = None
|
||||
best_dist = 999999
|
||||
for e in entries:
|
||||
# Simple string distance on dates (YYYY-MM-DD sorts lexicographically)
|
||||
try:
|
||||
t = datetime.strptime(target, "%Y-%m-%d")
|
||||
d = datetime.strptime(e["d"], "%Y-%m-%d")
|
||||
dist = abs((t - d).days)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if dist < best_dist:
|
||||
best_dist = dist
|
||||
best = e
|
||||
if dist == 0:
|
||||
break # exact date match
|
||||
|
||||
if best is None or best_dist > 90:
|
||||
return {} # no match within 3 months
|
||||
e = best
|
||||
|
||||
result = {}
|
||||
if e.get("city"):
|
||||
result["city"] = e["city"]
|
||||
if e.get("shape"):
|
||||
result["shape"] = e["shape"]
|
||||
result["shape_raw"] = e["shape"]
|
||||
if e.get("dur"):
|
||||
result["duration"] = e["dur"]
|
||||
if e.get("sum"):
|
||||
result["summary"] = e["sum"]
|
||||
return result
|
||||
@@ -8,14 +8,33 @@ full metadata (volume, end dates, descriptions, source badges).
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import urlencode
|
||||
from cachetools import TTLCache, cached
|
||||
|
||||
logger = logging.getLogger("services.data_fetcher")
|
||||
|
||||
_market_cache = TTLCache(maxsize=1, ttl=60) # 60-second TTL — markets change fast
|
||||
|
||||
# Delta tracking: {market_title: previous_consensus_pct}
|
||||
_prev_probabilities: dict[str, float] = {}
|
||||
_market_cache = TTLCache(maxsize=1, ttl=300)
|
||||
_POLYMARKET_PAGE_DELAY_S = float(os.environ.get("MESH_POLYMARKET_PAGE_DELAY_S", "0.02"))
|
||||
_KALSHI_PAGE_DELAY_S = float(os.environ.get("MESH_KALSHI_PAGE_DELAY_S", "0.08"))
|
||||
_provider_pace_lock = threading.Lock()
|
||||
_provider_last_request_at: dict[str, float] = {}
|
||||
|
||||
|
||||
def _pace_provider(provider: str, min_interval_s: float) -> None:
|
||||
if min_interval_s <= 0:
|
||||
return
|
||||
with _provider_pace_lock:
|
||||
now = time.monotonic()
|
||||
wait_s = min_interval_s - (now - _provider_last_request_at.get(provider, 0.0))
|
||||
if wait_s > 0:
|
||||
time.sleep(wait_s)
|
||||
now = time.monotonic()
|
||||
_provider_last_request_at[provider] = now
|
||||
|
||||
|
||||
def _finite_or_none(value):
|
||||
@@ -28,7 +47,7 @@ def _finite_or_none(value):
|
||||
# ---------------------------------------------------------------------------
|
||||
# Category classification
|
||||
# ---------------------------------------------------------------------------
|
||||
CATEGORIES = ["POLITICS", "CONFLICT", "NEWS", "FINANCE", "CRYPTO"]
|
||||
CATEGORIES = ["POLITICS", "CONFLICT", "NEWS", "FINANCE", "CRYPTO", "SPORTS"]
|
||||
|
||||
_KALSHI_CATEGORY_MAP = {
|
||||
"Politics": "POLITICS",
|
||||
@@ -38,7 +57,7 @@ _KALSHI_CATEGORY_MAP = {
|
||||
"Tech": "FINANCE",
|
||||
"Science": "NEWS",
|
||||
"Climate and Weather": "NEWS",
|
||||
"Sports": "NEWS",
|
||||
"Sports": "SPORTS",
|
||||
"Culture": "NEWS",
|
||||
}
|
||||
|
||||
@@ -62,7 +81,14 @@ _TAG_CATEGORY_MAP = {
|
||||
"Ethereum": "CRYPTO",
|
||||
"AI": "NEWS",
|
||||
"Science": "NEWS",
|
||||
"Sports": "NEWS",
|
||||
"Sports": "SPORTS",
|
||||
"NBA": "SPORTS",
|
||||
"NFL": "SPORTS",
|
||||
"MLB": "SPORTS",
|
||||
"NHL": "SPORTS",
|
||||
"Soccer": "SPORTS",
|
||||
"Tennis": "SPORTS",
|
||||
"Golf": "SPORTS",
|
||||
"Culture": "NEWS",
|
||||
"Entertainment": "NEWS",
|
||||
"Tech": "FINANCE",
|
||||
@@ -152,6 +178,26 @@ _KEYWORD_CATEGORIES = {
|
||||
"market cap",
|
||||
"revenue",
|
||||
],
|
||||
"SPORTS": [
|
||||
"nba",
|
||||
"nfl",
|
||||
"mlb",
|
||||
"nhl",
|
||||
"wnba",
|
||||
"soccer",
|
||||
"football",
|
||||
"basketball",
|
||||
"baseball",
|
||||
"hockey",
|
||||
"ufc",
|
||||
"mma",
|
||||
"tennis",
|
||||
"golf",
|
||||
"championship",
|
||||
"playoffs",
|
||||
"world cup",
|
||||
"super bowl",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@@ -177,21 +223,186 @@ def _classify_category(title: str, poly_tags: list[str], kalshi_category: str) -
|
||||
return "NEWS"
|
||||
|
||||
|
||||
def _polymarket_event_to_entry(ev: dict) -> dict | None:
|
||||
title = ev.get("title", "")
|
||||
if not title:
|
||||
return None
|
||||
|
||||
markets = ev.get("markets", [])
|
||||
best_pct = None
|
||||
total_volume = 0
|
||||
outcomes = []
|
||||
for m in markets:
|
||||
raw_op = m.get("outcomePrices")
|
||||
price = None
|
||||
try:
|
||||
op = json.loads(raw_op) if isinstance(raw_op, str) else raw_op
|
||||
if isinstance(op, list) and len(op) >= 1:
|
||||
price = _finite_or_none(op[0])
|
||||
except (json.JSONDecodeError, ValueError, TypeError):
|
||||
pass
|
||||
if price is None:
|
||||
price = _finite_or_none(m.get("lastTradePrice") or m.get("bestBid"))
|
||||
pct = None
|
||||
if price is not None:
|
||||
try:
|
||||
pct = round(price * 100, 1)
|
||||
if best_pct is None or pct > best_pct:
|
||||
best_pct = pct
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
volume = _finite_or_none(m.get("volume", 0) or 0)
|
||||
if volume is not None:
|
||||
total_volume += volume
|
||||
oname = m.get("groupItemTitle") or ""
|
||||
if oname and pct is not None:
|
||||
outcomes.append({"name": oname, "pct": pct})
|
||||
|
||||
if len(outcomes) > 2:
|
||||
outcomes.sort(key=lambda x: x["pct"], reverse=True)
|
||||
else:
|
||||
outcomes = []
|
||||
|
||||
tag_labels = [t.get("label", "") for t in ev.get("tags", []) if t.get("label")]
|
||||
return {
|
||||
"title": title,
|
||||
"source": "polymarket",
|
||||
"pct": best_pct,
|
||||
"slug": ev.get("slug", ""),
|
||||
"description": ev.get("description") or "",
|
||||
"end_date": ev.get("endDate"),
|
||||
"volume": round(total_volume, 2),
|
||||
"volume_24h": round(_finite_or_none(ev.get("volume24hr", 0) or 0) or 0, 2),
|
||||
"tags": tag_labels,
|
||||
"outcomes": outcomes,
|
||||
}
|
||||
|
||||
|
||||
def _kalshi_market_pct(m: dict) -> float | None:
|
||||
bid = _finite_or_none(m.get("yes_bid_dollars"))
|
||||
ask = _finite_or_none(m.get("yes_ask_dollars"))
|
||||
last = _finite_or_none(m.get("last_price_dollars"))
|
||||
if bid is not None and ask is not None and ask >= bid:
|
||||
return round(((bid + ask) / 2) * 100, 1)
|
||||
if last is not None:
|
||||
return round(last * 100, 1)
|
||||
cents = _finite_or_none(m.get("yes_price") or m.get("last_price"))
|
||||
if cents is None:
|
||||
return None
|
||||
return round(cents * 100, 1) if cents <= 1 else round(cents, 1)
|
||||
|
||||
|
||||
def _kalshi_market_volume(m: dict) -> float:
|
||||
for key in ("volume_24h_fp", "volume_fp", "dollar_volume", "volume"):
|
||||
value = _finite_or_none(m.get(key))
|
||||
if value is not None:
|
||||
return value
|
||||
return 0
|
||||
|
||||
|
||||
def _kalshi_market_category(m: dict) -> str:
|
||||
text = " ".join(
|
||||
str(m.get(k, "") or "")
|
||||
for k in ("ticker", "event_ticker", "mve_collection_ticker", "title", "yes_sub_title", "no_sub_title")
|
||||
).lower()
|
||||
if any(token in text for token in ("sports", "xnba", "xnfl", "xmlb", "xnhl", "soccer", "tennis", "golf")):
|
||||
return "Sports"
|
||||
return str(m.get("category", "") or "")
|
||||
|
||||
|
||||
def _kalshi_event_to_entry(ev: dict, markets: list[dict] | None = None) -> dict | None:
|
||||
title = ev.get("title", "")
|
||||
if not title:
|
||||
return None
|
||||
|
||||
markets = markets or ev.get("markets", []) or []
|
||||
best_pct = None
|
||||
total_volume = 0.0
|
||||
close_dates = []
|
||||
outcomes = []
|
||||
first_ticker = ""
|
||||
descriptions = []
|
||||
for m in markets:
|
||||
first_ticker = first_ticker or m.get("ticker", "")
|
||||
pct = _kalshi_market_pct(m)
|
||||
if pct is not None:
|
||||
if best_pct is None or pct > best_pct:
|
||||
best_pct = pct
|
||||
oname = m.get("yes_sub_title") or m.get("sub_title") or m.get("title") or ""
|
||||
if oname and oname != title:
|
||||
outcomes.append({"name": oname, "pct": pct})
|
||||
total_volume += _kalshi_market_volume(m)
|
||||
cd = m.get("close_time") or m.get("close_date") or m.get("expiration_time")
|
||||
if cd:
|
||||
close_dates.append(cd)
|
||||
desc = (m.get("rules_primary") or m.get("rules_secondary") or "").strip()
|
||||
if desc:
|
||||
descriptions.append(desc)
|
||||
|
||||
if len(outcomes) > 2:
|
||||
outcomes.sort(key=lambda x: x["pct"], reverse=True)
|
||||
else:
|
||||
outcomes = []
|
||||
|
||||
desc = (ev.get("settle_details") or ev.get("underlying") or "").strip()
|
||||
if not desc and descriptions:
|
||||
desc = descriptions[0]
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"source": "kalshi",
|
||||
"pct": best_pct,
|
||||
"ticker": first_ticker or ev.get("event_ticker", "") or ev.get("ticker", ""),
|
||||
"description": desc,
|
||||
"sub_title": ev.get("sub_title", ""),
|
||||
"end_date": max(close_dates) if close_dates else None,
|
||||
"volume": round(total_volume, 2),
|
||||
"category": ev.get("category", ""),
|
||||
"outcomes": outcomes,
|
||||
}
|
||||
|
||||
|
||||
def _kalshi_market_to_entry(m: dict) -> dict | None:
|
||||
title = m.get("title") or m.get("yes_sub_title") or ""
|
||||
if not title:
|
||||
return None
|
||||
pct = _kalshi_market_pct(m)
|
||||
volume = _kalshi_market_volume(m)
|
||||
desc = (m.get("rules_primary") or m.get("rules_secondary") or "").strip()
|
||||
end_date = m.get("close_time") or m.get("expiration_time") or m.get("expected_expiration_time")
|
||||
return {
|
||||
"title": title,
|
||||
"source": "kalshi",
|
||||
"pct": pct,
|
||||
"ticker": m.get("ticker", "") or m.get("event_ticker", ""),
|
||||
"description": desc,
|
||||
"sub_title": m.get("subtitle", ""),
|
||||
"end_date": end_date,
|
||||
"volume": round(volume, 2),
|
||||
"category": _kalshi_market_category(m),
|
||||
"outcomes": [],
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Polymarket
|
||||
# ---------------------------------------------------------------------------
|
||||
def _fetch_polymarket_events() -> list[dict]:
|
||||
"""Fetch active events from Polymarket Gamma API (no auth required).
|
||||
|
||||
Fetches up to 500 events (multiple pages) for better search coverage.
|
||||
Fetches paginated active events, bounded by MESH_POLYMARKET_MAX_EVENTS
|
||||
so boot-time refresh does not become unbounded.
|
||||
"""
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
all_events = []
|
||||
for offset in range(0, 500, 100):
|
||||
page_size = 250
|
||||
max_events = int(os.environ.get("MESH_POLYMARKET_MAX_EVENTS", "5000"))
|
||||
for offset in range(0, max_events, page_size):
|
||||
try:
|
||||
_pace_provider("polymarket", _POLYMARKET_PAGE_DELAY_S)
|
||||
resp = fetch_with_curl(
|
||||
f"https://gamma-api.polymarket.com/events?active=true&closed=false&limit=100&offset={offset}",
|
||||
f"https://gamma-api.polymarket.com/events?active=true&closed=false&limit={page_size}&offset={offset}",
|
||||
timeout=15,
|
||||
)
|
||||
if not resp or resp.status_code != 200:
|
||||
@@ -200,6 +411,8 @@ def _fetch_polymarket_events() -> list[dict]:
|
||||
if not isinstance(page, list) or not page:
|
||||
break
|
||||
all_events.extend(page)
|
||||
if len(page) < page_size:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning(f"Polymarket page offset={offset} error: {e}")
|
||||
break
|
||||
@@ -286,6 +499,42 @@ def _fetch_kalshi_events() -> list[dict]:
|
||||
"""Fetch active events from Kalshi public API (no auth required)."""
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
try:
|
||||
max_events = int(os.environ.get("MESH_KALSHI_MAX_EVENTS", "2000"))
|
||||
page_size = 200
|
||||
markets = []
|
||||
cursor = ""
|
||||
while len(markets) < max_events:
|
||||
params = {"status": "open", "limit": str(page_size)}
|
||||
if cursor:
|
||||
params["cursor"] = cursor
|
||||
_pace_provider("kalshi", _KALSHI_PAGE_DELAY_S)
|
||||
resp = fetch_with_curl(
|
||||
f"https://api.elections.kalshi.com/trade-api/v2/markets?{urlencode(params)}",
|
||||
timeout=15,
|
||||
)
|
||||
if not resp or resp.status_code != 200:
|
||||
break
|
||||
data = resp.json()
|
||||
page = data.get("markets", []) if isinstance(data, dict) else []
|
||||
if not page:
|
||||
break
|
||||
markets.extend(page)
|
||||
cursor = data.get("cursor") or ""
|
||||
if not cursor or len(page) < page_size:
|
||||
break
|
||||
|
||||
results = []
|
||||
for market in markets:
|
||||
entry = _kalshi_market_to_entry(market)
|
||||
if entry:
|
||||
results.append(entry)
|
||||
if results:
|
||||
logger.info(f"Kalshi: fetched {len(results)} active events from v2")
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning(f"Kalshi v2 fetch error, falling back to legacy v1: {e}")
|
||||
|
||||
try:
|
||||
resp = fetch_with_curl(
|
||||
"https://api.elections.kalshi.com/v1/events?status=open&limit=100",
|
||||
@@ -540,11 +789,11 @@ def fetch_prediction_markets():
|
||||
# ---------------------------------------------------------------------------
|
||||
# Direct API search (not limited to cached data)
|
||||
# ---------------------------------------------------------------------------
|
||||
def search_polymarket_direct(query: str, limit: int = 20) -> list[dict]:
|
||||
def search_polymarket_direct(query: str, limit: int = 20, offset: int = 0) -> list[dict]:
|
||||
"""Search Polymarket by scanning API pages for title matches.
|
||||
|
||||
The Gamma API has no text search parameter, so we scan cached events
|
||||
plus additional pages until we find enough matches or exhaust the scan.
|
||||
Prefer Polymarket's public search endpoint, then fall back to scanning
|
||||
Gamma event pages if search is unavailable.
|
||||
"""
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
@@ -552,11 +801,53 @@ def search_polymarket_direct(query: str, limit: int = 20) -> list[dict]:
|
||||
q_words = set(q_lower.split())
|
||||
results = []
|
||||
|
||||
try:
|
||||
params = urlencode({"q": query, "limit": str(limit), "offset": str(max(0, offset))})
|
||||
_pace_provider("polymarket", _POLYMARKET_PAGE_DELAY_S)
|
||||
resp = fetch_with_curl(
|
||||
f"https://gamma-api.polymarket.com/public-search?{params}",
|
||||
timeout=15,
|
||||
)
|
||||
if resp and resp.status_code == 200:
|
||||
data = resp.json()
|
||||
events = data.get("events", []) if isinstance(data, dict) else []
|
||||
for ev in events:
|
||||
if ev.get("closed") or ev.get("active") is False:
|
||||
continue
|
||||
entry = _polymarket_event_to_entry(ev)
|
||||
if not entry:
|
||||
continue
|
||||
category = _classify_category(entry["title"], entry.get("tags", []), "")
|
||||
pct = _finite_or_none(entry.get("pct"))
|
||||
sources = [{"name": "POLY", "pct": pct}] if pct is not None else []
|
||||
results.append(
|
||||
{
|
||||
"title": entry["title"],
|
||||
"polymarket_pct": pct,
|
||||
"kalshi_pct": None,
|
||||
"consensus_pct": pct,
|
||||
"description": entry.get("description", ""),
|
||||
"end_date": entry.get("end_date"),
|
||||
"volume": entry.get("volume", 0),
|
||||
"volume_24h": entry.get("volume_24h", 0),
|
||||
"kalshi_volume": 0,
|
||||
"category": category,
|
||||
"sources": sources,
|
||||
"slug": entry.get("slug", ""),
|
||||
"outcomes": entry.get("outcomes", []),
|
||||
}
|
||||
)
|
||||
logger.info(f"Polymarket search '{query}': {len(results)} results via public-search")
|
||||
return results[:limit]
|
||||
except Exception as e:
|
||||
logger.warning(f"Polymarket public-search '{query}' error: {e}")
|
||||
|
||||
# Scan up to 2000 events (10 pages of 200) looking for title matches
|
||||
for offset in range(0, 2000, 200):
|
||||
for scan_offset in range(0, 3000, 200):
|
||||
try:
|
||||
_pace_provider("polymarket", _POLYMARKET_PAGE_DELAY_S)
|
||||
resp = fetch_with_curl(
|
||||
f"https://gamma-api.polymarket.com/events?active=true&closed=false&limit=200&offset={offset}",
|
||||
f"https://gamma-api.polymarket.com/events?active=true&closed=false&limit=200&offset={scan_offset}",
|
||||
timeout=15,
|
||||
)
|
||||
if not resp or resp.status_code != 200:
|
||||
@@ -637,11 +928,168 @@ def search_polymarket_direct(query: str, limit: int = 20) -> list[dict]:
|
||||
}
|
||||
)
|
||||
# Stop scanning if we have enough results
|
||||
if len(results) >= limit:
|
||||
if len(results) >= offset + limit:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning(f"Polymarket search scan offset={offset} error: {e}")
|
||||
logger.warning(f"Polymarket search scan offset={scan_offset} error: {e}")
|
||||
break
|
||||
|
||||
logger.info(f"Polymarket search '{query}': {len(results)} results (scanned API)")
|
||||
return results[:limit]
|
||||
return results[offset : offset + limit]
|
||||
|
||||
|
||||
def search_kalshi_direct(query: str, limit: int = 20, offset: int = 0) -> list[dict]:
|
||||
"""Search Kalshi events by scanning API pages for title matches."""
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
q_lower = query.lower()
|
||||
q_words = set(q_lower.split())
|
||||
results = []
|
||||
|
||||
try:
|
||||
max_scan = int(os.environ.get("MESH_KALSHI_SEARCH_SCAN_EVENTS", "1200"))
|
||||
page_size = 200
|
||||
cursor = ""
|
||||
scanned = 0
|
||||
while scanned < max_scan and len(results) < offset + limit:
|
||||
params = {"status": "open", "limit": str(page_size)}
|
||||
if cursor:
|
||||
params["cursor"] = cursor
|
||||
_pace_provider("kalshi", _KALSHI_PAGE_DELAY_S)
|
||||
resp = fetch_with_curl(
|
||||
f"https://api.elections.kalshi.com/trade-api/v2/markets?{urlencode(params)}",
|
||||
timeout=15,
|
||||
)
|
||||
if not resp or resp.status_code != 200:
|
||||
break
|
||||
data = resp.json()
|
||||
markets = data.get("markets", []) if isinstance(data, dict) else []
|
||||
if not markets:
|
||||
break
|
||||
scanned += len(markets)
|
||||
for market in markets:
|
||||
haystack = " ".join(
|
||||
str(market.get(k, "") or "")
|
||||
for k in ("title", "yes_sub_title", "no_sub_title", "event_ticker", "ticker")
|
||||
).lower()
|
||||
if q_lower not in haystack and not any(w in haystack for w in q_words):
|
||||
continue
|
||||
entry = _kalshi_market_to_entry(market)
|
||||
if not entry:
|
||||
continue
|
||||
pct = _finite_or_none(entry.get("pct"))
|
||||
sources = [{"name": "KALSHI", "pct": pct}] if pct is not None else []
|
||||
category = _classify_category(entry["title"], [], entry.get("category", ""))
|
||||
results.append({
|
||||
"title": entry["title"],
|
||||
"polymarket_pct": None,
|
||||
"kalshi_pct": pct,
|
||||
"consensus_pct": pct,
|
||||
"description": entry.get("description", ""),
|
||||
"end_date": entry.get("end_date"),
|
||||
"volume": 0,
|
||||
"volume_24h": 0,
|
||||
"kalshi_volume": entry.get("volume", 0),
|
||||
"category": category,
|
||||
"sources": sources,
|
||||
"slug": "",
|
||||
"kalshi_ticker": entry.get("ticker", ""),
|
||||
"outcomes": entry.get("outcomes", []),
|
||||
})
|
||||
if len(results) >= offset + limit:
|
||||
break
|
||||
cursor = data.get("cursor") or ""
|
||||
if not cursor or len(markets) < page_size:
|
||||
break
|
||||
if results:
|
||||
logger.info(f"Kalshi search '{query}': {len(results)} results via v2 scan")
|
||||
return results[offset : offset + limit]
|
||||
except Exception as e:
|
||||
logger.warning(f"Kalshi v2 search '{query}' error, falling back to legacy v1: {e}")
|
||||
|
||||
try:
|
||||
resp = fetch_with_curl(
|
||||
"https://api.elections.kalshi.com/v1/events?status=open&limit=200",
|
||||
timeout=15,
|
||||
)
|
||||
if not resp or resp.status_code != 200:
|
||||
return []
|
||||
data = resp.json()
|
||||
events = data.get("events", []) if isinstance(data, dict) else []
|
||||
|
||||
for ev in events:
|
||||
title = ev.get("title", "")
|
||||
if not title:
|
||||
continue
|
||||
title_lower = title.lower()
|
||||
if q_lower not in title_lower and not any(w in title_lower for w in q_words):
|
||||
continue
|
||||
|
||||
markets = ev.get("markets", [])
|
||||
best_pct = None
|
||||
total_volume = 0
|
||||
close_dates = []
|
||||
outcomes = []
|
||||
for m in markets:
|
||||
price = m.get("yes_price") or m.get("last_price")
|
||||
pct = None
|
||||
if price is not None:
|
||||
try:
|
||||
price = _finite_or_none(price)
|
||||
if price is None:
|
||||
raise ValueError("non-finite")
|
||||
pct = round(price, 1)
|
||||
if pct <= 1:
|
||||
pct = round(pct * 100, 1)
|
||||
if best_pct is None or pct > best_pct:
|
||||
best_pct = pct
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
try:
|
||||
volume = _finite_or_none(
|
||||
m.get("dollar_volume", 0) or m.get("volume", 0) or 0
|
||||
)
|
||||
if volume is not None:
|
||||
total_volume += int(volume)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
cd = m.get("close_date")
|
||||
if cd:
|
||||
close_dates.append(cd)
|
||||
oname = m.get("title") or m.get("subtitle", "")
|
||||
if oname and pct is not None:
|
||||
outcomes.append({"name": oname, "pct": pct})
|
||||
if len(outcomes) > 2:
|
||||
outcomes.sort(key=lambda x: x["pct"], reverse=True)
|
||||
else:
|
||||
outcomes = []
|
||||
|
||||
desc = (ev.get("settle_details") or ev.get("underlying") or "").strip()
|
||||
category = _classify_category(title, [], ev.get("category", ""))
|
||||
sources = []
|
||||
if best_pct is not None:
|
||||
sources.append({"name": "KALSHI", "pct": best_pct})
|
||||
|
||||
results.append({
|
||||
"title": title,
|
||||
"polymarket_pct": None,
|
||||
"kalshi_pct": best_pct,
|
||||
"consensus_pct": best_pct,
|
||||
"description": desc,
|
||||
"end_date": max(close_dates) if close_dates else None,
|
||||
"volume": total_volume,
|
||||
"volume_24h": 0,
|
||||
"kalshi_volume": total_volume,
|
||||
"category": category,
|
||||
"sources": sources,
|
||||
"slug": "",
|
||||
"kalshi_ticker": ev.get("ticker", ""),
|
||||
"outcomes": outcomes,
|
||||
})
|
||||
if len(results) >= offset + limit:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning(f"Kalshi search '{query}' error: {e}")
|
||||
|
||||
logger.info(f"Kalshi search '{query}': {len(results)} results")
|
||||
return results[offset : offset + limit]
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Static route + airport database loaded from vrs-standing-data.adsb.lol.
|
||||
|
||||
Replaces the per-batch /api/0/routeset POST with a single daily bulk download.
|
||||
Routes change ~weekly when airlines update schedules, so a 24h refresh cadence
|
||||
is far more than sufficient and removes ~all live-API pressure on adsb.lol.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import gzip
|
||||
import io
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_ROUTES_URL = "https://vrs-standing-data.adsb.lol/routes.csv.gz"
|
||||
_AIRPORTS_URL = "https://vrs-standing-data.adsb.lol/airports.csv.gz"
|
||||
_REFRESH_INTERVAL_S = 5 * 24 * 3600
|
||||
_HTTP_TIMEOUT_S = 60
|
||||
|
||||
_USER_AGENT = (
|
||||
"ShadowBroker-OSINT/0.9.7 "
|
||||
"(+https://github.com/BigBodyCobain/Shadowbroker; "
|
||||
"contact: bigbodycobain@gmail.com)"
|
||||
)
|
||||
|
||||
_lock = threading.RLock()
|
||||
_routes_by_callsign: dict[str, dict[str, Any]] = {}
|
||||
_airports_by_icao: dict[str, dict[str, Any]] = {}
|
||||
_last_refresh = 0.0
|
||||
_refresh_in_progress = False
|
||||
|
||||
|
||||
def _fetch_csv_gz(url: str) -> list[dict[str, str]]:
|
||||
response = requests.get(
|
||||
url,
|
||||
timeout=_HTTP_TIMEOUT_S,
|
||||
headers={"User-Agent": _USER_AGENT, "Accept-Encoding": "gzip"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
text = gzip.decompress(response.content).decode("utf-8-sig")
|
||||
return list(csv.DictReader(io.StringIO(text)))
|
||||
|
||||
|
||||
def _build_route_index(rows: list[dict[str, str]]) -> dict[str, dict[str, Any]]:
|
||||
index: dict[str, dict[str, Any]] = {}
|
||||
for row in rows:
|
||||
callsign = (row.get("Callsign") or "").strip().upper()
|
||||
airport_codes = (row.get("AirportCodes") or "").strip()
|
||||
if not callsign or not airport_codes:
|
||||
continue
|
||||
icaos = [c.strip() for c in airport_codes.split("-") if c.strip()]
|
||||
if len(icaos) < 2:
|
||||
continue
|
||||
index[callsign] = {
|
||||
"airline_code": (row.get("AirlineCode") or "").strip(),
|
||||
"airport_codes": airport_codes,
|
||||
"airport_icaos": icaos,
|
||||
}
|
||||
return index
|
||||
|
||||
|
||||
def _build_airport_index(rows: list[dict[str, str]]) -> dict[str, dict[str, Any]]:
|
||||
index: dict[str, dict[str, Any]] = {}
|
||||
for row in rows:
|
||||
icao = (row.get("ICAO") or "").strip().upper()
|
||||
if not icao:
|
||||
continue
|
||||
try:
|
||||
lat = float(row.get("Latitude") or 0)
|
||||
lon = float(row.get("Longitude") or 0)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
index[icao] = {
|
||||
"name": (row.get("Name") or "").strip(),
|
||||
"iata": (row.get("IATA") or "").strip(),
|
||||
"country": (row.get("CountryISO2") or "").strip(),
|
||||
"lat": lat,
|
||||
"lon": lon,
|
||||
}
|
||||
return index
|
||||
|
||||
|
||||
def refresh_route_database(force: bool = False) -> bool:
|
||||
"""Pull routes.csv.gz + airports.csv.gz and rebuild the in-memory indexes.
|
||||
|
||||
Returns True if a refresh was performed (success or attempted), False if
|
||||
skipped because the cache is still fresh or another refresh is in flight.
|
||||
"""
|
||||
global _last_refresh, _refresh_in_progress
|
||||
|
||||
now = time.time()
|
||||
with _lock:
|
||||
if _refresh_in_progress:
|
||||
return False
|
||||
if not force and (now - _last_refresh) < _REFRESH_INTERVAL_S and _routes_by_callsign:
|
||||
return False
|
||||
_refresh_in_progress = True
|
||||
|
||||
try:
|
||||
started = time.time()
|
||||
airport_rows = _fetch_csv_gz(_AIRPORTS_URL)
|
||||
route_rows = _fetch_csv_gz(_ROUTES_URL)
|
||||
airports = _build_airport_index(airport_rows)
|
||||
routes = _build_route_index(route_rows)
|
||||
with _lock:
|
||||
_airports_by_icao.clear()
|
||||
_airports_by_icao.update(airports)
|
||||
_routes_by_callsign.clear()
|
||||
_routes_by_callsign.update(routes)
|
||||
_last_refresh = time.time()
|
||||
logger.info(
|
||||
"route database refreshed in %.1fs: %d routes, %d airports",
|
||||
time.time() - started,
|
||||
len(routes),
|
||||
len(airports),
|
||||
)
|
||||
return True
|
||||
except (requests.RequestException, OSError, ValueError) as exc:
|
||||
logger.warning("route database refresh failed: %s", exc)
|
||||
return True
|
||||
finally:
|
||||
with _lock:
|
||||
_refresh_in_progress = False
|
||||
|
||||
|
||||
def lookup_route(callsign: str) -> dict[str, Any] | None:
|
||||
"""Resolve a callsign to {orig_name, dest_name, orig_loc, dest_loc} or None.
|
||||
|
||||
Matches the shape produced by the legacy fetch_routes_background cache so
|
||||
the caller in flights.py can be a drop-in replacement.
|
||||
"""
|
||||
key = (callsign or "").strip().upper()
|
||||
if not key:
|
||||
return None
|
||||
with _lock:
|
||||
route = _routes_by_callsign.get(key)
|
||||
if not route:
|
||||
return None
|
||||
icaos = route["airport_icaos"]
|
||||
orig = _airports_by_icao.get(icaos[0].upper())
|
||||
dest = _airports_by_icao.get(icaos[-1].upper())
|
||||
if not orig or not dest:
|
||||
return None
|
||||
return {
|
||||
"orig_name": f"{orig['iata']}: {orig['name']}" if orig["iata"] else orig["name"],
|
||||
"dest_name": f"{dest['iata']}: {dest['name']}" if dest["iata"] else dest["name"],
|
||||
"orig_loc": [orig["lon"], orig["lat"]],
|
||||
"dest_loc": [dest["lon"], dest["lat"]],
|
||||
}
|
||||
|
||||
|
||||
def route_database_status() -> dict[str, Any]:
|
||||
with _lock:
|
||||
return {
|
||||
"last_refresh": _last_refresh,
|
||||
"routes": len(_routes_by_callsign),
|
||||
"airports": len(_airports_by_icao),
|
||||
"in_progress": _refresh_in_progress,
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
"""SAR catalog fetcher (Mode A — default-on, free, no account).
|
||||
|
||||
Hits ASF Search every hour for Sentinel-1 scenes that touched any of
|
||||
the operator-defined AOIs in the last ~36h. Pure metadata, no
|
||||
downloads.
|
||||
|
||||
Result is written to ``latest_data["sar_scenes"]`` and a per-AOI
|
||||
coverage summary to ``latest_data["sar_aoi_coverage"]``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from services.fetchers._store import _data_lock, _mark_fresh, is_any_active, latest_data
|
||||
from services.fetchers.retry import with_retry
|
||||
from services.sar.sar_aoi import load_aois
|
||||
from services.sar.sar_catalog_client import estimate_next_pass, search_scenes_for_aoi
|
||||
from services.sar.sar_config import catalog_enabled
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@with_retry(max_retries=1, base_delay=2)
|
||||
def fetch_sar_catalog() -> None:
|
||||
"""Refresh the SAR scene catalog for all configured AOIs."""
|
||||
if not catalog_enabled():
|
||||
return
|
||||
if not is_any_active("sar"):
|
||||
return
|
||||
aois = load_aois()
|
||||
if not aois:
|
||||
logger.debug("SAR catalog: no AOIs configured")
|
||||
return
|
||||
|
||||
all_scenes: list[dict] = []
|
||||
coverage: list[dict] = []
|
||||
for aoi in aois:
|
||||
try:
|
||||
scenes = search_scenes_for_aoi(aoi)
|
||||
except (ConnectionError, TimeoutError, OSError, ValueError) as exc:
|
||||
logger.debug("SAR catalog %s: %s", aoi.id, exc)
|
||||
scenes = []
|
||||
scene_dicts = [s.to_dict() for s in scenes]
|
||||
all_scenes.extend(scene_dicts)
|
||||
next_pass = estimate_next_pass(scenes)
|
||||
coverage.append(
|
||||
{
|
||||
"aoi_id": aoi.id,
|
||||
"aoi_name": aoi.name,
|
||||
"category": aoi.category,
|
||||
"center_lat": aoi.center_lat,
|
||||
"center_lon": aoi.center_lon,
|
||||
"radius_km": aoi.radius_km,
|
||||
"recent_scene_count": len(scene_dicts),
|
||||
"latest_scene_time": (
|
||||
max((s["time"] for s in scene_dicts), default="")
|
||||
if scene_dicts
|
||||
else ""
|
||||
),
|
||||
**next_pass,
|
||||
}
|
||||
)
|
||||
|
||||
with _data_lock:
|
||||
latest_data["sar_scenes"] = all_scenes
|
||||
latest_data["sar_aoi_coverage"] = coverage
|
||||
if all_scenes or coverage:
|
||||
_mark_fresh("sar_scenes", "sar_aoi_coverage")
|
||||
logger.info(
|
||||
"SAR catalog: %d scenes across %d AOIs",
|
||||
len(all_scenes),
|
||||
len(aois),
|
||||
)
|
||||
@@ -0,0 +1,103 @@
|
||||
"""SAR pre-processed product fetcher (Mode B — opt-in, free, account needed).
|
||||
|
||||
Pulls already-computed deformation, flood, water, and damage products
|
||||
from NASA OPERA, Copernicus EGMS, GFM, EMS, and UNOSAT. No local DSP.
|
||||
|
||||
Two-step opt-in: ``MESH_SAR_PRODUCTS_FETCH=allow`` AND
|
||||
``MESH_SAR_PRODUCTS_FETCH_ACKNOWLEDGE=true``. When either flag is
|
||||
unset, this fetcher logs a single startup hint and returns.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from services.fetchers._store import _data_lock, _mark_fresh, is_any_active, latest_data
|
||||
from services.fetchers.retry import with_retry
|
||||
from services.sar.sar_aoi import load_aois
|
||||
from services.sar.sar_config import products_fetch_enabled, products_fetch_status
|
||||
from services.sar.sar_normalize import SarAnomaly
|
||||
from services.sar.sar_products_client import (
|
||||
fetch_egms_for_aoi,
|
||||
fetch_ems_for_aoi,
|
||||
fetch_gfm_for_aoi,
|
||||
fetch_opera_for_aoi,
|
||||
fetch_unosat_for_aoi,
|
||||
)
|
||||
from services.sar.sar_signing import emit_signed_anomaly
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_LOGGED_DISABLED_HINT = False
|
||||
|
||||
|
||||
def _hint_disabled_once() -> None:
|
||||
global _LOGGED_DISABLED_HINT
|
||||
if _LOGGED_DISABLED_HINT:
|
||||
return
|
||||
_LOGGED_DISABLED_HINT = True
|
||||
status = products_fetch_status()
|
||||
missing = ", ".join(status.get("missing", [])) or "nothing"
|
||||
logger.info(
|
||||
"SAR Mode B (ground-change alerts) is disabled. Missing: %s. "
|
||||
"Enable in Settings → SAR or set the env vars listed in .env.example. "
|
||||
"Free signup: https://urs.earthdata.nasa.gov/users/new",
|
||||
missing,
|
||||
)
|
||||
|
||||
|
||||
@with_retry(max_retries=1, base_delay=3)
|
||||
def fetch_sar_products() -> None:
|
||||
"""Refresh pre-processed SAR anomalies for all configured AOIs."""
|
||||
if not products_fetch_enabled():
|
||||
_hint_disabled_once()
|
||||
return
|
||||
if not is_any_active("sar"):
|
||||
return
|
||||
aois = load_aois()
|
||||
if not aois:
|
||||
logger.debug("SAR products: no AOIs configured")
|
||||
return
|
||||
|
||||
seen_ids: set[str] = set()
|
||||
all_anomalies: list[dict[str, Any]] = []
|
||||
publish_summary = {"signed": 0, "skipped": 0, "reasons": {}}
|
||||
|
||||
for aoi in aois:
|
||||
for fetcher in (
|
||||
fetch_opera_for_aoi,
|
||||
fetch_egms_for_aoi,
|
||||
fetch_gfm_for_aoi,
|
||||
fetch_ems_for_aoi,
|
||||
fetch_unosat_for_aoi,
|
||||
):
|
||||
try:
|
||||
anomalies: list[SarAnomaly] = fetcher(aoi) or []
|
||||
except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as exc:
|
||||
logger.debug("SAR %s for %s failed: %s", fetcher.__name__, aoi.id, exc)
|
||||
anomalies = []
|
||||
for a in anomalies:
|
||||
if a.anomaly_id in seen_ids:
|
||||
continue
|
||||
seen_ids.add(a.anomaly_id)
|
||||
all_anomalies.append(a.to_dict())
|
||||
status = emit_signed_anomaly(a)
|
||||
if status.get("signed"):
|
||||
publish_summary["signed"] += 1
|
||||
else:
|
||||
publish_summary["skipped"] += 1
|
||||
reason = status.get("reason", "unknown")
|
||||
publish_summary["reasons"][reason] = (
|
||||
publish_summary["reasons"].get(reason, 0) + 1
|
||||
)
|
||||
|
||||
with _data_lock:
|
||||
latest_data["sar_anomalies"] = all_anomalies
|
||||
if all_anomalies:
|
||||
_mark_fresh("sar_anomalies")
|
||||
logger.info(
|
||||
"SAR products: %d anomalies (%d signed, %d skipped)",
|
||||
len(all_anomalies),
|
||||
publish_summary["signed"],
|
||||
publish_summary["skipped"],
|
||||
)
|
||||
@@ -5,6 +5,11 @@ CelesTrak Fair Use Policy (https://celestrak.org/NORAD/elements/):
|
||||
- Use If-Modified-Since headers for conditional requests
|
||||
- No parallel/concurrent connections — one request at a time
|
||||
- Set a descriptive User-Agent
|
||||
|
||||
Analysis features (derived from cached TLEs — no extra network requests):
|
||||
- Maneuver detection: TLE-to-TLE comparison per satellite
|
||||
- Decay anomaly: mean-motion change rate monitoring
|
||||
- Overflight counting: 24h ground-track sampling for a bounding box
|
||||
"""
|
||||
|
||||
import math
|
||||
@@ -41,6 +46,67 @@ _sat_classified_cache = {"data": None, "gp_fetch_ts": 0}
|
||||
_SAT_CACHE_PATH = Path(__file__).parent.parent.parent / "data" / "sat_gp_cache.json"
|
||||
_SAT_CACHE_META_PATH = Path(__file__).parent.parent.parent / "data" / "sat_gp_cache_meta.json"
|
||||
|
||||
# ── Historical TLE storage for maneuver & decay detection ───────────────────
|
||||
# Stores the previous TLE snapshot keyed by NORAD_CAT_ID.
|
||||
# Populated when a fresh CelesTrak fetch replaces cached data.
|
||||
# Persisted to disk so analysis survives restarts.
|
||||
_SAT_HISTORY_PATH = Path(__file__).parent.parent.parent / "data" / "sat_tle_history.json"
|
||||
_tle_history: dict[int, dict] = {} # {norad_id: {elements + "epoch_ts"}}
|
||||
|
||||
|
||||
def _load_tle_history():
|
||||
"""Load previous TLE snapshot from disk."""
|
||||
global _tle_history
|
||||
try:
|
||||
if _SAT_HISTORY_PATH.exists():
|
||||
with open(_SAT_HISTORY_PATH, "r") as f:
|
||||
raw = json.load(f)
|
||||
_tle_history = {int(k): v for k, v in raw.items()}
|
||||
logger.info(f"Satellites: Loaded TLE history for {len(_tle_history)} objects")
|
||||
except (IOError, OSError, json.JSONDecodeError, ValueError, KeyError) as e:
|
||||
logger.warning(f"Satellites: Failed to load TLE history: {e}")
|
||||
_tle_history = {}
|
||||
|
||||
|
||||
def _save_tle_history():
|
||||
"""Persist current TLE snapshot as history for next comparison."""
|
||||
try:
|
||||
_SAT_HISTORY_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(_SAT_HISTORY_PATH, "w") as f:
|
||||
json.dump(_tle_history, f)
|
||||
except (IOError, OSError) as e:
|
||||
logger.warning(f"Satellites: Failed to save TLE history: {e}")
|
||||
|
||||
|
||||
def _snapshot_current_tles(gp_data):
|
||||
"""Capture orbital elements from current GP data as the new 'previous' snapshot.
|
||||
|
||||
Called once per CelesTrak fetch (every 24h). The old snapshot becomes
|
||||
the comparison baseline for maneuver/decay detection.
|
||||
"""
|
||||
global _tle_history
|
||||
new_snapshot = {}
|
||||
for sat in gp_data:
|
||||
norad_id = sat.get("NORAD_CAT_ID")
|
||||
if norad_id is None:
|
||||
continue
|
||||
epoch_str = sat.get("EPOCH", "")
|
||||
try:
|
||||
epoch_dt = datetime.strptime(epoch_str[:19], "%Y-%m-%dT%H:%M:%S")
|
||||
epoch_ts = epoch_dt.timestamp()
|
||||
except (ValueError, TypeError):
|
||||
epoch_ts = 0
|
||||
new_snapshot[int(norad_id)] = {
|
||||
"MEAN_MOTION": sat.get("MEAN_MOTION"),
|
||||
"ECCENTRICITY": sat.get("ECCENTRICITY"),
|
||||
"INCLINATION": sat.get("INCLINATION"),
|
||||
"RA_OF_ASC_NODE": sat.get("RA_OF_ASC_NODE"),
|
||||
"BSTAR": sat.get("BSTAR"),
|
||||
"epoch_ts": epoch_ts,
|
||||
}
|
||||
_tle_history = new_snapshot
|
||||
_save_tle_history()
|
||||
|
||||
|
||||
def _load_sat_cache():
|
||||
"""Load satellite GP data from local disk cache."""
|
||||
@@ -99,360 +165,368 @@ def _save_cache_meta():
|
||||
|
||||
|
||||
# Satellite intelligence classification database
|
||||
# Matched by substring against OBJECT_NAME (case-insensitive).
|
||||
# Order matters — first match wins, so specific names go before generic prefixes.
|
||||
_SAT_INTEL_DB = [
|
||||
(
|
||||
"USA 224",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "KH-11 Reconnaissance",
|
||||
"wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
|
||||
},
|
||||
),
|
||||
(
|
||||
"USA 245",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "KH-11 Reconnaissance",
|
||||
"wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
|
||||
},
|
||||
),
|
||||
(
|
||||
"USA 290",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "KH-11 Reconnaissance",
|
||||
"wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
|
||||
},
|
||||
),
|
||||
(
|
||||
"USA 314",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "KH-11 Reconnaissance",
|
||||
"wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
|
||||
},
|
||||
),
|
||||
(
|
||||
"USA 338",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "Keyhole Successor",
|
||||
"wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN",
|
||||
},
|
||||
),
|
||||
(
|
||||
"TOPAZ",
|
||||
{
|
||||
"country": "Russia",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "Optical Reconnaissance",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)",
|
||||
},
|
||||
),
|
||||
(
|
||||
"PERSONA",
|
||||
{
|
||||
"country": "Russia",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "Optical Reconnaissance",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)",
|
||||
},
|
||||
),
|
||||
(
|
||||
"KONDOR",
|
||||
{
|
||||
"country": "Russia",
|
||||
"mission": "military_sar",
|
||||
"sat_type": "SAR Reconnaissance",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Kondor_(satellite)",
|
||||
},
|
||||
),
|
||||
(
|
||||
"BARS-M",
|
||||
{
|
||||
"country": "Russia",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "Mapping Reconnaissance",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Bars-M",
|
||||
},
|
||||
),
|
||||
(
|
||||
"YAOGAN",
|
||||
{
|
||||
"country": "China",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "Remote Sensing / ELINT",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Yaogan",
|
||||
},
|
||||
),
|
||||
(
|
||||
"GAOFEN",
|
||||
{
|
||||
"country": "China",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "High-Res Imaging",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Gaofen",
|
||||
},
|
||||
),
|
||||
(
|
||||
"JILIN",
|
||||
{
|
||||
"country": "China",
|
||||
"mission": "commercial_imaging",
|
||||
"sat_type": "Video / Imaging",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Jilin-1",
|
||||
},
|
||||
),
|
||||
(
|
||||
"OFEK",
|
||||
{
|
||||
"country": "Israel",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "Reconnaissance",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Ofeq",
|
||||
},
|
||||
),
|
||||
(
|
||||
"CSO",
|
||||
{
|
||||
"country": "France",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "Optical Reconnaissance",
|
||||
"wiki": "https://en.wikipedia.org/wiki/CSO_(satellite)",
|
||||
},
|
||||
),
|
||||
(
|
||||
"IGS",
|
||||
{
|
||||
"country": "Japan",
|
||||
"mission": "military_recon",
|
||||
"sat_type": "Intelligence Gathering",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Information_Gathering_Satellite",
|
||||
},
|
||||
),
|
||||
(
|
||||
"CAPELLA",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "sar",
|
||||
"sat_type": "SAR Imaging",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Capella_Space",
|
||||
},
|
||||
),
|
||||
(
|
||||
"ICEYE",
|
||||
{
|
||||
"country": "Finland",
|
||||
"mission": "sar",
|
||||
"sat_type": "SAR Microsatellite",
|
||||
"wiki": "https://en.wikipedia.org/wiki/ICEYE",
|
||||
},
|
||||
),
|
||||
(
|
||||
"COSMO-SKYMED",
|
||||
{
|
||||
"country": "Italy",
|
||||
"mission": "sar",
|
||||
"sat_type": "SAR Constellation",
|
||||
"wiki": "https://en.wikipedia.org/wiki/COSMO-SkyMed",
|
||||
},
|
||||
),
|
||||
(
|
||||
"TANDEM",
|
||||
{
|
||||
"country": "Germany",
|
||||
"mission": "sar",
|
||||
"sat_type": "SAR Interferometry",
|
||||
"wiki": "https://en.wikipedia.org/wiki/TanDEM-X",
|
||||
},
|
||||
),
|
||||
(
|
||||
"PAZ",
|
||||
{
|
||||
"country": "Spain",
|
||||
"mission": "sar",
|
||||
"sat_type": "SAR Imaging",
|
||||
"wiki": "https://en.wikipedia.org/wiki/PAZ_(satellite)",
|
||||
},
|
||||
),
|
||||
(
|
||||
"WORLDVIEW",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "commercial_imaging",
|
||||
"sat_type": "Maxar High-Res",
|
||||
"wiki": "https://en.wikipedia.org/wiki/WorldView-3",
|
||||
},
|
||||
),
|
||||
(
|
||||
"GEOEYE",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "commercial_imaging",
|
||||
"sat_type": "Maxar Imaging",
|
||||
"wiki": "https://en.wikipedia.org/wiki/GeoEye-1",
|
||||
},
|
||||
),
|
||||
(
|
||||
"PLEIADES",
|
||||
{
|
||||
"country": "France",
|
||||
"mission": "commercial_imaging",
|
||||
"sat_type": "Airbus Imaging",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Pl%C3%A9iades_(satellite)",
|
||||
},
|
||||
),
|
||||
(
|
||||
"SPOT",
|
||||
{
|
||||
"country": "France",
|
||||
"mission": "commercial_imaging",
|
||||
"sat_type": "Airbus Medium-Res",
|
||||
"wiki": "https://en.wikipedia.org/wiki/SPOT_(satellite)",
|
||||
},
|
||||
),
|
||||
(
|
||||
"PLANET",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "commercial_imaging",
|
||||
"sat_type": "PlanetScope",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Planet_Labs",
|
||||
},
|
||||
),
|
||||
(
|
||||
"SKYSAT",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "commercial_imaging",
|
||||
"sat_type": "Planet Video",
|
||||
"wiki": "https://en.wikipedia.org/wiki/SkySat",
|
||||
},
|
||||
),
|
||||
(
|
||||
"BLACKSKY",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "commercial_imaging",
|
||||
"sat_type": "BlackSky Imaging",
|
||||
"wiki": "https://en.wikipedia.org/wiki/BlackSky",
|
||||
},
|
||||
),
|
||||
(
|
||||
"NROL",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "sigint",
|
||||
"sat_type": "Classified NRO",
|
||||
"wiki": "https://en.wikipedia.org/wiki/National_Reconnaissance_Office",
|
||||
},
|
||||
),
|
||||
(
|
||||
"MENTOR",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "sigint",
|
||||
"sat_type": "SIGINT / ELINT",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Mentor_(satellite)",
|
||||
},
|
||||
),
|
||||
(
|
||||
"LUCH",
|
||||
{
|
||||
"country": "Russia",
|
||||
"mission": "sigint",
|
||||
"sat_type": "Relay / SIGINT",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Luch_(satellite)",
|
||||
},
|
||||
),
|
||||
(
|
||||
"SHIJIAN",
|
||||
{
|
||||
"country": "China",
|
||||
"mission": "sigint",
|
||||
"sat_type": "ELINT / Tech Demo",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Shijian",
|
||||
},
|
||||
),
|
||||
(
|
||||
"NAVSTAR",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "navigation",
|
||||
"sat_type": "GPS",
|
||||
"wiki": "https://en.wikipedia.org/wiki/GPS_satellite_blocks",
|
||||
},
|
||||
),
|
||||
(
|
||||
"GLONASS",
|
||||
{
|
||||
"country": "Russia",
|
||||
"mission": "navigation",
|
||||
"sat_type": "GLONASS",
|
||||
"wiki": "https://en.wikipedia.org/wiki/GLONASS",
|
||||
},
|
||||
),
|
||||
(
|
||||
"BEIDOU",
|
||||
{
|
||||
"country": "China",
|
||||
"mission": "navigation",
|
||||
"sat_type": "BeiDou",
|
||||
"wiki": "https://en.wikipedia.org/wiki/BeiDou",
|
||||
},
|
||||
),
|
||||
(
|
||||
"GALILEO",
|
||||
{
|
||||
"country": "EU",
|
||||
"mission": "navigation",
|
||||
"sat_type": "Galileo",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Galileo_(satellite_navigation)",
|
||||
},
|
||||
),
|
||||
(
|
||||
"SBIRS",
|
||||
{
|
||||
"country": "USA",
|
||||
"mission": "early_warning",
|
||||
"sat_type": "Missile Warning",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Space-Based_Infrared_System",
|
||||
},
|
||||
),
|
||||
(
|
||||
"TUNDRA",
|
||||
{
|
||||
"country": "Russia",
|
||||
"mission": "early_warning",
|
||||
"sat_type": "Missile Warning",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Tundra_(satellite)",
|
||||
},
|
||||
),
|
||||
(
|
||||
"ISS",
|
||||
{
|
||||
"country": "Intl",
|
||||
"mission": "space_station",
|
||||
"sat_type": "Space Station",
|
||||
"wiki": "https://en.wikipedia.org/wiki/International_Space_Station",
|
||||
},
|
||||
),
|
||||
(
|
||||
"TIANGONG",
|
||||
{
|
||||
"country": "China",
|
||||
"mission": "space_station",
|
||||
"sat_type": "Space Station",
|
||||
"wiki": "https://en.wikipedia.org/wiki/Tiangong_space_station",
|
||||
},
|
||||
),
|
||||
# ── USA Keyhole / Reconnaissance ────────────────────────────────────────
|
||||
("USA 224", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
|
||||
("USA 245", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
|
||||
("USA 290", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
|
||||
("USA 314", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
|
||||
("USA 338", {"country": "USA", "mission": "military_recon", "sat_type": "Keyhole Successor", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}),
|
||||
# ── USA SIGINT / NRO ────────────────────────────────────────────────────
|
||||
("NROL", {"country": "USA", "mission": "sigint", "sat_type": "Classified NRO", "wiki": "https://en.wikipedia.org/wiki/National_Reconnaissance_Office"}),
|
||||
("MENTOR", {"country": "USA", "mission": "sigint", "sat_type": "SIGINT / ELINT (Orion)", "wiki": "https://en.wikipedia.org/wiki/Mentor_(satellite)"}),
|
||||
("TRUMPET", {"country": "USA", "mission": "sigint", "sat_type": "SIGINT (HEO)", "wiki": "https://en.wikipedia.org/wiki/Trumpet_(satellite)"}),
|
||||
("INTRUDER", {"country": "USA", "mission": "sigint", "sat_type": "Naval SIGINT (NOSS)", "wiki": "https://en.wikipedia.org/wiki/Naval_Ocean_Surveillance_System"}),
|
||||
# ── USA Early Warning / Missile Defense ─────────────────────────────────
|
||||
("SBIRS", {"country": "USA", "mission": "early_warning", "sat_type": "Missile Warning", "wiki": "https://en.wikipedia.org/wiki/Space-Based_Infrared_System"}),
|
||||
("DSP", {"country": "USA", "mission": "early_warning", "sat_type": "Defense Support Program", "wiki": "https://en.wikipedia.org/wiki/Defense_Support_Program"}),
|
||||
# ── USA Communications (Military) ───────────────────────────────────────
|
||||
("MUOS", {"country": "USA", "mission": "military_comms", "sat_type": "Mobile User Objective System", "wiki": "https://en.wikipedia.org/wiki/Mobile_User_Objective_System"}),
|
||||
("AEHF", {"country": "USA", "mission": "military_comms", "sat_type": "Advanced EHF", "wiki": "https://en.wikipedia.org/wiki/Advanced_Extremely_High_Frequency"}),
|
||||
("WGS", {"country": "USA", "mission": "military_comms", "sat_type": "Wideband Global SATCOM", "wiki": "https://en.wikipedia.org/wiki/Wideband_Global_SATCOM"}),
|
||||
("MILSTAR", {"country": "USA", "mission": "military_comms", "sat_type": "Milstar Secure Comms", "wiki": "https://en.wikipedia.org/wiki/Milstar"}),
|
||||
# ── USA Navigation ──────────────────────────────────────────────────────
|
||||
("NAVSTAR", {"country": "USA", "mission": "navigation", "sat_type": "GPS", "wiki": "https://en.wikipedia.org/wiki/GPS_satellite_blocks"}),
|
||||
# ── Russia Reconnaissance ───────────────────────────────────────────────
|
||||
("TOPAZ", {"country": "Russia", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)"}),
|
||||
("PERSONA", {"country": "Russia", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)"}),
|
||||
("KONDOR", {"country": "Russia", "mission": "military_sar", "sat_type": "SAR Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Kondor_(satellite)"}),
|
||||
("BARS-M", {"country": "Russia", "mission": "military_recon", "sat_type": "Mapping Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Bars-M"}),
|
||||
("RAZDAN", {"country": "Russia", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Razdan_(satellite)"}),
|
||||
("LOTOS", {"country": "Russia", "mission": "sigint", "sat_type": "ELINT (Lotos-S)", "wiki": "https://en.wikipedia.org/wiki/Lotos-S"}),
|
||||
("PION", {"country": "Russia", "mission": "sigint", "sat_type": "Naval SIGINT/Radar", "wiki": "https://en.wikipedia.org/wiki/Pion-NKS"}),
|
||||
("LUCH", {"country": "Russia", "mission": "sigint", "sat_type": "Relay / SIGINT", "wiki": "https://en.wikipedia.org/wiki/Luch_(satellite)"}),
|
||||
# ── Russia Early Warning & Navigation ───────────────────────────────────
|
||||
("TUNDRA", {"country": "Russia", "mission": "early_warning", "sat_type": "Missile Warning (EKS)", "wiki": "https://en.wikipedia.org/wiki/Tundra_(satellite)"}),
|
||||
("GLONASS", {"country": "Russia", "mission": "navigation", "sat_type": "GLONASS", "wiki": "https://en.wikipedia.org/wiki/GLONASS"}),
|
||||
# ── China Military / Intel ──────────────────────────────────────────────
|
||||
("YAOGAN", {"country": "China", "mission": "military_recon", "sat_type": "Remote Sensing / ELINT", "wiki": "https://en.wikipedia.org/wiki/Yaogan"}),
|
||||
("GAOFEN", {"country": "China", "mission": "military_recon", "sat_type": "High-Res Imaging", "wiki": "https://en.wikipedia.org/wiki/Gaofen"}),
|
||||
("JILIN", {"country": "China", "mission": "commercial_imaging", "sat_type": "Video / Imaging", "wiki": "https://en.wikipedia.org/wiki/Jilin-1"}),
|
||||
("SHIJIAN", {"country": "China", "mission": "sigint", "sat_type": "ELINT / Tech Demo", "wiki": "https://en.wikipedia.org/wiki/Shijian"}),
|
||||
("TONGXIN JISHU SHIYAN", {"country": "China", "mission": "military_comms", "sat_type": "Military Comms Test", "wiki": "https://en.wikipedia.org/wiki/Tongxin_Jishu_Shiyan"}),
|
||||
("BEIDOU", {"country": "China", "mission": "navigation", "sat_type": "BeiDou", "wiki": "https://en.wikipedia.org/wiki/BeiDou"}),
|
||||
("TIANGONG", {"country": "China", "mission": "space_station", "sat_type": "Space Station", "wiki": "https://en.wikipedia.org/wiki/Tiangong_space_station"}),
|
||||
# ── Allied Military / Intel ─────────────────────────────────────────────
|
||||
("OFEK", {"country": "Israel", "mission": "military_recon", "sat_type": "Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Ofeq"}),
|
||||
("EROS", {"country": "Israel", "mission": "commercial_imaging", "sat_type": "High-Res Imaging", "wiki": "https://en.wikipedia.org/wiki/EROS_(satellite)"}),
|
||||
("CSO", {"country": "France", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/CSO_(satellite)"}),
|
||||
("HELIOS", {"country": "France", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Helios_(satellite)"}),
|
||||
("CERES", {"country": "France", "mission": "sigint", "sat_type": "ELINT Constellation", "wiki": "https://en.wikipedia.org/wiki/CERES_(satellite)"}),
|
||||
("IGS", {"country": "Japan", "mission": "military_recon", "sat_type": "Intelligence Gathering", "wiki": "https://en.wikipedia.org/wiki/Information_Gathering_Satellite"}),
|
||||
("KOMPSAT", {"country": "South Korea", "mission": "military_recon", "sat_type": "Multi-Purpose Satellite", "wiki": "https://en.wikipedia.org/wiki/KOMPSAT"}),
|
||||
("SAR-LUPE", {"country": "Germany", "mission": "military_sar", "sat_type": "SAR Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/SAR-Lupe"}),
|
||||
("SARAH", {"country": "Germany", "mission": "military_sar", "sat_type": "SAR Successor (SARah)", "wiki": "https://en.wikipedia.org/wiki/SARah"}),
|
||||
# ── Commercial SAR ──────────────────────────────────────────────────────
|
||||
("CAPELLA", {"country": "USA", "mission": "sar", "sat_type": "SAR Imaging", "wiki": "https://en.wikipedia.org/wiki/Capella_Space"}),
|
||||
("ICEYE", {"country": "Finland", "mission": "sar", "sat_type": "SAR Microsatellite", "wiki": "https://en.wikipedia.org/wiki/ICEYE"}),
|
||||
("COSMO-SKYMED", {"country": "Italy", "mission": "sar", "sat_type": "SAR Constellation", "wiki": "https://en.wikipedia.org/wiki/COSMO-SkyMed"}),
|
||||
("TANDEM", {"country": "Germany", "mission": "sar", "sat_type": "SAR Interferometry", "wiki": "https://en.wikipedia.org/wiki/TanDEM-X"}),
|
||||
("PAZ", {"country": "Spain", "mission": "sar", "sat_type": "SAR Imaging", "wiki": "https://en.wikipedia.org/wiki/PAZ_(satellite)"}),
|
||||
("UMBRA", {"country": "USA", "mission": "sar", "sat_type": "SAR Microsatellite", "wiki": "https://en.wikipedia.org/wiki/Umbra_(company)"}),
|
||||
# ── Commercial Optical Imaging ──────────────────────────────────────────
|
||||
("WORLDVIEW", {"country": "USA", "mission": "commercial_imaging", "sat_type": "Maxar High-Res", "wiki": "https://en.wikipedia.org/wiki/WorldView-3"}),
|
||||
("GEOEYE", {"country": "USA", "mission": "commercial_imaging", "sat_type": "Maxar Imaging", "wiki": "https://en.wikipedia.org/wiki/GeoEye-1"}),
|
||||
("LEGION", {"country": "USA", "mission": "commercial_imaging", "sat_type": "Maxar Legion", "wiki": "https://en.wikipedia.org/wiki/WorldView_Legion"}),
|
||||
("PLEIADES", {"country": "France", "mission": "commercial_imaging", "sat_type": "Airbus Imaging", "wiki": "https://en.wikipedia.org/wiki/Pl%C3%A9iades_(satellite)"}),
|
||||
("SPOT", {"country": "France", "mission": "commercial_imaging", "sat_type": "Airbus Medium-Res", "wiki": "https://en.wikipedia.org/wiki/SPOT_(satellite)"}),
|
||||
("SKYSAT", {"country": "USA", "mission": "commercial_imaging", "sat_type": "Planet Video", "wiki": "https://en.wikipedia.org/wiki/SkySat"}),
|
||||
("BLACKSKY", {"country": "USA", "mission": "commercial_imaging", "sat_type": "BlackSky Imaging", "wiki": "https://en.wikipedia.org/wiki/BlackSky"}),
|
||||
# ── Starlink (separate category) ────────────────────────────────────────
|
||||
("STARLINK", {"country": "USA", "mission": "starlink", "sat_type": "Starlink Mega-Constellation", "wiki": "https://en.wikipedia.org/wiki/Starlink"}),
|
||||
# ── Other Constellations ────────────────────────────────────────────────
|
||||
("ONEWEB", {"country": "UK", "mission": "constellation", "sat_type": "OneWeb LEO Broadband", "wiki": "https://en.wikipedia.org/wiki/OneWeb"}),
|
||||
("GALILEO", {"country": "EU", "mission": "navigation", "sat_type": "Galileo", "wiki": "https://en.wikipedia.org/wiki/Galileo_(satellite_navigation)"}),
|
||||
# ── Space Stations ──────────────────────────────────────────────────────
|
||||
("ISS", {"country": "Intl", "mission": "space_station", "sat_type": "Space Station", "wiki": "https://en.wikipedia.org/wiki/International_Space_Station"}),
|
||||
# ── Generic fallback patterns (last resort) ─────────────────────────────
|
||||
("PLANET", {"country": "USA", "mission": "commercial_imaging", "sat_type": "PlanetScope", "wiki": "https://en.wikipedia.org/wiki/Planet_Labs"}),
|
||||
]
|
||||
|
||||
# CelesTrak SATCAT owner codes → country mapping for satellites not matched by name.
|
||||
# Used as a secondary classifier alongside name-pattern matching.
|
||||
_OWNER_CODE_MAP = {
|
||||
"US": "USA", "CIS": "Russia", "PRC": "China", "ISS": "Intl",
|
||||
"FR": "France", "UK": "UK", "GER": "Germany", "JPN": "Japan",
|
||||
"IND": "India", "ISRA": "Israel", "IT": "Italy", "KOR": "South Korea",
|
||||
"ESA": "EU", "NATO": "NATO", "TURK": "Turkey", "UAE": "UAE",
|
||||
"AUS": "Australia", "CA": "Canada", "SPN": "Spain", "FIN": "Finland",
|
||||
"BRAZ": "Brazil", "IRAN": "Iran", "NKOR": "North Korea",
|
||||
}
|
||||
|
||||
# ── Maneuver detection thresholds (per Lemmens & Krag 2014, Kim et al. 2021) ─
|
||||
# These are above TLE fitting noise but low enough to catch real maneuvers.
|
||||
_MANEUVER_THRESHOLDS = {
|
||||
"period_min": 0.1, # minutes — above TLE noise (~0.01–0.05 min)
|
||||
"inclination_deg": 0.05, # degrees — above J2 secular drift (~0.001°/day)
|
||||
"eccentricity": 0.005, # above TLE fitting noise (~0.0001–0.001)
|
||||
"raan_residual_deg": 0.5, # degrees — only after J2 correction (Vallado §9.4)
|
||||
}
|
||||
|
||||
# ── Decay anomaly threshold ─────────────────────────────────────────────────
|
||||
# Flag if mean motion change rate exceeds this (rev/day per day).
|
||||
# Normal drag-induced decay is ~0.001 rev/day/day for LEO.
|
||||
_DECAY_MM_RATE_THRESHOLD = 0.01 # rev/day per day
|
||||
|
||||
|
||||
def _j2_raan_rate(inclination_deg, mean_motion_revday):
|
||||
"""Expected RAAN precession rate due to J2 (Vallado §9.4).
|
||||
|
||||
Returns degrees/day. Negative for prograde orbits.
|
||||
"""
|
||||
J2 = 1.08263e-3
|
||||
Re = 6378.137 # km
|
||||
mu = 398600.4418 # km^3/s^2
|
||||
n_rad_s = mean_motion_revday * 2 * math.pi / 86400.0
|
||||
if n_rad_s <= 0:
|
||||
return 0.0
|
||||
a = (mu / (n_rad_s ** 2)) ** (1.0 / 3.0) # semi-major axis in km
|
||||
if a <= Re:
|
||||
return 0.0
|
||||
cos_i = math.cos(math.radians(inclination_deg))
|
||||
raan_rate = -1.5 * n_rad_s * J2 * (Re / a) ** 2 * cos_i
|
||||
return math.degrees(raan_rate) * 86400.0 / (2 * math.pi) # deg/day
|
||||
|
||||
|
||||
def detect_maneuvers(current_gp_data):
|
||||
"""Compare current TLEs against stored history to detect orbital maneuvers.
|
||||
|
||||
Returns list of maneuver alert dicts. Only runs when _tle_history is populated
|
||||
(i.e., after the second CelesTrak fetch or from persisted history).
|
||||
|
||||
Thresholds from Lemmens & Krag (2014), Kim et al. (2021).
|
||||
"""
|
||||
if not _tle_history:
|
||||
return []
|
||||
|
||||
alerts = []
|
||||
for sat in current_gp_data:
|
||||
norad_id = sat.get("NORAD_CAT_ID")
|
||||
if norad_id is None:
|
||||
continue
|
||||
norad_id = int(norad_id)
|
||||
prev = _tle_history.get(norad_id)
|
||||
if prev is None:
|
||||
continue
|
||||
|
||||
cur_mm = sat.get("MEAN_MOTION")
|
||||
cur_inc = sat.get("INCLINATION")
|
||||
cur_ecc = sat.get("ECCENTRICITY")
|
||||
cur_raan = sat.get("RA_OF_ASC_NODE")
|
||||
prev_mm = prev.get("MEAN_MOTION")
|
||||
prev_inc = prev.get("INCLINATION")
|
||||
prev_ecc = prev.get("ECCENTRICITY")
|
||||
prev_raan = prev.get("RA_OF_ASC_NODE")
|
||||
|
||||
if any(v is None for v in (cur_mm, cur_inc, cur_ecc, cur_raan,
|
||||
prev_mm, prev_inc, prev_ecc, prev_raan)):
|
||||
continue
|
||||
|
||||
# Convert mean motion (rev/day) to period (minutes)
|
||||
cur_period = 1440.0 / cur_mm if cur_mm > 0 else 0
|
||||
prev_period = 1440.0 / prev_mm if prev_mm > 0 else 0
|
||||
|
||||
reasons = []
|
||||
t = _MANEUVER_THRESHOLDS
|
||||
|
||||
delta_period = abs(cur_period - prev_period)
|
||||
if delta_period > t["period_min"]:
|
||||
reasons.append(f"period Δ{delta_period:+.3f} min")
|
||||
|
||||
delta_inc = abs(cur_inc - prev_inc)
|
||||
if delta_inc > t["inclination_deg"]:
|
||||
reasons.append(f"inclination Δ{delta_inc:+.4f}°")
|
||||
|
||||
delta_ecc = abs(cur_ecc - prev_ecc)
|
||||
if delta_ecc > t["eccentricity"]:
|
||||
reasons.append(f"eccentricity Δ{delta_ecc:+.6f}")
|
||||
|
||||
# RAAN with J2 correction — only flag residual beyond expected precession
|
||||
epoch_str = sat.get("EPOCH", "")
|
||||
try:
|
||||
epoch_dt = datetime.strptime(epoch_str[:19], "%Y-%m-%dT%H:%M:%S")
|
||||
epoch_ts = epoch_dt.timestamp()
|
||||
except (ValueError, TypeError):
|
||||
epoch_ts = 0
|
||||
prev_epoch_ts = prev.get("epoch_ts", 0)
|
||||
dt_days = (epoch_ts - prev_epoch_ts) / 86400.0 if (epoch_ts and prev_epoch_ts) else 1.0
|
||||
if dt_days > 0:
|
||||
expected_raan_drift = _j2_raan_rate(cur_inc, cur_mm) * dt_days
|
||||
actual_raan_change = cur_raan - prev_raan
|
||||
# Normalize to [-180, 180]
|
||||
actual_raan_change = (actual_raan_change + 180) % 360 - 180
|
||||
raan_residual = abs(actual_raan_change - expected_raan_drift)
|
||||
if raan_residual > t["raan_residual_deg"]:
|
||||
reasons.append(f"RAAN residual {raan_residual:.3f}° (J2-corrected)")
|
||||
|
||||
if reasons:
|
||||
alerts.append({
|
||||
"norad_id": norad_id,
|
||||
"name": sat.get("OBJECT_NAME", "UNKNOWN"),
|
||||
"type": "maneuver",
|
||||
"reasons": reasons,
|
||||
"epoch": sat.get("EPOCH", ""),
|
||||
"delta_period_min": round(delta_period, 4),
|
||||
"delta_inclination_deg": round(delta_inc, 5),
|
||||
"delta_eccentricity": round(delta_ecc, 7),
|
||||
})
|
||||
|
||||
logger.info(f"Satellites: Maneuver scan — {len(alerts)} detections from {len(current_gp_data)} objects")
|
||||
return alerts
|
||||
|
||||
|
||||
def detect_decay_anomalies(current_gp_data):
|
||||
"""Flag satellites with abnormal mean-motion change rates (possible decay).
|
||||
|
||||
A rapidly increasing mean motion indicates orbital decay — the satellite
|
||||
is losing altitude. Normal LEO drag is ~0.001 rev/day/day.
|
||||
"""
|
||||
if not _tle_history:
|
||||
return []
|
||||
|
||||
alerts = []
|
||||
for sat in current_gp_data:
|
||||
norad_id = sat.get("NORAD_CAT_ID")
|
||||
if norad_id is None:
|
||||
continue
|
||||
norad_id = int(norad_id)
|
||||
prev = _tle_history.get(norad_id)
|
||||
if prev is None:
|
||||
continue
|
||||
|
||||
cur_mm = sat.get("MEAN_MOTION")
|
||||
prev_mm = prev.get("MEAN_MOTION")
|
||||
if cur_mm is None or prev_mm is None:
|
||||
continue
|
||||
|
||||
epoch_str = sat.get("EPOCH", "")
|
||||
try:
|
||||
epoch_dt = datetime.strptime(epoch_str[:19], "%Y-%m-%dT%H:%M:%S")
|
||||
epoch_ts = epoch_dt.timestamp()
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
prev_epoch_ts = prev.get("epoch_ts", 0)
|
||||
dt_days = (epoch_ts - prev_epoch_ts) / 86400.0 if (epoch_ts and prev_epoch_ts) else 0
|
||||
if dt_days < 0.5:
|
||||
continue # Need at least 12h between TLEs for meaningful comparison
|
||||
|
||||
mm_rate = (cur_mm - prev_mm) / dt_days # rev/day per day
|
||||
if abs(mm_rate) > _DECAY_MM_RATE_THRESHOLD:
|
||||
cur_alt_km = (8681663.7 / (cur_mm ** (2.0 / 3.0))) - 6371.0 if cur_mm > 0 else 0
|
||||
alerts.append({
|
||||
"norad_id": norad_id,
|
||||
"name": sat.get("OBJECT_NAME", "UNKNOWN"),
|
||||
"type": "decay_anomaly",
|
||||
"mm_rate": round(mm_rate, 6),
|
||||
"current_mm": round(cur_mm, 4),
|
||||
"approx_alt_km": round(cur_alt_km, 1),
|
||||
"epoch": sat.get("EPOCH", ""),
|
||||
"dt_days": round(dt_days, 2),
|
||||
})
|
||||
|
||||
logger.info(f"Satellites: Decay scan — {len(alerts)} anomalies detected")
|
||||
return alerts
|
||||
|
||||
|
||||
def compute_overflights(gp_data, bbox, hours=24, step_minutes=10):
|
||||
"""Count unique satellites whose ground track enters a bounding box.
|
||||
|
||||
Args:
|
||||
gp_data: Full GP catalog (list of dicts with orbital elements).
|
||||
bbox: Dict with keys 's', 'w', 'n', 'e' (degrees).
|
||||
hours: Look-back window (default 24h).
|
||||
step_minutes: Sampling interval (default 10 min).
|
||||
|
||||
Returns dict with total count and per-mission breakdown.
|
||||
Uses SGP4 propagation — CPU cost is ~O(catalog_size × timesteps).
|
||||
Only propagates satellites that could plausibly overfly the bbox latitude range.
|
||||
"""
|
||||
if not gp_data or not bbox:
|
||||
return {"total": 0, "by_mission": {}, "satellites": []}
|
||||
|
||||
south, west = bbox["s"], bbox["w"]
|
||||
north, east = bbox["n"], bbox["e"]
|
||||
now = datetime.utcnow()
|
||||
steps = int(hours * 60 / step_minutes)
|
||||
|
||||
# Pre-filter: only propagate sats whose inclination allows them to reach bbox latitude
|
||||
max_lat = max(abs(south), abs(north))
|
||||
candidates = [s for s in gp_data if s.get("INCLINATION") is not None
|
||||
and s.get("INCLINATION") >= max_lat * 0.8] # 20% margin
|
||||
|
||||
seen_ids = set()
|
||||
results = []
|
||||
by_mission = {}
|
||||
|
||||
for s in candidates:
|
||||
norad_id = s.get("NORAD_CAT_ID")
|
||||
mean_motion = s.get("MEAN_MOTION")
|
||||
ecc = s.get("ECCENTRICITY")
|
||||
incl = s.get("INCLINATION")
|
||||
raan = s.get("RA_OF_ASC_NODE")
|
||||
argp = s.get("ARG_OF_PERICENTER")
|
||||
ma = s.get("MEAN_ANOMALY")
|
||||
bstar = s.get("BSTAR", 0)
|
||||
epoch_str = s.get("EPOCH", "")
|
||||
|
||||
if any(v is None for v in (mean_motion, ecc, incl, raan, argp, ma, epoch_str)):
|
||||
continue
|
||||
|
||||
try:
|
||||
epoch_dt = datetime.strptime(epoch_str[:19], "%Y-%m-%dT%H:%M:%S")
|
||||
epoch_jd, epoch_fr = jday(
|
||||
epoch_dt.year, epoch_dt.month, epoch_dt.day,
|
||||
epoch_dt.hour, epoch_dt.minute, epoch_dt.second,
|
||||
)
|
||||
sat_obj = Satrec()
|
||||
sat_obj.sgp4init(
|
||||
WGS72, "i", norad_id or 0,
|
||||
(epoch_jd + epoch_fr) - 2433281.5,
|
||||
bstar, 0.0, 0.0, ecc,
|
||||
math.radians(argp), math.radians(incl), math.radians(ma),
|
||||
mean_motion * 2 * math.pi / 1440.0, math.radians(raan),
|
||||
)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
|
||||
for step in range(steps):
|
||||
t = now - timedelta(minutes=step * step_minutes)
|
||||
jd_t, fr_t = jday(t.year, t.month, t.day, t.hour, t.minute, t.second)
|
||||
e, r, _ = sat_obj.sgp4(jd_t, fr_t)
|
||||
if e != 0:
|
||||
continue
|
||||
x, y, z = r
|
||||
gmst = _gmst(jd_t + fr_t)
|
||||
lng_rad = math.atan2(y, x) - gmst
|
||||
lat_deg = math.degrees(math.atan2(z, math.sqrt(x * x + y * y)))
|
||||
lng_deg = math.degrees(lng_rad) % 360
|
||||
if lng_deg > 180:
|
||||
lng_deg -= 360
|
||||
|
||||
# Check bounding box (handles antimeridian crossing)
|
||||
lat_in = south <= lat_deg <= north
|
||||
if west <= east:
|
||||
lng_in = west <= lng_deg <= east
|
||||
else:
|
||||
lng_in = lng_deg >= west or lng_deg <= east
|
||||
|
||||
if lat_in and lng_in and norad_id not in seen_ids:
|
||||
seen_ids.add(norad_id)
|
||||
name = s.get("OBJECT_NAME", "UNKNOWN")
|
||||
# Classify for mission breakdown
|
||||
mission = "unknown"
|
||||
for key, meta in _SAT_INTEL_DB:
|
||||
if key.upper() in name.upper():
|
||||
mission = meta.get("mission", "unknown")
|
||||
break
|
||||
by_mission[mission] = by_mission.get(mission, 0) + 1
|
||||
results.append({"norad_id": norad_id, "name": name, "mission": mission})
|
||||
break # Already counted this sat, move to next
|
||||
|
||||
return {"total": len(results), "by_mission": by_mission, "satellites": results}
|
||||
|
||||
|
||||
def _parse_tle_to_gp(name, norad_id, line1, line2):
|
||||
"""Convert TLE two-line element to CelesTrak GP-style dict."""
|
||||
@@ -539,9 +613,18 @@ def fetch_satellites():
|
||||
if not is_any_active("satellites"):
|
||||
return
|
||||
sats = []
|
||||
maneuver_alerts = []
|
||||
decay_alerts = []
|
||||
starlink_summary = {}
|
||||
data = None
|
||||
classified = None
|
||||
try:
|
||||
now_ts = time.time()
|
||||
|
||||
# On first call, load TLE history from disk for maneuver detection
|
||||
if not _tle_history:
|
||||
_load_tle_history()
|
||||
|
||||
# On first call, try disk cache before hitting CelesTrak
|
||||
if _sat_gp_cache["data"] is None:
|
||||
disk_data = _load_sat_cache()
|
||||
@@ -594,6 +677,9 @@ def fetch_satellites():
|
||||
if lm:
|
||||
_sat_gp_cache["last_modified"] = lm
|
||||
_save_sat_cache(gp_data)
|
||||
# Snapshot current TLEs as history before overwriting
|
||||
# (the old _tle_history becomes the comparison baseline)
|
||||
_snapshot_current_tles(gp_data)
|
||||
logger.info(
|
||||
f"Satellites: Downloaded {len(gp_data)} GP records from CelesTrak"
|
||||
)
|
||||
@@ -651,11 +737,14 @@ def fetch_satellites():
|
||||
and _sat_classified_cache["data"]
|
||||
):
|
||||
classified = _sat_classified_cache["data"]
|
||||
starlink_summary = _sat_classified_cache.get("starlink_summary", {})
|
||||
logger.info(
|
||||
f"Satellites: Using cached classification ({len(classified)} sats, TLEs unchanged)"
|
||||
)
|
||||
else:
|
||||
classified = []
|
||||
starlink_count = 0
|
||||
starlink_shells = {} # inclination shell → count
|
||||
for sat in data:
|
||||
name = sat.get("OBJECT_NAME", "UNKNOWN").upper()
|
||||
intel = None
|
||||
@@ -663,8 +752,24 @@ def fetch_satellites():
|
||||
if key.upper() in name:
|
||||
intel = dict(meta)
|
||||
break
|
||||
if not intel:
|
||||
# Secondary classification via SATCAT owner code
|
||||
owner = sat.get("OWNER", sat.get("OBJECT_OWNER", ""))
|
||||
if owner in _OWNER_CODE_MAP:
|
||||
intel = {"country": _OWNER_CODE_MAP[owner], "mission": "general", "sat_type": "Unclassified"}
|
||||
if not intel:
|
||||
continue
|
||||
|
||||
# Starlink: count and summarize but don't propagate individually
|
||||
# (6000+ sats would be too expensive to position every 60s)
|
||||
if intel.get("mission") == "starlink":
|
||||
starlink_count += 1
|
||||
inc = sat.get("INCLINATION")
|
||||
if inc is not None:
|
||||
shell_key = f"{round(inc, 0):.0f}°"
|
||||
starlink_shells[shell_key] = starlink_shells.get(shell_key, 0) + 1
|
||||
continue # Skip individual propagation
|
||||
|
||||
entry = {
|
||||
"id": sat.get("NORAD_CAT_ID"),
|
||||
"name": sat.get("OBJECT_NAME", "UNKNOWN"),
|
||||
@@ -679,14 +784,35 @@ def fetch_satellites():
|
||||
}
|
||||
entry.update(intel)
|
||||
classified.append(entry)
|
||||
|
||||
starlink_summary = {
|
||||
"total": starlink_count,
|
||||
"shells": starlink_shells,
|
||||
}
|
||||
_sat_classified_cache["data"] = classified
|
||||
_sat_classified_cache["starlink_summary"] = starlink_summary
|
||||
_sat_classified_cache["gp_fetch_ts"] = _sat_gp_cache["last_fetch"]
|
||||
logger.info(
|
||||
f"Satellites: {len(classified)} intel-classified out of {len(data)} total in catalog"
|
||||
f"Satellites: {len(classified)} intel-classified, "
|
||||
f"{starlink_count} Starlink (summarized), "
|
||||
f"out of {len(data)} total in catalog"
|
||||
)
|
||||
|
||||
all_sats = classified
|
||||
|
||||
# ── Run analysis detectors against the full GP catalog ──────────────
|
||||
# These use cached TLEs only — no extra network requests.
|
||||
maneuver_alerts = []
|
||||
decay_alerts = []
|
||||
try:
|
||||
maneuver_alerts = detect_maneuvers(data)
|
||||
except (ValueError, TypeError, KeyError, ZeroDivisionError) as e:
|
||||
logger.error(f"Satellites: Maneuver detection error: {e}")
|
||||
try:
|
||||
decay_alerts = detect_decay_anomalies(data)
|
||||
except (ValueError, TypeError, KeyError, ZeroDivisionError) as e:
|
||||
logger.error(f"Satellites: Decay detection error: {e}")
|
||||
|
||||
now = datetime.utcnow()
|
||||
jd, fr = jday(
|
||||
now.year, now.month, now.day, now.hour, now.minute, now.second + now.microsecond / 1e6
|
||||
@@ -800,6 +926,13 @@ def fetch_satellites():
|
||||
with _data_lock:
|
||||
latest_data["satellites"] = sats
|
||||
latest_data["satellite_source"] = _sat_gp_cache.get("source", "none")
|
||||
latest_data["satellite_analysis"] = {
|
||||
"maneuvers": maneuver_alerts,
|
||||
"decay_anomalies": decay_alerts,
|
||||
"starlink": starlink_summary,
|
||||
"catalog_size": len(data) if data else 0,
|
||||
"classified_count": len(classified) if classified else 0,
|
||||
}
|
||||
_mark_fresh("satellites")
|
||||
else:
|
||||
with _data_lock:
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
"""WastewaterSCAN fetcher — pathogen surveillance via wastewater monitoring.
|
||||
|
||||
Data source: Stanford/Emory WastewaterSCAN project
|
||||
- Plant locations: https://storage.googleapis.com/wastewater-dev-data/json/plants.json
|
||||
- Time series: https://storage.googleapis.com/wastewater-dev-data/json/{uuid}.json
|
||||
|
||||
All data is public, no authentication required. ~192 treatment plants across
|
||||
the US with daily sampling for COVID (N Gene), Influenza A/B, RSV, Norovirus,
|
||||
MPXV, Measles, H5N1, and others.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
import concurrent.futures
|
||||
from datetime import datetime, timedelta
|
||||
from services.network_utils import fetch_with_curl
|
||||
from services.fetchers._store import latest_data, _data_lock, _mark_fresh
|
||||
from services.fetchers.retry import with_retry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_GCS_BASE = "https://storage.googleapis.com/wastewater-dev-data/json"
|
||||
|
||||
# Cache the plants list for 24 hours (it rarely changes)
|
||||
_plants_cache: list[dict] = []
|
||||
_plants_cache_ts: float = 0
|
||||
_PLANTS_CACHE_TTL = 86400 # 24 hours
|
||||
|
||||
# Key pathogen targets to extract — maps internal target name to display label
|
||||
_TARGET_DISPLAY: dict[str, str] = {
|
||||
"N Gene": "COVID-19",
|
||||
"Influenza A F1R1": "Influenza A",
|
||||
"Influenza B": "Influenza B",
|
||||
"RSV": "RSV",
|
||||
"Noro_G2": "Norovirus",
|
||||
"MPXV_G2R_WA": "Mpox",
|
||||
"InfA_H5": "H5N1 (Bird Flu)",
|
||||
"HMPV_4": "HMPV",
|
||||
"Rota": "Rotavirus",
|
||||
"HAV": "Hepatitis A",
|
||||
"C_auris": "Candida auris",
|
||||
"EVD68": "Enterovirus D68",
|
||||
}
|
||||
|
||||
# Activity categories that represent elevated/alert levels
|
||||
_ALERT_CATEGORIES = {"high", "very high", "above normal"}
|
||||
|
||||
|
||||
def _fetch_plants() -> list[dict]:
|
||||
"""Fetch the full plants list from GCS, with 24h caching."""
|
||||
global _plants_cache, _plants_cache_ts
|
||||
|
||||
if _plants_cache and (time.time() - _plants_cache_ts) < _PLANTS_CACHE_TTL:
|
||||
return _plants_cache
|
||||
|
||||
url = f"{_GCS_BASE}/plants.json"
|
||||
resp = fetch_with_curl(url, timeout=30)
|
||||
if resp.status_code != 200:
|
||||
logger.warning(f"WastewaterSCAN plants fetch failed: HTTP {resp.status_code}")
|
||||
return _plants_cache # return stale cache on failure
|
||||
|
||||
data = resp.json()
|
||||
plants = data.get("plants", [])
|
||||
_plants_cache = plants
|
||||
_plants_cache_ts = time.time()
|
||||
logger.info(f"WastewaterSCAN: cached {len(plants)} plant locations")
|
||||
return plants
|
||||
|
||||
|
||||
def _fetch_plant_latest(plant_id: str) -> dict | None:
|
||||
"""Fetch the most recent sample for a single plant.
|
||||
|
||||
Returns a dict with pathogen levels or None on failure.
|
||||
"""
|
||||
url = f"{_GCS_BASE}/{plant_id}.json"
|
||||
try:
|
||||
resp = fetch_with_curl(url, timeout=12)
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
data = resp.json()
|
||||
samples = data.get("samples", [])
|
||||
if not samples:
|
||||
return None
|
||||
|
||||
# Find the most recent sample (last element, sorted by date)
|
||||
latest = samples[-1]
|
||||
collection_date = latest.get("collection_date", "")
|
||||
|
||||
# Skip samples older than 30 days
|
||||
try:
|
||||
sample_dt = datetime.strptime(collection_date, "%Y-%m-%d")
|
||||
if sample_dt < datetime.utcnow() - timedelta(days=30):
|
||||
return None
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# Extract key pathogen levels
|
||||
targets = latest.get("targets", {})
|
||||
pathogens: list[dict] = []
|
||||
alert_count = 0
|
||||
|
||||
for target_key, display_name in _TARGET_DISPLAY.items():
|
||||
target_data = targets.get(target_key)
|
||||
if not target_data:
|
||||
continue
|
||||
|
||||
concentration = target_data.get("gc_g_dry_weight", 0) or 0
|
||||
activity = target_data.get("activity_category", "not calculated")
|
||||
normalized = target_data.get("gc_g_dry_weight_pmmov", 0) or 0
|
||||
|
||||
if concentration <= 0 and normalized <= 0:
|
||||
continue # no detection
|
||||
|
||||
is_alert = activity.lower() in _ALERT_CATEGORIES
|
||||
if is_alert:
|
||||
alert_count += 1
|
||||
|
||||
pathogens.append({
|
||||
"name": display_name,
|
||||
"target_key": target_key,
|
||||
"concentration": round(concentration, 1),
|
||||
"normalized": round(normalized, 6),
|
||||
"activity": activity,
|
||||
"alert": is_alert,
|
||||
})
|
||||
|
||||
if not pathogens:
|
||||
return None
|
||||
|
||||
return {
|
||||
"collection_date": collection_date,
|
||||
"pathogens": pathogens,
|
||||
"alert_count": alert_count,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.debug(f"WastewaterSCAN: failed to fetch plant {plant_id}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
@with_retry(max_retries=1, base_delay=5)
|
||||
def fetch_wastewater():
|
||||
"""Fetch WastewaterSCAN plant locations and latest pathogen levels.
|
||||
|
||||
1. Fetches the plant list (cached 24h) for locations.
|
||||
2. Concurrently fetches time series for all plants, extracting only
|
||||
the most recent sample's pathogen data.
|
||||
3. Merges into a flat list suitable for map rendering.
|
||||
"""
|
||||
from services.fetchers._store import is_any_active
|
||||
|
||||
if not is_any_active("wastewater"):
|
||||
return
|
||||
|
||||
plants = _fetch_plants()
|
||||
if not plants:
|
||||
logger.warning("WastewaterSCAN: no plant data available")
|
||||
return
|
||||
|
||||
# Build base records from plant metadata
|
||||
plant_map: dict[str, dict] = {}
|
||||
for p in plants:
|
||||
point = p.get("point") or {}
|
||||
coords = point.get("coordinates") or []
|
||||
if len(coords) < 2:
|
||||
continue
|
||||
|
||||
pid = p.get("id") or p.get("uuid", "")
|
||||
if not pid:
|
||||
continue
|
||||
|
||||
plant_map[pid] = {
|
||||
"id": pid,
|
||||
"name": p.get("name", ""),
|
||||
"site_name": p.get("site_name", ""),
|
||||
"city": p.get("city", ""),
|
||||
"state": p.get("state", ""),
|
||||
"country": p.get("country", "US"),
|
||||
"population": p.get("sewershed_pop"),
|
||||
"lat": coords[1],
|
||||
"lng": coords[0],
|
||||
"pathogens": [],
|
||||
"alert_count": 0,
|
||||
"collection_date": "",
|
||||
"source": "WastewaterSCAN",
|
||||
}
|
||||
|
||||
# Fetch latest samples concurrently (up to 12 threads)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
|
||||
futures = {
|
||||
pool.submit(_fetch_plant_latest, pid): pid
|
||||
for pid in plant_map
|
||||
}
|
||||
for fut in concurrent.futures.as_completed(futures, timeout=120):
|
||||
pid = futures[fut]
|
||||
try:
|
||||
result = fut.result()
|
||||
if result:
|
||||
plant_map[pid]["pathogens"] = result["pathogens"]
|
||||
plant_map[pid]["alert_count"] = result["alert_count"]
|
||||
plant_map[pid]["collection_date"] = result["collection_date"]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
nodes = list(plant_map.values())
|
||||
active_nodes = [n for n in nodes if n["pathogens"]]
|
||||
|
||||
logger.info(
|
||||
f"WastewaterSCAN: {len(nodes)} plants, "
|
||||
f"{len(active_nodes)} with recent pathogen data, "
|
||||
f"{sum(n['alert_count'] for n in nodes)} total alerts"
|
||||
)
|
||||
|
||||
with _data_lock:
|
||||
latest_data["wastewater"] = nodes
|
||||
if nodes:
|
||||
_mark_fresh("wastewater")
|
||||
Reference in New Issue
Block a user