mirror of
https://github.com/BigBodyCobain/Shadowbroker.git
synced 2026-07-16 00:37:25 +02:00
feat: Telegram OSINT map layer, Osiris intel ports, and maritime settings
Add Telegram OSINT with hourly incremental t.me scraping, metro geocoding separate from news centroids, threat-intercept popup UI with inline media, and HTML markers above alert boxes so pins stay clickable. Expose GFW_API_TOKEN in onboarding and Settings Maritime; harden GFW/CCTV/geo fetchers. Port Osiris- derived recon, SCM, entity graph, malware/cyber feeds, sanctions, and submarine cable layers with tests and documentation. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
BigBodyCobain
@@ -51,6 +51,15 @@ API_REGISTRY = [
|
||||
"url": "https://aisstream.io/",
|
||||
"required": True,
|
||||
},
|
||||
{
|
||||
"id": "gfw_api_token",
|
||||
"env_key": "GFW_API_TOKEN",
|
||||
"name": "Global Fishing Watch",
|
||||
"description": "Bearer token for Global Fishing Watch fishing-vessel activity events (Fishing Activity map layer). Free registration at globalfishingwatch.org.",
|
||||
"category": "Maritime",
|
||||
"url": "https://globalfishingwatch.org/our-apis/",
|
||||
"required": False,
|
||||
},
|
||||
{
|
||||
"id": "adsb_lol",
|
||||
"env_key": None,
|
||||
|
||||
+416
-109
@@ -17,6 +17,9 @@ _KNOWN_CCTV_MEDIA_HOST_ALIASES = {
|
||||
# Trusted upstream occasionally publishes a typo for this Georgia camera
|
||||
# host. Normalize it at ingest so the proxy and client stay consistent.
|
||||
"navigatos-c2c.dot.ga.gov": "navigator-c2c.dot.ga.gov",
|
||||
# TravelIQ staging hosts occasionally appear in 511 catalog metadata.
|
||||
"on.stage.traveliq.co": "511on.ca",
|
||||
"ab.stage.traveliq.co": "511.alberta.ca",
|
||||
}
|
||||
|
||||
_POINT_WKT_RE = re.compile(
|
||||
@@ -40,6 +43,17 @@ def _normalize_cctv_media_url(raw_url: str) -> str:
|
||||
return urlunparse(parsed._replace(netloc=netloc))
|
||||
|
||||
|
||||
def _ensure_https_url(raw_url: str) -> str:
|
||||
"""Upgrade http:// media/catalog URLs to https:// at ingest time."""
|
||||
candidate = _normalize_cctv_media_url(str(raw_url or "").strip())
|
||||
if not candidate:
|
||||
return ""
|
||||
parsed = urlparse(candidate)
|
||||
if parsed.scheme.lower() == "http":
|
||||
return urlunparse(parsed._replace(scheme="https"))
|
||||
return candidate
|
||||
|
||||
|
||||
def _looks_like_direct_cctv_media_url(url: str) -> bool:
|
||||
candidate = str(url or "").strip().lower()
|
||||
if not candidate.startswith(("http://", "https://")):
|
||||
@@ -93,6 +107,165 @@ def _parse_wkt_point(raw_point: str) -> tuple[float | None, float | None]:
|
||||
return lat, lon
|
||||
|
||||
|
||||
def _fetch_traveliq_v2_cameras(
|
||||
*,
|
||||
api_url: str,
|
||||
base_url: str,
|
||||
id_prefix: str,
|
||||
source_agency: str,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Parse TravelIQ-style GET /api/v2/get/cameras feeds (Ontario, Alberta)."""
|
||||
resp = fetch_with_curl(
|
||||
api_url,
|
||||
timeout=30,
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
if not resp or resp.status_code != 200:
|
||||
logger.error(
|
||||
"%s CCTV fetch failed: HTTP %s",
|
||||
source_agency,
|
||||
resp.status_code if resp else "no response",
|
||||
)
|
||||
return []
|
||||
|
||||
data = resp.json()
|
||||
if not isinstance(data, list):
|
||||
return []
|
||||
|
||||
cameras: List[Dict[str, Any]] = []
|
||||
for cam in data:
|
||||
if not isinstance(cam, dict):
|
||||
continue
|
||||
try:
|
||||
lat = float(cam.get("Latitude"))
|
||||
lon = float(cam.get("Longitude"))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
|
||||
site_id = cam.get("Id")
|
||||
location = str(cam.get("Location") or cam.get("Roadway") or "Camera")[:120]
|
||||
views = cam.get("Views") or []
|
||||
if not views:
|
||||
continue
|
||||
|
||||
for view in views:
|
||||
if not isinstance(view, dict):
|
||||
continue
|
||||
status = str(view.get("Status") or "enabled").strip().lower()
|
||||
if status and status not in {"enabled", "active"}:
|
||||
continue
|
||||
media_url = _ensure_https_url(
|
||||
urljoin(base_url, str(view.get("Url") or "").strip())
|
||||
)
|
||||
if not media_url:
|
||||
continue
|
||||
view_id = view.get("Id") or site_id
|
||||
if site_id is None or view_id is None:
|
||||
continue
|
||||
label = str(view.get("Description") or location or "Camera")[:120]
|
||||
cameras.append(
|
||||
{
|
||||
"id": f"{id_prefix}-{site_id}-{view_id}",
|
||||
"source_agency": source_agency,
|
||||
"lat": lat,
|
||||
"lon": lon,
|
||||
"direction_facing": label,
|
||||
"media_url": media_url,
|
||||
"media_type": "image",
|
||||
"refresh_rate_seconds": 60,
|
||||
}
|
||||
)
|
||||
return cameras
|
||||
|
||||
|
||||
def _fetch_511_datatables_cameras(
|
||||
*,
|
||||
list_url: str,
|
||||
base_url: str,
|
||||
id_prefix: str,
|
||||
source_agency: str,
|
||||
referer: str,
|
||||
page_size: int = 500,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Parse 511 DataTables POST /List/GetData/Cameras feeds (Georgia, Florida)."""
|
||||
cameras: List[Dict[str, Any]] = []
|
||||
start = 0
|
||||
draw = 1
|
||||
while True:
|
||||
resp = fetch_with_curl(
|
||||
list_url,
|
||||
method="POST",
|
||||
json_data={"draw": draw, "start": start, "length": page_size},
|
||||
timeout=30,
|
||||
headers={
|
||||
"Accept": "application/json",
|
||||
"Referer": referer,
|
||||
"Origin": base_url.rstrip("/"),
|
||||
},
|
||||
)
|
||||
if not resp or resp.status_code != 200:
|
||||
logger.error(
|
||||
"%s CCTV fetch failed: HTTP %s",
|
||||
source_agency,
|
||||
resp.status_code if resp else "no response",
|
||||
)
|
||||
break
|
||||
|
||||
data = resp.json()
|
||||
rows = data.get("data") or []
|
||||
if not rows:
|
||||
break
|
||||
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
site_id = row.get("id") or row.get("DT_RowId")
|
||||
location = row.get("location") or row.get("roadway") or source_agency
|
||||
lat_lng = row.get("latLng") or {}
|
||||
geography = lat_lng.get("geography") if isinstance(lat_lng, dict) else {}
|
||||
lat, lon = _parse_wkt_point(
|
||||
geography.get("wellKnownText") if isinstance(geography, dict) else ""
|
||||
)
|
||||
images = row.get("images") or []
|
||||
image = next(
|
||||
(
|
||||
candidate
|
||||
for candidate in images
|
||||
if str(candidate.get("imageUrl") or "").strip()
|
||||
and not bool(candidate.get("blocked"))
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not (site_id and image and lat is not None and lon is not None):
|
||||
continue
|
||||
media_url = _ensure_https_url(
|
||||
urljoin(base_url, str(image.get("imageUrl") or "").strip())
|
||||
)
|
||||
if not media_url:
|
||||
continue
|
||||
cameras.append(
|
||||
{
|
||||
"id": f"{id_prefix}-{site_id}",
|
||||
"source_agency": source_agency,
|
||||
"lat": lat,
|
||||
"lon": lon,
|
||||
"direction_facing": str(location)[:120],
|
||||
"media_url": media_url,
|
||||
"media_type": "image",
|
||||
"refresh_rate_seconds": 60,
|
||||
}
|
||||
)
|
||||
|
||||
start += len(rows)
|
||||
draw += 1
|
||||
total = int(data.get("recordsTotal") or 0)
|
||||
if total and start >= total:
|
||||
break
|
||||
if not total and len(rows) < page_size:
|
||||
break
|
||||
return cameras
|
||||
|
||||
|
||||
def init_db():
|
||||
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(str(DB_PATH))
|
||||
@@ -169,7 +342,7 @@ class BaseCCTVIngestor(ABC):
|
||||
cam.get("lat"),
|
||||
cam.get("lon"),
|
||||
cam.get("direction_facing", "Unknown"),
|
||||
cam.get("media_url"),
|
||||
_ensure_https_url(cam.get("media_url", "")),
|
||||
cam.get("media_type", _detect_media_type(cam.get("media_url", ""))),
|
||||
cam.get("refresh_rate_seconds", 60),
|
||||
),
|
||||
@@ -454,77 +627,14 @@ class WSDOTIngestor(BaseCCTVIngestor):
|
||||
class GeorgiaDOTIngestor(BaseCCTVIngestor):
|
||||
"""Georgia cameras via the public 511GA list feed."""
|
||||
|
||||
URL = "https://511ga.org/List/GetData/Cameras"
|
||||
BASE_URL = "https://511ga.org"
|
||||
PAGE_SIZE = 500
|
||||
|
||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||
cameras = []
|
||||
start = 0
|
||||
draw = 1
|
||||
while True:
|
||||
resp = fetch_with_curl(
|
||||
self.URL,
|
||||
method="POST",
|
||||
json_data={"draw": draw, "start": start, "length": self.PAGE_SIZE},
|
||||
timeout=30,
|
||||
headers={
|
||||
"Accept": "application/json",
|
||||
"Referer": "https://511ga.org/cctv",
|
||||
"Origin": "https://511ga.org",
|
||||
},
|
||||
)
|
||||
if not resp or resp.status_code != 200:
|
||||
logger.error(
|
||||
"Georgia CCTV fetch failed: HTTP %s",
|
||||
resp.status_code if resp else "no response",
|
||||
)
|
||||
break
|
||||
data = resp.json()
|
||||
rows = data.get("data") or []
|
||||
if not rows:
|
||||
break
|
||||
for row in rows:
|
||||
site_id = row.get("id") or row.get("DT_RowId")
|
||||
location = row.get("location") or row.get("roadway") or "GA Camera"
|
||||
lat_lng = row.get("latLng") or {}
|
||||
geography = lat_lng.get("geography") if isinstance(lat_lng, dict) else {}
|
||||
lat, lon = _parse_wkt_point(geography.get("wellKnownText") if isinstance(geography, dict) else "")
|
||||
images = row.get("images") or []
|
||||
image = next(
|
||||
(
|
||||
candidate
|
||||
for candidate in images
|
||||
if str(candidate.get("imageUrl") or "").strip()
|
||||
and not bool(candidate.get("blocked"))
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not (site_id and image and lat is not None and lon is not None):
|
||||
continue
|
||||
media_url = _normalize_cctv_media_url(
|
||||
urljoin(self.BASE_URL, str(image.get("imageUrl") or "").strip())
|
||||
)
|
||||
cameras.append(
|
||||
{
|
||||
"id": f"GDOT-{site_id}",
|
||||
"source_agency": "Georgia DOT",
|
||||
"lat": lat,
|
||||
"lon": lon,
|
||||
"direction_facing": str(location)[:120],
|
||||
"media_url": media_url,
|
||||
"media_type": "image",
|
||||
"refresh_rate_seconds": 60,
|
||||
}
|
||||
)
|
||||
start += len(rows)
|
||||
draw += 1
|
||||
total = int(data.get("recordsTotal") or 0)
|
||||
if total and start >= total:
|
||||
break
|
||||
if not total and len(rows) < self.PAGE_SIZE:
|
||||
break
|
||||
return cameras
|
||||
return _fetch_511_datatables_cameras(
|
||||
list_url="https://511ga.org/List/GetData/Cameras",
|
||||
base_url="https://511ga.org",
|
||||
id_prefix="GDOT",
|
||||
source_agency="Georgia DOT",
|
||||
referer="https://511ga.org/cctv",
|
||||
)
|
||||
|
||||
|
||||
class IllinoisDOTIngestor(BaseCCTVIngestor):
|
||||
@@ -1009,30 +1119,66 @@ def _extract_img_src(html_fragment: str):
|
||||
return None
|
||||
|
||||
|
||||
class AsfinagIngestor(BaseCCTVIngestor):
|
||||
"""Austria ASFINAG motorway webcams (Osiris port)."""
|
||||
|
||||
API_URL = "https://odo.asfinag.at/odo/rest/sec/resource/001/json/webcams?language=atDE"
|
||||
HEADERS = {
|
||||
"User-Agent": "Shadowbroker-CCTV/1.0",
|
||||
"Accept": "application/json",
|
||||
"Referer": "https://www.asfinag.at/",
|
||||
"Authorization": "Basic bWFwX3dpZGdldDp0ZWdkaXc=",
|
||||
}
|
||||
|
||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||
try:
|
||||
response = fetch_with_curl(self.API_URL, timeout=15, headers=self.HEADERS)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
except Exception as exc:
|
||||
logger.error("AsfinagIngestor: fetch failed: %s", exc)
|
||||
return []
|
||||
if not isinstance(payload, list):
|
||||
return []
|
||||
cameras: List[Dict[str, Any]] = []
|
||||
for cam in payload:
|
||||
cam_id = cam.get("wcs_id")
|
||||
lat = cam.get("wgs84_lat")
|
||||
lon = cam.get("wgs84_lon")
|
||||
image_url = cam.get("url_campic")
|
||||
if not cam_id or lat is None or lon is None or not image_url:
|
||||
continue
|
||||
if str(cam_id).startswith("Utinform"):
|
||||
continue
|
||||
label = cam.get("position_txt") or cam.get("direction_txt") or "ASFINAG Webcam"
|
||||
secure_url = _ensure_https_url(image_url)
|
||||
if not secure_url:
|
||||
continue
|
||||
cameras.append(
|
||||
{
|
||||
"id": f"ASFINAG-{cam_id}",
|
||||
"source_agency": "ASFINAG Austria",
|
||||
"lat": float(lat),
|
||||
"lon": float(lon),
|
||||
"direction_facing": label,
|
||||
"media_url": secure_url,
|
||||
"media_type": "image",
|
||||
"refresh_rate_seconds": 300,
|
||||
}
|
||||
)
|
||||
logger.info("AsfinagIngestor: parsed %s cameras", len(cameras))
|
||||
return cameras
|
||||
|
||||
|
||||
class MadridCityIngestor(BaseCCTVIngestor):
|
||||
"""Madrid City Hall traffic cameras from datos.madrid.es KML feed."""
|
||||
|
||||
KML_URL_HTTPS = "https://datos.madrid.es/egob/catalogo/202088-0-trafico-camaras.kml"
|
||||
KML_URL_HTTP = "http://datos.madrid.es/egob/catalogo/202088-0-trafico-camaras.kml"
|
||||
KML_URL = "https://datos.madrid.es/egob/catalogo/202088-0-trafico-camaras.kml"
|
||||
|
||||
def _fetch_kml(self):
|
||||
"""Prefer HTTPS; fall back to legacy HTTP if the catalog is HTTP-only (#363)."""
|
||||
last_error: Exception | None = None
|
||||
for url in (self.KML_URL_HTTPS, self.KML_URL_HTTP):
|
||||
try:
|
||||
response = fetch_with_curl(url, timeout=20)
|
||||
response.raise_for_status()
|
||||
if url == self.KML_URL_HTTP:
|
||||
logger.warning(
|
||||
"MadridCityIngestor: HTTPS KML unavailable, using HTTP catalog feed"
|
||||
)
|
||||
return response
|
||||
except Exception as e:
|
||||
last_error = e
|
||||
logger.debug("MadridCityIngestor: KML fetch failed for %s: %s", url, e)
|
||||
if last_error is not None:
|
||||
raise last_error
|
||||
raise RuntimeError("Madrid KML fetch failed")
|
||||
response = fetch_with_curl(self.KML_URL, timeout=20)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
|
||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||
import defusedxml.ElementTree as ET
|
||||
@@ -1074,6 +1220,9 @@ class MadridCityIngestor(BaseCCTVIngestor):
|
||||
if desc_el is not None and desc_el.text:
|
||||
image_url = _extract_img_src(desc_el.text)
|
||||
|
||||
if not image_url:
|
||||
continue
|
||||
image_url = _ensure_https_url(image_url)
|
||||
if not image_url:
|
||||
continue
|
||||
|
||||
@@ -1095,6 +1244,153 @@ class MadridCityIngestor(BaseCCTVIngestor):
|
||||
return cameras
|
||||
|
||||
|
||||
class Ontario511Ingestor(BaseCCTVIngestor):
|
||||
"""Ontario highway cameras via 511on.ca TravelIQ API."""
|
||||
|
||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||
return _fetch_traveliq_v2_cameras(
|
||||
api_url="https://511on.ca/api/v2/get/cameras",
|
||||
base_url="https://511on.ca",
|
||||
id_prefix="ON511",
|
||||
source_agency="511 Ontario",
|
||||
)
|
||||
|
||||
|
||||
class Alberta511Ingestor(BaseCCTVIngestor):
|
||||
"""Alberta highway cameras via 511 Alberta TravelIQ API."""
|
||||
|
||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||
return _fetch_traveliq_v2_cameras(
|
||||
api_url="https://511.alberta.ca/api/v2/get/cameras",
|
||||
base_url="https://511.alberta.ca",
|
||||
id_prefix="AB511",
|
||||
source_agency="511 Alberta",
|
||||
)
|
||||
|
||||
|
||||
class Florida511Ingestor(BaseCCTVIngestor):
|
||||
"""Florida cameras via FL511 DataTables feed (~4,800 sites)."""
|
||||
|
||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||
return _fetch_511_datatables_cameras(
|
||||
list_url="https://fl511.com/List/GetData/Cameras",
|
||||
base_url="https://fl511.com",
|
||||
id_prefix="FL511",
|
||||
source_agency="Florida 511",
|
||||
referer="https://fl511.com/",
|
||||
)
|
||||
|
||||
|
||||
class AustraliaLiveTrafficIngestor(BaseCCTVIngestor):
|
||||
"""NSW / Australia live traffic cameras via Transport for NSW JSON feed."""
|
||||
|
||||
URL = "https://www.livetraffic.com/datajson/all-feeds-web.json"
|
||||
|
||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||
resp = fetch_with_curl(self.URL, timeout=35, headers={"Accept": "application/json"})
|
||||
if not resp or resp.status_code != 200:
|
||||
logger.error(
|
||||
"Australia Live Traffic CCTV fetch failed: HTTP %s",
|
||||
resp.status_code if resp else "no response",
|
||||
)
|
||||
return []
|
||||
|
||||
data = resp.json()
|
||||
if not isinstance(data, list):
|
||||
return []
|
||||
|
||||
cameras: List[Dict[str, Any]] = []
|
||||
for item in data:
|
||||
if not isinstance(item, dict) or item.get("eventType") != "liveCams":
|
||||
continue
|
||||
geometry = item.get("geometry") if isinstance(item.get("geometry"), dict) else {}
|
||||
coords = geometry.get("coordinates") if isinstance(geometry.get("coordinates"), list) else []
|
||||
if len(coords) < 2:
|
||||
continue
|
||||
try:
|
||||
lon = float(coords[0])
|
||||
lat = float(coords[1])
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
|
||||
props = item.get("properties") if isinstance(item.get("properties"), dict) else {}
|
||||
media_url = _ensure_https_url(str(props.get("href") or "").strip())
|
||||
if not media_url:
|
||||
continue
|
||||
|
||||
cam_id = str(item.get("path") or props.get("id") or len(cameras)).strip("/")
|
||||
label = str(props.get("title") or props.get("headline") or "Australia Camera")[:120]
|
||||
cameras.append(
|
||||
{
|
||||
"id": f"AUS-{cam_id}",
|
||||
"source_agency": "NSW Live Traffic",
|
||||
"lat": lat,
|
||||
"lon": lon,
|
||||
"direction_facing": label,
|
||||
"media_url": media_url,
|
||||
"media_type": "image",
|
||||
"refresh_rate_seconds": 120,
|
||||
}
|
||||
)
|
||||
logger.info("AustraliaLiveTrafficIngestor: parsed %s cameras", len(cameras))
|
||||
return cameras
|
||||
|
||||
|
||||
class NetherlandsRWSIngestor(BaseCCTVIngestor):
|
||||
"""Netherlands Rijkswaterstaat cameras from legacy NDW open-data JSON.
|
||||
|
||||
The opendata.ndw.nu/cameras.json feed Osiris used is often offline; when
|
||||
unavailable this ingestor returns an empty set and logs a warning.
|
||||
"""
|
||||
|
||||
URL = "https://opendata.ndw.nu/cameras.json"
|
||||
MAX_CAMERAS = 1200
|
||||
|
||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||
resp = fetch_with_curl(self.URL, timeout=25, headers={"Accept": "application/json"})
|
||||
if not resp or resp.status_code != 200:
|
||||
logger.warning(
|
||||
"Netherlands RWS cameras.json unavailable (HTTP %s) — "
|
||||
"NDW retired this open-data endpoint; no cameras ingested",
|
||||
resp.status_code if resp else "no response",
|
||||
)
|
||||
return []
|
||||
|
||||
data = resp.json()
|
||||
if not isinstance(data, list):
|
||||
return []
|
||||
|
||||
cameras: List[Dict[str, Any]] = []
|
||||
for i, cam in enumerate(data[: self.MAX_CAMERAS]):
|
||||
if not isinstance(cam, dict):
|
||||
continue
|
||||
lat = cam.get("lat") if cam.get("lat") is not None else cam.get("latitude")
|
||||
lon = cam.get("lng") if cam.get("lng") is not None else cam.get("longitude")
|
||||
media_url = _ensure_https_url(
|
||||
str(cam.get("imageUrl") or cam.get("feed_url") or cam.get("url") or "").strip()
|
||||
)
|
||||
if lat is None or lon is None or not media_url:
|
||||
continue
|
||||
try:
|
||||
lat_f, lon_f = float(lat), float(lon)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
cameras.append(
|
||||
{
|
||||
"id": f"NLRWS-{cam.get('id') or i}",
|
||||
"source_agency": "Rijkswaterstaat",
|
||||
"lat": lat_f,
|
||||
"lon": lon_f,
|
||||
"direction_facing": str(cam.get("name") or "Netherlands Camera")[:120],
|
||||
"media_url": media_url,
|
||||
"media_type": "image",
|
||||
"refresh_rate_seconds": 120,
|
||||
}
|
||||
)
|
||||
logger.info("NetherlandsRWSIngestor: parsed %s cameras", len(cameras))
|
||||
return cameras
|
||||
|
||||
|
||||
def _detect_media_type(url: str) -> str:
|
||||
"""Detect the media type from a camera URL for proper frontend rendering."""
|
||||
if not url:
|
||||
@@ -1113,29 +1409,40 @@ def _detect_media_type(url: str) -> str:
|
||||
return "image"
|
||||
|
||||
|
||||
def scheduled_cctv_ingestors() -> List[tuple["BaseCCTVIngestor", str]]:
|
||||
"""Canonical list of CCTV ingestors for startup, scheduler, and DB seeding."""
|
||||
return [
|
||||
(TFLJamCamIngestor(), "cctv_tfl"),
|
||||
(LTASingaporeIngestor(), "cctv_lta"),
|
||||
(AustinTXIngestor(), "cctv_atx"),
|
||||
(NYCDOTIngestor(), "cctv_nyc"),
|
||||
(CaltransIngestor(), "cctv_caltrans"),
|
||||
(ColoradoDOTIngestor(), "cctv_codot"),
|
||||
(WSDOTIngestor(), "cctv_wsdot"),
|
||||
(GeorgiaDOTIngestor(), "cctv_gdot"),
|
||||
(IllinoisDOTIngestor(), "cctv_idot"),
|
||||
(MichiganDOTIngestor(), "cctv_mdot"),
|
||||
(WindyWebcamsIngestor(), "cctv_windy"),
|
||||
(DGTNationalIngestor(), "cctv_dgt"),
|
||||
(MadridCityIngestor(), "cctv_madrid"),
|
||||
(OSMTrafficCameraIngestor(), "cctv_osm"),
|
||||
(AsfinagIngestor(), "cctv_asfinag"),
|
||||
(OSMALPRCameraIngestor(), "cctv_osm_alpr"),
|
||||
(Ontario511Ingestor(), "cctv_on511"),
|
||||
(Alberta511Ingestor(), "cctv_ab511"),
|
||||
(Florida511Ingestor(), "cctv_fl511"),
|
||||
(AustraliaLiveTrafficIngestor(), "cctv_australia"),
|
||||
(NetherlandsRWSIngestor(), "cctv_nl_rws"),
|
||||
]
|
||||
|
||||
|
||||
def run_all_ingestors():
|
||||
"""Run all CCTV ingestors synchronously. Used for first-run DB seeding."""
|
||||
ingestors = [
|
||||
TFLJamCamIngestor(),
|
||||
LTASingaporeIngestor(),
|
||||
AustinTXIngestor(),
|
||||
NYCDOTIngestor(),
|
||||
CaltransIngestor(),
|
||||
ColoradoDOTIngestor(),
|
||||
WSDOTIngestor(),
|
||||
GeorgiaDOTIngestor(),
|
||||
IllinoisDOTIngestor(),
|
||||
MichiganDOTIngestor(),
|
||||
WindyWebcamsIngestor(),
|
||||
OSMTrafficCameraIngestor(),
|
||||
DGTNationalIngestor(),
|
||||
MadridCityIngestor(),
|
||||
]
|
||||
for ing in ingestors:
|
||||
for ingestor, _name in scheduled_cctv_ingestors():
|
||||
try:
|
||||
ing.ingest()
|
||||
ingestor.ingest()
|
||||
except Exception as e:
|
||||
logger.warning(f"Ingestor {ing.__class__.__name__} failed during seed: {e}")
|
||||
logger.warning(f"Ingestor {ingestor.__class__.__name__} failed during seed: {e}")
|
||||
|
||||
|
||||
def get_all_cameras() -> List[Dict[str, Any]]:
|
||||
|
||||
@@ -101,6 +101,10 @@ from services.fetchers.crowdthreat import fetch_crowdthreat # noqa: F401
|
||||
from services.fetchers.wastewater import fetch_wastewater # noqa: F401
|
||||
from services.fetchers.sar_catalog import fetch_sar_catalog # noqa: F401
|
||||
from services.fetchers.sar_products import fetch_sar_products # noqa: F401
|
||||
from services.fetchers.malware import fetch_malware_threats # noqa: F401
|
||||
from services.fetchers.telegram_osint import fetch_telegram_osint # noqa: F401
|
||||
from services.fetchers.cyber_status import fetch_cyber_threats # noqa: F401
|
||||
from services.scm.suppliers import fetch_scm_suppliers # noqa: F401
|
||||
from services.ais_stream import prune_stale_vessels # noqa: F401
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -480,6 +484,9 @@ def update_slow_data():
|
||||
fetch_fishing_activity,
|
||||
fetch_power_plants,
|
||||
fetch_ukraine_air_raid_alerts,
|
||||
fetch_malware_threats,
|
||||
fetch_cyber_threats,
|
||||
fetch_scm_suppliers,
|
||||
]
|
||||
_run_tasks("slow-tier", slow_funcs)
|
||||
# Run correlation engine after all data is fresh
|
||||
@@ -523,6 +530,15 @@ def _load_cctv_cache_for_startup() -> None:
|
||||
logger.warning("Startup CCTV cache load failed (non-fatal): %s", e)
|
||||
|
||||
|
||||
def _load_static_infrastructure_for_startup() -> None:
|
||||
"""Disk-backed reference layers — instant, no network."""
|
||||
for func in (fetch_datacenters, fetch_military_bases, fetch_power_plants):
|
||||
try:
|
||||
func()
|
||||
except Exception as e:
|
||||
logger.warning("Startup static infrastructure load failed for %s: %s", func.__name__, e)
|
||||
|
||||
|
||||
def _run_delayed_startup_heavy_refresh() -> None:
|
||||
if _STARTUP_HEAVY_REFRESH_DELAY_S > 0:
|
||||
logger.info(
|
||||
@@ -535,6 +551,7 @@ def _run_delayed_startup_heavy_refresh() -> None:
|
||||
"startup-heavy",
|
||||
[
|
||||
update_slow_data,
|
||||
fetch_telegram_osint,
|
||||
fetch_volcanoes,
|
||||
fetch_viirs_change_nodes,
|
||||
fetch_unusual_whales,
|
||||
@@ -573,6 +590,7 @@ def update_all_data(*, startup_mode: bool = False):
|
||||
logger.info("Full data update starting (parallel)...")
|
||||
# Preload Meshtastic map cache immediately (instant, from disk)
|
||||
seed_startup_caches()
|
||||
_load_static_infrastructure_for_startup()
|
||||
with _data_lock:
|
||||
meshtastic_seeded = bool(latest_data.get("meshtastic_map_nodes"))
|
||||
if startup_mode:
|
||||
@@ -649,22 +667,9 @@ def update_all_data(*, startup_mode: bool = False):
|
||||
# (the scheduled job also runs every 10 min for ongoing refresh).
|
||||
if startup_mode:
|
||||
try:
|
||||
from services.cctv_pipeline import (
|
||||
TFLJamCamIngestor, LTASingaporeIngestor, AustinTXIngestor,
|
||||
NYCDOTIngestor, CaltransIngestor, ColoradoDOTIngestor,
|
||||
WSDOTIngestor, GeorgiaDOTIngestor, IllinoisDOTIngestor,
|
||||
MichiganDOTIngestor, WindyWebcamsIngestor, DGTNationalIngestor,
|
||||
MadridCityIngestor, OSMTrafficCameraIngestor, get_all_cameras,
|
||||
)
|
||||
from services.cctv_pipeline import OSMALPRCameraIngestor
|
||||
_startup_ingestors = [
|
||||
TFLJamCamIngestor(), LTASingaporeIngestor(), AustinTXIngestor(),
|
||||
NYCDOTIngestor(), CaltransIngestor(), ColoradoDOTIngestor(),
|
||||
WSDOTIngestor(), GeorgiaDOTIngestor(), IllinoisDOTIngestor(),
|
||||
MichiganDOTIngestor(), WindyWebcamsIngestor(), DGTNationalIngestor(),
|
||||
MadridCityIngestor(), OSMTrafficCameraIngestor(),
|
||||
OSMALPRCameraIngestor(),
|
||||
]
|
||||
from services.cctv_pipeline import get_all_cameras, scheduled_cctv_ingestors
|
||||
|
||||
_startup_ingestors = [ing for ing, _name in scheduled_cctv_ingestors()]
|
||||
logger.info("Running CCTV ingest at startup (%d ingestors)...", len(_startup_ingestors))
|
||||
ingest_futures = {
|
||||
_SHARED_EXECUTOR.submit(ing.ingest): ing.__class__.__name__
|
||||
@@ -800,6 +805,18 @@ def start_scheduler():
|
||||
misfire_grace_time=120,
|
||||
)
|
||||
|
||||
# Telegram OSINT — hourly t.me/s channel scrape (kept off the 5-minute slow tier).
|
||||
_telegram_interval_m = max(15, int(os.environ.get("TELEGRAM_OSINT_INTERVAL_MINUTES", "60")))
|
||||
_scheduler.add_job(
|
||||
lambda: _run_task_with_health(fetch_telegram_osint, "fetch_telegram_osint"),
|
||||
"interval",
|
||||
minutes=_telegram_interval_m,
|
||||
next_run_time=datetime.utcnow() + timedelta(seconds=45),
|
||||
id="telegram_osint",
|
||||
max_instances=1,
|
||||
misfire_grace_time=600,
|
||||
)
|
||||
|
||||
# Prediction markets — own jittered cadence (Polymarket/Kalshi clearnet egress).
|
||||
# Kept off the fixed 5-minute slow tier so poll timing is less fingerprintable.
|
||||
from services.fetchers.prediction_markets import fetch_prediction_markets
|
||||
@@ -938,39 +955,9 @@ def start_scheduler():
|
||||
|
||||
# CCTV pipeline refresh — runs all ingestors, then refreshes in-memory data.
|
||||
# Delay the first run slightly so startup serves cached/DB-backed data first.
|
||||
from services.cctv_pipeline import (
|
||||
TFLJamCamIngestor,
|
||||
LTASingaporeIngestor,
|
||||
AustinTXIngestor,
|
||||
NYCDOTIngestor,
|
||||
CaltransIngestor,
|
||||
ColoradoDOTIngestor,
|
||||
WSDOTIngestor,
|
||||
GeorgiaDOTIngestor,
|
||||
IllinoisDOTIngestor,
|
||||
MichiganDOTIngestor,
|
||||
WindyWebcamsIngestor,
|
||||
DGTNationalIngestor,
|
||||
MadridCityIngestor,
|
||||
OSMTrafficCameraIngestor,
|
||||
)
|
||||
from services.cctv_pipeline import scheduled_cctv_ingestors
|
||||
|
||||
_cctv_ingestors = [
|
||||
(TFLJamCamIngestor(), "cctv_tfl"),
|
||||
(LTASingaporeIngestor(), "cctv_lta"),
|
||||
(AustinTXIngestor(), "cctv_atx"),
|
||||
(NYCDOTIngestor(), "cctv_nyc"),
|
||||
(CaltransIngestor(), "cctv_caltrans"),
|
||||
(ColoradoDOTIngestor(), "cctv_codot"),
|
||||
(WSDOTIngestor(), "cctv_wsdot"),
|
||||
(GeorgiaDOTIngestor(), "cctv_gdot"),
|
||||
(IllinoisDOTIngestor(), "cctv_idot"),
|
||||
(MichiganDOTIngestor(), "cctv_mdot"),
|
||||
(WindyWebcamsIngestor(), "cctv_windy"),
|
||||
(DGTNationalIngestor(), "cctv_dgt"),
|
||||
(MadridCityIngestor(), "cctv_madrid"),
|
||||
(OSMTrafficCameraIngestor(), "cctv_osm"),
|
||||
]
|
||||
_cctv_ingestors = scheduled_cctv_ingestors()
|
||||
|
||||
def _run_cctv_ingest_cycle():
|
||||
from services.fetchers._store import is_any_active
|
||||
|
||||
@@ -46,6 +46,7 @@ _CRITICAL_WARN = {
|
||||
|
||||
_OPTIONAL = {
|
||||
"AIS_API_KEY": "AIS vessel streaming (ships layer will be empty without it)",
|
||||
"GFW_API_TOKEN": "Global Fishing Watch fishing-vessel activity (fishing_activity layer)",
|
||||
"LTA_ACCOUNT_KEY": "Singapore LTA traffic cameras (CCTV layer)",
|
||||
"PUBLIC_API_KEY": "Optional client auth for public endpoints (recommended for exposed deployments)",
|
||||
}
|
||||
|
||||
@@ -70,6 +70,10 @@ class DashboardData(TypedDict, total=False):
|
||||
sar_anomalies: List[Dict[str, Any]]
|
||||
sar_aoi_coverage: List[Dict[str, Any]]
|
||||
road_corridor_trends: Dict[str, Any]
|
||||
malware_threats: Dict[str, Any]
|
||||
cyber_threats: Dict[str, Any]
|
||||
scm_suppliers: Dict[str, Any]
|
||||
telegram_osint: Dict[str, Any]
|
||||
|
||||
|
||||
# In-memory store
|
||||
@@ -121,6 +125,10 @@ latest_data: DashboardData = {
|
||||
"sar_anomalies": [],
|
||||
"sar_aoi_coverage": [],
|
||||
"road_corridor_trends": {"updated_at": None, "corridors": []},
|
||||
"malware_threats": {"threats": [], "total": 0, "timestamp": None},
|
||||
"cyber_threats": {"threats": [], "stats": {}},
|
||||
"scm_suppliers": {"suppliers": [], "total": 0, "critical_count": 0},
|
||||
"telegram_osint": {"posts": [], "total": 0, "geolocated": 0, "timestamp": None},
|
||||
}
|
||||
|
||||
# Per-source freshness timestamps
|
||||
@@ -331,6 +339,11 @@ active_layers: dict[str, bool] = {
|
||||
"crowdthreat": False,
|
||||
"sar": True,
|
||||
"road_corridor_trends": False,
|
||||
"malware_c2": False,
|
||||
"submarine_cables": False,
|
||||
"scm_suppliers": False,
|
||||
"cyber_threats": False,
|
||||
"telegram_osint": True,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
"""CISA KEV + cyber threat stats (Osiris port)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from services.fetchers._store import _data_lock, _mark_fresh, is_any_active, latest_data
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def fetch_cyber_threats() -> dict[str, Any]:
|
||||
if not is_any_active("cyber_threats"):
|
||||
return latest_data.get("cyber_threats") or {"threats": [], "stats": {}}
|
||||
|
||||
results: dict[str, Any] = {"threats": [], "stats": {}, "timestamp": datetime.now(timezone.utc).isoformat()}
|
||||
try:
|
||||
resp = fetch_with_curl(
|
||||
"https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json",
|
||||
timeout=15,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
vulns = data.get("vulnerabilities") or []
|
||||
results["stats"]["cisa_total"] = len(vulns)
|
||||
now = datetime.now(timezone.utc)
|
||||
recent = []
|
||||
for v in vulns:
|
||||
try:
|
||||
added = datetime.fromisoformat(v.get("dateAdded", "").replace("Z", "+00:00"))
|
||||
days = (now - added).total_seconds() / 86400
|
||||
except Exception:
|
||||
continue
|
||||
if days <= 30:
|
||||
recent.append(v)
|
||||
recent = recent[:10]
|
||||
results["threats"] = [
|
||||
{
|
||||
"id": v.get("cveID"),
|
||||
"name": v.get("vulnerabilityName"),
|
||||
"vendor": v.get("vendorProject"),
|
||||
"product": v.get("product"),
|
||||
"severity": "CRITICAL",
|
||||
"date": v.get("dateAdded"),
|
||||
"due": v.get("dueDate"),
|
||||
"source": "CISA KEV",
|
||||
}
|
||||
for v in recent
|
||||
]
|
||||
except Exception as exc:
|
||||
logger.warning("CISA KEV fetch failed: %s", exc)
|
||||
|
||||
count = len(results["threats"])
|
||||
results["stats"]["active_cves"] = count
|
||||
results["stats"]["threat_level"] = "CRITICAL" if count >= 8 else "HIGH" if count >= 4 else "ELEVATED"
|
||||
|
||||
with _data_lock:
|
||||
latest_data["cyber_threats"] = results
|
||||
_mark_fresh("cyber_threats")
|
||||
return results
|
||||
@@ -278,6 +278,16 @@ _FISHING_FETCH_INTERVAL_S = 3600 # once per hour — GFW data has ~5 day lag
|
||||
_last_fishing_fetch_ts: float = 0.0
|
||||
|
||||
|
||||
def _gfw_int_env(name: str, default: int, *, minimum: int = 1, maximum: int | None = None) -> int:
|
||||
try:
|
||||
value = int(os.environ.get(name, str(default)) or default)
|
||||
except (TypeError, ValueError):
|
||||
value = default
|
||||
if maximum is not None:
|
||||
value = min(maximum, value)
|
||||
return max(minimum, value)
|
||||
|
||||
|
||||
@with_retry(max_retries=1, base_delay=5)
|
||||
def fetch_fishing_activity():
|
||||
"""Fetch recent fishing events from Global Fishing Watch (~5 day lag)."""
|
||||
@@ -300,10 +310,16 @@ def fetch_fishing_activity():
|
||||
try:
|
||||
import datetime as _dt
|
||||
|
||||
# GFW publishes with ~5 day lag; windows shorter than ~7 days often return 0 events.
|
||||
lookback_days = _gfw_int_env("GFW_EVENTS_LOOKBACK_DAYS", 7, minimum=1, maximum=14)
|
||||
max_pages = _gfw_int_env("GFW_EVENTS_MAX_PAGES", 10, minimum=1, maximum=100)
|
||||
timeout_s = _gfw_int_env("GFW_EVENTS_TIMEOUT_S", 90, minimum=30, maximum=180)
|
||||
_end = _dt.date.today().isoformat()
|
||||
_start = (_dt.date.today() - _dt.timedelta(days=7)).isoformat()
|
||||
page_size = max(1, int(os.environ.get("GFW_EVENTS_PAGE_SIZE", "500") or "500"))
|
||||
_start = (_dt.date.today() - _dt.timedelta(days=lookback_days)).isoformat()
|
||||
page_size = _gfw_int_env("GFW_EVENTS_PAGE_SIZE", 500, minimum=1, maximum=1000)
|
||||
offset = 0
|
||||
pages_fetched = 0
|
||||
total_available: int | None = None
|
||||
seen_offsets: set[int] = set()
|
||||
seen_ids: set[str] = set()
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
@@ -324,7 +340,7 @@ def fetch_fishing_activity():
|
||||
}
|
||||
)
|
||||
url = f"https://gateway.api.globalfishingwatch.org/v3/events?{query}"
|
||||
response = fetch_with_curl(url, timeout=30, headers=headers)
|
||||
response = fetch_with_curl(url, timeout=timeout_s, headers=headers)
|
||||
if response.status_code != 200:
|
||||
logger.warning(
|
||||
"Fishing activity fetch failed at offset=%s: HTTP %s",
|
||||
@@ -334,10 +350,16 @@ def fetch_fishing_activity():
|
||||
break
|
||||
|
||||
payload = response.json() or {}
|
||||
if total_available is None:
|
||||
try:
|
||||
total_available = int(payload.get("total")) if payload.get("total") is not None else None
|
||||
except (TypeError, ValueError):
|
||||
total_available = None
|
||||
entries = payload.get("entries", [])
|
||||
if not entries:
|
||||
break
|
||||
|
||||
pages_fetched += 1
|
||||
added_this_page = 0
|
||||
for e in entries:
|
||||
pos = e.get("position", {})
|
||||
@@ -372,6 +394,15 @@ def fetch_fishing_activity():
|
||||
if len(entries) < page_size:
|
||||
break
|
||||
|
||||
if pages_fetched >= max_pages:
|
||||
logger.info(
|
||||
"Fishing activity: capped at %s pages (%s events fetched; GFW total=%s)",
|
||||
max_pages,
|
||||
len(events),
|
||||
total_available if total_available is not None else "unknown",
|
||||
)
|
||||
break
|
||||
|
||||
next_offset = payload.get("nextOffset")
|
||||
if next_offset is None:
|
||||
next_offset = (payload.get("pagination") or {}).get("nextOffset")
|
||||
|
||||
@@ -235,11 +235,11 @@ _DC_GEOCODED_PATH = Path(__file__).parent.parent.parent / "data" / "datacenters_
|
||||
|
||||
|
||||
def fetch_datacenters():
|
||||
"""Load geocoded data centers (5K+ street-level precise locations)."""
|
||||
from services.fetchers._store import is_any_active
|
||||
"""Load geocoded data centers (5K+ street-level precise locations).
|
||||
|
||||
if not is_any_active("datacenters"):
|
||||
return
|
||||
Always loads from disk; /api/live-data/slow gates the payload on the
|
||||
datacenters layer toggle so enabling the layer can render immediately.
|
||||
"""
|
||||
dcs = []
|
||||
try:
|
||||
if not _DC_GEOCODED_PATH.exists():
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Malware C2 / URLhaus feed (abuse.ch, Osiris port)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from services.fetchers._store import _data_lock, _mark_fresh, is_any_active, latest_data
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
COUNTRY_CENTROIDS: dict[str, tuple[float, float]] = {
|
||||
"AF": (65, 33), "AL": (20, 41), "DZ": (3, 28), "AR": (-64, -34), "AU": (134, -25),
|
||||
"AT": (14, 47.5), "BE": (4, 50.8), "BR": (-51, -10), "CA": (-96, 62), "CN": (105, 35),
|
||||
"DE": (10, 51), "FR": (2, 46), "GB": (-2, 54), "IN": (79, 22), "IR": (53, 32),
|
||||
"IT": (12.5, 42.8), "JP": (138, 36), "KR": (128, 36), "MX": (-102, 23.5), "NL": (5.5, 52.5),
|
||||
"PL": (19.5, 52), "RU": (100, 60), "SG": (103.8, 1.35), "TW": (121, 23.7), "UA": (32, 49),
|
||||
"US": (-97, 38), "VN": (106, 16),
|
||||
}
|
||||
|
||||
|
||||
def fetch_malware_threats() -> list[dict[str, Any]]:
|
||||
if not is_any_active("malware_c2"):
|
||||
return latest_data.get("malware_threats") or []
|
||||
|
||||
threats: list[dict[str, Any]] = []
|
||||
threat_id = 0
|
||||
|
||||
try:
|
||||
resp = fetch_with_curl(
|
||||
"https://feodotracker.abuse.ch/downloads/ipblocklist.json",
|
||||
timeout=10,
|
||||
headers={"User-Agent": "Shadowbroker/1.0", "Accept": "application/json"},
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
entries = resp.json()
|
||||
if not isinstance(entries, list):
|
||||
entries = []
|
||||
for entry in entries[:200]:
|
||||
cc = entry.get("country")
|
||||
if not cc or cc not in COUNTRY_CENTROIDS:
|
||||
continue
|
||||
lng, lat = COUNTRY_CENTROIDS[cc]
|
||||
j_lng = ((threat_id * 173.7) % 200 - 100) / 100 * 4
|
||||
j_lat = ((threat_id * 293.1) % 200 - 100) / 100 * 4
|
||||
threats.append(
|
||||
{
|
||||
"id": f"feodo-{threat_id}",
|
||||
"lat": lat + j_lat,
|
||||
"lng": lng + j_lng,
|
||||
"ip": entry.get("ip_address") or "unknown",
|
||||
"port": entry.get("dst_port") or 0,
|
||||
"malware": entry.get("malware") or "unknown",
|
||||
"status": entry.get("status") or "active",
|
||||
"first_seen": entry.get("first_seen"),
|
||||
"last_online": entry.get("last_online"),
|
||||
"country": cc,
|
||||
"threat_type": "botnet_c2",
|
||||
}
|
||||
)
|
||||
threat_id += 1
|
||||
except Exception as exc:
|
||||
logger.warning("Feodo fetch failed: %s", exc)
|
||||
|
||||
try:
|
||||
resp = fetch_with_curl(
|
||||
"https://urlhaus-api.abuse.ch/v1/urls/recent/limit/100/",
|
||||
timeout=8,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
urls = (resp.json() or {}).get("urls") or []
|
||||
for u in urls:
|
||||
cc = u.get("country")
|
||||
if not cc or cc not in COUNTRY_CENTROIDS:
|
||||
cc = next(iter(COUNTRY_CENTROIDS))
|
||||
lng, lat = COUNTRY_CENTROIDS[cc]
|
||||
j_lng = ((threat_id * 137.3) % 200 - 100) / 100 * 5
|
||||
j_lat = ((threat_id * 211.7) % 200 - 100) / 100 * 5
|
||||
threats.append(
|
||||
{
|
||||
"id": f"urlhaus-{threat_id}",
|
||||
"lat": lat + j_lat,
|
||||
"lng": lng + j_lng,
|
||||
"ip": u.get("host") or "unknown",
|
||||
"port": 0,
|
||||
"malware": ", ".join(u.get("tags") or []) or u.get("threat") or "malware",
|
||||
"status": u.get("url_status") or "online",
|
||||
"first_seen": u.get("dateadded"),
|
||||
"country": cc,
|
||||
"threat_type": "malware_url",
|
||||
}
|
||||
)
|
||||
threat_id += 1
|
||||
except Exception as exc:
|
||||
logger.debug("URLhaus supplement failed: %s", exc)
|
||||
|
||||
payload = {
|
||||
"threats": threats,
|
||||
"total": len(threats),
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"source": "abuse.ch Feodo Tracker + URLhaus",
|
||||
}
|
||||
with _data_lock:
|
||||
latest_data["malware_threats"] = payload
|
||||
_mark_fresh("malware_threats")
|
||||
return threats
|
||||
@@ -158,21 +158,26 @@ _KEYWORD_COORDS = {
|
||||
_SORTED_KEYWORDS = sorted(_KEYWORD_COORDS.items(), key=lambda x: len(x[0]), reverse=True)
|
||||
|
||||
|
||||
def resolve_coords_match(text: str) -> tuple[tuple[float, float], str] | None:
|
||||
"""Return ((lat, lng), matched_keyword) for the most specific keyword hit."""
|
||||
padded_text = f" {text} "
|
||||
for kw, coords in _SORTED_KEYWORDS:
|
||||
if kw.startswith(" ") or kw.endswith(" "):
|
||||
if kw in padded_text:
|
||||
return coords, kw
|
||||
elif re.search(r"\b" + re.escape(kw) + r"\b", text):
|
||||
return coords, kw
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_coords(text: str) -> tuple[float, float] | None:
|
||||
"""Return (lat, lng) for the most specific keyword match, or None.
|
||||
|
||||
Longer keywords are tried first. Space-padded keywords (" us ", " uk ")
|
||||
use substring matching on padded text; all others use word-boundary regex.
|
||||
"""
|
||||
padded_text = f" {text} "
|
||||
for kw, coords in _SORTED_KEYWORDS:
|
||||
if kw.startswith(" ") or kw.endswith(" "):
|
||||
if kw in padded_text:
|
||||
return coords
|
||||
else:
|
||||
if re.search(r'\b' + re.escape(kw) + r'\b', text):
|
||||
return coords
|
||||
return None
|
||||
match = resolve_coords_match(text)
|
||||
return match[0] if match else None
|
||||
|
||||
|
||||
@with_retry(max_retries=1, base_delay=2)
|
||||
|
||||
@@ -0,0 +1,381 @@
|
||||
"""Telegram OSINT — public channel web previews (t.me/s) with keyword geoparsing."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from services.fetchers._store import _data_lock, _mark_fresh, is_any_active, latest_data
|
||||
from services.fetchers.news import resolve_coords_match
|
||||
from services.network_utils import fetch_with_curl, outbound_user_agent
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_DEFAULT_CHANNELS = (
|
||||
"osintdefender",
|
||||
"insiderpaper",
|
||||
"aljazeeraenglish",
|
||||
"nexta_live",
|
||||
"war_monitor",
|
||||
"OSINTtechnical",
|
||||
"Liveuamap",
|
||||
)
|
||||
|
||||
_MESSAGE_BLOCK_RE = re.compile(
|
||||
r'<div class="tgme_widget_message_wrap js-widget_message_wrap"[\s\S]*?</div>\s*</div>\s*</div>',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_TEXT_RE = re.compile(
|
||||
r'<div class="tgme_widget_message_text[^>]*>([\s\S]*?)</div>',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_DATE_RE = re.compile(
|
||||
r'<a class="tgme_widget_message_date" href="(https://t\.me/[^"]+)".*?<time datetime="([^"]+)"',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_HAS_VIDEO_RE = re.compile(
|
||||
r'tgme_widget_message_video|js-message_video|<video\s',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_HAS_PHOTO_RE = re.compile(r'tgme_widget_message_photo_wrap', re.IGNORECASE)
|
||||
_VIDEO_SRC_RE = re.compile(r'<video[^>]+src="([^"]+)"', re.IGNORECASE)
|
||||
_BG_IMAGE_RE = re.compile(r"background-image:url\('([^']+)'\)", re.IGNORECASE)
|
||||
|
||||
_TELEGRAM_MEDIA_HOST_SUFFIXES = (".telesco.pe", ".telegram-cdn.org")
|
||||
|
||||
# Cyrillic / Arabic aliases for war-reporting channels (merged after English resolver).
|
||||
_EXTRA_PLACE_KEYWORDS: dict[str, tuple[float, float]] = {
|
||||
"киев": (50.450, 30.523),
|
||||
"київ": (50.450, 30.523),
|
||||
"харьков": (49.993, 36.231),
|
||||
"харків": (49.993, 36.231),
|
||||
"одесса": (46.482, 30.724),
|
||||
"одеса": (46.482, 30.724),
|
||||
"донецк": (48.015, 37.803),
|
||||
"донецьк": (48.015, 37.803),
|
||||
"луганск": (48.574, 39.307),
|
||||
"луганськ": (48.574, 39.307),
|
||||
"москва": (55.755, 37.617),
|
||||
"крым": (45.000, 34.000),
|
||||
"крим": (45.000, 34.000),
|
||||
"бахмут": (48.595, 38.000),
|
||||
"запорожье": (47.838, 35.139),
|
||||
"запоріжжя": (47.838, 35.139),
|
||||
"غزة": (31.416, 34.333),
|
||||
"دمشق": (33.513, 36.276),
|
||||
"بيروت": (33.893, 35.501),
|
||||
"tel aviv": (32.085, 34.781),
|
||||
"תל אביב": (32.085, 34.781),
|
||||
}
|
||||
|
||||
# Country-level news geocodes sit on national centroids that stack with threat alerts.
|
||||
# Telegram uses major metro anchors so pins land on a different map cell than news.
|
||||
_TELEGRAM_ANCHOR_OVERRIDES: dict[str, tuple[float, float]] = {
|
||||
"israel": (32.085, 34.781), # Tel Aviv (news uses central Israel ~Jerusalem corridor)
|
||||
"middle east": (32.085, 34.781),
|
||||
"china": (39.904, 116.407), # Beijing (news uses country centroid)
|
||||
"united states": (40.712, -74.006), # New York (news uses Washington DC)
|
||||
"usa": (40.712, -74.006),
|
||||
"us": (40.712, -74.006),
|
||||
"america": (40.712, -74.006),
|
||||
"uk": (51.507, -0.127), # London
|
||||
"iran": (35.689, 51.389), # Tehran
|
||||
"russia": (55.755, 37.617), # Moscow
|
||||
"ukraine": (50.450, 30.523), # Kyiv
|
||||
"france": (48.856, 2.352), # Paris
|
||||
"germany": (52.520, 13.405), # Berlin
|
||||
"lebanon": (34.433, 35.844), # Tripoli (news uses Beirut corridor)
|
||||
}
|
||||
|
||||
_RISK_KEYWORDS = (
|
||||
"war",
|
||||
"missile",
|
||||
"strike",
|
||||
"attack",
|
||||
"crisis",
|
||||
"tension",
|
||||
"military",
|
||||
"conflict",
|
||||
"defense",
|
||||
"clash",
|
||||
"nuclear",
|
||||
"invasion",
|
||||
"bomb",
|
||||
"drone",
|
||||
"weapon",
|
||||
"sanctions",
|
||||
"ceasefire",
|
||||
"escalation",
|
||||
"killed",
|
||||
"destroyed",
|
||||
"operation",
|
||||
"casualty",
|
||||
"frontline",
|
||||
"threat",
|
||||
"explosion",
|
||||
"shelling",
|
||||
)
|
||||
|
||||
|
||||
def telegram_osint_enabled() -> bool:
|
||||
return str(os.environ.get("TELEGRAM_OSINT_ENABLED", "true")).strip().lower() not in {
|
||||
"0",
|
||||
"false",
|
||||
"no",
|
||||
"off",
|
||||
"",
|
||||
}
|
||||
|
||||
|
||||
def _configured_channels() -> list[str]:
|
||||
raw = str(os.environ.get("TELEGRAM_OSINT_CHANNELS", "")).strip()
|
||||
if raw:
|
||||
return [part.strip().lstrip("@") for part in raw.split(",") if part.strip()]
|
||||
return list(_DEFAULT_CHANNELS)
|
||||
|
||||
|
||||
def telegram_media_host_allowed(hostname: str | None) -> bool:
|
||||
host = str(hostname or "").strip().lower()
|
||||
if not host:
|
||||
return False
|
||||
return any(host.endswith(suffix) for suffix in _TELEGRAM_MEDIA_HOST_SUFFIXES)
|
||||
|
||||
|
||||
def _extract_media(block: str, link: str) -> dict[str, Any]:
|
||||
has_video = bool(_HAS_VIDEO_RE.search(block))
|
||||
has_photo = bool(_HAS_PHOTO_RE.search(block))
|
||||
media_type: str | None = None
|
||||
media_url: str | None = None
|
||||
if has_video:
|
||||
media_type = "video"
|
||||
video_match = _VIDEO_SRC_RE.search(block)
|
||||
if video_match:
|
||||
media_url = video_match.group(1).strip()
|
||||
elif has_photo:
|
||||
media_type = "photo"
|
||||
photo_match = _BG_IMAGE_RE.search(block)
|
||||
if photo_match:
|
||||
media_url = photo_match.group(1).strip()
|
||||
|
||||
embed_url: str | None = None
|
||||
if media_type and link:
|
||||
embed_url = f"{link}?embed=1"
|
||||
|
||||
return {
|
||||
"media_type": media_type,
|
||||
"media_url": media_url,
|
||||
"embed_url": embed_url,
|
||||
}
|
||||
|
||||
|
||||
def _strip_html(text: str) -> str:
|
||||
cleaned = re.sub(r"<br\s*/?>", "\n", text, flags=re.IGNORECASE)
|
||||
cleaned = re.sub(r"<[^>]+>", "", cleaned)
|
||||
return (
|
||||
cleaned.replace(""", '"')
|
||||
.replace("&", "&")
|
||||
.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
.strip()
|
||||
)
|
||||
|
||||
|
||||
def _score_risk(text: str) -> int:
|
||||
lower = text.lower()
|
||||
score = 1
|
||||
for kw in _RISK_KEYWORDS:
|
||||
if kw in lower:
|
||||
score += 2
|
||||
return min(10, score)
|
||||
|
||||
|
||||
def _refresh_post_coords(post: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Re-apply geoparsing so stored posts pick up anchor updates."""
|
||||
text = "\n".join(
|
||||
str(part).strip()
|
||||
for part in (post.get("title"), post.get("description"))
|
||||
if part and str(part).strip()
|
||||
)
|
||||
if not text:
|
||||
return post
|
||||
coords = _resolve_telegram_coords(text)
|
||||
if not coords:
|
||||
return post
|
||||
updated = dict(post)
|
||||
updated["coords"] = [coords[0], coords[1]]
|
||||
return updated
|
||||
|
||||
|
||||
def _resolve_telegram_coords(text: str) -> tuple[float, float] | None:
|
||||
lower = text.lower()
|
||||
match = resolve_coords_match(lower)
|
||||
if match:
|
||||
_coords, keyword = match
|
||||
anchor = _TELEGRAM_ANCHOR_OVERRIDES.get(keyword.strip().lower())
|
||||
if anchor:
|
||||
return anchor
|
||||
return _coords
|
||||
for keyword, coords in sorted(_EXTRA_PLACE_KEYWORDS.items(), key=lambda x: len(x[0]), reverse=True):
|
||||
if keyword in lower:
|
||||
return coords
|
||||
return None
|
||||
|
||||
|
||||
def _post_link(post: dict[str, Any]) -> str:
|
||||
return str(post.get("link") or "").strip()
|
||||
|
||||
|
||||
def _extract_new_channel_posts(
|
||||
html: str,
|
||||
channel: str,
|
||||
known_links: set[str],
|
||||
*,
|
||||
bootstrap_limit: int = 12,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return unseen posts from a channel page; stop once we hit a stored link."""
|
||||
parsed = parse_telegram_channel_html(html, channel)
|
||||
if not parsed:
|
||||
return []
|
||||
if not known_links:
|
||||
return parsed[-bootstrap_limit:]
|
||||
|
||||
fresh: list[dict[str, Any]] = []
|
||||
for post in reversed(parsed):
|
||||
link = _post_link(post)
|
||||
if not link:
|
||||
continue
|
||||
if link in known_links:
|
||||
break
|
||||
fresh.append(post)
|
||||
fresh.reverse()
|
||||
return fresh
|
||||
|
||||
|
||||
def _merge_telegram_posts(
|
||||
existing: list[dict[str, Any]],
|
||||
incoming: list[dict[str, Any]],
|
||||
*,
|
||||
max_posts: int = 120,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
known_links = {_post_link(post) for post in existing if _post_link(post)}
|
||||
added = 0
|
||||
for post in incoming:
|
||||
link = _post_link(post)
|
||||
if not link or link in known_links:
|
||||
continue
|
||||
known_links.add(link)
|
||||
existing.append(post)
|
||||
added += 1
|
||||
existing.sort(key=lambda p: str(p.get("published") or ""), reverse=True)
|
||||
return existing[:max_posts], added
|
||||
|
||||
|
||||
def parse_telegram_channel_html(html: str, channel: str) -> list[dict[str, Any]]:
|
||||
"""Parse public t.me/s channel preview HTML into post dicts."""
|
||||
posts: list[dict[str, Any]] = []
|
||||
for block in _MESSAGE_BLOCK_RE.findall(html or ""):
|
||||
text_match = _TEXT_RE.search(block)
|
||||
if not text_match:
|
||||
continue
|
||||
text = _strip_html(text_match.group(1))
|
||||
if len(text) < 10:
|
||||
continue
|
||||
|
||||
date_match = _DATE_RE.search(block)
|
||||
link = date_match.group(1) if date_match else f"https://t.me/{channel}"
|
||||
published = date_match.group(2) if date_match else datetime.now(timezone.utc).isoformat()
|
||||
title = text.split("\n", 1)[0][:160]
|
||||
risk_score = _score_risk(text)
|
||||
coords = _resolve_telegram_coords(text)
|
||||
post_id = hashlib.sha1(f"{link}|{published}".encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
media = _extract_media(block, link)
|
||||
posts.append(
|
||||
{
|
||||
"id": post_id,
|
||||
"title": title,
|
||||
"description": text[:1200],
|
||||
"link": link,
|
||||
"published": published,
|
||||
"source": f"t.me/{channel}",
|
||||
"channel": channel,
|
||||
"risk_score": risk_score,
|
||||
"coords": [coords[0], coords[1]] if coords else None,
|
||||
**media,
|
||||
}
|
||||
)
|
||||
return posts
|
||||
|
||||
|
||||
def fetch_telegram_osint() -> dict[str, Any]:
|
||||
if not is_any_active("telegram_osint"):
|
||||
return latest_data.get("telegram_osint") or {"posts": [], "total": 0, "timestamp": None}
|
||||
|
||||
if not telegram_osint_enabled():
|
||||
with _data_lock:
|
||||
latest_data["telegram_osint"] = {"posts": [], "total": 0, "timestamp": None, "disabled": True}
|
||||
_mark_fresh("telegram_osint")
|
||||
return latest_data["telegram_osint"]
|
||||
|
||||
headers = {
|
||||
"User-Agent": (
|
||||
f"Mozilla/5.0 (compatible; {outbound_user_agent('telegram-osint')}) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": "text/html,application/xhtml+xml",
|
||||
}
|
||||
|
||||
with _data_lock:
|
||||
prior = latest_data.get("telegram_osint") or {}
|
||||
existing_posts = list(prior.get("posts") or [])
|
||||
|
||||
known_links = {_post_link(post) for post in existing_posts if _post_link(post)}
|
||||
incoming: list[dict[str, Any]] = []
|
||||
|
||||
for channel in _configured_channels():
|
||||
url = f"https://t.me/s/{channel}"
|
||||
try:
|
||||
resp = fetch_with_curl(url, timeout=15, headers=headers)
|
||||
if not resp or resp.status_code != 200:
|
||||
logger.warning(
|
||||
"Telegram channel %s fetch failed: HTTP %s",
|
||||
channel,
|
||||
resp.status_code if resp else "no response",
|
||||
)
|
||||
continue
|
||||
channel_new = _extract_new_channel_posts(resp.text, channel, known_links)
|
||||
for post in channel_new:
|
||||
link = _post_link(post)
|
||||
if not link or link in known_links:
|
||||
continue
|
||||
known_links.add(link)
|
||||
incoming.append(post)
|
||||
except Exception as exc:
|
||||
logger.warning("Telegram channel %s parse failed: %s", channel, exc)
|
||||
|
||||
merged_posts, added = _merge_telegram_posts(existing_posts, incoming)
|
||||
merged_posts = [_refresh_post_coords(post) for post in merged_posts]
|
||||
geolocated = sum(1 for p in merged_posts if p.get("coords"))
|
||||
|
||||
payload = {
|
||||
"posts": merged_posts,
|
||||
"total": len(merged_posts),
|
||||
"geolocated": geolocated,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"channels": _configured_channels(),
|
||||
"last_fetch_new": added,
|
||||
}
|
||||
|
||||
with _data_lock:
|
||||
latest_data["telegram_osint"] = payload
|
||||
_mark_fresh("telegram_osint")
|
||||
logger.info(
|
||||
"Telegram OSINT: +%s new, %s retained (%s geolocated)",
|
||||
added,
|
||||
len(merged_posts),
|
||||
geolocated,
|
||||
)
|
||||
return payload
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Country risk index (static scores + USGS quake enrichment)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
RISK_FACTORS: dict[str, dict[str, Any]] = {
|
||||
"UA": {"base": 85, "tags": ["active_conflict", "infrastructure_damage"]},
|
||||
"RU": {"base": 72, "tags": ["sanctions", "military_mobilization"]},
|
||||
"IL": {"base": 78, "tags": ["active_conflict", "regional_instability"]},
|
||||
"PS": {"base": 90, "tags": ["active_conflict", "humanitarian_crisis"]},
|
||||
"SY": {"base": 82, "tags": ["post_conflict", "infrastructure_damage"]},
|
||||
"YE": {"base": 88, "tags": ["active_conflict", "humanitarian_crisis"]},
|
||||
"MM": {"base": 76, "tags": ["civil_unrest", "military_junta"]},
|
||||
"SD": {"base": 84, "tags": ["active_conflict", "humanitarian_crisis"]},
|
||||
"AF": {"base": 80, "tags": ["post_conflict", "governance_collapse"]},
|
||||
"KP": {"base": 70, "tags": ["nuclear_risk", "isolation"]},
|
||||
"IR": {"base": 68, "tags": ["sanctions", "nuclear_program", "regional_proxy"]},
|
||||
"CN": {"base": 35, "tags": ["strategic_competition", "taiwan_tensions"]},
|
||||
"TW": {"base": 45, "tags": ["invasion_risk", "semiconductor_dependency"]},
|
||||
"VE": {"base": 60, "tags": ["economic_collapse", "political_instability"]},
|
||||
"HT": {"base": 85, "tags": ["gang_violence", "governance_collapse"]},
|
||||
"LB": {"base": 65, "tags": ["economic_crisis", "political_deadlock"]},
|
||||
"PK": {"base": 55, "tags": ["terrorism", "political_instability"]},
|
||||
"SO": {"base": 82, "tags": ["terrorism", "state_fragility"]},
|
||||
"LY": {"base": 72, "tags": ["divided_government", "militia_control"]},
|
||||
"ET": {"base": 62, "tags": ["ethnic_tensions", "regional_conflicts"]},
|
||||
}
|
||||
|
||||
EXCHANGES = [
|
||||
{"name": "NYSE", "tz": "America/New_York", "open": 9.5, "close": 16, "country": "US"},
|
||||
{"name": "NASDAQ", "tz": "America/New_York", "open": 9.5, "close": 16, "country": "US"},
|
||||
{"name": "LSE", "tz": "Europe/London", "open": 8, "close": 16.5, "country": "GB"},
|
||||
{"name": "TSE", "tz": "Asia/Tokyo", "open": 9, "close": 15, "country": "JP"},
|
||||
{"name": "SSE", "tz": "Asia/Shanghai", "open": 9.5, "close": 15, "country": "CN"},
|
||||
{"name": "HKEX", "tz": "Asia/Hong_Kong", "open": 9.5, "close": 16, "country": "HK"},
|
||||
{"name": "FRA", "tz": "Europe/Berlin", "open": 8, "close": 20, "country": "DE"},
|
||||
{"name": "TSX", "tz": "America/Toronto", "open": 9.5, "close": 16, "country": "CA"},
|
||||
{"name": "MOEX", "tz": "Europe/Moscow", "open": 10, "close": 18.5, "country": "RU"},
|
||||
]
|
||||
|
||||
|
||||
def _exchange_open(ex: dict[str, Any]) -> bool:
|
||||
try:
|
||||
now = datetime.now(ZoneInfo(ex["tz"]))
|
||||
if now.weekday() >= 5:
|
||||
return False
|
||||
decimal = now.hour + now.minute / 60
|
||||
return ex["open"] <= decimal < ex["close"]
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def build_country_risk_payload() -> dict[str, Any]:
|
||||
quake_risks: dict[str, float] = {}
|
||||
try:
|
||||
resp = fetch_with_curl(
|
||||
"https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/4.5_day.geojson",
|
||||
timeout=5,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
for f in resp.json().get("features") or []:
|
||||
place = (f.get("properties") or {}).get("place") or ""
|
||||
mag = (f.get("properties") or {}).get("mag") or 0
|
||||
for code in RISK_FACTORS:
|
||||
if code.lower() in place.lower():
|
||||
quake_risks[code] = quake_risks.get(code, 0) + mag
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
countries = []
|
||||
for code, data in RISK_FACTORS.items():
|
||||
base = data["base"]
|
||||
score = min(100, base + quake_risks.get(code, 0))
|
||||
countries.append(
|
||||
{
|
||||
"code": code,
|
||||
"risk_score": score,
|
||||
"risk_level": "CRITICAL" if base >= 80 else "HIGH" if base >= 60 else "ELEVATED" if base >= 40 else "LOW",
|
||||
"tags": data["tags"],
|
||||
}
|
||||
)
|
||||
countries.sort(key=lambda c: c["risk_score"], reverse=True)
|
||||
exchanges = [{"name": e["name"], "country": e["country"], "open": _exchange_open(e)} for e in EXCHANGES]
|
||||
return {
|
||||
"countries": countries,
|
||||
"exchanges": exchanges,
|
||||
"open_exchanges": sum(1 for e in exchanges if e["open"]),
|
||||
"total_exchanges": len(exchanges),
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
"""Operator-initiated OSINT lookups (server-side proxies)."""
|
||||
@@ -0,0 +1,492 @@
|
||||
"""Server-side OSINT lookups (Osiris port, HTTPS outbound only)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import socket
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from services.network_utils import fetch_with_curl
|
||||
from services.sanctions.ofac import match_exact, search_sanctions
|
||||
from services.ssrf_guard import safe_get, validate_domain, validate_host
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_IPV4_RE = re.compile(r"^(\d{1,3}\.){3}\d{1,3}$")
|
||||
_IPV6_RE = re.compile(r"^[0-9a-fA-F:]+$")
|
||||
_CVE_RE = re.compile(r"^CVE-\d{4}-\d{4,}$", re.I)
|
||||
_ASN_RE = re.compile(r"^(AS)?\d+$", re.I)
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _json_get(url: str, *, timeout: float = 8.0, headers: dict[str, str] | None = None) -> Any:
|
||||
resp = fetch_with_curl(url, timeout=timeout, headers=headers or {"Accept": "application/json"})
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
try:
|
||||
return resp.json()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _sanctions_hits(*values: str) -> list[dict[str, Any]] | None:
|
||||
hits: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for value in values:
|
||||
if not value or value in seen:
|
||||
continue
|
||||
seen.add(value)
|
||||
entries = match_exact(value)
|
||||
if entries:
|
||||
hits.append({"matched_value": value, "entries": entries})
|
||||
return hits or None
|
||||
|
||||
|
||||
def lookup_ip(ip: str) -> dict[str, Any]:
|
||||
if not _IPV4_RE.match(ip) and not _IPV6_RE.match(ip):
|
||||
raise ValueError("Invalid IP format")
|
||||
check = validate_host(ip.strip("[]"))
|
||||
if not check.get("ok"):
|
||||
raise ValueError(check.get("reason", "blocked IP"))
|
||||
|
||||
results: dict[str, Any] = {"ip": ip, "timestamp": _now_iso()}
|
||||
fields = (
|
||||
"status,message,continent,country,countryCode,region,regionName,city,zip,"
|
||||
"lat,lon,timezone,isp,org,as,asname,mobile,proxy,hosting,query"
|
||||
)
|
||||
geo = _json_get(f"https://ip-api.com/json/{quote(ip)}?fields={fields}", timeout=5)
|
||||
if isinstance(geo, dict) and geo.get("status") == "success":
|
||||
results["geo"] = {
|
||||
"country": geo.get("country"),
|
||||
"country_code": geo.get("countryCode"),
|
||||
"region": geo.get("regionName"),
|
||||
"city": geo.get("city"),
|
||||
"lat": geo.get("lat"),
|
||||
"lon": geo.get("lon"),
|
||||
"timezone": geo.get("timezone"),
|
||||
"isp": geo.get("isp"),
|
||||
"org": geo.get("org"),
|
||||
"as_number": geo.get("as"),
|
||||
"as_name": geo.get("asname"),
|
||||
"is_mobile": geo.get("mobile"),
|
||||
"is_proxy": geo.get("proxy"),
|
||||
"is_hosting": geo.get("hosting"),
|
||||
}
|
||||
results["reputation"] = {
|
||||
"is_proxy": bool(geo.get("proxy")),
|
||||
"is_hosting": bool(geo.get("hosting")),
|
||||
"is_mobile": bool(geo.get("mobile")),
|
||||
"risk_level": "HIGH" if geo.get("proxy") else "MEDIUM" if geo.get("hosting") else "LOW",
|
||||
}
|
||||
sm = _sanctions_hits(geo.get("org") or "", geo.get("isp") or "", geo.get("asname") or "")
|
||||
if sm:
|
||||
results["sanctions_match"] = {"source": "OFAC SDN", "hits": sm}
|
||||
return results
|
||||
|
||||
|
||||
def lookup_dns(domain: str) -> dict[str, Any]:
|
||||
if not validate_domain(domain):
|
||||
raise ValueError("Invalid domain format")
|
||||
results: dict[str, Any] = {"domain": domain, "records": {}, "timestamp": _now_iso()}
|
||||
for rtype in ("A", "AAAA", "MX", "NS", "TXT", "CNAME", "SOA"):
|
||||
data = _json_get(
|
||||
f"https://dns.google/resolve?name={quote(domain)}&type={rtype}",
|
||||
timeout=5,
|
||||
)
|
||||
answers = []
|
||||
if isinstance(data, dict):
|
||||
for ans in data.get("Answer") or []:
|
||||
answers.append(
|
||||
{
|
||||
"name": ans.get("name"),
|
||||
"type": ans.get("type"),
|
||||
"ttl": ans.get("TTL"),
|
||||
"data": ans.get("data"),
|
||||
}
|
||||
)
|
||||
results["records"][rtype] = answers
|
||||
a_records = results["records"].get("A") or []
|
||||
mx_records = results["records"].get("MX") or []
|
||||
ns_records = results["records"].get("NS") or []
|
||||
results["summary"] = {
|
||||
"ip_addresses": [r["data"] for r in a_records if r.get("data")],
|
||||
"mail_servers": [r["data"] for r in mx_records if r.get("data")],
|
||||
"nameservers": [r["data"] for r in ns_records if r.get("data")],
|
||||
"total_records": sum(len(v) for v in results["records"].values()),
|
||||
}
|
||||
return results
|
||||
|
||||
|
||||
def lookup_whois(domain: str) -> dict[str, Any]:
|
||||
if not validate_domain(domain):
|
||||
raise ValueError("Invalid domain format")
|
||||
results: dict[str, Any] = {"domain": domain, "timestamp": _now_iso()}
|
||||
rdap = _json_get(f"https://rdap.org/domain/{quote(domain)}", timeout=8)
|
||||
if isinstance(rdap, dict):
|
||||
entities = []
|
||||
for ent in rdap.get("entities") or []:
|
||||
vcard = ent.get("vcardArray")
|
||||
name = org = None
|
||||
if isinstance(vcard, list) and len(vcard) > 1:
|
||||
for row in vcard[1]:
|
||||
if row[0] == "fn":
|
||||
name = row[3]
|
||||
if row[0] == "org":
|
||||
org = row[3]
|
||||
if name or org:
|
||||
entities.append({"handle": ent.get("handle"), "roles": ent.get("roles"), "name": name, "org": org})
|
||||
events = [
|
||||
{"action": e.get("eventAction"), "date": e.get("eventDate")}
|
||||
for e in (rdap.get("events") or [])
|
||||
]
|
||||
results["rdap"] = {
|
||||
"handle": rdap.get("handle"),
|
||||
"name": rdap.get("ldhName"),
|
||||
"status": rdap.get("status"),
|
||||
"events": events,
|
||||
"nameservers": [ns.get("ldhName") for ns in (rdap.get("nameservers") or [])],
|
||||
"entities": entities,
|
||||
}
|
||||
results["registration"] = next((e["date"] for e in events if e["action"] == "registration"), None)
|
||||
results["expiration"] = next((e["date"] for e in events if e["action"] == "expiration"), None)
|
||||
results["last_changed"] = next((e["date"] for e in events if e["action"] == "last changed"), None)
|
||||
sm = _sanctions_hits(*(e.get("name") or "" for e in entities), *(e.get("org") or "" for e in entities))
|
||||
if sm:
|
||||
results["sanctions_match"] = {"source": "OFAC SDN", "hits": sm}
|
||||
|
||||
try:
|
||||
res = safe_get(f"https://{domain}", timeout=5, headers={"User-Agent": "Shadowbroker-OSINT/1.0"})
|
||||
headers = {}
|
||||
for h in (
|
||||
"server",
|
||||
"x-powered-by",
|
||||
"x-frame-options",
|
||||
"strict-transport-security",
|
||||
"content-security-policy",
|
||||
"x-content-type-options",
|
||||
"x-xss-protection",
|
||||
"referrer-policy",
|
||||
"permissions-policy",
|
||||
):
|
||||
val = res.headers.get(h)
|
||||
if val:
|
||||
headers[h] = val
|
||||
score = sum(
|
||||
1
|
||||
for k in (
|
||||
"strict-transport-security",
|
||||
"content-security-policy",
|
||||
"x-frame-options",
|
||||
"x-content-type-options",
|
||||
"referrer-policy",
|
||||
)
|
||||
if k in headers
|
||||
) + (2 if "strict-transport-security" in headers else 0) + (2 if "content-security-policy" in headers else 0)
|
||||
results["http"] = {"status": res.status_code, "headers": headers, "final_url": res.url}
|
||||
results["security_score"] = {
|
||||
"score": score,
|
||||
"max": 7,
|
||||
"grade": "A" if score >= 5 else "B" if score >= 3 else "C" if score >= 1 else "F",
|
||||
}
|
||||
except Exception as exc:
|
||||
logger.debug("WHOIS header probe failed for %s: %s", domain, exc)
|
||||
return results
|
||||
|
||||
|
||||
def lookup_certs(domain: str) -> dict[str, Any]:
|
||||
if not validate_domain(domain):
|
||||
raise ValueError("Invalid domain format")
|
||||
resp = fetch_with_curl(
|
||||
f"https://crt.sh/?q=%25.{quote(domain)}&output=json",
|
||||
timeout=10,
|
||||
headers={"User-Agent": "Shadowbroker-OSINT/1.0"},
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
return {"domain": domain, "certificates": [], "error": "crt.sh unavailable"}
|
||||
try:
|
||||
certs = resp.json()
|
||||
except Exception:
|
||||
certs = []
|
||||
seen: set[str] = set()
|
||||
subdomains: set[str] = set()
|
||||
unique: list[dict[str, Any]] = []
|
||||
for cert in (certs or [])[:200]:
|
||||
key = f"{cert.get('common_name')}-{cert.get('serial_number')}"
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
for name in (cert.get("name_value") or "").split("\n"):
|
||||
clean = name.strip().replace("*.", "")
|
||||
if clean.endswith(domain):
|
||||
subdomains.add(clean)
|
||||
unique.append(
|
||||
{
|
||||
"id": cert.get("id"),
|
||||
"issuer": cert.get("issuer_name"),
|
||||
"common_name": cert.get("common_name"),
|
||||
"not_before": cert.get("not_before"),
|
||||
"not_after": cert.get("not_after"),
|
||||
}
|
||||
)
|
||||
return {
|
||||
"domain": domain,
|
||||
"certificates": unique[:50],
|
||||
"subdomains": sorted(subdomains)[:100],
|
||||
"total_found": len(certs or []),
|
||||
"timestamp": _now_iso(),
|
||||
}
|
||||
|
||||
|
||||
def lookup_threats(query: str | None = None) -> dict[str, Any]:
|
||||
results: dict[str, Any] = {"timestamp": _now_iso()}
|
||||
pulses = _json_get("https://otx.alienvault.com/api/v1/pulses/activity?limit=10", timeout=8)
|
||||
if isinstance(pulses, dict):
|
||||
results["pulses"] = [
|
||||
{
|
||||
"name": p.get("name"),
|
||||
"description": (p.get("description") or "")[:200],
|
||||
"created": p.get("created"),
|
||||
"tags": (p.get("tags") or [])[:5],
|
||||
"adversary": p.get("adversary"),
|
||||
"indicators_count": p.get("indicator_count"),
|
||||
}
|
||||
for p in (pulses.get("results") or [])[:10]
|
||||
]
|
||||
if query:
|
||||
if _IPV4_RE.match(query):
|
||||
try:
|
||||
tor_resp = fetch_with_curl("https://check.torproject.org/torbulkexitlist", timeout=5)
|
||||
results["tor_exit_node"] = query in (tor_resp.text or "").splitlines() if tor_resp.status_code == 200 else None
|
||||
except Exception:
|
||||
results["tor_exit_node"] = None
|
||||
otx = _json_get(f"https://otx.alienvault.com/api/v1/indicators/IPv4/{quote(query)}/general", timeout=5)
|
||||
if isinstance(otx, dict):
|
||||
results["otx"] = {
|
||||
"reputation": otx.get("reputation"),
|
||||
"pulse_count": (otx.get("pulse_info") or {}).get("count", 0),
|
||||
"country": otx.get("country_name"),
|
||||
"asn": otx.get("asn"),
|
||||
}
|
||||
elif validate_domain(query):
|
||||
otx = _json_get(f"https://otx.alienvault.com/api/v1/indicators/domain/{quote(query)}/general", timeout=5)
|
||||
if isinstance(otx, dict):
|
||||
results["otx"] = {"pulse_count": (otx.get("pulse_info") or {}).get("count", 0)}
|
||||
pulse_count = (results.get("otx") or {}).get("pulse_count", 0)
|
||||
results["threat_level"] = "HIGH" if pulse_count > 5 else "MEDIUM" if pulse_count > 0 else "LOW"
|
||||
return results
|
||||
|
||||
|
||||
def lookup_bgp(query: str) -> dict[str, Any]:
|
||||
results: dict[str, Any] = {"query": query, "timestamp": _now_iso()}
|
||||
if _IPV4_RE.match(query):
|
||||
data = _json_get(f"https://api.bgpview.io/ip/{quote(query)}", timeout=8)
|
||||
if isinstance(data, dict) and data.get("status") == "ok":
|
||||
results["ip"] = data.get("data")
|
||||
results["type"] = "ip"
|
||||
return results
|
||||
if _ASN_RE.match(query):
|
||||
asn_num = re.sub(r"^AS", "", query, flags=re.I)
|
||||
asn = _json_get(f"https://api.bgpview.io/asn/{asn_num}", timeout=8)
|
||||
prefixes = _json_get(f"https://api.bgpview.io/asn/{asn_num}/prefixes", timeout=8)
|
||||
peers = _json_get(f"https://api.bgpview.io/asn/{asn_num}/peers", timeout=8)
|
||||
if isinstance(asn, dict) and asn.get("status") == "ok":
|
||||
results["asn"] = asn.get("data")
|
||||
if isinstance(prefixes, dict) and prefixes.get("status") == "ok":
|
||||
pdata = prefixes.get("data") or {}
|
||||
results["prefixes"] = {
|
||||
"ipv4": (pdata.get("ipv4_prefixes") or [])[:20],
|
||||
"ipv6": (pdata.get("ipv6_prefixes") or [])[:10],
|
||||
"total_v4": len(pdata.get("ipv4_prefixes") or []),
|
||||
"total_v6": len(pdata.get("ipv6_prefixes") or []),
|
||||
}
|
||||
if isinstance(peers, dict) and peers.get("status") == "ok":
|
||||
pdata = peers.get("data") or {}
|
||||
results["peers"] = {
|
||||
"upstream": (pdata.get("ipv4_peers") or [])[:10],
|
||||
"total": len(pdata.get("ipv4_peers") or []),
|
||||
}
|
||||
results["type"] = "asn"
|
||||
return results
|
||||
raise ValueError("Unrecognized query format. Use IP address or AS number.")
|
||||
|
||||
|
||||
def lookup_sanctions(query: str, *, schema: str | None = None, limit: int = 25) -> dict[str, Any]:
|
||||
matches = search_sanctions(query, schema=schema, limit=limit)
|
||||
return {
|
||||
"query": query,
|
||||
"schema": schema,
|
||||
"total": len(matches),
|
||||
"matches": matches,
|
||||
"source": "OpenSanctions / US OFAC SDN",
|
||||
"timestamp": _now_iso(),
|
||||
}
|
||||
|
||||
|
||||
def lookup_cve(cve: str) -> dict[str, Any]:
|
||||
if not _CVE_RE.match(cve):
|
||||
raise ValueError("Invalid CVE format")
|
||||
cve_id = cve.upper()
|
||||
data = _json_get(f"https://cveawg.mitre.org/api/cve/{quote(cve_id)}", timeout=8)
|
||||
if isinstance(data, dict) and data.get("cveMetadata"):
|
||||
meta = data["cveMetadata"]
|
||||
desc = ""
|
||||
for block in (data.get("containers") or {}).get("cna", {}).get("descriptions") or []:
|
||||
if block.get("lang") == "en":
|
||||
desc = block.get("value") or desc
|
||||
return {"id": meta.get("cveId", cve_id), "description": desc or "No description.", "timestamp": _now_iso()}
|
||||
fallback = _json_get(f"https://cve.circl.lu/api/cve/{quote(cve_id)}", timeout=8)
|
||||
if isinstance(fallback, dict):
|
||||
return {
|
||||
"id": fallback.get("id", cve_id),
|
||||
"description": fallback.get("summary") or "No description.",
|
||||
"cvss": fallback.get("cvss"),
|
||||
"references": (fallback.get("references") or [])[:5],
|
||||
"timestamp": _now_iso(),
|
||||
}
|
||||
raise ValueError("CVE not found")
|
||||
|
||||
|
||||
def lookup_mac(mac: str) -> dict[str, Any]:
|
||||
clean = mac.strip().upper()
|
||||
clean = re.sub(r"[^A-F0-9:-]", "", clean)
|
||||
data = _json_get(f"https://api.macvendors.com/{quote(clean)}", timeout=8)
|
||||
if isinstance(data, dict):
|
||||
return {"mac": clean, "vendor": data.get("company") or data.get("organization") or "Not Found"}
|
||||
if isinstance(data, str) and data:
|
||||
return {"mac": clean, "vendor": data}
|
||||
return {"mac": clean, "vendor": "Not Found"}
|
||||
|
||||
|
||||
def lookup_github(username: str) -> dict[str, Any]:
|
||||
user = _json_get(f"https://api.github.com/users/{quote(username)}", timeout=8)
|
||||
if not isinstance(user, dict) or user.get("message") == "Not Found":
|
||||
raise ValueError("GitHub user not found")
|
||||
repos = _json_get(f"https://api.github.com/users/{quote(username)}/repos?per_page=10&sort=updated", timeout=8)
|
||||
return {
|
||||
"username": username,
|
||||
"profile": {
|
||||
"name": user.get("name"),
|
||||
"bio": user.get("bio"),
|
||||
"company": user.get("company"),
|
||||
"location": user.get("location"),
|
||||
"public_repos": user.get("public_repos"),
|
||||
"followers": user.get("followers"),
|
||||
"created_at": user.get("created_at"),
|
||||
"html_url": user.get("html_url"),
|
||||
},
|
||||
"repos": [
|
||||
{"name": r.get("name"), "language": r.get("language"), "stars": r.get("stargazers_count")}
|
||||
for r in (repos or [])[:10]
|
||||
if isinstance(r, dict)
|
||||
],
|
||||
"timestamp": _now_iso(),
|
||||
}
|
||||
|
||||
|
||||
def lookup_leaks(email: str) -> dict[str, Any]:
|
||||
if "@" not in email or len(email) < 5:
|
||||
raise ValueError("Invalid email")
|
||||
# HIBP requires API key for v3; use public breach directory style via leak-lookup (rate limited)
|
||||
data = _json_get(f"https://leakcheck.io/api/public?check={quote(email)}", timeout=8)
|
||||
if isinstance(data, dict):
|
||||
return {
|
||||
"email": email,
|
||||
"found": bool(data.get("found")),
|
||||
"sources": data.get("sources") or [],
|
||||
"timestamp": _now_iso(),
|
||||
}
|
||||
return {"email": email, "found": False, "sources": [], "timestamp": _now_iso()}
|
||||
|
||||
|
||||
def sweep_init(ip: str, cidr: int = 24) -> dict[str, Any]:
|
||||
try:
|
||||
addr = ipaddress.IPv4Address(ip)
|
||||
except ValueError as exc:
|
||||
raise ValueError("Invalid IPv4 address format") from exc
|
||||
if addr.is_private or addr.is_loopback or addr.is_link_local or addr.is_reserved:
|
||||
raise ValueError("Private and reserved IP ranges are not allowed")
|
||||
if cidr < 24 or cidr > 32:
|
||||
raise ValueError("CIDR must be between 24 and 32")
|
||||
|
||||
fields = "status,message,country,countryCode,region,regionName,city,lat,lon,isp,org,as,proxy,hosting"
|
||||
geo = _json_get(f"https://ip-api.com/json/{quote(ip)}?fields={fields}", timeout=5)
|
||||
if not isinstance(geo, dict) or geo.get("status") != "success":
|
||||
raise ValueError(f"Geolocation failed: {(geo or {}).get('message', 'unknown')}")
|
||||
return {
|
||||
"center": {
|
||||
"lat": geo.get("lat"),
|
||||
"lng": geo.get("lon"),
|
||||
"city": geo.get("city"),
|
||||
"region": geo.get("regionName"),
|
||||
"country": geo.get("country"),
|
||||
"countryCode": geo.get("countryCode"),
|
||||
"isp": geo.get("isp"),
|
||||
"asn": geo.get("as") or "",
|
||||
"org": geo.get("org") or "",
|
||||
},
|
||||
"target_ip": ip,
|
||||
"cidr": cidr,
|
||||
}
|
||||
|
||||
|
||||
def _internetdb_lookup(ip: str) -> dict[str, Any] | None:
|
||||
try:
|
||||
resp = fetch_with_curl(
|
||||
f"https://internetdb.shodan.io/{quote(ip)}",
|
||||
timeout=4,
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
if resp.status_code == 404:
|
||||
return None
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
return resp.json()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def sweep_scan(subnet_start: str, cidr: int, *, max_workers: int = 12) -> dict[str, Any]:
|
||||
"""Scan a /24-/32 via Shodan InternetDB (server-side proxy)."""
|
||||
base = int(ipaddress.IPv4Address(subnet_start))
|
||||
host_count = 2 ** (32 - cidr)
|
||||
if host_count > 256:
|
||||
raise ValueError("Subnet too large")
|
||||
ips = [str(ipaddress.IPv4Address(base + i)) for i in range(host_count)]
|
||||
devices: list[dict[str, Any]] = []
|
||||
t0 = time.time()
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as pool:
|
||||
futures = {pool.submit(_internetdb_lookup, ip): ip for ip in ips}
|
||||
for fut in as_completed(futures):
|
||||
ip = futures[fut]
|
||||
data = fut.result()
|
||||
if not data:
|
||||
continue
|
||||
devices.append(
|
||||
{
|
||||
"ip": data.get("ip") or ip,
|
||||
"ports": data.get("ports") or [],
|
||||
"hostnames": data.get("hostnames") or [],
|
||||
"cpes": data.get("cpes") or [],
|
||||
"vulns": data.get("vulns") or [],
|
||||
"tags": data.get("tags") or [],
|
||||
}
|
||||
)
|
||||
return {
|
||||
"devices": devices,
|
||||
"summary": {"total_hosts": host_count, "total_responsive": len(devices)},
|
||||
"sweep_time_ms": int((time.time() - t0) * 1000),
|
||||
}
|
||||
|
||||
|
||||
def subnet_start_for(ip: str, cidr: int) -> str:
|
||||
net = ipaddress.IPv4Network(f"{ip}/{cidr}", strict=False)
|
||||
return str(net.network_address)
|
||||
@@ -0,0 +1 @@
|
||||
"""Entity graph resolution (Osiris intel layer port)."""
|
||||
@@ -0,0 +1,268 @@
|
||||
"""Entity graph resolver (Python port of Osiris intel/server.js)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from services.network_utils import fetch_with_curl
|
||||
from services.sanctions.ofac import match_exact, search_sanctions
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ALLOWED_TYPES = frozenset({"aircraft", "vessel", "company", "person", "ip", "country"})
|
||||
_WD_CACHE: dict[str, tuple[float, dict[str, Any]]] = {}
|
||||
_WD_LOCK = threading.Lock()
|
||||
_WD_TTL = 24 * 60 * 60
|
||||
_WD_UA = "Shadowbroker-Intel/1.0 (ontology engine)"
|
||||
|
||||
|
||||
def _dedup(nodes: list[dict], links: list[dict]) -> dict[str, Any]:
|
||||
node_map: dict[str, dict] = {}
|
||||
for n in nodes:
|
||||
node_map[n["id"]] = n
|
||||
seen_links: set[str] = set()
|
||||
out_links: list[dict] = []
|
||||
for link in links:
|
||||
key = f"{link['source']}→{link['target']}→{link['label']}"
|
||||
if key in seen_links:
|
||||
continue
|
||||
seen_links.add(key)
|
||||
out_links.append(link)
|
||||
return {"nodes": list(node_map.values()), "links": out_links}
|
||||
|
||||
|
||||
def _wd_cache_get(key: str) -> dict[str, Any] | None:
|
||||
with _WD_LOCK:
|
||||
entry = _WD_CACHE.get(key)
|
||||
if not entry:
|
||||
return None
|
||||
ts, data = entry
|
||||
if time.time() - ts > _WD_TTL:
|
||||
_WD_CACHE.pop(key, None)
|
||||
return None
|
||||
return data
|
||||
|
||||
|
||||
def _wd_cache_set(key: str, data: dict[str, Any]) -> None:
|
||||
with _WD_LOCK:
|
||||
if len(_WD_CACHE) > 5000:
|
||||
oldest = next(iter(_WD_CACHE))
|
||||
_WD_CACHE.pop(oldest, None)
|
||||
_WD_CACHE[key] = (time.time(), data)
|
||||
|
||||
|
||||
def _add_sanctions(id_label: str, root_id: str, nodes: list, links: list) -> None:
|
||||
for hit in search_sanctions(id_label, limit=3):
|
||||
sid = f"sanction:{hit['id']}"
|
||||
nodes.append(
|
||||
{
|
||||
"id": sid,
|
||||
"label": hit["name"],
|
||||
"type": "sanction",
|
||||
"properties": {"programs": hit.get("programs"), "source": "OFAC SDN"},
|
||||
}
|
||||
)
|
||||
links.append({"source": root_id, "target": sid, "label": "SANCTIONS MATCH"})
|
||||
|
||||
|
||||
def _sparql(query: str) -> list[dict[str, Any]]:
|
||||
url = f"https://query.wikidata.org/sparql?query={quote(query)}&format=json"
|
||||
resp = fetch_with_curl(url, timeout=10, headers={"User-Agent": _WD_UA, "Accept": "application/sparql-results+json"})
|
||||
if resp.status_code != 200:
|
||||
return []
|
||||
try:
|
||||
data = resp.json()
|
||||
except Exception:
|
||||
return []
|
||||
return data.get("results", {}).get("bindings", [])
|
||||
|
||||
|
||||
def _wd_search(label: str) -> str | None:
|
||||
url = (
|
||||
"https://www.wikidata.org/w/api.php?action=wbsearchentities"
|
||||
f"&search={quote(label)}&language=en&limit=1&format=json"
|
||||
)
|
||||
resp = fetch_with_curl(url, timeout=5, headers={"User-Agent": _WD_UA})
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
try:
|
||||
hits = resp.json().get("search") or []
|
||||
except Exception:
|
||||
return None
|
||||
return hits[0]["id"] if hits else None
|
||||
|
||||
|
||||
def _resolve_ip(id_value: str) -> dict[str, Any]:
|
||||
cache_key = f"ip:{id_value}"
|
||||
cached = _wd_cache_get(cache_key)
|
||||
if cached:
|
||||
return cached
|
||||
|
||||
root_id = f"ip:{id_value}"
|
||||
nodes: list[dict] = [{"id": root_id, "label": id_value, "type": "ip", "properties": {}}]
|
||||
links: list[dict] = []
|
||||
|
||||
geo = fetch_with_curl(
|
||||
f"https://ip-api.com/json/{quote(id_value)}"
|
||||
"?fields=status,country,countryCode,city,lat,lon,isp,org,as,asname,proxy,hosting,mobile",
|
||||
timeout=8,
|
||||
)
|
||||
if geo.status_code == 200:
|
||||
try:
|
||||
data = geo.json()
|
||||
except Exception:
|
||||
data = {}
|
||||
if data.get("status") == "success":
|
||||
nodes[0]["properties"] = {
|
||||
"proxy": bool(data.get("proxy")),
|
||||
"hosting": bool(data.get("hosting")),
|
||||
"mobile": bool(data.get("mobile")),
|
||||
"source": "ip-api.com",
|
||||
}
|
||||
if data.get("isp"):
|
||||
iid = f"company:{data['isp']}"
|
||||
nodes.append({"id": iid, "label": data["isp"], "type": "company", "properties": {"role": "ISP"}})
|
||||
links.append({"source": root_id, "target": iid, "label": "HOSTED_BY"})
|
||||
if data.get("country"):
|
||||
cid = f"country:{data['country']}"
|
||||
nodes.append(
|
||||
{
|
||||
"id": cid,
|
||||
"label": data["country"],
|
||||
"type": "country",
|
||||
"properties": {"code": data.get("countryCode")},
|
||||
}
|
||||
)
|
||||
links.append({"source": root_id, "target": cid, "label": "LOCATED_IN"})
|
||||
for val in (data.get("isp"), data.get("org"), data.get("asname")):
|
||||
if val:
|
||||
for entry in match_exact(val):
|
||||
sid = f"sanction:{entry['id']}"
|
||||
nodes.append({"id": sid, "label": entry["name"], "type": "sanction", "properties": {}})
|
||||
links.append({"source": root_id, "target": sid, "label": "SANCTIONS MATCH"})
|
||||
|
||||
whois = fetch_with_curl(
|
||||
f"https://stat.ripe.net/data/whois/data.json?resource={quote(id_value)}",
|
||||
timeout=8,
|
||||
)
|
||||
if whois.status_code == 200:
|
||||
try:
|
||||
records = whois.json().get("data", {}).get("records") or []
|
||||
except Exception:
|
||||
records = []
|
||||
for record in records:
|
||||
for field in record:
|
||||
if field.get("key") in ("netname", "NetName"):
|
||||
nid = f"company:{field['value']}"
|
||||
nodes.append({"id": nid, "label": field["value"], "type": "company", "properties": {"role": "Network"}})
|
||||
links.append({"source": root_id, "target": nid, "label": "HOSTED_BY"})
|
||||
|
||||
result = _dedup(nodes, links)
|
||||
_wd_cache_set(cache_key, result)
|
||||
return result
|
||||
|
||||
|
||||
def _resolve_company(id_value: str) -> dict[str, Any]:
|
||||
cache_key = f"company:{id_value}"
|
||||
cached = _wd_cache_get(cache_key)
|
||||
if cached:
|
||||
return cached
|
||||
root_id = f"company:{id_value}"
|
||||
nodes = [{"id": root_id, "label": id_value, "type": "company", "properties": {}}]
|
||||
links: list[dict] = []
|
||||
safe = re.sub(r'[^a-zA-Z0-9 \-._]', '', id_value).strip()
|
||||
qid = _wd_search(safe)
|
||||
filt = f"VALUES ?item {{ wd:{qid} }}" if qid else f'?item rdfs:label "{safe}"@en . ?item wdt:P31/wdt:P279* wd:Q4830453 .'
|
||||
rows = _sparql(
|
||||
f"""
|
||||
SELECT ?countryLabel ?parentLabel ?ceoLabel WHERE {{
|
||||
{filt}
|
||||
OPTIONAL {{ ?item wdt:P17 ?country . }}
|
||||
OPTIONAL {{ ?item wdt:P749 ?parent . }}
|
||||
OPTIONAL {{ ?item wdt:P169 ?ceo . }}
|
||||
SERVICE wikibase:label {{ bd:serviceParam wikibase:language "en" . }}
|
||||
}} LIMIT 10
|
||||
"""
|
||||
)
|
||||
for row in rows:
|
||||
if row.get("countryLabel", {}).get("value"):
|
||||
cid = f"country:{row['countryLabel']['value']}"
|
||||
nodes.append({"id": cid, "label": row["countryLabel"]["value"], "type": "country", "properties": {}})
|
||||
links.append({"source": root_id, "target": cid, "label": "HEADQUARTERED"})
|
||||
if row.get("parentLabel", {}).get("value"):
|
||||
pid = f"company:{row['parentLabel']['value']}"
|
||||
nodes.append({"id": pid, "label": row["parentLabel"]["value"], "type": "company", "properties": {}})
|
||||
links.append({"source": root_id, "target": pid, "label": "PARENT ORG"})
|
||||
if row.get("ceoLabel", {}).get("value"):
|
||||
pid = f"person:{row['ceoLabel']['value']}"
|
||||
nodes.append({"id": pid, "label": row["ceoLabel"]["value"], "type": "person", "properties": {"role": "CEO"}})
|
||||
links.append({"source": root_id, "target": pid, "label": "CEO"})
|
||||
_add_sanctions(id_value, root_id, nodes, links)
|
||||
result = _dedup(nodes, links)
|
||||
_wd_cache_set(cache_key, result)
|
||||
return result
|
||||
|
||||
|
||||
def _resolve_from_store(entity_type: str, id_value: str, props: dict[str, Any]) -> dict[str, Any]:
|
||||
from services.fetchers._store import get_latest_data_subset_refs
|
||||
|
||||
root_id = f"{entity_type}:{id_value}"
|
||||
nodes = [{"id": root_id, "label": props.get("label") or id_value, "type": entity_type, "properties": props}]
|
||||
links: list[dict] = []
|
||||
data = get_latest_data_subset_refs("flights", "ships", "military_flights", "tracked_flights")
|
||||
|
||||
if entity_type == "aircraft":
|
||||
icao = (props.get("icao24") or id_value).lower()
|
||||
for bucket in ("military_flights", "tracked_flights", "flights"):
|
||||
for f in data.get(bucket) or []:
|
||||
if str(f.get("icao24", "")).lower() == icao:
|
||||
if f.get("country"):
|
||||
cid = f"country:{f['country']}"
|
||||
nodes.append({"id": cid, "label": f["country"], "type": "country", "properties": {}})
|
||||
links.append({"source": root_id, "target": cid, "label": "REGISTERED_IN"})
|
||||
if f.get("registration"):
|
||||
nodes[0]["properties"]["registration"] = f["registration"]
|
||||
break
|
||||
elif entity_type == "vessel":
|
||||
mmsi = str(props.get("mmsi") or id_value)
|
||||
for ship in data.get("ships") or []:
|
||||
if str(ship.get("mmsi")) == mmsi:
|
||||
if ship.get("country"):
|
||||
cid = f"country:{ship['country']}"
|
||||
nodes.append({"id": cid, "label": ship["country"], "type": "country", "properties": {}})
|
||||
links.append({"source": root_id, "target": cid, "label": "FLAG"})
|
||||
break
|
||||
_add_sanctions(id_value, root_id, nodes, links)
|
||||
return _dedup(nodes, links)
|
||||
|
||||
|
||||
def resolve_entity(entity_type: str, id_value: str, properties: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
etype = (entity_type or "").lower().strip()
|
||||
eid = (id_value or "").strip()
|
||||
if etype not in ALLOWED_TYPES:
|
||||
raise ValueError(f"Invalid type. Allowed: {', '.join(sorted(ALLOWED_TYPES))}")
|
||||
if len(eid) < 2 or len(eid) > 200:
|
||||
raise ValueError("Invalid id (2-200 chars)")
|
||||
props = properties or {}
|
||||
|
||||
if etype == "ip":
|
||||
return _resolve_ip(eid)
|
||||
if etype in ("company", "person", "country"):
|
||||
if etype == "company":
|
||||
return _resolve_company(eid)
|
||||
if etype == "person":
|
||||
root_id = f"person:{eid}"
|
||||
nodes = [{"id": root_id, "label": eid, "type": "person", "properties": {}}]
|
||||
links: list[dict] = []
|
||||
_add_sanctions(eid, root_id, nodes, links)
|
||||
return _dedup(nodes, links)
|
||||
root_id = f"country:{eid}"
|
||||
nodes = [{"id": root_id, "label": eid, "type": "country", "properties": {}}]
|
||||
links = []
|
||||
_add_sanctions(eid, root_id, nodes, links)
|
||||
return _dedup(nodes, links)
|
||||
return _resolve_from_store(etype, eid, props)
|
||||
@@ -0,0 +1 @@
|
||||
"""Sanctions screening (OpenSanctions OFAC SDN)."""
|
||||
@@ -0,0 +1,154 @@
|
||||
"""OFAC SDN index via OpenSanctions (adapted from Osiris sanctions.ts)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SDN_CSV_URL = "https://data.opensanctions.org/datasets/latest/us_ofac_sdn/targets.simple.csv"
|
||||
TTL_S = 24 * 60 * 60
|
||||
|
||||
_lock = threading.Lock()
|
||||
_cache: dict[str, Any] | None = None
|
||||
_cache_at: float = 0.0
|
||||
_inflight: threading.Event | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SanctionEntry:
|
||||
id: str
|
||||
schema: str
|
||||
name: str
|
||||
aliases: list[str] = field(default_factory=list)
|
||||
countries: list[str] = field(default_factory=list)
|
||||
programs: list[str] = field(default_factory=list)
|
||||
sanctions: str = ""
|
||||
first_seen: str | None = None
|
||||
last_seen: str | None = None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"id": self.id,
|
||||
"schema": self.schema,
|
||||
"name": self.name,
|
||||
"aliases": self.aliases,
|
||||
"countries": self.countries,
|
||||
"programs": self.programs,
|
||||
"sanctions": self.sanctions,
|
||||
"first_seen": self.first_seen,
|
||||
"last_seen": self.last_seen,
|
||||
}
|
||||
|
||||
|
||||
def norm_name(s: str) -> str:
|
||||
s = re.sub(r"[^\w\s]+", " ", s.lower(), flags=re.UNICODE)
|
||||
return re.sub(r"\s+", " ", s).strip()
|
||||
|
||||
|
||||
def _split_semi(val: str) -> list[str]:
|
||||
return [x.strip() for x in (val or "").split(";") if x.strip()]
|
||||
|
||||
|
||||
def _load_list() -> dict[str, Any]:
|
||||
global _cache, _cache_at
|
||||
with _lock:
|
||||
if _cache and (time.time() - _cache_at) < TTL_S:
|
||||
return _cache
|
||||
|
||||
try:
|
||||
resp = fetch_with_curl(SDN_CSV_URL, timeout=45, headers={"Accept": "text/csv"})
|
||||
if resp.status_code != 200:
|
||||
raise RuntimeError(f"OpenSanctions HTTP {resp.status_code}")
|
||||
text = resp.text
|
||||
reader = csv.DictReader(io.StringIO(text))
|
||||
entries: list[SanctionEntry] = []
|
||||
by_norm: dict[str, list[SanctionEntry]] = {}
|
||||
for row in reader:
|
||||
name = (row.get("name") or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
entry = SanctionEntry(
|
||||
id=row.get("id") or "",
|
||||
schema=row.get("schema") or "LegalEntity",
|
||||
name=name,
|
||||
aliases=_split_semi(row.get("aliases") or ""),
|
||||
countries=_split_semi(row.get("countries") or ""),
|
||||
programs=_split_semi(row.get("program_ids") or ""),
|
||||
sanctions=row.get("sanctions") or "",
|
||||
first_seen=row.get("first_seen") or None,
|
||||
last_seen=row.get("last_seen") or None,
|
||||
)
|
||||
entries.append(entry)
|
||||
for key in {norm_name(name), *(norm_name(a) for a in entry.aliases)}:
|
||||
if not key:
|
||||
continue
|
||||
by_norm.setdefault(key, []).append(entry)
|
||||
loaded = {"entries": entries, "by_norm": by_norm, "fetched_at": time.time()}
|
||||
with _lock:
|
||||
_cache = loaded
|
||||
_cache_at = time.time()
|
||||
logger.info("OFAC SDN index loaded: %s entries", len(entries))
|
||||
return loaded
|
||||
except Exception as exc:
|
||||
logger.error("OFAC SDN load failed: %s", exc)
|
||||
with _lock:
|
||||
if _cache:
|
||||
return _cache
|
||||
raise
|
||||
|
||||
|
||||
def match_exact(query: str) -> list[dict[str, Any]]:
|
||||
if not query or len(query) < 3:
|
||||
return []
|
||||
data = _load_list()
|
||||
hits = data["by_norm"].get(norm_name(query), [])
|
||||
return [e.to_dict() for e in hits]
|
||||
|
||||
|
||||
def search_sanctions(query: str, *, schema: str | None = None, limit: int = 50) -> list[dict[str, Any]]:
|
||||
if not query or len(query) < 4:
|
||||
return []
|
||||
data = _load_list()
|
||||
q = norm_name(query)
|
||||
exact_name: list[SanctionEntry] = []
|
||||
exact_alias: list[SanctionEntry] = []
|
||||
sub_name: list[SanctionEntry] = []
|
||||
sub_alias: list[SanctionEntry] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def push(bucket: list[SanctionEntry], entry: SanctionEntry) -> None:
|
||||
if entry.id in seen:
|
||||
return
|
||||
if schema and entry.schema != schema:
|
||||
return
|
||||
seen.add(entry.id)
|
||||
bucket.append(entry)
|
||||
|
||||
for entry in data["entries"]:
|
||||
name_norm = norm_name(entry.name)
|
||||
if name_norm == q:
|
||||
push(exact_name, entry)
|
||||
elif any(norm_name(a) == q for a in entry.aliases):
|
||||
push(exact_alias, entry)
|
||||
elif q in name_norm:
|
||||
push(sub_name, entry)
|
||||
elif any(q in norm_name(a) for a in entry.aliases):
|
||||
push(sub_alias, entry)
|
||||
if len(seen) >= limit * 4:
|
||||
break
|
||||
|
||||
ordered = exact_name + exact_alias + sub_name + sub_alias
|
||||
return [e.to_dict() for e in ordered[:limit]]
|
||||
|
||||
|
||||
def index_size() -> int:
|
||||
return len(_load_list()["entries"])
|
||||
@@ -0,0 +1 @@
|
||||
"""Supply-chain risk overlay."""
|
||||
@@ -0,0 +1,154 @@
|
||||
"""SCM supplier risk overlay (Osiris port, uses in-memory dashboard data)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from services.fetchers._store import _data_lock, _mark_fresh, get_latest_data_subset_refs, is_any_active, latest_data
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
SUPPLIERS: list[dict[str, Any]] = [
|
||||
{"id": "sup-tsmc-hsinchu", "name": "TSMC Fab 12 (Tier 1)", "city": "Hsinchu", "country": "Taiwan", "lat": 24.774, "lng": 120.992, "category": "Semiconductor"},
|
||||
{"id": "sup-tsmc-tainan", "name": "TSMC Fab 14 (Tier 1)", "city": "Tainan", "country": "Taiwan", "lat": 23.111, "lng": 120.273, "category": "Semiconductor"},
|
||||
{"id": "sup-sec-giheung", "name": "Samsung Electronics (Tier 1)", "city": "Giheung", "country": "South Korea", "lat": 37.221, "lng": 127.098, "category": "Semiconductor"},
|
||||
{"id": "sup-sk-icheon", "name": "SK Hynix (Tier 1)", "city": "Icheon", "country": "South Korea", "lat": 37.256, "lng": 127.483, "category": "Semiconductor"},
|
||||
{"id": "sup-sony-kumamoto", "name": "Sony Semiconductor (Tier 2)", "city": "Kikuyo", "country": "Japan", "lat": 32.883, "lng": 130.825, "category": "Electronics"},
|
||||
{"id": "sup-mlcc-murata", "name": "Murata MLCC (Tier 2)", "city": "Izumo", "country": "Japan", "lat": 35.361, "lng": 132.756, "category": "Electronics"},
|
||||
{"id": "sup-bosch-stuttgart", "name": "Bosch Auto Parts (Tier 1)", "city": "Stuttgart", "country": "Germany", "lat": 48.815, "lng": 9.176, "category": "Automotive"},
|
||||
{"id": "sup-zf-bavaria", "name": "ZF Friedrichshafen (Tier 1)", "city": "Friedrichshafen", "country": "Germany", "lat": 47.662, "lng": 9.489, "category": "Automotive"},
|
||||
{"id": "sup-valeo-paris", "name": "Valeo R&D (Tier 2)", "city": "Paris", "country": "France", "lat": 48.878, "lng": 2.308, "category": "Automotive"},
|
||||
{"id": "sup-magna-celaya", "name": "Magna Assembly (Tier 2)", "city": "Celaya", "country": "Mexico", "lat": 20.525, "lng": -100.814, "category": "Automotive"},
|
||||
{"id": "sup-denso-monterrey", "name": "Denso Corp (Tier 1)", "city": "Monterrey", "country": "Mexico", "lat": 25.772, "lng": -100.174, "category": "Automotive"},
|
||||
{"id": "sup-catl-ningde", "name": "CATL Battery HQ (Tier 1)", "city": "Ningde", "country": "China", "lat": 26.666, "lng": 119.544, "category": "Battery"},
|
||||
{"id": "sup-byd-shenzhen", "name": "BYD Gigafactory (Tier 1)", "city": "Shenzhen", "country": "China", "lat": 22.684, "lng": 114.341, "category": "Battery"},
|
||||
{"id": "sup-panasonic-nevada", "name": "Panasonic Giga (Tier 1)", "city": "Sparks", "country": "US", "lat": 39.539, "lng": -119.439, "category": "Battery"},
|
||||
]
|
||||
|
||||
|
||||
def _distance_km(lat1: float, lng1: float, lat2: float, lng2: float) -> float:
|
||||
dx = (lng1 - lng2) * math.cos(math.radians((lat1 + lat2) / 2))
|
||||
dy = lat1 - lat2
|
||||
return math.sqrt(dx * dx + dy * dy) * 111.32
|
||||
|
||||
|
||||
def _seismic_risk_level(distance_km: float, magnitude: float) -> str | None:
|
||||
"""Meaningful fab impact only — ignore routine micro-quakes (e.g. Taiwan M3.x)."""
|
||||
if magnitude < 4.5:
|
||||
return None
|
||||
if magnitude >= 6.0 and distance_km <= 200:
|
||||
return "CRITICAL"
|
||||
if magnitude >= 5.5 and distance_km <= 75:
|
||||
return "CRITICAL"
|
||||
if magnitude >= 5.0 and distance_km <= 100:
|
||||
return "HIGH"
|
||||
if magnitude >= 4.5 and distance_km <= 40:
|
||||
return "HIGH"
|
||||
return None
|
||||
|
||||
|
||||
def _apply_seismic_threats(suppliers: list[dict[str, Any]], earthquakes: list[dict[str, Any]]) -> None:
|
||||
for sup in suppliers:
|
||||
best: tuple[str, float] | None = None
|
||||
for eq in earthquakes:
|
||||
lat = eq.get("lat")
|
||||
lng = eq.get("lng") or eq.get("lon")
|
||||
mag = float(eq.get("mag") or eq.get("magnitude") or 0)
|
||||
if lat is None or lng is None or mag < 4.5:
|
||||
continue
|
||||
dist = _distance_km(sup["lat"], sup["lng"], float(lat), float(lng))
|
||||
level = _seismic_risk_level(dist, mag)
|
||||
if not level:
|
||||
continue
|
||||
severity = {"HIGH": 1, "CRITICAL": 2}
|
||||
if best is None:
|
||||
best = (level, mag)
|
||||
else:
|
||||
cur = severity[level]
|
||||
prev = severity[best[0]]
|
||||
if cur > prev or (cur == prev and mag > best[1]):
|
||||
best = (level, mag)
|
||||
if best:
|
||||
level, mag = best
|
||||
if sup["risk_level"] == "NORMAL" or (
|
||||
level == "CRITICAL" and sup["risk_level"] != "CRITICAL"
|
||||
):
|
||||
sup["risk_level"] = level
|
||||
elif level == "CRITICAL" and sup["risk_level"] == "HIGH":
|
||||
sup["risk_level"] = "CRITICAL"
|
||||
sup["active_threats"].append(f"SEISMIC PROXIMITY (M{mag:.1f})")
|
||||
|
||||
|
||||
def build_scm_payload() -> dict[str, Any]:
|
||||
suppliers = [{**s, "risk_level": "NORMAL", "active_threats": []} for s in SUPPLIERS]
|
||||
refs = get_latest_data_subset_refs("earthquakes", "firms_fires", "gdelt")
|
||||
|
||||
earthquakes = refs.get("earthquakes") or []
|
||||
_apply_seismic_threats(suppliers, earthquakes)
|
||||
|
||||
fires = refs.get("firms_fires") or []
|
||||
for sup in suppliers:
|
||||
count = 0
|
||||
for fire in fires:
|
||||
lat = fire.get("lat") or fire.get("latitude")
|
||||
lng = fire.get("lng") or fire.get("lon") or fire.get("longitude")
|
||||
if lat is None or lng is None:
|
||||
continue
|
||||
if _distance_km(sup["lat"], sup["lng"], float(lat), float(lng)) < 50:
|
||||
count += 1
|
||||
if count:
|
||||
if sup["risk_level"] == "NORMAL":
|
||||
sup["risk_level"] = "HIGH"
|
||||
sup["active_threats"].append(f"WILDFIRE PROXIMITY ({count} hotspots)")
|
||||
|
||||
conflicts = refs.get("gdelt") or []
|
||||
for sup in suppliers:
|
||||
for event in conflicts:
|
||||
lat = event.get("lat")
|
||||
lng = event.get("lng") or event.get("lon")
|
||||
if lat is None or lng is None:
|
||||
continue
|
||||
if _distance_km(sup["lat"], sup["lng"], float(lat), float(lng)) < 100:
|
||||
sup["risk_level"] = "CRITICAL"
|
||||
sup["active_threats"].append("ARMED CONFLICT / RIOT")
|
||||
break
|
||||
|
||||
# USGS fallback if earthquakes empty
|
||||
if not earthquakes:
|
||||
try:
|
||||
resp = fetch_with_curl(
|
||||
"https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/4.5_day.geojson",
|
||||
timeout=5,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
features = resp.json().get("features") or []
|
||||
usgs_quakes = [
|
||||
{
|
||||
"lat": f.get("geometry", {}).get("coordinates", [None, None])[1],
|
||||
"lng": f.get("geometry", {}).get("coordinates", [None, None])[0],
|
||||
"mag": f.get("properties", {}).get("mag") or 0,
|
||||
}
|
||||
for f in features
|
||||
if len(f.get("geometry", {}).get("coordinates") or []) >= 2
|
||||
]
|
||||
_apply_seismic_threats(suppliers, usgs_quakes)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
critical = sum(1 for s in suppliers if s["risk_level"] == "CRITICAL")
|
||||
return {
|
||||
"suppliers": suppliers,
|
||||
"total": len(suppliers),
|
||||
"critical_count": critical,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def fetch_scm_suppliers() -> dict[str, Any]:
|
||||
if not is_any_active("scm_suppliers"):
|
||||
return latest_data.get("scm_suppliers") or {}
|
||||
payload = build_scm_payload()
|
||||
with _data_lock:
|
||||
latest_data["scm_suppliers"] = payload
|
||||
_mark_fresh("scm_suppliers")
|
||||
return payload
|
||||
@@ -0,0 +1,141 @@
|
||||
"""SSRF guard for operator-initiated recon (ported from Osiris ssrf-guard.ts)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import re
|
||||
import socket
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import requests
|
||||
|
||||
_IPV4_BLOCKS = [
|
||||
ipaddress.ip_network("0.0.0.0/8"),
|
||||
ipaddress.ip_network("10.0.0.0/8"),
|
||||
ipaddress.ip_network("100.64.0.0/10"),
|
||||
ipaddress.ip_network("127.0.0.0/8"),
|
||||
ipaddress.ip_network("169.254.0.0/16"),
|
||||
ipaddress.ip_network("172.16.0.0/12"),
|
||||
ipaddress.ip_network("192.0.0.0/24"),
|
||||
ipaddress.ip_network("192.0.2.0/24"),
|
||||
ipaddress.ip_network("192.168.0.0/16"),
|
||||
ipaddress.ip_network("198.18.0.0/15"),
|
||||
ipaddress.ip_network("198.51.100.0/24"),
|
||||
ipaddress.ip_network("203.0.113.0/24"),
|
||||
ipaddress.ip_network("224.0.0.0/4"),
|
||||
ipaddress.ip_network("240.0.0.0/4"),
|
||||
]
|
||||
|
||||
_NAME_BLOCKLIST = (
|
||||
re.compile(r"^localhost$", re.I),
|
||||
re.compile(r"\.localhost$", re.I),
|
||||
re.compile(r"^host\.docker\.internal$", re.I),
|
||||
re.compile(r"\.local$", re.I),
|
||||
re.compile(r"\.internal$", re.I),
|
||||
re.compile(r"^metadata\.google\.internal$", re.I),
|
||||
)
|
||||
|
||||
_HOSTNAME_RE = re.compile(
|
||||
r"^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?)*$"
|
||||
)
|
||||
|
||||
|
||||
def _ipv4_blocked(ip: str) -> bool:
|
||||
try:
|
||||
addr = ipaddress.ip_address(ip)
|
||||
except ValueError:
|
||||
return True
|
||||
if not isinstance(addr, ipaddress.IPv4Address):
|
||||
return False
|
||||
return any(addr in net for net in _IPV4_BLOCKS)
|
||||
|
||||
|
||||
def _ip_blocked(ip: str) -> bool:
|
||||
try:
|
||||
addr = ipaddress.ip_address(ip)
|
||||
except ValueError:
|
||||
return True
|
||||
if isinstance(addr, ipaddress.IPv4Address):
|
||||
return _ipv4_blocked(ip)
|
||||
return (
|
||||
addr.is_loopback
|
||||
or addr.is_private
|
||||
or addr.is_link_local
|
||||
or addr.is_multicast
|
||||
or addr.is_reserved
|
||||
or addr.is_unspecified
|
||||
)
|
||||
|
||||
|
||||
def validate_host(host: str) -> dict[str, Any]:
|
||||
trimmed = (host or "").strip()
|
||||
if not trimmed:
|
||||
return {"ok": False, "reason": "empty host"}
|
||||
bracketed = trimmed.strip("[]")
|
||||
lower = trimmed.lower()
|
||||
if any(p.search(lower) for p in _NAME_BLOCKLIST):
|
||||
return {"ok": False, "reason": "hostname matches reserved name pattern"}
|
||||
|
||||
try:
|
||||
ipaddress.ip_address(bracketed)
|
||||
is_literal = True
|
||||
except ValueError:
|
||||
is_literal = False
|
||||
|
||||
if is_literal:
|
||||
if _ip_blocked(bracketed):
|
||||
return {"ok": False, "reason": "IP in reserved range"}
|
||||
return {"ok": True, "resolved": [bracketed]}
|
||||
|
||||
if not _HOSTNAME_RE.match(trimmed):
|
||||
return {"ok": False, "reason": "invalid hostname syntax"}
|
||||
|
||||
try:
|
||||
infos = socket.getaddrinfo(trimmed, None, proto=socket.IPPROTO_TCP)
|
||||
except OSError as exc:
|
||||
return {"ok": False, "reason": f"DNS lookup failed: {exc}"}
|
||||
if not infos:
|
||||
return {"ok": False, "reason": "hostname has no A/AAAA records"}
|
||||
|
||||
resolved: list[str] = []
|
||||
for info in infos:
|
||||
addr = info[4][0]
|
||||
if _ip_blocked(addr):
|
||||
return {"ok": False, "reason": f"hostname resolves to reserved IP {addr}"}
|
||||
resolved.append(addr)
|
||||
return {"ok": True, "resolved": resolved}
|
||||
|
||||
|
||||
def safe_get(
|
||||
url: str,
|
||||
*,
|
||||
timeout: float = 8.0,
|
||||
headers: dict[str, str] | None = None,
|
||||
max_redirects: int = 3,
|
||||
) -> requests.Response:
|
||||
current = url
|
||||
for _ in range(max_redirects + 1):
|
||||
parsed = urlparse(current)
|
||||
if parsed.scheme not in ("http", "https"):
|
||||
raise ValueError(f"blocked protocol {parsed.scheme}")
|
||||
check = validate_host(parsed.hostname or "")
|
||||
if not check.get("ok"):
|
||||
raise ValueError(f"blocked target — {check.get('reason')}")
|
||||
res = requests.get(
|
||||
current,
|
||||
timeout=timeout,
|
||||
headers=headers or {},
|
||||
allow_redirects=False,
|
||||
)
|
||||
if 300 <= res.status_code < 400:
|
||||
loc = res.headers.get("location")
|
||||
if not loc:
|
||||
return res
|
||||
current = urljoin(current, loc)
|
||||
continue
|
||||
return res
|
||||
raise ValueError("too many redirects")
|
||||
|
||||
|
||||
def validate_domain(domain: str) -> bool:
|
||||
return bool(re.match(r"^[a-zA-Z0-9][a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$", domain or ""))
|
||||
Reference in New Issue
Block a user