mirror of
https://github.com/BigBodyCobain/Shadowbroker.git
synced 2026-09-27 19:21:53 +02:00
fix: restore CCTV layer ingestion and map rendering
This commit is contained in:
+147
-104
@@ -1,69 +1,89 @@
|
|||||||
import sqlite3
|
|
||||||
import requests
|
|
||||||
from services.network_utils import fetch_with_curl
|
|
||||||
import logging
|
import logging
|
||||||
|
import sqlite3
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from typing import List, Dict, Any
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
|
from services.network_utils import fetch_with_curl
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
DB_PATH = "cctv.db"
|
DB_PATH = Path(__file__).resolve().parent.parent / "data" / "cctv.db"
|
||||||
|
|
||||||
|
|
||||||
|
def _connect() -> sqlite3.Connection:
|
||||||
|
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
return sqlite3.connect(str(DB_PATH))
|
||||||
|
|
||||||
|
|
||||||
def init_db():
|
def init_db():
|
||||||
conn = sqlite3.connect(DB_PATH)
|
conn = _connect()
|
||||||
cursor = conn.cursor()
|
try:
|
||||||
cursor.execute("""
|
cursor = conn.cursor()
|
||||||
CREATE TABLE IF NOT EXISTS cameras (
|
cursor.execute("""
|
||||||
id TEXT PRIMARY KEY,
|
CREATE TABLE IF NOT EXISTS cameras (
|
||||||
source_agency TEXT,
|
id TEXT PRIMARY KEY,
|
||||||
lat REAL,
|
source_agency TEXT,
|
||||||
lon REAL,
|
lat REAL,
|
||||||
direction_facing TEXT,
|
lon REAL,
|
||||||
media_url TEXT,
|
direction_facing TEXT,
|
||||||
refresh_rate_seconds INTEGER,
|
media_url TEXT,
|
||||||
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
refresh_rate_seconds INTEGER,
|
||||||
)
|
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||||
""")
|
)
|
||||||
conn.commit()
|
""")
|
||||||
conn.close()
|
conn.commit()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
class BaseCCTVIngestor(ABC):
|
class BaseCCTVIngestor(ABC):
|
||||||
def __init__(self):
|
|
||||||
self.conn = sqlite3.connect(DB_PATH)
|
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def ingest(self):
|
def ingest(self):
|
||||||
|
conn = None
|
||||||
try:
|
try:
|
||||||
|
init_db()
|
||||||
cameras = self.fetch_data()
|
cameras = self.fetch_data()
|
||||||
cursor = self.conn.cursor()
|
conn = _connect()
|
||||||
|
cursor = conn.cursor()
|
||||||
for cam in cameras:
|
for cam in cameras:
|
||||||
cursor.execute("""
|
cursor.execute(
|
||||||
|
"""
|
||||||
INSERT INTO cameras
|
INSERT INTO cameras
|
||||||
(id, source_agency, lat, lon, direction_facing, media_url, refresh_rate_seconds)
|
(id, source_agency, lat, lon, direction_facing, media_url, refresh_rate_seconds)
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||||
ON CONFLICT(id) DO UPDATE SET
|
ON CONFLICT(id) DO UPDATE SET
|
||||||
media_url=excluded.media_url,
|
media_url=excluded.media_url,
|
||||||
last_updated=CURRENT_TIMESTAMP
|
last_updated=CURRENT_TIMESTAMP
|
||||||
""", (
|
""",
|
||||||
cam.get("id"),
|
(
|
||||||
cam.get("source_agency"),
|
cam.get("id"),
|
||||||
cam.get("lat"),
|
cam.get("source_agency"),
|
||||||
cam.get("lon"),
|
cam.get("lat"),
|
||||||
cam.get("direction_facing", "Unknown"),
|
cam.get("lon"),
|
||||||
cam.get("media_url"),
|
cam.get("direction_facing", "Unknown"),
|
||||||
cam.get("refresh_rate_seconds", 60)
|
cam.get("media_url"),
|
||||||
))
|
cam.get("refresh_rate_seconds", 60),
|
||||||
self.conn.commit()
|
),
|
||||||
logger.info(f"Successfully ingested {len(cameras)} cameras from {self.__class__.__name__}")
|
)
|
||||||
|
conn.commit()
|
||||||
|
logger.info(
|
||||||
|
f"Successfully ingested {len(cameras)} cameras from {self.__class__.__name__}"
|
||||||
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
try:
|
try:
|
||||||
self.conn.rollback()
|
if conn is not None:
|
||||||
|
conn.rollback()
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
logger.error(f"Failed to ingest cameras in {self.__class__.__name__}: {e}")
|
logger.error(f"Failed to ingest cameras in {self.__class__.__name__}: {e}")
|
||||||
|
finally:
|
||||||
|
if conn is not None:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
class TFLJamCamIngestor(BaseCCTVIngestor):
|
class TFLJamCamIngestor(BaseCCTVIngestor):
|
||||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||||
@@ -79,25 +99,28 @@ class TFLJamCamIngestor(BaseCCTVIngestor):
|
|||||||
vid_url = None
|
vid_url = None
|
||||||
img_url = None
|
img_url = None
|
||||||
|
|
||||||
for prop in item.get('additionalProperties', []):
|
for prop in item.get("additionalProperties", []):
|
||||||
if prop.get('key') == 'videoUrl':
|
if prop.get("key") == "videoUrl":
|
||||||
vid_url = prop.get('value')
|
vid_url = prop.get("value")
|
||||||
elif prop.get('key') == 'imageUrl':
|
elif prop.get("key") == "imageUrl":
|
||||||
img_url = prop.get('value')
|
img_url = prop.get("value")
|
||||||
|
|
||||||
media = vid_url if vid_url else img_url
|
media = vid_url if vid_url else img_url
|
||||||
if media:
|
if media:
|
||||||
cameras.append({
|
cameras.append(
|
||||||
"id": f"TFL-{item.get('id')}",
|
{
|
||||||
"source_agency": "TfL",
|
"id": f"TFL-{item.get('id')}",
|
||||||
"lat": item.get('lat'),
|
"source_agency": "TfL",
|
||||||
"lon": item.get('lon'),
|
"lat": item.get("lat"),
|
||||||
"direction_facing": item.get('commonName', 'Unknown'),
|
"lon": item.get("lon"),
|
||||||
"media_url": media,
|
"direction_facing": item.get("commonName", "Unknown"),
|
||||||
"refresh_rate_seconds": 15
|
"media_url": media,
|
||||||
})
|
"refresh_rate_seconds": 15,
|
||||||
|
}
|
||||||
|
)
|
||||||
return cameras
|
return cameras
|
||||||
|
|
||||||
|
|
||||||
class LTASingaporeIngestor(BaseCCTVIngestor):
|
class LTASingaporeIngestor(BaseCCTVIngestor):
|
||||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||||
# Singapore Land Transport Authority (LTA) Traffic Images API
|
# Singapore Land Transport Authority (LTA) Traffic Images API
|
||||||
@@ -111,19 +134,20 @@ class LTASingaporeIngestor(BaseCCTVIngestor):
|
|||||||
for item in data["items"][0].get("cameras", []):
|
for item in data["items"][0].get("cameras", []):
|
||||||
loc = item.get("location", {})
|
loc = item.get("location", {})
|
||||||
if "latitude" in loc and "longitude" in loc and "image" in item:
|
if "latitude" in loc and "longitude" in loc and "image" in item:
|
||||||
cameras.append({
|
cameras.append(
|
||||||
"id": f"SGP-{item.get('camera_id', 'UNK')}",
|
{
|
||||||
"source_agency": "Singapore LTA",
|
"id": f"SGP-{item.get('camera_id', 'UNK')}",
|
||||||
"lat": loc.get("latitude"),
|
"source_agency": "Singapore LTA",
|
||||||
"lon": loc.get("longitude"),
|
"lat": loc.get("latitude"),
|
||||||
"direction_facing": f"Camera {item.get('camera_id')}",
|
"lon": loc.get("longitude"),
|
||||||
"media_url": item.get("image"),
|
"direction_facing": f"Camera {item.get('camera_id')}",
|
||||||
"refresh_rate_seconds": 60
|
"media_url": item.get("image"),
|
||||||
})
|
"refresh_rate_seconds": 60,
|
||||||
|
}
|
||||||
|
)
|
||||||
return cameras
|
return cameras
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class AustinTXIngestor(BaseCCTVIngestor):
|
class AustinTXIngestor(BaseCCTVIngestor):
|
||||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||||
# City of Austin Traffic Cameras Open Data
|
# City of Austin Traffic Cameras Open Data
|
||||||
@@ -135,24 +159,30 @@ class AustinTXIngestor(BaseCCTVIngestor):
|
|||||||
cameras = []
|
cameras = []
|
||||||
for item in data:
|
for item in data:
|
||||||
cam_id = item.get("camera_id")
|
cam_id = item.get("camera_id")
|
||||||
if not cam_id: continue
|
if not cam_id:
|
||||||
|
continue
|
||||||
|
|
||||||
loc = item.get("location", {})
|
loc = item.get("location", {})
|
||||||
coords = loc.get("coordinates", [])
|
coords = loc.get("coordinates", [])
|
||||||
|
|
||||||
# coords is usually [lon, lat]
|
# coords is usually [lon, lat]
|
||||||
if len(coords) == 2:
|
if len(coords) == 2:
|
||||||
cameras.append({
|
cameras.append(
|
||||||
"id": f"ATX-{cam_id}",
|
{
|
||||||
"source_agency": "Austin TxDOT",
|
"id": f"ATX-{cam_id}",
|
||||||
"lat": coords[1],
|
"source_agency": "Austin TxDOT",
|
||||||
"lon": coords[0],
|
"lat": coords[1],
|
||||||
"direction_facing": item.get("location_name", "Austin TX Camera"),
|
"lon": coords[0],
|
||||||
"media_url": f"https://cctv.austinmobility.io/image/{cam_id}.jpg",
|
"direction_facing": item.get(
|
||||||
"refresh_rate_seconds": 60
|
"location_name", "Austin TX Camera"
|
||||||
})
|
),
|
||||||
|
"media_url": f"https://cctv.austinmobility.io/image/{cam_id}.jpg",
|
||||||
|
"refresh_rate_seconds": 60,
|
||||||
|
}
|
||||||
|
)
|
||||||
return cameras
|
return cameras
|
||||||
|
|
||||||
|
|
||||||
class NYCDOTIngestor(BaseCCTVIngestor):
|
class NYCDOTIngestor(BaseCCTVIngestor):
|
||||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||||
url = "https://webcams.nyctmc.org/api/cameras"
|
url = "https://webcams.nyctmc.org/api/cameras"
|
||||||
@@ -163,22 +193,26 @@ class NYCDOTIngestor(BaseCCTVIngestor):
|
|||||||
cameras = []
|
cameras = []
|
||||||
for item in data:
|
for item in data:
|
||||||
cam_id = item.get("id")
|
cam_id = item.get("id")
|
||||||
if not cam_id: continue
|
if not cam_id:
|
||||||
|
continue
|
||||||
|
|
||||||
lat = item.get("latitude")
|
lat = item.get("latitude")
|
||||||
lon = item.get("longitude")
|
lon = item.get("longitude")
|
||||||
if lat and lon:
|
if lat and lon:
|
||||||
cameras.append({
|
cameras.append(
|
||||||
"id": f"NYC-{cam_id}",
|
{
|
||||||
"source_agency": "NYC DOT",
|
"id": f"NYC-{cam_id}",
|
||||||
"lat": lat,
|
"source_agency": "NYC DOT",
|
||||||
"lon": lon,
|
"lat": lat,
|
||||||
"direction_facing": item.get("name", "NYC Camera"),
|
"lon": lon,
|
||||||
"media_url": f"https://webcams.nyctmc.org/api/cameras/{cam_id}/image",
|
"direction_facing": item.get("name", "NYC Camera"),
|
||||||
"refresh_rate_seconds": 30
|
"media_url": f"https://webcams.nyctmc.org/api/cameras/{cam_id}/image",
|
||||||
})
|
"refresh_rate_seconds": 30,
|
||||||
|
}
|
||||||
|
)
|
||||||
return cameras
|
return cameras
|
||||||
|
|
||||||
|
|
||||||
class GlobalOSMCrawlingIngestor(BaseCCTVIngestor):
|
class GlobalOSMCrawlingIngestor(BaseCCTVIngestor):
|
||||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||||
# This will pull physical street surveillance cameras across all global hotspots
|
# This will pull physical street surveillance cameras across all global hotspots
|
||||||
@@ -193,10 +227,12 @@ class GlobalOSMCrawlingIngestor(BaseCCTVIngestor):
|
|||||||
("25.1,55.2,25.3,55.4", "Dubai"),
|
("25.1,55.2,25.3,55.4", "Dubai"),
|
||||||
("19.3,-99.2,19.5,-99.0", "Mexico City"),
|
("19.3,-99.2,19.5,-99.0", "Mexico City"),
|
||||||
("-23.6,-46.7,-23.4,-46.5", "Sao Paulo"),
|
("-23.6,-46.7,-23.4,-46.5", "Sao Paulo"),
|
||||||
("39.6,-105.1,39.9,-104.8", "Denver")
|
("39.6,-105.1,39.9,-104.8", "Denver"),
|
||||||
]
|
]
|
||||||
|
|
||||||
query_parts = [f'node["man_made"="surveillance"]({bbox});' for bbox, city in regions]
|
query_parts = [
|
||||||
|
f'node["man_made"="surveillance"]({bbox});' for bbox, city in regions
|
||||||
|
]
|
||||||
query = "".join(query_parts)
|
query = "".join(query_parts)
|
||||||
url = f"https://overpass-api.de/api/interpreter?data=[out:json];({query});out%202000;"
|
url = f"https://overpass-api.de/api/interpreter?data=[out:json];({query});out%202000;"
|
||||||
|
|
||||||
@@ -206,7 +242,7 @@ class GlobalOSMCrawlingIngestor(BaseCCTVIngestor):
|
|||||||
data = response.json()
|
data = response.json()
|
||||||
|
|
||||||
cameras = []
|
cameras = []
|
||||||
for item in data.get('elements', []):
|
for item in data.get("elements", []):
|
||||||
lat = item.get("lat")
|
lat = item.get("lat")
|
||||||
lon = item.get("lon")
|
lon = item.get("lon")
|
||||||
cam_id = item.get("id")
|
cam_id = item.get("id")
|
||||||
@@ -215,7 +251,7 @@ class GlobalOSMCrawlingIngestor(BaseCCTVIngestor):
|
|||||||
# Find which city this belongs to
|
# Find which city this belongs to
|
||||||
source_city = "Global OSINT"
|
source_city = "Global OSINT"
|
||||||
for bbox, city in regions:
|
for bbox, city in regions:
|
||||||
s, w, n, e = map(float, bbox.split(','))
|
s, w, n, e = map(float, bbox.split(","))
|
||||||
if s <= lat <= n and w <= lon <= e:
|
if s <= lat <= n and w <= lon <= e:
|
||||||
source_city = f"OSINT: {city}"
|
source_city = f"OSINT: {city}"
|
||||||
break
|
break
|
||||||
@@ -230,40 +266,48 @@ class GlobalOSMCrawlingIngestor(BaseCCTVIngestor):
|
|||||||
mapbox_key = "YOUR_MAPBOX_TOKEN_HERE"
|
mapbox_key = "YOUR_MAPBOX_TOKEN_HERE"
|
||||||
mapbox_url = f"https://api.mapbox.com/styles/v1/mapbox/satellite-streets-v12/static/{lon},{lat},18,{bearing},60/600x400?access_token={mapbox_key}"
|
mapbox_url = f"https://api.mapbox.com/styles/v1/mapbox/satellite-streets-v12/static/{lon},{lat},18,{bearing},60/600x400?access_token={mapbox_key}"
|
||||||
|
|
||||||
cameras.append({
|
cameras.append(
|
||||||
"id": f"OSM-{cam_id}",
|
{
|
||||||
"source_agency": source_city,
|
"id": f"OSM-{cam_id}",
|
||||||
"lat": lat,
|
"source_agency": source_city,
|
||||||
"lon": lon,
|
"lat": lat,
|
||||||
"direction_facing": item.get("tags", {}).get("surveillance:type", "Street Level Camera"),
|
"lon": lon,
|
||||||
"media_url": mapbox_url,
|
"direction_facing": item.get("tags", {}).get(
|
||||||
"refresh_rate_seconds": 3600
|
"surveillance:type", "Street Level Camera"
|
||||||
})
|
),
|
||||||
|
"media_url": mapbox_url,
|
||||||
|
"refresh_rate_seconds": 3600,
|
||||||
|
}
|
||||||
|
)
|
||||||
return cameras
|
return cameras
|
||||||
except Exception:
|
except Exception:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def _detect_media_type(url: str) -> str:
|
def _detect_media_type(url: str) -> str:
|
||||||
"""Detect the media type from a camera URL for proper frontend rendering."""
|
"""Detect the media type from a camera URL for proper frontend rendering."""
|
||||||
if not url:
|
if not url:
|
||||||
return "image"
|
return "image"
|
||||||
url_lower = url.lower()
|
url_lower = url.lower()
|
||||||
if any(ext in url_lower for ext in ['.mp4', '.webm', '.ogg']):
|
if any(ext in url_lower for ext in [".mp4", ".webm", ".ogg"]):
|
||||||
return "video"
|
return "video"
|
||||||
if any(kw in url_lower for kw in ['.mjpg', '.mjpeg', 'mjpg', 'axis-cgi/mjpg', 'mode=motion']):
|
if any(
|
||||||
|
kw in url_lower
|
||||||
|
for kw in [".mjpg", ".mjpeg", "mjpg", "axis-cgi/mjpg", "mode=motion"]
|
||||||
|
):
|
||||||
return "mjpeg"
|
return "mjpeg"
|
||||||
if '.m3u8' in url_lower or 'hls' in url_lower:
|
if ".m3u8" in url_lower or "hls" in url_lower:
|
||||||
return "hls"
|
return "hls"
|
||||||
if any(kw in url_lower for kw in ['embed', 'maps/embed', 'iframe']):
|
if any(kw in url_lower for kw in ["embed", "maps/embed", "iframe"]):
|
||||||
return "embed"
|
return "embed"
|
||||||
if 'mapbox.com' in url_lower or 'satellite' in url_lower:
|
if "mapbox.com" in url_lower or "satellite" in url_lower:
|
||||||
return "satellite"
|
return "satellite"
|
||||||
return "image"
|
return "image"
|
||||||
|
|
||||||
|
|
||||||
def get_all_cameras() -> List[Dict[str, Any]]:
|
def get_all_cameras() -> List[Dict[str, Any]]:
|
||||||
conn = sqlite3.connect(DB_PATH)
|
init_db()
|
||||||
|
conn = _connect()
|
||||||
conn.row_factory = sqlite3.Row
|
conn.row_factory = sqlite3.Row
|
||||||
cursor = conn.cursor()
|
cursor = conn.cursor()
|
||||||
cursor.execute("SELECT * FROM cameras")
|
cursor.execute("SELECT * FROM cameras")
|
||||||
@@ -272,7 +316,6 @@ def get_all_cameras() -> List[Dict[str, Any]]:
|
|||||||
cameras = []
|
cameras = []
|
||||||
for row in rows:
|
for row in rows:
|
||||||
cam = dict(row)
|
cam = dict(row)
|
||||||
cam['media_type'] = _detect_media_type(cam.get('media_url', ''))
|
cam["media_type"] = _detect_media_type(cam.get("media_url", ""))
|
||||||
cameras.append(cam)
|
cameras.append(cam)
|
||||||
return cameras
|
return cameras
|
||||||
|
|
||||||
|
|||||||
@@ -13,10 +13,12 @@ Heavy logic has been extracted into services/fetchers/:
|
|||||||
- infrastructure.py — internet outages, data centers, CCTV, KiwiSDR
|
- infrastructure.py — internet outages, data centers, CCTV, KiwiSDR
|
||||||
- geo.py — ships, airports, frontlines, GDELT, LiveUAMap
|
- geo.py — ships, airports, frontlines, GDELT, LiveUAMap
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import concurrent.futures
|
import concurrent.futures
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
|
|
||||||
load_dotenv()
|
load_dotenv()
|
||||||
|
|
||||||
from apscheduler.schedulers.background import BackgroundScheduler
|
from apscheduler.schedulers.background import BackgroundScheduler
|
||||||
@@ -24,7 +26,10 @@ from services.cctv_pipeline import init_db
|
|||||||
|
|
||||||
# Shared state — all fetcher modules read/write through this
|
# Shared state — all fetcher modules read/write through this
|
||||||
from services.fetchers._store import (
|
from services.fetchers._store import (
|
||||||
latest_data, source_timestamps, _mark_fresh, _data_lock, # noqa: F401 — re-exported for main.py
|
latest_data,
|
||||||
|
source_timestamps,
|
||||||
|
_mark_fresh,
|
||||||
|
_data_lock, # noqa: F401 — re-exported for main.py
|
||||||
)
|
)
|
||||||
|
|
||||||
# Domain-specific fetcher modules (already extracted)
|
# Domain-specific fetcher modules (already extracted)
|
||||||
@@ -37,19 +42,55 @@ from services.fetchers.news import fetch_news # noqa: F401
|
|||||||
# Newly extracted fetcher modules
|
# Newly extracted fetcher modules
|
||||||
from services.fetchers.financial import fetch_defense_stocks, fetch_oil_prices # noqa: F401
|
from services.fetchers.financial import fetch_defense_stocks, fetch_oil_prices # noqa: F401
|
||||||
from services.fetchers.earth_observation import ( # noqa: F401
|
from services.fetchers.earth_observation import ( # noqa: F401
|
||||||
fetch_earthquakes, fetch_firms_fires, fetch_space_weather, fetch_weather,
|
fetch_earthquakes,
|
||||||
|
fetch_firms_fires,
|
||||||
|
fetch_space_weather,
|
||||||
|
fetch_weather,
|
||||||
)
|
)
|
||||||
from services.fetchers.infrastructure import ( # noqa: F401
|
from services.fetchers.infrastructure import ( # noqa: F401
|
||||||
fetch_internet_outages, fetch_datacenters, fetch_military_bases, fetch_power_plants,
|
fetch_internet_outages,
|
||||||
fetch_cctv, fetch_kiwisdr,
|
fetch_datacenters,
|
||||||
|
fetch_military_bases,
|
||||||
|
fetch_power_plants,
|
||||||
|
fetch_cctv,
|
||||||
|
fetch_kiwisdr,
|
||||||
)
|
)
|
||||||
from services.fetchers.geo import ( # noqa: F401
|
from services.fetchers.geo import ( # noqa: F401
|
||||||
fetch_ships, fetch_airports, find_nearest_airport, cached_airports,
|
fetch_ships,
|
||||||
fetch_frontlines, fetch_gdelt, fetch_geopolitics, update_liveuamap,
|
fetch_airports,
|
||||||
|
find_nearest_airport,
|
||||||
|
cached_airports,
|
||||||
|
fetch_frontlines,
|
||||||
|
fetch_gdelt,
|
||||||
|
fetch_geopolitics,
|
||||||
|
update_liveuamap,
|
||||||
)
|
)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def run_cctv_ingest_cycle():
|
||||||
|
"""Populate the CCTV database, then refresh the in-memory cache."""
|
||||||
|
logger.info("CCTV ingest cycle starting...")
|
||||||
|
from services.cctv_pipeline import (
|
||||||
|
AustinTXIngestor,
|
||||||
|
LTASingaporeIngestor,
|
||||||
|
NYCDOTIngestor,
|
||||||
|
TFLJamCamIngestor,
|
||||||
|
)
|
||||||
|
|
||||||
|
for ingestor_cls in (
|
||||||
|
TFLJamCamIngestor,
|
||||||
|
LTASingaporeIngestor,
|
||||||
|
AustinTXIngestor,
|
||||||
|
NYCDOTIngestor,
|
||||||
|
):
|
||||||
|
ingestor_cls().ingest()
|
||||||
|
|
||||||
|
fetch_cctv()
|
||||||
|
logger.info("CCTV ingest cycle complete.")
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Scheduler & Orchestration
|
# Scheduler & Orchestration
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -66,9 +107,10 @@ def update_fast_data():
|
|||||||
futures = [executor.submit(func) for func in fast_funcs]
|
futures = [executor.submit(func) for func in fast_funcs]
|
||||||
concurrent.futures.wait(futures)
|
concurrent.futures.wait(futures)
|
||||||
with _data_lock:
|
with _data_lock:
|
||||||
latest_data['last_updated'] = datetime.utcnow().isoformat()
|
latest_data["last_updated"] = datetime.utcnow().isoformat()
|
||||||
logger.info("Fast-tier update complete.")
|
logger.info("Fast-tier update complete.")
|
||||||
|
|
||||||
|
|
||||||
def update_slow_data():
|
def update_slow_data():
|
||||||
"""Slow-tier: contextual + enrichment data that refreshes less often (every 5–10 min)."""
|
"""Slow-tier: contextual + enrichment data that refreshes less often (every 5–10 min)."""
|
||||||
logger.info("Slow-tier data update starting...")
|
logger.info("Slow-tier data update starting...")
|
||||||
@@ -94,6 +136,7 @@ def update_slow_data():
|
|||||||
concurrent.futures.wait(futures)
|
concurrent.futures.wait(futures)
|
||||||
logger.info("Slow-tier update complete.")
|
logger.info("Slow-tier update complete.")
|
||||||
|
|
||||||
|
|
||||||
def update_all_data():
|
def update_all_data():
|
||||||
"""Full refresh — all tiers run IN PARALLEL for fastest startup."""
|
"""Full refresh — all tiers run IN PARALLEL for fastest startup."""
|
||||||
logger.info("Full data update starting (parallel)...")
|
logger.info("Full data update starting (parallel)...")
|
||||||
@@ -104,45 +147,73 @@ def update_all_data():
|
|||||||
concurrent.futures.wait([f0, f1, f2])
|
concurrent.futures.wait([f0, f1, f2])
|
||||||
logger.info("Full data update complete.")
|
logger.info("Full data update complete.")
|
||||||
|
|
||||||
|
|
||||||
_scheduler = None
|
_scheduler = None
|
||||||
|
|
||||||
|
|
||||||
def start_scheduler():
|
def start_scheduler():
|
||||||
global _scheduler
|
global _scheduler
|
||||||
init_db()
|
init_db()
|
||||||
_scheduler = BackgroundScheduler(daemon=True)
|
_scheduler = BackgroundScheduler(daemon=True)
|
||||||
|
|
||||||
# Fast tier — every 60 seconds
|
# Fast tier — every 60 seconds
|
||||||
_scheduler.add_job(update_fast_data, 'interval', seconds=60, id='fast_tier', max_instances=1, misfire_grace_time=30)
|
_scheduler.add_job(
|
||||||
|
update_fast_data,
|
||||||
|
"interval",
|
||||||
|
seconds=60,
|
||||||
|
id="fast_tier",
|
||||||
|
max_instances=1,
|
||||||
|
misfire_grace_time=30,
|
||||||
|
)
|
||||||
|
|
||||||
# Slow tier — every 5 minutes
|
# Slow tier — every 5 minutes
|
||||||
_scheduler.add_job(update_slow_data, 'interval', minutes=5, id='slow_tier', max_instances=1, misfire_grace_time=120)
|
_scheduler.add_job(
|
||||||
|
update_slow_data,
|
||||||
|
"interval",
|
||||||
|
minutes=5,
|
||||||
|
id="slow_tier",
|
||||||
|
max_instances=1,
|
||||||
|
misfire_grace_time=120,
|
||||||
|
)
|
||||||
|
|
||||||
# Very slow — every 15 minutes
|
# Very slow — every 15 minutes
|
||||||
_scheduler.add_job(fetch_gdelt, 'interval', minutes=15, id='gdelt', max_instances=1, misfire_grace_time=120)
|
_scheduler.add_job(
|
||||||
_scheduler.add_job(update_liveuamap, 'interval', minutes=15, id='liveuamap', max_instances=1, misfire_grace_time=120)
|
fetch_gdelt,
|
||||||
|
"interval",
|
||||||
|
minutes=15,
|
||||||
|
id="gdelt",
|
||||||
|
max_instances=1,
|
||||||
|
misfire_grace_time=120,
|
||||||
|
)
|
||||||
|
_scheduler.add_job(
|
||||||
|
update_liveuamap,
|
||||||
|
"interval",
|
||||||
|
minutes=15,
|
||||||
|
id="liveuamap",
|
||||||
|
max_instances=1,
|
||||||
|
misfire_grace_time=120,
|
||||||
|
)
|
||||||
|
|
||||||
# CCTV pipeline refresh — every 10 minutes
|
# CCTV pipeline refresh — hydrate DB-backed cameras immediately, then every 10 minutes.
|
||||||
# Instantiate once and reuse — avoids re-creating DB connections on every tick
|
_scheduler.add_job(
|
||||||
from services.cctv_pipeline import (
|
run_cctv_ingest_cycle,
|
||||||
TFLJamCamIngestor, LTASingaporeIngestor,
|
"interval",
|
||||||
AustinTXIngestor, NYCDOTIngestor,
|
minutes=10,
|
||||||
|
id="cctv_ingest",
|
||||||
|
max_instances=1,
|
||||||
|
misfire_grace_time=120,
|
||||||
|
next_run_time=datetime.utcnow(),
|
||||||
)
|
)
|
||||||
_cctv_tfl = TFLJamCamIngestor()
|
|
||||||
_cctv_lta = LTASingaporeIngestor()
|
|
||||||
_cctv_atx = AustinTXIngestor()
|
|
||||||
_cctv_nyc = NYCDOTIngestor()
|
|
||||||
_scheduler.add_job(_cctv_tfl.ingest, 'interval', minutes=10, id='cctv_tfl', max_instances=1, misfire_grace_time=120)
|
|
||||||
_scheduler.add_job(_cctv_lta.ingest, 'interval', minutes=10, id='cctv_lta', max_instances=1, misfire_grace_time=120)
|
|
||||||
_scheduler.add_job(_cctv_atx.ingest, 'interval', minutes=10, id='cctv_atx', max_instances=1, misfire_grace_time=120)
|
|
||||||
_scheduler.add_job(_cctv_nyc.ingest, 'interval', minutes=10, id='cctv_nyc', max_instances=1, misfire_grace_time=120)
|
|
||||||
|
|
||||||
_scheduler.start()
|
_scheduler.start()
|
||||||
logger.info("Scheduler started.")
|
logger.info("Scheduler started.")
|
||||||
|
|
||||||
|
|
||||||
def stop_scheduler():
|
def stop_scheduler():
|
||||||
if _scheduler:
|
if _scheduler:
|
||||||
_scheduler.shutdown(wait=False)
|
_scheduler.shutdown(wait=False)
|
||||||
|
|
||||||
|
|
||||||
def get_latest_data():
|
def get_latest_data():
|
||||||
with _data_lock:
|
with _data_lock:
|
||||||
return dict(latest_data)
|
return dict(latest_data)
|
||||||
|
|||||||
@@ -0,0 +1,79 @@
|
|||||||
|
"""Regression tests for CCTV ingestion and persistence."""
|
||||||
|
|
||||||
|
import threading
|
||||||
|
|
||||||
|
from services import cctv_pipeline
|
||||||
|
|
||||||
|
|
||||||
|
class DummyIngestor(cctv_pipeline.BaseCCTVIngestor):
|
||||||
|
def __init__(self, cameras):
|
||||||
|
self._cameras = cameras
|
||||||
|
|
||||||
|
def fetch_data(self):
|
||||||
|
return self._cameras
|
||||||
|
|
||||||
|
|
||||||
|
def test_ingestor_can_run_from_another_thread(tmp_path, monkeypatch):
|
||||||
|
db_path = tmp_path / "data" / "cctv.db"
|
||||||
|
monkeypatch.setattr(cctv_pipeline, "DB_PATH", db_path)
|
||||||
|
|
||||||
|
ingestor = DummyIngestor(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"id": "cam-1",
|
||||||
|
"source_agency": "Test",
|
||||||
|
"lat": 51.5,
|
||||||
|
"lon": -0.12,
|
||||||
|
"direction_facing": "North",
|
||||||
|
"media_url": "https://example.com/camera.jpg",
|
||||||
|
"refresh_rate_seconds": 30,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
thread = threading.Thread(target=ingestor.ingest)
|
||||||
|
thread.start()
|
||||||
|
thread.join()
|
||||||
|
|
||||||
|
cameras = cctv_pipeline.get_all_cameras()
|
||||||
|
assert len(cameras) == 1
|
||||||
|
assert cameras[0]["id"] == "cam-1"
|
||||||
|
assert cameras[0]["media_type"] == "image"
|
||||||
|
|
||||||
|
|
||||||
|
def test_ingest_updates_existing_rows_in_persistent_data_dir(tmp_path, monkeypatch):
|
||||||
|
db_path = tmp_path / "persistent" / "cctv.db"
|
||||||
|
monkeypatch.setattr(cctv_pipeline, "DB_PATH", db_path)
|
||||||
|
|
||||||
|
DummyIngestor(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"id": "cam-2",
|
||||||
|
"source_agency": "Test",
|
||||||
|
"lat": 40.71,
|
||||||
|
"lon": -74.0,
|
||||||
|
"direction_facing": "East",
|
||||||
|
"media_url": "https://example.com/old.jpg",
|
||||||
|
"refresh_rate_seconds": 60,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
).ingest()
|
||||||
|
DummyIngestor(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"id": "cam-2",
|
||||||
|
"source_agency": "Test",
|
||||||
|
"lat": 40.71,
|
||||||
|
"lon": -74.0,
|
||||||
|
"direction_facing": "East",
|
||||||
|
"media_url": "https://example.com/live.m3u8",
|
||||||
|
"refresh_rate_seconds": 60,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
).ingest()
|
||||||
|
|
||||||
|
cameras = cctv_pipeline.get_all_cameras()
|
||||||
|
assert db_path.exists()
|
||||||
|
assert len(cameras) == 1
|
||||||
|
assert cameras[0]["media_url"] == "https://example.com/live.m3u8"
|
||||||
|
assert cameras[0]["media_type"] == "hls"
|
||||||
@@ -593,6 +593,8 @@ const MaplibreViewer = ({ data, activeLayers, onEntityClick, flyToLocation, sele
|
|||||||
frontlineGeoJSON && 'ukraine-frontline-layer',
|
frontlineGeoJSON && 'ukraine-frontline-layer',
|
||||||
earthquakesGeoJSON && 'earthquakes-layer',
|
earthquakesGeoJSON && 'earthquakes-layer',
|
||||||
satellitesGeoJSON && 'satellites-layer',
|
satellitesGeoJSON && 'satellites-layer',
|
||||||
|
cctvGeoJSON && 'cctv-clusters',
|
||||||
|
cctvGeoJSON && 'cctv-cluster-count',
|
||||||
cctvGeoJSON && 'cctv-layer',
|
cctvGeoJSON && 'cctv-layer',
|
||||||
kiwisdrGeoJSON && 'kiwisdr-clusters',
|
kiwisdrGeoJSON && 'kiwisdr-clusters',
|
||||||
kiwisdrGeoJSON && 'kiwisdr-layer',
|
kiwisdrGeoJSON && 'kiwisdr-layer',
|
||||||
|
|||||||
Reference in New Issue
Block a user