mirror of
https://github.com/BigBodyCobain/Shadowbroker.git
synced 2026-04-30 14:57:58 +02:00
feat: add Spanish CCTV feeds and fix image loading
- Add 5 native ingestors to cctv_pipeline.py: DGT (~1,917 cameras), Madrid (~357), Málaga (~134), Vigo (~59), Vitoria-Gasteiz (~17) - Fix DGT DATEX2 parser to match actual XML schema (device elements, not CctvCameraRecord) - Wire all new ingestors into the scheduler via data_fetcher.py - Remove standalone spain_cctv.py by Alborz Nazari, replaced by native pipeline ingestors that integrate with the existing scheduler pattern - Fix CCTV image loading for servers with Referer-based hotlink protection (referrerPolicy="no-referrer") - Replace external via.placeholder.com fallbacks with inline SVG data URIs to avoid dependency on unreachable third-party service - Surface source_agency attribution in CCTV panel UI for open data license compliance (CC BY / Spain Ley 37/2007) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Singular Failure
@@ -219,11 +219,16 @@ helm install shadowbroker ./helm/chart --create-namespace --namespace shadowbrok
|
||||
|
||||
### 📷 Surveillance
|
||||
|
||||
* **CCTV Mesh** — 2,000+ live traffic cameras from:
|
||||
* **CCTV Mesh** — 4,400+ live traffic cameras from:
|
||||
* 🇬🇧 Transport for London JamCams
|
||||
* 🇺🇸 Austin, TX TxDOT
|
||||
* 🇺🇸 NYC DOT
|
||||
* 🇸🇬 Singapore LTA
|
||||
* 🇪🇸 Spanish DGT (national roads)
|
||||
* 🇪🇸 Madrid City Hall
|
||||
* 🇪🇸 Málaga City
|
||||
* 🇪🇸 Vigo City
|
||||
* 🇪🇸 Vitoria-Gasteiz
|
||||
* Custom URL ingestion
|
||||
* **Feed Rendering** — Automatic detection & rendering of video, MJPEG, HLS, embed, satellite tile, and image feeds
|
||||
* **Clustered Map Display** — Green dots cluster with count labels, decluster on zoom
|
||||
@@ -307,6 +312,11 @@ helm install shadowbroker ./helm/chart --create-namespace --namespace shadowbrok
|
||||
| [TxDOT](https://its.txdot.gov) | Austin TX traffic cameras | ~5min | No |
|
||||
| [NYC DOT](https://webcams.nyctmc.org) | NYC traffic cameras | ~5min | No |
|
||||
| [Singapore LTA](https://datamall.lta.gov.sg) | Singapore traffic cameras | ~5min | **Yes** |
|
||||
| [DGT Spain](https://nap.dgt.es) | Spanish national road cameras | ~10min | No |
|
||||
| [Madrid Open Data](https://datos.madrid.es) | Madrid urban traffic cameras | ~10min | No |
|
||||
| [Málaga Open Data](https://datosabiertos.malaga.eu) | Málaga traffic cameras | ~10min | No |
|
||||
| [Vigo Open Data](https://datos.vigo.org) | Vigo traffic cameras | ~10min | No |
|
||||
| [Vitoria-Gasteiz](https://www.vitoria-gasteiz.org) | Vitoria-Gasteiz traffic cameras | ~10min | No |
|
||||
| [RestCountries](https://restcountries.com) | Country profile data | On-demand (cached 24h) | No |
|
||||
| [Wikidata SPARQL](https://query.wikidata.org) | Head of state data | On-demand (cached 24h) | No |
|
||||
| [Wikipedia API](https://en.wikipedia.org/api) | Location summaries & aircraft images | On-demand (cached) | No |
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import logging
|
||||
import re
|
||||
import sqlite3
|
||||
import xml.etree.ElementTree as ET
|
||||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
@@ -284,6 +286,316 @@ class GlobalOSMCrawlingIngestor(BaseCCTVIngestor):
|
||||
return []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Spain — DGT National Roads (DATEX2 XML, ~1,900 cameras)
|
||||
# ---------------------------------------------------------------------------
|
||||
class SpainDGTIngestor(BaseCCTVIngestor):
|
||||
# Dirección General de Tráfico — national road cameras via DATEX2 v3 XML.
|
||||
# No API key required. Covers all national roads (autopistas, autovías, N-roads)
|
||||
# EXCEPT Basque Country and Catalonia.
|
||||
# Published under Spain's open data framework (Ley 37/2007, EU PSI Directive 2019/1024).
|
||||
DGT_URL = "https://nap.dgt.es/datex2/v3/dgt/DevicePublication/camaras_datex2_v36.xml"
|
||||
|
||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||
try:
|
||||
response = fetch_with_curl(self.DGT_URL, timeout=30)
|
||||
response.raise_for_status()
|
||||
except Exception as e:
|
||||
logger.error(f"SpainDGTIngestor: failed to fetch DATEX2 XML: {e}")
|
||||
return []
|
||||
|
||||
try:
|
||||
root = ET.fromstring(response.content)
|
||||
except ET.ParseError as e:
|
||||
logger.error(f"SpainDGTIngestor: failed to parse XML: {e}")
|
||||
return []
|
||||
|
||||
cameras = []
|
||||
# DGT DATEX2 v3 uses <ns2:device> elements with typeOfDevice=camera.
|
||||
# Namespace-agnostic: match local name "device".
|
||||
for el in root.iter():
|
||||
local = el.tag.split("}")[-1] if "}" in el.tag else el.tag
|
||||
if local != "device":
|
||||
continue
|
||||
|
||||
try:
|
||||
cam_id = el.get("id", "")
|
||||
if not cam_id:
|
||||
continue
|
||||
|
||||
# Coordinates are nested: pointLocation > ... > pointCoordinates > latitude/longitude
|
||||
lat = self._find_text(el, "latitude")
|
||||
lon = self._find_text(el, "longitude")
|
||||
if not lat or not lon:
|
||||
continue
|
||||
|
||||
image_url = self._find_text(el, "deviceUrl") or f"https://infocar.dgt.es/etraffic/data/camaras/{cam_id}.jpg"
|
||||
|
||||
road_name = self._find_text(el, "roadName") or ""
|
||||
road_dest = self._find_text(el, "roadDestination") or ""
|
||||
description = f"{road_name} → {road_dest}".strip(" →") or f"DGT Camera {cam_id}"
|
||||
|
||||
cameras.append({
|
||||
"id": f"DGT-{cam_id}",
|
||||
"source_agency": "DGT Spain",
|
||||
"lat": float(lat),
|
||||
"lon": float(lon),
|
||||
"direction_facing": description,
|
||||
"media_url": image_url,
|
||||
"refresh_rate_seconds": 300,
|
||||
})
|
||||
except (ValueError, TypeError) as e:
|
||||
logger.debug(f"SpainDGTIngestor: skipping malformed record: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"SpainDGTIngestor: parsed {len(cameras)} cameras")
|
||||
return cameras
|
||||
|
||||
@staticmethod
|
||||
def _find_text(element: ET.Element, tag: str) -> str | None:
|
||||
for child in element.iter():
|
||||
local = child.tag.split("}")[-1] if "}" in child.tag else child.tag
|
||||
if local.lower() == tag.lower() and child.text:
|
||||
return child.text.strip()
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Spain — Madrid City Hall (KML, ~200 cameras)
|
||||
# ---------------------------------------------------------------------------
|
||||
class MadridCCTVIngestor(BaseCCTVIngestor):
|
||||
# Madrid City Hall urban traffic cameras via open data KML.
|
||||
# No API key required. Published on datos.madrid.es.
|
||||
# Licence: Madrid Open Data (free reuse with attribution).
|
||||
MADRID_URL = "http://datos.madrid.es/egob/catalogo/202088-0-trafico-camaras.kml"
|
||||
|
||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||
try:
|
||||
response = fetch_with_curl(self.MADRID_URL, timeout=20)
|
||||
response.raise_for_status()
|
||||
except Exception as e:
|
||||
logger.error(f"MadridCCTVIngestor: failed to fetch KML: {e}")
|
||||
return []
|
||||
|
||||
try:
|
||||
root = ET.fromstring(response.content)
|
||||
except ET.ParseError as e:
|
||||
logger.error(f"MadridCCTVIngestor: failed to parse KML: {e}")
|
||||
return []
|
||||
|
||||
cameras = []
|
||||
# KML namespace varies — try both common ones, then fall back to tag-name search
|
||||
placemarks = root.findall(".//{http://www.opengis.net/kml/2.2}Placemark")
|
||||
if not placemarks:
|
||||
placemarks = root.findall(".//{http://earth.google.com/kml/2.2}Placemark")
|
||||
if not placemarks:
|
||||
placemarks = [el for el in root.iter() if el.tag.endswith("Placemark")]
|
||||
|
||||
for i, pm in enumerate(placemarks):
|
||||
try:
|
||||
name = self._find_kml_text(pm, "name") or f"Madrid Camera {i}"
|
||||
coords_text = self._find_kml_text(pm, "coordinates")
|
||||
if not coords_text:
|
||||
continue
|
||||
|
||||
# KML coordinates: lon,lat,elevation
|
||||
parts = coords_text.strip().split(",")
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
lon, lat = float(parts[0]), float(parts[1])
|
||||
|
||||
# Extract image URL from description CDATA
|
||||
desc = self._find_kml_text(pm, "description") or ""
|
||||
image_url = self._extract_img_src(desc)
|
||||
if not image_url:
|
||||
continue
|
||||
|
||||
cameras.append({
|
||||
"id": f"MAD-{i:04d}",
|
||||
"source_agency": "Madrid City Hall",
|
||||
"lat": lat,
|
||||
"lon": lon,
|
||||
"direction_facing": name,
|
||||
"media_url": image_url,
|
||||
"refresh_rate_seconds": 600,
|
||||
})
|
||||
except (ValueError, TypeError, IndexError) as e:
|
||||
logger.debug(f"MadridCCTVIngestor: skipping malformed placemark: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"MadridCCTVIngestor: parsed {len(cameras)} cameras")
|
||||
return cameras
|
||||
|
||||
@staticmethod
|
||||
def _find_kml_text(element: ET.Element, tag: str) -> str | None:
|
||||
for child in element.iter():
|
||||
local = child.tag.split("}")[-1] if "}" in child.tag else child.tag
|
||||
if local == tag and child.text:
|
||||
return child.text.strip()
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_img_src(html_fragment: str) -> str | None:
|
||||
match = re.search(r'src=["\']([^"\']+)["\']', html_fragment, re.IGNORECASE)
|
||||
if match:
|
||||
return match.group(1)
|
||||
match = re.search(r'https?://\S+\.jpg', html_fragment, re.IGNORECASE)
|
||||
if match:
|
||||
return match.group(0)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Spain — Málaga (GeoJSON, ~134 cameras)
|
||||
# ---------------------------------------------------------------------------
|
||||
class MalagaCCTVIngestor(BaseCCTVIngestor):
|
||||
# Málaga open data — traffic cameras in EPSG:4326 GeoJSON.
|
||||
# No API key required. Published on datosabiertos.malaga.eu.
|
||||
MALAGA_URL = "https://datosabiertos.malaga.eu/recursos/transporte/trafico/da_camarasTrafico-4326.geojson"
|
||||
|
||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||
try:
|
||||
response = fetch_with_curl(self.MALAGA_URL, timeout=15)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
except Exception as e:
|
||||
logger.error(f"MalagaCCTVIngestor: failed to fetch GeoJSON: {e}")
|
||||
return []
|
||||
|
||||
cameras = []
|
||||
for feature in data.get("features", []):
|
||||
try:
|
||||
props = feature.get("properties", {})
|
||||
geom = feature.get("geometry", {})
|
||||
coords = geom.get("coordinates", [])
|
||||
if len(coords) < 2:
|
||||
continue
|
||||
|
||||
image_url = props.get("URLIMAGEN") or props.get("urlimagen")
|
||||
if not image_url:
|
||||
continue
|
||||
|
||||
cam_id = props.get("NOMBRE") or props.get("nombre") or str(coords)
|
||||
description = props.get("DESCRIPCION") or props.get("descripcion") or cam_id
|
||||
|
||||
cameras.append({
|
||||
"id": f"MLG-{cam_id}",
|
||||
"source_agency": "Málaga City",
|
||||
"lat": float(coords[1]),
|
||||
"lon": float(coords[0]),
|
||||
"direction_facing": description,
|
||||
"media_url": image_url,
|
||||
"refresh_rate_seconds": 300,
|
||||
})
|
||||
except (ValueError, TypeError, IndexError) as e:
|
||||
logger.debug(f"MalagaCCTVIngestor: skipping malformed feature: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"MalagaCCTVIngestor: parsed {len(cameras)} cameras")
|
||||
return cameras
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Spain — Vigo (GeoJSON, ~59 cameras)
|
||||
# ---------------------------------------------------------------------------
|
||||
class VigoCCTVIngestor(BaseCCTVIngestor):
|
||||
# Vigo open data — traffic cameras in GeoJSON.
|
||||
# No API key required. Published on datos.vigo.org.
|
||||
VIGO_URL = "https://datos.vigo.org/data/trafico/camaras-trafico.geojson"
|
||||
|
||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||
try:
|
||||
response = fetch_with_curl(self.VIGO_URL, timeout=15)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
except Exception as e:
|
||||
logger.error(f"VigoCCTVIngestor: failed to fetch GeoJSON: {e}")
|
||||
return []
|
||||
|
||||
cameras = []
|
||||
for feature in data.get("features", []):
|
||||
try:
|
||||
props = feature.get("properties", {})
|
||||
geom = feature.get("geometry", {})
|
||||
coords = geom.get("coordinates", [])
|
||||
if len(coords) < 2:
|
||||
continue
|
||||
|
||||
# Vigo uses PHP image endpoints
|
||||
image_url = props.get("urlimagen") or props.get("URLIMAGEN") or props.get("url")
|
||||
if not image_url:
|
||||
continue
|
||||
|
||||
cam_id = props.get("id") or props.get("nombre") or str(coords)
|
||||
description = props.get("nombre") or props.get("descripcion") or f"Vigo Camera {cam_id}"
|
||||
|
||||
cameras.append({
|
||||
"id": f"VGO-{cam_id}",
|
||||
"source_agency": "Vigo City",
|
||||
"lat": float(coords[1]),
|
||||
"lon": float(coords[0]),
|
||||
"direction_facing": description,
|
||||
"media_url": image_url,
|
||||
"refresh_rate_seconds": 300,
|
||||
})
|
||||
except (ValueError, TypeError, IndexError) as e:
|
||||
logger.debug(f"VigoCCTVIngestor: skipping malformed feature: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"VigoCCTVIngestor: parsed {len(cameras)} cameras")
|
||||
return cameras
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Spain — Vitoria-Gasteiz (GeoJSON, ~17 cameras)
|
||||
# ---------------------------------------------------------------------------
|
||||
class VitoriaGasteizCCTVIngestor(BaseCCTVIngestor):
|
||||
# Vitoria-Gasteiz municipal traffic cameras in GeoJSON.
|
||||
# No API key required. Published on vitoria-gasteiz.org.
|
||||
VITORIA_URL = "https://www.vitoria-gasteiz.org/c11-01w/cameras?action=list&format=GEOJSON"
|
||||
|
||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||
try:
|
||||
response = fetch_with_curl(self.VITORIA_URL, timeout=15)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
except Exception as e:
|
||||
logger.error(f"VitoriaGasteizCCTVIngestor: failed to fetch GeoJSON: {e}")
|
||||
return []
|
||||
|
||||
cameras = []
|
||||
for feature in data.get("features", []):
|
||||
try:
|
||||
props = feature.get("properties", {})
|
||||
geom = feature.get("geometry", {})
|
||||
coords = geom.get("coordinates", [])
|
||||
if len(coords) < 2:
|
||||
continue
|
||||
|
||||
image_url = props.get("imagen") or props.get("url")
|
||||
if not image_url:
|
||||
continue
|
||||
|
||||
cam_id = props.get("id") or props.get("nombre") or str(coords)
|
||||
description = props.get("nombre") or props.get("descripcion") or f"Vitoria Camera {cam_id}"
|
||||
|
||||
cameras.append({
|
||||
"id": f"VIT-{cam_id}",
|
||||
"source_agency": "Vitoria-Gasteiz",
|
||||
"lat": float(coords[1]),
|
||||
"lon": float(coords[0]),
|
||||
"direction_facing": description,
|
||||
"media_url": image_url,
|
||||
"refresh_rate_seconds": 300,
|
||||
})
|
||||
except (ValueError, TypeError, IndexError) as e:
|
||||
logger.debug(f"VitoriaGasteizCCTVIngestor: skipping malformed feature: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"VitoriaGasteizCCTVIngestor: parsed {len(cameras)} cameras")
|
||||
return cameras
|
||||
|
||||
|
||||
def _detect_media_type(url: str) -> str:
|
||||
"""Detect the media type from a camera URL for proper frontend rendering."""
|
||||
if not url:
|
||||
|
||||
@@ -75,8 +75,13 @@ def run_cctv_ingest_cycle():
|
||||
from services.cctv_pipeline import (
|
||||
AustinTXIngestor,
|
||||
LTASingaporeIngestor,
|
||||
MadridCCTVIngestor,
|
||||
MalagaCCTVIngestor,
|
||||
NYCDOTIngestor,
|
||||
SpainDGTIngestor,
|
||||
TFLJamCamIngestor,
|
||||
VigoCCTVIngestor,
|
||||
VitoriaGasteizCCTVIngestor,
|
||||
)
|
||||
|
||||
for ingestor_cls in (
|
||||
@@ -84,6 +89,11 @@ def run_cctv_ingest_cycle():
|
||||
LTASingaporeIngestor,
|
||||
AustinTXIngestor,
|
||||
NYCDOTIngestor,
|
||||
SpainDGTIngestor,
|
||||
MadridCCTVIngestor,
|
||||
MalagaCCTVIngestor,
|
||||
VigoCCTVIngestor,
|
||||
VitoriaGasteizCCTVIngestor,
|
||||
):
|
||||
ingestor_cls().ingest()
|
||||
|
||||
|
||||
@@ -1,283 +0,0 @@
|
||||
"""
|
||||
Spain CCTV Ingestor
|
||||
===================
|
||||
Sources:
|
||||
- DGT (Dirección General de Tráfico) — national road cameras via DATEX2 XML
|
||||
No API key required. Covers all national roads EXCEPT Basque Country and Catalonia.
|
||||
~500-800 cameras across Spanish motorways and A-roads.
|
||||
|
||||
- Madrid City Hall — urban traffic cameras via open data KML
|
||||
No API key required. ~200 cameras across Madrid city centre.
|
||||
|
||||
Both sources are published under Spain's open data framework (Ley 37/2007 and
|
||||
EU PSI Directive 2019/1024). Free reuse with attribution required — source is
|
||||
credited via source_agency field which surfaces in the Shadowbroker UI.
|
||||
|
||||
Author: Alborz Nazari (github.com/AlborzNazari)
|
||||
"""
|
||||
|
||||
import logging
|
||||
import xml.etree.ElementTree as ET
|
||||
from typing import List, Dict, Any
|
||||
from services.cctv_pipeline import BaseCCTVIngestor
|
||||
from services.network_utils import fetch_with_curl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DGT National Roads — DATEX2 XML
|
||||
# ---------------------------------------------------------------------------
|
||||
# Full DATEX2 publication endpoint — no auth required, public open data.
|
||||
# Returns XML with <cctvCameraRecord> elements containing id, coords, image URL.
|
||||
# Note: excludes Basque Country (managed by Ertzaintza) and Catalonia (SCT).
|
||||
DGT_DATEX2_URL = (
|
||||
"http://infocar.dgt.es/datex2/dgt/PredefinedLocationsPublication/camaras/content.xml"
|
||||
)
|
||||
|
||||
# Still image URL pattern — substitute {id} with the camera serial from the XML.
|
||||
DGT_IMAGE_URL = "https://infocar.dgt.es/etraffic/data/camaras/{id}.jpg"
|
||||
|
||||
# DATEX2 namespace used by DGT's XML publication
|
||||
_NS = {
|
||||
"d2": "http://datex2.eu/schema/2/2_0",
|
||||
}
|
||||
|
||||
|
||||
class DGTNationalIngestor(BaseCCTVIngestor):
|
||||
"""
|
||||
DGT national road cameras using known working image URL pattern.
|
||||
Camera IDs 1-2000 cover the main national road network.
|
||||
Image URL pattern confirmed working: infocar.dgt.es/etraffic/data/camaras/{id}.jpg
|
||||
Coordinates sourced from the Madrid open data portal as a seed set.
|
||||
"""
|
||||
|
||||
# Confirmed working cameras with real coordinates (seed set)
|
||||
# Format: (id, lat, lon, description)
|
||||
KNOWN_CAMERAS = [
|
||||
(1398, 36.7213, -4.4214, "MA-19 Málaga"),
|
||||
(1001, 40.4168, -3.7038, "A-6 Madrid"),
|
||||
(1002, 40.4500, -3.6800, "A-2 Madrid"),
|
||||
(1003, 40.3800, -3.7200, "A-4 Madrid"),
|
||||
(1004, 40.4200, -3.8100, "A-5 Madrid"),
|
||||
(1005, 40.4600, -3.6600, "M-30 Madrid"),
|
||||
(1010, 41.3888, 2.1590, "AP-7 Barcelona"),
|
||||
(1011, 41.4100, 2.1800, "A-2 Barcelona"),
|
||||
(1020, 37.3891, -5.9845, "A-4 Sevilla"),
|
||||
(1021, 37.4000, -6.0000, "A-49 Sevilla"),
|
||||
(1030, 39.4699, -0.3763, "V-30 Valencia"),
|
||||
(1031, 39.4800, -0.3900, "A-3 Valencia"),
|
||||
(1040, 43.2630, -2.9350, "A-8 Bilbao"),
|
||||
(1050, 42.8782, -8.5448, "AG-55 Santiago"),
|
||||
(1060, 41.6488, -0.8891, "A-2 Zaragoza"),
|
||||
(1070, 37.9922, -1.1307, "A-30 Murcia"),
|
||||
(1080, 36.5271, -6.2886, "A-4 Cádiz"),
|
||||
(1090, 43.3623, -8.4115, "A-6 A Coruña"),
|
||||
(1100, 38.9942, -1.8585, "A-31 Albacete"),
|
||||
(1110, 39.8628, -4.0273, "A-4 Toledo"),
|
||||
]
|
||||
|
||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||
cameras = []
|
||||
for cam_id, lat, lon, description in self.KNOWN_CAMERAS:
|
||||
image_url = f"https://infocar.dgt.es/etraffic/data/camaras/{cam_id}.jpg"
|
||||
cameras.append({
|
||||
"id": f"DGT-{cam_id}",
|
||||
"source_agency": "DGT Spain",
|
||||
"lat": lat,
|
||||
"lon": lon,
|
||||
"direction_facing": description,
|
||||
"media_url": image_url,
|
||||
"refresh_rate_seconds": 300,
|
||||
})
|
||||
logger.info(f"DGTNationalIngestor: loaded {len(cameras)} cameras")
|
||||
return cameras
|
||||
cameras = []
|
||||
|
||||
# DATEX2 XML may or may not use a namespace prefix depending on the DGT
|
||||
# publication version. We try namespaced lookup first, then fall back to
|
||||
# a tag-name search that ignores namespaces entirely.
|
||||
records = root.findall(".//d2:cctvCameraRecord", _NS)
|
||||
if not records:
|
||||
# Fallback: namespace-agnostic search
|
||||
records = [el for el in root.iter() if el.tag.endswith("cctvCameraRecord")]
|
||||
|
||||
for record in records:
|
||||
try:
|
||||
cam_id = _find_text(record, "cctvCameraSerialNumber")
|
||||
if not cam_id:
|
||||
# Use the XML id attribute as fallback
|
||||
cam_id = record.get("id", "").replace("CAMERA_", "")
|
||||
if not cam_id:
|
||||
continue
|
||||
|
||||
lat = _find_text(record, "latitude")
|
||||
lon = _find_text(record, "longitude")
|
||||
if not lat or not lon:
|
||||
continue
|
||||
|
||||
# Prefer the stillImageUrl from the XML if present,
|
||||
# otherwise construct from the known DGT pattern.
|
||||
image_url = _find_text(record, "stillImageUrl")
|
||||
if not image_url:
|
||||
image_url = DGT_IMAGE_URL.format(id=cam_id)
|
||||
|
||||
# Road/description tag varies across DGT XML versions
|
||||
description = (
|
||||
_find_text(record, "locationDescription")
|
||||
or _find_text(record, "roadNumber")
|
||||
or f"DGT Camera {cam_id}"
|
||||
)
|
||||
|
||||
cameras.append({
|
||||
"id": f"DGT-{cam_id}",
|
||||
"source_agency": "DGT Spain",
|
||||
"lat": float(lat),
|
||||
"lon": float(lon),
|
||||
"direction_facing": description,
|
||||
"media_url": image_url,
|
||||
"refresh_rate_seconds": 300, # DGT updates stills every ~5 min
|
||||
})
|
||||
|
||||
except (ValueError, TypeError) as e:
|
||||
logger.debug(f"DGTNationalIngestor: skipping malformed record: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"DGTNationalIngestor: parsed {len(cameras)} cameras")
|
||||
return cameras
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Madrid City Hall — KML open data
|
||||
# ---------------------------------------------------------------------------
|
||||
# Published on datos.madrid.es. KML file with Placemark elements, each containing
|
||||
# camera location and a description with the image URL.
|
||||
# Licence: Madrid Open Data (free reuse with attribution).
|
||||
MADRID_KML_URL = (
|
||||
"http://datos.madrid.es/egob/catalogo/202088-0-trafico-camaras.kml"
|
||||
)
|
||||
|
||||
# KML namespace
|
||||
_KML_NS = {"kml": "http://www.opengis.net/kml/2.2"}
|
||||
|
||||
|
||||
class MadridCityIngestor(BaseCCTVIngestor):
|
||||
"""
|
||||
Fetches Madrid City Hall traffic cameras from the datos.madrid.es KML feed.
|
||||
|
||||
KML structure:
|
||||
<Placemark>
|
||||
<name>Camera name / road</name>
|
||||
<Point>
|
||||
<coordinates>-3.703790,40.416775,0</coordinates>
|
||||
</Point>
|
||||
<description><![CDATA[... image URL embedded ...]]></description>
|
||||
</Placemark>
|
||||
|
||||
Images are served as snapshots updated every 10 minutes.
|
||||
"""
|
||||
|
||||
def fetch_data(self) -> List[Dict[str, Any]]:
|
||||
try:
|
||||
response = fetch_with_curl(MADRID_KML_URL, timeout=20)
|
||||
response.raise_for_status()
|
||||
except Exception as e:
|
||||
logger.error(f"MadridCityIngestor: failed to fetch KML: {e}")
|
||||
return []
|
||||
|
||||
try:
|
||||
root = ET.fromstring(response.content)
|
||||
except ET.ParseError as e:
|
||||
logger.error(f"MadridCityIngestor: failed to parse KML: {e}")
|
||||
return []
|
||||
|
||||
cameras = []
|
||||
|
||||
# Try namespaced lookup, fall back to tag-name search
|
||||
placemarks = root.findall(".//kml:Placemark", _KML_NS)
|
||||
if not placemarks:
|
||||
placemarks = [el for el in root.iter() if el.tag.endswith("Placemark")]
|
||||
|
||||
for i, placemark in enumerate(placemarks):
|
||||
try:
|
||||
name_el = _find_element(placemark, "name")
|
||||
name = name_el.text.strip() if name_el is not None and name_el.text else f"Madrid Camera {i}"
|
||||
|
||||
coords_el = _find_element(placemark, "coordinates")
|
||||
if coords_el is None or not coords_el.text:
|
||||
continue
|
||||
|
||||
# KML coordinates are lon,lat,elevation
|
||||
parts = coords_el.text.strip().split(",")
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
lon = float(parts[0])
|
||||
lat = float(parts[1])
|
||||
|
||||
# Madrid KML embeds the image URL inside the description CDATA block.
|
||||
# It looks like: <img src="https://...jpg"> or a plain URL.
|
||||
# We extract the src attribute value if present.
|
||||
desc_el = _find_element(placemark, "description")
|
||||
image_url = None
|
||||
if desc_el is not None and desc_el.text:
|
||||
image_url = _extract_img_src(desc_el.text)
|
||||
|
||||
if not image_url:
|
||||
# No image available for this placemark — skip it
|
||||
continue
|
||||
|
||||
cameras.append({
|
||||
"id": f"MAD-{i:04d}",
|
||||
"source_agency": "Madrid City Hall",
|
||||
"lat": lat,
|
||||
"lon": lon,
|
||||
"direction_facing": name,
|
||||
"media_url": image_url,
|
||||
"refresh_rate_seconds": 600, # Madrid updates every 10 min
|
||||
})
|
||||
|
||||
except (ValueError, TypeError, IndexError) as e:
|
||||
logger.debug(f"MadridCityIngestor: skipping malformed placemark: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"MadridCityIngestor: parsed {len(cameras)} cameras")
|
||||
return cameras
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _find_text(element: ET.Element, tag: str) -> str | None:
|
||||
"""Find first child element matching tag (ignoring XML namespace) and return its text."""
|
||||
el = _find_element(element, tag)
|
||||
return el.text.strip() if el is not None and el.text else None
|
||||
|
||||
|
||||
def _find_element(element: ET.Element, tag: str) -> ET.Element | None:
|
||||
"""Find first descendant element matching tag, ignoring XML namespace prefix."""
|
||||
# Try exact match first (no namespace)
|
||||
el = element.find(f".//{tag}")
|
||||
if el is not None:
|
||||
return el
|
||||
# Try namespace-agnostic search
|
||||
for child in element.iter():
|
||||
if child.tag.endswith(f"}}{tag}") or child.tag == tag:
|
||||
return child
|
||||
return None
|
||||
|
||||
|
||||
def _extract_img_src(html_fragment: str) -> str | None:
|
||||
"""
|
||||
Extract src URL from an <img src="..."> HTML fragment.
|
||||
Falls back to finding any http/https URL in the string.
|
||||
"""
|
||||
import re
|
||||
# Look for src="..." or src='...'
|
||||
match = re.search(r'src=["\']([^"\']+)["\']', html_fragment, re.IGNORECASE)
|
||||
if match:
|
||||
return match.group(1)
|
||||
# Fallback: bare URL
|
||||
match = re.search(r'https?://\S+\.jpg', html_fragment, re.IGNORECASE)
|
||||
if match:
|
||||
return match.group(0)
|
||||
return None
|
||||
@@ -857,7 +857,7 @@ function NewsFeedInner({ data, selectedEntity, regionDossier, regionDossierLoadi
|
||||
? new Date(selectedEntity.extra.last_updated + 'Z').toLocaleString('en-US', { month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit', second: '2-digit', hour12: false, timeZoneName: 'short' }).toUpperCase() + ' — OPTIC INTERCEPT'
|
||||
: 'OPTIC INTERCEPT'}
|
||||
</h2>
|
||||
<span className="text-[10px] text-[var(--text-muted)] font-mono">ID: {selectedEntity.id}</span>
|
||||
<span className="text-[10px] text-[var(--text-muted)] font-mono">ID: {selectedEntity.id}{selectedEntity.extra?.source_agency ? ` | ${selectedEntity.extra.source_agency}` : ''}</span>
|
||||
</div>
|
||||
<div className="relative w-full h-48 bg-black flex items-center justify-center p-1">
|
||||
{(() => {
|
||||
@@ -898,22 +898,24 @@ function NewsFeedInner({ data, selectedEntity, regionDossier, regionDossierLoadi
|
||||
<img
|
||||
src={url}
|
||||
alt="MJPEG Feed"
|
||||
referrerPolicy="no-referrer"
|
||||
className="w-full h-full object-cover border border-cyan-900/50 rounded-sm filter contrast-125 saturate-50"
|
||||
onError={(e) => {
|
||||
const target = e.target as HTMLImageElement;
|
||||
target.src = "https://via.placeholder.com/400x300.png?text=FEED+UNAVAILABLE";
|
||||
target.src = "data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='400' height='300'%3E%3Crect fill='%23111' width='400' height='300'/%3E%3Ctext x='50%25' y='50%25' dominant-baseline='middle' text-anchor='middle' fill='%2306b6d4' font-family='monospace' font-size='14'%3EFEED UNAVAILABLE%3C/text%3E%3C/svg%3E";
|
||||
}}
|
||||
/>
|
||||
);
|
||||
// satellite / image — standard img with referrer policy for external tiles
|
||||
// satellite / image
|
||||
return (
|
||||
<img
|
||||
src={url}
|
||||
alt="CCTV Feed"
|
||||
referrerPolicy="no-referrer"
|
||||
className="w-full h-full object-cover border border-cyan-900/50 rounded-sm filter contrast-125 saturate-50"
|
||||
onError={(e) => {
|
||||
const target = e.target as HTMLImageElement;
|
||||
target.src = "https://via.placeholder.com/400x300.png?text=NO+SIGNAL";
|
||||
target.src = "data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='400' height='300'%3E%3Crect fill='%23111' width='400' height='300'/%3E%3Ctext x='50%25' y='50%25' dominant-baseline='middle' text-anchor='middle' fill='%2306b6d4' font-family='monospace' font-size='14'%3ENO SIGNAL%3C/text%3E%3C/svg%3E";
|
||||
}}
|
||||
/>
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user