From 3a2d8ddd7582a77e4ebbd74464c58031218f1817 Mon Sep 17 00:00:00 2001 From: Singular Failure Date: Sat, 21 Mar 2026 15:10:43 +0100 Subject: [PATCH] feat: add Spanish CCTV feeds and fix image loading MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add 5 native ingestors to cctv_pipeline.py: DGT (~1,917 cameras), Madrid (~357), Málaga (~134), Vigo (~59), Vitoria-Gasteiz (~17) - Fix DGT DATEX2 parser to match actual XML schema (device elements, not CctvCameraRecord) - Wire all new ingestors into the scheduler via data_fetcher.py - Remove standalone spain_cctv.py by Alborz Nazari, replaced by native pipeline ingestors that integrate with the existing scheduler pattern - Fix CCTV image loading for servers with Referer-based hotlink protection (referrerPolicy="no-referrer") - Replace external via.placeholder.com fallbacks with inline SVG data URIs to avoid dependency on unreachable third-party service - Surface source_agency attribution in CCTV panel UI for open data license compliance (CC BY / Spain Ley 37/2007) Co-Authored-By: Claude Opus 4.6 (1M context) --- README.md | 12 +- backend/services/cctv_pipeline.py | 312 +++++++++++++++++++++++++++ backend/services/data_fetcher.py | 10 + backend/services/spain_cctv.py | 283 ------------------------ frontend/src/components/NewsFeed.tsx | 10 +- 5 files changed, 339 insertions(+), 288 deletions(-) delete mode 100644 backend/services/spain_cctv.py diff --git a/README.md b/README.md index 0e3ab87..b577415 100644 --- a/README.md +++ b/README.md @@ -219,11 +219,16 @@ helm install shadowbroker ./helm/chart --create-namespace --namespace shadowbrok ### 📷 Surveillance -* **CCTV Mesh** — 2,000+ live traffic cameras from: +* **CCTV Mesh** — 4,400+ live traffic cameras from: * 🇬🇧 Transport for London JamCams * 🇺🇸 Austin, TX TxDOT * 🇺🇸 NYC DOT * 🇸🇬 Singapore LTA + * 🇪🇸 Spanish DGT (national roads) + * 🇪🇸 Madrid City Hall + * 🇪🇸 Málaga City + * 🇪🇸 Vigo City + * 🇪🇸 Vitoria-Gasteiz * Custom URL ingestion * **Feed Rendering** — Automatic detection & rendering of video, MJPEG, HLS, embed, satellite tile, and image feeds * **Clustered Map Display** — Green dots cluster with count labels, decluster on zoom @@ -307,6 +312,11 @@ helm install shadowbroker ./helm/chart --create-namespace --namespace shadowbrok | [TxDOT](https://its.txdot.gov) | Austin TX traffic cameras | ~5min | No | | [NYC DOT](https://webcams.nyctmc.org) | NYC traffic cameras | ~5min | No | | [Singapore LTA](https://datamall.lta.gov.sg) | Singapore traffic cameras | ~5min | **Yes** | +| [DGT Spain](https://nap.dgt.es) | Spanish national road cameras | ~10min | No | +| [Madrid Open Data](https://datos.madrid.es) | Madrid urban traffic cameras | ~10min | No | +| [Málaga Open Data](https://datosabiertos.malaga.eu) | Málaga traffic cameras | ~10min | No | +| [Vigo Open Data](https://datos.vigo.org) | Vigo traffic cameras | ~10min | No | +| [Vitoria-Gasteiz](https://www.vitoria-gasteiz.org) | Vitoria-Gasteiz traffic cameras | ~10min | No | | [RestCountries](https://restcountries.com) | Country profile data | On-demand (cached 24h) | No | | [Wikidata SPARQL](https://query.wikidata.org) | Head of state data | On-demand (cached 24h) | No | | [Wikipedia API](https://en.wikipedia.org/api) | Location summaries & aircraft images | On-demand (cached) | No | diff --git a/backend/services/cctv_pipeline.py b/backend/services/cctv_pipeline.py index 1a3b89d..9be09a8 100644 --- a/backend/services/cctv_pipeline.py +++ b/backend/services/cctv_pipeline.py @@ -1,5 +1,7 @@ import logging +import re import sqlite3 +import xml.etree.ElementTree as ET from abc import ABC, abstractmethod from pathlib import Path from typing import Any, Dict, List @@ -284,6 +286,316 @@ class GlobalOSMCrawlingIngestor(BaseCCTVIngestor): return [] +# --------------------------------------------------------------------------- +# Spain — DGT National Roads (DATEX2 XML, ~1,900 cameras) +# --------------------------------------------------------------------------- +class SpainDGTIngestor(BaseCCTVIngestor): + # Dirección General de Tráfico — national road cameras via DATEX2 v3 XML. + # No API key required. Covers all national roads (autopistas, autovías, N-roads) + # EXCEPT Basque Country and Catalonia. + # Published under Spain's open data framework (Ley 37/2007, EU PSI Directive 2019/1024). + DGT_URL = "https://nap.dgt.es/datex2/v3/dgt/DevicePublication/camaras_datex2_v36.xml" + + def fetch_data(self) -> List[Dict[str, Any]]: + try: + response = fetch_with_curl(self.DGT_URL, timeout=30) + response.raise_for_status() + except Exception as e: + logger.error(f"SpainDGTIngestor: failed to fetch DATEX2 XML: {e}") + return [] + + try: + root = ET.fromstring(response.content) + except ET.ParseError as e: + logger.error(f"SpainDGTIngestor: failed to parse XML: {e}") + return [] + + cameras = [] + # DGT DATEX2 v3 uses elements with typeOfDevice=camera. + # Namespace-agnostic: match local name "device". + for el in root.iter(): + local = el.tag.split("}")[-1] if "}" in el.tag else el.tag + if local != "device": + continue + + try: + cam_id = el.get("id", "") + if not cam_id: + continue + + # Coordinates are nested: pointLocation > ... > pointCoordinates > latitude/longitude + lat = self._find_text(el, "latitude") + lon = self._find_text(el, "longitude") + if not lat or not lon: + continue + + image_url = self._find_text(el, "deviceUrl") or f"https://infocar.dgt.es/etraffic/data/camaras/{cam_id}.jpg" + + road_name = self._find_text(el, "roadName") or "" + road_dest = self._find_text(el, "roadDestination") or "" + description = f"{road_name} → {road_dest}".strip(" →") or f"DGT Camera {cam_id}" + + cameras.append({ + "id": f"DGT-{cam_id}", + "source_agency": "DGT Spain", + "lat": float(lat), + "lon": float(lon), + "direction_facing": description, + "media_url": image_url, + "refresh_rate_seconds": 300, + }) + except (ValueError, TypeError) as e: + logger.debug(f"SpainDGTIngestor: skipping malformed record: {e}") + continue + + logger.info(f"SpainDGTIngestor: parsed {len(cameras)} cameras") + return cameras + + @staticmethod + def _find_text(element: ET.Element, tag: str) -> str | None: + for child in element.iter(): + local = child.tag.split("}")[-1] if "}" in child.tag else child.tag + if local.lower() == tag.lower() and child.text: + return child.text.strip() + return None + + +# --------------------------------------------------------------------------- +# Spain — Madrid City Hall (KML, ~200 cameras) +# --------------------------------------------------------------------------- +class MadridCCTVIngestor(BaseCCTVIngestor): + # Madrid City Hall urban traffic cameras via open data KML. + # No API key required. Published on datos.madrid.es. + # Licence: Madrid Open Data (free reuse with attribution). + MADRID_URL = "http://datos.madrid.es/egob/catalogo/202088-0-trafico-camaras.kml" + + def fetch_data(self) -> List[Dict[str, Any]]: + try: + response = fetch_with_curl(self.MADRID_URL, timeout=20) + response.raise_for_status() + except Exception as e: + logger.error(f"MadridCCTVIngestor: failed to fetch KML: {e}") + return [] + + try: + root = ET.fromstring(response.content) + except ET.ParseError as e: + logger.error(f"MadridCCTVIngestor: failed to parse KML: {e}") + return [] + + cameras = [] + # KML namespace varies — try both common ones, then fall back to tag-name search + placemarks = root.findall(".//{http://www.opengis.net/kml/2.2}Placemark") + if not placemarks: + placemarks = root.findall(".//{http://earth.google.com/kml/2.2}Placemark") + if not placemarks: + placemarks = [el for el in root.iter() if el.tag.endswith("Placemark")] + + for i, pm in enumerate(placemarks): + try: + name = self._find_kml_text(pm, "name") or f"Madrid Camera {i}" + coords_text = self._find_kml_text(pm, "coordinates") + if not coords_text: + continue + + # KML coordinates: lon,lat,elevation + parts = coords_text.strip().split(",") + if len(parts) < 2: + continue + lon, lat = float(parts[0]), float(parts[1]) + + # Extract image URL from description CDATA + desc = self._find_kml_text(pm, "description") or "" + image_url = self._extract_img_src(desc) + if not image_url: + continue + + cameras.append({ + "id": f"MAD-{i:04d}", + "source_agency": "Madrid City Hall", + "lat": lat, + "lon": lon, + "direction_facing": name, + "media_url": image_url, + "refresh_rate_seconds": 600, + }) + except (ValueError, TypeError, IndexError) as e: + logger.debug(f"MadridCCTVIngestor: skipping malformed placemark: {e}") + continue + + logger.info(f"MadridCCTVIngestor: parsed {len(cameras)} cameras") + return cameras + + @staticmethod + def _find_kml_text(element: ET.Element, tag: str) -> str | None: + for child in element.iter(): + local = child.tag.split("}")[-1] if "}" in child.tag else child.tag + if local == tag and child.text: + return child.text.strip() + return None + + @staticmethod + def _extract_img_src(html_fragment: str) -> str | None: + match = re.search(r'src=["\']([^"\']+)["\']', html_fragment, re.IGNORECASE) + if match: + return match.group(1) + match = re.search(r'https?://\S+\.jpg', html_fragment, re.IGNORECASE) + if match: + return match.group(0) + return None + + +# --------------------------------------------------------------------------- +# Spain — Málaga (GeoJSON, ~134 cameras) +# --------------------------------------------------------------------------- +class MalagaCCTVIngestor(BaseCCTVIngestor): + # Málaga open data — traffic cameras in EPSG:4326 GeoJSON. + # No API key required. Published on datosabiertos.malaga.eu. + MALAGA_URL = "https://datosabiertos.malaga.eu/recursos/transporte/trafico/da_camarasTrafico-4326.geojson" + + def fetch_data(self) -> List[Dict[str, Any]]: + try: + response = fetch_with_curl(self.MALAGA_URL, timeout=15) + response.raise_for_status() + data = response.json() + except Exception as e: + logger.error(f"MalagaCCTVIngestor: failed to fetch GeoJSON: {e}") + return [] + + cameras = [] + for feature in data.get("features", []): + try: + props = feature.get("properties", {}) + geom = feature.get("geometry", {}) + coords = geom.get("coordinates", []) + if len(coords) < 2: + continue + + image_url = props.get("URLIMAGEN") or props.get("urlimagen") + if not image_url: + continue + + cam_id = props.get("NOMBRE") or props.get("nombre") or str(coords) + description = props.get("DESCRIPCION") or props.get("descripcion") or cam_id + + cameras.append({ + "id": f"MLG-{cam_id}", + "source_agency": "Málaga City", + "lat": float(coords[1]), + "lon": float(coords[0]), + "direction_facing": description, + "media_url": image_url, + "refresh_rate_seconds": 300, + }) + except (ValueError, TypeError, IndexError) as e: + logger.debug(f"MalagaCCTVIngestor: skipping malformed feature: {e}") + continue + + logger.info(f"MalagaCCTVIngestor: parsed {len(cameras)} cameras") + return cameras + + +# --------------------------------------------------------------------------- +# Spain — Vigo (GeoJSON, ~59 cameras) +# --------------------------------------------------------------------------- +class VigoCCTVIngestor(BaseCCTVIngestor): + # Vigo open data — traffic cameras in GeoJSON. + # No API key required. Published on datos.vigo.org. + VIGO_URL = "https://datos.vigo.org/data/trafico/camaras-trafico.geojson" + + def fetch_data(self) -> List[Dict[str, Any]]: + try: + response = fetch_with_curl(self.VIGO_URL, timeout=15) + response.raise_for_status() + data = response.json() + except Exception as e: + logger.error(f"VigoCCTVIngestor: failed to fetch GeoJSON: {e}") + return [] + + cameras = [] + for feature in data.get("features", []): + try: + props = feature.get("properties", {}) + geom = feature.get("geometry", {}) + coords = geom.get("coordinates", []) + if len(coords) < 2: + continue + + # Vigo uses PHP image endpoints + image_url = props.get("urlimagen") or props.get("URLIMAGEN") or props.get("url") + if not image_url: + continue + + cam_id = props.get("id") or props.get("nombre") or str(coords) + description = props.get("nombre") or props.get("descripcion") or f"Vigo Camera {cam_id}" + + cameras.append({ + "id": f"VGO-{cam_id}", + "source_agency": "Vigo City", + "lat": float(coords[1]), + "lon": float(coords[0]), + "direction_facing": description, + "media_url": image_url, + "refresh_rate_seconds": 300, + }) + except (ValueError, TypeError, IndexError) as e: + logger.debug(f"VigoCCTVIngestor: skipping malformed feature: {e}") + continue + + logger.info(f"VigoCCTVIngestor: parsed {len(cameras)} cameras") + return cameras + + +# --------------------------------------------------------------------------- +# Spain — Vitoria-Gasteiz (GeoJSON, ~17 cameras) +# --------------------------------------------------------------------------- +class VitoriaGasteizCCTVIngestor(BaseCCTVIngestor): + # Vitoria-Gasteiz municipal traffic cameras in GeoJSON. + # No API key required. Published on vitoria-gasteiz.org. + VITORIA_URL = "https://www.vitoria-gasteiz.org/c11-01w/cameras?action=list&format=GEOJSON" + + def fetch_data(self) -> List[Dict[str, Any]]: + try: + response = fetch_with_curl(self.VITORIA_URL, timeout=15) + response.raise_for_status() + data = response.json() + except Exception as e: + logger.error(f"VitoriaGasteizCCTVIngestor: failed to fetch GeoJSON: {e}") + return [] + + cameras = [] + for feature in data.get("features", []): + try: + props = feature.get("properties", {}) + geom = feature.get("geometry", {}) + coords = geom.get("coordinates", []) + if len(coords) < 2: + continue + + image_url = props.get("imagen") or props.get("url") + if not image_url: + continue + + cam_id = props.get("id") or props.get("nombre") or str(coords) + description = props.get("nombre") or props.get("descripcion") or f"Vitoria Camera {cam_id}" + + cameras.append({ + "id": f"VIT-{cam_id}", + "source_agency": "Vitoria-Gasteiz", + "lat": float(coords[1]), + "lon": float(coords[0]), + "direction_facing": description, + "media_url": image_url, + "refresh_rate_seconds": 300, + }) + except (ValueError, TypeError, IndexError) as e: + logger.debug(f"VitoriaGasteizCCTVIngestor: skipping malformed feature: {e}") + continue + + logger.info(f"VitoriaGasteizCCTVIngestor: parsed {len(cameras)} cameras") + return cameras + + def _detect_media_type(url: str) -> str: """Detect the media type from a camera URL for proper frontend rendering.""" if not url: diff --git a/backend/services/data_fetcher.py b/backend/services/data_fetcher.py index 2e1a90a..24e90ac 100644 --- a/backend/services/data_fetcher.py +++ b/backend/services/data_fetcher.py @@ -75,8 +75,13 @@ def run_cctv_ingest_cycle(): from services.cctv_pipeline import ( AustinTXIngestor, LTASingaporeIngestor, + MadridCCTVIngestor, + MalagaCCTVIngestor, NYCDOTIngestor, + SpainDGTIngestor, TFLJamCamIngestor, + VigoCCTVIngestor, + VitoriaGasteizCCTVIngestor, ) for ingestor_cls in ( @@ -84,6 +89,11 @@ def run_cctv_ingest_cycle(): LTASingaporeIngestor, AustinTXIngestor, NYCDOTIngestor, + SpainDGTIngestor, + MadridCCTVIngestor, + MalagaCCTVIngestor, + VigoCCTVIngestor, + VitoriaGasteizCCTVIngestor, ): ingestor_cls().ingest() diff --git a/backend/services/spain_cctv.py b/backend/services/spain_cctv.py deleted file mode 100644 index c0745a8..0000000 --- a/backend/services/spain_cctv.py +++ /dev/null @@ -1,283 +0,0 @@ -""" -Spain CCTV Ingestor -=================== -Sources: - - DGT (Dirección General de Tráfico) — national road cameras via DATEX2 XML - No API key required. Covers all national roads EXCEPT Basque Country and Catalonia. - ~500-800 cameras across Spanish motorways and A-roads. - - - Madrid City Hall — urban traffic cameras via open data KML - No API key required. ~200 cameras across Madrid city centre. - -Both sources are published under Spain's open data framework (Ley 37/2007 and -EU PSI Directive 2019/1024). Free reuse with attribution required — source is -credited via source_agency field which surfaces in the Shadowbroker UI. - -Author: Alborz Nazari (github.com/AlborzNazari) -""" - -import logging -import xml.etree.ElementTree as ET -from typing import List, Dict, Any -from services.cctv_pipeline import BaseCCTVIngestor -from services.network_utils import fetch_with_curl - -logger = logging.getLogger(__name__) - -# --------------------------------------------------------------------------- -# DGT National Roads — DATEX2 XML -# --------------------------------------------------------------------------- -# Full DATEX2 publication endpoint — no auth required, public open data. -# Returns XML with elements containing id, coords, image URL. -# Note: excludes Basque Country (managed by Ertzaintza) and Catalonia (SCT). -DGT_DATEX2_URL = ( - "http://infocar.dgt.es/datex2/dgt/PredefinedLocationsPublication/camaras/content.xml" -) - -# Still image URL pattern — substitute {id} with the camera serial from the XML. -DGT_IMAGE_URL = "https://infocar.dgt.es/etraffic/data/camaras/{id}.jpg" - -# DATEX2 namespace used by DGT's XML publication -_NS = { - "d2": "http://datex2.eu/schema/2/2_0", -} - - -class DGTNationalIngestor(BaseCCTVIngestor): - """ - DGT national road cameras using known working image URL pattern. - Camera IDs 1-2000 cover the main national road network. - Image URL pattern confirmed working: infocar.dgt.es/etraffic/data/camaras/{id}.jpg - Coordinates sourced from the Madrid open data portal as a seed set. - """ - - # Confirmed working cameras with real coordinates (seed set) - # Format: (id, lat, lon, description) - KNOWN_CAMERAS = [ - (1398, 36.7213, -4.4214, "MA-19 Málaga"), - (1001, 40.4168, -3.7038, "A-6 Madrid"), - (1002, 40.4500, -3.6800, "A-2 Madrid"), - (1003, 40.3800, -3.7200, "A-4 Madrid"), - (1004, 40.4200, -3.8100, "A-5 Madrid"), - (1005, 40.4600, -3.6600, "M-30 Madrid"), - (1010, 41.3888, 2.1590, "AP-7 Barcelona"), - (1011, 41.4100, 2.1800, "A-2 Barcelona"), - (1020, 37.3891, -5.9845, "A-4 Sevilla"), - (1021, 37.4000, -6.0000, "A-49 Sevilla"), - (1030, 39.4699, -0.3763, "V-30 Valencia"), - (1031, 39.4800, -0.3900, "A-3 Valencia"), - (1040, 43.2630, -2.9350, "A-8 Bilbao"), - (1050, 42.8782, -8.5448, "AG-55 Santiago"), - (1060, 41.6488, -0.8891, "A-2 Zaragoza"), - (1070, 37.9922, -1.1307, "A-30 Murcia"), - (1080, 36.5271, -6.2886, "A-4 Cádiz"), - (1090, 43.3623, -8.4115, "A-6 A Coruña"), - (1100, 38.9942, -1.8585, "A-31 Albacete"), - (1110, 39.8628, -4.0273, "A-4 Toledo"), - ] - - def fetch_data(self) -> List[Dict[str, Any]]: - cameras = [] - for cam_id, lat, lon, description in self.KNOWN_CAMERAS: - image_url = f"https://infocar.dgt.es/etraffic/data/camaras/{cam_id}.jpg" - cameras.append({ - "id": f"DGT-{cam_id}", - "source_agency": "DGT Spain", - "lat": lat, - "lon": lon, - "direction_facing": description, - "media_url": image_url, - "refresh_rate_seconds": 300, - }) - logger.info(f"DGTNationalIngestor: loaded {len(cameras)} cameras") - return cameras - cameras = [] - - # DATEX2 XML may or may not use a namespace prefix depending on the DGT - # publication version. We try namespaced lookup first, then fall back to - # a tag-name search that ignores namespaces entirely. - records = root.findall(".//d2:cctvCameraRecord", _NS) - if not records: - # Fallback: namespace-agnostic search - records = [el for el in root.iter() if el.tag.endswith("cctvCameraRecord")] - - for record in records: - try: - cam_id = _find_text(record, "cctvCameraSerialNumber") - if not cam_id: - # Use the XML id attribute as fallback - cam_id = record.get("id", "").replace("CAMERA_", "") - if not cam_id: - continue - - lat = _find_text(record, "latitude") - lon = _find_text(record, "longitude") - if not lat or not lon: - continue - - # Prefer the stillImageUrl from the XML if present, - # otherwise construct from the known DGT pattern. - image_url = _find_text(record, "stillImageUrl") - if not image_url: - image_url = DGT_IMAGE_URL.format(id=cam_id) - - # Road/description tag varies across DGT XML versions - description = ( - _find_text(record, "locationDescription") - or _find_text(record, "roadNumber") - or f"DGT Camera {cam_id}" - ) - - cameras.append({ - "id": f"DGT-{cam_id}", - "source_agency": "DGT Spain", - "lat": float(lat), - "lon": float(lon), - "direction_facing": description, - "media_url": image_url, - "refresh_rate_seconds": 300, # DGT updates stills every ~5 min - }) - - except (ValueError, TypeError) as e: - logger.debug(f"DGTNationalIngestor: skipping malformed record: {e}") - continue - - logger.info(f"DGTNationalIngestor: parsed {len(cameras)} cameras") - return cameras - - -# --------------------------------------------------------------------------- -# Madrid City Hall — KML open data -# --------------------------------------------------------------------------- -# Published on datos.madrid.es. KML file with Placemark elements, each containing -# camera location and a description with the image URL. -# Licence: Madrid Open Data (free reuse with attribution). -MADRID_KML_URL = ( - "http://datos.madrid.es/egob/catalogo/202088-0-trafico-camaras.kml" -) - -# KML namespace -_KML_NS = {"kml": "http://www.opengis.net/kml/2.2"} - - -class MadridCityIngestor(BaseCCTVIngestor): - """ - Fetches Madrid City Hall traffic cameras from the datos.madrid.es KML feed. - - KML structure: - - Camera name / road - - -3.703790,40.416775,0 - - - - - Images are served as snapshots updated every 10 minutes. - """ - - def fetch_data(self) -> List[Dict[str, Any]]: - try: - response = fetch_with_curl(MADRID_KML_URL, timeout=20) - response.raise_for_status() - except Exception as e: - logger.error(f"MadridCityIngestor: failed to fetch KML: {e}") - return [] - - try: - root = ET.fromstring(response.content) - except ET.ParseError as e: - logger.error(f"MadridCityIngestor: failed to parse KML: {e}") - return [] - - cameras = [] - - # Try namespaced lookup, fall back to tag-name search - placemarks = root.findall(".//kml:Placemark", _KML_NS) - if not placemarks: - placemarks = [el for el in root.iter() if el.tag.endswith("Placemark")] - - for i, placemark in enumerate(placemarks): - try: - name_el = _find_element(placemark, "name") - name = name_el.text.strip() if name_el is not None and name_el.text else f"Madrid Camera {i}" - - coords_el = _find_element(placemark, "coordinates") - if coords_el is None or not coords_el.text: - continue - - # KML coordinates are lon,lat,elevation - parts = coords_el.text.strip().split(",") - if len(parts) < 2: - continue - lon = float(parts[0]) - lat = float(parts[1]) - - # Madrid KML embeds the image URL inside the description CDATA block. - # It looks like: or a plain URL. - # We extract the src attribute value if present. - desc_el = _find_element(placemark, "description") - image_url = None - if desc_el is not None and desc_el.text: - image_url = _extract_img_src(desc_el.text) - - if not image_url: - # No image available for this placemark — skip it - continue - - cameras.append({ - "id": f"MAD-{i:04d}", - "source_agency": "Madrid City Hall", - "lat": lat, - "lon": lon, - "direction_facing": name, - "media_url": image_url, - "refresh_rate_seconds": 600, # Madrid updates every 10 min - }) - - except (ValueError, TypeError, IndexError) as e: - logger.debug(f"MadridCityIngestor: skipping malformed placemark: {e}") - continue - - logger.info(f"MadridCityIngestor: parsed {len(cameras)} cameras") - return cameras - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def _find_text(element: ET.Element, tag: str) -> str | None: - """Find first child element matching tag (ignoring XML namespace) and return its text.""" - el = _find_element(element, tag) - return el.text.strip() if el is not None and el.text else None - - -def _find_element(element: ET.Element, tag: str) -> ET.Element | None: - """Find first descendant element matching tag, ignoring XML namespace prefix.""" - # Try exact match first (no namespace) - el = element.find(f".//{tag}") - if el is not None: - return el - # Try namespace-agnostic search - for child in element.iter(): - if child.tag.endswith(f"}}{tag}") or child.tag == tag: - return child - return None - - -def _extract_img_src(html_fragment: str) -> str | None: - """ - Extract src URL from an HTML fragment. - Falls back to finding any http/https URL in the string. - """ - import re - # Look for src="..." or src='...' - match = re.search(r'src=["\']([^"\']+)["\']', html_fragment, re.IGNORECASE) - if match: - return match.group(1) - # Fallback: bare URL - match = re.search(r'https?://\S+\.jpg', html_fragment, re.IGNORECASE) - if match: - return match.group(0) - return None diff --git a/frontend/src/components/NewsFeed.tsx b/frontend/src/components/NewsFeed.tsx index c9dfeff..f262fac 100644 --- a/frontend/src/components/NewsFeed.tsx +++ b/frontend/src/components/NewsFeed.tsx @@ -857,7 +857,7 @@ function NewsFeedInner({ data, selectedEntity, regionDossier, regionDossierLoadi ? new Date(selectedEntity.extra.last_updated + 'Z').toLocaleString('en-US', { month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit', second: '2-digit', hour12: false, timeZoneName: 'short' }).toUpperCase() + ' — OPTIC INTERCEPT' : 'OPTIC INTERCEPT'} - ID: {selectedEntity.id} + ID: {selectedEntity.id}{selectedEntity.extra?.source_agency ? ` | ${selectedEntity.extra.source_agency}` : ''}
{(() => { @@ -898,22 +898,24 @@ function NewsFeedInner({ data, selectedEntity, regionDossier, regionDossierLoadi MJPEG Feed { const target = e.target as HTMLImageElement; - target.src = "https://via.placeholder.com/400x300.png?text=FEED+UNAVAILABLE"; + target.src = "data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='400' height='300'%3E%3Crect fill='%23111' width='400' height='300'/%3E%3Ctext x='50%25' y='50%25' dominant-baseline='middle' text-anchor='middle' fill='%2306b6d4' font-family='monospace' font-size='14'%3EFEED UNAVAILABLE%3C/text%3E%3C/svg%3E"; }} /> ); - // satellite / image — standard img with referrer policy for external tiles + // satellite / image return ( CCTV Feed { const target = e.target as HTMLImageElement; - target.src = "https://via.placeholder.com/400x300.png?text=NO+SIGNAL"; + target.src = "data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='400' height='300'%3E%3Crect fill='%23111' width='400' height='300'/%3E%3Ctext x='50%25' y='50%25' dominant-baseline='middle' text-anchor='middle' fill='%2306b6d4' font-family='monospace' font-size='14'%3ENO SIGNAL%3C/text%3E%3C/svg%3E"; }} /> );