"""Train tracking fetchers with normalized metadata and non-redundant merging.""" from __future__ import annotations import logging import math from collections.abc import Callable from datetime import datetime, timezone from services.fetchers._store import _data_lock, _mark_fresh, latest_data from services.network_utils import fetch_with_curl logger = logging.getLogger(__name__) _EARTH_RADIUS_KM = 6371.0 _MERGE_DISTANCE_KM = 5.0 _MAX_INFERRED_SPEED_KMH = 350.0 _TRACK_CACHE_TTL_S = 6 * 60 * 60 _SOURCE_METADATA: dict[str, dict[str, object]] = { "amtrak": { "source_label": "Amtraker", "operator": "Amtrak", "country": "US", "telemetry_quality": "aggregated", "priority": 70, }, "digitraffic": { "source_label": "Digitraffic Finland", "operator": "Finnish Rail", "country": "FI", "telemetry_quality": "official", "priority": 100, }, # Future slots so better official feeds can be merged without changing the # rest of the train pipeline or duplicating map entities. "networkrail": { "source_label": "Network Rail Open Data", "operator": "Network Rail", "country": "GB", "telemetry_quality": "official", "priority": 98, }, "dbcargo": { "source_label": "DB Cargo link2rail", "operator": "DB Cargo", "country": "DE", "telemetry_quality": "commercial", "priority": 96, }, "railinc": { "source_label": "Railinc RailSight", "operator": "Railinc", "country": "US", "telemetry_quality": "commercial", "priority": 97, }, "sncf": { "source_label": "SNCF Open Data", "operator": "SNCF", "country": "FR", "telemetry_quality": "official", "priority": 94, }, } _TRAIN_TRACK_CACHE: dict[str, dict[str, float]] = {} def _safe_float(value) -> float | None: try: if value is None or value == "": return None return float(value) except (TypeError, ValueError): return None def _parse_observed_at(value) -> float | None: if value is None or value == "": return None if isinstance(value, (int, float)): raw = float(value) return raw / 1000.0 if raw > 1_000_000_000_000 else raw if not isinstance(value, str): return None text = value.strip() if not text: return None if text.endswith("Z"): text = f"{text[:-1]}+00:00" try: return datetime.fromisoformat(text).timestamp() except ValueError: return None def _haversine_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float: lat1_rad, lon1_rad = math.radians(lat1), math.radians(lon1) lat2_rad, lon2_rad = math.radians(lat2), math.radians(lon2) dlat = lat2_rad - lat1_rad dlon = lon2_rad - lon1_rad a = ( math.sin(dlat / 2.0) ** 2 + math.cos(lat1_rad) * math.cos(lat2_rad) * math.sin(dlon / 2.0) ** 2 ) return 2.0 * _EARTH_RADIUS_KM * math.asin(math.sqrt(a)) def _bearing_degrees(lat1: float, lon1: float, lat2: float, lon2: float) -> float | None: if lat1 == lat2 and lon1 == lon2: return None lat1_rad, lat2_rad = math.radians(lat1), math.radians(lat2) dlon_rad = math.radians(lon2 - lon1) y = math.sin(dlon_rad) * math.cos(lat2_rad) x = ( math.cos(lat1_rad) * math.sin(lat2_rad) - math.sin(lat1_rad) * math.cos(lat2_rad) * math.cos(dlon_rad) ) return (math.degrees(math.atan2(y, x)) + 360.0) % 360.0 def _source_meta(source: str) -> dict[str, object]: return dict(_SOURCE_METADATA.get(source, {})) def _normalize_train( *, source: str, raw_id: str, number: str, lat, lng, name: str = "", status: str = "Active", route: str = "", speed_kmh=None, heading=None, operator: str | None = None, country: str | None = None, source_label: str | None = None, telemetry_quality: str | None = None, observed_at=None, ) -> dict | None: lat_f = _safe_float(lat) lng_f = _safe_float(lng) if lat_f is None or lng_f is None: return None if not (-90.0 <= lat_f <= 90.0 and -180.0 <= lng_f <= 180.0): return None number_text = str(number or "").strip() meta = _source_meta(source) observed_ts = _parse_observed_at(observed_at) or datetime.now(timezone.utc).timestamp() speed_f = _safe_float(speed_kmh) heading_f = _safe_float(heading) normalized = { "id": str(raw_id or f"{source}-{number_text or 'unknown'}"), "name": str(name or f"Train {number_text or '?'}").strip(), "number": number_text, "source": source, "source_label": str(source_label or meta.get("source_label") or source.upper()), "operator": str(operator or meta.get("operator") or "").strip(), "country": str(country or meta.get("country") or "").strip(), "telemetry_quality": str( telemetry_quality or meta.get("telemetry_quality") or "unknown" ).strip(), "lat": lat_f, "lng": lng_f, "speed_kmh": speed_f, "heading": heading_f, "status": str(status or "Active").strip(), "route": str(route or "").strip(), "_source_priority": int(meta.get("priority") or 0), "_observed_ts": observed_ts, } _apply_motion_estimates(normalized) return normalized def _prune_track_cache(now_ts: float) -> None: stale_before = now_ts - _TRACK_CACHE_TTL_S stale_ids = [train_id for train_id, entry in _TRAIN_TRACK_CACHE.items() if entry["ts"] < stale_before] for train_id in stale_ids: _TRAIN_TRACK_CACHE.pop(train_id, None) def _apply_motion_estimates(train: dict) -> None: train_id = str(train.get("id") or "") if not train_id: return now_ts = float(train.get("_observed_ts") or datetime.now(timezone.utc).timestamp()) _prune_track_cache(now_ts) previous = _TRAIN_TRACK_CACHE.get(train_id) if previous: dt_s = now_ts - previous["ts"] if 5.0 <= dt_s <= 15.0 * 60.0: distance_km = _haversine_km( float(previous["lat"]), float(previous["lng"]), float(train["lat"]), float(train["lng"]), ) if 0.02 <= distance_km <= (_MAX_INFERRED_SPEED_KMH * (dt_s / 3600.0)): if train.get("speed_kmh") is None: inferred_speed = distance_km / (dt_s / 3600.0) train["speed_kmh"] = round(min(inferred_speed, _MAX_INFERRED_SPEED_KMH), 1) if train.get("heading") is None: inferred_heading = _bearing_degrees( float(previous["lat"]), float(previous["lng"]), float(train["lat"]), float(train["lng"]), ) if inferred_heading is not None: train["heading"] = round(inferred_heading, 1) _TRAIN_TRACK_CACHE[train_id] = { "lat": float(train["lat"]), "lng": float(train["lng"]), "ts": now_ts, } def _train_merge_key(train: dict) -> str: operator = str(train.get("operator") or "").strip().lower() country = str(train.get("country") or "").strip().lower() number = str(train.get("number") or "").strip().lower() if operator and number: return f"{country}|{operator}|{number}" return f"{str(train.get('source') or '').lower()}|{str(train.get('id') or '').lower()}" def _train_completeness(train: dict) -> tuple[int, int, int]: return ( 1 if train.get("speed_kmh") is not None else 0, 1 if train.get("heading") is not None else 0, 1 if train.get("route") else 0, ) def _should_merge(existing: dict, candidate: dict) -> bool: if _train_merge_key(existing) != _train_merge_key(candidate): return False return _haversine_km( float(existing["lat"]), float(existing["lng"]), float(candidate["lat"]), float(candidate["lng"]), ) <= _MERGE_DISTANCE_KM def _merge_train_pair(existing: dict, candidate: dict) -> dict: existing_priority = int(existing.get("_source_priority") or 0) candidate_priority = int(candidate.get("_source_priority") or 0) existing_score = (existing_priority, _train_completeness(existing)) candidate_score = (candidate_priority, _train_completeness(candidate)) primary = candidate if candidate_score > existing_score else existing secondary = existing if primary is candidate else candidate merged = dict(primary) for field in ( "speed_kmh", "heading", "route", "status", "operator", "country", "source_label", "telemetry_quality", ): if merged.get(field) in (None, "", "Active"): replacement = secondary.get(field) if replacement not in (None, ""): merged[field] = replacement if primary is not candidate and float(candidate.get("_observed_ts") or 0) > float( primary.get("_observed_ts") or 0 ): merged["lat"] = candidate["lat"] merged["lng"] = candidate["lng"] merged["_observed_ts"] = candidate["_observed_ts"] return merged def _merge_nonredundant_trains(*sources: list[dict]) -> list[dict]: merged: list[dict] = [] for source_trains in sources: for train in source_trains: exact_match = next( ( idx for idx, existing in enumerate(merged) if existing.get("source") == train.get("source") and existing.get("id") == train.get("id") ), None, ) if exact_match is not None: merged[exact_match] = _merge_train_pair(merged[exact_match], train) continue merged_idx = next( (idx for idx, existing in enumerate(merged) if _should_merge(existing, train)), None, ) if merged_idx is not None: merged[merged_idx] = _merge_train_pair(merged[merged_idx], train) continue merged.append(train) merged.sort( key=lambda train: ( str(train.get("country") or ""), str(train.get("operator") or ""), str(train.get("number") or ""), str(train.get("id") or ""), ) ) for train in merged: train.pop("_source_priority", None) train.pop("_observed_ts", None) return merged def _fetch_amtraker() -> list[dict]: """Fetch all active Amtrak trains from the Amtraker API.""" try: resp = fetch_with_curl( "https://api.amtraker.com/v3/trains", timeout=20, headers={ "User-Agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/136.0.0.0 Safari/537.36" ), "Accept": "application/json,text/plain,*/*", "Referer": "https://www.amtraker.com/", }, ) if resp.status_code != 200: logger.warning("Amtraker returned %s", resp.status_code) return [] raw = resp.json() trains: list[dict] = [] for train_num, variants in raw.items(): if not isinstance(variants, list): continue for item in variants: normalized = _normalize_train( source="amtrak", raw_id=f"AMTK-{item.get('trainID', train_num)}", name=item.get("routeName", f"Train {train_num}"), number=str(item.get("trainNum", train_num) or train_num), lat=item.get("lat"), lng=item.get("lon"), speed_kmh=item.get("velocity") or item.get("speed"), heading=item.get("heading") or item.get("bearing"), status=item.get("trainTimely") or "On Time", route=item.get("routeName", ""), observed_at=item.get("updatedAt") or item.get("lastValTS") or item.get("eventDT"), ) if normalized: trains.append(normalized) return trains except Exception as exc: logger.warning("Amtraker fetch error: %s", exc) return [] def _fetch_digitraffic() -> list[dict]: """Fetch live train positions from Finnish DigiTraffic API.""" try: resp = fetch_with_curl( "https://rata.digitraffic.fi/api/v1/train-locations/latest", timeout=15, headers={ "Accept-Encoding": "gzip", "User-Agent": "ShadowBroker-OSINT/1.0", }, ) if resp.status_code != 200: logger.warning("DigiTraffic returned %s", resp.status_code) return [] raw = resp.json() trains: list[dict] = [] for item in raw: location = item.get("location", {}) coords = location.get("coordinates") if not coords or len(coords) < 2: continue lon, lat = coords[0], coords[1] train_number = str(item.get("trainNumber", "") or "").strip() route_bits = [ str(item.get("departureStationShortCode") or "").strip(), str(item.get("stationShortCode") or "").strip(), ] route = " -> ".join([bit for bit in route_bits if bit]) train_type = str(item.get("trainType") or "").strip() normalized = _normalize_train( source="digitraffic", raw_id=f"FIN-{train_number or len(trains)}", name=f"{train_type} {train_number}".strip() or f"Train {train_number or '?'}", number=train_number, lat=lat, lng=lon, speed_kmh=item.get("speed"), heading=item.get("heading"), status="Active", route=route, observed_at=item.get("timestamp"), ) if normalized: trains.append(normalized) return trains except Exception as exc: logger.warning("DigiTraffic fetch error: %s", exc) return [] _TRAIN_FETCHERS: tuple[tuple[str, Callable[[], list[dict]]], ...] = ( ("amtrak", _fetch_amtraker), ("digitraffic", _fetch_digitraffic), ) def fetch_trains(): """Fetch trains from all configured sources and merge without duplicates.""" with _data_lock: existing_trains = list(latest_data.get("trains") or []) source_batches: list[list[dict]] = [] source_counts: list[str] = [] for source_name, fetcher in _TRAIN_FETCHERS: batch = fetcher() source_batches.append(batch) if batch: source_counts.append(f"{source_name}:{len(batch)}") trains = _merge_nonredundant_trains(*source_batches) if not trains and existing_trains: logger.warning( "Train refresh returned 0 records — preserving %s cached trains until the next successful poll", len(existing_trains), ) trains = existing_trains with _data_lock: latest_data["trains"] = trains _mark_fresh("trains") logger.info( "Trains: %s total%s", len(trains), f" ({', '.join(source_counts)})" if source_counts else "", )