Follow up on #375 review: dedupe live-data route and harden serializers.

Align full /api/live-data with slow-tier orjson options, remove dead main.py duplicate, cap slow batches to pool size, cancel queued work on timeout, and stop retrying HTTP 4xx/5xx.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/services/data_fetcher.py

@@ -347,7 +347,14 @@ def _run_task_with_health_on_executor(
 
         record_success(task_name, duration_s=duration)
         if duration > _SLOW_FETCH_S:
-            logger.warning("task slow: %s took %.2f}s", task_name, duration)
+            logger.warning("task slow: %s took %.2fs", task_name, duration)
+    except concurrent.futures.TimeoutError:
+        future.cancel()
+        duration = time.perf_counter() - start
+        from services.fetch_health import record_failure
+
+        record_failure(task_name, error=TimeoutError(f"{task_name} timed out"), duration_s=duration)
+        logger.error("task timed out: %s (%.2fs)", task_name, duration)
     except Exception as e:
         duration = time.perf_counter() - start
         from services.fetch_health import record_failure
@@ -368,7 +375,8 @@ def _run_tasks(label: str, funcs: list, *, max_concurrency: int | None = None):
             max_concurrency = _STARTUP_HEAVY_CONCURRENCY
         else:
             max_concurrency = len(funcs)
-    max_concurrency = max(1, min(max_concurrency, len(funcs)))
+    pool_workers = getattr(executor, "_max_workers", len(funcs))
+    max_concurrency = max(1, min(max_concurrency, len(funcs), pool_workers))
 
     remaining_funcs = list(funcs)
     while remaining_funcs:
@@ -390,6 +398,13 @@ def _drain_task_futures(label: str, futures: dict):
             record_success(name, duration_s=duration)
             if duration > _SLOW_FETCH_S:
                 logger.warning(f"{label} task slow: {name} took {duration:.2f}s")
+        except concurrent.futures.TimeoutError:
+            future.cancel()
+            duration = time.perf_counter() - start
+            from services.fetch_health import record_failure
+
+            record_failure(name, error=TimeoutError(f"{name} timed out"), duration_s=duration)
+            logger.error("%s task timed out: %s (%.2fs)", label, name, duration)
         except Exception as e:
             duration = time.perf_counter() - start
             from services.fetch_health import record_failure

backend/services/fetchers/_store.py

@@ -230,12 +230,14 @@ _active_layers_version: int = 0
 def bump_active_layers_version() -> None:
     """Increment the active-layer version when frontend toggles change response shape."""
     global _active_layers_version
-    _active_layers_version += 1
+    with _data_lock:
+        _active_layers_version += 1
 
 
 def get_active_layers_version() -> int:
     """Return the current active-layer version (for ETag generation)."""
-    return _active_layers_version
+    with _data_lock:
+        return _active_layers_version
 
 
 def get_latest_data_subset(*keys: str) -> DashboardData:

backend/services/fetchers/retry.py

@@ -11,15 +11,20 @@ import random
 import logging
 import functools
 import requests
+from requests.exceptions import ChunkedEncodingError, ConnectionError as RequestsConnectionError
+from requests.exceptions import Timeout as RequestsTimeout
 
 logger = logging.getLogger(__name__)
 
-# Only retry on transient network/OS errors — not on parse errors, key errors, etc.
+# Only retry on transient network/OS errors — not parse/key errors or HTTP 4xx/5xx.
+# requests.HTTPError (from raise_for_status) is intentionally excluded.
 TRANSIENT_ERRORS = (
     TimeoutError,
     ConnectionError,
     OSError,
-    requests.RequestException,
+    RequestsConnectionError,
+    RequestsTimeout,
+    ChunkedEncodingError,
 )
 
 
@@ -43,6 +48,8 @@ def with_retry(max_retries: int = 3, base_delay: float = 2.0, max_delay: float =
             for attempt in range(1 + max_retries):
                 try:
                     return func(*args, **kwargs)
+                except requests.HTTPError:
+                    raise
                 except TRANSIENT_ERRORS as exc:
                     last_exc = exc
                     if attempt < max_retries: