Follow up on #375 review: dedupe live-data route and harden serializers.

Align full /api/live-data with slow-tier orjson options, remove dead main.py duplicate, cap slow batches to pool size, cancel queued work on timeout, and stop retrying HTTP 4xx/5xx.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
BigBodyCobain
2026-06-06 20:10:59 -06:00
co-authored by Cursor
parent 9a0a9a116a
commit bd81a940ff
7 changed files with 100 additions and 27 deletions
@@ -38,10 +38,6 @@
"main",
"routers.health"
],
"GET /api/live-data": [
"main",
"routers.data"
],
"GET /api/live-data/fast": [
"main",
"routers.data"
@@ -65,3 +65,68 @@ class TestLiveDataFullEndpoint:
r2 = client.get("/api/live-data", headers={"If-None-Match": etag})
assert r2.status_code == 304
assert r2.headers.get("etag") == etag
def test_live_data_serializes_non_json_native_values(self, client):
from datetime import datetime, timezone
from services.fetchers import _store
with _store._data_lock:
prior = _store.latest_data.get("gdelt")
_store.latest_data["gdelt"] = [
{"observed": datetime(2026, 1, 1, tzinfo=timezone.utc)},
]
try:
r = client.get("/api/live-data")
assert r.status_code == 200
assert "2026-01-01" in r.text
finally:
with _store._data_lock:
_store.latest_data["gdelt"] = prior
class TestSlowTaskConcurrency:
def test_run_tasks_caps_batch_size_to_executor_workers(self, monkeypatch):
from unittest.mock import MagicMock
import services.data_fetcher as df
class _FakeExecutor:
_max_workers = 2
def submit(self, func):
return MagicMock()
mock_executor = _FakeExecutor()
monkeypatch.setattr(df, "_executor_for_task_label", lambda _label: mock_executor)
monkeypatch.setattr(df, "_SLOW_FETCH_CONCURRENCY", 8)
batch_sizes = []
def _capture_drain(_label, futures):
batch_sizes.append(len(futures))
monkeypatch.setattr(df, "_drain_task_futures", _capture_drain)
jobs = [lambda: None for _ in range(5)]
df._run_tasks("slow-tier-test", jobs)
assert batch_sizes == [2, 2, 1]
class TestFetcherRetryScope:
def test_http_error_is_not_retried(self, monkeypatch):
import requests
from services.fetchers.retry import with_retry
attempts = {"n": 0}
@with_retry(max_retries=2, base_delay=0.01)
def _raises_http():
attempts["n"] += 1
raise requests.HTTPError("403 Client Error")
with pytest.raises(requests.HTTPError):
_raises_http()
assert attempts["n"] == 1