mirror of
https://github.com/BigBodyCobain/Shadowbroker.git
synced 2026-08-05 18:28:39 +02:00
Follow up on #375 review: dedupe live-data route and harden serializers.
Align full /api/live-data with slow-tier orjson options, remove dead main.py duplicate, cap slow batches to pool size, cancel queued work on timeout, and stop retrying HTTP 4xx/5xx. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -38,10 +38,6 @@
|
||||
"main",
|
||||
"routers.health"
|
||||
],
|
||||
"GET /api/live-data": [
|
||||
"main",
|
||||
"routers.data"
|
||||
],
|
||||
"GET /api/live-data/fast": [
|
||||
"main",
|
||||
"routers.data"
|
||||
|
||||
@@ -65,3 +65,68 @@ class TestLiveDataFullEndpoint:
|
||||
r2 = client.get("/api/live-data", headers={"If-None-Match": etag})
|
||||
assert r2.status_code == 304
|
||||
assert r2.headers.get("etag") == etag
|
||||
|
||||
def test_live_data_serializes_non_json_native_values(self, client):
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from services.fetchers import _store
|
||||
|
||||
with _store._data_lock:
|
||||
prior = _store.latest_data.get("gdelt")
|
||||
_store.latest_data["gdelt"] = [
|
||||
{"observed": datetime(2026, 1, 1, tzinfo=timezone.utc)},
|
||||
]
|
||||
try:
|
||||
r = client.get("/api/live-data")
|
||||
assert r.status_code == 200
|
||||
assert "2026-01-01" in r.text
|
||||
finally:
|
||||
with _store._data_lock:
|
||||
_store.latest_data["gdelt"] = prior
|
||||
|
||||
|
||||
class TestSlowTaskConcurrency:
|
||||
def test_run_tasks_caps_batch_size_to_executor_workers(self, monkeypatch):
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import services.data_fetcher as df
|
||||
|
||||
class _FakeExecutor:
|
||||
_max_workers = 2
|
||||
|
||||
def submit(self, func):
|
||||
return MagicMock()
|
||||
|
||||
mock_executor = _FakeExecutor()
|
||||
monkeypatch.setattr(df, "_executor_for_task_label", lambda _label: mock_executor)
|
||||
monkeypatch.setattr(df, "_SLOW_FETCH_CONCURRENCY", 8)
|
||||
|
||||
batch_sizes = []
|
||||
|
||||
def _capture_drain(_label, futures):
|
||||
batch_sizes.append(len(futures))
|
||||
|
||||
monkeypatch.setattr(df, "_drain_task_futures", _capture_drain)
|
||||
|
||||
jobs = [lambda: None for _ in range(5)]
|
||||
df._run_tasks("slow-tier-test", jobs)
|
||||
|
||||
assert batch_sizes == [2, 2, 1]
|
||||
|
||||
|
||||
class TestFetcherRetryScope:
|
||||
def test_http_error_is_not_retried(self, monkeypatch):
|
||||
import requests
|
||||
|
||||
from services.fetchers.retry import with_retry
|
||||
|
||||
attempts = {"n": 0}
|
||||
|
||||
@with_retry(max_retries=2, base_delay=0.01)
|
||||
def _raises_http():
|
||||
attempts["n"] += 1
|
||||
raise requests.HTTPError("403 Client Error")
|
||||
|
||||
with pytest.raises(requests.HTTPError):
|
||||
_raises_http()
|
||||
assert attempts["n"] == 1
|
||||
|
||||
Reference in New Issue
Block a user