mirror of
https://github.com/BigBodyCobain/Shadowbroker.git
synced 2026-06-08 07:13:53 +02:00
v0.9.5: The Voltron Update — modular architecture, stable IDs, parallelized boot
- Parallelized startup (60s → 15s) via ThreadPoolExecutor - Adaptive polling engine with ETag caching (no more bbox interrupts) - useCallback optimization for interpolation functions - Sliding LAYERS/INTEL edge panels replace bulky Record Panel - Modular fetcher architecture (flights, geo, infrastructure, financial, earth_observation) - Stable entity IDs for GDELT & News popups (PR #63, credit @csysp) - Admin auth (X-Admin-Key), rate limiting (slowapi), auto-updater - Docker Swarm secrets support, env_check.py validation - 85+ vitest tests, CI pipeline, geoJSON builder extraction - Server-side viewport bbox filtering reduces payloads 80%+ Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> Former-commit-id: f2883150b5bc78ebc139d89cc966a76f7d7c0408
This commit is contained in:
anoracleofra-code
@@ -0,0 +1,159 @@
|
||||
"""Tests for network_utils — fetch_with_curl, circuit breaker, domain fail cache."""
|
||||
import time
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
from services.network_utils import fetch_with_curl, _circuit_breaker, _domain_fail_cache, _cb_lock, _DummyResponse
|
||||
|
||||
|
||||
class TestDummyResponse:
|
||||
"""Tests for the minimal response object used as curl fallback."""
|
||||
|
||||
def test_status_code_and_text(self):
|
||||
resp = _DummyResponse(200, '{"ok": true}')
|
||||
assert resp.status_code == 200
|
||||
assert resp.text == '{"ok": true}'
|
||||
|
||||
def test_json_parsing(self):
|
||||
resp = _DummyResponse(200, '{"key": "value", "num": 42}')
|
||||
data = resp.json()
|
||||
assert data["key"] == "value"
|
||||
assert data["num"] == 42
|
||||
|
||||
def test_content_bytes(self):
|
||||
resp = _DummyResponse(200, "hello")
|
||||
assert resp.content == b"hello"
|
||||
|
||||
def test_raise_for_status_ok(self):
|
||||
resp = _DummyResponse(200, "ok")
|
||||
resp.raise_for_status() # Should not raise
|
||||
|
||||
def test_raise_for_status_error(self):
|
||||
resp = _DummyResponse(500, "server error")
|
||||
with pytest.raises(Exception, match="HTTP 500"):
|
||||
resp.raise_for_status()
|
||||
|
||||
def test_raise_for_status_404(self):
|
||||
resp = _DummyResponse(404, "not found")
|
||||
with pytest.raises(Exception, match="HTTP 404"):
|
||||
resp.raise_for_status()
|
||||
|
||||
|
||||
class TestCircuitBreaker:
|
||||
"""Tests for the circuit breaker and domain fail cache."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Clear caches before each test."""
|
||||
with _cb_lock:
|
||||
_circuit_breaker.clear()
|
||||
_domain_fail_cache.clear()
|
||||
|
||||
def test_circuit_breaker_blocks_request(self):
|
||||
"""If a domain is in circuit breaker, fetch_with_curl should fail fast."""
|
||||
with _cb_lock:
|
||||
_circuit_breaker["example.com"] = time.time()
|
||||
|
||||
with pytest.raises(Exception, match="Circuit breaker open"):
|
||||
fetch_with_curl("https://example.com/test")
|
||||
|
||||
def test_circuit_breaker_expires_after_ttl(self):
|
||||
"""Circuit breaker entries older than TTL should be ignored."""
|
||||
with _cb_lock:
|
||||
_circuit_breaker["expired.com"] = time.time() - 200 # > 120s TTL
|
||||
|
||||
# Should not raise — circuit breaker expired
|
||||
# Will fail for other reasons (network) but won't raise circuit breaker
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.text = "ok"
|
||||
mock_resp.raise_for_status = MagicMock()
|
||||
|
||||
with patch("services.network_utils._session") as mock_session:
|
||||
mock_session.get.return_value = mock_resp
|
||||
result = fetch_with_curl("https://expired.com/test")
|
||||
assert result.status_code == 200
|
||||
|
||||
def test_domain_fail_cache_skips_to_curl(self):
|
||||
"""If a domain recently failed with requests, skip straight to curl."""
|
||||
with _cb_lock:
|
||||
_domain_fail_cache["skip-to-curl.com"] = time.time()
|
||||
|
||||
# Mock subprocess to simulate curl success
|
||||
mock_result = MagicMock()
|
||||
mock_result.returncode = 0
|
||||
mock_result.stdout = '{"data": true}\n200'
|
||||
mock_result.stderr = ''
|
||||
|
||||
with patch("subprocess.run", return_value=mock_result) as mock_run:
|
||||
result = fetch_with_curl("https://skip-to-curl.com/api")
|
||||
assert result.status_code == 200
|
||||
assert result.json()["data"] is True
|
||||
# Verify subprocess.run was called (curl fallback)
|
||||
mock_run.assert_called_once()
|
||||
|
||||
def test_successful_request_clears_caches(self):
|
||||
"""Successful requests should clear both domain_fail_cache and circuit_breaker."""
|
||||
domain = "success-clears.com"
|
||||
with _cb_lock:
|
||||
_domain_fail_cache[domain] = time.time() - 400 # Expired, won't skip
|
||||
_circuit_breaker[domain] = time.time() - 200 # Expired, won't block
|
||||
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.text = "ok"
|
||||
mock_resp.raise_for_status = MagicMock()
|
||||
|
||||
with patch("services.network_utils._session") as mock_session:
|
||||
mock_session.get.return_value = mock_resp
|
||||
fetch_with_curl(f"https://{domain}/test")
|
||||
|
||||
with _cb_lock:
|
||||
assert domain not in _domain_fail_cache
|
||||
assert domain not in _circuit_breaker
|
||||
|
||||
|
||||
class TestFetchWithCurl:
|
||||
"""Tests for the primary fetch_with_curl function."""
|
||||
|
||||
def setup_method(self):
|
||||
with _cb_lock:
|
||||
_circuit_breaker.clear()
|
||||
_domain_fail_cache.clear()
|
||||
|
||||
def test_successful_get_returns_response(self):
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.text = '{"result": 42}'
|
||||
mock_resp.raise_for_status = MagicMock()
|
||||
|
||||
with patch("services.network_utils._session") as mock_session:
|
||||
mock_session.get.return_value = mock_resp
|
||||
result = fetch_with_curl("https://api.example.com/data")
|
||||
assert result.status_code == 200
|
||||
|
||||
def test_post_with_json_data(self):
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.text = '{"created": true}'
|
||||
mock_resp.raise_for_status = MagicMock()
|
||||
|
||||
with patch("services.network_utils._session") as mock_session:
|
||||
mock_session.post.return_value = mock_resp
|
||||
result = fetch_with_curl("https://api.example.com/create",
|
||||
method="POST", json_data={"name": "test"})
|
||||
assert result.status_code == 200
|
||||
mock_session.post.assert_called_once()
|
||||
|
||||
def test_custom_headers_merged(self):
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.text = "ok"
|
||||
mock_resp.raise_for_status = MagicMock()
|
||||
|
||||
with patch("services.network_utils._session") as mock_session:
|
||||
mock_session.get.return_value = mock_resp
|
||||
fetch_with_curl("https://api.example.com/data",
|
||||
headers={"Authorization": "Bearer token123"})
|
||||
call_args = mock_session.get.call_args
|
||||
headers = call_args.kwargs.get("headers", {})
|
||||
assert "Authorization" in headers
|
||||
assert headers["Authorization"] == "Bearer token123"
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Tests for Pydantic response schemas."""
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
from services.schemas import HealthResponse, RefreshResponse, AisFeedResponse, RouteResponse
|
||||
|
||||
|
||||
class TestHealthResponse:
|
||||
def test_valid_health_response(self):
|
||||
data = {
|
||||
"status": "ok",
|
||||
"last_updated": "2024-01-01T00:00:00",
|
||||
"sources": {"flights": 150, "ships": 42},
|
||||
"freshness": {"flights": "2024-01-01T00:00:00", "ships": "2024-01-01T00:00:00"},
|
||||
"uptime_seconds": 3600
|
||||
}
|
||||
resp = HealthResponse(**data)
|
||||
assert resp.status == "ok"
|
||||
assert resp.sources["flights"] == 150
|
||||
assert resp.uptime_seconds == 3600
|
||||
|
||||
def test_health_response_optional_last_updated(self):
|
||||
data = {
|
||||
"status": "ok",
|
||||
"sources": {},
|
||||
"freshness": {},
|
||||
"uptime_seconds": 0
|
||||
}
|
||||
resp = HealthResponse(**data)
|
||||
assert resp.last_updated is None
|
||||
|
||||
def test_health_response_missing_required_field(self):
|
||||
with pytest.raises(ValidationError):
|
||||
HealthResponse(status="ok") # Missing sources, freshness, uptime_seconds
|
||||
|
||||
|
||||
class TestRefreshResponse:
|
||||
def test_valid_refresh(self):
|
||||
resp = RefreshResponse(status="refreshing")
|
||||
assert resp.status == "refreshing"
|
||||
|
||||
def test_missing_status(self):
|
||||
with pytest.raises(ValidationError):
|
||||
RefreshResponse()
|
||||
|
||||
|
||||
class TestAisFeedResponse:
|
||||
def test_valid_ais_feed(self):
|
||||
resp = AisFeedResponse(status="ok", ingested=42)
|
||||
assert resp.ingested == 42
|
||||
|
||||
def test_default_ingested_zero(self):
|
||||
resp = AisFeedResponse(status="ok")
|
||||
assert resp.ingested == 0
|
||||
|
||||
|
||||
class TestRouteResponse:
|
||||
def test_valid_route(self):
|
||||
resp = RouteResponse(
|
||||
orig_loc=[40.6413, -73.7781],
|
||||
dest_loc=[51.4700, -0.4543],
|
||||
origin_name="JFK",
|
||||
dest_name="LHR"
|
||||
)
|
||||
assert resp.origin_name == "JFK"
|
||||
assert len(resp.orig_loc) == 2
|
||||
|
||||
def test_all_optional(self):
|
||||
resp = RouteResponse()
|
||||
assert resp.orig_loc is None
|
||||
assert resp.dest_loc is None
|
||||
assert resp.origin_name is None
|
||||
assert resp.dest_name is None
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Tests for the shared in-memory data store."""
|
||||
import threading
|
||||
import time
|
||||
import pytest
|
||||
from services.fetchers._store import latest_data, source_timestamps, _mark_fresh, _data_lock
|
||||
|
||||
|
||||
class TestLatestDataStructure:
|
||||
"""Verify the store has the expected keys and default values."""
|
||||
|
||||
def test_has_all_required_keys(self):
|
||||
expected_keys = {
|
||||
"last_updated", "news", "stocks", "oil", "flights", "ships",
|
||||
"military_flights", "tracked_flights", "cctv", "weather",
|
||||
"earthquakes", "uavs", "frontlines", "gdelt", "liveuamap",
|
||||
"kiwisdr", "space_weather", "internet_outages", "firms_fires",
|
||||
"datacenters"
|
||||
}
|
||||
assert expected_keys.issubset(set(latest_data.keys()))
|
||||
|
||||
def test_list_keys_default_to_empty_list(self):
|
||||
list_keys = ["news", "flights", "ships", "military_flights",
|
||||
"tracked_flights", "cctv", "earthquakes", "uavs",
|
||||
"gdelt", "liveuamap", "kiwisdr", "internet_outages",
|
||||
"firms_fires", "datacenters"]
|
||||
for key in list_keys:
|
||||
assert isinstance(latest_data[key], list), f"{key} should default to list"
|
||||
|
||||
def test_dict_keys_default_to_empty_dict(self):
|
||||
dict_keys = ["stocks", "oil"]
|
||||
for key in dict_keys:
|
||||
assert isinstance(latest_data[key], dict), f"{key} should default to dict"
|
||||
|
||||
|
||||
class TestMarkFresh:
|
||||
"""Tests for _mark_fresh timestamp helper."""
|
||||
|
||||
def test_records_timestamp_for_single_key(self):
|
||||
_mark_fresh("test_key_1")
|
||||
assert "test_key_1" in source_timestamps
|
||||
assert isinstance(source_timestamps["test_key_1"], str)
|
||||
|
||||
def test_records_timestamps_for_multiple_keys(self):
|
||||
_mark_fresh("multi_a", "multi_b", "multi_c")
|
||||
assert "multi_a" in source_timestamps
|
||||
assert "multi_b" in source_timestamps
|
||||
assert "multi_c" in source_timestamps
|
||||
|
||||
def test_timestamps_are_iso_format(self):
|
||||
_mark_fresh("iso_test")
|
||||
ts = source_timestamps["iso_test"]
|
||||
# ISO format: YYYY-MM-DDTHH:MM:SS.ffffff
|
||||
assert "T" in ts
|
||||
assert len(ts) >= 19 # At least YYYY-MM-DDTHH:MM:SS
|
||||
|
||||
def test_successive_calls_update_timestamp(self):
|
||||
_mark_fresh("update_test")
|
||||
ts1 = source_timestamps["update_test"]
|
||||
time.sleep(0.01)
|
||||
_mark_fresh("update_test")
|
||||
ts2 = source_timestamps["update_test"]
|
||||
assert ts2 >= ts1
|
||||
|
||||
|
||||
class TestDataLock:
|
||||
"""Verify the data lock works for thread safety."""
|
||||
|
||||
def test_lock_exists_and_is_a_lock(self):
|
||||
assert isinstance(_data_lock, type(threading.Lock()))
|
||||
|
||||
def test_concurrent_writes_dont_corrupt(self):
|
||||
"""Simulate concurrent writes to latest_data through the lock."""
|
||||
errors = []
|
||||
|
||||
def writer(key, value, iterations=100):
|
||||
try:
|
||||
for _ in range(iterations):
|
||||
with _data_lock:
|
||||
latest_data[key] = value
|
||||
# Read back immediately — should be our value
|
||||
assert latest_data[key] == value
|
||||
except Exception as e:
|
||||
errors.append(e)
|
||||
|
||||
threads = [
|
||||
threading.Thread(target=writer, args=("test_concurrent", [1, 2, 3])),
|
||||
threading.Thread(target=writer, args=("test_concurrent", [4, 5, 6])),
|
||||
threading.Thread(target=writer, args=("test_concurrent", [7, 8, 9])),
|
||||
]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
assert len(errors) == 0, f"Thread safety errors: {errors}"
|
||||
# Restore default
|
||||
latest_data["test_concurrent"] = []
|
||||
Reference in New Issue
Block a user