mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-08-17 16:37:30 +02:00
291 lines
8.9 KiB
Python
291 lines
8.9 KiB
Python
"""Deterministic contracts for BESTIARY catalog resolution and filtering."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from obliteratus import models_client
|
|
|
|
|
|
def _catalog(*records: dict) -> dict:
|
|
return {"generated_at": "2026-08-16T00:00:00Z", "models": list(records)}
|
|
|
|
|
|
def _write_catalog(path: Path, payload: object) -> Path:
|
|
path.write_text(json.dumps(payload), encoding="utf-8")
|
|
return path
|
|
|
|
|
|
@pytest.fixture
|
|
def isolated_sources(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
|
monkeypatch.delenv("BESTIARY_CATALOG", raising=False)
|
|
monkeypatch.setattr(models_client, "_LOCAL", tmp_path / "local.json")
|
|
monkeypatch.setattr(models_client, "_CANONICAL", tmp_path / "canonical.json")
|
|
monkeypatch.setattr(models_client, "_HERE", tmp_path / "package")
|
|
|
|
|
|
def test_explicit_path_is_authoritative_and_does_not_fall_back(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
isolated_sources: None,
|
|
) -> None:
|
|
_write_catalog(models_client._LOCAL, _catalog({"id": "fallback"}))
|
|
missing = tmp_path / "operator-selected.json"
|
|
monkeypatch.setenv("BESTIARY_CATALOG", str(missing))
|
|
|
|
with pytest.raises(FileNotFoundError, match="operator-selected"):
|
|
models_client._load()
|
|
|
|
|
|
def test_default_resolution_uses_first_existing_packaged_source(
|
|
isolated_sources: None,
|
|
) -> None:
|
|
models_client._CANONICAL.parent.mkdir(parents=True, exist_ok=True)
|
|
_write_catalog(models_client._CANONICAL, _catalog({"id": "canonical"}))
|
|
state = models_client._HERE / "state" / "catalog.json"
|
|
state.parent.mkdir(parents=True)
|
|
_write_catalog(state, _catalog({"id": "state"}))
|
|
|
|
assert models_client.model_ids() == ["canonical"]
|
|
|
|
|
|
def test_http_resolution_is_timeout_and_size_bounded(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
isolated_sources: None,
|
|
) -> None:
|
|
payload = json.dumps(_catalog({"id": "remote"})).encode()
|
|
calls: dict[str, object] = {}
|
|
|
|
class Response:
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_args):
|
|
return False
|
|
|
|
def read(self, amount: int) -> bytes:
|
|
calls["read"] = amount
|
|
return payload
|
|
|
|
def urlopen(url: str, *, timeout: int):
|
|
calls["url"] = url
|
|
calls["timeout"] = timeout
|
|
return Response()
|
|
|
|
monkeypatch.setenv("BESTIARY_CATALOG", "https://catalog.invalid/catalog.json")
|
|
monkeypatch.setattr(models_client.urllib.request, "urlopen", urlopen)
|
|
|
|
assert models_client.model_ids() == ["remote"]
|
|
assert calls == {
|
|
"url": "https://catalog.invalid/catalog.json",
|
|
"timeout": models_client._CATALOG_TIMEOUT_SECONDS,
|
|
"read": models_client._MAX_CATALOG_BYTES + 1,
|
|
}
|
|
|
|
|
|
def test_http_resolution_rejects_oversized_catalog(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
isolated_sources: None,
|
|
) -> None:
|
|
class Response:
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_args):
|
|
return False
|
|
|
|
def read(self, amount: int) -> bytes:
|
|
return b"x" * amount
|
|
|
|
monkeypatch.setenv("BESTIARY_CATALOG", "https://catalog.invalid/catalog.json")
|
|
monkeypatch.setattr(
|
|
models_client.urllib.request,
|
|
"urlopen",
|
|
lambda *_args, **_kwargs: Response(),
|
|
)
|
|
|
|
with pytest.raises(ValueError, match="size limit"):
|
|
models_client._load()
|
|
|
|
|
|
@pytest.mark.parametrize("payload", [b"\xff", b"{not-json}"])
|
|
def test_path_resolution_rejects_invalid_utf8_or_json(
|
|
payload: bytes,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
isolated_sources: None,
|
|
) -> None:
|
|
path = tmp_path / "catalog.json"
|
|
path.write_bytes(payload)
|
|
monkeypatch.setenv("BESTIARY_CATALOG", str(path))
|
|
with pytest.raises(ValueError, match="not valid UTF-8 JSON"):
|
|
models_client._load()
|
|
|
|
|
|
def test_path_resolution_is_size_bounded(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
isolated_sources: None,
|
|
) -> None:
|
|
path = tmp_path / "catalog.json"
|
|
path.write_bytes(b"123")
|
|
monkeypatch.setattr(models_client, "_MAX_CATALOG_BYTES", 2)
|
|
monkeypatch.setenv("BESTIARY_CATALOG", str(path))
|
|
with pytest.raises(ValueError, match="size limit"):
|
|
models_client._load()
|
|
|
|
|
|
def test_filters_reject_non_string_arguments(isolated_sources: None) -> None:
|
|
with pytest.raises(TypeError, match="channel must be a string"):
|
|
models_client.models(channel=7) # type: ignore[arg-type]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"payload, message",
|
|
[
|
|
([], "root must be an object"),
|
|
({}, "models must be a list"),
|
|
({"models": ["not-a-record"]}, "record 0 must be an object"),
|
|
],
|
|
)
|
|
def test_catalog_schema_fails_closed(
|
|
payload: object,
|
|
message: str,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
isolated_sources: None,
|
|
) -> None:
|
|
path = _write_catalog(tmp_path / "catalog.json", payload)
|
|
monkeypatch.setenv("BESTIARY_CATALOG", str(path))
|
|
|
|
with pytest.raises(ValueError, match=message) as error:
|
|
models_client._load()
|
|
assert str(error.value) == f"BESTIARY catalog {message}"
|
|
|
|
|
|
def test_filters_normalize_aliases_and_catalog_values(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
isolated_sources: None,
|
|
) -> None:
|
|
path = _write_catalog(
|
|
tmp_path / "catalog.json",
|
|
_catalog(
|
|
{
|
|
"id": "Example/Alpha",
|
|
"vendor": "Example",
|
|
"channels": ["OpenRouter"],
|
|
"open_weight": False,
|
|
"capabilities": ["Tools"],
|
|
},
|
|
{
|
|
"id": "Example/Beta",
|
|
"vendor": "example",
|
|
"channels": ["huggingface"],
|
|
"open_weight": True,
|
|
"capabilities": ["vision"],
|
|
},
|
|
),
|
|
)
|
|
monkeypatch.setenv("BESTIARY_CATALOG", str(path))
|
|
|
|
assert models_client.model_ids(channel="API", vendor="EXAMPLE", capability="TOOLS") == [
|
|
"Example/Alpha"
|
|
]
|
|
assert models_client.model_ids(channel="open-weight") == ["Example/Beta"]
|
|
assert models_client.model_ids(channel="HF", open_weight=True) == ["Example/Beta"]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"record, message",
|
|
[
|
|
({"id": "bad", "channels": "api"}, "channels must be a list"),
|
|
({"id": "bad", "capabilities": [7]}, "capabilities must be a list"),
|
|
({"id": "bad", "vendor": 7}, "vendor must be a string"),
|
|
],
|
|
)
|
|
def test_filters_reject_malformed_catalog_fields(
|
|
record: dict,
|
|
message: str,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
isolated_sources: None,
|
|
) -> None:
|
|
path = _write_catalog(tmp_path / "catalog.json", _catalog(record))
|
|
monkeypatch.setenv("BESTIARY_CATALOG", str(path))
|
|
|
|
with pytest.raises(ValueError, match=message):
|
|
models_client.models()
|
|
|
|
|
|
@pytest.mark.parametrize("bad_id", [None, "", 7])
|
|
def test_model_ids_rejects_missing_or_non_string_ids(
|
|
bad_id: object,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
isolated_sources: None,
|
|
) -> None:
|
|
path = _write_catalog(tmp_path / "catalog.json", _catalog({"id": bad_id}))
|
|
monkeypatch.setenv("BESTIARY_CATALOG", str(path))
|
|
|
|
with pytest.raises(ValueError, match="non-empty string id"):
|
|
models_client.model_ids()
|
|
|
|
|
|
def test_newest_uses_injected_clock_release_fallback_and_stable_order(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
isolated_sources: None,
|
|
) -> None:
|
|
path = _write_catalog(
|
|
tmp_path / "catalog.json",
|
|
_catalog(
|
|
{"id": "released", "released": "2026-08-15"},
|
|
{"id": "newest", "first_seen": "2026-08-16"},
|
|
{"id": "cutoff", "first_seen": "2026-08-09"},
|
|
{"id": "old", "first_seen": "2026-08-08"},
|
|
{"id": "invalid", "first_seen": "yesterday"},
|
|
{"id": "undated"},
|
|
),
|
|
)
|
|
monkeypatch.setenv("BESTIARY_CATALOG", str(path))
|
|
monkeypatch.setattr(
|
|
models_client,
|
|
"_utc_now",
|
|
lambda: datetime(2026, 8, 16, tzinfo=timezone.utc),
|
|
)
|
|
|
|
assert models_client.model_ids() == [
|
|
"released", "newest", "cutoff", "old", "invalid", "undated"
|
|
]
|
|
assert [record["id"] for record in models_client.newest(7)] == [
|
|
"newest", "released", "cutoff"
|
|
]
|
|
|
|
|
|
def test_newest_rejects_non_string_catalog_dates(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
isolated_sources: None,
|
|
) -> None:
|
|
path = _write_catalog(
|
|
tmp_path / "catalog.json",
|
|
_catalog({"id": "bad-date", "first_seen": 20260816}),
|
|
)
|
|
monkeypatch.setenv("BESTIARY_CATALOG", str(path))
|
|
with pytest.raises(ValueError, match="dates must be strings"):
|
|
models_client.newest()
|
|
|
|
|
|
@pytest.mark.parametrize("days", [-1, True, 1.5, "7"])
|
|
def test_newest_rejects_invalid_day_windows(
|
|
days: object,
|
|
isolated_sources: None,
|
|
) -> None:
|
|
with pytest.raises((TypeError, ValueError), match="days"):
|
|
models_client.newest(days) # type: ignore[arg-type]
|