mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-08-18 00:47:23 +02:00
Joseph Magly
37d008d462
Establishes the mandatory testing, coverage, repeatability, mutation, packaging, supply-chain, and AIWG workspace baseline before feature integration.
296 lines
12 KiB
Python
296 lines
12 KiB
Python
"""Failure, storage, and Hub boundary contracts for opt-in telemetry."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import Mock
|
|
|
|
import pytest
|
|
import torch
|
|
|
|
import obliteratus.telemetry as telemetry
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _restore_telemetry_globals(monkeypatch, tmp_path):
|
|
monkeypatch.setattr(telemetry, "TELEMETRY_FILE", tmp_path / "telemetry.jsonl")
|
|
monkeypatch.setattr(telemetry, "_TELEMETRY_DIR", tmp_path)
|
|
monkeypatch.setattr(telemetry, "_TELEMETRY_REPO", "")
|
|
monkeypatch.setattr(telemetry, "_hub_repo_created", False)
|
|
monkeypatch.setattr(telemetry, "_hub_sync_last", 0.0)
|
|
monkeypatch.setattr(telemetry, "_restore_done", False)
|
|
telemetry._sync_in_progress.clear()
|
|
|
|
|
|
def test_public_text_sanitizes_windows_paths_and_truncates():
|
|
assert telemetry._sanitize_public_text(r"C:\private\model.bin") == "model.bin"
|
|
assert telemetry._sanitize_public_text("x" * 20, max_len=8) == "xxxxx..."
|
|
assert telemetry._sanitize_public_value(object())
|
|
|
|
|
|
def test_telemetry_directory_prefers_explicit_and_home(monkeypatch, tmp_path):
|
|
explicit = tmp_path / "explicit"
|
|
monkeypatch.setenv("OBLITERATUS_DATA_DIR", str(explicit))
|
|
monkeypatch.setattr(telemetry, "_ON_HF_SPACES", False)
|
|
assert telemetry._telemetry_dir() == explicit
|
|
|
|
monkeypatch.setattr(telemetry, "_test_writable", lambda path: path.name == ".obliteratus")
|
|
monkeypatch.setattr(Path, "home", staticmethod(lambda: tmp_path / "home"))
|
|
assert telemetry._telemetry_dir() == tmp_path / "home" / ".obliteratus"
|
|
|
|
|
|
def test_telemetry_directory_retries_hf_mount_then_uses_it(monkeypatch):
|
|
monkeypatch.delenv("OBLITERATUS_DATA_DIR", raising=False)
|
|
monkeypatch.setattr(telemetry, "_ON_HF_SPACES", True)
|
|
monkeypatch.setattr(Path, "exists", lambda self: str(self) == "/data")
|
|
attempts = iter([False, True])
|
|
monkeypatch.setattr(telemetry, "_test_writable", lambda _path: next(attempts))
|
|
sleep = Mock()
|
|
monkeypatch.setattr(telemetry.time, "sleep", sleep)
|
|
|
|
assert telemetry._telemetry_dir() == Path("/data/obliteratus")
|
|
sleep.assert_called_once_with(1)
|
|
|
|
|
|
def test_telemetry_directory_has_ephemeral_fallback(monkeypatch, tmp_path):
|
|
monkeypatch.delenv("OBLITERATUS_DATA_DIR", raising=False)
|
|
monkeypatch.setattr(telemetry, "_ON_HF_SPACES", False)
|
|
monkeypatch.setattr(telemetry, "_test_writable", lambda _path: False)
|
|
monkeypatch.setattr(Path, "home", staticmethod(lambda: tmp_path / "home"))
|
|
assert telemetry._telemetry_dir() == Path("/tmp/obliteratus_telemetry")
|
|
|
|
|
|
class _HubApi:
|
|
instances: list["_HubApi"] = []
|
|
|
|
def __init__(self, **kwargs):
|
|
self.kwargs = kwargs
|
|
self.create_repo = Mock()
|
|
self.repo_info = Mock()
|
|
self.upload_file = Mock()
|
|
self.list_repo_files = Mock(return_value=[])
|
|
self.instances.append(self)
|
|
|
|
|
|
def _hub_module(api_class=_HubApi, **members):
|
|
return SimpleNamespace(HfApi=api_class, **members)
|
|
|
|
|
|
def test_ensure_hub_repo_create_and_existing_fallback(monkeypatch):
|
|
_HubApi.instances.clear()
|
|
monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module())
|
|
assert telemetry._ensure_hub_repo("org/data") is True
|
|
_HubApi.instances[-1].create_repo.assert_called_once()
|
|
assert telemetry._ensure_hub_repo("org/data") is True
|
|
assert len(_HubApi.instances) == 1
|
|
|
|
telemetry._hub_repo_created = False
|
|
|
|
class ExistingApi(_HubApi):
|
|
def __init__(self, **kwargs):
|
|
super().__init__(**kwargs)
|
|
self.create_repo.side_effect = RuntimeError("cannot create")
|
|
|
|
monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module(ExistingApi))
|
|
assert telemetry._ensure_hub_repo("org/data") is True
|
|
ExistingApi.instances[-1].repo_info.assert_called_once()
|
|
|
|
|
|
def test_ensure_hub_repo_fails_closed(monkeypatch):
|
|
class FailingApi(_HubApi):
|
|
def __init__(self, **kwargs):
|
|
super().__init__(**kwargs)
|
|
self.create_repo.side_effect = RuntimeError("create")
|
|
self.repo_info.side_effect = RuntimeError("lookup")
|
|
|
|
monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module(FailingApi))
|
|
assert telemetry._ensure_hub_repo("org/data") is False
|
|
|
|
monkeypatch.setitem(sys.modules, "huggingface_hub", None)
|
|
assert telemetry._ensure_hub_repo("org/data") is False
|
|
|
|
|
|
def test_background_sync_short_circuits_and_uploads(monkeypatch, tmp_path):
|
|
telemetry._sync_in_progress.set()
|
|
telemetry._sync_to_hub_bg()
|
|
telemetry._sync_in_progress.clear()
|
|
|
|
telemetry._sync_to_hub_bg()
|
|
assert not telemetry._sync_in_progress.is_set()
|
|
|
|
telemetry._TELEMETRY_REPO = "org/data"
|
|
telemetry._sync_to_hub_bg()
|
|
assert not telemetry._sync_in_progress.is_set()
|
|
|
|
telemetry.TELEMETRY_FILE.write_text("{}\n")
|
|
_HubApi.instances.clear()
|
|
monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module())
|
|
monkeypatch.setattr(telemetry, "_ensure_hub_repo", lambda _repo: True)
|
|
monkeypatch.setattr(telemetry, "_instance_slug", lambda: "slug")
|
|
telemetry._sync_to_hub_bg()
|
|
_HubApi.instances[-1].upload_file.assert_called_once_with(
|
|
path_or_fileobj=str(telemetry.TELEMETRY_FILE),
|
|
path_in_repo="data/slug.jsonl",
|
|
repo_id="org/data",
|
|
repo_type="dataset",
|
|
commit_message="Auto-sync telemetry from slug",
|
|
)
|
|
assert not telemetry._sync_in_progress.is_set()
|
|
|
|
|
|
def test_sync_scheduler_enforces_configuration_enablement_and_debounce(monkeypatch):
|
|
thread = Mock()
|
|
monkeypatch.setattr(telemetry.threading, "Thread", Mock(return_value=thread))
|
|
monkeypatch.setattr(telemetry, "is_enabled", lambda: True)
|
|
|
|
telemetry._schedule_hub_sync()
|
|
thread.start.assert_not_called()
|
|
|
|
telemetry._TELEMETRY_REPO = "org/data"
|
|
monkeypatch.setattr(telemetry, "is_enabled", lambda: False)
|
|
telemetry._schedule_hub_sync()
|
|
thread.start.assert_not_called()
|
|
|
|
monkeypatch.setattr(telemetry, "is_enabled", lambda: True)
|
|
monkeypatch.setattr(telemetry.time, "time", lambda: 100.0)
|
|
telemetry._schedule_hub_sync()
|
|
thread.start.assert_called_once()
|
|
telemetry._schedule_hub_sync()
|
|
thread.start.assert_called_once()
|
|
|
|
|
|
def test_hf_api_fetch_handles_listing_errors_and_file_errors(monkeypatch, tmp_path):
|
|
class ListingApi(_HubApi):
|
|
def __init__(self, **kwargs):
|
|
super().__init__(**kwargs)
|
|
self.list_repo_files.return_value = ["README.md"]
|
|
|
|
monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module(ListingApi, hf_hub_download=Mock()))
|
|
assert telemetry._fetch_via_hf_api("org/data", 2) == []
|
|
|
|
class BrokenListingApi(_HubApi):
|
|
def __init__(self, **kwargs):
|
|
super().__init__(**kwargs)
|
|
self.list_repo_files.side_effect = RuntimeError("offline")
|
|
|
|
monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module(BrokenListingApi, hf_hub_download=Mock()))
|
|
assert telemetry._fetch_via_hf_api("org/data", 2) == []
|
|
|
|
|
|
def test_git_clone_fetch_parses_bounded_records_and_cleans_up(monkeypatch):
|
|
def fake_run(command, **_kwargs):
|
|
clone_dir = Path(command[-1])
|
|
data = clone_dir / "data"
|
|
data.mkdir()
|
|
(data / "a.jsonl").write_text('\n{"id": 1}\ninvalid\n{"id": 2}\n')
|
|
return SimpleNamespace(returncode=0, stderr="")
|
|
|
|
monkeypatch.setattr("subprocess.run", fake_run)
|
|
assert telemetry._fetch_via_git_clone("org/data", 1) == [{"id": 1}]
|
|
|
|
|
|
def test_git_clone_fetch_handles_failure_and_missing_data(monkeypatch):
|
|
monkeypatch.setattr(
|
|
"subprocess.run",
|
|
lambda *_args, **_kwargs: SimpleNamespace(returncode=1, stderr="denied"),
|
|
)
|
|
assert telemetry._fetch_via_git_clone("org/data", 2) == []
|
|
|
|
monkeypatch.setattr(
|
|
"subprocess.run",
|
|
lambda *_args, **_kwargs: SimpleNamespace(returncode=0, stderr=""),
|
|
)
|
|
assert telemetry._fetch_via_git_clone("org/data", 2) == []
|
|
|
|
|
|
def test_gpu_detection_and_peak_vram(monkeypatch):
|
|
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
|
|
monkeypatch.setattr(torch.cuda, "get_device_name", lambda _index: "GPU")
|
|
monkeypatch.setattr(
|
|
torch.cuda,
|
|
"get_device_properties",
|
|
lambda _index: SimpleNamespace(total_memory=8 * 1024**3),
|
|
)
|
|
monkeypatch.setattr(torch.cuda, "max_memory_allocated", lambda: 3 * 1024**3)
|
|
monkeypatch.setattr(torch.cuda, "max_memory_reserved", lambda: 4 * 1024**3)
|
|
assert telemetry._detect_gpu() == ("GPU", 8.0)
|
|
assert telemetry._get_peak_vram() == {
|
|
"peak_allocated_gb": 3.0,
|
|
"peak_reserved_gb": 4.0,
|
|
}
|
|
assert telemetry._detect_model_family("org/Qwen-model") == "qwen"
|
|
assert telemetry._detect_model_family("org/other") == "unknown"
|
|
|
|
|
|
def test_direction_stats_and_excise_details_cover_optional_techniques(monkeypatch):
|
|
pipeline = SimpleNamespace(
|
|
refusal_directions={0: torch.tensor([1.0, 0.0]), 1: torch.tensor([0.0, 1.0])},
|
|
refusal_subspaces={0: torch.eye(2)},
|
|
_excise_modified_count=2,
|
|
_refusal_heads={0: [1, 2]},
|
|
_sae_directions={0: torch.ones(2)},
|
|
_expert_safety_scores={0: 1.0},
|
|
_layer_excise_weights={0: 0.2, 1: 0.8},
|
|
_expert_directions={0: torch.ones(2)},
|
|
_steering_hooks=[object()],
|
|
invert_refusal=True,
|
|
project_embeddings=True,
|
|
activation_steering=True,
|
|
expert_transplant=True,
|
|
)
|
|
stats = telemetry._direction_stats(pipeline)
|
|
assert stats["direction_norms"] == {"0": 1.0, "1": 1.0}
|
|
assert stats["mean_direction_persistence"] == 0.0
|
|
assert stats["effective_ranks"] == {"0": 2.0}
|
|
|
|
details = telemetry._extract_excise_details(pipeline)
|
|
assert details["modified_count"] == 2
|
|
assert details["total_heads_projected"] == 2
|
|
assert details["adaptive_weight_min"] == 0.2
|
|
assert details["adaptive_weight_max"] == 0.8
|
|
assert set(details["used_techniques"]) == {
|
|
"head_surgery", "sae_features", "expert_gating", "layer_adaptive",
|
|
"per_expert", "activation_steering", "inversion", "embedding_projection",
|
|
"expert_transplant",
|
|
}
|
|
|
|
monkeypatch.setattr(torch.linalg, "svdvals", Mock(side_effect=RuntimeError("svd")))
|
|
assert "effective_ranks" not in telemetry._direction_stats(pipeline)
|
|
|
|
|
|
def test_send_and_pipeline_failures_are_best_effort(monkeypatch, tmp_path):
|
|
monkeypatch.setattr(telemetry, "is_enabled", lambda: True)
|
|
monkeypatch.setattr(telemetry, "TELEMETRY_FILE", tmp_path / "missing" / "file.jsonl")
|
|
telemetry._send_sync({"schema_version": 2})
|
|
|
|
logger = Mock()
|
|
monkeypatch.setattr(telemetry, "logger", logger)
|
|
pipeline = SimpleNamespace(handle=SimpleNamespace(summary=Mock(side_effect=RuntimeError("summary"))))
|
|
telemetry.maybe_send_pipeline_report(pipeline)
|
|
telemetry.maybe_send_informed_report(pipeline, SimpleNamespace())
|
|
assert logger.debug.call_count == 2
|
|
|
|
|
|
def test_push_to_hub_failure_paths(monkeypatch, tmp_path):
|
|
assert telemetry.push_to_hub() is False
|
|
|
|
telemetry.TELEMETRY_FILE.write_text('{}\n')
|
|
monkeypatch.setattr(telemetry, "read_telemetry", lambda: [{}])
|
|
monkeypatch.setattr(telemetry, "_ensure_hub_repo", lambda _repo: False)
|
|
assert telemetry.push_to_hub("org/data") is False
|
|
|
|
monkeypatch.setitem(sys.modules, "huggingface_hub", None)
|
|
monkeypatch.setattr(telemetry, "_ensure_hub_repo", lambda _repo: True)
|
|
assert telemetry.push_to_hub("org/data") is False
|
|
|
|
|
|
def test_restore_and_background_restore_absorb_boundary_failures(monkeypatch):
|
|
telemetry._TELEMETRY_REPO = "org/data"
|
|
monkeypatch.setattr(telemetry, "fetch_hub_records", Mock(side_effect=RuntimeError("offline")))
|
|
assert telemetry.restore_from_hub() == 0
|
|
|
|
monkeypatch.setattr(telemetry, "restore_from_hub", Mock(side_effect=RuntimeError("offline")))
|
|
telemetry._restore_from_hub_bg()
|