"""Failure, storage, and Hub boundary contracts for opt-in telemetry.""" from __future__ import annotations import sys from pathlib import Path from types import SimpleNamespace from unittest.mock import Mock import pytest import torch import obliteratus.telemetry as telemetry @pytest.fixture(autouse=True) def _restore_telemetry_globals(monkeypatch, tmp_path): monkeypatch.setattr(telemetry, "TELEMETRY_FILE", tmp_path / "telemetry.jsonl") monkeypatch.setattr(telemetry, "_TELEMETRY_DIR", tmp_path) monkeypatch.setattr(telemetry, "_TELEMETRY_REPO", "") monkeypatch.setattr(telemetry, "_hub_repo_created", False) monkeypatch.setattr(telemetry, "_hub_sync_last", 0.0) monkeypatch.setattr(telemetry, "_restore_done", False) telemetry._sync_in_progress.clear() def test_public_text_sanitizes_windows_paths_and_truncates(): assert telemetry._sanitize_public_text(r"C:\private\model.bin") == "model.bin" assert telemetry._sanitize_public_text("x" * 20, max_len=8) == "xxxxx..." assert telemetry._sanitize_public_value(object()) def test_telemetry_directory_prefers_explicit_and_home(monkeypatch, tmp_path): explicit = tmp_path / "explicit" monkeypatch.setenv("OBLITERATUS_DATA_DIR", str(explicit)) monkeypatch.setattr(telemetry, "_ON_HF_SPACES", False) assert telemetry._telemetry_dir() == explicit monkeypatch.setattr(telemetry, "_test_writable", lambda path: path.name == ".obliteratus") monkeypatch.setattr(Path, "home", staticmethod(lambda: tmp_path / "home")) assert telemetry._telemetry_dir() == tmp_path / "home" / ".obliteratus" def test_telemetry_directory_retries_hf_mount_then_uses_it(monkeypatch): monkeypatch.delenv("OBLITERATUS_DATA_DIR", raising=False) monkeypatch.setattr(telemetry, "_ON_HF_SPACES", True) monkeypatch.setattr(Path, "exists", lambda self: str(self) == "/data") attempts = iter([False, True]) monkeypatch.setattr(telemetry, "_test_writable", lambda _path: next(attempts)) sleep = Mock() monkeypatch.setattr(telemetry.time, "sleep", sleep) assert telemetry._telemetry_dir() == Path("/data/obliteratus") sleep.assert_called_once_with(1) def test_telemetry_directory_has_ephemeral_fallback(monkeypatch, tmp_path): monkeypatch.delenv("OBLITERATUS_DATA_DIR", raising=False) monkeypatch.setattr(telemetry, "_ON_HF_SPACES", False) monkeypatch.setattr(telemetry, "_test_writable", lambda _path: False) monkeypatch.setattr(Path, "home", staticmethod(lambda: tmp_path / "home")) assert telemetry._telemetry_dir() == Path("/tmp/obliteratus_telemetry") class _HubApi: instances: list["_HubApi"] = [] def __init__(self, **kwargs): self.kwargs = kwargs self.create_repo = Mock() self.repo_info = Mock() self.upload_file = Mock() self.list_repo_files = Mock(return_value=[]) self.instances.append(self) def _hub_module(api_class=_HubApi, **members): return SimpleNamespace(HfApi=api_class, **members) def test_ensure_hub_repo_create_and_existing_fallback(monkeypatch): _HubApi.instances.clear() monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module()) assert telemetry._ensure_hub_repo("org/data") is True _HubApi.instances[-1].create_repo.assert_called_once() assert telemetry._ensure_hub_repo("org/data") is True assert len(_HubApi.instances) == 1 telemetry._hub_repo_created = False class ExistingApi(_HubApi): def __init__(self, **kwargs): super().__init__(**kwargs) self.create_repo.side_effect = RuntimeError("cannot create") monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module(ExistingApi)) assert telemetry._ensure_hub_repo("org/data") is True ExistingApi.instances[-1].repo_info.assert_called_once() def test_ensure_hub_repo_fails_closed(monkeypatch): class FailingApi(_HubApi): def __init__(self, **kwargs): super().__init__(**kwargs) self.create_repo.side_effect = RuntimeError("create") self.repo_info.side_effect = RuntimeError("lookup") monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module(FailingApi)) assert telemetry._ensure_hub_repo("org/data") is False monkeypatch.setitem(sys.modules, "huggingface_hub", None) assert telemetry._ensure_hub_repo("org/data") is False def test_background_sync_short_circuits_and_uploads(monkeypatch, tmp_path): telemetry._sync_in_progress.set() telemetry._sync_to_hub_bg() telemetry._sync_in_progress.clear() telemetry._sync_to_hub_bg() assert not telemetry._sync_in_progress.is_set() telemetry._TELEMETRY_REPO = "org/data" telemetry._sync_to_hub_bg() assert not telemetry._sync_in_progress.is_set() telemetry.TELEMETRY_FILE.write_text("{}\n") _HubApi.instances.clear() monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module()) monkeypatch.setattr(telemetry, "_ensure_hub_repo", lambda _repo: True) monkeypatch.setattr(telemetry, "_instance_slug", lambda: "slug") telemetry._sync_to_hub_bg() _HubApi.instances[-1].upload_file.assert_called_once_with( path_or_fileobj=str(telemetry.TELEMETRY_FILE), path_in_repo="data/slug.jsonl", repo_id="org/data", repo_type="dataset", commit_message="Auto-sync telemetry from slug", ) assert not telemetry._sync_in_progress.is_set() def test_sync_scheduler_enforces_configuration_enablement_and_debounce(monkeypatch): thread = Mock() monkeypatch.setattr(telemetry.threading, "Thread", Mock(return_value=thread)) monkeypatch.setattr(telemetry, "is_enabled", lambda: True) telemetry._schedule_hub_sync() thread.start.assert_not_called() telemetry._TELEMETRY_REPO = "org/data" monkeypatch.setattr(telemetry, "is_enabled", lambda: False) telemetry._schedule_hub_sync() thread.start.assert_not_called() monkeypatch.setattr(telemetry, "is_enabled", lambda: True) monkeypatch.setattr(telemetry.time, "time", lambda: 100.0) telemetry._schedule_hub_sync() thread.start.assert_called_once() telemetry._schedule_hub_sync() thread.start.assert_called_once() def test_hf_api_fetch_handles_listing_errors_and_file_errors(monkeypatch, tmp_path): class ListingApi(_HubApi): def __init__(self, **kwargs): super().__init__(**kwargs) self.list_repo_files.return_value = ["README.md"] monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module(ListingApi, hf_hub_download=Mock())) assert telemetry._fetch_via_hf_api("org/data", 2) == [] class BrokenListingApi(_HubApi): def __init__(self, **kwargs): super().__init__(**kwargs) self.list_repo_files.side_effect = RuntimeError("offline") monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module(BrokenListingApi, hf_hub_download=Mock())) assert telemetry._fetch_via_hf_api("org/data", 2) == [] def test_git_clone_fetch_parses_bounded_records_and_cleans_up(monkeypatch): def fake_run(command, **_kwargs): clone_dir = Path(command[-1]) data = clone_dir / "data" data.mkdir() (data / "a.jsonl").write_text('\n{"id": 1}\ninvalid\n{"id": 2}\n') return SimpleNamespace(returncode=0, stderr="") monkeypatch.setattr("subprocess.run", fake_run) assert telemetry._fetch_via_git_clone("org/data", 1) == [{"id": 1}] def test_git_clone_fetch_handles_failure_and_missing_data(monkeypatch): monkeypatch.setattr( "subprocess.run", lambda *_args, **_kwargs: SimpleNamespace(returncode=1, stderr="denied"), ) assert telemetry._fetch_via_git_clone("org/data", 2) == [] monkeypatch.setattr( "subprocess.run", lambda *_args, **_kwargs: SimpleNamespace(returncode=0, stderr=""), ) assert telemetry._fetch_via_git_clone("org/data", 2) == [] def test_gpu_detection_and_peak_vram(monkeypatch): monkeypatch.setattr(torch.cuda, "is_available", lambda: True) monkeypatch.setattr(torch.cuda, "get_device_name", lambda _index: "GPU") monkeypatch.setattr( torch.cuda, "get_device_properties", lambda _index: SimpleNamespace(total_memory=8 * 1024**3), ) monkeypatch.setattr(torch.cuda, "max_memory_allocated", lambda: 3 * 1024**3) monkeypatch.setattr(torch.cuda, "max_memory_reserved", lambda: 4 * 1024**3) assert telemetry._detect_gpu() == ("GPU", 8.0) assert telemetry._get_peak_vram() == { "peak_allocated_gb": 3.0, "peak_reserved_gb": 4.0, } assert telemetry._detect_model_family("org/Qwen-model") == "qwen" assert telemetry._detect_model_family("org/other") == "unknown" def test_direction_stats_and_excise_details_cover_optional_techniques(monkeypatch): pipeline = SimpleNamespace( refusal_directions={0: torch.tensor([1.0, 0.0]), 1: torch.tensor([0.0, 1.0])}, refusal_subspaces={0: torch.eye(2)}, _excise_modified_count=2, _refusal_heads={0: [1, 2]}, _sae_directions={0: torch.ones(2)}, _expert_safety_scores={0: 1.0}, _layer_excise_weights={0: 0.2, 1: 0.8}, _expert_directions={0: torch.ones(2)}, _steering_hooks=[object()], invert_refusal=True, project_embeddings=True, activation_steering=True, expert_transplant=True, ) stats = telemetry._direction_stats(pipeline) assert stats["direction_norms"] == {"0": 1.0, "1": 1.0} assert stats["mean_direction_persistence"] == 0.0 assert stats["effective_ranks"] == {"0": 2.0} details = telemetry._extract_excise_details(pipeline) assert details["modified_count"] == 2 assert details["total_heads_projected"] == 2 assert details["adaptive_weight_min"] == 0.2 assert details["adaptive_weight_max"] == 0.8 assert set(details["used_techniques"]) == { "head_surgery", "sae_features", "expert_gating", "layer_adaptive", "per_expert", "activation_steering", "inversion", "embedding_projection", "expert_transplant", } monkeypatch.setattr(torch.linalg, "svdvals", Mock(side_effect=RuntimeError("svd"))) assert "effective_ranks" not in telemetry._direction_stats(pipeline) def test_send_and_pipeline_failures_are_best_effort(monkeypatch, tmp_path): monkeypatch.setattr(telemetry, "is_enabled", lambda: True) monkeypatch.setattr(telemetry, "TELEMETRY_FILE", tmp_path / "missing" / "file.jsonl") telemetry._send_sync({"schema_version": 2}) logger = Mock() monkeypatch.setattr(telemetry, "logger", logger) pipeline = SimpleNamespace(handle=SimpleNamespace(summary=Mock(side_effect=RuntimeError("summary")))) telemetry.maybe_send_pipeline_report(pipeline) telemetry.maybe_send_informed_report(pipeline, SimpleNamespace()) assert logger.debug.call_count == 2 def test_push_to_hub_failure_paths(monkeypatch, tmp_path): assert telemetry.push_to_hub() is False telemetry.TELEMETRY_FILE.write_text('{}\n') monkeypatch.setattr(telemetry, "read_telemetry", lambda: [{}]) monkeypatch.setattr(telemetry, "_ensure_hub_repo", lambda _repo: False) assert telemetry.push_to_hub("org/data") is False monkeypatch.setitem(sys.modules, "huggingface_hub", None) monkeypatch.setattr(telemetry, "_ensure_hub_repo", lambda _repo: True) assert telemetry.push_to_hub("org/data") is False def test_restore_and_background_restore_absorb_boundary_failures(monkeypatch): telemetry._TELEMETRY_REPO = "org/data" monkeypatch.setattr(telemetry, "fetch_hub_records", Mock(side_effect=RuntimeError("offline"))) assert telemetry.restore_from_hub() == 0 monkeypatch.setattr(telemetry, "restore_from_hub", Mock(side_effect=RuntimeError("offline"))) telemetry._restore_from_hub_bg()