test: enforce research integrity contracts

tests/test_telemetry.py

@@ -7,22 +7,33 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from unittest.mock import MagicMock, patch
 
+import pytest
 import torch
 
 from obliteratus.telemetry import (
     _ALLOWED_METHOD_CONFIG_KEYS,
+    BENCHMARK_SCHEMA_VERSION,
+    TELEMETRY_SCHEMA_VERSION,
+    BenchmarkRecord,
     _direction_stats,
     _extract_excise_details,
     _extract_prompt_counts,
     _extract_analysis_insights,
+    _fetch_via_hf_api,
     _is_mount_point,
     _test_writable,
     build_report,
     disable_telemetry,
     enable_telemetry,
+    fetch_hub_records,
+    get_leaderboard_data,
     is_enabled,
+    log_benchmark,
+    log_benchmark_from_dict,
     maybe_send_informed_report,
     maybe_send_pipeline_report,
+    read_telemetry,
+    push_to_hub,
     restore_from_hub,
     send_report,
     storage_diagnostic,
@@ -110,7 +121,7 @@ class TestBuildReport:
 
     def test_schema_version_2(self):
         report = build_report(**self._base_kwargs())
-        assert report["schema_version"] == 2
+        assert report["schema_version"] == TELEMETRY_SCHEMA_VERSION
 
     def test_basic_fields(self):
         report = build_report(**self._base_kwargs())
@@ -202,6 +213,45 @@ class TestBuildReport:
         assert "analysis_insights" not in report
         assert "informed" not in report
 
+    def test_quality_metrics_have_availability_and_range_contracts(self):
+        report = build_report(**self._base_kwargs(quality_metrics={
+            "refusal_rate": 0.0,
+            "perplexity": None,
+            "coherence": 2.0,
+            "unknown_metric": 4.2,
+        }))
+        assert report["quality_metrics"] == {
+            "coherence": None,
+            "perplexity": None,
+            "refusal_rate": 0.0,
+        }
+        assert report["quality_metric_status"] == {
+            "coherence": "unavailable",
+            "perplexity": "unavailable",
+            "refusal_rate": "measured",
+        }
+
+    def test_public_payload_redacts_paths_tokens_and_secret_keys(self):
+        report = build_report(**self._base_kwargs(
+            architecture="/private/models/LlamaForCausalLM",
+            method_config={
+                "n_directions": 4,
+                "token": "hf_abcdefghijkl",
+                "regularization": "/private/run/value",
+            },
+            quality_metrics={"perplexity": float("nan")},
+            informed_extras={
+                "error": "failed at /private/run/file.bin with sk-abcdefghijklmnop",
+                "api_key": "secret",
+            },
+        ))
+        encoded = json.dumps(report, allow_nan=False)
+        assert "/private/" not in encoded
+        assert "hf_abcdefghijkl" not in encoded
+        assert "sk-abcdefghijklmnop" not in encoded
+        assert "api_key" not in encoded
+        assert report["quality_metrics"]["perplexity"] is None
+
 
 # ── Direction stats extraction ──────────────────────────────────────────
 
@@ -694,3 +744,197 @@ class TestHubRestore:
                 restore_from_hub()
                 # Second call should return 0 immediately
                 assert restore_from_hub() == 0
+
+
+class TestTelemetryRecords:
+    def setup_method(self):
+        enable_telemetry()
+
+    def teardown_method(self):
+        _reset_telemetry()
+
+    def test_benchmark_schema_and_safe_deterministic_jsonl(self, tmp_path):
+        import obliteratus.telemetry as telemetry
+
+        output = tmp_path / "telemetry.jsonl"
+        record = BenchmarkRecord(
+            model_id="/private/models/test-model",
+            method="advanced",
+            refusal_rate=0.0,
+            perplexity=None,
+            error="failed at /private/run/file.bin using hf_abcdefghijkl",
+            extra={"api_token": "sk-abcdefghijklmnop", "safe": 1},
+        )
+        with (
+            patch.object(telemetry, "TELEMETRY_FILE", output),
+            patch("obliteratus.telemetry._schedule_hub_sync"),
+            patch("obliteratus.telemetry._detect_gpu", return_value=("", 0.0)),
+        ):
+            assert log_benchmark(record)
+
+        text = output.read_text()
+        data = json.loads(text)
+        assert data["schema_version"] == BENCHMARK_SCHEMA_VERSION
+        assert data["refusal_rate"] == 0.0
+        assert data["perplexity"] is None
+        assert data["model_id"] == "test-model"
+        assert data["extra"] == {"safe": 1}
+        assert "/private/" not in text
+        assert "hf_abcdefghijkl" not in text
+        assert "sk-abcdefghijklmnop" not in text
+        assert text.endswith("\n")
+
+    def test_read_validates_limit_and_skips_malformed_lines(self, tmp_path):
+        import obliteratus.telemetry as telemetry
+
+        output = tmp_path / "telemetry.jsonl"
+        output.write_text('{"timestamp":"1"}\nnot json\n{"timestamp":"2"}\n')
+        with patch.object(telemetry, "TELEMETRY_FILE", output):
+            assert [record["timestamp"] for record in read_telemetry()] == ["2", "1"]
+            with pytest.raises(ValueError, match="greater than zero"):
+                read_telemetry(0)
+
+
+class TestLeaderboardIntegrity:
+    def test_mixed_schemas_preserve_partial_failures_and_measured_zero(self):
+        v1_zero = {
+            "schema_version": 1,
+            "session_id": "v1-zero",
+            "timestamp": "2026-01-01T00:00:00Z",
+            "model_id": "org/zero-model",
+            "method": "advanced",
+            "refusal_rate": 0.0,
+            "perplexity": 4.0,
+            "coherence": 0.8,
+            "time_seconds": 5.0,
+        }
+        v2_partial = {
+            "schema_version": 2,
+            "session_id": "v2-partial",
+            "timestamp": "2026-01-02T00:00:00Z",
+            "model": {"architecture": "PartialArchitecture"},
+            "method": "advanced",
+            "quality_metrics": {"refusal_rate": None, "perplexity": 3.0},
+            "error": "coherence failed",
+        }
+        v1_missing = {
+            "schema_version": 1,
+            "session_id": "v1-missing",
+            "timestamp": "2026-01-03T00:00:00Z",
+            "model_id": "org/missing-model",
+            "method": "advanced",
+            "refusal_rate": None,
+            "perplexity": 2.0,
+        }
+        with (
+            patch("obliteratus.telemetry.read_telemetry", return_value=[v1_missing, v1_zero]),
+            patch("obliteratus.telemetry.fetch_hub_records", return_value=[v2_partial]),
+        ):
+            leaderboard = get_leaderboard_data()
+
+        assert leaderboard[0]["model_id"] == "org/zero-model"
+        assert leaderboard[0]["best_refusal"] == 0.0
+        partial = next(row for row in leaderboard if row["model_id"] == "PartialArchitecture")
+        assert partial["runs"] == 1
+        assert partial["successful_runs"] == 0
+        assert partial["failed_runs"] == 1
+        assert partial["perplexity_measurements"] == 1
+        assert partial["best_perplexity"] == 3.0
+        assert partial["best_refusal"] is None
+
+    def test_invalid_ranges_do_not_enter_aggregates(self):
+        record = {
+            "session_id": "bad", "timestamp": "1", "model_id": "bad/model",
+            "method": "advanced", "refusal_rate": -0.1,
+            "perplexity": float("nan"), "coherence": True,
+        }
+        with (
+            patch("obliteratus.telemetry.read_telemetry", return_value=[record]),
+            patch("obliteratus.telemetry.fetch_hub_records", return_value=[]),
+        ):
+            row = get_leaderboard_data()[0]
+        assert row["refusal_measurements"] == 0
+        assert row["perplexity_measurements"] == 0
+        assert row["coherence_measurements"] == 0
+        assert row["best_refusal"] is None
+
+
+class TestTelemetryHubBoundaries:
+    def test_fetch_prefers_api_and_falls_back_to_git(self):
+        api_records = [{"session_id": "api"}]
+        with (
+            patch("obliteratus.telemetry._fetch_via_hf_api", return_value=api_records),
+            patch("obliteratus.telemetry._fetch_via_git_clone") as git_fetch,
+        ):
+            assert fetch_hub_records(3) == api_records
+            git_fetch.assert_not_called()
+
+        git_records = [{"session_id": "git"}]
+        with (
+            patch("obliteratus.telemetry._fetch_via_hf_api", return_value=[]),
+            patch("obliteratus.telemetry._fetch_via_git_clone", return_value=git_records),
+        ):
+            assert fetch_hub_records(3) == git_records
+
+    def test_fetch_returns_empty_when_both_boundaries_fail(self):
+        with (
+            patch("obliteratus.telemetry._fetch_via_hf_api", side_effect=RuntimeError("api")),
+            patch("obliteratus.telemetry._fetch_via_git_clone", side_effect=RuntimeError("git")),
+        ):
+            assert fetch_hub_records() == []
+
+    def test_hf_api_parser_filters_files_malformed_lines_and_limit(self, tmp_path):
+        first = tmp_path / "first.jsonl"
+        second = tmp_path / "second.jsonl"
+        first.write_text('\n{"session_id":"one"}\nnot-json\n{"session_id":"two"}\n')
+        second.write_text('{"session_id":"three"}\n')
+        api = MagicMock()
+        api.list_repo_files.return_value = [
+            "README.md", "other/ignored.jsonl", "data/first.jsonl", "data/second.jsonl",
+        ]
+        with (
+            patch("huggingface_hub.HfApi", return_value=api),
+            patch("huggingface_hub.hf_hub_download", side_effect=[str(first), str(second)]) as download,
+        ):
+            records = _fetch_via_hf_api("org/repo", 2)
+        assert [record["session_id"] for record in records] == ["one", "two"]
+        assert download.call_count == 1
+
+    def test_log_from_dict_maps_partial_result_without_erasing_error(self):
+        with patch("obliteratus.telemetry.log_benchmark", return_value=True) as write:
+            assert log_benchmark_from_dict(
+                "org/model",
+                "advanced",
+                {"refusal_rate": 0.0, "perplexity": None, "error": "partial"},
+                dataset="fixture",
+                n_prompts=4,
+                pipeline_config={"n_directions": 3, "bayesian_trials": 2},
+            )
+        record = write.call_args.args[0]
+        assert record.refusal_rate == 0.0
+        assert record.perplexity is None
+        assert record.error == "partial"
+        assert record.n_directions == 3
+        assert record.use_bayesian is True
+
+    def test_push_to_hub_success_and_empty_short_circuits(self, tmp_path):
+        import obliteratus.telemetry as telemetry
+
+        output = tmp_path / "telemetry.jsonl"
+        output.write_text('{"session_id":"one"}\n')
+        api = MagicMock()
+        with (
+            patch.object(telemetry, "TELEMETRY_FILE", output),
+            patch("obliteratus.telemetry.read_telemetry", return_value=[{"session_id": "one"}]),
+            patch("obliteratus.telemetry._ensure_hub_repo", return_value=True),
+            patch("huggingface_hub.HfApi", return_value=api),
+            patch("obliteratus.telemetry._instance_slug", return_value="instance"),
+        ):
+            assert push_to_hub("org/repo") is True
+        api.upload_file.assert_called_once()
+        assert api.upload_file.call_args.kwargs["path_in_repo"] == "data/instance.jsonl"
+
+        with patch("obliteratus.telemetry.read_telemetry", return_value=[]):
+            assert push_to_hub("org/repo") is False
+    get_leaderboard_data,
+    read_telemetry,