feat: evaluate saved Qwen3.8 checkpoints

tests/test_checkpoint_evaluation.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import hashlib
+import json
+
+import pytest
+
+from obliteratus.checkpoint_evaluation import (
+    _partition_pairs,
+    _verify_checkpoint_inventory,
+)
+
+
+def test_protocol_evaluation_partitions_are_immutable_and_disjoint():
+    tune = _partition_pairs("optimizer_tune")
+    final = _partition_pairs("final_test")
+
+    assert len(tune) == 142
+    assert len(final) == 200
+    assert set(tune).isdisjoint(final)
+    with pytest.raises(ValueError, match="unsupported"):
+        _partition_pairs("training")
+
+
+def test_checkpoint_inventory_verifies_size_hash_and_managed_path(tmp_path):
+    run_dir = tmp_path / ("run-" + "a" * 32)
+    checkpoint = run_dir / "checkpoint"
+    checkpoint.mkdir(parents=True)
+    weights = checkpoint / "weights.bin"
+    weights.write_bytes(b"verified weights")
+    inventory = run_dir / "artifact-inventory.json"
+    inventory.write_text(
+        json.dumps(
+            {
+                "artifacts": [
+                    {
+                        "path": "checkpoint/weights.bin",
+                        "bytes": weights.stat().st_size,
+                        "sha256": hashlib.sha256(weights.read_bytes()).hexdigest(),
+                    }
+                ]
+            }
+        ),
+        encoding="utf-8",
+    )
+    manifest = {
+        "result": {"checkpoint": str(checkpoint), "inventory": str(inventory)}
+    }
+
+    assert _verify_checkpoint_inventory(run_dir, manifest) == checkpoint.resolve()
+    weights.write_bytes(b"tampered weights")
+    with pytest.raises(ValueError, match="hash changed"):
+        _verify_checkpoint_inventory(run_dir, manifest)

tests/test_run_archive.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import hashlib
 import json
 import os
 from pathlib import Path
@@ -200,6 +201,58 @@ def test_experiment_protocol_is_durable_without_raw_prompts(tmp_path):
     assert json.loads(path.read_text(encoding="utf-8")) == protocol
 
 
+def _completed_protocol_run(archive: RunArchive) -> str:
+    run_id = archive.begin(["org/model"])
+    archive.record_experiment_protocol(
+        run_id,
+        {
+            "protocol": "qwen38-v1",
+            "manifest_sha256": "a" * 64,
+            "counts": {"train": 500, "tune": 142, "test": 200},
+            "pair_ids": {"train": [], "tune": [], "test": []},
+        },
+    )
+    checkpoint = archive._run_dir(run_id) / "checkpoint"
+    checkpoint.mkdir()
+    (checkpoint / "weights.bin").write_bytes(b"weights")
+    archive.complete(run_id, checkpoint=checkpoint, metrics={})
+    return run_id
+
+
+def test_saved_checkpoint_evaluation_is_durable_and_hashed(tmp_path):
+    archive = RunArchive(tmp_path)
+    run_id = _completed_protocol_run(archive)
+
+    reservation = archive.begin_evaluation(
+        run_id,
+        partition="optimizer_tune",
+        evaluator="test-v1",
+    )
+    record = archive.finish_evaluation(
+        run_id,
+        reservation["evaluation_id"],
+        metrics={"refusal_rate": 0.1, "coherence": 0.9},
+        log=["verified"],
+    )
+
+    assert record["status"] == "succeeded"
+    manifest = archive.result(run_id)
+    entry = manifest["evaluations"][0]
+    path = archive._run_dir(run_id) / entry["path"]
+    assert path.is_file()
+    assert entry["sha256"] == hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def test_final_evaluation_reservation_is_single_use_even_if_interrupted(tmp_path):
+    archive = RunArchive(tmp_path)
+    run_id = _completed_protocol_run(archive)
+
+    archive.begin_evaluation(run_id, partition="final_test", evaluator="test-v1")
+
+    with pytest.raises(ValueError, match="already reserved"):
+        archive.begin_evaluation(run_id, partition="final_test", evaluator="test-v1")
+
+
 def test_failure_detail_redacts_huggingface_and_bearer_tokens(tmp_path):
     archive = RunArchive(tmp_path)
     run_id = archive.begin(["org/model"])