feat: add held-out Qwen3.8 experiment protocol (#185)

tests/test_abliterate.py

@@ -163,7 +163,7 @@ class TestStages:
 
 class TestMethods:
     def test_methods_exist(self):
-        assert set(METHODS.keys()) == {"basic", "advanced", "aggressive", "informed", "surgical", "inverted", "nuclear", "optimized", "failspy", "gabliteration", "heretic", "rdo", "spectral_cascade", "som"}
+        assert set(METHODS.keys()) == {"basic", "advanced", "aggressive", "informed", "surgical", "inverted", "nuclear", "optimized", "failspy", "gabliteration", "heretic", "rdo", "spectral_cascade", "som", "qwen38_e01"}
 
     def test_basic_single_direction(self):
         cfg = METHODS["basic"]
@@ -207,6 +207,23 @@ class TestPipelineInit:
         assert pipeline.harmful_prompts == harmful
         assert pipeline.harmless_prompts == harmless
 
+    def test_held_out_evaluation_prompts_are_distinct_and_recorded(self):
+        pipeline = AbliterationPipeline(
+            model_name="test-model",
+            harmful_prompts=["train harmful"],
+            harmless_prompts=["train harmless"],
+            evaluation_harmful_prompts=["test harmful one", "test harmful two"],
+            evaluation_harmless_prompts=["test harmless one", "test harmless two"],
+        )
+
+        assert pipeline.evaluation_harmful_prompts == [
+            "test harmful one",
+            "test harmful two",
+        ]
+        metadata = pipeline._build_metadata()
+        assert metadata["n_harmful_prompts"] == 1
+        assert metadata["n_evaluation_harmful_prompts"] == 2
+
     def test_defaults(self):
         pipeline = AbliterationPipeline(model_name="test-model")
         assert pipeline.device == "auto"

tests/test_experiment_protocol.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from obliteratus.experiment_protocol import build_qwen38_split
+
+
+def _corpus(size: int = 842) -> tuple[list[str], list[str]]:
+    return (
+        [f"harmful-{index}" for index in range(size)],
+        [f"harmless-{index}" for index in range(size)],
+    )
+
+
+def test_qwen38_split_is_exact_disjoint_and_reproducible():
+    harmful, harmless = _corpus()
+    first = build_qwen38_split(harmful, harmless)
+    second = build_qwen38_split(list(harmful), list(harmless))
+
+    assert [len(first.train), len(first.tune), len(first.test)] == [500, 142, 200]
+    assert first.manifest == second.manifest
+    identities = first.manifest["pair_ids"]
+    train = set(identities["train"])
+    tune = set(identities["tune"])
+    test = set(identities["test"])
+    assert not train & tune
+    assert not train & test
+    assert not tune & test
+    assert len(train | tune | test) == 842
+
+
+def test_qwen38_split_manifest_contains_no_prompt_text():
+    harmful, harmless = _corpus()
+    split = build_qwen38_split(harmful, harmless)
+    serialized = json.dumps(split.manifest)
+
+    assert "harmful-0" not in serialized
+    assert "harmless-0" not in serialized
+    assert len(split.manifest["manifest_sha256"]) == 64
+
+
+def test_qwen38_split_rejects_wrong_size_and_duplicate_pairs():
+    harmful, harmless = _corpus(4)
+    with pytest.raises(ValueError, match="split sizes total"):
+        build_qwen38_split(harmful, harmless)
+
+    harmful, harmless = _corpus()
+    harmful[1] = harmful[0]
+    harmless[1] = harmless[0]
+    with pytest.raises(ValueError, match="duplicate prompt pair"):
+        build_qwen38_split(harmful, harmless)

tests/test_run_archive.py

@@ -183,6 +183,23 @@ def test_dataset_manifest_records_hash_and_counts_without_prompt_text(tmp_path):
     assert len(manifest["dataset_inputs"][0]["sha256"]) == 64
 
 
+def test_experiment_protocol_is_durable_without_raw_prompts(tmp_path):
+    archive = RunArchive(tmp_path / "runs")
+    run_id = archive.begin(["model"])
+    protocol = {
+        "protocol": "qwen38-v1",
+        "manifest_sha256": "a" * 64,
+        "counts": {"train": 500, "tune": 142, "test": 200},
+        "pair_ids": {"train": ["b" * 64], "tune": [], "test": []},
+    }
+
+    manifest = archive.record_experiment_protocol(run_id, protocol)
+
+    assert manifest["experiment_protocol"]["protocol"] == "qwen38-v1"
+    path = tmp_path / "runs" / run_id / "experiment-protocol.json"
+    assert json.loads(path.read_text(encoding="utf-8")) == protocol
+
+
 def test_failure_detail_redacts_huggingface_and_bearer_tokens(tmp_path):
     archive = RunArchive(tmp_path)
     run_id = archive.begin(["org/model"])