test: add Gate 3 numerical oracle contracts

This commit is contained in:
Joseph Magly
2026-08-15 20:09:21 -04:00
parent 369417fbb4
commit 51529ad1d7
25 changed files with 3443 additions and 170 deletions
+200
View File
@@ -8,7 +8,10 @@ downloading real models or running any pipeline. They use
from __future__ import annotations
import json
from io import StringIO
from types import SimpleNamespace
from unittest.mock import MagicMock
from unittest.mock import patch
import pytest
@@ -155,6 +158,203 @@ class TestCLIDispatch:
assert args_passed.contribute is True
assert args_passed.contribute_notes == "Testing contribution system"
@pytest.mark.parametrize("flag", ["--prompt-pairs-file", "--prompt-pair-file"])
@pytest.mark.parametrize("command", ["obliterate", "abliterate"])
def test_prompt_pairs_file_flag_is_available_on_obliterate_and_alias(
self,
command,
flag,
tmp_path,
):
"""Explicit prompt-pair files are parsed for both commands and flag spellings."""
path = tmp_path / "pairs.json"
path.write_text("{}", encoding="utf-8")
with patch("obliteratus.cli._cmd_abliterate") as mock_cmd:
main([command, "fake/model", flag, str(path)])
args_passed = mock_cmd.call_args[0][0]
assert args_passed.prompt_pairs_file == str(path)
@pytest.mark.parametrize("flag", ["--prompt-pairs-file", "--prompt-pair-file"])
@pytest.mark.parametrize("command", ["obliterate", "abliterate"])
def test_prompt_pairs_file_is_mutually_exclusive_with_residue_files(
self,
command,
flag,
tmp_path,
):
"""Explicit prompt-pair files and mined residue construction cannot be mixed."""
path = tmp_path / "pairs.json"
path.write_text("{}", encoding="utf-8")
stderr_text = _capture_exit(
[
command,
"fake/model",
flag,
str(path),
"--residue-file",
"audit.json",
],
expect_code=2,
)
assert "not allowed with argument" in stderr_text.lower()
@pytest.mark.parametrize(
("option", "value"),
[
("--dataset", "custom"),
("--residue-weight", "7"),
("--residue-max", "3"),
],
)
@pytest.mark.parametrize("flag", ["--prompt-pairs-file", "--prompt-pair-file"])
@pytest.mark.parametrize("command", ["obliterate", "abliterate"])
def test_prompt_pairs_file_rejects_residue_only_options(
self,
command,
flag,
option,
value,
tmp_path,
):
"""Explicit prompt-pair files cannot be mixed with residue-only options."""
path = tmp_path / "pairs.json"
path.write_text("{}", encoding="utf-8")
stderr_text = _capture_exit(
[command, "fake/model", flag, str(path), option, value],
expect_code=2,
)
assert option in stderr_text
assert "can only be used with --residue-file" in stderr_text
def test_cmd_abliterate_wires_prompt_pairs_file_into_pipeline(tmp_path):
"""Loaded prompt-pair files are passed directly to AbliterationPipeline."""
path = tmp_path / "pairs.json"
path.write_text(
json.dumps(
{
"harmful": [f"harm {index}" for index in range(5)],
"harmless": [f"safe {index}" for index in range(5)],
}
),
encoding="utf-8",
)
result_path = tmp_path / "result"
result_path.mkdir()
pipeline = MagicMock()
pipeline.run.return_value = str(result_path)
class FakeLive:
def __init__(self, *_args, **_kwargs):
self.update = MagicMock()
def __enter__(self):
return self
def __exit__(self, *_args):
return False
args = SimpleNamespace(
model="org/model",
output_dir=str(tmp_path / "out"),
device="cpu",
dtype="float32",
method="basic",
n_directions=1,
direction_method=None,
regularization=None,
refinement_passes=1,
min_layer_fraction=None,
max_layer_fraction=None,
harmless_pc_count=None,
shield_concept_count=None,
shield_ridge=None,
shield_residualize=None,
shield_layer_penalty=None,
projection_target=None,
projection_row_fraction=None,
quantization=None,
gpu_memory_utilization=None,
large_model=False,
verify_sample_size=1,
refusal_max_tokens=1,
residue_file=[],
dataset="builtin",
residue_weight=5,
residue_max=None,
prompt_pairs_file=str(path),
contribute=False,
contribute_notes="",
)
with (
patch("rich.live.Live", FakeLive),
patch("obliteratus.abliterate.AbliterationPipeline", return_value=pipeline) as factory,
patch("obliteratus.telemetry.maybe_send_pipeline_report"),
):
from obliteratus import cli
cli._cmd_abliterate(args)
assert factory.call_args.kwargs["harmful_prompts"] == [f"harm {index}" for index in range(5)]
assert factory.call_args.kwargs["harmless_prompts"] == [f"safe {index}" for index in range(5)]
args.prompt_pairs_file = None
pipeline.reset_mock()
pipeline.run.return_value = str(result_path)
with (
patch("rich.live.Live", FakeLive),
patch("obliteratus.abliterate.AbliterationPipeline", return_value=pipeline) as factory,
patch("obliteratus.telemetry.maybe_send_pipeline_report"),
):
from obliteratus import cli
cli._cmd_abliterate(args)
assert "harmful_prompts" not in factory.call_args.kwargs
assert "harmless_prompts" not in factory.call_args.kwargs
def test_cmd_abliterate_reports_invalid_prompt_pairs_file(tmp_path):
from obliteratus import cli
args = SimpleNamespace(
model="org/model",
output_dir=str(tmp_path / "out"),
device="cpu",
dtype="float32",
method="basic",
n_directions=1,
direction_method=None,
regularization=None,
refinement_passes=1,
min_layer_fraction=None,
max_layer_fraction=None,
harmless_pc_count=None,
shield_concept_count=None,
shield_ridge=None,
shield_residualize=None,
shield_layer_penalty=None,
projection_target=None,
projection_row_fraction=None,
quantization=None,
gpu_memory_utilization=None,
large_model=False,
verify_sample_size=1,
refusal_max_tokens=1,
residue_file=[],
dataset="builtin",
residue_weight=5,
residue_max=None,
prompt_pairs_file=str(tmp_path / "missing.json"),
contribute=False,
contribute_notes="",
)
with pytest.raises(SystemExit) as exc:
cli._cmd_abliterate(args)
assert exc.value.code == 2
class _EncodingOnlyStdout:
"""Minimal stream stand-in for encoding-selection tests."""