mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-08-18 00:47:23 +02:00
test: add Gate 3 numerical oracle contracts
This commit is contained in:
@@ -8,7 +8,10 @@ downloading real models or running any pipeline. They use
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from io import StringIO
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
@@ -155,6 +158,203 @@ class TestCLIDispatch:
|
||||
assert args_passed.contribute is True
|
||||
assert args_passed.contribute_notes == "Testing contribution system"
|
||||
|
||||
@pytest.mark.parametrize("flag", ["--prompt-pairs-file", "--prompt-pair-file"])
|
||||
@pytest.mark.parametrize("command", ["obliterate", "abliterate"])
|
||||
def test_prompt_pairs_file_flag_is_available_on_obliterate_and_alias(
|
||||
self,
|
||||
command,
|
||||
flag,
|
||||
tmp_path,
|
||||
):
|
||||
"""Explicit prompt-pair files are parsed for both commands and flag spellings."""
|
||||
path = tmp_path / "pairs.json"
|
||||
path.write_text("{}", encoding="utf-8")
|
||||
with patch("obliteratus.cli._cmd_abliterate") as mock_cmd:
|
||||
main([command, "fake/model", flag, str(path)])
|
||||
args_passed = mock_cmd.call_args[0][0]
|
||||
assert args_passed.prompt_pairs_file == str(path)
|
||||
|
||||
@pytest.mark.parametrize("flag", ["--prompt-pairs-file", "--prompt-pair-file"])
|
||||
@pytest.mark.parametrize("command", ["obliterate", "abliterate"])
|
||||
def test_prompt_pairs_file_is_mutually_exclusive_with_residue_files(
|
||||
self,
|
||||
command,
|
||||
flag,
|
||||
tmp_path,
|
||||
):
|
||||
"""Explicit prompt-pair files and mined residue construction cannot be mixed."""
|
||||
path = tmp_path / "pairs.json"
|
||||
path.write_text("{}", encoding="utf-8")
|
||||
stderr_text = _capture_exit(
|
||||
[
|
||||
command,
|
||||
"fake/model",
|
||||
flag,
|
||||
str(path),
|
||||
"--residue-file",
|
||||
"audit.json",
|
||||
],
|
||||
expect_code=2,
|
||||
)
|
||||
assert "not allowed with argument" in stderr_text.lower()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("option", "value"),
|
||||
[
|
||||
("--dataset", "custom"),
|
||||
("--residue-weight", "7"),
|
||||
("--residue-max", "3"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("flag", ["--prompt-pairs-file", "--prompt-pair-file"])
|
||||
@pytest.mark.parametrize("command", ["obliterate", "abliterate"])
|
||||
def test_prompt_pairs_file_rejects_residue_only_options(
|
||||
self,
|
||||
command,
|
||||
flag,
|
||||
option,
|
||||
value,
|
||||
tmp_path,
|
||||
):
|
||||
"""Explicit prompt-pair files cannot be mixed with residue-only options."""
|
||||
path = tmp_path / "pairs.json"
|
||||
path.write_text("{}", encoding="utf-8")
|
||||
stderr_text = _capture_exit(
|
||||
[command, "fake/model", flag, str(path), option, value],
|
||||
expect_code=2,
|
||||
)
|
||||
assert option in stderr_text
|
||||
assert "can only be used with --residue-file" in stderr_text
|
||||
|
||||
|
||||
def test_cmd_abliterate_wires_prompt_pairs_file_into_pipeline(tmp_path):
|
||||
"""Loaded prompt-pair files are passed directly to AbliterationPipeline."""
|
||||
path = tmp_path / "pairs.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"harmful": [f"harm {index}" for index in range(5)],
|
||||
"harmless": [f"safe {index}" for index in range(5)],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
result_path = tmp_path / "result"
|
||||
result_path.mkdir()
|
||||
pipeline = MagicMock()
|
||||
pipeline.run.return_value = str(result_path)
|
||||
|
||||
class FakeLive:
|
||||
def __init__(self, *_args, **_kwargs):
|
||||
self.update = MagicMock()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
args = SimpleNamespace(
|
||||
model="org/model",
|
||||
output_dir=str(tmp_path / "out"),
|
||||
device="cpu",
|
||||
dtype="float32",
|
||||
method="basic",
|
||||
n_directions=1,
|
||||
direction_method=None,
|
||||
regularization=None,
|
||||
refinement_passes=1,
|
||||
min_layer_fraction=None,
|
||||
max_layer_fraction=None,
|
||||
harmless_pc_count=None,
|
||||
shield_concept_count=None,
|
||||
shield_ridge=None,
|
||||
shield_residualize=None,
|
||||
shield_layer_penalty=None,
|
||||
projection_target=None,
|
||||
projection_row_fraction=None,
|
||||
quantization=None,
|
||||
gpu_memory_utilization=None,
|
||||
large_model=False,
|
||||
verify_sample_size=1,
|
||||
refusal_max_tokens=1,
|
||||
residue_file=[],
|
||||
dataset="builtin",
|
||||
residue_weight=5,
|
||||
residue_max=None,
|
||||
prompt_pairs_file=str(path),
|
||||
contribute=False,
|
||||
contribute_notes="",
|
||||
)
|
||||
with (
|
||||
patch("rich.live.Live", FakeLive),
|
||||
patch("obliteratus.abliterate.AbliterationPipeline", return_value=pipeline) as factory,
|
||||
patch("obliteratus.telemetry.maybe_send_pipeline_report"),
|
||||
):
|
||||
from obliteratus import cli
|
||||
|
||||
cli._cmd_abliterate(args)
|
||||
|
||||
assert factory.call_args.kwargs["harmful_prompts"] == [f"harm {index}" for index in range(5)]
|
||||
assert factory.call_args.kwargs["harmless_prompts"] == [f"safe {index}" for index in range(5)]
|
||||
|
||||
args.prompt_pairs_file = None
|
||||
pipeline.reset_mock()
|
||||
pipeline.run.return_value = str(result_path)
|
||||
with (
|
||||
patch("rich.live.Live", FakeLive),
|
||||
patch("obliteratus.abliterate.AbliterationPipeline", return_value=pipeline) as factory,
|
||||
patch("obliteratus.telemetry.maybe_send_pipeline_report"),
|
||||
):
|
||||
from obliteratus import cli
|
||||
|
||||
cli._cmd_abliterate(args)
|
||||
|
||||
assert "harmful_prompts" not in factory.call_args.kwargs
|
||||
assert "harmless_prompts" not in factory.call_args.kwargs
|
||||
|
||||
|
||||
def test_cmd_abliterate_reports_invalid_prompt_pairs_file(tmp_path):
|
||||
from obliteratus import cli
|
||||
|
||||
args = SimpleNamespace(
|
||||
model="org/model",
|
||||
output_dir=str(tmp_path / "out"),
|
||||
device="cpu",
|
||||
dtype="float32",
|
||||
method="basic",
|
||||
n_directions=1,
|
||||
direction_method=None,
|
||||
regularization=None,
|
||||
refinement_passes=1,
|
||||
min_layer_fraction=None,
|
||||
max_layer_fraction=None,
|
||||
harmless_pc_count=None,
|
||||
shield_concept_count=None,
|
||||
shield_ridge=None,
|
||||
shield_residualize=None,
|
||||
shield_layer_penalty=None,
|
||||
projection_target=None,
|
||||
projection_row_fraction=None,
|
||||
quantization=None,
|
||||
gpu_memory_utilization=None,
|
||||
large_model=False,
|
||||
verify_sample_size=1,
|
||||
refusal_max_tokens=1,
|
||||
residue_file=[],
|
||||
dataset="builtin",
|
||||
residue_weight=5,
|
||||
residue_max=None,
|
||||
prompt_pairs_file=str(tmp_path / "missing.json"),
|
||||
contribute=False,
|
||||
contribute_notes="",
|
||||
)
|
||||
|
||||
with pytest.raises(SystemExit) as exc:
|
||||
cli._cmd_abliterate(args)
|
||||
|
||||
assert exc.value.code == 2
|
||||
|
||||
|
||||
class _EncodingOnlyStdout:
|
||||
"""Minimal stream stand-in for encoding-selection tests."""
|
||||
|
||||
Reference in New Issue
Block a user