mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-08-17 16:37:30 +02:00
test: add Gate 3 numerical oracle contracts
This commit is contained in:
Joseph Magly
@@ -8,7 +8,10 @@ downloading real models or running any pipeline. They use
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from io import StringIO
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
@@ -155,6 +158,203 @@ class TestCLIDispatch:
|
||||
assert args_passed.contribute is True
|
||||
assert args_passed.contribute_notes == "Testing contribution system"
|
||||
|
||||
@pytest.mark.parametrize("flag", ["--prompt-pairs-file", "--prompt-pair-file"])
|
||||
@pytest.mark.parametrize("command", ["obliterate", "abliterate"])
|
||||
def test_prompt_pairs_file_flag_is_available_on_obliterate_and_alias(
|
||||
self,
|
||||
command,
|
||||
flag,
|
||||
tmp_path,
|
||||
):
|
||||
"""Explicit prompt-pair files are parsed for both commands and flag spellings."""
|
||||
path = tmp_path / "pairs.json"
|
||||
path.write_text("{}", encoding="utf-8")
|
||||
with patch("obliteratus.cli._cmd_abliterate") as mock_cmd:
|
||||
main([command, "fake/model", flag, str(path)])
|
||||
args_passed = mock_cmd.call_args[0][0]
|
||||
assert args_passed.prompt_pairs_file == str(path)
|
||||
|
||||
@pytest.mark.parametrize("flag", ["--prompt-pairs-file", "--prompt-pair-file"])
|
||||
@pytest.mark.parametrize("command", ["obliterate", "abliterate"])
|
||||
def test_prompt_pairs_file_is_mutually_exclusive_with_residue_files(
|
||||
self,
|
||||
command,
|
||||
flag,
|
||||
tmp_path,
|
||||
):
|
||||
"""Explicit prompt-pair files and mined residue construction cannot be mixed."""
|
||||
path = tmp_path / "pairs.json"
|
||||
path.write_text("{}", encoding="utf-8")
|
||||
stderr_text = _capture_exit(
|
||||
[
|
||||
command,
|
||||
"fake/model",
|
||||
flag,
|
||||
str(path),
|
||||
"--residue-file",
|
||||
"audit.json",
|
||||
],
|
||||
expect_code=2,
|
||||
)
|
||||
assert "not allowed with argument" in stderr_text.lower()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("option", "value"),
|
||||
[
|
||||
("--dataset", "custom"),
|
||||
("--residue-weight", "7"),
|
||||
("--residue-max", "3"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("flag", ["--prompt-pairs-file", "--prompt-pair-file"])
|
||||
@pytest.mark.parametrize("command", ["obliterate", "abliterate"])
|
||||
def test_prompt_pairs_file_rejects_residue_only_options(
|
||||
self,
|
||||
command,
|
||||
flag,
|
||||
option,
|
||||
value,
|
||||
tmp_path,
|
||||
):
|
||||
"""Explicit prompt-pair files cannot be mixed with residue-only options."""
|
||||
path = tmp_path / "pairs.json"
|
||||
path.write_text("{}", encoding="utf-8")
|
||||
stderr_text = _capture_exit(
|
||||
[command, "fake/model", flag, str(path), option, value],
|
||||
expect_code=2,
|
||||
)
|
||||
assert option in stderr_text
|
||||
assert "can only be used with --residue-file" in stderr_text
|
||||
|
||||
|
||||
def test_cmd_abliterate_wires_prompt_pairs_file_into_pipeline(tmp_path):
|
||||
"""Loaded prompt-pair files are passed directly to AbliterationPipeline."""
|
||||
path = tmp_path / "pairs.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"harmful": [f"harm {index}" for index in range(5)],
|
||||
"harmless": [f"safe {index}" for index in range(5)],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
result_path = tmp_path / "result"
|
||||
result_path.mkdir()
|
||||
pipeline = MagicMock()
|
||||
pipeline.run.return_value = str(result_path)
|
||||
|
||||
class FakeLive:
|
||||
def __init__(self, *_args, **_kwargs):
|
||||
self.update = MagicMock()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
args = SimpleNamespace(
|
||||
model="org/model",
|
||||
output_dir=str(tmp_path / "out"),
|
||||
device="cpu",
|
||||
dtype="float32",
|
||||
method="basic",
|
||||
n_directions=1,
|
||||
direction_method=None,
|
||||
regularization=None,
|
||||
refinement_passes=1,
|
||||
min_layer_fraction=None,
|
||||
max_layer_fraction=None,
|
||||
harmless_pc_count=None,
|
||||
shield_concept_count=None,
|
||||
shield_ridge=None,
|
||||
shield_residualize=None,
|
||||
shield_layer_penalty=None,
|
||||
projection_target=None,
|
||||
projection_row_fraction=None,
|
||||
quantization=None,
|
||||
gpu_memory_utilization=None,
|
||||
large_model=False,
|
||||
verify_sample_size=1,
|
||||
refusal_max_tokens=1,
|
||||
residue_file=[],
|
||||
dataset="builtin",
|
||||
residue_weight=5,
|
||||
residue_max=None,
|
||||
prompt_pairs_file=str(path),
|
||||
contribute=False,
|
||||
contribute_notes="",
|
||||
)
|
||||
with (
|
||||
patch("rich.live.Live", FakeLive),
|
||||
patch("obliteratus.abliterate.AbliterationPipeline", return_value=pipeline) as factory,
|
||||
patch("obliteratus.telemetry.maybe_send_pipeline_report"),
|
||||
):
|
||||
from obliteratus import cli
|
||||
|
||||
cli._cmd_abliterate(args)
|
||||
|
||||
assert factory.call_args.kwargs["harmful_prompts"] == [f"harm {index}" for index in range(5)]
|
||||
assert factory.call_args.kwargs["harmless_prompts"] == [f"safe {index}" for index in range(5)]
|
||||
|
||||
args.prompt_pairs_file = None
|
||||
pipeline.reset_mock()
|
||||
pipeline.run.return_value = str(result_path)
|
||||
with (
|
||||
patch("rich.live.Live", FakeLive),
|
||||
patch("obliteratus.abliterate.AbliterationPipeline", return_value=pipeline) as factory,
|
||||
patch("obliteratus.telemetry.maybe_send_pipeline_report"),
|
||||
):
|
||||
from obliteratus import cli
|
||||
|
||||
cli._cmd_abliterate(args)
|
||||
|
||||
assert "harmful_prompts" not in factory.call_args.kwargs
|
||||
assert "harmless_prompts" not in factory.call_args.kwargs
|
||||
|
||||
|
||||
def test_cmd_abliterate_reports_invalid_prompt_pairs_file(tmp_path):
|
||||
from obliteratus import cli
|
||||
|
||||
args = SimpleNamespace(
|
||||
model="org/model",
|
||||
output_dir=str(tmp_path / "out"),
|
||||
device="cpu",
|
||||
dtype="float32",
|
||||
method="basic",
|
||||
n_directions=1,
|
||||
direction_method=None,
|
||||
regularization=None,
|
||||
refinement_passes=1,
|
||||
min_layer_fraction=None,
|
||||
max_layer_fraction=None,
|
||||
harmless_pc_count=None,
|
||||
shield_concept_count=None,
|
||||
shield_ridge=None,
|
||||
shield_residualize=None,
|
||||
shield_layer_penalty=None,
|
||||
projection_target=None,
|
||||
projection_row_fraction=None,
|
||||
quantization=None,
|
||||
gpu_memory_utilization=None,
|
||||
large_model=False,
|
||||
verify_sample_size=1,
|
||||
refusal_max_tokens=1,
|
||||
residue_file=[],
|
||||
dataset="builtin",
|
||||
residue_weight=5,
|
||||
residue_max=None,
|
||||
prompt_pairs_file=str(tmp_path / "missing.json"),
|
||||
contribute=False,
|
||||
contribute_notes="",
|
||||
)
|
||||
|
||||
with pytest.raises(SystemExit) as exc:
|
||||
cli._cmd_abliterate(args)
|
||||
|
||||
assert exc.value.code == 2
|
||||
|
||||
|
||||
class _EncodingOnlyStdout:
|
||||
"""Minimal stream stand-in for encoding-selection tests."""
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Isolated tests for the CI-only mutmut covered-line reuse hook."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).parents[1]
|
||||
SITECUSTOMIZE = ROOT / "scripts" / "mutmut_coverage_sitecustomize"
|
||||
|
||||
|
||||
def _write_fake_modules(tmp_path: Path, *, validation: str = "return None") -> None:
|
||||
mutmut_dir = tmp_path / "mutmut"
|
||||
scripts_dir = tmp_path / "scripts"
|
||||
mutmut_dir.mkdir()
|
||||
scripts_dir.mkdir()
|
||||
(mutmut_dir / "__init__.py").write_text("", encoding="utf-8")
|
||||
(mutmut_dir / "__main__.py").write_text(
|
||||
"def store_lines_covered_by_tests():\n"
|
||||
" print('original coverage collector')\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(scripts_dir / "__init__.py").write_text("", encoding="utf-8")
|
||||
(scripts_dir / "prepare_mutation_coverage.py").write_text(
|
||||
"def validate_manifest():\n"
|
||||
f" {validation}\n"
|
||||
"def validate_execution_manifest():\n"
|
||||
f" {validation}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _run_hook_probe(tmp_path: Path, *, env_flag: bool, validation: str = "return None"):
|
||||
_write_fake_modules(tmp_path, validation=validation)
|
||||
env = os.environ.copy()
|
||||
env.pop("OBLITERATUS_MUTMUT_REUSE_COVERAGE", None)
|
||||
env["PYTHONPATH"] = f"{SITECUSTOMIZE}:{tmp_path}"
|
||||
if env_flag:
|
||||
env["OBLITERATUS_MUTMUT_REUSE_COVERAGE"] = "1"
|
||||
return subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
"from mutmut import __main__ as m; m.store_lines_covered_by_tests()",
|
||||
],
|
||||
check=False,
|
||||
env=env,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
|
||||
|
||||
def test_sitecustomize_does_not_patch_without_reuse_env(tmp_path):
|
||||
result = _run_hook_probe(tmp_path, env_flag=False)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert result.stdout.strip() == "original coverage collector"
|
||||
|
||||
|
||||
def test_sitecustomize_patches_only_with_reuse_env_and_valid_manifest(tmp_path):
|
||||
result = _run_hook_probe(tmp_path, env_flag=True, validation="print('validated')")
|
||||
|
||||
assert result.returncode == 0
|
||||
assert "validated" in result.stdout
|
||||
assert "Reusing prepared mutmut covered-line artifacts" in result.stdout
|
||||
assert "original coverage collector" not in result.stdout
|
||||
|
||||
|
||||
def test_sitecustomize_fails_closed_when_manifest_validation_fails(tmp_path):
|
||||
result = _run_hook_probe(
|
||||
tmp_path,
|
||||
env_flag=True,
|
||||
validation="raise RuntimeError('stale manifest')",
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "stale manifest" in result.stderr
|
||||
|
||||
|
||||
def test_sitecustomize_reuses_prepared_stats_when_enabled(tmp_path):
|
||||
_write_fake_modules(tmp_path, validation="print('validated execution')")
|
||||
env = os.environ.copy()
|
||||
env.pop("OBLITERATUS_MUTMUT_REUSE_COVERAGE", None)
|
||||
env["OBLITERATUS_MUTMUT_REUSE_STATS"] = "1"
|
||||
env["PYTHONPATH"] = f"{SITECUSTOMIZE}:{tmp_path}"
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
(
|
||||
"from mutmut import __main__ as m\n"
|
||||
"m.load_stats = lambda: True\n"
|
||||
"m.collect_or_load_stats(object())\n"
|
||||
),
|
||||
],
|
||||
check=False,
|
||||
env=env,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert "validated execution" in result.stdout
|
||||
assert "Reusing prepared mutmut test-selection stats" in result.stdout
|
||||
@@ -58,6 +58,14 @@ def test_direction_count_must_be_a_positive_integer(n_directions):
|
||||
validate_whitened_request(2, 2, n_directions)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n_directions", [-1, 0])
|
||||
def test_non_positive_direction_count_uses_public_error_message(n_directions):
|
||||
with pytest.raises(ValueError) as excinfo:
|
||||
validate_whitened_request(2, 2, n_directions)
|
||||
|
||||
assert str(excinfo.value) == "n_directions must be a positive integer"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n_directions", [1, 2, 100])
|
||||
def test_valid_direction_count_is_returned(n_directions):
|
||||
assert validate_whitened_request(2, 2, n_directions) == n_directions
|
||||
|
||||
@@ -23,20 +23,62 @@ from tests.fixtures.tiny_offline_model import build_tiny_offline_model
|
||||
|
||||
pytestmark = [pytest.mark.cpu, pytest.mark.integration]
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
THREAD_BOUND_ENV = {
|
||||
"BLIS_NUM_THREADS": "1",
|
||||
"MKL_NUM_THREADS": "1",
|
||||
"NUMEXPR_NUM_THREADS": "1",
|
||||
"OMP_NUM_THREADS": "1",
|
||||
"OMP_THREAD_LIMIT": "1",
|
||||
"OPENBLAS_NUM_THREADS": "1",
|
||||
"VECLIB_MAXIMUM_THREADS": "1",
|
||||
}
|
||||
|
||||
|
||||
def _offline_cli_env(home: Path) -> dict[str, str]:
|
||||
return {
|
||||
**os.environ,
|
||||
**THREAD_BOUND_ENV,
|
||||
"CUDA_VISIBLE_DEVICES": "",
|
||||
"HOME": str(home),
|
||||
"HF_HOME": str(home / "hf"),
|
||||
"HF_DATASETS_OFFLINE": "1",
|
||||
"HF_HUB_DISABLE_TELEMETRY": "1",
|
||||
"HF_HUB_OFFLINE": "1",
|
||||
"TOKENIZERS_PARALLELISM": "false",
|
||||
"TRANSFORMERS_OFFLINE": "1",
|
||||
}
|
||||
|
||||
|
||||
def _assert_offline_cli_env_thread_bounded(
|
||||
workdir: Path,
|
||||
env: dict[str, str],
|
||||
) -> None:
|
||||
assert {key: env[key] for key in THREAD_BOUND_ENV} == THREAD_BOUND_ENV
|
||||
child_env = {
|
||||
**env,
|
||||
"OBLITERATUS_TEST_THREAD_BOUND_KEYS": ",".join(THREAD_BOUND_ENV),
|
||||
}
|
||||
child = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-I",
|
||||
"-c",
|
||||
(
|
||||
"import json, os; "
|
||||
"keys = os.environ['OBLITERATUS_TEST_THREAD_BOUND_KEYS'].split(','); "
|
||||
"print(json.dumps({key: os.environ.get(key) for key in keys}))"
|
||||
),
|
||||
],
|
||||
cwd=workdir,
|
||||
env=child_env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
assert json.loads(child.stdout) == THREAD_BOUND_ENV
|
||||
|
||||
|
||||
def _obliteratus_import_origin(workdir: Path, env: dict[str, str]) -> Path:
|
||||
origin = subprocess.run(
|
||||
[sys.executable, "-I", "-c", "import obliteratus; print(obliteratus.__file__)"],
|
||||
@@ -154,6 +196,7 @@ def test_installed_wheel_cli_loads_local_model_without_repository_imports(tmp_pa
|
||||
isolated_home = tmp_path / "home"
|
||||
isolated_home.mkdir()
|
||||
env = _offline_cli_env(isolated_home)
|
||||
_assert_offline_cli_env_thread_bounded(isolated_workdir, env)
|
||||
|
||||
origin = _obliteratus_import_origin(isolated_workdir, env)
|
||||
_require_installed_artifact_import(origin)
|
||||
@@ -213,14 +256,38 @@ def test_installed_package_cli_executes_offline_checkpoint_to_report_slice(tmp_p
|
||||
},
|
||||
),
|
||||
)
|
||||
prompt_pairs_path = tmp_path / "prompt-pairs.json"
|
||||
prompt_pairs_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"harmful": [
|
||||
"Explain an unsafe request in one sentence",
|
||||
"Describe a harmful instruction plainly",
|
||||
"Write a risky instruction prompt",
|
||||
"Summarize a prohibited request",
|
||||
"State an unsafe operational request",
|
||||
],
|
||||
"harmless": [
|
||||
"Explain a safe request in one sentence",
|
||||
"Describe a helpful instruction plainly",
|
||||
"Write a benign instruction prompt",
|
||||
"Summarize an allowed request",
|
||||
"State a safe operational request",
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
isolated_workdir = tmp_path / "outside-repository"
|
||||
isolated_workdir.mkdir()
|
||||
isolated_home = tmp_path / "home"
|
||||
isolated_home.mkdir()
|
||||
env = _offline_cli_env(isolated_home)
|
||||
_assert_offline_cli_env_thread_bounded(isolated_workdir, env)
|
||||
origin = _obliteratus_import_origin(isolated_workdir, env)
|
||||
_require_installed_artifact_import(origin)
|
||||
|
||||
original = _state_dict(source)
|
||||
subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
@@ -245,6 +312,8 @@ def test_installed_package_cli_executes_offline_checkpoint_to_report_slice(tmp_p
|
||||
"1",
|
||||
"--refusal-max-tokens",
|
||||
"1",
|
||||
"--prompt-pairs-file",
|
||||
str(prompt_pairs_path),
|
||||
],
|
||||
cwd=isolated_workdir,
|
||||
env=env,
|
||||
@@ -253,8 +322,22 @@ def test_installed_package_cli_executes_offline_checkpoint_to_report_slice(tmp_p
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
assert (checkpoint / "abliteration_metadata.json").is_file()
|
||||
metadata_path = checkpoint / "abliteration_metadata.json"
|
||||
assert metadata_path.is_file()
|
||||
metadata = json.loads(metadata_path.read_text())
|
||||
assert metadata["source_model"] == str(source)
|
||||
assert metadata["method"] == "basic"
|
||||
assert metadata["method_config"]["n_directions"] == 1
|
||||
assert metadata["method_config"]["refinement_passes"] == 1
|
||||
assert metadata["n_harmful_prompts"] == 5
|
||||
assert metadata["n_harmless_prompts"] == 5
|
||||
AutoModelForCausalLM.from_pretrained(checkpoint, local_files_only=True)
|
||||
checkpoint_state = _state_dict(checkpoint)
|
||||
assert original.keys() == checkpoint_state.keys()
|
||||
assert any(
|
||||
not torch.equal(original[name], tensor)
|
||||
for name, tensor in checkpoint_state.items()
|
||||
)
|
||||
|
||||
subprocess.run(
|
||||
[sys.executable, "-I", "-m", "obliteratus", "run", str(config_path)],
|
||||
|
||||
@@ -0,0 +1,711 @@
|
||||
"""Reference-oracle contracts for pure projection and orthogonalization math."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from obliteratus.abliterate import AbliterationPipeline
|
||||
from obliteratus.analysis.numerical_contracts import (
|
||||
orthogonalize_subspace_rows,
|
||||
project_weight_against_direction,
|
||||
remove_harmless_principal_components,
|
||||
residualize_against_shield_atoms,
|
||||
select_projection_coefficients,
|
||||
)
|
||||
|
||||
|
||||
def _canonical_rows(rows: torch.Tensor) -> torch.Tensor:
|
||||
result = rows.clone()
|
||||
for idx in range(result.shape[0]):
|
||||
pivot = result[idx].abs().argmax()
|
||||
if result[idx, pivot] < 0:
|
||||
result[idx] = -result[idx]
|
||||
return result
|
||||
|
||||
|
||||
def _reference_gram_schmidt(rows: torch.Tensor) -> torch.Tensor:
|
||||
basis: list[torch.Tensor] = []
|
||||
for row in rows.to(dtype=torch.float64):
|
||||
residual = row.clone()
|
||||
for prev in basis:
|
||||
residual = residual - (residual @ prev) * prev
|
||||
norm = residual.norm()
|
||||
if norm > 1e-8:
|
||||
basis.append(residual / norm)
|
||||
return torch.stack(basis)[: rows.shape[0]].to(dtype=rows.dtype, device=rows.device)
|
||||
|
||||
|
||||
def _reference_projection(weight: torch.Tensor, direction: torch.Tensor, scale: float) -> torch.Tensor:
|
||||
work = weight.to(dtype=torch.float64)
|
||||
d = direction.reshape(-1, 1).to(dtype=torch.float64, device=weight.device)
|
||||
d_norm = d.norm()
|
||||
if d_norm < 1e-8:
|
||||
return weight.clone()
|
||||
d = d / d_norm
|
||||
if weight.shape[-1] == d.shape[0]:
|
||||
coeff = work @ d
|
||||
return (work - d.T * (scale * coeff)).to(dtype=weight.dtype)
|
||||
coeff = d.T @ work
|
||||
return (work - (scale * d) * coeff).to(dtype=weight.dtype)
|
||||
|
||||
|
||||
def test_orthogonalize_matches_reference_and_preserves_primary_orientation():
|
||||
subspace = torch.tensor(
|
||||
[[2.0, 0.0, 0.0], [1.0, 3.0, 0.0], [1.0, 1.0, 4.0]],
|
||||
dtype=torch.float64,
|
||||
)
|
||||
|
||||
actual = orthogonalize_subspace_rows(subspace)
|
||||
expected = _reference_gram_schmidt(subspace)
|
||||
|
||||
assert actual.dtype == subspace.dtype
|
||||
assert torch.allclose(actual @ actual.T, torch.eye(3, dtype=torch.float64), atol=1e-12)
|
||||
assert torch.allclose(_canonical_rows(actual), _canonical_rows(expected), atol=1e-12)
|
||||
assert actual[0] @ subspace[0] > 0
|
||||
|
||||
|
||||
def test_orthogonalize_returns_degenerate_inputs_without_allocating_new_tensor():
|
||||
single_row = torch.tensor([[1.0, 2.0, 3.0]], dtype=torch.float64)
|
||||
empty = torch.empty((2, 0), dtype=torch.float64)
|
||||
|
||||
assert orthogonalize_subspace_rows(single_row) is single_row
|
||||
assert orthogonalize_subspace_rows(empty) is empty
|
||||
|
||||
|
||||
def test_orthogonalize_non_finite_subspace_is_explicit_noop():
|
||||
subspace = torch.tensor([[1.0, 0.0], [float("nan"), 1.0]], dtype=torch.float64)
|
||||
|
||||
actual = orthogonalize_subspace_rows(subspace)
|
||||
|
||||
assert actual is subspace
|
||||
assert torch.isnan(actual[1, 0])
|
||||
|
||||
|
||||
def test_orthogonalization_preserves_float64_precision_for_nearly_collinear_rows():
|
||||
subspace = torch.tensor(
|
||||
[[1.0, 1e-6, 0.0], [1.0, 0.0, 1e-6], [0.0, 1.0, 1.0]],
|
||||
dtype=torch.float64,
|
||||
)
|
||||
|
||||
actual = orthogonalize_subspace_rows(subspace)
|
||||
expected = _reference_gram_schmidt(subspace)
|
||||
|
||||
assert actual.dtype == torch.float64
|
||||
assert torch.allclose(_canonical_rows(actual), _canonical_rows(expected), atol=1e-10)
|
||||
|
||||
|
||||
def test_orthogonalize_two_half_precision_rows_uses_stable_compute_dtype():
|
||||
subspace = torch.tensor(
|
||||
[[1.0, 1.0, 0.0], [1.0, 0.0, 1.0]],
|
||||
dtype=torch.float16,
|
||||
)
|
||||
|
||||
actual = orthogonalize_subspace_rows(subspace)
|
||||
expected = _reference_gram_schmidt(subspace)
|
||||
|
||||
assert actual is not subspace
|
||||
assert actual.dtype == torch.float16
|
||||
assert torch.allclose(_canonical_rows(actual), _canonical_rows(expected), atol=1e-3)
|
||||
gram = actual.float() @ actual.float().T
|
||||
assert torch.allclose(gram, torch.eye(2), atol=1e-3)
|
||||
|
||||
|
||||
def test_integer_projection_uses_supported_float_compute_then_restores_weight_dtype():
|
||||
weight = torch.tensor([[2, 0], [0, 2]], dtype=torch.int64)
|
||||
direction = torch.tensor([1, 0], dtype=torch.int64)
|
||||
|
||||
projected = project_weight_against_direction(weight, direction)
|
||||
|
||||
assert projected.projected is True
|
||||
assert projected.weight.dtype == torch.int64
|
||||
assert torch.equal(projected.weight, torch.tensor([[0, 0], [0, 2]], dtype=torch.int64))
|
||||
|
||||
|
||||
def test_projection_full_removal_is_idempotent_and_orthogonal_to_direction():
|
||||
weight = torch.tensor([[3.0, 4.0, 0.0], [1.0, -2.0, 2.0]], dtype=torch.float64)
|
||||
direction = torch.tensor([0.6, 0.8, 0.0], dtype=torch.float64)
|
||||
|
||||
first = project_weight_against_direction(weight, direction, regularization=0.0)
|
||||
second = project_weight_against_direction(first.weight, direction, regularization=0.0)
|
||||
|
||||
assert first.projected
|
||||
assert torch.allclose(first.weight @ direction, torch.zeros(2, dtype=torch.float64), atol=1e-12)
|
||||
assert torch.allclose(first.weight, second.weight, atol=1e-12)
|
||||
assert torch.allclose(first.weight, _reference_projection(weight, direction, scale=1.0), atol=1e-12)
|
||||
|
||||
|
||||
def test_projection_normalizes_non_unit_directions_before_applying_formula():
|
||||
weight = torch.tensor([[3.0, 4.0], [-5.0, 6.0]], dtype=torch.float64)
|
||||
|
||||
unit = project_weight_against_direction(weight, torch.tensor([1.0, 0.0], dtype=torch.float64))
|
||||
non_unit = project_weight_against_direction(weight, torch.tensor([2.0, 0.0], dtype=torch.float64))
|
||||
|
||||
assert unit.projected
|
||||
assert non_unit.projected
|
||||
assert torch.allclose(non_unit.weight, unit.weight, atol=1e-12)
|
||||
assert torch.allclose(non_unit.weight, _reference_projection(weight, torch.tensor([2.0, 0.0]), 1.0))
|
||||
|
||||
|
||||
def test_projection_zero_direction_is_deterministic_no_op():
|
||||
weight = torch.tensor([[3.0, 4.0], [-5.0, 6.0]], dtype=torch.float64)
|
||||
direction = torch.zeros(2, dtype=torch.float64)
|
||||
|
||||
first = project_weight_against_direction(weight, direction)
|
||||
second = project_weight_against_direction(weight, direction)
|
||||
|
||||
assert not first.projected
|
||||
assert first.layout is None
|
||||
assert torch.equal(first.weight, weight)
|
||||
assert torch.equal(second.weight, first.weight)
|
||||
assert first.coefficient_norm_sq == 0.0
|
||||
|
||||
|
||||
def test_projection_projects_at_exact_tiny_direction_threshold():
|
||||
weight = torch.tensor([[3.0, 4.0]], dtype=torch.float64)
|
||||
direction = torch.tensor([1e-8, 0.0], dtype=torch.float64)
|
||||
|
||||
projected = project_weight_against_direction(weight, direction)
|
||||
|
||||
assert projected.projected is True
|
||||
assert projected.layout == "standard"
|
||||
assert torch.allclose(projected.weight, torch.tensor([[0.0, 4.0]], dtype=torch.float64), atol=1e-12)
|
||||
|
||||
|
||||
def test_projection_zero_direction_metadata_uses_strict_false_flag():
|
||||
result = project_weight_against_direction(
|
||||
torch.tensor([[3.0, 4.0]], dtype=torch.float64),
|
||||
torch.zeros(2, dtype=torch.float64),
|
||||
)
|
||||
|
||||
assert result.projected is False
|
||||
assert result.weight is not None
|
||||
assert result.coefficient_norm_sq == 0.0
|
||||
assert result.layout is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("projection_row_fraction", [0.0, -0.1, 1.01])
|
||||
def test_select_projection_coefficients_rejects_invalid_fraction(projection_row_fraction):
|
||||
coeff = torch.tensor([[1.0], [2.0]], dtype=torch.float64)
|
||||
|
||||
with pytest.raises(ValueError, match=r"projection_row_fraction must be in"):
|
||||
select_projection_coefficients(coeff, projection_row_fraction)
|
||||
|
||||
|
||||
def test_select_projection_coefficients_empty_and_singleton_inputs_are_noops():
|
||||
empty = torch.empty((0, 1), dtype=torch.float64)
|
||||
singleton = torch.tensor([[3.0]], dtype=torch.float64)
|
||||
|
||||
assert select_projection_coefficients(empty, 0.5) is empty
|
||||
assert select_projection_coefficients(singleton, 0.5) is singleton
|
||||
|
||||
|
||||
def test_projection_unsupported_layout_returns_complete_noop_metadata_clone():
|
||||
weight = torch.tensor([[1.0, 2.0, 3.0]], dtype=torch.float64)
|
||||
direction = torch.tensor([1.0, 0.0], dtype=torch.float64)
|
||||
|
||||
projected = project_weight_against_direction(weight, direction)
|
||||
|
||||
assert projected.projected is False
|
||||
assert projected.layout is None
|
||||
assert projected.coefficient_norm_sq == 0.0
|
||||
assert projected.weight is not weight
|
||||
assert torch.equal(projected.weight, weight)
|
||||
|
||||
|
||||
def test_projection_standard_layout_non_finite_coefficients_fail_closed():
|
||||
huge = torch.finfo(torch.float32).max
|
||||
weight = torch.tensor([[huge, huge]], dtype=torch.float32)
|
||||
direction = torch.tensor([1.0, 1.0], dtype=torch.float32)
|
||||
|
||||
projected = project_weight_against_direction(weight, direction)
|
||||
|
||||
assert projected.projected is False
|
||||
assert projected.layout == "standard"
|
||||
assert projected.coefficient_norm_sq == 0.0
|
||||
assert torch.equal(projected.weight, weight)
|
||||
|
||||
|
||||
def test_projection_transposed_layout_non_finite_coefficients_fail_closed():
|
||||
huge = torch.finfo(torch.float32).max
|
||||
weight = torch.tensor([[huge], [huge]], dtype=torch.float32)
|
||||
direction = torch.tensor([1.0, 1.0], dtype=torch.float32)
|
||||
|
||||
projected = project_weight_against_direction(weight, direction)
|
||||
|
||||
assert projected.projected is False
|
||||
assert projected.layout == "transposed"
|
||||
assert projected.coefficient_norm_sq == 0.0
|
||||
assert torch.equal(projected.weight, weight)
|
||||
|
||||
|
||||
def test_projection_preserves_orthogonal_coordinates_and_contracts_norm_without_restore():
|
||||
direction = torch.tensor([1.0, 0.0, 0.0])
|
||||
orthogonal_probe = torch.tensor([0.0, 2.0, -1.0])
|
||||
weight = torch.tensor([[5.0, 2.0, -1.0], [-3.0, 4.0, 7.0]])
|
||||
|
||||
projected = project_weight_against_direction(weight, direction, norm_preserve=False)
|
||||
|
||||
assert torch.allclose(projected.weight @ direction, torch.zeros(2))
|
||||
assert torch.allclose(projected.weight @ orthogonal_probe, weight @ orthogonal_probe)
|
||||
assert projected.weight.norm() <= weight.norm()
|
||||
|
||||
|
||||
def test_norm_preservation_uses_cap_when_projection_would_amplify_too_much():
|
||||
weight = torch.tensor([[100.0, 1.0], [0.0, 0.0]])
|
||||
direction = torch.tensor([1.0, 0.0])
|
||||
|
||||
projected = project_weight_against_direction(
|
||||
weight,
|
||||
direction,
|
||||
norm_preserve=True,
|
||||
max_norm_ratio=1.10,
|
||||
)
|
||||
|
||||
assert projected.projected
|
||||
assert torch.isclose(projected.weight.norm(), torch.tensor(1.10), atol=1e-6)
|
||||
|
||||
|
||||
def test_norm_preservation_keeps_zero_projection_without_restoration():
|
||||
weight = torch.tensor([[3.0, 0.0]], dtype=torch.float64)
|
||||
direction = torch.tensor([1.0, 0.0], dtype=torch.float64)
|
||||
|
||||
projected = project_weight_against_direction(weight, direction, norm_preserve=True)
|
||||
|
||||
assert projected.projected is True
|
||||
assert projected.layout == "standard"
|
||||
assert projected.coefficient_norm_sq == 9.0
|
||||
assert torch.allclose(projected.weight, torch.zeros_like(weight), atol=0.0, rtol=0.0)
|
||||
assert torch.isfinite(projected.weight).all()
|
||||
|
||||
|
||||
def test_norm_preservation_at_max_ratio_boundary_preserves_original_norm():
|
||||
direction = torch.tensor([1.0, 0.0], dtype=torch.float64)
|
||||
weight = torch.tensor([[math.sqrt(21.0), 10.0]], dtype=torch.float64)
|
||||
|
||||
projected = project_weight_against_direction(
|
||||
weight,
|
||||
direction,
|
||||
norm_preserve=True,
|
||||
max_norm_ratio=1.10,
|
||||
)
|
||||
|
||||
assert projected.projected
|
||||
assert torch.allclose(projected.weight.norm(), weight.norm(), atol=1e-12)
|
||||
|
||||
|
||||
def test_transposed_norm_preservation_reports_removed_coefficient_energy():
|
||||
direction = torch.tensor([1.0, 0.0], dtype=torch.float64)
|
||||
weight = torch.tensor([[3.0, 4.0, 0.0], [10.0, 20.0, 30.0]], dtype=torch.float64)
|
||||
|
||||
projected = project_weight_against_direction(weight, direction, norm_preserve=True)
|
||||
|
||||
assert projected.projected is True
|
||||
assert projected.layout == "transposed"
|
||||
assert projected.coefficient_norm_sq == 25.0
|
||||
assert torch.allclose(projected.weight[0], torch.zeros(3, dtype=torch.float64), atol=1e-12)
|
||||
|
||||
|
||||
def test_transposed_projection_without_norm_preservation_reports_zero_metadata_energy():
|
||||
direction = torch.tensor([1.0, 0.0], dtype=torch.float64)
|
||||
weight = torch.tensor([[3.0, 4.0, 0.0], [10.0, 20.0, 30.0]], dtype=torch.float64)
|
||||
|
||||
projected = project_weight_against_direction(weight, direction, norm_preserve=False)
|
||||
|
||||
assert projected.projected is True
|
||||
assert projected.layout == "transposed"
|
||||
assert projected.coefficient_norm_sq == 0.0
|
||||
assert torch.allclose(projected.weight[0], torch.zeros(3, dtype=torch.float64), atol=1e-12)
|
||||
|
||||
|
||||
def test_projection_supports_standard_and_transposed_layouts():
|
||||
direction = torch.tensor([1.0, 0.0])
|
||||
standard = torch.tensor([[3.0, 4.0], [5.0, 6.0], [7.0, 8.0]])
|
||||
transposed = standard.T.contiguous()
|
||||
|
||||
standard_result = project_weight_against_direction(standard, direction)
|
||||
transposed_result = project_weight_against_direction(transposed, direction)
|
||||
|
||||
assert torch.allclose(standard_result.weight[:, 0], torch.zeros(3))
|
||||
assert torch.allclose(transposed_result.weight[0, :], torch.zeros(3))
|
||||
assert torch.allclose(standard_result.weight, _reference_projection(standard, direction, 1.0))
|
||||
assert torch.allclose(transposed_result.weight, _reference_projection(transposed, direction, 1.0))
|
||||
|
||||
|
||||
def test_projection_rejects_orthogonal_direction_magnitude_as_a_signal():
|
||||
weight = torch.tensor([[4.0, 3.0], [2.0, -1.0]], dtype=torch.float64)
|
||||
unit = project_weight_against_direction(weight, torch.tensor([1.0, 0.0], dtype=torch.float64))
|
||||
scaled = project_weight_against_direction(weight, torch.tensor([5.0, 0.0], dtype=torch.float64))
|
||||
|
||||
assert torch.allclose(unit.weight, scaled.weight, atol=1e-12)
|
||||
assert unit.layout == scaled.layout == "standard"
|
||||
|
||||
|
||||
def test_row_fraction_selects_largest_coefficients_and_is_permutation_equivariant():
|
||||
coeff = torch.tensor([[0.5], [-3.0], [2.0], [0.1]])
|
||||
selected = select_projection_coefficients(coeff, 0.5)
|
||||
|
||||
assert selected.tolist() == [[0.0], [-3.0], [2.0], [0.0]]
|
||||
|
||||
permutation = torch.tensor([2, 0, 3, 1])
|
||||
permuted = select_projection_coefficients(coeff[permutation], 0.5)
|
||||
assert torch.allclose(permuted, selected[permutation])
|
||||
|
||||
|
||||
def test_projection_row_fraction_removes_only_selected_rows():
|
||||
weight = torch.tensor([[10.0, 1.0], [1.0, 7.0], [-5.0, 2.0], [0.2, 9.0]])
|
||||
direction = torch.tensor([1.0, 0.0])
|
||||
|
||||
projected = project_weight_against_direction(weight, direction, projection_row_fraction=0.5)
|
||||
|
||||
assert torch.allclose(projected.weight[:, 0], torch.tensor([0.0, 1.0, 0.0, 0.2]))
|
||||
assert torch.allclose(projected.weight[:, 1], weight[:, 1])
|
||||
|
||||
|
||||
def test_projection_row_fraction_keeps_only_the_two_largest_magnitudes():
|
||||
coeff = torch.tensor([[0.5], [-3.0], [2.0], [0.1]], dtype=torch.float64)
|
||||
selected = select_projection_coefficients(coeff, 0.5)
|
||||
|
||||
assert torch.equal(selected != 0, torch.tensor([[False], [True], [True], [False]]))
|
||||
assert torch.allclose(selected.abs().sum(), torch.tensor(5.0, dtype=torch.float64))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("regularization", [-1.0, 0.25, 1.25])
|
||||
def test_finite_regularization_values_are_applied_without_unit_interval_clamping(regularization):
|
||||
weight = torch.tensor([[4.0, 3.0]], dtype=torch.float64)
|
||||
direction = torch.tensor([1.0, 0.0], dtype=torch.float64)
|
||||
scale = 1.0 - regularization
|
||||
|
||||
projected = project_weight_against_direction(weight, direction, regularization=regularization)
|
||||
|
||||
assert projected.projected
|
||||
assert torch.allclose(projected.weight, _reference_projection(weight, direction, scale), atol=1e-12)
|
||||
|
||||
|
||||
def test_harmless_pc_removal_orthogonalizes_against_dominant_component():
|
||||
subspace = torch.tensor([[1.0, 1.0, 0.0], [0.5, 0.0, 1.0]], dtype=torch.float64)
|
||||
harmless = torch.tensor(
|
||||
[[-2.0, 0.0, 0.0], [0.0, 0.0, 0.0], [2.0, 0.0, 0.0], [4.0, 0.0, 0.0]],
|
||||
dtype=torch.float64,
|
||||
)
|
||||
|
||||
residual = remove_harmless_principal_components(subspace, harmless, pc_count=1)
|
||||
|
||||
assert torch.allclose(residual[:, 0], torch.zeros(2, dtype=torch.float64), atol=1e-12)
|
||||
assert torch.allclose(residual.norm(dim=-1), torch.ones(2, dtype=torch.float64), atol=1e-12)
|
||||
assert torch.allclose(residual @ residual.T, torch.eye(2, dtype=torch.float64), atol=1e-12)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("subspace", "harmless", "pc_count"),
|
||||
[
|
||||
(
|
||||
torch.tensor([[1.0, 0.0]], dtype=torch.float64),
|
||||
torch.eye(3, 2, dtype=torch.float64),
|
||||
0,
|
||||
),
|
||||
(
|
||||
torch.tensor([[1.0, 0.0]], dtype=torch.float64),
|
||||
torch.eye(2, dtype=torch.float64),
|
||||
1,
|
||||
),
|
||||
(
|
||||
torch.empty((0, 2), dtype=torch.float64),
|
||||
torch.eye(3, 2, dtype=torch.float64),
|
||||
1,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_harmless_pc_removal_noops_when_preconditions_are_not_met(
|
||||
subspace,
|
||||
harmless,
|
||||
pc_count,
|
||||
):
|
||||
residual = remove_harmless_principal_components(subspace, harmless, pc_count)
|
||||
|
||||
assert residual is subspace
|
||||
|
||||
|
||||
def test_harmless_pc_removal_returns_subspace_when_svd_fails(monkeypatch):
|
||||
subspace = torch.tensor([[1.0, 0.5]], dtype=torch.float64)
|
||||
harmless = torch.tensor([[-2.0, 0.0], [0.0, 0.0], [2.0, 0.0]], dtype=torch.float64)
|
||||
|
||||
def fail_svd(*_args, **_kwargs):
|
||||
raise RuntimeError("svd fixture failure")
|
||||
|
||||
monkeypatch.setattr(torch.linalg, "svd", fail_svd)
|
||||
|
||||
residual = remove_harmless_principal_components(subspace, harmless, pc_count=1)
|
||||
|
||||
assert residual is subspace
|
||||
|
||||
|
||||
def test_harmless_pc_removal_noops_when_svd_returns_no_components():
|
||||
subspace = torch.tensor([[1.0]], dtype=torch.float64)
|
||||
harmless = torch.empty((3, 0), dtype=torch.float64)
|
||||
|
||||
residual = remove_harmless_principal_components(subspace, harmless, pc_count=1)
|
||||
|
||||
assert residual is subspace
|
||||
|
||||
|
||||
def test_harmless_pc_removal_subtracts_pc_component_before_row_normalization():
|
||||
subspace = torch.tensor([[1.0, 0.5]], dtype=torch.float64)
|
||||
harmless = torch.tensor([[-2.0, 0.0], [0.0, 0.0], [2.0, 0.0]], dtype=torch.float64)
|
||||
|
||||
residual = remove_harmless_principal_components(subspace, harmless, pc_count=1)
|
||||
|
||||
assert torch.allclose(residual, torch.tensor([[0.0, 1.0]], dtype=torch.float64), atol=1e-12)
|
||||
assert torch.allclose(residual.norm(dim=-1), torch.ones(1, dtype=torch.float64), atol=1e-12)
|
||||
|
||||
|
||||
def test_harmless_pc_removal_treats_exact_epsilon_residual_as_usable_signal():
|
||||
subspace = torch.tensor([[1.0, 1e-8]], dtype=torch.float64)
|
||||
harmless = torch.tensor([[-2.0, 0.0], [0.0, 0.0], [2.0, 0.0]], dtype=torch.float64)
|
||||
|
||||
residual = remove_harmless_principal_components(subspace, harmless, pc_count=1)
|
||||
|
||||
assert torch.allclose(residual, torch.tensor([[0.0, 1.0]], dtype=torch.float64), atol=1e-12)
|
||||
|
||||
|
||||
def test_harmless_pc_removal_preserves_subunit_residuals_instead_of_restoring_original_row():
|
||||
subspace = torch.tensor([[1.0, 0.5]], dtype=torch.float64)
|
||||
harmless = torch.tensor([[-2.0, 0.0], [0.0, 0.0], [2.0, 0.0]], dtype=torch.float64)
|
||||
|
||||
residual = remove_harmless_principal_components(subspace, harmless, pc_count=1)
|
||||
|
||||
assert torch.allclose(residual[:, 0], torch.zeros(1, dtype=torch.float64), atol=1e-12)
|
||||
assert torch.allclose(residual[:, 1], torch.ones(1, dtype=torch.float64), atol=1e-12)
|
||||
|
||||
|
||||
def test_harmless_pc_removal_restores_only_rows_with_near_zero_residuals():
|
||||
subspace = torch.tensor([[1.0, 0.0], [1.0, 0.5]], dtype=torch.float64)
|
||||
harmless = torch.tensor([[-2.0, 0.0], [0.0, 0.0], [2.0, 0.0]], dtype=torch.float64)
|
||||
|
||||
residual = remove_harmless_principal_components(subspace, harmless, pc_count=1)
|
||||
|
||||
assert torch.allclose(residual @ residual.T, torch.eye(2, dtype=torch.float64), atol=1e-12)
|
||||
assert torch.allclose(residual[0], torch.tensor([1.0, 0.0], dtype=torch.float64), atol=1e-12)
|
||||
assert torch.allclose(residual[1], torch.tensor([0.0, 1.0], dtype=torch.float64), atol=1e-12)
|
||||
|
||||
|
||||
def test_harmless_pc_removal_near_zero_fallback_is_per_row_before_qr():
|
||||
subspace = torch.tensor([[1.0, 1e-9, 0.0], [0.2, 0.5, 1.0]], dtype=torch.float64)
|
||||
harmless = torch.tensor(
|
||||
[[-2.0, 0.0, 0.0], [0.0, 0.0, 0.0], [2.0, 0.0, 0.0]],
|
||||
dtype=torch.float64,
|
||||
)
|
||||
|
||||
residual = remove_harmless_principal_components(subspace, harmless, pc_count=1)
|
||||
|
||||
assert torch.allclose(residual[0], subspace[0], atol=1e-12)
|
||||
assert torch.allclose(residual @ residual.T, torch.eye(2, dtype=torch.float64), atol=1e-8)
|
||||
|
||||
|
||||
def test_harmless_pc_removal_replays_deterministically_for_singular_inputs():
|
||||
subspace = torch.tensor([[0.0, 1.0, 1.0], [0.0, 2.0, 2.0]], dtype=torch.float64)
|
||||
harmless = torch.ones((4, 3), dtype=torch.float64)
|
||||
|
||||
first = remove_harmless_principal_components(subspace, harmless, pc_count=2)
|
||||
second = remove_harmless_principal_components(subspace, harmless, pc_count=2)
|
||||
|
||||
assert torch.allclose(first, second, atol=0.0, rtol=0.0)
|
||||
assert torch.isfinite(first).all()
|
||||
|
||||
|
||||
def test_shield_atom_residualization_handles_rank_deficient_atoms():
|
||||
subspace = torch.tensor([[1.0, 1.0, 0.0], [1.0, 0.0, 1.0]], dtype=torch.float64)
|
||||
atoms = torch.tensor([[1.0, 0.0, 0.0], [2.0, 0.0, 0.0]], dtype=torch.float64)
|
||||
|
||||
residual = residualize_against_shield_atoms(subspace, atoms, ridge=1e-6)
|
||||
|
||||
assert torch.allclose(residual[:, 0], torch.zeros(2, dtype=torch.float64), atol=2e-6)
|
||||
assert torch.allclose(residual.norm(dim=-1), torch.ones(2, dtype=torch.float64), atol=1e-12)
|
||||
assert torch.allclose(residual @ residual.T, torch.eye(2, dtype=torch.float64), atol=1e-12)
|
||||
|
||||
|
||||
def test_shield_atom_residualization_noops_when_atoms_or_subspace_are_empty():
|
||||
subspace = torch.tensor([[1.0, 0.0]], dtype=torch.float64)
|
||||
atoms = torch.empty((0, 2), dtype=torch.float64)
|
||||
empty_subspace = torch.empty((0, 2), dtype=torch.float64)
|
||||
|
||||
assert residualize_against_shield_atoms(subspace, atoms, ridge=1e-3) is subspace
|
||||
assert residualize_against_shield_atoms(empty_subspace, torch.eye(2), ridge=1e-3) is empty_subspace
|
||||
|
||||
|
||||
def test_shield_atom_residualization_returns_subspace_when_solve_fails(monkeypatch):
|
||||
subspace = torch.tensor([[1.0, 1.0]], dtype=torch.float64)
|
||||
atoms = torch.tensor([[1.0, 0.0]], dtype=torch.float64)
|
||||
|
||||
def fail_solve(*_args, **_kwargs):
|
||||
raise RuntimeError("solve fixture failure")
|
||||
|
||||
monkeypatch.setattr(torch.linalg, "solve", fail_solve)
|
||||
|
||||
residual = residualize_against_shield_atoms(subspace, atoms, ridge=1e-6)
|
||||
|
||||
assert residual is subspace
|
||||
|
||||
|
||||
def test_shield_atom_residualization_uses_mixed_dtype_compute_but_returns_subspace_dtype():
|
||||
subspace = torch.tensor([[1.0, 1.0, 0.0]], dtype=torch.float32)
|
||||
atoms = torch.tensor([[1.0, 0.0, 0.0]], dtype=torch.float64)
|
||||
|
||||
residual = residualize_against_shield_atoms(subspace, atoms, ridge=1e-6)
|
||||
|
||||
assert residual.dtype == torch.float32
|
||||
assert residual.device == subspace.device
|
||||
assert torch.allclose(residual, torch.tensor([[0.0, 1.0, 0.0]], dtype=torch.float32), atol=2e-6)
|
||||
|
||||
|
||||
def test_shield_atom_residualization_upcasts_atoms_to_match_float64_subspace_compute():
|
||||
subspace = torch.tensor([[1.0, 1.0, 0.0]], dtype=torch.float64)
|
||||
atoms = torch.tensor([[1.0, 0.0, 0.0]], dtype=torch.float32)
|
||||
|
||||
residual = residualize_against_shield_atoms(subspace, atoms, ridge=1e-6)
|
||||
|
||||
assert residual.dtype == torch.float64
|
||||
assert torch.allclose(residual, torch.tensor([[0.0, 1.0, 0.0]], dtype=torch.float64), atol=2e-6)
|
||||
|
||||
|
||||
def test_shield_atom_residualization_keeps_float64_precision_with_float32_atoms():
|
||||
subspace = torch.tensor([[1.0, 1e-4, 1.0 - 1e-4]], dtype=torch.float64)
|
||||
atoms = torch.tensor([[1.0, 1e-4, 0.0], [1e-4, 1.0, 1e-4]], dtype=torch.float32)
|
||||
|
||||
residual = residualize_against_shield_atoms(subspace, atoms, ridge=1e-12)
|
||||
|
||||
assert residual.dtype == torch.float64
|
||||
assert residual[0, 0] > 1e-8
|
||||
assert torch.allclose(
|
||||
residual,
|
||||
torch.tensor(
|
||||
[[1.0000999534067183e-08, -9.9999997973787514e-05, 0.9999999950000001]],
|
||||
dtype=torch.float64,
|
||||
),
|
||||
atol=1e-15,
|
||||
)
|
||||
|
||||
|
||||
def test_harmless_pc_removal_upcasts_half_precision_for_svd_then_returns_input_dtype():
|
||||
subspace = torch.tensor([[1.0, 0.5]], dtype=torch.float16)
|
||||
harmless = torch.tensor([[-2.0, 0.0], [0.0, 0.0], [2.0, 0.0]], dtype=torch.float16)
|
||||
|
||||
residual = remove_harmless_principal_components(subspace, harmless, pc_count=1)
|
||||
|
||||
assert residual.dtype == torch.float16
|
||||
assert torch.allclose(residual.float(), torch.tensor([[0.0, 1.0]]), atol=1e-3)
|
||||
|
||||
|
||||
def test_dtype_and_device_are_preserved_for_projection_and_residualization():
|
||||
weight = torch.tensor([[1.0, 2.0]], dtype=torch.float32)
|
||||
direction = torch.tensor([1.0, 0.0], dtype=torch.float64)
|
||||
subspace = torch.tensor([[1.0, 1.0]], dtype=torch.float32)
|
||||
harmless = torch.tensor([[-1.0, 0.0], [0.0, 0.0], [1.0, 0.0]], dtype=torch.float64)
|
||||
|
||||
projected = project_weight_against_direction(weight, direction)
|
||||
residual = remove_harmless_principal_components(subspace, harmless, pc_count=1)
|
||||
|
||||
assert projected.weight.dtype == torch.float32
|
||||
assert projected.weight.device == weight.device
|
||||
assert residual.dtype == torch.float32
|
||||
assert residual.device == subspace.device
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad_value", [float("nan"), float("inf")])
|
||||
def test_projection_non_finite_policy_is_skip_without_mutation(bad_value):
|
||||
weight = torch.tensor([[1.0, 2.0], [bad_value, 4.0]])
|
||||
direction = torch.tensor([1.0, 0.0])
|
||||
|
||||
projected = project_weight_against_direction(weight, direction)
|
||||
|
||||
assert not projected.projected
|
||||
assert torch.allclose(projected.weight, weight, equal_nan=True)
|
||||
|
||||
|
||||
def test_zero_inputs_follow_existing_fallback_policy_without_non_finite_output():
|
||||
zero_subspace = torch.zeros((2, 3), dtype=torch.float64)
|
||||
harmless = torch.tensor([[-1.0, 0.0, 0.0], [0.0, 0.0, 0.0], [1.0, 0.0, 0.0]])
|
||||
atoms = torch.tensor([[1.0, 0.0, 0.0]])
|
||||
|
||||
pc_residual = remove_harmless_principal_components(zero_subspace, harmless, pc_count=1)
|
||||
shield_residual = residualize_against_shield_atoms(zero_subspace, atoms, ridge=1e-3)
|
||||
|
||||
assert torch.isfinite(pc_residual).all()
|
||||
assert torch.isfinite(shield_residual).all()
|
||||
assert torch.allclose(pc_residual, zero_subspace)
|
||||
assert torch.allclose(shield_residual, zero_subspace)
|
||||
|
||||
|
||||
def test_regularized_projection_replay_matches_closed_form_decay():
|
||||
weight = torch.tensor([[4.0, 3.0]], dtype=torch.float64)
|
||||
direction = torch.tensor([1.0, 0.0], dtype=torch.float64)
|
||||
regularization = 0.25
|
||||
scale = 1.0 - regularization
|
||||
|
||||
first = project_weight_against_direction(weight, direction, regularization=regularization)
|
||||
second = project_weight_against_direction(first.weight, direction, regularization=regularization)
|
||||
|
||||
assert torch.allclose(first.weight[:, 0], weight[:, 0] * regularization)
|
||||
assert torch.allclose(second.weight[:, 0], weight[:, 0] * math.pow(regularization, 2))
|
||||
assert torch.allclose(first.weight, _reference_projection(weight, direction, scale))
|
||||
|
||||
|
||||
def test_abliteration_pipeline_math_wrappers_delegate_to_contract_helpers():
|
||||
subspace = torch.tensor([[1.0, 0.0], [1.0, 1.0]], dtype=torch.float64)
|
||||
harmless = torch.tensor([[-2.0, 0.0], [0.0, 0.0], [2.0, 0.0]], dtype=torch.float64)
|
||||
atoms = torch.tensor([[1.0, 0.0]], dtype=torch.float64)
|
||||
coeff = torch.tensor([[0.1], [3.0], [-2.0]], dtype=torch.float64)
|
||||
|
||||
assert torch.allclose(
|
||||
AbliterationPipeline._orthogonalize_subspace(subspace),
|
||||
orthogonalize_subspace_rows(subspace),
|
||||
)
|
||||
assert torch.allclose(
|
||||
AbliterationPipeline(None)._remove_harmless_principal_components(
|
||||
subspace,
|
||||
harmless,
|
||||
1,
|
||||
),
|
||||
remove_harmless_principal_components(subspace, harmless, 1),
|
||||
)
|
||||
assert torch.allclose(
|
||||
AbliterationPipeline(None)._residualize_against_shield_atoms(subspace, atoms, 1e-6),
|
||||
residualize_against_shield_atoms(subspace, atoms, 1e-6),
|
||||
)
|
||||
assert torch.equal(
|
||||
AbliterationPipeline._select_projection_coefficients(coeff, 0.5),
|
||||
select_projection_coefficients(coeff, 0.5),
|
||||
)
|
||||
|
||||
|
||||
def test_project_out_advanced_replaces_quantized_weight_after_successful_projection(monkeypatch):
|
||||
linear = torch.nn.Linear(2, 2, bias=False)
|
||||
with torch.no_grad():
|
||||
linear.weight.copy_(torch.tensor([[2.0, 0.0], [0.0, 2.0]]))
|
||||
module = SimpleNamespace(o_proj=linear)
|
||||
replacement_calls = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
AbliterationPipeline,
|
||||
"_dequantize_weight",
|
||||
staticmethod(lambda proj: (proj.weight.data, True)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
AbliterationPipeline,
|
||||
"_replace_quantized_weight",
|
||||
staticmethod(lambda proj, weight: replacement_calls.append((proj, weight.clone()))),
|
||||
)
|
||||
|
||||
count = AbliterationPipeline._project_out_advanced(
|
||||
module,
|
||||
torch.tensor([1.0, 0.0]),
|
||||
["o_proj"],
|
||||
)
|
||||
|
||||
assert count == 1
|
||||
assert len(replacement_calls) == 1
|
||||
assert replacement_calls[0][0] is linear
|
||||
assert torch.allclose(replacement_calls[0][1][:, 0], torch.zeros(2))
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -122,6 +123,115 @@ def test_custom_prompt_validation_padding_and_registry_access():
|
||||
assert prompts.get_valid_volumes("harmbench")[-1] == "all (use entire dataset)"
|
||||
|
||||
|
||||
def test_prompt_pairs_file_loader_accepts_exact_schema(tmp_path):
|
||||
path = tmp_path / "pairs.json"
|
||||
path.write_text(
|
||||
"""{
|
||||
"harmful": ["harm 0", "harm 1", "harm 2", "harm 3", "harm 4"],
|
||||
"harmless": ["safe 0", "safe 1", "safe 2", "safe 3", "safe 4"]
|
||||
}""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
harmful, harmless = prompts.load_prompt_pairs_file(path)
|
||||
|
||||
assert harmful == [f"harm {index}" for index in range(5)]
|
||||
assert harmless == [f"safe {index}" for index in range(5)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("payload", "message"),
|
||||
[
|
||||
("not json", "malformed JSON"),
|
||||
("[]", "JSON object"),
|
||||
('{"harmful": ["h"], "harmless": ["s"], "extra": []}', "exactly"),
|
||||
('{"harmful": "h", "harmless": ["s"]}', "must be an array"),
|
||||
(
|
||||
'{"harmful": ["h0", "h1", "h2", "h3", 4], '
|
||||
'"harmless": ["s0", "s1", "s2", "s3", "s4"]}',
|
||||
"must be a string",
|
||||
),
|
||||
('{"harmful": ["h"], "harmless": ["s"]}', "at least 5"),
|
||||
(
|
||||
'{"harmful": ["h0", "h1", "h2", "h3", "h4"], '
|
||||
'"harmless": ["s0", "s1", "s2", "s3"]}',
|
||||
"equal length",
|
||||
),
|
||||
(
|
||||
'{"harmful": ["h0", "h1", "h2", "h3", "h4"], '
|
||||
'"harmless": ["s0", "s1", "s2", "s3", ""]}',
|
||||
"nonblank strings",
|
||||
),
|
||||
(
|
||||
'{"harmful": ["h0", "h1", "h2", "h3", "h4"], '
|
||||
'"harmless": ["s0", "s1", "s2", "s3", "s\\u0000"]}',
|
||||
"NUL",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_prompt_pairs_file_loader_rejects_invalid_schema(tmp_path, payload, message):
|
||||
path = tmp_path / "pairs.json"
|
||||
path.write_text(payload, encoding="utf-8")
|
||||
|
||||
with pytest.raises(ValueError, match=message):
|
||||
prompts.load_prompt_pairs_file(path)
|
||||
|
||||
|
||||
def test_prompt_pairs_file_loader_rejects_missing_non_regular_oversized_and_utf8(tmp_path):
|
||||
with pytest.raises(ValueError, match="does not exist"):
|
||||
prompts.load_prompt_pairs_file(tmp_path / "missing.json")
|
||||
|
||||
with pytest.raises(ValueError, match="regular file"):
|
||||
prompts.load_prompt_pairs_file(tmp_path)
|
||||
|
||||
oversized = tmp_path / "oversized.json"
|
||||
oversized.write_bytes(b" " * (prompts.MAX_PROMPT_PAIRS_FILE_BYTES + 1))
|
||||
with pytest.raises(ValueError, match="too large"):
|
||||
prompts.load_prompt_pairs_file(oversized)
|
||||
|
||||
invalid_utf8 = tmp_path / "invalid.json"
|
||||
invalid_utf8.write_bytes(b"\xff")
|
||||
with pytest.raises(ValueError, match="UTF-8"):
|
||||
prompts.load_prompt_pairs_file(invalid_utf8)
|
||||
|
||||
|
||||
def test_prompt_pairs_file_loader_rejects_filesystem_errors_and_pair_limit(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
path = tmp_path / "pairs.json"
|
||||
path.write_text('{"harmful": [], "harmless": []}', encoding="utf-8")
|
||||
|
||||
def stat_error(_self):
|
||||
raise OSError("denied")
|
||||
|
||||
with monkeypatch.context() as m:
|
||||
m.setattr(type(path), "stat", stat_error)
|
||||
with pytest.raises(ValueError, match="Cannot access"):
|
||||
prompts.load_prompt_pairs_file(path)
|
||||
|
||||
def read_error(*_args, **_kwargs):
|
||||
raise OSError("denied")
|
||||
|
||||
with monkeypatch.context() as m:
|
||||
m.setattr(type(path), "read_text", read_error)
|
||||
with pytest.raises(ValueError, match="Cannot read"):
|
||||
prompts.load_prompt_pairs_file(path)
|
||||
|
||||
monkeypatch.setattr(prompts, "MAX_PROMPT_PAIRS", 4)
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"harmful": [f"harm {index}" for index in range(5)],
|
||||
"harmless": [f"safe {index}" for index in range(5)],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
with pytest.raises(ValueError, match="at most"):
|
||||
prompts.load_prompt_pairs_file(path)
|
||||
|
||||
|
||||
def test_harmless_generator_cycles_deterministically():
|
||||
count = len(prompts._HARMLESS_POOL) + 1
|
||||
generated = prompts._generate_harmless_counterparts(count)
|
||||
|
||||
@@ -3,9 +3,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from scripts import check_mutation_score
|
||||
from scripts import check_mutation_targets
|
||||
from scripts import run_repeat_gate
|
||||
|
||||
|
||||
@@ -17,6 +19,13 @@ def test_mutation_score_accepts_exact_floor_and_rejects_regression():
|
||||
]
|
||||
|
||||
|
||||
def test_mutation_score_validator_default_matches_immutable_policy_floor():
|
||||
parser = check_mutation_score._parser()
|
||||
|
||||
assert check_mutation_score.DEFAULT_MUTATION_SCORE_MINIMUM == 85.0
|
||||
assert parser.parse_args(["stats.json"]).minimum == 85.0
|
||||
|
||||
|
||||
def test_mutation_score_rejects_malformed_and_interrupted_runs():
|
||||
assert check_mutation_score.validate_mutation_stats(
|
||||
{"killed": True, "total": 1}, minimum=70,
|
||||
@@ -26,6 +35,74 @@ def test_mutation_score_rejects_malformed_and_interrupted_runs():
|
||||
) == ["mutation run was interrupted"]
|
||||
|
||||
|
||||
def test_mutation_target_guard_invalidates_stale_copied_targets(tmp_path):
|
||||
pyproject = tmp_path / "pyproject.toml"
|
||||
pyproject.write_text(
|
||||
"""
|
||||
[tool.mutmut]
|
||||
only_mutate = [
|
||||
"obliteratus/config.py",
|
||||
"obliteratus/analysis/whitened_svd.py",
|
||||
]
|
||||
required_mutation_targets = [
|
||||
"obliteratus/analysis/whitened_svd.py",
|
||||
]
|
||||
""".lstrip(),
|
||||
encoding="utf-8",
|
||||
)
|
||||
for target in (
|
||||
"obliteratus/config.py",
|
||||
"obliteratus/analysis/whitened_svd.py",
|
||||
):
|
||||
source = tmp_path / target
|
||||
source.parent.mkdir(parents=True, exist_ok=True)
|
||||
source.write_text("pass\n", encoding="utf-8")
|
||||
|
||||
valid_meta = tmp_path / "mutants/obliteratus/config.py.meta"
|
||||
valid_meta.parent.mkdir(parents=True, exist_ok=True)
|
||||
valid_meta.write_text('{"exit_code_by_key": {"obliteratus.config.x__mutmut_1": null}}')
|
||||
|
||||
stale_copy = tmp_path / "mutants/obliteratus/analysis/whitened_svd.py"
|
||||
stale_copy.parent.mkdir(parents=True, exist_ok=True)
|
||||
stale_copy.write_text("pass\n", encoding="utf-8")
|
||||
|
||||
stale = check_mutation_targets.prepare_required_targets(pyproject)
|
||||
|
||||
assert stale == [Path("obliteratus/analysis/whitened_svd.py")]
|
||||
assert not stale_copy.exists()
|
||||
assert check_mutation_targets.validate_required_targets(pyproject) == [
|
||||
"configured mutation target produced zero mutants: obliteratus/analysis/whitened_svd.py",
|
||||
]
|
||||
|
||||
|
||||
def test_mutation_target_guard_passes_when_each_exact_target_has_mutants(tmp_path):
|
||||
pyproject = tmp_path / "pyproject.toml"
|
||||
pyproject.write_text(
|
||||
"""
|
||||
[tool.mutmut]
|
||||
only_mutate = [
|
||||
"obliteratus/analysis/*.py",
|
||||
"obliteratus/analysis/numerical_contracts.py",
|
||||
]
|
||||
required_mutation_targets = [
|
||||
"obliteratus/analysis/numerical_contracts.py",
|
||||
]
|
||||
""".lstrip(),
|
||||
encoding="utf-8",
|
||||
)
|
||||
source = tmp_path / "obliteratus/analysis/numerical_contracts.py"
|
||||
source.parent.mkdir(parents=True, exist_ok=True)
|
||||
source.write_text("pass\n", encoding="utf-8")
|
||||
meta = tmp_path / "mutants/obliteratus/analysis/numerical_contracts.py.meta"
|
||||
meta.parent.mkdir(parents=True, exist_ok=True)
|
||||
meta.write_text(
|
||||
'{"exit_code_by_key": {"obliteratus.analysis.numerical_contracts.x__mutmut_1": null}}',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
assert check_mutation_targets.validate_required_targets(pyproject) == []
|
||||
|
||||
|
||||
def test_repeat_orders_are_distinct_and_deterministic():
|
||||
paths = ["a", "b", "c", "d"]
|
||||
assert run_repeat_gate.test_orders(paths) == [
|
||||
|
||||
@@ -9,21 +9,42 @@ from pathlib import Path
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from scripts import check_mutation_targets
|
||||
from scripts import prepare_mutation_coverage
|
||||
from scripts import run_prepared_mutmut
|
||||
from scripts import check_quality_policy as quality
|
||||
|
||||
|
||||
def test_mutation_campaign_preloads_native_modules_before_covered_line_discovery():
|
||||
def _workflow_step(name: str) -> dict:
|
||||
workflow = yaml.safe_load(Path(".github/workflows/ci.yml").read_text())
|
||||
steps = workflow["jobs"]["quality-depth"]["steps"]
|
||||
matches = [step for step in steps if step.get("name") == name]
|
||||
assert len(matches) == 1
|
||||
return matches[0]
|
||||
|
||||
|
||||
def test_mutation_campaign_uses_fork_safe_native_runtime_policy():
|
||||
pyproject = Path("pyproject.toml").read_text()
|
||||
mutmut_config = pyproject.split("[tool.mutmut]", maxsplit=1)[1].split(
|
||||
"\n[", maxsplit=1,
|
||||
)[0]
|
||||
workflow = Path(".github/workflows/ci.yml").read_text()
|
||||
mutation_step = _workflow_step("Run bounded selective mutation gate")
|
||||
mutation_run = mutation_step["run"]
|
||||
mutation_env = mutation_step["env"]
|
||||
quality_job = yaml.safe_load(Path(".github/workflows/ci.yml").read_text())["jobs"][
|
||||
"quality-depth"
|
||||
]
|
||||
|
||||
assert "mutate_only_covered_lines = true" in mutmut_config
|
||||
assert "timeout_constant = 2.0" in mutmut_config
|
||||
assert '"obliteratus/runtime_contracts.py"' in mutmut_config
|
||||
assert '"obliteratus/persistence_contracts.py"' in mutmut_config
|
||||
assert '"obliteratus/evaluation/lm_eval_integration.py"' in mutmut_config
|
||||
assert '"obliteratus/analysis/numerical_contracts.py"' in mutmut_config
|
||||
assert '"obliteratus/analysis/whitened_svd.py"' in mutmut_config
|
||||
assert "required_mutation_targets" in mutmut_config
|
||||
assert '"obliteratus/reporting/report.py"' not in mutmut_config
|
||||
assert '"tests/test_runtime_contracts.py"' in mutmut_config
|
||||
assert '"tests/test_persistence_contracts.py"' in mutmut_config
|
||||
@@ -38,7 +59,98 @@ def test_mutation_campaign_preloads_native_modules_before_covered_line_discovery
|
||||
assert '"tests/test_telemetry_failure_contracts.py"' in Path(
|
||||
"scripts/run_repeat_gate.py",
|
||||
).read_text()
|
||||
assert "import torch, yaml; from mutmut.__main__ import cli; cli()" in workflow
|
||||
assert mutation_env == {
|
||||
"BLIS_NUM_THREADS": "1",
|
||||
"MKL_NUM_THREADS": "1",
|
||||
"NUMEXPR_NUM_THREADS": "1",
|
||||
"OMP_THREAD_LIMIT": "1",
|
||||
"OMP_NUM_THREADS": "1",
|
||||
"OPENBLAS_NUM_THREADS": "1",
|
||||
"VECLIB_MAXIMUM_THREADS": "1",
|
||||
}
|
||||
assert quality_job["timeout-minutes"] == 45
|
||||
assert "scripts/check_mutation_targets.py prepare" in mutation_run
|
||||
assert "scripts/prepare_mutation_coverage.py prepare-coverage --max-children 8" in mutation_run
|
||||
assert "scripts/prepare_mutation_coverage.py prepare-stats --max-children 8" in mutation_run
|
||||
assert "scripts/run_prepared_mutmut.py run --max-children 8" in mutation_run
|
||||
assert "OBLITERATUS_MUTMUT_REUSE_COVERAGE=1" not in mutation_run
|
||||
assert "scripts/check_mutation_targets.py check" in mutation_run
|
||||
assert '"$QUALITY_ENV/bin/mutmut" run --max-children 8' not in mutation_run
|
||||
assert "/usr/bin/time" in mutation_run
|
||||
timed_block = mutation_run.split("/usr/bin/time", maxsplit=1)[1]
|
||||
assert timed_block.index("scripts/prepare_mutation_coverage.py prepare-coverage") < (
|
||||
timed_block.index("scripts/prepare_mutation_coverage.py prepare-stats")
|
||||
) < (
|
||||
timed_block.index("scripts/run_prepared_mutmut.py run --max-children 8")
|
||||
)
|
||||
assert "quality-evidence/mutation-time.txt" in timed_block
|
||||
assert "import torch, yaml; from mutmut.__main__ import cli; cli()" not in mutation_run
|
||||
|
||||
|
||||
def test_prepared_mutmut_runner_fails_closed_without_executable(monkeypatch, capsys):
|
||||
monkeypatch.setattr(sys, "argv", ["run_prepared_mutmut.py", "run"])
|
||||
monkeypatch.setattr(run_prepared_mutmut.shutil, "which", lambda _name: None)
|
||||
|
||||
assert run_prepared_mutmut.main() == 1
|
||||
assert "mutmut executable is not on PATH" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_prepared_mutmut_runner_execs_with_all_isolation_hooks(monkeypatch):
|
||||
captured: dict[str, object] = {}
|
||||
existing_pythonpath = "/existing/pythonpath"
|
||||
monkeypatch.setenv("PYTHONPATH", existing_pythonpath)
|
||||
for name in (
|
||||
"OBLITERATUS_MUTMUT_REUSE_COVERAGE",
|
||||
"OBLITERATUS_MUTMUT_REUSE_STATS",
|
||||
"OBLITERATUS_MUTMUT_SUBPROCESS_PREFLIGHT",
|
||||
):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
monkeypatch.setattr(sys, "argv", ["run_prepared_mutmut.py", "run", "--max-children", "4"])
|
||||
monkeypatch.setattr(run_prepared_mutmut.shutil, "which", lambda _name: "/venv/bin/mutmut")
|
||||
|
||||
def fake_execv(executable, command):
|
||||
captured["executable"] = executable
|
||||
captured["command"] = command
|
||||
captured["pythonpath"] = run_prepared_mutmut.os.environ["PYTHONPATH"]
|
||||
captured["reuse_coverage"] = run_prepared_mutmut.os.environ[
|
||||
"OBLITERATUS_MUTMUT_REUSE_COVERAGE"
|
||||
]
|
||||
captured["reuse_stats"] = run_prepared_mutmut.os.environ[
|
||||
"OBLITERATUS_MUTMUT_REUSE_STATS"
|
||||
]
|
||||
captured["subprocess_preflight"] = run_prepared_mutmut.os.environ[
|
||||
"OBLITERATUS_MUTMUT_SUBPROCESS_PREFLIGHT"
|
||||
]
|
||||
raise RuntimeError("exec intercepted")
|
||||
|
||||
monkeypatch.setattr(run_prepared_mutmut.os, "execv", fake_execv)
|
||||
|
||||
with pytest.raises(RuntimeError, match="exec intercepted"):
|
||||
run_prepared_mutmut.main()
|
||||
|
||||
assert captured == {
|
||||
"executable": "/venv/bin/mutmut",
|
||||
"command": ["/venv/bin/mutmut", "run", "--max-children", "4"],
|
||||
"pythonpath": run_prepared_mutmut.os.pathsep.join(
|
||||
[
|
||||
str(run_prepared_mutmut.SITECUSTOMIZE),
|
||||
str(run_prepared_mutmut.PROJECT_ROOT),
|
||||
existing_pythonpath,
|
||||
],
|
||||
),
|
||||
"reuse_coverage": "1",
|
||||
"reuse_stats": "1",
|
||||
"subprocess_preflight": "1",
|
||||
}
|
||||
|
||||
|
||||
def test_mutation_score_floor_is_immutable_across_policy_ci_and_validator():
|
||||
policy = json.loads(Path("ci/test-quality-policy.json").read_text(encoding="utf-8"))
|
||||
workflow = Path(".github/workflows/ci.yml").read_text(encoding="utf-8")
|
||||
|
||||
assert quality.BASELINE_FLOORS["mutation_score"] == 85.0
|
||||
assert policy["minimums"]["mutation_score"] == 85.0
|
||||
assert "--minimum 85.0" in workflow
|
||||
|
||||
|
||||
def _policy():
|
||||
@@ -111,6 +223,421 @@ def _coverage():
|
||||
}
|
||||
|
||||
|
||||
def _mutation_project(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
mutmut_table: str | None = None,
|
||||
source_paths: tuple[str, ...] = ("obliteratus/analysis/numerical_contracts.py",),
|
||||
meta_by_path: dict[str, str] | None = None,
|
||||
) -> Path:
|
||||
"""Create a deterministic mutation-target fixture under tmp_path."""
|
||||
pyproject = tmp_path / "pyproject.toml"
|
||||
pyproject.write_text(
|
||||
mutmut_table
|
||||
or """
|
||||
[tool.mutmut]
|
||||
required_mutation_targets = [
|
||||
"obliteratus/analysis/numerical_contracts.py",
|
||||
]
|
||||
""".lstrip(),
|
||||
encoding="utf-8",
|
||||
)
|
||||
for source_path in source_paths:
|
||||
source = tmp_path / source_path
|
||||
source.parent.mkdir(parents=True, exist_ok=True)
|
||||
source.write_text("def covered_target():\n return 1\n", encoding="utf-8")
|
||||
for target, metadata in (meta_by_path or {}).items():
|
||||
meta = tmp_path / "mutants" / f"{target}.meta"
|
||||
meta.parent.mkdir(parents=True, exist_ok=True)
|
||||
meta.write_text(metadata, encoding="utf-8")
|
||||
return pyproject
|
||||
|
||||
|
||||
def test_mutation_target_guard_uses_required_exact_python_targets(tmp_path):
|
||||
pyproject = _mutation_project(
|
||||
tmp_path,
|
||||
mutmut_table="""
|
||||
[tool.mutmut]
|
||||
only_mutate = [
|
||||
"obliteratus/config.py",
|
||||
"obliteratus/analysis/*.py",
|
||||
]
|
||||
required_mutation_targets = [
|
||||
"obliteratus/analysis/numerical_contracts.py",
|
||||
]
|
||||
""".lstrip(),
|
||||
)
|
||||
|
||||
assert check_mutation_targets.exact_python_targets(
|
||||
check_mutation_targets.load_mutmut_config(pyproject),
|
||||
) == [Path("obliteratus/analysis/numerical_contracts.py")]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bad_target",
|
||||
[
|
||||
"/tmp/escape.py",
|
||||
"../escape.py",
|
||||
"obliteratus/../escape.py",
|
||||
"obliteratus/analysis/*.py",
|
||||
"README.md",
|
||||
],
|
||||
)
|
||||
def test_mutation_target_guard_rejects_required_target_escapes_and_non_exact_paths(
|
||||
tmp_path, bad_target,
|
||||
):
|
||||
pyproject = _mutation_project(
|
||||
tmp_path,
|
||||
mutmut_table=f"""
|
||||
[tool.mutmut]
|
||||
required_mutation_targets = [
|
||||
{bad_target!r},
|
||||
]
|
||||
""".lstrip(),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="invalid required mutation target"):
|
||||
check_mutation_targets.exact_python_targets(
|
||||
check_mutation_targets.load_mutmut_config(pyproject),
|
||||
)
|
||||
|
||||
|
||||
def test_mutation_target_guard_rejects_malformed_config_and_metadata(tmp_path):
|
||||
pyproject = _mutation_project(
|
||||
tmp_path,
|
||||
mutmut_table='[tool]\nmutmut = "not-a-table"\n',
|
||||
)
|
||||
with pytest.raises(ValueError, match=r"\[tool\.mutmut\] must be a table"):
|
||||
check_mutation_targets.load_mutmut_config(pyproject)
|
||||
|
||||
with pytest.raises(ValueError, match="list of strings"):
|
||||
check_mutation_targets.exact_python_targets({"required_mutation_targets": ["ok.py", 3]})
|
||||
|
||||
malformed = tmp_path / "mutants/bad.py.meta"
|
||||
malformed.parent.mkdir(parents=True)
|
||||
malformed.write_text("{", encoding="utf-8")
|
||||
with pytest.raises(ValueError, match="is not valid JSON"):
|
||||
check_mutation_targets.mutant_count(malformed)
|
||||
|
||||
missing_key = tmp_path / "mutants/missing.py.meta"
|
||||
missing_key.write_text('{"mutants": []}', encoding="utf-8")
|
||||
with pytest.raises(ValueError, match="exit_code_by_key object"):
|
||||
check_mutation_targets.mutant_count(missing_key)
|
||||
|
||||
|
||||
def test_mutation_target_guard_rejects_missing_configured_source(tmp_path):
|
||||
pyproject = _mutation_project(tmp_path, source_paths=())
|
||||
|
||||
with pytest.raises(ValueError, match="configured mutation target does not exist"):
|
||||
check_mutation_targets.stale_or_empty_targets(
|
||||
[Path("obliteratus/analysis/numerical_contracts.py")],
|
||||
project_root=pyproject.parent,
|
||||
)
|
||||
|
||||
|
||||
def test_mutation_target_guard_removes_all_stale_artifact_kinds(tmp_path):
|
||||
target = "obliteratus/analysis/numerical_contracts.py"
|
||||
pyproject = _mutation_project(
|
||||
tmp_path,
|
||||
source_paths=(target,),
|
||||
meta_by_path={target: '{"exit_code_by_key": {}}'},
|
||||
)
|
||||
artifact_base = tmp_path / "mutants" / target
|
||||
artifact_base.parent.mkdir(parents=True, exist_ok=True)
|
||||
for suffix in ("", ".spans"):
|
||||
(tmp_path / "mutants" / f"{target}{suffix}").write_text("stale", encoding="utf-8")
|
||||
|
||||
assert check_mutation_targets.prepare_required_targets(pyproject) == [Path(target)]
|
||||
assert not artifact_base.exists()
|
||||
assert not artifact_base.with_suffix(".py.meta").exists()
|
||||
assert not artifact_base.with_suffix(".py.spans").exists()
|
||||
|
||||
|
||||
def test_mutation_target_guard_never_unlinks_escaped_symlink_artifacts(tmp_path):
|
||||
target = "obliteratus/analysis/numerical_contracts.py"
|
||||
pyproject = _mutation_project(
|
||||
tmp_path,
|
||||
source_paths=(target,),
|
||||
meta_by_path={target: '{"exit_code_by_key": {}}'},
|
||||
)
|
||||
outside = tmp_path / "outside.py"
|
||||
outside.write_text("do not remove\n", encoding="utf-8")
|
||||
artifact = tmp_path / "mutants" / target
|
||||
artifact.parent.mkdir(parents=True, exist_ok=True)
|
||||
artifact.symlink_to(outside)
|
||||
|
||||
with pytest.raises(ValueError, match="escapes project-owned mutants"):
|
||||
check_mutation_targets.prepare_required_targets(pyproject)
|
||||
assert outside.exists()
|
||||
assert artifact.is_symlink()
|
||||
|
||||
|
||||
def test_mutation_target_guard_rejects_symlinked_mutants_root_without_deleting_outside(
|
||||
tmp_path,
|
||||
):
|
||||
target = "obliteratus/analysis/numerical_contracts.py"
|
||||
outside = tmp_path / "outside-mutants"
|
||||
outside.mkdir()
|
||||
sentinel = outside / "sentinel.txt"
|
||||
sentinel.write_text("do not delete\n", encoding="utf-8")
|
||||
(tmp_path / "mutants").symlink_to(outside, target_is_directory=True)
|
||||
pyproject = _mutation_project(
|
||||
tmp_path,
|
||||
source_paths=(target,),
|
||||
meta_by_path={},
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="literal project-owned mutants directory"):
|
||||
check_mutation_targets.prepare_required_targets(pyproject)
|
||||
with pytest.raises(ValueError, match="literal project-owned mutants directory"):
|
||||
check_mutation_targets.validate_required_targets(pyproject)
|
||||
assert sentinel.read_text(encoding="utf-8") == "do not delete\n"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mutants_dir", [Path("/tmp/mutants"), Path("../mutants"), Path("mutants-copy")])
|
||||
def test_mutation_target_guard_rejects_unowned_mutants_dir(tmp_path, mutants_dir):
|
||||
pyproject = _mutation_project(tmp_path)
|
||||
|
||||
with pytest.raises(ValueError, match="mutants directory must be project-owned"):
|
||||
check_mutation_targets.prepare_required_targets(pyproject, mutants_dir=mutants_dir)
|
||||
|
||||
|
||||
def test_mutation_target_guard_cli_prepare_check_and_failure_paths(tmp_path, monkeypatch, capsys):
|
||||
target = "obliteratus/analysis/numerical_contracts.py"
|
||||
pyproject = _mutation_project(
|
||||
tmp_path,
|
||||
source_paths=(target,),
|
||||
meta_by_path={target: '{"exit_code_by_key": {"target__mutmut_1": null}}'},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
["check_mutation_targets.py", "prepare", "--pyproject", str(pyproject)],
|
||||
)
|
||||
assert check_mutation_targets.main() == 0
|
||||
assert "found no stale required targets" in capsys.readouterr().out
|
||||
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
["check_mutation_targets.py", "check", "--pyproject", str(pyproject)],
|
||||
)
|
||||
assert check_mutation_targets.main() == 0
|
||||
assert "mutation target guard passed" in capsys.readouterr().out
|
||||
|
||||
(tmp_path / "mutants" / f"{target}.meta").unlink()
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
["check_mutation_targets.py", "check", "--pyproject", str(pyproject)],
|
||||
)
|
||||
assert check_mutation_targets.main() == 1
|
||||
assert "produced zero mutants" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_mutation_target_guard_rejects_no_test_required_mutants(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
check_mutation_targets,
|
||||
"_installed_mutmut_version",
|
||||
lambda: check_mutation_targets.SUPPORTED_MUTMUT_VERSION,
|
||||
)
|
||||
target = "obliteratus/analysis/numerical_contracts.py"
|
||||
pyproject = _mutation_project(
|
||||
tmp_path,
|
||||
source_paths=(target,),
|
||||
meta_by_path={
|
||||
target: '{"exit_code_by_key": {"target__mutmut_1": 5, "target__mutmut_2": 33}}',
|
||||
},
|
||||
)
|
||||
|
||||
assert check_mutation_targets.validate_required_targets(pyproject) == [
|
||||
"configured mutation target has 2 mutant(s) with no tests: "
|
||||
"obliteratus/analysis/numerical_contracts.py",
|
||||
]
|
||||
assert check_mutation_targets.MUTMUT_NO_TEST_EXIT_CODES == frozenset({5, 33})
|
||||
assert check_mutation_targets.SUPPORTED_MUTMUT_VERSION == "3.7.0"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version", [None, "3.6.0", "3.8.0"])
|
||||
def test_mutation_target_guard_fails_closed_before_no_test_code_interpretation(
|
||||
tmp_path, monkeypatch, version,
|
||||
):
|
||||
target = "obliteratus/analysis/numerical_contracts.py"
|
||||
pyproject = _mutation_project(
|
||||
tmp_path,
|
||||
source_paths=(target,),
|
||||
meta_by_path={target: '{"exit_code_by_key": {"target__mutmut_1": 33}}'},
|
||||
)
|
||||
monkeypatch.setattr(check_mutation_targets, "_installed_mutmut_version", lambda: version)
|
||||
|
||||
with pytest.raises(ValueError, match="unsupported mutmut version"):
|
||||
check_mutation_targets.validate_required_targets(pyproject)
|
||||
|
||||
|
||||
def test_mutation_target_guard_cli_reports_validation_errors(tmp_path, monkeypatch, capsys):
|
||||
pyproject = _mutation_project(tmp_path, source_paths=())
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
["check_mutation_targets.py", "prepare", "--pyproject", str(pyproject)],
|
||||
)
|
||||
|
||||
assert check_mutation_targets.main() == 1
|
||||
assert "configured mutation target does not exist" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_mutation_coverage_manifest_rejects_stale_source_hash(tmp_path, monkeypatch):
|
||||
source = tmp_path / "pkg/example.py"
|
||||
mutant = tmp_path / "mutants/pkg/example.py"
|
||||
source.parent.mkdir(parents=True)
|
||||
mutant.parent.mkdir(parents=True)
|
||||
source.write_text("def f():\n return 1\n", encoding="utf-8")
|
||||
mutant.write_text("mutant", encoding="utf-8")
|
||||
|
||||
manifest = {
|
||||
"version": 1,
|
||||
"mutmut_version": prepare_mutation_coverage.SUPPORTED_MUTMUT_VERSION,
|
||||
"mutate_only_covered_lines": True,
|
||||
"source_paths": ["pkg/"],
|
||||
"only_mutate": ["pkg/example.py"],
|
||||
"pytest_add_cli_args": ["--no-cov"],
|
||||
"pytest_add_cli_args_test_selection": ["tests/test_example.py"],
|
||||
"required_mutation_targets": ["pkg/example.py"],
|
||||
"mutatable_paths": ["pkg/example.py"],
|
||||
"source_hashes": {"pkg/example.py": "stale"},
|
||||
"selected_test_hashes": {"tests/test_example.py": "ok"},
|
||||
"hook_hashes": {
|
||||
"scripts/prepare_mutation_coverage.py": "ok",
|
||||
"scripts/mutmut_coverage_sitecustomize/sitecustomize.py": "ok",
|
||||
},
|
||||
}
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
(tmp_path / "mutants/.covered-lines-prepass.json").write_text(
|
||||
json.dumps(manifest),
|
||||
encoding="utf-8",
|
||||
)
|
||||
source.write_text("def f():\n return 2\n", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(prepare_mutation_coverage, "assert_supported_mutmut", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
prepare_mutation_coverage,
|
||||
"configured_paths",
|
||||
lambda: ([Path("pkg/example.py")], [Path("pkg/example.py")]),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
prepare_mutation_coverage,
|
||||
"validate_prepared_artifacts",
|
||||
lambda *, mutatable, required: {
|
||||
**manifest,
|
||||
"source_hashes": {
|
||||
"pkg/example.py": prepare_mutation_coverage._sha256(source),
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="manifest is stale for source_hashes"):
|
||||
prepare_mutation_coverage.validate_manifest()
|
||||
|
||||
|
||||
def test_mutation_coverage_prepass_rejects_escaped_cleanup_artifacts(tmp_path, monkeypatch):
|
||||
outside = tmp_path / "outside.py"
|
||||
outside.write_text("do not remove\n", encoding="utf-8")
|
||||
artifact = tmp_path / "mutants/pkg/example.py"
|
||||
artifact.parent.mkdir(parents=True, exist_ok=True)
|
||||
artifact.symlink_to(outside)
|
||||
monkeypatch.chdir(tmp_path)
|
||||
|
||||
with pytest.raises(RuntimeError, match="mutation artifact escapes project-owned mutants"):
|
||||
prepare_mutation_coverage.remove_mutation_artifacts([Path("pkg/example.py")])
|
||||
assert outside.exists()
|
||||
assert artifact.is_symlink()
|
||||
|
||||
|
||||
def test_mutation_coverage_manifest_rejects_stale_selected_test_hash(tmp_path, monkeypatch):
|
||||
source = tmp_path / "pkg/example.py"
|
||||
test_file = tmp_path / "tests/test_example.py"
|
||||
hook = tmp_path / "scripts/mutmut_coverage_sitecustomize/sitecustomize.py"
|
||||
prepass = tmp_path / "scripts/prepare_mutation_coverage.py"
|
||||
mutant = tmp_path / "mutants/pkg/example.py"
|
||||
for path in (source, test_file, hook, prepass, mutant):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("original\n", encoding="utf-8")
|
||||
manifest = {
|
||||
"version": 1,
|
||||
"mutmut_version": prepare_mutation_coverage.SUPPORTED_MUTMUT_VERSION,
|
||||
"mutate_only_covered_lines": True,
|
||||
"source_paths": ["pkg/"],
|
||||
"only_mutate": ["pkg/example.py"],
|
||||
"pytest_add_cli_args": ["--no-cov"],
|
||||
"pytest_add_cli_args_test_selection": ["tests/test_example.py"],
|
||||
"required_mutation_targets": ["pkg/example.py"],
|
||||
"mutatable_paths": ["pkg/example.py"],
|
||||
"source_hashes": {"pkg/example.py": prepare_mutation_coverage._sha256(source)},
|
||||
"selected_test_hashes": {"tests/test_example.py": "stale"},
|
||||
"hook_hashes": {
|
||||
"scripts/prepare_mutation_coverage.py": prepare_mutation_coverage._sha256(prepass),
|
||||
"scripts/mutmut_coverage_sitecustomize/sitecustomize.py": (
|
||||
prepare_mutation_coverage._sha256(hook)
|
||||
),
|
||||
},
|
||||
}
|
||||
monkeypatch.chdir(tmp_path)
|
||||
(tmp_path / "mutants/.covered-lines-prepass.json").write_text(
|
||||
json.dumps(manifest),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(prepare_mutation_coverage, "assert_supported_mutmut", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
prepare_mutation_coverage,
|
||||
"configured_paths",
|
||||
lambda: ([Path("pkg/example.py")], [Path("pkg/example.py")]),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
prepare_mutation_coverage,
|
||||
"validate_prepared_artifacts",
|
||||
lambda *, mutatable, required: {
|
||||
**manifest,
|
||||
"selected_test_hashes": {
|
||||
"tests/test_example.py": prepare_mutation_coverage._sha256(test_file),
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="manifest is stale for selected_test_hashes"):
|
||||
prepare_mutation_coverage.validate_manifest()
|
||||
|
||||
|
||||
def test_mutation_execution_manifest_rejects_stale_stats_hash(tmp_path, monkeypatch):
|
||||
stats = tmp_path / "mutants/mutmut-stats.json"
|
||||
stats.parent.mkdir(parents=True)
|
||||
stats.write_text(
|
||||
json.dumps({
|
||||
"tests_by_mangled_function_name": {"pkg.x_f": ["tests/test_example.py::test_f"]},
|
||||
"duration_by_test": {"tests/test_example.py::test_f": 0.01},
|
||||
"stats_time": 0.1,
|
||||
"function_hashes": {"pkg.x_f": "abc"},
|
||||
"function_dependencies": {},
|
||||
"config_fingerprint": {},
|
||||
"watched_file_hashes": {},
|
||||
"git_commit": None,
|
||||
}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
manifest = {"stats_hash": "stale"}
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setattr(
|
||||
prepare_mutation_coverage,
|
||||
"validate_coverage_manifest",
|
||||
lambda: manifest,
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="execution manifest is stale for stats_hash"):
|
||||
prepare_mutation_coverage.validate_execution_manifest()
|
||||
|
||||
|
||||
def test_policy_and_exact_mature_floors_pass():
|
||||
policy = _policy()
|
||||
assert quality.validate_policy(policy) == []
|
||||
@@ -122,13 +649,13 @@ def test_policy_and_exact_mature_floors_pass():
|
||||
|
||||
def test_floor_regression_requires_structured_reviewed_exception():
|
||||
policy = _policy()
|
||||
policy["minimums"]["mutation_score"] = 74
|
||||
policy["minimums"]["mutation_score"] = 84
|
||||
assert quality.validate_policy(policy) == [
|
||||
"quality minimum mutation_score cannot move below 75 without an explicit reviewed exception",
|
||||
"quality minimum mutation_score cannot move below 85 without an explicit reviewed exception",
|
||||
]
|
||||
policy["threshold_exceptions"] = [{
|
||||
"threshold": "mutation_score",
|
||||
"new_value": 74,
|
||||
"new_value": 84,
|
||||
"reason": "Temporary tool regression",
|
||||
"approved_issue": "https://github.com/elder-plinius/OBLITERATUS/issues/999",
|
||||
"expires": "2026-09-01",
|
||||
|
||||
@@ -0,0 +1,372 @@
|
||||
"""Reference-oracle tests for whitened SVD extraction.
|
||||
|
||||
## Test Context
|
||||
|
||||
- Code to test: `obliteratus/analysis/whitened_svd.py`
|
||||
- Testing framework: pytest with torch CPU tensors
|
||||
- Coverage target: repository default, minimum 80% Gate 3 target
|
||||
- Test types needed: deterministic unit and metamorphic tests
|
||||
- External dependencies to mock: none; these tests intentionally exercise the
|
||||
real CPU tensor math and production `WhitenedSVDExtractor`
|
||||
- Edge cases identified: paired sample permutation, feature-coordinate
|
||||
permutation, common translation, SVD sign ambiguity, dtype normalization,
|
||||
singular harmless covariance, zero signal, non-finite input, deterministic
|
||||
replay, over-requested directions, and `extract_all_layers` intersection order
|
||||
|
||||
The static fixtures below are tiny deterministic activation matrices. Dynamic
|
||||
fixture factories clone rows into the public list-of-tensors input shape so
|
||||
tests cannot pass by mutating shared tensors across cases.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from obliteratus.analysis.whitened_svd import WhitenedSVDExtractor
|
||||
|
||||
|
||||
REGULARIZATION_EPS = 1e-4
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReferenceWhitenedSVD:
|
||||
directions: torch.Tensor
|
||||
whitened_directions: torch.Tensor
|
||||
singular_values: torch.Tensor
|
||||
variance_explained: float
|
||||
|
||||
|
||||
def _activation_pair_fixture(
|
||||
dtype: torch.dtype = torch.float32,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
harmless = torch.tensor(
|
||||
[
|
||||
[-2.0, -1.0, 0.5, 1.0],
|
||||
[-1.0, 0.0, -0.5, -1.0],
|
||||
[0.0, 1.0, 1.5, 0.0],
|
||||
[1.0, -2.0, -1.5, 2.0],
|
||||
[2.0, 2.0, 0.0, -2.0],
|
||||
],
|
||||
dtype=dtype,
|
||||
)
|
||||
paired_delta = torch.tensor(
|
||||
[
|
||||
[1.25, -0.50, 0.75, 0.10],
|
||||
[0.80, 0.25, -0.10, 0.40],
|
||||
[1.60, -0.75, 0.35, -0.20],
|
||||
[0.45, 0.90, -0.60, 0.55],
|
||||
[1.10, -0.20, 0.95, -0.35],
|
||||
],
|
||||
dtype=dtype,
|
||||
)
|
||||
return harmless + paired_delta, harmless
|
||||
|
||||
|
||||
def _singular_covariance_fixture() -> tuple[torch.Tensor, torch.Tensor]:
|
||||
harmless = torch.tensor(
|
||||
[
|
||||
[-2.0, 0.0, 0.0, 1.0],
|
||||
[-1.0, 0.0, 0.0, 1.0],
|
||||
[0.0, 0.0, 0.0, 1.0],
|
||||
[1.0, 0.0, 0.0, 1.0],
|
||||
[2.0, 0.0, 0.0, 1.0],
|
||||
],
|
||||
)
|
||||
harmful = harmless + torch.tensor([1.5, 0.0, 0.0, 0.0])
|
||||
return harmful, harmless
|
||||
|
||||
|
||||
def _as_public_samples(matrix: torch.Tensor) -> list[torch.Tensor]:
|
||||
return [row.clone() for row in matrix]
|
||||
|
||||
|
||||
def _reference_whitened_svd(
|
||||
harmful: torch.Tensor,
|
||||
harmless: torch.Tensor,
|
||||
*,
|
||||
n_directions: int,
|
||||
min_variance_ratio: float = 0.0,
|
||||
) -> ReferenceWhitenedSVD:
|
||||
"""Independent tiny-tensor reference for the published mathematical contract."""
|
||||
harmful = harmful.to(torch.float32).to(torch.float64)
|
||||
harmless = harmless.to(torch.float32).to(torch.float64)
|
||||
baseline_mean = harmless.mean(dim=0, keepdim=True)
|
||||
centered_baseline = harmless - baseline_mean
|
||||
covariance = centered_baseline.T.mm(centered_baseline) / max(harmless.shape[0] - 1, 1)
|
||||
eigenvalues, eigenvectors = torch.linalg.eigh(covariance)
|
||||
eigenvalues = torch.clamp(eigenvalues, min=0.0)
|
||||
threshold = eigenvalues.max() * min_variance_ratio
|
||||
kept = eigenvalues >= threshold
|
||||
kept_values = eigenvalues[kept]
|
||||
kept_vectors = eigenvectors[:, kept]
|
||||
|
||||
whitening = kept_vectors.mm(torch.diag(torch.rsqrt(kept_values + REGULARIZATION_EPS)))
|
||||
whitened_delta = (harmful - baseline_mean).mm(whitening) - centered_baseline.mm(whitening)
|
||||
_, singular_values, right_vectors_t = torch.linalg.svd(whitened_delta, full_matrices=False)
|
||||
count = min(n_directions, whitened_delta.shape[0], whitened_delta.shape[1])
|
||||
whitened_directions = right_vectors_t[:count]
|
||||
|
||||
inverse_whitening = kept_vectors.mm(torch.diag(torch.sqrt(kept_values + REGULARIZATION_EPS)))
|
||||
original_directions = whitened_directions.mm(inverse_whitening.T)
|
||||
original_directions = torch.nn.functional.normalize(original_directions, dim=1)
|
||||
whitened_directions = torch.nn.functional.normalize(whitened_directions, dim=1)
|
||||
selected_singular_values = singular_values[:count]
|
||||
variance_explained = (
|
||||
selected_singular_values.square().sum() / singular_values.square().sum().clamp(min=1e-12)
|
||||
).item()
|
||||
return ReferenceWhitenedSVD(
|
||||
directions=original_directions,
|
||||
whitened_directions=whitened_directions,
|
||||
singular_values=selected_singular_values,
|
||||
variance_explained=variance_explained,
|
||||
)
|
||||
|
||||
|
||||
def _row_space_projector(rows: torch.Tensor, *, tolerance: float = 1e-9) -> torch.Tensor:
|
||||
rows = rows.to(torch.float64)
|
||||
_, singular_values, right_vectors_t = torch.linalg.svd(rows, full_matrices=False)
|
||||
basis = right_vectors_t[singular_values > tolerance]
|
||||
return basis.T.mm(basis)
|
||||
|
||||
|
||||
def _assert_same_subspace(left: torch.Tensor, right: torch.Tensor, *, atol: float = 3e-5) -> None:
|
||||
assert torch.allclose(
|
||||
_row_space_projector(left),
|
||||
_row_space_projector(right),
|
||||
atol=atol,
|
||||
rtol=0,
|
||||
)
|
||||
|
||||
|
||||
def test_matches_independent_reference_not_raw_svd_or_identity_whitening() -> None:
|
||||
harmful, harmless = _activation_pair_fixture()
|
||||
result = WhitenedSVDExtractor(
|
||||
regularization_eps=REGULARIZATION_EPS,
|
||||
min_variance_ratio=0.0,
|
||||
).extract(_as_public_samples(harmful), _as_public_samples(harmless), n_directions=3)
|
||||
reference = _reference_whitened_svd(harmful, harmless, n_directions=3)
|
||||
|
||||
_assert_same_subspace(result.directions, reference.directions)
|
||||
_assert_same_subspace(result.whitened_directions, reference.whitened_directions)
|
||||
assert result.singular_values.double() == pytest.approx(
|
||||
reference.singular_values,
|
||||
rel=3e-5,
|
||||
abs=3e-5,
|
||||
)
|
||||
assert result.variance_explained == pytest.approx(reference.variance_explained, abs=3e-6)
|
||||
|
||||
raw_delta = harmful - harmless
|
||||
_, _, raw_right_vectors_t = torch.linalg.svd(raw_delta, full_matrices=False)
|
||||
raw_primary_alignment = torch.dot(result.directions[0], raw_right_vectors_t[0]).abs()
|
||||
assert raw_primary_alignment < 0.95
|
||||
|
||||
|
||||
def test_joint_sample_permutation_preserves_sign_invariant_refusal_subspace() -> None:
|
||||
harmful, harmless = _activation_pair_fixture()
|
||||
permutation = torch.tensor([3, 0, 4, 1, 2])
|
||||
extractor = WhitenedSVDExtractor(
|
||||
regularization_eps=REGULARIZATION_EPS,
|
||||
min_variance_ratio=0.0,
|
||||
)
|
||||
|
||||
original = extractor.extract(_as_public_samples(harmful), _as_public_samples(harmless), 3)
|
||||
permuted = extractor.extract(
|
||||
_as_public_samples(harmful[permutation]),
|
||||
_as_public_samples(harmless[permutation]),
|
||||
3,
|
||||
)
|
||||
|
||||
_assert_same_subspace(original.directions, permuted.directions)
|
||||
assert permuted.singular_values == pytest.approx(original.singular_values, rel=3e-5, abs=3e-5)
|
||||
assert permuted.variance_explained == pytest.approx(original.variance_explained, abs=3e-6)
|
||||
|
||||
|
||||
def test_feature_coordinate_permutation_round_trips_through_inverse_mapping() -> None:
|
||||
harmful, harmless = _activation_pair_fixture()
|
||||
feature_permutation = torch.tensor([2, 0, 3, 1])
|
||||
extractor = WhitenedSVDExtractor(
|
||||
regularization_eps=REGULARIZATION_EPS,
|
||||
min_variance_ratio=0.0,
|
||||
)
|
||||
|
||||
original = extractor.extract(_as_public_samples(harmful), _as_public_samples(harmless), 3)
|
||||
permuted = extractor.extract(
|
||||
_as_public_samples(harmful[:, feature_permutation]),
|
||||
_as_public_samples(harmless[:, feature_permutation]),
|
||||
3,
|
||||
)
|
||||
mapped_back = torch.empty_like(permuted.directions)
|
||||
mapped_back[:, feature_permutation] = permuted.directions
|
||||
|
||||
_assert_same_subspace(original.directions, mapped_back)
|
||||
assert permuted.singular_values == pytest.approx(original.singular_values, rel=5e-5, abs=5e-5)
|
||||
|
||||
|
||||
def test_common_translation_cannot_change_whitened_svd_oracle_values() -> None:
|
||||
harmful, harmless = _activation_pair_fixture()
|
||||
offset = torch.tensor([8.0, -3.0, 0.25, 11.0])
|
||||
extractor = WhitenedSVDExtractor(
|
||||
regularization_eps=REGULARIZATION_EPS,
|
||||
min_variance_ratio=0.0,
|
||||
)
|
||||
|
||||
original = extractor.extract(_as_public_samples(harmful), _as_public_samples(harmless), 3)
|
||||
translated = extractor.extract(
|
||||
_as_public_samples(harmful + offset),
|
||||
_as_public_samples(harmless + offset),
|
||||
3,
|
||||
)
|
||||
|
||||
_assert_same_subspace(original.directions, translated.directions)
|
||||
_assert_same_subspace(original.whitened_directions, translated.whitened_directions)
|
||||
assert translated.singular_values == pytest.approx(original.singular_values, rel=3e-5, abs=3e-5)
|
||||
assert translated.variance_explained == pytest.approx(original.variance_explained, abs=3e-6)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.float64])
|
||||
def test_float_inputs_follow_float32_output_policy_with_dtype_tolerances(
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
harmful, harmless = _activation_pair_fixture(dtype)
|
||||
result = WhitenedSVDExtractor(
|
||||
regularization_eps=REGULARIZATION_EPS,
|
||||
min_variance_ratio=0.0,
|
||||
).extract(_as_public_samples(harmful), _as_public_samples(harmless), 3)
|
||||
reference = _reference_whitened_svd(harmful, harmless, n_directions=3)
|
||||
|
||||
assert result.directions.dtype is torch.float32
|
||||
assert result.whitened_directions.dtype is torch.float32
|
||||
assert result.singular_values.dtype is torch.float32
|
||||
_assert_same_subspace(result.directions, reference.directions, atol=4e-5)
|
||||
assert result.singular_values.double() == pytest.approx(
|
||||
reference.singular_values,
|
||||
rel=4e-5,
|
||||
abs=4e-5,
|
||||
)
|
||||
|
||||
|
||||
def test_singular_covariance_limits_over_requested_directions_to_effective_rank() -> None:
|
||||
harmful, harmless = _singular_covariance_fixture()
|
||||
|
||||
result = WhitenedSVDExtractor(
|
||||
regularization_eps=REGULARIZATION_EPS,
|
||||
min_variance_ratio=0.01,
|
||||
).extract(_as_public_samples(harmful), _as_public_samples(harmless), n_directions=5)
|
||||
|
||||
assert result.directions.shape == (1, 4)
|
||||
assert result.whitened_directions.shape == (1, 1)
|
||||
assert result.singular_values.shape == (1,)
|
||||
assert torch.count_nonzero(result.directions[0, 1:].abs() > 1e-6) == 0
|
||||
assert result.directions.norm() == pytest.approx(1.0)
|
||||
assert result.variance_explained == pytest.approx(1.0)
|
||||
assert result.effective_rank == pytest.approx(1.0, abs=1e-6)
|
||||
|
||||
|
||||
def test_identical_harmful_and_harmless_inputs_are_rejected_as_no_refusal_signal() -> None:
|
||||
_, harmless = _activation_pair_fixture()
|
||||
extractor = WhitenedSVDExtractor(
|
||||
regularization_eps=REGULARIZATION_EPS,
|
||||
min_variance_ratio=0.0,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="without activation difference"):
|
||||
extractor.extract(_as_public_samples(harmless), _as_public_samples(harmless), 2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("poisoned_side", "poisoned_value", "message"),
|
||||
[
|
||||
("harmful", float("nan"), "finite"),
|
||||
("harmless", float("inf"), "finite"),
|
||||
],
|
||||
)
|
||||
def test_non_finite_activation_values_are_rejected_before_linear_algebra(
|
||||
poisoned_side: str,
|
||||
poisoned_value: float,
|
||||
message: str,
|
||||
) -> None:
|
||||
harmful, harmless = _activation_pair_fixture()
|
||||
target = harmful if poisoned_side == "harmful" else harmless
|
||||
target[1, 2] = poisoned_value
|
||||
|
||||
with pytest.raises(ValueError, match=message):
|
||||
WhitenedSVDExtractor().extract(_as_public_samples(harmful), _as_public_samples(harmless), 1)
|
||||
|
||||
|
||||
def test_deterministic_replay_returns_byte_stable_cpu_outputs() -> None:
|
||||
harmful, harmless = _activation_pair_fixture()
|
||||
extractor = WhitenedSVDExtractor(
|
||||
regularization_eps=REGULARIZATION_EPS,
|
||||
min_variance_ratio=0.0,
|
||||
)
|
||||
|
||||
first = extractor.extract(_as_public_samples(harmful), _as_public_samples(harmless), 3)
|
||||
second = extractor.extract(_as_public_samples(harmful), _as_public_samples(harmless), 3)
|
||||
|
||||
assert torch.equal(first.directions, second.directions)
|
||||
assert torch.equal(first.whitened_directions, second.whitened_directions)
|
||||
assert torch.equal(first.singular_values, second.singular_values)
|
||||
assert first.variance_explained == second.variance_explained
|
||||
assert first.condition_number == second.condition_number
|
||||
assert first.effective_rank == second.effective_rank
|
||||
|
||||
|
||||
def test_extract_all_layers_returns_sorted_harmful_harmless_intersection_only() -> None:
|
||||
base_harmful, base_harmless = _activation_pair_fixture()
|
||||
harmful_by_layer = {
|
||||
8: _as_public_samples(base_harmful + 0.25),
|
||||
2: _as_public_samples(base_harmful),
|
||||
5: _as_public_samples(base_harmful * 1.5),
|
||||
}
|
||||
harmless_by_layer = {
|
||||
9: _as_public_samples(base_harmless),
|
||||
5: _as_public_samples(base_harmless * 1.5),
|
||||
2: _as_public_samples(base_harmless),
|
||||
}
|
||||
|
||||
results = WhitenedSVDExtractor(
|
||||
regularization_eps=REGULARIZATION_EPS,
|
||||
min_variance_ratio=0.0,
|
||||
).extract_all_layers(harmful_by_layer, harmless_by_layer, n_directions=2)
|
||||
|
||||
assert list(results) == [2, 5]
|
||||
assert [result.layer_idx for result in results.values()] == [2, 5]
|
||||
assert all(result.directions.shape == (2, 4) for result in results.values())
|
||||
|
||||
|
||||
def test_extract_all_layers_skips_missing_layers_without_stopping_later_matches() -> None:
|
||||
base_harmful, base_harmless = _activation_pair_fixture()
|
||||
harmful_by_layer = {
|
||||
2: _as_public_samples(base_harmful),
|
||||
5: _as_public_samples(base_harmful * 1.5),
|
||||
8: _as_public_samples(base_harmful + 0.25),
|
||||
}
|
||||
harmless_by_layer = {
|
||||
5: _as_public_samples(base_harmless * 1.5),
|
||||
8: _as_public_samples(base_harmless),
|
||||
}
|
||||
|
||||
results = WhitenedSVDExtractor(
|
||||
regularization_eps=REGULARIZATION_EPS,
|
||||
min_variance_ratio=0.0,
|
||||
).extract_all_layers(harmful_by_layer, harmless_by_layer, n_directions=2)
|
||||
|
||||
assert list(results) == [5, 8]
|
||||
assert [result.layer_idx for result in results.values()] == [5, 8]
|
||||
|
||||
|
||||
def test_extract_all_layers_uses_documented_default_direction_count() -> None:
|
||||
base_harmful, base_harmless = _activation_pair_fixture()
|
||||
harmful_by_layer = {5: _as_public_samples(base_harmful)}
|
||||
harmless_by_layer = {5: _as_public_samples(base_harmless)}
|
||||
|
||||
results = WhitenedSVDExtractor(
|
||||
regularization_eps=REGULARIZATION_EPS,
|
||||
min_variance_ratio=0.0,
|
||||
).extract_all_layers(harmful_by_layer, harmless_by_layer)
|
||||
|
||||
assert list(results) == [5]
|
||||
assert results[5].directions.shape == (4, 4)
|
||||
Reference in New Issue
Block a user