fix(surgery): guard unvalidated Qwen hybrid layouts

This commit is contained in:
Joseph Magly
2026-08-24 13:54:31 -04:00
parent 56442df2d2
commit ccba4f5d85
9 changed files with 399 additions and 14 deletions
@@ -2,6 +2,7 @@
from __future__ import annotations
import os
import uuid
import pytest
@@ -17,6 +18,7 @@ MODEL = "hf-internal-testing/tiny-random-gpt2"
REVISION = "71034c5d8bde858ff824298bdedc65515b97d2b9"
MISTRAL4_MODEL = "mistralai/Mistral-Small-4-119B-2603"
MISTRAL4_REVISION = "a11f36bebf709121056b1dbcc943d1c6afbe494d"
QWEN38_MODEL = "Qwen/Qwen3.8-27B"
def test_pinned_tiny_model_download_inference_and_offline_cache(monkeypatch):
@@ -71,3 +73,26 @@ def test_pinned_mistral4_config_resolves_composite_contract_without_remote_code(
assert profile.model_type == "mistral4"
assert profile.arch_class is ArchitectureClass.LARGE_MOE
assert (profile.num_experts, profile.num_active_experts) == (128, 4)
@pytest.mark.gpu
def test_qwen38_bf16_pristine_baseline_blocks_unvalidated_surgery(tmp_path):
"""Operator-gated 27B regression: healthy stock model, zero modified weights."""
if os.environ.get("OBLITERATUS_QWEN38_E2E") != "1":
pytest.skip("set OBLITERATUS_QWEN38_E2E=1 on a >=80 GiB GPU runner")
from obliteratus.abliterate import AbliterationPipeline, PipelineValidationError
pipeline = AbliterationPipeline(
QWEN38_MODEL,
output_dir=str(tmp_path / "qwen38-output"),
dtype="bfloat16",
quantization=None,
)
pipeline._summon()
pipeline._capture_stock_baseline()
assert pipeline._stock_baseline["perplexity"] > 0
assert pipeline._stock_baseline["coherence"] > 0
with pytest.raises(PipelineValidationError, match="no validated projection allowlist"):
pipeline._validate_architecture_surgery_support()
assert pipeline._excise_modified_count is None
+83
View File
@@ -215,8 +215,68 @@ class TestPipelineInit:
assert pipeline.trust_remote_code is False
assert pipeline.gpu_memory_utilization is None
assert pipeline.refusal_max_tokens == 128
assert pipeline.max_perplexity_increase == 3.0
assert pipeline.min_coherence_retention == 0.5
assert pipeline.max_degenerate_fraction == 0.2
assert pipeline.handle is None
@pytest.mark.parametrize(
("keyword", "value"),
[
("max_perplexity_increase", 0.99),
("min_coherence_retention", 1.01),
("max_degenerate_fraction", -0.01),
],
)
def test_quality_guardrails_validate_configuration(self, keyword, value):
with pytest.raises(ValueError):
AbliterationPipeline(model_name="test-model", **{keyword: value})
@pytest.mark.parametrize("architecture", ["qwen3_5", "qwen3_5_text", "qwen3_5_moe"])
def test_unvalidated_qwen35_hybrid_fails_before_surgery(self, architecture):
from types import SimpleNamespace
from obliteratus.abliterate import PipelineValidationError
pipeline = AbliterationPipeline(model_name="Qwen/Qwen3.8-27B")
pipeline.handle = SimpleNamespace(architecture=architecture)
with pytest.raises(PipelineValidationError, match="no validated projection allowlist"):
pipeline._validate_architecture_surgery_support()
assert pipeline._excise_modified_count is None
assert pipeline._quality_metrics["architecture_support"] == 0.0
def test_relative_perplexity_guardrail_fails_closed(self):
from obliteratus.abliterate import PipelineValidationError
pipeline = AbliterationPipeline(
model_name="test-model",
max_perplexity_increase=2.0,
)
pipeline._stock_baseline = {"perplexity": 10.0}
with pytest.raises(PipelineValidationError) as exc_info:
pipeline._enforce_perplexity_guardrail(21.0)
assert exc_info.value.metric == "perplexity_increase"
assert pipeline._quality_metrics["perplexity_increase"] == pytest.approx(2.1)
@pytest.mark.parametrize(
("coherence", "degenerate", "metric"),
[(0.3, 0.0, "coherence_retention"), (1.0, 0.21, "degenerate_fraction")],
)
def test_generation_guardrails_fail_closed(self, coherence, degenerate, metric):
from obliteratus.abliterate import PipelineValidationError
pipeline = AbliterationPipeline(model_name="test-model")
pipeline._stock_baseline = {"coherence": 1.0}
with pytest.raises(PipelineValidationError) as exc_info:
pipeline._enforce_generation_guardrails(coherence, degenerate)
assert exc_info.value.metric == metric
def test_cancellation_is_terminal_and_cleanup_unloads_model(self):
from threading import Event
@@ -237,6 +297,29 @@ class TestPipelineInit:
assert pipeline.handle.model is None
assert pipeline.handle.tokenizer is None
def test_public_run_cleans_up_terminal_pipeline_failure(self, monkeypatch):
from obliteratus.abliterate import PipelineValidationError
pipeline = AbliterationPipeline(model_name="test-model")
cleanup = Mock()
monkeypatch.setattr(pipeline, "cleanup_failed_run", cleanup)
monkeypatch.setattr(
pipeline,
"_run_pipeline",
Mock(
side_effect=PipelineValidationError(
"unsafe",
stage="verify",
metric="perplexity",
),
),
)
with pytest.raises(PipelineValidationError):
pipeline.run()
cleanup.assert_called_once_with()
def test_catastrophic_perplexity_aborts_before_generation(self):
from types import SimpleNamespace
+17
View File
@@ -80,6 +80,23 @@ def test_refusal_max_tokens_cli_default_and_positive_override(monkeypatch):
assert command.call_args.args[0].refusal_max_tokens == 512
def test_quality_guardrail_cli_overrides(monkeypatch):
command = Mock()
monkeypatch.setattr(cli, "_cmd_abliterate", command)
cli.main([
"abliterate", "local/model",
"--max-perplexity-increase", "2.5",
"--min-coherence-retention", "0.75",
"--max-degenerate-fraction", "0.1",
])
args = command.call_args.args[0]
assert args.max_perplexity_increase == 2.5
assert args.min_coherence_retention == 0.75
assert args.max_degenerate_fraction == 0.1
def test_trust_remote_code_requires_explicit_cli_opt_in(monkeypatch):
command = Mock()
monkeypatch.setattr(cli, "_cmd_abliterate", command)
@@ -26,6 +26,8 @@ def test_run_informed_executes_the_documented_stage_order(pipeline, monkeypatch)
output = pipeline.output_dir
for name in (
"_summon",
"_capture_stock_baseline",
"_validate_architecture_surgery_support",
"_probe",
"_analyze",
"_distill_informed",
@@ -46,6 +48,8 @@ def test_run_informed_executes_the_documented_stage_order(pipeline, monkeypatch)
assert result == output
assert calls == [
"_summon",
"_capture_stock_baseline",
"_validate_architecture_surgery_support",
"_probe",
"_analyze",
"_distill_informed",
+7 -1
View File
@@ -151,6 +151,8 @@ def test_full_pipeline_saves_and_reloads_a_real_offline_model(tmp_path):
n_directions=1,
max_seq_length=8,
verify_sample_size=1,
max_perplexity_increase=1000.0,
max_degenerate_fraction=1.0,
harmful_prompts=["harmful request"],
harmless_prompts=["harmless request"],
on_stage=events.append,
@@ -160,7 +162,9 @@ def test_full_pipeline_saves_and_reloads_a_real_offline_model(tmp_path):
assert result == output
assert [(event.stage, event.status) for event in events] == [
(stage, status)
for stage in ("summon", "probe", "distill", "excise", "verify", "rebirth")
for stage in (
"summon", "baseline", "probe", "distill", "excise", "verify", "rebirth",
)
for status in ("running", "done")
]
assert (output / "abliteration_metadata.json").is_file()
@@ -211,6 +215,8 @@ def test_multidirection_pipeline_restores_tiny_model_layer_norms(tmp_path):
max_seq_length=8,
verify_sample_size=1,
refusal_max_tokens=1,
max_perplexity_increase=1000.0,
max_degenerate_fraction=1.0,
harmful_prompts=["harmful request", "harmful answer"],
harmless_prompts=["harmless request", "harmless answer"],
)