mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-08-30 06:30:37 +02:00
fix(surgery): guard unvalidated Qwen hybrid layouts
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
@@ -17,6 +18,7 @@ MODEL = "hf-internal-testing/tiny-random-gpt2"
|
||||
REVISION = "71034c5d8bde858ff824298bdedc65515b97d2b9"
|
||||
MISTRAL4_MODEL = "mistralai/Mistral-Small-4-119B-2603"
|
||||
MISTRAL4_REVISION = "a11f36bebf709121056b1dbcc943d1c6afbe494d"
|
||||
QWEN38_MODEL = "Qwen/Qwen3.8-27B"
|
||||
|
||||
|
||||
def test_pinned_tiny_model_download_inference_and_offline_cache(monkeypatch):
|
||||
@@ -71,3 +73,26 @@ def test_pinned_mistral4_config_resolves_composite_contract_without_remote_code(
|
||||
assert profile.model_type == "mistral4"
|
||||
assert profile.arch_class is ArchitectureClass.LARGE_MOE
|
||||
assert (profile.num_experts, profile.num_active_experts) == (128, 4)
|
||||
|
||||
|
||||
@pytest.mark.gpu
|
||||
def test_qwen38_bf16_pristine_baseline_blocks_unvalidated_surgery(tmp_path):
|
||||
"""Operator-gated 27B regression: healthy stock model, zero modified weights."""
|
||||
if os.environ.get("OBLITERATUS_QWEN38_E2E") != "1":
|
||||
pytest.skip("set OBLITERATUS_QWEN38_E2E=1 on a >=80 GiB GPU runner")
|
||||
|
||||
from obliteratus.abliterate import AbliterationPipeline, PipelineValidationError
|
||||
|
||||
pipeline = AbliterationPipeline(
|
||||
QWEN38_MODEL,
|
||||
output_dir=str(tmp_path / "qwen38-output"),
|
||||
dtype="bfloat16",
|
||||
quantization=None,
|
||||
)
|
||||
pipeline._summon()
|
||||
pipeline._capture_stock_baseline()
|
||||
assert pipeline._stock_baseline["perplexity"] > 0
|
||||
assert pipeline._stock_baseline["coherence"] > 0
|
||||
with pytest.raises(PipelineValidationError, match="no validated projection allowlist"):
|
||||
pipeline._validate_architecture_surgery_support()
|
||||
assert pipeline._excise_modified_count is None
|
||||
|
||||
@@ -215,8 +215,68 @@ class TestPipelineInit:
|
||||
assert pipeline.trust_remote_code is False
|
||||
assert pipeline.gpu_memory_utilization is None
|
||||
assert pipeline.refusal_max_tokens == 128
|
||||
assert pipeline.max_perplexity_increase == 3.0
|
||||
assert pipeline.min_coherence_retention == 0.5
|
||||
assert pipeline.max_degenerate_fraction == 0.2
|
||||
assert pipeline.handle is None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("keyword", "value"),
|
||||
[
|
||||
("max_perplexity_increase", 0.99),
|
||||
("min_coherence_retention", 1.01),
|
||||
("max_degenerate_fraction", -0.01),
|
||||
],
|
||||
)
|
||||
def test_quality_guardrails_validate_configuration(self, keyword, value):
|
||||
with pytest.raises(ValueError):
|
||||
AbliterationPipeline(model_name="test-model", **{keyword: value})
|
||||
|
||||
@pytest.mark.parametrize("architecture", ["qwen3_5", "qwen3_5_text", "qwen3_5_moe"])
|
||||
def test_unvalidated_qwen35_hybrid_fails_before_surgery(self, architecture):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from obliteratus.abliterate import PipelineValidationError
|
||||
|
||||
pipeline = AbliterationPipeline(model_name="Qwen/Qwen3.8-27B")
|
||||
pipeline.handle = SimpleNamespace(architecture=architecture)
|
||||
|
||||
with pytest.raises(PipelineValidationError, match="no validated projection allowlist"):
|
||||
pipeline._validate_architecture_surgery_support()
|
||||
|
||||
assert pipeline._excise_modified_count is None
|
||||
assert pipeline._quality_metrics["architecture_support"] == 0.0
|
||||
|
||||
def test_relative_perplexity_guardrail_fails_closed(self):
|
||||
from obliteratus.abliterate import PipelineValidationError
|
||||
|
||||
pipeline = AbliterationPipeline(
|
||||
model_name="test-model",
|
||||
max_perplexity_increase=2.0,
|
||||
)
|
||||
pipeline._stock_baseline = {"perplexity": 10.0}
|
||||
|
||||
with pytest.raises(PipelineValidationError) as exc_info:
|
||||
pipeline._enforce_perplexity_guardrail(21.0)
|
||||
|
||||
assert exc_info.value.metric == "perplexity_increase"
|
||||
assert pipeline._quality_metrics["perplexity_increase"] == pytest.approx(2.1)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("coherence", "degenerate", "metric"),
|
||||
[(0.3, 0.0, "coherence_retention"), (1.0, 0.21, "degenerate_fraction")],
|
||||
)
|
||||
def test_generation_guardrails_fail_closed(self, coherence, degenerate, metric):
|
||||
from obliteratus.abliterate import PipelineValidationError
|
||||
|
||||
pipeline = AbliterationPipeline(model_name="test-model")
|
||||
pipeline._stock_baseline = {"coherence": 1.0}
|
||||
|
||||
with pytest.raises(PipelineValidationError) as exc_info:
|
||||
pipeline._enforce_generation_guardrails(coherence, degenerate)
|
||||
|
||||
assert exc_info.value.metric == metric
|
||||
|
||||
def test_cancellation_is_terminal_and_cleanup_unloads_model(self):
|
||||
from threading import Event
|
||||
|
||||
@@ -237,6 +297,29 @@ class TestPipelineInit:
|
||||
assert pipeline.handle.model is None
|
||||
assert pipeline.handle.tokenizer is None
|
||||
|
||||
def test_public_run_cleans_up_terminal_pipeline_failure(self, monkeypatch):
|
||||
from obliteratus.abliterate import PipelineValidationError
|
||||
|
||||
pipeline = AbliterationPipeline(model_name="test-model")
|
||||
cleanup = Mock()
|
||||
monkeypatch.setattr(pipeline, "cleanup_failed_run", cleanup)
|
||||
monkeypatch.setattr(
|
||||
pipeline,
|
||||
"_run_pipeline",
|
||||
Mock(
|
||||
side_effect=PipelineValidationError(
|
||||
"unsafe",
|
||||
stage="verify",
|
||||
metric="perplexity",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(PipelineValidationError):
|
||||
pipeline.run()
|
||||
|
||||
cleanup.assert_called_once_with()
|
||||
|
||||
def test_catastrophic_perplexity_aborts_before_generation(self):
|
||||
from types import SimpleNamespace
|
||||
|
||||
|
||||
@@ -80,6 +80,23 @@ def test_refusal_max_tokens_cli_default_and_positive_override(monkeypatch):
|
||||
assert command.call_args.args[0].refusal_max_tokens == 512
|
||||
|
||||
|
||||
def test_quality_guardrail_cli_overrides(monkeypatch):
|
||||
command = Mock()
|
||||
monkeypatch.setattr(cli, "_cmd_abliterate", command)
|
||||
|
||||
cli.main([
|
||||
"abliterate", "local/model",
|
||||
"--max-perplexity-increase", "2.5",
|
||||
"--min-coherence-retention", "0.75",
|
||||
"--max-degenerate-fraction", "0.1",
|
||||
])
|
||||
|
||||
args = command.call_args.args[0]
|
||||
assert args.max_perplexity_increase == 2.5
|
||||
assert args.min_coherence_retention == 0.75
|
||||
assert args.max_degenerate_fraction == 0.1
|
||||
|
||||
|
||||
def test_trust_remote_code_requires_explicit_cli_opt_in(monkeypatch):
|
||||
command = Mock()
|
||||
monkeypatch.setattr(cli, "_cmd_abliterate", command)
|
||||
|
||||
@@ -26,6 +26,8 @@ def test_run_informed_executes_the_documented_stage_order(pipeline, monkeypatch)
|
||||
output = pipeline.output_dir
|
||||
for name in (
|
||||
"_summon",
|
||||
"_capture_stock_baseline",
|
||||
"_validate_architecture_surgery_support",
|
||||
"_probe",
|
||||
"_analyze",
|
||||
"_distill_informed",
|
||||
@@ -46,6 +48,8 @@ def test_run_informed_executes_the_documented_stage_order(pipeline, monkeypatch)
|
||||
assert result == output
|
||||
assert calls == [
|
||||
"_summon",
|
||||
"_capture_stock_baseline",
|
||||
"_validate_architecture_surgery_support",
|
||||
"_probe",
|
||||
"_analyze",
|
||||
"_distill_informed",
|
||||
|
||||
@@ -151,6 +151,8 @@ def test_full_pipeline_saves_and_reloads_a_real_offline_model(tmp_path):
|
||||
n_directions=1,
|
||||
max_seq_length=8,
|
||||
verify_sample_size=1,
|
||||
max_perplexity_increase=1000.0,
|
||||
max_degenerate_fraction=1.0,
|
||||
harmful_prompts=["harmful request"],
|
||||
harmless_prompts=["harmless request"],
|
||||
on_stage=events.append,
|
||||
@@ -160,7 +162,9 @@ def test_full_pipeline_saves_and_reloads_a_real_offline_model(tmp_path):
|
||||
assert result == output
|
||||
assert [(event.stage, event.status) for event in events] == [
|
||||
(stage, status)
|
||||
for stage in ("summon", "probe", "distill", "excise", "verify", "rebirth")
|
||||
for stage in (
|
||||
"summon", "baseline", "probe", "distill", "excise", "verify", "rebirth",
|
||||
)
|
||||
for status in ("running", "done")
|
||||
]
|
||||
assert (output / "abliteration_metadata.json").is_file()
|
||||
@@ -211,6 +215,8 @@ def test_multidirection_pipeline_restores_tiny_model_layer_norms(tmp_path):
|
||||
max_seq_length=8,
|
||||
verify_sample_size=1,
|
||||
refusal_max_tokens=1,
|
||||
max_perplexity_increase=1000.0,
|
||||
max_degenerate_fraction=1.0,
|
||||
harmful_prompts=["harmful request", "harmful answer"],
|
||||
harmless_prompts=["harmless request", "harmless answer"],
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user