mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-08-30 06:30:37 +02:00
fix(surgery): guard unvalidated Qwen hybrid layouts
This commit is contained in:
@@ -549,6 +549,13 @@ obliteratus obliterate Qwen/Qwen3.8-27B --dtype bfloat16
|
||||
obliteratus obliterate Qwen/Qwen3.8-27B --dtype float16 --quantization 4bit
|
||||
```
|
||||
|
||||
> **Qwen3.8 safety status:** loading the native BF16 checkpoint is supported, but
|
||||
> permanent abliteration of its Qwen3.5 hybrid Gated DeltaNet/attention layout is
|
||||
> currently unvalidated. The pipeline checks pristine perplexity and coherence,
|
||||
> then stops before modifying weights with an unsupported-architecture error.
|
||||
> Qwen3.8 is not considered surgery-supported until projection-family validation
|
||||
> and a real post-surgery regression pass are published.
|
||||
|
||||
Install the optional backend before selecting a bitsandbytes mode:
|
||||
|
||||
```bash
|
||||
|
||||
+220
-13
@@ -145,6 +145,15 @@ _CORPUS_CONTAMINATION_RE = re.compile(
|
||||
r"(?im)(?:\bbrainly\b|^\s*(?:profile|answer|answered)\s*$|^\s*\d{2}\.\d{2}\.\d{4}\s*$)"
|
||||
)
|
||||
|
||||
_REFERENCE_TEXTS = (
|
||||
"The theory of general relativity describes gravity as a geometric property of space and time. "
|
||||
"Massive objects cause a distortion in space-time, which is felt as gravity by other objects.",
|
||||
"In computer science, a binary search algorithm finds the position of a target value within a "
|
||||
"sorted array. It compares the target value to the middle element of the array.",
|
||||
"Photosynthesis is the process by which plants convert light energy into chemical energy. "
|
||||
"This process occurs primarily in the leaves of plants using chlorophyll.",
|
||||
)
|
||||
|
||||
|
||||
def _is_coherent_completion(prompt: str, completion: str) -> bool:
|
||||
"""Return whether a completion is relevant, varied, and contamination-free."""
|
||||
@@ -164,6 +173,15 @@ def _is_coherent_completion(prompt: str, completion: str) -> bool:
|
||||
return all(any(term in lowered for term in alternatives) for alternatives in anchors)
|
||||
|
||||
|
||||
def _is_degenerate_completion(completion: str) -> bool:
|
||||
"""Detect empty, extremely repetitive, or punctuation-only generation."""
|
||||
text = completion.strip()
|
||||
words = re.findall(r"[\w'-]+", text.lower())
|
||||
if not words:
|
||||
return True
|
||||
return len(words) >= 5 and len(set(words)) / len(words) <= 0.2
|
||||
|
||||
|
||||
# ── Abliteration method presets ───────────────────────────────────────────
|
||||
|
||||
METHODS = {
|
||||
@@ -909,6 +927,9 @@ class AbliterationPipeline:
|
||||
# Verify stage sample size
|
||||
verify_sample_size: int | None = None,
|
||||
refusal_max_tokens: int | None = None,
|
||||
max_perplexity_increase: float = 3.0,
|
||||
min_coherence_retention: float = 0.5,
|
||||
max_degenerate_fraction: float = 0.2,
|
||||
on_stage: Callable[[StageResult], None] | None = None,
|
||||
on_log: Callable[[str], None] | None = None,
|
||||
cancellation_event: Event | None = None,
|
||||
@@ -1057,6 +1078,30 @@ class AbliterationPipeline:
|
||||
):
|
||||
raise ValueError("refusal_max_tokens must be a positive integer")
|
||||
self.refusal_max_tokens = refusal_max_tokens if refusal_max_tokens is not None else 128
|
||||
if (
|
||||
isinstance(max_perplexity_increase, bool)
|
||||
or not isinstance(max_perplexity_increase, (int, float))
|
||||
or not math.isfinite(max_perplexity_increase)
|
||||
or max_perplexity_increase < 1.0
|
||||
):
|
||||
raise ValueError("max_perplexity_increase must be at least 1.0")
|
||||
if (
|
||||
isinstance(min_coherence_retention, bool)
|
||||
or not isinstance(min_coherence_retention, (int, float))
|
||||
or not math.isfinite(min_coherence_retention)
|
||||
or not 0.0 <= min_coherence_retention <= 1.0
|
||||
):
|
||||
raise ValueError("min_coherence_retention must be in [0.0, 1.0]")
|
||||
if (
|
||||
isinstance(max_degenerate_fraction, bool)
|
||||
or not isinstance(max_degenerate_fraction, (int, float))
|
||||
or not math.isfinite(max_degenerate_fraction)
|
||||
or not 0.0 <= max_degenerate_fraction <= 1.0
|
||||
):
|
||||
raise ValueError("max_degenerate_fraction must be in [0.0, 1.0]")
|
||||
self.max_perplexity_increase = float(max_perplexity_increase)
|
||||
self.min_coherence_retention = float(min_coherence_retention)
|
||||
self.max_degenerate_fraction = float(max_degenerate_fraction)
|
||||
|
||||
# Large model mode: conservative defaults for 120B+ models.
|
||||
# Reduces memory footprint by limiting SAE features, directions,
|
||||
@@ -1079,6 +1124,7 @@ class AbliterationPipeline:
|
||||
self._harmless_means: dict[int, torch.Tensor] = {}
|
||||
self._shield_concept_atoms: dict[int, torch.Tensor] = {}
|
||||
self._quality_metrics: dict[str, float] = {}
|
||||
self._stock_baseline: dict[str, float] = {}
|
||||
|
||||
# LoRA ablation state (reversible adapters)
|
||||
self._lora_adapters: dict[str, tuple[torch.Tensor, torch.Tensor]] = {}
|
||||
@@ -1151,11 +1197,18 @@ class AbliterationPipeline:
|
||||
metric="cancellation",
|
||||
)
|
||||
|
||||
def _fail_validation(self, metric: str, value: float, reason: str) -> None:
|
||||
def _fail_validation(
|
||||
self,
|
||||
metric: str,
|
||||
value: float,
|
||||
reason: str,
|
||||
*,
|
||||
stage: str = "verify",
|
||||
) -> None:
|
||||
self._quality_metrics[metric] = value
|
||||
message = f"Validation failed: {reason} ({metric}={value!r})"
|
||||
self._emit("verify", "error", message, metric=metric, value=value)
|
||||
raise PipelineValidationError(message, stage="verify", metric=metric)
|
||||
self._emit(stage, "error", message, metric=metric, value=value)
|
||||
raise PipelineValidationError(message, stage=stage, metric=metric)
|
||||
|
||||
def cleanup_failed_run(self) -> None:
|
||||
"""Unload an unsafe/partial in-memory result and release transient state."""
|
||||
@@ -1384,6 +1437,14 @@ class AbliterationPipeline:
|
||||
|
||||
def run(self) -> Path:
|
||||
"""Execute the full abliteration pipeline. Returns path to saved model."""
|
||||
try:
|
||||
return self._run_pipeline()
|
||||
except PipelineFailure:
|
||||
self.cleanup_failed_run()
|
||||
raise
|
||||
|
||||
def _run_pipeline(self) -> Path:
|
||||
"""Execute stages while ``run`` owns terminal failure cleanup."""
|
||||
# Remove any steering hooks left from a previous run() call
|
||||
for h in self._steering_hooks:
|
||||
h.remove()
|
||||
@@ -1391,6 +1452,10 @@ class AbliterationPipeline:
|
||||
self._active_stage = "summon"
|
||||
self._summon()
|
||||
self._free_gpu_memory()
|
||||
self._active_stage = "baseline"
|
||||
self._capture_stock_baseline()
|
||||
self._validate_architecture_surgery_support()
|
||||
self._free_gpu_memory()
|
||||
self._active_stage = "probe"
|
||||
self._probe()
|
||||
self._free_gpu_memory()
|
||||
@@ -1455,6 +1520,151 @@ class AbliterationPipeline:
|
||||
self.log(f"Total parameters: {summary['total_params']:,}")
|
||||
self._emit("summon", "done", f"Loaded ({elapsed:.1f}s)", duration=elapsed, **summary)
|
||||
|
||||
def _reference_perplexity(self) -> float:
|
||||
"""Measure finite reference perplexity for baseline and post-surgery gates."""
|
||||
model = self.handle.model
|
||||
tokenizer = self.handle.tokenizer
|
||||
device = self._get_model_device(model)
|
||||
weighted_loss = 0.0
|
||||
token_count = 0
|
||||
for text in _REFERENCE_TEXTS:
|
||||
inputs = tokenizer(
|
||||
text,
|
||||
return_tensors="pt",
|
||||
truncation=True,
|
||||
max_length=self.max_seq_length or 256,
|
||||
)
|
||||
inputs = {key: value.to(device) for key, value in inputs.items()}
|
||||
with torch.no_grad():
|
||||
outputs = model(**inputs, labels=inputs["input_ids"])
|
||||
loss = float(outputs.loss.item())
|
||||
sequence_length = inputs["input_ids"].shape[1]
|
||||
del inputs, outputs
|
||||
if not math.isfinite(loss):
|
||||
return float("inf")
|
||||
weighted_loss += loss * sequence_length
|
||||
token_count += sequence_length
|
||||
return math.exp(min(weighted_loss / token_count, 100.0)) if token_count else float("inf")
|
||||
|
||||
def _stock_coherence(self) -> float:
|
||||
"""Run a small, deterministic coherence sample before any weight mutation."""
|
||||
model = self.handle.model
|
||||
tokenizer = self.handle.tokenizer
|
||||
device = self._get_model_device(model)
|
||||
prompts = (
|
||||
"The capital of France is",
|
||||
"Photosynthesis is the process by which",
|
||||
"A binary search algorithm works by",
|
||||
)
|
||||
coherent = 0
|
||||
for prompt in prompts:
|
||||
inputs = tokenizer(prompt, return_tensors="pt")
|
||||
input_length = inputs["input_ids"].shape[1]
|
||||
inputs = {key: value.to(device) for key, value in inputs.items()}
|
||||
with torch.no_grad():
|
||||
output = model.generate(**inputs, max_new_tokens=64, do_sample=False)
|
||||
completion = tokenizer.decode(
|
||||
output[0][input_length:],
|
||||
skip_special_tokens=True,
|
||||
).strip()[:200]
|
||||
del inputs, output
|
||||
coherent += int(_is_coherent_completion(prompt, completion))
|
||||
return coherent / len(prompts)
|
||||
|
||||
def _capture_stock_baseline(self) -> None:
|
||||
"""Prove the pristine checkpoint is healthy before surgery begins."""
|
||||
self._emit("baseline", "running", "Validating pristine checkpoint...")
|
||||
perplexity = self._reference_perplexity()
|
||||
if not math.isfinite(perplexity):
|
||||
self._fail_validation(
|
||||
"baseline_perplexity",
|
||||
perplexity,
|
||||
"the pristine checkpoint produces non-finite loss",
|
||||
stage="baseline",
|
||||
)
|
||||
try:
|
||||
coherence = self._stock_coherence()
|
||||
except RuntimeError as error:
|
||||
if not dev.is_oom_error(error):
|
||||
raise
|
||||
self._free_gpu_memory()
|
||||
coherence = float("nan")
|
||||
self.log("Stock generation baseline skipped because the KV cache exceeded device capacity.")
|
||||
self._stock_baseline = {
|
||||
"perplexity": perplexity,
|
||||
"coherence": coherence,
|
||||
}
|
||||
self._quality_metrics["baseline_perplexity"] = perplexity
|
||||
self._quality_metrics["baseline_coherence"] = coherence
|
||||
self._emit(
|
||||
"baseline",
|
||||
"done",
|
||||
"Pristine checkpoint is numerically healthy.",
|
||||
perplexity=perplexity,
|
||||
coherence=coherence,
|
||||
)
|
||||
|
||||
def _validate_architecture_surgery_support(self) -> None:
|
||||
"""Reject hybrid layouts whose permanent projection allowlist is unvalidated."""
|
||||
architecture = str(self.handle.architecture).lower()
|
||||
if architecture in {"qwen3_5", "qwen3_5_text", "qwen3_5_moe"}:
|
||||
coherence = self._stock_baseline.get("coherence")
|
||||
if coherence is not None and math.isfinite(coherence) and coherence <= 0.0:
|
||||
self._fail_validation(
|
||||
"baseline_coherence",
|
||||
coherence,
|
||||
"the pristine Qwen3.5/Qwen3.8 checkpoint failed every "
|
||||
"deterministic coherence prompt",
|
||||
stage="baseline",
|
||||
)
|
||||
self._fail_validation(
|
||||
"architecture_support",
|
||||
0.0,
|
||||
"Qwen3.5/Qwen3.8 hybrid Gated DeltaNet surgery has no validated "
|
||||
"projection allowlist; no weights were modified",
|
||||
stage="baseline",
|
||||
)
|
||||
|
||||
def _enforce_perplexity_guardrail(self, perplexity: float) -> None:
|
||||
baseline = self._stock_baseline.get("perplexity")
|
||||
if not baseline or not math.isfinite(baseline):
|
||||
return
|
||||
increase = perplexity / baseline
|
||||
self._quality_metrics["perplexity_increase"] = increase
|
||||
if increase > self.max_perplexity_increase:
|
||||
self._fail_validation(
|
||||
"perplexity_increase",
|
||||
increase,
|
||||
f"perplexity exceeded the configured {self.max_perplexity_increase:.2f}x "
|
||||
"stock-baseline limit",
|
||||
)
|
||||
|
||||
def _enforce_generation_guardrails(
|
||||
self,
|
||||
coherence: float,
|
||||
degenerate_fraction: float,
|
||||
) -> None:
|
||||
self._quality_metrics["degenerate_fraction"] = degenerate_fraction
|
||||
if degenerate_fraction > self.max_degenerate_fraction:
|
||||
self._fail_validation(
|
||||
"degenerate_fraction",
|
||||
degenerate_fraction,
|
||||
f"degenerate completions exceeded the configured "
|
||||
f"{self.max_degenerate_fraction:.0%} limit",
|
||||
)
|
||||
baseline = self._stock_baseline.get("coherence")
|
||||
if not baseline or not math.isfinite(baseline):
|
||||
return
|
||||
retention = coherence / baseline
|
||||
self._quality_metrics["coherence_retention"] = retention
|
||||
if retention < self.min_coherence_retention:
|
||||
self._fail_validation(
|
||||
"coherence_retention",
|
||||
retention,
|
||||
f"coherence fell below the configured "
|
||||
f"{self.min_coherence_retention:.0%} stock-baseline retention",
|
||||
)
|
||||
|
||||
# ── Stage 2: PROBE ──────────────────────────────────────────────────
|
||||
|
||||
def _probe(self):
|
||||
@@ -6515,20 +6725,11 @@ class AbliterationPipeline:
|
||||
self._free_gpu_memory()
|
||||
|
||||
# 1. Perplexity measurement on reference texts
|
||||
reference_texts = [
|
||||
"The theory of general relativity describes gravity as a geometric property of space and time. "
|
||||
"Massive objects cause a distortion in space-time, which is felt as gravity by other objects.",
|
||||
"In computer science, a binary search algorithm finds the position of a target value within a "
|
||||
"sorted array. It compares the target value to the middle element of the array.",
|
||||
"Photosynthesis is the process by which plants convert light energy into chemical energy. "
|
||||
"This process occurs primarily in the leaves of plants using chlorophyll.",
|
||||
]
|
||||
|
||||
self.log("Measuring perplexity on reference texts...")
|
||||
total_loss = 0.0
|
||||
n_tokens = 0
|
||||
has_nan_loss = False
|
||||
for text in reference_texts:
|
||||
for text in _REFERENCE_TEXTS:
|
||||
inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=self.max_seq_length or 256)
|
||||
inputs = {k: v.to(device) for k, v in inputs.items()}
|
||||
with torch.no_grad():
|
||||
@@ -6574,6 +6775,7 @@ class AbliterationPipeline:
|
||||
perplexity,
|
||||
"perplexity is non-finite; generation and export are blocked",
|
||||
)
|
||||
self._enforce_perplexity_guardrail(perplexity)
|
||||
|
||||
# 2. Generation coherence test
|
||||
test_prompts = [
|
||||
@@ -6641,6 +6843,7 @@ class AbliterationPipeline:
|
||||
|
||||
self.log("Generating test completions:")
|
||||
coherent_count = 0
|
||||
degenerate_count = 0
|
||||
generation_failed = False
|
||||
for prompt in test_prompts:
|
||||
if generation_failed:
|
||||
@@ -6666,6 +6869,8 @@ class AbliterationPipeline:
|
||||
|
||||
if _is_coherent_completion(prompt, completion):
|
||||
coherent_count += 1
|
||||
if _is_degenerate_completion(completion):
|
||||
degenerate_count += 1
|
||||
except (RuntimeError, Exception) as e:
|
||||
if dev.is_oom_error(e):
|
||||
self._free_gpu_memory()
|
||||
@@ -6685,7 +6890,9 @@ class AbliterationPipeline:
|
||||
if not generation_failed:
|
||||
coherence_score = coherent_count / len(test_prompts)
|
||||
self._quality_metrics["coherence"] = coherence_score
|
||||
degenerate_fraction = degenerate_count / len(test_prompts)
|
||||
self.log(f" Coherence: {coherence_score:.0%} ({coherent_count}/{len(test_prompts)} prompts)")
|
||||
self._enforce_generation_guardrails(coherence_score, degenerate_fraction)
|
||||
else:
|
||||
coherence_score = None
|
||||
self._quality_metrics["coherence"] = None
|
||||
|
||||
@@ -317,6 +317,18 @@ def main(argv: list[str] | None = None):
|
||||
"--refusal-max-tokens", type=_positive_int, default=None,
|
||||
help="Max new tokens to generate per response in the refusal test (default: 128).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--max-perplexity-increase", type=float, default=3.0,
|
||||
help="Fail if post-surgery perplexity exceeds this multiple of stock (default: 3.0).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--min-coherence-retention", type=float, default=0.5,
|
||||
help="Minimum post-surgery coherence divided by stock coherence (default: 0.5).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--max-degenerate-fraction", type=float, default=0.2,
|
||||
help="Fail if repetitive/empty completions exceed this fraction (default: 0.2).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--dataset", type=str, default="builtin",
|
||||
help="Prompt dataset source for contrastive extraction when using residue mining (default: builtin).",
|
||||
@@ -1272,6 +1284,9 @@ def _cmd_abliterate(args):
|
||||
large_model_mode=getattr(args, "large_model", False),
|
||||
verify_sample_size=getattr(args, "verify_sample_size", None),
|
||||
refusal_max_tokens=getattr(args, "refusal_max_tokens", None),
|
||||
max_perplexity_increase=getattr(args, "max_perplexity_increase", 3.0),
|
||||
min_coherence_retention=getattr(args, "min_coherence_retention", 0.5),
|
||||
max_degenerate_fraction=getattr(args, "max_degenerate_fraction", 0.2),
|
||||
on_stage=on_stage,
|
||||
on_log=on_log,
|
||||
**prompt_kwargs,
|
||||
|
||||
@@ -61,6 +61,7 @@ import torch
|
||||
|
||||
from obliteratus.abliterate import (
|
||||
AbliterationPipeline,
|
||||
PipelineFailure,
|
||||
StageResult,
|
||||
)
|
||||
|
||||
@@ -184,6 +185,9 @@ class InformedAbliterationPipeline(AbliterationPipeline):
|
||||
on_stage: Callable[[StageResult], None] | None = None,
|
||||
on_log: Callable[[str], None] | None = None,
|
||||
cancellation_event: Event | None = None,
|
||||
max_perplexity_increase: float = 3.0,
|
||||
min_coherence_retention: float = 0.5,
|
||||
max_degenerate_fraction: float = 0.2,
|
||||
# Base pipeline kwargs forwarded to AbliterationPipeline
|
||||
push_to_hub: str | None = None,
|
||||
hub_token: str | None = None,
|
||||
@@ -219,6 +223,9 @@ class InformedAbliterationPipeline(AbliterationPipeline):
|
||||
on_stage=on_stage,
|
||||
on_log=on_log,
|
||||
cancellation_event=cancellation_event,
|
||||
max_perplexity_increase=max_perplexity_increase,
|
||||
min_coherence_retention=min_coherence_retention,
|
||||
max_degenerate_fraction=max_degenerate_fraction,
|
||||
push_to_hub=push_to_hub,
|
||||
hub_token=hub_token,
|
||||
hub_community_org=hub_community_org,
|
||||
@@ -267,12 +274,26 @@ class InformedAbliterationPipeline(AbliterationPipeline):
|
||||
(output_path, report) tuple with saved model path and
|
||||
comprehensive analysis report.
|
||||
"""
|
||||
try:
|
||||
return self._run_informed_pipeline()
|
||||
except PipelineFailure:
|
||||
self.cleanup_failed_run()
|
||||
raise
|
||||
|
||||
def _run_informed_pipeline(self) -> tuple[Path, InformedPipelineReport]:
|
||||
"""Execute informed stages while the public method owns cleanup."""
|
||||
t0 = time.time()
|
||||
|
||||
# Stage 1: SUMMON
|
||||
self._active_stage = "summon"
|
||||
self._summon()
|
||||
|
||||
# Prove the pristine checkpoint is healthy and the permanent surgery
|
||||
# layout is supported before analysis can lead to weight mutation.
|
||||
self._active_stage = "baseline"
|
||||
self._capture_stock_baseline()
|
||||
self._validate_architecture_surgery_support()
|
||||
|
||||
# Stage 2: PROBE
|
||||
self._active_stage = "probe"
|
||||
self._probe()
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
@@ -17,6 +18,7 @@ MODEL = "hf-internal-testing/tiny-random-gpt2"
|
||||
REVISION = "71034c5d8bde858ff824298bdedc65515b97d2b9"
|
||||
MISTRAL4_MODEL = "mistralai/Mistral-Small-4-119B-2603"
|
||||
MISTRAL4_REVISION = "a11f36bebf709121056b1dbcc943d1c6afbe494d"
|
||||
QWEN38_MODEL = "Qwen/Qwen3.8-27B"
|
||||
|
||||
|
||||
def test_pinned_tiny_model_download_inference_and_offline_cache(monkeypatch):
|
||||
@@ -71,3 +73,26 @@ def test_pinned_mistral4_config_resolves_composite_contract_without_remote_code(
|
||||
assert profile.model_type == "mistral4"
|
||||
assert profile.arch_class is ArchitectureClass.LARGE_MOE
|
||||
assert (profile.num_experts, profile.num_active_experts) == (128, 4)
|
||||
|
||||
|
||||
@pytest.mark.gpu
|
||||
def test_qwen38_bf16_pristine_baseline_blocks_unvalidated_surgery(tmp_path):
|
||||
"""Operator-gated 27B regression: healthy stock model, zero modified weights."""
|
||||
if os.environ.get("OBLITERATUS_QWEN38_E2E") != "1":
|
||||
pytest.skip("set OBLITERATUS_QWEN38_E2E=1 on a >=80 GiB GPU runner")
|
||||
|
||||
from obliteratus.abliterate import AbliterationPipeline, PipelineValidationError
|
||||
|
||||
pipeline = AbliterationPipeline(
|
||||
QWEN38_MODEL,
|
||||
output_dir=str(tmp_path / "qwen38-output"),
|
||||
dtype="bfloat16",
|
||||
quantization=None,
|
||||
)
|
||||
pipeline._summon()
|
||||
pipeline._capture_stock_baseline()
|
||||
assert pipeline._stock_baseline["perplexity"] > 0
|
||||
assert pipeline._stock_baseline["coherence"] > 0
|
||||
with pytest.raises(PipelineValidationError, match="no validated projection allowlist"):
|
||||
pipeline._validate_architecture_surgery_support()
|
||||
assert pipeline._excise_modified_count is None
|
||||
|
||||
@@ -215,8 +215,68 @@ class TestPipelineInit:
|
||||
assert pipeline.trust_remote_code is False
|
||||
assert pipeline.gpu_memory_utilization is None
|
||||
assert pipeline.refusal_max_tokens == 128
|
||||
assert pipeline.max_perplexity_increase == 3.0
|
||||
assert pipeline.min_coherence_retention == 0.5
|
||||
assert pipeline.max_degenerate_fraction == 0.2
|
||||
assert pipeline.handle is None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("keyword", "value"),
|
||||
[
|
||||
("max_perplexity_increase", 0.99),
|
||||
("min_coherence_retention", 1.01),
|
||||
("max_degenerate_fraction", -0.01),
|
||||
],
|
||||
)
|
||||
def test_quality_guardrails_validate_configuration(self, keyword, value):
|
||||
with pytest.raises(ValueError):
|
||||
AbliterationPipeline(model_name="test-model", **{keyword: value})
|
||||
|
||||
@pytest.mark.parametrize("architecture", ["qwen3_5", "qwen3_5_text", "qwen3_5_moe"])
|
||||
def test_unvalidated_qwen35_hybrid_fails_before_surgery(self, architecture):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from obliteratus.abliterate import PipelineValidationError
|
||||
|
||||
pipeline = AbliterationPipeline(model_name="Qwen/Qwen3.8-27B")
|
||||
pipeline.handle = SimpleNamespace(architecture=architecture)
|
||||
|
||||
with pytest.raises(PipelineValidationError, match="no validated projection allowlist"):
|
||||
pipeline._validate_architecture_surgery_support()
|
||||
|
||||
assert pipeline._excise_modified_count is None
|
||||
assert pipeline._quality_metrics["architecture_support"] == 0.0
|
||||
|
||||
def test_relative_perplexity_guardrail_fails_closed(self):
|
||||
from obliteratus.abliterate import PipelineValidationError
|
||||
|
||||
pipeline = AbliterationPipeline(
|
||||
model_name="test-model",
|
||||
max_perplexity_increase=2.0,
|
||||
)
|
||||
pipeline._stock_baseline = {"perplexity": 10.0}
|
||||
|
||||
with pytest.raises(PipelineValidationError) as exc_info:
|
||||
pipeline._enforce_perplexity_guardrail(21.0)
|
||||
|
||||
assert exc_info.value.metric == "perplexity_increase"
|
||||
assert pipeline._quality_metrics["perplexity_increase"] == pytest.approx(2.1)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("coherence", "degenerate", "metric"),
|
||||
[(0.3, 0.0, "coherence_retention"), (1.0, 0.21, "degenerate_fraction")],
|
||||
)
|
||||
def test_generation_guardrails_fail_closed(self, coherence, degenerate, metric):
|
||||
from obliteratus.abliterate import PipelineValidationError
|
||||
|
||||
pipeline = AbliterationPipeline(model_name="test-model")
|
||||
pipeline._stock_baseline = {"coherence": 1.0}
|
||||
|
||||
with pytest.raises(PipelineValidationError) as exc_info:
|
||||
pipeline._enforce_generation_guardrails(coherence, degenerate)
|
||||
|
||||
assert exc_info.value.metric == metric
|
||||
|
||||
def test_cancellation_is_terminal_and_cleanup_unloads_model(self):
|
||||
from threading import Event
|
||||
|
||||
@@ -237,6 +297,29 @@ class TestPipelineInit:
|
||||
assert pipeline.handle.model is None
|
||||
assert pipeline.handle.tokenizer is None
|
||||
|
||||
def test_public_run_cleans_up_terminal_pipeline_failure(self, monkeypatch):
|
||||
from obliteratus.abliterate import PipelineValidationError
|
||||
|
||||
pipeline = AbliterationPipeline(model_name="test-model")
|
||||
cleanup = Mock()
|
||||
monkeypatch.setattr(pipeline, "cleanup_failed_run", cleanup)
|
||||
monkeypatch.setattr(
|
||||
pipeline,
|
||||
"_run_pipeline",
|
||||
Mock(
|
||||
side_effect=PipelineValidationError(
|
||||
"unsafe",
|
||||
stage="verify",
|
||||
metric="perplexity",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(PipelineValidationError):
|
||||
pipeline.run()
|
||||
|
||||
cleanup.assert_called_once_with()
|
||||
|
||||
def test_catastrophic_perplexity_aborts_before_generation(self):
|
||||
from types import SimpleNamespace
|
||||
|
||||
|
||||
@@ -80,6 +80,23 @@ def test_refusal_max_tokens_cli_default_and_positive_override(monkeypatch):
|
||||
assert command.call_args.args[0].refusal_max_tokens == 512
|
||||
|
||||
|
||||
def test_quality_guardrail_cli_overrides(monkeypatch):
|
||||
command = Mock()
|
||||
monkeypatch.setattr(cli, "_cmd_abliterate", command)
|
||||
|
||||
cli.main([
|
||||
"abliterate", "local/model",
|
||||
"--max-perplexity-increase", "2.5",
|
||||
"--min-coherence-retention", "0.75",
|
||||
"--max-degenerate-fraction", "0.1",
|
||||
])
|
||||
|
||||
args = command.call_args.args[0]
|
||||
assert args.max_perplexity_increase == 2.5
|
||||
assert args.min_coherence_retention == 0.75
|
||||
assert args.max_degenerate_fraction == 0.1
|
||||
|
||||
|
||||
def test_trust_remote_code_requires_explicit_cli_opt_in(monkeypatch):
|
||||
command = Mock()
|
||||
monkeypatch.setattr(cli, "_cmd_abliterate", command)
|
||||
|
||||
@@ -26,6 +26,8 @@ def test_run_informed_executes_the_documented_stage_order(pipeline, monkeypatch)
|
||||
output = pipeline.output_dir
|
||||
for name in (
|
||||
"_summon",
|
||||
"_capture_stock_baseline",
|
||||
"_validate_architecture_surgery_support",
|
||||
"_probe",
|
||||
"_analyze",
|
||||
"_distill_informed",
|
||||
@@ -46,6 +48,8 @@ def test_run_informed_executes_the_documented_stage_order(pipeline, monkeypatch)
|
||||
assert result == output
|
||||
assert calls == [
|
||||
"_summon",
|
||||
"_capture_stock_baseline",
|
||||
"_validate_architecture_surgery_support",
|
||||
"_probe",
|
||||
"_analyze",
|
||||
"_distill_informed",
|
||||
|
||||
@@ -151,6 +151,8 @@ def test_full_pipeline_saves_and_reloads_a_real_offline_model(tmp_path):
|
||||
n_directions=1,
|
||||
max_seq_length=8,
|
||||
verify_sample_size=1,
|
||||
max_perplexity_increase=1000.0,
|
||||
max_degenerate_fraction=1.0,
|
||||
harmful_prompts=["harmful request"],
|
||||
harmless_prompts=["harmless request"],
|
||||
on_stage=events.append,
|
||||
@@ -160,7 +162,9 @@ def test_full_pipeline_saves_and_reloads_a_real_offline_model(tmp_path):
|
||||
assert result == output
|
||||
assert [(event.stage, event.status) for event in events] == [
|
||||
(stage, status)
|
||||
for stage in ("summon", "probe", "distill", "excise", "verify", "rebirth")
|
||||
for stage in (
|
||||
"summon", "baseline", "probe", "distill", "excise", "verify", "rebirth",
|
||||
)
|
||||
for status in ("running", "done")
|
||||
]
|
||||
assert (output / "abliteration_metadata.json").is_file()
|
||||
@@ -211,6 +215,8 @@ def test_multidirection_pipeline_restores_tiny_model_layer_norms(tmp_path):
|
||||
max_seq_length=8,
|
||||
verify_sample_size=1,
|
||||
refusal_max_tokens=1,
|
||||
max_perplexity_increase=1000.0,
|
||||
max_degenerate_fraction=1.0,
|
||||
harmful_prompts=["harmful request", "harmful answer"],
|
||||
harmless_prompts=["harmless request", "harmless answer"],
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user