diff --git a/README.md b/README.md index 8bdbd6d..a0423ec 100644 --- a/README.md +++ b/README.md @@ -549,6 +549,13 @@ obliteratus obliterate Qwen/Qwen3.8-27B --dtype bfloat16 obliteratus obliterate Qwen/Qwen3.8-27B --dtype float16 --quantization 4bit ``` +> **Qwen3.8 safety status:** loading the native BF16 checkpoint is supported, but +> permanent abliteration of its Qwen3.5 hybrid Gated DeltaNet/attention layout is +> currently unvalidated. The pipeline checks pristine perplexity and coherence, +> then stops before modifying weights with an unsupported-architecture error. +> Qwen3.8 is not considered surgery-supported until projection-family validation +> and a real post-surgery regression pass are published. + Install the optional backend before selecting a bitsandbytes mode: ```bash diff --git a/obliteratus/abliterate.py b/obliteratus/abliterate.py index fce7fcb..f550e4e 100644 --- a/obliteratus/abliterate.py +++ b/obliteratus/abliterate.py @@ -145,6 +145,15 @@ _CORPUS_CONTAMINATION_RE = re.compile( r"(?im)(?:\bbrainly\b|^\s*(?:profile|answer|answered)\s*$|^\s*\d{2}\.\d{2}\.\d{4}\s*$)" ) +_REFERENCE_TEXTS = ( + "The theory of general relativity describes gravity as a geometric property of space and time. " + "Massive objects cause a distortion in space-time, which is felt as gravity by other objects.", + "In computer science, a binary search algorithm finds the position of a target value within a " + "sorted array. It compares the target value to the middle element of the array.", + "Photosynthesis is the process by which plants convert light energy into chemical energy. " + "This process occurs primarily in the leaves of plants using chlorophyll.", +) + def _is_coherent_completion(prompt: str, completion: str) -> bool: """Return whether a completion is relevant, varied, and contamination-free.""" @@ -164,6 +173,15 @@ def _is_coherent_completion(prompt: str, completion: str) -> bool: return all(any(term in lowered for term in alternatives) for alternatives in anchors) +def _is_degenerate_completion(completion: str) -> bool: + """Detect empty, extremely repetitive, or punctuation-only generation.""" + text = completion.strip() + words = re.findall(r"[\w'-]+", text.lower()) + if not words: + return True + return len(words) >= 5 and len(set(words)) / len(words) <= 0.2 + + # ── Abliteration method presets ─────────────────────────────────────────── METHODS = { @@ -909,6 +927,9 @@ class AbliterationPipeline: # Verify stage sample size verify_sample_size: int | None = None, refusal_max_tokens: int | None = None, + max_perplexity_increase: float = 3.0, + min_coherence_retention: float = 0.5, + max_degenerate_fraction: float = 0.2, on_stage: Callable[[StageResult], None] | None = None, on_log: Callable[[str], None] | None = None, cancellation_event: Event | None = None, @@ -1057,6 +1078,30 @@ class AbliterationPipeline: ): raise ValueError("refusal_max_tokens must be a positive integer") self.refusal_max_tokens = refusal_max_tokens if refusal_max_tokens is not None else 128 + if ( + isinstance(max_perplexity_increase, bool) + or not isinstance(max_perplexity_increase, (int, float)) + or not math.isfinite(max_perplexity_increase) + or max_perplexity_increase < 1.0 + ): + raise ValueError("max_perplexity_increase must be at least 1.0") + if ( + isinstance(min_coherence_retention, bool) + or not isinstance(min_coherence_retention, (int, float)) + or not math.isfinite(min_coherence_retention) + or not 0.0 <= min_coherence_retention <= 1.0 + ): + raise ValueError("min_coherence_retention must be in [0.0, 1.0]") + if ( + isinstance(max_degenerate_fraction, bool) + or not isinstance(max_degenerate_fraction, (int, float)) + or not math.isfinite(max_degenerate_fraction) + or not 0.0 <= max_degenerate_fraction <= 1.0 + ): + raise ValueError("max_degenerate_fraction must be in [0.0, 1.0]") + self.max_perplexity_increase = float(max_perplexity_increase) + self.min_coherence_retention = float(min_coherence_retention) + self.max_degenerate_fraction = float(max_degenerate_fraction) # Large model mode: conservative defaults for 120B+ models. # Reduces memory footprint by limiting SAE features, directions, @@ -1079,6 +1124,7 @@ class AbliterationPipeline: self._harmless_means: dict[int, torch.Tensor] = {} self._shield_concept_atoms: dict[int, torch.Tensor] = {} self._quality_metrics: dict[str, float] = {} + self._stock_baseline: dict[str, float] = {} # LoRA ablation state (reversible adapters) self._lora_adapters: dict[str, tuple[torch.Tensor, torch.Tensor]] = {} @@ -1151,11 +1197,18 @@ class AbliterationPipeline: metric="cancellation", ) - def _fail_validation(self, metric: str, value: float, reason: str) -> None: + def _fail_validation( + self, + metric: str, + value: float, + reason: str, + *, + stage: str = "verify", + ) -> None: self._quality_metrics[metric] = value message = f"Validation failed: {reason} ({metric}={value!r})" - self._emit("verify", "error", message, metric=metric, value=value) - raise PipelineValidationError(message, stage="verify", metric=metric) + self._emit(stage, "error", message, metric=metric, value=value) + raise PipelineValidationError(message, stage=stage, metric=metric) def cleanup_failed_run(self) -> None: """Unload an unsafe/partial in-memory result and release transient state.""" @@ -1384,6 +1437,14 @@ class AbliterationPipeline: def run(self) -> Path: """Execute the full abliteration pipeline. Returns path to saved model.""" + try: + return self._run_pipeline() + except PipelineFailure: + self.cleanup_failed_run() + raise + + def _run_pipeline(self) -> Path: + """Execute stages while ``run`` owns terminal failure cleanup.""" # Remove any steering hooks left from a previous run() call for h in self._steering_hooks: h.remove() @@ -1391,6 +1452,10 @@ class AbliterationPipeline: self._active_stage = "summon" self._summon() self._free_gpu_memory() + self._active_stage = "baseline" + self._capture_stock_baseline() + self._validate_architecture_surgery_support() + self._free_gpu_memory() self._active_stage = "probe" self._probe() self._free_gpu_memory() @@ -1455,6 +1520,151 @@ class AbliterationPipeline: self.log(f"Total parameters: {summary['total_params']:,}") self._emit("summon", "done", f"Loaded ({elapsed:.1f}s)", duration=elapsed, **summary) + def _reference_perplexity(self) -> float: + """Measure finite reference perplexity for baseline and post-surgery gates.""" + model = self.handle.model + tokenizer = self.handle.tokenizer + device = self._get_model_device(model) + weighted_loss = 0.0 + token_count = 0 + for text in _REFERENCE_TEXTS: + inputs = tokenizer( + text, + return_tensors="pt", + truncation=True, + max_length=self.max_seq_length or 256, + ) + inputs = {key: value.to(device) for key, value in inputs.items()} + with torch.no_grad(): + outputs = model(**inputs, labels=inputs["input_ids"]) + loss = float(outputs.loss.item()) + sequence_length = inputs["input_ids"].shape[1] + del inputs, outputs + if not math.isfinite(loss): + return float("inf") + weighted_loss += loss * sequence_length + token_count += sequence_length + return math.exp(min(weighted_loss / token_count, 100.0)) if token_count else float("inf") + + def _stock_coherence(self) -> float: + """Run a small, deterministic coherence sample before any weight mutation.""" + model = self.handle.model + tokenizer = self.handle.tokenizer + device = self._get_model_device(model) + prompts = ( + "The capital of France is", + "Photosynthesis is the process by which", + "A binary search algorithm works by", + ) + coherent = 0 + for prompt in prompts: + inputs = tokenizer(prompt, return_tensors="pt") + input_length = inputs["input_ids"].shape[1] + inputs = {key: value.to(device) for key, value in inputs.items()} + with torch.no_grad(): + output = model.generate(**inputs, max_new_tokens=64, do_sample=False) + completion = tokenizer.decode( + output[0][input_length:], + skip_special_tokens=True, + ).strip()[:200] + del inputs, output + coherent += int(_is_coherent_completion(prompt, completion)) + return coherent / len(prompts) + + def _capture_stock_baseline(self) -> None: + """Prove the pristine checkpoint is healthy before surgery begins.""" + self._emit("baseline", "running", "Validating pristine checkpoint...") + perplexity = self._reference_perplexity() + if not math.isfinite(perplexity): + self._fail_validation( + "baseline_perplexity", + perplexity, + "the pristine checkpoint produces non-finite loss", + stage="baseline", + ) + try: + coherence = self._stock_coherence() + except RuntimeError as error: + if not dev.is_oom_error(error): + raise + self._free_gpu_memory() + coherence = float("nan") + self.log("Stock generation baseline skipped because the KV cache exceeded device capacity.") + self._stock_baseline = { + "perplexity": perplexity, + "coherence": coherence, + } + self._quality_metrics["baseline_perplexity"] = perplexity + self._quality_metrics["baseline_coherence"] = coherence + self._emit( + "baseline", + "done", + "Pristine checkpoint is numerically healthy.", + perplexity=perplexity, + coherence=coherence, + ) + + def _validate_architecture_surgery_support(self) -> None: + """Reject hybrid layouts whose permanent projection allowlist is unvalidated.""" + architecture = str(self.handle.architecture).lower() + if architecture in {"qwen3_5", "qwen3_5_text", "qwen3_5_moe"}: + coherence = self._stock_baseline.get("coherence") + if coherence is not None and math.isfinite(coherence) and coherence <= 0.0: + self._fail_validation( + "baseline_coherence", + coherence, + "the pristine Qwen3.5/Qwen3.8 checkpoint failed every " + "deterministic coherence prompt", + stage="baseline", + ) + self._fail_validation( + "architecture_support", + 0.0, + "Qwen3.5/Qwen3.8 hybrid Gated DeltaNet surgery has no validated " + "projection allowlist; no weights were modified", + stage="baseline", + ) + + def _enforce_perplexity_guardrail(self, perplexity: float) -> None: + baseline = self._stock_baseline.get("perplexity") + if not baseline or not math.isfinite(baseline): + return + increase = perplexity / baseline + self._quality_metrics["perplexity_increase"] = increase + if increase > self.max_perplexity_increase: + self._fail_validation( + "perplexity_increase", + increase, + f"perplexity exceeded the configured {self.max_perplexity_increase:.2f}x " + "stock-baseline limit", + ) + + def _enforce_generation_guardrails( + self, + coherence: float, + degenerate_fraction: float, + ) -> None: + self._quality_metrics["degenerate_fraction"] = degenerate_fraction + if degenerate_fraction > self.max_degenerate_fraction: + self._fail_validation( + "degenerate_fraction", + degenerate_fraction, + f"degenerate completions exceeded the configured " + f"{self.max_degenerate_fraction:.0%} limit", + ) + baseline = self._stock_baseline.get("coherence") + if not baseline or not math.isfinite(baseline): + return + retention = coherence / baseline + self._quality_metrics["coherence_retention"] = retention + if retention < self.min_coherence_retention: + self._fail_validation( + "coherence_retention", + retention, + f"coherence fell below the configured " + f"{self.min_coherence_retention:.0%} stock-baseline retention", + ) + # ── Stage 2: PROBE ────────────────────────────────────────────────── def _probe(self): @@ -6515,20 +6725,11 @@ class AbliterationPipeline: self._free_gpu_memory() # 1. Perplexity measurement on reference texts - reference_texts = [ - "The theory of general relativity describes gravity as a geometric property of space and time. " - "Massive objects cause a distortion in space-time, which is felt as gravity by other objects.", - "In computer science, a binary search algorithm finds the position of a target value within a " - "sorted array. It compares the target value to the middle element of the array.", - "Photosynthesis is the process by which plants convert light energy into chemical energy. " - "This process occurs primarily in the leaves of plants using chlorophyll.", - ] - self.log("Measuring perplexity on reference texts...") total_loss = 0.0 n_tokens = 0 has_nan_loss = False - for text in reference_texts: + for text in _REFERENCE_TEXTS: inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=self.max_seq_length or 256) inputs = {k: v.to(device) for k, v in inputs.items()} with torch.no_grad(): @@ -6574,6 +6775,7 @@ class AbliterationPipeline: perplexity, "perplexity is non-finite; generation and export are blocked", ) + self._enforce_perplexity_guardrail(perplexity) # 2. Generation coherence test test_prompts = [ @@ -6641,6 +6843,7 @@ class AbliterationPipeline: self.log("Generating test completions:") coherent_count = 0 + degenerate_count = 0 generation_failed = False for prompt in test_prompts: if generation_failed: @@ -6666,6 +6869,8 @@ class AbliterationPipeline: if _is_coherent_completion(prompt, completion): coherent_count += 1 + if _is_degenerate_completion(completion): + degenerate_count += 1 except (RuntimeError, Exception) as e: if dev.is_oom_error(e): self._free_gpu_memory() @@ -6685,7 +6890,9 @@ class AbliterationPipeline: if not generation_failed: coherence_score = coherent_count / len(test_prompts) self._quality_metrics["coherence"] = coherence_score + degenerate_fraction = degenerate_count / len(test_prompts) self.log(f" Coherence: {coherence_score:.0%} ({coherent_count}/{len(test_prompts)} prompts)") + self._enforce_generation_guardrails(coherence_score, degenerate_fraction) else: coherence_score = None self._quality_metrics["coherence"] = None diff --git a/obliteratus/cli.py b/obliteratus/cli.py index 06abf58..2905b66 100644 --- a/obliteratus/cli.py +++ b/obliteratus/cli.py @@ -317,6 +317,18 @@ def main(argv: list[str] | None = None): "--refusal-max-tokens", type=_positive_int, default=None, help="Max new tokens to generate per response in the refusal test (default: 128).", ) + p.add_argument( + "--max-perplexity-increase", type=float, default=3.0, + help="Fail if post-surgery perplexity exceeds this multiple of stock (default: 3.0).", + ) + p.add_argument( + "--min-coherence-retention", type=float, default=0.5, + help="Minimum post-surgery coherence divided by stock coherence (default: 0.5).", + ) + p.add_argument( + "--max-degenerate-fraction", type=float, default=0.2, + help="Fail if repetitive/empty completions exceed this fraction (default: 0.2).", + ) p.add_argument( "--dataset", type=str, default="builtin", help="Prompt dataset source for contrastive extraction when using residue mining (default: builtin).", @@ -1272,6 +1284,9 @@ def _cmd_abliterate(args): large_model_mode=getattr(args, "large_model", False), verify_sample_size=getattr(args, "verify_sample_size", None), refusal_max_tokens=getattr(args, "refusal_max_tokens", None), + max_perplexity_increase=getattr(args, "max_perplexity_increase", 3.0), + min_coherence_retention=getattr(args, "min_coherence_retention", 0.5), + max_degenerate_fraction=getattr(args, "max_degenerate_fraction", 0.2), on_stage=on_stage, on_log=on_log, **prompt_kwargs, diff --git a/obliteratus/informed_pipeline.py b/obliteratus/informed_pipeline.py index 562c79f..3d70d54 100644 --- a/obliteratus/informed_pipeline.py +++ b/obliteratus/informed_pipeline.py @@ -61,6 +61,7 @@ import torch from obliteratus.abliterate import ( AbliterationPipeline, + PipelineFailure, StageResult, ) @@ -184,6 +185,9 @@ class InformedAbliterationPipeline(AbliterationPipeline): on_stage: Callable[[StageResult], None] | None = None, on_log: Callable[[str], None] | None = None, cancellation_event: Event | None = None, + max_perplexity_increase: float = 3.0, + min_coherence_retention: float = 0.5, + max_degenerate_fraction: float = 0.2, # Base pipeline kwargs forwarded to AbliterationPipeline push_to_hub: str | None = None, hub_token: str | None = None, @@ -219,6 +223,9 @@ class InformedAbliterationPipeline(AbliterationPipeline): on_stage=on_stage, on_log=on_log, cancellation_event=cancellation_event, + max_perplexity_increase=max_perplexity_increase, + min_coherence_retention=min_coherence_retention, + max_degenerate_fraction=max_degenerate_fraction, push_to_hub=push_to_hub, hub_token=hub_token, hub_community_org=hub_community_org, @@ -267,12 +274,26 @@ class InformedAbliterationPipeline(AbliterationPipeline): (output_path, report) tuple with saved model path and comprehensive analysis report. """ + try: + return self._run_informed_pipeline() + except PipelineFailure: + self.cleanup_failed_run() + raise + + def _run_informed_pipeline(self) -> tuple[Path, InformedPipelineReport]: + """Execute informed stages while the public method owns cleanup.""" t0 = time.time() # Stage 1: SUMMON self._active_stage = "summon" self._summon() + # Prove the pristine checkpoint is healthy and the permanent surgery + # layout is supported before analysis can lead to weight mutation. + self._active_stage = "baseline" + self._capture_stock_baseline() + self._validate_architecture_surgery_support() + # Stage 2: PROBE self._active_stage = "probe" self._probe() diff --git a/tests/conditional/test_model_download_runtime.py b/tests/conditional/test_model_download_runtime.py index 699de55..3015466 100644 --- a/tests/conditional/test_model_download_runtime.py +++ b/tests/conditional/test_model_download_runtime.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os import uuid import pytest @@ -17,6 +18,7 @@ MODEL = "hf-internal-testing/tiny-random-gpt2" REVISION = "71034c5d8bde858ff824298bdedc65515b97d2b9" MISTRAL4_MODEL = "mistralai/Mistral-Small-4-119B-2603" MISTRAL4_REVISION = "a11f36bebf709121056b1dbcc943d1c6afbe494d" +QWEN38_MODEL = "Qwen/Qwen3.8-27B" def test_pinned_tiny_model_download_inference_and_offline_cache(monkeypatch): @@ -71,3 +73,26 @@ def test_pinned_mistral4_config_resolves_composite_contract_without_remote_code( assert profile.model_type == "mistral4" assert profile.arch_class is ArchitectureClass.LARGE_MOE assert (profile.num_experts, profile.num_active_experts) == (128, 4) + + +@pytest.mark.gpu +def test_qwen38_bf16_pristine_baseline_blocks_unvalidated_surgery(tmp_path): + """Operator-gated 27B regression: healthy stock model, zero modified weights.""" + if os.environ.get("OBLITERATUS_QWEN38_E2E") != "1": + pytest.skip("set OBLITERATUS_QWEN38_E2E=1 on a >=80 GiB GPU runner") + + from obliteratus.abliterate import AbliterationPipeline, PipelineValidationError + + pipeline = AbliterationPipeline( + QWEN38_MODEL, + output_dir=str(tmp_path / "qwen38-output"), + dtype="bfloat16", + quantization=None, + ) + pipeline._summon() + pipeline._capture_stock_baseline() + assert pipeline._stock_baseline["perplexity"] > 0 + assert pipeline._stock_baseline["coherence"] > 0 + with pytest.raises(PipelineValidationError, match="no validated projection allowlist"): + pipeline._validate_architecture_surgery_support() + assert pipeline._excise_modified_count is None diff --git a/tests/test_abliterate.py b/tests/test_abliterate.py index ca3c51f..227d59c 100644 --- a/tests/test_abliterate.py +++ b/tests/test_abliterate.py @@ -215,8 +215,68 @@ class TestPipelineInit: assert pipeline.trust_remote_code is False assert pipeline.gpu_memory_utilization is None assert pipeline.refusal_max_tokens == 128 + assert pipeline.max_perplexity_increase == 3.0 + assert pipeline.min_coherence_retention == 0.5 + assert pipeline.max_degenerate_fraction == 0.2 assert pipeline.handle is None + @pytest.mark.parametrize( + ("keyword", "value"), + [ + ("max_perplexity_increase", 0.99), + ("min_coherence_retention", 1.01), + ("max_degenerate_fraction", -0.01), + ], + ) + def test_quality_guardrails_validate_configuration(self, keyword, value): + with pytest.raises(ValueError): + AbliterationPipeline(model_name="test-model", **{keyword: value}) + + @pytest.mark.parametrize("architecture", ["qwen3_5", "qwen3_5_text", "qwen3_5_moe"]) + def test_unvalidated_qwen35_hybrid_fails_before_surgery(self, architecture): + from types import SimpleNamespace + + from obliteratus.abliterate import PipelineValidationError + + pipeline = AbliterationPipeline(model_name="Qwen/Qwen3.8-27B") + pipeline.handle = SimpleNamespace(architecture=architecture) + + with pytest.raises(PipelineValidationError, match="no validated projection allowlist"): + pipeline._validate_architecture_surgery_support() + + assert pipeline._excise_modified_count is None + assert pipeline._quality_metrics["architecture_support"] == 0.0 + + def test_relative_perplexity_guardrail_fails_closed(self): + from obliteratus.abliterate import PipelineValidationError + + pipeline = AbliterationPipeline( + model_name="test-model", + max_perplexity_increase=2.0, + ) + pipeline._stock_baseline = {"perplexity": 10.0} + + with pytest.raises(PipelineValidationError) as exc_info: + pipeline._enforce_perplexity_guardrail(21.0) + + assert exc_info.value.metric == "perplexity_increase" + assert pipeline._quality_metrics["perplexity_increase"] == pytest.approx(2.1) + + @pytest.mark.parametrize( + ("coherence", "degenerate", "metric"), + [(0.3, 0.0, "coherence_retention"), (1.0, 0.21, "degenerate_fraction")], + ) + def test_generation_guardrails_fail_closed(self, coherence, degenerate, metric): + from obliteratus.abliterate import PipelineValidationError + + pipeline = AbliterationPipeline(model_name="test-model") + pipeline._stock_baseline = {"coherence": 1.0} + + with pytest.raises(PipelineValidationError) as exc_info: + pipeline._enforce_generation_guardrails(coherence, degenerate) + + assert exc_info.value.metric == metric + def test_cancellation_is_terminal_and_cleanup_unloads_model(self): from threading import Event @@ -237,6 +297,29 @@ class TestPipelineInit: assert pipeline.handle.model is None assert pipeline.handle.tokenizer is None + def test_public_run_cleans_up_terminal_pipeline_failure(self, monkeypatch): + from obliteratus.abliterate import PipelineValidationError + + pipeline = AbliterationPipeline(model_name="test-model") + cleanup = Mock() + monkeypatch.setattr(pipeline, "cleanup_failed_run", cleanup) + monkeypatch.setattr( + pipeline, + "_run_pipeline", + Mock( + side_effect=PipelineValidationError( + "unsafe", + stage="verify", + metric="perplexity", + ), + ), + ) + + with pytest.raises(PipelineValidationError): + pipeline.run() + + cleanup.assert_called_once_with() + def test_catastrophic_perplexity_aborts_before_generation(self): from types import SimpleNamespace diff --git a/tests/test_cli_boundaries.py b/tests/test_cli_boundaries.py index cc6285a..2ac834b 100644 --- a/tests/test_cli_boundaries.py +++ b/tests/test_cli_boundaries.py @@ -80,6 +80,23 @@ def test_refusal_max_tokens_cli_default_and_positive_override(monkeypatch): assert command.call_args.args[0].refusal_max_tokens == 512 +def test_quality_guardrail_cli_overrides(monkeypatch): + command = Mock() + monkeypatch.setattr(cli, "_cmd_abliterate", command) + + cli.main([ + "abliterate", "local/model", + "--max-perplexity-increase", "2.5", + "--min-coherence-retention", "0.75", + "--max-degenerate-fraction", "0.1", + ]) + + args = command.call_args.args[0] + assert args.max_perplexity_increase == 2.5 + assert args.min_coherence_retention == 0.75 + assert args.max_degenerate_fraction == 0.1 + + def test_trust_remote_code_requires_explicit_cli_opt_in(monkeypatch): command = Mock() monkeypatch.setattr(cli, "_cmd_abliterate", command) diff --git a/tests/test_informed_pipeline_contracts.py b/tests/test_informed_pipeline_contracts.py index defbbc3..8401e63 100644 --- a/tests/test_informed_pipeline_contracts.py +++ b/tests/test_informed_pipeline_contracts.py @@ -26,6 +26,8 @@ def test_run_informed_executes_the_documented_stage_order(pipeline, monkeypatch) output = pipeline.output_dir for name in ( "_summon", + "_capture_stock_baseline", + "_validate_architecture_surgery_support", "_probe", "_analyze", "_distill_informed", @@ -46,6 +48,8 @@ def test_run_informed_executes_the_documented_stage_order(pipeline, monkeypatch) assert result == output assert calls == [ "_summon", + "_capture_stock_baseline", + "_validate_architecture_surgery_support", "_probe", "_analyze", "_distill_informed", diff --git a/tests/test_offline_integration.py b/tests/test_offline_integration.py index cdc0c3b..58e383d 100644 --- a/tests/test_offline_integration.py +++ b/tests/test_offline_integration.py @@ -151,6 +151,8 @@ def test_full_pipeline_saves_and_reloads_a_real_offline_model(tmp_path): n_directions=1, max_seq_length=8, verify_sample_size=1, + max_perplexity_increase=1000.0, + max_degenerate_fraction=1.0, harmful_prompts=["harmful request"], harmless_prompts=["harmless request"], on_stage=events.append, @@ -160,7 +162,9 @@ def test_full_pipeline_saves_and_reloads_a_real_offline_model(tmp_path): assert result == output assert [(event.stage, event.status) for event in events] == [ (stage, status) - for stage in ("summon", "probe", "distill", "excise", "verify", "rebirth") + for stage in ( + "summon", "baseline", "probe", "distill", "excise", "verify", "rebirth", + ) for status in ("running", "done") ] assert (output / "abliteration_metadata.json").is_file() @@ -211,6 +215,8 @@ def test_multidirection_pipeline_restores_tiny_model_layer_norms(tmp_path): max_seq_length=8, verify_sample_size=1, refusal_max_tokens=1, + max_perplexity_increase=1000.0, + max_degenerate_fraction=1.0, harmful_prompts=["harmful request", "harmful answer"], harmless_prompts=["harmless request", "harmless answer"], ) @@ -356,6 +362,10 @@ def test_installed_package_cli_executes_offline_checkpoint_to_report_slice(tmp_p "1", "--refusal-max-tokens", "1", + "--max-perplexity-increase", + "1000", + "--max-degenerate-fraction", + "1", "--prompt-pairs-file", str(prompt_pairs_path), ],