fix(pipeline): fail closed on invalid output

tests/test_abliterate.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import json
+import math
 from pathlib import Path
 from unittest.mock import MagicMock, Mock, patch
 
@@ -216,6 +217,70 @@ class TestPipelineInit:
         assert pipeline.refusal_max_tokens == 128
         assert pipeline.handle is None
 
+    def test_cancellation_is_terminal_and_cleanup_unloads_model(self):
+        from threading import Event
+
+        from obliteratus.abliterate import PipelineCancelledError
+
+        cancellation = Event()
+        pipeline = AbliterationPipeline(
+            model_name="test-model",
+            cancellation_event=cancellation,
+        )
+        pipeline.handle = MagicMock()
+        cancellation.set()
+
+        with pytest.raises(PipelineCancelledError, match="cancelled during PROBE"):
+            pipeline._check_cancelled("probe")
+
+        pipeline.cleanup_failed_run()
+        assert pipeline.handle.model is None
+        assert pipeline.handle.tokenizer is None
+
+    def test_catastrophic_perplexity_aborts_before_generation(self):
+        from types import SimpleNamespace
+
+        from obliteratus.abliterate import PipelineValidationError
+
+        class NonFiniteModel(torch.nn.Module):
+            def __init__(self):
+                super().__init__()
+                self.anchor = torch.nn.Parameter(torch.zeros(1))
+                self.generate_calls = 0
+
+            def forward(self, **_kwargs):
+                return SimpleNamespace(loss=torch.tensor(float("nan")))
+
+            def generate(self, **_kwargs):
+                self.generate_calls += 1
+                raise AssertionError("generation must not run after catastrophic validation")
+
+        class Tokenizer:
+            def __call__(self, _text, **_kwargs):
+                return {
+                    "input_ids": torch.tensor([[1, 2, 3]]),
+                    "attention_mask": torch.ones((1, 3), dtype=torch.long),
+                }
+
+        model = NonFiniteModel()
+        pipeline = AbliterationPipeline(model_name="test-model")
+        pipeline.handle = SimpleNamespace(model=model, tokenizer=Tokenizer())
+        stages = []
+        pipeline._on_stage = stages.append
+
+        with pytest.raises(
+            PipelineValidationError,
+            match="all reference losses were non-finite",
+        ) as exc_info:
+            pipeline._verify()
+
+        assert exc_info.value.stage == "verify"
+        assert exc_info.value.metric == "perplexity"
+        assert math.isinf(pipeline._quality_metrics["perplexity"])
+        assert model.generate_calls == 0
+        assert stages[-1].status == "error"
+        assert stages[-1].details["metric"] == "perplexity"
+
     @pytest.mark.parametrize("invalid", [0, -1, 1.5, True])
     def test_refusal_max_tokens_must_be_positive_integer(self, invalid):
         with pytest.raises(
@@ -2063,6 +2128,36 @@ class TestExcise:
 # ---------------------------------------------------------------------------
 
 class TestRebirth:
+    def test_rebirth_cancellation_discards_staging_before_promotion(self, handle, tmp_path):
+        from threading import Event
+
+        from obliteratus.abliterate import PipelineCancelledError
+
+        cancellation = Event()
+        output = tmp_path / "output"
+        pipeline = AbliterationPipeline(
+            model_name="test-model",
+            output_dir=str(output),
+            cancellation_event=cancellation,
+        )
+        pipeline.handle = handle
+        pipeline._strong_layers = [0]
+
+        def save_then_cancel(path, **_kwargs):
+            (Path(path) / "config.json").write_text("{}", encoding="utf-8")
+            (Path(path) / "model.safetensors").write_bytes(b"partial")
+            cancellation.set()
+
+        handle.model.save_pretrained = MagicMock(side_effect=save_then_cancel)
+        handle.tokenizer.save_pretrained = MagicMock()
+
+        with pytest.raises(PipelineCancelledError, match="cancelled during REBIRTH"):
+            pipeline._rebirth()
+
+        assert not output.exists()
+        assert list(tmp_path.glob(".output.staging-*")) == []
+        handle.tokenizer.save_pretrained.assert_not_called()
+
     def test_rebirth_saves_metadata(self, handle, tmp_path):
         """Rebirth should save model and comprehensive metadata JSON."""
         pipeline = AbliterationPipeline(

tests/test_app_model_lifecycle.py

@@ -72,3 +72,100 @@ assert app._state["output_dir"] is None
     )
 
     assert result.returncode == 0, result.stdout + result.stderr
+
+
+@pytest.mark.operator_ui
+def test_pipeline_cancellation_and_local_checkpoint_reload_contracts(tmp_path):
+    """Cover timeout grace outcomes and local-only reload classification."""
+    script = r'''
+import pathlib
+import sys
+import threading
+import time
+from unittest.mock import Mock
+
+import app
+
+root = pathlib.Path(sys.argv[1])
+checkpoint = root / "completed"
+checkpoint.mkdir()
+(checkpoint / "abliteration_metadata.json").write_text("{}", encoding="utf-8")
+(checkpoint / "config.json").write_text("{}", encoding="utf-8")
+(checkpoint / "tokenizer_config.json").write_text("{}", encoding="utf-8")
+(checkpoint / "model.safetensors").write_bytes(b"weights")
+assert app._resolve_local_checkpoint(checkpoint) == checkpoint.resolve()
+
+calls = []
+model = Mock()
+model.to.return_value = model
+tokenizer = Mock(pad_token=None, eos_token="<eos>")
+app.AutoModelForCausalLM.from_pretrained = Mock(
+    side_effect=lambda source, **kwargs: (calls.append((source, kwargs)), model)[1]
+)
+app.AutoTokenizer.from_pretrained = Mock(return_value=tokenizer)
+app.dev.supports_device_map_auto = lambda: True
+app._load_model_to_device(checkpoint, local_files_only=True)
+assert calls[0][0] == checkpoint
+assert calls[0][1]["local_files_only"] is True
+
+quantization = object()
+loaded_model, loaded_tokenizer = app._reload_local_checkpoint(
+    checkpoint,
+    trust_remote_code=True,
+    model_kwargs={"quantization_config": quantization},
+)
+assert loaded_model is model and loaded_tokenizer is tokenizer
+assert calls[-1][1]["quantization_config"] is quantization
+assert calls[-1][1]["local_files_only"] is True
+assert tokenizer.pad_token == "<eos>"
+
+app._reload_local_checkpoint(
+    checkpoint,
+    trust_remote_code=True,
+    model_kwargs={"offload_folder": str(root / "offload")},
+)
+assert calls[-1][1]["offload_folder"] == str(root / "offload")
+assert calls[-1][1]["local_files_only"] is True
+
+assert app._format_checkpoint_reload_error(FileNotFoundError("gone")).startswith(
+    "Saved checkpoint is missing"
+)
+assert app._format_checkpoint_reload_error(ValueError("Checkpoint has no model weights")).startswith(
+    "Saved checkpoint is incomplete"
+)
+assert app._format_checkpoint_reload_error(ValueError("Checkpoint metadata is corrupt")).startswith(
+    "Saved checkpoint is corrupt"
+)
+assert app._format_checkpoint_reload_error(ValueError("Repo id is invalid")).startswith(
+    "Checkpoint Hub resolution failed"
+)
+
+def run_cooperative(cancel, delay):
+    cancel.wait()
+    time.sleep(delay)
+
+cancel = threading.Event()
+worker = threading.Thread(target=run_cooperative, args=(cancel, 0.02))
+worker.start()
+assert app._cancel_pipeline_worker(worker, cancel, grace_seconds=1.0) is True
+assert cancel.is_set()
+
+cancel = threading.Event()
+release = threading.Event()
+worker = threading.Thread(target=lambda: release.wait())
+worker.start()
+assert app._cancel_pipeline_worker(worker, cancel, grace_seconds=0.01) is False
+assert cancel.is_set() and worker.is_alive()
+release.set()
+worker.join(timeout=1.0)
+assert not worker.is_alive()
+'''
+
+    result = subprocess.run(
+        [sys.executable, "-c", script, str(tmp_path)],
+        capture_output=True,
+        text=True,
+        timeout=120,
+        check=False,
+    )
+    assert result.returncode == 0, result.stdout + result.stderr

tests/test_persistence_contracts.py

@@ -31,6 +31,33 @@ def _write_valid_local_checkpoint(
     (checkpoint_dir / "model.safetensors").write_bytes(b"weights")
 
 
+def test_reloadable_checkpoint_resolves_absolute_and_relative_paths(tmp_path, monkeypatch):
+    checkpoint = tmp_path / "completed"
+    checkpoint.mkdir()
+    _write_valid_local_checkpoint(checkpoint)
+
+    assert persistence.validate_reloadable_checkpoint(checkpoint) == checkpoint.resolve()
+    monkeypatch.chdir(tmp_path)
+    assert persistence.validate_reloadable_checkpoint("completed") == checkpoint.resolve()
+
+
+def test_reloadable_checkpoint_rejects_absent_incomplete_and_corrupt_paths(tmp_path):
+    with pytest.raises(FileNotFoundError, match="not a directory"):
+        persistence.validate_reloadable_checkpoint(tmp_path / "absent")
+
+    incomplete = tmp_path / "incomplete"
+    incomplete.mkdir()
+    (incomplete / "abliteration_metadata.json").write_text("{}", encoding="utf-8")
+    with pytest.raises(ValueError, match="model config is missing"):
+        persistence.validate_reloadable_checkpoint(incomplete)
+
+    corrupt = tmp_path / "corrupt"
+    corrupt.mkdir()
+    (corrupt / "abliteration_metadata.json").write_text("not-json", encoding="utf-8")
+    with pytest.raises(ValueError, match="metadata is corrupt"):
+        persistence.validate_reloadable_checkpoint(corrupt)
+
+
 @pytest.mark.parametrize(
     ("state_dict", "expected"),
     [