diff --git a/obliteratus/abliterate.py b/obliteratus/abliterate.py index 5883263..503599e 100644 --- a/obliteratus/abliterate.py +++ b/obliteratus/abliterate.py @@ -164,9 +164,9 @@ def _is_coherent_completion(prompt: str, completion: str) -> bool: """Return whether a completion is relevant, varied, and contamination-free.""" text = completion.strip() words = re.findall(r"[\w'-]+", text.lower()) - if len(text) <= 5 or len(words) < 3: + if not text or not words: return False - if len(set(words)) / len(words) <= 0.2: + if len(words) >= 5 and len(set(words)) / len(words) <= 0.2: return False if _CORPUS_CONTAMINATION_RE.search(text): return False @@ -1685,7 +1685,7 @@ class AbliterationPipeline: completion = tokenizer.decode( output[0][input_length:], skip_special_tokens=True, - ).strip()[:200] + ).strip() del inputs, output coherent += int(_is_coherent_completion(prompt, completion)) return coherent / len(prompts) @@ -1772,6 +1772,7 @@ class AbliterationPipeline: degenerate_fraction: float, ) -> None: self._quality_metrics["degenerate_fraction"] = degenerate_fraction + self._quality_metrics["coherence_degenerate_fraction"] = degenerate_fraction if degenerate_fraction > self.max_degenerate_fraction: self._fail_validation( "degenerate_fraction", @@ -7061,10 +7062,10 @@ class AbliterationPipeline: completion_ids = output[0][input_len:] completion = tokenizer.decode( completion_ids, skip_special_tokens=True, - ).strip()[:200] + ).strip() del inputs, output self._free_gpu_memory() - self.log(f' "{prompt}" -> {completion}') + self.log(f' "{prompt}" -> {completion[:200]}') if _is_coherent_completion(prompt, completion): coherent_count += 1 @@ -7285,6 +7286,10 @@ class AbliterationPipeline: # Count degenerate outputs (broken model indicator) n_degenerate = sum(1 for _, _, reason in prompt_verdicts if reason == "DEGENERATE") + self._quality_metrics["harmful_degenerate_count"] = n_degenerate + self._quality_metrics["harmful_degenerate_fraction"] = ( + n_degenerate / n_tested if n_tested else 0.0 + ) if n_degenerate > 0: self.log(f" WARNING: {n_degenerate}/{n_tested} responses are degenerate " f"(repetitive/broken output) — model may be damaged") diff --git a/tests/test_abliterate.py b/tests/test_abliterate.py index 745ee42..5398341 100644 --- a/tests/test_abliterate.py +++ b/tests/test_abliterate.py @@ -118,6 +118,14 @@ class TestCoherenceScoring: def test_accepts_relevant_completion(self, prompt, completion): assert _is_coherent_completion(prompt, completion) is True + def test_accepts_concise_anchored_completion(self): + assert _is_coherent_completion("The capital of France is", "Paris") is True + + def test_scores_anchors_after_log_preview_boundary(self): + completion = "The exact defined value follows. " + ("context " * 30) + "299,792,458 m/s" + assert len(completion) > 200 + assert _is_coherent_completion("The speed of light in a vacuum is", completion) is True + def test_rejects_fluent_but_irrelevant_completion(self): assert _is_coherent_completion( "The capital of France is", @@ -305,6 +313,7 @@ class TestPipelineInit: pipeline._enforce_generation_guardrails(coherence, degenerate) assert exc_info.value.metric == metric + assert pipeline._quality_metrics["coherence_degenerate_fraction"] == degenerate def test_cancellation_is_terminal_and_cleanup_unloads_model(self): from threading import Event