From bf8688e13e329f48341650193ad8a8e508cd8891 Mon Sep 17 00:00:00 2001
From: Agentirish <ai@vexusfiber.net>
Date: Tue, 4 Aug 2026 11:17:26 -0500
Subject: [PATCH] Fix causal LM evaluation and report output creation

---
 examples/gpt2_gpu_quick.yaml        | 23 +++++++
 obliteratus/cli.py                  |  1 +
 obliteratus/evaluation/evaluator.py | 93 +++++++++++++++++++++++++++--
 3 files changed, 111 insertions(+), 6 deletions(-)
 create mode 100644 examples/gpt2_gpu_quick.yaml

diff --git a/examples/gpt2_gpu_quick.yaml b/examples/gpt2_gpu_quick.yaml
new file mode 100644
index 0000000..1e16d04
--- /dev/null
+++ b/examples/gpt2_gpu_quick.yaml
@@ -0,0 +1,23 @@
+model:
+  name: gpt2
+  task: causal_lm
+  dtype: float16
+  device: cuda
+
+dataset:
+  name: Salesforce/wikitext
+  subset: wikitext-2-raw-v1
+  split: test
+  text_column: text
+  max_samples: 25
+
+strategies:
+  - name: layer_removal
+    params: {}
+
+metrics:
+  - perplexity
+
+batch_size: 4
+max_length: 256
+output_dir: results/gpt2_gpu_quick
diff --git a/obliteratus/cli.py b/obliteratus/cli.py
index 98548bf..4bb9996 100644
--- a/obliteratus/cli.py
+++ b/obliteratus/cli.py
@@ -762,6 +762,7 @@ def _cmd_report(args):
     report.print_summary()
 
     output_dir = Path(args.output_dir) if args.output_dir else path.parent
+    output_dir.mkdir(parents=True, exist_ok=True)
     metric_name = list(data["baseline_metrics"].keys())[0]
     try:
         report.plot_impact(metric=metric_name, output_path=output_dir / "impact.png")
diff --git a/obliteratus/evaluation/evaluator.py b/obliteratus/evaluation/evaluator.py
index b5bf6c9..d17615b 100644
--- a/obliteratus/evaluation/evaluator.py
+++ b/obliteratus/evaluation/evaluator.py
@@ -50,20 +50,54 @@ class Evaluator:
             raise ValueError(f"Unsupported task: {self.handle.task}")
 
     def _evaluate_causal_lm(self) -> dict[str, float]:
-
         model = self.handle.model
         tokenizer = self.handle.tokenizer
         device = next(model.parameters()).device
 
         ds = self.dataset
+
+        # WikiText and similar corpora contain empty separator rows.
+        # Filter them before max_samples so max_samples means usable texts.
+        raw_texts = ds[self.text_column]
+        valid_indices = [
+            index
+            for index, text in enumerate(raw_texts)
+            if isinstance(text, str) and text.strip()
+        ]
+
+        if not valid_indices:
+            raise ValueError(
+                f"Dataset contains no non-empty text in column "
+                f"{self.text_column!r}."
+            )
+
+        ds = ds.select(valid_indices)
+
         if self.max_samples is not None:
             ds = ds.select(range(min(self.max_samples, len(ds))))
 
         total_loss = 0.0
         total_tokens = 0
+        skipped_batches = 0
 
-        for i in tqdm(range(0, len(ds), self.batch_size), desc="Evaluating PPL"):
+        for i in tqdm(
+            range(0, len(ds), self.batch_size),
+            desc="Evaluating PPL",
+        ):
             batch_texts = ds[i : i + self.batch_size][self.text_column]
+
+            # Defensive filtering in case a custom dataset returns unexpected
+            # values after selection or transformation.
+            batch_texts = [
+                text
+                for text in batch_texts
+                if isinstance(text, str) and text.strip()
+            ]
+
+            if not batch_texts:
+                skipped_batches += 1
+                continue
+
             encodings = tokenizer(
                 batch_texts,
                 return_tensors="pt",
@@ -75,15 +109,62 @@ class Evaluator:
             input_ids = encodings["input_ids"]
             attention_mask = encodings["attention_mask"]
 
-            outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=input_ids)
-            # Mask out padding tokens for loss computation
-            num_tokens = attention_mask[:, 1:].sum().item()
+            # A causal-LM loss requires at least one prediction target after
+            # shifting. A sequence length below two cannot contribute.
+            if (
+                input_ids.ndim != 2
+                or input_ids.numel() == 0
+                or input_ids.shape[1] < 2
+            ):
+                skipped_batches += 1
+                continue
+
+            labels = input_ids.clone()
+
+            # Hugging Face causal-LM losses ignore labels set to -100.
+            # This prevents padded positions from affecting perplexity.
+            labels[attention_mask == 0] = -100
+
+            num_tokens = labels[:, 1:].ne(-100).sum().item()
+
+            if num_tokens <= 0:
+                skipped_batches += 1
+                continue
+
+            outputs = model(
+                input_ids=input_ids,
+                attention_mask=attention_mask,
+                labels=labels,
+            )
+
+            if outputs.loss is None:
+                raise RuntimeError(
+                    "The causal language model returned no loss value."
+                )
+
+            if not torch.isfinite(outputs.loss):
+                raise RuntimeError(
+                    f"Non-finite evaluation loss encountered at batch "
+                    f"offset {i}: {outputs.loss.item()}"
+                )
+
             total_loss += outputs.loss.item() * num_tokens
             total_tokens += num_tokens
 
+        if total_tokens <= 0:
+            raise RuntimeError(
+                "Perplexity evaluation produced zero valid prediction tokens. "
+                "Check the dataset text column and tokenizer configuration."
+            )
+
+        if skipped_batches:
+            tqdm.write(
+                f"Skipped {skipped_batches} empty or too-short batch(es)."
+            )
+
         import math
 
-        avg_loss = total_loss / max(total_tokens, 1)
+        avg_loss = total_loss / total_tokens
         return {"perplexity": math.exp(avg_loss)}
 
     def _evaluate_classification(self) -> dict[str, float]: