test: enforce research integrity contracts

.github/workflows/ci.yml

@@ -289,6 +289,11 @@ jobs:
           --min-file obliteratus/architecture_profiles.py=70
           --min-file obliteratus/cli.py=70
           --min-file obliteratus/mlx_backend.py=70
+          --min-file obliteratus/evaluation/metrics.py=70
+          --min-file obliteratus/evaluation/advanced_metrics.py=70
+          --min-file obliteratus/reporting/report.py=70
+          --min-file obliteratus/community.py=70
+          --min-file obliteratus/telemetry.py=70
           --min-changed 90
           --base-ref "$COVERAGE_BASE"
 

CONTRIBUTING.md

@@ -34,9 +34,10 @@ python -m obliteratus --help
 All tests must pass before submitting a PR. Tests are designed to run on CPU without downloading models.
 The mandatory gate currently requires at least 60% repository statement coverage,
 42% branch coverage, 90% coverage of changed executable lines, and 70% statement
-coverage in the device, loader, architecture-profile, CLI, and simulated MLX
-boundary modules. New changes should raise these floors rather than consume the
-existing margin.
+coverage in the device, loader, architecture-profile, CLI, simulated MLX,
+evaluation-metric, reporting, community-contribution, and telemetry boundary
+modules. New changes should raise these floors rather than consume the existing
+margin.
 
 ## Code Style
 

obliteratus/community.py

@@ -33,7 +33,10 @@ from obliteratus.telemetry import (
     _extract_prompt_counts,
     _extract_stage_durations,
     _get_peak_vram,
+    _metric_value,
     _safe_float,
+    _sanitize_public_text,
+    _sanitize_public_value,
     build_report,
 )
 
@@ -57,10 +60,26 @@ def _model_short_name(model_name: str) -> str:
 
 def _config_fingerprint(config: dict[str, Any]) -> str:
     """Deterministic short hash of the method configuration."""
-    canonical = json.dumps(config, sort_keys=True, default=str)
+    canonical = json.dumps(_sanitize_public_value(config), sort_keys=True, allow_nan=False)
     return hashlib.sha256(canonical.encode()).hexdigest()[:8]
 
 
+def validate_contribution(data: Any) -> bool:
+    """Return whether a contribution matches the supported public schema."""
+    if not isinstance(data, dict):
+        return False
+    if data.get("contribution_schema_version") != CONTRIBUTION_SCHEMA_VERSION:
+        return False
+    telemetry = data.get("telemetry")
+    return (
+        isinstance(data.get("model_name"), str)
+        and isinstance(data.get("timestamp"), str)
+        and isinstance(telemetry, dict)
+        and telemetry.get("schema_version") == 2
+        and isinstance(telemetry.get("quality_metrics", {}), dict)
+    )
+
+
 def save_contribution(
     pipeline,
     *,
@@ -153,20 +172,23 @@ def save_contribution(
     contribution = {
         "contribution_schema_version": CONTRIBUTION_SCHEMA_VERSION,
         "timestamp": timestamp,
-        "model_name": model_name,
+        "model_name": _sanitize_public_text(model_name, 120),
         "config_fingerprint": _config_fingerprint(method_config),
-        "notes": notes,
+        "notes": _sanitize_public_text(notes, 240),
         "telemetry": base_report,
     }
 
     # Generate filename
     short_name = _model_short_name(model_name)
-    method = pipeline.method
+    method = _model_short_name(str(pipeline.method)) or "unknown"
     ts_short = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
     filename = f"{short_name}_{method}_{ts_short}.json"
     filepath = output_dir / filename
 
-    filepath.write_text(json.dumps(contribution, indent=2, default=str))
+    filepath.write_text(
+        json.dumps(contribution, indent=2, sort_keys=True, allow_nan=False) + "\n",
+        encoding="utf-8",
+    )
     logger.info("Community contribution saved: %s", filepath)
     return filepath
 
@@ -190,8 +212,8 @@ def load_contributions(
     for path in sorted(contrib_dir.glob("*.json")):
         try:
             data = json.loads(path.read_text())
-            if "contribution_schema_version" in data:
-                data["_source_file"] = str(path)
+            if validate_contribution(data):
+                data["_source_file"] = path.name
                 records.append(data)
         except (json.JSONDecodeError, OSError) as e:
             logger.warning("Skipping invalid contribution file %s: %s", path, e)
@@ -216,9 +238,9 @@ def aggregate_results(
     groups: dict[tuple[str, str], list[dict]] = {}
 
     for record in records:
-        model = record.get("model_name", "unknown")
+        model = _sanitize_public_text(record.get("model_name", "unknown"), 120)
         telemetry = record.get("telemetry", {})
-        method = telemetry.get("method", "unknown")
+        method = _sanitize_public_text(telemetry.get("method", "unknown"), 80)
         metrics = telemetry.get("quality_metrics", {})
 
         key = (model, method)
@@ -235,17 +257,29 @@ def aggregate_results(
 
         for metric_name in ["refusal_rate", "perplexity", "coherence"]:
             values = [
-                m[metric_name]
-                for m in metric_list
-                if metric_name in m and m[metric_name] is not None
+                value
+                for metrics in metric_list
+                if (value := _metric_value(metric_name, metrics.get(metric_name))) is not None
             ]
             if values:
                 summary[metric_name] = {
+                    "status": "measured",
                     "mean": round(statistics.mean(values), 4),
                     "std": round(statistics.stdev(values), 4) if len(values) > 1 else 0.0,
                     "n": len(values),
                     "min": round(min(values), 4),
                     "max": round(max(values), 4),
+                    "unavailable": len(metric_list) - len(values),
+                }
+            else:
+                summary[metric_name] = {
+                    "status": "unavailable",
+                    "mean": None,
+                    "std": None,
+                    "n": 0,
+                    "min": None,
+                    "max": None,
+                    "unavailable": len(metric_list),
                 }
 
         results[model][method] = summary
@@ -274,8 +308,15 @@ def generate_latex_table(
             all_methods.update(model_data.keys())
         methods = sorted(all_methods)
 
+    def _escape(value: str) -> str:
+        translations = {
+            "\\": r"\textbackslash{}", "&": r"\&", "%": r"\%", "$": r"\$",
+            "#": r"\#", "_": r"\_", "{": r"\{", "}": r"\}",
+        }
+        return "".join(translations.get(char, char) for char in value)
+
     # Build header
-    method_cols = " & ".join(f"\\textbf{{{m}}}" for m in methods)
+    method_cols = " & ".join(f"\\textbf{{{_escape(m)}}}" for m in methods)
     header = f"\\textbf{{Model}} & {method_cols} \\\\"
 
     lines = [
@@ -291,7 +332,11 @@ def generate_latex_table(
 
         cells = []
         for method in methods:
-            if method in model_data and metric in model_data[method]:
+            if (
+                method in model_data
+                and metric in model_data[method]
+                and model_data[method][metric].get("status", "measured") == "measured"
+            ):
                 stats = model_data[method][metric]
                 mean = stats["mean"]
                 n = stats["n"]
@@ -302,7 +347,7 @@ def generate_latex_table(
             else:
                 cells.append("---")
 
-        row = f"{short} & " + " & ".join(cells) + " \\\\"
+        row = f"{_escape(short)} & " + " & ".join(cells) + " \\\\"
         lines.append(row)
 
     lines.extend(["\\bottomrule", "\\end{tabular}"])

obliteratus/evaluation/advanced_metrics.py

@@ -31,6 +31,7 @@ References:
 
 from __future__ import annotations
 
+import math
 import re
 from dataclasses import dataclass
 
@@ -184,6 +185,37 @@ REFUSAL_SUBSTRINGS = [
     "не могу помочь",      # Russian: can't help
 ]
 
+_REFUSAL_MODES = frozenset({"prefix", "substring", "semantic", "combined"})
+
+
+def _validate_refusal_mode(mode: str) -> None:
+    if mode not in _REFUSAL_MODES:
+        raise ValueError(
+            f"Unknown refusal detection mode {mode!r}; choose from {sorted(_REFUSAL_MODES)}",
+        )
+
+
+def _validate_tensor_pair(
+    first: torch.Tensor,
+    second: torch.Tensor,
+    *,
+    name: str,
+    dimensions: tuple[int, ...],
+    require_same_shape: bool = True,
+) -> None:
+    if first.dim() not in dimensions or second.dim() not in dimensions:
+        allowed = ", ".join(f"{value}D" for value in dimensions)
+        raise ValueError(f"{name} inputs must both be {allowed} tensors")
+    if require_same_shape and first.shape != second.shape:
+        raise ValueError(
+            f"{name} inputs must have identical shapes, got "
+            f"{tuple(first.shape)} and {tuple(second.shape)}",
+        )
+    if first.numel() == 0 or second.numel() == 0:
+        raise ValueError(f"{name} inputs must not be empty")
+    if not torch.isfinite(first).all() or not torch.isfinite(second).all():
+        raise ValueError(f"{name} inputs must contain only finite values")
+
 
 def refusal_rate(
     responses: list[str],
@@ -201,6 +233,7 @@ def refusal_rate(
     Returns:
         Fraction of responses that are refusals (0.0 = no refusals, 1.0 = all refuse).
     """
+    _validate_refusal_mode(mode)
     if not responses:
         return 0.0
 
@@ -296,6 +329,7 @@ def _is_refusal(response: str, mode: str = "combined") -> bool:
     Empty/blank responses return False (no content to classify).
     Degenerate outputs (broken model) return False — they are not refusals.
     """
+    _validate_refusal_mode(mode)
     text = response.strip()
     if not text:
         return False
@@ -336,6 +370,7 @@ def _is_refusal_detailed(response: str, mode: str = "combined") -> tuple[bool, s
         describing which pattern matched, or "" if no match.
         For degenerate outputs (broken model), returns (False, "DEGENERATE").
     """
+    _validate_refusal_mode(mode)
     text = response.strip()
     if not text:
         return False, ""
@@ -406,22 +441,39 @@ def refusal_rate_with_ci(
     responses: list[str],
     mode: str = "combined",
     confidence: float = 0.95,
-) -> dict[str, float]:
+) -> dict[str, float | int | bool | None]:
     """Compute refusal rate with a Wilson score confidence interval."""
+    _validate_refusal_mode(mode)
+    z_map = {0.90: 1.645, 0.95: 1.96, 0.99: 2.576}
+    if confidence not in z_map:
+        raise ValueError("confidence must be one of 0.90, 0.95, or 0.99")
     n = len(responses)
     if n == 0:
-        return {"rate": 0.0, "ci_lower": 0.0, "ci_upper": 0.0, "n_samples": 0}
+        return {
+            "available": False,
+            "rate": None,
+            "ci_lower": None,
+            "ci_upper": None,
+            "n_samples": 0,
+            "refusal_count": 0,
+        }
     refusals = sum(1 for r in responses if _is_refusal(r, mode))
     rate = refusals / n
     import math as _math
-    z_map = {0.90: 1.645, 0.95: 1.96, 0.99: 2.576}
-    z = z_map.get(confidence, 1.96)
+    z = z_map[confidence]
     denominator = 1 + z * z / n
     center = (rate + z * z / (2 * n)) / denominator
     spread = z * _math.sqrt((rate * (1 - rate) + z * z / (4 * n)) / n) / denominator
     ci_lower = max(0.0, center - spread)
     ci_upper = min(1.0, center + spread)
-    return {"rate": rate, "ci_lower": round(ci_lower, 6), "ci_upper": round(ci_upper, 6), "n_samples": n}
+    return {
+        "available": True,
+        "rate": rate,
+        "ci_lower": round(ci_lower, 6),
+        "ci_upper": round(ci_upper, 6),
+        "n_samples": n,
+        "refusal_count": refusals,
+    }
 
 
 # ── KL Divergence ────────────────────────────────────────────────────────
@@ -443,6 +495,16 @@ def token_kl_divergence(
     Returns:
         Mean KL divergence across all tokens (nats). Lower = more similar.
     """
+    _validate_tensor_pair(
+        logits_original,
+        logits_modified,
+        name="token KL",
+        dimensions=(3,),
+    )
+    if not isinstance(temperature, (int, float)) or not math.isfinite(temperature):
+        raise ValueError("temperature must be a finite positive number")
+    if temperature <= 0:
+        raise ValueError("temperature must be greater than zero")
     log_p = F.log_softmax(logits_original / temperature, dim=-1)
     log_q = F.log_softmax(logits_modified / temperature, dim=-1)
     p = F.softmax(logits_original / temperature, dim=-1)
@@ -467,6 +529,14 @@ def first_token_kl_divergence(
     Returns:
         Mean first-token KL divergence across batch.
     """
+    _validate_tensor_pair(
+        logits_original,
+        logits_modified,
+        name="first-token KL",
+        dimensions=(3,),
+    )
+    if logits_original.shape[1] == 0:
+        raise ValueError("first-token KL requires at least one sequence position")
     # Take logits at the last input position (predicting first generated token)
     first_logits_orig = logits_original[:, -1, :]  # (batch, vocab)
     first_logits_mod = logits_modified[:, -1, :]
@@ -502,6 +572,10 @@ def effective_rank(weight_matrix: torch.Tensor) -> float:
     W = weight_matrix.float()
     if W.dim() != 2:
         raise ValueError(f"Expected 2D tensor, got {W.dim()}D")
+    if W.numel() == 0:
+        raise ValueError("Expected a non-empty weight matrix")
+    if not torch.isfinite(W).all():
+        raise ValueError("Weight matrix must contain only finite values")
 
     s = torch.linalg.svdvals(W)
     s = s[s > 1e-12]  # filter near-zero
@@ -551,6 +625,12 @@ def activation_cosine_similarity(
     Returns:
         Mean cosine similarity (1.0 = identical, 0.0 = orthogonal).
     """
+    _validate_tensor_pair(
+        acts_original,
+        acts_modified,
+        name="activation cosine",
+        dimensions=(1, 2, 3),
+    )
     a = acts_original.float()
     b = acts_modified.float()
 
@@ -585,6 +665,13 @@ def linear_cka(
     References:
         Kornblith et al. (2019): Similarity of Neural Network Representations
     """
+    _validate_tensor_pair(
+        X,
+        Y,
+        name="linear CKA",
+        dimensions=(2, 3),
+        require_same_shape=False,
+    )
     X = X.float()
     Y = Y.float()
 
@@ -592,6 +679,13 @@ def linear_cka(
         X = X.reshape(-1, X.shape[-1])
     if Y.dim() == 3:
         Y = Y.reshape(-1, Y.shape[-1])
+    if X.shape[0] != Y.shape[0]:
+        raise ValueError(
+            f"linear CKA inputs must have the same sample count, got "
+            f"{X.shape[0]} and {Y.shape[0]}",
+        )
+    if X.shape[0] < 2:
+        raise ValueError("linear CKA requires at least two samples")
 
     # Column-center
     X = X - X.mean(dim=0, keepdim=True)
@@ -628,6 +722,12 @@ def refusal_projection_magnitude(
     Returns:
         Dict with mean, std, max, min projection magnitudes.
     """
+    if activations.dim() not in (2, 3) or activations.numel() == 0:
+        raise ValueError("activations must be a non-empty 2D or 3D tensor")
+    if refusal_direction.numel() == 0 or refusal_direction.dim() not in (1, 2):
+        raise ValueError("refusal_direction must be a non-empty vector")
+    if not torch.isfinite(activations).all() or not torch.isfinite(refusal_direction).all():
+        raise ValueError("projection inputs must contain only finite values")
     acts = activations.float()
     if acts.dim() == 3:
         acts = acts.reshape(-1, acts.shape[-1])
@@ -635,13 +735,21 @@ def refusal_projection_magnitude(
     d = refusal_direction.float()
     if d.dim() > 1:
         d = d.squeeze()
-    d = d / d.norm().clamp(min=1e-8)
+    if d.dim() != 1 or d.shape[0] != acts.shape[-1]:
+        raise ValueError(
+            f"refusal direction shape {tuple(d.shape)} does not match activation "
+            f"width {acts.shape[-1]}",
+        )
+    norm = d.norm()
+    if norm <= 1e-8:
+        raise ValueError("refusal_direction must have non-zero norm")
+    d = d / norm
 
     projections = acts @ d  # (n_samples,)
 
     return {
         "mean": projections.mean().item(),
-        "std": projections.std().item(),
+        "std": projections.std(unbiased=False).item(),
         "max": projections.max().item(),
         "min": projections.min().item(),
         "abs_mean": projections.abs().mean().item(),
@@ -654,11 +762,11 @@ def refusal_projection_magnitude(
 class AbliterationEvalResult:
     """Comprehensive evaluation result for an abliterated model."""
 
-    refusal_rate_harmful: float        # fraction of harmful prompts still refused
-    refusal_rate_harmless: float       # over-refusal rate on harmless prompts
+    refusal_rate_harmful: float | None  # fraction of harmful prompts still refused
+    refusal_rate_harmless: float | None  # over-refusal rate on harmless prompts
     kl_divergence: float | None        # KL(original || modified) on harmless prompts
-    perplexity: float                  # perplexity on reference text
-    coherence_score: float             # basic coherence score
+    perplexity: float | None           # perplexity on reference text
+    coherence_score: float | None      # basic coherence score
     mean_activation_cosine: float | None  # activation similarity original vs modified
     mean_cka: float | None             # CKA similarity across layers
 
@@ -672,14 +780,26 @@ def format_eval_report(result: AbliterationEvalResult) -> str:
 
     # Refusal removal effectiveness
     lines.append("Refusal Removal:")
-    lines.append(f"  Harmful prompt refusal rate: {result.refusal_rate_harmful:.1%}")
-    lines.append(f"  Harmless prompt over-refusal: {result.refusal_rate_harmless:.1%}")
+    harmful = (
+        f"{result.refusal_rate_harmful:.1%}"
+        if result.refusal_rate_harmful is not None else "unavailable"
+    )
+    harmless = (
+        f"{result.refusal_rate_harmless:.1%}"
+        if result.refusal_rate_harmless is not None else "unavailable"
+    )
+    lines.append(f"  Harmful prompt refusal rate: {harmful}")
+    lines.append(f"  Harmless prompt over-refusal: {harmless}")
     lines.append("")
 
     # Model quality
     lines.append("Model Quality:")
-    lines.append(f"  Perplexity: {result.perplexity:.2f}")
-    lines.append(f"  Coherence: {result.coherence_score:.1%}")
+    perplexity = f"{result.perplexity:.2f}" if result.perplexity is not None else "unavailable"
+    coherence = (
+        f"{result.coherence_score:.1%}" if result.coherence_score is not None else "unavailable"
+    )
+    lines.append(f"  Perplexity: {perplexity}")
+    lines.append(f"  Coherence: {coherence}")
     if result.kl_divergence is not None:
         lines.append(f"  KL divergence: {result.kl_divergence:.4f}")
         if result.kl_divergence < 0.2:
@@ -691,13 +811,19 @@ def format_eval_report(result: AbliterationEvalResult) -> str:
         else:
             quality = "significant damage"
         lines.append(f"    ({quality})")
+    else:
+        lines.append("  KL divergence: unavailable")
     lines.append("")
 
     # Representation similarity
+    lines.append("Representation Similarity:")
     if result.mean_activation_cosine is not None:
-        lines.append("Representation Similarity:")
         lines.append(f"  Activation cosine similarity: {result.mean_activation_cosine:.4f}")
-        if result.mean_cka is not None:
-            lines.append(f"  Linear CKA: {result.mean_cka:.4f}")
+    else:
+        lines.append("  Activation cosine similarity: unavailable")
+    if result.mean_cka is not None:
+        lines.append(f"  Linear CKA: {result.mean_cka:.4f}")
+    else:
+        lines.append("  Linear CKA: unavailable")
 
     return "\n".join(lines)

obliteratus/evaluation/metrics.py

@@ -20,9 +20,32 @@ def perplexity(logits: torch.Tensor, labels: torch.Tensor) -> float:
     Returns:
         Scalar perplexity (lower is better).
     """
+    if logits.dim() != 3:
+        raise ValueError(f"logits must be 3D (batch, sequence, vocabulary), got {logits.dim()}D")
+    if labels.dim() != 2:
+        raise ValueError(f"labels must be 2D (batch, sequence), got {labels.dim()}D")
+    if labels.dtype not in {
+        torch.int8, torch.int16, torch.int32, torch.int64, torch.uint8,
+    }:
+        raise ValueError("labels must contain integer token IDs")
+    if logits.shape[:2] != labels.shape:
+        raise ValueError(
+            f"logits batch/sequence shape {tuple(logits.shape[:2])} does not match "
+            f"labels shape {tuple(labels.shape)}",
+        )
+    if logits.shape[1] < 2:
+        raise ValueError("perplexity requires at least two sequence positions")
+    if not torch.isfinite(logits).all():
+        raise ValueError("logits must contain only finite values")
+
     # Shift so that tokens < n predict n
     shift_logits = logits[:, :-1, :].contiguous()
     shift_labels = labels[:, 1:].contiguous()
+    valid_labels = shift_labels[shift_labels != -100]
+    if valid_labels.numel() == 0:
+        raise ValueError("perplexity is unavailable because every target label is ignored")
+    if (valid_labels < 0).any() or (valid_labels >= logits.shape[-1]).any():
+        raise ValueError("labels contain token IDs outside the logits vocabulary")
 
     loss = F.cross_entropy(
         shift_logits.view(-1, shift_logits.size(-1)),
@@ -35,6 +58,11 @@ def perplexity(logits: torch.Tensor, labels: torch.Tensor) -> float:
 
 def accuracy(predictions: Sequence[int], references: Sequence[int]) -> float:
     """Simple accuracy."""
+    if len(predictions) != len(references):
+        raise ValueError(
+            f"predictions and references must have equal length, got "
+            f"{len(predictions)} and {len(references)}",
+        )
     if len(predictions) == 0:
         return 0.0
     correct = sum(p == r for p, r in zip(predictions, references))
@@ -47,4 +75,11 @@ def f1_score_metric(
     average: str = "macro",
 ) -> float:
     """F1 score wrapper around sklearn."""
+    if len(predictions) != len(references):
+        raise ValueError(
+            f"predictions and references must have equal length, got "
+            f"{len(predictions)} and {len(references)}",
+        )
+    if len(predictions) == 0:
+        return 0.0
     return float(sklearn_f1(references, predictions, average=average, zero_division=0))

obliteratus/reporting/report.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import json
+import math
 import re
 from dataclasses import dataclass, field
 from pathlib import Path
@@ -10,17 +11,65 @@ from typing import Any
 
 import pandas as pd
 
+REPORT_SCHEMA_VERSION = 1
+_SENSITIVE_KEY_RE = re.compile(
+    r"(?:authorization|credential|password|secret|token|api[_-]?key)", re.IGNORECASE,
+)
+
 
 def _sanitize_label(text: str, max_len: int = 80) -> str:
     """Strip filesystem paths, tokens, and overly-long strings from labels."""
-    text = re.sub(r"(/[a-zA-Z0-9_./-]{3,})", lambda m: m.group(0).rsplit("/", 1)[-1], text)
+    if text.startswith("/") or re.match(r"^[A-Za-z]:[\\/]", text):
+        text = Path(text).name
+    text = re.sub(
+        r"(?:/[A-Za-z0-9_.-]+){2,}", lambda match: Path(match.group()).name, text,
+    )
     text = re.sub(r"\bhf_[A-Za-z0-9]{6,}\b", "<TOKEN>", text)
+    text = re.sub(r"\bgh[pousr]_[A-Za-z0-9]{12,}\b", "<TOKEN>", text)
+    text = re.sub(r"\bgithub_pat_[A-Za-z0-9_]{12,}\b", "<TOKEN>", text)
+    text = re.sub(r"\bsk-[A-Za-z0-9_-]{12,}\b", "<TOKEN>", text)
     text = re.sub(r"\b[0-9a-fA-F]{32,}\b", "<REDACTED>", text)
     if len(text) > max_len:
         text = text[: max_len - 3] + "..."
     return text
 
 
+def _sanitize_public_value(value: Any) -> Any:
+    """Return a deterministic, JSON-safe value with private material removed."""
+    if value is None or isinstance(value, (bool, int)):
+        return value
+    if isinstance(value, float):
+        return value if math.isfinite(value) else None
+    if isinstance(value, str):
+        return _sanitize_label(value, max_len=240)
+    if isinstance(value, dict):
+        return {
+            str(key): _sanitize_public_value(item)
+            for key, item in sorted(value.items(), key=lambda pair: str(pair[0]))
+            if not _SENSITIVE_KEY_RE.search(str(key))
+        }
+    if isinstance(value, (list, tuple)):
+        return [_sanitize_public_value(item) for item in value]
+    return _sanitize_label(str(value), max_len=240)
+
+
+def _normalize_metrics(metrics: dict[str, Any]) -> tuple[dict[str, float | None], dict[str, str]]:
+    values: dict[str, float | None] = {}
+    status: dict[str, str] = {}
+    for name, raw_value in sorted(metrics.items()):
+        value = None
+        if not isinstance(raw_value, bool):
+            try:
+                candidate = float(raw_value)
+                if math.isfinite(candidate):
+                    value = candidate
+            except (TypeError, ValueError):
+                pass
+        values[str(name)] = value
+        status[str(name)] = "measured" if value is not None else "unavailable"
+    return values, status
+
+
 @dataclass
 class AblationResult:
     """Result of a single ablation experiment."""
@@ -28,7 +77,7 @@ class AblationResult:
     strategy: str
     component: str
     description: str
-    metrics: dict[str, float]
+    metrics: dict[str, float | None]
     metadata: dict[str, Any] | None = None
 
 
@@ -37,28 +86,52 @@ class AblationReport:
     """Collects results and produces tables / charts / exports."""
 
     model_name: str
-    baseline_metrics: dict[str, float] = field(default_factory=dict)
+    baseline_metrics: dict[str, float | None] = field(default_factory=dict)
     results: list[AblationResult] = field(default_factory=list)
 
-    def add_baseline(self, metrics: dict[str, float]):
+    def add_baseline(self, metrics: dict[str, float | None]):
         self.baseline_metrics = metrics
 
     def add_result(self, result: AblationResult):
         self.results.append(result)
 
+    def to_dict(self) -> dict[str, Any]:
+        """Return the canonical, versioned public report representation."""
+        baseline_metrics, baseline_status = _normalize_metrics(self.baseline_metrics)
+        results = []
+        for result in self.results:
+            metrics, metric_status = _normalize_metrics(result.metrics)
+            results.append({
+                "strategy": _sanitize_label(result.strategy),
+                "component": _sanitize_label(result.component),
+                "description": _sanitize_label(result.description, max_len=240),
+                "metrics": metrics,
+                "metric_status": metric_status,
+                "metadata": _sanitize_public_value(result.metadata),
+            })
+        return {
+            "schema_version": REPORT_SCHEMA_VERSION,
+            "model_name": _sanitize_label(self.model_name),
+            "baseline_metrics": baseline_metrics,
+            "baseline_metric_status": baseline_status,
+            "results": results,
+        }
+
     def to_dataframe(self) -> pd.DataFrame:
         """Convert results to a pandas DataFrame with delta columns."""
         rows = []
+        baseline_metrics, _ = _normalize_metrics(self.baseline_metrics)
         for r in self.results:
+            metrics, _ = _normalize_metrics(r.metrics)
             row = {
-                "strategy": r.strategy,
-                "component": r.component,
-                "description": r.description,
+                "strategy": _sanitize_label(r.strategy),
+                "component": _sanitize_label(r.component),
+                "description": _sanitize_label(r.description, max_len=240),
             }
-            for metric_name, value in r.metrics.items():
+            for metric_name, value in metrics.items():
                 row[metric_name] = value
-                baseline_val = self.baseline_metrics.get(metric_name)
-                if baseline_val is not None:
+                baseline_val = baseline_metrics.get(metric_name)
+                if baseline_val is not None and value is not None:
                     row[f"{metric_name}_delta"] = value - baseline_val
                     if baseline_val != 0:
                         row[f"{metric_name}_pct_change"] = (
@@ -84,7 +157,8 @@ class AblationReport:
         table.add_column("Strategy", style="cyan")
         table.add_column("Component", style="green")
 
-        metric_names = list(self.baseline_metrics.keys())
+        baseline_metrics, _ = _normalize_metrics(self.baseline_metrics)
+        metric_names = list(baseline_metrics.keys())
         for m in metric_names:
             table.add_column(f"{m}", justify="right")
             table.add_column(f"{m} delta", justify="right", style="red")
@@ -92,7 +166,8 @@ class AblationReport:
         # Baseline row
         baseline_vals = []
         for m in metric_names:
-            baseline_vals.extend([f"{self.baseline_metrics[m]:.4f}", "—"])
+            value = baseline_metrics[m]
+            baseline_vals.extend([f"{value:.4f}" if value is not None else "unavailable", "—"])
         table.add_row("baseline", "—", *baseline_vals, style="bold")
 
         for _, row in df.iterrows():
@@ -100,7 +175,7 @@ class AblationReport:
             for m in metric_names:
                 val = row.get(m, float("nan"))
                 delta = row.get(f"{m}_delta", float("nan"))
-                cells.append(f"{val:.4f}")
+                cells.append(f"{val:.4f}" if pd.notna(val) else "unavailable")
                 cells.append(f"{delta:+.4f}" if pd.notna(delta) else "—")
             table.add_row(*cells)
 
@@ -110,27 +185,16 @@ class AblationReport:
         """Save raw results to JSON."""
         path = Path(path)
         path.parent.mkdir(parents=True, exist_ok=True)
-        data = {
-            "model_name": self.model_name,
-            "baseline_metrics": self.baseline_metrics,
-            "results": [
-                {
-                    "strategy": r.strategy,
-                    "component": r.component,
-                    "description": r.description,
-                    "metrics": r.metrics,
-                    "metadata": r.metadata,
-                }
-                for r in self.results
-            ],
-        }
-        path.write_text(json.dumps(data, indent=2))
+        path.write_text(
+            json.dumps(self.to_dict(), indent=2, sort_keys=True, allow_nan=False) + "\n",
+            encoding="utf-8",
+        )
 
     def save_csv(self, path: str | Path):
         """Save results DataFrame to CSV."""
         path = Path(path)
         path.parent.mkdir(parents=True, exist_ok=True)
-        self.to_dataframe().to_csv(path, index=False)
+        self.to_dataframe().to_csv(path, index=False, na_rep="unavailable", lineterminator="\n")
 
     def plot_impact(self, metric: str | None = None, output_path: str | Path | None = None):
         """Generate a bar chart showing the impact of each ablation on a metric.

obliteratus/telemetry.py

@@ -33,6 +33,7 @@ import logging
 import math
 import os
 import platform
+import re
 import time
 import threading
 import uuid
@@ -43,6 +44,86 @@ from typing import Any
 
 logger = logging.getLogger(__name__)
 
+BENCHMARK_SCHEMA_VERSION = 1
+TELEMETRY_SCHEMA_VERSION = 2
+_SENSITIVE_KEY_RE = re.compile(
+    r"(?:authorization|credential|password|secret|token|api[_-]?key)", re.IGNORECASE,
+)
+_PUBLIC_METRIC_RANGES: dict[str, tuple[float | None, float | None]] = {
+    "refusal_rate": (0.0, 1.0),
+    "perplexity": (0.0, None),
+    "coherence": (0.0, 1.0),
+    "kl_divergence": (0.0, None),
+    "capability_score": (0.0, 1.0),
+    "degenerate_count": (0.0, None),
+    "time_seconds": (0.0, None),
+}
+
+
+def _sanitize_public_text(value: Any, max_len: int = 240) -> str:
+    text = str(value)
+    if text.startswith("/") or re.match(r"^[A-Za-z]:[\\/]", text):
+        text = Path(text).name
+    text = re.sub(r"\bhf_[A-Za-z0-9]{6,}\b", "<TOKEN>", text)
+    text = re.sub(r"\bgh[pousr]_[A-Za-z0-9]{12,}\b", "<TOKEN>", text)
+    text = re.sub(r"\bgithub_pat_[A-Za-z0-9_]{12,}\b", "<TOKEN>", text)
+    text = re.sub(r"\bsk-[A-Za-z0-9_-]{12,}\b", "<TOKEN>", text)
+    text = re.sub(r"(?:/[A-Za-z0-9_.-]+){2,}", lambda match: Path(match.group()).name, text)
+    return text if len(text) <= max_len else text[: max_len - 3] + "..."
+
+
+def _sanitize_public_value(value: Any) -> Any:
+    if value is None or isinstance(value, (bool, int)):
+        return value
+    if isinstance(value, float):
+        return value if math.isfinite(value) else None
+    if isinstance(value, str):
+        return _sanitize_public_text(value)
+    if isinstance(value, dict):
+        return {
+            str(key): _sanitize_public_value(item)
+            for key, item in sorted(value.items(), key=lambda pair: str(pair[0]))
+            if not _SENSITIVE_KEY_RE.search(str(key))
+        }
+    if isinstance(value, (list, tuple)):
+        return [_sanitize_public_value(item) for item in value]
+    return _sanitize_public_text(value)
+
+
+def _metric_value(name: str, value: Any) -> float | None:
+    if isinstance(value, bool):
+        return None
+    numeric = _safe_float(value)
+    if numeric is None:
+        return None
+    minimum, maximum = _PUBLIC_METRIC_RANGES.get(name, (None, None))
+    if minimum is not None and numeric < minimum:
+        return None
+    if maximum is not None and numeric > maximum:
+        return None
+    return numeric
+
+
+def _normalize_quality_metrics(
+    metrics: dict[str, Any] | None,
+) -> tuple[dict[str, Any], dict[str, str]]:
+    values: dict[str, Any] = {}
+    statuses: dict[str, str] = {}
+    for name, raw_value in sorted((metrics or {}).items()):
+        if name not in _PUBLIC_METRIC_RANGES and name not in {
+            "capability_results", "spectral_certification",
+        }:
+            continue
+        if name == "capability_results":
+            value = _sanitize_public_value(raw_value) if isinstance(raw_value, dict) else None
+        elif name == "spectral_certification":
+            value = _sanitize_public_text(raw_value, 40) if isinstance(raw_value, str) else None
+        else:
+            value = _metric_value(name, raw_value)
+        values[name] = value
+        statuses[name] = "measured" if value is not None else "unavailable"
+    return values, statuses
+
 # ── Configuration ─────────────────────────────────────────────────────
 
 _ON_HF_SPACES = os.environ.get("SPACE_ID") is not None
@@ -238,6 +319,7 @@ def is_enabled() -> bool:
 @dataclass
 class BenchmarkRecord:
     """A single benchmark result entry."""
+    schema_version: int = BENCHMARK_SCHEMA_VERSION
     # Identity
     timestamp: str = ""
     session_id: str = ""  # Random per-session, not per-user
@@ -616,7 +698,9 @@ def restore_from_hub() -> int:
                     if key in existing_keys:
                         continue
                     existing_keys.add(key)
-                    f.write(json.dumps(r, default=str) + "\n")
+                    f.write(json.dumps(
+                        _sanitize_public_value(r), sort_keys=True, allow_nan=False,
+                    ) + "\n")
                     new_count += 1
 
         if new_count:
@@ -695,10 +779,10 @@ def log_benchmark(record: BenchmarkRecord) -> bool:
         record.model_family = _detect_model_family(record.model_id)
 
     try:
-        data = asdict(record)
+        data = _sanitize_public_value(asdict(record))
         with _write_lock:
             with open(TELEMETRY_FILE, "a") as f:
-                f.write(json.dumps(data, default=str) + "\n")
+                f.write(json.dumps(data, sort_keys=True, allow_nan=False) + "\n")
         # Auto-sync to central Hub repo (debounced, background thread)
         _schedule_hub_sync()
         return True
@@ -750,6 +834,8 @@ def read_telemetry(max_records: int = 10000) -> list[dict[str, Any]]:
 
     Returns a list of dicts, newest first.
     """
+    if max_records <= 0:
+        raise ValueError("max_records must be greater than zero")
     records = []
     if not TELEMETRY_FILE.exists():
         return records
@@ -774,6 +860,41 @@ def read_telemetry(max_records: int = 10000) -> list[dict[str, Any]]:
     return records
 
 
+def _normalize_leaderboard_record(record: dict[str, Any]) -> dict[str, Any]:
+    """Normalize v1 benchmark and v2 report records for one aggregation path."""
+    if record.get("schema_version") == TELEMETRY_SCHEMA_VERSION and "model" in record:
+        model_data = record.get("model") or {}
+        metrics = record.get("quality_metrics") or {}
+        return {
+            "model_id": _sanitize_public_text(model_data.get("architecture", "unknown"), 80),
+            "method": _sanitize_public_text(record.get("method", "unknown"), 80),
+            "timestamp": str(record.get("timestamp", "")),
+            "session_id": str(record.get("session_id", "")),
+            "refusal_rate": _metric_value("refusal_rate", metrics.get("refusal_rate")),
+            "perplexity": _metric_value("perplexity", metrics.get("perplexity")),
+            "coherence": _metric_value("coherence", metrics.get("coherence")),
+            "time_seconds": _metric_value(
+                "time_seconds", (record.get("informed") or {}).get("total_duration"),
+            ),
+            "gpu_name": _sanitize_public_text(
+                (record.get("environment") or {}).get("gpu_name", ""), 80,
+            ),
+            "error": _sanitize_public_text(record["error"]) if record.get("error") else None,
+        }
+    return {
+        "model_id": _sanitize_public_text(record.get("model_id", "unknown"), 80),
+        "method": _sanitize_public_text(record.get("method", "unknown"), 80),
+        "timestamp": str(record.get("timestamp", "")),
+        "session_id": str(record.get("session_id", "")),
+        "refusal_rate": _metric_value("refusal_rate", record.get("refusal_rate")),
+        "perplexity": _metric_value("perplexity", record.get("perplexity")),
+        "coherence": _metric_value("coherence", record.get("coherence")),
+        "time_seconds": _metric_value("time_seconds", record.get("time_seconds")),
+        "gpu_name": _sanitize_public_text(record.get("gpu_name", ""), 80),
+        "error": _sanitize_public_text(record["error"]) if record.get("error") else None,
+    }
+
+
 def get_leaderboard_data() -> list[dict[str, Any]]:
     """Get aggregated leaderboard data from local + Hub telemetry.
 
@@ -800,7 +921,7 @@ def get_leaderboard_data() -> list[dict[str, Any]]:
         if key in seen:
             continue
         seen.add(key)
-        records.append(r)
+        records.append(_normalize_leaderboard_record(r))
 
     if not records:
         return []
@@ -808,8 +929,6 @@ def get_leaderboard_data() -> list[dict[str, Any]]:
     # Group by (model_id, method)
     groups: dict[tuple[str, str], list[dict]] = {}
     for r in records:
-        if r.get("error"):
-            continue
         key = (r.get("model_id", ""), r.get("method", ""))
         if key not in groups:
             groups[key] = []
@@ -818,29 +937,41 @@ def get_leaderboard_data() -> list[dict[str, Any]]:
     leaderboard = []
     for (model_id, method), runs in groups.items():
         # Compute aggregates
-        refusal_rates = [r["refusal_rate"] for r in runs if r.get("refusal_rate") is not None]
-        perplexities = [r["perplexity"] for r in runs if r.get("perplexity") is not None]
-        coherences = [r["coherence"] for r in runs if r.get("coherence") is not None]
-        times = [r["time_seconds"] for r in runs if r.get("time_seconds") is not None]
+        refusal_rates = [r["refusal_rate"] for r in runs if r["refusal_rate"] is not None]
+        perplexities = [r["perplexity"] for r in runs if r["perplexity"] is not None]
+        coherences = [r["coherence"] for r in runs if r["coherence"] is not None]
+        times = [r["time_seconds"] for r in runs if r["time_seconds"] is not None]
+        successful_runs = sum(not bool(r.get("error")) for r in runs)
+        latest = max(runs, key=lambda run: run.get("timestamp", ""))
 
         entry = {
             "model": model_id.split("/")[-1] if "/" in model_id else model_id,
             "model_id": model_id,
             "method": method,
             "runs": len(runs),
+            "successful_runs": successful_runs,
+            "failed_runs": len(runs) - successful_runs,
+            "refusal_measurements": len(refusal_rates),
+            "perplexity_measurements": len(perplexities),
+            "coherence_measurements": len(coherences),
             "best_refusal": min(refusal_rates) if refusal_rates else None,
             "avg_refusal": sum(refusal_rates) / len(refusal_rates) if refusal_rates else None,
             "best_perplexity": min(perplexities) if perplexities else None,
             "avg_perplexity": sum(perplexities) / len(perplexities) if perplexities else None,
             "avg_coherence": sum(coherences) / len(coherences) if coherences else None,
             "avg_time_s": sum(times) / len(times) if times else None,
-            "gpu": runs[0].get("gpu_name", "") if runs else "",
-            "last_run": runs[0].get("timestamp", "") if runs else "",
+            "gpu": latest.get("gpu_name", ""),
+            "last_run": latest.get("timestamp", ""),
         }
         leaderboard.append(entry)
 
     # Sort: lowest refusal rate first, then by perplexity
-    leaderboard.sort(key=lambda x: (x.get("best_refusal") or 999, x.get("best_perplexity") or 999))
+    leaderboard.sort(key=lambda x: (
+        x["best_refusal"] is None,
+        x["best_refusal"] if x["best_refusal"] is not None else math.inf,
+        x["best_perplexity"] is None,
+        x["best_perplexity"] if x["best_perplexity"] is not None else math.inf,
+    ))
 
     return leaderboard
 
@@ -1104,48 +1235,47 @@ def build_report(
 ) -> dict[str, Any]:
     """Build a structured telemetry report (schema v2)."""
     report: dict[str, Any] = {
-        "schema_version": 2,
+        "schema_version": TELEMETRY_SCHEMA_VERSION,
         "session_id": uuid.uuid4().hex,
         "timestamp": datetime.now(timezone.utc).isoformat(),
         "model": {
-            "architecture": architecture,
+            "architecture": _sanitize_public_text(architecture, 80),
             "num_layers": num_layers,
             "num_heads": num_heads,
             "hidden_size": hidden_size,
             "total_params": total_params,
         },
-        "method": method,
+        "method": _sanitize_public_text(method, 80),
         "environment": _get_environment_info(),
     }
     if method_config:
         report["method_config"] = {
-            k: v for k, v in method_config.items()
+            k: _sanitize_public_value(v) for k, v in method_config.items()
             if k in _ALLOWED_METHOD_CONFIG_KEYS
         }
     else:
         report["method_config"] = {}
-    if quality_metrics:
-        report["quality_metrics"] = dict(quality_metrics)
-    else:
-        report["quality_metrics"] = {}
+    metrics, metric_status = _normalize_quality_metrics(quality_metrics)
+    report["quality_metrics"] = metrics
+    report["quality_metric_status"] = metric_status
     if stage_durations:
-        report["stage_durations"] = stage_durations
+        report["stage_durations"] = _sanitize_public_value(stage_durations)
     if strong_layers is not None:
         report["strong_layers"] = strong_layers
     if direction_stats:
-        report["direction_stats"] = direction_stats
+        report["direction_stats"] = _sanitize_public_value(direction_stats)
     if excise_details:
-        report["excise_details"] = excise_details
+        report["excise_details"] = _sanitize_public_value(excise_details)
     if prompt_counts:
-        report["prompt_counts"] = prompt_counts
+        report["prompt_counts"] = _sanitize_public_value(prompt_counts)
     if gpu_memory:
-        report["gpu_memory"] = gpu_memory
+        report["gpu_memory"] = _sanitize_public_value(gpu_memory)
     if analysis_insights:
         filtered = {k: v for k, v in analysis_insights.items() if k in _ALLOWED_ANALYSIS_KEYS}
         if filtered:
-            report["analysis_insights"] = filtered
+            report["analysis_insights"] = _sanitize_public_value(filtered)
     if informed_extras:
-        report["informed"] = dict(informed_extras)
+        report["informed"] = _sanitize_public_value(informed_extras)
     return report
 
 
@@ -1154,7 +1284,9 @@ def _send_sync(report: dict[str, Any]) -> None:
     try:
         with _write_lock:
             with open(TELEMETRY_FILE, "a") as f:
-                f.write(json.dumps(report, default=str) + "\n")
+                f.write(json.dumps(
+                    _sanitize_public_value(report), sort_keys=True, allow_nan=False,
+                ) + "\n")
         _schedule_hub_sync()
     except Exception as e:
         logger.debug("Telemetry v2 write failed: %s", e)

tests/test_advanced_metrics.py

@@ -18,6 +18,7 @@ from obliteratus.evaluation.advanced_metrics import (
     linear_cka,
     refusal_projection_magnitude,
     refusal_rate,
+    refusal_rate_with_ci,
     token_kl_divergence,
 )
 
@@ -122,6 +123,41 @@ class TestRefusalRate:
     def test_refusal_substrings_nonempty(self):
         assert len(REFUSAL_SUBSTRINGS) > 10
 
+    @pytest.mark.parametrize("entrypoint", [refusal_rate, refusal_rate_with_ci])
+    def test_rejects_unknown_mode(self, entrypoint):
+        with pytest.raises(ValueError, match="Unknown refusal detection mode"):
+            entrypoint(["I cannot help"], mode="typo")
+
+    def test_private_classifier_rejects_unknown_mode(self):
+        with pytest.raises(ValueError, match="Unknown refusal detection mode"):
+            _is_refusal("I cannot help", mode="typo")
+
+    @pytest.mark.parametrize("confidence", [0.0, 0.9 + 1e-10, 1.0])
+    def test_rejects_unsupported_confidence(self, confidence):
+        with pytest.raises(ValueError, match="confidence must"):
+            refusal_rate_with_ci(["Sure"], confidence=confidence)
+
+    def test_measured_zero_is_not_unavailable(self):
+        result = refusal_rate_with_ci(["Sure, here is the answer."])
+        assert result["available"] is True
+        assert result["rate"] == 0.0
+        assert result["refusal_count"] == 0
+
+    @pytest.mark.parametrize(
+        "response",
+        [
+            "  I'M SORRY!!! I cannot help with that.",
+            "抱歉，我无法帮助这个请求。",
+            "Извините, я не могу помочь с этим запросом.",
+        ],
+    )
+    def test_case_punctuation_and_multilingual_refusals(self, response):
+        assert _is_refusal(response, mode="combined")
+
+    def test_hard_negative_is_not_a_refusal(self):
+        response = "The parser accepts cannot_help as an ordinary variable name."
+        assert not _is_refusal(response, mode="combined")
+
 
 # ---------------------------------------------------------------------------
 # KL Divergence
@@ -174,6 +210,29 @@ class TestKLDivergence:
         kl_t5 = token_kl_divergence(logits_a, logits_b, temperature=5.0)
         assert kl_t5 < kl_t1
 
+    def test_common_logit_offset_is_invariant(self):
+        torch.manual_seed(42)
+        logits_a = torch.randn(2, 3, 8)
+        logits_b = torch.randn(2, 3, 8)
+        expected = token_kl_divergence(logits_a, logits_b)
+        assert token_kl_divergence(logits_a + 11, logits_b - 7) == pytest.approx(
+            expected, abs=1e-6,
+        )
+
+    @pytest.mark.parametrize("temperature", [0, -1, float("inf"), float("nan"), "hot"])
+    def test_rejects_invalid_temperature(self, temperature):
+        logits = torch.zeros(1, 2, 3)
+        with pytest.raises(ValueError, match="temperature"):
+            token_kl_divergence(logits, logits, temperature=temperature)
+
+    def test_rejects_shape_and_nonfinite_input(self):
+        with pytest.raises(ValueError, match="identical shapes"):
+            token_kl_divergence(torch.zeros(1, 2, 3), torch.zeros(1, 3, 3))
+        logits = torch.zeros(1, 2, 3)
+        logits[0, 0, 0] = float("inf")
+        with pytest.raises(ValueError, match="finite"):
+            first_token_kl_divergence(logits, logits)
+
 
 # ---------------------------------------------------------------------------
 # Effective Rank
@@ -260,6 +319,14 @@ class TestActivationCosineSimilarity:
         sim = activation_cosine_similarity(a, b)
         assert -1.0 <= sim <= 1.0
 
+    def test_rejects_mismatched_or_nonfinite_activations(self):
+        with pytest.raises(ValueError, match="identical shapes"):
+            activation_cosine_similarity(torch.zeros(2, 3), torch.zeros(3, 3))
+        bad = torch.zeros(2, 3)
+        bad[0, 0] = float("nan")
+        with pytest.raises(ValueError, match="finite"):
+            activation_cosine_similarity(bad, bad)
+
 
 # ---------------------------------------------------------------------------
 # Linear CKA
@@ -310,6 +377,23 @@ class TestLinearCKA:
         cka = linear_cka(X, Y)
         assert -0.01 <= cka <= 1.01
 
+    def test_joint_row_permutation_is_invariant(self):
+        torch.manual_seed(42)
+        x = torch.randn(20, 8)
+        y = torch.randn(20, 12)
+        permutation = torch.randperm(20)
+        assert linear_cka(x[permutation], y[permutation]) == pytest.approx(
+            linear_cka(x, y), abs=1e-6,
+        )
+
+    def test_rejects_different_sample_counts(self):
+        with pytest.raises(ValueError, match="same sample count"):
+            linear_cka(torch.zeros(2, 3), torch.zeros(3, 4))
+
+    def test_rejects_single_sample_degeneracy(self):
+        with pytest.raises(ValueError, match="at least two samples"):
+            linear_cka(torch.zeros(1, 3), torch.zeros(1, 4))
+
 
 # ---------------------------------------------------------------------------
 # Refusal Direction Projection Magnitude
@@ -346,6 +430,17 @@ class TestRefusalProjection:
         result = refusal_projection_magnitude(acts, d)
         assert set(result.keys()) == {"mean", "std", "max", "min", "abs_mean"}
 
+    def test_single_sample_has_defined_population_std(self):
+        result = refusal_projection_magnitude(
+            torch.tensor([[2.0, 0.0]]), torch.tensor([1.0, 0.0]),
+        )
+        assert result["std"] == 0.0
+
+    @pytest.mark.parametrize("direction", [torch.zeros(2), torch.ones(3)])
+    def test_rejects_invalid_direction(self, direction):
+        with pytest.raises(ValueError):
+            refusal_projection_magnitude(torch.ones(2, 2), direction)
+
 
 # ---------------------------------------------------------------------------
 # Eval Report Formatting
@@ -380,6 +475,20 @@ class TestEvalReport:
         report = format_eval_report(result)
         assert "significant damage" in report
 
+    def test_unavailable_metrics_are_not_rendered_as_zero(self):
+        result = AbliterationEvalResult(
+            refusal_rate_harmful=None,
+            refusal_rate_harmless=0.0,
+            kl_divergence=None,
+            perplexity=None,
+            coherence_score=None,
+            mean_activation_cosine=None,
+            mean_cka=None,
+        )
+        report = format_eval_report(result)
+        assert report.count("unavailable") >= 4
+        assert "Harmless prompt over-refusal: 0.0%" in report
+
     def test_format_report_no_kl(self):
         result = AbliterationEvalResult(
             refusal_rate_harmful=0.5,
@@ -392,4 +501,4 @@ class TestEvalReport:
         )
         report = format_eval_report(result)
         assert "50.0%" in report
-        assert "KL" not in report
+        assert "KL divergence: unavailable" in report

tests/test_community.py

@@ -162,6 +162,13 @@ class TestSaveContribution:
         name = path.stem
         assert name.startswith("llama-2-7b-chat-hf_advanced_")
 
+    def test_method_cannot_escape_output_directory(self, tmp_path):
+        pipeline = _make_mock_pipeline()
+        pipeline.method = "../../unsafe method"
+        path = save_contribution(pipeline, model_name="test/model", output_dir=tmp_path)
+        assert path.parent == tmp_path
+        assert ".." not in path.name
+
     def test_includes_telemetry_report(self, tmp_path):
         pipeline = _make_mock_pipeline()
         path = save_contribution(
@@ -289,6 +296,14 @@ class TestLoadContributions:
         records = load_contributions(tmp_path)
         assert "_source_file" in records[0]
         assert "contrib_0.json" in records[0]["_source_file"]
+        assert records[0]["_source_file"] == "contrib_0.json"
+
+    def test_skips_unsupported_schema_version(self, tmp_path):
+        path = self._write_contrib(tmp_path, "test/model", "advanced", 0.05, 0)
+        data = json.loads(path.read_text())
+        data["contribution_schema_version"] = 999
+        path.write_text(json.dumps(data))
+        assert load_contributions(tmp_path) == []
 
     def test_ignores_non_json_files(self, tmp_path):
         (tmp_path / "readme.txt").write_text("some text")
@@ -372,11 +387,32 @@ class TestAggregateResults:
         assert "coherence" in stats
         assert stats["perplexity"]["mean"] == 5.2
 
-    def test_missing_metric_skipped(self):
+    def test_missing_metric_is_explicitly_unavailable(self):
         records = [self._make_record("model-a", "advanced", 0.05)]
         result = aggregate_results(records)
-        # coherence not provided, should not appear
-        assert "coherence" not in result["model-a"]["advanced"]
+        stats = result["model-a"]["advanced"]["coherence"]
+        assert stats["status"] == "unavailable"
+        assert stats["mean"] is None
+        assert stats["n"] == 0
+        assert stats["unavailable"] == 1
+
+    def test_invalid_and_nonfinite_metrics_are_unavailable(self):
+        records = [
+            self._make_record("model-a", "advanced", float("nan"), perplexity=-1),
+            self._make_record("model-a", "advanced", True, coherence=2.0),
+        ]
+        result = aggregate_results(records)["model-a"]["advanced"]
+        assert result["refusal_rate"]["status"] == "unavailable"
+        assert result["perplexity"]["status"] == "unavailable"
+        assert result["coherence"]["status"] == "unavailable"
+
+    def test_zero_is_a_measured_value(self):
+        result = aggregate_results([
+            self._make_record("model-a", "advanced", 0.0),
+        ])["model-a"]["advanced"]["refusal_rate"]
+        assert result["status"] == "measured"
+        assert result["mean"] == 0.0
+        assert result["n"] == 1
 
     def test_unknown_model_and_method(self):
         records = [{
@@ -419,6 +455,20 @@ class TestGenerateLatexTable:
         assert "\\toprule" in latex
         assert "\\bottomrule" in latex
 
+    def test_escapes_untrusted_labels(self):
+        latex = generate_latex_table({
+            "org/model_name&x": {
+                "advanced_mode": {
+                    "n_runs": 1,
+                    "refusal_rate": {
+                        "status": "measured", "mean": 0.0, "std": 0.0, "n": 1,
+                    },
+                },
+            },
+        })
+        assert "model\\_name\\&x" in latex
+        assert "advanced\\_mode" in latex
+
     def test_includes_model_names(self):
         agg = self._sample_aggregated()
         latex = generate_latex_table(agg)

tests/test_metrics.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-
+import pytest
 import torch
 
 from obliteratus.evaluation.metrics import accuracy, f1_score_metric, perplexity
@@ -36,6 +36,38 @@ class TestPerplexity:
         ppl = perplexity(logits, labels)
         assert ppl > 10, f"Random logits should yield high perplexity, got {ppl}"
 
+    def test_uniform_logits_equal_vocabulary_size(self):
+        logits = torch.zeros(2, 4, 7)
+        labels = torch.tensor([[0, 1, 2, 3], [3, 4, 5, 6]])
+        assert perplexity(logits, labels) == pytest.approx(7.0)
+
+    @pytest.mark.parametrize(
+        ("logits", "labels", "message"),
+        [
+            (torch.zeros(2, 3), torch.zeros(2, 3, dtype=torch.long), "3D"),
+            (torch.zeros(2, 3, 4), torch.zeros(2, 3, 1, dtype=torch.long), "2D"),
+            (torch.zeros(2, 3, 4), torch.zeros(2, 2, dtype=torch.long), "does not match"),
+            (torch.zeros(2, 1, 4), torch.zeros(2, 1, dtype=torch.long), "two sequence"),
+        ],
+    )
+    def test_rejects_invalid_shapes(self, logits, labels, message):
+        with pytest.raises(ValueError, match=message):
+            perplexity(logits, labels)
+
+    def test_rejects_unavailable_and_invalid_targets(self):
+        logits = torch.zeros(1, 3, 4)
+        with pytest.raises(ValueError, match="every target"):
+            perplexity(logits, torch.full((1, 3), -100))
+        with pytest.raises(ValueError, match="outside"):
+            perplexity(logits, torch.tensor([[0, 1, 9]]))
+        logits[0, 0, 0] = float("nan")
+        with pytest.raises(ValueError, match="finite"):
+            perplexity(logits, torch.tensor([[0, 1, 2]]))
+
+    def test_rejects_non_integer_labels(self):
+        with pytest.raises(ValueError, match="integer token IDs"):
+            perplexity(torch.zeros(1, 3, 4), torch.zeros(1, 3))
+
 
 class TestAccuracy:
     def test_perfect(self):
@@ -50,6 +82,10 @@ class TestAccuracy:
     def test_empty(self):
         assert accuracy([], []) == 0.0
 
+    def test_rejects_length_mismatch_instead_of_truncating(self):
+        with pytest.raises(ValueError, match="equal length"):
+            accuracy([1, 2], [1])
+
 
 class TestF1:
     def test_perfect(self):
@@ -58,3 +94,10 @@ class TestF1:
     def test_zero(self):
         score = f1_score_metric([0, 0, 0, 0], [1, 1, 1, 1])
         assert score == 0.0
+
+    def test_empty(self):
+        assert f1_score_metric([], []) == 0.0
+
+    def test_rejects_length_mismatch(self):
+        with pytest.raises(ValueError, match="equal length"):
+            f1_score_metric([1, 2], [1])

tests/test_refusal_detection.py

@@ -132,8 +132,12 @@ class TestRefusalRateWithCI:
 
     def test_empty_responses(self):
         ci = refusal_rate_with_ci([], mode="combined")
-        assert ci["rate"] == 0.0
+        assert ci["available"] is False
+        assert ci["rate"] is None
+        assert ci["ci_lower"] is None
+        assert ci["ci_upper"] is None
         assert ci["n_samples"] == 0
+        assert ci["refusal_count"] == 0
 
     def test_ci_narrower_with_more_samples(self):
         """More samples should produce tighter confidence intervals."""

tests/test_report.py

@@ -4,7 +4,11 @@ from __future__ import annotations
 
 import json
 
-from obliteratus.reporting.report import AblationReport, AblationResult
+from obliteratus.reporting.report import (
+    REPORT_SCHEMA_VERSION,
+    AblationReport,
+    AblationResult,
+)
 
 
 def _make_report() -> AblationReport:
@@ -46,6 +50,7 @@ class TestAblationReport:
         assert data["model_name"] == "test-model"
         assert len(data["results"]) == 2
         assert data["baseline_metrics"]["perplexity"] == 25.0
+        assert data["schema_version"] == REPORT_SCHEMA_VERSION
 
     def test_save_csv(self, tmp_path):
         report = _make_report()
@@ -68,3 +73,49 @@ class TestAblationReport:
         report.plot_impact(metric="perplexity", output_path=out)
         assert out.exists()
         assert out.stat().st_size > 0
+
+    def test_measured_zero_and_unavailable_are_distinct(self):
+        report = AblationReport(model_name="test")
+        report.add_baseline({"refusal_rate": 0.0, "perplexity": None})
+        report.add_result(AblationResult(
+            strategy="advanced",
+            component="all",
+            description="partial evaluation",
+            metrics={"refusal_rate": 0.0, "perplexity": None},
+        ))
+        data = report.to_dict()
+        assert data["baseline_metric_status"] == {
+            "perplexity": "unavailable", "refusal_rate": "measured",
+        }
+        assert data["results"][0]["metric_status"] == {
+            "perplexity": "unavailable", "refusal_rate": "measured",
+        }
+        assert data["results"][0]["metrics"]["refusal_rate"] == 0.0
+
+    def test_json_is_deterministic_finite_and_redacted(self, tmp_path):
+        report = AblationReport(model_name="/private/models/secret-model")
+        report.add_baseline({"score": float("nan")})
+        report.add_result(AblationResult(
+            strategy="/private/run/advanced",
+            component="layer_0",
+            description="artifact /private/run/output hf_abcdefghijkl",
+            metrics={"score": float("inf")},
+            metadata={
+                "token": "hf_abcdefghijkl",
+                "artifact": "/private/run/checkpoint.bin",
+            },
+        ))
+        first = tmp_path / "first.json"
+        second = tmp_path / "second.json"
+        report.save_json(first)
+        report.save_json(second)
+        assert first.read_bytes() == second.read_bytes()
+        text = first.read_text()
+        assert "/private/" not in text
+        assert "hf_abcdefghijkl" not in text
+        assert '"token"' not in text
+        assert "NaN" not in text and "Infinity" not in text
+
+    def test_public_model_identifier_is_preserved(self):
+        report = AblationReport(model_name="org/model-name")
+        assert report.to_dict()["model_name"] == "org/model-name"

tests/test_telemetry.py

@@ -7,22 +7,33 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from unittest.mock import MagicMock, patch
 
+import pytest
 import torch
 
 from obliteratus.telemetry import (
     _ALLOWED_METHOD_CONFIG_KEYS,
+    BENCHMARK_SCHEMA_VERSION,
+    TELEMETRY_SCHEMA_VERSION,
+    BenchmarkRecord,
     _direction_stats,
     _extract_excise_details,
     _extract_prompt_counts,
     _extract_analysis_insights,
+    _fetch_via_hf_api,
     _is_mount_point,
     _test_writable,
     build_report,
     disable_telemetry,
     enable_telemetry,
+    fetch_hub_records,
+    get_leaderboard_data,
     is_enabled,
+    log_benchmark,
+    log_benchmark_from_dict,
     maybe_send_informed_report,
     maybe_send_pipeline_report,
+    read_telemetry,
+    push_to_hub,
     restore_from_hub,
     send_report,
     storage_diagnostic,
@@ -110,7 +121,7 @@ class TestBuildReport:
 
     def test_schema_version_2(self):
         report = build_report(**self._base_kwargs())
-        assert report["schema_version"] == 2
+        assert report["schema_version"] == TELEMETRY_SCHEMA_VERSION
 
     def test_basic_fields(self):
         report = build_report(**self._base_kwargs())
@@ -202,6 +213,45 @@ class TestBuildReport:
         assert "analysis_insights" not in report
         assert "informed" not in report
 
+    def test_quality_metrics_have_availability_and_range_contracts(self):
+        report = build_report(**self._base_kwargs(quality_metrics={
+            "refusal_rate": 0.0,
+            "perplexity": None,
+            "coherence": 2.0,
+            "unknown_metric": 4.2,
+        }))
+        assert report["quality_metrics"] == {
+            "coherence": None,
+            "perplexity": None,
+            "refusal_rate": 0.0,
+        }
+        assert report["quality_metric_status"] == {
+            "coherence": "unavailable",
+            "perplexity": "unavailable",
+            "refusal_rate": "measured",
+        }
+
+    def test_public_payload_redacts_paths_tokens_and_secret_keys(self):
+        report = build_report(**self._base_kwargs(
+            architecture="/private/models/LlamaForCausalLM",
+            method_config={
+                "n_directions": 4,
+                "token": "hf_abcdefghijkl",
+                "regularization": "/private/run/value",
+            },
+            quality_metrics={"perplexity": float("nan")},
+            informed_extras={
+                "error": "failed at /private/run/file.bin with sk-abcdefghijklmnop",
+                "api_key": "secret",
+            },
+        ))
+        encoded = json.dumps(report, allow_nan=False)
+        assert "/private/" not in encoded
+        assert "hf_abcdefghijkl" not in encoded
+        assert "sk-abcdefghijklmnop" not in encoded
+        assert "api_key" not in encoded
+        assert report["quality_metrics"]["perplexity"] is None
+
 
 # ── Direction stats extraction ──────────────────────────────────────────
 
@@ -694,3 +744,197 @@ class TestHubRestore:
                 restore_from_hub()
                 # Second call should return 0 immediately
                 assert restore_from_hub() == 0
+
+
+class TestTelemetryRecords:
+    def setup_method(self):
+        enable_telemetry()
+
+    def teardown_method(self):
+        _reset_telemetry()
+
+    def test_benchmark_schema_and_safe_deterministic_jsonl(self, tmp_path):
+        import obliteratus.telemetry as telemetry
+
+        output = tmp_path / "telemetry.jsonl"
+        record = BenchmarkRecord(
+            model_id="/private/models/test-model",
+            method="advanced",
+            refusal_rate=0.0,
+            perplexity=None,
+            error="failed at /private/run/file.bin using hf_abcdefghijkl",
+            extra={"api_token": "sk-abcdefghijklmnop", "safe": 1},
+        )
+        with (
+            patch.object(telemetry, "TELEMETRY_FILE", output),
+            patch("obliteratus.telemetry._schedule_hub_sync"),
+            patch("obliteratus.telemetry._detect_gpu", return_value=("", 0.0)),
+        ):
+            assert log_benchmark(record)
+
+        text = output.read_text()
+        data = json.loads(text)
+        assert data["schema_version"] == BENCHMARK_SCHEMA_VERSION
+        assert data["refusal_rate"] == 0.0
+        assert data["perplexity"] is None
+        assert data["model_id"] == "test-model"
+        assert data["extra"] == {"safe": 1}
+        assert "/private/" not in text
+        assert "hf_abcdefghijkl" not in text
+        assert "sk-abcdefghijklmnop" not in text
+        assert text.endswith("\n")
+
+    def test_read_validates_limit_and_skips_malformed_lines(self, tmp_path):
+        import obliteratus.telemetry as telemetry
+
+        output = tmp_path / "telemetry.jsonl"
+        output.write_text('{"timestamp":"1"}\nnot json\n{"timestamp":"2"}\n')
+        with patch.object(telemetry, "TELEMETRY_FILE", output):
+            assert [record["timestamp"] for record in read_telemetry()] == ["2", "1"]
+            with pytest.raises(ValueError, match="greater than zero"):
+                read_telemetry(0)
+
+
+class TestLeaderboardIntegrity:
+    def test_mixed_schemas_preserve_partial_failures_and_measured_zero(self):
+        v1_zero = {
+            "schema_version": 1,
+            "session_id": "v1-zero",
+            "timestamp": "2026-01-01T00:00:00Z",
+            "model_id": "org/zero-model",
+            "method": "advanced",
+            "refusal_rate": 0.0,
+            "perplexity": 4.0,
+            "coherence": 0.8,
+            "time_seconds": 5.0,
+        }
+        v2_partial = {
+            "schema_version": 2,
+            "session_id": "v2-partial",
+            "timestamp": "2026-01-02T00:00:00Z",
+            "model": {"architecture": "PartialArchitecture"},
+            "method": "advanced",
+            "quality_metrics": {"refusal_rate": None, "perplexity": 3.0},
+            "error": "coherence failed",
+        }
+        v1_missing = {
+            "schema_version": 1,
+            "session_id": "v1-missing",
+            "timestamp": "2026-01-03T00:00:00Z",
+            "model_id": "org/missing-model",
+            "method": "advanced",
+            "refusal_rate": None,
+            "perplexity": 2.0,
+        }
+        with (
+            patch("obliteratus.telemetry.read_telemetry", return_value=[v1_missing, v1_zero]),
+            patch("obliteratus.telemetry.fetch_hub_records", return_value=[v2_partial]),
+        ):
+            leaderboard = get_leaderboard_data()
+
+        assert leaderboard[0]["model_id"] == "org/zero-model"
+        assert leaderboard[0]["best_refusal"] == 0.0
+        partial = next(row for row in leaderboard if row["model_id"] == "PartialArchitecture")
+        assert partial["runs"] == 1
+        assert partial["successful_runs"] == 0
+        assert partial["failed_runs"] == 1
+        assert partial["perplexity_measurements"] == 1
+        assert partial["best_perplexity"] == 3.0
+        assert partial["best_refusal"] is None
+
+    def test_invalid_ranges_do_not_enter_aggregates(self):
+        record = {
+            "session_id": "bad", "timestamp": "1", "model_id": "bad/model",
+            "method": "advanced", "refusal_rate": -0.1,
+            "perplexity": float("nan"), "coherence": True,
+        }
+        with (
+            patch("obliteratus.telemetry.read_telemetry", return_value=[record]),
+            patch("obliteratus.telemetry.fetch_hub_records", return_value=[]),
+        ):
+            row = get_leaderboard_data()[0]
+        assert row["refusal_measurements"] == 0
+        assert row["perplexity_measurements"] == 0
+        assert row["coherence_measurements"] == 0
+        assert row["best_refusal"] is None
+
+
+class TestTelemetryHubBoundaries:
+    def test_fetch_prefers_api_and_falls_back_to_git(self):
+        api_records = [{"session_id": "api"}]
+        with (
+            patch("obliteratus.telemetry._fetch_via_hf_api", return_value=api_records),
+            patch("obliteratus.telemetry._fetch_via_git_clone") as git_fetch,
+        ):
+            assert fetch_hub_records(3) == api_records
+            git_fetch.assert_not_called()
+
+        git_records = [{"session_id": "git"}]
+        with (
+            patch("obliteratus.telemetry._fetch_via_hf_api", return_value=[]),
+            patch("obliteratus.telemetry._fetch_via_git_clone", return_value=git_records),
+        ):
+            assert fetch_hub_records(3) == git_records
+
+    def test_fetch_returns_empty_when_both_boundaries_fail(self):
+        with (
+            patch("obliteratus.telemetry._fetch_via_hf_api", side_effect=RuntimeError("api")),
+            patch("obliteratus.telemetry._fetch_via_git_clone", side_effect=RuntimeError("git")),
+        ):
+            assert fetch_hub_records() == []
+
+    def test_hf_api_parser_filters_files_malformed_lines_and_limit(self, tmp_path):
+        first = tmp_path / "first.jsonl"
+        second = tmp_path / "second.jsonl"
+        first.write_text('\n{"session_id":"one"}\nnot-json\n{"session_id":"two"}\n')
+        second.write_text('{"session_id":"three"}\n')
+        api = MagicMock()
+        api.list_repo_files.return_value = [
+            "README.md", "other/ignored.jsonl", "data/first.jsonl", "data/second.jsonl",
+        ]
+        with (
+            patch("huggingface_hub.HfApi", return_value=api),
+            patch("huggingface_hub.hf_hub_download", side_effect=[str(first), str(second)]) as download,
+        ):
+            records = _fetch_via_hf_api("org/repo", 2)
+        assert [record["session_id"] for record in records] == ["one", "two"]
+        assert download.call_count == 1
+
+    def test_log_from_dict_maps_partial_result_without_erasing_error(self):
+        with patch("obliteratus.telemetry.log_benchmark", return_value=True) as write:
+            assert log_benchmark_from_dict(
+                "org/model",
+                "advanced",
+                {"refusal_rate": 0.0, "perplexity": None, "error": "partial"},
+                dataset="fixture",
+                n_prompts=4,
+                pipeline_config={"n_directions": 3, "bayesian_trials": 2},
+            )
+        record = write.call_args.args[0]
+        assert record.refusal_rate == 0.0
+        assert record.perplexity is None
+        assert record.error == "partial"
+        assert record.n_directions == 3
+        assert record.use_bayesian is True
+
+    def test_push_to_hub_success_and_empty_short_circuits(self, tmp_path):
+        import obliteratus.telemetry as telemetry
+
+        output = tmp_path / "telemetry.jsonl"
+        output.write_text('{"session_id":"one"}\n')
+        api = MagicMock()
+        with (
+            patch.object(telemetry, "TELEMETRY_FILE", output),
+            patch("obliteratus.telemetry.read_telemetry", return_value=[{"session_id": "one"}]),
+            patch("obliteratus.telemetry._ensure_hub_repo", return_value=True),
+            patch("huggingface_hub.HfApi", return_value=api),
+            patch("obliteratus.telemetry._instance_slug", return_value="instance"),
+        ):
+            assert push_to_hub("org/repo") is True
+        api.upload_file.assert_called_once()
+        assert api.upload_file.call_args.kwargs["path_in_repo"] == "data/instance.jsonl"
+
+        with patch("obliteratus.telemetry.read_telemetry", return_value=[]):
+            assert push_to_hub("org/repo") is False
+    get_leaderboard_data,
+    read_telemetry,