diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 561dcbb..2033345 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -289,6 +289,11 @@ jobs: --min-file obliteratus/architecture_profiles.py=70 --min-file obliteratus/cli.py=70 --min-file obliteratus/mlx_backend.py=70 + --min-file obliteratus/evaluation/metrics.py=70 + --min-file obliteratus/evaluation/advanced_metrics.py=70 + --min-file obliteratus/reporting/report.py=70 + --min-file obliteratus/community.py=70 + --min-file obliteratus/telemetry.py=70 --min-changed 90 --base-ref "$COVERAGE_BASE" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c604c4a..3103b2a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,9 +34,10 @@ python -m obliteratus --help All tests must pass before submitting a PR. Tests are designed to run on CPU without downloading models. The mandatory gate currently requires at least 60% repository statement coverage, 42% branch coverage, 90% coverage of changed executable lines, and 70% statement -coverage in the device, loader, architecture-profile, CLI, and simulated MLX -boundary modules. New changes should raise these floors rather than consume the -existing margin. +coverage in the device, loader, architecture-profile, CLI, simulated MLX, +evaluation-metric, reporting, community-contribution, and telemetry boundary +modules. New changes should raise these floors rather than consume the existing +margin. ## Code Style diff --git a/obliteratus/community.py b/obliteratus/community.py index fcbf58a..6b68fd2 100644 --- a/obliteratus/community.py +++ b/obliteratus/community.py @@ -33,7 +33,10 @@ from obliteratus.telemetry import ( _extract_prompt_counts, _extract_stage_durations, _get_peak_vram, + _metric_value, _safe_float, + _sanitize_public_text, + _sanitize_public_value, build_report, ) @@ -57,10 +60,26 @@ def _model_short_name(model_name: str) -> str: def _config_fingerprint(config: dict[str, Any]) -> str: """Deterministic short hash of the method configuration.""" - canonical = json.dumps(config, sort_keys=True, default=str) + canonical = json.dumps(_sanitize_public_value(config), sort_keys=True, allow_nan=False) return hashlib.sha256(canonical.encode()).hexdigest()[:8] +def validate_contribution(data: Any) -> bool: + """Return whether a contribution matches the supported public schema.""" + if not isinstance(data, dict): + return False + if data.get("contribution_schema_version") != CONTRIBUTION_SCHEMA_VERSION: + return False + telemetry = data.get("telemetry") + return ( + isinstance(data.get("model_name"), str) + and isinstance(data.get("timestamp"), str) + and isinstance(telemetry, dict) + and telemetry.get("schema_version") == 2 + and isinstance(telemetry.get("quality_metrics", {}), dict) + ) + + def save_contribution( pipeline, *, @@ -153,20 +172,23 @@ def save_contribution( contribution = { "contribution_schema_version": CONTRIBUTION_SCHEMA_VERSION, "timestamp": timestamp, - "model_name": model_name, + "model_name": _sanitize_public_text(model_name, 120), "config_fingerprint": _config_fingerprint(method_config), - "notes": notes, + "notes": _sanitize_public_text(notes, 240), "telemetry": base_report, } # Generate filename short_name = _model_short_name(model_name) - method = pipeline.method + method = _model_short_name(str(pipeline.method)) or "unknown" ts_short = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") filename = f"{short_name}_{method}_{ts_short}.json" filepath = output_dir / filename - filepath.write_text(json.dumps(contribution, indent=2, default=str)) + filepath.write_text( + json.dumps(contribution, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) logger.info("Community contribution saved: %s", filepath) return filepath @@ -190,8 +212,8 @@ def load_contributions( for path in sorted(contrib_dir.glob("*.json")): try: data = json.loads(path.read_text()) - if "contribution_schema_version" in data: - data["_source_file"] = str(path) + if validate_contribution(data): + data["_source_file"] = path.name records.append(data) except (json.JSONDecodeError, OSError) as e: logger.warning("Skipping invalid contribution file %s: %s", path, e) @@ -216,9 +238,9 @@ def aggregate_results( groups: dict[tuple[str, str], list[dict]] = {} for record in records: - model = record.get("model_name", "unknown") + model = _sanitize_public_text(record.get("model_name", "unknown"), 120) telemetry = record.get("telemetry", {}) - method = telemetry.get("method", "unknown") + method = _sanitize_public_text(telemetry.get("method", "unknown"), 80) metrics = telemetry.get("quality_metrics", {}) key = (model, method) @@ -235,17 +257,29 @@ def aggregate_results( for metric_name in ["refusal_rate", "perplexity", "coherence"]: values = [ - m[metric_name] - for m in metric_list - if metric_name in m and m[metric_name] is not None + value + for metrics in metric_list + if (value := _metric_value(metric_name, metrics.get(metric_name))) is not None ] if values: summary[metric_name] = { + "status": "measured", "mean": round(statistics.mean(values), 4), "std": round(statistics.stdev(values), 4) if len(values) > 1 else 0.0, "n": len(values), "min": round(min(values), 4), "max": round(max(values), 4), + "unavailable": len(metric_list) - len(values), + } + else: + summary[metric_name] = { + "status": "unavailable", + "mean": None, + "std": None, + "n": 0, + "min": None, + "max": None, + "unavailable": len(metric_list), } results[model][method] = summary @@ -274,8 +308,15 @@ def generate_latex_table( all_methods.update(model_data.keys()) methods = sorted(all_methods) + def _escape(value: str) -> str: + translations = { + "\\": r"\textbackslash{}", "&": r"\&", "%": r"\%", "$": r"\$", + "#": r"\#", "_": r"\_", "{": r"\{", "}": r"\}", + } + return "".join(translations.get(char, char) for char in value) + # Build header - method_cols = " & ".join(f"\\textbf{{{m}}}" for m in methods) + method_cols = " & ".join(f"\\textbf{{{_escape(m)}}}" for m in methods) header = f"\\textbf{{Model}} & {method_cols} \\\\" lines = [ @@ -291,7 +332,11 @@ def generate_latex_table( cells = [] for method in methods: - if method in model_data and metric in model_data[method]: + if ( + method in model_data + and metric in model_data[method] + and model_data[method][metric].get("status", "measured") == "measured" + ): stats = model_data[method][metric] mean = stats["mean"] n = stats["n"] @@ -302,7 +347,7 @@ def generate_latex_table( else: cells.append("---") - row = f"{short} & " + " & ".join(cells) + " \\\\" + row = f"{_escape(short)} & " + " & ".join(cells) + " \\\\" lines.append(row) lines.extend(["\\bottomrule", "\\end{tabular}"]) diff --git a/obliteratus/evaluation/advanced_metrics.py b/obliteratus/evaluation/advanced_metrics.py index f86a86f..9e9f8fe 100644 --- a/obliteratus/evaluation/advanced_metrics.py +++ b/obliteratus/evaluation/advanced_metrics.py @@ -31,6 +31,7 @@ References: from __future__ import annotations +import math import re from dataclasses import dataclass @@ -184,6 +185,37 @@ REFUSAL_SUBSTRINGS = [ "не могу помочь", # Russian: can't help ] +_REFUSAL_MODES = frozenset({"prefix", "substring", "semantic", "combined"}) + + +def _validate_refusal_mode(mode: str) -> None: + if mode not in _REFUSAL_MODES: + raise ValueError( + f"Unknown refusal detection mode {mode!r}; choose from {sorted(_REFUSAL_MODES)}", + ) + + +def _validate_tensor_pair( + first: torch.Tensor, + second: torch.Tensor, + *, + name: str, + dimensions: tuple[int, ...], + require_same_shape: bool = True, +) -> None: + if first.dim() not in dimensions or second.dim() not in dimensions: + allowed = ", ".join(f"{value}D" for value in dimensions) + raise ValueError(f"{name} inputs must both be {allowed} tensors") + if require_same_shape and first.shape != second.shape: + raise ValueError( + f"{name} inputs must have identical shapes, got " + f"{tuple(first.shape)} and {tuple(second.shape)}", + ) + if first.numel() == 0 or second.numel() == 0: + raise ValueError(f"{name} inputs must not be empty") + if not torch.isfinite(first).all() or not torch.isfinite(second).all(): + raise ValueError(f"{name} inputs must contain only finite values") + def refusal_rate( responses: list[str], @@ -201,6 +233,7 @@ def refusal_rate( Returns: Fraction of responses that are refusals (0.0 = no refusals, 1.0 = all refuse). """ + _validate_refusal_mode(mode) if not responses: return 0.0 @@ -296,6 +329,7 @@ def _is_refusal(response: str, mode: str = "combined") -> bool: Empty/blank responses return False (no content to classify). Degenerate outputs (broken model) return False — they are not refusals. """ + _validate_refusal_mode(mode) text = response.strip() if not text: return False @@ -336,6 +370,7 @@ def _is_refusal_detailed(response: str, mode: str = "combined") -> tuple[bool, s describing which pattern matched, or "" if no match. For degenerate outputs (broken model), returns (False, "DEGENERATE"). """ + _validate_refusal_mode(mode) text = response.strip() if not text: return False, "" @@ -406,22 +441,39 @@ def refusal_rate_with_ci( responses: list[str], mode: str = "combined", confidence: float = 0.95, -) -> dict[str, float]: +) -> dict[str, float | int | bool | None]: """Compute refusal rate with a Wilson score confidence interval.""" + _validate_refusal_mode(mode) + z_map = {0.90: 1.645, 0.95: 1.96, 0.99: 2.576} + if confidence not in z_map: + raise ValueError("confidence must be one of 0.90, 0.95, or 0.99") n = len(responses) if n == 0: - return {"rate": 0.0, "ci_lower": 0.0, "ci_upper": 0.0, "n_samples": 0} + return { + "available": False, + "rate": None, + "ci_lower": None, + "ci_upper": None, + "n_samples": 0, + "refusal_count": 0, + } refusals = sum(1 for r in responses if _is_refusal(r, mode)) rate = refusals / n import math as _math - z_map = {0.90: 1.645, 0.95: 1.96, 0.99: 2.576} - z = z_map.get(confidence, 1.96) + z = z_map[confidence] denominator = 1 + z * z / n center = (rate + z * z / (2 * n)) / denominator spread = z * _math.sqrt((rate * (1 - rate) + z * z / (4 * n)) / n) / denominator ci_lower = max(0.0, center - spread) ci_upper = min(1.0, center + spread) - return {"rate": rate, "ci_lower": round(ci_lower, 6), "ci_upper": round(ci_upper, 6), "n_samples": n} + return { + "available": True, + "rate": rate, + "ci_lower": round(ci_lower, 6), + "ci_upper": round(ci_upper, 6), + "n_samples": n, + "refusal_count": refusals, + } # ── KL Divergence ──────────────────────────────────────────────────────── @@ -443,6 +495,16 @@ def token_kl_divergence( Returns: Mean KL divergence across all tokens (nats). Lower = more similar. """ + _validate_tensor_pair( + logits_original, + logits_modified, + name="token KL", + dimensions=(3,), + ) + if not isinstance(temperature, (int, float)) or not math.isfinite(temperature): + raise ValueError("temperature must be a finite positive number") + if temperature <= 0: + raise ValueError("temperature must be greater than zero") log_p = F.log_softmax(logits_original / temperature, dim=-1) log_q = F.log_softmax(logits_modified / temperature, dim=-1) p = F.softmax(logits_original / temperature, dim=-1) @@ -467,6 +529,14 @@ def first_token_kl_divergence( Returns: Mean first-token KL divergence across batch. """ + _validate_tensor_pair( + logits_original, + logits_modified, + name="first-token KL", + dimensions=(3,), + ) + if logits_original.shape[1] == 0: + raise ValueError("first-token KL requires at least one sequence position") # Take logits at the last input position (predicting first generated token) first_logits_orig = logits_original[:, -1, :] # (batch, vocab) first_logits_mod = logits_modified[:, -1, :] @@ -502,6 +572,10 @@ def effective_rank(weight_matrix: torch.Tensor) -> float: W = weight_matrix.float() if W.dim() != 2: raise ValueError(f"Expected 2D tensor, got {W.dim()}D") + if W.numel() == 0: + raise ValueError("Expected a non-empty weight matrix") + if not torch.isfinite(W).all(): + raise ValueError("Weight matrix must contain only finite values") s = torch.linalg.svdvals(W) s = s[s > 1e-12] # filter near-zero @@ -551,6 +625,12 @@ def activation_cosine_similarity( Returns: Mean cosine similarity (1.0 = identical, 0.0 = orthogonal). """ + _validate_tensor_pair( + acts_original, + acts_modified, + name="activation cosine", + dimensions=(1, 2, 3), + ) a = acts_original.float() b = acts_modified.float() @@ -585,6 +665,13 @@ def linear_cka( References: Kornblith et al. (2019): Similarity of Neural Network Representations """ + _validate_tensor_pair( + X, + Y, + name="linear CKA", + dimensions=(2, 3), + require_same_shape=False, + ) X = X.float() Y = Y.float() @@ -592,6 +679,13 @@ def linear_cka( X = X.reshape(-1, X.shape[-1]) if Y.dim() == 3: Y = Y.reshape(-1, Y.shape[-1]) + if X.shape[0] != Y.shape[0]: + raise ValueError( + f"linear CKA inputs must have the same sample count, got " + f"{X.shape[0]} and {Y.shape[0]}", + ) + if X.shape[0] < 2: + raise ValueError("linear CKA requires at least two samples") # Column-center X = X - X.mean(dim=0, keepdim=True) @@ -628,6 +722,12 @@ def refusal_projection_magnitude( Returns: Dict with mean, std, max, min projection magnitudes. """ + if activations.dim() not in (2, 3) or activations.numel() == 0: + raise ValueError("activations must be a non-empty 2D or 3D tensor") + if refusal_direction.numel() == 0 or refusal_direction.dim() not in (1, 2): + raise ValueError("refusal_direction must be a non-empty vector") + if not torch.isfinite(activations).all() or not torch.isfinite(refusal_direction).all(): + raise ValueError("projection inputs must contain only finite values") acts = activations.float() if acts.dim() == 3: acts = acts.reshape(-1, acts.shape[-1]) @@ -635,13 +735,21 @@ def refusal_projection_magnitude( d = refusal_direction.float() if d.dim() > 1: d = d.squeeze() - d = d / d.norm().clamp(min=1e-8) + if d.dim() != 1 or d.shape[0] != acts.shape[-1]: + raise ValueError( + f"refusal direction shape {tuple(d.shape)} does not match activation " + f"width {acts.shape[-1]}", + ) + norm = d.norm() + if norm <= 1e-8: + raise ValueError("refusal_direction must have non-zero norm") + d = d / norm projections = acts @ d # (n_samples,) return { "mean": projections.mean().item(), - "std": projections.std().item(), + "std": projections.std(unbiased=False).item(), "max": projections.max().item(), "min": projections.min().item(), "abs_mean": projections.abs().mean().item(), @@ -654,11 +762,11 @@ def refusal_projection_magnitude( class AbliterationEvalResult: """Comprehensive evaluation result for an abliterated model.""" - refusal_rate_harmful: float # fraction of harmful prompts still refused - refusal_rate_harmless: float # over-refusal rate on harmless prompts + refusal_rate_harmful: float | None # fraction of harmful prompts still refused + refusal_rate_harmless: float | None # over-refusal rate on harmless prompts kl_divergence: float | None # KL(original || modified) on harmless prompts - perplexity: float # perplexity on reference text - coherence_score: float # basic coherence score + perplexity: float | None # perplexity on reference text + coherence_score: float | None # basic coherence score mean_activation_cosine: float | None # activation similarity original vs modified mean_cka: float | None # CKA similarity across layers @@ -672,14 +780,26 @@ def format_eval_report(result: AbliterationEvalResult) -> str: # Refusal removal effectiveness lines.append("Refusal Removal:") - lines.append(f" Harmful prompt refusal rate: {result.refusal_rate_harmful:.1%}") - lines.append(f" Harmless prompt over-refusal: {result.refusal_rate_harmless:.1%}") + harmful = ( + f"{result.refusal_rate_harmful:.1%}" + if result.refusal_rate_harmful is not None else "unavailable" + ) + harmless = ( + f"{result.refusal_rate_harmless:.1%}" + if result.refusal_rate_harmless is not None else "unavailable" + ) + lines.append(f" Harmful prompt refusal rate: {harmful}") + lines.append(f" Harmless prompt over-refusal: {harmless}") lines.append("") # Model quality lines.append("Model Quality:") - lines.append(f" Perplexity: {result.perplexity:.2f}") - lines.append(f" Coherence: {result.coherence_score:.1%}") + perplexity = f"{result.perplexity:.2f}" if result.perplexity is not None else "unavailable" + coherence = ( + f"{result.coherence_score:.1%}" if result.coherence_score is not None else "unavailable" + ) + lines.append(f" Perplexity: {perplexity}") + lines.append(f" Coherence: {coherence}") if result.kl_divergence is not None: lines.append(f" KL divergence: {result.kl_divergence:.4f}") if result.kl_divergence < 0.2: @@ -691,13 +811,19 @@ def format_eval_report(result: AbliterationEvalResult) -> str: else: quality = "significant damage" lines.append(f" ({quality})") + else: + lines.append(" KL divergence: unavailable") lines.append("") # Representation similarity + lines.append("Representation Similarity:") if result.mean_activation_cosine is not None: - lines.append("Representation Similarity:") lines.append(f" Activation cosine similarity: {result.mean_activation_cosine:.4f}") - if result.mean_cka is not None: - lines.append(f" Linear CKA: {result.mean_cka:.4f}") + else: + lines.append(" Activation cosine similarity: unavailable") + if result.mean_cka is not None: + lines.append(f" Linear CKA: {result.mean_cka:.4f}") + else: + lines.append(" Linear CKA: unavailable") return "\n".join(lines) diff --git a/obliteratus/evaluation/metrics.py b/obliteratus/evaluation/metrics.py index d42aded..46983db 100644 --- a/obliteratus/evaluation/metrics.py +++ b/obliteratus/evaluation/metrics.py @@ -20,9 +20,32 @@ def perplexity(logits: torch.Tensor, labels: torch.Tensor) -> float: Returns: Scalar perplexity (lower is better). """ + if logits.dim() != 3: + raise ValueError(f"logits must be 3D (batch, sequence, vocabulary), got {logits.dim()}D") + if labels.dim() != 2: + raise ValueError(f"labels must be 2D (batch, sequence), got {labels.dim()}D") + if labels.dtype not in { + torch.int8, torch.int16, torch.int32, torch.int64, torch.uint8, + }: + raise ValueError("labels must contain integer token IDs") + if logits.shape[:2] != labels.shape: + raise ValueError( + f"logits batch/sequence shape {tuple(logits.shape[:2])} does not match " + f"labels shape {tuple(labels.shape)}", + ) + if logits.shape[1] < 2: + raise ValueError("perplexity requires at least two sequence positions") + if not torch.isfinite(logits).all(): + raise ValueError("logits must contain only finite values") + # Shift so that tokens < n predict n shift_logits = logits[:, :-1, :].contiguous() shift_labels = labels[:, 1:].contiguous() + valid_labels = shift_labels[shift_labels != -100] + if valid_labels.numel() == 0: + raise ValueError("perplexity is unavailable because every target label is ignored") + if (valid_labels < 0).any() or (valid_labels >= logits.shape[-1]).any(): + raise ValueError("labels contain token IDs outside the logits vocabulary") loss = F.cross_entropy( shift_logits.view(-1, shift_logits.size(-1)), @@ -35,6 +58,11 @@ def perplexity(logits: torch.Tensor, labels: torch.Tensor) -> float: def accuracy(predictions: Sequence[int], references: Sequence[int]) -> float: """Simple accuracy.""" + if len(predictions) != len(references): + raise ValueError( + f"predictions and references must have equal length, got " + f"{len(predictions)} and {len(references)}", + ) if len(predictions) == 0: return 0.0 correct = sum(p == r for p, r in zip(predictions, references)) @@ -47,4 +75,11 @@ def f1_score_metric( average: str = "macro", ) -> float: """F1 score wrapper around sklearn.""" + if len(predictions) != len(references): + raise ValueError( + f"predictions and references must have equal length, got " + f"{len(predictions)} and {len(references)}", + ) + if len(predictions) == 0: + return 0.0 return float(sklearn_f1(references, predictions, average=average, zero_division=0)) diff --git a/obliteratus/reporting/report.py b/obliteratus/reporting/report.py index 81048cd..697b4f7 100644 --- a/obliteratus/reporting/report.py +++ b/obliteratus/reporting/report.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import math import re from dataclasses import dataclass, field from pathlib import Path @@ -10,17 +11,65 @@ from typing import Any import pandas as pd +REPORT_SCHEMA_VERSION = 1 +_SENSITIVE_KEY_RE = re.compile( + r"(?:authorization|credential|password|secret|token|api[_-]?key)", re.IGNORECASE, +) + def _sanitize_label(text: str, max_len: int = 80) -> str: """Strip filesystem paths, tokens, and overly-long strings from labels.""" - text = re.sub(r"(/[a-zA-Z0-9_./-]{3,})", lambda m: m.group(0).rsplit("/", 1)[-1], text) + if text.startswith("/") or re.match(r"^[A-Za-z]:[\\/]", text): + text = Path(text).name + text = re.sub( + r"(?:/[A-Za-z0-9_.-]+){2,}", lambda match: Path(match.group()).name, text, + ) text = re.sub(r"\bhf_[A-Za-z0-9]{6,}\b", "", text) + text = re.sub(r"\bgh[pousr]_[A-Za-z0-9]{12,}\b", "", text) + text = re.sub(r"\bgithub_pat_[A-Za-z0-9_]{12,}\b", "", text) + text = re.sub(r"\bsk-[A-Za-z0-9_-]{12,}\b", "", text) text = re.sub(r"\b[0-9a-fA-F]{32,}\b", "", text) if len(text) > max_len: text = text[: max_len - 3] + "..." return text +def _sanitize_public_value(value: Any) -> Any: + """Return a deterministic, JSON-safe value with private material removed.""" + if value is None or isinstance(value, (bool, int)): + return value + if isinstance(value, float): + return value if math.isfinite(value) else None + if isinstance(value, str): + return _sanitize_label(value, max_len=240) + if isinstance(value, dict): + return { + str(key): _sanitize_public_value(item) + for key, item in sorted(value.items(), key=lambda pair: str(pair[0])) + if not _SENSITIVE_KEY_RE.search(str(key)) + } + if isinstance(value, (list, tuple)): + return [_sanitize_public_value(item) for item in value] + return _sanitize_label(str(value), max_len=240) + + +def _normalize_metrics(metrics: dict[str, Any]) -> tuple[dict[str, float | None], dict[str, str]]: + values: dict[str, float | None] = {} + status: dict[str, str] = {} + for name, raw_value in sorted(metrics.items()): + value = None + if not isinstance(raw_value, bool): + try: + candidate = float(raw_value) + if math.isfinite(candidate): + value = candidate + except (TypeError, ValueError): + pass + values[str(name)] = value + status[str(name)] = "measured" if value is not None else "unavailable" + return values, status + + @dataclass class AblationResult: """Result of a single ablation experiment.""" @@ -28,7 +77,7 @@ class AblationResult: strategy: str component: str description: str - metrics: dict[str, float] + metrics: dict[str, float | None] metadata: dict[str, Any] | None = None @@ -37,28 +86,52 @@ class AblationReport: """Collects results and produces tables / charts / exports.""" model_name: str - baseline_metrics: dict[str, float] = field(default_factory=dict) + baseline_metrics: dict[str, float | None] = field(default_factory=dict) results: list[AblationResult] = field(default_factory=list) - def add_baseline(self, metrics: dict[str, float]): + def add_baseline(self, metrics: dict[str, float | None]): self.baseline_metrics = metrics def add_result(self, result: AblationResult): self.results.append(result) + def to_dict(self) -> dict[str, Any]: + """Return the canonical, versioned public report representation.""" + baseline_metrics, baseline_status = _normalize_metrics(self.baseline_metrics) + results = [] + for result in self.results: + metrics, metric_status = _normalize_metrics(result.metrics) + results.append({ + "strategy": _sanitize_label(result.strategy), + "component": _sanitize_label(result.component), + "description": _sanitize_label(result.description, max_len=240), + "metrics": metrics, + "metric_status": metric_status, + "metadata": _sanitize_public_value(result.metadata), + }) + return { + "schema_version": REPORT_SCHEMA_VERSION, + "model_name": _sanitize_label(self.model_name), + "baseline_metrics": baseline_metrics, + "baseline_metric_status": baseline_status, + "results": results, + } + def to_dataframe(self) -> pd.DataFrame: """Convert results to a pandas DataFrame with delta columns.""" rows = [] + baseline_metrics, _ = _normalize_metrics(self.baseline_metrics) for r in self.results: + metrics, _ = _normalize_metrics(r.metrics) row = { - "strategy": r.strategy, - "component": r.component, - "description": r.description, + "strategy": _sanitize_label(r.strategy), + "component": _sanitize_label(r.component), + "description": _sanitize_label(r.description, max_len=240), } - for metric_name, value in r.metrics.items(): + for metric_name, value in metrics.items(): row[metric_name] = value - baseline_val = self.baseline_metrics.get(metric_name) - if baseline_val is not None: + baseline_val = baseline_metrics.get(metric_name) + if baseline_val is not None and value is not None: row[f"{metric_name}_delta"] = value - baseline_val if baseline_val != 0: row[f"{metric_name}_pct_change"] = ( @@ -84,7 +157,8 @@ class AblationReport: table.add_column("Strategy", style="cyan") table.add_column("Component", style="green") - metric_names = list(self.baseline_metrics.keys()) + baseline_metrics, _ = _normalize_metrics(self.baseline_metrics) + metric_names = list(baseline_metrics.keys()) for m in metric_names: table.add_column(f"{m}", justify="right") table.add_column(f"{m} delta", justify="right", style="red") @@ -92,7 +166,8 @@ class AblationReport: # Baseline row baseline_vals = [] for m in metric_names: - baseline_vals.extend([f"{self.baseline_metrics[m]:.4f}", "—"]) + value = baseline_metrics[m] + baseline_vals.extend([f"{value:.4f}" if value is not None else "unavailable", "—"]) table.add_row("baseline", "—", *baseline_vals, style="bold") for _, row in df.iterrows(): @@ -100,7 +175,7 @@ class AblationReport: for m in metric_names: val = row.get(m, float("nan")) delta = row.get(f"{m}_delta", float("nan")) - cells.append(f"{val:.4f}") + cells.append(f"{val:.4f}" if pd.notna(val) else "unavailable") cells.append(f"{delta:+.4f}" if pd.notna(delta) else "—") table.add_row(*cells) @@ -110,27 +185,16 @@ class AblationReport: """Save raw results to JSON.""" path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) - data = { - "model_name": self.model_name, - "baseline_metrics": self.baseline_metrics, - "results": [ - { - "strategy": r.strategy, - "component": r.component, - "description": r.description, - "metrics": r.metrics, - "metadata": r.metadata, - } - for r in self.results - ], - } - path.write_text(json.dumps(data, indent=2)) + path.write_text( + json.dumps(self.to_dict(), indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) def save_csv(self, path: str | Path): """Save results DataFrame to CSV.""" path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) - self.to_dataframe().to_csv(path, index=False) + self.to_dataframe().to_csv(path, index=False, na_rep="unavailable", lineterminator="\n") def plot_impact(self, metric: str | None = None, output_path: str | Path | None = None): """Generate a bar chart showing the impact of each ablation on a metric. diff --git a/obliteratus/telemetry.py b/obliteratus/telemetry.py index a473bbb..d33c88a 100644 --- a/obliteratus/telemetry.py +++ b/obliteratus/telemetry.py @@ -33,6 +33,7 @@ import logging import math import os import platform +import re import time import threading import uuid @@ -43,6 +44,86 @@ from typing import Any logger = logging.getLogger(__name__) +BENCHMARK_SCHEMA_VERSION = 1 +TELEMETRY_SCHEMA_VERSION = 2 +_SENSITIVE_KEY_RE = re.compile( + r"(?:authorization|credential|password|secret|token|api[_-]?key)", re.IGNORECASE, +) +_PUBLIC_METRIC_RANGES: dict[str, tuple[float | None, float | None]] = { + "refusal_rate": (0.0, 1.0), + "perplexity": (0.0, None), + "coherence": (0.0, 1.0), + "kl_divergence": (0.0, None), + "capability_score": (0.0, 1.0), + "degenerate_count": (0.0, None), + "time_seconds": (0.0, None), +} + + +def _sanitize_public_text(value: Any, max_len: int = 240) -> str: + text = str(value) + if text.startswith("/") or re.match(r"^[A-Za-z]:[\\/]", text): + text = Path(text).name + text = re.sub(r"\bhf_[A-Za-z0-9]{6,}\b", "", text) + text = re.sub(r"\bgh[pousr]_[A-Za-z0-9]{12,}\b", "", text) + text = re.sub(r"\bgithub_pat_[A-Za-z0-9_]{12,}\b", "", text) + text = re.sub(r"\bsk-[A-Za-z0-9_-]{12,}\b", "", text) + text = re.sub(r"(?:/[A-Za-z0-9_.-]+){2,}", lambda match: Path(match.group()).name, text) + return text if len(text) <= max_len else text[: max_len - 3] + "..." + + +def _sanitize_public_value(value: Any) -> Any: + if value is None or isinstance(value, (bool, int)): + return value + if isinstance(value, float): + return value if math.isfinite(value) else None + if isinstance(value, str): + return _sanitize_public_text(value) + if isinstance(value, dict): + return { + str(key): _sanitize_public_value(item) + for key, item in sorted(value.items(), key=lambda pair: str(pair[0])) + if not _SENSITIVE_KEY_RE.search(str(key)) + } + if isinstance(value, (list, tuple)): + return [_sanitize_public_value(item) for item in value] + return _sanitize_public_text(value) + + +def _metric_value(name: str, value: Any) -> float | None: + if isinstance(value, bool): + return None + numeric = _safe_float(value) + if numeric is None: + return None + minimum, maximum = _PUBLIC_METRIC_RANGES.get(name, (None, None)) + if minimum is not None and numeric < minimum: + return None + if maximum is not None and numeric > maximum: + return None + return numeric + + +def _normalize_quality_metrics( + metrics: dict[str, Any] | None, +) -> tuple[dict[str, Any], dict[str, str]]: + values: dict[str, Any] = {} + statuses: dict[str, str] = {} + for name, raw_value in sorted((metrics or {}).items()): + if name not in _PUBLIC_METRIC_RANGES and name not in { + "capability_results", "spectral_certification", + }: + continue + if name == "capability_results": + value = _sanitize_public_value(raw_value) if isinstance(raw_value, dict) else None + elif name == "spectral_certification": + value = _sanitize_public_text(raw_value, 40) if isinstance(raw_value, str) else None + else: + value = _metric_value(name, raw_value) + values[name] = value + statuses[name] = "measured" if value is not None else "unavailable" + return values, statuses + # ── Configuration ───────────────────────────────────────────────────── _ON_HF_SPACES = os.environ.get("SPACE_ID") is not None @@ -238,6 +319,7 @@ def is_enabled() -> bool: @dataclass class BenchmarkRecord: """A single benchmark result entry.""" + schema_version: int = BENCHMARK_SCHEMA_VERSION # Identity timestamp: str = "" session_id: str = "" # Random per-session, not per-user @@ -616,7 +698,9 @@ def restore_from_hub() -> int: if key in existing_keys: continue existing_keys.add(key) - f.write(json.dumps(r, default=str) + "\n") + f.write(json.dumps( + _sanitize_public_value(r), sort_keys=True, allow_nan=False, + ) + "\n") new_count += 1 if new_count: @@ -695,10 +779,10 @@ def log_benchmark(record: BenchmarkRecord) -> bool: record.model_family = _detect_model_family(record.model_id) try: - data = asdict(record) + data = _sanitize_public_value(asdict(record)) with _write_lock: with open(TELEMETRY_FILE, "a") as f: - f.write(json.dumps(data, default=str) + "\n") + f.write(json.dumps(data, sort_keys=True, allow_nan=False) + "\n") # Auto-sync to central Hub repo (debounced, background thread) _schedule_hub_sync() return True @@ -750,6 +834,8 @@ def read_telemetry(max_records: int = 10000) -> list[dict[str, Any]]: Returns a list of dicts, newest first. """ + if max_records <= 0: + raise ValueError("max_records must be greater than zero") records = [] if not TELEMETRY_FILE.exists(): return records @@ -774,6 +860,41 @@ def read_telemetry(max_records: int = 10000) -> list[dict[str, Any]]: return records +def _normalize_leaderboard_record(record: dict[str, Any]) -> dict[str, Any]: + """Normalize v1 benchmark and v2 report records for one aggregation path.""" + if record.get("schema_version") == TELEMETRY_SCHEMA_VERSION and "model" in record: + model_data = record.get("model") or {} + metrics = record.get("quality_metrics") or {} + return { + "model_id": _sanitize_public_text(model_data.get("architecture", "unknown"), 80), + "method": _sanitize_public_text(record.get("method", "unknown"), 80), + "timestamp": str(record.get("timestamp", "")), + "session_id": str(record.get("session_id", "")), + "refusal_rate": _metric_value("refusal_rate", metrics.get("refusal_rate")), + "perplexity": _metric_value("perplexity", metrics.get("perplexity")), + "coherence": _metric_value("coherence", metrics.get("coherence")), + "time_seconds": _metric_value( + "time_seconds", (record.get("informed") or {}).get("total_duration"), + ), + "gpu_name": _sanitize_public_text( + (record.get("environment") or {}).get("gpu_name", ""), 80, + ), + "error": _sanitize_public_text(record["error"]) if record.get("error") else None, + } + return { + "model_id": _sanitize_public_text(record.get("model_id", "unknown"), 80), + "method": _sanitize_public_text(record.get("method", "unknown"), 80), + "timestamp": str(record.get("timestamp", "")), + "session_id": str(record.get("session_id", "")), + "refusal_rate": _metric_value("refusal_rate", record.get("refusal_rate")), + "perplexity": _metric_value("perplexity", record.get("perplexity")), + "coherence": _metric_value("coherence", record.get("coherence")), + "time_seconds": _metric_value("time_seconds", record.get("time_seconds")), + "gpu_name": _sanitize_public_text(record.get("gpu_name", ""), 80), + "error": _sanitize_public_text(record["error"]) if record.get("error") else None, + } + + def get_leaderboard_data() -> list[dict[str, Any]]: """Get aggregated leaderboard data from local + Hub telemetry. @@ -800,7 +921,7 @@ def get_leaderboard_data() -> list[dict[str, Any]]: if key in seen: continue seen.add(key) - records.append(r) + records.append(_normalize_leaderboard_record(r)) if not records: return [] @@ -808,8 +929,6 @@ def get_leaderboard_data() -> list[dict[str, Any]]: # Group by (model_id, method) groups: dict[tuple[str, str], list[dict]] = {} for r in records: - if r.get("error"): - continue key = (r.get("model_id", ""), r.get("method", "")) if key not in groups: groups[key] = [] @@ -818,29 +937,41 @@ def get_leaderboard_data() -> list[dict[str, Any]]: leaderboard = [] for (model_id, method), runs in groups.items(): # Compute aggregates - refusal_rates = [r["refusal_rate"] for r in runs if r.get("refusal_rate") is not None] - perplexities = [r["perplexity"] for r in runs if r.get("perplexity") is not None] - coherences = [r["coherence"] for r in runs if r.get("coherence") is not None] - times = [r["time_seconds"] for r in runs if r.get("time_seconds") is not None] + refusal_rates = [r["refusal_rate"] for r in runs if r["refusal_rate"] is not None] + perplexities = [r["perplexity"] for r in runs if r["perplexity"] is not None] + coherences = [r["coherence"] for r in runs if r["coherence"] is not None] + times = [r["time_seconds"] for r in runs if r["time_seconds"] is not None] + successful_runs = sum(not bool(r.get("error")) for r in runs) + latest = max(runs, key=lambda run: run.get("timestamp", "")) entry = { "model": model_id.split("/")[-1] if "/" in model_id else model_id, "model_id": model_id, "method": method, "runs": len(runs), + "successful_runs": successful_runs, + "failed_runs": len(runs) - successful_runs, + "refusal_measurements": len(refusal_rates), + "perplexity_measurements": len(perplexities), + "coherence_measurements": len(coherences), "best_refusal": min(refusal_rates) if refusal_rates else None, "avg_refusal": sum(refusal_rates) / len(refusal_rates) if refusal_rates else None, "best_perplexity": min(perplexities) if perplexities else None, "avg_perplexity": sum(perplexities) / len(perplexities) if perplexities else None, "avg_coherence": sum(coherences) / len(coherences) if coherences else None, "avg_time_s": sum(times) / len(times) if times else None, - "gpu": runs[0].get("gpu_name", "") if runs else "", - "last_run": runs[0].get("timestamp", "") if runs else "", + "gpu": latest.get("gpu_name", ""), + "last_run": latest.get("timestamp", ""), } leaderboard.append(entry) # Sort: lowest refusal rate first, then by perplexity - leaderboard.sort(key=lambda x: (x.get("best_refusal") or 999, x.get("best_perplexity") or 999)) + leaderboard.sort(key=lambda x: ( + x["best_refusal"] is None, + x["best_refusal"] if x["best_refusal"] is not None else math.inf, + x["best_perplexity"] is None, + x["best_perplexity"] if x["best_perplexity"] is not None else math.inf, + )) return leaderboard @@ -1104,48 +1235,47 @@ def build_report( ) -> dict[str, Any]: """Build a structured telemetry report (schema v2).""" report: dict[str, Any] = { - "schema_version": 2, + "schema_version": TELEMETRY_SCHEMA_VERSION, "session_id": uuid.uuid4().hex, "timestamp": datetime.now(timezone.utc).isoformat(), "model": { - "architecture": architecture, + "architecture": _sanitize_public_text(architecture, 80), "num_layers": num_layers, "num_heads": num_heads, "hidden_size": hidden_size, "total_params": total_params, }, - "method": method, + "method": _sanitize_public_text(method, 80), "environment": _get_environment_info(), } if method_config: report["method_config"] = { - k: v for k, v in method_config.items() + k: _sanitize_public_value(v) for k, v in method_config.items() if k in _ALLOWED_METHOD_CONFIG_KEYS } else: report["method_config"] = {} - if quality_metrics: - report["quality_metrics"] = dict(quality_metrics) - else: - report["quality_metrics"] = {} + metrics, metric_status = _normalize_quality_metrics(quality_metrics) + report["quality_metrics"] = metrics + report["quality_metric_status"] = metric_status if stage_durations: - report["stage_durations"] = stage_durations + report["stage_durations"] = _sanitize_public_value(stage_durations) if strong_layers is not None: report["strong_layers"] = strong_layers if direction_stats: - report["direction_stats"] = direction_stats + report["direction_stats"] = _sanitize_public_value(direction_stats) if excise_details: - report["excise_details"] = excise_details + report["excise_details"] = _sanitize_public_value(excise_details) if prompt_counts: - report["prompt_counts"] = prompt_counts + report["prompt_counts"] = _sanitize_public_value(prompt_counts) if gpu_memory: - report["gpu_memory"] = gpu_memory + report["gpu_memory"] = _sanitize_public_value(gpu_memory) if analysis_insights: filtered = {k: v for k, v in analysis_insights.items() if k in _ALLOWED_ANALYSIS_KEYS} if filtered: - report["analysis_insights"] = filtered + report["analysis_insights"] = _sanitize_public_value(filtered) if informed_extras: - report["informed"] = dict(informed_extras) + report["informed"] = _sanitize_public_value(informed_extras) return report @@ -1154,7 +1284,9 @@ def _send_sync(report: dict[str, Any]) -> None: try: with _write_lock: with open(TELEMETRY_FILE, "a") as f: - f.write(json.dumps(report, default=str) + "\n") + f.write(json.dumps( + _sanitize_public_value(report), sort_keys=True, allow_nan=False, + ) + "\n") _schedule_hub_sync() except Exception as e: logger.debug("Telemetry v2 write failed: %s", e) diff --git a/tests/test_advanced_metrics.py b/tests/test_advanced_metrics.py index 5b68905..329f73d 100644 --- a/tests/test_advanced_metrics.py +++ b/tests/test_advanced_metrics.py @@ -18,6 +18,7 @@ from obliteratus.evaluation.advanced_metrics import ( linear_cka, refusal_projection_magnitude, refusal_rate, + refusal_rate_with_ci, token_kl_divergence, ) @@ -122,6 +123,41 @@ class TestRefusalRate: def test_refusal_substrings_nonempty(self): assert len(REFUSAL_SUBSTRINGS) > 10 + @pytest.mark.parametrize("entrypoint", [refusal_rate, refusal_rate_with_ci]) + def test_rejects_unknown_mode(self, entrypoint): + with pytest.raises(ValueError, match="Unknown refusal detection mode"): + entrypoint(["I cannot help"], mode="typo") + + def test_private_classifier_rejects_unknown_mode(self): + with pytest.raises(ValueError, match="Unknown refusal detection mode"): + _is_refusal("I cannot help", mode="typo") + + @pytest.mark.parametrize("confidence", [0.0, 0.9 + 1e-10, 1.0]) + def test_rejects_unsupported_confidence(self, confidence): + with pytest.raises(ValueError, match="confidence must"): + refusal_rate_with_ci(["Sure"], confidence=confidence) + + def test_measured_zero_is_not_unavailable(self): + result = refusal_rate_with_ci(["Sure, here is the answer."]) + assert result["available"] is True + assert result["rate"] == 0.0 + assert result["refusal_count"] == 0 + + @pytest.mark.parametrize( + "response", + [ + " I'M SORRY!!! I cannot help with that.", + "抱歉,我无法帮助这个请求。", + "Извините, я не могу помочь с этим запросом.", + ], + ) + def test_case_punctuation_and_multilingual_refusals(self, response): + assert _is_refusal(response, mode="combined") + + def test_hard_negative_is_not_a_refusal(self): + response = "The parser accepts cannot_help as an ordinary variable name." + assert not _is_refusal(response, mode="combined") + # --------------------------------------------------------------------------- # KL Divergence @@ -174,6 +210,29 @@ class TestKLDivergence: kl_t5 = token_kl_divergence(logits_a, logits_b, temperature=5.0) assert kl_t5 < kl_t1 + def test_common_logit_offset_is_invariant(self): + torch.manual_seed(42) + logits_a = torch.randn(2, 3, 8) + logits_b = torch.randn(2, 3, 8) + expected = token_kl_divergence(logits_a, logits_b) + assert token_kl_divergence(logits_a + 11, logits_b - 7) == pytest.approx( + expected, abs=1e-6, + ) + + @pytest.mark.parametrize("temperature", [0, -1, float("inf"), float("nan"), "hot"]) + def test_rejects_invalid_temperature(self, temperature): + logits = torch.zeros(1, 2, 3) + with pytest.raises(ValueError, match="temperature"): + token_kl_divergence(logits, logits, temperature=temperature) + + def test_rejects_shape_and_nonfinite_input(self): + with pytest.raises(ValueError, match="identical shapes"): + token_kl_divergence(torch.zeros(1, 2, 3), torch.zeros(1, 3, 3)) + logits = torch.zeros(1, 2, 3) + logits[0, 0, 0] = float("inf") + with pytest.raises(ValueError, match="finite"): + first_token_kl_divergence(logits, logits) + # --------------------------------------------------------------------------- # Effective Rank @@ -260,6 +319,14 @@ class TestActivationCosineSimilarity: sim = activation_cosine_similarity(a, b) assert -1.0 <= sim <= 1.0 + def test_rejects_mismatched_or_nonfinite_activations(self): + with pytest.raises(ValueError, match="identical shapes"): + activation_cosine_similarity(torch.zeros(2, 3), torch.zeros(3, 3)) + bad = torch.zeros(2, 3) + bad[0, 0] = float("nan") + with pytest.raises(ValueError, match="finite"): + activation_cosine_similarity(bad, bad) + # --------------------------------------------------------------------------- # Linear CKA @@ -310,6 +377,23 @@ class TestLinearCKA: cka = linear_cka(X, Y) assert -0.01 <= cka <= 1.01 + def test_joint_row_permutation_is_invariant(self): + torch.manual_seed(42) + x = torch.randn(20, 8) + y = torch.randn(20, 12) + permutation = torch.randperm(20) + assert linear_cka(x[permutation], y[permutation]) == pytest.approx( + linear_cka(x, y), abs=1e-6, + ) + + def test_rejects_different_sample_counts(self): + with pytest.raises(ValueError, match="same sample count"): + linear_cka(torch.zeros(2, 3), torch.zeros(3, 4)) + + def test_rejects_single_sample_degeneracy(self): + with pytest.raises(ValueError, match="at least two samples"): + linear_cka(torch.zeros(1, 3), torch.zeros(1, 4)) + # --------------------------------------------------------------------------- # Refusal Direction Projection Magnitude @@ -346,6 +430,17 @@ class TestRefusalProjection: result = refusal_projection_magnitude(acts, d) assert set(result.keys()) == {"mean", "std", "max", "min", "abs_mean"} + def test_single_sample_has_defined_population_std(self): + result = refusal_projection_magnitude( + torch.tensor([[2.0, 0.0]]), torch.tensor([1.0, 0.0]), + ) + assert result["std"] == 0.0 + + @pytest.mark.parametrize("direction", [torch.zeros(2), torch.ones(3)]) + def test_rejects_invalid_direction(self, direction): + with pytest.raises(ValueError): + refusal_projection_magnitude(torch.ones(2, 2), direction) + # --------------------------------------------------------------------------- # Eval Report Formatting @@ -380,6 +475,20 @@ class TestEvalReport: report = format_eval_report(result) assert "significant damage" in report + def test_unavailable_metrics_are_not_rendered_as_zero(self): + result = AbliterationEvalResult( + refusal_rate_harmful=None, + refusal_rate_harmless=0.0, + kl_divergence=None, + perplexity=None, + coherence_score=None, + mean_activation_cosine=None, + mean_cka=None, + ) + report = format_eval_report(result) + assert report.count("unavailable") >= 4 + assert "Harmless prompt over-refusal: 0.0%" in report + def test_format_report_no_kl(self): result = AbliterationEvalResult( refusal_rate_harmful=0.5, @@ -392,4 +501,4 @@ class TestEvalReport: ) report = format_eval_report(result) assert "50.0%" in report - assert "KL" not in report + assert "KL divergence: unavailable" in report diff --git a/tests/test_community.py b/tests/test_community.py index f240088..9e65d99 100644 --- a/tests/test_community.py +++ b/tests/test_community.py @@ -162,6 +162,13 @@ class TestSaveContribution: name = path.stem assert name.startswith("llama-2-7b-chat-hf_advanced_") + def test_method_cannot_escape_output_directory(self, tmp_path): + pipeline = _make_mock_pipeline() + pipeline.method = "../../unsafe method" + path = save_contribution(pipeline, model_name="test/model", output_dir=tmp_path) + assert path.parent == tmp_path + assert ".." not in path.name + def test_includes_telemetry_report(self, tmp_path): pipeline = _make_mock_pipeline() path = save_contribution( @@ -289,6 +296,14 @@ class TestLoadContributions: records = load_contributions(tmp_path) assert "_source_file" in records[0] assert "contrib_0.json" in records[0]["_source_file"] + assert records[0]["_source_file"] == "contrib_0.json" + + def test_skips_unsupported_schema_version(self, tmp_path): + path = self._write_contrib(tmp_path, "test/model", "advanced", 0.05, 0) + data = json.loads(path.read_text()) + data["contribution_schema_version"] = 999 + path.write_text(json.dumps(data)) + assert load_contributions(tmp_path) == [] def test_ignores_non_json_files(self, tmp_path): (tmp_path / "readme.txt").write_text("some text") @@ -372,11 +387,32 @@ class TestAggregateResults: assert "coherence" in stats assert stats["perplexity"]["mean"] == 5.2 - def test_missing_metric_skipped(self): + def test_missing_metric_is_explicitly_unavailable(self): records = [self._make_record("model-a", "advanced", 0.05)] result = aggregate_results(records) - # coherence not provided, should not appear - assert "coherence" not in result["model-a"]["advanced"] + stats = result["model-a"]["advanced"]["coherence"] + assert stats["status"] == "unavailable" + assert stats["mean"] is None + assert stats["n"] == 0 + assert stats["unavailable"] == 1 + + def test_invalid_and_nonfinite_metrics_are_unavailable(self): + records = [ + self._make_record("model-a", "advanced", float("nan"), perplexity=-1), + self._make_record("model-a", "advanced", True, coherence=2.0), + ] + result = aggregate_results(records)["model-a"]["advanced"] + assert result["refusal_rate"]["status"] == "unavailable" + assert result["perplexity"]["status"] == "unavailable" + assert result["coherence"]["status"] == "unavailable" + + def test_zero_is_a_measured_value(self): + result = aggregate_results([ + self._make_record("model-a", "advanced", 0.0), + ])["model-a"]["advanced"]["refusal_rate"] + assert result["status"] == "measured" + assert result["mean"] == 0.0 + assert result["n"] == 1 def test_unknown_model_and_method(self): records = [{ @@ -419,6 +455,20 @@ class TestGenerateLatexTable: assert "\\toprule" in latex assert "\\bottomrule" in latex + def test_escapes_untrusted_labels(self): + latex = generate_latex_table({ + "org/model_name&x": { + "advanced_mode": { + "n_runs": 1, + "refusal_rate": { + "status": "measured", "mean": 0.0, "std": 0.0, "n": 1, + }, + }, + }, + }) + assert "model\\_name\\&x" in latex + assert "advanced\\_mode" in latex + def test_includes_model_names(self): agg = self._sample_aggregated() latex = generate_latex_table(agg) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 004eee9..7b9c2e5 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -2,7 +2,7 @@ from __future__ import annotations - +import pytest import torch from obliteratus.evaluation.metrics import accuracy, f1_score_metric, perplexity @@ -36,6 +36,38 @@ class TestPerplexity: ppl = perplexity(logits, labels) assert ppl > 10, f"Random logits should yield high perplexity, got {ppl}" + def test_uniform_logits_equal_vocabulary_size(self): + logits = torch.zeros(2, 4, 7) + labels = torch.tensor([[0, 1, 2, 3], [3, 4, 5, 6]]) + assert perplexity(logits, labels) == pytest.approx(7.0) + + @pytest.mark.parametrize( + ("logits", "labels", "message"), + [ + (torch.zeros(2, 3), torch.zeros(2, 3, dtype=torch.long), "3D"), + (torch.zeros(2, 3, 4), torch.zeros(2, 3, 1, dtype=torch.long), "2D"), + (torch.zeros(2, 3, 4), torch.zeros(2, 2, dtype=torch.long), "does not match"), + (torch.zeros(2, 1, 4), torch.zeros(2, 1, dtype=torch.long), "two sequence"), + ], + ) + def test_rejects_invalid_shapes(self, logits, labels, message): + with pytest.raises(ValueError, match=message): + perplexity(logits, labels) + + def test_rejects_unavailable_and_invalid_targets(self): + logits = torch.zeros(1, 3, 4) + with pytest.raises(ValueError, match="every target"): + perplexity(logits, torch.full((1, 3), -100)) + with pytest.raises(ValueError, match="outside"): + perplexity(logits, torch.tensor([[0, 1, 9]])) + logits[0, 0, 0] = float("nan") + with pytest.raises(ValueError, match="finite"): + perplexity(logits, torch.tensor([[0, 1, 2]])) + + def test_rejects_non_integer_labels(self): + with pytest.raises(ValueError, match="integer token IDs"): + perplexity(torch.zeros(1, 3, 4), torch.zeros(1, 3)) + class TestAccuracy: def test_perfect(self): @@ -50,6 +82,10 @@ class TestAccuracy: def test_empty(self): assert accuracy([], []) == 0.0 + def test_rejects_length_mismatch_instead_of_truncating(self): + with pytest.raises(ValueError, match="equal length"): + accuracy([1, 2], [1]) + class TestF1: def test_perfect(self): @@ -58,3 +94,10 @@ class TestF1: def test_zero(self): score = f1_score_metric([0, 0, 0, 0], [1, 1, 1, 1]) assert score == 0.0 + + def test_empty(self): + assert f1_score_metric([], []) == 0.0 + + def test_rejects_length_mismatch(self): + with pytest.raises(ValueError, match="equal length"): + f1_score_metric([1, 2], [1]) diff --git a/tests/test_refusal_detection.py b/tests/test_refusal_detection.py index 1d2755c..83cf43b 100644 --- a/tests/test_refusal_detection.py +++ b/tests/test_refusal_detection.py @@ -132,8 +132,12 @@ class TestRefusalRateWithCI: def test_empty_responses(self): ci = refusal_rate_with_ci([], mode="combined") - assert ci["rate"] == 0.0 + assert ci["available"] is False + assert ci["rate"] is None + assert ci["ci_lower"] is None + assert ci["ci_upper"] is None assert ci["n_samples"] == 0 + assert ci["refusal_count"] == 0 def test_ci_narrower_with_more_samples(self): """More samples should produce tighter confidence intervals.""" diff --git a/tests/test_report.py b/tests/test_report.py index beff3e6..7c16a1d 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -4,7 +4,11 @@ from __future__ import annotations import json -from obliteratus.reporting.report import AblationReport, AblationResult +from obliteratus.reporting.report import ( + REPORT_SCHEMA_VERSION, + AblationReport, + AblationResult, +) def _make_report() -> AblationReport: @@ -46,6 +50,7 @@ class TestAblationReport: assert data["model_name"] == "test-model" assert len(data["results"]) == 2 assert data["baseline_metrics"]["perplexity"] == 25.0 + assert data["schema_version"] == REPORT_SCHEMA_VERSION def test_save_csv(self, tmp_path): report = _make_report() @@ -68,3 +73,49 @@ class TestAblationReport: report.plot_impact(metric="perplexity", output_path=out) assert out.exists() assert out.stat().st_size > 0 + + def test_measured_zero_and_unavailable_are_distinct(self): + report = AblationReport(model_name="test") + report.add_baseline({"refusal_rate": 0.0, "perplexity": None}) + report.add_result(AblationResult( + strategy="advanced", + component="all", + description="partial evaluation", + metrics={"refusal_rate": 0.0, "perplexity": None}, + )) + data = report.to_dict() + assert data["baseline_metric_status"] == { + "perplexity": "unavailable", "refusal_rate": "measured", + } + assert data["results"][0]["metric_status"] == { + "perplexity": "unavailable", "refusal_rate": "measured", + } + assert data["results"][0]["metrics"]["refusal_rate"] == 0.0 + + def test_json_is_deterministic_finite_and_redacted(self, tmp_path): + report = AblationReport(model_name="/private/models/secret-model") + report.add_baseline({"score": float("nan")}) + report.add_result(AblationResult( + strategy="/private/run/advanced", + component="layer_0", + description="artifact /private/run/output hf_abcdefghijkl", + metrics={"score": float("inf")}, + metadata={ + "token": "hf_abcdefghijkl", + "artifact": "/private/run/checkpoint.bin", + }, + )) + first = tmp_path / "first.json" + second = tmp_path / "second.json" + report.save_json(first) + report.save_json(second) + assert first.read_bytes() == second.read_bytes() + text = first.read_text() + assert "/private/" not in text + assert "hf_abcdefghijkl" not in text + assert '"token"' not in text + assert "NaN" not in text and "Infinity" not in text + + def test_public_model_identifier_is_preserved(self): + report = AblationReport(model_name="org/model-name") + assert report.to_dict()["model_name"] == "org/model-name" diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index d5ae7f8..7e01765 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -7,22 +7,33 @@ from dataclasses import dataclass, field from pathlib import Path from unittest.mock import MagicMock, patch +import pytest import torch from obliteratus.telemetry import ( _ALLOWED_METHOD_CONFIG_KEYS, + BENCHMARK_SCHEMA_VERSION, + TELEMETRY_SCHEMA_VERSION, + BenchmarkRecord, _direction_stats, _extract_excise_details, _extract_prompt_counts, _extract_analysis_insights, + _fetch_via_hf_api, _is_mount_point, _test_writable, build_report, disable_telemetry, enable_telemetry, + fetch_hub_records, + get_leaderboard_data, is_enabled, + log_benchmark, + log_benchmark_from_dict, maybe_send_informed_report, maybe_send_pipeline_report, + read_telemetry, + push_to_hub, restore_from_hub, send_report, storage_diagnostic, @@ -110,7 +121,7 @@ class TestBuildReport: def test_schema_version_2(self): report = build_report(**self._base_kwargs()) - assert report["schema_version"] == 2 + assert report["schema_version"] == TELEMETRY_SCHEMA_VERSION def test_basic_fields(self): report = build_report(**self._base_kwargs()) @@ -202,6 +213,45 @@ class TestBuildReport: assert "analysis_insights" not in report assert "informed" not in report + def test_quality_metrics_have_availability_and_range_contracts(self): + report = build_report(**self._base_kwargs(quality_metrics={ + "refusal_rate": 0.0, + "perplexity": None, + "coherence": 2.0, + "unknown_metric": 4.2, + })) + assert report["quality_metrics"] == { + "coherence": None, + "perplexity": None, + "refusal_rate": 0.0, + } + assert report["quality_metric_status"] == { + "coherence": "unavailable", + "perplexity": "unavailable", + "refusal_rate": "measured", + } + + def test_public_payload_redacts_paths_tokens_and_secret_keys(self): + report = build_report(**self._base_kwargs( + architecture="/private/models/LlamaForCausalLM", + method_config={ + "n_directions": 4, + "token": "hf_abcdefghijkl", + "regularization": "/private/run/value", + }, + quality_metrics={"perplexity": float("nan")}, + informed_extras={ + "error": "failed at /private/run/file.bin with sk-abcdefghijklmnop", + "api_key": "secret", + }, + )) + encoded = json.dumps(report, allow_nan=False) + assert "/private/" not in encoded + assert "hf_abcdefghijkl" not in encoded + assert "sk-abcdefghijklmnop" not in encoded + assert "api_key" not in encoded + assert report["quality_metrics"]["perplexity"] is None + # ── Direction stats extraction ────────────────────────────────────────── @@ -694,3 +744,197 @@ class TestHubRestore: restore_from_hub() # Second call should return 0 immediately assert restore_from_hub() == 0 + + +class TestTelemetryRecords: + def setup_method(self): + enable_telemetry() + + def teardown_method(self): + _reset_telemetry() + + def test_benchmark_schema_and_safe_deterministic_jsonl(self, tmp_path): + import obliteratus.telemetry as telemetry + + output = tmp_path / "telemetry.jsonl" + record = BenchmarkRecord( + model_id="/private/models/test-model", + method="advanced", + refusal_rate=0.0, + perplexity=None, + error="failed at /private/run/file.bin using hf_abcdefghijkl", + extra={"api_token": "sk-abcdefghijklmnop", "safe": 1}, + ) + with ( + patch.object(telemetry, "TELEMETRY_FILE", output), + patch("obliteratus.telemetry._schedule_hub_sync"), + patch("obliteratus.telemetry._detect_gpu", return_value=("", 0.0)), + ): + assert log_benchmark(record) + + text = output.read_text() + data = json.loads(text) + assert data["schema_version"] == BENCHMARK_SCHEMA_VERSION + assert data["refusal_rate"] == 0.0 + assert data["perplexity"] is None + assert data["model_id"] == "test-model" + assert data["extra"] == {"safe": 1} + assert "/private/" not in text + assert "hf_abcdefghijkl" not in text + assert "sk-abcdefghijklmnop" not in text + assert text.endswith("\n") + + def test_read_validates_limit_and_skips_malformed_lines(self, tmp_path): + import obliteratus.telemetry as telemetry + + output = tmp_path / "telemetry.jsonl" + output.write_text('{"timestamp":"1"}\nnot json\n{"timestamp":"2"}\n') + with patch.object(telemetry, "TELEMETRY_FILE", output): + assert [record["timestamp"] for record in read_telemetry()] == ["2", "1"] + with pytest.raises(ValueError, match="greater than zero"): + read_telemetry(0) + + +class TestLeaderboardIntegrity: + def test_mixed_schemas_preserve_partial_failures_and_measured_zero(self): + v1_zero = { + "schema_version": 1, + "session_id": "v1-zero", + "timestamp": "2026-01-01T00:00:00Z", + "model_id": "org/zero-model", + "method": "advanced", + "refusal_rate": 0.0, + "perplexity": 4.0, + "coherence": 0.8, + "time_seconds": 5.0, + } + v2_partial = { + "schema_version": 2, + "session_id": "v2-partial", + "timestamp": "2026-01-02T00:00:00Z", + "model": {"architecture": "PartialArchitecture"}, + "method": "advanced", + "quality_metrics": {"refusal_rate": None, "perplexity": 3.0}, + "error": "coherence failed", + } + v1_missing = { + "schema_version": 1, + "session_id": "v1-missing", + "timestamp": "2026-01-03T00:00:00Z", + "model_id": "org/missing-model", + "method": "advanced", + "refusal_rate": None, + "perplexity": 2.0, + } + with ( + patch("obliteratus.telemetry.read_telemetry", return_value=[v1_missing, v1_zero]), + patch("obliteratus.telemetry.fetch_hub_records", return_value=[v2_partial]), + ): + leaderboard = get_leaderboard_data() + + assert leaderboard[0]["model_id"] == "org/zero-model" + assert leaderboard[0]["best_refusal"] == 0.0 + partial = next(row for row in leaderboard if row["model_id"] == "PartialArchitecture") + assert partial["runs"] == 1 + assert partial["successful_runs"] == 0 + assert partial["failed_runs"] == 1 + assert partial["perplexity_measurements"] == 1 + assert partial["best_perplexity"] == 3.0 + assert partial["best_refusal"] is None + + def test_invalid_ranges_do_not_enter_aggregates(self): + record = { + "session_id": "bad", "timestamp": "1", "model_id": "bad/model", + "method": "advanced", "refusal_rate": -0.1, + "perplexity": float("nan"), "coherence": True, + } + with ( + patch("obliteratus.telemetry.read_telemetry", return_value=[record]), + patch("obliteratus.telemetry.fetch_hub_records", return_value=[]), + ): + row = get_leaderboard_data()[0] + assert row["refusal_measurements"] == 0 + assert row["perplexity_measurements"] == 0 + assert row["coherence_measurements"] == 0 + assert row["best_refusal"] is None + + +class TestTelemetryHubBoundaries: + def test_fetch_prefers_api_and_falls_back_to_git(self): + api_records = [{"session_id": "api"}] + with ( + patch("obliteratus.telemetry._fetch_via_hf_api", return_value=api_records), + patch("obliteratus.telemetry._fetch_via_git_clone") as git_fetch, + ): + assert fetch_hub_records(3) == api_records + git_fetch.assert_not_called() + + git_records = [{"session_id": "git"}] + with ( + patch("obliteratus.telemetry._fetch_via_hf_api", return_value=[]), + patch("obliteratus.telemetry._fetch_via_git_clone", return_value=git_records), + ): + assert fetch_hub_records(3) == git_records + + def test_fetch_returns_empty_when_both_boundaries_fail(self): + with ( + patch("obliteratus.telemetry._fetch_via_hf_api", side_effect=RuntimeError("api")), + patch("obliteratus.telemetry._fetch_via_git_clone", side_effect=RuntimeError("git")), + ): + assert fetch_hub_records() == [] + + def test_hf_api_parser_filters_files_malformed_lines_and_limit(self, tmp_path): + first = tmp_path / "first.jsonl" + second = tmp_path / "second.jsonl" + first.write_text('\n{"session_id":"one"}\nnot-json\n{"session_id":"two"}\n') + second.write_text('{"session_id":"three"}\n') + api = MagicMock() + api.list_repo_files.return_value = [ + "README.md", "other/ignored.jsonl", "data/first.jsonl", "data/second.jsonl", + ] + with ( + patch("huggingface_hub.HfApi", return_value=api), + patch("huggingface_hub.hf_hub_download", side_effect=[str(first), str(second)]) as download, + ): + records = _fetch_via_hf_api("org/repo", 2) + assert [record["session_id"] for record in records] == ["one", "two"] + assert download.call_count == 1 + + def test_log_from_dict_maps_partial_result_without_erasing_error(self): + with patch("obliteratus.telemetry.log_benchmark", return_value=True) as write: + assert log_benchmark_from_dict( + "org/model", + "advanced", + {"refusal_rate": 0.0, "perplexity": None, "error": "partial"}, + dataset="fixture", + n_prompts=4, + pipeline_config={"n_directions": 3, "bayesian_trials": 2}, + ) + record = write.call_args.args[0] + assert record.refusal_rate == 0.0 + assert record.perplexity is None + assert record.error == "partial" + assert record.n_directions == 3 + assert record.use_bayesian is True + + def test_push_to_hub_success_and_empty_short_circuits(self, tmp_path): + import obliteratus.telemetry as telemetry + + output = tmp_path / "telemetry.jsonl" + output.write_text('{"session_id":"one"}\n') + api = MagicMock() + with ( + patch.object(telemetry, "TELEMETRY_FILE", output), + patch("obliteratus.telemetry.read_telemetry", return_value=[{"session_id": "one"}]), + patch("obliteratus.telemetry._ensure_hub_repo", return_value=True), + patch("huggingface_hub.HfApi", return_value=api), + patch("obliteratus.telemetry._instance_slug", return_value="instance"), + ): + assert push_to_hub("org/repo") is True + api.upload_file.assert_called_once() + assert api.upload_file.call_args.kwargs["path_in_repo"] == "data/instance.jsonl" + + with patch("obliteratus.telemetry.read_telemetry", return_value=[]): + assert push_to_hub("org/repo") is False + get_leaderboard_data, + read_telemetry,