mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-08-17 16:37:30 +02:00
test: add quality-depth gates and mature CPU coverage
This commit is contained in:
Joseph Magly
+6
-1
@@ -6,7 +6,6 @@ import socket
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
|
||||
_EXTERNAL_MARKERS = ("network", "download", "remote")
|
||||
@@ -46,6 +45,8 @@ def mock_model():
|
||||
- model.config with config.num_hidden_layers = 4
|
||||
- model.named_parameters() returning fake weight tensors
|
||||
"""
|
||||
import torch
|
||||
|
||||
model = MagicMock()
|
||||
|
||||
# Config with num_hidden_layers
|
||||
@@ -83,6 +84,8 @@ def mock_tokenizer():
|
||||
@pytest.fixture
|
||||
def refusal_direction():
|
||||
"""A normalized random torch tensor of shape (768,)."""
|
||||
import torch
|
||||
|
||||
t = torch.randn(768)
|
||||
return t / t.norm()
|
||||
|
||||
@@ -90,6 +93,8 @@ def refusal_direction():
|
||||
@pytest.fixture
|
||||
def activation_pair():
|
||||
"""A tuple of (harmful_activations, harmless_activations) as random tensors of shape (10, 768)."""
|
||||
import torch
|
||||
|
||||
harmful_activations = torch.randn(10, 768)
|
||||
harmless_activations = torch.randn(10, 768)
|
||||
return (harmful_activations, harmless_activations)
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
"""CPU-only behavioral tests for telemetry-driven adaptive defaults."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from obliteratus import adaptive_defaults as adaptive
|
||||
|
||||
|
||||
def _record(
|
||||
method="advanced", *, architecture="LlamaForCausalLM", params_b=7,
|
||||
refusal=0.1, coherence=0.9, config=None, session="session",
|
||||
):
|
||||
return {
|
||||
"session_id": session,
|
||||
"timestamp": session,
|
||||
"model": {
|
||||
"architecture": architecture,
|
||||
"num_layers": 32,
|
||||
"hidden_size": 4096,
|
||||
"total_params": int(params_b * 1e9),
|
||||
},
|
||||
"method": method,
|
||||
"method_config": (
|
||||
config if config is not None else {"strength": 1.0, "cot_aware": False}
|
||||
),
|
||||
"quality_metrics": {
|
||||
"refusal_rate": refusal,
|
||||
"coherence": coherence,
|
||||
"kl_divergence": 0.1,
|
||||
"perplexity": 10.0,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("params", "expected"),
|
||||
[(-1, "tiny"), (0.5, "tiny"), (4, "small"), (16, "medium"), (80, "large"), (81, "frontier")],
|
||||
)
|
||||
def test_parameter_buckets_have_closed_boundaries(params, expected):
|
||||
assert adaptive._param_bucket(params) == expected
|
||||
|
||||
|
||||
def test_architecture_key_handles_schema_estimates_moe_and_reasoning():
|
||||
assert adaptive._extract_arch_key({"model": "legacy"}) is None
|
||||
assert adaptive._extract_arch_key({"model": {"architecture": "unknown"}}) is None
|
||||
assert adaptive._extract_arch_key(_record()) == (
|
||||
"dense", "standard", "medium",
|
||||
)
|
||||
|
||||
record = _record(architecture="Qwen3_MoE_Reasoning", params_b=120)
|
||||
assert adaptive._extract_arch_key(record) == ("large_moe", "reasoning", "frontier")
|
||||
record = _record(params_b=8)
|
||||
record["model"]["total_params"] = 0
|
||||
record["method_config"] = {"per_expert_directions": True, "cot_aware": True}
|
||||
assert adaptive._extract_arch_key(record) == ("small_moe", "reasoning", "medium")
|
||||
|
||||
|
||||
def test_composite_score_defaults_and_complete_metrics():
|
||||
assert adaptive._composite_score({}) == pytest.approx(0.15)
|
||||
assert adaptive._composite_score({
|
||||
"refusal_rate": 0.0,
|
||||
"coherence": 1.0,
|
||||
"kl_divergence": 0.0,
|
||||
"perplexity": 0.0,
|
||||
}) == pytest.approx(1.0)
|
||||
|
||||
|
||||
def test_method_statistics_ranges_and_bucket_ranking():
|
||||
empty = adaptive.MethodStats("empty")
|
||||
assert (empty.mean_score, empty.best_score, empty.median_score) == (0, 0, 0)
|
||||
assert empty.best_config_ranges() == {}
|
||||
|
||||
stats = adaptive.MethodStats(
|
||||
"strong",
|
||||
n_runs=4,
|
||||
scores=[0.1, 0.9, 0.8, 0.7],
|
||||
configs=[
|
||||
{"strength": 1, "enabled": False, "label": "skip"},
|
||||
{"strength": 4, "enabled": True},
|
||||
{"strength": 3, "enabled": True},
|
||||
{"strength": 2, "enabled": False},
|
||||
],
|
||||
)
|
||||
assert stats.best_config_ranges() == {"strength": 4, "enabled": True}
|
||||
bucket = adaptive.BucketKnowledge(
|
||||
("dense", "standard", "small"),
|
||||
methods={"empty": empty, "strong": stats},
|
||||
total_runs=4,
|
||||
)
|
||||
assert bucket.best_method == "strong"
|
||||
assert bucket.ranked_methods[0][0] == "strong"
|
||||
assert adaptive.BucketKnowledge(("dense", "standard", "tiny")).best_method is None
|
||||
|
||||
|
||||
def test_build_knowledge_base_filters_invalid_runs_and_aggregates_metrics():
|
||||
records = [
|
||||
_record(session="one"),
|
||||
_record(method="basic", refusal=0.4, coherence=0.5, config={}, session="two"),
|
||||
{**_record(session="error"), "error": "failed"},
|
||||
{**_record(session="no-method"), "method": ""},
|
||||
{**_record(session="no-metrics"), "quality_metrics": {}},
|
||||
{**_record(session="legacy"), "model": "legacy"},
|
||||
]
|
||||
knowledge = adaptive.build_knowledge_base(records)
|
||||
bucket = knowledge[("dense", "standard", "medium")]
|
||||
assert bucket.total_runs == 2
|
||||
assert bucket.methods["advanced"].n_runs == 1
|
||||
assert bucket.methods["advanced"].refusal_rates == [0.1]
|
||||
assert bucket.methods["basic"].configs == []
|
||||
|
||||
|
||||
def test_fetch_records_caches_deduplicates_and_tolerates_sources(monkeypatch):
|
||||
adaptive._cache.clear()
|
||||
monkeypatch.setattr(adaptive, "_cache_ts", 0.0)
|
||||
import obliteratus.telemetry as telemetry
|
||||
|
||||
monkeypatch.setattr(telemetry, "read_telemetry", lambda: [_record(session="same")])
|
||||
monkeypatch.setattr(
|
||||
telemetry,
|
||||
"fetch_hub_records",
|
||||
lambda: [_record(session="same"), _record(session="other")],
|
||||
)
|
||||
first = adaptive._fetch_all_records()
|
||||
assert len(first) == 2
|
||||
monkeypatch.setattr(telemetry, "read_telemetry", lambda: (_ for _ in ()).throw(RuntimeError()))
|
||||
assert adaptive._fetch_all_records() is first
|
||||
|
||||
adaptive._cache.clear()
|
||||
monkeypatch.setattr(adaptive, "_cache_ts", 0.0)
|
||||
monkeypatch.setattr(telemetry, "fetch_hub_records", lambda: (_ for _ in ()).throw(RuntimeError()))
|
||||
assert adaptive._fetch_all_records() == []
|
||||
|
||||
|
||||
def test_recommendation_exact_fallback_confidence_and_formatting():
|
||||
records = [
|
||||
_record("advanced", refusal=0.05, coherence=0.95, session=f"a-{index}")
|
||||
for index in range(5)
|
||||
] + [
|
||||
_record("basic", refusal=0.5, coherence=0.4, session=f"b-{index}")
|
||||
for index in range(5)
|
||||
]
|
||||
knowledge = adaptive.build_knowledge_base(records)
|
||||
rec = adaptive.get_adaptive_recommendation("dense", "standard", 7, knowledge=knowledge)
|
||||
assert rec.recommended_method == "advanced"
|
||||
assert rec.confidence == "medium"
|
||||
assert rec.best_refusal_rate == 0.05
|
||||
assert "Runner-up" in rec.reason
|
||||
assert rec.to_dict()["arch_key"] == ["dense", "standard", "medium"]
|
||||
formatted = adaptive.format_recommendation(rec)
|
||||
assert "Adaptive Recommendation" in formatted
|
||||
assert "strength" in formatted
|
||||
|
||||
sparse_knowledge = adaptive.build_knowledge_base([
|
||||
_record("advanced", params_b=1, session="small"),
|
||||
_record("advanced", params_b=30, session="large"),
|
||||
])
|
||||
fallback = adaptive.get_adaptive_recommendation(
|
||||
"dense", "reasoning", 7, knowledge=sparse_knowledge,
|
||||
)
|
||||
assert fallback.confidence == "low"
|
||||
assert fallback.arch_key == ("dense", "*", "*")
|
||||
|
||||
none = adaptive.get_adaptive_recommendation("small_moe", "standard", 7, knowledge={})
|
||||
assert none.confidence == "none"
|
||||
assert "No telemetry data" in adaptive.format_recommendation(none)
|
||||
|
||||
|
||||
def test_global_insights_reports_rankings_buckets_and_hyperparameters():
|
||||
knowledge = adaptive.build_knowledge_base([
|
||||
_record("advanced", config={"strength": 1.5, "enabled": True}, session="one"),
|
||||
_record("basic", config={"strength": 0.5, "enabled": False}, session="two"),
|
||||
])
|
||||
insights = adaptive.get_global_insights(knowledge)
|
||||
assert insights["total_records"] == 2
|
||||
assert insights["overall_best_methods"][0]["method"] == "advanced"
|
||||
assert insights["hyperparameter_trends"]["strength"]["type"] == "numeric"
|
||||
assert insights["hyperparameter_trends"]["enabled"]["type"] == "bool"
|
||||
@@ -0,0 +1,108 @@
|
||||
"""CPU-only contracts for benchmark visualization generation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import pytest
|
||||
|
||||
from obliteratus.evaluation import benchmark_plots as plots
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def results():
|
||||
return [
|
||||
{
|
||||
"method": "advanced",
|
||||
"model": "org/model-a",
|
||||
"model_short": "model-a",
|
||||
"perplexity": 10.0,
|
||||
"refusal_rate": 0.1,
|
||||
"coherence": 0.9,
|
||||
"time_s": 2.0,
|
||||
"strong_layers": 3,
|
||||
"ega_expert_dirs": 2,
|
||||
"cot_preserved": 1,
|
||||
"expert_classified_layers": 4,
|
||||
},
|
||||
{
|
||||
"method": "unknown/hf_secret123456",
|
||||
"model": "org/model-b",
|
||||
"perplexity": 12.0,
|
||||
"refusal_rate": 0.2,
|
||||
"coherence": 0.8,
|
||||
"time_s": 3.0,
|
||||
"strong_layers": 0,
|
||||
"ega_expert_dirs": 0,
|
||||
"cot_preserved": 0,
|
||||
"ega_safety_layers": 0,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_label_sanitization_and_palette_fallback():
|
||||
label = plots._sanitize_label(
|
||||
"/private/path/model hf_secret123456 0123456789abcdef0123456789abcdef", max_len=40,
|
||||
)
|
||||
assert "/private/path" not in label
|
||||
assert "hf_secret" not in label
|
||||
assert label == "model <TOKEN> <REDACTED>"
|
||||
assert plots._get_color("basic") == plots.PALETTE["basic"]
|
||||
assert plots._get_color("custom", 9) == plots.MODEL_PALETTE[1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"factory",
|
||||
[
|
||||
plots.plot_pareto_frontier,
|
||||
plots.plot_method_radar,
|
||||
plots.plot_metric_bars,
|
||||
plots.plot_timing_efficiency,
|
||||
plots.plot_moe_metrics,
|
||||
plots.plot_model_scaling,
|
||||
],
|
||||
)
|
||||
def test_each_plot_returns_a_figure_for_empty_input(factory):
|
||||
figure = factory([])
|
||||
assert isinstance(figure, plt.Figure)
|
||||
plt.close(figure)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"factory",
|
||||
[
|
||||
plots.plot_pareto_frontier,
|
||||
plots.plot_method_radar,
|
||||
plots.plot_metric_bars,
|
||||
plots.plot_timing_efficiency,
|
||||
plots.plot_moe_metrics,
|
||||
plots.plot_model_scaling,
|
||||
],
|
||||
)
|
||||
def test_each_plot_renders_synthetic_results(factory, results):
|
||||
figure = factory(results, " — fixture")
|
||||
assert isinstance(figure, plt.Figure)
|
||||
assert figure.axes
|
||||
plt.close(figure)
|
||||
|
||||
|
||||
def test_pareto_handles_single_point_without_frontier_line(results):
|
||||
figure = plots.plot_pareto_frontier(results[:1])
|
||||
assert isinstance(figure, plt.Figure)
|
||||
plt.close(figure)
|
||||
|
||||
|
||||
def test_moe_plot_handles_non_moe_results(results):
|
||||
figure = plots.plot_moe_metrics(results[1:])
|
||||
assert "No MoE-specific features" in figure.axes[0].texts[0].get_text()
|
||||
plt.close(figure)
|
||||
|
||||
|
||||
def test_dashboard_modes_and_optional_moe_panel(results):
|
||||
assert plots.generate_benchmark_dashboard([]) == []
|
||||
method_figures = plots.generate_benchmark_dashboard(results, mode="multi_method")
|
||||
model_figures = plots.generate_benchmark_dashboard(results, mode="multi_model")
|
||||
assert len(method_figures) == 5
|
||||
assert len(model_figures) == 5
|
||||
assert plots.generate_benchmark_dashboard(results, mode="unsupported") == []
|
||||
for figure in method_figures + model_figures:
|
||||
plt.close(figure)
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Deterministic property contracts for configuration serialization."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from hypothesis import given, seed, settings, strategies as st
|
||||
|
||||
from obliteratus.config import StudyConfig
|
||||
|
||||
|
||||
PROPERTY_SETTINGS = settings(max_examples=60, deadline=None, database=None)
|
||||
|
||||
|
||||
@seed(7007)
|
||||
@PROPERTY_SETTINGS
|
||||
@given(
|
||||
batch_size=st.integers(1, 32),
|
||||
max_length=st.integers(2, 4096),
|
||||
strategy_names=st.lists(
|
||||
st.text(alphabet="abcdefghijklmnopqrstuvwxyz_", min_size=1, max_size=16),
|
||||
min_size=1,
|
||||
max_size=8,
|
||||
),
|
||||
)
|
||||
def test_study_config_public_roundtrip_preserves_explicit_values(
|
||||
batch_size, max_length, strategy_names,
|
||||
):
|
||||
raw = {
|
||||
"model": {"name": "fixture", "device": "cpu"},
|
||||
"dataset": {"name": "fixture", "max_samples": 12},
|
||||
"strategies": [{"name": name, "params": {"strength": 0.5}} for name in strategy_names],
|
||||
"metrics": ["perplexity", "accuracy"],
|
||||
"batch_size": batch_size,
|
||||
"max_length": max_length,
|
||||
"output_dir": "results/property",
|
||||
}
|
||||
config = StudyConfig.from_dict(raw)
|
||||
restored = StudyConfig.from_dict(config.to_dict())
|
||||
assert restored == config
|
||||
@@ -2,15 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).parents[1] / "scripts" / "check_coverage_thresholds.py"
|
||||
SPEC = importlib.util.spec_from_file_location("check_coverage_thresholds", SCRIPT)
|
||||
assert SPEC is not None and SPEC.loader is not None
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
from scripts import check_coverage_thresholds as MODULE
|
||||
|
||||
|
||||
def _report(line: float = 55.0, branch: float = 42.0) -> dict[str, object]:
|
||||
@@ -86,6 +78,30 @@ def test_validate_coverage_enforces_critical_file_floors():
|
||||
]
|
||||
|
||||
|
||||
def test_critical_file_validation_continues_and_rejects_malformed_values():
|
||||
report = _report()
|
||||
report["files"]["bool.py"] = {
|
||||
"summary": {"percent_statements_covered": True},
|
||||
}
|
||||
report["files"]["text.py"] = {
|
||||
"summary": {"percent_statements_covered": "70"},
|
||||
}
|
||||
assert MODULE.validate_coverage(
|
||||
report,
|
||||
min_line=55.0,
|
||||
min_branch=42.0,
|
||||
file_floors={
|
||||
"missing.py": 70.0,
|
||||
"bool.py": 70.0,
|
||||
"text.py": 70.0,
|
||||
},
|
||||
) == [
|
||||
"coverage report is missing critical file missing.py",
|
||||
"coverage report is missing numeric line coverage for bool.py",
|
||||
"coverage report is missing numeric line coverage for text.py",
|
||||
]
|
||||
|
||||
|
||||
def test_parse_changed_lines_and_measurement_ignore_non_executable_lines():
|
||||
diff = """diff --git a/obliteratus/example.py b/obliteratus/example.py
|
||||
+++ b/obliteratus/example.py
|
||||
@@ -111,3 +127,41 @@ def test_changed_line_gate_passes_when_diff_has_no_measured_source():
|
||||
0,
|
||||
100.0,
|
||||
)
|
||||
|
||||
|
||||
def test_changed_line_parser_ignores_hunks_before_paths_and_defaults_count_to_one():
|
||||
diff = """@@ -1 +99 @@
|
||||
+++ b/obliteratus/example.py
|
||||
@@ -2 +3 @@
|
||||
"""
|
||||
assert MODULE.parse_changed_lines(diff) == {"obliteratus/example.py": {3}}
|
||||
|
||||
|
||||
def test_changed_line_measurement_handles_missing_fields_and_multiple_files():
|
||||
report = {
|
||||
"files": {
|
||||
"obliteratus/no-fields.py": {"summary": {}},
|
||||
"obliteratus/first.py": {
|
||||
"executed_lines": [1],
|
||||
"missing_lines": [2],
|
||||
},
|
||||
"obliteratus/second.py": {
|
||||
"executed_lines": [4],
|
||||
"missing_lines": [5],
|
||||
},
|
||||
},
|
||||
}
|
||||
changed = {
|
||||
"missing-entry.py": {1},
|
||||
"obliteratus/no-fields.py": {1},
|
||||
"obliteratus/first.py": {1, 2, 99},
|
||||
"obliteratus/second.py": {4, 5, 100},
|
||||
}
|
||||
assert MODULE.changed_line_coverage(report, changed) == (2, 4, 50.0)
|
||||
|
||||
|
||||
def test_changed_line_gate_accepts_exact_floor():
|
||||
changed = {"obliteratus/example.py": {1, 2, 3, 4}}
|
||||
assert MODULE.validate_changed_coverage(
|
||||
_report(), changed, minimum=75.0,
|
||||
) == []
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Numerical and error-path tests for Fisher/LEACE direction extraction."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from obliteratus.analysis.leace import LEACEExtractor
|
||||
|
||||
|
||||
def _activations():
|
||||
harmful = [
|
||||
torch.tensor([2.0, 0.0, 1.0]),
|
||||
torch.tensor([3.0, 1.0, 0.0]),
|
||||
torch.tensor([4.0, -1.0, 1.0]),
|
||||
]
|
||||
harmless = [
|
||||
torch.tensor([-2.0, 0.0, 1.0]),
|
||||
torch.tensor([-3.0, 1.0, 0.0]),
|
||||
torch.tensor([-4.0, -1.0, 1.0]),
|
||||
]
|
||||
return harmful, harmless
|
||||
|
||||
|
||||
def test_extract_returns_normalized_reproducible_diagnostics():
|
||||
harmful, harmless = _activations()
|
||||
result = LEACEExtractor().extract(harmful, harmless, layer_idx=7)
|
||||
assert result.layer_idx == 7
|
||||
assert result.direction.norm().item() == pytest.approx(1.0)
|
||||
assert result.direction[0].abs().item() > 0.99
|
||||
assert result.generalized_eigenvalue > 0
|
||||
assert result.mean_diff_norm == pytest.approx(6.0)
|
||||
assert result.erasure_loss > 0
|
||||
assert result.within_class_condition >= 1
|
||||
|
||||
|
||||
def test_extract_accepts_singleton_sequence_axis_and_shrinkage():
|
||||
harmful, harmless = _activations()
|
||||
harmful_3d = [value.unsqueeze(0) for value in harmful]
|
||||
harmless_3d = [value.unsqueeze(0) for value in harmless]
|
||||
result = LEACEExtractor(shrinkage=0.5).extract(harmful_3d, harmless_3d)
|
||||
assert result.direction.shape == (3,)
|
||||
assert torch.isfinite(result.direction).all()
|
||||
|
||||
|
||||
def test_degenerate_classes_return_zero_direction():
|
||||
samples = [torch.ones(3), torch.ones(3)]
|
||||
result = LEACEExtractor().extract(samples, samples)
|
||||
assert torch.equal(result.direction, torch.zeros(3))
|
||||
assert result.generalized_eigenvalue == 0
|
||||
|
||||
|
||||
def test_solver_failure_uses_least_squares(monkeypatch):
|
||||
harmful, harmless = _activations()
|
||||
|
||||
def fail_solve(*_args, **_kwargs):
|
||||
raise torch.linalg.LinAlgError("fixture")
|
||||
|
||||
monkeypatch.setattr(torch.linalg, "solve", fail_solve)
|
||||
result = LEACEExtractor().extract(harmful, harmless)
|
||||
assert result.direction.norm().item() == pytest.approx(1.0)
|
||||
|
||||
|
||||
def test_condition_failure_is_reported_as_infinite(monkeypatch):
|
||||
harmful, harmless = _activations()
|
||||
original_solve = torch.linalg.solve
|
||||
monkeypatch.setattr(
|
||||
torch.linalg,
|
||||
"eigvalsh",
|
||||
lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("fixture")),
|
||||
)
|
||||
monkeypatch.setattr(torch.linalg, "solve", original_solve)
|
||||
result = LEACEExtractor().extract(harmful, harmless)
|
||||
assert result.within_class_condition == float("inf")
|
||||
|
||||
|
||||
def test_extract_all_layers_skips_unpaired_layers_and_sorts():
|
||||
harmful, harmless = _activations()
|
||||
results = LEACEExtractor().extract_all_layers(
|
||||
{2: harmful, 1: harmful, 3: harmful},
|
||||
{1: harmless, 2: harmless},
|
||||
)
|
||||
assert list(results) == [1, 2]
|
||||
assert results[1].layer_idx == 1
|
||||
|
||||
|
||||
def test_compare_with_diff_of_means_handles_regular_and_degenerate_difference():
|
||||
harmful, harmless = _activations()
|
||||
result = LEACEExtractor().extract(harmful, harmless)
|
||||
comparison = LEACEExtractor.compare_with_diff_of_means(
|
||||
result,
|
||||
torch.stack(harmful).mean(0),
|
||||
torch.stack(harmless).mean(0),
|
||||
)
|
||||
assert comparison["cosine_similarity"] == pytest.approx(1.0)
|
||||
degenerate = LEACEExtractor.compare_with_diff_of_means(
|
||||
result, torch.zeros(3), torch.zeros(3),
|
||||
)
|
||||
assert degenerate["cosine_similarity"] == 0
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Offline tests for prompt registry parsing and external schema boundaries."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from obliteratus import prompts
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clean_cache():
|
||||
prompts.clear_dataset_cache()
|
||||
yield
|
||||
prompts.clear_dataset_cache()
|
||||
|
||||
|
||||
def _datasets(monkeypatch, rows_or_error):
|
||||
def load_dataset(*_args, **_kwargs):
|
||||
if isinstance(rows_or_error, Exception):
|
||||
raise rows_or_error
|
||||
return rows_or_error
|
||||
|
||||
monkeypatch.setitem(sys.modules, "datasets", SimpleNamespace(load_dataset=load_dataset))
|
||||
|
||||
|
||||
def test_cache_copies_results_and_builtin_loader_isolated():
|
||||
calls = []
|
||||
|
||||
def loader():
|
||||
calls.append(True)
|
||||
return ["harm"], ["safe"]
|
||||
|
||||
first = prompts._cached_load("fixture", loader)
|
||||
first[0].append("mutation")
|
||||
assert prompts._cached_load("fixture", loader) == (["harm"], ["safe"])
|
||||
assert len(calls) == 1
|
||||
harmful, harmless = prompts._load_builtin()
|
||||
assert len(harmful) == len(harmless) == 842
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("loader", "rows", "expected"),
|
||||
[
|
||||
(prompts._load_harmbench, [{"Behavior": "A sufficiently long behavior"}], "A sufficiently long behavior"),
|
||||
(prompts._load_advbench, [{"goal": "A sufficiently long goal prompt"}], "A sufficiently long goal prompt"),
|
||||
],
|
||||
)
|
||||
def test_single_column_external_sources(monkeypatch, loader, rows, expected):
|
||||
_datasets(monkeypatch, rows)
|
||||
harmful, harmless = loader()
|
||||
assert harmful == [expected]
|
||||
assert len(harmless) == 1
|
||||
|
||||
|
||||
def test_anthropic_parser_deduplicates_and_uses_fallback(monkeypatch):
|
||||
calls = []
|
||||
|
||||
def load_dataset(*_args, **kwargs):
|
||||
calls.append(kwargs)
|
||||
if "data_dir" in kwargs:
|
||||
raise RuntimeError("primary unavailable")
|
||||
return [
|
||||
{"chosen": "Human: A unique and sufficiently long prompt Assistant: response"},
|
||||
{"rejected": "Human: A unique and sufficiently long prompt Assistant: other"},
|
||||
{"chosen": "no conversation markers"},
|
||||
]
|
||||
|
||||
monkeypatch.setitem(sys.modules, "datasets", SimpleNamespace(load_dataset=load_dataset))
|
||||
harmful, harmless = prompts._load_anthropic_redteam()
|
||||
assert harmful == ["A unique and sufficiently long prompt"]
|
||||
assert len(harmless) == 1
|
||||
assert len(calls) == 2
|
||||
|
||||
|
||||
def test_wildjailbreak_requires_pairs_and_deduplicates(monkeypatch):
|
||||
_datasets(monkeypatch, [
|
||||
{"adversarial_query": "A sufficiently long adversarial prompt", "vanilla_query": "safe"},
|
||||
{"adversarial": "A sufficiently long adversarial prompt", "vanilla": "duplicate"},
|
||||
{"adversarial": "missing pair"},
|
||||
])
|
||||
harmful, harmless = prompts._load_wildjailbreak()
|
||||
assert harmful == ["A sufficiently long adversarial prompt"]
|
||||
assert harmless == ["safe"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"loader",
|
||||
[prompts._load_harmbench, prompts._load_advbench, prompts._load_wildjailbreak],
|
||||
)
|
||||
def test_external_loaders_explain_empty_schemas(monkeypatch, loader):
|
||||
_datasets(monkeypatch, [{"unexpected": "value"}])
|
||||
with pytest.raises(RuntimeError, match="0 prompts extracted"):
|
||||
loader()
|
||||
|
||||
|
||||
def test_anthropic_empty_parse_is_rejected(monkeypatch):
|
||||
_datasets(monkeypatch, [{"chosen": "not a conversation"}])
|
||||
with pytest.raises(RuntimeError, match="0 prompts extracted"):
|
||||
prompts._load_anthropic_redteam()
|
||||
|
||||
|
||||
def test_custom_prompt_validation_padding_and_registry_access():
|
||||
harmful_text = "\n".join(f"harm {index}" for index in range(5))
|
||||
with pytest.raises(ValueError, match="at least 5"):
|
||||
prompts.load_custom_prompts("only one", "safe")
|
||||
|
||||
harmful, harmless = prompts.load_custom_prompts(harmful_text, "")
|
||||
assert len(harmful) == len(harmless) == 5
|
||||
harmful, harmless = prompts.load_custom_prompts(harmful_text, "safe one\nsafe two")
|
||||
assert harmless[:2] == ["safe one", "safe two"]
|
||||
assert len(harmless) == 5
|
||||
|
||||
with pytest.raises(ValueError, match="Unknown dataset source"):
|
||||
prompts.load_dataset_source("missing")
|
||||
assert prompts.get_source_key_from_label("missing label") == "builtin"
|
||||
assert prompts.get_source_key_from_label(prompts.DATASET_SOURCES["advbench"].label) == "advbench"
|
||||
assert len(prompts.get_source_choices()) == len(prompts.DATASET_SOURCES)
|
||||
assert prompts.get_valid_volumes("missing") == ["all (use entire dataset)"]
|
||||
assert prompts.get_valid_volumes("harmbench")[-1] == "all (use entire dataset)"
|
||||
|
||||
|
||||
def test_harmless_generator_cycles_deterministically():
|
||||
count = len(prompts._HARMLESS_POOL) + 1
|
||||
generated = prompts._generate_harmless_counterparts(count)
|
||||
assert generated[0] == generated[-1]
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Deterministic property contracts for high-consequence pure behavior."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from hypothesis import given, seed, settings, strategies as st
|
||||
|
||||
from obliteratus.evaluation.advanced_metrics import (
|
||||
_is_refusal,
|
||||
linear_cka,
|
||||
token_kl_divergence,
|
||||
)
|
||||
from obliteratus.evaluation.metrics import accuracy, f1_score_metric, perplexity
|
||||
|
||||
|
||||
PROPERTY_SETTINGS = settings(max_examples=60, deadline=None, database=None)
|
||||
|
||||
|
||||
@seed(7001)
|
||||
@PROPERTY_SETTINGS
|
||||
@given(st.lists(st.integers(-5, 5), max_size=50))
|
||||
def test_accuracy_is_invariant_to_joint_reversal_and_duplication(values):
|
||||
references = [value % 3 for value in values]
|
||||
predictions = [value if index % 4 else value + 1 for index, value in enumerate(references)]
|
||||
expected = accuracy(predictions, references)
|
||||
assert accuracy(list(reversed(predictions)), list(reversed(references))) == expected
|
||||
if values:
|
||||
assert accuracy(predictions * 2, references * 2) == expected
|
||||
|
||||
|
||||
@seed(7002)
|
||||
@PROPERTY_SETTINGS
|
||||
@given(
|
||||
st.lists(st.integers(0, 4), min_size=1, max_size=50),
|
||||
st.lists(st.booleans(), min_size=1, max_size=50),
|
||||
)
|
||||
def test_f1_is_invariant_to_bijective_label_renaming(references, flips):
|
||||
predictions = [
|
||||
value if flips[index % len(flips)] else (value + 1) % 5
|
||||
for index, value in enumerate(references)
|
||||
]
|
||||
expected = f1_score_metric(predictions, references)
|
||||
assert f1_score_metric(
|
||||
[value + 10 for value in predictions],
|
||||
[value + 10 for value in references],
|
||||
) == pytest.approx(expected)
|
||||
|
||||
|
||||
@seed(7003)
|
||||
@PROPERTY_SETTINGS
|
||||
@given(
|
||||
vocab_size=st.integers(2, 40),
|
||||
batch_size=st.integers(1, 4),
|
||||
sequence_length=st.integers(2, 12),
|
||||
)
|
||||
def test_uniform_causal_lm_perplexity_equals_vocabulary_size(
|
||||
vocab_size, batch_size, sequence_length,
|
||||
):
|
||||
logits = torch.zeros(batch_size, sequence_length, vocab_size)
|
||||
labels = torch.arange(batch_size * sequence_length).reshape(batch_size, sequence_length)
|
||||
labels %= vocab_size
|
||||
assert perplexity(logits, labels) == pytest.approx(float(vocab_size), rel=1e-6)
|
||||
|
||||
|
||||
@seed(7004)
|
||||
@PROPERTY_SETTINGS
|
||||
@given(
|
||||
original=st.lists(
|
||||
st.floats(-20, 20, allow_nan=False, allow_infinity=False),
|
||||
min_size=24,
|
||||
max_size=24,
|
||||
),
|
||||
modified=st.lists(
|
||||
st.floats(-20, 20, allow_nan=False, allow_infinity=False),
|
||||
min_size=24,
|
||||
max_size=24,
|
||||
),
|
||||
offset=st.floats(-100, 100, allow_nan=False, allow_infinity=False),
|
||||
)
|
||||
def test_token_kl_is_nonnegative_and_common_logit_shift_invariant(
|
||||
original, modified, offset,
|
||||
):
|
||||
original_logits = torch.tensor(original).reshape(2, 3, 4)
|
||||
modified_logits = torch.tensor(modified).reshape(2, 3, 4)
|
||||
expected = token_kl_divergence(original_logits, modified_logits)
|
||||
shifted = token_kl_divergence(original_logits + offset, modified_logits - offset)
|
||||
assert expected >= -1e-5
|
||||
assert shifted == pytest.approx(expected, abs=2e-5)
|
||||
|
||||
|
||||
@seed(7005)
|
||||
@PROPERTY_SETTINGS
|
||||
@given(
|
||||
scale=st.floats(
|
||||
min_value=0.1, max_value=20, allow_nan=False, allow_infinity=False,
|
||||
),
|
||||
offset=st.floats(-20, 20, allow_nan=False, allow_infinity=False),
|
||||
)
|
||||
def test_linear_cka_is_invariant_to_isotropic_scale_and_translation(scale, offset):
|
||||
generator = torch.Generator().manual_seed(7005)
|
||||
representation = torch.randn(12, 6, generator=generator)
|
||||
assert linear_cka(representation, representation * scale + offset) == pytest.approx(
|
||||
1.0, abs=1e-5,
|
||||
)
|
||||
|
||||
|
||||
@seed(7006)
|
||||
@PROPERTY_SETTINGS
|
||||
@given(
|
||||
refusal=st.sampled_from([
|
||||
"I'm sorry, I cannot help with that.",
|
||||
"I must decline this request.",
|
||||
"Je ne peux pas vous aider.",
|
||||
"我无法帮助这个请求。",
|
||||
]),
|
||||
leading_space=st.integers(0, 8),
|
||||
upper=st.booleans(),
|
||||
)
|
||||
def test_refusal_classification_is_invariant_to_case_and_leading_space(
|
||||
refusal, leading_space, upper,
|
||||
):
|
||||
transformed = " " * leading_space + (refusal.upper() if upper else refusal.lower())
|
||||
assert _is_refusal(transformed, mode="combined")
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Unit tests for quality-depth gate helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
from scripts import check_mutation_score
|
||||
from scripts import run_repeat_gate
|
||||
|
||||
|
||||
def test_mutation_score_accepts_exact_floor_and_rejects_regression():
|
||||
stats = {"killed": 70, "total": 100, "check_was_interrupted_by_user": 0}
|
||||
assert check_mutation_score.validate_mutation_stats(stats, minimum=70) == []
|
||||
assert check_mutation_score.validate_mutation_stats(stats, minimum=70.01) == [
|
||||
"mutation score 70.00% (70/100) is below the 70.01% floor",
|
||||
]
|
||||
|
||||
|
||||
def test_mutation_score_rejects_malformed_and_interrupted_runs():
|
||||
assert check_mutation_score.validate_mutation_stats(
|
||||
{"killed": True, "total": 1}, minimum=70,
|
||||
) == ["mutation statistics require a non-negative integer 'killed'"]
|
||||
assert check_mutation_score.validate_mutation_stats(
|
||||
{"killed": 1, "total": 1, "check_was_interrupted_by_user": 1}, minimum=70,
|
||||
) == ["mutation run was interrupted"]
|
||||
|
||||
|
||||
def test_repeat_orders_are_distinct_and_deterministic():
|
||||
paths = ["a", "b", "c", "d"]
|
||||
assert run_repeat_gate.test_orders(paths) == [
|
||||
["a", "b", "c", "d"],
|
||||
["d", "c", "b", "a"],
|
||||
["a", "c", "b", "d"],
|
||||
]
|
||||
|
||||
|
||||
def test_repeat_gate_records_each_pass(monkeypatch, tmp_path):
|
||||
calls = []
|
||||
|
||||
def fake_run(command, **kwargs):
|
||||
calls.append((command, kwargs["env"]["PYTHONHASHSEED"]))
|
||||
return SimpleNamespace(returncode=0, stdout="2 passed\n", stderr="")
|
||||
|
||||
monkeypatch.setattr(run_repeat_gate.subprocess, "run", fake_run)
|
||||
output = tmp_path / "repeat.json"
|
||||
assert run_repeat_gate.run_repeat_gate(
|
||||
["first.py", "second.py"], output=output, python="python-fixture",
|
||||
) == 0
|
||||
evidence = json.loads(output.read_text())
|
||||
assert evidence["status"] == "passed"
|
||||
assert [entry["python_hash_seed"] for entry in evidence["passes"]] == [
|
||||
"0", "1", "8675309",
|
||||
]
|
||||
assert calls[0][0] == [
|
||||
"python-fixture", "-m", "pytest", "--no-cov", "-q", "first.py", "second.py",
|
||||
]
|
||||
|
||||
|
||||
def test_repeat_gate_stops_and_preserves_failure_output(monkeypatch, tmp_path):
|
||||
def fake_run(*_args, **_kwargs):
|
||||
return SimpleNamespace(returncode=3, stdout="failed output", stderr="failure detail")
|
||||
|
||||
monkeypatch.setattr(run_repeat_gate.subprocess, "run", fake_run)
|
||||
output = tmp_path / "repeat.json"
|
||||
assert run_repeat_gate.run_repeat_gate(["test.py"], output=output) == 3
|
||||
evidence = json.loads(output.read_text())
|
||||
assert evidence["status"] == "failed"
|
||||
assert len(evidence["passes"]) == 1
|
||||
assert evidence["passes"][0]["stderr"] == "failure detail"
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Tests for quality-policy immutability and mature-scope measurement."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
|
||||
from scripts import check_quality_policy as quality
|
||||
|
||||
|
||||
def _policy():
|
||||
return {
|
||||
"minimums": dict(quality.BASELINE_FLOORS),
|
||||
"mature_cpu_scope": {
|
||||
"exclusions": [{
|
||||
"path": "obliteratus/external.py",
|
||||
"boundary": "network-service",
|
||||
"rationale": "Requires a live external service.",
|
||||
"conditional_issue": "https://github.com/elder-plinius/OBLITERATUS/issues/71",
|
||||
"conditional_gate": "network-services",
|
||||
}],
|
||||
},
|
||||
"threshold_exceptions": [],
|
||||
}
|
||||
|
||||
|
||||
def _coverage():
|
||||
return {
|
||||
"files": {
|
||||
"obliteratus/pure.py": {
|
||||
"summary": {
|
||||
"num_statements": 100,
|
||||
"covered_lines": 80,
|
||||
"num_branches": 20,
|
||||
"covered_branches": 15,
|
||||
},
|
||||
},
|
||||
"obliteratus/external.py": {
|
||||
"summary": {
|
||||
"num_statements": 1000,
|
||||
"covered_lines": 0,
|
||||
"num_branches": 500,
|
||||
"covered_branches": 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_policy_and_exact_mature_floors_pass():
|
||||
policy = _policy()
|
||||
assert quality.validate_policy(policy) == []
|
||||
measurement, failures = quality.validate_mature_cpu_scope(_coverage(), policy)
|
||||
assert failures == []
|
||||
assert measurement["line_percent"] == 80
|
||||
assert measurement["branch_percent"] == 75
|
||||
|
||||
|
||||
def test_floor_regression_requires_structured_reviewed_exception():
|
||||
policy = _policy()
|
||||
policy["minimums"]["mutation_score"] = 69
|
||||
assert quality.validate_policy(policy) == [
|
||||
"quality minimum mutation_score cannot move below 70 without an explicit reviewed exception",
|
||||
]
|
||||
policy["threshold_exceptions"] = [{
|
||||
"threshold": "mutation_score",
|
||||
"new_value": 69,
|
||||
"reason": "Temporary tool regression",
|
||||
"approved_issue": "https://github.com/elder-plinius/OBLITERATUS/issues/999",
|
||||
"expires": "2026-09-01",
|
||||
}]
|
||||
assert quality.validate_policy(policy) == []
|
||||
|
||||
|
||||
def test_exclusions_require_unique_traceable_environment_boundaries():
|
||||
policy = _policy()
|
||||
duplicate = deepcopy(policy["mature_cpu_scope"]["exclusions"][0])
|
||||
duplicate["rationale"] = ""
|
||||
policy["mature_cpu_scope"]["exclusions"].append(duplicate)
|
||||
failures = quality.validate_policy(policy)
|
||||
assert "mature CPU exclusion path is duplicated: obliteratus/external.py" in failures
|
||||
assert "mature CPU exclusion 1 requires non-empty rationale" in failures
|
||||
|
||||
|
||||
def test_mature_scope_rejects_regression_and_stale_exclusion():
|
||||
policy = _policy()
|
||||
report = _coverage()
|
||||
report["files"]["obliteratus/pure.py"]["summary"]["covered_lines"] = 79
|
||||
_, failures = quality.validate_mature_cpu_scope(report, policy)
|
||||
assert failures == [
|
||||
"mature CPU line coverage 79.00% is below the 80.00% floor",
|
||||
]
|
||||
del report["files"]["obliteratus/external.py"]
|
||||
_, failures = quality.measure_mature_cpu_scope(report, policy)
|
||||
assert failures == [
|
||||
"coverage report is missing excluded source file obliteratus/external.py",
|
||||
]
|
||||
Reference in New Issue
Block a user