mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-08-17 16:37:30 +02:00
129 lines
4.6 KiB
Python
129 lines
4.6 KiB
Python
"""Offline tests for prompt registry parsing and external schema boundaries."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from obliteratus import prompts
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def clean_cache():
|
|
prompts.clear_dataset_cache()
|
|
yield
|
|
prompts.clear_dataset_cache()
|
|
|
|
|
|
def _datasets(monkeypatch, rows_or_error):
|
|
def load_dataset(*_args, **_kwargs):
|
|
if isinstance(rows_or_error, Exception):
|
|
raise rows_or_error
|
|
return rows_or_error
|
|
|
|
monkeypatch.setitem(sys.modules, "datasets", SimpleNamespace(load_dataset=load_dataset))
|
|
|
|
|
|
def test_cache_copies_results_and_builtin_loader_isolated():
|
|
calls = []
|
|
|
|
def loader():
|
|
calls.append(True)
|
|
return ["harm"], ["safe"]
|
|
|
|
first = prompts._cached_load("fixture", loader)
|
|
first[0].append("mutation")
|
|
assert prompts._cached_load("fixture", loader) == (["harm"], ["safe"])
|
|
assert len(calls) == 1
|
|
harmful, harmless = prompts._load_builtin()
|
|
assert len(harmful) == len(harmless) == 842
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("loader", "rows", "expected"),
|
|
[
|
|
(prompts._load_harmbench, [{"Behavior": "A sufficiently long behavior"}], "A sufficiently long behavior"),
|
|
(prompts._load_advbench, [{"goal": "A sufficiently long goal prompt"}], "A sufficiently long goal prompt"),
|
|
],
|
|
)
|
|
def test_single_column_external_sources(monkeypatch, loader, rows, expected):
|
|
_datasets(monkeypatch, rows)
|
|
harmful, harmless = loader()
|
|
assert harmful == [expected]
|
|
assert len(harmless) == 1
|
|
|
|
|
|
def test_anthropic_parser_deduplicates_and_uses_fallback(monkeypatch):
|
|
calls = []
|
|
|
|
def load_dataset(*_args, **kwargs):
|
|
calls.append(kwargs)
|
|
if "data_dir" in kwargs:
|
|
raise RuntimeError("primary unavailable")
|
|
return [
|
|
{"chosen": "Human: A unique and sufficiently long prompt Assistant: response"},
|
|
{"rejected": "Human: A unique and sufficiently long prompt Assistant: other"},
|
|
{"chosen": "no conversation markers"},
|
|
]
|
|
|
|
monkeypatch.setitem(sys.modules, "datasets", SimpleNamespace(load_dataset=load_dataset))
|
|
harmful, harmless = prompts._load_anthropic_redteam()
|
|
assert harmful == ["A unique and sufficiently long prompt"]
|
|
assert len(harmless) == 1
|
|
assert len(calls) == 2
|
|
|
|
|
|
def test_wildjailbreak_requires_pairs_and_deduplicates(monkeypatch):
|
|
_datasets(monkeypatch, [
|
|
{"adversarial_query": "A sufficiently long adversarial prompt", "vanilla_query": "safe"},
|
|
{"adversarial": "A sufficiently long adversarial prompt", "vanilla": "duplicate"},
|
|
{"adversarial": "missing pair"},
|
|
])
|
|
harmful, harmless = prompts._load_wildjailbreak()
|
|
assert harmful == ["A sufficiently long adversarial prompt"]
|
|
assert harmless == ["safe"]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"loader",
|
|
[prompts._load_harmbench, prompts._load_advbench, prompts._load_wildjailbreak],
|
|
)
|
|
def test_external_loaders_explain_empty_schemas(monkeypatch, loader):
|
|
_datasets(monkeypatch, [{"unexpected": "value"}])
|
|
with pytest.raises(RuntimeError, match="0 prompts extracted"):
|
|
loader()
|
|
|
|
|
|
def test_anthropic_empty_parse_is_rejected(monkeypatch):
|
|
_datasets(monkeypatch, [{"chosen": "not a conversation"}])
|
|
with pytest.raises(RuntimeError, match="0 prompts extracted"):
|
|
prompts._load_anthropic_redteam()
|
|
|
|
|
|
def test_custom_prompt_validation_padding_and_registry_access():
|
|
harmful_text = "\n".join(f"harm {index}" for index in range(5))
|
|
with pytest.raises(ValueError, match="at least 5"):
|
|
prompts.load_custom_prompts("only one", "safe")
|
|
|
|
harmful, harmless = prompts.load_custom_prompts(harmful_text, "")
|
|
assert len(harmful) == len(harmless) == 5
|
|
harmful, harmless = prompts.load_custom_prompts(harmful_text, "safe one\nsafe two")
|
|
assert harmless[:2] == ["safe one", "safe two"]
|
|
assert len(harmless) == 5
|
|
|
|
with pytest.raises(ValueError, match="Unknown dataset source"):
|
|
prompts.load_dataset_source("missing")
|
|
assert prompts.get_source_key_from_label("missing label") == "builtin"
|
|
assert prompts.get_source_key_from_label(prompts.DATASET_SOURCES["advbench"].label) == "advbench"
|
|
assert len(prompts.get_source_choices()) == len(prompts.DATASET_SOURCES)
|
|
assert prompts.get_valid_volumes("missing") == ["all (use entire dataset)"]
|
|
assert prompts.get_valid_volumes("harmbench")[-1] == "all (use entire dataset)"
|
|
|
|
|
|
def test_harmless_generator_cycles_deterministically():
|
|
count = len(prompts._HARMLESS_POOL) + 1
|
|
generated = prompts._generate_harmless_counterparts(count)
|
|
assert generated[0] == generated[-1]
|