"""Offline tests for prompt registry parsing and external schema boundaries.""" from __future__ import annotations import sys from types import SimpleNamespace import pytest from obliteratus import prompts @pytest.fixture(autouse=True) def clean_cache(): prompts.clear_dataset_cache() yield prompts.clear_dataset_cache() def _datasets(monkeypatch, rows_or_error): def load_dataset(*_args, **_kwargs): if isinstance(rows_or_error, Exception): raise rows_or_error return rows_or_error monkeypatch.setitem(sys.modules, "datasets", SimpleNamespace(load_dataset=load_dataset)) def test_cache_copies_results_and_builtin_loader_isolated(): calls = [] def loader(): calls.append(True) return ["harm"], ["safe"] first = prompts._cached_load("fixture", loader) first[0].append("mutation") assert prompts._cached_load("fixture", loader) == (["harm"], ["safe"]) assert len(calls) == 1 harmful, harmless = prompts._load_builtin() assert len(harmful) == len(harmless) == 842 @pytest.mark.parametrize( ("loader", "rows", "expected"), [ (prompts._load_harmbench, [{"Behavior": "A sufficiently long behavior"}], "A sufficiently long behavior"), (prompts._load_advbench, [{"goal": "A sufficiently long goal prompt"}], "A sufficiently long goal prompt"), ], ) def test_single_column_external_sources(monkeypatch, loader, rows, expected): _datasets(monkeypatch, rows) harmful, harmless = loader() assert harmful == [expected] assert len(harmless) == 1 def test_anthropic_parser_deduplicates_and_uses_fallback(monkeypatch): calls = [] def load_dataset(*_args, **kwargs): calls.append(kwargs) if "data_dir" in kwargs: raise RuntimeError("primary unavailable") return [ {"chosen": "Human: A unique and sufficiently long prompt Assistant: response"}, {"rejected": "Human: A unique and sufficiently long prompt Assistant: other"}, {"chosen": "no conversation markers"}, ] monkeypatch.setitem(sys.modules, "datasets", SimpleNamespace(load_dataset=load_dataset)) harmful, harmless = prompts._load_anthropic_redteam() assert harmful == ["A unique and sufficiently long prompt"] assert len(harmless) == 1 assert len(calls) == 2 def test_wildjailbreak_requires_pairs_and_deduplicates(monkeypatch): _datasets(monkeypatch, [ {"adversarial_query": "A sufficiently long adversarial prompt", "vanilla_query": "safe"}, {"adversarial": "A sufficiently long adversarial prompt", "vanilla": "duplicate"}, {"adversarial": "missing pair"}, ]) harmful, harmless = prompts._load_wildjailbreak() assert harmful == ["A sufficiently long adversarial prompt"] assert harmless == ["safe"] @pytest.mark.parametrize( "loader", [prompts._load_harmbench, prompts._load_advbench, prompts._load_wildjailbreak], ) def test_external_loaders_explain_empty_schemas(monkeypatch, loader): _datasets(monkeypatch, [{"unexpected": "value"}]) with pytest.raises(RuntimeError, match="0 prompts extracted"): loader() def test_anthropic_empty_parse_is_rejected(monkeypatch): _datasets(monkeypatch, [{"chosen": "not a conversation"}]) with pytest.raises(RuntimeError, match="0 prompts extracted"): prompts._load_anthropic_redteam() def test_custom_prompt_validation_padding_and_registry_access(): harmful_text = "\n".join(f"harm {index}" for index in range(5)) with pytest.raises(ValueError, match="at least 5"): prompts.load_custom_prompts("only one", "safe") harmful, harmless = prompts.load_custom_prompts(harmful_text, "") assert len(harmful) == len(harmless) == 5 harmful, harmless = prompts.load_custom_prompts(harmful_text, "safe one\nsafe two") assert harmless[:2] == ["safe one", "safe two"] assert len(harmless) == 5 with pytest.raises(ValueError, match="Unknown dataset source"): prompts.load_dataset_source("missing") assert prompts.get_source_key_from_label("missing label") == "builtin" assert prompts.get_source_key_from_label(prompts.DATASET_SOURCES["advbench"].label) == "advbench" assert len(prompts.get_source_choices()) == len(prompts.DATASET_SOURCES) assert prompts.get_valid_volumes("missing") == ["all (use entire dataset)"] assert prompts.get_valid_volumes("harmbench")[-1] == "all (use entire dataset)" def test_harmless_generator_cycles_deterministically(): count = len(prompts._HARMLESS_POOL) + 1 generated = prompts._generate_harmless_counterparts(count) assert generated[0] == generated[-1]