mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-07-12 23:26:32 +02:00
faber
04b8ec60cb
New core modules: - auto_obliterate.py: Automated multi-iteration obliteration pipeline - watchtower.py: HF Hub model discovery and tracking - ui_watchtower.py: Gradio tabs for Watchtower (ready for app.py wiring) - hard_negative.py: Residue mining from refusal audits - model_profile.py: Parameter profiling from safetensors/config - bestiary_sync.py: Sync models from PlinyOS BESTIARY registry - models_client.py: Lightweight HF model list client Framework enhancements: - abliterate.py: ASPA source-tethering, step gradient blending, hard-negative residue support - cli.py: self-improve command, model profiling, hard-negative flags - prompts.py: Expanded 842-prompt refusal eval corpus across 10 categories - __init__.py: New exports (Watchtower, AutoObliterator) Reference implementations (14 scripts): - ASPA sweep, gradient search, coherence eval, MMLU benchmarks - Pareto controller, refusal sniper, stock comparisons Documentation: - README: Research framing, responsible use section, comprehensive disclaimer - docs/beyond_sota_roadmap.md, docs/recursive_self_improvement.md Tests: 4 new test files (354 lines) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
105 lines
3.2 KiB
Python
105 lines
3.2 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
from obliteratus.hard_negative import (
|
|
build_weighted_prompt_pairs,
|
|
classify_refusal_reason,
|
|
load_residue_file,
|
|
prompt_hash,
|
|
)
|
|
|
|
|
|
def test_load_residue_file_from_audit_summary(tmp_path):
|
|
audit = {
|
|
"n3_reg025": {
|
|
"refusals": [
|
|
{
|
|
"tier": "tier4_cyber_digital",
|
|
"global_index_1based": 2,
|
|
"prompt_sha256_12": "abc123",
|
|
"reason": "semantic: illegal privacy violation",
|
|
"response_preview": "This is illegal and a severe privacy violation.",
|
|
}
|
|
]
|
|
}
|
|
}
|
|
path = tmp_path / "summary.json"
|
|
path.write_text(json.dumps(audit))
|
|
|
|
rows = load_residue_file(path)
|
|
|
|
assert len(rows) == 1
|
|
assert rows[0].dataset == "builtin"
|
|
assert rows[0].index == 1
|
|
assert rows[0].tier == "tier4_cyber_digital"
|
|
assert "illegal_crime" in rows[0].tags
|
|
assert "privacy_intrusion" in rows[0].tags
|
|
|
|
|
|
def test_build_weighted_prompt_pairs_reconstructs_by_index(monkeypatch, tmp_path):
|
|
harmful = ["h0", "h1 stubborn", "h2"]
|
|
harmless = ["a0", "a1 benign counterpart", "a2"]
|
|
|
|
def fake_load_dataset_source(key):
|
|
assert key == "builtin"
|
|
return list(harmful), list(harmless)
|
|
|
|
monkeypatch.setattr("obliteratus.hard_negative.load_dataset_source", fake_load_dataset_source)
|
|
audit = {
|
|
"refusals": [
|
|
{
|
|
"global_index_1based": 2,
|
|
"prompt_sha256_12": prompt_hash("h1 stubborn"),
|
|
"reason": "do not attempt this",
|
|
"response_preview": "Do not attempt this.",
|
|
}
|
|
]
|
|
}
|
|
path = tmp_path / "audit.json"
|
|
path.write_text(json.dumps(audit))
|
|
|
|
h, a, meta = build_weighted_prompt_pairs(
|
|
base_dataset="builtin",
|
|
residue_files=[path],
|
|
residue_weight=3,
|
|
)
|
|
|
|
assert h[:3] == harmful
|
|
assert a[:3] == harmless
|
|
assert h[3:] == ["h1 stubborn"] * 3
|
|
assert a[3:] == ["a1 benign counterpart"] * 3
|
|
assert meta["residue_examples"] == 1
|
|
assert meta["residue_added_pairs"] == 3
|
|
assert meta["total_pairs"] == 6
|
|
meta_text = json.dumps(meta)
|
|
assert "h1 stubborn" not in meta_text
|
|
assert "a1 benign counterpart" not in meta_text
|
|
|
|
|
|
def test_hash_mismatch_skips_residue(monkeypatch, tmp_path):
|
|
monkeypatch.setattr(
|
|
"obliteratus.hard_negative.load_dataset_source",
|
|
lambda key: (["h0", "changed"], ["a0", "a1"]),
|
|
)
|
|
path = tmp_path / "audit.json"
|
|
path.write_text(json.dumps({"refusals": [{"global_index_1based": 2, "prompt_sha256_12": "badbadbadbad"}]}))
|
|
|
|
h, a, meta = build_weighted_prompt_pairs("builtin", [path], residue_weight=5)
|
|
|
|
assert h == ["h0", "changed"]
|
|
assert a == ["a0", "a1"]
|
|
assert meta["residue_examples"] == 0
|
|
assert meta["residue_added_pairs"] == 0
|
|
|
|
|
|
def test_classify_refusal_reason_tags():
|
|
tags = classify_refusal_reason(
|
|
"semantic illegal/crime pattern",
|
|
"Stop. This is a dangerous terrorism and privacy violation warning.",
|
|
)
|
|
assert "illegal_crime" in tags
|
|
assert "terrorism" in tags
|
|
assert "privacy_intrusion" in tags
|
|
assert "safety_banner" in tags
|