diff --git a/app.py b/app.py index 5c23316..a1a3b47 100644 --- a/app.py +++ b/app.py @@ -534,6 +534,7 @@ METHODS = { "adaptive (telemetry-recommended)": "adaptive", "Qwen3.8 E01 (held-out causal baseline)": "qwen38_e01", "Qwen3.8 E02 (tune-only multi-direction)": "qwen38_e02", + "Qwen3.8 E03 (tune-only coherence recovery)": "qwen38_e03", "advanced (recommended)": "advanced", "basic (fast, single direction)": "basic", "aggressive (maximum removal)": "aggressive", @@ -2285,8 +2286,12 @@ def obliterate(model_choice: str, method_choice: str, "harmful": hashlib.sha256((custom_harmful or "").encode()).hexdigest(), "harmless": hashlib.sha256((custom_harmless or "").encode()).hexdigest(), } - if method in {"qwen38_e01", "qwen38_e02"}: - experiment = "E01" if method == "qwen38_e01" else "E02" + if method in {"qwen38_e01", "qwen38_e02", "qwen38_e03"}: + experiment = { + "qwen38_e01": "E01", + "qwen38_e02": "E02", + "qwen38_e03": "E03", + }[method] experiment_settings = { "E01": { "direction_method": "diff_means", @@ -2313,6 +2318,20 @@ def obliterate(model_choice: str, method_choice: str, "verify_sample_size": 142, "evaluation_split": "optimizer_tune", }, + "E03": { + "direction_method": "svd", + "n_directions": 4, + "regularization": 0.1, + "refinement_passes": 1, + "norm_preserve": True, + "layer_selection": "middle60", + "projection_target": "output", + "rdo_refinement": True, + "winsorize_activations": True, + "use_kl_optimization": True, + "verify_sample_size": 142, + "evaluation_split": "optimizer_tune", + }, }[experiment] run_config["immutable_experiment"] = { "protocol": "qwen38-v1", @@ -2416,7 +2435,7 @@ def obliterate(model_choice: str, method_choice: str, evaluation_harmful = None evaluation_harmless = None - if method in {"qwen38_e01", "qwen38_e02"}: + if method in {"qwen38_e01", "qwen38_e02", "qwen38_e03"}: if model_id.rstrip("/").lower() != "qwen/qwen3.8-27b": raise ValueError("Qwen3.8 experiments require Qwen/Qwen3.8-27B") if use_custom or dataset_key != "builtin": @@ -2431,7 +2450,11 @@ def obliterate(model_choice: str, method_choice: str, split = build_qwen38_split(harmful_all, harmless_all) train_harmful = [pair[0] for pair in split.train] train_harmless = [pair[1] for pair in split.train] - experiment = "E01" if method == "qwen38_e01" else "E02" + experiment = { + "qwen38_e01": "E01", + "qwen38_e02": "E02", + "qwen38_e03": "E03", + }[method] evaluation_pairs = qwen38_evaluation_pairs( split, experiment, @@ -2448,7 +2471,7 @@ def obliterate(model_choice: str, method_choice: str, ) on_log( f"{experiment} evaluation partition: " - f"{'optimizer-tune' if experiment == 'E02' else 'final-test'} " + f"{'final-test' if experiment == 'E01' else 'optimizer-tune'} " f"({len(evaluation_pairs)} pairs)" ) on_log(f"Split manifest: {split.manifest['manifest_sha256']}") @@ -2492,13 +2515,13 @@ def obliterate(model_choice: str, method_choice: str, if immutable_qwen38: # Promotion experiments are immutable: ignore mutable # advanced controls and run the registered candidate. - is_e02 = method == "qwen38_e02" + is_tune_candidate = method in {"qwen38_e02", "qwen38_e03"} advanced_options = { - "n_directions": 4 if is_e02 else 1, - "direction_method": "svd" if is_e02 else "diff_means", - "regularization": 0.0, + "n_directions": 4 if is_tune_candidate else 1, + "direction_method": "svd" if is_tune_candidate else "diff_means", + "regularization": 0.1 if method == "qwen38_e03" else 0.0, "refinement_passes": 1, - "norm_preserve": is_e02, + "norm_preserve": is_tune_candidate, "project_biases": False, "use_chat_template": True, "use_whitened_svd": False, @@ -2515,13 +2538,13 @@ def obliterate(model_choice: str, method_choice: str, "expert_transplant": False, "use_wasserstein_optimal": False, "spectral_cascade": False, - "layer_selection": "middle60" if is_e02 else "all_except_first", - "winsorize_activations": is_e02, - "use_kl_optimization": is_e02, + "layer_selection": "middle60" if is_tune_candidate else "all_except_first", + "winsorize_activations": is_tune_candidate, + "use_kl_optimization": is_tune_candidate, "float_layer_interpolation": False, - "rdo_refinement": is_e02, + "rdo_refinement": is_tune_candidate, "cot_aware": False, - "verify_sample_size": 142 if is_e02 else 200, + "verify_sample_size": 142 if is_tune_candidate else 200, } else: advanced_options = { @@ -2577,12 +2600,12 @@ def obliterate(model_choice: str, method_choice: str, spectral_threshold=float(adv_spectral_threshold), winsorize_percentile=( 0.01 - if immutable_qwen38 and method == "qwen38_e02" + if immutable_qwen38 and method in {"qwen38_e02", "qwen38_e03"} else float(adv_winsorize_percentile) ), kl_budget=( 0.5 - if immutable_qwen38 and method == "qwen38_e02" + if immutable_qwen38 and method in {"qwen38_e02", "qwen38_e03"} else float(adv_kl_budget) ), n_sae_features=int(adv_n_sae_features), diff --git a/docs/QWEN38_27B_RESEARCH_ROADMAP.md b/docs/QWEN38_27B_RESEARCH_ROADMAP.md index 04db834..a3b3e96 100644 --- a/docs/QWEN38_27B_RESEARCH_ROADMAP.md +++ b/docs/QWEN38_27B_RESEARCH_ROADMAP.md @@ -215,7 +215,7 @@ method needs an isolated ablation and an interaction test before composition. | E00 | Is the pristine evaluator stable? | final-test only, 3 seeds | none | metric variance and baselines recorded | | E01 | Does the Arditi control work? | 400/100 | DIM, one layer/direction sweep | lower refusal with <=1.25x PPL | | E02 | Does a low-rank writer intervention improve E01? | 500/142; final test prohibited | SVD-4 + RDO, middle-60% residual writers, KL rollback | tune refusal <30% and coherence >=80% | -| E03 | Which architecture component carries refusal? | 400/100 | DeltaNet vs full-attn vs MLP groups | best causal effect per KL unit | +| E03 | Can slight attenuation recover E02 coherence? | 500/142; final test prohibited | E02 kernel with 0.10 regularization | tune refusal <30% and coherence >=80% | | E04 | Does SVD rank help? | 400/100 | k=1..7, joint held-out selection | nondominated gain over E01 | | E05 | Does RDO improve targeting? | 400/100 | one and multiple RDO directions | held-out gain over E01/E04 | | E06 | Do concept-cone combinations help? | large corpus + 128--512 trials | SOM/independent combinations | reproducible gain over E05 | @@ -256,11 +256,15 @@ from 60% refusal in S2 to 100% in several other strata. ## Recommended next operational run -Run the pre-registered E02 candidate on the 142-pair optimizer-tune partition. -E02 must not read the 200-pair final partition. Retain its complete archive even -if it fails. If it misses either target, use only tune evidence to define the -next isolated ablation. If it passes both targets, reload that saved checkpoint -and run the final partition exactly once for release qualification. +E02 completed on the optimizer-tune partition as run +`run-efb2f334197e48fb82587f39a84fe6c9`: refusal fell to 1% (1/142), but +coherence was 70% and 6/142 harmful responses were degenerate. Its checkpoint +and full archive are retained. E03 changes only regularization from 0.00 to +0.10, retaining 10% of the measured refusal subspace to recover coherence while +using E02's 29-point refusal margin. E03 must not read the 200-pair final +partition. Promote only if tune refusal remains below 30% and coherence reaches +at least 80%; then evaluate the final partition exactly once and independently +reload the saved checkpoint for release qualification. ## Evidence base diff --git a/docs/qwen38-experiment-matrix-v1.json b/docs/qwen38-experiment-matrix-v1.json index 4ca08d9..502d45c 100644 --- a/docs/qwen38-experiment-matrix-v1.json +++ b/docs/qwen38-experiment-matrix-v1.json @@ -59,8 +59,20 @@ }, { "id": "E03", - "purpose": "DeltaNet/full-attention/MLP writer ablation", - "depends_on": "E01" + "purpose": "tune-only coherence recovery by attenuating E02", + "depends_on": "E02", + "method": "qwen38_e03", + "evaluation_split": "optimizer_tune", + "direction_method": "svd", + "directions": 4, + "layer_selection": "middle60", + "projection_target": "output", + "regularization": 0.1, + "norm_preserve": true, + "refinement_passes": 1, + "rdo_refinement": true, + "winsorize_activations": true, + "kl_optimization": true } ] } diff --git a/obliteratus/abliterate.py b/obliteratus/abliterate.py index 24fe35d..5883263 100644 --- a/obliteratus/abliterate.py +++ b/obliteratus/abliterate.py @@ -249,6 +249,37 @@ METHODS = { "rdo_refinement": True, "verify_sample_size": 142, }, + "qwen38_e03": { + "label": "Qwen3.8 E03 (tune-only coherence recovery)", + "description": ( + "E02 follow-up on the same held-out tune split: retain 10% of the " + "validated residual-writer subspace to recover coherence while " + "preserving E02's refusal-removal margin." + ), + "n_directions": 4, + "direction_method": "svd", + "norm_preserve": True, + "regularization": 0.1, + "refinement_passes": 1, + "project_biases": False, + "use_chat_template": True, + "use_whitened_svd": False, + "true_iterative_refinement": False, + "use_jailbreak_contrast": False, + "layer_adaptive_strength": False, + "safety_neuron_masking": False, + "per_expert_directions": False, + "attention_head_surgery": False, + "use_sae_features": False, + "invert_refusal": False, + "use_kl_optimization": True, + "winsorize_activations": True, + "winsorize_percentile": 0.01, + "layer_selection": "middle60", + "projection_target": "output", + "rdo_refinement": True, + "verify_sample_size": 142, + }, "basic": { "label": "Basic (Arditi et al.)", "description": "Single refusal direction via difference-in-means", diff --git a/obliteratus/experiment_protocol.py b/obliteratus/experiment_protocol.py index 022c6fe..9a206bc 100644 --- a/obliteratus/experiment_protocol.py +++ b/obliteratus/experiment_protocol.py @@ -33,6 +33,7 @@ def qwen38_evaluation_pairs( partitions = { "E01": split.test, "E02": split.tune, + "E03": split.tune, } try: return partitions[experiment] diff --git a/tests/test_abliterate.py b/tests/test_abliterate.py index ef2c4ab..745ee42 100644 --- a/tests/test_abliterate.py +++ b/tests/test_abliterate.py @@ -163,7 +163,7 @@ class TestStages: class TestMethods: def test_methods_exist(self): - assert set(METHODS.keys()) == {"basic", "advanced", "aggressive", "informed", "surgical", "inverted", "nuclear", "optimized", "failspy", "gabliteration", "heretic", "rdo", "spectral_cascade", "som", "qwen38_e01", "qwen38_e02"} + assert set(METHODS.keys()) == {"basic", "advanced", "aggressive", "informed", "surgical", "inverted", "nuclear", "optimized", "failspy", "gabliteration", "heretic", "rdo", "spectral_cascade", "som", "qwen38_e01", "qwen38_e02", "qwen38_e03"} def test_basic_single_direction(self): cfg = METHODS["basic"] @@ -179,6 +179,18 @@ class TestMethods: assert cfg["regularization"] > 0 assert cfg["refinement_passes"] >= 2 + def test_qwen38_e03_is_single_variable_e02_attenuation(self): + e02 = METHODS["qwen38_e02"] + e03 = METHODS["qwen38_e03"] + changed = { + key for key in set(e02) | set(e03) + if e02.get(key) != e03.get(key) + } + + assert changed == {"label", "description", "regularization"} + assert e02["regularization"] == 0.0 + assert e03["regularization"] == 0.1 + def test_aggressive_full_gabliteration(self): cfg = METHODS["aggressive"] assert cfg["n_directions"] >= 8 diff --git a/tests/test_experiment_protocol.py b/tests/test_experiment_protocol.py index 6795cc1..3dd0054 100644 --- a/tests/test_experiment_protocol.py +++ b/tests/test_experiment_protocol.py @@ -50,9 +50,11 @@ def test_qwen38_experiment_evaluation_partitions_are_fail_closed(): assert qwen38_evaluation_pairs(split, "E01") is split.test assert qwen38_evaluation_pairs(split, "E02") is split.tune + assert qwen38_evaluation_pairs(split, "E03") is split.tune assert not set(split.test) & set(qwen38_evaluation_pairs(split, "E02")) + assert not set(split.test) & set(qwen38_evaluation_pairs(split, "E03")) with pytest.raises(ValueError, match="unregistered Qwen3.8 experiment"): - qwen38_evaluation_pairs(split, "E03") + qwen38_evaluation_pairs(split, "E04") def test_qwen38_split_rejects_wrong_size_and_duplicate_pairs():