diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 78d0475..9a0d2c2 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -4,7 +4,7 @@ ## Tests -- [ ] `python -m ruff check --select F obliteratus tests scripts/check_coverage_thresholds.py scripts/gemma4_12b_recursive_loop.py` +- [ ] `python -m ruff check --select F app.py obliteratus tests scripts` - [ ] `python -m ruff check --select E501 --statistics obliteratus tests scripts/gemma4_12b_recursive_loop.py` reviewed as known non-blocking line-length debt - [ ] `python -c 'import obliteratus; print(getattr(obliteratus, "__version__", "import ok"))'` - [ ] `python -m obliteratus --help` diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e25e958..e38da56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -198,22 +198,7 @@ jobs: - name: Enforce Ruff F gate run: >- - python -m ruff check --select F app.py obliteratus tests - scripts/check_coverage_thresholds.py - scripts/check_mutation_score.py - scripts/check_quality_policy.py - scripts/check_mutation_targets.py - scripts/prepare_mutation_coverage.py - scripts/run_prepared_mutmut.py - scripts/mutmut_coverage_sitecustomize/sitecustomize.py - scripts/check_conditional_policy.py - scripts/check_test_risk_map.py - scripts/conditional_gate_summary.py - scripts/run_conditional_gate.py - scripts/run_repeat_gate.py - scripts/write_test_evidence.py - scripts/check_supply_chain_policy.py - scripts/gemma4_12b_recursive_loop.py + python -m ruff check --select F app.py obliteratus tests scripts - name: Report E501 legacy baseline if: always() diff --git a/scripts/gemma4_aspa_sweep.py b/scripts/gemma4_aspa_sweep.py index 739bb4c..96b2f6e 100644 --- a/scripts/gemma4_aspa_sweep.py +++ b/scripts/gemma4_aspa_sweep.py @@ -9,7 +9,6 @@ refusal geometry removal with low KL (0.094). """ import argparse -import copy import gc import json import time @@ -139,7 +138,6 @@ def mmlu_pro_val70(model, tok, device): if gold_ids: probs = F.softmax(logits.float(), dim=-1) - gold_prob = max(probs[tid].item() for tid in gold_ids) best_prob = 0.0 best_letter = "?" for letter in LETTERS[:len(row["options"])]: diff --git a/scripts/gemma4_gradient_aspa.py b/scripts/gemma4_gradient_aspa.py index 1a4462a..4ca820b 100644 --- a/scripts/gemma4_gradient_aspa.py +++ b/scripts/gemma4_gradient_aspa.py @@ -9,7 +9,6 @@ This should let us recover MORE MMLU than uniform blending while keeping refusals at absolute zero. """ -import argparse import gc import json import time @@ -197,7 +196,7 @@ def main(): out_dir = Path("runs/gemma4-12b-surgery/gradient_aspa") out_dir.mkdir(parents=True, exist_ok=True) - print(f"Loading tokenizer and v1 model...", flush=True) + print("Loading tokenizer and v1 model...", flush=True) tok = AutoTokenizer.from_pretrained(v1_model_path, trust_remote_code=True) model = AutoModelForCausalLM.from_pretrained( v1_model_path, torch_dtype=torch.bfloat16, trust_remote_code=True diff --git a/scripts/gemma4_mmlu_head2head.py b/scripts/gemma4_mmlu_head2head.py index 2fcfb25..be4d83f 100644 --- a/scripts/gemma4_mmlu_head2head.py +++ b/scripts/gemma4_mmlu_head2head.py @@ -233,7 +233,7 @@ def main(): p_pool = (results["v2_test"]["correct"] + results["stock_test"]["correct"]) / (2 * n) se = math.sqrt(2 * p_pool * (1 - p_pool) / n) if p_pool > 0 and p_pool < 1 else 1 z = (p1 - p2) / se if se > 0 else 0 - print(f"\n Statistical test (test-500):", flush=True) + print("\n Statistical test (test-500):", flush=True) print(f" Z-score: {z:.3f} (|z| < 1.96 = NOT significant at p<0.05)", flush=True) print(f" Conclusion: {'PARITY CONFIRMED' if abs(z) < 1.96 else 'SIGNIFICANT DIFFERENCE'}", flush=True) diff --git a/scripts/gemma4_mmlu_pro_likelihood.py b/scripts/gemma4_mmlu_pro_likelihood.py index 07c62d8..542ca67 100644 --- a/scripts/gemma4_mmlu_pro_likelihood.py +++ b/scripts/gemma4_mmlu_pro_likelihood.py @@ -21,8 +21,6 @@ from gemma4_hard_tier_bench import ( apply_chat, load_model, load_tokenizer, - resolve_device, - resolve_dtype, ) diff --git a/scripts/gemma4_public_refusal_bench.py b/scripts/gemma4_public_refusal_bench.py index 1af534d..aa489a4 100644 --- a/scripts/gemma4_public_refusal_bench.py +++ b/scripts/gemma4_public_refusal_bench.py @@ -20,19 +20,15 @@ from datasets import load_dataset from gemma4_hard_tier_bench import ( DEFAULT_SYSTEM_PROMPT, - apply_chat, generate_one, load_model, load_tokenizer, repetition_flag, - resolve_device, - resolve_dtype, thinking_tag_flag, unique_word_ratio, - words, ) from obliteratus.evaluation.advanced_metrics import _is_refusal_detailed -from obliteratus.hard_negative import classify_refusal_reason, prompt_hash +from obliteratus.hard_negative import prompt_hash BENCHMARK_CHOICES = ( diff --git a/scripts/gemma4_refusal_sniper.py b/scripts/gemma4_refusal_sniper.py index 590b231..a4a043e 100644 --- a/scripts/gemma4_refusal_sniper.py +++ b/scripts/gemma4_refusal_sniper.py @@ -14,7 +14,6 @@ Strategy: import gc import json -import time from pathlib import Path import torch diff --git a/scripts/gemma4_stock_mmlu.py b/scripts/gemma4_stock_mmlu.py index 1d06e7c..751b0c0 100644 --- a/scripts/gemma4_stock_mmlu.py +++ b/scripts/gemma4_stock_mmlu.py @@ -125,7 +125,7 @@ def run_mmlu_pro(model, tok, device, split="validation", max_n=None, label=""): print(f"\n RESULT: {correct}/{len(rows)} ({accuracy:.1%})", flush=True) # Per-category breakdown - print(f"\n Per-category breakdown:", flush=True) + print("\n Per-category breakdown:", flush=True) for cat, stats in sorted(per_category.items()): cat_acc = stats["correct"] / stats["total"] if stats["total"] > 0 else 0 print(f" {cat}: {stats['correct']}/{stats['total']} ({cat_acc:.1%})", flush=True) diff --git a/scripts/gemma4_v2_final_gate.py b/scripts/gemma4_v2_final_gate.py index edbf9c9..915fefa 100644 --- a/scripts/gemma4_v2_final_gate.py +++ b/scripts/gemma4_v2_final_gate.py @@ -192,7 +192,7 @@ def main(): print(f"Model: {V2_MODEL}", flush=True) print(f"{'='*60}", flush=True) - print(f"\nLoading model...", flush=True) + print("\nLoading model...", flush=True) tok = AutoTokenizer.from_pretrained(V2_MODEL, trust_remote_code=True) t0 = time.time() model = AutoModelForCausalLM.from_pretrained( @@ -231,11 +231,11 @@ def main(): print(f"\n{'='*60}", flush=True) print("V2 FINAL VERDICT", flush=True) print(f"{'='*60}", flush=True) - print(f" Model: Step Gradient ASPA (55%/20%, layers 22-31/32-46)", flush=True) + print(" Model: Step Gradient ASPA (55%/20%, layers 22-31/32-46)", flush=True) print(f" Coherence: {coherence['passed']}/{coherence['total']} pass", flush=True) print(f" MMLU-Pro: {mmlu['correct']}/{mmlu['total']} ({mmlu['accuracy']:.1%})", flush=True) print(f" Refusal: {refusal['refused']}/{refusal['total']} ({refusal['rate']:.1%})", flush=True) - print(f" Stock MMLU: 46/70 (65.7%)", flush=True) + print(" Stock MMLU: 46/70 (65.7%)", flush=True) print(f" MMLU delta: {mmlu['correct'] - 46:+d} vs stock", flush=True) if refusal['refused'] == 0 and mmlu['accuracy'] >= 0.60 and coherence['passed'] >= 5: diff --git a/scripts/gemma4_v2_full_bench.py b/scripts/gemma4_v2_full_bench.py index 15699d1..33af41b 100644 --- a/scripts/gemma4_v2_full_bench.py +++ b/scripts/gemma4_v2_full_bench.py @@ -227,7 +227,7 @@ def main(): print(f"\n{'='*60}", flush=True) print("V2 BENCHMARK SUMMARY", flush=True) print(f"{'='*60}", flush=True) - print(f" Model: gamma=0.40 ASPA candidate", flush=True) + print(" Model: gamma=0.40 ASPA candidate", flush=True) print(f" Coherence: {coherence['passed']}/{coherence['total']} pass", flush=True) print(f" MMLU-Pro: {mmlu['correct']}/{mmlu['total']} ({mmlu['accuracy']:.1%})", flush=True) print(f" Refusal: {refusal['refused']}/{refusal['total']} ({refusal['rate']:.1%})", flush=True) diff --git a/tests/test_ci_policy.py b/tests/test_ci_policy.py index 4d1d3d8..795d8a3 100644 --- a/tests/test_ci_policy.py +++ b/tests/test_ci_policy.py @@ -48,6 +48,15 @@ def test_actionlint_version_and_checksum_match_manifest(): assert f'ACTIONLINT_SHA256: "{pin.removeprefix("sha256:")}"' in workflow +def test_ci_ruff_f_gate_covers_the_entire_scripts_tree(): + workflow = WORKFLOW.read_text(encoding="utf-8") + ruff_step = workflow.split(" - name: Enforce Ruff F gate\n", maxsplit=1)[1].split( + " - name:", maxsplit=1, + )[0] + + assert "python -m ruff check --select F app.py obliteratus tests scripts" in ruff_step + + def test_uv_and_gitleaks_pins_match_manifest(): workflow = WORKFLOW.read_text(encoding="utf-8") entries = _manifest_entries()