ci: enforce Ruff across all scripts

This commit is contained in:
Joseph Magly
2026-08-16 03:49:09 -04:00
parent aa182cc448
commit 3574800094
12 changed files with 19 additions and 35 deletions
+1 -1
View File
@@ -4,7 +4,7 @@
## Tests
- [ ] `python -m ruff check --select F obliteratus tests scripts/check_coverage_thresholds.py scripts/gemma4_12b_recursive_loop.py`
- [ ] `python -m ruff check --select F app.py obliteratus tests scripts`
- [ ] `python -m ruff check --select E501 --statistics obliteratus tests scripts/gemma4_12b_recursive_loop.py` reviewed as known non-blocking line-length debt
- [ ] `python -c 'import obliteratus; print(getattr(obliteratus, "__version__", "import ok"))'`
- [ ] `python -m obliteratus --help`
+1 -16
View File
@@ -198,22 +198,7 @@ jobs:
- name: Enforce Ruff F gate
run: >-
python -m ruff check --select F app.py obliteratus tests
scripts/check_coverage_thresholds.py
scripts/check_mutation_score.py
scripts/check_quality_policy.py
scripts/check_mutation_targets.py
scripts/prepare_mutation_coverage.py
scripts/run_prepared_mutmut.py
scripts/mutmut_coverage_sitecustomize/sitecustomize.py
scripts/check_conditional_policy.py
scripts/check_test_risk_map.py
scripts/conditional_gate_summary.py
scripts/run_conditional_gate.py
scripts/run_repeat_gate.py
scripts/write_test_evidence.py
scripts/check_supply_chain_policy.py
scripts/gemma4_12b_recursive_loop.py
python -m ruff check --select F app.py obliteratus tests scripts
- name: Report E501 legacy baseline
if: always()
-2
View File
@@ -9,7 +9,6 @@ refusal geometry removal with low KL (0.094).
"""
import argparse
import copy
import gc
import json
import time
@@ -139,7 +138,6 @@ def mmlu_pro_val70(model, tok, device):
if gold_ids:
probs = F.softmax(logits.float(), dim=-1)
gold_prob = max(probs[tid].item() for tid in gold_ids)
best_prob = 0.0
best_letter = "?"
for letter in LETTERS[:len(row["options"])]:
+1 -2
View File
@@ -9,7 +9,6 @@ This should let us recover MORE MMLU than uniform blending while
keeping refusals at absolute zero.
"""
import argparse
import gc
import json
import time
@@ -197,7 +196,7 @@ def main():
out_dir = Path("runs/gemma4-12b-surgery/gradient_aspa")
out_dir.mkdir(parents=True, exist_ok=True)
print(f"Loading tokenizer and v1 model...", flush=True)
print("Loading tokenizer and v1 model...", flush=True)
tok = AutoTokenizer.from_pretrained(v1_model_path, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(
v1_model_path, torch_dtype=torch.bfloat16, trust_remote_code=True
+1 -1
View File
@@ -233,7 +233,7 @@ def main():
p_pool = (results["v2_test"]["correct"] + results["stock_test"]["correct"]) / (2 * n)
se = math.sqrt(2 * p_pool * (1 - p_pool) / n) if p_pool > 0 and p_pool < 1 else 1
z = (p1 - p2) / se if se > 0 else 0
print(f"\n Statistical test (test-500):", flush=True)
print("\n Statistical test (test-500):", flush=True)
print(f" Z-score: {z:.3f} (|z| < 1.96 = NOT significant at p<0.05)", flush=True)
print(f" Conclusion: {'PARITY CONFIRMED' if abs(z) < 1.96 else 'SIGNIFICANT DIFFERENCE'}", flush=True)
-2
View File
@@ -21,8 +21,6 @@ from gemma4_hard_tier_bench import (
apply_chat,
load_model,
load_tokenizer,
resolve_device,
resolve_dtype,
)
+1 -5
View File
@@ -20,19 +20,15 @@ from datasets import load_dataset
from gemma4_hard_tier_bench import (
DEFAULT_SYSTEM_PROMPT,
apply_chat,
generate_one,
load_model,
load_tokenizer,
repetition_flag,
resolve_device,
resolve_dtype,
thinking_tag_flag,
unique_word_ratio,
words,
)
from obliteratus.evaluation.advanced_metrics import _is_refusal_detailed
from obliteratus.hard_negative import classify_refusal_reason, prompt_hash
from obliteratus.hard_negative import prompt_hash
BENCHMARK_CHOICES = (
-1
View File
@@ -14,7 +14,6 @@ Strategy:
import gc
import json
import time
from pathlib import Path
import torch
+1 -1
View File
@@ -125,7 +125,7 @@ def run_mmlu_pro(model, tok, device, split="validation", max_n=None, label=""):
print(f"\n RESULT: {correct}/{len(rows)} ({accuracy:.1%})", flush=True)
# Per-category breakdown
print(f"\n Per-category breakdown:", flush=True)
print("\n Per-category breakdown:", flush=True)
for cat, stats in sorted(per_category.items()):
cat_acc = stats["correct"] / stats["total"] if stats["total"] > 0 else 0
print(f" {cat}: {stats['correct']}/{stats['total']} ({cat_acc:.1%})", flush=True)
+3 -3
View File
@@ -192,7 +192,7 @@ def main():
print(f"Model: {V2_MODEL}", flush=True)
print(f"{'='*60}", flush=True)
print(f"\nLoading model...", flush=True)
print("\nLoading model...", flush=True)
tok = AutoTokenizer.from_pretrained(V2_MODEL, trust_remote_code=True)
t0 = time.time()
model = AutoModelForCausalLM.from_pretrained(
@@ -231,11 +231,11 @@ def main():
print(f"\n{'='*60}", flush=True)
print("V2 FINAL VERDICT", flush=True)
print(f"{'='*60}", flush=True)
print(f" Model: Step Gradient ASPA (55%/20%, layers 22-31/32-46)", flush=True)
print(" Model: Step Gradient ASPA (55%/20%, layers 22-31/32-46)", flush=True)
print(f" Coherence: {coherence['passed']}/{coherence['total']} pass", flush=True)
print(f" MMLU-Pro: {mmlu['correct']}/{mmlu['total']} ({mmlu['accuracy']:.1%})", flush=True)
print(f" Refusal: {refusal['refused']}/{refusal['total']} ({refusal['rate']:.1%})", flush=True)
print(f" Stock MMLU: 46/70 (65.7%)", flush=True)
print(" Stock MMLU: 46/70 (65.7%)", flush=True)
print(f" MMLU delta: {mmlu['correct'] - 46:+d} vs stock", flush=True)
if refusal['refused'] == 0 and mmlu['accuracy'] >= 0.60 and coherence['passed'] >= 5:
+1 -1
View File
@@ -227,7 +227,7 @@ def main():
print(f"\n{'='*60}", flush=True)
print("V2 BENCHMARK SUMMARY", flush=True)
print(f"{'='*60}", flush=True)
print(f" Model: gamma=0.40 ASPA candidate", flush=True)
print(" Model: gamma=0.40 ASPA candidate", flush=True)
print(f" Coherence: {coherence['passed']}/{coherence['total']} pass", flush=True)
print(f" MMLU-Pro: {mmlu['correct']}/{mmlu['total']} ({mmlu['accuracy']:.1%})", flush=True)
print(f" Refusal: {refusal['refused']}/{refusal['total']} ({refusal['rate']:.1%})", flush=True)
+9
View File
@@ -48,6 +48,15 @@ def test_actionlint_version_and_checksum_match_manifest():
assert f'ACTIONLINT_SHA256: "{pin.removeprefix("sha256:")}"' in workflow
def test_ci_ruff_f_gate_covers_the_entire_scripts_tree():
workflow = WORKFLOW.read_text(encoding="utf-8")
ruff_step = workflow.split(" - name: Enforce Ruff F gate\n", maxsplit=1)[1].split(
" - name:", maxsplit=1,
)[0]
assert "python -m ruff check --select F app.py obliteratus tests scripts" in ruff_step
def test_uv_and_gitleaks_pins_match_manifest():
workflow = WORKFLOW.read_text(encoding="utf-8")
entries = _manifest_entries()