ci: enforce Ruff across all scripts

This commit is contained in:
Joseph Magly
2026-08-16 03:49:09 -04:00
parent aa182cc448
commit 3574800094
12 changed files with 19 additions and 35 deletions
-2
View File
@@ -9,7 +9,6 @@ refusal geometry removal with low KL (0.094).
"""
import argparse
import copy
import gc
import json
import time
@@ -139,7 +138,6 @@ def mmlu_pro_val70(model, tok, device):
if gold_ids:
probs = F.softmax(logits.float(), dim=-1)
gold_prob = max(probs[tid].item() for tid in gold_ids)
best_prob = 0.0
best_letter = "?"
for letter in LETTERS[:len(row["options"])]:
+1 -2
View File
@@ -9,7 +9,6 @@ This should let us recover MORE MMLU than uniform blending while
keeping refusals at absolute zero.
"""
import argparse
import gc
import json
import time
@@ -197,7 +196,7 @@ def main():
out_dir = Path("runs/gemma4-12b-surgery/gradient_aspa")
out_dir.mkdir(parents=True, exist_ok=True)
print(f"Loading tokenizer and v1 model...", flush=True)
print("Loading tokenizer and v1 model...", flush=True)
tok = AutoTokenizer.from_pretrained(v1_model_path, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(
v1_model_path, torch_dtype=torch.bfloat16, trust_remote_code=True
+1 -1
View File
@@ -233,7 +233,7 @@ def main():
p_pool = (results["v2_test"]["correct"] + results["stock_test"]["correct"]) / (2 * n)
se = math.sqrt(2 * p_pool * (1 - p_pool) / n) if p_pool > 0 and p_pool < 1 else 1
z = (p1 - p2) / se if se > 0 else 0
print(f"\n Statistical test (test-500):", flush=True)
print("\n Statistical test (test-500):", flush=True)
print(f" Z-score: {z:.3f} (|z| < 1.96 = NOT significant at p<0.05)", flush=True)
print(f" Conclusion: {'PARITY CONFIRMED' if abs(z) < 1.96 else 'SIGNIFICANT DIFFERENCE'}", flush=True)
-2
View File
@@ -21,8 +21,6 @@ from gemma4_hard_tier_bench import (
apply_chat,
load_model,
load_tokenizer,
resolve_device,
resolve_dtype,
)
+1 -5
View File
@@ -20,19 +20,15 @@ from datasets import load_dataset
from gemma4_hard_tier_bench import (
DEFAULT_SYSTEM_PROMPT,
apply_chat,
generate_one,
load_model,
load_tokenizer,
repetition_flag,
resolve_device,
resolve_dtype,
thinking_tag_flag,
unique_word_ratio,
words,
)
from obliteratus.evaluation.advanced_metrics import _is_refusal_detailed
from obliteratus.hard_negative import classify_refusal_reason, prompt_hash
from obliteratus.hard_negative import prompt_hash
BENCHMARK_CHOICES = (
-1
View File
@@ -14,7 +14,6 @@ Strategy:
import gc
import json
import time
from pathlib import Path
import torch
+1 -1
View File
@@ -125,7 +125,7 @@ def run_mmlu_pro(model, tok, device, split="validation", max_n=None, label=""):
print(f"\n RESULT: {correct}/{len(rows)} ({accuracy:.1%})", flush=True)
# Per-category breakdown
print(f"\n Per-category breakdown:", flush=True)
print("\n Per-category breakdown:", flush=True)
for cat, stats in sorted(per_category.items()):
cat_acc = stats["correct"] / stats["total"] if stats["total"] > 0 else 0
print(f" {cat}: {stats['correct']}/{stats['total']} ({cat_acc:.1%})", flush=True)
+3 -3
View File
@@ -192,7 +192,7 @@ def main():
print(f"Model: {V2_MODEL}", flush=True)
print(f"{'='*60}", flush=True)
print(f"\nLoading model...", flush=True)
print("\nLoading model...", flush=True)
tok = AutoTokenizer.from_pretrained(V2_MODEL, trust_remote_code=True)
t0 = time.time()
model = AutoModelForCausalLM.from_pretrained(
@@ -231,11 +231,11 @@ def main():
print(f"\n{'='*60}", flush=True)
print("V2 FINAL VERDICT", flush=True)
print(f"{'='*60}", flush=True)
print(f" Model: Step Gradient ASPA (55%/20%, layers 22-31/32-46)", flush=True)
print(" Model: Step Gradient ASPA (55%/20%, layers 22-31/32-46)", flush=True)
print(f" Coherence: {coherence['passed']}/{coherence['total']} pass", flush=True)
print(f" MMLU-Pro: {mmlu['correct']}/{mmlu['total']} ({mmlu['accuracy']:.1%})", flush=True)
print(f" Refusal: {refusal['refused']}/{refusal['total']} ({refusal['rate']:.1%})", flush=True)
print(f" Stock MMLU: 46/70 (65.7%)", flush=True)
print(" Stock MMLU: 46/70 (65.7%)", flush=True)
print(f" MMLU delta: {mmlu['correct'] - 46:+d} vs stock", flush=True)
if refusal['refused'] == 0 and mmlu['accuracy'] >= 0.60 and coherence['passed'] >= 5:
+1 -1
View File
@@ -227,7 +227,7 @@ def main():
print(f"\n{'='*60}", flush=True)
print("V2 BENCHMARK SUMMARY", flush=True)
print(f"{'='*60}", flush=True)
print(f" Model: gamma=0.40 ASPA candidate", flush=True)
print(" Model: gamma=0.40 ASPA candidate", flush=True)
print(f" Coherence: {coherence['passed']}/{coherence['total']} pass", flush=True)
print(f" MMLU-Pro: {mmlu['correct']}/{mmlu['total']} ({mmlu['accuracy']:.1%})", flush=True)
print(f" Refusal: {refusal['refused']}/{refusal['total']} ({refusal['rate']:.1%})", flush=True)