mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 22:14:25 +01:00
0.1: eval:pass-rates --headroom, --reds and --run report views; 85% headroom alarm in --gate
A new pure module, scripts/lib/eval-history.ts, reports per case the slowest armed session against the timeout it armed (censored when it timed out, insufficient below 3 samples, unknown without a session ledger, with the case wall shown separately as an upper bound), the census critical path from the slice jobs' timings, the verdict red ledger per census by failure class and machine cause with shrunk per-case rates, and the all-green probability (an approximation: the product of 1 - p_i, with an interval). --run <id> lists one census's reds with their values and history and downloads only the slice artifacts those reds name, reporting any over 64 MB instead of skipping them. eval-flake-rank.ts only routes flags: --help works offline, an unknown flag or case id exits 2 before any fetch, --case narrows the JSON and the alarms, every view prints its run/branch scope, and --gate also alarms on a session above HEADROOM_FAIL (85%) of its budget.
This commit is contained in:
garrytan committed
1 parent
d725d94ea3
commit
c8115df6dd
4 files changed
+604
-16
No files matched your search
+45
-16
@@ -36,6 +36,8 @@
|
||||
* bun run eval:pass-rates --dir <path> # local eval dirs / downloaded artifacts (repeatable)
|
||||
* bun run eval:pass-rates --backfill # also import legacy slice artifacts, labeled pre-policy
|
||||
* bun run eval:pass-rates --json | --gate
|
||||
* bun run eval:pass-rates --headroom | --reds | --run <id> # report views (scripts/lib/eval-history.ts)
|
||||
* bun run eval:pass-rates --help # every flag, no network
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
@@ -51,6 +53,8 @@ import { CASE_QUARANTINE, EVAL_POLICY } from '../test/helpers/periodic-exclude-d
|
||||
import { matchGlob } from '../test/helpers/test-selection';
|
||||
import { CASE_TEST_NAMES } from './test-paid-shards';
|
||||
import { resolveStateRoot } from '../lib/state-root';
|
||||
import { GH_JOBS, PASS_RATES_USAGE, criticalPath, formatCriticalPath, formatHeadroom, formatRedLedger, headroom, headroomAlarms, parsePassRatesArgs,
|
||||
redLedger, triageRun, type CriticalPath } from './lib/eval-history';
|
||||
|
||||
interface TestSeries {
|
||||
name: string;
|
||||
@@ -316,7 +320,7 @@ export function holmRejections(pValues: number[], alpha: number): Set<number> {
|
||||
|
||||
export type PassRateLabel = 'INCONCLUSIVE' | 'BROKEN' | 'FLAKY' | 'FAILING' | 'PASSING';
|
||||
export type AlarmKind = 'drift' | 'rule-as-behavior' | 'regression' | 'quarantine-exit' | 'quarantine-expired'
|
||||
| 'quarantine-cap' | 'quarantine-invalid';
|
||||
| 'quarantine-cap' | 'quarantine-invalid' | 'headroom';
|
||||
|
||||
/** The EVAL_POLICY fields pass-rates reads (structural, so tests can vary them). */
|
||||
export interface PassRatePolicy {
|
||||
@@ -639,24 +643,22 @@ function repoSlug(): string {
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const argv = process.argv.slice(2);
|
||||
const flag = (name: string) => { const index = argv.indexOf(name); return index === -1 ? undefined : argv[index + 1]; };
|
||||
const dirs = argv.flatMap((arg, index) => arg === '--dir' && argv[index + 1] ? [argv[index + 1]!] : []);
|
||||
const asJson = argv.includes('--json');
|
||||
const gate = argv.includes('--gate');
|
||||
const backfill = argv.includes('--backfill');
|
||||
const caseFilter = flag('--case');
|
||||
const runsLimit = Number(flag('--runs')) || 10;
|
||||
const sinceDays = Number(flag('--since-days')) || 60;
|
||||
const repo = flag('--repo') ?? repoSlug();
|
||||
const workflow = flag('--workflow') ?? 'evals-periodic.yml';
|
||||
const branch = flag('--branch') ?? gitOutput(['rev-parse', '--abbrev-ref', 'HEAD']) ?? 'main';
|
||||
const parsed = parsePassRatesArgs(process.argv.slice(2), id => Object.hasOwn(E2E_TIERS, id) || Object.hasOwn(LLM_JUDGE_TOUCHFILES, id)
|
||||
|| (/^test\/[\w./-]+\.test\.ts$/.test(id) && fs.existsSync(id)));
|
||||
if ('help' in parsed) { console.log(PASS_RATES_USAGE); process.exit(0); }
|
||||
if ('error' in parsed) { console.error(`eval:pass-rates: ${parsed.error}\n${PASS_RATES_USAGE}`); process.exit(2); }
|
||||
const { dirs, json: asJson, gate, backfill, caseFilter, runs: runsLimit, sinceDays, view } = parsed;
|
||||
const repo = parsed.repo ?? repoSlug();
|
||||
const workflow = parsed.workflow ?? 'evals-periodic.yml';
|
||||
const branch = parsed.branch ?? gitOutput(['rev-parse', '--abbrev-ref', 'HEAD']) ?? 'main';
|
||||
|
||||
const records: TrialRecord[] = [];
|
||||
const unattributed = new Set<string>();
|
||||
const errors: string[] = [];
|
||||
let historyError: string | null = null;
|
||||
let weeklyRuns: string[] | undefined;
|
||||
let fetched: WeeklyRun[] = [];
|
||||
const cacheDir = path.join(path.resolve(resolveStateRoot()), 'eval-pass-rates-cache', repo.replace('/', '-'));
|
||||
|
||||
const manualReviews: string[] = [];
|
||||
const importDir = (dir: string, run: { run_id: string; sha?: string; timestamp?: string } | undefined, legacyDays: number) => {
|
||||
@@ -674,8 +676,9 @@ if (import.meta.main) {
|
||||
} else {
|
||||
try {
|
||||
const runs = listWeeklyRuns({ repo, workflow, branches: [...new Set([branch, 'main'])], limit: runsLimit });
|
||||
if (parsed.runId !== undefined && !runs.some(run => run.id === parsed.runId)) runs.push(GH_JOBS.getRun(repo, parsed.runId));
|
||||
weeklyRuns = runs.map(run => run.createdAt);
|
||||
const cacheDir = path.join(path.resolve(resolveStateRoot()), 'eval-pass-rates-cache', repo.replace('/', '-'));
|
||||
fetched = runs;
|
||||
const match = backfill
|
||||
? (name: string) => name.startsWith('trial-outcomes') || /^(paid-slice-\d+|gate-census-\d+)(-a\d+)?$/.test(name)
|
||||
: (name: string) => name.startsWith('trial-outcomes');
|
||||
@@ -688,11 +691,37 @@ if (import.meta.main) {
|
||||
}
|
||||
}
|
||||
|
||||
const report = analyzePassRates(records, { weeklyRuns, unattributed: [...unattributed].sort(), errors, manualReviews });
|
||||
const scope = dirs.length ? `local dirs ${dirs.join(', ')}` : `${repo} ${workflow} on ${[...new Set([branch, 'main'])].join(' + ')}, `
|
||||
+ `last ${runsLimit} completed run(s) per branch: ${fetched.map(run => run.id).join(', ') || 'none'}`;
|
||||
const scoped = caseFilter ? records.filter(record => record.case === caseFilter) : records;
|
||||
const caseHeadroom = headroom(scoped);
|
||||
if (view !== 'rates') {
|
||||
const critical: CriticalPath[] = [];
|
||||
if (view !== 'reds' && !dirs.length) {
|
||||
for (const run of fetched.filter(run => view === 'headroom' || run.id === parsed.runId)) {
|
||||
try { critical.push(criticalPath(String(run.id), GH_JOBS.listJobs(repo, run.id))); } catch { critical.push({ run: String(run.id), jobs: 0, slowest: null, wallMs: null }); }
|
||||
}
|
||||
}
|
||||
const lines = view === 'headroom' ? formatHeadroom(caseHeadroom, critical) : view === 'reds' ? formatRedLedger(redLedger(scoped))
|
||||
: historyError ? [] : triageRun({ repo, run: fetched.find(run => run.id === parsed.runId)!, records, caseFilter, cacheDir, fetcher: GH_HISTORY,
|
||||
download: (match, maxBytes) => downloadRunArtifacts({ repo, run: fetched.find(run => run.id === parsed.runId)!, match, cacheDir, maxBytes }) })
|
||||
.concat(formatCriticalPath(critical));
|
||||
if (asJson) console.log(JSON.stringify({ scope, historyError, view, lines }, null, 2));
|
||||
else {
|
||||
console.log(`scope: ${scope}`);
|
||||
if (historyError) console.log(`history unavailable (${historyError})`);
|
||||
for (const line of lines) console.log(line);
|
||||
}
|
||||
process.exit(historyError ? 1 : 0);
|
||||
}
|
||||
const full = analyzePassRates(records, { weeklyRuns, unattributed: [...unattributed].sort(), errors, manualReviews });
|
||||
const alarms = [...full.alarms, ...(gate ? headroomAlarms(caseHeadroom) : [])].filter(alarm => !caseFilter || alarm.case === caseFilter);
|
||||
const report = { ...full, cases: full.cases.filter(entry => !caseFilter || entry.case === caseFilter), alarms };
|
||||
const ledger = readFreeLedger();
|
||||
if (asJson) {
|
||||
console.log(JSON.stringify({ repo, workflow, branch, dirs, historyError, ...report, freeLedger: ledger }, null, 2));
|
||||
console.log(JSON.stringify({ repo, workflow, branch, scope, dirs, historyError, ...report, freeLedger: ledger }, null, 2));
|
||||
} else {
|
||||
console.log(`scope: ${scope}`);
|
||||
if (historyError) console.log(`pass-rates: history unavailable (${historyError}); every label below is INCONCLUSIVE`);
|
||||
console.log(formatPassRates(report, { caseFilter }));
|
||||
if (ledger.length > 0) {
|
||||
|
||||
@@ -0,0 +1,350 @@
|
||||
/**
|
||||
* eval:pass-rates report views over stored trial records (plan 0.1):
|
||||
* session headroom (--headroom), the verdict red ledger with the all-green
|
||||
* probability (--reds), and one census's triage view (--run <id>). Pure
|
||||
* functions plus the run-scoped fetches they need; scripts/eval-flake-rank.ts
|
||||
* parses flags, loads history and dispatches here.
|
||||
*
|
||||
* Headroom is per armed session (session ledger, `sessions[]` on a record):
|
||||
* elapsed over the timeout it armed. A case's headroom is its slowest
|
||||
* session; a timed-out sample is censored (its true duration is unknown).
|
||||
* Records without sessions are unknown, never scored; their case wall
|
||||
* (JUnit or shard clock) is shown separately as an upper bound.
|
||||
*
|
||||
* The all-green probability is an approximation that assumes independent
|
||||
* verdicts: P(green) = Π (1 - p_i) over the cases a census verdicts, where
|
||||
* p_i = (reds_i + K * p̄) / (verdicts_i + K) shrinks each case's red rate
|
||||
* toward the pooled rate p̄. Its interval recomputes the same product with p̄
|
||||
* replaced by the pooled rate's 95% Wilson bounds.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { panelVerdict, type TrialOutcomeRecord } from '../../test/helpers/eval-store';
|
||||
import { HEADROOM_FAIL, HEADROOM_WARN } from '../../test/helpers/eval-budgets';
|
||||
import { sanitizePublishedText } from './published-text';
|
||||
import type { HistoryFetcher, WeeklyRun } from '../eval-flake-rank';
|
||||
import { CASE_SHARDED_FILES, trialShardKey } from './paid-cases';
|
||||
import { shardSlug } from '../test-paid-shards';
|
||||
|
||||
/** Below this many session samples a case's headroom is insufficient, never a pass. */
|
||||
export const HEADROOM_MIN_SAMPLES = 3;
|
||||
/** Pseudo-verdicts of pooled rate added to each case's red rate. */
|
||||
export const RED_RATE_SHRINK = 4;
|
||||
/** Largest slice artifact --run downloads. */
|
||||
export const EVIDENCE_MAX_BYTES = 64 * 1024 * 1024;
|
||||
|
||||
type Rec = TrialOutcomeRecord;
|
||||
const pct = (x: number) => `${Math.round(x * 1000) / 10}%`;
|
||||
const secs = (ms: number) => `${Math.round(ms / 1000)}s`;
|
||||
|
||||
// --- Headroom ---
|
||||
|
||||
export interface SessionHeadroom { key: string; samples: number; censored: number; maxRatio: number; maxElapsedMs: number; budgetMs: number }
|
||||
export interface CaseHeadroom {
|
||||
case: string;
|
||||
status: 'over' | 'warn' | 'ok' | 'insufficient' | 'unknown';
|
||||
sessions: SessionHeadroom[];
|
||||
/** Slowest session's worst ratio, when any session was recorded. */
|
||||
worst?: SessionHeadroom;
|
||||
/** Largest case wall (JUnit or shard clock) over the window: an upper bound, never a session clock. */
|
||||
caseWallMs: number;
|
||||
runs: number;
|
||||
}
|
||||
|
||||
export function headroom(records: readonly Rec[]): CaseHeadroom[] {
|
||||
const byCase = new Map<string, Rec[]>();
|
||||
for (const r of records) if (r.outcome !== 'skipped' && r.execution !== 'reused') byCase.set(r.case, [...(byCase.get(r.case) ?? []), r]);
|
||||
return [...byCase.entries()].map(([id, rs]) => {
|
||||
const sessions = new Map<string, SessionHeadroom>();
|
||||
for (const s of rs.flatMap(r => r.sessions ?? [])) {
|
||||
if (!s.budget_ms) continue;
|
||||
const key = s.key;
|
||||
const entry = sessions.get(key) ?? { key, samples: 0, censored: 0, maxRatio: 0, maxElapsedMs: 0, budgetMs: s.budget_ms };
|
||||
entry.samples++;
|
||||
if (s.end === 'session_timeout' || s.end === 'observer_timeout') entry.censored++;
|
||||
const ratio = s.elapsed_ms / s.budget_ms;
|
||||
if (ratio > entry.maxRatio) Object.assign(entry, { maxRatio: ratio, maxElapsedMs: s.elapsed_ms, budgetMs: s.budget_ms });
|
||||
sessions.set(key, entry);
|
||||
}
|
||||
const list = [...sessions.values()].sort((a, b) => b.maxRatio - a.maxRatio);
|
||||
const worst = list[0];
|
||||
const status: CaseHeadroom['status'] = !worst ? 'unknown' : worst.samples < HEADROOM_MIN_SAMPLES ? 'insufficient'
|
||||
: worst.maxRatio > HEADROOM_FAIL ? 'over' : worst.maxRatio > HEADROOM_WARN ? 'warn' : 'ok';
|
||||
return { case: id, status, sessions: list, ...(worst ? { worst } : {}), caseWallMs: Math.max(0, ...rs.map(r => r.duration_ms)),
|
||||
runs: new Set(rs.map(r => r.run_id ?? 'local')).size };
|
||||
}).sort((a, b) => (b.worst?.maxRatio ?? -1) - (a.worst?.maxRatio ?? -1) || b.caseWallMs - a.caseWallMs);
|
||||
}
|
||||
|
||||
/** --gate alarms: a case whose slowest session ran above HEADROOM_FAIL of its armed budget. */
|
||||
export function headroomAlarms(cases: readonly CaseHeadroom[]): Array<{ kind: 'headroom'; case: string; message: string }> {
|
||||
return cases.filter(c => c.status === 'over').map(c => ({ kind: 'headroom' as const, case: c.case,
|
||||
message: `${c.case} session ${c.worst!.key}: max ${secs(c.worst!.maxElapsedMs)} of ${secs(c.worst!.budgetMs)} (${pct(c.worst!.maxRatio)})`
|
||||
+ `${c.worst!.censored ? `, ${c.worst!.censored} timed-out sample(s) censored` : ''} over ${c.worst!.samples} sample(s), above the ${pct(HEADROOM_FAIL)} cap. `
|
||||
+ 'Cut work in the skill or fixture; budgets are never raised (TESTING_INTERNALS "Timeout policy").' }));
|
||||
}
|
||||
|
||||
export function formatHeadroom(cases: readonly CaseHeadroom[], critical: readonly CriticalPath[] = []): string[] {
|
||||
const lines = [`headroom: slowest armed session per case (warn > ${pct(HEADROOM_WARN)}, alarm > ${pct(HEADROOM_FAIL)}; `
|
||||
+ `insufficient below ${HEADROOM_MIN_SAMPLES} samples; censored = timed out)`,
|
||||
' status session max/budget samples case wall (upper bound, not a session clock) case'];
|
||||
for (const c of cases) {
|
||||
const w = c.worst;
|
||||
const session = w ? `${pct(w.maxRatio)} ${secs(w.maxElapsedMs)}/${secs(w.budgetMs)}` : 'unknown (no session ledger)';
|
||||
lines.push(` ${c.status.padEnd(12)} ${session.padEnd(26)} ${(w ? `${w.samples}${w.censored ? ` (${w.censored} censored)` : ''}` : '-').padEnd(7)} `
|
||||
+ `${secs(c.caseWallMs).padEnd(45)} ${c.case}${w ? ` [${w.key}]` : ''}`);
|
||||
}
|
||||
return [...lines, ...formatCriticalPath(critical)];
|
||||
}
|
||||
|
||||
export function formatCriticalPath(critical: readonly CriticalPath[]): string[] {
|
||||
return critical.map(cp => `critical path ${cp.run}: ${cp.slowest ? `${cp.slowest.name} ${secs(cp.slowest.ms)}` : 'unknown'}`
|
||||
+ ` of ${cp.jobs} slice job(s), run wall ${cp.wallMs === null ? 'unknown' : secs(cp.wallMs)}${cp.files?.length ? `; slice files: ${cp.files.join(', ')}` : ''}`);
|
||||
}
|
||||
|
||||
// --- Critical path from runner slice timings ---
|
||||
|
||||
export interface JobTiming { name: string; startedAt: string | null; completedAt: string | null }
|
||||
export interface CriticalPath { run: string; jobs: number; slowest: { name: string; ms: number } | null; wallMs: number | null; files?: string[] }
|
||||
|
||||
const SLICE_JOB = /^(eval-slices|eval-codex-slices|gate-census) \((\d+)\)$/;
|
||||
|
||||
export function criticalPath(run: string, jobs: readonly JobTiming[], plan?: { entries: Array<{ file: string; slice?: number; status?: string }> }): CriticalPath {
|
||||
const timed = jobs.filter(j => SLICE_JOB.test(j.name) && j.startedAt && j.completedAt)
|
||||
.map(j => ({ name: j.name, ms: Date.parse(j.completedAt!) - Date.parse(j.startedAt!) }));
|
||||
const slowest = timed.sort((a, b) => b.ms - a.ms)[0] ?? null;
|
||||
const all = jobs.filter(j => j.startedAt && j.completedAt);
|
||||
const wallMs = all.length ? Math.max(...all.map(j => Date.parse(j.completedAt!))) - Math.min(...all.map(j => Date.parse(j.startedAt!))) : null;
|
||||
const slice = slowest ? Number(SLICE_JOB.exec(slowest.name)![2]) : null;
|
||||
const files = plan && slice !== null && slowest!.name.startsWith('gate-census')
|
||||
? plan.entries.filter(e => e.slice === slice && e.status === 'planned').map(e => e.file) : undefined;
|
||||
return { run, jobs: timed.length, slowest, wallMs, ...(files ? { files } : {}) };
|
||||
}
|
||||
|
||||
// --- Red ledger ---
|
||||
|
||||
export interface VerdictRed { run: string; lane: string; case: string; status: string; trials: Rec[] }
|
||||
export interface CensusReds { run: string; lane: string; verdicts: number; reds: VerdictRed[]; byClass: Record<string, number>; byCause: Record<string, number> }
|
||||
export interface RedLedger {
|
||||
censuses: CensusReds[];
|
||||
pooled: { verdicts: number; reds: number; rate: number; lo: number; hi: number };
|
||||
perCase: Array<{ case: string; verdicts: number; reds: number; rate: number }>;
|
||||
allGreen: { p: number; lo: number; hi: number; verdicts: number } | null;
|
||||
}
|
||||
|
||||
function wilson(k: number, n: number, z = 1.96): { lo: number; hi: number } {
|
||||
if (n === 0) return { lo: 0, hi: 1 };
|
||||
const p = k / n, d = 1 + z * z / n, c = p + z * z / (2 * n), m = z * Math.sqrt(p * (1 - p) / n + z * z / (4 * n * n));
|
||||
return { lo: Math.max(0, (c - m) / d), hi: Math.min(1, (c + m) / d) };
|
||||
}
|
||||
|
||||
/** One panelVerdict per case per census lane (earliest attempt), exactly as the report computes it. */
|
||||
export function verdictsOf(records: readonly Rec[]): Array<{ run: string; lane: string; case: string; status: string; failsLane: boolean; trials: Rec[] }> {
|
||||
const groups = new Map<string, Rec[]>();
|
||||
for (const r of records) {
|
||||
if (r.policy_version < 1 || r.execution === 'reused') continue;
|
||||
// A JUnit case is one verdict per shard file, as the report counts it; trial shards form one panel.
|
||||
const key = `${r.run_id ?? 'local'}\0${r.lane ?? r.tier}\0${r.case}\0${r.source === 'shard' ? '' : r.file}`;
|
||||
groups.set(key, [...(groups.get(key) ?? []), r]);
|
||||
}
|
||||
return [...groups.entries()].map(([key, rs]) => {
|
||||
const [run, lane, id] = key.split('\0') as [string, string, string, string];
|
||||
const attempt = Math.min(...rs.map(r => r.attempt));
|
||||
const trials = rs.filter(r => r.attempt === attempt);
|
||||
const v = panelVerdict({ case: id, kind: trials[0]!.kind, panel: trials[0]!.panel, quarantined: trials[0]!.quarantined,
|
||||
trials: trials.map(t => ({ trial: t.trial, outcome: t.outcome, attempt, ...(t.failure_class ? { failure_class: t.failure_class } : {}) })) });
|
||||
return { run, lane, case: id, status: v.status, failsLane: v.failsLane, trials };
|
||||
});
|
||||
}
|
||||
|
||||
export function redLedger(records: readonly Rec[]): RedLedger {
|
||||
const verdicts = verdictsOf(records).filter(v => v.status !== 'SKIPPED');
|
||||
const censusKey = (v: { run: string; lane: string }) => `${v.run}\0${v.lane}`;
|
||||
const censuses = new Map<string, CensusReds>();
|
||||
for (const v of verdicts) {
|
||||
const c = censuses.get(censusKey(v)) ?? { run: v.run, lane: v.lane, verdicts: 0, reds: [], byClass: {}, byCause: {} };
|
||||
c.verdicts++;
|
||||
if (v.failsLane) {
|
||||
c.reds.push({ run: v.run, lane: v.lane, case: v.case, status: v.status, trials: v.trials });
|
||||
for (const t of v.trials.filter(t => t.outcome === 'failed')) {
|
||||
c.byClass[t.failure_class ?? 'assertion'] = (c.byClass[t.failure_class ?? 'assertion'] ?? 0) + 1;
|
||||
c.byCause[t.failure_cause ?? 'unrecorded'] = (c.byCause[t.failure_cause ?? 'unrecorded'] ?? 0) + 1;
|
||||
}
|
||||
}
|
||||
censuses.set(censusKey(v), c);
|
||||
}
|
||||
const reds = verdicts.filter(v => v.failsLane).length;
|
||||
const rate = verdicts.length ? reds / verdicts.length : 0;
|
||||
const pooled = { verdicts: verdicts.length, reds, rate, ...wilson(reds, verdicts.length) };
|
||||
const byCase = new Map<string, { verdicts: number; reds: number }>();
|
||||
for (const v of verdicts) {
|
||||
const e = byCase.get(v.case) ?? { verdicts: 0, reds: 0 };
|
||||
e.verdicts++; if (v.failsLane) e.reds++;
|
||||
byCase.set(v.case, e);
|
||||
}
|
||||
const perCase = [...byCase.entries()].map(([id, e]) => ({ case: id, ...e, rate: (e.reds + RED_RATE_SHRINK * rate) / (e.verdicts + RED_RATE_SHRINK) }))
|
||||
.sort((a, b) => b.rate - a.rate || (a.case < b.case ? -1 : 1));
|
||||
const ordered = [...censuses.values()];
|
||||
const latestRun = ordered.map(c => c.run).sort().at(-1);
|
||||
const census = verdicts.filter(v => v.run === latestRun);
|
||||
const green = (prior: number) => census.reduce((p, v) => {
|
||||
const e = byCase.get(v.case)!;
|
||||
return p * (1 - (e.reds + RED_RATE_SHRINK * prior) / (e.verdicts + RED_RATE_SHRINK));
|
||||
}, 1);
|
||||
const allGreen = census.length ? { p: green(rate), lo: green(pooled.hi), hi: green(pooled.lo), verdicts: census.length } : null;
|
||||
return { censuses: ordered.sort((a, b) => (a.run < b.run ? -1 : a.run > b.run ? 1 : a.lane < b.lane ? -1 : 1)), pooled, perCase, allGreen };
|
||||
}
|
||||
|
||||
const tally = (counts: Record<string, number>) => Object.entries(counts).sort(([a, x], [b, y]) => y - x || (a < b ? -1 : 1)).map(([k, n]) => `${k} ${n}`).join(', ') || 'none';
|
||||
|
||||
export function formatRedLedger(ledger: RedLedger, opts: { top?: number } = {}): string[] {
|
||||
const lines = ['reds: verdict reds per census lane by failure class and machine cause (cause "unrecorded" = written before failure_cause existed)'];
|
||||
for (const c of ledger.censuses) {
|
||||
lines.push(` ${c.run} ${c.lane}: ${c.reds.length} red of ${c.verdicts} verdicts${c.reds.length ? ` — class: ${tally(c.byClass)}; cause: ${tally(c.byCause)}` : ''}`);
|
||||
for (const red of c.reds) {
|
||||
const failed = red.trials.filter(t => t.outcome === 'failed');
|
||||
lines.push(` ✗ ${red.case} ${red.status} ${failed.map(t => `t${t.trial} ${t.failure_class ?? 'assertion'}/${t.failure_cause ?? 'unrecorded'}`
|
||||
+ `${t.error ? ` — ${sanitizePublishedText(t.error, 160)}` : ''}`).join('; ')}`);
|
||||
}
|
||||
}
|
||||
const p = ledger.pooled;
|
||||
lines.push(`pooled: ${p.reds} red of ${p.verdicts} verdicts = ${pct(p.rate)} per verdict [95% ${pct(p.lo)}–${pct(p.hi)}]`);
|
||||
const top = ledger.perCase.filter(c => c.reds > 0).slice(0, opts.top ?? 15);
|
||||
if (top.length) lines.push(`per-case red rates (shrunk toward the pooled rate, K=${RED_RATE_SHRINK}):`, ...top.map(c => ` ${pct(c.rate).padStart(6)} ${c.reds}/${c.verdicts} ${c.case}`));
|
||||
const g = ledger.allGreen;
|
||||
lines.push(g ? `all-green probability (approximation, assumes independent verdicts): ${pct(g.p)} for the latest census's ${g.verdicts} verdicts [${pct(g.lo)}–${pct(g.hi)} with the pooled rate's 95% interval as prior]; `
|
||||
+ 'formula Π(1 - p_i) over its cases; rerun: bun run eval:pass-rates --reds' : 'all-green probability: no post-policy verdicts');
|
||||
return lines;
|
||||
}
|
||||
|
||||
// --- One census's triage view ---
|
||||
|
||||
/** Fetches the report views make beyond the history loader: a named run and its slice jobs. */
|
||||
export interface RunJobsFetcher { listJobs(repo: string, runId: number): JobTiming[]; getRun(repo: string, runId: number): WeeklyRun }
|
||||
|
||||
function ghJson(args: string[]): string[] {
|
||||
const result = spawnSync('gh', args, { timeout: 120_000, maxBuffer: 64 * 1024 * 1024 });
|
||||
if (result.status !== 0) throw new Error(`gh ${args.slice(0, 2).join(' ')} failed: ${String(result.stderr || result.error || '').trim()}`);
|
||||
return result.stdout.toString('utf8').split('\n').filter(Boolean);
|
||||
}
|
||||
|
||||
export const GH_JOBS: RunJobsFetcher = {
|
||||
listJobs: (repo, runId) => ghJson(['api', `repos/${repo}/actions/runs/${runId}/jobs?per_page=100`, '--paginate',
|
||||
'--jq', '.jobs[] | {name, startedAt: .started_at, completedAt: .completed_at}']).map(line => JSON.parse(line) as JobTiming),
|
||||
getRun: (repo, runId) => JSON.parse(ghJson(['api', `repos/${repo}/actions/runs/${runId}`,
|
||||
'--jq', '{id, attempt: .run_attempt, sha: .head_sha, branch: .head_branch, createdAt: .created_at}'])[0]!) as WeeklyRun,
|
||||
};
|
||||
|
||||
/** The manifest shard key a record ran under: its trial shard, its case shard, or its file. */
|
||||
export function recordShardKey(r: Pick<Rec, 'file' | 'case' | 'trial' | 'source'>): string {
|
||||
return r.source === 'shard' ? trialShardKey(r.file, r.case, r.trial) : CASE_SHARDED_FILES.includes(r.file) ? `${r.file}#${r.case}` : r.file;
|
||||
}
|
||||
|
||||
/**
|
||||
* The run's reds with their values and history, and the slice artifacts that
|
||||
* hold their transcripts. Only the slices a red names are downloaded; a slice
|
||||
* over EVIDENCE_MAX_BYTES is reported, never silently skipped.
|
||||
*/
|
||||
export function triageRun(opts: { repo: string; run: WeeklyRun; records: readonly Rec[]; caseFilter?: string; cacheDir: string;
|
||||
fetcher: HistoryFetcher; download: (match: (name: string) => boolean, maxBytes: number) => string[] }): string[] {
|
||||
const runId = String(opts.run.id);
|
||||
const mine = opts.records.filter(r => r.run_id === runId);
|
||||
const reds = verdictsOf(mine).filter(v => v.failsLane && (!opts.caseFilter || v.case === opts.caseFilter));
|
||||
const lines = [`run ${runId} (${opts.run.branch} @ ${opts.run.sha.slice(0, 9)}, ${opts.run.createdAt}): ${reds.length} verdict red(s)${opts.caseFilter ? ` for ${opts.caseFilter}` : ''}`];
|
||||
const history = verdictsOf(opts.records);
|
||||
const plans = new Map<string, Map<string, number>>();
|
||||
const planFor = (lane: string) => {
|
||||
const name = lane.startsWith('gate') && lane.includes('census') ? 'gate-census-plan' : 'paid-plan';
|
||||
if (!plans.has(name)) {
|
||||
const files = new Map<string, number>();
|
||||
for (const dir of opts.download(n => n === name, 8 * 1024 * 1024)) {
|
||||
try {
|
||||
const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'manifest.json'), 'utf8'));
|
||||
for (const e of manifest.entries ?? []) if (e.status === 'planned' && Number.isInteger(e.slice)) files.set(e.file, e.slice);
|
||||
} catch { /* no manifest: slices stay unknown */ }
|
||||
}
|
||||
plans.set(name, files);
|
||||
}
|
||||
return { prefix: name === 'gate-census-plan' ? 'gate-census' : 'paid-slice', files: plans.get(name)! };
|
||||
};
|
||||
const artifacts = opts.fetcher.listArtifacts(opts.repo, opts.run.id);
|
||||
for (const red of reds) {
|
||||
const runs = history.filter(v => v.case === red.case);
|
||||
lines.push(`✗ ${red.case} ${red.lane} ${red.status} (history: ${runs.filter(v => !v.failsLane).length}/${runs.length} verdicts green)`);
|
||||
for (const t of red.trials.filter(t => t.outcome === 'failed')) {
|
||||
const d = t.failure_detail;
|
||||
const detail = !d ? '' : 'judge' in d ? ` ${d.judge.map(j => `${j.dimension} ${j.mean} < ${j.threshold} (${j.samples} samples)${j.rationale ? ` "${j.rationale}"` : ''}`).join('; ')}`
|
||||
: ` Expected: ${d.expected} · Received: ${d.received}`;
|
||||
lines.push(sanitizePublishedText(` t${t.trial}: ${t.failure_class ?? 'assertion'} / cause ${t.failure_cause ?? 'unrecorded'}`
|
||||
+ `${t.failure_cause_evidence ? `: ${t.failure_cause_evidence}` : ''}${t.error ? ` — ${t.error}` : ''}${detail}`, 2000));
|
||||
const plan = planFor(red.lane);
|
||||
const slice = plan.files.get(recordShardKey(t));
|
||||
if (slice === undefined) { lines.push(' evidence: slice unknown (no plan artifact for this lane)'); continue; }
|
||||
const name = `${plan.prefix}-${slice}-a${t.attempt}`;
|
||||
const artifact = artifacts.find(a => a.name === name);
|
||||
if (!artifact) { lines.push(` evidence: ${name} (expired or missing)`); continue; }
|
||||
if (artifact.size > EVIDENCE_MAX_BYTES) { lines.push(` evidence: ${name} too large to fetch (${Math.round(artifact.size / 1024 / 1024)} MB)`); continue; }
|
||||
const [dir] = opts.download(n => n === name, EVIDENCE_MAX_BYTES);
|
||||
lines.push(` evidence: ${dir ? path.join(dir, 'shards', shardSlug([recordShardKey(t)])) : `${name} (download failed)`}`);
|
||||
}
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
// --- Flags ---
|
||||
|
||||
export const PASS_RATES_USAGE = `Usage: bun run eval:pass-rates [options]
|
||||
(no view flag) per-case trial pass rates with 95% Wilson intervals
|
||||
--headroom slowest armed session per case vs its budget, case wall upper bound, census critical path
|
||||
--reds verdict reds per census by failure class and cause, pooled and per-case rates, all-green probability
|
||||
--run <id> one census: each red with its values, history and fetched transcript evidence
|
||||
--case <id> restrict every view, the JSON and the alarms to one registered case
|
||||
--runs <n> completed runs per branch to read (default 10)
|
||||
--branch <name> branch whose runs to read beside main (default: current branch)
|
||||
--repo <owner/repo> --workflow <file> --dir <path> (repeatable, local artifacts; no network)
|
||||
--backfill also import legacy slice artifacts, labeled pre-policy
|
||||
--since-days <n> legacy eval files window for --dir (default 60)
|
||||
--json machine-readable output
|
||||
--gate exit 1 on ACTION REQUIRED (pass-rate alarms and sessions above ${pct(HEADROOM_FAIL)} of budget)
|
||||
--help this text (no network)`;
|
||||
|
||||
export interface PassRatesArgs {
|
||||
view: 'rates' | 'headroom' | 'reds' | 'run';
|
||||
runId?: number; caseFilter?: string; runs: number; sinceDays: number; dirs: string[];
|
||||
json: boolean; gate: boolean; backfill: boolean; repo?: string; workflow?: string; branch?: string;
|
||||
}
|
||||
|
||||
/** Parse eval:pass-rates flags; unknown flags, bad values and unknown case ids are errors (exit 2, before any fetch). */
|
||||
export function parsePassRatesArgs(argv: readonly string[], isKnownCase: (id: string) => boolean): { help: true } | { error: string } | PassRatesArgs {
|
||||
const out: PassRatesArgs = { view: 'rates', runs: 10, sinceDays: 60, dirs: [], json: false, gate: false, backfill: false };
|
||||
const views: string[] = [];
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const arg = argv[i]!;
|
||||
const value = () => { const v = argv[++i]; if (v === undefined || v.startsWith('--')) throw new Error(`${arg} needs a value`); return v; };
|
||||
const count = () => { const v = value(); if (!/^[1-9]\d*$/.test(v)) throw new Error(`${arg} needs a positive integer, got ${v}`); return Number(v); };
|
||||
try {
|
||||
switch (arg) {
|
||||
case '--help': case '-h': return { help: true };
|
||||
case '--headroom': views.push('headroom'); break;
|
||||
case '--reds': views.push('reds'); break;
|
||||
case '--run': views.push('run'); out.runId = count(); break;
|
||||
case '--case': out.caseFilter = value(); break;
|
||||
case '--runs': out.runs = count(); break;
|
||||
case '--since-days': out.sinceDays = count(); break;
|
||||
case '--dir': out.dirs.push(value()); break;
|
||||
case '--repo': out.repo = value(); break;
|
||||
case '--workflow': out.workflow = value(); break;
|
||||
case '--branch': out.branch = value(); break;
|
||||
case '--json': out.json = true; break;
|
||||
case '--gate': out.gate = true; break;
|
||||
case '--backfill': out.backfill = true; break;
|
||||
default: return { error: `unknown flag ${arg}` };
|
||||
}
|
||||
} catch (error) { return { error: (error as Error).message }; }
|
||||
}
|
||||
if (views.length > 1) return { error: `choose one of --headroom, --reds, --run (got ${views.join(', ')})` };
|
||||
if (views[0]) out.view = views[0] as PassRatesArgs['view'];
|
||||
if (out.caseFilter !== undefined && !isKnownCase(out.caseFilter)) return { error: `unknown case ${out.caseFilter} (not an E2E, judge or paid test file id)` };
|
||||
if (out.view === 'run' && out.dirs.length) return { error: '--run reads one GitHub run; it cannot combine with --dir' };
|
||||
return out;
|
||||
}
|
||||
Reference in new issue
Block a user