mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 14:06:50 +01:00
0.1: eval:pass-rates --headroom, --reds and --run report views; 85% headroom alarm in --gate
A new pure module, scripts/lib/eval-history.ts, reports per case the slowest armed session against the timeout it armed (censored when it timed out, insufficient below 3 samples, unknown without a session ledger, with the case wall shown separately as an upper bound), the census critical path from the slice jobs' timings, the verdict red ledger per census by failure class and machine cause with shrunk per-case rates, and the all-green probability (an approximation: the product of 1 - p_i, with an interval). --run <id> lists one census's reds with their values and history and downloads only the slice artifacts those reds name, reporting any over 64 MB instead of skipping them. eval-flake-rank.ts only routes flags: --help works offline, an unknown flag or case id exits 2 before any fetch, --case narrows the JSON and the alarms, every view prints its run/branch scope, and --gate also alarms on a session above HEADROOM_FAIL (85%) of its budget.
This commit is contained in:
garrytan committed
1 parent
d725d94ea3
commit
c8115df6dd
4 files changed
+604
-16
No files matched your search
+45
-16
@@ -36,6 +36,8 @@
|
||||
* bun run eval:pass-rates --dir <path> # local eval dirs / downloaded artifacts (repeatable)
|
||||
* bun run eval:pass-rates --backfill # also import legacy slice artifacts, labeled pre-policy
|
||||
* bun run eval:pass-rates --json | --gate
|
||||
* bun run eval:pass-rates --headroom | --reds | --run <id> # report views (scripts/lib/eval-history.ts)
|
||||
* bun run eval:pass-rates --help # every flag, no network
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
@@ -51,6 +53,8 @@ import { CASE_QUARANTINE, EVAL_POLICY } from '../test/helpers/periodic-exclude-d
|
||||
import { matchGlob } from '../test/helpers/test-selection';
|
||||
import { CASE_TEST_NAMES } from './test-paid-shards';
|
||||
import { resolveStateRoot } from '../lib/state-root';
|
||||
import { GH_JOBS, PASS_RATES_USAGE, criticalPath, formatCriticalPath, formatHeadroom, formatRedLedger, headroom, headroomAlarms, parsePassRatesArgs,
|
||||
redLedger, triageRun, type CriticalPath } from './lib/eval-history';
|
||||
|
||||
interface TestSeries {
|
||||
name: string;
|
||||
@@ -316,7 +320,7 @@ export function holmRejections(pValues: number[], alpha: number): Set<number> {
|
||||
|
||||
export type PassRateLabel = 'INCONCLUSIVE' | 'BROKEN' | 'FLAKY' | 'FAILING' | 'PASSING';
|
||||
export type AlarmKind = 'drift' | 'rule-as-behavior' | 'regression' | 'quarantine-exit' | 'quarantine-expired'
|
||||
| 'quarantine-cap' | 'quarantine-invalid';
|
||||
| 'quarantine-cap' | 'quarantine-invalid' | 'headroom';
|
||||
|
||||
/** The EVAL_POLICY fields pass-rates reads (structural, so tests can vary them). */
|
||||
export interface PassRatePolicy {
|
||||
@@ -639,24 +643,22 @@ function repoSlug(): string {
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const argv = process.argv.slice(2);
|
||||
const flag = (name: string) => { const index = argv.indexOf(name); return index === -1 ? undefined : argv[index + 1]; };
|
||||
const dirs = argv.flatMap((arg, index) => arg === '--dir' && argv[index + 1] ? [argv[index + 1]!] : []);
|
||||
const asJson = argv.includes('--json');
|
||||
const gate = argv.includes('--gate');
|
||||
const backfill = argv.includes('--backfill');
|
||||
const caseFilter = flag('--case');
|
||||
const runsLimit = Number(flag('--runs')) || 10;
|
||||
const sinceDays = Number(flag('--since-days')) || 60;
|
||||
const repo = flag('--repo') ?? repoSlug();
|
||||
const workflow = flag('--workflow') ?? 'evals-periodic.yml';
|
||||
const branch = flag('--branch') ?? gitOutput(['rev-parse', '--abbrev-ref', 'HEAD']) ?? 'main';
|
||||
const parsed = parsePassRatesArgs(process.argv.slice(2), id => Object.hasOwn(E2E_TIERS, id) || Object.hasOwn(LLM_JUDGE_TOUCHFILES, id)
|
||||
|| (/^test\/[\w./-]+\.test\.ts$/.test(id) && fs.existsSync(id)));
|
||||
if ('help' in parsed) { console.log(PASS_RATES_USAGE); process.exit(0); }
|
||||
if ('error' in parsed) { console.error(`eval:pass-rates: ${parsed.error}\n${PASS_RATES_USAGE}`); process.exit(2); }
|
||||
const { dirs, json: asJson, gate, backfill, caseFilter, runs: runsLimit, sinceDays, view } = parsed;
|
||||
const repo = parsed.repo ?? repoSlug();
|
||||
const workflow = parsed.workflow ?? 'evals-periodic.yml';
|
||||
const branch = parsed.branch ?? gitOutput(['rev-parse', '--abbrev-ref', 'HEAD']) ?? 'main';
|
||||
|
||||
const records: TrialRecord[] = [];
|
||||
const unattributed = new Set<string>();
|
||||
const errors: string[] = [];
|
||||
let historyError: string | null = null;
|
||||
let weeklyRuns: string[] | undefined;
|
||||
let fetched: WeeklyRun[] = [];
|
||||
const cacheDir = path.join(path.resolve(resolveStateRoot()), 'eval-pass-rates-cache', repo.replace('/', '-'));
|
||||
|
||||
const manualReviews: string[] = [];
|
||||
const importDir = (dir: string, run: { run_id: string; sha?: string; timestamp?: string } | undefined, legacyDays: number) => {
|
||||
@@ -674,8 +676,9 @@ if (import.meta.main) {
|
||||
} else {
|
||||
try {
|
||||
const runs = listWeeklyRuns({ repo, workflow, branches: [...new Set([branch, 'main'])], limit: runsLimit });
|
||||
if (parsed.runId !== undefined && !runs.some(run => run.id === parsed.runId)) runs.push(GH_JOBS.getRun(repo, parsed.runId));
|
||||
weeklyRuns = runs.map(run => run.createdAt);
|
||||
const cacheDir = path.join(path.resolve(resolveStateRoot()), 'eval-pass-rates-cache', repo.replace('/', '-'));
|
||||
fetched = runs;
|
||||
const match = backfill
|
||||
? (name: string) => name.startsWith('trial-outcomes') || /^(paid-slice-\d+|gate-census-\d+)(-a\d+)?$/.test(name)
|
||||
: (name: string) => name.startsWith('trial-outcomes');
|
||||
@@ -688,11 +691,37 @@ if (import.meta.main) {
|
||||
}
|
||||
}
|
||||
|
||||
const report = analyzePassRates(records, { weeklyRuns, unattributed: [...unattributed].sort(), errors, manualReviews });
|
||||
const scope = dirs.length ? `local dirs ${dirs.join(', ')}` : `${repo} ${workflow} on ${[...new Set([branch, 'main'])].join(' + ')}, `
|
||||
+ `last ${runsLimit} completed run(s) per branch: ${fetched.map(run => run.id).join(', ') || 'none'}`;
|
||||
const scoped = caseFilter ? records.filter(record => record.case === caseFilter) : records;
|
||||
const caseHeadroom = headroom(scoped);
|
||||
if (view !== 'rates') {
|
||||
const critical: CriticalPath[] = [];
|
||||
if (view !== 'reds' && !dirs.length) {
|
||||
for (const run of fetched.filter(run => view === 'headroom' || run.id === parsed.runId)) {
|
||||
try { critical.push(criticalPath(String(run.id), GH_JOBS.listJobs(repo, run.id))); } catch { critical.push({ run: String(run.id), jobs: 0, slowest: null, wallMs: null }); }
|
||||
}
|
||||
}
|
||||
const lines = view === 'headroom' ? formatHeadroom(caseHeadroom, critical) : view === 'reds' ? formatRedLedger(redLedger(scoped))
|
||||
: historyError ? [] : triageRun({ repo, run: fetched.find(run => run.id === parsed.runId)!, records, caseFilter, cacheDir, fetcher: GH_HISTORY,
|
||||
download: (match, maxBytes) => downloadRunArtifacts({ repo, run: fetched.find(run => run.id === parsed.runId)!, match, cacheDir, maxBytes }) })
|
||||
.concat(formatCriticalPath(critical));
|
||||
if (asJson) console.log(JSON.stringify({ scope, historyError, view, lines }, null, 2));
|
||||
else {
|
||||
console.log(`scope: ${scope}`);
|
||||
if (historyError) console.log(`history unavailable (${historyError})`);
|
||||
for (const line of lines) console.log(line);
|
||||
}
|
||||
process.exit(historyError ? 1 : 0);
|
||||
}
|
||||
const full = analyzePassRates(records, { weeklyRuns, unattributed: [...unattributed].sort(), errors, manualReviews });
|
||||
const alarms = [...full.alarms, ...(gate ? headroomAlarms(caseHeadroom) : [])].filter(alarm => !caseFilter || alarm.case === caseFilter);
|
||||
const report = { ...full, cases: full.cases.filter(entry => !caseFilter || entry.case === caseFilter), alarms };
|
||||
const ledger = readFreeLedger();
|
||||
if (asJson) {
|
||||
console.log(JSON.stringify({ repo, workflow, branch, dirs, historyError, ...report, freeLedger: ledger }, null, 2));
|
||||
console.log(JSON.stringify({ repo, workflow, branch, scope, dirs, historyError, ...report, freeLedger: ledger }, null, 2));
|
||||
} else {
|
||||
console.log(`scope: ${scope}`);
|
||||
if (historyError) console.log(`pass-rates: history unavailable (${historyError}); every label below is INCONCLUSIVE`);
|
||||
console.log(formatPassRates(report, { caseFilter }));
|
||||
if (ledger.length > 0) {
|
||||
|
||||
@@ -0,0 +1,350 @@
|
||||
/**
|
||||
* eval:pass-rates report views over stored trial records (plan 0.1):
|
||||
* session headroom (--headroom), the verdict red ledger with the all-green
|
||||
* probability (--reds), and one census's triage view (--run <id>). Pure
|
||||
* functions plus the run-scoped fetches they need; scripts/eval-flake-rank.ts
|
||||
* parses flags, loads history and dispatches here.
|
||||
*
|
||||
* Headroom is per armed session (session ledger, `sessions[]` on a record):
|
||||
* elapsed over the timeout it armed. A case's headroom is its slowest
|
||||
* session; a timed-out sample is censored (its true duration is unknown).
|
||||
* Records without sessions are unknown, never scored; their case wall
|
||||
* (JUnit or shard clock) is shown separately as an upper bound.
|
||||
*
|
||||
* The all-green probability is an approximation that assumes independent
|
||||
* verdicts: P(green) = Π (1 - p_i) over the cases a census verdicts, where
|
||||
* p_i = (reds_i + K * p̄) / (verdicts_i + K) shrinks each case's red rate
|
||||
* toward the pooled rate p̄. Its interval recomputes the same product with p̄
|
||||
* replaced by the pooled rate's 95% Wilson bounds.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { panelVerdict, type TrialOutcomeRecord } from '../../test/helpers/eval-store';
|
||||
import { HEADROOM_FAIL, HEADROOM_WARN } from '../../test/helpers/eval-budgets';
|
||||
import { sanitizePublishedText } from './published-text';
|
||||
import type { HistoryFetcher, WeeklyRun } from '../eval-flake-rank';
|
||||
import { CASE_SHARDED_FILES, trialShardKey } from './paid-cases';
|
||||
import { shardSlug } from '../test-paid-shards';
|
||||
|
||||
/** Below this many session samples a case's headroom is insufficient, never a pass. */
|
||||
export const HEADROOM_MIN_SAMPLES = 3;
|
||||
/** Pseudo-verdicts of pooled rate added to each case's red rate. */
|
||||
export const RED_RATE_SHRINK = 4;
|
||||
/** Largest slice artifact --run downloads. */
|
||||
export const EVIDENCE_MAX_BYTES = 64 * 1024 * 1024;
|
||||
|
||||
type Rec = TrialOutcomeRecord;
|
||||
const pct = (x: number) => `${Math.round(x * 1000) / 10}%`;
|
||||
const secs = (ms: number) => `${Math.round(ms / 1000)}s`;
|
||||
|
||||
// --- Headroom ---
|
||||
|
||||
export interface SessionHeadroom { key: string; samples: number; censored: number; maxRatio: number; maxElapsedMs: number; budgetMs: number }
|
||||
export interface CaseHeadroom {
|
||||
case: string;
|
||||
status: 'over' | 'warn' | 'ok' | 'insufficient' | 'unknown';
|
||||
sessions: SessionHeadroom[];
|
||||
/** Slowest session's worst ratio, when any session was recorded. */
|
||||
worst?: SessionHeadroom;
|
||||
/** Largest case wall (JUnit or shard clock) over the window: an upper bound, never a session clock. */
|
||||
caseWallMs: number;
|
||||
runs: number;
|
||||
}
|
||||
|
||||
export function headroom(records: readonly Rec[]): CaseHeadroom[] {
|
||||
const byCase = new Map<string, Rec[]>();
|
||||
for (const r of records) if (r.outcome !== 'skipped' && r.execution !== 'reused') byCase.set(r.case, [...(byCase.get(r.case) ?? []), r]);
|
||||
return [...byCase.entries()].map(([id, rs]) => {
|
||||
const sessions = new Map<string, SessionHeadroom>();
|
||||
for (const s of rs.flatMap(r => r.sessions ?? [])) {
|
||||
if (!s.budget_ms) continue;
|
||||
const key = s.key;
|
||||
const entry = sessions.get(key) ?? { key, samples: 0, censored: 0, maxRatio: 0, maxElapsedMs: 0, budgetMs: s.budget_ms };
|
||||
entry.samples++;
|
||||
if (s.end === 'session_timeout' || s.end === 'observer_timeout') entry.censored++;
|
||||
const ratio = s.elapsed_ms / s.budget_ms;
|
||||
if (ratio > entry.maxRatio) Object.assign(entry, { maxRatio: ratio, maxElapsedMs: s.elapsed_ms, budgetMs: s.budget_ms });
|
||||
sessions.set(key, entry);
|
||||
}
|
||||
const list = [...sessions.values()].sort((a, b) => b.maxRatio - a.maxRatio);
|
||||
const worst = list[0];
|
||||
const status: CaseHeadroom['status'] = !worst ? 'unknown' : worst.samples < HEADROOM_MIN_SAMPLES ? 'insufficient'
|
||||
: worst.maxRatio > HEADROOM_FAIL ? 'over' : worst.maxRatio > HEADROOM_WARN ? 'warn' : 'ok';
|
||||
return { case: id, status, sessions: list, ...(worst ? { worst } : {}), caseWallMs: Math.max(0, ...rs.map(r => r.duration_ms)),
|
||||
runs: new Set(rs.map(r => r.run_id ?? 'local')).size };
|
||||
}).sort((a, b) => (b.worst?.maxRatio ?? -1) - (a.worst?.maxRatio ?? -1) || b.caseWallMs - a.caseWallMs);
|
||||
}
|
||||
|
||||
/** --gate alarms: a case whose slowest session ran above HEADROOM_FAIL of its armed budget. */
|
||||
export function headroomAlarms(cases: readonly CaseHeadroom[]): Array<{ kind: 'headroom'; case: string; message: string }> {
|
||||
return cases.filter(c => c.status === 'over').map(c => ({ kind: 'headroom' as const, case: c.case,
|
||||
message: `${c.case} session ${c.worst!.key}: max ${secs(c.worst!.maxElapsedMs)} of ${secs(c.worst!.budgetMs)} (${pct(c.worst!.maxRatio)})`
|
||||
+ `${c.worst!.censored ? `, ${c.worst!.censored} timed-out sample(s) censored` : ''} over ${c.worst!.samples} sample(s), above the ${pct(HEADROOM_FAIL)} cap. `
|
||||
+ 'Cut work in the skill or fixture; budgets are never raised (TESTING_INTERNALS "Timeout policy").' }));
|
||||
}
|
||||
|
||||
export function formatHeadroom(cases: readonly CaseHeadroom[], critical: readonly CriticalPath[] = []): string[] {
|
||||
const lines = [`headroom: slowest armed session per case (warn > ${pct(HEADROOM_WARN)}, alarm > ${pct(HEADROOM_FAIL)}; `
|
||||
+ `insufficient below ${HEADROOM_MIN_SAMPLES} samples; censored = timed out)`,
|
||||
' status session max/budget samples case wall (upper bound, not a session clock) case'];
|
||||
for (const c of cases) {
|
||||
const w = c.worst;
|
||||
const session = w ? `${pct(w.maxRatio)} ${secs(w.maxElapsedMs)}/${secs(w.budgetMs)}` : 'unknown (no session ledger)';
|
||||
lines.push(` ${c.status.padEnd(12)} ${session.padEnd(26)} ${(w ? `${w.samples}${w.censored ? ` (${w.censored} censored)` : ''}` : '-').padEnd(7)} `
|
||||
+ `${secs(c.caseWallMs).padEnd(45)} ${c.case}${w ? ` [${w.key}]` : ''}`);
|
||||
}
|
||||
return [...lines, ...formatCriticalPath(critical)];
|
||||
}
|
||||
|
||||
export function formatCriticalPath(critical: readonly CriticalPath[]): string[] {
|
||||
return critical.map(cp => `critical path ${cp.run}: ${cp.slowest ? `${cp.slowest.name} ${secs(cp.slowest.ms)}` : 'unknown'}`
|
||||
+ ` of ${cp.jobs} slice job(s), run wall ${cp.wallMs === null ? 'unknown' : secs(cp.wallMs)}${cp.files?.length ? `; slice files: ${cp.files.join(', ')}` : ''}`);
|
||||
}
|
||||
|
||||
// --- Critical path from runner slice timings ---
|
||||
|
||||
export interface JobTiming { name: string; startedAt: string | null; completedAt: string | null }
|
||||
export interface CriticalPath { run: string; jobs: number; slowest: { name: string; ms: number } | null; wallMs: number | null; files?: string[] }
|
||||
|
||||
const SLICE_JOB = /^(eval-slices|eval-codex-slices|gate-census) \((\d+)\)$/;
|
||||
|
||||
export function criticalPath(run: string, jobs: readonly JobTiming[], plan?: { entries: Array<{ file: string; slice?: number; status?: string }> }): CriticalPath {
|
||||
const timed = jobs.filter(j => SLICE_JOB.test(j.name) && j.startedAt && j.completedAt)
|
||||
.map(j => ({ name: j.name, ms: Date.parse(j.completedAt!) - Date.parse(j.startedAt!) }));
|
||||
const slowest = timed.sort((a, b) => b.ms - a.ms)[0] ?? null;
|
||||
const all = jobs.filter(j => j.startedAt && j.completedAt);
|
||||
const wallMs = all.length ? Math.max(...all.map(j => Date.parse(j.completedAt!))) - Math.min(...all.map(j => Date.parse(j.startedAt!))) : null;
|
||||
const slice = slowest ? Number(SLICE_JOB.exec(slowest.name)![2]) : null;
|
||||
const files = plan && slice !== null && slowest!.name.startsWith('gate-census')
|
||||
? plan.entries.filter(e => e.slice === slice && e.status === 'planned').map(e => e.file) : undefined;
|
||||
return { run, jobs: timed.length, slowest, wallMs, ...(files ? { files } : {}) };
|
||||
}
|
||||
|
||||
// --- Red ledger ---
|
||||
|
||||
export interface VerdictRed { run: string; lane: string; case: string; status: string; trials: Rec[] }
|
||||
export interface CensusReds { run: string; lane: string; verdicts: number; reds: VerdictRed[]; byClass: Record<string, number>; byCause: Record<string, number> }
|
||||
export interface RedLedger {
|
||||
censuses: CensusReds[];
|
||||
pooled: { verdicts: number; reds: number; rate: number; lo: number; hi: number };
|
||||
perCase: Array<{ case: string; verdicts: number; reds: number; rate: number }>;
|
||||
allGreen: { p: number; lo: number; hi: number; verdicts: number } | null;
|
||||
}
|
||||
|
||||
function wilson(k: number, n: number, z = 1.96): { lo: number; hi: number } {
|
||||
if (n === 0) return { lo: 0, hi: 1 };
|
||||
const p = k / n, d = 1 + z * z / n, c = p + z * z / (2 * n), m = z * Math.sqrt(p * (1 - p) / n + z * z / (4 * n * n));
|
||||
return { lo: Math.max(0, (c - m) / d), hi: Math.min(1, (c + m) / d) };
|
||||
}
|
||||
|
||||
/** One panelVerdict per case per census lane (earliest attempt), exactly as the report computes it. */
|
||||
export function verdictsOf(records: readonly Rec[]): Array<{ run: string; lane: string; case: string; status: string; failsLane: boolean; trials: Rec[] }> {
|
||||
const groups = new Map<string, Rec[]>();
|
||||
for (const r of records) {
|
||||
if (r.policy_version < 1 || r.execution === 'reused') continue;
|
||||
// A JUnit case is one verdict per shard file, as the report counts it; trial shards form one panel.
|
||||
const key = `${r.run_id ?? 'local'}\0${r.lane ?? r.tier}\0${r.case}\0${r.source === 'shard' ? '' : r.file}`;
|
||||
groups.set(key, [...(groups.get(key) ?? []), r]);
|
||||
}
|
||||
return [...groups.entries()].map(([key, rs]) => {
|
||||
const [run, lane, id] = key.split('\0') as [string, string, string, string];
|
||||
const attempt = Math.min(...rs.map(r => r.attempt));
|
||||
const trials = rs.filter(r => r.attempt === attempt);
|
||||
const v = panelVerdict({ case: id, kind: trials[0]!.kind, panel: trials[0]!.panel, quarantined: trials[0]!.quarantined,
|
||||
trials: trials.map(t => ({ trial: t.trial, outcome: t.outcome, attempt, ...(t.failure_class ? { failure_class: t.failure_class } : {}) })) });
|
||||
return { run, lane, case: id, status: v.status, failsLane: v.failsLane, trials };
|
||||
});
|
||||
}
|
||||
|
||||
export function redLedger(records: readonly Rec[]): RedLedger {
|
||||
const verdicts = verdictsOf(records).filter(v => v.status !== 'SKIPPED');
|
||||
const censusKey = (v: { run: string; lane: string }) => `${v.run}\0${v.lane}`;
|
||||
const censuses = new Map<string, CensusReds>();
|
||||
for (const v of verdicts) {
|
||||
const c = censuses.get(censusKey(v)) ?? { run: v.run, lane: v.lane, verdicts: 0, reds: [], byClass: {}, byCause: {} };
|
||||
c.verdicts++;
|
||||
if (v.failsLane) {
|
||||
c.reds.push({ run: v.run, lane: v.lane, case: v.case, status: v.status, trials: v.trials });
|
||||
for (const t of v.trials.filter(t => t.outcome === 'failed')) {
|
||||
c.byClass[t.failure_class ?? 'assertion'] = (c.byClass[t.failure_class ?? 'assertion'] ?? 0) + 1;
|
||||
c.byCause[t.failure_cause ?? 'unrecorded'] = (c.byCause[t.failure_cause ?? 'unrecorded'] ?? 0) + 1;
|
||||
}
|
||||
}
|
||||
censuses.set(censusKey(v), c);
|
||||
}
|
||||
const reds = verdicts.filter(v => v.failsLane).length;
|
||||
const rate = verdicts.length ? reds / verdicts.length : 0;
|
||||
const pooled = { verdicts: verdicts.length, reds, rate, ...wilson(reds, verdicts.length) };
|
||||
const byCase = new Map<string, { verdicts: number; reds: number }>();
|
||||
for (const v of verdicts) {
|
||||
const e = byCase.get(v.case) ?? { verdicts: 0, reds: 0 };
|
||||
e.verdicts++; if (v.failsLane) e.reds++;
|
||||
byCase.set(v.case, e);
|
||||
}
|
||||
const perCase = [...byCase.entries()].map(([id, e]) => ({ case: id, ...e, rate: (e.reds + RED_RATE_SHRINK * rate) / (e.verdicts + RED_RATE_SHRINK) }))
|
||||
.sort((a, b) => b.rate - a.rate || (a.case < b.case ? -1 : 1));
|
||||
const ordered = [...censuses.values()];
|
||||
const latestRun = ordered.map(c => c.run).sort().at(-1);
|
||||
const census = verdicts.filter(v => v.run === latestRun);
|
||||
const green = (prior: number) => census.reduce((p, v) => {
|
||||
const e = byCase.get(v.case)!;
|
||||
return p * (1 - (e.reds + RED_RATE_SHRINK * prior) / (e.verdicts + RED_RATE_SHRINK));
|
||||
}, 1);
|
||||
const allGreen = census.length ? { p: green(rate), lo: green(pooled.hi), hi: green(pooled.lo), verdicts: census.length } : null;
|
||||
return { censuses: ordered.sort((a, b) => (a.run < b.run ? -1 : a.run > b.run ? 1 : a.lane < b.lane ? -1 : 1)), pooled, perCase, allGreen };
|
||||
}
|
||||
|
||||
const tally = (counts: Record<string, number>) => Object.entries(counts).sort(([a, x], [b, y]) => y - x || (a < b ? -1 : 1)).map(([k, n]) => `${k} ${n}`).join(', ') || 'none';
|
||||
|
||||
export function formatRedLedger(ledger: RedLedger, opts: { top?: number } = {}): string[] {
|
||||
const lines = ['reds: verdict reds per census lane by failure class and machine cause (cause "unrecorded" = written before failure_cause existed)'];
|
||||
for (const c of ledger.censuses) {
|
||||
lines.push(` ${c.run} ${c.lane}: ${c.reds.length} red of ${c.verdicts} verdicts${c.reds.length ? ` — class: ${tally(c.byClass)}; cause: ${tally(c.byCause)}` : ''}`);
|
||||
for (const red of c.reds) {
|
||||
const failed = red.trials.filter(t => t.outcome === 'failed');
|
||||
lines.push(` ✗ ${red.case} ${red.status} ${failed.map(t => `t${t.trial} ${t.failure_class ?? 'assertion'}/${t.failure_cause ?? 'unrecorded'}`
|
||||
+ `${t.error ? ` — ${sanitizePublishedText(t.error, 160)}` : ''}`).join('; ')}`);
|
||||
}
|
||||
}
|
||||
const p = ledger.pooled;
|
||||
lines.push(`pooled: ${p.reds} red of ${p.verdicts} verdicts = ${pct(p.rate)} per verdict [95% ${pct(p.lo)}–${pct(p.hi)}]`);
|
||||
const top = ledger.perCase.filter(c => c.reds > 0).slice(0, opts.top ?? 15);
|
||||
if (top.length) lines.push(`per-case red rates (shrunk toward the pooled rate, K=${RED_RATE_SHRINK}):`, ...top.map(c => ` ${pct(c.rate).padStart(6)} ${c.reds}/${c.verdicts} ${c.case}`));
|
||||
const g = ledger.allGreen;
|
||||
lines.push(g ? `all-green probability (approximation, assumes independent verdicts): ${pct(g.p)} for the latest census's ${g.verdicts} verdicts [${pct(g.lo)}–${pct(g.hi)} with the pooled rate's 95% interval as prior]; `
|
||||
+ 'formula Π(1 - p_i) over its cases; rerun: bun run eval:pass-rates --reds' : 'all-green probability: no post-policy verdicts');
|
||||
return lines;
|
||||
}
|
||||
|
||||
// --- One census's triage view ---
|
||||
|
||||
/** Fetches the report views make beyond the history loader: a named run and its slice jobs. */
|
||||
export interface RunJobsFetcher { listJobs(repo: string, runId: number): JobTiming[]; getRun(repo: string, runId: number): WeeklyRun }
|
||||
|
||||
function ghJson(args: string[]): string[] {
|
||||
const result = spawnSync('gh', args, { timeout: 120_000, maxBuffer: 64 * 1024 * 1024 });
|
||||
if (result.status !== 0) throw new Error(`gh ${args.slice(0, 2).join(' ')} failed: ${String(result.stderr || result.error || '').trim()}`);
|
||||
return result.stdout.toString('utf8').split('\n').filter(Boolean);
|
||||
}
|
||||
|
||||
export const GH_JOBS: RunJobsFetcher = {
|
||||
listJobs: (repo, runId) => ghJson(['api', `repos/${repo}/actions/runs/${runId}/jobs?per_page=100`, '--paginate',
|
||||
'--jq', '.jobs[] | {name, startedAt: .started_at, completedAt: .completed_at}']).map(line => JSON.parse(line) as JobTiming),
|
||||
getRun: (repo, runId) => JSON.parse(ghJson(['api', `repos/${repo}/actions/runs/${runId}`,
|
||||
'--jq', '{id, attempt: .run_attempt, sha: .head_sha, branch: .head_branch, createdAt: .created_at}'])[0]!) as WeeklyRun,
|
||||
};
|
||||
|
||||
/** The manifest shard key a record ran under: its trial shard, its case shard, or its file. */
|
||||
export function recordShardKey(r: Pick<Rec, 'file' | 'case' | 'trial' | 'source'>): string {
|
||||
return r.source === 'shard' ? trialShardKey(r.file, r.case, r.trial) : CASE_SHARDED_FILES.includes(r.file) ? `${r.file}#${r.case}` : r.file;
|
||||
}
|
||||
|
||||
/**
|
||||
* The run's reds with their values and history, and the slice artifacts that
|
||||
* hold their transcripts. Only the slices a red names are downloaded; a slice
|
||||
* over EVIDENCE_MAX_BYTES is reported, never silently skipped.
|
||||
*/
|
||||
export function triageRun(opts: { repo: string; run: WeeklyRun; records: readonly Rec[]; caseFilter?: string; cacheDir: string;
|
||||
fetcher: HistoryFetcher; download: (match: (name: string) => boolean, maxBytes: number) => string[] }): string[] {
|
||||
const runId = String(opts.run.id);
|
||||
const mine = opts.records.filter(r => r.run_id === runId);
|
||||
const reds = verdictsOf(mine).filter(v => v.failsLane && (!opts.caseFilter || v.case === opts.caseFilter));
|
||||
const lines = [`run ${runId} (${opts.run.branch} @ ${opts.run.sha.slice(0, 9)}, ${opts.run.createdAt}): ${reds.length} verdict red(s)${opts.caseFilter ? ` for ${opts.caseFilter}` : ''}`];
|
||||
const history = verdictsOf(opts.records);
|
||||
const plans = new Map<string, Map<string, number>>();
|
||||
const planFor = (lane: string) => {
|
||||
const name = lane.startsWith('gate') && lane.includes('census') ? 'gate-census-plan' : 'paid-plan';
|
||||
if (!plans.has(name)) {
|
||||
const files = new Map<string, number>();
|
||||
for (const dir of opts.download(n => n === name, 8 * 1024 * 1024)) {
|
||||
try {
|
||||
const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'manifest.json'), 'utf8'));
|
||||
for (const e of manifest.entries ?? []) if (e.status === 'planned' && Number.isInteger(e.slice)) files.set(e.file, e.slice);
|
||||
} catch { /* no manifest: slices stay unknown */ }
|
||||
}
|
||||
plans.set(name, files);
|
||||
}
|
||||
return { prefix: name === 'gate-census-plan' ? 'gate-census' : 'paid-slice', files: plans.get(name)! };
|
||||
};
|
||||
const artifacts = opts.fetcher.listArtifacts(opts.repo, opts.run.id);
|
||||
for (const red of reds) {
|
||||
const runs = history.filter(v => v.case === red.case);
|
||||
lines.push(`✗ ${red.case} ${red.lane} ${red.status} (history: ${runs.filter(v => !v.failsLane).length}/${runs.length} verdicts green)`);
|
||||
for (const t of red.trials.filter(t => t.outcome === 'failed')) {
|
||||
const d = t.failure_detail;
|
||||
const detail = !d ? '' : 'judge' in d ? ` ${d.judge.map(j => `${j.dimension} ${j.mean} < ${j.threshold} (${j.samples} samples)${j.rationale ? ` "${j.rationale}"` : ''}`).join('; ')}`
|
||||
: ` Expected: ${d.expected} · Received: ${d.received}`;
|
||||
lines.push(sanitizePublishedText(` t${t.trial}: ${t.failure_class ?? 'assertion'} / cause ${t.failure_cause ?? 'unrecorded'}`
|
||||
+ `${t.failure_cause_evidence ? `: ${t.failure_cause_evidence}` : ''}${t.error ? ` — ${t.error}` : ''}${detail}`, 2000));
|
||||
const plan = planFor(red.lane);
|
||||
const slice = plan.files.get(recordShardKey(t));
|
||||
if (slice === undefined) { lines.push(' evidence: slice unknown (no plan artifact for this lane)'); continue; }
|
||||
const name = `${plan.prefix}-${slice}-a${t.attempt}`;
|
||||
const artifact = artifacts.find(a => a.name === name);
|
||||
if (!artifact) { lines.push(` evidence: ${name} (expired or missing)`); continue; }
|
||||
if (artifact.size > EVIDENCE_MAX_BYTES) { lines.push(` evidence: ${name} too large to fetch (${Math.round(artifact.size / 1024 / 1024)} MB)`); continue; }
|
||||
const [dir] = opts.download(n => n === name, EVIDENCE_MAX_BYTES);
|
||||
lines.push(` evidence: ${dir ? path.join(dir, 'shards', shardSlug([recordShardKey(t)])) : `${name} (download failed)`}`);
|
||||
}
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
// --- Flags ---
|
||||
|
||||
export const PASS_RATES_USAGE = `Usage: bun run eval:pass-rates [options]
|
||||
(no view flag) per-case trial pass rates with 95% Wilson intervals
|
||||
--headroom slowest armed session per case vs its budget, case wall upper bound, census critical path
|
||||
--reds verdict reds per census by failure class and cause, pooled and per-case rates, all-green probability
|
||||
--run <id> one census: each red with its values, history and fetched transcript evidence
|
||||
--case <id> restrict every view, the JSON and the alarms to one registered case
|
||||
--runs <n> completed runs per branch to read (default 10)
|
||||
--branch <name> branch whose runs to read beside main (default: current branch)
|
||||
--repo <owner/repo> --workflow <file> --dir <path> (repeatable, local artifacts; no network)
|
||||
--backfill also import legacy slice artifacts, labeled pre-policy
|
||||
--since-days <n> legacy eval files window for --dir (default 60)
|
||||
--json machine-readable output
|
||||
--gate exit 1 on ACTION REQUIRED (pass-rate alarms and sessions above ${pct(HEADROOM_FAIL)} of budget)
|
||||
--help this text (no network)`;
|
||||
|
||||
export interface PassRatesArgs {
|
||||
view: 'rates' | 'headroom' | 'reds' | 'run';
|
||||
runId?: number; caseFilter?: string; runs: number; sinceDays: number; dirs: string[];
|
||||
json: boolean; gate: boolean; backfill: boolean; repo?: string; workflow?: string; branch?: string;
|
||||
}
|
||||
|
||||
/** Parse eval:pass-rates flags; unknown flags, bad values and unknown case ids are errors (exit 2, before any fetch). */
|
||||
export function parsePassRatesArgs(argv: readonly string[], isKnownCase: (id: string) => boolean): { help: true } | { error: string } | PassRatesArgs {
|
||||
const out: PassRatesArgs = { view: 'rates', runs: 10, sinceDays: 60, dirs: [], json: false, gate: false, backfill: false };
|
||||
const views: string[] = [];
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const arg = argv[i]!;
|
||||
const value = () => { const v = argv[++i]; if (v === undefined || v.startsWith('--')) throw new Error(`${arg} needs a value`); return v; };
|
||||
const count = () => { const v = value(); if (!/^[1-9]\d*$/.test(v)) throw new Error(`${arg} needs a positive integer, got ${v}`); return Number(v); };
|
||||
try {
|
||||
switch (arg) {
|
||||
case '--help': case '-h': return { help: true };
|
||||
case '--headroom': views.push('headroom'); break;
|
||||
case '--reds': views.push('reds'); break;
|
||||
case '--run': views.push('run'); out.runId = count(); break;
|
||||
case '--case': out.caseFilter = value(); break;
|
||||
case '--runs': out.runs = count(); break;
|
||||
case '--since-days': out.sinceDays = count(); break;
|
||||
case '--dir': out.dirs.push(value()); break;
|
||||
case '--repo': out.repo = value(); break;
|
||||
case '--workflow': out.workflow = value(); break;
|
||||
case '--branch': out.branch = value(); break;
|
||||
case '--json': out.json = true; break;
|
||||
case '--gate': out.gate = true; break;
|
||||
case '--backfill': out.backfill = true; break;
|
||||
default: return { error: `unknown flag ${arg}` };
|
||||
}
|
||||
} catch (error) { return { error: (error as Error).message }; }
|
||||
}
|
||||
if (views.length > 1) return { error: `choose one of --headroom, --reds, --run (got ${views.join(', ')})` };
|
||||
if (views[0]) out.view = views[0] as PassRatesArgs['view'];
|
||||
if (out.caseFilter !== undefined && !isKnownCase(out.caseFilter)) return { error: `unknown case ${out.caseFilter} (not an E2E, judge or paid test file id)` };
|
||||
if (out.view === 'run' && out.dirs.length) return { error: '--run reads one GitHub run; it cannot combine with --dir' };
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
/**
|
||||
* eval:pass-rates report views (plan 0.1): session headroom, the verdict red
|
||||
* ledger with the all-green probability, one census's triage view, and the
|
||||
* CLI contract (offline --help, unknown flag and case rejected before any
|
||||
* fetch, --case applied to JSON and alarms, the 85% alarm in --gate).
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { TRIAL_OUTCOME_SCHEMA, formatTrialOutcomes, type TrialOutcomeRecord } from './helpers/eval-store';
|
||||
import { EVAL_POLICY } from './helpers/periodic-exclude-data';
|
||||
import { HEADROOM_FAIL } from './helpers/eval-budgets';
|
||||
import { criticalPath, formatHeadroom, formatRedLedger, headroom, headroomAlarms, parsePassRatesArgs, redLedger, triageRun,
|
||||
verdictsOf, HEADROOM_MIN_SAMPLES, RED_RATE_SHRINK } from '../scripts/lib/eval-history';
|
||||
import type { HistoryFetcher } from '../scripts/eval-flake-rank';
|
||||
|
||||
const SCRIPT = path.resolve(import.meta.dir, '../scripts/eval-flake-rank.ts');
|
||||
|
||||
const rec = (over: Partial<TrialOutcomeRecord>): TrialOutcomeRecord => ({
|
||||
schema: TRIAL_OUTCOME_SCHEMA, case: 'rule-a', file: 'test/a.test.ts', tier: 'gate', kind: 'rule', trial: 1, panel: { n: 1, k: 1 },
|
||||
attempt: 1, outcome: 'passed', duration_ms: 1000, cost_usd: 0, policy_version: EVAL_POLICY.version, quarantined: false,
|
||||
execution: 'executed', source: 'junit', run_id: '1', lane: 'gate/full census', ...over,
|
||||
});
|
||||
const session = (elapsed: number, budget = 100_000, end = 'completed') => ({ key: 'pty:x#1', runner: 'pty', elapsed_ms: elapsed, budget_ms: budget, end });
|
||||
|
||||
describe('headroom', () => {
|
||||
const records = [
|
||||
...[1, 2, 3].map(run => rec({ case: 'slow', run_id: String(run), duration_ms: 95_000, sessions: [session(run === 3 ? 88_000 : 60_000)] })),
|
||||
...[1, 2, 3].map(run => rec({ case: 'fine', run_id: String(run), sessions: [session(70_000), { ...session(10_000), key: 'claude-p:y#1' }] })),
|
||||
...[1, 2].map(run => rec({ case: 'rare', run_id: String(run), sessions: [session(99_000)] })),
|
||||
rec({ case: 'legacy', duration_ms: 517_000 }),
|
||||
rec({ case: 'timedout', run_id: '1', outcome: 'failed', failure_class: 'timeout', sessions: [session(100_050, 100_000, 'session_timeout')] }),
|
||||
rec({ case: 'timedout', run_id: '2', sessions: [session(50_000)] }), rec({ case: 'timedout', run_id: '3', sessions: [session(50_000)] }),
|
||||
rec({ case: 'skipped-only', outcome: 'skipped', sessions: [session(99_000)] }),
|
||||
];
|
||||
const cases = headroom(records);
|
||||
const byCase = Object.fromEntries(cases.map(c => [c.case, c]));
|
||||
|
||||
test('88% alarms, 70% passes, too few samples is insufficient, a timeout is censored, no ledger is unknown, skipped never counts', () => {
|
||||
expect(byCase.slow).toMatchObject({ status: 'over', worst: { key: 'pty:x#1', samples: 3, maxRatio: 0.88, censored: 0 } });
|
||||
expect(byCase.fine).toMatchObject({ status: 'ok', worst: { maxRatio: 0.7 } });
|
||||
expect(byCase.rare!.status).toBe('insufficient');
|
||||
expect(HEADROOM_MIN_SAMPLES).toBe(3);
|
||||
expect(byCase.timedout).toMatchObject({ status: 'over', worst: { censored: 1, samples: 3 } });
|
||||
expect(byCase.legacy).toMatchObject({ status: 'unknown', caseWallMs: 517_000 });
|
||||
expect(byCase['skipped-only']).toBeUndefined();
|
||||
expect(headroomAlarms(cases).map(a => a.case).sort()).toEqual(['slow', 'timedout']);
|
||||
expect(headroomAlarms(cases)[0]!.message).toContain(`above the ${HEADROOM_FAIL * 100}% cap`);
|
||||
});
|
||||
|
||||
test('the session column and the case wall column are labelled differently; unknown sessions are never scored', () => {
|
||||
const lines = formatHeadroom(cases);
|
||||
expect(lines[1]).toContain('session max/budget');
|
||||
expect(lines[1]).toContain('case wall (upper bound, not a session clock)');
|
||||
expect(lines.find(l => l.endsWith('legacy'))).toMatch(/^ {2}unknown\s+unknown \(no session ledger\)\s+-\s+517s/);
|
||||
});
|
||||
|
||||
test('critical path is the slowest slice job; non-slice jobs only bound the run wall', () => {
|
||||
const job = (name: string, start: number, end: number) => ({ name, startedAt: new Date(start * 1000).toISOString(), completedAt: new Date(end * 1000).toISOString() });
|
||||
const cp = criticalPath('9', [job('build-image', 0, 60), job('eval-slices (1)', 60, 400), job('gate-census (11)', 70, 781), job('report', 781, 840)],
|
||||
{ entries: [{ file: 'test/skill-e2e-plan-mode-no-op.test.ts', slice: 11, status: 'planned' }, { file: 'x', slice: 2, status: 'planned' }] });
|
||||
expect(cp).toEqual({ run: '9', jobs: 2, slowest: { name: 'gate-census (11)', ms: 711_000 }, wallMs: 840_000, files: ['test/skill-e2e-plan-mode-no-op.test.ts'] });
|
||||
});
|
||||
});
|
||||
|
||||
describe('red ledger over the 8 wave censuses', () => {
|
||||
const fixture = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures', 'eval-history', 'wave-reds-8-censuses.json'), 'utf8'));
|
||||
// Every other verdict of each census lane, as a passing rule case (same ids across censuses).
|
||||
const passing = fixture.censuses.flatMap((c: any) => Array.from({ length: c.otherVerdicts }, (_, i) =>
|
||||
rec({ run_id: c.run, lane: c.lane, case: `green-${c.lane}-${i}`, file: `test/green-${i}.test.ts` })));
|
||||
const records: TrialOutcomeRecord[] = [...fixture.records, ...passing];
|
||||
const ledger = redLedger(records);
|
||||
|
||||
test('21 verdict reds across 8 censuses; 27 of 28 failed trials are class assertion; red trials carry no recorded cause', () => {
|
||||
expect(new Set(ledger.censuses.map(c => c.run)).size).toBe(8);
|
||||
expect(ledger.pooled.reds).toBe(21);
|
||||
const failed = records.filter(r => r.outcome === 'failed');
|
||||
expect(failed).toHaveLength(28);
|
||||
expect(failed.filter(r => r.failure_class === 'assertion')).toHaveLength(27);
|
||||
// 22 failed trials sit in red verdicts; the other 6 are the failed trials of split PASS panels.
|
||||
expect(ledger.censuses.reduce((sum, c) => sum + (c.byCause.unrecorded ?? 0), 0)).toBe(22);
|
||||
expect(ledger.censuses.map(c => c.reds.length).reduce((a, b) => a + b, 0)).toBe(21);
|
||||
});
|
||||
|
||||
test('the all-green probability follows its documented formula and sits inside its interval', () => {
|
||||
const latest = verdictsOf(records).filter(v => v.run === '37198445662');
|
||||
const rate = ledger.pooled.reds / ledger.pooled.verdicts;
|
||||
const expected = latest.reduce((p, v) => {
|
||||
const mine = verdictsOf(records).filter(x => x.case === v.case);
|
||||
return p * (1 - (mine.filter(x => x.failsLane).length + RED_RATE_SHRINK * rate) / (mine.length + RED_RATE_SHRINK));
|
||||
}, 1);
|
||||
expect(ledger.allGreen!.p).toBeCloseTo(expected, 12);
|
||||
expect(ledger.allGreen!.verdicts).toBe(latest.length);
|
||||
expect(ledger.allGreen!.lo).toBeLessThan(ledger.allGreen!.p);
|
||||
expect(ledger.allGreen!.hi).toBeGreaterThan(ledger.allGreen!.p);
|
||||
const text = formatRedLedger(ledger).join('\n');
|
||||
expect(text).toContain('all-green probability (approximation, assumes independent verdicts)');
|
||||
expect(text).toContain('rerun: bun run eval:pass-rates --reds');
|
||||
expect(text).toContain('37186854666 gate/full census: 1 red of');
|
||||
expect(text).toContain('class: timeout 1; cause: unrecorded 1');
|
||||
});
|
||||
|
||||
test('new records group reds by machine cause', () => {
|
||||
const l = redLedger([rec({ outcome: 'failed', failure_class: 'assertion', failure_cause: 'refusal' }), rec({ case: 'b' })]);
|
||||
expect(l.censuses[0]).toMatchObject({ byClass: { assertion: 1 }, byCause: { refusal: 1 } });
|
||||
});
|
||||
});
|
||||
|
||||
describe('triageRun (stub fetcher, no network)', () => {
|
||||
test('downloads only the slice a red names, reports an oversize slice, and shows values and history', () => {
|
||||
const cacheDir = fs.mkdtempSync(path.join(os.tmpdir(), 'triage-'));
|
||||
const downloaded: string[] = [];
|
||||
const fetcher: HistoryFetcher = {
|
||||
listRuns: () => [], downloadZip: () => { throw new Error('unused'); },
|
||||
listArtifacts: () => [{ id: 1, name: 'paid-plan', size: 100 }, { id: 2, name: 'paid-slice-5-a1', size: 10 }, { id: 3, name: 'paid-slice-7-a1', size: 70 * 1024 * 1024 },
|
||||
{ id: 4, name: 'paid-slice-9-a1', size: 10 }],
|
||||
};
|
||||
const download = (match: (name: string) => boolean) => {
|
||||
const names = ['paid-plan', 'paid-slice-5-a1', 'paid-slice-7-a1', 'paid-slice-9-a1'].filter(match);
|
||||
downloaded.push(...names);
|
||||
return names.map(name => {
|
||||
const dir = path.join(cacheDir, name);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
if (name === 'paid-plan') fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify({ entries: [
|
||||
{ file: 'test/skill-llm-eval.test.ts', slice: 5, status: 'planned' }, { file: 'test/big.test.ts', slice: 7, status: 'planned' },
|
||||
{ file: 'test/green.test.ts', slice: 9, status: 'planned' }] }));
|
||||
return dir;
|
||||
});
|
||||
};
|
||||
const lane = 'periodic/full census';
|
||||
const records = [
|
||||
rec({ run_id: '42', lane, case: 'qa-only/SKILL.md workflow', file: 'test/skill-llm-eval.test.ts', kind: 'judge', outcome: 'failed', failure_class: 'assertion',
|
||||
failure_cause: 'assertion', error: 'expect(received).toBeGreaterThanOrEqual(expected)',
|
||||
failure_detail: { judge: [{ dimension: 'clarity', mean: 3.67, threshold: 4, samples: 3, rationale: 'Clarity suffers from density.' }] } }),
|
||||
rec({ run_id: '41', lane, case: 'qa-only/SKILL.md workflow', file: 'test/skill-llm-eval.test.ts', kind: 'judge' }),
|
||||
rec({ run_id: '42', lane, case: 'big-case', file: 'test/big.test.ts', outcome: 'failed', failure_class: 'timeout' }),
|
||||
rec({ run_id: '42', lane, case: 'green', file: 'test/green.test.ts' }),
|
||||
];
|
||||
const lines = triageRun({ repo: 'o/r', run: { id: 42, attempt: 1, sha: 'abcdef1234567', branch: 'b', createdAt: 't' }, records, cacheDir, fetcher, download });
|
||||
fs.rmSync(cacheDir, { recursive: true, force: true });
|
||||
expect(downloaded).toEqual(['paid-plan', 'paid-slice-5-a1']);
|
||||
expect(lines[0]).toContain('run 42 (b @ abcdef123, t): 2 verdict red(s)');
|
||||
expect(lines).toContain('✗ qa-only/SKILL.md workflow periodic/full census FAIL (history: 1/2 verdicts green)');
|
||||
expect(lines.join('\n')).toContain('t1: assertion / cause assertion — expect(received).toBeGreaterThanOrEqual(expected) clarity 3.67 < 4 (3 samples) "Clarity suffers from density."');
|
||||
expect(lines.join('\n')).toContain(`evidence: ${path.join(cacheDir, 'paid-slice-5-a1', 'shards', 'skill-llm-eval')}`);
|
||||
expect(lines).toContain(' evidence: paid-slice-7-a1 too large to fetch (70 MB)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('eval:pass-rates CLI contract', () => {
|
||||
function withFakeGh(run: (env: NodeJS.ProcessEnv, called: () => boolean, dir: string) => void) {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'passrates-gh-'));
|
||||
const marker = path.join(dir, 'gh-called');
|
||||
fs.writeFileSync(path.join(dir, 'gh'), `#!/bin/sh\ntouch '${marker}'\nexit 1\n`, { mode: 0o755 });
|
||||
try {
|
||||
run({ ...process.env, PATH: `${dir}${path.delimiter}${process.env.PATH}`, GSTACK_STATE_ROOT: path.join(dir, 'state') }, () => fs.existsSync(marker), dir);
|
||||
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
}
|
||||
const cli = (args: string[], env: NodeJS.ProcessEnv) => spawnSync(process.execPath, [SCRIPT, ...args], { encoding: 'utf8', timeout: 30_000, env });
|
||||
|
||||
test('--help is offline; an unknown flag or case exits 2 before any fetch', () => {
|
||||
withFakeGh((env, called) => {
|
||||
const help = cli(['--help'], env);
|
||||
expect(help.status).toBe(0);
|
||||
for (const flag of ['--headroom', '--reds', '--run <id>', '--case <id>', '--gate', '--help']) expect(help.stdout).toContain(flag);
|
||||
for (const args of [['--bogus'], ['--case', 'no-such-case'], ['--runs', '0'], ['--headroom', '--reds'], ['--run']]) {
|
||||
const r = cli(args, env);
|
||||
expect(r.status, args.join(' ')).toBe(2);
|
||||
expect(r.stderr).toContain('Usage: bun run eval:pass-rates');
|
||||
}
|
||||
expect(called()).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
test('--case narrows the JSON and the alarms; --gate alarms above 85% of a session budget; scope is printed', () => {
|
||||
withFakeGh((env, called, dir) => {
|
||||
const drift = (id: string) => Array.from({ length: 12 }, (_, i) => rec({ case: id, kind: 'behavior', tier: 'periodic', panel: { n: 3, k: 2 },
|
||||
trial: (i % 3) + 1, run_id: String(Math.floor(i / 3)), outcome: i < 11 ? 'passed' : 'failed', ...(i < 11 ? {} : { failure_class: 'assertion' as const }) }));
|
||||
const slow = [1, 2, 3].map(run => rec({ case: 'plan-mode-no-op', run_id: String(run), sessions: [session(run === 1 ? 88_000 : 50_000)] }));
|
||||
fs.writeFileSync(path.join(dir, 'trial-outcomes.jsonl'), formatTrialOutcomes([...drift('plan-ceo-review-format-mode'), ...drift('plan-review-prosons-neutral-neg'), ...slow]));
|
||||
const json = cli(['--dir', dir, '--json', '--case', 'plan-review-prosons-neutral-neg'], env);
|
||||
expect(json.status, json.stderr).toBe(0);
|
||||
const parsed = JSON.parse(json.stdout);
|
||||
expect(parsed.scope).toBe(`local dirs ${dir}`);
|
||||
expect(parsed.cases.map((c: any) => c.case)).toEqual(['plan-review-prosons-neutral-neg']);
|
||||
expect(parsed.alarms.map((a: any) => a.case)).toEqual(['plan-review-prosons-neutral-neg']);
|
||||
const gate = cli(['--dir', dir, '--gate', '--case', 'plan-mode-no-op'], env);
|
||||
expect(gate.status).toBe(1);
|
||||
expect(gate.stdout).toContain(`scope: local dirs ${dir}`);
|
||||
expect(gate.stdout).toContain('[headroom] plan-mode-no-op session pty:x#1: max 88s of 100s (88%)');
|
||||
expect(gate.stdout).not.toContain('[drift]');
|
||||
const plain = cli(['--dir', dir, '--case', 'plan-mode-no-op'], env);
|
||||
expect(plain.status).toBe(0);
|
||||
expect(plain.stdout).not.toContain('[headroom]');
|
||||
expect(cli(['--dir', dir, '--headroom'], env).stdout).toContain('over 88% 88s/100s');
|
||||
expect(called()).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
test('flag parsing', () => {
|
||||
const known = (id: string) => id === 'a';
|
||||
expect(parsePassRatesArgs(['--run', '37198445662', '--case', 'a'], known)).toMatchObject({ view: 'run', runId: 37198445662, caseFilter: 'a' });
|
||||
expect(parsePassRatesArgs(['--run', '1', '--dir', 'x'], known)).toEqual({ error: '--run reads one GitHub run; it cannot combine with --dir' });
|
||||
expect(parsePassRatesArgs(['-h'], known)).toEqual({ help: true });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1 @@
|
||||
{"source":"trial-outcomes of censuses 37174266054..37198445662 (wave-trials.jsonl): every trial of each red verdict, plus per census lane the count of other verdicts","censuses":[{"run":"37174266054","lane":"gate/full census","otherVerdicts":105},{"run":"37174266054","lane":"periodic/full census","otherVerdicts":144},{"run":"37176837432","lane":"gate/full census","otherVerdicts":106},{"run":"37176837432","lane":"periodic/full census","otherVerdicts":142},{"run":"37179171083","lane":"gate/full census","otherVerdicts":105},{"run":"37179171083","lane":"periodic/full census","otherVerdicts":142},{"run":"37182865432","lane":"gate/full census","otherVerdicts":106},{"run":"37182865432","lane":"periodic/full census","otherVerdicts":143},{"run":"37186854666","lane":"gate/full census","otherVerdicts":105},{"run":"37186854666","lane":"periodic/full census","otherVerdicts":144},{"run":"37193478719","lane":"gate/full census","otherVerdicts":106},{"run":"37193478719","lane":"periodic/full census","otherVerdicts":142},{"run":"37195203538","lane":"gate/full census","otherVerdicts":106},{"run":"37195203538","lane":"periodic/full census","otherVerdicts":143},{"run":"37198445662","lane":"gate/full census","otherVerdicts":106},{"run":"37198445662","lane":"periodic/full census","otherVerdicts":144}],"records":[{"schema":"gstack-trial-outcome/v1","tier":"gate","attempt":1,"policy_version":1,"run_id":"37174266054","lane":"gate/full census","case":"shared-libs-read-only","file":"test/skill-e2e-shared-libs.test.ts","kind":"rule","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","exit_reason":"assertion_failed","error":"Error: expect(received).toEqual(expected)","duration_ms":175687,"cost_usd":1.16,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37174266054","lane":"periodic/full census","case":"plan-eng-review-artifact","file":"test/skill-e2e-plan-eng-artifact.test.ts","kind":"rule","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","error":"plan-eng-review-artifact: outcome=timeout (no terminal outcome within 600000ms total budget (including startup and 5000ms cleanup reserve; step0=6, review=0); idleFor=234ms)","duration_ms":595026,"cost_usd":0,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37174266054","lane":"periodic/full census","case":"shared-libs-plan-callers","file":"test/skill-e2e-shared-libs-periodic.test.ts","kind":"rule","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","exit_reason":"actor_contract","error":"Error: shared-libs-plan-callers actor: question expands beyond unchanged-helper reuse and its required proof: hardening","duration_ms":199864,"cost_usd":0,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37176837432","lane":"periodic/full census","case":"scrape-prototype-path","file":"test/skill-e2e-skillify.test.ts","kind":"behavior","trial":1,"panel":{"n":3,"k":2},"outcome":"failed","failure_class":"assertion","exit_reason":"success","duration_ms":39393,"cost_usd":0.37,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37176837432","lane":"periodic/full census","case":"scrape-prototype-path","file":"test/skill-e2e-skillify.test.ts","kind":"behavior","trial":2,"panel":{"n":3,"k":2},"outcome":"passed","duration_ms":34525,"cost_usd":0.35,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37176837432","lane":"periodic/full census","case":"scrape-prototype-path","file":"test/skill-e2e-skillify.test.ts","kind":"behavior","trial":3,"panel":{"n":3,"k":2},"outcome":"passed","duration_ms":38454,"cost_usd":0.37,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37176837432","lane":"periodic/full census","case":"plan-ceo-mode-routing","file":"test/skill-e2e-plan-ceo-mode-routing.test.ts","kind":"rule","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","error":"Mode \"HOLD SCOPE\" routing FAILED after sending option 2: no posture match for \\b(rigor|bulletproof|hold\\s*scope|maximum\\s+rigor)\\b.","duration_ms":636550,"cost_usd":0,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37176837432","lane":"periodic/full census","case":"setup-deploy/SKILL.md platform setup","file":"test/skill-llm-eval.test.ts","kind":"judge","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_clasLine truncated
|
||||
Reference in new issue
Block a user