diff --git a/scripts/eval-flake-rank.ts b/scripts/eval-flake-rank.ts index e2ee502f1..de8e17664 100644 --- a/scripts/eval-flake-rank.ts +++ b/scripts/eval-flake-rank.ts @@ -36,6 +36,8 @@ * bun run eval:pass-rates --dir # local eval dirs / downloaded artifacts (repeatable) * bun run eval:pass-rates --backfill # also import legacy slice artifacts, labeled pre-policy * bun run eval:pass-rates --json | --gate + * bun run eval:pass-rates --headroom | --reds | --run # report views (scripts/lib/eval-history.ts) + * bun run eval:pass-rates --help # every flag, no network */ import * as fs from 'node:fs'; @@ -51,6 +53,8 @@ import { CASE_QUARANTINE, EVAL_POLICY } from '../test/helpers/periodic-exclude-d import { matchGlob } from '../test/helpers/test-selection'; import { CASE_TEST_NAMES } from './test-paid-shards'; import { resolveStateRoot } from '../lib/state-root'; +import { GH_JOBS, PASS_RATES_USAGE, criticalPath, formatCriticalPath, formatHeadroom, formatRedLedger, headroom, headroomAlarms, parsePassRatesArgs, + redLedger, triageRun, type CriticalPath } from './lib/eval-history'; interface TestSeries { name: string; @@ -316,7 +320,7 @@ export function holmRejections(pValues: number[], alpha: number): Set { export type PassRateLabel = 'INCONCLUSIVE' | 'BROKEN' | 'FLAKY' | 'FAILING' | 'PASSING'; export type AlarmKind = 'drift' | 'rule-as-behavior' | 'regression' | 'quarantine-exit' | 'quarantine-expired' - | 'quarantine-cap' | 'quarantine-invalid'; + | 'quarantine-cap' | 'quarantine-invalid' | 'headroom'; /** The EVAL_POLICY fields pass-rates reads (structural, so tests can vary them). */ export interface PassRatePolicy { @@ -639,24 +643,22 @@ function repoSlug(): string { } if (import.meta.main) { - const argv = process.argv.slice(2); - const flag = (name: string) => { const index = argv.indexOf(name); return index === -1 ? undefined : argv[index + 1]; }; - const dirs = argv.flatMap((arg, index) => arg === '--dir' && argv[index + 1] ? [argv[index + 1]!] : []); - const asJson = argv.includes('--json'); - const gate = argv.includes('--gate'); - const backfill = argv.includes('--backfill'); - const caseFilter = flag('--case'); - const runsLimit = Number(flag('--runs')) || 10; - const sinceDays = Number(flag('--since-days')) || 60; - const repo = flag('--repo') ?? repoSlug(); - const workflow = flag('--workflow') ?? 'evals-periodic.yml'; - const branch = flag('--branch') ?? gitOutput(['rev-parse', '--abbrev-ref', 'HEAD']) ?? 'main'; + const parsed = parsePassRatesArgs(process.argv.slice(2), id => Object.hasOwn(E2E_TIERS, id) || Object.hasOwn(LLM_JUDGE_TOUCHFILES, id) + || (/^test\/[\w./-]+\.test\.ts$/.test(id) && fs.existsSync(id))); + if ('help' in parsed) { console.log(PASS_RATES_USAGE); process.exit(0); } + if ('error' in parsed) { console.error(`eval:pass-rates: ${parsed.error}\n${PASS_RATES_USAGE}`); process.exit(2); } + const { dirs, json: asJson, gate, backfill, caseFilter, runs: runsLimit, sinceDays, view } = parsed; + const repo = parsed.repo ?? repoSlug(); + const workflow = parsed.workflow ?? 'evals-periodic.yml'; + const branch = parsed.branch ?? gitOutput(['rev-parse', '--abbrev-ref', 'HEAD']) ?? 'main'; const records: TrialRecord[] = []; const unattributed = new Set(); const errors: string[] = []; let historyError: string | null = null; let weeklyRuns: string[] | undefined; + let fetched: WeeklyRun[] = []; + const cacheDir = path.join(path.resolve(resolveStateRoot()), 'eval-pass-rates-cache', repo.replace('/', '-')); const manualReviews: string[] = []; const importDir = (dir: string, run: { run_id: string; sha?: string; timestamp?: string } | undefined, legacyDays: number) => { @@ -674,8 +676,9 @@ if (import.meta.main) { } else { try { const runs = listWeeklyRuns({ repo, workflow, branches: [...new Set([branch, 'main'])], limit: runsLimit }); + if (parsed.runId !== undefined && !runs.some(run => run.id === parsed.runId)) runs.push(GH_JOBS.getRun(repo, parsed.runId)); weeklyRuns = runs.map(run => run.createdAt); - const cacheDir = path.join(path.resolve(resolveStateRoot()), 'eval-pass-rates-cache', repo.replace('/', '-')); + fetched = runs; const match = backfill ? (name: string) => name.startsWith('trial-outcomes') || /^(paid-slice-\d+|gate-census-\d+)(-a\d+)?$/.test(name) : (name: string) => name.startsWith('trial-outcomes'); @@ -688,11 +691,37 @@ if (import.meta.main) { } } - const report = analyzePassRates(records, { weeklyRuns, unattributed: [...unattributed].sort(), errors, manualReviews }); + const scope = dirs.length ? `local dirs ${dirs.join(', ')}` : `${repo} ${workflow} on ${[...new Set([branch, 'main'])].join(' + ')}, ` + + `last ${runsLimit} completed run(s) per branch: ${fetched.map(run => run.id).join(', ') || 'none'}`; + const scoped = caseFilter ? records.filter(record => record.case === caseFilter) : records; + const caseHeadroom = headroom(scoped); + if (view !== 'rates') { + const critical: CriticalPath[] = []; + if (view !== 'reds' && !dirs.length) { + for (const run of fetched.filter(run => view === 'headroom' || run.id === parsed.runId)) { + try { critical.push(criticalPath(String(run.id), GH_JOBS.listJobs(repo, run.id))); } catch { critical.push({ run: String(run.id), jobs: 0, slowest: null, wallMs: null }); } + } + } + const lines = view === 'headroom' ? formatHeadroom(caseHeadroom, critical) : view === 'reds' ? formatRedLedger(redLedger(scoped)) + : historyError ? [] : triageRun({ repo, run: fetched.find(run => run.id === parsed.runId)!, records, caseFilter, cacheDir, fetcher: GH_HISTORY, + download: (match, maxBytes) => downloadRunArtifacts({ repo, run: fetched.find(run => run.id === parsed.runId)!, match, cacheDir, maxBytes }) }) + .concat(formatCriticalPath(critical)); + if (asJson) console.log(JSON.stringify({ scope, historyError, view, lines }, null, 2)); + else { + console.log(`scope: ${scope}`); + if (historyError) console.log(`history unavailable (${historyError})`); + for (const line of lines) console.log(line); + } + process.exit(historyError ? 1 : 0); + } + const full = analyzePassRates(records, { weeklyRuns, unattributed: [...unattributed].sort(), errors, manualReviews }); + const alarms = [...full.alarms, ...(gate ? headroomAlarms(caseHeadroom) : [])].filter(alarm => !caseFilter || alarm.case === caseFilter); + const report = { ...full, cases: full.cases.filter(entry => !caseFilter || entry.case === caseFilter), alarms }; const ledger = readFreeLedger(); if (asJson) { - console.log(JSON.stringify({ repo, workflow, branch, dirs, historyError, ...report, freeLedger: ledger }, null, 2)); + console.log(JSON.stringify({ repo, workflow, branch, scope, dirs, historyError, ...report, freeLedger: ledger }, null, 2)); } else { + console.log(`scope: ${scope}`); if (historyError) console.log(`pass-rates: history unavailable (${historyError}); every label below is INCONCLUSIVE`); console.log(formatPassRates(report, { caseFilter })); if (ledger.length > 0) { diff --git a/scripts/lib/eval-history.ts b/scripts/lib/eval-history.ts new file mode 100644 index 000000000..d02e327d0 --- /dev/null +++ b/scripts/lib/eval-history.ts @@ -0,0 +1,350 @@ +/** + * eval:pass-rates report views over stored trial records (plan 0.1): + * session headroom (--headroom), the verdict red ledger with the all-green + * probability (--reds), and one census's triage view (--run ). Pure + * functions plus the run-scoped fetches they need; scripts/eval-flake-rank.ts + * parses flags, loads history and dispatches here. + * + * Headroom is per armed session (session ledger, `sessions[]` on a record): + * elapsed over the timeout it armed. A case's headroom is its slowest + * session; a timed-out sample is censored (its true duration is unknown). + * Records without sessions are unknown, never scored; their case wall + * (JUnit or shard clock) is shown separately as an upper bound. + * + * The all-green probability is an approximation that assumes independent + * verdicts: P(green) = Π (1 - p_i) over the cases a census verdicts, where + * p_i = (reds_i + K * p̄) / (verdicts_i + K) shrinks each case's red rate + * toward the pooled rate p̄. Its interval recomputes the same product with p̄ + * replaced by the pooled rate's 95% Wilson bounds. + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { panelVerdict, type TrialOutcomeRecord } from '../../test/helpers/eval-store'; +import { HEADROOM_FAIL, HEADROOM_WARN } from '../../test/helpers/eval-budgets'; +import { sanitizePublishedText } from './published-text'; +import type { HistoryFetcher, WeeklyRun } from '../eval-flake-rank'; +import { CASE_SHARDED_FILES, trialShardKey } from './paid-cases'; +import { shardSlug } from '../test-paid-shards'; + +/** Below this many session samples a case's headroom is insufficient, never a pass. */ +export const HEADROOM_MIN_SAMPLES = 3; +/** Pseudo-verdicts of pooled rate added to each case's red rate. */ +export const RED_RATE_SHRINK = 4; +/** Largest slice artifact --run downloads. */ +export const EVIDENCE_MAX_BYTES = 64 * 1024 * 1024; + +type Rec = TrialOutcomeRecord; +const pct = (x: number) => `${Math.round(x * 1000) / 10}%`; +const secs = (ms: number) => `${Math.round(ms / 1000)}s`; + +// --- Headroom --- + +export interface SessionHeadroom { key: string; samples: number; censored: number; maxRatio: number; maxElapsedMs: number; budgetMs: number } +export interface CaseHeadroom { + case: string; + status: 'over' | 'warn' | 'ok' | 'insufficient' | 'unknown'; + sessions: SessionHeadroom[]; + /** Slowest session's worst ratio, when any session was recorded. */ + worst?: SessionHeadroom; + /** Largest case wall (JUnit or shard clock) over the window: an upper bound, never a session clock. */ + caseWallMs: number; + runs: number; +} + +export function headroom(records: readonly Rec[]): CaseHeadroom[] { + const byCase = new Map(); + for (const r of records) if (r.outcome !== 'skipped' && r.execution !== 'reused') byCase.set(r.case, [...(byCase.get(r.case) ?? []), r]); + return [...byCase.entries()].map(([id, rs]) => { + const sessions = new Map(); + for (const s of rs.flatMap(r => r.sessions ?? [])) { + if (!s.budget_ms) continue; + const key = s.key; + const entry = sessions.get(key) ?? { key, samples: 0, censored: 0, maxRatio: 0, maxElapsedMs: 0, budgetMs: s.budget_ms }; + entry.samples++; + if (s.end === 'session_timeout' || s.end === 'observer_timeout') entry.censored++; + const ratio = s.elapsed_ms / s.budget_ms; + if (ratio > entry.maxRatio) Object.assign(entry, { maxRatio: ratio, maxElapsedMs: s.elapsed_ms, budgetMs: s.budget_ms }); + sessions.set(key, entry); + } + const list = [...sessions.values()].sort((a, b) => b.maxRatio - a.maxRatio); + const worst = list[0]; + const status: CaseHeadroom['status'] = !worst ? 'unknown' : worst.samples < HEADROOM_MIN_SAMPLES ? 'insufficient' + : worst.maxRatio > HEADROOM_FAIL ? 'over' : worst.maxRatio > HEADROOM_WARN ? 'warn' : 'ok'; + return { case: id, status, sessions: list, ...(worst ? { worst } : {}), caseWallMs: Math.max(0, ...rs.map(r => r.duration_ms)), + runs: new Set(rs.map(r => r.run_id ?? 'local')).size }; + }).sort((a, b) => (b.worst?.maxRatio ?? -1) - (a.worst?.maxRatio ?? -1) || b.caseWallMs - a.caseWallMs); +} + +/** --gate alarms: a case whose slowest session ran above HEADROOM_FAIL of its armed budget. */ +export function headroomAlarms(cases: readonly CaseHeadroom[]): Array<{ kind: 'headroom'; case: string; message: string }> { + return cases.filter(c => c.status === 'over').map(c => ({ kind: 'headroom' as const, case: c.case, + message: `${c.case} session ${c.worst!.key}: max ${secs(c.worst!.maxElapsedMs)} of ${secs(c.worst!.budgetMs)} (${pct(c.worst!.maxRatio)})` + + `${c.worst!.censored ? `, ${c.worst!.censored} timed-out sample(s) censored` : ''} over ${c.worst!.samples} sample(s), above the ${pct(HEADROOM_FAIL)} cap. ` + + 'Cut work in the skill or fixture; budgets are never raised (TESTING_INTERNALS "Timeout policy").' })); +} + +export function formatHeadroom(cases: readonly CaseHeadroom[], critical: readonly CriticalPath[] = []): string[] { + const lines = [`headroom: slowest armed session per case (warn > ${pct(HEADROOM_WARN)}, alarm > ${pct(HEADROOM_FAIL)}; ` + + `insufficient below ${HEADROOM_MIN_SAMPLES} samples; censored = timed out)`, + ' status session max/budget samples case wall (upper bound, not a session clock) case']; + for (const c of cases) { + const w = c.worst; + const session = w ? `${pct(w.maxRatio)} ${secs(w.maxElapsedMs)}/${secs(w.budgetMs)}` : 'unknown (no session ledger)'; + lines.push(` ${c.status.padEnd(12)} ${session.padEnd(26)} ${(w ? `${w.samples}${w.censored ? ` (${w.censored} censored)` : ''}` : '-').padEnd(7)} ` + + `${secs(c.caseWallMs).padEnd(45)} ${c.case}${w ? ` [${w.key}]` : ''}`); + } + return [...lines, ...formatCriticalPath(critical)]; +} + +export function formatCriticalPath(critical: readonly CriticalPath[]): string[] { + return critical.map(cp => `critical path ${cp.run}: ${cp.slowest ? `${cp.slowest.name} ${secs(cp.slowest.ms)}` : 'unknown'}` + + ` of ${cp.jobs} slice job(s), run wall ${cp.wallMs === null ? 'unknown' : secs(cp.wallMs)}${cp.files?.length ? `; slice files: ${cp.files.join(', ')}` : ''}`); +} + +// --- Critical path from runner slice timings --- + +export interface JobTiming { name: string; startedAt: string | null; completedAt: string | null } +export interface CriticalPath { run: string; jobs: number; slowest: { name: string; ms: number } | null; wallMs: number | null; files?: string[] } + +const SLICE_JOB = /^(eval-slices|eval-codex-slices|gate-census) \((\d+)\)$/; + +export function criticalPath(run: string, jobs: readonly JobTiming[], plan?: { entries: Array<{ file: string; slice?: number; status?: string }> }): CriticalPath { + const timed = jobs.filter(j => SLICE_JOB.test(j.name) && j.startedAt && j.completedAt) + .map(j => ({ name: j.name, ms: Date.parse(j.completedAt!) - Date.parse(j.startedAt!) })); + const slowest = timed.sort((a, b) => b.ms - a.ms)[0] ?? null; + const all = jobs.filter(j => j.startedAt && j.completedAt); + const wallMs = all.length ? Math.max(...all.map(j => Date.parse(j.completedAt!))) - Math.min(...all.map(j => Date.parse(j.startedAt!))) : null; + const slice = slowest ? Number(SLICE_JOB.exec(slowest.name)![2]) : null; + const files = plan && slice !== null && slowest!.name.startsWith('gate-census') + ? plan.entries.filter(e => e.slice === slice && e.status === 'planned').map(e => e.file) : undefined; + return { run, jobs: timed.length, slowest, wallMs, ...(files ? { files } : {}) }; +} + +// --- Red ledger --- + +export interface VerdictRed { run: string; lane: string; case: string; status: string; trials: Rec[] } +export interface CensusReds { run: string; lane: string; verdicts: number; reds: VerdictRed[]; byClass: Record; byCause: Record } +export interface RedLedger { + censuses: CensusReds[]; + pooled: { verdicts: number; reds: number; rate: number; lo: number; hi: number }; + perCase: Array<{ case: string; verdicts: number; reds: number; rate: number }>; + allGreen: { p: number; lo: number; hi: number; verdicts: number } | null; +} + +function wilson(k: number, n: number, z = 1.96): { lo: number; hi: number } { + if (n === 0) return { lo: 0, hi: 1 }; + const p = k / n, d = 1 + z * z / n, c = p + z * z / (2 * n), m = z * Math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)); + return { lo: Math.max(0, (c - m) / d), hi: Math.min(1, (c + m) / d) }; +} + +/** One panelVerdict per case per census lane (earliest attempt), exactly as the report computes it. */ +export function verdictsOf(records: readonly Rec[]): Array<{ run: string; lane: string; case: string; status: string; failsLane: boolean; trials: Rec[] }> { + const groups = new Map(); + for (const r of records) { + if (r.policy_version < 1 || r.execution === 'reused') continue; + // A JUnit case is one verdict per shard file, as the report counts it; trial shards form one panel. + const key = `${r.run_id ?? 'local'}\0${r.lane ?? r.tier}\0${r.case}\0${r.source === 'shard' ? '' : r.file}`; + groups.set(key, [...(groups.get(key) ?? []), r]); + } + return [...groups.entries()].map(([key, rs]) => { + const [run, lane, id] = key.split('\0') as [string, string, string, string]; + const attempt = Math.min(...rs.map(r => r.attempt)); + const trials = rs.filter(r => r.attempt === attempt); + const v = panelVerdict({ case: id, kind: trials[0]!.kind, panel: trials[0]!.panel, quarantined: trials[0]!.quarantined, + trials: trials.map(t => ({ trial: t.trial, outcome: t.outcome, attempt, ...(t.failure_class ? { failure_class: t.failure_class } : {}) })) }); + return { run, lane, case: id, status: v.status, failsLane: v.failsLane, trials }; + }); +} + +export function redLedger(records: readonly Rec[]): RedLedger { + const verdicts = verdictsOf(records).filter(v => v.status !== 'SKIPPED'); + const censusKey = (v: { run: string; lane: string }) => `${v.run}\0${v.lane}`; + const censuses = new Map(); + for (const v of verdicts) { + const c = censuses.get(censusKey(v)) ?? { run: v.run, lane: v.lane, verdicts: 0, reds: [], byClass: {}, byCause: {} }; + c.verdicts++; + if (v.failsLane) { + c.reds.push({ run: v.run, lane: v.lane, case: v.case, status: v.status, trials: v.trials }); + for (const t of v.trials.filter(t => t.outcome === 'failed')) { + c.byClass[t.failure_class ?? 'assertion'] = (c.byClass[t.failure_class ?? 'assertion'] ?? 0) + 1; + c.byCause[t.failure_cause ?? 'unrecorded'] = (c.byCause[t.failure_cause ?? 'unrecorded'] ?? 0) + 1; + } + } + censuses.set(censusKey(v), c); + } + const reds = verdicts.filter(v => v.failsLane).length; + const rate = verdicts.length ? reds / verdicts.length : 0; + const pooled = { verdicts: verdicts.length, reds, rate, ...wilson(reds, verdicts.length) }; + const byCase = new Map(); + for (const v of verdicts) { + const e = byCase.get(v.case) ?? { verdicts: 0, reds: 0 }; + e.verdicts++; if (v.failsLane) e.reds++; + byCase.set(v.case, e); + } + const perCase = [...byCase.entries()].map(([id, e]) => ({ case: id, ...e, rate: (e.reds + RED_RATE_SHRINK * rate) / (e.verdicts + RED_RATE_SHRINK) })) + .sort((a, b) => b.rate - a.rate || (a.case < b.case ? -1 : 1)); + const ordered = [...censuses.values()]; + const latestRun = ordered.map(c => c.run).sort().at(-1); + const census = verdicts.filter(v => v.run === latestRun); + const green = (prior: number) => census.reduce((p, v) => { + const e = byCase.get(v.case)!; + return p * (1 - (e.reds + RED_RATE_SHRINK * prior) / (e.verdicts + RED_RATE_SHRINK)); + }, 1); + const allGreen = census.length ? { p: green(rate), lo: green(pooled.hi), hi: green(pooled.lo), verdicts: census.length } : null; + return { censuses: ordered.sort((a, b) => (a.run < b.run ? -1 : a.run > b.run ? 1 : a.lane < b.lane ? -1 : 1)), pooled, perCase, allGreen }; +} + +const tally = (counts: Record) => Object.entries(counts).sort(([a, x], [b, y]) => y - x || (a < b ? -1 : 1)).map(([k, n]) => `${k} ${n}`).join(', ') || 'none'; + +export function formatRedLedger(ledger: RedLedger, opts: { top?: number } = {}): string[] { + const lines = ['reds: verdict reds per census lane by failure class and machine cause (cause "unrecorded" = written before failure_cause existed)']; + for (const c of ledger.censuses) { + lines.push(` ${c.run} ${c.lane}: ${c.reds.length} red of ${c.verdicts} verdicts${c.reds.length ? ` — class: ${tally(c.byClass)}; cause: ${tally(c.byCause)}` : ''}`); + for (const red of c.reds) { + const failed = red.trials.filter(t => t.outcome === 'failed'); + lines.push(` ✗ ${red.case} ${red.status} ${failed.map(t => `t${t.trial} ${t.failure_class ?? 'assertion'}/${t.failure_cause ?? 'unrecorded'}` + + `${t.error ? ` — ${sanitizePublishedText(t.error, 160)}` : ''}`).join('; ')}`); + } + } + const p = ledger.pooled; + lines.push(`pooled: ${p.reds} red of ${p.verdicts} verdicts = ${pct(p.rate)} per verdict [95% ${pct(p.lo)}–${pct(p.hi)}]`); + const top = ledger.perCase.filter(c => c.reds > 0).slice(0, opts.top ?? 15); + if (top.length) lines.push(`per-case red rates (shrunk toward the pooled rate, K=${RED_RATE_SHRINK}):`, ...top.map(c => ` ${pct(c.rate).padStart(6)} ${c.reds}/${c.verdicts} ${c.case}`)); + const g = ledger.allGreen; + lines.push(g ? `all-green probability (approximation, assumes independent verdicts): ${pct(g.p)} for the latest census's ${g.verdicts} verdicts [${pct(g.lo)}–${pct(g.hi)} with the pooled rate's 95% interval as prior]; ` + + 'formula Π(1 - p_i) over its cases; rerun: bun run eval:pass-rates --reds' : 'all-green probability: no post-policy verdicts'); + return lines; +} + +// --- One census's triage view --- + +/** Fetches the report views make beyond the history loader: a named run and its slice jobs. */ +export interface RunJobsFetcher { listJobs(repo: string, runId: number): JobTiming[]; getRun(repo: string, runId: number): WeeklyRun } + +function ghJson(args: string[]): string[] { + const result = spawnSync('gh', args, { timeout: 120_000, maxBuffer: 64 * 1024 * 1024 }); + if (result.status !== 0) throw new Error(`gh ${args.slice(0, 2).join(' ')} failed: ${String(result.stderr || result.error || '').trim()}`); + return result.stdout.toString('utf8').split('\n').filter(Boolean); +} + +export const GH_JOBS: RunJobsFetcher = { + listJobs: (repo, runId) => ghJson(['api', `repos/${repo}/actions/runs/${runId}/jobs?per_page=100`, '--paginate', + '--jq', '.jobs[] | {name, startedAt: .started_at, completedAt: .completed_at}']).map(line => JSON.parse(line) as JobTiming), + getRun: (repo, runId) => JSON.parse(ghJson(['api', `repos/${repo}/actions/runs/${runId}`, + '--jq', '{id, attempt: .run_attempt, sha: .head_sha, branch: .head_branch, createdAt: .created_at}'])[0]!) as WeeklyRun, +}; + +/** The manifest shard key a record ran under: its trial shard, its case shard, or its file. */ +export function recordShardKey(r: Pick): string { + return r.source === 'shard' ? trialShardKey(r.file, r.case, r.trial) : CASE_SHARDED_FILES.includes(r.file) ? `${r.file}#${r.case}` : r.file; +} + +/** + * The run's reds with their values and history, and the slice artifacts that + * hold their transcripts. Only the slices a red names are downloaded; a slice + * over EVIDENCE_MAX_BYTES is reported, never silently skipped. + */ +export function triageRun(opts: { repo: string; run: WeeklyRun; records: readonly Rec[]; caseFilter?: string; cacheDir: string; + fetcher: HistoryFetcher; download: (match: (name: string) => boolean, maxBytes: number) => string[] }): string[] { + const runId = String(opts.run.id); + const mine = opts.records.filter(r => r.run_id === runId); + const reds = verdictsOf(mine).filter(v => v.failsLane && (!opts.caseFilter || v.case === opts.caseFilter)); + const lines = [`run ${runId} (${opts.run.branch} @ ${opts.run.sha.slice(0, 9)}, ${opts.run.createdAt}): ${reds.length} verdict red(s)${opts.caseFilter ? ` for ${opts.caseFilter}` : ''}`]; + const history = verdictsOf(opts.records); + const plans = new Map>(); + const planFor = (lane: string) => { + const name = lane.startsWith('gate') && lane.includes('census') ? 'gate-census-plan' : 'paid-plan'; + if (!plans.has(name)) { + const files = new Map(); + for (const dir of opts.download(n => n === name, 8 * 1024 * 1024)) { + try { + const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'manifest.json'), 'utf8')); + for (const e of manifest.entries ?? []) if (e.status === 'planned' && Number.isInteger(e.slice)) files.set(e.file, e.slice); + } catch { /* no manifest: slices stay unknown */ } + } + plans.set(name, files); + } + return { prefix: name === 'gate-census-plan' ? 'gate-census' : 'paid-slice', files: plans.get(name)! }; + }; + const artifacts = opts.fetcher.listArtifacts(opts.repo, opts.run.id); + for (const red of reds) { + const runs = history.filter(v => v.case === red.case); + lines.push(`✗ ${red.case} ${red.lane} ${red.status} (history: ${runs.filter(v => !v.failsLane).length}/${runs.length} verdicts green)`); + for (const t of red.trials.filter(t => t.outcome === 'failed')) { + const d = t.failure_detail; + const detail = !d ? '' : 'judge' in d ? ` ${d.judge.map(j => `${j.dimension} ${j.mean} < ${j.threshold} (${j.samples} samples)${j.rationale ? ` "${j.rationale}"` : ''}`).join('; ')}` + : ` Expected: ${d.expected} · Received: ${d.received}`; + lines.push(sanitizePublishedText(` t${t.trial}: ${t.failure_class ?? 'assertion'} / cause ${t.failure_cause ?? 'unrecorded'}` + + `${t.failure_cause_evidence ? `: ${t.failure_cause_evidence}` : ''}${t.error ? ` — ${t.error}` : ''}${detail}`, 2000)); + const plan = planFor(red.lane); + const slice = plan.files.get(recordShardKey(t)); + if (slice === undefined) { lines.push(' evidence: slice unknown (no plan artifact for this lane)'); continue; } + const name = `${plan.prefix}-${slice}-a${t.attempt}`; + const artifact = artifacts.find(a => a.name === name); + if (!artifact) { lines.push(` evidence: ${name} (expired or missing)`); continue; } + if (artifact.size > EVIDENCE_MAX_BYTES) { lines.push(` evidence: ${name} too large to fetch (${Math.round(artifact.size / 1024 / 1024)} MB)`); continue; } + const [dir] = opts.download(n => n === name, EVIDENCE_MAX_BYTES); + lines.push(` evidence: ${dir ? path.join(dir, 'shards', shardSlug([recordShardKey(t)])) : `${name} (download failed)`}`); + } + } + return lines; +} + +// --- Flags --- + +export const PASS_RATES_USAGE = `Usage: bun run eval:pass-rates [options] + (no view flag) per-case trial pass rates with 95% Wilson intervals + --headroom slowest armed session per case vs its budget, case wall upper bound, census critical path + --reds verdict reds per census by failure class and cause, pooled and per-case rates, all-green probability + --run one census: each red with its values, history and fetched transcript evidence + --case restrict every view, the JSON and the alarms to one registered case + --runs completed runs per branch to read (default 10) + --branch branch whose runs to read beside main (default: current branch) + --repo --workflow --dir (repeatable, local artifacts; no network) + --backfill also import legacy slice artifacts, labeled pre-policy + --since-days legacy eval files window for --dir (default 60) + --json machine-readable output + --gate exit 1 on ACTION REQUIRED (pass-rate alarms and sessions above ${pct(HEADROOM_FAIL)} of budget) + --help this text (no network)`; + +export interface PassRatesArgs { + view: 'rates' | 'headroom' | 'reds' | 'run'; + runId?: number; caseFilter?: string; runs: number; sinceDays: number; dirs: string[]; + json: boolean; gate: boolean; backfill: boolean; repo?: string; workflow?: string; branch?: string; +} + +/** Parse eval:pass-rates flags; unknown flags, bad values and unknown case ids are errors (exit 2, before any fetch). */ +export function parsePassRatesArgs(argv: readonly string[], isKnownCase: (id: string) => boolean): { help: true } | { error: string } | PassRatesArgs { + const out: PassRatesArgs = { view: 'rates', runs: 10, sinceDays: 60, dirs: [], json: false, gate: false, backfill: false }; + const views: string[] = []; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]!; + const value = () => { const v = argv[++i]; if (v === undefined || v.startsWith('--')) throw new Error(`${arg} needs a value`); return v; }; + const count = () => { const v = value(); if (!/^[1-9]\d*$/.test(v)) throw new Error(`${arg} needs a positive integer, got ${v}`); return Number(v); }; + try { + switch (arg) { + case '--help': case '-h': return { help: true }; + case '--headroom': views.push('headroom'); break; + case '--reds': views.push('reds'); break; + case '--run': views.push('run'); out.runId = count(); break; + case '--case': out.caseFilter = value(); break; + case '--runs': out.runs = count(); break; + case '--since-days': out.sinceDays = count(); break; + case '--dir': out.dirs.push(value()); break; + case '--repo': out.repo = value(); break; + case '--workflow': out.workflow = value(); break; + case '--branch': out.branch = value(); break; + case '--json': out.json = true; break; + case '--gate': out.gate = true; break; + case '--backfill': out.backfill = true; break; + default: return { error: `unknown flag ${arg}` }; + } + } catch (error) { return { error: (error as Error).message }; } + } + if (views.length > 1) return { error: `choose one of --headroom, --reds, --run (got ${views.join(', ')})` }; + if (views[0]) out.view = views[0] as PassRatesArgs['view']; + if (out.caseFilter !== undefined && !isKnownCase(out.caseFilter)) return { error: `unknown case ${out.caseFilter} (not an E2E, judge or paid test file id)` }; + if (out.view === 'run' && out.dirs.length) return { error: '--run reads one GitHub run; it cannot combine with --dir' }; + return out; +} diff --git a/test/eval-history.test.ts b/test/eval-history.test.ts new file mode 100644 index 000000000..7d25b7439 --- /dev/null +++ b/test/eval-history.test.ts @@ -0,0 +1,208 @@ +/** + * eval:pass-rates report views (plan 0.1): session headroom, the verdict red + * ledger with the all-green probability, one census's triage view, and the + * CLI contract (offline --help, unknown flag and case rejected before any + * fetch, --case applied to JSON and alarms, the 85% alarm in --gate). + */ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { TRIAL_OUTCOME_SCHEMA, formatTrialOutcomes, type TrialOutcomeRecord } from './helpers/eval-store'; +import { EVAL_POLICY } from './helpers/periodic-exclude-data'; +import { HEADROOM_FAIL } from './helpers/eval-budgets'; +import { criticalPath, formatHeadroom, formatRedLedger, headroom, headroomAlarms, parsePassRatesArgs, redLedger, triageRun, + verdictsOf, HEADROOM_MIN_SAMPLES, RED_RATE_SHRINK } from '../scripts/lib/eval-history'; +import type { HistoryFetcher } from '../scripts/eval-flake-rank'; + +const SCRIPT = path.resolve(import.meta.dir, '../scripts/eval-flake-rank.ts'); + +const rec = (over: Partial): TrialOutcomeRecord => ({ + schema: TRIAL_OUTCOME_SCHEMA, case: 'rule-a', file: 'test/a.test.ts', tier: 'gate', kind: 'rule', trial: 1, panel: { n: 1, k: 1 }, + attempt: 1, outcome: 'passed', duration_ms: 1000, cost_usd: 0, policy_version: EVAL_POLICY.version, quarantined: false, + execution: 'executed', source: 'junit', run_id: '1', lane: 'gate/full census', ...over, +}); +const session = (elapsed: number, budget = 100_000, end = 'completed') => ({ key: 'pty:x#1', runner: 'pty', elapsed_ms: elapsed, budget_ms: budget, end }); + +describe('headroom', () => { + const records = [ + ...[1, 2, 3].map(run => rec({ case: 'slow', run_id: String(run), duration_ms: 95_000, sessions: [session(run === 3 ? 88_000 : 60_000)] })), + ...[1, 2, 3].map(run => rec({ case: 'fine', run_id: String(run), sessions: [session(70_000), { ...session(10_000), key: 'claude-p:y#1' }] })), + ...[1, 2].map(run => rec({ case: 'rare', run_id: String(run), sessions: [session(99_000)] })), + rec({ case: 'legacy', duration_ms: 517_000 }), + rec({ case: 'timedout', run_id: '1', outcome: 'failed', failure_class: 'timeout', sessions: [session(100_050, 100_000, 'session_timeout')] }), + rec({ case: 'timedout', run_id: '2', sessions: [session(50_000)] }), rec({ case: 'timedout', run_id: '3', sessions: [session(50_000)] }), + rec({ case: 'skipped-only', outcome: 'skipped', sessions: [session(99_000)] }), + ]; + const cases = headroom(records); + const byCase = Object.fromEntries(cases.map(c => [c.case, c])); + + test('88% alarms, 70% passes, too few samples is insufficient, a timeout is censored, no ledger is unknown, skipped never counts', () => { + expect(byCase.slow).toMatchObject({ status: 'over', worst: { key: 'pty:x#1', samples: 3, maxRatio: 0.88, censored: 0 } }); + expect(byCase.fine).toMatchObject({ status: 'ok', worst: { maxRatio: 0.7 } }); + expect(byCase.rare!.status).toBe('insufficient'); + expect(HEADROOM_MIN_SAMPLES).toBe(3); + expect(byCase.timedout).toMatchObject({ status: 'over', worst: { censored: 1, samples: 3 } }); + expect(byCase.legacy).toMatchObject({ status: 'unknown', caseWallMs: 517_000 }); + expect(byCase['skipped-only']).toBeUndefined(); + expect(headroomAlarms(cases).map(a => a.case).sort()).toEqual(['slow', 'timedout']); + expect(headroomAlarms(cases)[0]!.message).toContain(`above the ${HEADROOM_FAIL * 100}% cap`); + }); + + test('the session column and the case wall column are labelled differently; unknown sessions are never scored', () => { + const lines = formatHeadroom(cases); + expect(lines[1]).toContain('session max/budget'); + expect(lines[1]).toContain('case wall (upper bound, not a session clock)'); + expect(lines.find(l => l.endsWith('legacy'))).toMatch(/^ {2}unknown\s+unknown \(no session ledger\)\s+-\s+517s/); + }); + + test('critical path is the slowest slice job; non-slice jobs only bound the run wall', () => { + const job = (name: string, start: number, end: number) => ({ name, startedAt: new Date(start * 1000).toISOString(), completedAt: new Date(end * 1000).toISOString() }); + const cp = criticalPath('9', [job('build-image', 0, 60), job('eval-slices (1)', 60, 400), job('gate-census (11)', 70, 781), job('report', 781, 840)], + { entries: [{ file: 'test/skill-e2e-plan-mode-no-op.test.ts', slice: 11, status: 'planned' }, { file: 'x', slice: 2, status: 'planned' }] }); + expect(cp).toEqual({ run: '9', jobs: 2, slowest: { name: 'gate-census (11)', ms: 711_000 }, wallMs: 840_000, files: ['test/skill-e2e-plan-mode-no-op.test.ts'] }); + }); +}); + +describe('red ledger over the 8 wave censuses', () => { + const fixture = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures', 'eval-history', 'wave-reds-8-censuses.json'), 'utf8')); + // Every other verdict of each census lane, as a passing rule case (same ids across censuses). + const passing = fixture.censuses.flatMap((c: any) => Array.from({ length: c.otherVerdicts }, (_, i) => + rec({ run_id: c.run, lane: c.lane, case: `green-${c.lane}-${i}`, file: `test/green-${i}.test.ts` }))); + const records: TrialOutcomeRecord[] = [...fixture.records, ...passing]; + const ledger = redLedger(records); + + test('21 verdict reds across 8 censuses; 27 of 28 failed trials are class assertion; red trials carry no recorded cause', () => { + expect(new Set(ledger.censuses.map(c => c.run)).size).toBe(8); + expect(ledger.pooled.reds).toBe(21); + const failed = records.filter(r => r.outcome === 'failed'); + expect(failed).toHaveLength(28); + expect(failed.filter(r => r.failure_class === 'assertion')).toHaveLength(27); + // 22 failed trials sit in red verdicts; the other 6 are the failed trials of split PASS panels. + expect(ledger.censuses.reduce((sum, c) => sum + (c.byCause.unrecorded ?? 0), 0)).toBe(22); + expect(ledger.censuses.map(c => c.reds.length).reduce((a, b) => a + b, 0)).toBe(21); + }); + + test('the all-green probability follows its documented formula and sits inside its interval', () => { + const latest = verdictsOf(records).filter(v => v.run === '37198445662'); + const rate = ledger.pooled.reds / ledger.pooled.verdicts; + const expected = latest.reduce((p, v) => { + const mine = verdictsOf(records).filter(x => x.case === v.case); + return p * (1 - (mine.filter(x => x.failsLane).length + RED_RATE_SHRINK * rate) / (mine.length + RED_RATE_SHRINK)); + }, 1); + expect(ledger.allGreen!.p).toBeCloseTo(expected, 12); + expect(ledger.allGreen!.verdicts).toBe(latest.length); + expect(ledger.allGreen!.lo).toBeLessThan(ledger.allGreen!.p); + expect(ledger.allGreen!.hi).toBeGreaterThan(ledger.allGreen!.p); + const text = formatRedLedger(ledger).join('\n'); + expect(text).toContain('all-green probability (approximation, assumes independent verdicts)'); + expect(text).toContain('rerun: bun run eval:pass-rates --reds'); + expect(text).toContain('37186854666 gate/full census: 1 red of'); + expect(text).toContain('class: timeout 1; cause: unrecorded 1'); + }); + + test('new records group reds by machine cause', () => { + const l = redLedger([rec({ outcome: 'failed', failure_class: 'assertion', failure_cause: 'refusal' }), rec({ case: 'b' })]); + expect(l.censuses[0]).toMatchObject({ byClass: { assertion: 1 }, byCause: { refusal: 1 } }); + }); +}); + +describe('triageRun (stub fetcher, no network)', () => { + test('downloads only the slice a red names, reports an oversize slice, and shows values and history', () => { + const cacheDir = fs.mkdtempSync(path.join(os.tmpdir(), 'triage-')); + const downloaded: string[] = []; + const fetcher: HistoryFetcher = { + listRuns: () => [], downloadZip: () => { throw new Error('unused'); }, + listArtifacts: () => [{ id: 1, name: 'paid-plan', size: 100 }, { id: 2, name: 'paid-slice-5-a1', size: 10 }, { id: 3, name: 'paid-slice-7-a1', size: 70 * 1024 * 1024 }, + { id: 4, name: 'paid-slice-9-a1', size: 10 }], + }; + const download = (match: (name: string) => boolean) => { + const names = ['paid-plan', 'paid-slice-5-a1', 'paid-slice-7-a1', 'paid-slice-9-a1'].filter(match); + downloaded.push(...names); + return names.map(name => { + const dir = path.join(cacheDir, name); + fs.mkdirSync(dir, { recursive: true }); + if (name === 'paid-plan') fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify({ entries: [ + { file: 'test/skill-llm-eval.test.ts', slice: 5, status: 'planned' }, { file: 'test/big.test.ts', slice: 7, status: 'planned' }, + { file: 'test/green.test.ts', slice: 9, status: 'planned' }] })); + return dir; + }); + }; + const lane = 'periodic/full census'; + const records = [ + rec({ run_id: '42', lane, case: 'qa-only/SKILL.md workflow', file: 'test/skill-llm-eval.test.ts', kind: 'judge', outcome: 'failed', failure_class: 'assertion', + failure_cause: 'assertion', error: 'expect(received).toBeGreaterThanOrEqual(expected)', + failure_detail: { judge: [{ dimension: 'clarity', mean: 3.67, threshold: 4, samples: 3, rationale: 'Clarity suffers from density.' }] } }), + rec({ run_id: '41', lane, case: 'qa-only/SKILL.md workflow', file: 'test/skill-llm-eval.test.ts', kind: 'judge' }), + rec({ run_id: '42', lane, case: 'big-case', file: 'test/big.test.ts', outcome: 'failed', failure_class: 'timeout' }), + rec({ run_id: '42', lane, case: 'green', file: 'test/green.test.ts' }), + ]; + const lines = triageRun({ repo: 'o/r', run: { id: 42, attempt: 1, sha: 'abcdef1234567', branch: 'b', createdAt: 't' }, records, cacheDir, fetcher, download }); + fs.rmSync(cacheDir, { recursive: true, force: true }); + expect(downloaded).toEqual(['paid-plan', 'paid-slice-5-a1']); + expect(lines[0]).toContain('run 42 (b @ abcdef123, t): 2 verdict red(s)'); + expect(lines).toContain('✗ qa-only/SKILL.md workflow periodic/full census FAIL (history: 1/2 verdicts green)'); + expect(lines.join('\n')).toContain('t1: assertion / cause assertion — expect(received).toBeGreaterThanOrEqual(expected) clarity 3.67 < 4 (3 samples) "Clarity suffers from density."'); + expect(lines.join('\n')).toContain(`evidence: ${path.join(cacheDir, 'paid-slice-5-a1', 'shards', 'skill-llm-eval')}`); + expect(lines).toContain(' evidence: paid-slice-7-a1 too large to fetch (70 MB)'); + }); +}); + +describe('eval:pass-rates CLI contract', () => { + function withFakeGh(run: (env: NodeJS.ProcessEnv, called: () => boolean, dir: string) => void) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'passrates-gh-')); + const marker = path.join(dir, 'gh-called'); + fs.writeFileSync(path.join(dir, 'gh'), `#!/bin/sh\ntouch '${marker}'\nexit 1\n`, { mode: 0o755 }); + try { + run({ ...process.env, PATH: `${dir}${path.delimiter}${process.env.PATH}`, GSTACK_STATE_ROOT: path.join(dir, 'state') }, () => fs.existsSync(marker), dir); + } finally { fs.rmSync(dir, { recursive: true, force: true }); } + } + const cli = (args: string[], env: NodeJS.ProcessEnv) => spawnSync(process.execPath, [SCRIPT, ...args], { encoding: 'utf8', timeout: 30_000, env }); + + test('--help is offline; an unknown flag or case exits 2 before any fetch', () => { + withFakeGh((env, called) => { + const help = cli(['--help'], env); + expect(help.status).toBe(0); + for (const flag of ['--headroom', '--reds', '--run ', '--case ', '--gate', '--help']) expect(help.stdout).toContain(flag); + for (const args of [['--bogus'], ['--case', 'no-such-case'], ['--runs', '0'], ['--headroom', '--reds'], ['--run']]) { + const r = cli(args, env); + expect(r.status, args.join(' ')).toBe(2); + expect(r.stderr).toContain('Usage: bun run eval:pass-rates'); + } + expect(called()).toBe(false); + }); + }); + + test('--case narrows the JSON and the alarms; --gate alarms above 85% of a session budget; scope is printed', () => { + withFakeGh((env, called, dir) => { + const drift = (id: string) => Array.from({ length: 12 }, (_, i) => rec({ case: id, kind: 'behavior', tier: 'periodic', panel: { n: 3, k: 2 }, + trial: (i % 3) + 1, run_id: String(Math.floor(i / 3)), outcome: i < 11 ? 'passed' : 'failed', ...(i < 11 ? {} : { failure_class: 'assertion' as const }) })); + const slow = [1, 2, 3].map(run => rec({ case: 'plan-mode-no-op', run_id: String(run), sessions: [session(run === 1 ? 88_000 : 50_000)] })); + fs.writeFileSync(path.join(dir, 'trial-outcomes.jsonl'), formatTrialOutcomes([...drift('plan-ceo-review-format-mode'), ...drift('plan-review-prosons-neutral-neg'), ...slow])); + const json = cli(['--dir', dir, '--json', '--case', 'plan-review-prosons-neutral-neg'], env); + expect(json.status, json.stderr).toBe(0); + const parsed = JSON.parse(json.stdout); + expect(parsed.scope).toBe(`local dirs ${dir}`); + expect(parsed.cases.map((c: any) => c.case)).toEqual(['plan-review-prosons-neutral-neg']); + expect(parsed.alarms.map((a: any) => a.case)).toEqual(['plan-review-prosons-neutral-neg']); + const gate = cli(['--dir', dir, '--gate', '--case', 'plan-mode-no-op'], env); + expect(gate.status).toBe(1); + expect(gate.stdout).toContain(`scope: local dirs ${dir}`); + expect(gate.stdout).toContain('[headroom] plan-mode-no-op session pty:x#1: max 88s of 100s (88%)'); + expect(gate.stdout).not.toContain('[drift]'); + const plain = cli(['--dir', dir, '--case', 'plan-mode-no-op'], env); + expect(plain.status).toBe(0); + expect(plain.stdout).not.toContain('[headroom]'); + expect(cli(['--dir', dir, '--headroom'], env).stdout).toContain('over 88% 88s/100s'); + expect(called()).toBe(false); + }); + }); + + test('flag parsing', () => { + const known = (id: string) => id === 'a'; + expect(parsePassRatesArgs(['--run', '37198445662', '--case', 'a'], known)).toMatchObject({ view: 'run', runId: 37198445662, caseFilter: 'a' }); + expect(parsePassRatesArgs(['--run', '1', '--dir', 'x'], known)).toEqual({ error: '--run reads one GitHub run; it cannot combine with --dir' }); + expect(parsePassRatesArgs(['-h'], known)).toEqual({ help: true }); + }); +}); diff --git a/test/fixtures/eval-history/wave-reds-8-censuses.json b/test/fixtures/eval-history/wave-reds-8-censuses.json new file mode 100644 index 000000000..59e4b8592 --- /dev/null +++ b/test/fixtures/eval-history/wave-reds-8-censuses.json @@ -0,0 +1 @@ +{"source":"trial-outcomes of censuses 37174266054..37198445662 (wave-trials.jsonl): every trial of each red verdict, plus per census lane the count of other verdicts","censuses":[{"run":"37174266054","lane":"gate/full census","otherVerdicts":105},{"run":"37174266054","lane":"periodic/full census","otherVerdicts":144},{"run":"37176837432","lane":"gate/full census","otherVerdicts":106},{"run":"37176837432","lane":"periodic/full census","otherVerdicts":142},{"run":"37179171083","lane":"gate/full census","otherVerdicts":105},{"run":"37179171083","lane":"periodic/full census","otherVerdicts":142},{"run":"37182865432","lane":"gate/full census","otherVerdicts":106},{"run":"37182865432","lane":"periodic/full census","otherVerdicts":143},{"run":"37186854666","lane":"gate/full census","otherVerdicts":105},{"run":"37186854666","lane":"periodic/full census","otherVerdicts":144},{"run":"37193478719","lane":"gate/full census","otherVerdicts":106},{"run":"37193478719","lane":"periodic/full census","otherVerdicts":142},{"run":"37195203538","lane":"gate/full census","otherVerdicts":106},{"run":"37195203538","lane":"periodic/full census","otherVerdicts":143},{"run":"37198445662","lane":"gate/full census","otherVerdicts":106},{"run":"37198445662","lane":"periodic/full census","otherVerdicts":144}],"records":[{"schema":"gstack-trial-outcome/v1","tier":"gate","attempt":1,"policy_version":1,"run_id":"37174266054","lane":"gate/full census","case":"shared-libs-read-only","file":"test/skill-e2e-shared-libs.test.ts","kind":"rule","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","exit_reason":"assertion_failed","error":"Error: expect(received).toEqual(expected)","duration_ms":175687,"cost_usd":1.16,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37174266054","lane":"periodic/full census","case":"plan-eng-review-artifact","file":"test/skill-e2e-plan-eng-artifact.test.ts","kind":"rule","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","error":"plan-eng-review-artifact: outcome=timeout (no terminal outcome within 600000ms total budget (including startup and 5000ms cleanup reserve; step0=6, review=0); idleFor=234ms)","duration_ms":595026,"cost_usd":0,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37174266054","lane":"periodic/full census","case":"shared-libs-plan-callers","file":"test/skill-e2e-shared-libs-periodic.test.ts","kind":"rule","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","exit_reason":"actor_contract","error":"Error: shared-libs-plan-callers actor: question expands beyond unchanged-helper reuse and its required proof: hardening","duration_ms":199864,"cost_usd":0,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37176837432","lane":"periodic/full census","case":"scrape-prototype-path","file":"test/skill-e2e-skillify.test.ts","kind":"behavior","trial":1,"panel":{"n":3,"k":2},"outcome":"failed","failure_class":"assertion","exit_reason":"success","duration_ms":39393,"cost_usd":0.37,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37176837432","lane":"periodic/full census","case":"scrape-prototype-path","file":"test/skill-e2e-skillify.test.ts","kind":"behavior","trial":2,"panel":{"n":3,"k":2},"outcome":"passed","duration_ms":34525,"cost_usd":0.35,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37176837432","lane":"periodic/full census","case":"scrape-prototype-path","file":"test/skill-e2e-skillify.test.ts","kind":"behavior","trial":3,"panel":{"n":3,"k":2},"outcome":"passed","duration_ms":38454,"cost_usd":0.37,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37176837432","lane":"periodic/full census","case":"plan-ceo-mode-routing","file":"test/skill-e2e-plan-ceo-mode-routing.test.ts","kind":"rule","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","error":"Mode \"HOLD SCOPE\" routing FAILED after sending option 2: no posture match for \\b(rigor|bulletproof|hold\\s*scope|maximum\\s+rigor)\\b.","duration_ms":636550,"cost_usd":0,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37176837432","lane":"periodic/full census","case":"setup-deploy/SKILL.md platform setup","file":"test/skill-llm-eval.test.ts","kind":"judge","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","exit_reason":"validation_failed","error":"expect(received).toBeGreaterThanOrEqual(expected)","duration_ms":21971,"cost_usd":0.06,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37176837432","lane":"periodic/full census","case":"auto-decide-preserved","file":"test/skill-e2e-auto-decide-preserved.test.ts","kind":"rule","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","error":"AUTO_DECIDE preserve test inconclusive: outcome=timeout","duration_ms":149624,"cost_usd":0,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"gate","attempt":1,"policy_version":1,"run_id":"37179171083","lane":"gate/full census","case":"tpa-absent-darwin","file":"test/skill-e2e-third-party-actions.test.ts","kind":"rule","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","exit_reason":"success","error":"expect(received).toMatch(expected)","duration_ms":32867,"cost_usd":0.27,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37179171083","lane":"periodic/full census","case":"plan-ceo-review-format-mode","file":"test/skill-e2e-plan-format.test.ts","kind":"behavior","trial":1,"panel":{"n":3,"k":2},"outcome":"passed","duration_ms":83711,"cost_usd":0.65,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37179171083","lane":"periodic/full census","case":"plan-ceo-review-format-mode","file":"test/skill-e2e-plan-format.test.ts","kind":"behavior","trial":2,"panel":{"n":3,"k":2},"outcome":"passed","duration_ms":78163,"cost_usd":0.61,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37179171083","lane":"periodic/full census","case":"plan-ceo-review-format-mode","file":"test/skill-e2e-plan-format.test.ts","kind":"behavior","trial":3,"panel":{"n":3,"k":2},"outcome":"failed","failure_class":"assertion","exit_reason":"success","error":"expect(received).toBe(expected)","duration_ms":58634,"cost_usd":0.52,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37179171083","lane":"periodic/full census","case":"plan-review-prosons-neutral-neg","file":"test/skill-e2e-plan-prosons.test.ts","kind":"behavior","trial":1,"panel":{"n":3,"k":2},"outcome":"passed","duration_ms":51967,"cost_usd":0.49,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37179171083","lane":"periodic/full census","case":"plan-review-prosons-neutral-neg","file":"test/skill-e2e-plan-prosons.test.ts","kind":"behavior","trial":2,"panel":{"n":3,"k":2},"outcome":"passed","duration_ms":35587,"cost_usd":0.51,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37179171083","lane":"periodic/full census","case":"plan-review-prosons-neutral-neg","file":"test/skill-e2e-plan-prosons.test.ts","kind":"behavior","trial":3,"panel":{"n":3,"k":2},"outcome":"failed","failure_class":"assertion","exit_reason":"success","error":"expect(received).not.toMatch(expected)","duration_ms":29998,"cost_usd":0.32,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37179171083","lane":"periodic/full census","case":"plan-design-review-plan-mode","file":"test/skill-e2e-design.test.ts","kind":"rule","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","error":"expect(received).toBe(expected)","duration_ms":161532,"cost_usd":0,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37179171083","lane":"periodic/full census","case":"auto-decide-preserved","file":"test/skill-e2e-auto-decide-preserved.test.ts","kind":"rule","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","error":"AUTO_DECIDE preserve test inconclusive: outcome=timeout","duration_ms":186374,"cost_usd":0,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37182865432","lane":"periodic/full census","case":"plan-ceo-section-loading","file":"test/skill-e2e-plan-ceo-review-section-loading.test.ts","kind":"rule","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","exit_reason":"success","error":"Error: expect(received).toBe(expected)","duration_ms":273203,"cost_usd":2.42,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37182865432","lane":"periodic/full census","case":"test/skill-e2e-auq-matrix.test.ts","file":"test/skill-e2e-auq-matrix.test.ts","kind":"rule","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","error":"plan-design-review AUQ not at plan-ceo bar:","duration_ms":242435,"cost_usd":0,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37182865432","lane":"periodic/full census","case":"auto-decide-preserved","file":"test/skill-e2e-auto-decide-preserved.test.ts","kind":"rule","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","error":"AUTO_DECIDE preserve test inconclusive: outcome=timeout","duration_ms":96897,"cost_usd":0,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"gate","attempt":1,"policy_version":1,"run_id":"37186854666","lane":"gate/full census","case":"shared-libs-review-revalidation","file":"test/skill-e2e-shared-libs.test.ts","kind":"rule","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"timeout","exit_reason":"timeout","error":"Error: Claude Code process aborted by user","duration_ms":399091,"cost_usd":3.25,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37186854666","lane":"periodic/full census","case":"qa/SKILL.md workflow","file":"test/skill-llm-eval.test.ts","kind":"judge","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","error":"expect(received).toBeGreaterThanOrEqual(expected)","duration_ms":49150,"cost_usd":0.06,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37186854666","lane":"periodic/full census","case":"shared-libs-plan-callers","file":"test/skill-e2e-shared-libs-periodic.test.ts","kind":"rule","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","exit_reason":"actor_contract","error":"Error: shared-libs-plan-callers actor: question expands beyond unchanged-helper reuse and its required proof: hardening","duration_ms":157370,"cost_usd":0,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37193478719","lane":"periodic/full census","case":"shared-libs-opportunity-judgment","file":"test/skill-e2e-shared-libs-periodic.test.ts","kind":"behavior","trial":1,"panel":{"n":3,"k":2},"outcome":"passed","duration_ms":160909,"cost_usd":1.25,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37193478719","lane":"periodic/full census","case":"shared-libs-opportunity-judgment","file":"test/skill-e2e-shared-libs-periodic.test.ts","kind":"behavior","trial":2,"panel":{"n":3,"k":2},"outcome":"passed","duration_ms":124334,"cost_usd":1.09,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37193478719","lane":"periodic/full census","case":"shared-libs-opportunity-judgment","file":"test/skill-e2e-shared-libs-periodic.test.ts","kind":"behavior","trial":3,"panel":{"n":3,"k":2},"outcome":"failed","failure_class":"assertion","exit_reason":"assertion_failed","error":"Error: adoption_and_risk: adoption_and_risk fails: the top recommendation names migration steps (one-line re-exports, preserved export names) and test coverage, but its risk section states 'none found inside the repo' and does not describe a concrete behavioral uncertainty or shared-failure risk (e…","duration_ms":145832,"cost_usd":1.17,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37193478719","lane":"periodic/full census","case":"plan-review-prosons-neutral-neg","file":"test/skill-e2e-plan-prosons.test.ts","kind":"behavior","trial":1,"panel":{"n":3,"k":2},"outcome":"passed","duration_ms":33978,"cost_usd":0.51,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37193478719","lane":"periodic/full census","case":"plan-review-prosons-neutral-neg","file":"test/skill-e2e-plan-prosons.test.ts","kind":"behavior","trial":2,"panel":{"n":3,"k":2},"outcome":"passed","duration_ms":33356,"cost_usd":0.51,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37193478719","lane":"periodic/full census","case":"plan-review-prosons-neutral-neg","file":"test/skill-e2e-plan-prosons.test.ts","kind":"behavior","trial":3,"panel":{"n":3,"k":2},"outcome":"failed","failure_class":"assertion","exit_reason":"success","error":"expect(received).not.toMatch(expected)","duration_ms":45836,"cost_usd":0.37,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37193478719","lane":"periodic/full census","case":"review/SKILL.md workflow","file":"test/skill-llm-eval.test.ts","kind":"judge","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","exit_reason":"validation_failed","error":"expect(received).toBeGreaterThanOrEqual(expected)","duration_ms":55735,"cost_usd":0.06,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37193478719","lane":"periodic/full census","case":"plan-ceo-section-loading","file":"test/skill-e2e-plan-ceo-review-section-loading.test.ts","kind":"rule","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","exit_reason":"success","error":"Error: expect(received).toBe(expected)","duration_ms":320159,"cost_usd":2.6,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37195203538","lane":"periodic/full census","case":"shared-libs-opportunity-judgment","file":"test/skill-e2e-shared-libs-periodic.test.ts","kind":"behavior","trial":1,"panel":{"n":3,"k":2},"outcome":"failed","failure_class":"assertion","exit_reason":"assertion_failed","error":"Error: adoption_and_risk: adoption_and_risk: The top recommendation names migration steps (re-exports preserving names/paths) and existing test coverage, but it explicitly states 'Main risk: none in behavior' and only raises an external-import path concern that the re-exports already resolve. It do…","duration_ms":184629,"cost_usd":1.37,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37195203538","lane":"periodic/full census","case":"shared-libs-opportunity-judgment","file":"test/skill-e2e-shared-libs-periodic.test.ts","kind":"behavior","trial":2,"panel":{"n":3,"k":2},"outcome":"passed","duration_ms":147734,"cost_usd":1.19,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37195203538","lane":"periodic/full census","case":"shared-libs-opportunity-judgment","file":"test/skill-e2e-shared-libs-periodic.test.ts","kind":"behavior","trial":3,"panel":{"n":3,"k":2},"outcome":"passed","duration_ms":183272,"cost_usd":1.46,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37195203538","lane":"periodic/full census","case":"plan-ceo-review-format-approach","file":"test/skill-e2e-plan-format.test.ts","kind":"behavior","trial":1,"panel":{"n":3,"k":2},"outcome":"failed","failure_class":"assertion","exit_reason":"exit_code_1","error":"expect(received).toContain(expected)","duration_ms":16165,"cost_usd":0.22,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37195203538","lane":"periodic/full census","case":"plan-ceo-review-format-approach","file":"test/skill-e2e-plan-format.test.ts","kind":"behavior","trial":2,"panel":{"n":3,"k":2},"outcome":"failed","failure_class":"assertion","exit_reason":"exit_code_1","error":"expect(received).toContain(expected)","duration_ms":78090,"cost_usd":0.61,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37195203538","lane":"periodic/full census","case":"plan-ceo-review-format-approach","file":"test/skill-e2e-plan-format.test.ts","kind":"behavior","trial":3,"panel":{"n":3,"k":2},"outcome":"passed","duration_ms":93861,"cost_usd":0.71,"quarantined":false,"execution":"executed","source":"shard"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37195203538","lane":"periodic/full census","case":"shared-libs-plan-callers","file":"test/skill-e2e-shared-libs-periodic.test.ts","kind":"rule","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","exit_reason":"actor_contract","error":"Error: shared-libs-plan-callers actor: question expands beyond unchanged-helper reuse and its required proof: hardening","duration_ms":195904,"cost_usd":0,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37198445662","lane":"periodic/full census","case":"qa-only/SKILL.md workflow","file":"test/skill-llm-eval.test.ts","kind":"judge","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","exit_reason":"validation_failed","error":"expect(received).toBeGreaterThanOrEqual(expected)","duration_ms":34833,"cost_usd":0.06,"quarantined":false,"execution":"executed","source":"junit"},{"schema":"gstack-trial-outcome/v1","tier":"periodic","attempt":1,"policy_version":1,"run_id":"37198445662","lane":"periodic/full census","case":"overlay-harness-claude-dedicated-tools-vs-bash-sonnet","file":"test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-sonnet.test.ts","kind":"rule","trial":1,"panel":{"n":1,"k":1},"outcome":"failed","failure_class":"assertion","error":"expect(received).toBe(expected)","duration_ms":77914,"cost_usd":0,"quarantined":false,"execution":"executed","source":"junit"}]} \ No newline at end of file