mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-01 06:24:41 +01:00
garrytan
30d2904c66
evals.yml wraps collector-outcomes failures in a fixed three-backtick fence for the PR comment, so those lines (and eval:pass-rates output meant for the tracking issue) go through sanitizeFixedFenceLine: cleaned like every published line, with backticks turned into quotes. Inline published text also neutralizes backticks.
351 lines
22 KiB
TypeScript
351 lines
22 KiB
TypeScript
/**
|
||
* eval:pass-rates report views over stored trial records (plan 0.1):
|
||
* session headroom (--headroom), the verdict red ledger with the all-green
|
||
* probability (--reds), and one census's triage view (--run <id>). Pure
|
||
* functions plus the run-scoped fetches they need; scripts/eval-flake-rank.ts
|
||
* parses flags, loads history and dispatches here.
|
||
*
|
||
* Headroom is per armed session (session ledger, `sessions[]` on a record):
|
||
* elapsed over the timeout it armed. A case's headroom is its slowest
|
||
* session; a timed-out sample is censored (its true duration is unknown).
|
||
* Records without sessions are unknown, never scored; their case wall
|
||
* (JUnit or shard clock) is shown separately as an upper bound.
|
||
*
|
||
* The all-green probability is an approximation that assumes independent
|
||
* verdicts: P(green) = Π (1 - p_i) over the cases a census verdicts, where
|
||
* p_i = (reds_i + K * p̄) / (verdicts_i + K) shrinks each case's red rate
|
||
* toward the pooled rate p̄. Its interval recomputes the same product with p̄
|
||
* replaced by the pooled rate's 95% Wilson bounds.
|
||
*/
|
||
import * as fs from 'node:fs';
|
||
import * as path from 'node:path';
|
||
import { spawnSync } from 'node:child_process';
|
||
import { panelVerdict, type TrialOutcomeRecord } from '../../test/helpers/eval-store';
|
||
import { HEADROOM_FAIL, HEADROOM_WARN } from '../../test/helpers/eval-budgets';
|
||
import { sanitizeFixedFenceLine } from './published-text';
|
||
import type { HistoryFetcher, WeeklyRun } from '../eval-flake-rank';
|
||
import { CASE_SHARDED_FILES, trialShardKey } from './paid-cases';
|
||
import { shardSlug } from '../test-paid-shards';
|
||
|
||
/** Below this many session samples a case's headroom is insufficient, never a pass. */
|
||
export const HEADROOM_MIN_SAMPLES = 3;
|
||
/** Pseudo-verdicts of pooled rate added to each case's red rate. */
|
||
export const RED_RATE_SHRINK = 4;
|
||
/** Largest slice artifact --run downloads. */
|
||
export const EVIDENCE_MAX_BYTES = 64 * 1024 * 1024;
|
||
|
||
type Rec = TrialOutcomeRecord;
|
||
const pct = (x: number) => `${Math.round(x * 1000) / 10}%`;
|
||
const secs = (ms: number) => `${Math.round(ms / 1000)}s`;
|
||
|
||
// --- Headroom ---
|
||
|
||
export interface SessionHeadroom { key: string; samples: number; censored: number; maxRatio: number; maxElapsedMs: number; budgetMs: number }
|
||
export interface CaseHeadroom {
|
||
case: string;
|
||
status: 'over' | 'warn' | 'ok' | 'insufficient' | 'unknown';
|
||
sessions: SessionHeadroom[];
|
||
/** Slowest session's worst ratio, when any session was recorded. */
|
||
worst?: SessionHeadroom;
|
||
/** Largest case wall (JUnit or shard clock) over the window: an upper bound, never a session clock. */
|
||
caseWallMs: number;
|
||
runs: number;
|
||
}
|
||
|
||
export function headroom(records: readonly Rec[]): CaseHeadroom[] {
|
||
const byCase = new Map<string, Rec[]>();
|
||
for (const r of records) if (r.outcome !== 'skipped' && r.execution !== 'reused') byCase.set(r.case, [...(byCase.get(r.case) ?? []), r]);
|
||
return [...byCase.entries()].map(([id, rs]) => {
|
||
const sessions = new Map<string, SessionHeadroom>();
|
||
for (const s of rs.flatMap(r => r.sessions ?? [])) {
|
||
if (!s.budget_ms) continue;
|
||
const key = s.key;
|
||
const entry = sessions.get(key) ?? { key, samples: 0, censored: 0, maxRatio: 0, maxElapsedMs: 0, budgetMs: s.budget_ms };
|
||
entry.samples++;
|
||
if (s.end === 'session_timeout' || s.end === 'observer_timeout') entry.censored++;
|
||
const ratio = s.elapsed_ms / s.budget_ms;
|
||
if (ratio > entry.maxRatio) Object.assign(entry, { maxRatio: ratio, maxElapsedMs: s.elapsed_ms, budgetMs: s.budget_ms });
|
||
sessions.set(key, entry);
|
||
}
|
||
const list = [...sessions.values()].sort((a, b) => b.maxRatio - a.maxRatio);
|
||
const worst = list[0];
|
||
const status: CaseHeadroom['status'] = !worst ? 'unknown' : worst.samples < HEADROOM_MIN_SAMPLES ? 'insufficient'
|
||
: worst.maxRatio > HEADROOM_FAIL ? 'over' : worst.maxRatio > HEADROOM_WARN ? 'warn' : 'ok';
|
||
return { case: id, status, sessions: list, ...(worst ? { worst } : {}), caseWallMs: Math.max(0, ...rs.map(r => r.duration_ms)),
|
||
runs: new Set(rs.map(r => r.run_id ?? 'local')).size };
|
||
}).sort((a, b) => (b.worst?.maxRatio ?? -1) - (a.worst?.maxRatio ?? -1) || b.caseWallMs - a.caseWallMs);
|
||
}
|
||
|
||
/** --gate alarms: a case whose slowest session ran above HEADROOM_FAIL of its armed budget. */
|
||
export function headroomAlarms(cases: readonly CaseHeadroom[]): Array<{ kind: 'headroom'; case: string; message: string }> {
|
||
return cases.filter(c => c.status === 'over').map(c => ({ kind: 'headroom' as const, case: c.case,
|
||
message: `${c.case} session ${c.worst!.key}: max ${secs(c.worst!.maxElapsedMs)} of ${secs(c.worst!.budgetMs)} (${pct(c.worst!.maxRatio)})`
|
||
+ `${c.worst!.censored ? `, ${c.worst!.censored} timed-out sample(s) censored` : ''} over ${c.worst!.samples} sample(s), above the ${pct(HEADROOM_FAIL)} cap. `
|
||
+ 'Cut work in the skill or fixture; budgets are never raised (TESTING_INTERNALS "Timeout policy").' }));
|
||
}
|
||
|
||
export function formatHeadroom(cases: readonly CaseHeadroom[], critical: readonly CriticalPath[] = []): string[] {
|
||
const lines = [`headroom: slowest armed session per case (warn > ${pct(HEADROOM_WARN)}, alarm > ${pct(HEADROOM_FAIL)}; `
|
||
+ `insufficient below ${HEADROOM_MIN_SAMPLES} samples; censored = timed out)`,
|
||
' status session max/budget samples case wall (upper bound, not a session clock) case'];
|
||
for (const c of cases) {
|
||
const w = c.worst;
|
||
const session = w ? `${pct(w.maxRatio)} ${secs(w.maxElapsedMs)}/${secs(w.budgetMs)}` : 'unknown (no session ledger)';
|
||
lines.push(` ${c.status.padEnd(12)} ${session.padEnd(26)} ${(w ? `${w.samples}${w.censored ? ` (${w.censored} censored)` : ''}` : '-').padEnd(7)} `
|
||
+ `${secs(c.caseWallMs).padEnd(45)} ${c.case}${w ? ` [${w.key}]` : ''}`);
|
||
}
|
||
return [...lines, ...formatCriticalPath(critical)];
|
||
}
|
||
|
||
export function formatCriticalPath(critical: readonly CriticalPath[]): string[] {
|
||
return critical.map(cp => `critical path ${cp.run}: ${cp.slowest ? `${cp.slowest.name} ${secs(cp.slowest.ms)}` : 'unknown'}`
|
||
+ ` of ${cp.jobs} slice job(s), run wall ${cp.wallMs === null ? 'unknown' : secs(cp.wallMs)}${cp.files?.length ? `; slice files: ${cp.files.join(', ')}` : ''}`);
|
||
}
|
||
|
||
// --- Critical path from runner slice timings ---
|
||
|
||
export interface JobTiming { name: string; startedAt: string | null; completedAt: string | null }
|
||
export interface CriticalPath { run: string; jobs: number; slowest: { name: string; ms: number } | null; wallMs: number | null; files?: string[] }
|
||
|
||
const SLICE_JOB = /^(eval-slices|eval-codex-slices|gate-census) \((\d+)\)$/;
|
||
|
||
export function criticalPath(run: string, jobs: readonly JobTiming[], plan?: { entries: Array<{ file: string; slice?: number; status?: string }> }): CriticalPath {
|
||
const timed = jobs.filter(j => SLICE_JOB.test(j.name) && j.startedAt && j.completedAt)
|
||
.map(j => ({ name: j.name, ms: Date.parse(j.completedAt!) - Date.parse(j.startedAt!) }));
|
||
const slowest = timed.sort((a, b) => b.ms - a.ms)[0] ?? null;
|
||
const all = jobs.filter(j => j.startedAt && j.completedAt);
|
||
const wallMs = all.length ? Math.max(...all.map(j => Date.parse(j.completedAt!))) - Math.min(...all.map(j => Date.parse(j.startedAt!))) : null;
|
||
const slice = slowest ? Number(SLICE_JOB.exec(slowest.name)![2]) : null;
|
||
const files = plan && slice !== null && slowest!.name.startsWith('gate-census')
|
||
? plan.entries.filter(e => e.slice === slice && e.status === 'planned').map(e => e.file) : undefined;
|
||
return { run, jobs: timed.length, slowest, wallMs, ...(files ? { files } : {}) };
|
||
}
|
||
|
||
// --- Red ledger ---
|
||
|
||
export interface VerdictRed { run: string; lane: string; case: string; status: string; trials: Rec[] }
|
||
export interface CensusReds { run: string; lane: string; verdicts: number; reds: VerdictRed[]; byClass: Record<string, number>; byCause: Record<string, number> }
|
||
export interface RedLedger {
|
||
censuses: CensusReds[];
|
||
pooled: { verdicts: number; reds: number; rate: number; lo: number; hi: number };
|
||
perCase: Array<{ case: string; verdicts: number; reds: number; rate: number }>;
|
||
allGreen: { p: number; lo: number; hi: number; verdicts: number } | null;
|
||
}
|
||
|
||
function wilson(k: number, n: number, z = 1.96): { lo: number; hi: number } {
|
||
if (n === 0) return { lo: 0, hi: 1 };
|
||
const p = k / n, d = 1 + z * z / n, c = p + z * z / (2 * n), m = z * Math.sqrt(p * (1 - p) / n + z * z / (4 * n * n));
|
||
return { lo: Math.max(0, (c - m) / d), hi: Math.min(1, (c + m) / d) };
|
||
}
|
||
|
||
/** One panelVerdict per case per census lane (earliest attempt), exactly as the report computes it. */
|
||
export function verdictsOf(records: readonly Rec[]): Array<{ run: string; lane: string; case: string; status: string; failsLane: boolean; trials: Rec[] }> {
|
||
const groups = new Map<string, Rec[]>();
|
||
for (const r of records) {
|
||
if (r.policy_version < 1 || r.execution === 'reused') continue;
|
||
// A JUnit case is one verdict per shard file, as the report counts it; trial shards form one panel.
|
||
const key = `${r.run_id ?? 'local'}\0${r.lane ?? r.tier}\0${r.case}\0${r.source === 'shard' ? '' : r.file}`;
|
||
groups.set(key, [...(groups.get(key) ?? []), r]);
|
||
}
|
||
return [...groups.entries()].map(([key, rs]) => {
|
||
const [run, lane, id] = key.split('\0') as [string, string, string, string];
|
||
const attempt = Math.min(...rs.map(r => r.attempt));
|
||
const trials = rs.filter(r => r.attempt === attempt);
|
||
const v = panelVerdict({ case: id, kind: trials[0]!.kind, panel: trials[0]!.panel, quarantined: trials[0]!.quarantined,
|
||
trials: trials.map(t => ({ trial: t.trial, outcome: t.outcome, attempt, ...(t.failure_class ? { failure_class: t.failure_class } : {}) })) });
|
||
return { run, lane, case: id, status: v.status, failsLane: v.failsLane, trials };
|
||
});
|
||
}
|
||
|
||
export function redLedger(records: readonly Rec[]): RedLedger {
|
||
const verdicts = verdictsOf(records).filter(v => v.status !== 'SKIPPED');
|
||
const censusKey = (v: { run: string; lane: string }) => `${v.run}\0${v.lane}`;
|
||
const censuses = new Map<string, CensusReds>();
|
||
for (const v of verdicts) {
|
||
const c = censuses.get(censusKey(v)) ?? { run: v.run, lane: v.lane, verdicts: 0, reds: [], byClass: {}, byCause: {} };
|
||
c.verdicts++;
|
||
if (v.failsLane) {
|
||
c.reds.push({ run: v.run, lane: v.lane, case: v.case, status: v.status, trials: v.trials });
|
||
for (const t of v.trials.filter(t => t.outcome === 'failed')) {
|
||
c.byClass[t.failure_class ?? 'assertion'] = (c.byClass[t.failure_class ?? 'assertion'] ?? 0) + 1;
|
||
c.byCause[t.failure_cause ?? 'unrecorded'] = (c.byCause[t.failure_cause ?? 'unrecorded'] ?? 0) + 1;
|
||
}
|
||
}
|
||
censuses.set(censusKey(v), c);
|
||
}
|
||
const reds = verdicts.filter(v => v.failsLane).length;
|
||
const rate = verdicts.length ? reds / verdicts.length : 0;
|
||
const pooled = { verdicts: verdicts.length, reds, rate, ...wilson(reds, verdicts.length) };
|
||
const byCase = new Map<string, { verdicts: number; reds: number }>();
|
||
for (const v of verdicts) {
|
||
const e = byCase.get(v.case) ?? { verdicts: 0, reds: 0 };
|
||
e.verdicts++; if (v.failsLane) e.reds++;
|
||
byCase.set(v.case, e);
|
||
}
|
||
const perCase = [...byCase.entries()].map(([id, e]) => ({ case: id, ...e, rate: (e.reds + RED_RATE_SHRINK * rate) / (e.verdicts + RED_RATE_SHRINK) }))
|
||
.sort((a, b) => b.rate - a.rate || (a.case < b.case ? -1 : 1));
|
||
const ordered = [...censuses.values()];
|
||
const latestRun = ordered.map(c => c.run).sort().at(-1);
|
||
const census = verdicts.filter(v => v.run === latestRun);
|
||
const green = (prior: number) => census.reduce((p, v) => {
|
||
const e = byCase.get(v.case)!;
|
||
return p * (1 - (e.reds + RED_RATE_SHRINK * prior) / (e.verdicts + RED_RATE_SHRINK));
|
||
}, 1);
|
||
const allGreen = census.length ? { p: green(rate), lo: green(pooled.hi), hi: green(pooled.lo), verdicts: census.length } : null;
|
||
return { censuses: ordered.sort((a, b) => (a.run < b.run ? -1 : a.run > b.run ? 1 : a.lane < b.lane ? -1 : 1)), pooled, perCase, allGreen };
|
||
}
|
||
|
||
const tally = (counts: Record<string, number>) => Object.entries(counts).sort(([a, x], [b, y]) => y - x || (a < b ? -1 : 1)).map(([k, n]) => `${k} ${n}`).join(', ') || 'none';
|
||
|
||
export function formatRedLedger(ledger: RedLedger, opts: { top?: number } = {}): string[] {
|
||
const lines = ['reds: verdict reds per census lane by failure class and machine cause (cause "unrecorded" = written before failure_cause existed)'];
|
||
for (const c of ledger.censuses) {
|
||
lines.push(` ${c.run} ${c.lane}: ${c.reds.length} red of ${c.verdicts} verdicts${c.reds.length ? ` — class: ${tally(c.byClass)}; cause: ${tally(c.byCause)}` : ''}`);
|
||
for (const red of c.reds) {
|
||
const failed = red.trials.filter(t => t.outcome === 'failed');
|
||
lines.push(` ✗ ${red.case} ${red.status} ${failed.map(t => `t${t.trial} ${t.failure_class ?? 'assertion'}/${t.failure_cause ?? 'unrecorded'}`
|
||
+ `${t.error ? ` — ${sanitizeFixedFenceLine(t.error, 160)}` : ''}`).join('; ')}`);
|
||
}
|
||
}
|
||
const p = ledger.pooled;
|
||
lines.push(`pooled: ${p.reds} red of ${p.verdicts} verdicts = ${pct(p.rate)} per verdict [95% ${pct(p.lo)}–${pct(p.hi)}]`);
|
||
const top = ledger.perCase.filter(c => c.reds > 0).slice(0, opts.top ?? 15);
|
||
if (top.length) lines.push(`per-case red rates (shrunk toward the pooled rate, K=${RED_RATE_SHRINK}):`, ...top.map(c => ` ${pct(c.rate).padStart(6)} ${c.reds}/${c.verdicts} ${c.case}`));
|
||
const g = ledger.allGreen;
|
||
lines.push(g ? `all-green probability (approximation, assumes independent verdicts): ${pct(g.p)} for the latest census's ${g.verdicts} verdicts [${pct(g.lo)}–${pct(g.hi)} with the pooled rate's 95% interval as prior]; `
|
||
+ 'formula Π(1 - p_i) over its cases; rerun: bun run eval:pass-rates --reds' : 'all-green probability: no post-policy verdicts');
|
||
return lines;
|
||
}
|
||
|
||
// --- One census's triage view ---
|
||
|
||
/** Fetches the report views make beyond the history loader: a named run and its slice jobs. */
|
||
export interface RunJobsFetcher { listJobs(repo: string, runId: number): JobTiming[]; getRun(repo: string, runId: number): WeeklyRun }
|
||
|
||
function ghJson(args: string[]): string[] {
|
||
const result = spawnSync('gh', args, { timeout: 120_000, maxBuffer: 64 * 1024 * 1024 });
|
||
if (result.status !== 0) throw new Error(`gh ${args.slice(0, 2).join(' ')} failed: ${String(result.stderr || result.error || '').trim()}`);
|
||
return result.stdout.toString('utf8').split('\n').filter(Boolean);
|
||
}
|
||
|
||
export const GH_JOBS: RunJobsFetcher = {
|
||
listJobs: (repo, runId) => ghJson(['api', `repos/${repo}/actions/runs/${runId}/jobs?per_page=100`, '--paginate',
|
||
'--jq', '.jobs[] | {name, startedAt: .started_at, completedAt: .completed_at}']).map(line => JSON.parse(line) as JobTiming),
|
||
getRun: (repo, runId) => JSON.parse(ghJson(['api', `repos/${repo}/actions/runs/${runId}`,
|
||
'--jq', '{id, attempt: .run_attempt, sha: .head_sha, branch: .head_branch, createdAt: .created_at}'])[0]!) as WeeklyRun,
|
||
};
|
||
|
||
/** The manifest shard key a record ran under: its trial shard, its case shard, or its file. */
|
||
export function recordShardKey(r: Pick<Rec, 'file' | 'case' | 'trial' | 'source'>): string {
|
||
return r.source === 'shard' ? trialShardKey(r.file, r.case, r.trial) : CASE_SHARDED_FILES.includes(r.file) ? `${r.file}#${r.case}` : r.file;
|
||
}
|
||
|
||
/**
|
||
* The run's reds with their values and history, and the slice artifacts that
|
||
* hold their transcripts. Only the slices a red names are downloaded; a slice
|
||
* over EVIDENCE_MAX_BYTES is reported, never silently skipped.
|
||
*/
|
||
export function triageRun(opts: { repo: string; run: WeeklyRun; records: readonly Rec[]; caseFilter?: string; cacheDir: string;
|
||
fetcher: HistoryFetcher; download: (match: (name: string) => boolean, maxBytes: number) => string[] }): string[] {
|
||
const runId = String(opts.run.id);
|
||
const mine = opts.records.filter(r => r.run_id === runId);
|
||
const reds = verdictsOf(mine).filter(v => v.failsLane && (!opts.caseFilter || v.case === opts.caseFilter));
|
||
const lines = [`run ${runId} (${opts.run.branch} @ ${opts.run.sha.slice(0, 9)}, ${opts.run.createdAt}): ${reds.length} verdict red(s)${opts.caseFilter ? ` for ${opts.caseFilter}` : ''}`];
|
||
const history = verdictsOf(opts.records);
|
||
const plans = new Map<string, Map<string, number>>();
|
||
const planFor = (lane: string) => {
|
||
const name = lane.startsWith('gate') && lane.includes('census') ? 'gate-census-plan' : 'paid-plan';
|
||
if (!plans.has(name)) {
|
||
const files = new Map<string, number>();
|
||
for (const dir of opts.download(n => n === name, 8 * 1024 * 1024)) {
|
||
try {
|
||
const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'manifest.json'), 'utf8'));
|
||
for (const e of manifest.entries ?? []) if (e.status === 'planned' && Number.isInteger(e.slice)) files.set(e.file, e.slice);
|
||
} catch { /* no manifest: slices stay unknown */ }
|
||
}
|
||
plans.set(name, files);
|
||
}
|
||
return { prefix: name === 'gate-census-plan' ? 'gate-census' : 'paid-slice', files: plans.get(name)! };
|
||
};
|
||
const artifacts = opts.fetcher.listArtifacts(opts.repo, opts.run.id);
|
||
for (const red of reds) {
|
||
const runs = history.filter(v => v.case === red.case);
|
||
lines.push(`✗ ${red.case} ${red.lane} ${red.status} (history: ${runs.filter(v => !v.failsLane).length}/${runs.length} verdicts green)`);
|
||
for (const t of red.trials.filter(t => t.outcome === 'failed')) {
|
||
const d = t.failure_detail;
|
||
const detail = !d ? '' : 'judge' in d ? ` ${d.judge.map(j => `${j.dimension} ${j.mean} < ${j.threshold} (${j.samples} samples)${j.rationale ? ` "${j.rationale}"` : ''}`).join('; ')}`
|
||
: ` Expected: ${d.expected} · Received: ${d.received}`;
|
||
lines.push(sanitizeFixedFenceLine(` t${t.trial}: ${t.failure_class ?? 'assertion'} / cause ${t.failure_cause ?? 'unrecorded'}`
|
||
+ `${t.failure_cause_evidence ? `: ${t.failure_cause_evidence}` : ''}${t.error ? ` — ${t.error}` : ''}${detail}`, 2000));
|
||
const plan = planFor(red.lane);
|
||
const slice = plan.files.get(recordShardKey(t));
|
||
if (slice === undefined) { lines.push(' evidence: slice unknown (no plan artifact for this lane)'); continue; }
|
||
const name = `${plan.prefix}-${slice}-a${t.attempt}`;
|
||
const artifact = artifacts.find(a => a.name === name);
|
||
if (!artifact) { lines.push(` evidence: ${name} (expired or missing)`); continue; }
|
||
if (artifact.size > EVIDENCE_MAX_BYTES) { lines.push(` evidence: ${name} too large to fetch (${Math.round(artifact.size / 1024 / 1024)} MB)`); continue; }
|
||
const [dir] = opts.download(n => n === name, EVIDENCE_MAX_BYTES);
|
||
lines.push(` evidence: ${dir ? path.join(dir, 'shards', shardSlug([recordShardKey(t)])) : `${name} (download failed)`}`);
|
||
}
|
||
}
|
||
return lines;
|
||
}
|
||
|
||
// --- Flags ---
|
||
|
||
export const PASS_RATES_USAGE = `Usage: bun run eval:pass-rates [options]
|
||
(no view flag) per-case trial pass rates with 95% Wilson intervals
|
||
--headroom slowest armed session per case vs its budget, case wall upper bound, census critical path
|
||
--reds verdict reds per census by failure class and cause, pooled and per-case rates, all-green probability
|
||
--run <id> one census: each red with its values, history and fetched transcript evidence
|
||
--case <id> restrict every view, the JSON and the alarms to one registered case
|
||
--runs <n> completed runs per branch to read (default 10)
|
||
--branch <name> branch whose runs to read beside main (default: current branch)
|
||
--repo <owner/repo> --workflow <file> --dir <path> (repeatable, local artifacts; no network)
|
||
--backfill also import legacy slice artifacts, labeled pre-policy
|
||
--since-days <n> legacy eval files window for --dir (default 60)
|
||
--json machine-readable output
|
||
--gate exit 1 on ACTION REQUIRED (pass-rate alarms and sessions above ${pct(HEADROOM_FAIL)} of budget)
|
||
--help this text (no network)`;
|
||
|
||
export interface PassRatesArgs {
|
||
view: 'rates' | 'headroom' | 'reds' | 'run';
|
||
runId?: number; caseFilter?: string; runs: number; sinceDays: number; dirs: string[];
|
||
json: boolean; gate: boolean; backfill: boolean; repo?: string; workflow?: string; branch?: string;
|
||
}
|
||
|
||
/** Parse eval:pass-rates flags; unknown flags, bad values and unknown case ids are errors (exit 2, before any fetch). */
|
||
export function parsePassRatesArgs(argv: readonly string[], isKnownCase: (id: string) => boolean): { help: true } | { error: string } | PassRatesArgs {
|
||
const out: PassRatesArgs = { view: 'rates', runs: 10, sinceDays: 60, dirs: [], json: false, gate: false, backfill: false };
|
||
const views: string[] = [];
|
||
for (let i = 0; i < argv.length; i++) {
|
||
const arg = argv[i]!;
|
||
const value = () => { const v = argv[++i]; if (v === undefined || v.startsWith('--')) throw new Error(`${arg} needs a value`); return v; };
|
||
const count = () => { const v = value(); if (!/^[1-9]\d*$/.test(v)) throw new Error(`${arg} needs a positive integer, got ${v}`); return Number(v); };
|
||
try {
|
||
switch (arg) {
|
||
case '--help': case '-h': return { help: true };
|
||
case '--headroom': views.push('headroom'); break;
|
||
case '--reds': views.push('reds'); break;
|
||
case '--run': views.push('run'); out.runId = count(); break;
|
||
case '--case': out.caseFilter = value(); break;
|
||
case '--runs': out.runs = count(); break;
|
||
case '--since-days': out.sinceDays = count(); break;
|
||
case '--dir': out.dirs.push(value()); break;
|
||
case '--repo': out.repo = value(); break;
|
||
case '--workflow': out.workflow = value(); break;
|
||
case '--branch': out.branch = value(); break;
|
||
case '--json': out.json = true; break;
|
||
case '--gate': out.gate = true; break;
|
||
case '--backfill': out.backfill = true; break;
|
||
default: return { error: `unknown flag ${arg}` };
|
||
}
|
||
} catch (error) { return { error: (error as Error).message }; }
|
||
}
|
||
if (views.length > 1) return { error: `choose one of --headroom, --reds, --run (got ${views.join(', ')})` };
|
||
if (views[0]) out.view = views[0] as PassRatesArgs['view'];
|
||
if (out.caseFilter !== undefined && !isKnownCase(out.caseFilter)) return { error: `unknown case ${out.caseFilter} (not an E2E, judge or paid test file id)` };
|
||
if (out.view === 'run' && out.dirs.length) return { error: '--run reads one GitHub run; it cannot combine with --dir' };
|
||
return out;
|
||
}
|