From 0c0077100ac1ffbb92803b410f712d57ff3f6276 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 31 Aug 2026 04:50:31 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20eval:flake-rank=20=E2=80=94=20the=20fla?= =?UTF-8?q?ke-telemetry=20dial?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aggregates per-test series across every finalized eval-store run (shard dirs included) plus the free flake ledger: runs, fails, RETRIED PASSES (the flake signature), avg duration — ranked retries-first. This is the readable dial behind two policies: a flaky pass never blocks a merge but is always ranked here, and the WS16 required-check promotion needs weeks of clean flake-rank, not vibes. --json for machines, --dir for downloaded CI artifacts. Co-Authored-By: Claude Fable 5 --- package.json | 1 + scripts/eval-flake-rank.ts | 127 +++++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 scripts/eval-flake-rank.ts diff --git a/package.json b/package.json index 12e607cd2..b17cfcf7c 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "eval:list": "bun run scripts/eval-list.ts", "eval:compare": "bun run scripts/eval-compare.ts", "eval:summary": "bun run scripts/eval-summary.ts", + "eval:flake-rank": "bun run scripts/eval-flake-rank.ts", "eval:watch": "bun run scripts/eval-watch.ts", "eval:select": "bun run scripts/eval-select.ts", "analytics": "bun run scripts/analytics.ts", diff --git a/scripts/eval-flake-rank.ts b/scripts/eval-flake-rank.ts new file mode 100644 index 000000000..0885124ad --- /dev/null +++ b/scripts/eval-flake-rank.ts @@ -0,0 +1,127 @@ +#!/usr/bin/env bun +/** + * eval-flake-rank — the flake-telemetry dial (WS1). + * + * Aggregates per-test series across every FINALIZED eval-store run on this + * machine (default: ~/.gstack/projects//evals/, shard dirs included) + * plus the free suite's flake ledger, and ranks tests by flake signal: + * retried passes first (a test that needs attempt 2 to go green is the + * definition of a flake), then failure rate. + * + * This is the readable dial behind two policies: + * - a flaky pass never blocks a merge, but it is recorded and RANKED here; + * - the required-check promotion (WS16) needs weeks of clean flake-rank, + * not vibes. + * + * Usage: + * bun run eval:flake-rank # project eval dir + * bun run eval:flake-rank --dir # e.g. downloaded CI artifacts + * bun run eval:flake-rank --json # machine-readable + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { getProjectEvalDir, isPartialEval, type EvalResult } from '../test/helpers/eval-store'; +import { flakeLedgerPath, type FlakeLedgerEntry } from './test-free-shards'; + +interface TestSeries { + name: string; + runs: number; + passes: number; + fails: number; + retriedPasses: number; + totalAttempts: number; + totalCostUsd: number; + totalDurationMs: number; + lastSeen: string; +} + +export function aggregate(evalFiles: string[]): Map { + const series = new Map(); + for (const file of evalFiles) { + let run: EvalResult; + try { + run = JSON.parse(fs.readFileSync(file, 'utf-8')); + } catch { continue; } + if (isPartialEval(run, file)) continue; // in-progress accumulators are not runs + if (!Array.isArray(run.tests)) continue; + // Group this run's entries by name so N attempts = 1 run of that test. + const byName = new Map(); + for (const t of run.tests) { + const list = byName.get(t.name) ?? []; + list.push(t); + byName.set(t.name, list); + } + for (const [name, entries] of byName) { + const s = series.get(name) ?? { + name, runs: 0, passes: 0, fails: 0, retriedPasses: 0, + totalAttempts: 0, totalCostUsd: 0, totalDurationMs: 0, lastSeen: '', + }; + const final = entries[entries.length - 1]; + s.runs += 1; + s.totalAttempts += entries.length; + if (final.passed) s.passes += 1; else s.fails += 1; + if (final.passed && entries.length > 1) s.retriedPasses += 1; + for (const e of entries) { + s.totalCostUsd += e.cost_usd || 0; + s.totalDurationMs += e.duration_ms || 0; + } + if (run.timestamp > s.lastSeen) s.lastSeen = run.timestamp; + series.set(name, s); + } + } + return series; +} + +export function collectEvalFiles(dir: string): string[] { + if (!fs.existsSync(dir)) return []; + const out: string[] = []; + for (const name of fs.readdirSync(dir, { recursive: true }) as string[]) { + if (!name.endsWith('.json') || path.basename(name).startsWith('_partial')) continue; + out.push(path.join(dir, name)); + } + return out; +} + +function readFreeLedger(): FlakeLedgerEntry[] { + try { + return fs.readFileSync(flakeLedgerPath(), 'utf-8') + .split('\n').filter(Boolean).map((l) => JSON.parse(l)); + } catch { return []; } +} + +if (import.meta.main) { + const argv = process.argv.slice(2); + const dirFlag = argv.indexOf('--dir'); + const dir = dirFlag !== -1 ? argv[dirFlag + 1] : getProjectEvalDir(); + const asJson = argv.includes('--json'); + + const files = collectEvalFiles(dir); + const series = [...aggregate(files).values()] + .sort((a, b) => b.retriedPasses - a.retriedPasses || (b.fails / b.runs) - (a.fails / a.runs)); + const ledger = readFreeLedger(); + + if (asJson) { + console.log(JSON.stringify({ dir, runsScanned: files.length, tests: series, freeLedger: ledger }, null, 2)); + } else { + console.log(`flake-rank: ${files.length} finalized run file(s) under ${dir}`); + const flaky = series.filter((s) => s.retriedPasses > 0 || s.fails > 0); + if (flaky.length === 0) { + console.log(' no retried passes and no failures recorded — clean series'); + } else { + console.log(' retries fails/runs avg-dur test'); + for (const s of flaky.slice(0, 30)) { + console.log(` ${String(s.retriedPasses).padStart(7)} ${String(s.fails).padStart(5)}/${String(s.runs).padEnd(4)} ` + + `${Math.round(s.totalDurationMs / s.totalAttempts / 1000).toString().padStart(5)}s ${s.name}`); + } + } + if (ledger.length > 0) { + const byFile = new Map(); + for (const e of ledger) byFile.set(e.file, (byFile.get(e.file) ?? 0) + 1); + console.log(`free-suite flaky-passes (${flakeLedgerPath()}):`); + for (const [file, n] of [...byFile.entries()].sort((a, b) => b[1] - a[1])) { + console.log(` ${String(n).padStart(3)}x ${file}`); + } + } + } +}