From 2dabc024472e2e27e28171596c6c30ef1976484c Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 31 Aug 2026 04:50:12 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20paid-lane=20flake=20telemetry=20?= =?UTF-8?q?=E2=80=94=20record-level=20attempts,=20flaky=5Fretries,=20repor?= =?UTF-8?q?t=20surfacing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bun --retry leaves a retried pass INVISIBLE in its output: a fail-then-pass prints the error detail but no (fail) result line and recaps as a clean pass (probed live on 1.3.10). So attempts are recorded where they cannot lie: EvalCollector.addTest stamps a 1-based attempt on same-name re-records (a retried test runs its body again and re-records), finalized runs carry flaky_retries, printSummary warns loudly, and the fail-closed slices report lists every passed-only-on-retry test — recorded and ranked, never blocking and never silent. Cross-model confirmed (codex reached the same don't-parse -the-stream conclusion independently). Co-Authored-By: Claude Fable 5 --- scripts/test-paid-shards.ts | 17 +++++++++++++++++ test/helpers/eval-store.test.ts | 25 +++++++++++++++++++++++++ test/helpers/eval-store.ts | 33 ++++++++++++++++++++++++++++++++- 3 files changed, 74 insertions(+), 1 deletion(-) diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts index 077e598c9..bcbcf5851 100644 --- a/scripts/test-paid-shards.ts +++ b/scripts/test-paid-shards.ts @@ -1002,6 +1002,23 @@ async function main(): Promise { for (const problem of verdict.problems) console.error(` ✗ ${problem}`); return 1; } + // Flake honesty (WS1): surface every test that needed a retry to pass. + // WARNS, never fails — a flaky pass must not block merges; it must also + // never be invisible (bun's own output hides retried passes entirely). + // Source: the finalized eval-store JSONs inside the slice artifacts. + const flaky: Array<{ name: string; attempts: number; file: string }> = []; + for (const name of fs.readdirSync(options.reportDir, { recursive: true }) as string[]) { + if (!/\.json$/.test(name) || /manifest\.json$|slice-\d+\.json$|^_partial|\/_partial/.test(name)) continue; + try { + const parsed = JSON.parse(fs.readFileSync(path.join(options.reportDir, name), 'utf-8')) as { flaky_retries?: Array<{ name: string; attempts: number }> }; + for (const f of parsed.flaky_retries ?? []) flaky.push({ ...f, file: name }); + } catch { /* non-eval JSON — not this report's business */ } + } + if (flaky.length > 0) { + console.log(`[test:paid] report: ⚠ ${flaky.length} test(s) passed only on retry this run (recorded, not blocking):`); + for (const f of flaky) console.log(` ⚠ ${f.name} (x${f.attempts}) — ${f.file}`); + } + // Census honesty: a 'passed' shard whose every test skipped verified // nothing (external-service binary absent on the runner). Not a failure — // service availability is host state, not a repo regression — but the diff --git a/test/helpers/eval-store.test.ts b/test/helpers/eval-store.test.ts index bd1d3a631..cf02e28d8 100644 --- a/test/helpers/eval-store.test.ts +++ b/test/helpers/eval-store.test.ts @@ -121,6 +121,31 @@ describe('EvalCollector', () => { expect(data.claude_cli_version!.length).toBeGreaterThan(0); }); + test('a same-name re-record stamps attempts and surfaces flaky_retries', async () => { + // bun --retry re-runs the test BODY, so recordE2E fires again under the + // same name — the only reliable retry signal (bun's own output hides + // retried passes: fail→pass recaps as a clean pass, probed on 1.3.10). + const collector = new EvalCollector('e2e', tmpDir); + collector.addTest(makeEntry({ name: 'flaky-one', passed: false })); + collector.addTest(makeEntry({ name: 'flaky-one', passed: true })); + collector.addTest(makeEntry({ name: 'steady', passed: true })); + const filepath = await collector.finalize(); + + const data: EvalResult = JSON.parse(fs.readFileSync(filepath, 'utf-8')); + const attempts = data.tests.filter((t) => t.name === 'flaky-one').map((t) => t.attempt); + expect(attempts).toEqual([1, 2]); + expect(data.tests.find((t) => t.name === 'steady')?.attempt).toBe(1); + expect(data.flaky_retries).toEqual([{ name: 'flaky-one', attempts: 2 }]); + }); + + test('no retries → no flaky_retries field (absent, not empty)', async () => { + const collector = new EvalCollector('e2e', tmpDir); + collector.addTest(makeEntry({ name: 'only-once' })); + const filepath = await collector.finalize(); + const data: EvalResult = JSON.parse(fs.readFileSync(filepath, 'utf-8')); + expect('flaky_retries' in data).toBe(false); + }); + test('finalize creates directory if missing', async () => { const nestedDir = path.join(tmpDir, 'nested', 'deep', 'evals'); const collector = new EvalCollector('e2e', nestedDir); diff --git a/test/helpers/eval-store.ts b/test/helpers/eval-store.ts index 53a3d6201..1ca3edbf0 100644 --- a/test/helpers/eval-store.ts +++ b/test/helpers/eval-store.ts @@ -63,6 +63,12 @@ export interface EvalTestEntry { passed: boolean; duration_ms: number; cost_usd: number; + /** 1-based record attempt for this name in this run. bun's --retry leaves + * retried passes INVISIBLE in its text output (a fail→pass prints no + * (fail) line and recaps as a clean pass — probed on 1.3.10), so the ONLY + * reliable attempt signal is this in-process record: a retried test runs + * its body again and re-records under the same name. Set by addTest. */ + attempt?: number; // E2E transcript?: any[]; @@ -132,6 +138,11 @@ export interface EvalResult { tests: EvalTestEntry[]; /** Shard slug when the run was collected under /shards//. */ shard?: string; + /** Tests recorded more than once this run — the flake ledger for the paid + * lane. A test passing on attempt 2 every week used to read permanently + * green (the retry's entry was indistinguishable and bun's output hides + * retries entirely). Present only when non-empty. */ + flaky_retries?: Array<{ name: string; attempts: number }>; _partial?: boolean; // true for incremental saves, absent in final } @@ -825,10 +836,23 @@ export class EvalCollector { } addTest(entry: EvalTestEntry): void { - this.tests.push(entry); + // Same-name re-record = the test body ran again = bun retried it (test + // names are unique by convention). Stamp the 1-based attempt so a + // pass-on-attempt-2 stays visible forever — the stream hides it. + const prior = this.tests.filter((t) => t.name === entry.name).length; + this.tests.push({ ...entry, attempt: prior + 1 }); this.savePartial(); } + /** Names recorded more than once this run, with their attempt counts. */ + private flakyRetries(): Array<{ name: string; attempts: number }> { + const counts = new Map(); + for (const t of this.tests) counts.set(t.name, (counts.get(t.name) ?? 0) + 1); + return [...counts.entries()] + .filter(([, n]) => n > 1) + .map(([name, attempts]) => ({ name, attempts })); + } + /** Write incremental results after each test. Atomic write, non-fatal. */ savePartial(): void { try { @@ -893,6 +917,7 @@ export class EvalCollector { wall_clock_ms: Date.now() - this.createdAt, tests: this.tests, ...(this.shard ? { shard: this.shard } : {}), + ...(this.flakyRetries().length > 0 ? { flaky_retries: this.flakyRetries() } : {}), }; // Write eval file @@ -955,6 +980,12 @@ export class EvalCollector { const totalCost = `$${result.total_cost_usd.toFixed(2)}`; const totalDur = `${Math.round(result.total_duration_ms / 1000)}s`; lines.push(` Total: ${result.passed}/${result.total_tests} passed${' '.repeat(20)}${totalCost.padStart(6)} ${totalDur}`); + if (result.flaky_retries && result.flaky_retries.length > 0) { + // Loud, never fatal: a flaky pass must not block anyone, but it must + // never be silent either — that invisibility is how flakes calcified. + lines.push(` ⚠ FLAKY: ${result.flaky_retries.length} test(s) recorded multiple attempts this run: ` + + result.flaky_retries.map((f) => `${f.name} (x${f.attempts})`).join(', ')); + } lines.push(`Saved: ${filepath}`); process.stderr.write(lines.join('\n') + '\n');