feat: paid-lane flake telemetry — record-level attempts, flaky_retries, report surfacing

bun --retry leaves a retried pass INVISIBLE in its output: a fail-then-pass
prints the error detail but no (fail) result line and recaps as a clean
pass (probed live on 1.3.10). So attempts are recorded where they cannot
lie: EvalCollector.addTest stamps a 1-based attempt on same-name re-records
(a retried test runs its body again and re-records), finalized runs carry
flaky_retries, printSummary warns loudly, and the fail-closed slices report
lists every passed-only-on-retry test — recorded and ranked, never blocking
and never silent. Cross-model confirmed (codex reached the same don't-parse
-the-stream conclusion independently).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

scripts/test-paid-shards.ts

@@ -1002,6 +1002,23 @@ async function main(): Promise<number> {
       for (const problem of verdict.problems) console.error(`  ✗ ${problem}`);
       return 1;
     }
+    // Flake honesty (WS1): surface every test that needed a retry to pass.
+    // WARNS, never fails — a flaky pass must not block merges; it must also
+    // never be invisible (bun's own output hides retried passes entirely).
+    // Source: the finalized eval-store JSONs inside the slice artifacts.
+    const flaky: Array<{ name: string; attempts: number; file: string }> = [];
+    for (const name of fs.readdirSync(options.reportDir, { recursive: true }) as string[]) {
+      if (!/\.json$/.test(name) || /manifest\.json$|slice-\d+\.json$|^_partial|\/_partial/.test(name)) continue;
+      try {
+        const parsed = JSON.parse(fs.readFileSync(path.join(options.reportDir, name), 'utf-8')) as { flaky_retries?: Array<{ name: string; attempts: number }> };
+        for (const f of parsed.flaky_retries ?? []) flaky.push({ ...f, file: name });
+      } catch { /* non-eval JSON — not this report's business */ }
+    }
+    if (flaky.length > 0) {
+      console.log(`[test:paid] report: ⚠ ${flaky.length} test(s) passed only on retry this run (recorded, not blocking):`);
+      for (const f of flaky) console.log(`  ⚠ ${f.name} (x${f.attempts}) — ${f.file}`);
+    }
+
     // Census honesty: a 'passed' shard whose every test skipped verified
     // nothing (external-service binary absent on the runner). Not a failure —
     // service availability is host state, not a repo regression — but the

test/helpers/eval-store.test.ts

@@ -121,6 +121,31 @@ describe('EvalCollector', () => {
     expect(data.claude_cli_version!.length).toBeGreaterThan(0);
   });
 
+  test('a same-name re-record stamps attempts and surfaces flaky_retries', async () => {
+    // bun --retry re-runs the test BODY, so recordE2E fires again under the
+    // same name — the only reliable retry signal (bun's own output hides
+    // retried passes: fail→pass recaps as a clean pass, probed on 1.3.10).
+    const collector = new EvalCollector('e2e', tmpDir);
+    collector.addTest(makeEntry({ name: 'flaky-one', passed: false }));
+    collector.addTest(makeEntry({ name: 'flaky-one', passed: true }));
+    collector.addTest(makeEntry({ name: 'steady', passed: true }));
+    const filepath = await collector.finalize();
+
+    const data: EvalResult = JSON.parse(fs.readFileSync(filepath, 'utf-8'));
+    const attempts = data.tests.filter((t) => t.name === 'flaky-one').map((t) => t.attempt);
+    expect(attempts).toEqual([1, 2]);
+    expect(data.tests.find((t) => t.name === 'steady')?.attempt).toBe(1);
+    expect(data.flaky_retries).toEqual([{ name: 'flaky-one', attempts: 2 }]);
+  });
+
+  test('no retries → no flaky_retries field (absent, not empty)', async () => {
+    const collector = new EvalCollector('e2e', tmpDir);
+    collector.addTest(makeEntry({ name: 'only-once' }));
+    const filepath = await collector.finalize();
+    const data: EvalResult = JSON.parse(fs.readFileSync(filepath, 'utf-8'));
+    expect('flaky_retries' in data).toBe(false);
+  });
+
   test('finalize creates directory if missing', async () => {
     const nestedDir = path.join(tmpDir, 'nested', 'deep', 'evals');
     const collector = new EvalCollector('e2e', nestedDir);

test/helpers/eval-store.ts

@@ -63,6 +63,12 @@ export interface EvalTestEntry {
   passed: boolean;
   duration_ms: number;
   cost_usd: number;
+  /** 1-based record attempt for this name in this run. bun's --retry leaves
+   *  retried passes INVISIBLE in its text output (a fail→pass prints no
+   *  (fail) line and recaps as a clean pass — probed on 1.3.10), so the ONLY
+   *  reliable attempt signal is this in-process record: a retried test runs
+   *  its body again and re-records under the same name. Set by addTest. */
+  attempt?: number;
 
   // E2E
   transcript?: any[];
@@ -132,6 +138,11 @@ export interface EvalResult {
   tests: EvalTestEntry[];
   /** Shard slug when the run was collected under <evalDir>/shards/<slug>/. */
   shard?: string;
+  /** Tests recorded more than once this run — the flake ledger for the paid
+   *  lane. A test passing on attempt 2 every week used to read permanently
+   *  green (the retry's entry was indistinguishable and bun's output hides
+   *  retries entirely). Present only when non-empty. */
+  flaky_retries?: Array<{ name: string; attempts: number }>;
   _partial?: boolean;  // true for incremental saves, absent in final
 }
 
@@ -825,10 +836,23 @@ export class EvalCollector {
   }
 
   addTest(entry: EvalTestEntry): void {
-    this.tests.push(entry);
+    // Same-name re-record = the test body ran again = bun retried it (test
+    // names are unique by convention). Stamp the 1-based attempt so a
+    // pass-on-attempt-2 stays visible forever — the stream hides it.
+    const prior = this.tests.filter((t) => t.name === entry.name).length;
+    this.tests.push({ ...entry, attempt: prior + 1 });
     this.savePartial();
   }
 
+  /** Names recorded more than once this run, with their attempt counts. */
+  private flakyRetries(): Array<{ name: string; attempts: number }> {
+    const counts = new Map<string, number>();
+    for (const t of this.tests) counts.set(t.name, (counts.get(t.name) ?? 0) + 1);
+    return [...counts.entries()]
+      .filter(([, n]) => n > 1)
+      .map(([name, attempts]) => ({ name, attempts }));
+  }
+
   /** Write incremental results after each test. Atomic write, non-fatal. */
   savePartial(): void {
     try {
@@ -893,6 +917,7 @@ export class EvalCollector {
       wall_clock_ms: Date.now() - this.createdAt,
       tests: this.tests,
       ...(this.shard ? { shard: this.shard } : {}),
+      ...(this.flakyRetries().length > 0 ? { flaky_retries: this.flakyRetries() } : {}),
     };
 
     // Write eval file
@@ -955,6 +980,12 @@ export class EvalCollector {
     const totalCost = `$${result.total_cost_usd.toFixed(2)}`;
     const totalDur = `${Math.round(result.total_duration_ms / 1000)}s`;
     lines.push(`  Total: ${result.passed}/${result.total_tests} passed${' '.repeat(20)}${totalCost.padStart(6)}  ${totalDur}`);
+    if (result.flaky_retries && result.flaky_retries.length > 0) {
+      // Loud, never fatal: a flaky pass must not block anyone, but it must
+      // never be silent either — that invisibility is how flakes calcified.
+      lines.push(`  ⚠ FLAKY: ${result.flaky_retries.length} test(s) recorded multiple attempts this run: `
+        + result.flaky_retries.map((f) => `${f.name} (x${f.attempts})`).join(', '));
+    }
     lines.push(`Saved: ${filepath}`);
 
     process.stderr.write(lines.join('\n') + '\n');