feat: paid-lane flake telemetry — record-level attempts, flaky_retries, report surfacing

bun --retry leaves a retried pass INVISIBLE in its output: a fail-then-pass
prints the error detail but no (fail) result line and recaps as a clean
pass (probed live on 1.3.10). So attempts are recorded where they cannot
lie: EvalCollector.addTest stamps a 1-based attempt on same-name re-records
(a retried test runs its body again and re-records), finalized runs carry
flaky_retries, printSummary warns loudly, and the fail-closed slices report
lists every passed-only-on-retry test — recorded and ranked, never blocking
and never silent. Cross-model confirmed (codex reached the same don't-parse
-the-stream conclusion independently).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

test/helpers/eval-store.test.ts

@@ -121,6 +121,31 @@ describe('EvalCollector', () => {
     expect(data.claude_cli_version!.length).toBeGreaterThan(0);
   });
 
+  test('a same-name re-record stamps attempts and surfaces flaky_retries', async () => {
+    // bun --retry re-runs the test BODY, so recordE2E fires again under the
+    // same name — the only reliable retry signal (bun's own output hides
+    // retried passes: fail→pass recaps as a clean pass, probed on 1.3.10).
+    const collector = new EvalCollector('e2e', tmpDir);
+    collector.addTest(makeEntry({ name: 'flaky-one', passed: false }));
+    collector.addTest(makeEntry({ name: 'flaky-one', passed: true }));
+    collector.addTest(makeEntry({ name: 'steady', passed: true }));
+    const filepath = await collector.finalize();
+
+    const data: EvalResult = JSON.parse(fs.readFileSync(filepath, 'utf-8'));
+    const attempts = data.tests.filter((t) => t.name === 'flaky-one').map((t) => t.attempt);
+    expect(attempts).toEqual([1, 2]);
+    expect(data.tests.find((t) => t.name === 'steady')?.attempt).toBe(1);
+    expect(data.flaky_retries).toEqual([{ name: 'flaky-one', attempts: 2 }]);
+  });
+
+  test('no retries → no flaky_retries field (absent, not empty)', async () => {
+    const collector = new EvalCollector('e2e', tmpDir);
+    collector.addTest(makeEntry({ name: 'only-once' }));
+    const filepath = await collector.finalize();
+    const data: EvalResult = JSON.parse(fs.readFileSync(filepath, 'utf-8'));
+    expect('flaky_retries' in data).toBe(false);
+  });
+
   test('finalize creates directory if missing', async () => {
     const nestedDir = path.join(tmpDir, 'nested', 'deep', 'evals');
     const collector = new EvalCollector('e2e', nestedDir);