fix(test-free): a flaky pass keeps the failing assertion in the epilogue and the flake ledger

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## [Unreleased]
+
+**A free-suite flake now keeps the assertion that failed.** Four free-test flakes in the flaky ledger (plan-seed-submission, native-paste, qa-only preparation-late and the gstack-upgrade fresh-shell positive control) could not be diagnosed: each failed once on a Ubicloud or remote-pregate run, passed on the serial retry, and its log died with the machine that pulled it. The runner's console printed only the failing test's name, and the flake ledger recorded only the file.
+
+### Changed
+- **`scripts/test-free-shards.ts`**: the result reporter keeps the error bun printed before each `(fail)` line: from its first `error:` line, without blanks or CI annotations, at most 8 lines of 240 characters. The FAIL epilogue prints that excerpt under each `✗ <file> — <test>` line, so it reaches the console stream that remote pregates relay. A FLAKY-PASS ledger entry now records `tests: [{ name, excerpt }]` for each file. Existing ledger readers ignore the new field. Verdicts, retries and their cap are unchanged.
+
+### Tests
+- `test/test-free-shards.test.ts` replays the Windows team-init PowerShell failure as bun printed it (run 39448730953, attempt 1) and checks the excerpt and the epilogue. The real CLI flaky-retry routes (`retry-reader`, `retry-exclusive`) now check that the ledger entry names the failed test with `Expected: 2` / `Received: 1`.
+
 ## [1.91.173.0] - 2026-10-20
 
 **Merge trains can merge slot 2 onward.** After slot 1 merges, main carries the new stamp, so every later pin conflicts with main on the stamp files and GitHub can't squash it. A lane makes its PR mergeable by merging main into the pin before it stamps. `verify --pr <n> --train <nonce>` refused that head as moved off its pin. Now it also accepts a head whose tree equals the slot's train prefix (T<i>) modulo stamps: exactly the tree the train's gate covered, and what main becomes once the PR squash-merges. The first live train stopped at slot 2 on this (gate.train-ac4baf961759-43d6ed28f2bf-9a1dbb). Plan GSTA-49.

scripts/test-free-shards.ts

@@ -683,6 +683,15 @@ export interface FreeRunFailure {
   /** Planned relative path when attributable, else the raw header path, else null. */
   file: string | null;
   testName: string;
+  /** The error bun printed before the (fail) line (message, Expected/Received, location), bounded. */
+  excerpt?: string[];
+}
+
+/** The failing assertion from a test's own output: from its first error: line, without blanks or CI annotations. */
+export function failureExcerpt(output: string[]): string[] {
+  const lines = output.filter((line) => line.trim() !== '' && !line.startsWith('::'));
+  const start = lines.findIndex((line) => line.startsWith('error:'));
+  return (start < 0 ? lines.slice(-3) : lines.slice(start, start + 8)).map((line) => line.trimEnd().slice(0, 240));
 }
 
 export interface FreeRunReport {
@@ -759,6 +768,7 @@ export class FreeRunReporter {
   private filesRan: number | null = null;
   private sawSummary = false;
   private lastOutputLine: string | null = null;
+  private testOutput: string[] = [];
 
   constructor(
     private readonly plannedFiles: string[],
@@ -848,6 +858,7 @@ export class FreeRunReporter {
       if (this.currentFile && this.currentFile !== file) this.progressFor(this.currentFile).ended = true;
       this.currentFile = file;
       this.progressFor(file).headerSeen = true;
+      this.testOutput = [];
     } else {
       const fail = parseBunFailureResult(line);
       const retry = fail ? null : CRASH_RETRY_RE.exec(line);
@@ -862,8 +873,10 @@ export class FreeRunReporter {
         const key = `${this.currentFile ?? ''}\u0000${fail}`;
         if (!recapDuplicate && !this.failureKeys.has(key)) {
           this.failureKeys.add(key);
-          this.failures.push({ file: this.currentFile, testName: fail });
+          const excerpt = failureExcerpt(this.testOutput);
+          this.failures.push({ file: this.currentFile, testName: fail, ...(excerpt.length ? { excerpt } : {}) });
         }
+        this.testOutput = [];
       } else if (retry) {
         // The file will run again — a crash+retry does not end its chunk.
         visible = true;
@@ -873,7 +886,10 @@ export class FreeRunReporter {
         const file = this.canonicalize(final[1]);
         this.crashed.add(file);
         this.progressFor(file).ended = true;
+      } else if (/^\((?:pass|skip|todo)\) /.test(line)) {
+        this.testOutput = [];
       } else {
+        this.testOutput = [...this.testOutput.slice(-39), line];
         const summary = TERMINAL_SUMMARY_CAPTURE_RE.exec(line);
         if (summary) {
           visible = true;
@@ -943,6 +959,7 @@ export function buildRunEpilogue(
   ];
   for (const failure of report.failures) {
     lines.push(`  ✗ ${failure.file ?? '(unattributed)'} — ${failure.testName}`);
+    for (const line of failure.excerpt ?? []) lines.push(`      ${line}`);
   }
   if (report.unreportedFailures > 0) {
     lines.push(`  ⚠ ${report.unreportedFailures} failure(s) reported without named result lines`);
@@ -993,6 +1010,8 @@ export interface FlakeLedgerEntry {
    *  promotion evidence needs exactly that. */
   branch?: string;
   git_sha?: string;
+  /** The tests that failed in the first run, with the error each printed. */
+  tests?: Array<{ name: string; excerpt: string[] }>;
 }
 
 export function flakeLedgerPath(env: NodeJS.ProcessEnv = process.env): string {
@@ -1054,6 +1073,8 @@ export interface FreeShardOutcome {
    * hole the strict classifier exists to close.
    */
   unattributedFailures: number;
+  /** Named failures with their excerpts, so a flaky pass records what failed the first time. */
+  failures?: FreeRunFailure[];
 }
 
 export interface ShardCommand {
@@ -1722,6 +1743,7 @@ export async function runFreeShard(
       + (report.sawTerminalSummary ? 0 : 1);
   const outcome: FreeShardOutcome = {
     shard: shardNumber, files, status, exitCode, elapsedMs: Date.now() - startedAt, groupPid, failingFiles, unattributedFailures,
+    ...(status === 'passed' ? {} : { failures: report.failures }),
     summary: { testsRan: report.testsRan, filesRan: report.filesRan, sawTerminalSummary: report.sawTerminalSummary },
   };
   log(shardEpilogue(outcome, totalShards));
@@ -1856,6 +1878,7 @@ async function retryFailedFreeFiles(
             kind: 'flaky-pass' as const,
             file,
             shard: outcomes.find((o) => o.failingFiles.includes(file))?.shard,
+            tests: outcomes.flatMap((o) => o.failures ?? []).filter((f) => f.file === file).map((f) => ({ name: f.testName, excerpt: f.excerpt ?? [] })),
             ...(ledgerBranch ? { branch: ledgerBranch } : {}),
             ...(ledgerSha ? { git_sha: ledgerSha.slice(0, 12) } : {}),
           })),

test/test-free-shards.test.ts

@@ -14,6 +14,7 @@ import {
   runFreeShard,
   FreeRunReporter,
   buildRunEpilogue,
+  failureExcerpt,
   FREE_TEST_TIMEOUT_MS,
   DEFAULT_WALL_TIMEOUT_MS,
   PER_FILE_WALL_MS,
@@ -272,6 +273,12 @@ describe('test-free-shards: exclusive host-state phase', () => {
         expect(counts).toEqual(mode === 'retry-reader' ? [2, 1, 1] : mode === 'retry-exclusive' ? [1, 1, 2] : [1, 1, 1]);
         if (mode.startsWith('retry-')) {
           expect(output).toContain('FLAKY-PASS');
+          expect(output).toContain('      Expected: 2');
+          const [flake] = fs.readFileSync(path.join(directory, 'flakes.jsonl'), 'utf8').trim().split('\n').map(line => JSON.parse(line));
+          expect(flake.file).toBe(selected[mode === 'retry-reader' ? 0 : 2]);
+          expect(flake.tests).toHaveLength(1);
+          expect(flake.tests[0].name).toBe('selected fixture');
+          expect(flake.tests[0].excerpt.slice(0, 3)).toEqual(['error: expect(received).toBe(expected)', 'Expected: 2', 'Received: 1']);
           expect(events[6]).toEqual({ file: selected[mode === 'retry-reader' ? 0 : 2], event: 'start' });
         } else if (mode === 'truncated-exclusive') {
           expect(output).toContain('flaky-retry skipped');
@@ -826,6 +833,38 @@ describe('test-free-shards: output contract (log capture, quiet console, failure
     expect(lines.some((l) => l.includes('2 planned file(s) produced no output'))).toBe(true);
   });
 
+  // Windows run 39448730953 attempt 1, windows-free-shard (6): the team-init PowerShell case as bun printed it.
+  test('a failure keeps the assertion bun printed before its (fail) line, and the epilogue shows it', () => {
+    const reporter = new FreeRunReporter(['test/team-init-hook-shell-neutral.test.ts']);
+    reporter.write([
+      '::group::test\\team-init-hook-shell-neutral.test.ts:',
+      '(pass) required team hook: shell-neutral registration > through the default shell: deny with exit 2 when no install resolves, {} when one does [211.85ms]',
+      'killed 1 dangling process',
+      '66 |       const denied = runner(hookEnv({ HOME: home, USERPROFILE: home }));',
+      '67 |       expect(denied.status).toBe(denyStatus);',
+      '                                 ^',
+      'error: expect(received).toBe(expected)',
+      '',
+      'Expected: 1',
+      'Received: null',
+      '',
+      '      at <anonymous> (D:\\a\\gstack\\gstack\\test\\team-init-hook-shell-neutral.test.ts:67:29)',
+      '',
+      '::error file=test\\team-init-hook-shell-neutral.test.ts,line=67,col=29,title=error: expect(received).toBe(expected)::Expected: 1',
+      '(fail) required team hook: shell-neutral registration > through PowerShell: deny with exit 2 when no install resolves, {} when one does [30108.20ms]',
+      '  ^ this test timed out after 30000ms.',
+      '',
+    ].join('\n'), 'stderr');
+    reporter.end();
+    const [failure] = reporter.report().failures;
+    expect(failure.excerpt).toEqual(['error: expect(received).toBe(expected)', 'Expected: 1', 'Received: null',
+      '      at <anonymous> (D:\\a\\gstack\\gstack\\test\\team-init-hook-shell-neutral.test.ts:67:29)']);
+    const lines = buildRunEpilogue('failed', reporter.report(), 30_000, '/tmp/x.log');
+    expect(lines).toContain('      Received: null');
+    expect(failureExcerpt(['noise', 'one', 'two', 'three'])).toEqual(['one', 'two', 'three']);
+    expect(failureExcerpt(['error: x'.padEnd(300, 'y'), ...Array.from({ length: 10 }, (_, i) => 'line ' + i)])).toHaveLength(8);
+  });
+
   test('duplicate fail lines dedupe; pre-header failures are labeled unattributed', () => {
     const reporter = new FreeRunReporter(['test/a.test.ts']);
     reporter.write(`${failLine('early unattributed')}\n`, 'stderr');