From 4065e340e9a3556a0aa193f45e77dea2d5b3fbf6 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Wed, 21 Oct 2026 11:26:03 -0700 Subject: [PATCH] v1.91.175.0 fix(test-free): a flaky pass keeps the failing assertion in the epilogue and the flake ledger (#3331) * fix(test-free): a flaky pass keeps the failing assertion in the epilogue and the flake ledger Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * v1.91.175.0 stamp Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 10 +++++++++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- package.json | 2 +- scripts/test-free-shards.ts | 25 +++++++++++++++++++++- test/test-free-shards.test.ts | 39 ++++++++++++++++++++++++++++++++++ 6 files changed, 76 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7599e7af3..58d313b22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## [1.91.175.0] - 2026-10-21 + +**A free-suite flake now keeps the assertion that failed.** Four free-test flakes in the flaky ledger (plan-seed-submission, native-paste, qa-only preparation-late and the gstack-upgrade fresh-shell positive control) could not be diagnosed: each failed once on a Ubicloud or remote-pregate run, passed on the serial retry, and its log died with the machine that pulled it. The runner's console printed only the failing test's name, and the flake ledger recorded only the file. + +### Changed +- **`scripts/test-free-shards.ts`**: the result reporter keeps the error bun printed before each `(fail)` line: from its first `error:` line, without blanks or CI annotations, at most 8 lines of 240 characters. The FAIL epilogue prints that excerpt under each `✗ — ` line, so it reaches the console stream that remote pregates relay. A FLAKY-PASS ledger entry now records `tests: [{ name, excerpt }]` for each file. Existing ledger readers ignore the new field. Verdicts, retries and their cap are unchanged. + +### Tests +- `test/test-free-shards.test.ts` replays the Windows team-init PowerShell failure as bun printed it (run 39448730953, attempt 1) and checks the excerpt and the epilogue. The real CLI flaky-retry routes (`retry-reader`, `retry-exclusive`) now check that the ledger entry names the failed test with `Expected: 2` / `Received: 1`. + ## [1.91.174.0] - 2026-10-21 **Two paid eval flakes from the 2026-10-20 trains are repaired at their cause, with no threshold, trial count or case kind changed.** ship-local-hook-preservation failed in train gate 39440559183 even though the agent told the user "you'll need to set it up by hand": the manual-integration detector knew "set up" and "wire in" only with the particle right after the verb. ship-docsync-current lost a focused trial to an unquoted `grep … --include=*`, which the shared read-only grammar refuses on purpose because the shell expands the glob first; the grammar's declared interface said "unquoted glob matches included" and never said that an option's glob has to be quoted. diff --git a/VERSION b/VERSION index b62916591..a80eb989a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.174.0 +1.91.175.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 6cfe536af..dbaed5070 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.174.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.175.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/package.json b/package.json index a52f66ef7..3e5a43d26 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.174", + "version": "1.91.175", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/test-free-shards.ts b/scripts/test-free-shards.ts index 50528ef43..ba82c3a48 100755 --- a/scripts/test-free-shards.ts +++ b/scripts/test-free-shards.ts @@ -683,6 +683,15 @@ export interface FreeRunFailure { /** Planned relative path when attributable, else the raw header path, else null. */ file: string | null; testName: string; + /** The error bun printed before the (fail) line (message, Expected/Received, location), bounded. */ + excerpt?: string[]; +} + +/** The failing assertion from a test's own output: from its first error: line, without blanks or CI annotations. */ +export function failureExcerpt(output: string[]): string[] { + const lines = output.filter((line) => line.trim() !== '' && !line.startsWith('::')); + const start = lines.findIndex((line) => line.startsWith('error:')); + return (start < 0 ? lines.slice(-3) : lines.slice(start, start + 8)).map((line) => line.trimEnd().slice(0, 240)); } export interface FreeRunReport { @@ -759,6 +768,7 @@ export class FreeRunReporter { private filesRan: number | null = null; private sawSummary = false; private lastOutputLine: string | null = null; + private testOutput: string[] = []; constructor( private readonly plannedFiles: string[], @@ -848,6 +858,7 @@ export class FreeRunReporter { if (this.currentFile && this.currentFile !== file) this.progressFor(this.currentFile).ended = true; this.currentFile = file; this.progressFor(file).headerSeen = true; + this.testOutput = []; } else { const fail = parseBunFailureResult(line); const retry = fail ? null : CRASH_RETRY_RE.exec(line); @@ -862,8 +873,10 @@ export class FreeRunReporter { const key = `${this.currentFile ?? ''}\u0000${fail}`; if (!recapDuplicate && !this.failureKeys.has(key)) { this.failureKeys.add(key); - this.failures.push({ file: this.currentFile, testName: fail }); + const excerpt = failureExcerpt(this.testOutput); + this.failures.push({ file: this.currentFile, testName: fail, ...(excerpt.length ? { excerpt } : {}) }); } + this.testOutput = []; } else if (retry) { // The file will run again — a crash+retry does not end its chunk. visible = true; @@ -873,7 +886,10 @@ export class FreeRunReporter { const file = this.canonicalize(final[1]); this.crashed.add(file); this.progressFor(file).ended = true; + } else if (/^\((?:pass|skip|todo)\) /.test(line)) { + this.testOutput = []; } else { + this.testOutput = [...this.testOutput.slice(-39), line]; const summary = TERMINAL_SUMMARY_CAPTURE_RE.exec(line); if (summary) { visible = true; @@ -943,6 +959,7 @@ export function buildRunEpilogue( ]; for (const failure of report.failures) { lines.push(` ✗ ${failure.file ?? '(unattributed)'} — ${failure.testName}`); + for (const line of failure.excerpt ?? []) lines.push(` ${line}`); } if (report.unreportedFailures > 0) { lines.push(` ⚠ ${report.unreportedFailures} failure(s) reported without named result lines`); @@ -993,6 +1010,8 @@ export interface FlakeLedgerEntry { * promotion evidence needs exactly that. */ branch?: string; git_sha?: string; + /** The tests that failed in the first run, with the error each printed. */ + tests?: Array<{ name: string; excerpt: string[] }>; } export function flakeLedgerPath(env: NodeJS.ProcessEnv = process.env): string { @@ -1054,6 +1073,8 @@ export interface FreeShardOutcome { * hole the strict classifier exists to close. */ unattributedFailures: number; + /** Named failures with their excerpts, so a flaky pass records what failed the first time. */ + failures?: FreeRunFailure[]; } export interface ShardCommand { @@ -1722,6 +1743,7 @@ export async function runFreeShard( + (report.sawTerminalSummary ? 0 : 1); const outcome: FreeShardOutcome = { shard: shardNumber, files, status, exitCode, elapsedMs: Date.now() - startedAt, groupPid, failingFiles, unattributedFailures, + ...(status === 'passed' ? {} : { failures: report.failures }), summary: { testsRan: report.testsRan, filesRan: report.filesRan, sawTerminalSummary: report.sawTerminalSummary }, }; log(shardEpilogue(outcome, totalShards)); @@ -1856,6 +1878,7 @@ async function retryFailedFreeFiles( kind: 'flaky-pass' as const, file, shard: outcomes.find((o) => o.failingFiles.includes(file))?.shard, + tests: outcomes.flatMap((o) => o.failures ?? []).filter((f) => f.file === file).map((f) => ({ name: f.testName, excerpt: f.excerpt ?? [] })), ...(ledgerBranch ? { branch: ledgerBranch } : {}), ...(ledgerSha ? { git_sha: ledgerSha.slice(0, 12) } : {}), })), diff --git a/test/test-free-shards.test.ts b/test/test-free-shards.test.ts index 16272b9f8..620fbaf51 100644 --- a/test/test-free-shards.test.ts +++ b/test/test-free-shards.test.ts @@ -14,6 +14,7 @@ import { runFreeShard, FreeRunReporter, buildRunEpilogue, + failureExcerpt, FREE_TEST_TIMEOUT_MS, DEFAULT_WALL_TIMEOUT_MS, PER_FILE_WALL_MS, @@ -272,6 +273,12 @@ describe('test-free-shards: exclusive host-state phase', () => { expect(counts).toEqual(mode === 'retry-reader' ? [2, 1, 1] : mode === 'retry-exclusive' ? [1, 1, 2] : [1, 1, 1]); if (mode.startsWith('retry-')) { expect(output).toContain('FLAKY-PASS'); + expect(output).toContain(' Expected: 2'); + const [flake] = fs.readFileSync(path.join(directory, 'flakes.jsonl'), 'utf8').trim().split('\n').map(line => JSON.parse(line)); + expect(flake.file).toBe(selected[mode === 'retry-reader' ? 0 : 2]); + expect(flake.tests).toHaveLength(1); + expect(flake.tests[0].name).toBe('selected fixture'); + expect(flake.tests[0].excerpt.slice(0, 3)).toEqual(['error: expect(received).toBe(expected)', 'Expected: 2', 'Received: 1']); expect(events[6]).toEqual({ file: selected[mode === 'retry-reader' ? 0 : 2], event: 'start' }); } else if (mode === 'truncated-exclusive') { expect(output).toContain('flaky-retry skipped'); @@ -826,6 +833,38 @@ describe('test-free-shards: output contract (log capture, quiet console, failure expect(lines.some((l) => l.includes('2 planned file(s) produced no output'))).toBe(true); }); + // Windows run 39448730953 attempt 1, windows-free-shard (6): the team-init PowerShell case as bun printed it. + test('a failure keeps the assertion bun printed before its (fail) line, and the epilogue shows it', () => { + const reporter = new FreeRunReporter(['test/team-init-hook-shell-neutral.test.ts']); + reporter.write([ + '::group::test\\team-init-hook-shell-neutral.test.ts:', + '(pass) required team hook: shell-neutral registration > through the default shell: deny with exit 2 when no install resolves, {} when one does [211.85ms]', + 'killed 1 dangling process', + '66 | const denied = runner(hookEnv({ HOME: home, USERPROFILE: home }));', + '67 | expect(denied.status).toBe(denyStatus);', + ' ^', + 'error: expect(received).toBe(expected)', + '', + 'Expected: 1', + 'Received: null', + '', + ' at (D:\\a\\gstack\\gstack\\test\\team-init-hook-shell-neutral.test.ts:67:29)', + '', + '::error file=test\\team-init-hook-shell-neutral.test.ts,line=67,col=29,title=error: expect(received).toBe(expected)::Expected: 1', + '(fail) required team hook: shell-neutral registration > through PowerShell: deny with exit 2 when no install resolves, {} when one does [30108.20ms]', + ' ^ this test timed out after 30000ms.', + '', + ].join('\n'), 'stderr'); + reporter.end(); + const [failure] = reporter.report().failures; + expect(failure.excerpt).toEqual(['error: expect(received).toBe(expected)', 'Expected: 1', 'Received: null', + ' at (D:\\a\\gstack\\gstack\\test\\team-init-hook-shell-neutral.test.ts:67:29)']); + const lines = buildRunEpilogue('failed', reporter.report(), 30_000, '/tmp/x.log'); + expect(lines).toContain(' Received: null'); + expect(failureExcerpt(['noise', 'one', 'two', 'three'])).toEqual(['one', 'two', 'three']); + expect(failureExcerpt(['error: x'.padEnd(300, 'y'), ...Array.from({ length: 10 }, (_, i) => 'line ' + i)])).toHaveLength(8); + }); + test('duplicate fail lines dedupe; pre-header failures are labeled unattributed', () => { const reporter = new FreeRunReporter(['test/a.test.ts']); reporter.write(`${failLine('early unattributed')}\n`, 'stderr');