From a4ea2f3173a10f05a5d02497e8776a6e8fc1d8c7 Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 17:21:10 +0000 Subject: [PATCH] fix: whole-command rerun when coverage or runner-flaky tests can fail the run A coverage threshold or a fail-on-flaky policy fails a run for a reason a narrowed rerun cannot reproduce, so either forces a whole-command rerun. --junit refuses an existing file that is not a JUnit report instead of deleting it. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- bin/gstack-flaky | 2 ++ docs/flaky.md | 10 ++++++++-- lib/flaky-run.ts | 12 ++++++++---- test/flaky-run.test.ts | 22 ++++++++++++++++++++++ 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/bin/gstack-flaky b/bin/gstack-flaky index 602a3d4d6..4933c87cc 100755 --- a/bin/gstack-flaky +++ b/bin/gstack-flaky @@ -95,6 +95,8 @@ if (sub !== 'run') die(`unknown command ${sub}\n${USAGE}`); const o = parse(rest); if (!o.junit) die('run needs --junit '); if (!/\.xml$/i.test(o.junit) || fs.statSync(o.junit, { throwIfNoEntry: false })?.isDirectory()) die(`--junit must name a .xml file (it is deleted before every attempt), got ${JSON.stringify(o.junit)}`); +const existing = fs.statSync(o.junit, { throwIfNoEntry: false }) && fs.readFileSync(o.junit, 'utf-8').slice(0, 4096); +if (existing && existing.trim() && !//]/.test(existing)) die(`--junit ${JSON.stringify(o.junit)} exists and is not a JUnit report; gstack-flaky deletes that path before every attempt, so it refuses to touch it`); if (!o.command.length) die('run needs the test command after --'); let result; try { diff --git a/docs/flaky.md b/docs/flaky.md index 33761113a..29d07b475 100644 --- a/docs/flaky.md +++ b/docs/flaky.md @@ -60,7 +60,10 @@ one argument, so a title with shell or regex characters selects only itself. `interrupted`, `Testing stopped early` or a global timeout. Otherwise the rerun repeats the whole command (`FLAKY_RERUN: scope=command ()`), so a test the first run never reached (`--bail`, `--max-failures`, `-x`) - still has to pass. When the run is already red for another test, a + still has to pass. Coverage (`--coverage`, a coverage table in the output) + and runner-flaky tests in the same run also force a whole-command rerun: a + coverage threshold or `--fail-on-flaky-tests` fails the run for a reason a + narrowed rerun cannot reproduce. When the run is already red for another test, a whole-command rerun is skipped. - **A rerun clears a test only cleanly.** The rerun's command must exit 0 and the test must run once and pass on its first attempt; a rerun the runner @@ -74,7 +77,10 @@ one argument, so a title with shell or regex characters selects only itself. rerun. - **Reports.** After the run, `--junit` holds the first run's report, so a later upload or test reporter sees what failed. Each rerun's report is kept - beside it as `.rerun-.xml`. + beside it as `.rerun-.xml`. Other reporters' outputs (Playwright's + `test-results/` and HTML report) are rewritten by a rerun; upload the JUnit + report as the record. An existing `--junit` file that is not a JUnit report + is refused (usage, exit 2), never deleted. - **Ids.** A test is `:::: title>`, with the project empty outside Playwright, so a test in two browser projects is two tests. `gstack-flaky ids --junit ` prints them. diff --git a/lib/flaky-run.ts b/lib/flaky-run.ts index e8a9d60d6..d7be4bd25 100644 --- a/lib/flaky-run.ts +++ b/lib/flaky-run.ts @@ -152,13 +152,17 @@ export function unreportedFailures(runner: JunitRunner, output: string, reported /** * Why the first run cannot be trusted to have run every test, so a rerun must * repeat the whole command; null when the runner's own output proves it - * complete. A run that stopped early reports only the tests it reached, and a - * targeted rerun would never run the rest. + * complete and its exit status explained by the failed tests alone. A run that + * stopped early reports only the tests it reached, and a targeted rerun would + * never run the rest; a coverage threshold or a fail-on-flaky policy fails the + * run for a reason a narrowed rerun cannot reproduce. */ -export function incompleteRun(runner: JunitRunner, output: string, exit: number): string | null { +export function incompleteRun(runner: JunitRunner, output: string, exit: number, command: string[] = [], runnerFlaky = 0): string | null { if (runner === 'generic') return 'a generic runner cannot prove every test ran'; if (exit !== 1) return `the command exited ${exit}, not the runner's test-failure status 1`; const text = plain(output); + if (command.some(a => a === '--coverage' || a.startsWith('--coverage=')) || /%\s*Funcs\s*\|\s*%\s*Lines|coverage threshold/i.test(text)) return 'coverage is measured, and a narrowed rerun cannot reproduce a coverage failure'; + if (runnerFlaky) return `the runner retried ${runnerFlaky} flaky test(s), which a fail-on-flaky policy can fail the run on`; if (runner === 'bun') { if (/Bailed out after/.test(text)) return 'bun stopped early (--bail)'; if (!/^\s*Ran \d+ tests? across \d+ files?/m.test(text)) return 'bun printed no run summary'; @@ -281,7 +285,7 @@ export async function runFlaky(opts: RunOptions): Promise { else if (failed.length > opts.maxReruns) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `${failed.length} tests failed, more than --max-reruns ${opts.maxReruns}; not rerun` }); else eligible.push(t); } - const wholeRerun = eligible.length ? incompleteRun(report.runner, first.output, first.exit) ?? undefined : undefined; + const wholeRerun = eligible.length ? incompleteRun(report.runner, first.output, first.exit, opts.command, runnerFlaky.length) ?? undefined : undefined; const junit = path.resolve(cwd, opts.junit); const firstXml = fs.readFileSync(junit); let reruns = 0; diff --git a/test/flaky-run.test.ts b/test/flaky-run.test.ts index 3d99344cd..ab86c852b 100644 --- a/test/flaky-run.test.ts +++ b/test/flaky-run.test.ts @@ -91,6 +91,9 @@ describe('complete-run proof for a targeted rerun', () => { expect(incompleteRun('playwright', 'Timed out waiting 600s for the test suite to run\nTimed out waiting 600s for the entire test run\n 1 failed\n', 1)).toContain('stopped early'); expect(incompleteRun('playwright', '', 1)).toContain('no summary'); expect(incompleteRun('generic', 'Ran 5 tests', 1)).toContain('generic'); + expect(incompleteRun('bun', 'All files | % Funcs | % Lines |\nRan 5 tests across 2 files.\n', 1)).toContain('coverage'); + expect(incompleteRun('bun', 'Ran 5 tests across 2 files.\n', 1, ['bun', 'test', '--coverage'])).toContain('coverage'); + expect(incompleteRun('playwright', ' 1 failed\n 1 flaky\n 4 passed\n', 1, [], 1)).toContain('fail-on-flaky'); }); }); @@ -121,6 +124,22 @@ describe('runFlaky', () => { const base = (spawn: ReturnType, dir: string, extra: Partial[0]> = {}) => runFlaky({ command: ['bun', 'test'], junit: 'r.xml', maxReruns: 3, cwd: dir, spawn: spawn.spawn('r.xml'), env: {}, now: () => new Date('2026-10-29T00:00:00Z'), ...extra }); + test('a green run with no failures is green and reruns nothing', async () => { + const dir = tmp(); + const f = fakeRun(() => suite(pass('a') + pass('b')), () => 0); + expect(await base(f, dir)).toMatchObject({ exitCode: 0, reruns: 0, ledger: [] }); + expect(f.calls).toHaveLength(1); + }); + + test('a coverage failure beside a flaky test is not cleared by a narrowed rerun', async () => { + const dir = tmp(); + const cov = () => '----|---------|---------|\nAll files | % Funcs | % Lines |\nRan 2 tests across 1 file.\n'; + const f = fakeRun((_a, n) => n === 1 ? suite(pass('a') + fail('b')) : suite(pass('a') + pass('b')), () => 1, cov); + const r = await base(f, dir); + expect(f.calls[1]).toEqual(['bun', 'test']); + expect(r).toMatchObject({ exitCode: 1, wholeRerun: 'coverage is measured, and a narrowed rerun cannot reproduce a coverage failure' }); + }); + test('a failed test that passes on rerun is a flaky pass and goes on the ledger', async () => { const dir = tmp(); const f = fakeRun((argv, n) => n === 1 ? suite(pass('a') + fail('b')) : suite(pass('b')), (_a, n) => n === 1 ? 1 : 0); @@ -362,6 +381,9 @@ describe('bin/gstack-flaky', () => { fs.writeFileSync(path.join(dir, 'keep.ts'), 'x'); expect(spawnSync(bin, ['run', '--junit', 'keep.ts', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }).status).toBe(2); expect(fs.existsSync(path.join(dir, 'keep.ts'))).toBe(true); + fs.writeFileSync(path.join(dir, 'pom.xml'), '4.0.0'); + expect(spawnSync(bin, ['run', '--junit', 'pom.xml', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }).status).toBe(2); + expect(fs.existsSync(path.join(dir, 'pom.xml'))).toBe(true); const missing = spawnSync(bin, ['run', '--junit', 'r.xml', '--', path.join(dir, 'no-such-runner')], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); expect(missing.status).toBe(3); expect(missing.stderr).toContain('(FLAKY_COMMAND_NOT_STARTED)');