fix: whole-command rerun when coverage or runner-flaky tests can fail the run

A coverage threshold or a fail-on-flaky policy fails a run for a reason a
narrowed rerun cannot reproduce, so either forces a whole-command rerun.
--junit refuses an existing file that is not a JUnit report instead of
deleting it.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-29 17:21:10 +00:00
1 parent 0f8442cd1d
commit a4ea2f3173
4 files changed
+40 -6

No files matched your search

+2
View File
@@ -95,6 +95,8 @@ if (sub !== 'run') die(`unknown command ${sub}\n${USAGE}`);
const o = parse(rest);
if (!o.junit) die('run needs --junit <file.xml>');
if (!/\.xml$/i.test(o.junit) || fs.statSync(o.junit, { throwIfNoEntry: false })?.isDirectory()) die(`--junit must name a .xml file (it is deleted before every attempt), got ${JSON.stringify(o.junit)}`);
const existing = fs.statSync(o.junit, { throwIfNoEntry: false }) && fs.readFileSync(o.junit, 'utf-8').slice(0, 4096);
if (existing && existing.trim() && !/<testsuites?[\s>/]/.test(existing)) die(`--junit ${JSON.stringify(o.junit)} exists and is not a JUnit report; gstack-flaky deletes that path before every attempt, so it refuses to touch it`);
if (!o.command.length) die('run needs the test command after --');
let result;
try {
+8 -2
View File
@@ -60,7 +60,10 @@ one argument, so a title with shell or regex characters selects only itself.
`interrupted`, `Testing stopped early` or a global timeout. Otherwise the
rerun repeats the whole command (`FLAKY_RERUN: scope=command (<why>)`), so
a test the first run never reached (`--bail`, `--max-failures`, `-x`)
still has to pass. When the run is already red for another test, a
still has to pass. Coverage (`--coverage`, a coverage table in the output)
and runner-flaky tests in the same run also force a whole-command rerun: a
coverage threshold or `--fail-on-flaky-tests` fails the run for a reason a
narrowed rerun cannot reproduce. When the run is already red for another test, a
whole-command rerun is skipped.
- **A rerun clears a test only cleanly.** The rerun's command must exit 0 and
the test must run once and pass on its first attempt; a rerun the runner
@@ -74,7 +77,10 @@ one argument, so a title with shell or regex characters selects only itself.
rerun.
- **Reports.** After the run, `--junit` holds the first run's report, so a
later upload or test reporter sees what failed. Each rerun's report is kept
beside it as `<name>.rerun-<n>.xml`.
beside it as `<name>.rerun-<n>.xml`. Other reporters' outputs (Playwright's
`test-results/` and HTML report) are rewritten by a rerun; upload the JUnit
report as the record. An existing `--junit` file that is not a JUnit report
is refused (usage, exit 2), never deleted.
- **Ids.** A test is `<file>::<project>::<describe > title>`, with the project
empty outside Playwright, so a test in two browser projects is two tests.
`gstack-flaky ids --junit <file>` prints them.
+8 -4
View File
@@ -152,13 +152,17 @@ export function unreportedFailures(runner: JunitRunner, output: string, reported
/**
* Why the first run cannot be trusted to have run every test, so a rerun must
* repeat the whole command; null when the runner's own output proves it
* complete. A run that stopped early reports only the tests it reached, and a
* targeted rerun would never run the rest.
* complete and its exit status explained by the failed tests alone. A run that
* stopped early reports only the tests it reached, and a targeted rerun would
* never run the rest; a coverage threshold or a fail-on-flaky policy fails the
* run for a reason a narrowed rerun cannot reproduce.
*/
export function incompleteRun(runner: JunitRunner, output: string, exit: number): string | null {
export function incompleteRun(runner: JunitRunner, output: string, exit: number, command: string[] = [], runnerFlaky = 0): string | null {
if (runner === 'generic') return 'a generic runner cannot prove every test ran';
if (exit !== 1) return `the command exited ${exit}, not the runner's test-failure status 1`;
const text = plain(output);
if (command.some(a => a === '--coverage' || a.startsWith('--coverage=')) || /%\s*Funcs\s*\|\s*%\s*Lines|coverage threshold/i.test(text)) return 'coverage is measured, and a narrowed rerun cannot reproduce a coverage failure';
if (runnerFlaky) return `the runner retried ${runnerFlaky} flaky test(s), which a fail-on-flaky policy can fail the run on`;
if (runner === 'bun') {
if (/Bailed out after/.test(text)) return 'bun stopped early (--bail)';
if (!/^\s*Ran \d+ tests? across \d+ files?/m.test(text)) return 'bun printed no run summary';
@@ -281,7 +285,7 @@ export async function runFlaky(opts: RunOptions): Promise<RunResult> {
else if (failed.length > opts.maxReruns) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `${failed.length} tests failed, more than --max-reruns ${opts.maxReruns}; not rerun` });
else eligible.push(t);
}
const wholeRerun = eligible.length ? incompleteRun(report.runner, first.output, first.exit) ?? undefined : undefined;
const wholeRerun = eligible.length ? incompleteRun(report.runner, first.output, first.exit, opts.command, runnerFlaky.length) ?? undefined : undefined;
const junit = path.resolve(cwd, opts.junit);
const firstXml = fs.readFileSync(junit);
let reruns = 0;
+22
View File
@@ -91,6 +91,9 @@ describe('complete-run proof for a targeted rerun', () => {
expect(incompleteRun('playwright', 'Timed out waiting 600s for the test suite to run\nTimed out waiting 600s for the entire test run\n 1 failed\n', 1)).toContain('stopped early');
expect(incompleteRun('playwright', '', 1)).toContain('no summary');
expect(incompleteRun('generic', 'Ran 5 tests', 1)).toContain('generic');
expect(incompleteRun('bun', 'All files | % Funcs | % Lines |\nRan 5 tests across 2 files.\n', 1)).toContain('coverage');
expect(incompleteRun('bun', 'Ran 5 tests across 2 files.\n', 1, ['bun', 'test', '--coverage'])).toContain('coverage');
expect(incompleteRun('playwright', ' 1 failed\n 1 flaky\n 4 passed\n', 1, [], 1)).toContain('fail-on-flaky');
});
});
@@ -121,6 +124,22 @@ describe('runFlaky', () => {
const base = (spawn: ReturnType<typeof fakeRun>, dir: string, extra: Partial<Parameters<typeof runFlaky>[0]> = {}) =>
runFlaky({ command: ['bun', 'test'], junit: 'r.xml', maxReruns: 3, cwd: dir, spawn: spawn.spawn('r.xml'), env: {}, now: () => new Date('2026-10-29T00:00:00Z'), ...extra });
test('a green run with no failures is green and reruns nothing', async () => {
const dir = tmp();
const f = fakeRun(() => suite(pass('a') + pass('b')), () => 0);
expect(await base(f, dir)).toMatchObject({ exitCode: 0, reruns: 0, ledger: [] });
expect(f.calls).toHaveLength(1);
});
test('a coverage failure beside a flaky test is not cleared by a narrowed rerun', async () => {
const dir = tmp();
const cov = () => '----|---------|---------|\nAll files | % Funcs | % Lines |\nRan 2 tests across 1 file.\n';
const f = fakeRun((_a, n) => n === 1 ? suite(pass('a') + fail('b')) : suite(pass('a') + pass('b')), () => 1, cov);
const r = await base(f, dir);
expect(f.calls[1]).toEqual(['bun', 'test']);
expect(r).toMatchObject({ exitCode: 1, wholeRerun: 'coverage is measured, and a narrowed rerun cannot reproduce a coverage failure' });
});
test('a failed test that passes on rerun is a flaky pass and goes on the ledger', async () => {
const dir = tmp();
const f = fakeRun((argv, n) => n === 1 ? suite(pass('a') + fail('b')) : suite(pass('b')), (_a, n) => n === 1 ? 1 : 0);
@@ -362,6 +381,9 @@ describe('bin/gstack-flaky', () => {
fs.writeFileSync(path.join(dir, 'keep.ts'), 'x');
expect(spawnSync(bin, ['run', '--junit', 'keep.ts', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }).status).toBe(2);
expect(fs.existsSync(path.join(dir, 'keep.ts'))).toBe(true);
fs.writeFileSync(path.join(dir, 'pom.xml'), '<project><modelVersion>4.0.0</modelVersion></project>');
expect(spawnSync(bin, ['run', '--junit', 'pom.xml', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }).status).toBe(2);
expect(fs.existsSync(path.join(dir, 'pom.xml'))).toBe(true);
const missing = spawnSync(bin, ['run', '--junit', 'r.xml', '--', path.join(dir, 'no-such-runner')], { cwd: dir, encoding: 'utf-8', timeout: 30_000 });
expect(missing.status).toBe(3);
expect(missing.stderr).toContain('(FLAKY_COMMAND_NOT_STARTED)');