mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-19 14:44:44 +01:00
fix: whole-command rerun when coverage or runner-flaky tests can fail the run
A coverage threshold or a fail-on-flaky policy fails a run for a reason a narrowed rerun cannot reproduce, so either forces a whole-command rerun. --junit refuses an existing file that is not a JUnit report instead of deleting it. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
0f8442cd1d
commit
a4ea2f3173
4 files changed
+40
-6
No files matched your search
@@ -95,6 +95,8 @@ if (sub !== 'run') die(`unknown command ${sub}\n${USAGE}`);
|
||||
const o = parse(rest);
|
||||
if (!o.junit) die('run needs --junit <file.xml>');
|
||||
if (!/\.xml$/i.test(o.junit) || fs.statSync(o.junit, { throwIfNoEntry: false })?.isDirectory()) die(`--junit must name a .xml file (it is deleted before every attempt), got ${JSON.stringify(o.junit)}`);
|
||||
const existing = fs.statSync(o.junit, { throwIfNoEntry: false }) && fs.readFileSync(o.junit, 'utf-8').slice(0, 4096);
|
||||
if (existing && existing.trim() && !/<testsuites?[\s>/]/.test(existing)) die(`--junit ${JSON.stringify(o.junit)} exists and is not a JUnit report; gstack-flaky deletes that path before every attempt, so it refuses to touch it`);
|
||||
if (!o.command.length) die('run needs the test command after --');
|
||||
let result;
|
||||
try {
|
||||
|
||||
+8
-2
@@ -60,7 +60,10 @@ one argument, so a title with shell or regex characters selects only itself.
|
||||
`interrupted`, `Testing stopped early` or a global timeout. Otherwise the
|
||||
rerun repeats the whole command (`FLAKY_RERUN: scope=command (<why>)`), so
|
||||
a test the first run never reached (`--bail`, `--max-failures`, `-x`)
|
||||
still has to pass. When the run is already red for another test, a
|
||||
still has to pass. Coverage (`--coverage`, a coverage table in the output)
|
||||
and runner-flaky tests in the same run also force a whole-command rerun: a
|
||||
coverage threshold or `--fail-on-flaky-tests` fails the run for a reason a
|
||||
narrowed rerun cannot reproduce. When the run is already red for another test, a
|
||||
whole-command rerun is skipped.
|
||||
- **A rerun clears a test only cleanly.** The rerun's command must exit 0 and
|
||||
the test must run once and pass on its first attempt; a rerun the runner
|
||||
@@ -74,7 +77,10 @@ one argument, so a title with shell or regex characters selects only itself.
|
||||
rerun.
|
||||
- **Reports.** After the run, `--junit` holds the first run's report, so a
|
||||
later upload or test reporter sees what failed. Each rerun's report is kept
|
||||
beside it as `<name>.rerun-<n>.xml`.
|
||||
beside it as `<name>.rerun-<n>.xml`. Other reporters' outputs (Playwright's
|
||||
`test-results/` and HTML report) are rewritten by a rerun; upload the JUnit
|
||||
report as the record. An existing `--junit` file that is not a JUnit report
|
||||
is refused (usage, exit 2), never deleted.
|
||||
- **Ids.** A test is `<file>::<project>::<describe > title>`, with the project
|
||||
empty outside Playwright, so a test in two browser projects is two tests.
|
||||
`gstack-flaky ids --junit <file>` prints them.
|
||||
|
||||
+8
-4
@@ -152,13 +152,17 @@ export function unreportedFailures(runner: JunitRunner, output: string, reported
|
||||
/**
|
||||
* Why the first run cannot be trusted to have run every test, so a rerun must
|
||||
* repeat the whole command; null when the runner's own output proves it
|
||||
* complete. A run that stopped early reports only the tests it reached, and a
|
||||
* targeted rerun would never run the rest.
|
||||
* complete and its exit status explained by the failed tests alone. A run that
|
||||
* stopped early reports only the tests it reached, and a targeted rerun would
|
||||
* never run the rest; a coverage threshold or a fail-on-flaky policy fails the
|
||||
* run for a reason a narrowed rerun cannot reproduce.
|
||||
*/
|
||||
export function incompleteRun(runner: JunitRunner, output: string, exit: number): string | null {
|
||||
export function incompleteRun(runner: JunitRunner, output: string, exit: number, command: string[] = [], runnerFlaky = 0): string | null {
|
||||
if (runner === 'generic') return 'a generic runner cannot prove every test ran';
|
||||
if (exit !== 1) return `the command exited ${exit}, not the runner's test-failure status 1`;
|
||||
const text = plain(output);
|
||||
if (command.some(a => a === '--coverage' || a.startsWith('--coverage=')) || /%\s*Funcs\s*\|\s*%\s*Lines|coverage threshold/i.test(text)) return 'coverage is measured, and a narrowed rerun cannot reproduce a coverage failure';
|
||||
if (runnerFlaky) return `the runner retried ${runnerFlaky} flaky test(s), which a fail-on-flaky policy can fail the run on`;
|
||||
if (runner === 'bun') {
|
||||
if (/Bailed out after/.test(text)) return 'bun stopped early (--bail)';
|
||||
if (!/^\s*Ran \d+ tests? across \d+ files?/m.test(text)) return 'bun printed no run summary';
|
||||
@@ -281,7 +285,7 @@ export async function runFlaky(opts: RunOptions): Promise<RunResult> {
|
||||
else if (failed.length > opts.maxReruns) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `${failed.length} tests failed, more than --max-reruns ${opts.maxReruns}; not rerun` });
|
||||
else eligible.push(t);
|
||||
}
|
||||
const wholeRerun = eligible.length ? incompleteRun(report.runner, first.output, first.exit) ?? undefined : undefined;
|
||||
const wholeRerun = eligible.length ? incompleteRun(report.runner, first.output, first.exit, opts.command, runnerFlaky.length) ?? undefined : undefined;
|
||||
const junit = path.resolve(cwd, opts.junit);
|
||||
const firstXml = fs.readFileSync(junit);
|
||||
let reruns = 0;
|
||||
|
||||
@@ -91,6 +91,9 @@ describe('complete-run proof for a targeted rerun', () => {
|
||||
expect(incompleteRun('playwright', 'Timed out waiting 600s for the test suite to run\nTimed out waiting 600s for the entire test run\n 1 failed\n', 1)).toContain('stopped early');
|
||||
expect(incompleteRun('playwright', '', 1)).toContain('no summary');
|
||||
expect(incompleteRun('generic', 'Ran 5 tests', 1)).toContain('generic');
|
||||
expect(incompleteRun('bun', 'All files | % Funcs | % Lines |\nRan 5 tests across 2 files.\n', 1)).toContain('coverage');
|
||||
expect(incompleteRun('bun', 'Ran 5 tests across 2 files.\n', 1, ['bun', 'test', '--coverage'])).toContain('coverage');
|
||||
expect(incompleteRun('playwright', ' 1 failed\n 1 flaky\n 4 passed\n', 1, [], 1)).toContain('fail-on-flaky');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -121,6 +124,22 @@ describe('runFlaky', () => {
|
||||
const base = (spawn: ReturnType<typeof fakeRun>, dir: string, extra: Partial<Parameters<typeof runFlaky>[0]> = {}) =>
|
||||
runFlaky({ command: ['bun', 'test'], junit: 'r.xml', maxReruns: 3, cwd: dir, spawn: spawn.spawn('r.xml'), env: {}, now: () => new Date('2026-10-29T00:00:00Z'), ...extra });
|
||||
|
||||
test('a green run with no failures is green and reruns nothing', async () => {
|
||||
const dir = tmp();
|
||||
const f = fakeRun(() => suite(pass('a') + pass('b')), () => 0);
|
||||
expect(await base(f, dir)).toMatchObject({ exitCode: 0, reruns: 0, ledger: [] });
|
||||
expect(f.calls).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('a coverage failure beside a flaky test is not cleared by a narrowed rerun', async () => {
|
||||
const dir = tmp();
|
||||
const cov = () => '----|---------|---------|\nAll files | % Funcs | % Lines |\nRan 2 tests across 1 file.\n';
|
||||
const f = fakeRun((_a, n) => n === 1 ? suite(pass('a') + fail('b')) : suite(pass('a') + pass('b')), () => 1, cov);
|
||||
const r = await base(f, dir);
|
||||
expect(f.calls[1]).toEqual(['bun', 'test']);
|
||||
expect(r).toMatchObject({ exitCode: 1, wholeRerun: 'coverage is measured, and a narrowed rerun cannot reproduce a coverage failure' });
|
||||
});
|
||||
|
||||
test('a failed test that passes on rerun is a flaky pass and goes on the ledger', async () => {
|
||||
const dir = tmp();
|
||||
const f = fakeRun((argv, n) => n === 1 ? suite(pass('a') + fail('b')) : suite(pass('b')), (_a, n) => n === 1 ? 1 : 0);
|
||||
@@ -362,6 +381,9 @@ describe('bin/gstack-flaky', () => {
|
||||
fs.writeFileSync(path.join(dir, 'keep.ts'), 'x');
|
||||
expect(spawnSync(bin, ['run', '--junit', 'keep.ts', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }).status).toBe(2);
|
||||
expect(fs.existsSync(path.join(dir, 'keep.ts'))).toBe(true);
|
||||
fs.writeFileSync(path.join(dir, 'pom.xml'), '<project><modelVersion>4.0.0</modelVersion></project>');
|
||||
expect(spawnSync(bin, ['run', '--junit', 'pom.xml', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }).status).toBe(2);
|
||||
expect(fs.existsSync(path.join(dir, 'pom.xml'))).toBe(true);
|
||||
const missing = spawnSync(bin, ['run', '--junit', 'r.xml', '--', path.join(dir, 'no-such-runner')], { cwd: dir, encoding: 'utf-8', timeout: 30_000 });
|
||||
expect(missing.status).toBe(3);
|
||||
expect(missing.stderr).toContain('(FLAKY_COMMAND_NOT_STARTED)');
|
||||
|
||||
Reference in new issue
Block a user