From 12d596714c925fd81570884cd05b8bc9af3af7fa Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 17:30:16 +0000 Subject: [PATCH] feat(flaky): rerun the whole command by default; targeted rerun is opt-in A rerun counts only on the runner's own full green verdict: exit 0, no failed or missing test, nothing counted outside the report. That covers every way a run can fail, which a narrowed rerun cannot (a coverage threshold with an lcov reporter, a fail-on-flaky policy, a test never reached). --rerun-scope test reruns the failed test alone, with its preconditions documented and the complete-run checks kept as a safety net. An unreadable existing --junit file is a usage error, not a crash. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- bin/gstack-flaky | 31 +++++++++++------ docs/TESTING_INTERNALS.md | 3 +- docs/flaky.md | 73 ++++++++++++++++++++++++--------------- lib/flaky-run.ts | 26 +++++++++----- test/flaky-run.test.ts | 35 ++++++++++++++----- 5 files changed, 110 insertions(+), 58 deletions(-) diff --git a/bin/gstack-flaky b/bin/gstack-flaky index 4933c87cc..40239d156 100755 --- a/bin/gstack-flaky +++ b/bin/gstack-flaky @@ -1,6 +1,6 @@ #!/usr/bin/env -S bun --no-env-file --config=/dev/null /** - * gstack-flaky (GSTA-158): rerun a failed test once and record the flake, for + * gstack-flaky (GSTA-158): rerun failed tests once and record the flake, for * any runner that writes JUnit (Playwright, bun, jest, pytest, vitest). CREW * rule 16 for every repository: "you can just run the failed test again and * note that it is flaky". The rules live in lib/flaky-run.ts; docs/flaky.md is @@ -12,18 +12,20 @@ import { JunitError, readJunit, type JunitRunner } from '../lib/junit'; const USAGE = `Usage: gstack-flaky run --junit [--runner playwright|bun|generic] [--max-reruns ] - [--ledger ] -- + [--rerun-scope command|test] [--ledger ] -- gstack-flaky ids --junit [--runner playwright|bun|generic] gstack-flaky --help run: runs the command (an argv, no shell), reads the JUnit report it writes at --junit, and reruns each failed test once when the runner made no retries and -at most --max-reruns (${DEFAULT_MAX_RERUNS}) tests failed. When the runner's summary proves -the first run complete, a rerun appends the runner's selection to the same -command (playwright: --grep --project <p>; bun: <file> ---test-name-pattern <title>); otherwise, and always for generic runners, the -rerun is the whole command. A test that passes its rerun once, with the rerun -exiting 0, is a flaky pass: green, and one line on the ledger (--ledger, else +at most --max-reruns (${DEFAULT_MAX_RERUNS}) tests failed. The rerun repeats the whole +command and counts only on the runner's full green verdict (exit 0, no failed +or missing test). --rerun-scope test reruns the failed test alone (playwright: +<file> --grep <title> --project <p>; bun: <file> --test-name-pattern <title>) +when the runner's summary proves the first run complete and nothing but the +failed tests can fail it (no coverage, no runner-flaky test); otherwise, and +always for generic runners, the whole command reruns. A test that passes its +rerun once, with the rerun exiting 0, is a flaky pass: green, and one line on the ledger (--ledger, else GSTACK_FLAKE_LEDGER_JUNIT, else <state root>/flake-ledger-junit.jsonl). --junit holds the first run's report afterwards; reruns' reports sit beside it as <name>.rerun-<n>.xml. The command runs with PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1 @@ -59,7 +61,7 @@ function coded(e: JunitError): never { } function parse(argv: string[]) { - const o: { junit?: string; runner?: JunitRunner; maxReruns: number; ledger?: string; command: string[] } = { maxReruns: DEFAULT_MAX_RERUNS, command: [] }; + const o: { junit?: string; runner?: JunitRunner; maxReruns: number; rerunScope: 'command' | 'test'; ledger?: string; command: string[] } = { maxReruns: DEFAULT_MAX_RERUNS, rerunScope: 'command', command: [] }; for (let i = 0; i < argv.length; i++) { const a = argv[i]!; const val = () => { const v = argv[++i]; if (v === undefined) die(`${a} needs a value`); return v; }; @@ -73,6 +75,10 @@ function parse(argv: string[]) { const n = Number(val()); if (!Number.isInteger(n) || n < 0) die('--max-reruns must be a whole number'); o.maxReruns = n; + } else if (a === '--rerun-scope') { + const v = val(); + if (v !== 'command' && v !== 'test') die(`--rerun-scope must be command or test, got ${JSON.stringify(v)}`); + o.rerunScope = v; } else if (a === '--ledger') o.ledger = val(); else die(`unknown option ${a}\n${USAGE}`); } @@ -95,12 +101,15 @@ if (sub !== 'run') die(`unknown command ${sub}\n${USAGE}`); const o = parse(rest); if (!o.junit) die('run needs --junit <file.xml>'); if (!/\.xml$/i.test(o.junit) || fs.statSync(o.junit, { throwIfNoEntry: false })?.isDirectory()) die(`--junit must name a .xml file (it is deleted before every attempt), got ${JSON.stringify(o.junit)}`); -const existing = fs.statSync(o.junit, { throwIfNoEntry: false }) && fs.readFileSync(o.junit, 'utf-8').slice(0, 4096); +let existing: string | false | undefined; +try { existing = fs.statSync(o.junit, { throwIfNoEntry: false }) && fs.readFileSync(o.junit, 'utf-8').slice(0, 4096); } catch (e) { + die(`--junit ${JSON.stringify(o.junit)} exists but cannot be read (${(e as Error).message}); gstack-flaky deletes that path before every attempt, so it refuses to touch it`); +} if (existing && existing.trim() && !/<testsuites?[\s>/]/.test(existing)) die(`--junit ${JSON.stringify(o.junit)} exists and is not a JUnit report; gstack-flaky deletes that path before every attempt, so it refuses to touch it`); if (!o.command.length) die('run needs the test command after --'); let result; try { - result = await runFlaky({ command: o.command, junit: o.junit, runner: o.runner, maxReruns: o.maxReruns }); + result = await runFlaky({ command: o.command, junit: o.junit, runner: o.runner, maxReruns: o.maxReruns, rerunScope: o.rerunScope }); } catch (e) { if (e instanceof JunitError) coded(e); throw e; } const ledger = ledgerPath(o.ledger); appendLedger(result.ledger, ledger, s => process.stderr.write(`gstack-flaky: ${s}\n`)); diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index a5d29ddb0..d5d5356d1 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -1411,7 +1411,8 @@ never masks it. `bun run eval:pass-rates` lists only the local ledger Other repositories get the same rule per test with `bin/gstack-flaky run` ([docs/flaky.md](flaky.md)): it reads a JUnit report from any runner, reruns -each failed test once, and writes `flaky-pass` and `rerun-fail` lines to its +the command once when tests failed (counting only the runner's full green +verdict), and writes `flaky-pass` and `rerun-fail` lines to its own ledger (`flake-ledger-junit.jsonl`, or `GSTACK_FLAKE_LEDGER_JUNIT`), kept apart from this file-level ledger. diff --git a/docs/flaky.md b/docs/flaky.md index 29d07b475..1a56b798e 100644 --- a/docs/flaky.md +++ b/docs/flaky.md @@ -1,8 +1,8 @@ # Flaky tests: rerun once and record it (`gstack-flaky`) `gstack-flaky run` runs your test command, reads the JUnit report it writes, -and reruns each failed test once. A test that passes on its rerun is a -**flaky pass**: the job stays green and the flake is recorded on a ledger, so +and when tests failed, runs the command once more. A test that passes on its +rerun is a **flaky pass**: the job stays green and the flake is recorded on a ledger, so the daily flaky pass can fix it at the source. A test that fails twice is red. This is CREW rule 16 ("run the failed test again and note that it is flaky") for any runner that writes JUnit: Playwright, `bun test`, jest, vitest and @@ -37,50 +37,67 @@ the run before anyone reads the ledger. ## Runners -| Runner | Command after `--` | Rerun | +| Runner | Command after `--` | Notes | |---|---|---| -| Playwright | `npx playwright test --reporter=line,junit` with `PLAYWRIGHT_JUNIT_OUTPUT_NAME=<file>` | The failed test alone (`<file> --grep <title> --project <project>`) when the summary proves the run complete, else the whole command. `gstack-flaky` sets `PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1` so Playwright's own `retries` are reported; with retries configured, those attempts stand and nothing is rerun again. | -| bun | `bun test --reporter=junit --reporter-outfile=<file>` | The failed test alone (`<file> --test-name-pattern <title>`) when the summary proves the run complete, else the whole command. | -| jest, vitest, pytest (`--runner generic`) | your command, writing JUnit | Always the whole command: a generic runner's output cannot prove that every test ran. | +| Playwright | `npx playwright test --reporter=line,junit` with `PLAYWRIGHT_JUNIT_OUTPUT_NAME=<file>` | `gstack-flaky` sets `PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1` so Playwright's own `retries` are reported; with retries configured, those attempts stand and nothing is rerun again. | +| bun | `bun test --reporter=junit --reporter-outfile=<file>` | | +| jest, vitest, pytest (`--runner generic`) | your command, writing JUnit | Always reruns the whole command, even with `--rerun-scope test`. | The command after `--` runs without a shell. When it needs one, wrap it as `bash -c '<script> "$@"' _` so a targeted rerun's selection arguments reach -the runner. A targeted rerun appends its selection to the same command, each -one argument, so a title with shell or regex characters selects only itself. +the runner. + +## Rerun scope + +**Default: the whole command.** The rerun repeats your command exactly, and +counts only on the runner's own full green verdict: it exits 0, no test fails, +every test of the first run is in its report, and the runner counts no failure +outside the report. That verdict covers every reason a run can fail, including +the ones a narrowed rerun cannot see: a test the first run never reached +(`--bail`, `--max-failures`, `-x`), a coverage threshold (however coverage is +configured, `bunfig.toml` and lcov included), a fail-on-flaky policy, a +global setup error. A test that failed in the first run and passes once in this +rerun is a flaky pass. It costs one more run of the suite, which is still far +cheaper than rerunning the CI job by hand. + +**Opt-in: `--rerun-scope test`.** Reruns the failed test alone (Playwright: +`<file> --grep <title> --project <project>`; bun: `<file> --test-name-pattern <title>`) +and clears it when that rerun exits 0. Use it only when the first run's exit +status can come from failed tests alone: no coverage threshold in any form, no +`--fail-on-flaky-tests`, no other step inside the command that can fail. As a +safety net, `gstack-flaky` still reruns the whole command +(`FLAKY_RERUN: scope=command (<why>)`) when the runner exited anything but +1, its summary is missing, it stopped early (bun `Bailed out`, Playwright +`did not run`, `interrupted`, `Testing stopped early`, a global timeout), it +printed a coverage table or got `--coverage`, or it retried flaky tests itself. +It cannot see a coverage threshold configured without console output (for +example `coverageReporter = ["lcov"]` in `bunfig.toml`), which is why this +scope is opt-in. ## Rules - **One rerun, small failures only.** A failed test gets one rerun when the runner made no retries and at most `--max-reruns` (3) tests failed. More - failures than that is a real red, and nothing is rerun. -- **A targeted rerun needs a complete first run.** The failed test is rerun - alone only when the runner exited 1 (its test-failure status) and its own - summary is present with no early stop: bun's `Ran N tests` line without - `Bailed out`, Playwright's `N failed` line without `did not run`, - `interrupted`, `Testing stopped early` or a global timeout. Otherwise the - rerun repeats the whole command (`FLAKY_RERUN: scope=command (<why>)`), so - a test the first run never reached (`--bail`, `--max-failures`, `-x`) - still has to pass. Coverage (`--coverage`, a coverage table in the output) - and runner-flaky tests in the same run also force a whole-command rerun: a - coverage threshold or `--fail-on-flaky-tests` fails the run for a reason a - narrowed rerun cannot reproduce. When the run is already red for another test, a - whole-command rerun is skipped. + failures than that is a real red, and nothing is rerun. When the run is + already red for a test that cannot be rerun, a whole-command rerun is + skipped. - **A rerun clears a test only cleanly.** The rerun's command must exit 0 and the test must run once and pass on its first attempt; a rerun the runner retried again does not count. - **Never green by accident.** The report is deleted before every attempt, and a report that attempt did not write counts as missing. A missing, empty, zero-test or malformed report fails, as does a non-zero exit with no failed - test (a crash or a non-test step) and a runner killed by a signal. When the - runner's summary counts failures its JUnit omits (a bun file that fails to - load, a Playwright error outside any test), the run is red and nothing is - rerun. + test (a crash or a non-test step) and a runner killed by a signal. Cancelling + `gstack-flaky` (SIGINT, SIGTERM) is passed to the runner and latched: nothing + is rerun after it, whatever exit the runner chooses. When the runner's + summary counts failures its JUnit omits (a bun file that fails to load, a + Playwright error outside any test), the run is red and nothing is rerun. - **Reports.** After the run, `--junit` holds the first run's report, so a later upload or test reporter sees what failed. Each rerun's report is kept beside it as `<name>.rerun-<n>.xml`. Other reporters' outputs (Playwright's `test-results/` and HTML report) are rewritten by a rerun; upload the JUnit - report as the record. An existing `--junit` file that is not a JUnit report - is refused (usage, exit 2), never deleted. + report as the record. An existing `--junit` file that is not a JUnit report, + or cannot be read, is refused (usage, exit 2), never deleted. - **Ids.** A test is `<file>::<project>::<describe > title>`, with the project empty outside Playwright, so a test in two browser projects is two tests. `gstack-flaky ids --junit <file>` prints them. @@ -105,7 +122,7 @@ rerun repeated the whole command, then |---|---| | 0 | Every test passed, flaky passes included | | 1 | A test failed (after its rerun, or with no rerun), including `FLAKY_UNREPORTED_FAILURES` | -| 2 | Usage error, including a `--junit` path that is not a `.xml` file | +| 2 | Usage error, including a `--junit` path that is not a `.xml` file, or an existing file there that is not a readable JUnit report | | 3 | No usable report, a crash, or a killed runner (a code below on stderr) | | Code | Exit | Cause | Fix | diff --git a/lib/flaky-run.ts b/lib/flaky-run.ts index d7be4bd25..4c2f0d65e 100644 --- a/lib/flaky-run.ts +++ b/lib/flaky-run.ts @@ -1,6 +1,6 @@ /** - * gstack-flaky run (GSTA-158): run a test command, read its JUnit, rerun each - * failed test once, and record flaky passes on a ledger. This is CREW rule 16 + * gstack-flaky run (GSTA-158): run a test command, read its JUnit, rerun once + * when tests failed, and record flaky passes on a ledger. This is CREW rule 16 * ("run the failed test again and note that it is flaky") for any runner that * writes JUnit, the per-test form of the free suite's GSTACK_FREE_RETRY_FLAKY. * @@ -9,11 +9,15 @@ * at most maxReruns tests failed; more failures are a real red and nothing * is rerun. When the runner retried (Playwright `retries` with retry-aware * JUnit), its attempts stand and nothing is rerun again. - * - A targeted rerun (the failed test alone) needs proof that the first run - * was complete: the runner's own summary, its test-failure exit status 1 and - * no early stop (bail, max-failures, interrupt, global timeout). Without that - * proof, and always for generic runners, the rerun is the whole command, so a - * test the first run never reached still has to pass. + * - The rerun repeats the whole command by default, and counts only on the + * runner's own full green verdict: exit 0, no failed test, every first-run + * test present, nothing counted outside the report. Only that verdict covers + * every reason a run can fail (a test it never reached, a coverage + * threshold, a fail-on-flaky policy). + * - rerunScope 'test' (opt-in) reruns the failed test alone, and only with + * proof that the first run was complete: the runner's own summary, its + * test-failure exit status 1, no early stop, no coverage and no runner-flaky + * test; otherwise, and always for generic runners, the whole command reruns. * - A rerun makes a flaky pass only when its command exited 0 and the test ran * exactly once and passed on its first attempt. The command is an argv run * without a shell; a targeted rerun appends the adapter's selection, each one @@ -58,6 +62,8 @@ export interface RunOptions { junit: string; runner?: JunitRunner; maxReruns: number; + /** 'command' (default): rerun the whole command. 'test': rerun the failed test alone when the first run is proven complete. */ + rerunScope?: 'command' | 'test'; cwd?: string; env?: NodeJS.ProcessEnv; /** Runs one attempt; resolves with its exit status and the tail of its combined output. */ @@ -82,7 +88,7 @@ export interface RunResult { commandExit: number; outcomes: TestOutcome[]; reruns: number; - /** Why the rerun was the whole command; absent for targeted reruns. */ + /** Why the rerun was the whole command (the default, or why a requested targeted rerun was refused); absent for targeted reruns. */ wholeRerun?: string; ledger: LedgerLine[]; } @@ -285,7 +291,9 @@ export async function runFlaky(opts: RunOptions): Promise<RunResult> { else if (failed.length > opts.maxReruns) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `${failed.length} tests failed, more than --max-reruns ${opts.maxReruns}; not rerun` }); else eligible.push(t); } - const wholeRerun = eligible.length ? incompleteRun(report.runner, first.output, first.exit, opts.command, runnerFlaky.length) ?? undefined : undefined; + const wholeRerun = !eligible.length ? undefined + : opts.rerunScope !== 'test' ? 'the default; --rerun-scope test reruns the failed test alone' + : incompleteRun(report.runner, first.output, first.exit, opts.command, runnerFlaky.length) ?? undefined; const junit = path.resolve(cwd, opts.junit); const firstXml = fs.readFileSync(junit); let reruns = 0; diff --git a/test/flaky-run.test.ts b/test/flaky-run.test.ts index ab86c852b..c890b5883 100644 --- a/test/flaky-run.test.ts +++ b/test/flaky-run.test.ts @@ -135,15 +135,26 @@ describe('runFlaky', () => { const dir = tmp(); const cov = () => '----|---------|---------|\nAll files | % Funcs | % Lines |\nRan 2 tests across 1 file.\n'; const f = fakeRun((_a, n) => n === 1 ? suite(pass('a') + fail('b')) : suite(pass('a') + pass('b')), () => 1, cov); - const r = await base(f, dir); + const r = await base(f, dir, { rerunScope: 'test' }); expect(f.calls[1]).toEqual(['bun', 'test']); expect(r).toMatchObject({ exitCode: 1, wholeRerun: 'coverage is measured, and a narrowed rerun cannot reproduce a coverage failure' }); }); + test('by default the rerun is the whole command and counts only on its full green verdict', async () => { + const dir = tmp(); + const green = fakeRun((_a, n) => n === 1 ? suite(pass('a') + fail('b')) : suite(pass('a') + pass('b')), (_a, n) => n === 1 ? 1 : 0); + const r = await base(green, dir); + expect(green.calls).toEqual([['bun', 'test'], ['bun', 'test']]); + expect(r).toMatchObject({ exitCode: 0, wholeRerun: 'the default; --rerun-scope test reruns the failed test alone' }); + expect(r.ledger[0]).toMatchObject({ kind: 'flaky-pass', rerun_scope: 'command' }); + const otherFails = fakeRun((_a, n) => n === 1 ? suite(pass('a') + fail('b')) : suite(fail('a') + pass('b')), (_a, n) => n === 1 ? 1 : 0); + expect((await base(otherFails, dir)).outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: 1 test(s) failed in the whole-command rerun' }); + }); + test('a failed test that passes on rerun is a flaky pass and goes on the ledger', async () => { const dir = tmp(); const f = fakeRun((argv, n) => n === 1 ? suite(pass('a') + fail('b')) : suite(pass('b')), (_a, n) => n === 1 ? 1 : 0); - const r = await base(f, dir); + const r = await base(f, dir, { rerunScope: 'test' }); expect(r.exitCode).toBe(0); expect(f.calls[1]).toEqual(['bun', 'test', 'a.test.ts', '--test-name-pattern', '^b$']); expect(r.outcomes.map(o => [o.status, o.attempts.join(',')])).toEqual([['flaky-pass', 'fail,pass']]); @@ -218,7 +229,7 @@ describe('runFlaky', () => { const r = parseJunit(suite(pass('b') + fail('b'))); expect(r.tests[0]).toMatchObject({ executions: 2, attempts: ['pass', 'fail'] }); const f = fakeRun(() => suite(fail('b') + pass('b') + fail('c')), () => 1); - const out = await base(f, dir); + const out = await base(f, dir, { rerunScope: 'test' }); expect(out.outcomes.find(o => o.test.id.endsWith('::b'))).toMatchObject({ status: 'fail', reason: 'it ran 2 times in this report and one failed; not rerun' }); expect(f.calls).toHaveLength(2); }); @@ -262,19 +273,19 @@ describe('runFlaky', () => { const dir = tmp(); const bail = () => 'Bailed out after 1 failure\nRan 1 test across 1 file.\n'; const green = fakeRun((_a, n) => n === 1 ? suite(fail('b')) : suite(pass('b') + pass('never-reached')), (_a, n) => n === 1 ? 1 : 0, bail); - const r = await base(green, dir); + const r = await base(green, dir, { rerunScope: 'test' }); expect(green.calls).toEqual([['bun', 'test'], ['bun', 'test']]); expect(r).toMatchObject({ exitCode: 0, wholeRerun: 'bun stopped early (--bail)' }); expect(r.ledger[0]).toMatchObject({ kind: 'flaky-pass', rerun_scope: 'command' }); expect(renderResult(r, '/l')).toContain('FLAKY_RERUN: scope=command (bun stopped early (--bail))'); const red = fakeRun((_a, n) => n === 1 ? suite(fail('b')) : suite(pass('b') + fail('never-reached')), () => 1, bail); - expect((await base(red, dir)).outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: the command exited 1' }); + expect((await base(red, dir, { rerunScope: 'test' })).outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: the command exited 1' }); }); - test('generic runners always rerun the whole command', async () => { + test('generic runners rerun the whole command even when a targeted rerun is asked for', async () => { const dir = tmp(); const f = fakeRun((_a, n) => n === 1 ? suite(fail('b')) : suite(pass('b')), (_a, n) => n === 1 ? 1 : 0); - const r = await base(f, dir, { runner: 'generic' }); + const r = await base(f, dir, { runner: 'generic', rerunScope: 'test' }); expect(f.calls[1]).toEqual(['bun', 'test']); expect(r).toMatchObject({ exitCode: 0, wholeRerun: 'a generic runner cannot prove every test ran' }); }); @@ -309,7 +320,7 @@ describe('runFlaky', () => { const dir = tmp(); expect(await base(fakeRun(() => suite(fail('b')), () => 128, undefined, () => 'SIGTERM'), dir)).toMatchObject({ exitCode: 3, code: 'FLAKY_COMMAND_KILLED' }); const f = fakeRun((_a, n) => n === 1 ? suite(fail('b') + fail('c')) : suite(pass('b') + pass('c')), (_a, n) => n === 1 ? 1 : 128, undefined, n => n === 2 ? 'SIGTERM' : undefined); - const r = await base(f, dir); + const r = await base(f, dir, { rerunScope: 'test' }); expect(f.calls).toHaveLength(2); expect(r.outcomes.map(o => o.reason)).toEqual(['rerun: killed by SIGTERM', 'not rerun: an earlier rerun was killed']); expect(r.exitCode).toBe(1); @@ -367,7 +378,7 @@ describe('bin/gstack-flaky', () => { `printf '%s' '${suite(pass('b'))}' > r.xml`, ].join('\n'), { mode: 0o755 }); const ledger = path.join(dir, 'ledger.jsonl'); - const r = spawnSync(path.join(ROOT, 'bin', 'gstack-flaky'), ['run', '--junit', 'r.xml', '--runner', 'bun', '--ledger', ledger, '--', runner], { cwd: dir, encoding: 'utf-8', timeout: 30_000, env: { ...process.env, GITHUB_ACTIONS: '', GITHUB_STEP_SUMMARY: '' } }); + const r = spawnSync(path.join(ROOT, 'bin', 'gstack-flaky'), ['run', '--junit', 'r.xml', '--runner', 'bun', '--rerun-scope', 'test', '--ledger', ledger, '--', runner], { cwd: dir, encoding: 'utf-8', timeout: 30_000, env: { ...process.env, GITHUB_ACTIONS: '', GITHUB_STEP_SUMMARY: '' } }); expect(r.stdout).toContain('FLAKY: flaky-pass a.test.ts::::b attempts=fail,pass'); expect(r.stdout).toContain(`verdict=pass ledger=${ledger}`); expect(r.status).toBe(0); @@ -384,6 +395,12 @@ describe('bin/gstack-flaky', () => { fs.writeFileSync(path.join(dir, 'pom.xml'), '<project><modelVersion>4.0.0</modelVersion></project>'); expect(spawnSync(bin, ['run', '--junit', 'pom.xml', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }).status).toBe(2); expect(fs.existsSync(path.join(dir, 'pom.xml'))).toBe(true); + expect(spawnSync(bin, ['run', '--junit', 'r.xml', '--rerun-scope', 'file', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }).status).toBe(2); + fs.writeFileSync(path.join(dir, 'locked.xml'), '<testsuites/>', { mode: 0o000 }); + if (process.getuid?.() !== 0) { + const locked = spawnSync(bin, ['run', '--junit', 'locked.xml', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); + expect([locked.status, locked.stderr.includes('cannot be read')]).toEqual([2, true]); + } const missing = spawnSync(bin, ['run', '--junit', 'r.xml', '--', path.join(dir, 'no-such-runner')], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); expect(missing.status).toBe(3); expect(missing.stderr).toContain('(FLAKY_COMMAND_NOT_STARTED)');