diff --git a/lib/flaky-run.ts b/lib/flaky-run.ts index cf79f8ec5..e8a9d60d6 100644 --- a/lib/flaky-run.ts +++ b/lib/flaky-run.ts @@ -100,20 +100,27 @@ export function rerunSelection(runner: 'playwright' | 'bun', t: JunitTest): stri const OUTPUT_TAIL = 256 * 1024; +/** Set once gstack-flaky itself is asked to stop; every attempt after that counts as killed, whatever exit the runner chose. */ +let cancelled: string | undefined; + /** Stream the child's output live (a quiet runner trips CI idle watchdogs) and keep its tail for reconciliation. */ function defaultSpawn(argv: string[], env: NodeJS.ProcessEnv, cwd: string): Promise { return new Promise((resolve, reject) => { let tail = ''; const keep = (chunk: Buffer) => { tail = (tail + chunk.toString('utf-8')).slice(-OUTPUT_TAIL); }; const child = spawnChild(argv[0]!, argv.slice(1), { stdio: ['inherit', 'pipe', 'pipe'], env, cwd }); - const forward = (s: NodeJS.Signals) => { child.kill(s); }; + const forward = (s: NodeJS.Signals) => { cancelled ??= s; child.kill(s); }; const release = () => { process.off('SIGINT', forward); process.off('SIGTERM', forward); }; process.on('SIGINT', forward); process.on('SIGTERM', forward); child.stdout!.on('data', (c: Buffer) => { process.stdout.write(c); keep(c); }); child.stderr!.on('data', (c: Buffer) => { process.stderr.write(c); keep(c); }); child.on('error', e => { release(); reject(new JunitError('FLAKY_COMMAND_NOT_STARTED', `could not start ${argv[0]}: ${e.message}`)); }); - child.on('close', (code, signal) => { release(); resolve({ exit: code ?? 128, output: tail, ...(signal ? { signal } : {}) }); }); + child.on('close', (code, signal) => { + release(); + const killed = signal ?? cancelled; + resolve({ exit: code ?? 128, output: tail, ...(killed ? { signal: killed } : {}) }); + }); }); } @@ -208,13 +215,31 @@ async function attempt(opts: RunOptions, argv: string[], cwd: string, env: NodeJ } } +/** + * What keeps a rerun from clearing anything even when it exited 0: failures + * its runner counted outside the report and, for a whole-command rerun, any + * failed test or any first-run test missing from its report. + */ +function rerunProblem(again: Attempted & { report?: JunitReport }, first: JunitReport, whole: boolean): string | undefined { + if (!again.report) return undefined; + const failedNow = again.report.tests.filter(x => finalAttempt(x) === 'fail'); + const unreported = unreportedFailures(first.runner, again.output, failedNow.length); + if (unreported) return unreported; + if (!whole) return undefined; + if (failedNow.length) return `${failedNow.length} test(s) failed in the whole-command rerun`; + const seen = new Set(again.report.tests.map(x => x.id)); + const missing = first.tests.filter(x => !seen.has(x.id)).length; + return missing ? `${missing} test(s) of the first run are missing from the whole-command rerun` : undefined; +} + /** A rerun clears a test only when its command exited 0 and the test ran once and passed on its first attempt. */ -function rerunVerdict(again: Attempted & { report?: JunitReport; error?: JunitError }, t: JunitTest): { status: 'flaky-pass' | 'rerun-fail'; attempts: string[]; reason?: string } { +function rerunVerdict(again: Attempted & { report?: JunitReport; error?: JunitError }, t: JunitTest, problem?: string): { status: 'flaky-pass' | 'rerun-fail'; attempts: string[]; reason?: string } { const match = again.report?.tests.find(x => x.id === t.id); if (!match) return { status: 'rerun-fail', attempts: [...t.attempts, 'fail'], reason: again.error ? `rerun: ${again.error.message}` : 'rerun: the report does not contain this test' }; const attempts = [...t.attempts, ...match.attempts]; if (again.signal) return { status: 'rerun-fail', attempts, reason: `rerun: killed by ${again.signal}` }; if (again.exit !== 0) return { status: 'rerun-fail', attempts, reason: `rerun: the command exited ${again.exit}` }; + if (problem) return { status: 'rerun-fail', attempts, reason: `rerun: ${problem}` }; if (match.executions !== 1 || match.attempts.join() !== 'pass') return { status: 'rerun-fail', attempts, reason: `rerun: the test did not pass once on its first attempt (${match.attempts.join(',')})` }; return { status: 'flaky-pass', attempts }; } @@ -261,8 +286,8 @@ export async function runFlaky(opts: RunOptions): Promise { const firstXml = fs.readFileSync(junit); let reruns = 0; const rerun = async (argv: string[]) => { - const again = await attempt(opts, argv, cwd, env, report.runner); reruns++; + const again = await attempt(opts, argv, cwd, env, report.runner); if (fs.existsSync(junit)) fs.renameSync(junit, `${junit.replace(/\.xml$/i, '')}.rerun-${reruns}.xml`); return again; }; @@ -275,14 +300,15 @@ export async function runFlaky(opts: RunOptions): Promise { for (const t of eligible) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `not rerun: the run is already red, and a rerun would repeat the whole command (${wholeRerun})` }); } else if (wholeRerun) { const again = await rerun(opts.command); - for (const t of eligible) record(t, rerunVerdict(again, t), 'command'); + const problem = rerunProblem(again, report, true); + for (const t of eligible) record(t, rerunVerdict(again, t, problem), 'command'); } else if (eligible.length) { let interrupted = false; for (const t of eligible) { if (interrupted) { outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: 'not rerun: an earlier rerun was killed' }); continue; } const again = await rerun([...opts.command, ...rerunSelection(report.runner as 'playwright' | 'bun', t)]); interrupted = !!again.signal; - record(t, rerunVerdict(again, t), 'test'); + record(t, rerunVerdict(again, t, rerunProblem(again, report, false)), 'test'); } } } finally { diff --git a/test/flaky-run.test.ts b/test/flaky-run.test.ts index ab71531fd..3d99344cd 100644 --- a/test/flaky-run.test.ts +++ b/test/flaky-run.test.ts @@ -7,7 +7,7 @@ import { afterAll, describe, expect, test } from 'bun:test'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; -import { spawnSync } from 'child_process'; +import { spawn, spawnSync } from 'child_process'; import { JunitError, parseJunit, readJunit, testId } from '../lib/junit'; import { appendLedger, incompleteRun, ledgerPath, renderResult, rerunSelection, runFlaky, safeExcerpt, stepSummary, unreportedFailures, workflowEscape } from '../lib/flaky-run'; @@ -260,6 +260,18 @@ describe('runFlaky', () => { expect(r).toMatchObject({ exitCode: 0, wholeRerun: 'a generic runner cannot prove every test ran' }); }); + test('a whole-command rerun clears nothing when any test fails in it, a first-run test is missing, it is killed or it retried', async () => { + const dir = tmp(); + const generic = (second: string, exit = 0, signal?: string) => base(fakeRun((_a, n) => n === 1 ? suite(fail('a') + pass('b')) : second, () => exit && 1, undefined, n => n === 2 ? signal : undefined), dir, { runner: 'generic' }); + expect((await generic(suite(pass('a') + fail('b')))).outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: 1 test(s) failed in the whole-command rerun' }); + expect((await generic(suite(pass('a')))).outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: 1 test(s) of the first run are missing from the whole-command rerun' }); + expect((await generic(suite(pass('a') + pass('b')), 0, 'SIGTERM')).outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: killed by SIGTERM' }); + const retried = await generic(suite('' + pass('b'))); + expect(retried.outcomes.find(o => o.status === 'rerun-fail')).toMatchObject({ attempts: ['fail', 'fail', 'pass'] }); + expect(retried.ledger.find(l => l.kind === 'rerun-fail')).toMatchObject({ rerun_scope: 'command' }); + expect(retried.exitCode).toBe(1); + }); + test('an incomplete run that is already red is not rerun as a whole', async () => { const dir = tmp(); const f = fakeRun(() => suite(fail('b') + pass('b') + fail('c')), () => 1, () => ''); @@ -359,6 +371,40 @@ describe('bin/gstack-flaky', () => { expect(r.stderr).toContain('fix: '); }); + test('a runner that dies by a signal exits 3 FLAKY_COMMAND_KILLED and is not rerun', () => { + const dir = tmp(); + const runner = path.join(dir, 'runner.sh'); + fs.writeFileSync(runner, ['#!/usr/bin/env bash', 'echo x >> count', `printf '%s' '${suite(fail('b'))}' > r.xml`, 'kill -TERM $$'].join('\n'), { mode: 0o755 }); + const r = spawnSync(path.join(ROOT, 'bin', 'gstack-flaky'), ['run', '--junit', 'r.xml', '--runner', 'bun', '--ledger', path.join(dir, 'l.jsonl'), '--', runner], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); + expect(r.status).toBe(3); + expect(r.stderr).toContain('(FLAKY_COMMAND_KILLED)'); + expect(fs.readFileSync(path.join(dir, 'count'), 'utf-8').trim().split('\n')).toHaveLength(1); + }); + + test('cancelling gstack-flaky is latched even when the runner traps the signal and exits by code', async () => { + const dir = tmp(); + const runner = path.join(dir, 'runner.sh'); + fs.writeFileSync(runner, [ + '#!/usr/bin/env bash', + 'echo x >> count', + 'trap "cp report.xml r.xml; exit 143" TERM', + 'touch ready', + 'for i in $(seq 1 300); do sleep 0.1; done', + ].join('\n'), { mode: 0o755 }); + fs.writeFileSync(path.join(dir, 'report.xml'), suite(fail('b'))); + const child = spawn(path.join(ROOT, 'bin', 'gstack-flaky'), ['run', '--junit', 'r.xml', '--runner', 'generic', '--ledger', path.join(dir, 'l.jsonl'), '--', runner], { cwd: dir, stdio: ['ignore', 'ignore', 'pipe'] }); + let stderr = ''; + child.stderr.on('data', (c: Buffer) => { stderr += c.toString(); }); + const done = new Promise(resolve => child.on('close', code => resolve(code))); + const deadline = Date.now() + 20_000; + while (!fs.existsSync(path.join(dir, 'ready')) && Date.now() < deadline) await Bun.sleep(50); + child.kill('SIGTERM'); + expect(await done).toBe(3); + expect(stderr).toContain('(FLAKY_COMMAND_KILLED)'); + expect(fs.readFileSync(path.join(dir, 'count'), 'utf-8').trim().split('\n')).toHaveLength(1); + expect(fs.existsSync(path.join(dir, 'l.jsonl'))).toBe(false); + }, 30_000); + test('every FLAKY_ code the bin and the library emit is in the docs table', () => { const doc = fs.readFileSync(path.join(ROOT, 'docs', 'flaky.md'), 'utf-8'); const src = fs.readFileSync(path.join(ROOT, 'bin', 'gstack-flaky'), 'utf-8') + fs.readFileSync(path.join(ROOT, 'lib', 'flaky-run.ts'), 'utf-8') + fs.readFileSync(path.join(ROOT, 'lib', 'junit.ts'), 'utf-8');