fix: latch cancellation and check whole-command rerun reports

A cancel forwarded to a runner that traps the signal now counts as killed for
every later attempt, so a cancelled job is never rerun or greened. A
whole-command rerun clears nothing when its report has a failed test, misses a
first-run test, or its runner counted failures outside the report.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

lib/flaky-run.ts

@@ -100,20 +100,27 @@ export function rerunSelection(runner: 'playwright' | 'bun', t: JunitTest): stri
 
 const OUTPUT_TAIL = 256 * 1024;
 
+/** Set once gstack-flaky itself is asked to stop; every attempt after that counts as killed, whatever exit the runner chose. */
+let cancelled: string | undefined;
+
 /** Stream the child's output live (a quiet runner trips CI idle watchdogs) and keep its tail for reconciliation. */
 function defaultSpawn(argv: string[], env: NodeJS.ProcessEnv, cwd: string): Promise<Attempted> {
   return new Promise((resolve, reject) => {
     let tail = '';
     const keep = (chunk: Buffer) => { tail = (tail + chunk.toString('utf-8')).slice(-OUTPUT_TAIL); };
     const child = spawnChild(argv[0]!, argv.slice(1), { stdio: ['inherit', 'pipe', 'pipe'], env, cwd });
-    const forward = (s: NodeJS.Signals) => { child.kill(s); };
+    const forward = (s: NodeJS.Signals) => { cancelled ??= s; child.kill(s); };
     const release = () => { process.off('SIGINT', forward); process.off('SIGTERM', forward); };
     process.on('SIGINT', forward);
     process.on('SIGTERM', forward);
     child.stdout!.on('data', (c: Buffer) => { process.stdout.write(c); keep(c); });
     child.stderr!.on('data', (c: Buffer) => { process.stderr.write(c); keep(c); });
     child.on('error', e => { release(); reject(new JunitError('FLAKY_COMMAND_NOT_STARTED', `could not start ${argv[0]}: ${e.message}`)); });
-    child.on('close', (code, signal) => { release(); resolve({ exit: code ?? 128, output: tail, ...(signal ? { signal } : {}) }); });
+    child.on('close', (code, signal) => {
+      release();
+      const killed = signal ?? cancelled;
+      resolve({ exit: code ?? 128, output: tail, ...(killed ? { signal: killed } : {}) });
+    });
   });
 }
 
@@ -208,13 +215,31 @@ async function attempt(opts: RunOptions, argv: string[], cwd: string, env: NodeJ
   }
 }
 
+/**
+ * What keeps a rerun from clearing anything even when it exited 0: failures
+ * its runner counted outside the report and, for a whole-command rerun, any
+ * failed test or any first-run test missing from its report.
+ */
+function rerunProblem(again: Attempted & { report?: JunitReport }, first: JunitReport, whole: boolean): string | undefined {
+  if (!again.report) return undefined;
+  const failedNow = again.report.tests.filter(x => finalAttempt(x) === 'fail');
+  const unreported = unreportedFailures(first.runner, again.output, failedNow.length);
+  if (unreported) return unreported;
+  if (!whole) return undefined;
+  if (failedNow.length) return `${failedNow.length} test(s) failed in the whole-command rerun`;
+  const seen = new Set(again.report.tests.map(x => x.id));
+  const missing = first.tests.filter(x => !seen.has(x.id)).length;
+  return missing ? `${missing} test(s) of the first run are missing from the whole-command rerun` : undefined;
+}
+
 /** A rerun clears a test only when its command exited 0 and the test ran once and passed on its first attempt. */
-function rerunVerdict(again: Attempted & { report?: JunitReport; error?: JunitError }, t: JunitTest): { status: 'flaky-pass' | 'rerun-fail'; attempts: string[]; reason?: string } {
+function rerunVerdict(again: Attempted & { report?: JunitReport; error?: JunitError }, t: JunitTest, problem?: string): { status: 'flaky-pass' | 'rerun-fail'; attempts: string[]; reason?: string } {
   const match = again.report?.tests.find(x => x.id === t.id);
   if (!match) return { status: 'rerun-fail', attempts: [...t.attempts, 'fail'], reason: again.error ? `rerun: ${again.error.message}` : 'rerun: the report does not contain this test' };
   const attempts = [...t.attempts, ...match.attempts];
   if (again.signal) return { status: 'rerun-fail', attempts, reason: `rerun: killed by ${again.signal}` };
   if (again.exit !== 0) return { status: 'rerun-fail', attempts, reason: `rerun: the command exited ${again.exit}` };
+  if (problem) return { status: 'rerun-fail', attempts, reason: `rerun: ${problem}` };
   if (match.executions !== 1 || match.attempts.join() !== 'pass') return { status: 'rerun-fail', attempts, reason: `rerun: the test did not pass once on its first attempt (${match.attempts.join(',')})` };
   return { status: 'flaky-pass', attempts };
 }
@@ -261,8 +286,8 @@ export async function runFlaky(opts: RunOptions): Promise<RunResult> {
   const firstXml = fs.readFileSync(junit);
   let reruns = 0;
   const rerun = async (argv: string[]) => {
-    const again = await attempt(opts, argv, cwd, env, report.runner);
     reruns++;
+    const again = await attempt(opts, argv, cwd, env, report.runner);
     if (fs.existsSync(junit)) fs.renameSync(junit, `${junit.replace(/\.xml$/i, '')}.rerun-${reruns}.xml`);
     return again;
   };
@@ -275,14 +300,15 @@ export async function runFlaky(opts: RunOptions): Promise<RunResult> {
       for (const t of eligible) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `not rerun: the run is already red, and a rerun would repeat the whole command (${wholeRerun})` });
     } else if (wholeRerun) {
       const again = await rerun(opts.command);
-      for (const t of eligible) record(t, rerunVerdict(again, t), 'command');
+      const problem = rerunProblem(again, report, true);
+      for (const t of eligible) record(t, rerunVerdict(again, t, problem), 'command');
     } else if (eligible.length) {
       let interrupted = false;
       for (const t of eligible) {
         if (interrupted) { outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: 'not rerun: an earlier rerun was killed' }); continue; }
         const again = await rerun([...opts.command, ...rerunSelection(report.runner as 'playwright' | 'bun', t)]);
         interrupted = !!again.signal;
-        record(t, rerunVerdict(again, t), 'test');
+        record(t, rerunVerdict(again, t, rerunProblem(again, report, false)), 'test');
       }
     }
   } finally {

test/flaky-run.test.ts

@@ -7,7 +7,7 @@ import { afterAll, describe, expect, test } from 'bun:test';
 import * as fs from 'fs';
 import * as os from 'os';
 import * as path from 'path';
-import { spawnSync } from 'child_process';
+import { spawn, spawnSync } from 'child_process';
 import { JunitError, parseJunit, readJunit, testId } from '../lib/junit';
 import { appendLedger, incompleteRun, ledgerPath, renderResult, rerunSelection, runFlaky, safeExcerpt, stepSummary, unreportedFailures, workflowEscape } from '../lib/flaky-run';
 
@@ -260,6 +260,18 @@ describe('runFlaky', () => {
     expect(r).toMatchObject({ exitCode: 0, wholeRerun: 'a generic runner cannot prove every test ran' });
   });
 
+  test('a whole-command rerun clears nothing when any test fails in it, a first-run test is missing, it is killed or it retried', async () => {
+    const dir = tmp();
+    const generic = (second: string, exit = 0, signal?: string) => base(fakeRun((_a, n) => n === 1 ? suite(fail('a') + pass('b')) : second, () => exit && 1, undefined, n => n === 2 ? signal : undefined), dir, { runner: 'generic' });
+    expect((await generic(suite(pass('a') + fail('b')))).outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: 1 test(s) failed in the whole-command rerun' });
+    expect((await generic(suite(pass('a')))).outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: 1 test(s) of the first run are missing from the whole-command rerun' });
+    expect((await generic(suite(pass('a') + pass('b')), 0, 'SIGTERM')).outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: killed by SIGTERM' });
+    const retried = await generic(suite('<testcase name="a" classname="" file="a.test.ts"><flakyFailure message="x"/></testcase>' + pass('b')));
+    expect(retried.outcomes.find(o => o.status === 'rerun-fail')).toMatchObject({ attempts: ['fail', 'fail', 'pass'] });
+    expect(retried.ledger.find(l => l.kind === 'rerun-fail')).toMatchObject({ rerun_scope: 'command' });
+    expect(retried.exitCode).toBe(1);
+  });
+
   test('an incomplete run that is already red is not rerun as a whole', async () => {
     const dir = tmp();
     const f = fakeRun(() => suite(fail('b') + pass('b') + fail('c')), () => 1, () => '');
@@ -359,6 +371,40 @@ describe('bin/gstack-flaky', () => {
     expect(r.stderr).toContain('fix: ');
   });
 
+  test('a runner that dies by a signal exits 3 FLAKY_COMMAND_KILLED and is not rerun', () => {
+    const dir = tmp();
+    const runner = path.join(dir, 'runner.sh');
+    fs.writeFileSync(runner, ['#!/usr/bin/env bash', 'echo x >> count', `printf '%s' '${suite(fail('b'))}' > r.xml`, 'kill -TERM $$'].join('\n'), { mode: 0o755 });
+    const r = spawnSync(path.join(ROOT, 'bin', 'gstack-flaky'), ['run', '--junit', 'r.xml', '--runner', 'bun', '--ledger', path.join(dir, 'l.jsonl'), '--', runner], { cwd: dir, encoding: 'utf-8', timeout: 30_000 });
+    expect(r.status).toBe(3);
+    expect(r.stderr).toContain('(FLAKY_COMMAND_KILLED)');
+    expect(fs.readFileSync(path.join(dir, 'count'), 'utf-8').trim().split('\n')).toHaveLength(1);
+  });
+
+  test('cancelling gstack-flaky is latched even when the runner traps the signal and exits by code', async () => {
+    const dir = tmp();
+    const runner = path.join(dir, 'runner.sh');
+    fs.writeFileSync(runner, [
+      '#!/usr/bin/env bash',
+      'echo x >> count',
+      'trap "cp report.xml r.xml; exit 143" TERM',
+      'touch ready',
+      'for i in $(seq 1 300); do sleep 0.1; done',
+    ].join('\n'), { mode: 0o755 });
+    fs.writeFileSync(path.join(dir, 'report.xml'), suite(fail('b')));
+    const child = spawn(path.join(ROOT, 'bin', 'gstack-flaky'), ['run', '--junit', 'r.xml', '--runner', 'generic', '--ledger', path.join(dir, 'l.jsonl'), '--', runner], { cwd: dir, stdio: ['ignore', 'ignore', 'pipe'] });
+    let stderr = '';
+    child.stderr.on('data', (c: Buffer) => { stderr += c.toString(); });
+    const done = new Promise<number | null>(resolve => child.on('close', code => resolve(code)));
+    const deadline = Date.now() + 20_000;
+    while (!fs.existsSync(path.join(dir, 'ready')) && Date.now() < deadline) await Bun.sleep(50);
+    child.kill('SIGTERM');
+    expect(await done).toBe(3);
+    expect(stderr).toContain('(FLAKY_COMMAND_KILLED)');
+    expect(fs.readFileSync(path.join(dir, 'count'), 'utf-8').trim().split('\n')).toHaveLength(1);
+    expect(fs.existsSync(path.join(dir, 'l.jsonl'))).toBe(false);
+  }, 30_000);
+
   test('every FLAKY_ code the bin and the library emit is in the docs table', () => {
     const doc = fs.readFileSync(path.join(ROOT, 'docs', 'flaky.md'), 'utf-8');
     const src = fs.readFileSync(path.join(ROOT, 'bin', 'gstack-flaky'), 'utf-8') + fs.readFileSync(path.join(ROOT, 'lib', 'flaky-run.ts'), 'utf-8') + fs.readFileSync(path.join(ROOT, 'lib', 'junit.ts'), 'utf-8');