feat(evals): the gate reruns failed cases once and reports a flaky green (GSTA-49 F1)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

test/ci-native-evidence.test.ts

@@ -44,7 +44,7 @@ function render(template: string, fields: Record<string, string>): string {
 
 test('every direct CI paid executor binds a safe unique run/attempt/job/slice identity', () => {
   expect(executors.map(({ name, jobName }) => `${name}:${jobName}`)).toEqual([
-    'evals.yml:eval-slices', 'evals-periodic.yml:eval-slices', 'evals-periodic.yml:eval-codex-slices', 'evals-periodic.yml:gate-census', 'evals-marathon.yml:eval-slices',
+    'evals.yml:eval-slices', 'evals.yml:eval-rerun', 'evals-periodic.yml:eval-slices', 'evals-periodic.yml:eval-codex-slices', 'evals-periodic.yml:gate-census', 'evals-marathon.yml:eval-slices',
   ]);
   const ids = new Set<string>();
   for (const [workflowIndex, { job, step }] of executors.entries()) {
@@ -59,7 +59,7 @@ test('every direct CI paid executor binds a safe unique run/attempt/job/slice id
       for (const attempt of ['1', '2']) {
         // The planner sizes the matrix; cover more slices than any live plan.
         expect(armed ? job.strategy.matrix.include : job.strategy.matrix.slice)
-          .toMatch(armed ? /^\$\{\{ fromJSON\(needs\.plan-slices\.outputs\.gate_matrix\) \}\}$/ : /^\$\{\{ fromJSON\(needs\.plan-slices\.outputs\.(?:[a-z]+_)*slices\) \}\}$/);
+          .toMatch(armed ? /^\$\{\{ fromJSON\(needs\.plan-slices\.outputs\.gate_matrix\) \}\}$/ : /^\$\{\{ fromJSON\(needs\.(?:plan-slices|rerun-plan)\.outputs\.(?:[a-z]+_)*slices\) \}\}$/);
         for (const arm of armed ? Object.keys(CENSUS_ARMS) : ['']) {
           for (let slice = 1; slice <= 64; slice++) {
             const id = render(env.EVALS_RUN_ID, {

test/ci-paid-coordination.test.ts

@@ -145,8 +145,8 @@ describe('paid CI coordination stays off the eval image', () => {
       const captures = hiddenUploads.filter(step => step.with?.name === 'native-captures-${{ env.EVALS_RUN_ID }}');
       // evals-periodic's host-run Codex job mounts $RUNNER_TEMP/eval-home as the container's HOME.
       const hostHome = (pattern: string) => pattern.replace('${{ runner.temp }}/eval-home/', '~/');
-      // evals.yml: eval-slices and train-bisect (GSTA-49 T1b), each under its own EVALS_RUN_ID.
-      expect(captures).toHaveLength(name === 'evals.yml' ? 2 : name === 'evals-periodic.yml' ? 3 : 2);
+      // evals.yml: eval-slices, the rule-16 eval-rerun and train-bisect (GSTA-49 T1b), each under its own EVALS_RUN_ID.
+      expect(captures).toHaveLength(name === 'evals.yml' ? 3 : name === 'evals-periodic.yml' ? 3 : 2);
       for (const capture of captures) {
         expect(capture.if).toBe('always()');
         expect(String(capture.with?.path).trim().split('\n').map(hostHome)).toEqual([
@@ -165,7 +165,11 @@ describe('paid CI coordination stays off the eval image', () => {
       const bisectLogs = name === 'evals.yml'
         ? jobs['train-bisect'].steps.find(step => step.with?.name === 'train-bisect-logs-${{ matrix.slice }}-a${{ github.run_attempt }}') : undefined;
       if (name === 'evals.yml') expect(String(bisectLogs?.with?.path)).toBe(String(logs?.with?.path));
-      expect(hiddenUploads.filter(step => !captures.includes(step))).toEqual([logs!, ...(codexLogs ? [codexLogs] : []), ...(bisectLogs ? [bisectLogs] : [])]);
+      // F1: the rule-16 eval-rerun keeps the same shard-log upload under its own name.
+      const rerunLogs = name === 'evals.yml'
+        ? jobs['eval-rerun'].steps.find(step => step.with?.name === 'paid-rerun-logs-${{ matrix.slice }}-a${{ github.run_attempt }}') : undefined;
+      if (name === 'evals.yml') expect(String(rerunLogs?.with?.path)).toBe(String(logs?.with?.path));
+      expect(hiddenUploads.filter(step => !captures.includes(step))).toEqual([logs!, ...(codexLogs ? [codexLogs] : []), ...(rerunLogs ? [rerunLogs] : []), ...(bisectLogs ? [bisectLogs] : [])]);
     });
   }
 

test/evals-train-workflow.test.ts

@@ -146,8 +146,9 @@ const pr = (labels: string[] = []) => ({ event_name: 'pull_request', actor: 'gar
 const dispatch = (ref = 'main') => ({ event_name: 'workflow_dispatch', actor: 'garrytan', repository: 'garrytan/gstack', repository_id: 1, sha: SHA.main, run_id: 7, run_attempt: 1, ref_name: ref,
   event: { repository: { default_branch: 'main' } } });
 const INPUTS = { evals_all: true, base_ref: '', base_sha: '', expected_sha: '', nonce: '', pr_receipts: '', cases: '', prefix_shas: '', screen_pr: '', screen_merge_sha: '', screen_policy: '' };
-const TODAY_PR_JOBS = ['build-image', 'eval-slices', 'plan-slices', 'readiness', 'recover-receipts', 'slices-comment', 'slices-report'];
-const TODAY_DISPATCH_JOBS = ['build-image', 'eval-slices', 'plan-slices', 'slices-report'];
+// rerun-plan is the free rule-16 planner (F1); its paid eval-rerun runs only when it plans a rerun.
+const TODAY_PR_JOBS = ['build-image', 'eval-slices', 'plan-slices', 'readiness', 'recover-receipts', 'rerun-plan', 'slices-comment', 'slices-report'];
+const TODAY_DISPATCH_JOBS = ['build-image', 'eval-slices', 'plan-slices', 'rerun-plan', 'slices-report'];
 const TRAIN_STEPS = ['Write the train record', 'Upload the train record', "Check the screen's PR head (Free Tests)", 'Plan the bisect (one executor per prefix)', 'Hand the bisect plan to its executors'];
 const train = { record_artifact: 'gstack-train-record', full_gate_check: 'gstack/full-gate' };
 
@@ -206,7 +207,7 @@ describe('under a train block (the gate edits, each only on its mode)', () => {
     const inputs = { ...INPUTS, evals_all: false, nonce: 'pr.screen.narrow-x', expected_sha: SHA.merge, base_ref: 'main', base_sha: SHA.base, screen_pr: '42', screen_merge_sha: SHA.merge, screen_policy: 'narrow' };
     const route = { ...ROUTE, ...train, mode: 'screen', screen_head: SHA.screenHead, screen_base: SHA.screenBase, screen_branch: 'feature' };
     const sim = simulate(evals, { github: dispatch(), inputs, outputs: { route } });
-    expect(sim.jobs).toEqual(['build-image', 'eval-slices', 'plan-slices', 'recover-receipts', 'route', 'slices-comment', 'slices-report']);
+    expect(sim.jobs).toEqual(['build-image', 'eval-slices', 'plan-slices', 'recover-receipts', 'rerun-plan', 'route', 'slices-comment', 'slices-report']);
     expect(sim.group).toBe('evals-screen-42');
     expect(sim.runName).toBe('E2E Evals [eval-bg pr.screen.narrow-x] screen #42');
     expect(sim.steps('recover-receipts')).not.toContain('Recover receipts of cancelled runs (base-ref code)');
@@ -228,7 +229,7 @@ describe('under a train block (the gate edits, each only on its mode)', () => {
   test('a train gate runs the full gate, writes its record from the report and publishes the check; it never reuses or binds', () => {
     const inputs = { ...INPUTS, nonce: 'gate.train-x', expected_sha: SHA.head };
     const sim = simulate(evals, { github: dispatch('train/t1'), inputs, outputs: { route: { ...ROUTE, ...train, mode: 'train' } } });
-    expect(sim.jobs).toEqual(['build-image', 'eval-slices', 'plan-slices', 'route', 'slices-report', 'train-check']);
+    expect(sim.jobs).toEqual(['build-image', 'eval-slices', 'plan-slices', 'rerun-plan', 'route', 'slices-report', 'train-check']);
     expect(sim.steps('plan-slices')).not.toContain('Confirm gate verdict reuse (R2a/R2)');
     expect(sim.steps('slices-report')).not.toContain("Bind this run's verdict for reuse (R2a/R2)");
     expect(sim.steps('slices-report')).toEqual(expect.arrayContaining(['Write the train record', 'Upload the train record']));
@@ -279,6 +280,56 @@ describe('GSTA-48 open items', () => {
   });
 });
 
+describe('rule-16 rerun wiring (F1)', () => {
+  const planned = { planned: 'true', slices: '[1]', slice_timeouts: '[0,40]' };
+  test('the paid rerun runs only when rerun-plan planned one, on the same checkout as the slices, and the report waits for it', () => {
+    const quiet = simulate(evals, { github: pr(), inputs: {}, outputs: { route: { ...ROUTE, mode: 'pr' }, 'rerun-plan': { planned: 'false', slices: '', slice_timeouts: '' } } });
+    expect(quiet.jobs).not.toContain('eval-rerun');
+    const sim = simulate(evals, { github: pr(), inputs: {}, outputs: { route: { ...ROUTE, mode: 'pr' }, 'rerun-plan': planned } });
+    expect(sim.jobs).toEqual([...TODAY_PR_JOBS, 'eval-rerun', 'route'].sort());
+    for (const job of ['rerun-plan', 'eval-rerun']) expect(sim.checkout(job)?.ref ?? '', job).toBe(sim.checkout('eval-slices')?.ref ?? '');
+    expect([evals.jobs['slices-report']!.needs].flat()).toEqual(expect.arrayContaining(['rerun-plan', 'eval-rerun']));
+    const screen = simulate(evals, { github: dispatch(), inputs: { ...INPUTS, evals_all: false, nonce: 'pr.screen.narrow-x', expected_sha: SHA.merge, base_ref: 'main', base_sha: SHA.base, screen_pr: '42', screen_merge_sha: SHA.merge, screen_policy: 'narrow' },
+      outputs: { route: { ...ROUTE, ...train, mode: 'screen', screen_head: SHA.screenHead, screen_base: SHA.screenBase, screen_branch: 'feature' }, 'rerun-plan': planned } });
+    for (const job of ['rerun-plan', 'eval-rerun']) expect(screen.checkout(job)?.ref, job).toBe(SHA.merge);
+    const trainRun = simulate(evals, { github: dispatch('train/t1'), inputs: { ...INPUTS, nonce: 'gate.train-x', expected_sha: SHA.head }, outputs: { route: { ...ROUTE, ...train, mode: 'train' }, 'rerun-plan': planned } });
+    expect(trainRun.jobs).toEqual(expect.arrayContaining(['rerun-plan', 'eval-rerun', 'slices-report', 'train-check']));
+  });
+
+  test('a confirmed verdict reuse plans and runs no rerun', () => {
+    const sim = simulate(evals, { github: pr(), inputs: {}, outputs: { route: { ...ROUTE, mode: 'pr' }, 'plan-slices': { gate_reuse: 'true' }, 'rerun-plan': planned } });
+    expect(sim.jobs).not.toContain('rerun-plan');
+    expect(sim.jobs).not.toContain('eval-rerun');
+    expect(sim.jobs).not.toContain('eval-slices');
+  });
+
+  test('the rerun runs its own attempt-scoped manifest fresh: no receipts in, none out, results under paid-rerun-*', () => {
+    const job = evals.jobs['eval-rerun']!;
+    expect(job.strategy!.matrix.slice).toBe('${{ fromJSON(needs.rerun-plan.outputs.slices) }}');
+    expect(job.steps.find(st => st.uses?.startsWith('actions/download-artifact@'))!.with).toEqual({ name: 'rerun-plan-a${{ github.run_attempt }}', path: '/tmp/paid-plan' });
+    const run = job.steps.find(st => st.name === 'Rerun slice ${{ matrix.slice }} (rule 16, once)')!;
+    expect(run.run).toContain('--plan /tmp/paid-plan/manifest.json --slice ${{ matrix.slice }}');
+    expect(Object.keys(run.env!).filter(key => key.startsWith('EVALS_CACHE'))).toEqual([]);
+    expect(JSON.stringify(job)).not.toContain('receipts');
+    expect(job.steps.find(st => st.name === 'Upload rerun results')!.with).toMatchObject({ name: 'paid-rerun-${{ matrix.slice }}-a${{ github.run_attempt }}', path: '/tmp/paid-rerun-results' });
+    const plan = evals.jobs['rerun-plan']!;
+    expect(plan.permissions).toEqual({ contents: 'read' });
+    expect(JSON.stringify(plan)).not.toMatch(/secrets\.|restore-deps|bun install/);
+    expect(plan.steps.find(st => st.name === "Plan the rule-16 rerun of this attempt's failed cases")!.run!.replace(/\s*\\\n\s*/g, ' ')).toContain('--report /tmp/paid-report --rerun-plan /tmp/paid-rerun');
+  });
+
+  test('the report reads every attempt and rerun, publishes flake records, and the comment never hides a non-flaky failed record', () => {
+    const report = evals.jobs['slices-report']!;
+    const patterns = report.steps.filter(st => st.uses?.startsWith('actions/download-artifact@')).map(st => st.with?.pattern ?? st.with?.name);
+    expect(patterns).toEqual(expect.arrayContaining(['paid-slice-*', 'paid-rerun-*', 'rerun-plan-a*']));
+    expect(report.steps.find(st => st.name === 'Upload flake records for the daily flaky-fix pass')!.with).toMatchObject({ name: 'flake-records-a${{ github.run_attempt }}', path: '/tmp/paid-report/flake-records.jsonl' });
+    const comment = evals.jobs['slices-comment']!.steps.find(st => st.name === 'Post PR comment')!.run!;
+    expect(comment).toContain('[ "$FAILED" -gt "$FLAKY_FAILED" ]');
+    expect(comment).toContain("FLAKY_FAILED=$(jq -r '.flakyCollectorFailures // 0' \"$VERIFIED\")");
+    expect(comment).toContain('((.flakyCollectorFailures // 0) <= .totals.failed)');
+  });
+});
+
 describe.skipIf(!Bun.which('jq') || !Bun.which('bash'))('the route job reads the train policy from the base revision with git and jq only', () => {
   const route = evals.jobs.route!;
   const policyStep = route.steps.find(st => st.name === 'Read the train policy (base revision)')!;

test/fixtures/shard-cli-parity/results.json

@@ -461,6 +461,7 @@
           "planPath": null,
           "sliceIndex": null,
           "reportDir": null,
+          "rerunPlanDir": null,
           "writeDurations": false,
           "maxParallel": null,
           "caseId": null,
@@ -495,6 +496,7 @@
           "planPath": null,
           "sliceIndex": null,
           "reportDir": null,
+          "rerunPlanDir": null,
           "writeDurations": false,
           "maxParallel": null,
           "caseId": null,
@@ -529,6 +531,7 @@
           "planPath": null,
           "sliceIndex": null,
           "reportDir": null,
+          "rerunPlanDir": null,
           "writeDurations": false,
           "maxParallel": null,
           "caseId": null,
@@ -564,6 +567,7 @@
           "planPath": null,
           "sliceIndex": null,
           "reportDir": null,
+          "rerunPlanDir": null,
           "writeDurations": false,
           "maxParallel": null,
           "caseId": null,
@@ -598,6 +602,7 @@
           "planPath": null,
           "sliceIndex": null,
           "reportDir": null,
+          "rerunPlanDir": null,
           "writeDurations": false,
           "maxParallel": null,
           "caseId": null,
@@ -631,6 +636,7 @@
           "planPath": "manifest.json",
           "sliceIndex": 2,
           "reportDir": null,
+          "rerunPlanDir": null,
           "writeDurations": false,
           "maxParallel": null,
           "caseId": null,
@@ -663,6 +669,7 @@
           "planPath": null,
           "sliceIndex": null,
           "reportDir": "reports",
+          "rerunPlanDir": null,
           "writeDurations": true,
           "maxParallel": null,
           "caseId": null,
@@ -698,6 +705,7 @@
           "planPath": null,
           "sliceIndex": null,
           "reportDir": null,
+          "rerunPlanDir": null,
           "writeDurations": false,
           "maxParallel": null,
           "caseId": null,

test/gate-reuse.test.ts

@@ -9,7 +9,7 @@ import * as fs from 'node:fs';
 import * as os from 'node:os';
 import * as path from 'node:path';
 import {
-  BINDING_FILE, CANDIDATE_FILE, REUSE_FILE, buildBinding, confirmReuse, findReusableGate, parseReuse, reuseReport, selectionFingerprint,
+  BINDING_FILE, CANDIDATE_FILE, REUSE_FILE, buildBinding, confirmReuse, findReusableGate, parseBinding, parseReuse, reuseReport, selectionFingerprint,
   type GateBinding, type GateReuse,
 } from '../scripts/gate-reuse';
 import type { GhRunner, WorkflowRun } from '../scripts/recover-receipts';
@@ -558,3 +558,74 @@ describe('dispatched screens (GSTA-49 T1b): authenticated by PR, head, repositor
     expect(screen('narrow', [trainRun, screenRun(300)], { 300: screenBinding(sha.gated!), 310: screenBinding(sha.gated!) }).candidate?.reused_run_id).toBe(300);
   });
 });
+
+describe('rule-16 flaky GREENs are reusable (GSTA-53, F1)', () => {
+  const flaky = (over: Partial<GateBinding> = {}) => binding({ flaky_cases: ['tpa-absent-darwin'], ...over });
+  /** A run that bound once per attempt: gate-binding-a<n> each, download by name (a `gh run rerun --failed` attempt binds again). */
+  function attemptsGh(runs: WorkflowRun[], perAttempt: Record<number, GateBinding>): GhRunner {
+    const inner = fakeGh(runs, { 300: perAttempt[1]! });
+    return (args, timeoutMs) => {
+      if (args.some(a => /\/runs\/300\/artifacts$/.test(a))) {
+        return { status: 0, stdout: JSON.stringify({ artifacts: [...Object.keys(perAttempt).map(n => ({ name: `gate-binding-a${n}`, expired: false })), { name: 'paid-slice-1-a1', expired: false }] }), stderr: '' };
+      }
+      if (args[0] === 'run' && args[1] === 'download' && args[2] === '300') {
+        const dir = args[args.indexOf('-D') + 1]!;
+        fs.mkdirSync(dir, { recursive: true });
+        fs.writeFileSync(path.join(dir, BINDING_FILE), JSON.stringify(perAttempt[Number(args[args.indexOf('-n') + 1]!.slice('gate-binding-a'.length))]));
+        return { status: 0, stdout: '', stderr: '' };
+      }
+      return inner(args, timeoutMs);
+    };
+  }
+  const findWith = (head: string, gh: GhRunner, baseSha = sha.base!) => {
+    const out = path.join(scratch, 'out-' + ++outCount);
+    const result = findReusableGate({ repo: REPO, pr: 42, headRepo: REPO, branch: 'feature', selfRunId: 500, headSha: head, testedSha: tested(baseSha, head), repoRoot: receipt, out, gh });
+    return { ...result, out, text: result.lines.join('\n') };
+  };
+
+  test('R2a: a restamp of a flaky-annotated GREEN reuses it and carries the flaky case names', () => {
+    const { candidate } = find(sha.restamp!, [run(300, sha.gated!)], { 300: flaky() });
+    expect(candidate).toMatchObject({ kind: 'restamp-only', reused_run_id: 300, reused_flaky: ['tpa-absent-darwin'] });
+    expect(parseReuse(candidate)).toEqual(candidate);
+    expect(reuseReport(candidate!, REPO)).toContain('flaky (rule 16): tpa-absent-darwin failed once and passed on their one rerun in the reused gate');
+  });
+
+  test('R2: main moved with an identical own diff reuses a flaky-annotated GREEN', () => {
+    const { candidate } = find(sha.rebased!, [run(300, sha.gated!)], { 300: flaky() }, { baseSha: sha.moved });
+    expect(candidate).toMatchObject({ kind: 'main-moved', reused_run_id: 300, reused_flaky: ['tpa-absent-darwin'] });
+  });
+
+  test('a manual --failed attempt that went green: its newest binding is reused (R2a and R2), never the red first attempt', () => {
+    const attempts = { 1: binding({ verdict: 'RED', reconcile_exit: 1 }), 2: flaky({ run_attempt: 2 }) };
+    const restamp = findWith(sha.restamp!, attemptsGh([run(300, sha.gated!)], attempts));
+    expect(restamp.candidate, restamp.text).toMatchObject({ kind: 'restamp-only', reused_run_attempt: 2, reused_flaky: ['tpa-absent-darwin'] });
+    const moved = findWith(sha.rebased!, attemptsGh([run(300, sha.gated!)], attempts), sha.moved!);
+    expect(moved.candidate, moved.text).toMatchObject({ kind: 'main-moved', reused_run_attempt: 2 });
+  });
+
+  test('negative control: a run still red after its rerun is never reused, whatever it calls flaky', () => {
+    const stillRed = find(sha.restamp!, [run(300, sha.gated!, 'failure')], { 300: flaky({ verdict: 'RED', reconcile_exit: 1 }) });
+    expect(stillRed.candidate).toBeNull();
+    expect(stillRed.text).toContain('is RED, reconcile exit 1');
+    const moved = find(sha.rebased!, [run(300, sha.gated!, 'failure')], { 300: flaky({ verdict: 'RED', reconcile_exit: 1 }) }, { baseSha: sha.moved });
+    expect(moved.candidate).toBeNull();
+    const newestRed = findWith(sha.restamp!, attemptsGh([run(300, sha.gated!, 'failure')], { 1: flaky(), 2: binding({ run_attempt: 2, verdict: 'RED', reconcile_exit: 1 }) }));
+    expect(newestRed.candidate).toBeNull();
+    expect(newestRed.text).toContain('is RED');
+  });
+
+  test('bind records the report\'s flaky cases; a malformed flaky list is no binding', () => {
+    const dir = path.join(scratch, 'bind-flaky');
+    fs.mkdirSync(dir, { recursive: true });
+    fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify(manifest()));
+    fs.writeFileSync(path.join(dir, 'collector-outcomes.json'), JSON.stringify({ verdict: { verdict: 'GREEN' }, totals: { manual_accepted: 0 },
+      flaky: [{ case: 'tpa-absent-darwin' }, { case: 'plan-design-with-ui-scope' }] }));
+    const { binding: bound, reason } = buildBinding({ repository: REPO, pr: 42, runId: 300, runAttempt: 2, headSha: 'a'.repeat(40), baseSha: 'b'.repeat(40), profile: 'pr', reconcileExit: 0, reportDir: dir });
+    expect(bound).toMatchObject({ verdict: 'GREEN', run_attempt: 2, flaky_cases: ['tpa-absent-darwin', 'plan-design-with-ui-scope'] });
+    expect(reason).toContain('bound GREEN (flaky: tpa-absent-darwin, plan-design-with-ui-scope)');
+    expect(parseBinding({ ...binding(), flaky_cases: 'x' })).toBeNull();
+    expect(parseBinding({ ...binding(), flaky_cases: [''] })).toBeNull();
+    expect(parseBinding(binding())?.flaky_cases).toBeUndefined();
+  });
+});
+

test/helpers/periodic-exclude-data.ts

@@ -75,6 +75,13 @@ export const CASE_CI_EXCLUDE: Record<string, { reason: string; tracking: string
  *   infraRedispatch - a census whose every red verdict is machine-classified
  *                 INFRA or INCOMPLETE may be re-dispatched this many times as
  *                 a new run; both runs are reported.
+ *   flaky rerun - (CREW rule 16, Garry 2026-10-19: "you can just run the
+ *                 failed test again and note that it is flaky") the gate reruns
+ *                 a red case verdict once at the same head with the same trials
+ *                 and threshold; a pass is a flaky-annotated PASS, a second
+ *                 failure is red. Trial records and this version are unchanged;
+ *                 a rerun's trials stay in its own artifacts and flake record
+ *                 (scripts/lib/paid-rerun.ts, docs/TESTING_INTERNALS.md#flaky-reruns).
  */
 export const EVAL_POLICY = {
   version: 3,

test/paid-rerun.test.ts

@@ -0,0 +1,265 @@
+/**
+ * Rule-16 flaky reruns (CREW rule 16, Garry 2026-10-19): a red case verdict
+ * reruns once at the same head; a pass is GREEN with a flaky annotation and a
+ * flake record, a second failure stays red. Every case here goes through the
+ * real `--report` and `--rerun-plan` CLI paths the workflow runs, on a trimmed
+ * capture of run 39293281037 written below (plan-ceo-finding-floor timed out in
+ * attempt 1 and passed on `gh run rerun --failed` attempt 2; slice 6 renumbered
+ * to 1, the JUnit failure message cut to its first two lines).
+ */
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { type CaseObservation, FLAKE_RECORDS_FILE, RERUN_PLAN_ARTIFACT, resolveCases, rerunCandidates, rerunSpend } from '../scripts/lib/paid-rerun';
+import { sliceExitCode } from '../scripts/lib/paid-plan';
+
+const ROOT = path.resolve(import.meta.dir, '..');
+const CASE = 'plan-ceo-finding-floor';
+const SHARD = 'test/skill-e2e-plan-ceo-finding-floor.test.ts';
+const FAIL = 'paid-slice-1-a1';
+const PASS = 'paid-slice-1-a2';
+
+const selection = { e2e: [CASE], judges: [] };
+const budget = { timeoutMs: 1020000, source: 'registered', policyId: 'skill-e2e-plan-ceo-finding-floor-existing-retry-v1' };
+const MANIFEST = { version: 1, tier: 'gate', evalsAll: false, sliceCount: 1, selectionReason: 'trimmed replay of run 39293281037 (slice 6 -> 1)', profile: 'pr', selection,
+  prCoverage: { mode: 'dependents', e2e: [CASE], judges: [], deferred: [], unknownFiles: [], unknownFileLabels: [], directCases: [], derivedFiles: [], noConsumerFiles: [],
+    deferredPromptFiles: [], missingCoverage: [], needsFullValidation: false, reasons: [] },
+  plan: { sliceBudgetMs: 420000, jobs: 2, estimatedSliceMs: [418862], ciTimeoutMinutes: 50, sliceCiTimeoutMinutes: [50], codexSlices: [] },
+  entries: [{ file: SHARD, slice: 1, status: 'planned', estimatedMs: 248087, budget }] };
+const junit = (failure: string | null, seconds: string) => `<?xml version="1.0" encoding="UTF-8"?>
+<testsuites name="bun test" tests="1" assertions="0" failures="${failure ? 1 : 0}" skipped="0" time="${seconds}">
+  <testsuite name="${SHARD}" file="${SHARD}" tests="1" assertions="0" failures="${failure ? 1 : 0}" skipped="0" time="${seconds}">
+    <testsuite name="/plan-ceo-review AskUserQuestion floor (gate)" file="${SHARD}" line="15" tests="1" assertions="0" failures="${failure ? 1 : 0}" skipped="0" time="${seconds}">
+      <testcase name="seeded forcing finding causes the agent to fire at least one AskUserQuestion" classname="/plan-ceo-review AskUserQuestion floor (gate)" time="${seconds}" file="${SHARD}" line="17" assertions="0"${failure
+        ? `>\n        <failure type="Error" message="${failure}" />\n      </testcase>` : ' />'}
+    </testsuite>
+  </testsuite>
+</testsuites>
+`;
+/** The two captured attempts of slice 6: the slice result, the shard's JUnit and its session ledger. */
+const CAPTURE: Record<string, { slice: Record<string, unknown>; junit: string; ledger: Record<string, unknown> }> = {
+  [FAIL]: {
+    slice: { version: 1, tier: 'gate', profile: 'pr', selection, sliceIndex: 1, sliceCount: 1, attempt: 1, startedAt: 1792431197455, finishedAt: 1792431811508,
+      outcomes: [{ files: [SHARD], status: 'failed', exitCode: 1, elapsedMs: 610510, executedTests: 1, skippedTests: 0, budget }] },
+    junit: junit('floor test FAILED: outcome=timeout elapsed=609738ms&#10;summary: no qualifying finding question within 600000ms', '610.414157'),
+    ledger: { key: 'pty:session#1', runner: 'pty', started_at: '2026-10-19T17:33:21.040Z', budget_ms: 600000, elapsed_ms: 609726, end: 'observer_timeout', billed: false },
+  },
+  [PASS]: {
+    slice: { version: 1, tier: 'gate', profile: 'pr', selection, sliceIndex: 1, sliceCount: 1, attempt: 2, startedAt: 1792433125271, finishedAt: 1792433342167,
+      outcomes: [{ files: [SHARD], status: 'passed', exitCode: 0, elapsedMs: 117689, executedTests: 1, skippedTests: 0, budget }] },
+    junit: junit(null, '117.609903'),
+    ledger: { key: 'pty:session#1', runner: 'pty', started_at: '2026-10-19T18:05:41.000Z', budget_ms: 600000, elapsed_ms: 117600, end: 'completed', billed: false },
+  },
+};
+
+let base: string;
+beforeAll(() => { base = fs.mkdtempSync(path.join(os.tmpdir(), 'paid-rerun-')); });
+afterAll(() => { fs.rmSync(base, { recursive: true, force: true }); });
+
+type Placement = { from: string; to: string; attempt: number; edit?: (slice: any) => void };
+function place(dir: string, p: Placement, extra: Record<string, unknown> = {}) {
+  const capture = CAPTURE[p.from]!;
+  const shard = path.join(dir, p.to, 'shards', 'skill-e2e-plan-ceo-finding-floor');
+  fs.mkdirSync(shard, { recursive: true });
+  fs.writeFileSync(path.join(shard, 'junit.xml'), capture.junit);
+  fs.writeFileSync(path.join(shard, 'session-ledger.jsonl'), JSON.stringify(capture.ledger) + '\n');
+  const slice = structuredClone({ ...capture.slice, attempt: p.attempt, ...extra }) as any;
+  p.edit?.(slice);
+  fs.writeFileSync(path.join(dir, p.to, 'slice-1.json'), JSON.stringify(slice));
+}
+let counter = 0;
+/** A report dir holding the fixture manifest and the named first-execution slice artifacts. */
+function reportDir(placements: Placement[]): string {
+  const dir = path.join(base, `case-${++counter}`);
+  fs.mkdirSync(dir, { recursive: true });
+  fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify(MANIFEST));
+  for (const p of placements) place(dir, p);
+  return dir;
+}
+
+const env = (extra: NodeJS.ProcessEnv = {}) => ({ ...process.env, EVALS_TIER: 'gate', GITHUB_RUN_ID: '39293281037', GITHUB_SHA: '', GSTACK_GATE_PR: '', GSTACK_GATE_HEAD_SHA: '', ...extra });
+const cli = (args: string[], extra: NodeJS.ProcessEnv = {}) => spawnSync(process.execPath, ['--no-env-file', '--config=/dev/null', path.join(ROOT, 'scripts/test-paid-shards.ts'), '--tier', 'gate', ...args],
+  { cwd: ROOT, encoding: 'utf8', timeout: 60_000, env: env(extra) });
+function report(dir: string) {
+  const r = cli(['--report', dir]);
+  const collector = JSON.parse(fs.readFileSync(path.join(dir, 'collector-outcomes.json'), 'utf8'));
+  const flakes = fs.existsSync(path.join(dir, FLAKE_RECORDS_FILE))
+    ? fs.readFileSync(path.join(dir, FLAKE_RECORDS_FILE), 'utf8').trim().split('\n').map(line => JSON.parse(line)) : [];
+  return { status: r.status, out: `${r.stdout}\n${r.stderr}`, collector, flakes };
+}
+/** The rerun-plan job: writes `<dir>/rerun-plan-a<attempt>/manifest.json` when it plans a rerun. */
+function planRerun(dir: string, attempt: number) {
+  const out = path.join(dir, `${RERUN_PLAN_ARTIFACT}-a${attempt}`);
+  const r = cli(['--report', dir, '--rerun-plan', out], { GITHUB_RUN_ATTEMPT: String(attempt) });
+  expect(r.status, r.stdout + r.stderr).toBe(0);
+  const file = path.join(out, 'manifest.json');
+  return { out: r.stdout, manifest: fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, 'utf8')) : null };
+}
+/** Attempt 1 failed, the real planner wrote its rerun manifest, and the eval-rerun job's result is `from` (as paid-rerun-1-a1). */
+function withRerun(from: string, edit?: (slice: any) => void): string {
+  const dir = reportDir([{ from: FAIL, to: FAIL, attempt: 1 }]);
+  const plan = planRerun(dir, 1).manifest;
+  expect(plan).not.toBeNull();
+  fs.rmSync(path.join(dir, 'collector-outcomes.json'), { force: true });
+  place(dir, { from, to: 'paid-rerun-1-a1', attempt: 1, edit }, { rerun: true, sliceCount: plan.sliceCount, selection: plan.selection });
+  return dir;
+}
+
+describe('replay of run 39293281037 (trimmed capture)', () => {
+  test('attempt 1 alone is RED, and the planner reruns exactly the failed case once, fresh', () => {
+    const dir = reportDir([{ from: FAIL, to: FAIL, attempt: 1 }]);
+    const r = report(dir);
+    expect(r.status, r.out).toBe(1);
+    expect(r.out).toContain('VERDICT RED');
+    expect(r.flakes).toEqual([]);
+    const plan = planRerun(dir, 1);
+    expect(plan.out).toContain('rule-16 rerun of attempt 1: 1 case(s) once');
+    expect(plan.manifest.rerun).toEqual({ attempt: 1, cases: [CASE] });
+    expect(plan.manifest.entries.map((e: any) => e.file)).toEqual([SHARD]);
+    expect(plan.manifest.selection).toEqual({ e2e: [CASE], judges: [] });
+    expect(plan.manifest.tier).toBe('gate');
+  });
+
+  test('a manual `gh run rerun --failed` attempt that passes the red: GREEN, flaky-annotated, one flake record', () => {
+    const r = report(reportDir([{ from: FAIL, to: FAIL, attempt: 1 }, { from: PASS, to: PASS, attempt: 2 }]));
+    expect(r.status, r.out).toBe(0);
+    expect(r.out).toContain('VERDICT GREEN');
+    expect(r.out).toContain('FLAKY 1 (failed once, passed on its one rule-16 rerun)');
+    expect(r.collector.verdict.verdict).toBe('GREEN');
+    expect(r.collector.flakyLines).toHaveLength(1);
+    expect(r.collector.flakyLines[0]).toContain(`⚑ ${CASE}  rule  FLAKY: failed in attempt 1 (paid-slice-1-a1), passed on its one rerun in attempt 2 (paid-slice-1-a2); rerun cost unknown (no billing captured); first failure: assertion: floor test FAILED: outcome=timeout`);
+    expect(r.flakes).toEqual([expect.objectContaining({ schema: 1, kind: 'gate-flake', case: CASE, case_kind: 'rule', run_id: '39293281037',
+      first: { attempt: 1, phase: 'first', where: 'attempt 1 (paid-slice-1-a1)' }, rerun: { attempt: 2, phase: 'first', where: 'attempt 2 (paid-slice-1-a2)' },
+      symptom: expect.stringContaining('outcome=timeout elapsed=609738ms'), rerun_cost_usd: null })]);
+  });
+
+  test('the in-run rerun that passes: GREEN, flaky-annotated, its record names the rerun, and nothing reruns again', () => {
+    const dir = withRerun(PASS);
+    const r = report(dir);
+    expect(r.status, r.out).toBe(0);
+    expect(r.flakes).toEqual([expect.objectContaining({ case: CASE, first: { attempt: 1, phase: 'first', where: 'attempt 1 (paid-slice-1-a1)' },
+      rerun: { attempt: 1, phase: 'rerun', where: 'attempt 1 rerun (paid-rerun-1-a1)' } })]);
+    expect(planRerun(dir, 2).manifest).toBeNull();
+  });
+});
+
+describe('negative controls: rule 16 never turns a real red green', () => {
+  test('red twice stays RED, is named a real red, and is never rerun again (one rerun per case)', () => {
+    const dir = withRerun(FAIL);
+    const r = report(dir);
+    expect(r.status, r.out).toBe(1);
+    expect(r.out).toContain(`✗ ${CASE}  rule  FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-1-a1); again in attempt 1 rerun (paid-rerun-1-a1)`);
+    expect(r.flakes).toEqual([]);
+    expect(r.collector.flakyLines).toEqual([]);
+    const again = planRerun(dir, 2);
+    expect(again.manifest).toBeNull();
+    expect(again.out).toContain(`not rerun: ${CASE} (failed twice`);
+  });
+
+  test('a manual attempt after a failed rerun never makes it flaky: two failures stay RED', () => {
+    const dir = withRerun(FAIL);
+    place(dir, { from: PASS, to: PASS, attempt: 2 });
+    const r = report(dir);
+    expect(r.status, r.out).toBe(1);
+    expect(r.flakes).toEqual([]);
+  });
+
+  test('a later attempt never hides a new red: attempt 1 passed, attempt 2 failed, RED', () => {
+    const r = report(reportDir([{ from: PASS, to: FAIL, attempt: 1 }, { from: FAIL, to: PASS, attempt: 2 }]));
+    expect(r.status, r.out).toBe(1);
+    expect(r.out).toContain('VERDICT RED');
+    expect(r.flakes).toEqual([]);
+  });
+
+  test('a first-attempt pass: GREEN with no flake record and nothing to rerun', () => {
+    const dir = reportDir([{ from: PASS, to: PASS, attempt: 1 }]);
+    const r = report(dir);
+    expect(r.status, r.out).toBe(0);
+    expect(r.flakes).toEqual([]);
+    expect(r.out).not.toContain('FLAKY');
+    expect(fs.existsSync(path.join(dir, FLAKE_RECORDS_FILE))).toBe(false);
+    expect(planRerun(dir, 1).manifest).toBeNull();
+  });
+
+  for (const [label, edit] of [
+    ['a runner error (harness)', (s: any) => { s.outcomes[0].runnerError = 'spawn failed'; }],
+    ['a slice-deadline hang (INCOMPLETE)', (s: any) => { Object.assign(s.outcomes[0], { sliceDeadline: 'hung', status: 'timed-out', exitCode: null }); }],
+    ['a checkpoint the slice never finalized', (s: any) => { s.checkpoint = true; }],
+  ] as const) {
+    test(`${label} is never rerun and never flaky, even when a later attempt passes`, () => {
+      expect(planRerun(reportDir([{ from: FAIL, to: FAIL, attempt: 1, edit }]), 1).manifest).toBeNull();
+      const r = report(reportDir([{ from: FAIL, to: FAIL, attempt: 1, edit }, { from: PASS, to: PASS, attempt: 2 }]));
+      expect(r.status, r.out).toBe(1);
+      expect(r.flakes).toEqual([]);
+    });
+  }
+
+  test('rerun results without their rerun manifest are set aside and the red stays', () => {
+    const dir = withRerun(PASS);
+    fs.rmSync(path.join(dir, `${RERUN_PLAN_ARTIFACT}-a1`), { recursive: true });
+    const r = report(dir);
+    expect(r.status, r.out).toBe(1);
+    expect(r.out).toContain('rule-16 rerun ignored: attempt 1: rerun results without a readable rerun manifest');
+    expect(r.flakes).toEqual([]);
+  });
+
+  test('a rerun result that does not match its manifest is set aside and the red stays', () => {
+    const r = report(withRerun(PASS, slice => { slice.sliceCount = 9; }));
+    expect(r.status, r.out).toBe(1);
+    expect(r.out).toContain('1 rerun result(s) do not match the rerun manifest');
+    expect(r.flakes).toEqual([]);
+  });
+
+  test('a rerun counts only for the cases its manifest planned', () => {
+    const dir = withRerun(PASS);
+    const planFile = path.join(dir, `${RERUN_PLAN_ARTIFACT}-a1`, 'manifest.json');
+    fs.writeFileSync(planFile, JSON.stringify({ ...JSON.parse(fs.readFileSync(planFile, 'utf8')), rerun: { attempt: 1, cases: ['some-other-case'] } }));
+    const r = report(dir);
+    expect(r.status, r.out).toBe(1);
+    expect(r.flakes).toEqual([]);
+  });
+});
+
+describe('case history resolution (pure)', () => {
+  const o = (attempt: number, phase: 'first' | 'rerun', outcome: 'passed' | 'failed', extra: Partial<CaseObservation> = {}): CaseObservation => ({
+    case: 'c', kind: 'rule', attempt, phase, outcome, eligible: outcome === 'failed', keys: ['test/x.test.ts'], where: `a${attempt} ${phase}`,
+    costUsd: 0.5, costKnown: true, ...extra });
+
+  test('flaky only when the one failure is eligible and the very next observation passed', () => {
+    expect(resolveCases([o(1, 'first', 'failed'), o(1, 'rerun', 'passed')]).get('c')!.status).toBe('flaky');
+    expect(resolveCases([o(1, 'first', 'failed'), o(1, 'rerun', 'failed')]).get('c')!.status).toBe('red');
+    expect(resolveCases([o(1, 'first', 'failed', { eligible: false, reason: 'INFRA failure class' }), o(2, 'first', 'passed')]).get('c')!.reason).toBe('never rerun: INFRA failure class');
+    expect(resolveCases([o(1, 'first', 'failed')]).get('c')!.reason).toBe('no rerun verdict');
+    expect(resolveCases([o(1, 'first', 'passed')]).has('c')).toBe(false);
+    expect(resolveCases([o(1, 'first', 'passed'), o(2, 'first', 'failed')]).get('c')!.status).toBe('red');
+  });
+
+  test('candidates: only an eligible single failure nothing has re-executed, including an earlier attempt never reached', () => {
+    expect(rerunCandidates([o(1, 'first', 'failed')])).toHaveLength(1);
+    expect(rerunCandidates([o(1, 'first', 'failed'), o(2, 'first', 'passed')])).toEqual([]);
+    expect(rerunCandidates([o(1, 'first', 'failed'), o(1, 'rerun', 'failed')])).toEqual([]);
+    expect(rerunCandidates([o(1, 'first', 'failed', { eligible: false, reason: 'runner error' })])).toEqual([]);
+    expect(rerunCandidates([o(1, 'first', 'passed')])).toEqual([]);
+  });
+
+  test('rerun spend counts every execution after a case first failed', () => {
+    expect(rerunSpend([o(1, 'first', 'failed'), o(1, 'rerun', 'passed', { costKnown: false })])).toEqual({ executions: 1, usd: 0.5, unknown: 1 });
+    expect(rerunSpend([o(1, 'first', 'passed')])).toEqual({ executions: 0, usd: 0, unknown: 0 });
+  });
+});
+
+describe('gate slice exit (case verdicts)', () => {
+  const ran = { status: 'failed' as const, exitCode: 1, executedTests: 2, skippedTests: 0 };
+  test('a completed rule shard red is report input on the gate lane, so the run concludes as the report does', () => {
+    expect(sliceExitCode([ran], { caseVerdicts: true })).toBe(0);
+    expect(sliceExitCode([ran])).toBe(1);
+  });
+  test('execution failures still red the slice', () => {
+    for (const bad of [{ ...ran, runnerError: 'x' }, { ...ran, sliceDeadline: 'hung' as const }, { ...ran, executedTests: null }, { ...ran, skippedTests: 2 },
+      { ...ran, exitCode: null }, { ...ran, status: 'timed-out' as const }, { status: 'never-started' as const, exitCode: null, executedTests: null, skippedTests: null }]) {
+      expect(sliceExitCode([bad], { caseVerdicts: true }), JSON.stringify(bad)).toBe(1);
+    }
+  });
+});

test/train-gate.test.ts

@@ -158,6 +158,19 @@ describe('train record: the census comes from the run\'s own manifest, first att
     expect(record.panels.find(p => p.case === j1)).toMatchObject({ status: 'FAIL', red_class: 'VERDICT' });
     expect(classify(record)).toBe('RED');
   });
+
+  test('rule 16: a case the report resolved FLAKY is a PASS naming both runs, and the train is GREEN; without the record it stays red', () => {
+    const outcomes = [{ case: j1, kind: 'judge', attempt: 1, outcome: 'failed', failure_class: 'assertion' },
+      { case: j2, kind: 'judge', attempt: 1, outcome: 'passed' }, ...[1, 2, 3].map(trial => ({ case: behavior, kind: 'behavior', trial, attempt: 1, outcome: 'passed' }))];
+    const flake = { schema: 1, kind: 'gate-flake', case: j1, case_kind: 'judge', first: { attempt: 1, phase: 'first', where: 'attempt 1 (paid-slice-1-a1)' },
+      rerun: { attempt: 1, phase: 'rerun', where: 'attempt 1 rerun (paid-rerun-1-a1)' } };
+    const record = build(report({ outcomes, collector: { verdict: { verdict: 'GREEN' }, census: { unattributed: [] }, panels: [], flaky: [flake] } }));
+    expect(record.panels.find(p => p.case === j1)).toEqual({ case: j1, kind: 'judge', status: 'PASS', red_class: null, cost_usd: null,
+      flaky: { first: 'attempt 1 (paid-slice-1-a1)', rerun: 'attempt 1 rerun (paid-rerun-1-a1)' } });
+    expect(record.verdict).toBe('GREEN');
+    expect(classify(record)).toBe('GREEN');
+    expect(build(report({ outcomes, collector: { verdict: { verdict: 'RED' }, census: { unattributed: [] }, panels: [] } })).verdict).toBe('RED');
+  });
 });
 
 describe('prefix trees modulo stamps', () => {
@@ -237,6 +250,24 @@ describe('bisect: one executor per prefix, one trial per case', () => {
     await expect(runBisectPrefix(plan, 1, { rootDir: scratch, evalDir: scratch, checkedOut: B, runCase: async () => ({ status: 'PASS', redClass: null }) })).rejects.toThrow('not its prefix');
     await expect(runBisectPrefix(plan, 3, { rootDir: scratch, evalDir: scratch, checkedOut: B })).rejects.toThrow('names no prefix');
   });
+
+  test('rule 16 in a bisect: a case verdict FAIL reruns once on the prefix; a pass is a flaky PASS, a second FAIL stays red, INFRA and INCOMPLETE never rerun', async () => {
+    const script = (verdicts: Record<string, Array<{ status: string; redClass: string | null }>>) => {
+      const calls: string[] = [];
+      return { calls, runCase: async (id: string, _kind: string, evalDir: string) => { calls.push(`${id} ${path.basename(evalDir)}`); return verdicts[id]!.shift()!; } };
+    };
+    const fail = { status: 'FAIL', redClass: 'VERDICT' }, pass = { status: 'PASS', redClass: null };
+    const flaky = script({ 'case-a': [fail, pass], 'case-b': [fail, fail] });
+    const record = await runBisectPrefix(plan, 1, { rootDir: scratch, evalDir: path.join(scratch, 'bisect-flaky'), checkedOut: A, runCase: flaky.runCase });
+    expect(record.panels).toEqual([
+      { case: 'case-a', kind: 'rule', status: 'PASS', red_class: null, cost_usd: null, flaky: { first: `prefix ${A.slice(0, 12)} slice 1`, rerun: `prefix ${A.slice(0, 12)} slice 1 rerun` } },
+      { case: 'case-b', kind: 'judge', status: 'FAIL', red_class: 'VERDICT', cost_usd: null }]);
+    expect(flaky.calls).toEqual(['case-a case-a', 'case-a case-a.rerun', 'case-b case-b', 'case-b case-b.rerun']);
+    const never = script({ 'case-a': [{ status: 'FAIL', redClass: 'INFRA' }], 'case-b': [{ status: 'INCOMPLETE', redClass: 'INCOMPLETE' }] });
+    const kept = await runBisectPrefix(plan, 1, { rootDir: scratch, evalDir: path.join(scratch, 'bisect-never'), checkedOut: A, runCase: never.runCase });
+    expect(kept.panels.map(p => [p.case, p.status, p.red_class])).toEqual([['case-a', 'FAIL', 'INFRA'], ['case-b', 'INCOMPLETE', 'INCOMPLETE']]);
+    expect(never.calls).toHaveLength(2);
+  });
 });
 
 test('the free screening summary says first how to request the screen, then the selection and its estimated cost', () => {