From a592321668e26dcce54c1ca801d1e5eb2ba9b4aa Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 20 Oct 2026 03:19:36 +0000 Subject: [PATCH] test(evals): a case red twice stays a real red for the report, the run, reuse and the train verdict (GSTA-49 F1) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 2 +- test/evals-train-workflow.test.ts | 21 +++++++++++++++ test/paid-rerun.test.ts | 45 +++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1831bafe0..734cb78dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ - **Flaky GREENs are reusable (GSTA-53).** `gate-reuse` binds a flaky GREEN with its `flaky_cases`, reads each run's newest attempt binding, and R2a and R2 reuse it with `reused_flaky` named in the log and PR comment. On #3308 (run 39315041557), both attempts bound RED because the report kept attempt 1's verdict, so the next push paid for a full gate. A run still red after its rerun binds RED and is never reused. ### Changed -- **Gate integrity (a gate edit):** harness, INFRA, INCOMPLETE and unattributed reds never rerun and are never flaky. Neither is a case already red twice, or a case whose rerun could not turn the gate green. No threshold, trial count, case kind, judge model, `EVAL_POLICY` constant or `HARNESS_VERSION` changed (recorded non-behavioral). Pass-rate history still records each run's first execution unchanged; a rerun's trials stay in its `paid-rerun-*` artifacts and the flake record. On the gate lane, a rule shard that ran to completion with a red case no longer fails its slice job. The report still fails closed on it, so the run's conclusion follows the report and a flaky GREEN run concludes `success`. Execution failures (timeouts, runner errors, hollow or unfinished shards) still fail the slice. Train gates take the same rerun path. A bisect reruns a VERDICT FAIL once on its prefix (`scripts/train-gate.ts`), and a pass there is a flaky PASS naming both runs. The PR comment's verified-summary check also bounds `flakyCollectorFailures`, so a failed collector record that is not a flaky case still fails the comment. +- **Gate integrity (a gate edit):** harness, INFRA, INCOMPLETE and unattributed reds never rerun and are never flaky. Neither is a case already red twice, or a case whose rerun could not turn the gate green. No threshold, trial count, case kind, judge model, `EVAL_POLICY` constant or `HARNESS_VERSION` changed (recorded non-behavioral). Pass-rate history still records each run's first execution unchanged; a rerun's trials stay in its `paid-rerun-*` artifacts and the flake record. On the gate lane, a rule shard that ran to completion with a red case no longer fails its slice job. The report still fails closed on it, so the run's conclusion follows the report and a flaky GREEN run concludes `success`. Execution failures (timeouts, runner errors, hollow or unfinished shards) still fail the slice. Train gates take the same rerun path. A bisect reruns a VERDICT FAIL once on its prefix (`scripts/train-gate.ts`), and a pass there is a flaky PASS naming both runs. The PR comment's verified-summary check also bounds `flakyCollectorFailures`, so a failed collector record that is not a flaky case still fails the comment. Negative control for the slice-exit change: a case red on its first run and red again on its rerun keeps the real report CLI at exit 1 and `VERDICT RED`; `slices-report` still runs after every rerun path and its last step fails the job on reconcile exit 1, no job carries `continue-on-error`, so the run concludes `failure`; the binding is RED and `gate-reuse` refuses it; and the train record is RED, which T1 classifies RED whatever the run concluded (`test/paid-rerun.test.ts`, `test/evals-train-workflow.test.ts`). - **Free regressions:** `test/paid-rerun.test.ts` replays a trimmed capture of run 39293281037 through the real `--report` and `--rerun-plan` CLI. Its negative controls cover red twice, harness, INCOMPLETE and checkpoint reds, a new red on a later attempt, a first-attempt pass, rerun results with a missing or mismatched manifest, and reruns of unplanned cases. `test/gate-reuse.test.ts` covers R2a and R2 reuse of a flaky GREEN, a manual-attempt binding, and a run still red after its rerun. `test/train-gate.test.ts` covers the train record and the bisect rerun. Workflow wiring tests are in `test/evals-train-workflow.test.ts` and `test/ci-paid-coordination.test.ts`. ## [1.91.162.0] - 2026-10-19 diff --git a/test/evals-train-workflow.test.ts b/test/evals-train-workflow.test.ts index e9ce73254..5a7494826 100644 --- a/test/evals-train-workflow.test.ts +++ b/test/evals-train-workflow.test.ts @@ -296,6 +296,27 @@ describe('rule-16 rerun wiring (F1)', () => { expect(trainRun.jobs).toEqual(expect.arrayContaining(['rerun-plan', 'eval-rerun', 'slices-report', 'train-check'])); }); + test('a real red (reconcile exit 1, e.g. a case red twice) fails slices-report and so the run, after every rerun path', () => { + // slices-report runs whether the slices and the rerun passed, failed or were skipped, and no job may swallow its failure. + for (const results of [{}, { 'eval-slices': 'failure' }, { 'eval-rerun': 'failure' }, { 'eval-rerun': 'skipped' }] as Array>) { + const sim = simulate(evals, { github: pr(), inputs: {}, outputs: { route: { ...ROUTE, mode: 'pr' }, 'rerun-plan': planned }, results }); + expect(sim.jobs, JSON.stringify(results)).toContain('slices-report'); + } + for (const [id, job] of Object.entries(evals.jobs)) expect((job as Record)['continue-on-error'], id).toBeUndefined(); + const report = evals.jobs['slices-report']!; + const reconcile = report.steps.find(st => st.id === 'reconcile')!; + expect(reconcile.run).toContain('echo "exit=${PIPESTATUS[0]}"'); + const fail = report.steps.at(-1)!; + expect(fail.name).toBe('Fail the workflow when reconciliation failed'); + expect(fail.run).toBe('exit 1'); + const fires = (exit: string | null, reused: string | null = null) => truthy(evaluate(String(fail.if), { steps: { reconcile: { outputs: { exit } }, reused: { outputs: { exit: reused } } } }, { success: true, failure: false })); + expect(fires('1')).toBe(true); + expect(fires('0')).toBe(false); + // A red report also withholds the train check (the train verdict reads the run's failure too). + const trainRun = simulate(evals, { github: dispatch('train/t1'), inputs: { ...INPUTS, nonce: 'gate.train-x', expected_sha: SHA.head }, outputs: { route: { ...ROUTE, ...train, mode: 'train' }, 'rerun-plan': planned }, results: { 'slices-report': 'failure' } }); + expect(trainRun.jobs).not.toContain('train-check'); + }); + test('a confirmed verdict reuse plans and runs no rerun', () => { const sim = simulate(evals, { github: pr(), inputs: {}, outputs: { route: { ...ROUTE, mode: 'pr' }, 'plan-slices': { gate_reuse: 'true' }, 'rerun-plan': planned } }); expect(sim.jobs).not.toContain('rerun-plan'); diff --git a/test/paid-rerun.test.ts b/test/paid-rerun.test.ts index 64be4c21b..5c7a54cc0 100644 --- a/test/paid-rerun.test.ts +++ b/test/paid-rerun.test.ts @@ -14,6 +14,10 @@ import * as path from 'node:path'; import { spawnSync } from 'node:child_process'; import { type CaseObservation, FLAKE_RECORDS_FILE, RERUN_PLAN_ARTIFACT, resolveCases, rerunCandidates, rerunSpend } from '../scripts/lib/paid-rerun'; import { sliceExitCode } from '../scripts/lib/paid-plan'; +import { BINDING_FILE, buildBinding, findReusableGate } from '../scripts/gate-reuse'; +import type { GhRunner } from '../scripts/recover-receipts'; +import { buildTrainRecord } from '../scripts/train-gate'; +import { classifyGate } from '../lib/ship-queue-train-verdict'; const ROOT = path.resolve(import.meta.dir, '..'); const CASE = 'plan-ceo-finding-floor'; @@ -158,6 +162,47 @@ describe('negative controls: rule 16 never turns a real red green', () => { expect(again.out).toContain(`not rerun: ${CASE} (failed twice`); }); + test('red twice is a real red everywhere it is read: the report, the reuse binding and lookup, and the train verdict', () => { + const dir = withRerun(FAIL); + const r = report(dir); + expect(r.status, r.out).toBe(1); + expect(r.out).toContain('VERDICT RED'); + expect(r.collector.verdict.verdict).toBe('RED'); + // gate-reuse: slices-report binds what the report proved, and find refuses it (evals.yml also fails the run on reconcile exit 1). + const head = 'a'.repeat(40); + const { binding } = buildBinding({ repository: 'garrytan/gstack', pr: 42, runId: 300, runAttempt: 1, headSha: head, baseSha: 'b'.repeat(40), profile: 'pr', reconcileExit: r.status!, reportDir: dir }); + expect(binding).toMatchObject({ verdict: 'RED', reconcile_exit: 1 }); + expect(binding!.flaky_cases).toBeUndefined(); + const gh: GhRunner = (args) => { + const ok = (body: unknown) => ({ status: 0, stdout: JSON.stringify(body), stderr: '' }); + if (args.some(a => a.endsWith('/workflows/evals.yml/runs'))) { + return ok({ workflow_runs: [{ id: 300, status: 'completed', conclusion: 'failure', event: 'pull_request', head_sha: head, run_attempt: 1, + head_repository: { full_name: 'garrytan/gstack' }, pull_requests: [{ number: 42 }] }] }); + } + if (args.some(a => /\/runs\/300\/artifacts$/.test(a))) return ok({ artifacts: [{ name: 'gate-binding-a1', expired: false }, { name: 'paid-slice-1-a1', expired: false }] }); + if (args[0] === 'run' && args[1] === 'download') { + const out = args[args.indexOf('-D') + 1]!; + fs.mkdirSync(out, { recursive: true }); + fs.writeFileSync(path.join(out, BINDING_FILE), JSON.stringify(binding)); + return { status: 0, stdout: '', stderr: '' }; + } + return { status: 1, stdout: '', stderr: `unexpected gh ${args.join(' ')}` }; + }; + const found = findReusableGate({ repo: 'garrytan/gstack', pr: 42, headRepo: 'garrytan/gstack', branch: 'feature', selfRunId: 500, headSha: 'c'.repeat(40), + testedSha: 'd'.repeat(40), repoRoot: dir, out: path.join(dir, 'reuse-out'), gh }); + expect(found.candidate).toBeNull(); + expect(found.lines.join('\n')).toContain('is RED, reconcile exit 1, conclusion failure'); + // The train verdict: its record is RED with the case a VERDICT red, and T1 classifies it RED whatever the run concluded. + const record = buildTrainRecord({ reportDir: dir, nonce: 'gate.train-x', head, tree: 'e'.repeat(40), reconcileExit: r.status!, + prefixes: [], run: { id: 300, attempt: 1, url: 'https://example.invalid/runs/300' } }); + expect(record.verdict).toBe('RED'); + expect(record.panels.find(p => p.case === CASE)).toMatchObject({ status: 'FAIL', red_class: 'VERDICT' }); + expect(record.panels.find(p => p.case === CASE)!.flaky).toBeUndefined(); + for (const conclusion of ['failure', 'success']) { + expect(classifyGate(record, { nonce: 'gate.train-x', head, tree: 'e'.repeat(40) }, { status: 'completed', conclusion }, { status: 'completed', conclusion: 'success' } as never).outcome).toBe('RED'); + } + }); + test('a manual attempt after a failed rerun never makes it flaky: two failures stay RED', () => { const dir = withRerun(FAIL); place(dir, { from: PASS, to: PASS, attempt: 2 });