mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-16 21:31:58 +01:00
test(evals): a case red twice stays a real red for the report, the run, reuse and the train verdict (GSTA-49 F1)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
f1c0b30911
commit
a592321668
3 files changed
+67
-1
No files matched your search
@@ -296,6 +296,27 @@ describe('rule-16 rerun wiring (F1)', () => {
|
||||
expect(trainRun.jobs).toEqual(expect.arrayContaining(['rerun-plan', 'eval-rerun', 'slices-report', 'train-check']));
|
||||
});
|
||||
|
||||
test('a real red (reconcile exit 1, e.g. a case red twice) fails slices-report and so the run, after every rerun path', () => {
|
||||
// slices-report runs whether the slices and the rerun passed, failed or were skipped, and no job may swallow its failure.
|
||||
for (const results of [{}, { 'eval-slices': 'failure' }, { 'eval-rerun': 'failure' }, { 'eval-rerun': 'skipped' }] as Array<Record<string, string>>) {
|
||||
const sim = simulate(evals, { github: pr(), inputs: {}, outputs: { route: { ...ROUTE, mode: 'pr' }, 'rerun-plan': planned }, results });
|
||||
expect(sim.jobs, JSON.stringify(results)).toContain('slices-report');
|
||||
}
|
||||
for (const [id, job] of Object.entries(evals.jobs)) expect((job as Record<string, unknown>)['continue-on-error'], id).toBeUndefined();
|
||||
const report = evals.jobs['slices-report']!;
|
||||
const reconcile = report.steps.find(st => st.id === 'reconcile')!;
|
||||
expect(reconcile.run).toContain('echo "exit=${PIPESTATUS[0]}"');
|
||||
const fail = report.steps.at(-1)!;
|
||||
expect(fail.name).toBe('Fail the workflow when reconciliation failed');
|
||||
expect(fail.run).toBe('exit 1');
|
||||
const fires = (exit: string | null, reused: string | null = null) => truthy(evaluate(String(fail.if), { steps: { reconcile: { outputs: { exit } }, reused: { outputs: { exit: reused } } } }, { success: true, failure: false }));
|
||||
expect(fires('1')).toBe(true);
|
||||
expect(fires('0')).toBe(false);
|
||||
// A red report also withholds the train check (the train verdict reads the run's failure too).
|
||||
const trainRun = simulate(evals, { github: dispatch('train/t1'), inputs: { ...INPUTS, nonce: 'gate.train-x', expected_sha: SHA.head }, outputs: { route: { ...ROUTE, ...train, mode: 'train' }, 'rerun-plan': planned }, results: { 'slices-report': 'failure' } });
|
||||
expect(trainRun.jobs).not.toContain('train-check');
|
||||
});
|
||||
|
||||
test('a confirmed verdict reuse plans and runs no rerun', () => {
|
||||
const sim = simulate(evals, { github: pr(), inputs: {}, outputs: { route: { ...ROUTE, mode: 'pr' }, 'plan-slices': { gate_reuse: 'true' }, 'rerun-plan': planned } });
|
||||
expect(sim.jobs).not.toContain('rerun-plan');
|
||||
|
||||
@@ -14,6 +14,10 @@ import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { type CaseObservation, FLAKE_RECORDS_FILE, RERUN_PLAN_ARTIFACT, resolveCases, rerunCandidates, rerunSpend } from '../scripts/lib/paid-rerun';
|
||||
import { sliceExitCode } from '../scripts/lib/paid-plan';
|
||||
import { BINDING_FILE, buildBinding, findReusableGate } from '../scripts/gate-reuse';
|
||||
import type { GhRunner } from '../scripts/recover-receipts';
|
||||
import { buildTrainRecord } from '../scripts/train-gate';
|
||||
import { classifyGate } from '../lib/ship-queue-train-verdict';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const CASE = 'plan-ceo-finding-floor';
|
||||
@@ -158,6 +162,47 @@ describe('negative controls: rule 16 never turns a real red green', () => {
|
||||
expect(again.out).toContain(`not rerun: ${CASE} (failed twice`);
|
||||
});
|
||||
|
||||
test('red twice is a real red everywhere it is read: the report, the reuse binding and lookup, and the train verdict', () => {
|
||||
const dir = withRerun(FAIL);
|
||||
const r = report(dir);
|
||||
expect(r.status, r.out).toBe(1);
|
||||
expect(r.out).toContain('VERDICT RED');
|
||||
expect(r.collector.verdict.verdict).toBe('RED');
|
||||
// gate-reuse: slices-report binds what the report proved, and find refuses it (evals.yml also fails the run on reconcile exit 1).
|
||||
const head = 'a'.repeat(40);
|
||||
const { binding } = buildBinding({ repository: 'garrytan/gstack', pr: 42, runId: 300, runAttempt: 1, headSha: head, baseSha: 'b'.repeat(40), profile: 'pr', reconcileExit: r.status!, reportDir: dir });
|
||||
expect(binding).toMatchObject({ verdict: 'RED', reconcile_exit: 1 });
|
||||
expect(binding!.flaky_cases).toBeUndefined();
|
||||
const gh: GhRunner = (args) => {
|
||||
const ok = (body: unknown) => ({ status: 0, stdout: JSON.stringify(body), stderr: '' });
|
||||
if (args.some(a => a.endsWith('/workflows/evals.yml/runs'))) {
|
||||
return ok({ workflow_runs: [{ id: 300, status: 'completed', conclusion: 'failure', event: 'pull_request', head_sha: head, run_attempt: 1,
|
||||
head_repository: { full_name: 'garrytan/gstack' }, pull_requests: [{ number: 42 }] }] });
|
||||
}
|
||||
if (args.some(a => /\/runs\/300\/artifacts$/.test(a))) return ok({ artifacts: [{ name: 'gate-binding-a1', expired: false }, { name: 'paid-slice-1-a1', expired: false }] });
|
||||
if (args[0] === 'run' && args[1] === 'download') {
|
||||
const out = args[args.indexOf('-D') + 1]!;
|
||||
fs.mkdirSync(out, { recursive: true });
|
||||
fs.writeFileSync(path.join(out, BINDING_FILE), JSON.stringify(binding));
|
||||
return { status: 0, stdout: '', stderr: '' };
|
||||
}
|
||||
return { status: 1, stdout: '', stderr: `unexpected gh ${args.join(' ')}` };
|
||||
};
|
||||
const found = findReusableGate({ repo: 'garrytan/gstack', pr: 42, headRepo: 'garrytan/gstack', branch: 'feature', selfRunId: 500, headSha: 'c'.repeat(40),
|
||||
testedSha: 'd'.repeat(40), repoRoot: dir, out: path.join(dir, 'reuse-out'), gh });
|
||||
expect(found.candidate).toBeNull();
|
||||
expect(found.lines.join('\n')).toContain('is RED, reconcile exit 1, conclusion failure');
|
||||
// The train verdict: its record is RED with the case a VERDICT red, and T1 classifies it RED whatever the run concluded.
|
||||
const record = buildTrainRecord({ reportDir: dir, nonce: 'gate.train-x', head, tree: 'e'.repeat(40), reconcileExit: r.status!,
|
||||
prefixes: [], run: { id: 300, attempt: 1, url: 'https://example.invalid/runs/300' } });
|
||||
expect(record.verdict).toBe('RED');
|
||||
expect(record.panels.find(p => p.case === CASE)).toMatchObject({ status: 'FAIL', red_class: 'VERDICT' });
|
||||
expect(record.panels.find(p => p.case === CASE)!.flaky).toBeUndefined();
|
||||
for (const conclusion of ['failure', 'success']) {
|
||||
expect(classifyGate(record, { nonce: 'gate.train-x', head, tree: 'e'.repeat(40) }, { status: 'completed', conclusion }, { status: 'completed', conclusion: 'success' } as never).outcome).toBe('RED');
|
||||
}
|
||||
});
|
||||
|
||||
test('a manual attempt after a failed rerun never makes it flaky: two failures stay RED', () => {
|
||||
const dir = withRerun(FAIL);
|
||||
place(dir, { from: PASS, to: PASS, attempt: 2 });
|
||||
|
||||
Reference in new issue
Block a user