fix(evals): keep a lone rerun-plan artifact off the plan manifest (run 39363705692)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-20 04:00:49 +00:00
1 parent a592321668
commit 3f91ff3caa
6 files changed
+62 -17

No files matched your search

+1 -1
View File
@@ -167,7 +167,7 @@ describe('paid CI coordination stays off the eval image', () => {
if (name === 'evals.yml') expect(String(bisectLogs?.with?.path)).toBe(String(logs?.with?.path));
// F1: the rule-16 eval-rerun keeps the same shard-log upload under its own name.
const rerunLogs = name === 'evals.yml'
? jobs['eval-rerun'].steps.find(step => step.with?.name === 'paid-rerun-logs-${{ matrix.slice }}-a${{ github.run_attempt }}') : undefined;
? jobs['eval-rerun'].steps.find(step => step.with?.name === 'rerun-logs-${{ matrix.slice }}-a${{ github.run_attempt }}') : undefined;
if (name === 'evals.yml') expect(String(rerunLogs?.with?.path)).toBe(String(logs?.with?.path));
expect(hiddenUploads.filter(step => !captures.includes(step))).toEqual([logs!, ...(codexLogs ? [codexLogs] : []), ...(rerunLogs ? [rerunLogs] : []), ...(bisectLogs ? [bisectLogs] : [])]);
});
+19 -4
View File
@@ -327,16 +327,31 @@ describe('rule-16 rerun wiring (F1)', () => {
test('the rerun runs its own attempt-scoped manifest fresh: no receipts in, none out, results under paid-rerun-*', () => {
const job = evals.jobs['eval-rerun']!;
expect(job.strategy!.matrix.slice).toBe('${{ fromJSON(needs.rerun-plan.outputs.slices) }}');
expect(job.steps.find(st => st.uses?.startsWith('actions/download-artifact@'))!.with).toEqual({ name: 'rerun-plan-a${{ github.run_attempt }}', path: '/tmp/paid-plan' });
expect(job.steps.find(st => st.uses?.startsWith('actions/download-artifact@'))!.with).toEqual({ name: 'rerun-plan-a${{ github.run_attempt }}', path: '/tmp/paid-rerun-plan' });
const run = job.steps.find(st => st.name === 'Rerun slice ${{ matrix.slice }} (rule 16, once)')!;
expect(run.run).toContain('--plan /tmp/paid-plan/manifest.json --slice ${{ matrix.slice }}');
expect(run.run).toContain('--plan /tmp/paid-rerun-plan/rerun-plan-a${{ github.run_attempt }}/manifest.json --slice ${{ matrix.slice }}');
expect(Object.keys(run.env!).filter(key => key.startsWith('EVALS_CACHE'))).toEqual([]);
expect(JSON.stringify(job)).not.toContain('receipts');
expect(job.steps.find(st => st.name === 'Upload rerun results')!.with).toMatchObject({ name: 'paid-rerun-${{ matrix.slice }}-a${{ github.run_attempt }}', path: '/tmp/paid-rerun-results' });
expect(job.steps.find(st => st.name === 'Upload rerun results')!.with).toMatchObject({ name: 'paid-rerun-${{ matrix.slice }}-a${{ github.run_attempt }}', path: run.env!.GSTACK_EVAL_DIR });
const uploads = Object.values(evals.jobs).flatMap(j => j.steps).map(st => String(st.with?.name ?? '')).filter(Boolean);
expect(uploads.filter(name => name.startsWith('paid-rerun-'))).toEqual(['paid-rerun-${{ matrix.slice }}-a${{ github.run_attempt }}']);
const plan = evals.jobs['rerun-plan']!;
expect(plan.permissions).toEqual({ contents: 'read' });
expect(JSON.stringify(plan)).not.toMatch(/secrets\.|restore-deps|bun install/);
expect(plan.steps.find(st => st.name === "Plan the rule-16 rerun of this attempt's failed cases")!.run!.replace(/\s*\\\n\s*/g, ' ')).toContain('--report /tmp/paid-report --rerun-plan /tmp/paid-rerun');
expect(plan.steps.find(st => st.name === "Plan the rule-16 rerun of this attempt's failed cases")!.run!.replace(/\s*\\\n\s*/g, ' '))
.toContain('--report /tmp/paid-report --rerun-plan "/tmp/paid-rerun-plan/rerun-plan-a$GITHUB_RUN_ATTEMPT"');
expect(plan.steps.find(st => st.name === 'Hand the rerun manifest to its executors and the report')!.with).toMatchObject({ name: 'rerun-plan-a${{ github.run_attempt }}', path: '/tmp/paid-rerun-plan' });
});
test('a lone rerun artifact never unpacks over the report root or the plan manifest (run 39363705692 did)', () => {
for (const id of ['rerun-plan', 'slices-report']) {
const downloads = evals.jobs[id]!.steps.filter(st => st.uses?.startsWith('actions/download-artifact@') && /^(paid-rerun|rerun-plan)-/.test(String(st.with?.pattern ?? '')));
expect(downloads.map(st => st.with), id).toEqual([{ pattern: 'paid-rerun-*', path: '/tmp/paid-report/reruns' }, { pattern: 'rerun-plan-a*', path: '/tmp/paid-report' }]);
}
// The plan artifact's root is a directory holding rerun-plan-a<attempt>/manifest.json, written there by the planner.
const plan = evals.jobs['rerun-plan']!;
expect(plan.steps.find(st => st.name === 'Hand the rerun manifest to its executors and the report')!.with!.path).toBe('/tmp/paid-rerun-plan');
expect(plan.steps.find(st => st.name === "Plan the rule-16 rerun of this attempt's failed cases")!.run).toContain('PLAN="/tmp/paid-rerun-plan/rerun-plan-a$GITHUB_RUN_ATTEMPT/manifest.json"');
});
test('the report reads every attempt and rerun, publishes flake records, and the comment never hides a non-flaky failed record', () => {
+17
View File
@@ -250,6 +250,23 @@ describe('negative controls: rule 16 never turns a real red green', () => {
expect(r.flakes).toEqual([]);
});
test('the rerun reads the same in both download-artifact layouts: a lone match unpacked into its path, or each in its own directory', () => {
const lone = withRerun(PASS);
fs.mkdirSync(path.join(lone, 'reruns'));
fs.renameSync(path.join(lone, 'paid-rerun-1-a1'), path.join(lone, 'reruns', 'only'));
const several = withRerun(PASS);
const planDir = path.join(several, `${RERUN_PLAN_ARTIFACT}-a1`);
fs.renameSync(planDir, `${planDir}.tmp`);
fs.mkdirSync(planDir);
fs.renameSync(`${planDir}.tmp`, path.join(planDir, `${RERUN_PLAN_ARTIFACT}-a1`));
for (const dir of [lone, several]) {
const r = report(dir);
expect(r.status, r.out).toBe(0);
expect(r.out).toContain('FLAKY 1');
expect(r.flakes.map((f: any) => f.case)).toEqual([CASE]);
}
});
test('a rerun result that does not match its manifest is set aside and the red stays', () => {
const r = report(withRerun(PASS, slice => { slice.sliceCount = 9; }));
expect(r.status, r.out).toBe(1);