fix(evals): keep a lone rerun-plan artifact off the plan manifest (run 39363705692)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

.github/workflows/evals.yml

@@ -832,23 +832,28 @@ jobs:
         continue-on-error: true
         with:
           pattern: paid-rerun-*
-          path: /tmp/paid-report
+          # Its own directory: download-artifact unpacks a lone match into its path itself, so the
+          # report root would otherwise take a rerun slice's files (run 39363705692).
+          path: /tmp/paid-report/reruns
 
       - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
         continue-on-error: true
         with:
           pattern: rerun-plan-a*
+          # Each artifact holds rerun-plan-a<attempt>/manifest.json, so even a lone match never
+          # lands on the plan's own manifest.json (run 39363705692).
           path: /tmp/paid-report
 
       - name: Plan the rule-16 rerun of this attempt's failed cases
         id: plan
         run: |
           EVALS_TIER=gate bun --no-env-file --config=/dev/null --no-install run scripts/test-paid-shards.ts --tier gate --report /tmp/paid-report \
-            --rerun-plan /tmp/paid-rerun >> "$GITHUB_STEP_SUMMARY"
-          if [ -f /tmp/paid-rerun/manifest.json ]; then
+            --rerun-plan "/tmp/paid-rerun-plan/rerun-plan-a$GITHUB_RUN_ATTEMPT" >> "$GITHUB_STEP_SUMMARY"
+          PLAN="/tmp/paid-rerun-plan/rerun-plan-a$GITHUB_RUN_ATTEMPT/manifest.json"
+          if [ -f "$PLAN" ]; then
             { echo "planned=true"
-              echo "slices=$(jq -c '[range(1; .sliceCount + 1)]' /tmp/paid-rerun/manifest.json)"
-              echo "slice_timeouts=$(jq -ec '[0] + .plan.sliceCiTimeoutMinutes' /tmp/paid-rerun/manifest.json)"; } >> "$GITHUB_OUTPUT"
+              echo "slices=$(jq -c '[range(1; .sliceCount + 1)]' "$PLAN")"
+              echo "slice_timeouts=$(jq -ec '[0] + .plan.sliceCiTimeoutMinutes' "$PLAN")"; } >> "$GITHUB_OUTPUT"
           else
             echo "planned=false" >> "$GITHUB_OUTPUT"
           fi
@@ -858,7 +863,7 @@ jobs:
         uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
         with:
           name: rerun-plan-a${{ github.run_attempt }}
-          path: /tmp/paid-rerun/manifest.json
+          path: /tmp/paid-rerun-plan
           retention-days: 90
 
   eval-rerun:
@@ -911,7 +916,7 @@ jobs:
       - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
         with:
           name: rerun-plan-a${{ github.run_attempt }}
-          path: /tmp/paid-plan
+          path: /tmp/paid-rerun-plan
 
       - name: Authenticate the Codex CLI
         shell: bash
@@ -941,7 +946,7 @@ jobs:
           EVALS_JOBS: "2"
           EVALS_CONCURRENCY: "2"
           GSTACK_EVAL_DIR: /tmp/paid-rerun-results
-        run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/paid-plan/manifest.json --slice ${{ matrix.slice }}
+        run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/paid-rerun-plan/rerun-plan-a${{ github.run_attempt }}/manifest.json --slice ${{ matrix.slice }}
 
       - name: Upload rerun results
         if: always()
@@ -969,7 +974,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
         with:
-          name: paid-rerun-logs-${{ matrix.slice }}-a${{ github.run_attempt }}
+          name: rerun-logs-${{ matrix.slice }}-a${{ github.run_attempt }}
           include-hidden-files: true
           path: |
             /home/runner/.cache/gstack-paid-shard-*.log
@@ -1178,12 +1183,16 @@ jobs:
         continue-on-error: true
         with:
           pattern: paid-rerun-*
-          path: /tmp/paid-report
+          # Its own directory: download-artifact unpacks a lone match into its path itself, so the
+          # report root would otherwise take a rerun slice's files (run 39363705692).
+          path: /tmp/paid-report/reruns
 
       - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
         continue-on-error: true
         with:
           pattern: rerun-plan-a*
+          # Each artifact holds rerun-plan-a<attempt>/manifest.json, so even a lone match never
+          # lands on the plan's own manifest.json (run 39363705692).
           path: /tmp/paid-report
 
       - name: Reconcile slices against the manifest (fail-closed)

CHANGELOG.md

@@ -12,6 +12,7 @@
 ### Changed
 - **Gate integrity (a gate edit):** harness, INFRA, INCOMPLETE and unattributed reds never rerun and are never flaky. Neither is a case already red twice, or a case whose rerun could not turn the gate green. No threshold, trial count, case kind, judge model, `EVAL_POLICY` constant or `HARNESS_VERSION` changed (recorded non-behavioral). Pass-rate history still records each run's first execution unchanged; a rerun's trials stay in its `paid-rerun-*` artifacts and the flake record. On the gate lane, a rule shard that ran to completion with a red case no longer fails its slice job. The report still fails closed on it, so the run's conclusion follows the report and a flaky GREEN run concludes `success`. Execution failures (timeouts, runner errors, hollow or unfinished shards) still fail the slice. Train gates take the same rerun path. A bisect reruns a VERDICT FAIL once on its prefix (`scripts/train-gate.ts`), and a pass there is a flaky PASS naming both runs. The PR comment's verified-summary check also bounds `flakyCollectorFailures`, so a failed collector record that is not a flaky case still fails the comment. Negative control for the slice-exit change: a case red on its first run and red again on its rerun keeps the real report CLI at exit 1 and `VERDICT RED`; `slices-report` still runs after every rerun path and its last step fails the job on reconcile exit 1, no job carries `continue-on-error`, so the run concludes `failure`; the binding is RED and `gate-reuse` refuses it; and the train record is RED, which T1 classifies RED whatever the run concluded (`test/paid-rerun.test.ts`, `test/evals-train-workflow.test.ts`).
 - **Free regressions:** `test/paid-rerun.test.ts` replays a trimmed capture of run 39293281037 through the real `--report` and `--rerun-plan` CLI. Its negative controls cover red twice, harness, INCOMPLETE and checkpoint reds, a new red on a later attempt, a first-attempt pass, rerun results with a missing or mismatched manifest, and reruns of unplanned cases. `test/gate-reuse.test.ts` covers R2a and R2 reuse of a flaky GREEN, a manual-attempt binding, and a run still red after its rerun. `test/train-gate.test.ts` covers the train record and the bisect rerun. Workflow wiring tests are in `test/evals-train-workflow.test.ts` and `test/ci-paid-coordination.test.ts`.
+- **Rerun artifact layout:** the first gate of this change (run 39363705692) reran its one red case and passed, but `download-artifact` unpacked the lone `rerun-plan-a1` match into the report root, over the plan's own `manifest.json`, so the report read every case as unplanned and stayed red. The plan artifact now holds `rerun-plan-a<attempt>/manifest.json`, rerun results download under `reruns/`, the rerun logs artifact is `rerun-logs-*` so it no longer matches `paid-rerun-*`, and the report finds the rerun manifest in either layout `download-artifact` produces. That run's artifacts, replayed in the new layout, report `VERDICT GREEN` with `FLAKY 1`.
 
 ## [1.91.162.0] - 2026-10-19
 

scripts/lib/paid-rerun.ts

@@ -37,7 +37,7 @@ import { expectedPrCaseCount, shardSlug } from './paid-select';
 import { ROOT } from './paid-types';
 
 export const FLAKE_RECORDS_FILE = 'flake-records.jsonl';
-/** Each attempt's rerun manifest lands in the report dir as `<RERUN_PLAN_ARTIFACT>-a<attempt>/manifest.json`. */
+/** Each attempt's rerun manifest lands in the report dir as `<RERUN_PLAN_ARTIFACT>-a<attempt>/manifest.json` (one level deeper when several attempts' plans download). */
 export const RERUN_PLAN_ARTIFACT = 'rerun-plan';
 
 export type Phase = 'first' | 'rerun';
@@ -185,7 +185,10 @@ export function observationSets(reportDir: string, manifest: PaidRunManifest, fi
   const reruns = loadSliceArtifacts(reportDir, { reruns: true });
   for (const attempt of [...new Set(reruns.map(a => a.result.attempt ?? 1))].sort((a, b) => a - b)) {
     let rerunManifest: PaidRunManifest;
-    try { rerunManifest = parseRunManifest(fs.readFileSync(path.join(reportDir, `${RERUN_PLAN_ARTIFACT}-a${attempt}`, 'manifest.json'), 'utf8')); } catch (error) {
+    // download-artifact unpacks a lone pattern match into the report dir and each of several into its own directory.
+    const dir = path.join(reportDir, `${RERUN_PLAN_ARTIFACT}-a${attempt}`);
+    const file = [path.join(dir, 'manifest.json'), path.join(dir, path.basename(dir), 'manifest.json')].find(candidate => fs.existsSync(candidate)) ?? path.join(dir, 'manifest.json');
+    try { rerunManifest = parseRunManifest(fs.readFileSync(file, 'utf8')); } catch (error) {
       ignored.push(`attempt ${attempt}: rerun results without a readable rerun manifest (${error instanceof Error ? error.message : String(error)})`);
       continue;
     }

test/ci-paid-coordination.test.ts

@@ -167,7 +167,7 @@ describe('paid CI coordination stays off the eval image', () => {
       if (name === 'evals.yml') expect(String(bisectLogs?.with?.path)).toBe(String(logs?.with?.path));
       // F1: the rule-16 eval-rerun keeps the same shard-log upload under its own name.
       const rerunLogs = name === 'evals.yml'
-        ? jobs['eval-rerun'].steps.find(step => step.with?.name === 'paid-rerun-logs-${{ matrix.slice }}-a${{ github.run_attempt }}') : undefined;
+        ? jobs['eval-rerun'].steps.find(step => step.with?.name === 'rerun-logs-${{ matrix.slice }}-a${{ github.run_attempt }}') : undefined;
       if (name === 'evals.yml') expect(String(rerunLogs?.with?.path)).toBe(String(logs?.with?.path));
       expect(hiddenUploads.filter(step => !captures.includes(step))).toEqual([logs!, ...(codexLogs ? [codexLogs] : []), ...(rerunLogs ? [rerunLogs] : []), ...(bisectLogs ? [bisectLogs] : [])]);
     });

test/evals-train-workflow.test.ts

@@ -327,16 +327,31 @@ describe('rule-16 rerun wiring (F1)', () => {
   test('the rerun runs its own attempt-scoped manifest fresh: no receipts in, none out, results under paid-rerun-*', () => {
     const job = evals.jobs['eval-rerun']!;
     expect(job.strategy!.matrix.slice).toBe('${{ fromJSON(needs.rerun-plan.outputs.slices) }}');
-    expect(job.steps.find(st => st.uses?.startsWith('actions/download-artifact@'))!.with).toEqual({ name: 'rerun-plan-a${{ github.run_attempt }}', path: '/tmp/paid-plan' });
+    expect(job.steps.find(st => st.uses?.startsWith('actions/download-artifact@'))!.with).toEqual({ name: 'rerun-plan-a${{ github.run_attempt }}', path: '/tmp/paid-rerun-plan' });
     const run = job.steps.find(st => st.name === 'Rerun slice ${{ matrix.slice }} (rule 16, once)')!;
-    expect(run.run).toContain('--plan /tmp/paid-plan/manifest.json --slice ${{ matrix.slice }}');
+    expect(run.run).toContain('--plan /tmp/paid-rerun-plan/rerun-plan-a${{ github.run_attempt }}/manifest.json --slice ${{ matrix.slice }}');
     expect(Object.keys(run.env!).filter(key => key.startsWith('EVALS_CACHE'))).toEqual([]);
     expect(JSON.stringify(job)).not.toContain('receipts');
-    expect(job.steps.find(st => st.name === 'Upload rerun results')!.with).toMatchObject({ name: 'paid-rerun-${{ matrix.slice }}-a${{ github.run_attempt }}', path: '/tmp/paid-rerun-results' });
+    expect(job.steps.find(st => st.name === 'Upload rerun results')!.with).toMatchObject({ name: 'paid-rerun-${{ matrix.slice }}-a${{ github.run_attempt }}', path: run.env!.GSTACK_EVAL_DIR });
+    const uploads = Object.values(evals.jobs).flatMap(j => j.steps).map(st => String(st.with?.name ?? '')).filter(Boolean);
+    expect(uploads.filter(name => name.startsWith('paid-rerun-'))).toEqual(['paid-rerun-${{ matrix.slice }}-a${{ github.run_attempt }}']);
     const plan = evals.jobs['rerun-plan']!;
     expect(plan.permissions).toEqual({ contents: 'read' });
     expect(JSON.stringify(plan)).not.toMatch(/secrets\.|restore-deps|bun install/);
-    expect(plan.steps.find(st => st.name === "Plan the rule-16 rerun of this attempt's failed cases")!.run!.replace(/\s*\\\n\s*/g, ' ')).toContain('--report /tmp/paid-report --rerun-plan /tmp/paid-rerun');
+    expect(plan.steps.find(st => st.name === "Plan the rule-16 rerun of this attempt's failed cases")!.run!.replace(/\s*\\\n\s*/g, ' '))
+      .toContain('--report /tmp/paid-report --rerun-plan "/tmp/paid-rerun-plan/rerun-plan-a$GITHUB_RUN_ATTEMPT"');
+    expect(plan.steps.find(st => st.name === 'Hand the rerun manifest to its executors and the report')!.with).toMatchObject({ name: 'rerun-plan-a${{ github.run_attempt }}', path: '/tmp/paid-rerun-plan' });
+  });
+
+  test('a lone rerun artifact never unpacks over the report root or the plan manifest (run 39363705692 did)', () => {
+    for (const id of ['rerun-plan', 'slices-report']) {
+      const downloads = evals.jobs[id]!.steps.filter(st => st.uses?.startsWith('actions/download-artifact@') && /^(paid-rerun|rerun-plan)-/.test(String(st.with?.pattern ?? '')));
+      expect(downloads.map(st => st.with), id).toEqual([{ pattern: 'paid-rerun-*', path: '/tmp/paid-report/reruns' }, { pattern: 'rerun-plan-a*', path: '/tmp/paid-report' }]);
+    }
+    // The plan artifact's root is a directory holding rerun-plan-a<attempt>/manifest.json, written there by the planner.
+    const plan = evals.jobs['rerun-plan']!;
+    expect(plan.steps.find(st => st.name === 'Hand the rerun manifest to its executors and the report')!.with!.path).toBe('/tmp/paid-rerun-plan');
+    expect(plan.steps.find(st => st.name === "Plan the rule-16 rerun of this attempt's failed cases")!.run).toContain('PLAN="/tmp/paid-rerun-plan/rerun-plan-a$GITHUB_RUN_ATTEMPT/manifest.json"');
   });
 
   test('the report reads every attempt and rerun, publishes flake records, and the comment never hides a non-flaky failed record', () => {

test/paid-rerun.test.ts

@@ -250,6 +250,23 @@ describe('negative controls: rule 16 never turns a real red green', () => {
     expect(r.flakes).toEqual([]);
   });
 
+  test('the rerun reads the same in both download-artifact layouts: a lone match unpacked into its path, or each in its own directory', () => {
+    const lone = withRerun(PASS);
+    fs.mkdirSync(path.join(lone, 'reruns'));
+    fs.renameSync(path.join(lone, 'paid-rerun-1-a1'), path.join(lone, 'reruns', 'only'));
+    const several = withRerun(PASS);
+    const planDir = path.join(several, `${RERUN_PLAN_ARTIFACT}-a1`);
+    fs.renameSync(planDir, `${planDir}.tmp`);
+    fs.mkdirSync(planDir);
+    fs.renameSync(`${planDir}.tmp`, path.join(planDir, `${RERUN_PLAN_ARTIFACT}-a1`));
+    for (const dir of [lone, several]) {
+      const r = report(dir);
+      expect(r.status, r.out).toBe(0);
+      expect(r.out).toContain('FLAKY 1');
+      expect(r.flakes.map((f: any) => f.case)).toEqual([CASE]);
+    }
+  });
+
   test('a rerun result that does not match its manifest is set aside and the red stays', () => {
     const r = report(withRerun(PASS, slice => { slice.sliceCount = 9; }));
     expect(r.status, r.out).toBe(1);