diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index 1b0e9fa0b..ec8d4cb29 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -832,23 +832,28 @@ jobs: continue-on-error: true with: pattern: paid-rerun-* - path: /tmp/paid-report + # Its own directory: download-artifact unpacks a lone match into its path itself, so the + # report root would otherwise take a rerun slice's files (run 39363705692). + path: /tmp/paid-report/reruns - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 continue-on-error: true with: pattern: rerun-plan-a* + # Each artifact holds rerun-plan-a/manifest.json, so even a lone match never + # lands on the plan's own manifest.json (run 39363705692). path: /tmp/paid-report - name: Plan the rule-16 rerun of this attempt's failed cases id: plan run: | EVALS_TIER=gate bun --no-env-file --config=/dev/null --no-install run scripts/test-paid-shards.ts --tier gate --report /tmp/paid-report \ - --rerun-plan /tmp/paid-rerun >> "$GITHUB_STEP_SUMMARY" - if [ -f /tmp/paid-rerun/manifest.json ]; then + --rerun-plan "/tmp/paid-rerun-plan/rerun-plan-a$GITHUB_RUN_ATTEMPT" >> "$GITHUB_STEP_SUMMARY" + PLAN="/tmp/paid-rerun-plan/rerun-plan-a$GITHUB_RUN_ATTEMPT/manifest.json" + if [ -f "$PLAN" ]; then { echo "planned=true" - echo "slices=$(jq -c '[range(1; .sliceCount + 1)]' /tmp/paid-rerun/manifest.json)" - echo "slice_timeouts=$(jq -ec '[0] + .plan.sliceCiTimeoutMinutes' /tmp/paid-rerun/manifest.json)"; } >> "$GITHUB_OUTPUT" + echo "slices=$(jq -c '[range(1; .sliceCount + 1)]' "$PLAN")" + echo "slice_timeouts=$(jq -ec '[0] + .plan.sliceCiTimeoutMinutes' "$PLAN")"; } >> "$GITHUB_OUTPUT" else echo "planned=false" >> "$GITHUB_OUTPUT" fi @@ -858,7 +863,7 @@ jobs: uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: rerun-plan-a${{ github.run_attempt }} - path: /tmp/paid-rerun/manifest.json + path: /tmp/paid-rerun-plan retention-days: 90 eval-rerun: @@ -911,7 +916,7 @@ jobs: - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: rerun-plan-a${{ github.run_attempt }} - path: /tmp/paid-plan + path: /tmp/paid-rerun-plan - name: Authenticate the Codex CLI shell: bash @@ -941,7 +946,7 @@ jobs: EVALS_JOBS: "2" EVALS_CONCURRENCY: "2" GSTACK_EVAL_DIR: /tmp/paid-rerun-results - run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/paid-plan/manifest.json --slice ${{ matrix.slice }} + run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/paid-rerun-plan/rerun-plan-a${{ github.run_attempt }}/manifest.json --slice ${{ matrix.slice }} - name: Upload rerun results if: always() @@ -969,7 +974,7 @@ jobs: if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: - name: paid-rerun-logs-${{ matrix.slice }}-a${{ github.run_attempt }} + name: rerun-logs-${{ matrix.slice }}-a${{ github.run_attempt }} include-hidden-files: true path: | /home/runner/.cache/gstack-paid-shard-*.log @@ -1178,12 +1183,16 @@ jobs: continue-on-error: true with: pattern: paid-rerun-* - path: /tmp/paid-report + # Its own directory: download-artifact unpacks a lone match into its path itself, so the + # report root would otherwise take a rerun slice's files (run 39363705692). + path: /tmp/paid-report/reruns - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 continue-on-error: true with: pattern: rerun-plan-a* + # Each artifact holds rerun-plan-a/manifest.json, so even a lone match never + # lands on the plan's own manifest.json (run 39363705692). path: /tmp/paid-report - name: Reconcile slices against the manifest (fail-closed) diff --git a/CHANGELOG.md b/CHANGELOG.md index 734cb78dd..f9f6406ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ ### Changed - **Gate integrity (a gate edit):** harness, INFRA, INCOMPLETE and unattributed reds never rerun and are never flaky. Neither is a case already red twice, or a case whose rerun could not turn the gate green. No threshold, trial count, case kind, judge model, `EVAL_POLICY` constant or `HARNESS_VERSION` changed (recorded non-behavioral). Pass-rate history still records each run's first execution unchanged; a rerun's trials stay in its `paid-rerun-*` artifacts and the flake record. On the gate lane, a rule shard that ran to completion with a red case no longer fails its slice job. The report still fails closed on it, so the run's conclusion follows the report and a flaky GREEN run concludes `success`. Execution failures (timeouts, runner errors, hollow or unfinished shards) still fail the slice. Train gates take the same rerun path. A bisect reruns a VERDICT FAIL once on its prefix (`scripts/train-gate.ts`), and a pass there is a flaky PASS naming both runs. The PR comment's verified-summary check also bounds `flakyCollectorFailures`, so a failed collector record that is not a flaky case still fails the comment. Negative control for the slice-exit change: a case red on its first run and red again on its rerun keeps the real report CLI at exit 1 and `VERDICT RED`; `slices-report` still runs after every rerun path and its last step fails the job on reconcile exit 1, no job carries `continue-on-error`, so the run concludes `failure`; the binding is RED and `gate-reuse` refuses it; and the train record is RED, which T1 classifies RED whatever the run concluded (`test/paid-rerun.test.ts`, `test/evals-train-workflow.test.ts`). - **Free regressions:** `test/paid-rerun.test.ts` replays a trimmed capture of run 39293281037 through the real `--report` and `--rerun-plan` CLI. Its negative controls cover red twice, harness, INCOMPLETE and checkpoint reds, a new red on a later attempt, a first-attempt pass, rerun results with a missing or mismatched manifest, and reruns of unplanned cases. `test/gate-reuse.test.ts` covers R2a and R2 reuse of a flaky GREEN, a manual-attempt binding, and a run still red after its rerun. `test/train-gate.test.ts` covers the train record and the bisect rerun. Workflow wiring tests are in `test/evals-train-workflow.test.ts` and `test/ci-paid-coordination.test.ts`. +- **Rerun artifact layout:** the first gate of this change (run 39363705692) reran its one red case and passed, but `download-artifact` unpacked the lone `rerun-plan-a1` match into the report root, over the plan's own `manifest.json`, so the report read every case as unplanned and stayed red. The plan artifact now holds `rerun-plan-a/manifest.json`, rerun results download under `reruns/`, the rerun logs artifact is `rerun-logs-*` so it no longer matches `paid-rerun-*`, and the report finds the rerun manifest in either layout `download-artifact` produces. That run's artifacts, replayed in the new layout, report `VERDICT GREEN` with `FLAKY 1`. ## [1.91.162.0] - 2026-10-19 diff --git a/scripts/lib/paid-rerun.ts b/scripts/lib/paid-rerun.ts index 567ce9da6..d73a1204b 100644 --- a/scripts/lib/paid-rerun.ts +++ b/scripts/lib/paid-rerun.ts @@ -37,7 +37,7 @@ import { expectedPrCaseCount, shardSlug } from './paid-select'; import { ROOT } from './paid-types'; export const FLAKE_RECORDS_FILE = 'flake-records.jsonl'; -/** Each attempt's rerun manifest lands in the report dir as `-a/manifest.json`. */ +/** Each attempt's rerun manifest lands in the report dir as `-a/manifest.json` (one level deeper when several attempts' plans download). */ export const RERUN_PLAN_ARTIFACT = 'rerun-plan'; export type Phase = 'first' | 'rerun'; @@ -185,7 +185,10 @@ export function observationSets(reportDir: string, manifest: PaidRunManifest, fi const reruns = loadSliceArtifacts(reportDir, { reruns: true }); for (const attempt of [...new Set(reruns.map(a => a.result.attempt ?? 1))].sort((a, b) => a - b)) { let rerunManifest: PaidRunManifest; - try { rerunManifest = parseRunManifest(fs.readFileSync(path.join(reportDir, `${RERUN_PLAN_ARTIFACT}-a${attempt}`, 'manifest.json'), 'utf8')); } catch (error) { + // download-artifact unpacks a lone pattern match into the report dir and each of several into its own directory. + const dir = path.join(reportDir, `${RERUN_PLAN_ARTIFACT}-a${attempt}`); + const file = [path.join(dir, 'manifest.json'), path.join(dir, path.basename(dir), 'manifest.json')].find(candidate => fs.existsSync(candidate)) ?? path.join(dir, 'manifest.json'); + try { rerunManifest = parseRunManifest(fs.readFileSync(file, 'utf8')); } catch (error) { ignored.push(`attempt ${attempt}: rerun results without a readable rerun manifest (${error instanceof Error ? error.message : String(error)})`); continue; } diff --git a/test/ci-paid-coordination.test.ts b/test/ci-paid-coordination.test.ts index 8de15c6ee..0bb517205 100644 --- a/test/ci-paid-coordination.test.ts +++ b/test/ci-paid-coordination.test.ts @@ -167,7 +167,7 @@ describe('paid CI coordination stays off the eval image', () => { if (name === 'evals.yml') expect(String(bisectLogs?.with?.path)).toBe(String(logs?.with?.path)); // F1: the rule-16 eval-rerun keeps the same shard-log upload under its own name. const rerunLogs = name === 'evals.yml' - ? jobs['eval-rerun'].steps.find(step => step.with?.name === 'paid-rerun-logs-${{ matrix.slice }}-a${{ github.run_attempt }}') : undefined; + ? jobs['eval-rerun'].steps.find(step => step.with?.name === 'rerun-logs-${{ matrix.slice }}-a${{ github.run_attempt }}') : undefined; if (name === 'evals.yml') expect(String(rerunLogs?.with?.path)).toBe(String(logs?.with?.path)); expect(hiddenUploads.filter(step => !captures.includes(step))).toEqual([logs!, ...(codexLogs ? [codexLogs] : []), ...(rerunLogs ? [rerunLogs] : []), ...(bisectLogs ? [bisectLogs] : [])]); }); diff --git a/test/evals-train-workflow.test.ts b/test/evals-train-workflow.test.ts index 5a7494826..e52fd1986 100644 --- a/test/evals-train-workflow.test.ts +++ b/test/evals-train-workflow.test.ts @@ -327,16 +327,31 @@ describe('rule-16 rerun wiring (F1)', () => { test('the rerun runs its own attempt-scoped manifest fresh: no receipts in, none out, results under paid-rerun-*', () => { const job = evals.jobs['eval-rerun']!; expect(job.strategy!.matrix.slice).toBe('${{ fromJSON(needs.rerun-plan.outputs.slices) }}'); - expect(job.steps.find(st => st.uses?.startsWith('actions/download-artifact@'))!.with).toEqual({ name: 'rerun-plan-a${{ github.run_attempt }}', path: '/tmp/paid-plan' }); + expect(job.steps.find(st => st.uses?.startsWith('actions/download-artifact@'))!.with).toEqual({ name: 'rerun-plan-a${{ github.run_attempt }}', path: '/tmp/paid-rerun-plan' }); const run = job.steps.find(st => st.name === 'Rerun slice ${{ matrix.slice }} (rule 16, once)')!; - expect(run.run).toContain('--plan /tmp/paid-plan/manifest.json --slice ${{ matrix.slice }}'); + expect(run.run).toContain('--plan /tmp/paid-rerun-plan/rerun-plan-a${{ github.run_attempt }}/manifest.json --slice ${{ matrix.slice }}'); expect(Object.keys(run.env!).filter(key => key.startsWith('EVALS_CACHE'))).toEqual([]); expect(JSON.stringify(job)).not.toContain('receipts'); - expect(job.steps.find(st => st.name === 'Upload rerun results')!.with).toMatchObject({ name: 'paid-rerun-${{ matrix.slice }}-a${{ github.run_attempt }}', path: '/tmp/paid-rerun-results' }); + expect(job.steps.find(st => st.name === 'Upload rerun results')!.with).toMatchObject({ name: 'paid-rerun-${{ matrix.slice }}-a${{ github.run_attempt }}', path: run.env!.GSTACK_EVAL_DIR }); + const uploads = Object.values(evals.jobs).flatMap(j => j.steps).map(st => String(st.with?.name ?? '')).filter(Boolean); + expect(uploads.filter(name => name.startsWith('paid-rerun-'))).toEqual(['paid-rerun-${{ matrix.slice }}-a${{ github.run_attempt }}']); const plan = evals.jobs['rerun-plan']!; expect(plan.permissions).toEqual({ contents: 'read' }); expect(JSON.stringify(plan)).not.toMatch(/secrets\.|restore-deps|bun install/); - expect(plan.steps.find(st => st.name === "Plan the rule-16 rerun of this attempt's failed cases")!.run!.replace(/\s*\\\n\s*/g, ' ')).toContain('--report /tmp/paid-report --rerun-plan /tmp/paid-rerun'); + expect(plan.steps.find(st => st.name === "Plan the rule-16 rerun of this attempt's failed cases")!.run!.replace(/\s*\\\n\s*/g, ' ')) + .toContain('--report /tmp/paid-report --rerun-plan "/tmp/paid-rerun-plan/rerun-plan-a$GITHUB_RUN_ATTEMPT"'); + expect(plan.steps.find(st => st.name === 'Hand the rerun manifest to its executors and the report')!.with).toMatchObject({ name: 'rerun-plan-a${{ github.run_attempt }}', path: '/tmp/paid-rerun-plan' }); + }); + + test('a lone rerun artifact never unpacks over the report root or the plan manifest (run 39363705692 did)', () => { + for (const id of ['rerun-plan', 'slices-report']) { + const downloads = evals.jobs[id]!.steps.filter(st => st.uses?.startsWith('actions/download-artifact@') && /^(paid-rerun|rerun-plan)-/.test(String(st.with?.pattern ?? ''))); + expect(downloads.map(st => st.with), id).toEqual([{ pattern: 'paid-rerun-*', path: '/tmp/paid-report/reruns' }, { pattern: 'rerun-plan-a*', path: '/tmp/paid-report' }]); + } + // The plan artifact's root is a directory holding rerun-plan-a/manifest.json, written there by the planner. + const plan = evals.jobs['rerun-plan']!; + expect(plan.steps.find(st => st.name === 'Hand the rerun manifest to its executors and the report')!.with!.path).toBe('/tmp/paid-rerun-plan'); + expect(plan.steps.find(st => st.name === "Plan the rule-16 rerun of this attempt's failed cases")!.run).toContain('PLAN="/tmp/paid-rerun-plan/rerun-plan-a$GITHUB_RUN_ATTEMPT/manifest.json"'); }); test('the report reads every attempt and rerun, publishes flake records, and the comment never hides a non-flaky failed record', () => { diff --git a/test/paid-rerun.test.ts b/test/paid-rerun.test.ts index 5c7a54cc0..c4bb13682 100644 --- a/test/paid-rerun.test.ts +++ b/test/paid-rerun.test.ts @@ -250,6 +250,23 @@ describe('negative controls: rule 16 never turns a real red green', () => { expect(r.flakes).toEqual([]); }); + test('the rerun reads the same in both download-artifact layouts: a lone match unpacked into its path, or each in its own directory', () => { + const lone = withRerun(PASS); + fs.mkdirSync(path.join(lone, 'reruns')); + fs.renameSync(path.join(lone, 'paid-rerun-1-a1'), path.join(lone, 'reruns', 'only')); + const several = withRerun(PASS); + const planDir = path.join(several, `${RERUN_PLAN_ARTIFACT}-a1`); + fs.renameSync(planDir, `${planDir}.tmp`); + fs.mkdirSync(planDir); + fs.renameSync(`${planDir}.tmp`, path.join(planDir, `${RERUN_PLAN_ARTIFACT}-a1`)); + for (const dir of [lone, several]) { + const r = report(dir); + expect(r.status, r.out).toBe(0); + expect(r.out).toContain('FLAKY 1'); + expect(r.flakes.map((f: any) => f.case)).toEqual([CASE]); + } + }); + test('a rerun result that does not match its manifest is set aside and the red stays', () => { const r = report(withRerun(PASS, slice => { slice.sliceCount = 9; })); expect(r.status, r.out).toBe(1);