From eb4fd65ed356c99bd6528d9eb76627c287771f7b Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Wed, 9 Sep 2026 04:41:23 +0000 Subject: [PATCH] fix: use agent capture budgets for deploy evals Multi-turn deploy and benchmark sessions were incorrectly limited to the single-call judge timeout. Use the existing capture tier and leave outer-test cleanup headroom, with a free policy regression test. Keep all behavioral assertions and frontier models unchanged. Co-Authored-By: OpenAI Codex --- test/eval-budgets-policy.test.ts | 7 +++++++ test/skill-e2e-deploy.test.ts | 26 +++++++++++++------------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/test/eval-budgets-policy.test.ts b/test/eval-budgets-policy.test.ts index 6e7bff921..20da39011 100644 --- a/test/eval-budgets-policy.test.ts +++ b/test/eval-budgets-policy.test.ts @@ -40,6 +40,13 @@ describe('eval budget tiers', () => { expect(Math.max(...values)).toBe(PTY_LONG_MS); }); + test('deploy workflow sessions use capture budgets, not single-call judge budgets', () => { + const source = fs.readFileSync(path.join(ROOT, 'test/skill-e2e-deploy.test.ts'), 'utf8'); + expect(source).not.toContain('JUDGE_MS'); + expect([...source.matchAll(/timeout:\s*CAPTURE_MS/g)]).toHaveLength(6); + expect([...source.matchAll(/\},\s*CAPTURE_LONG_MS\);/g)]).toHaveLength(6); + }); + test('no paid-test timeout literal exceeds the ceiling tier', () => { const out = spawnSync('git', ['ls-files', 'test/*.test.ts'], { cwd: ROOT, encoding: 'utf-8', timeout: 30_000 }); const files = out.stdout.split('\n').filter((f) => f && isPaidTestFile(f)); diff --git a/test/skill-e2e-deploy.test.ts b/test/skill-e2e-deploy.test.ts index d9c657386..c52eac569 100644 --- a/test/skill-e2e-deploy.test.ts +++ b/test/skill-e2e-deploy.test.ts @@ -1,5 +1,5 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; -import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets'; +import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, browseBin, runId, evalsEnabled, @@ -68,7 +68,7 @@ Do NOT use AskUserQuestion. Do NOT run gh or fly commands.`, workingDirectory: landDir, maxTurns: 20, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], - timeout: JUDGE_MS, + timeout: CAPTURE_MS, testName: 'land-and-deploy-workflow', runId, }); @@ -86,7 +86,7 @@ Do NOT use AskUserQuestion. Do NOT run gh or fly commands.`, const reportDir = path.join(landDir, '.gstack', 'deploy-reports'); expect(fs.existsSync(reportDir)).toBe(true); - }, CAPTURE_MS); + }, CAPTURE_LONG_MS); }); // --- Land-and-Deploy First-Run E2E --- @@ -149,7 +149,7 @@ Just demonstrate the first-run dry-run output.`, workingDirectory: firstRunDir, maxTurns: 20, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], - timeout: JUDGE_MS, + timeout: CAPTURE_MS, testName: 'land-and-deploy-first-run', runId, }); @@ -168,7 +168,7 @@ Just demonstrate the first-run dry-run output.`, const reportContent = fs.readFileSync(path.join(reportDir, reportFiles[0]), 'utf-8'); const hasPlatform = reportContent.toLowerCase().includes('fly') || reportContent.toLowerCase().includes('first-run-app'); expect(hasPlatform).toBe(true); - }, CAPTURE_MS); + }, CAPTURE_LONG_MS); }); // --- Land-and-Deploy Review Gate E2E --- @@ -227,7 +227,7 @@ Show what the readiness gate output would look like.`, workingDirectory: reviewDir, maxTurns: 15, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], - timeout: JUDGE_MS, + timeout: CAPTURE_MS, testName: 'land-and-deploy-review-gate', runId, }); @@ -247,7 +247,7 @@ Show what the readiness gate output would look like.`, const hasReviewMention = reportContent.toLowerCase().includes('review') || reportContent.toLowerCase().includes('not run'); expect(hasReviewMention).toBe(true); - }, CAPTURE_MS); + }, CAPTURE_LONG_MS); }); // --- Canary skill E2E --- @@ -295,7 +295,7 @@ Just create the directory structure and report files showing the correct schema. workingDirectory: canaryDir, maxTurns: 15, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'], - timeout: JUDGE_MS, + timeout: CAPTURE_MS, testName: 'canary-workflow', runId, }); @@ -308,7 +308,7 @@ Just create the directory structure and report files showing the correct schema. const reportDir = path.join(canaryDir, '.gstack', 'canary-reports'); const files = fs.readdirSync(reportDir, { recursive: true }) as string[]; expect(files.length).toBeGreaterThan(0); - }, CAPTURE_MS); + }, CAPTURE_LONG_MS); }); // --- Benchmark skill E2E --- @@ -358,7 +358,7 @@ Just create the files showing the correct schema and report format.`, workingDirectory: benchDir, maxTurns: 15, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'], - timeout: JUDGE_MS, + timeout: CAPTURE_MS, testName: 'benchmark-workflow', runId, }); @@ -373,7 +373,7 @@ Just create the files showing the correct schema and report format.`, const files = fs.readdirSync(baselineDir); expect(files.length).toBeGreaterThan(0); } - }, CAPTURE_MS); + }, CAPTURE_LONG_MS); }); // --- Setup-Deploy skill E2E --- @@ -419,7 +419,7 @@ Just detect the platform and write the config.`, workingDirectory: setupDir, maxTurns: 15, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], - timeout: JUDGE_MS, + timeout: CAPTURE_MS, testName: 'setup-deploy-workflow', runId, }); @@ -435,7 +435,7 @@ Just detect the platform and write the config.`, expect(content.toLowerCase()).toContain('fly'); expect(content).toContain('my-cool-app'); expect(content).toContain('Deploy Configuration'); - }, CAPTURE_MS); + }, CAPTURE_LONG_MS); }); // Module-level afterAll — finalize eval collector after all tests complete