diff --git a/test/carve-section-loading.test.ts b/test/carve-section-loading.test.ts index 0cd521371..75a787cbd 100644 --- a/test/carve-section-loading.test.ts +++ b/test/carve-section-loading.test.ts @@ -20,6 +20,7 @@ */ import { test, expect } from 'bun:test'; +import { CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import { setupSkillDir, skillFromWorktree, captureSectionReads } from './helpers/auq-sdk-capture'; import { CARVE_GUARDS } from './helpers/carve-guards'; @@ -97,7 +98,7 @@ describeE2E('carve behavioral section-loading (periodic, SDK capture)', () => { }); expect(output.trim().length).toBeGreaterThan(200); }, - 540_000, + CAPTURE_LONG_MS, ); } }); diff --git a/test/codex-e2e-plan-format.test.ts b/test/codex-e2e-plan-format.test.ts index acf92f72e..3bfce0fdd 100644 --- a/test/codex-e2e-plan-format.test.ts +++ b/test/codex-e2e-plan-format.test.ts @@ -26,6 +26,7 @@ * Periodic tier (Codex non-determinism). Cost: ~$2-3 per full run. */ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { runCodexSkill, installSkillToTempHome } from './helpers/codex-session-runner'; import type { CodexResult } from './helpers/codex-session-runner'; import { EvalCollector } from './helpers/eval-store'; @@ -186,7 +187,7 @@ describeCodex('Codex Plan Format — CEO Mode Selection', () => { const result = await runCodexSkill({ skillDir, prompt: `Read the plan-ceo-review skill. Read plan.md (the plan to review). Proceed to Step 0F (Mode Selection) where the skill presents 4 mode options (SCOPE EXPANSION, SELECTIVE EXPANSION, HOLD SCOPE, SCOPE REDUCTION) via AskUserQuestion. These options differ in kind (review posture), not coverage. ${captureInstruction(outFile)}`, - timeoutMs: 300_000, + timeoutMs: CAPTURE_MS, cwd: planDir, skillName: 'gstack-plan-ceo-review', sandbox: 'workspace-write', @@ -208,7 +209,7 @@ describeCodex('Codex Plan Format — CEO Mode Selection', () => { // kind-differentiated: no fabricated score, must have note expect(captured).not.toMatch(COMPLETENESS_RE); expect(captured).toMatch(KIND_NOTE_RE); - }, 360_000); + }, CAPTURE_LONG_MS); }); describeCodex('Codex Plan Format — CEO Approach Menu', () => { @@ -226,7 +227,7 @@ describeCodex('Codex Plan Format — CEO Approach Menu', () => { const result = await runCodexSkill({ skillDir, prompt: `Read the plan-ceo-review skill. Read plan.md. Proceed to Step 0C-bis (Implementation Alternatives / Approach Menu) where the skill generates 2-3 approaches (minimal viable vs ideal architecture) and presents them via AskUserQuestion. These options differ in coverage so Completeness: N/10 applies. ${captureInstruction(outFile)}`, - timeoutMs: 300_000, + timeoutMs: CAPTURE_MS, cwd: planDir, skillName: 'gstack-plan-ceo-review', sandbox: 'workspace-write', @@ -245,7 +246,7 @@ describeCodex('Codex Plan Format — CEO Approach Menu', () => { expect(captured.length).toBeGreaterThan(ELI10_LENGTH_FLOOR); expect(captured).toMatch(RECOMMENDATION_RE); expect(captured).toMatch(COMPLETENESS_RE); - }, 360_000); + }, CAPTURE_LONG_MS); }); describeCodex('Codex Plan Format — Eng Coverage Issue', () => { @@ -263,7 +264,7 @@ describeCodex('Codex Plan Format — Eng Coverage Issue', () => { const result = await runCodexSkill({ skillDir, prompt: `Read the plan-eng-review skill. Read plan.md. In your Section 3 Test Review, generate ONE AskUserQuestion about test coverage depth where options are clearly coverage-differentiated: A) full coverage incl. edge + error paths (Completeness 10/10), B) happy path only (7/10), C) smoke test (3/10). ${captureInstruction(outFile)}`, - timeoutMs: 300_000, + timeoutMs: CAPTURE_MS, cwd: planDir, skillName: 'gstack-plan-eng-review', sandbox: 'workspace-write', @@ -282,7 +283,7 @@ describeCodex('Codex Plan Format — Eng Coverage Issue', () => { expect(captured.length).toBeGreaterThan(ELI10_LENGTH_FLOOR); expect(captured).toMatch(RECOMMENDATION_RE); expect(captured).toMatch(COMPLETENESS_RE); - }, 360_000); + }, CAPTURE_LONG_MS); }); describeCodex('Codex Plan Format — Eng Kind Issue', () => { @@ -300,7 +301,7 @@ describeCodex('Codex Plan Format — Eng Kind Issue', () => { const result = await runCodexSkill({ skillDir, prompt: `Read the plan-eng-review skill. Read plan.md. In your Section 1 Architecture review, generate ONE AskUserQuestion about an architectural choice where the options differ in kind (e.g. Redis vs Postgres materialized view vs in-process cache — different kinds of systems with different tradeoffs, NOT more-or-less-complete versions of the same thing). ${captureInstruction(outFile)}`, - timeoutMs: 300_000, + timeoutMs: CAPTURE_MS, cwd: planDir, skillName: 'gstack-plan-eng-review', sandbox: 'workspace-write', @@ -321,5 +322,5 @@ describeCodex('Codex Plan Format — Eng Kind Issue', () => { // kind-differentiated: no fabricated score expect(captured).not.toMatch(COMPLETENESS_RE); expect(captured).toMatch(KIND_NOTE_RE); - }, 360_000); + }, CAPTURE_LONG_MS); }); diff --git a/test/codex-e2e-recommendation-substance.test.ts b/test/codex-e2e-recommendation-substance.test.ts index b62c3daa5..f4a6530c0 100644 --- a/test/codex-e2e-recommendation-substance.test.ts +++ b/test/codex-e2e-recommendation-substance.test.ts @@ -21,6 +21,7 @@ * Periodic tier (Codex non-determinism, ~$2-3/run). */ import { describe, test, expect } from 'bun:test'; +import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import * as path from 'node:path'; import { e2eTierEnabled } from './helpers/e2e-gate'; import { runCodexSkill } from './helpers/codex-session-runner'; @@ -69,7 +70,7 @@ describeCodex('/codex recommendation substance (live, periodic)', () => { skillDir: path.join(ROOT, 'codex'), skillName: 'codex', prompt: FIXTURE_DIFF, - timeoutMs: 300_000, + timeoutMs: CAPTURE_MS, }); if (result.output.startsWith('SKIP:')) { @@ -98,6 +99,6 @@ describeCodex('/codex recommendation substance (live, periodic)', () => { ); } }, - 360_000, + CAPTURE_LONG_MS, ); }); diff --git a/test/codex-e2e-sol-scope.test.ts b/test/codex-e2e-sol-scope.test.ts index 0d00f1324..dd3c52aee 100644 --- a/test/codex-e2e-sol-scope.test.ts +++ b/test/codex-e2e-sol-scope.test.ts @@ -11,6 +11,7 @@ * golden), parallel shards (worktree copies), or live symlinked installs. */ import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import { CAPTURE_MS } from './helpers/eval-budgets'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; @@ -262,5 +263,5 @@ You are authorized to implement the minimal fix. The task boundary is src/parse- expect(readmeDecoyUntouched).toBe(true); console.log(`codex-sol-scope: ${result.tokens} tokens, ${result.toolCalls.length} tool calls, ${Math.round(result.durationMs / 1000)}s`); - }, 300_000); + }, CAPTURE_MS); }); diff --git a/test/codex-e2e.test.ts b/test/codex-e2e.test.ts index 696e66a5e..dc51d322d 100644 --- a/test/codex-e2e.test.ts +++ b/test/codex-e2e.test.ts @@ -14,6 +14,7 @@ */ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { JUDGE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { runCodexSkill, parseCodexJSONL, installSkillToTempHome } from './helpers/codex-session-runner'; import type { CodexResult } from './helpers/codex-session-runner'; import { CODEX_REVIEW_E2E_SECTIONS } from './helpers/skill-fixture'; @@ -150,7 +151,7 @@ describeCodex('Codex E2E', () => { const result = await runCodexSkill({ skillDir, prompt: 'List any skills or instructions you have available. Just list the names.', - timeoutMs: 60_000, + timeoutMs: JUDGE_MS, cwd: testWorktree, skillName: 'gstack-review', }); @@ -171,7 +172,7 @@ describeCodex('Codex E2E', () => { expect( outputLower.includes('review') || outputLower.includes('gstack') || outputLower.includes('skill'), ).toBe(true); - }, 120_000); + }, JUDGE_MS); // Validates that Codex can invoke the gstack-review skill, run a diff-based // code review, and produce structured review output with findings/issues. @@ -186,7 +187,7 @@ describeCodex('Codex E2E', () => { const result = await runCodexSkill({ skillDir, prompt: 'Run the gstack-review skill on this repository. Review the current branch diff and report your findings.', - timeoutMs: 540_000, + timeoutMs: CAPTURE_LONG_MS, cwd: testWorktree, skillName: 'gstack-review', sections: CODEX_REVIEW_E2E_SECTIONS, @@ -224,5 +225,5 @@ describeCodex('Codex E2E', () => { outputLower.includes('p1') || outputLower.includes('p2'); expect(hasReviewContent).toBe(true); - }, 600_000); + }, CAPTURE_LONG_MS); }); diff --git a/test/gemini-e2e.test.ts b/test/gemini-e2e.test.ts index afbb07f83..eb3c49b00 100644 --- a/test/gemini-e2e.test.ts +++ b/test/gemini-e2e.test.ts @@ -15,6 +15,7 @@ */ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { JUDGE_MS } from './helpers/eval-budgets'; import { runGeminiSkill } from './helpers/gemini-session-runner'; import type { GeminiResult } from './helpers/gemini-session-runner'; import { EvalCollector } from './helpers/eval-store'; @@ -151,7 +152,7 @@ describeGemini('Gemini E2E', () => { // Uses a simple prompt that doesn't require skill invocation or complex navigation. const result = await runGeminiSkill({ prompt: 'What is this project? Answer in one sentence based on the README.', - timeoutMs: 90_000, + timeoutMs: JUDGE_MS, cwd: testWorktree, }); @@ -163,5 +164,5 @@ describeGemini('Gemini E2E', () => { recordGeminiE2E('gemini-smoke', result, passed); expect(result.output.length, 'Gemini should produce output').toBeGreaterThan(10); - }, 120_000); + }, JUDGE_MS); }); diff --git a/test/llm-judge-recommendation.test.ts b/test/llm-judge-recommendation.test.ts index 04dac2dd7..438d1da37 100644 --- a/test/llm-judge-recommendation.test.ts +++ b/test/llm-judge-recommendation.test.ts @@ -12,6 +12,7 @@ */ import { expect } from 'bun:test'; +import { CAPTURE_MS } from './helpers/eval-budgets'; import { judgeRecommendation } from './helpers/llm-judge'; import { describeIfSelected, testIfSelected } from './helpers/e2e-helpers'; @@ -181,5 +182,5 @@ Net: ...`); `[hedge:${label}] expected commits=false; got ${score.commits}. text="${text}"`, ).toBe(false); } - }, 240_000); + }, CAPTURE_MS); }); diff --git a/test/skill-e2e-ask-user-question-format-compliance.test.ts b/test/skill-e2e-ask-user-question-format-compliance.test.ts index 2823febc3..a6ac84b6d 100644 --- a/test/skill-e2e-ask-user-question-format-compliance.test.ts +++ b/test/skill-e2e-ask-user-question-format-compliance.test.ts @@ -23,6 +23,7 @@ * A/B and matrix evals (test/helpers/auq-sdk-capture.ts). */ import { test, expect } from 'bun:test'; +import { CAPTURE_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; import { @@ -86,6 +87,6 @@ describeE2E('AskUserQuestion format compliance (gate)', () => { ); } }, - 300_000, + CAPTURE_MS, ); }); diff --git a/test/skill-e2e-auq-consistency.test.ts b/test/skill-e2e-auq-consistency.test.ts index ca1c9f93d..92e0f0b1c 100644 --- a/test/skill-e2e-auq-consistency.test.ts +++ b/test/skill-e2e-auq-consistency.test.ts @@ -16,6 +16,7 @@ * (N SDK runs, ~$0.50-1 each). */ import { test } from 'bun:test'; +import { CAPTURE_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; import { @@ -99,6 +100,6 @@ describeE2E('AUQ consistency across runs (periodic)', () => { `format elements every run; substance ${minSub}-${maxSub}`, ); }, - N_RUNS * 300_000 + 60_000, + N_RUNS * CAPTURE_MS + 60_000, ); }); diff --git a/test/skill-e2e-auq-matrix.test.ts b/test/skill-e2e-auq-matrix.test.ts index e8c5eef93..92674280d 100644 --- a/test/skill-e2e-auq-matrix.test.ts +++ b/test/skill-e2e-auq-matrix.test.ts @@ -23,6 +23,7 @@ * Run a subset in the foreground with AUQ_MATRIX_ONLY="plan-eng-review,cso". */ import { test } from 'bun:test'; +import { CAPTURE_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; import { @@ -174,7 +175,7 @@ describeE2E('AUQ behavioral matrix (periodic)', () => { ); } }, - 300_000, + CAPTURE_MS, ); } }); diff --git a/test/skill-e2e-auq-verbose-vs-carved-ab.test.ts b/test/skill-e2e-auq-verbose-vs-carved-ab.test.ts index a812785fa..595c8fb1e 100644 --- a/test/skill-e2e-auq-verbose-vs-carved-ab.test.ts +++ b/test/skill-e2e-auq-verbose-vs-carved-ab.test.ts @@ -23,6 +23,7 @@ * strictly less unrelated review-section text in context. */ import { test } from 'bun:test'; +import { CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; import { @@ -109,6 +110,6 @@ describeE2E('AUQ no-degradation: verbose vs carved (periodic)', () => { // eslint-disable-next-line no-console console.log('[AUQ-AB] NO DEGRADATION:\n' + summary); }, - 600_000, + CAPTURE_LONG_MS, ); }); diff --git a/test/skill-e2e-auto-decide-preserved.test.ts b/test/skill-e2e-auto-decide-preserved.test.ts index 0294a742e..2feca71cf 100644 --- a/test/skill-e2e-auto-decide-preserved.test.ts +++ b/test/skill-e2e-auto-decide-preserved.test.ts @@ -38,6 +38,7 @@ */ import { test, expect } from 'bun:test'; +import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillObservation } from './helpers/claude-pty-runner'; import * as fs from 'fs'; @@ -113,7 +114,7 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', () skillName: 'plan-ceo-review', inPlanMode: true, extraArgs: ['--disallowedTools', 'AskUserQuestion'], - timeoutMs: 540_000, + timeoutMs: CAPTURE_LONG_MS, env: { GSTACK_HOME: tmpHome, CONDUCTOR_WORKSPACE_PATH: tmpHome }, }); @@ -137,5 +138,5 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', () } finally { try { fs.rmSync(tmpHome, { recursive: true, force: true }); } catch { /* best-effort */ } } - }, 660_000); + }, PTY_MS); }); diff --git a/test/skill-e2e-autoplan-chain.test.ts b/test/skill-e2e-autoplan-chain.test.ts index 7f6fbea6d..d05219c45 100644 --- a/test/skill-e2e-autoplan-chain.test.ts +++ b/test/skill-e2e-autoplan-chain.test.ts @@ -25,6 +25,7 @@ */ import { test, expect } from 'bun:test'; +import { PTY_LONG_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import { spawnSync } from 'child_process'; import * as fs from 'fs'; @@ -174,6 +175,6 @@ describeE2E('/autoplan chain ordering (periodic)', () => { try { fs.rmSync(tempDir, { recursive: true, force: true }); } catch { /* ignore */ } } }, - 1_200_000, // 20 min absolute test ceiling + PTY_LONG_MS, // 20 min absolute test ceiling ); }); diff --git a/test/skill-e2e-autoplan-dual-voice.test.ts b/test/skill-e2e-autoplan-dual-voice.test.ts index d3e490f45..1efc6dfc6 100644 --- a/test/skill-e2e-autoplan-dual-voice.test.ts +++ b/test/skill-e2e-autoplan-dual-voice.test.ts @@ -1,4 +1,5 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, runId, evalsEnabled, @@ -98,7 +99,7 @@ Add a new /greet skill that prints a welcome message. testName: 'autoplan-dual-voice', workingDirectory: workDir, prompt: `/autoplan ${planPath}`, - timeout: 600_000, // 10 min + timeout: CAPTURE_LONG_MS, // 10 min // /autoplan spawns subagents and calls codex via Bash; it needs the // full tool set to get past Phase 1. Bash+Read+Write alone wasn't // enough — the skill stalled trying to invoke Agent/Skill. diff --git a/test/skill-e2e-benchmark-providers.test.ts b/test/skill-e2e-benchmark-providers.test.ts index 10395dea5..638f9a2c5 100644 --- a/test/skill-e2e-benchmark-providers.test.ts +++ b/test/skill-e2e-benchmark-providers.test.ts @@ -19,6 +19,7 @@ */ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets'; import { ClaudeAdapter } from './helpers/providers/claude'; import { GptAdapter } from './helpers/providers/gpt'; import { GeminiAdapter } from './helpers/providers/gemini'; @@ -94,7 +95,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => { process.stderr.write(`\nclaude live smoke: SKIPPED — ${check.reason}\n`); return; } - const result = await claude.run({ prompt: PROMPT, workdir, timeoutMs: 120_000 }); + const result = await claude.run({ prompt: PROMPT, workdir, timeoutMs: JUDGE_MS }); if (result.error) { throw new Error(`claude errored: ${result.error.code} — ${result.error.reason}`); } @@ -106,7 +107,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => { expect(result.modelUsed.length).toBeGreaterThan(0); const cost = claude.estimateCost(result.tokens, result.modelUsed); expect(cost).toBeGreaterThan(0); - }, 150_000); + }, CAPTURE_MS); test('gpt: trivial prompt produces parseable output', async () => { const check = await gpt.available(); @@ -114,7 +115,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => { process.stderr.write(`\ngpt live smoke: SKIPPED — ${check.reason}\n`); return; } - const result = await gpt.run({ prompt: PROMPT, workdir, timeoutMs: 120_000 }); + const result = await gpt.run({ prompt: PROMPT, workdir, timeoutMs: JUDGE_MS }); if (result.error) { throw new Error(`gpt errored: ${result.error.code} — ${result.error.reason}`); } @@ -125,7 +126,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => { expect(typeof result.modelUsed).toBe('string'); const cost = gpt.estimateCost(result.tokens, result.modelUsed); expect(cost).toBeGreaterThan(0); - }, 150_000); + }, CAPTURE_MS); test('gemini: trivial prompt produces parseable output', async () => { const check = await gemini.available(); @@ -133,7 +134,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => { process.stderr.write(`\ngemini live smoke: SKIPPED — ${check.reason}\n`); return; } - const result = await gemini.run({ prompt: PROMPT, workdir, timeoutMs: 120_000 }); + const result = await gemini.run({ prompt: PROMPT, workdir, timeoutMs: JUDGE_MS }); if (result.error) { // auth / rate_limit are ENVIRONMENT conditions the test can't act on // (e.g. Google deprecated the individual code-assist auth path — the @@ -155,7 +156,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => { expect(result.durationMs).toBeGreaterThan(0); expect(typeof result.modelUsed).toBe('string'); expect(result.modelUsed.length).toBeGreaterThan(0); - }, 150_000); + }, CAPTURE_MS); test('timeout error surfaces as error.code=timeout (no exception)', async () => { // Use whatever adapter is available first — all three should share timeout semantics. @@ -183,7 +184,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => { prompt: PROMPT, workdir, providers: ['claude', 'gpt', 'gemini'], - timeoutMs: 120_000, + timeoutMs: JUDGE_MS, skipUnavailable: false, }); expect(report.entries).toHaveLength(3); @@ -201,5 +202,5 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => { if (!hadSuccess) { process.stderr.write('\nrunBenchmark live: no provider produced a clean result (no auth?)\n'); } - }, 300_000); + }, CAPTURE_MS); }); diff --git a/test/skill-e2e-brain-privacy-gate.test.ts b/test/skill-e2e-brain-privacy-gate.test.ts index 1ce7c94de..200a1c1f2 100644 --- a/test/skill-e2e-brain-privacy-gate.test.ts +++ b/test/skill-e2e-brain-privacy-gate.test.ts @@ -21,6 +21,7 @@ */ import { test, expect } from 'bun:test'; +import { CAPTURE_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'fs'; import * as os from 'os'; @@ -150,7 +151,7 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => { fs.rmSync(fakeBinDir, { recursive: true, force: true }); fs.rmSync(tempHome, { recursive: true, force: true }); } - }, 180_000); + }, CAPTURE_MS); test('privacy gate does NOT fire when artifacts_sync_mode_prompted is already true', async () => { // Same staging, but prompted=true this time. Gate should be silent. @@ -228,5 +229,5 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => { fs.rmSync(fakeBinDir, { recursive: true, force: true }); fs.rmSync(tempHome, { recursive: true, force: true }); } - }, 180_000); + }, CAPTURE_MS); }); diff --git a/test/skill-e2e-bws.test.ts b/test/skill-e2e-bws.test.ts index 2a991faf5..57dd6dc0e 100644 --- a/test/skill-e2e-bws.test.ts +++ b/test/skill-e2e-bws.test.ts @@ -1,4 +1,5 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { JUDGE_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, browseBin, runId, evalsEnabled, @@ -47,7 +48,7 @@ describeIfSelected('Skill E2E tests', [ Report the results of each command.`, workingDirectory: tmpDir, maxTurns: 7, - timeout: 60_000, + timeout: JUDGE_MS, testName: 'browse-basic', runId, }); @@ -56,7 +57,7 @@ Report the results of each command.`, recordE2E(evalCollector, 'browse basic commands', 'Skill E2E tests', result); expect(result.browseErrors).toHaveLength(0); expect(result.exitReason).toBe('success'); - }, 90_000); + }, JUDGE_MS); testConcurrentIfSelected('browse-snapshot', async () => { const result = await runSkillTest({ @@ -69,7 +70,7 @@ Report the results of each command.`, Report what each command returned.`, workingDirectory: tmpDir, maxTurns: 9, - timeout: 60_000, + timeout: JUDGE_MS, testName: 'browse-snapshot', runId, }); @@ -81,7 +82,7 @@ Report what each command returned.`, console.warn('Browse errors (non-fatal):', result.browseErrors); } expect(result.exitReason).toBe('success'); - }, 90_000); + }, JUDGE_MS); testConcurrentIfSelected('skillmd-setup-discovery', async () => { // P2 (v1.2.0): the browse SETUP/binary-discovery block moved from the root @@ -104,7 +105,7 @@ Then run: $B text Report whether it worked.`, workingDirectory: tmpDir, maxTurns: 10, - timeout: 60_000, + timeout: JUDGE_MS, testName: 'skillmd-setup-discovery', runId, }); @@ -112,7 +113,7 @@ Report whether it worked.`, recordE2E(evalCollector, 'SKILL.md setup block discovery', 'Skill E2E tests', result); expect(result.browseErrors).toHaveLength(0); expect(result.exitReason).toBe('success'); - }, 90_000); + }, JUDGE_MS); testConcurrentIfSelected('skillmd-no-local-binary', async () => { // Create a tmpdir with no browse binary — no local .claude/skills/gstack/browse/dist/browse @@ -149,7 +150,7 @@ Report the exact output. Do NOT try to fix or install anything — just report w // Clean up try { fs.rmSync(emptyDir, { recursive: true, force: true }); } catch {} - }, 60_000); + }, JUDGE_MS); testConcurrentIfSelected('skillmd-outside-git', async () => { // Create a tmpdir outside any git repo @@ -182,7 +183,7 @@ Report the exact output — either "READY: " or "NEEDS_SETUP".`, // Clean up try { fs.rmSync(nonGitDir, { recursive: true, force: true }); } catch {} - }, 60_000); + }, JUDGE_MS); testConcurrentIfSelected('operational-learning', async () => { const opDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-oplearn-')); @@ -286,7 +287,7 @@ Log the operational learning now. Then say what you logged.`, // Clean up try { fs.rmSync(opDir, { recursive: true, force: true }); } catch {} - }, 90_000); + }, JUDGE_MS); testConcurrentIfSelected('session-awareness', async () => { const sessionDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-session-')); @@ -353,7 +354,7 @@ Since this is non-interactive, DO NOT actually call AskUserQuestion. Instead, wr Remember: _SESSIONS=4, so ELI16 mode is active. The user is juggling multiple windows and may not remember what this conversation is about. Re-ground them.`, workingDirectory: sessionDir, maxTurns: 8, - timeout: 60_000, + timeout: JUDGE_MS, testName: 'session-awareness', runId, }); @@ -394,7 +395,7 @@ Remember: _SESSIONS=4, so ELI16 mode is active. The user is juggling multiple wi // Clean up try { fs.rmSync(sessionDir, { recursive: true, force: true }); } catch {} - }, 90_000); + }, JUDGE_MS); }); // Module-level afterAll — finalize eval collector after all tests complete diff --git a/test/skill-e2e-conductor-prose.test.ts b/test/skill-e2e-conductor-prose.test.ts index 7b71677e4..7ddf04837 100644 --- a/test/skill-e2e-conductor-prose.test.ts +++ b/test/skill-e2e-conductor-prose.test.ts @@ -21,6 +21,7 @@ */ import { test, expect } from 'bun:test'; +import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillObservation } from './helpers/claude-pty-runner'; @@ -46,7 +47,7 @@ describeE2E('Conductor renders decisions as prose (periodic)', () => { extraArgs: ['--disallowedTools', 'AskUserQuestion'], env: { CONDUCTOR_WORKSPACE_PATH: '/tmp/conductor-prose-e2e' }, initialPlanContent: FLAWED_PLAN, - timeoutMs: 300_000, + timeoutMs: CAPTURE_MS, }); // The decision must reach the human as prose. 'silent_write' (wrote findings @@ -65,5 +66,5 @@ describeE2E('Conductor renders decisions as prose (periodic)', () => { } // A prose-rendered decision brief was observed at some point in the run. expect(obs.proseAUQEverObserved).toBe(true); - }, 360_000); + }, CAPTURE_LONG_MS); }); diff --git a/test/skill-e2e-context-skills.test.ts b/test/skill-e2e-context-skills.test.ts index d0896cc0f..32335ffe1 100644 --- a/test/skill-e2e-context-skills.test.ts +++ b/test/skill-e2e-context-skills.test.ts @@ -12,6 +12,7 @@ */ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, runId, evalsEnabled, @@ -163,7 +164,7 @@ describeIfSelected('Context Skills E2E (live-fire)', [ env: { GSTACK_HOME: gstackHome }, maxTurns: 12, allowedTools: ['Skill', 'Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], - timeout: 120_000, + timeout: JUDGE_MS, testName: 'context-save-routing', runId, }); @@ -185,7 +186,7 @@ describeIfSelected('Context Skills E2E (live-fire)', [ expect(routedToContextSave).toBe(true); expect(files.length).toBeGreaterThan(0); try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} - }, 180_000); + }, CAPTURE_MS); // ── 2. Round-trip: save then restore in the same session ───────────── testConcurrentIfSelected('context-save-then-restore-roundtrip', async () => { @@ -205,7 +206,7 @@ Do NOT use AskUserQuestion.`, env: { GSTACK_HOME: gstackHome }, maxTurns: 25, allowedTools: ['Skill', 'Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], - timeout: 240_000, + timeout: CAPTURE_MS, testName: 'context-save-then-restore-roundtrip', runId, }); @@ -232,7 +233,7 @@ Do NOT use AskUserQuestion.`, expect(files.length).toBeGreaterThan(0); expect(restoreMentionsTitle).toBe(true); try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} - }, 240_000); + }, CAPTURE_MS); // ── 3. /context-restore loads the matching save ─────────── testConcurrentIfSelected('context-restore-fragment-match', async () => { @@ -255,7 +256,7 @@ Do NOT use AskUserQuestion.`, env: { GSTACK_HOME: gstackHome }, maxTurns: 10, allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'], - timeout: 120_000, + timeout: JUDGE_MS, testName: 'context-restore-fragment-match', runId, }); @@ -279,7 +280,7 @@ Do NOT use AskUserQuestion.`, expect(loadedPayments).toBe(true); expect(didNotLoadOthers).toBe(true); try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} - }, 180_000); + }, CAPTURE_MS); // ── 4. /context-restore with zero saves → graceful empty-state ─────── testConcurrentIfSelected('context-restore-empty-state', async () => { @@ -294,7 +295,7 @@ Do NOT use AskUserQuestion.`, env: { GSTACK_HOME: gstackHome }, maxTurns: 8, allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'], - timeout: 90_000, + timeout: JUDGE_MS, testName: 'context-restore-empty-state', runId, }); @@ -319,7 +320,7 @@ Do NOT use AskUserQuestion.`, expect(routedToRestore).toBe(true); expect(gracefulMessage).toBe(true); try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} - }, 150_000); + }, CAPTURE_MS); // ── 5. /context-restore list redirects to /context-save list ───────── testConcurrentIfSelected('context-restore-list-delegates', async () => { @@ -334,7 +335,7 @@ Do NOT use AskUserQuestion.`, env: { GSTACK_HOME: gstackHome }, maxTurns: 8, allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'], - timeout: 90_000, + timeout: JUDGE_MS, testName: 'context-restore-list-delegates', runId, }); @@ -357,7 +358,7 @@ Do NOT use AskUserQuestion.`, expect(routedToRestore).toBe(true); expect(mentionsSaveList).toBe(true); try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} - }, 150_000); + }, CAPTURE_MS); // ── 6. Legacy compat: pre-rename save files still load ─────────────── testConcurrentIfSelected('context-restore-legacy-compat', async () => { @@ -381,7 +382,7 @@ Do NOT use AskUserQuestion.`, env: { GSTACK_HOME: gstackHome }, maxTurns: 8, allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'], - timeout: 120_000, + timeout: JUDGE_MS, testName: 'context-restore-legacy-compat', runId, }); @@ -414,7 +415,7 @@ Do NOT use AskUserQuestion.`, expect(routedToRestore).toBe(true); expect(loadedLegacy).toBe(true); try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} - }, 180_000); + }, CAPTURE_MS); // ── 7. /context-save list: default filters to current branch ───────── testConcurrentIfSelected('context-save-list-current-branch', async () => { @@ -437,7 +438,7 @@ Do NOT use AskUserQuestion.`, env: { GSTACK_HOME: gstackHome }, maxTurns: 10, allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'], - timeout: 120_000, + timeout: JUDGE_MS, testName: 'context-save-list-current-branch', runId, }); @@ -472,7 +473,7 @@ Do NOT use AskUserQuestion.`, expect(hidesAlpha).toBe(true); expect(hidesBeta).toBe(true); try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} - }, 180_000); + }, CAPTURE_MS); // ── 8. /context-save list --all: shows every branch ────────────────── testConcurrentIfSelected('context-save-list-all-branches', async () => { @@ -494,7 +495,7 @@ Do NOT use AskUserQuestion.`, env: { GSTACK_HOME: gstackHome }, maxTurns: 10, allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'], - timeout: 120_000, + timeout: JUDGE_MS, testName: 'context-save-list-all-branches', runId, }); @@ -520,5 +521,5 @@ Do NOT use AskUserQuestion.`, expect(routed).toBe(true); expect(filesShown).toBe(3); try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} - }, 180_000); + }, CAPTURE_MS); }); diff --git a/test/skill-e2e-coverage-audit.test.ts b/test/skill-e2e-coverage-audit.test.ts index 8e4f1af24..441610cbe 100644 --- a/test/skill-e2e-coverage-audit.test.ts +++ b/test/skill-e2e-coverage-audit.test.ts @@ -20,6 +20,7 @@ */ import { test, expect, beforeAll, afterAll } from 'bun:test'; +import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, runId, @@ -78,7 +79,7 @@ Output the diagram directly.`, workingDirectory: reviewCoverageDir, maxTurns: 15, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'], - timeout: 120_000, + timeout: JUDGE_MS, testName: 'review-coverage-audit', runId, }); @@ -107,7 +108,7 @@ Output the diagram directly.`, // At minimum, the agent should have read the source and test files const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read'); expect(readCalls.length).toBeGreaterThan(0); - }, 180_000); + }, CAPTURE_MS); }); // --- Plan Eng Review Coverage Audit E2E --- @@ -153,7 +154,7 @@ Output the diagram directly.`, workingDirectory: planCoverageDir, maxTurns: 15, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'], - timeout: 120_000, + timeout: JUDGE_MS, testName: 'plan-eng-coverage-audit', runId, }); @@ -182,7 +183,7 @@ Output the diagram directly.`, // At minimum, the agent should have read the source and test files const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read'); expect(readCalls.length).toBeGreaterThan(0); - }, 180_000); + }, CAPTURE_MS); }); // Module-level afterAll — finalize eval collector after all tests complete diff --git a/test/skill-e2e-cso.test.ts b/test/skill-e2e-cso.test.ts index 9bd0ed380..5f58a45d0 100644 --- a/test/skill-e2e-cso.test.ts +++ b/test/skill-e2e-cso.test.ts @@ -1,4 +1,5 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, runId, evalsEnabled, @@ -75,7 +76,7 @@ IMPORTANT: workingDirectory: csoDir, maxTurns: 30, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob', 'Agent'], - timeout: 300_000, + timeout: CAPTURE_MS, }); logCost('cso', result); @@ -106,7 +107,7 @@ IMPORTANT: } recordE2E(evalCollector, 'cso-full-audit', 'e2e-cso', result); - }, 300_000); + }, CAPTURE_MS); }); describeIfSelected('CSO v2 — diff mode', ['cso-diff-mode'], () => { @@ -181,7 +182,7 @@ IMPORTANT: ).toBe(true); recordE2E(evalCollector, 'cso-diff-mode', 'e2e-cso', result); - }, 400_000); + }, CAPTURE_LONG_MS); }); describeIfSelected('CSO v2 — infra scope', ['cso-infra-scope'], () => { @@ -245,7 +246,7 @@ IMPORTANT: workingDirectory: csoInfraDir, maxTurns: 30, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], - timeout: 360_000, + timeout: CAPTURE_LONG_MS, }); logCost('cso', result); @@ -259,5 +260,5 @@ IMPORTANT: ).toBe(true); recordE2E(evalCollector, 'cso-infra-scope', 'e2e-cso', result); - }, 360_000); + }, CAPTURE_LONG_MS); }); diff --git a/test/skill-e2e-deploy.test.ts b/test/skill-e2e-deploy.test.ts index 83fa7614f..77ef962d7 100644 --- a/test/skill-e2e-deploy.test.ts +++ b/test/skill-e2e-deploy.test.ts @@ -1,4 +1,5 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, browseBin, runId, evalsEnabled, @@ -67,7 +68,7 @@ Do NOT use AskUserQuestion. Do NOT run gh or fly commands.`, workingDirectory: landDir, maxTurns: 20, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], - timeout: 120_000, + timeout: JUDGE_MS, testName: 'land-and-deploy-workflow', runId, }); @@ -85,7 +86,7 @@ Do NOT use AskUserQuestion. Do NOT run gh or fly commands.`, const reportDir = path.join(landDir, '.gstack', 'deploy-reports'); expect(fs.existsSync(reportDir)).toBe(true); - }, 180_000); + }, CAPTURE_MS); }); // --- Land-and-Deploy First-Run E2E --- @@ -148,7 +149,7 @@ Just demonstrate the first-run dry-run output.`, workingDirectory: firstRunDir, maxTurns: 20, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], - timeout: 120_000, + timeout: JUDGE_MS, testName: 'land-and-deploy-first-run', runId, }); @@ -167,7 +168,7 @@ Just demonstrate the first-run dry-run output.`, const reportContent = fs.readFileSync(path.join(reportDir, reportFiles[0]), 'utf-8'); const hasPlatform = reportContent.toLowerCase().includes('fly') || reportContent.toLowerCase().includes('first-run-app'); expect(hasPlatform).toBe(true); - }, 180_000); + }, CAPTURE_MS); }); // --- Land-and-Deploy Review Gate E2E --- @@ -226,7 +227,7 @@ Show what the readiness gate output would look like.`, workingDirectory: reviewDir, maxTurns: 15, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], - timeout: 120_000, + timeout: JUDGE_MS, testName: 'land-and-deploy-review-gate', runId, }); @@ -246,7 +247,7 @@ Show what the readiness gate output would look like.`, const hasReviewMention = reportContent.toLowerCase().includes('review') || reportContent.toLowerCase().includes('not run'); expect(hasReviewMention).toBe(true); - }, 180_000); + }, CAPTURE_MS); }); // --- Canary skill E2E --- @@ -294,7 +295,7 @@ Just create the directory structure and report files showing the correct schema. workingDirectory: canaryDir, maxTurns: 15, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'], - timeout: 120_000, + timeout: JUDGE_MS, testName: 'canary-workflow', runId, }); @@ -307,7 +308,7 @@ Just create the directory structure and report files showing the correct schema. const reportDir = path.join(canaryDir, '.gstack', 'canary-reports'); const files = fs.readdirSync(reportDir, { recursive: true }) as string[]; expect(files.length).toBeGreaterThan(0); - }, 180_000); + }, CAPTURE_MS); }); // --- Benchmark skill E2E --- @@ -357,7 +358,7 @@ Just create the files showing the correct schema and report format.`, workingDirectory: benchDir, maxTurns: 15, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'], - timeout: 120_000, + timeout: JUDGE_MS, testName: 'benchmark-workflow', runId, }); @@ -372,7 +373,7 @@ Just create the files showing the correct schema and report format.`, const files = fs.readdirSync(baselineDir); expect(files.length).toBeGreaterThan(0); } - }, 180_000); + }, CAPTURE_MS); }); // --- Setup-Deploy skill E2E --- @@ -418,7 +419,7 @@ Just detect the platform and write the config.`, workingDirectory: setupDir, maxTurns: 15, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], - timeout: 120_000, + timeout: JUDGE_MS, testName: 'setup-deploy-workflow', runId, }); @@ -434,7 +435,7 @@ Just detect the platform and write the config.`, expect(content.toLowerCase()).toContain('fly'); expect(content).toContain('my-cool-app'); expect(content).toContain('Deploy Configuration'); - }, 180_000); + }, CAPTURE_MS); }); // Module-level afterAll — finalize eval collector after all tests complete diff --git a/test/skill-e2e-design.test.ts b/test/skill-e2e-design.test.ts index 221e39aa4..9c97fe665 100644 --- a/test/skill-e2e-design.test.ts +++ b/test/skill-e2e-design.test.ts @@ -1,4 +1,5 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { callJudge } from './helpers/llm-judge'; import { @@ -113,7 +114,7 @@ Skip research — work from your design knowledge. Skip the font preview page. S Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`, workingDirectory: designDir, maxTurns: 20, - timeout: 360_000, + timeout: CAPTURE_LONG_MS, testName: 'design-consultation-core', runId, model: 'claude-opus-4-7', @@ -178,7 +179,7 @@ Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`, const claude = fs.readFileSync(claudePath, 'utf-8'); expect(claude.toLowerCase()).toContain('design.md'); } - }, 420_000); + }, CAPTURE_LONG_MS); testConcurrentIfSelected('design-consultation-research', async () => { // Test WebSearch integration — research phase only, no DESIGN.md generation @@ -202,7 +203,7 @@ Do NOT generate a full DESIGN.md — just research notes.`, // queued past the budget under concurrent API load. 90s budgets cannot // absorb one slow first completion; 300s is the repo's standard floor // for CI SDK tests. Outer timeout below rises to 360s for headroom. - timeout: 300_000, + timeout: CAPTURE_MS, testName: 'design-consultation-research', runId, }); @@ -232,7 +233,7 @@ Do NOT generate a full DESIGN.md — just research notes.`, } try { fs.rmSync(researchDir, { recursive: true, force: true }); } catch {} - }, 360_000); + }, CAPTURE_LONG_MS); testConcurrentIfSelected('design-consultation-existing', async () => { // Pre-create a minimal DESIGN.md (independent of core test) @@ -250,7 +251,7 @@ There is already a DESIGN.md in this repo. Update it with a complete design syst Skip research. Skip font preview. Skip any AskUserQuestion calls — this is non-interactive.`, workingDirectory: designDir, maxTurns: 20, - timeout: 360_000, + timeout: CAPTURE_LONG_MS, testName: 'design-consultation-existing', runId, model: 'claude-opus-4-7', @@ -279,7 +280,7 @@ Skip research. Skip font preview. Skip any AskUserQuestion calls — this is non expect(hasColor).toBe(true); expect(hasSpacing).toBe(true); } - }, 420_000); + }, CAPTURE_LONG_MS); testConcurrentIfSelected('design-consultation-preview', async () => { // Test preview HTML generation only — no DESIGN.md (covered by core test) @@ -302,7 +303,7 @@ Do NOT write DESIGN.md — only the preview HTML.`, maxTurns: 8, // 300s, not 90s: this is the test that failed 3x at 0 turns/$0.00/93s // on PR #2533 CI — see the research test's comment for the class. - timeout: 300_000, + timeout: CAPTURE_MS, testName: 'design-consultation-preview', runId, }); @@ -331,7 +332,7 @@ Do NOT write DESIGN.md — only the preview HTML.`, } try { fs.rmSync(previewDir, { recursive: true, force: true }); } catch {} - }, 360_000); + }, CAPTURE_LONG_MS); }); // --- Plan Design Review E2E (plan-mode) --- @@ -398,7 +399,7 @@ Skip the preamble bash block. Skip any AskUserQuestion calls — this is non-int IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan review, not a live site audit. Just read the plan file, review it, and edit it to fix the gaps.`, workingDirectory: reviewDir, maxTurns: 15, - timeout: 300_000, + timeout: CAPTURE_MS, testName: 'plan-design-review-plan-mode', runId, }); @@ -437,7 +438,7 @@ IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan } finally { try { fs.rmSync(reviewDir, { recursive: true, force: true }); } catch {} } - }, 360_000); + }, CAPTURE_LONG_MS); testConcurrentIfSelected('plan-design-review-no-ui-scope', async () => { const reviewDir = setupReviewDir(); @@ -472,7 +473,7 @@ Skip the preamble bash block. Skip any AskUserQuestion calls — this is non-int IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan review, not a live site audit.`, workingDirectory: reviewDir, maxTurns: 10, - timeout: 180_000, + timeout: CAPTURE_MS, testName: 'plan-design-review-no-ui-scope', runId, }); @@ -496,7 +497,7 @@ IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan } finally { try { fs.rmSync(reviewDir, { recursive: true, force: true }); } catch {} } - }, 240_000); + }, CAPTURE_MS); }); // --- Design Review E2E (live-site audit + fix) --- @@ -602,7 +603,7 @@ Read design-review/SKILL.md for the design review + fix workflow. Review the site at ${serverUrl}. Use --quick mode. Skip any AskUserQuestion calls — this is non-interactive. Fix up to 3 issues max. Write your report to ./design-audit.md.`, workingDirectory: qaDesignDir, maxTurns: 30, - timeout: 360_000, + timeout: CAPTURE_LONG_MS, testName: 'design-review-fix', runId, }); @@ -634,7 +635,7 @@ Review the site at ${serverUrl}. Use --quick mode. Skip any AskUserQuestion call console.warn('No design-audit.md generated'); } console.log(`Design fix commits: ${designFixCommits.length}`); - }, 420_000); + }, CAPTURE_LONG_MS); }); // Module-level afterAll — finalize eval collector after all tests complete diff --git a/test/skill-e2e-diagram.test.ts b/test/skill-e2e-diagram.test.ts index 43f3dddfc..1e676510f 100644 --- a/test/skill-e2e-diagram.test.ts +++ b/test/skill-e2e-diagram.test.ts @@ -17,6 +17,7 @@ * with its preamble. */ import { describe, expect } from 'bun:test'; +import { CAPTURE_MS } from './helpers/eval-budgets'; import * as fs from 'node:fs'; import * as path from 'node:path'; import * as os from 'node:os'; @@ -73,7 +74,7 @@ describeIfSelected('/diagram skill E2E', ['diagram-triplet', 'diagram-authoring- workingDirectory: dir, maxTurns: 25, allowedTools: ['Bash', 'Read', 'Write'], - timeout: 240_000, + timeout: CAPTURE_MS, testName: 'diagram-triplet', runId, }); @@ -98,7 +99,7 @@ describeIfSelected('/diagram skill E2E', ['diagram-triplet', 'diagram-authoring- } finally { try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } } - }, 300_000); + }, CAPTURE_MS); testConcurrentIfSelected('diagram-authoring-quality', async () => { const dir = setupDir('diagram-quality-'); @@ -111,7 +112,7 @@ describeIfSelected('/diagram skill E2E', ['diagram-triplet', 'diagram-authoring- workingDirectory: dir, maxTurns: 25, allowedTools: ['Bash', 'Read', 'Write'], - timeout: 240_000, + timeout: CAPTURE_MS, testName: 'diagram-authoring-quality', runId, }); @@ -149,5 +150,5 @@ Respond with JSON: {"score": N, "reasoning": "..."}`, } finally { try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } } - }, 300_000); + }, CAPTURE_MS); }); diff --git a/test/skill-e2e-first-task-scaffold.test.ts b/test/skill-e2e-first-task-scaffold.test.ts index 871781fcb..619eba1ce 100644 --- a/test/skill-e2e-first-task-scaffold.test.ts +++ b/test/skill-e2e-first-task-scaffold.test.ts @@ -14,6 +14,7 @@ */ import { expect, afterAll } from 'bun:test'; +import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; @@ -55,7 +56,7 @@ async function detectVia(workDir: string, testName: string): Promise { workingDirectory: workDir, maxTurns: 3, allowedTools: ['Bash'], - timeout: 120_000, + timeout: JUDGE_MS, testName, runId, model: MODEL, @@ -91,7 +92,7 @@ describeIfSelected('first-run scaffold detection (E2E)', ['first-task-scaffold'] fs.rmSync(nodeDir, { recursive: true, force: true }); fs.rmSync(greenDir, { recursive: true, force: true }); } - }, 300_000); + }, CAPTURE_MS); }); afterAll(() => finalizeEvalCollector(evalCollector)); diff --git a/test/skill-e2e-gbrain-roundtrip-local.test.ts b/test/skill-e2e-gbrain-roundtrip-local.test.ts index 46e22b985..56fab1a73 100644 --- a/test/skill-e2e-gbrain-roundtrip-local.test.ts +++ b/test/skill-e2e-gbrain-roundtrip-local.test.ts @@ -29,6 +29,7 @@ */ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { JUDGE_MS } from './helpers/eval-budgets'; import { execFileSync } from 'child_process'; import { mkdtempSync, rmSync } from 'fs'; import { tmpdir } from 'os'; @@ -156,7 +157,7 @@ ${body}`; expect(retrieved).not.toContain('page_not_found'); expect(retrieved).not.toContain('Page not found'); }, - 120_000, + JUDGE_MS, ); }, ); diff --git a/test/skill-e2e-hermetic-canary.test.ts b/test/skill-e2e-hermetic-canary.test.ts index 06f1dc302..d14f2716a 100644 --- a/test/skill-e2e-hermetic-canary.test.ts +++ b/test/skill-e2e-hermetic-canary.test.ts @@ -31,6 +31,7 @@ */ import { expect, afterAll } from 'bun:test'; +import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; @@ -94,7 +95,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic- workingDirectory: workDir, maxTurns: 3, allowedTools: ['Bash'], - timeout: 120_000, + timeout: JUDGE_MS, testName: 'hermetic-canary', runId, model: CANARY_MODEL, @@ -129,7 +130,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic- } fs.rmSync(workDir, { recursive: true, force: true }); } - }, 180_000); + }, CAPTURE_MS); testIfSelected('hermetic-sentinel', async () => { if (!process.env.ANTHROPIC_API_KEY) { @@ -158,7 +159,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic- workingDirectory: workDir, maxTurns: 3, allowedTools: ['Bash'], - timeout: 120_000, + timeout: JUDGE_MS, testName: 'hermetic-sentinel', runId, model: CANARY_MODEL, @@ -188,7 +189,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic- fs.rmSync(workDir, { recursive: true, force: true }); fs.rmSync(poisonRoot, { recursive: true, force: true }); } - }, 180_000); + }, CAPTURE_MS); }); afterAll(() => finalizeEvalCollector(evalCollector)); diff --git a/test/skill-e2e-ios-device.test.ts b/test/skill-e2e-ios-device.test.ts index 200cb6557..678d80be7 100644 --- a/test/skill-e2e-ios-device.test.ts +++ b/test/skill-e2e-ios-device.test.ts @@ -21,6 +21,7 @@ // intentionally machine-specific. import { describe, test, expect } from 'bun:test'; +import { CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { spawnSync } from 'child_process'; import { cpSync, existsSync, mkdtempSync, readFileSync, rmSync, unlinkSync } from 'fs'; import { tmpdir } from 'os'; @@ -856,7 +857,7 @@ describe('ios device deployment (explicit opt-in)', () => { keepalive?.stop(); rmSync(workDir, { recursive: true, force: true }); } - }, 600_000); + }, CAPTURE_LONG_MS); }); // Always-on instructions if not paired. Surfaces actionable steps even when diff --git a/test/skill-e2e-ios-swift-build.test.ts b/test/skill-e2e-ios-swift-build.test.ts index 253b4cb84..8fd126e15 100644 --- a/test/skill-e2e-ios-swift-build.test.ts +++ b/test/skill-e2e-ios-swift-build.test.ts @@ -18,6 +18,7 @@ // gated (no compilation step for DebugBridgeCore/UI) import { describe, test, expect } from 'bun:test'; +import { CAPTURE_MS } from './helpers/eval-budgets'; import { spawnSync } from 'child_process'; import { readFileSync } from 'fs'; import { join } from 'path'; @@ -321,7 +322,7 @@ describeIfSwift('swift build invariants', () => { console.error('swift build stderr:', r.stderr?.toString().slice(0, 4000)); } expect(r.status).toBe(0); - }, 180_000); + }, CAPTURE_MS); test('XCTest suite for StateServer passes (validates real Swift impl)', () => { const r = spawnSync('swift', ['test', '--filter', 'DebugBridgeCoreTests'], { @@ -342,7 +343,7 @@ describeIfSwift('swift build invariants', () => { // Guard against an empty pass-by-no-tests (filter typo / target rename): // we expect at least one StateServer smoke test to actually execute. expect(combined).toContain('StateServerSmokeTests'); - }, 240_000); + }, CAPTURE_MS); // Codex-flagged: Release-build guard must be STRUCTURAL, not advisory. // The Package.swift's `.when(configuration: .debug)` setting causes Swift @@ -386,5 +387,5 @@ describeIfSwift('swift build invariants', () => { } } expect(foundForbidden).toBe(0); - }, 300_000); + }, CAPTURE_MS); }); diff --git a/test/skill-e2e-learnings.test.ts b/test/skill-e2e-learnings.test.ts index 8b6dec944..8672ef3c2 100644 --- a/test/skill-e2e-learnings.test.ts +++ b/test/skill-e2e-learnings.test.ts @@ -1,4 +1,5 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, runId, evalsEnabled, @@ -103,7 +104,7 @@ IMPORTANT: workingDirectory: workDir, maxTurns: 15, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], - timeout: 120_000, + timeout: JUDGE_MS, testName: 'learnings-show', runId, }); @@ -134,5 +135,5 @@ IMPORTANT: } else { console.warn(`Only ${foundCount}/3 learnings found (N+1: ${mentionsNPlusOne}, cache: ${mentionsCache}, rubocop: ${mentionsRubocop})`); } - }, 180_000); + }, CAPTURE_MS); }); diff --git a/test/skill-e2e-office-hours-auto-mode.test.ts b/test/skill-e2e-office-hours-auto-mode.test.ts index 49eb4f764..d32e835f4 100644 --- a/test/skill-e2e-office-hours-auto-mode.test.ts +++ b/test/skill-e2e-office-hours-auto-mode.test.ts @@ -17,6 +17,7 @@ */ import { test, expect } from 'bun:test'; +import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillObservation, planFileHasDecisionsSection } from './helpers/claude-pty-runner'; @@ -30,7 +31,7 @@ describeE2E('office-hours AskUserQuestion-blocked smoke (gate)', () => { skillName: 'office-hours', inPlanMode: true, extraArgs: ['--disallowedTools', 'AskUserQuestion'], - timeoutMs: 300_000, + timeoutMs: CAPTURE_MS, }); if ( @@ -55,5 +56,5 @@ describeE2E('office-hours AskUserQuestion-blocked smoke (gate)', () => { } } expect(['asked', 'plan_ready']).toContain(obs.outcome); - }, 360_000); + }, CAPTURE_LONG_MS); }); diff --git a/test/skill-e2e-office-hours-brain-writeback.test.ts b/test/skill-e2e-office-hours-brain-writeback.test.ts index 0dc085000..d8b711803 100644 --- a/test/skill-e2e-office-hours-brain-writeback.test.ts +++ b/test/skill-e2e-office-hours-brain-writeback.test.ts @@ -36,6 +36,7 @@ */ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { execFileSync, spawnSync } from 'child_process'; import { chmodSync, @@ -222,7 +223,7 @@ Generate the design doc per Phase 5. The feature-slug value to substitute into t This is a test of the brain-writeback path. Do NOT skip the gbrain save step under any circumstance — the runtime guard ("skip if gbrain not on PATH") does NOT apply here because gbrain IS available. Do NOT explore gbrain --help; follow the SAVE_RESULTS template's exact CLI shape. If you encounter any AskUserQuestion, auto-decide recommended.`, workingDirectory: workDir, maxTurns: 12, - timeout: 360_000, + timeout: CAPTURE_LONG_MS, testName: 'office-hours-brain-writeback', runId, model: 'claude-sonnet-4-6', @@ -315,7 +316,7 @@ This is a test of the brain-writeback path. Do NOT skip the gbrain save step und ); } }, - 420_000, + CAPTURE_LONG_MS, ); }, ); diff --git a/test/skill-e2e-office-hours-phase4.test.ts b/test/skill-e2e-office-hours-phase4.test.ts index 5777008b2..fae8333f6 100644 --- a/test/skill-e2e-office-hours-phase4.test.ts +++ b/test/skill-e2e-office-hours-phase4.test.ts @@ -20,6 +20,7 @@ * test turns out stable. */ import { expect, beforeAll, afterAll } from 'bun:test'; +import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, runId, @@ -133,7 +134,7 @@ ${captureInstruction(outFile)} After writing the file with that ONE Phase 4 question, stop. Do not continue to Phase 4.5 or Phase 5.`, workingDirectory: workDir, maxTurns: 12, - timeout: 300_000, + timeout: CAPTURE_MS, testName: 'office-hours-phase4-fork', runId, model: 'claude-opus-4-7', @@ -162,7 +163,7 @@ After writing the file with that ONE Phase 4 question, stop. Do not continue to result, passed: ['success', 'error_max_turns'].includes(result.exitReason), }); - }, 360_000); + }, CAPTURE_LONG_MS); }); afterAll(async () => { diff --git a/test/skill-e2e-office-hours.test.ts b/test/skill-e2e-office-hours.test.ts index d2e7700a9..c3ad6c57b 100644 --- a/test/skill-e2e-office-hours.test.ts +++ b/test/skill-e2e-office-hours.test.ts @@ -10,6 +10,7 @@ */ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, browseBin, runId, evalsEnabled, @@ -71,7 +72,7 @@ Assume the founder has already answered Q1 (strongest evidence = "got on a waitl Write Q3 output — the forcing question you would ask this founder — to ${workDir}/q3.md. Write ONLY the question prose. No conversational wrapper, no meta-commentary, no Q1/Q2 recap.`, workingDirectory: workDir, maxTurns: 8, - timeout: 240_000, + timeout: CAPTURE_MS, testName: 'office-hours-forcing-energy', runId, model: 'claude-sonnet-4-6', @@ -94,7 +95,7 @@ Write Q3 output — the forcing question you would ask this founder — to ${wor console.log('Forcing energy scores:', JSON.stringify(scores, null, 2)); expect(scores.axis_a).toBeGreaterThanOrEqual(4); // stacking_preserved expect(scores.axis_b).toBeGreaterThanOrEqual(4); // domain_matched_consequence - }, 360_000); + }, CAPTURE_LONG_MS); }); // --- Office Hours builder-mode wildness --- @@ -143,7 +144,7 @@ The user has confirmed the basic idea is "TypeScript + D3 web tool, start with J Write your response — the three adjacent unlocks — to ${workDir}/unlocks.md. Write ONLY the response prose. No meta-commentary, no mode recap. Lead with the fun; let me edit it down later.`, workingDirectory: workDir, maxTurns: 8, - timeout: 240_000, + timeout: CAPTURE_MS, testName: 'office-hours-builder-wildness', runId, model: 'claude-sonnet-4-6', @@ -166,7 +167,7 @@ Write your response — the three adjacent unlocks — to ${workDir}/unlocks.md. console.log('Builder wildness scores:', JSON.stringify(scores, null, 2)); expect(scores.axis_a).toBeGreaterThanOrEqual(4); // unexpected_combinations expect(scores.axis_b).toBeGreaterThanOrEqual(4); // excitement_over_optimization - }, 360_000); + }, CAPTURE_LONG_MS); }); // Finalize eval collector for this file diff --git a/test/skill-e2e-opus-47.test.ts b/test/skill-e2e-opus-47.test.ts index 8f4c0f3e6..7177b2898 100644 --- a/test/skill-e2e-opus-47.test.ts +++ b/test/skill-e2e-opus-47.test.ts @@ -18,6 +18,7 @@ */ import { describe, test, expect, afterAll } from 'bun:test'; +import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { EvalCollector } from './helpers/eval-store'; import { extractSkillHead } from './helpers/skill-fixture'; @@ -204,7 +205,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => { workingDirectory: armA, maxTurns: 5, allowedTools: ['Read', 'Bash', 'Glob', 'Grep'], - timeout: 90_000, + timeout: JUDGE_MS, testName: 'fanout-arm-overlay-on', runId, model: OPUS_47, @@ -214,7 +215,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => { workingDirectory: armB, maxTurns: 5, allowedTools: ['Read', 'Bash', 'Glob', 'Grep'], - timeout: 90_000, + timeout: JUDGE_MS, testName: 'fanout-arm-overlay-off', runId, model: OPUS_47, @@ -262,7 +263,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => { fs.rmSync(armB, { recursive: true, force: true }); } }, - 240_000, + CAPTURE_MS, ); test( @@ -281,7 +282,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => { workingDirectory: root, maxTurns: 3, allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], - timeout: 90_000, + timeout: JUDGE_MS, testName: `routing-${c.name}`, runId, model: OPUS_47, @@ -348,6 +349,6 @@ describeE2E('Opus 4.7 overlay behavior evals', () => { fs.rmSync(root, { recursive: true, force: true }); } }, - 360_000, + CAPTURE_LONG_MS, ); }); diff --git a/test/skill-e2e-plan-ceo-finding-floor.test.ts b/test/skill-e2e-plan-ceo-finding-floor.test.ts index 01753543e..742b51da6 100644 --- a/test/skill-e2e-plan-ceo-finding-floor.test.ts +++ b/test/skill-e2e-plan-ceo-finding-floor.test.ts @@ -5,6 +5,7 @@ */ import { test } from 'bun:test'; +import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner'; import { FORCING_FLOOR_CEO } from './fixtures/forcing-finding-seeds'; @@ -22,7 +23,7 @@ describeE2E('/plan-ceo-review AskUserQuestion floor (gate)', () => { // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). cwd: process.cwd(), - timeoutMs: 600_000, + timeoutMs: CAPTURE_LONG_MS, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, }); @@ -34,6 +35,6 @@ describeE2E('/plan-ceo-review AskUserQuestion floor (gate)', () => { ); } }, - 660_000, + PTY_MS, ); }); diff --git a/test/skill-e2e-plan-ceo-mode-routing.test.ts b/test/skill-e2e-plan-ceo-mode-routing.test.ts index 8b54a4d3c..ff113278c 100644 --- a/test/skill-e2e-plan-ceo-mode-routing.test.ts +++ b/test/skill-e2e-plan-ceo-mode-routing.test.ts @@ -31,6 +31,7 @@ */ import { test } from 'bun:test'; +import { CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import { launchClaudePty, @@ -151,7 +152,7 @@ describeE2E('/plan-ceo-review mode routing (gate)', () => { async () => { const session = await launchClaudePty({ permissionMode: 'plan', - timeoutMs: 540_000, + timeoutMs: CAPTURE_LONG_MS, seedSkills: true, }); try { @@ -207,7 +208,7 @@ describeE2E('/plan-ceo-review mode routing (gate)', () => { await session.close(); } }, - 600_000, + CAPTURE_LONG_MS, ); } }); diff --git a/test/skill-e2e-plan-ceo-plan-mode.test.ts b/test/skill-e2e-plan-ceo-plan-mode.test.ts index 0a358a0b1..59b96c829 100644 --- a/test/skill-e2e-plan-ceo-plan-mode.test.ts +++ b/test/skill-e2e-plan-ceo-plan-mode.test.ts @@ -34,6 +34,7 @@ */ import { test } from 'bun:test'; +import { CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillObservation, @@ -77,5 +78,5 @@ describeE2E('plan-ceo-review plan-mode smoke (gate)', () => { ); } assertReportAtBottomIfPlanWritten(obs); - }, 480_000); + }, CAPTURE_LONG_MS); }); diff --git a/test/skill-e2e-plan-ceo-review-section-loading.test.ts b/test/skill-e2e-plan-ceo-review-section-loading.test.ts index 7074fc141..6d59bd730 100644 --- a/test/skill-e2e-plan-ceo-review-section-loading.test.ts +++ b/test/skill-e2e-plan-ceo-review-section-loading.test.ts @@ -25,6 +25,7 @@ */ import { test, expect } from 'bun:test'; +import { CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import { setupSkillDir, @@ -87,6 +88,6 @@ describeE2E('/plan-ceo-review section-loading E2E (periodic, SDK capture)', () = // Guard against an empty pass: the report must have real content. expect(output.trim().length).toBeGreaterThan(200); }, - 360_000, + CAPTURE_LONG_MS, ); }); diff --git a/test/skill-e2e-plan-design-finding-floor.test.ts b/test/skill-e2e-plan-design-finding-floor.test.ts index 3f05ab36b..255f7db8c 100644 --- a/test/skill-e2e-plan-design-finding-floor.test.ts +++ b/test/skill-e2e-plan-design-finding-floor.test.ts @@ -5,6 +5,7 @@ */ import { test } from 'bun:test'; +import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner'; import { FORCING_FLOOR_DESIGN } from './fixtures/forcing-finding-seeds'; @@ -22,7 +23,7 @@ describeE2E('/plan-design-review AskUserQuestion floor (periodic)', () => { // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). cwd: process.cwd(), - timeoutMs: 600_000, + timeoutMs: CAPTURE_LONG_MS, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, }); @@ -34,6 +35,6 @@ describeE2E('/plan-design-review AskUserQuestion floor (periodic)', () => { ); } }, - 660_000, + PTY_MS, ); }); diff --git a/test/skill-e2e-plan-design-plan-mode.test.ts b/test/skill-e2e-plan-design-plan-mode.test.ts index 7d2a373fc..6ae438ae4 100644 --- a/test/skill-e2e-plan-design-plan-mode.test.ts +++ b/test/skill-e2e-plan-design-plan-mode.test.ts @@ -10,6 +10,7 @@ */ import { test, expect } from 'bun:test'; +import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillObservation, @@ -46,7 +47,7 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => { const obs = await runPlanSkillObservation({ skillName: 'plan-design-review', inPlanMode: true, - timeoutMs: 300_000, + timeoutMs: CAPTURE_MS, }); if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') { @@ -59,7 +60,7 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => { } expect(['asked', 'plan_ready']).toContain(obs.outcome); assertReportAtBottomIfPlanWritten(obs); - }, 360_000); + }, CAPTURE_LONG_MS); // Plan-mode scope-gate bypass: with a seeded UI-heavy plan in plan mode, // the gate must NOT render its "What should I review?" menu — it @@ -71,7 +72,7 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => { skillName: 'plan-design-review', inPlanMode: true, initialPlanContent: SEED_PLAN_UI_HEAVY, - timeoutMs: 300_000, + timeoutMs: CAPTURE_MS, }); if ( @@ -95,5 +96,5 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => { // though the seed arrives as a pasted user message). expect(obs.scopeGateQuestionObserved ?? false).toBe(false); expect(obs.scopeGateAutoSelectObserved ?? false).toBe(true); - }, 360_000); + }, CAPTURE_LONG_MS); }); diff --git a/test/skill-e2e-plan-design-with-ui.test.ts b/test/skill-e2e-plan-design-with-ui.test.ts index bdf25c75e..580891747 100644 --- a/test/skill-e2e-plan-design-with-ui.test.ts +++ b/test/skill-e2e-plan-design-with-ui.test.ts @@ -20,6 +20,7 @@ */ import { test } from 'bun:test'; +import { PTY_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import * as path from 'path'; import { @@ -46,7 +47,7 @@ describeE2E('/plan-design-review with UI scope (gate)', () => { // LIVE-REPO CWD: PTY session needs the repo cwd — skill registry, // hermetic pre-trusted dir, and the repo-relative fixture path above. cwd: ROOT, - timeoutMs: 720_000, + timeoutMs: PTY_MS, seedSkills: true, }); @@ -152,6 +153,6 @@ describeE2E('/plan-design-review with UI scope (gate)', () => { ); } }, - 780_000, + PTY_MS, ); }); diff --git a/test/skill-e2e-plan-devex-finding-floor.test.ts b/test/skill-e2e-plan-devex-finding-floor.test.ts index e1da9fe31..943ad8dc5 100644 --- a/test/skill-e2e-plan-devex-finding-floor.test.ts +++ b/test/skill-e2e-plan-devex-finding-floor.test.ts @@ -5,6 +5,7 @@ */ import { test } from 'bun:test'; +import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner'; import { FORCING_FLOOR_DEVEX } from './fixtures/forcing-finding-seeds'; @@ -22,7 +23,7 @@ describeE2E('/plan-devex-review AskUserQuestion floor (gate)', () => { // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). cwd: process.cwd(), - timeoutMs: 600_000, + timeoutMs: CAPTURE_LONG_MS, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, }); @@ -34,6 +35,6 @@ describeE2E('/plan-devex-review AskUserQuestion floor (gate)', () => { ); } }, - 660_000, + PTY_MS, ); }); diff --git a/test/skill-e2e-plan-devex-plan-mode.test.ts b/test/skill-e2e-plan-devex-plan-mode.test.ts index 2f1c73a8e..340db45ca 100644 --- a/test/skill-e2e-plan-devex-plan-mode.test.ts +++ b/test/skill-e2e-plan-devex-plan-mode.test.ts @@ -6,6 +6,7 @@ */ import { test, expect } from 'bun:test'; +import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillObservation, @@ -20,7 +21,7 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => { const obs = await runPlanSkillObservation({ skillName: 'plan-devex-review', inPlanMode: true, - timeoutMs: 300_000, + timeoutMs: CAPTURE_MS, }); if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') { @@ -33,7 +34,7 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => { } expect(['asked', 'plan_ready']).toContain(obs.outcome); assertReportAtBottomIfPlanWritten(obs); - }, 360_000); + }, CAPTURE_LONG_MS); // v1.21+ regression: see skill-e2e-plan-ceo-plan-mode.test.ts for the // contract. Pass envelope is ['asked', 'plan_ready']; failure signals @@ -44,7 +45,7 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => { skillName: 'plan-devex-review', inPlanMode: true, extraArgs: ['--disallowedTools', 'AskUserQuestion'], - timeoutMs: 300_000, + timeoutMs: CAPTURE_MS, }); if ( @@ -70,5 +71,5 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => { } expect(['asked', 'plan_ready']).toContain(obs.outcome); assertReportAtBottomIfPlanWritten(obs); - }, 360_000); + }, CAPTURE_LONG_MS); }); diff --git a/test/skill-e2e-plan-eng-finding-floor.test.ts b/test/skill-e2e-plan-eng-finding-floor.test.ts index 8e238f35f..f57c74312 100644 --- a/test/skill-e2e-plan-eng-finding-floor.test.ts +++ b/test/skill-e2e-plan-eng-finding-floor.test.ts @@ -16,6 +16,7 @@ */ import { test } from 'bun:test'; +import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner'; import { FORCING_FLOOR_ENG } from './fixtures/forcing-finding-seeds'; @@ -33,7 +34,7 @@ describeE2E('/plan-eng-review AskUserQuestion floor (periodic)', () => { // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). cwd: process.cwd(), - timeoutMs: 600_000, + timeoutMs: CAPTURE_LONG_MS, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, }); @@ -49,6 +50,6 @@ describeE2E('/plan-eng-review AskUserQuestion floor (periodic)', () => { ); } }, - 660_000, + PTY_MS, ); }); diff --git a/test/skill-e2e-plan-eng-plan-mode.test.ts b/test/skill-e2e-plan-eng-plan-mode.test.ts index c632abe6f..85eec6cc4 100644 --- a/test/skill-e2e-plan-eng-plan-mode.test.ts +++ b/test/skill-e2e-plan-eng-plan-mode.test.ts @@ -6,6 +6,7 @@ */ import { test, expect } from 'bun:test'; +import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillObservation, @@ -50,7 +51,7 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => { const obs = await runPlanSkillObservation({ skillName: 'plan-eng-review', inPlanMode: true, - timeoutMs: 300_000, + timeoutMs: CAPTURE_MS, }); if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') { @@ -63,7 +64,7 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => { } expect(['asked', 'plan_ready']).toContain(obs.outcome); assertReportAtBottomIfPlanWritten(obs); - }, 360_000); + }, CAPTURE_LONG_MS); // D3-B / D4-B: when a plan with guaranteed-finding-triggering complexity // is seeded, the skill MUST fire AskUserQuestion (or fall back to a @@ -79,7 +80,7 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => { // must use mcp__*__AskUserQuestion (outcome='asked') or fall back to // writing Decisions ('plan_ready'). extraArgs: ['--disallowedTools', 'AskUserQuestion'], - timeoutMs: 300_000, + timeoutMs: CAPTURE_MS, }); if ( @@ -118,5 +119,5 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => { // question. expect(obs.scopeGateQuestionObserved ?? false).toBe(false); expect(obs.scopeGateAutoSelectObserved ?? false).toBe(true); - }, 360_000); + }, CAPTURE_LONG_MS); }); diff --git a/test/skill-e2e-plan-format.test.ts b/test/skill-e2e-plan-format.test.ts index 8913348a6..05081bc65 100644 --- a/test/skill-e2e-plan-format.test.ts +++ b/test/skill-e2e-plan-format.test.ts @@ -18,6 +18,7 @@ * accordingly. */ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { CAPTURE_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, runId, @@ -134,7 +135,7 @@ ${captureInstruction(outFile)} After writing the file, stop. Do not continue the review.`, workingDirectory: planDir, maxTurns: 10, - timeout: 240_000, + timeout: CAPTURE_MS, testName: 'plan-ceo-review-format-mode', runId, model: 'claude-opus-4-7', @@ -160,7 +161,7 @@ After writing the file, stop. Do not continue the review.`, result, passed: ['success', 'error_max_turns'].includes(result.exitReason), }); - }, 300_000); + }, CAPTURE_MS); }); // --- Case 2: plan-ceo-review approach menu (coverage-differentiated) --- @@ -191,7 +192,7 @@ ${captureInstruction(outFile)} After writing the file, stop. Do not continue the review.`, workingDirectory: planDir, maxTurns: 10, - timeout: 240_000, + timeout: CAPTURE_MS, testName: 'plan-ceo-review-format-approach', runId, model: 'claude-opus-4-7', @@ -216,7 +217,7 @@ After writing the file, stop. Do not continue the review.`, result, passed: ['success', 'error_max_turns'].includes(result.exitReason), }); - }, 300_000); + }, CAPTURE_MS); }); // --- Case 3: plan-eng-review coverage-differentiated per-issue AskUserQuestion --- @@ -250,7 +251,7 @@ ${captureInstruction(outFile)} After writing the file with that ONE question, stop. Do not continue the review.`, workingDirectory: planDir, maxTurns: 10, - timeout: 240_000, + timeout: CAPTURE_MS, testName: 'plan-eng-review-format-coverage', runId, model: 'claude-opus-4-7', @@ -275,7 +276,7 @@ After writing the file with that ONE question, stop. Do not continue the review. result, passed: ['success', 'error_max_turns'].includes(result.exitReason), }); - }, 300_000); + }, CAPTURE_MS); }); // --- Case 4: plan-eng-review kind-differentiated per-issue AskUserQuestion --- @@ -306,7 +307,7 @@ ${captureInstruction(outFile)} After writing the file with that ONE question, stop. Do not continue the review.`, workingDirectory: planDir, maxTurns: 10, - timeout: 240_000, + timeout: CAPTURE_MS, testName: 'plan-eng-review-format-kind', runId, model: 'claude-opus-4-7', @@ -332,7 +333,7 @@ After writing the file with that ONE question, stop. Do not continue the review. result, passed: ['success', 'error_max_turns'].includes(result.exitReason), }); - }, 300_000); + }, CAPTURE_MS); }); afterAll(async () => { diff --git a/test/skill-e2e-plan-mode-no-op.test.ts b/test/skill-e2e-plan-mode-no-op.test.ts index bfc18d6f6..c76c83848 100644 --- a/test/skill-e2e-plan-mode-no-op.test.ts +++ b/test/skill-e2e-plan-mode-no-op.test.ts @@ -31,6 +31,7 @@ */ import { test, expect } from 'bun:test'; +import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillObservation } from './helpers/claude-pty-runner'; @@ -62,7 +63,7 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => { const obs = await runPlanSkillObservation({ skillName, inPlanMode: false, - timeoutMs: 300_000, + timeoutMs: CAPTURE_MS, // eng/design: force the prose-fallback path. The unconditional // gate-must-ask assert below pins the render shape the detector // anchors on, and only the --disallowedTools prose fallback makes @@ -115,7 +116,7 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => { ); } } - }, 360_000); + }, CAPTURE_LONG_MS); } // Named-target exception (outside plan mode): a pasted draft IS an @@ -130,7 +131,7 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => { inPlanMode: false, initialPlanContent: NAMED_TARGET_SEED, trackTokens: [SEED_TOKEN], - timeoutMs: 300_000, + timeoutMs: CAPTURE_MS, }); if ( @@ -159,5 +160,5 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => { // unreachable outside plan mode (extractPlanFilePath only matches // plan-mode save renders). expect(obs.tokensObserved?.[SEED_TOKEN] ?? false).toBe(true); - }, 360_000); + }, CAPTURE_LONG_MS); }); diff --git a/test/skill-e2e-plan-prosons.test.ts b/test/skill-e2e-plan-prosons.test.ts index 8fb68bc09..f092eb50f 100644 --- a/test/skill-e2e-plan-prosons.test.ts +++ b/test/skill-e2e-plan-prosons.test.ts @@ -27,6 +27,7 @@ * cases will land as follow-up PRs per skill. */ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { CAPTURE_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, runId, @@ -161,7 +162,7 @@ ${captureInstruction(outFile)} After writing the file, stop.`, workingDirectory: planDir, maxTurns: 10, - timeout: 240_000, + timeout: CAPTURE_MS, testName: 'plan-review-prosons-format', runId, model: 'claude-opus-4-7', @@ -191,7 +192,7 @@ After writing the file, stop.`, // (recommended) label on one option expect(captured).toMatch(RECOMMENDED_LABEL_RE); - }, 300_000); + }, CAPTURE_MS); }); // --- Case 2: Hard-stop escape NEGATIVE (CT2) --- @@ -220,7 +221,7 @@ ${captureInstruction(outFile)} After writing the file, stop.`, workingDirectory: planDir, maxTurns: 10, - timeout: 240_000, + timeout: CAPTURE_MS, testName: 'plan-review-prosons-hardstop-neg', runId, model: 'claude-opus-4-7', @@ -241,7 +242,7 @@ After writing the file, stop.`, // Must have real pros and cons (≥2 ✅ + ≥1 ❌ per option) expect(countChars(captured, '✅')).toBeGreaterThanOrEqual(4); expect(countChars(captured, '❌')).toBeGreaterThanOrEqual(2); - }, 300_000); + }, CAPTURE_MS); }); // --- Case 3: Neutral-posture NEGATIVE (CT2) --- @@ -270,7 +271,7 @@ ${captureInstruction(outFile)} After writing the file, stop.`, workingDirectory: planDir, maxTurns: 10, - timeout: 240_000, + timeout: CAPTURE_MS, testName: 'plan-review-prosons-neutral-neg', runId, model: 'claude-opus-4-7', @@ -292,7 +293,7 @@ After writing the file, stop.`, expect(captured).toMatch(RECOMMENDED_LABEL_RE); // Recommendation line must contain "because" (concrete reason, not "no preference") expect(captured).toMatch(/[Rr]ecommendation:.*because/); - }, 300_000); + }, CAPTURE_MS); }); // --- Case 4: Hard-stop POSITIVE (escape allowed when legitimately one-sided) --- @@ -321,7 +322,7 @@ ${captureInstruction(outFile)} After writing the file, stop.`, workingDirectory: planDir, maxTurns: 10, - timeout: 240_000, + timeout: CAPTURE_MS, testName: 'plan-ceo-review-prosons-cadence', runId, model: 'claude-opus-4-7', @@ -344,7 +345,7 @@ After writing the file, stop.`, const hasEscape = HARD_STOP_ESCAPE_RE.test(captured); const hasProsAndCons = countChars(captured, '✅') >= 1 && countChars(captured, '❌') >= 1; expect(hasEscape || hasProsAndCons).toBe(true); - }, 300_000); + }, CAPTURE_MS); }); afterAll(async () => { diff --git a/test/skill-e2e-plan-tune.test.ts b/test/skill-e2e-plan-tune.test.ts index dd7502088..17a7a2a0f 100644 --- a/test/skill-e2e-plan-tune.test.ts +++ b/test/skill-e2e-plan-tune.test.ts @@ -1,4 +1,5 @@ import { beforeAll, afterAll, expect } from 'bun:test'; +import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, runId, @@ -151,7 +152,7 @@ IMPORTANT: workingDirectory: workDir, maxTurns: 15, allowedTools: ['Bash', 'Read', 'Grep', 'Glob'], - timeout: 120_000, + timeout: JUDGE_MS, testName: 'plan-tune-inspect', runId, }); @@ -184,5 +185,5 @@ IMPORTANT: if (!noticedOverride) { console.warn('Agent did not surface override/skip behavior from the log'); } - }, 180_000); + }, CAPTURE_MS); }); diff --git a/test/skill-e2e-plan.test.ts b/test/skill-e2e-plan.test.ts index 7b2badde2..6cf6cc254 100644 --- a/test/skill-e2e-plan.test.ts +++ b/test/skill-e2e-plan.test.ts @@ -1,4 +1,5 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, browseBin, runId, evalsEnabled, @@ -103,7 +104,7 @@ Focus on reviewing the plan content: architecture, error handling, security, and const review = fs.readFileSync(reviewPath, 'utf-8'); expect(review.length).toBeGreaterThan(200); } - }, 660_000); + }, PTY_MS); }); // --- Plan CEO Review (SELECTIVE EXPANSION) E2E --- @@ -171,7 +172,7 @@ Write your complete review directly to ${planDir}/review-output-selective.md Focus on reviewing the plan content: architecture, error handling, security, and performance.`, workingDirectory: planDir, maxTurns: 15, - timeout: 540_000, + timeout: CAPTURE_LONG_MS, testName: 'plan-ceo-review-selective', runId, model: 'claude-opus-4-7', @@ -188,7 +189,7 @@ Focus on reviewing the plan content: architecture, error handling, security, and const review = fs.readFileSync(reviewPath, 'utf-8'); expect(review.length).toBeGreaterThan(200); } - }, 660_000); + }, PTY_MS); }); // --- Plan CEO Review SCOPE EXPANSION energy (V1.1 mode-posture regression gate) --- @@ -239,7 +240,7 @@ Choose SCOPE EXPANSION mode. Skip any AskUserQuestion calls — this is non-inte Write your expansion proposals to ${planDir}/proposals.md with ONLY the proposal text — no conversational wrapper, no review summary, no mode analysis. Each proposal separated by "---".`, workingDirectory: planDir, maxTurns: 15, - timeout: 360_000, + timeout: CAPTURE_LONG_MS, testName: 'plan-ceo-review-expansion-energy', runId, model: 'claude-opus-4-7', @@ -270,7 +271,7 @@ Write your expansion proposals to ${planDir}/proposals.md with ONLY the proposal // Pass threshold: 4/5 on both axes (good — matches posture with minor weakness). expect(scores.axis_a).toBeGreaterThanOrEqual(4); // surface_framing expect(scores.axis_b).toBeGreaterThanOrEqual(4); // decision_preservation - }, 600_000); + }, CAPTURE_LONG_MS); }); // --- Plan Eng Review E2E --- @@ -348,7 +349,7 @@ Write your complete review directly to ${planDir}/review-output.md Focus on architecture, code quality, tests, and performance sections.`, workingDirectory: planDir, maxTurns: 15, - timeout: 360_000, + timeout: CAPTURE_LONG_MS, testName: 'plan-eng-review', runId, model: 'claude-opus-4-7', @@ -366,7 +367,7 @@ Focus on architecture, code quality, tests, and performance sections.`, const review = fs.readFileSync(reviewPath, 'utf-8'); expect(review.length).toBeGreaterThan(200); } - }, 420_000); + }, CAPTURE_LONG_MS); }); // --- Plan-Eng-Review Test-Plan Artifact E2E --- @@ -476,7 +477,7 @@ Write your review to ${planDir}/review-output.md`, workingDirectory: planDir, maxTurns: 25, allowedTools: ['Bash', 'Read', 'Write', 'Glob', 'Grep'], - timeout: 360_000, + timeout: CAPTURE_LONG_MS, testName: 'plan-eng-review-artifact', runId, model: 'claude-opus-4-7', @@ -507,7 +508,7 @@ Write your review to ${planDir}/review-output.md`, if (newFiles.length === 0) { console.warn('SOFT FAIL: No test-plan artifact written — agent did not follow artifact instructions'); } - }, 420_000); + }, CAPTURE_LONG_MS); }); // --- Office Hours Spec Review E2E --- @@ -559,7 +560,7 @@ Write your summary to ${ohDir}/spec-review-summary.md`, // failures wrote a correct summary on tool-turn 8 and hit the cap on // the closing text turn (error_max_turns at 9 turns, deterministic). maxTurns: 12, - timeout: 120_000, + timeout: JUDGE_MS, testName: 'office-hours-spec-review', runId, }); @@ -575,7 +576,7 @@ Write your summary to ${ohDir}/spec-review-summary.md`, expect(summary).toMatch(/agent|subagent/); expect(summary).toMatch(/3.*iteration|iteration.*3|maximum.*3/); } - }, 180_000); + }, CAPTURE_MS); }); // --- Plan CEO Review Benefits-From E2E --- @@ -619,7 +620,7 @@ Summarize what happens when no design doc is found — specifically: Write your summary to ${benefitsDir}/benefits-summary.md`, workingDirectory: benefitsDir, maxTurns: 8, - timeout: 120_000, + timeout: JUDGE_MS, testName: 'plan-ceo-review-benefits', runId, }); @@ -634,7 +635,7 @@ Write your summary to ${benefitsDir}/benefits-summary.md`, expect(summary).toMatch(/office.hours/); expect(summary).toMatch(/design doc|no design/i); } - }, 180_000); + }, CAPTURE_MS); }); // --- Plan Review Report E2E --- @@ -706,7 +707,7 @@ CRITICAL REQUIREMENT: plan.md IS the plan file for this review session. After co This review report at the bottom of the plan is the MOST IMPORTANT deliverable of this test.`, workingDirectory: planDir, maxTurns: 20, - timeout: 360_000, + timeout: CAPTURE_LONG_MS, testName: 'plan-review-report', runId, model: 'claude-opus-4-7', @@ -768,7 +769,7 @@ This review report at the bottom of the plan is the MOST IMPORTANT deliverable o ).toBe(true); console.log('Plan review report found at bottom of plan.md (ends with unresolved status)'); - }, 420_000); + }, CAPTURE_LONG_MS); }); // --- Codex Offering E2E --- @@ -825,7 +826,7 @@ Summarize the Codex/${featureName} integration — answer these specific questio Write your summary to ${testDir}/${testName}-summary.md`, workingDirectory: testDir, maxTurns: 8, - timeout: 120_000, + timeout: JUDGE_MS, testName, runId, }); @@ -850,19 +851,19 @@ Write your summary to ${testDir}/${testName}-summary.md`, testConcurrentIfSelected('codex-offered-office-hours', async () => { await checkCodexOffering('office-hours', 'codex-offered-office-hours', 'second opinion'); - }, 180_000); + }, CAPTURE_MS); testConcurrentIfSelected('codex-offered-ceo-review', async () => { await checkCodexOffering('plan-ceo-review', 'codex-offered-ceo-review', 'outside voice'); - }, 180_000); + }, CAPTURE_MS); testConcurrentIfSelected('codex-offered-design-review', async () => { await checkCodexOffering('plan-design-review', 'codex-offered-design-review', 'design outside voices'); - }, 180_000); + }, CAPTURE_MS); testConcurrentIfSelected('codex-offered-eng-review', async () => { await checkCodexOffering('plan-eng-review', 'codex-offered-eng-review', 'outside voice'); - }, 180_000); + }, CAPTURE_MS); }); // Module-level afterAll — finalize eval collector after all tests complete diff --git a/test/skill-e2e-qa-bugs.test.ts b/test/skill-e2e-qa-bugs.test.ts index 93514295f..c2fcff155 100644 --- a/test/skill-e2e-qa-bugs.test.ts +++ b/test/skill-e2e-qa-bugs.test.ts @@ -1,4 +1,5 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { outcomeJudge } from './helpers/llm-judge'; import { judgePassed } from './helpers/eval-store'; @@ -97,7 +98,7 @@ CRITICAL RULES: - The report MUST exist at ${reportPath} when you finish`, workingDirectory: testWorkDir, maxTurns: 50, - timeout: 300_000, + timeout: CAPTURE_MS, testName: `qa-${label}`, runId, model: 'claude-opus-4-7', @@ -174,17 +175,17 @@ CRITICAL RULES: // B6: Static dashboard — broken link, disabled submit, overflow, missing alt, console error testConcurrentIfSelected('qa-b6-static', async () => { await runPlantedBugEval('qa-eval.html', 'qa-eval-ground-truth.json', 'b6-static'); - }, 360_000); + }, CAPTURE_LONG_MS); // B7: SPA — broken route, stale state, async race, missing aria, console warning testConcurrentIfSelected('qa-b7-spa', async () => { await runPlantedBugEval('qa-eval-spa.html', 'qa-eval-spa-ground-truth.json', 'b7-spa'); - }, 360_000); + }, CAPTURE_LONG_MS); // B8: Checkout — email regex, NaN total, CC overflow, missing required, stripe error testConcurrentIfSelected('qa-b8-checkout', async () => { await runPlantedBugEval('qa-eval-checkout.html', 'qa-eval-checkout-ground-truth.json', 'b8-checkout'); - }, 360_000); + }, CAPTURE_LONG_MS); }); diff --git a/test/skill-e2e-qa-workflow.test.ts b/test/skill-e2e-qa-workflow.test.ts index a6c471357..d2c028d46 100644 --- a/test/skill-e2e-qa-workflow.test.ts +++ b/test/skill-e2e-qa-workflow.test.ts @@ -1,4 +1,5 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, browseBin, runId, evalsEnabled, @@ -54,7 +55,7 @@ Do NOT try to start a server or discover ports — the URL above is ready. Write your report to ${qaDir}/qa-reports/qa-report.md`, workingDirectory: qaDir, maxTurns: 35, - timeout: 240_000, + timeout: CAPTURE_MS, testName: 'qa-quick', runId, }); @@ -69,7 +70,7 @@ Write your report to ${qaDir}/qa-reports/qa-report.md`, } // Accept error_max_turns — the agent doing thorough QA work is not a failure expect(['success', 'error_max_turns']).toContain(result.exitReason); - }, 300_000); + }, CAPTURE_MS); }); // --- QA-Only E2E (report-only, no fixes) --- @@ -124,7 +125,7 @@ Write your report to ${qaOnlyDir}/qa-reports/qa-only-report.md`, workingDirectory: qaOnlyDir, maxTurns: 40, allowedTools: ['Bash', 'Read', 'Write', 'Glob'], // NO Edit — the critical guardrail - timeout: 180_000, + timeout: CAPTURE_MS, testName: 'qa-only-no-fix', runId, }); @@ -156,7 +157,7 @@ Write your report to ${qaOnlyDir}/qa-reports/qa-only-report.md`, (l: string) => l.trim() && !l.includes('.prompt-tmp') && !l.includes('.gstack/') && !l.includes('qa-reports/'), ); expect(statusLines.filter((l: string) => l.startsWith(' M') || l.startsWith('M '))).toHaveLength(0); - }, 240_000); + }, CAPTURE_MS); }); // --- QA Fix Loop E2E --- @@ -247,7 +248,7 @@ This is a test+fix loop: find bugs, fix them in the source code, commit each fix workingDirectory: qaFixDir, maxTurns: 40, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'], - timeout: 420_000, + timeout: CAPTURE_LONG_MS, testName: 'qa-fix-loop', runId, }); @@ -271,7 +272,7 @@ This is a test+fix loop: find bugs, fix them in the source code, commit each fix // Verify Edit tool was used (agent actually modified source code) const editCalls = result.toolCalls.filter(tc => tc.tool === 'Edit'); expect(editCalls.length).toBeGreaterThan(0); - }, 480_000); + }, CAPTURE_LONG_MS); }); // --- Test Bootstrap E2E --- @@ -384,7 +385,7 @@ Do NOT fix any bugs. Do NOT use AskUserQuestion — just pick vitest.`, workingDirectory: bsDir, maxTurns: 12, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'], - timeout: 90_000, + timeout: JUDGE_MS, testName: 'qa-bootstrap', runId, }); @@ -405,7 +406,7 @@ Do NOT fix any bugs. Do NOT use AskUserQuestion — just pick vitest.`, console.log(`Test config: ${hasTestConfig}, Test file: ${hasTestFile}, TESTING.md: ${hasTestingMd}`); try { fs.rmSync(bsDir, { recursive: true, force: true }); } catch {} - }, 120_000); + }, JUDGE_MS); }); // Module-level afterAll — finalize eval collector after all tests complete. diff --git a/test/skill-e2e-retro.test.ts b/test/skill-e2e-retro.test.ts index 49f774ac0..e13a22d16 100644 --- a/test/skill-e2e-retro.test.ts +++ b/test/skill-e2e-retro.test.ts @@ -1,4 +1,5 @@ import { expect, beforeAll, afterAll } from 'bun:test'; +import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, runId, @@ -135,7 +136,7 @@ Write your retrospective to ${dir}/retro-output.md`, expect(wroteReport).toBe(true); const content = fs.readFileSync(retroPath, 'utf-8'); expect(content.length).toBeGreaterThan(100); - }, 480_000); + }, CAPTURE_LONG_MS); }); // --- Retro E2E --- @@ -198,7 +199,7 @@ Write your retrospective report to ${retroDir}/retro-output.md Analyze the git history and produce the narrative report as described in the SKILL.md.`, workingDirectory: retroDir, maxTurns: 30, - timeout: 300_000, + timeout: CAPTURE_MS, testName: 'retro', runId, model: 'claude-opus-4-7', @@ -217,7 +218,7 @@ Analyze the git history and produce the narrative report as described in the SKI expect(wroteReport).toBe(true); const retro = fs.readFileSync(retroPath, 'utf-8'); expect(retro.length).toBeGreaterThan(100); - }, 420_000); + }, CAPTURE_LONG_MS); }); // Module-level afterAll — finalize eval collector after all tests complete diff --git a/test/skill-e2e-review-army.test.ts b/test/skill-e2e-review-army.test.ts index 0bbe74a07..69cc1b95f 100644 --- a/test/skill-e2e-review-army.test.ts +++ b/test/skill-e2e-review-army.test.ts @@ -1,4 +1,5 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, runId, describeIfSelected, testConcurrentIfSelected, @@ -114,7 +115,7 @@ and apply it yourself against the diff (git diff main...HEAD). Write your findings to ${dir}/review-output.md`, workingDirectory: dir, maxTurns: 20, - timeout: 180_000, + timeout: CAPTURE_MS, testName: 'review-army-migration-safety', runId, }); @@ -135,7 +136,7 @@ Write your findings to ${dir}/review-output.md`, content.includes('column'); expect(hasMigrationFinding).toBe(true); } - }, 210_000); + }, CAPTURE_MS); }); // --- Review Army: N+1 Performance --- @@ -179,7 +180,7 @@ For the specialist dispatch, read review-specialists/performance.md and apply it Write your findings to ${dir}/review-output.md`, workingDirectory: dir, maxTurns: 20, - timeout: 180_000, + timeout: CAPTURE_MS, testName: 'review-army-perf-n-plus-one', runId, }); @@ -201,7 +202,7 @@ Write your findings to ${dir}/review-output.md`, content.includes('loop'); expect(hasN1Finding).toBe(true); } - }, 210_000); + }, CAPTURE_MS); }); // --- Review Army: Delivery Audit --- @@ -281,7 +282,7 @@ The email notification system should be classified as NOT DONE. Write your completion audit to ${dir}/review-output.md`, workingDirectory: dir, maxTurns: 15, - timeout: 120_000, + timeout: JUDGE_MS, testName: 'review-army-delivery-audit', runId, }); @@ -305,7 +306,7 @@ Write your completion audit to ${dir}/review-output.md`, expect(hasNotDone).toBe(true); expect(mentionsEmail).toBe(true); } - }, 150_000); + }, CAPTURE_MS); }); // --- Review Army: Quality Score --- @@ -356,7 +357,7 @@ Write your findings AND the computed quality score to ${dir}/review-output.md Include the line: "PR Quality Score: X/10" where X is the computed score.`, workingDirectory: dir, maxTurns: 15, - timeout: 120_000, + timeout: JUDGE_MS, testName: 'review-army-quality-score', runId, }); @@ -374,7 +375,7 @@ Include the line: "PR Quality Score: X/10" where X is the computed score.`, content.match(/\d+\/10/); expect(hasScore).toBeTruthy(); } - }, 150_000); + }, CAPTURE_MS); }); // --- Review Army: JSON Findings --- @@ -421,7 +422,7 @@ Output your findings as JSON objects, one per line, following the schema: Write ONLY JSON findings (no preamble) to ${dir}/findings.json`, workingDirectory: dir, maxTurns: 12, - timeout: 90_000, + timeout: JUDGE_MS, testName: 'review-army-json-findings', runId, }); @@ -450,7 +451,7 @@ Write ONLY JSON findings (no preamble) to ${dir}/findings.json`, break; // One valid line is enough for the gate test } } - }, 120_000); + }, JUDGE_MS); }); // --- Review Army: Red Team (periodic) --- @@ -499,7 +500,7 @@ Write your red team findings to ${dir}/review-output.md Start the file with "RED TEAM REVIEW" on the first line.`, workingDirectory: dir, maxTurns: 20, - timeout: 180_000, + timeout: CAPTURE_MS, testName: 'review-army-red-team', runId, }); @@ -513,7 +514,7 @@ Start the file with "RED TEAM REVIEW" on the first line.`, const content = fs.readFileSync(outputPath, 'utf-8'); expect(content.toLowerCase()).toMatch(/red team|adversarial/); } - }, 210_000); + }, CAPTURE_MS); }); // --- Review Army: Consensus (periodic) --- @@ -566,7 +567,7 @@ mark it as "MULTI-SPECIALIST CONFIRMED" with the confirming categories. Write findings to ${dir}/review-output.md`, workingDirectory: dir, maxTurns: 20, - timeout: 180_000, + timeout: CAPTURE_MS, testName: 'review-army-consensus', runId, }); @@ -585,7 +586,7 @@ Write findings to ${dir}/review-output.md`, content.includes('interpolat'); expect(hasSqlFinding).toBe(true); } - }, 210_000); + }, CAPTURE_MS); }); // Finalize eval collector diff --git a/test/skill-e2e-review-attribution.test.ts b/test/skill-e2e-review-attribution.test.ts index 5a5673cb9..6a60f1d1f 100644 --- a/test/skill-e2e-review-attribution.test.ts +++ b/test/skill-e2e-review-attribution.test.ts @@ -1,4 +1,5 @@ import { expect, beforeAll, afterAll } from 'bun:test'; +import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, runId, @@ -65,7 +66,7 @@ Then run git diff against the detected base branch and write a brief review. Write your findings to ${dir}/review-output.md`, workingDirectory: dir, maxTurns: 15, - timeout: 90_000, + timeout: JUDGE_MS, testName: 'review-base-branch', runId, }); @@ -84,7 +85,7 @@ Write your findings to ${dir}/review-output.md`, return cmd.includes('git diff'); }); expect(usedGitDiff).toBe(true); - }, 120_000); + }, JUDGE_MS); testConcurrentIfSelected('ship-base-branch', async () => { const dir = path.join(baseBranchDir, 'ship-base'); @@ -125,7 +126,7 @@ Write a summary to ${dir}/ship-preflight.md including: - The diff stat against the base branch`, workingDirectory: dir, maxTurns: 18, - timeout: 150_000, + timeout: CAPTURE_MS, testName: 'ship-base-branch', runId, }); @@ -155,7 +156,7 @@ Write a summary to ${dir}/ship-preflight.md including: return command.includes('git push') || command.includes('gh pr create'); }); expect(destructiveTools).toHaveLength(0); - }, 180_000); + }, CAPTURE_MS); }); // --- Review Dashboard Via Attribution E2E --- @@ -280,7 +281,7 @@ Write the dashboard output to ${dashDir}/dashboard-output.md`, ); // Ship dashboard should not gate when eng review is clear expect(gateQuestions).toHaveLength(0); - }, 480_000); + }, CAPTURE_LONG_MS); }); // Module-level afterAll — finalize eval collector after all tests complete diff --git a/test/skill-e2e-review.test.ts b/test/skill-e2e-review.test.ts index 2203e5200..75b9accd3 100644 --- a/test/skill-e2e-review.test.ts +++ b/test/skill-e2e-review.test.ts @@ -1,4 +1,5 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, browseBin, runId, evalsEnabled, selectedTests, @@ -66,7 +67,7 @@ Run /review on the current diff (git diff main...HEAD). Write your review findings to ${reviewDir}/review-output.md`, workingDirectory: reviewDir, maxTurns: 20, - timeout: 180_000, + timeout: CAPTURE_MS, testName: 'review-sql-injection', runId, }); @@ -89,7 +90,7 @@ Write your review findings to ${reviewDir}/review-output.md`, reviewContent.includes('unsanitized'); expect(hasSqlContent).toBe(true); } - }, 210_000); + }, CAPTURE_MS); }); // --- Review: Enum completeness E2E --- @@ -144,7 +145,7 @@ Write your review findings to ${enumDir}/review-output.md The diff adds a new "returned" status to the Order model. Your job is to check if all consumers handle it.`, workingDirectory: enumDir, maxTurns: 15, - timeout: 90_000, + timeout: JUDGE_MS, testName: 'review-enum-completeness', runId, }); @@ -164,7 +165,7 @@ The diff adds a new "returned" status to the Order model. Your job is to check i expect(mentionsReturned).toBe(true); expect(mentionsEnum || mentionsCritical).toBe(true); } - }, 120_000); + }, JUDGE_MS); }); // --- Review: Design review lite E2E --- @@ -229,7 +230,7 @@ Write your review findings to ${designDir}/review-output.md Important: The design checklist should catch issues like blacklisted fonts, small font sizes, outline:none, !important, AI slop patterns (purple gradients, generic hero copy, 3-column feature grid), etc.`, workingDirectory: designDir, maxTurns: 35, - timeout: 240_000, + timeout: CAPTURE_MS, testName: 'review-design-lite', runId, }); @@ -262,7 +263,7 @@ Important: The design checklist should catch issues like blacklisted fonts, smal console.log(`Design review detected ${detected}/7 planted issues`); expect(detected).toBeGreaterThanOrEqual(4); } - }, 300_000); + }, CAPTURE_MS); }); // Base branch detection tests for review/ship + the Review Dashboard Via diff --git a/test/skill-e2e-session-intelligence.test.ts b/test/skill-e2e-session-intelligence.test.ts index 10c1d8d76..f9e525fb8 100644 --- a/test/skill-e2e-session-intelligence.test.ts +++ b/test/skill-e2e-session-intelligence.test.ts @@ -1,4 +1,5 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, runId, evalsEnabled, @@ -166,7 +167,7 @@ IMPORTANT: workingDirectory: workDir, maxTurns: 10, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], - timeout: 120_000, + timeout: JUDGE_MS, testName: 'context-recovery-artifacts', runId, }); @@ -193,7 +194,7 @@ IMPORTANT: expect(foundCount).toBeGreaterThanOrEqual(1); console.log(`Context recovery: artifacts=${foundArtifacts}, lastSession=${foundLastSession}, timeline=${foundTimeline}`); - }, 180_000); + }, CAPTURE_MS); // --- Test 3: /context-save writes a file --- // Hand-feed the save section of context-save/SKILL.md to claude -p and verify @@ -231,7 +232,7 @@ IMPORTANT: workingDirectory: workDir, maxTurns: 10, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], - timeout: 120_000, + timeout: JUDGE_MS, testName: 'context-save-writes-file', runId, }); @@ -264,7 +265,7 @@ IMPORTANT: expect(hasYamlFrontmatter).toBe(true); console.log(`context-save: ${files.length} files created, YAML frontmatter: ${hasYamlFrontmatter}, branch: ${hasBranch}`); - }, 180_000); + }, CAPTURE_MS); // --- Test 4: /context-restore loads the newest file across branches --- // Seed two saved-context files with different YYYYMMDD-HHMMSS prefixes and @@ -340,7 +341,7 @@ IMPORTANT: workingDirectory: workDir, maxTurns: 8, allowedTools: ['Bash', 'Read', 'Grep', 'Glob'], - timeout: 120_000, + timeout: JUDGE_MS, testName: 'context-restore-loads-latest', runId, }); @@ -361,5 +362,5 @@ IMPORTANT: expect(loadedOlder).toBe(false); console.log(`context-restore: loadedNewer=${loadedNewer}, loadedOlder=${loadedOlder}`); - }, 180_000); + }, CAPTURE_MS); }); diff --git a/test/skill-e2e-setup-gbrain-bad-token.test.ts b/test/skill-e2e-setup-gbrain-bad-token.test.ts index 14e63fcb2..9ba2dede8 100644 --- a/test/skill-e2e-setup-gbrain-bad-token.test.ts +++ b/test/skill-e2e-setup-gbrain-bad-token.test.ts @@ -14,6 +14,7 @@ // on a failed verify the skill STOPs before any CLAUDE.md write. import { test, expect } from 'bun:test'; +import { CAPTURE_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'fs'; import * as os from 'os'; @@ -155,5 +156,5 @@ describeE2E('/setup-gbrain Path 4 — bad token STOPs cleanly', () => { fs.rmSync(gstackHome, { recursive: true, force: true }); fs.rmSync(fakeBinDir, { recursive: true, force: true }); } - }, 240_000); + }, CAPTURE_MS); }); diff --git a/test/skill-e2e-setup-gbrain-path4-local-pglite.test.ts b/test/skill-e2e-setup-gbrain-path4-local-pglite.test.ts index 0c4e72b81..727e2c529 100644 --- a/test/skill-e2e-setup-gbrain-path4-local-pglite.test.ts +++ b/test/skill-e2e-setup-gbrain-path4-local-pglite.test.ts @@ -19,6 +19,7 @@ // Cost: ~$0.50-$1.00 per run. Periodic-tier (EVALS=1 EVALS_TIER=periodic). import { test, expect } from 'bun:test'; +import { CAPTURE_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'fs'; import * as os from 'os'; @@ -264,5 +265,5 @@ describeE2E('/setup-gbrain Path 4 + Step 4.5 Yes → local PGLite for code', () fs.rmSync(sandboxHome, { recursive: true, force: true }); fs.rmSync(fakeBinDir, { recursive: true, force: true }); } - }, 300_000); + }, CAPTURE_MS); }); diff --git a/test/skill-e2e-setup-gbrain-remote.test.ts b/test/skill-e2e-setup-gbrain-remote.test.ts index 1429c151e..095e7c5bc 100644 --- a/test/skill-e2e-setup-gbrain-remote.test.ts +++ b/test/skill-e2e-setup-gbrain-remote.test.ts @@ -16,6 +16,7 @@ // block this test asserts on). import { test, expect } from 'bun:test'; +import { CAPTURE_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'fs'; import * as os from 'os'; @@ -245,5 +246,5 @@ describeE2E('/setup-gbrain Path 4 (Remote MCP) — happy path', () => { fs.rmSync(gstackHome, { recursive: true, force: true }); fs.rmSync(fakeBinDir, { recursive: true, force: true }); } - }, 240_000); + }, CAPTURE_MS); }); diff --git a/test/skill-e2e-ship-docsync.test.ts b/test/skill-e2e-ship-docsync.test.ts index 58d586b2b..6310e1537 100644 --- a/test/skill-e2e-ship-docsync.test.ts +++ b/test/skill-e2e-ship-docsync.test.ts @@ -49,6 +49,7 @@ * gate tier confirmed). */ import { expect, beforeAll, afterAll } from 'bun:test'; +import { CAPTURE_LONG_MS } from './helpers/eval-budgets'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; @@ -200,7 +201,7 @@ describeE2E('Ship doc-sync dispatch E2E (gate)', () => { workingDirectory: repoDir, maxTurns: 30, allowedTools: ['Bash', 'Read', 'Grep', 'Glob', 'Write', 'Agent', 'Task'], - timeout: 480_000, + timeout: CAPTURE_LONG_MS, env: { HOME: workDir, GSTACK_HOME: path.join(workDir, 'gstack-home'), @@ -273,7 +274,7 @@ describeE2E('Ship doc-sync dispatch E2E (gate)', () => { console.log( `dispatchIdx=${dispatchIdx} prCreateIdx=${prCreateIdx} readPrBody=${readPrBody} exit=${result.exitReason}` ); - }, 540_000); + }, CAPTURE_LONG_MS); }); }); diff --git a/test/skill-e2e-ship-idempotency.test.ts b/test/skill-e2e-ship-idempotency.test.ts index 9f035ee1f..a20020dfa 100644 --- a/test/skill-e2e-ship-idempotency.test.ts +++ b/test/skill-e2e-ship-idempotency.test.ts @@ -31,6 +31,7 @@ */ import { test, expect } from 'bun:test'; +import { PTY_LONG_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import { spawnSync } from 'child_process'; import * as fs from 'fs'; @@ -158,7 +159,7 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => { const session = await launchClaudePty({ permissionMode: 'plan', cwd: fixture.workTree, - timeoutMs: 1_080_000, + timeoutMs: PTY_LONG_MS, // Disable network-y pieces so the agent can't reach actual github. env: { GH_TOKEN: 'mock-not-real', NO_COLOR: '1' }, seedSkills: true, @@ -279,6 +280,6 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => { try { fs.rmSync(path.dirname(fixture.workTree), { recursive: true, force: true }); } catch { /* ignore */ } } }, - 1_200_000, // 20 min wall clock + PTY_LONG_MS, // 20 min wall clock ); }); diff --git a/test/skill-e2e-ship-section-loading.test.ts b/test/skill-e2e-ship-section-loading.test.ts index 23d9db9e8..ddd9bdc33 100644 --- a/test/skill-e2e-ship-section-loading.test.ts +++ b/test/skill-e2e-ship-section-loading.test.ts @@ -24,6 +24,7 @@ */ import { test, expect } from 'bun:test'; +import { CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import { setupSkillDir, @@ -78,6 +79,6 @@ describeE2E('/ship section-loading E2E (periodic, SDK capture)', () => { // Guard against an empty pass: the report must have real content. expect(output.trim().length).toBeGreaterThan(200); }, - 360_000, + CAPTURE_LONG_MS, ); }); diff --git a/test/skill-e2e-skillify.test.ts b/test/skill-e2e-skillify.test.ts index f92af6cdc..74df0034a 100644 --- a/test/skill-e2e-skillify.test.ts +++ b/test/skill-e2e-skillify.test.ts @@ -27,6 +27,7 @@ */ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, browseBin, runId, @@ -204,7 +205,7 @@ Do NOT enter the prototype phase. Do NOT use AskUserQuestion.`, env: { GSTACK_HOME: gstackHome }, maxTurns: 12, allowedTools: ['Skill', 'Bash', 'Read'], - timeout: 120_000, + timeout: JUDGE_MS, testName: 'scrape-match-path', runId, }); @@ -224,7 +225,7 @@ Do NOT enter the prototype phase. Do NOT use AskUserQuestion.`, expect(listedSkills).toBe(true); expect(ranBundledSkill).toBe(true); try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} - }, 180_000); + }, CAPTURE_MS); // ── 2. /scrape prototype path: drive $B primitives against fixture ──── testConcurrentIfSelected('scrape-prototype-path', async () => { @@ -248,7 +249,7 @@ Do NOT use AskUserQuestion.`, env: { GSTACK_HOME: gstackHome }, maxTurns: 18, allowedTools: ['Skill', 'Bash', 'Read'], - timeout: 180_000, + timeout: CAPTURE_MS, testName: 'scrape-prototype-path', runId, }); @@ -283,7 +284,7 @@ Do NOT use AskUserQuestion.`, expect(hasJsonItems).toBe(true); expect(mentionsSkillify).toBe(true); try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} - }, 240_000); + }, CAPTURE_MS); // ── 3. /skillify happy path: scrape then skillify in one session ───── testConcurrentIfSelected('skillify-happy-path', async () => { @@ -316,7 +317,7 @@ Do NOT halt for clarification.`, }, maxTurns: 40, allowedTools: ['Skill', 'Bash', 'Read', 'Write'], - timeout: 360_000, + timeout: CAPTURE_LONG_MS, testName: 'skillify-happy-path', runId, }); @@ -360,7 +361,7 @@ Do NOT halt for clarification.`, expect(hasAllFiles).toBe(true); expect(prosesClean).toBe(true); try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} - }, 420_000); + }, CAPTURE_LONG_MS); // ── 4. /skillify provenance refusal: D1 contract ───────────────────── testConcurrentIfSelected('skillify-provenance-refusal', async () => { @@ -379,7 +380,7 @@ write any files.`, }, maxTurns: 8, allowedTools: ['Skill', 'Bash', 'Read'], - timeout: 90_000, + timeout: JUDGE_MS, testName: 'skillify-provenance-refusal', runId, }); @@ -408,7 +409,7 @@ write any files.`, expect(noSkillsWritten).toBe(true); expect(noStaging).toBe(true); try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} - }, 120_000); + }, JUDGE_MS); // ── 5. /skillify approval-gate reject: D3 cleanup ──────────────────── testConcurrentIfSelected('skillify-approval-reject', async () => { @@ -435,7 +436,7 @@ Use HOME=${workDir}. Do NOT commit the skill.`, }, maxTurns: 35, allowedTools: ['Skill', 'Bash', 'Read', 'Write'], - timeout: 360_000, + timeout: CAPTURE_LONG_MS, testName: 'skillify-approval-reject', runId, }); @@ -462,5 +463,5 @@ Use HOME=${workDir}. Do NOT commit the skill.`, expect(writtenSkills.length).toBe(0); expect(stagingLeftovers.length).toBe(0); try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} - }, 420_000); + }, CAPTURE_LONG_MS); }); diff --git a/test/skill-e2e-triage.test.ts b/test/skill-e2e-triage.test.ts index 5b25526bd..e971c5e23 100644 --- a/test/skill-e2e-triage.test.ts +++ b/test/skill-e2e-triage.test.ts @@ -19,6 +19,7 @@ */ import { test, expect, beforeAll, afterAll } from 'bun:test'; +import { CAPTURE_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, runId, @@ -178,7 +179,7 @@ This is a solo repo (REPO_MODE=solo). For pre-existing failures, recommend fixin workingDirectory: triageDir, maxTurns: 20, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'], - timeout: 180_000, + timeout: CAPTURE_MS, testName: 'ship-triage', runId, }); @@ -229,7 +230,7 @@ This is a solo repo (REPO_MODE=solo). For pre-existing failures, recommend fixin // Must have actually run both test files (exercises both failure classes) expect(ranMathTest).toBe(true); expect(ranStringTest).toBe(true); - }, 240_000); + }, CAPTURE_MS); }); // Module-level afterAll — finalize eval collector after all tests complete diff --git a/test/skill-e2e-workflow.test.ts b/test/skill-e2e-workflow.test.ts index 5c4931e63..055974e9c 100644 --- a/test/skill-e2e-workflow.test.ts +++ b/test/skill-e2e-workflow.test.ts @@ -1,4 +1,5 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import { ROOT, browseBin, runId, evalsEnabled, @@ -83,7 +84,7 @@ IMPORTANT: // other rounds — marginal at 180s, same contention story as // review-dashboard-via and retro-base-branch. Outer bun timeout // rises to 360s for headroom. - timeout: 300_000, + timeout: CAPTURE_MS, testName: 'document-release', runId, }); @@ -120,7 +121,7 @@ IMPORTANT: } else { console.warn('README was NOT updated — agent may not have found the feature'); } - }, 360_000); + }, CAPTURE_LONG_MS); }); // --- Ship workflow with local bare remote --- @@ -174,7 +175,7 @@ describeIfSelected('Ship workflow E2E', ['ship-local-workflow'], () => { 4. Push to origin: git push origin feature/ship-test`, workingDirectory: shipWorkDir, maxTurns: 8, - timeout: 120_000, + timeout: JUDGE_MS, testName: 'ship-local-workflow', runId, }); @@ -198,7 +199,7 @@ describeIfSelected('Ship workflow E2E', ['ship-local-workflow'], () => { expect(branchExists).toBe(true); expect(versionBumped).toBe(true); console.log(`Branch pushed: ${branchExists}, VERSION: ${versionContent}, bumped: ${versionBumped}`); - }, 150_000); + }, CAPTURE_MS); }); // setup-cookies-detect REMOVED: The cookie-import-browser module has 30+ thorough @@ -297,7 +298,7 @@ Skip any AskUserQuestion calls — auto-approve the upgrade. Write a summary of IMPORTANT: The install directory is at ./.claude/skills/gstack — use that exact path.`, workingDirectory: upgradeDir, maxTurns: 20, - timeout: 180_000, + timeout: CAPTURE_MS, testName: 'gstack-upgrade-happy-path', runId, }); @@ -317,7 +318,7 @@ IMPORTANT: The install directory is at ./.claude/skills/gstack — use that exac expect(['success', 'error_max_turns']).toContain(result.exitReason); expect(versionAfter).toBe('0.6.0'); - }, 240_000); + }, CAPTURE_MS); }); // --- Test Coverage Audit E2E --- @@ -418,7 +419,7 @@ Output the diagram directly.`, workingDirectory: coverageDir, maxTurns: 15, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'], - timeout: 120_000, + timeout: JUDGE_MS, testName: 'ship-coverage-audit', runId, }); @@ -443,7 +444,7 @@ Output the diagram directly.`, // At minimum, the agent should have read the source and test files const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read'); expect(readCalls.length).toBeGreaterThan(0); - }, 180_000); + }, CAPTURE_MS); }); // --- Codex skill E2E --- @@ -520,7 +521,7 @@ Follow those instructions to run codex review against the diff on this branch. Write the full output (including the GATE verdict) to ${codexDir}/codex-output.md`, workingDirectory: codexDir, maxTurns: 25, - timeout: 300_000, + timeout: CAPTURE_MS, testName: 'codex-review', runId, model: 'claude-opus-4-7', @@ -538,7 +539,7 @@ Write the full output (including the GATE verdict) to ${codexDir}/codex-output.m const hasCodexOutput = output.includes('CODEX') || output.includes('GATE') || output.includes('codex'); expect(hasCodexOutput).toBe(true); } - }, 360_000); + }, CAPTURE_LONG_MS); }); // Module-level afterAll — finalize eval collector after all tests complete diff --git a/test/skill-llm-eval.test.ts b/test/skill-llm-eval.test.ts index 74e58de7c..cd845c291 100644 --- a/test/skill-llm-eval.test.ts +++ b/test/skill-llm-eval.test.ts @@ -11,6 +11,7 @@ */ import { afterAll, expect } from 'bun:test'; +import { JUDGE_MS } from './helpers/eval-budgets'; import Anthropic from '@anthropic-ai/sdk'; import * as fs from 'fs'; import * as path from 'path'; @@ -556,7 +557,7 @@ describeIfSelected('Baseline score pinning', ['baseline score pinning'], () => { if (!passed) { throw new Error(`Score regressions detected:\n${regressions.join('\n')}`); } - }, 60_000); + }, JUDGE_MS); }); // --- Workflow SKILL.md quality evals (10 new tests for 100% coverage) --- diff --git a/test/skill-routing-e2e.test.ts b/test/skill-routing-e2e.test.ts index 32ef55bb9..7e86b03f6 100644 --- a/test/skill-routing-e2e.test.ts +++ b/test/skill-routing-e2e.test.ts @@ -1,4 +1,5 @@ import { describe, test, expect, afterAll } from 'bun:test'; +import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; import type { SkillTestResult } from './helpers/session-runner'; import { EvalCollector } from './helpers/eval-store'; @@ -197,7 +198,7 @@ describeE2E('Skill Routing E2E — Developer Journey', () => { workingDirectory: tmpDir, maxTurns: 5, allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], - timeout: 60_000, + timeout: JUDGE_MS, testName, runId, }); @@ -213,7 +214,7 @@ describeE2E('Skill Routing E2E — Developer Journey', () => { } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } - }, 150_000); + }, CAPTURE_MS); testIfSelected('journey-plan-eng', async () => { const tmpDir = createRoutingWorkDir('plan-eng'); @@ -247,7 +248,7 @@ describeE2E('Skill Routing E2E — Developer Journey', () => { workingDirectory: tmpDir, maxTurns: 5, allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], - timeout: 60_000, + timeout: JUDGE_MS, testName, runId, }); @@ -263,7 +264,7 @@ describeE2E('Skill Routing E2E — Developer Journey', () => { } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } - }, 150_000); + }, CAPTURE_MS); // Removed: journey-think-bigger // Tested ambiguous routing ("think bigger" → plan-ceo-review) but Claude @@ -309,7 +310,7 @@ export default app; workingDirectory: tmpDir, maxTurns: 5, allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], - timeout: 60_000, + timeout: JUDGE_MS, testName, runId, }); @@ -326,7 +327,7 @@ export default app; } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } - }, 150_000); + }, CAPTURE_MS); testIfSelected('journey-qa', async () => { const tmpDir = createRoutingWorkDir('qa'); @@ -345,7 +346,7 @@ export default app; workingDirectory: tmpDir, maxTurns: 5, allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], - timeout: 60_000, + timeout: JUDGE_MS, testName, runId, }); @@ -362,7 +363,7 @@ export default app; } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } - }, 150_000); + }, CAPTURE_MS); testIfSelected('journey-code-review', async () => { const tmpDir = createRoutingWorkDir('code-review'); @@ -386,7 +387,7 @@ export default app; workingDirectory: tmpDir, maxTurns: 5, allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], - timeout: 120_000, + timeout: JUDGE_MS, testName, runId, }); @@ -402,7 +403,7 @@ export default app; } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } - }, 150_000); + }, CAPTURE_MS); testIfSelected('journey-ship', async () => { const tmpDir = createRoutingWorkDir('ship'); @@ -425,7 +426,7 @@ export default app; workingDirectory: tmpDir, maxTurns: 5, allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], - timeout: 60_000, + timeout: JUDGE_MS, testName, runId, }); @@ -441,7 +442,7 @@ export default app; } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } - }, 150_000); + }, CAPTURE_MS); testIfSelected('journey-docs', async () => { const tmpDir = createRoutingWorkDir('docs'); @@ -462,7 +463,7 @@ export default app; workingDirectory: tmpDir, maxTurns: 5, allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], - timeout: 60_000, + timeout: JUDGE_MS, testName, runId, }); @@ -478,7 +479,7 @@ export default app; } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } - }, 150_000); + }, CAPTURE_MS); testIfSelected('journey-retro', async () => { const tmpDir = createRoutingWorkDir('retro'); @@ -505,7 +506,7 @@ export default app; workingDirectory: tmpDir, maxTurns: 5, allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], - timeout: 120_000, + timeout: JUDGE_MS, testName, runId, }); @@ -521,7 +522,7 @@ export default app; } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } - }, 150_000); + }, CAPTURE_MS); testIfSelected('journey-design-system', async () => { const tmpDir = createRoutingWorkDir('design-system'); @@ -534,7 +535,7 @@ export default app; workingDirectory: tmpDir, maxTurns: 5, allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], - timeout: 60_000, + timeout: JUDGE_MS, testName, runId, }); @@ -550,7 +551,7 @@ export default app; } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } - }, 150_000); + }, CAPTURE_MS); testIfSelected('journey-visual-qa', async () => { const tmpDir = createRoutingWorkDir('visual-qa'); @@ -585,7 +586,7 @@ body { font-family: sans-serif; } workingDirectory: tmpDir, maxTurns: 5, allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], - timeout: 60_000, + timeout: JUDGE_MS, testName, runId, }); @@ -602,5 +603,5 @@ body { font-family: sans-serif; } } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } - }, 150_000); + }, CAPTURE_MS); });