refactor(test): mechanical sweep — 298 paid-test timeouts onto eval-budget tiers

69 files, both shapes (trailing bun-test budgets and runner
timeout/timeoutMs options), ROUND-UP ONLY so nothing that passed can
start failing: 75 → JUDGE_MS, 137 → CAPTURE_MS, 74 → CAPTURE_LONG_MS,
9 → PTY_MS, 3 → PTY_LONG_MS. Raw >=60s literal count in the paid scope:
395 → 97, of which 51 are non-timeout noise (fixture dates, run IDs)
and 46 are enumerated justified holds (comment-carrying calibrated
budgets, poll-loop constants, utility spawn waits, and the seven
physical-ceiling 1_500_000 sites). The eval-budgets policy ratchet
keeps the residue from regrowing.

Known collapse: where an inner runner budget and its enclosing test
budget now share a tier, the old stagger is gone — an overrun surfaces
as a bun test timeout instead of a graceful runner timeout
(diagnosability trade, not a correctness one).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-29 05:25:06 +00:00
co-authored by Claude Fable 5
parent 74ae357e0f
commit 6841183c35
69 changed files with 367 additions and 298 deletions
+2 -1
View File
@@ -20,6 +20,7 @@
*/ */
import { test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import { setupSkillDir, skillFromWorktree, captureSectionReads } from './helpers/auq-sdk-capture'; import { setupSkillDir, skillFromWorktree, captureSectionReads } from './helpers/auq-sdk-capture';
import { CARVE_GUARDS } from './helpers/carve-guards'; import { CARVE_GUARDS } from './helpers/carve-guards';
@@ -97,7 +98,7 @@ describeE2E('carve behavioral section-loading (periodic, SDK capture)', () => {
}); });
expect(output.trim().length).toBeGreaterThan(200); expect(output.trim().length).toBeGreaterThan(200);
}, },
540_000, CAPTURE_LONG_MS,
); );
} }
}); });
+9 -8
View File
@@ -26,6 +26,7 @@
* Periodic tier (Codex non-determinism). Cost: ~$2-3 per full run. * Periodic tier (Codex non-determinism). Cost: ~$2-3 per full run.
*/ */
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runCodexSkill, installSkillToTempHome } from './helpers/codex-session-runner'; import { runCodexSkill, installSkillToTempHome } from './helpers/codex-session-runner';
import type { CodexResult } from './helpers/codex-session-runner'; import type { CodexResult } from './helpers/codex-session-runner';
import { EvalCollector } from './helpers/eval-store'; import { EvalCollector } from './helpers/eval-store';
@@ -186,7 +187,7 @@ describeCodex('Codex Plan Format — CEO Mode Selection', () => {
const result = await runCodexSkill({ const result = await runCodexSkill({
skillDir, skillDir,
prompt: `Read the plan-ceo-review skill. Read plan.md (the plan to review). Proceed to Step 0F (Mode Selection) where the skill presents 4 mode options (SCOPE EXPANSION, SELECTIVE EXPANSION, HOLD SCOPE, SCOPE REDUCTION) via AskUserQuestion. These options differ in kind (review posture), not coverage. ${captureInstruction(outFile)}`, prompt: `Read the plan-ceo-review skill. Read plan.md (the plan to review). Proceed to Step 0F (Mode Selection) where the skill presents 4 mode options (SCOPE EXPANSION, SELECTIVE EXPANSION, HOLD SCOPE, SCOPE REDUCTION) via AskUserQuestion. These options differ in kind (review posture), not coverage. ${captureInstruction(outFile)}`,
timeoutMs: 300_000, timeoutMs: CAPTURE_MS,
cwd: planDir, cwd: planDir,
skillName: 'gstack-plan-ceo-review', skillName: 'gstack-plan-ceo-review',
sandbox: 'workspace-write', sandbox: 'workspace-write',
@@ -208,7 +209,7 @@ describeCodex('Codex Plan Format — CEO Mode Selection', () => {
// kind-differentiated: no fabricated score, must have note // kind-differentiated: no fabricated score, must have note
expect(captured).not.toMatch(COMPLETENESS_RE); expect(captured).not.toMatch(COMPLETENESS_RE);
expect(captured).toMatch(KIND_NOTE_RE); expect(captured).toMatch(KIND_NOTE_RE);
}, 360_000); }, CAPTURE_LONG_MS);
}); });
describeCodex('Codex Plan Format — CEO Approach Menu', () => { describeCodex('Codex Plan Format — CEO Approach Menu', () => {
@@ -226,7 +227,7 @@ describeCodex('Codex Plan Format — CEO Approach Menu', () => {
const result = await runCodexSkill({ const result = await runCodexSkill({
skillDir, skillDir,
prompt: `Read the plan-ceo-review skill. Read plan.md. Proceed to Step 0C-bis (Implementation Alternatives / Approach Menu) where the skill generates 2-3 approaches (minimal viable vs ideal architecture) and presents them via AskUserQuestion. These options differ in coverage so Completeness: N/10 applies. ${captureInstruction(outFile)}`, prompt: `Read the plan-ceo-review skill. Read plan.md. Proceed to Step 0C-bis (Implementation Alternatives / Approach Menu) where the skill generates 2-3 approaches (minimal viable vs ideal architecture) and presents them via AskUserQuestion. These options differ in coverage so Completeness: N/10 applies. ${captureInstruction(outFile)}`,
timeoutMs: 300_000, timeoutMs: CAPTURE_MS,
cwd: planDir, cwd: planDir,
skillName: 'gstack-plan-ceo-review', skillName: 'gstack-plan-ceo-review',
sandbox: 'workspace-write', sandbox: 'workspace-write',
@@ -245,7 +246,7 @@ describeCodex('Codex Plan Format — CEO Approach Menu', () => {
expect(captured.length).toBeGreaterThan(ELI10_LENGTH_FLOOR); expect(captured.length).toBeGreaterThan(ELI10_LENGTH_FLOOR);
expect(captured).toMatch(RECOMMENDATION_RE); expect(captured).toMatch(RECOMMENDATION_RE);
expect(captured).toMatch(COMPLETENESS_RE); expect(captured).toMatch(COMPLETENESS_RE);
}, 360_000); }, CAPTURE_LONG_MS);
}); });
describeCodex('Codex Plan Format — Eng Coverage Issue', () => { describeCodex('Codex Plan Format — Eng Coverage Issue', () => {
@@ -263,7 +264,7 @@ describeCodex('Codex Plan Format — Eng Coverage Issue', () => {
const result = await runCodexSkill({ const result = await runCodexSkill({
skillDir, skillDir,
prompt: `Read the plan-eng-review skill. Read plan.md. In your Section 3 Test Review, generate ONE AskUserQuestion about test coverage depth where options are clearly coverage-differentiated: A) full coverage incl. edge + error paths (Completeness 10/10), B) happy path only (7/10), C) smoke test (3/10). ${captureInstruction(outFile)}`, prompt: `Read the plan-eng-review skill. Read plan.md. In your Section 3 Test Review, generate ONE AskUserQuestion about test coverage depth where options are clearly coverage-differentiated: A) full coverage incl. edge + error paths (Completeness 10/10), B) happy path only (7/10), C) smoke test (3/10). ${captureInstruction(outFile)}`,
timeoutMs: 300_000, timeoutMs: CAPTURE_MS,
cwd: planDir, cwd: planDir,
skillName: 'gstack-plan-eng-review', skillName: 'gstack-plan-eng-review',
sandbox: 'workspace-write', sandbox: 'workspace-write',
@@ -282,7 +283,7 @@ describeCodex('Codex Plan Format — Eng Coverage Issue', () => {
expect(captured.length).toBeGreaterThan(ELI10_LENGTH_FLOOR); expect(captured.length).toBeGreaterThan(ELI10_LENGTH_FLOOR);
expect(captured).toMatch(RECOMMENDATION_RE); expect(captured).toMatch(RECOMMENDATION_RE);
expect(captured).toMatch(COMPLETENESS_RE); expect(captured).toMatch(COMPLETENESS_RE);
}, 360_000); }, CAPTURE_LONG_MS);
}); });
describeCodex('Codex Plan Format — Eng Kind Issue', () => { describeCodex('Codex Plan Format — Eng Kind Issue', () => {
@@ -300,7 +301,7 @@ describeCodex('Codex Plan Format — Eng Kind Issue', () => {
const result = await runCodexSkill({ const result = await runCodexSkill({
skillDir, skillDir,
prompt: `Read the plan-eng-review skill. Read plan.md. In your Section 1 Architecture review, generate ONE AskUserQuestion about an architectural choice where the options differ in kind (e.g. Redis vs Postgres materialized view vs in-process cache — different kinds of systems with different tradeoffs, NOT more-or-less-complete versions of the same thing). ${captureInstruction(outFile)}`, prompt: `Read the plan-eng-review skill. Read plan.md. In your Section 1 Architecture review, generate ONE AskUserQuestion about an architectural choice where the options differ in kind (e.g. Redis vs Postgres materialized view vs in-process cache — different kinds of systems with different tradeoffs, NOT more-or-less-complete versions of the same thing). ${captureInstruction(outFile)}`,
timeoutMs: 300_000, timeoutMs: CAPTURE_MS,
cwd: planDir, cwd: planDir,
skillName: 'gstack-plan-eng-review', skillName: 'gstack-plan-eng-review',
sandbox: 'workspace-write', sandbox: 'workspace-write',
@@ -321,5 +322,5 @@ describeCodex('Codex Plan Format — Eng Kind Issue', () => {
// kind-differentiated: no fabricated score // kind-differentiated: no fabricated score
expect(captured).not.toMatch(COMPLETENESS_RE); expect(captured).not.toMatch(COMPLETENESS_RE);
expect(captured).toMatch(KIND_NOTE_RE); expect(captured).toMatch(KIND_NOTE_RE);
}, 360_000); }, CAPTURE_LONG_MS);
}); });
@@ -21,6 +21,7 @@
* Periodic tier (Codex non-determinism, ~$2-3/run). * Periodic tier (Codex non-determinism, ~$2-3/run).
*/ */
import { describe, test, expect } from 'bun:test'; import { describe, test, expect } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import * as path from 'node:path'; import * as path from 'node:path';
import { e2eTierEnabled } from './helpers/e2e-gate'; import { e2eTierEnabled } from './helpers/e2e-gate';
import { runCodexSkill } from './helpers/codex-session-runner'; import { runCodexSkill } from './helpers/codex-session-runner';
@@ -69,7 +70,7 @@ describeCodex('/codex recommendation substance (live, periodic)', () => {
skillDir: path.join(ROOT, 'codex'), skillDir: path.join(ROOT, 'codex'),
skillName: 'codex', skillName: 'codex',
prompt: FIXTURE_DIFF, prompt: FIXTURE_DIFF,
timeoutMs: 300_000, timeoutMs: CAPTURE_MS,
}); });
if (result.output.startsWith('SKIP:')) { if (result.output.startsWith('SKIP:')) {
@@ -98,6 +99,6 @@ describeCodex('/codex recommendation substance (live, periodic)', () => {
); );
} }
}, },
360_000, CAPTURE_LONG_MS,
); );
}); });
+2 -1
View File
@@ -11,6 +11,7 @@
* golden), parallel shards (worktree copies), or live symlinked installs. * golden), parallel shards (worktree copies), or live symlinked installs.
*/ */
import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import * as fs from 'fs'; import * as fs from 'fs';
import * as os from 'os'; import * as os from 'os';
import * as path from 'path'; import * as path from 'path';
@@ -262,5 +263,5 @@ You are authorized to implement the minimal fix. The task boundary is src/parse-
expect(readmeDecoyUntouched).toBe(true); expect(readmeDecoyUntouched).toBe(true);
console.log(`codex-sol-scope: ${result.tokens} tokens, ${result.toolCalls.length} tool calls, ${Math.round(result.durationMs / 1000)}s`); console.log(`codex-sol-scope: ${result.tokens} tokens, ${result.toolCalls.length} tool calls, ${Math.round(result.durationMs / 1000)}s`);
}, 300_000); }, CAPTURE_MS);
}); });
+5 -4
View File
@@ -14,6 +14,7 @@
*/ */
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runCodexSkill, parseCodexJSONL, installSkillToTempHome } from './helpers/codex-session-runner'; import { runCodexSkill, parseCodexJSONL, installSkillToTempHome } from './helpers/codex-session-runner';
import type { CodexResult } from './helpers/codex-session-runner'; import type { CodexResult } from './helpers/codex-session-runner';
import { CODEX_REVIEW_E2E_SECTIONS } from './helpers/skill-fixture'; import { CODEX_REVIEW_E2E_SECTIONS } from './helpers/skill-fixture';
@@ -150,7 +151,7 @@ describeCodex('Codex E2E', () => {
const result = await runCodexSkill({ const result = await runCodexSkill({
skillDir, skillDir,
prompt: 'List any skills or instructions you have available. Just list the names.', prompt: 'List any skills or instructions you have available. Just list the names.',
timeoutMs: 60_000, timeoutMs: JUDGE_MS,
cwd: testWorktree, cwd: testWorktree,
skillName: 'gstack-review', skillName: 'gstack-review',
}); });
@@ -171,7 +172,7 @@ describeCodex('Codex E2E', () => {
expect( expect(
outputLower.includes('review') || outputLower.includes('gstack') || outputLower.includes('skill'), outputLower.includes('review') || outputLower.includes('gstack') || outputLower.includes('skill'),
).toBe(true); ).toBe(true);
}, 120_000); }, JUDGE_MS);
// Validates that Codex can invoke the gstack-review skill, run a diff-based // Validates that Codex can invoke the gstack-review skill, run a diff-based
// code review, and produce structured review output with findings/issues. // code review, and produce structured review output with findings/issues.
@@ -186,7 +187,7 @@ describeCodex('Codex E2E', () => {
const result = await runCodexSkill({ const result = await runCodexSkill({
skillDir, skillDir,
prompt: 'Run the gstack-review skill on this repository. Review the current branch diff and report your findings.', prompt: 'Run the gstack-review skill on this repository. Review the current branch diff and report your findings.',
timeoutMs: 540_000, timeoutMs: CAPTURE_LONG_MS,
cwd: testWorktree, cwd: testWorktree,
skillName: 'gstack-review', skillName: 'gstack-review',
sections: CODEX_REVIEW_E2E_SECTIONS, sections: CODEX_REVIEW_E2E_SECTIONS,
@@ -224,5 +225,5 @@ describeCodex('Codex E2E', () => {
outputLower.includes('p1') || outputLower.includes('p1') ||
outputLower.includes('p2'); outputLower.includes('p2');
expect(hasReviewContent).toBe(true); expect(hasReviewContent).toBe(true);
}, 600_000); }, CAPTURE_LONG_MS);
}); });
+3 -2
View File
@@ -15,6 +15,7 @@
*/ */
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS } from './helpers/eval-budgets';
import { runGeminiSkill } from './helpers/gemini-session-runner'; import { runGeminiSkill } from './helpers/gemini-session-runner';
import type { GeminiResult } from './helpers/gemini-session-runner'; import type { GeminiResult } from './helpers/gemini-session-runner';
import { EvalCollector } from './helpers/eval-store'; import { EvalCollector } from './helpers/eval-store';
@@ -151,7 +152,7 @@ describeGemini('Gemini E2E', () => {
// Uses a simple prompt that doesn't require skill invocation or complex navigation. // Uses a simple prompt that doesn't require skill invocation or complex navigation.
const result = await runGeminiSkill({ const result = await runGeminiSkill({
prompt: 'What is this project? Answer in one sentence based on the README.', prompt: 'What is this project? Answer in one sentence based on the README.',
timeoutMs: 90_000, timeoutMs: JUDGE_MS,
cwd: testWorktree, cwd: testWorktree,
}); });
@@ -163,5 +164,5 @@ describeGemini('Gemini E2E', () => {
recordGeminiE2E('gemini-smoke', result, passed); recordGeminiE2E('gemini-smoke', result, passed);
expect(result.output.length, 'Gemini should produce output').toBeGreaterThan(10); expect(result.output.length, 'Gemini should produce output').toBeGreaterThan(10);
}, 120_000); }, JUDGE_MS);
}); });
+2 -1
View File
@@ -12,6 +12,7 @@
*/ */
import { expect } from 'bun:test'; import { expect } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { judgeRecommendation } from './helpers/llm-judge'; import { judgeRecommendation } from './helpers/llm-judge';
import { describeIfSelected, testIfSelected } from './helpers/e2e-helpers'; import { describeIfSelected, testIfSelected } from './helpers/e2e-helpers';
@@ -181,5 +182,5 @@ Net: ...`);
`[hedge:${label}] expected commits=false; got ${score.commits}. text="${text}"`, `[hedge:${label}] expected commits=false; got ${score.commits}. text="${text}"`,
).toBe(false); ).toBe(false);
} }
}, 240_000); }, CAPTURE_MS);
}); });
@@ -23,6 +23,7 @@
* A/B and matrix evals (test/helpers/auq-sdk-capture.ts). * A/B and matrix evals (test/helpers/auq-sdk-capture.ts).
*/ */
import { test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import { import {
@@ -86,6 +87,6 @@ describeE2E('AskUserQuestion format compliance (gate)', () => {
); );
} }
}, },
300_000, CAPTURE_MS,
); );
}); });
+2 -1
View File
@@ -16,6 +16,7 @@
* (N SDK runs, ~$0.50-1 each). * (N SDK runs, ~$0.50-1 each).
*/ */
import { test } from 'bun:test'; import { test } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import { import {
@@ -99,6 +100,6 @@ describeE2E('AUQ consistency across runs (periodic)', () => {
`format elements every run; substance ${minSub}-${maxSub}`, `format elements every run; substance ${minSub}-${maxSub}`,
); );
}, },
N_RUNS * 300_000 + 60_000, N_RUNS * CAPTURE_MS + 60_000,
); );
}); });
+2 -1
View File
@@ -23,6 +23,7 @@
* Run a subset in the foreground with AUQ_MATRIX_ONLY="plan-eng-review,cso". * Run a subset in the foreground with AUQ_MATRIX_ONLY="plan-eng-review,cso".
*/ */
import { test } from 'bun:test'; import { test } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import { import {
@@ -174,7 +175,7 @@ describeE2E('AUQ behavioral matrix (periodic)', () => {
); );
} }
}, },
300_000, CAPTURE_MS,
); );
} }
}); });
@@ -23,6 +23,7 @@
* strictly less unrelated review-section text in context. * strictly less unrelated review-section text in context.
*/ */
import { test } from 'bun:test'; import { test } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import { import {
@@ -109,6 +110,6 @@ describeE2E('AUQ no-degradation: verbose vs carved (periodic)', () => {
// eslint-disable-next-line no-console // eslint-disable-next-line no-console
console.log('[AUQ-AB] NO DEGRADATION:\n' + summary); console.log('[AUQ-AB] NO DEGRADATION:\n' + summary);
}, },
600_000, CAPTURE_LONG_MS,
); );
}); });
+3 -2
View File
@@ -38,6 +38,7 @@
*/ */
import { test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillObservation } from './helpers/claude-pty-runner'; import { runPlanSkillObservation } from './helpers/claude-pty-runner';
import * as fs from 'fs'; import * as fs from 'fs';
@@ -113,7 +114,7 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', ()
skillName: 'plan-ceo-review', skillName: 'plan-ceo-review',
inPlanMode: true, inPlanMode: true,
extraArgs: ['--disallowedTools', 'AskUserQuestion'], extraArgs: ['--disallowedTools', 'AskUserQuestion'],
timeoutMs: 540_000, timeoutMs: CAPTURE_LONG_MS,
env: { GSTACK_HOME: tmpHome, CONDUCTOR_WORKSPACE_PATH: tmpHome }, env: { GSTACK_HOME: tmpHome, CONDUCTOR_WORKSPACE_PATH: tmpHome },
}); });
@@ -137,5 +138,5 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', ()
} finally { } finally {
try { fs.rmSync(tmpHome, { recursive: true, force: true }); } catch { /* best-effort */ } try { fs.rmSync(tmpHome, { recursive: true, force: true }); } catch { /* best-effort */ }
} }
}, 660_000); }, PTY_MS);
}); });
+2 -1
View File
@@ -25,6 +25,7 @@
*/ */
import { test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { PTY_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import { spawnSync } from 'child_process'; import { spawnSync } from 'child_process';
import * as fs from 'fs'; import * as fs from 'fs';
@@ -174,6 +175,6 @@ describeE2E('/autoplan chain ordering (periodic)', () => {
try { fs.rmSync(tempDir, { recursive: true, force: true }); } catch { /* ignore */ } try { fs.rmSync(tempDir, { recursive: true, force: true }); } catch { /* ignore */ }
} }
}, },
1_200_000, // 20 min absolute test ceiling PTY_LONG_MS, // 20 min absolute test ceiling
); );
}); });
+2 -1
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { import {
ROOT, runId, evalsEnabled, ROOT, runId, evalsEnabled,
@@ -98,7 +99,7 @@ Add a new /greet skill that prints a welcome message.
testName: 'autoplan-dual-voice', testName: 'autoplan-dual-voice',
workingDirectory: workDir, workingDirectory: workDir,
prompt: `/autoplan ${planPath}`, prompt: `/autoplan ${planPath}`,
timeout: 600_000, // 10 min timeout: CAPTURE_LONG_MS, // 10 min
// /autoplan spawns subagents and calls codex via Bash; it needs the // /autoplan spawns subagents and calls codex via Bash; it needs the
// full tool set to get past Phase 1. Bash+Read+Write alone wasn't // full tool set to get past Phase 1. Bash+Read+Write alone wasn't
// enough — the skill stalled trying to invoke Agent/Skill. // enough — the skill stalled trying to invoke Agent/Skill.
+9 -8
View File
@@ -19,6 +19,7 @@
*/ */
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { ClaudeAdapter } from './helpers/providers/claude'; import { ClaudeAdapter } from './helpers/providers/claude';
import { GptAdapter } from './helpers/providers/gpt'; import { GptAdapter } from './helpers/providers/gpt';
import { GeminiAdapter } from './helpers/providers/gemini'; import { GeminiAdapter } from './helpers/providers/gemini';
@@ -94,7 +95,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
process.stderr.write(`\nclaude live smoke: SKIPPED — ${check.reason}\n`); process.stderr.write(`\nclaude live smoke: SKIPPED — ${check.reason}\n`);
return; return;
} }
const result = await claude.run({ prompt: PROMPT, workdir, timeoutMs: 120_000 }); const result = await claude.run({ prompt: PROMPT, workdir, timeoutMs: JUDGE_MS });
if (result.error) { if (result.error) {
throw new Error(`claude errored: ${result.error.code}${result.error.reason}`); throw new Error(`claude errored: ${result.error.code}${result.error.reason}`);
} }
@@ -106,7 +107,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
expect(result.modelUsed.length).toBeGreaterThan(0); expect(result.modelUsed.length).toBeGreaterThan(0);
const cost = claude.estimateCost(result.tokens, result.modelUsed); const cost = claude.estimateCost(result.tokens, result.modelUsed);
expect(cost).toBeGreaterThan(0); expect(cost).toBeGreaterThan(0);
}, 150_000); }, CAPTURE_MS);
test('gpt: trivial prompt produces parseable output', async () => { test('gpt: trivial prompt produces parseable output', async () => {
const check = await gpt.available(); const check = await gpt.available();
@@ -114,7 +115,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
process.stderr.write(`\ngpt live smoke: SKIPPED — ${check.reason}\n`); process.stderr.write(`\ngpt live smoke: SKIPPED — ${check.reason}\n`);
return; return;
} }
const result = await gpt.run({ prompt: PROMPT, workdir, timeoutMs: 120_000 }); const result = await gpt.run({ prompt: PROMPT, workdir, timeoutMs: JUDGE_MS });
if (result.error) { if (result.error) {
throw new Error(`gpt errored: ${result.error.code}${result.error.reason}`); throw new Error(`gpt errored: ${result.error.code}${result.error.reason}`);
} }
@@ -125,7 +126,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
expect(typeof result.modelUsed).toBe('string'); expect(typeof result.modelUsed).toBe('string');
const cost = gpt.estimateCost(result.tokens, result.modelUsed); const cost = gpt.estimateCost(result.tokens, result.modelUsed);
expect(cost).toBeGreaterThan(0); expect(cost).toBeGreaterThan(0);
}, 150_000); }, CAPTURE_MS);
test('gemini: trivial prompt produces parseable output', async () => { test('gemini: trivial prompt produces parseable output', async () => {
const check = await gemini.available(); const check = await gemini.available();
@@ -133,7 +134,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
process.stderr.write(`\ngemini live smoke: SKIPPED — ${check.reason}\n`); process.stderr.write(`\ngemini live smoke: SKIPPED — ${check.reason}\n`);
return; return;
} }
const result = await gemini.run({ prompt: PROMPT, workdir, timeoutMs: 120_000 }); const result = await gemini.run({ prompt: PROMPT, workdir, timeoutMs: JUDGE_MS });
if (result.error) { if (result.error) {
// auth / rate_limit are ENVIRONMENT conditions the test can't act on // auth / rate_limit are ENVIRONMENT conditions the test can't act on
// (e.g. Google deprecated the individual code-assist auth path — the // (e.g. Google deprecated the individual code-assist auth path — the
@@ -155,7 +156,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
expect(result.durationMs).toBeGreaterThan(0); expect(result.durationMs).toBeGreaterThan(0);
expect(typeof result.modelUsed).toBe('string'); expect(typeof result.modelUsed).toBe('string');
expect(result.modelUsed.length).toBeGreaterThan(0); expect(result.modelUsed.length).toBeGreaterThan(0);
}, 150_000); }, CAPTURE_MS);
test('timeout error surfaces as error.code=timeout (no exception)', async () => { test('timeout error surfaces as error.code=timeout (no exception)', async () => {
// Use whatever adapter is available first — all three should share timeout semantics. // Use whatever adapter is available first — all three should share timeout semantics.
@@ -183,7 +184,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
prompt: PROMPT, prompt: PROMPT,
workdir, workdir,
providers: ['claude', 'gpt', 'gemini'], providers: ['claude', 'gpt', 'gemini'],
timeoutMs: 120_000, timeoutMs: JUDGE_MS,
skipUnavailable: false, skipUnavailable: false,
}); });
expect(report.entries).toHaveLength(3); expect(report.entries).toHaveLength(3);
@@ -201,5 +202,5 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
if (!hadSuccess) { if (!hadSuccess) {
process.stderr.write('\nrunBenchmark live: no provider produced a clean result (no auth?)\n'); process.stderr.write('\nrunBenchmark live: no provider produced a clean result (no auth?)\n');
} }
}, 300_000); }, CAPTURE_MS);
}); });
+3 -2
View File
@@ -21,6 +21,7 @@
*/ */
import { test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'fs'; import * as fs from 'fs';
import * as os from 'os'; import * as os from 'os';
@@ -150,7 +151,7 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
fs.rmSync(fakeBinDir, { recursive: true, force: true }); fs.rmSync(fakeBinDir, { recursive: true, force: true });
fs.rmSync(tempHome, { recursive: true, force: true }); fs.rmSync(tempHome, { recursive: true, force: true });
} }
}, 180_000); }, CAPTURE_MS);
test('privacy gate does NOT fire when artifacts_sync_mode_prompted is already true', async () => { test('privacy gate does NOT fire when artifacts_sync_mode_prompted is already true', async () => {
// Same staging, but prompted=true this time. Gate should be silent. // Same staging, but prompted=true this time. Gate should be silent.
@@ -228,5 +229,5 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
fs.rmSync(fakeBinDir, { recursive: true, force: true }); fs.rmSync(fakeBinDir, { recursive: true, force: true });
fs.rmSync(tempHome, { recursive: true, force: true }); fs.rmSync(tempHome, { recursive: true, force: true });
} }
}, 180_000); }, CAPTURE_MS);
}); });
+12 -11
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { import {
ROOT, browseBin, runId, evalsEnabled, ROOT, browseBin, runId, evalsEnabled,
@@ -47,7 +48,7 @@ describeIfSelected('Skill E2E tests', [
Report the results of each command.`, Report the results of each command.`,
workingDirectory: tmpDir, workingDirectory: tmpDir,
maxTurns: 7, maxTurns: 7,
timeout: 60_000, timeout: JUDGE_MS,
testName: 'browse-basic', testName: 'browse-basic',
runId, runId,
}); });
@@ -56,7 +57,7 @@ Report the results of each command.`,
recordE2E(evalCollector, 'browse basic commands', 'Skill E2E tests', result); recordE2E(evalCollector, 'browse basic commands', 'Skill E2E tests', result);
expect(result.browseErrors).toHaveLength(0); expect(result.browseErrors).toHaveLength(0);
expect(result.exitReason).toBe('success'); expect(result.exitReason).toBe('success');
}, 90_000); }, JUDGE_MS);
testConcurrentIfSelected('browse-snapshot', async () => { testConcurrentIfSelected('browse-snapshot', async () => {
const result = await runSkillTest({ const result = await runSkillTest({
@@ -69,7 +70,7 @@ Report the results of each command.`,
Report what each command returned.`, Report what each command returned.`,
workingDirectory: tmpDir, workingDirectory: tmpDir,
maxTurns: 9, maxTurns: 9,
timeout: 60_000, timeout: JUDGE_MS,
testName: 'browse-snapshot', testName: 'browse-snapshot',
runId, runId,
}); });
@@ -81,7 +82,7 @@ Report what each command returned.`,
console.warn('Browse errors (non-fatal):', result.browseErrors); console.warn('Browse errors (non-fatal):', result.browseErrors);
} }
expect(result.exitReason).toBe('success'); expect(result.exitReason).toBe('success');
}, 90_000); }, JUDGE_MS);
testConcurrentIfSelected('skillmd-setup-discovery', async () => { testConcurrentIfSelected('skillmd-setup-discovery', async () => {
// P2 (v1.2.0): the browse SETUP/binary-discovery block moved from the root // P2 (v1.2.0): the browse SETUP/binary-discovery block moved from the root
@@ -104,7 +105,7 @@ Then run: $B text
Report whether it worked.`, Report whether it worked.`,
workingDirectory: tmpDir, workingDirectory: tmpDir,
maxTurns: 10, maxTurns: 10,
timeout: 60_000, timeout: JUDGE_MS,
testName: 'skillmd-setup-discovery', testName: 'skillmd-setup-discovery',
runId, runId,
}); });
@@ -112,7 +113,7 @@ Report whether it worked.`,
recordE2E(evalCollector, 'SKILL.md setup block discovery', 'Skill E2E tests', result); recordE2E(evalCollector, 'SKILL.md setup block discovery', 'Skill E2E tests', result);
expect(result.browseErrors).toHaveLength(0); expect(result.browseErrors).toHaveLength(0);
expect(result.exitReason).toBe('success'); expect(result.exitReason).toBe('success');
}, 90_000); }, JUDGE_MS);
testConcurrentIfSelected('skillmd-no-local-binary', async () => { testConcurrentIfSelected('skillmd-no-local-binary', async () => {
// Create a tmpdir with no browse binary — no local .claude/skills/gstack/browse/dist/browse // Create a tmpdir with no browse binary — no local .claude/skills/gstack/browse/dist/browse
@@ -149,7 +150,7 @@ Report the exact output. Do NOT try to fix or install anything — just report w
// Clean up // Clean up
try { fs.rmSync(emptyDir, { recursive: true, force: true }); } catch {} try { fs.rmSync(emptyDir, { recursive: true, force: true }); } catch {}
}, 60_000); }, JUDGE_MS);
testConcurrentIfSelected('skillmd-outside-git', async () => { testConcurrentIfSelected('skillmd-outside-git', async () => {
// Create a tmpdir outside any git repo // Create a tmpdir outside any git repo
@@ -182,7 +183,7 @@ Report the exact output — either "READY: <path>" or "NEEDS_SETUP".`,
// Clean up // Clean up
try { fs.rmSync(nonGitDir, { recursive: true, force: true }); } catch {} try { fs.rmSync(nonGitDir, { recursive: true, force: true }); } catch {}
}, 60_000); }, JUDGE_MS);
testConcurrentIfSelected('operational-learning', async () => { testConcurrentIfSelected('operational-learning', async () => {
const opDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-oplearn-')); const opDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-oplearn-'));
@@ -286,7 +287,7 @@ Log the operational learning now. Then say what you logged.`,
// Clean up // Clean up
try { fs.rmSync(opDir, { recursive: true, force: true }); } catch {} try { fs.rmSync(opDir, { recursive: true, force: true }); } catch {}
}, 90_000); }, JUDGE_MS);
testConcurrentIfSelected('session-awareness', async () => { testConcurrentIfSelected('session-awareness', async () => {
const sessionDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-session-')); const sessionDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-session-'));
@@ -353,7 +354,7 @@ Since this is non-interactive, DO NOT actually call AskUserQuestion. Instead, wr
Remember: _SESSIONS=4, so ELI16 mode is active. The user is juggling multiple windows and may not remember what this conversation is about. Re-ground them.`, Remember: _SESSIONS=4, so ELI16 mode is active. The user is juggling multiple windows and may not remember what this conversation is about. Re-ground them.`,
workingDirectory: sessionDir, workingDirectory: sessionDir,
maxTurns: 8, maxTurns: 8,
timeout: 60_000, timeout: JUDGE_MS,
testName: 'session-awareness', testName: 'session-awareness',
runId, runId,
}); });
@@ -394,7 +395,7 @@ Remember: _SESSIONS=4, so ELI16 mode is active. The user is juggling multiple wi
// Clean up // Clean up
try { fs.rmSync(sessionDir, { recursive: true, force: true }); } catch {} try { fs.rmSync(sessionDir, { recursive: true, force: true }); } catch {}
}, 90_000); }, JUDGE_MS);
}); });
// Module-level afterAll — finalize eval collector after all tests complete // Module-level afterAll — finalize eval collector after all tests complete
+3 -2
View File
@@ -21,6 +21,7 @@
*/ */
import { test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillObservation } from './helpers/claude-pty-runner'; import { runPlanSkillObservation } from './helpers/claude-pty-runner';
@@ -46,7 +47,7 @@ describeE2E('Conductor renders decisions as prose (periodic)', () => {
extraArgs: ['--disallowedTools', 'AskUserQuestion'], extraArgs: ['--disallowedTools', 'AskUserQuestion'],
env: { CONDUCTOR_WORKSPACE_PATH: '/tmp/conductor-prose-e2e' }, env: { CONDUCTOR_WORKSPACE_PATH: '/tmp/conductor-prose-e2e' },
initialPlanContent: FLAWED_PLAN, initialPlanContent: FLAWED_PLAN,
timeoutMs: 300_000, timeoutMs: CAPTURE_MS,
}); });
// The decision must reach the human as prose. 'silent_write' (wrote findings // The decision must reach the human as prose. 'silent_write' (wrote findings
@@ -65,5 +66,5 @@ describeE2E('Conductor renders decisions as prose (periodic)', () => {
} }
// A prose-rendered decision brief was observed at some point in the run. // A prose-rendered decision brief was observed at some point in the run.
expect(obs.proseAUQEverObserved).toBe(true); expect(obs.proseAUQEverObserved).toBe(true);
}, 360_000); }, CAPTURE_LONG_MS);
}); });
+17 -16
View File
@@ -12,6 +12,7 @@
*/ */
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { import {
ROOT, runId, evalsEnabled, ROOT, runId, evalsEnabled,
@@ -163,7 +164,7 @@ describeIfSelected('Context Skills E2E (live-fire)', [
env: { GSTACK_HOME: gstackHome }, env: { GSTACK_HOME: gstackHome },
maxTurns: 12, maxTurns: 12,
allowedTools: ['Skill', 'Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], allowedTools: ['Skill', 'Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000, timeout: JUDGE_MS,
testName: 'context-save-routing', testName: 'context-save-routing',
runId, runId,
}); });
@@ -185,7 +186,7 @@ describeIfSelected('Context Skills E2E (live-fire)', [
expect(routedToContextSave).toBe(true); expect(routedToContextSave).toBe(true);
expect(files.length).toBeGreaterThan(0); expect(files.length).toBeGreaterThan(0);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 180_000); }, CAPTURE_MS);
// ── 2. Round-trip: save then restore in the same session ───────────── // ── 2. Round-trip: save then restore in the same session ─────────────
testConcurrentIfSelected('context-save-then-restore-roundtrip', async () => { testConcurrentIfSelected('context-save-then-restore-roundtrip', async () => {
@@ -205,7 +206,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome }, env: { GSTACK_HOME: gstackHome },
maxTurns: 25, maxTurns: 25,
allowedTools: ['Skill', 'Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], allowedTools: ['Skill', 'Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 240_000, timeout: CAPTURE_MS,
testName: 'context-save-then-restore-roundtrip', testName: 'context-save-then-restore-roundtrip',
runId, runId,
}); });
@@ -232,7 +233,7 @@ Do NOT use AskUserQuestion.`,
expect(files.length).toBeGreaterThan(0); expect(files.length).toBeGreaterThan(0);
expect(restoreMentionsTitle).toBe(true); expect(restoreMentionsTitle).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 240_000); }, CAPTURE_MS);
// ── 3. /context-restore <fragment> loads the matching save ─────────── // ── 3. /context-restore <fragment> loads the matching save ───────────
testConcurrentIfSelected('context-restore-fragment-match', async () => { testConcurrentIfSelected('context-restore-fragment-match', async () => {
@@ -255,7 +256,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome }, env: { GSTACK_HOME: gstackHome },
maxTurns: 10, maxTurns: 10,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'], allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 120_000, timeout: JUDGE_MS,
testName: 'context-restore-fragment-match', testName: 'context-restore-fragment-match',
runId, runId,
}); });
@@ -279,7 +280,7 @@ Do NOT use AskUserQuestion.`,
expect(loadedPayments).toBe(true); expect(loadedPayments).toBe(true);
expect(didNotLoadOthers).toBe(true); expect(didNotLoadOthers).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 180_000); }, CAPTURE_MS);
// ── 4. /context-restore with zero saves → graceful empty-state ─────── // ── 4. /context-restore with zero saves → graceful empty-state ───────
testConcurrentIfSelected('context-restore-empty-state', async () => { testConcurrentIfSelected('context-restore-empty-state', async () => {
@@ -294,7 +295,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome }, env: { GSTACK_HOME: gstackHome },
maxTurns: 8, maxTurns: 8,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'], allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 90_000, timeout: JUDGE_MS,
testName: 'context-restore-empty-state', testName: 'context-restore-empty-state',
runId, runId,
}); });
@@ -319,7 +320,7 @@ Do NOT use AskUserQuestion.`,
expect(routedToRestore).toBe(true); expect(routedToRestore).toBe(true);
expect(gracefulMessage).toBe(true); expect(gracefulMessage).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 150_000); }, CAPTURE_MS);
// ── 5. /context-restore list redirects to /context-save list ───────── // ── 5. /context-restore list redirects to /context-save list ─────────
testConcurrentIfSelected('context-restore-list-delegates', async () => { testConcurrentIfSelected('context-restore-list-delegates', async () => {
@@ -334,7 +335,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome }, env: { GSTACK_HOME: gstackHome },
maxTurns: 8, maxTurns: 8,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'], allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 90_000, timeout: JUDGE_MS,
testName: 'context-restore-list-delegates', testName: 'context-restore-list-delegates',
runId, runId,
}); });
@@ -357,7 +358,7 @@ Do NOT use AskUserQuestion.`,
expect(routedToRestore).toBe(true); expect(routedToRestore).toBe(true);
expect(mentionsSaveList).toBe(true); expect(mentionsSaveList).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 150_000); }, CAPTURE_MS);
// ── 6. Legacy compat: pre-rename save files still load ─────────────── // ── 6. Legacy compat: pre-rename save files still load ───────────────
testConcurrentIfSelected('context-restore-legacy-compat', async () => { testConcurrentIfSelected('context-restore-legacy-compat', async () => {
@@ -381,7 +382,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome }, env: { GSTACK_HOME: gstackHome },
maxTurns: 8, maxTurns: 8,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'], allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 120_000, timeout: JUDGE_MS,
testName: 'context-restore-legacy-compat', testName: 'context-restore-legacy-compat',
runId, runId,
}); });
@@ -414,7 +415,7 @@ Do NOT use AskUserQuestion.`,
expect(routedToRestore).toBe(true); expect(routedToRestore).toBe(true);
expect(loadedLegacy).toBe(true); expect(loadedLegacy).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 180_000); }, CAPTURE_MS);
// ── 7. /context-save list: default filters to current branch ───────── // ── 7. /context-save list: default filters to current branch ─────────
testConcurrentIfSelected('context-save-list-current-branch', async () => { testConcurrentIfSelected('context-save-list-current-branch', async () => {
@@ -437,7 +438,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome }, env: { GSTACK_HOME: gstackHome },
maxTurns: 10, maxTurns: 10,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'], allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 120_000, timeout: JUDGE_MS,
testName: 'context-save-list-current-branch', testName: 'context-save-list-current-branch',
runId, runId,
}); });
@@ -472,7 +473,7 @@ Do NOT use AskUserQuestion.`,
expect(hidesAlpha).toBe(true); expect(hidesAlpha).toBe(true);
expect(hidesBeta).toBe(true); expect(hidesBeta).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 180_000); }, CAPTURE_MS);
// ── 8. /context-save list --all: shows every branch ────────────────── // ── 8. /context-save list --all: shows every branch ──────────────────
testConcurrentIfSelected('context-save-list-all-branches', async () => { testConcurrentIfSelected('context-save-list-all-branches', async () => {
@@ -494,7 +495,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome }, env: { GSTACK_HOME: gstackHome },
maxTurns: 10, maxTurns: 10,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'], allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 120_000, timeout: JUDGE_MS,
testName: 'context-save-list-all-branches', testName: 'context-save-list-all-branches',
runId, runId,
}); });
@@ -520,5 +521,5 @@ Do NOT use AskUserQuestion.`,
expect(routed).toBe(true); expect(routed).toBe(true);
expect(filesShown).toBe(3); expect(filesShown).toBe(3);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 180_000); }, CAPTURE_MS);
}); });
+5 -4
View File
@@ -20,6 +20,7 @@
*/ */
import { test, expect, beforeAll, afterAll } from 'bun:test'; import { test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { import {
ROOT, runId, ROOT, runId,
@@ -78,7 +79,7 @@ Output the diagram directly.`,
workingDirectory: reviewCoverageDir, workingDirectory: reviewCoverageDir,
maxTurns: 15, maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'], allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
timeout: 120_000, timeout: JUDGE_MS,
testName: 'review-coverage-audit', testName: 'review-coverage-audit',
runId, runId,
}); });
@@ -107,7 +108,7 @@ Output the diagram directly.`,
// At minimum, the agent should have read the source and test files // At minimum, the agent should have read the source and test files
const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read'); const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read');
expect(readCalls.length).toBeGreaterThan(0); expect(readCalls.length).toBeGreaterThan(0);
}, 180_000); }, CAPTURE_MS);
}); });
// --- Plan Eng Review Coverage Audit E2E --- // --- Plan Eng Review Coverage Audit E2E ---
@@ -153,7 +154,7 @@ Output the diagram directly.`,
workingDirectory: planCoverageDir, workingDirectory: planCoverageDir,
maxTurns: 15, maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'], allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
timeout: 120_000, timeout: JUDGE_MS,
testName: 'plan-eng-coverage-audit', testName: 'plan-eng-coverage-audit',
runId, runId,
}); });
@@ -182,7 +183,7 @@ Output the diagram directly.`,
// At minimum, the agent should have read the source and test files // At minimum, the agent should have read the source and test files
const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read'); const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read');
expect(readCalls.length).toBeGreaterThan(0); expect(readCalls.length).toBeGreaterThan(0);
}, 180_000); }, CAPTURE_MS);
}); });
// Module-level afterAll — finalize eval collector after all tests complete // Module-level afterAll — finalize eval collector after all tests complete
+6 -5
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { import {
ROOT, runId, evalsEnabled, ROOT, runId, evalsEnabled,
@@ -75,7 +76,7 @@ IMPORTANT:
workingDirectory: csoDir, workingDirectory: csoDir,
maxTurns: 30, maxTurns: 30,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob', 'Agent'], allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob', 'Agent'],
timeout: 300_000, timeout: CAPTURE_MS,
}); });
logCost('cso', result); logCost('cso', result);
@@ -106,7 +107,7 @@ IMPORTANT:
} }
recordE2E(evalCollector, 'cso-full-audit', 'e2e-cso', result); recordE2E(evalCollector, 'cso-full-audit', 'e2e-cso', result);
}, 300_000); }, CAPTURE_MS);
}); });
describeIfSelected('CSO v2 — diff mode', ['cso-diff-mode'], () => { describeIfSelected('CSO v2 — diff mode', ['cso-diff-mode'], () => {
@@ -181,7 +182,7 @@ IMPORTANT:
).toBe(true); ).toBe(true);
recordE2E(evalCollector, 'cso-diff-mode', 'e2e-cso', result); recordE2E(evalCollector, 'cso-diff-mode', 'e2e-cso', result);
}, 400_000); }, CAPTURE_LONG_MS);
}); });
describeIfSelected('CSO v2 — infra scope', ['cso-infra-scope'], () => { describeIfSelected('CSO v2 — infra scope', ['cso-infra-scope'], () => {
@@ -245,7 +246,7 @@ IMPORTANT:
workingDirectory: csoInfraDir, workingDirectory: csoInfraDir,
maxTurns: 30, maxTurns: 30,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 360_000, timeout: CAPTURE_LONG_MS,
}); });
logCost('cso', result); logCost('cso', result);
@@ -259,5 +260,5 @@ IMPORTANT:
).toBe(true); ).toBe(true);
recordE2E(evalCollector, 'cso-infra-scope', 'e2e-cso', result); recordE2E(evalCollector, 'cso-infra-scope', 'e2e-cso', result);
}, 360_000); }, CAPTURE_LONG_MS);
}); });
+13 -12
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { import {
ROOT, browseBin, runId, evalsEnabled, ROOT, browseBin, runId, evalsEnabled,
@@ -67,7 +68,7 @@ Do NOT use AskUserQuestion. Do NOT run gh or fly commands.`,
workingDirectory: landDir, workingDirectory: landDir,
maxTurns: 20, maxTurns: 20,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000, timeout: JUDGE_MS,
testName: 'land-and-deploy-workflow', testName: 'land-and-deploy-workflow',
runId, runId,
}); });
@@ -85,7 +86,7 @@ Do NOT use AskUserQuestion. Do NOT run gh or fly commands.`,
const reportDir = path.join(landDir, '.gstack', 'deploy-reports'); const reportDir = path.join(landDir, '.gstack', 'deploy-reports');
expect(fs.existsSync(reportDir)).toBe(true); expect(fs.existsSync(reportDir)).toBe(true);
}, 180_000); }, CAPTURE_MS);
}); });
// --- Land-and-Deploy First-Run E2E --- // --- Land-and-Deploy First-Run E2E ---
@@ -148,7 +149,7 @@ Just demonstrate the first-run dry-run output.`,
workingDirectory: firstRunDir, workingDirectory: firstRunDir,
maxTurns: 20, maxTurns: 20,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000, timeout: JUDGE_MS,
testName: 'land-and-deploy-first-run', testName: 'land-and-deploy-first-run',
runId, runId,
}); });
@@ -167,7 +168,7 @@ Just demonstrate the first-run dry-run output.`,
const reportContent = fs.readFileSync(path.join(reportDir, reportFiles[0]), 'utf-8'); const reportContent = fs.readFileSync(path.join(reportDir, reportFiles[0]), 'utf-8');
const hasPlatform = reportContent.toLowerCase().includes('fly') || reportContent.toLowerCase().includes('first-run-app'); const hasPlatform = reportContent.toLowerCase().includes('fly') || reportContent.toLowerCase().includes('first-run-app');
expect(hasPlatform).toBe(true); expect(hasPlatform).toBe(true);
}, 180_000); }, CAPTURE_MS);
}); });
// --- Land-and-Deploy Review Gate E2E --- // --- Land-and-Deploy Review Gate E2E ---
@@ -226,7 +227,7 @@ Show what the readiness gate output would look like.`,
workingDirectory: reviewDir, workingDirectory: reviewDir,
maxTurns: 15, maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000, timeout: JUDGE_MS,
testName: 'land-and-deploy-review-gate', testName: 'land-and-deploy-review-gate',
runId, runId,
}); });
@@ -246,7 +247,7 @@ Show what the readiness gate output would look like.`,
const hasReviewMention = reportContent.toLowerCase().includes('review') || const hasReviewMention = reportContent.toLowerCase().includes('review') ||
reportContent.toLowerCase().includes('not run'); reportContent.toLowerCase().includes('not run');
expect(hasReviewMention).toBe(true); expect(hasReviewMention).toBe(true);
}, 180_000); }, CAPTURE_MS);
}); });
// --- Canary skill E2E --- // --- Canary skill E2E ---
@@ -294,7 +295,7 @@ Just create the directory structure and report files showing the correct schema.
workingDirectory: canaryDir, workingDirectory: canaryDir,
maxTurns: 15, maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'], allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'],
timeout: 120_000, timeout: JUDGE_MS,
testName: 'canary-workflow', testName: 'canary-workflow',
runId, runId,
}); });
@@ -307,7 +308,7 @@ Just create the directory structure and report files showing the correct schema.
const reportDir = path.join(canaryDir, '.gstack', 'canary-reports'); const reportDir = path.join(canaryDir, '.gstack', 'canary-reports');
const files = fs.readdirSync(reportDir, { recursive: true }) as string[]; const files = fs.readdirSync(reportDir, { recursive: true }) as string[];
expect(files.length).toBeGreaterThan(0); expect(files.length).toBeGreaterThan(0);
}, 180_000); }, CAPTURE_MS);
}); });
// --- Benchmark skill E2E --- // --- Benchmark skill E2E ---
@@ -357,7 +358,7 @@ Just create the files showing the correct schema and report format.`,
workingDirectory: benchDir, workingDirectory: benchDir,
maxTurns: 15, maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'], allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'],
timeout: 120_000, timeout: JUDGE_MS,
testName: 'benchmark-workflow', testName: 'benchmark-workflow',
runId, runId,
}); });
@@ -372,7 +373,7 @@ Just create the files showing the correct schema and report format.`,
const files = fs.readdirSync(baselineDir); const files = fs.readdirSync(baselineDir);
expect(files.length).toBeGreaterThan(0); expect(files.length).toBeGreaterThan(0);
} }
}, 180_000); }, CAPTURE_MS);
}); });
// --- Setup-Deploy skill E2E --- // --- Setup-Deploy skill E2E ---
@@ -418,7 +419,7 @@ Just detect the platform and write the config.`,
workingDirectory: setupDir, workingDirectory: setupDir,
maxTurns: 15, maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000, timeout: JUDGE_MS,
testName: 'setup-deploy-workflow', testName: 'setup-deploy-workflow',
runId, runId,
}); });
@@ -434,7 +435,7 @@ Just detect the platform and write the config.`,
expect(content.toLowerCase()).toContain('fly'); expect(content.toLowerCase()).toContain('fly');
expect(content).toContain('my-cool-app'); expect(content).toContain('my-cool-app');
expect(content).toContain('Deploy Configuration'); expect(content).toContain('Deploy Configuration');
}, 180_000); }, CAPTURE_MS);
}); });
// Module-level afterAll — finalize eval collector after all tests complete // Module-level afterAll — finalize eval collector after all tests complete
+15 -14
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { callJudge } from './helpers/llm-judge'; import { callJudge } from './helpers/llm-judge';
import { import {
@@ -113,7 +114,7 @@ Skip research — work from your design knowledge. Skip the font preview page. S
Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`, Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`,
workingDirectory: designDir, workingDirectory: designDir,
maxTurns: 20, maxTurns: 20,
timeout: 360_000, timeout: CAPTURE_LONG_MS,
testName: 'design-consultation-core', testName: 'design-consultation-core',
runId, runId,
model: 'claude-opus-4-7', model: 'claude-opus-4-7',
@@ -178,7 +179,7 @@ Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`,
const claude = fs.readFileSync(claudePath, 'utf-8'); const claude = fs.readFileSync(claudePath, 'utf-8');
expect(claude.toLowerCase()).toContain('design.md'); expect(claude.toLowerCase()).toContain('design.md');
} }
}, 420_000); }, CAPTURE_LONG_MS);
testConcurrentIfSelected('design-consultation-research', async () => { testConcurrentIfSelected('design-consultation-research', async () => {
// Test WebSearch integration — research phase only, no DESIGN.md generation // Test WebSearch integration — research phase only, no DESIGN.md generation
@@ -202,7 +203,7 @@ Do NOT generate a full DESIGN.md — just research notes.`,
// queued past the budget under concurrent API load. 90s budgets cannot // queued past the budget under concurrent API load. 90s budgets cannot
// absorb one slow first completion; 300s is the repo's standard floor // absorb one slow first completion; 300s is the repo's standard floor
// for CI SDK tests. Outer timeout below rises to 360s for headroom. // for CI SDK tests. Outer timeout below rises to 360s for headroom.
timeout: 300_000, timeout: CAPTURE_MS,
testName: 'design-consultation-research', testName: 'design-consultation-research',
runId, runId,
}); });
@@ -232,7 +233,7 @@ Do NOT generate a full DESIGN.md — just research notes.`,
} }
try { fs.rmSync(researchDir, { recursive: true, force: true }); } catch {} try { fs.rmSync(researchDir, { recursive: true, force: true }); } catch {}
}, 360_000); }, CAPTURE_LONG_MS);
testConcurrentIfSelected('design-consultation-existing', async () => { testConcurrentIfSelected('design-consultation-existing', async () => {
// Pre-create a minimal DESIGN.md (independent of core test) // Pre-create a minimal DESIGN.md (independent of core test)
@@ -250,7 +251,7 @@ There is already a DESIGN.md in this repo. Update it with a complete design syst
Skip research. Skip font preview. Skip any AskUserQuestion calls — this is non-interactive.`, Skip research. Skip font preview. Skip any AskUserQuestion calls — this is non-interactive.`,
workingDirectory: designDir, workingDirectory: designDir,
maxTurns: 20, maxTurns: 20,
timeout: 360_000, timeout: CAPTURE_LONG_MS,
testName: 'design-consultation-existing', testName: 'design-consultation-existing',
runId, runId,
model: 'claude-opus-4-7', model: 'claude-opus-4-7',
@@ -279,7 +280,7 @@ Skip research. Skip font preview. Skip any AskUserQuestion calls — this is non
expect(hasColor).toBe(true); expect(hasColor).toBe(true);
expect(hasSpacing).toBe(true); expect(hasSpacing).toBe(true);
} }
}, 420_000); }, CAPTURE_LONG_MS);
testConcurrentIfSelected('design-consultation-preview', async () => { testConcurrentIfSelected('design-consultation-preview', async () => {
// Test preview HTML generation only — no DESIGN.md (covered by core test) // Test preview HTML generation only — no DESIGN.md (covered by core test)
@@ -302,7 +303,7 @@ Do NOT write DESIGN.md — only the preview HTML.`,
maxTurns: 8, maxTurns: 8,
// 300s, not 90s: this is the test that failed 3x at 0 turns/$0.00/93s // 300s, not 90s: this is the test that failed 3x at 0 turns/$0.00/93s
// on PR #2533 CI — see the research test's comment for the class. // on PR #2533 CI — see the research test's comment for the class.
timeout: 300_000, timeout: CAPTURE_MS,
testName: 'design-consultation-preview', testName: 'design-consultation-preview',
runId, runId,
}); });
@@ -331,7 +332,7 @@ Do NOT write DESIGN.md — only the preview HTML.`,
} }
try { fs.rmSync(previewDir, { recursive: true, force: true }); } catch {} try { fs.rmSync(previewDir, { recursive: true, force: true }); } catch {}
}, 360_000); }, CAPTURE_LONG_MS);
}); });
// --- Plan Design Review E2E (plan-mode) --- // --- Plan Design Review E2E (plan-mode) ---
@@ -398,7 +399,7 @@ Skip the preamble bash block. Skip any AskUserQuestion calls — this is non-int
IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan review, not a live site audit. Just read the plan file, review it, and edit it to fix the gaps.`, IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan review, not a live site audit. Just read the plan file, review it, and edit it to fix the gaps.`,
workingDirectory: reviewDir, workingDirectory: reviewDir,
maxTurns: 15, maxTurns: 15,
timeout: 300_000, timeout: CAPTURE_MS,
testName: 'plan-design-review-plan-mode', testName: 'plan-design-review-plan-mode',
runId, runId,
}); });
@@ -437,7 +438,7 @@ IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan
} finally { } finally {
try { fs.rmSync(reviewDir, { recursive: true, force: true }); } catch {} try { fs.rmSync(reviewDir, { recursive: true, force: true }); } catch {}
} }
}, 360_000); }, CAPTURE_LONG_MS);
testConcurrentIfSelected('plan-design-review-no-ui-scope', async () => { testConcurrentIfSelected('plan-design-review-no-ui-scope', async () => {
const reviewDir = setupReviewDir(); const reviewDir = setupReviewDir();
@@ -472,7 +473,7 @@ Skip the preamble bash block. Skip any AskUserQuestion calls — this is non-int
IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan review, not a live site audit.`, IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan review, not a live site audit.`,
workingDirectory: reviewDir, workingDirectory: reviewDir,
maxTurns: 10, maxTurns: 10,
timeout: 180_000, timeout: CAPTURE_MS,
testName: 'plan-design-review-no-ui-scope', testName: 'plan-design-review-no-ui-scope',
runId, runId,
}); });
@@ -496,7 +497,7 @@ IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan
} finally { } finally {
try { fs.rmSync(reviewDir, { recursive: true, force: true }); } catch {} try { fs.rmSync(reviewDir, { recursive: true, force: true }); } catch {}
} }
}, 240_000); }, CAPTURE_MS);
}); });
// --- Design Review E2E (live-site audit + fix) --- // --- Design Review E2E (live-site audit + fix) ---
@@ -602,7 +603,7 @@ Read design-review/SKILL.md for the design review + fix workflow.
Review the site at ${serverUrl}. Use --quick mode. Skip any AskUserQuestion calls — this is non-interactive. Fix up to 3 issues max. Write your report to ./design-audit.md.`, Review the site at ${serverUrl}. Use --quick mode. Skip any AskUserQuestion calls — this is non-interactive. Fix up to 3 issues max. Write your report to ./design-audit.md.`,
workingDirectory: qaDesignDir, workingDirectory: qaDesignDir,
maxTurns: 30, maxTurns: 30,
timeout: 360_000, timeout: CAPTURE_LONG_MS,
testName: 'design-review-fix', testName: 'design-review-fix',
runId, runId,
}); });
@@ -634,7 +635,7 @@ Review the site at ${serverUrl}. Use --quick mode. Skip any AskUserQuestion call
console.warn('No design-audit.md generated'); console.warn('No design-audit.md generated');
} }
console.log(`Design fix commits: ${designFixCommits.length}`); console.log(`Design fix commits: ${designFixCommits.length}`);
}, 420_000); }, CAPTURE_LONG_MS);
}); });
// Module-level afterAll — finalize eval collector after all tests complete // Module-level afterAll — finalize eval collector after all tests complete
+5 -4
View File
@@ -17,6 +17,7 @@
* with its preamble. * with its preamble.
*/ */
import { describe, expect } from 'bun:test'; import { describe, expect } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import * as path from 'node:path'; import * as path from 'node:path';
import * as os from 'node:os'; import * as os from 'node:os';
@@ -73,7 +74,7 @@ describeIfSelected('/diagram skill E2E', ['diagram-triplet', 'diagram-authoring-
workingDirectory: dir, workingDirectory: dir,
maxTurns: 25, maxTurns: 25,
allowedTools: ['Bash', 'Read', 'Write'], allowedTools: ['Bash', 'Read', 'Write'],
timeout: 240_000, timeout: CAPTURE_MS,
testName: 'diagram-triplet', testName: 'diagram-triplet',
runId, runId,
}); });
@@ -98,7 +99,7 @@ describeIfSelected('/diagram skill E2E', ['diagram-triplet', 'diagram-authoring-
} finally { } finally {
try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
} }
}, 300_000); }, CAPTURE_MS);
testConcurrentIfSelected('diagram-authoring-quality', async () => { testConcurrentIfSelected('diagram-authoring-quality', async () => {
const dir = setupDir('diagram-quality-'); const dir = setupDir('diagram-quality-');
@@ -111,7 +112,7 @@ describeIfSelected('/diagram skill E2E', ['diagram-triplet', 'diagram-authoring-
workingDirectory: dir, workingDirectory: dir,
maxTurns: 25, maxTurns: 25,
allowedTools: ['Bash', 'Read', 'Write'], allowedTools: ['Bash', 'Read', 'Write'],
timeout: 240_000, timeout: CAPTURE_MS,
testName: 'diagram-authoring-quality', testName: 'diagram-authoring-quality',
runId, runId,
}); });
@@ -149,5 +150,5 @@ Respond with JSON: {"score": N, "reasoning": "..."}`,
} finally { } finally {
try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
} }
}, 300_000); }, CAPTURE_MS);
}); });
+3 -2
View File
@@ -14,6 +14,7 @@
*/ */
import { expect, afterAll } from 'bun:test'; import { expect, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import * as fs from 'fs'; import * as fs from 'fs';
import * as path from 'path'; import * as path from 'path';
import * as os from 'os'; import * as os from 'os';
@@ -55,7 +56,7 @@ async function detectVia(workDir: string, testName: string): Promise<string> {
workingDirectory: workDir, workingDirectory: workDir,
maxTurns: 3, maxTurns: 3,
allowedTools: ['Bash'], allowedTools: ['Bash'],
timeout: 120_000, timeout: JUDGE_MS,
testName, testName,
runId, runId,
model: MODEL, model: MODEL,
@@ -91,7 +92,7 @@ describeIfSelected('first-run scaffold detection (E2E)', ['first-task-scaffold']
fs.rmSync(nodeDir, { recursive: true, force: true }); fs.rmSync(nodeDir, { recursive: true, force: true });
fs.rmSync(greenDir, { recursive: true, force: true }); fs.rmSync(greenDir, { recursive: true, force: true });
} }
}, 300_000); }, CAPTURE_MS);
}); });
afterAll(() => finalizeEvalCollector(evalCollector)); afterAll(() => finalizeEvalCollector(evalCollector));
@@ -29,6 +29,7 @@
*/ */
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS } from './helpers/eval-budgets';
import { execFileSync } from 'child_process'; import { execFileSync } from 'child_process';
import { mkdtempSync, rmSync } from 'fs'; import { mkdtempSync, rmSync } from 'fs';
import { tmpdir } from 'os'; import { tmpdir } from 'os';
@@ -156,7 +157,7 @@ ${body}`;
expect(retrieved).not.toContain('page_not_found'); expect(retrieved).not.toContain('page_not_found');
expect(retrieved).not.toContain('Page not found'); expect(retrieved).not.toContain('Page not found');
}, },
120_000, JUDGE_MS,
); );
}, },
); );
+5 -4
View File
@@ -31,6 +31,7 @@
*/ */
import { expect, afterAll } from 'bun:test'; import { expect, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import * as fs from 'fs'; import * as fs from 'fs';
import * as path from 'path'; import * as path from 'path';
import * as os from 'os'; import * as os from 'os';
@@ -94,7 +95,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic-
workingDirectory: workDir, workingDirectory: workDir,
maxTurns: 3, maxTurns: 3,
allowedTools: ['Bash'], allowedTools: ['Bash'],
timeout: 120_000, timeout: JUDGE_MS,
testName: 'hermetic-canary', testName: 'hermetic-canary',
runId, runId,
model: CANARY_MODEL, model: CANARY_MODEL,
@@ -129,7 +130,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic-
} }
fs.rmSync(workDir, { recursive: true, force: true }); fs.rmSync(workDir, { recursive: true, force: true });
} }
}, 180_000); }, CAPTURE_MS);
testIfSelected('hermetic-sentinel', async () => { testIfSelected('hermetic-sentinel', async () => {
if (!process.env.ANTHROPIC_API_KEY) { if (!process.env.ANTHROPIC_API_KEY) {
@@ -158,7 +159,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic-
workingDirectory: workDir, workingDirectory: workDir,
maxTurns: 3, maxTurns: 3,
allowedTools: ['Bash'], allowedTools: ['Bash'],
timeout: 120_000, timeout: JUDGE_MS,
testName: 'hermetic-sentinel', testName: 'hermetic-sentinel',
runId, runId,
model: CANARY_MODEL, model: CANARY_MODEL,
@@ -188,7 +189,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic-
fs.rmSync(workDir, { recursive: true, force: true }); fs.rmSync(workDir, { recursive: true, force: true });
fs.rmSync(poisonRoot, { recursive: true, force: true }); fs.rmSync(poisonRoot, { recursive: true, force: true });
} }
}, 180_000); }, CAPTURE_MS);
}); });
afterAll(() => finalizeEvalCollector(evalCollector)); afterAll(() => finalizeEvalCollector(evalCollector));
+2 -1
View File
@@ -21,6 +21,7 @@
// intentionally machine-specific. // intentionally machine-specific.
import { describe, test, expect } from 'bun:test'; import { describe, test, expect } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { spawnSync } from 'child_process'; import { spawnSync } from 'child_process';
import { cpSync, existsSync, mkdtempSync, readFileSync, rmSync, unlinkSync } from 'fs'; import { cpSync, existsSync, mkdtempSync, readFileSync, rmSync, unlinkSync } from 'fs';
import { tmpdir } from 'os'; import { tmpdir } from 'os';
@@ -856,7 +857,7 @@ describe('ios device deployment (explicit opt-in)', () => {
keepalive?.stop(); keepalive?.stop();
rmSync(workDir, { recursive: true, force: true }); rmSync(workDir, { recursive: true, force: true });
} }
}, 600_000); }, CAPTURE_LONG_MS);
}); });
// Always-on instructions if not paired. Surfaces actionable steps even when // Always-on instructions if not paired. Surfaces actionable steps even when
+4 -3
View File
@@ -18,6 +18,7 @@
// gated (no compilation step for DebugBridgeCore/UI) // gated (no compilation step for DebugBridgeCore/UI)
import { describe, test, expect } from 'bun:test'; import { describe, test, expect } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { spawnSync } from 'child_process'; import { spawnSync } from 'child_process';
import { readFileSync } from 'fs'; import { readFileSync } from 'fs';
import { join } from 'path'; import { join } from 'path';
@@ -321,7 +322,7 @@ describeIfSwift('swift build invariants', () => {
console.error('swift build stderr:', r.stderr?.toString().slice(0, 4000)); console.error('swift build stderr:', r.stderr?.toString().slice(0, 4000));
} }
expect(r.status).toBe(0); expect(r.status).toBe(0);
}, 180_000); }, CAPTURE_MS);
test('XCTest suite for StateServer passes (validates real Swift impl)', () => { test('XCTest suite for StateServer passes (validates real Swift impl)', () => {
const r = spawnSync('swift', ['test', '--filter', 'DebugBridgeCoreTests'], { const r = spawnSync('swift', ['test', '--filter', 'DebugBridgeCoreTests'], {
@@ -342,7 +343,7 @@ describeIfSwift('swift build invariants', () => {
// Guard against an empty pass-by-no-tests (filter typo / target rename): // Guard against an empty pass-by-no-tests (filter typo / target rename):
// we expect at least one StateServer smoke test to actually execute. // we expect at least one StateServer smoke test to actually execute.
expect(combined).toContain('StateServerSmokeTests'); expect(combined).toContain('StateServerSmokeTests');
}, 240_000); }, CAPTURE_MS);
// Codex-flagged: Release-build guard must be STRUCTURAL, not advisory. // Codex-flagged: Release-build guard must be STRUCTURAL, not advisory.
// The Package.swift's `.when(configuration: .debug)` setting causes Swift // The Package.swift's `.when(configuration: .debug)` setting causes Swift
@@ -386,5 +387,5 @@ describeIfSwift('swift build invariants', () => {
} }
} }
expect(foundForbidden).toBe(0); expect(foundForbidden).toBe(0);
}, 300_000); }, CAPTURE_MS);
}); });
+3 -2
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { import {
ROOT, runId, evalsEnabled, ROOT, runId, evalsEnabled,
@@ -103,7 +104,7 @@ IMPORTANT:
workingDirectory: workDir, workingDirectory: workDir,
maxTurns: 15, maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000, timeout: JUDGE_MS,
testName: 'learnings-show', testName: 'learnings-show',
runId, runId,
}); });
@@ -134,5 +135,5 @@ IMPORTANT:
} else { } else {
console.warn(`Only ${foundCount}/3 learnings found (N+1: ${mentionsNPlusOne}, cache: ${mentionsCache}, rubocop: ${mentionsRubocop})`); console.warn(`Only ${foundCount}/3 learnings found (N+1: ${mentionsNPlusOne}, cache: ${mentionsCache}, rubocop: ${mentionsRubocop})`);
} }
}, 180_000); }, CAPTURE_MS);
}); });
@@ -17,6 +17,7 @@
*/ */
import { test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillObservation, planFileHasDecisionsSection } from './helpers/claude-pty-runner'; import { runPlanSkillObservation, planFileHasDecisionsSection } from './helpers/claude-pty-runner';
@@ -30,7 +31,7 @@ describeE2E('office-hours AskUserQuestion-blocked smoke (gate)', () => {
skillName: 'office-hours', skillName: 'office-hours',
inPlanMode: true, inPlanMode: true,
extraArgs: ['--disallowedTools', 'AskUserQuestion'], extraArgs: ['--disallowedTools', 'AskUserQuestion'],
timeoutMs: 300_000, timeoutMs: CAPTURE_MS,
}); });
if ( if (
@@ -55,5 +56,5 @@ describeE2E('office-hours AskUserQuestion-blocked smoke (gate)', () => {
} }
} }
expect(['asked', 'plan_ready']).toContain(obs.outcome); expect(['asked', 'plan_ready']).toContain(obs.outcome);
}, 360_000); }, CAPTURE_LONG_MS);
}); });
@@ -36,6 +36,7 @@
*/ */
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { execFileSync, spawnSync } from 'child_process'; import { execFileSync, spawnSync } from 'child_process';
import { import {
chmodSync, chmodSync,
@@ -222,7 +223,7 @@ Generate the design doc per Phase 5. The feature-slug value to substitute into t
This is a test of the brain-writeback path. Do NOT skip the gbrain save step under any circumstance the runtime guard ("skip if gbrain not on PATH") does NOT apply here because gbrain IS available. Do NOT explore gbrain --help; follow the SAVE_RESULTS template's exact CLI shape. If you encounter any AskUserQuestion, auto-decide recommended.`, This is a test of the brain-writeback path. Do NOT skip the gbrain save step under any circumstance the runtime guard ("skip if gbrain not on PATH") does NOT apply here because gbrain IS available. Do NOT explore gbrain --help; follow the SAVE_RESULTS template's exact CLI shape. If you encounter any AskUserQuestion, auto-decide recommended.`,
workingDirectory: workDir, workingDirectory: workDir,
maxTurns: 12, maxTurns: 12,
timeout: 360_000, timeout: CAPTURE_LONG_MS,
testName: 'office-hours-brain-writeback', testName: 'office-hours-brain-writeback',
runId, runId,
model: 'claude-sonnet-4-6', model: 'claude-sonnet-4-6',
@@ -315,7 +316,7 @@ This is a test of the brain-writeback path. Do NOT skip the gbrain save step und
); );
} }
}, },
420_000, CAPTURE_LONG_MS,
); );
}, },
); );
+3 -2
View File
@@ -20,6 +20,7 @@
* test turns out stable. * test turns out stable.
*/ */
import { expect, beforeAll, afterAll } from 'bun:test'; import { expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { import {
ROOT, runId, ROOT, runId,
@@ -133,7 +134,7 @@ ${captureInstruction(outFile)}
After writing the file with that ONE Phase 4 question, stop. Do not continue to Phase 4.5 or Phase 5.`, After writing the file with that ONE Phase 4 question, stop. Do not continue to Phase 4.5 or Phase 5.`,
workingDirectory: workDir, workingDirectory: workDir,
maxTurns: 12, maxTurns: 12,
timeout: 300_000, timeout: CAPTURE_MS,
testName: 'office-hours-phase4-fork', testName: 'office-hours-phase4-fork',
runId, runId,
model: 'claude-opus-4-7', model: 'claude-opus-4-7',
@@ -162,7 +163,7 @@ After writing the file with that ONE Phase 4 question, stop. Do not continue to
result, result,
passed: ['success', 'error_max_turns'].includes(result.exitReason), passed: ['success', 'error_max_turns'].includes(result.exitReason),
}); });
}, 360_000); }, CAPTURE_LONG_MS);
}); });
afterAll(async () => { afterAll(async () => {
+5 -4
View File
@@ -10,6 +10,7 @@
*/ */
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { import {
ROOT, browseBin, runId, evalsEnabled, ROOT, browseBin, runId, evalsEnabled,
@@ -71,7 +72,7 @@ Assume the founder has already answered Q1 (strongest evidence = "got on a waitl
Write Q3 output the forcing question you would ask this founder to ${workDir}/q3.md. Write ONLY the question prose. No conversational wrapper, no meta-commentary, no Q1/Q2 recap.`, Write Q3 output the forcing question you would ask this founder to ${workDir}/q3.md. Write ONLY the question prose. No conversational wrapper, no meta-commentary, no Q1/Q2 recap.`,
workingDirectory: workDir, workingDirectory: workDir,
maxTurns: 8, maxTurns: 8,
timeout: 240_000, timeout: CAPTURE_MS,
testName: 'office-hours-forcing-energy', testName: 'office-hours-forcing-energy',
runId, runId,
model: 'claude-sonnet-4-6', model: 'claude-sonnet-4-6',
@@ -94,7 +95,7 @@ Write Q3 output — the forcing question you would ask this founder — to ${wor
console.log('Forcing energy scores:', JSON.stringify(scores, null, 2)); console.log('Forcing energy scores:', JSON.stringify(scores, null, 2));
expect(scores.axis_a).toBeGreaterThanOrEqual(4); // stacking_preserved expect(scores.axis_a).toBeGreaterThanOrEqual(4); // stacking_preserved
expect(scores.axis_b).toBeGreaterThanOrEqual(4); // domain_matched_consequence expect(scores.axis_b).toBeGreaterThanOrEqual(4); // domain_matched_consequence
}, 360_000); }, CAPTURE_LONG_MS);
}); });
// --- Office Hours builder-mode wildness --- // --- Office Hours builder-mode wildness ---
@@ -143,7 +144,7 @@ The user has confirmed the basic idea is "TypeScript + D3 web tool, start with J
Write your response the three adjacent unlocks to ${workDir}/unlocks.md. Write ONLY the response prose. No meta-commentary, no mode recap. Lead with the fun; let me edit it down later.`, Write your response the three adjacent unlocks to ${workDir}/unlocks.md. Write ONLY the response prose. No meta-commentary, no mode recap. Lead with the fun; let me edit it down later.`,
workingDirectory: workDir, workingDirectory: workDir,
maxTurns: 8, maxTurns: 8,
timeout: 240_000, timeout: CAPTURE_MS,
testName: 'office-hours-builder-wildness', testName: 'office-hours-builder-wildness',
runId, runId,
model: 'claude-sonnet-4-6', model: 'claude-sonnet-4-6',
@@ -166,7 +167,7 @@ Write your response — the three adjacent unlocks — to ${workDir}/unlocks.md.
console.log('Builder wildness scores:', JSON.stringify(scores, null, 2)); console.log('Builder wildness scores:', JSON.stringify(scores, null, 2));
expect(scores.axis_a).toBeGreaterThanOrEqual(4); // unexpected_combinations expect(scores.axis_a).toBeGreaterThanOrEqual(4); // unexpected_combinations
expect(scores.axis_b).toBeGreaterThanOrEqual(4); // excitement_over_optimization expect(scores.axis_b).toBeGreaterThanOrEqual(4); // excitement_over_optimization
}, 360_000); }, CAPTURE_LONG_MS);
}); });
// Finalize eval collector for this file // Finalize eval collector for this file
+6 -5
View File
@@ -18,6 +18,7 @@
*/ */
import { describe, test, expect, afterAll } from 'bun:test'; import { describe, test, expect, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { EvalCollector } from './helpers/eval-store'; import { EvalCollector } from './helpers/eval-store';
import { extractSkillHead } from './helpers/skill-fixture'; import { extractSkillHead } from './helpers/skill-fixture';
@@ -204,7 +205,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
workingDirectory: armA, workingDirectory: armA,
maxTurns: 5, maxTurns: 5,
allowedTools: ['Read', 'Bash', 'Glob', 'Grep'], allowedTools: ['Read', 'Bash', 'Glob', 'Grep'],
timeout: 90_000, timeout: JUDGE_MS,
testName: 'fanout-arm-overlay-on', testName: 'fanout-arm-overlay-on',
runId, runId,
model: OPUS_47, model: OPUS_47,
@@ -214,7 +215,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
workingDirectory: armB, workingDirectory: armB,
maxTurns: 5, maxTurns: 5,
allowedTools: ['Read', 'Bash', 'Glob', 'Grep'], allowedTools: ['Read', 'Bash', 'Glob', 'Grep'],
timeout: 90_000, timeout: JUDGE_MS,
testName: 'fanout-arm-overlay-off', testName: 'fanout-arm-overlay-off',
runId, runId,
model: OPUS_47, model: OPUS_47,
@@ -262,7 +263,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
fs.rmSync(armB, { recursive: true, force: true }); fs.rmSync(armB, { recursive: true, force: true });
} }
}, },
240_000, CAPTURE_MS,
); );
test( test(
@@ -281,7 +282,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
workingDirectory: root, workingDirectory: root,
maxTurns: 3, maxTurns: 3,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 90_000, timeout: JUDGE_MS,
testName: `routing-${c.name}`, testName: `routing-${c.name}`,
runId, runId,
model: OPUS_47, model: OPUS_47,
@@ -348,6 +349,6 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
fs.rmSync(root, { recursive: true, force: true }); fs.rmSync(root, { recursive: true, force: true });
} }
}, },
360_000, CAPTURE_LONG_MS,
); );
}); });
@@ -5,6 +5,7 @@
*/ */
import { test } from 'bun:test'; import { test } from 'bun:test';
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner'; import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
import { FORCING_FLOOR_CEO } from './fixtures/forcing-finding-seeds'; import { FORCING_FLOOR_CEO } from './fixtures/forcing-finding-seeds';
@@ -22,7 +23,7 @@ describeE2E('/plan-ceo-review AskUserQuestion floor (gate)', () => {
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs). // registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(), cwd: process.cwd(),
timeoutMs: 600_000, timeoutMs: CAPTURE_LONG_MS,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
}); });
@@ -34,6 +35,6 @@ describeE2E('/plan-ceo-review AskUserQuestion floor (gate)', () => {
); );
} }
}, },
660_000, PTY_MS,
); );
}); });
+3 -2
View File
@@ -31,6 +31,7 @@
*/ */
import { test } from 'bun:test'; import { test } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import { import {
launchClaudePty, launchClaudePty,
@@ -151,7 +152,7 @@ describeE2E('/plan-ceo-review mode routing (gate)', () => {
async () => { async () => {
const session = await launchClaudePty({ const session = await launchClaudePty({
permissionMode: 'plan', permissionMode: 'plan',
timeoutMs: 540_000, timeoutMs: CAPTURE_LONG_MS,
seedSkills: true, seedSkills: true,
}); });
try { try {
@@ -207,7 +208,7 @@ describeE2E('/plan-ceo-review mode routing (gate)', () => {
await session.close(); await session.close();
} }
}, },
600_000, CAPTURE_LONG_MS,
); );
} }
}); });
+2 -1
View File
@@ -34,6 +34,7 @@
*/ */
import { test } from 'bun:test'; import { test } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import { import {
runPlanSkillObservation, runPlanSkillObservation,
@@ -77,5 +78,5 @@ describeE2E('plan-ceo-review plan-mode smoke (gate)', () => {
); );
} }
assertReportAtBottomIfPlanWritten(obs); assertReportAtBottomIfPlanWritten(obs);
}, 480_000); }, CAPTURE_LONG_MS);
}); });
@@ -25,6 +25,7 @@
*/ */
import { test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import { import {
setupSkillDir, setupSkillDir,
@@ -87,6 +88,6 @@ describeE2E('/plan-ceo-review section-loading E2E (periodic, SDK capture)', () =
// Guard against an empty pass: the report must have real content. // Guard against an empty pass: the report must have real content.
expect(output.trim().length).toBeGreaterThan(200); expect(output.trim().length).toBeGreaterThan(200);
}, },
360_000, CAPTURE_LONG_MS,
); );
}); });
@@ -5,6 +5,7 @@
*/ */
import { test } from 'bun:test'; import { test } from 'bun:test';
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner'; import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
import { FORCING_FLOOR_DESIGN } from './fixtures/forcing-finding-seeds'; import { FORCING_FLOOR_DESIGN } from './fixtures/forcing-finding-seeds';
@@ -22,7 +23,7 @@ describeE2E('/plan-design-review AskUserQuestion floor (periodic)', () => {
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs). // registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(), cwd: process.cwd(),
timeoutMs: 600_000, timeoutMs: CAPTURE_LONG_MS,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
}); });
@@ -34,6 +35,6 @@ describeE2E('/plan-design-review AskUserQuestion floor (periodic)', () => {
); );
} }
}, },
660_000, PTY_MS,
); );
}); });
+5 -4
View File
@@ -10,6 +10,7 @@
*/ */
import { test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import { import {
runPlanSkillObservation, runPlanSkillObservation,
@@ -46,7 +47,7 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => {
const obs = await runPlanSkillObservation({ const obs = await runPlanSkillObservation({
skillName: 'plan-design-review', skillName: 'plan-design-review',
inPlanMode: true, inPlanMode: true,
timeoutMs: 300_000, timeoutMs: CAPTURE_MS,
}); });
if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') { if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') {
@@ -59,7 +60,7 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => {
} }
expect(['asked', 'plan_ready']).toContain(obs.outcome); expect(['asked', 'plan_ready']).toContain(obs.outcome);
assertReportAtBottomIfPlanWritten(obs); assertReportAtBottomIfPlanWritten(obs);
}, 360_000); }, CAPTURE_LONG_MS);
// Plan-mode scope-gate bypass: with a seeded UI-heavy plan in plan mode, // Plan-mode scope-gate bypass: with a seeded UI-heavy plan in plan mode,
// the gate must NOT render its "What should I review?" menu — it // the gate must NOT render its "What should I review?" menu — it
@@ -71,7 +72,7 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => {
skillName: 'plan-design-review', skillName: 'plan-design-review',
inPlanMode: true, inPlanMode: true,
initialPlanContent: SEED_PLAN_UI_HEAVY, initialPlanContent: SEED_PLAN_UI_HEAVY,
timeoutMs: 300_000, timeoutMs: CAPTURE_MS,
}); });
if ( if (
@@ -95,5 +96,5 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => {
// though the seed arrives as a pasted user message). // though the seed arrives as a pasted user message).
expect(obs.scopeGateQuestionObserved ?? false).toBe(false); expect(obs.scopeGateQuestionObserved ?? false).toBe(false);
expect(obs.scopeGateAutoSelectObserved ?? false).toBe(true); expect(obs.scopeGateAutoSelectObserved ?? false).toBe(true);
}, 360_000); }, CAPTURE_LONG_MS);
}); });
+3 -2
View File
@@ -20,6 +20,7 @@
*/ */
import { test } from 'bun:test'; import { test } from 'bun:test';
import { PTY_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import * as path from 'path'; import * as path from 'path';
import { import {
@@ -46,7 +47,7 @@ describeE2E('/plan-design-review with UI scope (gate)', () => {
// LIVE-REPO CWD: PTY session needs the repo cwd — skill registry, // LIVE-REPO CWD: PTY session needs the repo cwd — skill registry,
// hermetic pre-trusted dir, and the repo-relative fixture path above. // hermetic pre-trusted dir, and the repo-relative fixture path above.
cwd: ROOT, cwd: ROOT,
timeoutMs: 720_000, timeoutMs: PTY_MS,
seedSkills: true, seedSkills: true,
}); });
@@ -152,6 +153,6 @@ describeE2E('/plan-design-review with UI scope (gate)', () => {
); );
} }
}, },
780_000, PTY_MS,
); );
}); });
@@ -5,6 +5,7 @@
*/ */
import { test } from 'bun:test'; import { test } from 'bun:test';
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner'; import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
import { FORCING_FLOOR_DEVEX } from './fixtures/forcing-finding-seeds'; import { FORCING_FLOOR_DEVEX } from './fixtures/forcing-finding-seeds';
@@ -22,7 +23,7 @@ describeE2E('/plan-devex-review AskUserQuestion floor (gate)', () => {
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs). // registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(), cwd: process.cwd(),
timeoutMs: 600_000, timeoutMs: CAPTURE_LONG_MS,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
}); });
@@ -34,6 +35,6 @@ describeE2E('/plan-devex-review AskUserQuestion floor (gate)', () => {
); );
} }
}, },
660_000, PTY_MS,
); );
}); });
+5 -4
View File
@@ -6,6 +6,7 @@
*/ */
import { test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import { import {
runPlanSkillObservation, runPlanSkillObservation,
@@ -20,7 +21,7 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => {
const obs = await runPlanSkillObservation({ const obs = await runPlanSkillObservation({
skillName: 'plan-devex-review', skillName: 'plan-devex-review',
inPlanMode: true, inPlanMode: true,
timeoutMs: 300_000, timeoutMs: CAPTURE_MS,
}); });
if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') { if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') {
@@ -33,7 +34,7 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => {
} }
expect(['asked', 'plan_ready']).toContain(obs.outcome); expect(['asked', 'plan_ready']).toContain(obs.outcome);
assertReportAtBottomIfPlanWritten(obs); assertReportAtBottomIfPlanWritten(obs);
}, 360_000); }, CAPTURE_LONG_MS);
// v1.21+ regression: see skill-e2e-plan-ceo-plan-mode.test.ts for the // v1.21+ regression: see skill-e2e-plan-ceo-plan-mode.test.ts for the
// contract. Pass envelope is ['asked', 'plan_ready']; failure signals // contract. Pass envelope is ['asked', 'plan_ready']; failure signals
@@ -44,7 +45,7 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => {
skillName: 'plan-devex-review', skillName: 'plan-devex-review',
inPlanMode: true, inPlanMode: true,
extraArgs: ['--disallowedTools', 'AskUserQuestion'], extraArgs: ['--disallowedTools', 'AskUserQuestion'],
timeoutMs: 300_000, timeoutMs: CAPTURE_MS,
}); });
if ( if (
@@ -70,5 +71,5 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => {
} }
expect(['asked', 'plan_ready']).toContain(obs.outcome); expect(['asked', 'plan_ready']).toContain(obs.outcome);
assertReportAtBottomIfPlanWritten(obs); assertReportAtBottomIfPlanWritten(obs);
}, 360_000); }, CAPTURE_LONG_MS);
}); });
@@ -16,6 +16,7 @@
*/ */
import { test } from 'bun:test'; import { test } from 'bun:test';
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner'; import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
import { FORCING_FLOOR_ENG } from './fixtures/forcing-finding-seeds'; import { FORCING_FLOOR_ENG } from './fixtures/forcing-finding-seeds';
@@ -33,7 +34,7 @@ describeE2E('/plan-eng-review AskUserQuestion floor (periodic)', () => {
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs). // registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(), cwd: process.cwd(),
timeoutMs: 600_000, timeoutMs: CAPTURE_LONG_MS,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
}); });
@@ -49,6 +50,6 @@ describeE2E('/plan-eng-review AskUserQuestion floor (periodic)', () => {
); );
} }
}, },
660_000, PTY_MS,
); );
}); });
+5 -4
View File
@@ -6,6 +6,7 @@
*/ */
import { test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import { import {
runPlanSkillObservation, runPlanSkillObservation,
@@ -50,7 +51,7 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => {
const obs = await runPlanSkillObservation({ const obs = await runPlanSkillObservation({
skillName: 'plan-eng-review', skillName: 'plan-eng-review',
inPlanMode: true, inPlanMode: true,
timeoutMs: 300_000, timeoutMs: CAPTURE_MS,
}); });
if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') { if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') {
@@ -63,7 +64,7 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => {
} }
expect(['asked', 'plan_ready']).toContain(obs.outcome); expect(['asked', 'plan_ready']).toContain(obs.outcome);
assertReportAtBottomIfPlanWritten(obs); assertReportAtBottomIfPlanWritten(obs);
}, 360_000); }, CAPTURE_LONG_MS);
// D3-B / D4-B: when a plan with guaranteed-finding-triggering complexity // D3-B / D4-B: when a plan with guaranteed-finding-triggering complexity
// is seeded, the skill MUST fire AskUserQuestion (or fall back to a // is seeded, the skill MUST fire AskUserQuestion (or fall back to a
@@ -79,7 +80,7 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => {
// must use mcp__*__AskUserQuestion (outcome='asked') or fall back to // must use mcp__*__AskUserQuestion (outcome='asked') or fall back to
// writing Decisions ('plan_ready'). // writing Decisions ('plan_ready').
extraArgs: ['--disallowedTools', 'AskUserQuestion'], extraArgs: ['--disallowedTools', 'AskUserQuestion'],
timeoutMs: 300_000, timeoutMs: CAPTURE_MS,
}); });
if ( if (
@@ -118,5 +119,5 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => {
// question. // question.
expect(obs.scopeGateQuestionObserved ?? false).toBe(false); expect(obs.scopeGateQuestionObserved ?? false).toBe(false);
expect(obs.scopeGateAutoSelectObserved ?? false).toBe(true); expect(obs.scopeGateAutoSelectObserved ?? false).toBe(true);
}, 360_000); }, CAPTURE_LONG_MS);
}); });
+9 -8
View File
@@ -18,6 +18,7 @@
* accordingly. * accordingly.
*/ */
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { import {
ROOT, runId, ROOT, runId,
@@ -134,7 +135,7 @@ ${captureInstruction(outFile)}
After writing the file, stop. Do not continue the review.`, After writing the file, stop. Do not continue the review.`,
workingDirectory: planDir, workingDirectory: planDir,
maxTurns: 10, maxTurns: 10,
timeout: 240_000, timeout: CAPTURE_MS,
testName: 'plan-ceo-review-format-mode', testName: 'plan-ceo-review-format-mode',
runId, runId,
model: 'claude-opus-4-7', model: 'claude-opus-4-7',
@@ -160,7 +161,7 @@ After writing the file, stop. Do not continue the review.`,
result, result,
passed: ['success', 'error_max_turns'].includes(result.exitReason), passed: ['success', 'error_max_turns'].includes(result.exitReason),
}); });
}, 300_000); }, CAPTURE_MS);
}); });
// --- Case 2: plan-ceo-review approach menu (coverage-differentiated) --- // --- Case 2: plan-ceo-review approach menu (coverage-differentiated) ---
@@ -191,7 +192,7 @@ ${captureInstruction(outFile)}
After writing the file, stop. Do not continue the review.`, After writing the file, stop. Do not continue the review.`,
workingDirectory: planDir, workingDirectory: planDir,
maxTurns: 10, maxTurns: 10,
timeout: 240_000, timeout: CAPTURE_MS,
testName: 'plan-ceo-review-format-approach', testName: 'plan-ceo-review-format-approach',
runId, runId,
model: 'claude-opus-4-7', model: 'claude-opus-4-7',
@@ -216,7 +217,7 @@ After writing the file, stop. Do not continue the review.`,
result, result,
passed: ['success', 'error_max_turns'].includes(result.exitReason), passed: ['success', 'error_max_turns'].includes(result.exitReason),
}); });
}, 300_000); }, CAPTURE_MS);
}); });
// --- Case 3: plan-eng-review coverage-differentiated per-issue AskUserQuestion --- // --- Case 3: plan-eng-review coverage-differentiated per-issue AskUserQuestion ---
@@ -250,7 +251,7 @@ ${captureInstruction(outFile)}
After writing the file with that ONE question, stop. Do not continue the review.`, After writing the file with that ONE question, stop. Do not continue the review.`,
workingDirectory: planDir, workingDirectory: planDir,
maxTurns: 10, maxTurns: 10,
timeout: 240_000, timeout: CAPTURE_MS,
testName: 'plan-eng-review-format-coverage', testName: 'plan-eng-review-format-coverage',
runId, runId,
model: 'claude-opus-4-7', model: 'claude-opus-4-7',
@@ -275,7 +276,7 @@ After writing the file with that ONE question, stop. Do not continue the review.
result, result,
passed: ['success', 'error_max_turns'].includes(result.exitReason), passed: ['success', 'error_max_turns'].includes(result.exitReason),
}); });
}, 300_000); }, CAPTURE_MS);
}); });
// --- Case 4: plan-eng-review kind-differentiated per-issue AskUserQuestion --- // --- Case 4: plan-eng-review kind-differentiated per-issue AskUserQuestion ---
@@ -306,7 +307,7 @@ ${captureInstruction(outFile)}
After writing the file with that ONE question, stop. Do not continue the review.`, After writing the file with that ONE question, stop. Do not continue the review.`,
workingDirectory: planDir, workingDirectory: planDir,
maxTurns: 10, maxTurns: 10,
timeout: 240_000, timeout: CAPTURE_MS,
testName: 'plan-eng-review-format-kind', testName: 'plan-eng-review-format-kind',
runId, runId,
model: 'claude-opus-4-7', model: 'claude-opus-4-7',
@@ -332,7 +333,7 @@ After writing the file with that ONE question, stop. Do not continue the review.
result, result,
passed: ['success', 'error_max_turns'].includes(result.exitReason), passed: ['success', 'error_max_turns'].includes(result.exitReason),
}); });
}, 300_000); }, CAPTURE_MS);
}); });
afterAll(async () => { afterAll(async () => {
+5 -4
View File
@@ -31,6 +31,7 @@
*/ */
import { test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillObservation } from './helpers/claude-pty-runner'; import { runPlanSkillObservation } from './helpers/claude-pty-runner';
@@ -62,7 +63,7 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
const obs = await runPlanSkillObservation({ const obs = await runPlanSkillObservation({
skillName, skillName,
inPlanMode: false, inPlanMode: false,
timeoutMs: 300_000, timeoutMs: CAPTURE_MS,
// eng/design: force the prose-fallback path. The unconditional // eng/design: force the prose-fallback path. The unconditional
// gate-must-ask assert below pins the render shape the detector // gate-must-ask assert below pins the render shape the detector
// anchors on, and only the --disallowedTools prose fallback makes // anchors on, and only the --disallowedTools prose fallback makes
@@ -115,7 +116,7 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
); );
} }
} }
}, 360_000); }, CAPTURE_LONG_MS);
} }
// Named-target exception (outside plan mode): a pasted draft IS an // Named-target exception (outside plan mode): a pasted draft IS an
@@ -130,7 +131,7 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
inPlanMode: false, inPlanMode: false,
initialPlanContent: NAMED_TARGET_SEED, initialPlanContent: NAMED_TARGET_SEED,
trackTokens: [SEED_TOKEN], trackTokens: [SEED_TOKEN],
timeoutMs: 300_000, timeoutMs: CAPTURE_MS,
}); });
if ( if (
@@ -159,5 +160,5 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
// unreachable outside plan mode (extractPlanFilePath only matches // unreachable outside plan mode (extractPlanFilePath only matches
// plan-mode save renders). // plan-mode save renders).
expect(obs.tokensObserved?.[SEED_TOKEN] ?? false).toBe(true); expect(obs.tokensObserved?.[SEED_TOKEN] ?? false).toBe(true);
}, 360_000); }, CAPTURE_LONG_MS);
}); });
+9 -8
View File
@@ -27,6 +27,7 @@
* cases will land as follow-up PRs per skill. * cases will land as follow-up PRs per skill.
*/ */
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { import {
ROOT, runId, ROOT, runId,
@@ -161,7 +162,7 @@ ${captureInstruction(outFile)}
After writing the file, stop.`, After writing the file, stop.`,
workingDirectory: planDir, workingDirectory: planDir,
maxTurns: 10, maxTurns: 10,
timeout: 240_000, timeout: CAPTURE_MS,
testName: 'plan-review-prosons-format', testName: 'plan-review-prosons-format',
runId, runId,
model: 'claude-opus-4-7', model: 'claude-opus-4-7',
@@ -191,7 +192,7 @@ After writing the file, stop.`,
// (recommended) label on one option // (recommended) label on one option
expect(captured).toMatch(RECOMMENDED_LABEL_RE); expect(captured).toMatch(RECOMMENDED_LABEL_RE);
}, 300_000); }, CAPTURE_MS);
}); });
// --- Case 2: Hard-stop escape NEGATIVE (CT2) --- // --- Case 2: Hard-stop escape NEGATIVE (CT2) ---
@@ -220,7 +221,7 @@ ${captureInstruction(outFile)}
After writing the file, stop.`, After writing the file, stop.`,
workingDirectory: planDir, workingDirectory: planDir,
maxTurns: 10, maxTurns: 10,
timeout: 240_000, timeout: CAPTURE_MS,
testName: 'plan-review-prosons-hardstop-neg', testName: 'plan-review-prosons-hardstop-neg',
runId, runId,
model: 'claude-opus-4-7', model: 'claude-opus-4-7',
@@ -241,7 +242,7 @@ After writing the file, stop.`,
// Must have real pros and cons (≥2 ✅ + ≥1 ❌ per option) // Must have real pros and cons (≥2 ✅ + ≥1 ❌ per option)
expect(countChars(captured, '✅')).toBeGreaterThanOrEqual(4); expect(countChars(captured, '✅')).toBeGreaterThanOrEqual(4);
expect(countChars(captured, '❌')).toBeGreaterThanOrEqual(2); expect(countChars(captured, '❌')).toBeGreaterThanOrEqual(2);
}, 300_000); }, CAPTURE_MS);
}); });
// --- Case 3: Neutral-posture NEGATIVE (CT2) --- // --- Case 3: Neutral-posture NEGATIVE (CT2) ---
@@ -270,7 +271,7 @@ ${captureInstruction(outFile)}
After writing the file, stop.`, After writing the file, stop.`,
workingDirectory: planDir, workingDirectory: planDir,
maxTurns: 10, maxTurns: 10,
timeout: 240_000, timeout: CAPTURE_MS,
testName: 'plan-review-prosons-neutral-neg', testName: 'plan-review-prosons-neutral-neg',
runId, runId,
model: 'claude-opus-4-7', model: 'claude-opus-4-7',
@@ -292,7 +293,7 @@ After writing the file, stop.`,
expect(captured).toMatch(RECOMMENDED_LABEL_RE); expect(captured).toMatch(RECOMMENDED_LABEL_RE);
// Recommendation line must contain "because" (concrete reason, not "no preference") // Recommendation line must contain "because" (concrete reason, not "no preference")
expect(captured).toMatch(/[Rr]ecommendation:.*because/); expect(captured).toMatch(/[Rr]ecommendation:.*because/);
}, 300_000); }, CAPTURE_MS);
}); });
// --- Case 4: Hard-stop POSITIVE (escape allowed when legitimately one-sided) --- // --- Case 4: Hard-stop POSITIVE (escape allowed when legitimately one-sided) ---
@@ -321,7 +322,7 @@ ${captureInstruction(outFile)}
After writing the file, stop.`, After writing the file, stop.`,
workingDirectory: planDir, workingDirectory: planDir,
maxTurns: 10, maxTurns: 10,
timeout: 240_000, timeout: CAPTURE_MS,
testName: 'plan-ceo-review-prosons-cadence', testName: 'plan-ceo-review-prosons-cadence',
runId, runId,
model: 'claude-opus-4-7', model: 'claude-opus-4-7',
@@ -344,7 +345,7 @@ After writing the file, stop.`,
const hasEscape = HARD_STOP_ESCAPE_RE.test(captured); const hasEscape = HARD_STOP_ESCAPE_RE.test(captured);
const hasProsAndCons = countChars(captured, '✅') >= 1 && countChars(captured, '❌') >= 1; const hasProsAndCons = countChars(captured, '✅') >= 1 && countChars(captured, '❌') >= 1;
expect(hasEscape || hasProsAndCons).toBe(true); expect(hasEscape || hasProsAndCons).toBe(true);
}, 300_000); }, CAPTURE_MS);
}); });
afterAll(async () => { afterAll(async () => {
+3 -2
View File
@@ -1,4 +1,5 @@
import { beforeAll, afterAll, expect } from 'bun:test'; import { beforeAll, afterAll, expect } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { import {
ROOT, runId, ROOT, runId,
@@ -151,7 +152,7 @@ IMPORTANT:
workingDirectory: workDir, workingDirectory: workDir,
maxTurns: 15, maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Grep', 'Glob'], allowedTools: ['Bash', 'Read', 'Grep', 'Glob'],
timeout: 120_000, timeout: JUDGE_MS,
testName: 'plan-tune-inspect', testName: 'plan-tune-inspect',
runId, runId,
}); });
@@ -184,5 +185,5 @@ IMPORTANT:
if (!noticedOverride) { if (!noticedOverride) {
console.warn('Agent did not surface override/skip behavior from the log'); console.warn('Agent did not surface override/skip behavior from the log');
} }
}, 180_000); }, CAPTURE_MS);
}); });
+21 -20
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { import {
ROOT, browseBin, runId, evalsEnabled, ROOT, browseBin, runId, evalsEnabled,
@@ -103,7 +104,7 @@ Focus on reviewing the plan content: architecture, error handling, security, and
const review = fs.readFileSync(reviewPath, 'utf-8'); const review = fs.readFileSync(reviewPath, 'utf-8');
expect(review.length).toBeGreaterThan(200); expect(review.length).toBeGreaterThan(200);
} }
}, 660_000); }, PTY_MS);
}); });
// --- Plan CEO Review (SELECTIVE EXPANSION) E2E --- // --- Plan CEO Review (SELECTIVE EXPANSION) E2E ---
@@ -171,7 +172,7 @@ Write your complete review directly to ${planDir}/review-output-selective.md
Focus on reviewing the plan content: architecture, error handling, security, and performance.`, Focus on reviewing the plan content: architecture, error handling, security, and performance.`,
workingDirectory: planDir, workingDirectory: planDir,
maxTurns: 15, maxTurns: 15,
timeout: 540_000, timeout: CAPTURE_LONG_MS,
testName: 'plan-ceo-review-selective', testName: 'plan-ceo-review-selective',
runId, runId,
model: 'claude-opus-4-7', model: 'claude-opus-4-7',
@@ -188,7 +189,7 @@ Focus on reviewing the plan content: architecture, error handling, security, and
const review = fs.readFileSync(reviewPath, 'utf-8'); const review = fs.readFileSync(reviewPath, 'utf-8');
expect(review.length).toBeGreaterThan(200); expect(review.length).toBeGreaterThan(200);
} }
}, 660_000); }, PTY_MS);
}); });
// --- Plan CEO Review SCOPE EXPANSION energy (V1.1 mode-posture regression gate) --- // --- Plan CEO Review SCOPE EXPANSION energy (V1.1 mode-posture regression gate) ---
@@ -239,7 +240,7 @@ Choose SCOPE EXPANSION mode. Skip any AskUserQuestion calls — this is non-inte
Write your expansion proposals to ${planDir}/proposals.md with ONLY the proposal text no conversational wrapper, no review summary, no mode analysis. Each proposal separated by "---".`, Write your expansion proposals to ${planDir}/proposals.md with ONLY the proposal text no conversational wrapper, no review summary, no mode analysis. Each proposal separated by "---".`,
workingDirectory: planDir, workingDirectory: planDir,
maxTurns: 15, maxTurns: 15,
timeout: 360_000, timeout: CAPTURE_LONG_MS,
testName: 'plan-ceo-review-expansion-energy', testName: 'plan-ceo-review-expansion-energy',
runId, runId,
model: 'claude-opus-4-7', model: 'claude-opus-4-7',
@@ -270,7 +271,7 @@ Write your expansion proposals to ${planDir}/proposals.md with ONLY the proposal
// Pass threshold: 4/5 on both axes (good — matches posture with minor weakness). // Pass threshold: 4/5 on both axes (good — matches posture with minor weakness).
expect(scores.axis_a).toBeGreaterThanOrEqual(4); // surface_framing expect(scores.axis_a).toBeGreaterThanOrEqual(4); // surface_framing
expect(scores.axis_b).toBeGreaterThanOrEqual(4); // decision_preservation expect(scores.axis_b).toBeGreaterThanOrEqual(4); // decision_preservation
}, 600_000); }, CAPTURE_LONG_MS);
}); });
// --- Plan Eng Review E2E --- // --- Plan Eng Review E2E ---
@@ -348,7 +349,7 @@ Write your complete review directly to ${planDir}/review-output.md
Focus on architecture, code quality, tests, and performance sections.`, Focus on architecture, code quality, tests, and performance sections.`,
workingDirectory: planDir, workingDirectory: planDir,
maxTurns: 15, maxTurns: 15,
timeout: 360_000, timeout: CAPTURE_LONG_MS,
testName: 'plan-eng-review', testName: 'plan-eng-review',
runId, runId,
model: 'claude-opus-4-7', model: 'claude-opus-4-7',
@@ -366,7 +367,7 @@ Focus on architecture, code quality, tests, and performance sections.`,
const review = fs.readFileSync(reviewPath, 'utf-8'); const review = fs.readFileSync(reviewPath, 'utf-8');
expect(review.length).toBeGreaterThan(200); expect(review.length).toBeGreaterThan(200);
} }
}, 420_000); }, CAPTURE_LONG_MS);
}); });
// --- Plan-Eng-Review Test-Plan Artifact E2E --- // --- Plan-Eng-Review Test-Plan Artifact E2E ---
@@ -476,7 +477,7 @@ Write your review to ${planDir}/review-output.md`,
workingDirectory: planDir, workingDirectory: planDir,
maxTurns: 25, maxTurns: 25,
allowedTools: ['Bash', 'Read', 'Write', 'Glob', 'Grep'], allowedTools: ['Bash', 'Read', 'Write', 'Glob', 'Grep'],
timeout: 360_000, timeout: CAPTURE_LONG_MS,
testName: 'plan-eng-review-artifact', testName: 'plan-eng-review-artifact',
runId, runId,
model: 'claude-opus-4-7', model: 'claude-opus-4-7',
@@ -507,7 +508,7 @@ Write your review to ${planDir}/review-output.md`,
if (newFiles.length === 0) { if (newFiles.length === 0) {
console.warn('SOFT FAIL: No test-plan artifact written — agent did not follow artifact instructions'); console.warn('SOFT FAIL: No test-plan artifact written — agent did not follow artifact instructions');
} }
}, 420_000); }, CAPTURE_LONG_MS);
}); });
// --- Office Hours Spec Review E2E --- // --- Office Hours Spec Review E2E ---
@@ -559,7 +560,7 @@ Write your summary to ${ohDir}/spec-review-summary.md`,
// failures wrote a correct summary on tool-turn 8 and hit the cap on // failures wrote a correct summary on tool-turn 8 and hit the cap on
// the closing text turn (error_max_turns at 9 turns, deterministic). // the closing text turn (error_max_turns at 9 turns, deterministic).
maxTurns: 12, maxTurns: 12,
timeout: 120_000, timeout: JUDGE_MS,
testName: 'office-hours-spec-review', testName: 'office-hours-spec-review',
runId, runId,
}); });
@@ -575,7 +576,7 @@ Write your summary to ${ohDir}/spec-review-summary.md`,
expect(summary).toMatch(/agent|subagent/); expect(summary).toMatch(/agent|subagent/);
expect(summary).toMatch(/3.*iteration|iteration.*3|maximum.*3/); expect(summary).toMatch(/3.*iteration|iteration.*3|maximum.*3/);
} }
}, 180_000); }, CAPTURE_MS);
}); });
// --- Plan CEO Review Benefits-From E2E --- // --- Plan CEO Review Benefits-From E2E ---
@@ -619,7 +620,7 @@ Summarize what happens when no design doc is found — specifically:
Write your summary to ${benefitsDir}/benefits-summary.md`, Write your summary to ${benefitsDir}/benefits-summary.md`,
workingDirectory: benefitsDir, workingDirectory: benefitsDir,
maxTurns: 8, maxTurns: 8,
timeout: 120_000, timeout: JUDGE_MS,
testName: 'plan-ceo-review-benefits', testName: 'plan-ceo-review-benefits',
runId, runId,
}); });
@@ -634,7 +635,7 @@ Write your summary to ${benefitsDir}/benefits-summary.md`,
expect(summary).toMatch(/office.hours/); expect(summary).toMatch(/office.hours/);
expect(summary).toMatch(/design doc|no design/i); expect(summary).toMatch(/design doc|no design/i);
} }
}, 180_000); }, CAPTURE_MS);
}); });
// --- Plan Review Report E2E --- // --- Plan Review Report E2E ---
@@ -706,7 +707,7 @@ CRITICAL REQUIREMENT: plan.md IS the plan file for this review session. After co
This review report at the bottom of the plan is the MOST IMPORTANT deliverable of this test.`, This review report at the bottom of the plan is the MOST IMPORTANT deliverable of this test.`,
workingDirectory: planDir, workingDirectory: planDir,
maxTurns: 20, maxTurns: 20,
timeout: 360_000, timeout: CAPTURE_LONG_MS,
testName: 'plan-review-report', testName: 'plan-review-report',
runId, runId,
model: 'claude-opus-4-7', model: 'claude-opus-4-7',
@@ -768,7 +769,7 @@ This review report at the bottom of the plan is the MOST IMPORTANT deliverable o
).toBe(true); ).toBe(true);
console.log('Plan review report found at bottom of plan.md (ends with unresolved status)'); console.log('Plan review report found at bottom of plan.md (ends with unresolved status)');
}, 420_000); }, CAPTURE_LONG_MS);
}); });
// --- Codex Offering E2E --- // --- Codex Offering E2E ---
@@ -825,7 +826,7 @@ Summarize the Codex/${featureName} integration — answer these specific questio
Write your summary to ${testDir}/${testName}-summary.md`, Write your summary to ${testDir}/${testName}-summary.md`,
workingDirectory: testDir, workingDirectory: testDir,
maxTurns: 8, maxTurns: 8,
timeout: 120_000, timeout: JUDGE_MS,
testName, testName,
runId, runId,
}); });
@@ -850,19 +851,19 @@ Write your summary to ${testDir}/${testName}-summary.md`,
testConcurrentIfSelected('codex-offered-office-hours', async () => { testConcurrentIfSelected('codex-offered-office-hours', async () => {
await checkCodexOffering('office-hours', 'codex-offered-office-hours', 'second opinion'); await checkCodexOffering('office-hours', 'codex-offered-office-hours', 'second opinion');
}, 180_000); }, CAPTURE_MS);
testConcurrentIfSelected('codex-offered-ceo-review', async () => { testConcurrentIfSelected('codex-offered-ceo-review', async () => {
await checkCodexOffering('plan-ceo-review', 'codex-offered-ceo-review', 'outside voice'); await checkCodexOffering('plan-ceo-review', 'codex-offered-ceo-review', 'outside voice');
}, 180_000); }, CAPTURE_MS);
testConcurrentIfSelected('codex-offered-design-review', async () => { testConcurrentIfSelected('codex-offered-design-review', async () => {
await checkCodexOffering('plan-design-review', 'codex-offered-design-review', 'design outside voices'); await checkCodexOffering('plan-design-review', 'codex-offered-design-review', 'design outside voices');
}, 180_000); }, CAPTURE_MS);
testConcurrentIfSelected('codex-offered-eng-review', async () => { testConcurrentIfSelected('codex-offered-eng-review', async () => {
await checkCodexOffering('plan-eng-review', 'codex-offered-eng-review', 'outside voice'); await checkCodexOffering('plan-eng-review', 'codex-offered-eng-review', 'outside voice');
}, 180_000); }, CAPTURE_MS);
}); });
// Module-level afterAll — finalize eval collector after all tests complete // Module-level afterAll — finalize eval collector after all tests complete
+5 -4
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { outcomeJudge } from './helpers/llm-judge'; import { outcomeJudge } from './helpers/llm-judge';
import { judgePassed } from './helpers/eval-store'; import { judgePassed } from './helpers/eval-store';
@@ -97,7 +98,7 @@ CRITICAL RULES:
- The report MUST exist at ${reportPath} when you finish`, - The report MUST exist at ${reportPath} when you finish`,
workingDirectory: testWorkDir, workingDirectory: testWorkDir,
maxTurns: 50, maxTurns: 50,
timeout: 300_000, timeout: CAPTURE_MS,
testName: `qa-${label}`, testName: `qa-${label}`,
runId, runId,
model: 'claude-opus-4-7', model: 'claude-opus-4-7',
@@ -174,17 +175,17 @@ CRITICAL RULES:
// B6: Static dashboard — broken link, disabled submit, overflow, missing alt, console error // B6: Static dashboard — broken link, disabled submit, overflow, missing alt, console error
testConcurrentIfSelected('qa-b6-static', async () => { testConcurrentIfSelected('qa-b6-static', async () => {
await runPlantedBugEval('qa-eval.html', 'qa-eval-ground-truth.json', 'b6-static'); await runPlantedBugEval('qa-eval.html', 'qa-eval-ground-truth.json', 'b6-static');
}, 360_000); }, CAPTURE_LONG_MS);
// B7: SPA — broken route, stale state, async race, missing aria, console warning // B7: SPA — broken route, stale state, async race, missing aria, console warning
testConcurrentIfSelected('qa-b7-spa', async () => { testConcurrentIfSelected('qa-b7-spa', async () => {
await runPlantedBugEval('qa-eval-spa.html', 'qa-eval-spa-ground-truth.json', 'b7-spa'); await runPlantedBugEval('qa-eval-spa.html', 'qa-eval-spa-ground-truth.json', 'b7-spa');
}, 360_000); }, CAPTURE_LONG_MS);
// B8: Checkout — email regex, NaN total, CC overflow, missing required, stripe error // B8: Checkout — email regex, NaN total, CC overflow, missing required, stripe error
testConcurrentIfSelected('qa-b8-checkout', async () => { testConcurrentIfSelected('qa-b8-checkout', async () => {
await runPlantedBugEval('qa-eval-checkout.html', 'qa-eval-checkout-ground-truth.json', 'b8-checkout'); await runPlantedBugEval('qa-eval-checkout.html', 'qa-eval-checkout-ground-truth.json', 'b8-checkout');
}, 360_000); }, CAPTURE_LONG_MS);
}); });
+9 -8
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { import {
ROOT, browseBin, runId, evalsEnabled, ROOT, browseBin, runId, evalsEnabled,
@@ -54,7 +55,7 @@ Do NOT try to start a server or discover ports — the URL above is ready.
Write your report to ${qaDir}/qa-reports/qa-report.md`, Write your report to ${qaDir}/qa-reports/qa-report.md`,
workingDirectory: qaDir, workingDirectory: qaDir,
maxTurns: 35, maxTurns: 35,
timeout: 240_000, timeout: CAPTURE_MS,
testName: 'qa-quick', testName: 'qa-quick',
runId, runId,
}); });
@@ -69,7 +70,7 @@ Write your report to ${qaDir}/qa-reports/qa-report.md`,
} }
// Accept error_max_turns — the agent doing thorough QA work is not a failure // Accept error_max_turns — the agent doing thorough QA work is not a failure
expect(['success', 'error_max_turns']).toContain(result.exitReason); expect(['success', 'error_max_turns']).toContain(result.exitReason);
}, 300_000); }, CAPTURE_MS);
}); });
// --- QA-Only E2E (report-only, no fixes) --- // --- QA-Only E2E (report-only, no fixes) ---
@@ -124,7 +125,7 @@ Write your report to ${qaOnlyDir}/qa-reports/qa-only-report.md`,
workingDirectory: qaOnlyDir, workingDirectory: qaOnlyDir,
maxTurns: 40, maxTurns: 40,
allowedTools: ['Bash', 'Read', 'Write', 'Glob'], // NO Edit — the critical guardrail allowedTools: ['Bash', 'Read', 'Write', 'Glob'], // NO Edit — the critical guardrail
timeout: 180_000, timeout: CAPTURE_MS,
testName: 'qa-only-no-fix', testName: 'qa-only-no-fix',
runId, runId,
}); });
@@ -156,7 +157,7 @@ Write your report to ${qaOnlyDir}/qa-reports/qa-only-report.md`,
(l: string) => l.trim() && !l.includes('.prompt-tmp') && !l.includes('.gstack/') && !l.includes('qa-reports/'), (l: string) => l.trim() && !l.includes('.prompt-tmp') && !l.includes('.gstack/') && !l.includes('qa-reports/'),
); );
expect(statusLines.filter((l: string) => l.startsWith(' M') || l.startsWith('M '))).toHaveLength(0); expect(statusLines.filter((l: string) => l.startsWith(' M') || l.startsWith('M '))).toHaveLength(0);
}, 240_000); }, CAPTURE_MS);
}); });
// --- QA Fix Loop E2E --- // --- QA Fix Loop E2E ---
@@ -247,7 +248,7 @@ This is a test+fix loop: find bugs, fix them in the source code, commit each fix
workingDirectory: qaFixDir, workingDirectory: qaFixDir,
maxTurns: 40, maxTurns: 40,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'], allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
timeout: 420_000, timeout: CAPTURE_LONG_MS,
testName: 'qa-fix-loop', testName: 'qa-fix-loop',
runId, runId,
}); });
@@ -271,7 +272,7 @@ This is a test+fix loop: find bugs, fix them in the source code, commit each fix
// Verify Edit tool was used (agent actually modified source code) // Verify Edit tool was used (agent actually modified source code)
const editCalls = result.toolCalls.filter(tc => tc.tool === 'Edit'); const editCalls = result.toolCalls.filter(tc => tc.tool === 'Edit');
expect(editCalls.length).toBeGreaterThan(0); expect(editCalls.length).toBeGreaterThan(0);
}, 480_000); }, CAPTURE_LONG_MS);
}); });
// --- Test Bootstrap E2E --- // --- Test Bootstrap E2E ---
@@ -384,7 +385,7 @@ Do NOT fix any bugs. Do NOT use AskUserQuestion — just pick vitest.`,
workingDirectory: bsDir, workingDirectory: bsDir,
maxTurns: 12, maxTurns: 12,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'], allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'],
timeout: 90_000, timeout: JUDGE_MS,
testName: 'qa-bootstrap', testName: 'qa-bootstrap',
runId, runId,
}); });
@@ -405,7 +406,7 @@ Do NOT fix any bugs. Do NOT use AskUserQuestion — just pick vitest.`,
console.log(`Test config: ${hasTestConfig}, Test file: ${hasTestFile}, TESTING.md: ${hasTestingMd}`); console.log(`Test config: ${hasTestConfig}, Test file: ${hasTestFile}, TESTING.md: ${hasTestingMd}`);
try { fs.rmSync(bsDir, { recursive: true, force: true }); } catch {} try { fs.rmSync(bsDir, { recursive: true, force: true }); } catch {}
}, 120_000); }, JUDGE_MS);
}); });
// Module-level afterAll — finalize eval collector after all tests complete. // Module-level afterAll — finalize eval collector after all tests complete.
+4 -3
View File
@@ -1,4 +1,5 @@
import { expect, beforeAll, afterAll } from 'bun:test'; import { expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { import {
ROOT, runId, ROOT, runId,
@@ -135,7 +136,7 @@ Write your retrospective to ${dir}/retro-output.md`,
expect(wroteReport).toBe(true); expect(wroteReport).toBe(true);
const content = fs.readFileSync(retroPath, 'utf-8'); const content = fs.readFileSync(retroPath, 'utf-8');
expect(content.length).toBeGreaterThan(100); expect(content.length).toBeGreaterThan(100);
}, 480_000); }, CAPTURE_LONG_MS);
}); });
// --- Retro E2E --- // --- Retro E2E ---
@@ -198,7 +199,7 @@ Write your retrospective report to ${retroDir}/retro-output.md
Analyze the git history and produce the narrative report as described in the SKILL.md.`, Analyze the git history and produce the narrative report as described in the SKILL.md.`,
workingDirectory: retroDir, workingDirectory: retroDir,
maxTurns: 30, maxTurns: 30,
timeout: 300_000, timeout: CAPTURE_MS,
testName: 'retro', testName: 'retro',
runId, runId,
model: 'claude-opus-4-7', model: 'claude-opus-4-7',
@@ -217,7 +218,7 @@ Analyze the git history and produce the narrative report as described in the SKI
expect(wroteReport).toBe(true); expect(wroteReport).toBe(true);
const retro = fs.readFileSync(retroPath, 'utf-8'); const retro = fs.readFileSync(retroPath, 'utf-8');
expect(retro.length).toBeGreaterThan(100); expect(retro.length).toBeGreaterThan(100);
}, 420_000); }, CAPTURE_LONG_MS);
}); });
// Module-level afterAll — finalize eval collector after all tests complete // Module-level afterAll — finalize eval collector after all tests complete
+15 -14
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { import {
ROOT, runId, describeIfSelected, testConcurrentIfSelected, ROOT, runId, describeIfSelected, testConcurrentIfSelected,
@@ -114,7 +115,7 @@ and apply it yourself against the diff (git diff main...HEAD).
Write your findings to ${dir}/review-output.md`, Write your findings to ${dir}/review-output.md`,
workingDirectory: dir, workingDirectory: dir,
maxTurns: 20, maxTurns: 20,
timeout: 180_000, timeout: CAPTURE_MS,
testName: 'review-army-migration-safety', testName: 'review-army-migration-safety',
runId, runId,
}); });
@@ -135,7 +136,7 @@ Write your findings to ${dir}/review-output.md`,
content.includes('column'); content.includes('column');
expect(hasMigrationFinding).toBe(true); expect(hasMigrationFinding).toBe(true);
} }
}, 210_000); }, CAPTURE_MS);
}); });
// --- Review Army: N+1 Performance --- // --- Review Army: N+1 Performance ---
@@ -179,7 +180,7 @@ For the specialist dispatch, read review-specialists/performance.md and apply it
Write your findings to ${dir}/review-output.md`, Write your findings to ${dir}/review-output.md`,
workingDirectory: dir, workingDirectory: dir,
maxTurns: 20, maxTurns: 20,
timeout: 180_000, timeout: CAPTURE_MS,
testName: 'review-army-perf-n-plus-one', testName: 'review-army-perf-n-plus-one',
runId, runId,
}); });
@@ -201,7 +202,7 @@ Write your findings to ${dir}/review-output.md`,
content.includes('loop'); content.includes('loop');
expect(hasN1Finding).toBe(true); expect(hasN1Finding).toBe(true);
} }
}, 210_000); }, CAPTURE_MS);
}); });
// --- Review Army: Delivery Audit --- // --- Review Army: Delivery Audit ---
@@ -281,7 +282,7 @@ The email notification system should be classified as NOT DONE.
Write your completion audit to ${dir}/review-output.md`, Write your completion audit to ${dir}/review-output.md`,
workingDirectory: dir, workingDirectory: dir,
maxTurns: 15, maxTurns: 15,
timeout: 120_000, timeout: JUDGE_MS,
testName: 'review-army-delivery-audit', testName: 'review-army-delivery-audit',
runId, runId,
}); });
@@ -305,7 +306,7 @@ Write your completion audit to ${dir}/review-output.md`,
expect(hasNotDone).toBe(true); expect(hasNotDone).toBe(true);
expect(mentionsEmail).toBe(true); expect(mentionsEmail).toBe(true);
} }
}, 150_000); }, CAPTURE_MS);
}); });
// --- Review Army: Quality Score --- // --- Review Army: Quality Score ---
@@ -356,7 +357,7 @@ Write your findings AND the computed quality score to ${dir}/review-output.md
Include the line: "PR Quality Score: X/10" where X is the computed score.`, Include the line: "PR Quality Score: X/10" where X is the computed score.`,
workingDirectory: dir, workingDirectory: dir,
maxTurns: 15, maxTurns: 15,
timeout: 120_000, timeout: JUDGE_MS,
testName: 'review-army-quality-score', testName: 'review-army-quality-score',
runId, runId,
}); });
@@ -374,7 +375,7 @@ Include the line: "PR Quality Score: X/10" where X is the computed score.`,
content.match(/\d+\/10/); content.match(/\d+\/10/);
expect(hasScore).toBeTruthy(); expect(hasScore).toBeTruthy();
} }
}, 150_000); }, CAPTURE_MS);
}); });
// --- Review Army: JSON Findings --- // --- Review Army: JSON Findings ---
@@ -421,7 +422,7 @@ Output your findings as JSON objects, one per line, following the schema:
Write ONLY JSON findings (no preamble) to ${dir}/findings.json`, Write ONLY JSON findings (no preamble) to ${dir}/findings.json`,
workingDirectory: dir, workingDirectory: dir,
maxTurns: 12, maxTurns: 12,
timeout: 90_000, timeout: JUDGE_MS,
testName: 'review-army-json-findings', testName: 'review-army-json-findings',
runId, runId,
}); });
@@ -450,7 +451,7 @@ Write ONLY JSON findings (no preamble) to ${dir}/findings.json`,
break; // One valid line is enough for the gate test break; // One valid line is enough for the gate test
} }
} }
}, 120_000); }, JUDGE_MS);
}); });
// --- Review Army: Red Team (periodic) --- // --- Review Army: Red Team (periodic) ---
@@ -499,7 +500,7 @@ Write your red team findings to ${dir}/review-output.md
Start the file with "RED TEAM REVIEW" on the first line.`, Start the file with "RED TEAM REVIEW" on the first line.`,
workingDirectory: dir, workingDirectory: dir,
maxTurns: 20, maxTurns: 20,
timeout: 180_000, timeout: CAPTURE_MS,
testName: 'review-army-red-team', testName: 'review-army-red-team',
runId, runId,
}); });
@@ -513,7 +514,7 @@ Start the file with "RED TEAM REVIEW" on the first line.`,
const content = fs.readFileSync(outputPath, 'utf-8'); const content = fs.readFileSync(outputPath, 'utf-8');
expect(content.toLowerCase()).toMatch(/red team|adversarial/); expect(content.toLowerCase()).toMatch(/red team|adversarial/);
} }
}, 210_000); }, CAPTURE_MS);
}); });
// --- Review Army: Consensus (periodic) --- // --- Review Army: Consensus (periodic) ---
@@ -566,7 +567,7 @@ mark it as "MULTI-SPECIALIST CONFIRMED" with the confirming categories.
Write findings to ${dir}/review-output.md`, Write findings to ${dir}/review-output.md`,
workingDirectory: dir, workingDirectory: dir,
maxTurns: 20, maxTurns: 20,
timeout: 180_000, timeout: CAPTURE_MS,
testName: 'review-army-consensus', testName: 'review-army-consensus',
runId, runId,
}); });
@@ -585,7 +586,7 @@ Write findings to ${dir}/review-output.md`,
content.includes('interpolat'); content.includes('interpolat');
expect(hasSqlFinding).toBe(true); expect(hasSqlFinding).toBe(true);
} }
}, 210_000); }, CAPTURE_MS);
}); });
// Finalize eval collector // Finalize eval collector
+6 -5
View File
@@ -1,4 +1,5 @@
import { expect, beforeAll, afterAll } from 'bun:test'; import { expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { import {
ROOT, runId, ROOT, runId,
@@ -65,7 +66,7 @@ Then run git diff against the detected base branch and write a brief review.
Write your findings to ${dir}/review-output.md`, Write your findings to ${dir}/review-output.md`,
workingDirectory: dir, workingDirectory: dir,
maxTurns: 15, maxTurns: 15,
timeout: 90_000, timeout: JUDGE_MS,
testName: 'review-base-branch', testName: 'review-base-branch',
runId, runId,
}); });
@@ -84,7 +85,7 @@ Write your findings to ${dir}/review-output.md`,
return cmd.includes('git diff'); return cmd.includes('git diff');
}); });
expect(usedGitDiff).toBe(true); expect(usedGitDiff).toBe(true);
}, 120_000); }, JUDGE_MS);
testConcurrentIfSelected('ship-base-branch', async () => { testConcurrentIfSelected('ship-base-branch', async () => {
const dir = path.join(baseBranchDir, 'ship-base'); const dir = path.join(baseBranchDir, 'ship-base');
@@ -125,7 +126,7 @@ Write a summary to ${dir}/ship-preflight.md including:
- The diff stat against the base branch`, - The diff stat against the base branch`,
workingDirectory: dir, workingDirectory: dir,
maxTurns: 18, maxTurns: 18,
timeout: 150_000, timeout: CAPTURE_MS,
testName: 'ship-base-branch', testName: 'ship-base-branch',
runId, runId,
}); });
@@ -155,7 +156,7 @@ Write a summary to ${dir}/ship-preflight.md including:
return command.includes('git push') || command.includes('gh pr create'); return command.includes('git push') || command.includes('gh pr create');
}); });
expect(destructiveTools).toHaveLength(0); expect(destructiveTools).toHaveLength(0);
}, 180_000); }, CAPTURE_MS);
}); });
// --- Review Dashboard Via Attribution E2E --- // --- Review Dashboard Via Attribution E2E ---
@@ -280,7 +281,7 @@ Write the dashboard output to ${dashDir}/dashboard-output.md`,
); );
// Ship dashboard should not gate when eng review is clear // Ship dashboard should not gate when eng review is clear
expect(gateQuestions).toHaveLength(0); expect(gateQuestions).toHaveLength(0);
}, 480_000); }, CAPTURE_LONG_MS);
}); });
// Module-level afterAll — finalize eval collector after all tests complete // Module-level afterAll — finalize eval collector after all tests complete
+7 -6
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { import {
ROOT, browseBin, runId, evalsEnabled, selectedTests, ROOT, browseBin, runId, evalsEnabled, selectedTests,
@@ -66,7 +67,7 @@ Run /review on the current diff (git diff main...HEAD).
Write your review findings to ${reviewDir}/review-output.md`, Write your review findings to ${reviewDir}/review-output.md`,
workingDirectory: reviewDir, workingDirectory: reviewDir,
maxTurns: 20, maxTurns: 20,
timeout: 180_000, timeout: CAPTURE_MS,
testName: 'review-sql-injection', testName: 'review-sql-injection',
runId, runId,
}); });
@@ -89,7 +90,7 @@ Write your review findings to ${reviewDir}/review-output.md`,
reviewContent.includes('unsanitized'); reviewContent.includes('unsanitized');
expect(hasSqlContent).toBe(true); expect(hasSqlContent).toBe(true);
} }
}, 210_000); }, CAPTURE_MS);
}); });
// --- Review: Enum completeness E2E --- // --- Review: Enum completeness E2E ---
@@ -144,7 +145,7 @@ Write your review findings to ${enumDir}/review-output.md
The diff adds a new "returned" status to the Order model. Your job is to check if all consumers handle it.`, The diff adds a new "returned" status to the Order model. Your job is to check if all consumers handle it.`,
workingDirectory: enumDir, workingDirectory: enumDir,
maxTurns: 15, maxTurns: 15,
timeout: 90_000, timeout: JUDGE_MS,
testName: 'review-enum-completeness', testName: 'review-enum-completeness',
runId, runId,
}); });
@@ -164,7 +165,7 @@ The diff adds a new "returned" status to the Order model. Your job is to check i
expect(mentionsReturned).toBe(true); expect(mentionsReturned).toBe(true);
expect(mentionsEnum || mentionsCritical).toBe(true); expect(mentionsEnum || mentionsCritical).toBe(true);
} }
}, 120_000); }, JUDGE_MS);
}); });
// --- Review: Design review lite E2E --- // --- Review: Design review lite E2E ---
@@ -229,7 +230,7 @@ Write your review findings to ${designDir}/review-output.md
Important: The design checklist should catch issues like blacklisted fonts, small font sizes, outline:none, !important, AI slop patterns (purple gradients, generic hero copy, 3-column feature grid), etc.`, Important: The design checklist should catch issues like blacklisted fonts, small font sizes, outline:none, !important, AI slop patterns (purple gradients, generic hero copy, 3-column feature grid), etc.`,
workingDirectory: designDir, workingDirectory: designDir,
maxTurns: 35, maxTurns: 35,
timeout: 240_000, timeout: CAPTURE_MS,
testName: 'review-design-lite', testName: 'review-design-lite',
runId, runId,
}); });
@@ -262,7 +263,7 @@ Important: The design checklist should catch issues like blacklisted fonts, smal
console.log(`Design review detected ${detected}/7 planted issues`); console.log(`Design review detected ${detected}/7 planted issues`);
expect(detected).toBeGreaterThanOrEqual(4); expect(detected).toBeGreaterThanOrEqual(4);
} }
}, 300_000); }, CAPTURE_MS);
}); });
// Base branch detection tests for review/ship + the Review Dashboard Via // Base branch detection tests for review/ship + the Review Dashboard Via
+7 -6
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { import {
ROOT, runId, evalsEnabled, ROOT, runId, evalsEnabled,
@@ -166,7 +167,7 @@ IMPORTANT:
workingDirectory: workDir, workingDirectory: workDir,
maxTurns: 10, maxTurns: 10,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000, timeout: JUDGE_MS,
testName: 'context-recovery-artifacts', testName: 'context-recovery-artifacts',
runId, runId,
}); });
@@ -193,7 +194,7 @@ IMPORTANT:
expect(foundCount).toBeGreaterThanOrEqual(1); expect(foundCount).toBeGreaterThanOrEqual(1);
console.log(`Context recovery: artifacts=${foundArtifacts}, lastSession=${foundLastSession}, timeline=${foundTimeline}`); console.log(`Context recovery: artifacts=${foundArtifacts}, lastSession=${foundLastSession}, timeline=${foundTimeline}`);
}, 180_000); }, CAPTURE_MS);
// --- Test 3: /context-save writes a file --- // --- Test 3: /context-save writes a file ---
// Hand-feed the save section of context-save/SKILL.md to claude -p and verify // Hand-feed the save section of context-save/SKILL.md to claude -p and verify
@@ -231,7 +232,7 @@ IMPORTANT:
workingDirectory: workDir, workingDirectory: workDir,
maxTurns: 10, maxTurns: 10,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000, timeout: JUDGE_MS,
testName: 'context-save-writes-file', testName: 'context-save-writes-file',
runId, runId,
}); });
@@ -264,7 +265,7 @@ IMPORTANT:
expect(hasYamlFrontmatter).toBe(true); expect(hasYamlFrontmatter).toBe(true);
console.log(`context-save: ${files.length} files created, YAML frontmatter: ${hasYamlFrontmatter}, branch: ${hasBranch}`); console.log(`context-save: ${files.length} files created, YAML frontmatter: ${hasYamlFrontmatter}, branch: ${hasBranch}`);
}, 180_000); }, CAPTURE_MS);
// --- Test 4: /context-restore loads the newest file across branches --- // --- Test 4: /context-restore loads the newest file across branches ---
// Seed two saved-context files with different YYYYMMDD-HHMMSS prefixes and // Seed two saved-context files with different YYYYMMDD-HHMMSS prefixes and
@@ -340,7 +341,7 @@ IMPORTANT:
workingDirectory: workDir, workingDirectory: workDir,
maxTurns: 8, maxTurns: 8,
allowedTools: ['Bash', 'Read', 'Grep', 'Glob'], allowedTools: ['Bash', 'Read', 'Grep', 'Glob'],
timeout: 120_000, timeout: JUDGE_MS,
testName: 'context-restore-loads-latest', testName: 'context-restore-loads-latest',
runId, runId,
}); });
@@ -361,5 +362,5 @@ IMPORTANT:
expect(loadedOlder).toBe(false); expect(loadedOlder).toBe(false);
console.log(`context-restore: loadedNewer=${loadedNewer}, loadedOlder=${loadedOlder}`); console.log(`context-restore: loadedNewer=${loadedNewer}, loadedOlder=${loadedOlder}`);
}, 180_000); }, CAPTURE_MS);
}); });
@@ -14,6 +14,7 @@
// on a failed verify the skill STOPs before any CLAUDE.md write. // on a failed verify the skill STOPs before any CLAUDE.md write.
import { test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'fs'; import * as fs from 'fs';
import * as os from 'os'; import * as os from 'os';
@@ -155,5 +156,5 @@ describeE2E('/setup-gbrain Path 4 — bad token STOPs cleanly', () => {
fs.rmSync(gstackHome, { recursive: true, force: true }); fs.rmSync(gstackHome, { recursive: true, force: true });
fs.rmSync(fakeBinDir, { recursive: true, force: true }); fs.rmSync(fakeBinDir, { recursive: true, force: true });
} }
}, 240_000); }, CAPTURE_MS);
}); });
@@ -19,6 +19,7 @@
// Cost: ~$0.50-$1.00 per run. Periodic-tier (EVALS=1 EVALS_TIER=periodic). // Cost: ~$0.50-$1.00 per run. Periodic-tier (EVALS=1 EVALS_TIER=periodic).
import { test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'fs'; import * as fs from 'fs';
import * as os from 'os'; import * as os from 'os';
@@ -264,5 +265,5 @@ describeE2E('/setup-gbrain Path 4 + Step 4.5 Yes → local PGLite for code', ()
fs.rmSync(sandboxHome, { recursive: true, force: true }); fs.rmSync(sandboxHome, { recursive: true, force: true });
fs.rmSync(fakeBinDir, { recursive: true, force: true }); fs.rmSync(fakeBinDir, { recursive: true, force: true });
} }
}, 300_000); }, CAPTURE_MS);
}); });
+2 -1
View File
@@ -16,6 +16,7 @@
// block this test asserts on). // block this test asserts on).
import { test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'fs'; import * as fs from 'fs';
import * as os from 'os'; import * as os from 'os';
@@ -245,5 +246,5 @@ describeE2E('/setup-gbrain Path 4 (Remote MCP) — happy path', () => {
fs.rmSync(gstackHome, { recursive: true, force: true }); fs.rmSync(gstackHome, { recursive: true, force: true });
fs.rmSync(fakeBinDir, { recursive: true, force: true }); fs.rmSync(fakeBinDir, { recursive: true, force: true });
} }
}, 240_000); }, CAPTURE_MS);
}); });
+3 -2
View File
@@ -49,6 +49,7 @@
* gate tier confirmed). * gate tier confirmed).
*/ */
import { expect, beforeAll, afterAll } from 'bun:test'; import { expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import * as fs from 'fs'; import * as fs from 'fs';
import * as os from 'os'; import * as os from 'os';
import * as path from 'path'; import * as path from 'path';
@@ -200,7 +201,7 @@ describeE2E('Ship doc-sync dispatch E2E (gate)', () => {
workingDirectory: repoDir, workingDirectory: repoDir,
maxTurns: 30, maxTurns: 30,
allowedTools: ['Bash', 'Read', 'Grep', 'Glob', 'Write', 'Agent', 'Task'], allowedTools: ['Bash', 'Read', 'Grep', 'Glob', 'Write', 'Agent', 'Task'],
timeout: 480_000, timeout: CAPTURE_LONG_MS,
env: { env: {
HOME: workDir, HOME: workDir,
GSTACK_HOME: path.join(workDir, 'gstack-home'), GSTACK_HOME: path.join(workDir, 'gstack-home'),
@@ -273,7 +274,7 @@ describeE2E('Ship doc-sync dispatch E2E (gate)', () => {
console.log( console.log(
`dispatchIdx=${dispatchIdx} prCreateIdx=${prCreateIdx} readPrBody=${readPrBody} exit=${result.exitReason}` `dispatchIdx=${dispatchIdx} prCreateIdx=${prCreateIdx} readPrBody=${readPrBody} exit=${result.exitReason}`
); );
}, 540_000); }, CAPTURE_LONG_MS);
}); });
}); });
+3 -2
View File
@@ -31,6 +31,7 @@
*/ */
import { test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { PTY_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import { spawnSync } from 'child_process'; import { spawnSync } from 'child_process';
import * as fs from 'fs'; import * as fs from 'fs';
@@ -158,7 +159,7 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => {
const session = await launchClaudePty({ const session = await launchClaudePty({
permissionMode: 'plan', permissionMode: 'plan',
cwd: fixture.workTree, cwd: fixture.workTree,
timeoutMs: 1_080_000, timeoutMs: PTY_LONG_MS,
// Disable network-y pieces so the agent can't reach actual github. // Disable network-y pieces so the agent can't reach actual github.
env: { GH_TOKEN: 'mock-not-real', NO_COLOR: '1' }, env: { GH_TOKEN: 'mock-not-real', NO_COLOR: '1' },
seedSkills: true, seedSkills: true,
@@ -279,6 +280,6 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => {
try { fs.rmSync(path.dirname(fixture.workTree), { recursive: true, force: true }); } catch { /* ignore */ } try { fs.rmSync(path.dirname(fixture.workTree), { recursive: true, force: true }); } catch { /* ignore */ }
} }
}, },
1_200_000, // 20 min wall clock PTY_LONG_MS, // 20 min wall clock
); );
}); });
+2 -1
View File
@@ -24,6 +24,7 @@
*/ */
import { test, expect } from 'bun:test'; import { test, expect } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import { import {
setupSkillDir, setupSkillDir,
@@ -78,6 +79,6 @@ describeE2E('/ship section-loading E2E (periodic, SDK capture)', () => {
// Guard against an empty pass: the report must have real content. // Guard against an empty pass: the report must have real content.
expect(output.trim().length).toBeGreaterThan(200); expect(output.trim().length).toBeGreaterThan(200);
}, },
360_000, CAPTURE_LONG_MS,
); );
}); });
+11 -10
View File
@@ -27,6 +27,7 @@
*/ */
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { import {
ROOT, browseBin, runId, ROOT, browseBin, runId,
@@ -204,7 +205,7 @@ Do NOT enter the prototype phase. Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome }, env: { GSTACK_HOME: gstackHome },
maxTurns: 12, maxTurns: 12,
allowedTools: ['Skill', 'Bash', 'Read'], allowedTools: ['Skill', 'Bash', 'Read'],
timeout: 120_000, timeout: JUDGE_MS,
testName: 'scrape-match-path', testName: 'scrape-match-path',
runId, runId,
}); });
@@ -224,7 +225,7 @@ Do NOT enter the prototype phase. Do NOT use AskUserQuestion.`,
expect(listedSkills).toBe(true); expect(listedSkills).toBe(true);
expect(ranBundledSkill).toBe(true); expect(ranBundledSkill).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 180_000); }, CAPTURE_MS);
// ── 2. /scrape prototype path: drive $B primitives against fixture ──── // ── 2. /scrape prototype path: drive $B primitives against fixture ────
testConcurrentIfSelected('scrape-prototype-path', async () => { testConcurrentIfSelected('scrape-prototype-path', async () => {
@@ -248,7 +249,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome }, env: { GSTACK_HOME: gstackHome },
maxTurns: 18, maxTurns: 18,
allowedTools: ['Skill', 'Bash', 'Read'], allowedTools: ['Skill', 'Bash', 'Read'],
timeout: 180_000, timeout: CAPTURE_MS,
testName: 'scrape-prototype-path', testName: 'scrape-prototype-path',
runId, runId,
}); });
@@ -283,7 +284,7 @@ Do NOT use AskUserQuestion.`,
expect(hasJsonItems).toBe(true); expect(hasJsonItems).toBe(true);
expect(mentionsSkillify).toBe(true); expect(mentionsSkillify).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 240_000); }, CAPTURE_MS);
// ── 3. /skillify happy path: scrape then skillify in one session ───── // ── 3. /skillify happy path: scrape then skillify in one session ─────
testConcurrentIfSelected('skillify-happy-path', async () => { testConcurrentIfSelected('skillify-happy-path', async () => {
@@ -316,7 +317,7 @@ Do NOT halt for clarification.`,
}, },
maxTurns: 40, maxTurns: 40,
allowedTools: ['Skill', 'Bash', 'Read', 'Write'], allowedTools: ['Skill', 'Bash', 'Read', 'Write'],
timeout: 360_000, timeout: CAPTURE_LONG_MS,
testName: 'skillify-happy-path', testName: 'skillify-happy-path',
runId, runId,
}); });
@@ -360,7 +361,7 @@ Do NOT halt for clarification.`,
expect(hasAllFiles).toBe(true); expect(hasAllFiles).toBe(true);
expect(prosesClean).toBe(true); expect(prosesClean).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 420_000); }, CAPTURE_LONG_MS);
// ── 4. /skillify provenance refusal: D1 contract ───────────────────── // ── 4. /skillify provenance refusal: D1 contract ─────────────────────
testConcurrentIfSelected('skillify-provenance-refusal', async () => { testConcurrentIfSelected('skillify-provenance-refusal', async () => {
@@ -379,7 +380,7 @@ write any files.`,
}, },
maxTurns: 8, maxTurns: 8,
allowedTools: ['Skill', 'Bash', 'Read'], allowedTools: ['Skill', 'Bash', 'Read'],
timeout: 90_000, timeout: JUDGE_MS,
testName: 'skillify-provenance-refusal', testName: 'skillify-provenance-refusal',
runId, runId,
}); });
@@ -408,7 +409,7 @@ write any files.`,
expect(noSkillsWritten).toBe(true); expect(noSkillsWritten).toBe(true);
expect(noStaging).toBe(true); expect(noStaging).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 120_000); }, JUDGE_MS);
// ── 5. /skillify approval-gate reject: D3 cleanup ──────────────────── // ── 5. /skillify approval-gate reject: D3 cleanup ────────────────────
testConcurrentIfSelected('skillify-approval-reject', async () => { testConcurrentIfSelected('skillify-approval-reject', async () => {
@@ -435,7 +436,7 @@ Use HOME=${workDir}. Do NOT commit the skill.`,
}, },
maxTurns: 35, maxTurns: 35,
allowedTools: ['Skill', 'Bash', 'Read', 'Write'], allowedTools: ['Skill', 'Bash', 'Read', 'Write'],
timeout: 360_000, timeout: CAPTURE_LONG_MS,
testName: 'skillify-approval-reject', testName: 'skillify-approval-reject',
runId, runId,
}); });
@@ -462,5 +463,5 @@ Use HOME=${workDir}. Do NOT commit the skill.`,
expect(writtenSkills.length).toBe(0); expect(writtenSkills.length).toBe(0);
expect(stagingLeftovers.length).toBe(0); expect(stagingLeftovers.length).toBe(0);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {} try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 420_000); }, CAPTURE_LONG_MS);
}); });
+3 -2
View File
@@ -19,6 +19,7 @@
*/ */
import { test, expect, beforeAll, afterAll } from 'bun:test'; import { test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { import {
ROOT, runId, ROOT, runId,
@@ -178,7 +179,7 @@ This is a solo repo (REPO_MODE=solo). For pre-existing failures, recommend fixin
workingDirectory: triageDir, workingDirectory: triageDir,
maxTurns: 20, maxTurns: 20,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'], allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
timeout: 180_000, timeout: CAPTURE_MS,
testName: 'ship-triage', testName: 'ship-triage',
runId, runId,
}); });
@@ -229,7 +230,7 @@ This is a solo repo (REPO_MODE=solo). For pre-existing failures, recommend fixin
// Must have actually run both test files (exercises both failure classes) // Must have actually run both test files (exercises both failure classes)
expect(ranMathTest).toBe(true); expect(ranMathTest).toBe(true);
expect(ranStringTest).toBe(true); expect(ranStringTest).toBe(true);
}, 240_000); }, CAPTURE_MS);
}); });
// Module-level afterAll — finalize eval collector after all tests complete // Module-level afterAll — finalize eval collector after all tests complete
+11 -10
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import { import {
ROOT, browseBin, runId, evalsEnabled, ROOT, browseBin, runId, evalsEnabled,
@@ -83,7 +84,7 @@ IMPORTANT:
// other rounds — marginal at 180s, same contention story as // other rounds — marginal at 180s, same contention story as
// review-dashboard-via and retro-base-branch. Outer bun timeout // review-dashboard-via and retro-base-branch. Outer bun timeout
// rises to 360s for headroom. // rises to 360s for headroom.
timeout: 300_000, timeout: CAPTURE_MS,
testName: 'document-release', testName: 'document-release',
runId, runId,
}); });
@@ -120,7 +121,7 @@ IMPORTANT:
} else { } else {
console.warn('README was NOT updated — agent may not have found the feature'); console.warn('README was NOT updated — agent may not have found the feature');
} }
}, 360_000); }, CAPTURE_LONG_MS);
}); });
// --- Ship workflow with local bare remote --- // --- Ship workflow with local bare remote ---
@@ -174,7 +175,7 @@ describeIfSelected('Ship workflow E2E', ['ship-local-workflow'], () => {
4. Push to origin: git push origin feature/ship-test`, 4. Push to origin: git push origin feature/ship-test`,
workingDirectory: shipWorkDir, workingDirectory: shipWorkDir,
maxTurns: 8, maxTurns: 8,
timeout: 120_000, timeout: JUDGE_MS,
testName: 'ship-local-workflow', testName: 'ship-local-workflow',
runId, runId,
}); });
@@ -198,7 +199,7 @@ describeIfSelected('Ship workflow E2E', ['ship-local-workflow'], () => {
expect(branchExists).toBe(true); expect(branchExists).toBe(true);
expect(versionBumped).toBe(true); expect(versionBumped).toBe(true);
console.log(`Branch pushed: ${branchExists}, VERSION: ${versionContent}, bumped: ${versionBumped}`); console.log(`Branch pushed: ${branchExists}, VERSION: ${versionContent}, bumped: ${versionBumped}`);
}, 150_000); }, CAPTURE_MS);
}); });
// setup-cookies-detect REMOVED: The cookie-import-browser module has 30+ thorough // setup-cookies-detect REMOVED: The cookie-import-browser module has 30+ thorough
@@ -297,7 +298,7 @@ Skip any AskUserQuestion calls — auto-approve the upgrade. Write a summary of
IMPORTANT: The install directory is at ./.claude/skills/gstack use that exact path.`, IMPORTANT: The install directory is at ./.claude/skills/gstack use that exact path.`,
workingDirectory: upgradeDir, workingDirectory: upgradeDir,
maxTurns: 20, maxTurns: 20,
timeout: 180_000, timeout: CAPTURE_MS,
testName: 'gstack-upgrade-happy-path', testName: 'gstack-upgrade-happy-path',
runId, runId,
}); });
@@ -317,7 +318,7 @@ IMPORTANT: The install directory is at ./.claude/skills/gstack — use that exac
expect(['success', 'error_max_turns']).toContain(result.exitReason); expect(['success', 'error_max_turns']).toContain(result.exitReason);
expect(versionAfter).toBe('0.6.0'); expect(versionAfter).toBe('0.6.0');
}, 240_000); }, CAPTURE_MS);
}); });
// --- Test Coverage Audit E2E --- // --- Test Coverage Audit E2E ---
@@ -418,7 +419,7 @@ Output the diagram directly.`,
workingDirectory: coverageDir, workingDirectory: coverageDir,
maxTurns: 15, maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'], allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
timeout: 120_000, timeout: JUDGE_MS,
testName: 'ship-coverage-audit', testName: 'ship-coverage-audit',
runId, runId,
}); });
@@ -443,7 +444,7 @@ Output the diagram directly.`,
// At minimum, the agent should have read the source and test files // At minimum, the agent should have read the source and test files
const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read'); const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read');
expect(readCalls.length).toBeGreaterThan(0); expect(readCalls.length).toBeGreaterThan(0);
}, 180_000); }, CAPTURE_MS);
}); });
// --- Codex skill E2E --- // --- Codex skill E2E ---
@@ -520,7 +521,7 @@ Follow those instructions to run codex review against the diff on this branch.
Write the full output (including the GATE verdict) to ${codexDir}/codex-output.md`, Write the full output (including the GATE verdict) to ${codexDir}/codex-output.md`,
workingDirectory: codexDir, workingDirectory: codexDir,
maxTurns: 25, maxTurns: 25,
timeout: 300_000, timeout: CAPTURE_MS,
testName: 'codex-review', testName: 'codex-review',
runId, runId,
model: 'claude-opus-4-7', model: 'claude-opus-4-7',
@@ -538,7 +539,7 @@ Write the full output (including the GATE verdict) to ${codexDir}/codex-output.m
const hasCodexOutput = output.includes('CODEX') || output.includes('GATE') || output.includes('codex'); const hasCodexOutput = output.includes('CODEX') || output.includes('GATE') || output.includes('codex');
expect(hasCodexOutput).toBe(true); expect(hasCodexOutput).toBe(true);
} }
}, 360_000); }, CAPTURE_LONG_MS);
}); });
// Module-level afterAll — finalize eval collector after all tests complete // Module-level afterAll — finalize eval collector after all tests complete
+2 -1
View File
@@ -11,6 +11,7 @@
*/ */
import { afterAll, expect } from 'bun:test'; import { afterAll, expect } from 'bun:test';
import { JUDGE_MS } from './helpers/eval-budgets';
import Anthropic from '@anthropic-ai/sdk'; import Anthropic from '@anthropic-ai/sdk';
import * as fs from 'fs'; import * as fs from 'fs';
import * as path from 'path'; import * as path from 'path';
@@ -556,7 +557,7 @@ describeIfSelected('Baseline score pinning', ['baseline score pinning'], () => {
if (!passed) { if (!passed) {
throw new Error(`Score regressions detected:\n${regressions.join('\n')}`); throw new Error(`Score regressions detected:\n${regressions.join('\n')}`);
} }
}, 60_000); }, JUDGE_MS);
}); });
// --- Workflow SKILL.md quality evals (10 new tests for 100% coverage) --- // --- Workflow SKILL.md quality evals (10 new tests for 100% coverage) ---
+21 -20
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, afterAll } from 'bun:test'; import { describe, test, expect, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner'; import { runSkillTest } from './helpers/session-runner';
import type { SkillTestResult } from './helpers/session-runner'; import type { SkillTestResult } from './helpers/session-runner';
import { EvalCollector } from './helpers/eval-store'; import { EvalCollector } from './helpers/eval-store';
@@ -197,7 +198,7 @@ describeE2E('Skill Routing E2E — Developer Journey', () => {
workingDirectory: tmpDir, workingDirectory: tmpDir,
maxTurns: 5, maxTurns: 5,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 60_000, timeout: JUDGE_MS,
testName, testName,
runId, runId,
}); });
@@ -213,7 +214,7 @@ describeE2E('Skill Routing E2E — Developer Journey', () => {
} finally { } finally {
fs.rmSync(tmpDir, { recursive: true, force: true }); fs.rmSync(tmpDir, { recursive: true, force: true });
} }
}, 150_000); }, CAPTURE_MS);
testIfSelected('journey-plan-eng', async () => { testIfSelected('journey-plan-eng', async () => {
const tmpDir = createRoutingWorkDir('plan-eng'); const tmpDir = createRoutingWorkDir('plan-eng');
@@ -247,7 +248,7 @@ describeE2E('Skill Routing E2E — Developer Journey', () => {
workingDirectory: tmpDir, workingDirectory: tmpDir,
maxTurns: 5, maxTurns: 5,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 60_000, timeout: JUDGE_MS,
testName, testName,
runId, runId,
}); });
@@ -263,7 +264,7 @@ describeE2E('Skill Routing E2E — Developer Journey', () => {
} finally { } finally {
fs.rmSync(tmpDir, { recursive: true, force: true }); fs.rmSync(tmpDir, { recursive: true, force: true });
} }
}, 150_000); }, CAPTURE_MS);
// Removed: journey-think-bigger // Removed: journey-think-bigger
// Tested ambiguous routing ("think bigger" → plan-ceo-review) but Claude // Tested ambiguous routing ("think bigger" → plan-ceo-review) but Claude
@@ -309,7 +310,7 @@ export default app;
workingDirectory: tmpDir, workingDirectory: tmpDir,
maxTurns: 5, maxTurns: 5,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 60_000, timeout: JUDGE_MS,
testName, testName,
runId, runId,
}); });
@@ -326,7 +327,7 @@ export default app;
} finally { } finally {
fs.rmSync(tmpDir, { recursive: true, force: true }); fs.rmSync(tmpDir, { recursive: true, force: true });
} }
}, 150_000); }, CAPTURE_MS);
testIfSelected('journey-qa', async () => { testIfSelected('journey-qa', async () => {
const tmpDir = createRoutingWorkDir('qa'); const tmpDir = createRoutingWorkDir('qa');
@@ -345,7 +346,7 @@ export default app;
workingDirectory: tmpDir, workingDirectory: tmpDir,
maxTurns: 5, maxTurns: 5,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 60_000, timeout: JUDGE_MS,
testName, testName,
runId, runId,
}); });
@@ -362,7 +363,7 @@ export default app;
} finally { } finally {
fs.rmSync(tmpDir, { recursive: true, force: true }); fs.rmSync(tmpDir, { recursive: true, force: true });
} }
}, 150_000); }, CAPTURE_MS);
testIfSelected('journey-code-review', async () => { testIfSelected('journey-code-review', async () => {
const tmpDir = createRoutingWorkDir('code-review'); const tmpDir = createRoutingWorkDir('code-review');
@@ -386,7 +387,7 @@ export default app;
workingDirectory: tmpDir, workingDirectory: tmpDir,
maxTurns: 5, maxTurns: 5,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 120_000, timeout: JUDGE_MS,
testName, testName,
runId, runId,
}); });
@@ -402,7 +403,7 @@ export default app;
} finally { } finally {
fs.rmSync(tmpDir, { recursive: true, force: true }); fs.rmSync(tmpDir, { recursive: true, force: true });
} }
}, 150_000); }, CAPTURE_MS);
testIfSelected('journey-ship', async () => { testIfSelected('journey-ship', async () => {
const tmpDir = createRoutingWorkDir('ship'); const tmpDir = createRoutingWorkDir('ship');
@@ -425,7 +426,7 @@ export default app;
workingDirectory: tmpDir, workingDirectory: tmpDir,
maxTurns: 5, maxTurns: 5,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 60_000, timeout: JUDGE_MS,
testName, testName,
runId, runId,
}); });
@@ -441,7 +442,7 @@ export default app;
} finally { } finally {
fs.rmSync(tmpDir, { recursive: true, force: true }); fs.rmSync(tmpDir, { recursive: true, force: true });
} }
}, 150_000); }, CAPTURE_MS);
testIfSelected('journey-docs', async () => { testIfSelected('journey-docs', async () => {
const tmpDir = createRoutingWorkDir('docs'); const tmpDir = createRoutingWorkDir('docs');
@@ -462,7 +463,7 @@ export default app;
workingDirectory: tmpDir, workingDirectory: tmpDir,
maxTurns: 5, maxTurns: 5,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 60_000, timeout: JUDGE_MS,
testName, testName,
runId, runId,
}); });
@@ -478,7 +479,7 @@ export default app;
} finally { } finally {
fs.rmSync(tmpDir, { recursive: true, force: true }); fs.rmSync(tmpDir, { recursive: true, force: true });
} }
}, 150_000); }, CAPTURE_MS);
testIfSelected('journey-retro', async () => { testIfSelected('journey-retro', async () => {
const tmpDir = createRoutingWorkDir('retro'); const tmpDir = createRoutingWorkDir('retro');
@@ -505,7 +506,7 @@ export default app;
workingDirectory: tmpDir, workingDirectory: tmpDir,
maxTurns: 5, maxTurns: 5,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 120_000, timeout: JUDGE_MS,
testName, testName,
runId, runId,
}); });
@@ -521,7 +522,7 @@ export default app;
} finally { } finally {
fs.rmSync(tmpDir, { recursive: true, force: true }); fs.rmSync(tmpDir, { recursive: true, force: true });
} }
}, 150_000); }, CAPTURE_MS);
testIfSelected('journey-design-system', async () => { testIfSelected('journey-design-system', async () => {
const tmpDir = createRoutingWorkDir('design-system'); const tmpDir = createRoutingWorkDir('design-system');
@@ -534,7 +535,7 @@ export default app;
workingDirectory: tmpDir, workingDirectory: tmpDir,
maxTurns: 5, maxTurns: 5,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 60_000, timeout: JUDGE_MS,
testName, testName,
runId, runId,
}); });
@@ -550,7 +551,7 @@ export default app;
} finally { } finally {
fs.rmSync(tmpDir, { recursive: true, force: true }); fs.rmSync(tmpDir, { recursive: true, force: true });
} }
}, 150_000); }, CAPTURE_MS);
testIfSelected('journey-visual-qa', async () => { testIfSelected('journey-visual-qa', async () => {
const tmpDir = createRoutingWorkDir('visual-qa'); const tmpDir = createRoutingWorkDir('visual-qa');
@@ -585,7 +586,7 @@ body { font-family: sans-serif; }
workingDirectory: tmpDir, workingDirectory: tmpDir,
maxTurns: 5, maxTurns: 5,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'], allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 60_000, timeout: JUDGE_MS,
testName, testName,
runId, runId,
}); });
@@ -602,5 +603,5 @@ body { font-family: sans-serif; }
} finally { } finally {
fs.rmSync(tmpDir, { recursive: true, force: true }); fs.rmSync(tmpDir, { recursive: true, force: true });
} }
}, 150_000); }, CAPTURE_MS);
}); });