refactor(test): mechanical sweep — 298 paid-test timeouts onto eval-budget tiers

69 files, both shapes (trailing bun-test budgets and runner
timeout/timeoutMs options), ROUND-UP ONLY so nothing that passed can
start failing: 75 → JUDGE_MS, 137 → CAPTURE_MS, 74 → CAPTURE_LONG_MS,
9 → PTY_MS, 3 → PTY_LONG_MS. Raw >=60s literal count in the paid scope:
395 → 97, of which 51 are non-timeout noise (fixture dates, run IDs)
and 46 are enumerated justified holds (comment-carrying calibrated
budgets, poll-loop constants, utility spawn waits, and the seven
physical-ceiling 1_500_000 sites). The eval-budgets policy ratchet
keeps the residue from regrowing.

Known collapse: where an inner runner budget and its enclosing test
budget now share a tier, the old stagger is gone — an overrun surfaces
as a bun test timeout instead of a graceful runner timeout
(diagnosability trade, not a correctness one).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-29 05:25:06 +00:00
co-authored by Claude Fable 5
parent 74ae357e0f
commit 6841183c35
69 changed files with 367 additions and 298 deletions
+2 -1
View File
@@ -20,6 +20,7 @@
*/
import { test, expect } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { setupSkillDir, skillFromWorktree, captureSectionReads } from './helpers/auq-sdk-capture';
import { CARVE_GUARDS } from './helpers/carve-guards';
@@ -97,7 +98,7 @@ describeE2E('carve behavioral section-loading (periodic, SDK capture)', () => {
});
expect(output.trim().length).toBeGreaterThan(200);
},
540_000,
CAPTURE_LONG_MS,
);
}
});
+9 -8
View File
@@ -26,6 +26,7 @@
* Periodic tier (Codex non-determinism). Cost: ~$2-3 per full run.
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runCodexSkill, installSkillToTempHome } from './helpers/codex-session-runner';
import type { CodexResult } from './helpers/codex-session-runner';
import { EvalCollector } from './helpers/eval-store';
@@ -186,7 +187,7 @@ describeCodex('Codex Plan Format — CEO Mode Selection', () => {
const result = await runCodexSkill({
skillDir,
prompt: `Read the plan-ceo-review skill. Read plan.md (the plan to review). Proceed to Step 0F (Mode Selection) where the skill presents 4 mode options (SCOPE EXPANSION, SELECTIVE EXPANSION, HOLD SCOPE, SCOPE REDUCTION) via AskUserQuestion. These options differ in kind (review posture), not coverage. ${captureInstruction(outFile)}`,
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
cwd: planDir,
skillName: 'gstack-plan-ceo-review',
sandbox: 'workspace-write',
@@ -208,7 +209,7 @@ describeCodex('Codex Plan Format — CEO Mode Selection', () => {
// kind-differentiated: no fabricated score, must have note
expect(captured).not.toMatch(COMPLETENESS_RE);
expect(captured).toMatch(KIND_NOTE_RE);
}, 360_000);
}, CAPTURE_LONG_MS);
});
describeCodex('Codex Plan Format — CEO Approach Menu', () => {
@@ -226,7 +227,7 @@ describeCodex('Codex Plan Format — CEO Approach Menu', () => {
const result = await runCodexSkill({
skillDir,
prompt: `Read the plan-ceo-review skill. Read plan.md. Proceed to Step 0C-bis (Implementation Alternatives / Approach Menu) where the skill generates 2-3 approaches (minimal viable vs ideal architecture) and presents them via AskUserQuestion. These options differ in coverage so Completeness: N/10 applies. ${captureInstruction(outFile)}`,
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
cwd: planDir,
skillName: 'gstack-plan-ceo-review',
sandbox: 'workspace-write',
@@ -245,7 +246,7 @@ describeCodex('Codex Plan Format — CEO Approach Menu', () => {
expect(captured.length).toBeGreaterThan(ELI10_LENGTH_FLOOR);
expect(captured).toMatch(RECOMMENDATION_RE);
expect(captured).toMatch(COMPLETENESS_RE);
}, 360_000);
}, CAPTURE_LONG_MS);
});
describeCodex('Codex Plan Format — Eng Coverage Issue', () => {
@@ -263,7 +264,7 @@ describeCodex('Codex Plan Format — Eng Coverage Issue', () => {
const result = await runCodexSkill({
skillDir,
prompt: `Read the plan-eng-review skill. Read plan.md. In your Section 3 Test Review, generate ONE AskUserQuestion about test coverage depth where options are clearly coverage-differentiated: A) full coverage incl. edge + error paths (Completeness 10/10), B) happy path only (7/10), C) smoke test (3/10). ${captureInstruction(outFile)}`,
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
cwd: planDir,
skillName: 'gstack-plan-eng-review',
sandbox: 'workspace-write',
@@ -282,7 +283,7 @@ describeCodex('Codex Plan Format — Eng Coverage Issue', () => {
expect(captured.length).toBeGreaterThan(ELI10_LENGTH_FLOOR);
expect(captured).toMatch(RECOMMENDATION_RE);
expect(captured).toMatch(COMPLETENESS_RE);
}, 360_000);
}, CAPTURE_LONG_MS);
});
describeCodex('Codex Plan Format — Eng Kind Issue', () => {
@@ -300,7 +301,7 @@ describeCodex('Codex Plan Format — Eng Kind Issue', () => {
const result = await runCodexSkill({
skillDir,
prompt: `Read the plan-eng-review skill. Read plan.md. In your Section 1 Architecture review, generate ONE AskUserQuestion about an architectural choice where the options differ in kind (e.g. Redis vs Postgres materialized view vs in-process cache — different kinds of systems with different tradeoffs, NOT more-or-less-complete versions of the same thing). ${captureInstruction(outFile)}`,
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
cwd: planDir,
skillName: 'gstack-plan-eng-review',
sandbox: 'workspace-write',
@@ -321,5 +322,5 @@ describeCodex('Codex Plan Format — Eng Kind Issue', () => {
// kind-differentiated: no fabricated score
expect(captured).not.toMatch(COMPLETENESS_RE);
expect(captured).toMatch(KIND_NOTE_RE);
}, 360_000);
}, CAPTURE_LONG_MS);
});
@@ -21,6 +21,7 @@
* Periodic tier (Codex non-determinism, ~$2-3/run).
*/
import { describe, test, expect } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import * as path from 'node:path';
import { e2eTierEnabled } from './helpers/e2e-gate';
import { runCodexSkill } from './helpers/codex-session-runner';
@@ -69,7 +70,7 @@ describeCodex('/codex recommendation substance (live, periodic)', () => {
skillDir: path.join(ROOT, 'codex'),
skillName: 'codex',
prompt: FIXTURE_DIFF,
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
});
if (result.output.startsWith('SKIP:')) {
@@ -98,6 +99,6 @@ describeCodex('/codex recommendation substance (live, periodic)', () => {
);
}
},
360_000,
CAPTURE_LONG_MS,
);
});
+2 -1
View File
@@ -11,6 +11,7 @@
* golden), parallel shards (worktree copies), or live symlinked installs.
*/
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
@@ -262,5 +263,5 @@ You are authorized to implement the minimal fix. The task boundary is src/parse-
expect(readmeDecoyUntouched).toBe(true);
console.log(`codex-sol-scope: ${result.tokens} tokens, ${result.toolCalls.length} tool calls, ${Math.round(result.durationMs / 1000)}s`);
}, 300_000);
}, CAPTURE_MS);
});
+5 -4
View File
@@ -14,6 +14,7 @@
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runCodexSkill, parseCodexJSONL, installSkillToTempHome } from './helpers/codex-session-runner';
import type { CodexResult } from './helpers/codex-session-runner';
import { CODEX_REVIEW_E2E_SECTIONS } from './helpers/skill-fixture';
@@ -150,7 +151,7 @@ describeCodex('Codex E2E', () => {
const result = await runCodexSkill({
skillDir,
prompt: 'List any skills or instructions you have available. Just list the names.',
timeoutMs: 60_000,
timeoutMs: JUDGE_MS,
cwd: testWorktree,
skillName: 'gstack-review',
});
@@ -171,7 +172,7 @@ describeCodex('Codex E2E', () => {
expect(
outputLower.includes('review') || outputLower.includes('gstack') || outputLower.includes('skill'),
).toBe(true);
}, 120_000);
}, JUDGE_MS);
// Validates that Codex can invoke the gstack-review skill, run a diff-based
// code review, and produce structured review output with findings/issues.
@@ -186,7 +187,7 @@ describeCodex('Codex E2E', () => {
const result = await runCodexSkill({
skillDir,
prompt: 'Run the gstack-review skill on this repository. Review the current branch diff and report your findings.',
timeoutMs: 540_000,
timeoutMs: CAPTURE_LONG_MS,
cwd: testWorktree,
skillName: 'gstack-review',
sections: CODEX_REVIEW_E2E_SECTIONS,
@@ -224,5 +225,5 @@ describeCodex('Codex E2E', () => {
outputLower.includes('p1') ||
outputLower.includes('p2');
expect(hasReviewContent).toBe(true);
}, 600_000);
}, CAPTURE_LONG_MS);
});
+3 -2
View File
@@ -15,6 +15,7 @@
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS } from './helpers/eval-budgets';
import { runGeminiSkill } from './helpers/gemini-session-runner';
import type { GeminiResult } from './helpers/gemini-session-runner';
import { EvalCollector } from './helpers/eval-store';
@@ -151,7 +152,7 @@ describeGemini('Gemini E2E', () => {
// Uses a simple prompt that doesn't require skill invocation or complex navigation.
const result = await runGeminiSkill({
prompt: 'What is this project? Answer in one sentence based on the README.',
timeoutMs: 90_000,
timeoutMs: JUDGE_MS,
cwd: testWorktree,
});
@@ -163,5 +164,5 @@ describeGemini('Gemini E2E', () => {
recordGeminiE2E('gemini-smoke', result, passed);
expect(result.output.length, 'Gemini should produce output').toBeGreaterThan(10);
}, 120_000);
}, JUDGE_MS);
});
+2 -1
View File
@@ -12,6 +12,7 @@
*/
import { expect } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { judgeRecommendation } from './helpers/llm-judge';
import { describeIfSelected, testIfSelected } from './helpers/e2e-helpers';
@@ -181,5 +182,5 @@ Net: ...`);
`[hedge:${label}] expected commits=false; got ${score.commits}. text="${text}"`,
).toBe(false);
}
}, 240_000);
}, CAPTURE_MS);
});
@@ -23,6 +23,7 @@
* A/B and matrix evals (test/helpers/auq-sdk-capture.ts).
*/
import { test, expect } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs';
import {
@@ -86,6 +87,6 @@ describeE2E('AskUserQuestion format compliance (gate)', () => {
);
}
},
300_000,
CAPTURE_MS,
);
});
+2 -1
View File
@@ -16,6 +16,7 @@
* (N SDK runs, ~$0.50-1 each).
*/
import { test } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs';
import {
@@ -99,6 +100,6 @@ describeE2E('AUQ consistency across runs (periodic)', () => {
`format elements every run; substance ${minSub}-${maxSub}`,
);
},
N_RUNS * 300_000 + 60_000,
N_RUNS * CAPTURE_MS + 60_000,
);
});
+2 -1
View File
@@ -23,6 +23,7 @@
* Run a subset in the foreground with AUQ_MATRIX_ONLY="plan-eng-review,cso".
*/
import { test } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs';
import {
@@ -174,7 +175,7 @@ describeE2E('AUQ behavioral matrix (periodic)', () => {
);
}
},
300_000,
CAPTURE_MS,
);
}
});
@@ -23,6 +23,7 @@
* strictly less unrelated review-section text in context.
*/
import { test } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs';
import {
@@ -109,6 +110,6 @@ describeE2E('AUQ no-degradation: verbose vs carved (periodic)', () => {
// eslint-disable-next-line no-console
console.log('[AUQ-AB] NO DEGRADATION:\n' + summary);
},
600_000,
CAPTURE_LONG_MS,
);
});
+3 -2
View File
@@ -38,6 +38,7 @@
*/
import { test, expect } from 'bun:test';
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillObservation } from './helpers/claude-pty-runner';
import * as fs from 'fs';
@@ -113,7 +114,7 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', ()
skillName: 'plan-ceo-review',
inPlanMode: true,
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
timeoutMs: 540_000,
timeoutMs: CAPTURE_LONG_MS,
env: { GSTACK_HOME: tmpHome, CONDUCTOR_WORKSPACE_PATH: tmpHome },
});
@@ -137,5 +138,5 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', ()
} finally {
try { fs.rmSync(tmpHome, { recursive: true, force: true }); } catch { /* best-effort */ }
}
}, 660_000);
}, PTY_MS);
});
+2 -1
View File
@@ -25,6 +25,7 @@
*/
import { test, expect } from 'bun:test';
import { PTY_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { spawnSync } from 'child_process';
import * as fs from 'fs';
@@ -174,6 +175,6 @@ describeE2E('/autoplan chain ordering (periodic)', () => {
try { fs.rmSync(tempDir, { recursive: true, force: true }); } catch { /* ignore */ }
}
},
1_200_000, // 20 min absolute test ceiling
PTY_LONG_MS, // 20 min absolute test ceiling
);
});
+2 -1
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId, evalsEnabled,
@@ -98,7 +99,7 @@ Add a new /greet skill that prints a welcome message.
testName: 'autoplan-dual-voice',
workingDirectory: workDir,
prompt: `/autoplan ${planPath}`,
timeout: 600_000, // 10 min
timeout: CAPTURE_LONG_MS, // 10 min
// /autoplan spawns subagents and calls codex via Bash; it needs the
// full tool set to get past Phase 1. Bash+Read+Write alone wasn't
// enough — the skill stalled trying to invoke Agent/Skill.
+9 -8
View File
@@ -19,6 +19,7 @@
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { ClaudeAdapter } from './helpers/providers/claude';
import { GptAdapter } from './helpers/providers/gpt';
import { GeminiAdapter } from './helpers/providers/gemini';
@@ -94,7 +95,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
process.stderr.write(`\nclaude live smoke: SKIPPED — ${check.reason}\n`);
return;
}
const result = await claude.run({ prompt: PROMPT, workdir, timeoutMs: 120_000 });
const result = await claude.run({ prompt: PROMPT, workdir, timeoutMs: JUDGE_MS });
if (result.error) {
throw new Error(`claude errored: ${result.error.code}${result.error.reason}`);
}
@@ -106,7 +107,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
expect(result.modelUsed.length).toBeGreaterThan(0);
const cost = claude.estimateCost(result.tokens, result.modelUsed);
expect(cost).toBeGreaterThan(0);
}, 150_000);
}, CAPTURE_MS);
test('gpt: trivial prompt produces parseable output', async () => {
const check = await gpt.available();
@@ -114,7 +115,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
process.stderr.write(`\ngpt live smoke: SKIPPED — ${check.reason}\n`);
return;
}
const result = await gpt.run({ prompt: PROMPT, workdir, timeoutMs: 120_000 });
const result = await gpt.run({ prompt: PROMPT, workdir, timeoutMs: JUDGE_MS });
if (result.error) {
throw new Error(`gpt errored: ${result.error.code}${result.error.reason}`);
}
@@ -125,7 +126,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
expect(typeof result.modelUsed).toBe('string');
const cost = gpt.estimateCost(result.tokens, result.modelUsed);
expect(cost).toBeGreaterThan(0);
}, 150_000);
}, CAPTURE_MS);
test('gemini: trivial prompt produces parseable output', async () => {
const check = await gemini.available();
@@ -133,7 +134,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
process.stderr.write(`\ngemini live smoke: SKIPPED — ${check.reason}\n`);
return;
}
const result = await gemini.run({ prompt: PROMPT, workdir, timeoutMs: 120_000 });
const result = await gemini.run({ prompt: PROMPT, workdir, timeoutMs: JUDGE_MS });
if (result.error) {
// auth / rate_limit are ENVIRONMENT conditions the test can't act on
// (e.g. Google deprecated the individual code-assist auth path — the
@@ -155,7 +156,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
expect(result.durationMs).toBeGreaterThan(0);
expect(typeof result.modelUsed).toBe('string');
expect(result.modelUsed.length).toBeGreaterThan(0);
}, 150_000);
}, CAPTURE_MS);
test('timeout error surfaces as error.code=timeout (no exception)', async () => {
// Use whatever adapter is available first — all three should share timeout semantics.
@@ -183,7 +184,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
prompt: PROMPT,
workdir,
providers: ['claude', 'gpt', 'gemini'],
timeoutMs: 120_000,
timeoutMs: JUDGE_MS,
skipUnavailable: false,
});
expect(report.entries).toHaveLength(3);
@@ -201,5 +202,5 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
if (!hadSuccess) {
process.stderr.write('\nrunBenchmark live: no provider produced a clean result (no auth?)\n');
}
}, 300_000);
}, CAPTURE_MS);
});
+3 -2
View File
@@ -21,6 +21,7 @@
*/
import { test, expect } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'fs';
import * as os from 'os';
@@ -150,7 +151,7 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
fs.rmSync(fakeBinDir, { recursive: true, force: true });
fs.rmSync(tempHome, { recursive: true, force: true });
}
}, 180_000);
}, CAPTURE_MS);
test('privacy gate does NOT fire when artifacts_sync_mode_prompted is already true', async () => {
// Same staging, but prompted=true this time. Gate should be silent.
@@ -228,5 +229,5 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
fs.rmSync(fakeBinDir, { recursive: true, force: true });
fs.rmSync(tempHome, { recursive: true, force: true });
}
}, 180_000);
}, CAPTURE_MS);
});
+12 -11
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, browseBin, runId, evalsEnabled,
@@ -47,7 +48,7 @@ describeIfSelected('Skill E2E tests', [
Report the results of each command.`,
workingDirectory: tmpDir,
maxTurns: 7,
timeout: 60_000,
timeout: JUDGE_MS,
testName: 'browse-basic',
runId,
});
@@ -56,7 +57,7 @@ Report the results of each command.`,
recordE2E(evalCollector, 'browse basic commands', 'Skill E2E tests', result);
expect(result.browseErrors).toHaveLength(0);
expect(result.exitReason).toBe('success');
}, 90_000);
}, JUDGE_MS);
testConcurrentIfSelected('browse-snapshot', async () => {
const result = await runSkillTest({
@@ -69,7 +70,7 @@ Report the results of each command.`,
Report what each command returned.`,
workingDirectory: tmpDir,
maxTurns: 9,
timeout: 60_000,
timeout: JUDGE_MS,
testName: 'browse-snapshot',
runId,
});
@@ -81,7 +82,7 @@ Report what each command returned.`,
console.warn('Browse errors (non-fatal):', result.browseErrors);
}
expect(result.exitReason).toBe('success');
}, 90_000);
}, JUDGE_MS);
testConcurrentIfSelected('skillmd-setup-discovery', async () => {
// P2 (v1.2.0): the browse SETUP/binary-discovery block moved from the root
@@ -104,7 +105,7 @@ Then run: $B text
Report whether it worked.`,
workingDirectory: tmpDir,
maxTurns: 10,
timeout: 60_000,
timeout: JUDGE_MS,
testName: 'skillmd-setup-discovery',
runId,
});
@@ -112,7 +113,7 @@ Report whether it worked.`,
recordE2E(evalCollector, 'SKILL.md setup block discovery', 'Skill E2E tests', result);
expect(result.browseErrors).toHaveLength(0);
expect(result.exitReason).toBe('success');
}, 90_000);
}, JUDGE_MS);
testConcurrentIfSelected('skillmd-no-local-binary', async () => {
// Create a tmpdir with no browse binary — no local .claude/skills/gstack/browse/dist/browse
@@ -149,7 +150,7 @@ Report the exact output. Do NOT try to fix or install anything — just report w
// Clean up
try { fs.rmSync(emptyDir, { recursive: true, force: true }); } catch {}
}, 60_000);
}, JUDGE_MS);
testConcurrentIfSelected('skillmd-outside-git', async () => {
// Create a tmpdir outside any git repo
@@ -182,7 +183,7 @@ Report the exact output — either "READY: <path>" or "NEEDS_SETUP".`,
// Clean up
try { fs.rmSync(nonGitDir, { recursive: true, force: true }); } catch {}
}, 60_000);
}, JUDGE_MS);
testConcurrentIfSelected('operational-learning', async () => {
const opDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-oplearn-'));
@@ -286,7 +287,7 @@ Log the operational learning now. Then say what you logged.`,
// Clean up
try { fs.rmSync(opDir, { recursive: true, force: true }); } catch {}
}, 90_000);
}, JUDGE_MS);
testConcurrentIfSelected('session-awareness', async () => {
const sessionDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-session-'));
@@ -353,7 +354,7 @@ Since this is non-interactive, DO NOT actually call AskUserQuestion. Instead, wr
Remember: _SESSIONS=4, so ELI16 mode is active. The user is juggling multiple windows and may not remember what this conversation is about. Re-ground them.`,
workingDirectory: sessionDir,
maxTurns: 8,
timeout: 60_000,
timeout: JUDGE_MS,
testName: 'session-awareness',
runId,
});
@@ -394,7 +395,7 @@ Remember: _SESSIONS=4, so ELI16 mode is active. The user is juggling multiple wi
// Clean up
try { fs.rmSync(sessionDir, { recursive: true, force: true }); } catch {}
}, 90_000);
}, JUDGE_MS);
});
// Module-level afterAll — finalize eval collector after all tests complete
+3 -2
View File
@@ -21,6 +21,7 @@
*/
import { test, expect } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillObservation } from './helpers/claude-pty-runner';
@@ -46,7 +47,7 @@ describeE2E('Conductor renders decisions as prose (periodic)', () => {
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
env: { CONDUCTOR_WORKSPACE_PATH: '/tmp/conductor-prose-e2e' },
initialPlanContent: FLAWED_PLAN,
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
});
// The decision must reach the human as prose. 'silent_write' (wrote findings
@@ -65,5 +66,5 @@ describeE2E('Conductor renders decisions as prose (periodic)', () => {
}
// A prose-rendered decision brief was observed at some point in the run.
expect(obs.proseAUQEverObserved).toBe(true);
}, 360_000);
}, CAPTURE_LONG_MS);
});
+17 -16
View File
@@ -12,6 +12,7 @@
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId, evalsEnabled,
@@ -163,7 +164,7 @@ describeIfSelected('Context Skills E2E (live-fire)', [
env: { GSTACK_HOME: gstackHome },
maxTurns: 12,
allowedTools: ['Skill', 'Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'context-save-routing',
runId,
});
@@ -185,7 +186,7 @@ describeIfSelected('Context Skills E2E (live-fire)', [
expect(routedToContextSave).toBe(true);
expect(files.length).toBeGreaterThan(0);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 180_000);
}, CAPTURE_MS);
// ── 2. Round-trip: save then restore in the same session ─────────────
testConcurrentIfSelected('context-save-then-restore-roundtrip', async () => {
@@ -205,7 +206,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 25,
allowedTools: ['Skill', 'Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'context-save-then-restore-roundtrip',
runId,
});
@@ -232,7 +233,7 @@ Do NOT use AskUserQuestion.`,
expect(files.length).toBeGreaterThan(0);
expect(restoreMentionsTitle).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 240_000);
}, CAPTURE_MS);
// ── 3. /context-restore <fragment> loads the matching save ───────────
testConcurrentIfSelected('context-restore-fragment-match', async () => {
@@ -255,7 +256,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 10,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'context-restore-fragment-match',
runId,
});
@@ -279,7 +280,7 @@ Do NOT use AskUserQuestion.`,
expect(loadedPayments).toBe(true);
expect(didNotLoadOthers).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 180_000);
}, CAPTURE_MS);
// ── 4. /context-restore with zero saves → graceful empty-state ───────
testConcurrentIfSelected('context-restore-empty-state', async () => {
@@ -294,7 +295,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 8,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 90_000,
timeout: JUDGE_MS,
testName: 'context-restore-empty-state',
runId,
});
@@ -319,7 +320,7 @@ Do NOT use AskUserQuestion.`,
expect(routedToRestore).toBe(true);
expect(gracefulMessage).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 150_000);
}, CAPTURE_MS);
// ── 5. /context-restore list redirects to /context-save list ─────────
testConcurrentIfSelected('context-restore-list-delegates', async () => {
@@ -334,7 +335,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 8,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 90_000,
timeout: JUDGE_MS,
testName: 'context-restore-list-delegates',
runId,
});
@@ -357,7 +358,7 @@ Do NOT use AskUserQuestion.`,
expect(routedToRestore).toBe(true);
expect(mentionsSaveList).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 150_000);
}, CAPTURE_MS);
// ── 6. Legacy compat: pre-rename save files still load ───────────────
testConcurrentIfSelected('context-restore-legacy-compat', async () => {
@@ -381,7 +382,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 8,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'context-restore-legacy-compat',
runId,
});
@@ -414,7 +415,7 @@ Do NOT use AskUserQuestion.`,
expect(routedToRestore).toBe(true);
expect(loadedLegacy).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 180_000);
}, CAPTURE_MS);
// ── 7. /context-save list: default filters to current branch ─────────
testConcurrentIfSelected('context-save-list-current-branch', async () => {
@@ -437,7 +438,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 10,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'context-save-list-current-branch',
runId,
});
@@ -472,7 +473,7 @@ Do NOT use AskUserQuestion.`,
expect(hidesAlpha).toBe(true);
expect(hidesBeta).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 180_000);
}, CAPTURE_MS);
// ── 8. /context-save list --all: shows every branch ──────────────────
testConcurrentIfSelected('context-save-list-all-branches', async () => {
@@ -494,7 +495,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 10,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'context-save-list-all-branches',
runId,
});
@@ -520,5 +521,5 @@ Do NOT use AskUserQuestion.`,
expect(routed).toBe(true);
expect(filesShown).toBe(3);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 180_000);
}, CAPTURE_MS);
});
+5 -4
View File
@@ -20,6 +20,7 @@
*/
import { test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId,
@@ -78,7 +79,7 @@ Output the diagram directly.`,
workingDirectory: reviewCoverageDir,
maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'review-coverage-audit',
runId,
});
@@ -107,7 +108,7 @@ Output the diagram directly.`,
// At minimum, the agent should have read the source and test files
const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read');
expect(readCalls.length).toBeGreaterThan(0);
}, 180_000);
}, CAPTURE_MS);
});
// --- Plan Eng Review Coverage Audit E2E ---
@@ -153,7 +154,7 @@ Output the diagram directly.`,
workingDirectory: planCoverageDir,
maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'plan-eng-coverage-audit',
runId,
});
@@ -182,7 +183,7 @@ Output the diagram directly.`,
// At minimum, the agent should have read the source and test files
const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read');
expect(readCalls.length).toBeGreaterThan(0);
}, 180_000);
}, CAPTURE_MS);
});
// Module-level afterAll — finalize eval collector after all tests complete
+6 -5
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId, evalsEnabled,
@@ -75,7 +76,7 @@ IMPORTANT:
workingDirectory: csoDir,
maxTurns: 30,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob', 'Agent'],
timeout: 300_000,
timeout: CAPTURE_MS,
});
logCost('cso', result);
@@ -106,7 +107,7 @@ IMPORTANT:
}
recordE2E(evalCollector, 'cso-full-audit', 'e2e-cso', result);
}, 300_000);
}, CAPTURE_MS);
});
describeIfSelected('CSO v2 — diff mode', ['cso-diff-mode'], () => {
@@ -181,7 +182,7 @@ IMPORTANT:
).toBe(true);
recordE2E(evalCollector, 'cso-diff-mode', 'e2e-cso', result);
}, 400_000);
}, CAPTURE_LONG_MS);
});
describeIfSelected('CSO v2 — infra scope', ['cso-infra-scope'], () => {
@@ -245,7 +246,7 @@ IMPORTANT:
workingDirectory: csoInfraDir,
maxTurns: 30,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 360_000,
timeout: CAPTURE_LONG_MS,
});
logCost('cso', result);
@@ -259,5 +260,5 @@ IMPORTANT:
).toBe(true);
recordE2E(evalCollector, 'cso-infra-scope', 'e2e-cso', result);
}, 360_000);
}, CAPTURE_LONG_MS);
});
+13 -12
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, browseBin, runId, evalsEnabled,
@@ -67,7 +68,7 @@ Do NOT use AskUserQuestion. Do NOT run gh or fly commands.`,
workingDirectory: landDir,
maxTurns: 20,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'land-and-deploy-workflow',
runId,
});
@@ -85,7 +86,7 @@ Do NOT use AskUserQuestion. Do NOT run gh or fly commands.`,
const reportDir = path.join(landDir, '.gstack', 'deploy-reports');
expect(fs.existsSync(reportDir)).toBe(true);
}, 180_000);
}, CAPTURE_MS);
});
// --- Land-and-Deploy First-Run E2E ---
@@ -148,7 +149,7 @@ Just demonstrate the first-run dry-run output.`,
workingDirectory: firstRunDir,
maxTurns: 20,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'land-and-deploy-first-run',
runId,
});
@@ -167,7 +168,7 @@ Just demonstrate the first-run dry-run output.`,
const reportContent = fs.readFileSync(path.join(reportDir, reportFiles[0]), 'utf-8');
const hasPlatform = reportContent.toLowerCase().includes('fly') || reportContent.toLowerCase().includes('first-run-app');
expect(hasPlatform).toBe(true);
}, 180_000);
}, CAPTURE_MS);
});
// --- Land-and-Deploy Review Gate E2E ---
@@ -226,7 +227,7 @@ Show what the readiness gate output would look like.`,
workingDirectory: reviewDir,
maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'land-and-deploy-review-gate',
runId,
});
@@ -246,7 +247,7 @@ Show what the readiness gate output would look like.`,
const hasReviewMention = reportContent.toLowerCase().includes('review') ||
reportContent.toLowerCase().includes('not run');
expect(hasReviewMention).toBe(true);
}, 180_000);
}, CAPTURE_MS);
});
// --- Canary skill E2E ---
@@ -294,7 +295,7 @@ Just create the directory structure and report files showing the correct schema.
workingDirectory: canaryDir,
maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'canary-workflow',
runId,
});
@@ -307,7 +308,7 @@ Just create the directory structure and report files showing the correct schema.
const reportDir = path.join(canaryDir, '.gstack', 'canary-reports');
const files = fs.readdirSync(reportDir, { recursive: true }) as string[];
expect(files.length).toBeGreaterThan(0);
}, 180_000);
}, CAPTURE_MS);
});
// --- Benchmark skill E2E ---
@@ -357,7 +358,7 @@ Just create the files showing the correct schema and report format.`,
workingDirectory: benchDir,
maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'benchmark-workflow',
runId,
});
@@ -372,7 +373,7 @@ Just create the files showing the correct schema and report format.`,
const files = fs.readdirSync(baselineDir);
expect(files.length).toBeGreaterThan(0);
}
}, 180_000);
}, CAPTURE_MS);
});
// --- Setup-Deploy skill E2E ---
@@ -418,7 +419,7 @@ Just detect the platform and write the config.`,
workingDirectory: setupDir,
maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'setup-deploy-workflow',
runId,
});
@@ -434,7 +435,7 @@ Just detect the platform and write the config.`,
expect(content.toLowerCase()).toContain('fly');
expect(content).toContain('my-cool-app');
expect(content).toContain('Deploy Configuration');
}, 180_000);
}, CAPTURE_MS);
});
// Module-level afterAll — finalize eval collector after all tests complete
+15 -14
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import { callJudge } from './helpers/llm-judge';
import {
@@ -113,7 +114,7 @@ Skip research — work from your design knowledge. Skip the font preview page. S
Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`,
workingDirectory: designDir,
maxTurns: 20,
timeout: 360_000,
timeout: CAPTURE_LONG_MS,
testName: 'design-consultation-core',
runId,
model: 'claude-opus-4-7',
@@ -178,7 +179,7 @@ Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`,
const claude = fs.readFileSync(claudePath, 'utf-8');
expect(claude.toLowerCase()).toContain('design.md');
}
}, 420_000);
}, CAPTURE_LONG_MS);
testConcurrentIfSelected('design-consultation-research', async () => {
// Test WebSearch integration — research phase only, no DESIGN.md generation
@@ -202,7 +203,7 @@ Do NOT generate a full DESIGN.md — just research notes.`,
// queued past the budget under concurrent API load. 90s budgets cannot
// absorb one slow first completion; 300s is the repo's standard floor
// for CI SDK tests. Outer timeout below rises to 360s for headroom.
timeout: 300_000,
timeout: CAPTURE_MS,
testName: 'design-consultation-research',
runId,
});
@@ -232,7 +233,7 @@ Do NOT generate a full DESIGN.md — just research notes.`,
}
try { fs.rmSync(researchDir, { recursive: true, force: true }); } catch {}
}, 360_000);
}, CAPTURE_LONG_MS);
testConcurrentIfSelected('design-consultation-existing', async () => {
// Pre-create a minimal DESIGN.md (independent of core test)
@@ -250,7 +251,7 @@ There is already a DESIGN.md in this repo. Update it with a complete design syst
Skip research. Skip font preview. Skip any AskUserQuestion calls — this is non-interactive.`,
workingDirectory: designDir,
maxTurns: 20,
timeout: 360_000,
timeout: CAPTURE_LONG_MS,
testName: 'design-consultation-existing',
runId,
model: 'claude-opus-4-7',
@@ -279,7 +280,7 @@ Skip research. Skip font preview. Skip any AskUserQuestion calls — this is non
expect(hasColor).toBe(true);
expect(hasSpacing).toBe(true);
}
}, 420_000);
}, CAPTURE_LONG_MS);
testConcurrentIfSelected('design-consultation-preview', async () => {
// Test preview HTML generation only — no DESIGN.md (covered by core test)
@@ -302,7 +303,7 @@ Do NOT write DESIGN.md — only the preview HTML.`,
maxTurns: 8,
// 300s, not 90s: this is the test that failed 3x at 0 turns/$0.00/93s
// on PR #2533 CI — see the research test's comment for the class.
timeout: 300_000,
timeout: CAPTURE_MS,
testName: 'design-consultation-preview',
runId,
});
@@ -331,7 +332,7 @@ Do NOT write DESIGN.md — only the preview HTML.`,
}
try { fs.rmSync(previewDir, { recursive: true, force: true }); } catch {}
}, 360_000);
}, CAPTURE_LONG_MS);
});
// --- Plan Design Review E2E (plan-mode) ---
@@ -398,7 +399,7 @@ Skip the preamble bash block. Skip any AskUserQuestion calls — this is non-int
IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan review, not a live site audit. Just read the plan file, review it, and edit it to fix the gaps.`,
workingDirectory: reviewDir,
maxTurns: 15,
timeout: 300_000,
timeout: CAPTURE_MS,
testName: 'plan-design-review-plan-mode',
runId,
});
@@ -437,7 +438,7 @@ IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan
} finally {
try { fs.rmSync(reviewDir, { recursive: true, force: true }); } catch {}
}
}, 360_000);
}, CAPTURE_LONG_MS);
testConcurrentIfSelected('plan-design-review-no-ui-scope', async () => {
const reviewDir = setupReviewDir();
@@ -472,7 +473,7 @@ Skip the preamble bash block. Skip any AskUserQuestion calls — this is non-int
IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan review, not a live site audit.`,
workingDirectory: reviewDir,
maxTurns: 10,
timeout: 180_000,
timeout: CAPTURE_MS,
testName: 'plan-design-review-no-ui-scope',
runId,
});
@@ -496,7 +497,7 @@ IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan
} finally {
try { fs.rmSync(reviewDir, { recursive: true, force: true }); } catch {}
}
}, 240_000);
}, CAPTURE_MS);
});
// --- Design Review E2E (live-site audit + fix) ---
@@ -602,7 +603,7 @@ Read design-review/SKILL.md for the design review + fix workflow.
Review the site at ${serverUrl}. Use --quick mode. Skip any AskUserQuestion calls — this is non-interactive. Fix up to 3 issues max. Write your report to ./design-audit.md.`,
workingDirectory: qaDesignDir,
maxTurns: 30,
timeout: 360_000,
timeout: CAPTURE_LONG_MS,
testName: 'design-review-fix',
runId,
});
@@ -634,7 +635,7 @@ Review the site at ${serverUrl}. Use --quick mode. Skip any AskUserQuestion call
console.warn('No design-audit.md generated');
}
console.log(`Design fix commits: ${designFixCommits.length}`);
}, 420_000);
}, CAPTURE_LONG_MS);
});
// Module-level afterAll — finalize eval collector after all tests complete
+5 -4
View File
@@ -17,6 +17,7 @@
* with its preamble.
*/
import { describe, expect } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
@@ -73,7 +74,7 @@ describeIfSelected('/diagram skill E2E', ['diagram-triplet', 'diagram-authoring-
workingDirectory: dir,
maxTurns: 25,
allowedTools: ['Bash', 'Read', 'Write'],
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'diagram-triplet',
runId,
});
@@ -98,7 +99,7 @@ describeIfSelected('/diagram skill E2E', ['diagram-triplet', 'diagram-authoring-
} finally {
try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
}
}, 300_000);
}, CAPTURE_MS);
testConcurrentIfSelected('diagram-authoring-quality', async () => {
const dir = setupDir('diagram-quality-');
@@ -111,7 +112,7 @@ describeIfSelected('/diagram skill E2E', ['diagram-triplet', 'diagram-authoring-
workingDirectory: dir,
maxTurns: 25,
allowedTools: ['Bash', 'Read', 'Write'],
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'diagram-authoring-quality',
runId,
});
@@ -149,5 +150,5 @@ Respond with JSON: {"score": N, "reasoning": "..."}`,
} finally {
try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
}
}, 300_000);
}, CAPTURE_MS);
});
+3 -2
View File
@@ -14,6 +14,7 @@
*/
import { expect, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
@@ -55,7 +56,7 @@ async function detectVia(workDir: string, testName: string): Promise<string> {
workingDirectory: workDir,
maxTurns: 3,
allowedTools: ['Bash'],
timeout: 120_000,
timeout: JUDGE_MS,
testName,
runId,
model: MODEL,
@@ -91,7 +92,7 @@ describeIfSelected('first-run scaffold detection (E2E)', ['first-task-scaffold']
fs.rmSync(nodeDir, { recursive: true, force: true });
fs.rmSync(greenDir, { recursive: true, force: true });
}
}, 300_000);
}, CAPTURE_MS);
});
afterAll(() => finalizeEvalCollector(evalCollector));
@@ -29,6 +29,7 @@
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS } from './helpers/eval-budgets';
import { execFileSync } from 'child_process';
import { mkdtempSync, rmSync } from 'fs';
import { tmpdir } from 'os';
@@ -156,7 +157,7 @@ ${body}`;
expect(retrieved).not.toContain('page_not_found');
expect(retrieved).not.toContain('Page not found');
},
120_000,
JUDGE_MS,
);
},
);
+5 -4
View File
@@ -31,6 +31,7 @@
*/
import { expect, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
@@ -94,7 +95,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic-
workingDirectory: workDir,
maxTurns: 3,
allowedTools: ['Bash'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'hermetic-canary',
runId,
model: CANARY_MODEL,
@@ -129,7 +130,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic-
}
fs.rmSync(workDir, { recursive: true, force: true });
}
}, 180_000);
}, CAPTURE_MS);
testIfSelected('hermetic-sentinel', async () => {
if (!process.env.ANTHROPIC_API_KEY) {
@@ -158,7 +159,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic-
workingDirectory: workDir,
maxTurns: 3,
allowedTools: ['Bash'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'hermetic-sentinel',
runId,
model: CANARY_MODEL,
@@ -188,7 +189,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic-
fs.rmSync(workDir, { recursive: true, force: true });
fs.rmSync(poisonRoot, { recursive: true, force: true });
}
}, 180_000);
}, CAPTURE_MS);
});
afterAll(() => finalizeEvalCollector(evalCollector));
+2 -1
View File
@@ -21,6 +21,7 @@
// intentionally machine-specific.
import { describe, test, expect } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { spawnSync } from 'child_process';
import { cpSync, existsSync, mkdtempSync, readFileSync, rmSync, unlinkSync } from 'fs';
import { tmpdir } from 'os';
@@ -856,7 +857,7 @@ describe('ios device deployment (explicit opt-in)', () => {
keepalive?.stop();
rmSync(workDir, { recursive: true, force: true });
}
}, 600_000);
}, CAPTURE_LONG_MS);
});
// Always-on instructions if not paired. Surfaces actionable steps even when
+4 -3
View File
@@ -18,6 +18,7 @@
// gated (no compilation step for DebugBridgeCore/UI)
import { describe, test, expect } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { spawnSync } from 'child_process';
import { readFileSync } from 'fs';
import { join } from 'path';
@@ -321,7 +322,7 @@ describeIfSwift('swift build invariants', () => {
console.error('swift build stderr:', r.stderr?.toString().slice(0, 4000));
}
expect(r.status).toBe(0);
}, 180_000);
}, CAPTURE_MS);
test('XCTest suite for StateServer passes (validates real Swift impl)', () => {
const r = spawnSync('swift', ['test', '--filter', 'DebugBridgeCoreTests'], {
@@ -342,7 +343,7 @@ describeIfSwift('swift build invariants', () => {
// Guard against an empty pass-by-no-tests (filter typo / target rename):
// we expect at least one StateServer smoke test to actually execute.
expect(combined).toContain('StateServerSmokeTests');
}, 240_000);
}, CAPTURE_MS);
// Codex-flagged: Release-build guard must be STRUCTURAL, not advisory.
// The Package.swift's `.when(configuration: .debug)` setting causes Swift
@@ -386,5 +387,5 @@ describeIfSwift('swift build invariants', () => {
}
}
expect(foundForbidden).toBe(0);
}, 300_000);
}, CAPTURE_MS);
});
+3 -2
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId, evalsEnabled,
@@ -103,7 +104,7 @@ IMPORTANT:
workingDirectory: workDir,
maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'learnings-show',
runId,
});
@@ -134,5 +135,5 @@ IMPORTANT:
} else {
console.warn(`Only ${foundCount}/3 learnings found (N+1: ${mentionsNPlusOne}, cache: ${mentionsCache}, rubocop: ${mentionsRubocop})`);
}
}, 180_000);
}, CAPTURE_MS);
});
@@ -17,6 +17,7 @@
*/
import { test, expect } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillObservation, planFileHasDecisionsSection } from './helpers/claude-pty-runner';
@@ -30,7 +31,7 @@ describeE2E('office-hours AskUserQuestion-blocked smoke (gate)', () => {
skillName: 'office-hours',
inPlanMode: true,
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
});
if (
@@ -55,5 +56,5 @@ describeE2E('office-hours AskUserQuestion-blocked smoke (gate)', () => {
}
}
expect(['asked', 'plan_ready']).toContain(obs.outcome);
}, 360_000);
}, CAPTURE_LONG_MS);
});
@@ -36,6 +36,7 @@
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { execFileSync, spawnSync } from 'child_process';
import {
chmodSync,
@@ -222,7 +223,7 @@ Generate the design doc per Phase 5. The feature-slug value to substitute into t
This is a test of the brain-writeback path. Do NOT skip the gbrain save step under any circumstance — the runtime guard ("skip if gbrain not on PATH") does NOT apply here because gbrain IS available. Do NOT explore gbrain --help; follow the SAVE_RESULTS template's exact CLI shape. If you encounter any AskUserQuestion, auto-decide recommended.`,
workingDirectory: workDir,
maxTurns: 12,
timeout: 360_000,
timeout: CAPTURE_LONG_MS,
testName: 'office-hours-brain-writeback',
runId,
model: 'claude-sonnet-4-6',
@@ -315,7 +316,7 @@ This is a test of the brain-writeback path. Do NOT skip the gbrain save step und
);
}
},
420_000,
CAPTURE_LONG_MS,
);
},
);
+3 -2
View File
@@ -20,6 +20,7 @@
* test turns out stable.
*/
import { expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId,
@@ -133,7 +134,7 @@ ${captureInstruction(outFile)}
After writing the file with that ONE Phase 4 question, stop. Do not continue to Phase 4.5 or Phase 5.`,
workingDirectory: workDir,
maxTurns: 12,
timeout: 300_000,
timeout: CAPTURE_MS,
testName: 'office-hours-phase4-fork',
runId,
model: 'claude-opus-4-7',
@@ -162,7 +163,7 @@ After writing the file with that ONE Phase 4 question, stop. Do not continue to
result,
passed: ['success', 'error_max_turns'].includes(result.exitReason),
});
}, 360_000);
}, CAPTURE_LONG_MS);
});
afterAll(async () => {
+5 -4
View File
@@ -10,6 +10,7 @@
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, browseBin, runId, evalsEnabled,
@@ -71,7 +72,7 @@ Assume the founder has already answered Q1 (strongest evidence = "got on a waitl
Write Q3 output the forcing question you would ask this founder to ${workDir}/q3.md. Write ONLY the question prose. No conversational wrapper, no meta-commentary, no Q1/Q2 recap.`,
workingDirectory: workDir,
maxTurns: 8,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'office-hours-forcing-energy',
runId,
model: 'claude-sonnet-4-6',
@@ -94,7 +95,7 @@ Write Q3 output — the forcing question you would ask this founder — to ${wor
console.log('Forcing energy scores:', JSON.stringify(scores, null, 2));
expect(scores.axis_a).toBeGreaterThanOrEqual(4); // stacking_preserved
expect(scores.axis_b).toBeGreaterThanOrEqual(4); // domain_matched_consequence
}, 360_000);
}, CAPTURE_LONG_MS);
});
// --- Office Hours builder-mode wildness ---
@@ -143,7 +144,7 @@ The user has confirmed the basic idea is "TypeScript + D3 web tool, start with J
Write your response the three adjacent unlocks to ${workDir}/unlocks.md. Write ONLY the response prose. No meta-commentary, no mode recap. Lead with the fun; let me edit it down later.`,
workingDirectory: workDir,
maxTurns: 8,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'office-hours-builder-wildness',
runId,
model: 'claude-sonnet-4-6',
@@ -166,7 +167,7 @@ Write your response — the three adjacent unlocks — to ${workDir}/unlocks.md.
console.log('Builder wildness scores:', JSON.stringify(scores, null, 2));
expect(scores.axis_a).toBeGreaterThanOrEqual(4); // unexpected_combinations
expect(scores.axis_b).toBeGreaterThanOrEqual(4); // excitement_over_optimization
}, 360_000);
}, CAPTURE_LONG_MS);
});
// Finalize eval collector for this file
+6 -5
View File
@@ -18,6 +18,7 @@
*/
import { describe, test, expect, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import { EvalCollector } from './helpers/eval-store';
import { extractSkillHead } from './helpers/skill-fixture';
@@ -204,7 +205,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
workingDirectory: armA,
maxTurns: 5,
allowedTools: ['Read', 'Bash', 'Glob', 'Grep'],
timeout: 90_000,
timeout: JUDGE_MS,
testName: 'fanout-arm-overlay-on',
runId,
model: OPUS_47,
@@ -214,7 +215,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
workingDirectory: armB,
maxTurns: 5,
allowedTools: ['Read', 'Bash', 'Glob', 'Grep'],
timeout: 90_000,
timeout: JUDGE_MS,
testName: 'fanout-arm-overlay-off',
runId,
model: OPUS_47,
@@ -262,7 +263,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
fs.rmSync(armB, { recursive: true, force: true });
}
},
240_000,
CAPTURE_MS,
);
test(
@@ -281,7 +282,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
workingDirectory: root,
maxTurns: 3,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 90_000,
timeout: JUDGE_MS,
testName: `routing-${c.name}`,
runId,
model: OPUS_47,
@@ -348,6 +349,6 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
fs.rmSync(root, { recursive: true, force: true });
}
},
360_000,
CAPTURE_LONG_MS,
);
});
@@ -5,6 +5,7 @@
*/
import { test } from 'bun:test';
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
import { FORCING_FLOOR_CEO } from './fixtures/forcing-finding-seeds';
@@ -22,7 +23,7 @@ describeE2E('/plan-ceo-review AskUserQuestion floor (gate)', () => {
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 600_000,
timeoutMs: CAPTURE_LONG_MS,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
@@ -34,6 +35,6 @@ describeE2E('/plan-ceo-review AskUserQuestion floor (gate)', () => {
);
}
},
660_000,
PTY_MS,
);
});
+3 -2
View File
@@ -31,6 +31,7 @@
*/
import { test } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import {
launchClaudePty,
@@ -151,7 +152,7 @@ describeE2E('/plan-ceo-review mode routing (gate)', () => {
async () => {
const session = await launchClaudePty({
permissionMode: 'plan',
timeoutMs: 540_000,
timeoutMs: CAPTURE_LONG_MS,
seedSkills: true,
});
try {
@@ -207,7 +208,7 @@ describeE2E('/plan-ceo-review mode routing (gate)', () => {
await session.close();
}
},
600_000,
CAPTURE_LONG_MS,
);
}
});
+2 -1
View File
@@ -34,6 +34,7 @@
*/
import { test } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import {
runPlanSkillObservation,
@@ -77,5 +78,5 @@ describeE2E('plan-ceo-review plan-mode smoke (gate)', () => {
);
}
assertReportAtBottomIfPlanWritten(obs);
}, 480_000);
}, CAPTURE_LONG_MS);
});
@@ -25,6 +25,7 @@
*/
import { test, expect } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import {
setupSkillDir,
@@ -87,6 +88,6 @@ describeE2E('/plan-ceo-review section-loading E2E (periodic, SDK capture)', () =
// Guard against an empty pass: the report must have real content.
expect(output.trim().length).toBeGreaterThan(200);
},
360_000,
CAPTURE_LONG_MS,
);
});
@@ -5,6 +5,7 @@
*/
import { test } from 'bun:test';
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
import { FORCING_FLOOR_DESIGN } from './fixtures/forcing-finding-seeds';
@@ -22,7 +23,7 @@ describeE2E('/plan-design-review AskUserQuestion floor (periodic)', () => {
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 600_000,
timeoutMs: CAPTURE_LONG_MS,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
@@ -34,6 +35,6 @@ describeE2E('/plan-design-review AskUserQuestion floor (periodic)', () => {
);
}
},
660_000,
PTY_MS,
);
});
+5 -4
View File
@@ -10,6 +10,7 @@
*/
import { test, expect } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import {
runPlanSkillObservation,
@@ -46,7 +47,7 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => {
const obs = await runPlanSkillObservation({
skillName: 'plan-design-review',
inPlanMode: true,
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
});
if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') {
@@ -59,7 +60,7 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => {
}
expect(['asked', 'plan_ready']).toContain(obs.outcome);
assertReportAtBottomIfPlanWritten(obs);
}, 360_000);
}, CAPTURE_LONG_MS);
// Plan-mode scope-gate bypass: with a seeded UI-heavy plan in plan mode,
// the gate must NOT render its "What should I review?" menu — it
@@ -71,7 +72,7 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => {
skillName: 'plan-design-review',
inPlanMode: true,
initialPlanContent: SEED_PLAN_UI_HEAVY,
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
});
if (
@@ -95,5 +96,5 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => {
// though the seed arrives as a pasted user message).
expect(obs.scopeGateQuestionObserved ?? false).toBe(false);
expect(obs.scopeGateAutoSelectObserved ?? false).toBe(true);
}, 360_000);
}, CAPTURE_LONG_MS);
});
+3 -2
View File
@@ -20,6 +20,7 @@
*/
import { test } from 'bun:test';
import { PTY_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import * as path from 'path';
import {
@@ -46,7 +47,7 @@ describeE2E('/plan-design-review with UI scope (gate)', () => {
// LIVE-REPO CWD: PTY session needs the repo cwd — skill registry,
// hermetic pre-trusted dir, and the repo-relative fixture path above.
cwd: ROOT,
timeoutMs: 720_000,
timeoutMs: PTY_MS,
seedSkills: true,
});
@@ -152,6 +153,6 @@ describeE2E('/plan-design-review with UI scope (gate)', () => {
);
}
},
780_000,
PTY_MS,
);
});
@@ -5,6 +5,7 @@
*/
import { test } from 'bun:test';
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
import { FORCING_FLOOR_DEVEX } from './fixtures/forcing-finding-seeds';
@@ -22,7 +23,7 @@ describeE2E('/plan-devex-review AskUserQuestion floor (gate)', () => {
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 600_000,
timeoutMs: CAPTURE_LONG_MS,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
@@ -34,6 +35,6 @@ describeE2E('/plan-devex-review AskUserQuestion floor (gate)', () => {
);
}
},
660_000,
PTY_MS,
);
});
+5 -4
View File
@@ -6,6 +6,7 @@
*/
import { test, expect } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import {
runPlanSkillObservation,
@@ -20,7 +21,7 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => {
const obs = await runPlanSkillObservation({
skillName: 'plan-devex-review',
inPlanMode: true,
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
});
if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') {
@@ -33,7 +34,7 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => {
}
expect(['asked', 'plan_ready']).toContain(obs.outcome);
assertReportAtBottomIfPlanWritten(obs);
}, 360_000);
}, CAPTURE_LONG_MS);
// v1.21+ regression: see skill-e2e-plan-ceo-plan-mode.test.ts for the
// contract. Pass envelope is ['asked', 'plan_ready']; failure signals
@@ -44,7 +45,7 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => {
skillName: 'plan-devex-review',
inPlanMode: true,
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
});
if (
@@ -70,5 +71,5 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => {
}
expect(['asked', 'plan_ready']).toContain(obs.outcome);
assertReportAtBottomIfPlanWritten(obs);
}, 360_000);
}, CAPTURE_LONG_MS);
});
@@ -16,6 +16,7 @@
*/
import { test } from 'bun:test';
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
import { FORCING_FLOOR_ENG } from './fixtures/forcing-finding-seeds';
@@ -33,7 +34,7 @@ describeE2E('/plan-eng-review AskUserQuestion floor (periodic)', () => {
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 600_000,
timeoutMs: CAPTURE_LONG_MS,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
@@ -49,6 +50,6 @@ describeE2E('/plan-eng-review AskUserQuestion floor (periodic)', () => {
);
}
},
660_000,
PTY_MS,
);
});
+5 -4
View File
@@ -6,6 +6,7 @@
*/
import { test, expect } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import {
runPlanSkillObservation,
@@ -50,7 +51,7 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => {
const obs = await runPlanSkillObservation({
skillName: 'plan-eng-review',
inPlanMode: true,
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
});
if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') {
@@ -63,7 +64,7 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => {
}
expect(['asked', 'plan_ready']).toContain(obs.outcome);
assertReportAtBottomIfPlanWritten(obs);
}, 360_000);
}, CAPTURE_LONG_MS);
// D3-B / D4-B: when a plan with guaranteed-finding-triggering complexity
// is seeded, the skill MUST fire AskUserQuestion (or fall back to a
@@ -79,7 +80,7 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => {
// must use mcp__*__AskUserQuestion (outcome='asked') or fall back to
// writing Decisions ('plan_ready').
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
});
if (
@@ -118,5 +119,5 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => {
// question.
expect(obs.scopeGateQuestionObserved ?? false).toBe(false);
expect(obs.scopeGateAutoSelectObserved ?? false).toBe(true);
}, 360_000);
}, CAPTURE_LONG_MS);
});
+9 -8
View File
@@ -18,6 +18,7 @@
* accordingly.
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId,
@@ -134,7 +135,7 @@ ${captureInstruction(outFile)}
After writing the file, stop. Do not continue the review.`,
workingDirectory: planDir,
maxTurns: 10,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'plan-ceo-review-format-mode',
runId,
model: 'claude-opus-4-7',
@@ -160,7 +161,7 @@ After writing the file, stop. Do not continue the review.`,
result,
passed: ['success', 'error_max_turns'].includes(result.exitReason),
});
}, 300_000);
}, CAPTURE_MS);
});
// --- Case 2: plan-ceo-review approach menu (coverage-differentiated) ---
@@ -191,7 +192,7 @@ ${captureInstruction(outFile)}
After writing the file, stop. Do not continue the review.`,
workingDirectory: planDir,
maxTurns: 10,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'plan-ceo-review-format-approach',
runId,
model: 'claude-opus-4-7',
@@ -216,7 +217,7 @@ After writing the file, stop. Do not continue the review.`,
result,
passed: ['success', 'error_max_turns'].includes(result.exitReason),
});
}, 300_000);
}, CAPTURE_MS);
});
// --- Case 3: plan-eng-review coverage-differentiated per-issue AskUserQuestion ---
@@ -250,7 +251,7 @@ ${captureInstruction(outFile)}
After writing the file with that ONE question, stop. Do not continue the review.`,
workingDirectory: planDir,
maxTurns: 10,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'plan-eng-review-format-coverage',
runId,
model: 'claude-opus-4-7',
@@ -275,7 +276,7 @@ After writing the file with that ONE question, stop. Do not continue the review.
result,
passed: ['success', 'error_max_turns'].includes(result.exitReason),
});
}, 300_000);
}, CAPTURE_MS);
});
// --- Case 4: plan-eng-review kind-differentiated per-issue AskUserQuestion ---
@@ -306,7 +307,7 @@ ${captureInstruction(outFile)}
After writing the file with that ONE question, stop. Do not continue the review.`,
workingDirectory: planDir,
maxTurns: 10,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'plan-eng-review-format-kind',
runId,
model: 'claude-opus-4-7',
@@ -332,7 +333,7 @@ After writing the file with that ONE question, stop. Do not continue the review.
result,
passed: ['success', 'error_max_turns'].includes(result.exitReason),
});
}, 300_000);
}, CAPTURE_MS);
});
afterAll(async () => {
+5 -4
View File
@@ -31,6 +31,7 @@
*/
import { test, expect } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillObservation } from './helpers/claude-pty-runner';
@@ -62,7 +63,7 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
const obs = await runPlanSkillObservation({
skillName,
inPlanMode: false,
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
// eng/design: force the prose-fallback path. The unconditional
// gate-must-ask assert below pins the render shape the detector
// anchors on, and only the --disallowedTools prose fallback makes
@@ -115,7 +116,7 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
);
}
}
}, 360_000);
}, CAPTURE_LONG_MS);
}
// Named-target exception (outside plan mode): a pasted draft IS an
@@ -130,7 +131,7 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
inPlanMode: false,
initialPlanContent: NAMED_TARGET_SEED,
trackTokens: [SEED_TOKEN],
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
});
if (
@@ -159,5 +160,5 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
// unreachable outside plan mode (extractPlanFilePath only matches
// plan-mode save renders).
expect(obs.tokensObserved?.[SEED_TOKEN] ?? false).toBe(true);
}, 360_000);
}, CAPTURE_LONG_MS);
});
+9 -8
View File
@@ -27,6 +27,7 @@
* cases will land as follow-up PRs per skill.
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId,
@@ -161,7 +162,7 @@ ${captureInstruction(outFile)}
After writing the file, stop.`,
workingDirectory: planDir,
maxTurns: 10,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'plan-review-prosons-format',
runId,
model: 'claude-opus-4-7',
@@ -191,7 +192,7 @@ After writing the file, stop.`,
// (recommended) label on one option
expect(captured).toMatch(RECOMMENDED_LABEL_RE);
}, 300_000);
}, CAPTURE_MS);
});
// --- Case 2: Hard-stop escape NEGATIVE (CT2) ---
@@ -220,7 +221,7 @@ ${captureInstruction(outFile)}
After writing the file, stop.`,
workingDirectory: planDir,
maxTurns: 10,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'plan-review-prosons-hardstop-neg',
runId,
model: 'claude-opus-4-7',
@@ -241,7 +242,7 @@ After writing the file, stop.`,
// Must have real pros and cons (≥2 ✅ + ≥1 ❌ per option)
expect(countChars(captured, '✅')).toBeGreaterThanOrEqual(4);
expect(countChars(captured, '❌')).toBeGreaterThanOrEqual(2);
}, 300_000);
}, CAPTURE_MS);
});
// --- Case 3: Neutral-posture NEGATIVE (CT2) ---
@@ -270,7 +271,7 @@ ${captureInstruction(outFile)}
After writing the file, stop.`,
workingDirectory: planDir,
maxTurns: 10,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'plan-review-prosons-neutral-neg',
runId,
model: 'claude-opus-4-7',
@@ -292,7 +293,7 @@ After writing the file, stop.`,
expect(captured).toMatch(RECOMMENDED_LABEL_RE);
// Recommendation line must contain "because" (concrete reason, not "no preference")
expect(captured).toMatch(/[Rr]ecommendation:.*because/);
}, 300_000);
}, CAPTURE_MS);
});
// --- Case 4: Hard-stop POSITIVE (escape allowed when legitimately one-sided) ---
@@ -321,7 +322,7 @@ ${captureInstruction(outFile)}
After writing the file, stop.`,
workingDirectory: planDir,
maxTurns: 10,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'plan-ceo-review-prosons-cadence',
runId,
model: 'claude-opus-4-7',
@@ -344,7 +345,7 @@ After writing the file, stop.`,
const hasEscape = HARD_STOP_ESCAPE_RE.test(captured);
const hasProsAndCons = countChars(captured, '✅') >= 1 && countChars(captured, '❌') >= 1;
expect(hasEscape || hasProsAndCons).toBe(true);
}, 300_000);
}, CAPTURE_MS);
});
afterAll(async () => {
+3 -2
View File
@@ -1,4 +1,5 @@
import { beforeAll, afterAll, expect } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId,
@@ -151,7 +152,7 @@ IMPORTANT:
workingDirectory: workDir,
maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'plan-tune-inspect',
runId,
});
@@ -184,5 +185,5 @@ IMPORTANT:
if (!noticedOverride) {
console.warn('Agent did not surface override/skip behavior from the log');
}
}, 180_000);
}, CAPTURE_MS);
});
+21 -20
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, browseBin, runId, evalsEnabled,
@@ -103,7 +104,7 @@ Focus on reviewing the plan content: architecture, error handling, security, and
const review = fs.readFileSync(reviewPath, 'utf-8');
expect(review.length).toBeGreaterThan(200);
}
}, 660_000);
}, PTY_MS);
});
// --- Plan CEO Review (SELECTIVE EXPANSION) E2E ---
@@ -171,7 +172,7 @@ Write your complete review directly to ${planDir}/review-output-selective.md
Focus on reviewing the plan content: architecture, error handling, security, and performance.`,
workingDirectory: planDir,
maxTurns: 15,
timeout: 540_000,
timeout: CAPTURE_LONG_MS,
testName: 'plan-ceo-review-selective',
runId,
model: 'claude-opus-4-7',
@@ -188,7 +189,7 @@ Focus on reviewing the plan content: architecture, error handling, security, and
const review = fs.readFileSync(reviewPath, 'utf-8');
expect(review.length).toBeGreaterThan(200);
}
}, 660_000);
}, PTY_MS);
});
// --- Plan CEO Review SCOPE EXPANSION energy (V1.1 mode-posture regression gate) ---
@@ -239,7 +240,7 @@ Choose SCOPE EXPANSION mode. Skip any AskUserQuestion calls — this is non-inte
Write your expansion proposals to ${planDir}/proposals.md with ONLY the proposal text no conversational wrapper, no review summary, no mode analysis. Each proposal separated by "---".`,
workingDirectory: planDir,
maxTurns: 15,
timeout: 360_000,
timeout: CAPTURE_LONG_MS,
testName: 'plan-ceo-review-expansion-energy',
runId,
model: 'claude-opus-4-7',
@@ -270,7 +271,7 @@ Write your expansion proposals to ${planDir}/proposals.md with ONLY the proposal
// Pass threshold: 4/5 on both axes (good — matches posture with minor weakness).
expect(scores.axis_a).toBeGreaterThanOrEqual(4); // surface_framing
expect(scores.axis_b).toBeGreaterThanOrEqual(4); // decision_preservation
}, 600_000);
}, CAPTURE_LONG_MS);
});
// --- Plan Eng Review E2E ---
@@ -348,7 +349,7 @@ Write your complete review directly to ${planDir}/review-output.md
Focus on architecture, code quality, tests, and performance sections.`,
workingDirectory: planDir,
maxTurns: 15,
timeout: 360_000,
timeout: CAPTURE_LONG_MS,
testName: 'plan-eng-review',
runId,
model: 'claude-opus-4-7',
@@ -366,7 +367,7 @@ Focus on architecture, code quality, tests, and performance sections.`,
const review = fs.readFileSync(reviewPath, 'utf-8');
expect(review.length).toBeGreaterThan(200);
}
}, 420_000);
}, CAPTURE_LONG_MS);
});
// --- Plan-Eng-Review Test-Plan Artifact E2E ---
@@ -476,7 +477,7 @@ Write your review to ${planDir}/review-output.md`,
workingDirectory: planDir,
maxTurns: 25,
allowedTools: ['Bash', 'Read', 'Write', 'Glob', 'Grep'],
timeout: 360_000,
timeout: CAPTURE_LONG_MS,
testName: 'plan-eng-review-artifact',
runId,
model: 'claude-opus-4-7',
@@ -507,7 +508,7 @@ Write your review to ${planDir}/review-output.md`,
if (newFiles.length === 0) {
console.warn('SOFT FAIL: No test-plan artifact written — agent did not follow artifact instructions');
}
}, 420_000);
}, CAPTURE_LONG_MS);
});
// --- Office Hours Spec Review E2E ---
@@ -559,7 +560,7 @@ Write your summary to ${ohDir}/spec-review-summary.md`,
// failures wrote a correct summary on tool-turn 8 and hit the cap on
// the closing text turn (error_max_turns at 9 turns, deterministic).
maxTurns: 12,
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'office-hours-spec-review',
runId,
});
@@ -575,7 +576,7 @@ Write your summary to ${ohDir}/spec-review-summary.md`,
expect(summary).toMatch(/agent|subagent/);
expect(summary).toMatch(/3.*iteration|iteration.*3|maximum.*3/);
}
}, 180_000);
}, CAPTURE_MS);
});
// --- Plan CEO Review Benefits-From E2E ---
@@ -619,7 +620,7 @@ Summarize what happens when no design doc is found — specifically:
Write your summary to ${benefitsDir}/benefits-summary.md`,
workingDirectory: benefitsDir,
maxTurns: 8,
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'plan-ceo-review-benefits',
runId,
});
@@ -634,7 +635,7 @@ Write your summary to ${benefitsDir}/benefits-summary.md`,
expect(summary).toMatch(/office.hours/);
expect(summary).toMatch(/design doc|no design/i);
}
}, 180_000);
}, CAPTURE_MS);
});
// --- Plan Review Report E2E ---
@@ -706,7 +707,7 @@ CRITICAL REQUIREMENT: plan.md IS the plan file for this review session. After co
This review report at the bottom of the plan is the MOST IMPORTANT deliverable of this test.`,
workingDirectory: planDir,
maxTurns: 20,
timeout: 360_000,
timeout: CAPTURE_LONG_MS,
testName: 'plan-review-report',
runId,
model: 'claude-opus-4-7',
@@ -768,7 +769,7 @@ This review report at the bottom of the plan is the MOST IMPORTANT deliverable o
).toBe(true);
console.log('Plan review report found at bottom of plan.md (ends with unresolved status)');
}, 420_000);
}, CAPTURE_LONG_MS);
});
// --- Codex Offering E2E ---
@@ -825,7 +826,7 @@ Summarize the Codex/${featureName} integration — answer these specific questio
Write your summary to ${testDir}/${testName}-summary.md`,
workingDirectory: testDir,
maxTurns: 8,
timeout: 120_000,
timeout: JUDGE_MS,
testName,
runId,
});
@@ -850,19 +851,19 @@ Write your summary to ${testDir}/${testName}-summary.md`,
testConcurrentIfSelected('codex-offered-office-hours', async () => {
await checkCodexOffering('office-hours', 'codex-offered-office-hours', 'second opinion');
}, 180_000);
}, CAPTURE_MS);
testConcurrentIfSelected('codex-offered-ceo-review', async () => {
await checkCodexOffering('plan-ceo-review', 'codex-offered-ceo-review', 'outside voice');
}, 180_000);
}, CAPTURE_MS);
testConcurrentIfSelected('codex-offered-design-review', async () => {
await checkCodexOffering('plan-design-review', 'codex-offered-design-review', 'design outside voices');
}, 180_000);
}, CAPTURE_MS);
testConcurrentIfSelected('codex-offered-eng-review', async () => {
await checkCodexOffering('plan-eng-review', 'codex-offered-eng-review', 'outside voice');
}, 180_000);
}, CAPTURE_MS);
});
// Module-level afterAll — finalize eval collector after all tests complete
+5 -4
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import { outcomeJudge } from './helpers/llm-judge';
import { judgePassed } from './helpers/eval-store';
@@ -97,7 +98,7 @@ CRITICAL RULES:
- The report MUST exist at ${reportPath} when you finish`,
workingDirectory: testWorkDir,
maxTurns: 50,
timeout: 300_000,
timeout: CAPTURE_MS,
testName: `qa-${label}`,
runId,
model: 'claude-opus-4-7',
@@ -174,17 +175,17 @@ CRITICAL RULES:
// B6: Static dashboard — broken link, disabled submit, overflow, missing alt, console error
testConcurrentIfSelected('qa-b6-static', async () => {
await runPlantedBugEval('qa-eval.html', 'qa-eval-ground-truth.json', 'b6-static');
}, 360_000);
}, CAPTURE_LONG_MS);
// B7: SPA — broken route, stale state, async race, missing aria, console warning
testConcurrentIfSelected('qa-b7-spa', async () => {
await runPlantedBugEval('qa-eval-spa.html', 'qa-eval-spa-ground-truth.json', 'b7-spa');
}, 360_000);
}, CAPTURE_LONG_MS);
// B8: Checkout — email regex, NaN total, CC overflow, missing required, stripe error
testConcurrentIfSelected('qa-b8-checkout', async () => {
await runPlantedBugEval('qa-eval-checkout.html', 'qa-eval-checkout-ground-truth.json', 'b8-checkout');
}, 360_000);
}, CAPTURE_LONG_MS);
});
+9 -8
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, browseBin, runId, evalsEnabled,
@@ -54,7 +55,7 @@ Do NOT try to start a server or discover ports — the URL above is ready.
Write your report to ${qaDir}/qa-reports/qa-report.md`,
workingDirectory: qaDir,
maxTurns: 35,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'qa-quick',
runId,
});
@@ -69,7 +70,7 @@ Write your report to ${qaDir}/qa-reports/qa-report.md`,
}
// Accept error_max_turns — the agent doing thorough QA work is not a failure
expect(['success', 'error_max_turns']).toContain(result.exitReason);
}, 300_000);
}, CAPTURE_MS);
});
// --- QA-Only E2E (report-only, no fixes) ---
@@ -124,7 +125,7 @@ Write your report to ${qaOnlyDir}/qa-reports/qa-only-report.md`,
workingDirectory: qaOnlyDir,
maxTurns: 40,
allowedTools: ['Bash', 'Read', 'Write', 'Glob'], // NO Edit — the critical guardrail
timeout: 180_000,
timeout: CAPTURE_MS,
testName: 'qa-only-no-fix',
runId,
});
@@ -156,7 +157,7 @@ Write your report to ${qaOnlyDir}/qa-reports/qa-only-report.md`,
(l: string) => l.trim() && !l.includes('.prompt-tmp') && !l.includes('.gstack/') && !l.includes('qa-reports/'),
);
expect(statusLines.filter((l: string) => l.startsWith(' M') || l.startsWith('M '))).toHaveLength(0);
}, 240_000);
}, CAPTURE_MS);
});
// --- QA Fix Loop E2E ---
@@ -247,7 +248,7 @@ This is a test+fix loop: find bugs, fix them in the source code, commit each fix
workingDirectory: qaFixDir,
maxTurns: 40,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
timeout: 420_000,
timeout: CAPTURE_LONG_MS,
testName: 'qa-fix-loop',
runId,
});
@@ -271,7 +272,7 @@ This is a test+fix loop: find bugs, fix them in the source code, commit each fix
// Verify Edit tool was used (agent actually modified source code)
const editCalls = result.toolCalls.filter(tc => tc.tool === 'Edit');
expect(editCalls.length).toBeGreaterThan(0);
}, 480_000);
}, CAPTURE_LONG_MS);
});
// --- Test Bootstrap E2E ---
@@ -384,7 +385,7 @@ Do NOT fix any bugs. Do NOT use AskUserQuestion — just pick vitest.`,
workingDirectory: bsDir,
maxTurns: 12,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'],
timeout: 90_000,
timeout: JUDGE_MS,
testName: 'qa-bootstrap',
runId,
});
@@ -405,7 +406,7 @@ Do NOT fix any bugs. Do NOT use AskUserQuestion — just pick vitest.`,
console.log(`Test config: ${hasTestConfig}, Test file: ${hasTestFile}, TESTING.md: ${hasTestingMd}`);
try { fs.rmSync(bsDir, { recursive: true, force: true }); } catch {}
}, 120_000);
}, JUDGE_MS);
});
// Module-level afterAll — finalize eval collector after all tests complete.
+4 -3
View File
@@ -1,4 +1,5 @@
import { expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId,
@@ -135,7 +136,7 @@ Write your retrospective to ${dir}/retro-output.md`,
expect(wroteReport).toBe(true);
const content = fs.readFileSync(retroPath, 'utf-8');
expect(content.length).toBeGreaterThan(100);
}, 480_000);
}, CAPTURE_LONG_MS);
});
// --- Retro E2E ---
@@ -198,7 +199,7 @@ Write your retrospective report to ${retroDir}/retro-output.md
Analyze the git history and produce the narrative report as described in the SKILL.md.`,
workingDirectory: retroDir,
maxTurns: 30,
timeout: 300_000,
timeout: CAPTURE_MS,
testName: 'retro',
runId,
model: 'claude-opus-4-7',
@@ -217,7 +218,7 @@ Analyze the git history and produce the narrative report as described in the SKI
expect(wroteReport).toBe(true);
const retro = fs.readFileSync(retroPath, 'utf-8');
expect(retro.length).toBeGreaterThan(100);
}, 420_000);
}, CAPTURE_LONG_MS);
});
// Module-level afterAll — finalize eval collector after all tests complete
+15 -14
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId, describeIfSelected, testConcurrentIfSelected,
@@ -114,7 +115,7 @@ and apply it yourself against the diff (git diff main...HEAD).
Write your findings to ${dir}/review-output.md`,
workingDirectory: dir,
maxTurns: 20,
timeout: 180_000,
timeout: CAPTURE_MS,
testName: 'review-army-migration-safety',
runId,
});
@@ -135,7 +136,7 @@ Write your findings to ${dir}/review-output.md`,
content.includes('column');
expect(hasMigrationFinding).toBe(true);
}
}, 210_000);
}, CAPTURE_MS);
});
// --- Review Army: N+1 Performance ---
@@ -179,7 +180,7 @@ For the specialist dispatch, read review-specialists/performance.md and apply it
Write your findings to ${dir}/review-output.md`,
workingDirectory: dir,
maxTurns: 20,
timeout: 180_000,
timeout: CAPTURE_MS,
testName: 'review-army-perf-n-plus-one',
runId,
});
@@ -201,7 +202,7 @@ Write your findings to ${dir}/review-output.md`,
content.includes('loop');
expect(hasN1Finding).toBe(true);
}
}, 210_000);
}, CAPTURE_MS);
});
// --- Review Army: Delivery Audit ---
@@ -281,7 +282,7 @@ The email notification system should be classified as NOT DONE.
Write your completion audit to ${dir}/review-output.md`,
workingDirectory: dir,
maxTurns: 15,
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'review-army-delivery-audit',
runId,
});
@@ -305,7 +306,7 @@ Write your completion audit to ${dir}/review-output.md`,
expect(hasNotDone).toBe(true);
expect(mentionsEmail).toBe(true);
}
}, 150_000);
}, CAPTURE_MS);
});
// --- Review Army: Quality Score ---
@@ -356,7 +357,7 @@ Write your findings AND the computed quality score to ${dir}/review-output.md
Include the line: "PR Quality Score: X/10" where X is the computed score.`,
workingDirectory: dir,
maxTurns: 15,
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'review-army-quality-score',
runId,
});
@@ -374,7 +375,7 @@ Include the line: "PR Quality Score: X/10" where X is the computed score.`,
content.match(/\d+\/10/);
expect(hasScore).toBeTruthy();
}
}, 150_000);
}, CAPTURE_MS);
});
// --- Review Army: JSON Findings ---
@@ -421,7 +422,7 @@ Output your findings as JSON objects, one per line, following the schema:
Write ONLY JSON findings (no preamble) to ${dir}/findings.json`,
workingDirectory: dir,
maxTurns: 12,
timeout: 90_000,
timeout: JUDGE_MS,
testName: 'review-army-json-findings',
runId,
});
@@ -450,7 +451,7 @@ Write ONLY JSON findings (no preamble) to ${dir}/findings.json`,
break; // One valid line is enough for the gate test
}
}
}, 120_000);
}, JUDGE_MS);
});
// --- Review Army: Red Team (periodic) ---
@@ -499,7 +500,7 @@ Write your red team findings to ${dir}/review-output.md
Start the file with "RED TEAM REVIEW" on the first line.`,
workingDirectory: dir,
maxTurns: 20,
timeout: 180_000,
timeout: CAPTURE_MS,
testName: 'review-army-red-team',
runId,
});
@@ -513,7 +514,7 @@ Start the file with "RED TEAM REVIEW" on the first line.`,
const content = fs.readFileSync(outputPath, 'utf-8');
expect(content.toLowerCase()).toMatch(/red team|adversarial/);
}
}, 210_000);
}, CAPTURE_MS);
});
// --- Review Army: Consensus (periodic) ---
@@ -566,7 +567,7 @@ mark it as "MULTI-SPECIALIST CONFIRMED" with the confirming categories.
Write findings to ${dir}/review-output.md`,
workingDirectory: dir,
maxTurns: 20,
timeout: 180_000,
timeout: CAPTURE_MS,
testName: 'review-army-consensus',
runId,
});
@@ -585,7 +586,7 @@ Write findings to ${dir}/review-output.md`,
content.includes('interpolat');
expect(hasSqlFinding).toBe(true);
}
}, 210_000);
}, CAPTURE_MS);
});
// Finalize eval collector
+6 -5
View File
@@ -1,4 +1,5 @@
import { expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId,
@@ -65,7 +66,7 @@ Then run git diff against the detected base branch and write a brief review.
Write your findings to ${dir}/review-output.md`,
workingDirectory: dir,
maxTurns: 15,
timeout: 90_000,
timeout: JUDGE_MS,
testName: 'review-base-branch',
runId,
});
@@ -84,7 +85,7 @@ Write your findings to ${dir}/review-output.md`,
return cmd.includes('git diff');
});
expect(usedGitDiff).toBe(true);
}, 120_000);
}, JUDGE_MS);
testConcurrentIfSelected('ship-base-branch', async () => {
const dir = path.join(baseBranchDir, 'ship-base');
@@ -125,7 +126,7 @@ Write a summary to ${dir}/ship-preflight.md including:
- The diff stat against the base branch`,
workingDirectory: dir,
maxTurns: 18,
timeout: 150_000,
timeout: CAPTURE_MS,
testName: 'ship-base-branch',
runId,
});
@@ -155,7 +156,7 @@ Write a summary to ${dir}/ship-preflight.md including:
return command.includes('git push') || command.includes('gh pr create');
});
expect(destructiveTools).toHaveLength(0);
}, 180_000);
}, CAPTURE_MS);
});
// --- Review Dashboard Via Attribution E2E ---
@@ -280,7 +281,7 @@ Write the dashboard output to ${dashDir}/dashboard-output.md`,
);
// Ship dashboard should not gate when eng review is clear
expect(gateQuestions).toHaveLength(0);
}, 480_000);
}, CAPTURE_LONG_MS);
});
// Module-level afterAll — finalize eval collector after all tests complete
+7 -6
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, browseBin, runId, evalsEnabled, selectedTests,
@@ -66,7 +67,7 @@ Run /review on the current diff (git diff main...HEAD).
Write your review findings to ${reviewDir}/review-output.md`,
workingDirectory: reviewDir,
maxTurns: 20,
timeout: 180_000,
timeout: CAPTURE_MS,
testName: 'review-sql-injection',
runId,
});
@@ -89,7 +90,7 @@ Write your review findings to ${reviewDir}/review-output.md`,
reviewContent.includes('unsanitized');
expect(hasSqlContent).toBe(true);
}
}, 210_000);
}, CAPTURE_MS);
});
// --- Review: Enum completeness E2E ---
@@ -144,7 +145,7 @@ Write your review findings to ${enumDir}/review-output.md
The diff adds a new "returned" status to the Order model. Your job is to check if all consumers handle it.`,
workingDirectory: enumDir,
maxTurns: 15,
timeout: 90_000,
timeout: JUDGE_MS,
testName: 'review-enum-completeness',
runId,
});
@@ -164,7 +165,7 @@ The diff adds a new "returned" status to the Order model. Your job is to check i
expect(mentionsReturned).toBe(true);
expect(mentionsEnum || mentionsCritical).toBe(true);
}
}, 120_000);
}, JUDGE_MS);
});
// --- Review: Design review lite E2E ---
@@ -229,7 +230,7 @@ Write your review findings to ${designDir}/review-output.md
Important: The design checklist should catch issues like blacklisted fonts, small font sizes, outline:none, !important, AI slop patterns (purple gradients, generic hero copy, 3-column feature grid), etc.`,
workingDirectory: designDir,
maxTurns: 35,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'review-design-lite',
runId,
});
@@ -262,7 +263,7 @@ Important: The design checklist should catch issues like blacklisted fonts, smal
console.log(`Design review detected ${detected}/7 planted issues`);
expect(detected).toBeGreaterThanOrEqual(4);
}
}, 300_000);
}, CAPTURE_MS);
});
// Base branch detection tests for review/ship + the Review Dashboard Via
+7 -6
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId, evalsEnabled,
@@ -166,7 +167,7 @@ IMPORTANT:
workingDirectory: workDir,
maxTurns: 10,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'context-recovery-artifacts',
runId,
});
@@ -193,7 +194,7 @@ IMPORTANT:
expect(foundCount).toBeGreaterThanOrEqual(1);
console.log(`Context recovery: artifacts=${foundArtifacts}, lastSession=${foundLastSession}, timeline=${foundTimeline}`);
}, 180_000);
}, CAPTURE_MS);
// --- Test 3: /context-save writes a file ---
// Hand-feed the save section of context-save/SKILL.md to claude -p and verify
@@ -231,7 +232,7 @@ IMPORTANT:
workingDirectory: workDir,
maxTurns: 10,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'context-save-writes-file',
runId,
});
@@ -264,7 +265,7 @@ IMPORTANT:
expect(hasYamlFrontmatter).toBe(true);
console.log(`context-save: ${files.length} files created, YAML frontmatter: ${hasYamlFrontmatter}, branch: ${hasBranch}`);
}, 180_000);
}, CAPTURE_MS);
// --- Test 4: /context-restore loads the newest file across branches ---
// Seed two saved-context files with different YYYYMMDD-HHMMSS prefixes and
@@ -340,7 +341,7 @@ IMPORTANT:
workingDirectory: workDir,
maxTurns: 8,
allowedTools: ['Bash', 'Read', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'context-restore-loads-latest',
runId,
});
@@ -361,5 +362,5 @@ IMPORTANT:
expect(loadedOlder).toBe(false);
console.log(`context-restore: loadedNewer=${loadedNewer}, loadedOlder=${loadedOlder}`);
}, 180_000);
}, CAPTURE_MS);
});
@@ -14,6 +14,7 @@
// on a failed verify the skill STOPs before any CLAUDE.md write.
import { test, expect } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'fs';
import * as os from 'os';
@@ -155,5 +156,5 @@ describeE2E('/setup-gbrain Path 4 — bad token STOPs cleanly', () => {
fs.rmSync(gstackHome, { recursive: true, force: true });
fs.rmSync(fakeBinDir, { recursive: true, force: true });
}
}, 240_000);
}, CAPTURE_MS);
});
@@ -19,6 +19,7 @@
// Cost: ~$0.50-$1.00 per run. Periodic-tier (EVALS=1 EVALS_TIER=periodic).
import { test, expect } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'fs';
import * as os from 'os';
@@ -264,5 +265,5 @@ describeE2E('/setup-gbrain Path 4 + Step 4.5 Yes → local PGLite for code', ()
fs.rmSync(sandboxHome, { recursive: true, force: true });
fs.rmSync(fakeBinDir, { recursive: true, force: true });
}
}, 300_000);
}, CAPTURE_MS);
});
+2 -1
View File
@@ -16,6 +16,7 @@
// block this test asserts on).
import { test, expect } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'fs';
import * as os from 'os';
@@ -245,5 +246,5 @@ describeE2E('/setup-gbrain Path 4 (Remote MCP) — happy path', () => {
fs.rmSync(gstackHome, { recursive: true, force: true });
fs.rmSync(fakeBinDir, { recursive: true, force: true });
}
}, 240_000);
}, CAPTURE_MS);
});
+3 -2
View File
@@ -49,6 +49,7 @@
* gate tier confirmed).
*/
import { expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
@@ -200,7 +201,7 @@ describeE2E('Ship doc-sync dispatch E2E (gate)', () => {
workingDirectory: repoDir,
maxTurns: 30,
allowedTools: ['Bash', 'Read', 'Grep', 'Glob', 'Write', 'Agent', 'Task'],
timeout: 480_000,
timeout: CAPTURE_LONG_MS,
env: {
HOME: workDir,
GSTACK_HOME: path.join(workDir, 'gstack-home'),
@@ -273,7 +274,7 @@ describeE2E('Ship doc-sync dispatch E2E (gate)', () => {
console.log(
`dispatchIdx=${dispatchIdx} prCreateIdx=${prCreateIdx} readPrBody=${readPrBody} exit=${result.exitReason}`
);
}, 540_000);
}, CAPTURE_LONG_MS);
});
});
+3 -2
View File
@@ -31,6 +31,7 @@
*/
import { test, expect } from 'bun:test';
import { PTY_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { spawnSync } from 'child_process';
import * as fs from 'fs';
@@ -158,7 +159,7 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => {
const session = await launchClaudePty({
permissionMode: 'plan',
cwd: fixture.workTree,
timeoutMs: 1_080_000,
timeoutMs: PTY_LONG_MS,
// Disable network-y pieces so the agent can't reach actual github.
env: { GH_TOKEN: 'mock-not-real', NO_COLOR: '1' },
seedSkills: true,
@@ -279,6 +280,6 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => {
try { fs.rmSync(path.dirname(fixture.workTree), { recursive: true, force: true }); } catch { /* ignore */ }
}
},
1_200_000, // 20 min wall clock
PTY_LONG_MS, // 20 min wall clock
);
});
+2 -1
View File
@@ -24,6 +24,7 @@
*/
import { test, expect } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import {
setupSkillDir,
@@ -78,6 +79,6 @@ describeE2E('/ship section-loading E2E (periodic, SDK capture)', () => {
// Guard against an empty pass: the report must have real content.
expect(output.trim().length).toBeGreaterThan(200);
},
360_000,
CAPTURE_LONG_MS,
);
});
+11 -10
View File
@@ -27,6 +27,7 @@
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, browseBin, runId,
@@ -204,7 +205,7 @@ Do NOT enter the prototype phase. Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 12,
allowedTools: ['Skill', 'Bash', 'Read'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'scrape-match-path',
runId,
});
@@ -224,7 +225,7 @@ Do NOT enter the prototype phase. Do NOT use AskUserQuestion.`,
expect(listedSkills).toBe(true);
expect(ranBundledSkill).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 180_000);
}, CAPTURE_MS);
// ── 2. /scrape prototype path: drive $B primitives against fixture ────
testConcurrentIfSelected('scrape-prototype-path', async () => {
@@ -248,7 +249,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 18,
allowedTools: ['Skill', 'Bash', 'Read'],
timeout: 180_000,
timeout: CAPTURE_MS,
testName: 'scrape-prototype-path',
runId,
});
@@ -283,7 +284,7 @@ Do NOT use AskUserQuestion.`,
expect(hasJsonItems).toBe(true);
expect(mentionsSkillify).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 240_000);
}, CAPTURE_MS);
// ── 3. /skillify happy path: scrape then skillify in one session ─────
testConcurrentIfSelected('skillify-happy-path', async () => {
@@ -316,7 +317,7 @@ Do NOT halt for clarification.`,
},
maxTurns: 40,
allowedTools: ['Skill', 'Bash', 'Read', 'Write'],
timeout: 360_000,
timeout: CAPTURE_LONG_MS,
testName: 'skillify-happy-path',
runId,
});
@@ -360,7 +361,7 @@ Do NOT halt for clarification.`,
expect(hasAllFiles).toBe(true);
expect(prosesClean).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 420_000);
}, CAPTURE_LONG_MS);
// ── 4. /skillify provenance refusal: D1 contract ─────────────────────
testConcurrentIfSelected('skillify-provenance-refusal', async () => {
@@ -379,7 +380,7 @@ write any files.`,
},
maxTurns: 8,
allowedTools: ['Skill', 'Bash', 'Read'],
timeout: 90_000,
timeout: JUDGE_MS,
testName: 'skillify-provenance-refusal',
runId,
});
@@ -408,7 +409,7 @@ write any files.`,
expect(noSkillsWritten).toBe(true);
expect(noStaging).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 120_000);
}, JUDGE_MS);
// ── 5. /skillify approval-gate reject: D3 cleanup ────────────────────
testConcurrentIfSelected('skillify-approval-reject', async () => {
@@ -435,7 +436,7 @@ Use HOME=${workDir}. Do NOT commit the skill.`,
},
maxTurns: 35,
allowedTools: ['Skill', 'Bash', 'Read', 'Write'],
timeout: 360_000,
timeout: CAPTURE_LONG_MS,
testName: 'skillify-approval-reject',
runId,
});
@@ -462,5 +463,5 @@ Use HOME=${workDir}. Do NOT commit the skill.`,
expect(writtenSkills.length).toBe(0);
expect(stagingLeftovers.length).toBe(0);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 420_000);
}, CAPTURE_LONG_MS);
});
+3 -2
View File
@@ -19,6 +19,7 @@
*/
import { test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId,
@@ -178,7 +179,7 @@ This is a solo repo (REPO_MODE=solo). For pre-existing failures, recommend fixin
workingDirectory: triageDir,
maxTurns: 20,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
timeout: 180_000,
timeout: CAPTURE_MS,
testName: 'ship-triage',
runId,
});
@@ -229,7 +230,7 @@ This is a solo repo (REPO_MODE=solo). For pre-existing failures, recommend fixin
// Must have actually run both test files (exercises both failure classes)
expect(ranMathTest).toBe(true);
expect(ranStringTest).toBe(true);
}, 240_000);
}, CAPTURE_MS);
});
// Module-level afterAll — finalize eval collector after all tests complete
+11 -10
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, browseBin, runId, evalsEnabled,
@@ -83,7 +84,7 @@ IMPORTANT:
// other rounds — marginal at 180s, same contention story as
// review-dashboard-via and retro-base-branch. Outer bun timeout
// rises to 360s for headroom.
timeout: 300_000,
timeout: CAPTURE_MS,
testName: 'document-release',
runId,
});
@@ -120,7 +121,7 @@ IMPORTANT:
} else {
console.warn('README was NOT updated — agent may not have found the feature');
}
}, 360_000);
}, CAPTURE_LONG_MS);
});
// --- Ship workflow with local bare remote ---
@@ -174,7 +175,7 @@ describeIfSelected('Ship workflow E2E', ['ship-local-workflow'], () => {
4. Push to origin: git push origin feature/ship-test`,
workingDirectory: shipWorkDir,
maxTurns: 8,
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'ship-local-workflow',
runId,
});
@@ -198,7 +199,7 @@ describeIfSelected('Ship workflow E2E', ['ship-local-workflow'], () => {
expect(branchExists).toBe(true);
expect(versionBumped).toBe(true);
console.log(`Branch pushed: ${branchExists}, VERSION: ${versionContent}, bumped: ${versionBumped}`);
}, 150_000);
}, CAPTURE_MS);
});
// setup-cookies-detect REMOVED: The cookie-import-browser module has 30+ thorough
@@ -297,7 +298,7 @@ Skip any AskUserQuestion calls — auto-approve the upgrade. Write a summary of
IMPORTANT: The install directory is at ./.claude/skills/gstack use that exact path.`,
workingDirectory: upgradeDir,
maxTurns: 20,
timeout: 180_000,
timeout: CAPTURE_MS,
testName: 'gstack-upgrade-happy-path',
runId,
});
@@ -317,7 +318,7 @@ IMPORTANT: The install directory is at ./.claude/skills/gstack — use that exac
expect(['success', 'error_max_turns']).toContain(result.exitReason);
expect(versionAfter).toBe('0.6.0');
}, 240_000);
}, CAPTURE_MS);
});
// --- Test Coverage Audit E2E ---
@@ -418,7 +419,7 @@ Output the diagram directly.`,
workingDirectory: coverageDir,
maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'ship-coverage-audit',
runId,
});
@@ -443,7 +444,7 @@ Output the diagram directly.`,
// At minimum, the agent should have read the source and test files
const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read');
expect(readCalls.length).toBeGreaterThan(0);
}, 180_000);
}, CAPTURE_MS);
});
// --- Codex skill E2E ---
@@ -520,7 +521,7 @@ Follow those instructions to run codex review against the diff on this branch.
Write the full output (including the GATE verdict) to ${codexDir}/codex-output.md`,
workingDirectory: codexDir,
maxTurns: 25,
timeout: 300_000,
timeout: CAPTURE_MS,
testName: 'codex-review',
runId,
model: 'claude-opus-4-7',
@@ -538,7 +539,7 @@ Write the full output (including the GATE verdict) to ${codexDir}/codex-output.m
const hasCodexOutput = output.includes('CODEX') || output.includes('GATE') || output.includes('codex');
expect(hasCodexOutput).toBe(true);
}
}, 360_000);
}, CAPTURE_LONG_MS);
});
// Module-level afterAll — finalize eval collector after all tests complete
+2 -1
View File
@@ -11,6 +11,7 @@
*/
import { afterAll, expect } from 'bun:test';
import { JUDGE_MS } from './helpers/eval-budgets';
import Anthropic from '@anthropic-ai/sdk';
import * as fs from 'fs';
import * as path from 'path';
@@ -556,7 +557,7 @@ describeIfSelected('Baseline score pinning', ['baseline score pinning'], () => {
if (!passed) {
throw new Error(`Score regressions detected:\n${regressions.join('\n')}`);
}
}, 60_000);
}, JUDGE_MS);
});
// --- Workflow SKILL.md quality evals (10 new tests for 100% coverage) ---
+21 -20
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import type { SkillTestResult } from './helpers/session-runner';
import { EvalCollector } from './helpers/eval-store';
@@ -197,7 +198,7 @@ describeE2E('Skill Routing E2E — Developer Journey', () => {
workingDirectory: tmpDir,
maxTurns: 5,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 60_000,
timeout: JUDGE_MS,
testName,
runId,
});
@@ -213,7 +214,7 @@ describeE2E('Skill Routing E2E — Developer Journey', () => {
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
}, 150_000);
}, CAPTURE_MS);
testIfSelected('journey-plan-eng', async () => {
const tmpDir = createRoutingWorkDir('plan-eng');
@@ -247,7 +248,7 @@ describeE2E('Skill Routing E2E — Developer Journey', () => {
workingDirectory: tmpDir,
maxTurns: 5,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 60_000,
timeout: JUDGE_MS,
testName,
runId,
});
@@ -263,7 +264,7 @@ describeE2E('Skill Routing E2E — Developer Journey', () => {
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
}, 150_000);
}, CAPTURE_MS);
// Removed: journey-think-bigger
// Tested ambiguous routing ("think bigger" → plan-ceo-review) but Claude
@@ -309,7 +310,7 @@ export default app;
workingDirectory: tmpDir,
maxTurns: 5,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 60_000,
timeout: JUDGE_MS,
testName,
runId,
});
@@ -326,7 +327,7 @@ export default app;
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
}, 150_000);
}, CAPTURE_MS);
testIfSelected('journey-qa', async () => {
const tmpDir = createRoutingWorkDir('qa');
@@ -345,7 +346,7 @@ export default app;
workingDirectory: tmpDir,
maxTurns: 5,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 60_000,
timeout: JUDGE_MS,
testName,
runId,
});
@@ -362,7 +363,7 @@ export default app;
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
}, 150_000);
}, CAPTURE_MS);
testIfSelected('journey-code-review', async () => {
const tmpDir = createRoutingWorkDir('code-review');
@@ -386,7 +387,7 @@ export default app;
workingDirectory: tmpDir,
maxTurns: 5,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 120_000,
timeout: JUDGE_MS,
testName,
runId,
});
@@ -402,7 +403,7 @@ export default app;
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
}, 150_000);
}, CAPTURE_MS);
testIfSelected('journey-ship', async () => {
const tmpDir = createRoutingWorkDir('ship');
@@ -425,7 +426,7 @@ export default app;
workingDirectory: tmpDir,
maxTurns: 5,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 60_000,
timeout: JUDGE_MS,
testName,
runId,
});
@@ -441,7 +442,7 @@ export default app;
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
}, 150_000);
}, CAPTURE_MS);
testIfSelected('journey-docs', async () => {
const tmpDir = createRoutingWorkDir('docs');
@@ -462,7 +463,7 @@ export default app;
workingDirectory: tmpDir,
maxTurns: 5,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 60_000,
timeout: JUDGE_MS,
testName,
runId,
});
@@ -478,7 +479,7 @@ export default app;
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
}, 150_000);
}, CAPTURE_MS);
testIfSelected('journey-retro', async () => {
const tmpDir = createRoutingWorkDir('retro');
@@ -505,7 +506,7 @@ export default app;
workingDirectory: tmpDir,
maxTurns: 5,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 120_000,
timeout: JUDGE_MS,
testName,
runId,
});
@@ -521,7 +522,7 @@ export default app;
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
}, 150_000);
}, CAPTURE_MS);
testIfSelected('journey-design-system', async () => {
const tmpDir = createRoutingWorkDir('design-system');
@@ -534,7 +535,7 @@ export default app;
workingDirectory: tmpDir,
maxTurns: 5,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 60_000,
timeout: JUDGE_MS,
testName,
runId,
});
@@ -550,7 +551,7 @@ export default app;
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
}, 150_000);
}, CAPTURE_MS);
testIfSelected('journey-visual-qa', async () => {
const tmpDir = createRoutingWorkDir('visual-qa');
@@ -585,7 +586,7 @@ body { font-family: sans-serif; }
workingDirectory: tmpDir,
maxTurns: 5,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 60_000,
timeout: JUDGE_MS,
testName,
runId,
});
@@ -602,5 +603,5 @@ body { font-family: sans-serif; }
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
}, 150_000);
}, CAPTURE_MS);
});