mirror of
https://github.com/garrytan/gstack.git
synced 2026-08-31 10:20:42 +02:00
refactor(test): mechanical sweep — 298 paid-test timeouts onto eval-budget tiers
69 files, both shapes (trailing bun-test budgets and runner timeout/timeoutMs options), ROUND-UP ONLY so nothing that passed can start failing: 75 → JUDGE_MS, 137 → CAPTURE_MS, 74 → CAPTURE_LONG_MS, 9 → PTY_MS, 3 → PTY_LONG_MS. Raw >=60s literal count in the paid scope: 395 → 97, of which 51 are non-timeout noise (fixture dates, run IDs) and 46 are enumerated justified holds (comment-carrying calibrated budgets, poll-loop constants, utility spawn waits, and the seven physical-ceiling 1_500_000 sites). The eval-budgets policy ratchet keeps the residue from regrowing. Known collapse: where an inner runner budget and its enclosing test budget now share a tier, the old stagger is gone — an overrun surfaces as a bun test timeout instead of a graceful runner timeout (diagnosability trade, not a correctness one). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
74ae357e0f
commit
6841183c35
@@ -20,6 +20,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { setupSkillDir, skillFromWorktree, captureSectionReads } from './helpers/auq-sdk-capture';
|
||||
import { CARVE_GUARDS } from './helpers/carve-guards';
|
||||
@@ -97,7 +98,7 @@ describeE2E('carve behavioral section-loading (periodic, SDK capture)', () => {
|
||||
});
|
||||
expect(output.trim().length).toBeGreaterThan(200);
|
||||
},
|
||||
540_000,
|
||||
CAPTURE_LONG_MS,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -26,6 +26,7 @@
|
||||
* Periodic tier (Codex non-determinism). Cost: ~$2-3 per full run.
|
||||
*/
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runCodexSkill, installSkillToTempHome } from './helpers/codex-session-runner';
|
||||
import type { CodexResult } from './helpers/codex-session-runner';
|
||||
import { EvalCollector } from './helpers/eval-store';
|
||||
@@ -186,7 +187,7 @@ describeCodex('Codex Plan Format — CEO Mode Selection', () => {
|
||||
const result = await runCodexSkill({
|
||||
skillDir,
|
||||
prompt: `Read the plan-ceo-review skill. Read plan.md (the plan to review). Proceed to Step 0F (Mode Selection) where the skill presents 4 mode options (SCOPE EXPANSION, SELECTIVE EXPANSION, HOLD SCOPE, SCOPE REDUCTION) via AskUserQuestion. These options differ in kind (review posture), not coverage. ${captureInstruction(outFile)}`,
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
cwd: planDir,
|
||||
skillName: 'gstack-plan-ceo-review',
|
||||
sandbox: 'workspace-write',
|
||||
@@ -208,7 +209,7 @@ describeCodex('Codex Plan Format — CEO Mode Selection', () => {
|
||||
// kind-differentiated: no fabricated score, must have note
|
||||
expect(captured).not.toMatch(COMPLETENESS_RE);
|
||||
expect(captured).toMatch(KIND_NOTE_RE);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
describeCodex('Codex Plan Format — CEO Approach Menu', () => {
|
||||
@@ -226,7 +227,7 @@ describeCodex('Codex Plan Format — CEO Approach Menu', () => {
|
||||
const result = await runCodexSkill({
|
||||
skillDir,
|
||||
prompt: `Read the plan-ceo-review skill. Read plan.md. Proceed to Step 0C-bis (Implementation Alternatives / Approach Menu) where the skill generates 2-3 approaches (minimal viable vs ideal architecture) and presents them via AskUserQuestion. These options differ in coverage so Completeness: N/10 applies. ${captureInstruction(outFile)}`,
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
cwd: planDir,
|
||||
skillName: 'gstack-plan-ceo-review',
|
||||
sandbox: 'workspace-write',
|
||||
@@ -245,7 +246,7 @@ describeCodex('Codex Plan Format — CEO Approach Menu', () => {
|
||||
expect(captured.length).toBeGreaterThan(ELI10_LENGTH_FLOOR);
|
||||
expect(captured).toMatch(RECOMMENDATION_RE);
|
||||
expect(captured).toMatch(COMPLETENESS_RE);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
describeCodex('Codex Plan Format — Eng Coverage Issue', () => {
|
||||
@@ -263,7 +264,7 @@ describeCodex('Codex Plan Format — Eng Coverage Issue', () => {
|
||||
const result = await runCodexSkill({
|
||||
skillDir,
|
||||
prompt: `Read the plan-eng-review skill. Read plan.md. In your Section 3 Test Review, generate ONE AskUserQuestion about test coverage depth where options are clearly coverage-differentiated: A) full coverage incl. edge + error paths (Completeness 10/10), B) happy path only (7/10), C) smoke test (3/10). ${captureInstruction(outFile)}`,
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
cwd: planDir,
|
||||
skillName: 'gstack-plan-eng-review',
|
||||
sandbox: 'workspace-write',
|
||||
@@ -282,7 +283,7 @@ describeCodex('Codex Plan Format — Eng Coverage Issue', () => {
|
||||
expect(captured.length).toBeGreaterThan(ELI10_LENGTH_FLOOR);
|
||||
expect(captured).toMatch(RECOMMENDATION_RE);
|
||||
expect(captured).toMatch(COMPLETENESS_RE);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
describeCodex('Codex Plan Format — Eng Kind Issue', () => {
|
||||
@@ -300,7 +301,7 @@ describeCodex('Codex Plan Format — Eng Kind Issue', () => {
|
||||
const result = await runCodexSkill({
|
||||
skillDir,
|
||||
prompt: `Read the plan-eng-review skill. Read plan.md. In your Section 1 Architecture review, generate ONE AskUserQuestion about an architectural choice where the options differ in kind (e.g. Redis vs Postgres materialized view vs in-process cache — different kinds of systems with different tradeoffs, NOT more-or-less-complete versions of the same thing). ${captureInstruction(outFile)}`,
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
cwd: planDir,
|
||||
skillName: 'gstack-plan-eng-review',
|
||||
sandbox: 'workspace-write',
|
||||
@@ -321,5 +322,5 @@ describeCodex('Codex Plan Format — Eng Kind Issue', () => {
|
||||
// kind-differentiated: no fabricated score
|
||||
expect(captured).not.toMatch(COMPLETENESS_RE);
|
||||
expect(captured).toMatch(KIND_NOTE_RE);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
* Periodic tier (Codex non-determinism, ~$2-3/run).
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import * as path from 'node:path';
|
||||
import { e2eTierEnabled } from './helpers/e2e-gate';
|
||||
import { runCodexSkill } from './helpers/codex-session-runner';
|
||||
@@ -69,7 +70,7 @@ describeCodex('/codex recommendation substance (live, periodic)', () => {
|
||||
skillDir: path.join(ROOT, 'codex'),
|
||||
skillName: 'codex',
|
||||
prompt: FIXTURE_DIFF,
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
});
|
||||
|
||||
if (result.output.startsWith('SKIP:')) {
|
||||
@@ -98,6 +99,6 @@ describeCodex('/codex recommendation substance (live, periodic)', () => {
|
||||
);
|
||||
}
|
||||
},
|
||||
360_000,
|
||||
CAPTURE_LONG_MS,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
* golden), parallel shards (worktree copies), or live symlinked installs.
|
||||
*/
|
||||
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
@@ -262,5 +263,5 @@ You are authorized to implement the minimal fix. The task boundary is src/parse-
|
||||
expect(readmeDecoyUntouched).toBe(true);
|
||||
|
||||
console.log(`codex-sol-scope: ${result.tokens} tokens, ${result.toolCalls.length} tool calls, ${Math.round(result.durationMs / 1000)}s`);
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runCodexSkill, parseCodexJSONL, installSkillToTempHome } from './helpers/codex-session-runner';
|
||||
import type { CodexResult } from './helpers/codex-session-runner';
|
||||
import { CODEX_REVIEW_E2E_SECTIONS } from './helpers/skill-fixture';
|
||||
@@ -150,7 +151,7 @@ describeCodex('Codex E2E', () => {
|
||||
const result = await runCodexSkill({
|
||||
skillDir,
|
||||
prompt: 'List any skills or instructions you have available. Just list the names.',
|
||||
timeoutMs: 60_000,
|
||||
timeoutMs: JUDGE_MS,
|
||||
cwd: testWorktree,
|
||||
skillName: 'gstack-review',
|
||||
});
|
||||
@@ -171,7 +172,7 @@ describeCodex('Codex E2E', () => {
|
||||
expect(
|
||||
outputLower.includes('review') || outputLower.includes('gstack') || outputLower.includes('skill'),
|
||||
).toBe(true);
|
||||
}, 120_000);
|
||||
}, JUDGE_MS);
|
||||
|
||||
// Validates that Codex can invoke the gstack-review skill, run a diff-based
|
||||
// code review, and produce structured review output with findings/issues.
|
||||
@@ -186,7 +187,7 @@ describeCodex('Codex E2E', () => {
|
||||
const result = await runCodexSkill({
|
||||
skillDir,
|
||||
prompt: 'Run the gstack-review skill on this repository. Review the current branch diff and report your findings.',
|
||||
timeoutMs: 540_000,
|
||||
timeoutMs: CAPTURE_LONG_MS,
|
||||
cwd: testWorktree,
|
||||
skillName: 'gstack-review',
|
||||
sections: CODEX_REVIEW_E2E_SECTIONS,
|
||||
@@ -224,5 +225,5 @@ describeCodex('Codex E2E', () => {
|
||||
outputLower.includes('p1') ||
|
||||
outputLower.includes('p2');
|
||||
expect(hasReviewContent).toBe(true);
|
||||
}, 600_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS } from './helpers/eval-budgets';
|
||||
import { runGeminiSkill } from './helpers/gemini-session-runner';
|
||||
import type { GeminiResult } from './helpers/gemini-session-runner';
|
||||
import { EvalCollector } from './helpers/eval-store';
|
||||
@@ -151,7 +152,7 @@ describeGemini('Gemini E2E', () => {
|
||||
// Uses a simple prompt that doesn't require skill invocation or complex navigation.
|
||||
const result = await runGeminiSkill({
|
||||
prompt: 'What is this project? Answer in one sentence based on the README.',
|
||||
timeoutMs: 90_000,
|
||||
timeoutMs: JUDGE_MS,
|
||||
cwd: testWorktree,
|
||||
});
|
||||
|
||||
@@ -163,5 +164,5 @@ describeGemini('Gemini E2E', () => {
|
||||
recordGeminiE2E('gemini-smoke', result, passed);
|
||||
|
||||
expect(result.output.length, 'Gemini should produce output').toBeGreaterThan(10);
|
||||
}, 120_000);
|
||||
}, JUDGE_MS);
|
||||
});
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
*/
|
||||
|
||||
import { expect } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { judgeRecommendation } from './helpers/llm-judge';
|
||||
import { describeIfSelected, testIfSelected } from './helpers/e2e-helpers';
|
||||
|
||||
@@ -181,5 +182,5 @@ Net: ...`);
|
||||
`[hedge:${label}] expected commits=false; got ${score.commits}. text="${text}"`,
|
||||
).toBe(false);
|
||||
}
|
||||
}, 240_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
* A/B and matrix evals (test/helpers/auq-sdk-capture.ts).
|
||||
*/
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import {
|
||||
@@ -86,6 +87,6 @@ describeE2E('AskUserQuestion format compliance (gate)', () => {
|
||||
);
|
||||
}
|
||||
},
|
||||
300_000,
|
||||
CAPTURE_MS,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
* (N SDK runs, ~$0.50-1 each).
|
||||
*/
|
||||
import { test } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import {
|
||||
@@ -99,6 +100,6 @@ describeE2E('AUQ consistency across runs (periodic)', () => {
|
||||
`format elements every run; substance ${minSub}-${maxSub}`,
|
||||
);
|
||||
},
|
||||
N_RUNS * 300_000 + 60_000,
|
||||
N_RUNS * CAPTURE_MS + 60_000,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
* Run a subset in the foreground with AUQ_MATRIX_ONLY="plan-eng-review,cso".
|
||||
*/
|
||||
import { test } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import {
|
||||
@@ -174,7 +175,7 @@ describeE2E('AUQ behavioral matrix (periodic)', () => {
|
||||
);
|
||||
}
|
||||
},
|
||||
300_000,
|
||||
CAPTURE_MS,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
* strictly less unrelated review-section text in context.
|
||||
*/
|
||||
import { test } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import {
|
||||
@@ -109,6 +110,6 @@ describeE2E('AUQ no-degradation: verbose vs carved (periodic)', () => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('[AUQ-AB] NO DEGRADATION:\n' + summary);
|
||||
},
|
||||
600_000,
|
||||
CAPTURE_LONG_MS,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -38,6 +38,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillObservation } from './helpers/claude-pty-runner';
|
||||
import * as fs from 'fs';
|
||||
@@ -113,7 +114,7 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', ()
|
||||
skillName: 'plan-ceo-review',
|
||||
inPlanMode: true,
|
||||
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
|
||||
timeoutMs: 540_000,
|
||||
timeoutMs: CAPTURE_LONG_MS,
|
||||
env: { GSTACK_HOME: tmpHome, CONDUCTOR_WORKSPACE_PATH: tmpHome },
|
||||
});
|
||||
|
||||
@@ -137,5 +138,5 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', ()
|
||||
} finally {
|
||||
try { fs.rmSync(tmpHome, { recursive: true, force: true }); } catch { /* best-effort */ }
|
||||
}
|
||||
}, 660_000);
|
||||
}, PTY_MS);
|
||||
});
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { PTY_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
@@ -174,6 +175,6 @@ describeE2E('/autoplan chain ordering (periodic)', () => {
|
||||
try { fs.rmSync(tempDir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
}
|
||||
},
|
||||
1_200_000, // 20 min absolute test ceiling
|
||||
PTY_LONG_MS, // 20 min absolute test ceiling
|
||||
);
|
||||
});
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId, evalsEnabled,
|
||||
@@ -98,7 +99,7 @@ Add a new /greet skill that prints a welcome message.
|
||||
testName: 'autoplan-dual-voice',
|
||||
workingDirectory: workDir,
|
||||
prompt: `/autoplan ${planPath}`,
|
||||
timeout: 600_000, // 10 min
|
||||
timeout: CAPTURE_LONG_MS, // 10 min
|
||||
// /autoplan spawns subagents and calls codex via Bash; it needs the
|
||||
// full tool set to get past Phase 1. Bash+Read+Write alone wasn't
|
||||
// enough — the skill stalled trying to invoke Agent/Skill.
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { ClaudeAdapter } from './helpers/providers/claude';
|
||||
import { GptAdapter } from './helpers/providers/gpt';
|
||||
import { GeminiAdapter } from './helpers/providers/gemini';
|
||||
@@ -94,7 +95,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
|
||||
process.stderr.write(`\nclaude live smoke: SKIPPED — ${check.reason}\n`);
|
||||
return;
|
||||
}
|
||||
const result = await claude.run({ prompt: PROMPT, workdir, timeoutMs: 120_000 });
|
||||
const result = await claude.run({ prompt: PROMPT, workdir, timeoutMs: JUDGE_MS });
|
||||
if (result.error) {
|
||||
throw new Error(`claude errored: ${result.error.code} — ${result.error.reason}`);
|
||||
}
|
||||
@@ -106,7 +107,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
|
||||
expect(result.modelUsed.length).toBeGreaterThan(0);
|
||||
const cost = claude.estimateCost(result.tokens, result.modelUsed);
|
||||
expect(cost).toBeGreaterThan(0);
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
test('gpt: trivial prompt produces parseable output', async () => {
|
||||
const check = await gpt.available();
|
||||
@@ -114,7 +115,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
|
||||
process.stderr.write(`\ngpt live smoke: SKIPPED — ${check.reason}\n`);
|
||||
return;
|
||||
}
|
||||
const result = await gpt.run({ prompt: PROMPT, workdir, timeoutMs: 120_000 });
|
||||
const result = await gpt.run({ prompt: PROMPT, workdir, timeoutMs: JUDGE_MS });
|
||||
if (result.error) {
|
||||
throw new Error(`gpt errored: ${result.error.code} — ${result.error.reason}`);
|
||||
}
|
||||
@@ -125,7 +126,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
|
||||
expect(typeof result.modelUsed).toBe('string');
|
||||
const cost = gpt.estimateCost(result.tokens, result.modelUsed);
|
||||
expect(cost).toBeGreaterThan(0);
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
test('gemini: trivial prompt produces parseable output', async () => {
|
||||
const check = await gemini.available();
|
||||
@@ -133,7 +134,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
|
||||
process.stderr.write(`\ngemini live smoke: SKIPPED — ${check.reason}\n`);
|
||||
return;
|
||||
}
|
||||
const result = await gemini.run({ prompt: PROMPT, workdir, timeoutMs: 120_000 });
|
||||
const result = await gemini.run({ prompt: PROMPT, workdir, timeoutMs: JUDGE_MS });
|
||||
if (result.error) {
|
||||
// auth / rate_limit are ENVIRONMENT conditions the test can't act on
|
||||
// (e.g. Google deprecated the individual code-assist auth path — the
|
||||
@@ -155,7 +156,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
|
||||
expect(result.durationMs).toBeGreaterThan(0);
|
||||
expect(typeof result.modelUsed).toBe('string');
|
||||
expect(result.modelUsed.length).toBeGreaterThan(0);
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
test('timeout error surfaces as error.code=timeout (no exception)', async () => {
|
||||
// Use whatever adapter is available first — all three should share timeout semantics.
|
||||
@@ -183,7 +184,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
|
||||
prompt: PROMPT,
|
||||
workdir,
|
||||
providers: ['claude', 'gpt', 'gemini'],
|
||||
timeoutMs: 120_000,
|
||||
timeoutMs: JUDGE_MS,
|
||||
skipUnavailable: false,
|
||||
});
|
||||
expect(report.entries).toHaveLength(3);
|
||||
@@ -201,5 +202,5 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
|
||||
if (!hadSuccess) {
|
||||
process.stderr.write('\nrunBenchmark live: no provider produced a clean result (no auth?)\n');
|
||||
}
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
@@ -150,7 +151,7 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
|
||||
fs.rmSync(fakeBinDir, { recursive: true, force: true });
|
||||
fs.rmSync(tempHome, { recursive: true, force: true });
|
||||
}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
test('privacy gate does NOT fire when artifacts_sync_mode_prompted is already true', async () => {
|
||||
// Same staging, but prompted=true this time. Gate should be silent.
|
||||
@@ -228,5 +229,5 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
|
||||
fs.rmSync(fakeBinDir, { recursive: true, force: true });
|
||||
fs.rmSync(tempHome, { recursive: true, force: true });
|
||||
}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
+12
-11
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, browseBin, runId, evalsEnabled,
|
||||
@@ -47,7 +48,7 @@ describeIfSelected('Skill E2E tests', [
|
||||
Report the results of each command.`,
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 7,
|
||||
timeout: 60_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'browse-basic',
|
||||
runId,
|
||||
});
|
||||
@@ -56,7 +57,7 @@ Report the results of each command.`,
|
||||
recordE2E(evalCollector, 'browse basic commands', 'Skill E2E tests', result);
|
||||
expect(result.browseErrors).toHaveLength(0);
|
||||
expect(result.exitReason).toBe('success');
|
||||
}, 90_000);
|
||||
}, JUDGE_MS);
|
||||
|
||||
testConcurrentIfSelected('browse-snapshot', async () => {
|
||||
const result = await runSkillTest({
|
||||
@@ -69,7 +70,7 @@ Report the results of each command.`,
|
||||
Report what each command returned.`,
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 9,
|
||||
timeout: 60_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'browse-snapshot',
|
||||
runId,
|
||||
});
|
||||
@@ -81,7 +82,7 @@ Report what each command returned.`,
|
||||
console.warn('Browse errors (non-fatal):', result.browseErrors);
|
||||
}
|
||||
expect(result.exitReason).toBe('success');
|
||||
}, 90_000);
|
||||
}, JUDGE_MS);
|
||||
|
||||
testConcurrentIfSelected('skillmd-setup-discovery', async () => {
|
||||
// P2 (v1.2.0): the browse SETUP/binary-discovery block moved from the root
|
||||
@@ -104,7 +105,7 @@ Then run: $B text
|
||||
Report whether it worked.`,
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 10,
|
||||
timeout: 60_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'skillmd-setup-discovery',
|
||||
runId,
|
||||
});
|
||||
@@ -112,7 +113,7 @@ Report whether it worked.`,
|
||||
recordE2E(evalCollector, 'SKILL.md setup block discovery', 'Skill E2E tests', result);
|
||||
expect(result.browseErrors).toHaveLength(0);
|
||||
expect(result.exitReason).toBe('success');
|
||||
}, 90_000);
|
||||
}, JUDGE_MS);
|
||||
|
||||
testConcurrentIfSelected('skillmd-no-local-binary', async () => {
|
||||
// Create a tmpdir with no browse binary — no local .claude/skills/gstack/browse/dist/browse
|
||||
@@ -149,7 +150,7 @@ Report the exact output. Do NOT try to fix or install anything — just report w
|
||||
|
||||
// Clean up
|
||||
try { fs.rmSync(emptyDir, { recursive: true, force: true }); } catch {}
|
||||
}, 60_000);
|
||||
}, JUDGE_MS);
|
||||
|
||||
testConcurrentIfSelected('skillmd-outside-git', async () => {
|
||||
// Create a tmpdir outside any git repo
|
||||
@@ -182,7 +183,7 @@ Report the exact output — either "READY: <path>" or "NEEDS_SETUP".`,
|
||||
|
||||
// Clean up
|
||||
try { fs.rmSync(nonGitDir, { recursive: true, force: true }); } catch {}
|
||||
}, 60_000);
|
||||
}, JUDGE_MS);
|
||||
|
||||
testConcurrentIfSelected('operational-learning', async () => {
|
||||
const opDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-oplearn-'));
|
||||
@@ -286,7 +287,7 @@ Log the operational learning now. Then say what you logged.`,
|
||||
|
||||
// Clean up
|
||||
try { fs.rmSync(opDir, { recursive: true, force: true }); } catch {}
|
||||
}, 90_000);
|
||||
}, JUDGE_MS);
|
||||
|
||||
testConcurrentIfSelected('session-awareness', async () => {
|
||||
const sessionDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-session-'));
|
||||
@@ -353,7 +354,7 @@ Since this is non-interactive, DO NOT actually call AskUserQuestion. Instead, wr
|
||||
Remember: _SESSIONS=4, so ELI16 mode is active. The user is juggling multiple windows and may not remember what this conversation is about. Re-ground them.`,
|
||||
workingDirectory: sessionDir,
|
||||
maxTurns: 8,
|
||||
timeout: 60_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'session-awareness',
|
||||
runId,
|
||||
});
|
||||
@@ -394,7 +395,7 @@ Remember: _SESSIONS=4, so ELI16 mode is active. The user is juggling multiple wi
|
||||
|
||||
// Clean up
|
||||
try { fs.rmSync(sessionDir, { recursive: true, force: true }); } catch {}
|
||||
}, 90_000);
|
||||
}, JUDGE_MS);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillObservation } from './helpers/claude-pty-runner';
|
||||
|
||||
@@ -46,7 +47,7 @@ describeE2E('Conductor renders decisions as prose (periodic)', () => {
|
||||
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
|
||||
env: { CONDUCTOR_WORKSPACE_PATH: '/tmp/conductor-prose-e2e' },
|
||||
initialPlanContent: FLAWED_PLAN,
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
});
|
||||
|
||||
// The decision must reach the human as prose. 'silent_write' (wrote findings
|
||||
@@ -65,5 +66,5 @@ describeE2E('Conductor renders decisions as prose (periodic)', () => {
|
||||
}
|
||||
// A prose-rendered decision brief was observed at some point in the run.
|
||||
expect(obs.proseAUQEverObserved).toBe(true);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId, evalsEnabled,
|
||||
@@ -163,7 +164,7 @@ describeIfSelected('Context Skills E2E (live-fire)', [
|
||||
env: { GSTACK_HOME: gstackHome },
|
||||
maxTurns: 12,
|
||||
allowedTools: ['Skill', 'Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'context-save-routing',
|
||||
runId,
|
||||
});
|
||||
@@ -185,7 +186,7 @@ describeIfSelected('Context Skills E2E (live-fire)', [
|
||||
expect(routedToContextSave).toBe(true);
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// ── 2. Round-trip: save then restore in the same session ─────────────
|
||||
testConcurrentIfSelected('context-save-then-restore-roundtrip', async () => {
|
||||
@@ -205,7 +206,7 @@ Do NOT use AskUserQuestion.`,
|
||||
env: { GSTACK_HOME: gstackHome },
|
||||
maxTurns: 25,
|
||||
allowedTools: ['Skill', 'Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'context-save-then-restore-roundtrip',
|
||||
runId,
|
||||
});
|
||||
@@ -232,7 +233,7 @@ Do NOT use AskUserQuestion.`,
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
expect(restoreMentionsTitle).toBe(true);
|
||||
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
|
||||
}, 240_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// ── 3. /context-restore <fragment> loads the matching save ───────────
|
||||
testConcurrentIfSelected('context-restore-fragment-match', async () => {
|
||||
@@ -255,7 +256,7 @@ Do NOT use AskUserQuestion.`,
|
||||
env: { GSTACK_HOME: gstackHome },
|
||||
maxTurns: 10,
|
||||
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'context-restore-fragment-match',
|
||||
runId,
|
||||
});
|
||||
@@ -279,7 +280,7 @@ Do NOT use AskUserQuestion.`,
|
||||
expect(loadedPayments).toBe(true);
|
||||
expect(didNotLoadOthers).toBe(true);
|
||||
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// ── 4. /context-restore with zero saves → graceful empty-state ───────
|
||||
testConcurrentIfSelected('context-restore-empty-state', async () => {
|
||||
@@ -294,7 +295,7 @@ Do NOT use AskUserQuestion.`,
|
||||
env: { GSTACK_HOME: gstackHome },
|
||||
maxTurns: 8,
|
||||
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
|
||||
timeout: 90_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'context-restore-empty-state',
|
||||
runId,
|
||||
});
|
||||
@@ -319,7 +320,7 @@ Do NOT use AskUserQuestion.`,
|
||||
expect(routedToRestore).toBe(true);
|
||||
expect(gracefulMessage).toBe(true);
|
||||
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// ── 5. /context-restore list redirects to /context-save list ─────────
|
||||
testConcurrentIfSelected('context-restore-list-delegates', async () => {
|
||||
@@ -334,7 +335,7 @@ Do NOT use AskUserQuestion.`,
|
||||
env: { GSTACK_HOME: gstackHome },
|
||||
maxTurns: 8,
|
||||
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
|
||||
timeout: 90_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'context-restore-list-delegates',
|
||||
runId,
|
||||
});
|
||||
@@ -357,7 +358,7 @@ Do NOT use AskUserQuestion.`,
|
||||
expect(routedToRestore).toBe(true);
|
||||
expect(mentionsSaveList).toBe(true);
|
||||
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// ── 6. Legacy compat: pre-rename save files still load ───────────────
|
||||
testConcurrentIfSelected('context-restore-legacy-compat', async () => {
|
||||
@@ -381,7 +382,7 @@ Do NOT use AskUserQuestion.`,
|
||||
env: { GSTACK_HOME: gstackHome },
|
||||
maxTurns: 8,
|
||||
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'context-restore-legacy-compat',
|
||||
runId,
|
||||
});
|
||||
@@ -414,7 +415,7 @@ Do NOT use AskUserQuestion.`,
|
||||
expect(routedToRestore).toBe(true);
|
||||
expect(loadedLegacy).toBe(true);
|
||||
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// ── 7. /context-save list: default filters to current branch ─────────
|
||||
testConcurrentIfSelected('context-save-list-current-branch', async () => {
|
||||
@@ -437,7 +438,7 @@ Do NOT use AskUserQuestion.`,
|
||||
env: { GSTACK_HOME: gstackHome },
|
||||
maxTurns: 10,
|
||||
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'context-save-list-current-branch',
|
||||
runId,
|
||||
});
|
||||
@@ -472,7 +473,7 @@ Do NOT use AskUserQuestion.`,
|
||||
expect(hidesAlpha).toBe(true);
|
||||
expect(hidesBeta).toBe(true);
|
||||
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// ── 8. /context-save list --all: shows every branch ──────────────────
|
||||
testConcurrentIfSelected('context-save-list-all-branches', async () => {
|
||||
@@ -494,7 +495,7 @@ Do NOT use AskUserQuestion.`,
|
||||
env: { GSTACK_HOME: gstackHome },
|
||||
maxTurns: 10,
|
||||
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'context-save-list-all-branches',
|
||||
runId,
|
||||
});
|
||||
@@ -520,5 +521,5 @@ Do NOT use AskUserQuestion.`,
|
||||
expect(routed).toBe(true);
|
||||
expect(filesShown).toBe(3);
|
||||
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId,
|
||||
@@ -78,7 +79,7 @@ Output the diagram directly.`,
|
||||
workingDirectory: reviewCoverageDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'review-coverage-audit',
|
||||
runId,
|
||||
});
|
||||
@@ -107,7 +108,7 @@ Output the diagram directly.`,
|
||||
// At minimum, the agent should have read the source and test files
|
||||
const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read');
|
||||
expect(readCalls.length).toBeGreaterThan(0);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Plan Eng Review Coverage Audit E2E ---
|
||||
@@ -153,7 +154,7 @@ Output the diagram directly.`,
|
||||
workingDirectory: planCoverageDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'plan-eng-coverage-audit',
|
||||
runId,
|
||||
});
|
||||
@@ -182,7 +183,7 @@ Output the diagram directly.`,
|
||||
// At minimum, the agent should have read the source and test files
|
||||
const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read');
|
||||
expect(readCalls.length).toBeGreaterThan(0);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId, evalsEnabled,
|
||||
@@ -75,7 +76,7 @@ IMPORTANT:
|
||||
workingDirectory: csoDir,
|
||||
maxTurns: 30,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob', 'Agent'],
|
||||
timeout: 300_000,
|
||||
timeout: CAPTURE_MS,
|
||||
});
|
||||
|
||||
logCost('cso', result);
|
||||
@@ -106,7 +107,7 @@ IMPORTANT:
|
||||
}
|
||||
|
||||
recordE2E(evalCollector, 'cso-full-audit', 'e2e-cso', result);
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
describeIfSelected('CSO v2 — diff mode', ['cso-diff-mode'], () => {
|
||||
@@ -181,7 +182,7 @@ IMPORTANT:
|
||||
).toBe(true);
|
||||
|
||||
recordE2E(evalCollector, 'cso-diff-mode', 'e2e-cso', result);
|
||||
}, 400_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
describeIfSelected('CSO v2 — infra scope', ['cso-infra-scope'], () => {
|
||||
@@ -245,7 +246,7 @@ IMPORTANT:
|
||||
workingDirectory: csoInfraDir,
|
||||
maxTurns: 30,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 360_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
});
|
||||
|
||||
logCost('cso', result);
|
||||
@@ -259,5 +260,5 @@ IMPORTANT:
|
||||
).toBe(true);
|
||||
|
||||
recordE2E(evalCollector, 'cso-infra-scope', 'e2e-cso', result);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, browseBin, runId, evalsEnabled,
|
||||
@@ -67,7 +68,7 @@ Do NOT use AskUserQuestion. Do NOT run gh or fly commands.`,
|
||||
workingDirectory: landDir,
|
||||
maxTurns: 20,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'land-and-deploy-workflow',
|
||||
runId,
|
||||
});
|
||||
@@ -85,7 +86,7 @@ Do NOT use AskUserQuestion. Do NOT run gh or fly commands.`,
|
||||
|
||||
const reportDir = path.join(landDir, '.gstack', 'deploy-reports');
|
||||
expect(fs.existsSync(reportDir)).toBe(true);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Land-and-Deploy First-Run E2E ---
|
||||
@@ -148,7 +149,7 @@ Just demonstrate the first-run dry-run output.`,
|
||||
workingDirectory: firstRunDir,
|
||||
maxTurns: 20,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'land-and-deploy-first-run',
|
||||
runId,
|
||||
});
|
||||
@@ -167,7 +168,7 @@ Just demonstrate the first-run dry-run output.`,
|
||||
const reportContent = fs.readFileSync(path.join(reportDir, reportFiles[0]), 'utf-8');
|
||||
const hasPlatform = reportContent.toLowerCase().includes('fly') || reportContent.toLowerCase().includes('first-run-app');
|
||||
expect(hasPlatform).toBe(true);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Land-and-Deploy Review Gate E2E ---
|
||||
@@ -226,7 +227,7 @@ Show what the readiness gate output would look like.`,
|
||||
workingDirectory: reviewDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'land-and-deploy-review-gate',
|
||||
runId,
|
||||
});
|
||||
@@ -246,7 +247,7 @@ Show what the readiness gate output would look like.`,
|
||||
const hasReviewMention = reportContent.toLowerCase().includes('review') ||
|
||||
reportContent.toLowerCase().includes('not run');
|
||||
expect(hasReviewMention).toBe(true);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Canary skill E2E ---
|
||||
@@ -294,7 +295,7 @@ Just create the directory structure and report files showing the correct schema.
|
||||
workingDirectory: canaryDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'canary-workflow',
|
||||
runId,
|
||||
});
|
||||
@@ -307,7 +308,7 @@ Just create the directory structure and report files showing the correct schema.
|
||||
const reportDir = path.join(canaryDir, '.gstack', 'canary-reports');
|
||||
const files = fs.readdirSync(reportDir, { recursive: true }) as string[];
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Benchmark skill E2E ---
|
||||
@@ -357,7 +358,7 @@ Just create the files showing the correct schema and report format.`,
|
||||
workingDirectory: benchDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'benchmark-workflow',
|
||||
runId,
|
||||
});
|
||||
@@ -372,7 +373,7 @@ Just create the files showing the correct schema and report format.`,
|
||||
const files = fs.readdirSync(baselineDir);
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Setup-Deploy skill E2E ---
|
||||
@@ -418,7 +419,7 @@ Just detect the platform and write the config.`,
|
||||
workingDirectory: setupDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'setup-deploy-workflow',
|
||||
runId,
|
||||
});
|
||||
@@ -434,7 +435,7 @@ Just detect the platform and write the config.`,
|
||||
expect(content.toLowerCase()).toContain('fly');
|
||||
expect(content).toContain('my-cool-app');
|
||||
expect(content).toContain('Deploy Configuration');
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import { callJudge } from './helpers/llm-judge';
|
||||
import {
|
||||
@@ -113,7 +114,7 @@ Skip research — work from your design knowledge. Skip the font preview page. S
|
||||
Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`,
|
||||
workingDirectory: designDir,
|
||||
maxTurns: 20,
|
||||
timeout: 360_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'design-consultation-core',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -178,7 +179,7 @@ Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`,
|
||||
const claude = fs.readFileSync(claudePath, 'utf-8');
|
||||
expect(claude.toLowerCase()).toContain('design.md');
|
||||
}
|
||||
}, 420_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
|
||||
testConcurrentIfSelected('design-consultation-research', async () => {
|
||||
// Test WebSearch integration — research phase only, no DESIGN.md generation
|
||||
@@ -202,7 +203,7 @@ Do NOT generate a full DESIGN.md — just research notes.`,
|
||||
// queued past the budget under concurrent API load. 90s budgets cannot
|
||||
// absorb one slow first completion; 300s is the repo's standard floor
|
||||
// for CI SDK tests. Outer timeout below rises to 360s for headroom.
|
||||
timeout: 300_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'design-consultation-research',
|
||||
runId,
|
||||
});
|
||||
@@ -232,7 +233,7 @@ Do NOT generate a full DESIGN.md — just research notes.`,
|
||||
}
|
||||
|
||||
try { fs.rmSync(researchDir, { recursive: true, force: true }); } catch {}
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
|
||||
testConcurrentIfSelected('design-consultation-existing', async () => {
|
||||
// Pre-create a minimal DESIGN.md (independent of core test)
|
||||
@@ -250,7 +251,7 @@ There is already a DESIGN.md in this repo. Update it with a complete design syst
|
||||
Skip research. Skip font preview. Skip any AskUserQuestion calls — this is non-interactive.`,
|
||||
workingDirectory: designDir,
|
||||
maxTurns: 20,
|
||||
timeout: 360_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'design-consultation-existing',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -279,7 +280,7 @@ Skip research. Skip font preview. Skip any AskUserQuestion calls — this is non
|
||||
expect(hasColor).toBe(true);
|
||||
expect(hasSpacing).toBe(true);
|
||||
}
|
||||
}, 420_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
|
||||
testConcurrentIfSelected('design-consultation-preview', async () => {
|
||||
// Test preview HTML generation only — no DESIGN.md (covered by core test)
|
||||
@@ -302,7 +303,7 @@ Do NOT write DESIGN.md — only the preview HTML.`,
|
||||
maxTurns: 8,
|
||||
// 300s, not 90s: this is the test that failed 3x at 0 turns/$0.00/93s
|
||||
// on PR #2533 CI — see the research test's comment for the class.
|
||||
timeout: 300_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'design-consultation-preview',
|
||||
runId,
|
||||
});
|
||||
@@ -331,7 +332,7 @@ Do NOT write DESIGN.md — only the preview HTML.`,
|
||||
}
|
||||
|
||||
try { fs.rmSync(previewDir, { recursive: true, force: true }); } catch {}
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// --- Plan Design Review E2E (plan-mode) ---
|
||||
@@ -398,7 +399,7 @@ Skip the preamble bash block. Skip any AskUserQuestion calls — this is non-int
|
||||
IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan review, not a live site audit. Just read the plan file, review it, and edit it to fix the gaps.`,
|
||||
workingDirectory: reviewDir,
|
||||
maxTurns: 15,
|
||||
timeout: 300_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'plan-design-review-plan-mode',
|
||||
runId,
|
||||
});
|
||||
@@ -437,7 +438,7 @@ IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan
|
||||
} finally {
|
||||
try { fs.rmSync(reviewDir, { recursive: true, force: true }); } catch {}
|
||||
}
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
|
||||
testConcurrentIfSelected('plan-design-review-no-ui-scope', async () => {
|
||||
const reviewDir = setupReviewDir();
|
||||
@@ -472,7 +473,7 @@ Skip the preamble bash block. Skip any AskUserQuestion calls — this is non-int
|
||||
IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan review, not a live site audit.`,
|
||||
workingDirectory: reviewDir,
|
||||
maxTurns: 10,
|
||||
timeout: 180_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'plan-design-review-no-ui-scope',
|
||||
runId,
|
||||
});
|
||||
@@ -496,7 +497,7 @@ IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan
|
||||
} finally {
|
||||
try { fs.rmSync(reviewDir, { recursive: true, force: true }); } catch {}
|
||||
}
|
||||
}, 240_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Design Review E2E (live-site audit + fix) ---
|
||||
@@ -602,7 +603,7 @@ Read design-review/SKILL.md for the design review + fix workflow.
|
||||
Review the site at ${serverUrl}. Use --quick mode. Skip any AskUserQuestion calls — this is non-interactive. Fix up to 3 issues max. Write your report to ./design-audit.md.`,
|
||||
workingDirectory: qaDesignDir,
|
||||
maxTurns: 30,
|
||||
timeout: 360_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'design-review-fix',
|
||||
runId,
|
||||
});
|
||||
@@ -634,7 +635,7 @@ Review the site at ${serverUrl}. Use --quick mode. Skip any AskUserQuestion call
|
||||
console.warn('No design-audit.md generated');
|
||||
}
|
||||
console.log(`Design fix commits: ${designFixCommits.length}`);
|
||||
}, 420_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
* with its preamble.
|
||||
*/
|
||||
import { describe, expect } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
@@ -73,7 +74,7 @@ describeIfSelected('/diagram skill E2E', ['diagram-triplet', 'diagram-authoring-
|
||||
workingDirectory: dir,
|
||||
maxTurns: 25,
|
||||
allowedTools: ['Bash', 'Read', 'Write'],
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'diagram-triplet',
|
||||
runId,
|
||||
});
|
||||
@@ -98,7 +99,7 @@ describeIfSelected('/diagram skill E2E', ['diagram-triplet', 'diagram-authoring-
|
||||
} finally {
|
||||
try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
}
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
testConcurrentIfSelected('diagram-authoring-quality', async () => {
|
||||
const dir = setupDir('diagram-quality-');
|
||||
@@ -111,7 +112,7 @@ describeIfSelected('/diagram skill E2E', ['diagram-triplet', 'diagram-authoring-
|
||||
workingDirectory: dir,
|
||||
maxTurns: 25,
|
||||
allowedTools: ['Bash', 'Read', 'Write'],
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'diagram-authoring-quality',
|
||||
runId,
|
||||
});
|
||||
@@ -149,5 +150,5 @@ Respond with JSON: {"score": N, "reasoning": "..."}`,
|
||||
} finally {
|
||||
try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
}
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
*/
|
||||
|
||||
import { expect, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
@@ -55,7 +56,7 @@ async function detectVia(workDir: string, testName: string): Promise<string> {
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 3,
|
||||
allowedTools: ['Bash'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
model: MODEL,
|
||||
@@ -91,7 +92,7 @@ describeIfSelected('first-run scaffold detection (E2E)', ['first-task-scaffold']
|
||||
fs.rmSync(nodeDir, { recursive: true, force: true });
|
||||
fs.rmSync(greenDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
afterAll(() => finalizeEvalCollector(evalCollector));
|
||||
|
||||
@@ -29,6 +29,7 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS } from './helpers/eval-budgets';
|
||||
import { execFileSync } from 'child_process';
|
||||
import { mkdtempSync, rmSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
@@ -156,7 +157,7 @@ ${body}`;
|
||||
expect(retrieved).not.toContain('page_not_found');
|
||||
expect(retrieved).not.toContain('Page not found');
|
||||
},
|
||||
120_000,
|
||||
JUDGE_MS,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -31,6 +31,7 @@
|
||||
*/
|
||||
|
||||
import { expect, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
@@ -94,7 +95,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic-
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 3,
|
||||
allowedTools: ['Bash'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'hermetic-canary',
|
||||
runId,
|
||||
model: CANARY_MODEL,
|
||||
@@ -129,7 +130,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic-
|
||||
}
|
||||
fs.rmSync(workDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
testIfSelected('hermetic-sentinel', async () => {
|
||||
if (!process.env.ANTHROPIC_API_KEY) {
|
||||
@@ -158,7 +159,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic-
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 3,
|
||||
allowedTools: ['Bash'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'hermetic-sentinel',
|
||||
runId,
|
||||
model: CANARY_MODEL,
|
||||
@@ -188,7 +189,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic-
|
||||
fs.rmSync(workDir, { recursive: true, force: true });
|
||||
fs.rmSync(poisonRoot, { recursive: true, force: true });
|
||||
}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
afterAll(() => finalizeEvalCollector(evalCollector));
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
// intentionally machine-specific.
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { cpSync, existsSync, mkdtempSync, readFileSync, rmSync, unlinkSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
@@ -856,7 +857,7 @@ describe('ios device deployment (explicit opt-in)', () => {
|
||||
keepalive?.stop();
|
||||
rmSync(workDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 600_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// Always-on instructions if not paired. Surfaces actionable steps even when
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
// gated (no compilation step for DebugBridgeCore/UI)
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { readFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
@@ -321,7 +322,7 @@ describeIfSwift('swift build invariants', () => {
|
||||
console.error('swift build stderr:', r.stderr?.toString().slice(0, 4000));
|
||||
}
|
||||
expect(r.status).toBe(0);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
test('XCTest suite for StateServer passes (validates real Swift impl)', () => {
|
||||
const r = spawnSync('swift', ['test', '--filter', 'DebugBridgeCoreTests'], {
|
||||
@@ -342,7 +343,7 @@ describeIfSwift('swift build invariants', () => {
|
||||
// Guard against an empty pass-by-no-tests (filter typo / target rename):
|
||||
// we expect at least one StateServer smoke test to actually execute.
|
||||
expect(combined).toContain('StateServerSmokeTests');
|
||||
}, 240_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// Codex-flagged: Release-build guard must be STRUCTURAL, not advisory.
|
||||
// The Package.swift's `.when(configuration: .debug)` setting causes Swift
|
||||
@@ -386,5 +387,5 @@ describeIfSwift('swift build invariants', () => {
|
||||
}
|
||||
}
|
||||
expect(foundForbidden).toBe(0);
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId, evalsEnabled,
|
||||
@@ -103,7 +104,7 @@ IMPORTANT:
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'learnings-show',
|
||||
runId,
|
||||
});
|
||||
@@ -134,5 +135,5 @@ IMPORTANT:
|
||||
} else {
|
||||
console.warn(`Only ${foundCount}/3 learnings found (N+1: ${mentionsNPlusOne}, cache: ${mentionsCache}, rubocop: ${mentionsRubocop})`);
|
||||
}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillObservation, planFileHasDecisionsSection } from './helpers/claude-pty-runner';
|
||||
|
||||
@@ -30,7 +31,7 @@ describeE2E('office-hours AskUserQuestion-blocked smoke (gate)', () => {
|
||||
skillName: 'office-hours',
|
||||
inPlanMode: true,
|
||||
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
});
|
||||
|
||||
if (
|
||||
@@ -55,5 +56,5 @@ describeE2E('office-hours AskUserQuestion-blocked smoke (gate)', () => {
|
||||
}
|
||||
}
|
||||
expect(['asked', 'plan_ready']).toContain(obs.outcome);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
@@ -36,6 +36,7 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { execFileSync, spawnSync } from 'child_process';
|
||||
import {
|
||||
chmodSync,
|
||||
@@ -222,7 +223,7 @@ Generate the design doc per Phase 5. The feature-slug value to substitute into t
|
||||
This is a test of the brain-writeback path. Do NOT skip the gbrain save step under any circumstance — the runtime guard ("skip if gbrain not on PATH") does NOT apply here because gbrain IS available. Do NOT explore gbrain --help; follow the SAVE_RESULTS template's exact CLI shape. If you encounter any AskUserQuestion, auto-decide recommended.`,
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 12,
|
||||
timeout: 360_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'office-hours-brain-writeback',
|
||||
runId,
|
||||
model: 'claude-sonnet-4-6',
|
||||
@@ -315,7 +316,7 @@ This is a test of the brain-writeback path. Do NOT skip the gbrain save step und
|
||||
);
|
||||
}
|
||||
},
|
||||
420_000,
|
||||
CAPTURE_LONG_MS,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
* test turns out stable.
|
||||
*/
|
||||
import { expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId,
|
||||
@@ -133,7 +134,7 @@ ${captureInstruction(outFile)}
|
||||
After writing the file with that ONE Phase 4 question, stop. Do not continue to Phase 4.5 or Phase 5.`,
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 12,
|
||||
timeout: 300_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'office-hours-phase4-fork',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -162,7 +163,7 @@ After writing the file with that ONE Phase 4 question, stop. Do not continue to
|
||||
result,
|
||||
passed: ['success', 'error_max_turns'].includes(result.exitReason),
|
||||
});
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, browseBin, runId, evalsEnabled,
|
||||
@@ -71,7 +72,7 @@ Assume the founder has already answered Q1 (strongest evidence = "got on a waitl
|
||||
Write Q3 output — the forcing question you would ask this founder — to ${workDir}/q3.md. Write ONLY the question prose. No conversational wrapper, no meta-commentary, no Q1/Q2 recap.`,
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 8,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'office-hours-forcing-energy',
|
||||
runId,
|
||||
model: 'claude-sonnet-4-6',
|
||||
@@ -94,7 +95,7 @@ Write Q3 output — the forcing question you would ask this founder — to ${wor
|
||||
console.log('Forcing energy scores:', JSON.stringify(scores, null, 2));
|
||||
expect(scores.axis_a).toBeGreaterThanOrEqual(4); // stacking_preserved
|
||||
expect(scores.axis_b).toBeGreaterThanOrEqual(4); // domain_matched_consequence
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// --- Office Hours builder-mode wildness ---
|
||||
@@ -143,7 +144,7 @@ The user has confirmed the basic idea is "TypeScript + D3 web tool, start with J
|
||||
Write your response — the three adjacent unlocks — to ${workDir}/unlocks.md. Write ONLY the response prose. No meta-commentary, no mode recap. Lead with the fun; let me edit it down later.`,
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 8,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'office-hours-builder-wildness',
|
||||
runId,
|
||||
model: 'claude-sonnet-4-6',
|
||||
@@ -166,7 +167,7 @@ Write your response — the three adjacent unlocks — to ${workDir}/unlocks.md.
|
||||
console.log('Builder wildness scores:', JSON.stringify(scores, null, 2));
|
||||
expect(scores.axis_a).toBeGreaterThanOrEqual(4); // unexpected_combinations
|
||||
expect(scores.axis_b).toBeGreaterThanOrEqual(4); // excitement_over_optimization
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// Finalize eval collector for this file
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import { EvalCollector } from './helpers/eval-store';
|
||||
import { extractSkillHead } from './helpers/skill-fixture';
|
||||
@@ -204,7 +205,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
|
||||
workingDirectory: armA,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 90_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'fanout-arm-overlay-on',
|
||||
runId,
|
||||
model: OPUS_47,
|
||||
@@ -214,7 +215,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
|
||||
workingDirectory: armB,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 90_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'fanout-arm-overlay-off',
|
||||
runId,
|
||||
model: OPUS_47,
|
||||
@@ -262,7 +263,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
|
||||
fs.rmSync(armB, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
240_000,
|
||||
CAPTURE_MS,
|
||||
);
|
||||
|
||||
test(
|
||||
@@ -281,7 +282,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
|
||||
workingDirectory: root,
|
||||
maxTurns: 3,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 90_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: `routing-${c.name}`,
|
||||
runId,
|
||||
model: OPUS_47,
|
||||
@@ -348,6 +349,6 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
360_000,
|
||||
CAPTURE_LONG_MS,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
*/
|
||||
|
||||
import { test } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
|
||||
import { FORCING_FLOOR_CEO } from './fixtures/forcing-finding-seeds';
|
||||
@@ -22,7 +23,7 @@ describeE2E('/plan-ceo-review AskUserQuestion floor (gate)', () => {
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 600_000,
|
||||
timeoutMs: CAPTURE_LONG_MS,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
@@ -34,6 +35,6 @@ describeE2E('/plan-ceo-review AskUserQuestion floor (gate)', () => {
|
||||
);
|
||||
}
|
||||
},
|
||||
660_000,
|
||||
PTY_MS,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -31,6 +31,7 @@
|
||||
*/
|
||||
|
||||
import { test } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import {
|
||||
launchClaudePty,
|
||||
@@ -151,7 +152,7 @@ describeE2E('/plan-ceo-review mode routing (gate)', () => {
|
||||
async () => {
|
||||
const session = await launchClaudePty({
|
||||
permissionMode: 'plan',
|
||||
timeoutMs: 540_000,
|
||||
timeoutMs: CAPTURE_LONG_MS,
|
||||
seedSkills: true,
|
||||
});
|
||||
try {
|
||||
@@ -207,7 +208,7 @@ describeE2E('/plan-ceo-review mode routing (gate)', () => {
|
||||
await session.close();
|
||||
}
|
||||
},
|
||||
600_000,
|
||||
CAPTURE_LONG_MS,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -34,6 +34,7 @@
|
||||
*/
|
||||
|
||||
import { test } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import {
|
||||
runPlanSkillObservation,
|
||||
@@ -77,5 +78,5 @@ describeE2E('plan-ceo-review plan-mode smoke (gate)', () => {
|
||||
);
|
||||
}
|
||||
assertReportAtBottomIfPlanWritten(obs);
|
||||
}, 480_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import {
|
||||
setupSkillDir,
|
||||
@@ -87,6 +88,6 @@ describeE2E('/plan-ceo-review section-loading E2E (periodic, SDK capture)', () =
|
||||
// Guard against an empty pass: the report must have real content.
|
||||
expect(output.trim().length).toBeGreaterThan(200);
|
||||
},
|
||||
360_000,
|
||||
CAPTURE_LONG_MS,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
*/
|
||||
|
||||
import { test } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
|
||||
import { FORCING_FLOOR_DESIGN } from './fixtures/forcing-finding-seeds';
|
||||
@@ -22,7 +23,7 @@ describeE2E('/plan-design-review AskUserQuestion floor (periodic)', () => {
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 600_000,
|
||||
timeoutMs: CAPTURE_LONG_MS,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
@@ -34,6 +35,6 @@ describeE2E('/plan-design-review AskUserQuestion floor (periodic)', () => {
|
||||
);
|
||||
}
|
||||
},
|
||||
660_000,
|
||||
PTY_MS,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import {
|
||||
runPlanSkillObservation,
|
||||
@@ -46,7 +47,7 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => {
|
||||
const obs = await runPlanSkillObservation({
|
||||
skillName: 'plan-design-review',
|
||||
inPlanMode: true,
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
});
|
||||
|
||||
if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') {
|
||||
@@ -59,7 +60,7 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => {
|
||||
}
|
||||
expect(['asked', 'plan_ready']).toContain(obs.outcome);
|
||||
assertReportAtBottomIfPlanWritten(obs);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
|
||||
// Plan-mode scope-gate bypass: with a seeded UI-heavy plan in plan mode,
|
||||
// the gate must NOT render its "What should I review?" menu — it
|
||||
@@ -71,7 +72,7 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => {
|
||||
skillName: 'plan-design-review',
|
||||
inPlanMode: true,
|
||||
initialPlanContent: SEED_PLAN_UI_HEAVY,
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
});
|
||||
|
||||
if (
|
||||
@@ -95,5 +96,5 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => {
|
||||
// though the seed arrives as a pasted user message).
|
||||
expect(obs.scopeGateQuestionObserved ?? false).toBe(false);
|
||||
expect(obs.scopeGateAutoSelectObserved ?? false).toBe(true);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
*/
|
||||
|
||||
import { test } from 'bun:test';
|
||||
import { PTY_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as path from 'path';
|
||||
import {
|
||||
@@ -46,7 +47,7 @@ describeE2E('/plan-design-review with UI scope (gate)', () => {
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — skill registry,
|
||||
// hermetic pre-trusted dir, and the repo-relative fixture path above.
|
||||
cwd: ROOT,
|
||||
timeoutMs: 720_000,
|
||||
timeoutMs: PTY_MS,
|
||||
seedSkills: true,
|
||||
});
|
||||
|
||||
@@ -152,6 +153,6 @@ describeE2E('/plan-design-review with UI scope (gate)', () => {
|
||||
);
|
||||
}
|
||||
},
|
||||
780_000,
|
||||
PTY_MS,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
*/
|
||||
|
||||
import { test } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
|
||||
import { FORCING_FLOOR_DEVEX } from './fixtures/forcing-finding-seeds';
|
||||
@@ -22,7 +23,7 @@ describeE2E('/plan-devex-review AskUserQuestion floor (gate)', () => {
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 600_000,
|
||||
timeoutMs: CAPTURE_LONG_MS,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
@@ -34,6 +35,6 @@ describeE2E('/plan-devex-review AskUserQuestion floor (gate)', () => {
|
||||
);
|
||||
}
|
||||
},
|
||||
660_000,
|
||||
PTY_MS,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import {
|
||||
runPlanSkillObservation,
|
||||
@@ -20,7 +21,7 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => {
|
||||
const obs = await runPlanSkillObservation({
|
||||
skillName: 'plan-devex-review',
|
||||
inPlanMode: true,
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
});
|
||||
|
||||
if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') {
|
||||
@@ -33,7 +34,7 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => {
|
||||
}
|
||||
expect(['asked', 'plan_ready']).toContain(obs.outcome);
|
||||
assertReportAtBottomIfPlanWritten(obs);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
|
||||
// v1.21+ regression: see skill-e2e-plan-ceo-plan-mode.test.ts for the
|
||||
// contract. Pass envelope is ['asked', 'plan_ready']; failure signals
|
||||
@@ -44,7 +45,7 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => {
|
||||
skillName: 'plan-devex-review',
|
||||
inPlanMode: true,
|
||||
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
});
|
||||
|
||||
if (
|
||||
@@ -70,5 +71,5 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => {
|
||||
}
|
||||
expect(['asked', 'plan_ready']).toContain(obs.outcome);
|
||||
assertReportAtBottomIfPlanWritten(obs);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
*/
|
||||
|
||||
import { test } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
|
||||
import { FORCING_FLOOR_ENG } from './fixtures/forcing-finding-seeds';
|
||||
@@ -33,7 +34,7 @@ describeE2E('/plan-eng-review AskUserQuestion floor (periodic)', () => {
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 600_000,
|
||||
timeoutMs: CAPTURE_LONG_MS,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
@@ -49,6 +50,6 @@ describeE2E('/plan-eng-review AskUserQuestion floor (periodic)', () => {
|
||||
);
|
||||
}
|
||||
},
|
||||
660_000,
|
||||
PTY_MS,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import {
|
||||
runPlanSkillObservation,
|
||||
@@ -50,7 +51,7 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => {
|
||||
const obs = await runPlanSkillObservation({
|
||||
skillName: 'plan-eng-review',
|
||||
inPlanMode: true,
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
});
|
||||
|
||||
if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') {
|
||||
@@ -63,7 +64,7 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => {
|
||||
}
|
||||
expect(['asked', 'plan_ready']).toContain(obs.outcome);
|
||||
assertReportAtBottomIfPlanWritten(obs);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
|
||||
// D3-B / D4-B: when a plan with guaranteed-finding-triggering complexity
|
||||
// is seeded, the skill MUST fire AskUserQuestion (or fall back to a
|
||||
@@ -79,7 +80,7 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => {
|
||||
// must use mcp__*__AskUserQuestion (outcome='asked') or fall back to
|
||||
// writing Decisions ('plan_ready').
|
||||
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
});
|
||||
|
||||
if (
|
||||
@@ -118,5 +119,5 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => {
|
||||
// question.
|
||||
expect(obs.scopeGateQuestionObserved ?? false).toBe(false);
|
||||
expect(obs.scopeGateAutoSelectObserved ?? false).toBe(true);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
* accordingly.
|
||||
*/
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId,
|
||||
@@ -134,7 +135,7 @@ ${captureInstruction(outFile)}
|
||||
After writing the file, stop. Do not continue the review.`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 10,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'plan-ceo-review-format-mode',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -160,7 +161,7 @@ After writing the file, stop. Do not continue the review.`,
|
||||
result,
|
||||
passed: ['success', 'error_max_turns'].includes(result.exitReason),
|
||||
});
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Case 2: plan-ceo-review approach menu (coverage-differentiated) ---
|
||||
@@ -191,7 +192,7 @@ ${captureInstruction(outFile)}
|
||||
After writing the file, stop. Do not continue the review.`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 10,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'plan-ceo-review-format-approach',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -216,7 +217,7 @@ After writing the file, stop. Do not continue the review.`,
|
||||
result,
|
||||
passed: ['success', 'error_max_turns'].includes(result.exitReason),
|
||||
});
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Case 3: plan-eng-review coverage-differentiated per-issue AskUserQuestion ---
|
||||
@@ -250,7 +251,7 @@ ${captureInstruction(outFile)}
|
||||
After writing the file with that ONE question, stop. Do not continue the review.`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 10,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'plan-eng-review-format-coverage',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -275,7 +276,7 @@ After writing the file with that ONE question, stop. Do not continue the review.
|
||||
result,
|
||||
passed: ['success', 'error_max_turns'].includes(result.exitReason),
|
||||
});
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Case 4: plan-eng-review kind-differentiated per-issue AskUserQuestion ---
|
||||
@@ -306,7 +307,7 @@ ${captureInstruction(outFile)}
|
||||
After writing the file with that ONE question, stop. Do not continue the review.`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 10,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'plan-eng-review-format-kind',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -332,7 +333,7 @@ After writing the file with that ONE question, stop. Do not continue the review.
|
||||
result,
|
||||
passed: ['success', 'error_max_turns'].includes(result.exitReason),
|
||||
});
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
|
||||
@@ -31,6 +31,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillObservation } from './helpers/claude-pty-runner';
|
||||
|
||||
@@ -62,7 +63,7 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
|
||||
const obs = await runPlanSkillObservation({
|
||||
skillName,
|
||||
inPlanMode: false,
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
// eng/design: force the prose-fallback path. The unconditional
|
||||
// gate-must-ask assert below pins the render shape the detector
|
||||
// anchors on, and only the --disallowedTools prose fallback makes
|
||||
@@ -115,7 +116,7 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
|
||||
);
|
||||
}
|
||||
}
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
}
|
||||
|
||||
// Named-target exception (outside plan mode): a pasted draft IS an
|
||||
@@ -130,7 +131,7 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
|
||||
inPlanMode: false,
|
||||
initialPlanContent: NAMED_TARGET_SEED,
|
||||
trackTokens: [SEED_TOKEN],
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
});
|
||||
|
||||
if (
|
||||
@@ -159,5 +160,5 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
|
||||
// unreachable outside plan mode (extractPlanFilePath only matches
|
||||
// plan-mode save renders).
|
||||
expect(obs.tokensObserved?.[SEED_TOKEN] ?? false).toBe(true);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
@@ -27,6 +27,7 @@
|
||||
* cases will land as follow-up PRs per skill.
|
||||
*/
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId,
|
||||
@@ -161,7 +162,7 @@ ${captureInstruction(outFile)}
|
||||
After writing the file, stop.`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 10,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'plan-review-prosons-format',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -191,7 +192,7 @@ After writing the file, stop.`,
|
||||
|
||||
// (recommended) label on one option
|
||||
expect(captured).toMatch(RECOMMENDED_LABEL_RE);
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Case 2: Hard-stop escape NEGATIVE (CT2) ---
|
||||
@@ -220,7 +221,7 @@ ${captureInstruction(outFile)}
|
||||
After writing the file, stop.`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 10,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'plan-review-prosons-hardstop-neg',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -241,7 +242,7 @@ After writing the file, stop.`,
|
||||
// Must have real pros and cons (≥2 ✅ + ≥1 ❌ per option)
|
||||
expect(countChars(captured, '✅')).toBeGreaterThanOrEqual(4);
|
||||
expect(countChars(captured, '❌')).toBeGreaterThanOrEqual(2);
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Case 3: Neutral-posture NEGATIVE (CT2) ---
|
||||
@@ -270,7 +271,7 @@ ${captureInstruction(outFile)}
|
||||
After writing the file, stop.`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 10,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'plan-review-prosons-neutral-neg',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -292,7 +293,7 @@ After writing the file, stop.`,
|
||||
expect(captured).toMatch(RECOMMENDED_LABEL_RE);
|
||||
// Recommendation line must contain "because" (concrete reason, not "no preference")
|
||||
expect(captured).toMatch(/[Rr]ecommendation:.*because/);
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Case 4: Hard-stop POSITIVE (escape allowed when legitimately one-sided) ---
|
||||
@@ -321,7 +322,7 @@ ${captureInstruction(outFile)}
|
||||
After writing the file, stop.`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 10,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'plan-ceo-review-prosons-cadence',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -344,7 +345,7 @@ After writing the file, stop.`,
|
||||
const hasEscape = HARD_STOP_ESCAPE_RE.test(captured);
|
||||
const hasProsAndCons = countChars(captured, '✅') >= 1 && countChars(captured, '❌') >= 1;
|
||||
expect(hasEscape || hasProsAndCons).toBe(true);
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { beforeAll, afterAll, expect } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId,
|
||||
@@ -151,7 +152,7 @@ IMPORTANT:
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'plan-tune-inspect',
|
||||
runId,
|
||||
});
|
||||
@@ -184,5 +185,5 @@ IMPORTANT:
|
||||
if (!noticedOverride) {
|
||||
console.warn('Agent did not surface override/skip behavior from the log');
|
||||
}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
+21
-20
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, browseBin, runId, evalsEnabled,
|
||||
@@ -103,7 +104,7 @@ Focus on reviewing the plan content: architecture, error handling, security, and
|
||||
const review = fs.readFileSync(reviewPath, 'utf-8');
|
||||
expect(review.length).toBeGreaterThan(200);
|
||||
}
|
||||
}, 660_000);
|
||||
}, PTY_MS);
|
||||
});
|
||||
|
||||
// --- Plan CEO Review (SELECTIVE EXPANSION) E2E ---
|
||||
@@ -171,7 +172,7 @@ Write your complete review directly to ${planDir}/review-output-selective.md
|
||||
Focus on reviewing the plan content: architecture, error handling, security, and performance.`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 15,
|
||||
timeout: 540_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'plan-ceo-review-selective',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -188,7 +189,7 @@ Focus on reviewing the plan content: architecture, error handling, security, and
|
||||
const review = fs.readFileSync(reviewPath, 'utf-8');
|
||||
expect(review.length).toBeGreaterThan(200);
|
||||
}
|
||||
}, 660_000);
|
||||
}, PTY_MS);
|
||||
});
|
||||
|
||||
// --- Plan CEO Review SCOPE EXPANSION energy (V1.1 mode-posture regression gate) ---
|
||||
@@ -239,7 +240,7 @@ Choose SCOPE EXPANSION mode. Skip any AskUserQuestion calls — this is non-inte
|
||||
Write your expansion proposals to ${planDir}/proposals.md with ONLY the proposal text — no conversational wrapper, no review summary, no mode analysis. Each proposal separated by "---".`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 15,
|
||||
timeout: 360_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'plan-ceo-review-expansion-energy',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -270,7 +271,7 @@ Write your expansion proposals to ${planDir}/proposals.md with ONLY the proposal
|
||||
// Pass threshold: 4/5 on both axes (good — matches posture with minor weakness).
|
||||
expect(scores.axis_a).toBeGreaterThanOrEqual(4); // surface_framing
|
||||
expect(scores.axis_b).toBeGreaterThanOrEqual(4); // decision_preservation
|
||||
}, 600_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// --- Plan Eng Review E2E ---
|
||||
@@ -348,7 +349,7 @@ Write your complete review directly to ${planDir}/review-output.md
|
||||
Focus on architecture, code quality, tests, and performance sections.`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 15,
|
||||
timeout: 360_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'plan-eng-review',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -366,7 +367,7 @@ Focus on architecture, code quality, tests, and performance sections.`,
|
||||
const review = fs.readFileSync(reviewPath, 'utf-8');
|
||||
expect(review.length).toBeGreaterThan(200);
|
||||
}
|
||||
}, 420_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// --- Plan-Eng-Review Test-Plan Artifact E2E ---
|
||||
@@ -476,7 +477,7 @@ Write your review to ${planDir}/review-output.md`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 25,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Glob', 'Grep'],
|
||||
timeout: 360_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'plan-eng-review-artifact',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -507,7 +508,7 @@ Write your review to ${planDir}/review-output.md`,
|
||||
if (newFiles.length === 0) {
|
||||
console.warn('SOFT FAIL: No test-plan artifact written — agent did not follow artifact instructions');
|
||||
}
|
||||
}, 420_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// --- Office Hours Spec Review E2E ---
|
||||
@@ -559,7 +560,7 @@ Write your summary to ${ohDir}/spec-review-summary.md`,
|
||||
// failures wrote a correct summary on tool-turn 8 and hit the cap on
|
||||
// the closing text turn (error_max_turns at 9 turns, deterministic).
|
||||
maxTurns: 12,
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'office-hours-spec-review',
|
||||
runId,
|
||||
});
|
||||
@@ -575,7 +576,7 @@ Write your summary to ${ohDir}/spec-review-summary.md`,
|
||||
expect(summary).toMatch(/agent|subagent/);
|
||||
expect(summary).toMatch(/3.*iteration|iteration.*3|maximum.*3/);
|
||||
}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Plan CEO Review Benefits-From E2E ---
|
||||
@@ -619,7 +620,7 @@ Summarize what happens when no design doc is found — specifically:
|
||||
Write your summary to ${benefitsDir}/benefits-summary.md`,
|
||||
workingDirectory: benefitsDir,
|
||||
maxTurns: 8,
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'plan-ceo-review-benefits',
|
||||
runId,
|
||||
});
|
||||
@@ -634,7 +635,7 @@ Write your summary to ${benefitsDir}/benefits-summary.md`,
|
||||
expect(summary).toMatch(/office.hours/);
|
||||
expect(summary).toMatch(/design doc|no design/i);
|
||||
}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Plan Review Report E2E ---
|
||||
@@ -706,7 +707,7 @@ CRITICAL REQUIREMENT: plan.md IS the plan file for this review session. After co
|
||||
This review report at the bottom of the plan is the MOST IMPORTANT deliverable of this test.`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 20,
|
||||
timeout: 360_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'plan-review-report',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -768,7 +769,7 @@ This review report at the bottom of the plan is the MOST IMPORTANT deliverable o
|
||||
).toBe(true);
|
||||
|
||||
console.log('Plan review report found at bottom of plan.md (ends with unresolved status)');
|
||||
}, 420_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// --- Codex Offering E2E ---
|
||||
@@ -825,7 +826,7 @@ Summarize the Codex/${featureName} integration — answer these specific questio
|
||||
Write your summary to ${testDir}/${testName}-summary.md`,
|
||||
workingDirectory: testDir,
|
||||
maxTurns: 8,
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
@@ -850,19 +851,19 @@ Write your summary to ${testDir}/${testName}-summary.md`,
|
||||
|
||||
testConcurrentIfSelected('codex-offered-office-hours', async () => {
|
||||
await checkCodexOffering('office-hours', 'codex-offered-office-hours', 'second opinion');
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
testConcurrentIfSelected('codex-offered-ceo-review', async () => {
|
||||
await checkCodexOffering('plan-ceo-review', 'codex-offered-ceo-review', 'outside voice');
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
testConcurrentIfSelected('codex-offered-design-review', async () => {
|
||||
await checkCodexOffering('plan-design-review', 'codex-offered-design-review', 'design outside voices');
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
testConcurrentIfSelected('codex-offered-eng-review', async () => {
|
||||
await checkCodexOffering('plan-eng-review', 'codex-offered-eng-review', 'outside voice');
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import { outcomeJudge } from './helpers/llm-judge';
|
||||
import { judgePassed } from './helpers/eval-store';
|
||||
@@ -97,7 +98,7 @@ CRITICAL RULES:
|
||||
- The report MUST exist at ${reportPath} when you finish`,
|
||||
workingDirectory: testWorkDir,
|
||||
maxTurns: 50,
|
||||
timeout: 300_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: `qa-${label}`,
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -174,17 +175,17 @@ CRITICAL RULES:
|
||||
// B6: Static dashboard — broken link, disabled submit, overflow, missing alt, console error
|
||||
testConcurrentIfSelected('qa-b6-static', async () => {
|
||||
await runPlantedBugEval('qa-eval.html', 'qa-eval-ground-truth.json', 'b6-static');
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
|
||||
// B7: SPA — broken route, stale state, async race, missing aria, console warning
|
||||
testConcurrentIfSelected('qa-b7-spa', async () => {
|
||||
await runPlantedBugEval('qa-eval-spa.html', 'qa-eval-spa-ground-truth.json', 'b7-spa');
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
|
||||
// B8: Checkout — email regex, NaN total, CC overflow, missing required, stripe error
|
||||
testConcurrentIfSelected('qa-b8-checkout', async () => {
|
||||
await runPlantedBugEval('qa-eval-checkout.html', 'qa-eval-checkout-ground-truth.json', 'b8-checkout');
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
|
||||
});
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, browseBin, runId, evalsEnabled,
|
||||
@@ -54,7 +55,7 @@ Do NOT try to start a server or discover ports — the URL above is ready.
|
||||
Write your report to ${qaDir}/qa-reports/qa-report.md`,
|
||||
workingDirectory: qaDir,
|
||||
maxTurns: 35,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'qa-quick',
|
||||
runId,
|
||||
});
|
||||
@@ -69,7 +70,7 @@ Write your report to ${qaDir}/qa-reports/qa-report.md`,
|
||||
}
|
||||
// Accept error_max_turns — the agent doing thorough QA work is not a failure
|
||||
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- QA-Only E2E (report-only, no fixes) ---
|
||||
@@ -124,7 +125,7 @@ Write your report to ${qaOnlyDir}/qa-reports/qa-only-report.md`,
|
||||
workingDirectory: qaOnlyDir,
|
||||
maxTurns: 40,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Glob'], // NO Edit — the critical guardrail
|
||||
timeout: 180_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'qa-only-no-fix',
|
||||
runId,
|
||||
});
|
||||
@@ -156,7 +157,7 @@ Write your report to ${qaOnlyDir}/qa-reports/qa-only-report.md`,
|
||||
(l: string) => l.trim() && !l.includes('.prompt-tmp') && !l.includes('.gstack/') && !l.includes('qa-reports/'),
|
||||
);
|
||||
expect(statusLines.filter((l: string) => l.startsWith(' M') || l.startsWith('M '))).toHaveLength(0);
|
||||
}, 240_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- QA Fix Loop E2E ---
|
||||
@@ -247,7 +248,7 @@ This is a test+fix loop: find bugs, fix them in the source code, commit each fix
|
||||
workingDirectory: qaFixDir,
|
||||
maxTurns: 40,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
|
||||
timeout: 420_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'qa-fix-loop',
|
||||
runId,
|
||||
});
|
||||
@@ -271,7 +272,7 @@ This is a test+fix loop: find bugs, fix them in the source code, commit each fix
|
||||
// Verify Edit tool was used (agent actually modified source code)
|
||||
const editCalls = result.toolCalls.filter(tc => tc.tool === 'Edit');
|
||||
expect(editCalls.length).toBeGreaterThan(0);
|
||||
}, 480_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// --- Test Bootstrap E2E ---
|
||||
@@ -384,7 +385,7 @@ Do NOT fix any bugs. Do NOT use AskUserQuestion — just pick vitest.`,
|
||||
workingDirectory: bsDir,
|
||||
maxTurns: 12,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'],
|
||||
timeout: 90_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'qa-bootstrap',
|
||||
runId,
|
||||
});
|
||||
@@ -405,7 +406,7 @@ Do NOT fix any bugs. Do NOT use AskUserQuestion — just pick vitest.`,
|
||||
console.log(`Test config: ${hasTestConfig}, Test file: ${hasTestFile}, TESTING.md: ${hasTestingMd}`);
|
||||
|
||||
try { fs.rmSync(bsDir, { recursive: true, force: true }); } catch {}
|
||||
}, 120_000);
|
||||
}, JUDGE_MS);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete.
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId,
|
||||
@@ -135,7 +136,7 @@ Write your retrospective to ${dir}/retro-output.md`,
|
||||
expect(wroteReport).toBe(true);
|
||||
const content = fs.readFileSync(retroPath, 'utf-8');
|
||||
expect(content.length).toBeGreaterThan(100);
|
||||
}, 480_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// --- Retro E2E ---
|
||||
@@ -198,7 +199,7 @@ Write your retrospective report to ${retroDir}/retro-output.md
|
||||
Analyze the git history and produce the narrative report as described in the SKILL.md.`,
|
||||
workingDirectory: retroDir,
|
||||
maxTurns: 30,
|
||||
timeout: 300_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'retro',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -217,7 +218,7 @@ Analyze the git history and produce the narrative report as described in the SKI
|
||||
expect(wroteReport).toBe(true);
|
||||
const retro = fs.readFileSync(retroPath, 'utf-8');
|
||||
expect(retro.length).toBeGreaterThan(100);
|
||||
}, 420_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId, describeIfSelected, testConcurrentIfSelected,
|
||||
@@ -114,7 +115,7 @@ and apply it yourself against the diff (git diff main...HEAD).
|
||||
Write your findings to ${dir}/review-output.md`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 20,
|
||||
timeout: 180_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'review-army-migration-safety',
|
||||
runId,
|
||||
});
|
||||
@@ -135,7 +136,7 @@ Write your findings to ${dir}/review-output.md`,
|
||||
content.includes('column');
|
||||
expect(hasMigrationFinding).toBe(true);
|
||||
}
|
||||
}, 210_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Review Army: N+1 Performance ---
|
||||
@@ -179,7 +180,7 @@ For the specialist dispatch, read review-specialists/performance.md and apply it
|
||||
Write your findings to ${dir}/review-output.md`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 20,
|
||||
timeout: 180_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'review-army-perf-n-plus-one',
|
||||
runId,
|
||||
});
|
||||
@@ -201,7 +202,7 @@ Write your findings to ${dir}/review-output.md`,
|
||||
content.includes('loop');
|
||||
expect(hasN1Finding).toBe(true);
|
||||
}
|
||||
}, 210_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Review Army: Delivery Audit ---
|
||||
@@ -281,7 +282,7 @@ The email notification system should be classified as NOT DONE.
|
||||
Write your completion audit to ${dir}/review-output.md`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 15,
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'review-army-delivery-audit',
|
||||
runId,
|
||||
});
|
||||
@@ -305,7 +306,7 @@ Write your completion audit to ${dir}/review-output.md`,
|
||||
expect(hasNotDone).toBe(true);
|
||||
expect(mentionsEmail).toBe(true);
|
||||
}
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Review Army: Quality Score ---
|
||||
@@ -356,7 +357,7 @@ Write your findings AND the computed quality score to ${dir}/review-output.md
|
||||
Include the line: "PR Quality Score: X/10" where X is the computed score.`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 15,
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'review-army-quality-score',
|
||||
runId,
|
||||
});
|
||||
@@ -374,7 +375,7 @@ Include the line: "PR Quality Score: X/10" where X is the computed score.`,
|
||||
content.match(/\d+\/10/);
|
||||
expect(hasScore).toBeTruthy();
|
||||
}
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Review Army: JSON Findings ---
|
||||
@@ -421,7 +422,7 @@ Output your findings as JSON objects, one per line, following the schema:
|
||||
Write ONLY JSON findings (no preamble) to ${dir}/findings.json`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 12,
|
||||
timeout: 90_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'review-army-json-findings',
|
||||
runId,
|
||||
});
|
||||
@@ -450,7 +451,7 @@ Write ONLY JSON findings (no preamble) to ${dir}/findings.json`,
|
||||
break; // One valid line is enough for the gate test
|
||||
}
|
||||
}
|
||||
}, 120_000);
|
||||
}, JUDGE_MS);
|
||||
});
|
||||
|
||||
// --- Review Army: Red Team (periodic) ---
|
||||
@@ -499,7 +500,7 @@ Write your red team findings to ${dir}/review-output.md
|
||||
Start the file with "RED TEAM REVIEW" on the first line.`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 20,
|
||||
timeout: 180_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'review-army-red-team',
|
||||
runId,
|
||||
});
|
||||
@@ -513,7 +514,7 @@ Start the file with "RED TEAM REVIEW" on the first line.`,
|
||||
const content = fs.readFileSync(outputPath, 'utf-8');
|
||||
expect(content.toLowerCase()).toMatch(/red team|adversarial/);
|
||||
}
|
||||
}, 210_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Review Army: Consensus (periodic) ---
|
||||
@@ -566,7 +567,7 @@ mark it as "MULTI-SPECIALIST CONFIRMED" with the confirming categories.
|
||||
Write findings to ${dir}/review-output.md`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 20,
|
||||
timeout: 180_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'review-army-consensus',
|
||||
runId,
|
||||
});
|
||||
@@ -585,7 +586,7 @@ Write findings to ${dir}/review-output.md`,
|
||||
content.includes('interpolat');
|
||||
expect(hasSqlFinding).toBe(true);
|
||||
}
|
||||
}, 210_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// Finalize eval collector
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId,
|
||||
@@ -65,7 +66,7 @@ Then run git diff against the detected base branch and write a brief review.
|
||||
Write your findings to ${dir}/review-output.md`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 15,
|
||||
timeout: 90_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'review-base-branch',
|
||||
runId,
|
||||
});
|
||||
@@ -84,7 +85,7 @@ Write your findings to ${dir}/review-output.md`,
|
||||
return cmd.includes('git diff');
|
||||
});
|
||||
expect(usedGitDiff).toBe(true);
|
||||
}, 120_000);
|
||||
}, JUDGE_MS);
|
||||
|
||||
testConcurrentIfSelected('ship-base-branch', async () => {
|
||||
const dir = path.join(baseBranchDir, 'ship-base');
|
||||
@@ -125,7 +126,7 @@ Write a summary to ${dir}/ship-preflight.md including:
|
||||
- The diff stat against the base branch`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 18,
|
||||
timeout: 150_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'ship-base-branch',
|
||||
runId,
|
||||
});
|
||||
@@ -155,7 +156,7 @@ Write a summary to ${dir}/ship-preflight.md including:
|
||||
return command.includes('git push') || command.includes('gh pr create');
|
||||
});
|
||||
expect(destructiveTools).toHaveLength(0);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Review Dashboard Via Attribution E2E ---
|
||||
@@ -280,7 +281,7 @@ Write the dashboard output to ${dashDir}/dashboard-output.md`,
|
||||
);
|
||||
// Ship dashboard should not gate when eng review is clear
|
||||
expect(gateQuestions).toHaveLength(0);
|
||||
}, 480_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, browseBin, runId, evalsEnabled, selectedTests,
|
||||
@@ -66,7 +67,7 @@ Run /review on the current diff (git diff main...HEAD).
|
||||
Write your review findings to ${reviewDir}/review-output.md`,
|
||||
workingDirectory: reviewDir,
|
||||
maxTurns: 20,
|
||||
timeout: 180_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'review-sql-injection',
|
||||
runId,
|
||||
});
|
||||
@@ -89,7 +90,7 @@ Write your review findings to ${reviewDir}/review-output.md`,
|
||||
reviewContent.includes('unsanitized');
|
||||
expect(hasSqlContent).toBe(true);
|
||||
}
|
||||
}, 210_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Review: Enum completeness E2E ---
|
||||
@@ -144,7 +145,7 @@ Write your review findings to ${enumDir}/review-output.md
|
||||
The diff adds a new "returned" status to the Order model. Your job is to check if all consumers handle it.`,
|
||||
workingDirectory: enumDir,
|
||||
maxTurns: 15,
|
||||
timeout: 90_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'review-enum-completeness',
|
||||
runId,
|
||||
});
|
||||
@@ -164,7 +165,7 @@ The diff adds a new "returned" status to the Order model. Your job is to check i
|
||||
expect(mentionsReturned).toBe(true);
|
||||
expect(mentionsEnum || mentionsCritical).toBe(true);
|
||||
}
|
||||
}, 120_000);
|
||||
}, JUDGE_MS);
|
||||
});
|
||||
|
||||
// --- Review: Design review lite E2E ---
|
||||
@@ -229,7 +230,7 @@ Write your review findings to ${designDir}/review-output.md
|
||||
Important: The design checklist should catch issues like blacklisted fonts, small font sizes, outline:none, !important, AI slop patterns (purple gradients, generic hero copy, 3-column feature grid), etc.`,
|
||||
workingDirectory: designDir,
|
||||
maxTurns: 35,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'review-design-lite',
|
||||
runId,
|
||||
});
|
||||
@@ -262,7 +263,7 @@ Important: The design checklist should catch issues like blacklisted fonts, smal
|
||||
console.log(`Design review detected ${detected}/7 planted issues`);
|
||||
expect(detected).toBeGreaterThanOrEqual(4);
|
||||
}
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// Base branch detection tests for review/ship + the Review Dashboard Via
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId, evalsEnabled,
|
||||
@@ -166,7 +167,7 @@ IMPORTANT:
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 10,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'context-recovery-artifacts',
|
||||
runId,
|
||||
});
|
||||
@@ -193,7 +194,7 @@ IMPORTANT:
|
||||
expect(foundCount).toBeGreaterThanOrEqual(1);
|
||||
|
||||
console.log(`Context recovery: artifacts=${foundArtifacts}, lastSession=${foundLastSession}, timeline=${foundTimeline}`);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// --- Test 3: /context-save writes a file ---
|
||||
// Hand-feed the save section of context-save/SKILL.md to claude -p and verify
|
||||
@@ -231,7 +232,7 @@ IMPORTANT:
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 10,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'context-save-writes-file',
|
||||
runId,
|
||||
});
|
||||
@@ -264,7 +265,7 @@ IMPORTANT:
|
||||
expect(hasYamlFrontmatter).toBe(true);
|
||||
|
||||
console.log(`context-save: ${files.length} files created, YAML frontmatter: ${hasYamlFrontmatter}, branch: ${hasBranch}`);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// --- Test 4: /context-restore loads the newest file across branches ---
|
||||
// Seed two saved-context files with different YYYYMMDD-HHMMSS prefixes and
|
||||
@@ -340,7 +341,7 @@ IMPORTANT:
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 8,
|
||||
allowedTools: ['Bash', 'Read', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'context-restore-loads-latest',
|
||||
runId,
|
||||
});
|
||||
@@ -361,5 +362,5 @@ IMPORTANT:
|
||||
expect(loadedOlder).toBe(false);
|
||||
|
||||
console.log(`context-restore: loadedNewer=${loadedNewer}, loadedOlder=${loadedOlder}`);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
// on a failed verify the skill STOPs before any CLAUDE.md write.
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
@@ -155,5 +156,5 @@ describeE2E('/setup-gbrain Path 4 — bad token STOPs cleanly', () => {
|
||||
fs.rmSync(gstackHome, { recursive: true, force: true });
|
||||
fs.rmSync(fakeBinDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 240_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
// Cost: ~$0.50-$1.00 per run. Periodic-tier (EVALS=1 EVALS_TIER=periodic).
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
@@ -264,5 +265,5 @@ describeE2E('/setup-gbrain Path 4 + Step 4.5 Yes → local PGLite for code', ()
|
||||
fs.rmSync(sandboxHome, { recursive: true, force: true });
|
||||
fs.rmSync(fakeBinDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
// block this test asserts on).
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
@@ -245,5 +246,5 @@ describeE2E('/setup-gbrain Path 4 (Remote MCP) — happy path', () => {
|
||||
fs.rmSync(gstackHome, { recursive: true, force: true });
|
||||
fs.rmSync(fakeBinDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 240_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
@@ -49,6 +49,7 @@
|
||||
* gate tier confirmed).
|
||||
*/
|
||||
import { expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
@@ -200,7 +201,7 @@ describeE2E('Ship doc-sync dispatch E2E (gate)', () => {
|
||||
workingDirectory: repoDir,
|
||||
maxTurns: 30,
|
||||
allowedTools: ['Bash', 'Read', 'Grep', 'Glob', 'Write', 'Agent', 'Task'],
|
||||
timeout: 480_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
env: {
|
||||
HOME: workDir,
|
||||
GSTACK_HOME: path.join(workDir, 'gstack-home'),
|
||||
@@ -273,7 +274,7 @@ describeE2E('Ship doc-sync dispatch E2E (gate)', () => {
|
||||
console.log(
|
||||
`dispatchIdx=${dispatchIdx} prCreateIdx=${prCreateIdx} readPrBody=${readPrBody} exit=${result.exitReason}`
|
||||
);
|
||||
}, 540_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { PTY_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
@@ -158,7 +159,7 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => {
|
||||
const session = await launchClaudePty({
|
||||
permissionMode: 'plan',
|
||||
cwd: fixture.workTree,
|
||||
timeoutMs: 1_080_000,
|
||||
timeoutMs: PTY_LONG_MS,
|
||||
// Disable network-y pieces so the agent can't reach actual github.
|
||||
env: { GH_TOKEN: 'mock-not-real', NO_COLOR: '1' },
|
||||
seedSkills: true,
|
||||
@@ -279,6 +280,6 @@ describeE2E('/ship idempotency E2E (periodic, real-PTY)', () => {
|
||||
try { fs.rmSync(path.dirname(fixture.workTree), { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
}
|
||||
},
|
||||
1_200_000, // 20 min wall clock
|
||||
PTY_LONG_MS, // 20 min wall clock
|
||||
);
|
||||
});
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import {
|
||||
setupSkillDir,
|
||||
@@ -78,6 +79,6 @@ describeE2E('/ship section-loading E2E (periodic, SDK capture)', () => {
|
||||
// Guard against an empty pass: the report must have real content.
|
||||
expect(output.trim().length).toBeGreaterThan(200);
|
||||
},
|
||||
360_000,
|
||||
CAPTURE_LONG_MS,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -27,6 +27,7 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, browseBin, runId,
|
||||
@@ -204,7 +205,7 @@ Do NOT enter the prototype phase. Do NOT use AskUserQuestion.`,
|
||||
env: { GSTACK_HOME: gstackHome },
|
||||
maxTurns: 12,
|
||||
allowedTools: ['Skill', 'Bash', 'Read'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'scrape-match-path',
|
||||
runId,
|
||||
});
|
||||
@@ -224,7 +225,7 @@ Do NOT enter the prototype phase. Do NOT use AskUserQuestion.`,
|
||||
expect(listedSkills).toBe(true);
|
||||
expect(ranBundledSkill).toBe(true);
|
||||
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// ── 2. /scrape prototype path: drive $B primitives against fixture ────
|
||||
testConcurrentIfSelected('scrape-prototype-path', async () => {
|
||||
@@ -248,7 +249,7 @@ Do NOT use AskUserQuestion.`,
|
||||
env: { GSTACK_HOME: gstackHome },
|
||||
maxTurns: 18,
|
||||
allowedTools: ['Skill', 'Bash', 'Read'],
|
||||
timeout: 180_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'scrape-prototype-path',
|
||||
runId,
|
||||
});
|
||||
@@ -283,7 +284,7 @@ Do NOT use AskUserQuestion.`,
|
||||
expect(hasJsonItems).toBe(true);
|
||||
expect(mentionsSkillify).toBe(true);
|
||||
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
|
||||
}, 240_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// ── 3. /skillify happy path: scrape then skillify in one session ─────
|
||||
testConcurrentIfSelected('skillify-happy-path', async () => {
|
||||
@@ -316,7 +317,7 @@ Do NOT halt for clarification.`,
|
||||
},
|
||||
maxTurns: 40,
|
||||
allowedTools: ['Skill', 'Bash', 'Read', 'Write'],
|
||||
timeout: 360_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'skillify-happy-path',
|
||||
runId,
|
||||
});
|
||||
@@ -360,7 +361,7 @@ Do NOT halt for clarification.`,
|
||||
expect(hasAllFiles).toBe(true);
|
||||
expect(prosesClean).toBe(true);
|
||||
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
|
||||
}, 420_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
|
||||
// ── 4. /skillify provenance refusal: D1 contract ─────────────────────
|
||||
testConcurrentIfSelected('skillify-provenance-refusal', async () => {
|
||||
@@ -379,7 +380,7 @@ write any files.`,
|
||||
},
|
||||
maxTurns: 8,
|
||||
allowedTools: ['Skill', 'Bash', 'Read'],
|
||||
timeout: 90_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'skillify-provenance-refusal',
|
||||
runId,
|
||||
});
|
||||
@@ -408,7 +409,7 @@ write any files.`,
|
||||
expect(noSkillsWritten).toBe(true);
|
||||
expect(noStaging).toBe(true);
|
||||
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
|
||||
}, 120_000);
|
||||
}, JUDGE_MS);
|
||||
|
||||
// ── 5. /skillify approval-gate reject: D3 cleanup ────────────────────
|
||||
testConcurrentIfSelected('skillify-approval-reject', async () => {
|
||||
@@ -435,7 +436,7 @@ Use HOME=${workDir}. Do NOT commit the skill.`,
|
||||
},
|
||||
maxTurns: 35,
|
||||
allowedTools: ['Skill', 'Bash', 'Read', 'Write'],
|
||||
timeout: 360_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'skillify-approval-reject',
|
||||
runId,
|
||||
});
|
||||
@@ -462,5 +463,5 @@ Use HOME=${workDir}. Do NOT commit the skill.`,
|
||||
expect(writtenSkills.length).toBe(0);
|
||||
expect(stagingLeftovers.length).toBe(0);
|
||||
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
|
||||
}, 420_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId,
|
||||
@@ -178,7 +179,7 @@ This is a solo repo (REPO_MODE=solo). For pre-existing failures, recommend fixin
|
||||
workingDirectory: triageDir,
|
||||
maxTurns: 20,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
|
||||
timeout: 180_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'ship-triage',
|
||||
runId,
|
||||
});
|
||||
@@ -229,7 +230,7 @@ This is a solo repo (REPO_MODE=solo). For pre-existing failures, recommend fixin
|
||||
// Must have actually run both test files (exercises both failure classes)
|
||||
expect(ranMathTest).toBe(true);
|
||||
expect(ranStringTest).toBe(true);
|
||||
}, 240_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, browseBin, runId, evalsEnabled,
|
||||
@@ -83,7 +84,7 @@ IMPORTANT:
|
||||
// other rounds — marginal at 180s, same contention story as
|
||||
// review-dashboard-via and retro-base-branch. Outer bun timeout
|
||||
// rises to 360s for headroom.
|
||||
timeout: 300_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'document-release',
|
||||
runId,
|
||||
});
|
||||
@@ -120,7 +121,7 @@ IMPORTANT:
|
||||
} else {
|
||||
console.warn('README was NOT updated — agent may not have found the feature');
|
||||
}
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// --- Ship workflow with local bare remote ---
|
||||
@@ -174,7 +175,7 @@ describeIfSelected('Ship workflow E2E', ['ship-local-workflow'], () => {
|
||||
4. Push to origin: git push origin feature/ship-test`,
|
||||
workingDirectory: shipWorkDir,
|
||||
maxTurns: 8,
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'ship-local-workflow',
|
||||
runId,
|
||||
});
|
||||
@@ -198,7 +199,7 @@ describeIfSelected('Ship workflow E2E', ['ship-local-workflow'], () => {
|
||||
expect(branchExists).toBe(true);
|
||||
expect(versionBumped).toBe(true);
|
||||
console.log(`Branch pushed: ${branchExists}, VERSION: ${versionContent}, bumped: ${versionBumped}`);
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// setup-cookies-detect REMOVED: The cookie-import-browser module has 30+ thorough
|
||||
@@ -297,7 +298,7 @@ Skip any AskUserQuestion calls — auto-approve the upgrade. Write a summary of
|
||||
IMPORTANT: The install directory is at ./.claude/skills/gstack — use that exact path.`,
|
||||
workingDirectory: upgradeDir,
|
||||
maxTurns: 20,
|
||||
timeout: 180_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'gstack-upgrade-happy-path',
|
||||
runId,
|
||||
});
|
||||
@@ -317,7 +318,7 @@ IMPORTANT: The install directory is at ./.claude/skills/gstack — use that exac
|
||||
|
||||
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
||||
expect(versionAfter).toBe('0.6.0');
|
||||
}, 240_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Test Coverage Audit E2E ---
|
||||
@@ -418,7 +419,7 @@ Output the diagram directly.`,
|
||||
workingDirectory: coverageDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'ship-coverage-audit',
|
||||
runId,
|
||||
});
|
||||
@@ -443,7 +444,7 @@ Output the diagram directly.`,
|
||||
// At minimum, the agent should have read the source and test files
|
||||
const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read');
|
||||
expect(readCalls.length).toBeGreaterThan(0);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Codex skill E2E ---
|
||||
@@ -520,7 +521,7 @@ Follow those instructions to run codex review against the diff on this branch.
|
||||
Write the full output (including the GATE verdict) to ${codexDir}/codex-output.md`,
|
||||
workingDirectory: codexDir,
|
||||
maxTurns: 25,
|
||||
timeout: 300_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'codex-review',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -538,7 +539,7 @@ Write the full output (including the GATE verdict) to ${codexDir}/codex-output.m
|
||||
const hasCodexOutput = output.includes('CODEX') || output.includes('GATE') || output.includes('codex');
|
||||
expect(hasCodexOutput).toBe(true);
|
||||
}
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
*/
|
||||
|
||||
import { afterAll, expect } from 'bun:test';
|
||||
import { JUDGE_MS } from './helpers/eval-budgets';
|
||||
import Anthropic from '@anthropic-ai/sdk';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
@@ -556,7 +557,7 @@ describeIfSelected('Baseline score pinning', ['baseline score pinning'], () => {
|
||||
if (!passed) {
|
||||
throw new Error(`Score regressions detected:\n${regressions.join('\n')}`);
|
||||
}
|
||||
}, 60_000);
|
||||
}, JUDGE_MS);
|
||||
});
|
||||
|
||||
// --- Workflow SKILL.md quality evals (10 new tests for 100% coverage) ---
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import type { SkillTestResult } from './helpers/session-runner';
|
||||
import { EvalCollector } from './helpers/eval-store';
|
||||
@@ -197,7 +198,7 @@ describeE2E('Skill Routing E2E — Developer Journey', () => {
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 60_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
@@ -213,7 +214,7 @@ describeE2E('Skill Routing E2E — Developer Journey', () => {
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
testIfSelected('journey-plan-eng', async () => {
|
||||
const tmpDir = createRoutingWorkDir('plan-eng');
|
||||
@@ -247,7 +248,7 @@ describeE2E('Skill Routing E2E — Developer Journey', () => {
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 60_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
@@ -263,7 +264,7 @@ describeE2E('Skill Routing E2E — Developer Journey', () => {
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// Removed: journey-think-bigger
|
||||
// Tested ambiguous routing ("think bigger" → plan-ceo-review) but Claude
|
||||
@@ -309,7 +310,7 @@ export default app;
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 60_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
@@ -326,7 +327,7 @@ export default app;
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
testIfSelected('journey-qa', async () => {
|
||||
const tmpDir = createRoutingWorkDir('qa');
|
||||
@@ -345,7 +346,7 @@ export default app;
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 60_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
@@ -362,7 +363,7 @@ export default app;
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
testIfSelected('journey-code-review', async () => {
|
||||
const tmpDir = createRoutingWorkDir('code-review');
|
||||
@@ -386,7 +387,7 @@ export default app;
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
@@ -402,7 +403,7 @@ export default app;
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
testIfSelected('journey-ship', async () => {
|
||||
const tmpDir = createRoutingWorkDir('ship');
|
||||
@@ -425,7 +426,7 @@ export default app;
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 60_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
@@ -441,7 +442,7 @@ export default app;
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
testIfSelected('journey-docs', async () => {
|
||||
const tmpDir = createRoutingWorkDir('docs');
|
||||
@@ -462,7 +463,7 @@ export default app;
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 60_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
@@ -478,7 +479,7 @@ export default app;
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
testIfSelected('journey-retro', async () => {
|
||||
const tmpDir = createRoutingWorkDir('retro');
|
||||
@@ -505,7 +506,7 @@ export default app;
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
@@ -521,7 +522,7 @@ export default app;
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
testIfSelected('journey-design-system', async () => {
|
||||
const tmpDir = createRoutingWorkDir('design-system');
|
||||
@@ -534,7 +535,7 @@ export default app;
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 60_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
@@ -550,7 +551,7 @@ export default app;
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
testIfSelected('journey-visual-qa', async () => {
|
||||
const tmpDir = createRoutingWorkDir('visual-qa');
|
||||
@@ -585,7 +586,7 @@ body { font-family: sans-serif; }
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 60_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
@@ -602,5 +603,5 @@ body { font-family: sans-serif; }
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user