test: isolate plan-count fixtures before starting review workflows

Seed the complete test plan in a private git repository before launching Claude, so a bare slash command cannot review the live workspace while a delayed fixture message remains queued. Preserve count thresholds, parsers and budgets. Add initial-context and installed-discovery tests, and retain startup/terminal diagnostics on failed evaluations.
This commit is contained in:
Garry Tan committed 2026-09-08 05:59:29 +00:00
1 parent db72f47498
commit b8c388bd1f
11 files changed
+338 -58

No files matched your search

+30 -23
View File
@@ -26,6 +26,7 @@ import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { hermeticChildEnv, hermeticSkillsConfigDir, isHermeticEnabled } from './hermetic-env';
import { createPlanCountFixture } from './plan-count-fixture';
/** Strip ANSI escapes for pattern-matching against visible text. */
export function stripAnsi(s: string): string {
@@ -1929,14 +1930,13 @@ export interface PlanSkillCountObservation {
* AskUserQuestions until a terminal signal fires.
*
* Flow:
* 1. Boot PTY in plan mode (8s grace + auto-trust dialog).
* 2. Send `slashCommand` alone. Sleep ~3s.
* 3. Send `followUpPrompt` as a chat message — this is the plan content
* the skill reviews. Slash commands with trailing args are rejected by
* Claude Code unless the skill defines them, so the plan goes as a
* follow-up message (the proven pattern at
* skill-e2e-plan-design-with-ui.test.ts:57-71).
* 4. Poll loop:
* 1. Seed the complete fixture request in an isolated git repository's
* PLAN.md and initial CLAUDE.md context, then boot the PTY in that cwd
* (8s grace + auto-trust dialog). Skills remain registered in user scope.
* 2. Send `slashCommand` alone. The fixture is already in context; sending
* it later can queue it behind the skill's first question while the
* review incorrectly starts against the operator's live branch.
* 3. Poll loop:
* - Skip permission dialogs (auto-grant with `defaultPick`).
* - On a new numbered-option list, parse prompt + options, build
* fingerprint via `auqFingerprint`. Empty-prompt parses are skipped
@@ -1970,7 +1970,7 @@ export async function runPlanSkillCounting(opts: {
skillName: string;
/** Slash command to send alone, e.g. '/plan-ceo-review'. No trailing args. */
slashCommand: string;
/** Plan content sent as a follow-up message ~3s after the slash command. */
/** Fixture request seeded in initial project context before the slash command. */
followUpPrompt: string;
/** Per-skill predicate: which answered AUQ is the last Step-0 question. */
isLastStep0AUQ: Step0BoundaryPredicate;
@@ -1989,11 +1989,9 @@ export async function runPlanSkillCounting(opts: {
* interview". Pressing the default 1 routes to "branch diff" (the wrong
* review target for a seeded fixture). firstAUQPick lets the test pick
* "Skip interview" or "describe inline" so the agent reviews the
* follow-up plan content the test sent, not the git diff.
* fixture plan content, not the git diff.
*/
firstAUQPick?: (fp: AskUserQuestionFingerprint) => number;
/** Working directory. Default process.cwd() (repo cwd holds skill registry). */
cwd?: string;
/** Total budget for skill to reach a terminal outcome. Default 1_500_000 (25 min). */
timeoutMs?: number;
/** Extra env merged into the spawned `claude` process. */
@@ -2005,14 +2003,21 @@ export async function runPlanSkillCounting(opts: {
const defaultPick = opts.defaultPick ?? 1;
const timeoutMs = opts.timeoutMs ?? 1_500_000;
const session = await launchClaudePty({
permissionMode: 'plan',
cwd: opts.cwd,
timeoutMs: timeoutMs + 60_000,
env: opts.env,
model: opts.model,
seedSkills: true,
});
const fixture = createPlanCountFixture(opts.followUpPrompt);
let session: ClaudePtySession;
try {
session = await launchClaudePty({
permissionMode: 'plan',
cwd: fixture.cwd,
timeoutMs: timeoutMs + 60_000,
env: opts.env,
model: opts.model,
seedSkills: true,
});
} catch (error) {
fixture.cleanup();
throw error;
}
const fingerprints: AskUserQuestionFingerprint[] = [];
const seen = new Set<string>();
@@ -2042,8 +2047,6 @@ export async function runPlanSkillCounting(opts: {
await Bun.sleep(8000); // boot grace + auto-trust handler window
const since = session.mark();
session.send(`${opts.slashCommand}\r`);
await Bun.sleep(3000);
session.send(`${opts.followUpPrompt}\r`);
const budgetStart = Date.now();
while (Date.now() - budgetStart < timeoutMs) {
@@ -2170,7 +2173,11 @@ export async function runPlanSkillCounting(opts: {
session.visibleSince(since),
);
} finally {
await session.close();
try {
await session.close();
} finally {
fixture.cleanup();
}
}
}
+53
View File
@@ -0,0 +1,53 @@
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { spawnSync } from 'node:child_process';
/**
* Count evals review a seeded plan, never the checkout that supplies skills.
* Put the complete request in Claude's initial project context before the
* bare slash command starts: a later message can remain queued behind the
* skill's first AskUserQuestion and leave it reviewing the live branch.
*/
export function createPlanCountFixture(prompt: string): {
cwd: string;
cleanup(): void;
} {
const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-plan-count-'));
const cleanup = () => fs.rmSync(cwd, { recursive: true, force: true });
try {
fs.writeFileSync(path.join(cwd, 'PLAN.md'), prompt);
fs.writeFileSync(path.join(cwd, 'CLAUDE.md'), [
'# Plan review fixture',
'',
'This repository contains the plan under review. Use PLAN.md as the',
'current plan for the requested plan-review skill. The skill installation',
'supplies the workflow; its source checkout is not the review target.',
'',
'The complete user request is available from the start of this session:',
'',
prompt,
'',
].join('\n'));
const git = (args: string[]) => {
const result = spawnSync('git', args, {
cwd,
encoding: 'utf8',
timeout: 10_000,
});
if (result.error || result.status !== 0) {
throw new Error(`Could not initialize plan-count fixture: ${result.error?.message ?? result.stderr}`);
}
};
git(['init', '-b', 'main']);
git(['add', 'PLAN.md', 'CLAUDE.md']);
git(['-c', 'user.name=Plan Count Fixture', '-c', 'user.email=plan-count@example.test',
'-c', 'commit.gpgsign=false', 'commit', '--no-verify', '-m', 'Seed review plan']);
git(['update-ref', 'refs/remotes/origin/main', 'HEAD']);
return { cwd, cleanup };
} catch (error) {
cleanup();
throw error;
}
}
+6 -6
View File
@@ -160,10 +160,10 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
// Each test drives its skill end-to-end; touchfiles include preamble +
// completion-status resolvers because they affect question cadence and
// terminal output (the regression surface this test catches).
'plan-ceo-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-finding-count.test.ts'],
'plan-eng-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-eng-finding-count.test.ts'],
'plan-design-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-finding-count.test.ts'],
'plan-devex-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts'],
'plan-ceo-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-ceo-finding-count.test.ts'],
'plan-eng-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-eng-finding-count.test.ts'],
'plan-design-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-design-finding-count.test.ts'],
'plan-devex-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts'],
// Gate-tier reviewCount-floor counterparts. Catch the May 2026 transcript
// bug (model wrote a plan-mode plan and ExitPlanMode'd without firing any
@@ -180,8 +180,8 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
// a model fires one AUQ then batches the rest into a "## Decisions to
// confirm" plan write. runPlanSkillFloorCheck cannot detect that shape
// (it exits on first AUQ); runPlanSkillCounting can.
'plan-eng-multi-finding-batching': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts'],
'plan-ceo-split-overflow': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts'],
'plan-eng-multi-finding-batching': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts'],
'plan-ceo-split-overflow': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts'],
'brain-privacy-gate': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'bin/gstack-brain-sync', 'bin/gstack-artifacts-init', 'bin/gstack-config', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-brain-privacy-gate.test.ts'],
// /setup-gbrain Path 4 (Remote MCP) — happy + bad-token end-to-end via
+222
View File
@@ -0,0 +1,222 @@
/** Free behavioral coverage for the isolated, preloaded plan-count workspace. */
import { describe, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { pathToFileURL } from 'node:url';
import { createPlanCountFixture } from './helpers/plan-count-fixture';
const ROOT = path.resolve(import.meta.dir, '..');
const PROMPT = '# Seeded settings plan\n\nReview each issue separately.\n' +
'Literal text: "quotes" \'single quotes\' `touch never` $(touch never)\n';
function gitRoot(cwd: string): string {
const result = spawnSync('git', ['rev-parse', '--show-toplevel'], { cwd, encoding: 'utf8', timeout: 10_000 });
expect(result.status, result.stderr).toBe(0);
return fs.realpathSync(result.stdout.trim());
}
describe('plan-count fixtures', () => {
test('preloads the literal plan and initial context in a separate git repository', () => {
const fixture = createPlanCountFixture(PROMPT);
try {
expect(fs.realpathSync(fixture.cwd)).not.toBe(fs.realpathSync(ROOT));
expect(gitRoot(fixture.cwd)).toBe(fs.realpathSync(fixture.cwd));
expect(fs.readFileSync(path.join(fixture.cwd, 'PLAN.md'), 'utf8')).toBe(PROMPT);
const context = fs.readFileSync(path.join(fixture.cwd, 'CLAUDE.md'), 'utf8');
expect(context).toContain('PLAN.md');
expect(context).toContain(PROMPT);
expect(fs.existsSync(path.join(fixture.cwd, 'DESIGN.md'))).toBe(false);
expect(fs.existsSync(path.join(fixture.cwd, 'TODOS.md'))).toBe(false);
expect(fs.existsSync(path.join(fixture.cwd, 'never'))).toBe(false);
} finally {
fixture.cleanup();
}
expect(fs.existsSync(fixture.cwd)).toBe(false);
});
test('concurrent fixtures have independent content and cleanup owns only its directory', () => {
const first = createPlanCountFixture('first plan');
const second = createPlanCountFixture('second plan');
try {
expect(first.cwd).not.toBe(second.cwd);
first.cleanup();
first.cleanup();
expect(fs.existsSync(first.cwd)).toBe(false);
expect(fs.readFileSync(path.join(second.cwd, 'PLAN.md'), 'utf8')).toBe('second plan');
expect(gitRoot(second.cwd)).toBe(fs.realpathSync(second.cwd));
} finally {
first.cleanup();
second.cleanup();
}
expect(fs.existsSync(second.cwd)).toBe(false);
});
test.skipIf(process.platform === 'win32')('cleans the fixture when the PTY executable cannot launch', () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-launch-failure-'));
const fixtureTmp = path.join(dir, 'fixtures');
const workerPath = path.join(dir, 'worker.ts');
const brokenCli = path.join(dir, 'not-executable');
fs.mkdirSync(fixtureTmp);
fs.writeFileSync(brokenCli, 'This file deliberately has no executable permission.', { mode: 0o644 });
const runnerUrl = pathToFileURL(path.join(ROOT, 'test/helpers/claude-pty-runner.ts')).href;
fs.writeFileSync(workerPath, `
import { runPlanSkillCounting } from ${JSON.stringify(runnerUrl)};
try {
await runPlanSkillCounting({
skillName: 'plan-design-review', slashCommand: '/plan-design-review',
followUpPrompt: 'Review this launch-failure fixture.',
isLastStep0AUQ: () => false, reviewCountCeiling: 8,
});
process.exitCode = 1;
} catch (error) {
process.stdout.write('launch failed as expected');
}
`);
try {
const result = spawnSync(process.execPath, [workerPath], {
cwd: ROOT,
env: { ...process.env, BROWSE_TERMINAL_BINARY: brokenCli, EVALS_HERMETIC: '1', TMPDIR: fixtureTmp },
encoding: 'utf8',
timeout: 10_000,
});
expect(result.status, result.stderr).toBe(0);
expect(result.stdout).toBe('launch failed as expected');
expect(fs.readdirSync(fixtureTmp)).toEqual([]);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
}, 12_000);
// Bun's real PTY spawn and executable shebangs are POSIX-only. The direct
// fixture tests above still exercise repository setup/cleanup on Windows.
test.skipIf(process.platform === 'win32')(
'real PTY children see preloaded isolated plans, shipped skills, and only the bare slash command',
async () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-subprocess-'));
const fakePath = path.join(dir, 'fake-claude');
const workerPath = path.join(dir, 'worker.ts');
const resultPath = path.join(dir, 'result.json');
const cases = [
{ name: 'design', skillName: 'plan-design-review', prompt: PROMPT, mode: 'complete' },
{ name: 'ceo', skillName: 'plan-ceo-review', prompt: '# Independent CEO plan\nUnique product context.', mode: 'complete' },
{ name: 'exited', skillName: 'plan-eng-review', prompt: '# Early-exit plan\nStill clean up.', mode: 'exit' },
].map((item) => ({ ...item, record: path.join(dir, `${item.name}.jsonl`) }));
// The fake snapshots its environment BEFORE installing the first stdin
// handler. It never dispatches a model or invokes the real Claude CLI.
fs.writeFileSync(fakePath, `#!${process.execPath}\n` + String.raw`
import * as fs from 'node:fs';
import * as path from 'node:path';
import { spawnSync } from 'node:child_process';
const record = (event) => fs.appendFileSync(process.env.FIXTURE_RECORD, JSON.stringify(event) + '\n');
const skillDir = path.join(process.env.CLAUDE_CONFIG_DIR, 'skills', process.env.FIXTURE_SKILL);
const planPath = path.join(process.cwd(), 'PLAN.md');
const contextPath = path.join(process.cwd(), 'CLAUDE.md');
record({
type: 'startup', pid: process.pid, cwd: process.cwd(), argv: process.argv.slice(2),
gitRoot: spawnSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8', timeout: 10_000 }).stdout.trim(),
plan: fs.existsSync(planPath) ? fs.readFileSync(planPath, 'utf8') : null,
context: fs.existsSync(contextPath) ? fs.readFileSync(contextPath, 'utf8') : null,
inheritedDesign: fs.existsSync('DESIGN.md'), inheritedTodos: fs.existsSync('TODOS.md'),
skill: fs.existsSync(path.join(skillDir, 'SKILL.md')) ? fs.readFileSync(path.join(skillDir, 'SKILL.md'), 'utf8') : null,
sections: fs.existsSync(path.join(skillDir, 'sections', 'review-sections.md'))
? fs.readFileSync(path.join(skillDir, 'sections', 'review-sections.md'), 'utf8') : null,
});
process.stdin.setRawMode?.(true);
let firstInput = true;
let completion;
process.stdin.on('data', (data) => {
record({ type: 'input', data: data.toString('utf8') });
if (!firstInput) return;
firstInput = false;
if (process.env.FIXTURE_MODE === 'exit') process.exit(7);
// Longer than the old 3 s delayed fixture send: record that regression
// even if the helper would otherwise return on our completion marker.
completion = setTimeout(() => process.stdout.write('\nGSTACK REVIEW REPORT\n'), 4100);
});
process.on('SIGINT', () => {
clearTimeout(completion);
record({ type: 'closed', at: Date.now() });
process.exit(0);
});
process.stdin.resume();
`);
fs.chmodSync(fakePath, 0o755);
const runnerUrl = pathToFileURL(path.join(ROOT, 'test/helpers/claude-pty-runner.ts')).href;
fs.writeFileSync(workerPath, `
import { runPlanSkillCounting } from ${JSON.stringify(runnerUrl)};
const cases = ${JSON.stringify(cases)};
const results = await Promise.all(cases.map(async (item) => ({
name: item.name,
observation: await runPlanSkillCounting({
skillName: item.skillName,
slashCommand: '/' + item.skillName,
followUpPrompt: item.prompt,
isLastStep0AUQ: () => false,
reviewCountCeiling: 8,
timeoutMs: 15000,
env: { FIXTURE_RECORD: item.record, FIXTURE_SKILL: item.skillName, FIXTURE_MODE: item.mode },
}),
})));
await Bun.write(${JSON.stringify(resultPath)}, JSON.stringify(results));
`);
const startedAt = Date.now();
const child = Bun.spawn([process.execPath, workerPath], {
cwd: ROOT,
env: { ...process.env, BROWSE_TERMINAL_BINARY: fakePath, EVALS_HERMETIC: '1' },
stdout: 'pipe',
stderr: 'pipe',
});
const killer = setTimeout(() => child.kill('SIGKILL'), 35_000);
try {
const [exitCode, stdout, stderr] = await Promise.all([
child.exited,
new Response(child.stdout).text(),
new Response(child.stderr).text(),
]);
expect(exitCode, stdout + stderr).toBe(0);
expect(Date.now() - startedAt).toBeLessThan(35_000);
const results = JSON.parse(fs.readFileSync(resultPath, 'utf8'));
const cwds = new Set<string>();
for (const item of cases) {
const events = fs.readFileSync(item.record, 'utf8').trim().split('\n').map((line) => JSON.parse(line));
const startup = events[0];
expect(startup.type).toBe('startup');
expect(startup.cwd).not.toBe(ROOT);
expect(startup.gitRoot).toBe(startup.cwd);
expect(startup.plan).toBe(item.prompt);
expect(startup.context).toContain('PLAN.md');
expect(startup.context).toContain(item.prompt);
expect(startup.argv[startup.argv.indexOf('--permission-mode') + 1]).toBe('plan');
expect(startup.inheritedDesign).toBe(false);
expect(startup.inheritedTodos).toBe(false);
expect(startup.skill).toContain(`name: ${item.skillName}`);
expect(startup.sections).toBe(fs.readFileSync(path.join(ROOT, item.skillName, 'sections/review-sections.md'), 'utf8'));
expect(events.filter((event) => event.type === 'input').map((event) => event.data).join(''))
.toBe(`/${item.skillName}\r`);
expect(fs.existsSync(startup.cwd)).toBe(false);
expect(() => process.kill(startup.pid, 0)).toThrow();
const result = results.find((result) => result.name === item.name);
expect(result.observation.outcome).toBe(item.mode === 'exit' ? 'exited' : 'completion_summary');
if (item.mode === 'complete') expect(events.at(-1).type).toBe('closed');
cwds.add(startup.cwd);
}
expect(cwds.size).toBe(cases.length);
} finally {
clearTimeout(killer);
child.kill('SIGKILL');
// A failed worker assertion must not leave its PTY fakes running.
for (const item of cases) {
if (!fs.existsSync(item.record)) continue;
const startup = JSON.parse(fs.readFileSync(item.record, 'utf8').split('\n')[0]);
try { process.kill(startup.pid, 'SIGKILL'); } catch { /* already reaped */ }
}
fs.rmSync(dir, { recursive: true, force: true });
}
},
40_000,
);
});
+16 -2
View File
@@ -32,6 +32,7 @@ import { spawnSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { stripVTControlCharacters } from 'node:util';
import {
launchClaudePty,
isPlanReadyVisible,
@@ -49,6 +50,12 @@ interface PhaseHit {
ts: number;
}
function diagnosticTail(text: string): string {
return stripVTControlCharacters(text)
.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, '')
.slice(-3000);
}
describeE2E('/autoplan chain ordering (periodic)', () => {
test(
'phases run sequentially: Phase 1 (CEO) before Phase 3 (Eng), Phase 2 (Design) between when present',
@@ -79,6 +86,8 @@ describeE2E('/autoplan chain ordering (periodic)', () => {
const hits: PhaseHit[] = [];
let outcome: 'chain_complete' | 'plan_ready' | 'timeout' | 'exited' = 'timeout';
let evidence = '';
let fullSessionEvidence = '';
let exitCode: number | null = null;
try {
await Bun.sleep(8000);
@@ -144,13 +153,18 @@ describeE2E('/autoplan chain ordering (periodic)', () => {
}
}
} finally {
// Preserve boot failures omitted by mark(), and observe the real exit
// status before close() deliberately terminates a live session.
exitCode = session.exitCode();
fullSessionEvidence = diagnosticTail(session.visibleText());
await session.close();
}
if (outcome === 'exited' || outcome === 'timeout') {
throw new Error(
`autoplan chain test FAILED: outcome=${outcome}, hits=${JSON.stringify(hits)}\n` +
`--- evidence (last 3KB) ---\n${evidence}`,
`autoplan chain test FAILED: outcome=${outcome}, exitCode=${exitCode}, hits=${JSON.stringify(hits)}\n` +
`--- post-command evidence (last 3KB) ---\n${diagnosticTail(evidence)}\n` +
`--- full-session visible tail, including startup (last 3KB) ---\n${fullSessionEvidence}`,
);
}
@@ -38,8 +38,8 @@ import {
* The default pick (1) routes to "branch diff vs main" — the wrong target
* for our seeded fixture (the agent would review the gstack PR itself,
* recursively). Picking "Skip interview and plan immediately" bypasses
* Step 0 and routes the agent to review the chat context (where our
* follow-up plan was pasted).
* Step 0 and routes the agent to review the fixture request, which is
* already present in its initial project context.
*/
function pickSkipInterview(fp: AskUserQuestionFingerprint): number {
const skipOpt = fp.options.find((o) =>
@@ -122,9 +122,6 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', ()
isLastStep0AUQ: ceoStep0Boundary,
reviewCountCeiling: CEILING_DISTINCT + 1, // hard cap above assertion ceiling
firstAUQPick: pickSkipInterview, // bypass scope-selection, route to review
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 1_500_000, // 25 min
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
@@ -211,9 +208,6 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', ()
followUpPrompt: planCeo2PairedFindings(planPath),
isLastStep0AUQ: ceoStep0Boundary,
reviewCountCeiling: CEILING_PAIRED + 1,
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 1_500_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
@@ -73,9 +73,6 @@ describeE2E('/plan-ceo-review split-overflow regression (periodic)', () => {
followUpPrompt,
isLastStep0AUQ: ceoStep0Boundary,
reviewCountCeiling: N + 3, // hard cap above floor + tolerance
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 1_500_000, // 25 min
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
@@ -69,9 +69,6 @@ describeE2E('/plan-design-review per-finding AskUserQuestion count (periodic)',
followUpPrompt: planDesign5Findings(planPath),
isLastStep0AUQ: designStep0Boundary,
reviewCountCeiling: CEILING + 1,
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 1_500_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
@@ -94,11 +91,16 @@ describeE2E('/plan-design-review per-finding AskUserQuestion count (periodic)',
if (obs.reviewCount < FLOOR) {
throw new Error(
`BAND FAIL (below floor): reviewCount=${obs.reviewCount} < FLOOR=${FLOOR}.\n` +
`Likely batching regression. Review-phase fingerprints:\n` +
`outcome=${obs.outcome} step0=${obs.step0Count} elapsed=${obs.elapsedMs}ms\n` +
`summary: ${obs.summary}\n` +
`All captured fingerprints (including Step 0):\n` +
obs.fingerprints
.filter((f) => !f.preReview)
.map((f) => ` - "${f.promptSnippet.slice(0, 80)}"`)
.join('\n'),
.map(
(f) =>
` - preReview=${f.preReview} sig=${f.signature.slice(0, 12)} prompt="${f.promptSnippet}"`,
)
.join('\n') +
`\n--- evidence (last 3KB) ---\n${obs.evidence}`,
);
}
if (obs.reviewCount > CEILING) {
@@ -69,9 +69,6 @@ describeE2E('/plan-devex-review per-finding AskUserQuestion count (periodic)', (
followUpPrompt: planDevex5Findings(planPath),
isLastStep0AUQ: devexStep0Boundary,
reviewCountCeiling: CEILING + 1,
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 1_500_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
@@ -68,9 +68,6 @@ describeE2E('/plan-eng-review per-finding AskUserQuestion count (periodic)', ()
followUpPrompt: planEng5Findings(planPath),
isLastStep0AUQ: engStep0Boundary,
reviewCountCeiling: CEILING + 1,
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 1_500_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
@@ -65,9 +65,6 @@ describeE2E('/plan-eng-review multi-finding batching regression (periodic)', ()
followUpPrompt,
isLastStep0AUQ: engStep0Boundary,
reviewCountCeiling: N + 3, // hard cap above floor + tolerance
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 1_500_000, // 25 min
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});