diff --git a/test/helpers/claude-pty-runner.ts b/test/helpers/claude-pty-runner.ts index ef625a194..825b01bbf 100644 --- a/test/helpers/claude-pty-runner.ts +++ b/test/helpers/claude-pty-runner.ts @@ -26,6 +26,7 @@ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import { hermeticChildEnv, hermeticSkillsConfigDir, isHermeticEnabled } from './hermetic-env'; +import { createPlanCountFixture } from './plan-count-fixture'; /** Strip ANSI escapes for pattern-matching against visible text. */ export function stripAnsi(s: string): string { @@ -1929,14 +1930,13 @@ export interface PlanSkillCountObservation { * AskUserQuestions until a terminal signal fires. * * Flow: - * 1. Boot PTY in plan mode (8s grace + auto-trust dialog). - * 2. Send `slashCommand` alone. Sleep ~3s. - * 3. Send `followUpPrompt` as a chat message — this is the plan content - * the skill reviews. Slash commands with trailing args are rejected by - * Claude Code unless the skill defines them, so the plan goes as a - * follow-up message (the proven pattern at - * skill-e2e-plan-design-with-ui.test.ts:57-71). - * 4. Poll loop: + * 1. Seed the complete fixture request in an isolated git repository's + * PLAN.md and initial CLAUDE.md context, then boot the PTY in that cwd + * (8s grace + auto-trust dialog). Skills remain registered in user scope. + * 2. Send `slashCommand` alone. The fixture is already in context; sending + * it later can queue it behind the skill's first question while the + * review incorrectly starts against the operator's live branch. + * 3. Poll loop: * - Skip permission dialogs (auto-grant with `defaultPick`). * - On a new numbered-option list, parse prompt + options, build * fingerprint via `auqFingerprint`. Empty-prompt parses are skipped @@ -1970,7 +1970,7 @@ export async function runPlanSkillCounting(opts: { skillName: string; /** Slash command to send alone, e.g. '/plan-ceo-review'. No trailing args. */ slashCommand: string; - /** Plan content sent as a follow-up message ~3s after the slash command. */ + /** Fixture request seeded in initial project context before the slash command. */ followUpPrompt: string; /** Per-skill predicate: which answered AUQ is the last Step-0 question. */ isLastStep0AUQ: Step0BoundaryPredicate; @@ -1989,11 +1989,9 @@ export async function runPlanSkillCounting(opts: { * interview". Pressing the default 1 routes to "branch diff" (the wrong * review target for a seeded fixture). firstAUQPick lets the test pick * "Skip interview" or "describe inline" so the agent reviews the - * follow-up plan content the test sent, not the git diff. + * fixture plan content, not the git diff. */ firstAUQPick?: (fp: AskUserQuestionFingerprint) => number; - /** Working directory. Default process.cwd() (repo cwd holds skill registry). */ - cwd?: string; /** Total budget for skill to reach a terminal outcome. Default 1_500_000 (25 min). */ timeoutMs?: number; /** Extra env merged into the spawned `claude` process. */ @@ -2005,14 +2003,21 @@ export async function runPlanSkillCounting(opts: { const defaultPick = opts.defaultPick ?? 1; const timeoutMs = opts.timeoutMs ?? 1_500_000; - const session = await launchClaudePty({ - permissionMode: 'plan', - cwd: opts.cwd, - timeoutMs: timeoutMs + 60_000, - env: opts.env, - model: opts.model, - seedSkills: true, - }); + const fixture = createPlanCountFixture(opts.followUpPrompt); + let session: ClaudePtySession; + try { + session = await launchClaudePty({ + permissionMode: 'plan', + cwd: fixture.cwd, + timeoutMs: timeoutMs + 60_000, + env: opts.env, + model: opts.model, + seedSkills: true, + }); + } catch (error) { + fixture.cleanup(); + throw error; + } const fingerprints: AskUserQuestionFingerprint[] = []; const seen = new Set(); @@ -2042,8 +2047,6 @@ export async function runPlanSkillCounting(opts: { await Bun.sleep(8000); // boot grace + auto-trust handler window const since = session.mark(); session.send(`${opts.slashCommand}\r`); - await Bun.sleep(3000); - session.send(`${opts.followUpPrompt}\r`); const budgetStart = Date.now(); while (Date.now() - budgetStart < timeoutMs) { @@ -2170,7 +2173,11 @@ export async function runPlanSkillCounting(opts: { session.visibleSince(since), ); } finally { - await session.close(); + try { + await session.close(); + } finally { + fixture.cleanup(); + } } } diff --git a/test/helpers/plan-count-fixture.ts b/test/helpers/plan-count-fixture.ts new file mode 100644 index 000000000..7570a6e76 --- /dev/null +++ b/test/helpers/plan-count-fixture.ts @@ -0,0 +1,53 @@ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { spawnSync } from 'node:child_process'; + +/** + * Count evals review a seeded plan, never the checkout that supplies skills. + * Put the complete request in Claude's initial project context before the + * bare slash command starts: a later message can remain queued behind the + * skill's first AskUserQuestion and leave it reviewing the live branch. + */ +export function createPlanCountFixture(prompt: string): { + cwd: string; + cleanup(): void; +} { + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-plan-count-')); + const cleanup = () => fs.rmSync(cwd, { recursive: true, force: true }); + try { + fs.writeFileSync(path.join(cwd, 'PLAN.md'), prompt); + fs.writeFileSync(path.join(cwd, 'CLAUDE.md'), [ + '# Plan review fixture', + '', + 'This repository contains the plan under review. Use PLAN.md as the', + 'current plan for the requested plan-review skill. The skill installation', + 'supplies the workflow; its source checkout is not the review target.', + '', + 'The complete user request is available from the start of this session:', + '', + prompt, + '', + ].join('\n')); + + const git = (args: string[]) => { + const result = spawnSync('git', args, { + cwd, + encoding: 'utf8', + timeout: 10_000, + }); + if (result.error || result.status !== 0) { + throw new Error(`Could not initialize plan-count fixture: ${result.error?.message ?? result.stderr}`); + } + }; + git(['init', '-b', 'main']); + git(['add', 'PLAN.md', 'CLAUDE.md']); + git(['-c', 'user.name=Plan Count Fixture', '-c', 'user.email=plan-count@example.test', + '-c', 'commit.gpgsign=false', 'commit', '--no-verify', '-m', 'Seed review plan']); + git(['update-ref', 'refs/remotes/origin/main', 'HEAD']); + return { cwd, cleanup }; + } catch (error) { + cleanup(); + throw error; + } +} diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 3d609a6be..cf2dce00d 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -160,10 +160,10 @@ export const E2E_TOUCHFILES: Record = { // Each test drives its skill end-to-end; touchfiles include preamble + // completion-status resolvers because they affect question cadence and // terminal output (the regression surface this test catches). - 'plan-ceo-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-finding-count.test.ts'], - 'plan-eng-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-eng-finding-count.test.ts'], - 'plan-design-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-finding-count.test.ts'], - 'plan-devex-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts'], + 'plan-ceo-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-ceo-finding-count.test.ts'], + 'plan-eng-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-eng-finding-count.test.ts'], + 'plan-design-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-design-finding-count.test.ts'], + 'plan-devex-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts'], // Gate-tier reviewCount-floor counterparts. Catch the May 2026 transcript // bug (model wrote a plan-mode plan and ExitPlanMode'd without firing any @@ -180,8 +180,8 @@ export const E2E_TOUCHFILES: Record = { // a model fires one AUQ then batches the rest into a "## Decisions to // confirm" plan write. runPlanSkillFloorCheck cannot detect that shape // (it exits on first AUQ); runPlanSkillCounting can. - 'plan-eng-multi-finding-batching': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts'], - 'plan-ceo-split-overflow': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts'], + 'plan-eng-multi-finding-batching': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts'], + 'plan-ceo-split-overflow': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts'], 'brain-privacy-gate': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'bin/gstack-brain-sync', 'bin/gstack-artifacts-init', 'bin/gstack-config', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-brain-privacy-gate.test.ts'], // /setup-gbrain Path 4 (Remote MCP) — happy + bad-token end-to-end via diff --git a/test/plan-count-fixture.test.ts b/test/plan-count-fixture.test.ts new file mode 100644 index 000000000..61e80199e --- /dev/null +++ b/test/plan-count-fixture.test.ts @@ -0,0 +1,222 @@ +/** Free behavioral coverage for the isolated, preloaded plan-count workspace. */ +import { describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { createPlanCountFixture } from './helpers/plan-count-fixture'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const PROMPT = '# Seeded settings plan\n\nReview each issue separately.\n' + + 'Literal text: "quotes" \'single quotes\' `touch never` $(touch never)\n'; + +function gitRoot(cwd: string): string { + const result = spawnSync('git', ['rev-parse', '--show-toplevel'], { cwd, encoding: 'utf8', timeout: 10_000 }); + expect(result.status, result.stderr).toBe(0); + return fs.realpathSync(result.stdout.trim()); +} + +describe('plan-count fixtures', () => { + test('preloads the literal plan and initial context in a separate git repository', () => { + const fixture = createPlanCountFixture(PROMPT); + try { + expect(fs.realpathSync(fixture.cwd)).not.toBe(fs.realpathSync(ROOT)); + expect(gitRoot(fixture.cwd)).toBe(fs.realpathSync(fixture.cwd)); + expect(fs.readFileSync(path.join(fixture.cwd, 'PLAN.md'), 'utf8')).toBe(PROMPT); + const context = fs.readFileSync(path.join(fixture.cwd, 'CLAUDE.md'), 'utf8'); + expect(context).toContain('PLAN.md'); + expect(context).toContain(PROMPT); + expect(fs.existsSync(path.join(fixture.cwd, 'DESIGN.md'))).toBe(false); + expect(fs.existsSync(path.join(fixture.cwd, 'TODOS.md'))).toBe(false); + expect(fs.existsSync(path.join(fixture.cwd, 'never'))).toBe(false); + } finally { + fixture.cleanup(); + } + expect(fs.existsSync(fixture.cwd)).toBe(false); + }); + + test('concurrent fixtures have independent content and cleanup owns only its directory', () => { + const first = createPlanCountFixture('first plan'); + const second = createPlanCountFixture('second plan'); + try { + expect(first.cwd).not.toBe(second.cwd); + first.cleanup(); + first.cleanup(); + expect(fs.existsSync(first.cwd)).toBe(false); + expect(fs.readFileSync(path.join(second.cwd, 'PLAN.md'), 'utf8')).toBe('second plan'); + expect(gitRoot(second.cwd)).toBe(fs.realpathSync(second.cwd)); + } finally { + first.cleanup(); + second.cleanup(); + } + expect(fs.existsSync(second.cwd)).toBe(false); + }); + + test.skipIf(process.platform === 'win32')('cleans the fixture when the PTY executable cannot launch', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-launch-failure-')); + const fixtureTmp = path.join(dir, 'fixtures'); + const workerPath = path.join(dir, 'worker.ts'); + const brokenCli = path.join(dir, 'not-executable'); + fs.mkdirSync(fixtureTmp); + fs.writeFileSync(brokenCli, 'This file deliberately has no executable permission.', { mode: 0o644 }); + const runnerUrl = pathToFileURL(path.join(ROOT, 'test/helpers/claude-pty-runner.ts')).href; + fs.writeFileSync(workerPath, ` +import { runPlanSkillCounting } from ${JSON.stringify(runnerUrl)}; +try { + await runPlanSkillCounting({ + skillName: 'plan-design-review', slashCommand: '/plan-design-review', + followUpPrompt: 'Review this launch-failure fixture.', + isLastStep0AUQ: () => false, reviewCountCeiling: 8, + }); + process.exitCode = 1; +} catch (error) { + process.stdout.write('launch failed as expected'); +} +`); + try { + const result = spawnSync(process.execPath, [workerPath], { + cwd: ROOT, + env: { ...process.env, BROWSE_TERMINAL_BINARY: brokenCli, EVALS_HERMETIC: '1', TMPDIR: fixtureTmp }, + encoding: 'utf8', + timeout: 10_000, + }); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toBe('launch failed as expected'); + expect(fs.readdirSync(fixtureTmp)).toEqual([]); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }, 12_000); + + // Bun's real PTY spawn and executable shebangs are POSIX-only. The direct + // fixture tests above still exercise repository setup/cleanup on Windows. + test.skipIf(process.platform === 'win32')( + 'real PTY children see preloaded isolated plans, shipped skills, and only the bare slash command', + async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-subprocess-')); + const fakePath = path.join(dir, 'fake-claude'); + const workerPath = path.join(dir, 'worker.ts'); + const resultPath = path.join(dir, 'result.json'); + const cases = [ + { name: 'design', skillName: 'plan-design-review', prompt: PROMPT, mode: 'complete' }, + { name: 'ceo', skillName: 'plan-ceo-review', prompt: '# Independent CEO plan\nUnique product context.', mode: 'complete' }, + { name: 'exited', skillName: 'plan-eng-review', prompt: '# Early-exit plan\nStill clean up.', mode: 'exit' }, + ].map((item) => ({ ...item, record: path.join(dir, `${item.name}.jsonl`) })); + + // The fake snapshots its environment BEFORE installing the first stdin + // handler. It never dispatches a model or invokes the real Claude CLI. + fs.writeFileSync(fakePath, `#!${process.execPath}\n` + String.raw` +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { spawnSync } from 'node:child_process'; +const record = (event) => fs.appendFileSync(process.env.FIXTURE_RECORD, JSON.stringify(event) + '\n'); +const skillDir = path.join(process.env.CLAUDE_CONFIG_DIR, 'skills', process.env.FIXTURE_SKILL); +const planPath = path.join(process.cwd(), 'PLAN.md'); +const contextPath = path.join(process.cwd(), 'CLAUDE.md'); +record({ + type: 'startup', pid: process.pid, cwd: process.cwd(), argv: process.argv.slice(2), + gitRoot: spawnSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8', timeout: 10_000 }).stdout.trim(), + plan: fs.existsSync(planPath) ? fs.readFileSync(planPath, 'utf8') : null, + context: fs.existsSync(contextPath) ? fs.readFileSync(contextPath, 'utf8') : null, + inheritedDesign: fs.existsSync('DESIGN.md'), inheritedTodos: fs.existsSync('TODOS.md'), + skill: fs.existsSync(path.join(skillDir, 'SKILL.md')) ? fs.readFileSync(path.join(skillDir, 'SKILL.md'), 'utf8') : null, + sections: fs.existsSync(path.join(skillDir, 'sections', 'review-sections.md')) + ? fs.readFileSync(path.join(skillDir, 'sections', 'review-sections.md'), 'utf8') : null, +}); +process.stdin.setRawMode?.(true); +let firstInput = true; +let completion; +process.stdin.on('data', (data) => { + record({ type: 'input', data: data.toString('utf8') }); + if (!firstInput) return; + firstInput = false; + if (process.env.FIXTURE_MODE === 'exit') process.exit(7); + // Longer than the old 3 s delayed fixture send: record that regression + // even if the helper would otherwise return on our completion marker. + completion = setTimeout(() => process.stdout.write('\nGSTACK REVIEW REPORT\n'), 4100); +}); +process.on('SIGINT', () => { + clearTimeout(completion); + record({ type: 'closed', at: Date.now() }); + process.exit(0); +}); +process.stdin.resume(); +`); + fs.chmodSync(fakePath, 0o755); + const runnerUrl = pathToFileURL(path.join(ROOT, 'test/helpers/claude-pty-runner.ts')).href; + fs.writeFileSync(workerPath, ` +import { runPlanSkillCounting } from ${JSON.stringify(runnerUrl)}; +const cases = ${JSON.stringify(cases)}; +const results = await Promise.all(cases.map(async (item) => ({ + name: item.name, + observation: await runPlanSkillCounting({ + skillName: item.skillName, + slashCommand: '/' + item.skillName, + followUpPrompt: item.prompt, + isLastStep0AUQ: () => false, + reviewCountCeiling: 8, + timeoutMs: 15000, + env: { FIXTURE_RECORD: item.record, FIXTURE_SKILL: item.skillName, FIXTURE_MODE: item.mode }, + }), +}))); +await Bun.write(${JSON.stringify(resultPath)}, JSON.stringify(results)); +`); + + const startedAt = Date.now(); + const child = Bun.spawn([process.execPath, workerPath], { + cwd: ROOT, + env: { ...process.env, BROWSE_TERMINAL_BINARY: fakePath, EVALS_HERMETIC: '1' }, + stdout: 'pipe', + stderr: 'pipe', + }); + const killer = setTimeout(() => child.kill('SIGKILL'), 35_000); + try { + const [exitCode, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]); + expect(exitCode, stdout + stderr).toBe(0); + expect(Date.now() - startedAt).toBeLessThan(35_000); + const results = JSON.parse(fs.readFileSync(resultPath, 'utf8')); + const cwds = new Set(); + for (const item of cases) { + const events = fs.readFileSync(item.record, 'utf8').trim().split('\n').map((line) => JSON.parse(line)); + const startup = events[0]; + expect(startup.type).toBe('startup'); + expect(startup.cwd).not.toBe(ROOT); + expect(startup.gitRoot).toBe(startup.cwd); + expect(startup.plan).toBe(item.prompt); + expect(startup.context).toContain('PLAN.md'); + expect(startup.context).toContain(item.prompt); + expect(startup.argv[startup.argv.indexOf('--permission-mode') + 1]).toBe('plan'); + expect(startup.inheritedDesign).toBe(false); + expect(startup.inheritedTodos).toBe(false); + expect(startup.skill).toContain(`name: ${item.skillName}`); + expect(startup.sections).toBe(fs.readFileSync(path.join(ROOT, item.skillName, 'sections/review-sections.md'), 'utf8')); + expect(events.filter((event) => event.type === 'input').map((event) => event.data).join('')) + .toBe(`/${item.skillName}\r`); + expect(fs.existsSync(startup.cwd)).toBe(false); + expect(() => process.kill(startup.pid, 0)).toThrow(); + const result = results.find((result) => result.name === item.name); + expect(result.observation.outcome).toBe(item.mode === 'exit' ? 'exited' : 'completion_summary'); + if (item.mode === 'complete') expect(events.at(-1).type).toBe('closed'); + cwds.add(startup.cwd); + } + expect(cwds.size).toBe(cases.length); + } finally { + clearTimeout(killer); + child.kill('SIGKILL'); + // A failed worker assertion must not leave its PTY fakes running. + for (const item of cases) { + if (!fs.existsSync(item.record)) continue; + const startup = JSON.parse(fs.readFileSync(item.record, 'utf8').split('\n')[0]); + try { process.kill(startup.pid, 'SIGKILL'); } catch { /* already reaped */ } + } + fs.rmSync(dir, { recursive: true, force: true }); + } + }, + 40_000, + ); +}); diff --git a/test/skill-e2e-autoplan-chain.test.ts b/test/skill-e2e-autoplan-chain.test.ts index 72cf0513e..9d9e669c7 100644 --- a/test/skill-e2e-autoplan-chain.test.ts +++ b/test/skill-e2e-autoplan-chain.test.ts @@ -32,6 +32,7 @@ import { spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; +import { stripVTControlCharacters } from 'node:util'; import { launchClaudePty, isPlanReadyVisible, @@ -49,6 +50,12 @@ interface PhaseHit { ts: number; } +function diagnosticTail(text: string): string { + return stripVTControlCharacters(text) + .replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, '') + .slice(-3000); +} + describeE2E('/autoplan chain ordering (periodic)', () => { test( 'phases run sequentially: Phase 1 (CEO) before Phase 3 (Eng), Phase 2 (Design) between when present', @@ -79,6 +86,8 @@ describeE2E('/autoplan chain ordering (periodic)', () => { const hits: PhaseHit[] = []; let outcome: 'chain_complete' | 'plan_ready' | 'timeout' | 'exited' = 'timeout'; let evidence = ''; + let fullSessionEvidence = ''; + let exitCode: number | null = null; try { await Bun.sleep(8000); @@ -144,13 +153,18 @@ describeE2E('/autoplan chain ordering (periodic)', () => { } } } finally { + // Preserve boot failures omitted by mark(), and observe the real exit + // status before close() deliberately terminates a live session. + exitCode = session.exitCode(); + fullSessionEvidence = diagnosticTail(session.visibleText()); await session.close(); } if (outcome === 'exited' || outcome === 'timeout') { throw new Error( - `autoplan chain test FAILED: outcome=${outcome}, hits=${JSON.stringify(hits)}\n` + - `--- evidence (last 3KB) ---\n${evidence}`, + `autoplan chain test FAILED: outcome=${outcome}, exitCode=${exitCode}, hits=${JSON.stringify(hits)}\n` + + `--- post-command evidence (last 3KB) ---\n${diagnosticTail(evidence)}\n` + + `--- full-session visible tail, including startup (last 3KB) ---\n${fullSessionEvidence}`, ); } diff --git a/test/skill-e2e-plan-ceo-finding-count.test.ts b/test/skill-e2e-plan-ceo-finding-count.test.ts index 4b8038b02..6ed054bc2 100644 --- a/test/skill-e2e-plan-ceo-finding-count.test.ts +++ b/test/skill-e2e-plan-ceo-finding-count.test.ts @@ -38,8 +38,8 @@ import { * The default pick (1) routes to "branch diff vs main" — the wrong target * for our seeded fixture (the agent would review the gstack PR itself, * recursively). Picking "Skip interview and plan immediately" bypasses - * Step 0 and routes the agent to review the chat context (where our - * follow-up plan was pasted). + * Step 0 and routes the agent to review the fixture request, which is + * already present in its initial project context. */ function pickSkipInterview(fp: AskUserQuestionFingerprint): number { const skipOpt = fp.options.find((o) => @@ -122,9 +122,6 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', () isLastStep0AUQ: ceoStep0Boundary, reviewCountCeiling: CEILING_DISTINCT + 1, // hard cap above assertion ceiling firstAUQPick: pickSkipInterview, // bypass scope-selection, route to review - // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill - // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). - cwd: process.cwd(), timeoutMs: 1_500_000, // 25 min env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, }); @@ -211,9 +208,6 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', () followUpPrompt: planCeo2PairedFindings(planPath), isLastStep0AUQ: ceoStep0Boundary, reviewCountCeiling: CEILING_PAIRED + 1, - // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill - // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). - cwd: process.cwd(), timeoutMs: 1_500_000, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, }); diff --git a/test/skill-e2e-plan-ceo-split-overflow.test.ts b/test/skill-e2e-plan-ceo-split-overflow.test.ts index 9f9060d68..94bc9abc4 100644 --- a/test/skill-e2e-plan-ceo-split-overflow.test.ts +++ b/test/skill-e2e-plan-ceo-split-overflow.test.ts @@ -73,9 +73,6 @@ describeE2E('/plan-ceo-review split-overflow regression (periodic)', () => { followUpPrompt, isLastStep0AUQ: ceoStep0Boundary, reviewCountCeiling: N + 3, // hard cap above floor + tolerance - // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill - // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). - cwd: process.cwd(), timeoutMs: 1_500_000, // 25 min env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, }); diff --git a/test/skill-e2e-plan-design-finding-count.test.ts b/test/skill-e2e-plan-design-finding-count.test.ts index 6f2398d99..073b68470 100644 --- a/test/skill-e2e-plan-design-finding-count.test.ts +++ b/test/skill-e2e-plan-design-finding-count.test.ts @@ -69,9 +69,6 @@ describeE2E('/plan-design-review per-finding AskUserQuestion count (periodic)', followUpPrompt: planDesign5Findings(planPath), isLastStep0AUQ: designStep0Boundary, reviewCountCeiling: CEILING + 1, - // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill - // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). - cwd: process.cwd(), timeoutMs: 1_500_000, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, }); @@ -94,11 +91,16 @@ describeE2E('/plan-design-review per-finding AskUserQuestion count (periodic)', if (obs.reviewCount < FLOOR) { throw new Error( `BAND FAIL (below floor): reviewCount=${obs.reviewCount} < FLOOR=${FLOOR}.\n` + - `Likely batching regression. Review-phase fingerprints:\n` + + `outcome=${obs.outcome} step0=${obs.step0Count} elapsed=${obs.elapsedMs}ms\n` + + `summary: ${obs.summary}\n` + + `All captured fingerprints (including Step 0):\n` + obs.fingerprints - .filter((f) => !f.preReview) - .map((f) => ` - "${f.promptSnippet.slice(0, 80)}"`) - .join('\n'), + .map( + (f) => + ` - preReview=${f.preReview} sig=${f.signature.slice(0, 12)} prompt="${f.promptSnippet}"`, + ) + .join('\n') + + `\n--- evidence (last 3KB) ---\n${obs.evidence}`, ); } if (obs.reviewCount > CEILING) { diff --git a/test/skill-e2e-plan-devex-finding-count.test.ts b/test/skill-e2e-plan-devex-finding-count.test.ts index e80bdac5d..1e4edb068 100644 --- a/test/skill-e2e-plan-devex-finding-count.test.ts +++ b/test/skill-e2e-plan-devex-finding-count.test.ts @@ -69,9 +69,6 @@ describeE2E('/plan-devex-review per-finding AskUserQuestion count (periodic)', ( followUpPrompt: planDevex5Findings(planPath), isLastStep0AUQ: devexStep0Boundary, reviewCountCeiling: CEILING + 1, - // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill - // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). - cwd: process.cwd(), timeoutMs: 1_500_000, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, }); diff --git a/test/skill-e2e-plan-eng-finding-count.test.ts b/test/skill-e2e-plan-eng-finding-count.test.ts index f51018c77..566776dd9 100644 --- a/test/skill-e2e-plan-eng-finding-count.test.ts +++ b/test/skill-e2e-plan-eng-finding-count.test.ts @@ -68,9 +68,6 @@ describeE2E('/plan-eng-review per-finding AskUserQuestion count (periodic)', () followUpPrompt: planEng5Findings(planPath), isLastStep0AUQ: engStep0Boundary, reviewCountCeiling: CEILING + 1, - // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill - // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). - cwd: process.cwd(), timeoutMs: 1_500_000, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, }); diff --git a/test/skill-e2e-plan-eng-multi-finding-batching.test.ts b/test/skill-e2e-plan-eng-multi-finding-batching.test.ts index 89d2b594d..d6c423d36 100644 --- a/test/skill-e2e-plan-eng-multi-finding-batching.test.ts +++ b/test/skill-e2e-plan-eng-multi-finding-batching.test.ts @@ -65,9 +65,6 @@ describeE2E('/plan-eng-review multi-finding batching regression (periodic)', () followUpPrompt, isLastStep0AUQ: engStep0Boundary, reviewCountCeiling: N + 3, // hard cap above floor + tolerance - // LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill - // registry + hermetic pre-trusted dir (hermetic-env trustedDirs). - cwd: process.cwd(), timeoutMs: 1_500_000, // 25 min env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, });