Files
gstack/test/plan-count-fixture.test.ts
T
Garry Tan 35dd014c58 v1.87.5.0 perf: remove idle waits from tests and CI planning (#2897)
* v1.87.5.0 perf: remove idle waits from tests and CI planning

* fix: settle split PTY redraws before routing input

* docs: record final burst-safe test benchmarks

* fix: keep cold-setup snapshot metadata dependency-free

* fix: avoid early-reader pipe races in artifact URL parsing

* fix: preserve safety matches for multiline command payloads

* fix: recognize concurrent CSO publication removal

* test: preload the UI design-review target before invocation

* docs: record validation blocker fixes

* fix: bind plan observer rejection to the invoked command

* fix: count only native design decisions in the UI gate

* docs: clarify UI-positive eval evidence requirements

* test: recognize native UI decisions without weakening finding counts

* test: decouple native UI evidence from question punctuation

* test: recognize concrete native UI decisions independently of prose format

* fix: retain failed eval logs under the hidden CI cache

* test: await telemetry completion instead of racing disk writes
2026-09-21 12:27:25 -04:00

726 lines
44 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/** Free behavioral coverage for the isolated, preloaded plan-count workspace. */
import { describe, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { pathToFileURL } from 'node:url';
import { createPlanCountFixture } from './helpers/plan-count-fixture';
import { getHermeticDirs } from './helpers/hermetic-env';
import { nativePlanCallFingerprint } from './helpers/claude-pty-runner';
import { isDesignCountFirstReview } from './helpers/design-count-review';
import { isDesignUIScopeReview } from './helpers/design-ui-scope';
import designUICapture from './fixtures/plan-design-ui-scope.json';
const ROOT = path.resolve(import.meta.dir, '..');
const PROMPT = '# Seeded settings plan\n\nReview each issue separately.\n' +
'Literal text: "quotes" \'single quotes\' `touch never` $(touch never)\n';
function gitRoot(cwd: string): string {
const result = spawnSync('git', ['rev-parse', '--show-toplevel'], { cwd, encoding: 'utf8', timeout: 10_000 });
expect(result.status, result.stderr).toBe(0);
return fs.realpathSync(result.stdout.trim());
}
describe('plan-count fixtures', () => {
test('preloads the literal plan and initial context in a separate git repository', () => {
const fixture = createPlanCountFixture(PROMPT);
try {
expect(fs.realpathSync(fixture.cwd)).not.toBe(fs.realpathSync(ROOT));
expect(gitRoot(fixture.cwd)).toBe(fs.realpathSync(fixture.cwd));
expect(fs.readFileSync(path.join(fixture.cwd, 'PLAN.md'), 'utf8')).toBe(PROMPT);
const context = fs.readFileSync(path.join(fixture.cwd, 'CLAUDE.md'), 'utf8');
expect(context).toContain('PLAN.md');
expect(context).toContain(PROMPT);
expect(fs.existsSync(path.join(fixture.cwd, 'DESIGN.md'))).toBe(false);
expect(fs.existsSync(path.join(fixture.cwd, 'TODOS.md'))).toBe(false);
expect(fs.existsSync(path.join(fixture.cwd, 'never'))).toBe(false);
expect(fixture.env).toEqual({}); // Mode-routing callers keep normal config.
} finally {
fixture.cleanup();
}
expect(fs.existsSync(fixture.cwd)).toBe(false);
});
test('native count config owns independent state and preserves shared onboarding seeds', () => {
const shared = getHermeticDirs().gstackHome;
const before = fs.readFileSync(path.join(shared, 'config.yaml'), 'utf8');
const markers = fs.readdirSync(shared).filter(name => name.startsWith('.'));
const first = createPlanCountFixture('first native plan', { nativeReviewOnly: true });
const second = createPlanCountFixture('second native plan', { nativeReviewOnly: true });
try {
expect(first.env.GSTACK_HOME).not.toBe(shared);
expect(first.env.GSTACK_HOME).not.toBe(second.env.GSTACK_HOME);
expect(first.env.GSTACK_STATE_ROOT).toBe(first.env.GSTACK_HOME);
expect(fs.readFileSync(path.join(first.env.GSTACK_HOME, 'config.yaml'), 'utf8'))
.toBe(before.replace(/^codex_reviews:.*(?:\r?\n|$)/gm, '') + '\ncodex_reviews: disabled\n');
for (const marker of markers) {
expect(fs.readFileSync(path.join(first.env.GSTACK_HOME, marker), 'utf8'))
.toBe(fs.readFileSync(path.join(shared, marker), 'utf8'));
}
first.cleanup();
first.cleanup();
expect(fs.existsSync(first.env.GSTACK_HOME)).toBe(false);
expect(fs.existsSync(second.env.GSTACK_HOME)).toBe(true);
expect(fs.readFileSync(path.join(shared, 'config.yaml'), 'utf8')).toBe(before);
} finally {
first.cleanup();
second.cleanup();
}
expect(fs.existsSync(second.env.GSTACK_HOME)).toBe(false);
});
test('concurrent fixtures have independent content and cleanup owns only its directory', () => {
const first = createPlanCountFixture('first plan');
const second = createPlanCountFixture('second plan');
try {
expect(first.cwd).not.toBe(second.cwd);
first.cleanup();
first.cleanup();
expect(fs.existsSync(first.cwd)).toBe(false);
expect(fs.readFileSync(path.join(second.cwd, 'PLAN.md'), 'utf8')).toBe('second plan');
expect(gitRoot(second.cwd)).toBe(fs.realpathSync(second.cwd));
} finally {
first.cleanup();
second.cleanup();
}
expect(fs.existsSync(second.cwd)).toBe(false);
});
test('additional design context exists in the initial clean fixture commit', () => {
const design = '# Approved design system\nUse the existing components.\n' + PROMPT;
const fixture = createPlanCountFixture(PROMPT, { files: { 'DESIGN.md': design, 'docs/accepted behavior.md': 'Save is atomic.\n' } });
try {
const committed = spawnSync('git', ['show', 'HEAD:DESIGN.md'], { cwd: fixture.cwd, encoding: 'utf8', timeout: 10_000 });
expect(committed.status, committed.stderr).toBe(0);
expect(committed.stdout).toBe(design);
expect(fs.readFileSync(path.join(fixture.cwd, 'docs/accepted behavior.md'), 'utf8')).toBe('Save is atomic.\n');
expect(spawnSync('git', ['status', '--porcelain'], { cwd: fixture.cwd, encoding: 'utf8', timeout: 10_000 }).stdout).toBe('');
} finally {
fixture.cleanup();
}
expect(fs.existsSync(fixture.cwd)).toBe(false);
});
test('additional context cannot escape the fixture or replace its instructions and Git metadata', () => {
const outside = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-outside-'));
const sentinel = path.join(outside, 'untouched.md');
fs.writeFileSync(sentinel, 'user-owned');
try {
for (const name of [sentinel, '../escape.md', 'nested/../escape.md', '..\\escape.md',
'C:\\escape.md', '/escape.md', '.git/config', 'nested/.git/config', 'PLAN.md', 'CLAUDE.md']) {
expect(() => createPlanCountFixture(PROMPT, { files: { [name]: 'overwrite' } }))
.toThrow('Invalid plan-count fixture file:');
}
expect(fs.readFileSync(sentinel, 'utf8')).toBe('user-owned');
} finally {
fs.rmSync(outside, { recursive: true, force: true });
}
});
test.skipIf(process.platform === 'win32')('cleans the fixture when the PTY executable cannot launch', () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-launch-failure-'));
const fixtureTmp = path.join(dir, 'fixtures');
const workerPath = path.join(dir, 'worker.ts');
const brokenCli = path.join(dir, 'not-executable');
fs.mkdirSync(fixtureTmp);
fs.writeFileSync(brokenCli, 'This file deliberately has no executable permission.', { mode: 0o644 });
const runnerUrl = pathToFileURL(path.join(ROOT, 'test/helpers/claude-pty-runner.ts')).href;
fs.writeFileSync(workerPath, `
import { runPlanSkillCounting } from ${JSON.stringify(runnerUrl)};
try {
await runPlanSkillCounting({
skillName: 'plan-design-review', slashCommand: '/plan-design-review',
followUpPrompt: 'Review this launch-failure fixture.',
isLastStep0AUQ: () => false, reviewCountCeiling: 8,
});
process.exitCode = 1;
} catch (error) {
process.stdout.write('launch failed as expected');
}
`);
try {
const result = spawnSync(process.execPath, [workerPath], {
cwd: ROOT,
env: { ...process.env, BROWSE_TERMINAL_BINARY: brokenCli, EVALS_HERMETIC: '1', TMPDIR: fixtureTmp },
encoding: 'utf8',
timeout: 10_000,
});
expect(result.status, result.stderr).toBe(0);
expect(result.stdout).toBe('launch failed as expected');
expect(fs.readdirSync(fixtureTmp)).toEqual([]);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
}, 12_000);
// Bun's real PTY spawn and executable shebangs are POSIX-only. The direct
// fixture tests above still exercise repository setup/cleanup on Windows.
test.skipIf(process.platform === 'win32')(
'real PTY children see preloaded isolated plans, shipped skills, and only the bare slash command',
async () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-subprocess-'));
const fakePath = path.join(dir, 'fake-claude');
const workerPath = path.join(dir, 'worker.ts');
const resultPath = path.join(dir, 'result.json');
const hostState = path.join(dir, 'host-state');
const hostConfig = 'codex_reviews: enabled\nexplain_level: expert\n';
fs.mkdirSync(hostState);
fs.writeFileSync(path.join(hostState, 'config.yaml'), hostConfig);
const cases = [
{ name: 'design', skillName: 'plan-design-review', prompt: PROMPT, mode: 'complete', files: { 'DESIGN.md': '# Approved design\nKeep the existing layout.\n' } },
{ name: 'design-direct', skillName: 'plan-design-review', prompt: PROMPT, mode: 'direct-finding' },
{ name: 'design-named-target', skillName: 'plan-design-review', prompt: fs.readFileSync(path.join(ROOT, 'test/fixtures/plans/ui-heavy-feature.md'), 'utf8'), mode: 'direct-finding', namedTarget: true },
{ name: 'design-tool-diagnostic', skillName: 'plan-design-review', prompt: PROMPT, mode: 'direct-finding', toolDiagnostic: true },
{ name: 'design-gate-positive', skillName: 'plan-design-review', prompt: PROMPT, mode: 'direct-finding', gateFilter: true },
{ name: 'design-ui-captured', skillName: 'plan-design-review', prompt: fs.readFileSync(path.join(ROOT, 'test/fixtures/plans/ui-heavy-feature.md'), 'utf8'), mode: 'direct-finding', gateFilter: true, namedTarget: true, capturedQuestions: designUICapture.calls[3]!.questions },
{ name: 'design-ui-captured-separators', skillName: 'plan-design-review', prompt: fs.readFileSync(path.join(ROOT, 'test/fixtures/plans/ui-heavy-feature.md'), 'utf8'), mode: 'direct-finding', gateFilter: true, namedTarget: true, capturedQuestions: designUICapture.additionalCaptures[0]!.calls[2]!.questions },
{ name: 'design-ui-captured-decision', skillName: 'plan-design-review', prompt: fs.readFileSync(path.join(ROOT, 'test/fixtures/plans/ui-heavy-feature.md'), 'utf8'), mode: 'direct-finding', gateFilter: true, namedTarget: true, capturedQuestions: [designUICapture.additionalQuestionCaptures[0]!.question] },
{ name: 'design-batched', skillName: 'plan-design-review', prompt: PROMPT, mode: 'batched-finding' },
{ name: 'failed-native', skillName: 'plan-design-review', prompt: PROMPT, mode: 'failed-call' },
{ name: 'native-permission-policy', skillName: 'plan-eng-review', prompt: PROMPT, mode: 'native-permission-policy', report: path.join(dir, 'native-policy-report.md') },
{ name: 'permission-lifecycle', skillName: 'plan-design-review', prompt: PROMPT, mode: 'permission-lifecycle' },
{ name: 'permission', skillName: 'plan-design-review', prompt: PROMPT, mode: 'permission' },
{ name: 'missing-transcript', skillName: 'plan-design-review', prompt: PROMPT, mode: 'missing-transcript' },
{ name: 'ceo', skillName: 'plan-ceo-review', prompt: '# Independent CEO plan\nUnique product context.', mode: 'complete' },
{ name: 'exited', skillName: 'plan-eng-review', prompt: '# Early-exit plan\nStill clean up.', mode: 'exit' },
{ name: 'skip-first', skillName: 'plan-devex-review', prompt: '# Native DX plan', mode: 'prerequisite', skipIndex: 1 },
{ name: 'skip-second', skillName: 'plan-devex-review', prompt: '# Native DX plan', mode: 'prerequisite', skipIndex: 2 },
{ name: 'skip-review-now', skillName: 'plan-ceo-review', prompt: '# Native CEO plan', mode: 'prerequisite', skipIndex: 2, skipLabel: 'Skip — review now (Recommended)' },
{ name: 'caller-policy', skillName: 'plan-devex-review', prompt: '# Native DX plan', mode: 'prerequisite', skipIndex: 2, custom: true },
{ name: 'late-mode', skillName: 'plan-devex-review', prompt: '# Native DX plan', mode: 'late-mode' },
{ name: 'batched-mode', skillName: 'plan-devex-review', prompt: '# Native DX plan', mode: 'batched-mode' },
{ name: 'damaged-submit', skillName: 'plan-devex-review', prompt: '# Native DX plan', mode: 'damaged-submit' },
{ name: 'damaged-menu', skillName: 'plan-ceo-review', prompt: '# Native CEO plan', mode: 'damaged-menu' },
].map((item) => ({ ...item, record: path.join(dir, `${item.name}.jsonl`) }));
// The fake snapshots its environment BEFORE installing the first stdin
// handler. It never dispatches a model or invokes the real Claude CLI.
fs.writeFileSync(fakePath, `#!${process.execPath}\n` + String.raw`
import * as fs from 'node:fs';
import * as path from 'node:path';
import { spawnSync } from 'node:child_process';
const record = (event) => fs.appendFileSync(process.env.FIXTURE_RECORD, JSON.stringify(event) + '\n');
// Older fake menus used CR as a line separator. Emit actual terminal newlines;
// real cursor/erase behavior is exercised separately in pty-screen-session.test.ts.
const render = text => process.stdout.write(text.replace(/\r(?!\n)/g, '\r\n'));
const sessionId = 'fixture-' + process.pid;
const project = path.join(process.env.CLAUDE_CONFIG_DIR, 'projects', sessionId);
fs.mkdirSync(project, { recursive: true });
const native = (role, content, extra = {}) => {
if (process.env.FIXTURE_MODE === 'missing-transcript') return;
fs.appendFileSync(path.join(project, sessionId + '.jsonl'), JSON.stringify({
cwd: process.cwd(), sessionId, isSidechain: false, message: { role, content },
...(process.env.FIXTURE_MODE === 'native-permission-policy' ? { timestamp: new Date().toISOString() } : {}), ...extra,
}) + '\n');
};
native('assistant', [{ type: 'text', text: 'Fixture CLI started.' }]);
let callId = 0;
let nativeQuestions = [];
const ask = (questions) => {
nativeQuestions = questions;
native('assistant', [{ type: 'tool_use', id: 'question-' + (++callId), name: 'AskUserQuestion', input: { questions } }]);
};
const answer = () => native('user', [{ type: 'tool_result', tool_use_id: 'question-' + callId, content: 'Your questions have been answered.' }],
{ toolUseResult: { answers: Object.fromEntries(nativeQuestions.map(q => [q.question, q.options[0].label])) } });
const questionMetadata = (header, question, labels) => ({ header, question, options: labels.map(label => ({ label })) });
const modeQuestion = questionMetadata('Review mode', 'How deep should this DX review go? <gstack-qid:plan-devex-review-mode>', ['DX EXPANSION', 'DX POLISH', 'DX TRIAGE']);
const submitQuestions = [
questionMetadata('Routing setup', 'Should gstack add skill routing rules? <gstack-qid:routing-injection>', ['Add routing rules', 'Skip']),
questionMetadata('Cross-project', 'Enable cross-project learnings?', ['Enable cross-project', 'Keep project-scoped']),
];
const submitPanel = (answered) => '\r← ☒ Routing setup ' + (answered ? '☒' : '☐') + ' Cross-project ✔ Submit →\r' +
'Review your answers\r' + (answered ? '' : '⚠You have not answere all questions\r') +
'│ ●D1 — Should gstack add skill routing rules?\r→Add routing rules\r' +
'Ready to submit your answers?\r1.Sbmi answers\r2Cancel\r';
const skillDir = path.join(process.env.CLAUDE_CONFIG_DIR, 'skills', process.env.FIXTURE_SKILL);
const planPath = path.join(process.cwd(), 'PLAN.md');
const contextPath = path.join(process.cwd(), 'CLAUDE.md');
const config = (key) => {
const result = spawnSync('bash', [process.env.FIXTURE_CONFIG_BIN, 'get', key], { encoding: 'utf8', timeout: 10000 });
if (result.status !== 0) throw new Error('Fixture config failed: ' + result.stderr);
return result.stdout;
};
record({
type: 'startup', pid: process.pid, cwd: process.cwd(), argv: process.argv.slice(2),
stateRoot: process.env.GSTACK_STATE_ROOT, gstackHome: process.env.GSTACK_HOME,
codexReviews: config('codex_reviews'), explainLevel: config('explain_level'),
onboarding: fs.readdirSync(process.env.GSTACK_HOME).filter(name => name.startsWith('.')),
gitRoot: spawnSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8', timeout: 10_000 }).stdout.trim(),
plan: fs.existsSync(planPath) ? fs.readFileSync(planPath, 'utf8') : null,
context: fs.existsSync(contextPath) ? fs.readFileSync(contextPath, 'utf8') : null,
inheritedDesign: fs.existsSync('DESIGN.md'), inheritedTodos: fs.existsSync('TODOS.md'),
design: fs.existsSync('DESIGN.md') ? fs.readFileSync('DESIGN.md', 'utf8') : null,
skill: fs.existsSync(path.join(skillDir, 'SKILL.md')) ? fs.readFileSync(path.join(skillDir, 'SKILL.md'), 'utf8') : null,
sections: fs.existsSync(path.join(skillDir, 'sections', 'review-sections.md'))
? fs.readFileSync(path.join(skillDir, 'sections', 'review-sections.md'), 'utf8') : null,
});
fs.writeFileSync(path.join(process.env.GSTACK_HOME, 'child-owned'), 'isolated');
if (process.env.FIXTURE_MODE === 'exit') render('\x1b[?25lSTARTUP_DIAGNOSTIC fixture CLI booted\x1b[?25h\n');
process.stdin.setRawMode?.(true);
let firstInput = true;
let completion;
let question = 0;
let selected = '';
let permissionStage = 'first';
const filePermission = () => '\nDo you want to make this edit to gstack-test-plan-design.md?\n' +
'1.Yes\n2.Yes,andswitchtoacceptedits(auto-approvefileeditsandcommonfilecommands)forthissession;Yes,and\n' +
'alwaysallowaccessto/tmp/fixtureforthissession\n3.No\nEsctocancel·Tabtoamend\n';
process.stdin.on('data', (data) => {
const rawInput = data.toString('utf8');
record({ type: 'input', data: rawInput });
// These complete native panels accept a numeric shortcut immediately.
// Keep the fixture's existing answer handler sentinel internal; assertions
// below still check the exact bytes the real driver sent.
const instantNative = ['native-permission-policy', 'damaged-menu'].includes(process.env.FIXTURE_MODE)
|| (process.env.FIXTURE_MODE === 'permission-lifecycle' && permissionStage === 'question')
|| (process.env.FIXTURE_MODE === 'prerequisite' && (question === 2 || process.env.FIXTURE_CUSTOM !== 'true'));
const input = instantNative && /^[1-9]$/.test(rawInput) ? rawInput + '\r' : rawInput;
if (!firstInput) {
if (process.env.FIXTURE_MODE === 'native-permission-policy') {
if (permissionStage === 'done') { record({ type: 'unexpected-policy-input', input }); return; }
selected += input.replace(/\r/g, '');
if (input.includes('\r')) {
permissionStage = 'done';
const label = nativeQuestions[0].options[Number(selected) - 1]?.label;
record({ type: 'native-policy-answer', selected, label });
native('user', [{ type: 'tool_result', tool_use_id: 'question-' + callId, content: 'Answered.' }],
{ toolUseResult: { answers: { [nativeQuestions[0].question]: label } } });
// The answered menu survives a provisional heading and partial report.
// Neither may turn its permission wording into a fresh default1.
const summary = 'Completion Summary:\n- Architecture Review: file policy issue resolved\n';
render('\n● ' + summary);
native('assistant', [{ type: 'text', text: summary }], { timestamp: new Date(Date.now() + 1).toISOString() });
setTimeout(() => {
fs.writeFileSync(process.env.FIXTURE_EXPECTED_REPORT, '## GSTACK REVIEW REPORT\n');
record({ type: 'partial-policy-report' });
}, 3000);
completion = setTimeout(() => {
fs.writeFileSync(process.env.FIXTURE_EXPECTED_REPORT, '# Reviewed plan\n\n## GSTACK REVIEW REPORT\n\n' +
'| Review | Status | Findings |\n|---|---|---|\n| Eng Review | clean | policy resolved |\n\n' +
'VERDICT: ENG CLEARED\n\nNO UNRESOLVED DECISIONS\n');
record({ type: 'complete-policy-report' });
render('\nGSTACK REVIEW REPORT\n');
}, 6500);
}
return;
}
if (process.env.FIXTURE_MODE === 'damaged-menu') {
if (question === 0) record({ type: 'input-during-prose', input });
else if (input.includes('\r')) {
record({ type: 'damaged-menu-answer', input });
answer();
render('\nGSTACK REVIEW REPORT\n');
}
return;
}
if (process.env.FIXTURE_MODE === 'damaged-submit') {
if (question === 0 && input === '\x1b[Z') {
question = 1;
record({ type: 'returned-to-unanswered' });
render('\r← ☒ Routing setup ☐ Cross-project ✔ Submit →\r' +
'│ Enable cross-project learnings?\r1.Enable cross-project\r2.Keep project-scoped\r');
} else if (question === 1) {
selected += input.replace(/\r/g, '');
if (input.includes('\r')) {
question = 2;
record({ type: 'answered-remaining-tab', selected });
render(submitPanel(true));
}
} else if (question === 2 && input === '\r') {
// Native JSONL can arrive only after the whole packet is submitted.
ask(submitQuestions);
native('user', [{ type: 'tool_result', tool_use_id: 'question-' + callId, content: 'Your questions have been answered.' }],
{ toolUseResult: { answers: {
[submitQuestions[0].question]: submitQuestions[0].options[0].label,
[submitQuestions[1].question]: submitQuestions[1].options[Number(selected) - 1]?.label,
} } });
record({ type: 'submitted-both-answers' });
render('\nGSTACK REVIEW REPORT\n');
} else record({ type: 'unexpected-submit-input', question, input });
return;
}
if (['late-mode', 'batched-mode'].includes(process.env.FIXTURE_MODE)) {
selected += input.replace(/\r/g, '');
if (input.includes('\r')) {
// The real CLI can defer its native call record until after the UI
// answer. A known multi-question call must still block the override.
if (process.env.FIXTURE_MODE === 'late-mode') ask([modeQuestion]);
const label = modeQuestion.options[Number(selected) - 1]?.label;
record({ type: 'mode-answer', selected, label });
native('user', [{ type: 'tool_result', tool_use_id: 'question-' + callId, content: 'Your questions have been answered.' }],
{ toolUseResult: { answers: { [modeQuestion.question]: label } } });
render('\nGSTACK REVIEW REPORT\n');
}
return;
}
if (['direct-finding', 'failed-call'].includes(process.env.FIXTURE_MODE) && input.includes('\r')) {
answer();
render('\nGSTACK REVIEW REPORT\n');
return;
}
if (process.env.FIXTURE_MODE === 'batched-finding' && input.includes('\r')) {
if (question === 1) {
question = 2;
render('\r☐Loading\rD2 — Define the loading state <gstack-qid:plan-design-review-loading>\r1.Add spinner\r2.Keep blank\r');
} else {
answer();
render('\nGSTACK REVIEW REPORT\n');
}
return;
}
if (process.env.FIXTURE_MODE === 'permission-lifecycle' && input.includes('\r')) {
if (permissionStage === 'first') {
record({ type: 'permission-grant', number: 1, input });
permissionStage = 'pending';
render(filePermission()); // Same pending menu, appended redraw.
completion = setTimeout(() => {
permissionStage = 'completed';
render('\n⎿ Wrote320linesto../fixture/gstack-test-plan-design.md\n' + '·'.repeat(1600));
completion = setTimeout(() => {
permissionStage = 'second';
render(filePermission()); // New request, identical file/text.
}, 4500);
}, 4500);
} else if (permissionStage === 'second') {
record({ type: 'permission-grant', number: 2, input });
permissionStage = 'question';
render('\n⎿ Added2lines\n');
ask([questionMetadata('File policy', 'Do you want to create gstack-test-plan-design.md?', ['Yes', 'No'])]);
render('\n☐ File policy\nDo you want to create gstack-test-plan-design.md?\n1.Yes\n2.No\n' +
'Enter to select · ↑/↓ to navigate · Esc to cancel\n');
} else if (permissionStage === 'question') {
record({ type: 'file-policy-answer', input });
native('user', [{ type: 'tool_result', tool_use_id: 'question-' + callId, content: 'Your question has been answered.' }],
{ toolUseResult: { answers: { [nativeQuestions[0].question]: input.startsWith('2') ? 'No' : 'Yes' } } });
permissionStage = 'done';
render('\nGSTACK REVIEW REPORT\n');
} else record({ type: 'unexpected-permission-input', stage: permissionStage, input });
return;
}
if (process.env.FIXTURE_MODE === 'permission' && input.includes('\r')) {
render('\nGSTACK REVIEW REPORT\n');
return;
}
if (process.env.FIXTURE_MODE !== 'prerequisite') return;
if (question === 1 && input.includes('\r')) {
answer();
record({ type: 'generic-answer', input });
question = 2;
selected = '';
const labels = ['Run /office-hours now', process.env.FIXTURE_SKIP_LABEL || 'Skip — proceed with standard review'];
if (process.env.FIXTURE_SKIP_INDEX === '1') labels.reverse();
ask([questionMetadata('Prerequisite', 'No design doc found. Run /office-hours first?', labels)]);
render('\r☐ Prerequisite\rNo design doc found. Run /office-hours first?\r1.' + labels[0] + '\r2.' + labels[1] + '\r');
return;
}
selected += input.replace(/\r/g, '');
if (input.includes('\r')) {
answer();
record({ type: 'prerequisite-answer', selected });
render(selected === process.env.FIXTURE_SKIP_INDEX ? '\nGSTACK REVIEW REPORT\n' : '\nWRONG_PREREQUISITE_CHOICE\n');
}
return;
}
firstInput = false;
if (process.env.FIXTURE_MODE === 'exit') process.exit(7);
if (process.env.FIXTURE_TOOL_DIAGNOSTIC === 'true') render('Unknown command: --help\n');
if (process.env.FIXTURE_GATE_FILTER === 'true') {
ask([questionMetadata('Focus', 'D2 — Review all 7 design dimensions, or focus on specific areas?', ['All 7 dimensions', 'Choose areas'])]);
answer();
ask([questionMetadata('Outside voices', 'D3 — Want outside design voices before the detailed review?\nA fresh reviewer checks completeness. <gstack-qid:outside-voices-design>', ['Yes, run outside voices (recommended)', 'No, proceed without'])]);
answer();
}
if (process.env.FIXTURE_MODE === 'damaged-menu') {
render('☐Stripe event types\nWhich event should the handler accept?\n1.Specify one canonical event\n2.Accept all events\n' +
'·'.repeat(4200) + '\nMinimum required test cases:\n1.Happy path\n2.Email failure\n3.DB timeout\n4.Unknown event\n5.Unknown user\n1\n');
completion = setTimeout(() => {
question = 1;
ask([questionMetadata('Test scope', 'Should the plan include an integration test? <gstack-qid:plan-ceo-test-scope>',
['Unit tests only for now (recommended)', 'Unit tests + one integration test', 'Integration test only'])]);
render('\n☐Test scope\nShould the plan include an integration test? <gstack-qid:plan-ceo-test-scope>\n' +
'1.Unittestsonlyfornow(recommended)\n2Uni tsts + one integration test\n3.Integrationtestonly\n' +
'Enter to select · ↑/↓ to navigate · Esc to cancel\n');
}, 4100);
return;
}
if (process.env.FIXTURE_MODE === 'damaged-submit') {
render(submitPanel(false));
return;
}
if (['late-mode', 'batched-mode'].includes(process.env.FIXTURE_MODE)) {
if (process.env.FIXTURE_MODE === 'batched-mode') ask([modeQuestion,
questionMetadata('Separate decision', 'Which fixture target should be used?', ['First target', 'Second target'])]);
render('\r☐ Review mode\r' + modeQuestion.question + '\r1.DX EXPANSION\r2.DX POLISH\r3.DX TRIAGE\r');
return;
}
if (['direct-finding', 'batched-finding', 'failed-call'].includes(process.env.FIXTURE_MODE)) {
question = 1;
if (process.env.FIXTURE_MODE === 'failed-call') {
ask([questionMetadata('Missing answer', 'Should the save retry be idempotent?', ['Yes', 'No'])]);
native('user', [{ type: 'tool_result', tool_use_id: 'question-' + callId, is_error: true, content: 'Question rejected' }]);
}
const detail = process.env.FIXTURE_GATE_FILTER === 'true' ? ' Specify the primary button treatment.'.repeat(30) : '';
const questions = process.env.FIXTURE_CAPTURED_QUESTIONS ? JSON.parse(process.env.FIXTURE_CAPTURED_QUESTIONS)
: [questionMetadata('Button style', 'D1 — How should the four header buttons differ?' + detail + ' <gstack-qid:plan-design-review-button-hierarchy>', ['Filled primary', 'Ghost buttons'])];
if (process.env.FIXTURE_MODE === 'batched-finding') questions.push(questionMetadata('Loading', 'D2 — Define the loading state <gstack-qid:plan-design-review-loading>', ['Add spinner', 'Keep blank']));
ask(questions);
if (process.env.FIXTURE_CAPTURED_QUESTIONS) {
const q = questions[0];
render('\r☐' + q.header + '\r' + q.question.split('\n')[0] + '\r' + q.options.map((o, i) => (i === 0 ? '' : '') + (i + 1) + '.' + o.label).join('\r') + '\r');
return;
}
render('\r☐Buttonstyle\r│D1—Howshouldthe4headerbuttonsbedifferentiated?<gstack-qid:plan-design-review-butn-hierarchy>\r1.Filledprimary\r2.Ghostbuttons\r');
return;
}
if (process.env.FIXTURE_MODE === 'prerequisite') {
question = 1;
const suffix = process.env.FIXTURE_CUSTOM === 'true' ? ' Context '.repeat(50) + 'routing-proof-after-240' : '';
ask([questionMetadata('Setup', 'Which fixture setup should be used?' + suffix, ['First setup', 'Second setup'])]);
render('\r☐ Setup\rWhich fixture setup should be used?\r1.First setup\r2.Second setup\r');
return;
}
if (process.env.FIXTURE_MODE === 'native-permission-policy') {
const prompt = 'D1 — Should we create a file that documents always allow access to the project? <gstack-qid:plan-eng-review-file-policy>';
ask([questionMetadata('File policy', prompt, ['Create it', 'Keep current policy'])]);
render('\n☐ File policy\n' + prompt + '\n1.Create it\n2.Keep current policy\nEnter to select · ↑/↓ to navigte · Esc to cancel\n');
return;
}
if (process.env.FIXTURE_MODE === 'permission-lifecycle') {
render(filePermission());
return;
}
if (process.env.FIXTURE_MODE === 'permission') {
render('\rDo you want to create PLAN.md?\r1.Yes\r2.Yes, and switch to accept edits\r3.No\r');
return;
}
// Longer than the old 3 s delayed fixture send: record that regression
// even if the helper would otherwise return on our completion marker.
completion = setTimeout(() => render('\nGSTACK REVIEW REPORT\n'), 4100);
});
process.on('SIGINT', () => {
clearTimeout(completion);
record({ type: 'closed', at: Date.now() });
process.exit(0);
});
process.stdin.resume();
`);
fs.chmodSync(fakePath, 0o755);
const runnerUrl = pathToFileURL(path.join(ROOT, 'test/helpers/claude-pty-runner.ts')).href;
const hermeticUrl = pathToFileURL(path.join(ROOT, 'test/helpers/hermetic-env.ts')).href;
const devexUrl = pathToFileURL(path.join(ROOT, 'test/helpers/devex-count-fixture.ts')).href;
const designUrl = pathToFileURL(path.join(ROOT, 'test/helpers/design-count-review.ts')).href;
const designUIUrl = pathToFileURL(path.join(ROOT, 'test/helpers/design-ui-scope.ts')).href;
fs.writeFileSync(workerPath, `
import { runPlanSkillCounting, designFirstReviewAUQ } from ${JSON.stringify(runnerUrl)};
import { getHermeticDirs } from ${JSON.stringify(hermeticUrl)};
import { devexReviewModePick } from ${JSON.stringify(devexUrl)};
import { isDesignCountFirstReview } from ${JSON.stringify(designUrl)};
import { isDesignUIScopeReview } from ${JSON.stringify(designUIUrl)};
import * as fs from 'node:fs';
import * as path from 'node:path';
const shared = getHermeticDirs().gstackHome;
fs.appendFileSync(path.join(shared, 'config.yaml'), 'codex_reviews: enabled\\nexplain_level: beginner\\n');
const sharedBefore = fs.readFileSync(path.join(shared, 'config.yaml'), 'utf8');
const onboarding = fs.readdirSync(shared).filter(name => name.startsWith('.'));
const cases = ${JSON.stringify(cases)};
const results = await Promise.all(cases.map(async (item) => ({
name: item.name,
observation: await runPlanSkillCounting({
skillName: item.skillName,
slashCommand: '/' + item.skillName + (item.namedTarget ? ' PLAN.md' : ''),
followUpPrompt: item.prompt,
fixtureFiles: item.files,
expectedPlanPath: item.report,
isLastStep0AUQ: item.gateFilter ? fp => fp.nativeCall?.questions[0]?.header === 'Focus' : () => false,
isFirstReviewAUQ: ['direct-finding', 'batched-finding', 'failed-call'].includes(item.mode) ? designFirstReviewAUQ : undefined,
isReviewAUQ: item.capturedQuestions ? isDesignUIScopeReview : item.gateFilter ? isDesignCountFirstReview : item.custom ? fp => fp.promptSnippet.includes('routing-proof-after-240') : undefined,
pickAUQ: item.mode === 'native-permission-policy' ? () => 2
: ['late-mode', 'batched-mode'].includes(item.mode) ? devexReviewModePick
: item.custom ? fp => fp.promptSnippet.includes('routing-proof-after-240') ? 1 : null : undefined,
reviewCountCeiling: item.gateFilter ? 1 : 8,
timeoutMs: item.mode === 'permission-lifecycle' ? 35000 : 28000,
firstAUQPick: () => ['late-mode', 'batched-mode'].includes(item.mode) ? 1 : 2,
env: {
FIXTURE_RECORD: item.record, FIXTURE_SKILL: item.skillName, FIXTURE_MODE: item.mode,
FIXTURE_EXPECTED_REPORT: item.report ?? '',
FIXTURE_CUSTOM: String(item.custom ?? false),
FIXTURE_TOOL_DIAGNOSTIC: String(item.toolDiagnostic ?? false), FIXTURE_GATE_FILTER: String(item.gateFilter ?? false),
FIXTURE_CAPTURED_QUESTIONS: item.capturedQuestions ? JSON.stringify(item.capturedQuestions) : '',
FIXTURE_SKIP_INDEX: String(item.skipIndex ?? ''), FIXTURE_CONFIG_BIN: ${JSON.stringify(path.join(ROOT, 'bin/gstack-config'))},
FIXTURE_SKIP_LABEL: item.skipLabel ?? '',
GSTACK_HOME: ${JSON.stringify(hostState)}, GSTACK_STATE_ROOT: ${JSON.stringify(hostState)},
},
}),
})));
await Bun.write(${JSON.stringify(resultPath)}, JSON.stringify({ results, onboarding, sharedBefore,
sharedAfter: fs.readFileSync(path.join(shared, 'config.yaml'), 'utf8'),
sharedHasChildFile: fs.existsSync(path.join(shared, 'child-owned')),
}));
`);
const startedAt = Date.now();
const child = Bun.spawn([process.execPath, workerPath], {
cwd: ROOT,
env: { ...process.env, BROWSE_TERMINAL_BINARY: fakePath, EVALS_HERMETIC: '1', GSTACK_HOME: hostState, GSTACK_STATE_ROOT: hostState,
EVALS_RUN_ID: 'fixture-integration', GSTACK_EVAL_DIR: path.join(dir, 'eval-artifacts') },
stdout: 'pipe',
stderr: 'pipe',
});
const killer = setTimeout(() => child.kill('SIGKILL'), 35_000);
try {
const [exitCode, stdout, stderr] = await Promise.all([
child.exited,
new Response(child.stdout).text(),
new Response(child.stderr).text(),
]);
expect(exitCode, stdout + stderr).toBe(0);
expect(Date.now() - startedAt).toBeLessThan(35_000);
const report = JSON.parse(fs.readFileSync(resultPath, 'utf8'));
const results = report.results;
expect(report.sharedAfter).toBe(report.sharedBefore);
expect(report.sharedHasChildFile).toBe(false);
expect(fs.readFileSync(path.join(hostState, 'config.yaml'), 'utf8')).toBe(hostConfig);
expect(fs.readdirSync(hostState)).toEqual(['config.yaml']);
const cwds = new Set<string>();
const stateRoots = new Set<string>();
for (const item of cases) {
const events = fs.readFileSync(item.record, 'utf8').trim().split('\n').map((line) => JSON.parse(line));
const startup = events[0];
expect(startup.type).toBe('startup');
expect(startup.cwd).not.toBe(ROOT);
expect(startup.gitRoot).toBe(startup.cwd);
expect(startup.stateRoot).toBe(startup.gstackHome);
expect(startup.stateRoot).not.toBe(hostState);
expect(startup.codexReviews).toBe('disabled');
expect(startup.explainLevel).toBe('beginner');
expect(startup.onboarding).toEqual(report.onboarding);
expect(startup.plan).toBe(item.prompt);
expect(startup.context).toContain('PLAN.md');
expect(startup.context).toContain(item.prompt);
expect(startup.argv[startup.argv.indexOf('--permission-mode') + 1]).toBe('plan');
expect(startup.inheritedDesign).toBe(Boolean(item.files?.['DESIGN.md']));
expect(startup.design).toBe(item.files?.['DESIGN.md'] ?? null);
expect(startup.inheritedTodos).toBe(false);
expect(startup.skill).toContain(`name: ${item.skillName}`);
expect(startup.sections).toBe(fs.readFileSync(path.join(ROOT, item.skillName, 'sections/review-sections.md'), 'utf8'));
expect(events.filter((event) => event.type === 'input').map((event) => event.data).join(''))
.toBe(`/${item.skillName}${item.namedTarget ? ' PLAN.md' : ''}\r` + (item.mode === 'prerequisite' ? `${item.custom ? '1\r' : '2'}${item.skipIndex}`
: item.mode === 'permission-lifecycle' ? '1\r1\r2'
: item.mode === 'damaged-submit' ? '\x1b[Z2\r\r'
: item.mode === 'batched-finding' ? '2\r1\r' : item.mode === 'batched-mode' ? '1\r'
: ['damaged-menu', 'native-permission-policy'].includes(item.mode) ? '2'
: ['direct-finding', 'failed-call', 'permission', 'late-mode'].includes(item.mode) ? '2\r' : ''));
expect(fs.existsSync(startup.cwd)).toBe(false);
expect(fs.existsSync(startup.stateRoot)).toBe(false);
expect(() => process.kill(startup.pid, 0)).toThrow();
const result = results.find((result) => result.name === item.name);
expect(result.observation.outcome, `${item.name}: ${JSON.stringify(result.observation)}`).toBe(item.mode === 'exit' ? 'exited'
: ['missing-transcript', 'failed-call'].includes(item.mode) ? 'transcript_unavailable' : item.gateFilter ? 'ceiling_reached' : 'completion_summary');
const artifacts = result.observation.artifactDir;
expect(result.observation.artifactError).toBeUndefined();
expect(fs.existsSync(artifacts)).toBe(true); // Survives the temporary fixture's cleanup.
const captured = JSON.parse(fs.readFileSync(path.join(artifacts, 'observation.json'), 'utf8'));
expect(captured.outcome).toBe(result.observation.outcome);
expect(captured.capture.cwd).toBe(startup.cwd);
expect(fs.readFileSync(path.join(artifacts, 'terminal.raw.log'), 'utf8')).toContain(item.mode === 'exit' ? 'STARTUP_DIAGNOSTIC' : 'GSTACK REVIEW REPORT');
expect(fs.readFileSync(path.join(artifacts, 'terminal.visible.log'), 'utf8')).toContain(item.mode === 'exit' ? 'STARTUP_DIAGNOSTIC' : 'GSTACK REVIEW REPORT');
if (['direct-finding', 'batched-finding', 'failed-call'].includes(item.mode) && !item.gateFilter) {
expect(result.observation.reviewCount).toBe(1);
expect(result.observation.step0Count).toBe(0);
expect(result.observation.fingerprints).toHaveLength(1);
expect(result.observation.fingerprints[0].nativeCall.questions).toHaveLength(item.mode === 'batched-finding' ? 2 : 1);
}
if (item.gateFilter) {
expect(result.observation.reviewCount).toBe(1);
expect(result.observation.step0Count).toBe(2);
expect(result.observation.fingerprints.map(fp => fp.preReview)).toEqual([true, true, false]);
expect(result.observation.fingerprints.at(-1).nativeCall.questions[0].header).toBe(item.capturedQuestions?.[0].header ?? 'Button style');
const finding = result.observation.fingerprints.at(-1);
expect(finding.promptSnippet.length).toBe(240);
expect((item.capturedQuestions ? isDesignUIScopeReview : isDesignCountFirstReview)(nativePlanCallFingerprint(finding.nativeCall, finding.observedAtMs, finding.preReview))).toBe(true);
}
if (item.mode === 'damaged-menu') {
expect(events.filter(event => event.type === 'input-during-prose')).toEqual([]);
expect(events.filter(event => event.type === 'damaged-menu-answer').map(event => event.input)).toEqual(['2\r']);
expect(result.observation.fingerprints).toHaveLength(1);
expect(result.observation.fingerprints[0].nativeCall.questions[0].header).toBe('Test scope');
}
if (item.mode === 'failed-call') {
expect(result.observation.transcript.calls[0].failure).toContain('is_error');
expect(result.observation.transcript.calls[0].answered).toBe(false);
}
if (item.mode === 'native-permission-policy') {
expect(events.filter(event => event.type === 'unexpected-policy-input')).toEqual([]);
expect(events.filter(event => event.type.endsWith('-policy-report')).map(event => event.type))
.toEqual(['partial-policy-report', 'complete-policy-report']);
expect(fs.readFileSync(item.report!, 'utf8')).toContain('NO UNRESOLVED DECISIONS');
expect(events.filter(event => event.type === 'native-policy-answer').map(event => event.selected)).toEqual(['2']);
expect(result.observation.transcript.calls).toHaveLength(1);
expect(Object.values(result.observation.transcript.calls[0].answers)).toEqual(['Keep current policy']);
}
if (item.mode === 'permission-lifecycle') {
expect(events.filter(event => event.type === 'unexpected-permission-input')).toEqual([]);
expect(events.filter(event => event.type === 'permission-grant').map(event => event.input)).toEqual(['1\r', '1\r']);
expect(events.filter(event => event.type === 'file-policy-answer').map(event => event.input)).toEqual(['2\r']);
expect(result.observation.step0Count).toBe(1);
expect(result.observation.reviewCount).toBe(0);
expect(result.observation.transcript.calls).toHaveLength(1);
expect(result.observation.transcript.calls[0].answers['Do you want to create gstack-test-plan-design.md?']).toBe('No');
}
if (['permission', 'missing-transcript'].includes(item.mode)) {
expect(result.observation.reviewCount).toBe(0);
expect(result.observation.step0Count).toBe(0);
}
if (item.mode === 'exit') {
expect(result.observation.evidence).toContain('exitCode=7');
expect(result.observation.evidence).toContain('STARTUP_DIAGNOSTIC fixture CLI booted');
expect(result.observation.evidence).not.toContain('\x1b');
}
if (item.mode === 'complete') expect(events.at(-1).type).toBe('closed');
if (item.mode === 'prerequisite') {
expect(events.find(event => event.type === 'prerequisite-answer').selected).toBe(String(item.skipIndex));
expect(result.observation.step0Count).toBe(item.custom ? 1 : 2);
expect(result.observation.reviewCount).toBe(item.custom ? 1 : 0);
}
if (['late-mode', 'batched-mode'].includes(item.mode)) {
const expectedLabel = item.mode === 'late-mode' ? 'DX POLISH' : 'DX EXPANSION';
expect(events.find(event => event.type === 'mode-answer').label).toBe(expectedLabel);
expect(result.observation.step0Count).toBe(1);
expect(result.observation.reviewCount).toBe(0);
expect(result.observation.transcript.calls).toHaveLength(1);
const call = result.observation.transcript.calls[0];
expect(call.answers[call.questions[0].question]).toBe(expectedLabel);
expect(call.questions).toHaveLength(item.mode === 'late-mode' ? 1 : 2);
expect(call.unansweredQuestionIndices).toEqual(item.mode === 'late-mode' ? [] : [1]);
}
if (item.mode === 'damaged-submit') {
expect(events.filter(event => ['returned-to-unanswered', 'answered-remaining-tab', 'submitted-both-answers', 'unexpected-submit-input'].includes(event.type)))
.toEqual([{ type: 'returned-to-unanswered' }, { type: 'answered-remaining-tab', selected: '2' }, { type: 'submitted-both-answers' }]);
expect(result.observation.step0Count).toBe(1);
expect(result.observation.reviewCount).toBe(0);
expect(result.observation.transcript.calls).toHaveLength(1);
expect(result.observation.transcript.calls[0].unansweredQuestionIndices).toEqual([]);
expect(result.observation.transcript.calls[0].questions).toHaveLength(2);
}
cwds.add(startup.cwd);
stateRoots.add(startup.stateRoot);
}
expect(cwds.size).toBe(cases.length);
expect(stateRoots.size).toBe(cases.length);
} finally {
clearTimeout(killer);
child.kill('SIGKILL');
// A failed worker assertion must not leave its PTY fakes running.
for (const item of cases) {
if (!fs.existsSync(item.record)) continue;
const startup = JSON.parse(fs.readFileSync(item.record, 'utf8').split('\n')[0]);
try { process.kill(startup.pid, 'SIGKILL'); } catch { /* already reaped */ }
}
fs.rmSync(dir, { recursive: true, force: true });
}
},
40_000,
);
});