mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-07 01:15:47 +01:00
test: isolate plan-count fixtures before starting review workflows
Seed the complete test plan in a private git repository before launching Claude, so a bare slash command cannot review the live workspace while a delayed fixture message remains queued. Preserve count thresholds, parsers and budgets. Add initial-context and installed-discovery tests, and retain startup/terminal diagnostics on failed evaluations.
This commit is contained in:
1 parent
db72f47498
commit
b8c388bd1f
11 files changed
+338
-58
No files matched your search
@@ -26,6 +26,7 @@ import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { hermeticChildEnv, hermeticSkillsConfigDir, isHermeticEnabled } from './hermetic-env';
|
||||
import { createPlanCountFixture } from './plan-count-fixture';
|
||||
|
||||
/** Strip ANSI escapes for pattern-matching against visible text. */
|
||||
export function stripAnsi(s: string): string {
|
||||
@@ -1929,14 +1930,13 @@ export interface PlanSkillCountObservation {
|
||||
* AskUserQuestions until a terminal signal fires.
|
||||
*
|
||||
* Flow:
|
||||
* 1. Boot PTY in plan mode (8s grace + auto-trust dialog).
|
||||
* 2. Send `slashCommand` alone. Sleep ~3s.
|
||||
* 3. Send `followUpPrompt` as a chat message — this is the plan content
|
||||
* the skill reviews. Slash commands with trailing args are rejected by
|
||||
* Claude Code unless the skill defines them, so the plan goes as a
|
||||
* follow-up message (the proven pattern at
|
||||
* skill-e2e-plan-design-with-ui.test.ts:57-71).
|
||||
* 4. Poll loop:
|
||||
* 1. Seed the complete fixture request in an isolated git repository's
|
||||
* PLAN.md and initial CLAUDE.md context, then boot the PTY in that cwd
|
||||
* (8s grace + auto-trust dialog). Skills remain registered in user scope.
|
||||
* 2. Send `slashCommand` alone. The fixture is already in context; sending
|
||||
* it later can queue it behind the skill's first question while the
|
||||
* review incorrectly starts against the operator's live branch.
|
||||
* 3. Poll loop:
|
||||
* - Skip permission dialogs (auto-grant with `defaultPick`).
|
||||
* - On a new numbered-option list, parse prompt + options, build
|
||||
* fingerprint via `auqFingerprint`. Empty-prompt parses are skipped
|
||||
@@ -1970,7 +1970,7 @@ export async function runPlanSkillCounting(opts: {
|
||||
skillName: string;
|
||||
/** Slash command to send alone, e.g. '/plan-ceo-review'. No trailing args. */
|
||||
slashCommand: string;
|
||||
/** Plan content sent as a follow-up message ~3s after the slash command. */
|
||||
/** Fixture request seeded in initial project context before the slash command. */
|
||||
followUpPrompt: string;
|
||||
/** Per-skill predicate: which answered AUQ is the last Step-0 question. */
|
||||
isLastStep0AUQ: Step0BoundaryPredicate;
|
||||
@@ -1989,11 +1989,9 @@ export async function runPlanSkillCounting(opts: {
|
||||
* interview". Pressing the default 1 routes to "branch diff" (the wrong
|
||||
* review target for a seeded fixture). firstAUQPick lets the test pick
|
||||
* "Skip interview" or "describe inline" so the agent reviews the
|
||||
* follow-up plan content the test sent, not the git diff.
|
||||
* fixture plan content, not the git diff.
|
||||
*/
|
||||
firstAUQPick?: (fp: AskUserQuestionFingerprint) => number;
|
||||
/** Working directory. Default process.cwd() (repo cwd holds skill registry). */
|
||||
cwd?: string;
|
||||
/** Total budget for skill to reach a terminal outcome. Default 1_500_000 (25 min). */
|
||||
timeoutMs?: number;
|
||||
/** Extra env merged into the spawned `claude` process. */
|
||||
@@ -2005,14 +2003,21 @@ export async function runPlanSkillCounting(opts: {
|
||||
const defaultPick = opts.defaultPick ?? 1;
|
||||
const timeoutMs = opts.timeoutMs ?? 1_500_000;
|
||||
|
||||
const session = await launchClaudePty({
|
||||
permissionMode: 'plan',
|
||||
cwd: opts.cwd,
|
||||
timeoutMs: timeoutMs + 60_000,
|
||||
env: opts.env,
|
||||
model: opts.model,
|
||||
seedSkills: true,
|
||||
});
|
||||
const fixture = createPlanCountFixture(opts.followUpPrompt);
|
||||
let session: ClaudePtySession;
|
||||
try {
|
||||
session = await launchClaudePty({
|
||||
permissionMode: 'plan',
|
||||
cwd: fixture.cwd,
|
||||
timeoutMs: timeoutMs + 60_000,
|
||||
env: opts.env,
|
||||
model: opts.model,
|
||||
seedSkills: true,
|
||||
});
|
||||
} catch (error) {
|
||||
fixture.cleanup();
|
||||
throw error;
|
||||
}
|
||||
|
||||
const fingerprints: AskUserQuestionFingerprint[] = [];
|
||||
const seen = new Set<string>();
|
||||
@@ -2042,8 +2047,6 @@ export async function runPlanSkillCounting(opts: {
|
||||
await Bun.sleep(8000); // boot grace + auto-trust handler window
|
||||
const since = session.mark();
|
||||
session.send(`${opts.slashCommand}\r`);
|
||||
await Bun.sleep(3000);
|
||||
session.send(`${opts.followUpPrompt}\r`);
|
||||
|
||||
const budgetStart = Date.now();
|
||||
while (Date.now() - budgetStart < timeoutMs) {
|
||||
@@ -2170,7 +2173,11 @@ export async function runPlanSkillCounting(opts: {
|
||||
session.visibleSince(since),
|
||||
);
|
||||
} finally {
|
||||
await session.close();
|
||||
try {
|
||||
await session.close();
|
||||
} finally {
|
||||
fixture.cleanup();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
|
||||
/**
|
||||
* Count evals review a seeded plan, never the checkout that supplies skills.
|
||||
* Put the complete request in Claude's initial project context before the
|
||||
* bare slash command starts: a later message can remain queued behind the
|
||||
* skill's first AskUserQuestion and leave it reviewing the live branch.
|
||||
*/
|
||||
export function createPlanCountFixture(prompt: string): {
|
||||
cwd: string;
|
||||
cleanup(): void;
|
||||
} {
|
||||
const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-plan-count-'));
|
||||
const cleanup = () => fs.rmSync(cwd, { recursive: true, force: true });
|
||||
try {
|
||||
fs.writeFileSync(path.join(cwd, 'PLAN.md'), prompt);
|
||||
fs.writeFileSync(path.join(cwd, 'CLAUDE.md'), [
|
||||
'# Plan review fixture',
|
||||
'',
|
||||
'This repository contains the plan under review. Use PLAN.md as the',
|
||||
'current plan for the requested plan-review skill. The skill installation',
|
||||
'supplies the workflow; its source checkout is not the review target.',
|
||||
'',
|
||||
'The complete user request is available from the start of this session:',
|
||||
'',
|
||||
prompt,
|
||||
'',
|
||||
].join('\n'));
|
||||
|
||||
const git = (args: string[]) => {
|
||||
const result = spawnSync('git', args, {
|
||||
cwd,
|
||||
encoding: 'utf8',
|
||||
timeout: 10_000,
|
||||
});
|
||||
if (result.error || result.status !== 0) {
|
||||
throw new Error(`Could not initialize plan-count fixture: ${result.error?.message ?? result.stderr}`);
|
||||
}
|
||||
};
|
||||
git(['init', '-b', 'main']);
|
||||
git(['add', 'PLAN.md', 'CLAUDE.md']);
|
||||
git(['-c', 'user.name=Plan Count Fixture', '-c', 'user.email=plan-count@example.test',
|
||||
'-c', 'commit.gpgsign=false', 'commit', '--no-verify', '-m', 'Seed review plan']);
|
||||
git(['update-ref', 'refs/remotes/origin/main', 'HEAD']);
|
||||
return { cwd, cleanup };
|
||||
} catch (error) {
|
||||
cleanup();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -160,10 +160,10 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// Each test drives its skill end-to-end; touchfiles include preamble +
|
||||
// completion-status resolvers because they affect question cadence and
|
||||
// terminal output (the regression surface this test catches).
|
||||
'plan-ceo-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-finding-count.test.ts'],
|
||||
'plan-eng-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-eng-finding-count.test.ts'],
|
||||
'plan-design-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-finding-count.test.ts'],
|
||||
'plan-devex-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts'],
|
||||
'plan-ceo-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-ceo-finding-count.test.ts'],
|
||||
'plan-eng-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-eng-finding-count.test.ts'],
|
||||
'plan-design-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-design-finding-count.test.ts'],
|
||||
'plan-devex-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts'],
|
||||
|
||||
// Gate-tier reviewCount-floor counterparts. Catch the May 2026 transcript
|
||||
// bug (model wrote a plan-mode plan and ExitPlanMode'd without firing any
|
||||
@@ -180,8 +180,8 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// a model fires one AUQ then batches the rest into a "## Decisions to
|
||||
// confirm" plan write. runPlanSkillFloorCheck cannot detect that shape
|
||||
// (it exits on first AUQ); runPlanSkillCounting can.
|
||||
'plan-eng-multi-finding-batching': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts'],
|
||||
'plan-ceo-split-overflow': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts'],
|
||||
'plan-eng-multi-finding-batching': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts'],
|
||||
'plan-ceo-split-overflow': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts'],
|
||||
'brain-privacy-gate': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'bin/gstack-brain-sync', 'bin/gstack-artifacts-init', 'bin/gstack-config', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-brain-privacy-gate.test.ts'],
|
||||
|
||||
// /setup-gbrain Path 4 (Remote MCP) — happy + bad-token end-to-end via
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
/** Free behavioral coverage for the isolated, preloaded plan-count workspace. */
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { createPlanCountFixture } from './helpers/plan-count-fixture';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const PROMPT = '# Seeded settings plan\n\nReview each issue separately.\n' +
|
||||
'Literal text: "quotes" \'single quotes\' `touch never` $(touch never)\n';
|
||||
|
||||
function gitRoot(cwd: string): string {
|
||||
const result = spawnSync('git', ['rev-parse', '--show-toplevel'], { cwd, encoding: 'utf8', timeout: 10_000 });
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
return fs.realpathSync(result.stdout.trim());
|
||||
}
|
||||
|
||||
describe('plan-count fixtures', () => {
|
||||
test('preloads the literal plan and initial context in a separate git repository', () => {
|
||||
const fixture = createPlanCountFixture(PROMPT);
|
||||
try {
|
||||
expect(fs.realpathSync(fixture.cwd)).not.toBe(fs.realpathSync(ROOT));
|
||||
expect(gitRoot(fixture.cwd)).toBe(fs.realpathSync(fixture.cwd));
|
||||
expect(fs.readFileSync(path.join(fixture.cwd, 'PLAN.md'), 'utf8')).toBe(PROMPT);
|
||||
const context = fs.readFileSync(path.join(fixture.cwd, 'CLAUDE.md'), 'utf8');
|
||||
expect(context).toContain('PLAN.md');
|
||||
expect(context).toContain(PROMPT);
|
||||
expect(fs.existsSync(path.join(fixture.cwd, 'DESIGN.md'))).toBe(false);
|
||||
expect(fs.existsSync(path.join(fixture.cwd, 'TODOS.md'))).toBe(false);
|
||||
expect(fs.existsSync(path.join(fixture.cwd, 'never'))).toBe(false);
|
||||
} finally {
|
||||
fixture.cleanup();
|
||||
}
|
||||
expect(fs.existsSync(fixture.cwd)).toBe(false);
|
||||
});
|
||||
|
||||
test('concurrent fixtures have independent content and cleanup owns only its directory', () => {
|
||||
const first = createPlanCountFixture('first plan');
|
||||
const second = createPlanCountFixture('second plan');
|
||||
try {
|
||||
expect(first.cwd).not.toBe(second.cwd);
|
||||
first.cleanup();
|
||||
first.cleanup();
|
||||
expect(fs.existsSync(first.cwd)).toBe(false);
|
||||
expect(fs.readFileSync(path.join(second.cwd, 'PLAN.md'), 'utf8')).toBe('second plan');
|
||||
expect(gitRoot(second.cwd)).toBe(fs.realpathSync(second.cwd));
|
||||
} finally {
|
||||
first.cleanup();
|
||||
second.cleanup();
|
||||
}
|
||||
expect(fs.existsSync(second.cwd)).toBe(false);
|
||||
});
|
||||
|
||||
test.skipIf(process.platform === 'win32')('cleans the fixture when the PTY executable cannot launch', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-launch-failure-'));
|
||||
const fixtureTmp = path.join(dir, 'fixtures');
|
||||
const workerPath = path.join(dir, 'worker.ts');
|
||||
const brokenCli = path.join(dir, 'not-executable');
|
||||
fs.mkdirSync(fixtureTmp);
|
||||
fs.writeFileSync(brokenCli, 'This file deliberately has no executable permission.', { mode: 0o644 });
|
||||
const runnerUrl = pathToFileURL(path.join(ROOT, 'test/helpers/claude-pty-runner.ts')).href;
|
||||
fs.writeFileSync(workerPath, `
|
||||
import { runPlanSkillCounting } from ${JSON.stringify(runnerUrl)};
|
||||
try {
|
||||
await runPlanSkillCounting({
|
||||
skillName: 'plan-design-review', slashCommand: '/plan-design-review',
|
||||
followUpPrompt: 'Review this launch-failure fixture.',
|
||||
isLastStep0AUQ: () => false, reviewCountCeiling: 8,
|
||||
});
|
||||
process.exitCode = 1;
|
||||
} catch (error) {
|
||||
process.stdout.write('launch failed as expected');
|
||||
}
|
||||
`);
|
||||
try {
|
||||
const result = spawnSync(process.execPath, [workerPath], {
|
||||
cwd: ROOT,
|
||||
env: { ...process.env, BROWSE_TERMINAL_BINARY: brokenCli, EVALS_HERMETIC: '1', TMPDIR: fixtureTmp },
|
||||
encoding: 'utf8',
|
||||
timeout: 10_000,
|
||||
});
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
expect(result.stdout).toBe('launch failed as expected');
|
||||
expect(fs.readdirSync(fixtureTmp)).toEqual([]);
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}, 12_000);
|
||||
|
||||
// Bun's real PTY spawn and executable shebangs are POSIX-only. The direct
|
||||
// fixture tests above still exercise repository setup/cleanup on Windows.
|
||||
test.skipIf(process.platform === 'win32')(
|
||||
'real PTY children see preloaded isolated plans, shipped skills, and only the bare slash command',
|
||||
async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-subprocess-'));
|
||||
const fakePath = path.join(dir, 'fake-claude');
|
||||
const workerPath = path.join(dir, 'worker.ts');
|
||||
const resultPath = path.join(dir, 'result.json');
|
||||
const cases = [
|
||||
{ name: 'design', skillName: 'plan-design-review', prompt: PROMPT, mode: 'complete' },
|
||||
{ name: 'ceo', skillName: 'plan-ceo-review', prompt: '# Independent CEO plan\nUnique product context.', mode: 'complete' },
|
||||
{ name: 'exited', skillName: 'plan-eng-review', prompt: '# Early-exit plan\nStill clean up.', mode: 'exit' },
|
||||
].map((item) => ({ ...item, record: path.join(dir, `${item.name}.jsonl`) }));
|
||||
|
||||
// The fake snapshots its environment BEFORE installing the first stdin
|
||||
// handler. It never dispatches a model or invokes the real Claude CLI.
|
||||
fs.writeFileSync(fakePath, `#!${process.execPath}\n` + String.raw`
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
const record = (event) => fs.appendFileSync(process.env.FIXTURE_RECORD, JSON.stringify(event) + '\n');
|
||||
const skillDir = path.join(process.env.CLAUDE_CONFIG_DIR, 'skills', process.env.FIXTURE_SKILL);
|
||||
const planPath = path.join(process.cwd(), 'PLAN.md');
|
||||
const contextPath = path.join(process.cwd(), 'CLAUDE.md');
|
||||
record({
|
||||
type: 'startup', pid: process.pid, cwd: process.cwd(), argv: process.argv.slice(2),
|
||||
gitRoot: spawnSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8', timeout: 10_000 }).stdout.trim(),
|
||||
plan: fs.existsSync(planPath) ? fs.readFileSync(planPath, 'utf8') : null,
|
||||
context: fs.existsSync(contextPath) ? fs.readFileSync(contextPath, 'utf8') : null,
|
||||
inheritedDesign: fs.existsSync('DESIGN.md'), inheritedTodos: fs.existsSync('TODOS.md'),
|
||||
skill: fs.existsSync(path.join(skillDir, 'SKILL.md')) ? fs.readFileSync(path.join(skillDir, 'SKILL.md'), 'utf8') : null,
|
||||
sections: fs.existsSync(path.join(skillDir, 'sections', 'review-sections.md'))
|
||||
? fs.readFileSync(path.join(skillDir, 'sections', 'review-sections.md'), 'utf8') : null,
|
||||
});
|
||||
process.stdin.setRawMode?.(true);
|
||||
let firstInput = true;
|
||||
let completion;
|
||||
process.stdin.on('data', (data) => {
|
||||
record({ type: 'input', data: data.toString('utf8') });
|
||||
if (!firstInput) return;
|
||||
firstInput = false;
|
||||
if (process.env.FIXTURE_MODE === 'exit') process.exit(7);
|
||||
// Longer than the old 3 s delayed fixture send: record that regression
|
||||
// even if the helper would otherwise return on our completion marker.
|
||||
completion = setTimeout(() => process.stdout.write('\nGSTACK REVIEW REPORT\n'), 4100);
|
||||
});
|
||||
process.on('SIGINT', () => {
|
||||
clearTimeout(completion);
|
||||
record({ type: 'closed', at: Date.now() });
|
||||
process.exit(0);
|
||||
});
|
||||
process.stdin.resume();
|
||||
`);
|
||||
fs.chmodSync(fakePath, 0o755);
|
||||
const runnerUrl = pathToFileURL(path.join(ROOT, 'test/helpers/claude-pty-runner.ts')).href;
|
||||
fs.writeFileSync(workerPath, `
|
||||
import { runPlanSkillCounting } from ${JSON.stringify(runnerUrl)};
|
||||
const cases = ${JSON.stringify(cases)};
|
||||
const results = await Promise.all(cases.map(async (item) => ({
|
||||
name: item.name,
|
||||
observation: await runPlanSkillCounting({
|
||||
skillName: item.skillName,
|
||||
slashCommand: '/' + item.skillName,
|
||||
followUpPrompt: item.prompt,
|
||||
isLastStep0AUQ: () => false,
|
||||
reviewCountCeiling: 8,
|
||||
timeoutMs: 15000,
|
||||
env: { FIXTURE_RECORD: item.record, FIXTURE_SKILL: item.skillName, FIXTURE_MODE: item.mode },
|
||||
}),
|
||||
})));
|
||||
await Bun.write(${JSON.stringify(resultPath)}, JSON.stringify(results));
|
||||
`);
|
||||
|
||||
const startedAt = Date.now();
|
||||
const child = Bun.spawn([process.execPath, workerPath], {
|
||||
cwd: ROOT,
|
||||
env: { ...process.env, BROWSE_TERMINAL_BINARY: fakePath, EVALS_HERMETIC: '1' },
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe',
|
||||
});
|
||||
const killer = setTimeout(() => child.kill('SIGKILL'), 35_000);
|
||||
try {
|
||||
const [exitCode, stdout, stderr] = await Promise.all([
|
||||
child.exited,
|
||||
new Response(child.stdout).text(),
|
||||
new Response(child.stderr).text(),
|
||||
]);
|
||||
expect(exitCode, stdout + stderr).toBe(0);
|
||||
expect(Date.now() - startedAt).toBeLessThan(35_000);
|
||||
const results = JSON.parse(fs.readFileSync(resultPath, 'utf8'));
|
||||
const cwds = new Set<string>();
|
||||
for (const item of cases) {
|
||||
const events = fs.readFileSync(item.record, 'utf8').trim().split('\n').map((line) => JSON.parse(line));
|
||||
const startup = events[0];
|
||||
expect(startup.type).toBe('startup');
|
||||
expect(startup.cwd).not.toBe(ROOT);
|
||||
expect(startup.gitRoot).toBe(startup.cwd);
|
||||
expect(startup.plan).toBe(item.prompt);
|
||||
expect(startup.context).toContain('PLAN.md');
|
||||
expect(startup.context).toContain(item.prompt);
|
||||
expect(startup.argv[startup.argv.indexOf('--permission-mode') + 1]).toBe('plan');
|
||||
expect(startup.inheritedDesign).toBe(false);
|
||||
expect(startup.inheritedTodos).toBe(false);
|
||||
expect(startup.skill).toContain(`name: ${item.skillName}`);
|
||||
expect(startup.sections).toBe(fs.readFileSync(path.join(ROOT, item.skillName, 'sections/review-sections.md'), 'utf8'));
|
||||
expect(events.filter((event) => event.type === 'input').map((event) => event.data).join(''))
|
||||
.toBe(`/${item.skillName}\r`);
|
||||
expect(fs.existsSync(startup.cwd)).toBe(false);
|
||||
expect(() => process.kill(startup.pid, 0)).toThrow();
|
||||
const result = results.find((result) => result.name === item.name);
|
||||
expect(result.observation.outcome).toBe(item.mode === 'exit' ? 'exited' : 'completion_summary');
|
||||
if (item.mode === 'complete') expect(events.at(-1).type).toBe('closed');
|
||||
cwds.add(startup.cwd);
|
||||
}
|
||||
expect(cwds.size).toBe(cases.length);
|
||||
} finally {
|
||||
clearTimeout(killer);
|
||||
child.kill('SIGKILL');
|
||||
// A failed worker assertion must not leave its PTY fakes running.
|
||||
for (const item of cases) {
|
||||
if (!fs.existsSync(item.record)) continue;
|
||||
const startup = JSON.parse(fs.readFileSync(item.record, 'utf8').split('\n')[0]);
|
||||
try { process.kill(startup.pid, 'SIGKILL'); } catch { /* already reaped */ }
|
||||
}
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
40_000,
|
||||
);
|
||||
});
|
||||
@@ -32,6 +32,7 @@ import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import { stripVTControlCharacters } from 'node:util';
|
||||
import {
|
||||
launchClaudePty,
|
||||
isPlanReadyVisible,
|
||||
@@ -49,6 +50,12 @@ interface PhaseHit {
|
||||
ts: number;
|
||||
}
|
||||
|
||||
function diagnosticTail(text: string): string {
|
||||
return stripVTControlCharacters(text)
|
||||
.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, '')
|
||||
.slice(-3000);
|
||||
}
|
||||
|
||||
describeE2E('/autoplan chain ordering (periodic)', () => {
|
||||
test(
|
||||
'phases run sequentially: Phase 1 (CEO) before Phase 3 (Eng), Phase 2 (Design) between when present',
|
||||
@@ -79,6 +86,8 @@ describeE2E('/autoplan chain ordering (periodic)', () => {
|
||||
const hits: PhaseHit[] = [];
|
||||
let outcome: 'chain_complete' | 'plan_ready' | 'timeout' | 'exited' = 'timeout';
|
||||
let evidence = '';
|
||||
let fullSessionEvidence = '';
|
||||
let exitCode: number | null = null;
|
||||
|
||||
try {
|
||||
await Bun.sleep(8000);
|
||||
@@ -144,13 +153,18 @@ describeE2E('/autoplan chain ordering (periodic)', () => {
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
// Preserve boot failures omitted by mark(), and observe the real exit
|
||||
// status before close() deliberately terminates a live session.
|
||||
exitCode = session.exitCode();
|
||||
fullSessionEvidence = diagnosticTail(session.visibleText());
|
||||
await session.close();
|
||||
}
|
||||
|
||||
if (outcome === 'exited' || outcome === 'timeout') {
|
||||
throw new Error(
|
||||
`autoplan chain test FAILED: outcome=${outcome}, hits=${JSON.stringify(hits)}\n` +
|
||||
`--- evidence (last 3KB) ---\n${evidence}`,
|
||||
`autoplan chain test FAILED: outcome=${outcome}, exitCode=${exitCode}, hits=${JSON.stringify(hits)}\n` +
|
||||
`--- post-command evidence (last 3KB) ---\n${diagnosticTail(evidence)}\n` +
|
||||
`--- full-session visible tail, including startup (last 3KB) ---\n${fullSessionEvidence}`,
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -38,8 +38,8 @@ import {
|
||||
* The default pick (1) routes to "branch diff vs main" — the wrong target
|
||||
* for our seeded fixture (the agent would review the gstack PR itself,
|
||||
* recursively). Picking "Skip interview and plan immediately" bypasses
|
||||
* Step 0 and routes the agent to review the chat context (where our
|
||||
* follow-up plan was pasted).
|
||||
* Step 0 and routes the agent to review the fixture request, which is
|
||||
* already present in its initial project context.
|
||||
*/
|
||||
function pickSkipInterview(fp: AskUserQuestionFingerprint): number {
|
||||
const skipOpt = fp.options.find((o) =>
|
||||
@@ -122,9 +122,6 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', ()
|
||||
isLastStep0AUQ: ceoStep0Boundary,
|
||||
reviewCountCeiling: CEILING_DISTINCT + 1, // hard cap above assertion ceiling
|
||||
firstAUQPick: pickSkipInterview, // bypass scope-selection, route to review
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000, // 25 min
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
@@ -211,9 +208,6 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', ()
|
||||
followUpPrompt: planCeo2PairedFindings(planPath),
|
||||
isLastStep0AUQ: ceoStep0Boundary,
|
||||
reviewCountCeiling: CEILING_PAIRED + 1,
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
@@ -73,9 +73,6 @@ describeE2E('/plan-ceo-review split-overflow regression (periodic)', () => {
|
||||
followUpPrompt,
|
||||
isLastStep0AUQ: ceoStep0Boundary,
|
||||
reviewCountCeiling: N + 3, // hard cap above floor + tolerance
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000, // 25 min
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
@@ -69,9 +69,6 @@ describeE2E('/plan-design-review per-finding AskUserQuestion count (periodic)',
|
||||
followUpPrompt: planDesign5Findings(planPath),
|
||||
isLastStep0AUQ: designStep0Boundary,
|
||||
reviewCountCeiling: CEILING + 1,
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
@@ -94,11 +91,16 @@ describeE2E('/plan-design-review per-finding AskUserQuestion count (periodic)',
|
||||
if (obs.reviewCount < FLOOR) {
|
||||
throw new Error(
|
||||
`BAND FAIL (below floor): reviewCount=${obs.reviewCount} < FLOOR=${FLOOR}.\n` +
|
||||
`Likely batching regression. Review-phase fingerprints:\n` +
|
||||
`outcome=${obs.outcome} step0=${obs.step0Count} elapsed=${obs.elapsedMs}ms\n` +
|
||||
`summary: ${obs.summary}\n` +
|
||||
`All captured fingerprints (including Step 0):\n` +
|
||||
obs.fingerprints
|
||||
.filter((f) => !f.preReview)
|
||||
.map((f) => ` - "${f.promptSnippet.slice(0, 80)}"`)
|
||||
.join('\n'),
|
||||
.map(
|
||||
(f) =>
|
||||
` - preReview=${f.preReview} sig=${f.signature.slice(0, 12)} prompt="${f.promptSnippet}"`,
|
||||
)
|
||||
.join('\n') +
|
||||
`\n--- evidence (last 3KB) ---\n${obs.evidence}`,
|
||||
);
|
||||
}
|
||||
if (obs.reviewCount > CEILING) {
|
||||
|
||||
@@ -69,9 +69,6 @@ describeE2E('/plan-devex-review per-finding AskUserQuestion count (periodic)', (
|
||||
followUpPrompt: planDevex5Findings(planPath),
|
||||
isLastStep0AUQ: devexStep0Boundary,
|
||||
reviewCountCeiling: CEILING + 1,
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
@@ -68,9 +68,6 @@ describeE2E('/plan-eng-review per-finding AskUserQuestion count (periodic)', ()
|
||||
followUpPrompt: planEng5Findings(planPath),
|
||||
isLastStep0AUQ: engStep0Boundary,
|
||||
reviewCountCeiling: CEILING + 1,
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
@@ -65,9 +65,6 @@ describeE2E('/plan-eng-review multi-finding batching regression (periodic)', ()
|
||||
followUpPrompt,
|
||||
isLastStep0AUQ: engStep0Boundary,
|
||||
reviewCountCeiling: N + 3, // hard cap above floor + tolerance
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000, // 25 min
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user