test(evals): autoplan-guard-pty fails on any permission card after the reviewer's notice turn

The scripted boundary now mirrors Step 1's layout (restore point and artifacts
in the project's .gstack/tmp/autoplan/) and continues past the Phase 2 entry to
dispatch the Phase 2 reviewer after the CEO reviewer's completion notice. The
case asserts four verified guard allows, that the close packet Read, the Phase 2
section Read and the Phase 2 dispatch came after a task-notification turn, that
the Phase 2 reviewer read its input, and that no permission card appeared at
all. Still one trial on haiku, about 75-85 seconds.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-08 08:08:13 +00:00
1 parent 95fc35e585
commit ccbe0948fc
2 files changed
+67 -26

No files matched your search

+44 -14
View File
@@ -5,7 +5,9 @@
* carries /autoplan's own PreToolUse hook block and allowed tools. Its body is
* a minimal scripted phase boundary: init, Phase 1 entry, a prepared close
* packet, the report published with the `true autoplan-published` no-op, then the
* Phase 2 entry in a later message. Outcomes come from the guard decision log.
* Phase 2 entry in a later message and the Phase 2 reviewer. The restore point
* and phase artifacts live in the project's `.gstack/tmp/autoplan/`, as /autoplan
* Step 1 places them. Outcomes come from the guard decision log and the journals.
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
@@ -30,32 +32,40 @@ export interface GuardSession {
readLog(): Array<Record<string, any>>;
/** Content-bounded journal summary for a failure message: tool names, inputs' first keys, result heads. */
journalSummary(): string[];
/** Parent tool calls from the first background completion notice on, the notice count, and reviewers' successful Read paths. */
noticeEvidence(): { notices: number; afterNotice: string[]; reviewerReads: string[] };
cleanup(): void;
}
export function prepareGuardSession(apiKey: string | undefined): GuardSession {
const base = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'ap-guard-pty-')));
const home = path.join(base, 'home'), config = path.join(home, '.claude'), project = path.join(base, 'project');
const stateRoot = path.join(base, 'state'), artifacts = path.join(base, 'artifacts');
const stateRoot = path.join(base, 'state'), artifacts = path.join(project, '.gstack', 'tmp', 'autoplan');
for (const dir of [config, project, stateRoot, artifacts, path.join(config, 'skills')]) fs.mkdirSync(dir, { recursive: true });
fs.symlinkSync(ROOT, path.join(config, 'skills', 'gstack'), 'dir');
fs.writeFileSync(path.join(config, '.claude.json'), JSON.stringify(buildSeedConfig({ apiKey, trustedDirs: [project] })));
const source = path.join(project, 'PLAN.md'), active = path.join(artifacts, 'active.md'), restore = path.join(artifacts, 'restore.md');
const source = path.join(project, 'PLAN.md'), active = path.join(project, 'ACTIVE.md'), restore = path.join(artifacts, 'restore.md');
fs.writeFileSync(source, '# Plan: greet\nAdd a greet command that prints "hello".\n');
// Stands in for the CEO review the scripted boundary skips: methodology,
// checkpoint, accepted block and close packet, made after the model's own init.
const review = path.join(artifacts, 'review-ceo.ts'), reviewSkill = path.join(artifacts, 'ceo-skill', 'SKILL.md');
fs.mkdirSync(path.dirname(reviewSkill));
fs.writeFileSync(reviewSkill, '---\nname: plan-ceo-review\n---\n## Review Sections\nApply every current review criterion.\n');
// checkpoint, accepted block and close packet, made after the model's own init,
// then the Phase 2 reviewer input, so no helper Bash runs after the first notice.
const review = path.join(base, 'review.ts'), skills = path.join(base, 'review-skills');
for (const name of ['plan-ceo-review', 'plan-design-review']) {
fs.mkdirSync(path.join(skills, name), { recursive: true });
fs.writeFileSync(path.join(skills, name, 'SKILL.md'), `---\nname: ${name}\n---\n## Review Sections\nApply every current review criterion.\n`);
}
fs.writeFileSync(review, `import { appendFileSync, writeFileSync } from 'node:fs';
import { prepareMethodology, createSnapshot, preparePhaseClose } from ${JSON.stringify(path.join(ROOT, 'bin', 'gstack-autoplan-snapshot.ts'))};
const [active, restore, skill] = ${JSON.stringify([active, restore, reviewSkill])};
const method = prepareMethodology('ceo', skill, restore).methodologyPath;
const [active, restore, skills, artifacts] = ${JSON.stringify([active, restore, skills, artifacts])};
const method = prepareMethodology('ceo', skills + '/plan-ceo-review/SKILL.md', restore).methodologyPath;
const snapshot = createSnapshot('ceo', active, restore, method), checkpoint = snapshot.snapshotPath;
writeFileSync(${JSON.stringify(path.join(artifacts, 'dispatch.txt'))}, snapshot.nativeDispatchPrompt);
writeFileSync(artifacts + '/dispatch-ceo.txt', snapshot.nativeDispatchPrompt);
appendFileSync(active, '<!-- autoplan-accepted:ceo -->\\nNone: retain the current behavior.\\n<!-- /autoplan-accepted:ceo -->\\n');
console.log(preparePhaseClose('ceo', active, checkpoint, restore, method).closePacketPath);
const close = preparePhaseClose('ceo', active, checkpoint, restore, method).closePacketPath;
const design = prepareMethodology('design', skills + '/plan-design-review/SKILL.md', restore).methodologyPath;
writeFileSync(artifacts + '/dispatch-design.txt', createSnapshot('design', active, restore, design).nativeDispatchPrompt);
console.log(close);
`);
const tool = path.join(ROOT, 'bin', 'gstack-autoplan-snapshot.ts');
@@ -65,15 +75,19 @@ console.log(preparePhaseClose('ceo', active, checkpoint, restore, method).closeP
`1. Run this Bash command alone: bun "${tool}" init "${source}" "${active}" "${restore}"`,
`2. Read the file ${path.join(sections, 'ceo-phase.md')} with offset 1 and limit 20.`,
`3. Run this Bash command alone: bun "${review}"`,
`4. Run this Bash command alone: cat "${path.join(artifacts, 'dispatch.txt')}"`,
`4. Run this Bash command alone: cat "${path.join(artifacts, 'dispatch-ceo.txt')}"`,
'5. Call the Agent tool once with these inputs: prompt = the exact, complete output of step 4 (every line, unchanged), ' +
'description "CEO review", subagent_type "general-purpose", and run_in_background false. Include run_in_background even ' +
'if the tool schema lacks it. If the agent runs in the background, wait for its completion notification.',
'if the tool schema lacks it. If the agent runs in the background, end your turn and wait for its completion notification.',
'6. Read the file whose path step 3 printed, completely (no offset or limit).',
'7. In one message, write exactly this line as your own text: **Phase 1 complete.** and make exactly one tool call in that ' +
'same message: the Bash command `true autoplan-published ceo`.',
`8. In a later message, Read the file ${path.join(sections, 'design-phase.md')} with offset 1 and limit 20.`,
'9. Reply with the single word DONE. If any tool call is denied, quote the denial text and stop.',
`9. Run this Bash command alone: cat "${path.join(artifacts, 'dispatch-design.txt')}"`,
'10. Call the Agent tool once with these inputs: prompt = the exact, complete output of step 9 (every line, unchanged), ' +
'description "Design review", subagent_type "general-purpose". If the agent runs in the background, end your turn and wait ' +
'for its completion notification.',
'11. Reply with the single word DONE. If any tool call is denied, quote the denial text and stop.',
].join('\n');
const skillDir = path.join(project, '.claude', 'skills', GUARD_SKILL);
fs.mkdirSync(skillDir, { recursive: true });
@@ -96,6 +110,22 @@ console.log(preparePhaseClose('ceo', active, checkpoint, restore, method).closeP
: b.type === 'text' ? `text ${String(b.text).slice(0, 80)}` : b.type);
})));
},
noticeEvidence: () => {
const projects = path.join(config, 'projects'), sub = `${path.sep}subagents${path.sep}`;
const files = fs.existsSync(projects) ? fs.readdirSync(projects, { recursive: true }).map(String)
.filter(f => f.endsWith('.jsonl')).map(f => path.join(projects, f)) : [];
const rows = (file: string): any[] => fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line));
const blocks = (rs: any[], type: string): any[] => rs.flatMap(r => Array.isArray(r.message?.content) ? r.message.content.filter((b: any) => b.type === type) : []);
const parent = files.filter(f => !f.includes(sub)).flatMap(rows);
const notice = parent.findIndex(r => r.origin?.kind === 'task-notification');
const afterNotice = notice < 0 ? [] : blocks(parent.slice(notice), 'tool_use')
.map(b => `${b.name} ${b.input?.file_path ?? b.input?.command ?? b.input?.description ?? ''}`);
const reviewerReads = files.filter(f => f.includes(sub)).flatMap(f => {
const rs = rows(f), ok = new Set(blocks(rs, 'tool_result').filter(b => !b.is_error).map(b => b.tool_use_id));
return blocks(rs, 'tool_use').filter(b => b.name === 'Read' && ok.has(b.id)).map(b => String(b.input?.file_path));
});
return { notices: parent.filter(r => r.origin?.kind === 'task-notification').length, afterNotice, reviewerReads };
},
readLog: () => fs.existsSync(log) ? fs.readFileSync(log, 'utf8').trim().split('\n').filter(Boolean).map(line => JSON.parse(line)) : [],
cleanup: () => fs.rmSync(base, { recursive: true, force: true }),
};
+23 -12
View File
@@ -4,12 +4,18 @@
* On the pinned Claude Code, in an interactive PTY session with default
* settings, a project skill carrying /autoplan's own hook block enters Phase 1,
* dispatches the CEO reviewer, reads a prepared close packet, publishes the Phase 1 report in a message
* whose only tool call is `true autoplan-published ceo`, and enters Phase 2 in a
* later message. The reviewer dispatch sends run_in_background, which the
* whose only tool call is `true autoplan-published ceo`, enters Phase 2 in a
* later message and dispatches the Phase 2 reviewer. The reviewer dispatch sends run_in_background, which the
* 2.1.29x default fork-subagent schema strips. Claude Code 2.1.29x can write a tool call to the journal only
* after PreToolUse returns, so this is where the guard's payload path and its
* flush rule meet the real CLI. The outcome is read from the guard decision
* log, never the screen; the no-op must run without a permission prompt.
* log, never the screen.
*
* The reviewer runs in the background, so its completion notice starts a new
* turn and Claude Code stops applying the skill's allowed tools. Every later
* Read of /autoplan's own files (the close packet and the Phase 2 reviewer's
* input in the project's .gstack/tmp/autoplan/, the installed Phase 2 section)
* must still raise no permission card.
*/
import { expect } from 'bun:test';
import { describeIfSelected, testIfSelected } from './helpers/e2e-helpers';
@@ -34,29 +40,34 @@ describeIfSelected('autoplan publication guard live PTY', ['autoplan-guard-pty']
await Bun.sleep(1_000);
const log = s.readLog();
if (phase1Ms === null && log.length) phase1Ms = Math.round(performance.now() - started);
// A permission card is recorded and approved; the no-op must never raise one (DX-13).
// A permission card is recorded and approved so the run can finish; any card fails the case.
const screen = await session.currentScreen();
if (/Do you want to proceed\?/.test(screen) && /❯\s*1\.\s*Yes/.test(screen)) {
prompts.push(screen);
await selectPtyNumberedOption(session, 1);
await Bun.sleep(1_500);
}
if (log.some(e => e.decision === 'deny') || log.length >= 3) break;
if (log.some(e => e.decision === 'deny') || (log.length >= 4 && s.noticeEvidence().notices >= 2)) break;
if (session.exited()) break;
}
await Bun.sleep(3_000);
const log = s.readLog();
const summary = { prompts: prompts.length, phase1Ms, totalMs: Math.round(performance.now() - started),
const log = s.readLog(), evidence = s.noticeEvidence();
const summary = { prompts: prompts.length, phase1Ms, totalMs: Math.round(performance.now() - started), evidence,
decisions: log.map(e => ({ decision: e.decision, code: e.code, path: e.path, version: e.claude_code_version })) };
console.log(`[autoplan-guard-pty] ${JSON.stringify(summary)}`);
if (log.length !== 3 || log.some(e => e.decision === 'deny'))
if (log.length !== 4 || log.some(e => e.decision === 'deny') || prompts.length)
console.log(`[autoplan-guard-pty] journal:\n${s.journalSummary().join('\n')}\n[autoplan-guard-pty] screen:\n${session.visibleText().slice(-2_000)}`);
for (const prompt of prompts) console.log(`[autoplan-guard-pty] permission card approved:\n${prompt}`);
expect(prompts.filter(p => p.replace(/\s+/g, '').includes('autoplan-published'))).toEqual([]);
expect(log.filter(e => e.decision === 'deny')).toEqual([]);
// Phase 1 entry, the CEO reviewer dispatch (Cause A: run_in_background sent, stripped from the payload)
// and the Phase 2 transition: three guarded allows, each verified (not an unverified allow).
expect(log.map(e => [e.decision, e.disposition])).toEqual([['allow', 'allow'], ['allow', 'allow'], ['allow', 'allow']]);
// Phase 1 entry, the CEO reviewer dispatch (Cause A: run_in_background sent, stripped from the payload),
// the Phase 2 transition and the Phase 2 reviewer dispatch: four guarded allows, each verified.
expect(log.map(e => [e.decision, e.disposition])).toEqual(Array(4).fill(['allow', 'allow']));
// The CEO reviewer's notice started a new turn before the close packet, the Phase 2 entry and the Phase 2 dispatch.
expect(evidence.afterNotice.filter(c => /^Read .*close-packet\.md$/.test(c))).not.toEqual([]);
expect(evidence.afterNotice.filter(c => /^Read .*autoplan\/sections\/design-phase\.md$/.test(c))).not.toEqual([]);
expect(evidence.afterNotice).toContain('Agent Design review');
expect(evidence.reviewerReads.filter(f => /\.gstack\/tmp\/autoplan\/autoplan-design-[^/]+\/native-prompt\.md$/.test(f))).not.toEqual([]);
expect(prompts).toEqual([]);
} finally {
await session.close();
s.cleanup();