diff --git a/test/helpers/autoplan-guard-pty.ts b/test/helpers/autoplan-guard-pty.ts index a8f7128d9..6ec2da015 100644 --- a/test/helpers/autoplan-guard-pty.ts +++ b/test/helpers/autoplan-guard-pty.ts @@ -5,7 +5,9 @@ * carries /autoplan's own PreToolUse hook block and allowed tools. Its body is * a minimal scripted phase boundary: init, Phase 1 entry, a prepared close * packet, the report published with the `true autoplan-published` no-op, then the - * Phase 2 entry in a later message. Outcomes come from the guard decision log. + * Phase 2 entry in a later message and the Phase 2 reviewer. The restore point + * and phase artifacts live in the project's `.gstack/tmp/autoplan/`, as /autoplan + * Step 1 places them. Outcomes come from the guard decision log and the journals. */ import * as fs from 'node:fs'; import * as path from 'node:path'; @@ -30,32 +32,40 @@ export interface GuardSession { readLog(): Array>; /** Content-bounded journal summary for a failure message: tool names, inputs' first keys, result heads. */ journalSummary(): string[]; + /** Parent tool calls from the first background completion notice on, the notice count, and reviewers' successful Read paths. */ + noticeEvidence(): { notices: number; afterNotice: string[]; reviewerReads: string[] }; cleanup(): void; } export function prepareGuardSession(apiKey: string | undefined): GuardSession { const base = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'ap-guard-pty-'))); const home = path.join(base, 'home'), config = path.join(home, '.claude'), project = path.join(base, 'project'); - const stateRoot = path.join(base, 'state'), artifacts = path.join(base, 'artifacts'); + const stateRoot = path.join(base, 'state'), artifacts = path.join(project, '.gstack', 'tmp', 'autoplan'); for (const dir of [config, project, stateRoot, artifacts, path.join(config, 'skills')]) fs.mkdirSync(dir, { recursive: true }); fs.symlinkSync(ROOT, path.join(config, 'skills', 'gstack'), 'dir'); fs.writeFileSync(path.join(config, '.claude.json'), JSON.stringify(buildSeedConfig({ apiKey, trustedDirs: [project] }))); - const source = path.join(project, 'PLAN.md'), active = path.join(artifacts, 'active.md'), restore = path.join(artifacts, 'restore.md'); + const source = path.join(project, 'PLAN.md'), active = path.join(project, 'ACTIVE.md'), restore = path.join(artifacts, 'restore.md'); fs.writeFileSync(source, '# Plan: greet\nAdd a greet command that prints "hello".\n'); // Stands in for the CEO review the scripted boundary skips: methodology, - // checkpoint, accepted block and close packet, made after the model's own init. - const review = path.join(artifacts, 'review-ceo.ts'), reviewSkill = path.join(artifacts, 'ceo-skill', 'SKILL.md'); - fs.mkdirSync(path.dirname(reviewSkill)); - fs.writeFileSync(reviewSkill, '---\nname: plan-ceo-review\n---\n## Review Sections\nApply every current review criterion.\n'); + // checkpoint, accepted block and close packet, made after the model's own init, + // then the Phase 2 reviewer input, so no helper Bash runs after the first notice. + const review = path.join(base, 'review.ts'), skills = path.join(base, 'review-skills'); + for (const name of ['plan-ceo-review', 'plan-design-review']) { + fs.mkdirSync(path.join(skills, name), { recursive: true }); + fs.writeFileSync(path.join(skills, name, 'SKILL.md'), `---\nname: ${name}\n---\n## Review Sections\nApply every current review criterion.\n`); + } fs.writeFileSync(review, `import { appendFileSync, writeFileSync } from 'node:fs'; import { prepareMethodology, createSnapshot, preparePhaseClose } from ${JSON.stringify(path.join(ROOT, 'bin', 'gstack-autoplan-snapshot.ts'))}; -const [active, restore, skill] = ${JSON.stringify([active, restore, reviewSkill])}; -const method = prepareMethodology('ceo', skill, restore).methodologyPath; +const [active, restore, skills, artifacts] = ${JSON.stringify([active, restore, skills, artifacts])}; +const method = prepareMethodology('ceo', skills + '/plan-ceo-review/SKILL.md', restore).methodologyPath; const snapshot = createSnapshot('ceo', active, restore, method), checkpoint = snapshot.snapshotPath; -writeFileSync(${JSON.stringify(path.join(artifacts, 'dispatch.txt'))}, snapshot.nativeDispatchPrompt); +writeFileSync(artifacts + '/dispatch-ceo.txt', snapshot.nativeDispatchPrompt); appendFileSync(active, '\\nNone: retain the current behavior.\\n\\n'); -console.log(preparePhaseClose('ceo', active, checkpoint, restore, method).closePacketPath); +const close = preparePhaseClose('ceo', active, checkpoint, restore, method).closePacketPath; +const design = prepareMethodology('design', skills + '/plan-design-review/SKILL.md', restore).methodologyPath; +writeFileSync(artifacts + '/dispatch-design.txt', createSnapshot('design', active, restore, design).nativeDispatchPrompt); +console.log(close); `); const tool = path.join(ROOT, 'bin', 'gstack-autoplan-snapshot.ts'); @@ -65,15 +75,19 @@ console.log(preparePhaseClose('ceo', active, checkpoint, restore, method).closeP `1. Run this Bash command alone: bun "${tool}" init "${source}" "${active}" "${restore}"`, `2. Read the file ${path.join(sections, 'ceo-phase.md')} with offset 1 and limit 20.`, `3. Run this Bash command alone: bun "${review}"`, - `4. Run this Bash command alone: cat "${path.join(artifacts, 'dispatch.txt')}"`, + `4. Run this Bash command alone: cat "${path.join(artifacts, 'dispatch-ceo.txt')}"`, '5. Call the Agent tool once with these inputs: prompt = the exact, complete output of step 4 (every line, unchanged), ' + 'description "CEO review", subagent_type "general-purpose", and run_in_background false. Include run_in_background even ' + - 'if the tool schema lacks it. If the agent runs in the background, wait for its completion notification.', + 'if the tool schema lacks it. If the agent runs in the background, end your turn and wait for its completion notification.', '6. Read the file whose path step 3 printed, completely (no offset or limit).', '7. In one message, write exactly this line as your own text: **Phase 1 complete.** and make exactly one tool call in that ' + 'same message: the Bash command `true autoplan-published ceo`.', `8. In a later message, Read the file ${path.join(sections, 'design-phase.md')} with offset 1 and limit 20.`, - '9. Reply with the single word DONE. If any tool call is denied, quote the denial text and stop.', + `9. Run this Bash command alone: cat "${path.join(artifacts, 'dispatch-design.txt')}"`, + '10. Call the Agent tool once with these inputs: prompt = the exact, complete output of step 9 (every line, unchanged), ' + + 'description "Design review", subagent_type "general-purpose". If the agent runs in the background, end your turn and wait ' + + 'for its completion notification.', + '11. Reply with the single word DONE. If any tool call is denied, quote the denial text and stop.', ].join('\n'); const skillDir = path.join(project, '.claude', 'skills', GUARD_SKILL); fs.mkdirSync(skillDir, { recursive: true }); @@ -96,6 +110,22 @@ console.log(preparePhaseClose('ceo', active, checkpoint, restore, method).closeP : b.type === 'text' ? `text ${String(b.text).slice(0, 80)}` : b.type); }))); }, + noticeEvidence: () => { + const projects = path.join(config, 'projects'), sub = `${path.sep}subagents${path.sep}`; + const files = fs.existsSync(projects) ? fs.readdirSync(projects, { recursive: true }).map(String) + .filter(f => f.endsWith('.jsonl')).map(f => path.join(projects, f)) : []; + const rows = (file: string): any[] => fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line)); + const blocks = (rs: any[], type: string): any[] => rs.flatMap(r => Array.isArray(r.message?.content) ? r.message.content.filter((b: any) => b.type === type) : []); + const parent = files.filter(f => !f.includes(sub)).flatMap(rows); + const notice = parent.findIndex(r => r.origin?.kind === 'task-notification'); + const afterNotice = notice < 0 ? [] : blocks(parent.slice(notice), 'tool_use') + .map(b => `${b.name} ${b.input?.file_path ?? b.input?.command ?? b.input?.description ?? ''}`); + const reviewerReads = files.filter(f => f.includes(sub)).flatMap(f => { + const rs = rows(f), ok = new Set(blocks(rs, 'tool_result').filter(b => !b.is_error).map(b => b.tool_use_id)); + return blocks(rs, 'tool_use').filter(b => b.name === 'Read' && ok.has(b.id)).map(b => String(b.input?.file_path)); + }); + return { notices: parent.filter(r => r.origin?.kind === 'task-notification').length, afterNotice, reviewerReads }; + }, readLog: () => fs.existsSync(log) ? fs.readFileSync(log, 'utf8').trim().split('\n').filter(Boolean).map(line => JSON.parse(line)) : [], cleanup: () => fs.rmSync(base, { recursive: true, force: true }), }; diff --git a/test/skill-e2e-autoplan-guard-pty.test.ts b/test/skill-e2e-autoplan-guard-pty.test.ts index 030c32abd..b875eb58b 100644 --- a/test/skill-e2e-autoplan-guard-pty.test.ts +++ b/test/skill-e2e-autoplan-guard-pty.test.ts @@ -4,12 +4,18 @@ * On the pinned Claude Code, in an interactive PTY session with default * settings, a project skill carrying /autoplan's own hook block enters Phase 1, * dispatches the CEO reviewer, reads a prepared close packet, publishes the Phase 1 report in a message - * whose only tool call is `true autoplan-published ceo`, and enters Phase 2 in a - * later message. The reviewer dispatch sends run_in_background, which the + * whose only tool call is `true autoplan-published ceo`, enters Phase 2 in a + * later message and dispatches the Phase 2 reviewer. The reviewer dispatch sends run_in_background, which the * 2.1.29x default fork-subagent schema strips. Claude Code 2.1.29x can write a tool call to the journal only * after PreToolUse returns, so this is where the guard's payload path and its * flush rule meet the real CLI. The outcome is read from the guard decision - * log, never the screen; the no-op must run without a permission prompt. + * log, never the screen. + * + * The reviewer runs in the background, so its completion notice starts a new + * turn and Claude Code stops applying the skill's allowed tools. Every later + * Read of /autoplan's own files (the close packet and the Phase 2 reviewer's + * input in the project's .gstack/tmp/autoplan/, the installed Phase 2 section) + * must still raise no permission card. */ import { expect } from 'bun:test'; import { describeIfSelected, testIfSelected } from './helpers/e2e-helpers'; @@ -34,29 +40,34 @@ describeIfSelected('autoplan publication guard live PTY', ['autoplan-guard-pty'] await Bun.sleep(1_000); const log = s.readLog(); if (phase1Ms === null && log.length) phase1Ms = Math.round(performance.now() - started); - // A permission card is recorded and approved; the no-op must never raise one (DX-13). + // A permission card is recorded and approved so the run can finish; any card fails the case. const screen = await session.currentScreen(); if (/Do you want to proceed\?/.test(screen) && /❯\s*1\.\s*Yes/.test(screen)) { prompts.push(screen); await selectPtyNumberedOption(session, 1); await Bun.sleep(1_500); } - if (log.some(e => e.decision === 'deny') || log.length >= 3) break; + if (log.some(e => e.decision === 'deny') || (log.length >= 4 && s.noticeEvidence().notices >= 2)) break; if (session.exited()) break; } await Bun.sleep(3_000); - const log = s.readLog(); - const summary = { prompts: prompts.length, phase1Ms, totalMs: Math.round(performance.now() - started), + const log = s.readLog(), evidence = s.noticeEvidence(); + const summary = { prompts: prompts.length, phase1Ms, totalMs: Math.round(performance.now() - started), evidence, decisions: log.map(e => ({ decision: e.decision, code: e.code, path: e.path, version: e.claude_code_version })) }; console.log(`[autoplan-guard-pty] ${JSON.stringify(summary)}`); - if (log.length !== 3 || log.some(e => e.decision === 'deny')) + if (log.length !== 4 || log.some(e => e.decision === 'deny') || prompts.length) console.log(`[autoplan-guard-pty] journal:\n${s.journalSummary().join('\n')}\n[autoplan-guard-pty] screen:\n${session.visibleText().slice(-2_000)}`); for (const prompt of prompts) console.log(`[autoplan-guard-pty] permission card approved:\n${prompt}`); - expect(prompts.filter(p => p.replace(/\s+/g, '').includes('autoplan-published'))).toEqual([]); expect(log.filter(e => e.decision === 'deny')).toEqual([]); - // Phase 1 entry, the CEO reviewer dispatch (Cause A: run_in_background sent, stripped from the payload) - // and the Phase 2 transition: three guarded allows, each verified (not an unverified allow). - expect(log.map(e => [e.decision, e.disposition])).toEqual([['allow', 'allow'], ['allow', 'allow'], ['allow', 'allow']]); + // Phase 1 entry, the CEO reviewer dispatch (Cause A: run_in_background sent, stripped from the payload), + // the Phase 2 transition and the Phase 2 reviewer dispatch: four guarded allows, each verified. + expect(log.map(e => [e.decision, e.disposition])).toEqual(Array(4).fill(['allow', 'allow'])); + // The CEO reviewer's notice started a new turn before the close packet, the Phase 2 entry and the Phase 2 dispatch. + expect(evidence.afterNotice.filter(c => /^Read .*close-packet\.md$/.test(c))).not.toEqual([]); + expect(evidence.afterNotice.filter(c => /^Read .*autoplan\/sections\/design-phase\.md$/.test(c))).not.toEqual([]); + expect(evidence.afterNotice).toContain('Agent Design review'); + expect(evidence.reviewerReads.filter(f => /\.gstack\/tmp\/autoplan\/autoplan-design-[^/]+\/native-prompt\.md$/.test(f))).not.toEqual([]); + expect(prompts).toEqual([]); } finally { await session.close(); s.cleanup();