mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-20 15:13:47 +01:00
test(evals): autoplan-guard-pty fails on any permission card after the reviewer's notice turn
The scripted boundary now mirrors Step 1's layout (restore point and artifacts in the project's .gstack/tmp/autoplan/) and continues past the Phase 2 entry to dispatch the Phase 2 reviewer after the CEO reviewer's completion notice. The case asserts four verified guard allows, that the close packet Read, the Phase 2 section Read and the Phase 2 dispatch came after a task-notification turn, that the Phase 2 reviewer read its input, and that no permission card appeared at all. Still one trial on haiku, about 75-85 seconds. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
95fc35e585
commit
ccbe0948fc
2 files changed
+67
-26
No files matched your search
@@ -5,7 +5,9 @@
|
||||
* carries /autoplan's own PreToolUse hook block and allowed tools. Its body is
|
||||
* a minimal scripted phase boundary: init, Phase 1 entry, a prepared close
|
||||
* packet, the report published with the `true autoplan-published` no-op, then the
|
||||
* Phase 2 entry in a later message. Outcomes come from the guard decision log.
|
||||
* Phase 2 entry in a later message and the Phase 2 reviewer. The restore point
|
||||
* and phase artifacts live in the project's `.gstack/tmp/autoplan/`, as /autoplan
|
||||
* Step 1 places them. Outcomes come from the guard decision log and the journals.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
@@ -30,32 +32,40 @@ export interface GuardSession {
|
||||
readLog(): Array<Record<string, any>>;
|
||||
/** Content-bounded journal summary for a failure message: tool names, inputs' first keys, result heads. */
|
||||
journalSummary(): string[];
|
||||
/** Parent tool calls from the first background completion notice on, the notice count, and reviewers' successful Read paths. */
|
||||
noticeEvidence(): { notices: number; afterNotice: string[]; reviewerReads: string[] };
|
||||
cleanup(): void;
|
||||
}
|
||||
|
||||
export function prepareGuardSession(apiKey: string | undefined): GuardSession {
|
||||
const base = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'ap-guard-pty-')));
|
||||
const home = path.join(base, 'home'), config = path.join(home, '.claude'), project = path.join(base, 'project');
|
||||
const stateRoot = path.join(base, 'state'), artifacts = path.join(base, 'artifacts');
|
||||
const stateRoot = path.join(base, 'state'), artifacts = path.join(project, '.gstack', 'tmp', 'autoplan');
|
||||
for (const dir of [config, project, stateRoot, artifacts, path.join(config, 'skills')]) fs.mkdirSync(dir, { recursive: true });
|
||||
fs.symlinkSync(ROOT, path.join(config, 'skills', 'gstack'), 'dir');
|
||||
fs.writeFileSync(path.join(config, '.claude.json'), JSON.stringify(buildSeedConfig({ apiKey, trustedDirs: [project] })));
|
||||
|
||||
const source = path.join(project, 'PLAN.md'), active = path.join(artifacts, 'active.md'), restore = path.join(artifacts, 'restore.md');
|
||||
const source = path.join(project, 'PLAN.md'), active = path.join(project, 'ACTIVE.md'), restore = path.join(artifacts, 'restore.md');
|
||||
fs.writeFileSync(source, '# Plan: greet\nAdd a greet command that prints "hello".\n');
|
||||
// Stands in for the CEO review the scripted boundary skips: methodology,
|
||||
// checkpoint, accepted block and close packet, made after the model's own init.
|
||||
const review = path.join(artifacts, 'review-ceo.ts'), reviewSkill = path.join(artifacts, 'ceo-skill', 'SKILL.md');
|
||||
fs.mkdirSync(path.dirname(reviewSkill));
|
||||
fs.writeFileSync(reviewSkill, '---\nname: plan-ceo-review\n---\n## Review Sections\nApply every current review criterion.\n');
|
||||
// checkpoint, accepted block and close packet, made after the model's own init,
|
||||
// then the Phase 2 reviewer input, so no helper Bash runs after the first notice.
|
||||
const review = path.join(base, 'review.ts'), skills = path.join(base, 'review-skills');
|
||||
for (const name of ['plan-ceo-review', 'plan-design-review']) {
|
||||
fs.mkdirSync(path.join(skills, name), { recursive: true });
|
||||
fs.writeFileSync(path.join(skills, name, 'SKILL.md'), `---\nname: ${name}\n---\n## Review Sections\nApply every current review criterion.\n`);
|
||||
}
|
||||
fs.writeFileSync(review, `import { appendFileSync, writeFileSync } from 'node:fs';
|
||||
import { prepareMethodology, createSnapshot, preparePhaseClose } from ${JSON.stringify(path.join(ROOT, 'bin', 'gstack-autoplan-snapshot.ts'))};
|
||||
const [active, restore, skill] = ${JSON.stringify([active, restore, reviewSkill])};
|
||||
const method = prepareMethodology('ceo', skill, restore).methodologyPath;
|
||||
const [active, restore, skills, artifacts] = ${JSON.stringify([active, restore, skills, artifacts])};
|
||||
const method = prepareMethodology('ceo', skills + '/plan-ceo-review/SKILL.md', restore).methodologyPath;
|
||||
const snapshot = createSnapshot('ceo', active, restore, method), checkpoint = snapshot.snapshotPath;
|
||||
writeFileSync(${JSON.stringify(path.join(artifacts, 'dispatch.txt'))}, snapshot.nativeDispatchPrompt);
|
||||
writeFileSync(artifacts + '/dispatch-ceo.txt', snapshot.nativeDispatchPrompt);
|
||||
appendFileSync(active, '<!-- autoplan-accepted:ceo -->\\nNone: retain the current behavior.\\n<!-- /autoplan-accepted:ceo -->\\n');
|
||||
console.log(preparePhaseClose('ceo', active, checkpoint, restore, method).closePacketPath);
|
||||
const close = preparePhaseClose('ceo', active, checkpoint, restore, method).closePacketPath;
|
||||
const design = prepareMethodology('design', skills + '/plan-design-review/SKILL.md', restore).methodologyPath;
|
||||
writeFileSync(artifacts + '/dispatch-design.txt', createSnapshot('design', active, restore, design).nativeDispatchPrompt);
|
||||
console.log(close);
|
||||
`);
|
||||
|
||||
const tool = path.join(ROOT, 'bin', 'gstack-autoplan-snapshot.ts');
|
||||
@@ -65,15 +75,19 @@ console.log(preparePhaseClose('ceo', active, checkpoint, restore, method).closeP
|
||||
`1. Run this Bash command alone: bun "${tool}" init "${source}" "${active}" "${restore}"`,
|
||||
`2. Read the file ${path.join(sections, 'ceo-phase.md')} with offset 1 and limit 20.`,
|
||||
`3. Run this Bash command alone: bun "${review}"`,
|
||||
`4. Run this Bash command alone: cat "${path.join(artifacts, 'dispatch.txt')}"`,
|
||||
`4. Run this Bash command alone: cat "${path.join(artifacts, 'dispatch-ceo.txt')}"`,
|
||||
'5. Call the Agent tool once with these inputs: prompt = the exact, complete output of step 4 (every line, unchanged), ' +
|
||||
'description "CEO review", subagent_type "general-purpose", and run_in_background false. Include run_in_background even ' +
|
||||
'if the tool schema lacks it. If the agent runs in the background, wait for its completion notification.',
|
||||
'if the tool schema lacks it. If the agent runs in the background, end your turn and wait for its completion notification.',
|
||||
'6. Read the file whose path step 3 printed, completely (no offset or limit).',
|
||||
'7. In one message, write exactly this line as your own text: **Phase 1 complete.** and make exactly one tool call in that ' +
|
||||
'same message: the Bash command `true autoplan-published ceo`.',
|
||||
`8. In a later message, Read the file ${path.join(sections, 'design-phase.md')} with offset 1 and limit 20.`,
|
||||
'9. Reply with the single word DONE. If any tool call is denied, quote the denial text and stop.',
|
||||
`9. Run this Bash command alone: cat "${path.join(artifacts, 'dispatch-design.txt')}"`,
|
||||
'10. Call the Agent tool once with these inputs: prompt = the exact, complete output of step 9 (every line, unchanged), ' +
|
||||
'description "Design review", subagent_type "general-purpose". If the agent runs in the background, end your turn and wait ' +
|
||||
'for its completion notification.',
|
||||
'11. Reply with the single word DONE. If any tool call is denied, quote the denial text and stop.',
|
||||
].join('\n');
|
||||
const skillDir = path.join(project, '.claude', 'skills', GUARD_SKILL);
|
||||
fs.mkdirSync(skillDir, { recursive: true });
|
||||
@@ -96,6 +110,22 @@ console.log(preparePhaseClose('ceo', active, checkpoint, restore, method).closeP
|
||||
: b.type === 'text' ? `text ${String(b.text).slice(0, 80)}` : b.type);
|
||||
})));
|
||||
},
|
||||
noticeEvidence: () => {
|
||||
const projects = path.join(config, 'projects'), sub = `${path.sep}subagents${path.sep}`;
|
||||
const files = fs.existsSync(projects) ? fs.readdirSync(projects, { recursive: true }).map(String)
|
||||
.filter(f => f.endsWith('.jsonl')).map(f => path.join(projects, f)) : [];
|
||||
const rows = (file: string): any[] => fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line));
|
||||
const blocks = (rs: any[], type: string): any[] => rs.flatMap(r => Array.isArray(r.message?.content) ? r.message.content.filter((b: any) => b.type === type) : []);
|
||||
const parent = files.filter(f => !f.includes(sub)).flatMap(rows);
|
||||
const notice = parent.findIndex(r => r.origin?.kind === 'task-notification');
|
||||
const afterNotice = notice < 0 ? [] : blocks(parent.slice(notice), 'tool_use')
|
||||
.map(b => `${b.name} ${b.input?.file_path ?? b.input?.command ?? b.input?.description ?? ''}`);
|
||||
const reviewerReads = files.filter(f => f.includes(sub)).flatMap(f => {
|
||||
const rs = rows(f), ok = new Set(blocks(rs, 'tool_result').filter(b => !b.is_error).map(b => b.tool_use_id));
|
||||
return blocks(rs, 'tool_use').filter(b => b.name === 'Read' && ok.has(b.id)).map(b => String(b.input?.file_path));
|
||||
});
|
||||
return { notices: parent.filter(r => r.origin?.kind === 'task-notification').length, afterNotice, reviewerReads };
|
||||
},
|
||||
readLog: () => fs.existsSync(log) ? fs.readFileSync(log, 'utf8').trim().split('\n').filter(Boolean).map(line => JSON.parse(line)) : [],
|
||||
cleanup: () => fs.rmSync(base, { recursive: true, force: true }),
|
||||
};
|
||||
|
||||
@@ -4,12 +4,18 @@
|
||||
* On the pinned Claude Code, in an interactive PTY session with default
|
||||
* settings, a project skill carrying /autoplan's own hook block enters Phase 1,
|
||||
* dispatches the CEO reviewer, reads a prepared close packet, publishes the Phase 1 report in a message
|
||||
* whose only tool call is `true autoplan-published ceo`, and enters Phase 2 in a
|
||||
* later message. The reviewer dispatch sends run_in_background, which the
|
||||
* whose only tool call is `true autoplan-published ceo`, enters Phase 2 in a
|
||||
* later message and dispatches the Phase 2 reviewer. The reviewer dispatch sends run_in_background, which the
|
||||
* 2.1.29x default fork-subagent schema strips. Claude Code 2.1.29x can write a tool call to the journal only
|
||||
* after PreToolUse returns, so this is where the guard's payload path and its
|
||||
* flush rule meet the real CLI. The outcome is read from the guard decision
|
||||
* log, never the screen; the no-op must run without a permission prompt.
|
||||
* log, never the screen.
|
||||
*
|
||||
* The reviewer runs in the background, so its completion notice starts a new
|
||||
* turn and Claude Code stops applying the skill's allowed tools. Every later
|
||||
* Read of /autoplan's own files (the close packet and the Phase 2 reviewer's
|
||||
* input in the project's .gstack/tmp/autoplan/, the installed Phase 2 section)
|
||||
* must still raise no permission card.
|
||||
*/
|
||||
import { expect } from 'bun:test';
|
||||
import { describeIfSelected, testIfSelected } from './helpers/e2e-helpers';
|
||||
@@ -34,29 +40,34 @@ describeIfSelected('autoplan publication guard live PTY', ['autoplan-guard-pty']
|
||||
await Bun.sleep(1_000);
|
||||
const log = s.readLog();
|
||||
if (phase1Ms === null && log.length) phase1Ms = Math.round(performance.now() - started);
|
||||
// A permission card is recorded and approved; the no-op must never raise one (DX-13).
|
||||
// A permission card is recorded and approved so the run can finish; any card fails the case.
|
||||
const screen = await session.currentScreen();
|
||||
if (/Do you want to proceed\?/.test(screen) && /❯\s*1\.\s*Yes/.test(screen)) {
|
||||
prompts.push(screen);
|
||||
await selectPtyNumberedOption(session, 1);
|
||||
await Bun.sleep(1_500);
|
||||
}
|
||||
if (log.some(e => e.decision === 'deny') || log.length >= 3) break;
|
||||
if (log.some(e => e.decision === 'deny') || (log.length >= 4 && s.noticeEvidence().notices >= 2)) break;
|
||||
if (session.exited()) break;
|
||||
}
|
||||
await Bun.sleep(3_000);
|
||||
const log = s.readLog();
|
||||
const summary = { prompts: prompts.length, phase1Ms, totalMs: Math.round(performance.now() - started),
|
||||
const log = s.readLog(), evidence = s.noticeEvidence();
|
||||
const summary = { prompts: prompts.length, phase1Ms, totalMs: Math.round(performance.now() - started), evidence,
|
||||
decisions: log.map(e => ({ decision: e.decision, code: e.code, path: e.path, version: e.claude_code_version })) };
|
||||
console.log(`[autoplan-guard-pty] ${JSON.stringify(summary)}`);
|
||||
if (log.length !== 3 || log.some(e => e.decision === 'deny'))
|
||||
if (log.length !== 4 || log.some(e => e.decision === 'deny') || prompts.length)
|
||||
console.log(`[autoplan-guard-pty] journal:\n${s.journalSummary().join('\n')}\n[autoplan-guard-pty] screen:\n${session.visibleText().slice(-2_000)}`);
|
||||
for (const prompt of prompts) console.log(`[autoplan-guard-pty] permission card approved:\n${prompt}`);
|
||||
expect(prompts.filter(p => p.replace(/\s+/g, '').includes('autoplan-published'))).toEqual([]);
|
||||
expect(log.filter(e => e.decision === 'deny')).toEqual([]);
|
||||
// Phase 1 entry, the CEO reviewer dispatch (Cause A: run_in_background sent, stripped from the payload)
|
||||
// and the Phase 2 transition: three guarded allows, each verified (not an unverified allow).
|
||||
expect(log.map(e => [e.decision, e.disposition])).toEqual([['allow', 'allow'], ['allow', 'allow'], ['allow', 'allow']]);
|
||||
// Phase 1 entry, the CEO reviewer dispatch (Cause A: run_in_background sent, stripped from the payload),
|
||||
// the Phase 2 transition and the Phase 2 reviewer dispatch: four guarded allows, each verified.
|
||||
expect(log.map(e => [e.decision, e.disposition])).toEqual(Array(4).fill(['allow', 'allow']));
|
||||
// The CEO reviewer's notice started a new turn before the close packet, the Phase 2 entry and the Phase 2 dispatch.
|
||||
expect(evidence.afterNotice.filter(c => /^Read .*close-packet\.md$/.test(c))).not.toEqual([]);
|
||||
expect(evidence.afterNotice.filter(c => /^Read .*autoplan\/sections\/design-phase\.md$/.test(c))).not.toEqual([]);
|
||||
expect(evidence.afterNotice).toContain('Agent Design review');
|
||||
expect(evidence.reviewerReads.filter(f => /\.gstack\/tmp\/autoplan\/autoplan-design-[^/]+\/native-prompt\.md$/.test(f))).not.toEqual([]);
|
||||
expect(prompts).toEqual([]);
|
||||
} finally {
|
||||
await session.close();
|
||||
s.cleanup();
|
||||
|
||||
Reference in new issue
Block a user