From 6c5ed933451ca58ccf587aa0223767d1eb52b6ff Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 17 Oct 2026 21:39:13 +0000 Subject: [PATCH] test(autoplan-guard-pty): the scripted actor ends its turn after a background CEO dispatch before reading the close packet PR gate 39070276914 (dc739be) failed at the close-packet check while gate 39069171159 (b789898) passed with byte-identical autoplan inputs. The evidence differs only in order: the failing actor read the close packet in the dispatch turn, so the first tool call after the CEO notice was the publication. The product guard allowed all four guarded calls in both runs. The fixture step 5 told the actor to end its turn only conditionally; step 5 now ends the turn as soon as the result says the agent runs in the background, and step 6 reads the close packet only after the reviewer finished. The close-packet check moves to closePacketReadAfterNotice, replayed in a free test against both captured orders; a second free test pins the actor's steps 5 and 6. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- test/autoplan-guard-pty-fixture.test.ts | 35 +++++++++++++++++++++++ test/helpers/autoplan-guard-pty.ts | 9 ++++-- test/skill-e2e-autoplan-guard-pty.test.ts | 4 +-- 3 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 test/autoplan-guard-pty-fixture.test.ts diff --git a/test/autoplan-guard-pty-fixture.test.ts b/test/autoplan-guard-pty-fixture.test.ts new file mode 100644 index 000000000..93a3790d1 --- /dev/null +++ b/test/autoplan-guard-pty-fixture.test.ts @@ -0,0 +1,35 @@ +import { expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { closePacketReadAfterNotice, prepareGuardSession } from './helpers/autoplan-guard-pty'; + +const base = '/home/runner/.cache/gstack-paid-shard-x/tmp/ap-guard-pty-y'; + +test('the close-packet check accepts a Read in the notice turn and rejects one made before the notice', () => { + // PR gate 39069171159 (b789898, pass): the CEO reviewer's notice started the turn that read the close packet. + expect(closePacketReadAfterNotice([ + `Read ${base}/project/.gstack/tmp/autoplan/autoplan-ceo-oMVeYz/close-packet.md`, + 'Bash true autoplan-published ceo', + `Read ${base}/home/.claude/skills/gstack/autoplan/sections/design-phase.md`, + 'Agent Design review', + ])).toBe(true); + // PR gate 39070276914 (dc739be, fail): the actor read the close packet in the dispatch turn, before the notice. + expect(closePacketReadAfterNotice([ + 'Bash true autoplan-published ceo', + `Read ${base}/home/.claude/skills/gstack/autoplan/sections/design-phase.md`, + `Bash cat "${base}/project/.gstack/tmp/autoplan/dispatch-design.txt"`, + 'Agent Design review', + ])).toBe(false); +}); + +test('the scripted actor ends its turn after a background CEO dispatch and reads the close packet only after the reviewer finishes', () => { + const s = prepareGuardSession(undefined); + try { + const skill = fs.readFileSync(path.join(s.project, '.claude', 'skills', fs.readdirSync(path.join(s.project, '.claude', 'skills'))[0]!, 'SKILL.md'), 'utf8'); + const step = (n: number) => skill.slice(skill.indexOf(`\n${n}. `) + 1, skill.indexOf(`\n${n + 1}. `)); + expect(step(5)).toContain('If its result says the agent is running in the background, end your turn at once with no further tool call; the completion notification starts your next turn.'); + expect(step(5)).not.toContain('If the agent runs in the background, end your turn and wait'); + expect(step(6).startsWith('6. Only after the CEO review agent has finished (in the turn its completion notification starts, when it ran in the background), Read the file whose path step 3 printed')).toBe(true); + expect(skill.indexOf('\n6. ')).toBeGreaterThan(skill.indexOf('\n5. ')); + } finally { s.cleanup(); } +}); diff --git a/test/helpers/autoplan-guard-pty.ts b/test/helpers/autoplan-guard-pty.ts index 3223510d2..4266ab352 100644 --- a/test/helpers/autoplan-guard-pty.ts +++ b/test/helpers/autoplan-guard-pty.ts @@ -37,6 +37,9 @@ export interface GuardSession { cleanup(): void; } +/** The close packet Read (step 6) came after the CEO reviewer's completion notice, so it ran in the notice's new turn. */ +export const closePacketReadAfterNotice = (afterNotice: string[]): boolean => afterNotice.some(c => /^Read .*close-packet\.md$/.test(c)); + export function prepareGuardSession(apiKey: string | undefined): GuardSession { const base = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'ap-guard-pty-'))); const home = path.join(base, 'home'), config = path.join(home, '.claude'), project = path.join(base, 'project'); @@ -78,8 +81,10 @@ console.log(close); `4. Run this Bash command alone: cat "${path.join(artifacts, 'dispatch-ceo.txt')}"`, '5. Call the Agent tool once with these inputs: prompt = the exact, complete output of step 4 (every line, unchanged), ' + 'description "CEO review", subagent_type "general-purpose", and run_in_background false. Include run_in_background even ' + - 'if the tool schema lacks it. If the agent runs in the background, end your turn and wait for its completion notification.', - '6. Read the file whose path step 3 printed, completely (no offset or limit).', + 'if the tool schema lacks it. If its result says the agent is running in the background, end your turn at once with no ' + + 'further tool call; the completion notification starts your next turn.', + '6. Only after the CEO review agent has finished (in the turn its completion notification starts, when it ran in the ' + + 'background), Read the file whose path step 3 printed, completely (no offset or limit).', '7. In one message, write exactly this line as your own text: **Phase 1 complete.** and make exactly one tool call in that ' + 'same message: the Bash command `true autoplan-published ceo`.', `8. In your next assistant message, without waiting for any user input, Read the file ${path.join(sections, 'design-phase.md')} with offset 1 and limit 20.`, diff --git a/test/skill-e2e-autoplan-guard-pty.test.ts b/test/skill-e2e-autoplan-guard-pty.test.ts index c114151d4..652f7c81d 100644 --- a/test/skill-e2e-autoplan-guard-pty.test.ts +++ b/test/skill-e2e-autoplan-guard-pty.test.ts @@ -20,7 +20,7 @@ import { expect } from 'bun:test'; import { describeIfSelected, testIfSelected } from './helpers/e2e-helpers'; import { launchClaudePty, selectPtyNumberedOption } from './helpers/claude-pty-runner'; -import { prepareGuardSession } from './helpers/autoplan-guard-pty'; +import { closePacketReadAfterNotice, prepareGuardSession } from './helpers/autoplan-guard-pty'; const MODEL = 'claude-haiku-4-5-20251001'; @@ -63,7 +63,7 @@ describeIfSelected('autoplan publication guard live PTY', ['autoplan-guard-pty'] // the Phase 2 transition and the Phase 2 reviewer dispatch: four guarded allows, each verified. expect(log.map(e => [e.decision, e.disposition])).toEqual(Array(4).fill(['allow', 'allow'])); // The CEO reviewer's notice started a new turn before the close packet, the Phase 2 entry and the Phase 2 dispatch. - expect(evidence.afterNotice.filter(c => /^Read .*close-packet\.md$/.test(c))).not.toEqual([]); + expect(closePacketReadAfterNotice(evidence.afterNotice), evidence.afterNotice.join('\n')).toBe(true); expect(evidence.afterNotice.filter(c => /^Read .*autoplan\/sections\/design-phase\.md$/.test(c))).not.toEqual([]); expect(evidence.afterNotice).toContain('Agent Design review'); expect(evidence.reviewerReads.filter(f => /\.gstack\/tmp\/autoplan\/autoplan-design-[^/]+\/native-prompt\.md$/.test(f))).not.toEqual([]);