test(autoplan-guard-pty): the scripted actor ends its turn after a background CEO dispatch before reading the close packet

PR gate 39070276914 (dc739be) failed at the close-packet check while gate
39069171159 (b789898) passed with byte-identical autoplan inputs. The evidence
differs only in order: the failing actor read the close packet in the dispatch
turn, so the first tool call after the CEO notice was the publication. The
product guard allowed all four guarded calls in both runs. The fixture step 5
told the actor to end its turn only conditionally; step 5 now ends the turn as
soon as the result says the agent runs in the background, and step 6 reads the
close packet only after the reviewer finished. The close-packet check moves to
closePacketReadAfterNotice, replayed in a free test against both captured
orders; a second free test pins the actor's steps 5 and 6.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-17 21:39:13 +00:00
1 parent dc739be05a
commit 6c5ed93345
3 files changed
+44 -4

No files matched your search

+35
View File
@@ -0,0 +1,35 @@
import { expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { closePacketReadAfterNotice, prepareGuardSession } from './helpers/autoplan-guard-pty';
const base = '/home/runner/.cache/gstack-paid-shard-x/tmp/ap-guard-pty-y';
test('the close-packet check accepts a Read in the notice turn and rejects one made before the notice', () => {
// PR gate 39069171159 (b789898, pass): the CEO reviewer's notice started the turn that read the close packet.
expect(closePacketReadAfterNotice([
`Read ${base}/project/.gstack/tmp/autoplan/autoplan-ceo-oMVeYz/close-packet.md`,
'Bash true autoplan-published ceo',
`Read ${base}/home/.claude/skills/gstack/autoplan/sections/design-phase.md`,
'Agent Design review',
])).toBe(true);
// PR gate 39070276914 (dc739be, fail): the actor read the close packet in the dispatch turn, before the notice.
expect(closePacketReadAfterNotice([
'Bash true autoplan-published ceo',
`Read ${base}/home/.claude/skills/gstack/autoplan/sections/design-phase.md`,
`Bash cat "${base}/project/.gstack/tmp/autoplan/dispatch-design.txt"`,
'Agent Design review',
])).toBe(false);
});
test('the scripted actor ends its turn after a background CEO dispatch and reads the close packet only after the reviewer finishes', () => {
const s = prepareGuardSession(undefined);
try {
const skill = fs.readFileSync(path.join(s.project, '.claude', 'skills', fs.readdirSync(path.join(s.project, '.claude', 'skills'))[0]!, 'SKILL.md'), 'utf8');
const step = (n: number) => skill.slice(skill.indexOf(`\n${n}. `) + 1, skill.indexOf(`\n${n + 1}. `));
expect(step(5)).toContain('If its result says the agent is running in the background, end your turn at once with no further tool call; the completion notification starts your next turn.');
expect(step(5)).not.toContain('If the agent runs in the background, end your turn and wait');
expect(step(6).startsWith('6. Only after the CEO review agent has finished (in the turn its completion notification starts, when it ran in the background), Read the file whose path step 3 printed')).toBe(true);
expect(skill.indexOf('\n6. ')).toBeGreaterThan(skill.indexOf('\n5. '));
} finally { s.cleanup(); }
});
+7 -2
View File
@@ -37,6 +37,9 @@ export interface GuardSession {
cleanup(): void;
}
/** The close packet Read (step 6) came after the CEO reviewer's completion notice, so it ran in the notice's new turn. */
export const closePacketReadAfterNotice = (afterNotice: string[]): boolean => afterNotice.some(c => /^Read .*close-packet\.md$/.test(c));
export function prepareGuardSession(apiKey: string | undefined): GuardSession {
const base = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'ap-guard-pty-')));
const home = path.join(base, 'home'), config = path.join(home, '.claude'), project = path.join(base, 'project');
@@ -78,8 +81,10 @@ console.log(close);
`4. Run this Bash command alone: cat "${path.join(artifacts, 'dispatch-ceo.txt')}"`,
'5. Call the Agent tool once with these inputs: prompt = the exact, complete output of step 4 (every line, unchanged), ' +
'description "CEO review", subagent_type "general-purpose", and run_in_background false. Include run_in_background even ' +
'if the tool schema lacks it. If the agent runs in the background, end your turn and wait for its completion notification.',
'6. Read the file whose path step 3 printed, completely (no offset or limit).',
'if the tool schema lacks it. If its result says the agent is running in the background, end your turn at once with no ' +
'further tool call; the completion notification starts your next turn.',
'6. Only after the CEO review agent has finished (in the turn its completion notification starts, when it ran in the ' +
'background), Read the file whose path step 3 printed, completely (no offset or limit).',
'7. In one message, write exactly this line as your own text: **Phase 1 complete.** and make exactly one tool call in that ' +
'same message: the Bash command `true autoplan-published ceo`.',
`8. In your next assistant message, without waiting for any user input, Read the file ${path.join(sections, 'design-phase.md')} with offset 1 and limit 20.`,
+2 -2
View File
@@ -20,7 +20,7 @@
import { expect } from 'bun:test';
import { describeIfSelected, testIfSelected } from './helpers/e2e-helpers';
import { launchClaudePty, selectPtyNumberedOption } from './helpers/claude-pty-runner';
import { prepareGuardSession } from './helpers/autoplan-guard-pty';
import { closePacketReadAfterNotice, prepareGuardSession } from './helpers/autoplan-guard-pty';
const MODEL = 'claude-haiku-4-5-20251001';
@@ -63,7 +63,7 @@ describeIfSelected('autoplan publication guard live PTY', ['autoplan-guard-pty']
// the Phase 2 transition and the Phase 2 reviewer dispatch: four guarded allows, each verified.
expect(log.map(e => [e.decision, e.disposition])).toEqual(Array(4).fill(['allow', 'allow']));
// The CEO reviewer's notice started a new turn before the close packet, the Phase 2 entry and the Phase 2 dispatch.
expect(evidence.afterNotice.filter(c => /^Read .*close-packet\.md$/.test(c))).not.toEqual([]);
expect(closePacketReadAfterNotice(evidence.afterNotice), evidence.afterNotice.join('\n')).toBe(true);
expect(evidence.afterNotice.filter(c => /^Read .*autoplan\/sections\/design-phase\.md$/.test(c))).not.toEqual([]);
expect(evidence.afterNotice).toContain('Agent Design review');
expect(evidence.reviewerReads.filter(f => /\.gstack\/tmp\/autoplan\/autoplan-design-[^/]+\/native-prompt\.md$/.test(f))).not.toEqual([]);