mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 14:06:50 +01:00
test(autoplan-guard-pty): the scripted actor ends its turn after a background CEO dispatch before reading the close packet
PR gate 39070276914 (dc739be) failed at the close-packet check while gate 39069171159 (b789898) passed with byte-identical autoplan inputs. The evidence differs only in order: the failing actor read the close packet in the dispatch turn, so the first tool call after the CEO notice was the publication. The product guard allowed all four guarded calls in both runs. The fixture step 5 told the actor to end its turn only conditionally; step 5 now ends the turn as soon as the result says the agent runs in the background, and step 6 reads the close packet only after the reviewer finished. The close-packet check moves to closePacketReadAfterNotice, replayed in a free test against both captured orders; a second free test pins the actor's steps 5 and 6. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
dc739be05a
commit
6c5ed93345
3 files changed
+44
-4
No files matched your search
@@ -0,0 +1,35 @@
|
||||
import { expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { closePacketReadAfterNotice, prepareGuardSession } from './helpers/autoplan-guard-pty';
|
||||
|
||||
const base = '/home/runner/.cache/gstack-paid-shard-x/tmp/ap-guard-pty-y';
|
||||
|
||||
test('the close-packet check accepts a Read in the notice turn and rejects one made before the notice', () => {
|
||||
// PR gate 39069171159 (b789898, pass): the CEO reviewer's notice started the turn that read the close packet.
|
||||
expect(closePacketReadAfterNotice([
|
||||
`Read ${base}/project/.gstack/tmp/autoplan/autoplan-ceo-oMVeYz/close-packet.md`,
|
||||
'Bash true autoplan-published ceo',
|
||||
`Read ${base}/home/.claude/skills/gstack/autoplan/sections/design-phase.md`,
|
||||
'Agent Design review',
|
||||
])).toBe(true);
|
||||
// PR gate 39070276914 (dc739be, fail): the actor read the close packet in the dispatch turn, before the notice.
|
||||
expect(closePacketReadAfterNotice([
|
||||
'Bash true autoplan-published ceo',
|
||||
`Read ${base}/home/.claude/skills/gstack/autoplan/sections/design-phase.md`,
|
||||
`Bash cat "${base}/project/.gstack/tmp/autoplan/dispatch-design.txt"`,
|
||||
'Agent Design review',
|
||||
])).toBe(false);
|
||||
});
|
||||
|
||||
test('the scripted actor ends its turn after a background CEO dispatch and reads the close packet only after the reviewer finishes', () => {
|
||||
const s = prepareGuardSession(undefined);
|
||||
try {
|
||||
const skill = fs.readFileSync(path.join(s.project, '.claude', 'skills', fs.readdirSync(path.join(s.project, '.claude', 'skills'))[0]!, 'SKILL.md'), 'utf8');
|
||||
const step = (n: number) => skill.slice(skill.indexOf(`\n${n}. `) + 1, skill.indexOf(`\n${n + 1}. `));
|
||||
expect(step(5)).toContain('If its result says the agent is running in the background, end your turn at once with no further tool call; the completion notification starts your next turn.');
|
||||
expect(step(5)).not.toContain('If the agent runs in the background, end your turn and wait');
|
||||
expect(step(6).startsWith('6. Only after the CEO review agent has finished (in the turn its completion notification starts, when it ran in the background), Read the file whose path step 3 printed')).toBe(true);
|
||||
expect(skill.indexOf('\n6. ')).toBeGreaterThan(skill.indexOf('\n5. '));
|
||||
} finally { s.cleanup(); }
|
||||
});
|
||||
@@ -37,6 +37,9 @@ export interface GuardSession {
|
||||
cleanup(): void;
|
||||
}
|
||||
|
||||
/** The close packet Read (step 6) came after the CEO reviewer's completion notice, so it ran in the notice's new turn. */
|
||||
export const closePacketReadAfterNotice = (afterNotice: string[]): boolean => afterNotice.some(c => /^Read .*close-packet\.md$/.test(c));
|
||||
|
||||
export function prepareGuardSession(apiKey: string | undefined): GuardSession {
|
||||
const base = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'ap-guard-pty-')));
|
||||
const home = path.join(base, 'home'), config = path.join(home, '.claude'), project = path.join(base, 'project');
|
||||
@@ -78,8 +81,10 @@ console.log(close);
|
||||
`4. Run this Bash command alone: cat "${path.join(artifacts, 'dispatch-ceo.txt')}"`,
|
||||
'5. Call the Agent tool once with these inputs: prompt = the exact, complete output of step 4 (every line, unchanged), ' +
|
||||
'description "CEO review", subagent_type "general-purpose", and run_in_background false. Include run_in_background even ' +
|
||||
'if the tool schema lacks it. If the agent runs in the background, end your turn and wait for its completion notification.',
|
||||
'6. Read the file whose path step 3 printed, completely (no offset or limit).',
|
||||
'if the tool schema lacks it. If its result says the agent is running in the background, end your turn at once with no ' +
|
||||
'further tool call; the completion notification starts your next turn.',
|
||||
'6. Only after the CEO review agent has finished (in the turn its completion notification starts, when it ran in the ' +
|
||||
'background), Read the file whose path step 3 printed, completely (no offset or limit).',
|
||||
'7. In one message, write exactly this line as your own text: **Phase 1 complete.** and make exactly one tool call in that ' +
|
||||
'same message: the Bash command `true autoplan-published ceo`.',
|
||||
`8. In your next assistant message, without waiting for any user input, Read the file ${path.join(sections, 'design-phase.md')} with offset 1 and limit 20.`,
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
import { expect } from 'bun:test';
|
||||
import { describeIfSelected, testIfSelected } from './helpers/e2e-helpers';
|
||||
import { launchClaudePty, selectPtyNumberedOption } from './helpers/claude-pty-runner';
|
||||
import { prepareGuardSession } from './helpers/autoplan-guard-pty';
|
||||
import { closePacketReadAfterNotice, prepareGuardSession } from './helpers/autoplan-guard-pty';
|
||||
|
||||
const MODEL = 'claude-haiku-4-5-20251001';
|
||||
|
||||
@@ -63,7 +63,7 @@ describeIfSelected('autoplan publication guard live PTY', ['autoplan-guard-pty']
|
||||
// the Phase 2 transition and the Phase 2 reviewer dispatch: four guarded allows, each verified.
|
||||
expect(log.map(e => [e.decision, e.disposition])).toEqual(Array(4).fill(['allow', 'allow']));
|
||||
// The CEO reviewer's notice started a new turn before the close packet, the Phase 2 entry and the Phase 2 dispatch.
|
||||
expect(evidence.afterNotice.filter(c => /^Read .*close-packet\.md$/.test(c))).not.toEqual([]);
|
||||
expect(closePacketReadAfterNotice(evidence.afterNotice), evidence.afterNotice.join('\n')).toBe(true);
|
||||
expect(evidence.afterNotice.filter(c => /^Read .*autoplan\/sections\/design-phase\.md$/.test(c))).not.toEqual([]);
|
||||
expect(evidence.afterNotice).toContain('Agent Design review');
|
||||
expect(evidence.reviewerReads.filter(f => /\.gstack\/tmp\/autoplan\/autoplan-design-[^/]+\/native-prompt\.md$/.test(f))).not.toEqual([]);
|
||||
|
||||
Reference in new issue
Block a user