fix(evals): autoplan-guard-pty actor continues through the phase boundary; name PHASE_BOUNDARY_STOP

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

CHANGELOG.md

@@ -13,6 +13,7 @@
 
 ### Fixed
 - **The tree receipt no longer drops diff lines that start with `--` or `++`.** Its unified-diff parser treated `---`/`+++` inside a hunk as file headers, so a release-output line beginning `++` or `--` could pass as a stamp. It now reads headers only before a file's first hunk, and a file mode change is never a stamp.
+- **`autoplan-guard-pty` no longer stalls at the phase boundary.** Its scripted actor (haiku) published Phase 1, printed "Phase 1 complete." and ended its turn, so the case idled to its 480 s deadline with 2 of 4 guarded allows. This is the 489 s red seen on #3190, #3258, #3260 and this PR's stamped gate (run 39099195754). The script now runs steps 1 to 11 in one continuous run, and says the Phase 1 line is a boundary, not the end. The publication and the Phase 2 entry stay in separate messages. The case also stops waiting as soon as the actor ends its turn there, and names the cause `PHASE_BOUNDARY_STOP`.
 
 ### If you…
 - **…want a restamp to reuse its gate:** put the CHANGELOG entry in the gated push (under `## [Unreleased]` or the tentative version), so the stamp only re-heads it.

test/autoplan-guard-pty-fixture.test.ts

@@ -1,7 +1,7 @@
 import { expect, test } from 'bun:test';
 import * as fs from 'node:fs';
 import * as path from 'node:path';
-import { closePacketReadAfterNotice, prepareGuardSession } from './helpers/autoplan-guard-pty';
+import { closePacketReadAfterNotice, prepareGuardSession, stoppedAtPhaseBoundary } from './helpers/autoplan-guard-pty';
 
 const base = '/home/runner/.cache/gstack-paid-shard-x/tmp/ap-guard-pty-y';
 
@@ -33,3 +33,34 @@ test('the scripted actor ends its turn after a background CEO dispatch and reads
     expect(skill.indexOf('\n6. ')).toBeGreaterThan(skill.indexOf('\n5. '));
   } finally { s.cleanup(); }
 });
+
+// PR gate 39099195754 (ce1da4e, fail): the actor published Phase 1, printed "Phase 1 complete." and ended its
+// turn, so the run idled to its 480 s deadline with 2 of 4 guarded allows (the same 489 s shape as 38976253774).
+const STOP_SCREEN = fs.readFileSync(path.join(import.meta.dir, 'fixtures', 'autoplan-guard-pty-phase-boundary-stop-screen.txt'), 'utf8');
+
+test('the captured phase-boundary stop is named; a run still working or already in Phase 2 is not', () => {
+  expect(STOP_SCREEN).toContain('Phase 1 complete.');
+  expect(stoppedAtPhaseBoundary(STOP_SCREEN)).toBe(true);
+  const working = STOP_SCREEN.slice(0, STOP_SCREEN.indexOf('✻ Brewed'));
+  expect(stoppedAtPhaseBoundary(working)).toBe(false);
+  const continued = working + '●Step8:ReadingtheDesignphasefile\n\nRead 1 file\n\n●Step9:ReadingtheDesigndispatchfile\n\n✻ Brewed for 80s · done 5:17 AM\n❯\n';
+  expect(stoppedAtPhaseBoundary(continued)).toBe(false);
+  expect(stoppedAtPhaseBoundary('✻ Brewed for 10s · done 5:00 AM\n❯\n')).toBe(false);
+});
+
+test('the scripted actor keeps going through the phase boundary: nothing between step 7 and step 11 lets it end its turn there', () => {
+  const s = prepareGuardSession(undefined);
+  try {
+    const skill = fs.readFileSync(path.join(s.project, '.claude', 'skills', fs.readdirSync(path.join(s.project, '.claude', 'skills'))[0]!, 'SKILL.md'), 'utf8');
+    const step = (n: number) => skill.slice(skill.indexOf(`\n${n}. `) + 1, skill.indexOf(`\n${n + 1}. `));
+    const intro = skill.slice(skill.indexOf('\n---\n', 4) + 5, skill.indexOf('\n1. '));
+    expect(intro).toContain('from step 1 through step 11 in one continuous run');
+    expect(intro).toContain('End your turn before step 11 only where step 5 or step 10 says to wait for a background agent.');
+    expect(intro).toContain('**Phase 1 complete.** is a phase boundary, not the end of the task.');
+    expect(step(7)).toContain('Do not end your turn after it: when its result arrives, go straight on to step 8.');
+    expect(step(8).startsWith("8. As your next assistant message, in the same turn as step 7's result (no user input is coming), Read the file")).toBe(true);
+    // The publication and the Phase 2 entry stay in separate messages: the guard's boundary is unchanged.
+    expect(step(7)).toContain('make exactly one tool call in that same message: the Bash command `true autoplan-published ceo`');
+    for (const n of [6, 7, 8, 9]) expect(step(n)).not.toMatch(/end your turn(?! after it)|wait for (its|a) completion/i);
+  } finally { s.cleanup(); }
+});

test/fixtures/autoplan-guard-pty-phase-boundary-stop-screen.txt

@@ -0,0 +1,122 @@
+pletion
+ Running1shellcommand…
+·400
+25
+50
+●
+✢75
+●Step2:ReadingtheCEOphasefile
+
+Read 1 file
+
+●Step3:Runningthereviewgenerationcommand
+
+Ran 1 shell command
+
+●Step4:ReadingtheCEOdispatchfile
+
+Read 1 file
+
+●Step5:CallingtheCEOreviewagentwiththedispatchinstructions
+
+●Agent(CEOreview)
+  ⎿  Backgrounded agent (↓ to manage · ctrl+o to expand)
+
+●TheCEOreviewagentisrunninginthebackground.Waitingforitscompletionnotification.
+
+✻Waiting for 1 background agent to finish
+
+●Agent"CEOreview"finished · 27s
+
+●Step6:Readingtheclosepacketfilecompletely
+
+Read 1 file
+
+●Step7:Publishingphase1completion
+
+●Running1shellcommand…
+  ⎿  $ true autoplan-published ceo
+Wandering…
+Wandering…512
+*Wandering…837
+✶62
+✻87
+
+Wandering…625
+Wandering…
+thinking)
+✽43thinking
+5thinking
+60thinking
+✻1thinking
+●
+4
+9
+
+Read 1 file
+
+●Step3:Runningthereviewgenerationcommand
+
+Ran 1 shell command
+
+●Step4:ReadingtheCEOdispatchfile
+
+Read 1 file
+
+●Step5:CallingtheCEOreviewagentwiththedispatchinstructions
+
+●Agent(CEOreview)
+  ⎿  Backgrounded agent (↓ to manage · ctrl+o to expand)
+
+●TheCEOreviewagentisrunninginthebackground.Waitingforitscompletionnotification.
+
+✻Waiting for 1 background agent to finish
+
+●Agent"CEOreview"finished · 27s
+
+●Step6:Readingtheclosepacketfilecompletely
+
+Read 1 file
+
+●Step7:Publishingphase1completion
+
+Ran 1 shell command
+
+● Phase 1 complete.
+●Step2:ReadingtheCEOphasefile
+
+Read 1 file
+
+●Step3:Runningthereviewgenerationcommand
+
+Ran 1 shell command
+
+●Step4:ReadingtheCEOdispatchfile
+
+Read 1 file
+
+●Step5:CallingtheCEOreviewagentwiththedispatchinstructions
+
+● Agent(CEO review)
+  ⎿  Backgroundedagent (↓ to manage · ctrl+o to expand)
+
+●TheCEOreviewagentisrunninginthebackground.Waitingforitscompletionnotification.
+
+✻Waiting for 1 background agent to finish
+
+●Agent"CEOreview"finished · 27s
+
+●Step6:Readingtheclosepacketfilecompletely
+
+Read 1 file
+
+●Step7:Publishingphase1completion
+
+Ran 1 shell command
+
+●Phase 1 complete.
+
+✻ Brewed for 50s · done 5:16 AM
+❯
+← for agents
+? for shrtcut · ← foragents

test/helpers/autoplan-guard-pty.ts

@@ -37,6 +37,20 @@ export interface GuardSession {
   cleanup(): void;
 }
 
+/**
+ * The actor ended its turn at the Phase 1 publication instead of going on to
+ * Phase 2: the screen after the last "Phase 1 complete." line shows a finished
+ * turn and no Phase 2 step (PR gate 39099195754, the 489 s "hang" first seen
+ * in 38976253774). Names the failure; it never passes a run.
+ */
+export function stoppedAtPhaseBoundary(screen: string): boolean {
+  const marks = [...screen.matchAll(/Phase\s*1\s*complete\./g)];
+  if (!marks.length) return false;
+  const last = marks[marks.length - 1]!;
+  const after = screen.slice(last.index! + last[0].length);
+  return /✻[^\n]*\bdone\b/.test(after) && !/Step\s*8|design-phase|Design\s*review/i.test(after);
+}
+
 /** The close packet Read (step 6) came after the CEO reviewer's completion notice, so it ran in the notice's new turn. */
 export const closePacketReadAfterNotice = (afterNotice: string[]): boolean => afterNotice.some(c => /^Read .*close-packet\.md$/.test(c));
 
@@ -74,7 +88,9 @@ console.log(close);
   const tool = path.join(ROOT, 'bin', 'gstack-autoplan-snapshot.ts');
   const sections = path.join(config, 'skills', 'gstack', 'autoplan', 'sections');
   const body = [
-    'Run this scripted phase boundary exactly. Each numbered step is its own assistant message; do nothing else.',
+    'Run this scripted phase boundary exactly, from step 1 through step 11 in one continuous run. Each numbered step is its own ' +
+      'assistant message; do nothing else. End your turn before step 11 only where step 5 or step 10 says to wait for a ' +
+      'background agent. **Phase 1 complete.** is a phase boundary, not the end of the task.',
     `1. Run this Bash command alone: bun "${tool}" init "${source}" "${active}" "${restore}"`,
     `2. Read the file ${path.join(sections, 'ceo-phase.md')} with offset 1 and limit 20.`,
     `3. Run this Bash command alone: bun "${review}"`,
@@ -86,8 +102,9 @@ console.log(close);
     '6. Only after the CEO review agent has finished (in the turn its completion notification starts, when it ran in the ' +
       'background), Read the file whose path step 3 printed, completely (no offset or limit).',
     '7. In one message, write exactly this line as your own text: **Phase 1 complete.** and make exactly one tool call in that ' +
-      'same message: the Bash command `true autoplan-published ceo`.',
-    `8. In your next assistant message, without waiting for any user input, Read the file ${path.join(sections, 'design-phase.md')} with offset 1 and limit 20.`,
+      'same message: the Bash command `true autoplan-published ceo`. Do not end your turn after it: when its result arrives, go ' +
+      'straight on to step 8.',
+    `8. As your next assistant message, in the same turn as step 7's result (no user input is coming), Read the file ${path.join(sections, 'design-phase.md')} with offset 1 and limit 20.`,
     `9. Run this Bash command alone: cat "${path.join(artifacts, 'dispatch-design.txt')}"`,
     '10. Call the Agent tool once with these inputs: prompt = the exact, complete output of step 9 (every line, unchanged), ' +
       'description "Design review", subagent_type "general-purpose". If the agent runs in the background, end your turn and wait ' +

test/helpers/free-fixtures-data.ts

@@ -16,6 +16,7 @@
 export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
   'test/fixtures/auq-matrix/**': ['test/auq-native-capture.test.ts'],
   'test/fixtures/autoplan-edit-digests-al.json': ['test/autoplan-edit-digests-al.test.ts'],
+  'test/fixtures/autoplan-guard-pty-phase-boundary-stop-screen.txt': ['test/autoplan-guard-pty-fixture.test.ts'],
   'test/fixtures/autoplan-home-phase-entry-fb10.json': ['test/autoplan-method-read-audit.test.ts'],
   'test/fixtures/autoplan-owned-state-edit.json': ['test/autoplan-owned-state.test.ts'],
   'test/fixtures/autoplan-pairing/run-20261014-pairs.json': ['test/autoplan-pairing.test.ts'],

test/skill-e2e-autoplan-guard-pty.test.ts

@@ -20,7 +20,7 @@
 import { expect } from 'bun:test';
 import { describeIfSelected, testIfSelected } from './helpers/e2e-helpers';
 import { launchClaudePty, selectPtyNumberedOption } from './helpers/claude-pty-runner';
-import { closePacketReadAfterNotice, prepareGuardSession } from './helpers/autoplan-guard-pty';
+import { closePacketReadAfterNotice, prepareGuardSession, stoppedAtPhaseBoundary } from './helpers/autoplan-guard-pty';
 
 const MODEL = 'claude-haiku-4-5-20251001';
 
@@ -48,6 +48,8 @@ describeIfSelected('autoplan publication guard live PTY', ['autoplan-guard-pty']
           await Bun.sleep(1_500);
         }
         if (log.some(e => e.decision === 'deny') || (log.length >= 4 && s.noticeEvidence().notices >= 2)) break;
+        // The actor ended its turn at the Phase 1 publication; no input is coming, so it cannot reach Phase 2.
+        if (log.length < 4 && stoppedAtPhaseBoundary(screen)) break;
         if (session.exited()) break;
       }
       await Bun.sleep(3_000);
@@ -55,6 +57,7 @@ describeIfSelected('autoplan publication guard live PTY', ['autoplan-guard-pty']
       const summary = { prompts: prompts.length, phase1Ms, totalMs: Math.round(performance.now() - started), evidence,
         decisions: log.map(e => ({ decision: e.decision, code: e.code, path: e.path, version: e.claude_code_version })) };
       console.log(`[autoplan-guard-pty] ${JSON.stringify(summary)}`);
+      if (log.length < 4 && stoppedAtPhaseBoundary(session.visibleText())) console.log('[autoplan-guard-pty] PHASE_BOUNDARY_STOP: the actor ended its turn after "Phase 1 complete." instead of going on to step 8');
       if (log.length !== 4 || log.some(e => e.decision === 'deny') || prompts.length)
         console.log(`[autoplan-guard-pty] journal:\n${s.journalSummary().join('\n')}\n[autoplan-guard-pty] screen:\n${session.visibleText().slice(-2_000)}\n[autoplan-guard-pty] last non-blank screen:\n${session.lastScreen?.() ?? ''}`);
       for (const prompt of prompts) console.log(`[autoplan-guard-pty] permission card approved:\n${prompt}`);