CalvinBackup/gstack
1
0 Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-11-04 15:57:33 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor(pty): one runPtySession loop drives observation, counting and floor

Browse Source 
test/helpers/pty/session.ts owns launch -> start -> (poll -> tick)* ->
timeout and the failure contract the three runners each hand-rolled: the
run's own error wins over capture and close errors, close always runs, owned
fixture cleanup runs last (also when launch fails). Each runner now supplies a
PtySessionPlan: its boot/command step, poll cadence (2s observation/floor
sleep; counting's output wake + 250ms coalesce), tick policy (permission
handling, native identity, terminal rules stay per runner because they differ)
and capture hooks. The runner bodies are decomposed into top-level steps so no
function exceeds 150 lines; behavior is unchanged and the fake-driver cases
from the first W4 commit pass unmodified.

The counting capture step and the native completion-summary predicate are now
named functions (countingCapture, isNativeCompletionSummary), so
plan-create-prepublication and plan-count-completion call them directly
instead of executing sliced source. The two harnesses that still execute a
sliced runner body with injected boundaries (eng-seeded-completion-ai,
plan-floor-permission) pass the PtyDriver seam instead of overriding
Date/Bun.sleep.
This commit is contained in:
garrytan committed 2026-09-30 06:15:24 +00:00
1 parent 5df542dc3f
commit a1b21407eb
9 files changed
+1274 -1055
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

test/eng-seeded-completion-ai.test.ts
+5 -1
View File
@@ -8,6 +8,7 @@ import { fakePlanSeedPrelude } from './helpers/fake-plan-seed';
import fixture from './fixtures/eng-seeded-completion-ai.json';
import { classifyVisible, extractPlanFilePath } from './helpers/claude-pty-runner';
import * as predicates from './helpers/claude-pty-runner';
import type { PtyDriver } from './helpers/claude-pty-runner';
const gate = '─────\nClaude has written up a plan and is ready to execute. Would you like to proceed?\n❯ 1. Yes, and use auto mode\n2. Yes, manually approve edits\n3. Tell Claude what to change';
const compactGate = 'Exit plan mode?\nClaude wants to exit plan mode\n❯ 1. Yes, and switch to default (ask each time) for this session\n2. No';
const question = 'Which runner should the plan use?\nA) Use the built-in runner\nB) Build a custom runner\nRecommendation: A because it avoids duplicate scheduling logic.\nReply with A or B.';
@@ -141,9 +142,12 @@ async function mockedObservation(frames: string[], verdict: 'waiting' | 'working
isScopeGateAutoSelectVisible: predicates.isScopeGateAutoSelectVisible,
classifyVisible, extractPlanFilePath, findNativeAutoDecision: () => null,
judgePtyState: () => { judged++; return { state: verdict, reasoning: 'synthetic current-frame verdict' }; },
runPtySession: predicates.runPtySession,
};
const run = new Function(...Object.keys(args), js)(...Object.values(args));
const obs = await run({ skillName: 'plan-eng-review', timeoutMs: 70000,
const clockArgs = args as { Date: { now(): number }; Bun: { sleep(ms: number): Promise<void> }; launchClaudePty: PtyDriver['launch'] };
const driver: PtyDriver = { launch: clockArgs.launchClaudePty, now: clockArgs.Date.now, monotonic: clockArgs.Date.now, sleep: clockArgs.Bun.sleep };
const obs = await run({ skillName: 'plan-eng-review', timeoutMs: 70000, driver,
...(seeded ? { initialPlanContent: '# Plan: Required draft' } : {}) });
expect(closed).toBe(1);
return { obs, judged, seedSubmittedAt };
test/helpers/claude-pty-runner.ts
+5 -4
View File
@@ -11,7 +11,8 @@ export { resolveClaudeBinary } from './pty/binary';
export { stripAnsi, isRejectedSlashCommand, isPlanReadyVisible, isAutoDecidedVisible, extractPlanFilePath, planFileHasDecisionsSection, TAIL_SCAN_BYTES, isPermissionDialogVisible, stripPtyResidue, isNumberedOptionListVisible } from './pty/screen';
export { selectPtyNumberedOption, launchClaudePty } from './pty/launch';
export type { ClaudePtyOptions, ClaudePtySession } from './pty/launch';
export type { PtyDriver } from './pty/session';
export { runPtySession, realPtyDriver } from './pty/session';
export type { PtyDriver, PtyStep, PtySessionPlan } from './pty/session';
export { logPtySnapshot, judgePtyState } from './pty/judge';
export type { PtyStateVerdict } from './pty/judge';
export { isProseAUQVisible, isScopeGateQuestionVisible, isScopeGateAutoSelectVisible, parseNumberedOptions, MODE_RE, optionsSignature, classifyVisible, COMPLETION_SUMMARY_RE, classifyPlanCountFrame, planCountSubmissionInput } from './pty/classify';
@@ -22,8 +23,8 @@ export { assertReviewReportAtBottom, hasNativePlanCompletion, isQuestionlessNati
export type { ReviewReportAtBottomResult, NativePlanTerminalReview, NativePlanTerminalAssessment, NativePlanTerminalEvaluator } from './pty/plan-native';
export { ceoStep0Boundary, engSetupAUQ, engFirstReviewAUQ, engStep0Boundary, designReviewSetupAUQ } from './pty/boundaries';
export { runPlanSkillObservation } from './pty/runners/observation';
export type { PlanSkillObservation } from './pty/runners/observation';
export { runPlanSkillCounting } from './pty/runners/counting';
export type { PlanSkillCountObservation } from './pty/runners/counting';
export type { PlanSkillObservation, PlanSkillObservationOptions } from './pty/runners/observation';
export { runPlanSkillCounting, countingCapture, isNativeCompletionSummary } from './pty/runners/counting';
export type { PlanSkillCountObservation, PlanSkillCountingOptions, CountingRun } from './pty/runners/counting';
export { planFloorDXPane, planFloorDXReplyInput, runPlanSkillFloorCheck } from './pty/runners/floor';
export type { PlanSkillFloorObservation, PlanFloorDXReply } from './pty/runners/floor';
test/helpers/pty/runners/counting.ts
+524 -458
View File
File diff suppressed because it is too large. Load diff
test/helpers/pty/runners/floor.ts
+335 -289
View File
@@ -18,10 +18,9 @@ import { capturePlanCountQuestion, createPlanCountPermissionGuard, matchesNative
import { resolveClaudeBinary } from '../binary';
import { designReviewSetupAUQ } from '../boundaries';
import { SANCTIONED_WRITE_SUBSTRINGS, isProseAUQVisible, planCountSubmissionInput } from '../classify';
import { launchClaudePty } from '../launch';
import type { ClaudePtySession } from '../launch';
import { isNumberedOptionListVisible, isPermissionDialogVisible, isPlanReadyVisible, isRejectedSlashCommand, stripPtyResidue } from '../screen';
import type { PtyDriver } from '../session';
import { realPtyDriver, runPtySession, type PtyDriver, type PtyStep } from '../session';
// ────────────────────────────────────────────────────────────────────────────
// runPlanSkillFloorCheck — stop at the first substantive seeded question.
@@ -130,7 +129,8 @@ export function planFloorDXReplyInput(visible: string, call: NativePlanQuestionC
: { input: '\r', stage: 'done' };
}
export async function runPlanSkillFloorCheck(opts: {
/** Options for runPlanSkillFloorCheck. */
export interface PlanSkillFloorOptions {
/** Skill name, e.g. 'plan-eng-review'. Used for diagnostic strings only. */
skillName: string;
/** Slash command; the owned PLAN.md target is supplied in the same submission. */
@@ -153,9 +153,10 @@ export async function runPlanSkillFloorCheck(opts: {
model?: string;
/** Launch seam and clock; tests pass the fake driver. Default: real launcher and clocks. */
driver?: PtyDriver;
}): Promise<PlanSkillFloorObservation> {
const driver = opts.driver ?? { launch: launchClaudePty, now: () => Date.now(),
monotonic: () => performance.now(), sleep: (ms: number) => Bun.sleep(ms) };
}
export async function runPlanSkillFloorCheck(opts: PlanSkillFloorOptions): Promise<PlanSkillFloorObservation> {
const driver = opts.driver ?? realPtyDriver;
const startedAt = driver.now();
const timeoutMs = opts.timeoutMs ?? 600_000;
const dxContext = opts.devexSetupContext;
@@ -177,9 +178,18 @@ export async function runPlanSkillFloorCheck(opts: {
const fixture = createPlanCountFixture(request, { requestedPlanPath: opts.requestedPlanPath,
nativeReviewOnly: true, preconfiguredReviewActor: true });
const sessionId = randomUUID();
let session: ClaudePtySession;
try {
session = await driver.launch({
const run = { opts, driver, startedAt, timeoutMs, dxContext, fixture, sessionId,
ownedFilePermissions: [], planningDirectory: undefined, transcript: { status: 'missing', calls: [], assistantMessages: [] },
viewport: '', publicTools: [], pendingQuestion: undefined, floorReview: undefined, floorAssessment: undefined,
setupChoices: new Map(), submittedSetup: new Set(), assessed: new Map(), dxReplies: new Map(),
since: 0, commandStartedAt: 0, targetDelivery: undefined, saveSnapshot: undefined, nativeCandidates: [],
validatedPendingQuestion: undefined, sampledAt: undefined, lastCheckpointAt: 0, lastCheckpointState: '',
artifactError: undefined, finished: false, started: false, start: 0, deadlineAt: 0, screenDeadlineAt: 0,
} as unknown as FloorRun;
return runPtySession<PlanSkillFloorObservation>({
driver,
cleanup: () => fixture.cleanup(),
launch: {
permissionMode: 'plan',
cwd: fixture.cwd,
timeoutMs: timeoutMs + 60_000,
@@ -191,291 +201,327 @@ export async function runPlanSkillFloorCheck(opts: {
...(dxContext ? { rows: 80 } : {}),
observeFilePermissions: fixture.workingPlanPath ? [fixture.workingPlanPath] : undefined,
extraArgs: ['--session-id', sessionId],
});
} catch (error) {
fixture.cleanup();
throw error;
}
const ownedFilePermissions = (session.pendingFilePermissionFiles ?? []).map(binding =>
({ ...binding, guard: createPlanCountPermissionGuard() }));
const planningDirectory = session.hermeticConfigDir ? path.join(session.hermeticConfigDir, 'plans') : undefined;
let transcript: PlanCountTranscript = { status: 'missing', calls: [], assistantMessages: [] };
let viewport = '';
let publicTools: NativePublicToolEvent[] = [];
let pendingQuestion: NativePlanQuestionCall | undefined;
let floorReview: PlanFloorReview | undefined;
let floorAssessment: PlanFloorAssessment | undefined;
const setupChoices = new Map<string, Set<number>>();
const submittedSetup = new Set<string>();
const assessed = new Map<string, PlanFloorAssessment>();
const dxReplies = new Map<string, PlanFloorDXReply>();
let captureBeforeClose: ((error?: unknown) => void) | undefined;
let floorFailed = false;
let floorError: unknown;
try {
await driver.sleep(8000); // boot grace + auto-trust handler window
const since = session.mark();
const commandStartedAt = driver.now();
session.send(`${opts.slashCommand} PLAN.md\r`);
const deliveryOptions = { seed: fixture.seed, sessionId,
slashCommand: opts.slashCommand, startedAt: commandStartedAt };
let targetDelivery = readPlanFloorTarget(session.hermeticConfigDir, fixture.cwd,
{ ...deliveryOptions, now: driver.now() });
const saveSnapshot = createPlanCountSnapshotWriter();
let nativeCandidates: NativePlanQuestionCall[] = [];
let validatedPendingQuestion: ReturnType<typeof readPendingQuestion>;
let sampledAt: number | undefined;
let lastCheckpointAt = 0, lastCheckpointState = '';
let artifactError: string | undefined;
let finished = false;
const capture = (observation: object) => {
const recorderStatus = pendingQuestionRecorderStatus(session.pendingQuestionFile, fixture.cwd, session.hermeticConfigDir);
const artifacts = saveSnapshot({ skillName: opts.skillName, cwd: fixture.cwd,
claudeConfigDir: session.hermeticConfigDir, raw: session.rawOutput(),
visible: session.visibleSince(since), viewport,
observation: { ...observation, transcript, publicTools, pendingQuestion, floorReview, floorAssessment, targetDelivery, commandStartedAt,
pendingWriteInputs: ownedFilePermissions.flatMap(binding => {
const input = readPendingWriteInput(binding.file, binding.expected, fixture.cwd, session.hermeticConfigDir, startedAt);
return input ? [input] : [];
}),
questionDiagnostics: { sampledAt, parentSessionId: sessionId, recorderStatus, recorderStatusAt: driver.now(), nativeCandidates, validatedPendingQuestion },
...(dxContext ? { setupContextReplies: [...dxReplies.values()] } : {}), artifactError } });
if (artifacts.artifactError) {
artifactError ??= artifacts.artifactError;
console.error(`PTY artifact write failed: ${artifacts.artifactError}`);
}
return { ...artifacts, ...(artifactError ? { artifactError } : {}) };
};
const checkpoint = () => {
const state = JSON.stringify([pendingQuestionRecorderStatus(session.pendingQuestionFile, fixture.cwd, session.hermeticConfigDir),
targetDelivery.status, nativeCandidates, validatedPendingQuestion, pendingQuestion, [...dxReplies.values()]]);
if (state === lastCheckpointState && driver.now() - lastCheckpointAt < 15_000) return;
lastCheckpointState = state; lastCheckpointAt = driver.now();
capture({ state: 'in_progress', elapsedMs: driver.now() - startedAt });
};
captureBeforeClose = (error) => {
if (floorFailed && session.hermeticConfigDir) {
publicTools = [];
transcript = readPlanCountTranscript(session.hermeticConfigDir, fixture.cwd, event => publicTools.push(event));
}
if (!finished) capture({ state: floorFailed ? 'threw' : 'in_progress', error: floorFailed ? String(error) : undefined,
captureReason: 'before_cleanup', elapsedMs: driver.now() - startedAt });
};
const finish = (observation: PlanSkillFloorObservation): PlanSkillFloorObservation => {
const artifacts = capture(observation);
finished = true;
return { ...observation, targetDelivery, ...artifacts };
};
const start = driver.now();
const deadlineAt = start + timeoutMs;
const screenDeadlineAt = driver.monotonic() + timeoutMs;
while (driver.now() - start < timeoutMs) {
},
start: session => floorStart(run, session),
poll: async () => {
if (!(driver.now() - run.start < timeoutMs)) return false;
await driver.sleep(2000);
const visible = session.visibleSince(since);
if (session.exited()) {
return finish({
auqObserved: false,
outcome: 'exited',
summary: `claude exited (code=${session.exitCode()}) before a qualifying finding`,
evidence: visible.slice(-3000),
elapsedMs: driver.now() - startedAt,
});
}
if (isRejectedSlashCommand(visible, opts.slashCommand)) {
return finish({
auqObserved: false,
outcome: 'exited',
summary: `claude rejected ${opts.slashCommand} as unknown command`,
evidence: visible.slice(-3000),
elapsedMs: driver.now() - startedAt,
});
}
if (targetDelivery.status !== 'ready') {
targetDelivery = readPlanFloorTarget(session.hermeticConfigDir, fixture.cwd,
{ ...deliveryOptions, now: driver.now() });
if (targetDelivery.status !== 'ready') {
viewport = await session.currentScreen(screenDeadlineAt);
checkpoint();
continue;
}
}
// Current native identity precedes permission handling and finding assessment.
floorReview = undefined; floorAssessment = undefined;
viewport = await session.currentScreen(screenDeadlineAt);
publicTools = [];
transcript = session.hermeticConfigDir
? readPlanCountTranscript(session.hermeticConfigDir, fixture.cwd, event => publicTools.push(event))
: { status: 'missing', calls: [], assistantMessages: [] };
const hook = readPendingQuestion(session.pendingQuestionFile, fixture.cwd,
session.hermeticConfigDir, commandStartedAt, transcript);
const currentCalls = transcript.status === 'ready' ? transcript.calls.filter(call =>
call.sessionId === sessionId && !call.answered && !call.failed && publicTools.filter(event =>
event.sessionId === sessionId && event.kind === 'use' && event.name === 'AskUserQuestion' &&
event.toolUseId === call.toolUseId && Date.parse(event.timestamp) >= commandStartedAt &&
Date.parse(event.timestamp) <= driver.now() && isDeepStrictEqual(event.input?.questions, call.questions)).length === 1) : [];
nativeCandidates = currentCalls.slice();
validatedPendingQuestion = hook;
sampledAt = driver.now();
if (hook?.sessionId === sessionId && !transcript.calls.some(call => call.toolUseId === hook.toolUseId)) currentCalls.push(hook);
const activeReply = currentCalls.length === 1 && dxReplies.get(`${currentCalls[0]!.sessionId}:${currentCalls[0]!.toolUseId}`);
if (activeReply) {
pendingQuestion = undefined;
const next = planFloorDXReplyInput(viewport, currentCalls[0]!, activeReply);
if (next) { session.send(next.input); activeReply.stage = next.stage; }
checkpoint();
continue;
}
const matching = currentCalls.filter(call => dxContext
? planFloorDXPane(viewport, call) !== null : matchesNativePlanQuestion(viewport, call, planningDirectory));
pendingQuestion = matching.length === 1 ? matching[0] : undefined;
checkpoint();
const nativeQuestionVisible = Boolean(pendingQuestion);
const permissionIsActiveRender = !nativeQuestionVisible &&
(isPermissionDialogVisible(viewport) || isCroppedEditPermissionVisible(viewport));
if (permissionIsActiveRender) {
// An authorized file write enables the review, but never supplies its
// finding question. Exclude the permission's old render after approval.
const owned = currentFilePermissionBinding(ownedFilePermissions, fixture.cwd,
session.hermeticConfigDir, startedAt, transcript, viewport);
let ordinaryOwnedTarget = false;
if (fixture.workingPlanPath) try {
const parent = fs.realpathSync(path.dirname(fixture.workingPlanPath));
let target: fs.Stats | undefined;
try { target = fs.lstatSync(fixture.workingPlanPath); }
catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; }
ordinaryOwnedTarget = parent === fs.realpathSync(fixture.cwd) && (!target || target.isFile());
} catch { /* No authority for a linked, foreign, or unreadable target. */ }
if (owned && ordinaryOwnedTarget && owned.binding.guard(viewport, session.visibleText(), owned.epoch) === 'grant')
session.send('1\r');
}
if (permissionIsActiveRender) continue;
const questionViewport = dxContext && pendingQuestion ? planFloorDXPane(viewport, pendingQuestion)! : viewport;
const fp = pendingQuestion && capturePlanCountQuestion(questionViewport, new Set(), driver.now() - start, true, pendingQuestion, planningDirectory);
if (fp && pendingQuestion) {
const index = fp.nativeQuestionIndex ?? 0;
const question = pendingQuestion.questions[index]!;
const key = `${pendingQuestion.sessionId}:${pendingQuestion.toolUseId}`;
const chosen = setupChoices.get(key) ?? new Set<number>();
const allDesign = opts.skillName === 'plan-design-review' && designReviewSetupAUQ(fp)
? question.options.flatMap((option, i) => /^(?:Review )?All 7 (?:design )?(?:dimensions|passes)(?:\s*\(recommended\))?$/i.test(option.label.trim()) ? [i + 1] : []) : [];
const pick = pickPlanFloorMode(opts.skillName, question) ?? planCountPrerequisitePick(fp, fp)
?? (opts.skillName === 'plan-devex-review' ? pickPlanFloorProductType(question, opts.productType) : null)
?? (allDesign.length === 1 ? allDesign[0]! : null);
if (pick !== null) {
if (!chosen.has(index)) {
session.send(planCountQuestionInput(viewport, fp, pick));
chosen.add(index); setupChoices.set(key, chosen);
}
continue;
}
floorReview = { seed: fixture.seed, candidate: { transport: 'native', identity: `${key}:question:${index}`,
question: structuredClone(question) } };
} else {
// Public fallback must be a complete current question, not scrollback,
// a generic idle prompt, permission, tool result or quoted example.
floorReview = undefined;
const message = transcript.assistantMessages.filter(m => m.sessionId === sessionId &&
Date.parse(m.timestamp) >= commandStartedAt && Date.parse(m.timestamp) <= driver.now()).at(-1);
const compact = (text: string) => text.replace(/\s+/g, '');
if (!currentCalls.length && message && isProseAUQVisible(viewport) && isProseAUQVisible(message.text) &&
!/^\s*(?:>|`{3,}|~{3,})/m.test(message.text) && compact(viewport).includes(compact(message.text)))
floorReview = { seed: fixture.seed, candidate: { transport: 'prose',
identity: `${message.sessionId}:${message.timestamp}`, text: message.text } };
const submit = planCountSubmissionInput(viewport);
const packet = currentCalls.find(call => setupChoices.get(`${call.sessionId}:${call.toolUseId}`)?.size === call.questions.length);
if (submit && packet) {
const key = `${packet.sessionId}:${packet.toolUseId}`;
if (!submittedSetup.has(key)) { session.send(submit); submittedSetup.add(key); }
continue;
}
}
if (floorReview) {
const key = JSON.stringify(floorReview);
floorAssessment = assessed.get(key);
if (!floorAssessment) {
try {
floorAssessment = judgePlanFloorReview(floorReview, {
binary: resolveClaudeBinary() ?? 'claude', model: resolveEvalModel('warmup'), deadlineAt });
} catch (error) {
return finish({ auqObserved: false, outcome: 'assessment_error',
summary: `Finding assessment failed: ${error instanceof Error ? error.message : String(error)}`,
evidence: viewport, elapsedMs: driver.now() - startedAt });
}
assessed.set(key, floorAssessment);
}
if (dxContext && floorAssessment.kind === 'setup' && pendingQuestion?.questions.length === 1 &&
!pendingQuestion.questions[0]!.multiSelect && floorReview.candidate.transport === 'native') {
const key = `${pendingQuestion.sessionId}:${pendingQuestion.toolUseId}`;
dxReplies.set(key, { call: structuredClone(pendingQuestion), pane: questionViewport, reply: dxContext, stage: 'focus' });
}
if (floorAssessment.kind === 'finding') return finish({
auqObserved: true, outcome: 'auq_observed',
summary: `Current ${floorReview.candidate.transport} question addresses the owned seeded finding: ${floorAssessment.reason}`,
evidence: viewport, elapsedMs: driver.now() - startedAt,
});
}
// Silent write outside sanctioned dirs is the transcript-bug shape.
const writeRe = /⏺\s*(?:Write|Edit)\(([^)]+)\)/g;
let m: RegExpExecArray | null;
while ((m = writeRe.exec(visible)) !== null) {
const target = m[1] ?? '';
const sanctioned = SANCTIONED_WRITE_SUBSTRINGS.some((s) => target.includes(s));
if (!sanctioned && !isNumberedOptionListVisible(visible)) {
return finish({
auqObserved: false,
outcome: 'silent_write',
summary: `Write/Edit to ${target} fired before any AskUserQuestion`,
evidence: visible.slice(-3000),
elapsedMs: driver.now() - startedAt,
});
}
}
// Reached terminal without AUQ → transcript-bug regression.
// Note: COMPLETION_SUMMARY_RE is intentionally NOT checked here — it
// matches "GSTACK REVIEW REPORT" anywhere in the buffer, including
// when the agent does recon by reading existing plan files (which
// contain that string as a generated section). The plan_ready check
// (claude's actual "Ready to execute" confirmation) is the reliable
// terminal signal for "agent finished without asking."
if (isPlanReadyVisible(visible)) {
return finish({
auqObserved: false,
outcome: 'plan_ready',
summary: 'agent reached plan_ready without a qualifying finding question',
evidence: visible.slice(-3000),
elapsedMs: driver.now() - startedAt,
});
}
}
return finish({
return true;
},
tick: session => floorTick(run, session),
timeout: session => floorFinish(run, session, {
auqObserved: false,
outcome: 'timeout',
summary: targetDelivery.status === 'ready'
summary: run.targetDelivery.status === 'ready'
? `no qualifying finding question within ${timeoutMs}ms`
: `seeded target delivery unavailable within ${timeoutMs}ms: ${targetDelivery.reason ?? targetDelivery.status}`,
evidence: session.visibleSince(since).slice(-3000),
: `seeded target delivery unavailable within ${timeoutMs}ms: ${run.targetDelivery.reason ?? run.targetDelivery.status}`,
evidence: session.visibleSince(run.since).slice(-3000),
elapsedMs: driver.now() - startedAt,
});
} catch (error) {
floorFailed = true;
floorError = error;
throw error;
} finally {
try { captureBeforeClose?.(floorError); }
catch (error) { if (!floorFailed) { floorFailed = true; throw error; } }
finally {
try { await session.close(); }
catch (error) { if (!floorFailed) throw error; }
finally { fixture.cleanup(); }
}),
beforeClose: (session, failure) => floorBeforeClose(run, session, failure),
});
}
/** Mutable state of one floor attempt, shared by its session-loop steps. */
interface FloorRun {
opts: PlanSkillFloorOptions;
driver: PtyDriver;
startedAt: number;
timeoutMs: number;
dxContext: string | undefined;
fixture: ReturnType<typeof createPlanCountFixture>;
sessionId: string;
ownedFilePermissions: Array<{ expected: string; file: string; guard: ReturnType<typeof createPlanCountPermissionGuard> }>;
planningDirectory: string | undefined;
transcript: PlanCountTranscript;
viewport: string;
publicTools: NativePublicToolEvent[];
pendingQuestion: NativePlanQuestionCall | undefined;
floorReview: PlanFloorReview | undefined;
floorAssessment: PlanFloorAssessment | undefined;
setupChoices: Map<string, Set<number>>;
submittedSetup: Set<string>;
assessed: Map<string, PlanFloorAssessment>;
dxReplies: Map<string, PlanFloorDXReply>;
since: number;
commandStartedAt: number;
deliveryOptions: { seed: string; sessionId: string; slashCommand: string; startedAt: number };
targetDelivery: PlanFloorTargetDelivery;
saveSnapshot: ReturnType<typeof createPlanCountSnapshotWriter>;
nativeCandidates: NativePlanQuestionCall[];
validatedPendingQuestion: ReturnType<typeof readPendingQuestion>;
sampledAt: number | undefined;
lastCheckpointAt: number;
lastCheckpointState: string;
artifactError: string | undefined;
finished: boolean;
/** Command sent and capture state ready; before-close capture applies from here. */
started: boolean;
/** Loop start after the command; the finding clock and assessment deadline. */
start: number;
deadlineAt: number;
screenDeadlineAt: number;
}
async function floorStart(run: FloorRun, session: ClaudePtySession): Promise<undefined> {
const { opts, driver, fixture } = run;
run.ownedFilePermissions = (session.pendingFilePermissionFiles ?? []).map(binding =>
({ ...binding, guard: createPlanCountPermissionGuard() }));
run.planningDirectory = session.hermeticConfigDir ? path.join(session.hermeticConfigDir, 'plans') : undefined;
await driver.sleep(8000); // boot grace + auto-trust handler window
run.since = session.mark();
run.commandStartedAt = driver.now();
session.send(`${opts.slashCommand} PLAN.md\r`);
run.deliveryOptions = { seed: fixture.seed, sessionId: run.sessionId,
slashCommand: opts.slashCommand, startedAt: run.commandStartedAt };
run.targetDelivery = readPlanFloorTarget(session.hermeticConfigDir, fixture.cwd,
{ ...run.deliveryOptions, now: driver.now() });
run.saveSnapshot = createPlanCountSnapshotWriter();
run.started = true;
run.start = driver.now();
run.deadlineAt = run.start + run.timeoutMs;
run.screenDeadlineAt = driver.monotonic() + run.timeoutMs;
return undefined;
}
function floorCapture(run: FloorRun, session: ClaudePtySession, observation: object) {
const { fixture } = run;
const recorderStatus = pendingQuestionRecorderStatus(session.pendingQuestionFile, fixture.cwd, session.hermeticConfigDir);
const artifacts = run.saveSnapshot({ skillName: run.opts.skillName, cwd: fixture.cwd,
claudeConfigDir: session.hermeticConfigDir, raw: session.rawOutput(),
visible: session.visibleSince(run.since), viewport: run.viewport,
observation: { ...observation, transcript: run.transcript, publicTools: run.publicTools, pendingQuestion: run.pendingQuestion,
floorReview: run.floorReview, floorAssessment: run.floorAssessment, targetDelivery: run.targetDelivery, commandStartedAt: run.commandStartedAt,
pendingWriteInputs: run.ownedFilePermissions.flatMap(binding => {
const input = readPendingWriteInput(binding.file, binding.expected, fixture.cwd, session.hermeticConfigDir, run.startedAt);
return input ? [input] : [];
}),
questionDiagnostics: { sampledAt: run.sampledAt, parentSessionId: run.sessionId, recorderStatus, recorderStatusAt: run.driver.now(),
nativeCandidates: run.nativeCandidates, validatedPendingQuestion: run.validatedPendingQuestion },
...(run.dxContext ? { setupContextReplies: [...run.dxReplies.values()] } : {}), artifactError: run.artifactError } });
if (artifacts.artifactError) {
run.artifactError ??= artifacts.artifactError;
console.error(`PTY artifact write failed: ${artifacts.artifactError}`);
}
return { ...artifacts, ...(run.artifactError ? { artifactError: run.artifactError } : {}) };
}
function floorCheckpoint(run: FloorRun, session: ClaudePtySession): void {
const state = JSON.stringify([pendingQuestionRecorderStatus(session.pendingQuestionFile, run.fixture.cwd, session.hermeticConfigDir),
run.targetDelivery.status, run.nativeCandidates, run.validatedPendingQuestion, run.pendingQuestion, [...run.dxReplies.values()]]);
if (state === run.lastCheckpointState && run.driver.now() - run.lastCheckpointAt < 15_000) return;
run.lastCheckpointState = state; run.lastCheckpointAt = run.driver.now();
floorCapture(run, session, { state: 'in_progress', elapsedMs: run.driver.now() - run.startedAt });
}
function floorBeforeClose(run: FloorRun, session: ClaudePtySession, failure: { error: unknown } | undefined): void {
if (!run.started) return;
if (failure && session.hermeticConfigDir) {
const publicTools: NativePublicToolEvent[] = [];
run.publicTools = publicTools;
run.transcript = readPlanCountTranscript(session.hermeticConfigDir, run.fixture.cwd, event => publicTools.push(event));
}
if (!run.finished) floorCapture(run, session, { state: failure ? 'threw' : 'in_progress', error: failure ? String(failure.error) : undefined,
captureReason: 'before_cleanup', elapsedMs: run.driver.now() - run.startedAt });
}
function floorFinish(run: FloorRun, session: ClaudePtySession, observation: PlanSkillFloorObservation): PlanSkillFloorObservation {
const artifacts = floorCapture(run, session, observation);
run.finished = true;
return { ...observation, targetDelivery: run.targetDelivery, ...artifacts };
}
async function floorTick(run: FloorRun, session: ClaudePtySession): Promise<PtyStep<PlanSkillFloorObservation>> {
const { opts, driver, fixture, sessionId } = run;
const visible = session.visibleSince(run.since);
const finish = (observation: PlanSkillFloorObservation) => ({ done: floorFinish(run, session, observation) });
if (session.exited()) return finish({ auqObserved: false, outcome: 'exited',
summary: `claude exited (code=${session.exitCode()}) before a qualifying finding`,
evidence: visible.slice(-3000), elapsedMs: driver.now() - run.startedAt });
if (isRejectedSlashCommand(visible, opts.slashCommand)) return finish({ auqObserved: false, outcome: 'exited',
summary: `claude rejected ${opts.slashCommand} as unknown command`,
evidence: visible.slice(-3000), elapsedMs: driver.now() - run.startedAt });
if (run.targetDelivery.status !== 'ready') {
run.targetDelivery = readPlanFloorTarget(session.hermeticConfigDir, fixture.cwd,
{ ...run.deliveryOptions, now: driver.now() });
if (run.targetDelivery.status !== 'ready') {
run.viewport = await session.currentScreen(run.screenDeadlineAt);
floorCheckpoint(run, session);
return 'continue';
}
}
// Current native identity precedes permission handling and finding assessment.
run.floorReview = undefined; run.floorAssessment = undefined;
run.viewport = await session.currentScreen(run.screenDeadlineAt);
const publicTools: NativePublicToolEvent[] = [];
run.publicTools = publicTools;
run.transcript = session.hermeticConfigDir
? readPlanCountTranscript(session.hermeticConfigDir, fixture.cwd, event => publicTools.push(event))
: { status: 'missing', calls: [], assistantMessages: [] };
const hook = readPendingQuestion(session.pendingQuestionFile, fixture.cwd,
session.hermeticConfigDir, run.commandStartedAt, run.transcript);
const currentCalls = run.transcript.status === 'ready' ? run.transcript.calls.filter(call =>
call.sessionId === sessionId && !call.answered && !call.failed && publicTools.filter(event =>
event.sessionId === sessionId && event.kind === 'use' && event.name === 'AskUserQuestion' &&
event.toolUseId === call.toolUseId && Date.parse(event.timestamp) >= run.commandStartedAt &&
Date.parse(event.timestamp) <= driver.now() && isDeepStrictEqual(event.input?.questions, call.questions)).length === 1) : [];
run.nativeCandidates = currentCalls.slice();
run.validatedPendingQuestion = hook;
run.sampledAt = driver.now();
if (hook?.sessionId === sessionId && !run.transcript.calls.some(call => call.toolUseId === hook.toolUseId)) currentCalls.push(hook);
const activeReply = currentCalls.length === 1 && run.dxReplies.get(`${currentCalls[0]!.sessionId}:${currentCalls[0]!.toolUseId}`);
if (activeReply) {
run.pendingQuestion = undefined;
const next = planFloorDXReplyInput(run.viewport, currentCalls[0]!, activeReply);
if (next) { session.send(next.input); activeReply.stage = next.stage; }
floorCheckpoint(run, session);
return 'continue';
}
const matching = currentCalls.filter(call => run.dxContext
? planFloorDXPane(run.viewport, call) !== null : matchesNativePlanQuestion(run.viewport, call, run.planningDirectory));
run.pendingQuestion = matching.length === 1 ? matching[0] : undefined;
floorCheckpoint(run, session);
if (floorPermission(run, session)) return 'continue';
const question = floorQuestion(run, session, currentCalls);
if (question) return question;
return floorAssess(run, session, visible);
}
/** An authorized owned file write enables the review, but never supplies its
* finding question. True when an active permission render owns this tick. */
function floorPermission(run: FloorRun, session: ClaudePtySession): boolean {
const { fixture, viewport } = run;
const permissionIsActiveRender = !run.pendingQuestion &&
(isPermissionDialogVisible(viewport) || isCroppedEditPermissionVisible(viewport));
if (!permissionIsActiveRender) return false;
// Exclude the permission's old render after approval.
const owned = currentFilePermissionBinding(run.ownedFilePermissions, fixture.cwd,
session.hermeticConfigDir, run.startedAt, run.transcript, viewport);
let ordinaryOwnedTarget = false;
if (fixture.workingPlanPath) try {
const parent = fs.realpathSync(path.dirname(fixture.workingPlanPath));
let target: fs.Stats | undefined;
try { target = fs.lstatSync(fixture.workingPlanPath); }
catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; }
ordinaryOwnedTarget = parent === fs.realpathSync(fixture.cwd) && (!target || target.isFile());
} catch { /* No authority for a linked, foreign, or unreadable target. */ }
if (owned && ordinaryOwnedTarget && owned.binding.guard(viewport, session.visibleText(), owned.epoch) === 'grant')
session.send('1\r');
return true;
}
/** Answer declared setup choices; otherwise stage the current question (native
* or complete prose) as the floor review candidate. A step ends this tick. */
function floorQuestion(run: FloorRun, session: ClaudePtySession, currentCalls: NativePlanQuestionCall[]): PtyStep<PlanSkillFloorObservation> | undefined {
const { opts, fixture, viewport, pendingQuestion } = run;
const questionViewport = run.dxContext && pendingQuestion ? planFloorDXPane(viewport, pendingQuestion)! : viewport;
const fp = pendingQuestion && capturePlanCountQuestion(questionViewport, new Set(), run.driver.now() - run.start, true, pendingQuestion, run.planningDirectory);
if (fp && pendingQuestion) {
const index = fp.nativeQuestionIndex ?? 0;
const question = pendingQuestion.questions[index]!;
const key = `${pendingQuestion.sessionId}:${pendingQuestion.toolUseId}`;
const chosen = run.setupChoices.get(key) ?? new Set<number>();
const allDesign = opts.skillName === 'plan-design-review' && designReviewSetupAUQ(fp)
? question.options.flatMap((option, i) => /^(?:Review )?All 7 (?:design )?(?:dimensions|passes)(?:\s*\(recommended\))?$/i.test(option.label.trim()) ? [i + 1] : []) : [];
const pick = pickPlanFloorMode(opts.skillName, question) ?? planCountPrerequisitePick(fp, fp)
?? (opts.skillName === 'plan-devex-review' ? pickPlanFloorProductType(question, opts.productType) : null)
?? (allDesign.length === 1 ? allDesign[0]! : null);
if (pick !== null) {
if (!chosen.has(index)) {
session.send(planCountQuestionInput(viewport, fp, pick));
chosen.add(index); run.setupChoices.set(key, chosen);
}
return 'continue';
}
run.floorReview = { seed: fixture.seed, candidate: { transport: 'native', identity: `${key}:question:${index}`,
question: structuredClone(question) } };
return undefined;
}
// Public fallback must be a complete current question, not scrollback,
// a generic idle prompt, permission, tool result or quoted example.
run.floorReview = undefined;
const message = run.transcript.assistantMessages.filter(m => m.sessionId === run.sessionId &&
Date.parse(m.timestamp) >= run.commandStartedAt && Date.parse(m.timestamp) <= run.driver.now()).at(-1);
const compact = (text: string) => text.replace(/\s+/g, '');
if (!currentCalls.length && message && isProseAUQVisible(viewport) && isProseAUQVisible(message.text) &&
!/^\s*(?:>|`{3,}|~{3,})/m.test(message.text) && compact(viewport).includes(compact(message.text)))
run.floorReview = { seed: fixture.seed, candidate: { transport: 'prose',
identity: `${message.sessionId}:${message.timestamp}`, text: message.text } };
const submit = planCountSubmissionInput(viewport);
const packet = currentCalls.find(call => run.setupChoices.get(`${call.sessionId}:${call.toolUseId}`)?.size === call.questions.length);
if (submit && packet) {
const key = `${packet.sessionId}:${packet.toolUseId}`;
if (!run.submittedSetup.has(key)) { session.send(submit); run.submittedSetup.add(key); }
return 'continue';
}
return undefined;
}
/** Assess a staged candidate once; then the silent-write and plan-ready terminals. */
function floorAssess(run: FloorRun, session: ClaudePtySession, visible: string): PtyStep<PlanSkillFloorObservation> {
const { driver, floorReview, pendingQuestion } = run;
const finish = (observation: PlanSkillFloorObservation) => ({ done: floorFinish(run, session, observation) });
if (floorReview) {
const key = JSON.stringify(floorReview);
run.floorAssessment = run.assessed.get(key);
if (!run.floorAssessment) {
try {
run.floorAssessment = judgePlanFloorReview(floorReview, {
binary: resolveClaudeBinary() ?? 'claude', model: resolveEvalModel('warmup'), deadlineAt: run.deadlineAt });
} catch (error) {
return finish({ auqObserved: false, outcome: 'assessment_error',
summary: `Finding assessment failed: ${error instanceof Error ? error.message : String(error)}`,
evidence: run.viewport, elapsedMs: driver.now() - run.startedAt });
}
run.assessed.set(key, run.floorAssessment);
}
if (run.dxContext && run.floorAssessment.kind === 'setup' && pendingQuestion?.questions.length === 1 &&
!pendingQuestion.questions[0]!.multiSelect && floorReview.candidate.transport === 'native') {
const replyKey = `${pendingQuestion.sessionId}:${pendingQuestion.toolUseId}`;
const questionViewport = planFloorDXPane(run.viewport, pendingQuestion)!;
run.dxReplies.set(replyKey, { call: structuredClone(pendingQuestion), pane: questionViewport, reply: run.dxContext, stage: 'focus' });
}
if (run.floorAssessment.kind === 'finding') return finish({
auqObserved: true, outcome: 'auq_observed',
summary: `Current ${floorReview.candidate.transport} question addresses the owned seeded finding: ${run.floorAssessment.reason}`,
evidence: run.viewport, elapsedMs: driver.now() - run.startedAt,
});
}
// Silent write outside sanctioned dirs is the transcript-bug shape.
const writeRe = /⏺\s*(?:Write|Edit)\(([^)]+)\)/g;
let m: RegExpExecArray | null;
while ((m = writeRe.exec(visible)) !== null) {
const target = m[1] ?? '';
const sanctioned = SANCTIONED_WRITE_SUBSTRINGS.some((s) => target.includes(s));
if (!sanctioned && !isNumberedOptionListVisible(visible)) {
return finish({
auqObserved: false,
outcome: 'silent_write',
summary: `Write/Edit to ${target} fired before any AskUserQuestion`,
evidence: visible.slice(-3000),
elapsedMs: driver.now() - run.startedAt,
});
}
}
// Reached terminal without AUQ → transcript-bug regression.
// Note: COMPLETION_SUMMARY_RE is intentionally NOT checked here — it
// matches "GSTACK REVIEW REPORT" anywhere in the buffer, including
// when the agent does recon by reading existing plan files (which
// contain that string as a generated section). The plan_ready check
// (claude's actual "Ready to execute" confirmation) is the reliable
// terminal signal for "agent finished without asking."
if (isPlanReadyVisible(visible)) {
return finish({
auqObserved: false,
outcome: 'plan_ready',
summary: 'agent reached plan_ready without a qualifying finding question',
evidence: visible.slice(-3000),
elapsedMs: driver.now() - run.startedAt,
});
}
return 'continue';
}
test/helpers/pty/runners/observation.ts
+317 -281
View File
@@ -13,9 +13,9 @@ import { randomUUID } from 'node:crypto';
import { classifyVisible, isProseAUQVisible, isScopeGateAutoSelectVisible, isScopeGateQuestionVisible } from '../classify';
import { judgePtyState, logPtySnapshot } from '../judge';
import type { PtyStateVerdict } from '../judge';
import { launchClaudePty } from '../launch';
import type { ClaudePtySession } from '../launch';
import { extractPlanFilePath, isNumberedOptionListVisible, isPermissionDialogVisible, isPlanReadyVisible, isRejectedSlashCommand } from '../screen';
import type { PtyDriver } from '../session';
import { realPtyDriver, runPtySession, type PtyDriver, type PtyStep } from '../session';
// ---------------------------------------------------------------------------
// High-level skill-mode test contract
@@ -116,24 +116,8 @@ export interface PlanSkillObservation {
tokensObserved?: Record<string, boolean>;
}
/**
* The contract for "skill X invoked in plan mode behaves correctly."
*
* PASS: outcome is 'asked' or 'plan_ready'.
* - 'asked' = the skill is gating decisions on the user, as expected.
* - 'plan_ready' = the skill ran end-to-end, wrote a plan file, and
* surfaced claude's native confirmation. Some skills (like
* plan-design-review on a no-UI branch) legitimately reach plan_ready
* without firing AskUserQuestion because they short-circuit.
*
* FAIL: 'silent_write' or 'exited' or 'timeout'.
*
* This replaces the SDK-based runPlanModeSkillTest which never worked
* because plan mode renders its native confirmation as TTY UI, not via
* the AskUserQuestion tool — so canUseTool never fired and the assertion
* counted zero questions.
*/
export async function runPlanSkillObservation(opts: {
/** Options for runPlanSkillObservation. */
export interface PlanSkillObservationOptions {
/** Skill name, e.g. 'plan-ceo-review'. */
skillName: string;
/** Whether to launch in plan mode. Default true. The no-op regression
@@ -186,279 +170,331 @@ export async function runPlanSkillObservation(opts: {
trackTokens?: string[];
/** Launch seam and clock; tests pass the fake driver. Default: real launcher and clocks. */
driver?: PtyDriver;
}): Promise<PlanSkillObservation> {
const driver = opts.driver ?? { launch: launchClaudePty, now: () => Date.now(),
monotonic: () => performance.now(), sleep: (ms: number) => Bun.sleep(ms) };
}
/**
* The contract for "skill X invoked in plan mode behaves correctly."
*
* PASS: outcome is 'asked' or 'plan_ready'.
* - 'asked' = the skill is gating decisions on the user, as expected.
* - 'plan_ready' = the skill ran end-to-end, wrote a plan file, and
* surfaced claude's native confirmation. Some skills (like
* plan-design-review on a no-UI branch) legitimately reach plan_ready
* without firing AskUserQuestion because they short-circuit.
*
* FAIL: 'silent_write' or 'exited' or 'timeout'.
*
* This replaces the SDK-based runPlanModeSkillTest which never worked
* because plan mode renders its native confirmation as TTY UI, not via
* the AskUserQuestion tool — so canUseTool never fired and the assertion
* counted zero questions.
*/
export async function runPlanSkillObservation(opts: PlanSkillObservationOptions): Promise<PlanSkillObservation> {
const driver = opts.driver ?? realPtyDriver;
const startedAt = driver.now();
const budgetMs = opts.timeoutMs ?? 180_000;
const deadlineAt = startedAt + budgetMs;
const screenDeadlineAt = driver.monotonic() + budgetMs;
// Explicitly identify only a new seeded plan-mode session. Caller-owned
// resume/session arguments retain their existing behavior.
const scopeSessionId = opts.initialPlanContent && opts.inPlanMode !== false &&
!opts.extraArgs?.some(arg => /^(?:--session-id|--resume|--continue|-r|-c)(?:=|$)/.test(arg))
? randomUUID() : undefined;
const readAutoDecisionState = opts.autoDecisionState
? bindAutoDecisionState(opts.autoDecisionState, opts.env, opts.skillName) : undefined;
const saveSnapshot = createPlanCountSnapshotWriter();
const session = await driver.launch({
permissionMode: opts.inPlanMode === false ? null : 'plan',
cwd: opts.cwd,
timeoutMs: (opts.timeoutMs ?? 180_000) + 30_000,
extraArgs: [...(opts.extraArgs ?? []), ...(scopeSessionId ? ['--session-id', scopeSessionId] : [])],
env: {
...(opts.inPlanMode !== false && !opts.extraArgs?.some(arg => /^--permission-mode(?:=|$)/.test(arg))
? { GSTACK_PLAN_MODE: 'active' } : {}),
...opts.env,
const run: ObservationRun = {
opts, driver, startedAt, budgetMs, deadlineAt: startedAt + budgetMs, scopeSessionId,
readAutoDecisionState: opts.autoDecisionState
? bindAutoDecisionState(opts.autoDecisionState, opts.env, opts.skillName) : undefined,
saveSnapshot: createPlanCountSnapshotWriter(),
since: 0, commandStartedAt: 0, start: 0, lastJudgeAt: 0, lastJudgeVerdict: null,
proseAUQEverObserved: false, waitingEverObserved: false,
scopeGateQuestionObserved: false, scopeGateAutoSelectObserved: false,
scopeTranscript: { status: 'missing', calls: [], assistantMessages: [] }, scopeTools: [],
nativeAutoDecide: null, nativePolledAt: null,
tokensObserved: Object.fromEntries((opts.trackTokens ?? []).map(t => [t, false])),
};
return runPtySession<PlanSkillObservation>({
driver,
launch: {
permissionMode: opts.inPlanMode === false ? null : 'plan',
cwd: opts.cwd,
timeoutMs: (opts.timeoutMs ?? 180_000) + 30_000,
extraArgs: [...(opts.extraArgs ?? []), ...(scopeSessionId ? ['--session-id', scopeSessionId] : [])],
env: {
...(opts.inPlanMode !== false && !opts.extraArgs?.some(arg => /^--permission-mode(?:=|$)/.test(arg))
? { GSTACK_PLAN_MODE: 'active' } : {}),
...opts.env,
},
model: opts.model,
seedSkills: true,
observeScreen: !!opts.initialPlanContent,
screenDeadlineAt: driver.monotonic() + budgetMs,
},
model: opts.model,
seedSkills: true,
observeScreen: !!opts.initialPlanContent,
screenDeadlineAt,
start: session => observationStart(run, session),
poll: async () => {
if (!(driver.now() < run.deadlineAt)) return false;
await driver.sleep(Math.min(2000, Math.max(0, run.deadlineAt - driver.now())));
return true;
},
tick: session => observationTick(run, session),
timeout: session => observationTimeout(run, session),
onError: (session, error) => observationFailure(run, session, error),
});
}
let observationFailed = false;
try {
const preflightTimeout = async (summary: string): Promise<PlanSkillObservation> => {
let viewport: string | undefined, viewportError: string | undefined;
try { viewport = (await session.currentScreenFrame())?.text; } catch (error) { viewportError = String(error); }
const artifacts = saveSnapshot({ skillName: opts.skillName, cwd: path.resolve(opts.cwd ?? process.cwd()),
claudeConfigDir: session.hermeticConfigDir, raw: session.rawOutput(), visible: session.visibleText(), viewport,
observation: { state: 'plan_skill_preflight_timeout', summary, scopeSessionId, startedAt, deadlineAt, viewportError } });
return {
outcome: 'timeout', summary, evidence: session.visibleText().slice(-2000),
elapsedMs: driver.now() - startedAt,
proseAUQEverObserved: false, waitingEverObserved: false,
scopeGateQuestionObserved: false, scopeGateAutoSelectObserved: false,
...(opts.trackTokens?.length ? { tokensObserved: Object.fromEntries(opts.trackTokens.map(t => [t, false])) } : {}),
...artifacts,
};
};
// Entry deadline → boot → owned paste/receipt/ack → slash → observation.
// Setup consumes the existing case budget; cleanup has its separate grace.
if (!opts.initialPlanContent) await driver.sleep(Math.min(8000, Math.max(0, deadlineAt - driver.now())));
if (opts.initialPlanContent) {
const seed = `Keep this draft plan as context. Briefly acknowledge receipt, then wait for my next message containing a slash command. Do not start the review or call tools yet.\n\n${opts.initialPlanContent}`;
try {
await submitPlanSeed({...session, currentScreen: session.currentScreenFrame}, seed, {
cwd: opts.cwd ?? process.cwd(), launchedAt: startedAt, deadlineAt,
isQuestionOrPermission: text => isProseAUQVisible(text) || isNumberedOptionListVisible(text) || isPermissionDialogVisible(text),
});
} catch (error) {
if (!(error instanceof PlanSeedTimeout)) throw error;
return await preflightTimeout(`Plan seed submission failed: ${error.message}`);
}
}
if (driver.now() >= deadlineAt) return await preflightTimeout('Boot or seed preflight exhausted the existing case budget');
const commandStartedAt = driver.now();
const since = session.mark();
session.send(`/${opts.skillName}\r`);
interface ObservationRun {
opts: PlanSkillObservationOptions;
driver: PtyDriver;
startedAt: number;
budgetMs: number;
deadlineAt: number;
scopeSessionId: string | undefined;
readAutoDecisionState: ReturnType<typeof bindAutoDecisionState> | undefined;
saveSnapshot: ReturnType<typeof createPlanCountSnapshotWriter>;
since: number;
commandStartedAt: number;
start: number;
lastJudgeAt: number;
lastJudgeVerdict: PtyStateVerdict | null;
// High-water marks: did we EVER see a prose-AUQ surface or a judge
// 'waiting' verdict during the run? Models may surface options
// briefly, then resume thinking when no user response comes (test
// env has no responder). At timeout we trust historical signals
// even if the current state is 'working'.
proseAUQEverObserved: boolean;
waitingEverObserved: boolean;
scopeGateQuestionObserved: boolean;
scopeGateAutoSelectObserved: boolean;
scopeTranscript: PlanCountTranscript;
scopeTools: NativePublicToolEvent[];
nativeAutoDecide: NativeAutoDecision | null;
nativePolledAt: number | null;
tokensObserved: Record<string, boolean>;
}
const start = driver.now();
let lastJudgeAt = 0;
let lastJudgeVerdict: PtyStateVerdict | null = null;
// High-water marks: did we EVER see a prose-AUQ surface or a judge
// 'waiting' verdict during the run? Models may surface options
// briefly, then resume thinking when no user response comes (test
// env has no responder). At timeout we trust historical signals
// even if the current state is 'working'.
let proseAUQEverObserved = false;
let waitingEverObserved = false;
let scopeGateQuestionObserved = false;
let scopeGateAutoSelectObserved = false;
let scopeTranscript: PlanCountTranscript = { status: 'missing', calls: [], assistantMessages: [] };
let scopeTools: NativePublicToolEvent[] = [];
let nativeAutoDecide: NativeAutoDecision | null = null;
let nativePolledAt: number | null = null;
const tokensObserved: Record<string, boolean> = {};
for (const t of opts.trackTokens ?? []) tokensObserved[t] = false;
// Single source for the high-water flags at EVERY return site. Hand-
// spreading them per-site already drifted once (the judge-waiting return
// omitted the prose/waiting flags); a site that forgets a must-stay-false
// flag makes `obs.flag ?? false` negative assertions pass vacuously.
const highWaterFlags = () => {
const flags = { proseAUQEverObserved, waitingEverObserved,
scopeGateQuestionObserved, scopeGateAutoSelectObserved,
...(nativeAutoDecide ? { nativeAutoDecide } : {}),
...(opts.trackTokens?.length ? { tokensObserved } : {}) };
// Preserve the measured terminal flags and public evidence before the
// hermetic session is removed, including when a later assertion fails.
const artifacts = saveSnapshot({ skillName: opts.skillName,
cwd: path.resolve(opts.cwd ?? process.cwd()), claudeConfigDir: session.hermeticConfigDir,
raw: session.rawOutput(), visible: session.visibleSince(since),
observation: { state: 'plan_skill_observation_terminal', ...flags,
commandStartedAt, scopeSessionId, native: scopeTranscript, publicTools: scopeTools,
nativePolledAt, nativeScope: 'Latest owned native poll, refreshed while observing; not an exhaustive terminal journal.' } });
return { ...flags, ...artifacts };
};
const JUDGE_AFTER_MS = 60_000;
const JUDGE_INTERVAL_MS = 30_000;
while (driver.now() < deadlineAt) {
await driver.sleep(Math.min(2000, Math.max(0, deadlineAt - driver.now())));
const visible = session.visibleSince(since);
async function observationPreflightTimeout(run: ObservationRun, session: ClaudePtySession, summary: string): Promise<PlanSkillObservation> {
const { opts } = run;
let viewport: string | undefined, viewportError: string | undefined;
try { viewport = (await session.currentScreenFrame())?.text; } catch (error) { viewportError = String(error); }
const artifacts = run.saveSnapshot({ skillName: opts.skillName, cwd: path.resolve(opts.cwd ?? process.cwd()),
claudeConfigDir: session.hermeticConfigDir, raw: session.rawOutput(), visible: session.visibleText(), viewport,
observation: { state: 'plan_skill_preflight_timeout', summary, scopeSessionId: run.scopeSessionId,
startedAt: run.startedAt, deadlineAt: run.deadlineAt, viewportError } });
return {
outcome: 'timeout', summary, evidence: session.visibleText().slice(-2000),
elapsedMs: run.driver.now() - run.startedAt,
proseAUQEverObserved: false, waitingEverObserved: false,
scopeGateQuestionObserved: false, scopeGateAutoSelectObserved: false,
...(opts.trackTokens?.length ? { tokensObserved: Object.fromEntries(opts.trackTokens.map(t => [t, false])) } : {}),
...artifacts,
};
}
if (session.exited()) {
return {
outcome: 'exited',
summary: `claude exited (code=${session.exitCode()}) before reaching a terminal outcome`,
evidence: visible.slice(-2000),
elapsedMs: driver.now() - startedAt,
...highWaterFlags(),
};
}
if (isRejectedSlashCommand(visible, `/${opts.skillName}`)) {
return {
outcome: 'exited',
summary: `claude rejected /${opts.skillName} as unknown command (skill not registered in this cwd)`,
evidence: visible.slice(-2000),
elapsedMs: driver.now() - startedAt,
...highWaterFlags(),
};
}
const classified = classifyVisible(visible, {
strictPlanWrites: !!opts.initialPlanContent,
currentScreen: opts.initialPlanContent ? await session.currentScreen() : undefined,
});
const pendingSeededCompletion = !!opts.initialPlanContent &&
isPlanReadyVisible(visible) && classified === null;
// Cheap surface-tracking: did the model ever surface a prose AUQ in
// this tick's recent buffer? Track once-true (high water).
if (!proseAUQEverObserved && !pendingSeededCompletion && isProseAUQVisible(visible)) {
proseAUQEverObserved = true;
logPtySnapshot(visible, {
testName: opts.skillName,
elapsedMs: driver.now() - start,
tag: 'prose-auq-surfaced',
});
}
// Scope-gate render tracking (same high-water shape). Full-run
// detection matters because the 2KB evidence tail usually scrolls
// past the gate render before the outcome fires.
if (!scopeGateQuestionObserved && isScopeGateQuestionVisible(visible)) {
scopeGateQuestionObserved = true;
}
if (!scopeGateAutoSelectObserved && isScopeGateAutoSelectVisible(visible)) {
scopeGateAutoSelectObserved = true;
}
// Keep reading after scope selection: an AUTO_DECIDE annotation may
// arrive later, and a prior poll cannot establish its current ownership.
if (scopeSessionId && opts.initialPlanContent && session.hermeticConfigDir) {
scopeTools = [];
scopeTranscript = readPlanCountTranscript(session.hermeticConfigDir,
path.resolve(opts.cwd ?? process.cwd()), event => scopeTools.push(event));
nativePolledAt = driver.now();
if (!scopeGateAutoSelectObserved) scopeGateAutoSelectObserved = nativeSeededPlanSelection(scopeTranscript, scopeTools, {
seed: opts.initialPlanContent, skillName: opts.skillName, sessionId: scopeSessionId, commandStartedAt,
});
}
for (const t of opts.trackTokens ?? []) {
if (!tokensObserved[t] && visible.includes(t)) tokensObserved[t] = true;
}
if (classified) {
const obs: PlanSkillObservation = {
...classified,
evidence: visible.slice(-2000),
elapsedMs: driver.now() - startedAt,
...highWaterFlags(),
};
// Capture the plan file path on any outcome where one may have been
// written. Gating only on 'plan_ready' missed two cases: (1) the
// 'asked' outcome where the model wrote a plan partway through then
// paused on a question, and (2) 'wrote_findings_before_asking' where
// the bug is precisely that the plan was written. The
// assertReviewReportAtBottom checks downstream gate on planFile
// existing, not on the outcome.
const planFile = extractPlanFilePath(visible);
if (planFile) obs.planFile = planFile;
return obs;
}
// Terminal classification retains precedence (including actual questions
// and writes). Only an unclassified frame may use owned native auto-decision evidence.
if (scopeSessionId) {
nativeAutoDecide = findNativeAutoDecision(scopeTranscript, scopeTools, {
skillName: opts.skillName, sessionId: scopeSessionId, commandStartedAt, now: driver.now(), proseQuestionObserved: proseAUQEverObserved,
stateEvidence: readAutoDecisionState?.(),
});
if (nativeAutoDecide) return {
outcome: 'auto_decided',
summary: 'owned native session completed the saved-preference auto-decision',
evidence: visible.slice(-2000), elapsedMs: driver.now() - startedAt,
...highWaterFlags(),
};
}
// LLM judge fallback: if regex detectors didn't classify and we've
// burned >60s with periodic ticks, ask Haiku "is the model waiting,
// working, or hung?" Treat 'waiting' as 'asked' (model surfaced a
// question via prose the regex couldn't reassemble). Snapshot the
// visible buffer at each judge call when GSTACK_PTY_LOG=1.
const elapsed = driver.now() - start;
if (elapsed > JUDGE_AFTER_MS && driver.now() - lastJudgeAt > JUDGE_INTERVAL_MS) {
lastJudgeAt = driver.now();
logPtySnapshot(visible, { testName: opts.skillName, elapsedMs: elapsed, tag: 'judge-tick' });
lastJudgeVerdict = judgePtyState(visible, { testName: opts.skillName });
if (lastJudgeVerdict.state === 'waiting' && !pendingSeededCompletion) {
waitingEverObserved = true;
if (opts.requireProseEvidence && !proseAUQEverObserved) continue;
return {
outcome: 'asked',
summary: `LLM judge: ${lastJudgeVerdict.reasoning} (state=waiting after ${Math.round(elapsed / 1000)}s)`,
evidence: visible.slice(-2000),
elapsedMs: driver.now() - startedAt,
...highWaterFlags(),
};
}
}
}
// Timeout fallback: if we observed a prose-AUQ surface OR a judge
// 'waiting' verdict at any point during the run, treat as 'asked'.
// This catches the model-surfaced-then-resumed-thinking case where
// by the time the timeout fires, the buffer has moved past the
// options into spinner state but the question DID surface earlier.
const finalVisible = session.visibleSince(since);
if (proseAUQEverObserved || waitingEverObserved && !opts.requireProseEvidence) {
return {
outcome: 'asked',
summary:
`prose-AUQ surface observed during run (proseAUQEverObserved=${proseAUQEverObserved}, waitingEverObserved=${waitingEverObserved}); model surfaced the question and the test budget elapsed without a follow-up classification` +
(lastJudgeVerdict
? ` (last LLM judge: ${lastJudgeVerdict.state} — ${lastJudgeVerdict.reasoning})`
: ''),
evidence: finalVisible.slice(-2000),
elapsedMs: driver.now() - startedAt,
...highWaterFlags(),
};
}
return {
outcome: 'timeout',
summary:
`no terminal outcome within ${budgetMs}ms` +
(lastJudgeVerdict
? ` (last LLM judge: state=${lastJudgeVerdict.state} — ${lastJudgeVerdict.reasoning})`
: ''),
evidence: finalVisible.slice(-2000),
elapsedMs: driver.now() - startedAt,
...highWaterFlags(),
};
} catch (error) {
observationFailed = true;
/** Entry deadline → boot → owned paste/receipt/ack → slash → observation.
* Setup consumes the existing case budget; cleanup has its separate grace. */
async function observationStart(run: ObservationRun, session: ClaudePtySession): Promise<PlanSkillObservation | undefined> {
const { opts, driver } = run;
if (!opts.initialPlanContent) await driver.sleep(Math.min(8000, Math.max(0, run.deadlineAt - driver.now())));
if (opts.initialPlanContent) {
const seed = `Keep this draft plan as context. Briefly acknowledge receipt, then wait for my next message containing a slash command. Do not start the review or call tools yet.\n\n${opts.initialPlanContent}`;
try {
const publicTools: NativePublicToolEvent[] = [];
const transcript = session.hermeticConfigDir ? readPlanCountTranscript(session.hermeticConfigDir,
path.resolve(opts.cwd ?? process.cwd()), event => publicTools.push(event)) : undefined;
const saved = saveSnapshot({ skillName: opts.skillName, cwd: path.resolve(opts.cwd ?? process.cwd()),
claudeConfigDir: session.hermeticConfigDir, raw: session.rawOutput(), visible: session.visibleText(),
observation: { state: 'threw', error: String(error), transcript, publicTools } });
if (saved.artifactError) console.error(`PTY artifact write failed: ${saved.artifactError}`);
} catch (captureError) { console.error(`PTY failure capture failed: ${String(captureError)}`); }
throw error;
} finally {
try { await session.close(); }
catch (error) { if (!observationFailed) throw error; }
await submitPlanSeed({...session, currentScreen: session.currentScreenFrame}, seed, {
cwd: opts.cwd ?? process.cwd(), launchedAt: run.startedAt, deadlineAt: run.deadlineAt,
isQuestionOrPermission: text => isProseAUQVisible(text) || isNumberedOptionListVisible(text) || isPermissionDialogVisible(text),
});
} catch (error) {
if (!(error instanceof PlanSeedTimeout)) throw error;
return await observationPreflightTimeout(run, session, `Plan seed submission failed: ${error.message}`);
}
}
if (driver.now() >= run.deadlineAt) return await observationPreflightTimeout(run, session, 'Boot or seed preflight exhausted the existing case budget');
run.commandStartedAt = driver.now();
run.since = session.mark();
session.send(`/${opts.skillName}\r`);
run.start = driver.now();
return undefined;
}
/** Single source for the high-water flags at EVERY return site. Hand-
* spreading them per-site already drifted once (the judge-waiting return
* omitted the prose/waiting flags); a site that forgets a must-stay-false
* flag makes `obs.flag ?? false` negative assertions pass vacuously. */
function observationFlags(run: ObservationRun, session: ClaudePtySession) {
const { opts } = run;
const flags = { proseAUQEverObserved: run.proseAUQEverObserved, waitingEverObserved: run.waitingEverObserved,
scopeGateQuestionObserved: run.scopeGateQuestionObserved, scopeGateAutoSelectObserved: run.scopeGateAutoSelectObserved,
...(run.nativeAutoDecide ? { nativeAutoDecide: run.nativeAutoDecide } : {}),
...(opts.trackTokens?.length ? { tokensObserved: run.tokensObserved } : {}) };
// Preserve the measured terminal flags and public evidence before the
// hermetic session is removed, including when a later assertion fails.
const artifacts = run.saveSnapshot({ skillName: opts.skillName,
cwd: path.resolve(opts.cwd ?? process.cwd()), claudeConfigDir: session.hermeticConfigDir,
raw: session.rawOutput(), visible: session.visibleSince(run.since),
observation: { state: 'plan_skill_observation_terminal', ...flags,
commandStartedAt: run.commandStartedAt, scopeSessionId: run.scopeSessionId, native: run.scopeTranscript, publicTools: run.scopeTools,
nativePolledAt: run.nativePolledAt, nativeScope: 'Latest owned native poll, refreshed while observing; not an exhaustive terminal journal.' } });
return { ...flags, ...artifacts };
}
const JUDGE_AFTER_MS = 60_000;
const JUDGE_INTERVAL_MS = 30_000;
async function observationTick(run: ObservationRun, session: ClaudePtySession): Promise<PtyStep<PlanSkillObservation>> {
const { opts, driver } = run;
const visible = session.visibleSince(run.since);
const ended = (summary: string): PtyStep<PlanSkillObservation> => ({ done: {
outcome: 'exited', summary, evidence: visible.slice(-2000), elapsedMs: driver.now() - run.startedAt,
...observationFlags(run, session),
} });
if (session.exited()) return ended(`claude exited (code=${session.exitCode()}) before reaching a terminal outcome`);
if (isRejectedSlashCommand(visible, `/${opts.skillName}`)) {
return ended(`claude rejected /${opts.skillName} as unknown command (skill not registered in this cwd)`);
}
const classified = classifyVisible(visible, {
strictPlanWrites: !!opts.initialPlanContent,
currentScreen: opts.initialPlanContent ? await session.currentScreen() : undefined,
});
const pendingSeededCompletion = !!opts.initialPlanContent &&
isPlanReadyVisible(visible) && classified === null;
observeHighWaterMarks(run, session, visible, pendingSeededCompletion);
if (classified) {
const obs: PlanSkillObservation = {
...classified,
evidence: visible.slice(-2000),
elapsedMs: driver.now() - run.startedAt,
...observationFlags(run, session),
};
// Capture the plan file path on any outcome where one may have been
// written. Gating only on 'plan_ready' missed two cases: (1) the
// 'asked' outcome where the model wrote a plan partway through then
// paused on a question, and (2) 'wrote_findings_before_asking' where
// the bug is precisely that the plan was written. The
// assertReviewReportAtBottom checks downstream gate on planFile
// existing, not on the outcome.
const planFile = extractPlanFilePath(visible);
if (planFile) obs.planFile = planFile;
return { done: obs };
}
// Terminal classification retains precedence (including actual questions
// and writes). Only an unclassified frame may use owned native auto-decision evidence.
if (run.scopeSessionId) {
run.nativeAutoDecide = findNativeAutoDecision(run.scopeTranscript, run.scopeTools, {
skillName: opts.skillName, sessionId: run.scopeSessionId, commandStartedAt: run.commandStartedAt, now: driver.now(), proseQuestionObserved: run.proseAUQEverObserved,
stateEvidence: run.readAutoDecisionState?.(),
});
if (run.nativeAutoDecide) return { done: {
outcome: 'auto_decided',
summary: 'owned native session completed the saved-preference auto-decision',
evidence: visible.slice(-2000), elapsedMs: driver.now() - run.startedAt,
...observationFlags(run, session),
} };
}
// LLM judge fallback: if regex detectors didn't classify and we've
// burned >60s with periodic ticks, ask Haiku "is the model waiting,
// working, or hung?" Treat 'waiting' as 'asked' (model surfaced a
// question via prose the regex couldn't reassemble). Snapshot the
// visible buffer at each judge call when GSTACK_PTY_LOG=1.
const elapsed = driver.now() - run.start;
if (elapsed > JUDGE_AFTER_MS && driver.now() - run.lastJudgeAt > JUDGE_INTERVAL_MS) {
run.lastJudgeAt = driver.now();
logPtySnapshot(visible, { testName: opts.skillName, elapsedMs: elapsed, tag: 'judge-tick' });
run.lastJudgeVerdict = judgePtyState(visible, { testName: opts.skillName });
if (run.lastJudgeVerdict.state === 'waiting' && !pendingSeededCompletion) {
run.waitingEverObserved = true;
if (opts.requireProseEvidence && !run.proseAUQEverObserved) return 'continue';
return { done: {
outcome: 'asked',
summary: `LLM judge: ${run.lastJudgeVerdict.reasoning} (state=waiting after ${Math.round(elapsed / 1000)}s)`,
evidence: visible.slice(-2000),
elapsedMs: driver.now() - run.startedAt,
...observationFlags(run, session),
} };
}
}
return 'continue';
}
/** Cheap per-tick surface tracking; each flag is once-true (high water). */
function observeHighWaterMarks(run: ObservationRun, session: ClaudePtySession, visible: string, pendingSeededCompletion: boolean): void {
const { opts, driver } = run;
if (!run.proseAUQEverObserved && !pendingSeededCompletion && isProseAUQVisible(visible)) {
run.proseAUQEverObserved = true;
logPtySnapshot(visible, {
testName: opts.skillName,
elapsedMs: driver.now() - run.start,
tag: 'prose-auq-surfaced',
});
}
// Scope-gate render tracking (same high-water shape). Full-run
// detection matters because the 2KB evidence tail usually scrolls
// past the gate render before the outcome fires.
if (!run.scopeGateQuestionObserved && isScopeGateQuestionVisible(visible)) {
run.scopeGateQuestionObserved = true;
}
if (!run.scopeGateAutoSelectObserved && isScopeGateAutoSelectVisible(visible)) {
run.scopeGateAutoSelectObserved = true;
}
// Keep reading after scope selection: an AUTO_DECIDE annotation may
// arrive later, and a prior poll cannot establish its current ownership.
if (run.scopeSessionId && opts.initialPlanContent && session.hermeticConfigDir) {
const scopeTools: NativePublicToolEvent[] = [];
run.scopeTools = scopeTools;
run.scopeTranscript = readPlanCountTranscript(session.hermeticConfigDir,
path.resolve(opts.cwd ?? process.cwd()), event => scopeTools.push(event));
run.nativePolledAt = driver.now();
if (!run.scopeGateAutoSelectObserved) run.scopeGateAutoSelectObserved = nativeSeededPlanSelection(run.scopeTranscript, run.scopeTools, {
seed: opts.initialPlanContent, skillName: opts.skillName, sessionId: run.scopeSessionId, commandStartedAt: run.commandStartedAt,
});
}
for (const t of opts.trackTokens ?? []) {
if (!run.tokensObserved[t] && visible.includes(t)) run.tokensObserved[t] = true;
}
}
/** Timeout fallback: if we observed a prose-AUQ surface OR a judge
* 'waiting' verdict at any point during the run, treat as 'asked'.
* This catches the model-surfaced-then-resumed-thinking case where
* by the time the timeout fires, the buffer has moved past the
* options into spinner state but the question DID surface earlier. */
function observationTimeout(run: ObservationRun, session: ClaudePtySession): PlanSkillObservation {
const { opts, driver, lastJudgeVerdict } = run;
const finalVisible = session.visibleSince(run.since);
if (run.proseAUQEverObserved || run.waitingEverObserved && !opts.requireProseEvidence) {
return {
outcome: 'asked',
summary:
`prose-AUQ surface observed during run (proseAUQEverObserved=${run.proseAUQEverObserved}, waitingEverObserved=${run.waitingEverObserved}); model surfaced the question and the test budget elapsed without a follow-up classification` +
(lastJudgeVerdict
? ` (last LLM judge: ${lastJudgeVerdict.state} — ${lastJudgeVerdict.reasoning})`
: ''),
evidence: finalVisible.slice(-2000),
elapsedMs: driver.now() - run.startedAt,
...observationFlags(run, session),
};
}
return {
outcome: 'timeout',
summary:
`no terminal outcome within ${run.budgetMs}ms` +
(lastJudgeVerdict
? ` (last LLM judge: state=${lastJudgeVerdict.state} — ${lastJudgeVerdict.reasoning})`
: ''),
evidence: finalVisible.slice(-2000),
elapsedMs: driver.now() - run.startedAt,
...observationFlags(run, session),
};
}
function observationFailure(run: ObservationRun, session: ClaudePtySession, error: unknown): void {
const { opts } = run;
try {
const publicTools: NativePublicToolEvent[] = [];
const transcript = session.hermeticConfigDir ? readPlanCountTranscript(session.hermeticConfigDir,
path.resolve(opts.cwd ?? process.cwd()), event => publicTools.push(event)) : undefined;
const saved = run.saveSnapshot({ skillName: opts.skillName, cwd: path.resolve(opts.cwd ?? process.cwd()),
claudeConfigDir: session.hermeticConfigDir, raw: session.rawOutput(), visible: session.visibleText(),
observation: { state: 'threw', error: String(error), transcript, publicTools } });
if (saved.artifactError) console.error(`PTY artifact write failed: ${saved.artifactError}`);
} catch (captureError) { console.error(`PTY failure capture failed: ${String(captureError)}`); }
}
test/helpers/pty/session.ts
+76 -3
View File
@@ -1,8 +1,13 @@
/**
* The runners' launch seam and clock (PtyDriver). Moved from claude-pty-runner.ts.
* Import through test/helpers/claude-pty-runner.ts from tests; pty/ modules import siblings directly.
* The one PTY session loop (runPtySession) and its launch seam (PtyDriver).
* Owns launch, the poll/deadline loop, failure capture and close/cleanup
* ordering; each runner in pty/runners/ supplies its policy as a PtySessionPlan
* (start, poll, tick, timeout, capture hooks). New runner: build a plan and
* `return runPtySession(plan)`; tests pass the fake driver from fake-session.ts.
* Size ratchet (c) binds this module. Moved from the three runner loops of the
* former claude-pty-runner.ts. Tests import through claude-pty-runner.ts.
*/
import type { ClaudePtyOptions, ClaudePtySession } from './launch';
import { launchClaudePty, type ClaudePtyOptions, type ClaudePtySession } from './launch';
/**
* The runners' launch seam: how a session starts, and the clock the runner
@@ -18,3 +23,71 @@ export interface PtyDriver {
monotonic(): number;
sleep(ms: number): Promise<void>;
}
export const realPtyDriver: PtyDriver = {
launch: opts => launchClaudePty(opts),
now: () => Date.now(),
monotonic: () => performance.now(),
sleep: ms => Bun.sleep(ms),
};
/** One poll step: finish with a result, keep polling, or end the loop as a timeout. */
export type PtyStep<R> = { done: R } | 'continue' | 'break';
export interface PtySessionPlan<R> {
driver: PtyDriver;
launch: ClaudePtyOptions;
/** Owned fixture cleanup; runs after close, and when launch itself fails. */
cleanup?: () => void;
/** Boot and command submission. A returned result ends the run before polling. */
start(session: ClaudePtySession): Promise<R | undefined>;
/** Wait for the next poll. False means the deadline passed: the run times out. */
poll(session: ClaudePtySession): Promise<boolean>;
tick(session: ClaudePtySession): Promise<PtyStep<R>>;
timeout(session: ClaudePtySession): R;
/** Snapshot a thrown run before close; the original error is rethrown. */
onError?(session: ClaudePtySession, error: unknown): void;
/** Always runs before close. A throw here fails an otherwise successful run. */
beforeClose?(session: ClaudePtySession, failure: { error: unknown } | undefined): void;
/** Close failed after a successful run; the close error is rethrown. */
onCloseError?(session: ClaudePtySession, error: unknown): void;
}
/**
* launch → start → (poll → tick)* → timeout, with one failure contract: a
* run's own error wins over capture and close errors, close always runs, and
* owned cleanup runs last.
*/
export async function runPtySession<R>(plan: PtySessionPlan<R>): Promise<R> {
let session: ClaudePtySession;
try {
session = await plan.driver.launch(plan.launch);
} catch (error) {
plan.cleanup?.();
throw error;
}
let failure: { error: unknown } | undefined;
try {
const early = await plan.start(session);
if (early !== undefined) return early;
while (await plan.poll(session)) {
const step = await plan.tick(session);
if (step === 'break') break;
if (step !== 'continue') return step.done;
}
return plan.timeout(session);
} catch (error) {
failure = { error };
plan.onError?.(session, error);
throw error;
} finally {
try {
try { plan.beforeClose?.(session, failure); }
catch (error) { if (!failure) { failure = { error }; throw error; } }
finally {
try { await session.close(); }
catch (error) { if (!failure) { plan.onCloseError?.(session, error); throw error; } }
}
} finally { plan.cleanup?.(); }
}
}
test/plan-count-completion.test.ts
+3 -7
View File
@@ -9,7 +9,7 @@ import type { PlanCountTranscript } from './helpers/plan-count-transcript';
import capturedL from './fixtures/devex-review-l-calls.json';
import designStatusCapture from './fixtures/design-count-native-issue-fields.json';
import designEnvelope from './fixtures/design-completion-envelope-90f.json';
import { nativePlanCallFingerprint } from './helpers/claude-pty-runner';
import { nativePlanCallFingerprint, isNativeCompletionSummary } from './helpers/claude-pty-runner';
import type { NativePlanQuestionCall } from './helpers/plan-count-transcript';
import captured_ceo_completion_handoff_m from './fixtures/ceo-completion-handoff-m-call.json';
import nextStepCapture_ceo_completion_handoff_m from './fixtures/ceo-handoff-n-calls.json';
@@ -48,13 +48,9 @@ describe('captured Design completion envelope', () => {
});
test('the actual runner branch recognizes the idle native envelope without answering an exit gate',()=>{
const f=completedEnvelope();try {
const source=fs.readFileSync(path.join(import.meta.dir,'helpers/pty/runners/counting.ts'),'utf8');
const expression=/const nativeSummary = ([\s\S]+?);\n const terminalFrame/.exec(source)?.[1];expect(expression).toBeTruthy();
const evaluate=new Function('opts','nativeCompletion','renderedFrame','visible','transcript','startedAt','administrative',
'isNumberedOptionListVisible','isPermissionDialogVisible','isProseAUQVisible','hasNativePlanTerminal',`return (${expression});`);
const screen=designEnvelope.screen.replaceAll(designEnvelope.originalPlanPath,f.file);
const check=()=>evaluate({expectedPlanPath:f.file},hasNativePlanCompletion(f.transcript,f.file,designEnvelope.startedAt),classifyPlanCountFrame(screen),screen,f.transcript,
designEnvelope.startedAt,new Set(),isNumberedOptionListVisible,isPermissionDialogVisible,isProseAUQVisible,hasNativePlanTerminal);
const check=()=>isNativeCompletionSummary({expectedPlanPath:f.file,nativeCompletion:hasNativePlanCompletion(f.transcript,f.file,designEnvelope.startedAt),
renderedFrame:classifyPlanCountFrame(screen),visible:screen,transcript:f.transcript,startedAt:designEnvelope.startedAt,administrative:new Set()});
expect(check()).toBe(true);
f.transcript.calls[0]!.answered=false;expect(check()).toBe(false);
}finally{f.cleanup();}
test/plan-create-prepublication.test.ts
+7 -11
View File
@@ -5,7 +5,7 @@ import * as path from 'node:path';
import {createHash} from 'node:crypto';
import {createFilePermissionRecorder,recordFilePermission,currentFilePermissionEpoch,readPendingWriteInput} from './helpers/plan-count-file-permission';
import {readPlanCountTranscript} from './helpers/plan-count-transcript';
import {createPlanCountPermissionGuard} from './helpers/claude-pty-runner';
import {createPlanCountPermissionGuard,countingCapture,type CountingRun} from './helpers/claude-pty-runner';
import {createPlanCountSnapshotWriter} from './helpers/plan-count-artifacts';
import captured from './fixtures/plan-create-prepublication-491.json';
@@ -137,22 +137,18 @@ test('legacy published Write metadata remains sufficient without a sidecar',()=>
});
// Execute only the actual count capture assembly with its real reader/writer.
// This is not a full counting-loop execution; existing count throw/timeout tests
// This calls the counting runner's capture step directly, not a full loop; existing count throw/timeout tests
// separately prove that the lifecycle invokes capture before fixture disposal.
test('count capture assembly retains exact prepublication input through refresh, throw and fixture cleanup',()=>{
const f=fixture(),evalDir=fs.mkdtempSync(path.join(os.tmpdir(),'count-write-retention-'));
try {
f.hook();const expected=readPendingWriteInput(f.recorder.file,f.expected,f.cwd,f.config,f.startedAt);
expect(expected).toBeDefined();
const source=fs.readFileSync(path.join(import.meta.dir,'helpers/pty/runners/counting.ts'),'utf8');
const start=source.indexOf(' const capture = (observation: object) => {',source.indexOf('export async function runPlanSkillCounting('));
const end=source.indexOf('\n function snapshot(',start);
expect(start).toBeGreaterThan(0);expect(end).toBeGreaterThan(start);
const code=new Bun.Transpiler({loader:'ts'}).transformSync(source.slice(start,end)+'\nreturn capture;');
const capture=new Function('saveSnapshot','opts','ownedFilePermissions','readPendingWriteInput','readPlanCountTranscript','fixture','session','startedAt','viewport',code)(
createPlanCountSnapshotWriter({EVALS_RUN_ID:'count-prepublication-free',GSTACK_EVAL_DIR:evalDir}),
{skillName:'plan-eng-review'},[{file:f.recorder.file,expected:f.expected}],readPendingWriteInput,readPlanCountTranscript,
{cwd:f.cwd},{hermeticConfigDir:f.config,rawOutput:()=>f.screen,visibleText:()=>f.screen},f.startedAt,f.screen);
// The runner's own capture step, called with the state it builds for an attempt.
const run={opts:{skillName:'plan-eng-review'},fixture:{cwd:f.cwd},startedAt:f.startedAt,viewport:f.screen,
ownedFilePermissions:[{file:f.recorder.file,expected:f.expected}],
saveSnapshot:createPlanCountSnapshotWriter({EVALS_RUN_ID:'count-prepublication-free',GSTACK_EVAL_DIR:evalDir})} as unknown as CountingRun;
const capture=(observation:object)=>countingCapture(run,{hermeticConfigDir:f.config,rawOutput:()=>f.screen,visibleText:()=>f.screen},observation);
const progress=capture({state:'in_progress'});
expect(progress.artifactError).toBeUndefined();expect(progress.artifactDir).toBeDefined();
const initial=JSON.parse(fs.readFileSync(path.join(progress.artifactDir,'observation.json'),'utf8'));
test/plan-floor-permission.test.ts
+2 -1
View File
@@ -259,13 +259,14 @@ async function exercise(mode: Mode, kind: keyof typeof SEEDS = 'ceo', capture?:
},
};
const run = new Function(...Object.keys(boundary), body + '\nreturn runPlanSkillFloorCheck;')(...Object.values(boundary));
const driver: runner.PtyDriver = {launch: boundary.launchClaudePty, now: () => now, monotonic: () => now, sleep: boundary.Bun.sleep};
try {
let result: any, error: unknown;
try {
result = await run({skillName:`plan-${kind}-review`, slashCommand:`/plan-${kind}-review`, followUpPrompt:SEEDS[kind],
productType:mode==='product-type'||mode.startsWith('dx-')?'sdk-documentation':undefined,
devexSetupContext:mode.startsWith('dx-')&&mode!=='dx-undeclared'?dxCustom.reply:undefined,
requestedPlanPath:mode === 'captured' ? undefined : `/tmp/gstack-test-plan-${kind}-floor.md`, timeoutMs:100_000});
requestedPlanPath:mode === 'captured' ? undefined : `/tmp/gstack-test-plan-${kind}-floor.md`, timeoutMs:100_000, driver});
} catch(caught) { if(!snapshotOptions?.interrupt)throw caught; error=caught; }
expect(closed).toBe(1); expect(fs.existsSync(fixture!.cwd)).toBe(false);
expect(recorders.every(recorder=>!fs.existsSync(recorder.file))).toBe(true);