diff --git a/test/eng-seeded-completion-ai.test.ts b/test/eng-seeded-completion-ai.test.ts index 05bf944a0..feff4a337 100644 --- a/test/eng-seeded-completion-ai.test.ts +++ b/test/eng-seeded-completion-ai.test.ts @@ -8,6 +8,7 @@ import { fakePlanSeedPrelude } from './helpers/fake-plan-seed'; import fixture from './fixtures/eng-seeded-completion-ai.json'; import { classifyVisible, extractPlanFilePath } from './helpers/claude-pty-runner'; import * as predicates from './helpers/claude-pty-runner'; +import type { PtyDriver } from './helpers/claude-pty-runner'; const gate = '─────\nClaude has written up a plan and is ready to execute. Would you like to proceed?\n❯ 1. Yes, and use auto mode\n2. Yes, manually approve edits\n3. Tell Claude what to change'; const compactGate = 'Exit plan mode?\nClaude wants to exit plan mode\n❯ 1. Yes, and switch to default (ask each time) for this session\n2. No'; const question = 'Which runner should the plan use?\nA) Use the built-in runner\nB) Build a custom runner\nRecommendation: A because it avoids duplicate scheduling logic.\nReply with A or B.'; @@ -141,9 +142,12 @@ async function mockedObservation(frames: string[], verdict: 'waiting' | 'working isScopeGateAutoSelectVisible: predicates.isScopeGateAutoSelectVisible, classifyVisible, extractPlanFilePath, findNativeAutoDecision: () => null, judgePtyState: () => { judged++; return { state: verdict, reasoning: 'synthetic current-frame verdict' }; }, + runPtySession: predicates.runPtySession, }; const run = new Function(...Object.keys(args), js)(...Object.values(args)); - const obs = await run({ skillName: 'plan-eng-review', timeoutMs: 70000, + const clockArgs = args as { Date: { now(): number }; Bun: { sleep(ms: number): Promise }; launchClaudePty: PtyDriver['launch'] }; + const driver: PtyDriver = { launch: clockArgs.launchClaudePty, now: clockArgs.Date.now, monotonic: clockArgs.Date.now, sleep: clockArgs.Bun.sleep }; + const obs = await run({ skillName: 'plan-eng-review', timeoutMs: 70000, driver, ...(seeded ? { initialPlanContent: '# Plan: Required draft' } : {}) }); expect(closed).toBe(1); return { obs, judged, seedSubmittedAt }; diff --git a/test/helpers/claude-pty-runner.ts b/test/helpers/claude-pty-runner.ts index 14e0ca54a..847d712a4 100644 --- a/test/helpers/claude-pty-runner.ts +++ b/test/helpers/claude-pty-runner.ts @@ -11,7 +11,8 @@ export { resolveClaudeBinary } from './pty/binary'; export { stripAnsi, isRejectedSlashCommand, isPlanReadyVisible, isAutoDecidedVisible, extractPlanFilePath, planFileHasDecisionsSection, TAIL_SCAN_BYTES, isPermissionDialogVisible, stripPtyResidue, isNumberedOptionListVisible } from './pty/screen'; export { selectPtyNumberedOption, launchClaudePty } from './pty/launch'; export type { ClaudePtyOptions, ClaudePtySession } from './pty/launch'; -export type { PtyDriver } from './pty/session'; +export { runPtySession, realPtyDriver } from './pty/session'; +export type { PtyDriver, PtyStep, PtySessionPlan } from './pty/session'; export { logPtySnapshot, judgePtyState } from './pty/judge'; export type { PtyStateVerdict } from './pty/judge'; export { isProseAUQVisible, isScopeGateQuestionVisible, isScopeGateAutoSelectVisible, parseNumberedOptions, MODE_RE, optionsSignature, classifyVisible, COMPLETION_SUMMARY_RE, classifyPlanCountFrame, planCountSubmissionInput } from './pty/classify'; @@ -22,8 +23,8 @@ export { assertReviewReportAtBottom, hasNativePlanCompletion, isQuestionlessNati export type { ReviewReportAtBottomResult, NativePlanTerminalReview, NativePlanTerminalAssessment, NativePlanTerminalEvaluator } from './pty/plan-native'; export { ceoStep0Boundary, engSetupAUQ, engFirstReviewAUQ, engStep0Boundary, designReviewSetupAUQ } from './pty/boundaries'; export { runPlanSkillObservation } from './pty/runners/observation'; -export type { PlanSkillObservation } from './pty/runners/observation'; -export { runPlanSkillCounting } from './pty/runners/counting'; -export type { PlanSkillCountObservation } from './pty/runners/counting'; +export type { PlanSkillObservation, PlanSkillObservationOptions } from './pty/runners/observation'; +export { runPlanSkillCounting, countingCapture, isNativeCompletionSummary } from './pty/runners/counting'; +export type { PlanSkillCountObservation, PlanSkillCountingOptions, CountingRun } from './pty/runners/counting'; export { planFloorDXPane, planFloorDXReplyInput, runPlanSkillFloorCheck } from './pty/runners/floor'; export type { PlanSkillFloorObservation, PlanFloorDXReply } from './pty/runners/floor'; diff --git a/test/helpers/pty/runners/counting.ts b/test/helpers/pty/runners/counting.ts index bf7ebcea2..d3a04c737 100644 --- a/test/helpers/pty/runners/counting.ts +++ b/test/helpers/pty/runners/counting.ts @@ -14,12 +14,12 @@ import { stripVTControlCharacters } from 'node:util'; import { capturePlanCountQuestion, createPlanCountPermissionGuard, matchesNativePlanQuestion, nativePlanCallFingerprint, planCountPrerequisitePick, planCountQuestionInput, planCountQuestionPhase } from '../auq'; import type { AskUserQuestionFingerprint, Step0BoundaryPredicate } from '../auq'; import { SANCTIONED_WRITE_SUBSTRINGS, classifyPlanCountFrame, isProseAUQVisible, planCountSubmissionInput } from '../classify'; -import { launchClaudePty, selectPtyNumberedOption } from '../launch'; +import { selectPtyNumberedOption } from '../launch'; import type { ClaudePtySession } from '../launch'; import { evaluateOwnedNativePlanTerminal, hasNativePlanCompletion, hasNativePlanTerminal, isQuestionlessNativePlanExit } from '../plan-native'; import type { NativePlanTerminalEvaluator } from '../plan-native'; import { isNumberedOptionListVisible, isPermissionDialogVisible, isRejectedSlashCommand } from '../screen'; -import type { PtyDriver } from '../session'; +import { realPtyDriver, runPtySession, type PtyDriver, type PtyStep } from '../session'; // ──────────────────────────────────────────────────────────────────────────── // runPlanSkillCounting — drives a plan-* skill end-to-end through Step 0 then @@ -64,51 +64,8 @@ export interface PlanSkillCountObservation { administrativeCount: number; } -/** - * Drive a plan-* skill in plan mode and count distinct native review-phase - * AskUserQuestions until a terminal signal fires. Each run disables the - * extra outside review in its own gstack config: independent reviewers can - * add valid findings unrelated to the seeded-N cadence band. These counts - * do not assert outside-review dispatch or its approval-question cadence. - * - * Flow: - * 1. Seed the complete fixture request in an isolated git repository's - * PLAN.md and initial CLAUDE.md context, with owned native-only config, - * then boot the PTY in that cwd - * (8s grace + auto-trust dialog). Skills remain registered in user scope. - * 2. Send `slashCommand` alone. The fixture is already in context; sending - * it later can queue it behind the skill's first question while the - * review incorrectly starts against the operator's live branch. - * 3. Poll loop: - * - Skip permission dialogs (auto-grant with `defaultPick`). - * - Read fixture-scoped native JSONL. Count each AskUserQuestion call - * once after its matching successful answer record, regardless of - * how many questions the call batches. Full native metadata feeds - * phase predicates; ANSI redraws and permissions cannot add counts. - * - On a new numbered-option list, keep the existing PTY answer driver. - * Decline only the recognized optional office-hours - * prerequisite by label so a different skill does not rewrite the - * seeded plan before this review begins. - * - After the native answer, evaluate `isLastStep0AUQ(fingerprint)`. If true, - * subsequent AUQs are review-phase unless the caller positively identifies native setup. - * - Hard ceiling: if `reviewCount >= reviewCountCeiling`, return - * `ceiling_reached`. This bounds runaway counts; tests should set - * the ceiling above their assertion CEILING. - * - Soft terminals: `COMPLETION_SUMMARY_RE` match → `completion_summary`; - * plan-ready confirmation → `plan_ready`; silent write outside - * sanctioned dirs → `silent_write`; process exited → `exited`; - * wall clock exceeded → `timeout`. - * - * Boundary detection (D14): event-based, fired against the answered AUQ's - * fingerprint, not against later rendered content. This avoids the race - * where Step-0-final and Section-1-first AUQs straddle a section header - * regex match. - * - * UI fingerprints still dedupe redraws. Counted fingerprints use the - * native session/tool call IDs, so shared answer labels never collapse - * different calls and one multi-question call remains one finding. - */ -export async function runPlanSkillCounting(opts: { +/** Options for runPlanSkillCounting. */ +export interface PlanSkillCountingOptions { /** Skill name, e.g. 'plan-ceo-review'. Used for diagnostic strings only. */ skillName: string; /** Slash command to send alone, e.g. '/plan-ceo-review'. No trailing args. */ @@ -185,9 +142,54 @@ export async function runPlanSkillCounting(opts: { model?: string; /** Launch seam and clock; tests pass the fake driver. Default: real launcher and clocks. */ driver?: PtyDriver; -}): Promise { - const driver = opts.driver ?? { launch: launchClaudePty, now: () => Date.now(), - monotonic: () => performance.now(), sleep: (ms: number) => Bun.sleep(ms) }; +} + +/** + * Drive a plan-* skill in plan mode and count distinct native review-phase + * AskUserQuestions until a terminal signal fires. Each run disables the + * extra outside review in its own gstack config: independent reviewers can + * add valid findings unrelated to the seeded-N cadence band. These counts + * do not assert outside-review dispatch or its approval-question cadence. + * + * Flow: + * 1. Seed the complete fixture request in an isolated git repository's + * PLAN.md and initial CLAUDE.md context, with owned native-only config, + * then boot the PTY in that cwd + * (8s grace + auto-trust dialog). Skills remain registered in user scope. + * 2. Send `slashCommand` alone. The fixture is already in context; sending + * it later can queue it behind the skill's first question while the + * review incorrectly starts against the operator's live branch. + * 3. Poll loop: + * - Skip permission dialogs (auto-grant with `defaultPick`). + * - Read fixture-scoped native JSONL. Count each AskUserQuestion call + * once after its matching successful answer record, regardless of + * how many questions the call batches. Full native metadata feeds + * phase predicates; ANSI redraws and permissions cannot add counts. + * - On a new numbered-option list, keep the existing PTY answer driver. + * Decline only the recognized optional office-hours + * prerequisite by label so a different skill does not rewrite the + * seeded plan before this review begins. + * - After the native answer, evaluate `isLastStep0AUQ(fingerprint)`. If true, + * subsequent AUQs are review-phase unless the caller positively identifies native setup. + * - Hard ceiling: if `reviewCount >= reviewCountCeiling`, return + * `ceiling_reached`. This bounds runaway counts; tests should set + * the ceiling above their assertion CEILING. + * - Soft terminals: `COMPLETION_SUMMARY_RE` match → `completion_summary`; + * plan-ready confirmation → `plan_ready`; silent write outside + * sanctioned dirs → `silent_write`; process exited → `exited`; + * wall clock exceeded → `timeout`. + * + * Boundary detection (D14): event-based, fired against the answered AUQ's + * fingerprint, not against later rendered content. This avoids the race + * where Step-0-final and Section-1-first AUQs straddle a section header + * regex match. + * + * UI fingerprints still dedupe redraws. Counted fingerprints use the + * native session/tool call IDs, so shared answer labels never collapse + * different calls and one multi-question call remains one finding. + */ +export async function runPlanSkillCounting(opts: PlanSkillCountingOptions): Promise { + const driver = opts.driver ?? realPtyDriver; if (opts.requireNativePicker && !opts.pickAUQ) throw Error('Native picker binding requires a declared picker'); if (opts.bindDesignBoardState && (opts.skillName !== 'plan-design-review' || !opts.pickAUQ)) @@ -198,7 +200,6 @@ export async function runPlanSkillCounting(opts: { throw Error('Collection-only completion cannot replace the final report contract'); const budgetStarted = driver.monotonic(); const startedAt = driver.now(); - const defaultPick = opts.defaultPick ?? 1; const timeoutMs = opts.timeoutMs ?? 1_500_000; if (opts.startupReadyMarker !== undefined && !opts.startupReadyMarker.length) { throw new RangeError('Plan counting startup-ready marker must not be empty'); @@ -211,32 +212,30 @@ export async function runPlanSkillCounting(opts: { throw new RangeError('Plan counting timeout must exceed the 5000ms cleanup reserve'); } const workDeadline = budgetStarted + timeoutMs - cleanupReserveMs; - const remainingWork = () => Math.max(0, workDeadline - driver.monotonic()); - async function waitForWork(ms: number): Promise { - const remaining = remainingWork(); - if (remaining <= 0) return false; - const clipped = ms >= remaining; - await driver.sleep(Math.min(ms, remaining)); - // A clipped wait cannot finish the requested interval. Timers may wake - // just before the fractional deadline; that is no license to advance. - return !clipped && remainingWork() > 0; - } - const fixture = createPlanCountFixture(opts.followUpPrompt, { nativeReviewOnly: true, files: opts.fixtureFiles, preconfiguredReviewActor: opts.preconfiguredReviewActor }); - const pickerContext = Object.freeze({cwd: fixture.cwd, deadlineAt: startedAt + timeoutMs - cleanupReserveMs}); + const run: CountingRun = { + opts, driver, startedAt, timeoutMs, cleanupReserveMs, workDeadline, fixture, defaultPick: opts.defaultPick ?? 1, + pickerContext: Object.freeze({cwd: fixture.cwd, deadlineAt: startedAt + timeoutMs - cleanupReserveMs}), + saveSnapshot: createPlanCountSnapshotWriter(), fingerprints: [], planningDirectory: undefined, + seen: new Set(), countedCalls: new Set(), filePermission: createPlanCountPermissionGuard(), ownedFilePermissions: [], + lastMatchedNativeQuestion: undefined, transcript: { status: 'missing', calls: [], assistantMessages: [] }, + boundaryFired: false, step0Count: 0, reviewCount: 0, administrativeCount: 0, isFirstAUQ: true, + lastCheckpointAt: 0, viewport: '', observedOutput: 0, lastObservationAt: -Infinity, + }; const permissionPaths = [ ...(opts.expectedPlanPath ? [opts.expectedPlanPath, path.join(fixture.cwd, 'PLAN.md')] : []), ...(opts.permissionPlanPath ? [opts.permissionPlanPath] : []), ]; - let session: ClaudePtySession; - try { - session = await driver.launch({ + return runPtySession({ + driver, + cleanup: () => fixture.cleanup(), + launch: { permissionMode: 'plan', cwd: fixture.cwd, // Stop new output at the work cutoff so screen drain cannot consume // the reserve while the CLI continues streaming. - timeoutMs: Math.max(1, remainingWork()), + timeoutMs: Math.max(1, remainingWork(run)), screenDeadlineAt: workDeadline, env: { ...opts.env, ...fixture.env, // The renderer may cd into its artifact directory before starting the daemon. @@ -249,408 +248,475 @@ export async function runPlanSkillCounting(opts: { observeSetupQuestions: opts.observeSetupQuestions, observeFilePermissions: permissionPaths.length ? [...new Set(permissionPaths)] : undefined, ...(opts.approveEngTestPlanEdits ? { observeAutoplanArtifacts: true, approveAutoplanArtifactEdits: true, engTestPlanArtifactOnly: true } : {}), - }); - } catch (error) { - fixture.cleanup(); - throw error; - } + }, + start: session => countingStart(run, session), + poll: session => countingPoll(run, session), + tick: session => countingTick(run, session), + timeout: session => countingSnapshot(run, session, 'timeout', + `no terminal outcome within ${timeoutMs}ms total budget (including startup and ${cleanupReserveMs}ms cleanup reserve; step0=${run.step0Count}, review=${run.reviewCount})`, + run.viewport), + onError: (session, error) => countingFailure(run, session, error), + onCloseError: (session, error) => { + countingCapture(run, session, { state: 'cleanup_failed', error: String(error), transcript: run.transcript, fingerprints: run.fingerprints }); + }, + }); +} - const fingerprints: AskUserQuestionFingerprint[] = []; - const planningDirectory = session.hermeticConfigDir ? path.join(session.hermeticConfigDir, 'plans') : undefined; - const seen = new Set(); - const countedCalls = new Set(); - const filePermission = createPlanCountPermissionGuard(); +/** Mutable state of one counting attempt, shared by its session-loop steps. */ +export interface CountingRun { + opts: PlanSkillCountingOptions; + driver: PtyDriver; + startedAt: number; + timeoutMs: number; + cleanupReserveMs: number; + /** Monotonic work cutoff: budget start + timeout - cleanup reserve. */ + workDeadline: number; + fixture: { cwd: string }; + defaultPick: number; + pickerContext: Readonly<{ cwd: string; deadlineAt: number }>; + saveSnapshot: ReturnType; + fingerprints: AskUserQuestionFingerprint[]; + planningDirectory: string | undefined; + seen: Set; + countedCalls: Set; + filePermission: ReturnType; // Each owned path keeps its own grant history when the workflow switches // between the active plan and the separate caller-owned final report. - const ownedFilePermissions = (session.pendingFilePermissionFiles ?? []).map(binding => - ({ ...binding, guard: createPlanCountPermissionGuard() })); - let lastMatchedNativeQuestion: NativePlanQuestionCall | undefined; - let transcript: PlanCountTranscript = { status: 'missing', calls: [], assistantMessages: [] }; - let boundaryFired = false; - let step0Count = 0; - let reviewCount = 0; - let administrativeCount = 0; - let isFirstAUQ = true; - const saveSnapshot = createPlanCountSnapshotWriter(); - let lastCheckpointAt = driver.now(); - let viewport = ''; + ownedFilePermissions: Array<{ expected: string; file: string; guard: ReturnType }>; + lastMatchedNativeQuestion: NativePlanQuestionCall | undefined; + transcript: PlanCountTranscript; + boundaryFired: boolean; + step0Count: number; + reviewCount: number; + administrativeCount: number; + isFirstAUQ: boolean; + lastCheckpointAt: number; + viewport: string; + observedOutput: number; + lastObservationAt: number; +} - const capture = (observation: object) => { - const publicTools: NativePublicToolEvent[] = []; - if (session.hermeticConfigDir) readPlanCountTranscript(session.hermeticConfigDir, fixture.cwd, - event => publicTools.push(event)); - return saveSnapshot({ - skillName: opts.skillName, observation: { ...observation, - publicTools, - pendingWriteInputs: ownedFilePermissions.flatMap(binding => { - const input = readPendingWriteInput(binding.file, binding.expected, fixture.cwd, session.hermeticConfigDir, startedAt); - return input ? [input] : []; - }) }, raw: session.rawOutput(), visible: session.visibleText(), viewport, - cwd: fixture.cwd, claudeConfigDir: session.hermeticConfigDir, - }); +function remainingWork(run: CountingRun): number { + return Math.max(0, run.workDeadline - run.driver.monotonic()); +} + +async function waitForWork(run: CountingRun, ms: number): Promise { + const remaining = remainingWork(run); + if (remaining <= 0) return false; + const clipped = ms >= remaining; + await run.driver.sleep(Math.min(ms, remaining)); + // A clipped wait cannot finish the requested interval. Timers may wake + // just before the fractional deadline; that is no license to advance. + return !clipped && remainingWork(run) > 0; +} + +/** Durable observation with public tool events and pending owned write inputs. */ +export function countingCapture(run: CountingRun, session: Pick, + observation: object) { + const publicTools: NativePublicToolEvent[] = []; + if (session.hermeticConfigDir) readPlanCountTranscript(session.hermeticConfigDir, run.fixture.cwd, + event => publicTools.push(event)); + return run.saveSnapshot({ + skillName: run.opts.skillName, observation: { ...observation, + publicTools, + pendingWriteInputs: run.ownedFilePermissions.flatMap(binding => { + const input = readPendingWriteInput(binding.file, binding.expected, run.fixture.cwd, session.hermeticConfigDir, run.startedAt); + return input ? [input] : []; + }) }, raw: session.rawOutput(), visible: session.visibleText(), viewport: run.viewport, + cwd: run.fixture.cwd, claudeConfigDir: session.hermeticConfigDir, + }); +} + +function countingSnapshot( + run: CountingRun, + session: ClaudePtySession, + outcome: PlanSkillCountObservation['outcome'], + summary: string, + visible: string, +): PlanSkillCountObservation { + const clean = (text: string) => stripVTControlCharacters(text) + .replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, ''); + const failed = outcome === 'exited' || outcome === 'timeout'; + const observation: PlanSkillCountObservation = { + outcome, + summary, + evidence: failed + ? `exitCode=${session.exitCode()}\n--- post-command evidence (last 3KB) ---\n${clean(visible).slice(-3000)}` + + `\n--- full-session evidence, including startup (last 6KB) ---\n${clean(session.visibleText()).slice(-6000)}` + : visible.slice(-3000), + elapsedMs: run.driver.now() - run.startedAt, + fingerprints: run.fingerprints, + transcript: run.transcript, + step0Count: run.step0Count, + reviewCount: run.reviewCount, + administrativeCount: run.administrativeCount, }; + const artifacts = countingCapture(run, session, observation); + Object.assign(observation, artifacts); + if (artifacts.artifactDir) observation.evidence += `\nFull PTY artifacts: ${artifacts.artifactDir}`; + if (artifacts.artifactError) observation.evidence += `\nPTY artifact write failed: ${artifacts.artifactError}`; + return observation; +} - function snapshot( - outcome: PlanSkillCountObservation['outcome'], - summary: string, - visible: string, - ): PlanSkillCountObservation { - const clean = (text: string) => stripVTControlCharacters(text) - .replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, ''); - const failed = outcome === 'exited' || outcome === 'timeout'; - const observation: PlanSkillCountObservation = { - outcome, - summary, - evidence: failed - ? `exitCode=${session.exitCode()}\n--- post-command evidence (last 3KB) ---\n${clean(visible).slice(-3000)}` + - `\n--- full-session evidence, including startup (last 6KB) ---\n${clean(session.visibleText()).slice(-6000)}` - : visible.slice(-3000), - elapsedMs: driver.now() - startedAt, - fingerprints, - transcript, - step0Count, - reviewCount, - administrativeCount, - }; - const artifacts = capture(observation); - Object.assign(observation, artifacts); - if (artifacts.artifactDir) observation.evidence += `\nFull PTY artifacts: ${artifacts.artifactDir}`; - if (artifacts.artifactError) observation.evidence += `\nPTY artifact write failed: ${artifacts.artifactError}`; - return observation; +async function countingStart(run: CountingRun, session: ClaudePtySession): Promise { + const { opts } = run; + run.planningDirectory = session.hermeticConfigDir ? path.join(session.hermeticConfigDir, 'plans') : undefined; + run.ownedFilePermissions = (session.pendingFilePermissionFiles ?? []).map(binding => + ({ ...binding, guard: createPlanCountPermissionGuard() })); + run.lastCheckpointAt = run.driver.now(); + run.observedOutput = session.mark(); + let startupReady: boolean; + if (opts.startupReadyMarker !== undefined) { + await session.waitFor(opts.startupReadyMarker, { timeoutMs: Math.min(8000, remainingWork(run)) }); + startupReady = remainingWork(run) > 0; + } else { + startupReady = await waitForWork(run, 8000); } - - let observedOutput = session.mark(); - let lastObservationAt = -Infinity; - let countingFailed = false; - try { - let startupReady: boolean; - if (opts.startupReadyMarker !== undefined) { - await session.waitFor(opts.startupReadyMarker, { timeoutMs: Math.min(8000, remainingWork()) }); - startupReady = remainingWork() > 0; - } else { - startupReady = await waitForWork(8000); - } - if (startupReady) { - observedOutput = session.mark(); - if (opts.approveEngTestPlanEdits) { - if (!session.startAutoplanArtifactEditApproval) throw Error('Owned Eng test-plan approval hook unavailable'); - session.startAutoplanArtifactEditApproval(driver.now()); - } - session.send(`${opts.slashCommand}\r`); + if (startupReady) { + run.observedOutput = session.mark(); + if (opts.approveEngTestPlanEdits) { + if (!session.startAutoplanArtifactEditApproval) throw Error('Owned Eng test-plan approval hook unavailable'); + session.startAutoplanArtifactEditApproval(run.driver.now()); } + session.send(`${opts.slashCommand}\r`); + } + return undefined; +} - while (remainingWork() > 0) { - await session.waitForOutput(observedOutput, Math.min(2000, remainingWork())); - if (remainingWork() <= 0) break; - const coalesceMs = session.rawOutput().length > observedOutput - ? 250 : 250 - (driver.monotonic() - lastObservationAt); - if (coalesceMs > 0 && !await waitForWork(coalesceMs)) break; - observedOutput = session.mark(); - lastObservationAt = driver.monotonic(); - const visible = viewport = await session.currentScreen(); - if (remainingWork() <= 0) break; - transcript = session.hermeticConfigDir - ? readPlanCountTranscript(session.hermeticConfigDir, fixture.cwd) - : { status: 'error', calls: [], assistantMessages: [], error: 'Claude count session has no isolated transcript directory' }; - transcript = withPendingExit(transcript, session.pendingPlanReadyFile, fixture.cwd, - session.hermeticConfigDir, startedAt, visible); - if (opts.approveEngTestPlanEdits) { - const status = autoplanArtifactRecorderStatus(session.pendingAutoplanArtifactFile, fixture.cwd, - session.hermeticConfigDir, session.autoplanArtifactStateRoot); - const boundary = autoplanArtifactApprovalBoundary(status); - if (boundary === 'failed') return snapshot('artifact_permission_failed', - `Owned Eng QA test-plan approval failed: ${JSON.stringify(status)}`, visible); - // Native approval owns this Edit. Never answer its repaint or count a - // metadata-only pending request; resume only after its actual result. - if (boundary === 'pending') { - if (driver.now() - lastCheckpointAt >= 30_000) { - lastCheckpointAt = driver.now(); - const saved = capture({ state: 'artifact_pending', elapsedMs: driver.now() - startedAt, - fingerprints, step0Count, reviewCount, administrativeCount, transcript, artifactStatus: status }); - if (saved.artifactError) console.error(`PTY artifact write failed: ${saved.artifactError}`); - } - continue; - } - } - if (transcript.status === 'error') { - return snapshot('transcript_unavailable', transcript.error!, visible); - } - for (const [callIndex, call] of transcript.calls.entries()) { - const signature = `${call.sessionId}:${call.toolUseId}`; - if (!call.answered || countedCalls.has(signature)) continue; - const fp = nativePlanCallFingerprint(call, driver.now() - startedAt, !boundaryFired); - const phase = planCountQuestionPhase(fp, boundaryFired, opts.isLastStep0AUQ, opts.isFirstReviewAUQ, opts.isSetupAUQ, opts.isCompletionHandoffAUQ, opts.isArtifactGenerationAUQ); - if (phase.administrative) { - fp.preReview = false; - fp.administrative = phase.administrative; - administrativeCount += 1; - } else { - fp.preReview = opts.isReviewAUQ ? !opts.isReviewAUQ(fp, transcript.calls.slice(0, callIndex)) : phase.preReview; - if (fp.preReview) step0Count += 1; - else reviewCount += 1; - } - fp.promptSnippet = fp.promptSnippet.replace(/\s+/g, ' ').slice(0, 240); - fingerprints.push(fp); - countedCalls.add(signature); - boundaryFired = phase.reviewStarted; - } - if (reviewCount >= opts.reviewCountCeiling) { - if (unresolvedPlanQuestionCalls(transcript.calls).length) { - return snapshot('transcript_unavailable', 'Question count reached its ceiling with unresolved failed native calls', visible); - } - return snapshot('ceiling_reached', `review-phase AUQ count reached ceiling (${opts.reviewCountCeiling})`, visible); - } - // An outer test timeout/cancellation may prevent a terminal snapshot. - // Keep bounded-cadence evidence without adding a timer to clean up. - if (driver.now() - lastCheckpointAt >= 30_000) { - lastCheckpointAt = driver.now(); - const saved = capture({ state: 'in_progress', elapsedMs: driver.now() - startedAt, - fingerprints, step0Count, reviewCount, administrativeCount, transcript }); +/** Wake on output (or 2s), coalesce redraws for 250ms, then read the viewport. */ +async function countingPoll(run: CountingRun, session: ClaudePtySession): Promise { + if (remainingWork(run) <= 0) return false; + await session.waitForOutput(run.observedOutput, Math.min(2000, remainingWork(run))); + if (remainingWork(run) <= 0) return false; + const coalesceMs = session.rawOutput().length > run.observedOutput + ? 250 : 250 - (run.driver.monotonic() - run.lastObservationAt); + if (coalesceMs > 0 && !await waitForWork(run, coalesceMs)) return false; + run.observedOutput = session.mark(); + run.lastObservationAt = run.driver.monotonic(); + run.viewport = await session.currentScreen(); + return remainingWork(run) > 0; +} + +async function countingTick(run: CountingRun, session: ClaudePtySession): Promise> { + const visible = run.viewport; + const observed = countingObserve(run, session, visible); + if (observed) return observed; + const { opts } = run; + const transcript = run.transcript; + // A native AUQ can render before its JSONL tool-use record is flushed. + // The opt-in hook supplies pending identity only; answered counts above + // still come exclusively from the published native transcript. + const pending = transcript.calls.find(c => !c.answered && !c.failed) + ?? readPendingQuestion(session.pendingQuestionFile, run.fixture.cwd, + session.hermeticConfigDir, run.startedAt, transcript); + // Native ACKs → complete collection → caller validation. A process failure + // above or a pending native question still prevents this early collection stop. + if (opts.isCollectionComplete && !pending && transcript.status === 'ready' && + transcript.calls.length > 0 && transcript.calls.every(call => call.answered && !call.failed) && + !unresolvedPlanQuestionCalls(transcript.calls).length && remainingWork(run) > 0 && + opts.isCollectionComplete(transcript, run.fingerprints) && remainingWork(run) > 0) { + return { done: countingSnapshot(run, session, 'collection_complete', 'Caller-defined native collection is complete; final validation remains required', visible) }; + } + const terminal = await countingTerminal(run, visible, pending); + if ('done' in terminal) return { done: countingSnapshot(run, session, terminal.done[0], terminal.done[1], visible) }; + return countingRespond(run, session, visible, pending, terminal); +} + +/** Read and count native calls; process-level terminals. Undefined keeps routing. */ +function countingObserve(run: CountingRun, session: ClaudePtySession, visible: string): PtyStep | undefined { + const { opts, fixture, startedAt } = run; + const finish = (outcome: PlanSkillCountObservation['outcome'], summary: string) => + ({ done: countingSnapshot(run, session, outcome, summary, visible) }); + let transcript: PlanCountTranscript = session.hermeticConfigDir + ? readPlanCountTranscript(session.hermeticConfigDir, fixture.cwd) + : { status: 'error', calls: [], assistantMessages: [], error: 'Claude count session has no isolated transcript directory' }; + transcript = run.transcript = withPendingExit(transcript, session.pendingPlanReadyFile, fixture.cwd, + session.hermeticConfigDir, startedAt, visible); + if (opts.approveEngTestPlanEdits) { + const status = autoplanArtifactRecorderStatus(session.pendingAutoplanArtifactFile, fixture.cwd, + session.hermeticConfigDir, session.autoplanArtifactStateRoot); + const boundary = autoplanArtifactApprovalBoundary(status); + if (boundary === 'failed') return finish('artifact_permission_failed', + `Owned Eng QA test-plan approval failed: ${JSON.stringify(status)}`); + // Native approval owns this Edit. Never answer its repaint or count a + // metadata-only pending request; resume only after its actual result. + if (boundary === 'pending') { + if (run.driver.now() - run.lastCheckpointAt >= 30_000) { + run.lastCheckpointAt = run.driver.now(); + const saved = countingCapture(run, session, { state: 'artifact_pending', elapsedMs: run.driver.now() - startedAt, + fingerprints: run.fingerprints, step0Count: run.step0Count, reviewCount: run.reviewCount, + administrativeCount: run.administrativeCount, transcript, artifactStatus: status }); if (saved.artifactError) console.error(`PTY artifact write failed: ${saved.artifactError}`); } - - // Process exited? - if (session.exited()) { - return snapshot( - 'exited', - `claude exited (code=${session.exitCode()}) during counting (step0=${step0Count}, review=${reviewCount})`, - visible, - ); - } - - if (isRejectedSlashCommand(visible, opts.slashCommand)) { - return snapshot( - 'exited', - `claude rejected ${opts.slashCommand} as unknown command (skill not registered in this cwd)`, - visible, - ); - } - - // A native AUQ can render before its JSONL tool-use record is flushed. - // The opt-in hook supplies pending identity only; answered counts above - // still come exclusively from the published native transcript. - const pending = transcript.calls.find(c => !c.answered && !c.failed) - ?? readPendingQuestion(session.pendingQuestionFile, fixture.cwd, - session.hermeticConfigDir, startedAt, transcript); - // Native ACKs → complete collection → caller validation. A process failure - // above or a pending native question still prevents this early collection stop. - if (opts.isCollectionComplete && !pending && transcript.status === 'ready' && - transcript.calls.length > 0 && transcript.calls.every(call => call.answered && !call.failed) && - !unresolvedPlanQuestionCalls(transcript.calls).length && remainingWork() > 0 && - opts.isCollectionComplete(transcript, fingerprints) && remainingWork() > 0) { - return snapshot('collection_complete', 'Caller-defined native collection is complete; final validation remains required', visible); - } - const newlyMatched = pending && matchesNativePlanQuestion(visible, pending, planningDirectory); - if (newlyMatched) lastMatchedNativeQuestion = pending; - const renderedFrame = classifyPlanCountFrame(visible); - const administrative = new Set(fingerprints.filter(fp => fp.administrative === 'completion-handoff').map(fp => fp.signature)); - if (opts.evaluateTerminal && opts.expectedPlanPath && renderedFrame === 'plan_ready' && !newlyMatched) { - const reviewed = await evaluateOwnedNativePlanTerminal(transcript, opts.expectedPlanPath, startedAt, - startedAt + timeoutMs - cleanupReserveMs, opts.evaluateTerminal); - if (reviewed) { - // Once semantics are validated, lexical phase labels have no veto. - // The earlier progress snapshots remain unchanged diagnostic evidence. - administrative.clear(); - for (const identity of reviewed.administrative) administrative.add(identity); - for (const fp of fingerprints) { - fp.preReview = !reviewed.substantive.has(fp.signature) && !administrative.has(fp.signature); - if (administrative.has(fp.signature)) fp.administrative = 'completion-handoff'; - else delete fp.administrative; - } - reviewCount = reviewed.substantive.size; - administrativeCount = administrative.size; - step0Count = fingerprints.length - reviewCount - administrativeCount; - } - } - - // A long completed summary may scroll its heading off the viewport. - // With no active input UI, retain the existing native/report validator; - // neither display text nor a missing heading supplies completion evidence. - const nativeCompletion = opts.expectedPlanPath && hasNativePlanCompletion(transcript, opts.expectedPlanPath, startedAt); - const nativeSummary = !nativeCompletion && renderedFrame === null && opts.expectedPlanPath && - !isNumberedOptionListVisible(visible) && !isPermissionDialogVisible(visible) && !isProseAUQVisible(visible) && - hasNativePlanTerminal(transcript, opts.expectedPlanPath, startedAt, 'completion_summary', administrative); - const terminalFrame = nativeSummary ? 'completion_summary' : renderedFrame; - const isTerminalHint = terminalFrame === 'completion_summary' || terminalFrame === 'plan_ready'; - const verifiedTerminal = nativeSummary || (opts.expectedPlanPath && isTerminalHint && - hasNativePlanTerminal(transcript, opts.expectedPlanPath, startedAt, terminalFrame, administrative)); - if (reviewCount === 0 && fingerprints.some(fp => fp.administrative === 'artifact-generation') && - (nativeCompletion || verifiedTerminal)) { - return snapshot('no_review_questions', 'Completed artifact generation supplied no review finding decisions', visible); - } - // A streamed heading cannot dismiss the last bound native question. - // Only an accepted terminal may supersede its answered redraw; otherwise - // permission wording inside that question could queue a stray answer. - const acceptedTerminal = isTerminalHint && (!opts.expectedPlanPath || verifiedTerminal); - const nativeQuestionVisible = newlyMatched || (!acceptedTerminal && - lastMatchedNativeQuestion && matchesNativePlanQuestion(visible, lastMatchedNativeQuestion, planningDirectory)); - const terminalHint = nativeQuestionVisible ? null : terminalFrame; - let frame = terminalHint; - // Clear unverified hints before routing active permissions and Submit. - if (opts.expectedPlanPath && isTerminalHint && !verifiedTerminal) frame = null; - // A real approval gate is never an AUQ or a file permission. Wait for - // its report/native evidence before input routing; verified gates still - // pass the existing silent-write check below. A positively matched - // native question above takes precedence over gate text. - const nonReviewCalls = new Set(fingerprints.filter(fp => fp.preReview || fp.administrative) - .map(fp => fp.signature)); - if (opts.expectedPlanPath && terminalHint === 'plan_ready' && reviewCount === 0 && - isQuestionlessNativePlanExit(transcript, opts.expectedPlanPath, startedAt, visible, nonReviewCalls)) { - return snapshot('no_review_questions', - 'Native plan approval reached with zero review-phase AskUserQuestion calls; review coverage is missing', visible); - } - if (opts.expectedPlanPath && terminalHint === 'plan_ready' && !verifiedTerminal) continue; - let permissionGuard = filePermission; - let permissionEpoch: FilePermissionEpoch | null | undefined; - const currentBinding = currentFilePermissionBinding(ownedFilePermissions, fixture.cwd, - session.hermeticConfigDir, startedAt, transcript, visible); - if (currentBinding) { permissionGuard = currentBinding.binding.guard; permissionEpoch = currentBinding.epoch; } - else permissionEpoch = currentBinding; - const permission = nativeQuestionVisible || terminalHint === 'plan_ready' - ? null : permissionGuard(visible, session.visibleText(), permissionEpoch); - if (frame === 'permission' || (permission === 'grant' && frame === null)) { - if (remainingWork() <= 0) break; - if (permission !== 'handled') session.send(`${defaultPick}\r`); - await waitForWork(1500); - continue; - } - - const submissionInput = frame === null ? planCountSubmissionInput(visible) : null; - if (submissionInput !== null) { - if (remainingWork() <= 0) break; - session.send(submissionInput); - await waitForWork(1500); - continue; - } - - // Silent write detection — only fires if no numbered prompt is on - // screen (otherwise the write is gated by a permission/AUQ). - const writeRe = /⏺\s*(?:Write|Edit)\(([^)]+)\)/g; - let m: RegExpExecArray | null; - while ((m = writeRe.exec(visible)) !== null) { - const target = m[1] ?? ''; - const sanctioned = SANCTIONED_WRITE_SUBSTRINGS.some((s) => - target.includes(s), - ); - if (!sanctioned && !isNumberedOptionListVisible(visible)) { - return snapshot( - 'silent_write', - `Write/Edit to ${target} fired before any AskUserQuestion`, - visible, - ); - } - } - - if (opts.expectedPlanPath && verifiedTerminal && (frame === 'completion_summary' || frame === 'plan_ready')) { - return snapshot(frame, `native ${frame} and final report verified (step0=${step0Count}, review=${reviewCount})`, visible); - } - - // Legacy callers without a report contract retain their terminal policy. - if (frame === 'completion_summary') { - if (transcript.status !== 'ready' || transcript.calls.some(c => !c.answered && !c.failed) || - unresolvedPlanQuestionCalls(transcript.calls).length) { - return snapshot('transcript_unavailable', 'Completion has no complete native question transcript', visible); - } - return snapshot( - 'completion_summary', - `skill emitted completion summary / verdict / status line (step0=${step0Count}, review=${reviewCount})`, - visible, - ); - } - if (frame === 'plan_ready') { - if (transcript.status !== 'ready' || transcript.calls.some(c => !c.answered && !c.failed) || - unresolvedPlanQuestionCalls(transcript.calls).length) { - return snapshot('transcript_unavailable', 'Plan-ready gate has no complete native question transcript', visible); - } - return snapshot( - 'plan_ready', - `skill emitted plan-mode "Ready to execute" confirmation (step0=${step0Count}, review=${reviewCount})`, - visible, - ); - } - - if (opts.expectedPlanPath && !transcript.planReadyRequests?.length && !isNumberedOptionListVisible(visible) && - hasNativePlanCompletion(transcript, opts.expectedPlanPath, startedAt)) { - return snapshot('completion_summary', - `native review completion and final report verified (step0=${step0Count}, review=${reviewCount})`, visible); - } - - // A dismissed or repainted permission is never a native question. - if (permission === 'handled') continue; - - // Dedupe the complete question, not just its answer labels: separate - // findings often reuse the same Add to plan / Defer / Skip menu. - if (opts.requireNativePicker && !newlyMatched) continue; - const capturedSeen = opts.requireNativePicker ? new Set(seen) : seen; - const fp = capturePlanCountQuestion(visible, capturedSeen, driver.now() - startedAt, !boundaryFired, pending, planningDirectory); - if (!fp) continue; - const boundNativeTab = pending && fp.nativeCall === pending && fp.nativeQuestionIndex !== undefined; - if (opts.requireNativePicker && !boundNativeTab) continue; - // Press to advance — first AUQ may use the override pick. - const routing = pending?.questions.length === 1 - ? nativePlanCallFingerprint(pending, fp.observedAtMs, fp.preReview) : fp; - const prerequisitePick = planCountPrerequisitePick(routing, fp); - // Native tool records may flush only after the answer. Let a guarded - // caller recognize that visible menu. A known packet needs a positively - // matched active tab before a caller can change that tab's choice. - // The captured fingerprint alone proves whether native metadata matched - // this active UI; an unrelated pending record is not a routing identity. - let callerPick: number | null = null; - if ((!opts.requireNativePicker || prerequisitePick === null) && - (!pending || pending.questions.length === 1 || boundNativeTab)) { - callerPick = opts.pickAUQ?.(routing, fp, pickerContext) ?? null; - } - if (opts.requireNativePicker && prerequisitePick === null && callerPick === null) - throw Error('Declared native picker returned no authorized choice'); - const pickIdx = prerequisitePick ?? callerPick ?? - (isFirstAUQ && opts.firstAUQPick ? opts.firstAUQPick(routing) : defaultPick); - if (opts.requireNativePicker) for (const signature of capturedSeen) seen.add(signature); - isFirstAUQ = false; - const questionInput = planCountQuestionInput(visible, fp, pickIdx); - if (remainingWork() <= 0) break; - if (questionInput.includes('\r')) { - // The helper separates digit and Enter by 500ms. Do not let that - // delayed confirmation send input after this counting window closes. - await selectPtyNumberedOption({ send: input => { - if (remainingWork() > 0) session.send(input); - } }, pickIdx); - } else session.send(questionInput); - - // Give the agent a beat to advance to the next state. - await waitForWork(2000); - } - - return snapshot( - 'timeout', - `no terminal outcome within ${timeoutMs}ms total budget (including startup and ${cleanupReserveMs}ms cleanup reserve; step0=${step0Count}, review=${reviewCount})`, - viewport, - ); - } catch (error) { - countingFailed = true; - // Caller/actor errors used to leave only the preceding 30s checkpoint. - // Retain the actual throw frame and public native state before close() - // removes the hook and fixture, without replacing the original failure. - try { - // Keep the exact frame/transcript that the throwing caller observed; - // awaiting a redraw here would erase that ordering evidence. - const saved = capture({ state: 'threw', error: error instanceof Error ? error.message : String(error), - elapsedMs: driver.now() - startedAt, fingerprints, step0Count, reviewCount, administrativeCount, transcript, - pendingQuestion: readPendingQuestion(session.pendingQuestionFile, fixture.cwd, - session.hermeticConfigDir, startedAt, transcript) }); - if (saved.artifactDir) console.error(`Full PTY artifacts: ${saved.artifactDir}`); - if (saved.artifactError) console.error(`PTY artifact write failed: ${saved.artifactError}`); - } catch (captureError) { console.error(`PTY failure capture failed: ${String(captureError)}`); } - throw error; - } finally { - try { - await session.close(); - } catch (error) { - if (!countingFailed) { - capture({ state: 'cleanup_failed', error: String(error), transcript, fingerprints }); - throw error; - } - } finally { - fixture.cleanup(); + return 'continue'; } } + if (transcript.status === 'error') return finish('transcript_unavailable', transcript.error!); + countAnsweredCalls(run, transcript); + if (run.reviewCount >= opts.reviewCountCeiling) { + if (unresolvedPlanQuestionCalls(transcript.calls).length) { + return finish('transcript_unavailable', 'Question count reached its ceiling with unresolved failed native calls'); + } + return finish('ceiling_reached', `review-phase AUQ count reached ceiling (${opts.reviewCountCeiling})`); + } + // An outer test timeout/cancellation may prevent a terminal snapshot. + // Keep bounded-cadence evidence without adding a timer to clean up. + if (run.driver.now() - run.lastCheckpointAt >= 30_000) { + run.lastCheckpointAt = run.driver.now(); + const saved = countingCapture(run, session, { state: 'in_progress', elapsedMs: run.driver.now() - startedAt, + fingerprints: run.fingerprints, step0Count: run.step0Count, reviewCount: run.reviewCount, + administrativeCount: run.administrativeCount, transcript }); + if (saved.artifactError) console.error(`PTY artifact write failed: ${saved.artifactError}`); + } + if (session.exited()) { + return finish('exited', `claude exited (code=${session.exitCode()}) during counting (step0=${run.step0Count}, review=${run.reviewCount})`); + } + if (isRejectedSlashCommand(visible, opts.slashCommand)) { + return finish('exited', `claude rejected ${opts.slashCommand} as unknown command (skill not registered in this cwd)`); + } + return undefined; +} + +/** Count each answered native call once; phase comes from the caller's predicates. */ +function countAnsweredCalls(run: CountingRun, transcript: PlanCountTranscript): void { + const { opts } = run; + for (const [callIndex, call] of transcript.calls.entries()) { + const signature = `${call.sessionId}:${call.toolUseId}`; + if (!call.answered || run.countedCalls.has(signature)) continue; + const fp = nativePlanCallFingerprint(call, run.driver.now() - run.startedAt, !run.boundaryFired); + const phase = planCountQuestionPhase(fp, run.boundaryFired, opts.isLastStep0AUQ, opts.isFirstReviewAUQ, opts.isSetupAUQ, opts.isCompletionHandoffAUQ, opts.isArtifactGenerationAUQ); + if (phase.administrative) { + fp.preReview = false; + fp.administrative = phase.administrative; + run.administrativeCount += 1; + } else { + fp.preReview = opts.isReviewAUQ ? !opts.isReviewAUQ(fp, transcript.calls.slice(0, callIndex)) : phase.preReview; + if (fp.preReview) run.step0Count += 1; + else run.reviewCount += 1; + } + fp.promptSnippet = fp.promptSnippet.replace(/\s+/g, ' ').slice(0, 240); + run.fingerprints.push(fp); + run.countedCalls.add(signature); + run.boundaryFired = phase.reviewStarted; + } +} + +/** + * A long completed summary may scroll its heading off the viewport. With no + * active input UI, retain the existing native/report validator; neither + * display text nor a missing heading supplies completion evidence. + */ +export function isNativeCompletionSummary(input: { expectedPlanPath?: string; nativeCompletion: boolean; + renderedFrame: ReturnType; visible: string; transcript: PlanCountTranscript; startedAt: number; + administrative: ReadonlySet }): boolean { + return Boolean(!input.nativeCompletion && input.renderedFrame === null && input.expectedPlanPath && + !isNumberedOptionListVisible(input.visible) && !isPermissionDialogVisible(input.visible) && !isProseAUQVisible(input.visible) && + hasNativePlanTerminal(input.transcript, input.expectedPlanPath, input.startedAt, 'completion_summary', input.administrative)); +} + +interface CountingTerminal { + newlyMatched: boolean; + nativeCompletion: boolean; + verifiedTerminal: boolean; + nativeQuestionVisible: boolean; + terminalHint: 'permission' | 'completion_summary' | 'plan_ready' | null; + frame: 'permission' | 'completion_summary' | 'plan_ready' | null; +} + +/** Terminal evidence for this frame, before any input is routed. */ +async function countingTerminal(run: CountingRun, visible: string, pending: NativePlanQuestionCall | undefined): + Promise { + const { opts, startedAt, transcript, fingerprints } = run; + const newlyMatched = Boolean(pending && matchesNativePlanQuestion(visible, pending, run.planningDirectory)); + if (newlyMatched) run.lastMatchedNativeQuestion = pending; + const renderedFrame = classifyPlanCountFrame(visible); + const administrative = new Set(fingerprints.filter(fp => fp.administrative === 'completion-handoff').map(fp => fp.signature)); + if (opts.evaluateTerminal && opts.expectedPlanPath && renderedFrame === 'plan_ready' && !newlyMatched) { + const reviewed = await evaluateOwnedNativePlanTerminal(transcript, opts.expectedPlanPath, startedAt, + startedAt + run.timeoutMs - run.cleanupReserveMs, opts.evaluateTerminal); + if (reviewed) { + // Once semantics are validated, lexical phase labels have no veto. + // The earlier progress snapshots remain unchanged diagnostic evidence. + administrative.clear(); + for (const identity of reviewed.administrative) administrative.add(identity); + for (const fp of fingerprints) { + fp.preReview = !reviewed.substantive.has(fp.signature) && !administrative.has(fp.signature); + if (administrative.has(fp.signature)) fp.administrative = 'completion-handoff'; + else delete fp.administrative; + } + run.reviewCount = reviewed.substantive.size; + run.administrativeCount = administrative.size; + run.step0Count = fingerprints.length - run.reviewCount - run.administrativeCount; + } + } + const nativeCompletion = Boolean(opts.expectedPlanPath && hasNativePlanCompletion(transcript, opts.expectedPlanPath, startedAt)); + const nativeSummary = isNativeCompletionSummary({ expectedPlanPath: opts.expectedPlanPath, nativeCompletion, + renderedFrame, visible, transcript, startedAt, administrative }); + const terminalFrame = nativeSummary ? 'completion_summary' : renderedFrame; + const isTerminalHint = terminalFrame === 'completion_summary' || terminalFrame === 'plan_ready'; + const verifiedTerminal = nativeSummary || Boolean(opts.expectedPlanPath && isTerminalHint && + hasNativePlanTerminal(transcript, opts.expectedPlanPath, startedAt, terminalFrame, administrative)); + if (run.reviewCount === 0 && fingerprints.some(fp => fp.administrative === 'artifact-generation') && + (nativeCompletion || verifiedTerminal)) { + return { done: ['no_review_questions', 'Completed artifact generation supplied no review finding decisions'] }; + } + // A streamed heading cannot dismiss the last bound native question. + // Only an accepted terminal may supersede its answered redraw; otherwise + // permission wording inside that question could queue a stray answer. + const acceptedTerminal = isTerminalHint && (!opts.expectedPlanPath || verifiedTerminal); + const nativeQuestionVisible = newlyMatched || (!acceptedTerminal && + Boolean(run.lastMatchedNativeQuestion && matchesNativePlanQuestion(visible, run.lastMatchedNativeQuestion, run.planningDirectory))); + const terminalHint = nativeQuestionVisible ? null : terminalFrame; + let frame = terminalHint; + // Clear unverified hints before routing active permissions and Submit. + if (opts.expectedPlanPath && isTerminalHint && !verifiedTerminal) frame = null; + // A real approval gate is never an AUQ or a file permission. Wait for + // its report/native evidence before input routing; verified gates still + // pass the existing silent-write check below. A positively matched + // native question above takes precedence over gate text. + const nonReviewCalls = new Set(fingerprints.filter(fp => fp.preReview || fp.administrative) + .map(fp => fp.signature)); + if (opts.expectedPlanPath && terminalHint === 'plan_ready' && run.reviewCount === 0 && + isQuestionlessNativePlanExit(transcript, opts.expectedPlanPath, startedAt, visible, nonReviewCalls)) { + return { done: ['no_review_questions', + 'Native plan approval reached with zero review-phase AskUserQuestion calls; review coverage is missing'] }; + } + return { newlyMatched, nativeCompletion, verifiedTerminal, nativeQuestionVisible, terminalHint, frame }; +} + +/** Route permissions, Submit and the current question; then terminal outcomes. */ +async function countingRespond(run: CountingRun, session: ClaudePtySession, visible: string, + pending: NativePlanQuestionCall | undefined, t: CountingTerminal): Promise> { + const { opts, fixture, startedAt, transcript } = run; + const finish = (outcome: PlanSkillCountObservation['outcome'], summary: string) => + ({ done: countingSnapshot(run, session, outcome, summary, visible) }); + const { frame } = t; + if (opts.expectedPlanPath && t.terminalHint === 'plan_ready' && !t.verifiedTerminal) return 'continue'; + let permissionGuard = run.filePermission; + let permissionEpoch: FilePermissionEpoch | null | undefined; + const currentBinding = currentFilePermissionBinding(run.ownedFilePermissions, fixture.cwd, + session.hermeticConfigDir, startedAt, transcript, visible); + if (currentBinding) { permissionGuard = currentBinding.binding.guard; permissionEpoch = currentBinding.epoch; } + else permissionEpoch = currentBinding; + const permission = t.nativeQuestionVisible || t.terminalHint === 'plan_ready' + ? null : permissionGuard(visible, session.visibleText(), permissionEpoch); + if (frame === 'permission' || (permission === 'grant' && frame === null)) { + if (remainingWork(run) <= 0) return 'break'; + if (permission !== 'handled') session.send(`${run.defaultPick}\r`); + await waitForWork(run, 1500); + return 'continue'; + } + + const submissionInput = frame === null ? planCountSubmissionInput(visible) : null; + if (submissionInput !== null) { + if (remainingWork(run) <= 0) return 'break'; + session.send(submissionInput); + await waitForWork(run, 1500); + return 'continue'; + } + + // Silent write detection — only fires if no numbered prompt is on + // screen (otherwise the write is gated by a permission/AUQ). + const writeRe = /⏺\s*(?:Write|Edit)\(([^)]+)\)/g; + let m: RegExpExecArray | null; + while ((m = writeRe.exec(visible)) !== null) { + const target = m[1] ?? ''; + const sanctioned = SANCTIONED_WRITE_SUBSTRINGS.some((s) => + target.includes(s), + ); + if (!sanctioned && !isNumberedOptionListVisible(visible)) { + return finish('silent_write', `Write/Edit to ${target} fired before any AskUserQuestion`); + } + } + + if (opts.expectedPlanPath && t.verifiedTerminal && (frame === 'completion_summary' || frame === 'plan_ready')) { + return finish(frame, `native ${frame} and final report verified (step0=${run.step0Count}, review=${run.reviewCount})`); + } + + // Legacy callers without a report contract retain their terminal policy. + if (frame === 'completion_summary' || frame === 'plan_ready') { + if (transcript.status !== 'ready' || transcript.calls.some(c => !c.answered && !c.failed) || + unresolvedPlanQuestionCalls(transcript.calls).length) { + return finish('transcript_unavailable', frame === 'completion_summary' + ? 'Completion has no complete native question transcript' + : 'Plan-ready gate has no complete native question transcript'); + } + return finish(frame, frame === 'completion_summary' + ? `skill emitted completion summary / verdict / status line (step0=${run.step0Count}, review=${run.reviewCount})` + : `skill emitted plan-mode "Ready to execute" confirmation (step0=${run.step0Count}, review=${run.reviewCount})`); + } + + if (opts.expectedPlanPath && !transcript.planReadyRequests?.length && !isNumberedOptionListVisible(visible) && + hasNativePlanCompletion(transcript, opts.expectedPlanPath, startedAt)) { + return finish('completion_summary', + `native review completion and final report verified (step0=${run.step0Count}, review=${run.reviewCount})`); + } + + // A dismissed or repainted permission is never a native question. + if (permission === 'handled') return 'continue'; + return countingAnswer(run, session, visible, pending, t.newlyMatched); +} + +/** Dedupe the complete question and press its policy-selected option. */ +async function countingAnswer(run: CountingRun, session: ClaudePtySession, visible: string, + pending: NativePlanQuestionCall | undefined, newlyMatched: boolean): Promise> { + const { opts, startedAt } = run; + // Dedupe the complete question, not just its answer labels: separate + // findings often reuse the same Add to plan / Defer / Skip menu. + if (opts.requireNativePicker && !newlyMatched) return 'continue'; + const capturedSeen = opts.requireNativePicker ? new Set(run.seen) : run.seen; + const fp = capturePlanCountQuestion(visible, capturedSeen, run.driver.now() - startedAt, !run.boundaryFired, pending, run.planningDirectory); + if (!fp) return 'continue'; + const boundNativeTab = pending && fp.nativeCall === pending && fp.nativeQuestionIndex !== undefined; + if (opts.requireNativePicker && !boundNativeTab) return 'continue'; + // Press to advance — first AUQ may use the override pick. + const routing = pending?.questions.length === 1 + ? nativePlanCallFingerprint(pending, fp.observedAtMs, fp.preReview) : fp; + const prerequisitePick = planCountPrerequisitePick(routing, fp); + // Native tool records may flush only after the answer. Let a guarded + // caller recognize that visible menu. A known packet needs a positively + // matched active tab before a caller can change that tab's choice. + // The captured fingerprint alone proves whether native metadata matched + // this active UI; an unrelated pending record is not a routing identity. + let callerPick: number | null = null; + if ((!opts.requireNativePicker || prerequisitePick === null) && + (!pending || pending.questions.length === 1 || boundNativeTab)) { + callerPick = opts.pickAUQ?.(routing, fp, run.pickerContext) ?? null; + } + if (opts.requireNativePicker && prerequisitePick === null && callerPick === null) + throw Error('Declared native picker returned no authorized choice'); + const pickIdx = prerequisitePick ?? callerPick ?? + (run.isFirstAUQ && opts.firstAUQPick ? opts.firstAUQPick(routing) : run.defaultPick); + if (opts.requireNativePicker) for (const signature of capturedSeen) run.seen.add(signature); + run.isFirstAUQ = false; + const questionInput = planCountQuestionInput(visible, fp, pickIdx); + if (remainingWork(run) <= 0) return 'break'; + if (questionInput.includes('\r')) { + // The helper separates digit and Enter by 500ms. Do not let that + // delayed confirmation send input after this counting window closes. + await selectPtyNumberedOption({ send: input => { + if (remainingWork(run) > 0) session.send(input); + } }, pickIdx); + } else session.send(questionInput); + + // Give the agent a beat to advance to the next state. + await waitForWork(run, 2000); + return 'continue'; +} + +/** Caller/actor errors used to leave only the preceding 30s checkpoint. + * Retain the actual throw frame and public native state before close() + * removes the hook and fixture, without replacing the original failure. */ +function countingFailure(run: CountingRun, session: ClaudePtySession, error: unknown): void { + try { + // Keep the exact frame/transcript that the throwing caller observed; + // awaiting a redraw here would erase that ordering evidence. + const saved = countingCapture(run, session, { state: 'threw', error: error instanceof Error ? error.message : String(error), + elapsedMs: run.driver.now() - run.startedAt, fingerprints: run.fingerprints, step0Count: run.step0Count, + reviewCount: run.reviewCount, administrativeCount: run.administrativeCount, transcript: run.transcript, + pendingQuestion: readPendingQuestion(session.pendingQuestionFile, run.fixture.cwd, + session.hermeticConfigDir, run.startedAt, run.transcript) }); + if (saved.artifactDir) console.error(`Full PTY artifacts: ${saved.artifactDir}`); + if (saved.artifactError) console.error(`PTY artifact write failed: ${saved.artifactError}`); + } catch (captureError) { console.error(`PTY failure capture failed: ${String(captureError)}`); } } diff --git a/test/helpers/pty/runners/floor.ts b/test/helpers/pty/runners/floor.ts index b6e8a7a18..b9e739086 100644 --- a/test/helpers/pty/runners/floor.ts +++ b/test/helpers/pty/runners/floor.ts @@ -18,10 +18,9 @@ import { capturePlanCountQuestion, createPlanCountPermissionGuard, matchesNative import { resolveClaudeBinary } from '../binary'; import { designReviewSetupAUQ } from '../boundaries'; import { SANCTIONED_WRITE_SUBSTRINGS, isProseAUQVisible, planCountSubmissionInput } from '../classify'; -import { launchClaudePty } from '../launch'; import type { ClaudePtySession } from '../launch'; import { isNumberedOptionListVisible, isPermissionDialogVisible, isPlanReadyVisible, isRejectedSlashCommand, stripPtyResidue } from '../screen'; -import type { PtyDriver } from '../session'; +import { realPtyDriver, runPtySession, type PtyDriver, type PtyStep } from '../session'; // ──────────────────────────────────────────────────────────────────────────── // runPlanSkillFloorCheck — stop at the first substantive seeded question. @@ -130,7 +129,8 @@ export function planFloorDXReplyInput(visible: string, call: NativePlanQuestionC : { input: '\r', stage: 'done' }; } -export async function runPlanSkillFloorCheck(opts: { +/** Options for runPlanSkillFloorCheck. */ +export interface PlanSkillFloorOptions { /** Skill name, e.g. 'plan-eng-review'. Used for diagnostic strings only. */ skillName: string; /** Slash command; the owned PLAN.md target is supplied in the same submission. */ @@ -153,9 +153,10 @@ export async function runPlanSkillFloorCheck(opts: { model?: string; /** Launch seam and clock; tests pass the fake driver. Default: real launcher and clocks. */ driver?: PtyDriver; -}): Promise { - const driver = opts.driver ?? { launch: launchClaudePty, now: () => Date.now(), - monotonic: () => performance.now(), sleep: (ms: number) => Bun.sleep(ms) }; +} + +export async function runPlanSkillFloorCheck(opts: PlanSkillFloorOptions): Promise { + const driver = opts.driver ?? realPtyDriver; const startedAt = driver.now(); const timeoutMs = opts.timeoutMs ?? 600_000; const dxContext = opts.devexSetupContext; @@ -177,9 +178,18 @@ export async function runPlanSkillFloorCheck(opts: { const fixture = createPlanCountFixture(request, { requestedPlanPath: opts.requestedPlanPath, nativeReviewOnly: true, preconfiguredReviewActor: true }); const sessionId = randomUUID(); - let session: ClaudePtySession; - try { - session = await driver.launch({ + const run = { opts, driver, startedAt, timeoutMs, dxContext, fixture, sessionId, + ownedFilePermissions: [], planningDirectory: undefined, transcript: { status: 'missing', calls: [], assistantMessages: [] }, + viewport: '', publicTools: [], pendingQuestion: undefined, floorReview: undefined, floorAssessment: undefined, + setupChoices: new Map(), submittedSetup: new Set(), assessed: new Map(), dxReplies: new Map(), + since: 0, commandStartedAt: 0, targetDelivery: undefined, saveSnapshot: undefined, nativeCandidates: [], + validatedPendingQuestion: undefined, sampledAt: undefined, lastCheckpointAt: 0, lastCheckpointState: '', + artifactError: undefined, finished: false, started: false, start: 0, deadlineAt: 0, screenDeadlineAt: 0, + } as unknown as FloorRun; + return runPtySession({ + driver, + cleanup: () => fixture.cleanup(), + launch: { permissionMode: 'plan', cwd: fixture.cwd, timeoutMs: timeoutMs + 60_000, @@ -191,291 +201,327 @@ export async function runPlanSkillFloorCheck(opts: { ...(dxContext ? { rows: 80 } : {}), observeFilePermissions: fixture.workingPlanPath ? [fixture.workingPlanPath] : undefined, extraArgs: ['--session-id', sessionId], - }); - } catch (error) { - fixture.cleanup(); - throw error; - } - - const ownedFilePermissions = (session.pendingFilePermissionFiles ?? []).map(binding => - ({ ...binding, guard: createPlanCountPermissionGuard() })); - const planningDirectory = session.hermeticConfigDir ? path.join(session.hermeticConfigDir, 'plans') : undefined; - let transcript: PlanCountTranscript = { status: 'missing', calls: [], assistantMessages: [] }; - let viewport = ''; - let publicTools: NativePublicToolEvent[] = []; - let pendingQuestion: NativePlanQuestionCall | undefined; - let floorReview: PlanFloorReview | undefined; - let floorAssessment: PlanFloorAssessment | undefined; - const setupChoices = new Map>(); - const submittedSetup = new Set(); - const assessed = new Map(); - const dxReplies = new Map(); - let captureBeforeClose: ((error?: unknown) => void) | undefined; - let floorFailed = false; - let floorError: unknown; - try { - await driver.sleep(8000); // boot grace + auto-trust handler window - const since = session.mark(); - const commandStartedAt = driver.now(); - session.send(`${opts.slashCommand} PLAN.md\r`); - const deliveryOptions = { seed: fixture.seed, sessionId, - slashCommand: opts.slashCommand, startedAt: commandStartedAt }; - let targetDelivery = readPlanFloorTarget(session.hermeticConfigDir, fixture.cwd, - { ...deliveryOptions, now: driver.now() }); - const saveSnapshot = createPlanCountSnapshotWriter(); - let nativeCandidates: NativePlanQuestionCall[] = []; - let validatedPendingQuestion: ReturnType; - let sampledAt: number | undefined; - let lastCheckpointAt = 0, lastCheckpointState = ''; - let artifactError: string | undefined; - let finished = false; - const capture = (observation: object) => { - const recorderStatus = pendingQuestionRecorderStatus(session.pendingQuestionFile, fixture.cwd, session.hermeticConfigDir); - const artifacts = saveSnapshot({ skillName: opts.skillName, cwd: fixture.cwd, - claudeConfigDir: session.hermeticConfigDir, raw: session.rawOutput(), - visible: session.visibleSince(since), viewport, - observation: { ...observation, transcript, publicTools, pendingQuestion, floorReview, floorAssessment, targetDelivery, commandStartedAt, - pendingWriteInputs: ownedFilePermissions.flatMap(binding => { - const input = readPendingWriteInput(binding.file, binding.expected, fixture.cwd, session.hermeticConfigDir, startedAt); - return input ? [input] : []; - }), - questionDiagnostics: { sampledAt, parentSessionId: sessionId, recorderStatus, recorderStatusAt: driver.now(), nativeCandidates, validatedPendingQuestion }, - ...(dxContext ? { setupContextReplies: [...dxReplies.values()] } : {}), artifactError } }); - if (artifacts.artifactError) { - artifactError ??= artifacts.artifactError; - console.error(`PTY artifact write failed: ${artifacts.artifactError}`); - } - return { ...artifacts, ...(artifactError ? { artifactError } : {}) }; - }; - const checkpoint = () => { - const state = JSON.stringify([pendingQuestionRecorderStatus(session.pendingQuestionFile, fixture.cwd, session.hermeticConfigDir), - targetDelivery.status, nativeCandidates, validatedPendingQuestion, pendingQuestion, [...dxReplies.values()]]); - if (state === lastCheckpointState && driver.now() - lastCheckpointAt < 15_000) return; - lastCheckpointState = state; lastCheckpointAt = driver.now(); - capture({ state: 'in_progress', elapsedMs: driver.now() - startedAt }); - }; - captureBeforeClose = (error) => { - if (floorFailed && session.hermeticConfigDir) { - publicTools = []; - transcript = readPlanCountTranscript(session.hermeticConfigDir, fixture.cwd, event => publicTools.push(event)); - } - if (!finished) capture({ state: floorFailed ? 'threw' : 'in_progress', error: floorFailed ? String(error) : undefined, - captureReason: 'before_cleanup', elapsedMs: driver.now() - startedAt }); - }; - const finish = (observation: PlanSkillFloorObservation): PlanSkillFloorObservation => { - const artifacts = capture(observation); - finished = true; - return { ...observation, targetDelivery, ...artifacts }; - }; - - const start = driver.now(); - const deadlineAt = start + timeoutMs; - const screenDeadlineAt = driver.monotonic() + timeoutMs; - while (driver.now() - start < timeoutMs) { + }, + start: session => floorStart(run, session), + poll: async () => { + if (!(driver.now() - run.start < timeoutMs)) return false; await driver.sleep(2000); - const visible = session.visibleSince(since); - - if (session.exited()) { - return finish({ - auqObserved: false, - outcome: 'exited', - summary: `claude exited (code=${session.exitCode()}) before a qualifying finding`, - evidence: visible.slice(-3000), - elapsedMs: driver.now() - startedAt, - }); - } - if (isRejectedSlashCommand(visible, opts.slashCommand)) { - return finish({ - auqObserved: false, - outcome: 'exited', - summary: `claude rejected ${opts.slashCommand} as unknown command`, - evidence: visible.slice(-3000), - elapsedMs: driver.now() - startedAt, - }); - } - - if (targetDelivery.status !== 'ready') { - targetDelivery = readPlanFloorTarget(session.hermeticConfigDir, fixture.cwd, - { ...deliveryOptions, now: driver.now() }); - if (targetDelivery.status !== 'ready') { - viewport = await session.currentScreen(screenDeadlineAt); - checkpoint(); - continue; - } - } - - // Current native identity precedes permission handling and finding assessment. - floorReview = undefined; floorAssessment = undefined; - viewport = await session.currentScreen(screenDeadlineAt); - publicTools = []; - transcript = session.hermeticConfigDir - ? readPlanCountTranscript(session.hermeticConfigDir, fixture.cwd, event => publicTools.push(event)) - : { status: 'missing', calls: [], assistantMessages: [] }; - const hook = readPendingQuestion(session.pendingQuestionFile, fixture.cwd, - session.hermeticConfigDir, commandStartedAt, transcript); - const currentCalls = transcript.status === 'ready' ? transcript.calls.filter(call => - call.sessionId === sessionId && !call.answered && !call.failed && publicTools.filter(event => - event.sessionId === sessionId && event.kind === 'use' && event.name === 'AskUserQuestion' && - event.toolUseId === call.toolUseId && Date.parse(event.timestamp) >= commandStartedAt && - Date.parse(event.timestamp) <= driver.now() && isDeepStrictEqual(event.input?.questions, call.questions)).length === 1) : []; - nativeCandidates = currentCalls.slice(); - validatedPendingQuestion = hook; - sampledAt = driver.now(); - if (hook?.sessionId === sessionId && !transcript.calls.some(call => call.toolUseId === hook.toolUseId)) currentCalls.push(hook); - const activeReply = currentCalls.length === 1 && dxReplies.get(`${currentCalls[0]!.sessionId}:${currentCalls[0]!.toolUseId}`); - if (activeReply) { - pendingQuestion = undefined; - const next = planFloorDXReplyInput(viewport, currentCalls[0]!, activeReply); - if (next) { session.send(next.input); activeReply.stage = next.stage; } - checkpoint(); - continue; - } - const matching = currentCalls.filter(call => dxContext - ? planFloorDXPane(viewport, call) !== null : matchesNativePlanQuestion(viewport, call, planningDirectory)); - pendingQuestion = matching.length === 1 ? matching[0] : undefined; - checkpoint(); - const nativeQuestionVisible = Boolean(pendingQuestion); - const permissionIsActiveRender = !nativeQuestionVisible && - (isPermissionDialogVisible(viewport) || isCroppedEditPermissionVisible(viewport)); - if (permissionIsActiveRender) { - // An authorized file write enables the review, but never supplies its - // finding question. Exclude the permission's old render after approval. - const owned = currentFilePermissionBinding(ownedFilePermissions, fixture.cwd, - session.hermeticConfigDir, startedAt, transcript, viewport); - let ordinaryOwnedTarget = false; - if (fixture.workingPlanPath) try { - const parent = fs.realpathSync(path.dirname(fixture.workingPlanPath)); - let target: fs.Stats | undefined; - try { target = fs.lstatSync(fixture.workingPlanPath); } - catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; } - ordinaryOwnedTarget = parent === fs.realpathSync(fixture.cwd) && (!target || target.isFile()); - } catch { /* No authority for a linked, foreign, or unreadable target. */ } - if (owned && ordinaryOwnedTarget && owned.binding.guard(viewport, session.visibleText(), owned.epoch) === 'grant') - session.send('1\r'); - } - if (permissionIsActiveRender) continue; - - const questionViewport = dxContext && pendingQuestion ? planFloorDXPane(viewport, pendingQuestion)! : viewport; - const fp = pendingQuestion && capturePlanCountQuestion(questionViewport, new Set(), driver.now() - start, true, pendingQuestion, planningDirectory); - if (fp && pendingQuestion) { - const index = fp.nativeQuestionIndex ?? 0; - const question = pendingQuestion.questions[index]!; - const key = `${pendingQuestion.sessionId}:${pendingQuestion.toolUseId}`; - const chosen = setupChoices.get(key) ?? new Set(); - const allDesign = opts.skillName === 'plan-design-review' && designReviewSetupAUQ(fp) - ? question.options.flatMap((option, i) => /^(?:Review )?All 7 (?:design )?(?:dimensions|passes)(?:\s*\(recommended\))?$/i.test(option.label.trim()) ? [i + 1] : []) : []; - const pick = pickPlanFloorMode(opts.skillName, question) ?? planCountPrerequisitePick(fp, fp) - ?? (opts.skillName === 'plan-devex-review' ? pickPlanFloorProductType(question, opts.productType) : null) - ?? (allDesign.length === 1 ? allDesign[0]! : null); - if (pick !== null) { - if (!chosen.has(index)) { - session.send(planCountQuestionInput(viewport, fp, pick)); - chosen.add(index); setupChoices.set(key, chosen); - } - continue; - } - floorReview = { seed: fixture.seed, candidate: { transport: 'native', identity: `${key}:question:${index}`, - question: structuredClone(question) } }; - } else { - // Public fallback must be a complete current question, not scrollback, - // a generic idle prompt, permission, tool result or quoted example. - floorReview = undefined; - const message = transcript.assistantMessages.filter(m => m.sessionId === sessionId && - Date.parse(m.timestamp) >= commandStartedAt && Date.parse(m.timestamp) <= driver.now()).at(-1); - const compact = (text: string) => text.replace(/\s+/g, ''); - if (!currentCalls.length && message && isProseAUQVisible(viewport) && isProseAUQVisible(message.text) && - !/^\s*(?:>|`{3,}|~{3,})/m.test(message.text) && compact(viewport).includes(compact(message.text))) - floorReview = { seed: fixture.seed, candidate: { transport: 'prose', - identity: `${message.sessionId}:${message.timestamp}`, text: message.text } }; - const submit = planCountSubmissionInput(viewport); - const packet = currentCalls.find(call => setupChoices.get(`${call.sessionId}:${call.toolUseId}`)?.size === call.questions.length); - if (submit && packet) { - const key = `${packet.sessionId}:${packet.toolUseId}`; - if (!submittedSetup.has(key)) { session.send(submit); submittedSetup.add(key); } - continue; - } - } - if (floorReview) { - const key = JSON.stringify(floorReview); - floorAssessment = assessed.get(key); - if (!floorAssessment) { - try { - floorAssessment = judgePlanFloorReview(floorReview, { - binary: resolveClaudeBinary() ?? 'claude', model: resolveEvalModel('warmup'), deadlineAt }); - } catch (error) { - return finish({ auqObserved: false, outcome: 'assessment_error', - summary: `Finding assessment failed: ${error instanceof Error ? error.message : String(error)}`, - evidence: viewport, elapsedMs: driver.now() - startedAt }); - } - assessed.set(key, floorAssessment); - } - if (dxContext && floorAssessment.kind === 'setup' && pendingQuestion?.questions.length === 1 && - !pendingQuestion.questions[0]!.multiSelect && floorReview.candidate.transport === 'native') { - const key = `${pendingQuestion.sessionId}:${pendingQuestion.toolUseId}`; - dxReplies.set(key, { call: structuredClone(pendingQuestion), pane: questionViewport, reply: dxContext, stage: 'focus' }); - } - if (floorAssessment.kind === 'finding') return finish({ - auqObserved: true, outcome: 'auq_observed', - summary: `Current ${floorReview.candidate.transport} question addresses the owned seeded finding: ${floorAssessment.reason}`, - evidence: viewport, elapsedMs: driver.now() - startedAt, - }); - } - - // Silent write outside sanctioned dirs is the transcript-bug shape. - const writeRe = /⏺\s*(?:Write|Edit)\(([^)]+)\)/g; - let m: RegExpExecArray | null; - while ((m = writeRe.exec(visible)) !== null) { - const target = m[1] ?? ''; - const sanctioned = SANCTIONED_WRITE_SUBSTRINGS.some((s) => target.includes(s)); - if (!sanctioned && !isNumberedOptionListVisible(visible)) { - return finish({ - auqObserved: false, - outcome: 'silent_write', - summary: `Write/Edit to ${target} fired before any AskUserQuestion`, - evidence: visible.slice(-3000), - elapsedMs: driver.now() - startedAt, - }); - } - } - - // Reached terminal without AUQ → transcript-bug regression. - // Note: COMPLETION_SUMMARY_RE is intentionally NOT checked here — it - // matches "GSTACK REVIEW REPORT" anywhere in the buffer, including - // when the agent does recon by reading existing plan files (which - // contain that string as a generated section). The plan_ready check - // (claude's actual "Ready to execute" confirmation) is the reliable - // terminal signal for "agent finished without asking." - if (isPlanReadyVisible(visible)) { - return finish({ - auqObserved: false, - outcome: 'plan_ready', - summary: 'agent reached plan_ready without a qualifying finding question', - evidence: visible.slice(-3000), - elapsedMs: driver.now() - startedAt, - }); - } - } - - return finish({ + return true; + }, + tick: session => floorTick(run, session), + timeout: session => floorFinish(run, session, { auqObserved: false, outcome: 'timeout', - summary: targetDelivery.status === 'ready' + summary: run.targetDelivery.status === 'ready' ? `no qualifying finding question within ${timeoutMs}ms` - : `seeded target delivery unavailable within ${timeoutMs}ms: ${targetDelivery.reason ?? targetDelivery.status}`, - evidence: session.visibleSince(since).slice(-3000), + : `seeded target delivery unavailable within ${timeoutMs}ms: ${run.targetDelivery.reason ?? run.targetDelivery.status}`, + evidence: session.visibleSince(run.since).slice(-3000), elapsedMs: driver.now() - startedAt, - }); - } catch (error) { - floorFailed = true; - floorError = error; - throw error; - } finally { - try { captureBeforeClose?.(floorError); } - catch (error) { if (!floorFailed) { floorFailed = true; throw error; } } - finally { - try { await session.close(); } - catch (error) { if (!floorFailed) throw error; } - finally { fixture.cleanup(); } + }), + beforeClose: (session, failure) => floorBeforeClose(run, session, failure), + }); +} + +/** Mutable state of one floor attempt, shared by its session-loop steps. */ +interface FloorRun { + opts: PlanSkillFloorOptions; + driver: PtyDriver; + startedAt: number; + timeoutMs: number; + dxContext: string | undefined; + fixture: ReturnType; + sessionId: string; + ownedFilePermissions: Array<{ expected: string; file: string; guard: ReturnType }>; + planningDirectory: string | undefined; + transcript: PlanCountTranscript; + viewport: string; + publicTools: NativePublicToolEvent[]; + pendingQuestion: NativePlanQuestionCall | undefined; + floorReview: PlanFloorReview | undefined; + floorAssessment: PlanFloorAssessment | undefined; + setupChoices: Map>; + submittedSetup: Set; + assessed: Map; + dxReplies: Map; + since: number; + commandStartedAt: number; + deliveryOptions: { seed: string; sessionId: string; slashCommand: string; startedAt: number }; + targetDelivery: PlanFloorTargetDelivery; + saveSnapshot: ReturnType; + nativeCandidates: NativePlanQuestionCall[]; + validatedPendingQuestion: ReturnType; + sampledAt: number | undefined; + lastCheckpointAt: number; + lastCheckpointState: string; + artifactError: string | undefined; + finished: boolean; + /** Command sent and capture state ready; before-close capture applies from here. */ + started: boolean; + /** Loop start after the command; the finding clock and assessment deadline. */ + start: number; + deadlineAt: number; + screenDeadlineAt: number; +} + +async function floorStart(run: FloorRun, session: ClaudePtySession): Promise { + const { opts, driver, fixture } = run; + run.ownedFilePermissions = (session.pendingFilePermissionFiles ?? []).map(binding => + ({ ...binding, guard: createPlanCountPermissionGuard() })); + run.planningDirectory = session.hermeticConfigDir ? path.join(session.hermeticConfigDir, 'plans') : undefined; + await driver.sleep(8000); // boot grace + auto-trust handler window + run.since = session.mark(); + run.commandStartedAt = driver.now(); + session.send(`${opts.slashCommand} PLAN.md\r`); + run.deliveryOptions = { seed: fixture.seed, sessionId: run.sessionId, + slashCommand: opts.slashCommand, startedAt: run.commandStartedAt }; + run.targetDelivery = readPlanFloorTarget(session.hermeticConfigDir, fixture.cwd, + { ...run.deliveryOptions, now: driver.now() }); + run.saveSnapshot = createPlanCountSnapshotWriter(); + run.started = true; + run.start = driver.now(); + run.deadlineAt = run.start + run.timeoutMs; + run.screenDeadlineAt = driver.monotonic() + run.timeoutMs; + return undefined; +} + +function floorCapture(run: FloorRun, session: ClaudePtySession, observation: object) { + const { fixture } = run; + const recorderStatus = pendingQuestionRecorderStatus(session.pendingQuestionFile, fixture.cwd, session.hermeticConfigDir); + const artifacts = run.saveSnapshot({ skillName: run.opts.skillName, cwd: fixture.cwd, + claudeConfigDir: session.hermeticConfigDir, raw: session.rawOutput(), + visible: session.visibleSince(run.since), viewport: run.viewport, + observation: { ...observation, transcript: run.transcript, publicTools: run.publicTools, pendingQuestion: run.pendingQuestion, + floorReview: run.floorReview, floorAssessment: run.floorAssessment, targetDelivery: run.targetDelivery, commandStartedAt: run.commandStartedAt, + pendingWriteInputs: run.ownedFilePermissions.flatMap(binding => { + const input = readPendingWriteInput(binding.file, binding.expected, fixture.cwd, session.hermeticConfigDir, run.startedAt); + return input ? [input] : []; + }), + questionDiagnostics: { sampledAt: run.sampledAt, parentSessionId: run.sessionId, recorderStatus, recorderStatusAt: run.driver.now(), + nativeCandidates: run.nativeCandidates, validatedPendingQuestion: run.validatedPendingQuestion }, + ...(run.dxContext ? { setupContextReplies: [...run.dxReplies.values()] } : {}), artifactError: run.artifactError } }); + if (artifacts.artifactError) { + run.artifactError ??= artifacts.artifactError; + console.error(`PTY artifact write failed: ${artifacts.artifactError}`); + } + return { ...artifacts, ...(run.artifactError ? { artifactError: run.artifactError } : {}) }; +} + +function floorCheckpoint(run: FloorRun, session: ClaudePtySession): void { + const state = JSON.stringify([pendingQuestionRecorderStatus(session.pendingQuestionFile, run.fixture.cwd, session.hermeticConfigDir), + run.targetDelivery.status, run.nativeCandidates, run.validatedPendingQuestion, run.pendingQuestion, [...run.dxReplies.values()]]); + if (state === run.lastCheckpointState && run.driver.now() - run.lastCheckpointAt < 15_000) return; + run.lastCheckpointState = state; run.lastCheckpointAt = run.driver.now(); + floorCapture(run, session, { state: 'in_progress', elapsedMs: run.driver.now() - run.startedAt }); +} + +function floorBeforeClose(run: FloorRun, session: ClaudePtySession, failure: { error: unknown } | undefined): void { + if (!run.started) return; + if (failure && session.hermeticConfigDir) { + const publicTools: NativePublicToolEvent[] = []; + run.publicTools = publicTools; + run.transcript = readPlanCountTranscript(session.hermeticConfigDir, run.fixture.cwd, event => publicTools.push(event)); + } + if (!run.finished) floorCapture(run, session, { state: failure ? 'threw' : 'in_progress', error: failure ? String(failure.error) : undefined, + captureReason: 'before_cleanup', elapsedMs: run.driver.now() - run.startedAt }); +} + +function floorFinish(run: FloorRun, session: ClaudePtySession, observation: PlanSkillFloorObservation): PlanSkillFloorObservation { + const artifacts = floorCapture(run, session, observation); + run.finished = true; + return { ...observation, targetDelivery: run.targetDelivery, ...artifacts }; +} + +async function floorTick(run: FloorRun, session: ClaudePtySession): Promise> { + const { opts, driver, fixture, sessionId } = run; + const visible = session.visibleSince(run.since); + const finish = (observation: PlanSkillFloorObservation) => ({ done: floorFinish(run, session, observation) }); + if (session.exited()) return finish({ auqObserved: false, outcome: 'exited', + summary: `claude exited (code=${session.exitCode()}) before a qualifying finding`, + evidence: visible.slice(-3000), elapsedMs: driver.now() - run.startedAt }); + if (isRejectedSlashCommand(visible, opts.slashCommand)) return finish({ auqObserved: false, outcome: 'exited', + summary: `claude rejected ${opts.slashCommand} as unknown command`, + evidence: visible.slice(-3000), elapsedMs: driver.now() - run.startedAt }); + + if (run.targetDelivery.status !== 'ready') { + run.targetDelivery = readPlanFloorTarget(session.hermeticConfigDir, fixture.cwd, + { ...run.deliveryOptions, now: driver.now() }); + if (run.targetDelivery.status !== 'ready') { + run.viewport = await session.currentScreen(run.screenDeadlineAt); + floorCheckpoint(run, session); + return 'continue'; } } + + // Current native identity precedes permission handling and finding assessment. + run.floorReview = undefined; run.floorAssessment = undefined; + run.viewport = await session.currentScreen(run.screenDeadlineAt); + const publicTools: NativePublicToolEvent[] = []; + run.publicTools = publicTools; + run.transcript = session.hermeticConfigDir + ? readPlanCountTranscript(session.hermeticConfigDir, fixture.cwd, event => publicTools.push(event)) + : { status: 'missing', calls: [], assistantMessages: [] }; + const hook = readPendingQuestion(session.pendingQuestionFile, fixture.cwd, + session.hermeticConfigDir, run.commandStartedAt, run.transcript); + const currentCalls = run.transcript.status === 'ready' ? run.transcript.calls.filter(call => + call.sessionId === sessionId && !call.answered && !call.failed && publicTools.filter(event => + event.sessionId === sessionId && event.kind === 'use' && event.name === 'AskUserQuestion' && + event.toolUseId === call.toolUseId && Date.parse(event.timestamp) >= run.commandStartedAt && + Date.parse(event.timestamp) <= driver.now() && isDeepStrictEqual(event.input?.questions, call.questions)).length === 1) : []; + run.nativeCandidates = currentCalls.slice(); + run.validatedPendingQuestion = hook; + run.sampledAt = driver.now(); + if (hook?.sessionId === sessionId && !run.transcript.calls.some(call => call.toolUseId === hook.toolUseId)) currentCalls.push(hook); + const activeReply = currentCalls.length === 1 && run.dxReplies.get(`${currentCalls[0]!.sessionId}:${currentCalls[0]!.toolUseId}`); + if (activeReply) { + run.pendingQuestion = undefined; + const next = planFloorDXReplyInput(run.viewport, currentCalls[0]!, activeReply); + if (next) { session.send(next.input); activeReply.stage = next.stage; } + floorCheckpoint(run, session); + return 'continue'; + } + const matching = currentCalls.filter(call => run.dxContext + ? planFloorDXPane(run.viewport, call) !== null : matchesNativePlanQuestion(run.viewport, call, run.planningDirectory)); + run.pendingQuestion = matching.length === 1 ? matching[0] : undefined; + floorCheckpoint(run, session); + if (floorPermission(run, session)) return 'continue'; + const question = floorQuestion(run, session, currentCalls); + if (question) return question; + return floorAssess(run, session, visible); +} + +/** An authorized owned file write enables the review, but never supplies its + * finding question. True when an active permission render owns this tick. */ +function floorPermission(run: FloorRun, session: ClaudePtySession): boolean { + const { fixture, viewport } = run; + const permissionIsActiveRender = !run.pendingQuestion && + (isPermissionDialogVisible(viewport) || isCroppedEditPermissionVisible(viewport)); + if (!permissionIsActiveRender) return false; + // Exclude the permission's old render after approval. + const owned = currentFilePermissionBinding(run.ownedFilePermissions, fixture.cwd, + session.hermeticConfigDir, run.startedAt, run.transcript, viewport); + let ordinaryOwnedTarget = false; + if (fixture.workingPlanPath) try { + const parent = fs.realpathSync(path.dirname(fixture.workingPlanPath)); + let target: fs.Stats | undefined; + try { target = fs.lstatSync(fixture.workingPlanPath); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; } + ordinaryOwnedTarget = parent === fs.realpathSync(fixture.cwd) && (!target || target.isFile()); + } catch { /* No authority for a linked, foreign, or unreadable target. */ } + if (owned && ordinaryOwnedTarget && owned.binding.guard(viewport, session.visibleText(), owned.epoch) === 'grant') + session.send('1\r'); + return true; +} + +/** Answer declared setup choices; otherwise stage the current question (native + * or complete prose) as the floor review candidate. A step ends this tick. */ +function floorQuestion(run: FloorRun, session: ClaudePtySession, currentCalls: NativePlanQuestionCall[]): PtyStep | undefined { + const { opts, fixture, viewport, pendingQuestion } = run; + const questionViewport = run.dxContext && pendingQuestion ? planFloorDXPane(viewport, pendingQuestion)! : viewport; + const fp = pendingQuestion && capturePlanCountQuestion(questionViewport, new Set(), run.driver.now() - run.start, true, pendingQuestion, run.planningDirectory); + if (fp && pendingQuestion) { + const index = fp.nativeQuestionIndex ?? 0; + const question = pendingQuestion.questions[index]!; + const key = `${pendingQuestion.sessionId}:${pendingQuestion.toolUseId}`; + const chosen = run.setupChoices.get(key) ?? new Set(); + const allDesign = opts.skillName === 'plan-design-review' && designReviewSetupAUQ(fp) + ? question.options.flatMap((option, i) => /^(?:Review )?All 7 (?:design )?(?:dimensions|passes)(?:\s*\(recommended\))?$/i.test(option.label.trim()) ? [i + 1] : []) : []; + const pick = pickPlanFloorMode(opts.skillName, question) ?? planCountPrerequisitePick(fp, fp) + ?? (opts.skillName === 'plan-devex-review' ? pickPlanFloorProductType(question, opts.productType) : null) + ?? (allDesign.length === 1 ? allDesign[0]! : null); + if (pick !== null) { + if (!chosen.has(index)) { + session.send(planCountQuestionInput(viewport, fp, pick)); + chosen.add(index); run.setupChoices.set(key, chosen); + } + return 'continue'; + } + run.floorReview = { seed: fixture.seed, candidate: { transport: 'native', identity: `${key}:question:${index}`, + question: structuredClone(question) } }; + return undefined; + } + // Public fallback must be a complete current question, not scrollback, + // a generic idle prompt, permission, tool result or quoted example. + run.floorReview = undefined; + const message = run.transcript.assistantMessages.filter(m => m.sessionId === run.sessionId && + Date.parse(m.timestamp) >= run.commandStartedAt && Date.parse(m.timestamp) <= run.driver.now()).at(-1); + const compact = (text: string) => text.replace(/\s+/g, ''); + if (!currentCalls.length && message && isProseAUQVisible(viewport) && isProseAUQVisible(message.text) && + !/^\s*(?:>|`{3,}|~{3,})/m.test(message.text) && compact(viewport).includes(compact(message.text))) + run.floorReview = { seed: fixture.seed, candidate: { transport: 'prose', + identity: `${message.sessionId}:${message.timestamp}`, text: message.text } }; + const submit = planCountSubmissionInput(viewport); + const packet = currentCalls.find(call => run.setupChoices.get(`${call.sessionId}:${call.toolUseId}`)?.size === call.questions.length); + if (submit && packet) { + const key = `${packet.sessionId}:${packet.toolUseId}`; + if (!run.submittedSetup.has(key)) { session.send(submit); run.submittedSetup.add(key); } + return 'continue'; + } + return undefined; +} + +/** Assess a staged candidate once; then the silent-write and plan-ready terminals. */ +function floorAssess(run: FloorRun, session: ClaudePtySession, visible: string): PtyStep { + const { driver, floorReview, pendingQuestion } = run; + const finish = (observation: PlanSkillFloorObservation) => ({ done: floorFinish(run, session, observation) }); + if (floorReview) { + const key = JSON.stringify(floorReview); + run.floorAssessment = run.assessed.get(key); + if (!run.floorAssessment) { + try { + run.floorAssessment = judgePlanFloorReview(floorReview, { + binary: resolveClaudeBinary() ?? 'claude', model: resolveEvalModel('warmup'), deadlineAt: run.deadlineAt }); + } catch (error) { + return finish({ auqObserved: false, outcome: 'assessment_error', + summary: `Finding assessment failed: ${error instanceof Error ? error.message : String(error)}`, + evidence: run.viewport, elapsedMs: driver.now() - run.startedAt }); + } + run.assessed.set(key, run.floorAssessment); + } + if (run.dxContext && run.floorAssessment.kind === 'setup' && pendingQuestion?.questions.length === 1 && + !pendingQuestion.questions[0]!.multiSelect && floorReview.candidate.transport === 'native') { + const replyKey = `${pendingQuestion.sessionId}:${pendingQuestion.toolUseId}`; + const questionViewport = planFloorDXPane(run.viewport, pendingQuestion)!; + run.dxReplies.set(replyKey, { call: structuredClone(pendingQuestion), pane: questionViewport, reply: run.dxContext, stage: 'focus' }); + } + if (run.floorAssessment.kind === 'finding') return finish({ + auqObserved: true, outcome: 'auq_observed', + summary: `Current ${floorReview.candidate.transport} question addresses the owned seeded finding: ${run.floorAssessment.reason}`, + evidence: run.viewport, elapsedMs: driver.now() - run.startedAt, + }); + } + + // Silent write outside sanctioned dirs is the transcript-bug shape. + const writeRe = /⏺\s*(?:Write|Edit)\(([^)]+)\)/g; + let m: RegExpExecArray | null; + while ((m = writeRe.exec(visible)) !== null) { + const target = m[1] ?? ''; + const sanctioned = SANCTIONED_WRITE_SUBSTRINGS.some((s) => target.includes(s)); + if (!sanctioned && !isNumberedOptionListVisible(visible)) { + return finish({ + auqObserved: false, + outcome: 'silent_write', + summary: `Write/Edit to ${target} fired before any AskUserQuestion`, + evidence: visible.slice(-3000), + elapsedMs: driver.now() - run.startedAt, + }); + } + } + + // Reached terminal without AUQ → transcript-bug regression. + // Note: COMPLETION_SUMMARY_RE is intentionally NOT checked here — it + // matches "GSTACK REVIEW REPORT" anywhere in the buffer, including + // when the agent does recon by reading existing plan files (which + // contain that string as a generated section). The plan_ready check + // (claude's actual "Ready to execute" confirmation) is the reliable + // terminal signal for "agent finished without asking." + if (isPlanReadyVisible(visible)) { + return finish({ + auqObserved: false, + outcome: 'plan_ready', + summary: 'agent reached plan_ready without a qualifying finding question', + evidence: visible.slice(-3000), + elapsedMs: driver.now() - run.startedAt, + }); + } + return 'continue'; } diff --git a/test/helpers/pty/runners/observation.ts b/test/helpers/pty/runners/observation.ts index c50ed4f3b..d1141df8e 100644 --- a/test/helpers/pty/runners/observation.ts +++ b/test/helpers/pty/runners/observation.ts @@ -13,9 +13,9 @@ import { randomUUID } from 'node:crypto'; import { classifyVisible, isProseAUQVisible, isScopeGateAutoSelectVisible, isScopeGateQuestionVisible } from '../classify'; import { judgePtyState, logPtySnapshot } from '../judge'; import type { PtyStateVerdict } from '../judge'; -import { launchClaudePty } from '../launch'; +import type { ClaudePtySession } from '../launch'; import { extractPlanFilePath, isNumberedOptionListVisible, isPermissionDialogVisible, isPlanReadyVisible, isRejectedSlashCommand } from '../screen'; -import type { PtyDriver } from '../session'; +import { realPtyDriver, runPtySession, type PtyDriver, type PtyStep } from '../session'; // --------------------------------------------------------------------------- // High-level skill-mode test contract @@ -116,24 +116,8 @@ export interface PlanSkillObservation { tokensObserved?: Record; } -/** - * The contract for "skill X invoked in plan mode behaves correctly." - * - * PASS: outcome is 'asked' or 'plan_ready'. - * - 'asked' = the skill is gating decisions on the user, as expected. - * - 'plan_ready' = the skill ran end-to-end, wrote a plan file, and - * surfaced claude's native confirmation. Some skills (like - * plan-design-review on a no-UI branch) legitimately reach plan_ready - * without firing AskUserQuestion because they short-circuit. - * - * FAIL: 'silent_write' or 'exited' or 'timeout'. - * - * This replaces the SDK-based runPlanModeSkillTest which never worked - * because plan mode renders its native confirmation as TTY UI, not via - * the AskUserQuestion tool — so canUseTool never fired and the assertion - * counted zero questions. - */ -export async function runPlanSkillObservation(opts: { +/** Options for runPlanSkillObservation. */ +export interface PlanSkillObservationOptions { /** Skill name, e.g. 'plan-ceo-review'. */ skillName: string; /** Whether to launch in plan mode. Default true. The no-op regression @@ -186,279 +170,331 @@ export async function runPlanSkillObservation(opts: { trackTokens?: string[]; /** Launch seam and clock; tests pass the fake driver. Default: real launcher and clocks. */ driver?: PtyDriver; -}): Promise { - const driver = opts.driver ?? { launch: launchClaudePty, now: () => Date.now(), - monotonic: () => performance.now(), sleep: (ms: number) => Bun.sleep(ms) }; +} + +/** + * The contract for "skill X invoked in plan mode behaves correctly." + * + * PASS: outcome is 'asked' or 'plan_ready'. + * - 'asked' = the skill is gating decisions on the user, as expected. + * - 'plan_ready' = the skill ran end-to-end, wrote a plan file, and + * surfaced claude's native confirmation. Some skills (like + * plan-design-review on a no-UI branch) legitimately reach plan_ready + * without firing AskUserQuestion because they short-circuit. + * + * FAIL: 'silent_write' or 'exited' or 'timeout'. + * + * This replaces the SDK-based runPlanModeSkillTest which never worked + * because plan mode renders its native confirmation as TTY UI, not via + * the AskUserQuestion tool — so canUseTool never fired and the assertion + * counted zero questions. + */ +export async function runPlanSkillObservation(opts: PlanSkillObservationOptions): Promise { + const driver = opts.driver ?? realPtyDriver; const startedAt = driver.now(); const budgetMs = opts.timeoutMs ?? 180_000; - const deadlineAt = startedAt + budgetMs; - const screenDeadlineAt = driver.monotonic() + budgetMs; // Explicitly identify only a new seeded plan-mode session. Caller-owned // resume/session arguments retain their existing behavior. const scopeSessionId = opts.initialPlanContent && opts.inPlanMode !== false && !opts.extraArgs?.some(arg => /^(?:--session-id|--resume|--continue|-r|-c)(?:=|$)/.test(arg)) ? randomUUID() : undefined; - const readAutoDecisionState = opts.autoDecisionState - ? bindAutoDecisionState(opts.autoDecisionState, opts.env, opts.skillName) : undefined; - const saveSnapshot = createPlanCountSnapshotWriter(); - const session = await driver.launch({ - permissionMode: opts.inPlanMode === false ? null : 'plan', - cwd: opts.cwd, - timeoutMs: (opts.timeoutMs ?? 180_000) + 30_000, - extraArgs: [...(opts.extraArgs ?? []), ...(scopeSessionId ? ['--session-id', scopeSessionId] : [])], - env: { - ...(opts.inPlanMode !== false && !opts.extraArgs?.some(arg => /^--permission-mode(?:=|$)/.test(arg)) - ? { GSTACK_PLAN_MODE: 'active' } : {}), - ...opts.env, + const run: ObservationRun = { + opts, driver, startedAt, budgetMs, deadlineAt: startedAt + budgetMs, scopeSessionId, + readAutoDecisionState: opts.autoDecisionState + ? bindAutoDecisionState(opts.autoDecisionState, opts.env, opts.skillName) : undefined, + saveSnapshot: createPlanCountSnapshotWriter(), + since: 0, commandStartedAt: 0, start: 0, lastJudgeAt: 0, lastJudgeVerdict: null, + proseAUQEverObserved: false, waitingEverObserved: false, + scopeGateQuestionObserved: false, scopeGateAutoSelectObserved: false, + scopeTranscript: { status: 'missing', calls: [], assistantMessages: [] }, scopeTools: [], + nativeAutoDecide: null, nativePolledAt: null, + tokensObserved: Object.fromEntries((opts.trackTokens ?? []).map(t => [t, false])), + }; + return runPtySession({ + driver, + launch: { + permissionMode: opts.inPlanMode === false ? null : 'plan', + cwd: opts.cwd, + timeoutMs: (opts.timeoutMs ?? 180_000) + 30_000, + extraArgs: [...(opts.extraArgs ?? []), ...(scopeSessionId ? ['--session-id', scopeSessionId] : [])], + env: { + ...(opts.inPlanMode !== false && !opts.extraArgs?.some(arg => /^--permission-mode(?:=|$)/.test(arg)) + ? { GSTACK_PLAN_MODE: 'active' } : {}), + ...opts.env, + }, + model: opts.model, + seedSkills: true, + observeScreen: !!opts.initialPlanContent, + screenDeadlineAt: driver.monotonic() + budgetMs, }, - model: opts.model, - seedSkills: true, - observeScreen: !!opts.initialPlanContent, - screenDeadlineAt, + start: session => observationStart(run, session), + poll: async () => { + if (!(driver.now() < run.deadlineAt)) return false; + await driver.sleep(Math.min(2000, Math.max(0, run.deadlineAt - driver.now()))); + return true; + }, + tick: session => observationTick(run, session), + timeout: session => observationTimeout(run, session), + onError: (session, error) => observationFailure(run, session, error), }); +} - let observationFailed = false; - try { - const preflightTimeout = async (summary: string): Promise => { - let viewport: string | undefined, viewportError: string | undefined; - try { viewport = (await session.currentScreenFrame())?.text; } catch (error) { viewportError = String(error); } - const artifacts = saveSnapshot({ skillName: opts.skillName, cwd: path.resolve(opts.cwd ?? process.cwd()), - claudeConfigDir: session.hermeticConfigDir, raw: session.rawOutput(), visible: session.visibleText(), viewport, - observation: { state: 'plan_skill_preflight_timeout', summary, scopeSessionId, startedAt, deadlineAt, viewportError } }); - return { - outcome: 'timeout', summary, evidence: session.visibleText().slice(-2000), - elapsedMs: driver.now() - startedAt, - proseAUQEverObserved: false, waitingEverObserved: false, - scopeGateQuestionObserved: false, scopeGateAutoSelectObserved: false, - ...(opts.trackTokens?.length ? { tokensObserved: Object.fromEntries(opts.trackTokens.map(t => [t, false])) } : {}), - ...artifacts, - }; - }; - // Entry deadline → boot → owned paste/receipt/ack → slash → observation. - // Setup consumes the existing case budget; cleanup has its separate grace. - if (!opts.initialPlanContent) await driver.sleep(Math.min(8000, Math.max(0, deadlineAt - driver.now()))); - if (opts.initialPlanContent) { - const seed = `Keep this draft plan as context. Briefly acknowledge receipt, then wait for my next message containing a slash command. Do not start the review or call tools yet.\n\n${opts.initialPlanContent}`; - try { - await submitPlanSeed({...session, currentScreen: session.currentScreenFrame}, seed, { - cwd: opts.cwd ?? process.cwd(), launchedAt: startedAt, deadlineAt, - isQuestionOrPermission: text => isProseAUQVisible(text) || isNumberedOptionListVisible(text) || isPermissionDialogVisible(text), - }); - } catch (error) { - if (!(error instanceof PlanSeedTimeout)) throw error; - return await preflightTimeout(`Plan seed submission failed: ${error.message}`); - } - } - if (driver.now() >= deadlineAt) return await preflightTimeout('Boot or seed preflight exhausted the existing case budget'); - const commandStartedAt = driver.now(); - const since = session.mark(); - session.send(`/${opts.skillName}\r`); +interface ObservationRun { + opts: PlanSkillObservationOptions; + driver: PtyDriver; + startedAt: number; + budgetMs: number; + deadlineAt: number; + scopeSessionId: string | undefined; + readAutoDecisionState: ReturnType | undefined; + saveSnapshot: ReturnType; + since: number; + commandStartedAt: number; + start: number; + lastJudgeAt: number; + lastJudgeVerdict: PtyStateVerdict | null; + // High-water marks: did we EVER see a prose-AUQ surface or a judge + // 'waiting' verdict during the run? Models may surface options + // briefly, then resume thinking when no user response comes (test + // env has no responder). At timeout we trust historical signals + // even if the current state is 'working'. + proseAUQEverObserved: boolean; + waitingEverObserved: boolean; + scopeGateQuestionObserved: boolean; + scopeGateAutoSelectObserved: boolean; + scopeTranscript: PlanCountTranscript; + scopeTools: NativePublicToolEvent[]; + nativeAutoDecide: NativeAutoDecision | null; + nativePolledAt: number | null; + tokensObserved: Record; +} - const start = driver.now(); - let lastJudgeAt = 0; - let lastJudgeVerdict: PtyStateVerdict | null = null; - // High-water marks: did we EVER see a prose-AUQ surface or a judge - // 'waiting' verdict during the run? Models may surface options - // briefly, then resume thinking when no user response comes (test - // env has no responder). At timeout we trust historical signals - // even if the current state is 'working'. - let proseAUQEverObserved = false; - let waitingEverObserved = false; - let scopeGateQuestionObserved = false; - let scopeGateAutoSelectObserved = false; - let scopeTranscript: PlanCountTranscript = { status: 'missing', calls: [], assistantMessages: [] }; - let scopeTools: NativePublicToolEvent[] = []; - let nativeAutoDecide: NativeAutoDecision | null = null; - let nativePolledAt: number | null = null; - const tokensObserved: Record = {}; - for (const t of opts.trackTokens ?? []) tokensObserved[t] = false; - // Single source for the high-water flags at EVERY return site. Hand- - // spreading them per-site already drifted once (the judge-waiting return - // omitted the prose/waiting flags); a site that forgets a must-stay-false - // flag makes `obs.flag ?? false` negative assertions pass vacuously. - const highWaterFlags = () => { - const flags = { proseAUQEverObserved, waitingEverObserved, - scopeGateQuestionObserved, scopeGateAutoSelectObserved, - ...(nativeAutoDecide ? { nativeAutoDecide } : {}), - ...(opts.trackTokens?.length ? { tokensObserved } : {}) }; - // Preserve the measured terminal flags and public evidence before the - // hermetic session is removed, including when a later assertion fails. - const artifacts = saveSnapshot({ skillName: opts.skillName, - cwd: path.resolve(opts.cwd ?? process.cwd()), claudeConfigDir: session.hermeticConfigDir, - raw: session.rawOutput(), visible: session.visibleSince(since), - observation: { state: 'plan_skill_observation_terminal', ...flags, - commandStartedAt, scopeSessionId, native: scopeTranscript, publicTools: scopeTools, - nativePolledAt, nativeScope: 'Latest owned native poll, refreshed while observing; not an exhaustive terminal journal.' } }); - return { ...flags, ...artifacts }; - }; - const JUDGE_AFTER_MS = 60_000; - const JUDGE_INTERVAL_MS = 30_000; - while (driver.now() < deadlineAt) { - await driver.sleep(Math.min(2000, Math.max(0, deadlineAt - driver.now()))); - const visible = session.visibleSince(since); +async function observationPreflightTimeout(run: ObservationRun, session: ClaudePtySession, summary: string): Promise { + const { opts } = run; + let viewport: string | undefined, viewportError: string | undefined; + try { viewport = (await session.currentScreenFrame())?.text; } catch (error) { viewportError = String(error); } + const artifacts = run.saveSnapshot({ skillName: opts.skillName, cwd: path.resolve(opts.cwd ?? process.cwd()), + claudeConfigDir: session.hermeticConfigDir, raw: session.rawOutput(), visible: session.visibleText(), viewport, + observation: { state: 'plan_skill_preflight_timeout', summary, scopeSessionId: run.scopeSessionId, + startedAt: run.startedAt, deadlineAt: run.deadlineAt, viewportError } }); + return { + outcome: 'timeout', summary, evidence: session.visibleText().slice(-2000), + elapsedMs: run.driver.now() - run.startedAt, + proseAUQEverObserved: false, waitingEverObserved: false, + scopeGateQuestionObserved: false, scopeGateAutoSelectObserved: false, + ...(opts.trackTokens?.length ? { tokensObserved: Object.fromEntries(opts.trackTokens.map(t => [t, false])) } : {}), + ...artifacts, + }; +} - if (session.exited()) { - return { - outcome: 'exited', - summary: `claude exited (code=${session.exitCode()}) before reaching a terminal outcome`, - evidence: visible.slice(-2000), - elapsedMs: driver.now() - startedAt, - ...highWaterFlags(), - }; - } - if (isRejectedSlashCommand(visible, `/${opts.skillName}`)) { - return { - outcome: 'exited', - summary: `claude rejected /${opts.skillName} as unknown command (skill not registered in this cwd)`, - evidence: visible.slice(-2000), - elapsedMs: driver.now() - startedAt, - ...highWaterFlags(), - }; - } - - const classified = classifyVisible(visible, { - strictPlanWrites: !!opts.initialPlanContent, - currentScreen: opts.initialPlanContent ? await session.currentScreen() : undefined, - }); - const pendingSeededCompletion = !!opts.initialPlanContent && - isPlanReadyVisible(visible) && classified === null; - - // Cheap surface-tracking: did the model ever surface a prose AUQ in - // this tick's recent buffer? Track once-true (high water). - if (!proseAUQEverObserved && !pendingSeededCompletion && isProseAUQVisible(visible)) { - proseAUQEverObserved = true; - logPtySnapshot(visible, { - testName: opts.skillName, - elapsedMs: driver.now() - start, - tag: 'prose-auq-surfaced', - }); - } - // Scope-gate render tracking (same high-water shape). Full-run - // detection matters because the 2KB evidence tail usually scrolls - // past the gate render before the outcome fires. - if (!scopeGateQuestionObserved && isScopeGateQuestionVisible(visible)) { - scopeGateQuestionObserved = true; - } - if (!scopeGateAutoSelectObserved && isScopeGateAutoSelectVisible(visible)) { - scopeGateAutoSelectObserved = true; - } - // Keep reading after scope selection: an AUTO_DECIDE annotation may - // arrive later, and a prior poll cannot establish its current ownership. - if (scopeSessionId && opts.initialPlanContent && session.hermeticConfigDir) { - scopeTools = []; - scopeTranscript = readPlanCountTranscript(session.hermeticConfigDir, - path.resolve(opts.cwd ?? process.cwd()), event => scopeTools.push(event)); - nativePolledAt = driver.now(); - if (!scopeGateAutoSelectObserved) scopeGateAutoSelectObserved = nativeSeededPlanSelection(scopeTranscript, scopeTools, { - seed: opts.initialPlanContent, skillName: opts.skillName, sessionId: scopeSessionId, commandStartedAt, - }); - } - for (const t of opts.trackTokens ?? []) { - if (!tokensObserved[t] && visible.includes(t)) tokensObserved[t] = true; - } - - if (classified) { - const obs: PlanSkillObservation = { - ...classified, - evidence: visible.slice(-2000), - elapsedMs: driver.now() - startedAt, - ...highWaterFlags(), - }; - // Capture the plan file path on any outcome where one may have been - // written. Gating only on 'plan_ready' missed two cases: (1) the - // 'asked' outcome where the model wrote a plan partway through then - // paused on a question, and (2) 'wrote_findings_before_asking' where - // the bug is precisely that the plan was written. The - // assertReviewReportAtBottom checks downstream gate on planFile - // existing, not on the outcome. - const planFile = extractPlanFilePath(visible); - if (planFile) obs.planFile = planFile; - return obs; - } - - // Terminal classification retains precedence (including actual questions - // and writes). Only an unclassified frame may use owned native auto-decision evidence. - if (scopeSessionId) { - nativeAutoDecide = findNativeAutoDecision(scopeTranscript, scopeTools, { - skillName: opts.skillName, sessionId: scopeSessionId, commandStartedAt, now: driver.now(), proseQuestionObserved: proseAUQEverObserved, - stateEvidence: readAutoDecisionState?.(), - }); - if (nativeAutoDecide) return { - outcome: 'auto_decided', - summary: 'owned native session completed the saved-preference auto-decision', - evidence: visible.slice(-2000), elapsedMs: driver.now() - startedAt, - ...highWaterFlags(), - }; - } - - // LLM judge fallback: if regex detectors didn't classify and we've - // burned >60s with periodic ticks, ask Haiku "is the model waiting, - // working, or hung?" Treat 'waiting' as 'asked' (model surfaced a - // question via prose the regex couldn't reassemble). Snapshot the - // visible buffer at each judge call when GSTACK_PTY_LOG=1. - const elapsed = driver.now() - start; - if (elapsed > JUDGE_AFTER_MS && driver.now() - lastJudgeAt > JUDGE_INTERVAL_MS) { - lastJudgeAt = driver.now(); - logPtySnapshot(visible, { testName: opts.skillName, elapsedMs: elapsed, tag: 'judge-tick' }); - lastJudgeVerdict = judgePtyState(visible, { testName: opts.skillName }); - if (lastJudgeVerdict.state === 'waiting' && !pendingSeededCompletion) { - waitingEverObserved = true; - if (opts.requireProseEvidence && !proseAUQEverObserved) continue; - return { - outcome: 'asked', - summary: `LLM judge: ${lastJudgeVerdict.reasoning} (state=waiting after ${Math.round(elapsed / 1000)}s)`, - evidence: visible.slice(-2000), - elapsedMs: driver.now() - startedAt, - ...highWaterFlags(), - }; - } - } - } - - // Timeout fallback: if we observed a prose-AUQ surface OR a judge - // 'waiting' verdict at any point during the run, treat as 'asked'. - // This catches the model-surfaced-then-resumed-thinking case where - // by the time the timeout fires, the buffer has moved past the - // options into spinner state but the question DID surface earlier. - const finalVisible = session.visibleSince(since); - if (proseAUQEverObserved || waitingEverObserved && !opts.requireProseEvidence) { - return { - outcome: 'asked', - summary: - `prose-AUQ surface observed during run (proseAUQEverObserved=${proseAUQEverObserved}, waitingEverObserved=${waitingEverObserved}); model surfaced the question and the test budget elapsed without a follow-up classification` + - (lastJudgeVerdict - ? ` (last LLM judge: ${lastJudgeVerdict.state} — ${lastJudgeVerdict.reasoning})` - : ''), - evidence: finalVisible.slice(-2000), - elapsedMs: driver.now() - startedAt, - ...highWaterFlags(), - }; - } - return { - outcome: 'timeout', - summary: - `no terminal outcome within ${budgetMs}ms` + - (lastJudgeVerdict - ? ` (last LLM judge: state=${lastJudgeVerdict.state} — ${lastJudgeVerdict.reasoning})` - : ''), - evidence: finalVisible.slice(-2000), - elapsedMs: driver.now() - startedAt, - ...highWaterFlags(), - }; - } catch (error) { - observationFailed = true; +/** Entry deadline → boot → owned paste/receipt/ack → slash → observation. + * Setup consumes the existing case budget; cleanup has its separate grace. */ +async function observationStart(run: ObservationRun, session: ClaudePtySession): Promise { + const { opts, driver } = run; + if (!opts.initialPlanContent) await driver.sleep(Math.min(8000, Math.max(0, run.deadlineAt - driver.now()))); + if (opts.initialPlanContent) { + const seed = `Keep this draft plan as context. Briefly acknowledge receipt, then wait for my next message containing a slash command. Do not start the review or call tools yet.\n\n${opts.initialPlanContent}`; try { - const publicTools: NativePublicToolEvent[] = []; - const transcript = session.hermeticConfigDir ? readPlanCountTranscript(session.hermeticConfigDir, - path.resolve(opts.cwd ?? process.cwd()), event => publicTools.push(event)) : undefined; - const saved = saveSnapshot({ skillName: opts.skillName, cwd: path.resolve(opts.cwd ?? process.cwd()), - claudeConfigDir: session.hermeticConfigDir, raw: session.rawOutput(), visible: session.visibleText(), - observation: { state: 'threw', error: String(error), transcript, publicTools } }); - if (saved.artifactError) console.error(`PTY artifact write failed: ${saved.artifactError}`); - } catch (captureError) { console.error(`PTY failure capture failed: ${String(captureError)}`); } - throw error; - } finally { - try { await session.close(); } - catch (error) { if (!observationFailed) throw error; } + await submitPlanSeed({...session, currentScreen: session.currentScreenFrame}, seed, { + cwd: opts.cwd ?? process.cwd(), launchedAt: run.startedAt, deadlineAt: run.deadlineAt, + isQuestionOrPermission: text => isProseAUQVisible(text) || isNumberedOptionListVisible(text) || isPermissionDialogVisible(text), + }); + } catch (error) { + if (!(error instanceof PlanSeedTimeout)) throw error; + return await observationPreflightTimeout(run, session, `Plan seed submission failed: ${error.message}`); + } + } + if (driver.now() >= run.deadlineAt) return await observationPreflightTimeout(run, session, 'Boot or seed preflight exhausted the existing case budget'); + run.commandStartedAt = driver.now(); + run.since = session.mark(); + session.send(`/${opts.skillName}\r`); + run.start = driver.now(); + return undefined; +} + +/** Single source for the high-water flags at EVERY return site. Hand- + * spreading them per-site already drifted once (the judge-waiting return + * omitted the prose/waiting flags); a site that forgets a must-stay-false + * flag makes `obs.flag ?? false` negative assertions pass vacuously. */ +function observationFlags(run: ObservationRun, session: ClaudePtySession) { + const { opts } = run; + const flags = { proseAUQEverObserved: run.proseAUQEverObserved, waitingEverObserved: run.waitingEverObserved, + scopeGateQuestionObserved: run.scopeGateQuestionObserved, scopeGateAutoSelectObserved: run.scopeGateAutoSelectObserved, + ...(run.nativeAutoDecide ? { nativeAutoDecide: run.nativeAutoDecide } : {}), + ...(opts.trackTokens?.length ? { tokensObserved: run.tokensObserved } : {}) }; + // Preserve the measured terminal flags and public evidence before the + // hermetic session is removed, including when a later assertion fails. + const artifacts = run.saveSnapshot({ skillName: opts.skillName, + cwd: path.resolve(opts.cwd ?? process.cwd()), claudeConfigDir: session.hermeticConfigDir, + raw: session.rawOutput(), visible: session.visibleSince(run.since), + observation: { state: 'plan_skill_observation_terminal', ...flags, + commandStartedAt: run.commandStartedAt, scopeSessionId: run.scopeSessionId, native: run.scopeTranscript, publicTools: run.scopeTools, + nativePolledAt: run.nativePolledAt, nativeScope: 'Latest owned native poll, refreshed while observing; not an exhaustive terminal journal.' } }); + return { ...flags, ...artifacts }; +} + +const JUDGE_AFTER_MS = 60_000; +const JUDGE_INTERVAL_MS = 30_000; + +async function observationTick(run: ObservationRun, session: ClaudePtySession): Promise> { + const { opts, driver } = run; + const visible = session.visibleSince(run.since); + const ended = (summary: string): PtyStep => ({ done: { + outcome: 'exited', summary, evidence: visible.slice(-2000), elapsedMs: driver.now() - run.startedAt, + ...observationFlags(run, session), + } }); + if (session.exited()) return ended(`claude exited (code=${session.exitCode()}) before reaching a terminal outcome`); + if (isRejectedSlashCommand(visible, `/${opts.skillName}`)) { + return ended(`claude rejected /${opts.skillName} as unknown command (skill not registered in this cwd)`); + } + + const classified = classifyVisible(visible, { + strictPlanWrites: !!opts.initialPlanContent, + currentScreen: opts.initialPlanContent ? await session.currentScreen() : undefined, + }); + const pendingSeededCompletion = !!opts.initialPlanContent && + isPlanReadyVisible(visible) && classified === null; + observeHighWaterMarks(run, session, visible, pendingSeededCompletion); + + if (classified) { + const obs: PlanSkillObservation = { + ...classified, + evidence: visible.slice(-2000), + elapsedMs: driver.now() - run.startedAt, + ...observationFlags(run, session), + }; + // Capture the plan file path on any outcome where one may have been + // written. Gating only on 'plan_ready' missed two cases: (1) the + // 'asked' outcome where the model wrote a plan partway through then + // paused on a question, and (2) 'wrote_findings_before_asking' where + // the bug is precisely that the plan was written. The + // assertReviewReportAtBottom checks downstream gate on planFile + // existing, not on the outcome. + const planFile = extractPlanFilePath(visible); + if (planFile) obs.planFile = planFile; + return { done: obs }; + } + + // Terminal classification retains precedence (including actual questions + // and writes). Only an unclassified frame may use owned native auto-decision evidence. + if (run.scopeSessionId) { + run.nativeAutoDecide = findNativeAutoDecision(run.scopeTranscript, run.scopeTools, { + skillName: opts.skillName, sessionId: run.scopeSessionId, commandStartedAt: run.commandStartedAt, now: driver.now(), proseQuestionObserved: run.proseAUQEverObserved, + stateEvidence: run.readAutoDecisionState?.(), + }); + if (run.nativeAutoDecide) return { done: { + outcome: 'auto_decided', + summary: 'owned native session completed the saved-preference auto-decision', + evidence: visible.slice(-2000), elapsedMs: driver.now() - run.startedAt, + ...observationFlags(run, session), + } }; + } + + // LLM judge fallback: if regex detectors didn't classify and we've + // burned >60s with periodic ticks, ask Haiku "is the model waiting, + // working, or hung?" Treat 'waiting' as 'asked' (model surfaced a + // question via prose the regex couldn't reassemble). Snapshot the + // visible buffer at each judge call when GSTACK_PTY_LOG=1. + const elapsed = driver.now() - run.start; + if (elapsed > JUDGE_AFTER_MS && driver.now() - run.lastJudgeAt > JUDGE_INTERVAL_MS) { + run.lastJudgeAt = driver.now(); + logPtySnapshot(visible, { testName: opts.skillName, elapsedMs: elapsed, tag: 'judge-tick' }); + run.lastJudgeVerdict = judgePtyState(visible, { testName: opts.skillName }); + if (run.lastJudgeVerdict.state === 'waiting' && !pendingSeededCompletion) { + run.waitingEverObserved = true; + if (opts.requireProseEvidence && !run.proseAUQEverObserved) return 'continue'; + return { done: { + outcome: 'asked', + summary: `LLM judge: ${run.lastJudgeVerdict.reasoning} (state=waiting after ${Math.round(elapsed / 1000)}s)`, + evidence: visible.slice(-2000), + elapsedMs: driver.now() - run.startedAt, + ...observationFlags(run, session), + } }; + } + } + return 'continue'; +} + +/** Cheap per-tick surface tracking; each flag is once-true (high water). */ +function observeHighWaterMarks(run: ObservationRun, session: ClaudePtySession, visible: string, pendingSeededCompletion: boolean): void { + const { opts, driver } = run; + if (!run.proseAUQEverObserved && !pendingSeededCompletion && isProseAUQVisible(visible)) { + run.proseAUQEverObserved = true; + logPtySnapshot(visible, { + testName: opts.skillName, + elapsedMs: driver.now() - run.start, + tag: 'prose-auq-surfaced', + }); + } + // Scope-gate render tracking (same high-water shape). Full-run + // detection matters because the 2KB evidence tail usually scrolls + // past the gate render before the outcome fires. + if (!run.scopeGateQuestionObserved && isScopeGateQuestionVisible(visible)) { + run.scopeGateQuestionObserved = true; + } + if (!run.scopeGateAutoSelectObserved && isScopeGateAutoSelectVisible(visible)) { + run.scopeGateAutoSelectObserved = true; + } + // Keep reading after scope selection: an AUTO_DECIDE annotation may + // arrive later, and a prior poll cannot establish its current ownership. + if (run.scopeSessionId && opts.initialPlanContent && session.hermeticConfigDir) { + const scopeTools: NativePublicToolEvent[] = []; + run.scopeTools = scopeTools; + run.scopeTranscript = readPlanCountTranscript(session.hermeticConfigDir, + path.resolve(opts.cwd ?? process.cwd()), event => scopeTools.push(event)); + run.nativePolledAt = driver.now(); + if (!run.scopeGateAutoSelectObserved) run.scopeGateAutoSelectObserved = nativeSeededPlanSelection(run.scopeTranscript, run.scopeTools, { + seed: opts.initialPlanContent, skillName: opts.skillName, sessionId: run.scopeSessionId, commandStartedAt: run.commandStartedAt, + }); + } + for (const t of opts.trackTokens ?? []) { + if (!run.tokensObserved[t] && visible.includes(t)) run.tokensObserved[t] = true; } } + +/** Timeout fallback: if we observed a prose-AUQ surface OR a judge + * 'waiting' verdict at any point during the run, treat as 'asked'. + * This catches the model-surfaced-then-resumed-thinking case where + * by the time the timeout fires, the buffer has moved past the + * options into spinner state but the question DID surface earlier. */ +function observationTimeout(run: ObservationRun, session: ClaudePtySession): PlanSkillObservation { + const { opts, driver, lastJudgeVerdict } = run; + const finalVisible = session.visibleSince(run.since); + if (run.proseAUQEverObserved || run.waitingEverObserved && !opts.requireProseEvidence) { + return { + outcome: 'asked', + summary: + `prose-AUQ surface observed during run (proseAUQEverObserved=${run.proseAUQEverObserved}, waitingEverObserved=${run.waitingEverObserved}); model surfaced the question and the test budget elapsed without a follow-up classification` + + (lastJudgeVerdict + ? ` (last LLM judge: ${lastJudgeVerdict.state} — ${lastJudgeVerdict.reasoning})` + : ''), + evidence: finalVisible.slice(-2000), + elapsedMs: driver.now() - run.startedAt, + ...observationFlags(run, session), + }; + } + return { + outcome: 'timeout', + summary: + `no terminal outcome within ${run.budgetMs}ms` + + (lastJudgeVerdict + ? ` (last LLM judge: state=${lastJudgeVerdict.state} — ${lastJudgeVerdict.reasoning})` + : ''), + evidence: finalVisible.slice(-2000), + elapsedMs: driver.now() - run.startedAt, + ...observationFlags(run, session), + }; +} + +function observationFailure(run: ObservationRun, session: ClaudePtySession, error: unknown): void { + const { opts } = run; + try { + const publicTools: NativePublicToolEvent[] = []; + const transcript = session.hermeticConfigDir ? readPlanCountTranscript(session.hermeticConfigDir, + path.resolve(opts.cwd ?? process.cwd()), event => publicTools.push(event)) : undefined; + const saved = run.saveSnapshot({ skillName: opts.skillName, cwd: path.resolve(opts.cwd ?? process.cwd()), + claudeConfigDir: session.hermeticConfigDir, raw: session.rawOutput(), visible: session.visibleText(), + observation: { state: 'threw', error: String(error), transcript, publicTools } }); + if (saved.artifactError) console.error(`PTY artifact write failed: ${saved.artifactError}`); + } catch (captureError) { console.error(`PTY failure capture failed: ${String(captureError)}`); } +} diff --git a/test/helpers/pty/session.ts b/test/helpers/pty/session.ts index 3ee82bb5a..68f56d907 100644 --- a/test/helpers/pty/session.ts +++ b/test/helpers/pty/session.ts @@ -1,8 +1,13 @@ /** - * The runners' launch seam and clock (PtyDriver). Moved from claude-pty-runner.ts. - * Import through test/helpers/claude-pty-runner.ts from tests; pty/ modules import siblings directly. + * The one PTY session loop (runPtySession) and its launch seam (PtyDriver). + * Owns launch, the poll/deadline loop, failure capture and close/cleanup + * ordering; each runner in pty/runners/ supplies its policy as a PtySessionPlan + * (start, poll, tick, timeout, capture hooks). New runner: build a plan and + * `return runPtySession(plan)`; tests pass the fake driver from fake-session.ts. + * Size ratchet (c) binds this module. Moved from the three runner loops of the + * former claude-pty-runner.ts. Tests import through claude-pty-runner.ts. */ -import type { ClaudePtyOptions, ClaudePtySession } from './launch'; +import { launchClaudePty, type ClaudePtyOptions, type ClaudePtySession } from './launch'; /** * The runners' launch seam: how a session starts, and the clock the runner @@ -18,3 +23,71 @@ export interface PtyDriver { monotonic(): number; sleep(ms: number): Promise; } + +export const realPtyDriver: PtyDriver = { + launch: opts => launchClaudePty(opts), + now: () => Date.now(), + monotonic: () => performance.now(), + sleep: ms => Bun.sleep(ms), +}; + +/** One poll step: finish with a result, keep polling, or end the loop as a timeout. */ +export type PtyStep = { done: R } | 'continue' | 'break'; + +export interface PtySessionPlan { + driver: PtyDriver; + launch: ClaudePtyOptions; + /** Owned fixture cleanup; runs after close, and when launch itself fails. */ + cleanup?: () => void; + /** Boot and command submission. A returned result ends the run before polling. */ + start(session: ClaudePtySession): Promise; + /** Wait for the next poll. False means the deadline passed: the run times out. */ + poll(session: ClaudePtySession): Promise; + tick(session: ClaudePtySession): Promise>; + timeout(session: ClaudePtySession): R; + /** Snapshot a thrown run before close; the original error is rethrown. */ + onError?(session: ClaudePtySession, error: unknown): void; + /** Always runs before close. A throw here fails an otherwise successful run. */ + beforeClose?(session: ClaudePtySession, failure: { error: unknown } | undefined): void; + /** Close failed after a successful run; the close error is rethrown. */ + onCloseError?(session: ClaudePtySession, error: unknown): void; +} + +/** + * launch → start → (poll → tick)* → timeout, with one failure contract: a + * run's own error wins over capture and close errors, close always runs, and + * owned cleanup runs last. + */ +export async function runPtySession(plan: PtySessionPlan): Promise { + let session: ClaudePtySession; + try { + session = await plan.driver.launch(plan.launch); + } catch (error) { + plan.cleanup?.(); + throw error; + } + let failure: { error: unknown } | undefined; + try { + const early = await plan.start(session); + if (early !== undefined) return early; + while (await plan.poll(session)) { + const step = await plan.tick(session); + if (step === 'break') break; + if (step !== 'continue') return step.done; + } + return plan.timeout(session); + } catch (error) { + failure = { error }; + plan.onError?.(session, error); + throw error; + } finally { + try { + try { plan.beforeClose?.(session, failure); } + catch (error) { if (!failure) { failure = { error }; throw error; } } + finally { + try { await session.close(); } + catch (error) { if (!failure) { plan.onCloseError?.(session, error); throw error; } } + } + } finally { plan.cleanup?.(); } + } +} diff --git a/test/plan-count-completion.test.ts b/test/plan-count-completion.test.ts index 1b3d2fca3..a59688a5c 100644 --- a/test/plan-count-completion.test.ts +++ b/test/plan-count-completion.test.ts @@ -9,7 +9,7 @@ import type { PlanCountTranscript } from './helpers/plan-count-transcript'; import capturedL from './fixtures/devex-review-l-calls.json'; import designStatusCapture from './fixtures/design-count-native-issue-fields.json'; import designEnvelope from './fixtures/design-completion-envelope-90f.json'; -import { nativePlanCallFingerprint } from './helpers/claude-pty-runner'; +import { nativePlanCallFingerprint, isNativeCompletionSummary } from './helpers/claude-pty-runner'; import type { NativePlanQuestionCall } from './helpers/plan-count-transcript'; import captured_ceo_completion_handoff_m from './fixtures/ceo-completion-handoff-m-call.json'; import nextStepCapture_ceo_completion_handoff_m from './fixtures/ceo-handoff-n-calls.json'; @@ -48,13 +48,9 @@ describe('captured Design completion envelope', () => { }); test('the actual runner branch recognizes the idle native envelope without answering an exit gate',()=>{ const f=completedEnvelope();try { - const source=fs.readFileSync(path.join(import.meta.dir,'helpers/pty/runners/counting.ts'),'utf8'); - const expression=/const nativeSummary = ([\s\S]+?);\n const terminalFrame/.exec(source)?.[1];expect(expression).toBeTruthy(); - const evaluate=new Function('opts','nativeCompletion','renderedFrame','visible','transcript','startedAt','administrative', - 'isNumberedOptionListVisible','isPermissionDialogVisible','isProseAUQVisible','hasNativePlanTerminal',`return (${expression});`); const screen=designEnvelope.screen.replaceAll(designEnvelope.originalPlanPath,f.file); - const check=()=>evaluate({expectedPlanPath:f.file},hasNativePlanCompletion(f.transcript,f.file,designEnvelope.startedAt),classifyPlanCountFrame(screen),screen,f.transcript, - designEnvelope.startedAt,new Set(),isNumberedOptionListVisible,isPermissionDialogVisible,isProseAUQVisible,hasNativePlanTerminal); + const check=()=>isNativeCompletionSummary({expectedPlanPath:f.file,nativeCompletion:hasNativePlanCompletion(f.transcript,f.file,designEnvelope.startedAt), + renderedFrame:classifyPlanCountFrame(screen),visible:screen,transcript:f.transcript,startedAt:designEnvelope.startedAt,administrative:new Set()}); expect(check()).toBe(true); f.transcript.calls[0]!.answered=false;expect(check()).toBe(false); }finally{f.cleanup();} diff --git a/test/plan-create-prepublication.test.ts b/test/plan-create-prepublication.test.ts index 5724134c1..0cad80e06 100644 --- a/test/plan-create-prepublication.test.ts +++ b/test/plan-create-prepublication.test.ts @@ -5,7 +5,7 @@ import * as path from 'node:path'; import {createHash} from 'node:crypto'; import {createFilePermissionRecorder,recordFilePermission,currentFilePermissionEpoch,readPendingWriteInput} from './helpers/plan-count-file-permission'; import {readPlanCountTranscript} from './helpers/plan-count-transcript'; -import {createPlanCountPermissionGuard} from './helpers/claude-pty-runner'; +import {createPlanCountPermissionGuard,countingCapture,type CountingRun} from './helpers/claude-pty-runner'; import {createPlanCountSnapshotWriter} from './helpers/plan-count-artifacts'; import captured from './fixtures/plan-create-prepublication-491.json'; @@ -137,22 +137,18 @@ test('legacy published Write metadata remains sufficient without a sidecar',()=> }); // Execute only the actual count capture assembly with its real reader/writer. -// This is not a full counting-loop execution; existing count throw/timeout tests +// This calls the counting runner's capture step directly, not a full loop; existing count throw/timeout tests // separately prove that the lifecycle invokes capture before fixture disposal. test('count capture assembly retains exact prepublication input through refresh, throw and fixture cleanup',()=>{ const f=fixture(),evalDir=fs.mkdtempSync(path.join(os.tmpdir(),'count-write-retention-')); try { f.hook();const expected=readPendingWriteInput(f.recorder.file,f.expected,f.cwd,f.config,f.startedAt); expect(expected).toBeDefined(); - const source=fs.readFileSync(path.join(import.meta.dir,'helpers/pty/runners/counting.ts'),'utf8'); - const start=source.indexOf(' const capture = (observation: object) => {',source.indexOf('export async function runPlanSkillCounting(')); - const end=source.indexOf('\n function snapshot(',start); - expect(start).toBeGreaterThan(0);expect(end).toBeGreaterThan(start); - const code=new Bun.Transpiler({loader:'ts'}).transformSync(source.slice(start,end)+'\nreturn capture;'); - const capture=new Function('saveSnapshot','opts','ownedFilePermissions','readPendingWriteInput','readPlanCountTranscript','fixture','session','startedAt','viewport',code)( - createPlanCountSnapshotWriter({EVALS_RUN_ID:'count-prepublication-free',GSTACK_EVAL_DIR:evalDir}), - {skillName:'plan-eng-review'},[{file:f.recorder.file,expected:f.expected}],readPendingWriteInput,readPlanCountTranscript, - {cwd:f.cwd},{hermeticConfigDir:f.config,rawOutput:()=>f.screen,visibleText:()=>f.screen},f.startedAt,f.screen); + // The runner's own capture step, called with the state it builds for an attempt. + const run={opts:{skillName:'plan-eng-review'},fixture:{cwd:f.cwd},startedAt:f.startedAt,viewport:f.screen, + ownedFilePermissions:[{file:f.recorder.file,expected:f.expected}], + saveSnapshot:createPlanCountSnapshotWriter({EVALS_RUN_ID:'count-prepublication-free',GSTACK_EVAL_DIR:evalDir})} as unknown as CountingRun; + const capture=(observation:object)=>countingCapture(run,{hermeticConfigDir:f.config,rawOutput:()=>f.screen,visibleText:()=>f.screen},observation); const progress=capture({state:'in_progress'}); expect(progress.artifactError).toBeUndefined();expect(progress.artifactDir).toBeDefined(); const initial=JSON.parse(fs.readFileSync(path.join(progress.artifactDir,'observation.json'),'utf8')); diff --git a/test/plan-floor-permission.test.ts b/test/plan-floor-permission.test.ts index ecdb5c9b7..4593bd5a4 100644 --- a/test/plan-floor-permission.test.ts +++ b/test/plan-floor-permission.test.ts @@ -259,13 +259,14 @@ async function exercise(mode: Mode, kind: keyof typeof SEEDS = 'ceo', capture?: }, }; const run = new Function(...Object.keys(boundary), body + '\nreturn runPlanSkillFloorCheck;')(...Object.values(boundary)); + const driver: runner.PtyDriver = {launch: boundary.launchClaudePty, now: () => now, monotonic: () => now, sleep: boundary.Bun.sleep}; try { let result: any, error: unknown; try { result = await run({skillName:`plan-${kind}-review`, slashCommand:`/plan-${kind}-review`, followUpPrompt:SEEDS[kind], productType:mode==='product-type'||mode.startsWith('dx-')?'sdk-documentation':undefined, devexSetupContext:mode.startsWith('dx-')&&mode!=='dx-undeclared'?dxCustom.reply:undefined, - requestedPlanPath:mode === 'captured' ? undefined : `/tmp/gstack-test-plan-${kind}-floor.md`, timeoutMs:100_000}); + requestedPlanPath:mode === 'captured' ? undefined : `/tmp/gstack-test-plan-${kind}-floor.md`, timeoutMs:100_000, driver}); } catch(caught) { if(!snapshotOptions?.interrupt)throw caught; error=caught; } expect(closed).toBe(1); expect(fs.existsSync(fixture!.cwd)).toBe(false); expect(recorders.every(recorder=>!fs.existsSync(recorder.file))).toBe(true);