Files
gstack/test/helpers/pty/runners/floor.ts
Garry Tanandcapy-ai[bot] 6f7488a70f v1.91.185.0 fix(evals): plan-design focus rating in plain words, docsync own-call rule where --help is named, floor Create crop and 0D menu (GSTA-65) (#3358)
* fix(evals): plan-design focus rating in plain words, docsync own-call rule at the candidate --help, floor Create menu cropped at its rule (GSTA-65)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(evals): floor actor answers Step 0D's focus menu in the template's own words (GSTA-65)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(evals): floor actor identifies Step 0D's focus menu by its structure, not one wording (GSTA-65)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs(changelog): GSTA-65 paid-side flaky repairs under [Unreleased]

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(evals): the 0D focus picker moves into plan-design-finding-floor's owned fixture; no harness series reset (GSTA-65)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test(evals): the 0D picker rows join the floor case's owner test (test-of-test ratchet)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* v1.91.185.0 fix(evals): plan-design focus rating in plain words, docsync own-call rule, floor Create crop and 0D menu (GSTA-65) (#3358)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
2026-10-23 18:35:59 +00:00

546 lines
32 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* runPlanSkillFloorCheck. Moved from claude-pty-runner.ts.
* Import through test/helpers/claude-pty-runner.ts from tests; pty/ modules import siblings directly.
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import { resolveEvalModel } from '../../../../lib/eval-model';
import { createPlanCountFixture } from '../../plan-count-fixture';
import { createPlanCountSnapshotWriter } from '../../plan-count-artifacts';
import { readPlanFloorTarget, type PlanFloorTargetDelivery } from '../../plan-floor-target';
import { judgePlanFloorReview, pickPlanFloorMode, pickPlanFloorProductType, type PlanFloorReview, type PlanFloorAssessment } from '../../plan-floor-review';
import { readPlanCountTranscript, type NativePlanQuestionCall, type PlanCountTranscript, type NativePublicToolEvent } from '../../plan-count-transcript';
import { readPendingQuestion, pendingQuestionRecorderStatus } from '../../plan-count-pending-question';
import { currentFilePermissionBinding, readPendingWriteInput, isCroppedEditPermissionVisible } from '../../plan-count-file-permission';
import { isDeepStrictEqual } from 'node:util';
import { randomUUID } from 'node:crypto';
import { capturePlanCountQuestion, createPlanCountPermissionGuard, matchesNativePlanQuestion, planCountPrerequisitePick, planCountQuestionInput } from '../auq';
import { resolveClaudeBinary } from '../binary';
import { SANCTIONED_WRITE_SUBSTRINGS, isProseAUQVisible, planCountSubmissionInput } from '../classify';
import type { ClaudePtySession } from '../launch';
import { isNumberedOptionListVisible, isPermissionDialogVisible, isPlanReadyVisible, isRejectedSlashCommand, stripPtyResidue } from '../screen';
import { realPtyDriver, runPtySession, type PtyDriver, type PtyStep } from '../session';
// ────────────────────────────────────────────────────────────────────────────
// runPlanSkillFloorCheck — stop at the first substantive seeded question.
// Current owned input → complete question → evidence-backed assessment → outcome.
// Setup answers advance only the predeclared review interface. Findings are
// observed without answering them; permissions and generic waiting never count.
// The existing model-work deadline also bounds the replacement waiting judge.
// ────────────────────────────────────────────────────────────────────────────
export interface PlanSkillFloorObservation {
/** True iff a review-phase AUQ render was observed. */
auqObserved: boolean;
/** Owned native acknowledgment of the exact seeded target command. */
targetDelivery?: PlanFloorTargetDelivery;
outcome:
| 'auq_observed'
| 'plan_ready'
| 'silent_write'
| 'exited'
| 'timeout'
| 'assessment_error'
| 'permission_unanswered';
summary: string;
/** Public current viewport for an accepted finding; terminal tail otherwise. */
evidence: string;
/** Wall time (ms) until the outcome was decided. */
elapsedMs: number;
}
/**
* Drive a plan-* skill and qualify its first current seeded finding question.
* The actor answers only its declared optional-prerequisite, mode and product-type choices.
*/
/** DX's long empathy prompt needs its whole native pane, not an interior crop.
* Retain the actual header and complete pinned renderer prefix; the shared
* matcher still authenticates the pending question and native menu. */
export function planFloorDXPane(visible: string, call: NativePlanQuestionCall): string | null {
if (call.answered || call.failed || call.questions.length !== 1) return null;
const text = stripPtyResidue(visible).replace(/\r+\n?/g, '\n')
.replace(/((?:^|\n)Enter\s+to\s+select\s*·\s*↑\/↓\s+to\s+navigate\s*·\s*)ctrl\+g\s+to\s+edit\s+in[ \t]+[^\s·\x00-\x1f\x7f][^·\x00-\x1f\x7f]*?\s*·\s*(Esc\s+to\s+cancel\s*)$/, '$1$2');
const headers = [...text.matchAll(/(?:^|\n)[\t ]*[☐□][^\n]*\n/g)];
const header = headers.at(-1);
if (!header) return null;
const preceding = text.slice(0, header.index).trimEnd();
if (preceding && !/(?:^|\n)[ \t]*[─━]{10,}[ \t]*$/.test(preceding)) return null;
let fence: string | undefined;
for (const line of preceding.split('\n')) {
const marker = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(line);
if (!marker) continue;
if (!fence) fence = marker[1];
else if (marker[1]![0] === fence[0] && marker[1]!.length >= fence.length && !marker[2]!.trim()) fence = undefined;
}
if (fence) return null;
const pane = text.slice(header.index).trimStart();
const cursor = /(?:^|\n)❯\s*1\./.exec(pane);
if (!cursor) return null;
const body = pane.slice(pane.indexOf('\n') + 1, cursor.index)
.replace(/^[ \t]*[│┃] ?|[│┃][ \t]*$/gm, '').trim();
const compact = (value: string) => value.replace(/\s+/g, '');
const q = call.questions[0]!;
const displayed = q.question.length > 2000 ? q.question.slice(0, 2000) + '…' : q.question;
if (compact(body) !== compact(displayed) || !matchesNativePlanQuestion(pane, call)) return null;
return pane;
}
export interface PlanFloorDXReply {
call: NativePlanQuestionCall;
pane: string;
reply: string;
stage: 'focus' | 'paste' | 'submit' | 'done';
}
/** The native custom field focuses first, then accepts literal bracketed paste.
* Each step binds the same unanswered call and unchanged complete pane. */
export function planFloorDXReplyInput(visible: string, call: NativePlanQuestionCall,
state: PlanFloorDXReply): { input: string; stage: PlanFloorDXReply['stage'] } | null {
if (state.stage === 'done' || call.answered || call.failed || call.questions.length !== 1 ||
call.questions[0]!.multiSelect || call.sessionId !== state.call.sessionId || call.toolUseId !== state.call.toolUseId ||
!isDeepStrictEqual(call.questions, state.call.questions) || !state.reply.trim() || state.reply.length > 1400 ||
/[\x00-\x1f\x7f]/.test(state.reply)) return null;
const compact = (value: string) => value.replace(/\s+/g, '');
const index = call.questions[0]!.options.length + 1;
if (state.stage === 'focus') {
const pane = planFloorDXPane(visible, call);
if (!pane || compact(pane) !== compact(state.pane) ||
!new RegExp(`(?:^|\\n) ${index}\\. Type something\\.[ \\t]*(?:\\n|$)`).test(pane)) return null;
return { input: String(index), stage: 'paste' };
}
const lines = stripPtyResidue(visible).replace(/\r+\n?/g, '\n').split('\n');
const start = lines.findIndex(line => new RegExp(`^❯ ${index}\\. `).test(line));
if (start < 0 || lines.filter(line => /^❯ [1-9]\. /.test(line)).length !== 1) return null;
let end = start + 1;
while (end < lines.length && /^ {5}\S|^ {5,}\S/.test(lines[end]!)) end++;
const field = [lines[start]!.replace(new RegExp(`^❯ ${index}\\. `), ''),
...lines.slice(start + 1, end).map(line => line.trim())].join(' ').trim();
if (field !== (state.stage === 'paste' ? 'Type something.' : state.reply)) return null;
lines.splice(start, end - start, ` ${index}. Type something.`);
const first = lines.findIndex(line => /^ 1\. /.test(line));
if (first < 0) return null;
lines[first] = lines[first]!.replace(/^ 1\./, '❯ 1.');
// Only the final footer line is presentation; question text that resembles it stays verbatim.
const footer = lines.findLastIndex(line => line.trim().length > 0);
lines[footer] = lines[footer]!.replace(/^[ \t]*(Enter to select · ↑\/↓ to navigate(?: · n to add notes)?) · ctrl\+g to edit in [^\x00-\x1f\x7f·]+( · Esc to cancel)[ \t]*$/, '$1$2');
const pane = planFloorDXPane(lines.join('\n'), call);
if (!pane || compact(pane) !== compact(state.pane)) return null;
return state.stage === 'paste'
? { input: '\x1b[200~' + state.reply + '\x1b[201~', stage: 'submit' }
: { input: '\r', stage: 'done' };
}
/** Options for runPlanSkillFloorCheck. */
export interface PlanSkillFloorOptions {
/** Skill name, e.g. 'plan-eng-review'. Used for diagnostic strings only. */
skillName: string;
/** Slash command; the owned PLAN.md target is supplied in the same submission. */
slashCommand: string;
/** Complete request seeded in an isolated project before the command starts. */
followUpPrompt: string;
/** Explicit working-plan request relocated into the owned fixture before launch. */
requestedPlanPath?: string;
/** Predeclared DX fixture classification; never inferred from a recommendation. */
productType?: 'sdk-documentation';
/** Literal persona/journey correction declared before DX setup; approves no offered remedy. */
devexSetupContext?: string;
/** Case-owned answer to a declared setup menu (the design seed's Step 0D focus choice); null leaves the question to the floor review. */
pickSetupChoice?: (question: NativePlanQuestionCall['questions'][number]) => number | null;
/** Installation cwd retained for caller compatibility; review uses an owned seeded project. */
cwd?: string;
/** Total budget. Default 600000 (10 min). Tests exit early on AUQ. */
timeoutMs?: number;
/** Extra env merged into the spawned `claude` process. */
env?: Record<string, string>;
/** Override the spawned model. Defaults via launchClaudePty's chain. */
model?: string;
/** Launch seam and clock; tests pass the fake driver. Default: real launcher and clocks. */
driver?: PtyDriver;
}
export async function runPlanSkillFloorCheck(opts: PlanSkillFloorOptions): Promise<PlanSkillFloorObservation> {
const driver = opts.driver ?? realPtyDriver;
const startedAt = driver.now();
const timeoutMs = opts.timeoutMs ?? 600_000;
const dxContext = opts.devexSetupContext;
if (dxContext !== undefined && (opts.skillName !== 'plan-devex-review' || opts.productType !== 'sdk-documentation' ||
!dxContext.trim() || dxContext.length > 1400 || /[\x00-\x1f\x7f]/.test(dxContext)))
throw Error('DX setup context requires the declared SDK-documentation actor and a bounded literal single line');
const request = [
'Proceed directly to the requested review; skip the optional /office-hours prerequisite.',
'This actor has already declined routing setup, cross-project recall and outside reviewers.',
'Preserve the supplied product scope. For review-mode questions choose HOLD SCOPE (CEO), DX POLISH (DX), or the full BIG CHANGE review (Eng). Design: review all seven dimensions.',
// Runs 36626737820 and 36776104571: CEO read scope preservation as approving
// approach A, so the seeded premise gap never reached a question. Run 36794871032:
// CEO read this request as supplying every answer up front and asked nothing.
...(opts.skillName === 'plan-ceo-review' ? ['Preserving scope does not approve the plan\'s premise, approach or any remedy. The routing, recall, outside-reviewer and review-mode choices above are already answered. Premise, approach and remedy decisions remain unanswered; follow the skill\'s normal interactive decision workflow for those choices.'] : []),
...(opts.productType === 'sdk-documentation' ? [
'Product type is confirmed: SDK quickstart documentation, with the complete journey to the first SDK call as context. If asked to classify, choose SDK + Docs when offered, otherwise Documentation. This confirms the review lens; it does not expand the plan.',
'Target persona is confirmed: a hands-on developer integrating this SDK for the first time, trying to make one successful call. Product type and persona setup are already answered; proceed to reviewing the supplied plan.',
] : []),
...(dxContext ? ['For setup confirmations, this actor can supply only the following persona/journey correction through the native custom answer. It does not approve a proposed narrative, remedy, or scope change: ' + dxContext] : []),
opts.followUpPrompt,
].join('\n\n');
const fixture = createPlanCountFixture(request, { requestedPlanPath: opts.requestedPlanPath,
nativeReviewOnly: true, preconfiguredReviewActor: true });
const sessionId = randomUUID();
const run = { opts, driver, startedAt, timeoutMs, dxContext, fixture, sessionId,
ownedFilePermissions: [], planningDirectory: undefined, transcript: { status: 'missing', calls: [], assistantMessages: [] },
viewport: '', publicTools: [], pendingQuestion: undefined, floorReview: undefined, floorAssessment: undefined,
setupChoices: new Map(), submittedSetup: new Set(), assessed: new Map(), dxReplies: new Map(),
since: 0, commandStartedAt: 0, targetDelivery: undefined, saveSnapshot: undefined, nativeCandidates: [],
validatedPendingQuestion: undefined, sampledAt: undefined, lastCheckpointAt: 0, lastCheckpointState: '',
artifactError: undefined, finished: false, started: false, start: 0, deadlineAt: 0, screenDeadlineAt: 0,
permissionUnansweredSince: undefined,
} as unknown as FloorRun;
return runPtySession<PlanSkillFloorObservation>({
driver, budgetMs: timeoutMs,
cleanup: () => fixture.cleanup(),
launch: {
permissionMode: 'plan',
cwd: fixture.cwd,
timeoutMs: timeoutMs + 60_000,
env: { ...opts.env, ...fixture.env },
model: opts.model,
seedSkills: true,
observeScreen: true,
observeSetupQuestions: true,
...(dxContext ? { rows: 80 } : {}),
observeFilePermissions: fixture.workingPlanPath ? [fixture.workingPlanPath] : undefined,
extraArgs: ['--session-id', sessionId],
},
start: session => floorStart(run, session),
poll: async () => {
if (!(driver.now() - run.start < timeoutMs)) return false;
await driver.sleep(2000);
return true;
},
tick: session => floorTick(run, session),
timeout: session => floorFinish(run, session, {
auqObserved: false,
outcome: 'timeout',
summary: run.targetDelivery.status === 'ready'
? `no qualifying finding question within ${timeoutMs}ms`
: `seeded target delivery unavailable within ${timeoutMs}ms: ${run.targetDelivery.reason ?? run.targetDelivery.status}`,
evidence: session.visibleSince(run.since).slice(-3000),
elapsedMs: driver.now() - startedAt,
}),
beforeClose: (session, failure) => floorBeforeClose(run, session, failure),
});
}
/** Mutable state of one floor attempt, shared by its session-loop steps. */
interface FloorRun {
opts: PlanSkillFloorOptions;
driver: PtyDriver;
startedAt: number;
timeoutMs: number;
dxContext: string | undefined;
fixture: ReturnType<typeof createPlanCountFixture>;
sessionId: string;
ownedFilePermissions: Array<{ expected: string; file: string; guard: ReturnType<typeof createPlanCountPermissionGuard> }>;
planningDirectory: string | undefined;
transcript: PlanCountTranscript;
viewport: string;
publicTools: NativePublicToolEvent[];
pendingQuestion: NativePlanQuestionCall | undefined;
floorReview: PlanFloorReview | undefined;
floorAssessment: PlanFloorAssessment | undefined;
setupChoices: Map<string, Set<number>>;
submittedSetup: Set<string>;
assessed: Map<string, PlanFloorAssessment>;
dxReplies: Map<string, PlanFloorDXReply>;
since: number;
commandStartedAt: number;
deliveryOptions: { seed: string; sessionId: string; slashCommand: string; startedAt: number };
targetDelivery: PlanFloorTargetDelivery;
saveSnapshot: ReturnType<typeof createPlanCountSnapshotWriter>;
nativeCandidates: NativePlanQuestionCall[];
validatedPendingQuestion: ReturnType<typeof readPendingQuestion>;
sampledAt: number | undefined;
lastCheckpointAt: number;
lastCheckpointState: string;
artifactError: string | undefined;
finished: boolean;
/** Command sent and capture state ready; before-close capture applies from here. */
started: boolean;
/** Loop start after the command; the finding clock and assessment deadline. */
start: number;
deadlineAt: number;
screenDeadlineAt: number;
/** When the current permission render was first left unanswered by this actor. */
permissionUnansweredSince: number | undefined;
}
async function floorStart(run: FloorRun, session: ClaudePtySession): Promise<undefined> {
const { opts, driver, fixture } = run;
run.ownedFilePermissions = (session.pendingFilePermissionFiles ?? []).map(binding =>
({ ...binding, guard: createPlanCountPermissionGuard() }));
run.planningDirectory = session.hermeticConfigDir ? path.join(session.hermeticConfigDir, 'plans') : undefined;
await driver.sleep(8000); // boot grace + auto-trust handler window
run.since = session.mark();
run.commandStartedAt = driver.now();
session.send(`${opts.slashCommand} PLAN.md\r`);
run.deliveryOptions = { seed: fixture.seed, sessionId: run.sessionId,
slashCommand: opts.slashCommand, startedAt: run.commandStartedAt };
run.targetDelivery = readPlanFloorTarget(session.hermeticConfigDir, fixture.cwd,
{ ...run.deliveryOptions, now: driver.now() });
run.saveSnapshot = createPlanCountSnapshotWriter();
run.started = true;
run.start = driver.now();
run.deadlineAt = run.start + run.timeoutMs;
run.screenDeadlineAt = driver.monotonic() + run.timeoutMs;
return undefined;
}
function floorCapture(run: FloorRun, session: ClaudePtySession, observation: object) {
const { fixture } = run;
const recorderStatus = pendingQuestionRecorderStatus(session.pendingQuestionFile, fixture.cwd, session.hermeticConfigDir);
const artifacts = run.saveSnapshot({ skillName: run.opts.skillName, cwd: fixture.cwd,
claudeConfigDir: session.hermeticConfigDir, raw: session.rawOutput(),
visible: session.visibleSince(run.since), viewport: run.viewport,
observation: { ...observation, transcript: run.transcript, publicTools: run.publicTools, pendingQuestion: run.pendingQuestion,
floorReview: run.floorReview, floorAssessment: run.floorAssessment, targetDelivery: run.targetDelivery, commandStartedAt: run.commandStartedAt,
pendingWriteInputs: run.ownedFilePermissions.flatMap(binding => {
const input = readPendingWriteInput(binding.file, binding.expected, fixture.cwd, session.hermeticConfigDir, run.startedAt);
return input ? [input] : [];
}),
questionDiagnostics: { sampledAt: run.sampledAt, parentSessionId: run.sessionId, recorderStatus, recorderStatusAt: run.driver.now(),
nativeCandidates: run.nativeCandidates, validatedPendingQuestion: run.validatedPendingQuestion },
...(run.dxContext ? { setupContextReplies: [...run.dxReplies.values()] } : {}), artifactError: run.artifactError } });
if (artifacts.artifactError) {
run.artifactError ??= artifacts.artifactError;
console.error(`PTY artifact write failed: ${artifacts.artifactError}`);
}
return { ...artifacts, ...(run.artifactError ? { artifactError: run.artifactError } : {}) };
}
function floorCheckpoint(run: FloorRun, session: ClaudePtySession): void {
const state = JSON.stringify([pendingQuestionRecorderStatus(session.pendingQuestionFile, run.fixture.cwd, session.hermeticConfigDir),
run.targetDelivery.status, run.nativeCandidates, run.validatedPendingQuestion, run.pendingQuestion, [...run.dxReplies.values()]]);
if (state === run.lastCheckpointState && run.driver.now() - run.lastCheckpointAt < 15_000) return;
run.lastCheckpointState = state; run.lastCheckpointAt = run.driver.now();
floorCapture(run, session, { state: 'in_progress', elapsedMs: run.driver.now() - run.startedAt });
}
function floorBeforeClose(run: FloorRun, session: ClaudePtySession, failure: { error: unknown } | undefined): void {
if (!run.started) return;
if (failure && session.hermeticConfigDir) {
const publicTools: NativePublicToolEvent[] = [];
run.publicTools = publicTools;
run.transcript = readPlanCountTranscript(session.hermeticConfigDir, run.fixture.cwd, event => publicTools.push(event));
}
if (!run.finished) floorCapture(run, session, { state: failure ? 'threw' : 'in_progress', error: failure ? String(failure.error) : undefined,
captureReason: 'before_cleanup', elapsedMs: run.driver.now() - run.startedAt });
}
function floorFinish(run: FloorRun, session: ClaudePtySession, observation: PlanSkillFloorObservation): PlanSkillFloorObservation {
const artifacts = floorCapture(run, session, observation);
run.finished = true;
return { ...observation, targetDelivery: run.targetDelivery, ...artifacts };
}
async function floorTick(run: FloorRun, session: ClaudePtySession): Promise<PtyStep<PlanSkillFloorObservation>> {
const { opts, driver, fixture, sessionId } = run;
const visible = session.visibleSince(run.since);
const finish = (observation: PlanSkillFloorObservation) => ({ done: floorFinish(run, session, observation) });
if (session.exited()) return finish({ auqObserved: false, outcome: 'exited',
summary: `claude exited (code=${session.exitCode()}) before a qualifying finding`,
evidence: visible.slice(-3000), elapsedMs: driver.now() - run.startedAt });
if (isRejectedSlashCommand(visible, opts.slashCommand)) return finish({ auqObserved: false, outcome: 'exited',
summary: `claude rejected ${opts.slashCommand} as unknown command`,
evidence: visible.slice(-3000), elapsedMs: driver.now() - run.startedAt });
if (run.targetDelivery.status !== 'ready') {
run.targetDelivery = readPlanFloorTarget(session.hermeticConfigDir, fixture.cwd,
{ ...run.deliveryOptions, now: driver.now() });
if (run.targetDelivery.status !== 'ready') {
run.viewport = await session.currentScreen(run.screenDeadlineAt);
floorCheckpoint(run, session);
return 'continue';
}
}
// Current native identity precedes permission handling and finding assessment.
run.floorReview = undefined; run.floorAssessment = undefined;
run.viewport = await session.currentScreen(run.screenDeadlineAt);
const publicTools: NativePublicToolEvent[] = [];
run.publicTools = publicTools;
run.transcript = session.hermeticConfigDir
? readPlanCountTranscript(session.hermeticConfigDir, fixture.cwd, event => publicTools.push(event))
: { status: 'missing', calls: [], assistantMessages: [] };
const hook = readPendingQuestion(session.pendingQuestionFile, fixture.cwd,
session.hermeticConfigDir, run.commandStartedAt, run.transcript);
const currentCalls = run.transcript.status === 'ready' ? run.transcript.calls.filter(call =>
call.sessionId === sessionId && !call.answered && !call.failed && publicTools.filter(event =>
event.sessionId === sessionId && event.kind === 'use' && event.name === 'AskUserQuestion' &&
event.toolUseId === call.toolUseId && Date.parse(event.timestamp) >= run.commandStartedAt &&
Date.parse(event.timestamp) <= driver.now() && isDeepStrictEqual(event.input?.questions, call.questions)).length === 1) : [];
run.nativeCandidates = currentCalls.slice();
run.validatedPendingQuestion = hook;
run.sampledAt = driver.now();
if (hook?.sessionId === sessionId && !run.transcript.calls.some(call => call.toolUseId === hook.toolUseId)) currentCalls.push(hook);
const activeReply = currentCalls.length === 1 && run.dxReplies.get(`${currentCalls[0]!.sessionId}:${currentCalls[0]!.toolUseId}`);
if (activeReply) {
run.pendingQuestion = undefined;
const next = planFloorDXReplyInput(run.viewport, currentCalls[0]!, activeReply);
if (next) { session.send(next.input); activeReply.stage = next.stage; }
floorCheckpoint(run, session);
return 'continue';
}
const matching = currentCalls.filter(call => run.dxContext
? planFloorDXPane(run.viewport, call) !== null : matchesNativePlanQuestion(run.viewport, call, run.planningDirectory));
run.pendingQuestion = matching.length === 1 ? matching[0] : undefined;
floorCheckpoint(run, session);
if (floorPermission(run, session)) {
const waited = run.permissionUnansweredSince === undefined ? 0 : driver.now() - run.permissionUnansweredSince;
return waited < PERMISSION_UNANSWERED_MS ? 'continue' : finish({ auqObserved: false, outcome: 'permission_unanswered',
summary: `PERMISSION_DIALOG_UNANSWERED: a Claude Code permission dialog stayed open for ${waited}ms without an owned grant`,
evidence: run.viewport.slice(-3000), elapsedMs: driver.now() - run.startedAt });
}
const question = floorQuestion(run, session, currentCalls);
if (question) return question;
return floorAssess(run, session, visible);
}
/** This actor answers only owned file grants; any other permission render fails fast with a named cause. */
const PERMISSION_UNANSWERED_MS = 20_000;
/** An authorized owned file write enables the review, but never supplies its
* finding question. True when an active permission render owns this tick. */
function floorPermission(run: FloorRun, session: ClaudePtySession): boolean {
const { fixture, viewport } = run;
const permissionIsActiveRender = !run.pendingQuestion &&
(isPermissionDialogVisible(viewport) || isCroppedEditPermissionVisible(viewport));
if (!permissionIsActiveRender) { run.permissionUnansweredSince = undefined; return false; }
// Exclude the permission's old render after approval.
const owned = currentFilePermissionBinding(run.ownedFilePermissions, fixture.cwd,
session.hermeticConfigDir, run.startedAt, run.transcript, viewport);
let ordinaryOwnedTarget = false;
if (fixture.workingPlanPath) try {
const parent = fs.realpathSync(path.dirname(fixture.workingPlanPath));
let target: fs.Stats | undefined;
try { target = fs.lstatSync(fixture.workingPlanPath); }
catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; }
ordinaryOwnedTarget = parent === fs.realpathSync(fixture.cwd) && (!target || target.isFile());
} catch { /* No authority for a linked, foreign, or unreadable target. */ }
const decision = owned && ordinaryOwnedTarget ? owned.binding.guard(viewport, session.visibleText(), owned.epoch) : null;
if (decision === 'grant') session.send('1\r');
// A grant or its lingering render is answered; anything else waits on no one.
run.permissionUnansweredSince = decision ? undefined : run.permissionUnansweredSince ?? run.driver.now();
return true;
}
/** Answer declared setup choices; otherwise stage the current question (native
* or complete prose) as the floor review candidate. A step ends this tick. */
function floorQuestion(run: FloorRun, session: ClaudePtySession, currentCalls: NativePlanQuestionCall[]): PtyStep<PlanSkillFloorObservation> | undefined {
const { opts, fixture, viewport, pendingQuestion } = run;
const questionViewport = run.dxContext && pendingQuestion ? planFloorDXPane(viewport, pendingQuestion)! : viewport;
const fp = pendingQuestion && capturePlanCountQuestion(questionViewport, new Set(), run.driver.now() - run.start, true, pendingQuestion, run.planningDirectory);
if (fp && pendingQuestion) {
const index = fp.nativeQuestionIndex ?? 0;
const question = pendingQuestion.questions[index]!;
const key = `${pendingQuestion.sessionId}:${pendingQuestion.toolUseId}`;
const chosen = run.setupChoices.get(key) ?? new Set<number>();
const pick = pickPlanFloorMode(opts.skillName, question) ?? planCountPrerequisitePick(fp, fp)
?? (opts.skillName === 'plan-devex-review' ? pickPlanFloorProductType(question, opts.productType) : null)
?? opts.pickSetupChoice?.(question) ?? null;
if (pick !== null) {
if (!chosen.has(index)) {
session.send(planCountQuestionInput(viewport, fp, pick));
chosen.add(index); run.setupChoices.set(key, chosen);
}
return 'continue';
}
run.floorReview = { seed: fixture.seed, candidate: { transport: 'native', identity: `${key}:question:${index}`,
question: structuredClone(question) } };
return undefined;
}
// Public fallback must be a complete current question, not scrollback,
// a generic idle prompt, permission, tool result or quoted example.
run.floorReview = undefined;
const message = run.transcript.assistantMessages.filter(m => m.sessionId === run.sessionId &&
Date.parse(m.timestamp) >= run.commandStartedAt && Date.parse(m.timestamp) <= run.driver.now()).at(-1);
const compact = (text: string) => text.replace(/\s+/g, '');
if (!currentCalls.length && message && isProseAUQVisible(viewport) && isProseAUQVisible(message.text) &&
!/^\s*(?:>|`{3,}|~{3,})/m.test(message.text) && compact(viewport).includes(compact(message.text)))
run.floorReview = { seed: fixture.seed, candidate: { transport: 'prose',
identity: `${message.sessionId}:${message.timestamp}`, text: message.text } };
const submit = planCountSubmissionInput(viewport);
const packet = currentCalls.find(call => run.setupChoices.get(`${call.sessionId}:${call.toolUseId}`)?.size === call.questions.length);
if (submit && packet) {
const key = `${packet.sessionId}:${packet.toolUseId}`;
if (!run.submittedSetup.has(key)) { session.send(submit); run.submittedSetup.add(key); }
return 'continue';
}
return undefined;
}
/** Assess a staged candidate once; then the silent-write and plan-ready terminals. */
function floorAssess(run: FloorRun, session: ClaudePtySession, visible: string): PtyStep<PlanSkillFloorObservation> {
const { driver, floorReview, pendingQuestion } = run;
const finish = (observation: PlanSkillFloorObservation) => ({ done: floorFinish(run, session, observation) });
if (floorReview) {
const key = JSON.stringify(floorReview);
run.floorAssessment = run.assessed.get(key);
if (!run.floorAssessment) {
try {
run.floorAssessment = judgePlanFloorReview(floorReview, {
binary: resolveClaudeBinary() ?? 'claude', model: resolveEvalModel('warmup'), deadlineAt: run.deadlineAt });
} catch (error) {
return finish({ auqObserved: false, outcome: 'assessment_error',
summary: `Finding assessment failed: ${error instanceof Error ? error.message : String(error)}`,
evidence: run.viewport, elapsedMs: driver.now() - run.startedAt });
}
run.assessed.set(key, run.floorAssessment);
}
if (run.dxContext && run.floorAssessment.kind === 'setup' && pendingQuestion?.questions.length === 1 &&
!pendingQuestion.questions[0]!.multiSelect && floorReview.candidate.transport === 'native') {
const replyKey = `${pendingQuestion.sessionId}:${pendingQuestion.toolUseId}`;
const questionViewport = planFloorDXPane(run.viewport, pendingQuestion)!;
run.dxReplies.set(replyKey, { call: structuredClone(pendingQuestion), pane: questionViewport, reply: run.dxContext, stage: 'focus' });
}
if (run.floorAssessment.kind === 'finding') return finish({
auqObserved: true, outcome: 'auq_observed',
summary: `Current ${floorReview.candidate.transport} question addresses the owned seeded finding: ${run.floorAssessment.reason}`,
evidence: run.viewport, elapsedMs: driver.now() - run.startedAt,
});
}
// Silent write outside sanctioned dirs is the transcript-bug shape.
const writeRe = /⏺\s*(?:Write|Edit)\(([^)]+)\)/g;
let m: RegExpExecArray | null;
while ((m = writeRe.exec(visible)) !== null) {
const target = m[1] ?? '';
const sanctioned = SANCTIONED_WRITE_SUBSTRINGS.some((s) => target.includes(s));
if (!sanctioned && !isNumberedOptionListVisible(visible)) {
return finish({
auqObserved: false,
outcome: 'silent_write',
summary: `Write/Edit to ${target} fired before any AskUserQuestion`,
evidence: visible.slice(-3000),
elapsedMs: driver.now() - run.startedAt,
});
}
}
// Reached terminal without AUQ → transcript-bug regression.
// Note: COMPLETION_SUMMARY_RE is intentionally NOT checked here — it
// matches "GSTACK REVIEW REPORT" anywhere in the buffer, including
// when the agent does recon by reading existing plan files (which
// contain that string as a generated section). The plan_ready check
// (claude's actual "Ready to execute" confirmation) is the reliable
// terminal signal for "agent finished without asking."
if (isPlanReadyVisible(visible)) {
return finish({
auqObserved: false,
outcome: 'plan_ready',
summary: 'agent reached plan_ready without a qualifying finding question',
evidence: visible.slice(-3000),
elapsedMs: driver.now() - run.startedAt,
});
}
return 'continue';
}