Files
gstack/test/helpers/plan-count-pending-question.ts
T
Garry TanandOpenAI Codex 06ed920a97 v1.89.0.0 feat: add shared-code extraction audit (#2925)
* feat: bind shared-code review advice to source and branch

* feat: add shared-code extraction audit and scoped review checks

* test: recognize complete source reads and explicit coverage legends

* chore: bump version and changelog (v1.88.0.0)

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* test: capture native review questions and retain public evidence

Capture the actual first public native question with strict ownership and display matching. Preserve terminal failures and raw evidence, and retain SDK completion checks.

* test: recognize verified review evidence and complete fixtures

Recognize complete source and diagram evidence, concrete design and developer-experience decisions, and the complete planted scenario contracts. Preserve negative controls and grading thresholds.

* fix: preserve decision brief structure in native questions

Keep the required pros-and-cons heading and final Net field in native question text. Regenerate host outputs and document the release and evaluation repairs.

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* docs: update project documentation for v1.88.0.0

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* fix: correct eval retry accounting and ship workflow gates

* fix: capture native eval evidence and stabilize CI fixtures

* fix: keep shared-code eval skips read-only

Choose explicit no-change answers instead of mixed fix/preservation options.
Reuse the bounded revalidation prompt for path fixtures so required review
metadata is available without repeated discovery. Preserve source checks,
retry limits, and failed native terminal outcomes.

Add captured-question and callback regressions, plus evaluation selection
coverage for the affected fixtures.

---------

Co-authored-by: OpenAI Codex <noreply@openai.com>
2026-09-24 01:53:58 -04:00

240 lines
14 KiB
TypeScript

/** Pending native AUQ identity only; answers and coverage always come from JSONL. */
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import type { NativePlanQuestion, NativePlanQuestionCall, PlanCountTranscript } from './plan-count-transcript';
const MAX_BYTES = 128 * 1024;
const MAX_IDS = 128;
const REASONS = ['invalid_event', 'input_overflow', 'stdin_timeout', 'record_error', 'concurrent_pending',
'conflicting_replay', 'record_overflow', 'lock_conflict', 'hook_error', 'unknown'] as const;
type RecorderReason = typeof REASONS[number];
const identifier = (v: unknown): v is string => typeof v === 'string' && /^[A-Za-z0-9_-]{1,160}$/.test(v);
const object = (v: unknown): v is Record<string, any> => v !== null && typeof v === 'object' && !Array.isArray(v);
const quote = (v: string) => `'${v.replaceAll("'", "'\\''")}'`;
const keysOnly = (v: Record<string, unknown>, keys: string[]) => Object.keys(v).every(k => keys.includes(k));
function questions(value: unknown): value is NativePlanQuestion[] {
return Array.isArray(value) && value.length >= 1 && value.length <= 4 && value.every(q =>
object(q) && keysOnly(q, ['header', 'question', 'options', 'multiSelect']) &&
typeof q.header === 'string' && q.header.trim() && typeof q.question === 'string' && q.question.trim() &&
(q.multiSelect === undefined || typeof q.multiSelect === 'boolean') &&
Array.isArray(q.options) && q.options.length >= 2 && q.options.length <= 4 &&
q.options.every((o: unknown) => object(o) && keysOnly(o, ['label', 'description']) &&
typeof o.label === 'string' && o.label.trim() && (o.description === undefined || typeof o.description === 'string')) &&
new Set(q.options.map((o: {label:string}) => o.label)).size === q.options.length) &&
new Set(value.map(q => q.header)).size === value.length && new Set(value.map(q => q.question)).size === value.length;
}
/** Completion-only fields from Claude's permission UI; never retained as answers. */
function completionInput(input: Record<string, any>): boolean {
if (!keysOnly(input, ['questions', 'answers', 'annotations'])) return false;
const texts = new Set(input.questions.map((q: NativePlanQuestion) => q.question));
if (input.answers !== undefined && (!object(input.answers) || Object.entries(input.answers).some(([key, value]) =>
!texts.has(key) || typeof value !== 'string'))) return false;
if (input.annotations !== undefined && (!object(input.annotations) || Object.entries(input.annotations).some(([key, value]) =>
!texts.has(key) || !object(value) || !keysOnly(value, ['preview', 'notes']) ||
Object.values(value).some(field => typeof field !== 'string')))) return false;
return true;
}
function questionIdentity(value: NativePlanQuestion[]): string {
// Claude normalizes object-key order after permission collection; question
// order, option order and every supported field must still match.
return JSON.stringify(value.map(q => ({header:q.header, question:q.question, multiSelect:q.multiSelect,
options:q.options.map(o => ({label:o.label, description:o.description}))})));
}
/** One real parent JSONL, inside the owned config; never a subagent or symlink. */
function scopedTranscript(file: unknown, configDir: string, session: string): file is string {
if (typeof file !== 'string' || !path.isAbsolute(file)) return false;
const projects = path.join(configDir, 'projects');
const rel = path.relative(projects, file).split(path.sep);
if (rel.length !== 2 || !rel[0] || rel[0] === '..' || rel[0] === '.' || rel[1] !== `${session}.jsonl`) return false;
try {
return fs.lstatSync(file).isFile() && !fs.lstatSync(projects).isSymbolicLink() &&
!fs.lstatSync(path.dirname(file)).isSymbolicLink() &&
fs.realpathSync(file) === path.join(fs.realpathSync(configDir), 'projects', ...rel);
} catch { return false; }
}
interface Pending {
sessionId: string; toolUseId: string; transcriptPath: string; timestamp: string; questions: NativePlanQuestion[];
}
interface State {
version: 1; cwd: string; configDir: string; sessionId?: string; seenIds: string[]; pending: Pending | null;
}
function readState(file: string, cwd: string, configDir: string): State {
const stat = fs.lstatSync(file);
if (!stat.isFile() || stat.size > MAX_BYTES) throw Error('invalid recorder file');
const s = JSON.parse(fs.readFileSync(file, 'utf8'));
if (!object(s) || s.version !== 1 || s.cwd !== cwd || s.configDir !== configDir ||
(s.sessionId !== undefined && !identifier(s.sessionId)) || !Array.isArray(s.seenIds) ||
s.seenIds.length > MAX_IDS || !s.seenIds.every(identifier) || new Set(s.seenIds).size !== s.seenIds.length ||
(s.pending !== null && (!object(s.pending) || !identifier(s.pending.sessionId) ||
s.pending.sessionId !== s.sessionId || !identifier(s.pending.toolUseId) ||
!s.seenIds.includes(s.pending.toolUseId) || !questions(s.pending.questions) ||
typeof s.pending.timestamp !== 'string' || !scopedTranscript(s.pending.transcriptPath, configDir, s.pending.sessionId)))) {
throw Error('invalid recorder state');
}
return s as State;
}
export function createPendingQuestionRecorder(cwd: string, configDir: string) {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-pending-question-'));
const file = path.join(dir, 'state.json');
fs.writeFileSync(file, JSON.stringify({version:1, cwd, configDir, seenIds:[], pending:null}) + '\n', {mode:0o600});
// Git Bash needs slash-separated command paths. Recorder arguments retain
// their native spelling because the owned event/state identity is exact.
const shellPath = (value: string) => process.platform === 'win32' ? value.replaceAll('\\', '/') : value;
const command = [shellPath(process.execPath), shellPath(import.meta.path), '--record', file, cwd, configDir].map(quote).join(' ');
const hook = {matcher:'^AskUserQuestion$', hooks:[{type:'command', command, timeout:5}]};
return {file, hooks:{PreToolUse:[hook], PostToolUse:[hook], PostToolUseFailure:[hook]},
dispose: () => fs.rmSync(dir, {recursive:true, force:true})};
}
/** Malformed/ambiguous owned input stays closed; later replay cannot revive it. */
function poison(file: string, reason: RecorderReason) {
try { fs.writeFileSync(file + '.invalid', JSON.stringify({reason})+'\n', {mode:0o600, flag:'wx'}); } catch { /* already invalid or disposed */ }
}
/** Content-free diagnostic survives in caller snapshots even after close removes the recorder. */
export function pendingQuestionRecorderStatus(file: string | undefined, cwd: string, configDir: string | null):
{status:'disabled'|'missing'|'busy'|'idle'|'pending'|'invalid'; reason?:RecorderReason} {
if (!file || !configDir) return {status:'disabled'};
try {
if (fs.existsSync(file+'.invalid')) {
const stat = fs.lstatSync(file+'.invalid');
const reason = stat.isFile() && stat.size <= 1024 ? JSON.parse(fs.readFileSync(file+'.invalid','utf8')).reason : 'unknown';
return {status:'invalid', reason:REASONS.includes(reason) ? reason : 'unknown'};
}
if (fs.existsSync(file+'.lock')) return {status:'busy'};
if (!fs.existsSync(file)) return {status:'missing'};
return {status:readState(file,cwd,configDir).pending ? 'pending' : 'idle'};
} catch { return {status:'invalid',reason:'record_error'}; }
}
/** Silent observation: no approval, answers, input rewrite, or model context. */
export function recordPendingQuestion(input: string, file: string, cwd: string, configDir: string): void {
let lock: number | undefined;
let reason: RecorderReason = 'input_overflow';
const temporary = `${file}.${process.pid}.tmp`;
try {
if (Buffer.byteLength(input) > MAX_BYTES) throw Error('oversized hook input');
reason = 'invalid_event';
const e = JSON.parse(input);
// Foreign and sidechain events cannot cancel or supply the parent request.
if (object(e) && (e.agent_id !== undefined || (typeof e.cwd === 'string' && e.cwd !== cwd))) return;
if (fs.existsSync(file + '.invalid')) return;
reason = 'lock_conflict';
lock = fs.openSync(file + '.lock', 'wx', 0o600);
reason = 'record_error';
const old = readState(file, cwd, configDir);
reason = 'invalid_event';
if (!object(e) || e.cwd !== cwd || !['PreToolUse', 'PostToolUse', 'PostToolUseFailure'].includes(e.hook_event_name) ||
e.tool_name !== 'AskUserQuestion' || !identifier(e.session_id) || !identifier(e.tool_use_id) ||
!scopedTranscript(e.transcript_path, configDir, e.session_id) ||
!object(e.tool_input) || !questions(e.tool_input.questions) ||
!(e.hook_event_name === 'PreToolUse' ? keysOnly(e.tool_input, ['questions']) : completionInput(e.tool_input))) {
throw Error('invalid owned hook event');
}
if (old.sessionId !== undefined && old.sessionId !== e.session_id) return;
const state: State = {...old, sessionId:e.session_id};
const pending = old.pending;
if (e.hook_event_name !== 'PreToolUse' && pending?.toolUseId === e.tool_use_id &&
(pending.transcriptPath !== e.transcript_path || questionIdentity(pending.questions) !== questionIdentity(e.tool_input.questions))) {
throw Error('completion does not match pending request');
}
if (e.hook_event_name === 'PreToolUse') {
if (old.seenIds.includes(e.tool_use_id)) {
if (pending?.toolUseId === e.tool_use_id && JSON.stringify(pending.questions) !== JSON.stringify(e.tool_input.questions)) {
reason = 'conflicting_replay';
throw Error('conflicting replay');
}
return;
}
if (pending) { reason = 'concurrent_pending'; throw Error('concurrent pending questions'); }
state.pending = {sessionId:e.session_id, toolUseId:e.tool_use_id, transcriptPath:e.transcript_path,
timestamp:new Date().toISOString(), questions:e.tool_input.questions};
} else {
// A late result cannot clear another request. Remember completion even
// if its Pre was never observed, so replay cannot reopen that old ID.
if (pending?.toolUseId === e.tool_use_id) state.pending = null;
}
if (!state.seenIds.includes(e.tool_use_id)) state.seenIds = [...state.seenIds, e.tool_use_id];
const serialized = JSON.stringify(state) + '\n';
reason = 'record_overflow';
if (state.seenIds.length > MAX_IDS || Buffer.byteLength(serialized) > MAX_BYTES) throw Error('recorder overflow');
reason = 'record_error';
fs.writeFileSync(temporary, serialized, {mode:0o600, flag:'wx'});
fs.renameSync(temporary, file);
} catch { poison(file, reason); }
finally {
try { fs.rmSync(temporary, {force:true}); } catch { /* no evidence */ }
if (lock !== undefined) {
try { fs.closeSync(lock); fs.unlinkSync(file + '.lock'); } catch { poison(file, 'record_error'); }
}
}
}
/** Pending-only source. A published JSONL call, including failure, always wins. */
export function readPendingQuestion(file: string | undefined, cwd: string, configDir: string | null,
startedAt: number, transcript: PlanCountTranscript,
): (NativePlanQuestionCall & {source:'pre_tool_use'}) | undefined {
if (!file || !configDir || transcript.status !== 'ready' || !Number.isFinite(startedAt)) return undefined;
try {
if (fs.existsSync(file + '.invalid') || fs.existsSync(file + '.lock')) return undefined;
const p = readState(file, cwd, configDir).pending;
if (!p) return undefined;
const time = Date.parse(p.timestamp);
const sessions = new Set([...transcript.calls.map(c => c.sessionId), ...transcript.assistantMessages.map(m => m.sessionId)]);
if (sessions.size !== 1 || !sessions.has(p.sessionId) || !Number.isFinite(time) || time < startedAt || time > Date.now() ||
transcript.calls.some(c => c.sessionId === p.sessionId && c.toolUseId === p.toolUseId)) return undefined;
if (!transcript.calls.length && !transcript.assistantMessages.some(m => m.sessionId === p.sessionId && m.text.trim() &&
Number.isFinite(Date.parse(m.timestamp)) && Date.parse(m.timestamp) >= startedAt && Date.parse(m.timestamp) <= time)) return undefined;
return {sessionId:p.sessionId, toolUseId:p.toolUseId, questions:p.questions, answered:false, failed:false, source:'pre_tool_use'};
} catch { return undefined; }
}
/**
* Display-only first-call identity for AUQ format capture. The caller must bind
* this payload to the current native viewport before grading it. Unlike the
* counting reader, this never opens a transcript or supplies answer evidence.
* A launcher-assigned session and a fresh recorder exclude other invocations;
* a completed or skipped first call cannot be replaced with a later question.
*/
export function readFirstPendingQuestionForDisplay(file: string | undefined, cwd: string,
configDir: string | null, startedAt: number, sessionId: string,
): (NativePlanQuestionCall & {source:'pre_tool_use'}) | undefined {
if (!file || !configDir || !identifier(sessionId) || !Number.isFinite(startedAt)) return undefined;
try {
if (fs.existsSync(file + '.invalid') || fs.existsSync(file + '.lock')) return undefined;
const state = readState(file, cwd, configDir);
const p = state.pending;
if (!p || p.sessionId !== sessionId || state.seenIds.length !== 1 || state.seenIds[0] !== p.toolUseId) return undefined;
const time = Date.parse(p.timestamp);
if (!Number.isFinite(time) || time < startedAt || time > Date.now()) return undefined;
return {sessionId:p.sessionId, toolUseId:p.toolUseId, questions:p.questions,
answered:false, failed:false, source:'pre_tool_use'};
} catch { return undefined; }
}
if (import.meta.main && process.argv[2] === '--record') {
const [file, cwd, configDir] = process.argv.slice(3);
if (file && cwd && configDir) {
const timer = setTimeout(() => { poison(file, 'stdin_timeout'); process.exit(0); }, 4000);
try {
const chunks: Uint8Array[] = [];
let size = 0;
for await (const chunk of Bun.stdin.stream()) {
size += chunk.byteLength;
if (size > MAX_BYTES) { poison(file, 'input_overflow'); process.exit(0); }
chunks.push(chunk);
}
recordPendingQuestion(Buffer.concat(chunks).toString('utf8'), file, cwd, configDir);
} catch { poison(file, 'hook_error'); }
finally { clearTimeout(timer); }
}
}