/** * The Codex host for runSkillTest (EVALS_HOST=codex, GSTA-49 TM2): runs a * Claude-authored gate case through `codex exec --json` and returns the same * SkillTestResult shape, so the case's own assertions judge the outcome. * * Parity with the Claude runner (session-runner.ts): * - runtime: the full gstack Codex runtime (`./setup --host codex`), installed * once per process into a private CODEX_HOME holding the login's auth.json; * - events: command_execution -> Bash ({ command } unwrapped from Codex's * `bash -lc`, its output); file_change -> Write (add), Edit (update) or * Delete per path; web_search -> WebSearch; mcp_tool_call -> mcp__server__tool; * the last agent message is the output; * - transcript: the Claude stream-json shape the cases' observers read: one * system/init event (cwd, the Codex thread id as session_id), assistant text * and tool_use events, user tool_result events (is_error from the exit code) * and a final result event; * - sandbox: GSTACK_CODEX_SANDBOX, default danger-full-access (the counterpart * of claude -p --dangerously-skip-permissions; cases write outside their * working directory, start loopback servers and browsers); workspace-write * gets network access; * - effort: GSTACK_CODEX_EFFORT sets model_reasoning_effort; unset keeps the * model's default; * - model: a non-Claude options.model, else GSTACK_CODEX_MODEL, else * CODEX_FRONTIER_MODEL; a case's Claude model pin does not apply here; * - cost: real accounting from `turn.completed` usage at lib/pricing.ts rates, * recorded on the result, the eval record and the session ledger. * - nativeLifecycle: onSpawn gets the codex process-group owner right after * spawn; onSettled runs after its exit and group kill, within the drain grace. * Not translated: maxTurns (Codex has no turn cap), allowedTools/tools (Codex * has its own tool surface) and startupGraceMs (the timeout bounds the session). */ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import { spawnSync } from 'child_process'; import { CODEX_DRAIN_GRACE_MS, CodexHarnessError, codexUsageCostUsd, execCodex, type CodexResult } from './codex-session-runner'; import { getProjectEvalDir } from './eval-store'; import { hermeticChildEnv } from './hermetic-env'; import { appendSessionLedger, sessionKey, type SessionEnd } from './session-ledger'; import { sanitizeTestName, scanBrowseErrors, type SkillTestResult } from './session-runner'; import { CODEX_FRONTIER_MODEL } from '../../scripts/resolvers/constants'; const ROOT = path.resolve(import.meta.dir, '..', '..'); const SETUP_TIMEOUT_MS = 300_000; export const CODEX_SANDBOXES = ['read-only', 'workspace-write', 'danger-full-access'] as const; const EFFORT = /^[a-z]+$/; /** The Codex model for a session: a Claude pin never reaches `codex --model`. */ export function codexHostModel(explicit: string | undefined, env: NodeJS.ProcessEnv = process.env): string { if (explicit && !/^claude-/.test(explicit)) return explicit; return env.GSTACK_CODEX_MODEL || CODEX_FRONTIER_MODEL; } /** `codex exec` arguments for one session; the prompt is the last argument. */ export function codexHostExecArgs(model: string, prompt: string, env: NodeJS.ProcessEnv = process.env): string[] { const sandbox = env.GSTACK_CODEX_SANDBOX || 'danger-full-access'; if (!(CODEX_SANDBOXES as readonly string[]).includes(sandbox)) { throw new Error(`GSTACK_CODEX_SANDBOX must be one of ${CODEX_SANDBOXES.join(', ')}; got "${sandbox}"`); } const effort = env.GSTACK_CODEX_EFFORT; if (effort && !EFFORT.test(effort)) throw new Error(`GSTACK_CODEX_EFFORT must be a bare effort name; got "${effort}"`); return ['exec', '--json', '-s', sandbox, '--skip-git-repo-check', '--model', model, ...(sandbox === 'workspace-write' ? ['-c', 'sandbox_workspace_write.network_access=true'] : []), ...(effort ? ['-c', `model_reasoning_effort="${effort}"`] : []), prompt]; } /** The command inside Codex's `/bin/bash -lc ` wrapper, else the command unchanged. */ export function unwrapShellCommand(command: string): string { const match = /^(?:\/(?:usr\/)?bin\/)?(?:ba|z)?sh -lc (.+)$/s.exec(command); if (!match) return command; const word = match[1]!; let out = ''; let i = 0; while (i < word.length) { const c = word[i]!; if (c === "'") { const end = word.indexOf("'", i + 1); if (end < 0) return command; out += word.slice(i + 1, end); i = end + 1; } else if (c === '"') { i++; let closed = false; while (i < word.length) { if (word[i] === '\\' && i + 1 < word.length && '"\\$`\n'.includes(word[i + 1]!)) { out += word[i + 1]; i += 2; } else if (word[i] === '"') { i++; closed = true; break; } else out += word[i++]; } if (!closed) return command; } else if (c === '\\' && i + 1 < word.length) { out += word[i + 1]; i += 2; } else if (/\s/.test(c)) { return command; } else { out += c; i++; } } return out; } export type HostToolCall = SkillTestResult['toolCalls'][number]; const FILE_CHANGE_TOOLS: Record = { add: 'Write', update: 'Edit', delete: 'Delete' }; /** Claude-shaped tool calls from completed Codex items, in order. */ export function codexHostToolCalls(rawLines: readonly string[]): HostToolCall[] { const calls: HostToolCall[] = []; for (const line of rawLines) { let event: any; try { event = JSON.parse(line); } catch { continue; } if (event?.type !== 'item.completed' || !event.item) continue; const item = event.item; if (item.type === 'command_execution') { calls.push({ tool: 'Bash', input: { command: unwrapShellCommand(String(item.command ?? '')) }, output: String(item.aggregated_output ?? '') }); } else if (item.type === 'file_change') { for (const change of Array.isArray(item.changes) ? item.changes : []) { calls.push({ tool: FILE_CHANGE_TOOLS[change?.kind] ?? 'Edit', input: { file_path: String(change?.path ?? ''), kind: change?.kind }, output: '' }); } } else if (item.type === 'web_search') { calls.push({ tool: 'WebSearch', input: { query: item.query }, output: '' }); } else if (item.type === 'mcp_tool_call') { calls.push({ tool: `mcp__${item.server}__${item.tool}`, input: item.arguments ?? {}, output: typeof item.result === 'string' ? item.result : JSON.stringify(item.result ?? '') }); } } return calls; } /** * Codex events as the Claude stream-json transcript the cases' observers read * (system/init, assistant text and tool_use, user tool_result, result). */ export function codexHostTranscript(rawLines: readonly string[], cwd: string, output: string): any[] { const events: any[] = []; let session = ''; const say = (role: 'assistant' | 'user', content: any[]) => events.push({ type: role, session_id: session, parent_tool_use_id: null, message: { role, content } }); for (const line of rawLines) { let event: any; try { event = JSON.parse(line); } catch { continue; } if (event?.type === 'thread.started') { session = String(event.thread_id ?? ''); events.push({ type: 'system', subtype: 'init', cwd, session_id: session, tools: [] }); continue; } if (event?.type !== 'item.completed' || !event.item) continue; const item = event.item; const id = String(item.id ?? `item_${events.length}`); if (item.type === 'agent_message' && item.text) say('assistant', [{ type: 'text', text: String(item.text) }]); else if (item.type === 'command_execution') { say('assistant', [{ type: 'tool_use', id, name: 'Bash', input: { command: unwrapShellCommand(String(item.command ?? '')) } }]); say('user', [{ type: 'tool_result', tool_use_id: id, content: String(item.aggregated_output ?? ''), is_error: item.exit_code !== 0 }]); } else if (item.type === 'file_change') { (Array.isArray(item.changes) ? item.changes : []).forEach((change: any, index: number) => { const useId = `${id}_${index}`; say('assistant', [{ type: 'tool_use', id: useId, name: FILE_CHANGE_TOOLS[change?.kind] ?? 'Edit', input: { file_path: String(change?.path ?? '') } }]); say('user', [{ type: 'tool_result', tool_use_id: useId, content: '', is_error: item.status !== 'completed' }]); }); } } events.push({ type: 'result', subtype: 'success', session_id: session, result: output }); return events; } let runtimeRoot: string | undefined; /** * Install the full gstack Codex runtime once per process: `./setup --host codex` * into a private HOME/CODEX_HOME, with the login home's auth.json copied in. * Returns the root holding home/.codex and tmp/ (Codex will not create its * helper aliases when CODEX_HOME sits under its own TMPDIR). A caller that * already prepared one passes it as GSTACK_CODEX_HOST_ROOT. */ export function prepareCodexHostRuntime(env: NodeJS.ProcessEnv = process.env): string { if (runtimeRoot) return runtimeRoot; const preset = env.GSTACK_CODEX_HOST_ROOT; if (preset) { if (!fs.existsSync(path.join(preset, 'home', '.codex'))) throw new Error(`GSTACK_CODEX_HOST_ROOT ${preset} has no home/.codex`); fs.mkdirSync(path.join(preset, 'tmp'), { recursive: true }); return runtimeRoot = preset; } const root = fs.mkdtempSync(path.join(os.tmpdir(), 'codex-host-')); const home = path.join(root, 'home'); const codexHome = path.join(home, '.codex'); const tmp = path.join(root, 'tmp'); for (const dir of [codexHome, tmp]) fs.mkdirSync(dir, { recursive: true }); const auth = path.join(env.CODEX_HOME || path.join(os.homedir(), '.codex'), 'auth.json'); if (fs.existsSync(auth)) fs.copyFileSync(auth, path.join(codexHome, 'auth.json')); const setup = spawnSync(path.join(ROOT, 'setup'), ['--host', 'codex'], { cwd: ROOT, encoding: 'utf8', timeout: SETUP_TIMEOUT_MS, env: hermeticChildEnv({ HOME: home, CODEX_HOME: codexHome, TMPDIR: tmp }, { extraAllow: ['OPENAI_API_KEY', 'CODEX_*'] }), }); if (setup.status !== 0) { fs.rmSync(root, { recursive: true, force: true }); throw new Error(`./setup --host codex failed (exit ${setup.status}${setup.error ? `, ${setup.error.message}` : ''}): ${(setup.stderr || setup.stdout || '').slice(-2000)}`); } process.once('exit', () => { try { fs.rmSync(root, { recursive: true, force: true }); } catch { /* best effort */ } }); runtimeRoot = root; return root; } type RunSkillTestOptions = Parameters[0]; function exitReasonOf(result: CodexResult, aborted: boolean): string { if (result.exitCode === 124) return 'timeout'; const failedTurn = result.rawLines.some(line => /"type":"(?:turn\.failed|error)"/.test(line)); if (result.exitCode === 0) return failedTurn ? 'error_api' : 'success'; if (aborted) return 'timeout'; return failedTurn && !result.rawLines.some(line => line.includes('"type":"item.completed"')) ? 'error_api' : `exit_code_${result.exitCode}`; } const SESSION_ENDS: Record = { success: 'completed', timeout: 'session_timeout', error_api: 'api_error' }; /** runSkillTest on the Codex host. `appendSystemPrompt` arrives with any completion notice already applied. */ export async function runCodexHostSkillTest(options: RunSkillTestOptions): Promise { const startTime = Date.now(); const startMono = performance.now(); const { workingDirectory, timeout = 120_000, testName, runId, signal } = options; const deadline = startTime + timeout; const model = codexHostModel(options.model); const prompt = options.appendSystemPrompt ? `${options.appendSystemPrompt}\n\n${options.prompt}` : options.prompt; const root = prepareCodexHostRuntime(); const safeName = testName ? sanitizeTestName(testName) : null; let runDir: string | null = null; if (runId) { try { runDir = path.join(path.dirname(getProjectEvalDir()), 'e2e-runs', runId); fs.mkdirSync(runDir, { recursive: true }); } catch { runDir = null; } } let firstResponseMs = 0; let lastItemMs = 0; let maxInterTurnMs = 0; const onLine = (line: string, elapsedMs: number) => { if (!firstResponseMs) firstResponseMs = Math.max(1, elapsedMs); if (runDir && safeName) { try { fs.appendFileSync(path.join(runDir, `${safeName}.ndjson`), line + '\n'); } catch { /* non-fatal */ } } if (!line.includes('"type":"item.completed"') || !/"type":"(?:command_execution|file_change)"/.test(line)) return; if (lastItemMs) maxInterTurnMs = Math.max(maxInterTurnMs, elapsedMs - lastItemMs); lastItemMs = elapsedMs; }; let result: CodexResult; let harnessReason: string | undefined; let started = true; const lifecycle = options.nativeLifecycle; try { result = await execCodex(codexHostExecArgs(model, prompt), { cwd: workingDirectory, startTime, deadline, signal, onLine, ...(lifecycle ? { onSpawn: (pid: number) => lifecycle.onSpawn(pid) } : {}), env: hermeticChildEnv( { GSTACK_HEADLESS: '1', ...options.env, CODEX_HOME: path.join(root, 'home', '.codex'), TMPDIR: options.env?.TMPDIR ?? path.join(root, 'tmp') }, { extraAllow: ['OPENAI_API_KEY', 'CODEX_*'] }, ), }); } catch (error) { if (!(error instanceof CodexHarnessError) || !error.result) throw error; result = error.result; started = !error.message.startsWith('Could not start Codex'); harnessReason = /drain/.test(error.message) ? 'error_output_drain' : 'error_output_stream'; } if (lifecycle) { const settleDeadline = Date.now() + CODEX_DRAIN_GRACE_MS; let hookTimer: ReturnType | undefined; try { await Promise.race([ lifecycle.onSettled({ deadline: settleDeadline, exited: started }), new Promise((_, reject) => { hookTimer = setTimeout(() => reject(new Error('native lifecycle settlement deadline exceeded')), CODEX_DRAIN_GRACE_MS); }), ]); } finally { clearTimeout(hookTimer); } } const exitReason = harnessReason ?? exitReasonOf(result, !!signal?.aborted); const toolCalls = codexHostToolCalls(result.rawLines); const events = result.rawLines.flatMap(line => { try { return [JSON.parse(line)]; } catch { return []; } }); const messages = events.filter(event => event?.type === 'item.completed' && event.item?.type === 'agent_message'); const output = String(messages.at(-1)?.item?.text ?? ''); const turnsUsed = events.filter(event => event?.type === 'item.completed' && event.item?.type !== 'reasoning').length; const transcript = codexHostTranscript(result.rawLines, workingDirectory, output); const cost = result.usage ? codexUsageCostUsd(result.usage, model) : null; const browseErrors = scanBrowseErrors(toolCalls.filter(call => call.tool === 'Bash').map(call => call.output).join('\n') + '\n' + result.stderr); const duration = Date.now() - startTime; appendSessionLedger({ key: sessionKey('codex', testName), ...(testName ? { test_name: testName } : {}), runner: 'codex', started_at: new Date(startTime).toISOString(), budget_ms: timeout, elapsed_ms: performance.now() - startMono, end: exitReason === 'timeout' && signal?.aborted ? 'aborted' : SESSION_ENDS[exitReason] ?? 'error', ...(exitReason === 'timeout' ? { evidence: `armed ${timeout}ms session timeout fired` } : exitReason !== 'success' && result.stderr ? { evidence: result.stderr } : {}), billed: cost !== null, model, ...(cost !== null ? { cost_usd: cost, tokens: result.usage! } : {}) }); if (exitReason !== 'success' || browseErrors.length > 0) { try { const failureDir = runDir || path.join(workingDirectory, '.gstack', 'test-transcripts'); fs.mkdirSync(failureDir, { recursive: true }); fs.writeFileSync(path.join(failureDir, safeName ? `${safeName}-failure.json` : `e2e-${new Date().toISOString().replace(/[:.]/g, '-')}.json`), JSON.stringify({ host: 'codex', model, prompt: prompt.slice(0, 500), testName: testName || 'unknown', exitReason, browseErrors, duration, stderr: result.stderr.slice(0, 2000), output: output.slice(0, 500) }, null, 2)); } catch { /* non-fatal */ } } return { toolCalls, browseErrors, exitReason, duration, output, transcript, model, firstResponseMs, maxInterTurnMs, costEstimate: { inputChars: prompt.length, outputChars: output.length, estimatedTokens: (result.usage?.input ?? 0) + (result.usage?.output ?? 0), estimatedCost: cost ?? 0, turnsUsed }, ...(result.usage ? { tokenUsage: result.usage } : {}), ...(cost === null ? { costKnown: false } : {}), }; }