v1.87.0.0 feat: add verified CSO audits and replayable repair bundles (#2852)

* feat(cso): add verified audits and replayable repair bundles

* fix(cso): harden qualification and setup boundaries

* fix(cso): assemble security canaries at runtime

* fix(cso): bound release proof and maintenance work

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* fix(cso): require complete evaluation reports

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* fix(cso): replay expired snapshots from supplied source

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* test(cso): synchronize DNS cancellation assertion

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* chore(ship): exempt repository owner from liveness proof

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* test(cso): make recheck retention overlap deterministic

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* chore: bump version and changelog (v1.85.0.0)

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* fix(cso): pass native release gates

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* chore: move release to v1.86.0.0

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* fix(cso): resolve rechecks by finding

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* chore: move release to v1.87.0.0

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* fix(cso): pass macOS and Windows release gates

Normalize BSD wc output, compare Windows paths by filesystem identity, preserve portable snapshot race coverage, and narrow POSIX-only Windows fixtures.

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* fix(cso): harden native verification gates

* fix(cso): refine Windows native diagnostics

* test(cso): isolate Windows Git startup failure

* test(cso): stabilize Windows native diagnostics

* fix(cso): support hardened Git on Windows

* fix(cso): close final verification gaps

* test(cso): bound cold Docker fixture setup

* fix(cso): restore cross-platform free-suite gates

---------

Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Garry Tan
2026-09-14 15:14:58 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent 9f81911136
commit 4a3c6a8a3c
160 changed files with 24697 additions and 2288 deletions
+129 -47
View File
@@ -1,4 +1,15 @@
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
import {
csoProducerChildEnvironment,
csoProducerHelperHome,
csoProducerHelperLauncher,
csoProducerProviderCommand,
csoProducerSourceDirectory,
csoProducerStateDirectory,
type ProviderAdapter,
type RunOpts,
type RunResult,
type AvailabilityCheck,
} from './types';
import { estimateCostUsd } from '../pricing';
import { execFileSync, spawnSync } from 'child_process';
import * as fs from 'fs';
@@ -19,10 +30,11 @@ export class ClaudeAdapter implements ProviderAdapter {
readonly name = 'claude';
readonly family = 'claude' as const;
async available(): Promise<AvailabilityCheck> {
async available(opts?: RunOpts): Promise<AvailabilityCheck> {
// Binary on PATH (or GSTACK_CLAUDE_BIN override). Routes through the shared
// resolver so Windows + override paths behave the same as production sites.
const resolved = resolveClaudeCommand();
const producerCommand = csoProducerProviderCommand(opts ?? { prompt: '', workdir: '/', timeoutMs: 1 });
const resolved = producerCommand ? { command: producerCommand.executable, argsPrefix: producerCommand.argsPrefix } : resolveClaudeCommand();
if (!resolved) {
return { ok: false, reason: 'claude CLI not found on PATH. Install from https://claude.ai/download or npm i -g @anthropic-ai/claude-code (or set GSTACK_CLAUDE_BIN)' };
}
@@ -35,8 +47,8 @@ export class ClaudeAdapter implements ProviderAdapter {
// secret), and any failure of `security` itself falls through to the
// not-found reason rather than throwing.
const credsPath = path.join(os.homedir(), '.claude', '.credentials.json');
const hasCreds = fs.existsSync(credsPath);
const hasKey = !!process.env.ANTHROPIC_API_KEY;
const hasCreds = !opts?.csoProducer && fs.existsSync(credsPath);
const hasKey = !!(process.env.ANTHROPIC_API_KEY || process.env.CLAUDE_CODE_OAUTH_TOKEN);
let hasKeychain = false;
if (!hasCreds && !hasKey && process.platform === 'darwin') {
try {
@@ -50,41 +62,40 @@ export class ClaudeAdapter implements ProviderAdapter {
}
}
if (!hasCreds && !hasKey && !hasKeychain) {
return { ok: false, reason: 'No Claude auth found. Log in via `claude` interactive session, or export ANTHROPIC_API_KEY.' };
return { ok: false, reason: opts?.csoProducer
? 'No Claude producer auth found. Export ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN; macOS Keychain auth is also supported.'
: 'No Claude auth found. Log in via `claude` interactive session, or export ANTHROPIC_API_KEY.' };
}
return { ok: true };
}
async run(opts: RunOpts): Promise<RunResult> {
const start = Date.now();
const resolved = resolveClaudeCommand();
const producerCommand = csoProducerProviderCommand(opts);
const resolved = producerCommand ? { command: producerCommand.executable, argsPrefix: producerCommand.argsPrefix } : resolveClaudeCommand();
if (!resolved) {
throw new Error('claude CLI not resolvable (set GSTACK_CLAUDE_BIN or install)');
}
const model = opts.model ?? process.env.EVALS_MODEL ?? resolveEvalModel('capture');
const args = [...resolved.argsPrefix, '-p', '--output-format', 'json'];
args.push('--model', model);
if (opts.extraArgs) args.push(...opts.extraArgs);
const stateDirectory = csoProducerStateDirectory(opts);
if (stateDirectory) {
assertClaudeProducerPrerequisites();
}
try {
if (stateDirectory) prepareClaudeProducerState(stateDirectory);
const args = claudeExecArgs(opts, model, resolved.argsPrefix);
const out = execFileSync(resolved.command, args, {
input: opts.prompt,
cwd: opts.workdir,
cwd: claudeExecWorkingDirectory(opts),
timeout: opts.timeoutMs,
encoding: 'utf-8',
maxBuffer: 32 * 1024 * 1024,
// Default GSTACK_HEADLESS=1 so a benchmark run classifies as headless (an
// AskUserQuestion failure BLOCKs rather than emitting unanswerable prose).
env: { ...process.env, GSTACK_HEADLESS: '1' },
env: claudeExecEnvironment(opts, process.env),
});
const parsed = this.parseOutput(out);
return {
output: parsed.output,
tokens: parsed.tokens,
durationMs: Date.now() - start,
toolCalls: parsed.toolCalls,
modelUsed: parsed.modelUsed || model,
};
return resultFromClaudeOutput(out,{model,durationMs:Date.now()-start,producer:!!opts.csoProducer});
} catch (err: unknown) {
const durationMs = Date.now() - start;
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
@@ -99,6 +110,8 @@ export class ClaudeAdapter implements ProviderAdapter {
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, model);
}
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, model);
} finally {
if (stateDirectory) removeClaudeProducerState(stateDirectory);
}
}
@@ -106,33 +119,6 @@ export class ClaudeAdapter implements ProviderAdapter {
return estimateCostUsd(tokens, model ?? resolveEvalModel('capture'));
}
/**
* Parse claude -p --output-format json output. Shape (as of 2026-04):
* { type: "result", result: "<assistant text>", usage: { input_tokens, output_tokens, ... },
* num_turns, session_id, ... }
* Older formats may differ — adapter is best-effort.
*/
private parseOutput(raw: string): { output: string; tokens: { input: number; output: number; cached?: number }; toolCalls: number; modelUsed?: string } {
try {
const obj = JSON.parse(raw);
const result = typeof obj.result === 'string' ? obj.result : String(obj.result ?? '');
const u = obj.usage ?? {};
return {
output: result,
tokens: {
input: u.input_tokens ?? 0,
output: u.output_tokens ?? 0,
cached: u.cache_read_input_tokens,
},
toolCalls: obj.num_turns ?? 0,
modelUsed: obj.model,
};
} catch {
// Non-JSON output: treat as plain text.
return { output: raw, tokens: { input: 0, output: 0 }, toolCalls: 0 };
}
}
private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
return {
output: '',
@@ -144,3 +130,99 @@ export class ClaudeAdapter implements ProviderAdapter {
};
}
}
/** Map Claude JSON output; paid producers require the documented nonblank `result` envelope. */
export function resultFromClaudeOutput(raw:string,opts:{model?:string;durationMs?:number;producer?:boolean}={}):RunResult{
const durationMs=opts.durationMs??0,defaultModel=opts.model??resolveEvalModel('capture');
try{
const obj=JSON.parse(raw),explicitError=obj?.is_error===true||(typeof obj?.subtype==='string'&&obj.subtype!=='success'),validResult=!explicitError&&typeof obj?.result==='string'&&obj.result.trim().length>0;
if(opts.producer&&!validResult)return{output:'',tokens:{input:0,output:0},durationMs,toolCalls:0,modelUsed:typeof obj?.model==='string'&&obj.model?obj.model:defaultModel,error:{code:'unknown',reason:'empty or invalid output from claude CLI (exit 0)'}};
const output=typeof obj?.result==='string'?obj.result:String(obj?.result??''),usage=obj?.usage??{};
return{output,tokens:{input:usage.input_tokens??0,output:usage.output_tokens??0,cached:usage.cache_read_input_tokens},durationMs,toolCalls:obj?.num_turns??0,modelUsed:typeof obj?.model==='string'&&obj.model?obj.model:defaultModel};
}catch{
if(opts.producer)return{output:'',tokens:{input:0,output:0},durationMs,toolCalls:0,modelUsed:defaultModel,error:{code:'unknown',reason:'empty or invalid output from claude CLI (exit 0)'}};
return{output:raw,tokens:{input:0,output:0},durationMs,toolCalls:0,modelUsed:defaultModel};
}
}
export interface ClaudeProducerPaths { root: string; home: string; workdir: string }
export function assertClaudeProducerPrerequisites(): void {
if (process.platform !== 'linux') return;
const executable = '/usr/bin/bwrap';
let stat: fs.Stats;
try { stat = fs.lstatSync(executable); } catch { throw new Error('CLAUDE_PRODUCER_REQUIRES_TRUSTED_BWRAP'); }
if (!stat.isFile() || stat.isSymbolicLink() || fs.realpathSync(executable) !== executable || stat.uid !== 0 || (stat.mode & 0o022) !== 0 || (stat.mode & 0o111) === 0) {
throw new Error('CLAUDE_PRODUCER_REQUIRES_TRUSTED_BWRAP');
}
}
export function claudeProducerPaths(stateDirectory: string): ClaudeProducerPaths {
const root = path.join(stateDirectory, 'claude-provider');
return { root, home: path.join(root, 'home'), workdir: path.join(root, 'work') };
}
export function prepareClaudeProducerState(stateDirectory: string): ClaudeProducerPaths {
const paths = claudeProducerPaths(stateDirectory);
if (fs.existsSync(paths.root)) throw new Error('CLAUDE_PRODUCER_STATE_EXISTS');
fs.mkdirSync(paths.root, { mode: 0o700 });
fs.mkdirSync(paths.home, { mode: 0o700 });
fs.mkdirSync(paths.workdir, { mode: 0o700 });
return paths;
}
export function removeClaudeProducerState(stateDirectory: string): void {
fs.rmSync(claudeProducerPaths(stateDirectory).root, { recursive: true, force: true });
}
export function claudeExecWorkingDirectory(opts: RunOpts): string {
const stateDirectory = csoProducerStateDirectory(opts);
return stateDirectory ? claudeProducerPaths(stateDirectory).workdir : opts.workdir;
}
export function claudeProducerTools(opts: RunOpts): string {
const launcher = csoProducerHelperLauncher(opts);
if (!launcher) throw new Error('CSO producer helper launcher is required');
return [`Bash(${launcher})`, `Bash(${launcher} *)`, 'Write'].join(',');
}
export function claudeExecArgs(opts: RunOpts, model: string, argsPrefix: readonly string[] = []): string[] {
const args = [...argsPrefix, '-p', '--output-format', 'json', '--model', model];
const stateDirectory = csoProducerStateDirectory(opts);
if (stateDirectory) {
if (opts.extraArgs?.length) throw new Error('CSO producer does not accept extra provider arguments');
const tools = claudeProducerTools(opts);
const sourceDirectory = csoProducerSourceDirectory(opts)!;
const helperHome = csoProducerHelperHome(opts)!;
args.push(
'--restricted',
'--safe-mode',
'--no-session-persistence',
'--permission-prompts', 'none',
'--permission-mode', 'dontAsk',
'--tools', 'Bash,Write',
'--allowed-tools', tools,
'--add-dir', claudeProducerPaths(stateDirectory).workdir,
'--add-dir', sourceDirectory,
'--add-dir', helperHome,
'--strict-mcp-config',
'--no-chrome',
);
}
if (opts.extraArgs) args.push(...opts.extraArgs);
return args;
}
export function claudeExecEnvironment(
opts: RunOpts,
source: NodeJS.ProcessEnv = process.env,
): Record<string, string> {
const stateDirectory = csoProducerStateDirectory(opts);
const paths = stateDirectory ? claudeProducerPaths(stateDirectory) : undefined;
return {
...(opts.csoProducer ? csoProducerChildEnvironment('claude', source) : source),
...(paths ? { HOME: paths.home, GSTACK_HOME: csoProducerHelperHome(opts)! } : {}),
...(stateDirectory ? { CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: '1' } : {}),
GSTACK_HEADLESS: '1',
} as Record<string, string>;
}
+144 -17
View File
@@ -1,9 +1,22 @@
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
import {
csoProducerChildEnvironment,
csoProducerHelperHome,
csoProducerHelperLauncher,
csoProducerProviderCommand,
csoProducerStateDirectory,
validateCsoProducerStateDirectory,
type ProviderAdapter,
type RunOpts,
type RunResult,
type AvailabilityCheck,
} from './types';
import { estimateCostUsd } from '../pricing';
import { execFileSync, spawnSync } from 'child_process';
import { randomBytes } from 'node:crypto';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { atomicWriteSync } from '../../../lib/fs-atomic';
export type GeminiStreamParse = {
output: string;
@@ -112,15 +125,16 @@ export class GeminiAdapter implements ProviderAdapter {
readonly name = 'gemini';
readonly family = 'gemini' as const;
async available(): Promise<AvailabilityCheck> {
const res = spawnSync('sh', ['-c', 'command -v gemini'], { timeout: 2000 });
async available(opts?: RunOpts): Promise<AvailabilityCheck> {
const producerCommand = csoProducerProviderCommand(opts ?? { prompt: '', workdir: '/', timeoutMs: 1 });
const res = producerCommand ? { status: 0 } : spawnSync('sh', ['-c', 'command -v gemini'], { timeout: 2000 });
if (res.status !== 0) {
return { ok: false, reason: 'gemini CLI not found on PATH. Install per https://github.com/google-gemini/gemini-cli' };
}
const legacyCfgDir = path.join(os.homedir(), '.config', 'gemini');
const newCfgDir = path.join(os.homedir(), '.gemini');
const newOauth = path.join(newCfgDir, 'oauth_creds.json');
const hasCfg = fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth);
const hasCfg = !opts?.csoProducer && (fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth));
// CLI accepts either name; Google AI Studio keys are usually GEMINI_API_KEY.
const hasKey = !!(process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY);
if (!hasCfg && !hasKey) {
@@ -138,23 +152,15 @@ export class GeminiAdapter implements ProviderAdapter {
// Default to --yolo (non-interactive) and stream-json output so we can parse
// tokens + tool calls. Callers can override via extraArgs. (--skip-trust was
// removed in gemini-cli 0.34; passing it errors at argv parse.)
const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--yolo'];
if (opts.model) args.push('--model', opts.model);
if (opts.extraArgs) args.push(...opts.extraArgs);
try {
const out = execFileSync('gemini', args, {
cwd: opts.workdir,
const args = geminiExecArgs(opts);
const command = csoProducerProviderCommand(opts);
const out = execFileSync(command?.executable ?? 'gemini', [...(command?.argsPrefix ?? []), ...args], {
cwd: geminiExecWorkingDirectory(opts),
timeout: opts.timeoutMs,
encoding: 'utf-8',
maxBuffer: 32 * 1024 * 1024,
env: {
...process.env,
// Prefer GEMINI_API_KEY when only that is set (CLI reads both).
...(process.env.GEMINI_API_KEY && !process.env.GOOGLE_API_KEY
? { GOOGLE_API_KEY: process.env.GEMINI_API_KEY }
: {}),
},
env: geminiExecEnvironment(opts, process.env),
});
return resultFromGeminiStream(out, { model: opts.model, durationMs: Date.now() - start });
} catch (err: unknown) {
@@ -189,3 +195,124 @@ export class GeminiAdapter implements ProviderAdapter {
};
}
}
export interface GeminiProducerPaths {
root: string;
home: string;
workdir: string;
systemDefaults: string;
systemSettings: string;
}
export function geminiProducerPaths(stateDirectory: string): GeminiProducerPaths {
validateCsoProducerStateDirectory(stateDirectory);
const root = path.join(stateDirectory, 'gemini-provider');
return {
root,
home: path.join(root, 'home'),
workdir: path.join(root, 'work'),
systemDefaults: path.join(root, 'system-defaults.json'),
systemSettings: path.join(root, 'system-settings.json'),
};
}
/** Highest-precedence Gemini policy for a clean, one-cell producer host. */
const GEMINI_PRODUCER_BLOCKED_ENVIRONMENT = [
'GEMINI_API_KEY', 'GOOGLE_API_KEY', 'GOOGLE_CLOUD_PROJECT', 'GOOGLE_CLOUD_LOCATION',
] as const;
export function geminiProducerSystemSettings(contextFileName: string, helperLauncher: string) {
if (!/^\.gstack-cso-context-[a-f0-9]{32}\.md$/.test(contextFileName)) throw new Error('INVALID_GEMINI_CONTEXT_FILENAME');
const helper = csoProducerHelperLauncher({ prompt: '', workdir: '/', timeoutMs: 1, csoProducer: { stateDirectory: '/', sourceDirectory: '/source', helperLauncher, helperGeneration: path.join(path.dirname(helperLauncher), '.gstack-cso-generation'), providerCommand: { executable: '/provider', argsPrefix: [] } } });
if (!helper) throw new Error('CSO producer helper launcher is required');
const tools = [`run_shell_command(${helper})`, 'write_file'];
return {
advanced: { ignoreLocalEnv: true },
admin: {
extensions: { enabled: false },
mcp: { enabled: false },
skills: { enabled: false },
},
context: {
fileName: [contextFileName],
includeDirectoryTree: false,
loadMemoryFromIncludeDirectories: false,
memoryBoundaryMarkers: [],
},
hooksConfig: { enabled: false },
privacy: { usageStatisticsEnabled: false },
security: {
environmentVariableRedaction: { allowed: [], blocked: [...GEMINI_PRODUCER_BLOCKED_ENVIRONMENT], enabled: true },
folderTrust: { enabled: false },
toolSandboxing: false,
},
skills: { enabled: false },
telemetry: { enabled: false, logPrompts: false },
tools: { allowed: tools, core: tools, sandbox: false },
};
}
export function prepareGeminiProducerState(stateDirectory: string, helperLauncher: string): GeminiProducerPaths {
const paths = geminiProducerPaths(stateDirectory);
if (fs.existsSync(paths.root)) throw new Error('GEMINI_PRODUCER_STATE_EXISTS');
fs.mkdirSync(paths.root, { mode: 0o700 });
fs.mkdirSync(paths.home, { mode: 0o700 });
fs.mkdirSync(paths.workdir, { mode: 0o700 });
const contextFileName = `.gstack-cso-context-${randomBytes(16).toString('hex')}.md`;
atomicWriteSync(paths.systemDefaults, '{}\n', { mode: 0o600, noReplace: true });
atomicWriteSync(paths.systemSettings, `${JSON.stringify(geminiProducerSystemSettings(contextFileName, helperLauncher), null, 2)}\n`, { mode: 0o600, noReplace: true });
return paths;
}
export function removeGeminiProducerState(stateDirectory: string): void {
const paths = geminiProducerPaths(stateDirectory);
fs.rmSync(paths.root, { recursive: true, force: true });
}
export function geminiExecArgs(opts: RunOpts): string[] {
const stateDirectory = csoProducerStateDirectory(opts);
const args = ['-p', opts.prompt, '--output-format', 'stream-json'];
if (stateDirectory) {
if (opts.extraArgs?.length) throw new Error('CSO producer does not accept extra provider arguments');
args.push(
'--approval-mode', 'yolo',
'--include-directories', geminiProducerPaths(stateDirectory).workdir,
'-e', 'none',
);
} else {
args.push('--yolo');
}
if (opts.model) args.push('--model', opts.model);
if (opts.extraArgs) args.push(...opts.extraArgs);
return args;
}
export function geminiExecWorkingDirectory(opts: RunOpts): string {
const stateDirectory = csoProducerStateDirectory(opts);
return stateDirectory ? geminiProducerPaths(stateDirectory).workdir : opts.workdir;
}
export function geminiExecEnvironment(
opts: RunOpts,
source: NodeJS.ProcessEnv = process.env,
): Record<string, string> {
const stateDirectory = csoProducerStateDirectory(opts);
const paths = stateDirectory ? geminiProducerPaths(stateDirectory) : undefined;
const env = paths ? csoProducerChildEnvironment('gemini', source) : { ...source } as Record<string, string>;
// Prefer GEMINI_API_KEY when only that is set (CLI reads both).
if (env.GEMINI_API_KEY && !env.GOOGLE_API_KEY) env.GOOGLE_API_KEY = env.GEMINI_API_KEY;
if (paths) {
env.HOME = paths.home;
env.GSTACK_HOME = csoProducerHelperHome(opts)!;
env.GEMINI_SANDBOX = 'false';
env.GEMINI_TELEMETRY_ENABLED = 'false';
env.GEMINI_TELEMETRY_LOG_PROMPTS = 'false';
env.GEMINI_CLI_TRUST_WORKSPACE = 'true';
env.GEMINI_SYSTEM_MD = 'false';
env.GEMINI_WRITE_SYSTEM_MD = 'false';
env.GEMINI_CLI_HOME = paths.home;
env.GEMINI_CLI_SYSTEM_DEFAULTS_PATH = paths.systemDefaults;
env.GEMINI_CLI_SYSTEM_SETTINGS_PATH = paths.systemSettings;
}
return env;
}
+154 -60
View File
@@ -1,10 +1,23 @@
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
import {
csoProducerChildEnvironment,
csoProducerHelperGeneration,
csoProducerHelperHome,
csoProducerProviderCommand,
csoProducerSourceDirectory,
csoProducerStateDirectory,
CSO_PRODUCER_SHELL_ENV,
type ProviderAdapter,
type RunOpts,
type RunResult,
type AvailabilityCheck,
} from './types';
import { estimateCostUsd } from '../pricing';
import { execFileSync, spawnSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { CODEX_FRONTIER_MODEL } from '../../../scripts/resolvers/constants';
import { atomicWriteSync } from '../../../lib/fs-atomic';
/**
* GPT adapter — wraps the OpenAI `codex` CLI (codex exec with --json output).
@@ -17,45 +30,40 @@ export class GptAdapter implements ProviderAdapter {
readonly name = 'gpt';
readonly family = 'gpt' as const;
async available(): Promise<AvailabilityCheck> {
const res = spawnSync('sh', ['-c', 'command -v codex'], { timeout: 2000 });
async available(opts?: RunOpts): Promise<AvailabilityCheck> {
const producerCommand = csoProducerProviderCommand(opts ?? { prompt: '', workdir: '/', timeoutMs: 1 });
const res = producerCommand ? { status: 0 } : spawnSync('sh', ['-c', 'command -v codex'], { timeout: 2000 });
if (res.status !== 0) {
return { ok: false, reason: 'codex CLI not found on PATH. Install: npm i -g @openai/codex' };
}
// Auth sniff: ~/.codex/ should contain auth state after `codex login`
const codexDir = path.join(os.homedir(), '.codex');
if (!fs.existsSync(codexDir)) {
return { ok: false, reason: 'No ~/.codex/ found. Run `codex login` to authenticate via ChatGPT.' };
const hasFileAuth = !opts?.csoProducer && fs.existsSync(codexDir);
if (!hasFileAuth && !process.env.OPENAI_API_KEY) {
return { ok: false, reason: 'No Codex auth found. Paid CSO producers require OPENAI_API_KEY; other evals may use `codex login`.' };
}
return { ok: true };
}
async run(opts: RunOpts): Promise<RunResult> {
const start = Date.now();
// `-s read-only` is load-bearing safety. With `--skip-git-repo-check` we
// bypass codex's interactive trust prompt for unknown directories (benchmarks
// often run in temp dirs / non-git paths), so the read-only sandbox is now
// the only boundary preventing codex from mutating the workdir. If you ever
// remove `-s read-only`, drop `--skip-git-repo-check` too.
// Existing callers retain `-s read-only`. CSO producer calls use an isolated
// CODEX_HOME with a reviewed permission profile written below.
const model = opts.model ?? process.env.GSTACK_CODEX_MODEL ?? CODEX_FRONTIER_MODEL;
const args = ['exec', opts.prompt, '-C', opts.workdir, '-s', 'read-only', '--skip-git-repo-check', '--json', '-m', model];
if (opts.extraArgs) args.push(...opts.extraArgs);
const stateDirectory = csoProducerStateDirectory(opts);
try {
const out = execFileSync('codex', args, {
cwd: opts.workdir,
if (stateDirectory) prepareCodexProducerState(opts);
const args = codexExecArgs(opts, model);
const command = csoProducerProviderCommand(opts);
const out = execFileSync(command?.executable ?? 'codex', [...(command?.argsPrefix ?? []), ...args], {
cwd: codexExecWorkingDirectory(opts),
timeout: opts.timeoutMs,
encoding: 'utf-8',
maxBuffer: 32 * 1024 * 1024,
...(opts.csoProducer ? { env: codexExecEnvironment(opts, process.env) } : {}),
});
const parsed = this.parseJsonl(out);
return {
output: parsed.output,
tokens: parsed.tokens,
durationMs: Date.now() - start,
toolCalls: parsed.toolCalls,
modelUsed: parsed.modelUsed || model,
};
return resultFromCodexStream(out,{model,durationMs:Date.now()-start,producer:!!opts.csoProducer});
} catch (err: unknown) {
const durationMs = Date.now() - start;
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
@@ -70,6 +78,8 @@ export class GptAdapter implements ProviderAdapter {
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, model);
}
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, model);
} finally {
if (stateDirectory) removeCodexProducerState(stateDirectory);
}
}
@@ -77,44 +87,6 @@ export class GptAdapter implements ProviderAdapter {
return estimateCostUsd(tokens, model ?? CODEX_FRONTIER_MODEL);
}
/**
* Parse codex exec --json JSONL stream.
* Key events:
* - item.completed with item.type === 'agent_message' → text output
* - item.completed with item.type === 'command_execution' → tool call
* - turn.completed → usage.input_tokens, usage.output_tokens
* - thread.started → session id (not used here)
*/
private parseJsonl(raw: string): { output: string; tokens: { input: number; output: number }; toolCalls: number; modelUsed?: string } {
let output = '';
let input = 0;
let out = 0;
let toolCalls = 0;
let modelUsed: string | undefined;
for (const line of raw.split('\n')) {
const s = line.trim();
if (!s) continue;
try {
const obj = JSON.parse(s);
if (obj.type === 'item.completed' && obj.item) {
if (obj.item.type === 'agent_message' && typeof obj.item.text === 'string') {
output += (output ? '\n' : '') + obj.item.text;
} else if (obj.item.type === 'command_execution') {
toolCalls += 1;
}
} else if (obj.type === 'turn.completed') {
const u = obj.usage ?? {};
input += u.input_tokens ?? 0;
out += u.output_tokens ?? 0;
if (obj.model) modelUsed = obj.model;
}
} catch {
// skip malformed lines — codex stderr can leak in
}
}
return { output, tokens: { input, output: out }, toolCalls, modelUsed };
}
private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
return {
output: '',
@@ -126,3 +98,125 @@ export class GptAdapter implements ProviderAdapter {
};
}
}
/** Map `codex exec --json` output while requiring evidence-bearing text for paid producers. */
export function resultFromCodexStream(raw:string,opts:{model?:string;durationMs?:number;producer?:boolean}={}):RunResult{
let output='',input=0,out=0,toolCalls=0,modelUsed:string|undefined;
for(const line of raw.split('\n')){
const text=line.trim();if(!text)continue;
try{
const obj=JSON.parse(text);
if(obj.type==='item.completed'&&obj.item){
if(obj.item.type==='agent_message'&&typeof obj.item.text==='string')output+=(output?'\n':'')+obj.item.text;
else if(obj.item.type==='command_execution')toolCalls++;
}else if(obj.type==='turn.completed'){
const usage=obj.usage??{};input+=usage.input_tokens??0;out+=usage.output_tokens??0;
if(typeof obj.model==='string'&&obj.model)modelUsed=obj.model;
}
}catch{/* Codex can mix diagnostic text into the JSONL stream. */}
}
const durationMs=opts.durationMs??0,resolvedModel=modelUsed||opts.model||CODEX_FRONTIER_MODEL;
if(opts.producer&&!output.trim())return{output:'',tokens:{input:0,output:0},durationMs,toolCalls:0,modelUsed:resolvedModel,error:{code:'unknown',reason:'empty output from codex CLI (exit 0)'}};
return{output,tokens:{input,output:out},durationMs,toolCalls,modelUsed:resolvedModel};
}
export interface CodexProducerPaths {
root: string;
home: string;
workdir: string;
config: string;
}
export function codexProducerPaths(stateDirectory: string): CodexProducerPaths {
const root = path.join(stateDirectory, 'codex-provider');
return { root, home: path.join(root, 'home'), workdir: path.join(root, 'work'), config: path.join(root, 'home', 'config.toml') };
}
export function codexProducerConfig(opts: RunOpts): string {
const launcher = opts.csoProducer?.helperLauncher;
if (!launcher) throw new Error('CSO producer helper launcher is required');
const sourceDirectory = csoProducerSourceDirectory(opts);
if (!sourceDirectory) throw new Error('CSO producer source directory is required');
const helperHome = csoProducerHelperHome(opts);
if (!helperHome) throw new Error('CSO producer helper home is required');
const generation = csoProducerHelperGeneration(opts);
if (!generation) throw new Error('CSO producer helper generation is required');
const directory = path.dirname(launcher), suffix = process.platform === 'win32' ? '.exe' : '';
const trustedArtifacts = [
path.join(directory, `cso-eval-producer${suffix}`), launcher,
path.join(directory, `gstack-cso-core${suffix}`), path.join(directory, `gstack-cso-watchdog${suffix}`),
generation,
];
const grants = [...trustedArtifacts, sourceDirectory].map(file => `${JSON.stringify(file)} = "read"`).join('\n');
return `approval_policy = "never"
default_permissions = "cso-producer"
allow_login_shell = false
check_for_update_on_startup = false
[shell_environment_policy]
inherit = "all"
include_only = ${JSON.stringify(CSO_PRODUCER_SHELL_ENV)}
ignore_default_excludes = false
experimental_use_profile = false
[permissions.cso-producer]
description = "CSO producer: private state plus one immutable source snapshot"
[permissions.cso-producer.filesystem]
":root" = "deny"
":minimal" = "read"
${grants}
${JSON.stringify(helperHome)} = "write"
[permissions.cso-producer.filesystem.":workspace_roots"]
"." = "write"
[permissions.cso-producer.network]
enabled = false
`;
}
export function prepareCodexProducerState(opts: RunOpts): CodexProducerPaths {
const stateDirectory = csoProducerStateDirectory(opts);
if (!stateDirectory) throw new Error('CSO producer state directory is required');
const paths = codexProducerPaths(stateDirectory);
if (fs.existsSync(paths.root)) throw new Error('CODEX_PRODUCER_STATE_EXISTS');
fs.mkdirSync(paths.root, { mode: 0o700 });
fs.mkdirSync(paths.home, { mode: 0o700 });
fs.mkdirSync(paths.workdir, { mode: 0o700 });
atomicWriteSync(paths.config, codexProducerConfig(opts), { mode: 0o600, noReplace: true });
return paths;
}
export function removeCodexProducerState(stateDirectory: string): void {
fs.rmSync(codexProducerPaths(stateDirectory).root, { recursive: true, force: true });
}
/** Exported so free tests can inspect the exact paid-CLI boundary without running it. */
export function codexExecArgs(opts: RunOpts, model: string): string[] {
const stateDirectory = csoProducerStateDirectory(opts);
const args = ['exec', opts.prompt, '-C', stateDirectory ? codexProducerPaths(stateDirectory).workdir : opts.workdir];
if (stateDirectory) {
if (opts.extraArgs?.length) throw new Error('CSO producer does not accept extra provider arguments');
args.push(
'--strict-config', '--ephemeral', '--ignore-rules', '--skip-git-repo-check',
);
} else {
args.push('-s', 'read-only', '--skip-git-repo-check');
}
args.push('--json', '-m', model);
if (opts.extraArgs) args.push(...opts.extraArgs);
return args;
}
export function codexExecWorkingDirectory(opts: RunOpts): string {
const stateDirectory = csoProducerStateDirectory(opts);
return stateDirectory ? codexProducerPaths(stateDirectory).workdir : opts.workdir;
}
export function codexExecEnvironment(opts: RunOpts, source: NodeJS.ProcessEnv = process.env): Record<string, string> {
if (!opts.csoProducer) return source as Record<string, string>;
const stateDirectory = csoProducerStateDirectory(opts)!;
const paths = codexProducerPaths(stateDirectory);
return { ...csoProducerChildEnvironment('gpt', source), HOME: paths.home, CODEX_HOME: paths.home, GSTACK_HOME: csoProducerHelperHome(opts)! };
}
+99 -1
View File
@@ -1,3 +1,5 @@
import * as path from 'node:path';
/**
* Provider adapter interface — uniform contract for Claude, GPT, Gemini.
*
@@ -18,6 +20,19 @@ export interface RunOpts {
model?: string;
/** Extra flags per-provider (escape hatch for rare cases). Prefer staying generic. */
extraArgs?: string[];
/** Producer-only execution policy. Omit to preserve each adapter's defaults. */
csoProducer?: {
/** The one explicit state directory exposed outside the source worktree. */
stateDirectory: string;
/** Exact validated per-cell source root. Providers may grant only this path read-only. */
sourceDirectory: string;
/** Absolute immutable launcher path admitted by the producer. */
helperLauncher: string;
/** Exact adjacent generation manifest read by the launcher. */
helperGeneration: string;
/** Exact provider command whose bytes/version are bound into the receipt. */
providerCommand: { executable: string; argsPrefix: string[] };
};
}
export interface TokenUsage {
@@ -57,6 +72,89 @@ export interface AvailabilityCheck {
export type Family = 'claude' | 'gpt' | 'gemini';
export const CSO_PRODUCER_SHELL_ENV = [
'PATH', 'HOME', 'LANG', 'LC_ALL', 'TZ',
'GSTACK_HOME', 'GSTACK_SESSION_KIND', 'GSTACK_HEADLESS',
] as const;
const CSO_AUTH_ENV: Record<Family, readonly string[]> = {
claude: ['ANTHROPIC_API_KEY', 'CLAUDE_CODE_OAUTH_TOKEN'],
gpt: ['OPENAI_API_KEY'],
gemini: ['GEMINI_API_KEY', 'GOOGLE_API_KEY', 'GOOGLE_CLOUD_PROJECT', 'GOOGLE_CLOUD_LOCATION'],
};
export function validateCsoProducerStateDirectory(stateDirectory: string): string {
if (!path.isAbsolute(stateDirectory) || path.normalize(stateDirectory) !== stateDirectory) {
throw new Error('CSO producer state directory must be an absolute normalized path');
}
return stateDirectory;
}
export function csoProducerStateDirectory(opts: RunOpts): string | undefined {
if (!opts.csoProducer) return undefined;
csoProducerHelperLauncher(opts);
csoProducerHelperGeneration(opts);
csoProducerSourceDirectory(opts);
csoProducerProviderCommand(opts);
return validateCsoProducerStateDirectory(opts.csoProducer.stateDirectory);
}
export function csoProducerSourceDirectory(opts: RunOpts): string | undefined {
if (!opts.csoProducer) return undefined;
return validateCsoProducerStateDirectory(opts.csoProducer.sourceDirectory);
}
export function csoProducerHelperHome(opts: RunOpts): string | undefined {
const stateDirectory = csoProducerStateDirectory(opts);
return stateDirectory ? path.join(stateDirectory, 'cso-home') : undefined;
}
export function csoProducerProviderCommand(opts: RunOpts): { executable: string; argsPrefix: string[] } | undefined {
if (!opts.csoProducer) return undefined;
const command = opts.csoProducer.providerCommand;
if (!command || !path.isAbsolute(command.executable) || path.normalize(command.executable) !== command.executable ||
!Array.isArray(command.argsPrefix) || command.argsPrefix.some(value => typeof value !== 'string' || value.includes('\0'))) {
throw new Error('CSO producer provider command must be an absolute normalized trusted path');
}
return { executable: command.executable, argsPrefix: [...command.argsPrefix] };
}
export function csoProducerHelperLauncher(opts: RunOpts): string | undefined {
if (!opts.csoProducer) return undefined;
const launcher = opts.csoProducer.helperLauncher;
if (typeof launcher !== 'string') throw new Error('CSO producer helper launcher must be an absolute normalized trusted path');
const suffix = process.platform === 'win32' ? '.exe' : '';
if (!path.isAbsolute(launcher) || path.normalize(launcher) !== launcher || path.basename(launcher) !== `gstack-cso-launcher${suffix}` ||
!/^[A-Za-z0-9_./:\\-]+$/.test(launcher)) {
throw new Error('CSO producer helper launcher must be an absolute normalized trusted path');
}
return launcher;
}
export function csoProducerHelperGeneration(opts: RunOpts): string | undefined {
if (!opts.csoProducer) return undefined;
const generation = opts.csoProducer.helperGeneration;
const launcher = csoProducerHelperLauncher(opts);
if (typeof generation !== 'string' || !path.isAbsolute(generation) || path.normalize(generation) !== generation ||
path.basename(generation) !== '.gstack-cso-generation' || path.dirname(generation) !== path.dirname(launcher!) ||
!/^[A-Za-z0-9_./:\\-]+$/.test(generation)) {
throw new Error('CSO producer helper generation must be the exact adjacent absolute normalized trusted path');
}
return generation;
}
/** Copy only execution inputs needed by the selected producer host. */
export function csoProducerChildEnvironment(
family: Family,
source: NodeJS.ProcessEnv = process.env,
): Record<string, string> {
const output: Record<string, string> = {};
for (const key of [...CSO_PRODUCER_SHELL_ENV, ...CSO_AUTH_ENV[family]]) {
const value = source[key];
if (typeof value === 'string' && !value.includes('\0')) output[key] = value;
}
return output;
}
export interface ProviderAdapter {
/** Stable name used in output tables and config (e.g., 'claude', 'gpt', 'gemini'). */
readonly name: string;
@@ -66,7 +164,7 @@ export interface ProviderAdapter {
* Check whether the provider's CLI binary is present and authenticated.
* Should never block >2s. Non-throwing: returns { ok: false, reason } on failure.
*/
available(): Promise<AvailabilityCheck>;
available(opts?: RunOpts): Promise<AvailabilityCheck>;
/** Run a prompt and return normalized RunResult. Non-throwing. Errors go in result.error. */
run(opts: RunOpts): Promise<RunResult>;
/** Estimate USD cost for the reported token usage and model. */