v1.87.0.0 feat: add verified CSO audits and replayable repair bundles (#2852)

* feat(cso): add verified audits and replayable repair bundles

* fix(cso): harden qualification and setup boundaries

* fix(cso): assemble security canaries at runtime

* fix(cso): bound release proof and maintenance work

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* fix(cso): require complete evaluation reports

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* fix(cso): replay expired snapshots from supplied source

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* test(cso): synchronize DNS cancellation assertion

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* chore(ship): exempt repository owner from liveness proof

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* test(cso): make recheck retention overlap deterministic

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* chore: bump version and changelog (v1.85.0.0)

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* fix(cso): pass native release gates

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* chore: move release to v1.86.0.0

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* fix(cso): resolve rechecks by finding

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* chore: move release to v1.87.0.0

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* fix(cso): pass macOS and Windows release gates

Normalize BSD wc output, compare Windows paths by filesystem identity, preserve portable snapshot race coverage, and narrow POSIX-only Windows fixtures.

Co-Authored-By: OpenAI Codex <noreply@openai.com>

* fix(cso): harden native verification gates

* fix(cso): refine Windows native diagnostics

* test(cso): isolate Windows Git startup failure

* test(cso): stabilize Windows native diagnostics

* fix(cso): support hardened Git on Windows

* fix(cso): close final verification gaps

* test(cso): bound cold Docker fixture setup

* fix(cso): restore cross-platform free-suite gates

---------

Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Garry Tan
2026-09-14 15:14:58 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent 9f81911136
commit 4a3c6a8a3c
160 changed files with 24697 additions and 2288 deletions
+12 -11
View File
@@ -363,7 +363,7 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
scenario:
'Run a security audit on this repository in --owasp mode (OWASP Top 10 only). Resolve the mode, do the Phase 0 stack detection and Phase 1 attack-surface census, then run the scoped audit phases and produce the findings report. Skip any step that needs network access.',
staticInvariants: {
// Dispatch + always-run + FP-filtering phases are ALWAYS loaded (security).
// Dispatch, trusted execution, evidence/proof, reporting and recovery stay always loaded.
mustStayInSkeleton: [
'## Arguments',
'## Mode Resolution',
@@ -372,6 +372,9 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
'### Phase 12',
'### Phase 13',
'### Phase 14',
'**Private startup.**',
'CSO evidence rubric',
'identical security assertion',
],
// Earliest-use: mode must be resolvable before any section is read (codex #6).
mustPrecedeStop: ['## Arguments', '## Mode Resolution'],
@@ -383,17 +386,15 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
],
gateAfterStop: undefined,
},
behavioral: 'prompt',
// +Conductor AUQ-default-prose rule + one-way/continuation safety in the
// always-loaded AskUserQuestion Format section.
// v1.2.0 activation lift: first-run-guidance section in the shared preamble.
maxSkeletonBytes: 61_800, // + v2.0 {{ASIDE_RESEARCH}} (Aside first, WebSearch fallback); measured 60_628
minUnionBytes: 64_200, // token-reduction Phases 1-2 (v1.69.x branch); measured union 71,379
// v3 requires a trusted helper and private state, absent from generic prompt fixtures.
// The full-audit E2E asserts actual section loading alongside report/proof behavior.
behavioral: 'external',
externalTest: 'test/skill-e2e-cso.test.ts',
maxSkeletonBytes: 18_000,
minUnionBytes: 30_000, // v3 deliberately removes the shared export/startup preamble.
mustContain: ['OWASP', 'STRIDE', 'daily', 'comprehensive', 'verif'],
// cso keeps its mode-dispatch + FP-filtering phases always-loaded, so the
// cross-cutting preamble growth (v1.57.2.0 AUQ-failure prose fallback ~2KB + the
// decision-memory nudge) lands it just over 1.05; headroom for the shared additions.
// v1.64+v1.65 merge sums both waves' preamble growth; measured 1.073.
// Existing baseline comparison remains an upper bound; absolute limits above
// preserve the compact controller and its complete domain reference.
maxSizeRatio: 1.08,
},
// ── Token-reduction Phase 4 wave 1 (v1.69.x branch) ──────────────────────
+108
View File
@@ -0,0 +1,108 @@
/** Private evaluator inputs. Never copy this module or its JSON output into producer snapshots. */
import { createHash } from 'node:crypto';
import { CORPUS_VERSION, type EvalFamily, type EvalStack } from '../fixtures/cso-eval/materialize';
export interface OracleRequest { method: 'GET' | 'POST'; path: string; body?: string; headers?: Record<string, string> }
export interface OracleResponse { status: number; body: string; headers?: Record<string, string> }
export interface PrivateOracle {
version: string;
controls: Array<{ request: OracleRequest; response: (value: OracleResponse) => boolean }>;
attack: OracleRequest;
vulnerable: (value: OracleResponse) => boolean;
fixed: (value: OracleResponse) => boolean;
/** More than the exposed training assertion: alternative attack payloads. */
heldOut: Array<{ request: OracleRequest; fixed: (value: OracleResponse) => boolean }>;
}
const get = (params: Record<string, string>): OracleRequest => ({ method: 'GET', path: `/action?${new URLSearchParams(params)}` });
const post = (value: object): OracleRequest => ({ method: 'POST', path: '/action', body: JSON.stringify(value), headers: { 'content-type': 'application/json' } });
const contains = (status: number, expected: string) => (response: OracleResponse) => response.status === status && response.body.includes(expected);
const rejects = (status: number) => (response: OracleResponse) => response.status === status;
const jsonValue = (value: OracleResponse): any => { try { return JSON.parse(value.body); } catch { return null; } };
export function oracleFor(family: EvalFamily): PrivateOracle {
const input: Record<EvalFamily, Omit<PrivateOracle, 'version'>> = {
'sql-injection': {
controls: [{ request: get({ name: 'alice' }), response: contains(200, 'HELLO_ALICE') }],
attack: get({ name: "nobody' OR 1=1 --" }), vulnerable: contains(200, 'SECRET_ADMIN_NOTE'),
fixed: value => value.status === 200 && Array.isArray(jsonValue(value)) && jsonValue(value).length === 0,
heldOut: [{ request: get({ name: "' UNION SELECT 'ORACLE_SQL_MARKER' --" }), fixed: value => value.status === 200 && !value.body.includes('ORACLE_SQL_MARKER') }],
},
'command-injection': {
controls: [{ request: get({ name: 'Ada' }), response: value => value.status === 200 && jsonValue(value)?.output === 'Ada' }],
attack: get({ name: '"; printf CSO_COMMAND_EXECUTED; #' }), vulnerable: value => value.status === 200 && jsonValue(value)?.output === 'CSO_COMMAND_EXECUTED', fixed: rejects(400),
heldOut: [{ request: get({ name: '$(printf ORACLE_COMMAND_MARKER)' }), fixed: rejects(400) }],
},
'path-traversal': {
controls: [{ request: get({ file: 'hello.txt' }), response: contains(200, 'PUBLIC_HELLO') }],
attack: get({ file: '../private.txt' }), vulnerable: contains(200, 'PRIVATE_DOCUMENT_SECRET'), fixed: rejects(403),
heldOut: [{ request: get({ file: 'subdir/../../private.txt' }), fixed: rejects(403) }],
},
ssrf: {
controls: [{ request: get({ url: 'http://127.0.0.1:8001/public' }), response: contains(200, 'PUBLIC_STATUS') }],
attack: get({ url: 'http://127.0.0.1:8001/private' }), vulnerable: contains(200, 'INTERNAL_SERVICE_SECRET'), fixed: rejects(403),
heldOut: [{ request: get({ url: 'http://localhost:8001/private' }), fixed: rejects(403) }],
},
'object-authorization': {
controls: [{ request: get({ id: 'invoice-1' }), response: value => value.status === 200 && jsonValue(value)?.total === 12 }],
attack: get({ id: 'invoice-2' }), vulnerable: value => value.status === 200 && jsonValue(value)?.total === 9800, fixed: rejects(403),
heldOut: [{ request: get({ id: 'invoice-2', owner: 'member-1' }), fixed: rejects(403) }],
},
'tenant-isolation': {
controls: [{ request: get({ tenant: 'tenant-a' }), response: contains(200, 'OWN_TENANT_DOCUMENT') }],
attack: get({ tenant: 'tenant-b' }), vulnerable: contains(200, 'OTHER_TENANT_SECRET'),
fixed: value => value.status === 200 && value.body.includes('OWN_TENANT_DOCUMENT') && !value.body.includes('OTHER_TENANT_SECRET'),
heldOut: [{ request: get({ tenant: 'tenant-b', actor_tenant: 'tenant-b' }), fixed: value => value.status === 200 && !value.body.includes('OTHER_TENANT_SECRET') }],
},
'html-injection': {
controls: [{ request: get({ name: 'Alice' }), response: contains(200, '<main>Welcome Alice</main>') }],
attack: get({ name: '<img src=x onerror="window.CSO_XSS=1">' }), vulnerable: contains(200, '<img src=x onerror='),
fixed: value => value.status === 200 && !value.body.includes('<img') && value.body.includes('&lt;img'),
heldOut: [{ request: get({ name: '</main><svg onload="window.ORACLE_XSS=1">' }), fixed: value => value.status === 200 && !value.body.includes('<svg') && value.body.includes('&lt;svg') }],
},
'open-redirect': {
controls: [{ request: get({ next: '/account' }), response: value => value.status === 302 && jsonValue(value)?.next === '/account' && value.headers?.location === '/account' }],
attack: get({ next: 'https://attacker.invalid/collect' }), vulnerable: value => value.status === 302 && value.headers?.location === 'https://attacker.invalid/collect', fixed: rejects(400),
heldOut: [{ request: get({ next: '//attacker.invalid/collect' }), fixed: rejects(400) }, { request: get({ next: '/\\attacker.invalid/collect' }), fixed: rejects(400) }],
},
'mass-assignment': {
controls: [{ request: post({ display_name: 'Ada' }), response: value => value.status === 200 && jsonValue(value)?.display_name === 'Ada' }],
attack: post({ role: 'admin' }), vulnerable: value => value.status === 200 && jsonValue(value)?.role === 'admin', fixed: value => value.status === 200 && jsonValue(value)?.role === 'member',
heldOut: [{ request: post({ id: 'member-2', role: 'owner', display_name: 'Grace' }), fixed: value => value.status === 200 && jsonValue(value)?.id === 'member-1' && jsonValue(value)?.role === 'member' && jsonValue(value)?.display_name === 'Grace' }],
},
'resource-exhaustion': {
controls: [{ request: get({ count: '5' }), response: value => value.status === 200 && jsonValue(value)?.count === 5 }],
// Bounded proof of missing admission, not an attempt to exhaust the runner.
attack: get({ count: '250' }), vulnerable: value => value.status === 200 && jsonValue(value)?.count === 250, fixed: rejects(400),
heldOut: [{ request: get({ count: '101' }), fixed: rejects(400) }, { request: get({ count: '0' }), fixed: rejects(400) }],
},
};
return { version: CORPUS_VERSION, ...input[family] };
}
export interface Observation { request: OracleRequest; response: OracleResponse }
export interface PrivateEvidence {
original: { booted: boolean; controls: Observation[]; attack: Observation };
patched: { booted: boolean; controls: Observation[]; attack: Observation; heldOut: Observation[]; existingTestsPassed: boolean };
immutableVerifier: boolean;
independentRootCauseReview: boolean;
featurePreserved: boolean;
boundaryMocks: boolean;
}
const sameRequest = (left: OracleRequest, right: OracleRequest) => JSON.stringify(left) === JSON.stringify(right);
/** Only trusted runner observations may enter this function; producer claims are not observations. */
export function judgeRepair(family: EvalFamily, evidence: PrivateEvidence): { reproduced: boolean; correctRepair: boolean; evidenceHash: string } {
const oracle = oracleFor(family);
const controls = (observations: Observation[]) => oracle.controls.every(control => observations.some(observation => sameRequest(control.request, observation.request) && control.response(observation.response)));
const original = evidence.original, patched = evidence.patched;
const reproduced = original.booted && controls(original.controls) && sameRequest(original.attack.request, oracle.attack) && oracle.vulnerable(original.attack.response);
const correctRepair = reproduced && patched.booted && controls(patched.controls) && sameRequest(patched.attack.request, oracle.attack) && oracle.fixed(patched.attack.response)
&& oracle.heldOut.every(assertion => patched.heldOut.some(observation => sameRequest(assertion.request, observation.request) && assertion.fixed(observation.response)))
&& patched.existingTestsPassed && evidence.immutableVerifier && evidence.independentRootCauseReview && evidence.featurePreserved && !evidence.boundaryMocks;
return { reproduced, correctRepair, evidenceHash: createHash('sha256').update(JSON.stringify({ version: oracle.version, family, evidence })).digest('hex') };
}
export function runtimeStart(stack: EvalStack): { executable: string; args: string[]; port: 8000; environment: Record<string, string> } {
return stack === 'node' ? { executable: '/usr/local/bin/node', args: ['app.mjs'], port: 8000, environment: {} }
: stack === 'bun' ? { executable: '/usr/local/bin/bun', args: ['--no-install', 'app.ts'], port: 8000, environment: {} }
: stack === 'python' ? { executable: '/usr/local/bin/python', args: ['-I', 'app.py'], port: 8000, environment: {} }
: { executable: '/usr/local/bin/ruby', args: ['bin/rails', 'server', '-e', 'test', '-b', '127.0.0.1', '-p', '8000'], port: 8000, environment: { RAILS_ENV: 'test', RACK_ENV: 'test' } };
}
+60
View File
@@ -0,0 +1,60 @@
import * as fs from 'node:fs';
import * as path from 'node:path';
import { dispatchCsoCommand, type CsoCliDependencies } from '../../lib/cso/cli';
import { canonical, sha256 } from '../../lib/cso/contracts';
import { validateRuntimeCatalog, type QualifiedRuntime, type RuntimeCatalog, type RuntimePlatform } from '../../lib/cso/runtime-catalog';
import { completeRuntimeCatalogFixture } from './cso-runtime-catalog';
export interface QualifiedCsoCli {
readonly catalog: RuntimeCatalog;
readonly runtime: QualifiedRuntime;
command<T = unknown>(args: string[]): Promise<T>;
}
/** Bind a staged Node image to the real command dispatcher without adding a production override. */
export function qualifiedNodeCli(options: {
image: string;
versions: Record<string, string>;
watchdogPath: string;
platform: RuntimePlatform;
}): QualifiedCsoCli {
if (!path.isAbsolute(options.watchdogPath) || !fs.statSync(options.watchdogPath).isFile()) {
throw new Error('Node lifecycle qualification requires an absolute compiled watchdog path');
}
for (const key of ['node', 'npm', 'cso-preparation']) {
if (!/^\d+\.\d+\.\d+$/.test(options.versions[key] ?? '')) {
throw new Error(`Node lifecycle qualification requires exact ${key} version metadata`);
}
}
if (options.versions['cso-preparation'] !== '1.0.0') {
throw new Error('Node lifecycle qualification requires cso-preparation 1.0.0');
}
const catalog = completeRuntimeCatalogFixture('node-lifecycle-fixture');
const runtimeIndex = catalog.runtimes.findIndex(item => item.stack === 'node' && item.platform === options.platform);
const profile = catalog.profiles.find(item => item.stack === 'node' && item.platform === options.platform);
if (runtimeIndex < 0 || !profile || !catalog.promotion) throw new Error('Node lifecycle catalog fixture is incomplete');
const runtime: QualifiedRuntime = {
...catalog.runtimes[runtimeIndex],
image: options.image,
versions: { ...options.versions },
};
catalog.runtimes[runtimeIndex] = runtime;
profile.versions = { ...runtime.versions };
catalog.promotion.evidenceDigest = `sha256:${sha256(canonical(catalog.runtimes))}`;
validateRuntimeCatalog(catalog);
const dependencies: CsoCliDependencies = Object.freeze({
runtimeCatalog: catalog,
watchdogPath: () => options.watchdogPath,
});
return Object.freeze({
catalog,
runtime,
async command<T = unknown>(args: string[]): Promise<T> {
const [command, ...commandArgs] = args;
if (!command) throw new Error('CSO qualification command is required');
return await dispatchCsoCommand(command, commandArgs, dependencies) as T;
},
});
}
+102
View File
@@ -0,0 +1,102 @@
import type { CsoStack } from '../../lib/cso/preparation';
import { canonical, sha256 } from '../../lib/cso/contracts';
import {
CSO_HELPER_ABI,
type QualifiedRuntime,
type RuntimeCatalog,
type RuntimePlatform,
} from '../../lib/cso/runtime-catalog';
const STACKS = ['node', 'bun', 'python', 'rails', 'postgresql'] as const;
const PLATFORMS = ['linux/amd64', 'linux/arm64'] as const;
const SOURCE_COMMIT = 'b'.repeat(40);
const WORKFLOW = 'https://github.com/garrytan/gstack/actions/runs/1';
const DIGEST = `sha256:${'a'.repeat(64)}`;
const VERSIONS: Record<CsoStack | 'postgresql', Record<string, string>> = {
node: { node: '24.1.0', npm: '11.3.0', 'cso-preparation': '1.0.0' },
bun: { bun: '1.3.10', 'cso-preparation': '1.0.0' },
python: { python: '3.12.9', uv: '0.8.0', 'cso-preparation': '1.0.0' },
rails: { ruby: '3.3.6', bundler: '2.6.9', 'cso-preparation': '1.0.0' },
postgresql: { postgresql: '17.2' },
};
function runtimeId(stack: CsoStack | 'postgresql', platform: RuntimePlatform): string {
return `${stack}-qualified-test-${platform === 'linux/amd64' ? 'amd64' : 'arm64'}`;
}
export function qualifiedRuntimeFixture(
stack: CsoStack | 'postgresql',
platform: RuntimePlatform = 'linux/amd64',
): QualifiedRuntime {
const arch = platform === 'linux/amd64' ? 'amd64' : 'arm64';
const qualification = stack === 'postgresql'
? {
kind: 'postgresql' as const,
sourceCommit: SOURCE_COMMIT,
workflow: WORKFLOW,
sbomDigest: DIGEST,
provenanceDigest: DIGEST,
verifiedProvenance: true as const,
containmentPassed: true as const,
coldStartPassed: true as const,
multiDatabasePassed: true as const,
readinessPassed: true as const,
}
: {
kind: 'application' as const,
sourceCommit: SOURCE_COMMIT,
workflow: WORKFLOW,
sbomDigest: DIGEST,
provenanceDigest: DIGEST,
verifiedProvenance: true as const,
containmentPassed: true as const,
coldStartPassed: true as const,
positiveNegativeAssertionsPassed: true as const,
heldOutRepairPassed: true as const,
};
return {
id: runtimeId(stack, platform),
stack,
platform,
state: 'qualified',
image: `ghcr.io/garrytan/gstack/cso-staging/${stack}-${arch}@${DIGEST}`,
entrypoint: '/opt/cso/entrypoint',
helperAbi: CSO_HELPER_ABI,
versions: { ...VERSIONS[stack] },
policyVersion: 'cso-isolation-v1',
qualifiedAt: '2026-09-09T00:00:00Z',
qualification,
};
}
/** A fresh catalog satisfying the complete reviewed and promoted runtime matrices. */
export function completeRuntimeCatalogFixture(
revision = 'runtime-test-v1',
previousRevision: string | null = null,
): RuntimeCatalog {
const runtimes = STACKS.flatMap(stack => PLATFORMS.map(platform => qualifiedRuntimeFixture(stack, platform)));
return {
schemaVersion: 1,
revision,
previousRevision,
helperAbi: CSO_HELPER_ABI,
buildRevision: 'runtime-test-build-v1',
profiles: runtimes.map(runtime => ({
id: runtime.id,
stack: runtime.stack,
platform: runtime.platform,
state: 'build_reviewed' as const,
versions: { ...runtime.versions },
reviewedAt: '2026-09-08T00:00:00Z',
})),
promotion: {
sourceCommit: SOURCE_COMMIT,
workflow: WORKFLOW,
evidenceDigest: `sha256:${sha256(canonical(runtimes))}`,
qualificationEvidenceDigest: DIGEST,
},
runtimes,
};
}
+3 -1
View File
@@ -241,7 +241,9 @@ const CARVED_INVARIANTS: ParityInvariant[] = Object.values(CARVE_GUARDS).map((g)
maxSkeletonBytes: g.maxSkeletonBytes,
minBytes: g.minUnionBytes,
mustContain: g.mustContain,
mustHaveHeadings: ['## Preamble', '## When to invoke'],
// CSO's helper trust boundary requires its private startup; demanding the
// shared Preamble here would silently reintroduce conflicting policy.
mustHaveHeadings: g.skill === 'cso' ? ['## When to invoke'] : ['## Preamble', '## When to invoke'],
maxSizeRatio: g.maxSizeRatio ?? 1.05,
}));
+129 -47
View File
@@ -1,4 +1,15 @@
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
import {
csoProducerChildEnvironment,
csoProducerHelperHome,
csoProducerHelperLauncher,
csoProducerProviderCommand,
csoProducerSourceDirectory,
csoProducerStateDirectory,
type ProviderAdapter,
type RunOpts,
type RunResult,
type AvailabilityCheck,
} from './types';
import { estimateCostUsd } from '../pricing';
import { execFileSync, spawnSync } from 'child_process';
import * as fs from 'fs';
@@ -19,10 +30,11 @@ export class ClaudeAdapter implements ProviderAdapter {
readonly name = 'claude';
readonly family = 'claude' as const;
async available(): Promise<AvailabilityCheck> {
async available(opts?: RunOpts): Promise<AvailabilityCheck> {
// Binary on PATH (or GSTACK_CLAUDE_BIN override). Routes through the shared
// resolver so Windows + override paths behave the same as production sites.
const resolved = resolveClaudeCommand();
const producerCommand = csoProducerProviderCommand(opts ?? { prompt: '', workdir: '/', timeoutMs: 1 });
const resolved = producerCommand ? { command: producerCommand.executable, argsPrefix: producerCommand.argsPrefix } : resolveClaudeCommand();
if (!resolved) {
return { ok: false, reason: 'claude CLI not found on PATH. Install from https://claude.ai/download or npm i -g @anthropic-ai/claude-code (or set GSTACK_CLAUDE_BIN)' };
}
@@ -35,8 +47,8 @@ export class ClaudeAdapter implements ProviderAdapter {
// secret), and any failure of `security` itself falls through to the
// not-found reason rather than throwing.
const credsPath = path.join(os.homedir(), '.claude', '.credentials.json');
const hasCreds = fs.existsSync(credsPath);
const hasKey = !!process.env.ANTHROPIC_API_KEY;
const hasCreds = !opts?.csoProducer && fs.existsSync(credsPath);
const hasKey = !!(process.env.ANTHROPIC_API_KEY || process.env.CLAUDE_CODE_OAUTH_TOKEN);
let hasKeychain = false;
if (!hasCreds && !hasKey && process.platform === 'darwin') {
try {
@@ -50,41 +62,40 @@ export class ClaudeAdapter implements ProviderAdapter {
}
}
if (!hasCreds && !hasKey && !hasKeychain) {
return { ok: false, reason: 'No Claude auth found. Log in via `claude` interactive session, or export ANTHROPIC_API_KEY.' };
return { ok: false, reason: opts?.csoProducer
? 'No Claude producer auth found. Export ANTHROPIC_API_KEY or CLAUDE_CODE_OAUTH_TOKEN; macOS Keychain auth is also supported.'
: 'No Claude auth found. Log in via `claude` interactive session, or export ANTHROPIC_API_KEY.' };
}
return { ok: true };
}
async run(opts: RunOpts): Promise<RunResult> {
const start = Date.now();
const resolved = resolveClaudeCommand();
const producerCommand = csoProducerProviderCommand(opts);
const resolved = producerCommand ? { command: producerCommand.executable, argsPrefix: producerCommand.argsPrefix } : resolveClaudeCommand();
if (!resolved) {
throw new Error('claude CLI not resolvable (set GSTACK_CLAUDE_BIN or install)');
}
const model = opts.model ?? process.env.EVALS_MODEL ?? resolveEvalModel('capture');
const args = [...resolved.argsPrefix, '-p', '--output-format', 'json'];
args.push('--model', model);
if (opts.extraArgs) args.push(...opts.extraArgs);
const stateDirectory = csoProducerStateDirectory(opts);
if (stateDirectory) {
assertClaudeProducerPrerequisites();
}
try {
if (stateDirectory) prepareClaudeProducerState(stateDirectory);
const args = claudeExecArgs(opts, model, resolved.argsPrefix);
const out = execFileSync(resolved.command, args, {
input: opts.prompt,
cwd: opts.workdir,
cwd: claudeExecWorkingDirectory(opts),
timeout: opts.timeoutMs,
encoding: 'utf-8',
maxBuffer: 32 * 1024 * 1024,
// Default GSTACK_HEADLESS=1 so a benchmark run classifies as headless (an
// AskUserQuestion failure BLOCKs rather than emitting unanswerable prose).
env: { ...process.env, GSTACK_HEADLESS: '1' },
env: claudeExecEnvironment(opts, process.env),
});
const parsed = this.parseOutput(out);
return {
output: parsed.output,
tokens: parsed.tokens,
durationMs: Date.now() - start,
toolCalls: parsed.toolCalls,
modelUsed: parsed.modelUsed || model,
};
return resultFromClaudeOutput(out,{model,durationMs:Date.now()-start,producer:!!opts.csoProducer});
} catch (err: unknown) {
const durationMs = Date.now() - start;
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
@@ -99,6 +110,8 @@ export class ClaudeAdapter implements ProviderAdapter {
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, model);
}
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, model);
} finally {
if (stateDirectory) removeClaudeProducerState(stateDirectory);
}
}
@@ -106,33 +119,6 @@ export class ClaudeAdapter implements ProviderAdapter {
return estimateCostUsd(tokens, model ?? resolveEvalModel('capture'));
}
/**
* Parse claude -p --output-format json output. Shape (as of 2026-04):
* { type: "result", result: "<assistant text>", usage: { input_tokens, output_tokens, ... },
* num_turns, session_id, ... }
* Older formats may differ — adapter is best-effort.
*/
private parseOutput(raw: string): { output: string; tokens: { input: number; output: number; cached?: number }; toolCalls: number; modelUsed?: string } {
try {
const obj = JSON.parse(raw);
const result = typeof obj.result === 'string' ? obj.result : String(obj.result ?? '');
const u = obj.usage ?? {};
return {
output: result,
tokens: {
input: u.input_tokens ?? 0,
output: u.output_tokens ?? 0,
cached: u.cache_read_input_tokens,
},
toolCalls: obj.num_turns ?? 0,
modelUsed: obj.model,
};
} catch {
// Non-JSON output: treat as plain text.
return { output: raw, tokens: { input: 0, output: 0 }, toolCalls: 0 };
}
}
private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
return {
output: '',
@@ -144,3 +130,99 @@ export class ClaudeAdapter implements ProviderAdapter {
};
}
}
/** Map Claude JSON output; paid producers require the documented nonblank `result` envelope. */
export function resultFromClaudeOutput(raw:string,opts:{model?:string;durationMs?:number;producer?:boolean}={}):RunResult{
const durationMs=opts.durationMs??0,defaultModel=opts.model??resolveEvalModel('capture');
try{
const obj=JSON.parse(raw),explicitError=obj?.is_error===true||(typeof obj?.subtype==='string'&&obj.subtype!=='success'),validResult=!explicitError&&typeof obj?.result==='string'&&obj.result.trim().length>0;
if(opts.producer&&!validResult)return{output:'',tokens:{input:0,output:0},durationMs,toolCalls:0,modelUsed:typeof obj?.model==='string'&&obj.model?obj.model:defaultModel,error:{code:'unknown',reason:'empty or invalid output from claude CLI (exit 0)'}};
const output=typeof obj?.result==='string'?obj.result:String(obj?.result??''),usage=obj?.usage??{};
return{output,tokens:{input:usage.input_tokens??0,output:usage.output_tokens??0,cached:usage.cache_read_input_tokens},durationMs,toolCalls:obj?.num_turns??0,modelUsed:typeof obj?.model==='string'&&obj.model?obj.model:defaultModel};
}catch{
if(opts.producer)return{output:'',tokens:{input:0,output:0},durationMs,toolCalls:0,modelUsed:defaultModel,error:{code:'unknown',reason:'empty or invalid output from claude CLI (exit 0)'}};
return{output:raw,tokens:{input:0,output:0},durationMs,toolCalls:0,modelUsed:defaultModel};
}
}
export interface ClaudeProducerPaths { root: string; home: string; workdir: string }
export function assertClaudeProducerPrerequisites(): void {
if (process.platform !== 'linux') return;
const executable = '/usr/bin/bwrap';
let stat: fs.Stats;
try { stat = fs.lstatSync(executable); } catch { throw new Error('CLAUDE_PRODUCER_REQUIRES_TRUSTED_BWRAP'); }
if (!stat.isFile() || stat.isSymbolicLink() || fs.realpathSync(executable) !== executable || stat.uid !== 0 || (stat.mode & 0o022) !== 0 || (stat.mode & 0o111) === 0) {
throw new Error('CLAUDE_PRODUCER_REQUIRES_TRUSTED_BWRAP');
}
}
export function claudeProducerPaths(stateDirectory: string): ClaudeProducerPaths {
const root = path.join(stateDirectory, 'claude-provider');
return { root, home: path.join(root, 'home'), workdir: path.join(root, 'work') };
}
export function prepareClaudeProducerState(stateDirectory: string): ClaudeProducerPaths {
const paths = claudeProducerPaths(stateDirectory);
if (fs.existsSync(paths.root)) throw new Error('CLAUDE_PRODUCER_STATE_EXISTS');
fs.mkdirSync(paths.root, { mode: 0o700 });
fs.mkdirSync(paths.home, { mode: 0o700 });
fs.mkdirSync(paths.workdir, { mode: 0o700 });
return paths;
}
export function removeClaudeProducerState(stateDirectory: string): void {
fs.rmSync(claudeProducerPaths(stateDirectory).root, { recursive: true, force: true });
}
export function claudeExecWorkingDirectory(opts: RunOpts): string {
const stateDirectory = csoProducerStateDirectory(opts);
return stateDirectory ? claudeProducerPaths(stateDirectory).workdir : opts.workdir;
}
export function claudeProducerTools(opts: RunOpts): string {
const launcher = csoProducerHelperLauncher(opts);
if (!launcher) throw new Error('CSO producer helper launcher is required');
return [`Bash(${launcher})`, `Bash(${launcher} *)`, 'Write'].join(',');
}
export function claudeExecArgs(opts: RunOpts, model: string, argsPrefix: readonly string[] = []): string[] {
const args = [...argsPrefix, '-p', '--output-format', 'json', '--model', model];
const stateDirectory = csoProducerStateDirectory(opts);
if (stateDirectory) {
if (opts.extraArgs?.length) throw new Error('CSO producer does not accept extra provider arguments');
const tools = claudeProducerTools(opts);
const sourceDirectory = csoProducerSourceDirectory(opts)!;
const helperHome = csoProducerHelperHome(opts)!;
args.push(
'--restricted',
'--safe-mode',
'--no-session-persistence',
'--permission-prompts', 'none',
'--permission-mode', 'dontAsk',
'--tools', 'Bash,Write',
'--allowed-tools', tools,
'--add-dir', claudeProducerPaths(stateDirectory).workdir,
'--add-dir', sourceDirectory,
'--add-dir', helperHome,
'--strict-mcp-config',
'--no-chrome',
);
}
if (opts.extraArgs) args.push(...opts.extraArgs);
return args;
}
export function claudeExecEnvironment(
opts: RunOpts,
source: NodeJS.ProcessEnv = process.env,
): Record<string, string> {
const stateDirectory = csoProducerStateDirectory(opts);
const paths = stateDirectory ? claudeProducerPaths(stateDirectory) : undefined;
return {
...(opts.csoProducer ? csoProducerChildEnvironment('claude', source) : source),
...(paths ? { HOME: paths.home, GSTACK_HOME: csoProducerHelperHome(opts)! } : {}),
...(stateDirectory ? { CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: '1' } : {}),
GSTACK_HEADLESS: '1',
} as Record<string, string>;
}
+144 -17
View File
@@ -1,9 +1,22 @@
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
import {
csoProducerChildEnvironment,
csoProducerHelperHome,
csoProducerHelperLauncher,
csoProducerProviderCommand,
csoProducerStateDirectory,
validateCsoProducerStateDirectory,
type ProviderAdapter,
type RunOpts,
type RunResult,
type AvailabilityCheck,
} from './types';
import { estimateCostUsd } from '../pricing';
import { execFileSync, spawnSync } from 'child_process';
import { randomBytes } from 'node:crypto';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { atomicWriteSync } from '../../../lib/fs-atomic';
export type GeminiStreamParse = {
output: string;
@@ -112,15 +125,16 @@ export class GeminiAdapter implements ProviderAdapter {
readonly name = 'gemini';
readonly family = 'gemini' as const;
async available(): Promise<AvailabilityCheck> {
const res = spawnSync('sh', ['-c', 'command -v gemini'], { timeout: 2000 });
async available(opts?: RunOpts): Promise<AvailabilityCheck> {
const producerCommand = csoProducerProviderCommand(opts ?? { prompt: '', workdir: '/', timeoutMs: 1 });
const res = producerCommand ? { status: 0 } : spawnSync('sh', ['-c', 'command -v gemini'], { timeout: 2000 });
if (res.status !== 0) {
return { ok: false, reason: 'gemini CLI not found on PATH. Install per https://github.com/google-gemini/gemini-cli' };
}
const legacyCfgDir = path.join(os.homedir(), '.config', 'gemini');
const newCfgDir = path.join(os.homedir(), '.gemini');
const newOauth = path.join(newCfgDir, 'oauth_creds.json');
const hasCfg = fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth);
const hasCfg = !opts?.csoProducer && (fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth));
// CLI accepts either name; Google AI Studio keys are usually GEMINI_API_KEY.
const hasKey = !!(process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY);
if (!hasCfg && !hasKey) {
@@ -138,23 +152,15 @@ export class GeminiAdapter implements ProviderAdapter {
// Default to --yolo (non-interactive) and stream-json output so we can parse
// tokens + tool calls. Callers can override via extraArgs. (--skip-trust was
// removed in gemini-cli 0.34; passing it errors at argv parse.)
const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--yolo'];
if (opts.model) args.push('--model', opts.model);
if (opts.extraArgs) args.push(...opts.extraArgs);
try {
const out = execFileSync('gemini', args, {
cwd: opts.workdir,
const args = geminiExecArgs(opts);
const command = csoProducerProviderCommand(opts);
const out = execFileSync(command?.executable ?? 'gemini', [...(command?.argsPrefix ?? []), ...args], {
cwd: geminiExecWorkingDirectory(opts),
timeout: opts.timeoutMs,
encoding: 'utf-8',
maxBuffer: 32 * 1024 * 1024,
env: {
...process.env,
// Prefer GEMINI_API_KEY when only that is set (CLI reads both).
...(process.env.GEMINI_API_KEY && !process.env.GOOGLE_API_KEY
? { GOOGLE_API_KEY: process.env.GEMINI_API_KEY }
: {}),
},
env: geminiExecEnvironment(opts, process.env),
});
return resultFromGeminiStream(out, { model: opts.model, durationMs: Date.now() - start });
} catch (err: unknown) {
@@ -189,3 +195,124 @@ export class GeminiAdapter implements ProviderAdapter {
};
}
}
export interface GeminiProducerPaths {
root: string;
home: string;
workdir: string;
systemDefaults: string;
systemSettings: string;
}
export function geminiProducerPaths(stateDirectory: string): GeminiProducerPaths {
validateCsoProducerStateDirectory(stateDirectory);
const root = path.join(stateDirectory, 'gemini-provider');
return {
root,
home: path.join(root, 'home'),
workdir: path.join(root, 'work'),
systemDefaults: path.join(root, 'system-defaults.json'),
systemSettings: path.join(root, 'system-settings.json'),
};
}
/** Highest-precedence Gemini policy for a clean, one-cell producer host. */
const GEMINI_PRODUCER_BLOCKED_ENVIRONMENT = [
'GEMINI_API_KEY', 'GOOGLE_API_KEY', 'GOOGLE_CLOUD_PROJECT', 'GOOGLE_CLOUD_LOCATION',
] as const;
export function geminiProducerSystemSettings(contextFileName: string, helperLauncher: string) {
if (!/^\.gstack-cso-context-[a-f0-9]{32}\.md$/.test(contextFileName)) throw new Error('INVALID_GEMINI_CONTEXT_FILENAME');
const helper = csoProducerHelperLauncher({ prompt: '', workdir: '/', timeoutMs: 1, csoProducer: { stateDirectory: '/', sourceDirectory: '/source', helperLauncher, helperGeneration: path.join(path.dirname(helperLauncher), '.gstack-cso-generation'), providerCommand: { executable: '/provider', argsPrefix: [] } } });
if (!helper) throw new Error('CSO producer helper launcher is required');
const tools = [`run_shell_command(${helper})`, 'write_file'];
return {
advanced: { ignoreLocalEnv: true },
admin: {
extensions: { enabled: false },
mcp: { enabled: false },
skills: { enabled: false },
},
context: {
fileName: [contextFileName],
includeDirectoryTree: false,
loadMemoryFromIncludeDirectories: false,
memoryBoundaryMarkers: [],
},
hooksConfig: { enabled: false },
privacy: { usageStatisticsEnabled: false },
security: {
environmentVariableRedaction: { allowed: [], blocked: [...GEMINI_PRODUCER_BLOCKED_ENVIRONMENT], enabled: true },
folderTrust: { enabled: false },
toolSandboxing: false,
},
skills: { enabled: false },
telemetry: { enabled: false, logPrompts: false },
tools: { allowed: tools, core: tools, sandbox: false },
};
}
export function prepareGeminiProducerState(stateDirectory: string, helperLauncher: string): GeminiProducerPaths {
const paths = geminiProducerPaths(stateDirectory);
if (fs.existsSync(paths.root)) throw new Error('GEMINI_PRODUCER_STATE_EXISTS');
fs.mkdirSync(paths.root, { mode: 0o700 });
fs.mkdirSync(paths.home, { mode: 0o700 });
fs.mkdirSync(paths.workdir, { mode: 0o700 });
const contextFileName = `.gstack-cso-context-${randomBytes(16).toString('hex')}.md`;
atomicWriteSync(paths.systemDefaults, '{}\n', { mode: 0o600, noReplace: true });
atomicWriteSync(paths.systemSettings, `${JSON.stringify(geminiProducerSystemSettings(contextFileName, helperLauncher), null, 2)}\n`, { mode: 0o600, noReplace: true });
return paths;
}
export function removeGeminiProducerState(stateDirectory: string): void {
const paths = geminiProducerPaths(stateDirectory);
fs.rmSync(paths.root, { recursive: true, force: true });
}
export function geminiExecArgs(opts: RunOpts): string[] {
const stateDirectory = csoProducerStateDirectory(opts);
const args = ['-p', opts.prompt, '--output-format', 'stream-json'];
if (stateDirectory) {
if (opts.extraArgs?.length) throw new Error('CSO producer does not accept extra provider arguments');
args.push(
'--approval-mode', 'yolo',
'--include-directories', geminiProducerPaths(stateDirectory).workdir,
'-e', 'none',
);
} else {
args.push('--yolo');
}
if (opts.model) args.push('--model', opts.model);
if (opts.extraArgs) args.push(...opts.extraArgs);
return args;
}
export function geminiExecWorkingDirectory(opts: RunOpts): string {
const stateDirectory = csoProducerStateDirectory(opts);
return stateDirectory ? geminiProducerPaths(stateDirectory).workdir : opts.workdir;
}
export function geminiExecEnvironment(
opts: RunOpts,
source: NodeJS.ProcessEnv = process.env,
): Record<string, string> {
const stateDirectory = csoProducerStateDirectory(opts);
const paths = stateDirectory ? geminiProducerPaths(stateDirectory) : undefined;
const env = paths ? csoProducerChildEnvironment('gemini', source) : { ...source } as Record<string, string>;
// Prefer GEMINI_API_KEY when only that is set (CLI reads both).
if (env.GEMINI_API_KEY && !env.GOOGLE_API_KEY) env.GOOGLE_API_KEY = env.GEMINI_API_KEY;
if (paths) {
env.HOME = paths.home;
env.GSTACK_HOME = csoProducerHelperHome(opts)!;
env.GEMINI_SANDBOX = 'false';
env.GEMINI_TELEMETRY_ENABLED = 'false';
env.GEMINI_TELEMETRY_LOG_PROMPTS = 'false';
env.GEMINI_CLI_TRUST_WORKSPACE = 'true';
env.GEMINI_SYSTEM_MD = 'false';
env.GEMINI_WRITE_SYSTEM_MD = 'false';
env.GEMINI_CLI_HOME = paths.home;
env.GEMINI_CLI_SYSTEM_DEFAULTS_PATH = paths.systemDefaults;
env.GEMINI_CLI_SYSTEM_SETTINGS_PATH = paths.systemSettings;
}
return env;
}
+154 -60
View File
@@ -1,10 +1,23 @@
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
import {
csoProducerChildEnvironment,
csoProducerHelperGeneration,
csoProducerHelperHome,
csoProducerProviderCommand,
csoProducerSourceDirectory,
csoProducerStateDirectory,
CSO_PRODUCER_SHELL_ENV,
type ProviderAdapter,
type RunOpts,
type RunResult,
type AvailabilityCheck,
} from './types';
import { estimateCostUsd } from '../pricing';
import { execFileSync, spawnSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { CODEX_FRONTIER_MODEL } from '../../../scripts/resolvers/constants';
import { atomicWriteSync } from '../../../lib/fs-atomic';
/**
* GPT adapter — wraps the OpenAI `codex` CLI (codex exec with --json output).
@@ -17,45 +30,40 @@ export class GptAdapter implements ProviderAdapter {
readonly name = 'gpt';
readonly family = 'gpt' as const;
async available(): Promise<AvailabilityCheck> {
const res = spawnSync('sh', ['-c', 'command -v codex'], { timeout: 2000 });
async available(opts?: RunOpts): Promise<AvailabilityCheck> {
const producerCommand = csoProducerProviderCommand(opts ?? { prompt: '', workdir: '/', timeoutMs: 1 });
const res = producerCommand ? { status: 0 } : spawnSync('sh', ['-c', 'command -v codex'], { timeout: 2000 });
if (res.status !== 0) {
return { ok: false, reason: 'codex CLI not found on PATH. Install: npm i -g @openai/codex' };
}
// Auth sniff: ~/.codex/ should contain auth state after `codex login`
const codexDir = path.join(os.homedir(), '.codex');
if (!fs.existsSync(codexDir)) {
return { ok: false, reason: 'No ~/.codex/ found. Run `codex login` to authenticate via ChatGPT.' };
const hasFileAuth = !opts?.csoProducer && fs.existsSync(codexDir);
if (!hasFileAuth && !process.env.OPENAI_API_KEY) {
return { ok: false, reason: 'No Codex auth found. Paid CSO producers require OPENAI_API_KEY; other evals may use `codex login`.' };
}
return { ok: true };
}
async run(opts: RunOpts): Promise<RunResult> {
const start = Date.now();
// `-s read-only` is load-bearing safety. With `--skip-git-repo-check` we
// bypass codex's interactive trust prompt for unknown directories (benchmarks
// often run in temp dirs / non-git paths), so the read-only sandbox is now
// the only boundary preventing codex from mutating the workdir. If you ever
// remove `-s read-only`, drop `--skip-git-repo-check` too.
// Existing callers retain `-s read-only`. CSO producer calls use an isolated
// CODEX_HOME with a reviewed permission profile written below.
const model = opts.model ?? process.env.GSTACK_CODEX_MODEL ?? CODEX_FRONTIER_MODEL;
const args = ['exec', opts.prompt, '-C', opts.workdir, '-s', 'read-only', '--skip-git-repo-check', '--json', '-m', model];
if (opts.extraArgs) args.push(...opts.extraArgs);
const stateDirectory = csoProducerStateDirectory(opts);
try {
const out = execFileSync('codex', args, {
cwd: opts.workdir,
if (stateDirectory) prepareCodexProducerState(opts);
const args = codexExecArgs(opts, model);
const command = csoProducerProviderCommand(opts);
const out = execFileSync(command?.executable ?? 'codex', [...(command?.argsPrefix ?? []), ...args], {
cwd: codexExecWorkingDirectory(opts),
timeout: opts.timeoutMs,
encoding: 'utf-8',
maxBuffer: 32 * 1024 * 1024,
...(opts.csoProducer ? { env: codexExecEnvironment(opts, process.env) } : {}),
});
const parsed = this.parseJsonl(out);
return {
output: parsed.output,
tokens: parsed.tokens,
durationMs: Date.now() - start,
toolCalls: parsed.toolCalls,
modelUsed: parsed.modelUsed || model,
};
return resultFromCodexStream(out,{model,durationMs:Date.now()-start,producer:!!opts.csoProducer});
} catch (err: unknown) {
const durationMs = Date.now() - start;
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
@@ -70,6 +78,8 @@ export class GptAdapter implements ProviderAdapter {
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, model);
}
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, model);
} finally {
if (stateDirectory) removeCodexProducerState(stateDirectory);
}
}
@@ -77,44 +87,6 @@ export class GptAdapter implements ProviderAdapter {
return estimateCostUsd(tokens, model ?? CODEX_FRONTIER_MODEL);
}
/**
* Parse codex exec --json JSONL stream.
* Key events:
* - item.completed with item.type === 'agent_message' → text output
* - item.completed with item.type === 'command_execution' → tool call
* - turn.completed → usage.input_tokens, usage.output_tokens
* - thread.started → session id (not used here)
*/
private parseJsonl(raw: string): { output: string; tokens: { input: number; output: number }; toolCalls: number; modelUsed?: string } {
let output = '';
let input = 0;
let out = 0;
let toolCalls = 0;
let modelUsed: string | undefined;
for (const line of raw.split('\n')) {
const s = line.trim();
if (!s) continue;
try {
const obj = JSON.parse(s);
if (obj.type === 'item.completed' && obj.item) {
if (obj.item.type === 'agent_message' && typeof obj.item.text === 'string') {
output += (output ? '\n' : '') + obj.item.text;
} else if (obj.item.type === 'command_execution') {
toolCalls += 1;
}
} else if (obj.type === 'turn.completed') {
const u = obj.usage ?? {};
input += u.input_tokens ?? 0;
out += u.output_tokens ?? 0;
if (obj.model) modelUsed = obj.model;
}
} catch {
// skip malformed lines — codex stderr can leak in
}
}
return { output, tokens: { input, output: out }, toolCalls, modelUsed };
}
private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
return {
output: '',
@@ -126,3 +98,125 @@ export class GptAdapter implements ProviderAdapter {
};
}
}
/** Map `codex exec --json` output while requiring evidence-bearing text for paid producers. */
export function resultFromCodexStream(raw:string,opts:{model?:string;durationMs?:number;producer?:boolean}={}):RunResult{
let output='',input=0,out=0,toolCalls=0,modelUsed:string|undefined;
for(const line of raw.split('\n')){
const text=line.trim();if(!text)continue;
try{
const obj=JSON.parse(text);
if(obj.type==='item.completed'&&obj.item){
if(obj.item.type==='agent_message'&&typeof obj.item.text==='string')output+=(output?'\n':'')+obj.item.text;
else if(obj.item.type==='command_execution')toolCalls++;
}else if(obj.type==='turn.completed'){
const usage=obj.usage??{};input+=usage.input_tokens??0;out+=usage.output_tokens??0;
if(typeof obj.model==='string'&&obj.model)modelUsed=obj.model;
}
}catch{/* Codex can mix diagnostic text into the JSONL stream. */}
}
const durationMs=opts.durationMs??0,resolvedModel=modelUsed||opts.model||CODEX_FRONTIER_MODEL;
if(opts.producer&&!output.trim())return{output:'',tokens:{input:0,output:0},durationMs,toolCalls:0,modelUsed:resolvedModel,error:{code:'unknown',reason:'empty output from codex CLI (exit 0)'}};
return{output,tokens:{input,output:out},durationMs,toolCalls,modelUsed:resolvedModel};
}
export interface CodexProducerPaths {
root: string;
home: string;
workdir: string;
config: string;
}
export function codexProducerPaths(stateDirectory: string): CodexProducerPaths {
const root = path.join(stateDirectory, 'codex-provider');
return { root, home: path.join(root, 'home'), workdir: path.join(root, 'work'), config: path.join(root, 'home', 'config.toml') };
}
export function codexProducerConfig(opts: RunOpts): string {
const launcher = opts.csoProducer?.helperLauncher;
if (!launcher) throw new Error('CSO producer helper launcher is required');
const sourceDirectory = csoProducerSourceDirectory(opts);
if (!sourceDirectory) throw new Error('CSO producer source directory is required');
const helperHome = csoProducerHelperHome(opts);
if (!helperHome) throw new Error('CSO producer helper home is required');
const generation = csoProducerHelperGeneration(opts);
if (!generation) throw new Error('CSO producer helper generation is required');
const directory = path.dirname(launcher), suffix = process.platform === 'win32' ? '.exe' : '';
const trustedArtifacts = [
path.join(directory, `cso-eval-producer${suffix}`), launcher,
path.join(directory, `gstack-cso-core${suffix}`), path.join(directory, `gstack-cso-watchdog${suffix}`),
generation,
];
const grants = [...trustedArtifacts, sourceDirectory].map(file => `${JSON.stringify(file)} = "read"`).join('\n');
return `approval_policy = "never"
default_permissions = "cso-producer"
allow_login_shell = false
check_for_update_on_startup = false
[shell_environment_policy]
inherit = "all"
include_only = ${JSON.stringify(CSO_PRODUCER_SHELL_ENV)}
ignore_default_excludes = false
experimental_use_profile = false
[permissions.cso-producer]
description = "CSO producer: private state plus one immutable source snapshot"
[permissions.cso-producer.filesystem]
":root" = "deny"
":minimal" = "read"
${grants}
${JSON.stringify(helperHome)} = "write"
[permissions.cso-producer.filesystem.":workspace_roots"]
"." = "write"
[permissions.cso-producer.network]
enabled = false
`;
}
export function prepareCodexProducerState(opts: RunOpts): CodexProducerPaths {
const stateDirectory = csoProducerStateDirectory(opts);
if (!stateDirectory) throw new Error('CSO producer state directory is required');
const paths = codexProducerPaths(stateDirectory);
if (fs.existsSync(paths.root)) throw new Error('CODEX_PRODUCER_STATE_EXISTS');
fs.mkdirSync(paths.root, { mode: 0o700 });
fs.mkdirSync(paths.home, { mode: 0o700 });
fs.mkdirSync(paths.workdir, { mode: 0o700 });
atomicWriteSync(paths.config, codexProducerConfig(opts), { mode: 0o600, noReplace: true });
return paths;
}
export function removeCodexProducerState(stateDirectory: string): void {
fs.rmSync(codexProducerPaths(stateDirectory).root, { recursive: true, force: true });
}
/** Exported so free tests can inspect the exact paid-CLI boundary without running it. */
export function codexExecArgs(opts: RunOpts, model: string): string[] {
const stateDirectory = csoProducerStateDirectory(opts);
const args = ['exec', opts.prompt, '-C', stateDirectory ? codexProducerPaths(stateDirectory).workdir : opts.workdir];
if (stateDirectory) {
if (opts.extraArgs?.length) throw new Error('CSO producer does not accept extra provider arguments');
args.push(
'--strict-config', '--ephemeral', '--ignore-rules', '--skip-git-repo-check',
);
} else {
args.push('-s', 'read-only', '--skip-git-repo-check');
}
args.push('--json', '-m', model);
if (opts.extraArgs) args.push(...opts.extraArgs);
return args;
}
export function codexExecWorkingDirectory(opts: RunOpts): string {
const stateDirectory = csoProducerStateDirectory(opts);
return stateDirectory ? codexProducerPaths(stateDirectory).workdir : opts.workdir;
}
export function codexExecEnvironment(opts: RunOpts, source: NodeJS.ProcessEnv = process.env): Record<string, string> {
if (!opts.csoProducer) return source as Record<string, string>;
const stateDirectory = csoProducerStateDirectory(opts)!;
const paths = codexProducerPaths(stateDirectory);
return { ...csoProducerChildEnvironment('gpt', source), HOME: paths.home, CODEX_HOME: paths.home, GSTACK_HOME: csoProducerHelperHome(opts)! };
}
+99 -1
View File
@@ -1,3 +1,5 @@
import * as path from 'node:path';
/**
* Provider adapter interface — uniform contract for Claude, GPT, Gemini.
*
@@ -18,6 +20,19 @@ export interface RunOpts {
model?: string;
/** Extra flags per-provider (escape hatch for rare cases). Prefer staying generic. */
extraArgs?: string[];
/** Producer-only execution policy. Omit to preserve each adapter's defaults. */
csoProducer?: {
/** The one explicit state directory exposed outside the source worktree. */
stateDirectory: string;
/** Exact validated per-cell source root. Providers may grant only this path read-only. */
sourceDirectory: string;
/** Absolute immutable launcher path admitted by the producer. */
helperLauncher: string;
/** Exact adjacent generation manifest read by the launcher. */
helperGeneration: string;
/** Exact provider command whose bytes/version are bound into the receipt. */
providerCommand: { executable: string; argsPrefix: string[] };
};
}
export interface TokenUsage {
@@ -57,6 +72,89 @@ export interface AvailabilityCheck {
export type Family = 'claude' | 'gpt' | 'gemini';
export const CSO_PRODUCER_SHELL_ENV = [
'PATH', 'HOME', 'LANG', 'LC_ALL', 'TZ',
'GSTACK_HOME', 'GSTACK_SESSION_KIND', 'GSTACK_HEADLESS',
] as const;
const CSO_AUTH_ENV: Record<Family, readonly string[]> = {
claude: ['ANTHROPIC_API_KEY', 'CLAUDE_CODE_OAUTH_TOKEN'],
gpt: ['OPENAI_API_KEY'],
gemini: ['GEMINI_API_KEY', 'GOOGLE_API_KEY', 'GOOGLE_CLOUD_PROJECT', 'GOOGLE_CLOUD_LOCATION'],
};
export function validateCsoProducerStateDirectory(stateDirectory: string): string {
if (!path.isAbsolute(stateDirectory) || path.normalize(stateDirectory) !== stateDirectory) {
throw new Error('CSO producer state directory must be an absolute normalized path');
}
return stateDirectory;
}
export function csoProducerStateDirectory(opts: RunOpts): string | undefined {
if (!opts.csoProducer) return undefined;
csoProducerHelperLauncher(opts);
csoProducerHelperGeneration(opts);
csoProducerSourceDirectory(opts);
csoProducerProviderCommand(opts);
return validateCsoProducerStateDirectory(opts.csoProducer.stateDirectory);
}
export function csoProducerSourceDirectory(opts: RunOpts): string | undefined {
if (!opts.csoProducer) return undefined;
return validateCsoProducerStateDirectory(opts.csoProducer.sourceDirectory);
}
export function csoProducerHelperHome(opts: RunOpts): string | undefined {
const stateDirectory = csoProducerStateDirectory(opts);
return stateDirectory ? path.join(stateDirectory, 'cso-home') : undefined;
}
export function csoProducerProviderCommand(opts: RunOpts): { executable: string; argsPrefix: string[] } | undefined {
if (!opts.csoProducer) return undefined;
const command = opts.csoProducer.providerCommand;
if (!command || !path.isAbsolute(command.executable) || path.normalize(command.executable) !== command.executable ||
!Array.isArray(command.argsPrefix) || command.argsPrefix.some(value => typeof value !== 'string' || value.includes('\0'))) {
throw new Error('CSO producer provider command must be an absolute normalized trusted path');
}
return { executable: command.executable, argsPrefix: [...command.argsPrefix] };
}
export function csoProducerHelperLauncher(opts: RunOpts): string | undefined {
if (!opts.csoProducer) return undefined;
const launcher = opts.csoProducer.helperLauncher;
if (typeof launcher !== 'string') throw new Error('CSO producer helper launcher must be an absolute normalized trusted path');
const suffix = process.platform === 'win32' ? '.exe' : '';
if (!path.isAbsolute(launcher) || path.normalize(launcher) !== launcher || path.basename(launcher) !== `gstack-cso-launcher${suffix}` ||
!/^[A-Za-z0-9_./:\\-]+$/.test(launcher)) {
throw new Error('CSO producer helper launcher must be an absolute normalized trusted path');
}
return launcher;
}
export function csoProducerHelperGeneration(opts: RunOpts): string | undefined {
if (!opts.csoProducer) return undefined;
const generation = opts.csoProducer.helperGeneration;
const launcher = csoProducerHelperLauncher(opts);
if (typeof generation !== 'string' || !path.isAbsolute(generation) || path.normalize(generation) !== generation ||
path.basename(generation) !== '.gstack-cso-generation' || path.dirname(generation) !== path.dirname(launcher!) ||
!/^[A-Za-z0-9_./:\\-]+$/.test(generation)) {
throw new Error('CSO producer helper generation must be the exact adjacent absolute normalized trusted path');
}
return generation;
}
/** Copy only execution inputs needed by the selected producer host. */
export function csoProducerChildEnvironment(
family: Family,
source: NodeJS.ProcessEnv = process.env,
): Record<string, string> {
const output: Record<string, string> = {};
for (const key of [...CSO_PRODUCER_SHELL_ENV, ...CSO_AUTH_ENV[family]]) {
const value = source[key];
if (typeof value === 'string' && !value.includes('\0')) output[key] = value;
}
return output;
}
export interface ProviderAdapter {
/** Stable name used in output tables and config (e.g., 'claude', 'gpt', 'gemini'). */
readonly name: string;
@@ -66,7 +164,7 @@ export interface ProviderAdapter {
* Check whether the provider's CLI binary is present and authenticated.
* Should never block >2s. Non-throwing: returns { ok: false, reason } on failure.
*/
available(): Promise<AvailabilityCheck>;
available(opts?: RunOpts): Promise<AvailabilityCheck>;
/** Run a prompt and return normalized RunResult. Non-throwing. Errors go in result.error. */
run(opts: RunOpts): Promise<RunResult>;
/** Estimate USD cost for the reported token usage and model. */