v1.91.68.0 fix(cso-eval): Claude producers keep the source sealed, run the launcher outside the sandbox, and keep usage on failed checks (#3093)

* v1.91.68.0 fix(cso-eval): failed receipts with usage for post-run integrity failures; Claude producers stop mutating the sealed source

Claude Code adds <dir>/.mcp.json as a sandbox deny path for every --add-dir.
The producer passed the sealed source as one, so bubblewrap left an empty
0444 .mcp.json mount point there that Claude Code cannot unlink from the 0555
directory, and the post-run source check failed with no receipt.

- claudeExecArgs no longer adds the source directory
- post-run integrity failures write status:failed receipts carrying the
  check code/reason plus usage, tokens and estimated cost (no output or
  artifacts)
- INVALID_PRODUCER_SOURCE / PRODUCER_CHANGED_SOURCE_MODE name the first
  offending relative path and the rule it broke

* fix(cso-eval): run the trusted launcher outside Claude's sandbox; keep every other command sandboxed

Claude Code 2.1.263 forces every Bash command into its bubblewrap sandbox
while CLAUDE_CODE_SUBPROCESS_ENV_SCRUB is set, excludedCommands
notwithstanding. Inside it the launcher's owner check sees root as nobody
and the Docker socket is unreachable, so comprehensive cells degraded to
static-only results.

- drop CLAUDE_CODE_SUBPROCESS_ENV_SCRUB for producers; pass --settings with
  sandbox enabled, failIfUnavailable, no unsandboxed fallback, no
  auto-allow, and excludedCommands naming only the exact launcher path
- ClaudeAdapter.run refuses to start the agent
  (CLAUDE_PRODUCER_SANDBOX_POLICY_MISMATCH) unless args and env carry
  exactly that policy
- README containment note and CHANGELOG updated

* test(doctor): split the two multi-run doctor tests so each fits Windows' 30s per-test budget

On windows-latest each bin/gstack-doctor run costs ~4-5s. The cached/live probe test made 7 runs and the artifacts-sync test 5; both hit the 30s per-test timeout, which killed doctor mid-output ("no row codex probe") and then EBUSY on cleanup. Same failure on #3093 and the fix/runtime-root-source-guard PR. Every assertion is kept; each test now makes at most 4 runs.
This commit is contained in:
Garry Tan authored and GitHub committed 2026-10-09 05:32:45 +00:00
1 parent 54efba6dd5
commit 20eb6202fa
10 files changed
+239 -68

No files matched your search

+50 -10
View File
@@ -1,6 +1,6 @@
import { afterEach, describe, expect, spyOn, test } from 'bun:test';
import * as childProcess from 'node:child_process';
import { chmodSync, cpSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, symlinkSync, writeFileSync } from 'node:fs';
import { chmodSync, cpSync, existsSync, linkSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, symlinkSync, writeFileSync } from 'node:fs';
import { createHash } from 'node:crypto';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
@@ -899,18 +899,24 @@ describe('CSO matched producer orchestration', () => {
}
});
test('withholds the receipt when the bound provider executable changes during a cell', async () => {
const expectIntegrityReceipt = (receipt: ProducerReceipt, receiptPath: string, error: { code: string; reason: string }) => {
expect(receipt).toMatchObject({ status: 'failed', error, output: '', outputHash: sha256(''), durationMs: 1234, toolCalls: 4, usage: { inputTokens: 120, outputTokens: 30, cachedTokens: 10, estimatedCostUSD: 0.0042 } });
expect(receipt.artifacts.entries).toEqual([]);
expect(JSON.parse(readFileSync(receiptPath, 'utf8'))).toEqual(receipt);
};
test('records a failed receipt with usage when the bound provider executable changes during a cell', async () => {
const destination = join(root(), 'provider-race-prepared'), cell = selected[0];
prepareEvalJobs(producerMatrix, skills, destination, [cell.id]);
const isolated = isolate(destination, cell), launcher = helperBundle();
const outputRoot = join(root(), 'provider-race-receipts'); mkdirSync(outputRoot);
const receiptPath = join(outputRoot, `${cell.id}.json`);
const adapter = new FakeAdapter(opts => writeFileSync(opts.csoProducer!.providerCommand.executable, '#!/bin/sh\nexit 1\n'));
await expect(runProducerCell(isolated.input, receiptPath, withHelper({ adapter, paidExecutionAuthorized: true }, launcher))).rejects.toThrow('PRODUCER_PROVIDER_INSTALLATION_RACE');
expect(existsSync(receiptPath)).toBe(false);
const receipt = await runProducerCell(isolated.input, receiptPath, withHelper({ adapter, paidExecutionAuthorized: true }, launcher));
expectIntegrityReceipt(receipt, receiptPath, { code: 'PRODUCER_PROVIDER_INSTALLATION_RACE', reason: 'provider executable changed during the run' });
});
test('withholds the receipt when the helper generation changes during a cell', async () => {
test('records a failed receipt with usage when the helper generation changes during a cell', async () => {
const destination = join(root(), 'helper-generation-race-prepared'), cell = selected[0];
prepareEvalJobs(producerMatrix, skills, destination, [cell.id]);
const isolated = isolate(destination, cell), launcher = helperBundle();
@@ -921,8 +927,40 @@ describe('CSO matched producer orchestration', () => {
writeFileSync(core, process.platform === 'win32' ? 'new test executable\n' : '#!/bin/sh\nexit 1\n', { mode: 0o755 });
writeFileSync(opts.csoProducer!.helperGeneration, `${sha256(readFileSync(core))}\n`);
});
await expect(runProducerCell(isolated.input, receiptPath, withHelper({ adapter, paidExecutionAuthorized: true }, launcher))).rejects.toThrow('PRODUCER_HELPER_GENERATION_CHANGED');
expect(existsSync(receiptPath)).toBe(false);
const receipt = await runProducerCell(isolated.input, receiptPath, withHelper({ adapter, paidExecutionAuthorized: true }, launcher));
expectIntegrityReceipt(receipt, receiptPath, { code: 'PRODUCER_HELPER_GENERATION_CHANGED', reason: 'installed helper identity changed during the run' });
});
test('records a failed receipt with usage when the provider output exceeds the limit', async () => {
const destination = join(root(), 'output-limit-prepared'), cell = selected[0];
prepareEvalJobs(producerMatrix, skills, destination, [cell.id]);
const isolated = isolate(destination, cell), receiptPath = join(root(), `${cell.id}.json`);
const output = 'x'.repeat(32 * 1024 * 1024 + 1);
const receipt = await runProducerCell(isolated.input, receiptPath, withHelper({ adapter: new FakeAdapter(() => {}, 'gpt-5.4', 'gpt', { output }), paidExecutionAuthorized: true }));
expectIntegrityReceipt(receipt, receiptPath, { code: 'PRODUCER_OUTPUT_TOO_LARGE', reason: `output is ${output.length} bytes; the limit is ${32 * 1024 * 1024}` });
});
test('names the first offending source path and rule in a failed receipt that keeps usage', async () => {
if (process.platform === 'win32') return;
const writable = (directory: string, change: () => void) => { chmodSync(directory, 0o755); try { change(); } finally { chmodSync(directory, 0o555); } };
const cases: Array<[string, (source: string, outside: string) => void, string]> = [
['mount-point', source => writable(source, () => writeFileSync(join(source, '.mcp.json'), '', { mode: 0o444 })), '".mcp.json" is not in the expected source listing'],
['missing', source => writable(source, () => rmSync(join(source, 'app.mjs'))), '"app.mjs" is missing from the source'],
['hard-link', (source, outside) => linkSync(join(source, 'app.mjs'), join(outside, 'app.mjs')), '"app.mjs" has 2 hard links; expected 1'],
['content', source => {
const file = join(source, 'app.mjs'), size = statSync(file).size;
chmodSync(file, 0o644); writeFileSync(file, 'y'.repeat(size)); chmodSync(file, 0o444);
}, '"app.mjs" does not match its expected sha256'],
];
for (const [name, mutate, reason] of cases) {
const destination = join(root(), `${name}-prepared`), cell = selected[0], outside = root();
const schedule = prepareEvalJobs(producerMatrix, skills, destination, [cell.id]);
const isolated = isolate(destination, cell), receiptPath = join(root(), `${cell.id}.json`);
const receipt = await runProducerCell(isolated.input, receiptPath, withHelper({ adapter: new FakeAdapter(opts => mutate(opts.csoProducer!.sourceDirectory, outside)), paidExecutionAuthorized: true }));
expectIntegrityReceipt(receipt, receiptPath, { code: 'INVALID_PRODUCER_SOURCE', reason });
const group = collectProducerReceipts(producerMatrix, schedule, [receipt]).summary.groups.find(item => item.version === cell.version && item.mode === cell.mode);
expect(group).toMatchObject({ submitted: 1, succeeded: 0, failed: 1, estimatedCost: { measured: 1, totalUSD: 0.0042 } });
}
});
test('fails closed without a receipt or raw console reason when redaction cannot inspect output', async () => {
@@ -986,7 +1024,7 @@ describe('CSO matched producer orchestration', () => {
expect(written).not.toMatch(/caseId|variant|vulnerable|"fixed"/); expect(written).not.toContain(native.caseId);
});
test('seals source read-only and withholds a receipt after a mode/content mutation', async () => {
test('seals source read-only and records a failed receipt after a mode/content mutation', async () => {
const destination = join(root(), 'prepared');
const cell = selected[0];
prepareEvalJobs(producerMatrix, skills, destination, [cell.id]);
@@ -1004,8 +1042,10 @@ describe('CSO matched producer orchestration', () => {
}
writeFileSync(join(source, 'app.mjs'), 'changed by producer\n');
});
await expect(runProducerCell(isolated.input, receipt, withHelper({ adapter, paidExecutionAuthorized: true }))).rejects.toThrow(process.platform === 'win32' ? 'INVALID_PRODUCER_SOURCE' : 'PRODUCER_CHANGED_SOURCE_MODE');
const written = await runProducerCell(isolated.input, receipt, withHelper({ adapter, paidExecutionAuthorized: true }));
expectIntegrityReceipt(written, receipt, process.platform === 'win32'
? { code: 'INVALID_PRODUCER_SOURCE', reason: `"app.mjs" is ${'changed by producer\n'.length} bytes; expected ${Buffer.byteLength(original)}` }
: { code: 'PRODUCER_CHANGED_SOURCE_MODE', reason: '"app.mjs" has mode 644; expected 444' });
expect(readFileSync(join(isolated.source, 'app.mjs'), 'utf8')).not.toBe(original);
expect(existsSync(receipt)).toBe(false);
});
});
+8 -2
View File
@@ -195,7 +195,7 @@ describe('gstack-doctor', () => {
expect(broken.status).toBe(1);
});
test('the cached probe is reported with its age; only --live runs probe-model', () => {
test('the cached probe is reported with its age and is never re-run without --live', () => {
const f = makeFixture();
const now = Math.floor(Date.now() / 1000);
write(path.join(f.state, '.codex-model-probe'), `MODEL_OK ${now - 7300} 123456789\n`);
@@ -210,7 +210,10 @@ describe('gstack-doctor', () => {
write(path.join(f.state, '.codex-model-probe'), `$(rm -rf ~) ${now} x\n`);
expect(doctor(f).row('codex probe').detail).toContain('unrecognized');
expect(probeCalls(f)).not.toContain('probe-model');
});
test('--live runs probe-model once and maps its exit to the codex probe row', () => {
const f = makeFixture();
const live = doctor(f, ['--live']);
expect(probeCalls(f).match(/probe-model exec/g)?.length).toBe(1);
expect(live.row('codex probe').state).toBe('ok');
@@ -245,7 +248,7 @@ describe('gstack-doctor', () => {
expect(r.out).not.toContain('.brain-sync-status.json');
});
test('artifacts sync: held, ok, stale push and stale drain', () => {
test('artifacts sync: held, then ok', () => {
const f = makeFixture();
syncOn(f, { status: 'held', message: 'held 2 file(s): sk-ant-secret', held_count: 2, held: [{ path: 'projects/p/notes.md', rule: 'anthropic_key', dependents: [], fixes: ['edit', 'skip'] }, { path: 'projects/p/b.md', rule: 'anthropic_key', dependents: [], fixes: ['edit', 'skip'] }], drainable: 0, last_drain_at: iso(60), last_push_at: iso(60) });
const held = doctor(f);
@@ -255,7 +258,10 @@ describe('gstack-doctor', () => {
syncOn(f, { status: 'ok', message: 'pushed 3 file(s)', drainable: 0, last_drain_at: iso(60), last_push_at: iso(60) });
expect(doctor(f).row('artifacts sync').state).toBe('ok');
});
test('artifacts sync: stale push and stale drain warn, a fresh push clears them', () => {
const f = makeFixture();
syncOn(f, { status: 'ok', message: 'x', drainable: 4, last_drain_at: iso(60), last_push_at: iso(90_000) });
const stalePush = doctor(f).row('artifacts sync');
expect(stalePush.state).toBe('warn');
+39 -6
View File
@@ -3,7 +3,6 @@ import {
csoProducerHelperHome,
csoProducerHelperLauncher,
csoProducerProviderCommand,
csoProducerSourceDirectory,
csoProducerStateDirectory,
type ProviderAdapter,
type RunOpts,
@@ -78,13 +77,15 @@ export class ClaudeAdapter implements ProviderAdapter {
}
const model = opts.model ?? process.env.EVALS_MODEL ?? resolveEvalModel('capture');
const stateDirectory = csoProducerStateDirectory(opts);
const args = claudeExecArgs(opts, model, resolved.argsPrefix);
const env = claudeExecEnvironment(opts, process.env);
if (stateDirectory) {
assertClaudeProducerPrerequisites();
assertClaudeProducerSandboxPolicy(args, env, csoProducerHelperLauncher(opts)!);
}
try {
if (stateDirectory) prepareClaudeProducerState(stateDirectory);
const args = claudeExecArgs(opts, model, resolved.argsPrefix);
const out = execFileSync(resolved.command, args, {
input: opts.prompt,
cwd: claudeExecWorkingDirectory(opts),
@@ -93,7 +94,7 @@ export class ClaudeAdapter implements ProviderAdapter {
maxBuffer: 32 * 1024 * 1024,
// Default GSTACK_HEADLESS=1 so a benchmark run classifies as headless (an
// AskUserQuestion failure BLOCKs rather than emitting unanswerable prose).
env: claudeExecEnvironment(opts, process.env),
env,
});
return resultFromClaudeOutput(out,{model,durationMs:Date.now()-start,producer:!!opts.csoProducer});
} catch (err: unknown) {
@@ -186,13 +187,46 @@ export function claudeProducerTools(opts: RunOpts): string {
return [`Bash(${launcher})`, `Bash(${launcher} *)`, 'Write'].join(',');
}
/** Claude's sandbox wraps every Bash command except the trusted launcher, which must run where users run it. */
export function claudeProducerSettings(launcher: string): { sandbox: Record<string, unknown> } {
return {
sandbox: {
enabled: true,
failIfUnavailable: true,
allowUnsandboxedCommands: false,
autoAllowBashIfSandboxed: false,
excludedCommands: [launcher, `${launcher} *`],
},
};
}
export function assertClaudeProducerSandboxPolicy(args: readonly string[], env: Record<string, string>, launcher: string): void {
const fail = (detail: string): never => { throw new Error(`CLAUDE_PRODUCER_SANDBOX_POLICY_MISMATCH: ${detail}`); };
const value = (flag: string): string => {
const positions = args.flatMap((arg, index) => arg === flag || arg.startsWith(`${flag}=`) ? [index] : []);
if (positions.length !== 1 || args[positions[0]] !== flag || positions[0] + 1 >= args.length) fail(`expected exactly one ${flag} <value>`);
return args[positions[0] + 1];
};
if (value('--allowed-tools') !== `Bash(${launcher}),Bash(${launcher} *),Write`) fail('--allowed-tools must name only the launcher and Write');
const raw = value('--settings');
let settings: unknown;
try { settings = JSON.parse(raw); } catch { fail('--settings is not JSON'); }
const expected = { enabled: true, failIfUnavailable: true, allowUnsandboxedCommands: false, autoAllowBashIfSandboxed: false, excludedCommands: [launcher, `${launcher} *`] };
const sandbox = (settings as { sandbox?: Record<string, unknown> } | null)?.sandbox;
if (!settings || typeof settings !== 'object' || Object.keys(settings).join() !== 'sandbox' || !sandbox || typeof sandbox !== 'object') fail('--settings must hold only a sandbox object');
if (Object.keys(sandbox!).sort().join() !== Object.keys(expected).sort().join()) fail('sandbox settings keys differ from the launcher-only exclusion policy');
for (const [key, wanted] of Object.entries(expected)) {
if (JSON.stringify(sandbox![key]) !== JSON.stringify(wanted)) fail(`sandbox.${key} must be ${JSON.stringify(wanted)}`);
}
if (env.CLAUDE_CODE_SUBPROCESS_ENV_SCRUB !== undefined) fail('CLAUDE_CODE_SUBPROCESS_ENV_SCRUB forces every command, the launcher included, into the sandbox');
}
export function claudeExecArgs(opts: RunOpts, model: string, argsPrefix: readonly string[] = []): string[] {
const args = [...argsPrefix, '-p', '--output-format', 'json', '--model', model];
const stateDirectory = csoProducerStateDirectory(opts);
if (stateDirectory) {
if (opts.extraArgs?.length) throw new Error('CSO producer does not accept extra provider arguments');
const tools = claudeProducerTools(opts);
const sourceDirectory = csoProducerSourceDirectory(opts)!;
const helperHome = csoProducerHelperHome(opts)!;
args.push(
'--restricted',
@@ -203,8 +237,8 @@ export function claudeExecArgs(opts: RunOpts, model: string, argsPrefix: readonl
'--tools', 'Bash,Write',
'--allowed-tools', tools,
'--add-dir', claudeProducerPaths(stateDirectory).workdir,
'--add-dir', sourceDirectory,
'--add-dir', helperHome,
'--settings', JSON.stringify(claudeProducerSettings(csoProducerHelperLauncher(opts)!)),
'--strict-mcp-config',
'--no-chrome',
);
@@ -222,7 +256,6 @@ export function claudeExecEnvironment(
return {
...(opts.csoProducer ? csoProducerChildEnvironment('claude', source) : source),
...(paths ? { HOME: paths.home, GSTACK_HOME: csoProducerHelperHome(opts)! } : {}),
...(stateDirectory ? { CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: '1' } : {}),
GSTACK_HEADLESS: '1',
} as Record<string, string>;
}
+35 -3
View File
@@ -3,7 +3,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { execFileSync } from 'node:child_process';
import { ClaudeAdapter, claudeExecArgs, claudeExecEnvironment, claudeExecWorkingDirectory, claudeProducerPaths, resultFromClaudeOutput } from './helpers/providers/claude';
import { assertClaudeProducerSandboxPolicy, ClaudeAdapter, claudeExecArgs, claudeExecEnvironment, claudeExecWorkingDirectory, claudeProducerPaths, resultFromClaudeOutput } from './helpers/providers/claude';
import { codexExecArgs, codexExecEnvironment, codexExecWorkingDirectory, codexProducerConfig, codexProducerPaths, GptAdapter, resultFromCodexStream } from './helpers/providers/gpt';
import { geminiExecArgs, geminiExecEnvironment, geminiExecWorkingDirectory, geminiProducerPaths, geminiProducerSystemSettings } from './helpers/providers/gemini';
import { CSO_PRODUCER_SHELL_ENV, csoProducerChildEnvironment } from './helpers/providers/types';
@@ -79,15 +79,47 @@ describe('CSO producer provider policies', () => {
'--tools', 'Bash,Write',
'--allowed-tools', `Bash(${helper}),Bash(${helper} *),Write`,
'--add-dir', claudeProducerPaths(state).workdir,
'--add-dir', source,
'--add-dir', join(state, 'cso-home'),
'--settings', JSON.stringify({ sandbox: { enabled: true, failIfUnavailable: true, allowUnsandboxedCommands: false, autoAllowBashIfSandboxed: false, excludedCommands: [helper, `${helper} *`] } }),
'--strict-mcp-config', '--no-chrome',
]);
expect(claudeExecArgs(producer, 'claude-test')).not.toContain(source);
expect(claudeExecWorkingDirectory(common)).toBe(work);
expect(claudeExecWorkingDirectory(producer)).toBe(claudeProducerPaths(state).workdir);
expect(claudeExecArgs(producer, 'claude-test').join(' ')).not.toMatch(/bypassPermissions|danger/);
});
test('fails the Claude producer closed unless its sandbox exempts exactly the launcher', () => {
const work = join(tmpdir(), 'gstack-claude-sandbox-policy');
const state = join(work, 'state');
const helper = helperAt(join(work, 'installed'));
const producer = { prompt: '', workdir: work, timeoutMs: 1, csoProducer: policy(state, join(work, 'source'), helper) };
const args = claudeExecArgs(producer, 'claude-test'), env = claudeExecEnvironment(producer, { PATH: '/bin', ANTHROPIC_API_KEY: 'key' });
expect(() => assertClaudeProducerSandboxPolicy(args, env, helper)).not.toThrow();
const at = args.indexOf('--settings'), settings = JSON.parse(args[at + 1]);
const withSettings = (change: (sandbox: Record<string, unknown>) => void) => {
const next = structuredClone(settings); change(next.sandbox);
return args.map((arg, index) => index === at + 1 ? JSON.stringify(next) : arg);
};
const withoutSettings = args.filter((_, index) => index !== at && index !== at + 1);
for (const [tampered, tamperedEnv, detail] of [
[withoutSettings, env, 'expected exactly one --settings'],
[[...args, '--settings', args[at + 1]], env, 'expected exactly one --settings'],
[[...withoutSettings, `--settings=${args[at + 1]}`], env, 'expected exactly one --settings'],
[withSettings(sandbox => { sandbox.excludedCommands = [helper, `${helper} *`, 'cat']; }), env, 'sandbox.excludedCommands'],
[withSettings(sandbox => { sandbox.excludedCommands = [`${helper}-link`, `${helper}-link *`]; }), env, 'sandbox.excludedCommands'],
[withSettings(sandbox => { sandbox.allowUnsandboxedCommands = true; }), env, 'sandbox.allowUnsandboxedCommands'],
[withSettings(sandbox => { delete sandbox.autoAllowBashIfSandboxed; }), env, 'keys differ'],
[withSettings(sandbox => { sandbox.enabled = false; }), env, 'sandbox.enabled'],
[withSettings(sandbox => { sandbox.dangerouslyDisableSandbox = true; }), env, 'keys differ'],
[args.map(arg => arg.startsWith('Bash(') ? `${arg},Bash(cat *)` : arg), env, '--allowed-tools'],
[args, { ...env, CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: '1' }, 'CLAUDE_CODE_SUBPROCESS_ENV_SCRUB'],
] as Array<[string[], Record<string, string>, string]>) {
expect(() => assertClaudeProducerSandboxPolicy(tampered, tamperedEnv, helper)).toThrow(`CLAUDE_PRODUCER_SANDBOX_POLICY_MISMATCH: `);
expect(() => assertClaudeProducerSandboxPolicy(tampered, tamperedEnv, helper)).toThrow(detail);
}
});
test('keeps exact Gemini defaults and replaces deprecated yolo only for the producer', () => {
const work = join(tmpdir(), 'gstack-gemini-work');
const state = join(work, 'state');
@@ -154,7 +186,7 @@ describe('CSO producer provider policies', () => {
const claudePaths = claudeProducerPaths(state);
expect(claudeExecEnvironment(common, sourceEnv)).toEqual({
...safe, HOME: claudePaths.home, GSTACK_HOME: join(state, 'cso-home'),
ANTHROPIC_API_KEY: 'anthropic-auth', CLAUDE_CODE_OAUTH_TOKEN: 'claude-auth', CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: '1',
ANTHROPIC_API_KEY: 'anthropic-auth', CLAUDE_CODE_OAUTH_TOKEN: 'claude-auth',
});
const geminiPaths = geminiProducerPaths(state);
expect(geminiExecEnvironment(common, sourceEnv)).toEqual({