From 4350c0621d1a886852f4aaef40bae9010d7d15d3 Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 9 Oct 2026 04:55:49 +0000 Subject: [PATCH] v1.91.68.0 fix(cso-eval): failed receipts with usage for post-run integrity failures; Claude producers stop mutating the sealed source Claude Code adds /.mcp.json as a sandbox deny path for every --add-dir. The producer passed the sealed source as one, so bubblewrap left an empty 0444 .mcp.json mount point there that Claude Code cannot unlink from the 0555 directory, and the post-run source check failed with no receipt. - claudeExecArgs no longer adds the source directory - post-run integrity failures write status:failed receipts carrying the check code/reason plus usage, tokens and estimated cost (no output or artifacts) - INVALID_PRODUCER_SOURCE / PRODUCER_CHANGED_SOURCE_MODE name the first offending relative path and the rule it broke --- CHANGELOG.md | 22 +++++ README.md | 2 +- VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- package.json | 2 +- scripts/cso-eval-producer.ts | 119 +++++++++++++++++---------- test/cso-eval.test.ts | 60 +++++++++++--- test/helpers/providers/claude.ts | 3 - test/provider-model-defaults.test.ts | 2 +- 9 files changed, 153 insertions(+), 61 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d8d7967b..debeb865f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## [1.91.68.0] - 2026-10-08 + +**A /cso eval cell whose post-run checks fail now still records what it spent, and Claude producers no longer break their own source check.** + +The first paid baseline smoke cells ran the agent to completion, then stopped with a bare `INVALID_PRODUCER_SOURCE` and no receipt. That meant no tokens, no cost and no clue which file was at fault. The cause was the producer's own Claude launch. It passed the sealed source copy to Claude as an `--add-dir`. Claude Code treats each added directory as a sandbox write root and blocks writes to its `.mcp.json`. With no `.mcp.json` present, bubblewrap creates an empty one on the host as a mount point. Claude Code can't remove it from the read-only directory afterwards, so the source check rightly failed. + +### What this means for you + +- Claude producers no longer receive the source as an added directory. The constrained launcher reads the sealed copy through the sandbox's read-only root, so the source stays byte-for-byte unchanged. +- When a post-run integrity check fails, the producer writes a receipt with `status: "failed"` instead of no receipt. The receipt carries the check's code and reason, plus the run's usage, tokens, estimated cost, duration and tool calls. This applies to a changed source, mode or Git state, a changed helper or provider installation, an artifact problem, and oversized output. That receipt has no output and no artifacts, and it never counts as succeeded. +- `INVALID_PRODUCER_SOURCE` and `PRODUCER_CHANGED_SOURCE_MODE` name the first offending relative path and the rule it broke, for example `".mcp.json" is not in the expected source listing`. They never include file contents. + +### Itemized changes + +#### Fixed +- `claudeExecArgs` drops `--add-dir ` for CSO producers. The README's containment note now explains why. +- `runProducerCell` turns post-run check failures into failed receipts that keep usage. A redaction failure still withholds the receipt, as before. +- Source validation reports a missing or extra path, the hard-link count, a size or hash mismatch, symlinks and special files, and a Git `include` directive, each with its path. + +#### For contributors +- New tests cover a failed receipt for a provider race, a helper-generation change, oversized output, four source mutations (including a leftover empty `.mcp.json`), and a mode change. Each test checks the usage numbers and confirms collection counts the cell as failed. + ## [1.91.67.0] - 2026-10-08 **Antigravity CLI is now a gstack host.** diff --git a/README.md b/README.md index 5ad4ab569..23580fd72 100644 --- a/README.md +++ b/README.md @@ -412,7 +412,7 @@ Beyond the slash-command skills, gstack ships standalone CLIs for workflows that The private paid CSO evaluation producer is a packaging contract, not an ordinary `bun run build` artifact. On macOS or Linux, a release operator compiles `cso-eval-producer` with the documented hardened Bun flags in the same clean build session as `bun run build:cso`, then installs it beside `gstack-cso-launcher`, `gstack-cso-core`, `gstack-cso-watchdog`, and the hidden `.gstack-cso-generation` manifest as one root-owned, nonwritable five-artifact unit. The producer rejects root execution, writable/symlinked/incomplete installations, and unreviewed provider CLI versions; each receipt binds all five artifact hashes, and collection rejects receipts from different unit identities. See the [clean producer procedure](test/fixtures/cso-eval/README.md#trusted-five-artifact-producer-unit). Paid producer evaluation remains unavailable on Windows because the detached watchdog has no Windows build. -Each producer gets a curated one-cell source copy with directories sealed to `0555` and files to `0444`, plus pre/post content, Git, and mode checks. Claude receives that exact copy as a restricted read-only add-directory so its constrained launcher command can reach it. Gemini receives no source working directory or include-directory. Codex technically receives read-only filesystem access to the exact curated source root because the trusted helper inherits the Codex permission profile; its private provider work directory and exact `cso-home` artifact directory are the only write roots, every other root path remains denied, and the producer prompt requires source access through the helper. This is evaluation containment for an immutable public fixture, not a claim that Codex cannot directly read that fixture. +Each producer gets a curated one-cell source copy with directories sealed to `0555` and files to `0444`, plus pre/post content, Git, and mode checks. Claude receives no source add-directory: its constrained launcher command reads the sealed copy through the sandbox's read-only root, and an add-directory would be a sandbox write root where Claude Code's bubblewrap leaves an empty `.mcp.json` mount point. Gemini receives no source working directory or include-directory. Codex technically receives read-only filesystem access to the exact curated source root because the trusted helper inherits the Codex permission profile; its private provider work directory and exact `cso-home` artifact directory are the only write roots, every other root path remains denied, and the producer prompt requires source access through the helper. This is evaluation containment for an immutable public fixture, not a claim that Codex cannot directly read that fixture. Copy each completed cell's receipt together with `state/cso-home/security/cso/` to the trusted adjudication host. Receipt entries are sorted paths relative to that directory and bind every retained file's size and SHA-256 plus an aggregate inventory hash. Provider homes, settings, sessions, and credentials live under separate disposable directories and are removed after the cell; they are never part of the retained artifact tree. The adjudicator must re-hash the transported tree against the receipt before trusting reports or repair bundles. diff --git a/VERSION b/VERSION index e72f1c71d..5c9f16c24 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.67.0 +1.91.68.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 09ac7532c..b82e32f29 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.67.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.68.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/package.json b/package.json index d2380061d..ba58629d2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.67", + "version": "1.91.68", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/cso-eval-producer.ts b/scripts/cso-eval-producer.ts index 32ea5592e..fc8294387 100644 --- a/scripts/cso-eval-producer.ts +++ b/scripts/cso-eval-producer.ts @@ -48,9 +48,14 @@ function inside(parent: string, child: string): boolean { return path === '' || (!path.startsWith(`..${sep}`) && path !== '..' && !isAbsolute(path)); } -function regularFile(path: string): fs.Stats { +function sourceError(path: string, rule: string): Error { + return new Error(`INVALID_PRODUCER_SOURCE: ${JSON.stringify(path.length > 200 ? `${path.slice(0, 200)}...` : path)} ${rule}`); +} + +function regularFile(path: string, relativePath: string): fs.Stats { const stat = fs.lstatSync(path); - if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1) throw new Error('INVALID_PRODUCER_SOURCE'); + if (!stat.isFile() || stat.isSymbolicLink()) throw sourceError(relativePath, 'is not a regular file'); + if (stat.nlink !== 1) throw sourceError(relativePath, `has ${stat.nlink} hard links; expected 1`); return stat; } @@ -77,15 +82,15 @@ function assertReceiptDestination(path: string): void { function walk(root: string, directory = root): string[] { const output: string[] = []; for (const entry of fs.readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) { + const full = join(directory, entry.name), relativePath = relative(root, full).split(sep).join('/'); if (directory === root && entry.name === '.git') { - if (!entry.isDirectory() || entry.isSymbolicLink() || fs.realpathSync(join(root, '.git')) !== join(root, '.git')) throw new Error('INVALID_PRODUCER_SOURCE'); + if (!entry.isDirectory() || entry.isSymbolicLink() || fs.realpathSync(join(root, '.git')) !== join(root, '.git')) throw sourceError('.git', 'is not a real directory'); continue; } - if (entry.isSymbolicLink()) throw new Error('INVALID_PRODUCER_SOURCE'); - const full = join(directory, entry.name); + if (entry.isSymbolicLink()) throw sourceError(relativePath, 'is a symbolic link'); if (entry.isDirectory()) output.push(...walk(root, full)); - else if (entry.isFile()) output.push(relative(root, full).split(sep).join('/')); - else throw new Error('INVALID_PRODUCER_SOURCE'); + else if (entry.isFile()) output.push(relativePath); + else throw sourceError(relativePath, 'is not a regular file or directory'); } return output.sort(); } @@ -96,19 +101,24 @@ function validateSource(root: string, entries: ProducerSourceEntry[], expectedHa if (!entry || typeof entry.path !== 'string' || !entry.path || entry.path.includes('\\') || entry.path.startsWith('/') || entry.path.split('/').some(part => !part || part === '.' || part === '..') || seen.has(entry.path) || !HEX.test(entry.sha256) || !Number.isSafeInteger(entry.bytes) || entry.bytes < 0) throw new Error('INVALID_PRODUCER_INPUT'); seen.add(entry.path); } - const actualPaths = walk(root); - const expectedPaths = entries.map(entry => entry.path).sort(); - if (JSON.stringify(actualPaths) !== JSON.stringify(expectedPaths)) throw new Error('INVALID_PRODUCER_SOURCE'); + const actualPaths = walk(root), actual = new Set(actualPaths); + const expectedPaths = entries.map(entry => entry.path).sort(), expected = new Set(expectedPaths); + const firstMismatch = [ + ...actualPaths.filter(path => !expected.has(path)).map(path => [path, 'is not in the expected source listing']), + ...expectedPaths.filter(path => !actual.has(path)).map(path => [path, 'is missing from the source']), + ].sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0)[0]; + if (firstMismatch) throw sourceError(firstMismatch[0], firstMismatch[1]); const hashed: Array<[string, string]> = []; for (const entry of [...entries].sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0)) { const path = resolve(root, ...entry.path.split('/')); - if (!inside(root, path)) throw new Error('INVALID_PRODUCER_SOURCE'); - const stat = regularFile(path); + if (!inside(root, path)) throw sourceError(entry.path, 'resolves outside the source root'); + const stat = regularFile(path, entry.path); const contents = readBoundedStable(path, 2 * 1024 * 1024, 'Producer source file'); - if (stat.size !== entry.bytes || contents.byteLength !== entry.bytes || sha256(contents) !== entry.sha256) throw new Error('INVALID_PRODUCER_SOURCE'); + if (stat.size !== entry.bytes || contents.byteLength !== entry.bytes) throw sourceError(entry.path, `is ${contents.byteLength} bytes; expected ${entry.bytes}`); + if (sha256(contents) !== entry.sha256) throw sourceError(entry.path, 'does not match its expected sha256'); hashed.push([entry.path, entry.sha256]); } - if (sha256(JSON.stringify(hashed)) !== expectedHash) throw new Error('INVALID_PRODUCER_SOURCE'); + if (sha256(JSON.stringify(hashed)) !== expectedHash) throw sourceError('.', 'listing does not match the cell sourceHash'); } /** Seal the one-cell source copy before a provider starts. Production producers are macOS/Linux only. */ @@ -118,14 +128,14 @@ export function sealProducerSource(root: string): void { const files: string[] = []; const visit = (directory: string): void => { const directoryStat = fs.lstatSync(directory); - if (!directoryStat.isDirectory() || directoryStat.isSymbolicLink()) throw new Error('INVALID_PRODUCER_SOURCE'); + if (!directoryStat.isDirectory() || directoryStat.isSymbolicLink()) throw sourceError(relative(root, directory) || '.', 'is not a real directory'); directories.push(directory); for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { const full = join(directory, entry.name); - if (entry.isSymbolicLink()) throw new Error('INVALID_PRODUCER_SOURCE'); + if (entry.isSymbolicLink()) throw sourceError(relative(root, full), 'is a symbolic link'); if (entry.isDirectory()) visit(full); else if (entry.isFile()) files.push(full); - else throw new Error('INVALID_PRODUCER_SOURCE'); + else throw sourceError(relative(root, full), 'is not a regular file or directory'); } }; visit(root); @@ -138,28 +148,32 @@ export function sealProducerSource(root: string): void { export function assertProducerSourceSealed(root: string): void { if (process.platform === 'win32') return; const visit = (directory: string): void => { + const modeError = (path: string, rule: string) => new Error(`PRODUCER_CHANGED_SOURCE_MODE: ${JSON.stringify(relative(root, path) || '.')} ${rule}`); const directoryStat = fs.lstatSync(directory); - if (!directoryStat.isDirectory() || directoryStat.isSymbolicLink() || (directoryStat.mode & 0o777) !== 0o555) throw new Error('PRODUCER_CHANGED_SOURCE_MODE'); + if (!directoryStat.isDirectory() || directoryStat.isSymbolicLink()) throw modeError(directory, 'is not a real directory'); + if ((directoryStat.mode & 0o777) !== 0o555) throw modeError(directory, `has mode ${(directoryStat.mode & 0o777).toString(8)}; expected 555`); for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { const full = join(directory, entry.name), stat = fs.lstatSync(full); - if (entry.isSymbolicLink()) throw new Error('PRODUCER_CHANGED_SOURCE_MODE'); + if (entry.isSymbolicLink()) throw modeError(full, 'is a symbolic link'); if (entry.isDirectory()) visit(full); - else if (!entry.isFile() || (stat.mode & 0o777) !== 0o444) throw new Error('PRODUCER_CHANGED_SOURCE_MODE'); + else if (!entry.isFile()) throw modeError(full, 'is not a regular file or directory'); + else if ((stat.mode & 0o777) !== 0o444) throw modeError(full, `has mode ${(stat.mode & 0o777).toString(8)}; expected 444`); } }; visit(root); } +function emptyArtifactInventory(): ProducerArtifactInventory { + const base = { schemaVersion: 1 as const, root: 'security/cso' as const, entries: [], totalBytes: 0 }; + return { ...base, identityHash: producerArtifactInventoryHash(base) }; +} + export function inventoryProducerArtifacts(helperHome: string): ProducerArtifactInventory { - const empty = (): ProducerArtifactInventory => { - const base = { schemaVersion: 1 as const, root: 'security/cso' as const, entries: [], totalBytes: 0 }; - return { ...base, identityHash: producerArtifactInventoryHash(base) }; - }; const security = join(helperHome, 'security'), artifactRoot = join(security, 'cso'); for (const directory of [security, artifactRoot]) { let stat: fs.Stats; try { stat = fs.lstatSync(directory); } - catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return empty(); throw error; } + catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return emptyArtifactInventory(); throw error; } if (!stat.isDirectory() || stat.isSymbolicLink() || fs.realpathSync(directory) !== directory) throw new Error('INVALID_PRODUCER_ARTIFACTS'); } const entries: ProducerSourceEntry[] = []; @@ -193,7 +207,7 @@ function repositoryIdentity(root: string): string { const env = { PATH: process.env.PATH ?? '', LANG: 'C', LC_ALL: 'C', GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: nullPath, GIT_ATTR_NOSYSTEM: '1', GIT_OPTIONAL_LOCKS: '0' }; const git = executable('git'); const config = readBoundedStable(join(root, '.git', 'config'), 1024 * 1024, 'Producer Git configuration').toString('utf8'); - if (/^\s*\[\s*include(?:if)?(?=[\s."\]])/im.test(config)) throw new Error('INVALID_PRODUCER_SOURCE'); + if (/^\s*\[\s*include(?:if)?(?=[\s."\]])/im.test(config)) throw sourceError('.git/config', 'contains an include directive'); const common = ['-c', 'core.fsmonitor=false', '-c', 'core.untrackedCache=false', '-c', `core.hooksPath=${nullPath}`, '-c', `core.attributesFile=${nullPath}`, '-c', `core.excludesFile=${nullPath}`, '-c', 'core.pager=cat', '-C', root]; const read = (args: string[]) => execFileSync(git, [...common, ...args], { env, encoding: 'utf8', timeout: 10_000, maxBuffer: 1024 * 1024 }); return sha256(JSON.stringify({ config: sha256(config), head: read(['rev-parse', '--verify', 'HEAD']).trim(), branch: read(['symbolic-ref', '--short', 'HEAD']).trim(), status: read(['status', '--porcelain=v2', '--untracked-files=all']) })); @@ -478,6 +492,14 @@ export function producerFailureMessage(error: unknown): string { catch { return 'REDACTION_FAILED: producer error withheld'; } } +/** A post-run check failed: the receipt records it as a failed cell with no output or artifacts, keeping the run's usage. */ +export function producerIntegrityFailure(error: unknown): { code: string; reason: string } { + if (error instanceof CsoError) return { code: error.code, reason: producerFailureMessage(error) }; + const message = producerFailureMessage(error); + const named = error instanceof Error && !(error as NodeJS.ErrnoException).code ? /^([A-Z][A-Z0-9_]+)(?::\s*([\s\S]*))?$/.exec(message) : null; + return named ? { code: named[1], reason: named[2] || named[1] } : { code: 'PRODUCER_POST_RUN_CHECK_FAILED', reason: message }; +} + export async function runProducerCell(inputPath: string, receiptPath: string, options: { adapter?: ProviderAdapter; paidExecutionAuthorized: boolean; @@ -560,23 +582,34 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op } } const finishedAt = new Date().toISOString(); - const installationAfter = producerInstallationIdentity(helper); - if (installationAfter.identityHash !== installationIdentity.identityHash) { - throw new Error('PRODUCER_HELPER_GENERATION_CHANGED'); + let artifacts = emptyArtifactInventory(); + let integrityFailure: { code: string; reason: string } | undefined; + try { + const installationAfter = producerInstallationIdentity(helper); + if (installationAfter.identityHash !== installationIdentity.identityHash) { + throw new Error('PRODUCER_HELPER_GENERATION_CHANGED: installed helper identity changed during the run'); + } + const providerAfter = artifactIdentity(provider.command.executable); + if (providerAfter.sha256 !== providerIdentity.executable.sha256 || providerAfter.bytes !== providerIdentity.executable.bytes) { + throw new Error('PRODUCER_PROVIDER_INSTALLATION_RACE: provider executable changed during the run'); + } + assertProducerSourceSealed(sourceRoot); + validateSource(sourceRoot, input.source, cell.sourceHash); + if (repositoryIdentity(sourceRoot) !== originalRepositoryIdentity) throw new Error('PRODUCER_CHANGED_SOURCE: Git HEAD, branch, config or status changed during the run'); + artifacts = inventoryProducerArtifacts(helperHome); + const outputBytes = typeof run.output === 'string' ? Buffer.byteLength(run.output) : 0; + if (outputBytes > OUTPUT_LIMIT) throw new Error(`PRODUCER_OUTPUT_TOO_LARGE: output is ${outputBytes} bytes; the limit is ${OUTPUT_LIMIT}`); + } catch (error) { + integrityFailure = producerIntegrityFailure(error); + artifacts = emptyArtifactInventory(); } - const providerAfter = artifactIdentity(provider.command.executable); - if (providerAfter.sha256 !== providerIdentity.executable.sha256 || providerAfter.bytes !== providerIdentity.executable.bytes) { - throw new Error('PRODUCER_PROVIDER_INSTALLATION_RACE'); + if (!integrityFailure) { + run = sanitizeProducerRun(run); + if(!run.error&&!run.output.trim())run={...run,error:{code:'unknown',reason:'empty output from provider CLI (exit 0)'}}; } - assertProducerSourceSealed(sourceRoot); - validateSource(sourceRoot, input.source, cell.sourceHash); - if (repositoryIdentity(sourceRoot) !== originalRepositoryIdentity) throw new Error('PRODUCER_CHANGED_SOURCE'); - const artifacts = inventoryProducerArtifacts(helperHome); - run = sanitizeProducerRun(run); - if(!run.error&&!run.output.trim())run={...run,error:{code:'unknown',reason:'empty output from provider CLI (exit 0)'}}; - if (Buffer.byteLength(run.output) > OUTPUT_LIMIT) throw new Error('PRODUCER_OUTPUT_TOO_LARGE'); const tokensReported = run.tokens.input > 0 || run.tokens.output > 0 || (run.tokens.cached ?? 0) > 0; const estimatedCostUSD = tokensReported && PRICING[run.modelUsed] ? adapter.estimateCost(run.tokens, run.modelUsed) : null; + const failure = integrityFailure ?? run.error, output = integrityFailure ? '' : run.output; const withoutHash: Omit = { schemaVersion: 2, cellRef: input.cellRef, @@ -586,22 +619,22 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op artifacts, startedAt, finishedAt, - status: run.error ? 'failed' : 'succeeded', + status: failure ? 'failed' : 'succeeded', requestedModel: cell.model, modelUsed: run.modelUsed, modelIdentitySource: run.modelUsed === cell.model ? 'requested_pin' : 'provider_reported', durationMs: run.durationMs, firstUsefulResultMs: null, toolCalls: run.toolCalls, - output: run.output, - outputHash: sha256(run.output), + output, + outputHash: sha256(output), usage: { inputTokens: tokensReported ? run.tokens.input : null, outputTokens: tokensReported ? run.tokens.output : null, cachedTokens: tokensReported && run.tokens.cached !== undefined ? run.tokens.cached : null, estimatedCostUSD, }, - ...(run.error ? { error: run.error } : {}), + ...(failure ? { error: failure } : {}), }; const result = { ...withoutHash, receiptHash: producerReceiptHash(withoutHash) }; writeExclusiveAtomic(receipt, result); diff --git a/test/cso-eval.test.ts b/test/cso-eval.test.ts index 444b43422..e7fde57fd 100644 --- a/test/cso-eval.test.ts +++ b/test/cso-eval.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, spyOn, test } from 'bun:test'; import * as childProcess from 'node:child_process'; -import { chmodSync, cpSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, symlinkSync, writeFileSync } from 'node:fs'; +import { chmodSync, cpSync, existsSync, linkSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, symlinkSync, writeFileSync } from 'node:fs'; import { createHash } from 'node:crypto'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; @@ -899,18 +899,24 @@ describe('CSO matched producer orchestration', () => { } }); - test('withholds the receipt when the bound provider executable changes during a cell', async () => { + const expectIntegrityReceipt = (receipt: ProducerReceipt, receiptPath: string, error: { code: string; reason: string }) => { + expect(receipt).toMatchObject({ status: 'failed', error, output: '', outputHash: sha256(''), durationMs: 1234, toolCalls: 4, usage: { inputTokens: 120, outputTokens: 30, cachedTokens: 10, estimatedCostUSD: 0.0042 } }); + expect(receipt.artifacts.entries).toEqual([]); + expect(JSON.parse(readFileSync(receiptPath, 'utf8'))).toEqual(receipt); + }; + + test('records a failed receipt with usage when the bound provider executable changes during a cell', async () => { const destination = join(root(), 'provider-race-prepared'), cell = selected[0]; prepareEvalJobs(producerMatrix, skills, destination, [cell.id]); const isolated = isolate(destination, cell), launcher = helperBundle(); const outputRoot = join(root(), 'provider-race-receipts'); mkdirSync(outputRoot); const receiptPath = join(outputRoot, `${cell.id}.json`); const adapter = new FakeAdapter(opts => writeFileSync(opts.csoProducer!.providerCommand.executable, '#!/bin/sh\nexit 1\n')); - await expect(runProducerCell(isolated.input, receiptPath, withHelper({ adapter, paidExecutionAuthorized: true }, launcher))).rejects.toThrow('PRODUCER_PROVIDER_INSTALLATION_RACE'); - expect(existsSync(receiptPath)).toBe(false); + const receipt = await runProducerCell(isolated.input, receiptPath, withHelper({ adapter, paidExecutionAuthorized: true }, launcher)); + expectIntegrityReceipt(receipt, receiptPath, { code: 'PRODUCER_PROVIDER_INSTALLATION_RACE', reason: 'provider executable changed during the run' }); }); - test('withholds the receipt when the helper generation changes during a cell', async () => { + test('records a failed receipt with usage when the helper generation changes during a cell', async () => { const destination = join(root(), 'helper-generation-race-prepared'), cell = selected[0]; prepareEvalJobs(producerMatrix, skills, destination, [cell.id]); const isolated = isolate(destination, cell), launcher = helperBundle(); @@ -921,8 +927,40 @@ describe('CSO matched producer orchestration', () => { writeFileSync(core, process.platform === 'win32' ? 'new test executable\n' : '#!/bin/sh\nexit 1\n', { mode: 0o755 }); writeFileSync(opts.csoProducer!.helperGeneration, `${sha256(readFileSync(core))}\n`); }); - await expect(runProducerCell(isolated.input, receiptPath, withHelper({ adapter, paidExecutionAuthorized: true }, launcher))).rejects.toThrow('PRODUCER_HELPER_GENERATION_CHANGED'); - expect(existsSync(receiptPath)).toBe(false); + const receipt = await runProducerCell(isolated.input, receiptPath, withHelper({ adapter, paidExecutionAuthorized: true }, launcher)); + expectIntegrityReceipt(receipt, receiptPath, { code: 'PRODUCER_HELPER_GENERATION_CHANGED', reason: 'installed helper identity changed during the run' }); + }); + + test('records a failed receipt with usage when the provider output exceeds the limit', async () => { + const destination = join(root(), 'output-limit-prepared'), cell = selected[0]; + prepareEvalJobs(producerMatrix, skills, destination, [cell.id]); + const isolated = isolate(destination, cell), receiptPath = join(root(), `${cell.id}.json`); + const output = 'x'.repeat(32 * 1024 * 1024 + 1); + const receipt = await runProducerCell(isolated.input, receiptPath, withHelper({ adapter: new FakeAdapter(() => {}, 'gpt-5.4', 'gpt', { output }), paidExecutionAuthorized: true })); + expectIntegrityReceipt(receipt, receiptPath, { code: 'PRODUCER_OUTPUT_TOO_LARGE', reason: `output is ${output.length} bytes; the limit is ${32 * 1024 * 1024}` }); + }); + + test('names the first offending source path and rule in a failed receipt that keeps usage', async () => { + if (process.platform === 'win32') return; + const writable = (directory: string, change: () => void) => { chmodSync(directory, 0o755); try { change(); } finally { chmodSync(directory, 0o555); } }; + const cases: Array<[string, (source: string, outside: string) => void, string]> = [ + ['mount-point', source => writable(source, () => writeFileSync(join(source, '.mcp.json'), '', { mode: 0o444 })), '".mcp.json" is not in the expected source listing'], + ['missing', source => writable(source, () => rmSync(join(source, 'app.mjs'))), '"app.mjs" is missing from the source'], + ['hard-link', (source, outside) => linkSync(join(source, 'app.mjs'), join(outside, 'app.mjs')), '"app.mjs" has 2 hard links; expected 1'], + ['content', source => { + const file = join(source, 'app.mjs'), size = statSync(file).size; + chmodSync(file, 0o644); writeFileSync(file, 'y'.repeat(size)); chmodSync(file, 0o444); + }, '"app.mjs" does not match its expected sha256'], + ]; + for (const [name, mutate, reason] of cases) { + const destination = join(root(), `${name}-prepared`), cell = selected[0], outside = root(); + const schedule = prepareEvalJobs(producerMatrix, skills, destination, [cell.id]); + const isolated = isolate(destination, cell), receiptPath = join(root(), `${cell.id}.json`); + const receipt = await runProducerCell(isolated.input, receiptPath, withHelper({ adapter: new FakeAdapter(opts => mutate(opts.csoProducer!.sourceDirectory, outside)), paidExecutionAuthorized: true })); + expectIntegrityReceipt(receipt, receiptPath, { code: 'INVALID_PRODUCER_SOURCE', reason }); + const group = collectProducerReceipts(producerMatrix, schedule, [receipt]).summary.groups.find(item => item.version === cell.version && item.mode === cell.mode); + expect(group).toMatchObject({ submitted: 1, succeeded: 0, failed: 1, estimatedCost: { measured: 1, totalUSD: 0.0042 } }); + } }); test('fails closed without a receipt or raw console reason when redaction cannot inspect output', async () => { @@ -986,7 +1024,7 @@ describe('CSO matched producer orchestration', () => { expect(written).not.toMatch(/caseId|variant|vulnerable|"fixed"/); expect(written).not.toContain(native.caseId); }); - test('seals source read-only and withholds a receipt after a mode/content mutation', async () => { + test('seals source read-only and records a failed receipt after a mode/content mutation', async () => { const destination = join(root(), 'prepared'); const cell = selected[0]; prepareEvalJobs(producerMatrix, skills, destination, [cell.id]); @@ -1004,8 +1042,10 @@ describe('CSO matched producer orchestration', () => { } writeFileSync(join(source, 'app.mjs'), 'changed by producer\n'); }); - await expect(runProducerCell(isolated.input, receipt, withHelper({ adapter, paidExecutionAuthorized: true }))).rejects.toThrow(process.platform === 'win32' ? 'INVALID_PRODUCER_SOURCE' : 'PRODUCER_CHANGED_SOURCE_MODE'); + const written = await runProducerCell(isolated.input, receipt, withHelper({ adapter, paidExecutionAuthorized: true })); + expectIntegrityReceipt(written, receipt, process.platform === 'win32' + ? { code: 'INVALID_PRODUCER_SOURCE', reason: `"app.mjs" is ${'changed by producer\n'.length} bytes; expected ${Buffer.byteLength(original)}` } + : { code: 'PRODUCER_CHANGED_SOURCE_MODE', reason: '"app.mjs" has mode 644; expected 444' }); expect(readFileSync(join(isolated.source, 'app.mjs'), 'utf8')).not.toBe(original); - expect(existsSync(receipt)).toBe(false); }); }); diff --git a/test/helpers/providers/claude.ts b/test/helpers/providers/claude.ts index ba95e0764..0e094279f 100644 --- a/test/helpers/providers/claude.ts +++ b/test/helpers/providers/claude.ts @@ -3,7 +3,6 @@ import { csoProducerHelperHome, csoProducerHelperLauncher, csoProducerProviderCommand, - csoProducerSourceDirectory, csoProducerStateDirectory, type ProviderAdapter, type RunOpts, @@ -192,7 +191,6 @@ export function claudeExecArgs(opts: RunOpts, model: string, argsPrefix: readonl if (stateDirectory) { if (opts.extraArgs?.length) throw new Error('CSO producer does not accept extra provider arguments'); const tools = claudeProducerTools(opts); - const sourceDirectory = csoProducerSourceDirectory(opts)!; const helperHome = csoProducerHelperHome(opts)!; args.push( '--restricted', @@ -203,7 +201,6 @@ export function claudeExecArgs(opts: RunOpts, model: string, argsPrefix: readonl '--tools', 'Bash,Write', '--allowed-tools', tools, '--add-dir', claudeProducerPaths(stateDirectory).workdir, - '--add-dir', sourceDirectory, '--add-dir', helperHome, '--strict-mcp-config', '--no-chrome', diff --git a/test/provider-model-defaults.test.ts b/test/provider-model-defaults.test.ts index a957d1486..71e22b649 100644 --- a/test/provider-model-defaults.test.ts +++ b/test/provider-model-defaults.test.ts @@ -79,10 +79,10 @@ describe('CSO producer provider policies', () => { '--tools', 'Bash,Write', '--allowed-tools', `Bash(${helper}),Bash(${helper} *),Write`, '--add-dir', claudeProducerPaths(state).workdir, - '--add-dir', source, '--add-dir', join(state, 'cso-home'), '--strict-mcp-config', '--no-chrome', ]); + expect(claudeExecArgs(producer, 'claude-test')).not.toContain(source); expect(claudeExecWorkingDirectory(common)).toBe(work); expect(claudeExecWorkingDirectory(producer)).toBe(claudeProducerPaths(state).workdir); expect(claudeExecArgs(producer, 'claude-test').join(' ')).not.toMatch(/bypassPermissions|danger/);