From 91bbd9e6c4c582f9404464517a8845b8618dc654 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 10 Oct 2026 11:59:17 -0700 Subject: [PATCH] v1.91.69.0 fix(cso-eval): producers keep usage on timeout, give the agent reporting time, and accept the helper's own artifact names (#3111) - every helper artifact path tripped the secret redactor: the run ID's 13-digit epoch reads as a phone number, some 32-hex IDs as wallets. Validate paths against the helper layout; schema-shaped ID components are accepted by shape, every other component still goes through redact - the provider was killed at exactly --budget after cell start, before the helper deadline (start + budget) and before finish; give it the budget plus a 300 s reporting grace (evaluator graceSeconds) - Claude producers stream events (stream-json --verbose --include-partial-messages) and add usage per message, so a timed-out receipt keeps tokens, cost, turns and the last text - failed post-run checks name the check in the receipt reason --- CHANGELOG.md | 28 +++++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- package.json | 2 +- scripts/cso-eval-producer.ts | 61 +++++++++-- test/claude-producer-stream.test.ts | 68 +++++++++++++ test/cso-eval.test.ts | 68 +++++++++++-- test/helpers/providers/claude-stream.ts | 130 ++++++++++++++++++++++++ test/helpers/providers/claude.ts | 14 ++- test/provider-model-defaults.test.ts | 2 +- 10 files changed, 352 insertions(+), 25 deletions(-) create mode 100644 test/claude-producer-stream.test.ts create mode 100644 test/helpers/providers/claude-stream.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 992e66188..2910520fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,33 @@ # Changelog +## [1.91.69.0] - 2026-10-10 + +**/cso eval cells now get time to finish their report, keep their spend when they run out of time, and stop mislabeling the helper's own files as secrets.** + +The first paid smoke cells that ran real scanners both ended as `REDACTION_FAILED` with no usage and no artifacts after the full 30 minutes. There were three causes. + +- **Every artifact path tripped the secret check.** The helper names each run `-`, and the secret redactor reads a 13-digit epoch as a phone number. Some 32-hex review IDs also read as wallet addresses. So the first run that actually wrote artifacts failed the post-run inventory, every time. +- **The provider was killed at the budget.** The producer stopped Claude at exactly `--budget` after the cell started. The helper's own deadline is the budget after `start`, and its `finish` writes the report after that deadline. Both agents started the helper within 30 seconds, worked until the deadline, and were killed before they could call `finish`. +- **A kill threw the spend away.** `--output-format json` reports usage only when Claude exits, so a timeout recorded zero tokens. + +### What this means for you + +- Helper artifact names are checked against the helper's layout: `//...`, `public-cache/` and `legacy-imports/`. A path component that is wholly a helper identifier is accepted by its shape. Every other component, including source paths mirrored under `snapshot/` and `readable/`, still goes through the secret redactor. +- The producer gives the provider the budget plus 300 seconds of reporting time (`PRODUCER_REPORTING_GRACE_SECONDS`, the evaluator's frozen `graceSeconds`). The helper still refuses evidence after its deadline, so the measured budget is unchanged. The agent just gets to call `finish`, as a user's agent would. +- Claude producers stream events (`stream-json --verbose --include-partial-messages`) and add up usage message by message. A timed-out receipt carries its tokens, estimated cost, turns and the agent's last text, marked `failed` with `timeout`. A run with no successful, nonblank final result still fails. +- A failed post-run check now names the check in the receipt's reason, for example `artifact inventory check: ...`. A redaction rejection says which component failed and never includes the component itself. + +### Itemized changes + +#### Fixed +- `inventoryProducerArtifacts` validates paths with `assertProducerArtifactPath`. Paths outside the helper layout fail as `INVALID_PRODUCER_ARTIFACTS`. A component that looks like a secret fails as `REDACTION_FAILED` with its position and masked parent. +- `runProducerCell` passes `producerProviderTimeoutMs(budget)`, which is the budget plus 300 s. Integrity failures read ` check: `, and add the provider's own error code when there was one. +- The Claude producer runs through `runClaudeProducerStream` in `test/helpers/providers/claude-stream.ts`. It folds the event stream as it arrives, so no output buffer can overflow. It resolves even when a grandchild holds stdout, and keeps the `Command failed: ` reason on a nonzero exit. Non-producer Claude evals are unchanged. + +#### For contributors +- `test/claude-producer-stream.test.ts` covers stream accounting, the paid result contract, a timed-out stub, and nonzero and clean exits. `test/cso-eval.test.ts` covers the real helper layout (an epoch run ID and a wallet-shaped review ID), a planted secret-shaped file name, and a timed-out receipt that keeps its usage. +- The evaluator's cells need a matching change: an outer timeout of the budget plus 600 s, and the new Claude flags in the stub preflight prefix. + ## [1.91.68.0] - 2026-10-08 **Claude /cso eval cells run the real scanners again, never break their own source check, and still record what they spent when a post-run check fails.** diff --git a/VERSION b/VERSION index 5c9f16c24..1425d44f3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.68.0 +1.91.69.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index b82e32f29..918da2797 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.68.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.69.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/package.json b/package.json index ba58629d2..e51ad6509 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.68", + "version": "1.91.69", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/cso-eval-producer.ts b/scripts/cso-eval-producer.ts index fc8294387..380c319ca 100644 --- a/scripts/cso-eval-producer.ts +++ b/scripts/cso-eval-producer.ts @@ -42,6 +42,16 @@ const OUTPUT_LIMIT = 32 * 1024 * 1024; const ARTIFACT_FILE_LIMIT = 32 * 1024 * 1024; const ARTIFACT_TOTAL_LIMIT = 128 * 1024 * 1024; const ARTIFACT_COUNT_LIMIT = 4096; +/** + * `/cso --budget` bounds the helper run: it accepts evidence until its deadline (start + budget) and `finish` writes the + * report after it. A user's agent is never killed at the budget, so the provider gets the budget, the agent's time + * before `start`, and the reporting time after the deadline. Matches the evaluator's frozen `graceSeconds`. + */ +export const PRODUCER_REPORTING_GRACE_SECONDS = 300; + +export function producerProviderTimeoutMs(budgetSeconds: number): number { + return (budgetSeconds + PRODUCER_REPORTING_GRACE_SECONDS) * 1000; +} function inside(parent: string, child: string): boolean { const path = relative(parent, child); @@ -168,6 +178,34 @@ function emptyArtifactInventory(): ProducerArtifactInventory { return { ...base, identityHash: producerArtifactInventoryHash(base) }; } +/** A whole path component that is one of the helper's own identifiers: `-` run and replay IDs, retired runs, and hex artifact IDs with an optional lowercase-word prefix (`gitleaks--.json`). */ +const HELPER_ID_COMPONENT = /^(?:\d{13}-[a-f0-9]{16}|\.retired-\d{13}-[a-f0-9]{16}-[a-f0-9]{32}|(?:[a-z]+-)*[a-f0-9]{16,64}(?:-[a-f0-9]{16})*(?:\.[a-z]+)?)$/; + +/** + * Helper artifact paths are `//...` plus the helper's public-cache/ and legacy-imports/. Components that + * are wholly a helper identifier are checked by shape, because the secret redactor reads them as data (a 13-digit epoch + * as a phone number, some 32-hex IDs as wallets). Every other component (fixed helper names, and source paths mirrored + * under snapshot/ and readable/) still goes through the redactor. + */ +export function assertProducerArtifactPath(relativePath: string): void { + const parts = relativePath.split('/'); + const reject = (rule: string): never => { throw new Error(`INVALID_PRODUCER_ARTIFACTS: ${rule}`); }; + if (!relativePath || relativePath.length > 1024) reject('an artifact path is empty or longer than 1024 bytes'); + if (!/^[A-Za-z0-9._/-]+$/.test(relativePath) || parts.some(part => !part || part === '.' || part === '..')) reject('an artifact path has a component outside [A-Za-z0-9._-]'); + const inRun = /^[a-f0-9]{24}$/.test(parts[0]) && /^(?:\d{13}-[a-f0-9]{16}|\.retired-\d{13}-[a-f0-9]{16}-[a-f0-9]{32})$/.test(parts[1] ?? ''); + if (parts.length < 2 || !(inRun ? parts.length >= 3 : ['public-cache', 'legacy-imports'].includes(parts[0]))) reject('an artifact lies outside //, public-cache/ and legacy-imports/'); + const schemaId = (part: string, index: number) => index === 0 ? /^[a-f0-9]{24}$/.test(part) : HELPER_ID_COMPONENT.test(part); + for (const [index, part] of parts.entries()) { + if (schemaId(part, index)) continue; + let clean: boolean; + try { clean = redact(part) === part; } catch { clean = false; } + if (!clean) { + const parent = parts.slice(0, index).map((item, at) => schemaId(item, at) ? 'ID' : item).join('/'); + throw new CsoError('REDACTION_FAILED', `artifact path component ${index + 1} under ${JSON.stringify(parent)} looks like a secret; the component is withheld`); + } + } +} + export function inventoryProducerArtifacts(helperHome: string): ProducerArtifactInventory { const security = join(helperHome, 'security'), artifactRoot = join(security, 'cso'); for (const directory of [security, artifactRoot]) { @@ -187,9 +225,7 @@ export function inventoryProducerArtifacts(helperHome: string): ProducerArtifact if (entry.isDirectory()) { visit(full, depth + 1); continue; } if (!entry.isFile() || entries.length >= ARTIFACT_COUNT_LIMIT) throw new Error('INVALID_PRODUCER_ARTIFACTS'); const relativePath = relative(artifactRoot, full).split(sep).join('/'); - if (!relativePath || relativePath.length > 1024 || !/^[A-Za-z0-9._/-]+$/.test(relativePath) || relativePath.split('/').some(part => !part || part === '.' || part === '..') || redact(relativePath) !== relativePath) { - throw new CsoError('REDACTION_FAILED', 'Producer artifact path withheld'); - } + assertProducerArtifactPath(relativePath); const contents = readBoundedStable(full, ARTIFACT_FILE_LIMIT, 'Producer artifact'); totalBytes += contents.byteLength; if (totalBytes > ARTIFACT_TOTAL_LIMIT) throw new Error('PRODUCER_ARTIFACTS_TOO_LARGE'); @@ -494,7 +530,11 @@ export function producerFailureMessage(error: unknown): string { /** A post-run check failed: the receipt records it as a failed cell with no output or artifacts, keeping the run's usage. */ export function producerIntegrityFailure(error: unknown): { code: string; reason: string } { - if (error instanceof CsoError) return { code: error.code, reason: producerFailureMessage(error) }; + if (error instanceof CsoError) { + let reason: string; + try { reason = redact(error.message); } catch { reason = 'detail withheld'; } + return { code: error.code, reason }; + } const message = producerFailureMessage(error); const named = error instanceof Error && !(error as NodeJS.ErrnoException).code ? /^([A-Z][A-Z0-9_]+)(?::\s*([\s\S]*))?$/.exec(message) : null; return named ? { code: named[1], reason: named[2] || named[1] } : { code: 'PRODUCER_POST_RUN_CHECK_FAILED', reason: message }; @@ -544,7 +584,7 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op const runOptions = { prompt: promptFor(cell, input.skill, sourceRoot, helper), workdir: stateRoot, - timeoutMs: cell.budgetSeconds * 1000, + timeoutMs: producerProviderTimeoutMs(cell.budgetSeconds), model: cell.model, csoProducer: { stateDirectory: stateRoot, @@ -584,23 +624,32 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op const finishedAt = new Date().toISOString(); let artifacts = emptyArtifactInventory(); let integrityFailure: { code: string; reason: string } | undefined; + let check = ''; try { + check = 'helper installation'; const installationAfter = producerInstallationIdentity(helper); if (installationAfter.identityHash !== installationIdentity.identityHash) { throw new Error('PRODUCER_HELPER_GENERATION_CHANGED: installed helper identity changed during the run'); } + check = 'provider installation'; const providerAfter = artifactIdentity(provider.command.executable); if (providerAfter.sha256 !== providerIdentity.executable.sha256 || providerAfter.bytes !== providerIdentity.executable.bytes) { throw new Error('PRODUCER_PROVIDER_INSTALLATION_RACE: provider executable changed during the run'); } + check = 'source modes'; assertProducerSourceSealed(sourceRoot); + check = 'source content'; validateSource(sourceRoot, input.source, cell.sourceHash); + check = 'source Git state'; if (repositoryIdentity(sourceRoot) !== originalRepositoryIdentity) throw new Error('PRODUCER_CHANGED_SOURCE: Git HEAD, branch, config or status changed during the run'); + check = 'artifact inventory'; artifacts = inventoryProducerArtifacts(helperHome); + check = 'output size'; const outputBytes = typeof run.output === 'string' ? Buffer.byteLength(run.output) : 0; if (outputBytes > OUTPUT_LIMIT) throw new Error(`PRODUCER_OUTPUT_TOO_LARGE: output is ${outputBytes} bytes; the limit is ${OUTPUT_LIMIT}`); } catch (error) { - integrityFailure = producerIntegrityFailure(error); + const failure = producerIntegrityFailure(error); + integrityFailure = { code: failure.code, reason: `${check} check: ${failure.reason}${run.error ? `; the provider run also failed (${run.error.code})` : ''}` }; artifacts = emptyArtifactInventory(); } if (!integrityFailure) { diff --git a/test/claude-producer-stream.test.ts b/test/claude-producer-stream.test.ts new file mode 100644 index 000000000..5d064d192 --- /dev/null +++ b/test/claude-producer-stream.test.ts @@ -0,0 +1,68 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { claudeStreamState, consumeClaudeStreamLine, resultFromClaudeStream, runClaudeProducerStream } from './helpers/providers/claude-stream'; + +// The event shapes claude 2.1.263 prints with -p --output-format stream-json --verbose --include-partial-messages. +const start = (id: string, input: number, cached = 7) => ({ type: 'stream_event', parent_tool_use_id: null, event: { type: 'message_start', message: { id, usage: { input_tokens: input, output_tokens: 1, cache_read_input_tokens: cached } } } }); +const delta = (output: number) => ({ type: 'stream_event', parent_tool_use_id: null, event: { type: 'message_delta', delta: { stop_reason: 'tool_use' }, usage: { output_tokens: output } } }); +const stop = { type: 'stream_event', parent_tool_use_id: null, event: { type: 'message_stop' } }; +const text = (id: string, value: string) => ({ type: 'assistant', message: { id, content: [{ type: 'text', text: value }], usage: { input_tokens: 1, output_tokens: 1 } } }); +const result = (extra: Record) => ({ type: 'result', subtype: 'success', is_error: false, num_turns: 2, result: 'REPORT', usage: { input_tokens: 2001, output_tokens: 201, cache_read_input_tokens: 14 }, ...extra }); +const lines = (...events: unknown[]) => events.map(event => JSON.stringify(event)); + +const fixtures: string[] = []; +afterEach(() => { for (const path of fixtures.splice(0)) rmSync(path, { recursive: true, force: true }); }); +const stub = (body: string) => { + const directory = mkdtempSync(join(tmpdir(), 'claude-stream-')); fixtures.push(directory); + const script = join(directory, 'claude'); + writeFileSync(script, `#!/bin/sh\ncat > /dev/null\n${body}\n`, { mode: 0o755 }); + return { script, directory }; +}; +const printf = (events: unknown[]) => lines(...events).map(line => `printf '%s\\n' '${line}'`).join('\n'); + +describe('Claude producer stream accounting', () => { + test('counts every message from its stream usage, including one a kill interrupted', () => { + const state = claudeStreamState(); + for (const line of lines(start('a', 1000), text('a', 'first note'), delta(100), stop, start('b', 1001), text('b', 'second note'), delta(40))) consumeClaudeStreamLine(state, line); + const killed = resultFromClaudeStream(state, { model: 'claude-test', durationMs: 5, error: { code: 'timeout', reason: 'exceeded 5ms' } }); + expect(killed).toMatchObject({ tokens: { input: 2001, output: 140, cached: 14 }, toolCalls: 2, output: 'second note', error: { code: 'timeout' }, modelUsed: 'claude-test' }); + }); + + test('takes usage and turns from the final result and keeps the paid result contract', () => { + const state = claudeStreamState(); + for (const line of lines(start('a', 1000), delta(100), stop, result({}))) consumeClaudeStreamLine(state, line); + expect(resultFromClaudeStream(state, { model: 'claude-test', durationMs: 5 })).toEqual({ output: 'REPORT', tokens: { input: 2001, output: 201, cached: 14 }, durationMs: 5, toolCalls: 2, modelUsed: 'claude-test' }); + for (const ending of [[], [result({ result: ' ' })], [result({ result: 7 })], [result({ is_error: true })], [result({ subtype: 'error_max_turns' })]]) { + const partial = claudeStreamState(); + for (const line of lines(start('a', 10), delta(3), stop, ...ending)) consumeClaudeStreamLine(partial, line); + const outcome = resultFromClaudeStream(partial, { model: 'claude-test', durationMs: 1 }); + expect(outcome).toMatchObject({ output: '', error: { code: 'unknown', reason: 'empty or invalid output from claude CLI (exit 0)' } }); + expect(outcome.tokens.input).toBeGreaterThan(0); + } + }); + + test.skipIf(process.platform === 'win32')('a timed-out run returns the usage and last text it streamed before the kill', async () => { + const { script, directory } = stub(`${printf([start('a', 1000), text('a', 'investigating'), delta(100), stop, start('b', 1001), delta(9)])}\nsleep 30`); + const began = Date.now(); + const outcome = await runClaudeProducerStream(script, ['-p'], { input: 'go', cwd: directory, env: { PATH: '/usr/bin:/bin' }, timeoutMs: 1500, model: 'claude-test' }); + expect(Date.now() - began).toBeLessThan(10_000); + expect(outcome).toMatchObject({ output: 'investigating', tokens: { input: 2001, output: 109, cached: 14 }, toolCalls: 2, error: { code: 'timeout', reason: 'exceeded 1500ms' } }); + }); + + test.skipIf(process.platform === 'win32')('keeps the failed-command reason shape and the stream usage on a nonzero exit', async () => { + const { script, directory } = stub(`${printf([result({ is_error: true, result: 'Invalid API key', usage: { input_tokens: 0, output_tokens: 0 } })])}\nexit 1`); + const outcome = await runClaudeProducerStream(script, ['-p', '--model', 'm'], { input: 'go', cwd: directory, env: { PATH: '/usr/bin:/bin' }, timeoutMs: 5000, model: 'claude-test' }); + expect(outcome.error?.code).toBe('unknown'); + expect(outcome.error?.reason.startsWith(`Command failed: ${script} -p --model m`)).toBe(true); + expect(outcome.tokens).toEqual({ input: 0, output: 0, cached: 0 }); + }); + + test.skipIf(process.platform === 'win32')('a clean exit with a final result succeeds', async () => { + const { script, directory } = stub(printf([start('a', 10), delta(2), stop, result({})])); + const outcome = await runClaudeProducerStream(script, [], { input: 'go', cwd: directory, env: { PATH: '/usr/bin:/bin' }, timeoutMs: 5000, model: 'claude-test' }); + expect(outcome).toMatchObject({ output: 'REPORT', tokens: { input: 2001, output: 201, cached: 14 }, toolCalls: 2 }); + expect(outcome.error).toBeUndefined(); + }); +}); diff --git a/test/cso-eval.test.ts b/test/cso-eval.test.ts index e7fde57fd..372c8614d 100644 --- a/test/cso-eval.test.ts +++ b/test/cso-eval.test.ts @@ -5,7 +5,7 @@ import { createHash } from 'node:crypto'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { collectProducerReceipts, createEvalBaseline, createEvalMatrix, createPortableSkillPayload, loadPortableSkillPayload, prepareEvalJobs, REQUIRED_CONTAINMENT, scoreCollectedEval, scoreEval, validateMatrix, validatePortableSkillPayload, type EvalBaseline, type EvalCell, type EvalMatrix, type EvalResult, type PreparedEvalSchedule } from '../scripts/cso-eval'; -import { PRODUCER_PROVIDER_POLICY, producerFailureMessage, producerInstallationIdentity, resolveProducerHelperBinding, runProducerCell, validateProductionProducerInstallation } from '../scripts/cso-eval-producer'; +import { assertProducerArtifactPath, PRODUCER_PROVIDER_POLICY, producerFailureMessage, producerProviderTimeoutMs, producerInstallationIdentity, resolveProducerHelperBinding, runProducerCell, validateProductionProducerInstallation } from '../scripts/cso-eval-producer'; import { producerArtifactInventoryHash, producerHostPlatform, producerInstallationIdentityHash, producerProviderIdentityHash, producerReceiptHash, sha256, type ProducerArtifactInventory, type ProducerInstallationIdentity, type ProducerProviderIdentity, type ProducerReceipt } from '../scripts/cso-eval-protocol'; import type { Family, ProviderAdapter, RunOpts, RunResult } from './helpers/providers/types'; import { CORPUS_VERSION, FAMILIES, STACKS, defineEvalCorpus, loadCorpusManifest, materializeCase, sourceFiles, sourceHash, validateCorpusManifest, type CorpusManifest, type EvalVariant } from './fixtures/cso-eval/materialize'; @@ -17,6 +17,7 @@ import { CsoError } from '../lib/cso/contracts'; import { geminiProducerPaths, geminiProducerSystemSettings } from './helpers/providers/gemini'; const temporary: string[] = []; +const randomHex = (bytes: number) => createHash('sha256').update(`${Math.random()}:${process.hrtime.bigint()}`).digest('hex').slice(0, bytes * 2); const root = () => { const path = mkdtempSync(join(tmpdir(), 'cso-eval-')); temporary.push(path); return path; }; function unlockTemporary(path: string): void { if (!existsSync(path)) return; @@ -526,6 +527,8 @@ describe('CSO matched producer orchestration', () => { testProviderCommand: providerCommandFor(launcher), }); + // The helper's real layout: `/-`; the secret redactor reads that epoch as a phone number. + const helperRun = `${'e'.repeat(24)}/${Date.now()}-${'0123456789abcdef'}`; class FakeAdapter implements ProviderAdapter { readonly name = 'fake'; constructor( @@ -729,7 +732,7 @@ describe('CSO matched producer orchestration', () => { expect(existsSync(input)).toBe(false); expect(opts).toMatchObject({ model: cell.model, - timeoutMs: cell.budgetSeconds * 1000, + timeoutMs: (cell.budgetSeconds + 300) * 1000, workdir: join(isolated.job, 'state'), csoProducer: { stateDirectory: join(isolated.job, 'state'), @@ -751,7 +754,7 @@ describe('CSO matched producer orchestration', () => { expect(opts.prompt).not.toContain(cell.caseId); expect(opts.prompt).not.toContain(cell.variant); expect(opts.prompt).not.toContain('ORACLE_SQL_MARKER'); - const artifactRoot = join(process.env.GSTACK_HOME!, 'security', 'cso', 'run-1'); + const artifactRoot = join(process.env.GSTACK_HOME!, 'security', 'cso', helperRun); mkdirSync(artifactRoot, { recursive: true }); writeFileSync(join(artifactRoot, 'report.json'), '{"status":"complete"}\n'); writeFileSync(join(artifactRoot, 'repair.patch'), 'diff --git a/app.mjs b/app.mjs\n'); @@ -780,10 +783,10 @@ describe('CSO matched producer orchestration', () => { expect(receipt.providerIdentity).toEqual(providerIdentityFor('gpt', launcher)); expect(receipt.artifacts).toMatchObject({ schemaVersion: 1, root: 'security/cso' }); expect(receipt.artifacts.totalBytes).toBe(receipt.artifacts.entries.reduce((sum, entry) => sum + entry.bytes, 0)); - expect(receipt.artifacts.entries.map(entry => entry.path)).toEqual(['run-1/repair.patch', 'run-1/report.json']); + expect(receipt.artifacts.entries.map(entry => entry.path)).toEqual([`${helperRun}/repair.patch`, `${helperRun}/report.json`]); expect(receipt.artifacts.identityHash).toBe(producerArtifactInventoryHash({ schemaVersion: 1, root: 'security/cso', entries: receipt.artifacts.entries, totalBytes: receipt.artifacts.totalBytes })); const retainedHome = join(isolated.job, 'state', 'cso-home'); - expect(readFileSync(join(retainedHome, 'security', 'cso', 'run-1', 'report.json'), 'utf8')).toContain('complete'); + expect(readFileSync(join(retainedHome, 'security', 'cso', helperRun, 'report.json'), 'utf8')).toContain('complete'); expect(receipt.artifacts.entries.every(entry => !/provider|credential|session/i.test(entry.path))).toBe(true); expect(JSON.parse(readFileSync(output, 'utf8'))).toEqual(receipt); const { receiptHash, ...receiptWithoutHash } = receipt; @@ -913,7 +916,7 @@ describe('CSO matched producer orchestration', () => { const receiptPath = join(outputRoot, `${cell.id}.json`); const adapter = new FakeAdapter(opts => writeFileSync(opts.csoProducer!.providerCommand.executable, '#!/bin/sh\nexit 1\n')); const receipt = await runProducerCell(isolated.input, receiptPath, withHelper({ adapter, paidExecutionAuthorized: true }, launcher)); - expectIntegrityReceipt(receipt, receiptPath, { code: 'PRODUCER_PROVIDER_INSTALLATION_RACE', reason: 'provider executable changed during the run' }); + expectIntegrityReceipt(receipt, receiptPath, { code: 'PRODUCER_PROVIDER_INSTALLATION_RACE', reason: 'provider installation check: provider executable changed during the run' }); }); test('records a failed receipt with usage when the helper generation changes during a cell', async () => { @@ -928,7 +931,7 @@ describe('CSO matched producer orchestration', () => { writeFileSync(opts.csoProducer!.helperGeneration, `${sha256(readFileSync(core))}\n`); }); const receipt = await runProducerCell(isolated.input, receiptPath, withHelper({ adapter, paidExecutionAuthorized: true }, launcher)); - expectIntegrityReceipt(receipt, receiptPath, { code: 'PRODUCER_HELPER_GENERATION_CHANGED', reason: 'installed helper identity changed during the run' }); + expectIntegrityReceipt(receipt, receiptPath, { code: 'PRODUCER_HELPER_GENERATION_CHANGED', reason: 'helper installation check: installed helper identity changed during the run' }); }); test('records a failed receipt with usage when the provider output exceeds the limit', async () => { @@ -937,7 +940,50 @@ describe('CSO matched producer orchestration', () => { const isolated = isolate(destination, cell), receiptPath = join(root(), `${cell.id}.json`); const output = 'x'.repeat(32 * 1024 * 1024 + 1); const receipt = await runProducerCell(isolated.input, receiptPath, withHelper({ adapter: new FakeAdapter(() => {}, 'gpt-5.4', 'gpt', { output }), paidExecutionAuthorized: true })); - expectIntegrityReceipt(receipt, receiptPath, { code: 'PRODUCER_OUTPUT_TOO_LARGE', reason: `output is ${output.length} bytes; the limit is ${32 * 1024 * 1024}` }); + expectIntegrityReceipt(receipt, receiptPath, { code: 'PRODUCER_OUTPUT_TOO_LARGE', reason: `output size check: output is ${output.length} bytes; the limit is ${32 * 1024 * 1024}` }); + }); + + test('accepts real helper artifact names the secret redactor misreads, and names the check that rejects the rest', async () => { + const { redact } = await import('../lib/cso/process'); + const runId = `${Date.now()}-${'0123456789abcdef'}`; + let walletShaped = ''; + while (!walletShaped) { const id = randomHex(16); if (redact(id) !== id) walletShaped = id; } + expect(redact(runId)).not.toBe(runId); + const repo = 'e'.repeat(24); + const accepted = [`${repo}/${runId}/report.json`, `${repo}/${runId}/reviews/${walletShaped}.json`, `${repo}/${runId}/scanner-outcomes/gitleaks-${randomHex(8)}-${randomHex(8)}.json`, + `${repo}/${runId}/snapshot/test/fixtures/cso-canary-token.txt`, 'public-cache/osv/index.json']; + for (const path of accepted) expect(() => assertProducerArtifactPath(path)).not.toThrow(); + for (const [path, message] of [ + [`run-1/report.json`, 'INVALID_PRODUCER_ARTIFACTS: an artifact lies outside'], + [`${repo}/report.json`, 'INVALID_PRODUCER_ARTIFACTS: an artifact lies outside'], + [`${repo}/${runId}/bad name.json`, 'INVALID_PRODUCER_ARTIFACTS: an artifact path has a component outside'], + ] as const) expect(() => assertProducerArtifactPath(path)).toThrow(message); + const planted = `sk-ant-api03-${randomHex(24)}`; + expect(redact(planted)).not.toBe(planted); + let failure: unknown; + try { assertProducerArtifactPath(`${repo}/${runId}/notes/${planted}.txt`); } catch (error) { failure = error; } + expect((failure as CsoError).code).toBe('REDACTION_FAILED'); + expect((failure as Error).message).toBe('artifact path component 4 under "ID/ID/notes" looks like a secret; the component is withheld'); + const destination = join(root(), 'planted-artifact-prepared'), cell = selected[0]; + prepareEvalJobs(producerMatrix, skills, destination, [cell.id]); + const isolated = isolate(destination, cell), receiptPath = join(root(), `${cell.id}.json`); + const adapter = new FakeAdapter(() => { + const directory = join(process.env.GSTACK_HOME!, 'security', 'cso', repo, runId, 'notes'); + mkdirSync(directory, { recursive: true }); writeFileSync(join(directory, `${planted}.txt`), 'x'); + }, 'gpt-5.4', 'gpt', { error: { code: 'timeout', reason: 'exceeded 1ms' } }); + const receipt = await runProducerCell(isolated.input, receiptPath, withHelper({ adapter, paidExecutionAuthorized: true })); + expectIntegrityReceipt(receipt, receiptPath, { code: 'REDACTION_FAILED', reason: 'artifact inventory check: artifact path component 4 under "ID/ID/notes" looks like a secret; the component is withheld; the provider run also failed (timeout)' }); + expect(readFileSync(receiptPath, 'utf8')).not.toContain(planted); + }); + + test('gives the provider the budget plus the reporting grace and keeps a timed-out run\'s usage and last result', async () => { + expect(producerProviderTimeoutMs(1800)).toBe(2100 * 1000); + const destination = join(root(), 'timeout-prepared'), cell = selected[0]; + prepareEvalJobs(producerMatrix, skills, destination, [cell.id]); + const isolated = isolate(destination, cell), receiptPath = join(root(), `${cell.id}.json`); + const adapter = new FakeAdapter(() => {}, 'gpt-5.4', 'gpt', { output: 'last assistant text before the kill', error: { code: 'timeout', reason: 'exceeded 2100000ms' } }); + const receipt = await runProducerCell(isolated.input, receiptPath, withHelper({ adapter, paidExecutionAuthorized: true })); + expect(receipt).toMatchObject({ status: 'failed', error: { code: 'timeout', reason: 'exceeded 2100000ms' }, output: 'last assistant text before the kill', toolCalls: 4, usage: { inputTokens: 120, outputTokens: 30, cachedTokens: 10, estimatedCostUSD: 0.0042 } }); }); test('names the first offending source path and rule in a failed receipt that keeps usage', async () => { @@ -957,7 +1003,7 @@ describe('CSO matched producer orchestration', () => { const schedule = prepareEvalJobs(producerMatrix, skills, destination, [cell.id]); const isolated = isolate(destination, cell), receiptPath = join(root(), `${cell.id}.json`); const receipt = await runProducerCell(isolated.input, receiptPath, withHelper({ adapter: new FakeAdapter(opts => mutate(opts.csoProducer!.sourceDirectory, outside)), paidExecutionAuthorized: true })); - expectIntegrityReceipt(receipt, receiptPath, { code: 'INVALID_PRODUCER_SOURCE', reason }); + expectIntegrityReceipt(receipt, receiptPath, { code: 'INVALID_PRODUCER_SOURCE', reason: `source content check: ${reason}` }); const group = collectProducerReceipts(producerMatrix, schedule, [receipt]).summary.groups.find(item => item.version === cell.version && item.mode === cell.mode); expect(group).toMatchObject({ submitted: 1, succeeded: 0, failed: 1, estimatedCost: { measured: 1, totalUSD: 0.0042 } }); } @@ -1044,8 +1090,8 @@ describe('CSO matched producer orchestration', () => { }); const written = await runProducerCell(isolated.input, receipt, withHelper({ adapter, paidExecutionAuthorized: true })); expectIntegrityReceipt(written, receipt, process.platform === 'win32' - ? { code: 'INVALID_PRODUCER_SOURCE', reason: `"app.mjs" is ${'changed by producer\n'.length} bytes; expected ${Buffer.byteLength(original)}` } - : { code: 'PRODUCER_CHANGED_SOURCE_MODE', reason: '"app.mjs" has mode 644; expected 444' }); + ? { code: 'INVALID_PRODUCER_SOURCE', reason: `source content check: "app.mjs" is ${'changed by producer\n'.length} bytes; expected ${Buffer.byteLength(original)}` } + : { code: 'PRODUCER_CHANGED_SOURCE_MODE', reason: 'source modes check: "app.mjs" has mode 644; expected 444' }); expect(readFileSync(join(isolated.source, 'app.mjs'), 'utf8')).not.toBe(original); }); }); diff --git a/test/helpers/providers/claude-stream.ts b/test/helpers/providers/claude-stream.ts new file mode 100644 index 000000000..19221e965 --- /dev/null +++ b/test/helpers/providers/claude-stream.ts @@ -0,0 +1,130 @@ +import { spawn } from 'child_process'; +import type { RunResult } from './types'; + +/** Usage, turns and the latest text from `claude -p --output-format stream-json --verbose --include-partial-messages`. */ +export interface ClaudeStreamState { + totals: { input: number; output: number; cached: number }; + /** The message each stream (main thread or a tool's subagent) is currently receiving. */ + open: Map; + turns: number; + lastText: string; + result?: Record; + invalidLines: number; +} + +export function claudeStreamState(): ClaudeStreamState { + return { totals: { input: 0, output: 0, cached: 0 }, open: new Map(), turns: 0, lastText: '', invalidLines: 0 }; +} + +const count = (value: unknown): number => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : 0; + +function close(state: ClaudeStreamState, key: string): void { + const message = state.open.get(key); + if (!message) return; + state.totals.input += message.input; state.totals.output += message.output; state.totals.cached += message.cached; + state.open.delete(key); +} + +export function consumeClaudeStreamLine(state: ClaudeStreamState, line: string): void { + if (!line.trim()) return; + let event: any; + try { event = JSON.parse(line); } catch { state.invalidLines++; return; } + if (!event || typeof event !== 'object') { state.invalidLines++; return; } + if (event.type === 'result') { state.result = event; return; } + if (event.type === 'assistant' && Array.isArray(event.message?.content)) { + for (const block of event.message.content) if (block?.type === 'text' && typeof block.text === 'string' && block.text.trim()) state.lastText = block.text; + return; + } + if (event.type !== 'stream_event' || !event.event) return; + const key = typeof event.parent_tool_use_id === 'string' ? event.parent_tool_use_id : 'main'; + const stream = event.event; + if (stream.type === 'message_start') { + close(state, key); + const usage = stream.message?.usage ?? {}; + state.open.set(key, { input: count(usage.input_tokens), output: count(usage.output_tokens), cached: count(usage.cache_read_input_tokens) }); + if (key === 'main') state.turns++; + } else if (stream.type === 'message_delta' && stream.usage) { + const message = state.open.get(key); + if (!message) return; + // Anthropic message_delta usage is cumulative for the message. + if (stream.usage.output_tokens !== undefined) message.output = count(stream.usage.output_tokens); + if (stream.usage.input_tokens !== undefined) message.input = count(stream.usage.input_tokens); + if (stream.usage.cache_read_input_tokens !== undefined) message.cached = count(stream.usage.cache_read_input_tokens); + } else if (stream.type === 'message_stop') { + close(state, key); + } +} + +/** Every message seen so far, including ones a kill interrupted mid-stream. */ +export function claudeStreamUsage(state: ClaudeStreamState): { input: number; output: number; cached: number } { + const totals = { ...state.totals }; + for (const message of state.open.values()) { totals.input += message.input; totals.output += message.output; totals.cached += message.cached; } + return totals; +} + +/** + * The final `result` event is authoritative for usage and turns; without it (a killed run) the per-message stream + * usage stands in. A paid producer still needs a successful result event with a nonblank `result`. + */ +export function resultFromClaudeStream(state: ClaudeStreamState, opts: { model: string; durationMs: number; error?: RunResult['error'] }): RunResult { + const result = state.result, usage = (result?.usage ?? undefined) as Record | undefined; + const tokens = usage + ? { input: count(usage.input_tokens), output: count(usage.output_tokens), cached: count(usage.cache_read_input_tokens) } + : claudeStreamUsage(state); + const toolCalls = result && Number.isSafeInteger(result.num_turns) ? result.num_turns as number : state.turns; + const modelUsed = typeof result?.model === 'string' && result.model ? result.model : opts.model; + const base = { tokens, durationMs: opts.durationMs, toolCalls, modelUsed }; + if (opts.error) return { ...base, output: typeof result?.result === 'string' ? result.result : state.lastText, error: opts.error }; + const explicitError = result?.is_error === true || (typeof result?.subtype === 'string' && result.subtype !== 'success'); + if (!result || explicitError || typeof result.result !== 'string' || !result.result.trim()) { + return { ...base, output: '', error: { code: 'unknown', reason: 'empty or invalid output from claude CLI (exit 0)' } }; + } + return { ...base, output: result.result }; +} + +const LINE_LIMIT = 32 * 1024 * 1024, STDERR_LIMIT = 64 * 1024, KILL_GRACE_MS = 10_000; + +/** Run the producer's claude, folding its event stream as it arrives so a timeout still leaves usage behind. */ +export function runClaudeProducerStream(command: string, args: string[], opts: { input: string; cwd: string; env: Record; timeoutMs: number; model: string }): Promise { + return new Promise(resolve => { + const started = Date.now(), state = claudeStreamState(); + let pending = '', stderr = '', timedOut = false, overflow = false, spawnError: Error | undefined; + let killTimer: ReturnType | undefined; + const child = spawn(command, args, { cwd: opts.cwd, env: opts.env, stdio: ['pipe', 'pipe', 'pipe'] }); + const stop = () => { child.kill('SIGTERM'); killTimer = setTimeout(() => child.kill('SIGKILL'), KILL_GRACE_MS); }; + const timer = setTimeout(() => { timedOut = true; stop(); }, opts.timeoutMs); + child.stdout.setEncoding('utf8'); + child.stdout.on('data', (chunk: string) => { + pending += chunk; + let newline: number; + while ((newline = pending.indexOf('\n')) >= 0) { consumeClaudeStreamLine(state, pending.slice(0, newline)); pending = pending.slice(newline + 1); } + if (pending.length > LINE_LIMIT && !overflow) { overflow = true; pending = ''; stop(); } + }); + child.stderr.setEncoding('utf8'); + child.stderr.on('data', (chunk: string) => { if (stderr.length < STDERR_LIMIT) stderr += chunk.slice(0, STDERR_LIMIT - stderr.length); }); + child.stdin.on('error', () => {}); + let settled = false, exitTimer: ReturnType | undefined; + child.on('error', error => { spawnError = error; if (child.pid === undefined) finish(null, null); }); + // A grandchild that inherited stdout must not hold the result open after claude itself exits. + child.on('exit', (code, signal) => { exitTimer = setTimeout(() => { child.stdout.destroy(); child.stderr.destroy(); finish(code, signal); }, 2000); }); + child.on('close', (code, signal) => finish(code, signal)); + function finish(code: number | null, signal: NodeJS.Signals | null): void { + if (settled) return; + settled = true; + clearTimeout(timer); if (killTimer) clearTimeout(killTimer); if (exitTimer) clearTimeout(exitTimer); + if (!overflow) consumeClaudeStreamLine(state, pending); + const durationMs = Date.now() - started; + const fail = (error: NonNullable) => resolve(resultFromClaudeStream(state, { model: opts.model, durationMs, error })); + if (timedOut) return fail({ code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }); + if (overflow) return fail({ code: 'unknown', reason: `claude stream-json line exceeded ${LINE_LIMIT} bytes` }); + if (spawnError) return fail({ code: 'unknown', reason: spawnError.message.slice(0, 400) }); + if (code !== 0) { + if (/unauthorized|auth|login/i.test(stderr)) return fail({ code: 'auth', reason: stderr.slice(0, 400) }); + if (/rate[- ]?limit|429/i.test(stderr)) return fail({ code: 'rate_limit', reason: stderr.slice(0, 400) }); + return fail({ code: 'unknown', reason: `Command failed: ${[command, ...args].join(' ')}${signal ? ` (signal ${signal})` : ''}\n${stderr}`.slice(0, 400) }); + } + resolve(resultFromClaudeStream(state, { model: opts.model, durationMs })); + } + child.stdin.end(opts.input); + }); +} diff --git a/test/helpers/providers/claude.ts b/test/helpers/providers/claude.ts index e2aa9fa2d..90256c7ad 100644 --- a/test/helpers/providers/claude.ts +++ b/test/helpers/providers/claude.ts @@ -16,14 +16,15 @@ import * as path from 'path'; import * as os from 'os'; import { resolveClaudeCommand } from '../../../lib/claude-bin'; import { resolveEvalModel } from '../../../lib/eval-model'; +import { runClaudeProducerStream } from './claude-stream'; /** * Claude adapter — wraps the `claude` CLI via claude -p. * * For brevity and to avoid duplicating the full stream-json parser, this adapter * uses claude CLI in non-interactive mode (--print) with the simpler JSON output - * format. If richer event-level metrics are needed (per-tool timing etc.), - * swap to session-runner's full stream-json parser. + * format. CSO producers stream events instead (claude-stream.ts) so a timed-out + * run still reports its usage. */ export class ClaudeAdapter implements ProviderAdapter { readonly name = 'claude'; @@ -85,7 +86,10 @@ export class ClaudeAdapter implements ProviderAdapter { } try { - if (stateDirectory) prepareClaudeProducerState(stateDirectory); + if (stateDirectory) { + prepareClaudeProducerState(stateDirectory); + return await runClaudeProducerStream(resolved.command, args, { input: opts.prompt, cwd: claudeExecWorkingDirectory(opts), env, timeoutMs: opts.timeoutMs, model }); + } const out = execFileSync(resolved.command, args, { input: opts.prompt, cwd: claudeExecWorkingDirectory(opts), @@ -222,8 +226,10 @@ export function assertClaudeProducerSandboxPolicy(args: readonly string[], env: } export function claudeExecArgs(opts: RunOpts, model: string, argsPrefix: readonly string[] = []): string[] { - const args = [...argsPrefix, '-p', '--output-format', 'json', '--model', model]; const stateDirectory = csoProducerStateDirectory(opts); + // Producers stream events so a timed-out or killed run still reports the usage it spent. + const output = stateDirectory ? ['stream-json', '--verbose', '--include-partial-messages'] : ['json']; + const args = [...argsPrefix, '-p', '--output-format', ...output, '--model', model]; if (stateDirectory) { if (opts.extraArgs?.length) throw new Error('CSO producer does not accept extra provider arguments'); const tools = claudeProducerTools(opts); diff --git a/test/provider-model-defaults.test.ts b/test/provider-model-defaults.test.ts index c1e1192c3..1af08191a 100644 --- a/test/provider-model-defaults.test.ts +++ b/test/provider-model-defaults.test.ts @@ -73,7 +73,7 @@ describe('CSO producer provider policies', () => { ]); const producer = { ...common, csoProducer: policy(state, source, helper) }; expect(claudeExecArgs(producer, 'claude-test')).toEqual([ - '-p', '--output-format', 'json', '--model', 'claude-test', + '-p', '--output-format', 'stream-json', '--verbose', '--include-partial-messages', '--model', 'claude-test', '--restricted', '--safe-mode', '--no-session-persistence', '--permission-prompts', 'none', '--permission-mode', 'dontAsk', '--tools', 'Bash,Write',