From 1e3a3e83f24f3ae1e793635ae422ced366e1e8bd Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 11 Oct 2026 09:21:14 -0700 Subject: [PATCH] v1.91.87.0 fix(cso-eval): give the paid /cso producer the Agent tool users have, and count subagent tokens (#3141) The paid CSO producer gives `/cso` the Agent tool users have, so independent review and repair verification can run in the evaluation. The producer's receipt now counts subagent tokens. --- CHANGELOG.md | 28 +++++++++++++++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- package.json | 2 +- scripts/cso-eval-producer.ts | 2 +- test/claude-producer-stream.test.ts | 18 ++++++++++ test/cso-eval.test.ts | 2 +- test/helpers/providers/claude-stream.ts | 48 +++++++++++++++++++------ test/helpers/providers/claude.ts | 10 +++++- test/provider-model-defaults.test.ts | 12 +++++-- 10 files changed, 107 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14269c851..a810bb6a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,33 @@ # Changelog +## [1.91.87.0] - 2026-10-11 + +**The paid CSO producer gives `/cso` the Agent tool users have, so independent review and repair verification can run in the evaluation. The producer's receipt now counts subagent tokens.** + +The current-skill smoke reports (run 38090891646) said repair verification was "not performed": `record-review` refused the review with "Repair review must be explicitly independent". The producer started Claude with `--tools Bash,Write`, so the Agent tool was missing. `/cso` could start no independent reviewer, honestly declined to attest independence, and so every repair stayed `not_attempted`. Users' Claude Code has the Agent tool. The v2 skill's parallel finding verification uses it too, and fell back to self-verification in the v2 baseline for the same reason. + +### What this means for you + +- **The producer allows `Bash,Write,Agent`.** Subagents inherit the whole policy. Claude Code 2.1.263, replayed against a mock API through the real producer, shows this in practice: + - A subagent has only Agent, Bash and Write, and no MCP servers. + - Its launcher calls run unsandboxed and reach Docker. Its other Bash is denied, including chained, path-variant and `dangerouslyDisableSandbox` commands. + - `--restricted` keeps its Write calls inside the producer's work and helper directories. + - Read, WebFetch, ListMcpResourcesTool and `mcp__*` tools are unavailable to it. + - A subagent's `record-review` is accepted (`self_attested`), and `verify` then passes the review check. +- **Policy revision `claude-2.1.263-cso-v2`.** The tool set changed, so the producer policy identity changed with it. A recorded v2 baseline bound to `cso-v1` no longer matches, and the baseline must be measured again under this policy. +- **Subagent tokens are counted.** A `result` event's `usage` covers only the main thread's last turn. The receipt now takes the session total from `modelUsage` and sums turns across every `result`. A background subagent gives the main thread a second result. A killed run also counts the subagent messages it streamed. + +### Itemized changes + +#### Changed +- `test/helpers/providers/claude.ts`: `--tools Bash,Write,Agent` (`CLAUDE_PRODUCER_BUILTIN_TOOLS`). `assertClaudeProducerSandboxPolicy` also pins `--tools`, `--permission-mode dontAsk` and exactly one each of `--restricted`, `--safe-mode` and `--strict-mcp-config`. It refuses `--mcp-config`, `--agents`, `--plugin-dir`, `--setting-sources`, `--disallowed-tools` and the permission-bypass flags. +- `test/helpers/providers/claude-stream.ts`: tokens from `modelUsage`, turns summed over results, subagent messages counted once by message id, and only main-thread text kept as a killed run's output. +- `scripts/cso-eval-producer.ts`: Claude policy revision `claude-2.1.263-cso-v2`. + +#### For contributors +- `test/provider-model-defaults.test.ts`: the tool list and each new policy refusal. +- `test/claude-producer-stream.test.ts`: subagent accounting with the event shapes Claude Code 2.1.263 printed in the replay, including a background subagent's second result and a killed run. + ## [1.91.86.0] - 2026-10-11 **browse follows the proxy your shell already uses, writes PDFs on Windows machines whose temp dir is an 8.3 short path, and stops spawning `--config=NUL` children that crashed Bun 1.3.10 (Wave 9, lane E).** diff --git a/VERSION b/VERSION index 4279168d0..96604cf14 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.86.0 +1.91.87.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index bcc38ef39..d7bacbb05 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.86.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.87.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/package.json b/package.json index 5ebe68b97..9ff86cbbd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.86", + "version": "1.91.87", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/cso-eval-producer.ts b/scripts/cso-eval-producer.ts index 58e9978ff..333e71268 100644 --- a/scripts/cso-eval-producer.ts +++ b/scripts/cso-eval-producer.ts @@ -433,7 +433,7 @@ export function producerInstallationIdentity(binding: ProducerHelperBinding): Pr } export const PRODUCER_PROVIDER_POLICY: Record = { - claude: { family: 'claude', policyRevision: 'claude-2.1.263-cso-v1', version: '2.1.263 (Claude Code)' }, + claude: { family: 'claude', policyRevision: 'claude-2.1.263-cso-v2', version: '2.1.263 (Claude Code)' }, codex: { family: 'gpt', policyRevision: 'codex-0.153.4-cso-v3-generation', version: 'codex-cli 0.153.4' }, gemini: { family: 'gemini', policyRevision: 'gemini-0.59.0-cso-v1', version: '0.59.0' }, }; diff --git a/test/claude-producer-stream.test.ts b/test/claude-producer-stream.test.ts index 5d064d192..badfbe836 100644 --- a/test/claude-producer-stream.test.ts +++ b/test/claude-producer-stream.test.ts @@ -43,6 +43,24 @@ describe('Claude producer stream accounting', () => { } }); + // Shapes claude 2.1.263 printed in the mock-API replay when the producer's agent ran an Agent subagent in the background. + const subagent = (id: string, input: number) => ({ type: 'assistant', parent_tool_use_id: 'toolu_agent', message: { id, model: 'claude-test', content: [{ type: 'text', text: 'subagent note' }], usage: { input_tokens: input, output_tokens: 1, cache_read_input_tokens: 7 } } }); + const sessionUsage = { 'claude-test': { inputTokens: 153009, outputTokens: 15308, cacheReadInputTokens: 49, cacheCreationInputTokens: 0, costUSD: 2.3 } }; + + test('counts subagent usage: the session modelUsage, every result turn, and the main thread text', () => { + const state = claudeStreamState(); + for (const line of lines(start('a', 1000), delta(100), stop, subagent('s1', 50000), subagent('s1', 50000), subagent('s2', 50001), + result({ result: 'MAIN ENDED TURN', modelUsage: sessionUsage }), start('b', 1002), delta(102), stop, + result({ num_turns: 1, result: 'REPORT', usage: { input_tokens: 1002, output_tokens: 102, cache_read_input_tokens: 7 }, modelUsage: sessionUsage }))) consumeClaudeStreamLine(state, line); + expect(resultFromClaudeStream(state, { model: 'claude-test', durationMs: 5 })).toEqual({ output: 'REPORT', tokens: { input: 153009, output: 15308, cached: 49 }, durationMs: 5, toolCalls: 3, modelUsed: 'claude-test' }); + }); + + test('a killed run counts each subagent message once and never reports subagent text as the producer output', () => { + const state = claudeStreamState(); + for (const line of lines(start('a', 1000), text('a', 'main note'), delta(100), stop, subagent('s1', 50000), subagent('s1', 50000), subagent('s2', 50001))) consumeClaudeStreamLine(state, line); + expect(resultFromClaudeStream(state, { model: 'claude-test', durationMs: 5, error: { code: 'timeout', reason: 'exceeded 5ms' } })).toMatchObject({ output: 'main note', tokens: { input: 101001, output: 102, cached: 21 }, toolCalls: 1 }); + }); + test.skipIf(process.platform === 'win32')('a timed-out run returns the usage and last text it streamed before the kill', async () => { const { script, directory } = stub(`${printf([start('a', 1000), text('a', 'investigating'), delta(100), stop, start('b', 1001), delta(9)])}\nsleep 30`); const began = Date.now(); diff --git a/test/cso-eval.test.ts b/test/cso-eval.test.ts index e75ef38f3..75f0b77bb 100644 --- a/test/cso-eval.test.ts +++ b/test/cso-eval.test.ts @@ -629,7 +629,7 @@ describe('CSO matched producer orchestration', () => { test('pins reviewed provider CLI versions and policy identities', () => { expect(PRODUCER_PROVIDER_POLICY).toEqual({ - claude: { family: 'claude', policyRevision: 'claude-2.1.263-cso-v1', version: '2.1.263 (Claude Code)' }, + claude: { family: 'claude', policyRevision: 'claude-2.1.263-cso-v2', version: '2.1.263 (Claude Code)' }, codex: { family: 'gpt', policyRevision: 'codex-0.153.4-cso-v3-generation', version: 'codex-cli 0.153.4' }, gemini: { family: 'gemini', policyRevision: 'gemini-0.59.0-cso-v1', version: '0.59.0' }, }); diff --git a/test/helpers/providers/claude-stream.ts b/test/helpers/providers/claude-stream.ts index 19221e965..18374b592 100644 --- a/test/helpers/providers/claude-stream.ts +++ b/test/helpers/providers/claude-stream.ts @@ -6,14 +6,18 @@ export interface ClaudeStreamState { totals: { input: number; output: number; cached: number }; /** The message each stream (main thread or a tool's subagent) is currently receiving. */ open: Map; + /** Subagent messages, by message id: claude prints them only as `assistant` events tagged with their Agent call. */ + subagents: Map; turns: number; + /** Turns summed over every `result` event: a background subagent's completion gives the main thread another result. */ + resultTurns: number; lastText: string; result?: Record; invalidLines: number; } export function claudeStreamState(): ClaudeStreamState { - return { totals: { input: 0, output: 0, cached: 0 }, open: new Map(), turns: 0, lastText: '', invalidLines: 0 }; + return { totals: { input: 0, output: 0, cached: 0 }, open: new Map(), subagents: new Map(), turns: 0, resultTurns: 0, lastText: '', invalidLines: 0 }; } const count = (value: unknown): number => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : 0; @@ -30,7 +34,16 @@ export function consumeClaudeStreamLine(state: ClaudeStreamState, line: string): let event: any; try { event = JSON.parse(line); } catch { state.invalidLines++; return; } if (!event || typeof event !== 'object') { state.invalidLines++; return; } - if (event.type === 'result') { state.result = event; return; } + if (event.type === 'result') { + state.result = event; + if (Number.isSafeInteger(event.num_turns) && event.num_turns >= 0) state.resultTurns += event.num_turns; + return; + } + if (event.type === 'assistant' && typeof event.parent_tool_use_id === 'string') { + const usage = event.message?.usage; + if (typeof event.message?.id === 'string' && usage) state.subagents.set(event.message.id, { input: count(usage.input_tokens), output: count(usage.output_tokens), cached: count(usage.cache_read_input_tokens) }); + return; + } if (event.type === 'assistant' && Array.isArray(event.message?.content)) { for (const block of event.message.content) if (block?.type === 'text' && typeof block.text === 'string' && block.text.trim()) state.lastText = block.text; return; @@ -55,23 +68,36 @@ export function consumeClaudeStreamLine(state: ClaudeStreamState, line: string): } } -/** Every message seen so far, including ones a kill interrupted mid-stream. */ +/** Every message seen so far, subagents' included, and ones a kill interrupted mid-stream. */ export function claudeStreamUsage(state: ClaudeStreamState): { input: number; output: number; cached: number } { const totals = { ...state.totals }; - for (const message of state.open.values()) { totals.input += message.input; totals.output += message.output; totals.cached += message.cached; } + for (const message of [...state.open.values(), ...state.subagents.values()]) { totals.input += message.input; totals.output += message.output; totals.cached += message.cached; } return totals; } +/** `modelUsage` is the session total across models, subagents included; `usage` covers only the main thread's last turn. */ +function resultTokens(result: Record): { input: number; output: number; cached: number } | undefined { + const models = result.modelUsage; + if (models && typeof models === 'object' && Object.keys(models).length) { + const totals = { input: 0, output: 0, cached: 0 }; + for (const usage of Object.values(models as Record>)) { + totals.input += count(usage?.inputTokens); totals.output += count(usage?.outputTokens); totals.cached += count(usage?.cacheReadInputTokens); + } + return totals; + } + const usage = result.usage as Record | undefined; + return usage ? { input: count(usage.input_tokens), output: count(usage.output_tokens), cached: count(usage.cache_read_input_tokens) } : undefined; +} + /** - * The final `result` event is authoritative for usage and turns; without it (a killed run) the per-message stream - * usage stands in. A paid producer still needs a successful result event with a nonblank `result`. + * The final `result` event is authoritative for usage (its `modelUsage` session totals) and its text; turns sum over + * every result. Without one (a killed run) the per-message stream usage stands in. A paid producer still needs a + * successful result event with a nonblank `result`. */ export function resultFromClaudeStream(state: ClaudeStreamState, opts: { model: string; durationMs: number; error?: RunResult['error'] }): RunResult { - const result = state.result, usage = (result?.usage ?? undefined) as Record | undefined; - const tokens = usage - ? { input: count(usage.input_tokens), output: count(usage.output_tokens), cached: count(usage.cache_read_input_tokens) } - : claudeStreamUsage(state); - const toolCalls = result && Number.isSafeInteger(result.num_turns) ? result.num_turns as number : state.turns; + const result = state.result; + const tokens = (result && resultTokens(result)) ?? claudeStreamUsage(state); + const toolCalls = result ? state.resultTurns : state.turns; const modelUsed = typeof result?.model === 'string' && result.model ? result.model : opts.model; const base = { tokens, durationMs: opts.durationMs, toolCalls, modelUsed }; if (opts.error) return { ...base, output: typeof result?.result === 'string' ? result.result : state.lastText, error: opts.error }; diff --git a/test/helpers/providers/claude.ts b/test/helpers/providers/claude.ts index 90256c7ad..945c82393 100644 --- a/test/helpers/providers/claude.ts +++ b/test/helpers/providers/claude.ts @@ -191,6 +191,9 @@ export function claudeProducerTools(opts: RunOpts): string { return [`Bash(${launcher})`, `Bash(${launcher} *)`, 'Write'].join(','); } +/** Built-in tools: users' /cso starts its independent reviewers with Agent; subagents inherit this whole policy. */ +export const CLAUDE_PRODUCER_BUILTIN_TOOLS = 'Bash,Write,Agent'; + /** Claude's sandbox wraps every Bash command except the trusted launcher, which must run where users run it. */ export function claudeProducerSettings(launcher: string): { sandbox: Record } { return { @@ -211,7 +214,12 @@ export function assertClaudeProducerSandboxPolicy(args: readonly string[], env: if (positions.length !== 1 || args[positions[0]] !== flag || positions[0] + 1 >= args.length) fail(`expected exactly one ${flag} `); return args[positions[0] + 1]; }; + if (value('--tools') !== CLAUDE_PRODUCER_BUILTIN_TOOLS) fail(`--tools must be exactly ${CLAUDE_PRODUCER_BUILTIN_TOOLS}`); if (value('--allowed-tools') !== `Bash(${launcher}),Bash(${launcher} *),Write`) fail('--allowed-tools must name only the launcher and Write'); + if (value('--permission-mode') !== 'dontAsk') fail('--permission-mode must be dontAsk'); + for (const flag of ['--restricted', '--safe-mode', '--strict-mcp-config']) if (args.filter(arg => arg === flag).length !== 1) fail(`expected exactly one ${flag}`); + const widening = args.find(arg => /^--(mcp-config|agents?|plugin-dir|dangerously-skip-permissions|allow-dangerously-skip-permissions|disallowed-tools|disallowedTools|setting-sources)(=|$)/.test(arg)); + if (widening) fail(`${widening.split('=')[0]} would change what the agent and its subagents may do`); const raw = value('--settings'); let settings: unknown; try { settings = JSON.parse(raw); } catch { fail('--settings is not JSON'); } @@ -240,7 +248,7 @@ export function claudeExecArgs(opts: RunOpts, model: string, argsPrefix: readonl '--no-session-persistence', '--permission-prompts', 'none', '--permission-mode', 'dontAsk', - '--tools', 'Bash,Write', + '--tools', CLAUDE_PRODUCER_BUILTIN_TOOLS, '--allowed-tools', tools, '--add-dir', claudeProducerPaths(stateDirectory).workdir, '--add-dir', helperHome, diff --git a/test/provider-model-defaults.test.ts b/test/provider-model-defaults.test.ts index 1af08191a..07ff031e1 100644 --- a/test/provider-model-defaults.test.ts +++ b/test/provider-model-defaults.test.ts @@ -76,7 +76,7 @@ describe('CSO producer provider policies', () => { '-p', '--output-format', 'stream-json', '--verbose', '--include-partial-messages', '--model', 'claude-test', '--restricted', '--safe-mode', '--no-session-persistence', '--permission-prompts', 'none', '--permission-mode', 'dontAsk', - '--tools', 'Bash,Write', + '--tools', 'Bash,Write,Agent', '--allowed-tools', `Bash(${helper}),Bash(${helper} *),Write`, '--add-dir', claudeProducerPaths(state).workdir, '--add-dir', join(state, 'cso-home'), @@ -89,7 +89,7 @@ describe('CSO producer provider policies', () => { expect(claudeExecArgs(producer, 'claude-test').join(' ')).not.toMatch(/bypassPermissions|danger/); }); - test('fails the Claude producer closed unless its sandbox exempts exactly the launcher', () => { + test('fails the Claude producer closed unless its sandbox exempts exactly the launcher and its tool policy is the one subagents inherit', () => { const work = join(tmpdir(), 'gstack-claude-sandbox-policy'); const state = join(work, 'state'); const helper = helperAt(join(work, 'installed')); @@ -114,6 +114,14 @@ describe('CSO producer provider policies', () => { [withSettings(sandbox => { sandbox.dangerouslyDisableSandbox = true; }), env, 'keys differ'], [args.map(arg => arg.startsWith('Bash(') ? `${arg},Bash(cat *)` : arg), env, '--allowed-tools'], [args, { ...env, CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: '1' }, 'CLAUDE_CODE_SUBPROCESS_ENV_SCRUB'], + [args.map(arg => arg === 'Bash,Write,Agent' ? 'Bash,Write' : arg), env, '--tools must be exactly Bash,Write,Agent'], + [args.map(arg => arg === 'Bash,Write,Agent' ? 'Bash,Write,Agent,Read' : arg), env, '--tools must be exactly Bash,Write,Agent'], + [args.map(arg => arg === 'dontAsk' ? 'acceptEdits' : arg), env, '--permission-mode must be dontAsk'], + [args.filter(arg => arg !== '--restricted'), env, 'expected exactly one --restricted'], + [args.filter(arg => arg !== '--strict-mcp-config'), env, 'expected exactly one --strict-mcp-config'], + [[...args, '--mcp-config', '{}'], env, '--mcp-config would change'], + [[...args, '--agents', '{}'], env, '--agents would change'], + [[...args, '--setting-sources=user'], env, '--setting-sources would change'], ] as Array<[string[], Record, string]>) { expect(() => assertClaudeProducerSandboxPolicy(tampered, tamperedEnv, helper)).toThrow(`CLAUDE_PRODUCER_SANDBOX_POLICY_MISMATCH: `); expect(() => assertClaudeProducerSandboxPolicy(tampered, tamperedEnv, helper)).toThrow(detail);