From 6a432f00a093886b03ec62a67c3adf18c26f62fa Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 17 Oct 2026 05:10:18 +0000 Subject: [PATCH] feat(cso): helper replies carry remainingSeconds; producer receipts record a content-free tool timeline --- cso/SKILL.md | 2 +- cso/SKILL.md.tmpl | 2 +- lib/cso/cli.ts | 20 ++++++++++++++++++ scripts/cso-eval-producer.ts | 1 + scripts/cso-eval-protocol.ts | 4 +++- scripts/cso-eval.ts | 2 +- test/claude-producer-stream.test.ts | 16 ++++++++++++-- test/cso-cli.test.ts | 1 + test/helpers/providers/claude-stream.ts | 28 +++++++++++++++++++++++-- test/helpers/providers/types.ts | 2 ++ 10 files changed, 70 insertions(+), 8 deletions(-) diff --git a/cso/SKILL.md b/cso/SKILL.md index 46e19240b..478fc6e44 100644 --- a/cso/SKILL.md +++ b/cso/SKILL.md @@ -99,7 +99,7 @@ Start first; inspect snapshot, readiness, deadline, transformations, and coverag When updating coverage, copy every record's `domain` and `scope` exactly from `inspect`; a new scope leaves the planned scope unassessed. Only helper commands may update helper-owned records. -Budgets are ten minutes daily and thirty comprehensive, including one minute for reporting. Use at most three investigation workers and two reproduction groups per Docker endpoint. Attempts get five minutes and three harness repairs per finding, within the run deadline. Prioritize unresolved high impact. Report model usage only when exposed. +Budgets: ten minutes daily, thirty comprehensive. Use at most three investigation workers and two reproduction groups per Docker endpoint. Attempts get five minutes and three harness repairs per finding. Prioritize unresolved high impact. Report model usage only when exposed. Helper replies carry `remainingSeconds`: start only what fits; below 120, stop waiting and `finish`. --- ## Section index — Read each section when its situation applies diff --git a/cso/SKILL.md.tmpl b/cso/SKILL.md.tmpl index 9f936311a..09e5c67ac 100644 --- a/cso/SKILL.md.tmpl +++ b/cso/SKILL.md.tmpl @@ -99,7 +99,7 @@ Start first; inspect snapshot, readiness, deadline, transformations, and coverag When updating coverage, copy every record's `domain` and `scope` exactly from `inspect`; a new scope leaves the planned scope unassessed. Only helper commands may update helper-owned records. -Budgets are ten minutes daily and thirty comprehensive, including one minute for reporting. Use at most three investigation workers and two reproduction groups per Docker endpoint. Attempts get five minutes and three harness repairs per finding, within the run deadline. Prioritize unresolved high impact. Report model usage only when exposed. +Budgets: ten minutes daily, thirty comprehensive. Use at most three investigation workers and two reproduction groups per Docker endpoint. Attempts get five minutes and three harness repairs per finding. Prioritize unresolved high impact. Report model usage only when exposed. Helper replies carry `remainingSeconds`: start only what fits; below 120, stop waiting and `finish`. --- {{SECTION_INDEX:cso}} diff --git a/lib/cso/cli.ts b/lib/cso/cli.ts index 672272742..04dd00f58 100644 --- a/lib/cso/cli.ts +++ b/lib/cso/cli.ts @@ -2802,6 +2802,7 @@ export async function dispatchCsoCommand( dependencies: CsoCliDependencies, ): Promise { args = [...args]; + const runId = args[0]; if (!['start', 'recheck', 'replay', 'doctor', 'provision-images'].includes(command)) { const started = Date.now(); retention(started, { deadlineMs: started + RETENTION_MAINTENANCE_MS, maxEntries: RETENTION_MAX_ENTRIES }); @@ -2876,8 +2877,27 @@ export async function dispatchCsoCommand( default: throw new CsoError('INVALID_ARGUMENT', `Unknown command: ${command}`); } + if (RUN_CLOCK_COMMANDS.has(command) && result && typeof result === 'object' && !Array.isArray(result)) { + const deadline = (result as { deadline?: unknown }).deadline; + const ends = Date.parse( + typeof deadline === 'string' ? deadline : loadReport(runDirectory(runId)).deadline, + ); + result = { ...result, remainingSeconds: Math.max(0, Math.floor((ends - Date.now()) / 1000)) }; + } return result; } +const RUN_CLOCK_COMMANDS = new Set([ + 'start', + 'resume', + 'submit', + 'scan', + 'scanner-outcome', + 'import-sarif', + 'record-review', + 'test-plan', + 'runtime-plan', + 'verify', +]); const PRODUCTION_CLI_DEPENDENCIES: CsoCliDependencies = Object.freeze({ runtimeCatalog: RUNTIME_CATALOG, scannerCatalog: SCANNER_CATALOG, diff --git a/scripts/cso-eval-producer.ts b/scripts/cso-eval-producer.ts index 333e71268..a2995473b 100644 --- a/scripts/cso-eval-producer.ts +++ b/scripts/cso-eval-producer.ts @@ -686,6 +686,7 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op durationMs: run.durationMs, firstUsefulResultMs: null, toolCalls: run.toolCalls, + ...(run.toolTimeline ? { toolTimeline: run.toolTimeline } : {}), output, outputHash: sha256(output), usage: { diff --git a/scripts/cso-eval-protocol.ts b/scripts/cso-eval-protocol.ts index 88174ecff..fdeb35d2e 100644 --- a/scripts/cso-eval-protocol.ts +++ b/scripts/cso-eval-protocol.ts @@ -130,6 +130,8 @@ export interface ProducerReceipt { durationMs: number; firstUsefulResultMs: null; toolCalls: number; + /** Content-free tool timeline (name, offset, thread) for diagnosing overruns; absent when the adapter has none. */ + toolTimeline?: Array<{ atMs: number; tool: string; agent: 'main' | 'subagent' }>; output: string; outputHash: string; usage: { @@ -148,7 +150,7 @@ export interface ProducerReceipt { * Compact trusted index; raw output remains in the separately retained receipt. * Collection adds the matrix cell that the receipt's cellRef names. */ -export type ProducerReceiptIndex = Omit & { +export type ProducerReceiptIndex = Omit & { cell: ProducerCell; error?: { code: string }; }; diff --git a/scripts/cso-eval.ts b/scripts/cso-eval.ts index ad02a9d2f..9fb6665a0 100644 --- a/scripts/cso-eval.ts +++ b/scripts/cso-eval.ts @@ -530,7 +530,7 @@ export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEv estimatedCost: { measured: costs.length, denominator: expected.length, totalUSD: costs.length ? costs.reduce((sum, value) => sum + value, 0) : null, source: 'pricing-table-estimate' }, }); } - const indexes: ProducerReceiptIndex[] = bound.map(({ output: _output, error, ...receipt }) => ({ ...receipt, ...(error ? { error: { code: error.code } } : {}) })); + const indexes: ProducerReceiptIndex[] = bound.map(({ output: _output, toolTimeline: _toolTimeline, error, ...receipt }) => ({ ...receipt, ...(error ? { error: { code: error.code } } : {}) })); const base = { schemaVersion: 1 as const, matrixHash: matrixHash(matrix), scheduleHash: digest(JSON.stringify(schedule)), receipts: indexes.sort((left, right) => left.cell.id.localeCompare(right.cell.id)), summary: { scheduled: matrix.cells.length, prepared: schedule.preparedCells, submitted: bound.length, missing: matrix.cells.length - bound.length, matchedPairsExpected: expectedMatchedPairs(matrix), matchedPairsSubmitted: completePairs.length, modelMismatches, groups, diff --git a/test/claude-producer-stream.test.ts b/test/claude-producer-stream.test.ts index badfbe836..87e21c2f5 100644 --- a/test/claude-producer-stream.test.ts +++ b/test/claude-producer-stream.test.ts @@ -33,7 +33,7 @@ describe('Claude producer stream accounting', () => { test('takes usage and turns from the final result and keeps the paid result contract', () => { const state = claudeStreamState(); for (const line of lines(start('a', 1000), delta(100), stop, result({}))) consumeClaudeStreamLine(state, line); - expect(resultFromClaudeStream(state, { model: 'claude-test', durationMs: 5 })).toEqual({ output: 'REPORT', tokens: { input: 2001, output: 201, cached: 14 }, durationMs: 5, toolCalls: 2, modelUsed: 'claude-test' }); + expect(resultFromClaudeStream(state, { model: 'claude-test', durationMs: 5 })).toEqual({ output: 'REPORT', tokens: { input: 2001, output: 201, cached: 14 }, durationMs: 5, toolCalls: 2, toolTimeline: [], modelUsed: 'claude-test' }); for (const ending of [[], [result({ result: ' ' })], [result({ result: 7 })], [result({ is_error: true })], [result({ subtype: 'error_max_turns' })]]) { const partial = claudeStreamState(); for (const line of lines(start('a', 10), delta(3), stop, ...ending)) consumeClaudeStreamLine(partial, line); @@ -52,7 +52,19 @@ describe('Claude producer stream accounting', () => { for (const line of lines(start('a', 1000), delta(100), stop, subagent('s1', 50000), subagent('s1', 50000), subagent('s2', 50001), result({ result: 'MAIN ENDED TURN', modelUsage: sessionUsage }), start('b', 1002), delta(102), stop, result({ num_turns: 1, result: 'REPORT', usage: { input_tokens: 1002, output_tokens: 102, cache_read_input_tokens: 7 }, modelUsage: sessionUsage }))) consumeClaudeStreamLine(state, line); - expect(resultFromClaudeStream(state, { model: 'claude-test', durationMs: 5 })).toEqual({ output: 'REPORT', tokens: { input: 153009, output: 15308, cached: 49 }, durationMs: 5, toolCalls: 3, modelUsed: 'claude-test' }); + expect(resultFromClaudeStream(state, { model: 'claude-test', durationMs: 5 })).toEqual({ output: 'REPORT', tokens: { input: 153009, output: 15308, cached: 49 }, durationMs: 5, toolCalls: 3, toolTimeline: [], modelUsed: 'claude-test' }); + }); + + test('records a content-free tool timeline: main and subagent calls once each, never their inputs', () => { + const toolStart = (id: string, name: string) => ({ type: 'stream_event', parent_tool_use_id: null, event: { type: 'content_block_start', index: 1, content_block: { type: 'tool_use', id, name, input: {} } } }); + const mainCall = { type: 'assistant', message: { id: 'a', content: [{ type: 'tool_use', id: 'toolu_bash', name: 'Bash', input: { command: 'gstack-cso inspect SECRET_PATH' } }] } }; + const subCall = { type: 'assistant', parent_tool_use_id: 'toolu_agent', message: { id: 's1', content: [{ type: 'tool_use', id: 'toolu_sub', name: 'Bash', input: { command: 'cat SECRET_PATH' } }], usage: { input_tokens: 1, output_tokens: 1 } } }; + const state = claudeStreamState(); + for (const line of lines(start('a', 10), toolStart('toolu_bash', 'Bash'), toolStart('toolu_agent', 'Agent'), mainCall, subCall, subCall, toolStart('toolu_odd', 'mcp__x/../y'))) consumeClaudeStreamLine(state, line); + const timeline = resultFromClaudeStream(state, { model: 'claude-test', durationMs: 5, error: { code: 'timeout', reason: 'exceeded 5ms' } }).toolTimeline!; + expect(timeline.map(({ tool, agent }) => [tool, agent])).toEqual([['Bash', 'main'], ['Agent', 'main'], ['Bash', 'subagent'], ['other', 'main']]); + expect(timeline.every(entry => Number.isSafeInteger(entry.atMs) && entry.atMs >= 0 && Object.keys(entry).length === 3)).toBe(true); + expect(JSON.stringify(timeline)).not.toContain('SECRET_PATH'); }); test('a killed run counts each subagent message once and never reports subagent text as the producer output', () => { diff --git a/test/cso-cli.test.ts b/test/cso-cli.test.ts index 4a14016df..2892038d8 100644 --- a/test/cso-cli.test.ts +++ b/test/cso-cli.test.ts @@ -69,6 +69,7 @@ describe('compiled CSO command workflow',()=>{ runGit('init','-q');runGit('config','user.email','fixture@example.test');runGit('config','user.name','Fixture');fs.writeFileSync(path.join(planRepo,'package.json'),JSON.stringify({name:'sensitive-plan',version:'1.0.0',scripts:{start:'node app.js',test:'node --test'}}));fs.writeFileSync(path.join(planRepo,'package-lock.json'),JSON.stringify({name:'sensitive-plan',version:'1.0.0',lockfileVersion:3,packages:{'':{name:'sensitive-plan',version:'1.0.0'}}}));fs.writeFileSync(path.join(planRepo,'app.js'),'export const app = true\n');fs.writeFileSync(path.join(planRepo,testPath),'const test=require("node:test");test("control",()=>{})\n');runGit('add','.');runGit('commit','-qm','sensitive test path'); const started=command(['start','--repo',planRepo,'--comprehensive','--offline']);expect(started.status).toBe(0);const run=JSON.parse(started.stdout),planned=command(['runtime-plan',run.runId,'node','--port','3456']);expect(planned.status).toBe(0);const plan=JSON.parse(planned.stdout),serialized=JSON.stringify(plan); expect(serialized).not.toContain(wallet);const testHandle=plan.tests.files.find((value:string)=>value.startsWith('@cso-path//'));expect(testHandle).toMatch(/^@cso-path\/\/[a-f0-9]{32}$/);expect(plan.tests.commands[0].args).toContain(`./${testHandle}`);expect(plan.start.command).toEqual({executable:'/usr/local/bin/node',args:['app.js']}); + const budget=(Date.parse(run.deadline)-Date.parse(run.createdAt))/1000;for(const remaining of [run.remainingSeconds,plan.remainingSeconds]){expect(Number.isSafeInteger(remaining)).toBe(true);expect(remaining).toBeGreaterThan(budget-120);expect(remaining).toBeLessThanOrEqual(budget);}expect(plan.remainingSeconds).toBeLessThanOrEqual(run.remainingSeconds); }); test('wallet-shaped helper finding IDs survive submit, review, and verification admission',()=>{const started=command(['start','--repo',repo,'--comprehensive','--offline']);expect(started.status).toBe(0);const run=JSON.parse(started.stdout),base:any={title:'Wallet-shaped helper ID',rootCause:'Tenant query omits caller tenant predicate 0',location:{path:'src/users.ts',line:1,symbol:'tenantQuery'},advisoryIds:[],severity:'high',confidence:'high',confidenceRationale:'Caller-to-sink trace directly establishes the missing predicate',evidence:'supported',attackerControl:'Tenant chooses the record ID',impact:'Cross-tenant record disclosure',scenario:'A tenant requests another tenant record by ID',trace:['route','tenantQuery','database'],references:['src/users.ts:1'],recommendation:'Bind the query to the authenticated tenant',challenge:{reviewer:'reviewer-session',independent:true,mode:'independent_agent',callers:'route caller checked',controls:'authentication lacks authorization',counterevidence:'none found',conclusion:'tenant predicate is absent'}};let helperId='';for(let i=0;i<10_000;i++){base.rootCause=`Tenant query omits caller tenant predicate ${i}`;helperId=fingerprint(base);if(redact(helperId)!==helperId)break;}expect(redact(helperId)).not.toBe(helperId);const input=path.join(root,`wallet-id-${run.runId}.json`);fs.writeFileSync(input,JSON.stringify({findings:[base]}));expect(command(['submit',run.runId,input]).status).toBe(0);const reportPath=path.join(state,'security','cso',run.repoId,run.runId,'report.json'),stored=JSON.parse(fs.readFileSync(reportPath,'utf8'));expect(stored.findings[0].id).toBe(helperId);const before=fs.readFileSync(path.join(repo,'src','users.ts'),'utf8'),request:any={findingId:helperId,runtimeProfile:'node-v1',port:3456,start:{executable:'/usr/local/bin/node',args:['src/users.ts']},legitimate:[{name:'control',path:'/health',method:'GET',expected:{status:200}}],security:{name:'tenant boundary',path:'/users/other',method:'GET',expected:{status:403},vulnerable:{status:200}},existingTests:[{executable:'/usr/local/bin/node',args:['--test']}],testFiles:['src/control.test.ts'],fixtures:{},boundaryFiles:['src/users.ts'],changes:[{path:'src/users.ts',beforeSha256:sha256(before),after:before+'// tenant predicate\n',effect:'source'}],review:{reviewer:'reviewer-session',independent:true,rootCauseRepaired:true,featurePreserved:true,boundaryMocks:false,rationale:'The tenant predicate is added while the control remains intact',reviewedPatchHash:''}};request.review.reviewedPatchHash=patchHash(request);fs.writeFileSync(input,JSON.stringify(request));const reviewed=command(['record-review',run.runId,input,'--producer','producer-session']);expect(reviewed.status).toBe(0);request.review.artifactId=JSON.parse(reviewed.stdout).reviewArtifactId;fs.writeFileSync(input,JSON.stringify(request));const verified=command(['verify',run.runId,input]);expect(verified.status).not.toBe(0);expect(verified.stderr).toContain('PREREQUISITE');expect(verified.stderr).not.toContain('REDACTION_FAILED');const review=JSON.parse(fs.readFileSync(path.join(path.dirname(reportPath),'reviews',`${request.review.artifactId}.json`),'utf8'));expect(review.findingId).toBe(helperId);}); test('daily find, early submit, and finish persist a truthful report without changing source',()=>{const before=git('status','--porcelain=v1','-z'),started=command(['start','--repo',repo,'--scope','auth','--offline']);expect(started.status).toBe(0);const run=JSON.parse(started.stdout),runId=run.runId;expect(run.completeness).toBe('not assessed');const reportPath=path.join(state,'security','cso',run.repoId,runId,'report.json');expect(fs.existsSync(reportPath)).toBe(true);const current=JSON.parse(fs.readFileSync(reportPath,'utf8')); diff --git a/test/helpers/providers/claude-stream.ts b/test/helpers/providers/claude-stream.ts index 18374b592..de08d558e 100644 --- a/test/helpers/providers/claude-stream.ts +++ b/test/helpers/providers/claude-stream.ts @@ -14,14 +14,33 @@ export interface ClaudeStreamState { lastText: string; result?: Record; invalidLines: number; + startedAt: number; + /** Content-free: each tool call's name, start offset and thread, never its input or output. */ + toolTimeline: NonNullable; + toolUseIds: Set; } export function claudeStreamState(): ClaudeStreamState { - return { totals: { input: 0, output: 0, cached: 0 }, open: new Map(), subagents: new Map(), turns: 0, resultTurns: 0, lastText: '', invalidLines: 0 }; + return { totals: { input: 0, output: 0, cached: 0 }, open: new Map(), subagents: new Map(), turns: 0, resultTurns: 0, lastText: '', invalidLines: 0, startedAt: Date.now(), toolTimeline: [], toolUseIds: new Set() }; } +const TOOL_TIMELINE_LIMIT = 4096; + const count = (value: unknown): number => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : 0; +function recordToolUse(state: ClaudeStreamState, block: { id?: unknown; name?: unknown }, agent: 'main' | 'subagent'): void { + if (typeof block.id === 'string') { + if (state.toolUseIds.has(block.id)) return; + state.toolUseIds.add(block.id); + } + if (state.toolTimeline.length >= TOOL_TIMELINE_LIMIT) return; + state.toolTimeline.push({ + atMs: Date.now() - state.startedAt, + tool: typeof block.name === 'string' && /^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(block.name) ? block.name : 'other', + agent, + }); +} + function close(state: ClaudeStreamState, key: string): void { const message = state.open.get(key); if (!message) return; @@ -39,6 +58,9 @@ export function consumeClaudeStreamLine(state: ClaudeStreamState, line: string): if (Number.isSafeInteger(event.num_turns) && event.num_turns >= 0) state.resultTurns += event.num_turns; return; } + if (event.type === 'assistant' && Array.isArray(event.message?.content)) { + for (const block of event.message.content) if (block?.type === 'tool_use') recordToolUse(state, block, typeof event.parent_tool_use_id === 'string' ? 'subagent' : 'main'); + } if (event.type === 'assistant' && typeof event.parent_tool_use_id === 'string') { const usage = event.message?.usage; if (typeof event.message?.id === 'string' && usage) state.subagents.set(event.message.id, { input: count(usage.input_tokens), output: count(usage.output_tokens), cached: count(usage.cache_read_input_tokens) }); @@ -56,6 +78,8 @@ export function consumeClaudeStreamLine(state: ClaudeStreamState, line: string): const usage = stream.message?.usage ?? {}; state.open.set(key, { input: count(usage.input_tokens), output: count(usage.output_tokens), cached: count(usage.cache_read_input_tokens) }); if (key === 'main') state.turns++; + } else if (stream.type === 'content_block_start' && stream.content_block?.type === 'tool_use') { + recordToolUse(state, stream.content_block, key === 'main' ? 'main' : 'subagent'); } else if (stream.type === 'message_delta' && stream.usage) { const message = state.open.get(key); if (!message) return; @@ -99,7 +123,7 @@ export function resultFromClaudeStream(state: ClaudeStreamState, opts: { model: const tokens = (result && resultTokens(result)) ?? claudeStreamUsage(state); const toolCalls = result ? state.resultTurns : state.turns; const modelUsed = typeof result?.model === 'string' && result.model ? result.model : opts.model; - const base = { tokens, durationMs: opts.durationMs, toolCalls, modelUsed }; + const base = { tokens, durationMs: opts.durationMs, toolCalls, toolTimeline: state.toolTimeline, modelUsed }; if (opts.error) return { ...base, output: typeof result?.result === 'string' ? result.result : state.lastText, error: opts.error }; const explicitError = result?.is_error === true || (typeof result?.subtype === 'string' && result.subtype !== 'success'); if (!result || explicitError || typeof result.result !== 'string' || !result.result.trim()) { diff --git a/test/helpers/providers/types.ts b/test/helpers/providers/types.ts index 59063e7b2..0a9247508 100644 --- a/test/helpers/providers/types.ts +++ b/test/helpers/providers/types.ts @@ -58,6 +58,8 @@ export interface RunResult { durationMs: number; /** Count of tool/function calls made during the run (0 if unsupported). */ toolCalls: number; + /** Tool-call starts in order: name, offset from run start, and thread; no inputs or outputs. Unset if unsupported. */ + toolTimeline?: Array<{ atMs: number; tool: string; agent: 'main' | 'subagent' }>; /** Actual model ID the provider reports using (may be a variant of the family). */ modelUsed: string; /** If the run failed, error code + human reason. output/tokens may be partial. */