fix(cso): count remainingSeconds to the investigation deadline; bound and validate the tool timeline

remainingSeconds now measures to deadline - 60s, the point where scan,
verify and record-review refuse with DEADLINE, and inspect carries it.
The post-command clock lookup can only omit the field, never fail a
command whose state change is already saved. The stream timeline records
end times from tool_result, indexes only recorded calls so it stays at
the 4096 cap, and receipts reject malformed timeline entries.
This commit is contained in:
garrytan committed 2026-10-17 05:37:17 +00:00
1 parent 6a432f00a0
commit d804fbb2fc
11 files changed
+90 -34

No files matched your search

+12
View File
@@ -67,6 +67,18 @@ describe('Claude producer stream accounting', () => {
expect(JSON.stringify(timeline)).not.toContain('SECRET_PATH');
});
test('stamps a tool call end once from its tool_result and keeps the timeline and its id index at the cap', () => {
const state = claudeStreamState();
const toolResult = (id: string) => ({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: id, content: 'SECRET_OUTPUT' }] } });
const call = (id: string) => ({ type: 'assistant', message: { id: `m${id}`, content: [{ type: 'tool_use', id, name: 'Bash', input: {} }] } });
for (const line of lines(call('t0'), toolResult('t0'), toolResult('t0'), toolResult('unknown'), call('t1'))) consumeClaudeStreamLine(state, line);
expect(state.toolTimeline.map(entry => entry.endMs === undefined)).toEqual([false, true]);
expect(JSON.stringify(state.toolTimeline)).not.toContain('SECRET_OUTPUT');
for (let i = 2; i < 5000; i++) consumeClaudeStreamLine(state, JSON.stringify(call(`t${i}`)));
expect(state.toolTimeline).toHaveLength(4096);
expect(state.toolUseIndex.size).toBe(4096);
});
test('a killed run counts each subagent message once and never reports subagent text as the producer output', () => {
const state = claudeStreamState();
for (const line of lines(start('a', 1000), text('a', 'main note'), delta(100), stop, subagent('s1', 50000), subagent('s1', 50000), subagent('s2', 50001))) consumeClaudeStreamLine(state, line);
+11 -1
View File
@@ -69,7 +69,17 @@ describe('compiled CSO command workflow',()=>{
runGit('init','-q');runGit('config','user.email','fixture@example.test');runGit('config','user.name','Fixture');fs.writeFileSync(path.join(planRepo,'package.json'),JSON.stringify({name:'sensitive-plan',version:'1.0.0',scripts:{start:'node app.js',test:'node --test'}}));fs.writeFileSync(path.join(planRepo,'package-lock.json'),JSON.stringify({name:'sensitive-plan',version:'1.0.0',lockfileVersion:3,packages:{'':{name:'sensitive-plan',version:'1.0.0'}}}));fs.writeFileSync(path.join(planRepo,'app.js'),'export const app = true\n');fs.writeFileSync(path.join(planRepo,testPath),'const test=require("node:test");test("control",()=>{})\n');runGit('add','.');runGit('commit','-qm','sensitive test path');
const started=command(['start','--repo',planRepo,'--comprehensive','--offline']);expect(started.status).toBe(0);const run=JSON.parse(started.stdout),planned=command(['runtime-plan',run.runId,'node','--port','3456']);expect(planned.status).toBe(0);const plan=JSON.parse(planned.stdout),serialized=JSON.stringify(plan);
expect(serialized).not.toContain(wallet);const testHandle=plan.tests.files.find((value:string)=>value.startsWith('@cso-path//'));expect(testHandle).toMatch(/^@cso-path\/\/[a-f0-9]{32}$/);expect(plan.tests.commands[0].args).toContain(`./${testHandle}`);expect(plan.start.command).toEqual({executable:'/usr/local/bin/node',args:['app.js']});
const budget=(Date.parse(run.deadline)-Date.parse(run.createdAt))/1000;for(const remaining of [run.remainingSeconds,plan.remainingSeconds]){expect(Number.isSafeInteger(remaining)).toBe(true);expect(remaining).toBeGreaterThan(budget-120);expect(remaining).toBeLessThanOrEqual(budget);}expect(plan.remainingSeconds).toBeLessThanOrEqual(run.remainingSeconds);
const budget=(Date.parse(run.deadline)-Date.parse(run.createdAt))/1000;for(const remaining of [run.remainingSeconds,plan.remainingSeconds]){expect(Number.isSafeInteger(remaining)).toBe(true);expect(remaining).toBeGreaterThan(budget-120);expect(remaining).toBeLessThanOrEqual(budget-60);}expect(plan.remainingSeconds).toBeLessThanOrEqual(run.remainingSeconds);
});
test('run clock counts to the investigation deadline: zero in the reporting minute, which still accepts submit until the audit deadline',()=>{
const started=command(['start','--repo',repo,'--scope','auth','--offline']);expect(started.status).toBe(0);const run=JSON.parse(started.stdout),reportPath=path.join(state,'security','cso',run.repoId,run.runId,'report.json'),input=path.join(root,`clock-${run.runId}.json`);fs.writeFileSync(input,'{}');
const submitted=command(['submit',run.runId,input]);expect(submitted.status).toBe(0);const inspected=command(['inspect',run.runId]);expect(inspected.status).toBe(0);
for(const remaining of [JSON.parse(submitted.stdout).remainingSeconds,JSON.parse(inspected.stdout).remainingSeconds]){expect(Number.isSafeInteger(remaining)).toBe(true);expect(remaining).toBeGreaterThan(0);expect(remaining).toBeLessThanOrEqual(run.remainingSeconds);}
const report=JSON.parse(fs.readFileSync(reportPath,'utf8'));report.deadline=new Date(Date.now()+30_000).toISOString();fs.writeFileSync(reportPath,JSON.stringify(report,null,2)+'\n');
const late=command(['inspect',run.runId]);expect(late.status).toBe(0);expect(JSON.parse(late.stdout).remainingSeconds).toBe(0);
const reporting=command(['submit',run.runId,input]);expect(reporting.status).toBe(0);expect(JSON.parse(reporting.stdout).remainingSeconds).toBe(0);
report.deadline=new Date(Date.now()-1_000).toISOString();fs.writeFileSync(reportPath,JSON.stringify({...JSON.parse(fs.readFileSync(reportPath,'utf8')),deadline:report.deadline},null,2)+'\n');
const refused=command(['submit',run.runId,input]);expect(refused.status).not.toBe(0);expect(refused.stderr).toContain('DEADLINE');expect(refused.stdout).toBe('');
});
test('wallet-shaped helper finding IDs survive submit, review, and verification admission',()=>{const started=command(['start','--repo',repo,'--comprehensive','--offline']);expect(started.status).toBe(0);const run=JSON.parse(started.stdout),base:any={title:'Wallet-shaped helper ID',rootCause:'Tenant query omits caller tenant predicate 0',location:{path:'src/users.ts',line:1,symbol:'tenantQuery'},advisoryIds:[],severity:'high',confidence:'high',confidenceRationale:'Caller-to-sink trace directly establishes the missing predicate',evidence:'supported',attackerControl:'Tenant chooses the record ID',impact:'Cross-tenant record disclosure',scenario:'A tenant requests another tenant record by ID',trace:['route','tenantQuery','database'],references:['src/users.ts:1'],recommendation:'Bind the query to the authenticated tenant',challenge:{reviewer:'reviewer-session',independent:true,mode:'independent_agent',callers:'route caller checked',controls:'authentication lacks authorization',counterevidence:'none found',conclusion:'tenant predicate is absent'}};let helperId='';for(let i=0;i<10_000;i++){base.rootCause=`Tenant query omits caller tenant predicate ${i}`;helperId=fingerprint(base);if(redact(helperId)!==helperId)break;}expect(redact(helperId)).not.toBe(helperId);const input=path.join(root,`wallet-id-${run.runId}.json`);fs.writeFileSync(input,JSON.stringify({findings:[base]}));expect(command(['submit',run.runId,input]).status).toBe(0);const reportPath=path.join(state,'security','cso',run.repoId,run.runId,'report.json'),stored=JSON.parse(fs.readFileSync(reportPath,'utf8'));expect(stored.findings[0].id).toBe(helperId);const before=fs.readFileSync(path.join(repo,'src','users.ts'),'utf8'),request:any={findingId:helperId,runtimeProfile:'node-v1',port:3456,start:{executable:'/usr/local/bin/node',args:['src/users.ts']},legitimate:[{name:'control',path:'/health',method:'GET',expected:{status:200}}],security:{name:'tenant boundary',path:'/users/other',method:'GET',expected:{status:403},vulnerable:{status:200}},existingTests:[{executable:'/usr/local/bin/node',args:['--test']}],testFiles:['src/control.test.ts'],fixtures:{},boundaryFiles:['src/users.ts'],changes:[{path:'src/users.ts',beforeSha256:sha256(before),after:before+'// tenant predicate\n',effect:'source'}],review:{reviewer:'reviewer-session',independent:true,rootCauseRepaired:true,featurePreserved:true,boundaryMocks:false,rationale:'The tenant predicate is added while the control remains intact',reviewedPatchHash:''}};request.review.reviewedPatchHash=patchHash(request);fs.writeFileSync(input,JSON.stringify(request));const reviewed=command(['record-review',run.runId,input,'--producer','producer-session']);expect(reviewed.status).toBe(0);request.review.artifactId=JSON.parse(reviewed.stdout).reviewArtifactId;fs.writeFileSync(input,JSON.stringify(request));const verified=command(['verify',run.runId,input]);expect(verified.status).not.toBe(0);expect(verified.stderr).toContain('PREREQUISITE');expect(verified.stderr).not.toContain('REDACTION_FAILED');const review=JSON.parse(fs.readFileSync(path.join(path.dirname(reportPath),'reviews',`${request.review.artifactId}.json`),'utf8'));expect(review.findingId).toBe(helperId);});
test('daily find, early submit, and finish persist a truthful report without changing source',()=>{const before=git('status','--porcelain=v1','-z'),started=command(['start','--repo',repo,'--scope','auth','--offline']);expect(started.status).toBe(0);const run=JSON.parse(started.stdout),runId=run.runId;expect(run.completeness).toBe('not assessed');const reportPath=path.join(state,'security','cso',run.repoId,runId,'report.json');expect(fs.existsSync(reportPath)).toBe(true);const current=JSON.parse(fs.readFileSync(reportPath,'utf8'));
+5
View File
@@ -268,6 +268,11 @@ describe('CSO matched evaluation accounting', () => {
artifactTamper.artifacts.totalBytes = 1;
expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [rehashReceipt(artifactTamper)])).toThrow('INVALID_PRODUCER_RECEIPT');
const timed = rehashReceipt({ ...structuredClone(first), toolTimeline: [{ atMs: 5, endMs: 9, tool: 'Bash', agent: 'main' }, { atMs: 7, tool: 'Agent', agent: 'subagent' }] });
expect(collectProducerReceipts(matrix, syntheticSchedule(), [timed]).receipts[0]).not.toHaveProperty('toolTimeline');
for (const entry of [{ atMs: 9, endMs: 5, tool: 'Bash', agent: 'main' }, { atMs: 1, tool: 'cat /secret', agent: 'main' }, { atMs: 1, tool: 'Bash', agent: 'main', input: 'x' }])
expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [rehashReceipt({ ...structuredClone(first), toolTimeline: [entry] } as ProducerReceipt)])).toThrow('INVALID_PRODUCER_RECEIPT');
const mixedInstallation = syntheticReceipt(cells[1]);
mixedInstallation.installationIdentity = syntheticInstallationIdentity('different-installation');
expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [first, rehashReceipt(mixedInstallation)])).toThrow('UNMATCHED_PRODUCER_INSTALLATIONS');
+17 -10
View File
@@ -1,5 +1,5 @@
import { spawn } from 'child_process';
import type { RunResult } from './types';
import type { RunResult, ToolTimelineEntry } from './types';
/** Usage, turns and the latest text from `claude -p --output-format stream-json --verbose --include-partial-messages`. */
export interface ClaudeStreamState {
@@ -15,13 +15,14 @@ export interface ClaudeStreamState {
result?: Record<string, unknown>;
invalidLines: number;
startedAt: number;
/** Content-free: each tool call's name, start offset and thread, never its input or output. */
toolTimeline: NonNullable<RunResult['toolTimeline']>;
toolUseIds: Set<string>;
/** Content-free: each tool call's name, start and end offsets and thread, never its input or output. */
toolTimeline: ToolTimelineEntry[];
/** Timeline index by tool-use id, for recorded calls only, so it stays within the timeline cap. */
toolUseIndex: Map<string, number>;
}
export function claudeStreamState(): ClaudeStreamState {
return { totals: { input: 0, output: 0, cached: 0 }, open: new Map(), subagents: new Map(), turns: 0, resultTurns: 0, lastText: '', invalidLines: 0, startedAt: Date.now(), toolTimeline: [], toolUseIds: new Set() };
return { totals: { input: 0, output: 0, cached: 0 }, open: new Map(), subagents: new Map(), turns: 0, resultTurns: 0, lastText: '', invalidLines: 0, startedAt: Date.now(), toolTimeline: [], toolUseIndex: new Map() };
}
const TOOL_TIMELINE_LIMIT = 4096;
@@ -29,11 +30,8 @@ const TOOL_TIMELINE_LIMIT = 4096;
const count = (value: unknown): number => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : 0;
function recordToolUse(state: ClaudeStreamState, block: { id?: unknown; name?: unknown }, agent: 'main' | 'subagent'): void {
if (typeof block.id === 'string') {
if (state.toolUseIds.has(block.id)) return;
state.toolUseIds.add(block.id);
}
if (state.toolTimeline.length >= TOOL_TIMELINE_LIMIT) return;
if (typeof block.id !== 'string' || state.toolUseIndex.has(block.id) || state.toolTimeline.length >= TOOL_TIMELINE_LIMIT) return;
state.toolUseIndex.set(block.id, state.toolTimeline.length);
state.toolTimeline.push({
atMs: Date.now() - state.startedAt,
tool: typeof block.name === 'string' && /^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(block.name) ? block.name : 'other',
@@ -41,6 +39,11 @@ function recordToolUse(state: ClaudeStreamState, block: { id?: unknown; name?: u
});
}
function recordToolResult(state: ClaudeStreamState, block: { tool_use_id?: unknown }): void {
const index = typeof block.tool_use_id === 'string' ? state.toolUseIndex.get(block.tool_use_id) : undefined;
if (index !== undefined && state.toolTimeline[index].endMs === undefined) state.toolTimeline[index].endMs = Date.now() - state.startedAt;
}
function close(state: ClaudeStreamState, key: string): void {
const message = state.open.get(key);
if (!message) return;
@@ -61,6 +64,10 @@ export function consumeClaudeStreamLine(state: ClaudeStreamState, line: string):
if (event.type === 'assistant' && Array.isArray(event.message?.content)) {
for (const block of event.message.content) if (block?.type === 'tool_use') recordToolUse(state, block, typeof event.parent_tool_use_id === 'string' ? 'subagent' : 'main');
}
if (event.type === 'user' && Array.isArray(event.message?.content)) {
for (const block of event.message.content) if (block?.type === 'tool_result') recordToolResult(state, block);
return;
}
if (event.type === 'assistant' && typeof event.parent_tool_use_id === 'string') {
const usage = event.message?.usage;
if (typeof event.message?.id === 'string' && usage) state.subagents.set(event.message.id, { input: count(usage.input_tokens), output: count(usage.output_tokens), cached: count(usage.cache_read_input_tokens) });
+10 -2
View File
@@ -49,6 +49,14 @@ export type RunError =
| 'binary_missing' // CLI not found on PATH.
| 'unknown'; // Catch-all with reason populated.
export interface ToolTimelineEntry {
atMs: number;
/** Set when the call's tool_result arrived; absent for a call still running when the stream ended. */
endMs?: number;
tool: string;
agent: 'main' | 'subagent';
}
export interface RunResult {
/** Provider's textual output for the prompt. */
output: string;
@@ -58,8 +66,8 @@ export interface RunResult {
durationMs: number;
/** Count of tool/function calls made during the run (0 if unsupported). */
toolCalls: number;
/** Tool-call starts in order: name, offset from run start, and thread; no inputs or outputs. Unset if unsupported. */
toolTimeline?: Array<{ atMs: number; tool: string; agent: 'main' | 'subagent' }>;
/** Tool calls in start order: name, start and end offsets from run start, and thread; no inputs or outputs. Unset if unsupported. */
toolTimeline?: ToolTimelineEntry[];
/** Actual model ID the provider reports using (may be a variant of the family). */
modelUsed: string;
/** If the run failed, error code + human reason. output/tokens may be partial. */