mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-05 08:23:59 +01:00
fix(cso): count remainingSeconds to the investigation deadline; bound and validate the tool timeline
remainingSeconds now measures to deadline - 60s, the point where scan, verify and record-review refuse with DEADLINE, and inspect carries it. The post-command clock lookup can only omit the field, never fail a command whose state change is already saved. The stream timeline records end times from tool_result, indexes only recorded calls so it stays at the 4096 cap, and receipts reject malformed timeline entries.
This commit is contained in:
1 parent
6a432f00a0
commit
d804fbb2fc
11 files changed
+90
-34
No files matched your search
@@ -67,6 +67,18 @@ describe('Claude producer stream accounting', () => {
|
||||
expect(JSON.stringify(timeline)).not.toContain('SECRET_PATH');
|
||||
});
|
||||
|
||||
test('stamps a tool call end once from its tool_result and keeps the timeline and its id index at the cap', () => {
|
||||
const state = claudeStreamState();
|
||||
const toolResult = (id: string) => ({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: id, content: 'SECRET_OUTPUT' }] } });
|
||||
const call = (id: string) => ({ type: 'assistant', message: { id: `m${id}`, content: [{ type: 'tool_use', id, name: 'Bash', input: {} }] } });
|
||||
for (const line of lines(call('t0'), toolResult('t0'), toolResult('t0'), toolResult('unknown'), call('t1'))) consumeClaudeStreamLine(state, line);
|
||||
expect(state.toolTimeline.map(entry => entry.endMs === undefined)).toEqual([false, true]);
|
||||
expect(JSON.stringify(state.toolTimeline)).not.toContain('SECRET_OUTPUT');
|
||||
for (let i = 2; i < 5000; i++) consumeClaudeStreamLine(state, JSON.stringify(call(`t${i}`)));
|
||||
expect(state.toolTimeline).toHaveLength(4096);
|
||||
expect(state.toolUseIndex.size).toBe(4096);
|
||||
});
|
||||
|
||||
test('a killed run counts each subagent message once and never reports subagent text as the producer output', () => {
|
||||
const state = claudeStreamState();
|
||||
for (const line of lines(start('a', 1000), text('a', 'main note'), delta(100), stop, subagent('s1', 50000), subagent('s1', 50000), subagent('s2', 50001))) consumeClaudeStreamLine(state, line);
|
||||
|
||||
+11
-1
@@ -69,7 +69,17 @@ describe('compiled CSO command workflow',()=>{
|
||||
runGit('init','-q');runGit('config','user.email','fixture@example.test');runGit('config','user.name','Fixture');fs.writeFileSync(path.join(planRepo,'package.json'),JSON.stringify({name:'sensitive-plan',version:'1.0.0',scripts:{start:'node app.js',test:'node --test'}}));fs.writeFileSync(path.join(planRepo,'package-lock.json'),JSON.stringify({name:'sensitive-plan',version:'1.0.0',lockfileVersion:3,packages:{'':{name:'sensitive-plan',version:'1.0.0'}}}));fs.writeFileSync(path.join(planRepo,'app.js'),'export const app = true\n');fs.writeFileSync(path.join(planRepo,testPath),'const test=require("node:test");test("control",()=>{})\n');runGit('add','.');runGit('commit','-qm','sensitive test path');
|
||||
const started=command(['start','--repo',planRepo,'--comprehensive','--offline']);expect(started.status).toBe(0);const run=JSON.parse(started.stdout),planned=command(['runtime-plan',run.runId,'node','--port','3456']);expect(planned.status).toBe(0);const plan=JSON.parse(planned.stdout),serialized=JSON.stringify(plan);
|
||||
expect(serialized).not.toContain(wallet);const testHandle=plan.tests.files.find((value:string)=>value.startsWith('@cso-path//'));expect(testHandle).toMatch(/^@cso-path\/\/[a-f0-9]{32}$/);expect(plan.tests.commands[0].args).toContain(`./${testHandle}`);expect(plan.start.command).toEqual({executable:'/usr/local/bin/node',args:['app.js']});
|
||||
const budget=(Date.parse(run.deadline)-Date.parse(run.createdAt))/1000;for(const remaining of [run.remainingSeconds,plan.remainingSeconds]){expect(Number.isSafeInteger(remaining)).toBe(true);expect(remaining).toBeGreaterThan(budget-120);expect(remaining).toBeLessThanOrEqual(budget);}expect(plan.remainingSeconds).toBeLessThanOrEqual(run.remainingSeconds);
|
||||
const budget=(Date.parse(run.deadline)-Date.parse(run.createdAt))/1000;for(const remaining of [run.remainingSeconds,plan.remainingSeconds]){expect(Number.isSafeInteger(remaining)).toBe(true);expect(remaining).toBeGreaterThan(budget-120);expect(remaining).toBeLessThanOrEqual(budget-60);}expect(plan.remainingSeconds).toBeLessThanOrEqual(run.remainingSeconds);
|
||||
});
|
||||
test('run clock counts to the investigation deadline: zero in the reporting minute, which still accepts submit until the audit deadline',()=>{
|
||||
const started=command(['start','--repo',repo,'--scope','auth','--offline']);expect(started.status).toBe(0);const run=JSON.parse(started.stdout),reportPath=path.join(state,'security','cso',run.repoId,run.runId,'report.json'),input=path.join(root,`clock-${run.runId}.json`);fs.writeFileSync(input,'{}');
|
||||
const submitted=command(['submit',run.runId,input]);expect(submitted.status).toBe(0);const inspected=command(['inspect',run.runId]);expect(inspected.status).toBe(0);
|
||||
for(const remaining of [JSON.parse(submitted.stdout).remainingSeconds,JSON.parse(inspected.stdout).remainingSeconds]){expect(Number.isSafeInteger(remaining)).toBe(true);expect(remaining).toBeGreaterThan(0);expect(remaining).toBeLessThanOrEqual(run.remainingSeconds);}
|
||||
const report=JSON.parse(fs.readFileSync(reportPath,'utf8'));report.deadline=new Date(Date.now()+30_000).toISOString();fs.writeFileSync(reportPath,JSON.stringify(report,null,2)+'\n');
|
||||
const late=command(['inspect',run.runId]);expect(late.status).toBe(0);expect(JSON.parse(late.stdout).remainingSeconds).toBe(0);
|
||||
const reporting=command(['submit',run.runId,input]);expect(reporting.status).toBe(0);expect(JSON.parse(reporting.stdout).remainingSeconds).toBe(0);
|
||||
report.deadline=new Date(Date.now()-1_000).toISOString();fs.writeFileSync(reportPath,JSON.stringify({...JSON.parse(fs.readFileSync(reportPath,'utf8')),deadline:report.deadline},null,2)+'\n');
|
||||
const refused=command(['submit',run.runId,input]);expect(refused.status).not.toBe(0);expect(refused.stderr).toContain('DEADLINE');expect(refused.stdout).toBe('');
|
||||
});
|
||||
test('wallet-shaped helper finding IDs survive submit, review, and verification admission',()=>{const started=command(['start','--repo',repo,'--comprehensive','--offline']);expect(started.status).toBe(0);const run=JSON.parse(started.stdout),base:any={title:'Wallet-shaped helper ID',rootCause:'Tenant query omits caller tenant predicate 0',location:{path:'src/users.ts',line:1,symbol:'tenantQuery'},advisoryIds:[],severity:'high',confidence:'high',confidenceRationale:'Caller-to-sink trace directly establishes the missing predicate',evidence:'supported',attackerControl:'Tenant chooses the record ID',impact:'Cross-tenant record disclosure',scenario:'A tenant requests another tenant record by ID',trace:['route','tenantQuery','database'],references:['src/users.ts:1'],recommendation:'Bind the query to the authenticated tenant',challenge:{reviewer:'reviewer-session',independent:true,mode:'independent_agent',callers:'route caller checked',controls:'authentication lacks authorization',counterevidence:'none found',conclusion:'tenant predicate is absent'}};let helperId='';for(let i=0;i<10_000;i++){base.rootCause=`Tenant query omits caller tenant predicate ${i}`;helperId=fingerprint(base);if(redact(helperId)!==helperId)break;}expect(redact(helperId)).not.toBe(helperId);const input=path.join(root,`wallet-id-${run.runId}.json`);fs.writeFileSync(input,JSON.stringify({findings:[base]}));expect(command(['submit',run.runId,input]).status).toBe(0);const reportPath=path.join(state,'security','cso',run.repoId,run.runId,'report.json'),stored=JSON.parse(fs.readFileSync(reportPath,'utf8'));expect(stored.findings[0].id).toBe(helperId);const before=fs.readFileSync(path.join(repo,'src','users.ts'),'utf8'),request:any={findingId:helperId,runtimeProfile:'node-v1',port:3456,start:{executable:'/usr/local/bin/node',args:['src/users.ts']},legitimate:[{name:'control',path:'/health',method:'GET',expected:{status:200}}],security:{name:'tenant boundary',path:'/users/other',method:'GET',expected:{status:403},vulnerable:{status:200}},existingTests:[{executable:'/usr/local/bin/node',args:['--test']}],testFiles:['src/control.test.ts'],fixtures:{},boundaryFiles:['src/users.ts'],changes:[{path:'src/users.ts',beforeSha256:sha256(before),after:before+'// tenant predicate\n',effect:'source'}],review:{reviewer:'reviewer-session',independent:true,rootCauseRepaired:true,featurePreserved:true,boundaryMocks:false,rationale:'The tenant predicate is added while the control remains intact',reviewedPatchHash:''}};request.review.reviewedPatchHash=patchHash(request);fs.writeFileSync(input,JSON.stringify(request));const reviewed=command(['record-review',run.runId,input,'--producer','producer-session']);expect(reviewed.status).toBe(0);request.review.artifactId=JSON.parse(reviewed.stdout).reviewArtifactId;fs.writeFileSync(input,JSON.stringify(request));const verified=command(['verify',run.runId,input]);expect(verified.status).not.toBe(0);expect(verified.stderr).toContain('PREREQUISITE');expect(verified.stderr).not.toContain('REDACTION_FAILED');const review=JSON.parse(fs.readFileSync(path.join(path.dirname(reportPath),'reviews',`${request.review.artifactId}.json`),'utf8'));expect(review.findingId).toBe(helperId);});
|
||||
test('daily find, early submit, and finish persist a truthful report without changing source',()=>{const before=git('status','--porcelain=v1','-z'),started=command(['start','--repo',repo,'--scope','auth','--offline']);expect(started.status).toBe(0);const run=JSON.parse(started.stdout),runId=run.runId;expect(run.completeness).toBe('not assessed');const reportPath=path.join(state,'security','cso',run.repoId,runId,'report.json');expect(fs.existsSync(reportPath)).toBe(true);const current=JSON.parse(fs.readFileSync(reportPath,'utf8'));
|
||||
|
||||
@@ -268,6 +268,11 @@ describe('CSO matched evaluation accounting', () => {
|
||||
artifactTamper.artifacts.totalBytes = 1;
|
||||
expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [rehashReceipt(artifactTamper)])).toThrow('INVALID_PRODUCER_RECEIPT');
|
||||
|
||||
const timed = rehashReceipt({ ...structuredClone(first), toolTimeline: [{ atMs: 5, endMs: 9, tool: 'Bash', agent: 'main' }, { atMs: 7, tool: 'Agent', agent: 'subagent' }] });
|
||||
expect(collectProducerReceipts(matrix, syntheticSchedule(), [timed]).receipts[0]).not.toHaveProperty('toolTimeline');
|
||||
for (const entry of [{ atMs: 9, endMs: 5, tool: 'Bash', agent: 'main' }, { atMs: 1, tool: 'cat /secret', agent: 'main' }, { atMs: 1, tool: 'Bash', agent: 'main', input: 'x' }])
|
||||
expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [rehashReceipt({ ...structuredClone(first), toolTimeline: [entry] } as ProducerReceipt)])).toThrow('INVALID_PRODUCER_RECEIPT');
|
||||
|
||||
const mixedInstallation = syntheticReceipt(cells[1]);
|
||||
mixedInstallation.installationIdentity = syntheticInstallationIdentity('different-installation');
|
||||
expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [first, rehashReceipt(mixedInstallation)])).toThrow('UNMATCHED_PRODUCER_INSTALLATIONS');
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { spawn } from 'child_process';
|
||||
import type { RunResult } from './types';
|
||||
import type { RunResult, ToolTimelineEntry } from './types';
|
||||
|
||||
/** Usage, turns and the latest text from `claude -p --output-format stream-json --verbose --include-partial-messages`. */
|
||||
export interface ClaudeStreamState {
|
||||
@@ -15,13 +15,14 @@ export interface ClaudeStreamState {
|
||||
result?: Record<string, unknown>;
|
||||
invalidLines: number;
|
||||
startedAt: number;
|
||||
/** Content-free: each tool call's name, start offset and thread, never its input or output. */
|
||||
toolTimeline: NonNullable<RunResult['toolTimeline']>;
|
||||
toolUseIds: Set<string>;
|
||||
/** Content-free: each tool call's name, start and end offsets and thread, never its input or output. */
|
||||
toolTimeline: ToolTimelineEntry[];
|
||||
/** Timeline index by tool-use id, for recorded calls only, so it stays within the timeline cap. */
|
||||
toolUseIndex: Map<string, number>;
|
||||
}
|
||||
|
||||
export function claudeStreamState(): ClaudeStreamState {
|
||||
return { totals: { input: 0, output: 0, cached: 0 }, open: new Map(), subagents: new Map(), turns: 0, resultTurns: 0, lastText: '', invalidLines: 0, startedAt: Date.now(), toolTimeline: [], toolUseIds: new Set() };
|
||||
return { totals: { input: 0, output: 0, cached: 0 }, open: new Map(), subagents: new Map(), turns: 0, resultTurns: 0, lastText: '', invalidLines: 0, startedAt: Date.now(), toolTimeline: [], toolUseIndex: new Map() };
|
||||
}
|
||||
|
||||
const TOOL_TIMELINE_LIMIT = 4096;
|
||||
@@ -29,11 +30,8 @@ const TOOL_TIMELINE_LIMIT = 4096;
|
||||
const count = (value: unknown): number => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : 0;
|
||||
|
||||
function recordToolUse(state: ClaudeStreamState, block: { id?: unknown; name?: unknown }, agent: 'main' | 'subagent'): void {
|
||||
if (typeof block.id === 'string') {
|
||||
if (state.toolUseIds.has(block.id)) return;
|
||||
state.toolUseIds.add(block.id);
|
||||
}
|
||||
if (state.toolTimeline.length >= TOOL_TIMELINE_LIMIT) return;
|
||||
if (typeof block.id !== 'string' || state.toolUseIndex.has(block.id) || state.toolTimeline.length >= TOOL_TIMELINE_LIMIT) return;
|
||||
state.toolUseIndex.set(block.id, state.toolTimeline.length);
|
||||
state.toolTimeline.push({
|
||||
atMs: Date.now() - state.startedAt,
|
||||
tool: typeof block.name === 'string' && /^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(block.name) ? block.name : 'other',
|
||||
@@ -41,6 +39,11 @@ function recordToolUse(state: ClaudeStreamState, block: { id?: unknown; name?: u
|
||||
});
|
||||
}
|
||||
|
||||
function recordToolResult(state: ClaudeStreamState, block: { tool_use_id?: unknown }): void {
|
||||
const index = typeof block.tool_use_id === 'string' ? state.toolUseIndex.get(block.tool_use_id) : undefined;
|
||||
if (index !== undefined && state.toolTimeline[index].endMs === undefined) state.toolTimeline[index].endMs = Date.now() - state.startedAt;
|
||||
}
|
||||
|
||||
function close(state: ClaudeStreamState, key: string): void {
|
||||
const message = state.open.get(key);
|
||||
if (!message) return;
|
||||
@@ -61,6 +64,10 @@ export function consumeClaudeStreamLine(state: ClaudeStreamState, line: string):
|
||||
if (event.type === 'assistant' && Array.isArray(event.message?.content)) {
|
||||
for (const block of event.message.content) if (block?.type === 'tool_use') recordToolUse(state, block, typeof event.parent_tool_use_id === 'string' ? 'subagent' : 'main');
|
||||
}
|
||||
if (event.type === 'user' && Array.isArray(event.message?.content)) {
|
||||
for (const block of event.message.content) if (block?.type === 'tool_result') recordToolResult(state, block);
|
||||
return;
|
||||
}
|
||||
if (event.type === 'assistant' && typeof event.parent_tool_use_id === 'string') {
|
||||
const usage = event.message?.usage;
|
||||
if (typeof event.message?.id === 'string' && usage) state.subagents.set(event.message.id, { input: count(usage.input_tokens), output: count(usage.output_tokens), cached: count(usage.cache_read_input_tokens) });
|
||||
|
||||
@@ -49,6 +49,14 @@ export type RunError =
|
||||
| 'binary_missing' // CLI not found on PATH.
|
||||
| 'unknown'; // Catch-all with reason populated.
|
||||
|
||||
export interface ToolTimelineEntry {
|
||||
atMs: number;
|
||||
/** Set when the call's tool_result arrived; absent for a call still running when the stream ended. */
|
||||
endMs?: number;
|
||||
tool: string;
|
||||
agent: 'main' | 'subagent';
|
||||
}
|
||||
|
||||
export interface RunResult {
|
||||
/** Provider's textual output for the prompt. */
|
||||
output: string;
|
||||
@@ -58,8 +66,8 @@ export interface RunResult {
|
||||
durationMs: number;
|
||||
/** Count of tool/function calls made during the run (0 if unsupported). */
|
||||
toolCalls: number;
|
||||
/** Tool-call starts in order: name, offset from run start, and thread; no inputs or outputs. Unset if unsupported. */
|
||||
toolTimeline?: Array<{ atMs: number; tool: string; agent: 'main' | 'subagent' }>;
|
||||
/** Tool calls in start order: name, start and end offsets from run start, and thread; no inputs or outputs. Unset if unsupported. */
|
||||
toolTimeline?: ToolTimelineEntry[];
|
||||
/** Actual model ID the provider reports using (may be a variant of the family). */
|
||||
modelUsed: string;
|
||||
/** If the run failed, error code + human reason. output/tokens may be partial. */
|
||||
|
||||
Reference in new issue
Block a user