From 56f25ef82bb7cb5cb2b0efaeec4e8843eb2a3594 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 17 Oct 2026 10:03:41 -0700 Subject: [PATCH 1/3] v1.91.123.0 fix(cso): comprehensive verify can patch read-only snapshots and accepts runtime-plan's profile (#3240) * fix(cso): comprehensive verify can patch read-only snapshots and accepts runtime-plan's profile * docs(changelog): unreleased entry for the /cso verify repair path * v1.91.123.0 stamp --- CHANGELOG.md | 11 +++++++++++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- lib/cso/cli.ts | 15 ++++++++++++--- lib/cso/verification.ts | 2 ++ package.json | 2 +- test/cso-contracts.test.ts | 1 + test/cso-node-lifecycle-integration.test.ts | 3 ++- 8 files changed, 31 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dfa0a6e9..679805e64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## [1.91.123.0] - 2026-10-17 + +**/cso comprehensive repairs run again.** In the first paid runtime qualification, the agent recorded 148 repair reviews across 42 comprehensive runs, but not one verification attempt started. Every runtime scored 0/24 on held-out repairs. Two helper bugs stacked: the patch write failed on the snapshot's read-only files, and `runtime-plan` handed the agent a profile name that `verify` refused. + +### Fixed +- **`verify` can patch read-only snapshot files.** Snapshot copies keep their 0444 modes, so writing the patched file failed with EACCES for any non-root user, which is how real users and the evaluator run. The error surfaced as a generic "unexpected or unsafe input". The helper now makes the file owner-writable for the write and restores its mode afterwards. +- **`runtime-plan` returns the qualified runtime ID** (for example `node-24.4.0-amd64`) instead of the stack alias, and the skill copies that value into the verification request. `verify` rejects any other profile with an `INVALID_ARGUMENT` that names the expected ID, before an attempt is counted. + +### If you… +- **…build a verification request by hand:** use the `runtimeProfile` that `runtime-plan` printed. The stack alias is refused. + ## [1.91.122.0] - 2026-10-17 **Main no longer goes red on 2026-11-16.** All seven `.osv-scanner.toml` suppressions expired on 2026-11-30, and `test/osv-ignore-expiry.test.ts` fails 14 days early. Six are gone and the seventh is re-dated with an upstream link. Dependabot's grouped dev bump (#3228) is replaced by hand, and Version Gate stops failing every dependabot package.json PR. diff --git a/VERSION b/VERSION index e0b7c3105..48617770e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.122.0 +1.91.123.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 47072b96d..957e4470a 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.122.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.123.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/lib/cso/cli.ts b/lib/cso/cli.ts index 6061afbd0..672272742 100644 --- a/lib/cso/cli.ts +++ b/lib/cso/cli.ts @@ -2056,7 +2056,7 @@ function testPlan(args: string[]) { ...publicTestPlan(manifest, canonicalTestPlan(join(dir, 'snapshot'), stack)), }; } -function runtimePlan(args: string[]) { +function runtimePlan(args: string[], dependencies: CsoCliDependencies) { const { dir } = run(args); if (!args.length || !['node', 'bun', 'python', 'rails'].includes(args[0])) throw new CsoError('INVALID_ARGUMENT', 'runtime-plan requires one supported stack and --port PORT'); @@ -2075,9 +2075,13 @@ function runtimePlan(args: string[]) { preparation.prerequisites.map((item) => item.message).join('; ') || `${stack} preparation metadata is incomplete`, ); + let runtimeProfile = preparation.runtimeProfile; + try { + runtimeProfile = selectRuntime(runtimeProfile, platform(), dependencies.runtimeCatalog).id; + } catch {} return { stack, - runtimeProfile: preparation.runtimeProfile, + runtimeProfile, start: publicStartPlan(manifest, canonicalStartPlan(join(dir, 'snapshot'), stack, port)), tests: publicTestPlan(manifest, canonicalTestPlan(join(dir, 'snapshot'), stack)), }; @@ -2339,6 +2343,11 @@ async function verify(args: string[], dependencies: CsoCliDependencies) { } catch (error: any) { throw new CsoError('PREREQUISITE', error?.message || 'Qualified runtime is unavailable'); } + if (request.runtimeProfile !== runtime.id) + throw new CsoError( + 'INVALID_ARGUMENT', + `runtimeProfile must be the qualified runtime ID ${runtime.id} returned by runtime-plan`, + ); const verifier = runtime; if (!['node', 'bun', 'python', 'rails'].includes(runtime.stack)) throw new CsoError( @@ -2848,7 +2857,7 @@ export async function dispatchCsoCommand( result = testPlan(args); break; case 'runtime-plan': - result = runtimePlan(args); + result = runtimePlan(args, dependencies); break; case 'recheck': result = await recheck(args, dependencies); diff --git a/lib/cso/verification.ts b/lib/cso/verification.ts index 5b0eda27b..6ccf9ef83 100644 --- a/lib/cso/verification.ts +++ b/lib/cso/verification.ts @@ -1035,7 +1035,9 @@ export function preparePatchedSource(snapshot: string, target: string, request: ); const mode = exists ? fs.statSync(file).mode & 0o777 : 0o600; secureDirectory(dirname(file)); + if (exists) fs.chmodSync(file, mode | 0o200); fs.writeFileSync(file, change.after, { mode }); + fs.chmodSync(file, mode); } } export function certify(params: { diff --git a/package.json b/package.json index 41bab4c69..a130d5382 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.122", + "version": "1.91.123", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/test/cso-contracts.test.ts b/test/cso-contracts.test.ts index e9513eddd..a3434a54f 100644 --- a/test/cso-contracts.test.ts +++ b/test/cso-contracts.test.ts @@ -189,6 +189,7 @@ describe('tested repair certificate gate',()=>{ test('review is a separate content-bound but explicitly self-attested artifact',()=>{const raw:any=request('vulnerable\n','fixed\n');raw.review.reviewedPatchHash=patchHash(raw);const req=validateVerificationRequest(raw),artifact=makeReviewArtifact('run',req,'producer-session');raw.review.artifactId=artifact.id;const bound=validateVerificationRequest(raw);expect(validateReviewArtifact(artifact,'run',bound)).toMatchObject({producer:'producer-session',assurance:'self_attested'});expect(()=>makeReviewArtifact('run',req,req.review.reviewer)).toThrow('distinct');expect(()=>validateReviewArtifact({...artifact,rationale:'changed'},'run',bound)).toThrow('does not bind');}); test('executor observations have a strict helper-owned shape',()=>{expect(()=>validateVerificationObservation({booted:true,legitimate:true,security:'pass',existingTests:true,output:'ok',inputHash:'',tested:true})).toThrow('Unexpected');expect(()=>validateVerificationObservation({booted:'yes',legitimate:true,security:'pass',existingTests:true,output:'ok',inputHash:''})).toThrow('booleans');}); test('secret-bearing fixtures cannot enter a replayable verification bundle',async()=>{const runDir=tmp(),snapshot=path.join(runDir,'snapshot');fs.mkdirSync(snapshot);fs.writeFileSync(path.join(snapshot,'app.js'),'vulnerable\n');const raw:any=request('vulnerable\n','fixed\n');raw.review.reviewedPatchHash=patchHash(raw);raw.fixtures={'secret.txt':CREDENTIAL_CANARY};const manifest:any={entries:[{path:'app.js',transformation:undefined}]};const runtime:any={id:'node',image:'runtime@sha256:'+'a'.repeat(64),platform:'linux/amd64'};expect(verifyRepair({runId:'run',runDir,manifest,rawRequest:raw,runtime,verifier:runtime,policyHash:'c'.repeat(64),archives:[],executor:{observe:async()=>{throw new Error('must not execute');}},persist:false})).rejects.toMatchObject({code:'REDACTION_FAILED'});}); + test('a non-root producer patches a read-only snapshot file and keeps its mode',()=>{const root=tmp(),before=path.join(root,'before'),after=path.join(root,'after');fs.mkdirSync(before);fs.writeFileSync(path.join(before,'app.js'),'vulnerable\n');fs.chmodSync(path.join(before,'app.js'),0o444);const raw:any=request('vulnerable\n','fixed\n');raw.review.reviewedPatchHash=patchHash(raw);preparePatchedSource(before,after,validateVerificationRequest(raw));expect(fs.readFileSync(path.join(after,'app.js'),'utf8')).toBe('fixed\n');expect(fs.statSync(path.join(after,'app.js')).mode&0o777).toBe(0o444);}); test('declared patches require exact pristine preimages',()=>{const root=tmp(),before=path.join(root,'before'),after=path.join(root,'after');fs.mkdirSync(before);fs.writeFileSync(path.join(before,'app.js'),'changed\n');const raw:any=request('vulnerable\n','fixed\n');raw.review.reviewedPatchHash=patchHash(raw);expect(()=>preparePatchedSource(before,after,validateVerificationRequest(raw))).toThrow('preimage');}); test('helper derives dependency and configuration patch effects',()=>{expect(fileEffect('package.json')).toBe('dependency');expect(fileEffect('apps/x/pyproject.toml')).toBe('dependency');expect(fileEffect('config/environments/test.rb')).toBe('configuration');const root=tmp(),before=path.join(root,'before'),after=path.join(root,'after');fs.mkdirSync(before);fs.writeFileSync(path.join(before,'package.json'),'{}');const raw:any=request('vulnerable\n','fixed\n');raw.changes=[{path:'package.json',beforeSha256:sha256('{}'),after:'{"dependencies":{}}',effect:'configuration'}];raw.boundaryFiles=['package.json'];raw.review.reviewedPatchHash=patchHash(raw);expect(()=>preparePatchedSource(before,after,validateVerificationRequest(raw))).toThrow('must be declared as dependency');}); test('verification evidence stays outside both source trees and temporary copies are deleted on errors',async()=>{const runDir=tmp(),snapshot=path.join(runDir,'snapshot');fs.mkdirSync(snapshot);fs.writeFileSync(path.join(snapshot,'app.js'),'vulnerable\n',{mode:0o755});const testBody='test("ok",()=>{})\n';fs.writeFileSync(path.join(snapshot,'app.test.js'),testBody);const packageJson='{"scripts":{"test":"node --test"}}';fs.writeFileSync(path.join(snapshot,'package.json'),packageJson);const raw:any=request('vulnerable\n','fixed\n');raw.findingId='1abcdefabcdefabcdefabcdefabcdefa';raw.existingTests=[{executable:'/usr/local/bin/node',args:['--test','--test-reporter=tap','./app.test.js']}];raw.testFiles=['app.test.js'];raw.changes.push({path:'security/policy.js',beforeSha256:null,after:'export const enforced = true\n',effect:'source'});raw.review.reviewedPatchHash=patchHash(raw);const manifest:any={version:3,createdAt:'2026-01-01T00:00:00Z',expiresAt:'2026-01-08T00:00:00Z',root:'/repo',headCommit:'a'.repeat(40),originalHash:'b'.repeat(64),executionHash:treeHash(snapshot),entries:[{path:'app.js',originalHash:sha256('vulnerable\n'),executionHash:sha256('vulnerable\n'),bytes:11,mode:0o755},{path:'app.test.js',originalHash:sha256(testBody),executionHash:sha256(testBody),bytes:Buffer.byteLength(testBody),mode:0o600},{path:'package.json',originalHash:sha256(packageJson),executionHash:sha256(packageJson),bytes:packageJson.length,mode:0o600}]};const runtime:any={id:'node',stack:'node',image:'runtime@sha256:'+'a'.repeat(64),platform:'linux/amd64'};const executor:any={observe:async(source:string,phase:string,_request:any,_runtime:any,_verifier:any,work:string)=>{expect(work.startsWith(source+path.sep)).toBe(false);fs.mkdirSync(path.join(work,phase),{recursive:true});fs.writeFileSync(path.join(work,phase,'policy.json'),'evidence');return{booted:true,legitimate:true,security:phase==='before'?'intended_failure':'pass',existingTests:true,output:'ok',inputHash:''};}};let unattested:any;try{await verifyRepair({runId:'run',runDir,manifest,rawRequest:raw,runtime,verifier:runtime,policyHash:'c'.repeat(64),archives:[],executor,persist:false});}catch(error){unattested=error;}expect(unattested).toMatchObject({code:'PREREQUISITE',attempt:{artifactKind:'repair_candidate',findingId:raw.findingId,reproduction:'reproduced',repair:'proposed',bundleIssued:false,testCompletionAssurance:'self_reported'}});expect(unattested.message).toContain('out-of-process external assertion witness');expect(fs.existsSync(path.join(runDir,'bundles'))).toBe(false);expect(fs.readdirSync(path.join(runDir,'verification'))).toEqual([]);expect(fs.statSync(path.join(snapshot,'app.js')).mode&0o777).toBe(0o755);const failing:any={observe:async()=>{throw new Error('stop');}};expect(verifyRepair({runId:'run',runDir,manifest,rawRequest:raw,runtime,verifier:runtime,policyHash:'c'.repeat(64),archives:[],executor:failing,persist:false})).rejects.toThrow('stop');expect(fs.readdirSync(path.join(runDir,'verification'))).toEqual([]);const afterFailure:any={observe:async(_source:string,phase:string)=>{if(phase==='after')throw new CsoError('TOOL_FAILED','after setup failed');return{booted:true,legitimate:true,security:'intended_failure',existingTests:true,output:'reproduced safely',inputHash:''};}};let failed:any;try{await verifyRepair({runId:'run',runDir,manifest,rawRequest:raw,runtime,verifier:runtime,policyHash:'c'.repeat(64),archives:[],executor:afterFailure});}catch(error){failed=error;}expect(failed).toMatchObject({code:'TOOL_FAILED',attempt:{findingId:raw.findingId,reproduction:'reproduced',repair:'failed',bundleIssued:false}});const attempt=failed.attempt,artifact=JSON.parse(fs.readFileSync(path.join(runDir,'verification-attempts',`${attempt.id}.json`),'utf8'));expect(artifact).toMatchObject({id:attempt.id,findingId:raw.findingId,reproduction:'reproduced',repair:'failed',bundleIssued:false});expect(fs.existsSync(path.join(runDir,'bundles'))).toBe(false);expect(fs.readdirSync(path.join(runDir,'verification'))).toEqual([]);const assertionFailure:any={observe:async(_source:string,phase:string)=>({booted:true,legitimate:true,security:'intended_failure',existingTests:true,output:phase==='before'?'reproduced safely':'repair remained vulnerable',inputHash:''})};let notRepaired:any;try{await verifyRepair({runId:'run',runDir,manifest,rawRequest:raw,runtime,verifier:runtime,policyHash:'c'.repeat(64),archives:[],executor:assertionFailure});}catch(error){notRepaired=error;}expect(notRepaired).toMatchObject({code:'ASSERTION_FAILED',attempt:{reproduction:'reproduced',repair:'failed',bundleIssued:false,after:{security:'intended_failure'}}});expect(fs.existsSync(path.join(runDir,'bundles'))).toBe(false);expect(fs.readdirSync(path.join(runDir,'verification-attempts'))).toHaveLength(2);const brokenStartup:any={observe:async()=>({booted:false,legitimate:false,security:'pass',existingTests:false,output:'application never started',inputHash:''})};let blocked:any;try{await verifyRepair({runId:'run',runDir,manifest,rawRequest:raw,runtime,verifier:runtime,policyHash:'c'.repeat(64),archives:[],executor:brokenStartup});}catch(error){blocked=error;}expect(blocked).toMatchObject({attempt:{reproduction:'blocked',repair:'failed',bundleIssued:false}});expect(fs.readdirSync(path.join(runDir,'verification-attempts'))).toHaveLength(3);fs.writeFileSync(path.join(snapshot,'unmanifested.txt'),'changed after snapshot identity\n');let executed=false;const mustNotExecute:any={observe:async()=>{executed=true;return{booted:true,legitimate:true,security:'pass',existingTests:true,output:'',inputHash:''};}};await expect(verifyRepair({runId:'run',runDir,manifest,rawRequest:raw,runtime,verifier:runtime,policyHash:'c'.repeat(64),archives:[],executor:mustNotExecute,persist:false})).rejects.toMatchObject({code:'INCOMPATIBLE_INPUT'});expect(executed).toBe(false);}); diff --git a/test/cso-node-lifecycle-integration.test.ts b/test/cso-node-lifecycle-integration.test.ts index 69c616526..9f73c26fa 100644 --- a/test/cso-node-lifecycle-integration.test.ts +++ b/test/cso-node-lifecycle-integration.test.ts @@ -191,11 +191,12 @@ suite('CSO Node producer-visible repair lifecycle fixture', () => { const submittedFinding = submitted.findings[0]; expect(submittedFinding).toMatchObject({ reproduction: 'not_attempted', repair: 'not_attempted', closure: 'open' }); const plan = await cli.command(['runtime-plan', original.runId, 'node', '--port', '34569']); + expect(plan.runtimeProfile).toBe(cli.runtime.id); const vulnerable = fs.readFileSync(path.join(originalRunDirectory, 'snapshot', 'policy.js'), 'utf8'); const repaired = "module.exports={status:403,body:'DENIED'};\n"; const request: VerificationRequest = { findingId: submittedFinding.id, - runtimeProfile: cli.runtime.id, + runtimeProfile: plan.runtimeProfile, port: 34569, start: plan.start.command, legitimate: [{ name: 'legitimate control', path: '/control', method: 'GET', expected: { status: 200, includes: 'CONTROL_OK' } }], From 501ff8310a1e13e6bf7f7804907d58b5cd37ef2e Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 17 Oct 2026 10:19:52 -0700 Subject: [PATCH 2/3] v1.91.124.0 feat(cso): helper run clock (remainingSeconds), /cso time discipline, content-free producer tool timeline (#3242) * fix(cso): comprehensive verify can patch read-only snapshots and accepts runtime-plan's profile * feat(cso): helper replies carry remainingSeconds; producer receipts record a content-free tool timeline * fix(cso): count remainingSeconds to the investigation deadline; bound and validate the tool timeline remainingSeconds now measures to deadline - 60s, the point where scan, verify and record-review refuse with DEADLINE, and inspect carries it. The post-command clock lookup can only omit the field, never fail a command whose state change is already saved. The stream timeline records end times from tool_result, indexes only recorded calls so it stays at the 4096 cap, and receipts reject malformed timeline entries. * docs(changelog): unreleased entry for the /cso run clock * v1.91.124.0 stamp --- CHANGELOG.md | 12 ++++++++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- cso/SKILL.md | 2 +- cso/SKILL.md.tmpl | 2 +- lib/cso/cli.ts | 34 ++++++++++++++++++++++- lib/cso/state.ts | 2 +- package.json | 2 +- scripts/cso-eval-producer.ts | 1 + scripts/cso-eval-protocol.ts | 4 ++- scripts/cso-eval.ts | 4 ++- test/claude-producer-stream.test.ts | 28 +++++++++++++++++-- test/cso-cli.test.ts | 11 ++++++++ test/cso-eval.test.ts | 5 ++++ test/helpers/providers/claude-stream.ts | 37 +++++++++++++++++++++++-- test/helpers/providers/types.ts | 10 +++++++ 16 files changed, 144 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 679805e64..f8d22287c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [1.91.124.0] - 2026-10-17 + +**/cso runs finish on time.** In the first paid runtime qualification, one comprehensive run submitted its findings ten minutes in, kept working for another 25 minutes without calling the helper, and was killed before it called `finish`. Its report was never finished, so the cold-start gate scored 63/64. The agent had no clock to follow. + +### Added +- **Helper replies carry `remainingSeconds`.** `start`, `resume`, `submit`, `scan`, `scanner-outcome`, `import-sarif`, `record-review`, `test-plan`, `runtime-plan`, `verify` and `inspect` report the whole seconds of investigation time left. The count stops where `scan`, `verify` and `record-review` start refusing with DEADLINE, so the final reporting minute stays free for `submit` and `finish`. The field is added after the command and only omitted if the deadline can't be read, so it never turns a saved change into an error. +- **The /cso skill follows the clock.** It now says to start only work that fits and, below 120 seconds, to stop waiting and `finish`. +- **Producer receipts record a content-free tool timeline.** For each tool call the eval producer records the name, its start and end offsets, and whether a subagent made it. It never records inputs or outputs. The timeline is capped at 4096 entries, validated on read, and left out of the batch index. Scoring is unchanged. + +### If you… +- **…drive `gstack-cso` from your own agent:** read `remainingSeconds` from each reply and stop starting work below two minutes. `inspect` refreshes it without side effects. + ## [1.91.123.0] - 2026-10-17 **/cso comprehensive repairs run again.** In the first paid runtime qualification, the agent recorded 148 repair reviews across 42 comprehensive runs, but not one verification attempt started. Every runtime scored 0/24 on held-out repairs. Two helper bugs stacked: the patch write failed on the snapshot's read-only files, and `runtime-plan` handed the agent a profile name that `verify` refused. diff --git a/VERSION b/VERSION index 48617770e..114fc58b8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.123.0 +1.91.124.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 957e4470a..9a47949cd 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.123.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.124.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/cso/SKILL.md b/cso/SKILL.md index 46e19240b..47293f45f 100644 --- a/cso/SKILL.md +++ b/cso/SKILL.md @@ -99,7 +99,7 @@ Start first; inspect snapshot, readiness, deadline, transformations, and coverag When updating coverage, copy every record's `domain` and `scope` exactly from `inspect`; a new scope leaves the planned scope unassessed. Only helper commands may update helper-owned records. -Budgets are ten minutes daily and thirty comprehensive, including one minute for reporting. Use at most three investigation workers and two reproduction groups per Docker endpoint. Attempts get five minutes and three harness repairs per finding, within the run deadline. Prioritize unresolved high impact. Report model usage only when exposed. +Budgets: ten minutes daily, thirty comprehensive. Use at most three investigation workers and two reproduction groups per Docker endpoint. Attempts get five minutes and three harness repairs per finding. Prioritize unresolved high impact. Report model usage only when exposed. Helper replies carry investigation `remainingSeconds`: start only what fits; below 120, stop and `finish`. --- ## Section index — Read each section when its situation applies diff --git a/cso/SKILL.md.tmpl b/cso/SKILL.md.tmpl index 9f936311a..765027bc5 100644 --- a/cso/SKILL.md.tmpl +++ b/cso/SKILL.md.tmpl @@ -99,7 +99,7 @@ Start first; inspect snapshot, readiness, deadline, transformations, and coverag When updating coverage, copy every record's `domain` and `scope` exactly from `inspect`; a new scope leaves the planned scope unassessed. Only helper commands may update helper-owned records. -Budgets are ten minutes daily and thirty comprehensive, including one minute for reporting. Use at most three investigation workers and two reproduction groups per Docker endpoint. Attempts get five minutes and three harness repairs per finding, within the run deadline. Prioritize unresolved high impact. Report model usage only when exposed. +Budgets: ten minutes daily, thirty comprehensive. Use at most three investigation workers and two reproduction groups per Docker endpoint. Attempts get five minutes and three harness repairs per finding. Prioritize unresolved high impact. Report model usage only when exposed. Helper replies carry investigation `remainingSeconds`: start only what fits; below 120, stop and `finish`. --- {{SECTION_INDEX:cso}} diff --git a/lib/cso/cli.ts b/lib/cso/cli.ts index 672272742..4cac4e3f8 100644 --- a/lib/cso/cli.ts +++ b/lib/cso/cli.ts @@ -57,6 +57,7 @@ import { readJson, reclaimDeadAttemptScratch, repoId, + executionDeadline, requireTime, retention, runDirectory, @@ -1609,6 +1610,7 @@ async function inspect(args: string[]) { ? readJson(join(dir, 'preparation.json')) : undefined, recovery: recoveryEvents(dir), + remainingSeconds: remainingSeconds(report.deadline), }); } async function read(args: string[]) { @@ -2796,12 +2798,31 @@ async function replay(args: string[], dependencies: CsoCliDependencies) { }); } +/** In-run replies that report the investigation time left; every one but `start` takes the run ID first. */ +const RUN_CLOCK_COMMANDS = new Set([ + 'start', + 'resume', + 'submit', + 'scan', + 'scanner-outcome', + 'import-sarif', + 'record-review', + 'test-plan', + 'runtime-plan', + 'verify', +]); +/** Whole seconds until the investigation deadline (the final reporting minute excluded), or undefined if unknown. */ +function remainingSeconds(deadline: string): number | undefined { + const ends = executionDeadline({ deadline }); + return Number.isFinite(ends) ? Math.max(0, Math.floor((ends - Date.now()) / 1000)) : undefined; +} export async function dispatchCsoCommand( command: string, args: string[], dependencies: CsoCliDependencies, ): Promise { args = [...args]; + const runId = args[0]; if (!['start', 'recheck', 'replay', 'doctor', 'provision-images'].includes(command)) { const started = Date.now(); retention(started, { deadlineMs: started + RETENTION_MAINTENANCE_MS, maxEntries: RETENTION_MAX_ENTRIES }); @@ -2876,7 +2897,18 @@ export async function dispatchCsoCommand( default: throw new CsoError('INVALID_ARGUMENT', `Unknown command: ${command}`); } - return result; + if (!RUN_CLOCK_COMMANDS.has(command) || !result || typeof result !== 'object' || Array.isArray(result)) + return result; + const deadline = (result as { deadline?: unknown }).deadline; + let seconds: number | undefined; + try { + seconds = remainingSeconds( + typeof deadline === 'string' ? deadline : loadReport(runDirectory(runId)).deadline, + ); + } catch { + seconds = undefined; + } + return seconds === undefined ? result : { ...result, remainingSeconds: seconds }; } const PRODUCTION_CLI_DEPENDENCIES: CsoCliDependencies = Object.freeze({ runtimeCatalog: RUNTIME_CATALOG, diff --git a/lib/cso/state.ts b/lib/cso/state.ts index 66ac7f6eb..aaaca9f79 100644 --- a/lib/cso/state.ts +++ b/lib/cso/state.ts @@ -944,7 +944,7 @@ export function loadReport(dir: string): RunReportV3 { export function event(report: RunReportV3, kind: string, message: string): void { report.events.push({ at: new Date().toISOString(), kind, message: redact(message) }); } -export function executionDeadline(report: RunReportV3): number { +export function executionDeadline(report: Pick): number { return Date.parse(report.deadline) - 60_000; } export function requireTime(report: RunReportV3): void { diff --git a/package.json b/package.json index a130d5382..1b48d7977 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.123", + "version": "1.91.124", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/cso-eval-producer.ts b/scripts/cso-eval-producer.ts index 333e71268..a2995473b 100644 --- a/scripts/cso-eval-producer.ts +++ b/scripts/cso-eval-producer.ts @@ -686,6 +686,7 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op durationMs: run.durationMs, firstUsefulResultMs: null, toolCalls: run.toolCalls, + ...(run.toolTimeline ? { toolTimeline: run.toolTimeline } : {}), output, outputHash: sha256(output), usage: { diff --git a/scripts/cso-eval-protocol.ts b/scripts/cso-eval-protocol.ts index 88174ecff..58e76d50c 100644 --- a/scripts/cso-eval-protocol.ts +++ b/scripts/cso-eval-protocol.ts @@ -130,6 +130,8 @@ export interface ProducerReceipt { durationMs: number; firstUsefulResultMs: null; toolCalls: number; + /** Content-free tool timeline (name, offset, thread) for diagnosing overruns; absent when the adapter has none. */ + toolTimeline?: Array<{ atMs: number; endMs?: number; tool: string; agent: 'main' | 'subagent' }>; output: string; outputHash: string; usage: { @@ -148,7 +150,7 @@ export interface ProducerReceipt { * Compact trusted index; raw output remains in the separately retained receipt. * Collection adds the matrix cell that the receipt's cellRef names. */ -export type ProducerReceiptIndex = Omit & { +export type ProducerReceiptIndex = Omit & { cell: ProducerCell; error?: { code: string }; }; diff --git a/scripts/cso-eval.ts b/scripts/cso-eval.ts index ad02a9d2f..eef53751d 100644 --- a/scripts/cso-eval.ts +++ b/scripts/cso-eval.ts @@ -400,6 +400,8 @@ function validateReceipt(receipt: ProducerReceipt, cell: EvalCell, inputHash: st validateInstallationIdentity(receipt.installationIdentity, 'INVALID_PRODUCER_RECEIPT'); validateProviderIdentity(receipt.providerIdentity, cell.host === 'codex' ? 'gpt' : cell.host, 'INVALID_PRODUCER_RECEIPT'); validateArtifactInventory(receipt.artifacts, 'INVALID_PRODUCER_RECEIPT'); + const timeline = receipt.toolTimeline; + if (timeline !== undefined && (!Array.isArray(timeline) || timeline.length > 4096 || timeline.some(entry => !entry || !Number.isSafeInteger(entry.atMs) || entry.atMs < 0 || (entry.endMs !== undefined && (!Number.isSafeInteger(entry.endMs) || entry.endMs < entry.atMs)) || typeof entry.tool !== 'string' || !/^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(entry.tool) || !['main', 'subagent'].includes(entry.agent) || Object.keys(entry).some(key => !['atMs', 'endMs', 'tool', 'agent'].includes(key))))) throw new Error('INVALID_PRODUCER_RECEIPT'); } function validateInstallationIdentity(identity: ProducerInstallationIdentity, error: string): void { @@ -530,7 +532,7 @@ export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEv estimatedCost: { measured: costs.length, denominator: expected.length, totalUSD: costs.length ? costs.reduce((sum, value) => sum + value, 0) : null, source: 'pricing-table-estimate' }, }); } - const indexes: ProducerReceiptIndex[] = bound.map(({ output: _output, error, ...receipt }) => ({ ...receipt, ...(error ? { error: { code: error.code } } : {}) })); + const indexes: ProducerReceiptIndex[] = bound.map(({ output: _output, toolTimeline: _toolTimeline, error, ...receipt }) => ({ ...receipt, ...(error ? { error: { code: error.code } } : {}) })); const base = { schemaVersion: 1 as const, matrixHash: matrixHash(matrix), scheduleHash: digest(JSON.stringify(schedule)), receipts: indexes.sort((left, right) => left.cell.id.localeCompare(right.cell.id)), summary: { scheduled: matrix.cells.length, prepared: schedule.preparedCells, submitted: bound.length, missing: matrix.cells.length - bound.length, matchedPairsExpected: expectedMatchedPairs(matrix), matchedPairsSubmitted: completePairs.length, modelMismatches, groups, diff --git a/test/claude-producer-stream.test.ts b/test/claude-producer-stream.test.ts index badfbe836..014801c12 100644 --- a/test/claude-producer-stream.test.ts +++ b/test/claude-producer-stream.test.ts @@ -33,7 +33,7 @@ describe('Claude producer stream accounting', () => { test('takes usage and turns from the final result and keeps the paid result contract', () => { const state = claudeStreamState(); for (const line of lines(start('a', 1000), delta(100), stop, result({}))) consumeClaudeStreamLine(state, line); - expect(resultFromClaudeStream(state, { model: 'claude-test', durationMs: 5 })).toEqual({ output: 'REPORT', tokens: { input: 2001, output: 201, cached: 14 }, durationMs: 5, toolCalls: 2, modelUsed: 'claude-test' }); + expect(resultFromClaudeStream(state, { model: 'claude-test', durationMs: 5 })).toEqual({ output: 'REPORT', tokens: { input: 2001, output: 201, cached: 14 }, durationMs: 5, toolCalls: 2, toolTimeline: [], modelUsed: 'claude-test' }); for (const ending of [[], [result({ result: ' ' })], [result({ result: 7 })], [result({ is_error: true })], [result({ subtype: 'error_max_turns' })]]) { const partial = claudeStreamState(); for (const line of lines(start('a', 10), delta(3), stop, ...ending)) consumeClaudeStreamLine(partial, line); @@ -52,7 +52,31 @@ describe('Claude producer stream accounting', () => { for (const line of lines(start('a', 1000), delta(100), stop, subagent('s1', 50000), subagent('s1', 50000), subagent('s2', 50001), result({ result: 'MAIN ENDED TURN', modelUsage: sessionUsage }), start('b', 1002), delta(102), stop, result({ num_turns: 1, result: 'REPORT', usage: { input_tokens: 1002, output_tokens: 102, cache_read_input_tokens: 7 }, modelUsage: sessionUsage }))) consumeClaudeStreamLine(state, line); - expect(resultFromClaudeStream(state, { model: 'claude-test', durationMs: 5 })).toEqual({ output: 'REPORT', tokens: { input: 153009, output: 15308, cached: 49 }, durationMs: 5, toolCalls: 3, modelUsed: 'claude-test' }); + expect(resultFromClaudeStream(state, { model: 'claude-test', durationMs: 5 })).toEqual({ output: 'REPORT', tokens: { input: 153009, output: 15308, cached: 49 }, durationMs: 5, toolCalls: 3, toolTimeline: [], modelUsed: 'claude-test' }); + }); + + test('records a content-free tool timeline: main and subagent calls once each, never their inputs', () => { + const toolStart = (id: string, name: string) => ({ type: 'stream_event', parent_tool_use_id: null, event: { type: 'content_block_start', index: 1, content_block: { type: 'tool_use', id, name, input: {} } } }); + const mainCall = { type: 'assistant', message: { id: 'a', content: [{ type: 'tool_use', id: 'toolu_bash', name: 'Bash', input: { command: 'gstack-cso inspect SECRET_PATH' } }] } }; + const subCall = { type: 'assistant', parent_tool_use_id: 'toolu_agent', message: { id: 's1', content: [{ type: 'tool_use', id: 'toolu_sub', name: 'Bash', input: { command: 'cat SECRET_PATH' } }], usage: { input_tokens: 1, output_tokens: 1 } } }; + const state = claudeStreamState(); + for (const line of lines(start('a', 10), toolStart('toolu_bash', 'Bash'), toolStart('toolu_agent', 'Agent'), mainCall, subCall, subCall, toolStart('toolu_odd', 'mcp__x/../y'))) consumeClaudeStreamLine(state, line); + const timeline = resultFromClaudeStream(state, { model: 'claude-test', durationMs: 5, error: { code: 'timeout', reason: 'exceeded 5ms' } }).toolTimeline!; + expect(timeline.map(({ tool, agent }) => [tool, agent])).toEqual([['Bash', 'main'], ['Agent', 'main'], ['Bash', 'subagent'], ['other', 'main']]); + expect(timeline.every(entry => Number.isSafeInteger(entry.atMs) && entry.atMs >= 0 && Object.keys(entry).length === 3)).toBe(true); + expect(JSON.stringify(timeline)).not.toContain('SECRET_PATH'); + }); + + test('stamps a tool call end once from its tool_result and keeps the timeline and its id index at the cap', () => { + const state = claudeStreamState(); + const toolResult = (id: string) => ({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: id, content: 'SECRET_OUTPUT' }] } }); + const call = (id: string) => ({ type: 'assistant', message: { id: `m${id}`, content: [{ type: 'tool_use', id, name: 'Bash', input: {} }] } }); + for (const line of lines(call('t0'), toolResult('t0'), toolResult('t0'), toolResult('unknown'), call('t1'))) consumeClaudeStreamLine(state, line); + expect(state.toolTimeline.map(entry => entry.endMs === undefined)).toEqual([false, true]); + expect(JSON.stringify(state.toolTimeline)).not.toContain('SECRET_OUTPUT'); + for (let i = 2; i < 5000; i++) consumeClaudeStreamLine(state, JSON.stringify(call(`t${i}`))); + expect(state.toolTimeline).toHaveLength(4096); + expect(state.toolUseIndex.size).toBe(4096); }); test('a killed run counts each subagent message once and never reports subagent text as the producer output', () => { diff --git a/test/cso-cli.test.ts b/test/cso-cli.test.ts index 4a14016df..b3b10735b 100644 --- a/test/cso-cli.test.ts +++ b/test/cso-cli.test.ts @@ -69,6 +69,17 @@ describe('compiled CSO command workflow',()=>{ runGit('init','-q');runGit('config','user.email','fixture@example.test');runGit('config','user.name','Fixture');fs.writeFileSync(path.join(planRepo,'package.json'),JSON.stringify({name:'sensitive-plan',version:'1.0.0',scripts:{start:'node app.js',test:'node --test'}}));fs.writeFileSync(path.join(planRepo,'package-lock.json'),JSON.stringify({name:'sensitive-plan',version:'1.0.0',lockfileVersion:3,packages:{'':{name:'sensitive-plan',version:'1.0.0'}}}));fs.writeFileSync(path.join(planRepo,'app.js'),'export const app = true\n');fs.writeFileSync(path.join(planRepo,testPath),'const test=require("node:test");test("control",()=>{})\n');runGit('add','.');runGit('commit','-qm','sensitive test path'); const started=command(['start','--repo',planRepo,'--comprehensive','--offline']);expect(started.status).toBe(0);const run=JSON.parse(started.stdout),planned=command(['runtime-plan',run.runId,'node','--port','3456']);expect(planned.status).toBe(0);const plan=JSON.parse(planned.stdout),serialized=JSON.stringify(plan); expect(serialized).not.toContain(wallet);const testHandle=plan.tests.files.find((value:string)=>value.startsWith('@cso-path//'));expect(testHandle).toMatch(/^@cso-path\/\/[a-f0-9]{32}$/);expect(plan.tests.commands[0].args).toContain(`./${testHandle}`);expect(plan.start.command).toEqual({executable:'/usr/local/bin/node',args:['app.js']}); + const budget=(Date.parse(run.deadline)-Date.parse(run.createdAt))/1000;for(const remaining of [run.remainingSeconds,plan.remainingSeconds]){expect(Number.isSafeInteger(remaining)).toBe(true);expect(remaining).toBeGreaterThan(budget-120);expect(remaining).toBeLessThanOrEqual(budget-60);}expect(plan.remainingSeconds).toBeLessThanOrEqual(run.remainingSeconds); + }); + test('run clock counts to the investigation deadline: zero in the reporting minute, which still accepts submit until the audit deadline',()=>{ + const started=command(['start','--repo',repo,'--scope','auth','--offline']);expect(started.status).toBe(0);const run=JSON.parse(started.stdout),reportPath=path.join(state,'security','cso',run.repoId,run.runId,'report.json'),input=path.join(root,`clock-${run.runId}.json`);fs.writeFileSync(input,'{}'); + const submitted=command(['submit',run.runId,input]);expect(submitted.status).toBe(0);const inspected=command(['inspect',run.runId]);expect(inspected.status).toBe(0); + for(const remaining of [JSON.parse(submitted.stdout).remainingSeconds,JSON.parse(inspected.stdout).remainingSeconds]){expect(Number.isSafeInteger(remaining)).toBe(true);expect(remaining).toBeGreaterThan(0);expect(remaining).toBeLessThanOrEqual(run.remainingSeconds);} + const report=JSON.parse(fs.readFileSync(reportPath,'utf8'));report.deadline=new Date(Date.now()+30_000).toISOString();fs.writeFileSync(reportPath,JSON.stringify(report,null,2)+'\n'); + const late=command(['inspect',run.runId]);expect(late.status).toBe(0);expect(JSON.parse(late.stdout).remainingSeconds).toBe(0); + const reporting=command(['submit',run.runId,input]);expect(reporting.status).toBe(0);expect(JSON.parse(reporting.stdout).remainingSeconds).toBe(0); + report.deadline=new Date(Date.now()-1_000).toISOString();fs.writeFileSync(reportPath,JSON.stringify({...JSON.parse(fs.readFileSync(reportPath,'utf8')),deadline:report.deadline},null,2)+'\n'); + const refused=command(['submit',run.runId,input]);expect(refused.status).not.toBe(0);expect(refused.stderr).toContain('DEADLINE');expect(refused.stdout).toBe(''); }); test('wallet-shaped helper finding IDs survive submit, review, and verification admission',()=>{const started=command(['start','--repo',repo,'--comprehensive','--offline']);expect(started.status).toBe(0);const run=JSON.parse(started.stdout),base:any={title:'Wallet-shaped helper ID',rootCause:'Tenant query omits caller tenant predicate 0',location:{path:'src/users.ts',line:1,symbol:'tenantQuery'},advisoryIds:[],severity:'high',confidence:'high',confidenceRationale:'Caller-to-sink trace directly establishes the missing predicate',evidence:'supported',attackerControl:'Tenant chooses the record ID',impact:'Cross-tenant record disclosure',scenario:'A tenant requests another tenant record by ID',trace:['route','tenantQuery','database'],references:['src/users.ts:1'],recommendation:'Bind the query to the authenticated tenant',challenge:{reviewer:'reviewer-session',independent:true,mode:'independent_agent',callers:'route caller checked',controls:'authentication lacks authorization',counterevidence:'none found',conclusion:'tenant predicate is absent'}};let helperId='';for(let i=0;i<10_000;i++){base.rootCause=`Tenant query omits caller tenant predicate ${i}`;helperId=fingerprint(base);if(redact(helperId)!==helperId)break;}expect(redact(helperId)).not.toBe(helperId);const input=path.join(root,`wallet-id-${run.runId}.json`);fs.writeFileSync(input,JSON.stringify({findings:[base]}));expect(command(['submit',run.runId,input]).status).toBe(0);const reportPath=path.join(state,'security','cso',run.repoId,run.runId,'report.json'),stored=JSON.parse(fs.readFileSync(reportPath,'utf8'));expect(stored.findings[0].id).toBe(helperId);const before=fs.readFileSync(path.join(repo,'src','users.ts'),'utf8'),request:any={findingId:helperId,runtimeProfile:'node-v1',port:3456,start:{executable:'/usr/local/bin/node',args:['src/users.ts']},legitimate:[{name:'control',path:'/health',method:'GET',expected:{status:200}}],security:{name:'tenant boundary',path:'/users/other',method:'GET',expected:{status:403},vulnerable:{status:200}},existingTests:[{executable:'/usr/local/bin/node',args:['--test']}],testFiles:['src/control.test.ts'],fixtures:{},boundaryFiles:['src/users.ts'],changes:[{path:'src/users.ts',beforeSha256:sha256(before),after:before+'// tenant predicate\n',effect:'source'}],review:{reviewer:'reviewer-session',independent:true,rootCauseRepaired:true,featurePreserved:true,boundaryMocks:false,rationale:'The tenant predicate is added while the control remains intact',reviewedPatchHash:''}};request.review.reviewedPatchHash=patchHash(request);fs.writeFileSync(input,JSON.stringify(request));const reviewed=command(['record-review',run.runId,input,'--producer','producer-session']);expect(reviewed.status).toBe(0);request.review.artifactId=JSON.parse(reviewed.stdout).reviewArtifactId;fs.writeFileSync(input,JSON.stringify(request));const verified=command(['verify',run.runId,input]);expect(verified.status).not.toBe(0);expect(verified.stderr).toContain('PREREQUISITE');expect(verified.stderr).not.toContain('REDACTION_FAILED');const review=JSON.parse(fs.readFileSync(path.join(path.dirname(reportPath),'reviews',`${request.review.artifactId}.json`),'utf8'));expect(review.findingId).toBe(helperId);}); test('daily find, early submit, and finish persist a truthful report without changing source',()=>{const before=git('status','--porcelain=v1','-z'),started=command(['start','--repo',repo,'--scope','auth','--offline']);expect(started.status).toBe(0);const run=JSON.parse(started.stdout),runId=run.runId;expect(run.completeness).toBe('not assessed');const reportPath=path.join(state,'security','cso',run.repoId,runId,'report.json');expect(fs.existsSync(reportPath)).toBe(true);const current=JSON.parse(fs.readFileSync(reportPath,'utf8')); diff --git a/test/cso-eval.test.ts b/test/cso-eval.test.ts index 6d6559576..34b773c84 100644 --- a/test/cso-eval.test.ts +++ b/test/cso-eval.test.ts @@ -268,6 +268,11 @@ describe('CSO matched evaluation accounting', () => { artifactTamper.artifacts.totalBytes = 1; expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [rehashReceipt(artifactTamper)])).toThrow('INVALID_PRODUCER_RECEIPT'); + const timed = rehashReceipt({ ...structuredClone(first), toolTimeline: [{ atMs: 5, endMs: 9, tool: 'Bash', agent: 'main' }, { atMs: 7, tool: 'Agent', agent: 'subagent' }] }); + expect(collectProducerReceipts(matrix, syntheticSchedule(), [timed]).receipts[0]).not.toHaveProperty('toolTimeline'); + for (const entry of [{ atMs: 9, endMs: 5, tool: 'Bash', agent: 'main' }, { atMs: 1, tool: 'cat /secret', agent: 'main' }, { atMs: 1, tool: 'Bash', agent: 'main', input: 'x' }]) + expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [rehashReceipt({ ...structuredClone(first), toolTimeline: [entry] } as ProducerReceipt)])).toThrow('INVALID_PRODUCER_RECEIPT'); + const mixedInstallation = syntheticReceipt(cells[1]); mixedInstallation.installationIdentity = syntheticInstallationIdentity('different-installation'); expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [first, rehashReceipt(mixedInstallation)])).toThrow('UNMATCHED_PRODUCER_INSTALLATIONS'); diff --git a/test/helpers/providers/claude-stream.ts b/test/helpers/providers/claude-stream.ts index 18374b592..5cdb1da6e 100644 --- a/test/helpers/providers/claude-stream.ts +++ b/test/helpers/providers/claude-stream.ts @@ -1,5 +1,5 @@ import { spawn } from 'child_process'; -import type { RunResult } from './types'; +import type { RunResult, ToolTimelineEntry } from './types'; /** Usage, turns and the latest text from `claude -p --output-format stream-json --verbose --include-partial-messages`. */ export interface ClaudeStreamState { @@ -14,14 +14,36 @@ export interface ClaudeStreamState { lastText: string; result?: Record; invalidLines: number; + startedAt: number; + /** Content-free: each tool call's name, start and end offsets and thread, never its input or output. */ + toolTimeline: ToolTimelineEntry[]; + /** Timeline index by tool-use id, for recorded calls only, so it stays within the timeline cap. */ + toolUseIndex: Map; } export function claudeStreamState(): ClaudeStreamState { - return { totals: { input: 0, output: 0, cached: 0 }, open: new Map(), subagents: new Map(), turns: 0, resultTurns: 0, lastText: '', invalidLines: 0 }; + return { totals: { input: 0, output: 0, cached: 0 }, open: new Map(), subagents: new Map(), turns: 0, resultTurns: 0, lastText: '', invalidLines: 0, startedAt: Date.now(), toolTimeline: [], toolUseIndex: new Map() }; } +const TOOL_TIMELINE_LIMIT = 4096; + const count = (value: unknown): number => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : 0; +function recordToolUse(state: ClaudeStreamState, block: { id?: unknown; name?: unknown }, agent: 'main' | 'subagent'): void { + if (typeof block.id !== 'string' || state.toolUseIndex.has(block.id) || state.toolTimeline.length >= TOOL_TIMELINE_LIMIT) return; + state.toolUseIndex.set(block.id, state.toolTimeline.length); + state.toolTimeline.push({ + atMs: Date.now() - state.startedAt, + tool: typeof block.name === 'string' && /^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(block.name) ? block.name : 'other', + agent, + }); +} + +function recordToolResult(state: ClaudeStreamState, block: { tool_use_id?: unknown }): void { + const index = typeof block.tool_use_id === 'string' ? state.toolUseIndex.get(block.tool_use_id) : undefined; + if (index !== undefined && state.toolTimeline[index].endMs === undefined) state.toolTimeline[index].endMs = Date.now() - state.startedAt; +} + function close(state: ClaudeStreamState, key: string): void { const message = state.open.get(key); if (!message) return; @@ -39,6 +61,13 @@ export function consumeClaudeStreamLine(state: ClaudeStreamState, line: string): if (Number.isSafeInteger(event.num_turns) && event.num_turns >= 0) state.resultTurns += event.num_turns; return; } + if (event.type === 'assistant' && Array.isArray(event.message?.content)) { + for (const block of event.message.content) if (block?.type === 'tool_use') recordToolUse(state, block, typeof event.parent_tool_use_id === 'string' ? 'subagent' : 'main'); + } + if (event.type === 'user' && Array.isArray(event.message?.content)) { + for (const block of event.message.content) if (block?.type === 'tool_result') recordToolResult(state, block); + return; + } if (event.type === 'assistant' && typeof event.parent_tool_use_id === 'string') { const usage = event.message?.usage; if (typeof event.message?.id === 'string' && usage) state.subagents.set(event.message.id, { input: count(usage.input_tokens), output: count(usage.output_tokens), cached: count(usage.cache_read_input_tokens) }); @@ -56,6 +85,8 @@ export function consumeClaudeStreamLine(state: ClaudeStreamState, line: string): const usage = stream.message?.usage ?? {}; state.open.set(key, { input: count(usage.input_tokens), output: count(usage.output_tokens), cached: count(usage.cache_read_input_tokens) }); if (key === 'main') state.turns++; + } else if (stream.type === 'content_block_start' && stream.content_block?.type === 'tool_use') { + recordToolUse(state, stream.content_block, key === 'main' ? 'main' : 'subagent'); } else if (stream.type === 'message_delta' && stream.usage) { const message = state.open.get(key); if (!message) return; @@ -99,7 +130,7 @@ export function resultFromClaudeStream(state: ClaudeStreamState, opts: { model: const tokens = (result && resultTokens(result)) ?? claudeStreamUsage(state); const toolCalls = result ? state.resultTurns : state.turns; const modelUsed = typeof result?.model === 'string' && result.model ? result.model : opts.model; - const base = { tokens, durationMs: opts.durationMs, toolCalls, modelUsed }; + const base = { tokens, durationMs: opts.durationMs, toolCalls, toolTimeline: state.toolTimeline, modelUsed }; if (opts.error) return { ...base, output: typeof result?.result === 'string' ? result.result : state.lastText, error: opts.error }; const explicitError = result?.is_error === true || (typeof result?.subtype === 'string' && result.subtype !== 'success'); if (!result || explicitError || typeof result.result !== 'string' || !result.result.trim()) { diff --git a/test/helpers/providers/types.ts b/test/helpers/providers/types.ts index 59063e7b2..e9b51527e 100644 --- a/test/helpers/providers/types.ts +++ b/test/helpers/providers/types.ts @@ -49,6 +49,14 @@ export type RunError = | 'binary_missing' // CLI not found on PATH. | 'unknown'; // Catch-all with reason populated. +export interface ToolTimelineEntry { + atMs: number; + /** Set when the call's tool_result arrived; absent for a call still running when the stream ended. */ + endMs?: number; + tool: string; + agent: 'main' | 'subagent'; +} + export interface RunResult { /** Provider's textual output for the prompt. */ output: string; @@ -58,6 +66,8 @@ export interface RunResult { durationMs: number; /** Count of tool/function calls made during the run (0 if unsupported). */ toolCalls: number; + /** Tool calls in start order: name, start and end offsets from run start, and thread; no inputs or outputs. Unset if unsupported. */ + toolTimeline?: ToolTimelineEntry[]; /** Actual model ID the provider reports using (may be a variant of the family). */ modelUsed: string; /** If the run failed, error code + human reason. output/tokens may be partial. */ From c10c139ed6472ea965f0be4af1567aa580100d7f Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 17 Oct 2026 10:32:11 -0700 Subject: [PATCH 3/3] v1.91.125.0 feat(cso-eval): daily precision counts confirmed incidental issues and leaves out declared fixture instrumentation (#3251) * fix(cso): comprehensive verify can patch read-only snapshots and accepts runtime-plan's profile * feat(cso): helper replies carry remainingSeconds; producer receipts record a content-free tool timeline * fix(cso): count remainingSeconds to the investigation deadline; bound and validate the tool timeline remainingSeconds now measures to deadline - 60s, the point where scan, verify and record-review refuse with DEADLINE, and inspect carries it. The post-command clock lookup can only omit the field, never fail a command whose state change is already saved. The stream timeline records end times from tool_result, indexes only recorded calls so it stays at the 4096 cap, and receipts reject malformed timeline entries. * feat(cso-eval): daily precision counts confirmed incidental issues and leaves out declared fixture instrumentation * docs(changelog): unreleased entry for the cso-eval precision definition * v1.91.125.0 stamp --- CHANGELOG.md | 12 ++++++++++++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- docs/cso-release-qualification.md | 12 ++++++++++++ package.json | 2 +- scripts/cso-eval.ts | 18 ++++++++++++++---- test/cso-eval.test.ts | 21 ++++++++++++++++++++- 7 files changed, 61 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8d22287c..db8fc10dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [1.91.125.0] - 2026-10-17 + +**/cso release qualification measures precision on the application, not on the evaluator's own test rigging.** In the first paid runtime qualification, the daily precision gate failed on every stack (8/34 to 8/49). 116 of the 132 findings counted as false positives pointed at instrumentation the evaluator plants in each fixture: canary credentials and the egress probe route. Most of the other 16 were real issues the fixtures happen to contain. Garry chose to keep the 0.95 bar and fix what it counts. + +### Changed +- **Daily precision leaves out findings on declared fixture instrumentation.** The trusted evaluator marks them `instrumentation`, and they count in neither term. +- **A confirmed real issue counts as a true positive.** When a unanimous judge panel confirms a non-seeded finding is a real issue in that source, the evaluator marks it `incidental` with an opaque key. Each distinct key adds one true positive per cell, so restating it never inflates precision. It never counts toward recall. +- **Recall, the v2 baseline and every threshold are unchanged.** Validation rejects either new judgment on a hypothesis or with a seeded match, and rejects a missing or malformed key. + +### If you… +- **…write an evaluator for `cso-eval.ts`:** `incidental` needs a 64-hex `incidentalKey`, and no other judgment may carry one. + ## [1.91.124.0] - 2026-10-17 **/cso runs finish on time.** In the first paid runtime qualification, one comprehensive run submitted its findings ten minutes in, kept working for another 25 minutes without calling the helper, and was killed before it called `finish`. Its report was never finished, so the cold-start gate scored 63/64. The agent had no clock to follow. diff --git a/VERSION b/VERSION index 114fc58b8..c8708ac73 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.124.0 +1.91.125.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 9a47949cd..8a4b4971f 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.124.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.125.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/cso-release-qualification.md b/docs/cso-release-qualification.md index dc7786f86..191130310 100644 --- a/docs/cso-release-qualification.md +++ b/docs/cso-release-qualification.md @@ -294,6 +294,18 @@ requalified catalog or withdraws the catalog's runtimes in the same pull request. Independently, the helper refuses to select a runtime recorded under another isolation policy. +The daily precision gate (`dailyPrecision95` in +[`cso-eval.ts`](../scripts/cso-eval.ts)) divides true positives by reported +findings. Hypotheses count in neither term. A finding the evaluator marks +`instrumentation` also counts in neither term: it targets the evaluator's +declared fixture instrumentation, such as the canary overlay files or the egress +probe route. The seeded issue adds one true positive per cell, however many +times it is restated. A finding the evaluator marks `incidental` is a different +issue that a unanimous judge panel confirmed is real in that source. It adds one +true positive per distinct `incidentalKey` in its cell, never counts toward +recall, and needs no seeded match. Recall still measures only the seeded issue, +and the thresholds are unchanged. + The judge observes held-out attacks in one boot: a `VerificationRequest`'s `security` field is either one assertion or an array of up to eight assertions with distinct names, judged in a single before/after pair. The before phase is diff --git a/package.json b/package.json index 1b48d7977..737c1d2ae 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.124", + "version": "1.91.125", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/cso-eval.ts b/scripts/cso-eval.ts index eef53751d..87d3280b9 100644 --- a/scripts/cso-eval.ts +++ b/scripts/cso-eval.ts @@ -98,8 +98,15 @@ export interface PortableSkillPayloadIdentity { } export interface EvalFinding { id: string; evidence: 'supported' | 'hypothesis' | 'legacy_review'; claimedTested: boolean; - /** Written only by the independent evaluator after reviewing the reported trace. */ - judgment: 'correct' | 'incorrect' | 'unadjudicated'; matchedCaseId?: string; + /** + * Written only by the independent evaluator after reviewing the reported trace. `correct`: the seeded issue. + * `incidental`: a different issue a unanimous judge panel confirmed is real in this source; a precision true + * positive, never recall. `instrumentation`: it targets the evaluator's declared fixture instrumentation + * (canary overlay files, egress probe), so precision leaves it out of both terms. + */ + judgment: 'correct' | 'incidental' | 'instrumentation' | 'incorrect' | 'unadjudicated'; matchedCaseId?: string; + /** `incidental` only: the evaluator's opaque key for the confirmed issue, so restatements count once per cell. */ + incidentalKey?: string; /** * Written only by the independent evaluator. A producer's tested claim is * valid only when this finding, rather than merely another finding in the @@ -642,9 +649,11 @@ function validateResult(result: EvalResult, cell: EvalCell, corpus: EvalCorpusMa if (result.producerReceiptHash !== undefined && !HEX.test(result.producerReceiptHash)) throw new Error('INVALID_PRODUCER_RECEIPT_PROVENANCE'); const ids = new Set(), repairEvidence = new Set(), recheckEvidence = new Set(); for (const finding of result.findings) { - if (!finding.id || ids.has(finding.id) || !['supported', 'hypothesis', 'legacy_review'].includes(finding.evidence) || !['correct', 'incorrect', 'unadjudicated'].includes(finding.judgment) || typeof finding.claimedTested !== 'boolean') throw new Error('INVALID_FINDING_JUDGMENT'); + if (!finding.id || ids.has(finding.id) || !['supported', 'hypothesis', 'legacy_review'].includes(finding.evidence) || !['correct', 'incidental', 'instrumentation', 'incorrect', 'unadjudicated'].includes(finding.judgment) || typeof finding.claimedTested !== 'boolean') throw new Error('INVALID_FINDING_JUDGMENT'); ids.add(finding.id); if (finding.judgment === 'correct' && (cell.variant !== 'vulnerable' || finding.matchedCaseId !== cell.caseId)) throw new Error('INVALID_ORACLE_MATCH'); + if ((finding.judgment === 'incidental' || finding.judgment === 'instrumentation') && (finding.evidence === 'hypothesis' || finding.matchedCaseId !== undefined)) throw new Error('INVALID_FINDING_JUDGMENT'); + if ((finding.judgment === 'incidental') !== (finding.incidentalKey !== undefined) || (finding.incidentalKey !== undefined && !HEX.test(finding.incidentalKey))) throw new Error('INVALID_FINDING_JUDGMENT'); if (finding.trustedVerification !== undefined) { const verification = finding.trustedVerification as Record, allowed = new Set(['repair', 'repairEvidenceHash', 'recheck', 'recheckEvidenceHash']); if (!verification || typeof verification !== 'object' || Array.isArray(verification) || Object.keys(verification).some(key => !allowed.has(key)) || @@ -745,10 +754,11 @@ export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualif let correct = 0, reported = 0, unadjudicated = 0; for (const cell of selected) { const result = results.get(cell.id); - const findings = result?.reportFinished ? result.findings.filter(finding => finding.evidence !== 'hypothesis') : []; + const findings = result?.reportFinished ? result.findings.filter(finding => finding.evidence !== 'hypothesis' && finding.judgment !== 'instrumentation') : []; reported += findings.length; unadjudicated += findings.filter(finding => finding.judgment === 'unadjudicated').length; correct += Number(findings.some(finding => eligibleFinding(cell, finding) && finding.judgment === 'correct')); // Duplicate reports do not increase true positives. + correct += new Set(findings.filter(finding => eligibleFinding(cell, finding) && finding.judgment === 'incidental').map(finding => finding.incidentalKey)).size; } const firstUseful = completed.flatMap(result => result.firstUsefulResultMs === null ? [] : [result.firstUsefulResultMs]); const costs = completed.flatMap(result => result.usage?.costUSD === undefined ? [] : [result.usage.costUSD]); diff --git a/test/cso-eval.test.ts b/test/cso-eval.test.ts index 34b773c84..d1892ba87 100644 --- a/test/cso-eval.test.ts +++ b/test/cso-eval.test.ts @@ -4,7 +4,7 @@ import { chmodSync, cpSync, existsSync, linkSync, lstatSync, mkdirSync, mkdtempS import { createHash } from 'node:crypto'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; -import { collectProducerReceipts, createEvalBaseline, createEvalMatrix, createPortableSkillPayload, loadPortableSkillPayload, prepareEvalJobs, REQUIRED_CONTAINMENT, scoreCollectedEval, scoreEval, validateMatrix, validatePortableSkillPayload, type EvalBaseline, type EvalCell, type EvalMatrix, type EvalResult, type PreparedEvalSchedule } from '../scripts/cso-eval'; +import { collectProducerReceipts, createEvalBaseline, createEvalMatrix, createPortableSkillPayload, loadPortableSkillPayload, prepareEvalJobs, REQUIRED_CONTAINMENT, scoreCollectedEval, scoreEval, validateMatrix, validatePortableSkillPayload, type EvalBaseline, type EvalCell, type EvalFinding, type EvalMatrix, type EvalResult, type PreparedEvalSchedule } from '../scripts/cso-eval'; import { assertProducerArtifactPath, PRODUCER_PROVIDER_POLICY, producerFailureMessage, producerProviderTimeoutMs, producerInstallationIdentity, resolveProducerHelperBinding, runProducerCell, sanitizeProducerRun, validateProductionProducerInstallation } from '../scripts/cso-eval-producer'; import { producerArtifactInventoryHash, producerHostPlatform, producerInstallationIdentityHash, producerProviderIdentityHash, producerReceiptHash, sha256, type ProducerArtifactInventory, type ProducerInstallationIdentity, type ProducerProviderIdentity, type ProducerReceipt } from '../scripts/cso-eval-protocol'; import type { Family, ProviderAdapter, RunOpts, RunResult } from './helpers/providers/types'; @@ -316,6 +316,25 @@ describe('CSO matched evaluation accounting', () => { const two = syntheticResult(negative); two.findings.push({ id: 'false-positive', evidence: 'supported', claimedTested: false, judgment: 'incorrect' }); expect(group(scoreEval(matrix, [one, two]), 'v3', 'daily').precision).toEqual({ numerator: 1, denominator: 3, value: 1 / 3 }); }); + test('confirmed incidental issues count once each for precision, declared instrumentation counts in neither term, and recall still needs the seeded issue', () => { + const positive = matrix.cells.find(item => item.version === 'v3' && item.mode === 'daily' && item.variant === 'vulnerable')!; + const negative = matrix.cells.find(item => item.version === 'v3' && item.mode === 'daily' && item.variant === 'fixed')!; + const key = (seed: string) => createHash('sha256').update(seed).digest('hex'); + const one = syntheticResult(positive); + one.findings.push({ id: 'jwt', evidence: 'supported', claimedTested: false, judgment: 'incidental', incidentalKey: key('jwt') }, { id: 'jwt-again', evidence: 'supported', claimedTested: false, judgment: 'incidental', incidentalKey: key('jwt') }); + one.findings.push({ id: 'canary', evidence: 'supported', claimedTested: false, judgment: 'instrumentation' }, { id: 'guess', evidence: 'hypothesis', claimedTested: false, judgment: 'incidental', incidentalKey: key('guess') }); + const two = syntheticResult(negative); + two.findings.push({ id: 'admin', evidence: 'supported', claimedTested: false, judgment: 'incidental', incidentalKey: key('admin') }, { id: 'probe', evidence: 'supported', claimedTested: false, judgment: 'instrumentation' }, { id: 'noise', evidence: 'supported', claimedTested: false, judgment: 'incorrect' }); + expect(() => scoreEval(matrix, [one])).toThrow('INVALID_FINDING_JUDGMENT'); + one.findings.pop(); + expect(group(scoreEval(matrix, [one, two]), 'v3', 'daily').precision).toEqual({ numerator: 3, denominator: 5, value: 3 / 5 }); + const missed = syntheticResult(positive); missed.findings = [{ id: 'only-incidental', evidence: 'supported', claimedTested: false, judgment: 'incidental', incidentalKey: key('x') }]; + expect(group(scoreEval(matrix, [missed]), 'v3', 'daily').recall.numerator).toBe(0); + for (const bad of [{ judgment: 'incidental' }, { judgment: 'incidental', incidentalKey: 'short' }, { judgment: 'incorrect', incidentalKey: key('y') }, { judgment: 'instrumentation', matchedCaseId: positive.caseId }, { judgment: 'incidental', incidentalKey: key('z'), matchedCaseId: positive.caseId }]) { + const result = syntheticResult(positive); result.findings.push({ id: 'bad', evidence: 'supported', claimedTested: false, ...bad } as EvalFinding); + expect(() => scoreEval(matrix, [result])).toThrow('INVALID_FINDING_JUDGMENT'); + } + }); test('producer tested claims require independent successful held-out repair evidence', () => { const cell = matrix.cells.find(item => item.version === 'v3' && item.mode === 'comprehensive' && item.variant === 'vulnerable')!; const result = syntheticResult(cell); delete result.oracleEvidenceHash;