Files
gstack/test/cso-verifier.test.ts
T
Garry Tan 857466ff8b v1.91.25.0 feat(cso): private two-architecture release scoring + self-checking runtime release gates (#3036)
* v1.91.23.0 feat(cso): private two-architecture release scoring and self-checking runtime release gates

* Keep the Linux-only native-evidence and requalification tests out of the Windows lane

* Skip the two Linux-release-only checks on Windows in-file instead of growing the shard list

* fix(cso): opaque producer inputs and receipts; requalification triggers independent of cwd

Producer inputs are {schemaVersion: 2, cellRef, skill, source, execution} and
receipts bind cellRef + inputHash; collection rebuilds the cell from the
matrix, so a cell VM never learns its case or variant. requalificationTriggers
resolves Bun's cwd-relative metafile inputs against the repository root.

* test(evals): bound the review judge's thinking; guarded browser cleanup is not a probe

review/SKILL.md workflow: the ~64k-token bundle let default-effort thinking
spend all 8192 output tokens without an answer in CI. Medium effort with a
64k streaming cap kept 24/24 measured samples under 5k tokens and 63 s with
the same gating scores (as ship already does).

qa-only-no-fix: the fixture prompt routes browser cleanup through the deadline
guard, but the checkpoint detector treated every guarded run as a probe that
needs a checkpoint. A child that only closes the tab with the supplied browse
binary, after the last probe, is cleanup; anything else stays a probe. The
captured CI transcript (run 37252022510) now passes and new scenarios pin
the negative controls.

* test(evals): keep the review judge at default effort; only lift its output cap

Truncation was the defect: thinking and the answer share max_tokens, and the
~64k-token review bundle once spent all 8192. The registration keeps default
effort and response format and gets ship's 64k streaming cap; the guard test
pins exactly that.
2026-10-04 21:47:21 -07:00

77 lines
4.2 KiB
TypeScript

import { describe, expect, test } from 'bun:test';
import type { HttpAssertion } from '../lib/cso/contracts';
import { boundedResponseBody, judgeSecurity } from '../lib/cso/verifier';
describe('CSO bounded loopback verifier response reader',()=>{
test('accepts a response exactly at the configured byte limit',async()=>{
const body='x'.repeat(65_536);
expect(await boundedResponseBody(new Response(body))).toBe(body);
});
test('cancels an endless chunked response immediately after the byte limit',async()=>{
let pulls=0,cancelled=false;
const stream=new ReadableStream<Uint8Array>({
pull(controller){pulls++;controller.enqueue(new Uint8Array(8192));},
cancel(){cancelled=true;},
});
await expect(boundedResponseBody(new Response(stream))).rejects.toThrow('response too large');
expect(cancelled).toBe(true);
expect(pulls).toBeLessThanOrEqual(10);
});
test('rejects an oversized declared body before consuming it',async()=>{
let cancelled=false;
const stream=new ReadableStream<Uint8Array>({pull(controller){controller.enqueue(new Uint8Array(1));},cancel(){cancelled=true;}});
const response=new Response(stream,{headers:{'content-length':'65537'}});
await expect(boundedResponseBody(response)).rejects.toThrow('response too large');
expect(cancelled).toBe(true);
});
});
describe('CSO verifier security judgement',()=>{
const payloads:HttpAssertion[]=[
{name:'id parameter',path:'/user?id=2',method:'GET',expected:{status:403},vulnerable:{status:200,includes:'tenant-b'}},
{name:'tenant header',path:'/user',method:'GET',headers:{'x-tenant':'b'},expected:{status:403},vulnerable:{status:200,includes:'tenant-b'}},
{name:'write path',path:'/user/2',method:'PUT',body:'{}',expected:{status:404},vulnerable:{status:204}},
];
const serve=(fixed:Set<string>)=>{
const seen:string[]=[];
const server=Bun.serve({port:0,hostname:'127.0.0.1',fetch(req){
const url=new URL(req.url),key=`${req.method} ${url.pathname}${url.search}`;seen.push(key);
if(req.method==='PUT')return new Response(null,{status:fixed.has(key)?404:204});
return fixed.has(key)?new Response('denied',{status:403}):new Response('tenant-b secret',{status:200});
}});
return {server,seen,port:Number(server.port)};
};
const ALL=new Set(['GET /user?id=2','GET /user','PUT /user/2']);
test('one booted server answers every array member and the before phase needs every vulnerable expectation',async()=>{
const {server,seen,port}=serve(new Set());
try{
expect(await judgeSecurity('before',payloads,port)).toEqual({security:'intended_failure',members:['intended_failure','intended_failure','intended_failure']});
expect(seen).toEqual(['GET /user?id=2','GET /user','PUT /user/2']);
}finally{server.stop(true);}
});
test('the after phase passes only when every array member passes its fixed expectation',async()=>{
const fixed=serve(ALL);
try{expect(await judgeSecurity('after',payloads,fixed.port)).toEqual({security:'pass',members:['pass','pass','pass']});}finally{fixed.server.stop(true);}
const partial=serve(new Set(['GET /user?id=2','PUT /user/2']));
try{
expect(await judgeSecurity('after',payloads,partial.port)).toEqual({security:'inconclusive',members:['pass','inconclusive','pass']});
expect(await judgeSecurity('before',payloads,partial.port)).toEqual({security:'inconclusive',members:['pass','intended_failure','pass']});
}finally{partial.server.stop(true);}
const disproved=serve(ALL);
try{expect((await judgeSecurity('before',payloads,disproved.port)).security).toBe('pass');}finally{disproved.server.stop(true);}
});
test('a single assertion keeps its original before and after outcomes',async()=>{
const vulnerable=serve(new Set()),fixed=serve(ALL);
try{
expect(await judgeSecurity('before',payloads[0],vulnerable.port)).toEqual({security:'intended_failure',members:['intended_failure']});
expect(await judgeSecurity('after',payloads[0],vulnerable.port)).toEqual({security:'inconclusive',members:['inconclusive']});
expect(await judgeSecurity('after',payloads[0],fixed.port)).toEqual({security:'pass',members:['pass']});
}finally{vulnerable.server.stop(true);fixed.server.stop(true);}
});
});