Files
gstack/lib/cso/verifier.ts
T
Garry Tan 857466ff8b v1.91.25.0 feat(cso): private two-architecture release scoring + self-checking runtime release gates (#3036)
* v1.91.23.0 feat(cso): private two-architecture release scoring and self-checking runtime release gates

* Keep the Linux-only native-evidence and requalification tests out of the Windows lane

* Skip the two Linux-release-only checks on Windows in-file instead of growing the shard list

* fix(cso): opaque producer inputs and receipts; requalification triggers independent of cwd

Producer inputs are {schemaVersion: 2, cellRef, skill, source, execution} and
receipts bind cellRef + inputHash; collection rebuilds the cell from the
matrix, so a cell VM never learns its case or variant. requalificationTriggers
resolves Bun's cwd-relative metafile inputs against the repository root.

* test(evals): bound the review judge's thinking; guarded browser cleanup is not a probe

review/SKILL.md workflow: the ~64k-token bundle let default-effort thinking
spend all 8192 output tokens without an answer in CI. Medium effort with a
64k streaming cap kept 24/24 measured samples under 5k tokens and 63 s with
the same gating scores (as ship already does).

qa-only-no-fix: the fixture prompt routes browser cleanup through the deadline
guard, but the checkpoint detector treated every guarded run as a probe that
needs a checkpoint. A child that only closes the tab with the supplied browse
binary, after the last probe, is cleanup; anything else stays a probe. The
captured CI transcript (run 37252022510) now passes and new scenarios pin
the negative controls.

* test(evals): keep the review judge at default effort; only lift its output cap

Truncation was the defect: thinking and the answer share max_tokens, and the
~64k-token review bundle once spent all 8192. The registration keeps default
effort and response format and gets ship's 64k streaming cap; the guard test
pins exactly that.
2026-10-04 21:47:21 -07:00

189 lines
5.8 KiB
TypeScript

#!/usr/bin/env bun
import * as fs from 'node:fs';
import { connect } from 'node:net';
import {
HttpAssertion,
MAX_SECURITY_ASSERTIONS,
VerificationObservation,
VerificationRequest,
object,
securityAssertions,
} from './contracts';
interface Config {
phase: 'before' | 'after';
port: number;
legitimate: HttpAssertion[];
security: VerificationRequest['security'];
}
function matches(status: number, body: string, oracle: HttpAssertion['expected']): boolean {
return (
status === oracle.status &&
(oracle.includes === undefined || body.includes(oracle.includes)) &&
(oracle.excludes === undefined || !body.includes(oracle.excludes))
);
}
export async function boundedResponseBody(response: Response, limit = 65536): Promise<string> {
if (!Number.isSafeInteger(limit) || limit < 1) throw new Error('invalid response limit');
const declared = response.headers.get('content-length');
if (declared !== null && (/^\d+$/.test(declared) ? Number(declared) > limit : true)) {
await response.body?.cancel();
throw new Error('response too large');
}
if (!response.body) return '';
const reader = response.body.getReader(),
chunks: Uint8Array[] = [];
let total = 0;
try {
for (;;) {
const next = await reader.read();
if (next.done) break;
if (!next.value) continue;
total += next.value.byteLength;
if (total > limit) {
await reader.cancel();
throw new Error('response too large');
}
chunks.push(next.value);
}
} finally {
reader.releaseLock();
}
const bytes = new Uint8Array(total);
let offset = 0;
for (const chunk of chunks) {
bytes.set(chunk, offset);
offset += chunk.byteLength;
}
return new TextDecoder().decode(bytes);
}
async function request(a: HttpAssertion, port: number): Promise<{ status: number; body: string }> {
const controller = new AbortController(),
timer = setTimeout(() => controller.abort(), 5000);
try {
const response = await fetch(`http://127.0.0.1:${port}${a.path}`, {
method: a.method,
headers: a.headers,
body: ['GET'].includes(a.method) ? undefined : a.body,
redirect: 'manual',
signal: controller.signal,
});
return { status: response.status, body: await boundedResponseBody(response) };
} finally {
clearTimeout(timer);
}
}
export async function judgeSecurity(
phase: Config['phase'],
security: Config['security'],
port: number,
): Promise<{
security: VerificationObservation['security'];
members: VerificationObservation['security'][];
}> {
const members: VerificationObservation['security'][] = [];
for (const assertion of securityAssertions(security)) {
const r = await request(assertion, port),
fixed = matches(r.status, r.body, assertion.expected),
vulnerable = matches(r.status, r.body, assertion.vulnerable!);
members.push(
phase === 'before'
? vulnerable && !fixed
? 'intended_failure'
: fixed && !vulnerable
? 'pass'
: 'inconclusive'
: fixed && !vulnerable
? 'pass'
: 'inconclusive',
);
}
const agreed = members.every((member) => member === members[0]) ? members[0] : 'inconclusive';
return { security: agreed, members };
}
async function ready(port: number): Promise<boolean> {
return await new Promise((resolve) => {
const socket = connect({ host: '127.0.0.1', port }),
done = (value: boolean) => {
socket.removeAllListeners();
socket.destroy();
resolve(value);
},
timer = setTimeout(() => done(false), 500);
socket.once('connect', () => {
clearTimeout(timer);
done(true);
});
socket.once('error', () => {
clearTimeout(timer);
done(false);
});
});
}
async function main() {
const file = process.argv[2];
if (!file || !file.startsWith('/policy/')) throw new Error('trusted policy path required');
const raw = fs.readFileSync(file, 'utf8');
if (Buffer.byteLength(raw) > 1024 * 1024) throw new Error('policy too large');
const v = object(JSON.parse(raw), 'verifier policy') as any;
if (
!['before', 'after'].includes(v.phase) ||
!Number.isInteger(v.port) ||
v.port < 1024 ||
v.port > 65535 ||
!Array.isArray(v.legitimate) ||
!v.security ||
(Array.isArray(v.security) && (!v.security.length || v.security.length > MAX_SECURITY_ASSERTIONS))
)
throw new Error('invalid verifier policy');
const config = v as Config;
let booted = false;
for (let attempt = 0; attempt < 60; attempt++) {
if (await ready(config.port)) {
booted = true;
break;
}
await Bun.sleep(250);
}
let legitimate = false,
security: VerificationObservation['security'] = 'inconclusive',
summary = 'application did not answer a legitimate control';
if (booted) {
try {
legitimate = (
await Promise.all(
config.legitimate.map(async (a) => {
const r = await request(a, config.port);
return matches(r.status, r.body, a.expected);
}),
)
).every(Boolean);
const judged = await judgeSecurity(config.phase, config.security, config.port);
security = judged.security;
summary = `boot=true legitimate=${legitimate} security=${security}${
Array.isArray(config.security) ? ` members=${judged.members.join(',')}` : ''
}`;
} catch {
summary = 'bounded verifier request failed';
}
}
process.stdout.write(
JSON.stringify({ booted, legitimate, security, existingTests: false, output: summary, inputHash: '' }) +
'\n',
);
}
if (import.meta.main)
main().catch(() => {
process.stdout.write(
JSON.stringify({
booted: false,
legitimate: false,
security: 'inconclusive',
existingTests: false,
output: 'verifier setup failed',
inputHash: '',
}) + '\n',
);
process.exitCode = 1;
});