CalvinBackup/gstack
1
0 Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-11-13 03:48:06 +01:00
Code Issues Packages Projects Releases Wiki Activity

v1.91.23.0 feat(cso): private two-architecture release scoring and self-checking runtime release gates

Browse Source
This commit is contained in:
garrytan committed 2026-10-05 01:07:53 +00:00
1 parent f97aefe654
commit 9b21621462
35 files changed
+2284 -235
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

test/cso-contracts.test.ts
+45 -1
View File
@@ -4,7 +4,7 @@ import * as os from 'node:os';
import * as path from 'node:path';
import * as crypto from 'node:crypto';
import { canonical, completeness, CsoError, fingerprint, importLegacy, renderReport, sha256, snapshotPathHandle, snapshotPathId, validateCoverage, validateFinding, validateVerificationObservation, validateVerificationRequest } from '../lib/cso/contracts';
import { assertCanonicalStartPlan, canonicalStartPlan, canonicalTestPlan, certify, fileEffect, makeReviewArtifact, patchHash, preparePatchedSource, resolveVerificationRequestPaths, testExecutionPassed, treeHash, validateRepairBundle, validateReviewArtifact, verificationIdentity, verifyRepair } from '../lib/cso/verification';
import { assertCanonicalStartPlan, canonicalStartPlan, canonicalTestPlan, certify, fileEffect, makeReviewArtifact, patchHash, preparePatchedSource, resolveVerificationRequestPaths, testExecutionPassed, treeHash, validateRepairBundle, validateReviewArtifact, verificationHarnessHash, verificationIdentity, verifyRepair } from '../lib/cso/verification';
import { assertionWitnessPairHash, witnessObservationHash } from '../lib/cso/witness';
import { admit, machinePoolRoot, markSupervised, release } from '../lib/cso/admission';
import { sanitizeHelperForJson } from '../lib/cso/process';
@@ -211,6 +211,50 @@ describe('tested repair certificate gate',()=>{
const drift=structuredClone(bundle);drift.preparation!.after.preparedDependencyHash=h('b');drift.verification.preparationHash=sha256(canonical(drift.preparation));drift.verification.id='';const rebound=verificationIdentity(drift.verification);drift.verification.id=rebound;drift.id=rebound;expect(()=>validateRepairBundle(drift,rebound,before)).toThrow('test toolchain changed');
expect(()=>certify({...params,preparation:undefined})).toThrow('require before/after prepared dependency proofs');expect(()=>certify({...params,preparation:{before:proof(h('9')),after:proof(h('b'))}})).toThrow('toolchain bytes changed');
});
const securityArray=()=>[
{name:'tenant isolation',path:'/user?id=2',method:'GET',expected:{status:403},vulnerable:{status:200,includes:'tenant-b'}},
{name:'tenant isolation by header',path:'/user',method:'GET',headers:{'x-tenant':'b'},expected:{status:403},vulnerable:{status:200,includes:'tenant-b'}},
{name:'tenant isolation on write',path:'/user/2',method:'PUT',body:'{"name":"x"}',expected:{status:404},vulnerable:{status:204}},
];
test('single-object security keeps its pre-array request, patch, assertion, and harness bytes',()=>{
const root=tmp();fs.writeFileSync(path.join(root,'test.js'),'pass\n');fs.chmodSync(path.join(root,'test.js'),0o644);
const raw:any=request('vulnerable\n','fixed\n');raw.review.reviewedPatchHash=patchHash(raw);const req=validateVerificationRequest(raw);
expect(Array.isArray(req.security)).toBe(false);expect(canonical(req.security)).toBe(canonical(raw.security));
expect({patch:patchHash(req),request:sha256(canonical(req)),assertion:sha256(canonical({legitimate:req.legitimate,security:req.security})),harness:verificationHarnessHash(req,root)}).toEqual({
patch:'cc6ab00ee0160ce484a86130f94a5e6e5f588f3ec975c781c4d077647adff483',
request:'85175d755f1b297eb7d15dc1d9850e1ac453d1d129ceee58213d89cbab0ee59c',
assertion:'7e0f792881bc14137f2aa70b0662a013939749903e49eb507c1ecf48561b0771',
harness:'2a74b61982270fd10c071f0bdd32ae37f04de6e3eca1db6c9225132de29b4309',
});
});
test('security accepts a bounded array of distinct assertions that each carry an exclusive vulnerable oracle',()=>{
const make=(security:unknown)=>{const raw:any=request('vulnerable\n','fixed\n');raw.security=security;raw.review.reviewedPatchHash=patchHash(raw);return raw;};
const accepted=validateVerificationRequest(make(securityArray()));expect(accepted.security).toEqual(securityArray() as any);
expect(validateVerificationRequest(make(securityArray().slice(0,1))).security).toHaveLength(1);
const full=Array.from({length:8},(_,i)=>({...securityArray()[0],name:`payload ${i}`}));expect(validateVerificationRequest(make(full)).security).toHaveLength(8);
expect(()=>validateVerificationRequest(make([]))).toThrow('1..8 assertions');
expect(()=>validateVerificationRequest(make([...full,{...full[0],name:'payload 8'}]))).toThrow('1..8 assertions');
expect(()=>validateVerificationRequest(make([securityArray()[0],{...securityArray()[1],name:'tenant isolation'}]))).toThrow('names must be distinct');
const missing=securityArray();delete (missing[2] as any).vulnerable;expect(()=>validateVerificationRequest(make(missing))).toThrow('distinct before/fixed security oracles');
const overlapping=securityArray();overlapping[1].vulnerable={...overlapping[1].expected} as any;expect(()=>validateVerificationRequest(make(overlapping))).toThrow('mutually exclusive');
const extra:any=securityArray();extra[0].expected.untrusted='yes';expect(()=>validateVerificationRequest(make(extra))).toThrow('Unexpected security[0].expected field');
expect(()=>validateVerificationRequest(make([null]))).toThrow();
});
test('an array security request is observed in exactly one before/after pair and binds every member',async()=>{
const runDir=tmp(),snapshot=path.join(runDir,'snapshot');fs.mkdirSync(snapshot);fs.writeFileSync(path.join(snapshot,'app.js'),'vulnerable\n');const testBody='test("ok",()=>{})\n';fs.writeFileSync(path.join(snapshot,'app.test.js'),testBody);const packageJson='{"scripts":{"test":"node --test"}}';fs.writeFileSync(path.join(snapshot,'package.json'),packageJson);
const raw:any=request('vulnerable\n','fixed\n');raw.findingId='1abcdefabcdefabcdefabcdefabcdefa';raw.security=securityArray();raw.existingTests=[{executable:'/usr/local/bin/node',args:['--test','--test-reporter=tap','./app.test.js']}];raw.testFiles=['app.test.js'];raw.review.reviewedPatchHash=patchHash(raw);
const manifest:any={version:3,createdAt:'2026-01-01T00:00:00Z',expiresAt:'2026-01-08T00:00:00Z',root:'/repo',headCommit:'a'.repeat(40),originalHash:'b'.repeat(64),executionHash:treeHash(snapshot),entries:[{path:'app.js',originalHash:sha256('vulnerable\n'),executionHash:sha256('vulnerable\n'),bytes:11,mode:0o600},{path:'app.test.js',originalHash:sha256(testBody),executionHash:sha256(testBody),bytes:Buffer.byteLength(testBody),mode:0o600},{path:'package.json',originalHash:sha256(packageJson),executionHash:sha256(packageJson),bytes:packageJson.length,mode:0o600}]};
const runtime:any={id:'node',stack:'node',image:'runtime@sha256:'+'a'.repeat(64),platform:'linux/amd64'},phases:string[]=[];
const executor:any={observe:async(_source:string,phase:string,observed:any)=>{phases.push(phase);expect(observed.security).toEqual(securityArray());return{booted:true,legitimate:true,security:phase==='before'?'intended_failure':'pass',existingTests:true,output:'ok',inputHash:''};}};
let unattested:any;try{await verifyRepair({runId:'run',runDir,manifest,rawRequest:raw,runtime,verifier:runtime,policyHash:'c'.repeat(64),archives:[],executor,persist:false});}catch(error){unattested=error;}
expect(phases).toEqual(['before','after']);expect(unattested).toMatchObject({code:'PREREQUISITE',attempt:{reproduction:'reproduced',repair:'proposed',bundleIssued:false}});
const root=tmp(),before=path.join(root,'before'),after=path.join(root,'after');fs.mkdirSync(before);fs.writeFileSync(path.join(before,'app.js'),'vulnerable\n');fs.writeFileSync(path.join(before,'test.js'),'pass\n');
const plain:any=request('vulnerable\n','fixed\n');plain.security=securityArray();plain.review.reviewedPatchHash=patchHash(plain);const req=validateVerificationRequest(plain);preparePatchedSource(before,after,req);
const params:any={runId:'run',manifest:{originalHash:'d'.repeat(64),executionHash:treeHash(before),entries:[]},request:req,runtime:{id:'node',image:'runtime@sha256:'+'a'.repeat(64),platform:'linux/amd64'},verifier:{image:'runtime@sha256:'+'a'.repeat(64)},before:{booted:true,legitimate:true,security:'intended_failure',existingTests:true,output:'before',inputHash:''},after:{booted:true,legitimate:true,security:'pass',existingTests:true,output:'after',inputHash:''},beforeRoot:before,afterRoot:after,policyHash:'c'.repeat(64),auditPolicyHash:'e'.repeat(64),archives:[],testToolchain:'runtime'};
const bundle=authenticatedBundleFixture(certify(params),params);expect(validateRepairBundle(bundle,bundle.id,before).id).toBe(bundle.id);
const dropped=structuredClone(bundle);(dropped.request.security as any[]).pop();expect(()=>validateRepairBundle(dropped,dropped.id,before)).toThrow('harness');
expect(certify({...params,after:{...params.after,security:'inconclusive'}}).manifest.result).toBe('inconclusive');
});
});
function tree(root:string){return sha256(JSON.stringify(fs.readdirSync(root).sort().map(p=>{const file=path.join(root,p),stat=fs.statSync(file);return[p,sha256(fs.readFileSync(file)),stat.mode&0o777];})));}
test/cso-distribution.test.ts
+248 -12
View File
@@ -1,10 +1,17 @@
import { afterEach, describe, expect, test } from 'bun:test';
import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { createHash } from 'node:crypto';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { dirname, join, resolve } from 'node:path';
import { spawnSync } from 'node:child_process';
import { imageBuildMatrix } from '../scripts/cso-image-matrix';
import committedCatalog from '../lib/cso/runtime-catalog.json';
import buildInputs from '../lib/cso/images/build-inputs.json';
import { committedImageBuildMatrix, imageBuildMatrix } from '../scripts/cso-image-matrix';
import { catalogPromotionCandidate, requiredChecks } from '../scripts/cso-runtime-promotion';
import { runBashScript } from './helpers/bash-script';
import { requalificationTriggers } from '../scripts/cso-requalification';
const CURRENT_TRIGGERS = await requalificationTriggers();
const ROOT = resolve(import.meta.dir, '..');
const temps: string[] = [];
@@ -250,12 +257,75 @@ describe('CSO build and distribution wiring', () => {
});
});
const EVALUATION_REF = `sha256:${'e'.repeat(64)}`;
const STAGING_RUN = '777';
function dispatchedStatements(): any[] {
return committedImageBuildMatrix().include.map(row => ({
schemaVersion: 1, helperAbi: 3, state: 'qualified', buildRevision: row.inputRevision, runtimeId: row.runtimeId,
stack: row.stack, platform: row.platform, versions: row.versions, sourceCommit: 'a'.repeat(40),
image: `ghcr.io/garrytan/gstack/cso-staging/${row.stack}-${row.arch}@sha256:${Bun.hash(row.runtimeId).toString(16).padStart(64, '0')}`,
checks: Object.fromEntries(requiredChecks(row.stack).map(key => [key, true])),
evaluationRef: EVALUATION_REF,
}));
}
function ingressStep(name: string): string {
const job = (Bun.YAML.parse(readFileSync(join(ROOT, '.github/workflows/cso-runtime-qualification.yml'), 'utf8')) as any).jobs.qualify;
return (job.steps.find((step: any) => step.name === name).run as string).replaceAll('bun run scripts/', `bun run ${quote(ROOT)}/scripts/`);
}
function ingressFixture() {
const dir = mkdtempSync(join(tmpdir(), 'cso-ingress-')); temps.push(dir);
const bin = join(dir, 'bin'), work = join(dir, 'work'), fake = join(dir, 'gh-fixtures');
for (const path of [bin, work, fake]) mkdirSync(path);
writeFileSync(join(bin, 'gh'), [
'#!/bin/sh',
'set -eu',
'case "$1 $2" in',
' "attestation verify")',
' case "$*" in',
' *https://slsa.dev/provenance/v1*) jq --arg image "$3" \'.[0].verificationResult.statement.subject[0].name = $image\' "$FAKE_GH/provenance.json" ;;',
' *) jq -n --arg image "$3" \'[{verificationResult:{statement:{predicateType:"https://spdx.dev/Document/v2.3",subject:[{name:$image}]}}}]\' ;;',
' esac ;;',
' "api --paginate") cat "$FAKE_GH/jobs.json" ;;',
' "run download") while [ "$#" -gt 0 ]; do [ "$1" != --dir ] || dir=$2; shift; done; mkdir -p "$dir"; cp "$FAKE_GH/requalification.json" "$dir/" ;;',
' "api repos/"*) cat "$FAKE_GH/run.json" ;;',
' *) echo "unexpected gh $*" >&2; exit 2 ;;',
'esac',
'',
].join('\n'));
chmodSync(join(bin, 'gh'), 0o755);
const invocation = `https://github.com/garrytan/gstack/actions/runs/${STAGING_RUN}/attempts/1`;
writeFileSync(join(fake, 'provenance.json'), JSON.stringify([{ verificationResult: {
signature: { certificate: { runInvocationURI: invocation } },
statement: { predicateType: 'https://slsa.dev/provenance/v1', subject: [{ name: '' }], predicate: { runDetails: { metadata: { invocationId: invocation } } } },
} }]));
const run = (overrides: Record<string, unknown> = {}) => writeFileSync(join(fake, 'run.json'), JSON.stringify({
id: Number(STAGING_RUN), path: '.github/workflows/cso-runtime-images.yml', event: 'workflow_dispatch', head_branch: 'main',
head_sha: 'a'.repeat(40), status: 'completed', conclusion: 'success',
repository: { full_name: 'garrytan/gstack' }, head_repository: { full_name: 'garrytan/gstack' }, ...overrides,
}));
const jobs = (conclusion = 'success') => {
const list = committedImageBuildMatrix().include.map((row, index) => ({
name: `qualify-native ${row.runtimeId}`, run_id: Number(STAGING_RUN), status: 'completed', conclusion: index === 0 ? conclusion : 'success',
}));
writeFileSync(join(fake, 'jobs.json'), JSON.stringify([{ total_count: list.length, jobs: list }]));
};
run(); jobs();
writeFileSync(join(fake, 'requalification.json'), JSON.stringify(CURRENT_TRIGGERS));
const event = join(dir, 'event.json');
writeFileSync(event, JSON.stringify({ action: 'cso-runtime-qualified', client_payload: { evaluationRef: EVALUATION_REF, statements: dispatchedStatements() } }));
const env = { ...process.env, PATH: `${bin}:${dirname(process.execPath)}:${process.env.PATH}`, FAKE_GH: fake, GITHUB_EVENT_PATH: event,
GITHUB_REPOSITORY: 'garrytan/gstack', GITHUB_RUN_ID: '555' };
const step = (name: string) => runBashScript(ingressStep(name), { cwd: work, env, timeout: 60_000 });
return { work, run, jobs, step };
}
describe('CSO runtime staging gates', () => {
test('runtime workflow shell blocks parse after GitHub expressions are substituted', () => {
for (const relative of [
'.github/workflows/cso-runtime-images.yml',
'.github/workflows/cso-runtime-qualification.yml',
'.github/workflows/cso-runtime-promote.yml',
'.github/workflows/cso-scanner-images.yml',
]) {
const raw = readFileSync(join(ROOT, relative), 'utf8');
const workflow = Bun.YAML.parse(raw) as any;
@@ -384,13 +454,15 @@ describe('CSO runtime staging gates', () => {
expect(workflow.jobs['qualify-native'].needs).toEqual(['reviewed-inputs', 'stage']);
expect(raw).toContain('cso-staged-postgresql-${{ matrix.arch }}');
expect(raw).toContain('GSTACK_CSO_TEST_POSTGRES_IMAGE');
expect(raw).toContain('acquisitionPublicOnlyPassed:true');
expect(raw).toContain('positiveNegativeAssertionsPassed:true');
expect(raw).toContain('heldOutRepairPassed:"pending"');
expect(raw).toContain('railsSqlitePassed:true');
expect(raw).toContain('railsPostgresqlPassed:true');
expect(raw).toContain('nativeExtensionsPassed:true');
expect(raw).toContain('qualified:false');
const qualify = workflow.jobs['qualify-native'];
expect(qualify.name).toBe('qualify-native ${{ matrix.runtimeId }}');
expect(raw).not.toMatch(/Passed:\s*true/);
const tested = qualify.steps.findIndex((s: any) => s.run?.includes('bun run test:cso:docker --reporter=junit --reporter-outfile=native-tests.junit.xml'));
const derived = qualify.steps.findIndex((s: any) => s.run?.includes('scripts/cso-native-evidence.ts --stack "$CSO_STACK" --platform "$CSO_PLATFORM"'));
expect(tested).toBeGreaterThanOrEqual(0);
expect(derived).toBe(tested + 1);
expect(qualify.steps[derived].run).toContain('--junit native-tests.junit.xml --staged staged-application/staged-image.json');
expect(qualify.steps[derived + 1].with.path.split('\n').filter(Boolean)).toEqual(['qualification-evidence.json', 'native-tests.junit.xml']);
expect(raw).not.toContain('setup-qemu');
expect(raw).not.toContain('git push');
expect(raw).not.toContain('gh pr create');
@@ -412,7 +484,7 @@ describe('CSO runtime staging gates', () => {
expect(Object.keys(workflow.on)).toEqual(['workflow_dispatch']);
expect(job.if).toContain("github.ref == 'refs/heads/main'");
expect(job.environment).toBe('cso-runtime-release');
expect(job.permissions).toMatchObject({ contents: 'write', 'pull-requests': 'write', actions: 'read', packages: 'read', 'id-token': 'write', attestations: 'write' });
expect(job.permissions).toMatchObject({ contents: 'write', 'pull-requests': 'write', actions: 'write', packages: 'read', 'id-token': 'write', attestations: 'write' });
expect(raw).toContain('.head_branch == "main"');
expect(raw).toContain('.path == ".github/workflows/cso-runtime-qualification.yml"');
expect(raw).toContain('.event == "repository_dispatch"');
@@ -444,15 +516,51 @@ describe('CSO runtime staging gates', () => {
for (const step of job.steps) if (step.uses) expect(step.uses).toMatch(/@[a-f0-9]{40}$/);
});
test('catalog promotion PRs dispatch the required free-tests check on the pushed branch', () => {
const freeTests = Bun.YAML.parse(readFileSync(join(ROOT, '.github/workflows/free-tests.yml'), 'utf8')) as any;
expect(Object.keys(freeTests.on)).toContain('workflow_dispatch');
expect(freeTests.jobs['free-tests'].if).toBe('always()');
expect(freeTests.jobs['free-tests'].name).toBeUndefined();
for (const [relative, jobName] of [
['.github/workflows/cso-runtime-promote.yml', 'propose'],
['.github/workflows/cso-scanner-images.yml', 'promote-catalog'],
]) {
const job = (Bun.YAML.parse(readFileSync(join(ROOT, relative), 'utf8')) as any).jobs[jobName];
expect(job.permissions.actions, relative).toBe('write');
const script = job.steps.find((step: any) => step.run?.includes('gh pr create')).run as string;
const pushed = script.indexOf('git push --set-upstream origin "$branch"');
const dispatched = script.indexOf('gh workflow run free-tests.yml --repo "$GITHUB_REPOSITORY" --ref "$branch"');
expect(pushed, relative).toBeGreaterThan(0);
expect(dispatched, relative).toBeGreaterThan(pushed);
expect(script.indexOf('gh pr create'), relative).toBeGreaterThan(dispatched);
}
});
test('private qualification enters only through an actor-restricted protected environment and re-verifies staged attestations', () => {
const raw = readFileSync(join(ROOT, '.github/workflows/cso-runtime-qualification.yml'), 'utf8');
const workflow = Bun.YAML.parse(raw) as any;
const job = workflow.jobs.qualify;
expect(Object.keys(workflow.on)).toEqual(['repository_dispatch']);
expect(job.environment).toBe('cso-runtime-release');
expect(job.permissions).toEqual({ contents: 'read', packages: 'read', attestations: 'read' });
expect(job.permissions).toEqual({ contents: 'read', packages: 'read', attestations: 'read', actions: 'read' });
expect(job.needs).toBe('dispatch-summary');
const summary = workflow.jobs['dispatch-summary'];
expect(summary.permissions).toEqual({});
expect(summary.environment).toBeUndefined();
expect(summary.steps.every((step: any) => !step.uses)).toBe(true);
expect(raw).toContain('test "$GITHUB_ACTOR" = "$CSO_QUALIFICATION_ACTOR"');
expect(raw).toContain('.client_payload.statements | type == "array" and length == 10');
expect(raw).not.toContain('^ghcr.io/');
expect(raw).toContain('^ghcr\\\\.io/garrytan/gstack/cso-staging/');
const names = job.steps.map((step: any) => step.name);
const attested = names.indexOf('Verify each staged image attestation and normalize qualification statements');
const staging = names.indexOf('Require one successful protected-main staging run with every native gate passed');
const contract = names.indexOf('Enforce the complete release contract before retaining evidence');
expect(attested).toBeGreaterThan(0);
expect(staging).toBe(attested + 1);
expect(contract).toBe(staging + 1);
expect(job.steps[staging].run).toContain('gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/jobs?filter=latest&per_page=100"');
expect(job.steps.at(-1).with.path).toContain('staging-run-evidence');
expect(raw).toContain('--cert-identity "$signer"');
expect(raw).toContain('--source-digest "$source_commit"');
expect(raw).toContain('--deny-self-hosted-runners');
@@ -489,4 +597,132 @@ describe('CSO runtime staging gates', () => {
expect(qualify.steps[loaded].run).toContain('scripts/cso-public-ghcr.ts verify');
expect(qualify.steps[loaded].env.GH_TOKEN).toBe('${{ github.token }}');
});
test('the ingress payload gate accepts an optional evaluation reference and summarizes it before approval', () => {
const script = (Bun.YAML.parse(readFileSync(join(ROOT, '.github/workflows/cso-runtime-qualification.yml'), 'utf8')) as any).jobs['dispatch-summary'].steps[0].run;
const dir = mkdtempSync(join(tmpdir(), 'cso-dispatch-summary-')); temps.push(dir);
const event = join(dir, 'event.json'), summary = join(dir, 'summary.md');
const dispatch = (payload: unknown) => {
writeFileSync(event, JSON.stringify({ action: 'cso-runtime-qualified', client_payload: payload }));
writeFileSync(summary, '');
const result = runBashScript(script, { cwd: dir, env: { ...process.env, GITHUB_EVENT_PATH: event, GITHUB_STEP_SUMMARY: summary }, timeout: 30_000 });
return { status: result.status, summary: readFileSync(summary, 'utf8') };
};
const statements = dispatchedStatements();
expect(dispatch({ statements }).summary).toContain('- evaluationRef: `none`');
const accepted = dispatch({ evaluationRef: EVALUATION_REF, statements });
expect(accepted.status).toBe(0);
expect(accepted.summary).toContain(`- evaluationRef: \`${EVALUATION_REF}\``);
expect(accepted.summary).toContain(`- sourceCommit: \`${'a'.repeat(40)}\``);
for (const statement of statements) expect(accepted.summary).toContain(`| \`${statement.runtimeId}\` | \`${statement.image}\` |`);
const rejected = (payload: unknown) => expect(dispatch(payload).status).not.toBe(0);
rejected({ evaluationRef: 'sha256:abc', statements });
rejected({ evaluationRef: EVALUATION_REF.toUpperCase(), statements });
rejected({ evaluationRef: null, statements });
rejected({ evaluationRef: EVALUATION_REF, statements, finding: 'held-out detail' });
rejected({ statements: statements.slice(1) });
rejected({ statements: statements.map((item, index) => index ? item : { ...item, sourceCommit: 'b'.repeat(40) }) });
rejected({ statements: statements.map((item, index) => index ? item : { ...item, image: item.image.replace('ghcr.io', 'ghcrxio') }) });
rejected({ statements: statements.map((item, index) => index ? item : { ...item, runtimeId: 'node`|injected' }) });
});
test('the protected ingress binds verified statements to one successful staging run and never carries the evaluation reference', () => {
const fixture = ingressFixture();
const normalized = fixture.step('Verify each staged image attestation and normalize qualification statements');
expect(normalized.status, normalized.stderr).toBe(0);
const staging = fixture.step('Require one successful protected-main staging run with every native gate passed');
expect(staging.status, staging.stderr).toBe(0);
expect(staging.stdout).toContain('STAGING RUN VERIFIED');
const contract = fixture.step('Enforce the complete release contract before retaining evidence');
expect(contract.status, contract.stderr).toBe(0);
const statements = committedImageBuildMatrix().include.map(row =>
JSON.parse(readFileSync(join(fixture.work, 'qualification-evidence', row.runtimeId, 'qualified-runtime.json'), 'utf8')));
for (const statement of statements) {
expect(statement).not.toHaveProperty('evaluationRef');
expect(statement.workflow).toBe('https://github.com/garrytan/gstack/actions/runs/555');
}
expect(JSON.parse(readFileSync(join(fixture.work, 'qualification-evidence', 'requalification.json'), 'utf8'))).toEqual(CURRENT_TRIGGERS);
const candidate = JSON.stringify(catalogPromotionCandidate(committedCatalog, buildInputs, statements, CURRENT_TRIGGERS));
expect(candidate).not.toContain('evaluationRef');
expect(candidate).not.toContain('e'.repeat(64));
for (const [mutate, error] of [
[() => fixture.run({ conclusion: 'failure' }), 'STAGING_RUN_NOT_SUCCESSFUL'],
[() => fixture.jobs('failure'), 'STAGING_NATIVE_GATE_NOT_PASSED'],
] as const) {
fixture.run(); fixture.jobs(); mutate();
rmSync(join(fixture.work, 'staging-run-evidence'), { recursive: true, force: true });
const rejected = fixture.step('Require one successful protected-main staging run with every native gate passed');
expect(rejected.status).not.toBe(0);
expect(rejected.stderr).toContain(error);
}
});
});
describe('CSO evaluation-only candidate builds', () => {
const revision = `cso-eval-123456-${'a'.repeat(12)}`;
const sha = (path: string) => createHash('sha256').update(readFileSync(path)).digest('hex');
function candidateFixture(catalogRevision = revision) {
const dir = buildFixture(), outside = mkdtempSync(join(tmpdir(), 'gstack-cso-eval-out-')); temps.push(outside);
mkdirSync(join(dir, 'test/helpers'), { recursive: true });
const catalog = join(outside, 'candidate.json');
writeFileSync(catalog, JSON.stringify({ schemaVersion: 1, revision: catalogRevision, runtimes: [] }, null, 2) + '\n');
return { dir, outside, catalog, output: join(outside, 'unit') };
}
const candidateBuild = (dir: string, catalog: string, output: string, extra = '') =>
runBashScript(`${fakeBuildCommand(dir, extra)} --evaluation-candidate ${quote(catalog)} --output ${quote(output)}`, { timeout: 20_000 });
test.skipIf(process.platform === 'win32')('builds a marked five-artifact producer unit outside the checkout without touching bin/', () => {
const { dir, outside, catalog, output } = candidateFixture();
const result = candidateBuild(dir, catalog, output);
expect(result.status).toBe(0);
expect(JSON.parse(result.stdout.trim())).toEqual({ output, revision, evaluationOnly: true });
expect(readdirSync(output).sort()).toEqual(['.gstack-cso-evaluation', '.gstack-cso-generation', 'cso-eval-producer', 'gstack-cso-core', 'gstack-cso-launcher', 'gstack-cso-watchdog']);
const marker = JSON.parse(readFileSync(join(output, '.gstack-cso-evaluation'), 'utf8'));
expect(marker).toEqual({ schemaVersion: 1, evaluationOnly: true, runtimeCatalogRevision: revision, runtimeCatalogSha256: sha(catalog), coreSha256: sha(join(output, 'gstack-cso-core')) });
expect(readFileSync(join(output, '.gstack-cso-generation'), 'utf8')).toBe(`${marker.coreSha256}\n`);
expect(readdirSync(join(dir, 'bin'))).toEqual([]);
expect(readdirSync(outside).sort()).toEqual(['candidate.json', 'unit']);
const args = readFileSync(join(dir, 'compiler.log'), 'utf8').trim().split('\n');
const check = args.indexOf('check-evaluation');
expect(check).toBeGreaterThan(0); expect(args[check + 1]).toBe('lib/cso/runtime-catalog.json');
expect(check).toBeLessThan(args.indexOf('lib/cso/cli.ts'));
expect(args).toContain('scripts/cso-eval-producer.ts');
expect(args.filter(arg => arg === '--no-compile-autoload-package-json')).toHaveLength(2);
});
test.skipIf(process.platform === 'win32')('refuses unmarked catalogs, in-checkout or existing outputs, and leaves nothing on a failed check', () => {
const plain = candidateFixture('cso-v3-123456-aaaaaaaaaaaa');
expect(candidateBuild(plain.dir, plain.catalog, plain.output).status).toBe(65);
const inside = candidateFixture();
const insideResult = candidateBuild(inside.dir, inside.catalog, join(inside.dir, 'unit'));
expect(insideResult.status).toBe(73); expect(existsSync(join(inside.dir, 'unit'))).toBe(false);
const existing = candidateFixture(); mkdirSync(existing.output);
expect(candidateBuild(existing.dir, existing.catalog, existing.output).status).toBe(73);
const failed = candidateFixture();
expect(candidateBuild(failed.dir, failed.catalog, failed.output, 'CSO_FAIL_COMPILER_N=1').status).toBe(42);
expect(readdirSync(failed.outside)).toEqual(['candidate.json']);
});
test.skipIf(process.platform === 'win32')('distribution builds refuse an evaluation catalog in the source tree', () => {
const dir = buildFixture();
writeFileSync(join(dir, 'lib/cso/runtime-catalog.json'), JSON.stringify({ revision }, null, 2));
const result = fakeBuild(dir);
expect(result.status).toBe(65); expect(result.stderr).toContain('evaluation-only catalog');
expect(existsSync(join(dir, 'compiler.log'))).toBe(false);
});
test('the candidate gate rejects the committed release catalog', () => {
const result = spawnSync(process.execPath, ['scripts/cso-runtime-promotion.ts', 'check-evaluation', 'lib/cso/runtime-catalog.json'], { cwd: ROOT, encoding: 'utf8', timeout: 30_000 });
expect(result.status).toBe(1); expect(result.stderr).toContain('NOT_AN_EVALUATION_CATALOG');
expect(String((committedCatalog as { revision: string }).revision).startsWith('cso-eval-')).toBe(false);
});
test('setup refuses a marked or self-reporting evaluation helper', () => {
const setup = readFileSync(join(ROOT, 'setup'), 'utf8');
expect(setup).toContain('[ -e "$SOURCE_GSTACK_DIR/bin/.gstack-cso-evaluation" ]');
expect(setup).toContain(`--version 2>/dev/null | grep -q '"evaluationOnly": *true'`);
expect(setup.indexOf('bin/ holds an evaluation-only CSO helper')).toBeLessThan(setup.indexOf('provision-images --setup-summary'));
});
});
test/cso-docker-integration.test.ts
+2 -3
View File
@@ -38,9 +38,8 @@ suite('CSO Docker containment integration',()=>{
expect(spawnSync('/usr/bin/docker',['--host',endpoint.uri,'inspect',client],{timeout:30_000}).status).not.toBe(0);
});
test('machine-wide admission allows only two groups per endpoint',()=>{const a=admit(endpoint.uri,'a',Date.now()+60_000),b=admit(endpoint.uri,'b',Date.now()+60_000);try{expect(()=>admit(endpoint.uri,'c',Date.now()+60_000)).toThrow('Two reproduction groups');}finally{release(a);release(b);}});
test('staged runtime executes its trusted verifier and checks every declared tool version', async () => {
const staged = process.env.GSTACK_CSO_TEST_IMAGE;
if (!staged) return;
test.skipIf(!process.env.GSTACK_CSO_TEST_IMAGE)('staged runtime executes its trusted verifier and checks every declared tool version', async () => {
const staged = process.env.GSTACK_CSO_TEST_IMAGE ?? '';
expect(staged).toMatch(/@sha256:[a-f0-9]{64}$/);
expect(process.env.GSTACK_CSO_TEST_PLATFORM).toBe(process.arch === 'arm64' ? 'linux/arm64' : 'linux/amd64');
const versions = JSON.parse(process.env.GSTACK_CSO_EXPECTED_VERSIONS || '{}');
test/cso-eval.test.ts
+157 -11
View File
@@ -4,11 +4,11 @@ import { chmodSync, cpSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readd
import { createHash } from 'node:crypto';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { collectProducerReceipts, createEvalMatrix, createPortableSkillPayload, loadPortableSkillPayload, prepareEvalJobs, REQUIRED_CONTAINMENT, scoreCollectedEval, scoreEval, validateMatrix, validatePortableSkillPayload, type EvalCell, type EvalResult, type PreparedEvalSchedule } from '../scripts/cso-eval';
import { collectProducerReceipts, createEvalBaseline, createEvalMatrix, createPortableSkillPayload, loadPortableSkillPayload, prepareEvalJobs, REQUIRED_CONTAINMENT, scoreCollectedEval, scoreEval, validateMatrix, validatePortableSkillPayload, type EvalBaseline, type EvalCell, type EvalMatrix, type EvalResult, type PreparedEvalSchedule } from '../scripts/cso-eval';
import { PRODUCER_PROVIDER_POLICY, producerFailureMessage, producerInstallationIdentity, resolveProducerHelperBinding, runProducerCell, validateProductionProducerInstallation } from '../scripts/cso-eval-producer';
import { producerArtifactInventoryHash, producerInstallationIdentityHash, producerProviderIdentityHash, producerReceiptHash, sha256, type ProducerArtifactInventory, type ProducerInstallationIdentity, type ProducerProviderIdentity, type ProducerReceipt } from '../scripts/cso-eval-protocol';
import { producerArtifactInventoryHash, producerHostPlatform, producerInstallationIdentityHash, producerProviderIdentityHash, producerReceiptHash, sha256, type ProducerArtifactInventory, type ProducerInstallationIdentity, type ProducerProviderIdentity, type ProducerReceipt } from '../scripts/cso-eval-protocol';
import type { Family, ProviderAdapter, RunOpts, RunResult } from './helpers/providers/types';
import { CORPUS_VERSION, FAMILIES, STACKS, loadCorpusManifest, materializeCase, sourceFiles, sourceHash } from './fixtures/cso-eval/materialize';
import { CORPUS_VERSION, FAMILIES, STACKS, defineEvalCorpus, loadCorpusManifest, materializeCase, sourceFiles, sourceHash, validateCorpusManifest, type CorpusManifest, type EvalVariant } from './fixtures/cso-eval/materialize';
import { judgeRepair, oracleFor, type PrivateEvidence } from './helpers/cso-eval-oracles';
import { inspectPreparation } from '../lib/cso/preparation';
import { assertRuntimeCompatible, RUNTIME_CATALOG } from '../lib/cso/runtime-catalog';
@@ -41,7 +41,8 @@ function portableSkill(version: 'v2' | 'v3', sectionMarker: string): string {
}
/** Synthetic accounting records exercise the scorer. These are not measured agent results. */
function syntheticResult(cell: EvalCell): EvalResult {
const syntheticResult = (cell: EvalCell): EvalResult => syntheticResultFor(cell, corpus);
function syntheticResultFor(cell: EvalCell, manifest: CorpusManifest<string>): EvalResult {
const runtime = cell.mode === 'comprehensive';
const positive = cell.variant === 'vulnerable';
const repairEvidenceHash = 'c'.repeat(64), recheckEvidenceHash = 'd'.repeat(64);
@@ -50,12 +51,12 @@ function syntheticResult(cell: EvalCell): EvalResult {
findings: positive ? [{ id: 'finding-1', evidence: cell.version === 'v2' ? 'legacy_review' : 'supported', claimedTested: runtime && cell.version === 'v3', judgment: 'correct', matchedCaseId: cell.caseId,
...(runtime && cell.version === 'v3' ? { trustedVerification: { repair: 'passed' as const, repairEvidenceHash, recheck: 'passed' as const, recheckEvidenceHash } } : {}) }] : [],
setup: runtime ? 'passed' : 'not_attempted', reproduction: runtime && positive ? 'passed' : 'not_attempted', repair: runtime && positive ? 'passed' : 'not_attempted', recheck: runtime && positive ? 'passed' : 'not_attempted',
...(runtime && positive ? { oracleEvidenceHash: repairEvidenceHash, oracleVersion: CORPUS_VERSION, currentSourceHash: corpus.cases.find(fixture => fixture.id === cell.caseId)!.filesHash.fixed, recheckEvidenceHash } : {}),
...(runtime && positive ? { oracleEvidenceHash: repairEvidenceHash, oracleVersion: manifest.version, currentSourceHash: manifest.cases.find(fixture => fixture.id === cell.caseId)!.filesHash.fixed, recheckEvidenceHash } : {}),
heldOutAssertionsPassed: runtime && positive, freshRecheck: runtime && positive, latencyMs: 1500, firstUsefulResultMs: positive ? 500 : null,
};
}
const group = (score: ReturnType<typeof scoreEval>, version: string, mode: string) => score.groups.find(item => item.version === version && item.mode === mode)!;
function syntheticInstallationIdentity(seed = '1'): ProducerInstallationIdentity {
function syntheticInstallationIdentity(seed = '1', catalogSeed = seed): ProducerInstallationIdentity {
const artifact = (offset: number) => ({ sha256: sha256(`${seed}:artifact:${offset}`), bytes: 100 + offset });
const core = artifact(3);
const withoutHash: Omit<ProducerInstallationIdentity, 'identityHash'> = {
@@ -63,8 +64,8 @@ function syntheticInstallationIdentity(seed = '1'): ProducerInstallationIdentity
producer: artifact(1), launcher: artifact(2), core, watchdog: artifact(4),
generation: { coreSha256: core.sha256, manifest: { sha256: sha256(`${core.sha256}\n`), bytes: 65 } },
embeddedCatalogs: {
runtimeRevision: 'runtime-test', runtimeBuildRevision: 'runtime-build-test', runtimeSha256: sha256(`${seed}:runtime`),
scannerRevision: 'scanner-test', scannerSha256: sha256(`${seed}:scanner`),
runtimeRevision: 'runtime-test', runtimeBuildRevision: 'runtime-build-test', runtimeSha256: sha256(`${catalogSeed}:runtime`),
scannerRevision: 'scanner-test', scannerSha256: sha256(`${catalogSeed}:scanner`),
},
};
return { ...withoutHash, identityHash: producerInstallationIdentityHash(withoutHash) };
@@ -80,12 +81,14 @@ function syntheticArtifactInventory(): ProducerArtifactInventory {
const base = { schemaVersion: 1 as const, root: 'security/cso' as const, entries: [], totalBytes: 0 };
return { ...base, identityHash: producerArtifactInventoryHash(base) };
}
/** Platform-pinned cells get one native installation and provider binary per platform, sharing catalogs and provider policy. */
function syntheticReceipt(cell: EvalCell): ProducerReceipt {
const withoutHash: Omit<ProducerReceipt, 'receiptHash'> = { schemaVersion: 1, cell, inputHash: 'e'.repeat(64), installationIdentity: syntheticInstallationIdentity(), providerIdentity: syntheticProviderIdentity(cell.host === 'codex' ? 'gpt' : cell.host), artifacts: syntheticArtifactInventory(), startedAt: '2026-01-01T00:00:00.000Z', finishedAt: '2026-01-01T00:00:01.000Z', status: 'succeeded', requestedModel: cell.model, modelUsed: `resolved-${cell.model}`, modelIdentitySource: 'provider_reported', durationMs: 1000, firstUsefulResultMs: null, toolCalls: 1, output: 'synthetic producer transcript', outputHash: sha256('synthetic producer transcript'), usage: { inputTokens: 10, outputTokens: 5, cachedTokens: 0, estimatedCostUSD: 0.001 } };
const seed = cell.platform ?? '1';
const withoutHash: Omit<ProducerReceipt, 'receiptHash'> = { schemaVersion: 1, cell, inputHash: 'e'.repeat(64), installationIdentity: syntheticInstallationIdentity(seed, '1'), providerIdentity: syntheticProviderIdentity(cell.host === 'codex' ? 'gpt' : cell.host, seed), artifacts: syntheticArtifactInventory(), startedAt: '2026-01-01T00:00:00.000Z', finishedAt: '2026-01-01T00:00:01.000Z', status: 'succeeded', requestedModel: cell.model, modelUsed: `resolved-${cell.model}`, modelIdentitySource: 'provider_reported', durationMs: 1000, firstUsefulResultMs: null, toolCalls: 1, output: 'synthetic producer transcript', outputHash: sha256('synthetic producer transcript'), usage: { inputTokens: 10, outputTokens: 5, cachedTokens: 0, estimatedCostUSD: 0.001 } };
return { ...withoutHash, receiptHash: producerReceiptHash(withoutHash) };
}
function syntheticSchedule(): PreparedEvalSchedule {
return { schemaVersion: 1, matrixHash: createHash('sha256').update(JSON.stringify(matrix)).digest('hex'), scheduledCells: matrix.cells.length, preparedCells: matrix.cells.length, jobs: matrix.cells.map(cell => ({ cellId: cell.id, relativePath: `jobs/${cell.id}`, inputHash: 'e'.repeat(64) })) };
function syntheticSchedule(target: EvalMatrix = matrix): PreparedEvalSchedule {
return { schemaVersion: 1, matrixHash: createHash('sha256').update(JSON.stringify(target)).digest('hex'), scheduledCells: target.cells.length, preparedCells: target.cells.length, jobs: target.cells.map(cell => ({ cellId: cell.id, relativePath: `jobs/${cell.id}`, inputHash: 'e'.repeat(64) })) };
}
function rehashReceipt(receipt: ProducerReceipt): ProducerReceipt {
const { receiptHash: _receiptHash, ...withoutHash } = receipt;
@@ -333,6 +336,131 @@ describe('CSO matched evaluation accounting', () => {
});
});
/** Synthetic stand-in with the private corpus layout. It is not held-out content. */
const privateFiles = (id: string, variant: EvalVariant): Record<string, string> => ({ 'README.md': '# Synthetic service\n', 'src/app.txt': `synthetic ${id} ${variant}\n` });
const privateManifest: CorpusManifest<string> = { schemaVersion: 1, version: 'synthetic-private-1', cases: STACKS.flatMap(stack => ['alpha-family', 'beta-family'].map((family, index) => {
const id = `${stack}-${family}`;
return { id, stack, family, severity: index === 0 ? 'high' as const : 'medium' as const, rootCause: family, location: { path: 'src/app.txt', symbol: 'action' },
coreColdStart: true as const, heldOut: true as const, filesHash: { vulnerable: sourceHash(privateFiles(id, 'vulnerable')), fixed: sourceHash(privateFiles(id, 'fixed')) } };
})) };
const privateCorpus = defineEvalCorpus(privateManifest, privateFiles);
const releasePlatforms = { daily: 'linux/amd64' as const, comprehensive: Object.fromEntries(privateManifest.cases.map((fixture, index) => [fixture.id, index % 2 ? 'linux/arm64' as const : 'linux/amd64' as const])) };
const releaseOptions = { profile: 'release' as const, model: 'matched-test-model', host: 'codex' as const, skillHashes: { v3: 'b'.repeat(64) }, platforms: releasePlatforms };
function privateBaseline(mutate: (results: EvalResult[], cells: EvalCell[]) => void = () => {}): EvalBaseline {
const baselineMatrix = createEvalMatrix({ profile: 'baseline', model: 'matched-test-model', host: 'codex', skillHashes: { v2: 'a'.repeat(64) }, platforms: { comprehensive: releasePlatforms.comprehensive } }, privateManifest);
const batch = collectProducerReceipts(baselineMatrix, syntheticSchedule(baselineMatrix), baselineMatrix.cells.map(syntheticReceipt), privateManifest);
const results = baselineMatrix.cells.map(cell => ({ ...syntheticResultFor(cell, privateManifest), producerReceiptHash: batch.receipts.find(receipt => receipt.cell.id === cell.id)!.receiptHash }));
mutate(results, baselineMatrix.cells);
return createEvalBaseline(baselineMatrix, batch, results, privateManifest);
}
function releaseRun(baseline: EvalBaseline | null = privateBaseline()) {
const release = createEvalMatrix({ ...releaseOptions, ...(baseline ? { baseline } : {}) }, privateManifest);
const batch = collectProducerReceipts(release, syntheticSchedule(release), release.cells.map(syntheticReceipt), privateManifest);
const results = release.cells.map(cell => ({ ...syntheticResultFor(cell, privateManifest), producerReceiptHash: batch.receipts.find(receipt => receipt.cell.id === cell.id)!.receiptHash }));
return { release, batch, results };
}
describe('CSO release profile on an injected corpus', () => {
test('the full profile matrix is byte-identical to its pre-release-profile form', () => {
expect(createHash('sha256').update(JSON.stringify(matrix)).digest('hex')).toBe('a0f20a127c8c92c73bb2a277f740295c153ccf95901e92cffab34a4553ef976b');
expect('profile' in matrix).toBe(false); expect(matrix.cells.every(cell => !('platform' in cell))).toBe(true);
});
test('an injected corpus shares the public layout and binds every generated source', () => {
expect(validateCorpusManifest(structuredClone(corpus))).toEqual(corpus);
const destination = join(root(), 'case');
expect(materializeCase('rails-beta-family', 'fixed', destination, privateCorpus).sourceHash).toBe(privateManifest.cases.find(item => item.id === 'rails-beta-family')!.filesHash.fixed);
expect(readFileSync(join(destination, 'src/app.txt'), 'utf8')).toBe('synthetic rails-beta-family fixed\n');
expect(() => defineEvalCorpus({ ...privateManifest, version: CORPUS_VERSION }, privateFiles)).toThrow('INJECTED_CORPUS_REUSES_PUBLIC_VERSION');
expect(() => defineEvalCorpus(privateManifest, (id, variant) => ({ ...privateFiles(id, variant), extra: 'drift' }))).toThrow('CORPUS_INTEGRITY_MISMATCH');
expect(() => defineEvalCorpus(privateManifest, () => ({ '../escape.txt': 'x' }))).toThrow('INVALID_CORPUS_SOURCE');
const unknownStack = structuredClone(privateManifest); (unknownStack.cases[0] as any).stack = 'php';
expect(() => validateCorpusManifest(unknownStack)).toThrow('INVALID_CORPUS_MANIFEST');
const extraField = structuredClone(privateManifest); (extraField.cases[0] as any).answer = 'oracle';
expect(() => validateCorpusManifest(extraField)).toThrow('INVALID_CORPUS_MANIFEST');
});
test('release pins v3, one repetition, and one native platform per cell', () => {
const release = createEvalMatrix(releaseOptions, privateManifest);
expect(release.cells).toHaveLength(privateManifest.cases.length * 4); expect(release.repetitions).toBe(1); expect(release.baselineHash).toBeNull();
expect(release.cells.every(cell => cell.version === 'v3' && cell.repetition === 1)).toBe(true);
expect(release.cells.filter(cell => cell.mode === 'daily').every(cell => cell.platform === 'linux/amd64')).toBe(true);
for (const cell of release.cells.filter(item => item.mode === 'comprehensive')) expect(cell.platform).toBe(releasePlatforms.comprehensive[cell.caseId]);
expect(new Set(release.cells.filter(cell => cell.mode === 'comprehensive').map(cell => cell.platform))).toEqual(new Set(['linux/amd64', 'linux/arm64']));
validateMatrix(release, privateManifest);
const moved = structuredClone(release); moved.platforms.comprehensive[privateManifest.cases[0].id] = 'linux/arm64';
expect(() => validateMatrix(moved, privateManifest)).toThrow('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX');
const { [privateManifest.cases[0].id]: _omitted, ...partial } = releasePlatforms.comprehensive;
expect(() => createEvalMatrix({ ...releaseOptions, platforms: { daily: 'linux/amd64', comprehensive: partial } }, privateManifest)).toThrow('INVALID_EVAL_PLATFORMS');
expect(() => createEvalMatrix({ ...releaseOptions, platforms: { daily: 'linux/riscv64' as any, comprehensive: releasePlatforms.comprehensive } }, privateManifest)).toThrow('INVALID_EVAL_PLATFORMS');
expect(() => createEvalMatrix({ ...releaseOptions, skillHashes: { v2: 'a'.repeat(64), v3: 'b'.repeat(64) } as any }, privateManifest)).toThrow('INVALID_MATCHED_EVAL_INPUT');
});
test('a two-platform batch collects and scores qualified against a hash-bound v2 baseline', () => {
const { release, batch, results } = releaseRun();
expect(new Set(batch.receipts.map(receipt => receipt.installationIdentity.identityHash)).size).toBe(2);
expect(batch.summary.groups.map(group => `${group.version}:${group.mode}`)).toEqual(['v3:daily', 'v3:comprehensive']);
const baseline = privateBaseline();
const score = scoreCollectedEval(release, batch, results, qualification, privateManifest, baseline);
expect(score.status).toBe('qualified'); expect(Object.values(score.gates).every(value => value === 'pass')).toBe(true);
expect(score.profile).toBe('release'); expect(score.baseline).toEqual({ baselineHash: baseline.baselineHash, highCriticalRecall: { numerator: 4, denominator: 4, value: 1 } });
expect(score.groups.map(item => `${item.version}:${item.mode}`)).toEqual(['v3:daily', 'v3:comprehensive']);
});
test('a mixed-corpus batch is rejected', () => {
const { release, batch, results } = releaseRun();
expect(() => scoreCollectedEval(release, batch, results, qualification)).toThrow('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX');
expect(() => collectProducerReceipts(release, syntheticSchedule(release), release.cells.map(syntheticReceipt))).toThrow('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX');
const mixed = structuredClone(release); mixed.cells[0] = { ...matrix.cells.find(cell => cell.version === 'v3' && cell.repetition === 1)!, platform: 'linux/amd64' };
expect(() => validateMatrix(mixed, privateManifest)).toThrow('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX');
const publicCell = matrix.cells.find(cell => cell.version === 'v3')!;
expect(() => collectProducerReceipts(release, syntheticSchedule(release), [syntheticReceipt(publicCell)], privateManifest)).toThrow('UNKNOWN_OR_DUPLICATE_PRODUCER_RECEIPT');
expect(() => scoreCollectedEval(release, syntheticBatch(), results, qualification, privateManifest)).toThrow('INVALID_PRODUCER_BATCH');
expect(() => scoreEval(release, [syntheticResult(publicCell)], qualification, privateManifest)).toThrow('UNKNOWN_OR_DUPLICATE_EVAL_RESULT');
});
test('identity checks hold per platform and require shared catalogs and provider policy across platforms', () => {
const release = createEvalMatrix(releaseOptions, privateManifest), schedule = syntheticSchedule(release);
const amd = release.cells.filter(cell => cell.platform === 'linux/amd64'), arm = release.cells.find(cell => cell.platform === 'linux/arm64')!;
const relabel = (cell: EvalCell, change: (receipt: ProducerReceipt) => void) => { const receipt = syntheticReceipt(cell); change(receipt); return rehashReceipt(receipt); };
expect(() => collectProducerReceipts(release, schedule, [syntheticReceipt(amd[0]), relabel(amd[1], receipt => { receipt.installationIdentity = syntheticInstallationIdentity('second-amd64', '1'); })], privateManifest)).toThrow('UNMATCHED_PRODUCER_INSTALLATIONS');
expect(() => collectProducerReceipts(release, schedule, [syntheticReceipt(amd[0]), relabel(amd[1], receipt => { receipt.providerIdentity = syntheticProviderIdentity('gpt', 'second-amd64'); })], privateManifest)).toThrow('UNMATCHED_PRODUCER_PROVIDERS');
expect(() => collectProducerReceipts(release, schedule, [syntheticReceipt(amd[0]), relabel(arm, receipt => { receipt.installationIdentity = syntheticInstallationIdentity('linux/amd64', '1'); })], privateManifest)).toThrow('UNMATCHED_PRODUCER_PLATFORMS');
expect(() => collectProducerReceipts(release, schedule, [syntheticReceipt(amd[0]), relabel(arm, receipt => { receipt.installationIdentity = syntheticInstallationIdentity('linux/arm64', 'other-catalog'); })], privateManifest)).toThrow('UNMATCHED_PRODUCER_INSTALLATIONS');
const otherPolicy = (() => { const { identityHash: _hash, ...base } = syntheticProviderIdentity('gpt', 'linux/arm64'); const changed = { ...base, policyRevision: 'gpt-other-policy' }; return { ...changed, identityHash: producerProviderIdentityHash(changed) }; })();
expect(() => collectProducerReceipts(release, schedule, [syntheticReceipt(amd[0]), relabel(arm, receipt => { receipt.providerIdentity = otherPolicy; })], privateManifest)).toThrow('UNMATCHED_PRODUCER_PROVIDERS');
expect(() => collectProducerReceipts(release, schedule, [syntheticReceipt(amd[0]), relabel(amd[1], receipt => { receipt.modelUsed = 'resolved-other-model'; })], privateManifest)).toThrow('UNMATCHED_EFFECTIVE_MODELS');
});
test('the external v2 baseline is pre-registered, complete, and bound to the release inputs', () => {
const unpinned = releaseRun(null);
const unmeasured = scoreCollectedEval(unpinned.release, unpinned.batch, unpinned.results, qualification, privateManifest);
expect(unmeasured.gates.noHighCriticalRecallRegression).toBe('unmeasured'); expect(unmeasured.status).toBe('partial'); expect(unmeasured.baseline).toBeNull();
const baseline = privateBaseline();
expect(() => scoreCollectedEval(unpinned.release, unpinned.batch, unpinned.results, qualification, privateManifest, baseline)).toThrow('UNBOUND_EVAL_BASELINE');
const { release, batch, results } = releaseRun(baseline);
const forged = structuredClone(baseline); forged.cases[0].found = false;
expect(() => scoreCollectedEval(release, batch, results, qualification, privateManifest, forged)).toThrow('UNBOUND_EVAL_BASELINE');
const weaker = privateBaseline(found => { found.forEach(result => { result.findings = []; }); });
expect(() => scoreCollectedEval(release, batch, results, qualification, privateManifest, weaker)).toThrow('UNBOUND_EVAL_BASELINE');
expect(() => createEvalMatrix({ ...releaseOptions, model: 'other-model', baseline }, privateManifest)).toThrow('UNBOUND_EVAL_BASELINE');
expect(() => createEvalMatrix({ ...releaseOptions, budgets: { daily: 600, comprehensive: 1200 }, baseline }, privateManifest)).toThrow('UNBOUND_EVAL_BASELINE');
const otherProvider = structuredClone(baseline); otherProvider.provider.version = 'gpt-other-version';
const { baselineHash: _hash, ...record } = otherProvider; otherProvider.baselineHash = createHash('sha256').update(JSON.stringify(record)).digest('hex');
const rebound = releaseRun(otherProvider);
expect(() => scoreCollectedEval(rebound.release, rebound.batch, rebound.results, qualification, privateManifest, otherProvider)).toThrow('UNBOUND_EVAL_BASELINE');
const highId = privateManifest.cases.find(item => item.severity === 'high')!.id;
for (let index = 0; index < release.cells.length; index++) {
const cell = release.cells[index];
if (cell.mode === 'comprehensive' && cell.variant === 'vulnerable' && cell.caseId === highId) results[index].findings = [];
}
const regressed = scoreCollectedEval(release, batch, results, qualification, privateManifest, baseline);
expect(regressed.gates.noHighCriticalRecallRegression).toBe('fail'); expect(regressed.baseline!.highCriticalRecall.value).toBe(1);
});
test('baseline records refuse incomplete or unadjudicated runs and baseline matrices never qualify', () => {
expect(() => privateBaseline(results => { results.pop(); })).toThrow('INCOMPLETE_EVAL_BASELINE');
expect(() => privateBaseline((results, cells) => { results[cells.findIndex(cell => cell.variant === 'vulnerable')].findings[0].judgment = 'unadjudicated'; })).toThrow('UNADJUDICATED_EVAL_BASELINE');
const baselineMatrix = createEvalMatrix({ profile: 'baseline', model: 'matched-test-model', host: 'codex', skillHashes: { v2: 'a'.repeat(64) }, platforms: { comprehensive: releasePlatforms.comprehensive } }, privateManifest);
expect(baselineMatrix.cells.every(cell => cell.version === 'v2' && cell.mode === 'comprehensive')).toBe(true);
expect(() => scoreEval(baselineMatrix, [], qualification, privateManifest)).toThrow('BASELINE_PROFILE_NOT_QUALIFIABLE');
expect(() => scoreEval(matrix, [], qualification, corpus, privateBaseline())).toThrow('UNBOUND_EVAL_BASELINE');
});
});
describe('CSO matched producer orchestration', () => {
const skills = { v2: portableSkill('v2', 'V2_SECTION_ONLY'), v3: portableSkill('v3', 'V3_SECTION_ONLY') };
const producerMatrix = createEvalMatrix({ model: 'exact-eval-model', host: 'codex', skillHashes: { v2: sha256(skills.v2), v3: sha256(skills.v3) } });
@@ -815,6 +943,24 @@ describe('CSO matched producer orchestration', () => {
expect(() => collectProducerReceipts(producerMatrix, schedule, [tampered])).toThrow('INVALID_PRODUCER_RECEIPT');
});
test('a platform-pinned release cell runs only on its native producer platform', async () => {
const release = createEvalMatrix({ ...releaseOptions, skillHashes: { v3: sha256(skills.v3) } }, privateManifest);
const host = producerHostPlatform(), foreign = host === 'linux/amd64' ? 'linux/arm64' : 'linux/amd64';
const native = release.cells.find(cell => cell.mode === 'comprehensive' && cell.platform === host);
const other = release.cells.find(cell => cell.mode === 'comprehensive' && cell.platform === foreign)!;
const destination = join(root(), 'prepared');
expect(() => prepareEvalJobs(release, skills, join(root(), 'both'), [other.id], privateCorpus)).toThrow('EVAL_SKILL_HASH_MISMATCH');
prepareEvalJobs(release, { v3: skills.v3 }, destination, [other.id, ...(native ? [native.id] : [])], privateCorpus);
const receipts = join(root(), 'receipts'); mkdirSync(receipts);
const isolatedOther = isolate(destination, other);
expect(readFileSync(join(isolatedOther.source, 'src/app.txt'), 'utf8')).toBe(`synthetic ${other.caseId} ${other.variant}\n`);
await expect(runProducerCell(isolatedOther.input, join(receipts, `${other.id}.json`), withHelper({ adapter: new FakeAdapter(() => {}), paidExecutionAuthorized: true }))).rejects.toThrow('UNMATCHED_PRODUCER_PLATFORM');
if (!native) return;
const isolatedNative = isolate(destination, native);
const receipt = await runProducerCell(isolatedNative.input, join(receipts, `${native.id}.json`), withHelper({ adapter: new FakeAdapter(() => {}), paidExecutionAuthorized: true }));
expect(receipt.cell).toEqual(native);
});
test('seals source read-only and withholds a receipt after a mode/content mutation', async () => {
const destination = join(root(), 'prepared');
const cell = selected[0];
test/cso-native-evidence.test.ts
+103
View File
@@ -0,0 +1,103 @@
import { afterEach, describe, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { deriveNativeChecks, nativeEvidence, nativeTestRequirements, parseJUnit, type JUnitCase } from '../scripts/cso-native-evidence';
import { requiredChecks } from '../scripts/cso-runtime-promotion';
const ROOT = resolve(import.meta.dir, '..');
const STACKS = ['node', 'bun', 'python', 'rails', 'postgresql'] as const;
const temps: string[] = [];
afterEach(() => { for (const dir of temps.splice(0)) rmSync(dir, { recursive: true, force: true }); });
const escape = (value: string) => value.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;').replaceAll('"', '&quot;');
// Mirrors Bun 1.4.0 `bun test --reporter=junit` output: passing cases self-close, skips and failures carry a child element.
function report(cases: JUnitCase[]): string {
const body = cases.map(item => {
const open = `<testcase name="${escape(item.name)}" classname="suite" time="0.1" file="${item.file}" line="1" assertions="${item.assertions}"`;
if (item.status === 'passed') return ` ${open} />`;
const child = item.status === 'skipped' ? '<skipped />' : '<failure type="AssertionError" message="expected">AssertionError</failure>';
return ` ${open}>\n ${child}\n </testcase>`;
}).join('\n');
const failures = cases.filter(item => item.status === 'failed').length;
return `<?xml version="1.0" encoding="UTF-8"?>\n<testsuites name="bun test" tests="${cases.length}" failures="${failures}">\n <testsuite name="suite">\n${body}\n </testsuite>\n</testsuites>\n`;
}
function passing(stack: typeof STACKS[number]): JUnitCase[] {
const seen = new Map<string, JUnitCase>();
for (const refs of Object.values(nativeTestRequirements(stack))) for (const ref of refs) seen.set(`${ref.file}\0${ref.name}`, { ...ref, status: 'passed', assertions: 3 });
return [...seen.values()];
}
function staged(stack: string, platform = 'linux/amd64') {
return { schemaVersion: 1, state: 'staged', stack, platform, image: `ghcr.io/garrytan/gstack/cso-staging/${stack}-amd64@sha256:${'a'.repeat(64)}` };
}
describe('CSO native gate evidence', () => {
test('native check keys are release gates of their stack, and private checks are exactly the remainder', () => {
for (const stack of STACKS) {
const native = Object.keys(nativeTestRequirements(stack));
expect(native.every(key => requiredChecks(stack).includes(key))).toBe(true);
const evidence = nativeEvidence(staged(stack), stack, 'linux/amd64', report(passing(stack))) as any;
expect(Object.values(evidence.nativeChecks).every(value => value === true)).toBe(true);
expect(Object.keys(evidence.privateChecks).sort()).toEqual(requiredChecks(stack).filter(key => !native.includes(key)));
expect(Object.values(evidence.privateChecks).every(value => value === 'pending')).toBe(true);
expect(evidence).toMatchObject({ state: 'native-gates-passed', qualified: false, stack });
expect(evidence.nativeTestReport).toMatch(/^sha256:[a-f0-9]{64}$/);
}
const postgres = nativeEvidence(staged('postgresql'), 'postgresql', 'linux/amd64', report(passing('postgresql'))) as any;
expect(Object.keys(postgres.privateChecks).sort()).toEqual(['secretCanaryPassed', 'watchdogCleanupPassed']);
const rails = nativeEvidence(staged('rails'), 'rails', 'linux/amd64', report(passing('rails'))) as any;
expect(Object.keys(rails.privateChecks).sort()).toEqual(['accuracyGatesPassed', 'heldOutRepairPassed', 'secretCanaryPassed', 'watchdogCleanupPassed']);
});
test('a skipped, missing, failed, duplicated, or assertion-free named test leaves its checks false', () => {
const named = nativeTestRequirements('node').positiveNegativeAssertionsPassed[0];
const mutate = (change: (cases: JUnitCase[]) => JUnitCase[]) => deriveNativeChecks('node', parseJUnit(report(change(passing('node')))));
const skipped = mutate(cases => cases.map(item => item.name === named.name ? { ...item, status: 'skipped' as const, assertions: 0 } : item));
expect(skipped.positiveNegativeAssertionsPassed).toBe(false);
expect(skipped.containmentPassed).toBe(false);
expect(skipped.coldStartPassed).toBe(true);
expect(mutate(cases => cases.filter(item => item.name !== named.name)).positiveNegativeAssertionsPassed).toBe(false);
expect(mutate(cases => [...cases, cases.find(item => item.name === named.name)!]).positiveNegativeAssertionsPassed).toBe(false);
expect(mutate(cases => cases.map(item => item.name === named.name ? { ...item, assertions: 0 } : item)).positiveNegativeAssertionsPassed).toBe(false);
const unrelatedFailure = mutate(cases => [...cases, { file: 'test/other.test.ts', name: '(unnamed)', status: 'failed', assertions: 0 }]);
expect(Object.values(unrelatedFailure).every(value => value === false)).toBe(true);
expect(() => nativeEvidence(staged('node'), 'node', 'linux/amd64', report(passing('node').map(item => item.name === named.name ? { ...item, status: 'skipped', assertions: 0 } : item))))
.toThrow('NATIVE_GATE_NOT_PASSED: containmentPassed, positiveNegativeAssertionsPassed');
});
test('a different file with the same test name does not satisfy a gate', () => {
const cases = passing('bun').map(item => item.file === 'test/cso-stack-cold-integration.test.ts' ? { ...item, file: 'test/elsewhere.test.ts' } : item);
const checks = deriveNativeChecks('bun', cases);
expect(checks.coldStartPassed).toBe(false);
expect(checks.acquisitionPublicOnlyPassed).toBe(false);
expect(checks.positiveNegativeAssertionsPassed).toBe(true);
});
test('staged evidence must describe the same stack and platform', () => {
expect(() => nativeEvidence(staged('bun'), 'node', 'linux/amd64', report(passing('node')))).toThrow('STAGED_EVIDENCE_MISMATCH');
expect(() => nativeEvidence(staged('node', 'linux/arm64'), 'node', 'linux/amd64', report(passing('node')))).toThrow('STAGED_EVIDENCE_MISMATCH');
expect(() => nativeEvidence({ ...staged('node'), state: 'qualified' }, 'node', 'linux/amd64', report(passing('node')))).toThrow('STAGED_EVIDENCE_MISMATCH');
expect(() => parseJUnit('<testsuites/>')).toThrow('INVALID_JUNIT_REPORT');
expect(() => parseJUnit('<?xml version="1.0"?>\n<testsuites name="bun test"></testsuites>')).toThrow('EMPTY_JUNIT_REPORT');
});
test('the real Bun JUnit report of the Docker suites names every required test, and skipped suites prove nothing', () => {
const dir = mkdtempSync(join(tmpdir(), 'cso-native-junit-')); temps.push(dir);
const outfile = join(dir, 'native.xml');
const env = { ...process.env };
for (const key of ['GSTACK_CSO_DOCKER_TESTS', 'GSTACK_CSO_TEST_IMAGE', 'GSTACK_CSO_TEST_STACK']) delete env[key];
const run = spawnSync(process.execPath, ['run', 'test:cso:docker', '--reporter=junit', `--reporter-outfile=${outfile}`], { cwd: ROOT, env, encoding: 'utf8', timeout: 120_000 });
expect(run.status, run.stderr).toBe(0);
const cases = parseJUnit(readFileSync(outfile, 'utf8'));
for (const stack of STACKS) {
for (const refs of Object.values(nativeTestRequirements(stack))) for (const ref of refs) {
expect(cases.filter(item => item.file === ref.file && item.name === ref.name).map(item => item.status)).toEqual(['skipped']);
}
expect(Object.values(deriveNativeChecks(stack, cases)).every(value => value === false)).toBe(true);
}
});
});
test/cso-requalification.test.ts
+64
View File
@@ -0,0 +1,64 @@
import { afterEach, describe, expect, test } from 'bun:test';
import { cpSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { spawnSync } from 'node:child_process';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import committedCatalog from '../lib/cso/runtime-catalog.json';
import buildInputs from '../lib/cso/images/build-inputs.json';
import { ISOLATION_POLICY_HASH } from '../lib/cso/docker';
import { selectRuntime, validateRuntimeCatalog, type RuntimeCatalog } from '../lib/cso/runtime-catalog';
import { imageSourceFiles, requalificationTriggers, staleRequalificationTriggers } from '../scripts/cso-requalification';
import { catalogPromotionCandidate } from '../scripts/cso-runtime-promotion';
import { completeRuntimeCatalogFixture } from './helpers/cso-runtime-catalog';
const ROOT = resolve(import.meta.dir, '..');
const temps: string[] = [];
afterEach(() => { for (const dir of temps.splice(0)) rmSync(dir, { recursive: true, force: true }); });
const current = await requalificationTriggers();
describe('CSO runtime requalification triggers', () => {
test('the committed catalog is not older than its trigger inputs', () => {
expect(staleRequalificationTriggers(committedCatalog as unknown as RuntimeCatalog, current)).toEqual([]);
const cli = spawnSync(process.execPath, ['scripts/cso-requalification.ts', 'check', 'lib/cso/runtime-catalog.json'], { cwd: ROOT, encoding: 'utf8', timeout: 30_000 });
expect(cli.status, cli.stderr).toBe(0); expect(cli.stdout).toBe('REQUALIFICATION CURRENT\n');
});
test('triggers cover the helper ABI, isolation policy, every image-embedded source byte, and the build inputs', async () => {
expect(current).toMatchObject({ helperAbi: 3, isolationPolicyHash: ISOLATION_POLICY_HASH, buildInputsRevision: buildInputs.revision });
const files = await imageSourceFiles();
for (const file of ['lib/cso/verifier.ts', 'lib/cso/preparation-container.ts', 'lib/cso/contracts.ts', 'lib/cso/images/entrypoint', 'lib/cso/images/rails.Dockerfile']) expect(files).toContain(file);
for (const file of ['lib/cso/images/README.md', 'lib/cso/images/qualification.json', 'lib/cso/images/build-inputs.json', 'lib/cso/cli.ts']) expect(files).not.toContain(file);
const copy = mkdtempSync(join(tmpdir(), 'cso-requalification-')); temps.push(copy);
cpSync(join(ROOT, 'lib'), join(copy, 'lib'), { recursive: true });
expect(await requalificationTriggers(copy)).toEqual(current);
writeFileSync(join(copy, 'lib/cso/verifier.ts'), '\n', { flag: 'a' });
expect((await requalificationTriggers(copy)).preparationSha256).not.toBe(current.preparationSha256);
});
test('a promoted catalog names each trigger that changed after qualification', () => {
const catalog = completeRuntimeCatalogFixture();
const recorded = catalog.promotion!.requalification;
expect(staleRequalificationTriggers(catalog, { ...recorded })).toEqual([]);
expect(staleRequalificationTriggers(catalog, { ...recorded, preparationSha256: 'd'.repeat(64), buildInputsRevision: 'next-inputs' })).toEqual(['buildInputsRevision', 'preparationSha256']);
const withdrawn = { ...catalog, runtimes: [], promotion: undefined };
expect(staleRequalificationTriggers(withdrawn, { ...recorded, preparationSha256: 'd'.repeat(64) })).toEqual([]);
});
test('promoted catalogs must record valid triggers, and the helper refuses runtimes qualified under another isolation policy', () => {
const catalog = completeRuntimeCatalogFixture();
validateRuntimeCatalog(catalog);
expect(selectRuntime('node', 'linux/amd64', catalog).stack).toBe('node');
const missing = structuredClone(catalog) as any; delete missing.promotion.requalification;
expect(() => validateRuntimeCatalog(missing)).toThrow('INVALID_RUNTIME_PROMOTION');
const otherBuild = structuredClone(catalog); otherBuild.promotion!.requalification.buildInputsRevision = 'other-build';
expect(() => validateRuntimeCatalog(otherBuild)).toThrow('INVALID_RUNTIME_PROMOTION');
const otherPolicy = structuredClone(catalog); otherPolicy.promotion!.requalification.isolationPolicyHash = 'e'.repeat(64);
expect(() => selectRuntime('node', 'linux/amd64', otherPolicy)).toThrow('STALE_RUNTIME_QUALIFICATION');
});
test('promotion records the staged triggers and refuses ones measured under another isolation policy', () => {
expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, [], { ...current, isolationPolicyHash: 'e'.repeat(64) })).toThrow('STALE_RUNTIME_QUALIFICATION');
expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, [], { ...current, buildInputsRevision: 'other' })).toThrow('INVALID_REQUALIFICATION_TRIGGERS');
expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, [], { ...current, extra: true } as any)).toThrow('INVALID_REQUALIFICATION_TRIGGERS');
});
});
test/cso-runtime-promotion.test.ts
+63 -11
View File
@@ -2,11 +2,13 @@ import { describe, expect, test } from 'bun:test';
import { createHash } from 'node:crypto';
import committedCatalog from '../lib/cso/runtime-catalog.json';
import buildInputs from '../lib/cso/images/build-inputs.json';
import { validateRuntimeCatalog } from '../lib/cso/runtime-catalog';
import { isEvaluationRuntimeCatalog, validateRuntimeCatalog } from '../lib/cso/runtime-catalog';
import { imageBuildMatrix, type ImageBuildRow } from '../scripts/cso-image-matrix';
import { catalogPromotionCandidate, validateRuntimeCatalogTransition, type RuntimeQualificationStatement } from '../scripts/cso-runtime-promotion';
import { catalogPromotionCandidate, evaluationCandidateCatalog, validateEvaluationCandidate, validateRuntimeCatalogTransition, type RuntimeQualificationStatement } from '../scripts/cso-runtime-promotion';
import { assertVersionOutput, probesForBuildRow } from '../scripts/cso-verify-runtime-base';
import { requalificationTriggers } from '../scripts/cso-requalification';
const TRIGGERS = await requalificationTriggers();
const digest = (value: string) => `sha256:${createHash('sha256').update(value).digest('hex')}`;
function checks(row: ImageBuildRow): Record<string, true> {
@@ -69,7 +71,7 @@ describe('CSO runtime catalog promotion', () => {
});
test('complete same-run evidence generates a validated review candidate bound to its evidence', () => {
const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, statements());
const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, statements(), TRIGGERS);
expect(candidate.previousRevision).toBe(committedCatalog.revision);
expect(candidate.runtimes).toHaveLength(10);
expect(candidate.promotion).toMatchObject({
@@ -85,30 +87,30 @@ describe('CSO runtime catalog promotion', () => {
test('missing release checks, mutable images, and split qualification runs cannot be promoted', () => {
const missing = statements();
delete missing[0].checks.heldOutRepairPassed;
expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, missing)).toThrow('MISSING_RELEASE_GATES');
expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, missing, TRIGGERS)).toThrow('MISSING_RELEASE_GATES');
const mutable = statements();
mutable[0].image = 'ghcr.io/garrytan/gstack/cso-staging/node-amd64:latest';
expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, mutable)).toThrow('INVALID_QUALIFIED_IMAGE');
expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, mutable, TRIGGERS)).toThrow('INVALID_QUALIFIED_IMAGE');
const split = statements();
split[0].workflow = 'https://github.com/garrytan/gstack/actions/runs/999999';
expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, split)).toThrow('SPLIT_QUALIFICATION_RUN');
expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, split, TRIGGERS)).toThrow('SPLIT_QUALIFICATION_RUN');
const extra = statements() as Array<RuntimeQualificationStatement & { finding?: string }>;
extra[0].finding = 'must not enter a public release artifact';
expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, extra)).toThrow('INVALID_QUALIFICATION_STATEMENT_FIELDS');
expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, extra, TRIGGERS)).toThrow('INVALID_QUALIFICATION_STATEMENT_FIELDS');
});
test('qualification statements must match reviewed versions and every expected platform row', () => {
const drift = statements();
drift[0].versions = { ...drift[0].versions, node: '24.5.0' };
expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, drift)).toThrow('QUALIFICATION_BUILD_MISMATCH');
expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, statements().slice(1))).toThrow('INCOMPLETE_QUALIFICATION_MATRIX');
expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, drift, TRIGGERS)).toThrow('QUALIFICATION_BUILD_MISMATCH');
expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, statements().slice(1), TRIGGERS)).toThrow('INCOMPLETE_QUALIFICATION_MATRIX');
});
test('promotion is a compare-and-swap against the exact reviewed catalog revision', () => {
const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, statements());
const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, statements(), TRIGGERS);
expect(() => validateRuntimeCatalogTransition(committedCatalog, candidate)).not.toThrow();
expect(() => validateRuntimeCatalogTransition(
{ ...committedCatalog, revision: 'catalog-advanced-concurrently' },
@@ -123,7 +125,7 @@ describe('CSO runtime catalog promotion', () => {
});
test('retained promotion evidence rejects a post-generation runtime image substitution', () => {
const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, statements());
const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, statements(), TRIGGERS);
const altered = structuredClone(candidate);
altered.runtimes[0].image = altered.runtimes[0].image.replace(/[a-f0-9]{64}$/, 'e'.repeat(64));
expect(altered.promotion!.evidenceDigest).toBe(candidate.promotion!.evidenceDigest);
@@ -148,3 +150,53 @@ describe('CSO reviewed base version probes', () => {
expect(() => assertVersionOutput('bundler', '2.6.7', 'Bundler version 2.6.7\n')).not.toThrow();
});
});
/** staged-image.json records exactly as the stage job writes them. */
function stagedRecords(): Record<string, unknown>[] {
return statements().map(({ checks: _checks, qualifiedAt: _qualifiedAt, state: _state, ...statement }) => ({
...statement, state: 'staged', verifiedProvenance: true, qualification: 'pending native and private release gates; no automatic promotion',
}));
}
describe('CSO evaluation-only runtime catalogs', () => {
test('one staging run yields an evaluation catalog with no measured release gate', () => {
const candidate = evaluationCandidateCatalog(committedCatalog, buildInputs, stagedRecords(), '2026-10-05T00:00:00.000Z');
expect(candidate.revision).toBe(`cso-eval-123456-${'a'.repeat(12)}`);
expect(isEvaluationRuntimeCatalog(candidate)).toBe(true); expect(isEvaluationRuntimeCatalog(committedCatalog as any)).toBe(false);
expect(candidate.promotion).toBeUndefined(); expect(candidate.evaluation?.workflow).toBe('https://github.com/garrytan/gstack/actions/runs/123456');
expect(candidate.runtimes.every(runtime => runtime.state === 'evaluation_candidate' && runtime.qualification.kind === 'evaluation')).toBe(true);
expect(JSON.stringify(candidate)).not.toMatch(/Passed/);
expect(validateEvaluationCandidate(candidate, buildInputs)).toBe(candidate);
expect(() => validateEvaluationCandidate(committedCatalog, buildInputs)).toThrow('NOT_AN_EVALUATION_CATALOG');
});
test('staging records must come from one run and match the reviewed build rows', () => {
const split = stagedRecords(); split[1].workflow = 'https://github.com/garrytan/gstack/actions/runs/999';
expect(() => evaluationCandidateCatalog(committedCatalog, buildInputs, split, '2026-10-05T00:00:00.000Z')).toThrow('SPLIT_STAGING_RUN');
const unstaged = stagedRecords(); unstaged[0].state = 'qualified';
expect(() => evaluationCandidateCatalog(committedCatalog, buildInputs, unstaged, '2026-10-05T00:00:00.000Z')).toThrow('INVALID_STAGED_RUNTIME');
const moved = stagedRecords(); moved[0].buildRevision = 'other-build-revision';
expect(() => evaluationCandidateCatalog(committedCatalog, buildInputs, moved, '2026-10-05T00:00:00.000Z')).toThrow('QUALIFICATION_BUILD_MISMATCH');
expect(() => evaluationCandidateCatalog(committedCatalog, buildInputs, stagedRecords().slice(1), '2026-10-05T00:00:00.000Z')).toThrow('INCOMPLETE_QUALIFICATION_MATRIX');
});
test('evaluation and release catalogs cannot borrow each other\'s shape', () => {
const candidate = evaluationCandidateCatalog(committedCatalog, buildInputs, stagedRecords(), '2026-10-05T00:00:00.000Z');
const rebound = structuredClone(candidate); rebound.revision = `cso-eval-999-${'a'.repeat(12)}`;
expect(() => validateRuntimeCatalog(rebound)).toThrow('INVALID_EVALUATION_RUNTIME_CATALOG');
const promoted = structuredClone(candidate) as any; promoted.promotion = { ...promoted.evaluation, qualificationEvidenceDigest: digest('x'), requalification: TRIGGERS }; delete promoted.evaluation;
expect(() => validateRuntimeCatalog(promoted)).toThrow('INVALID_EVALUATION_RUNTIME_CATALOG');
const disguised = structuredClone(candidate) as any; disguised.revision = `cso-v3-123456-${'a'.repeat(12)}`;
expect(() => validateRuntimeCatalog(disguised)).toThrow('UNQUALIFIED_RUNTIME');
const gated = structuredClone(candidate) as any; gated.runtimes[0].qualification.heldOutRepairPassed = true;
expect(() => validateRuntimeCatalog(gated)).toThrow('INVALID_EVALUATION_RUNTIME');
const released = catalogPromotionCandidate(committedCatalog, buildInputs, statements(), TRIGGERS) as any;
released.revision = `cso-eval-123456-${'a'.repeat(12)}`;
expect(() => validateRuntimeCatalog(released)).toThrow('UNQUALIFIED_RUNTIME');
});
test('promotion and catalog transitions refuse evaluation catalogs', () => {
const candidate = evaluationCandidateCatalog(committedCatalog, buildInputs, stagedRecords(), '2026-10-05T00:00:00.000Z');
expect(() => catalogPromotionCandidate(candidate, buildInputs, statements(), TRIGGERS)).toThrow('EVALUATION_CATALOG_NOT_PROMOTABLE');
expect(() => validateRuntimeCatalogTransition(committedCatalog, candidate)).toThrow('EVALUATION_CATALOG_NOT_PROMOTABLE');
expect(() => validateRuntimeCatalogTransition(candidate, catalogPromotionCandidate(committedCatalog, buildInputs, statements(), TRIGGERS))).toThrow('EVALUATION_CATALOG_NOT_PROMOTABLE');
expect(isEvaluationRuntimeCatalog(committedCatalog as any)).toBe(false);
});
});
test/cso-staging-run.test.ts
+164
View File
@@ -0,0 +1,164 @@
import { afterEach, describe, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { committedImageBuildMatrix } from '../scripts/cso-image-matrix';
import { dispatchSourceCommit, stagingRunId, verifyStagingRun, type StagingRunEvidence } from '../scripts/cso-staging-run';
const ROOT = resolve(import.meta.dir, '..');
const REPOSITORY = 'garrytan/gstack';
const RUN_ID = '34980146643';
const COMMIT = 'a'.repeat(40);
const RUNTIME_IDS = committedImageBuildMatrix().include.map(row => row.runtimeId);
const temps: string[] = [];
afterEach(() => { for (const dir of temps.splice(0)) rmSync(dir, { recursive: true, force: true }); });
function invocation(runId = RUN_ID, attempt = 1, repository = REPOSITORY): string {
return `https://github.com/${repository}/actions/runs/${runId}/attempts/${attempt}`;
}
// Shape of one `gh attestation verify --format json` entry for an actions/attest provenance statement.
function verified(predicateInvocation = invocation(), certificateInvocation = predicateInvocation): unknown[] {
return [{
attestation: { bundle: {} },
verificationResult: {
mediaType: 'application/vnd.dev.sigstore.verificationresult+json;version=0.1',
signature: { certificate: {
subjectAlternativeName: `https://github.com/${REPOSITORY}/.github/workflows/cso-runtime-images.yml@refs/heads/main`,
githubWorkflowTrigger: 'workflow_dispatch',
sourceRepositoryDigest: COMMIT,
runInvocationURI: certificateInvocation,
} },
statement: {
_type: 'https://in-toto.io/Statement/v1',
predicateType: 'https://slsa.dev/provenance/v1',
subject: [{ name: 'ghcr.io/garrytan/gstack/cso-staging/node-amd64', digest: { sha256: 'b'.repeat(64) } }],
predicate: {
buildDefinition: { externalParameters: { workflow: { path: '.github/workflows/cso-runtime-images.yml', ref: 'refs/heads/main' } } },
runDetails: { builder: { id: 'builder' }, metadata: { invocationId: predicateInvocation } },
},
},
},
}];
}
function run(overrides: Record<string, unknown> = {}): Record<string, unknown> {
return {
id: Number(RUN_ID), path: '.github/workflows/cso-runtime-images.yml', event: 'workflow_dispatch',
head_branch: 'main', head_sha: COMMIT, status: 'completed', conclusion: 'success', run_attempt: 2,
repository: { full_name: REPOSITORY }, head_repository: { full_name: REPOSITORY }, ...overrides,
};
}
function jobs(): Array<Record<string, unknown>> {
const job = (name: string) => ({ name, run_id: Number(RUN_ID), status: 'completed', conclusion: 'success' });
return [
job('reviewed-inputs'),
{ ...job('validate-native'), conclusion: 'skipped' },
...RUNTIME_IDS.map(id => job(`stage (${id})`)),
...RUNTIME_IDS.map(id => job(`qualify-native ${id}`)),
];
}
function pages(list = jobs(), size = 100): unknown[] {
const result: unknown[] = [];
for (let start = 0; start < list.length; start += size) result.push({ total_count: list.length, jobs: list.slice(start, start + size) });
return result;
}
function evidence(overrides: Partial<StagingRunEvidence> = {}): StagingRunEvidence {
return { repository: REPOSITORY, runId: RUN_ID, sourceCommit: COMMIT, runtimeIds: RUNTIME_IDS, run: run(), jobPages: pages(), ...overrides };
}
describe('CSO staging run binding', () => {
test('the committed matrix has the ten native rows the ingress requires', () => {
expect(RUNTIME_IDS).toHaveLength(10);
});
test('resolves one staging run from every image provenance, across rerun attempts', () => {
const all = RUNTIME_IDS.map((_, index) => verified(invocation(RUN_ID, index === 0 ? 2 : 1)));
expect(stagingRunId(REPOSITORY, all)).toBe(RUN_ID);
});
test('rejects statements that resolve to different runs, foreign repositories, or forged invocations', () => {
const split = RUNTIME_IDS.map((_, index) => verified(invocation(index === 3 ? '999' : RUN_ID)));
expect(() => stagingRunId(REPOSITORY, split)).toThrow('STAGING_RUN_NOT_UNIQUE');
const twoInOne = [[...verified(), ...verified(invocation('999'))]];
expect(() => stagingRunId(REPOSITORY, twoInOne)).toThrow('STAGING_RUN_NOT_UNIQUE');
expect(() => stagingRunId(REPOSITORY, [verified(invocation(RUN_ID, 1, 'someone/fork'))])).toThrow('STAGING_INVOCATION_MISMATCH');
expect(() => stagingRunId(REPOSITORY, [verified(invocation(), invocation('999'))])).toThrow('STAGING_INVOCATION_MISMATCH');
expect(() => stagingRunId(REPOSITORY, [verified(`${invocation()}/extra`)])).toThrow('STAGING_INVOCATION_MISMATCH');
const sbom = verified() as any[];
sbom[0].verificationResult.statement.predicateType = 'https://spdx.dev/Document/v2.3';
expect(() => stagingRunId(REPOSITORY, [sbom])).toThrow('INVALID_STAGING_PROVENANCE');
expect(() => stagingRunId(REPOSITORY, [[]])).toThrow('INVALID_STAGING_PROVENANCE');
expect(() => stagingRunId(REPOSITORY, [])).toThrow('INVALID_STAGING_PROVENANCE');
});
test('accepts the successful protected-main staging dispatch with every native gate passed, across job pages', () => {
expect(() => verifyStagingRun(evidence())).not.toThrow();
expect(() => verifyStagingRun(evidence({ jobPages: pages(jobs(), 7) }))).not.toThrow();
});
test('rejects a run that is not the successful protected-main staging dispatch at the statements commit', () => {
expect(() => verifyStagingRun(evidence({ run: run({ path: '.github/workflows/cso-runtime-qualification.yml' }) }))).toThrow('STAGING_WORKFLOW_MISMATCH');
expect(() => verifyStagingRun(evidence({ run: run({ head_branch: 'feature' }) }))).toThrow('STAGING_RUN_NOT_PROTECTED_MAIN_DISPATCH');
expect(() => verifyStagingRun(evidence({ run: run({ event: 'pull_request' }) }))).toThrow('STAGING_RUN_NOT_PROTECTED_MAIN_DISPATCH');
expect(() => verifyStagingRun(evidence({ run: run({ conclusion: 'failure' }) }))).toThrow('STAGING_RUN_NOT_SUCCESSFUL');
expect(() => verifyStagingRun(evidence({ run: run({ status: 'in_progress', conclusion: null }) }))).toThrow('STAGING_RUN_NOT_SUCCESSFUL');
expect(() => verifyStagingRun(evidence({ run: run({ head_sha: 'c'.repeat(40) }) }))).toThrow('STAGING_SOURCE_COMMIT_MISMATCH');
expect(() => verifyStagingRun(evidence({ run: run({ id: 1 }) }))).toThrow('STAGING_RUN_ID_MISMATCH');
expect(() => verifyStagingRun(evidence({ run: run({ head_repository: { full_name: 'someone/fork' } }) }))).toThrow('STAGING_REPOSITORY_MISMATCH');
});
test('rejects a failed, missing, extra, or truncated native gate job', () => {
const failed = jobs();
failed[failed.length - 1].conclusion = 'failure';
expect(() => verifyStagingRun(evidence({ jobPages: pages(failed) }))).toThrow('STAGING_NATIVE_GATE_NOT_PASSED');
const skipped = jobs();
skipped[skipped.length - 1].conclusion = 'skipped';
expect(() => verifyStagingRun(evidence({ jobPages: pages(skipped) }))).toThrow('STAGING_NATIVE_GATE_NOT_PASSED');
const missing = jobs().slice(0, -1);
expect(() => verifyStagingRun(evidence({ jobPages: pages(missing) }))).toThrow('STAGING_NATIVE_JOBS_MISMATCH');
const extra = [...jobs(), { name: 'qualify-native unexpected', run_id: Number(RUN_ID), status: 'completed', conclusion: 'success' }];
expect(() => verifyStagingRun(evidence({ jobPages: pages(extra) }))).toThrow('STAGING_NATIVE_JOBS_MISMATCH');
const truncated = pages(jobs(), 7).slice(0, -1);
expect(() => verifyStagingRun(evidence({ jobPages: truncated }))).toThrow('INCOMPLETE_STAGING_JOBS');
const otherRun = jobs();
otherRun[otherRun.length - 1].run_id = 1;
expect(() => verifyStagingRun(evidence({ jobPages: pages(otherRun) }))).toThrow('STAGING_NATIVE_GATE_NOT_PASSED');
expect(() => verifyStagingRun(evidence({ jobPages: [] }))).toThrow('INVALID_STAGING_JOBS');
});
test('a dispatch must name one exact source commit', () => {
const statements = RUNTIME_IDS.map(runtimeId => ({ runtimeId, sourceCommit: COMMIT }));
expect(dispatchSourceCommit({ client_payload: { statements } })).toBe(COMMIT);
statements[4].sourceCommit = 'c'.repeat(40);
expect(() => dispatchSourceCommit({ client_payload: { statements } })).toThrow('STAGING_SOURCE_COMMIT_MISMATCH');
expect(() => dispatchSourceCommit({ client_payload: { statements: [{ sourceCommit: 'main' }] } })).toThrow('STAGING_SOURCE_COMMIT_MISMATCH');
});
test('the workflow command line resolves and verifies the run from retained files', () => {
const dir = mkdtempSync(join(tmpdir(), 'cso-staging-run-')); temps.push(dir);
const files = RUNTIME_IDS.map(id => {
const file = join(dir, `${id}.json`);
writeFileSync(file, JSON.stringify(verified()));
return file;
});
const script = join(ROOT, 'scripts/cso-staging-run.ts');
const resolved = spawnSync(process.execPath, ['run', script, 'run-id', '--repository', REPOSITORY, ...files], { encoding: 'utf8', timeout: 30_000 });
expect(resolved.status, resolved.stderr).toBe(0);
expect(resolved.stdout).toBe(`${RUN_ID}\n`);
writeFileSync(join(dir, 'event.json'), JSON.stringify({ client_payload: { statements: RUNTIME_IDS.map(runtimeId => ({ runtimeId, sourceCommit: COMMIT })) } }));
writeFileSync(join(dir, 'run.json'), JSON.stringify(run()));
writeFileSync(join(dir, 'jobs.json'), JSON.stringify(pages()));
const args = ['run', script, 'verify', '--repository', REPOSITORY, '--run-id', RUN_ID, '--event', join(dir, 'event.json'), '--run', join(dir, 'run.json'), '--jobs', join(dir, 'jobs.json')];
const accepted = spawnSync(process.execPath, args, { encoding: 'utf8', timeout: 30_000 });
expect(accepted.status, accepted.stderr).toBe(0);
writeFileSync(join(dir, 'run.json'), JSON.stringify(run({ conclusion: 'failure' })));
const rejected = spawnSync(process.execPath, args, { encoding: 'utf8', timeout: 30_000 });
expect(rejected.status).toBe(1);
expect(rejected.stderr).toContain('STAGING_RUN_NOT_SUCCESSFUL');
});
});
test/cso-verifier.test.ts
+49 -1
View File
@@ -1,5 +1,6 @@
import { describe, expect, test } from 'bun:test';
import { boundedResponseBody } from '../lib/cso/verifier';
import type { HttpAssertion } from '../lib/cso/contracts';
import { boundedResponseBody, judgeSecurity } from '../lib/cso/verifier';
describe('CSO bounded loopback verifier response reader',()=>{
test('accepts a response exactly at the configured byte limit',async()=>{
@@ -26,3 +27,50 @@ describe('CSO bounded loopback verifier response reader',()=>{
expect(cancelled).toBe(true);
});
});
describe('CSO verifier security judgement',()=>{
const payloads:HttpAssertion[]=[
{name:'id parameter',path:'/user?id=2',method:'GET',expected:{status:403},vulnerable:{status:200,includes:'tenant-b'}},
{name:'tenant header',path:'/user',method:'GET',headers:{'x-tenant':'b'},expected:{status:403},vulnerable:{status:200,includes:'tenant-b'}},
{name:'write path',path:'/user/2',method:'PUT',body:'{}',expected:{status:404},vulnerable:{status:204}},
];
const serve=(fixed:Set<string>)=>{
const seen:string[]=[];
const server=Bun.serve({port:0,hostname:'127.0.0.1',fetch(req){
const url=new URL(req.url),key=`${req.method} ${url.pathname}${url.search}`;seen.push(key);
if(req.method==='PUT')return new Response(null,{status:fixed.has(key)?404:204});
return fixed.has(key)?new Response('denied',{status:403}):new Response('tenant-b secret',{status:200});
}});
return {server,seen,port:Number(server.port)};
};
const ALL=new Set(['GET /user?id=2','GET /user','PUT /user/2']);
test('one booted server answers every array member and the before phase needs every vulnerable expectation',async()=>{
const {server,seen,port}=serve(new Set());
try{
expect(await judgeSecurity('before',payloads,port)).toEqual({security:'intended_failure',members:['intended_failure','intended_failure','intended_failure']});
expect(seen).toEqual(['GET /user?id=2','GET /user','PUT /user/2']);
}finally{server.stop(true);}
});
test('the after phase passes only when every array member passes its fixed expectation',async()=>{
const fixed=serve(ALL);
try{expect(await judgeSecurity('after',payloads,fixed.port)).toEqual({security:'pass',members:['pass','pass','pass']});}finally{fixed.server.stop(true);}
const partial=serve(new Set(['GET /user?id=2','PUT /user/2']));
try{
expect(await judgeSecurity('after',payloads,partial.port)).toEqual({security:'inconclusive',members:['pass','inconclusive','pass']});
expect(await judgeSecurity('before',payloads,partial.port)).toEqual({security:'inconclusive',members:['pass','intended_failure','pass']});
}finally{partial.server.stop(true);}
const disproved=serve(ALL);
try{expect((await judgeSecurity('before',payloads,disproved.port)).security).toBe('pass');}finally{disproved.server.stop(true);}
});
test('a single assertion keeps its original before and after outcomes',async()=>{
const vulnerable=serve(new Set()),fixed=serve(ALL);
try{
expect(await judgeSecurity('before',payloads[0],vulnerable.port)).toEqual({security:'intended_failure',members:['intended_failure']});
expect(await judgeSecurity('after',payloads[0],vulnerable.port)).toEqual({security:'inconclusive',members:['inconclusive']});
expect(await judgeSecurity('after',payloads[0],fixed.port)).toEqual({security:'pass',members:['pass']});
}finally{vulnerable.server.stop(true);fixed.server.stop(true);}
});
});
test/fixtures/cso-eval/README.md
Vendored
+41
View File
@@ -249,6 +249,47 @@ reports, containment/canary receipts, and enforced oracle separation. Missing
evaluations return
`unmeasured` or `partial`, never passing release gates.
## Release profile, private corpora, and two architectures
The matrix above is the `full` profile: matched v2/v3 cells, three repetitions,
and one producer installation. This public corpus is a development set, because
its oracles are public. A private evaluator scores a held-out corpus with the
same layout through the same code:
- **Injected corpus.** `defineEvalCorpus(manifest, sourceFiles)` accepts a
manifest with this `manifest.json` layout and a deterministic source generator,
checks every variant's source hash, and refuses the public corpus version.
`prepareEvalJobs(..., corpus)` and `materializeCase(..., corpus)` materialize
from it; `createEvalMatrix`, `validateMatrix`, `collectProducerReceipts`,
`scoreCollectedEval`, and `createEvalBaseline` take its manifest. Its oracles
use the `PrivateOracle` shape from `test/helpers/cso-eval-oracles.ts` and stay
in the private evaluator.
- **`release` profile.** `createEvalMatrix({ profile: 'release', skillHashes: { v3 },
platforms: { daily, comprehensive } })` builds v3-only cells with one repetition.
Daily cells run on one platform; each comprehensive case is fixed to
`linux/amd64` or `linux/arm64`. Every cell carries its `platform`, and the
producer refuses a cell whose platform differs from its host.
- **Per-platform identities.** One batch may hold both architectures. Each
platform must have exactly one producer installation and one provider binary;
platforms must use different installations with identical embedded catalogs
and identical provider family, policy revision, version, and arguments. All
cells must report one effective model.
- **External v2 baseline.** A `baseline` profile matrix holds v2 comprehensive
cells with the release platform assignment. `createEvalBaseline` turns its
complete, fully adjudicated, receipt-bound results into an `EvalBaseline`
record whose `baselineHash` covers the corpus hash, model, host, budget,
platform assignment, v2 payload hash, producer batch, effective model, provider
policy and version, and each case's outcome. A release matrix pins that hash
when it is created; scoring then requires the exact record and the same
effective model and provider policy. Without a pinned baseline the
regression gate stays `unmeasured`.
A matrix, batch, or result from one corpus never validates against another:
the matrix binds the corpus hash, so a mixed-corpus batch is rejected. The
`collect`, `score`, and `baseline` commands accept `--corpus-manifest <file>`,
and `score` accepts `--baseline <file>`. The `full` profile's matrix bytes and
gates are unchanged.
The free accounting tests use clearly labeled synthetic observations to test the
scorer. They are **not measured agent performance**. No paid comparison, fixture
cold-start qualification, or containment qualification is claimed by this corpus.
test/fixtures/cso-eval/materialize.ts
Vendored
+68 -8
View File
@@ -9,13 +9,22 @@ export const FAMILIES = ['sql-injection', 'command-injection', 'path-traversal',
export type EvalStack = typeof STACKS[number];
export type EvalFamily = typeof FAMILIES[number];
export type EvalVariant = 'vulnerable' | 'fixed';
export interface CorpusCase {
id: string; stack: EvalStack; family: EvalFamily; severity: 'critical' | 'high' | 'medium';
export interface CorpusCase<Family extends string = EvalFamily> {
id: string; stack: EvalStack; family: Family; severity: 'critical' | 'high' | 'medium';
rootCause: string; location: { path: string; symbol: string };
coreColdStart: true; heldOut: true;
filesHash: { vulnerable: string; fixed: string };
}
export interface CorpusManifest { schemaVersion: 1; version: string; cases: CorpusCase[] }
export interface CorpusManifest<Family extends string = EvalFamily> { schemaVersion: 1; version: string; cases: CorpusCase<Family>[] }
/**
* A corpus with the public layout: a manifest plus a deterministic source
* generator for each case variant. Private evaluators inject their own; the
* default is this public development corpus.
*/
export interface EvalCorpus {
manifest: CorpusManifest<string>;
sourceFiles(id: string, variant: EvalVariant): Record<string, string>;
}
const hash = (input: string) => createHash('sha256').update(input).digest('hex');
export function sourceHash(files: Record<string, string>): string {
return hash(JSON.stringify(Object.keys(files).sort().map(path => [path, hash(files[path])])));
@@ -304,16 +313,67 @@ export function loadCorpusManifest(): CorpusManifest {
if (JSON.stringify(manifest) !== JSON.stringify(createManifest())) throw new Error('CORPUS_INTEGRITY_MISMATCH');
return manifest;
}
export function materializeCase(id: string, variant: EvalVariant, destination: string): { path: string; sourceHash: string } {
const manifest = loadCorpusManifest();
const spec = manifest.cases.find(item => item.id === id);
const SLUG = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
const HEX = /^[a-f0-9]{64}$/;
const SOURCE_PATH = /^[A-Za-z0-9._-]+(?:\/[A-Za-z0-9._-]+)*$/;
const safeSourcePath = (path: string) => SOURCE_PATH.test(path) && path.length <= 256 && path.split('/').every(part => part !== '.' && part !== '..');
/** Validate the manifest layout of a public or injected corpus without reading its sources. */
export function validateCorpusManifest(value: unknown): CorpusManifest<string> {
const manifest = value as CorpusManifest<string>;
if (!manifest || typeof manifest !== 'object' || manifest.schemaVersion !== 1 || typeof manifest.version !== 'string' || !/^[a-z0-9][a-z0-9.-]{0,63}$/.test(manifest.version) ||
!Array.isArray(manifest.cases) || manifest.cases.length === 0 || canonicalKeys(manifest) !== 'cases,schemaVersion,version') throw new Error('INVALID_CORPUS_MANIFEST');
const ids = new Set<string>();
for (const item of manifest.cases) {
if (!item || canonicalKeys(item) !== 'coreColdStart,family,filesHash,heldOut,id,location,rootCause,severity,stack' ||
typeof item.id !== 'string' || !SLUG.test(item.id) || ids.has(item.id) || !STACKS.includes(item.stack) ||
typeof item.family !== 'string' || !SLUG.test(item.family) || !['critical', 'high', 'medium'].includes(item.severity) ||
typeof item.rootCause !== 'string' || !item.rootCause.trim() || !item.location || typeof item.location.path !== 'string' || !safeSourcePath(item.location.path) ||
typeof item.location.symbol !== 'string' || !item.location.symbol.trim() || item.coreColdStart !== true || item.heldOut !== true ||
!item.filesHash || !HEX.test(item.filesHash.vulnerable) || !HEX.test(item.filesHash.fixed) || item.filesHash.vulnerable === item.filesHash.fixed) throw new Error('INVALID_CORPUS_MANIFEST');
ids.add(item.id);
}
return manifest;
}
function canonicalKeys(value: object): string { return Object.keys(value).sort().join(','); }
/** The public development corpus. Its oracles are public, so it is never held out. */
export function publicCorpus(): EvalCorpus {
return { manifest: loadCorpusManifest(), sourceFiles };
}
/**
* Bind an injected corpus to its manifest: every variant's generated sources
* must hash to the pinned value. An injected corpus cannot reuse the public
* corpus version, so its matrices and results never alias public ones.
*/
export function defineEvalCorpus(manifestValue: unknown, generate: EvalCorpus['sourceFiles']): EvalCorpus {
const manifest = validateCorpusManifest(manifestValue);
if (manifest.version === CORPUS_VERSION) throw new Error('INJECTED_CORPUS_REUSES_PUBLIC_VERSION');
const read = (id: string, variant: EvalVariant) => {
const files = generate(id, variant);
if (!files || typeof files !== 'object' || !Object.keys(files).length || Object.entries(files).some(([path, contents]) => !safeSourcePath(path) || typeof contents !== 'string')) throw new Error('INVALID_CORPUS_SOURCE');
return files;
};
for (const item of manifest.cases) for (const variant of ['vulnerable', 'fixed'] as const) {
if (sourceHash(read(item.id, variant)) !== item.filesHash[variant]) throw new Error('CORPUS_INTEGRITY_MISMATCH');
}
return { manifest, sourceFiles: (id, variant) => {
if (!manifest.cases.some(item => item.id === id)) throw new Error('UNKNOWN_CORPUS_CASE');
if (variant !== 'vulnerable' && variant !== 'fixed') throw new Error('INVALID_CORPUS_VARIANT');
return read(id, variant);
} };
}
export function materializeCase(id: string, variant: EvalVariant, destination: string, corpus: EvalCorpus = publicCorpus()): { path: string; sourceHash: string } {
const spec = corpus.manifest.cases.find(item => item.id === id);
if (!spec) throw new Error('UNKNOWN_CORPUS_CASE');
const path = resolve(destination);
if (existsSync(path)) throw new Error('CORPUS_DESTINATION_EXISTS');
const parent = dirname(path);
if (realpathSync(parent) !== parent || !lstatSync(parent).isDirectory()) throw new Error('UNSAFE_CORPUS_DESTINATION');
const files = sourceFiles(id, variant);
if (sourceHash(files) !== spec.filesHash[variant]) throw new Error('CORPUS_INTEGRITY_MISMATCH');
const files = corpus.sourceFiles(id, variant);
if (sourceHash(files) !== spec.filesHash[variant] || Object.keys(files).some(file => !safeSourcePath(file))) throw new Error('CORPUS_INTEGRITY_MISMATCH');
mkdirSync(path, { mode: 0o700 });
for (const [file, contents] of Object.entries(files)) {
const output = join(path, file); mkdirSync(dirname(output), { recursive: true, mode: 0o700 });
test/helpers/cso-runtime-catalog.ts
+7
View File
@@ -1,5 +1,6 @@
import type { CsoStack } from '../../lib/cso/preparation';
import { canonical, sha256 } from '../../lib/cso/contracts';
import { ISOLATION_POLICY_HASH } from '../../lib/cso/docker';
import {
CSO_HELPER_ABI,
type QualifiedRuntime,
@@ -96,6 +97,12 @@ export function completeRuntimeCatalogFixture(
workflow: WORKFLOW,
evidenceDigest: `sha256:${sha256(canonical(runtimes))}`,
qualificationEvidenceDigest: DIGEST,
requalification: {
helperAbi: CSO_HELPER_ABI,
isolationPolicyHash: ISOLATION_POLICY_HASH,
preparationSha256: 'c'.repeat(64),
buildInputsRevision: 'runtime-test-build-v1',
},
},
runtimes,
};