From bd49e540f619c09c0e4db55b31b9c11df01f2ae6 Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 22 Oct 2026 20:24:36 +0000 Subject: [PATCH] test(cso): stop pinning the committed runtime catalog to empty --- CHANGELOG.md | 3 +++ test/cso-audit-paths.test.ts | 3 ++- test/cso-cli-lifecycle.test.ts | 13 +++++++------ test/cso-preparation-executor.test.ts | 6 +++--- test/cso-preparation.test.ts | 11 +++++++---- test/cso-runtime-promotion.test.ts | 5 ++++- test/cso-scanner-executor.test.ts | 4 ++-- test/helpers/cso-runtime-catalog.ts | 21 +++++++++++++++++++++ 8 files changed, 49 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5222c7347..321fb3b7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ ### Added - **`lib/cso/runtime-catalog.json`** is now catalog `cso-v3-39674065531-133cb360842d`, which replaces `cso-v3-build-reviewed-2026-09-10`. Promote run 39841502986 generated it from qualification run 39674065531's authenticated statements and attested it. GitHub Actions committed the attested bytes (`sha256:078e0bf6…`). Each runtime's image is pinned to a public `ghcr.io/garrytan/gstack/cso-staging/…@sha256` digest, and each entry records its source commit (133cb36), SBOM and provenance digests, and requalification triggers (`preparationSha256` `c721b585…`). +### For contributors +- **The /cso free tests no longer assume the committed runtime catalog is empty.** Six tests pinned `runtimes: []` and broke as soon as a promotion landed. Tests that need an unqualified catalog now build one with `unqualifiedRuntimeCatalog()`. The replay tests expect whichever fail-closed rejection the committed catalog produces, so the suite passes with the catalog both before and after a promotion. + ## [1.91.177.0] - 2026-10-22 **The /cso runtime promotion workflow can finish.** Promote run 39691485727 failed after its `cso-runtime-release` approval. Its "Generate a fail-closed catalog candidate" step calls `gh api …/compare` to prove the staged commit is an ancestor of the qualification head, but the step had no `GH_TOKEN`. On a hosted runner `gh` then exits 4 ("set the GH_TOKEN environment variable"). The run stopped before it signed, pushed or opened anything. diff --git a/test/cso-audit-paths.test.ts b/test/cso-audit-paths.test.ts index c4636487c..33dba7464 100644 --- a/test/cso-audit-paths.test.ts +++ b/test/cso-audit-paths.test.ts @@ -11,6 +11,7 @@ import { newRun, readJson, saveReport } from '../lib/cso/state'; import { MAX_OUTPUT, canonical, sha256, type VerificationObservation } from '../lib/cso/contracts'; import { ISOLATION_POLICY_HASH } from '../lib/cso/docker'; import { canonicalStartPlan, canonicalTestPlan, patchHash, verifyRepair } from '../lib/cso/verification'; +import { replayRuntimeRejection } from './helpers/cso-runtime-catalog'; const ROOT = path.resolve(import.meta.dir, '..'); const launcher = path.join(ROOT, 'bin', process.platform === 'win32' ? 'gstack-cso-launcher.exe' : 'gstack-cso-launcher'); @@ -342,6 +343,6 @@ describe('CSO --paths audit scope', () => { const supplied = cso(repo, ['replay', verified.bundle.id, '--source', repo]); expect(supplied.status).not.toBe(0); expect(supplied.stderr).not.toContain('does not match the bundle input hashes'); - expect(supplied.stderr).toContain('MISSING_QUALIFIED_RUNTIME'); + expect(supplied.stderr).toContain(replayRuntimeRejection(request.runtimeProfile, `linux/${arch}`).message); }, 60_000); }); diff --git a/test/cso-cli-lifecycle.test.ts b/test/cso-cli-lifecycle.test.ts index d3d502f34..8b54b0d66 100644 --- a/test/cso-cli-lifecycle.test.ts +++ b/test/cso-cli-lifecycle.test.ts @@ -6,6 +6,7 @@ import { spawnSync } from 'node:child_process'; import { canonical, sha256, type VerificationObservation } from '../lib/cso/contracts'; import { ISOLATION_POLICY_HASH } from '../lib/cso/docker'; import { saveReport, withLock } from '../lib/cso/state'; +import { replayRuntimeRejection } from './helpers/cso-runtime-catalog'; import { canonicalStartPlan, canonicalTestPlan, patchHash, verifyRepair } from '../lib/cso/verification'; const ROOT=path.resolve(import.meta.dir,'..'),launcher=path.join(ROOT,'bin',process.platform==='win32'?'gstack-cso-launcher.exe':'gstack-cso-launcher'); @@ -64,23 +65,23 @@ describe('CSO interrupted-run and replay commands',()=>{ const finding={title:'Unprotected record read',rootCause:'Record lookup omits the caller authorization predicate',location:{path:'app.js',line:1,symbol:'recordLookup'},advisoryIds:[],severity:'high',confidence:'high',confidenceRationale:'The caller-to-record trace directly establishes the missing authorization predicate',evidence:'supported',attackerControl:'Authenticated caller chooses the record identifier',impact:'Another tenant record is returned',scenario:'A tenant supplies another tenant record identifier and receives the record',trace:['GET /records/:id','recordLookup','record response'],references:['app.js:1','OWASP API1:2023'],recommendation:'Bind the lookup to the authenticated caller before returning the record',challenge:{reviewer:'independent-reviewer',independent:true,mode:'independent_agent',callers:'Authenticated route forwards the record identifier',controls:'Authentication exists without record authorization',counterevidence:'Opaque identifiers reduce guessing but do not authorize known identifiers',conclusion:'The record lookup lacks a caller authorization predicate'}}; expect(command(['submit',run.runId,writeInput('replay-finding.json',{findings:[finding]})]).status).toBe(0);const reportPath=path.join(dir,'report.json'),report=JSON.parse(fs.readFileSync(reportPath,'utf8')),findingId=report.findings[0].id,start=canonicalStartPlan(snapshot,'node',3456),tests=canonicalTestPlan(snapshot,'node'); const request:any={findingId,runtimeProfile:`node-24.4.0-${process.arch==='arm64'?'arm64':'amd64'}`,port:3456,start:start.command,legitimate:[{name:'legitimate control',path:'/control',method:'GET',expected:{status:200,includes:'CONTROL_OK'}}],security:{name:'unauthorized record is denied',path:'/records/other',method:'GET',expected:{status:403,includes:'DENIED'},vulnerable:{status:200,includes:'RECORD'}},existingTests:tests.commands,fixtures:{},boundaryFiles:['app.js'],testFiles:tests.files,changes:[{path:'app.js',beforeSha256:sha256(fs.readFileSync(path.join(snapshot,'app.js'))),after:'module.exports = "fixed"\n',effect:'source'}],review:{reviewer:'independent-reviewer',independent:true,rootCauseRepaired:true,featurePreserved:true,boundaryMocks:false,rationale:'The caller predicate is added while the legitimate control and existing test remain unchanged.',reviewedPatchHash:''}};request.review.reviewedPatchHash=patchHash(request); - const platform=process.arch==='arm64'?'linux/arm64':'linux/amd64',runtime:any={id:request.runtimeProfile,stack:'node',platform,image:`ghcr.io/garrytan/gstack/cso-staging/node-${process.arch==='arm64'?'arm64':'amd64'}@sha256:${'b'.repeat(64)}`},executor={observe:async(_source:string,phase:'before'|'after',received:any,_runtime:any,_verifier:any,_work:string,_control:string,_execution:any,evidence:any,witness:any)=>{const observation:VerificationObservation={booted:true,legitimate:true,security:phase==='before'?'intended_failure':'pass',existingTests:false,output:`external ${phase} assertions passed`,inputHash:''},receipt=await witness.attest(observation,[{command:received.existingTests[0],code:0,output:tap,minimumPassingTests:evidence.minimumPassingTests[0]}]);return{observation:{...observation,existingTests:receipt.diagnosticTestsPassed,inputHash:witness.binding.harnessHash},witness:receipt};}}; + const platform=process.arch==='arm64'?'linux/arm64':'linux/amd64',rejection=replayRuntimeRejection(request.runtimeProfile,platform),runtime:any={id:request.runtimeProfile,stack:'node',platform,image:`ghcr.io/garrytan/gstack/cso-staging/node-${process.arch==='arm64'?'arm64':'amd64'}@sha256:${'b'.repeat(64)}`},executor={observe:async(_source:string,phase:'before'|'after',received:any,_runtime:any,_verifier:any,_work:string,_control:string,_execution:any,evidence:any,witness:any)=>{const observation:VerificationObservation={booted:true,legitimate:true,security:phase==='before'?'intended_failure':'pass',existingTests:false,output:`external ${phase} assertions passed`,inputHash:''},receipt=await witness.attest(observation,[{command:received.existingTests[0],code:0,output:tap,minimumPassingTests:evidence.minimumPassingTests[0]}]);return{observation:{...observation,existingTests:receipt.diagnosticTestsPassed,inputHash:witness.binding.harnessHash},witness:receipt};}}; const verified=await verifyRepair({runId:run.runId,runDir:dir,manifest,rawRequest:request,runtime,verifier:runtime,policyHash:ISOLATION_POLICY_HASH,auditPolicyHash:sha256(canonical(report.policy)),archives:[],executor}); report.findings[0].reproduction='reproduced';report.findings[0].repair='runtime_tested';report.findings[0].verificationId=verified.bundle.id;report.findings[0].verificationAssurance={assertions:'authenticated_out_of_process',testCompletion:'self_reported',review:'self_attested'};saveReport(dir,report); const leased=withLock(dir,()=>command(['replay',verified.bundle.id])) as ReturnType;expect(leased.status).not.toBe(0);expect(leased.stderr).toContain('INSUFFICIENT_CAPACITY');expect(leased.stderr).toContain('Another helper is updating this run'); - const retained=command(['replay',verified.bundle.id]);expect(retained.status).not.toBe(0);expect(retained.stderr).toContain('PREREQUISITE');expect(retained.stderr).toContain('MISSING_QUALIFIED_RUNTIME'); + const retained=command(['replay',verified.bundle.id]);expect(retained.status).not.toBe(0);expect(retained.stderr).toContain(rejection.code);expect(retained.stderr).toContain(rejection.message); const manifestPath=path.join(dir,'snapshot.json'),expiredManifest={...manifest,expiresAt:new Date(Date.now()-1_000).toISOString()};fs.writeFileSync(manifestPath,`${JSON.stringify(expiredManifest,null,2)}\n`); const expiredPresent=command(['replay',verified.bundle.id]);expect(expiredPresent.status).not.toBe(0);expect(expiredPresent.stderr).toContain('MISSING_INPUT');expect(expiredPresent.stderr).toContain('Retained source expired'); - const suppliedExpired=command(['replay',verified.bundle.id,'--source',repo]);expect(suppliedExpired.status).not.toBe(0);expect(suppliedExpired.stderr).toContain('PREREQUISITE');expect(suppliedExpired.stderr).toContain('MISSING_QUALIFIED_RUNTIME');expect(suppliedExpired.stderr).not.toContain('Retained source expired'); + const suppliedExpired=command(['replay',verified.bundle.id,'--source',repo]);expect(suppliedExpired.status).not.toBe(0);expect(suppliedExpired.stderr).toContain(rejection.code);expect(suppliedExpired.stderr).toContain(rejection.message);expect(suppliedExpired.stderr).not.toContain('Retained source expired'); const noncanonicalExpiry=new Date(Date.now()+86400_000).toISOString().replace(/\.\d{3}Z$/,'Z');fs.writeFileSync(manifestPath,`${JSON.stringify({...manifest,expiresAt:noncanonicalExpiry},null,2)}\n`); - const invalidExpiry=command(['replay',verified.bundle.id,'--source',repo]);expect(invalidExpiry.status).not.toBe(0);expect(invalidExpiry.stderr).toContain('INCOMPATIBLE_INPUT');expect(invalidExpiry.stderr).toContain('Retained snapshot expiry is invalid');expect(invalidExpiry.stderr).not.toContain('MISSING_QUALIFIED_RUNTIME'); + const invalidExpiry=command(['replay',verified.bundle.id,'--source',repo]);expect(invalidExpiry.status).not.toBe(0);expect(invalidExpiry.stderr).toContain('INCOMPATIBLE_INPUT');expect(invalidExpiry.stderr).toContain('Retained snapshot expiry is invalid');expect(invalidExpiry.stderr).not.toContain(rejection.message); fs.writeFileSync(manifestPath,`${JSON.stringify({...manifest,expiresAt:new Date(Date.now()+86400_000).toISOString()},null,2)}\n`);fs.writeFileSync(path.join(snapshot,'app.js'),'module.exports = "tampered"\n'); - const corruptRetained=command(['replay',verified.bundle.id,'--source',repo]);expect(corruptRetained.status).not.toBe(0);expect(corruptRetained.stderr).toContain('INCOMPATIBLE_INPUT');expect(corruptRetained.stderr).toContain('Retained snapshot changed');expect(corruptRetained.stderr).not.toContain('MISSING_QUALIFIED_RUNTIME'); + const corruptRetained=command(['replay',verified.bundle.id,'--source',repo]);expect(corruptRetained.status).not.toBe(0);expect(corruptRetained.stderr).toContain('INCOMPATIBLE_INPUT');expect(corruptRetained.stderr).toContain('Retained snapshot changed');expect(corruptRetained.stderr).not.toContain(rejection.message); fs.rmSync(snapshot,{recursive:true,force:true});fs.rmSync(path.join(dir,'readable'),{recursive:true,force:true}); const expired=command(['replay',verified.bundle.id]);expect(expired.status).not.toBe(0);expect(expired.stderr).toContain('MISSING_INPUT');expect(expired.stderr).toContain('Retained source expired'); fs.writeFileSync(path.join(repo,'app.js'),'module.exports = "changed"\n');const changed=command(['replay',verified.bundle.id,'--source',repo]);expect(changed.status).not.toBe(0);expect(changed.stderr).toContain('INCOMPATIBLE_INPUT');expect(changed.stderr).toContain('does not match the bundle input hashes'); - fs.writeFileSync(path.join(repo,'app.js'),'module.exports = "vulnerable"\n');const supplied=command(['replay',verified.bundle.id,'--source',repo]);expect(supplied.status).not.toBe(0);expect(supplied.stderr).toContain('PREREQUISITE');expect(supplied.stderr).toContain('MISSING_QUALIFIED_RUNTIME');expect(supplied.stderr).not.toContain('does not match the bundle input hashes'); + fs.writeFileSync(path.join(repo,'app.js'),'module.exports = "vulnerable"\n');const supplied=command(['replay',verified.bundle.id,'--source',repo]);expect(supplied.status).not.toBe(0);expect(supplied.stderr).toContain(rejection.code);expect(supplied.stderr).toContain(rejection.message);expect(supplied.stderr).not.toContain('does not match the bundle input hashes'); },30_000); }); diff --git a/test/cso-preparation-executor.test.ts b/test/cso-preparation-executor.test.ts index 5aa47d25e..5123df45b 100644 --- a/test/cso-preparation-executor.test.ts +++ b/test/cso-preparation-executor.test.ts @@ -13,7 +13,7 @@ import { type PreparationSandboxRunner, type OfflinePreparationRequest, preparedIdentityEntry } from '../lib/cso/preparation-executor'; import { CSO_HELPER_ABI } from '../lib/cso/runtime-catalog'; import { DockerPreparationSandboxRunner, isBlockedRegistryAddress, RegistryEgressBroker } from '../lib/cso/preparation-docker'; -import { completeRuntimeCatalogFixture, qualifiedRuntimeFixture } from './helpers/cso-runtime-catalog'; +import { completeRuntimeCatalogFixture, qualifiedRuntimeFixture, unqualifiedRuntimeCatalog } from './helpers/cso-runtime-catalog'; const roots: string[] = []; const digest = 'a'.repeat(64); @@ -138,9 +138,9 @@ function errorCode(error: unknown): string | undefined { return error instanceof afterEach(() => { for (const value of roots.splice(0)) fs.rmSync(value, { recursive: true, force: true }); }); describe('CSO constrained dependency preparation executor', () => { - test('the unqualified default catalog and forged admissions fail closed', async () => { + test('an unqualified catalog and forged admissions fail closed', async () => { const source = snapshot('node'), plan = inspectPreparation(source, 'node'), runner = new FakeRunner(), cache = cacheFixture(); - expect(() => admitPreparationRuntime({ plan, platform: 'linux/amd64' })).toThrow('MISSING_QUALIFIED_RUNTIME'); + expect(() => admitPreparationRuntime({ plan, platform: 'linux/amd64', catalog: unqualifiedRuntimeCatalog() })).toThrow('MISSING_QUALIFIED_RUNTIME'); const executor = new PreparationExecutor({ cache, runner }); try { await executor.acquire({ plan, admission: { schemaVersion: 1, catalogRevision: 'forged', runtime: runtime('node') }, diff --git a/test/cso-preparation.test.ts b/test/cso-preparation.test.ts index f9ca4271a..6addab47f 100644 --- a/test/cso-preparation.test.ts +++ b/test/cso-preparation.test.ts @@ -4,7 +4,7 @@ import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { inspectPreparation, railsTestConfiguration } from '../lib/cso/preparation'; import { assertRuntimeCompatible, RUNTIME_CATALOG, rollbackCatalog, selectRuntime, validateRuntimeCatalog } from '../lib/cso/runtime-catalog'; -import { completeRuntimeCatalogFixture, qualifiedRuntimeFixture } from './helpers/cso-runtime-catalog'; +import { completeRuntimeCatalogFixture, qualifiedRuntimeFixture, unqualifiedRuntimeCatalog } from './helpers/cso-runtime-catalog'; const roots: string[] = []; const hash = 'a'.repeat(64); @@ -231,10 +231,13 @@ describe('CSO inert Rails preparation', () => { describe('CSO runtime catalog admission', () => { const runtime = () => qualifiedRuntimeFixture('node'); const catalog = () => completeRuntimeCatalogFixture('test-1'); - test('default catalog cannot execute unqualified images', () => { + test('default catalog admits only digest-pinned reviewed profiles and unqualified profiles cannot execute', () => { expect(RUNTIME_CATALOG.profiles).toHaveLength(10); - expect(RUNTIME_CATALOG.runtimes).toEqual([]); - expect(() => selectRuntime('node', 'linux/amd64')).toThrow('Reviewed build profile node-24.4.0-amd64 is awaiting a qualified image promotion'); + for (const runtime of RUNTIME_CATALOG.runtimes) { + expect(RUNTIME_CATALOG.profiles.filter(profile => profile.id === runtime.id && profile.stack === runtime.stack && profile.platform === runtime.platform)).toHaveLength(1); + expect(runtime.image).toMatch(/@sha256:[a-f0-9]{64}$/); + } + expect(() => selectRuntime('node', 'linux/amd64', unqualifiedRuntimeCatalog())).toThrow('Reviewed build profile node-24.4.0-amd64 is awaiting a qualified image promotion'); }); test('selects exact qualified digest and platform', () => { expect(selectRuntime('node', 'linux/amd64', catalog()).image).toContain('@sha256:'); diff --git a/test/cso-runtime-promotion.test.ts b/test/cso-runtime-promotion.test.ts index 74383245d..212e34a66 100644 --- a/test/cso-runtime-promotion.test.ts +++ b/test/cso-runtime-promotion.test.ts @@ -69,7 +69,10 @@ describe('CSO runtime catalog promotion', () => { const matrix = imageBuildMatrix(buildInputs); expect(matrix.include).toHaveLength(10); expect(committedCatalog.profiles).toHaveLength(10); - expect(committedCatalog.runtimes).toEqual([]); + for (const runtime of committedCatalog.runtimes) { + expect(committedCatalog.profiles.filter(profile => profile.id === runtime.id && profile.stack === runtime.stack && profile.platform === runtime.platform)).toHaveLength(1); + expect(runtime.image).toMatch(/@sha256:[a-f0-9]{64}$/); + } expect(committedCatalog.buildRevision).toBe(buildInputs.revision); expect(matrix.include.find(row => row.stack === 'bun')!.versions).not.toHaveProperty('node'); validateRuntimeCatalog(committedCatalog); diff --git a/test/cso-scanner-executor.test.ts b/test/cso-scanner-executor.test.ts index 6024462d8..bae145946 100644 --- a/test/cso-scanner-executor.test.ts +++ b/test/cso-scanner-executor.test.ts @@ -10,7 +10,7 @@ import { total } from '../lib/cso/admission'; import { SCANNER_IDS, ScannerExecution, ScannerId, scannerPlans } from '../lib/cso/scanners'; import { type PreparationSandboxRunner } from '../lib/cso/preparation-executor'; import { canonicalStartPlan } from '../lib/cso/verification'; -import { completeRuntimeCatalogFixture } from './helpers/cso-runtime-catalog'; +import { completeRuntimeCatalogFixture, unqualifiedRuntimeCatalog } from './helpers/cso-runtime-catalog'; const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); }); @@ -186,7 +186,7 @@ describe('Schemathesis application admission', () => { expect(calls).toBe(0); expect(out.outcome.gaps[0].message).toContain('boundary is missing or transformed'); }); test('no runtime profile means a prerequisite, not a scanner success', async () => { - const out = await executeScanner(input('schemathesis'), { catalog: catalog('schemathesis') }); + const out = await executeScanner(input('schemathesis'), { catalog: catalog('schemathesis'), runtimes: unqualifiedRuntimeCatalog() }); expect(out.outcome.status).toBe('not_assessed'); expect(out.outcome.gaps[0].message).toContain('runtime is unavailable'); }); test.each(['remote-ref', 'server', 'hook', 'callback', 'operation', 'unbounded', 'source-command'])('rejects hostile or ambiguous %s harness input', kind => { diff --git a/test/helpers/cso-runtime-catalog.ts b/test/helpers/cso-runtime-catalog.ts index f004896b7..1e55877d1 100644 --- a/test/helpers/cso-runtime-catalog.ts +++ b/test/helpers/cso-runtime-catalog.ts @@ -5,6 +5,8 @@ import { canonical, sha256 } from '../../lib/cso/contracts'; import { ISOLATION_POLICY_HASH } from '../../lib/cso/docker'; import { CSO_HELPER_ABI, + RUNTIME_CATALOG, + selectRuntime, type QualifiedRuntime, type RuntimeCatalog, type RuntimePlatform, @@ -125,3 +127,22 @@ export function installStagedRuntime(catalog: RuntimeCatalog, runtime: Qualified catalog.promotion.evidenceDigest = `sha256:${sha256(canonical(catalog.runtimes))}`; return installed; } + +/** The given catalog with its promotion removed: every reviewed profile is still awaiting a qualified image. */ +export function unqualifiedRuntimeCatalog(base: RuntimeCatalog = RUNTIME_CATALOG): RuntimeCatalog { + const { promotion: _promotion, ...reviewed } = base; + return { ...reviewed, profiles: reviewed.profiles.map(profile => ({ ...profile, versions: { ...profile.versions } })), runtimes: [] }; +} + +/** + * The fail-closed rejection replay reaches at runtime admission for a fixture + * bundle whose image digest no qualified catalog entry carries. + */ +export function replayRuntimeRejection(profile: string, platform: RuntimePlatform): { code: string; message: string } { + try { + selectRuntime(profile, platform); + } catch (error) { + return { code: 'PREREQUISITE', message: String((error as Error).message).split(':')[0] }; + } + return { code: 'INCOMPATIBLE_INPUT', message: 'Qualified runtime digest does not match the bundle' }; +}