test(cso): stop pinning the committed runtime catalog to empty

CHANGELOG.md

@@ -7,6 +7,9 @@
 ### Added
 - **`lib/cso/runtime-catalog.json`** is now catalog `cso-v3-39674065531-133cb360842d`, which replaces `cso-v3-build-reviewed-2026-09-10`. Promote run 39841502986 generated it from qualification run 39674065531's authenticated statements and attested it. GitHub Actions committed the attested bytes (`sha256:078e0bf6…`). Each runtime's image is pinned to a public `ghcr.io/garrytan/gstack/cso-staging/…@sha256` digest, and each entry records its source commit (133cb36), SBOM and provenance digests, and requalification triggers (`preparationSha256` `c721b585…`).
 
+### For contributors
+- **The /cso free tests no longer assume the committed runtime catalog is empty.** Six tests pinned `runtimes: []` and broke as soon as a promotion landed. Tests that need an unqualified catalog now build one with `unqualifiedRuntimeCatalog()`. The replay tests expect whichever fail-closed rejection the committed catalog produces, so the suite passes with the catalog both before and after a promotion.
+
 ## [1.91.177.0] - 2026-10-22
 
 **The /cso runtime promotion workflow can finish.** Promote run 39691485727 failed after its `cso-runtime-release` approval. Its "Generate a fail-closed catalog candidate" step calls `gh api …/compare` to prove the staged commit is an ancestor of the qualification head, but the step had no `GH_TOKEN`. On a hosted runner `gh` then exits 4 ("set the GH_TOKEN environment variable"). The run stopped before it signed, pushed or opened anything.

test/cso-audit-paths.test.ts

@@ -11,6 +11,7 @@ import { newRun, readJson, saveReport } from '../lib/cso/state';
 import { MAX_OUTPUT, canonical, sha256, type VerificationObservation } from '../lib/cso/contracts';
 import { ISOLATION_POLICY_HASH } from '../lib/cso/docker';
 import { canonicalStartPlan, canonicalTestPlan, patchHash, verifyRepair } from '../lib/cso/verification';
+import { replayRuntimeRejection } from './helpers/cso-runtime-catalog';
 
 const ROOT = path.resolve(import.meta.dir, '..');
 const launcher = path.join(ROOT, 'bin', process.platform === 'win32' ? 'gstack-cso-launcher.exe' : 'gstack-cso-launcher');
@@ -342,6 +343,6 @@ describe('CSO --paths audit scope', () => {
     const supplied = cso(repo, ['replay', verified.bundle.id, '--source', repo]);
     expect(supplied.status).not.toBe(0);
     expect(supplied.stderr).not.toContain('does not match the bundle input hashes');
-    expect(supplied.stderr).toContain('MISSING_QUALIFIED_RUNTIME');
+    expect(supplied.stderr).toContain(replayRuntimeRejection(request.runtimeProfile, `linux/${arch}`).message);
   }, 60_000);
 });

test/cso-cli-lifecycle.test.ts

@@ -6,6 +6,7 @@ import { spawnSync } from 'node:child_process';
 import { canonical, sha256, type VerificationObservation } from '../lib/cso/contracts';
 import { ISOLATION_POLICY_HASH } from '../lib/cso/docker';
 import { saveReport, withLock } from '../lib/cso/state';
+import { replayRuntimeRejection } from './helpers/cso-runtime-catalog';
 import { canonicalStartPlan, canonicalTestPlan, patchHash, verifyRepair } from '../lib/cso/verification';
 
 const ROOT=path.resolve(import.meta.dir,'..'),launcher=path.join(ROOT,'bin',process.platform==='win32'?'gstack-cso-launcher.exe':'gstack-cso-launcher');
@@ -64,23 +65,23 @@ describe('CSO interrupted-run and replay commands',()=>{
     const finding={title:'Unprotected record read',rootCause:'Record lookup omits the caller authorization predicate',location:{path:'app.js',line:1,symbol:'recordLookup'},advisoryIds:[],severity:'high',confidence:'high',confidenceRationale:'The caller-to-record trace directly establishes the missing authorization predicate',evidence:'supported',attackerControl:'Authenticated caller chooses the record identifier',impact:'Another tenant record is returned',scenario:'A tenant supplies another tenant record identifier and receives the record',trace:['GET /records/:id','recordLookup','record response'],references:['app.js:1','OWASP API1:2023'],recommendation:'Bind the lookup to the authenticated caller before returning the record',challenge:{reviewer:'independent-reviewer',independent:true,mode:'independent_agent',callers:'Authenticated route forwards the record identifier',controls:'Authentication exists without record authorization',counterevidence:'Opaque identifiers reduce guessing but do not authorize known identifiers',conclusion:'The record lookup lacks a caller authorization predicate'}};
     expect(command(['submit',run.runId,writeInput('replay-finding.json',{findings:[finding]})]).status).toBe(0);const reportPath=path.join(dir,'report.json'),report=JSON.parse(fs.readFileSync(reportPath,'utf8')),findingId=report.findings[0].id,start=canonicalStartPlan(snapshot,'node',3456),tests=canonicalTestPlan(snapshot,'node');
     const request:any={findingId,runtimeProfile:`node-24.4.0-${process.arch==='arm64'?'arm64':'amd64'}`,port:3456,start:start.command,legitimate:[{name:'legitimate control',path:'/control',method:'GET',expected:{status:200,includes:'CONTROL_OK'}}],security:{name:'unauthorized record is denied',path:'/records/other',method:'GET',expected:{status:403,includes:'DENIED'},vulnerable:{status:200,includes:'RECORD'}},existingTests:tests.commands,fixtures:{},boundaryFiles:['app.js'],testFiles:tests.files,changes:[{path:'app.js',beforeSha256:sha256(fs.readFileSync(path.join(snapshot,'app.js'))),after:'module.exports = "fixed"\n',effect:'source'}],review:{reviewer:'independent-reviewer',independent:true,rootCauseRepaired:true,featurePreserved:true,boundaryMocks:false,rationale:'The caller predicate is added while the legitimate control and existing test remain unchanged.',reviewedPatchHash:''}};request.review.reviewedPatchHash=patchHash(request);
-    const platform=process.arch==='arm64'?'linux/arm64':'linux/amd64',runtime:any={id:request.runtimeProfile,stack:'node',platform,image:`ghcr.io/garrytan/gstack/cso-staging/node-${process.arch==='arm64'?'arm64':'amd64'}@sha256:${'b'.repeat(64)}`},executor={observe:async(_source:string,phase:'before'|'after',received:any,_runtime:any,_verifier:any,_work:string,_control:string,_execution:any,evidence:any,witness:any)=>{const observation:VerificationObservation={booted:true,legitimate:true,security:phase==='before'?'intended_failure':'pass',existingTests:false,output:`external ${phase} assertions passed`,inputHash:''},receipt=await witness.attest(observation,[{command:received.existingTests[0],code:0,output:tap,minimumPassingTests:evidence.minimumPassingTests[0]}]);return{observation:{...observation,existingTests:receipt.diagnosticTestsPassed,inputHash:witness.binding.harnessHash},witness:receipt};}};
+    const platform=process.arch==='arm64'?'linux/arm64':'linux/amd64',rejection=replayRuntimeRejection(request.runtimeProfile,platform),runtime:any={id:request.runtimeProfile,stack:'node',platform,image:`ghcr.io/garrytan/gstack/cso-staging/node-${process.arch==='arm64'?'arm64':'amd64'}@sha256:${'b'.repeat(64)}`},executor={observe:async(_source:string,phase:'before'|'after',received:any,_runtime:any,_verifier:any,_work:string,_control:string,_execution:any,evidence:any,witness:any)=>{const observation:VerificationObservation={booted:true,legitimate:true,security:phase==='before'?'intended_failure':'pass',existingTests:false,output:`external ${phase} assertions passed`,inputHash:''},receipt=await witness.attest(observation,[{command:received.existingTests[0],code:0,output:tap,minimumPassingTests:evidence.minimumPassingTests[0]}]);return{observation:{...observation,existingTests:receipt.diagnosticTestsPassed,inputHash:witness.binding.harnessHash},witness:receipt};}};
     const verified=await verifyRepair({runId:run.runId,runDir:dir,manifest,rawRequest:request,runtime,verifier:runtime,policyHash:ISOLATION_POLICY_HASH,auditPolicyHash:sha256(canonical(report.policy)),archives:[],executor});
     report.findings[0].reproduction='reproduced';report.findings[0].repair='runtime_tested';report.findings[0].verificationId=verified.bundle.id;report.findings[0].verificationAssurance={assertions:'authenticated_out_of_process',testCompletion:'self_reported',review:'self_attested'};saveReport(dir,report);
 
     const leased=withLock(dir,()=>command(['replay',verified.bundle.id])) as ReturnType<typeof command>;expect(leased.status).not.toBe(0);expect(leased.stderr).toContain('INSUFFICIENT_CAPACITY');expect(leased.stderr).toContain('Another helper is updating this run');
-    const retained=command(['replay',verified.bundle.id]);expect(retained.status).not.toBe(0);expect(retained.stderr).toContain('PREREQUISITE');expect(retained.stderr).toContain('MISSING_QUALIFIED_RUNTIME');
+    const retained=command(['replay',verified.bundle.id]);expect(retained.status).not.toBe(0);expect(retained.stderr).toContain(rejection.code);expect(retained.stderr).toContain(rejection.message);
     const manifestPath=path.join(dir,'snapshot.json'),expiredManifest={...manifest,expiresAt:new Date(Date.now()-1_000).toISOString()};fs.writeFileSync(manifestPath,`${JSON.stringify(expiredManifest,null,2)}\n`);
     const expiredPresent=command(['replay',verified.bundle.id]);expect(expiredPresent.status).not.toBe(0);expect(expiredPresent.stderr).toContain('MISSING_INPUT');expect(expiredPresent.stderr).toContain('Retained source expired');
-    const suppliedExpired=command(['replay',verified.bundle.id,'--source',repo]);expect(suppliedExpired.status).not.toBe(0);expect(suppliedExpired.stderr).toContain('PREREQUISITE');expect(suppliedExpired.stderr).toContain('MISSING_QUALIFIED_RUNTIME');expect(suppliedExpired.stderr).not.toContain('Retained source expired');
+    const suppliedExpired=command(['replay',verified.bundle.id,'--source',repo]);expect(suppliedExpired.status).not.toBe(0);expect(suppliedExpired.stderr).toContain(rejection.code);expect(suppliedExpired.stderr).toContain(rejection.message);expect(suppliedExpired.stderr).not.toContain('Retained source expired');
     const noncanonicalExpiry=new Date(Date.now()+86400_000).toISOString().replace(/\.\d{3}Z$/,'Z');fs.writeFileSync(manifestPath,`${JSON.stringify({...manifest,expiresAt:noncanonicalExpiry},null,2)}\n`);
-    const invalidExpiry=command(['replay',verified.bundle.id,'--source',repo]);expect(invalidExpiry.status).not.toBe(0);expect(invalidExpiry.stderr).toContain('INCOMPATIBLE_INPUT');expect(invalidExpiry.stderr).toContain('Retained snapshot expiry is invalid');expect(invalidExpiry.stderr).not.toContain('MISSING_QUALIFIED_RUNTIME');
+    const invalidExpiry=command(['replay',verified.bundle.id,'--source',repo]);expect(invalidExpiry.status).not.toBe(0);expect(invalidExpiry.stderr).toContain('INCOMPATIBLE_INPUT');expect(invalidExpiry.stderr).toContain('Retained snapshot expiry is invalid');expect(invalidExpiry.stderr).not.toContain(rejection.message);
     fs.writeFileSync(manifestPath,`${JSON.stringify({...manifest,expiresAt:new Date(Date.now()+86400_000).toISOString()},null,2)}\n`);fs.writeFileSync(path.join(snapshot,'app.js'),'module.exports = "tampered"\n');
-    const corruptRetained=command(['replay',verified.bundle.id,'--source',repo]);expect(corruptRetained.status).not.toBe(0);expect(corruptRetained.stderr).toContain('INCOMPATIBLE_INPUT');expect(corruptRetained.stderr).toContain('Retained snapshot changed');expect(corruptRetained.stderr).not.toContain('MISSING_QUALIFIED_RUNTIME');
+    const corruptRetained=command(['replay',verified.bundle.id,'--source',repo]);expect(corruptRetained.status).not.toBe(0);expect(corruptRetained.stderr).toContain('INCOMPATIBLE_INPUT');expect(corruptRetained.stderr).toContain('Retained snapshot changed');expect(corruptRetained.stderr).not.toContain(rejection.message);
     fs.rmSync(snapshot,{recursive:true,force:true});fs.rmSync(path.join(dir,'readable'),{recursive:true,force:true});
     const expired=command(['replay',verified.bundle.id]);expect(expired.status).not.toBe(0);expect(expired.stderr).toContain('MISSING_INPUT');expect(expired.stderr).toContain('Retained source expired');
     fs.writeFileSync(path.join(repo,'app.js'),'module.exports = "changed"\n');const changed=command(['replay',verified.bundle.id,'--source',repo]);expect(changed.status).not.toBe(0);expect(changed.stderr).toContain('INCOMPATIBLE_INPUT');expect(changed.stderr).toContain('does not match the bundle input hashes');
-    fs.writeFileSync(path.join(repo,'app.js'),'module.exports = "vulnerable"\n');const supplied=command(['replay',verified.bundle.id,'--source',repo]);expect(supplied.status).not.toBe(0);expect(supplied.stderr).toContain('PREREQUISITE');expect(supplied.stderr).toContain('MISSING_QUALIFIED_RUNTIME');expect(supplied.stderr).not.toContain('does not match the bundle input hashes');
+    fs.writeFileSync(path.join(repo,'app.js'),'module.exports = "vulnerable"\n');const supplied=command(['replay',verified.bundle.id,'--source',repo]);expect(supplied.status).not.toBe(0);expect(supplied.stderr).toContain(rejection.code);expect(supplied.stderr).toContain(rejection.message);expect(supplied.stderr).not.toContain('does not match the bundle input hashes');
   },30_000);
 });
 

test/cso-preparation-executor.test.ts

@@ -13,7 +13,7 @@ import {
   type PreparationSandboxRunner, type OfflinePreparationRequest, preparedIdentityEntry } from '../lib/cso/preparation-executor';
 import { CSO_HELPER_ABI } from '../lib/cso/runtime-catalog';
 import { DockerPreparationSandboxRunner, isBlockedRegistryAddress, RegistryEgressBroker } from '../lib/cso/preparation-docker';
-import { completeRuntimeCatalogFixture, qualifiedRuntimeFixture } from './helpers/cso-runtime-catalog';
+import { completeRuntimeCatalogFixture, qualifiedRuntimeFixture, unqualifiedRuntimeCatalog } from './helpers/cso-runtime-catalog';
 
 const roots: string[] = [];
 const digest = 'a'.repeat(64);
@@ -138,9 +138,9 @@ function errorCode(error: unknown): string | undefined { return error instanceof
 afterEach(() => { for (const value of roots.splice(0)) fs.rmSync(value, { recursive: true, force: true }); });
 
 describe('CSO constrained dependency preparation executor', () => {
-  test('the unqualified default catalog and forged admissions fail closed', async () => {
+  test('an unqualified catalog and forged admissions fail closed', async () => {
     const source = snapshot('node'), plan = inspectPreparation(source, 'node'), runner = new FakeRunner(), cache = cacheFixture();
-    expect(() => admitPreparationRuntime({ plan, platform: 'linux/amd64' })).toThrow('MISSING_QUALIFIED_RUNTIME');
+    expect(() => admitPreparationRuntime({ plan, platform: 'linux/amd64', catalog: unqualifiedRuntimeCatalog() })).toThrow('MISSING_QUALIFIED_RUNTIME');
     const executor = new PreparationExecutor({ cache, runner });
     try {
       await executor.acquire({ plan, admission: { schemaVersion: 1, catalogRevision: 'forged', runtime: runtime('node') },

test/cso-preparation.test.ts

@@ -4,7 +4,7 @@ import { join } from 'node:path';
 import { tmpdir } from 'node:os';
 import { inspectPreparation, railsTestConfiguration } from '../lib/cso/preparation';
 import { assertRuntimeCompatible, RUNTIME_CATALOG, rollbackCatalog, selectRuntime, validateRuntimeCatalog } from '../lib/cso/runtime-catalog';
-import { completeRuntimeCatalogFixture, qualifiedRuntimeFixture } from './helpers/cso-runtime-catalog';
+import { completeRuntimeCatalogFixture, qualifiedRuntimeFixture, unqualifiedRuntimeCatalog } from './helpers/cso-runtime-catalog';
 
 const roots: string[] = [];
 const hash = 'a'.repeat(64);
@@ -231,10 +231,13 @@ describe('CSO inert Rails preparation', () => {
 describe('CSO runtime catalog admission', () => {
   const runtime = () => qualifiedRuntimeFixture('node');
   const catalog = () => completeRuntimeCatalogFixture('test-1');
-  test('default catalog cannot execute unqualified images', () => {
+  test('default catalog admits only digest-pinned reviewed profiles and unqualified profiles cannot execute', () => {
     expect(RUNTIME_CATALOG.profiles).toHaveLength(10);
-    expect(RUNTIME_CATALOG.runtimes).toEqual([]);
-    expect(() => selectRuntime('node', 'linux/amd64')).toThrow('Reviewed build profile node-24.4.0-amd64 is awaiting a qualified image promotion');
+    for (const runtime of RUNTIME_CATALOG.runtimes) {
+      expect(RUNTIME_CATALOG.profiles.filter(profile => profile.id === runtime.id && profile.stack === runtime.stack && profile.platform === runtime.platform)).toHaveLength(1);
+      expect(runtime.image).toMatch(/@sha256:[a-f0-9]{64}$/);
+    }
+    expect(() => selectRuntime('node', 'linux/amd64', unqualifiedRuntimeCatalog())).toThrow('Reviewed build profile node-24.4.0-amd64 is awaiting a qualified image promotion');
   });
   test('selects exact qualified digest and platform', () => {
     expect(selectRuntime('node', 'linux/amd64', catalog()).image).toContain('@sha256:');

test/cso-runtime-promotion.test.ts

@@ -69,7 +69,10 @@ describe('CSO runtime catalog promotion', () => {
     const matrix = imageBuildMatrix(buildInputs);
     expect(matrix.include).toHaveLength(10);
     expect(committedCatalog.profiles).toHaveLength(10);
-    expect(committedCatalog.runtimes).toEqual([]);
+    for (const runtime of committedCatalog.runtimes) {
+      expect(committedCatalog.profiles.filter(profile => profile.id === runtime.id && profile.stack === runtime.stack && profile.platform === runtime.platform)).toHaveLength(1);
+      expect(runtime.image).toMatch(/@sha256:[a-f0-9]{64}$/);
+    }
     expect(committedCatalog.buildRevision).toBe(buildInputs.revision);
     expect(matrix.include.find(row => row.stack === 'bun')!.versions).not.toHaveProperty('node');
     validateRuntimeCatalog(committedCatalog);

test/cso-scanner-executor.test.ts

@@ -10,7 +10,7 @@ import { total } from '../lib/cso/admission';
 import { SCANNER_IDS, ScannerExecution, ScannerId, scannerPlans } from '../lib/cso/scanners';
 import { type PreparationSandboxRunner } from '../lib/cso/preparation-executor';
 import { canonicalStartPlan } from '../lib/cso/verification';
-import { completeRuntimeCatalogFixture } from './helpers/cso-runtime-catalog';
+import { completeRuntimeCatalogFixture, unqualifiedRuntimeCatalog } from './helpers/cso-runtime-catalog';
 
 const roots: string[] = [];
 afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); });
@@ -186,7 +186,7 @@ describe('Schemathesis application admission', () => {
     expect(calls).toBe(0); expect(out.outcome.gaps[0].message).toContain('boundary is missing or transformed');
   });
   test('no runtime profile means a prerequisite, not a scanner success', async () => {
-    const out = await executeScanner(input('schemathesis'), { catalog: catalog('schemathesis') });
+    const out = await executeScanner(input('schemathesis'), { catalog: catalog('schemathesis'), runtimes: unqualifiedRuntimeCatalog() });
     expect(out.outcome.status).toBe('not_assessed'); expect(out.outcome.gaps[0].message).toContain('runtime is unavailable');
   });
   test.each(['remote-ref', 'server', 'hook', 'callback', 'operation', 'unbounded', 'source-command'])('rejects hostile or ambiguous %s harness input', kind => {

test/helpers/cso-runtime-catalog.ts

@@ -5,6 +5,8 @@ import { canonical, sha256 } from '../../lib/cso/contracts';
 import { ISOLATION_POLICY_HASH } from '../../lib/cso/docker';
 import {
   CSO_HELPER_ABI,
+  RUNTIME_CATALOG,
+  selectRuntime,
   type QualifiedRuntime,
   type RuntimeCatalog,
   type RuntimePlatform,
@@ -125,3 +127,22 @@ export function installStagedRuntime(catalog: RuntimeCatalog, runtime: Qualified
   catalog.promotion.evidenceDigest = `sha256:${sha256(canonical(catalog.runtimes))}`;
   return installed;
 }
+
+/** The given catalog with its promotion removed: every reviewed profile is still awaiting a qualified image. */
+export function unqualifiedRuntimeCatalog(base: RuntimeCatalog = RUNTIME_CATALOG): RuntimeCatalog {
+  const { promotion: _promotion, ...reviewed } = base;
+  return { ...reviewed, profiles: reviewed.profiles.map(profile => ({ ...profile, versions: { ...profile.versions } })), runtimes: [] };
+}
+
+/**
+ * The fail-closed rejection replay reaches at runtime admission for a fixture
+ * bundle whose image digest no qualified catalog entry carries.
+ */
+export function replayRuntimeRejection(profile: string, platform: RuntimePlatform): { code: string; message: string } {
+  try {
+    selectRuntime(profile, platform);
+  } catch (error) {
+    return { code: 'PREREQUISITE', message: String((error as Error).message).split(':')[0] };
+  }
+  return { code: 'INCOMPATIBLE_INPUT', message: 'Qualified runtime digest does not match the bundle' };
+}