v1.91.23.0 feat(cso): private two-architecture release scoring and self-checking runtime release gates

This commit is contained in:
garrytan committed 2026-10-05 01:07:53 +00:00
1 parent f97aefe654
commit 9b21621462
35 files changed
+2284 -235

No files matched your search

+98 -20
View File
@@ -27,6 +27,7 @@ CSO_NEW_LAUNCHER_INSTALLED=0
CSO_NEW_CORE_INSTALLED=0
CSO_NEW_WATCHDOG_INSTALLED=0
CSO_NEW_GENERATION_INSTALLED=0
CSO_EVALUATION_UNIT=""
cso_checkpoint() {
[ "${GSTACK_CSO_BUILD_TESTING:-0}" = 1 ] || return 0
@@ -87,6 +88,7 @@ cso_cleanup() {
if ! cso_restore_previous; then status=1; cleanup_stage=0; fi
fi
if [ "$cleanup_stage" -eq 1 ] && [ -n "$CSO_STAGE" ] && [ ! -f "$CSO_STAGE/.retain-recovery" ]; then rm -rf "$CSO_STAGE"; fi
if [ -n "$CSO_EVALUATION_UNIT" ]; then rm -rf "$CSO_EVALUATION_UNIT"; fi
exit "$status"
}
trap 'cso_cleanup $?' EXIT
@@ -143,6 +145,89 @@ cso_sha256() {
printf '%s\n' "$digest"
}
# Bun's compile-time switches stop configuration discovery in the audited
# working directory before the helper can apply its own policy.
cso_compile_bun() {
"$BUN_CMD" build --compile \
--no-compile-autoload-dotenv \
--no-compile-autoload-bunfig \
--no-compile-autoload-tsconfig \
--no-compile-autoload-package-json \
"$1" --outfile "$2"
chmod +x "$2"
}
cso_compile_watchdog() {
"$CSO_CC" -std=c11 -D_POSIX_C_SOURCE=200809L -O2 -Wall -Wextra \
lib/cso/watchdog.c -o "$1"
chmod +x "$1"
}
cso_compile_posix_launcher() {
CSO_LAUNCHER_FLAGS=""
case "$(uname -s)" in
Linux) CSO_LAUNCHER_FLAGS="-static" ;;
# Local ad-hoc signatures do not distinguish the launcher from another
# ad-hoc dylib. This section makes dyld prune DYLD_* before constructors.
Darwin) CSO_LAUNCHER_FLAGS="-Wl,-sectcreate,__RESTRICT,__restrict,/dev/null" ;;
esac
# shellcheck disable=SC2086 -- the optional platform flags are fixed literals.
"$CSO_CC" -std=c11 -D_POSIX_C_SOURCE=200809L -O2 -Wall -Wextra $CSO_LAUNCHER_FLAGS \
"-DGSTACK_CSO_CORE_SHA256=\"$2\"" lib/cso/launcher.c -o "$1"
}
# An evaluation candidate embeds a cso-eval-* runtime catalog in a private
# five-artifact producer unit outside this checkout. It never touches bin/, and
# its .gstack-cso-evaluation manifest marks it evaluation-only for setup.
cso_build_evaluation_candidate() {
[ "$#" -eq 4 ] && [ "$1" = --evaluation-candidate ] && [ "$3" = --output ] || {
echo 'Usage: build-cso.sh --evaluation-candidate <cso-eval-catalog.json> --output <new-directory>' >&2;return 64; }
[ -z "$CSO_EXE" ] || { echo 'CSO evaluation candidates are built only on Linux and macOS.' >&2;return 69; }
command -v "$CSO_CC" >/dev/null 2>&1 || { echo 'CSO build requires a C compiler (cc/clang/gcc) for the trusted launcher and watchdog.' >&2;return 1; }
candidate_catalog=$2
[ -f "$candidate_catalog" ] && [ ! -L "$candidate_catalog" ] || { echo 'CSO evaluation catalog must be one regular file.' >&2;return 66; }
candidate_parent="$(cd "$(dirname "$4")" 2>/dev/null && pwd -P)" || { echo 'CSO evaluation output parent must be an existing directory.' >&2;return 73; }
candidate_output="$candidate_parent/$(basename "$4")"
if cso_present "$candidate_output";then echo 'CSO evaluation output must be a new directory.' >&2;return 73;fi
case "$candidate_output/" in "$CSO_BUILD_ROOT"/*) echo 'CSO evaluation output must be outside the gstack checkout, where setup and distribution never look.' >&2;return 73;;esac
candidate_revision="$(grep -o '"revision": *"cso-eval-[0-9]\{1,\}-[a-f0-9]\{12\}"' "$candidate_catalog" | head -n 1 | sed 's/.*"\(cso-eval-[^"]*\)"$/\1/' || true)"
[ -n "$candidate_revision" ] || { echo 'CSO evaluation candidates require a cso-eval-<runId>-<sha12> catalog revision.' >&2;return 65; }
CSO_STAGE="$(mktemp -d "${TMPDIR:-/tmp}/gstack-cso-evaluation.XXXXXX")"
CSO_EVALUATION_UNIT="$(mktemp -d "$candidate_parent/.gstack-cso-evaluation.XXXXXX")"
source_root="$CSO_STAGE/source"
mkdir -m 700 "$source_root" "$source_root/test"
cp -R lib scripts "$source_root/"
cp -R test/helpers "$source_root/test/"
cp "$candidate_catalog" "$source_root/lib/cso/runtime-catalog.json"
(cd "$source_root" && "$BUN_CMD" scripts/cso-runtime-promotion.ts check-evaluation lib/cso/runtime-catalog.json)
(cd "$source_root" && cso_compile_bun lib/cso/cli.ts "$CSO_EVALUATION_UNIT/gstack-cso-core")
(cd "$source_root" && cso_compile_bun scripts/cso-eval-producer.ts "$CSO_EVALUATION_UNIT/cso-eval-producer")
cso_compile_watchdog "$CSO_EVALUATION_UNIT/gstack-cso-watchdog"
if [ "$(uname -s)" = Darwin ]; then
cso_sign_macos_artifact "$CSO_EVALUATION_UNIT/gstack-cso-core" 0
cso_sign_macos_artifact "$CSO_EVALUATION_UNIT/gstack-cso-watchdog" 0
fi
candidate_core_sha256="$(cso_sha256 "$CSO_EVALUATION_UNIT/gstack-cso-core")"
cso_compile_posix_launcher "$CSO_EVALUATION_UNIT/gstack-cso-launcher" "$candidate_core_sha256"
chmod +x "$CSO_EVALUATION_UNIT/gstack-cso-launcher"
if [ "$(uname -s)" = Darwin ];then cso_sign_macos_artifact "$CSO_EVALUATION_UNIT/gstack-cso-launcher" 1;fi
printf '%s\n' "$candidate_core_sha256" > "$CSO_EVALUATION_UNIT/.gstack-cso-generation"
printf '{"schemaVersion":1,"evaluationOnly":true,"runtimeCatalogRevision":"%s","runtimeCatalogSha256":"%s","coreSha256":"%s"}\n' \
"$candidate_revision" "$(cso_sha256 "$source_root/lib/cso/runtime-catalog.json")" "$candidate_core_sha256" \
> "$CSO_EVALUATION_UNIT/.gstack-cso-evaluation"
for artifact in gstack-cso-core gstack-cso-launcher gstack-cso-watchdog cso-eval-producer;do
cso_valid_artifact "$CSO_EVALUATION_UNIT/$artifact" || { echo "Evaluation $artifact is not one executable regular file." >&2;return 1; }
done
cso_valid_generation "$CSO_EVALUATION_UNIT/.gstack-cso-generation" || { echo 'Evaluation generation manifest is invalid.' >&2;return 1; }
[ "$(cso_sha256 "$CSO_EVALUATION_UNIT/gstack-cso-core")" = "$candidate_core_sha256" ] || { echo 'Evaluation core changed after launcher binding.' >&2;return 1; }
"$CSO_EVALUATION_UNIT/gstack-cso-launcher" --version >/dev/null
chmod 0555 "$CSO_EVALUATION_UNIT"/gstack-cso-core "$CSO_EVALUATION_UNIT"/gstack-cso-launcher "$CSO_EVALUATION_UNIT"/gstack-cso-watchdog "$CSO_EVALUATION_UNIT"/cso-eval-producer
chmod 0444 "$CSO_EVALUATION_UNIT/.gstack-cso-generation" "$CSO_EVALUATION_UNIT/.gstack-cso-evaluation"
chmod 0755 "$CSO_EVALUATION_UNIT"
mv "$CSO_EVALUATION_UNIT" "$candidate_output"
CSO_EVALUATION_UNIT=""
printf '{"output":"%s","revision":"%s","evaluationOnly":true}\n' "$candidate_output" "$candidate_revision"
}
cso_publish_locked() {
[ "${GSTACK_CSO_PUBLISH_LOCKED:-}" = 1 ] || { echo 'CSO publication requires the native publisher lock.' >&2;return 73; }
[ "$#" -eq 1 ] && [ -d "$1" ] && [ ! -L "$1" ] || { echo 'CSO publication stage is invalid.' >&2;return 69; }
@@ -201,6 +286,16 @@ if [ "${1:-}" = __publish_locked ];then
cso_publish_locked "$@"
exit 0
fi
if [ "${1:-}" = --evaluation-candidate ];then
cso_build_evaluation_candidate "$@"
exit 0
fi
[ "$#" -eq 0 ] || { echo 'Usage: build-cso.sh [--evaluation-candidate <cso-eval-catalog.json> --output <new-directory>]' >&2;exit 64; }
# Distribution builds embed only the committed catalog, never an evaluation one.
if [ -f lib/cso/runtime-catalog.json ] && grep -q '"revision": *"cso-eval-' lib/cso/runtime-catalog.json;then
echo 'lib/cso/runtime-catalog.json is an evaluation-only catalog; distribution builds refuse it. Restore the committed catalog.' >&2
exit 65
fi
# This entrypoint can be run directly. Once it may publish a CSO generation,
# the previous whole-build completion proof no longer describes all outputs.
@@ -226,21 +321,13 @@ if [ -n "$CSO_EXE" ];then : > "$CSO_STAGE/.gstack-cso-generation.lock";fi
# These are compile-time switches. Bun otherwise discovers configuration in
# the audited working directory before the helper can apply its own policy.
"$BUN_CMD" build --compile \
--no-compile-autoload-dotenv \
--no-compile-autoload-bunfig \
--no-compile-autoload-tsconfig \
--no-compile-autoload-package-json \
lib/cso/cli.ts --outfile "$CSO_STAGE_CORE"
chmod +x "$CSO_STAGE_CORE"
cso_compile_bun lib/cso/cli.ts "$CSO_STAGE_CORE"
cso_checkpoint core
# POSIX process groups are required by the detached watchdog. Windows keeps
# comprehensive execution unavailable instead of substituting a weaker timer.
if [ -z "$CSO_EXE" ]; then
"$CSO_CC" -std=c11 -D_POSIX_C_SOURCE=200809L -O2 -Wall -Wextra \
lib/cso/watchdog.c -o "$CSO_STAGE_WATCHDOG"
chmod +x "$CSO_STAGE_WATCHDOG"
cso_compile_watchdog "$CSO_STAGE_WATCHDOG"
cso_checkpoint watchdog
fi
@@ -265,16 +352,7 @@ if [ -n "$CSO_EXE" ];then
-CoreSha256 "$CSO_CORE_SHA256" \
-GitExePath "$(cygpath -aw "$CSO_WINDOWS_GIT")"
else
CSO_LAUNCHER_FLAGS=""
case "$(uname -s)" in
Linux) CSO_LAUNCHER_FLAGS="-static" ;;
# Local ad-hoc signatures do not distinguish the launcher from another
# ad-hoc dylib. This section makes dyld prune DYLD_* before constructors.
Darwin) CSO_LAUNCHER_FLAGS="-Wl,-sectcreate,__RESTRICT,__restrict,/dev/null" ;;
esac
# shellcheck disable=SC2086 -- the optional platform flags are fixed literals.
"$CSO_CC" -std=c11 -D_POSIX_C_SOURCE=200809L -O2 -Wall -Wextra $CSO_LAUNCHER_FLAGS \
"-DGSTACK_CSO_CORE_SHA256=\"$CSO_CORE_SHA256\"" lib/cso/launcher.c -o "$CSO_STAGE_LAUNCHER"
cso_compile_posix_launcher "$CSO_STAGE_LAUNCHER" "$CSO_CORE_SHA256"
"$CSO_CC" -std=c11 -D_POSIX_C_SOURCE=200809L -O2 -Wall -Wextra lib/cso/publish-lock.c -o "$CSO_STAGE_LOCKER"
fi
chmod +x "$CSO_STAGE_LAUNCHER" "$CSO_STAGE_LOCKER"
+4 -1
View File
@@ -17,6 +17,8 @@ import { GeminiAdapter, prepareGeminiProducerState, removeGeminiProducerState }
import { PRICING } from '../test/helpers/pricing';
import type { ProviderAdapter, RunOpts, RunResult } from '../test/helpers/providers/types';
import {
PRODUCER_PLATFORMS,
producerHostPlatform,
producerInputHash,
producerArtifactInventoryHash,
producerInstallationIdentityHash,
@@ -198,7 +200,8 @@ function repositoryIdentity(root: string): string {
}
function validateCell(cell: ProducerCell): void {
if (!cell || !HEX.test(cell.id) || !cell.caseId || !['node', 'bun', 'python', 'rails'].includes(cell.stack) || !['vulnerable', 'fixed'].includes(cell.variant) || !['v2', 'v3'].includes(cell.version) || !['daily', 'comprehensive'].includes(cell.mode) || ![1, 2, 3].includes(cell.repetition) || !cell.model || !['claude', 'codex', 'gemini'].includes(cell.host) || !Number.isInteger(cell.budgetSeconds) || cell.budgetSeconds <= 60 || cell.budgetSeconds > 3600 || !HEX.test(cell.sourceHash) || !HEX.test(cell.skillHash)) throw new Error('INVALID_PRODUCER_INPUT');
if (!cell || !HEX.test(cell.id) || !cell.caseId || !['node', 'bun', 'python', 'rails'].includes(cell.stack) || !['vulnerable', 'fixed'].includes(cell.variant) || !['v2', 'v3'].includes(cell.version) || !['daily', 'comprehensive'].includes(cell.mode) || ![1, 2, 3].includes(cell.repetition) || !cell.model || !['claude', 'codex', 'gemini'].includes(cell.host) || !Number.isInteger(cell.budgetSeconds) || cell.budgetSeconds <= 60 || cell.budgetSeconds > 3600 || !HEX.test(cell.sourceHash) || !HEX.test(cell.skillHash) || (cell.platform !== undefined && !PRODUCER_PLATFORMS.includes(cell.platform))) throw new Error('INVALID_PRODUCER_INPUT');
if (cell.platform !== undefined && cell.platform !== producerHostPlatform()) throw new Error('UNMATCHED_PRODUCER_PLATFORM');
}
export interface ProducerHelperBinding {
+10
View File
@@ -5,6 +5,8 @@ export type ProducerVersion = 'v2' | 'v3';
export type ProducerMode = 'daily' | 'comprehensive';
export type ProducerStack = 'node' | 'bun' | 'python' | 'rails';
export type ProducerVariant = 'vulnerable' | 'fixed';
export type ProducerPlatform = 'linux/amd64' | 'linux/arm64';
export const PRODUCER_PLATFORMS: readonly ProducerPlatform[] = ['linux/amd64', 'linux/arm64'];
export interface ProducerCell {
id: string;
@@ -19,6 +21,8 @@ export interface ProducerCell {
budgetSeconds: number;
sourceHash: string;
skillHash: string;
/** Release and baseline cells pin the native producer platform; full-profile cells omit it. */
platform?: ProducerPlatform;
}
export interface ProducerSourceEntry {
@@ -122,6 +126,12 @@ export type ProducerReceiptIndex = Omit<ProducerReceipt, 'output' | 'error'> & {
export const sha256 = (value: string | Buffer): string => createHash('sha256').update(value).digest('hex');
/** The native platform this producer process runs on, or null when no CSO runtime platform matches it. */
export function producerHostPlatform(platform: string = process.platform, arch: string = process.arch): ProducerPlatform | null {
if (platform !== 'linux') return null;
return arch === 'x64' ? 'linux/amd64' : arch === 'arm64' ? 'linux/arm64' : null;
}
export function producerInputHash(input: ProducerInput): string {
return sha256(JSON.stringify(input));
}
+292 -69
View File
@@ -7,8 +7,9 @@ import { dirname, join, resolve } from 'node:path';
import { readBoundedStable } from '../lib/cso/bounded-file';
import { executable } from '../lib/cso/process';
import { atomicWriteSync } from '../lib/fs-atomic';
import { loadCorpusManifest, materializeCase, sourceFiles, STACKS, type CorpusManifest, type EvalStack, type EvalVariant } from '../test/fixtures/cso-eval/materialize';
import { loadCorpusManifest, materializeCase, publicCorpus, STACKS, validateCorpusManifest, type CorpusManifest, type EvalCorpus, type EvalStack, type EvalVariant } from '../test/fixtures/cso-eval/materialize';
import {
PRODUCER_PLATFORMS,
producerArtifactInventoryHash,
producerInputHash,
producerInstallationIdentityHash,
@@ -20,6 +21,7 @@ import {
type ProducerHost,
type ProducerInput,
type ProducerInstallationIdentity,
type ProducerPlatform,
type ProducerProviderIdentity,
type ProducerReceipt,
type ProducerReceiptIndex,
@@ -29,13 +31,53 @@ import {
export type EvalVersion = 'v2' | 'v3';
export type EvalMode = 'daily' | 'comprehensive';
export type Outcome = 'passed' | 'failed' | 'blocked' | 'not_attempted';
/**
* full: matched v2/v3 × 3 repetitions on one producer installation.
* release: v3 only, 1 repetition, each cell pinned to a native platform.
* baseline: v2 comprehensive only, 1 repetition; scored into a reusable EvalBaseline.
*/
export type EvalProfile = 'full' | 'release' | 'baseline';
export type EvalPlatform = ProducerPlatform;
type EvalCorpusManifest = CorpusManifest<string>;
export interface EvalCell extends ProducerCell {
id: string; caseId: string; stack: EvalStack; variant: EvalVariant; version: EvalVersion; mode: EvalMode;
repetition: 1 | 2 | 3; model: string; host: ProducerHost; budgetSeconds: number; sourceHash: string; skillHash: string;
repetition: 1 | 2 | 3; model: string; host: ProducerHost; budgetSeconds: number; sourceHash: string; skillHash: string; platform?: EvalPlatform;
}
export interface EvalMatrix {
interface EvalMatrixBase {
schemaVersion: 1; corpusVersion: string; corpusHash: string; model: string; host: ProducerHost;
budgets: { daily: number; comprehensive: number }; skillHashes: { v2: string; v3: string }; repetitions: 3; cells: EvalCell[];
budgets: { daily: number; comprehensive: number }; cells: EvalCell[];
}
export interface FullEvalMatrix extends EvalMatrixBase { profile?: undefined; skillHashes: { v2: string; v3: string }; repetitions: 3 }
export interface ReleaseEvalMatrix extends EvalMatrixBase {
profile: 'release'; skillHashes: { v3: string }; repetitions: 1;
/** Daily cells run on one platform; each comprehensive case is fixed to one platform. */
platforms: { daily: EvalPlatform; comprehensive: Record<string, EvalPlatform> };
/** Pre-registered external v2 baseline, or null when the regression gate stays unmeasured. */
baselineHash: string | null;
}
export interface BaselineEvalMatrix extends EvalMatrixBase {
profile: 'baseline'; skillHashes: { v2: string }; repetitions: 1;
platforms: { comprehensive: Record<string, EvalPlatform> };
}
export type EvalMatrix = FullEvalMatrix | ReleaseEvalMatrix | BaselineEvalMatrix;
interface EvalMatrixCommonOptions { model: string; host: ProducerHost; budgets?: { daily: number; comprehensive: number } }
export interface FullEvalMatrixOptions extends EvalMatrixCommonOptions { profile?: 'full'; skillHashes: { v2: string; v3: string } }
export interface ReleaseEvalMatrixOptions extends EvalMatrixCommonOptions { profile: 'release'; skillHashes: { v3: string }; platforms: ReleaseEvalMatrix['platforms']; baseline?: EvalBaseline }
export interface BaselineEvalMatrixOptions extends EvalMatrixCommonOptions { profile: 'baseline'; skillHashes: { v2: string }; platforms: BaselineEvalMatrix['platforms'] }
export type EvalMatrixOptions = FullEvalMatrixOptions | ReleaseEvalMatrixOptions | BaselineEvalMatrixOptions;
/**
* A scored v2 comprehensive run, reused by release matrices while every bound
* input is unchanged. Release matrices pin its hash before results exist.
*/
export interface EvalBaseline {
schemaVersion: 1; version: 'v2';
corpusVersion: string; corpusHash: string; model: string; host: ProducerHost; budgetSeconds: number; skillHash: string;
platforms: Record<string, EvalPlatform>;
matrixHash: string; producerBatchHash: string; modelUsed: string;
provider: { family: ProducerProviderIdentity['family']; policyRevision: string; version: string };
/** One entry per corpus case, in manifest order: whether the vulnerable cell found the seeded issue. */
cases: Array<{ caseId: string; found: boolean }>;
baselineHash: string;
}
export interface PortableSkillFile {
path: string;
@@ -280,8 +322,8 @@ function initializeFixtureRepository(path: string): void {
execFileSync(git, [...safe, 'commit', '--quiet', '-m', 'immutable evaluation fixture'], { cwd: path, env, stdio: 'ignore' });
}
function sourceEntries(caseId: string, variant: EvalVariant): ProducerSourceEntry[] {
return Object.entries(sourceFiles(caseId, variant)).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0).map(([path, contents]) => ({ path, sha256: sha256(contents), bytes: Buffer.byteLength(contents) }));
function sourceEntries(corpus: EvalCorpus, caseId: string, variant: EvalVariant): ProducerSourceEntry[] {
return Object.entries(corpus.sourceFiles(caseId, variant)).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0).map(([path, contents]) => ({ path, sha256: sha256(contents), bytes: Buffer.byteLength(contents) }));
}
/**
@@ -289,10 +331,15 @@ function sourceEntries(caseId: string, variant: EvalVariant): ProducerSourceEntr
* evaluator metadata; transfer one job at a time and never give the schedule
* or sibling jobs to the producing agent.
*/
export function prepareEvalJobs(matrix: EvalMatrix, payloads: { v2: string; v3: string }, destination: string, selectedCellIds?: string[]): PreparedEvalSchedule {
validateMatrix(matrix);
validatePortableSkillPayload(payloads.v2, 'v2'); validatePortableSkillPayload(payloads.v3, 'v3');
if (sha256(payloads.v2) !== matrix.skillHashes.v2 || sha256(payloads.v3) !== matrix.skillHashes.v3) throw new Error('EVAL_SKILL_HASH_MISMATCH');
export function prepareEvalJobs(matrix: EvalMatrix, payloads: Partial<Record<EvalVersion, string>>, destination: string, selectedCellIds?: string[], corpus: EvalCorpus = publicCorpus()): PreparedEvalSchedule {
validateMatrix(matrix, corpus.manifest);
const skillHashes: Partial<Record<EvalVersion, string>> = matrix.skillHashes;
const versions = (['v2', 'v3'] as const).filter(version => skillHashes[version] !== undefined);
if (!payloads || Object.keys(payloads).sort().join(',') !== versions.join(',')) throw new Error('EVAL_SKILL_HASH_MISMATCH');
for (const version of versions) {
validatePortableSkillPayload(payloads[version]!, version);
if (sha256(payloads[version]!) !== skillHashes[version]) throw new Error('EVAL_SKILL_HASH_MISMATCH');
}
const root = resolve(destination);
if (existsSync(root)) throw new Error('EVAL_DESTINATION_EXISTS');
const parent = dirname(root);
@@ -308,10 +355,10 @@ export function prepareEvalJobs(matrix: EvalMatrix, payloads: { v2: string; v3:
const jobs: PreparedEvalSchedule['jobs'] = [];
for (const cell of selected) {
const jobRoot = join(jobsRoot, cell.id); mkdirSync(jobRoot, { mode: 0o700 });
const source = materializeCase(cell.caseId, cell.variant, join(jobRoot, 'source'));
const source = materializeCase(cell.caseId, cell.variant, join(jobRoot, 'source'), corpus);
if (source.sourceHash !== cell.sourceHash) throw new Error('CORPUS_INTEGRITY_MISMATCH');
initializeFixtureRepository(source.path);
const input: ProducerInput = { schemaVersion: 1, cell, skill: payloads[cell.version], source: sourceEntries(cell.caseId, cell.variant) };
const input: ProducerInput = { schemaVersion: 1, cell, skill: payloads[cell.version]!, source: sourceEntries(corpus, cell.caseId, cell.variant) };
const inputHash = producerInputHash(input);
safeWriteNew(join(jobRoot, 'producer-input.json'), input);
jobs.push({ cellId: cell.id, relativePath: `jobs/${cell.id}`, inputHash });
@@ -386,9 +433,41 @@ function validateReceiptIndex(receipt: ProducerReceiptIndex, cell: EvalCell): vo
}
const matchedPairKey = (cell: EvalCell): string => [cell.caseId, cell.variant, cell.mode, cell.repetition, cell.model, cell.host, cell.budgetSeconds, cell.sourceHash].join('|');
const PROFILE_GROUPS: Record<EvalProfile, ReadonlyArray<readonly [EvalVersion, EvalMode]>> = {
full: [['v2', 'daily'], ['v2', 'comprehensive'], ['v3', 'daily'], ['v3', 'comprehensive']],
release: [['v3', 'daily'], ['v3', 'comprehensive']],
baseline: [['v2', 'comprehensive']],
};
const profileOf = (matrix: EvalMatrix): EvalProfile => matrix.profile ?? 'full';
const expectedMatchedPairs = (matrix: EvalMatrix): number => profileOf(matrix) === 'full' ? matrix.cells.length / 2 : 0;
export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEvalSchedule, receipts: ProducerReceipt[]): ProducerBatch {
validateMatrix(matrix); validateSchedule(matrix, schedule);
type IdentityBearing = Pick<ProducerReceiptIndex, 'cell' | 'installationIdentity' | 'providerIdentity'>;
/**
* Each platform has exactly one producer installation and provider binary.
* Platforms differ only in native binaries: the embedded catalogs and the
* reviewed provider policy must be identical across them.
*/
function assertPlatformIdentities(receipts: IdentityBearing[]): void {
const byPlatform = new Map<string, IdentityBearing[]>();
for (const receipt of receipts) {
const key = receipt.cell.platform ?? 'unpinned', list = byPlatform.get(key) ?? [];
list.push(receipt); byPlatform.set(key, list);
}
for (const list of byPlatform.values()) {
if (new Set(list.map(receipt => receipt.installationIdentity.identityHash)).size > 1) throw new Error('UNMATCHED_PRODUCER_INSTALLATIONS');
if (new Set(list.map(receipt => receipt.providerIdentity.identityHash)).size > 1) throw new Error('UNMATCHED_PRODUCER_PROVIDERS');
}
const representatives = [...byPlatform.values()].map(list => list[0]);
if (representatives.length < 2) return;
if (new Set(representatives.map(receipt => receipt.installationIdentity.identityHash)).size !== representatives.length) throw new Error('UNMATCHED_PRODUCER_PLATFORMS');
const catalogs = (receipt: IdentityBearing) => JSON.stringify(receipt.installationIdentity.embeddedCatalogs);
const policy = ({ providerIdentity: { family, policyRevision, version, argsPrefix } }: IdentityBearing) => JSON.stringify({ family, policyRevision, version, argsPrefix });
if (new Set(representatives.map(catalogs)).size !== 1) throw new Error('UNMATCHED_PRODUCER_INSTALLATIONS');
if (new Set(representatives.map(policy)).size !== 1) throw new Error('UNMATCHED_PRODUCER_PROVIDERS');
}
export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEvalSchedule, receipts: ProducerReceipt[], corpus: EvalCorpusManifest = loadCorpusManifest()): ProducerBatch {
validateMatrix(matrix, corpus); validateSchedule(matrix, schedule);
if (!Array.isArray(receipts)) throw new Error('INVALID_PRODUCER_RECEIPTS');
const cells = new Map(matrix.cells.map(cell => [cell.id, cell]));
const jobs = new Map(schedule.jobs.map(job => [job.cellId, job]));
@@ -398,18 +477,18 @@ export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEv
if (!cell || !job || seen.has(cell.id)) throw new Error('UNKNOWN_OR_DUPLICATE_PRODUCER_RECEIPT');
validateReceipt(receipt, cell, job.inputHash); seen.add(cell.id);
}
if (new Set(receipts.map(receipt => receipt.installationIdentity.identityHash)).size > 1) throw new Error('UNMATCHED_PRODUCER_INSTALLATIONS');
if (new Set(receipts.map(receipt => receipt.providerIdentity.identityHash)).size > 1) throw new Error('UNMATCHED_PRODUCER_PROVIDERS');
assertPlatformIdentities(receipts);
const pairs = new Map<string, Partial<Record<EvalVersion, ProducerReceipt>>>();
for (const receipt of receipts) {
if (profileOf(matrix) === 'full') for (const receipt of receipts) {
const key = matchedPairKey(receipt.cell as EvalCell);
const pair = pairs.get(key) ?? {}; pair[receipt.cell.version] = receipt; pairs.set(key, pair);
}
else if (new Set(receipts.map(receipt => receipt.modelUsed)).size > 1) throw new Error('UNMATCHED_EFFECTIVE_MODELS: single-version cells used different normalized model identities');
const completePairs = [...pairs.entries()].filter(([, pair]) => pair.v2 && pair.v3) as Array<[string, { v2: ProducerReceipt; v3: ProducerReceipt }]>;
const modelMismatches = completePairs.filter(([, pair]) => pair.v2.modelUsed !== pair.v3.modelUsed).map(([pair, value]) => ({ pair: digest(pair), v2: value.v2.modelUsed, v3: value.v3.modelUsed }));
if (modelMismatches.length) throw new Error(`UNMATCHED_EFFECTIVE_MODELS: ${modelMismatches.length} matched v2/v3 pair(s) used different normalized model identities`);
const groups: ProducerGroupSummary[] = [];
for (const version of ['v2', 'v3'] as const) for (const mode of ['daily', 'comprehensive'] as const) {
for (const [version, mode] of PROFILE_GROUPS[profileOf(matrix)]) {
const expected = matrix.cells.filter(cell => cell.version === version && cell.mode === mode);
const submitted = receipts.filter(receipt => receipt.cell.version === version && receipt.cell.mode === mode);
const tokens = submitted.flatMap(receipt => receipt.usage.inputTokens === null || receipt.usage.outputTokens === null ? [] : [receipt.usage.inputTokens + receipt.usage.outputTokens]);
@@ -424,29 +503,99 @@ export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEv
const indexes: ProducerReceiptIndex[] = receipts.map(({ output: _output, error, ...receipt }) => ({ ...receipt, ...(error ? { error: { code: error.code } } : {}) }));
const base = { schemaVersion: 1 as const, matrixHash: matrixHash(matrix), scheduleHash: digest(JSON.stringify(schedule)), receipts: indexes.sort((left, right) => left.cell.id.localeCompare(right.cell.id)), summary: {
scheduled: matrix.cells.length, prepared: schedule.preparedCells, submitted: receipts.length, missing: matrix.cells.length - receipts.length,
matchedPairsExpected: matrix.cells.length / 2, matchedPairsSubmitted: completePairs.length, modelMismatches, groups,
matchedPairsExpected: expectedMatchedPairs(matrix), matchedPairsSubmitted: completePairs.length, modelMismatches, groups,
note: 'Costs are pricing-table estimates. First-useful timing is unmeasured because the reused provider adapters return completed runs. Trusted findings and runtime outcomes require separate oracle adjudication.',
} };
return { ...base, batchHash: digest(JSON.stringify(base)) };
}
export function createEvalMatrix(options: { model: string; host: ProducerHost; skillHashes: { v2: string; v3: string }; budgets?: { daily: number; comprehensive: number } }, corpus: CorpusManifest = loadCorpusManifest()): EvalMatrix {
if (!options.model?.trim() || !['claude', 'codex', 'gemini'].includes(options.host) || !HEX.test(options.skillHashes.v2) || !HEX.test(options.skillHashes.v3) || options.skillHashes.v2 === options.skillHashes.v3) throw new Error('INVALID_MATCHED_EVAL_INPUT');
const budgets = options.budgets ?? { daily: 600, comprehensive: 1800 };
if (![budgets.daily, budgets.comprehensive].every(value => Number.isInteger(value) && value > 60 && value <= 3600)) throw new Error('INVALID_EVAL_BUDGET');
const cells: EvalCell[] = [];
for (const fixture of corpus.cases) for (const variant of ['vulnerable', 'fixed'] as const) for (const mode of ['daily', 'comprehensive'] as const) for (const repetition of [1, 2, 3] as const) for (const version of ['v2', 'v3'] as const) {
const cell = { caseId: fixture.id, stack: fixture.stack, variant, version, mode, repetition, model: options.model, host: options.host, budgetSeconds: budgets[mode], sourceHash: fixture.filesHash[variant], skillHash: options.skillHashes[version] };
cells.push({ id: digest(JSON.stringify(cell)), ...cell });
}
return { schemaVersion: 1, corpusVersion: corpus.version, corpusHash: digest(JSON.stringify(corpus)), model: options.model, host: options.host, budgets, skillHashes: options.skillHashes, repetitions: 3, cells };
function normalizedPlatforms(corpus: EvalCorpusManifest, value: unknown): Record<string, EvalPlatform> {
const assignment = value as Record<string, EvalPlatform>;
if (!assignment || typeof assignment !== 'object' || Array.isArray(assignment) || Object.keys(assignment).length !== corpus.cases.length ||
corpus.cases.some(fixture => !PRODUCER_PLATFORMS.includes(assignment[fixture.id]))) throw new Error('INVALID_EVAL_PLATFORMS');
return Object.fromEntries(corpus.cases.map(fixture => [fixture.id, assignment[fixture.id]]));
}
export function validateMatrix(matrix: EvalMatrix, corpus: CorpusManifest = loadCorpusManifest()): void {
const expected = createEvalMatrix({ model: matrix.model, host: matrix.host, skillHashes: matrix.skillHashes, budgets: matrix.budgets }, corpus);
function buildEvalMatrix(options: EvalMatrixOptions, corpus: EvalCorpusManifest, baselineHash: string | null): EvalMatrix {
const profile = options.profile ?? 'full';
const skillHashes: Partial<Record<EvalVersion, string>> = options.skillHashes ?? {};
const versions = profile === 'full' ? ['v2', 'v3'] : profile === 'release' ? ['v3'] : ['v2'];
if (!options.model?.trim() || !['claude', 'codex', 'gemini'].includes(options.host) || !['full', 'release', 'baseline'].includes(profile) ||
Object.keys(skillHashes).sort().join(',') !== versions.join(',') || versions.some(version => !HEX.test(skillHashes[version as EvalVersion] ?? '')) ||
(profile === 'full' && skillHashes.v2 === skillHashes.v3)) throw new Error('INVALID_MATCHED_EVAL_INPUT');
const budgets = options.budgets ?? { daily: 600, comprehensive: 1800 };
if (![budgets.daily, budgets.comprehensive].every(value => Number.isInteger(value) && value > 60 && value <= 3600)) throw new Error('INVALID_EVAL_BUDGET');
const corpusHash = digest(JSON.stringify(corpus));
const cells: EvalCell[] = [];
if (options.profile === undefined || options.profile === 'full') {
for (const fixture of corpus.cases) for (const variant of ['vulnerable', 'fixed'] as const) for (const mode of ['daily', 'comprehensive'] as const) for (const repetition of [1, 2, 3] as const) for (const version of ['v2', 'v3'] as const) {
const cell = { caseId: fixture.id, stack: fixture.stack, variant, version, mode, repetition, model: options.model, host: options.host, budgetSeconds: budgets[mode], sourceHash: fixture.filesHash[variant], skillHash: options.skillHashes[version] };
cells.push({ id: digest(JSON.stringify(cell)), ...cell });
}
return { schemaVersion: 1, corpusVersion: corpus.version, corpusHash, model: options.model, host: options.host, budgets, skillHashes: options.skillHashes, repetitions: 3, cells };
}
const pinned = options as ReleaseEvalMatrixOptions | BaselineEvalMatrixOptions;
const platformKeys = pinned.profile === 'release' ? 'comprehensive,daily' : 'comprehensive';
if (!pinned.platforms || Object.keys(pinned.platforms).sort().join(',') !== platformKeys) throw new Error('INVALID_EVAL_PLATFORMS');
const comprehensive = normalizedPlatforms(corpus, pinned.platforms.comprehensive);
const daily = pinned.profile === 'release' ? pinned.platforms.daily : undefined;
if (pinned.profile === 'release' && !PRODUCER_PLATFORMS.includes(daily!)) throw new Error('INVALID_EVAL_PLATFORMS');
const version: EvalVersion = pinned.profile === 'release' ? 'v3' : 'v2';
const modes = pinned.profile === 'release' ? ['daily', 'comprehensive'] as const : ['comprehensive'] as const;
for (const fixture of corpus.cases) for (const variant of ['vulnerable', 'fixed'] as const) for (const mode of modes) {
const cell = { caseId: fixture.id, stack: fixture.stack, variant, version, mode, repetition: 1 as const, model: pinned.model, host: pinned.host, budgetSeconds: budgets[mode], sourceHash: fixture.filesHash[variant], skillHash: skillHashes[version]!, platform: mode === 'daily' ? daily! : comprehensive[fixture.id] };
cells.push({ id: digest(JSON.stringify(cell)), ...cell });
}
const common = { schemaVersion: 1 as const, corpusVersion: corpus.version, corpusHash, model: pinned.model, host: pinned.host, budgets };
if (pinned.profile === 'release') {
if (baselineHash !== null && !HEX.test(baselineHash)) throw new Error('UNBOUND_EVAL_BASELINE');
return { ...common, profile: 'release', skillHashes: { v3: pinned.skillHashes.v3 }, repetitions: 1, platforms: { daily: daily!, comprehensive }, baselineHash, cells };
}
return { ...common, profile: 'baseline', skillHashes: { v2: pinned.skillHashes.v2 }, repetitions: 1, platforms: { comprehensive }, cells };
}
const BASELINE_FIELDS = 'baselineHash,budgetSeconds,cases,corpusHash,corpusVersion,host,matrixHash,model,modelUsed,platforms,producerBatchHash,provider,schemaVersion,skillHash,version';
/** Check that a baseline was measured under the same corpus, model, host, budget and platform assignment. */
function assertBaselineBinding(baseline: EvalBaseline, target: { corpus: EvalCorpusManifest; model: string; host: ProducerHost; budgetSeconds: number; platforms: Record<string, EvalPlatform> }): void {
if (!baseline || typeof baseline !== 'object' || Object.keys(baseline).sort().join(',') !== BASELINE_FIELDS) throw new Error('UNBOUND_EVAL_BASELINE');
const { baselineHash, ...record } = baseline;
const provider = baseline.provider as unknown as Record<string, unknown>;
if (baseline.schemaVersion !== 1 || baseline.version !== 'v2' || !HEX.test(baselineHash) || digest(JSON.stringify(record)) !== baselineHash ||
baseline.corpusVersion !== target.corpus.version || baseline.corpusHash !== digest(JSON.stringify(target.corpus)) ||
baseline.model !== target.model || baseline.host !== target.host || baseline.budgetSeconds !== target.budgetSeconds ||
JSON.stringify(baseline.platforms) !== JSON.stringify(target.platforms) || !HEX.test(baseline.skillHash) || !HEX.test(baseline.matrixHash) ||
!HEX.test(baseline.producerBatchHash) || typeof baseline.modelUsed !== 'string' || !baseline.modelUsed ||
!provider || typeof provider !== 'object' || Object.keys(provider).sort().join(',') !== 'family,policyRevision,version' ||
!['claude', 'gpt', 'gemini'].includes(String(provider.family)) || typeof provider.policyRevision !== 'string' || !provider.policyRevision ||
typeof provider.version !== 'string' || !provider.version || !Array.isArray(baseline.cases) || baseline.cases.length !== target.corpus.cases.length ||
baseline.cases.some((entry, index) => !entry || Object.keys(entry).sort().join(',') !== 'caseId,found' || entry.caseId !== target.corpus.cases[index].id || typeof entry.found !== 'boolean'))
throw new Error('UNBOUND_EVAL_BASELINE');
}
export function createEvalMatrix(options: FullEvalMatrixOptions, corpus?: EvalCorpusManifest): FullEvalMatrix;
export function createEvalMatrix(options: ReleaseEvalMatrixOptions, corpus?: EvalCorpusManifest): ReleaseEvalMatrix;
export function createEvalMatrix(options: BaselineEvalMatrixOptions, corpus?: EvalCorpusManifest): BaselineEvalMatrix;
export function createEvalMatrix(options: EvalMatrixOptions, corpus: EvalCorpusManifest = loadCorpusManifest()): EvalMatrix {
let baselineHash: string | null = null;
if (options.profile === 'release' && options.baseline !== undefined) {
assertBaselineBinding(options.baseline, { corpus, model: options.model, host: options.host, budgetSeconds: (options.budgets ?? { comprehensive: 1800 }).comprehensive, platforms: normalizedPlatforms(corpus, options.platforms?.comprehensive) });
baselineHash = options.baseline.baselineHash;
}
return buildEvalMatrix(options, corpus, baselineHash);
}
export function validateMatrix(matrix: EvalMatrix, corpus: EvalCorpusManifest = loadCorpusManifest()): void {
if (!matrix || typeof matrix !== 'object') throw new Error('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX');
const common = { model: matrix.model, host: matrix.host, budgets: matrix.budgets };
let expected: EvalMatrix;
try {
expected = matrix.profile === 'release' ? buildEvalMatrix({ ...common, profile: 'release', skillHashes: matrix.skillHashes, platforms: matrix.platforms }, corpus, matrix.baselineHash)
: matrix.profile === 'baseline' ? buildEvalMatrix({ ...common, profile: 'baseline', skillHashes: matrix.skillHashes, platforms: matrix.platforms }, corpus, null)
: buildEvalMatrix({ ...common, skillHashes: matrix.skillHashes }, corpus, null);
} catch { throw new Error('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX'); }
if (JSON.stringify(matrix) !== JSON.stringify(expected)) throw new Error('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX');
}
function validateResult(result: EvalResult, cell: EvalCell, corpus: CorpusManifest): void {
function validateResult(result: EvalResult, cell: EvalCell, corpus: EvalCorpusManifest): void {
if (result.sourceHash !== cell.sourceHash || result.skillHash !== cell.skillHash || result.model !== cell.model || result.host !== cell.host || result.budgetSeconds !== cell.budgetSeconds) throw new Error('UNMATCHED_EVAL_RESULT');
for (const field of ['setup', 'reproduction', 'repair', 'recheck'] as const) if (!['passed', 'failed', 'blocked', 'not_attempted'].includes(result[field])) throw new Error('INVALID_EVAL_OUTCOME');
for (const field of ['reportPresent', 'reportComplete', 'heldOutAssertionsPassed', 'freshRecheck'] as const) if (typeof result[field] !== 'boolean') throw new Error('INVALID_EVAL_RESULT');
@@ -493,7 +642,7 @@ function quantile(values: number[], fraction: number): number | null {
if (!values.length) return null;
return [...values].sort((left, right) => left - right)[Math.ceil(values.length * fraction) - 1];
}
function measuredRate(cells: EvalCell[], results: Map<string, EvalResult>, field: 'setup' | 'reproduction' | 'repair' | 'recheck', corpus: CorpusManifest): Rate {
function measuredRate(cells: EvalCell[], results: Map<string, EvalResult>, field: 'setup' | 'reproduction' | 'repair' | 'recheck', corpus: EvalCorpusManifest): Rate {
return rate(cells.filter(cell => {
const result = results.get(cell.id);
if (!result?.reportPresent || !result.reportComplete || result[field] !== 'passed') return false;
@@ -511,7 +660,7 @@ function measuredRate(cells: EvalCell[], results: Map<string, EvalResult>, field
}).length, cells.length);
}
function trustedFindingVerification(cell: EvalCell, result: EvalResult, finding: EvalFinding, stage: 'repair' | 'recheck', corpus: CorpusManifest): boolean {
function trustedFindingVerification(cell: EvalCell, result: EvalResult, finding: EvalFinding, stage: 'repair' | 'recheck', corpus: EvalCorpusManifest): boolean {
const verification = finding.trustedVerification;
if (!result.reportPresent || !result.reportComplete || cell.version !== 'v3' || cell.mode !== 'comprehensive' || cell.variant !== 'vulnerable' || finding.evidence !== 'supported' ||
finding.judgment !== 'correct' || finding.matchedCaseId !== cell.caseId || !finding.claimedTested || !verification ||
@@ -524,8 +673,7 @@ function trustedFindingVerification(cell: EvalCell, result: EvalResult, finding:
!!result.recheckEvidenceHash && result.recheckEvidenceHash !== result.oracleEvidenceHash;
}
export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualification: ReleaseQualification = { containment: {} }, corpus: CorpusManifest = loadCorpusManifest()) {
validateMatrix(matrix, corpus);
function validatedResults(matrix: EvalMatrix, observations: EvalResult[], corpus: EvalCorpusManifest): Map<string, EvalResult> {
if (!Array.isArray(observations)) throw new Error('INVALID_EVAL_RESULTS');
const cells = new Map(matrix.cells.map(cell => [cell.id, cell]));
const results = new Map<string, EvalResult>();
@@ -534,18 +682,34 @@ export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualif
if (!cell || results.has(result.cellId)) throw new Error('UNKNOWN_OR_DUPLICATE_EVAL_RESULT');
validateResult(result, cell, corpus); results.set(result.cellId, result);
}
const eligibleFinding = (cell: EvalCell, finding: EvalFinding) => finding.evidence === 'supported' || (cell.version === 'v2' && finding.evidence === 'legacy_review');
const found = (cell: EvalCell) => {
const result = results.get(cell.id);
return result?.reportPresent === true && result.reportComplete === true &&
result.findings.some(finding => eligibleFinding(cell, finding) && finding.judgment === 'correct');
};
return results;
}
const eligibleFinding = (cell: EvalCell, finding: EvalFinding) => finding.evidence === 'supported' || (cell.version === 'v2' && finding.evidence === 'legacy_review');
const foundSeededIssue = (cell: EvalCell, result: EvalResult | undefined) => result?.reportPresent === true && result.reportComplete === true &&
result.findings.some(finding => eligibleFinding(cell, finding) && finding.judgment === 'correct');
const isHighCritical = (corpus: EvalCorpusManifest, caseId: string) => ['critical', 'high'].includes(corpus.cases.find(fixture => fixture.id === caseId)!.severity);
/** High/critical recall of a pre-registered external baseline, after proving it binds to this release matrix. */
function externalBaselineRecall(matrix: ReleaseEvalMatrix, baseline: EvalBaseline, corpus: EvalCorpusManifest): Rate {
if (matrix.baselineHash === null || baseline?.baselineHash !== matrix.baselineHash) throw new Error('UNBOUND_EVAL_BASELINE');
assertBaselineBinding(baseline, { corpus, model: matrix.model, host: matrix.host, budgetSeconds: matrix.budgets.comprehensive, platforms: matrix.platforms.comprehensive });
const highCritical = baseline.cases.filter(entry => isHighCritical(corpus, entry.caseId));
return rate(highCritical.filter(entry => entry.found).length, highCritical.length);
}
export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualification: ReleaseQualification = { containment: {} }, corpus: EvalCorpusManifest = loadCorpusManifest(), baseline?: EvalBaseline) {
validateMatrix(matrix, corpus);
const profile = profileOf(matrix);
if (profile === 'baseline') throw new Error('BASELINE_PROFILE_NOT_QUALIFIABLE: use createEvalBaseline');
if (profile !== 'release' && baseline !== undefined) throw new Error('UNBOUND_EVAL_BASELINE');
const results = validatedResults(matrix, observations, corpus);
const found = (cell: EvalCell) => foundSeededIssue(cell, results.get(cell.id));
const groups: any[] = [];
for (const version of ['v2', 'v3'] as const) for (const mode of ['daily', 'comprehensive'] as const) {
for (const [version, mode] of PROFILE_GROUPS[profile]) {
const selected = matrix.cells.filter(cell => cell.version === version && cell.mode === mode);
const completed = selected.map(cell => results.get(cell.id)).filter((result): result is EvalResult => !!result);
const expected = selected.filter(cell => cell.variant === 'vulnerable');
const highCritical = expected.filter(cell => ['critical','high'].includes(corpus.cases.find(fixture => fixture.id === cell.caseId)!.severity));
const highCritical = expected.filter(cell => isHighCritical(corpus, cell.caseId));
let correct = 0, reported = 0, unadjudicated = 0;
for (const cell of selected) {
const result = results.get(cell.id);
@@ -576,7 +740,8 @@ export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualif
}
const daily = groups.find(group => group.version === 'v3' && group.mode === 'daily');
const comprehensive = groups.find(group => group.version === 'v3' && group.mode === 'comprehensive');
const baseline = groups.find(group => group.version === 'v2' && group.mode === 'comprehensive');
const inBatchBaseline = groups.find(group => group.version === 'v2' && group.mode === 'comprehensive');
const external = matrix.profile === 'release' && baseline !== undefined ? externalBaselineRecall(matrix, baseline, corpus) : null;
const perStack = Object.fromEntries(STACKS.map(stack => {
const eligible = matrix.cells.filter(cell => cell.version === 'v3' && cell.mode === 'comprehensive' && cell.variant === 'vulnerable' && cell.stack === stack);
// A repaired oracle case demonstrates the full find-to-repair workflow only
@@ -596,24 +761,27 @@ export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualif
mandatoryReports: gate(groups.every(group => group.reports.numerator === group.reports.denominator), observations.length === matrix.cells.length),
dailyPrecision95: gate(daily.precision.value === null ? null : daily.precision.value >= 0.95, daily.submitted === daily.cells && !daily.unadjudicated),
comprehensiveHighCriticalRecall80: gate(comprehensive.highCriticalRecall.value >= 0.8, comprehensive.submitted === comprehensive.cells && !comprehensive.unadjudicated),
noHighCriticalRecallRegression: gate(comprehensive.highCriticalRecall.value >= baseline.highCriticalRecall.value, comprehensive.submitted === comprehensive.cells && baseline.submitted === baseline.cells && !comprehensive.unadjudicated && !baseline.unadjudicated),
noHighCriticalRecallRegression: profile === 'full'
? gate(comprehensive.highCriticalRecall.value >= inBatchBaseline.highCriticalRecall.value, comprehensive.submitted === comprehensive.cells && inBatchBaseline.submitted === inBatchBaseline.cells && !comprehensive.unadjudicated && !inBatchBaseline.unadjudicated)
: gate(external === null ? null : comprehensive.highCriticalRecall.value >= external.value!, comprehensive.submitted === comprehensive.cells && !comprehensive.unadjudicated && external !== null),
allCoreColdStarts: gate(comprehensive.setup.numerator === comprehensive.setup.denominator, comprehensive.submitted === comprehensive.cells),
zeroFalselyTestedRepairs: gate(comprehensive.falseTested === 0 && daily.falseTested === 0, comprehensive.submitted === comprehensive.cells && daily.submitted === daily.cells),
heldOutRepairEachStack: gate(Object.values(perStack).every(value => value.correctHeldOutRepairs >= 1), comprehensive.submitted === comprehensive.cells),
containmentAndCanaries: containmentValues.includes('failed') ? 'fail' as const : containmentValues.every(value => value === 'passed') ? 'pass' as const : 'unmeasured' as const,
};
return { schemaVersion: 1, corpusVersion: corpus.version, model: matrix.model, host: matrix.host,
...(profile === 'release' ? { profile, baseline: external === null ? null : { baselineHash: baseline!.baselineHash, highCriticalRecall: external } } : {}),
status: Object.values(gates).every(value => value === 'pass') ? 'qualified' : observations.length ? 'partial' : 'unmeasured', groups, perStack, gates,
notes: ['Missing and setup-blocked supported scenarios remain in recall and workflow denominators.', 'Costs and tokens come only from host-reported usage; wall-clock budgets are not model-spend caps.', 'Oracle judgments and qualification receipts must come from the trusted evaluator, never producing agents.'] };
}
/** Release scoring path: every trusted judgment must bind to a collected producer receipt. */
export function scoreCollectedEval(matrix: EvalMatrix, batch: ProducerBatch, observations: EvalResult[], qualification: ReleaseQualification = { containment: {} }, corpus: CorpusManifest = loadCorpusManifest()) {
validateMatrix(matrix, corpus);
/** Bind a complete collected batch and every trusted judgment to the matrix's producer receipts. */
function validateCollectedBatch(matrix: EvalMatrix, batch: ProducerBatch, observations: EvalResult[]): Map<string, ProducerReceiptIndex> {
if (!Array.isArray(observations)) throw new Error('INVALID_EVAL_RESULTS');
if (!batch || batch.schemaVersion !== 1 || batch.matrixHash !== matrixHash(matrix) || !HEX.test(batch.scheduleHash) || !HEX.test(batch.batchHash)) throw new Error('INVALID_PRODUCER_BATCH');
const { batchHash, ...withoutHash } = batch;
if (digest(JSON.stringify(withoutHash)) !== batchHash || !Array.isArray(batch.receipts) || batch.receipts.length !== matrix.cells.length || batch.summary.scheduled !== matrix.cells.length || batch.summary.prepared !== matrix.cells.length || batch.summary.submitted !== matrix.cells.length || batch.summary.missing !== 0 || batch.summary.matchedPairsExpected !== matrix.cells.length / 2 || batch.summary.matchedPairsSubmitted !== matrix.cells.length / 2 || batch.summary.modelMismatches.length !== 0) throw new Error('INCOMPLETE_PRODUCER_BATCH');
const pairs = expectedMatchedPairs(matrix);
if (digest(JSON.stringify(withoutHash)) !== batchHash || !Array.isArray(batch.receipts) || batch.receipts.length !== matrix.cells.length || batch.summary.scheduled !== matrix.cells.length || batch.summary.prepared !== matrix.cells.length || batch.summary.submitted !== matrix.cells.length || batch.summary.missing !== 0 || batch.summary.matchedPairsExpected !== pairs || batch.summary.matchedPairsSubmitted !== pairs || batch.summary.modelMismatches.length !== 0) throw new Error('INCOMPLETE_PRODUCER_BATCH');
const cells = new Map(matrix.cells.map(cell => [cell.id, cell]));
const receipts = new Map<string, ProducerReceiptIndex>();
for (const receipt of batch.receipts) {
@@ -621,22 +789,67 @@ export function scoreCollectedEval(matrix: EvalMatrix, batch: ProducerBatch, obs
if (!cell || receipts.has(cell.id)) throw new Error('INVALID_PRODUCER_BATCH');
validateReceiptIndex(receipt, cell); receipts.set(cell.id, receipt);
}
if (new Set(batch.receipts.map(receipt => receipt.installationIdentity.identityHash)).size !== 1) throw new Error('UNMATCHED_PRODUCER_INSTALLATIONS');
if (new Set(batch.receipts.map(receipt => receipt.providerIdentity.identityHash)).size !== 1) throw new Error('UNMATCHED_PRODUCER_PROVIDERS');
const actualModels = new Map<string, Partial<Record<EvalVersion, string>>>();
for (const receipt of batch.receipts) {
const key = matchedPairKey(receipt.cell as EvalCell), pair = actualModels.get(key) ?? {};
pair[receipt.cell.version] = receipt.modelUsed; actualModels.set(key, pair);
}
if (actualModels.size !== matrix.cells.length / 2 || [...actualModels.values()].some(pair => !pair.v2 || !pair.v3 || pair.v2 !== pair.v3)) throw new Error('UNMATCHED_EFFECTIVE_MODELS');
assertPlatformIdentities(batch.receipts);
if (profileOf(matrix) === 'full') {
const actualModels = new Map<string, Partial<Record<EvalVersion, string>>>();
for (const receipt of batch.receipts) {
const key = matchedPairKey(receipt.cell as EvalCell), pair = actualModels.get(key) ?? {};
pair[receipt.cell.version] = receipt.modelUsed; actualModels.set(key, pair);
}
if (actualModels.size !== pairs || [...actualModels.values()].some(pair => !pair.v2 || !pair.v3 || pair.v2 !== pair.v3)) throw new Error('UNMATCHED_EFFECTIVE_MODELS');
} else if (new Set(batch.receipts.map(receipt => receipt.modelUsed)).size !== 1) throw new Error('UNMATCHED_EFFECTIVE_MODELS');
for (const result of observations) {
const receipt = receipts.get(result.cellId);
if (!receipt || result.producerReceiptHash !== receipt.receiptHash) throw new Error('UNBOUND_EVAL_RESULT');
}
const score = scoreEval(matrix, observations, qualification, corpus);
return receipts;
}
/** Release scoring path: every trusted judgment must bind to a collected producer receipt. */
export function scoreCollectedEval(matrix: EvalMatrix, batch: ProducerBatch, observations: EvalResult[], qualification: ReleaseQualification = { containment: {} }, corpus: EvalCorpusManifest = loadCorpusManifest(), baseline?: EvalBaseline) {
validateMatrix(matrix, corpus);
if (profileOf(matrix) === 'baseline') throw new Error('BASELINE_PROFILE_NOT_QUALIFIABLE: use createEvalBaseline');
validateCollectedBatch(matrix, batch, observations);
if (baseline !== undefined) {
const { modelUsed, providerIdentity: { family, policyRevision, version } } = batch.receipts[0];
if (baseline?.modelUsed !== modelUsed || JSON.stringify(baseline?.provider) !== JSON.stringify({ family, policyRevision, version })) throw new Error('UNBOUND_EVAL_BASELINE');
}
const score = scoreEval(matrix, observations, qualification, corpus, baseline);
return { ...score, producerBatchHash: batch.batchHash };
}
/**
* Turn a complete, fully adjudicated v2 baseline run into a reusable record.
* Missing reports count as misses; an incomplete or unadjudicated run is refused.
*/
export function createEvalBaseline(matrix: BaselineEvalMatrix, batch: ProducerBatch, observations: EvalResult[], corpus: EvalCorpusManifest = loadCorpusManifest()): EvalBaseline {
if (matrix?.profile !== 'baseline') throw new Error('INVALID_EVAL_BASELINE_MATRIX');
validateMatrix(matrix, corpus);
validateCollectedBatch(matrix, batch, observations);
const results = validatedResults(matrix, observations, corpus);
if (results.size !== matrix.cells.length) throw new Error('INCOMPLETE_EVAL_BASELINE');
if ([...results.values()].some(result => result.reportPresent && result.reportComplete && result.findings.some(finding => finding.evidence !== 'hypothesis' && finding.judgment === 'unadjudicated'))) throw new Error('UNADJUDICATED_EVAL_BASELINE');
const vulnerable = new Map(matrix.cells.filter(cell => cell.variant === 'vulnerable').map(cell => [cell.caseId, cell]));
const { modelUsed, providerIdentity: { family, policyRevision, version } } = batch.receipts[0];
const record: Omit<EvalBaseline, 'baselineHash'> = {
schemaVersion: 1, version: 'v2', corpusVersion: corpus.version, corpusHash: matrix.corpusHash, model: matrix.model, host: matrix.host,
budgetSeconds: matrix.budgets.comprehensive, skillHash: matrix.skillHashes.v2, platforms: matrix.platforms.comprehensive,
matrixHash: matrixHash(matrix), producerBatchHash: batch.batchHash, modelUsed, provider: { family, policyRevision, version },
cases: corpus.cases.map(fixture => ({ caseId: fixture.id, found: foundSeededIssue(vulnerable.get(fixture.id)!, results.get(vulnerable.get(fixture.id)!.id)) })),
};
return { ...record, baselineHash: digest(JSON.stringify(record)) };
}
function splitFlags(args: string[], allowed: string[]): { positional: string[]; flags: Record<string, string> } {
const positional: string[] = [], flags: Record<string, string> = {};
for (let index = 0; index < args.length; index++) {
if (!args[index].startsWith('--')) { positional.push(args[index]); continue; }
if (!allowed.includes(args[index]) || !args[index + 1] || flags[args[index]]) throw new Error(`Unsupported or repeated flag: ${args[index]}`);
flags[args[index]] = args[++index];
}
return { positional, flags };
}
function cli(args: string[]): void {
const [command, ...rest] = args;
if (command === 'payload') {
@@ -669,25 +882,35 @@ function cli(args: string[]): void {
const schedule = prepareEvalJobs(matrix, { v2: readBoundedStable(resolve(rest[2]), PORTABLE_PAYLOAD_LIMIT, 'v2 portable skill payload').toString('utf8'), v3: readBoundedStable(resolve(rest[4]), PORTABLE_PAYLOAD_LIMIT, 'v3 portable skill payload').toString('utf8') }, rest[6]);
console.log(JSON.stringify({ scheduled: schedule.scheduledCells, prepared: schedule.preparedCells, output: resolve(rest[6]), paidCalls: 0 })); return;
}
const { positional, flags } = splitFlags(rest, ['--corpus-manifest', '--baseline']);
const corpus = flags['--corpus-manifest'] ? validateCorpusManifest(readJsonBounded(flags['--corpus-manifest'], 16 * 1024 * 1024, 'corpus manifest')) : loadCorpusManifest();
if (command === 'collect') {
if (rest.length !== 4) throw new Error('Usage: cso-eval collect <matrix.json> <schedule.json> <receipts-directory> <new-batch.json>');
const matrix = readJsonBounded(rest[0], 16 * 1024 * 1024, 'evaluation matrix') as EvalMatrix;
const schedule = readJsonBounded(rest[1], 16 * 1024 * 1024, 'evaluation schedule') as PreparedEvalSchedule;
const receiptRoot = resolve(rest[2]), stat = lstatSync(receiptRoot);
if (positional.length !== 4 || flags['--baseline']) throw new Error('Usage: cso-eval collect <matrix.json> <schedule.json> <receipts-directory> <new-batch.json> [--corpus-manifest <manifest.json>]');
const [matrixPath, schedulePath, receiptDirectory, output] = positional;
const matrix = readJsonBounded(matrixPath, 16 * 1024 * 1024, 'evaluation matrix') as EvalMatrix;
const schedule = readJsonBounded(schedulePath, 16 * 1024 * 1024, 'evaluation schedule') as PreparedEvalSchedule;
const receiptRoot = resolve(receiptDirectory), stat = lstatSync(receiptRoot);
if (!stat.isDirectory() || stat.isSymbolicLink() || realpathSync(receiptRoot) !== receiptRoot) throw new Error('UNSAFE_RECEIPT_DIRECTORY');
const receipts: ProducerReceipt[] = [];
for (const entry of readdirSync(receiptRoot, { withFileTypes: true })) {
if (!entry.isFile() || entry.isSymbolicLink() || !/^[a-f0-9]{64}\.json$/.test(entry.name)) throw new Error('UNSAFE_RECEIPT_DIRECTORY');
receipts.push(readJsonBounded(join(receiptRoot, entry.name), 40 * 1024 * 1024, 'producer receipt'));
}
const batch = collectProducerReceipts(matrix, schedule, receipts);
safeWriteNew(resolve(rest[3]), batch);
console.log(JSON.stringify({ output: resolve(rest[3]), submitted: batch.summary.submitted, missing: batch.summary.missing, modelMismatches: batch.summary.modelMismatches.length, paidCalls: 0 })); return;
const batch = collectProducerReceipts(matrix, schedule, receipts, corpus);
safeWriteNew(resolve(output), batch);
console.log(JSON.stringify({ output: resolve(output), submitted: batch.summary.submitted, missing: batch.summary.missing, modelMismatches: batch.summary.modelMismatches.length, paidCalls: 0 })); return;
}
if (command === 'score') {
if (rest.length < 3 || rest.length > 4) throw new Error('Usage: cso-eval score <matrix.json> <producer-batch.json> <trusted-results.json> [qualification.json]');
console.log(JSON.stringify(scoreCollectedEval(readJsonBounded(rest[0], 16 * 1024 * 1024, 'evaluation matrix'), readJsonBounded(rest[1], 16 * 1024 * 1024, 'producer batch'), readJsonBounded(rest[2], 64 * 1024 * 1024, 'trusted results'), rest[3] ? readJsonBounded(rest[3], 4 * 1024 * 1024, 'qualification') : undefined), null, 2)); return;
if (positional.length < 3 || positional.length > 4) throw new Error('Usage: cso-eval score <matrix.json> <producer-batch.json> <trusted-results.json> [qualification.json] [--corpus-manifest <manifest.json>] [--baseline <baseline.json>]');
const baseline = flags['--baseline'] ? readJsonBounded(flags['--baseline'], 4 * 1024 * 1024, 'evaluation baseline') as EvalBaseline : undefined;
console.log(JSON.stringify(scoreCollectedEval(readJsonBounded(positional[0], 16 * 1024 * 1024, 'evaluation matrix'), readJsonBounded(positional[1], 16 * 1024 * 1024, 'producer batch'), readJsonBounded(positional[2], 64 * 1024 * 1024, 'trusted results'), positional[3] ? readJsonBounded(positional[3], 4 * 1024 * 1024, 'qualification') : undefined, corpus, baseline), null, 2)); return;
}
throw new Error('Usage: cso-eval <payload|matrix|prepare|materialize|collect|score>');
if (command === 'baseline') {
if (positional.length !== 4 || flags['--baseline']) throw new Error('Usage: cso-eval baseline <baseline-matrix.json> <producer-batch.json> <trusted-results.json> <new-baseline.json> [--corpus-manifest <manifest.json>]');
const baseline = createEvalBaseline(readJsonBounded(positional[0], 16 * 1024 * 1024, 'evaluation matrix'), readJsonBounded(positional[1], 16 * 1024 * 1024, 'producer batch'), readJsonBounded(positional[2], 64 * 1024 * 1024, 'trusted results'), corpus);
safeWriteNew(resolve(positional[3]), baseline);
console.log(JSON.stringify({ output: resolve(positional[3]), baselineHash: baseline.baselineHash, paidCalls: 0 })); return;
}
throw new Error('Usage: cso-eval <payload|matrix|prepare|materialize|collect|score|baseline>');
}
if (import.meta.main) { try { cli(process.argv.slice(2)); } catch (error) { console.error(error instanceof Error ? error.message : 'CSO evaluation failed'); process.exitCode = 1; } }
+131
View File
@@ -0,0 +1,131 @@
#!/usr/bin/env bun
/** Derive native runtime gate evidence from the named Docker tests that actually executed and passed. */
import * as fs from 'node:fs';
import * as path from 'node:path';
import { sha256 } from '../lib/cso/contracts';
import type { ImageBuildRow } from './cso-image-matrix';
import { requiredChecks } from './cso-runtime-promotion';
type Stack = ImageBuildRow['stack'];
export interface TestRef { file: string; name: string }
export interface JUnitCase extends TestRef { status: 'passed' | 'failed' | 'skipped'; assertions: number }
const MAX_REPORT_BYTES = 4 * 1024 * 1024;
const DOCKER = 'test/cso-docker-integration.test.ts';
const CONTAINMENT: TestRef[] = [
{ file: DOCKER, name: 'hard fails when local daemon enforcement prerequisites are absent' },
{ file: DOCKER, name: 'rejects image-declared writable volumes before container creation' },
{ file: DOCKER, name: 'shares only loopback while denying egress, privileges, and daemon logs, and reads private source/policy mounts' },
{ file: DOCKER, name: 'machine-wide admission allows only two groups per endpoint' },
];
const STAGED: TestRef = { file: DOCKER, name: 'staged runtime executes its trusted verifier and checks every declared tool version' };
const NODE_LIFECYCLE: TestRef = {
file: 'test/cso-node-lifecycle-integration.test.ts',
name: 'finds, reproduces, bundles, replays, and closes against current source through the command dispatcher',
};
const COLD_JOURNEY: TestRef = {
file: 'test/cso-stack-cold-integration.test.ts',
name: 'acquires, prepares, boots, controls, and tests the requested staged stack with no target egress',
};
/** Each native check key names the tests whose execution proves it for one stack. */
export function nativeTestRequirements(stack: Stack): Record<string, TestRef[]> {
if (stack === 'postgresql') return {
containmentPassed: [...CONTAINMENT, STAGED],
coldStartPassed: [STAGED],
multiDatabasePassed: [STAGED],
readinessPassed: [STAGED],
};
const journey = stack === 'node' ? NODE_LIFECYCLE : COLD_JOURNEY;
const application = {
containmentPassed: [...CONTAINMENT, STAGED],
coldStartPassed: [journey],
positiveNegativeAssertionsPassed: [STAGED],
acquisitionPublicOnlyPassed: [journey],
offlineLifecyclePassed: [journey],
};
return stack === 'rails'
? { ...application, railsSqlitePassed: [COLD_JOURNEY], railsPostgresqlPassed: [COLD_JOURNEY], nativeExtensionsPassed: [COLD_JOURNEY] }
: application;
}
function decode(value: string): string {
return value.replace(/&(#x[0-9a-fA-F]+|#[0-9]+|amp|lt|gt|quot|apos);/g, (_, entity: string) => {
if (entity.startsWith('#x')) return String.fromCodePoint(Number.parseInt(entity.slice(2), 16));
if (entity.startsWith('#')) return String.fromCodePoint(Number.parseInt(entity.slice(1), 10));
return { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }[entity]!;
});
}
/** Parse the testcases of Bun's `--reporter=junit` output. */
export function parseJUnit(xml: string): JUnitCase[] {
if (!/^<\?xml[^>]*\?>\s*<testsuites\b/.test(xml)) throw new Error('INVALID_JUNIT_REPORT');
const cases: JUnitCase[] = [];
for (const match of xml.matchAll(/<testcase\b([^>]*?)(?:\/>|>([\s\S]*?)<\/testcase>)/g)) {
const attributes = Object.fromEntries([...match[1].matchAll(/([A-Za-z_:][\w:.-]*)="([^"]*)"/g)].map(([, key, value]) => [key, decode(value)]));
const body = match[2] ?? '';
const status = /<(?:failure|error)\b/.test(body) ? 'failed' : /<skipped\b/.test(body) ? 'skipped' : 'passed';
const assertions = Number(attributes.assertions ?? '0');
if (typeof attributes.name !== 'string' || typeof attributes.file !== 'string' || !Number.isInteger(assertions) || assertions < 0) {
throw new Error('INVALID_JUNIT_TESTCASE');
}
cases.push({ file: attributes.file, name: attributes.name, status, assertions });
}
if (cases.length === 0) throw new Error('EMPTY_JUNIT_REPORT');
return cases;
}
/** A check passes only when the whole report has no failure and every named test ran once, passed, and asserted. */
export function deriveNativeChecks(stack: Stack, cases: JUnitCase[]): Record<string, boolean> {
const clean = cases.some(item => item.status === 'passed') && !cases.some(item => item.status === 'failed');
const passed = (ref: TestRef) => {
const matches = cases.filter(item => item.file === ref.file && item.name === ref.name);
return matches.length === 1 && matches[0].status === 'passed' && matches[0].assertions > 0;
};
return Object.fromEntries(Object.entries(nativeTestRequirements(stack))
.map(([key, refs]) => [key, clean && refs.every(passed)]));
}
/** Native evidence for one staged row; private checks are every required release gate the native job cannot prove. */
export function nativeEvidence(staged: unknown, stack: Stack, platform: string, report: string): Record<string, unknown> {
const record = staged as Record<string, unknown>;
if (!record || typeof record !== 'object' || record.state !== 'staged' || record.stack !== stack || record.platform !== platform) {
throw new Error('STAGED_EVIDENCE_MISMATCH');
}
const required = requiredChecks(stack);
const nativeChecks = deriveNativeChecks(stack, parseJUnit(report));
const failed = Object.entries(nativeChecks).filter(([key, value]) => !value || !required.includes(key)).map(([key]) => key);
if (failed.length) throw new Error(`NATIVE_GATE_NOT_PASSED: ${failed.sort().join(', ')}`);
return {
...record,
state: 'native-gates-passed',
nativeChecks,
nativeTestReport: `sha256:${sha256(report)}`,
privateChecks: Object.fromEntries(required.filter(key => !(key in nativeChecks)).map(key => [key, 'pending'])),
qualified: false,
promotion: 'prohibited until authenticated private qualification evidence completes every release gate',
};
}
function readFile(file: string): string {
const stat = fs.lstatSync(path.resolve(file));
if (!stat.isFile() || stat.isSymbolicLink() || stat.size <= 0 || stat.size > MAX_REPORT_BYTES) throw new Error('UNSAFE_NATIVE_EVIDENCE_INPUT');
return fs.readFileSync(path.resolve(file), 'utf8');
}
if (import.meta.main) {
try {
const args = process.argv.slice(2);
const names = ['--stack', '--platform', '--junit', '--staged', '--output'];
if (args.length !== names.length * 2 || names.some((name, index) => args[index * 2] !== name)) {
throw new Error('Usage: cso-native-evidence.ts --stack STACK --platform PLATFORM --junit REPORT.xml --staged staged-image.json --output EVIDENCE.json');
}
const [stack, platform, junit, staged, output] = names.map((_, index) => args[index * 2 + 1]);
if (!['node', 'bun', 'python', 'rails', 'postgresql'].includes(stack)) throw new Error('INVALID_STACK');
const evidence = nativeEvidence(JSON.parse(readFile(staged)), stack as Stack, platform, readFile(junit));
fs.writeFileSync(path.resolve(output), JSON.stringify(evidence, null, 2) + '\n', { flag: 'wx', mode: 0o600 });
} catch (error) {
process.stderr.write((error instanceof Error ? error.message : 'NATIVE_EVIDENCE_FAILED') + '\n');
process.exitCode = 1;
}
}
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env bun
/**
* Requalification triggers: the inputs a runtime qualification was measured
* under. A promoted catalog records them; a catalog whose recorded triggers
* differ from the current source is older than its inputs and must be
* requalified (or its runtimes withdrawn) before it ships.
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import { canonical, sha256 } from '../lib/cso/contracts';
import { ISOLATION_POLICY_HASH } from '../lib/cso/docker';
import { CSO_HELPER_ABI, isEvaluationRuntimeCatalog, type RequalificationTriggers, type RuntimeCatalog } from '../lib/cso/runtime-catalog';
const ROOT = path.resolve(import.meta.dir, '..');
/** Compiled into every runtime image as gstack-cso-verifier and gstack-cso-preparation. */
export const IMAGE_HELPER_ENTRYPOINTS = ['lib/cso/verifier.ts', 'lib/cso/preparation-container.ts'] as const;
export const IMAGE_CONTEXT = 'lib/cso/images';
/** Image-context files that describe qualification rather than enter an image; build-inputs.json is its own trigger. */
const IMAGE_CONTEXT_EXCLUDED = new Set(['README.md', 'qualification.json', 'build-inputs.json']);
/** Every repository file whose bytes enter a runtime image: the helpers' import closure plus the image build context. */
export async function imageSourceFiles(root = ROOT): Promise<string[]> {
const build = await Bun.build({ entrypoints: IMAGE_HELPER_ENTRYPOINTS.map(entry => path.join(root, entry)), target: 'bun', metafile: true });
if (!build.success || !build.metafile) throw new Error('REQUALIFICATION_SOURCE_CLOSURE_FAILED');
const closure = Object.keys(build.metafile.inputs).map(input => path.relative(root, path.resolve(root, input)).split(path.sep).join('/'));
if (closure.some(file => file.startsWith('..') || file.includes('node_modules/'))) throw new Error('REQUALIFICATION_SOURCE_OUTSIDE_REPOSITORY');
const context = fs.readdirSync(path.join(root, IMAGE_CONTEXT), { withFileTypes: true })
.filter(entry => entry.isFile() && !IMAGE_CONTEXT_EXCLUDED.has(entry.name))
.map(entry => `${IMAGE_CONTEXT}/${entry.name}`);
return [...new Set([...closure, ...context])].sort();
}
export async function requalificationTriggers(root = ROOT): Promise<RequalificationTriggers> {
const files = await imageSourceFiles(root);
const buildInputs = JSON.parse(fs.readFileSync(path.join(root, IMAGE_CONTEXT, 'build-inputs.json'), 'utf8'));
if (typeof buildInputs?.revision !== 'string' || !buildInputs.revision) throw new Error('INVALID_BUILD_INPUTS_REVISION');
return {
helperAbi: CSO_HELPER_ABI,
isolationPolicyHash: ISOLATION_POLICY_HASH,
preparationSha256: sha256(canonical(files.map(file => [file, sha256(fs.readFileSync(path.join(root, file)))]))),
buildInputsRevision: buildInputs.revision,
};
}
/** Names of the triggers that changed since the catalog's runtimes were qualified; empty when nothing is promoted. */
export function staleRequalificationTriggers(catalog: RuntimeCatalog, current: RequalificationTriggers): string[] {
if (isEvaluationRuntimeCatalog(catalog) || catalog.runtimes.length === 0) return [];
const recorded = catalog.promotion?.requalification;
if (!recorded) return Object.keys(current).sort();
return (Object.keys(current) as Array<keyof RequalificationTriggers>).filter(key => recorded[key] !== current[key]).sort();
}
if (import.meta.main) {
try {
const [command, file, ...rest] = process.argv.slice(2);
if (command === 'triggers' && file === undefined) {
process.stdout.write(JSON.stringify(await requalificationTriggers(), null, 2) + '\n');
} else if (command === 'check' && file && rest.length === 0) {
const stale = staleRequalificationTriggers(JSON.parse(fs.readFileSync(path.resolve(file), 'utf8')), await requalificationTriggers());
if (stale.length) throw new Error(`STALE_RUNTIME_QUALIFICATION: ${stale.join(', ')} changed since qualification; requalify the runtimes or withdraw them from the catalog`);
process.stdout.write('REQUALIFICATION CURRENT\n');
} else throw new Error('Usage: cso-requalification.ts triggers | check <runtime-catalog.json>');
} catch (error) {
process.stderr.write((error instanceof Error ? error.message : 'REQUALIFICATION_CHECK_FAILED') + '\n');
process.exitCode = 1;
}
}
+131 -15
View File
@@ -7,12 +7,18 @@ import committedCatalog from '../lib/cso/runtime-catalog.json';
import buildInputs from '../lib/cso/images/build-inputs.json';
import {
CSO_HELPER_ABI,
EVALUATION_REVISION_PREFIX,
isEvaluationRuntimeCatalog,
validRequalificationTriggers,
type RequalificationTriggers,
type QualifiedRuntime,
type RuntimeCatalog,
type RuntimeQualification,
validateRuntimeCatalog,
} from '../lib/cso/runtime-catalog';
import { ISOLATION_POLICY_HASH } from '../lib/cso/docker';
import { imageBuildMatrix, type ImageBuildRow } from './cso-image-matrix';
import { requalificationTriggers, staleRequalificationTriggers } from './cso-requalification';
const DIGEST = /^sha256:[a-f0-9]{64}$/;
const WORKFLOW = /^https:\/\/github\.com\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+\/actions\/runs\/[0-9]+$/;
@@ -46,7 +52,7 @@ function versionsKey(value: Record<string, string>): string {
return canonical(value);
}
function requiredChecks(stack: ImageBuildRow['stack']): string[] {
export function requiredChecks(stack: ImageBuildRow['stack']): string[] {
if (stack === 'postgresql') return [
'containmentPassed', 'coldStartPassed', 'multiDatabasePassed', 'readinessPassed',
'secretCanaryPassed', 'watchdogCleanupPassed',
@@ -112,13 +118,15 @@ function validateStatement(raw: unknown, row: ImageBuildRow): RuntimeQualificati
return statement;
}
export function catalogPromotionCandidate(
currentValue: unknown,
inputValue: unknown,
evidenceValues: unknown[],
): RuntimeCatalog {
/** The committed (never evaluation-only) catalog and the reviewed build matrix it must describe. */
function reviewedMatrix(currentValue: unknown, inputValue: unknown): { current: RuntimeCatalog; rows: ImageBuildRow[] } {
validateRuntimeCatalog(currentValue);
const current = currentValue as RuntimeCatalog;
if (isEvaluationRuntimeCatalog(current)) throw new Error('EVALUATION_CATALOG_NOT_PROMOTABLE');
return { current, rows: profilesMatchingRows(current, inputValue) };
}
function profilesMatchingRows(current: RuntimeCatalog, inputValue: unknown): ImageBuildRow[] {
const rows = imageBuildMatrix(inputValue).include;
if (!current.profiles || current.buildRevision !== rows[0].inputRevision) throw new Error('CATALOG_BUILD_REVISION_MISMATCH');
const profiles = new Map(current.profiles.map(profile => [profile.id, profile]));
@@ -128,17 +136,100 @@ export function catalogPromotionCandidate(
if (!profile || profile.stack !== row.stack || profile.platform !== row.platform ||
versionsKey(profile.versions) !== versionsKey(row.versions)) throw new Error(`CATALOG_PROFILE_MATRIX_MISMATCH: ${row.runtimeId}`);
}
if (evidenceValues.length !== rows.length) throw new Error('INCOMPLETE_QUALIFICATION_MATRIX');
return rows;
}
/** The candidate build's gate: a valid evaluation-only catalog for the reviewed build inputs. */
export function validateEvaluationCandidate(catalogValue: unknown, inputValue: unknown): RuntimeCatalog {
validateRuntimeCatalog(catalogValue);
const catalog = catalogValue as RuntimeCatalog;
if (!isEvaluationRuntimeCatalog(catalog)) throw new Error('NOT_AN_EVALUATION_CATALOG');
const rows = profilesMatchingRows(catalog, inputValue);
for (const row of rows) {
const runtime = catalog.runtimes.find(item => item.id === row.runtimeId);
if (!runtime || !OUTPUT_IMAGE.test(runtime.image) || !runtime.image.includes(`/${row.stack}-${row.arch}@`)) throw new Error(`INVALID_QUALIFIED_IMAGE: ${row.runtimeId}`);
}
return catalog;
}
function matchRows(rows: ImageBuildRow[], values: unknown[]): Array<[ImageBuildRow, unknown]> {
if (values.length !== rows.length) throw new Error('INCOMPLETE_QUALIFICATION_MATRIX');
const byId = new Map<string, unknown>();
for (const value of evidenceValues) {
for (const value of values) {
const id = (value as any)?.runtimeId;
if (typeof id !== 'string' || byId.has(id)) throw new Error('DUPLICATE_QUALIFICATION_STATEMENT');
byId.set(id, value);
}
const statements = rows.map(row => {
return rows.map(row => {
if (!byId.has(row.runtimeId)) throw new Error(`MISSING_QUALIFICATION_STATEMENT: ${row.runtimeId}`);
return validateStatement(byId.get(row.runtimeId), row);
return [row, byId.get(row.runtimeId)];
});
}
const STAGED_FIELDS = [
'buildRevision', 'helperAbi', 'image', 'platform', 'provenanceDigest', 'qualification', 'runtimeId',
'sbomDigest', 'schemaVersion', 'sourceCommit', 'stack', 'state', 'verifiedProvenance', 'versions', 'workflow',
].sort();
/**
* Build an evaluation-only catalog from one staging run's staged-image.json
* records. It measures no release gate: every runtime is an
* evaluation_candidate, and setup, distribution builds and promotion refuse it.
*/
export function evaluationCandidateCatalog(currentValue: unknown, inputValue: unknown, stagedValues: unknown[], createdAt: string): RuntimeCatalog {
const { current, rows } = reviewedMatrix(currentValue, inputValue);
if (!Number.isFinite(Date.parse(createdAt))) throw new Error('INVALID_EVALUATION_TIME');
const staged = matchRows(rows, stagedValues).map(([row, value]) => {
const record = value as RuntimeQualificationStatement;
if (!record || typeof record !== 'object' || canonical(Object.keys(record).sort()) !== canonical(STAGED_FIELDS) ||
record.schemaVersion !== 1 || record.helperAbi !== CSO_HELPER_ABI || (record.state as string) !== 'staged' || (record as any).verifiedProvenance !== true) throw new Error(`INVALID_STAGED_RUNTIME: ${row.runtimeId}`);
if (record.buildRevision !== row.inputRevision || record.stack !== row.stack || record.platform !== row.platform ||
versionsKey(record.versions) !== versionsKey(row.versions)) throw new Error(`QUALIFICATION_BUILD_MISMATCH: ${row.runtimeId}`);
if (!OUTPUT_IMAGE.test(record.image) || !record.image.includes(`/${row.stack}-${row.arch}@`)) throw new Error(`INVALID_QUALIFIED_IMAGE: ${row.runtimeId}`);
if (!/^[a-f0-9]{40}$/.test(record.sourceCommit) || !WORKFLOW.test(record.workflow) || !DIGEST.test(record.sbomDigest) ||
!DIGEST.test(record.provenanceDigest)) throw new Error(`MISSING_QUALIFICATION_PROVENANCE: ${row.runtimeId}`);
return record;
});
if (new Set(staged.map(item => item.sourceCommit)).size !== 1 || new Set(staged.map(item => item.workflow)).size !== 1) throw new Error('SPLIT_STAGING_RUN');
const { sourceCommit, workflow } = staged[0];
const runtimes: QualifiedRuntime[] = staged.map(record => ({
id: record.runtimeId,
stack: record.stack,
platform: record.platform,
state: 'evaluation_candidate',
image: record.image,
entrypoint: '/opt/cso/entrypoint',
helperAbi: CSO_HELPER_ABI,
versions: record.versions,
policyVersion: 'cso-isolation-v1',
qualifiedAt: createdAt,
qualification: { kind: 'evaluation', sourceCommit, workflow, sbomDigest: record.sbomDigest, provenanceDigest: record.provenanceDigest, verifiedProvenance: true },
}));
const candidate: RuntimeCatalog = {
schemaVersion: 1,
revision: `${EVALUATION_REVISION_PREFIX}${workflow.slice(workflow.lastIndexOf('/') + 1)}-${sourceCommit.slice(0, 12)}`,
previousRevision: current.revision,
helperAbi: CSO_HELPER_ABI,
buildRevision: current.buildRevision,
profiles: current.profiles,
evaluation: { sourceCommit, workflow, evidenceDigest: sha256(canonical(runtimes)) },
runtimes,
};
validateRuntimeCatalog(candidate);
return candidate;
}
export function catalogPromotionCandidate(
currentValue: unknown,
inputValue: unknown,
evidenceValues: unknown[],
/** Measured by the staging run at sourceCommit (scripts/cso-requalification.ts triggers). */
requalification: RequalificationTriggers,
): RuntimeCatalog {
const { current, rows } = reviewedMatrix(currentValue, inputValue);
if (!validRequalificationTriggers(requalification, current.buildRevision)) throw new Error('INVALID_REQUALIFICATION_TRIGGERS');
if (requalification.isolationPolicyHash !== ISOLATION_POLICY_HASH) throw new Error('STALE_RUNTIME_QUALIFICATION: isolationPolicyHash');
const statements = matchRows(rows, evidenceValues).map(([row, value]) => validateStatement(value, row));
const sourceCommits = new Set(statements.map(item => item.sourceCommit));
const workflows = new Set(statements.map(item => item.workflow));
if (sourceCommits.size !== 1 || workflows.size !== 1) throw new Error('SPLIT_QUALIFICATION_RUN');
@@ -167,7 +258,15 @@ export function catalogPromotionCandidate(
helperAbi: CSO_HELPER_ABI,
buildRevision: current.buildRevision,
profiles: current.profiles,
promotion: { sourceCommit, workflow, evidenceDigest, qualificationEvidenceDigest },
promotion: {
sourceCommit, workflow, evidenceDigest, qualificationEvidenceDigest,
requalification: {
helperAbi: requalification.helperAbi,
isolationPolicyHash: requalification.isolationPolicyHash,
preparationSha256: requalification.preparationSha256,
buildInputsRevision: requalification.buildInputsRevision,
},
},
runtimes,
};
validateRuntimeCatalog(candidate);
@@ -180,6 +279,7 @@ export function validateRuntimeCatalogTransition(currentValue: unknown, proposed
validateRuntimeCatalog(proposedValue);
const current = currentValue as RuntimeCatalog;
const proposed = proposedValue as RuntimeCatalog;
if (isEvaluationRuntimeCatalog(current) || isEvaluationRuntimeCatalog(proposed)) throw new Error('EVALUATION_CATALOG_NOT_PROMOTABLE');
if (!proposed.promotion || proposed.runtimes.length === 0) throw new Error('UNQUALIFIED_RUNTIME_CATALOG_PROMOTION');
if (proposed.revision === current.revision || proposed.previousRevision !== current.revision) {
throw new Error('RUNTIME_CATALOG_BASE_REVISION_MISMATCH');
@@ -190,7 +290,7 @@ export function validateRuntimeCatalogTransition(currentValue: unknown, proposed
}
}
function collectEvidence(root: string): unknown[] {
function collectEvidence(root: string, name = 'qualified-runtime.json'): unknown[] {
const absolute = path.resolve(root);
const rootStat = fs.lstatSync(absolute);
if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) throw new Error('UNSAFE_EVIDENCE_ROOT');
@@ -201,7 +301,7 @@ function collectEvidence(root: string): unknown[] {
const file = path.join(directory, entry.name);
if (entry.isSymbolicLink() || (!entry.isDirectory() && !entry.isFile())) throw new Error('UNSAFE_EVIDENCE_ENTRY');
if (entry.isDirectory()) visit(file, depth + 1);
else if (entry.name === 'qualified-runtime.json') files.push(file);
else if (entry.name === name) files.push(file);
if (files.length > MAX_EVIDENCE_FILES) throw new Error('TOO_MANY_QUALIFICATION_STATEMENTS');
}
};
@@ -234,7 +334,19 @@ function readCatalogArtifact(file: string): unknown {
if (import.meta.main) {
try {
const args = process.argv.slice(2);
if (args[0] === 'validate-transition') {
if (args[0] === 'check-evaluation') {
if (args.length !== 2) throw new Error('Usage: cso-runtime-promotion.ts check-evaluation <cso-eval-catalog>');
const catalog = validateEvaluationCandidate(readCatalogArtifact(args[1]), buildInputs);
process.stdout.write(`EVALUATION-ONLY ${catalog.revision}\n`);
} else if (args[0] === 'evaluation-candidate') {
if (args.length !== 5 || args[1] !== '--staged-root' || args[3] !== '--output') {
throw new Error('Usage: cso-runtime-promotion.ts evaluation-candidate --staged-root <directory> --output <new-file>');
}
const candidate = evaluationCandidateCatalog(committedCatalog, buildInputs, collectEvidence(args[2], 'staged-image.json'), new Date().toISOString());
const output = path.resolve(args[4]);
fs.writeFileSync(output, JSON.stringify(candidate, null, 2) + '\n', { flag: 'wx', mode: 0o600 });
process.stdout.write(`${output}\n`);
} else if (args[0] === 'validate-transition') {
if (args.length !== 3) throw new Error('Usage: cso-runtime-promotion.ts validate-transition <current-catalog> <proposed-catalog>');
validateRuntimeCatalogTransition(readCatalogArtifact(args[1]), readCatalogArtifact(args[2]));
process.stdout.write('VALID TRANSITION\n');
@@ -242,7 +354,11 @@ if (import.meta.main) {
if (args.length !== 4 || args[0] !== '--evidence-root' || args[2] !== '--output') {
throw new Error('Usage: cso-runtime-promotion.ts --evidence-root <directory> --output <new-file>');
}
const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, collectEvidence(args[1]));
const triggers = collectEvidence(args[1], 'requalification.json');
if (triggers.length !== 1) throw new Error('MISSING_REQUALIFICATION_TRIGGERS');
const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, collectEvidence(args[1]), triggers[0] as RequalificationTriggers);
const stale = staleRequalificationTriggers(candidate, await requalificationTriggers());
if (stale.length) throw new Error(`STALE_RUNTIME_QUALIFICATION: ${stale.join(', ')} changed since the staged commit; re-stage and requalify`);
const output = path.resolve(args[3]);
fs.writeFileSync(output, JSON.stringify(candidate, null, 2) + '\n', { flag: 'wx', mode: 0o600 });
process.stdout.write(`${output}\n`);
+124
View File
@@ -0,0 +1,124 @@
#!/usr/bin/env bun
/** Bind a qualification dispatch to the one successful protected-main staging run that signed its images. */
import * as fs from 'node:fs';
import * as path from 'node:path';
import { canonical } from '../lib/cso/contracts';
import { committedImageBuildMatrix } from './cso-image-matrix';
export const STAGING_WORKFLOW_PATH = '.github/workflows/cso-runtime-images.yml';
export const NATIVE_JOB = 'qualify-native';
const SLSA_PROVENANCE = 'https://slsa.dev/provenance/v1';
const REPOSITORY = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/;
const INVOCATION = /^https:\/\/github\.com\/([A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+)\/actions\/runs\/([1-9][0-9]*)\/attempts\/[1-9][0-9]*$/;
const MAX_FILE_BYTES = 4 * 1024 * 1024;
export interface StagingRunEvidence {
repository: string;
runId: string;
sourceCommit: string;
runtimeIds: string[];
run: unknown;
jobPages: unknown;
}
/**
* Resolve the staging run from verified SLSA provenance. The predicate's
* invocation and the Fulcio certificate's run URI must agree, and every
* verified statement for every image must name the same run.
*/
export function stagingRunId(repository: string, verifications: unknown[]): string {
if (!REPOSITORY.test(repository) || verifications.length === 0) throw new Error('INVALID_STAGING_PROVENANCE');
const runs = new Set<string>();
for (const verification of verifications) {
if (!Array.isArray(verification) || verification.length === 0) throw new Error('INVALID_STAGING_PROVENANCE');
for (const item of verification) {
const result = (item as any)?.verificationResult;
if (result?.statement?.predicateType !== SLSA_PROVENANCE) throw new Error('INVALID_STAGING_PROVENANCE');
const invocation = result.statement.predicate?.runDetails?.metadata?.invocationId;
const match = typeof invocation === 'string' ? INVOCATION.exec(invocation) : null;
if (!match || match[1] !== repository || result.signature?.certificate?.runInvocationURI !== invocation) {
throw new Error('STAGING_INVOCATION_MISMATCH');
}
runs.add(match[2]);
}
}
if (runs.size !== 1) throw new Error('STAGING_RUN_NOT_UNIQUE');
return [...runs][0];
}
/** Every statement in one qualification dispatch names the same staged source commit. */
export function dispatchSourceCommit(event: unknown): string {
const statements = (event as any)?.client_payload?.statements;
if (!Array.isArray(statements) || statements.length === 0) throw new Error('INVALID_QUALIFICATION_DISPATCH');
const commits = new Set(statements.map((statement: any) => statement?.sourceCommit));
const [commit] = [...commits];
if (commits.size !== 1 || typeof commit !== 'string' || !/^[a-f0-9]{40}$/.test(commit)) throw new Error('STAGING_SOURCE_COMMIT_MISMATCH');
return commit;
}
/** The resolved run must be the protected-main staging dispatch at the statements' commit, with every native gate passed. */
export function verifyStagingRun(evidence: StagingRunEvidence): void {
const { repository, runId, sourceCommit, runtimeIds } = evidence;
const run = evidence.run as any;
if (!run || typeof run !== 'object' || String(run.id) !== runId) throw new Error('STAGING_RUN_ID_MISMATCH');
if (run.path !== STAGING_WORKFLOW_PATH) throw new Error('STAGING_WORKFLOW_MISMATCH');
if (run.repository?.full_name !== repository || run.head_repository?.full_name !== repository) throw new Error('STAGING_REPOSITORY_MISMATCH');
if (run.event !== 'workflow_dispatch' || run.head_branch !== 'main') throw new Error('STAGING_RUN_NOT_PROTECTED_MAIN_DISPATCH');
if (run.status !== 'completed' || run.conclusion !== 'success') throw new Error('STAGING_RUN_NOT_SUCCESSFUL');
if (run.head_sha !== sourceCommit) throw new Error('STAGING_SOURCE_COMMIT_MISMATCH');
const pages = evidence.jobPages;
if (!Array.isArray(pages) || pages.length === 0) throw new Error('INVALID_STAGING_JOBS');
const totals = new Set(pages.map((page: any) => page?.total_count));
if (pages.some((page: any) => !Array.isArray(page?.jobs)) || totals.size !== 1) throw new Error('INVALID_STAGING_JOBS');
const jobs = pages.flatMap((page: any) => page.jobs);
if (jobs.length !== [...totals][0]) throw new Error('INCOMPLETE_STAGING_JOBS');
const native = jobs.filter((job: any) => typeof job?.name === 'string' && job.name.startsWith(NATIVE_JOB));
const expected = runtimeIds.map(id => `${NATIVE_JOB} ${id}`).sort();
if (runtimeIds.length === 0 || canonical(native.map((job: any) => job.name).sort()) !== canonical(expected)) {
throw new Error('STAGING_NATIVE_JOBS_MISMATCH');
}
if (native.some((job: any) => String(job.run_id) !== runId || job.status !== 'completed' || job.conclusion !== 'success')) {
throw new Error('STAGING_NATIVE_GATE_NOT_PASSED');
}
}
function readJson(file: string): unknown {
const absolute = path.resolve(file);
const stat = fs.lstatSync(absolute);
if (!stat.isFile() || stat.isSymbolicLink() || stat.size <= 0 || stat.size > MAX_FILE_BYTES) throw new Error('UNSAFE_STAGING_EVIDENCE_FILE');
try { return JSON.parse(fs.readFileSync(absolute, 'utf8')); } catch { throw new Error('INVALID_STAGING_EVIDENCE_FILE'); }
}
function option(args: string[], name: string): string {
const index = args.indexOf(name);
const value = index >= 0 ? args[index + 1] : undefined;
if (!value || value.startsWith('--')) throw new Error(`Missing ${name}`);
return value;
}
if (import.meta.main) {
try {
const [command, ...args] = process.argv.slice(2);
if (command === 'run-id' && args[0] === '--repository' && args.length >= 3) {
process.stdout.write(`${stagingRunId(args[1], args.slice(2).map(readJson))}\n`);
} else if (command === 'verify' && args.length === 10) {
const event = readJson(option(args, '--event'));
verifyStagingRun({
repository: option(args, '--repository'),
runId: option(args, '--run-id'),
sourceCommit: dispatchSourceCommit(event),
runtimeIds: committedImageBuildMatrix().include.map(row => row.runtimeId),
run: readJson(option(args, '--run')),
jobPages: readJson(option(args, '--jobs')),
});
process.stdout.write('STAGING RUN VERIFIED\n');
} else {
throw new Error('Usage: cso-staging-run.ts run-id --repository OWNER/REPO PROVENANCE.json... | verify --repository OWNER/REPO --run-id ID --event EVENT.json --run RUN.json --jobs JOB-PAGES.json');
}
} catch (error) {
process.stderr.write((error instanceof Error ? error.message : 'STAGING_RUN_VERIFICATION_FAILED') + '\n');
process.exitCode = 1;
}
}