mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-10 10:40:32 +01:00
v1.91.23.0 feat(cso): private two-architecture release scoring and self-checking runtime release gates
This commit is contained in:
1 parent
f97aefe654
commit
9b21621462
35 files changed
+2284
-235
No files matched your search
+98
-20
@@ -27,6 +27,7 @@ CSO_NEW_LAUNCHER_INSTALLED=0
|
||||
CSO_NEW_CORE_INSTALLED=0
|
||||
CSO_NEW_WATCHDOG_INSTALLED=0
|
||||
CSO_NEW_GENERATION_INSTALLED=0
|
||||
CSO_EVALUATION_UNIT=""
|
||||
|
||||
cso_checkpoint() {
|
||||
[ "${GSTACK_CSO_BUILD_TESTING:-0}" = 1 ] || return 0
|
||||
@@ -87,6 +88,7 @@ cso_cleanup() {
|
||||
if ! cso_restore_previous; then status=1; cleanup_stage=0; fi
|
||||
fi
|
||||
if [ "$cleanup_stage" -eq 1 ] && [ -n "$CSO_STAGE" ] && [ ! -f "$CSO_STAGE/.retain-recovery" ]; then rm -rf "$CSO_STAGE"; fi
|
||||
if [ -n "$CSO_EVALUATION_UNIT" ]; then rm -rf "$CSO_EVALUATION_UNIT"; fi
|
||||
exit "$status"
|
||||
}
|
||||
trap 'cso_cleanup $?' EXIT
|
||||
@@ -143,6 +145,89 @@ cso_sha256() {
|
||||
printf '%s\n' "$digest"
|
||||
}
|
||||
|
||||
# Bun's compile-time switches stop configuration discovery in the audited
|
||||
# working directory before the helper can apply its own policy.
|
||||
cso_compile_bun() {
|
||||
"$BUN_CMD" build --compile \
|
||||
--no-compile-autoload-dotenv \
|
||||
--no-compile-autoload-bunfig \
|
||||
--no-compile-autoload-tsconfig \
|
||||
--no-compile-autoload-package-json \
|
||||
"$1" --outfile "$2"
|
||||
chmod +x "$2"
|
||||
}
|
||||
cso_compile_watchdog() {
|
||||
"$CSO_CC" -std=c11 -D_POSIX_C_SOURCE=200809L -O2 -Wall -Wextra \
|
||||
lib/cso/watchdog.c -o "$1"
|
||||
chmod +x "$1"
|
||||
}
|
||||
cso_compile_posix_launcher() {
|
||||
CSO_LAUNCHER_FLAGS=""
|
||||
case "$(uname -s)" in
|
||||
Linux) CSO_LAUNCHER_FLAGS="-static" ;;
|
||||
# Local ad-hoc signatures do not distinguish the launcher from another
|
||||
# ad-hoc dylib. This section makes dyld prune DYLD_* before constructors.
|
||||
Darwin) CSO_LAUNCHER_FLAGS="-Wl,-sectcreate,__RESTRICT,__restrict,/dev/null" ;;
|
||||
esac
|
||||
# shellcheck disable=SC2086 -- the optional platform flags are fixed literals.
|
||||
"$CSO_CC" -std=c11 -D_POSIX_C_SOURCE=200809L -O2 -Wall -Wextra $CSO_LAUNCHER_FLAGS \
|
||||
"-DGSTACK_CSO_CORE_SHA256=\"$2\"" lib/cso/launcher.c -o "$1"
|
||||
}
|
||||
|
||||
# An evaluation candidate embeds a cso-eval-* runtime catalog in a private
|
||||
# five-artifact producer unit outside this checkout. It never touches bin/, and
|
||||
# its .gstack-cso-evaluation manifest marks it evaluation-only for setup.
|
||||
cso_build_evaluation_candidate() {
|
||||
[ "$#" -eq 4 ] && [ "$1" = --evaluation-candidate ] && [ "$3" = --output ] || {
|
||||
echo 'Usage: build-cso.sh --evaluation-candidate <cso-eval-catalog.json> --output <new-directory>' >&2;return 64; }
|
||||
[ -z "$CSO_EXE" ] || { echo 'CSO evaluation candidates are built only on Linux and macOS.' >&2;return 69; }
|
||||
command -v "$CSO_CC" >/dev/null 2>&1 || { echo 'CSO build requires a C compiler (cc/clang/gcc) for the trusted launcher and watchdog.' >&2;return 1; }
|
||||
candidate_catalog=$2
|
||||
[ -f "$candidate_catalog" ] && [ ! -L "$candidate_catalog" ] || { echo 'CSO evaluation catalog must be one regular file.' >&2;return 66; }
|
||||
candidate_parent="$(cd "$(dirname "$4")" 2>/dev/null && pwd -P)" || { echo 'CSO evaluation output parent must be an existing directory.' >&2;return 73; }
|
||||
candidate_output="$candidate_parent/$(basename "$4")"
|
||||
if cso_present "$candidate_output";then echo 'CSO evaluation output must be a new directory.' >&2;return 73;fi
|
||||
case "$candidate_output/" in "$CSO_BUILD_ROOT"/*) echo 'CSO evaluation output must be outside the gstack checkout, where setup and distribution never look.' >&2;return 73;;esac
|
||||
candidate_revision="$(grep -o '"revision": *"cso-eval-[0-9]\{1,\}-[a-f0-9]\{12\}"' "$candidate_catalog" | head -n 1 | sed 's/.*"\(cso-eval-[^"]*\)"$/\1/' || true)"
|
||||
[ -n "$candidate_revision" ] || { echo 'CSO evaluation candidates require a cso-eval-<runId>-<sha12> catalog revision.' >&2;return 65; }
|
||||
|
||||
CSO_STAGE="$(mktemp -d "${TMPDIR:-/tmp}/gstack-cso-evaluation.XXXXXX")"
|
||||
CSO_EVALUATION_UNIT="$(mktemp -d "$candidate_parent/.gstack-cso-evaluation.XXXXXX")"
|
||||
source_root="$CSO_STAGE/source"
|
||||
mkdir -m 700 "$source_root" "$source_root/test"
|
||||
cp -R lib scripts "$source_root/"
|
||||
cp -R test/helpers "$source_root/test/"
|
||||
cp "$candidate_catalog" "$source_root/lib/cso/runtime-catalog.json"
|
||||
(cd "$source_root" && "$BUN_CMD" scripts/cso-runtime-promotion.ts check-evaluation lib/cso/runtime-catalog.json)
|
||||
(cd "$source_root" && cso_compile_bun lib/cso/cli.ts "$CSO_EVALUATION_UNIT/gstack-cso-core")
|
||||
(cd "$source_root" && cso_compile_bun scripts/cso-eval-producer.ts "$CSO_EVALUATION_UNIT/cso-eval-producer")
|
||||
cso_compile_watchdog "$CSO_EVALUATION_UNIT/gstack-cso-watchdog"
|
||||
if [ "$(uname -s)" = Darwin ]; then
|
||||
cso_sign_macos_artifact "$CSO_EVALUATION_UNIT/gstack-cso-core" 0
|
||||
cso_sign_macos_artifact "$CSO_EVALUATION_UNIT/gstack-cso-watchdog" 0
|
||||
fi
|
||||
candidate_core_sha256="$(cso_sha256 "$CSO_EVALUATION_UNIT/gstack-cso-core")"
|
||||
cso_compile_posix_launcher "$CSO_EVALUATION_UNIT/gstack-cso-launcher" "$candidate_core_sha256"
|
||||
chmod +x "$CSO_EVALUATION_UNIT/gstack-cso-launcher"
|
||||
if [ "$(uname -s)" = Darwin ];then cso_sign_macos_artifact "$CSO_EVALUATION_UNIT/gstack-cso-launcher" 1;fi
|
||||
printf '%s\n' "$candidate_core_sha256" > "$CSO_EVALUATION_UNIT/.gstack-cso-generation"
|
||||
printf '{"schemaVersion":1,"evaluationOnly":true,"runtimeCatalogRevision":"%s","runtimeCatalogSha256":"%s","coreSha256":"%s"}\n' \
|
||||
"$candidate_revision" "$(cso_sha256 "$source_root/lib/cso/runtime-catalog.json")" "$candidate_core_sha256" \
|
||||
> "$CSO_EVALUATION_UNIT/.gstack-cso-evaluation"
|
||||
for artifact in gstack-cso-core gstack-cso-launcher gstack-cso-watchdog cso-eval-producer;do
|
||||
cso_valid_artifact "$CSO_EVALUATION_UNIT/$artifact" || { echo "Evaluation $artifact is not one executable regular file." >&2;return 1; }
|
||||
done
|
||||
cso_valid_generation "$CSO_EVALUATION_UNIT/.gstack-cso-generation" || { echo 'Evaluation generation manifest is invalid.' >&2;return 1; }
|
||||
[ "$(cso_sha256 "$CSO_EVALUATION_UNIT/gstack-cso-core")" = "$candidate_core_sha256" ] || { echo 'Evaluation core changed after launcher binding.' >&2;return 1; }
|
||||
"$CSO_EVALUATION_UNIT/gstack-cso-launcher" --version >/dev/null
|
||||
chmod 0555 "$CSO_EVALUATION_UNIT"/gstack-cso-core "$CSO_EVALUATION_UNIT"/gstack-cso-launcher "$CSO_EVALUATION_UNIT"/gstack-cso-watchdog "$CSO_EVALUATION_UNIT"/cso-eval-producer
|
||||
chmod 0444 "$CSO_EVALUATION_UNIT/.gstack-cso-generation" "$CSO_EVALUATION_UNIT/.gstack-cso-evaluation"
|
||||
chmod 0755 "$CSO_EVALUATION_UNIT"
|
||||
mv "$CSO_EVALUATION_UNIT" "$candidate_output"
|
||||
CSO_EVALUATION_UNIT=""
|
||||
printf '{"output":"%s","revision":"%s","evaluationOnly":true}\n' "$candidate_output" "$candidate_revision"
|
||||
}
|
||||
|
||||
cso_publish_locked() {
|
||||
[ "${GSTACK_CSO_PUBLISH_LOCKED:-}" = 1 ] || { echo 'CSO publication requires the native publisher lock.' >&2;return 73; }
|
||||
[ "$#" -eq 1 ] && [ -d "$1" ] && [ ! -L "$1" ] || { echo 'CSO publication stage is invalid.' >&2;return 69; }
|
||||
@@ -201,6 +286,16 @@ if [ "${1:-}" = __publish_locked ];then
|
||||
cso_publish_locked "$@"
|
||||
exit 0
|
||||
fi
|
||||
if [ "${1:-}" = --evaluation-candidate ];then
|
||||
cso_build_evaluation_candidate "$@"
|
||||
exit 0
|
||||
fi
|
||||
[ "$#" -eq 0 ] || { echo 'Usage: build-cso.sh [--evaluation-candidate <cso-eval-catalog.json> --output <new-directory>]' >&2;exit 64; }
|
||||
# Distribution builds embed only the committed catalog, never an evaluation one.
|
||||
if [ -f lib/cso/runtime-catalog.json ] && grep -q '"revision": *"cso-eval-' lib/cso/runtime-catalog.json;then
|
||||
echo 'lib/cso/runtime-catalog.json is an evaluation-only catalog; distribution builds refuse it. Restore the committed catalog.' >&2
|
||||
exit 65
|
||||
fi
|
||||
|
||||
# This entrypoint can be run directly. Once it may publish a CSO generation,
|
||||
# the previous whole-build completion proof no longer describes all outputs.
|
||||
@@ -226,21 +321,13 @@ if [ -n "$CSO_EXE" ];then : > "$CSO_STAGE/.gstack-cso-generation.lock";fi
|
||||
|
||||
# These are compile-time switches. Bun otherwise discovers configuration in
|
||||
# the audited working directory before the helper can apply its own policy.
|
||||
"$BUN_CMD" build --compile \
|
||||
--no-compile-autoload-dotenv \
|
||||
--no-compile-autoload-bunfig \
|
||||
--no-compile-autoload-tsconfig \
|
||||
--no-compile-autoload-package-json \
|
||||
lib/cso/cli.ts --outfile "$CSO_STAGE_CORE"
|
||||
chmod +x "$CSO_STAGE_CORE"
|
||||
cso_compile_bun lib/cso/cli.ts "$CSO_STAGE_CORE"
|
||||
cso_checkpoint core
|
||||
|
||||
# POSIX process groups are required by the detached watchdog. Windows keeps
|
||||
# comprehensive execution unavailable instead of substituting a weaker timer.
|
||||
if [ -z "$CSO_EXE" ]; then
|
||||
"$CSO_CC" -std=c11 -D_POSIX_C_SOURCE=200809L -O2 -Wall -Wextra \
|
||||
lib/cso/watchdog.c -o "$CSO_STAGE_WATCHDOG"
|
||||
chmod +x "$CSO_STAGE_WATCHDOG"
|
||||
cso_compile_watchdog "$CSO_STAGE_WATCHDOG"
|
||||
cso_checkpoint watchdog
|
||||
fi
|
||||
|
||||
@@ -265,16 +352,7 @@ if [ -n "$CSO_EXE" ];then
|
||||
-CoreSha256 "$CSO_CORE_SHA256" \
|
||||
-GitExePath "$(cygpath -aw "$CSO_WINDOWS_GIT")"
|
||||
else
|
||||
CSO_LAUNCHER_FLAGS=""
|
||||
case "$(uname -s)" in
|
||||
Linux) CSO_LAUNCHER_FLAGS="-static" ;;
|
||||
# Local ad-hoc signatures do not distinguish the launcher from another
|
||||
# ad-hoc dylib. This section makes dyld prune DYLD_* before constructors.
|
||||
Darwin) CSO_LAUNCHER_FLAGS="-Wl,-sectcreate,__RESTRICT,__restrict,/dev/null" ;;
|
||||
esac
|
||||
# shellcheck disable=SC2086 -- the optional platform flags are fixed literals.
|
||||
"$CSO_CC" -std=c11 -D_POSIX_C_SOURCE=200809L -O2 -Wall -Wextra $CSO_LAUNCHER_FLAGS \
|
||||
"-DGSTACK_CSO_CORE_SHA256=\"$CSO_CORE_SHA256\"" lib/cso/launcher.c -o "$CSO_STAGE_LAUNCHER"
|
||||
cso_compile_posix_launcher "$CSO_STAGE_LAUNCHER" "$CSO_CORE_SHA256"
|
||||
"$CSO_CC" -std=c11 -D_POSIX_C_SOURCE=200809L -O2 -Wall -Wextra lib/cso/publish-lock.c -o "$CSO_STAGE_LOCKER"
|
||||
fi
|
||||
chmod +x "$CSO_STAGE_LAUNCHER" "$CSO_STAGE_LOCKER"
|
||||
|
||||
@@ -17,6 +17,8 @@ import { GeminiAdapter, prepareGeminiProducerState, removeGeminiProducerState }
|
||||
import { PRICING } from '../test/helpers/pricing';
|
||||
import type { ProviderAdapter, RunOpts, RunResult } from '../test/helpers/providers/types';
|
||||
import {
|
||||
PRODUCER_PLATFORMS,
|
||||
producerHostPlatform,
|
||||
producerInputHash,
|
||||
producerArtifactInventoryHash,
|
||||
producerInstallationIdentityHash,
|
||||
@@ -198,7 +200,8 @@ function repositoryIdentity(root: string): string {
|
||||
}
|
||||
|
||||
function validateCell(cell: ProducerCell): void {
|
||||
if (!cell || !HEX.test(cell.id) || !cell.caseId || !['node', 'bun', 'python', 'rails'].includes(cell.stack) || !['vulnerable', 'fixed'].includes(cell.variant) || !['v2', 'v3'].includes(cell.version) || !['daily', 'comprehensive'].includes(cell.mode) || ![1, 2, 3].includes(cell.repetition) || !cell.model || !['claude', 'codex', 'gemini'].includes(cell.host) || !Number.isInteger(cell.budgetSeconds) || cell.budgetSeconds <= 60 || cell.budgetSeconds > 3600 || !HEX.test(cell.sourceHash) || !HEX.test(cell.skillHash)) throw new Error('INVALID_PRODUCER_INPUT');
|
||||
if (!cell || !HEX.test(cell.id) || !cell.caseId || !['node', 'bun', 'python', 'rails'].includes(cell.stack) || !['vulnerable', 'fixed'].includes(cell.variant) || !['v2', 'v3'].includes(cell.version) || !['daily', 'comprehensive'].includes(cell.mode) || ![1, 2, 3].includes(cell.repetition) || !cell.model || !['claude', 'codex', 'gemini'].includes(cell.host) || !Number.isInteger(cell.budgetSeconds) || cell.budgetSeconds <= 60 || cell.budgetSeconds > 3600 || !HEX.test(cell.sourceHash) || !HEX.test(cell.skillHash) || (cell.platform !== undefined && !PRODUCER_PLATFORMS.includes(cell.platform))) throw new Error('INVALID_PRODUCER_INPUT');
|
||||
if (cell.platform !== undefined && cell.platform !== producerHostPlatform()) throw new Error('UNMATCHED_PRODUCER_PLATFORM');
|
||||
}
|
||||
|
||||
export interface ProducerHelperBinding {
|
||||
|
||||
@@ -5,6 +5,8 @@ export type ProducerVersion = 'v2' | 'v3';
|
||||
export type ProducerMode = 'daily' | 'comprehensive';
|
||||
export type ProducerStack = 'node' | 'bun' | 'python' | 'rails';
|
||||
export type ProducerVariant = 'vulnerable' | 'fixed';
|
||||
export type ProducerPlatform = 'linux/amd64' | 'linux/arm64';
|
||||
export const PRODUCER_PLATFORMS: readonly ProducerPlatform[] = ['linux/amd64', 'linux/arm64'];
|
||||
|
||||
export interface ProducerCell {
|
||||
id: string;
|
||||
@@ -19,6 +21,8 @@ export interface ProducerCell {
|
||||
budgetSeconds: number;
|
||||
sourceHash: string;
|
||||
skillHash: string;
|
||||
/** Release and baseline cells pin the native producer platform; full-profile cells omit it. */
|
||||
platform?: ProducerPlatform;
|
||||
}
|
||||
|
||||
export interface ProducerSourceEntry {
|
||||
@@ -122,6 +126,12 @@ export type ProducerReceiptIndex = Omit<ProducerReceipt, 'output' | 'error'> & {
|
||||
|
||||
export const sha256 = (value: string | Buffer): string => createHash('sha256').update(value).digest('hex');
|
||||
|
||||
/** The native platform this producer process runs on, or null when no CSO runtime platform matches it. */
|
||||
export function producerHostPlatform(platform: string = process.platform, arch: string = process.arch): ProducerPlatform | null {
|
||||
if (platform !== 'linux') return null;
|
||||
return arch === 'x64' ? 'linux/amd64' : arch === 'arm64' ? 'linux/arm64' : null;
|
||||
}
|
||||
|
||||
export function producerInputHash(input: ProducerInput): string {
|
||||
return sha256(JSON.stringify(input));
|
||||
}
|
||||
|
||||
+292
-69
@@ -7,8 +7,9 @@ import { dirname, join, resolve } from 'node:path';
|
||||
import { readBoundedStable } from '../lib/cso/bounded-file';
|
||||
import { executable } from '../lib/cso/process';
|
||||
import { atomicWriteSync } from '../lib/fs-atomic';
|
||||
import { loadCorpusManifest, materializeCase, sourceFiles, STACKS, type CorpusManifest, type EvalStack, type EvalVariant } from '../test/fixtures/cso-eval/materialize';
|
||||
import { loadCorpusManifest, materializeCase, publicCorpus, STACKS, validateCorpusManifest, type CorpusManifest, type EvalCorpus, type EvalStack, type EvalVariant } from '../test/fixtures/cso-eval/materialize';
|
||||
import {
|
||||
PRODUCER_PLATFORMS,
|
||||
producerArtifactInventoryHash,
|
||||
producerInputHash,
|
||||
producerInstallationIdentityHash,
|
||||
@@ -20,6 +21,7 @@ import {
|
||||
type ProducerHost,
|
||||
type ProducerInput,
|
||||
type ProducerInstallationIdentity,
|
||||
type ProducerPlatform,
|
||||
type ProducerProviderIdentity,
|
||||
type ProducerReceipt,
|
||||
type ProducerReceiptIndex,
|
||||
@@ -29,13 +31,53 @@ import {
|
||||
export type EvalVersion = 'v2' | 'v3';
|
||||
export type EvalMode = 'daily' | 'comprehensive';
|
||||
export type Outcome = 'passed' | 'failed' | 'blocked' | 'not_attempted';
|
||||
/**
|
||||
* full: matched v2/v3 × 3 repetitions on one producer installation.
|
||||
* release: v3 only, 1 repetition, each cell pinned to a native platform.
|
||||
* baseline: v2 comprehensive only, 1 repetition; scored into a reusable EvalBaseline.
|
||||
*/
|
||||
export type EvalProfile = 'full' | 'release' | 'baseline';
|
||||
export type EvalPlatform = ProducerPlatform;
|
||||
type EvalCorpusManifest = CorpusManifest<string>;
|
||||
export interface EvalCell extends ProducerCell {
|
||||
id: string; caseId: string; stack: EvalStack; variant: EvalVariant; version: EvalVersion; mode: EvalMode;
|
||||
repetition: 1 | 2 | 3; model: string; host: ProducerHost; budgetSeconds: number; sourceHash: string; skillHash: string;
|
||||
repetition: 1 | 2 | 3; model: string; host: ProducerHost; budgetSeconds: number; sourceHash: string; skillHash: string; platform?: EvalPlatform;
|
||||
}
|
||||
export interface EvalMatrix {
|
||||
interface EvalMatrixBase {
|
||||
schemaVersion: 1; corpusVersion: string; corpusHash: string; model: string; host: ProducerHost;
|
||||
budgets: { daily: number; comprehensive: number }; skillHashes: { v2: string; v3: string }; repetitions: 3; cells: EvalCell[];
|
||||
budgets: { daily: number; comprehensive: number }; cells: EvalCell[];
|
||||
}
|
||||
export interface FullEvalMatrix extends EvalMatrixBase { profile?: undefined; skillHashes: { v2: string; v3: string }; repetitions: 3 }
|
||||
export interface ReleaseEvalMatrix extends EvalMatrixBase {
|
||||
profile: 'release'; skillHashes: { v3: string }; repetitions: 1;
|
||||
/** Daily cells run on one platform; each comprehensive case is fixed to one platform. */
|
||||
platforms: { daily: EvalPlatform; comprehensive: Record<string, EvalPlatform> };
|
||||
/** Pre-registered external v2 baseline, or null when the regression gate stays unmeasured. */
|
||||
baselineHash: string | null;
|
||||
}
|
||||
export interface BaselineEvalMatrix extends EvalMatrixBase {
|
||||
profile: 'baseline'; skillHashes: { v2: string }; repetitions: 1;
|
||||
platforms: { comprehensive: Record<string, EvalPlatform> };
|
||||
}
|
||||
export type EvalMatrix = FullEvalMatrix | ReleaseEvalMatrix | BaselineEvalMatrix;
|
||||
interface EvalMatrixCommonOptions { model: string; host: ProducerHost; budgets?: { daily: number; comprehensive: number } }
|
||||
export interface FullEvalMatrixOptions extends EvalMatrixCommonOptions { profile?: 'full'; skillHashes: { v2: string; v3: string } }
|
||||
export interface ReleaseEvalMatrixOptions extends EvalMatrixCommonOptions { profile: 'release'; skillHashes: { v3: string }; platforms: ReleaseEvalMatrix['platforms']; baseline?: EvalBaseline }
|
||||
export interface BaselineEvalMatrixOptions extends EvalMatrixCommonOptions { profile: 'baseline'; skillHashes: { v2: string }; platforms: BaselineEvalMatrix['platforms'] }
|
||||
export type EvalMatrixOptions = FullEvalMatrixOptions | ReleaseEvalMatrixOptions | BaselineEvalMatrixOptions;
|
||||
/**
|
||||
* A scored v2 comprehensive run, reused by release matrices while every bound
|
||||
* input is unchanged. Release matrices pin its hash before results exist.
|
||||
*/
|
||||
export interface EvalBaseline {
|
||||
schemaVersion: 1; version: 'v2';
|
||||
corpusVersion: string; corpusHash: string; model: string; host: ProducerHost; budgetSeconds: number; skillHash: string;
|
||||
platforms: Record<string, EvalPlatform>;
|
||||
matrixHash: string; producerBatchHash: string; modelUsed: string;
|
||||
provider: { family: ProducerProviderIdentity['family']; policyRevision: string; version: string };
|
||||
/** One entry per corpus case, in manifest order: whether the vulnerable cell found the seeded issue. */
|
||||
cases: Array<{ caseId: string; found: boolean }>;
|
||||
baselineHash: string;
|
||||
}
|
||||
export interface PortableSkillFile {
|
||||
path: string;
|
||||
@@ -280,8 +322,8 @@ function initializeFixtureRepository(path: string): void {
|
||||
execFileSync(git, [...safe, 'commit', '--quiet', '-m', 'immutable evaluation fixture'], { cwd: path, env, stdio: 'ignore' });
|
||||
}
|
||||
|
||||
function sourceEntries(caseId: string, variant: EvalVariant): ProducerSourceEntry[] {
|
||||
return Object.entries(sourceFiles(caseId, variant)).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0).map(([path, contents]) => ({ path, sha256: sha256(contents), bytes: Buffer.byteLength(contents) }));
|
||||
function sourceEntries(corpus: EvalCorpus, caseId: string, variant: EvalVariant): ProducerSourceEntry[] {
|
||||
return Object.entries(corpus.sourceFiles(caseId, variant)).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0).map(([path, contents]) => ({ path, sha256: sha256(contents), bytes: Buffer.byteLength(contents) }));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -289,10 +331,15 @@ function sourceEntries(caseId: string, variant: EvalVariant): ProducerSourceEntr
|
||||
* evaluator metadata; transfer one job at a time and never give the schedule
|
||||
* or sibling jobs to the producing agent.
|
||||
*/
|
||||
export function prepareEvalJobs(matrix: EvalMatrix, payloads: { v2: string; v3: string }, destination: string, selectedCellIds?: string[]): PreparedEvalSchedule {
|
||||
validateMatrix(matrix);
|
||||
validatePortableSkillPayload(payloads.v2, 'v2'); validatePortableSkillPayload(payloads.v3, 'v3');
|
||||
if (sha256(payloads.v2) !== matrix.skillHashes.v2 || sha256(payloads.v3) !== matrix.skillHashes.v3) throw new Error('EVAL_SKILL_HASH_MISMATCH');
|
||||
export function prepareEvalJobs(matrix: EvalMatrix, payloads: Partial<Record<EvalVersion, string>>, destination: string, selectedCellIds?: string[], corpus: EvalCorpus = publicCorpus()): PreparedEvalSchedule {
|
||||
validateMatrix(matrix, corpus.manifest);
|
||||
const skillHashes: Partial<Record<EvalVersion, string>> = matrix.skillHashes;
|
||||
const versions = (['v2', 'v3'] as const).filter(version => skillHashes[version] !== undefined);
|
||||
if (!payloads || Object.keys(payloads).sort().join(',') !== versions.join(',')) throw new Error('EVAL_SKILL_HASH_MISMATCH');
|
||||
for (const version of versions) {
|
||||
validatePortableSkillPayload(payloads[version]!, version);
|
||||
if (sha256(payloads[version]!) !== skillHashes[version]) throw new Error('EVAL_SKILL_HASH_MISMATCH');
|
||||
}
|
||||
const root = resolve(destination);
|
||||
if (existsSync(root)) throw new Error('EVAL_DESTINATION_EXISTS');
|
||||
const parent = dirname(root);
|
||||
@@ -308,10 +355,10 @@ export function prepareEvalJobs(matrix: EvalMatrix, payloads: { v2: string; v3:
|
||||
const jobs: PreparedEvalSchedule['jobs'] = [];
|
||||
for (const cell of selected) {
|
||||
const jobRoot = join(jobsRoot, cell.id); mkdirSync(jobRoot, { mode: 0o700 });
|
||||
const source = materializeCase(cell.caseId, cell.variant, join(jobRoot, 'source'));
|
||||
const source = materializeCase(cell.caseId, cell.variant, join(jobRoot, 'source'), corpus);
|
||||
if (source.sourceHash !== cell.sourceHash) throw new Error('CORPUS_INTEGRITY_MISMATCH');
|
||||
initializeFixtureRepository(source.path);
|
||||
const input: ProducerInput = { schemaVersion: 1, cell, skill: payloads[cell.version], source: sourceEntries(cell.caseId, cell.variant) };
|
||||
const input: ProducerInput = { schemaVersion: 1, cell, skill: payloads[cell.version]!, source: sourceEntries(corpus, cell.caseId, cell.variant) };
|
||||
const inputHash = producerInputHash(input);
|
||||
safeWriteNew(join(jobRoot, 'producer-input.json'), input);
|
||||
jobs.push({ cellId: cell.id, relativePath: `jobs/${cell.id}`, inputHash });
|
||||
@@ -386,9 +433,41 @@ function validateReceiptIndex(receipt: ProducerReceiptIndex, cell: EvalCell): vo
|
||||
}
|
||||
|
||||
const matchedPairKey = (cell: EvalCell): string => [cell.caseId, cell.variant, cell.mode, cell.repetition, cell.model, cell.host, cell.budgetSeconds, cell.sourceHash].join('|');
|
||||
const PROFILE_GROUPS: Record<EvalProfile, ReadonlyArray<readonly [EvalVersion, EvalMode]>> = {
|
||||
full: [['v2', 'daily'], ['v2', 'comprehensive'], ['v3', 'daily'], ['v3', 'comprehensive']],
|
||||
release: [['v3', 'daily'], ['v3', 'comprehensive']],
|
||||
baseline: [['v2', 'comprehensive']],
|
||||
};
|
||||
const profileOf = (matrix: EvalMatrix): EvalProfile => matrix.profile ?? 'full';
|
||||
const expectedMatchedPairs = (matrix: EvalMatrix): number => profileOf(matrix) === 'full' ? matrix.cells.length / 2 : 0;
|
||||
|
||||
export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEvalSchedule, receipts: ProducerReceipt[]): ProducerBatch {
|
||||
validateMatrix(matrix); validateSchedule(matrix, schedule);
|
||||
type IdentityBearing = Pick<ProducerReceiptIndex, 'cell' | 'installationIdentity' | 'providerIdentity'>;
|
||||
/**
|
||||
* Each platform has exactly one producer installation and provider binary.
|
||||
* Platforms differ only in native binaries: the embedded catalogs and the
|
||||
* reviewed provider policy must be identical across them.
|
||||
*/
|
||||
function assertPlatformIdentities(receipts: IdentityBearing[]): void {
|
||||
const byPlatform = new Map<string, IdentityBearing[]>();
|
||||
for (const receipt of receipts) {
|
||||
const key = receipt.cell.platform ?? 'unpinned', list = byPlatform.get(key) ?? [];
|
||||
list.push(receipt); byPlatform.set(key, list);
|
||||
}
|
||||
for (const list of byPlatform.values()) {
|
||||
if (new Set(list.map(receipt => receipt.installationIdentity.identityHash)).size > 1) throw new Error('UNMATCHED_PRODUCER_INSTALLATIONS');
|
||||
if (new Set(list.map(receipt => receipt.providerIdentity.identityHash)).size > 1) throw new Error('UNMATCHED_PRODUCER_PROVIDERS');
|
||||
}
|
||||
const representatives = [...byPlatform.values()].map(list => list[0]);
|
||||
if (representatives.length < 2) return;
|
||||
if (new Set(representatives.map(receipt => receipt.installationIdentity.identityHash)).size !== representatives.length) throw new Error('UNMATCHED_PRODUCER_PLATFORMS');
|
||||
const catalogs = (receipt: IdentityBearing) => JSON.stringify(receipt.installationIdentity.embeddedCatalogs);
|
||||
const policy = ({ providerIdentity: { family, policyRevision, version, argsPrefix } }: IdentityBearing) => JSON.stringify({ family, policyRevision, version, argsPrefix });
|
||||
if (new Set(representatives.map(catalogs)).size !== 1) throw new Error('UNMATCHED_PRODUCER_INSTALLATIONS');
|
||||
if (new Set(representatives.map(policy)).size !== 1) throw new Error('UNMATCHED_PRODUCER_PROVIDERS');
|
||||
}
|
||||
|
||||
export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEvalSchedule, receipts: ProducerReceipt[], corpus: EvalCorpusManifest = loadCorpusManifest()): ProducerBatch {
|
||||
validateMatrix(matrix, corpus); validateSchedule(matrix, schedule);
|
||||
if (!Array.isArray(receipts)) throw new Error('INVALID_PRODUCER_RECEIPTS');
|
||||
const cells = new Map(matrix.cells.map(cell => [cell.id, cell]));
|
||||
const jobs = new Map(schedule.jobs.map(job => [job.cellId, job]));
|
||||
@@ -398,18 +477,18 @@ export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEv
|
||||
if (!cell || !job || seen.has(cell.id)) throw new Error('UNKNOWN_OR_DUPLICATE_PRODUCER_RECEIPT');
|
||||
validateReceipt(receipt, cell, job.inputHash); seen.add(cell.id);
|
||||
}
|
||||
if (new Set(receipts.map(receipt => receipt.installationIdentity.identityHash)).size > 1) throw new Error('UNMATCHED_PRODUCER_INSTALLATIONS');
|
||||
if (new Set(receipts.map(receipt => receipt.providerIdentity.identityHash)).size > 1) throw new Error('UNMATCHED_PRODUCER_PROVIDERS');
|
||||
assertPlatformIdentities(receipts);
|
||||
const pairs = new Map<string, Partial<Record<EvalVersion, ProducerReceipt>>>();
|
||||
for (const receipt of receipts) {
|
||||
if (profileOf(matrix) === 'full') for (const receipt of receipts) {
|
||||
const key = matchedPairKey(receipt.cell as EvalCell);
|
||||
const pair = pairs.get(key) ?? {}; pair[receipt.cell.version] = receipt; pairs.set(key, pair);
|
||||
}
|
||||
else if (new Set(receipts.map(receipt => receipt.modelUsed)).size > 1) throw new Error('UNMATCHED_EFFECTIVE_MODELS: single-version cells used different normalized model identities');
|
||||
const completePairs = [...pairs.entries()].filter(([, pair]) => pair.v2 && pair.v3) as Array<[string, { v2: ProducerReceipt; v3: ProducerReceipt }]>;
|
||||
const modelMismatches = completePairs.filter(([, pair]) => pair.v2.modelUsed !== pair.v3.modelUsed).map(([pair, value]) => ({ pair: digest(pair), v2: value.v2.modelUsed, v3: value.v3.modelUsed }));
|
||||
if (modelMismatches.length) throw new Error(`UNMATCHED_EFFECTIVE_MODELS: ${modelMismatches.length} matched v2/v3 pair(s) used different normalized model identities`);
|
||||
const groups: ProducerGroupSummary[] = [];
|
||||
for (const version of ['v2', 'v3'] as const) for (const mode of ['daily', 'comprehensive'] as const) {
|
||||
for (const [version, mode] of PROFILE_GROUPS[profileOf(matrix)]) {
|
||||
const expected = matrix.cells.filter(cell => cell.version === version && cell.mode === mode);
|
||||
const submitted = receipts.filter(receipt => receipt.cell.version === version && receipt.cell.mode === mode);
|
||||
const tokens = submitted.flatMap(receipt => receipt.usage.inputTokens === null || receipt.usage.outputTokens === null ? [] : [receipt.usage.inputTokens + receipt.usage.outputTokens]);
|
||||
@@ -424,29 +503,99 @@ export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEv
|
||||
const indexes: ProducerReceiptIndex[] = receipts.map(({ output: _output, error, ...receipt }) => ({ ...receipt, ...(error ? { error: { code: error.code } } : {}) }));
|
||||
const base = { schemaVersion: 1 as const, matrixHash: matrixHash(matrix), scheduleHash: digest(JSON.stringify(schedule)), receipts: indexes.sort((left, right) => left.cell.id.localeCompare(right.cell.id)), summary: {
|
||||
scheduled: matrix.cells.length, prepared: schedule.preparedCells, submitted: receipts.length, missing: matrix.cells.length - receipts.length,
|
||||
matchedPairsExpected: matrix.cells.length / 2, matchedPairsSubmitted: completePairs.length, modelMismatches, groups,
|
||||
matchedPairsExpected: expectedMatchedPairs(matrix), matchedPairsSubmitted: completePairs.length, modelMismatches, groups,
|
||||
note: 'Costs are pricing-table estimates. First-useful timing is unmeasured because the reused provider adapters return completed runs. Trusted findings and runtime outcomes require separate oracle adjudication.',
|
||||
} };
|
||||
return { ...base, batchHash: digest(JSON.stringify(base)) };
|
||||
}
|
||||
|
||||
export function createEvalMatrix(options: { model: string; host: ProducerHost; skillHashes: { v2: string; v3: string }; budgets?: { daily: number; comprehensive: number } }, corpus: CorpusManifest = loadCorpusManifest()): EvalMatrix {
|
||||
if (!options.model?.trim() || !['claude', 'codex', 'gemini'].includes(options.host) || !HEX.test(options.skillHashes.v2) || !HEX.test(options.skillHashes.v3) || options.skillHashes.v2 === options.skillHashes.v3) throw new Error('INVALID_MATCHED_EVAL_INPUT');
|
||||
const budgets = options.budgets ?? { daily: 600, comprehensive: 1800 };
|
||||
if (![budgets.daily, budgets.comprehensive].every(value => Number.isInteger(value) && value > 60 && value <= 3600)) throw new Error('INVALID_EVAL_BUDGET');
|
||||
const cells: EvalCell[] = [];
|
||||
for (const fixture of corpus.cases) for (const variant of ['vulnerable', 'fixed'] as const) for (const mode of ['daily', 'comprehensive'] as const) for (const repetition of [1, 2, 3] as const) for (const version of ['v2', 'v3'] as const) {
|
||||
const cell = { caseId: fixture.id, stack: fixture.stack, variant, version, mode, repetition, model: options.model, host: options.host, budgetSeconds: budgets[mode], sourceHash: fixture.filesHash[variant], skillHash: options.skillHashes[version] };
|
||||
cells.push({ id: digest(JSON.stringify(cell)), ...cell });
|
||||
}
|
||||
return { schemaVersion: 1, corpusVersion: corpus.version, corpusHash: digest(JSON.stringify(corpus)), model: options.model, host: options.host, budgets, skillHashes: options.skillHashes, repetitions: 3, cells };
|
||||
function normalizedPlatforms(corpus: EvalCorpusManifest, value: unknown): Record<string, EvalPlatform> {
|
||||
const assignment = value as Record<string, EvalPlatform>;
|
||||
if (!assignment || typeof assignment !== 'object' || Array.isArray(assignment) || Object.keys(assignment).length !== corpus.cases.length ||
|
||||
corpus.cases.some(fixture => !PRODUCER_PLATFORMS.includes(assignment[fixture.id]))) throw new Error('INVALID_EVAL_PLATFORMS');
|
||||
return Object.fromEntries(corpus.cases.map(fixture => [fixture.id, assignment[fixture.id]]));
|
||||
}
|
||||
|
||||
export function validateMatrix(matrix: EvalMatrix, corpus: CorpusManifest = loadCorpusManifest()): void {
|
||||
const expected = createEvalMatrix({ model: matrix.model, host: matrix.host, skillHashes: matrix.skillHashes, budgets: matrix.budgets }, corpus);
|
||||
function buildEvalMatrix(options: EvalMatrixOptions, corpus: EvalCorpusManifest, baselineHash: string | null): EvalMatrix {
|
||||
const profile = options.profile ?? 'full';
|
||||
const skillHashes: Partial<Record<EvalVersion, string>> = options.skillHashes ?? {};
|
||||
const versions = profile === 'full' ? ['v2', 'v3'] : profile === 'release' ? ['v3'] : ['v2'];
|
||||
if (!options.model?.trim() || !['claude', 'codex', 'gemini'].includes(options.host) || !['full', 'release', 'baseline'].includes(profile) ||
|
||||
Object.keys(skillHashes).sort().join(',') !== versions.join(',') || versions.some(version => !HEX.test(skillHashes[version as EvalVersion] ?? '')) ||
|
||||
(profile === 'full' && skillHashes.v2 === skillHashes.v3)) throw new Error('INVALID_MATCHED_EVAL_INPUT');
|
||||
const budgets = options.budgets ?? { daily: 600, comprehensive: 1800 };
|
||||
if (![budgets.daily, budgets.comprehensive].every(value => Number.isInteger(value) && value > 60 && value <= 3600)) throw new Error('INVALID_EVAL_BUDGET');
|
||||
const corpusHash = digest(JSON.stringify(corpus));
|
||||
const cells: EvalCell[] = [];
|
||||
if (options.profile === undefined || options.profile === 'full') {
|
||||
for (const fixture of corpus.cases) for (const variant of ['vulnerable', 'fixed'] as const) for (const mode of ['daily', 'comprehensive'] as const) for (const repetition of [1, 2, 3] as const) for (const version of ['v2', 'v3'] as const) {
|
||||
const cell = { caseId: fixture.id, stack: fixture.stack, variant, version, mode, repetition, model: options.model, host: options.host, budgetSeconds: budgets[mode], sourceHash: fixture.filesHash[variant], skillHash: options.skillHashes[version] };
|
||||
cells.push({ id: digest(JSON.stringify(cell)), ...cell });
|
||||
}
|
||||
return { schemaVersion: 1, corpusVersion: corpus.version, corpusHash, model: options.model, host: options.host, budgets, skillHashes: options.skillHashes, repetitions: 3, cells };
|
||||
}
|
||||
const pinned = options as ReleaseEvalMatrixOptions | BaselineEvalMatrixOptions;
|
||||
const platformKeys = pinned.profile === 'release' ? 'comprehensive,daily' : 'comprehensive';
|
||||
if (!pinned.platforms || Object.keys(pinned.platforms).sort().join(',') !== platformKeys) throw new Error('INVALID_EVAL_PLATFORMS');
|
||||
const comprehensive = normalizedPlatforms(corpus, pinned.platforms.comprehensive);
|
||||
const daily = pinned.profile === 'release' ? pinned.platforms.daily : undefined;
|
||||
if (pinned.profile === 'release' && !PRODUCER_PLATFORMS.includes(daily!)) throw new Error('INVALID_EVAL_PLATFORMS');
|
||||
const version: EvalVersion = pinned.profile === 'release' ? 'v3' : 'v2';
|
||||
const modes = pinned.profile === 'release' ? ['daily', 'comprehensive'] as const : ['comprehensive'] as const;
|
||||
for (const fixture of corpus.cases) for (const variant of ['vulnerable', 'fixed'] as const) for (const mode of modes) {
|
||||
const cell = { caseId: fixture.id, stack: fixture.stack, variant, version, mode, repetition: 1 as const, model: pinned.model, host: pinned.host, budgetSeconds: budgets[mode], sourceHash: fixture.filesHash[variant], skillHash: skillHashes[version]!, platform: mode === 'daily' ? daily! : comprehensive[fixture.id] };
|
||||
cells.push({ id: digest(JSON.stringify(cell)), ...cell });
|
||||
}
|
||||
const common = { schemaVersion: 1 as const, corpusVersion: corpus.version, corpusHash, model: pinned.model, host: pinned.host, budgets };
|
||||
if (pinned.profile === 'release') {
|
||||
if (baselineHash !== null && !HEX.test(baselineHash)) throw new Error('UNBOUND_EVAL_BASELINE');
|
||||
return { ...common, profile: 'release', skillHashes: { v3: pinned.skillHashes.v3 }, repetitions: 1, platforms: { daily: daily!, comprehensive }, baselineHash, cells };
|
||||
}
|
||||
return { ...common, profile: 'baseline', skillHashes: { v2: pinned.skillHashes.v2 }, repetitions: 1, platforms: { comprehensive }, cells };
|
||||
}
|
||||
|
||||
const BASELINE_FIELDS = 'baselineHash,budgetSeconds,cases,corpusHash,corpusVersion,host,matrixHash,model,modelUsed,platforms,producerBatchHash,provider,schemaVersion,skillHash,version';
|
||||
/** Check that a baseline was measured under the same corpus, model, host, budget and platform assignment. */
|
||||
function assertBaselineBinding(baseline: EvalBaseline, target: { corpus: EvalCorpusManifest; model: string; host: ProducerHost; budgetSeconds: number; platforms: Record<string, EvalPlatform> }): void {
|
||||
if (!baseline || typeof baseline !== 'object' || Object.keys(baseline).sort().join(',') !== BASELINE_FIELDS) throw new Error('UNBOUND_EVAL_BASELINE');
|
||||
const { baselineHash, ...record } = baseline;
|
||||
const provider = baseline.provider as unknown as Record<string, unknown>;
|
||||
if (baseline.schemaVersion !== 1 || baseline.version !== 'v2' || !HEX.test(baselineHash) || digest(JSON.stringify(record)) !== baselineHash ||
|
||||
baseline.corpusVersion !== target.corpus.version || baseline.corpusHash !== digest(JSON.stringify(target.corpus)) ||
|
||||
baseline.model !== target.model || baseline.host !== target.host || baseline.budgetSeconds !== target.budgetSeconds ||
|
||||
JSON.stringify(baseline.platforms) !== JSON.stringify(target.platforms) || !HEX.test(baseline.skillHash) || !HEX.test(baseline.matrixHash) ||
|
||||
!HEX.test(baseline.producerBatchHash) || typeof baseline.modelUsed !== 'string' || !baseline.modelUsed ||
|
||||
!provider || typeof provider !== 'object' || Object.keys(provider).sort().join(',') !== 'family,policyRevision,version' ||
|
||||
!['claude', 'gpt', 'gemini'].includes(String(provider.family)) || typeof provider.policyRevision !== 'string' || !provider.policyRevision ||
|
||||
typeof provider.version !== 'string' || !provider.version || !Array.isArray(baseline.cases) || baseline.cases.length !== target.corpus.cases.length ||
|
||||
baseline.cases.some((entry, index) => !entry || Object.keys(entry).sort().join(',') !== 'caseId,found' || entry.caseId !== target.corpus.cases[index].id || typeof entry.found !== 'boolean'))
|
||||
throw new Error('UNBOUND_EVAL_BASELINE');
|
||||
}
|
||||
|
||||
export function createEvalMatrix(options: FullEvalMatrixOptions, corpus?: EvalCorpusManifest): FullEvalMatrix;
|
||||
export function createEvalMatrix(options: ReleaseEvalMatrixOptions, corpus?: EvalCorpusManifest): ReleaseEvalMatrix;
|
||||
export function createEvalMatrix(options: BaselineEvalMatrixOptions, corpus?: EvalCorpusManifest): BaselineEvalMatrix;
|
||||
export function createEvalMatrix(options: EvalMatrixOptions, corpus: EvalCorpusManifest = loadCorpusManifest()): EvalMatrix {
|
||||
let baselineHash: string | null = null;
|
||||
if (options.profile === 'release' && options.baseline !== undefined) {
|
||||
assertBaselineBinding(options.baseline, { corpus, model: options.model, host: options.host, budgetSeconds: (options.budgets ?? { comprehensive: 1800 }).comprehensive, platforms: normalizedPlatforms(corpus, options.platforms?.comprehensive) });
|
||||
baselineHash = options.baseline.baselineHash;
|
||||
}
|
||||
return buildEvalMatrix(options, corpus, baselineHash);
|
||||
}
|
||||
|
||||
export function validateMatrix(matrix: EvalMatrix, corpus: EvalCorpusManifest = loadCorpusManifest()): void {
|
||||
if (!matrix || typeof matrix !== 'object') throw new Error('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX');
|
||||
const common = { model: matrix.model, host: matrix.host, budgets: matrix.budgets };
|
||||
let expected: EvalMatrix;
|
||||
try {
|
||||
expected = matrix.profile === 'release' ? buildEvalMatrix({ ...common, profile: 'release', skillHashes: matrix.skillHashes, platforms: matrix.platforms }, corpus, matrix.baselineHash)
|
||||
: matrix.profile === 'baseline' ? buildEvalMatrix({ ...common, profile: 'baseline', skillHashes: matrix.skillHashes, platforms: matrix.platforms }, corpus, null)
|
||||
: buildEvalMatrix({ ...common, skillHashes: matrix.skillHashes }, corpus, null);
|
||||
} catch { throw new Error('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX'); }
|
||||
if (JSON.stringify(matrix) !== JSON.stringify(expected)) throw new Error('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX');
|
||||
}
|
||||
function validateResult(result: EvalResult, cell: EvalCell, corpus: CorpusManifest): void {
|
||||
function validateResult(result: EvalResult, cell: EvalCell, corpus: EvalCorpusManifest): void {
|
||||
if (result.sourceHash !== cell.sourceHash || result.skillHash !== cell.skillHash || result.model !== cell.model || result.host !== cell.host || result.budgetSeconds !== cell.budgetSeconds) throw new Error('UNMATCHED_EVAL_RESULT');
|
||||
for (const field of ['setup', 'reproduction', 'repair', 'recheck'] as const) if (!['passed', 'failed', 'blocked', 'not_attempted'].includes(result[field])) throw new Error('INVALID_EVAL_OUTCOME');
|
||||
for (const field of ['reportPresent', 'reportComplete', 'heldOutAssertionsPassed', 'freshRecheck'] as const) if (typeof result[field] !== 'boolean') throw new Error('INVALID_EVAL_RESULT');
|
||||
@@ -493,7 +642,7 @@ function quantile(values: number[], fraction: number): number | null {
|
||||
if (!values.length) return null;
|
||||
return [...values].sort((left, right) => left - right)[Math.ceil(values.length * fraction) - 1];
|
||||
}
|
||||
function measuredRate(cells: EvalCell[], results: Map<string, EvalResult>, field: 'setup' | 'reproduction' | 'repair' | 'recheck', corpus: CorpusManifest): Rate {
|
||||
function measuredRate(cells: EvalCell[], results: Map<string, EvalResult>, field: 'setup' | 'reproduction' | 'repair' | 'recheck', corpus: EvalCorpusManifest): Rate {
|
||||
return rate(cells.filter(cell => {
|
||||
const result = results.get(cell.id);
|
||||
if (!result?.reportPresent || !result.reportComplete || result[field] !== 'passed') return false;
|
||||
@@ -511,7 +660,7 @@ function measuredRate(cells: EvalCell[], results: Map<string, EvalResult>, field
|
||||
}).length, cells.length);
|
||||
}
|
||||
|
||||
function trustedFindingVerification(cell: EvalCell, result: EvalResult, finding: EvalFinding, stage: 'repair' | 'recheck', corpus: CorpusManifest): boolean {
|
||||
function trustedFindingVerification(cell: EvalCell, result: EvalResult, finding: EvalFinding, stage: 'repair' | 'recheck', corpus: EvalCorpusManifest): boolean {
|
||||
const verification = finding.trustedVerification;
|
||||
if (!result.reportPresent || !result.reportComplete || cell.version !== 'v3' || cell.mode !== 'comprehensive' || cell.variant !== 'vulnerable' || finding.evidence !== 'supported' ||
|
||||
finding.judgment !== 'correct' || finding.matchedCaseId !== cell.caseId || !finding.claimedTested || !verification ||
|
||||
@@ -524,8 +673,7 @@ function trustedFindingVerification(cell: EvalCell, result: EvalResult, finding:
|
||||
!!result.recheckEvidenceHash && result.recheckEvidenceHash !== result.oracleEvidenceHash;
|
||||
}
|
||||
|
||||
export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualification: ReleaseQualification = { containment: {} }, corpus: CorpusManifest = loadCorpusManifest()) {
|
||||
validateMatrix(matrix, corpus);
|
||||
function validatedResults(matrix: EvalMatrix, observations: EvalResult[], corpus: EvalCorpusManifest): Map<string, EvalResult> {
|
||||
if (!Array.isArray(observations)) throw new Error('INVALID_EVAL_RESULTS');
|
||||
const cells = new Map(matrix.cells.map(cell => [cell.id, cell]));
|
||||
const results = new Map<string, EvalResult>();
|
||||
@@ -534,18 +682,34 @@ export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualif
|
||||
if (!cell || results.has(result.cellId)) throw new Error('UNKNOWN_OR_DUPLICATE_EVAL_RESULT');
|
||||
validateResult(result, cell, corpus); results.set(result.cellId, result);
|
||||
}
|
||||
const eligibleFinding = (cell: EvalCell, finding: EvalFinding) => finding.evidence === 'supported' || (cell.version === 'v2' && finding.evidence === 'legacy_review');
|
||||
const found = (cell: EvalCell) => {
|
||||
const result = results.get(cell.id);
|
||||
return result?.reportPresent === true && result.reportComplete === true &&
|
||||
result.findings.some(finding => eligibleFinding(cell, finding) && finding.judgment === 'correct');
|
||||
};
|
||||
return results;
|
||||
}
|
||||
const eligibleFinding = (cell: EvalCell, finding: EvalFinding) => finding.evidence === 'supported' || (cell.version === 'v2' && finding.evidence === 'legacy_review');
|
||||
const foundSeededIssue = (cell: EvalCell, result: EvalResult | undefined) => result?.reportPresent === true && result.reportComplete === true &&
|
||||
result.findings.some(finding => eligibleFinding(cell, finding) && finding.judgment === 'correct');
|
||||
const isHighCritical = (corpus: EvalCorpusManifest, caseId: string) => ['critical', 'high'].includes(corpus.cases.find(fixture => fixture.id === caseId)!.severity);
|
||||
|
||||
/** High/critical recall of a pre-registered external baseline, after proving it binds to this release matrix. */
|
||||
function externalBaselineRecall(matrix: ReleaseEvalMatrix, baseline: EvalBaseline, corpus: EvalCorpusManifest): Rate {
|
||||
if (matrix.baselineHash === null || baseline?.baselineHash !== matrix.baselineHash) throw new Error('UNBOUND_EVAL_BASELINE');
|
||||
assertBaselineBinding(baseline, { corpus, model: matrix.model, host: matrix.host, budgetSeconds: matrix.budgets.comprehensive, platforms: matrix.platforms.comprehensive });
|
||||
const highCritical = baseline.cases.filter(entry => isHighCritical(corpus, entry.caseId));
|
||||
return rate(highCritical.filter(entry => entry.found).length, highCritical.length);
|
||||
}
|
||||
|
||||
export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualification: ReleaseQualification = { containment: {} }, corpus: EvalCorpusManifest = loadCorpusManifest(), baseline?: EvalBaseline) {
|
||||
validateMatrix(matrix, corpus);
|
||||
const profile = profileOf(matrix);
|
||||
if (profile === 'baseline') throw new Error('BASELINE_PROFILE_NOT_QUALIFIABLE: use createEvalBaseline');
|
||||
if (profile !== 'release' && baseline !== undefined) throw new Error('UNBOUND_EVAL_BASELINE');
|
||||
const results = validatedResults(matrix, observations, corpus);
|
||||
const found = (cell: EvalCell) => foundSeededIssue(cell, results.get(cell.id));
|
||||
const groups: any[] = [];
|
||||
for (const version of ['v2', 'v3'] as const) for (const mode of ['daily', 'comprehensive'] as const) {
|
||||
for (const [version, mode] of PROFILE_GROUPS[profile]) {
|
||||
const selected = matrix.cells.filter(cell => cell.version === version && cell.mode === mode);
|
||||
const completed = selected.map(cell => results.get(cell.id)).filter((result): result is EvalResult => !!result);
|
||||
const expected = selected.filter(cell => cell.variant === 'vulnerable');
|
||||
const highCritical = expected.filter(cell => ['critical','high'].includes(corpus.cases.find(fixture => fixture.id === cell.caseId)!.severity));
|
||||
const highCritical = expected.filter(cell => isHighCritical(corpus, cell.caseId));
|
||||
let correct = 0, reported = 0, unadjudicated = 0;
|
||||
for (const cell of selected) {
|
||||
const result = results.get(cell.id);
|
||||
@@ -576,7 +740,8 @@ export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualif
|
||||
}
|
||||
const daily = groups.find(group => group.version === 'v3' && group.mode === 'daily');
|
||||
const comprehensive = groups.find(group => group.version === 'v3' && group.mode === 'comprehensive');
|
||||
const baseline = groups.find(group => group.version === 'v2' && group.mode === 'comprehensive');
|
||||
const inBatchBaseline = groups.find(group => group.version === 'v2' && group.mode === 'comprehensive');
|
||||
const external = matrix.profile === 'release' && baseline !== undefined ? externalBaselineRecall(matrix, baseline, corpus) : null;
|
||||
const perStack = Object.fromEntries(STACKS.map(stack => {
|
||||
const eligible = matrix.cells.filter(cell => cell.version === 'v3' && cell.mode === 'comprehensive' && cell.variant === 'vulnerable' && cell.stack === stack);
|
||||
// A repaired oracle case demonstrates the full find-to-repair workflow only
|
||||
@@ -596,24 +761,27 @@ export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualif
|
||||
mandatoryReports: gate(groups.every(group => group.reports.numerator === group.reports.denominator), observations.length === matrix.cells.length),
|
||||
dailyPrecision95: gate(daily.precision.value === null ? null : daily.precision.value >= 0.95, daily.submitted === daily.cells && !daily.unadjudicated),
|
||||
comprehensiveHighCriticalRecall80: gate(comprehensive.highCriticalRecall.value >= 0.8, comprehensive.submitted === comprehensive.cells && !comprehensive.unadjudicated),
|
||||
noHighCriticalRecallRegression: gate(comprehensive.highCriticalRecall.value >= baseline.highCriticalRecall.value, comprehensive.submitted === comprehensive.cells && baseline.submitted === baseline.cells && !comprehensive.unadjudicated && !baseline.unadjudicated),
|
||||
noHighCriticalRecallRegression: profile === 'full'
|
||||
? gate(comprehensive.highCriticalRecall.value >= inBatchBaseline.highCriticalRecall.value, comprehensive.submitted === comprehensive.cells && inBatchBaseline.submitted === inBatchBaseline.cells && !comprehensive.unadjudicated && !inBatchBaseline.unadjudicated)
|
||||
: gate(external === null ? null : comprehensive.highCriticalRecall.value >= external.value!, comprehensive.submitted === comprehensive.cells && !comprehensive.unadjudicated && external !== null),
|
||||
allCoreColdStarts: gate(comprehensive.setup.numerator === comprehensive.setup.denominator, comprehensive.submitted === comprehensive.cells),
|
||||
zeroFalselyTestedRepairs: gate(comprehensive.falseTested === 0 && daily.falseTested === 0, comprehensive.submitted === comprehensive.cells && daily.submitted === daily.cells),
|
||||
heldOutRepairEachStack: gate(Object.values(perStack).every(value => value.correctHeldOutRepairs >= 1), comprehensive.submitted === comprehensive.cells),
|
||||
containmentAndCanaries: containmentValues.includes('failed') ? 'fail' as const : containmentValues.every(value => value === 'passed') ? 'pass' as const : 'unmeasured' as const,
|
||||
};
|
||||
return { schemaVersion: 1, corpusVersion: corpus.version, model: matrix.model, host: matrix.host,
|
||||
...(profile === 'release' ? { profile, baseline: external === null ? null : { baselineHash: baseline!.baselineHash, highCriticalRecall: external } } : {}),
|
||||
status: Object.values(gates).every(value => value === 'pass') ? 'qualified' : observations.length ? 'partial' : 'unmeasured', groups, perStack, gates,
|
||||
notes: ['Missing and setup-blocked supported scenarios remain in recall and workflow denominators.', 'Costs and tokens come only from host-reported usage; wall-clock budgets are not model-spend caps.', 'Oracle judgments and qualification receipts must come from the trusted evaluator, never producing agents.'] };
|
||||
}
|
||||
|
||||
/** Release scoring path: every trusted judgment must bind to a collected producer receipt. */
|
||||
export function scoreCollectedEval(matrix: EvalMatrix, batch: ProducerBatch, observations: EvalResult[], qualification: ReleaseQualification = { containment: {} }, corpus: CorpusManifest = loadCorpusManifest()) {
|
||||
validateMatrix(matrix, corpus);
|
||||
/** Bind a complete collected batch and every trusted judgment to the matrix's producer receipts. */
|
||||
function validateCollectedBatch(matrix: EvalMatrix, batch: ProducerBatch, observations: EvalResult[]): Map<string, ProducerReceiptIndex> {
|
||||
if (!Array.isArray(observations)) throw new Error('INVALID_EVAL_RESULTS');
|
||||
if (!batch || batch.schemaVersion !== 1 || batch.matrixHash !== matrixHash(matrix) || !HEX.test(batch.scheduleHash) || !HEX.test(batch.batchHash)) throw new Error('INVALID_PRODUCER_BATCH');
|
||||
const { batchHash, ...withoutHash } = batch;
|
||||
if (digest(JSON.stringify(withoutHash)) !== batchHash || !Array.isArray(batch.receipts) || batch.receipts.length !== matrix.cells.length || batch.summary.scheduled !== matrix.cells.length || batch.summary.prepared !== matrix.cells.length || batch.summary.submitted !== matrix.cells.length || batch.summary.missing !== 0 || batch.summary.matchedPairsExpected !== matrix.cells.length / 2 || batch.summary.matchedPairsSubmitted !== matrix.cells.length / 2 || batch.summary.modelMismatches.length !== 0) throw new Error('INCOMPLETE_PRODUCER_BATCH');
|
||||
const pairs = expectedMatchedPairs(matrix);
|
||||
if (digest(JSON.stringify(withoutHash)) !== batchHash || !Array.isArray(batch.receipts) || batch.receipts.length !== matrix.cells.length || batch.summary.scheduled !== matrix.cells.length || batch.summary.prepared !== matrix.cells.length || batch.summary.submitted !== matrix.cells.length || batch.summary.missing !== 0 || batch.summary.matchedPairsExpected !== pairs || batch.summary.matchedPairsSubmitted !== pairs || batch.summary.modelMismatches.length !== 0) throw new Error('INCOMPLETE_PRODUCER_BATCH');
|
||||
const cells = new Map(matrix.cells.map(cell => [cell.id, cell]));
|
||||
const receipts = new Map<string, ProducerReceiptIndex>();
|
||||
for (const receipt of batch.receipts) {
|
||||
@@ -621,22 +789,67 @@ export function scoreCollectedEval(matrix: EvalMatrix, batch: ProducerBatch, obs
|
||||
if (!cell || receipts.has(cell.id)) throw new Error('INVALID_PRODUCER_BATCH');
|
||||
validateReceiptIndex(receipt, cell); receipts.set(cell.id, receipt);
|
||||
}
|
||||
if (new Set(batch.receipts.map(receipt => receipt.installationIdentity.identityHash)).size !== 1) throw new Error('UNMATCHED_PRODUCER_INSTALLATIONS');
|
||||
if (new Set(batch.receipts.map(receipt => receipt.providerIdentity.identityHash)).size !== 1) throw new Error('UNMATCHED_PRODUCER_PROVIDERS');
|
||||
const actualModels = new Map<string, Partial<Record<EvalVersion, string>>>();
|
||||
for (const receipt of batch.receipts) {
|
||||
const key = matchedPairKey(receipt.cell as EvalCell), pair = actualModels.get(key) ?? {};
|
||||
pair[receipt.cell.version] = receipt.modelUsed; actualModels.set(key, pair);
|
||||
}
|
||||
if (actualModels.size !== matrix.cells.length / 2 || [...actualModels.values()].some(pair => !pair.v2 || !pair.v3 || pair.v2 !== pair.v3)) throw new Error('UNMATCHED_EFFECTIVE_MODELS');
|
||||
assertPlatformIdentities(batch.receipts);
|
||||
if (profileOf(matrix) === 'full') {
|
||||
const actualModels = new Map<string, Partial<Record<EvalVersion, string>>>();
|
||||
for (const receipt of batch.receipts) {
|
||||
const key = matchedPairKey(receipt.cell as EvalCell), pair = actualModels.get(key) ?? {};
|
||||
pair[receipt.cell.version] = receipt.modelUsed; actualModels.set(key, pair);
|
||||
}
|
||||
if (actualModels.size !== pairs || [...actualModels.values()].some(pair => !pair.v2 || !pair.v3 || pair.v2 !== pair.v3)) throw new Error('UNMATCHED_EFFECTIVE_MODELS');
|
||||
} else if (new Set(batch.receipts.map(receipt => receipt.modelUsed)).size !== 1) throw new Error('UNMATCHED_EFFECTIVE_MODELS');
|
||||
for (const result of observations) {
|
||||
const receipt = receipts.get(result.cellId);
|
||||
if (!receipt || result.producerReceiptHash !== receipt.receiptHash) throw new Error('UNBOUND_EVAL_RESULT');
|
||||
}
|
||||
const score = scoreEval(matrix, observations, qualification, corpus);
|
||||
return receipts;
|
||||
}
|
||||
|
||||
/** Release scoring path: every trusted judgment must bind to a collected producer receipt. */
|
||||
export function scoreCollectedEval(matrix: EvalMatrix, batch: ProducerBatch, observations: EvalResult[], qualification: ReleaseQualification = { containment: {} }, corpus: EvalCorpusManifest = loadCorpusManifest(), baseline?: EvalBaseline) {
|
||||
validateMatrix(matrix, corpus);
|
||||
if (profileOf(matrix) === 'baseline') throw new Error('BASELINE_PROFILE_NOT_QUALIFIABLE: use createEvalBaseline');
|
||||
validateCollectedBatch(matrix, batch, observations);
|
||||
if (baseline !== undefined) {
|
||||
const { modelUsed, providerIdentity: { family, policyRevision, version } } = batch.receipts[0];
|
||||
if (baseline?.modelUsed !== modelUsed || JSON.stringify(baseline?.provider) !== JSON.stringify({ family, policyRevision, version })) throw new Error('UNBOUND_EVAL_BASELINE');
|
||||
}
|
||||
const score = scoreEval(matrix, observations, qualification, corpus, baseline);
|
||||
return { ...score, producerBatchHash: batch.batchHash };
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn a complete, fully adjudicated v2 baseline run into a reusable record.
|
||||
* Missing reports count as misses; an incomplete or unadjudicated run is refused.
|
||||
*/
|
||||
export function createEvalBaseline(matrix: BaselineEvalMatrix, batch: ProducerBatch, observations: EvalResult[], corpus: EvalCorpusManifest = loadCorpusManifest()): EvalBaseline {
|
||||
if (matrix?.profile !== 'baseline') throw new Error('INVALID_EVAL_BASELINE_MATRIX');
|
||||
validateMatrix(matrix, corpus);
|
||||
validateCollectedBatch(matrix, batch, observations);
|
||||
const results = validatedResults(matrix, observations, corpus);
|
||||
if (results.size !== matrix.cells.length) throw new Error('INCOMPLETE_EVAL_BASELINE');
|
||||
if ([...results.values()].some(result => result.reportPresent && result.reportComplete && result.findings.some(finding => finding.evidence !== 'hypothesis' && finding.judgment === 'unadjudicated'))) throw new Error('UNADJUDICATED_EVAL_BASELINE');
|
||||
const vulnerable = new Map(matrix.cells.filter(cell => cell.variant === 'vulnerable').map(cell => [cell.caseId, cell]));
|
||||
const { modelUsed, providerIdentity: { family, policyRevision, version } } = batch.receipts[0];
|
||||
const record: Omit<EvalBaseline, 'baselineHash'> = {
|
||||
schemaVersion: 1, version: 'v2', corpusVersion: corpus.version, corpusHash: matrix.corpusHash, model: matrix.model, host: matrix.host,
|
||||
budgetSeconds: matrix.budgets.comprehensive, skillHash: matrix.skillHashes.v2, platforms: matrix.platforms.comprehensive,
|
||||
matrixHash: matrixHash(matrix), producerBatchHash: batch.batchHash, modelUsed, provider: { family, policyRevision, version },
|
||||
cases: corpus.cases.map(fixture => ({ caseId: fixture.id, found: foundSeededIssue(vulnerable.get(fixture.id)!, results.get(vulnerable.get(fixture.id)!.id)) })),
|
||||
};
|
||||
return { ...record, baselineHash: digest(JSON.stringify(record)) };
|
||||
}
|
||||
|
||||
function splitFlags(args: string[], allowed: string[]): { positional: string[]; flags: Record<string, string> } {
|
||||
const positional: string[] = [], flags: Record<string, string> = {};
|
||||
for (let index = 0; index < args.length; index++) {
|
||||
if (!args[index].startsWith('--')) { positional.push(args[index]); continue; }
|
||||
if (!allowed.includes(args[index]) || !args[index + 1] || flags[args[index]]) throw new Error(`Unsupported or repeated flag: ${args[index]}`);
|
||||
flags[args[index]] = args[++index];
|
||||
}
|
||||
return { positional, flags };
|
||||
}
|
||||
|
||||
function cli(args: string[]): void {
|
||||
const [command, ...rest] = args;
|
||||
if (command === 'payload') {
|
||||
@@ -669,25 +882,35 @@ function cli(args: string[]): void {
|
||||
const schedule = prepareEvalJobs(matrix, { v2: readBoundedStable(resolve(rest[2]), PORTABLE_PAYLOAD_LIMIT, 'v2 portable skill payload').toString('utf8'), v3: readBoundedStable(resolve(rest[4]), PORTABLE_PAYLOAD_LIMIT, 'v3 portable skill payload').toString('utf8') }, rest[6]);
|
||||
console.log(JSON.stringify({ scheduled: schedule.scheduledCells, prepared: schedule.preparedCells, output: resolve(rest[6]), paidCalls: 0 })); return;
|
||||
}
|
||||
const { positional, flags } = splitFlags(rest, ['--corpus-manifest', '--baseline']);
|
||||
const corpus = flags['--corpus-manifest'] ? validateCorpusManifest(readJsonBounded(flags['--corpus-manifest'], 16 * 1024 * 1024, 'corpus manifest')) : loadCorpusManifest();
|
||||
if (command === 'collect') {
|
||||
if (rest.length !== 4) throw new Error('Usage: cso-eval collect <matrix.json> <schedule.json> <receipts-directory> <new-batch.json>');
|
||||
const matrix = readJsonBounded(rest[0], 16 * 1024 * 1024, 'evaluation matrix') as EvalMatrix;
|
||||
const schedule = readJsonBounded(rest[1], 16 * 1024 * 1024, 'evaluation schedule') as PreparedEvalSchedule;
|
||||
const receiptRoot = resolve(rest[2]), stat = lstatSync(receiptRoot);
|
||||
if (positional.length !== 4 || flags['--baseline']) throw new Error('Usage: cso-eval collect <matrix.json> <schedule.json> <receipts-directory> <new-batch.json> [--corpus-manifest <manifest.json>]');
|
||||
const [matrixPath, schedulePath, receiptDirectory, output] = positional;
|
||||
const matrix = readJsonBounded(matrixPath, 16 * 1024 * 1024, 'evaluation matrix') as EvalMatrix;
|
||||
const schedule = readJsonBounded(schedulePath, 16 * 1024 * 1024, 'evaluation schedule') as PreparedEvalSchedule;
|
||||
const receiptRoot = resolve(receiptDirectory), stat = lstatSync(receiptRoot);
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink() || realpathSync(receiptRoot) !== receiptRoot) throw new Error('UNSAFE_RECEIPT_DIRECTORY');
|
||||
const receipts: ProducerReceipt[] = [];
|
||||
for (const entry of readdirSync(receiptRoot, { withFileTypes: true })) {
|
||||
if (!entry.isFile() || entry.isSymbolicLink() || !/^[a-f0-9]{64}\.json$/.test(entry.name)) throw new Error('UNSAFE_RECEIPT_DIRECTORY');
|
||||
receipts.push(readJsonBounded(join(receiptRoot, entry.name), 40 * 1024 * 1024, 'producer receipt'));
|
||||
}
|
||||
const batch = collectProducerReceipts(matrix, schedule, receipts);
|
||||
safeWriteNew(resolve(rest[3]), batch);
|
||||
console.log(JSON.stringify({ output: resolve(rest[3]), submitted: batch.summary.submitted, missing: batch.summary.missing, modelMismatches: batch.summary.modelMismatches.length, paidCalls: 0 })); return;
|
||||
const batch = collectProducerReceipts(matrix, schedule, receipts, corpus);
|
||||
safeWriteNew(resolve(output), batch);
|
||||
console.log(JSON.stringify({ output: resolve(output), submitted: batch.summary.submitted, missing: batch.summary.missing, modelMismatches: batch.summary.modelMismatches.length, paidCalls: 0 })); return;
|
||||
}
|
||||
if (command === 'score') {
|
||||
if (rest.length < 3 || rest.length > 4) throw new Error('Usage: cso-eval score <matrix.json> <producer-batch.json> <trusted-results.json> [qualification.json]');
|
||||
console.log(JSON.stringify(scoreCollectedEval(readJsonBounded(rest[0], 16 * 1024 * 1024, 'evaluation matrix'), readJsonBounded(rest[1], 16 * 1024 * 1024, 'producer batch'), readJsonBounded(rest[2], 64 * 1024 * 1024, 'trusted results'), rest[3] ? readJsonBounded(rest[3], 4 * 1024 * 1024, 'qualification') : undefined), null, 2)); return;
|
||||
if (positional.length < 3 || positional.length > 4) throw new Error('Usage: cso-eval score <matrix.json> <producer-batch.json> <trusted-results.json> [qualification.json] [--corpus-manifest <manifest.json>] [--baseline <baseline.json>]');
|
||||
const baseline = flags['--baseline'] ? readJsonBounded(flags['--baseline'], 4 * 1024 * 1024, 'evaluation baseline') as EvalBaseline : undefined;
|
||||
console.log(JSON.stringify(scoreCollectedEval(readJsonBounded(positional[0], 16 * 1024 * 1024, 'evaluation matrix'), readJsonBounded(positional[1], 16 * 1024 * 1024, 'producer batch'), readJsonBounded(positional[2], 64 * 1024 * 1024, 'trusted results'), positional[3] ? readJsonBounded(positional[3], 4 * 1024 * 1024, 'qualification') : undefined, corpus, baseline), null, 2)); return;
|
||||
}
|
||||
throw new Error('Usage: cso-eval <payload|matrix|prepare|materialize|collect|score>');
|
||||
if (command === 'baseline') {
|
||||
if (positional.length !== 4 || flags['--baseline']) throw new Error('Usage: cso-eval baseline <baseline-matrix.json> <producer-batch.json> <trusted-results.json> <new-baseline.json> [--corpus-manifest <manifest.json>]');
|
||||
const baseline = createEvalBaseline(readJsonBounded(positional[0], 16 * 1024 * 1024, 'evaluation matrix'), readJsonBounded(positional[1], 16 * 1024 * 1024, 'producer batch'), readJsonBounded(positional[2], 64 * 1024 * 1024, 'trusted results'), corpus);
|
||||
safeWriteNew(resolve(positional[3]), baseline);
|
||||
console.log(JSON.stringify({ output: resolve(positional[3]), baselineHash: baseline.baselineHash, paidCalls: 0 })); return;
|
||||
}
|
||||
throw new Error('Usage: cso-eval <payload|matrix|prepare|materialize|collect|score|baseline>');
|
||||
}
|
||||
if (import.meta.main) { try { cli(process.argv.slice(2)); } catch (error) { console.error(error instanceof Error ? error.message : 'CSO evaluation failed'); process.exitCode = 1; } }
|
||||
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env bun
|
||||
/** Derive native runtime gate evidence from the named Docker tests that actually executed and passed. */
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { sha256 } from '../lib/cso/contracts';
|
||||
import type { ImageBuildRow } from './cso-image-matrix';
|
||||
import { requiredChecks } from './cso-runtime-promotion';
|
||||
|
||||
type Stack = ImageBuildRow['stack'];
|
||||
export interface TestRef { file: string; name: string }
|
||||
export interface JUnitCase extends TestRef { status: 'passed' | 'failed' | 'skipped'; assertions: number }
|
||||
|
||||
const MAX_REPORT_BYTES = 4 * 1024 * 1024;
|
||||
const DOCKER = 'test/cso-docker-integration.test.ts';
|
||||
const CONTAINMENT: TestRef[] = [
|
||||
{ file: DOCKER, name: 'hard fails when local daemon enforcement prerequisites are absent' },
|
||||
{ file: DOCKER, name: 'rejects image-declared writable volumes before container creation' },
|
||||
{ file: DOCKER, name: 'shares only loopback while denying egress, privileges, and daemon logs, and reads private source/policy mounts' },
|
||||
{ file: DOCKER, name: 'machine-wide admission allows only two groups per endpoint' },
|
||||
];
|
||||
const STAGED: TestRef = { file: DOCKER, name: 'staged runtime executes its trusted verifier and checks every declared tool version' };
|
||||
const NODE_LIFECYCLE: TestRef = {
|
||||
file: 'test/cso-node-lifecycle-integration.test.ts',
|
||||
name: 'finds, reproduces, bundles, replays, and closes against current source through the command dispatcher',
|
||||
};
|
||||
const COLD_JOURNEY: TestRef = {
|
||||
file: 'test/cso-stack-cold-integration.test.ts',
|
||||
name: 'acquires, prepares, boots, controls, and tests the requested staged stack with no target egress',
|
||||
};
|
||||
|
||||
/** Each native check key names the tests whose execution proves it for one stack. */
|
||||
export function nativeTestRequirements(stack: Stack): Record<string, TestRef[]> {
|
||||
if (stack === 'postgresql') return {
|
||||
containmentPassed: [...CONTAINMENT, STAGED],
|
||||
coldStartPassed: [STAGED],
|
||||
multiDatabasePassed: [STAGED],
|
||||
readinessPassed: [STAGED],
|
||||
};
|
||||
const journey = stack === 'node' ? NODE_LIFECYCLE : COLD_JOURNEY;
|
||||
const application = {
|
||||
containmentPassed: [...CONTAINMENT, STAGED],
|
||||
coldStartPassed: [journey],
|
||||
positiveNegativeAssertionsPassed: [STAGED],
|
||||
acquisitionPublicOnlyPassed: [journey],
|
||||
offlineLifecyclePassed: [journey],
|
||||
};
|
||||
return stack === 'rails'
|
||||
? { ...application, railsSqlitePassed: [COLD_JOURNEY], railsPostgresqlPassed: [COLD_JOURNEY], nativeExtensionsPassed: [COLD_JOURNEY] }
|
||||
: application;
|
||||
}
|
||||
|
||||
function decode(value: string): string {
|
||||
return value.replace(/&(#x[0-9a-fA-F]+|#[0-9]+|amp|lt|gt|quot|apos);/g, (_, entity: string) => {
|
||||
if (entity.startsWith('#x')) return String.fromCodePoint(Number.parseInt(entity.slice(2), 16));
|
||||
if (entity.startsWith('#')) return String.fromCodePoint(Number.parseInt(entity.slice(1), 10));
|
||||
return { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }[entity]!;
|
||||
});
|
||||
}
|
||||
|
||||
/** Parse the testcases of Bun's `--reporter=junit` output. */
|
||||
export function parseJUnit(xml: string): JUnitCase[] {
|
||||
if (!/^<\?xml[^>]*\?>\s*<testsuites\b/.test(xml)) throw new Error('INVALID_JUNIT_REPORT');
|
||||
const cases: JUnitCase[] = [];
|
||||
for (const match of xml.matchAll(/<testcase\b([^>]*?)(?:\/>|>([\s\S]*?)<\/testcase>)/g)) {
|
||||
const attributes = Object.fromEntries([...match[1].matchAll(/([A-Za-z_:][\w:.-]*)="([^"]*)"/g)].map(([, key, value]) => [key, decode(value)]));
|
||||
const body = match[2] ?? '';
|
||||
const status = /<(?:failure|error)\b/.test(body) ? 'failed' : /<skipped\b/.test(body) ? 'skipped' : 'passed';
|
||||
const assertions = Number(attributes.assertions ?? '0');
|
||||
if (typeof attributes.name !== 'string' || typeof attributes.file !== 'string' || !Number.isInteger(assertions) || assertions < 0) {
|
||||
throw new Error('INVALID_JUNIT_TESTCASE');
|
||||
}
|
||||
cases.push({ file: attributes.file, name: attributes.name, status, assertions });
|
||||
}
|
||||
if (cases.length === 0) throw new Error('EMPTY_JUNIT_REPORT');
|
||||
return cases;
|
||||
}
|
||||
|
||||
/** A check passes only when the whole report has no failure and every named test ran once, passed, and asserted. */
|
||||
export function deriveNativeChecks(stack: Stack, cases: JUnitCase[]): Record<string, boolean> {
|
||||
const clean = cases.some(item => item.status === 'passed') && !cases.some(item => item.status === 'failed');
|
||||
const passed = (ref: TestRef) => {
|
||||
const matches = cases.filter(item => item.file === ref.file && item.name === ref.name);
|
||||
return matches.length === 1 && matches[0].status === 'passed' && matches[0].assertions > 0;
|
||||
};
|
||||
return Object.fromEntries(Object.entries(nativeTestRequirements(stack))
|
||||
.map(([key, refs]) => [key, clean && refs.every(passed)]));
|
||||
}
|
||||
|
||||
/** Native evidence for one staged row; private checks are every required release gate the native job cannot prove. */
|
||||
export function nativeEvidence(staged: unknown, stack: Stack, platform: string, report: string): Record<string, unknown> {
|
||||
const record = staged as Record<string, unknown>;
|
||||
if (!record || typeof record !== 'object' || record.state !== 'staged' || record.stack !== stack || record.platform !== platform) {
|
||||
throw new Error('STAGED_EVIDENCE_MISMATCH');
|
||||
}
|
||||
const required = requiredChecks(stack);
|
||||
const nativeChecks = deriveNativeChecks(stack, parseJUnit(report));
|
||||
const failed = Object.entries(nativeChecks).filter(([key, value]) => !value || !required.includes(key)).map(([key]) => key);
|
||||
if (failed.length) throw new Error(`NATIVE_GATE_NOT_PASSED: ${failed.sort().join(', ')}`);
|
||||
return {
|
||||
...record,
|
||||
state: 'native-gates-passed',
|
||||
nativeChecks,
|
||||
nativeTestReport: `sha256:${sha256(report)}`,
|
||||
privateChecks: Object.fromEntries(required.filter(key => !(key in nativeChecks)).map(key => [key, 'pending'])),
|
||||
qualified: false,
|
||||
promotion: 'prohibited until authenticated private qualification evidence completes every release gate',
|
||||
};
|
||||
}
|
||||
|
||||
function readFile(file: string): string {
|
||||
const stat = fs.lstatSync(path.resolve(file));
|
||||
if (!stat.isFile() || stat.isSymbolicLink() || stat.size <= 0 || stat.size > MAX_REPORT_BYTES) throw new Error('UNSAFE_NATIVE_EVIDENCE_INPUT');
|
||||
return fs.readFileSync(path.resolve(file), 'utf8');
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
try {
|
||||
const args = process.argv.slice(2);
|
||||
const names = ['--stack', '--platform', '--junit', '--staged', '--output'];
|
||||
if (args.length !== names.length * 2 || names.some((name, index) => args[index * 2] !== name)) {
|
||||
throw new Error('Usage: cso-native-evidence.ts --stack STACK --platform PLATFORM --junit REPORT.xml --staged staged-image.json --output EVIDENCE.json');
|
||||
}
|
||||
const [stack, platform, junit, staged, output] = names.map((_, index) => args[index * 2 + 1]);
|
||||
if (!['node', 'bun', 'python', 'rails', 'postgresql'].includes(stack)) throw new Error('INVALID_STACK');
|
||||
const evidence = nativeEvidence(JSON.parse(readFile(staged)), stack as Stack, platform, readFile(junit));
|
||||
fs.writeFileSync(path.resolve(output), JSON.stringify(evidence, null, 2) + '\n', { flag: 'wx', mode: 0o600 });
|
||||
} catch (error) {
|
||||
process.stderr.write((error instanceof Error ? error.message : 'NATIVE_EVIDENCE_FAILED') + '\n');
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Requalification triggers: the inputs a runtime qualification was measured
|
||||
* under. A promoted catalog records them; a catalog whose recorded triggers
|
||||
* differ from the current source is older than its inputs and must be
|
||||
* requalified (or its runtimes withdrawn) before it ships.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { canonical, sha256 } from '../lib/cso/contracts';
|
||||
import { ISOLATION_POLICY_HASH } from '../lib/cso/docker';
|
||||
import { CSO_HELPER_ABI, isEvaluationRuntimeCatalog, type RequalificationTriggers, type RuntimeCatalog } from '../lib/cso/runtime-catalog';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
/** Compiled into every runtime image as gstack-cso-verifier and gstack-cso-preparation. */
|
||||
export const IMAGE_HELPER_ENTRYPOINTS = ['lib/cso/verifier.ts', 'lib/cso/preparation-container.ts'] as const;
|
||||
export const IMAGE_CONTEXT = 'lib/cso/images';
|
||||
/** Image-context files that describe qualification rather than enter an image; build-inputs.json is its own trigger. */
|
||||
const IMAGE_CONTEXT_EXCLUDED = new Set(['README.md', 'qualification.json', 'build-inputs.json']);
|
||||
|
||||
/** Every repository file whose bytes enter a runtime image: the helpers' import closure plus the image build context. */
|
||||
export async function imageSourceFiles(root = ROOT): Promise<string[]> {
|
||||
const build = await Bun.build({ entrypoints: IMAGE_HELPER_ENTRYPOINTS.map(entry => path.join(root, entry)), target: 'bun', metafile: true });
|
||||
if (!build.success || !build.metafile) throw new Error('REQUALIFICATION_SOURCE_CLOSURE_FAILED');
|
||||
const closure = Object.keys(build.metafile.inputs).map(input => path.relative(root, path.resolve(root, input)).split(path.sep).join('/'));
|
||||
if (closure.some(file => file.startsWith('..') || file.includes('node_modules/'))) throw new Error('REQUALIFICATION_SOURCE_OUTSIDE_REPOSITORY');
|
||||
const context = fs.readdirSync(path.join(root, IMAGE_CONTEXT), { withFileTypes: true })
|
||||
.filter(entry => entry.isFile() && !IMAGE_CONTEXT_EXCLUDED.has(entry.name))
|
||||
.map(entry => `${IMAGE_CONTEXT}/${entry.name}`);
|
||||
return [...new Set([...closure, ...context])].sort();
|
||||
}
|
||||
|
||||
export async function requalificationTriggers(root = ROOT): Promise<RequalificationTriggers> {
|
||||
const files = await imageSourceFiles(root);
|
||||
const buildInputs = JSON.parse(fs.readFileSync(path.join(root, IMAGE_CONTEXT, 'build-inputs.json'), 'utf8'));
|
||||
if (typeof buildInputs?.revision !== 'string' || !buildInputs.revision) throw new Error('INVALID_BUILD_INPUTS_REVISION');
|
||||
return {
|
||||
helperAbi: CSO_HELPER_ABI,
|
||||
isolationPolicyHash: ISOLATION_POLICY_HASH,
|
||||
preparationSha256: sha256(canonical(files.map(file => [file, sha256(fs.readFileSync(path.join(root, file)))]))),
|
||||
buildInputsRevision: buildInputs.revision,
|
||||
};
|
||||
}
|
||||
|
||||
/** Names of the triggers that changed since the catalog's runtimes were qualified; empty when nothing is promoted. */
|
||||
export function staleRequalificationTriggers(catalog: RuntimeCatalog, current: RequalificationTriggers): string[] {
|
||||
if (isEvaluationRuntimeCatalog(catalog) || catalog.runtimes.length === 0) return [];
|
||||
const recorded = catalog.promotion?.requalification;
|
||||
if (!recorded) return Object.keys(current).sort();
|
||||
return (Object.keys(current) as Array<keyof RequalificationTriggers>).filter(key => recorded[key] !== current[key]).sort();
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
try {
|
||||
const [command, file, ...rest] = process.argv.slice(2);
|
||||
if (command === 'triggers' && file === undefined) {
|
||||
process.stdout.write(JSON.stringify(await requalificationTriggers(), null, 2) + '\n');
|
||||
} else if (command === 'check' && file && rest.length === 0) {
|
||||
const stale = staleRequalificationTriggers(JSON.parse(fs.readFileSync(path.resolve(file), 'utf8')), await requalificationTriggers());
|
||||
if (stale.length) throw new Error(`STALE_RUNTIME_QUALIFICATION: ${stale.join(', ')} changed since qualification; requalify the runtimes or withdraw them from the catalog`);
|
||||
process.stdout.write('REQUALIFICATION CURRENT\n');
|
||||
} else throw new Error('Usage: cso-requalification.ts triggers | check <runtime-catalog.json>');
|
||||
} catch (error) {
|
||||
process.stderr.write((error instanceof Error ? error.message : 'REQUALIFICATION_CHECK_FAILED') + '\n');
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
@@ -7,12 +7,18 @@ import committedCatalog from '../lib/cso/runtime-catalog.json';
|
||||
import buildInputs from '../lib/cso/images/build-inputs.json';
|
||||
import {
|
||||
CSO_HELPER_ABI,
|
||||
EVALUATION_REVISION_PREFIX,
|
||||
isEvaluationRuntimeCatalog,
|
||||
validRequalificationTriggers,
|
||||
type RequalificationTriggers,
|
||||
type QualifiedRuntime,
|
||||
type RuntimeCatalog,
|
||||
type RuntimeQualification,
|
||||
validateRuntimeCatalog,
|
||||
} from '../lib/cso/runtime-catalog';
|
||||
import { ISOLATION_POLICY_HASH } from '../lib/cso/docker';
|
||||
import { imageBuildMatrix, type ImageBuildRow } from './cso-image-matrix';
|
||||
import { requalificationTriggers, staleRequalificationTriggers } from './cso-requalification';
|
||||
|
||||
const DIGEST = /^sha256:[a-f0-9]{64}$/;
|
||||
const WORKFLOW = /^https:\/\/github\.com\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+\/actions\/runs\/[0-9]+$/;
|
||||
@@ -46,7 +52,7 @@ function versionsKey(value: Record<string, string>): string {
|
||||
return canonical(value);
|
||||
}
|
||||
|
||||
function requiredChecks(stack: ImageBuildRow['stack']): string[] {
|
||||
export function requiredChecks(stack: ImageBuildRow['stack']): string[] {
|
||||
if (stack === 'postgresql') return [
|
||||
'containmentPassed', 'coldStartPassed', 'multiDatabasePassed', 'readinessPassed',
|
||||
'secretCanaryPassed', 'watchdogCleanupPassed',
|
||||
@@ -112,13 +118,15 @@ function validateStatement(raw: unknown, row: ImageBuildRow): RuntimeQualificati
|
||||
return statement;
|
||||
}
|
||||
|
||||
export function catalogPromotionCandidate(
|
||||
currentValue: unknown,
|
||||
inputValue: unknown,
|
||||
evidenceValues: unknown[],
|
||||
): RuntimeCatalog {
|
||||
/** The committed (never evaluation-only) catalog and the reviewed build matrix it must describe. */
|
||||
function reviewedMatrix(currentValue: unknown, inputValue: unknown): { current: RuntimeCatalog; rows: ImageBuildRow[] } {
|
||||
validateRuntimeCatalog(currentValue);
|
||||
const current = currentValue as RuntimeCatalog;
|
||||
if (isEvaluationRuntimeCatalog(current)) throw new Error('EVALUATION_CATALOG_NOT_PROMOTABLE');
|
||||
return { current, rows: profilesMatchingRows(current, inputValue) };
|
||||
}
|
||||
|
||||
function profilesMatchingRows(current: RuntimeCatalog, inputValue: unknown): ImageBuildRow[] {
|
||||
const rows = imageBuildMatrix(inputValue).include;
|
||||
if (!current.profiles || current.buildRevision !== rows[0].inputRevision) throw new Error('CATALOG_BUILD_REVISION_MISMATCH');
|
||||
const profiles = new Map(current.profiles.map(profile => [profile.id, profile]));
|
||||
@@ -128,17 +136,100 @@ export function catalogPromotionCandidate(
|
||||
if (!profile || profile.stack !== row.stack || profile.platform !== row.platform ||
|
||||
versionsKey(profile.versions) !== versionsKey(row.versions)) throw new Error(`CATALOG_PROFILE_MATRIX_MISMATCH: ${row.runtimeId}`);
|
||||
}
|
||||
if (evidenceValues.length !== rows.length) throw new Error('INCOMPLETE_QUALIFICATION_MATRIX');
|
||||
return rows;
|
||||
}
|
||||
|
||||
/** The candidate build's gate: a valid evaluation-only catalog for the reviewed build inputs. */
|
||||
export function validateEvaluationCandidate(catalogValue: unknown, inputValue: unknown): RuntimeCatalog {
|
||||
validateRuntimeCatalog(catalogValue);
|
||||
const catalog = catalogValue as RuntimeCatalog;
|
||||
if (!isEvaluationRuntimeCatalog(catalog)) throw new Error('NOT_AN_EVALUATION_CATALOG');
|
||||
const rows = profilesMatchingRows(catalog, inputValue);
|
||||
for (const row of rows) {
|
||||
const runtime = catalog.runtimes.find(item => item.id === row.runtimeId);
|
||||
if (!runtime || !OUTPUT_IMAGE.test(runtime.image) || !runtime.image.includes(`/${row.stack}-${row.arch}@`)) throw new Error(`INVALID_QUALIFIED_IMAGE: ${row.runtimeId}`);
|
||||
}
|
||||
return catalog;
|
||||
}
|
||||
|
||||
function matchRows(rows: ImageBuildRow[], values: unknown[]): Array<[ImageBuildRow, unknown]> {
|
||||
if (values.length !== rows.length) throw new Error('INCOMPLETE_QUALIFICATION_MATRIX');
|
||||
const byId = new Map<string, unknown>();
|
||||
for (const value of evidenceValues) {
|
||||
for (const value of values) {
|
||||
const id = (value as any)?.runtimeId;
|
||||
if (typeof id !== 'string' || byId.has(id)) throw new Error('DUPLICATE_QUALIFICATION_STATEMENT');
|
||||
byId.set(id, value);
|
||||
}
|
||||
const statements = rows.map(row => {
|
||||
return rows.map(row => {
|
||||
if (!byId.has(row.runtimeId)) throw new Error(`MISSING_QUALIFICATION_STATEMENT: ${row.runtimeId}`);
|
||||
return validateStatement(byId.get(row.runtimeId), row);
|
||||
return [row, byId.get(row.runtimeId)];
|
||||
});
|
||||
}
|
||||
|
||||
const STAGED_FIELDS = [
|
||||
'buildRevision', 'helperAbi', 'image', 'platform', 'provenanceDigest', 'qualification', 'runtimeId',
|
||||
'sbomDigest', 'schemaVersion', 'sourceCommit', 'stack', 'state', 'verifiedProvenance', 'versions', 'workflow',
|
||||
].sort();
|
||||
|
||||
/**
|
||||
* Build an evaluation-only catalog from one staging run's staged-image.json
|
||||
* records. It measures no release gate: every runtime is an
|
||||
* evaluation_candidate, and setup, distribution builds and promotion refuse it.
|
||||
*/
|
||||
export function evaluationCandidateCatalog(currentValue: unknown, inputValue: unknown, stagedValues: unknown[], createdAt: string): RuntimeCatalog {
|
||||
const { current, rows } = reviewedMatrix(currentValue, inputValue);
|
||||
if (!Number.isFinite(Date.parse(createdAt))) throw new Error('INVALID_EVALUATION_TIME');
|
||||
const staged = matchRows(rows, stagedValues).map(([row, value]) => {
|
||||
const record = value as RuntimeQualificationStatement;
|
||||
if (!record || typeof record !== 'object' || canonical(Object.keys(record).sort()) !== canonical(STAGED_FIELDS) ||
|
||||
record.schemaVersion !== 1 || record.helperAbi !== CSO_HELPER_ABI || (record.state as string) !== 'staged' || (record as any).verifiedProvenance !== true) throw new Error(`INVALID_STAGED_RUNTIME: ${row.runtimeId}`);
|
||||
if (record.buildRevision !== row.inputRevision || record.stack !== row.stack || record.platform !== row.platform ||
|
||||
versionsKey(record.versions) !== versionsKey(row.versions)) throw new Error(`QUALIFICATION_BUILD_MISMATCH: ${row.runtimeId}`);
|
||||
if (!OUTPUT_IMAGE.test(record.image) || !record.image.includes(`/${row.stack}-${row.arch}@`)) throw new Error(`INVALID_QUALIFIED_IMAGE: ${row.runtimeId}`);
|
||||
if (!/^[a-f0-9]{40}$/.test(record.sourceCommit) || !WORKFLOW.test(record.workflow) || !DIGEST.test(record.sbomDigest) ||
|
||||
!DIGEST.test(record.provenanceDigest)) throw new Error(`MISSING_QUALIFICATION_PROVENANCE: ${row.runtimeId}`);
|
||||
return record;
|
||||
});
|
||||
if (new Set(staged.map(item => item.sourceCommit)).size !== 1 || new Set(staged.map(item => item.workflow)).size !== 1) throw new Error('SPLIT_STAGING_RUN');
|
||||
const { sourceCommit, workflow } = staged[0];
|
||||
const runtimes: QualifiedRuntime[] = staged.map(record => ({
|
||||
id: record.runtimeId,
|
||||
stack: record.stack,
|
||||
platform: record.platform,
|
||||
state: 'evaluation_candidate',
|
||||
image: record.image,
|
||||
entrypoint: '/opt/cso/entrypoint',
|
||||
helperAbi: CSO_HELPER_ABI,
|
||||
versions: record.versions,
|
||||
policyVersion: 'cso-isolation-v1',
|
||||
qualifiedAt: createdAt,
|
||||
qualification: { kind: 'evaluation', sourceCommit, workflow, sbomDigest: record.sbomDigest, provenanceDigest: record.provenanceDigest, verifiedProvenance: true },
|
||||
}));
|
||||
const candidate: RuntimeCatalog = {
|
||||
schemaVersion: 1,
|
||||
revision: `${EVALUATION_REVISION_PREFIX}${workflow.slice(workflow.lastIndexOf('/') + 1)}-${sourceCommit.slice(0, 12)}`,
|
||||
previousRevision: current.revision,
|
||||
helperAbi: CSO_HELPER_ABI,
|
||||
buildRevision: current.buildRevision,
|
||||
profiles: current.profiles,
|
||||
evaluation: { sourceCommit, workflow, evidenceDigest: sha256(canonical(runtimes)) },
|
||||
runtimes,
|
||||
};
|
||||
validateRuntimeCatalog(candidate);
|
||||
return candidate;
|
||||
}
|
||||
|
||||
export function catalogPromotionCandidate(
|
||||
currentValue: unknown,
|
||||
inputValue: unknown,
|
||||
evidenceValues: unknown[],
|
||||
/** Measured by the staging run at sourceCommit (scripts/cso-requalification.ts triggers). */
|
||||
requalification: RequalificationTriggers,
|
||||
): RuntimeCatalog {
|
||||
const { current, rows } = reviewedMatrix(currentValue, inputValue);
|
||||
if (!validRequalificationTriggers(requalification, current.buildRevision)) throw new Error('INVALID_REQUALIFICATION_TRIGGERS');
|
||||
if (requalification.isolationPolicyHash !== ISOLATION_POLICY_HASH) throw new Error('STALE_RUNTIME_QUALIFICATION: isolationPolicyHash');
|
||||
const statements = matchRows(rows, evidenceValues).map(([row, value]) => validateStatement(value, row));
|
||||
const sourceCommits = new Set(statements.map(item => item.sourceCommit));
|
||||
const workflows = new Set(statements.map(item => item.workflow));
|
||||
if (sourceCommits.size !== 1 || workflows.size !== 1) throw new Error('SPLIT_QUALIFICATION_RUN');
|
||||
@@ -167,7 +258,15 @@ export function catalogPromotionCandidate(
|
||||
helperAbi: CSO_HELPER_ABI,
|
||||
buildRevision: current.buildRevision,
|
||||
profiles: current.profiles,
|
||||
promotion: { sourceCommit, workflow, evidenceDigest, qualificationEvidenceDigest },
|
||||
promotion: {
|
||||
sourceCommit, workflow, evidenceDigest, qualificationEvidenceDigest,
|
||||
requalification: {
|
||||
helperAbi: requalification.helperAbi,
|
||||
isolationPolicyHash: requalification.isolationPolicyHash,
|
||||
preparationSha256: requalification.preparationSha256,
|
||||
buildInputsRevision: requalification.buildInputsRevision,
|
||||
},
|
||||
},
|
||||
runtimes,
|
||||
};
|
||||
validateRuntimeCatalog(candidate);
|
||||
@@ -180,6 +279,7 @@ export function validateRuntimeCatalogTransition(currentValue: unknown, proposed
|
||||
validateRuntimeCatalog(proposedValue);
|
||||
const current = currentValue as RuntimeCatalog;
|
||||
const proposed = proposedValue as RuntimeCatalog;
|
||||
if (isEvaluationRuntimeCatalog(current) || isEvaluationRuntimeCatalog(proposed)) throw new Error('EVALUATION_CATALOG_NOT_PROMOTABLE');
|
||||
if (!proposed.promotion || proposed.runtimes.length === 0) throw new Error('UNQUALIFIED_RUNTIME_CATALOG_PROMOTION');
|
||||
if (proposed.revision === current.revision || proposed.previousRevision !== current.revision) {
|
||||
throw new Error('RUNTIME_CATALOG_BASE_REVISION_MISMATCH');
|
||||
@@ -190,7 +290,7 @@ export function validateRuntimeCatalogTransition(currentValue: unknown, proposed
|
||||
}
|
||||
}
|
||||
|
||||
function collectEvidence(root: string): unknown[] {
|
||||
function collectEvidence(root: string, name = 'qualified-runtime.json'): unknown[] {
|
||||
const absolute = path.resolve(root);
|
||||
const rootStat = fs.lstatSync(absolute);
|
||||
if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) throw new Error('UNSAFE_EVIDENCE_ROOT');
|
||||
@@ -201,7 +301,7 @@ function collectEvidence(root: string): unknown[] {
|
||||
const file = path.join(directory, entry.name);
|
||||
if (entry.isSymbolicLink() || (!entry.isDirectory() && !entry.isFile())) throw new Error('UNSAFE_EVIDENCE_ENTRY');
|
||||
if (entry.isDirectory()) visit(file, depth + 1);
|
||||
else if (entry.name === 'qualified-runtime.json') files.push(file);
|
||||
else if (entry.name === name) files.push(file);
|
||||
if (files.length > MAX_EVIDENCE_FILES) throw new Error('TOO_MANY_QUALIFICATION_STATEMENTS');
|
||||
}
|
||||
};
|
||||
@@ -234,7 +334,19 @@ function readCatalogArtifact(file: string): unknown {
|
||||
if (import.meta.main) {
|
||||
try {
|
||||
const args = process.argv.slice(2);
|
||||
if (args[0] === 'validate-transition') {
|
||||
if (args[0] === 'check-evaluation') {
|
||||
if (args.length !== 2) throw new Error('Usage: cso-runtime-promotion.ts check-evaluation <cso-eval-catalog>');
|
||||
const catalog = validateEvaluationCandidate(readCatalogArtifact(args[1]), buildInputs);
|
||||
process.stdout.write(`EVALUATION-ONLY ${catalog.revision}\n`);
|
||||
} else if (args[0] === 'evaluation-candidate') {
|
||||
if (args.length !== 5 || args[1] !== '--staged-root' || args[3] !== '--output') {
|
||||
throw new Error('Usage: cso-runtime-promotion.ts evaluation-candidate --staged-root <directory> --output <new-file>');
|
||||
}
|
||||
const candidate = evaluationCandidateCatalog(committedCatalog, buildInputs, collectEvidence(args[2], 'staged-image.json'), new Date().toISOString());
|
||||
const output = path.resolve(args[4]);
|
||||
fs.writeFileSync(output, JSON.stringify(candidate, null, 2) + '\n', { flag: 'wx', mode: 0o600 });
|
||||
process.stdout.write(`${output}\n`);
|
||||
} else if (args[0] === 'validate-transition') {
|
||||
if (args.length !== 3) throw new Error('Usage: cso-runtime-promotion.ts validate-transition <current-catalog> <proposed-catalog>');
|
||||
validateRuntimeCatalogTransition(readCatalogArtifact(args[1]), readCatalogArtifact(args[2]));
|
||||
process.stdout.write('VALID TRANSITION\n');
|
||||
@@ -242,7 +354,11 @@ if (import.meta.main) {
|
||||
if (args.length !== 4 || args[0] !== '--evidence-root' || args[2] !== '--output') {
|
||||
throw new Error('Usage: cso-runtime-promotion.ts --evidence-root <directory> --output <new-file>');
|
||||
}
|
||||
const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, collectEvidence(args[1]));
|
||||
const triggers = collectEvidence(args[1], 'requalification.json');
|
||||
if (triggers.length !== 1) throw new Error('MISSING_REQUALIFICATION_TRIGGERS');
|
||||
const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, collectEvidence(args[1]), triggers[0] as RequalificationTriggers);
|
||||
const stale = staleRequalificationTriggers(candidate, await requalificationTriggers());
|
||||
if (stale.length) throw new Error(`STALE_RUNTIME_QUALIFICATION: ${stale.join(', ')} changed since the staged commit; re-stage and requalify`);
|
||||
const output = path.resolve(args[3]);
|
||||
fs.writeFileSync(output, JSON.stringify(candidate, null, 2) + '\n', { flag: 'wx', mode: 0o600 });
|
||||
process.stdout.write(`${output}\n`);
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
#!/usr/bin/env bun
|
||||
/** Bind a qualification dispatch to the one successful protected-main staging run that signed its images. */
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { canonical } from '../lib/cso/contracts';
|
||||
import { committedImageBuildMatrix } from './cso-image-matrix';
|
||||
|
||||
export const STAGING_WORKFLOW_PATH = '.github/workflows/cso-runtime-images.yml';
|
||||
export const NATIVE_JOB = 'qualify-native';
|
||||
const SLSA_PROVENANCE = 'https://slsa.dev/provenance/v1';
|
||||
const REPOSITORY = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/;
|
||||
const INVOCATION = /^https:\/\/github\.com\/([A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+)\/actions\/runs\/([1-9][0-9]*)\/attempts\/[1-9][0-9]*$/;
|
||||
const MAX_FILE_BYTES = 4 * 1024 * 1024;
|
||||
|
||||
export interface StagingRunEvidence {
|
||||
repository: string;
|
||||
runId: string;
|
||||
sourceCommit: string;
|
||||
runtimeIds: string[];
|
||||
run: unknown;
|
||||
jobPages: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the staging run from verified SLSA provenance. The predicate's
|
||||
* invocation and the Fulcio certificate's run URI must agree, and every
|
||||
* verified statement for every image must name the same run.
|
||||
*/
|
||||
export function stagingRunId(repository: string, verifications: unknown[]): string {
|
||||
if (!REPOSITORY.test(repository) || verifications.length === 0) throw new Error('INVALID_STAGING_PROVENANCE');
|
||||
const runs = new Set<string>();
|
||||
for (const verification of verifications) {
|
||||
if (!Array.isArray(verification) || verification.length === 0) throw new Error('INVALID_STAGING_PROVENANCE');
|
||||
for (const item of verification) {
|
||||
const result = (item as any)?.verificationResult;
|
||||
if (result?.statement?.predicateType !== SLSA_PROVENANCE) throw new Error('INVALID_STAGING_PROVENANCE');
|
||||
const invocation = result.statement.predicate?.runDetails?.metadata?.invocationId;
|
||||
const match = typeof invocation === 'string' ? INVOCATION.exec(invocation) : null;
|
||||
if (!match || match[1] !== repository || result.signature?.certificate?.runInvocationURI !== invocation) {
|
||||
throw new Error('STAGING_INVOCATION_MISMATCH');
|
||||
}
|
||||
runs.add(match[2]);
|
||||
}
|
||||
}
|
||||
if (runs.size !== 1) throw new Error('STAGING_RUN_NOT_UNIQUE');
|
||||
return [...runs][0];
|
||||
}
|
||||
|
||||
/** Every statement in one qualification dispatch names the same staged source commit. */
|
||||
export function dispatchSourceCommit(event: unknown): string {
|
||||
const statements = (event as any)?.client_payload?.statements;
|
||||
if (!Array.isArray(statements) || statements.length === 0) throw new Error('INVALID_QUALIFICATION_DISPATCH');
|
||||
const commits = new Set(statements.map((statement: any) => statement?.sourceCommit));
|
||||
const [commit] = [...commits];
|
||||
if (commits.size !== 1 || typeof commit !== 'string' || !/^[a-f0-9]{40}$/.test(commit)) throw new Error('STAGING_SOURCE_COMMIT_MISMATCH');
|
||||
return commit;
|
||||
}
|
||||
|
||||
/** The resolved run must be the protected-main staging dispatch at the statements' commit, with every native gate passed. */
|
||||
export function verifyStagingRun(evidence: StagingRunEvidence): void {
|
||||
const { repository, runId, sourceCommit, runtimeIds } = evidence;
|
||||
const run = evidence.run as any;
|
||||
if (!run || typeof run !== 'object' || String(run.id) !== runId) throw new Error('STAGING_RUN_ID_MISMATCH');
|
||||
if (run.path !== STAGING_WORKFLOW_PATH) throw new Error('STAGING_WORKFLOW_MISMATCH');
|
||||
if (run.repository?.full_name !== repository || run.head_repository?.full_name !== repository) throw new Error('STAGING_REPOSITORY_MISMATCH');
|
||||
if (run.event !== 'workflow_dispatch' || run.head_branch !== 'main') throw new Error('STAGING_RUN_NOT_PROTECTED_MAIN_DISPATCH');
|
||||
if (run.status !== 'completed' || run.conclusion !== 'success') throw new Error('STAGING_RUN_NOT_SUCCESSFUL');
|
||||
if (run.head_sha !== sourceCommit) throw new Error('STAGING_SOURCE_COMMIT_MISMATCH');
|
||||
|
||||
const pages = evidence.jobPages;
|
||||
if (!Array.isArray(pages) || pages.length === 0) throw new Error('INVALID_STAGING_JOBS');
|
||||
const totals = new Set(pages.map((page: any) => page?.total_count));
|
||||
if (pages.some((page: any) => !Array.isArray(page?.jobs)) || totals.size !== 1) throw new Error('INVALID_STAGING_JOBS');
|
||||
const jobs = pages.flatMap((page: any) => page.jobs);
|
||||
if (jobs.length !== [...totals][0]) throw new Error('INCOMPLETE_STAGING_JOBS');
|
||||
|
||||
const native = jobs.filter((job: any) => typeof job?.name === 'string' && job.name.startsWith(NATIVE_JOB));
|
||||
const expected = runtimeIds.map(id => `${NATIVE_JOB} ${id}`).sort();
|
||||
if (runtimeIds.length === 0 || canonical(native.map((job: any) => job.name).sort()) !== canonical(expected)) {
|
||||
throw new Error('STAGING_NATIVE_JOBS_MISMATCH');
|
||||
}
|
||||
if (native.some((job: any) => String(job.run_id) !== runId || job.status !== 'completed' || job.conclusion !== 'success')) {
|
||||
throw new Error('STAGING_NATIVE_GATE_NOT_PASSED');
|
||||
}
|
||||
}
|
||||
|
||||
function readJson(file: string): unknown {
|
||||
const absolute = path.resolve(file);
|
||||
const stat = fs.lstatSync(absolute);
|
||||
if (!stat.isFile() || stat.isSymbolicLink() || stat.size <= 0 || stat.size > MAX_FILE_BYTES) throw new Error('UNSAFE_STAGING_EVIDENCE_FILE');
|
||||
try { return JSON.parse(fs.readFileSync(absolute, 'utf8')); } catch { throw new Error('INVALID_STAGING_EVIDENCE_FILE'); }
|
||||
}
|
||||
|
||||
function option(args: string[], name: string): string {
|
||||
const index = args.indexOf(name);
|
||||
const value = index >= 0 ? args[index + 1] : undefined;
|
||||
if (!value || value.startsWith('--')) throw new Error(`Missing ${name}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
try {
|
||||
const [command, ...args] = process.argv.slice(2);
|
||||
if (command === 'run-id' && args[0] === '--repository' && args.length >= 3) {
|
||||
process.stdout.write(`${stagingRunId(args[1], args.slice(2).map(readJson))}\n`);
|
||||
} else if (command === 'verify' && args.length === 10) {
|
||||
const event = readJson(option(args, '--event'));
|
||||
verifyStagingRun({
|
||||
repository: option(args, '--repository'),
|
||||
runId: option(args, '--run-id'),
|
||||
sourceCommit: dispatchSourceCommit(event),
|
||||
runtimeIds: committedImageBuildMatrix().include.map(row => row.runtimeId),
|
||||
run: readJson(option(args, '--run')),
|
||||
jobPages: readJson(option(args, '--jobs')),
|
||||
});
|
||||
process.stdout.write('STAGING RUN VERIFIED\n');
|
||||
} else {
|
||||
throw new Error('Usage: cso-staging-run.ts run-id --repository OWNER/REPO PROVENANCE.json... | verify --repository OWNER/REPO --run-id ID --event EVENT.json --run RUN.json --jobs JOB-PAGES.json');
|
||||
}
|
||||
} catch (error) {
|
||||
process.stderr.write((error instanceof Error ? error.message : 'STAGING_RUN_VERIFICATION_FAILED') + '\n');
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user