diff --git a/.github/workflows/cso-runtime-images.yml b/.github/workflows/cso-runtime-images.yml index 1f59c136c..6505a1910 100644 --- a/.github/workflows/cso-runtime-images.yml +++ b/.github/workflows/cso-runtime-images.yml @@ -12,6 +12,7 @@ on: - 'scripts/cso-image-matrix.ts' - 'scripts/cso-verify-runtime-base.ts' - 'scripts/cso-runtime-promotion.ts' + - 'scripts/cso-requalification.ts' - 'scripts/cso-public-ghcr.ts' - 'test/cso-*.test.ts' workflow_dispatch: @@ -42,6 +43,15 @@ jobs: set -eu matrix="$(bun run scripts/cso-image-matrix.ts)" printf 'matrix=%s\n' "$matrix" >> "$GITHUB_OUTPUT" + - name: Record the requalification trigger inputs of this source commit + run: bun run scripts/cso-requalification.ts triggers > requalification.json + - if: github.event_name == 'workflow_dispatch' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: cso-requalification-triggers + path: requalification.json + if-no-files-found: error + retention-days: 30 validate-native: needs: reviewed-inputs @@ -279,6 +289,7 @@ jobs: retention-days: 30 qualify-native: + name: qualify-native ${{ matrix.runtimeId }} needs: [reviewed-inputs, stage] if: github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' environment: cso-runtime-release @@ -344,31 +355,21 @@ jobs: set -eu bun install --frozen-lockfile --ignore-scripts bun run build:cso - bun run test:cso:docker - - name: Record native gate evidence without claiming runtime qualification + bun run test:cso:docker --reporter=junit --reporter-outfile=native-tests.junit.xml + - name: Derive native gate evidence from the executed test report env: CSO_STACK: ${{ matrix.stack }} CSO_PLATFORM: ${{ matrix.platform }} run: | set -eu - jq --arg stack "$CSO_STACK" --arg platform "$CSO_PLATFORM" ' - . + { - state:"native-gates-passed", - nativeChecks:(if $stack == "postgresql" then { - containmentPassed:true,coldStartPassed:true,multiDatabasePassed:true,readinessPassed:true - } else { - containmentPassed:true,coldStartPassed:true,positiveNegativeAssertionsPassed:true, - acquisitionPublicOnlyPassed:true,offlineLifecyclePassed:true - } + (if $stack == "rails" then { - railsSqlitePassed:true,railsPostgresqlPassed:true,nativeExtensionsPassed:true - } else {} end) end), - privateChecks:{heldOutRepairPassed:"pending",accuracyGatesPassed:"pending",secretCanaryPassed:"pending",watchdogCleanupPassed:"pending"}, - qualified:false, - promotion:"prohibited until authenticated private qualification evidence completes every release gate" - }' staged-application/staged-image.json > qualification-evidence.json + bun run scripts/cso-native-evidence.ts --stack "$CSO_STACK" --platform "$CSO_PLATFORM" \ + --junit native-tests.junit.xml --staged staged-application/staged-image.json \ + --output qualification-evidence.json - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: cso-qualification-gates-${{ matrix.stack }}-${{ matrix.arch }} - path: qualification-evidence.json + path: | + qualification-evidence.json + native-tests.junit.xml if-no-files-found: error retention-days: 30 diff --git a/.github/workflows/cso-runtime-promote.yml b/.github/workflows/cso-runtime-promote.yml index 5585be419..5a012ec19 100644 --- a/.github/workflows/cso-runtime-promote.yml +++ b/.github/workflows/cso-runtime-promote.yml @@ -3,6 +3,9 @@ name: Propose CSO Runtime Catalog Promotion # This workflow never deploys a catalog. It converts authenticated # same-repository qualification artifacts into an attested candidate, verifies # those exact bytes, and opens a normal source-review PR from a protected job. +# Pushes and PRs made with GITHUB_TOKEN start no workflows, so the job +# dispatches the required free-tests check on the pushed branch head; a +# workflow_dispatch is the documented exception to that rule. on: workflow_dispatch: inputs: @@ -28,7 +31,7 @@ jobs: permissions: contents: write pull-requests: write - actions: read + actions: write packages: read id-token: write attestations: write @@ -161,6 +164,7 @@ jobs: cmp runtime-catalog.candidate.json committed-runtime-catalog.json test "$expected_candidate_sha256" = "sha256:$(sha256sum committed-runtime-catalog.json | cut -d ' ' -f 1)" git push --set-upstream origin "$branch" + gh workflow run free-tests.yml --repo "$GITHUB_REPOSITORY" --ref "$branch" cat > pr-body.md <> "$GITHUB_STEP_SUMMARY" + qualify: + needs: dispatch-summary if: github.ref == 'refs/heads/main' && github.event_name == 'repository_dispatch' runs-on: ubuntu-24.04 timeout-minutes: 40 @@ -22,6 +60,7 @@ jobs: contents: read packages: read attestations: read + actions: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: @@ -29,18 +68,13 @@ jobs: - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 with: bun-version: 1.4.0 - - name: Require the configured private-evaluator actor and bounded matrix + - name: Require the configured private-evaluator actor env: CSO_QUALIFICATION_ACTOR: ${{ vars.CSO_QUALIFICATION_ACTOR }} run: | set -euo pipefail test -n "$CSO_QUALIFICATION_ACTOR" test "$GITHUB_ACTOR" = "$CSO_QUALIFICATION_ACTOR" - jq -e ' - .action == "cso-runtime-qualified" and - (.client_payload | keys == ["statements"]) and - (.client_payload.statements | type == "array" and length == 10) - ' "$GITHUB_EVENT_PATH" - name: Recheck public visibility and anonymous pulls for every qualified digest env: GH_TOKEN: ${{ github.token }} @@ -49,7 +83,7 @@ jobs: mkdir public-image-evidence jq -c '.client_payload.statements[]' "$GITHUB_EVENT_PATH" | while IFS= read -r statement; do runtime_id="$(printf '%s' "$statement" | jq -er '.runtimeId | select(test("^[a-z0-9][a-z0-9._-]{0,100}$"))')" - image="$(printf '%s' "$statement" | jq -er '.image | select(test("^ghcr.io/garrytan/gstack/cso-staging/[a-z0-9-]+-(amd64|arm64)@sha256:[a-f0-9]{64}$"))')" + image="$(printf '%s' "$statement" | jq -er '.image | select(test("^ghcr\\.io/garrytan/gstack/cso-staging/[a-z0-9-]+-(amd64|arm64)@sha256:[a-f0-9]{64}$"))')" platform="$(printf '%s' "$statement" | jq -er '.platform | select(. == "linux/amd64" or . == "linux/arm64")')" bun run scripts/cso-public-ghcr.ts verify \ --image "$image" --platform "$platform" --repository "$GITHUB_REPOSITORY" \ @@ -66,7 +100,7 @@ jobs: mkdir qualification-evidence jq -c '.client_payload.statements[]' "$GITHUB_EVENT_PATH" | while IFS= read -r statement; do runtime_id="$(printf '%s' "$statement" | jq -er '.runtimeId | select(test("^[a-z0-9][a-z0-9._-]{0,100}$"))')" - image="$(printf '%s' "$statement" | jq -er '.image | select(test("^ghcr.io/garrytan/gstack/cso-staging/[a-z0-9-]+-(amd64|arm64)@sha256:[a-f0-9]{64}$"))')" + image="$(printf '%s' "$statement" | jq -er '.image | select(test("^ghcr\\.io/garrytan/gstack/cso-staging/[a-z0-9-]+-(amd64|arm64)@sha256:[a-f0-9]{64}$"))')" source_commit="$(printf '%s' "$statement" | jq -er '.sourceCommit | select(test("^[a-f0-9]{40}$"))')" output="qualification-evidence/$runtime_id" mkdir "$output" @@ -92,6 +126,23 @@ jobs: }' \ > "$output/qualified-runtime.json" done + - name: Require one successful protected-main staging run with every native gate passed + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir staging-run-evidence + run_id="$(bun run scripts/cso-staging-run.ts run-id --repository "$GITHUB_REPOSITORY" \ + qualification-evidence/*/provenance-verification.json)" + gh api "repos/$GITHUB_REPOSITORY/actions/runs/$run_id" > staging-run-evidence/run.json + gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/jobs?filter=latest&per_page=100" \ + > staging-run-evidence/jobs.json + bun run scripts/cso-staging-run.ts verify --repository "$GITHUB_REPOSITORY" --run-id "$run_id" \ + --event "$GITHUB_EVENT_PATH" --run staging-run-evidence/run.json --jobs staging-run-evidence/jobs.json + gh run download "$run_id" --repo "$GITHUB_REPOSITORY" \ + --name cso-requalification-triggers --dir requalification-download + mv requalification-download/requalification.json qualification-evidence/requalification.json + rmdir requalification-download - name: Enforce the complete release contract before retaining evidence run: | set -euo pipefail @@ -105,5 +156,6 @@ jobs: path: | qualification-evidence public-image-evidence + staging-run-evidence if-no-files-found: error retention-days: 30 diff --git a/.github/workflows/cso-scanner-images.yml b/.github/workflows/cso-scanner-images.yml index e825f6849..2d7a172e2 100644 --- a/.github/workflows/cso-scanner-images.yml +++ b/.github/workflows/cso-scanner-images.yml @@ -319,7 +319,7 @@ jobs: permissions: contents: write pull-requests: write - actions: read + actions: write packages: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -365,6 +365,7 @@ jobs: git add lib/cso/scanner-images/catalog.json git commit -m "Promote qualified CSO scanner catalog" git push --set-upstream origin "$branch" + gh workflow run free-tests.yml --repo "$GITHUB_REPOSITORY" --ref "$branch" cat > pr-body.md <<'EOF' Promotes the complete scanner catalog produced by native amd64/arm64 qualification for all six CSO scanners. Each profile binds its immutable image, exact version-output hash, offline asset hash when required, containment and adapter gates, SBOM, provenance, source commit, and workflow run. diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f5b52182..45c917e58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## [1.91.25.0] - 2026-10-05 + +**gstack can grade a private, two-architecture `/cso` release run, and the runtime release gates check more for themselves.** + +These are the gstack-side pieces the private CSO runtime evaluator needs before it can qualify Node, Bun, Python and Rails runtimes. The release pipeline (`cso-runtime-images.yml` → private evaluator → `cso-runtime-qualification.yml` → `cso-runtime-promote.yml`) now verifies more of its own evidence, and nothing about a user's audit changes until a qualified runtime catalog merges. + +### What changes for you + +- **Private corpora score through the public scorer.** `scripts/cso-eval.ts` accepts an injected corpus with the public layout (`defineEvalCorpus`), per-cell `linux/amd64`/`linux/arm64` platforms with one producer installation and provider identity per platform, a `release` profile (v3, one repetition) and a hash-bound external v2 baseline. The `full` profile matrix is byte-identical to before. +- **Evaluation-only helper builds.** `bun run build:cso -- --evaluation-candidate --output ` builds a helper unit from a `cso-eval-` candidate catalog outside the checkout. It reports `evaluationOnly: true`; setup, promotion and normal builds refuse it. +- **The ingress checks the staging run itself.** `cso-runtime-qualification.yml` resolves the staging run from verified provenance and requires a successful protected-main `workflow_dispatch` run whose ten `qualify-native` jobs all passed. A dispatch summary job shows the approver the claimed images and an optional `evaluationRef` before approval. +- **Native gate evidence comes from test results.** Booleans are derived from JUnit executed-and-passed counts (`scripts/cso-native-evidence.ts`); skipped Docker tests no longer read as passing, and PostgreSQL rows list only the checks they need. +- **Catalog promotion PRs get their required checks.** Both promote jobs dispatch `free-tests.yml` on the new branch. +- **Multi-payload security proofs.** A verification request's `security` may hold 1–8 assertions judged in one before/after pair; a single assertion hashes exactly as before. +- **Producer cells are opaque.** A producer's input and receipt name only an opaque cell id; the evaluator maps it back to the case and variant, so an agent with root on its cell cannot learn which answer it is graded against. +- **Requalification triggers.** Qualified catalogs record the helper ABI, isolation policy hash, image preparation digest and build-inputs revision; a free test fails when the committed catalog no longer matches the code. + ## [1.91.24.0] - 2026-10-05 **PR evals stop throwing away work, and a red weekly census means something again.** diff --git a/VERSION b/VERSION index 5073fc2dd..aae93d7ed 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.24.0 +1.91.25.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index e63eea0d2..d816a5c03 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.24.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.25.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/cso-release-qualification.md b/docs/cso-release-qualification.md index 8b6317c09..239e8dadb 100644 --- a/docs/cso-release-qualification.md +++ b/docs/cso-release-qualification.md @@ -125,7 +125,20 @@ the embedded verifier against a loopback service with both a passing control and an intentionally failing control, alongside the containment suite. After all staging rows finish, native qualification jobs run the matching cold-start journey described above. The Node row additionally runs the complete lifecycle -journey. +journey. Each native job is named `qualify-native ` and runs the +Docker suites with Bun's JUnit reporter +(`--reporter=junit --reporter-outfile=native-tests.junit.xml`). +[`cso-native-evidence.ts`](../scripts/cso-native-evidence.ts) maps every native +check key to the named tests that prove it for that stack and records the key +true only when the report has no failed testcase and each named test appears +once, passed, and made at least one assertion. A test that the environment +skips, such as the staged-runtime smoke test without `GSTACK_CSO_TEST_IMAGE`, +reports as skipped and leaves its checks false; any false native check fails the +job. The pending private checks are the stack's required release gates from +`cso-runtime-promotion.ts` minus its native keys: `secretCanaryPassed` and +`watchdogCleanupPassed` for PostgreSQL, plus `heldOutRepairPassed` and +`accuracyGatesPassed` for application stacks. The job uploads the JUnit report +beside `qualification-evidence.json`, which records its digest. Registry publication is staging, not runtime qualification. GitHub Container Registry creates new packages private. Before a staging digest @@ -143,6 +156,12 @@ staging, qualification, and promotion. Both runtime and scanner promotion repeat this anonymous pull check on fresh hosted runners before opening a catalog PR. +The first protected-main staging run therefore stops at the public-package +check by design: its push creates the ten `cso-staging` packages as private. A +package administrator makes those ten packages public, then dispatches the +staging workflow again. Later runs publish new digests into the same public +packages. + Review the retained staging artifacts together with all checks in [`qualification.json`](../lib/cso/images/qualification.json). Cold-start fixtures, held-out repair evaluations, precision/recall thresholds, crash cleanup, and all @@ -158,7 +177,16 @@ the committed build profiles, and emits an attested candidate attestation against its workflow identity, protected-main source ref, source commit, and exact file digest. It then compares `previousRevision` with the source catalog, commits those same bytes to a fresh branch, and opens a -normal review PR. A concurrent catalog promotion fails the compare-and-swap or +normal review PR. Pushes and pull requests made with the workflow's +`GITHUB_TOKEN` start no workflows, so after pushing the branch the job dispatches +[`free-tests.yml`](../.github/workflows/free-tests.yml) on it with +`gh workflow run free-tests.yml --ref ` (its `actions: write` grant). +That run reports the required `free-tests` check on the branch head the PR +proposes. Opening the PR also requires the repository setting **Allow GitHub +Actions to create and approve pull requests** (Settings → Actions → General → +Workflow permissions). Without it `gh pr create` fails after the push and the +dispatch, and a maintainer opens the PR from the pushed branch. A concurrent +catalog promotion fails the compare-and-swap or produces a merge conflict instead of silently replacing the newer matrix. The catalog stores a separately recomputable digest of the retained runtime matrix and the digest of the complete external qualification statements. The @@ -171,16 +199,99 @@ The public qualification ingress is Set the protected environment variable `CSO_QUALIFICATION_ACTOR` to the GitHub service account used by the private evaluator. After its held-out assertions and accuracy run pass, that account sends a `repository_dispatch` event of type -`cso-runtime-qualified`. The payload has one key, `statements`, containing ten -objects with `schemaVersion`, `helperAbi`, `state`, `buildRevision`, -`runtimeId`, `stack`, `platform`, immutable `image`, exact `versions`, -`sourceCommit`, and the true-valued `checks` accepted by -`cso-runtime-promotion.ts`. The ingress checks the actor and matrix size, -re-verifies both OCI attestations against the protected staging workflow and -source commit, replaces the workflow, time, SBOM digest, and provenance digest -with values it observed, then runs the catalog generator as its final schema -gate. It uploads the statements only after that gate passes. The payload carries -no held-out assertion, application source, finding, or repair bundle. +`cso-runtime-qualified`. The payload has the key `statements` and, optionally, +`evaluationRef`; no other key is accepted. `statements` contains ten objects +with `schemaVersion`, `helperAbi`, `state`, `buildRevision`, `runtimeId`, +`stack`, `platform`, immutable `image`, exact `versions`, `sourceCommit`, and +the true-valued `checks` accepted by `cso-runtime-promotion.ts`. All ten name +one `sourceCommit`. `evaluationRef` is an opaque `sha256:<64 hex>` reference +that lets the approver match the dispatch to the private evaluator's verdict. +An unprotected `dispatch-summary` job with no token permissions validates the +payload shape and writes `evaluationRef`, the source commit, and each +`runtimeId` and `image` pair to the run's job summary. The reviewer reads it +there before approving the protected `qualify` job. `evaluationRef` never enters +the normalized statements, the uploaded evidence, or the catalog. + +The protected job checks the actor, re-verifies both OCI attestations against +the protected staging workflow and source commit, and replaces the workflow, +time, SBOM digest, and provenance digest with values it observed. Attestations +are signed in the staging job, before native qualification runs, so a valid +attestation alone does not show that native gates passed. +[`cso-staging-run.ts`](../scripts/cso-staging-run.ts) therefore resolves the +staging run from each verified SLSA provenance statement: the predicate's +`runDetails.metadata.invocationId` must equal the signing certificate's +`runInvocationURI`, and all ten images must resolve to the same run of this +repository. Through the Actions API (`actions: read`) the ingress then requires +that run to be `.github/workflows/cso-runtime-images.yml`, started by +`workflow_dispatch` on `main` in this repository, completed with conclusion +`success`, and built from the statements' `sourceCommit`. The latest attempt's +`qualify-native ` jobs, read across every page of the jobs API, must +be exactly one per committed matrix row (ten) and all `success`. The run and job +records are retained as `staging-run-evidence`. The catalog generator runs last +as the final schema gate, and the ingress uploads the statements only after that +gate passes. The payload carries no held-out assertion, application source, +finding, or repair bundle. + +A qualification stays valid only while the inputs it was measured under are +unchanged. [`cso-requalification.ts`](../scripts/cso-requalification.ts) defines +four requalification triggers: the helper ABI (`CSO_HELPER_ABI`), the isolation +policy (`ISOLATION_POLICY_HASH`, which covers the `cso-isolation-v1` policy +version and every limit), `preparationSha256` (a digest of every source byte +compiled or copied into a runtime image: the import closure of the image's +verifier and preparation helpers plus the image build context), and the +`build-inputs.json` revision. The staging run's `reviewed-inputs` job records +these values for its source commit as the `cso-requalification-triggers` +artifact. The ingress downloads that artifact from the verified staging run +into the evidence, and the promotion generator writes it into +`promotion.requalification`. Generation refuses triggers that differ from the +promoting checkout, so a trigger that changed between staging and promotion +requires a new staging run. The free suite's +[`cso-requalification.test.ts`](../test/cso-requalification.test.ts) compares +the committed catalog's recorded triggers with the current source and fails +when any differs. A change to a trigger therefore either lands with a +requalified catalog or withdraws the catalog's runtimes in the same pull +request. Independently, the helper refuses to select a runtime recorded under +another isolation policy. + +The judge observes held-out attacks in one boot: a `VerificationRequest`'s +`security` field is either one assertion or an array of up to eight assertions +with distinct names, judged in a single before/after pair. The before phase is +`intended_failure` only when every assertion shows its vulnerable expectation, +and the after phase passes only when every assertion passes its fixed +expectation. A single assertion keeps its original hashes. The verifier is +compiled into the runtime images, so array requests need images staged from a +commit that includes it. + +### Evaluation-only helper builds + +The private evaluator drives the real helper against staged digests before +any gate is measured. It never edits the committed catalog. Instead, +`bun scripts/cso-runtime-promotion.ts evaluation-candidate --staged-root +--output ` converts one staging run's ten `staged-image.json` records +into an evaluation-only catalog. Its revision is `cso-eval--`, it +carries an `evaluation` record instead of `promotion`, and every runtime is an +`evaluation_candidate` whose qualification has `kind: "evaluation"` and no +release-gate field. Then + +```bash +bun run build:cso -- --evaluation-candidate --output +``` + +compiles the helper core and `cso-eval-producer` from a private copy of the +source with that catalog embedded, adds the native launcher and watchdog, and +writes the five-artifact producer unit to a new directory outside the checkout. +Beside the unchanged `.gstack-cso-generation` manifest, it writes +`.gstack-cso-evaluation`, which marks the unit evaluation-only and records the +catalog revision, catalog SHA-256, and core SHA-256. The helper reports +`"evaluationOnly": true` from `--version`, labels its catalog evaluation-only +in runtime readiness, and refuses `provision-images --setup-summary`. +Distribution refuses it at every step. A normal `build:cso` stops when +`lib/cso/runtime-catalog.json` carries a `cso-eval-` revision. Setup fails when +`bin/` holds an `.gstack-cso-evaluation` manifest or a launcher that reports +`evaluationOnly`. Catalog promotion and transitions reject evaluation catalogs, +and the runtime catalog validator rejects an evaluation catalog that carries +`promotion`, release-gate fields, or a revision that does not name its staging +run and source commit. The current reviewed profiles and unpromoted catalog establish this release process; they do not claim published gstack images, passed containment, or diff --git a/docs/cso-scanners.md b/docs/cso-scanners.md index 342ef23ee..f1a904c7d 100644 --- a/docs/cso-scanners.md +++ b/docs/cso-scanners.md @@ -99,7 +99,15 @@ deployments restricted to protected `main`. GitHub otherwise creates a reference missing environment without protection rules. Promotion requires an explicit dispatch from `main` and approval through that configured environment. It opens a pull request containing only the proposed source catalog, so code review remains the final -authorization step. `previousRevision` records the prior compatible catalog for +authorization step. Pushes and pull requests made with the workflow's +`GITHUB_TOKEN` start no workflows, so after pushing the promotion branch the job +dispatches [`free-tests.yml`](../.github/workflows/free-tests.yml) on it +(`actions: write`); that run reports the required `free-tests` check on the +branch head. Opening the PR also requires the repository setting **Allow GitHub +Actions to create and approve pull requests** (Settings → Actions → General → +Workflow permissions); without it `gh pr create` fails after the push and the +dispatch, and a maintainer opens the PR from the pushed branch. +`previousRevision` records the prior compatible catalog for rollback and must still equal the current `main` catalog when promotion runs. Scanner flags follow the primary documentation linked in diff --git a/lib/cso/cli.ts b/lib/cso/cli.ts index e42008414..f27b1db8d 100644 --- a/lib/cso/cli.ts +++ b/lib/cso/cli.ts @@ -61,6 +61,7 @@ import { executable, git, redact, sanitizeForJson, sanitizeHelperForJson } from import { inspectPreparation } from './preparation'; import { assertRuntimeCompatible, + isEvaluationRuntimeCatalog, RUNTIME_CATALOG, selectRuntime, validateRuntimeCatalog, @@ -307,7 +308,7 @@ const SCHEMA = { }, ], security: { - name: 'security assertion', + name: 'security assertion; or send an array of 1..8 assertions with distinct names, all judged in one before/after boot pair', path: '/path', method: 'GET|POST|PUT|PATCH|DELETE', expected: { status: 'fixed status', includes: 'optional', excludes: 'optional' }, @@ -669,7 +670,9 @@ async function start( try { validateRuntimeCatalog(dependencies.runtimeCatalog); selectRuntime(plan.runtimeProfile, platform(), dependencies.runtimeCatalog); - c.evidence.push(`Qualified runtime catalog: ${dependencies.runtimeCatalog.revision}`); + c.evidence.push( + `${isEvaluationRuntimeCatalog(dependencies.runtimeCatalog) ? 'Evaluation-only' : 'Qualified'} runtime catalog: ${dependencies.runtimeCatalog.revision}`, + ); } catch (error: any) { c.gaps.push( error?.message?.startsWith('MISSING_QUALIFIED_RUNTIME') @@ -864,6 +867,8 @@ async function provisionImages(args: string[], dependencies: CsoCliDependencies) const setupSummary = take(args, '--setup-summary'), requestedSeconds = args.includes('--per-image-seconds') ? need(args, '--per-image-seconds') : undefined; if (args.length) throw new CsoError('INVALID_ARGUMENT', `Unknown argument: ${args[0]}`); + if (setupSummary && isEvaluationRuntimeCatalog(dependencies.runtimeCatalog)) + throw new CsoError('INCOMPATIBLE_INPUT', 'Setup never installs an evaluation-only CSO helper build'); if (requestedSeconds !== undefined) catalogImageProvisioningPolicy(0, requestedSeconds); let targetPlatform: 'linux/amd64' | 'linux/arm64'; try { @@ -2740,7 +2745,11 @@ async function main() { return; } if (command === '--version') { - emit({ version: VERSION, abi: ABI }); + emit({ + version: VERSION, + abi: ABI, + ...(isEvaluationRuntimeCatalog(RUNTIME_CATALOG) ? { evaluationOnly: true } : {}), + }); return; } if (command === 'schema') { diff --git a/lib/cso/contracts.ts b/lib/cso/contracts.ts index 52610986f..743537345 100644 --- a/lib/cso/contracts.ts +++ b/lib/cso/contracts.ts @@ -184,7 +184,7 @@ export interface VerificationRequest { port: number; start: Command; legitimate: HttpAssertion[]; - security: HttpAssertion; + security: HttpAssertion | HttpAssertion[]; existingTests: Command[]; fixtures: Record; boundaryFiles: string[]; @@ -678,6 +678,22 @@ function assertion(value: unknown, name: string): HttpAssertion { ...(v.vulnerable === undefined ? {} : { vulnerable: oracle(object(v.vulnerable), `${name}.vulnerable`) }), }; } +export const MAX_SECURITY_ASSERTIONS = 8; +export function securityAssertions(security: VerificationRequest['security']): HttpAssertion[] { + return Array.isArray(security) ? security : [security]; +} +function securityAssertion(value: unknown): VerificationRequest['security'] { + if (!Array.isArray(value)) return assertion(value, 'security'); + if (!value.length || value.length > MAX_SECURITY_ASSERTIONS) + throw new CsoError( + 'INVALID_SCHEMA', + `security must be one assertion or 1..${MAX_SECURITY_ASSERTIONS} assertions`, + ); + const assertions = value.map((x, i) => assertion(x, `security[${i}]`)); + if (new Set(assertions.map((x) => x.name)).size !== assertions.length) + throw new CsoError('INVALID_SCHEMA', 'security assertion names must be distinct'); + return assertions; +} export function validateVerificationRequest(input: unknown): VerificationRequest { const v = object(input, 'verification request'), changes = v.changes, @@ -729,7 +745,7 @@ export function validateVerificationRequest(input: unknown): VerificationRequest legitimate: (Array.isArray(v.legitimate) ? v.legitimate : []).map((x, i) => assertion(x, `legitimate[${i}]`), ), - security: assertion(v.security, 'security'), + security: securityAssertion(v.security), existingTests: (Array.isArray(v.existingTests) ? v.existingTests : []).map((x, i) => validateCommand(x, `existingTests[${i}]`), ), @@ -770,7 +786,7 @@ export function validateVerificationRequest(input: unknown): VerificationRequest throw new CsoError('INVALID_SCHEMA', 'port must be 1024..65535'); if ( !request.legitimate.length || - !request.security.vulnerable || + securityAssertions(request.security).some((x) => !x.vulnerable) || !request.existingTests.length || !request.boundaryFiles.length || !request.testFiles.length @@ -779,17 +795,15 @@ export function validateVerificationRequest(input: unknown): VerificationRequest 'INVALID_SCHEMA', 'Verification needs a legitimate control, distinct before/fixed security oracles, existing tests, immutable test files, and boundary files', ); - const secure = request.security.expected, - vulnerable = request.security.vulnerable; - const mutuallyExclusive = - secure.status !== vulnerable.status || + const mutuallyExclusive = ({ expected: secure, vulnerable }: HttpAssertion) => + secure.status !== vulnerable!.status || (secure.includes !== undefined && - vulnerable.excludes !== undefined && - secure.includes.includes(vulnerable.excludes)) || - (vulnerable.includes !== undefined && + vulnerable!.excludes !== undefined && + secure.includes.includes(vulnerable!.excludes)) || + (vulnerable!.includes !== undefined && secure.excludes !== undefined && - vulnerable.includes.includes(secure.excludes)); - if (!mutuallyExclusive) + vulnerable!.includes.includes(secure.excludes)); + if (!securityAssertions(request.security).every(mutuallyExclusive)) throw new CsoError( 'INVALID_SCHEMA', 'The vulnerable and fixed security oracles must be provably mutually exclusive', diff --git a/lib/cso/runtime-catalog.ts b/lib/cso/runtime-catalog.ts index e1859bcc9..b0ddfde07 100644 --- a/lib/cso/runtime-catalog.ts +++ b/lib/cso/runtime-catalog.ts @@ -2,6 +2,7 @@ import committedCatalog from './runtime-catalog.json'; import type { CsoStack, PreparationPlan } from './preparation'; import { CsoError, canonical, sha256 } from './contracts'; +import { ISOLATION_POLICY_HASH } from './docker'; export const CSO_HELPER_ABI = 3; export type RuntimePlatform = 'linux/amd64' | 'linux/arm64'; @@ -29,12 +30,15 @@ export type RuntimeQualification = RuntimeQualificationProvenance & multiDatabasePassed: true; readinessPassed: true; } + /** Staged and attested, with no release gate measured. Only evaluation catalogs carry it. */ + | { kind: 'evaluation' } ); export interface QualifiedRuntime { id: string; stack: CsoStack | 'postgresql'; platform: RuntimePlatform; - state: 'qualified'; + /** evaluation_candidate appears only in evaluation-only catalogs, never in a distributed helper. */ + state: 'qualified' | 'evaluation_candidate'; image: string; entrypoint: '/opt/cso/entrypoint'; helperAbi: number; @@ -43,6 +47,18 @@ export interface QualifiedRuntime { qualifiedAt: string; qualification: RuntimeQualification; } +/** + * The inputs a qualification was measured under. Changing any of them after + * promotion makes the catalog older than its inputs: requalify or withdraw. + */ +export interface RequalificationTriggers { + helperAbi: number; + /** ISOLATION_POLICY_HASH, which covers the cso-isolation-v1 policy version and every limit. */ + isolationPolicyHash: string; + /** Digest of every source byte compiled or copied into the runtime images (preparation and verifier helpers). */ + preparationSha256: string; + buildInputsRevision: string; +} /** Reviewed build metadata is informative. It never makes an image executable. */ export interface ReviewedRuntimeProfile { id: string; @@ -66,10 +82,28 @@ export interface RuntimeCatalog { evidenceDigest: string; /** Canonical digest of the complete release-gate statements retained externally. */ qualificationEvidenceDigest: string; + /** Trigger inputs measured by the staging run at sourceCommit. */ + requalification: RequalificationTriggers; + }; + /** + * Present only on an evaluation-only catalog (revision cso-eval--): + * staged digests from one protected staging run, built into a private + * evaluation helper. Setup, distribution builds and promotion refuse it. + */ + evaluation?: { + sourceCommit: string; + workflow: string; + /** Canonical digest of the staged runtime matrix. */ + evidenceDigest: string; }; runtimes: QualifiedRuntime[]; } +export const EVALUATION_REVISION_PREFIX = 'cso-eval-'; +export function isEvaluationRuntimeCatalog(catalog: Pick): boolean { + return typeof catalog?.revision === 'string' && catalog.revision.startsWith(EVALUATION_REVISION_PREFIX); +} + const DIGEST = /^sha256:[a-f0-9]{64}$/; const IMAGE = /^(?:[a-z0-9.-]+(?::[0-9]+)?\/)?[a-z0-9][a-z0-9._/-]*@sha256:[a-f0-9]{64}$/; const ID = /^[a-z0-9][a-z0-9._-]{0,100}$/; @@ -120,6 +154,20 @@ function validateRuntimeIdentity(value: { throw new Error('INCOMPATIBLE_PREPARATION_HELPER'); } +export function validRequalificationTriggers(value: unknown, buildRevision: string): boolean { + const triggers = value as RequalificationTriggers; + return ( + !!triggers && + typeof triggers === 'object' && + Object.keys(triggers).sort().join(',') === + 'buildInputsRevision,helperAbi,isolationPolicyHash,preparationSha256' && + triggers.helperAbi === CSO_HELPER_ABI && + /^[a-f0-9]{64}$/.test(triggers.isolationPolicyHash) && + /^[a-f0-9]{64}$/.test(triggers.preparationSha256) && + triggers.buildInputsRevision === buildRevision + ); +} + export function validateRuntimeCatalog(value: unknown): asserts value is RuntimeCatalog { const catalog = value as RuntimeCatalog; if ( @@ -131,6 +179,7 @@ export function validateRuntimeCatalog(value: unknown): asserts value is Runtime !Array.isArray(catalog.runtimes) ) throw new Error('INCOMPATIBLE_RUNTIME_CATALOG'); + const evaluation = isEvaluationRuntimeCatalog(catalog); if ( catalog.previousRevision !== null && (typeof catalog.previousRevision !== 'string' || !BUILD_REVISION.test(catalog.previousRevision)) @@ -170,8 +219,11 @@ export function validateRuntimeCatalog(value: unknown): asserts value is Runtime !QUALIFICATION_WORKFLOW.test(catalog.promotion.workflow) || !DIGEST.test(catalog.promotion.evidenceDigest) || !DIGEST.test(catalog.promotion.qualificationEvidenceDigest) || + !validRequalificationTriggers(catalog.promotion.requalification, catalog.buildRevision) || Object.keys(catalog.promotion).sort().join(',') !== - ['evidenceDigest', 'qualificationEvidenceDigest', 'sourceCommit', 'workflow'].sort().join(',') + ['evidenceDigest', 'qualificationEvidenceDigest', 'requalification', 'sourceCommit', 'workflow'] + .sort() + .join(',') ) { throw new Error('INVALID_RUNTIME_PROMOTION'); } @@ -194,7 +246,7 @@ export function validateRuntimeCatalog(value: unknown): asserts value is Runtime `^ghcr\\.io/garrytan/gstack/cso-staging/${runtime.stack}-${arch}@sha256:[a-f0-9]{64}$`, ); if ( - runtime.state !== 'qualified' || + runtime.state !== (evaluation ? 'evaluation_candidate' : 'qualified') || !IMAGE.test(runtime.image) || !expectedImage.test(runtime.image) || runtime.entrypoint !== '/opt/cso/entrypoint' || @@ -229,7 +281,10 @@ export function validateRuntimeCatalog(value: unknown): asserts value is Runtime 'provenanceDigest', 'verifiedProvenance', ]; - if (['node', 'bun', 'python', 'rails'].includes(runtime.stack)) { + if (evaluation) { + if (qualification.kind !== 'evaluation' || keys.join(',') !== [...common].sort().join(',')) + throw new Error('INVALID_EVALUATION_RUNTIME'); + } else if (['node', 'bun', 'python', 'rails'].includes(runtime.stack)) { if ( qualification.kind !== 'application' || qualification.containmentPassed !== true || @@ -266,6 +321,30 @@ export function validateRuntimeCatalog(value: unknown): asserts value is Runtime if (catalog.runtimes.length > 0) { for (const identity of profileIdentities) if (!runtimeIdentities.has(identity)) throw new Error('INCOMPLETE_QUALIFIED_RUNTIME_MATRIX'); + } + if (evaluation) { + const record = catalog.evaluation; + if ( + !record || + catalog.promotion !== undefined || + catalog.runtimes.length === 0 || + Object.keys(record).sort().join(',') !== 'evidenceDigest,sourceCommit,workflow' || + !/^[a-f0-9]{40}$/.test(record.sourceCommit) || + !QUALIFICATION_WORKFLOW.test(record.workflow) || + catalog.revision !== + `${EVALUATION_REVISION_PREFIX}${record.workflow.slice(record.workflow.lastIndexOf('/') + 1)}-${record.sourceCommit.slice(0, 12)}` || + catalog.runtimes.some( + (runtime) => + runtime.qualification.sourceCommit !== record.sourceCommit || + runtime.qualification.workflow !== record.workflow, + ) || + record.evidenceDigest !== `sha256:${sha256(canonical(catalog.runtimes))}` + ) + throw new Error('INVALID_EVALUATION_RUNTIME_CATALOG'); + return; + } + if (catalog.evaluation !== undefined) throw new Error('INVALID_EVALUATION_RUNTIME_CATALOG'); + if (catalog.runtimes.length > 0) { if (!catalog.promotion) throw new Error('MISSING_RUNTIME_PROMOTION'); if ( catalog.runtimes.some( @@ -326,6 +405,11 @@ export function selectRuntime( catalog: RuntimeCatalog = RUNTIME_CATALOG, ): QualifiedRuntime { validateRuntimeCatalog(catalog); + const recorded = catalog.promotion?.requalification; + if (recorded && recorded.isolationPolicyHash !== ISOLATION_POLICY_HASH) + throw new Error( + 'STALE_RUNTIME_QUALIFICATION: the isolation policy changed after these runtimes were qualified; requalify before target execution.', + ); const matches = catalog.runtimes.filter( (runtime) => runtime.platform === platform && (runtime.id === profile || runtime.stack === profile), ); diff --git a/lib/cso/verifier.ts b/lib/cso/verifier.ts index e14deb84d..f3364b764 100644 --- a/lib/cso/verifier.ts +++ b/lib/cso/verifier.ts @@ -1,13 +1,20 @@ #!/usr/bin/env bun import * as fs from 'node:fs'; import { connect } from 'node:net'; -import { HttpAssertion, VerificationObservation, object } from './contracts'; +import { + HttpAssertion, + MAX_SECURITY_ASSERTIONS, + VerificationObservation, + VerificationRequest, + object, + securityAssertions, +} from './contracts'; interface Config { phase: 'before' | 'after'; port: number; legitimate: HttpAssertion[]; - security: HttpAssertion; + security: VerificationRequest['security']; } function matches(status: number, body: string, oracle: HttpAssertion['expected']): boolean { return ( @@ -66,6 +73,34 @@ async function request(a: HttpAssertion, port: number): Promise<{ status: number clearTimeout(timer); } } +export async function judgeSecurity( + phase: Config['phase'], + security: Config['security'], + port: number, +): Promise<{ + security: VerificationObservation['security']; + members: VerificationObservation['security'][]; +}> { + const members: VerificationObservation['security'][] = []; + for (const assertion of securityAssertions(security)) { + const r = await request(assertion, port), + fixed = matches(r.status, r.body, assertion.expected), + vulnerable = matches(r.status, r.body, assertion.vulnerable!); + members.push( + phase === 'before' + ? vulnerable && !fixed + ? 'intended_failure' + : fixed && !vulnerable + ? 'pass' + : 'inconclusive' + : fixed && !vulnerable + ? 'pass' + : 'inconclusive', + ); + } + const agreed = members.every((member) => member === members[0]) ? members[0] : 'inconclusive'; + return { security: agreed, members }; +} async function ready(port: number): Promise { return await new Promise((resolve) => { const socket = connect({ host: '127.0.0.1', port }), @@ -97,7 +132,8 @@ async function main() { v.port < 1024 || v.port > 65535 || !Array.isArray(v.legitimate) || - !v.security + !v.security || + (Array.isArray(v.security) && (!v.security.length || v.security.length > MAX_SECURITY_ASSERTIONS)) ) throw new Error('invalid verifier policy'); const config = v as Config; @@ -122,20 +158,11 @@ async function main() { }), ) ).every(Boolean); - const r = await request(config.security, config.port), - fixed = matches(r.status, r.body, config.security.expected), - vulnerable = matches(r.status, r.body, config.security.vulnerable!); - security = - config.phase === 'before' - ? vulnerable && !fixed - ? 'intended_failure' - : fixed && !vulnerable - ? 'pass' - : 'inconclusive' - : fixed && !vulnerable - ? 'pass' - : 'inconclusive'; - summary = `boot=true legitimate=${legitimate} security=${security}`; + const judged = await judgeSecurity(config.phase, config.security, config.port); + security = judged.security; + summary = `boot=true legitimate=${legitimate} security=${security}${ + Array.isArray(config.security) ? ` members=${judged.members.join(',')}` : '' + }`; } catch { summary = 'bounded verifier request failed'; } diff --git a/package.json b/package.json index 5ab31e35c..2df04d323 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.24", + "version": "1.91.25", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/build-cso.sh b/scripts/build-cso.sh index bc7a692d6..e0d1084d6 100644 --- a/scripts/build-cso.sh +++ b/scripts/build-cso.sh @@ -27,6 +27,7 @@ CSO_NEW_LAUNCHER_INSTALLED=0 CSO_NEW_CORE_INSTALLED=0 CSO_NEW_WATCHDOG_INSTALLED=0 CSO_NEW_GENERATION_INSTALLED=0 +CSO_EVALUATION_UNIT="" cso_checkpoint() { [ "${GSTACK_CSO_BUILD_TESTING:-0}" = 1 ] || return 0 @@ -87,6 +88,7 @@ cso_cleanup() { if ! cso_restore_previous; then status=1; cleanup_stage=0; fi fi if [ "$cleanup_stage" -eq 1 ] && [ -n "$CSO_STAGE" ] && [ ! -f "$CSO_STAGE/.retain-recovery" ]; then rm -rf "$CSO_STAGE"; fi + if [ -n "$CSO_EVALUATION_UNIT" ]; then rm -rf "$CSO_EVALUATION_UNIT"; fi exit "$status" } trap 'cso_cleanup $?' EXIT @@ -143,6 +145,89 @@ cso_sha256() { printf '%s\n' "$digest" } +# Bun's compile-time switches stop configuration discovery in the audited +# working directory before the helper can apply its own policy. +cso_compile_bun() { + "$BUN_CMD" build --compile \ + --no-compile-autoload-dotenv \ + --no-compile-autoload-bunfig \ + --no-compile-autoload-tsconfig \ + --no-compile-autoload-package-json \ + "$1" --outfile "$2" + chmod +x "$2" +} +cso_compile_watchdog() { + "$CSO_CC" -std=c11 -D_POSIX_C_SOURCE=200809L -O2 -Wall -Wextra \ + lib/cso/watchdog.c -o "$1" + chmod +x "$1" +} +cso_compile_posix_launcher() { + CSO_LAUNCHER_FLAGS="" + case "$(uname -s)" in + Linux) CSO_LAUNCHER_FLAGS="-static" ;; + # Local ad-hoc signatures do not distinguish the launcher from another + # ad-hoc dylib. This section makes dyld prune DYLD_* before constructors. + Darwin) CSO_LAUNCHER_FLAGS="-Wl,-sectcreate,__RESTRICT,__restrict,/dev/null" ;; + esac + # shellcheck disable=SC2086 -- the optional platform flags are fixed literals. + "$CSO_CC" -std=c11 -D_POSIX_C_SOURCE=200809L -O2 -Wall -Wextra $CSO_LAUNCHER_FLAGS \ + "-DGSTACK_CSO_CORE_SHA256=\"$2\"" lib/cso/launcher.c -o "$1" +} + +# An evaluation candidate embeds a cso-eval-* runtime catalog in a private +# five-artifact producer unit outside this checkout. It never touches bin/, and +# its .gstack-cso-evaluation manifest marks it evaluation-only for setup. +cso_build_evaluation_candidate() { + [ "$#" -eq 4 ] && [ "$1" = --evaluation-candidate ] && [ "$3" = --output ] || { + echo 'Usage: build-cso.sh --evaluation-candidate --output ' >&2;return 64; } + [ -z "$CSO_EXE" ] || { echo 'CSO evaluation candidates are built only on Linux and macOS.' >&2;return 69; } + command -v "$CSO_CC" >/dev/null 2>&1 || { echo 'CSO build requires a C compiler (cc/clang/gcc) for the trusted launcher and watchdog.' >&2;return 1; } + candidate_catalog=$2 + [ -f "$candidate_catalog" ] && [ ! -L "$candidate_catalog" ] || { echo 'CSO evaluation catalog must be one regular file.' >&2;return 66; } + candidate_parent="$(cd "$(dirname "$4")" 2>/dev/null && pwd -P)" || { echo 'CSO evaluation output parent must be an existing directory.' >&2;return 73; } + candidate_output="$candidate_parent/$(basename "$4")" + if cso_present "$candidate_output";then echo 'CSO evaluation output must be a new directory.' >&2;return 73;fi + case "$candidate_output/" in "$CSO_BUILD_ROOT"/*) echo 'CSO evaluation output must be outside the gstack checkout, where setup and distribution never look.' >&2;return 73;;esac + candidate_revision="$(grep -o '"revision": *"cso-eval-[0-9]\{1,\}-[a-f0-9]\{12\}"' "$candidate_catalog" | head -n 1 | sed 's/.*"\(cso-eval-[^"]*\)"$/\1/' || true)" + [ -n "$candidate_revision" ] || { echo 'CSO evaluation candidates require a cso-eval-- catalog revision.' >&2;return 65; } + + CSO_STAGE="$(mktemp -d "${TMPDIR:-/tmp}/gstack-cso-evaluation.XXXXXX")" + CSO_EVALUATION_UNIT="$(mktemp -d "$candidate_parent/.gstack-cso-evaluation.XXXXXX")" + source_root="$CSO_STAGE/source" + mkdir -m 700 "$source_root" "$source_root/test" + cp -R lib scripts "$source_root/" + cp -R test/helpers "$source_root/test/" + cp "$candidate_catalog" "$source_root/lib/cso/runtime-catalog.json" + (cd "$source_root" && "$BUN_CMD" scripts/cso-runtime-promotion.ts check-evaluation lib/cso/runtime-catalog.json) + (cd "$source_root" && cso_compile_bun lib/cso/cli.ts "$CSO_EVALUATION_UNIT/gstack-cso-core") + (cd "$source_root" && cso_compile_bun scripts/cso-eval-producer.ts "$CSO_EVALUATION_UNIT/cso-eval-producer") + cso_compile_watchdog "$CSO_EVALUATION_UNIT/gstack-cso-watchdog" + if [ "$(uname -s)" = Darwin ]; then + cso_sign_macos_artifact "$CSO_EVALUATION_UNIT/gstack-cso-core" 0 + cso_sign_macos_artifact "$CSO_EVALUATION_UNIT/gstack-cso-watchdog" 0 + fi + candidate_core_sha256="$(cso_sha256 "$CSO_EVALUATION_UNIT/gstack-cso-core")" + cso_compile_posix_launcher "$CSO_EVALUATION_UNIT/gstack-cso-launcher" "$candidate_core_sha256" + chmod +x "$CSO_EVALUATION_UNIT/gstack-cso-launcher" + if [ "$(uname -s)" = Darwin ];then cso_sign_macos_artifact "$CSO_EVALUATION_UNIT/gstack-cso-launcher" 1;fi + printf '%s\n' "$candidate_core_sha256" > "$CSO_EVALUATION_UNIT/.gstack-cso-generation" + printf '{"schemaVersion":1,"evaluationOnly":true,"runtimeCatalogRevision":"%s","runtimeCatalogSha256":"%s","coreSha256":"%s"}\n' \ + "$candidate_revision" "$(cso_sha256 "$source_root/lib/cso/runtime-catalog.json")" "$candidate_core_sha256" \ + > "$CSO_EVALUATION_UNIT/.gstack-cso-evaluation" + for artifact in gstack-cso-core gstack-cso-launcher gstack-cso-watchdog cso-eval-producer;do + cso_valid_artifact "$CSO_EVALUATION_UNIT/$artifact" || { echo "Evaluation $artifact is not one executable regular file." >&2;return 1; } + done + cso_valid_generation "$CSO_EVALUATION_UNIT/.gstack-cso-generation" || { echo 'Evaluation generation manifest is invalid.' >&2;return 1; } + [ "$(cso_sha256 "$CSO_EVALUATION_UNIT/gstack-cso-core")" = "$candidate_core_sha256" ] || { echo 'Evaluation core changed after launcher binding.' >&2;return 1; } + "$CSO_EVALUATION_UNIT/gstack-cso-launcher" --version >/dev/null + chmod 0555 "$CSO_EVALUATION_UNIT"/gstack-cso-core "$CSO_EVALUATION_UNIT"/gstack-cso-launcher "$CSO_EVALUATION_UNIT"/gstack-cso-watchdog "$CSO_EVALUATION_UNIT"/cso-eval-producer + chmod 0444 "$CSO_EVALUATION_UNIT/.gstack-cso-generation" "$CSO_EVALUATION_UNIT/.gstack-cso-evaluation" + chmod 0755 "$CSO_EVALUATION_UNIT" + mv "$CSO_EVALUATION_UNIT" "$candidate_output" + CSO_EVALUATION_UNIT="" + printf '{"output":"%s","revision":"%s","evaluationOnly":true}\n' "$candidate_output" "$candidate_revision" +} + cso_publish_locked() { [ "${GSTACK_CSO_PUBLISH_LOCKED:-}" = 1 ] || { echo 'CSO publication requires the native publisher lock.' >&2;return 73; } [ "$#" -eq 1 ] && [ -d "$1" ] && [ ! -L "$1" ] || { echo 'CSO publication stage is invalid.' >&2;return 69; } @@ -201,6 +286,16 @@ if [ "${1:-}" = __publish_locked ];then cso_publish_locked "$@" exit 0 fi +if [ "${1:-}" = --evaluation-candidate ];then + cso_build_evaluation_candidate "$@" + exit 0 +fi +[ "$#" -eq 0 ] || { echo 'Usage: build-cso.sh [--evaluation-candidate --output ]' >&2;exit 64; } +# Distribution builds embed only the committed catalog, never an evaluation one. +if [ -f lib/cso/runtime-catalog.json ] && grep -q '"revision": *"cso-eval-' lib/cso/runtime-catalog.json;then + echo 'lib/cso/runtime-catalog.json is an evaluation-only catalog; distribution builds refuse it. Restore the committed catalog.' >&2 + exit 65 +fi # This entrypoint can be run directly. Once it may publish a CSO generation, # the previous whole-build completion proof no longer describes all outputs. @@ -226,21 +321,13 @@ if [ -n "$CSO_EXE" ];then : > "$CSO_STAGE/.gstack-cso-generation.lock";fi # These are compile-time switches. Bun otherwise discovers configuration in # the audited working directory before the helper can apply its own policy. -"$BUN_CMD" build --compile \ - --no-compile-autoload-dotenv \ - --no-compile-autoload-bunfig \ - --no-compile-autoload-tsconfig \ - --no-compile-autoload-package-json \ - lib/cso/cli.ts --outfile "$CSO_STAGE_CORE" -chmod +x "$CSO_STAGE_CORE" +cso_compile_bun lib/cso/cli.ts "$CSO_STAGE_CORE" cso_checkpoint core # POSIX process groups are required by the detached watchdog. Windows keeps # comprehensive execution unavailable instead of substituting a weaker timer. if [ -z "$CSO_EXE" ]; then - "$CSO_CC" -std=c11 -D_POSIX_C_SOURCE=200809L -O2 -Wall -Wextra \ - lib/cso/watchdog.c -o "$CSO_STAGE_WATCHDOG" - chmod +x "$CSO_STAGE_WATCHDOG" + cso_compile_watchdog "$CSO_STAGE_WATCHDOG" cso_checkpoint watchdog fi @@ -265,16 +352,7 @@ if [ -n "$CSO_EXE" ];then -CoreSha256 "$CSO_CORE_SHA256" \ -GitExePath "$(cygpath -aw "$CSO_WINDOWS_GIT")" else - CSO_LAUNCHER_FLAGS="" - case "$(uname -s)" in - Linux) CSO_LAUNCHER_FLAGS="-static" ;; - # Local ad-hoc signatures do not distinguish the launcher from another - # ad-hoc dylib. This section makes dyld prune DYLD_* before constructors. - Darwin) CSO_LAUNCHER_FLAGS="-Wl,-sectcreate,__RESTRICT,__restrict,/dev/null" ;; - esac - # shellcheck disable=SC2086 -- the optional platform flags are fixed literals. - "$CSO_CC" -std=c11 -D_POSIX_C_SOURCE=200809L -O2 -Wall -Wextra $CSO_LAUNCHER_FLAGS \ - "-DGSTACK_CSO_CORE_SHA256=\"$CSO_CORE_SHA256\"" lib/cso/launcher.c -o "$CSO_STAGE_LAUNCHER" + cso_compile_posix_launcher "$CSO_STAGE_LAUNCHER" "$CSO_CORE_SHA256" "$CSO_CC" -std=c11 -D_POSIX_C_SOURCE=200809L -O2 -Wall -Wextra lib/cso/publish-lock.c -o "$CSO_STAGE_LOCKER" fi chmod +x "$CSO_STAGE_LAUNCHER" "$CSO_STAGE_LOCKER" diff --git a/scripts/cso-eval-producer.ts b/scripts/cso-eval-producer.ts index 0162f8a35..32ea5592e 100644 --- a/scripts/cso-eval-producer.ts +++ b/scripts/cso-eval-producer.ts @@ -17,13 +17,15 @@ import { GeminiAdapter, prepareGeminiProducerState, removeGeminiProducerState } import { PRICING } from '../test/helpers/pricing'; import type { ProviderAdapter, RunOpts, RunResult } from '../test/helpers/providers/types'; import { + PRODUCER_PLATFORMS, + producerHostPlatform, producerInputHash, producerArtifactInventoryHash, producerInstallationIdentityHash, producerProviderIdentityHash, producerReceiptHash, sha256, - type ProducerCell, + type ProducerExecution, type ProducerHost, type ProducerInput, type ProducerArtifactInventory, @@ -197,8 +199,18 @@ function repositoryIdentity(root: string): string { return sha256(JSON.stringify({ config: sha256(config), head: read(['rev-parse', '--verify', 'HEAD']).trim(), branch: read(['symbolic-ref', '--short', 'HEAD']).trim(), status: read(['status', '--porcelain=v2', '--untracked-files=all']) })); } -function validateCell(cell: ProducerCell): void { - if (!cell || !HEX.test(cell.id) || !cell.caseId || !['node', 'bun', 'python', 'rails'].includes(cell.stack) || !['vulnerable', 'fixed'].includes(cell.variant) || !['v2', 'v3'].includes(cell.version) || !['daily', 'comprehensive'].includes(cell.mode) || ![1, 2, 3].includes(cell.repetition) || !cell.model || !['claude', 'codex', 'gemini'].includes(cell.host) || !Number.isInteger(cell.budgetSeconds) || cell.budgetSeconds <= 60 || cell.budgetSeconds > 3600 || !HEX.test(cell.sourceHash) || !HEX.test(cell.skillHash)) throw new Error('INVALID_PRODUCER_INPUT'); +const INPUT_FIELDS = 'cellRef,execution,schemaVersion,skill,source'; +const EXECUTION_FIELDS = 'budgetSeconds,host,mode,model,skillHash,sourceHash'; +/** The input has exactly the opaque fields: no case, variant, version, or repetition can ride along. */ +function validateInput(input: ProducerInput): void { + const execution = input?.execution; + if (!input || typeof input !== 'object' || Object.keys(input).sort().join(',') !== INPUT_FIELDS || input.schemaVersion !== 2 || !HEX.test(input.cellRef) || + typeof input.skill !== 'string' || Buffer.byteLength(input.skill) > INPUT_LIMIT || !Array.isArray(input.source) || + !execution || typeof execution !== 'object' || Object.keys(execution).filter(key => key !== 'platform').sort().join(',') !== EXECUTION_FIELDS || + !['daily', 'comprehensive'].includes(execution.mode) || typeof execution.model !== 'string' || !execution.model || !['claude', 'codex', 'gemini'].includes(execution.host) || + !Number.isInteger(execution.budgetSeconds) || execution.budgetSeconds <= 60 || execution.budgetSeconds > 3600 || !HEX.test(execution.sourceHash) || !HEX.test(execution.skillHash) || + ('platform' in execution && !PRODUCER_PLATFORMS.includes(execution.platform!))) throw new Error('INVALID_PRODUCER_INPUT'); + if (execution.platform !== undefined && execution.platform !== producerHostPlatform()) throw new Error('UNMATCHED_PRODUCER_PLATFORM'); } export interface ProducerHelperBinding { @@ -417,7 +429,7 @@ export function resolveProducerProviderIdentity(host: ProducerHost, stateRoot: s }; } -function promptFor(cell: ProducerCell, skill: string, sourceRoot: string, helper: ProducerHelperBinding): string { +function promptFor(cell: ProducerExecution, skill: string, sourceRoot: string, helper: ProducerHelperBinding): string { const invocation = `/cso${cell.mode === 'comprehensive' ? ' --comprehensive' : ''} --budget ${cell.budgetSeconds}`; const sourceAccess = cell.host === 'codex' ? 'The Codex permission profile grants read-only access to exactly that immutable snapshot so the trusted helper can read it. Use only the trusted helper to inspect or act on source; direct provider file or shell access is outside this evaluation policy.' @@ -484,20 +496,20 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op const bytes = readBoundedStable(controlPath, INPUT_LIMIT, 'Producer input'); let input: ProducerInput; try { input = JSON.parse(bytes.toString('utf8')); } catch { throw new Error('INVALID_PRODUCER_INPUT'); } - if (input.schemaVersion !== 1 || typeof input.skill !== 'string' || Buffer.byteLength(input.skill) > INPUT_LIMIT || !Array.isArray(input.source)) throw new Error('INVALID_PRODUCER_INPUT'); - validateCell(input.cell); - if (basename(receipt) !== `${input.cell.id}.json`) throw new Error('UNMATCHED_RECEIPT_DESTINATION'); - if (sha256(input.skill) !== input.cell.skillHash) throw new Error('INVALID_PRODUCER_SKILL'); - validateSource(sourceRoot, input.source, input.cell.sourceHash); + validateInput(input); + const cell = input.execution; + if (basename(receipt) !== `${input.cellRef}.json`) throw new Error('UNMATCHED_RECEIPT_DESTINATION'); + if (sha256(input.skill) !== cell.skillHash) throw new Error('INVALID_PRODUCER_SKILL'); + validateSource(sourceRoot, input.source, cell.sourceHash); sealProducerSource(sourceRoot); - validateSource(sourceRoot, input.source, input.cell.sourceHash); + validateSource(sourceRoot, input.source, cell.sourceHash); const originalRepositoryIdentity = repositoryIdentity(sourceRoot); const stateRoot = join(jobRoot, 'state'); const helperHome = join(stateRoot, 'cso-home'); const helper = resolveProducerHelperBinding(sourceRoot, stateRoot, options.testHelperLauncherPath); const installationIdentity = producerInstallationIdentity(helper); - const adapter = options.adapter ?? adapterFor(input.cell.host); - if ((input.cell.host === 'codex' ? 'gpt' : input.cell.host) !== adapter.family) throw new Error('UNMATCHED_PRODUCER_ADAPTER'); + const adapter = options.adapter ?? adapterFor(cell.host); + if ((cell.host === 'codex' ? 'gpt' : cell.host) !== adapter.family) throw new Error('UNMATCHED_PRODUCER_ADAPTER'); fs.mkdirSync(stateRoot, { recursive: false, mode: 0o700 }); fs.mkdirSync(helperHome, { recursive: false, mode: 0o700 }); const provider = options.testProviderIdentity @@ -505,13 +517,13 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op identity: validateProviderIdentity(options.testProviderIdentity, adapter.family), command: options.testProviderCommand ?? { executable: process.execPath, argsPrefix: [] }, } - : resolveProducerProviderIdentity(input.cell.host, stateRoot); + : resolveProducerProviderIdentity(cell.host, stateRoot); const providerIdentity = provider.identity; const runOptions = { - prompt: promptFor(input.cell, input.skill, sourceRoot, helper), + prompt: promptFor(cell, input.skill, sourceRoot, helper), workdir: stateRoot, - timeoutMs: input.cell.budgetSeconds * 1000, - model: input.cell.model, + timeoutMs: cell.budgetSeconds * 1000, + model: cell.model, csoProducer: { stateDirectory: stateRoot, sourceDirectory: sourceRoot, @@ -521,7 +533,7 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op }, } satisfies RunOpts; const availability = await adapter.available(runOptions); - if (!availability.ok) throw new Error(`PRODUCER_UNAVAILABLE: ${availability.reason ?? input.cell.host}`); + if (!availability.ok) throw new Error(`PRODUCER_UNAVAILABLE: ${availability.reason ?? cell.host}`); const inputHash = producerInputHash(input); // Load the opaque metadata into memory, then remove it before starting the @@ -536,11 +548,11 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op const startedAt = new Date().toISOString(); let run: RunResult; try { - if (input.cell.host === 'gemini') prepareGeminiProducerState(stateRoot, helper.launcher); + if (cell.host === 'gemini') prepareGeminiProducerState(stateRoot, helper.launcher); run = await adapter.run(runOptions); } finally { try { - if (input.cell.host === 'gemini') removeGeminiProducerState(stateRoot); + if (cell.host === 'gemini') removeGeminiProducerState(stateRoot); } finally { if (previousHome === undefined) delete process.env.GSTACK_HOME; else process.env.GSTACK_HOME = previousHome; if (previousSessionKind === undefined) delete process.env.GSTACK_SESSION_KIND; else process.env.GSTACK_SESSION_KIND = previousSessionKind; @@ -557,7 +569,7 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op throw new Error('PRODUCER_PROVIDER_INSTALLATION_RACE'); } assertProducerSourceSealed(sourceRoot); - validateSource(sourceRoot, input.source, input.cell.sourceHash); + validateSource(sourceRoot, input.source, cell.sourceHash); if (repositoryIdentity(sourceRoot) !== originalRepositoryIdentity) throw new Error('PRODUCER_CHANGED_SOURCE'); const artifacts = inventoryProducerArtifacts(helperHome); run = sanitizeProducerRun(run); @@ -566,8 +578,8 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op const tokensReported = run.tokens.input > 0 || run.tokens.output > 0 || (run.tokens.cached ?? 0) > 0; const estimatedCostUSD = tokensReported && PRICING[run.modelUsed] ? adapter.estimateCost(run.tokens, run.modelUsed) : null; const withoutHash: Omit = { - schemaVersion: 1, - cell: input.cell, + schemaVersion: 2, + cellRef: input.cellRef, inputHash, installationIdentity, providerIdentity, @@ -575,9 +587,9 @@ export async function runProducerCell(inputPath: string, receiptPath: string, op startedAt, finishedAt, status: run.error ? 'failed' : 'succeeded', - requestedModel: input.cell.model, + requestedModel: cell.model, modelUsed: run.modelUsed, - modelIdentitySource: run.modelUsed === input.cell.model ? 'requested_pin' : 'provider_reported', + modelIdentitySource: run.modelUsed === cell.model ? 'requested_pin' : 'provider_reported', durationMs: run.durationMs, firstUsefulResultMs: null, toolCalls: run.toolCalls, @@ -600,7 +612,7 @@ async function cli(args: string[]): Promise { if (args.length !== 4 || args[0] !== 'run' || args[3] !== '--execute-paid') throw new Error('Usage: cso-eval-producer run --execute-paid'); if (process.env.CSO_EVAL_PAID !== '1') throw new Error('PAID_EXECUTION_NOT_AUTHORIZED: also set CSO_EVAL_PAID=1 on the isolated producer host'); const receipt = await runProducerCell(args[1], args[2], { paidExecutionAuthorized: true }); - console.log(JSON.stringify({ cellId: receipt.cell.id, status: receipt.status, durationMs: receipt.durationMs, receiptHash: receipt.receiptHash })); + console.log(JSON.stringify({ cellRef: receipt.cellRef, status: receipt.status, durationMs: receipt.durationMs, receiptHash: receipt.receiptHash })); } if (import.meta.main) { diff --git a/scripts/cso-eval-protocol.ts b/scripts/cso-eval-protocol.ts index d6b71cbc3..88174ecff 100644 --- a/scripts/cso-eval-protocol.ts +++ b/scripts/cso-eval-protocol.ts @@ -5,6 +5,8 @@ export type ProducerVersion = 'v2' | 'v3'; export type ProducerMode = 'daily' | 'comprehensive'; export type ProducerStack = 'node' | 'bun' | 'python' | 'rails'; export type ProducerVariant = 'vulnerable' | 'fixed'; +export type ProducerPlatform = 'linux/amd64' | 'linux/arm64'; +export const PRODUCER_PLATFORMS: readonly ProducerPlatform[] = ['linux/amd64', 'linux/arm64']; export interface ProducerCell { id: string; @@ -19,6 +21,8 @@ export interface ProducerCell { budgetSeconds: number; sourceHash: string; skillHash: string; + /** Release and baseline cells pin the native producer platform; full-profile cells omit it. */ + platform?: ProducerPlatform; } export interface ProducerSourceEntry { @@ -27,16 +31,40 @@ export interface ProducerSourceEntry { bytes: number; } +/** + * Only what a producer needs to run one cell. It never names the corpus case, + * variant, version, or repetition: an agent that can read its own control file + * or receipt still cannot tell which answer it is being graded against. + */ +export interface ProducerExecution { + mode: ProducerMode; + model: string; + host: ProducerHost; + budgetSeconds: number; + sourceHash: string; + skillHash: string; + platform?: ProducerPlatform; +} + /** * This control file is consumed before a producing agent starts. It must never * be placed inside the application repository or retained on the producer. */ export interface ProducerInput { - schemaVersion: 1; - cell: ProducerCell; + schemaVersion: 2; + /** The opaque matrix cell id; only the trusted evaluator maps it back to a cell. */ + cellRef: string; /** Exact canonical portable payload: root SKILL.md plus its manifest-listed sections. */ skill: string; source: ProducerSourceEntry[]; + execution: ProducerExecution; +} + +export function producerExecution(cell: ProducerCell): ProducerExecution { + return { + mode: cell.mode, model: cell.model, host: cell.host, budgetSeconds: cell.budgetSeconds, + sourceHash: cell.sourceHash, skillHash: cell.skillHash, ...(cell.platform === undefined ? {} : { platform: cell.platform }), + }; } export interface ProducerArtifactIdentity { @@ -85,8 +113,9 @@ export interface ProducerArtifactInventory { } export interface ProducerReceipt { - schemaVersion: 1; - cell: ProducerCell; + schemaVersion: 2; + /** The opaque cell id from the consumed input; bound together with inputHash. */ + cellRef: string; inputHash: string; installationIdentity: ProducerInstallationIdentity; providerIdentity: ProducerProviderIdentity; @@ -115,13 +144,23 @@ export interface ProducerReceipt { receiptHash: string; } -/** Compact trusted index; raw output remains in the separately retained receipt. */ +/** + * Compact trusted index; raw output remains in the separately retained receipt. + * Collection adds the matrix cell that the receipt's cellRef names. + */ export type ProducerReceiptIndex = Omit & { + cell: ProducerCell; error?: { code: string }; }; export const sha256 = (value: string | Buffer): string => createHash('sha256').update(value).digest('hex'); +/** The native platform this producer process runs on, or null when no CSO runtime platform matches it. */ +export function producerHostPlatform(platform: string = process.platform, arch: string = process.arch): ProducerPlatform | null { + if (platform !== 'linux') return null; + return arch === 'x64' ? 'linux/amd64' : arch === 'arm64' ? 'linux/arm64' : null; +} + export function producerInputHash(input: ProducerInput): string { return sha256(JSON.stringify(input)); } diff --git a/scripts/cso-eval.ts b/scripts/cso-eval.ts index 122a850dc..69e475a61 100644 --- a/scripts/cso-eval.ts +++ b/scripts/cso-eval.ts @@ -7,9 +7,11 @@ import { dirname, join, resolve } from 'node:path'; import { readBoundedStable } from '../lib/cso/bounded-file'; import { executable } from '../lib/cso/process'; import { atomicWriteSync } from '../lib/fs-atomic'; -import { loadCorpusManifest, materializeCase, sourceFiles, STACKS, type CorpusManifest, type EvalStack, type EvalVariant } from '../test/fixtures/cso-eval/materialize'; +import { loadCorpusManifest, materializeCase, publicCorpus, STACKS, validateCorpusManifest, type CorpusManifest, type EvalCorpus, type EvalStack, type EvalVariant } from '../test/fixtures/cso-eval/materialize'; import { + PRODUCER_PLATFORMS, producerArtifactInventoryHash, + producerExecution, producerInputHash, producerInstallationIdentityHash, producerProviderIdentityHash, @@ -20,6 +22,7 @@ import { type ProducerHost, type ProducerInput, type ProducerInstallationIdentity, + type ProducerPlatform, type ProducerProviderIdentity, type ProducerReceipt, type ProducerReceiptIndex, @@ -29,13 +32,53 @@ import { export type EvalVersion = 'v2' | 'v3'; export type EvalMode = 'daily' | 'comprehensive'; export type Outcome = 'passed' | 'failed' | 'blocked' | 'not_attempted'; +/** + * full: matched v2/v3 × 3 repetitions on one producer installation. + * release: v3 only, 1 repetition, each cell pinned to a native platform. + * baseline: v2 comprehensive only, 1 repetition; scored into a reusable EvalBaseline. + */ +export type EvalProfile = 'full' | 'release' | 'baseline'; +export type EvalPlatform = ProducerPlatform; +type EvalCorpusManifest = CorpusManifest; export interface EvalCell extends ProducerCell { id: string; caseId: string; stack: EvalStack; variant: EvalVariant; version: EvalVersion; mode: EvalMode; - repetition: 1 | 2 | 3; model: string; host: ProducerHost; budgetSeconds: number; sourceHash: string; skillHash: string; + repetition: 1 | 2 | 3; model: string; host: ProducerHost; budgetSeconds: number; sourceHash: string; skillHash: string; platform?: EvalPlatform; } -export interface EvalMatrix { +interface EvalMatrixBase { schemaVersion: 1; corpusVersion: string; corpusHash: string; model: string; host: ProducerHost; - budgets: { daily: number; comprehensive: number }; skillHashes: { v2: string; v3: string }; repetitions: 3; cells: EvalCell[]; + budgets: { daily: number; comprehensive: number }; cells: EvalCell[]; +} +export interface FullEvalMatrix extends EvalMatrixBase { profile?: undefined; skillHashes: { v2: string; v3: string }; repetitions: 3 } +export interface ReleaseEvalMatrix extends EvalMatrixBase { + profile: 'release'; skillHashes: { v3: string }; repetitions: 1; + /** Daily cells run on one platform; each comprehensive case is fixed to one platform. */ + platforms: { daily: EvalPlatform; comprehensive: Record }; + /** Pre-registered external v2 baseline, or null when the regression gate stays unmeasured. */ + baselineHash: string | null; +} +export interface BaselineEvalMatrix extends EvalMatrixBase { + profile: 'baseline'; skillHashes: { v2: string }; repetitions: 1; + platforms: { comprehensive: Record }; +} +export type EvalMatrix = FullEvalMatrix | ReleaseEvalMatrix | BaselineEvalMatrix; +interface EvalMatrixCommonOptions { model: string; host: ProducerHost; budgets?: { daily: number; comprehensive: number } } +export interface FullEvalMatrixOptions extends EvalMatrixCommonOptions { profile?: 'full'; skillHashes: { v2: string; v3: string } } +export interface ReleaseEvalMatrixOptions extends EvalMatrixCommonOptions { profile: 'release'; skillHashes: { v3: string }; platforms: ReleaseEvalMatrix['platforms']; baseline?: EvalBaseline } +export interface BaselineEvalMatrixOptions extends EvalMatrixCommonOptions { profile: 'baseline'; skillHashes: { v2: string }; platforms: BaselineEvalMatrix['platforms'] } +export type EvalMatrixOptions = FullEvalMatrixOptions | ReleaseEvalMatrixOptions | BaselineEvalMatrixOptions; +/** + * A scored v2 comprehensive run, reused by release matrices while every bound + * input is unchanged. Release matrices pin its hash before results exist. + */ +export interface EvalBaseline { + schemaVersion: 1; version: 'v2'; + corpusVersion: string; corpusHash: string; model: string; host: ProducerHost; budgetSeconds: number; skillHash: string; + platforms: Record; + matrixHash: string; producerBatchHash: string; modelUsed: string; + provider: { family: ProducerProviderIdentity['family']; policyRevision: string; version: string }; + /** One entry per corpus case, in manifest order: whether the vulnerable cell found the seeded issue. */ + cases: Array<{ caseId: string; found: boolean }>; + baselineHash: string; } export interface PortableSkillFile { path: string; @@ -280,8 +323,8 @@ function initializeFixtureRepository(path: string): void { execFileSync(git, [...safe, 'commit', '--quiet', '-m', 'immutable evaluation fixture'], { cwd: path, env, stdio: 'ignore' }); } -function sourceEntries(caseId: string, variant: EvalVariant): ProducerSourceEntry[] { - return Object.entries(sourceFiles(caseId, variant)).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0).map(([path, contents]) => ({ path, sha256: sha256(contents), bytes: Buffer.byteLength(contents) })); +function sourceEntries(corpus: EvalCorpus, caseId: string, variant: EvalVariant): ProducerSourceEntry[] { + return Object.entries(corpus.sourceFiles(caseId, variant)).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0).map(([path, contents]) => ({ path, sha256: sha256(contents), bytes: Buffer.byteLength(contents) })); } /** @@ -289,10 +332,15 @@ function sourceEntries(caseId: string, variant: EvalVariant): ProducerSourceEntr * evaluator metadata; transfer one job at a time and never give the schedule * or sibling jobs to the producing agent. */ -export function prepareEvalJobs(matrix: EvalMatrix, payloads: { v2: string; v3: string }, destination: string, selectedCellIds?: string[]): PreparedEvalSchedule { - validateMatrix(matrix); - validatePortableSkillPayload(payloads.v2, 'v2'); validatePortableSkillPayload(payloads.v3, 'v3'); - if (sha256(payloads.v2) !== matrix.skillHashes.v2 || sha256(payloads.v3) !== matrix.skillHashes.v3) throw new Error('EVAL_SKILL_HASH_MISMATCH'); +export function prepareEvalJobs(matrix: EvalMatrix, payloads: Partial>, destination: string, selectedCellIds?: string[], corpus: EvalCorpus = publicCorpus()): PreparedEvalSchedule { + validateMatrix(matrix, corpus.manifest); + const skillHashes: Partial> = matrix.skillHashes; + const versions = (['v2', 'v3'] as const).filter(version => skillHashes[version] !== undefined); + if (!payloads || Object.keys(payloads).sort().join(',') !== versions.join(',')) throw new Error('EVAL_SKILL_HASH_MISMATCH'); + for (const version of versions) { + validatePortableSkillPayload(payloads[version]!, version); + if (sha256(payloads[version]!) !== skillHashes[version]) throw new Error('EVAL_SKILL_HASH_MISMATCH'); + } const root = resolve(destination); if (existsSync(root)) throw new Error('EVAL_DESTINATION_EXISTS'); const parent = dirname(root); @@ -308,10 +356,10 @@ export function prepareEvalJobs(matrix: EvalMatrix, payloads: { v2: string; v3: const jobs: PreparedEvalSchedule['jobs'] = []; for (const cell of selected) { const jobRoot = join(jobsRoot, cell.id); mkdirSync(jobRoot, { mode: 0o700 }); - const source = materializeCase(cell.caseId, cell.variant, join(jobRoot, 'source')); + const source = materializeCase(cell.caseId, cell.variant, join(jobRoot, 'source'), corpus); if (source.sourceHash !== cell.sourceHash) throw new Error('CORPUS_INTEGRITY_MISMATCH'); initializeFixtureRepository(source.path); - const input: ProducerInput = { schemaVersion: 1, cell, skill: payloads[cell.version], source: sourceEntries(cell.caseId, cell.variant) }; + const input: ProducerInput = { schemaVersion: 2, cellRef: cell.id, skill: payloads[cell.version]!, source: sourceEntries(corpus, cell.caseId, cell.variant), execution: producerExecution(cell) }; const inputHash = producerInputHash(input); safeWriteNew(join(jobRoot, 'producer-input.json'), input); jobs.push({ cellId: cell.id, relativePath: `jobs/${cell.id}`, inputHash }); @@ -332,7 +380,7 @@ function validateSchedule(matrix: EvalMatrix, schedule: PreparedEvalSchedule): v } function validateReceipt(receipt: ProducerReceipt, cell: EvalCell, inputHash: string): void { - if (receipt?.schemaVersion !== 1 || JSON.stringify(receipt.cell) !== JSON.stringify(cell) || receipt.inputHash !== inputHash || receipt.requestedModel !== cell.model || !['succeeded', 'failed'].includes(receipt.status) || typeof receipt.modelUsed !== 'string' || !receipt.modelUsed || !['provider_reported', 'requested_pin'].includes(receipt.modelIdentitySource) || (receipt.modelIdentitySource === 'requested_pin' && receipt.modelUsed !== receipt.requestedModel) || (receipt.modelIdentitySource === 'provider_reported' && receipt.modelUsed === receipt.requestedModel) || !validNumber(receipt.durationMs) || receipt.firstUsefulResultMs !== null || !Number.isSafeInteger(receipt.toolCalls) || receipt.toolCalls < 0 || typeof receipt.output !== 'string' || receipt.outputHash !== sha256(receipt.output) || !HEX.test(receipt.receiptHash)) throw new Error('INVALID_PRODUCER_RECEIPT'); + if (receipt?.schemaVersion !== 2 || receipt.cellRef !== cell.id || 'cell' in receipt || receipt.inputHash !== inputHash || receipt.requestedModel !== cell.model || !['succeeded', 'failed'].includes(receipt.status) || typeof receipt.modelUsed !== 'string' || !receipt.modelUsed || !['provider_reported', 'requested_pin'].includes(receipt.modelIdentitySource) || (receipt.modelIdentitySource === 'requested_pin' && receipt.modelUsed !== receipt.requestedModel) || (receipt.modelIdentitySource === 'provider_reported' && receipt.modelUsed === receipt.requestedModel) || !validNumber(receipt.durationMs) || receipt.firstUsefulResultMs !== null || !Number.isSafeInteger(receipt.toolCalls) || receipt.toolCalls < 0 || typeof receipt.output !== 'string' || receipt.outputHash !== sha256(receipt.output) || !HEX.test(receipt.receiptHash)) throw new Error('INVALID_PRODUCER_RECEIPT'); const { receiptHash, ...withoutHash } = receipt; if (producerReceiptHash(withoutHash) !== receiptHash) throw new Error('INVALID_PRODUCER_RECEIPT'); const started = Date.parse(receipt.startedAt), finished = Date.parse(receipt.finishedAt); @@ -377,7 +425,7 @@ function validateArtifactInventory(inventory: ProducerArtifactInventory, error: } function validateReceiptIndex(receipt: ProducerReceiptIndex, cell: EvalCell): void { - if (!receipt || receipt.schemaVersion !== 1 || JSON.stringify(receipt.cell) !== JSON.stringify(cell) || !HEX.test(receipt.inputHash) || receipt.requestedModel !== cell.model || !receipt.modelUsed || !['provider_reported', 'requested_pin'].includes(receipt.modelIdentitySource) || (receipt.modelIdentitySource === 'requested_pin' && receipt.modelUsed !== receipt.requestedModel) || (receipt.modelIdentitySource === 'provider_reported' && receipt.modelUsed === receipt.requestedModel) || !['succeeded', 'failed'].includes(receipt.status) || !HEX.test(receipt.outputHash) || !HEX.test(receipt.receiptHash) || !validNumber(receipt.durationMs) || receipt.firstUsefulResultMs !== null || !Number.isSafeInteger(receipt.toolCalls) || receipt.toolCalls < 0 || !receipt.usage) throw new Error('INVALID_PRODUCER_BATCH'); + if (!receipt || receipt.schemaVersion !== 2 || receipt.cellRef !== cell.id || JSON.stringify(receipt.cell) !== JSON.stringify(cell) || !HEX.test(receipt.inputHash) || receipt.requestedModel !== cell.model || !receipt.modelUsed || !['provider_reported', 'requested_pin'].includes(receipt.modelIdentitySource) || (receipt.modelIdentitySource === 'requested_pin' && receipt.modelUsed !== receipt.requestedModel) || (receipt.modelIdentitySource === 'provider_reported' && receipt.modelUsed === receipt.requestedModel) || !['succeeded', 'failed'].includes(receipt.status) || !HEX.test(receipt.outputHash) || !HEX.test(receipt.receiptHash) || !validNumber(receipt.durationMs) || receipt.firstUsefulResultMs !== null || !Number.isSafeInteger(receipt.toolCalls) || receipt.toolCalls < 0 || !receipt.usage) throw new Error('INVALID_PRODUCER_BATCH'); const started = Date.parse(receipt.startedAt), finished = Date.parse(receipt.finishedAt); if (!Number.isFinite(started) || !Number.isFinite(finished) || finished < started || !['inputTokens', 'outputTokens', 'cachedTokens', 'estimatedCostUSD'].every(field => receipt.usage[field as keyof typeof receipt.usage] === null || validNumber(receipt.usage[field as keyof typeof receipt.usage])) || ((receipt.status === 'failed') !== !!receipt.error) || (receipt.error && !receipt.error.code)) throw new Error('INVALID_PRODUCER_BATCH'); validateInstallationIdentity(receipt.installationIdentity, 'INVALID_PRODUCER_BATCH'); @@ -386,32 +434,66 @@ function validateReceiptIndex(receipt: ProducerReceiptIndex, cell: EvalCell): vo } const matchedPairKey = (cell: EvalCell): string => [cell.caseId, cell.variant, cell.mode, cell.repetition, cell.model, cell.host, cell.budgetSeconds, cell.sourceHash].join('|'); +const PROFILE_GROUPS: Record> = { + full: [['v2', 'daily'], ['v2', 'comprehensive'], ['v3', 'daily'], ['v3', 'comprehensive']], + release: [['v3', 'daily'], ['v3', 'comprehensive']], + baseline: [['v2', 'comprehensive']], +}; +const profileOf = (matrix: EvalMatrix): EvalProfile => matrix.profile ?? 'full'; +const expectedMatchedPairs = (matrix: EvalMatrix): number => profileOf(matrix) === 'full' ? matrix.cells.length / 2 : 0; -export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEvalSchedule, receipts: ProducerReceipt[]): ProducerBatch { - validateMatrix(matrix); validateSchedule(matrix, schedule); +type IdentityBearing = Pick; +/** + * Each platform has exactly one producer installation and provider binary. + * Platforms differ only in native binaries: the embedded catalogs and the + * reviewed provider policy must be identical across them. + */ +function assertPlatformIdentities(receipts: IdentityBearing[]): void { + const byPlatform = new Map(); + for (const receipt of receipts) { + const key = receipt.cell.platform ?? 'unpinned', list = byPlatform.get(key) ?? []; + list.push(receipt); byPlatform.set(key, list); + } + for (const list of byPlatform.values()) { + if (new Set(list.map(receipt => receipt.installationIdentity.identityHash)).size > 1) throw new Error('UNMATCHED_PRODUCER_INSTALLATIONS'); + if (new Set(list.map(receipt => receipt.providerIdentity.identityHash)).size > 1) throw new Error('UNMATCHED_PRODUCER_PROVIDERS'); + } + const representatives = [...byPlatform.values()].map(list => list[0]); + if (representatives.length < 2) return; + if (new Set(representatives.map(receipt => receipt.installationIdentity.identityHash)).size !== representatives.length) throw new Error('UNMATCHED_PRODUCER_PLATFORMS'); + const catalogs = (receipt: IdentityBearing) => JSON.stringify(receipt.installationIdentity.embeddedCatalogs); + const policy = ({ providerIdentity: { family, policyRevision, version, argsPrefix } }: IdentityBearing) => JSON.stringify({ family, policyRevision, version, argsPrefix }); + if (new Set(representatives.map(catalogs)).size !== 1) throw new Error('UNMATCHED_PRODUCER_INSTALLATIONS'); + if (new Set(representatives.map(policy)).size !== 1) throw new Error('UNMATCHED_PRODUCER_PROVIDERS'); +} + +export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEvalSchedule, receipts: ProducerReceipt[], corpus: EvalCorpusManifest = loadCorpusManifest()): ProducerBatch { + validateMatrix(matrix, corpus); validateSchedule(matrix, schedule); if (!Array.isArray(receipts)) throw new Error('INVALID_PRODUCER_RECEIPTS'); const cells = new Map(matrix.cells.map(cell => [cell.id, cell])); const jobs = new Map(schedule.jobs.map(job => [job.cellId, job])); const seen = new Set(); for (const receipt of receipts) { - const cell = cells.get(receipt?.cell?.id), job = jobs.get(receipt?.cell?.id); + const cell = cells.get(receipt?.cellRef), job = jobs.get(receipt?.cellRef); if (!cell || !job || seen.has(cell.id)) throw new Error('UNKNOWN_OR_DUPLICATE_PRODUCER_RECEIPT'); validateReceipt(receipt, cell, job.inputHash); seen.add(cell.id); } - if (new Set(receipts.map(receipt => receipt.installationIdentity.identityHash)).size > 1) throw new Error('UNMATCHED_PRODUCER_INSTALLATIONS'); - if (new Set(receipts.map(receipt => receipt.providerIdentity.identityHash)).size > 1) throw new Error('UNMATCHED_PRODUCER_PROVIDERS'); - const pairs = new Map>>(); - for (const receipt of receipts) { + // Receipts name only the opaque cellRef; the trusted matrix supplies the cell. + const bound = receipts.map(receipt => ({ ...receipt, cell: cells.get(receipt.cellRef)! })); + assertPlatformIdentities(bound); + const pairs = new Map>>(); + if (profileOf(matrix) === 'full') for (const receipt of bound) { const key = matchedPairKey(receipt.cell as EvalCell); const pair = pairs.get(key) ?? {}; pair[receipt.cell.version] = receipt; pairs.set(key, pair); } - const completePairs = [...pairs.entries()].filter(([, pair]) => pair.v2 && pair.v3) as Array<[string, { v2: ProducerReceipt; v3: ProducerReceipt }]>; + else if (new Set(bound.map(receipt => receipt.modelUsed)).size > 1) throw new Error('UNMATCHED_EFFECTIVE_MODELS: single-version cells used different normalized model identities'); + const completePairs = [...pairs.entries()].filter(([, pair]) => pair.v2 && pair.v3) as Array<[string, { v2: (typeof bound)[number]; v3: (typeof bound)[number] }]>; const modelMismatches = completePairs.filter(([, pair]) => pair.v2.modelUsed !== pair.v3.modelUsed).map(([pair, value]) => ({ pair: digest(pair), v2: value.v2.modelUsed, v3: value.v3.modelUsed })); if (modelMismatches.length) throw new Error(`UNMATCHED_EFFECTIVE_MODELS: ${modelMismatches.length} matched v2/v3 pair(s) used different normalized model identities`); const groups: ProducerGroupSummary[] = []; - for (const version of ['v2', 'v3'] as const) for (const mode of ['daily', 'comprehensive'] as const) { + for (const [version, mode] of PROFILE_GROUPS[profileOf(matrix)]) { const expected = matrix.cells.filter(cell => cell.version === version && cell.mode === mode); - const submitted = receipts.filter(receipt => receipt.cell.version === version && receipt.cell.mode === mode); + const submitted = bound.filter(receipt => receipt.cell.version === version && receipt.cell.mode === mode); const tokens = submitted.flatMap(receipt => receipt.usage.inputTokens === null || receipt.usage.outputTokens === null ? [] : [receipt.usage.inputTokens + receipt.usage.outputTokens]); const costs = submitted.flatMap(receipt => receipt.usage.estimatedCostUSD === null ? [] : [receipt.usage.estimatedCostUSD]); groups.push({ version, mode, scheduled: expected.length, submitted: submitted.length, missing: expected.length - submitted.length, succeeded: submitted.filter(receipt => receipt.status === 'succeeded').length, failed: submitted.filter(receipt => receipt.status === 'failed').length, @@ -421,32 +503,102 @@ export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEv estimatedCost: { measured: costs.length, denominator: expected.length, totalUSD: costs.length ? costs.reduce((sum, value) => sum + value, 0) : null, source: 'pricing-table-estimate' }, }); } - const indexes: ProducerReceiptIndex[] = receipts.map(({ output: _output, error, ...receipt }) => ({ ...receipt, ...(error ? { error: { code: error.code } } : {}) })); + const indexes: ProducerReceiptIndex[] = bound.map(({ output: _output, error, ...receipt }) => ({ ...receipt, ...(error ? { error: { code: error.code } } : {}) })); const base = { schemaVersion: 1 as const, matrixHash: matrixHash(matrix), scheduleHash: digest(JSON.stringify(schedule)), receipts: indexes.sort((left, right) => left.cell.id.localeCompare(right.cell.id)), summary: { - scheduled: matrix.cells.length, prepared: schedule.preparedCells, submitted: receipts.length, missing: matrix.cells.length - receipts.length, - matchedPairsExpected: matrix.cells.length / 2, matchedPairsSubmitted: completePairs.length, modelMismatches, groups, + scheduled: matrix.cells.length, prepared: schedule.preparedCells, submitted: bound.length, missing: matrix.cells.length - bound.length, + matchedPairsExpected: expectedMatchedPairs(matrix), matchedPairsSubmitted: completePairs.length, modelMismatches, groups, note: 'Costs are pricing-table estimates. First-useful timing is unmeasured because the reused provider adapters return completed runs. Trusted findings and runtime outcomes require separate oracle adjudication.', } }; return { ...base, batchHash: digest(JSON.stringify(base)) }; } -export function createEvalMatrix(options: { model: string; host: ProducerHost; skillHashes: { v2: string; v3: string }; budgets?: { daily: number; comprehensive: number } }, corpus: CorpusManifest = loadCorpusManifest()): EvalMatrix { - if (!options.model?.trim() || !['claude', 'codex', 'gemini'].includes(options.host) || !HEX.test(options.skillHashes.v2) || !HEX.test(options.skillHashes.v3) || options.skillHashes.v2 === options.skillHashes.v3) throw new Error('INVALID_MATCHED_EVAL_INPUT'); - const budgets = options.budgets ?? { daily: 600, comprehensive: 1800 }; - if (![budgets.daily, budgets.comprehensive].every(value => Number.isInteger(value) && value > 60 && value <= 3600)) throw new Error('INVALID_EVAL_BUDGET'); - const cells: EvalCell[] = []; - for (const fixture of corpus.cases) for (const variant of ['vulnerable', 'fixed'] as const) for (const mode of ['daily', 'comprehensive'] as const) for (const repetition of [1, 2, 3] as const) for (const version of ['v2', 'v3'] as const) { - const cell = { caseId: fixture.id, stack: fixture.stack, variant, version, mode, repetition, model: options.model, host: options.host, budgetSeconds: budgets[mode], sourceHash: fixture.filesHash[variant], skillHash: options.skillHashes[version] }; - cells.push({ id: digest(JSON.stringify(cell)), ...cell }); - } - return { schemaVersion: 1, corpusVersion: corpus.version, corpusHash: digest(JSON.stringify(corpus)), model: options.model, host: options.host, budgets, skillHashes: options.skillHashes, repetitions: 3, cells }; +function normalizedPlatforms(corpus: EvalCorpusManifest, value: unknown): Record { + const assignment = value as Record; + if (!assignment || typeof assignment !== 'object' || Array.isArray(assignment) || Object.keys(assignment).length !== corpus.cases.length || + corpus.cases.some(fixture => !PRODUCER_PLATFORMS.includes(assignment[fixture.id]))) throw new Error('INVALID_EVAL_PLATFORMS'); + return Object.fromEntries(corpus.cases.map(fixture => [fixture.id, assignment[fixture.id]])); } -export function validateMatrix(matrix: EvalMatrix, corpus: CorpusManifest = loadCorpusManifest()): void { - const expected = createEvalMatrix({ model: matrix.model, host: matrix.host, skillHashes: matrix.skillHashes, budgets: matrix.budgets }, corpus); +function buildEvalMatrix(options: EvalMatrixOptions, corpus: EvalCorpusManifest, baselineHash: string | null): EvalMatrix { + const profile = options.profile ?? 'full'; + const skillHashes: Partial> = options.skillHashes ?? {}; + const versions = profile === 'full' ? ['v2', 'v3'] : profile === 'release' ? ['v3'] : ['v2']; + if (!options.model?.trim() || !['claude', 'codex', 'gemini'].includes(options.host) || !['full', 'release', 'baseline'].includes(profile) || + Object.keys(skillHashes).sort().join(',') !== versions.join(',') || versions.some(version => !HEX.test(skillHashes[version as EvalVersion] ?? '')) || + (profile === 'full' && skillHashes.v2 === skillHashes.v3)) throw new Error('INVALID_MATCHED_EVAL_INPUT'); + const budgets = options.budgets ?? { daily: 600, comprehensive: 1800 }; + if (![budgets.daily, budgets.comprehensive].every(value => Number.isInteger(value) && value > 60 && value <= 3600)) throw new Error('INVALID_EVAL_BUDGET'); + const corpusHash = digest(JSON.stringify(corpus)); + const cells: EvalCell[] = []; + if (options.profile === undefined || options.profile === 'full') { + for (const fixture of corpus.cases) for (const variant of ['vulnerable', 'fixed'] as const) for (const mode of ['daily', 'comprehensive'] as const) for (const repetition of [1, 2, 3] as const) for (const version of ['v2', 'v3'] as const) { + const cell = { caseId: fixture.id, stack: fixture.stack, variant, version, mode, repetition, model: options.model, host: options.host, budgetSeconds: budgets[mode], sourceHash: fixture.filesHash[variant], skillHash: options.skillHashes[version] }; + cells.push({ id: digest(JSON.stringify(cell)), ...cell }); + } + return { schemaVersion: 1, corpusVersion: corpus.version, corpusHash, model: options.model, host: options.host, budgets, skillHashes: options.skillHashes, repetitions: 3, cells }; + } + const pinned = options as ReleaseEvalMatrixOptions | BaselineEvalMatrixOptions; + const platformKeys = pinned.profile === 'release' ? 'comprehensive,daily' : 'comprehensive'; + if (!pinned.platforms || Object.keys(pinned.platforms).sort().join(',') !== platformKeys) throw new Error('INVALID_EVAL_PLATFORMS'); + const comprehensive = normalizedPlatforms(corpus, pinned.platforms.comprehensive); + const daily = pinned.profile === 'release' ? pinned.platforms.daily : undefined; + if (pinned.profile === 'release' && !PRODUCER_PLATFORMS.includes(daily!)) throw new Error('INVALID_EVAL_PLATFORMS'); + const version: EvalVersion = pinned.profile === 'release' ? 'v3' : 'v2'; + const modes = pinned.profile === 'release' ? ['daily', 'comprehensive'] as const : ['comprehensive'] as const; + for (const fixture of corpus.cases) for (const variant of ['vulnerable', 'fixed'] as const) for (const mode of modes) { + const cell = { caseId: fixture.id, stack: fixture.stack, variant, version, mode, repetition: 1 as const, model: pinned.model, host: pinned.host, budgetSeconds: budgets[mode], sourceHash: fixture.filesHash[variant], skillHash: skillHashes[version]!, platform: mode === 'daily' ? daily! : comprehensive[fixture.id] }; + cells.push({ id: digest(JSON.stringify(cell)), ...cell }); + } + const common = { schemaVersion: 1 as const, corpusVersion: corpus.version, corpusHash, model: pinned.model, host: pinned.host, budgets }; + if (pinned.profile === 'release') { + if (baselineHash !== null && !HEX.test(baselineHash)) throw new Error('UNBOUND_EVAL_BASELINE'); + return { ...common, profile: 'release', skillHashes: { v3: pinned.skillHashes.v3 }, repetitions: 1, platforms: { daily: daily!, comprehensive }, baselineHash, cells }; + } + return { ...common, profile: 'baseline', skillHashes: { v2: pinned.skillHashes.v2 }, repetitions: 1, platforms: { comprehensive }, cells }; +} + +const BASELINE_FIELDS = 'baselineHash,budgetSeconds,cases,corpusHash,corpusVersion,host,matrixHash,model,modelUsed,platforms,producerBatchHash,provider,schemaVersion,skillHash,version'; +/** Check that a baseline was measured under the same corpus, model, host, budget and platform assignment. */ +function assertBaselineBinding(baseline: EvalBaseline, target: { corpus: EvalCorpusManifest; model: string; host: ProducerHost; budgetSeconds: number; platforms: Record }): void { + if (!baseline || typeof baseline !== 'object' || Object.keys(baseline).sort().join(',') !== BASELINE_FIELDS) throw new Error('UNBOUND_EVAL_BASELINE'); + const { baselineHash, ...record } = baseline; + const provider = baseline.provider as unknown as Record; + if (baseline.schemaVersion !== 1 || baseline.version !== 'v2' || !HEX.test(baselineHash) || digest(JSON.stringify(record)) !== baselineHash || + baseline.corpusVersion !== target.corpus.version || baseline.corpusHash !== digest(JSON.stringify(target.corpus)) || + baseline.model !== target.model || baseline.host !== target.host || baseline.budgetSeconds !== target.budgetSeconds || + JSON.stringify(baseline.platforms) !== JSON.stringify(target.platforms) || !HEX.test(baseline.skillHash) || !HEX.test(baseline.matrixHash) || + !HEX.test(baseline.producerBatchHash) || typeof baseline.modelUsed !== 'string' || !baseline.modelUsed || + !provider || typeof provider !== 'object' || Object.keys(provider).sort().join(',') !== 'family,policyRevision,version' || + !['claude', 'gpt', 'gemini'].includes(String(provider.family)) || typeof provider.policyRevision !== 'string' || !provider.policyRevision || + typeof provider.version !== 'string' || !provider.version || !Array.isArray(baseline.cases) || baseline.cases.length !== target.corpus.cases.length || + baseline.cases.some((entry, index) => !entry || Object.keys(entry).sort().join(',') !== 'caseId,found' || entry.caseId !== target.corpus.cases[index].id || typeof entry.found !== 'boolean')) + throw new Error('UNBOUND_EVAL_BASELINE'); +} + +export function createEvalMatrix(options: FullEvalMatrixOptions, corpus?: EvalCorpusManifest): FullEvalMatrix; +export function createEvalMatrix(options: ReleaseEvalMatrixOptions, corpus?: EvalCorpusManifest): ReleaseEvalMatrix; +export function createEvalMatrix(options: BaselineEvalMatrixOptions, corpus?: EvalCorpusManifest): BaselineEvalMatrix; +export function createEvalMatrix(options: EvalMatrixOptions, corpus: EvalCorpusManifest = loadCorpusManifest()): EvalMatrix { + let baselineHash: string | null = null; + if (options.profile === 'release' && options.baseline !== undefined) { + assertBaselineBinding(options.baseline, { corpus, model: options.model, host: options.host, budgetSeconds: (options.budgets ?? { comprehensive: 1800 }).comprehensive, platforms: normalizedPlatforms(corpus, options.platforms?.comprehensive) }); + baselineHash = options.baseline.baselineHash; + } + return buildEvalMatrix(options, corpus, baselineHash); +} + +export function validateMatrix(matrix: EvalMatrix, corpus: EvalCorpusManifest = loadCorpusManifest()): void { + if (!matrix || typeof matrix !== 'object') throw new Error('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX'); + const common = { model: matrix.model, host: matrix.host, budgets: matrix.budgets }; + let expected: EvalMatrix; + try { + expected = matrix.profile === 'release' ? buildEvalMatrix({ ...common, profile: 'release', skillHashes: matrix.skillHashes, platforms: matrix.platforms }, corpus, matrix.baselineHash) + : matrix.profile === 'baseline' ? buildEvalMatrix({ ...common, profile: 'baseline', skillHashes: matrix.skillHashes, platforms: matrix.platforms }, corpus, null) + : buildEvalMatrix({ ...common, skillHashes: matrix.skillHashes }, corpus, null); + } catch { throw new Error('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX'); } if (JSON.stringify(matrix) !== JSON.stringify(expected)) throw new Error('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX'); } -function validateResult(result: EvalResult, cell: EvalCell, corpus: CorpusManifest): void { +function validateResult(result: EvalResult, cell: EvalCell, corpus: EvalCorpusManifest): void { if (result.sourceHash !== cell.sourceHash || result.skillHash !== cell.skillHash || result.model !== cell.model || result.host !== cell.host || result.budgetSeconds !== cell.budgetSeconds) throw new Error('UNMATCHED_EVAL_RESULT'); for (const field of ['setup', 'reproduction', 'repair', 'recheck'] as const) if (!['passed', 'failed', 'blocked', 'not_attempted'].includes(result[field])) throw new Error('INVALID_EVAL_OUTCOME'); for (const field of ['reportPresent', 'reportComplete', 'heldOutAssertionsPassed', 'freshRecheck'] as const) if (typeof result[field] !== 'boolean') throw new Error('INVALID_EVAL_RESULT'); @@ -493,7 +645,7 @@ function quantile(values: number[], fraction: number): number | null { if (!values.length) return null; return [...values].sort((left, right) => left - right)[Math.ceil(values.length * fraction) - 1]; } -function measuredRate(cells: EvalCell[], results: Map, field: 'setup' | 'reproduction' | 'repair' | 'recheck', corpus: CorpusManifest): Rate { +function measuredRate(cells: EvalCell[], results: Map, field: 'setup' | 'reproduction' | 'repair' | 'recheck', corpus: EvalCorpusManifest): Rate { return rate(cells.filter(cell => { const result = results.get(cell.id); if (!result?.reportPresent || !result.reportComplete || result[field] !== 'passed') return false; @@ -511,7 +663,7 @@ function measuredRate(cells: EvalCell[], results: Map, field }).length, cells.length); } -function trustedFindingVerification(cell: EvalCell, result: EvalResult, finding: EvalFinding, stage: 'repair' | 'recheck', corpus: CorpusManifest): boolean { +function trustedFindingVerification(cell: EvalCell, result: EvalResult, finding: EvalFinding, stage: 'repair' | 'recheck', corpus: EvalCorpusManifest): boolean { const verification = finding.trustedVerification; if (!result.reportPresent || !result.reportComplete || cell.version !== 'v3' || cell.mode !== 'comprehensive' || cell.variant !== 'vulnerable' || finding.evidence !== 'supported' || finding.judgment !== 'correct' || finding.matchedCaseId !== cell.caseId || !finding.claimedTested || !verification || @@ -524,8 +676,7 @@ function trustedFindingVerification(cell: EvalCell, result: EvalResult, finding: !!result.recheckEvidenceHash && result.recheckEvidenceHash !== result.oracleEvidenceHash; } -export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualification: ReleaseQualification = { containment: {} }, corpus: CorpusManifest = loadCorpusManifest()) { - validateMatrix(matrix, corpus); +function validatedResults(matrix: EvalMatrix, observations: EvalResult[], corpus: EvalCorpusManifest): Map { if (!Array.isArray(observations)) throw new Error('INVALID_EVAL_RESULTS'); const cells = new Map(matrix.cells.map(cell => [cell.id, cell])); const results = new Map(); @@ -534,18 +685,34 @@ export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualif if (!cell || results.has(result.cellId)) throw new Error('UNKNOWN_OR_DUPLICATE_EVAL_RESULT'); validateResult(result, cell, corpus); results.set(result.cellId, result); } - const eligibleFinding = (cell: EvalCell, finding: EvalFinding) => finding.evidence === 'supported' || (cell.version === 'v2' && finding.evidence === 'legacy_review'); - const found = (cell: EvalCell) => { - const result = results.get(cell.id); - return result?.reportPresent === true && result.reportComplete === true && - result.findings.some(finding => eligibleFinding(cell, finding) && finding.judgment === 'correct'); - }; + return results; +} +const eligibleFinding = (cell: EvalCell, finding: EvalFinding) => finding.evidence === 'supported' || (cell.version === 'v2' && finding.evidence === 'legacy_review'); +const foundSeededIssue = (cell: EvalCell, result: EvalResult | undefined) => result?.reportPresent === true && result.reportComplete === true && + result.findings.some(finding => eligibleFinding(cell, finding) && finding.judgment === 'correct'); +const isHighCritical = (corpus: EvalCorpusManifest, caseId: string) => ['critical', 'high'].includes(corpus.cases.find(fixture => fixture.id === caseId)!.severity); + +/** High/critical recall of a pre-registered external baseline, after proving it binds to this release matrix. */ +function externalBaselineRecall(matrix: ReleaseEvalMatrix, baseline: EvalBaseline, corpus: EvalCorpusManifest): Rate { + if (matrix.baselineHash === null || baseline?.baselineHash !== matrix.baselineHash) throw new Error('UNBOUND_EVAL_BASELINE'); + assertBaselineBinding(baseline, { corpus, model: matrix.model, host: matrix.host, budgetSeconds: matrix.budgets.comprehensive, platforms: matrix.platforms.comprehensive }); + const highCritical = baseline.cases.filter(entry => isHighCritical(corpus, entry.caseId)); + return rate(highCritical.filter(entry => entry.found).length, highCritical.length); +} + +export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualification: ReleaseQualification = { containment: {} }, corpus: EvalCorpusManifest = loadCorpusManifest(), baseline?: EvalBaseline) { + validateMatrix(matrix, corpus); + const profile = profileOf(matrix); + if (profile === 'baseline') throw new Error('BASELINE_PROFILE_NOT_QUALIFIABLE: use createEvalBaseline'); + if (profile !== 'release' && baseline !== undefined) throw new Error('UNBOUND_EVAL_BASELINE'); + const results = validatedResults(matrix, observations, corpus); + const found = (cell: EvalCell) => foundSeededIssue(cell, results.get(cell.id)); const groups: any[] = []; - for (const version of ['v2', 'v3'] as const) for (const mode of ['daily', 'comprehensive'] as const) { + for (const [version, mode] of PROFILE_GROUPS[profile]) { const selected = matrix.cells.filter(cell => cell.version === version && cell.mode === mode); const completed = selected.map(cell => results.get(cell.id)).filter((result): result is EvalResult => !!result); const expected = selected.filter(cell => cell.variant === 'vulnerable'); - const highCritical = expected.filter(cell => ['critical','high'].includes(corpus.cases.find(fixture => fixture.id === cell.caseId)!.severity)); + const highCritical = expected.filter(cell => isHighCritical(corpus, cell.caseId)); let correct = 0, reported = 0, unadjudicated = 0; for (const cell of selected) { const result = results.get(cell.id); @@ -576,7 +743,8 @@ export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualif } const daily = groups.find(group => group.version === 'v3' && group.mode === 'daily'); const comprehensive = groups.find(group => group.version === 'v3' && group.mode === 'comprehensive'); - const baseline = groups.find(group => group.version === 'v2' && group.mode === 'comprehensive'); + const inBatchBaseline = groups.find(group => group.version === 'v2' && group.mode === 'comprehensive'); + const external = matrix.profile === 'release' && baseline !== undefined ? externalBaselineRecall(matrix, baseline, corpus) : null; const perStack = Object.fromEntries(STACKS.map(stack => { const eligible = matrix.cells.filter(cell => cell.version === 'v3' && cell.mode === 'comprehensive' && cell.variant === 'vulnerable' && cell.stack === stack); // A repaired oracle case demonstrates the full find-to-repair workflow only @@ -596,47 +764,95 @@ export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualif mandatoryReports: gate(groups.every(group => group.reports.numerator === group.reports.denominator), observations.length === matrix.cells.length), dailyPrecision95: gate(daily.precision.value === null ? null : daily.precision.value >= 0.95, daily.submitted === daily.cells && !daily.unadjudicated), comprehensiveHighCriticalRecall80: gate(comprehensive.highCriticalRecall.value >= 0.8, comprehensive.submitted === comprehensive.cells && !comprehensive.unadjudicated), - noHighCriticalRecallRegression: gate(comprehensive.highCriticalRecall.value >= baseline.highCriticalRecall.value, comprehensive.submitted === comprehensive.cells && baseline.submitted === baseline.cells && !comprehensive.unadjudicated && !baseline.unadjudicated), + noHighCriticalRecallRegression: profile === 'full' + ? gate(comprehensive.highCriticalRecall.value >= inBatchBaseline.highCriticalRecall.value, comprehensive.submitted === comprehensive.cells && inBatchBaseline.submitted === inBatchBaseline.cells && !comprehensive.unadjudicated && !inBatchBaseline.unadjudicated) + : gate(external === null ? null : comprehensive.highCriticalRecall.value >= external.value!, comprehensive.submitted === comprehensive.cells && !comprehensive.unadjudicated && external !== null), allCoreColdStarts: gate(comprehensive.setup.numerator === comprehensive.setup.denominator, comprehensive.submitted === comprehensive.cells), zeroFalselyTestedRepairs: gate(comprehensive.falseTested === 0 && daily.falseTested === 0, comprehensive.submitted === comprehensive.cells && daily.submitted === daily.cells), heldOutRepairEachStack: gate(Object.values(perStack).every(value => value.correctHeldOutRepairs >= 1), comprehensive.submitted === comprehensive.cells), containmentAndCanaries: containmentValues.includes('failed') ? 'fail' as const : containmentValues.every(value => value === 'passed') ? 'pass' as const : 'unmeasured' as const, }; return { schemaVersion: 1, corpusVersion: corpus.version, model: matrix.model, host: matrix.host, + ...(profile === 'release' ? { profile, baseline: external === null ? null : { baselineHash: baseline!.baselineHash, highCriticalRecall: external } } : {}), status: Object.values(gates).every(value => value === 'pass') ? 'qualified' : observations.length ? 'partial' : 'unmeasured', groups, perStack, gates, notes: ['Missing and setup-blocked supported scenarios remain in recall and workflow denominators.', 'Costs and tokens come only from host-reported usage; wall-clock budgets are not model-spend caps.', 'Oracle judgments and qualification receipts must come from the trusted evaluator, never producing agents.'] }; } -/** Release scoring path: every trusted judgment must bind to a collected producer receipt. */ -export function scoreCollectedEval(matrix: EvalMatrix, batch: ProducerBatch, observations: EvalResult[], qualification: ReleaseQualification = { containment: {} }, corpus: CorpusManifest = loadCorpusManifest()) { - validateMatrix(matrix, corpus); +/** Bind a complete collected batch and every trusted judgment to the matrix's producer receipts. */ +function validateCollectedBatch(matrix: EvalMatrix, batch: ProducerBatch, observations: EvalResult[]): Map { if (!Array.isArray(observations)) throw new Error('INVALID_EVAL_RESULTS'); if (!batch || batch.schemaVersion !== 1 || batch.matrixHash !== matrixHash(matrix) || !HEX.test(batch.scheduleHash) || !HEX.test(batch.batchHash)) throw new Error('INVALID_PRODUCER_BATCH'); const { batchHash, ...withoutHash } = batch; - if (digest(JSON.stringify(withoutHash)) !== batchHash || !Array.isArray(batch.receipts) || batch.receipts.length !== matrix.cells.length || batch.summary.scheduled !== matrix.cells.length || batch.summary.prepared !== matrix.cells.length || batch.summary.submitted !== matrix.cells.length || batch.summary.missing !== 0 || batch.summary.matchedPairsExpected !== matrix.cells.length / 2 || batch.summary.matchedPairsSubmitted !== matrix.cells.length / 2 || batch.summary.modelMismatches.length !== 0) throw new Error('INCOMPLETE_PRODUCER_BATCH'); + const pairs = expectedMatchedPairs(matrix); + if (digest(JSON.stringify(withoutHash)) !== batchHash || !Array.isArray(batch.receipts) || batch.receipts.length !== matrix.cells.length || batch.summary.scheduled !== matrix.cells.length || batch.summary.prepared !== matrix.cells.length || batch.summary.submitted !== matrix.cells.length || batch.summary.missing !== 0 || batch.summary.matchedPairsExpected !== pairs || batch.summary.matchedPairsSubmitted !== pairs || batch.summary.modelMismatches.length !== 0) throw new Error('INCOMPLETE_PRODUCER_BATCH'); const cells = new Map(matrix.cells.map(cell => [cell.id, cell])); const receipts = new Map(); for (const receipt of batch.receipts) { - const cell = cells.get(receipt?.cell?.id); + const cell = cells.get(receipt?.cellRef); if (!cell || receipts.has(cell.id)) throw new Error('INVALID_PRODUCER_BATCH'); validateReceiptIndex(receipt, cell); receipts.set(cell.id, receipt); } - if (new Set(batch.receipts.map(receipt => receipt.installationIdentity.identityHash)).size !== 1) throw new Error('UNMATCHED_PRODUCER_INSTALLATIONS'); - if (new Set(batch.receipts.map(receipt => receipt.providerIdentity.identityHash)).size !== 1) throw new Error('UNMATCHED_PRODUCER_PROVIDERS'); - const actualModels = new Map>>(); - for (const receipt of batch.receipts) { - const key = matchedPairKey(receipt.cell as EvalCell), pair = actualModels.get(key) ?? {}; - pair[receipt.cell.version] = receipt.modelUsed; actualModels.set(key, pair); - } - if (actualModels.size !== matrix.cells.length / 2 || [...actualModels.values()].some(pair => !pair.v2 || !pair.v3 || pair.v2 !== pair.v3)) throw new Error('UNMATCHED_EFFECTIVE_MODELS'); + assertPlatformIdentities(batch.receipts); + if (profileOf(matrix) === 'full') { + const actualModels = new Map>>(); + for (const receipt of batch.receipts) { + const key = matchedPairKey(receipt.cell as EvalCell), pair = actualModels.get(key) ?? {}; + pair[receipt.cell.version] = receipt.modelUsed; actualModels.set(key, pair); + } + if (actualModels.size !== pairs || [...actualModels.values()].some(pair => !pair.v2 || !pair.v3 || pair.v2 !== pair.v3)) throw new Error('UNMATCHED_EFFECTIVE_MODELS'); + } else if (new Set(batch.receipts.map(receipt => receipt.modelUsed)).size !== 1) throw new Error('UNMATCHED_EFFECTIVE_MODELS'); for (const result of observations) { const receipt = receipts.get(result.cellId); if (!receipt || result.producerReceiptHash !== receipt.receiptHash) throw new Error('UNBOUND_EVAL_RESULT'); } - const score = scoreEval(matrix, observations, qualification, corpus); + return receipts; +} + +/** Release scoring path: every trusted judgment must bind to a collected producer receipt. */ +export function scoreCollectedEval(matrix: EvalMatrix, batch: ProducerBatch, observations: EvalResult[], qualification: ReleaseQualification = { containment: {} }, corpus: EvalCorpusManifest = loadCorpusManifest(), baseline?: EvalBaseline) { + validateMatrix(matrix, corpus); + if (profileOf(matrix) === 'baseline') throw new Error('BASELINE_PROFILE_NOT_QUALIFIABLE: use createEvalBaseline'); + validateCollectedBatch(matrix, batch, observations); + if (baseline !== undefined) { + const { modelUsed, providerIdentity: { family, policyRevision, version } } = batch.receipts[0]; + if (baseline?.modelUsed !== modelUsed || JSON.stringify(baseline?.provider) !== JSON.stringify({ family, policyRevision, version })) throw new Error('UNBOUND_EVAL_BASELINE'); + } + const score = scoreEval(matrix, observations, qualification, corpus, baseline); return { ...score, producerBatchHash: batch.batchHash }; } +/** + * Turn a complete, fully adjudicated v2 baseline run into a reusable record. + * Missing reports count as misses; an incomplete or unadjudicated run is refused. + */ +export function createEvalBaseline(matrix: BaselineEvalMatrix, batch: ProducerBatch, observations: EvalResult[], corpus: EvalCorpusManifest = loadCorpusManifest()): EvalBaseline { + if (matrix?.profile !== 'baseline') throw new Error('INVALID_EVAL_BASELINE_MATRIX'); + validateMatrix(matrix, corpus); + validateCollectedBatch(matrix, batch, observations); + const results = validatedResults(matrix, observations, corpus); + if (results.size !== matrix.cells.length) throw new Error('INCOMPLETE_EVAL_BASELINE'); + if ([...results.values()].some(result => result.reportPresent && result.reportComplete && result.findings.some(finding => finding.evidence !== 'hypothesis' && finding.judgment === 'unadjudicated'))) throw new Error('UNADJUDICATED_EVAL_BASELINE'); + const vulnerable = new Map(matrix.cells.filter(cell => cell.variant === 'vulnerable').map(cell => [cell.caseId, cell])); + const { modelUsed, providerIdentity: { family, policyRevision, version } } = batch.receipts[0]; + const record: Omit = { + schemaVersion: 1, version: 'v2', corpusVersion: corpus.version, corpusHash: matrix.corpusHash, model: matrix.model, host: matrix.host, + budgetSeconds: matrix.budgets.comprehensive, skillHash: matrix.skillHashes.v2, platforms: matrix.platforms.comprehensive, + matrixHash: matrixHash(matrix), producerBatchHash: batch.batchHash, modelUsed, provider: { family, policyRevision, version }, + cases: corpus.cases.map(fixture => ({ caseId: fixture.id, found: foundSeededIssue(vulnerable.get(fixture.id)!, results.get(vulnerable.get(fixture.id)!.id)) })), + }; + return { ...record, baselineHash: digest(JSON.stringify(record)) }; +} + +function splitFlags(args: string[], allowed: string[]): { positional: string[]; flags: Record } { + const positional: string[] = [], flags: Record = {}; + for (let index = 0; index < args.length; index++) { + if (!args[index].startsWith('--')) { positional.push(args[index]); continue; } + if (!allowed.includes(args[index]) || !args[index + 1] || flags[args[index]]) throw new Error(`Unsupported or repeated flag: ${args[index]}`); + flags[args[index]] = args[++index]; + } + return { positional, flags }; +} + function cli(args: string[]): void { const [command, ...rest] = args; if (command === 'payload') { @@ -669,25 +885,35 @@ function cli(args: string[]): void { const schedule = prepareEvalJobs(matrix, { v2: readBoundedStable(resolve(rest[2]), PORTABLE_PAYLOAD_LIMIT, 'v2 portable skill payload').toString('utf8'), v3: readBoundedStable(resolve(rest[4]), PORTABLE_PAYLOAD_LIMIT, 'v3 portable skill payload').toString('utf8') }, rest[6]); console.log(JSON.stringify({ scheduled: schedule.scheduledCells, prepared: schedule.preparedCells, output: resolve(rest[6]), paidCalls: 0 })); return; } + const { positional, flags } = splitFlags(rest, ['--corpus-manifest', '--baseline']); + const corpus = flags['--corpus-manifest'] ? validateCorpusManifest(readJsonBounded(flags['--corpus-manifest'], 16 * 1024 * 1024, 'corpus manifest')) : loadCorpusManifest(); if (command === 'collect') { - if (rest.length !== 4) throw new Error('Usage: cso-eval collect '); - const matrix = readJsonBounded(rest[0], 16 * 1024 * 1024, 'evaluation matrix') as EvalMatrix; - const schedule = readJsonBounded(rest[1], 16 * 1024 * 1024, 'evaluation schedule') as PreparedEvalSchedule; - const receiptRoot = resolve(rest[2]), stat = lstatSync(receiptRoot); + if (positional.length !== 4 || flags['--baseline']) throw new Error('Usage: cso-eval collect [--corpus-manifest ]'); + const [matrixPath, schedulePath, receiptDirectory, output] = positional; + const matrix = readJsonBounded(matrixPath, 16 * 1024 * 1024, 'evaluation matrix') as EvalMatrix; + const schedule = readJsonBounded(schedulePath, 16 * 1024 * 1024, 'evaluation schedule') as PreparedEvalSchedule; + const receiptRoot = resolve(receiptDirectory), stat = lstatSync(receiptRoot); if (!stat.isDirectory() || stat.isSymbolicLink() || realpathSync(receiptRoot) !== receiptRoot) throw new Error('UNSAFE_RECEIPT_DIRECTORY'); const receipts: ProducerReceipt[] = []; for (const entry of readdirSync(receiptRoot, { withFileTypes: true })) { if (!entry.isFile() || entry.isSymbolicLink() || !/^[a-f0-9]{64}\.json$/.test(entry.name)) throw new Error('UNSAFE_RECEIPT_DIRECTORY'); receipts.push(readJsonBounded(join(receiptRoot, entry.name), 40 * 1024 * 1024, 'producer receipt')); } - const batch = collectProducerReceipts(matrix, schedule, receipts); - safeWriteNew(resolve(rest[3]), batch); - console.log(JSON.stringify({ output: resolve(rest[3]), submitted: batch.summary.submitted, missing: batch.summary.missing, modelMismatches: batch.summary.modelMismatches.length, paidCalls: 0 })); return; + const batch = collectProducerReceipts(matrix, schedule, receipts, corpus); + safeWriteNew(resolve(output), batch); + console.log(JSON.stringify({ output: resolve(output), submitted: batch.summary.submitted, missing: batch.summary.missing, modelMismatches: batch.summary.modelMismatches.length, paidCalls: 0 })); return; } if (command === 'score') { - if (rest.length < 3 || rest.length > 4) throw new Error('Usage: cso-eval score [qualification.json]'); - console.log(JSON.stringify(scoreCollectedEval(readJsonBounded(rest[0], 16 * 1024 * 1024, 'evaluation matrix'), readJsonBounded(rest[1], 16 * 1024 * 1024, 'producer batch'), readJsonBounded(rest[2], 64 * 1024 * 1024, 'trusted results'), rest[3] ? readJsonBounded(rest[3], 4 * 1024 * 1024, 'qualification') : undefined), null, 2)); return; + if (positional.length < 3 || positional.length > 4) throw new Error('Usage: cso-eval score [qualification.json] [--corpus-manifest ] [--baseline ]'); + const baseline = flags['--baseline'] ? readJsonBounded(flags['--baseline'], 4 * 1024 * 1024, 'evaluation baseline') as EvalBaseline : undefined; + console.log(JSON.stringify(scoreCollectedEval(readJsonBounded(positional[0], 16 * 1024 * 1024, 'evaluation matrix'), readJsonBounded(positional[1], 16 * 1024 * 1024, 'producer batch'), readJsonBounded(positional[2], 64 * 1024 * 1024, 'trusted results'), positional[3] ? readJsonBounded(positional[3], 4 * 1024 * 1024, 'qualification') : undefined, corpus, baseline), null, 2)); return; } - throw new Error('Usage: cso-eval '); + if (command === 'baseline') { + if (positional.length !== 4 || flags['--baseline']) throw new Error('Usage: cso-eval baseline [--corpus-manifest ]'); + const baseline = createEvalBaseline(readJsonBounded(positional[0], 16 * 1024 * 1024, 'evaluation matrix'), readJsonBounded(positional[1], 16 * 1024 * 1024, 'producer batch'), readJsonBounded(positional[2], 64 * 1024 * 1024, 'trusted results'), corpus); + safeWriteNew(resolve(positional[3]), baseline); + console.log(JSON.stringify({ output: resolve(positional[3]), baselineHash: baseline.baselineHash, paidCalls: 0 })); return; + } + throw new Error('Usage: cso-eval '); } if (import.meta.main) { try { cli(process.argv.slice(2)); } catch (error) { console.error(error instanceof Error ? error.message : 'CSO evaluation failed'); process.exitCode = 1; } } diff --git a/scripts/cso-native-evidence.ts b/scripts/cso-native-evidence.ts new file mode 100644 index 000000000..7c3de1b4f --- /dev/null +++ b/scripts/cso-native-evidence.ts @@ -0,0 +1,131 @@ +#!/usr/bin/env bun +/** Derive native runtime gate evidence from the named Docker tests that actually executed and passed. */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { sha256 } from '../lib/cso/contracts'; +import type { ImageBuildRow } from './cso-image-matrix'; +import { requiredChecks } from './cso-runtime-promotion'; + +type Stack = ImageBuildRow['stack']; +export interface TestRef { file: string; name: string } +export interface JUnitCase extends TestRef { status: 'passed' | 'failed' | 'skipped'; assertions: number } + +const MAX_REPORT_BYTES = 4 * 1024 * 1024; +const DOCKER = 'test/cso-docker-integration.test.ts'; +const CONTAINMENT: TestRef[] = [ + { file: DOCKER, name: 'hard fails when local daemon enforcement prerequisites are absent' }, + { file: DOCKER, name: 'rejects image-declared writable volumes before container creation' }, + { file: DOCKER, name: 'shares only loopback while denying egress, privileges, and daemon logs, and reads private source/policy mounts' }, + { file: DOCKER, name: 'machine-wide admission allows only two groups per endpoint' }, +]; +const STAGED: TestRef = { file: DOCKER, name: 'staged runtime executes its trusted verifier and checks every declared tool version' }; +const NODE_LIFECYCLE: TestRef = { + file: 'test/cso-node-lifecycle-integration.test.ts', + name: 'finds, reproduces, bundles, replays, and closes against current source through the command dispatcher', +}; +const COLD_JOURNEY: TestRef = { + file: 'test/cso-stack-cold-integration.test.ts', + name: 'acquires, prepares, boots, controls, and tests the requested staged stack with no target egress', +}; + +/** Each native check key names the tests whose execution proves it for one stack. */ +export function nativeTestRequirements(stack: Stack): Record { + if (stack === 'postgresql') return { + containmentPassed: [...CONTAINMENT, STAGED], + coldStartPassed: [STAGED], + multiDatabasePassed: [STAGED], + readinessPassed: [STAGED], + }; + const journey = stack === 'node' ? NODE_LIFECYCLE : COLD_JOURNEY; + const application = { + containmentPassed: [...CONTAINMENT, STAGED], + coldStartPassed: [journey], + positiveNegativeAssertionsPassed: [STAGED], + acquisitionPublicOnlyPassed: [journey], + offlineLifecyclePassed: [journey], + }; + return stack === 'rails' + ? { ...application, railsSqlitePassed: [COLD_JOURNEY], railsPostgresqlPassed: [COLD_JOURNEY], nativeExtensionsPassed: [COLD_JOURNEY] } + : application; +} + +function decode(value: string): string { + return value.replace(/&(#x[0-9a-fA-F]+|#[0-9]+|amp|lt|gt|quot|apos);/g, (_, entity: string) => { + if (entity.startsWith('#x')) return String.fromCodePoint(Number.parseInt(entity.slice(2), 16)); + if (entity.startsWith('#')) return String.fromCodePoint(Number.parseInt(entity.slice(1), 10)); + return { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }[entity]!; + }); +} + +/** Parse the testcases of Bun's `--reporter=junit` output. */ +export function parseJUnit(xml: string): JUnitCase[] { + if (!/^<\?xml[^>]*\?>\s*]*?)(?:\/>|>([\s\S]*?)<\/testcase>)/g)) { + const attributes = Object.fromEntries([...match[1].matchAll(/([A-Za-z_:][\w:.-]*)="([^"]*)"/g)].map(([, key, value]) => [key, decode(value)])); + const body = match[2] ?? ''; + const status = /<(?:failure|error)\b/.test(body) ? 'failed' : / { + const clean = cases.some(item => item.status === 'passed') && !cases.some(item => item.status === 'failed'); + const passed = (ref: TestRef) => { + const matches = cases.filter(item => item.file === ref.file && item.name === ref.name); + return matches.length === 1 && matches[0].status === 'passed' && matches[0].assertions > 0; + }; + return Object.fromEntries(Object.entries(nativeTestRequirements(stack)) + .map(([key, refs]) => [key, clean && refs.every(passed)])); +} + +/** Native evidence for one staged row; private checks are every required release gate the native job cannot prove. */ +export function nativeEvidence(staged: unknown, stack: Stack, platform: string, report: string): Record { + const record = staged as Record; + if (!record || typeof record !== 'object' || record.state !== 'staged' || record.stack !== stack || record.platform !== platform) { + throw new Error('STAGED_EVIDENCE_MISMATCH'); + } + const required = requiredChecks(stack); + const nativeChecks = deriveNativeChecks(stack, parseJUnit(report)); + const failed = Object.entries(nativeChecks).filter(([key, value]) => !value || !required.includes(key)).map(([key]) => key); + if (failed.length) throw new Error(`NATIVE_GATE_NOT_PASSED: ${failed.sort().join(', ')}`); + return { + ...record, + state: 'native-gates-passed', + nativeChecks, + nativeTestReport: `sha256:${sha256(report)}`, + privateChecks: Object.fromEntries(required.filter(key => !(key in nativeChecks)).map(key => [key, 'pending'])), + qualified: false, + promotion: 'prohibited until authenticated private qualification evidence completes every release gate', + }; +} + +function readFile(file: string): string { + const stat = fs.lstatSync(path.resolve(file)); + if (!stat.isFile() || stat.isSymbolicLink() || stat.size <= 0 || stat.size > MAX_REPORT_BYTES) throw new Error('UNSAFE_NATIVE_EVIDENCE_INPUT'); + return fs.readFileSync(path.resolve(file), 'utf8'); +} + +if (import.meta.main) { + try { + const args = process.argv.slice(2); + const names = ['--stack', '--platform', '--junit', '--staged', '--output']; + if (args.length !== names.length * 2 || names.some((name, index) => args[index * 2] !== name)) { + throw new Error('Usage: cso-native-evidence.ts --stack STACK --platform PLATFORM --junit REPORT.xml --staged staged-image.json --output EVIDENCE.json'); + } + const [stack, platform, junit, staged, output] = names.map((_, index) => args[index * 2 + 1]); + if (!['node', 'bun', 'python', 'rails', 'postgresql'].includes(stack)) throw new Error('INVALID_STACK'); + const evidence = nativeEvidence(JSON.parse(readFile(staged)), stack as Stack, platform, readFile(junit)); + fs.writeFileSync(path.resolve(output), JSON.stringify(evidence, null, 2) + '\n', { flag: 'wx', mode: 0o600 }); + } catch (error) { + process.stderr.write((error instanceof Error ? error.message : 'NATIVE_EVIDENCE_FAILED') + '\n'); + process.exitCode = 1; + } +} diff --git a/scripts/cso-requalification.ts b/scripts/cso-requalification.ts new file mode 100644 index 000000000..d64110687 --- /dev/null +++ b/scripts/cso-requalification.ts @@ -0,0 +1,68 @@ +#!/usr/bin/env bun +/** + * Requalification triggers: the inputs a runtime qualification was measured + * under. A promoted catalog records them; a catalog whose recorded triggers + * differ from the current source is older than its inputs and must be + * requalified (or its runtimes withdrawn) before it ships. + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { canonical, sha256 } from '../lib/cso/contracts'; +import { ISOLATION_POLICY_HASH } from '../lib/cso/docker'; +import { CSO_HELPER_ABI, isEvaluationRuntimeCatalog, type RequalificationTriggers, type RuntimeCatalog } from '../lib/cso/runtime-catalog'; + +const ROOT = path.resolve(import.meta.dir, '..'); +/** Compiled into every runtime image as gstack-cso-verifier and gstack-cso-preparation. */ +export const IMAGE_HELPER_ENTRYPOINTS = ['lib/cso/verifier.ts', 'lib/cso/preparation-container.ts'] as const; +export const IMAGE_CONTEXT = 'lib/cso/images'; +/** Image-context files that describe qualification rather than enter an image; build-inputs.json is its own trigger. */ +const IMAGE_CONTEXT_EXCLUDED = new Set(['README.md', 'qualification.json', 'build-inputs.json']); + +/** Every repository file whose bytes enter a runtime image: the helpers' import closure plus the image build context. */ +export async function imageSourceFiles(root = ROOT): Promise { + const build = await Bun.build({ entrypoints: IMAGE_HELPER_ENTRYPOINTS.map(entry => path.join(root, entry)), target: 'bun', metafile: true }); + if (!build.success || !build.metafile) throw new Error('REQUALIFICATION_SOURCE_CLOSURE_FAILED'); + // Bun reports metafile inputs relative to the process working directory, not to the entrypoints. + const closure = Object.keys(build.metafile.inputs).map(input => path.relative(root, path.resolve(process.cwd(), input)).split(path.sep).join('/')); + if (closure.some(file => file.startsWith('..') || file.includes('node_modules/'))) throw new Error('REQUALIFICATION_SOURCE_OUTSIDE_REPOSITORY'); + const context = fs.readdirSync(path.join(root, IMAGE_CONTEXT), { withFileTypes: true }) + .filter(entry => entry.isFile() && !IMAGE_CONTEXT_EXCLUDED.has(entry.name)) + .map(entry => `${IMAGE_CONTEXT}/${entry.name}`); + return [...new Set([...closure, ...context])].sort(); +} + +export async function requalificationTriggers(root = ROOT): Promise { + const files = await imageSourceFiles(root); + const buildInputs = JSON.parse(fs.readFileSync(path.join(root, IMAGE_CONTEXT, 'build-inputs.json'), 'utf8')); + if (typeof buildInputs?.revision !== 'string' || !buildInputs.revision) throw new Error('INVALID_BUILD_INPUTS_REVISION'); + return { + helperAbi: CSO_HELPER_ABI, + isolationPolicyHash: ISOLATION_POLICY_HASH, + preparationSha256: sha256(canonical(files.map(file => [file, sha256(fs.readFileSync(path.join(root, file)))]))), + buildInputsRevision: buildInputs.revision, + }; +} + +/** Names of the triggers that changed since the catalog's runtimes were qualified; empty when nothing is promoted. */ +export function staleRequalificationTriggers(catalog: RuntimeCatalog, current: RequalificationTriggers): string[] { + if (isEvaluationRuntimeCatalog(catalog) || catalog.runtimes.length === 0) return []; + const recorded = catalog.promotion?.requalification; + if (!recorded) return Object.keys(current).sort(); + return (Object.keys(current) as Array).filter(key => recorded[key] !== current[key]).sort(); +} + +if (import.meta.main) { + try { + const [command, file, ...rest] = process.argv.slice(2); + if (command === 'triggers' && file === undefined) { + process.stdout.write(JSON.stringify(await requalificationTriggers(), null, 2) + '\n'); + } else if (command === 'check' && file && rest.length === 0) { + const stale = staleRequalificationTriggers(JSON.parse(fs.readFileSync(path.resolve(file), 'utf8')), await requalificationTriggers()); + if (stale.length) throw new Error(`STALE_RUNTIME_QUALIFICATION: ${stale.join(', ')} changed since qualification; requalify the runtimes or withdraw them from the catalog`); + process.stdout.write('REQUALIFICATION CURRENT\n'); + } else throw new Error('Usage: cso-requalification.ts triggers | check '); + } catch (error) { + process.stderr.write((error instanceof Error ? error.message : 'REQUALIFICATION_CHECK_FAILED') + '\n'); + process.exitCode = 1; + } +} diff --git a/scripts/cso-runtime-promotion.ts b/scripts/cso-runtime-promotion.ts index e576bd182..4ba5c2e7d 100644 --- a/scripts/cso-runtime-promotion.ts +++ b/scripts/cso-runtime-promotion.ts @@ -7,12 +7,18 @@ import committedCatalog from '../lib/cso/runtime-catalog.json'; import buildInputs from '../lib/cso/images/build-inputs.json'; import { CSO_HELPER_ABI, + EVALUATION_REVISION_PREFIX, + isEvaluationRuntimeCatalog, + validRequalificationTriggers, + type RequalificationTriggers, type QualifiedRuntime, type RuntimeCatalog, type RuntimeQualification, validateRuntimeCatalog, } from '../lib/cso/runtime-catalog'; +import { ISOLATION_POLICY_HASH } from '../lib/cso/docker'; import { imageBuildMatrix, type ImageBuildRow } from './cso-image-matrix'; +import { requalificationTriggers, staleRequalificationTriggers } from './cso-requalification'; const DIGEST = /^sha256:[a-f0-9]{64}$/; const WORKFLOW = /^https:\/\/github\.com\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+\/actions\/runs\/[0-9]+$/; @@ -46,7 +52,7 @@ function versionsKey(value: Record): string { return canonical(value); } -function requiredChecks(stack: ImageBuildRow['stack']): string[] { +export function requiredChecks(stack: ImageBuildRow['stack']): string[] { if (stack === 'postgresql') return [ 'containmentPassed', 'coldStartPassed', 'multiDatabasePassed', 'readinessPassed', 'secretCanaryPassed', 'watchdogCleanupPassed', @@ -112,13 +118,15 @@ function validateStatement(raw: unknown, row: ImageBuildRow): RuntimeQualificati return statement; } -export function catalogPromotionCandidate( - currentValue: unknown, - inputValue: unknown, - evidenceValues: unknown[], -): RuntimeCatalog { +/** The committed (never evaluation-only) catalog and the reviewed build matrix it must describe. */ +function reviewedMatrix(currentValue: unknown, inputValue: unknown): { current: RuntimeCatalog; rows: ImageBuildRow[] } { validateRuntimeCatalog(currentValue); const current = currentValue as RuntimeCatalog; + if (isEvaluationRuntimeCatalog(current)) throw new Error('EVALUATION_CATALOG_NOT_PROMOTABLE'); + return { current, rows: profilesMatchingRows(current, inputValue) }; +} + +function profilesMatchingRows(current: RuntimeCatalog, inputValue: unknown): ImageBuildRow[] { const rows = imageBuildMatrix(inputValue).include; if (!current.profiles || current.buildRevision !== rows[0].inputRevision) throw new Error('CATALOG_BUILD_REVISION_MISMATCH'); const profiles = new Map(current.profiles.map(profile => [profile.id, profile])); @@ -128,17 +136,100 @@ export function catalogPromotionCandidate( if (!profile || profile.stack !== row.stack || profile.platform !== row.platform || versionsKey(profile.versions) !== versionsKey(row.versions)) throw new Error(`CATALOG_PROFILE_MATRIX_MISMATCH: ${row.runtimeId}`); } - if (evidenceValues.length !== rows.length) throw new Error('INCOMPLETE_QUALIFICATION_MATRIX'); + return rows; +} + +/** The candidate build's gate: a valid evaluation-only catalog for the reviewed build inputs. */ +export function validateEvaluationCandidate(catalogValue: unknown, inputValue: unknown): RuntimeCatalog { + validateRuntimeCatalog(catalogValue); + const catalog = catalogValue as RuntimeCatalog; + if (!isEvaluationRuntimeCatalog(catalog)) throw new Error('NOT_AN_EVALUATION_CATALOG'); + const rows = profilesMatchingRows(catalog, inputValue); + for (const row of rows) { + const runtime = catalog.runtimes.find(item => item.id === row.runtimeId); + if (!runtime || !OUTPUT_IMAGE.test(runtime.image) || !runtime.image.includes(`/${row.stack}-${row.arch}@`)) throw new Error(`INVALID_QUALIFIED_IMAGE: ${row.runtimeId}`); + } + return catalog; +} + +function matchRows(rows: ImageBuildRow[], values: unknown[]): Array<[ImageBuildRow, unknown]> { + if (values.length !== rows.length) throw new Error('INCOMPLETE_QUALIFICATION_MATRIX'); const byId = new Map(); - for (const value of evidenceValues) { + for (const value of values) { const id = (value as any)?.runtimeId; if (typeof id !== 'string' || byId.has(id)) throw new Error('DUPLICATE_QUALIFICATION_STATEMENT'); byId.set(id, value); } - const statements = rows.map(row => { + return rows.map(row => { if (!byId.has(row.runtimeId)) throw new Error(`MISSING_QUALIFICATION_STATEMENT: ${row.runtimeId}`); - return validateStatement(byId.get(row.runtimeId), row); + return [row, byId.get(row.runtimeId)]; }); +} + +const STAGED_FIELDS = [ + 'buildRevision', 'helperAbi', 'image', 'platform', 'provenanceDigest', 'qualification', 'runtimeId', + 'sbomDigest', 'schemaVersion', 'sourceCommit', 'stack', 'state', 'verifiedProvenance', 'versions', 'workflow', +].sort(); + +/** + * Build an evaluation-only catalog from one staging run's staged-image.json + * records. It measures no release gate: every runtime is an + * evaluation_candidate, and setup, distribution builds and promotion refuse it. + */ +export function evaluationCandidateCatalog(currentValue: unknown, inputValue: unknown, stagedValues: unknown[], createdAt: string): RuntimeCatalog { + const { current, rows } = reviewedMatrix(currentValue, inputValue); + if (!Number.isFinite(Date.parse(createdAt))) throw new Error('INVALID_EVALUATION_TIME'); + const staged = matchRows(rows, stagedValues).map(([row, value]) => { + const record = value as RuntimeQualificationStatement; + if (!record || typeof record !== 'object' || canonical(Object.keys(record).sort()) !== canonical(STAGED_FIELDS) || + record.schemaVersion !== 1 || record.helperAbi !== CSO_HELPER_ABI || (record.state as string) !== 'staged' || (record as any).verifiedProvenance !== true) throw new Error(`INVALID_STAGED_RUNTIME: ${row.runtimeId}`); + if (record.buildRevision !== row.inputRevision || record.stack !== row.stack || record.platform !== row.platform || + versionsKey(record.versions) !== versionsKey(row.versions)) throw new Error(`QUALIFICATION_BUILD_MISMATCH: ${row.runtimeId}`); + if (!OUTPUT_IMAGE.test(record.image) || !record.image.includes(`/${row.stack}-${row.arch}@`)) throw new Error(`INVALID_QUALIFIED_IMAGE: ${row.runtimeId}`); + if (!/^[a-f0-9]{40}$/.test(record.sourceCommit) || !WORKFLOW.test(record.workflow) || !DIGEST.test(record.sbomDigest) || + !DIGEST.test(record.provenanceDigest)) throw new Error(`MISSING_QUALIFICATION_PROVENANCE: ${row.runtimeId}`); + return record; + }); + if (new Set(staged.map(item => item.sourceCommit)).size !== 1 || new Set(staged.map(item => item.workflow)).size !== 1) throw new Error('SPLIT_STAGING_RUN'); + const { sourceCommit, workflow } = staged[0]; + const runtimes: QualifiedRuntime[] = staged.map(record => ({ + id: record.runtimeId, + stack: record.stack, + platform: record.platform, + state: 'evaluation_candidate', + image: record.image, + entrypoint: '/opt/cso/entrypoint', + helperAbi: CSO_HELPER_ABI, + versions: record.versions, + policyVersion: 'cso-isolation-v1', + qualifiedAt: createdAt, + qualification: { kind: 'evaluation', sourceCommit, workflow, sbomDigest: record.sbomDigest, provenanceDigest: record.provenanceDigest, verifiedProvenance: true }, + })); + const candidate: RuntimeCatalog = { + schemaVersion: 1, + revision: `${EVALUATION_REVISION_PREFIX}${workflow.slice(workflow.lastIndexOf('/') + 1)}-${sourceCommit.slice(0, 12)}`, + previousRevision: current.revision, + helperAbi: CSO_HELPER_ABI, + buildRevision: current.buildRevision, + profiles: current.profiles, + evaluation: { sourceCommit, workflow, evidenceDigest: sha256(canonical(runtimes)) }, + runtimes, + }; + validateRuntimeCatalog(candidate); + return candidate; +} + +export function catalogPromotionCandidate( + currentValue: unknown, + inputValue: unknown, + evidenceValues: unknown[], + /** Measured by the staging run at sourceCommit (scripts/cso-requalification.ts triggers). */ + requalification: RequalificationTriggers, +): RuntimeCatalog { + const { current, rows } = reviewedMatrix(currentValue, inputValue); + if (!validRequalificationTriggers(requalification, current.buildRevision)) throw new Error('INVALID_REQUALIFICATION_TRIGGERS'); + if (requalification.isolationPolicyHash !== ISOLATION_POLICY_HASH) throw new Error('STALE_RUNTIME_QUALIFICATION: isolationPolicyHash'); + const statements = matchRows(rows, evidenceValues).map(([row, value]) => validateStatement(value, row)); const sourceCommits = new Set(statements.map(item => item.sourceCommit)); const workflows = new Set(statements.map(item => item.workflow)); if (sourceCommits.size !== 1 || workflows.size !== 1) throw new Error('SPLIT_QUALIFICATION_RUN'); @@ -167,7 +258,15 @@ export function catalogPromotionCandidate( helperAbi: CSO_HELPER_ABI, buildRevision: current.buildRevision, profiles: current.profiles, - promotion: { sourceCommit, workflow, evidenceDigest, qualificationEvidenceDigest }, + promotion: { + sourceCommit, workflow, evidenceDigest, qualificationEvidenceDigest, + requalification: { + helperAbi: requalification.helperAbi, + isolationPolicyHash: requalification.isolationPolicyHash, + preparationSha256: requalification.preparationSha256, + buildInputsRevision: requalification.buildInputsRevision, + }, + }, runtimes, }; validateRuntimeCatalog(candidate); @@ -180,6 +279,7 @@ export function validateRuntimeCatalogTransition(currentValue: unknown, proposed validateRuntimeCatalog(proposedValue); const current = currentValue as RuntimeCatalog; const proposed = proposedValue as RuntimeCatalog; + if (isEvaluationRuntimeCatalog(current) || isEvaluationRuntimeCatalog(proposed)) throw new Error('EVALUATION_CATALOG_NOT_PROMOTABLE'); if (!proposed.promotion || proposed.runtimes.length === 0) throw new Error('UNQUALIFIED_RUNTIME_CATALOG_PROMOTION'); if (proposed.revision === current.revision || proposed.previousRevision !== current.revision) { throw new Error('RUNTIME_CATALOG_BASE_REVISION_MISMATCH'); @@ -190,7 +290,7 @@ export function validateRuntimeCatalogTransition(currentValue: unknown, proposed } } -function collectEvidence(root: string): unknown[] { +function collectEvidence(root: string, name = 'qualified-runtime.json'): unknown[] { const absolute = path.resolve(root); const rootStat = fs.lstatSync(absolute); if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) throw new Error('UNSAFE_EVIDENCE_ROOT'); @@ -201,7 +301,7 @@ function collectEvidence(root: string): unknown[] { const file = path.join(directory, entry.name); if (entry.isSymbolicLink() || (!entry.isDirectory() && !entry.isFile())) throw new Error('UNSAFE_EVIDENCE_ENTRY'); if (entry.isDirectory()) visit(file, depth + 1); - else if (entry.name === 'qualified-runtime.json') files.push(file); + else if (entry.name === name) files.push(file); if (files.length > MAX_EVIDENCE_FILES) throw new Error('TOO_MANY_QUALIFICATION_STATEMENTS'); } }; @@ -234,7 +334,19 @@ function readCatalogArtifact(file: string): unknown { if (import.meta.main) { try { const args = process.argv.slice(2); - if (args[0] === 'validate-transition') { + if (args[0] === 'check-evaluation') { + if (args.length !== 2) throw new Error('Usage: cso-runtime-promotion.ts check-evaluation '); + const catalog = validateEvaluationCandidate(readCatalogArtifact(args[1]), buildInputs); + process.stdout.write(`EVALUATION-ONLY ${catalog.revision}\n`); + } else if (args[0] === 'evaluation-candidate') { + if (args.length !== 5 || args[1] !== '--staged-root' || args[3] !== '--output') { + throw new Error('Usage: cso-runtime-promotion.ts evaluation-candidate --staged-root --output '); + } + const candidate = evaluationCandidateCatalog(committedCatalog, buildInputs, collectEvidence(args[2], 'staged-image.json'), new Date().toISOString()); + const output = path.resolve(args[4]); + fs.writeFileSync(output, JSON.stringify(candidate, null, 2) + '\n', { flag: 'wx', mode: 0o600 }); + process.stdout.write(`${output}\n`); + } else if (args[0] === 'validate-transition') { if (args.length !== 3) throw new Error('Usage: cso-runtime-promotion.ts validate-transition '); validateRuntimeCatalogTransition(readCatalogArtifact(args[1]), readCatalogArtifact(args[2])); process.stdout.write('VALID TRANSITION\n'); @@ -242,7 +354,11 @@ if (import.meta.main) { if (args.length !== 4 || args[0] !== '--evidence-root' || args[2] !== '--output') { throw new Error('Usage: cso-runtime-promotion.ts --evidence-root --output '); } - const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, collectEvidence(args[1])); + const triggers = collectEvidence(args[1], 'requalification.json'); + if (triggers.length !== 1) throw new Error('MISSING_REQUALIFICATION_TRIGGERS'); + const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, collectEvidence(args[1]), triggers[0] as RequalificationTriggers); + const stale = staleRequalificationTriggers(candidate, await requalificationTriggers()); + if (stale.length) throw new Error(`STALE_RUNTIME_QUALIFICATION: ${stale.join(', ')} changed since the staged commit; re-stage and requalify`); const output = path.resolve(args[3]); fs.writeFileSync(output, JSON.stringify(candidate, null, 2) + '\n', { flag: 'wx', mode: 0o600 }); process.stdout.write(`${output}\n`); diff --git a/scripts/cso-staging-run.ts b/scripts/cso-staging-run.ts new file mode 100644 index 000000000..d22713e86 --- /dev/null +++ b/scripts/cso-staging-run.ts @@ -0,0 +1,124 @@ +#!/usr/bin/env bun +/** Bind a qualification dispatch to the one successful protected-main staging run that signed its images. */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { canonical } from '../lib/cso/contracts'; +import { committedImageBuildMatrix } from './cso-image-matrix'; + +export const STAGING_WORKFLOW_PATH = '.github/workflows/cso-runtime-images.yml'; +export const NATIVE_JOB = 'qualify-native'; +const SLSA_PROVENANCE = 'https://slsa.dev/provenance/v1'; +const REPOSITORY = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; +const INVOCATION = /^https:\/\/github\.com\/([A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+)\/actions\/runs\/([1-9][0-9]*)\/attempts\/[1-9][0-9]*$/; +const MAX_FILE_BYTES = 4 * 1024 * 1024; + +export interface StagingRunEvidence { + repository: string; + runId: string; + sourceCommit: string; + runtimeIds: string[]; + run: unknown; + jobPages: unknown; +} + +/** + * Resolve the staging run from verified SLSA provenance. The predicate's + * invocation and the Fulcio certificate's run URI must agree, and every + * verified statement for every image must name the same run. + */ +export function stagingRunId(repository: string, verifications: unknown[]): string { + if (!REPOSITORY.test(repository) || verifications.length === 0) throw new Error('INVALID_STAGING_PROVENANCE'); + const runs = new Set(); + for (const verification of verifications) { + if (!Array.isArray(verification) || verification.length === 0) throw new Error('INVALID_STAGING_PROVENANCE'); + for (const item of verification) { + const result = (item as any)?.verificationResult; + if (result?.statement?.predicateType !== SLSA_PROVENANCE) throw new Error('INVALID_STAGING_PROVENANCE'); + const invocation = result.statement.predicate?.runDetails?.metadata?.invocationId; + const match = typeof invocation === 'string' ? INVOCATION.exec(invocation) : null; + if (!match || match[1] !== repository || result.signature?.certificate?.runInvocationURI !== invocation) { + throw new Error('STAGING_INVOCATION_MISMATCH'); + } + runs.add(match[2]); + } + } + if (runs.size !== 1) throw new Error('STAGING_RUN_NOT_UNIQUE'); + return [...runs][0]; +} + +/** Every statement in one qualification dispatch names the same staged source commit. */ +export function dispatchSourceCommit(event: unknown): string { + const statements = (event as any)?.client_payload?.statements; + if (!Array.isArray(statements) || statements.length === 0) throw new Error('INVALID_QUALIFICATION_DISPATCH'); + const commits = new Set(statements.map((statement: any) => statement?.sourceCommit)); + const [commit] = [...commits]; + if (commits.size !== 1 || typeof commit !== 'string' || !/^[a-f0-9]{40}$/.test(commit)) throw new Error('STAGING_SOURCE_COMMIT_MISMATCH'); + return commit; +} + +/** The resolved run must be the protected-main staging dispatch at the statements' commit, with every native gate passed. */ +export function verifyStagingRun(evidence: StagingRunEvidence): void { + const { repository, runId, sourceCommit, runtimeIds } = evidence; + const run = evidence.run as any; + if (!run || typeof run !== 'object' || String(run.id) !== runId) throw new Error('STAGING_RUN_ID_MISMATCH'); + if (run.path !== STAGING_WORKFLOW_PATH) throw new Error('STAGING_WORKFLOW_MISMATCH'); + if (run.repository?.full_name !== repository || run.head_repository?.full_name !== repository) throw new Error('STAGING_REPOSITORY_MISMATCH'); + if (run.event !== 'workflow_dispatch' || run.head_branch !== 'main') throw new Error('STAGING_RUN_NOT_PROTECTED_MAIN_DISPATCH'); + if (run.status !== 'completed' || run.conclusion !== 'success') throw new Error('STAGING_RUN_NOT_SUCCESSFUL'); + if (run.head_sha !== sourceCommit) throw new Error('STAGING_SOURCE_COMMIT_MISMATCH'); + + const pages = evidence.jobPages; + if (!Array.isArray(pages) || pages.length === 0) throw new Error('INVALID_STAGING_JOBS'); + const totals = new Set(pages.map((page: any) => page?.total_count)); + if (pages.some((page: any) => !Array.isArray(page?.jobs)) || totals.size !== 1) throw new Error('INVALID_STAGING_JOBS'); + const jobs = pages.flatMap((page: any) => page.jobs); + if (jobs.length !== [...totals][0]) throw new Error('INCOMPLETE_STAGING_JOBS'); + + const native = jobs.filter((job: any) => typeof job?.name === 'string' && job.name.startsWith(NATIVE_JOB)); + const expected = runtimeIds.map(id => `${NATIVE_JOB} ${id}`).sort(); + if (runtimeIds.length === 0 || canonical(native.map((job: any) => job.name).sort()) !== canonical(expected)) { + throw new Error('STAGING_NATIVE_JOBS_MISMATCH'); + } + if (native.some((job: any) => String(job.run_id) !== runId || job.status !== 'completed' || job.conclusion !== 'success')) { + throw new Error('STAGING_NATIVE_GATE_NOT_PASSED'); + } +} + +function readJson(file: string): unknown { + const absolute = path.resolve(file); + const stat = fs.lstatSync(absolute); + if (!stat.isFile() || stat.isSymbolicLink() || stat.size <= 0 || stat.size > MAX_FILE_BYTES) throw new Error('UNSAFE_STAGING_EVIDENCE_FILE'); + try { return JSON.parse(fs.readFileSync(absolute, 'utf8')); } catch { throw new Error('INVALID_STAGING_EVIDENCE_FILE'); } +} + +function option(args: string[], name: string): string { + const index = args.indexOf(name); + const value = index >= 0 ? args[index + 1] : undefined; + if (!value || value.startsWith('--')) throw new Error(`Missing ${name}`); + return value; +} + +if (import.meta.main) { + try { + const [command, ...args] = process.argv.slice(2); + if (command === 'run-id' && args[0] === '--repository' && args.length >= 3) { + process.stdout.write(`${stagingRunId(args[1], args.slice(2).map(readJson))}\n`); + } else if (command === 'verify' && args.length === 10) { + const event = readJson(option(args, '--event')); + verifyStagingRun({ + repository: option(args, '--repository'), + runId: option(args, '--run-id'), + sourceCommit: dispatchSourceCommit(event), + runtimeIds: committedImageBuildMatrix().include.map(row => row.runtimeId), + run: readJson(option(args, '--run')), + jobPages: readJson(option(args, '--jobs')), + }); + process.stdout.write('STAGING RUN VERIFIED\n'); + } else { + throw new Error('Usage: cso-staging-run.ts run-id --repository OWNER/REPO PROVENANCE.json... | verify --repository OWNER/REPO --run-id ID --event EVENT.json --run RUN.json --jobs JOB-PAGES.json'); + } + } catch (error) { + process.stderr.write((error instanceof Error ? error.message : 'STAGING_RUN_VERIFICATION_FAILED') + '\n'); + process.exitCode = 1; + } +} diff --git a/setup b/setup index 70947d524..84c67a3e1 100755 --- a/setup +++ b/setup @@ -1806,6 +1806,12 @@ if [ "$CSO_BUILD_AVAILABLE" -eq 1 ]; then echo "gstack setup failed: CSO build completed without its required native artifact set" >&2 exit 1 fi + # Evaluation-only helpers embed an unqualified cso-eval-* catalog for the + # private evaluator. Setup never installs one, whether marked or copied bare. + if [ -e "$SOURCE_GSTACK_DIR/bin/.gstack-cso-evaluation" ] || "$SOURCE_GSTACK_DIR/bin/gstack-cso-launcher$_EXE" --version 2>/dev/null | grep -q '"evaluationOnly": *true'; then + echo "gstack setup failed: bin/ holds an evaluation-only CSO helper; remove the bin/gstack-cso-* artifacts and re-run setup" >&2 + exit 1 + fi # Runtime/scanner execution never pulls. Setup is the sole automatic # acquisition path: the trusted launcher validates committed catalogs, # rejects remote Docker contexts, and uses an empty Docker config for diff --git a/test/cso-contracts.test.ts b/test/cso-contracts.test.ts index be474ceda..e9513eddd 100644 --- a/test/cso-contracts.test.ts +++ b/test/cso-contracts.test.ts @@ -4,7 +4,7 @@ import * as os from 'node:os'; import * as path from 'node:path'; import * as crypto from 'node:crypto'; import { canonical, completeness, CsoError, fingerprint, importLegacy, renderReport, sha256, snapshotPathHandle, snapshotPathId, validateCoverage, validateFinding, validateVerificationObservation, validateVerificationRequest } from '../lib/cso/contracts'; -import { assertCanonicalStartPlan, canonicalStartPlan, canonicalTestPlan, certify, fileEffect, makeReviewArtifact, patchHash, preparePatchedSource, resolveVerificationRequestPaths, testExecutionPassed, treeHash, validateRepairBundle, validateReviewArtifact, verificationIdentity, verifyRepair } from '../lib/cso/verification'; +import { assertCanonicalStartPlan, canonicalStartPlan, canonicalTestPlan, certify, fileEffect, makeReviewArtifact, patchHash, preparePatchedSource, resolveVerificationRequestPaths, testExecutionPassed, treeHash, validateRepairBundle, validateReviewArtifact, verificationHarnessHash, verificationIdentity, verifyRepair } from '../lib/cso/verification'; import { assertionWitnessPairHash, witnessObservationHash } from '../lib/cso/witness'; import { admit, machinePoolRoot, markSupervised, release } from '../lib/cso/admission'; import { sanitizeHelperForJson } from '../lib/cso/process'; @@ -211,6 +211,50 @@ describe('tested repair certificate gate',()=>{ const drift=structuredClone(bundle);drift.preparation!.after.preparedDependencyHash=h('b');drift.verification.preparationHash=sha256(canonical(drift.preparation));drift.verification.id='';const rebound=verificationIdentity(drift.verification);drift.verification.id=rebound;drift.id=rebound;expect(()=>validateRepairBundle(drift,rebound,before)).toThrow('test toolchain changed'); expect(()=>certify({...params,preparation:undefined})).toThrow('require before/after prepared dependency proofs');expect(()=>certify({...params,preparation:{before:proof(h('9')),after:proof(h('b'))}})).toThrow('toolchain bytes changed'); }); + const securityArray=()=>[ + {name:'tenant isolation',path:'/user?id=2',method:'GET',expected:{status:403},vulnerable:{status:200,includes:'tenant-b'}}, + {name:'tenant isolation by header',path:'/user',method:'GET',headers:{'x-tenant':'b'},expected:{status:403},vulnerable:{status:200,includes:'tenant-b'}}, + {name:'tenant isolation on write',path:'/user/2',method:'PUT',body:'{"name":"x"}',expected:{status:404},vulnerable:{status:204}}, + ]; + test('single-object security keeps its pre-array request, patch, assertion, and harness bytes',()=>{ + const root=tmp();fs.writeFileSync(path.join(root,'test.js'),'pass\n');fs.chmodSync(path.join(root,'test.js'),0o644); + const raw:any=request('vulnerable\n','fixed\n');raw.review.reviewedPatchHash=patchHash(raw);const req=validateVerificationRequest(raw); + expect(Array.isArray(req.security)).toBe(false);expect(canonical(req.security)).toBe(canonical(raw.security)); + expect({patch:patchHash(req),request:sha256(canonical(req)),assertion:sha256(canonical({legitimate:req.legitimate,security:req.security})),harness:verificationHarnessHash(req,root)}).toEqual({ + patch:'cc6ab00ee0160ce484a86130f94a5e6e5f588f3ec975c781c4d077647adff483', + request:'85175d755f1b297eb7d15dc1d9850e1ac453d1d129ceee58213d89cbab0ee59c', + assertion:'7e0f792881bc14137f2aa70b0662a013939749903e49eb507c1ecf48561b0771', + harness:'2a74b61982270fd10c071f0bdd32ae37f04de6e3eca1db6c9225132de29b4309', + }); + }); + test('security accepts a bounded array of distinct assertions that each carry an exclusive vulnerable oracle',()=>{ + const make=(security:unknown)=>{const raw:any=request('vulnerable\n','fixed\n');raw.security=security;raw.review.reviewedPatchHash=patchHash(raw);return raw;}; + const accepted=validateVerificationRequest(make(securityArray()));expect(accepted.security).toEqual(securityArray() as any); + expect(validateVerificationRequest(make(securityArray().slice(0,1))).security).toHaveLength(1); + const full=Array.from({length:8},(_,i)=>({...securityArray()[0],name:`payload ${i}`}));expect(validateVerificationRequest(make(full)).security).toHaveLength(8); + expect(()=>validateVerificationRequest(make([]))).toThrow('1..8 assertions'); + expect(()=>validateVerificationRequest(make([...full,{...full[0],name:'payload 8'}]))).toThrow('1..8 assertions'); + expect(()=>validateVerificationRequest(make([securityArray()[0],{...securityArray()[1],name:'tenant isolation'}]))).toThrow('names must be distinct'); + const missing=securityArray();delete (missing[2] as any).vulnerable;expect(()=>validateVerificationRequest(make(missing))).toThrow('distinct before/fixed security oracles'); + const overlapping=securityArray();overlapping[1].vulnerable={...overlapping[1].expected} as any;expect(()=>validateVerificationRequest(make(overlapping))).toThrow('mutually exclusive'); + const extra:any=securityArray();extra[0].expected.untrusted='yes';expect(()=>validateVerificationRequest(make(extra))).toThrow('Unexpected security[0].expected field'); + expect(()=>validateVerificationRequest(make([null]))).toThrow(); + }); + test('an array security request is observed in exactly one before/after pair and binds every member',async()=>{ + const runDir=tmp(),snapshot=path.join(runDir,'snapshot');fs.mkdirSync(snapshot);fs.writeFileSync(path.join(snapshot,'app.js'),'vulnerable\n');const testBody='test("ok",()=>{})\n';fs.writeFileSync(path.join(snapshot,'app.test.js'),testBody);const packageJson='{"scripts":{"test":"node --test"}}';fs.writeFileSync(path.join(snapshot,'package.json'),packageJson); + const raw:any=request('vulnerable\n','fixed\n');raw.findingId='1abcdefabcdefabcdefabcdefabcdefa';raw.security=securityArray();raw.existingTests=[{executable:'/usr/local/bin/node',args:['--test','--test-reporter=tap','./app.test.js']}];raw.testFiles=['app.test.js'];raw.review.reviewedPatchHash=patchHash(raw); + const manifest:any={version:3,createdAt:'2026-01-01T00:00:00Z',expiresAt:'2026-01-08T00:00:00Z',root:'/repo',headCommit:'a'.repeat(40),originalHash:'b'.repeat(64),executionHash:treeHash(snapshot),entries:[{path:'app.js',originalHash:sha256('vulnerable\n'),executionHash:sha256('vulnerable\n'),bytes:11,mode:0o600},{path:'app.test.js',originalHash:sha256(testBody),executionHash:sha256(testBody),bytes:Buffer.byteLength(testBody),mode:0o600},{path:'package.json',originalHash:sha256(packageJson),executionHash:sha256(packageJson),bytes:packageJson.length,mode:0o600}]}; + const runtime:any={id:'node',stack:'node',image:'runtime@sha256:'+'a'.repeat(64),platform:'linux/amd64'},phases:string[]=[]; + const executor:any={observe:async(_source:string,phase:string,observed:any)=>{phases.push(phase);expect(observed.security).toEqual(securityArray());return{booted:true,legitimate:true,security:phase==='before'?'intended_failure':'pass',existingTests:true,output:'ok',inputHash:''};}}; + let unattested:any;try{await verifyRepair({runId:'run',runDir,manifest,rawRequest:raw,runtime,verifier:runtime,policyHash:'c'.repeat(64),archives:[],executor,persist:false});}catch(error){unattested=error;} + expect(phases).toEqual(['before','after']);expect(unattested).toMatchObject({code:'PREREQUISITE',attempt:{reproduction:'reproduced',repair:'proposed',bundleIssued:false}}); + const root=tmp(),before=path.join(root,'before'),after=path.join(root,'after');fs.mkdirSync(before);fs.writeFileSync(path.join(before,'app.js'),'vulnerable\n');fs.writeFileSync(path.join(before,'test.js'),'pass\n'); + const plain:any=request('vulnerable\n','fixed\n');plain.security=securityArray();plain.review.reviewedPatchHash=patchHash(plain);const req=validateVerificationRequest(plain);preparePatchedSource(before,after,req); + const params:any={runId:'run',manifest:{originalHash:'d'.repeat(64),executionHash:treeHash(before),entries:[]},request:req,runtime:{id:'node',image:'runtime@sha256:'+'a'.repeat(64),platform:'linux/amd64'},verifier:{image:'runtime@sha256:'+'a'.repeat(64)},before:{booted:true,legitimate:true,security:'intended_failure',existingTests:true,output:'before',inputHash:''},after:{booted:true,legitimate:true,security:'pass',existingTests:true,output:'after',inputHash:''},beforeRoot:before,afterRoot:after,policyHash:'c'.repeat(64),auditPolicyHash:'e'.repeat(64),archives:[],testToolchain:'runtime'}; + const bundle=authenticatedBundleFixture(certify(params),params);expect(validateRepairBundle(bundle,bundle.id,before).id).toBe(bundle.id); + const dropped=structuredClone(bundle);(dropped.request.security as any[]).pop();expect(()=>validateRepairBundle(dropped,dropped.id,before)).toThrow('harness'); + expect(certify({...params,after:{...params.after,security:'inconclusive'}}).manifest.result).toBe('inconclusive'); + }); }); function tree(root:string){return sha256(JSON.stringify(fs.readdirSync(root).sort().map(p=>{const file=path.join(root,p),stat=fs.statSync(file);return[p,sha256(fs.readFileSync(file)),stat.mode&0o777];})));} diff --git a/test/cso-distribution.test.ts b/test/cso-distribution.test.ts index 812a1708e..2233d2a54 100644 --- a/test/cso-distribution.test.ts +++ b/test/cso-distribution.test.ts @@ -1,10 +1,17 @@ import { afterEach, describe, expect, test } from 'bun:test'; -import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { createHash } from 'node:crypto'; import { tmpdir } from 'node:os'; -import { join, resolve } from 'node:path'; +import { dirname, join, resolve } from 'node:path'; import { spawnSync } from 'node:child_process'; -import { imageBuildMatrix } from '../scripts/cso-image-matrix'; +import committedCatalog from '../lib/cso/runtime-catalog.json'; +import buildInputs from '../lib/cso/images/build-inputs.json'; +import { committedImageBuildMatrix, imageBuildMatrix } from '../scripts/cso-image-matrix'; +import { catalogPromotionCandidate, requiredChecks } from '../scripts/cso-runtime-promotion'; import { runBashScript } from './helpers/bash-script'; +import { requalificationTriggers } from '../scripts/cso-requalification'; + +const CURRENT_TRIGGERS = await requalificationTriggers(); const ROOT = resolve(import.meta.dir, '..'); const temps: string[] = []; @@ -250,12 +257,75 @@ describe('CSO build and distribution wiring', () => { }); }); +const EVALUATION_REF = `sha256:${'e'.repeat(64)}`; +const STAGING_RUN = '777'; +function dispatchedStatements(): any[] { + return committedImageBuildMatrix().include.map(row => ({ + schemaVersion: 1, helperAbi: 3, state: 'qualified', buildRevision: row.inputRevision, runtimeId: row.runtimeId, + stack: row.stack, platform: row.platform, versions: row.versions, sourceCommit: 'a'.repeat(40), + image: `ghcr.io/garrytan/gstack/cso-staging/${row.stack}-${row.arch}@sha256:${Bun.hash(row.runtimeId).toString(16).padStart(64, '0')}`, + checks: Object.fromEntries(requiredChecks(row.stack).map(key => [key, true])), + evaluationRef: EVALUATION_REF, + })); +} +function ingressStep(name: string): string { + const job = (Bun.YAML.parse(readFileSync(join(ROOT, '.github/workflows/cso-runtime-qualification.yml'), 'utf8')) as any).jobs.qualify; + return (job.steps.find((step: any) => step.name === name).run as string).replaceAll('bun run scripts/', `bun run ${quote(ROOT)}/scripts/`); +} +function ingressFixture() { + const dir = mkdtempSync(join(tmpdir(), 'cso-ingress-')); temps.push(dir); + const bin = join(dir, 'bin'), work = join(dir, 'work'), fake = join(dir, 'gh-fixtures'); + for (const path of [bin, work, fake]) mkdirSync(path); + writeFileSync(join(bin, 'gh'), [ + '#!/bin/sh', + 'set -eu', + 'case "$1 $2" in', + ' "attestation verify")', + ' case "$*" in', + ' *https://slsa.dev/provenance/v1*) jq --arg image "$3" \'.[0].verificationResult.statement.subject[0].name = $image\' "$FAKE_GH/provenance.json" ;;', + ' *) jq -n --arg image "$3" \'[{verificationResult:{statement:{predicateType:"https://spdx.dev/Document/v2.3",subject:[{name:$image}]}}}]\' ;;', + ' esac ;;', + ' "api --paginate") cat "$FAKE_GH/jobs.json" ;;', + ' "run download") while [ "$#" -gt 0 ]; do [ "$1" != --dir ] || dir=$2; shift; done; mkdir -p "$dir"; cp "$FAKE_GH/requalification.json" "$dir/" ;;', + ' "api repos/"*) cat "$FAKE_GH/run.json" ;;', + ' *) echo "unexpected gh $*" >&2; exit 2 ;;', + 'esac', + '', + ].join('\n')); + chmodSync(join(bin, 'gh'), 0o755); + const invocation = `https://github.com/garrytan/gstack/actions/runs/${STAGING_RUN}/attempts/1`; + writeFileSync(join(fake, 'provenance.json'), JSON.stringify([{ verificationResult: { + signature: { certificate: { runInvocationURI: invocation } }, + statement: { predicateType: 'https://slsa.dev/provenance/v1', subject: [{ name: '' }], predicate: { runDetails: { metadata: { invocationId: invocation } } } }, + } }])); + const run = (overrides: Record = {}) => writeFileSync(join(fake, 'run.json'), JSON.stringify({ + id: Number(STAGING_RUN), path: '.github/workflows/cso-runtime-images.yml', event: 'workflow_dispatch', head_branch: 'main', + head_sha: 'a'.repeat(40), status: 'completed', conclusion: 'success', + repository: { full_name: 'garrytan/gstack' }, head_repository: { full_name: 'garrytan/gstack' }, ...overrides, + })); + const jobs = (conclusion = 'success') => { + const list = committedImageBuildMatrix().include.map((row, index) => ({ + name: `qualify-native ${row.runtimeId}`, run_id: Number(STAGING_RUN), status: 'completed', conclusion: index === 0 ? conclusion : 'success', + })); + writeFileSync(join(fake, 'jobs.json'), JSON.stringify([{ total_count: list.length, jobs: list }])); + }; + run(); jobs(); + writeFileSync(join(fake, 'requalification.json'), JSON.stringify(CURRENT_TRIGGERS)); + const event = join(dir, 'event.json'); + writeFileSync(event, JSON.stringify({ action: 'cso-runtime-qualified', client_payload: { evaluationRef: EVALUATION_REF, statements: dispatchedStatements() } })); + const env = { ...process.env, PATH: `${bin}:${dirname(process.execPath)}:${process.env.PATH}`, FAKE_GH: fake, GITHUB_EVENT_PATH: event, + GITHUB_REPOSITORY: 'garrytan/gstack', GITHUB_RUN_ID: '555' }; + const step = (name: string) => runBashScript(ingressStep(name), { cwd: work, env, timeout: 60_000 }); + return { work, run, jobs, step }; +} + describe('CSO runtime staging gates', () => { test('runtime workflow shell blocks parse after GitHub expressions are substituted', () => { for (const relative of [ '.github/workflows/cso-runtime-images.yml', '.github/workflows/cso-runtime-qualification.yml', '.github/workflows/cso-runtime-promote.yml', + '.github/workflows/cso-scanner-images.yml', ]) { const raw = readFileSync(join(ROOT, relative), 'utf8'); const workflow = Bun.YAML.parse(raw) as any; @@ -384,13 +454,15 @@ describe('CSO runtime staging gates', () => { expect(workflow.jobs['qualify-native'].needs).toEqual(['reviewed-inputs', 'stage']); expect(raw).toContain('cso-staged-postgresql-${{ matrix.arch }}'); expect(raw).toContain('GSTACK_CSO_TEST_POSTGRES_IMAGE'); - expect(raw).toContain('acquisitionPublicOnlyPassed:true'); - expect(raw).toContain('positiveNegativeAssertionsPassed:true'); - expect(raw).toContain('heldOutRepairPassed:"pending"'); - expect(raw).toContain('railsSqlitePassed:true'); - expect(raw).toContain('railsPostgresqlPassed:true'); - expect(raw).toContain('nativeExtensionsPassed:true'); - expect(raw).toContain('qualified:false'); + const qualify = workflow.jobs['qualify-native']; + expect(qualify.name).toBe('qualify-native ${{ matrix.runtimeId }}'); + expect(raw).not.toMatch(/Passed:\s*true/); + const tested = qualify.steps.findIndex((s: any) => s.run?.includes('bun run test:cso:docker --reporter=junit --reporter-outfile=native-tests.junit.xml')); + const derived = qualify.steps.findIndex((s: any) => s.run?.includes('scripts/cso-native-evidence.ts --stack "$CSO_STACK" --platform "$CSO_PLATFORM"')); + expect(tested).toBeGreaterThanOrEqual(0); + expect(derived).toBe(tested + 1); + expect(qualify.steps[derived].run).toContain('--junit native-tests.junit.xml --staged staged-application/staged-image.json'); + expect(qualify.steps[derived + 1].with.path.split('\n').filter(Boolean)).toEqual(['qualification-evidence.json', 'native-tests.junit.xml']); expect(raw).not.toContain('setup-qemu'); expect(raw).not.toContain('git push'); expect(raw).not.toContain('gh pr create'); @@ -412,7 +484,7 @@ describe('CSO runtime staging gates', () => { expect(Object.keys(workflow.on)).toEqual(['workflow_dispatch']); expect(job.if).toContain("github.ref == 'refs/heads/main'"); expect(job.environment).toBe('cso-runtime-release'); - expect(job.permissions).toMatchObject({ contents: 'write', 'pull-requests': 'write', actions: 'read', packages: 'read', 'id-token': 'write', attestations: 'write' }); + expect(job.permissions).toMatchObject({ contents: 'write', 'pull-requests': 'write', actions: 'write', packages: 'read', 'id-token': 'write', attestations: 'write' }); expect(raw).toContain('.head_branch == "main"'); expect(raw).toContain('.path == ".github/workflows/cso-runtime-qualification.yml"'); expect(raw).toContain('.event == "repository_dispatch"'); @@ -444,15 +516,51 @@ describe('CSO runtime staging gates', () => { for (const step of job.steps) if (step.uses) expect(step.uses).toMatch(/@[a-f0-9]{40}$/); }); + test('catalog promotion PRs dispatch the required free-tests check on the pushed branch', () => { + const freeTests = Bun.YAML.parse(readFileSync(join(ROOT, '.github/workflows/free-tests.yml'), 'utf8')) as any; + expect(Object.keys(freeTests.on)).toContain('workflow_dispatch'); + expect(freeTests.jobs['free-tests'].if).toBe('always()'); + expect(freeTests.jobs['free-tests'].name).toBeUndefined(); + for (const [relative, jobName] of [ + ['.github/workflows/cso-runtime-promote.yml', 'propose'], + ['.github/workflows/cso-scanner-images.yml', 'promote-catalog'], + ]) { + const job = (Bun.YAML.parse(readFileSync(join(ROOT, relative), 'utf8')) as any).jobs[jobName]; + expect(job.permissions.actions, relative).toBe('write'); + const script = job.steps.find((step: any) => step.run?.includes('gh pr create')).run as string; + const pushed = script.indexOf('git push --set-upstream origin "$branch"'); + const dispatched = script.indexOf('gh workflow run free-tests.yml --repo "$GITHUB_REPOSITORY" --ref "$branch"'); + expect(pushed, relative).toBeGreaterThan(0); + expect(dispatched, relative).toBeGreaterThan(pushed); + expect(script.indexOf('gh pr create'), relative).toBeGreaterThan(dispatched); + } + }); + test('private qualification enters only through an actor-restricted protected environment and re-verifies staged attestations', () => { const raw = readFileSync(join(ROOT, '.github/workflows/cso-runtime-qualification.yml'), 'utf8'); const workflow = Bun.YAML.parse(raw) as any; const job = workflow.jobs.qualify; expect(Object.keys(workflow.on)).toEqual(['repository_dispatch']); expect(job.environment).toBe('cso-runtime-release'); - expect(job.permissions).toEqual({ contents: 'read', packages: 'read', attestations: 'read' }); + expect(job.permissions).toEqual({ contents: 'read', packages: 'read', attestations: 'read', actions: 'read' }); + expect(job.needs).toBe('dispatch-summary'); + const summary = workflow.jobs['dispatch-summary']; + expect(summary.permissions).toEqual({}); + expect(summary.environment).toBeUndefined(); + expect(summary.steps.every((step: any) => !step.uses)).toBe(true); expect(raw).toContain('test "$GITHUB_ACTOR" = "$CSO_QUALIFICATION_ACTOR"'); expect(raw).toContain('.client_payload.statements | type == "array" and length == 10'); + expect(raw).not.toContain('^ghcr.io/'); + expect(raw).toContain('^ghcr\\\\.io/garrytan/gstack/cso-staging/'); + const names = job.steps.map((step: any) => step.name); + const attested = names.indexOf('Verify each staged image attestation and normalize qualification statements'); + const staging = names.indexOf('Require one successful protected-main staging run with every native gate passed'); + const contract = names.indexOf('Enforce the complete release contract before retaining evidence'); + expect(attested).toBeGreaterThan(0); + expect(staging).toBe(attested + 1); + expect(contract).toBe(staging + 1); + expect(job.steps[staging].run).toContain('gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/jobs?filter=latest&per_page=100"'); + expect(job.steps.at(-1).with.path).toContain('staging-run-evidence'); expect(raw).toContain('--cert-identity "$signer"'); expect(raw).toContain('--source-digest "$source_commit"'); expect(raw).toContain('--deny-self-hosted-runners'); @@ -489,4 +597,132 @@ describe('CSO runtime staging gates', () => { expect(qualify.steps[loaded].run).toContain('scripts/cso-public-ghcr.ts verify'); expect(qualify.steps[loaded].env.GH_TOKEN).toBe('${{ github.token }}'); }); + + test('the ingress payload gate accepts an optional evaluation reference and summarizes it before approval', () => { + const script = (Bun.YAML.parse(readFileSync(join(ROOT, '.github/workflows/cso-runtime-qualification.yml'), 'utf8')) as any).jobs['dispatch-summary'].steps[0].run; + const dir = mkdtempSync(join(tmpdir(), 'cso-dispatch-summary-')); temps.push(dir); + const event = join(dir, 'event.json'), summary = join(dir, 'summary.md'); + const dispatch = (payload: unknown) => { + writeFileSync(event, JSON.stringify({ action: 'cso-runtime-qualified', client_payload: payload })); + writeFileSync(summary, ''); + const result = runBashScript(script, { cwd: dir, env: { ...process.env, GITHUB_EVENT_PATH: event, GITHUB_STEP_SUMMARY: summary }, timeout: 30_000 }); + return { status: result.status, summary: readFileSync(summary, 'utf8') }; + }; + const statements = dispatchedStatements(); + expect(dispatch({ statements }).summary).toContain('- evaluationRef: `none`'); + const accepted = dispatch({ evaluationRef: EVALUATION_REF, statements }); + expect(accepted.status).toBe(0); + expect(accepted.summary).toContain(`- evaluationRef: \`${EVALUATION_REF}\``); + expect(accepted.summary).toContain(`- sourceCommit: \`${'a'.repeat(40)}\``); + for (const statement of statements) expect(accepted.summary).toContain(`| \`${statement.runtimeId}\` | \`${statement.image}\` |`); + const rejected = (payload: unknown) => expect(dispatch(payload).status).not.toBe(0); + rejected({ evaluationRef: 'sha256:abc', statements }); + rejected({ evaluationRef: EVALUATION_REF.toUpperCase(), statements }); + rejected({ evaluationRef: null, statements }); + rejected({ evaluationRef: EVALUATION_REF, statements, finding: 'held-out detail' }); + rejected({ statements: statements.slice(1) }); + rejected({ statements: statements.map((item, index) => index ? item : { ...item, sourceCommit: 'b'.repeat(40) }) }); + rejected({ statements: statements.map((item, index) => index ? item : { ...item, image: item.image.replace('ghcr.io', 'ghcrxio') }) }); + rejected({ statements: statements.map((item, index) => index ? item : { ...item, runtimeId: 'node`|injected' }) }); + }); + + test('the protected ingress binds verified statements to one successful staging run and never carries the evaluation reference', () => { + const fixture = ingressFixture(); + const normalized = fixture.step('Verify each staged image attestation and normalize qualification statements'); + expect(normalized.status, normalized.stderr).toBe(0); + const staging = fixture.step('Require one successful protected-main staging run with every native gate passed'); + expect(staging.status, staging.stderr).toBe(0); + expect(staging.stdout).toContain('STAGING RUN VERIFIED'); + const contract = fixture.step('Enforce the complete release contract before retaining evidence'); + expect(contract.status, contract.stderr).toBe(0); + + const statements = committedImageBuildMatrix().include.map(row => + JSON.parse(readFileSync(join(fixture.work, 'qualification-evidence', row.runtimeId, 'qualified-runtime.json'), 'utf8'))); + for (const statement of statements) { + expect(statement).not.toHaveProperty('evaluationRef'); + expect(statement.workflow).toBe('https://github.com/garrytan/gstack/actions/runs/555'); + } + expect(JSON.parse(readFileSync(join(fixture.work, 'qualification-evidence', 'requalification.json'), 'utf8'))).toEqual(CURRENT_TRIGGERS); + const candidate = JSON.stringify(catalogPromotionCandidate(committedCatalog, buildInputs, statements, CURRENT_TRIGGERS)); + expect(candidate).not.toContain('evaluationRef'); + expect(candidate).not.toContain('e'.repeat(64)); + + for (const [mutate, error] of [ + [() => fixture.run({ conclusion: 'failure' }), 'STAGING_RUN_NOT_SUCCESSFUL'], + [() => fixture.jobs('failure'), 'STAGING_NATIVE_GATE_NOT_PASSED'], + ] as const) { + fixture.run(); fixture.jobs(); mutate(); + rmSync(join(fixture.work, 'staging-run-evidence'), { recursive: true, force: true }); + const rejected = fixture.step('Require one successful protected-main staging run with every native gate passed'); + expect(rejected.status).not.toBe(0); + expect(rejected.stderr).toContain(error); + } + }); +}); + +describe('CSO evaluation-only candidate builds', () => { + const revision = `cso-eval-123456-${'a'.repeat(12)}`; + const sha = (path: string) => createHash('sha256').update(readFileSync(path)).digest('hex'); + function candidateFixture(catalogRevision = revision) { + const dir = buildFixture(), outside = mkdtempSync(join(tmpdir(), 'gstack-cso-eval-out-')); temps.push(outside); + mkdirSync(join(dir, 'test/helpers'), { recursive: true }); + const catalog = join(outside, 'candidate.json'); + writeFileSync(catalog, JSON.stringify({ schemaVersion: 1, revision: catalogRevision, runtimes: [] }, null, 2) + '\n'); + return { dir, outside, catalog, output: join(outside, 'unit') }; + } + const candidateBuild = (dir: string, catalog: string, output: string, extra = '') => + runBashScript(`${fakeBuildCommand(dir, extra)} --evaluation-candidate ${quote(catalog)} --output ${quote(output)}`, { timeout: 20_000 }); + + test.skipIf(process.platform === 'win32')('builds a marked five-artifact producer unit outside the checkout without touching bin/', () => { + const { dir, outside, catalog, output } = candidateFixture(); + const result = candidateBuild(dir, catalog, output); + expect(result.status).toBe(0); + expect(JSON.parse(result.stdout.trim())).toEqual({ output, revision, evaluationOnly: true }); + expect(readdirSync(output).sort()).toEqual(['.gstack-cso-evaluation', '.gstack-cso-generation', 'cso-eval-producer', 'gstack-cso-core', 'gstack-cso-launcher', 'gstack-cso-watchdog']); + const marker = JSON.parse(readFileSync(join(output, '.gstack-cso-evaluation'), 'utf8')); + expect(marker).toEqual({ schemaVersion: 1, evaluationOnly: true, runtimeCatalogRevision: revision, runtimeCatalogSha256: sha(catalog), coreSha256: sha(join(output, 'gstack-cso-core')) }); + expect(readFileSync(join(output, '.gstack-cso-generation'), 'utf8')).toBe(`${marker.coreSha256}\n`); + expect(readdirSync(join(dir, 'bin'))).toEqual([]); + expect(readdirSync(outside).sort()).toEqual(['candidate.json', 'unit']); + const args = readFileSync(join(dir, 'compiler.log'), 'utf8').trim().split('\n'); + const check = args.indexOf('check-evaluation'); + expect(check).toBeGreaterThan(0); expect(args[check + 1]).toBe('lib/cso/runtime-catalog.json'); + expect(check).toBeLessThan(args.indexOf('lib/cso/cli.ts')); + expect(args).toContain('scripts/cso-eval-producer.ts'); + expect(args.filter(arg => arg === '--no-compile-autoload-package-json')).toHaveLength(2); + }); + + test.skipIf(process.platform === 'win32')('refuses unmarked catalogs, in-checkout or existing outputs, and leaves nothing on a failed check', () => { + const plain = candidateFixture('cso-v3-123456-aaaaaaaaaaaa'); + expect(candidateBuild(plain.dir, plain.catalog, plain.output).status).toBe(65); + const inside = candidateFixture(); + const insideResult = candidateBuild(inside.dir, inside.catalog, join(inside.dir, 'unit')); + expect(insideResult.status).toBe(73); expect(existsSync(join(inside.dir, 'unit'))).toBe(false); + const existing = candidateFixture(); mkdirSync(existing.output); + expect(candidateBuild(existing.dir, existing.catalog, existing.output).status).toBe(73); + const failed = candidateFixture(); + expect(candidateBuild(failed.dir, failed.catalog, failed.output, 'CSO_FAIL_COMPILER_N=1').status).toBe(42); + expect(readdirSync(failed.outside)).toEqual(['candidate.json']); + }); + + test.skipIf(process.platform === 'win32')('distribution builds refuse an evaluation catalog in the source tree', () => { + const dir = buildFixture(); + writeFileSync(join(dir, 'lib/cso/runtime-catalog.json'), JSON.stringify({ revision }, null, 2)); + const result = fakeBuild(dir); + expect(result.status).toBe(65); expect(result.stderr).toContain('evaluation-only catalog'); + expect(existsSync(join(dir, 'compiler.log'))).toBe(false); + }); + + test('the candidate gate rejects the committed release catalog', () => { + const result = spawnSync(process.execPath, ['scripts/cso-runtime-promotion.ts', 'check-evaluation', 'lib/cso/runtime-catalog.json'], { cwd: ROOT, encoding: 'utf8', timeout: 30_000 }); + expect(result.status).toBe(1); expect(result.stderr).toContain('NOT_AN_EVALUATION_CATALOG'); + expect(String((committedCatalog as { revision: string }).revision).startsWith('cso-eval-')).toBe(false); + }); + + test('setup refuses a marked or self-reporting evaluation helper', () => { + const setup = readFileSync(join(ROOT, 'setup'), 'utf8'); + expect(setup).toContain('[ -e "$SOURCE_GSTACK_DIR/bin/.gstack-cso-evaluation" ]'); + expect(setup).toContain(`--version 2>/dev/null | grep -q '"evaluationOnly": *true'`); + expect(setup.indexOf('bin/ holds an evaluation-only CSO helper')).toBeLessThan(setup.indexOf('provision-images --setup-summary')); + }); }); diff --git a/test/cso-eval.test.ts b/test/cso-eval.test.ts index 1b887361f..444b43422 100644 --- a/test/cso-eval.test.ts +++ b/test/cso-eval.test.ts @@ -4,11 +4,11 @@ import { chmodSync, cpSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readd import { createHash } from 'node:crypto'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; -import { collectProducerReceipts, createEvalMatrix, createPortableSkillPayload, loadPortableSkillPayload, prepareEvalJobs, REQUIRED_CONTAINMENT, scoreCollectedEval, scoreEval, validateMatrix, validatePortableSkillPayload, type EvalCell, type EvalResult, type PreparedEvalSchedule } from '../scripts/cso-eval'; +import { collectProducerReceipts, createEvalBaseline, createEvalMatrix, createPortableSkillPayload, loadPortableSkillPayload, prepareEvalJobs, REQUIRED_CONTAINMENT, scoreCollectedEval, scoreEval, validateMatrix, validatePortableSkillPayload, type EvalBaseline, type EvalCell, type EvalMatrix, type EvalResult, type PreparedEvalSchedule } from '../scripts/cso-eval'; import { PRODUCER_PROVIDER_POLICY, producerFailureMessage, producerInstallationIdentity, resolveProducerHelperBinding, runProducerCell, validateProductionProducerInstallation } from '../scripts/cso-eval-producer'; -import { producerArtifactInventoryHash, producerInstallationIdentityHash, producerProviderIdentityHash, producerReceiptHash, sha256, type ProducerArtifactInventory, type ProducerInstallationIdentity, type ProducerProviderIdentity, type ProducerReceipt } from '../scripts/cso-eval-protocol'; +import { producerArtifactInventoryHash, producerHostPlatform, producerInstallationIdentityHash, producerProviderIdentityHash, producerReceiptHash, sha256, type ProducerArtifactInventory, type ProducerInstallationIdentity, type ProducerProviderIdentity, type ProducerReceipt } from '../scripts/cso-eval-protocol'; import type { Family, ProviderAdapter, RunOpts, RunResult } from './helpers/providers/types'; -import { CORPUS_VERSION, FAMILIES, STACKS, loadCorpusManifest, materializeCase, sourceFiles, sourceHash } from './fixtures/cso-eval/materialize'; +import { CORPUS_VERSION, FAMILIES, STACKS, defineEvalCorpus, loadCorpusManifest, materializeCase, sourceFiles, sourceHash, validateCorpusManifest, type CorpusManifest, type EvalVariant } from './fixtures/cso-eval/materialize'; import { judgeRepair, oracleFor, type PrivateEvidence } from './helpers/cso-eval-oracles'; import { inspectPreparation } from '../lib/cso/preparation'; import { assertRuntimeCompatible, RUNTIME_CATALOG } from '../lib/cso/runtime-catalog'; @@ -41,7 +41,8 @@ function portableSkill(version: 'v2' | 'v3', sectionMarker: string): string { } /** Synthetic accounting records exercise the scorer. These are not measured agent results. */ -function syntheticResult(cell: EvalCell): EvalResult { +const syntheticResult = (cell: EvalCell): EvalResult => syntheticResultFor(cell, corpus); +function syntheticResultFor(cell: EvalCell, manifest: CorpusManifest): EvalResult { const runtime = cell.mode === 'comprehensive'; const positive = cell.variant === 'vulnerable'; const repairEvidenceHash = 'c'.repeat(64), recheckEvidenceHash = 'd'.repeat(64); @@ -50,12 +51,12 @@ function syntheticResult(cell: EvalCell): EvalResult { findings: positive ? [{ id: 'finding-1', evidence: cell.version === 'v2' ? 'legacy_review' : 'supported', claimedTested: runtime && cell.version === 'v3', judgment: 'correct', matchedCaseId: cell.caseId, ...(runtime && cell.version === 'v3' ? { trustedVerification: { repair: 'passed' as const, repairEvidenceHash, recheck: 'passed' as const, recheckEvidenceHash } } : {}) }] : [], setup: runtime ? 'passed' : 'not_attempted', reproduction: runtime && positive ? 'passed' : 'not_attempted', repair: runtime && positive ? 'passed' : 'not_attempted', recheck: runtime && positive ? 'passed' : 'not_attempted', - ...(runtime && positive ? { oracleEvidenceHash: repairEvidenceHash, oracleVersion: CORPUS_VERSION, currentSourceHash: corpus.cases.find(fixture => fixture.id === cell.caseId)!.filesHash.fixed, recheckEvidenceHash } : {}), + ...(runtime && positive ? { oracleEvidenceHash: repairEvidenceHash, oracleVersion: manifest.version, currentSourceHash: manifest.cases.find(fixture => fixture.id === cell.caseId)!.filesHash.fixed, recheckEvidenceHash } : {}), heldOutAssertionsPassed: runtime && positive, freshRecheck: runtime && positive, latencyMs: 1500, firstUsefulResultMs: positive ? 500 : null, }; } const group = (score: ReturnType, version: string, mode: string) => score.groups.find(item => item.version === version && item.mode === mode)!; -function syntheticInstallationIdentity(seed = '1'): ProducerInstallationIdentity { +function syntheticInstallationIdentity(seed = '1', catalogSeed = seed): ProducerInstallationIdentity { const artifact = (offset: number) => ({ sha256: sha256(`${seed}:artifact:${offset}`), bytes: 100 + offset }); const core = artifact(3); const withoutHash: Omit = { @@ -63,8 +64,8 @@ function syntheticInstallationIdentity(seed = '1'): ProducerInstallationIdentity producer: artifact(1), launcher: artifact(2), core, watchdog: artifact(4), generation: { coreSha256: core.sha256, manifest: { sha256: sha256(`${core.sha256}\n`), bytes: 65 } }, embeddedCatalogs: { - runtimeRevision: 'runtime-test', runtimeBuildRevision: 'runtime-build-test', runtimeSha256: sha256(`${seed}:runtime`), - scannerRevision: 'scanner-test', scannerSha256: sha256(`${seed}:scanner`), + runtimeRevision: 'runtime-test', runtimeBuildRevision: 'runtime-build-test', runtimeSha256: sha256(`${catalogSeed}:runtime`), + scannerRevision: 'scanner-test', scannerSha256: sha256(`${catalogSeed}:scanner`), }, }; return { ...withoutHash, identityHash: producerInstallationIdentityHash(withoutHash) }; @@ -80,12 +81,14 @@ function syntheticArtifactInventory(): ProducerArtifactInventory { const base = { schemaVersion: 1 as const, root: 'security/cso' as const, entries: [], totalBytes: 0 }; return { ...base, identityHash: producerArtifactInventoryHash(base) }; } +/** Platform-pinned cells get one native installation and provider binary per platform, sharing catalogs and provider policy. */ function syntheticReceipt(cell: EvalCell): ProducerReceipt { - const withoutHash: Omit = { schemaVersion: 1, cell, inputHash: 'e'.repeat(64), installationIdentity: syntheticInstallationIdentity(), providerIdentity: syntheticProviderIdentity(cell.host === 'codex' ? 'gpt' : cell.host), artifacts: syntheticArtifactInventory(), startedAt: '2026-01-01T00:00:00.000Z', finishedAt: '2026-01-01T00:00:01.000Z', status: 'succeeded', requestedModel: cell.model, modelUsed: `resolved-${cell.model}`, modelIdentitySource: 'provider_reported', durationMs: 1000, firstUsefulResultMs: null, toolCalls: 1, output: 'synthetic producer transcript', outputHash: sha256('synthetic producer transcript'), usage: { inputTokens: 10, outputTokens: 5, cachedTokens: 0, estimatedCostUSD: 0.001 } }; + const seed = cell.platform ?? '1'; + const withoutHash: Omit = { schemaVersion: 2, cellRef: cell.id, inputHash: 'e'.repeat(64), installationIdentity: syntheticInstallationIdentity(seed, '1'), providerIdentity: syntheticProviderIdentity(cell.host === 'codex' ? 'gpt' : cell.host, seed), artifacts: syntheticArtifactInventory(), startedAt: '2026-01-01T00:00:00.000Z', finishedAt: '2026-01-01T00:00:01.000Z', status: 'succeeded', requestedModel: cell.model, modelUsed: `resolved-${cell.model}`, modelIdentitySource: 'provider_reported', durationMs: 1000, firstUsefulResultMs: null, toolCalls: 1, output: 'synthetic producer transcript', outputHash: sha256('synthetic producer transcript'), usage: { inputTokens: 10, outputTokens: 5, cachedTokens: 0, estimatedCostUSD: 0.001 } }; return { ...withoutHash, receiptHash: producerReceiptHash(withoutHash) }; } -function syntheticSchedule(): PreparedEvalSchedule { - return { schemaVersion: 1, matrixHash: createHash('sha256').update(JSON.stringify(matrix)).digest('hex'), scheduledCells: matrix.cells.length, preparedCells: matrix.cells.length, jobs: matrix.cells.map(cell => ({ cellId: cell.id, relativePath: `jobs/${cell.id}`, inputHash: 'e'.repeat(64) })) }; +function syntheticSchedule(target: EvalMatrix = matrix): PreparedEvalSchedule { + return { schemaVersion: 1, matrixHash: createHash('sha256').update(JSON.stringify(target)).digest('hex'), scheduledCells: target.cells.length, preparedCells: target.cells.length, jobs: target.cells.map(cell => ({ cellId: cell.id, relativePath: `jobs/${cell.id}`, inputHash: 'e'.repeat(64) })) }; } function rehashReceipt(receipt: ProducerReceipt): ProducerReceipt { const { receiptHash: _receiptHash, ...withoutHash } = receipt; @@ -240,6 +243,20 @@ describe('CSO matched evaluation accounting', () => { mixedProvider.providerIdentity = syntheticProviderIdentity('gpt', 'different-provider'); expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [first, rehashReceipt(mixedProvider)])).toThrow('UNMATCHED_PRODUCER_PROVIDERS'); }); + test('receipts name only an opaque cellRef that the trusted matrix resolves', () => { + const cell = matrix.cells.find(item => item.version === 'v3')!, receipt = syntheticReceipt(cell); + expect(Object.keys(receipt)).not.toContain('cell'); expect(JSON.stringify(receipt)).not.toMatch(/caseId|variant|vulnerable|"fixed"/); + expect(JSON.stringify(receipt)).not.toContain(cell.caseId); + const unknown = structuredClone(receipt); unknown.cellRef = 'f'.repeat(64); + expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [rehashReceipt(unknown)])).toThrow('UNKNOWN_OR_DUPLICATE_PRODUCER_RECEIPT'); + const smuggled = rehashReceipt({ ...structuredClone(receipt), cell } as ProducerReceipt); + expect(() => collectProducerReceipts(matrix, syntheticSchedule(), [smuggled])).toThrow('INVALID_PRODUCER_RECEIPT'); + const batch = syntheticBatch(); + expect(batch.receipts.every(index => index.cellRef === index.cell.id && JSON.stringify(index.cell) === JSON.stringify(matrix.cells.find(item => item.id === index.cellRef)))).toBe(true); + const relabeled = structuredClone(batch); relabeled.receipts[0].cellRef = relabeled.receipts[1].cellRef; + const { batchHash: _hash, ...rest } = relabeled; relabeled.batchHash = createHash('sha256').update(JSON.stringify(rest)).digest('hex'); + expect(() => scoreCollectedEval(matrix, relabeled, [], qualification)).toThrow('INVALID_PRODUCER_BATCH'); + }); test('high/critical recall includes critical cases and detects a v3 regression',()=>{ const criticalCorpus=structuredClone(corpus);criticalCorpus.cases.find(item=>item.severity==='medium')!.severity='critical'; const criticalMatrix=createEvalMatrix({model:'matched-test-model',host:'codex',skillHashes:{v2:'a'.repeat(64),v3:'b'.repeat(64)}},criticalCorpus),complete=criticalMatrix.cells.map(syntheticResult); @@ -333,6 +350,131 @@ describe('CSO matched evaluation accounting', () => { }); }); +/** Synthetic stand-in with the private corpus layout. It is not held-out content. */ +const privateFiles = (id: string, variant: EvalVariant): Record => ({ 'README.md': '# Synthetic service\n', 'src/app.txt': `synthetic ${id} ${variant}\n` }); +const privateManifest: CorpusManifest = { schemaVersion: 1, version: 'synthetic-private-1', cases: STACKS.flatMap(stack => ['alpha-family', 'beta-family'].map((family, index) => { + const id = `${stack}-${family}`; + return { id, stack, family, severity: index === 0 ? 'high' as const : 'medium' as const, rootCause: family, location: { path: 'src/app.txt', symbol: 'action' }, + coreColdStart: true as const, heldOut: true as const, filesHash: { vulnerable: sourceHash(privateFiles(id, 'vulnerable')), fixed: sourceHash(privateFiles(id, 'fixed')) } }; +})) }; +const privateCorpus = defineEvalCorpus(privateManifest, privateFiles); +const releasePlatforms = { daily: 'linux/amd64' as const, comprehensive: Object.fromEntries(privateManifest.cases.map((fixture, index) => [fixture.id, index % 2 ? 'linux/arm64' as const : 'linux/amd64' as const])) }; +const releaseOptions = { profile: 'release' as const, model: 'matched-test-model', host: 'codex' as const, skillHashes: { v3: 'b'.repeat(64) }, platforms: releasePlatforms }; +function privateBaseline(mutate: (results: EvalResult[], cells: EvalCell[]) => void = () => {}): EvalBaseline { + const baselineMatrix = createEvalMatrix({ profile: 'baseline', model: 'matched-test-model', host: 'codex', skillHashes: { v2: 'a'.repeat(64) }, platforms: { comprehensive: releasePlatforms.comprehensive } }, privateManifest); + const batch = collectProducerReceipts(baselineMatrix, syntheticSchedule(baselineMatrix), baselineMatrix.cells.map(syntheticReceipt), privateManifest); + const results = baselineMatrix.cells.map(cell => ({ ...syntheticResultFor(cell, privateManifest), producerReceiptHash: batch.receipts.find(receipt => receipt.cell.id === cell.id)!.receiptHash })); + mutate(results, baselineMatrix.cells); + return createEvalBaseline(baselineMatrix, batch, results, privateManifest); +} +function releaseRun(baseline: EvalBaseline | null = privateBaseline()) { + const release = createEvalMatrix({ ...releaseOptions, ...(baseline ? { baseline } : {}) }, privateManifest); + const batch = collectProducerReceipts(release, syntheticSchedule(release), release.cells.map(syntheticReceipt), privateManifest); + const results = release.cells.map(cell => ({ ...syntheticResultFor(cell, privateManifest), producerReceiptHash: batch.receipts.find(receipt => receipt.cell.id === cell.id)!.receiptHash })); + return { release, batch, results }; +} + +describe('CSO release profile on an injected corpus', () => { + test('the full profile matrix is byte-identical to its pre-release-profile form', () => { + expect(createHash('sha256').update(JSON.stringify(matrix)).digest('hex')).toBe('a0f20a127c8c92c73bb2a277f740295c153ccf95901e92cffab34a4553ef976b'); + expect('profile' in matrix).toBe(false); expect(matrix.cells.every(cell => !('platform' in cell))).toBe(true); + }); + test('an injected corpus shares the public layout and binds every generated source', () => { + expect(validateCorpusManifest(structuredClone(corpus))).toEqual(corpus); + const destination = join(root(), 'case'); + expect(materializeCase('rails-beta-family', 'fixed', destination, privateCorpus).sourceHash).toBe(privateManifest.cases.find(item => item.id === 'rails-beta-family')!.filesHash.fixed); + expect(readFileSync(join(destination, 'src/app.txt'), 'utf8')).toBe('synthetic rails-beta-family fixed\n'); + expect(() => defineEvalCorpus({ ...privateManifest, version: CORPUS_VERSION }, privateFiles)).toThrow('INJECTED_CORPUS_REUSES_PUBLIC_VERSION'); + expect(() => defineEvalCorpus(privateManifest, (id, variant) => ({ ...privateFiles(id, variant), extra: 'drift' }))).toThrow('CORPUS_INTEGRITY_MISMATCH'); + expect(() => defineEvalCorpus(privateManifest, () => ({ '../escape.txt': 'x' }))).toThrow('INVALID_CORPUS_SOURCE'); + const unknownStack = structuredClone(privateManifest); (unknownStack.cases[0] as any).stack = 'php'; + expect(() => validateCorpusManifest(unknownStack)).toThrow('INVALID_CORPUS_MANIFEST'); + const extraField = structuredClone(privateManifest); (extraField.cases[0] as any).answer = 'oracle'; + expect(() => validateCorpusManifest(extraField)).toThrow('INVALID_CORPUS_MANIFEST'); + }); + test('release pins v3, one repetition, and one native platform per cell', () => { + const release = createEvalMatrix(releaseOptions, privateManifest); + expect(release.cells).toHaveLength(privateManifest.cases.length * 4); expect(release.repetitions).toBe(1); expect(release.baselineHash).toBeNull(); + expect(release.cells.every(cell => cell.version === 'v3' && cell.repetition === 1)).toBe(true); + expect(release.cells.filter(cell => cell.mode === 'daily').every(cell => cell.platform === 'linux/amd64')).toBe(true); + for (const cell of release.cells.filter(item => item.mode === 'comprehensive')) expect(cell.platform).toBe(releasePlatforms.comprehensive[cell.caseId]); + expect(new Set(release.cells.filter(cell => cell.mode === 'comprehensive').map(cell => cell.platform))).toEqual(new Set(['linux/amd64', 'linux/arm64'])); + validateMatrix(release, privateManifest); + const moved = structuredClone(release); moved.platforms.comprehensive[privateManifest.cases[0].id] = 'linux/arm64'; + expect(() => validateMatrix(moved, privateManifest)).toThrow('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX'); + const { [privateManifest.cases[0].id]: _omitted, ...partial } = releasePlatforms.comprehensive; + expect(() => createEvalMatrix({ ...releaseOptions, platforms: { daily: 'linux/amd64', comprehensive: partial } }, privateManifest)).toThrow('INVALID_EVAL_PLATFORMS'); + expect(() => createEvalMatrix({ ...releaseOptions, platforms: { daily: 'linux/riscv64' as any, comprehensive: releasePlatforms.comprehensive } }, privateManifest)).toThrow('INVALID_EVAL_PLATFORMS'); + expect(() => createEvalMatrix({ ...releaseOptions, skillHashes: { v2: 'a'.repeat(64), v3: 'b'.repeat(64) } as any }, privateManifest)).toThrow('INVALID_MATCHED_EVAL_INPUT'); + }); + test('a two-platform batch collects and scores qualified against a hash-bound v2 baseline', () => { + const { release, batch, results } = releaseRun(); + expect(new Set(batch.receipts.map(receipt => receipt.installationIdentity.identityHash)).size).toBe(2); + expect(batch.summary.groups.map(group => `${group.version}:${group.mode}`)).toEqual(['v3:daily', 'v3:comprehensive']); + const baseline = privateBaseline(); + const score = scoreCollectedEval(release, batch, results, qualification, privateManifest, baseline); + expect(score.status).toBe('qualified'); expect(Object.values(score.gates).every(value => value === 'pass')).toBe(true); + expect(score.profile).toBe('release'); expect(score.baseline).toEqual({ baselineHash: baseline.baselineHash, highCriticalRecall: { numerator: 4, denominator: 4, value: 1 } }); + expect(score.groups.map(item => `${item.version}:${item.mode}`)).toEqual(['v3:daily', 'v3:comprehensive']); + }); + test('a mixed-corpus batch is rejected', () => { + const { release, batch, results } = releaseRun(); + expect(() => scoreCollectedEval(release, batch, results, qualification)).toThrow('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX'); + expect(() => collectProducerReceipts(release, syntheticSchedule(release), release.cells.map(syntheticReceipt))).toThrow('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX'); + const mixed = structuredClone(release); mixed.cells[0] = { ...matrix.cells.find(cell => cell.version === 'v3' && cell.repetition === 1)!, platform: 'linux/amd64' }; + expect(() => validateMatrix(mixed, privateManifest)).toThrow('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX'); + const publicCell = matrix.cells.find(cell => cell.version === 'v3')!; + expect(() => collectProducerReceipts(release, syntheticSchedule(release), [syntheticReceipt(publicCell)], privateManifest)).toThrow('UNKNOWN_OR_DUPLICATE_PRODUCER_RECEIPT'); + expect(() => scoreCollectedEval(release, syntheticBatch(), results, qualification, privateManifest)).toThrow('INVALID_PRODUCER_BATCH'); + expect(() => scoreEval(release, [syntheticResult(publicCell)], qualification, privateManifest)).toThrow('UNKNOWN_OR_DUPLICATE_EVAL_RESULT'); + }); + test('identity checks hold per platform and require shared catalogs and provider policy across platforms', () => { + const release = createEvalMatrix(releaseOptions, privateManifest), schedule = syntheticSchedule(release); + const amd = release.cells.filter(cell => cell.platform === 'linux/amd64'), arm = release.cells.find(cell => cell.platform === 'linux/arm64')!; + const relabel = (cell: EvalCell, change: (receipt: ProducerReceipt) => void) => { const receipt = syntheticReceipt(cell); change(receipt); return rehashReceipt(receipt); }; + expect(() => collectProducerReceipts(release, schedule, [syntheticReceipt(amd[0]), relabel(amd[1], receipt => { receipt.installationIdentity = syntheticInstallationIdentity('second-amd64', '1'); })], privateManifest)).toThrow('UNMATCHED_PRODUCER_INSTALLATIONS'); + expect(() => collectProducerReceipts(release, schedule, [syntheticReceipt(amd[0]), relabel(amd[1], receipt => { receipt.providerIdentity = syntheticProviderIdentity('gpt', 'second-amd64'); })], privateManifest)).toThrow('UNMATCHED_PRODUCER_PROVIDERS'); + expect(() => collectProducerReceipts(release, schedule, [syntheticReceipt(amd[0]), relabel(arm, receipt => { receipt.installationIdentity = syntheticInstallationIdentity('linux/amd64', '1'); })], privateManifest)).toThrow('UNMATCHED_PRODUCER_PLATFORMS'); + expect(() => collectProducerReceipts(release, schedule, [syntheticReceipt(amd[0]), relabel(arm, receipt => { receipt.installationIdentity = syntheticInstallationIdentity('linux/arm64', 'other-catalog'); })], privateManifest)).toThrow('UNMATCHED_PRODUCER_INSTALLATIONS'); + const otherPolicy = (() => { const { identityHash: _hash, ...base } = syntheticProviderIdentity('gpt', 'linux/arm64'); const changed = { ...base, policyRevision: 'gpt-other-policy' }; return { ...changed, identityHash: producerProviderIdentityHash(changed) }; })(); + expect(() => collectProducerReceipts(release, schedule, [syntheticReceipt(amd[0]), relabel(arm, receipt => { receipt.providerIdentity = otherPolicy; })], privateManifest)).toThrow('UNMATCHED_PRODUCER_PROVIDERS'); + expect(() => collectProducerReceipts(release, schedule, [syntheticReceipt(amd[0]), relabel(amd[1], receipt => { receipt.modelUsed = 'resolved-other-model'; })], privateManifest)).toThrow('UNMATCHED_EFFECTIVE_MODELS'); + }); + test('the external v2 baseline is pre-registered, complete, and bound to the release inputs', () => { + const unpinned = releaseRun(null); + const unmeasured = scoreCollectedEval(unpinned.release, unpinned.batch, unpinned.results, qualification, privateManifest); + expect(unmeasured.gates.noHighCriticalRecallRegression).toBe('unmeasured'); expect(unmeasured.status).toBe('partial'); expect(unmeasured.baseline).toBeNull(); + const baseline = privateBaseline(); + expect(() => scoreCollectedEval(unpinned.release, unpinned.batch, unpinned.results, qualification, privateManifest, baseline)).toThrow('UNBOUND_EVAL_BASELINE'); + const { release, batch, results } = releaseRun(baseline); + const forged = structuredClone(baseline); forged.cases[0].found = false; + expect(() => scoreCollectedEval(release, batch, results, qualification, privateManifest, forged)).toThrow('UNBOUND_EVAL_BASELINE'); + const weaker = privateBaseline(found => { found.forEach(result => { result.findings = []; }); }); + expect(() => scoreCollectedEval(release, batch, results, qualification, privateManifest, weaker)).toThrow('UNBOUND_EVAL_BASELINE'); + expect(() => createEvalMatrix({ ...releaseOptions, model: 'other-model', baseline }, privateManifest)).toThrow('UNBOUND_EVAL_BASELINE'); + expect(() => createEvalMatrix({ ...releaseOptions, budgets: { daily: 600, comprehensive: 1200 }, baseline }, privateManifest)).toThrow('UNBOUND_EVAL_BASELINE'); + const otherProvider = structuredClone(baseline); otherProvider.provider.version = 'gpt-other-version'; + const { baselineHash: _hash, ...record } = otherProvider; otherProvider.baselineHash = createHash('sha256').update(JSON.stringify(record)).digest('hex'); + const rebound = releaseRun(otherProvider); + expect(() => scoreCollectedEval(rebound.release, rebound.batch, rebound.results, qualification, privateManifest, otherProvider)).toThrow('UNBOUND_EVAL_BASELINE'); + const highId = privateManifest.cases.find(item => item.severity === 'high')!.id; + for (let index = 0; index < release.cells.length; index++) { + const cell = release.cells[index]; + if (cell.mode === 'comprehensive' && cell.variant === 'vulnerable' && cell.caseId === highId) results[index].findings = []; + } + const regressed = scoreCollectedEval(release, batch, results, qualification, privateManifest, baseline); + expect(regressed.gates.noHighCriticalRecallRegression).toBe('fail'); expect(regressed.baseline!.highCriticalRecall.value).toBe(1); + }); + test('baseline records refuse incomplete or unadjudicated runs and baseline matrices never qualify', () => { + expect(() => privateBaseline(results => { results.pop(); })).toThrow('INCOMPLETE_EVAL_BASELINE'); + expect(() => privateBaseline((results, cells) => { results[cells.findIndex(cell => cell.variant === 'vulnerable')].findings[0].judgment = 'unadjudicated'; })).toThrow('UNADJUDICATED_EVAL_BASELINE'); + const baselineMatrix = createEvalMatrix({ profile: 'baseline', model: 'matched-test-model', host: 'codex', skillHashes: { v2: 'a'.repeat(64) }, platforms: { comprehensive: releasePlatforms.comprehensive } }, privateManifest); + expect(baselineMatrix.cells.every(cell => cell.version === 'v2' && cell.mode === 'comprehensive')).toBe(true); + expect(() => scoreEval(baselineMatrix, [], qualification, privateManifest)).toThrow('BASELINE_PROFILE_NOT_QUALIFIABLE'); + expect(() => scoreEval(matrix, [], qualification, corpus, privateBaseline())).toThrow('UNBOUND_EVAL_BASELINE'); + }); +}); + describe('CSO matched producer orchestration', () => { const skills = { v2: portableSkill('v2', 'V2_SECTION_ONLY'), v3: portableSkill('v3', 'V3_SECTION_ONLY') }; const producerMatrix = createEvalMatrix({ model: 'exact-eval-model', host: 'codex', skillHashes: { v2: sha256(skills.v2), v3: sha256(skills.v3) } }); @@ -422,7 +564,8 @@ describe('CSO matched producer orchestration', () => { expect(existsSync(join(destination, 'jobs', cell.id, 'producer-input.json'))).toBe(true); const input = JSON.parse(readFileSync(join(destination, 'jobs', cell.id, 'producer-input.json'), 'utf8')); expect(input.skill).toBe(skills[cell.version]); - expect(input.cell.skillHash).toBe(sha256(input.skill)); + expect(input.execution.skillHash).toBe(sha256(input.skill)); + expect(input.cellRef).toBe(cell.id); expect(input.skill).toContain(`${cell.version.toUpperCase()}_SECTION_ONLY`); expect(input.skill).not.toContain(cell.version === 'v2' ? 'V3_SECTION_ONLY' : 'V2_SECTION_ONLY'); expect(validatePortableSkillPayload(input.skill, cell.version).files.map(file => file.path)).toEqual(['SKILL.md', 'sections/manifest.json', 'sections/audit-phases.md']); @@ -815,6 +958,34 @@ describe('CSO matched producer orchestration', () => { expect(() => collectProducerReceipts(producerMatrix, schedule, [tampered])).toThrow('INVALID_PRODUCER_RECEIPT'); }); + test('a platform-pinned release cell runs only on its native producer platform', async () => { + const release = createEvalMatrix({ ...releaseOptions, skillHashes: { v3: sha256(skills.v3) } }, privateManifest); + const host = producerHostPlatform(), foreign = host === 'linux/amd64' ? 'linux/arm64' : 'linux/amd64'; + const native = release.cells.find(cell => cell.mode === 'comprehensive' && cell.platform === host); + const other = release.cells.find(cell => cell.mode === 'comprehensive' && cell.platform === foreign)!; + const destination = join(root(), 'prepared'); + expect(() => prepareEvalJobs(release, skills, join(root(), 'both'), [other.id], privateCorpus)).toThrow('EVAL_SKILL_HASH_MISMATCH'); + prepareEvalJobs(release, { v3: skills.v3 }, destination, [other.id, ...(native ? [native.id] : [])], privateCorpus); + const receipts = join(root(), 'receipts'); mkdirSync(receipts); + const isolatedOther = isolate(destination, other); + const opaque = readFileSync(isolatedOther.input, 'utf8'), parsed = JSON.parse(opaque); + expect(Object.keys(parsed).sort()).toEqual(['cellRef', 'execution', 'schemaVersion', 'skill', 'source']); + expect(parsed).toMatchObject({ schemaVersion: 2, cellRef: other.id, execution: { mode: 'comprehensive', platform: other.platform, sourceHash: other.sourceHash } }); + expect(opaque).not.toMatch(/caseId|variant|vulnerable|"fixed"|"version"|repetition/); expect(opaque).not.toContain(other.caseId); + const smuggled = isolate(destination, other); + writeFileSync(smuggled.input, JSON.stringify({ ...parsed, caseId: other.caseId })); + await expect(runProducerCell(smuggled.input, join(receipts, `${other.id}.json`), withHelper({ adapter: new FakeAdapter(() => {}), paidExecutionAuthorized: true }))).rejects.toThrow('INVALID_PRODUCER_INPUT'); + expect(readFileSync(join(isolatedOther.source, 'src/app.txt'), 'utf8')).toBe(`synthetic ${other.caseId} ${other.variant}\n`); + await expect(runProducerCell(isolatedOther.input, join(receipts, `${other.id}.json`), withHelper({ adapter: new FakeAdapter(() => {}), paidExecutionAuthorized: true }))).rejects.toThrow('UNMATCHED_PRODUCER_PLATFORM'); + if (!native) return; + const isolatedNative = isolate(destination, native); + const receipt = await runProducerCell(isolatedNative.input, join(receipts, `${native.id}.json`), withHelper({ adapter: new FakeAdapter(() => {}), paidExecutionAuthorized: true })); + expect(receipt.cellRef).toBe(native.id); + const written = readFileSync(join(receipts, `${native.id}.json`), 'utf8'); + expect(JSON.parse(written)).not.toHaveProperty('cell'); + expect(written).not.toMatch(/caseId|variant|vulnerable|"fixed"/); expect(written).not.toContain(native.caseId); + }); + test('seals source read-only and withholds a receipt after a mode/content mutation', async () => { const destination = join(root(), 'prepared'); const cell = selected[0]; diff --git a/test/cso-native-evidence.test.ts b/test/cso-native-evidence.test.ts new file mode 100644 index 000000000..721e9db3b --- /dev/null +++ b/test/cso-native-evidence.test.ts @@ -0,0 +1,103 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { deriveNativeChecks, nativeEvidence, nativeTestRequirements, parseJUnit, type JUnitCase } from '../scripts/cso-native-evidence'; +import { requiredChecks } from '../scripts/cso-runtime-promotion'; + +const ROOT = resolve(import.meta.dir, '..'); +const STACKS = ['node', 'bun', 'python', 'rails', 'postgresql'] as const; +const temps: string[] = []; +afterEach(() => { for (const dir of temps.splice(0)) rmSync(dir, { recursive: true, force: true }); }); + +const escape = (value: string) => value.replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>').replaceAll('"', '"'); + +// Mirrors Bun 1.4.0 `bun test --reporter=junit` output: passing cases self-close, skips and failures carry a child element. +function report(cases: JUnitCase[]): string { + const body = cases.map(item => { + const open = ``; + const child = item.status === 'skipped' ? '' : 'AssertionError'; + return ` ${open}>\n ${child}\n `; + }).join('\n'); + const failures = cases.filter(item => item.status === 'failed').length; + return `\n\n \n${body}\n \n\n`; +} + +function passing(stack: typeof STACKS[number]): JUnitCase[] { + const seen = new Map(); + for (const refs of Object.values(nativeTestRequirements(stack))) for (const ref of refs) seen.set(`${ref.file}\0${ref.name}`, { ...ref, status: 'passed', assertions: 3 }); + return [...seen.values()]; +} + +function staged(stack: string, platform = 'linux/amd64') { + return { schemaVersion: 1, state: 'staged', stack, platform, image: `ghcr.io/garrytan/gstack/cso-staging/${stack}-amd64@sha256:${'a'.repeat(64)}` }; +} + +describe('CSO native gate evidence', () => { + test('native check keys are release gates of their stack, and private checks are exactly the remainder', () => { + for (const stack of STACKS) { + const native = Object.keys(nativeTestRequirements(stack)); + expect(native.every(key => requiredChecks(stack).includes(key))).toBe(true); + const evidence = nativeEvidence(staged(stack), stack, 'linux/amd64', report(passing(stack))) as any; + expect(Object.values(evidence.nativeChecks).every(value => value === true)).toBe(true); + expect(Object.keys(evidence.privateChecks).sort()).toEqual(requiredChecks(stack).filter(key => !native.includes(key))); + expect(Object.values(evidence.privateChecks).every(value => value === 'pending')).toBe(true); + expect(evidence).toMatchObject({ state: 'native-gates-passed', qualified: false, stack }); + expect(evidence.nativeTestReport).toMatch(/^sha256:[a-f0-9]{64}$/); + } + const postgres = nativeEvidence(staged('postgresql'), 'postgresql', 'linux/amd64', report(passing('postgresql'))) as any; + expect(Object.keys(postgres.privateChecks).sort()).toEqual(['secretCanaryPassed', 'watchdogCleanupPassed']); + const rails = nativeEvidence(staged('rails'), 'rails', 'linux/amd64', report(passing('rails'))) as any; + expect(Object.keys(rails.privateChecks).sort()).toEqual(['accuracyGatesPassed', 'heldOutRepairPassed', 'secretCanaryPassed', 'watchdogCleanupPassed']); + }); + + test('a skipped, missing, failed, duplicated, or assertion-free named test leaves its checks false', () => { + const named = nativeTestRequirements('node').positiveNegativeAssertionsPassed[0]; + const mutate = (change: (cases: JUnitCase[]) => JUnitCase[]) => deriveNativeChecks('node', parseJUnit(report(change(passing('node'))))); + const skipped = mutate(cases => cases.map(item => item.name === named.name ? { ...item, status: 'skipped' as const, assertions: 0 } : item)); + expect(skipped.positiveNegativeAssertionsPassed).toBe(false); + expect(skipped.containmentPassed).toBe(false); + expect(skipped.coldStartPassed).toBe(true); + expect(mutate(cases => cases.filter(item => item.name !== named.name)).positiveNegativeAssertionsPassed).toBe(false); + expect(mutate(cases => [...cases, cases.find(item => item.name === named.name)!]).positiveNegativeAssertionsPassed).toBe(false); + expect(mutate(cases => cases.map(item => item.name === named.name ? { ...item, assertions: 0 } : item)).positiveNegativeAssertionsPassed).toBe(false); + const unrelatedFailure = mutate(cases => [...cases, { file: 'test/other.test.ts', name: '(unnamed)', status: 'failed', assertions: 0 }]); + expect(Object.values(unrelatedFailure).every(value => value === false)).toBe(true); + expect(() => nativeEvidence(staged('node'), 'node', 'linux/amd64', report(passing('node').map(item => item.name === named.name ? { ...item, status: 'skipped', assertions: 0 } : item)))) + .toThrow('NATIVE_GATE_NOT_PASSED: containmentPassed, positiveNegativeAssertionsPassed'); + }); + + test('a different file with the same test name does not satisfy a gate', () => { + const cases = passing('bun').map(item => item.file === 'test/cso-stack-cold-integration.test.ts' ? { ...item, file: 'test/elsewhere.test.ts' } : item); + const checks = deriveNativeChecks('bun', cases); + expect(checks.coldStartPassed).toBe(false); + expect(checks.acquisitionPublicOnlyPassed).toBe(false); + expect(checks.positiveNegativeAssertionsPassed).toBe(true); + }); + + test('staged evidence must describe the same stack and platform', () => { + expect(() => nativeEvidence(staged('bun'), 'node', 'linux/amd64', report(passing('node')))).toThrow('STAGED_EVIDENCE_MISMATCH'); + expect(() => nativeEvidence(staged('node', 'linux/arm64'), 'node', 'linux/amd64', report(passing('node')))).toThrow('STAGED_EVIDENCE_MISMATCH'); + expect(() => nativeEvidence({ ...staged('node'), state: 'qualified' }, 'node', 'linux/amd64', report(passing('node')))).toThrow('STAGED_EVIDENCE_MISMATCH'); + expect(() => parseJUnit('')).toThrow('INVALID_JUNIT_REPORT'); + expect(() => parseJUnit('\n')).toThrow('EMPTY_JUNIT_REPORT'); + }); + + test.skipIf(process.platform==='win32')('the real Bun JUnit report of the Docker suites names every required test, and skipped suites prove nothing', () => { + const dir = mkdtempSync(join(tmpdir(), 'cso-native-junit-')); temps.push(dir); + const outfile = join(dir, 'native.xml'); + const env = { ...process.env }; + for (const key of ['GSTACK_CSO_DOCKER_TESTS', 'GSTACK_CSO_TEST_IMAGE', 'GSTACK_CSO_TEST_STACK']) delete env[key]; + const run = spawnSync(process.execPath, ['run', 'test:cso:docker', '--reporter=junit', `--reporter-outfile=${outfile}`], { cwd: ROOT, env, encoding: 'utf8', timeout: 120_000 }); + expect(run.status, run.stderr).toBe(0); + const cases = parseJUnit(readFileSync(outfile, 'utf8')); + for (const stack of STACKS) { + for (const refs of Object.values(nativeTestRequirements(stack))) for (const ref of refs) { + expect(cases.filter(item => item.file === ref.file && item.name === ref.name).map(item => item.status)).toEqual(['skipped']); + } + expect(Object.values(deriveNativeChecks(stack, cases)).every(value => value === false)).toBe(true); + } + }); +}); diff --git a/test/cso-requalification.test.ts b/test/cso-requalification.test.ts new file mode 100644 index 000000000..03cc5c85b --- /dev/null +++ b/test/cso-requalification.test.ts @@ -0,0 +1,75 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import committedCatalog from '../lib/cso/runtime-catalog.json'; +import buildInputs from '../lib/cso/images/build-inputs.json'; +import { ISOLATION_POLICY_HASH } from '../lib/cso/docker'; +import { selectRuntime, validateRuntimeCatalog, type RuntimeCatalog } from '../lib/cso/runtime-catalog'; +import { imageSourceFiles, requalificationTriggers, staleRequalificationTriggers } from '../scripts/cso-requalification'; +import { catalogPromotionCandidate } from '../scripts/cso-runtime-promotion'; +import { completeRuntimeCatalogFixture } from './helpers/cso-runtime-catalog'; + +const ROOT = resolve(import.meta.dir, '..'); +const temps: string[] = []; +afterEach(() => { for (const dir of temps.splice(0)) rmSync(dir, { recursive: true, force: true }); }); +const current = await requalificationTriggers(); + +describe('CSO runtime requalification triggers', () => { + test('the committed catalog is not older than its trigger inputs', () => { + expect(staleRequalificationTriggers(committedCatalog as unknown as RuntimeCatalog, current)).toEqual([]); + const cli = spawnSync(process.execPath, ['scripts/cso-requalification.ts', 'check', 'lib/cso/runtime-catalog.json'], { cwd: ROOT, encoding: 'utf8', timeout: 30_000 }); + expect(cli.status, cli.stderr).toBe(0); expect(cli.stdout).toBe('REQUALIFICATION CURRENT\n'); + }); + + test.skipIf(process.platform==='win32')('triggers cover the helper ABI, isolation policy, every image-embedded source byte, and the build inputs', async () => { + expect(current).toMatchObject({ helperAbi: 3, isolationPolicyHash: ISOLATION_POLICY_HASH, buildInputsRevision: buildInputs.revision }); + const files = await imageSourceFiles(); + for (const file of ['lib/cso/verifier.ts', 'lib/cso/preparation-container.ts', 'lib/cso/contracts.ts', 'lib/cso/images/entrypoint', 'lib/cso/images/rails.Dockerfile']) expect(files).toContain(file); + for (const file of ['lib/cso/images/README.md', 'lib/cso/images/qualification.json', 'lib/cso/images/build-inputs.json', 'lib/cso/cli.ts']) expect(files).not.toContain(file); + const copy = mkdtempSync(join(tmpdir(), 'cso-requalification-')); temps.push(copy); + cpSync(join(ROOT, 'lib'), join(copy, 'lib'), { recursive: true }); + expect(await requalificationTriggers(copy)).toEqual(current); + writeFileSync(join(copy, 'lib/cso/verifier.ts'), '\n', { flag: 'a' }); + expect((await requalificationTriggers(copy)).preparationSha256).not.toBe(current.preparationSha256); + }); + + test.skipIf(process.platform==='win32')('triggers resolve against the repository root from any working directory', async () => { + const elsewhere = mkdtempSync(join(tmpdir(), 'cso-requalification-cwd-')); temps.push(elsewhere); + const nested = join(elsewhere, 'a', 'b', 'c'); mkdirSync(nested, { recursive: true }); + const cli = spawnSync(process.execPath, [join(ROOT, 'scripts/cso-requalification.ts'), 'triggers'], { cwd: nested, encoding: 'utf8', timeout: 30_000 }); + expect(cli.status, cli.stderr).toBe(0); expect(JSON.parse(cli.stdout)).toEqual(current); + const copy = join(elsewhere, 'deeper', 'checkout'); mkdirSync(copy, { recursive: true }); + cpSync(join(ROOT, 'lib'), join(copy, 'lib'), { recursive: true }); + expect(await requalificationTriggers(copy)).toEqual(current); + expect(await imageSourceFiles(copy)).toEqual(await imageSourceFiles()); + }); + + test('a promoted catalog names each trigger that changed after qualification', () => { + const catalog = completeRuntimeCatalogFixture(); + const recorded = catalog.promotion!.requalification; + expect(staleRequalificationTriggers(catalog, { ...recorded })).toEqual([]); + expect(staleRequalificationTriggers(catalog, { ...recorded, preparationSha256: 'd'.repeat(64), buildInputsRevision: 'next-inputs' })).toEqual(['buildInputsRevision', 'preparationSha256']); + const withdrawn = { ...catalog, runtimes: [], promotion: undefined }; + expect(staleRequalificationTriggers(withdrawn, { ...recorded, preparationSha256: 'd'.repeat(64) })).toEqual([]); + }); + + test('promoted catalogs must record valid triggers, and the helper refuses runtimes qualified under another isolation policy', () => { + const catalog = completeRuntimeCatalogFixture(); + validateRuntimeCatalog(catalog); + expect(selectRuntime('node', 'linux/amd64', catalog).stack).toBe('node'); + const missing = structuredClone(catalog) as any; delete missing.promotion.requalification; + expect(() => validateRuntimeCatalog(missing)).toThrow('INVALID_RUNTIME_PROMOTION'); + const otherBuild = structuredClone(catalog); otherBuild.promotion!.requalification.buildInputsRevision = 'other-build'; + expect(() => validateRuntimeCatalog(otherBuild)).toThrow('INVALID_RUNTIME_PROMOTION'); + const otherPolicy = structuredClone(catalog); otherPolicy.promotion!.requalification.isolationPolicyHash = 'e'.repeat(64); + expect(() => selectRuntime('node', 'linux/amd64', otherPolicy)).toThrow('STALE_RUNTIME_QUALIFICATION'); + }); + + test('promotion records the staged triggers and refuses ones measured under another isolation policy', () => { + expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, [], { ...current, isolationPolicyHash: 'e'.repeat(64) })).toThrow('STALE_RUNTIME_QUALIFICATION'); + expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, [], { ...current, buildInputsRevision: 'other' })).toThrow('INVALID_REQUALIFICATION_TRIGGERS'); + expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, [], { ...current, extra: true } as any)).toThrow('INVALID_REQUALIFICATION_TRIGGERS'); + }); +}); diff --git a/test/cso-runtime-promotion.test.ts b/test/cso-runtime-promotion.test.ts index 2becd10da..5725762aa 100644 --- a/test/cso-runtime-promotion.test.ts +++ b/test/cso-runtime-promotion.test.ts @@ -2,11 +2,13 @@ import { describe, expect, test } from 'bun:test'; import { createHash } from 'node:crypto'; import committedCatalog from '../lib/cso/runtime-catalog.json'; import buildInputs from '../lib/cso/images/build-inputs.json'; -import { validateRuntimeCatalog } from '../lib/cso/runtime-catalog'; +import { isEvaluationRuntimeCatalog, validateRuntimeCatalog } from '../lib/cso/runtime-catalog'; import { imageBuildMatrix, type ImageBuildRow } from '../scripts/cso-image-matrix'; -import { catalogPromotionCandidate, validateRuntimeCatalogTransition, type RuntimeQualificationStatement } from '../scripts/cso-runtime-promotion'; +import { catalogPromotionCandidate, evaluationCandidateCatalog, validateEvaluationCandidate, validateRuntimeCatalogTransition, type RuntimeQualificationStatement } from '../scripts/cso-runtime-promotion'; import { assertVersionOutput, probesForBuildRow } from '../scripts/cso-verify-runtime-base'; +import { requalificationTriggers } from '../scripts/cso-requalification'; +const TRIGGERS = await requalificationTriggers(); const digest = (value: string) => `sha256:${createHash('sha256').update(value).digest('hex')}`; function checks(row: ImageBuildRow): Record { @@ -69,7 +71,7 @@ describe('CSO runtime catalog promotion', () => { }); test('complete same-run evidence generates a validated review candidate bound to its evidence', () => { - const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, statements()); + const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, statements(), TRIGGERS); expect(candidate.previousRevision).toBe(committedCatalog.revision); expect(candidate.runtimes).toHaveLength(10); expect(candidate.promotion).toMatchObject({ @@ -85,30 +87,30 @@ describe('CSO runtime catalog promotion', () => { test('missing release checks, mutable images, and split qualification runs cannot be promoted', () => { const missing = statements(); delete missing[0].checks.heldOutRepairPassed; - expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, missing)).toThrow('MISSING_RELEASE_GATES'); + expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, missing, TRIGGERS)).toThrow('MISSING_RELEASE_GATES'); const mutable = statements(); mutable[0].image = 'ghcr.io/garrytan/gstack/cso-staging/node-amd64:latest'; - expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, mutable)).toThrow('INVALID_QUALIFIED_IMAGE'); + expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, mutable, TRIGGERS)).toThrow('INVALID_QUALIFIED_IMAGE'); const split = statements(); split[0].workflow = 'https://github.com/garrytan/gstack/actions/runs/999999'; - expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, split)).toThrow('SPLIT_QUALIFICATION_RUN'); + expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, split, TRIGGERS)).toThrow('SPLIT_QUALIFICATION_RUN'); const extra = statements() as Array; extra[0].finding = 'must not enter a public release artifact'; - expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, extra)).toThrow('INVALID_QUALIFICATION_STATEMENT_FIELDS'); + expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, extra, TRIGGERS)).toThrow('INVALID_QUALIFICATION_STATEMENT_FIELDS'); }); test('qualification statements must match reviewed versions and every expected platform row', () => { const drift = statements(); drift[0].versions = { ...drift[0].versions, node: '24.5.0' }; - expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, drift)).toThrow('QUALIFICATION_BUILD_MISMATCH'); - expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, statements().slice(1))).toThrow('INCOMPLETE_QUALIFICATION_MATRIX'); + expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, drift, TRIGGERS)).toThrow('QUALIFICATION_BUILD_MISMATCH'); + expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, statements().slice(1), TRIGGERS)).toThrow('INCOMPLETE_QUALIFICATION_MATRIX'); }); test('promotion is a compare-and-swap against the exact reviewed catalog revision', () => { - const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, statements()); + const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, statements(), TRIGGERS); expect(() => validateRuntimeCatalogTransition(committedCatalog, candidate)).not.toThrow(); expect(() => validateRuntimeCatalogTransition( { ...committedCatalog, revision: 'catalog-advanced-concurrently' }, @@ -123,7 +125,7 @@ describe('CSO runtime catalog promotion', () => { }); test('retained promotion evidence rejects a post-generation runtime image substitution', () => { - const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, statements()); + const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, statements(), TRIGGERS); const altered = structuredClone(candidate); altered.runtimes[0].image = altered.runtimes[0].image.replace(/[a-f0-9]{64}$/, 'e'.repeat(64)); expect(altered.promotion!.evidenceDigest).toBe(candidate.promotion!.evidenceDigest); @@ -148,3 +150,53 @@ describe('CSO reviewed base version probes', () => { expect(() => assertVersionOutput('bundler', '2.6.7', 'Bundler version 2.6.7\n')).not.toThrow(); }); }); + +/** staged-image.json records exactly as the stage job writes them. */ +function stagedRecords(): Record[] { + return statements().map(({ checks: _checks, qualifiedAt: _qualifiedAt, state: _state, ...statement }) => ({ + ...statement, state: 'staged', verifiedProvenance: true, qualification: 'pending native and private release gates; no automatic promotion', + })); +} + +describe('CSO evaluation-only runtime catalogs', () => { + test('one staging run yields an evaluation catalog with no measured release gate', () => { + const candidate = evaluationCandidateCatalog(committedCatalog, buildInputs, stagedRecords(), '2026-10-05T00:00:00.000Z'); + expect(candidate.revision).toBe(`cso-eval-123456-${'a'.repeat(12)}`); + expect(isEvaluationRuntimeCatalog(candidate)).toBe(true); expect(isEvaluationRuntimeCatalog(committedCatalog as any)).toBe(false); + expect(candidate.promotion).toBeUndefined(); expect(candidate.evaluation?.workflow).toBe('https://github.com/garrytan/gstack/actions/runs/123456'); + expect(candidate.runtimes.every(runtime => runtime.state === 'evaluation_candidate' && runtime.qualification.kind === 'evaluation')).toBe(true); + expect(JSON.stringify(candidate)).not.toMatch(/Passed/); + expect(validateEvaluationCandidate(candidate, buildInputs)).toBe(candidate); + expect(() => validateEvaluationCandidate(committedCatalog, buildInputs)).toThrow('NOT_AN_EVALUATION_CATALOG'); + }); + test('staging records must come from one run and match the reviewed build rows', () => { + const split = stagedRecords(); split[1].workflow = 'https://github.com/garrytan/gstack/actions/runs/999'; + expect(() => evaluationCandidateCatalog(committedCatalog, buildInputs, split, '2026-10-05T00:00:00.000Z')).toThrow('SPLIT_STAGING_RUN'); + const unstaged = stagedRecords(); unstaged[0].state = 'qualified'; + expect(() => evaluationCandidateCatalog(committedCatalog, buildInputs, unstaged, '2026-10-05T00:00:00.000Z')).toThrow('INVALID_STAGED_RUNTIME'); + const moved = stagedRecords(); moved[0].buildRevision = 'other-build-revision'; + expect(() => evaluationCandidateCatalog(committedCatalog, buildInputs, moved, '2026-10-05T00:00:00.000Z')).toThrow('QUALIFICATION_BUILD_MISMATCH'); + expect(() => evaluationCandidateCatalog(committedCatalog, buildInputs, stagedRecords().slice(1), '2026-10-05T00:00:00.000Z')).toThrow('INCOMPLETE_QUALIFICATION_MATRIX'); + }); + test('evaluation and release catalogs cannot borrow each other\'s shape', () => { + const candidate = evaluationCandidateCatalog(committedCatalog, buildInputs, stagedRecords(), '2026-10-05T00:00:00.000Z'); + const rebound = structuredClone(candidate); rebound.revision = `cso-eval-999-${'a'.repeat(12)}`; + expect(() => validateRuntimeCatalog(rebound)).toThrow('INVALID_EVALUATION_RUNTIME_CATALOG'); + const promoted = structuredClone(candidate) as any; promoted.promotion = { ...promoted.evaluation, qualificationEvidenceDigest: digest('x'), requalification: TRIGGERS }; delete promoted.evaluation; + expect(() => validateRuntimeCatalog(promoted)).toThrow('INVALID_EVALUATION_RUNTIME_CATALOG'); + const disguised = structuredClone(candidate) as any; disguised.revision = `cso-v3-123456-${'a'.repeat(12)}`; + expect(() => validateRuntimeCatalog(disguised)).toThrow('UNQUALIFIED_RUNTIME'); + const gated = structuredClone(candidate) as any; gated.runtimes[0].qualification.heldOutRepairPassed = true; + expect(() => validateRuntimeCatalog(gated)).toThrow('INVALID_EVALUATION_RUNTIME'); + const released = catalogPromotionCandidate(committedCatalog, buildInputs, statements(), TRIGGERS) as any; + released.revision = `cso-eval-123456-${'a'.repeat(12)}`; + expect(() => validateRuntimeCatalog(released)).toThrow('UNQUALIFIED_RUNTIME'); + }); + test('promotion and catalog transitions refuse evaluation catalogs', () => { + const candidate = evaluationCandidateCatalog(committedCatalog, buildInputs, stagedRecords(), '2026-10-05T00:00:00.000Z'); + expect(() => catalogPromotionCandidate(candidate, buildInputs, statements(), TRIGGERS)).toThrow('EVALUATION_CATALOG_NOT_PROMOTABLE'); + expect(() => validateRuntimeCatalogTransition(committedCatalog, candidate)).toThrow('EVALUATION_CATALOG_NOT_PROMOTABLE'); + expect(() => validateRuntimeCatalogTransition(candidate, catalogPromotionCandidate(committedCatalog, buildInputs, statements(), TRIGGERS))).toThrow('EVALUATION_CATALOG_NOT_PROMOTABLE'); + expect(isEvaluationRuntimeCatalog(committedCatalog as any)).toBe(false); + }); +}); diff --git a/test/cso-staging-run.test.ts b/test/cso-staging-run.test.ts new file mode 100644 index 000000000..ce09cf8a6 --- /dev/null +++ b/test/cso-staging-run.test.ts @@ -0,0 +1,164 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { committedImageBuildMatrix } from '../scripts/cso-image-matrix'; +import { dispatchSourceCommit, stagingRunId, verifyStagingRun, type StagingRunEvidence } from '../scripts/cso-staging-run'; + +const ROOT = resolve(import.meta.dir, '..'); +const REPOSITORY = 'garrytan/gstack'; +const RUN_ID = '34980146643'; +const COMMIT = 'a'.repeat(40); +const RUNTIME_IDS = committedImageBuildMatrix().include.map(row => row.runtimeId); +const temps: string[] = []; +afterEach(() => { for (const dir of temps.splice(0)) rmSync(dir, { recursive: true, force: true }); }); + +function invocation(runId = RUN_ID, attempt = 1, repository = REPOSITORY): string { + return `https://github.com/${repository}/actions/runs/${runId}/attempts/${attempt}`; +} + +// Shape of one `gh attestation verify --format json` entry for an actions/attest provenance statement. +function verified(predicateInvocation = invocation(), certificateInvocation = predicateInvocation): unknown[] { + return [{ + attestation: { bundle: {} }, + verificationResult: { + mediaType: 'application/vnd.dev.sigstore.verificationresult+json;version=0.1', + signature: { certificate: { + subjectAlternativeName: `https://github.com/${REPOSITORY}/.github/workflows/cso-runtime-images.yml@refs/heads/main`, + githubWorkflowTrigger: 'workflow_dispatch', + sourceRepositoryDigest: COMMIT, + runInvocationURI: certificateInvocation, + } }, + statement: { + _type: 'https://in-toto.io/Statement/v1', + predicateType: 'https://slsa.dev/provenance/v1', + subject: [{ name: 'ghcr.io/garrytan/gstack/cso-staging/node-amd64', digest: { sha256: 'b'.repeat(64) } }], + predicate: { + buildDefinition: { externalParameters: { workflow: { path: '.github/workflows/cso-runtime-images.yml', ref: 'refs/heads/main' } } }, + runDetails: { builder: { id: 'builder' }, metadata: { invocationId: predicateInvocation } }, + }, + }, + }, + }]; +} + +function run(overrides: Record = {}): Record { + return { + id: Number(RUN_ID), path: '.github/workflows/cso-runtime-images.yml', event: 'workflow_dispatch', + head_branch: 'main', head_sha: COMMIT, status: 'completed', conclusion: 'success', run_attempt: 2, + repository: { full_name: REPOSITORY }, head_repository: { full_name: REPOSITORY }, ...overrides, + }; +} + +function jobs(): Array> { + const job = (name: string) => ({ name, run_id: Number(RUN_ID), status: 'completed', conclusion: 'success' }); + return [ + job('reviewed-inputs'), + { ...job('validate-native'), conclusion: 'skipped' }, + ...RUNTIME_IDS.map(id => job(`stage (${id})`)), + ...RUNTIME_IDS.map(id => job(`qualify-native ${id}`)), + ]; +} + +function pages(list = jobs(), size = 100): unknown[] { + const result: unknown[] = []; + for (let start = 0; start < list.length; start += size) result.push({ total_count: list.length, jobs: list.slice(start, start + size) }); + return result; +} + +function evidence(overrides: Partial = {}): StagingRunEvidence { + return { repository: REPOSITORY, runId: RUN_ID, sourceCommit: COMMIT, runtimeIds: RUNTIME_IDS, run: run(), jobPages: pages(), ...overrides }; +} + +describe('CSO staging run binding', () => { + test('the committed matrix has the ten native rows the ingress requires', () => { + expect(RUNTIME_IDS).toHaveLength(10); + }); + + test('resolves one staging run from every image provenance, across rerun attempts', () => { + const all = RUNTIME_IDS.map((_, index) => verified(invocation(RUN_ID, index === 0 ? 2 : 1))); + expect(stagingRunId(REPOSITORY, all)).toBe(RUN_ID); + }); + + test('rejects statements that resolve to different runs, foreign repositories, or forged invocations', () => { + const split = RUNTIME_IDS.map((_, index) => verified(invocation(index === 3 ? '999' : RUN_ID))); + expect(() => stagingRunId(REPOSITORY, split)).toThrow('STAGING_RUN_NOT_UNIQUE'); + const twoInOne = [[...verified(), ...verified(invocation('999'))]]; + expect(() => stagingRunId(REPOSITORY, twoInOne)).toThrow('STAGING_RUN_NOT_UNIQUE'); + expect(() => stagingRunId(REPOSITORY, [verified(invocation(RUN_ID, 1, 'someone/fork'))])).toThrow('STAGING_INVOCATION_MISMATCH'); + expect(() => stagingRunId(REPOSITORY, [verified(invocation(), invocation('999'))])).toThrow('STAGING_INVOCATION_MISMATCH'); + expect(() => stagingRunId(REPOSITORY, [verified(`${invocation()}/extra`)])).toThrow('STAGING_INVOCATION_MISMATCH'); + const sbom = verified() as any[]; + sbom[0].verificationResult.statement.predicateType = 'https://spdx.dev/Document/v2.3'; + expect(() => stagingRunId(REPOSITORY, [sbom])).toThrow('INVALID_STAGING_PROVENANCE'); + expect(() => stagingRunId(REPOSITORY, [[]])).toThrow('INVALID_STAGING_PROVENANCE'); + expect(() => stagingRunId(REPOSITORY, [])).toThrow('INVALID_STAGING_PROVENANCE'); + }); + + test('accepts the successful protected-main staging dispatch with every native gate passed, across job pages', () => { + expect(() => verifyStagingRun(evidence())).not.toThrow(); + expect(() => verifyStagingRun(evidence({ jobPages: pages(jobs(), 7) }))).not.toThrow(); + }); + + test('rejects a run that is not the successful protected-main staging dispatch at the statements commit', () => { + expect(() => verifyStagingRun(evidence({ run: run({ path: '.github/workflows/cso-runtime-qualification.yml' }) }))).toThrow('STAGING_WORKFLOW_MISMATCH'); + expect(() => verifyStagingRun(evidence({ run: run({ head_branch: 'feature' }) }))).toThrow('STAGING_RUN_NOT_PROTECTED_MAIN_DISPATCH'); + expect(() => verifyStagingRun(evidence({ run: run({ event: 'pull_request' }) }))).toThrow('STAGING_RUN_NOT_PROTECTED_MAIN_DISPATCH'); + expect(() => verifyStagingRun(evidence({ run: run({ conclusion: 'failure' }) }))).toThrow('STAGING_RUN_NOT_SUCCESSFUL'); + expect(() => verifyStagingRun(evidence({ run: run({ status: 'in_progress', conclusion: null }) }))).toThrow('STAGING_RUN_NOT_SUCCESSFUL'); + expect(() => verifyStagingRun(evidence({ run: run({ head_sha: 'c'.repeat(40) }) }))).toThrow('STAGING_SOURCE_COMMIT_MISMATCH'); + expect(() => verifyStagingRun(evidence({ run: run({ id: 1 }) }))).toThrow('STAGING_RUN_ID_MISMATCH'); + expect(() => verifyStagingRun(evidence({ run: run({ head_repository: { full_name: 'someone/fork' } }) }))).toThrow('STAGING_REPOSITORY_MISMATCH'); + }); + + test('rejects a failed, missing, extra, or truncated native gate job', () => { + const failed = jobs(); + failed[failed.length - 1].conclusion = 'failure'; + expect(() => verifyStagingRun(evidence({ jobPages: pages(failed) }))).toThrow('STAGING_NATIVE_GATE_NOT_PASSED'); + const skipped = jobs(); + skipped[skipped.length - 1].conclusion = 'skipped'; + expect(() => verifyStagingRun(evidence({ jobPages: pages(skipped) }))).toThrow('STAGING_NATIVE_GATE_NOT_PASSED'); + const missing = jobs().slice(0, -1); + expect(() => verifyStagingRun(evidence({ jobPages: pages(missing) }))).toThrow('STAGING_NATIVE_JOBS_MISMATCH'); + const extra = [...jobs(), { name: 'qualify-native unexpected', run_id: Number(RUN_ID), status: 'completed', conclusion: 'success' }]; + expect(() => verifyStagingRun(evidence({ jobPages: pages(extra) }))).toThrow('STAGING_NATIVE_JOBS_MISMATCH'); + const truncated = pages(jobs(), 7).slice(0, -1); + expect(() => verifyStagingRun(evidence({ jobPages: truncated }))).toThrow('INCOMPLETE_STAGING_JOBS'); + const otherRun = jobs(); + otherRun[otherRun.length - 1].run_id = 1; + expect(() => verifyStagingRun(evidence({ jobPages: pages(otherRun) }))).toThrow('STAGING_NATIVE_GATE_NOT_PASSED'); + expect(() => verifyStagingRun(evidence({ jobPages: [] }))).toThrow('INVALID_STAGING_JOBS'); + }); + + test('a dispatch must name one exact source commit', () => { + const statements = RUNTIME_IDS.map(runtimeId => ({ runtimeId, sourceCommit: COMMIT })); + expect(dispatchSourceCommit({ client_payload: { statements } })).toBe(COMMIT); + statements[4].sourceCommit = 'c'.repeat(40); + expect(() => dispatchSourceCommit({ client_payload: { statements } })).toThrow('STAGING_SOURCE_COMMIT_MISMATCH'); + expect(() => dispatchSourceCommit({ client_payload: { statements: [{ sourceCommit: 'main' }] } })).toThrow('STAGING_SOURCE_COMMIT_MISMATCH'); + }); + + test('the workflow command line resolves and verifies the run from retained files', () => { + const dir = mkdtempSync(join(tmpdir(), 'cso-staging-run-')); temps.push(dir); + const files = RUNTIME_IDS.map(id => { + const file = join(dir, `${id}.json`); + writeFileSync(file, JSON.stringify(verified())); + return file; + }); + const script = join(ROOT, 'scripts/cso-staging-run.ts'); + const resolved = spawnSync(process.execPath, ['run', script, 'run-id', '--repository', REPOSITORY, ...files], { encoding: 'utf8', timeout: 30_000 }); + expect(resolved.status, resolved.stderr).toBe(0); + expect(resolved.stdout).toBe(`${RUN_ID}\n`); + writeFileSync(join(dir, 'event.json'), JSON.stringify({ client_payload: { statements: RUNTIME_IDS.map(runtimeId => ({ runtimeId, sourceCommit: COMMIT })) } })); + writeFileSync(join(dir, 'run.json'), JSON.stringify(run())); + writeFileSync(join(dir, 'jobs.json'), JSON.stringify(pages())); + const args = ['run', script, 'verify', '--repository', REPOSITORY, '--run-id', RUN_ID, '--event', join(dir, 'event.json'), '--run', join(dir, 'run.json'), '--jobs', join(dir, 'jobs.json')]; + const accepted = spawnSync(process.execPath, args, { encoding: 'utf8', timeout: 30_000 }); + expect(accepted.status, accepted.stderr).toBe(0); + writeFileSync(join(dir, 'run.json'), JSON.stringify(run({ conclusion: 'failure' }))); + const rejected = spawnSync(process.execPath, args, { encoding: 'utf8', timeout: 30_000 }); + expect(rejected.status).toBe(1); + expect(rejected.stderr).toContain('STAGING_RUN_NOT_SUCCESSFUL'); + }); +}); diff --git a/test/cso-verifier.test.ts b/test/cso-verifier.test.ts index 6448441ca..48ab1e1b9 100644 --- a/test/cso-verifier.test.ts +++ b/test/cso-verifier.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from 'bun:test'; -import { boundedResponseBody } from '../lib/cso/verifier'; +import type { HttpAssertion } from '../lib/cso/contracts'; +import { boundedResponseBody, judgeSecurity } from '../lib/cso/verifier'; describe('CSO bounded loopback verifier response reader',()=>{ test('accepts a response exactly at the configured byte limit',async()=>{ @@ -26,3 +27,50 @@ describe('CSO bounded loopback verifier response reader',()=>{ expect(cancelled).toBe(true); }); }); + +describe('CSO verifier security judgement',()=>{ + const payloads:HttpAssertion[]=[ + {name:'id parameter',path:'/user?id=2',method:'GET',expected:{status:403},vulnerable:{status:200,includes:'tenant-b'}}, + {name:'tenant header',path:'/user',method:'GET',headers:{'x-tenant':'b'},expected:{status:403},vulnerable:{status:200,includes:'tenant-b'}}, + {name:'write path',path:'/user/2',method:'PUT',body:'{}',expected:{status:404},vulnerable:{status:204}}, + ]; + const serve=(fixed:Set)=>{ + const seen:string[]=[]; + const server=Bun.serve({port:0,hostname:'127.0.0.1',fetch(req){ + const url=new URL(req.url),key=`${req.method} ${url.pathname}${url.search}`;seen.push(key); + if(req.method==='PUT')return new Response(null,{status:fixed.has(key)?404:204}); + return fixed.has(key)?new Response('denied',{status:403}):new Response('tenant-b secret',{status:200}); + }}); + return {server,seen,port:Number(server.port)}; + }; + const ALL=new Set(['GET /user?id=2','GET /user','PUT /user/2']); + + test('one booted server answers every array member and the before phase needs every vulnerable expectation',async()=>{ + const {server,seen,port}=serve(new Set()); + try{ + expect(await judgeSecurity('before',payloads,port)).toEqual({security:'intended_failure',members:['intended_failure','intended_failure','intended_failure']}); + expect(seen).toEqual(['GET /user?id=2','GET /user','PUT /user/2']); + }finally{server.stop(true);} + }); + + test('the after phase passes only when every array member passes its fixed expectation',async()=>{ + const fixed=serve(ALL); + try{expect(await judgeSecurity('after',payloads,fixed.port)).toEqual({security:'pass',members:['pass','pass','pass']});}finally{fixed.server.stop(true);} + const partial=serve(new Set(['GET /user?id=2','PUT /user/2'])); + try{ + expect(await judgeSecurity('after',payloads,partial.port)).toEqual({security:'inconclusive',members:['pass','inconclusive','pass']}); + expect(await judgeSecurity('before',payloads,partial.port)).toEqual({security:'inconclusive',members:['pass','intended_failure','pass']}); + }finally{partial.server.stop(true);} + const disproved=serve(ALL); + try{expect((await judgeSecurity('before',payloads,disproved.port)).security).toBe('pass');}finally{disproved.server.stop(true);} + }); + + test('a single assertion keeps its original before and after outcomes',async()=>{ + const vulnerable=serve(new Set()),fixed=serve(ALL); + try{ + expect(await judgeSecurity('before',payloads[0],vulnerable.port)).toEqual({security:'intended_failure',members:['intended_failure']}); + expect(await judgeSecurity('after',payloads[0],vulnerable.port)).toEqual({security:'inconclusive',members:['inconclusive']}); + expect(await judgeSecurity('after',payloads[0],fixed.port)).toEqual({security:'pass',members:['pass']}); + }finally{vulnerable.server.stop(true);fixed.server.stop(true);} + }); +}); diff --git a/test/fixtures/cso-eval/README.md b/test/fixtures/cso-eval/README.md index 66067f32b..c05915c84 100644 --- a/test/fixtures/cso-eval/README.md +++ b/test/fixtures/cso-eval/README.md @@ -127,8 +127,12 @@ gstack helper/runtime files. Do not install or copy any CSO `SKILL.md`, carved section, generated skill tree, schedule, this repository checkout, sibling jobs, fixed alternatives, or evaluator code to that filesystem. The producer runner reads its opaque control file into memory and deletes it before -the agent process starts, so the case identifier and vulnerable/fixed label are -not agent inputs. Copy the selected hash-named directory as the literal path +the agent process starts. That file never names the case, variant, version, or +repetition: it is `{schemaVersion: 2, cellRef, skill, source, execution}`, where +`cellRef` is the matrix cell id and `execution` holds only the mode, model, host, +budget, source and skill hashes, and platform. The receipt binds `cellRef` and +`inputHash`, and `collect` rebuilds each cell from the matrix, so neither file +tells an agent with root on its cell VM which answer it is graded against. Copy the selected hash-named directory as the literal path `/producer/job`; `/producer` must contain only `job`. Put the runner in a system tool directory and use a separate receipt mount. The runner rejects the full prepared batch layout, extra files beside `job`, a reused job with prior state, @@ -249,6 +253,47 @@ reports, containment/canary receipts, and enforced oracle separation. Missing evaluations return `unmeasured` or `partial`, never passing release gates. +## Release profile, private corpora, and two architectures + +The matrix above is the `full` profile: matched v2/v3 cells, three repetitions, +and one producer installation. This public corpus is a development set, because +its oracles are public. A private evaluator scores a held-out corpus with the +same layout through the same code: + +- **Injected corpus.** `defineEvalCorpus(manifest, sourceFiles)` accepts a + manifest with this `manifest.json` layout and a deterministic source generator, + checks every variant's source hash, and refuses the public corpus version. + `prepareEvalJobs(..., corpus)` and `materializeCase(..., corpus)` materialize + from it; `createEvalMatrix`, `validateMatrix`, `collectProducerReceipts`, + `scoreCollectedEval`, and `createEvalBaseline` take its manifest. Its oracles + use the `PrivateOracle` shape from `test/helpers/cso-eval-oracles.ts` and stay + in the private evaluator. +- **`release` profile.** `createEvalMatrix({ profile: 'release', skillHashes: { v3 }, + platforms: { daily, comprehensive } })` builds v3-only cells with one repetition. + Daily cells run on one platform; each comprehensive case is fixed to + `linux/amd64` or `linux/arm64`. Every cell carries its `platform`, and the + producer refuses a cell whose platform differs from its host. +- **Per-platform identities.** One batch may hold both architectures. Each + platform must have exactly one producer installation and one provider binary; + platforms must use different installations with identical embedded catalogs + and identical provider family, policy revision, version, and arguments. All + cells must report one effective model. +- **External v2 baseline.** A `baseline` profile matrix holds v2 comprehensive + cells with the release platform assignment. `createEvalBaseline` turns its + complete, fully adjudicated, receipt-bound results into an `EvalBaseline` + record whose `baselineHash` covers the corpus hash, model, host, budget, + platform assignment, v2 payload hash, producer batch, effective model, provider + policy and version, and each case's outcome. A release matrix pins that hash + when it is created; scoring then requires the exact record and the same + effective model and provider policy. Without a pinned baseline the + regression gate stays `unmeasured`. + +A matrix, batch, or result from one corpus never validates against another: +the matrix binds the corpus hash, so a mixed-corpus batch is rejected. The +`collect`, `score`, and `baseline` commands accept `--corpus-manifest `, +and `score` accepts `--baseline `. The `full` profile's matrix bytes and +gates are unchanged. + The free accounting tests use clearly labeled synthetic observations to test the scorer. They are **not measured agent performance**. No paid comparison, fixture cold-start qualification, or containment qualification is claimed by this corpus. diff --git a/test/fixtures/cso-eval/materialize.ts b/test/fixtures/cso-eval/materialize.ts index 53b8be23a..d9c96b994 100644 --- a/test/fixtures/cso-eval/materialize.ts +++ b/test/fixtures/cso-eval/materialize.ts @@ -9,13 +9,22 @@ export const FAMILIES = ['sql-injection', 'command-injection', 'path-traversal', export type EvalStack = typeof STACKS[number]; export type EvalFamily = typeof FAMILIES[number]; export type EvalVariant = 'vulnerable' | 'fixed'; -export interface CorpusCase { - id: string; stack: EvalStack; family: EvalFamily; severity: 'critical' | 'high' | 'medium'; +export interface CorpusCase { + id: string; stack: EvalStack; family: Family; severity: 'critical' | 'high' | 'medium'; rootCause: string; location: { path: string; symbol: string }; coreColdStart: true; heldOut: true; filesHash: { vulnerable: string; fixed: string }; } -export interface CorpusManifest { schemaVersion: 1; version: string; cases: CorpusCase[] } +export interface CorpusManifest { schemaVersion: 1; version: string; cases: CorpusCase[] } +/** + * A corpus with the public layout: a manifest plus a deterministic source + * generator for each case variant. Private evaluators inject their own; the + * default is this public development corpus. + */ +export interface EvalCorpus { + manifest: CorpusManifest; + sourceFiles(id: string, variant: EvalVariant): Record; +} const hash = (input: string) => createHash('sha256').update(input).digest('hex'); export function sourceHash(files: Record): string { return hash(JSON.stringify(Object.keys(files).sort().map(path => [path, hash(files[path])]))); @@ -304,16 +313,67 @@ export function loadCorpusManifest(): CorpusManifest { if (JSON.stringify(manifest) !== JSON.stringify(createManifest())) throw new Error('CORPUS_INTEGRITY_MISMATCH'); return manifest; } -export function materializeCase(id: string, variant: EvalVariant, destination: string): { path: string; sourceHash: string } { - const manifest = loadCorpusManifest(); - const spec = manifest.cases.find(item => item.id === id); +const SLUG = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; +const HEX = /^[a-f0-9]{64}$/; +const SOURCE_PATH = /^[A-Za-z0-9._-]+(?:\/[A-Za-z0-9._-]+)*$/; +const safeSourcePath = (path: string) => SOURCE_PATH.test(path) && path.length <= 256 && path.split('/').every(part => part !== '.' && part !== '..'); + +/** Validate the manifest layout of a public or injected corpus without reading its sources. */ +export function validateCorpusManifest(value: unknown): CorpusManifest { + const manifest = value as CorpusManifest; + if (!manifest || typeof manifest !== 'object' || manifest.schemaVersion !== 1 || typeof manifest.version !== 'string' || !/^[a-z0-9][a-z0-9.-]{0,63}$/.test(manifest.version) || + !Array.isArray(manifest.cases) || manifest.cases.length === 0 || canonicalKeys(manifest) !== 'cases,schemaVersion,version') throw new Error('INVALID_CORPUS_MANIFEST'); + const ids = new Set(); + for (const item of manifest.cases) { + if (!item || canonicalKeys(item) !== 'coreColdStart,family,filesHash,heldOut,id,location,rootCause,severity,stack' || + typeof item.id !== 'string' || !SLUG.test(item.id) || ids.has(item.id) || !STACKS.includes(item.stack) || + typeof item.family !== 'string' || !SLUG.test(item.family) || !['critical', 'high', 'medium'].includes(item.severity) || + typeof item.rootCause !== 'string' || !item.rootCause.trim() || !item.location || typeof item.location.path !== 'string' || !safeSourcePath(item.location.path) || + typeof item.location.symbol !== 'string' || !item.location.symbol.trim() || item.coreColdStart !== true || item.heldOut !== true || + !item.filesHash || !HEX.test(item.filesHash.vulnerable) || !HEX.test(item.filesHash.fixed) || item.filesHash.vulnerable === item.filesHash.fixed) throw new Error('INVALID_CORPUS_MANIFEST'); + ids.add(item.id); + } + return manifest; +} +function canonicalKeys(value: object): string { return Object.keys(value).sort().join(','); } + +/** The public development corpus. Its oracles are public, so it is never held out. */ +export function publicCorpus(): EvalCorpus { + return { manifest: loadCorpusManifest(), sourceFiles }; +} + +/** + * Bind an injected corpus to its manifest: every variant's generated sources + * must hash to the pinned value. An injected corpus cannot reuse the public + * corpus version, so its matrices and results never alias public ones. + */ +export function defineEvalCorpus(manifestValue: unknown, generate: EvalCorpus['sourceFiles']): EvalCorpus { + const manifest = validateCorpusManifest(manifestValue); + if (manifest.version === CORPUS_VERSION) throw new Error('INJECTED_CORPUS_REUSES_PUBLIC_VERSION'); + const read = (id: string, variant: EvalVariant) => { + const files = generate(id, variant); + if (!files || typeof files !== 'object' || !Object.keys(files).length || Object.entries(files).some(([path, contents]) => !safeSourcePath(path) || typeof contents !== 'string')) throw new Error('INVALID_CORPUS_SOURCE'); + return files; + }; + for (const item of manifest.cases) for (const variant of ['vulnerable', 'fixed'] as const) { + if (sourceHash(read(item.id, variant)) !== item.filesHash[variant]) throw new Error('CORPUS_INTEGRITY_MISMATCH'); + } + return { manifest, sourceFiles: (id, variant) => { + if (!manifest.cases.some(item => item.id === id)) throw new Error('UNKNOWN_CORPUS_CASE'); + if (variant !== 'vulnerable' && variant !== 'fixed') throw new Error('INVALID_CORPUS_VARIANT'); + return read(id, variant); + } }; +} + +export function materializeCase(id: string, variant: EvalVariant, destination: string, corpus: EvalCorpus = publicCorpus()): { path: string; sourceHash: string } { + const spec = corpus.manifest.cases.find(item => item.id === id); if (!spec) throw new Error('UNKNOWN_CORPUS_CASE'); const path = resolve(destination); if (existsSync(path)) throw new Error('CORPUS_DESTINATION_EXISTS'); const parent = dirname(path); if (realpathSync(parent) !== parent || !lstatSync(parent).isDirectory()) throw new Error('UNSAFE_CORPUS_DESTINATION'); - const files = sourceFiles(id, variant); - if (sourceHash(files) !== spec.filesHash[variant]) throw new Error('CORPUS_INTEGRITY_MISMATCH'); + const files = corpus.sourceFiles(id, variant); + if (sourceHash(files) !== spec.filesHash[variant] || Object.keys(files).some(file => !safeSourcePath(file))) throw new Error('CORPUS_INTEGRITY_MISMATCH'); mkdirSync(path, { mode: 0o700 }); for (const [file, contents] of Object.entries(files)) { const output = join(path, file); mkdirSync(dirname(output), { recursive: true, mode: 0o700 }); diff --git a/test/helpers/cso-runtime-catalog.ts b/test/helpers/cso-runtime-catalog.ts index 0bcf9e2bd..c7bdd90bf 100644 --- a/test/helpers/cso-runtime-catalog.ts +++ b/test/helpers/cso-runtime-catalog.ts @@ -1,5 +1,6 @@ import type { CsoStack } from '../../lib/cso/preparation'; import { canonical, sha256 } from '../../lib/cso/contracts'; +import { ISOLATION_POLICY_HASH } from '../../lib/cso/docker'; import { CSO_HELPER_ABI, type QualifiedRuntime, @@ -96,6 +97,12 @@ export function completeRuntimeCatalogFixture( workflow: WORKFLOW, evidenceDigest: `sha256:${sha256(canonical(runtimes))}`, qualificationEvidenceDigest: DIGEST, + requalification: { + helperAbi: CSO_HELPER_ABI, + isolationPolicyHash: ISOLATION_POLICY_HASH, + preparationSha256: 'c'.repeat(64), + buildInputsRevision: 'runtime-test-build-v1', + }, }, runtimes, }; diff --git a/test/helpers/qa-browser-deadline-evidence.ts b/test/helpers/qa-browser-deadline-evidence.ts index 2da105051..512dc6c43 100644 --- a/test/helpers/qa-browser-deadline-evidence.ts +++ b/test/helpers/qa-browser-deadline-evidence.ts @@ -105,18 +105,33 @@ export function assertQaBrowserPreparation(transcript: unknown[], options: Pick< if (!prepared) throw new Error('QA preparation: owned nonempty report Write must complete before guard start/baseline'); } -export function assertQaBrowserCheckpoints(transcript: unknown[], options: Pick) { +/** + * The fixture prompt routes browser cleanup through the guard, and a probe + * excludes bookkeeping. Only closing the tab with the supplied browse binary, + * as the whole child, is cleanup; anything else in the child is a probe. + */ +function browserCleanupChild(child: string[], browse: string): boolean { + if (child.length === 2) return child[0] === browse && child[1] === 'closetab'; + return child.length === 3 && child[0] === 'bash' && child[1] === '-c' + && [`B="${browse}"; "$B" closetab`, `"${browse}" closetab`].includes(child[2].trim()); +} + +export function assertQaBrowserCheckpoints(transcript: unknown[], options: Pick) { const fail = (reason: string): never => { throw new Error('QA checkpoint: ' + reason); }; const root = path.join(options.directory, 'qa-reports'); const files = readQACheckpointFiles(root); const calls = browserNativeCalls(transcript); - const runs = calls.filter(call => { - if (call.parent !== null || call.name !== 'Bash' || typeof call.input.command !== 'string') return false; + const guarded = calls.flatMap(call => { + if (call.parent !== null || call.name !== 'Bash' || typeof call.input.command !== 'string') return []; let argv: string[]; - try { argv = literalArgv(call.input.command); } catch { return false; } + try { argv = literalArgv(call.input.command); } catch { return []; } return ['bun', process.execPath].includes(argv[0]) && argv[1] === options.guard - && argv[2] === 'run' && argv[3] === path.join(root, 'deadline.json'); + && argv[2] === 'run' && argv[3] === path.join(root, 'deadline.json') + ? [{ call, cleanup: argv[4] === '--' && browserCleanupChild(argv.slice(5), options.browse) }] : []; }); + const runs = guarded.filter(run => !run.cleanup).map(run => run.call); + const lastProbe = runs.at(-1); + if (lastProbe && guarded.some(run => run.cleanup && run.call.start < lastProbe.end)) fail('browser cleanup preceded a probe'); const notes: Array<{ call: (typeof calls)[number]; value: any }> = []; const written = new Set(); for (const call of calls) { diff --git a/test/qa-browser-deadline-evidence.test.ts b/test/qa-browser-deadline-evidence.test.ts index ced94fdcf..5a019d8ad 100644 --- a/test/qa-browser-deadline-evidence.test.ts +++ b/test/qa-browser-deadline-evidence.test.ts @@ -58,11 +58,37 @@ test.each(['captured-rewrite', 'verbatim-text', 'decoded-json', 'wrapped-json', fs.writeFileSync(write.input.file_path, scenario === 'disk-mismatch' ? '{}' : write.input.content); if (scenario === 'missing-ack') events.splice(3, 1); if (scenario === 'late-write') events.push(...events.splice(2, 2)); - const check = () => assertQaBrowserCheckpoints(events, { directory: f.directory, guard }); + const check = () => assertQaBrowserCheckpoints(events, { directory: f.directory, guard, browse }); if (scenario === 'verbatim-text' || scenario === 'decoded-json') expect(check).not.toThrow(); else expect(check).toThrow(); }); +// CI run 37252022510 (qa-only-no-fix): after its last probe the agent closed the tab +// through the guard, as the fixture prompt directs, and the clock had expired. +test.each(['cleanup-after-last-probe', 'cleanup-direct-argv', 'probe-after-cleanup', 'compound-cleanup', 'other-browser-cleanup']) + ('guarded browser cleanup after the last probe needs no checkpoint; anything more is a probe: %s', scenario => { + const f = fixture(); + const events = JSON.parse(JSON.stringify(capturedObservation.events) + .replaceAll('__QA_FIXTURE__', f.directory).replaceAll('__QA_GUARD__', guard).replaceAll('__QA_BROWSE__', browse)); + const baseline = events[1].message.content[0], write = events[2].message.content[0]; + const lines = baseline.content.split('\n'); + const first = lines.findIndex((line: string) => line.startsWith('QA_DEADLINE ')); + const last = lines.findLastIndex((line: string) => line.startsWith('QA_DEADLINE ')); + write.input.content = JSON.stringify({ ...JSON.parse(write.input.content), observed: lines.slice(first + 1, last).join('\n') }); + fs.writeFileSync(write.input.file_path, write.input.content); + const child = scenario === 'cleanup-direct-argv' ? quote(browse) + ' closetab' + : `bash -c 'B="${scenario === 'other-browser-cleanup' ? '/usr/bin/true' : browse}"; "$B" closetab${scenario === 'compound-cleanup' ? '; "$B" goto http://127.0.0.1:1/' : ''}'`; + const expired = events[5].message.content[0].content; + const cleanup = [ + { type: 'assistant', message: { content: [{ type: 'tool_use', id: 'toolu_cleanup', name: 'Bash', input: { command: `bun ${quote(guard)} run ${quote(path.join(f.directory, 'qa-reports/deadline.json'))} -- ${child}` } }] } }, + { type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'toolu_cleanup', content: expired, is_error: true }] } }, + ]; + if (scenario === 'probe-after-cleanup') events.splice(4, 0, ...cleanup); else events.push(...cleanup); + const check = () => assertQaBrowserCheckpoints(events, { directory: f.directory, guard, browse }); + if (['cleanup-after-last-probe', 'cleanup-direct-argv'].includes(scenario)) expect(check).not.toThrow(); + else expect(check).toThrow(scenario === 'probe-after-cleanup' ? 'browser cleanup preceded a probe' : 'follow-up lacks one acknowledged preceding checkpoint'); + }); + test('R70 retained public ordering has no completed report Write before its clock/baseline', () => { const f = fixture(); const events = JSON.parse(JSON.stringify(capturedPreparation.events) diff --git a/test/skill-e2e-qa-workflow.test.ts b/test/skill-e2e-qa-workflow.test.ts index 57d8e8988..d2ebc644b 100644 --- a/test/skill-e2e-qa-workflow.test.ts +++ b/test/skill-e2e-qa-workflow.test.ts @@ -188,7 +188,7 @@ Write your report to ${qaOnlyDir}/qa-reports/qa-only-report.md`, directory: qaOnlyDir, guard: path.join(ROOT, 'bin/gstack-qa-deadline'), }); assertQaBrowserCheckpoints(result.transcript, { - directory: qaOnlyDir, guard: path.join(ROOT, 'bin/gstack-qa-deadline'), + directory: qaOnlyDir, guard: path.join(ROOT, 'bin/gstack-qa-deadline'), browse: browseBin, }); passed = true; } catch (error) { diff --git a/test/skill-llm-eval.test.ts b/test/skill-llm-eval.test.ts index 9cfa011eb..668c59a12 100644 --- a/test/skill-llm-eval.test.ts +++ b/test/skill-llm-eval.test.ts @@ -669,6 +669,11 @@ describeIfSelected('Other skill evals', [ judgeContext: 'a pre-landing review with bounded exploratory QA', judgeGoal: 'how to review and explore changed behavior even for small diffs without a plan or server, preserve report-only discovery and the test_stub ASK gate, handle incomplete probes honestly, and rerun affected evidence after approved repairs', agentCapability: 'frontier', + // This ~64k-token bundle is the largest workflow input. Thinking and the answer share max_tokens: + // at default effort, 24 measured samples used up to 6.9k output tokens and a CI sample spent all + // 8192 without an answer. The ship-sized streaming cap removes truncation; effort stays default. + maxTokens: 65_536, + stream: true, }); }, WORKFLOW_JUDGE_TEST_MS); diff --git a/test/workflow-judge-cache.test.ts b/test/workflow-judge-cache.test.ts index a049470a1..a11f1cc70 100644 --- a/test/workflow-judge-cache.test.ts +++ b/test/workflow-judge-cache.test.ts @@ -400,6 +400,12 @@ test('Ship sends its authorized 64k cap and compact response contract through th expect(options.stream).toBe(true); expect(options.effort).toBe('medium'); expect(source.match(/^\s+effort: '/gm)).toHaveLength(1); + // The ~64k-token review bundle gets only ship's output cap and streaming; its effort and response format stay default. + const review = source.match(/testIfSelected\('review\/SKILL\.md workflow',[\s\S]*?await runWorkflowJudge\(\{([\s\S]*?)\n \}\);/); + expect(review).not.toBeNull(); + const reviewOptions = new Function('QA_DISCOVERY_REFERENCES', `return ({${review![1]}});`)(QA_DISCOVERY_REFERENCES); + expect([reviewOptions.maxTokens, reviewOptions.stream, reviewOptions.effort, reviewOptions.schemaTransport, reviewOptions.compactReasoning]) + .toEqual([65_536, true, undefined, undefined, undefined]); expect(WORKFLOW_JUDGE_RESPONSE_SCHEMA.properties.reasoning).not.toHaveProperty('pattern'); expect(WORKFLOW_JUDGE_RESPONSE_SCHEMA.properties.reasoning).not.toHaveProperty('maxLength'); expect(source.match(/compactReasoning: true/g)).toHaveLength(1);