From 9b21621462f396ddedc5daa6bcf4c624ec22962e Mon Sep 17 00:00:00 2001 From: garrytan Date: Mon, 5 Oct 2026 01:07:53 +0000 Subject: [PATCH] v1.91.23.0 feat(cso): private two-architecture release scoring and self-checking runtime release gates --- .github/workflows/cso-runtime-images.yml | 37 +- .github/workflows/cso-runtime-promote.yml | 6 +- .../workflows/cso-runtime-qualification.yml | 74 +++- .github/workflows/cso-scanner-images.yml | 3 +- CHANGELOG.md | 16 + VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- docs/cso-release-qualification.md | 135 ++++++- docs/cso-scanners.md | 10 +- lib/cso/cli.ts | 15 +- lib/cso/contracts.ts | 38 +- lib/cso/runtime-catalog.ts | 92 ++++- lib/cso/verifier.ts | 61 ++- package.json | 2 +- scripts/build-cso.sh | 118 +++++- scripts/cso-eval-producer.ts | 5 +- scripts/cso-eval-protocol.ts | 10 + scripts/cso-eval.ts | 361 ++++++++++++++---- scripts/cso-native-evidence.ts | 131 +++++++ scripts/cso-requalification.ts | 67 ++++ scripts/cso-runtime-promotion.ts | 146 ++++++- scripts/cso-staging-run.ts | 124 ++++++ setup | 6 + test/cso-contracts.test.ts | 46 ++- test/cso-distribution.test.ts | 260 ++++++++++++- test/cso-docker-integration.test.ts | 5 +- test/cso-eval.test.ts | 168 +++++++- test/cso-native-evidence.test.ts | 103 +++++ test/cso-requalification.test.ts | 64 ++++ test/cso-runtime-promotion.test.ts | 74 +++- test/cso-staging-run.test.ts | 164 ++++++++ test/cso-verifier.test.ts | 50 ++- test/fixtures/cso-eval/README.md | 41 ++ test/fixtures/cso-eval/materialize.ts | 76 +++- test/helpers/cso-runtime-catalog.ts | 7 + 35 files changed, 2284 insertions(+), 235 deletions(-) create mode 100644 scripts/cso-native-evidence.ts create mode 100644 scripts/cso-requalification.ts create mode 100644 scripts/cso-staging-run.ts create mode 100644 test/cso-native-evidence.test.ts create mode 100644 test/cso-requalification.test.ts create mode 100644 test/cso-staging-run.test.ts diff --git a/.github/workflows/cso-runtime-images.yml b/.github/workflows/cso-runtime-images.yml index 1f59c136c..6505a1910 100644 --- a/.github/workflows/cso-runtime-images.yml +++ b/.github/workflows/cso-runtime-images.yml @@ -12,6 +12,7 @@ on: - 'scripts/cso-image-matrix.ts' - 'scripts/cso-verify-runtime-base.ts' - 'scripts/cso-runtime-promotion.ts' + - 'scripts/cso-requalification.ts' - 'scripts/cso-public-ghcr.ts' - 'test/cso-*.test.ts' workflow_dispatch: @@ -42,6 +43,15 @@ jobs: set -eu matrix="$(bun run scripts/cso-image-matrix.ts)" printf 'matrix=%s\n' "$matrix" >> "$GITHUB_OUTPUT" + - name: Record the requalification trigger inputs of this source commit + run: bun run scripts/cso-requalification.ts triggers > requalification.json + - if: github.event_name == 'workflow_dispatch' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: cso-requalification-triggers + path: requalification.json + if-no-files-found: error + retention-days: 30 validate-native: needs: reviewed-inputs @@ -279,6 +289,7 @@ jobs: retention-days: 30 qualify-native: + name: qualify-native ${{ matrix.runtimeId }} needs: [reviewed-inputs, stage] if: github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' environment: cso-runtime-release @@ -344,31 +355,21 @@ jobs: set -eu bun install --frozen-lockfile --ignore-scripts bun run build:cso - bun run test:cso:docker - - name: Record native gate evidence without claiming runtime qualification + bun run test:cso:docker --reporter=junit --reporter-outfile=native-tests.junit.xml + - name: Derive native gate evidence from the executed test report env: CSO_STACK: ${{ matrix.stack }} CSO_PLATFORM: ${{ matrix.platform }} run: | set -eu - jq --arg stack "$CSO_STACK" --arg platform "$CSO_PLATFORM" ' - . + { - state:"native-gates-passed", - nativeChecks:(if $stack == "postgresql" then { - containmentPassed:true,coldStartPassed:true,multiDatabasePassed:true,readinessPassed:true - } else { - containmentPassed:true,coldStartPassed:true,positiveNegativeAssertionsPassed:true, - acquisitionPublicOnlyPassed:true,offlineLifecyclePassed:true - } + (if $stack == "rails" then { - railsSqlitePassed:true,railsPostgresqlPassed:true,nativeExtensionsPassed:true - } else {} end) end), - privateChecks:{heldOutRepairPassed:"pending",accuracyGatesPassed:"pending",secretCanaryPassed:"pending",watchdogCleanupPassed:"pending"}, - qualified:false, - promotion:"prohibited until authenticated private qualification evidence completes every release gate" - }' staged-application/staged-image.json > qualification-evidence.json + bun run scripts/cso-native-evidence.ts --stack "$CSO_STACK" --platform "$CSO_PLATFORM" \ + --junit native-tests.junit.xml --staged staged-application/staged-image.json \ + --output qualification-evidence.json - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: cso-qualification-gates-${{ matrix.stack }}-${{ matrix.arch }} - path: qualification-evidence.json + path: | + qualification-evidence.json + native-tests.junit.xml if-no-files-found: error retention-days: 30 diff --git a/.github/workflows/cso-runtime-promote.yml b/.github/workflows/cso-runtime-promote.yml index 5585be419..5a012ec19 100644 --- a/.github/workflows/cso-runtime-promote.yml +++ b/.github/workflows/cso-runtime-promote.yml @@ -3,6 +3,9 @@ name: Propose CSO Runtime Catalog Promotion # This workflow never deploys a catalog. It converts authenticated # same-repository qualification artifacts into an attested candidate, verifies # those exact bytes, and opens a normal source-review PR from a protected job. +# Pushes and PRs made with GITHUB_TOKEN start no workflows, so the job +# dispatches the required free-tests check on the pushed branch head; a +# workflow_dispatch is the documented exception to that rule. on: workflow_dispatch: inputs: @@ -28,7 +31,7 @@ jobs: permissions: contents: write pull-requests: write - actions: read + actions: write packages: read id-token: write attestations: write @@ -161,6 +164,7 @@ jobs: cmp runtime-catalog.candidate.json committed-runtime-catalog.json test "$expected_candidate_sha256" = "sha256:$(sha256sum committed-runtime-catalog.json | cut -d ' ' -f 1)" git push --set-upstream origin "$branch" + gh workflow run free-tests.yml --repo "$GITHUB_REPOSITORY" --ref "$branch" cat > pr-body.md <> "$GITHUB_STEP_SUMMARY" + qualify: + needs: dispatch-summary if: github.ref == 'refs/heads/main' && github.event_name == 'repository_dispatch' runs-on: ubuntu-24.04 timeout-minutes: 40 @@ -22,6 +60,7 @@ jobs: contents: read packages: read attestations: read + actions: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: @@ -29,18 +68,13 @@ jobs: - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 with: bun-version: 1.4.0 - - name: Require the configured private-evaluator actor and bounded matrix + - name: Require the configured private-evaluator actor env: CSO_QUALIFICATION_ACTOR: ${{ vars.CSO_QUALIFICATION_ACTOR }} run: | set -euo pipefail test -n "$CSO_QUALIFICATION_ACTOR" test "$GITHUB_ACTOR" = "$CSO_QUALIFICATION_ACTOR" - jq -e ' - .action == "cso-runtime-qualified" and - (.client_payload | keys == ["statements"]) and - (.client_payload.statements | type == "array" and length == 10) - ' "$GITHUB_EVENT_PATH" - name: Recheck public visibility and anonymous pulls for every qualified digest env: GH_TOKEN: ${{ github.token }} @@ -49,7 +83,7 @@ jobs: mkdir public-image-evidence jq -c '.client_payload.statements[]' "$GITHUB_EVENT_PATH" | while IFS= read -r statement; do runtime_id="$(printf '%s' "$statement" | jq -er '.runtimeId | select(test("^[a-z0-9][a-z0-9._-]{0,100}$"))')" - image="$(printf '%s' "$statement" | jq -er '.image | select(test("^ghcr.io/garrytan/gstack/cso-staging/[a-z0-9-]+-(amd64|arm64)@sha256:[a-f0-9]{64}$"))')" + image="$(printf '%s' "$statement" | jq -er '.image | select(test("^ghcr\\.io/garrytan/gstack/cso-staging/[a-z0-9-]+-(amd64|arm64)@sha256:[a-f0-9]{64}$"))')" platform="$(printf '%s' "$statement" | jq -er '.platform | select(. == "linux/amd64" or . == "linux/arm64")')" bun run scripts/cso-public-ghcr.ts verify \ --image "$image" --platform "$platform" --repository "$GITHUB_REPOSITORY" \ @@ -66,7 +100,7 @@ jobs: mkdir qualification-evidence jq -c '.client_payload.statements[]' "$GITHUB_EVENT_PATH" | while IFS= read -r statement; do runtime_id="$(printf '%s' "$statement" | jq -er '.runtimeId | select(test("^[a-z0-9][a-z0-9._-]{0,100}$"))')" - image="$(printf '%s' "$statement" | jq -er '.image | select(test("^ghcr.io/garrytan/gstack/cso-staging/[a-z0-9-]+-(amd64|arm64)@sha256:[a-f0-9]{64}$"))')" + image="$(printf '%s' "$statement" | jq -er '.image | select(test("^ghcr\\.io/garrytan/gstack/cso-staging/[a-z0-9-]+-(amd64|arm64)@sha256:[a-f0-9]{64}$"))')" source_commit="$(printf '%s' "$statement" | jq -er '.sourceCommit | select(test("^[a-f0-9]{40}$"))')" output="qualification-evidence/$runtime_id" mkdir "$output" @@ -92,6 +126,23 @@ jobs: }' \ > "$output/qualified-runtime.json" done + - name: Require one successful protected-main staging run with every native gate passed + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir staging-run-evidence + run_id="$(bun run scripts/cso-staging-run.ts run-id --repository "$GITHUB_REPOSITORY" \ + qualification-evidence/*/provenance-verification.json)" + gh api "repos/$GITHUB_REPOSITORY/actions/runs/$run_id" > staging-run-evidence/run.json + gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/jobs?filter=latest&per_page=100" \ + > staging-run-evidence/jobs.json + bun run scripts/cso-staging-run.ts verify --repository "$GITHUB_REPOSITORY" --run-id "$run_id" \ + --event "$GITHUB_EVENT_PATH" --run staging-run-evidence/run.json --jobs staging-run-evidence/jobs.json + gh run download "$run_id" --repo "$GITHUB_REPOSITORY" \ + --name cso-requalification-triggers --dir requalification-download + mv requalification-download/requalification.json qualification-evidence/requalification.json + rmdir requalification-download - name: Enforce the complete release contract before retaining evidence run: | set -euo pipefail @@ -105,5 +156,6 @@ jobs: path: | qualification-evidence public-image-evidence + staging-run-evidence if-no-files-found: error retention-days: 30 diff --git a/.github/workflows/cso-scanner-images.yml b/.github/workflows/cso-scanner-images.yml index e825f6849..2d7a172e2 100644 --- a/.github/workflows/cso-scanner-images.yml +++ b/.github/workflows/cso-scanner-images.yml @@ -319,7 +319,7 @@ jobs: permissions: contents: write pull-requests: write - actions: read + actions: write packages: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -365,6 +365,7 @@ jobs: git add lib/cso/scanner-images/catalog.json git commit -m "Promote qualified CSO scanner catalog" git push --set-upstream origin "$branch" + gh workflow run free-tests.yml --repo "$GITHUB_REPOSITORY" --ref "$branch" cat > pr-body.md <<'EOF' Promotes the complete scanner catalog produced by native amd64/arm64 qualification for all six CSO scanners. Each profile binds its immutable image, exact version-output hash, offline asset hash when required, containment and adapter gates, SBOM, provenance, source commit, and workflow run. diff --git a/CHANGELOG.md b/CHANGELOG.md index 07f6a13fd..68b407081 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## [1.91.23.0] - 2026-10-05 + +**gstack can grade a private, two-architecture `/cso` release run, and the runtime release gates check more for themselves.** + +These are the gstack-side pieces the private CSO runtime evaluator needs before it can qualify Node, Bun, Python and Rails runtimes. The release pipeline (`cso-runtime-images.yml` → private evaluator → `cso-runtime-qualification.yml` → `cso-runtime-promote.yml`) now verifies more of its own evidence, and nothing about a user's audit changes until a qualified runtime catalog merges. + +### What changes for you + +- **Private corpora score through the public scorer.** `scripts/cso-eval.ts` accepts an injected corpus with the public layout (`defineEvalCorpus`), per-cell `linux/amd64`/`linux/arm64` platforms with one producer installation and provider identity per platform, a `release` profile (v3, one repetition) and a hash-bound external v2 baseline. The `full` profile matrix is byte-identical to before. +- **Evaluation-only helper builds.** `bun run build:cso -- --evaluation-candidate --output ` builds a helper unit from a `cso-eval-` candidate catalog outside the checkout. It reports `evaluationOnly: true`; setup, promotion and normal builds refuse it. +- **The ingress checks the staging run itself.** `cso-runtime-qualification.yml` resolves the staging run from verified provenance and requires a successful protected-main `workflow_dispatch` run whose ten `qualify-native` jobs all passed. A dispatch summary job shows the approver the claimed images and an optional `evaluationRef` before approval. +- **Native gate evidence comes from test results.** Booleans are derived from JUnit executed-and-passed counts (`scripts/cso-native-evidence.ts`); skipped Docker tests no longer read as passing, and PostgreSQL rows list only the checks they need. +- **Catalog promotion PRs get their required checks.** Both promote jobs dispatch `free-tests.yml` on the new branch. +- **Multi-payload security proofs.** A verification request's `security` may hold 1–8 assertions judged in one before/after pair; a single assertion hashes exactly as before. +- **Requalification triggers.** Qualified catalogs record the helper ABI, isolation policy hash, image preparation digest and build-inputs revision; a free test fails when the committed catalog no longer matches the code. + ## [1.91.22.0] - 2026-10-05 **`/cso` scanner execution has reviewed, signed scanner images to build from.** diff --git a/VERSION b/VERSION index c55ab8cf7..a47cfa8fd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.22.0 +1.91.23.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 556b33e73..f60ca1b16 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.22.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.23.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/cso-release-qualification.md b/docs/cso-release-qualification.md index e13dd8e0d..7bfdf6eb2 100644 --- a/docs/cso-release-qualification.md +++ b/docs/cso-release-qualification.md @@ -117,7 +117,20 @@ the embedded verifier against a loopback service with both a passing control and an intentionally failing control, alongside the containment suite. After all staging rows finish, native qualification jobs run the matching cold-start journey described above. The Node row additionally runs the complete lifecycle -journey. +journey. Each native job is named `qualify-native ` and runs the +Docker suites with Bun's JUnit reporter +(`--reporter=junit --reporter-outfile=native-tests.junit.xml`). +[`cso-native-evidence.ts`](../scripts/cso-native-evidence.ts) maps every native +check key to the named tests that prove it for that stack and records the key +true only when the report has no failed testcase and each named test appears +once, passed, and made at least one assertion. A test that the environment +skips, such as the staged-runtime smoke test without `GSTACK_CSO_TEST_IMAGE`, +reports as skipped and leaves its checks false; any false native check fails the +job. The pending private checks are the stack's required release gates from +`cso-runtime-promotion.ts` minus its native keys: `secretCanaryPassed` and +`watchdogCleanupPassed` for PostgreSQL, plus `heldOutRepairPassed` and +`accuracyGatesPassed` for application stacks. The job uploads the JUnit report +beside `qualification-evidence.json`, which records its digest. Registry publication is staging, not runtime qualification. GitHub Container Registry creates new packages private. Before a staging digest @@ -135,6 +148,12 @@ staging, qualification, and promotion. Both runtime and scanner promotion repeat this anonymous pull check on fresh hosted runners before opening a catalog PR. +The first protected-main staging run therefore stops at the public-package +check by design: its push creates the ten `cso-staging` packages as private. A +package administrator makes those ten packages public, then dispatches the +staging workflow again. Later runs publish new digests into the same public +packages. + Review the retained staging artifacts together with all checks in [`qualification.json`](../lib/cso/images/qualification.json). Cold-start fixtures, held-out repair evaluations, precision/recall thresholds, crash cleanup, and all @@ -150,7 +169,16 @@ the committed build profiles, and emits an attested candidate attestation against its workflow identity, protected-main source ref, source commit, and exact file digest. It then compares `previousRevision` with the source catalog, commits those same bytes to a fresh branch, and opens a -normal review PR. A concurrent catalog promotion fails the compare-and-swap or +normal review PR. Pushes and pull requests made with the workflow's +`GITHUB_TOKEN` start no workflows, so after pushing the branch the job dispatches +[`free-tests.yml`](../.github/workflows/free-tests.yml) on it with +`gh workflow run free-tests.yml --ref ` (its `actions: write` grant). +That run reports the required `free-tests` check on the branch head the PR +proposes. Opening the PR also requires the repository setting **Allow GitHub +Actions to create and approve pull requests** (Settings → Actions → General → +Workflow permissions). Without it `gh pr create` fails after the push and the +dispatch, and a maintainer opens the PR from the pushed branch. A concurrent +catalog promotion fails the compare-and-swap or produces a merge conflict instead of silently replacing the newer matrix. The catalog stores a separately recomputable digest of the retained runtime matrix and the digest of the complete external qualification statements. The @@ -163,16 +191,99 @@ The public qualification ingress is Set the protected environment variable `CSO_QUALIFICATION_ACTOR` to the GitHub service account used by the private evaluator. After its held-out assertions and accuracy run pass, that account sends a `repository_dispatch` event of type -`cso-runtime-qualified`. The payload has one key, `statements`, containing ten -objects with `schemaVersion`, `helperAbi`, `state`, `buildRevision`, -`runtimeId`, `stack`, `platform`, immutable `image`, exact `versions`, -`sourceCommit`, and the true-valued `checks` accepted by -`cso-runtime-promotion.ts`. The ingress checks the actor and matrix size, -re-verifies both OCI attestations against the protected staging workflow and -source commit, replaces the workflow, time, SBOM digest, and provenance digest -with values it observed, then runs the catalog generator as its final schema -gate. It uploads the statements only after that gate passes. The payload carries -no held-out assertion, application source, finding, or repair bundle. +`cso-runtime-qualified`. The payload has the key `statements` and, optionally, +`evaluationRef`; no other key is accepted. `statements` contains ten objects +with `schemaVersion`, `helperAbi`, `state`, `buildRevision`, `runtimeId`, +`stack`, `platform`, immutable `image`, exact `versions`, `sourceCommit`, and +the true-valued `checks` accepted by `cso-runtime-promotion.ts`. All ten name +one `sourceCommit`. `evaluationRef` is an opaque `sha256:<64 hex>` reference +that lets the approver match the dispatch to the private evaluator's verdict. +An unprotected `dispatch-summary` job with no token permissions validates the +payload shape and writes `evaluationRef`, the source commit, and each +`runtimeId` and `image` pair to the run's job summary. The reviewer reads it +there before approving the protected `qualify` job. `evaluationRef` never enters +the normalized statements, the uploaded evidence, or the catalog. + +The protected job checks the actor, re-verifies both OCI attestations against +the protected staging workflow and source commit, and replaces the workflow, +time, SBOM digest, and provenance digest with values it observed. Attestations +are signed in the staging job, before native qualification runs, so a valid +attestation alone does not show that native gates passed. +[`cso-staging-run.ts`](../scripts/cso-staging-run.ts) therefore resolves the +staging run from each verified SLSA provenance statement: the predicate's +`runDetails.metadata.invocationId` must equal the signing certificate's +`runInvocationURI`, and all ten images must resolve to the same run of this +repository. Through the Actions API (`actions: read`) the ingress then requires +that run to be `.github/workflows/cso-runtime-images.yml`, started by +`workflow_dispatch` on `main` in this repository, completed with conclusion +`success`, and built from the statements' `sourceCommit`. The latest attempt's +`qualify-native ` jobs, read across every page of the jobs API, must +be exactly one per committed matrix row (ten) and all `success`. The run and job +records are retained as `staging-run-evidence`. The catalog generator runs last +as the final schema gate, and the ingress uploads the statements only after that +gate passes. The payload carries no held-out assertion, application source, +finding, or repair bundle. + +A qualification stays valid only while the inputs it was measured under are +unchanged. [`cso-requalification.ts`](../scripts/cso-requalification.ts) defines +four requalification triggers: the helper ABI (`CSO_HELPER_ABI`), the isolation +policy (`ISOLATION_POLICY_HASH`, which covers the `cso-isolation-v1` policy +version and every limit), `preparationSha256` (a digest of every source byte +compiled or copied into a runtime image: the import closure of the image's +verifier and preparation helpers plus the image build context), and the +`build-inputs.json` revision. The staging run's `reviewed-inputs` job records +these values for its source commit as the `cso-requalification-triggers` +artifact. The ingress downloads that artifact from the verified staging run +into the evidence, and the promotion generator writes it into +`promotion.requalification`. Generation refuses triggers that differ from the +promoting checkout, so a trigger that changed between staging and promotion +requires a new staging run. The free suite's +[`cso-requalification.test.ts`](../test/cso-requalification.test.ts) compares +the committed catalog's recorded triggers with the current source and fails +when any differs. A change to a trigger therefore either lands with a +requalified catalog or withdraws the catalog's runtimes in the same pull +request. Independently, the helper refuses to select a runtime recorded under +another isolation policy. + +The judge observes held-out attacks in one boot: a `VerificationRequest`'s +`security` field is either one assertion or an array of up to eight assertions +with distinct names, judged in a single before/after pair. The before phase is +`intended_failure` only when every assertion shows its vulnerable expectation, +and the after phase passes only when every assertion passes its fixed +expectation. A single assertion keeps its original hashes. The verifier is +compiled into the runtime images, so array requests need images staged from a +commit that includes it. + +### Evaluation-only helper builds + +The private evaluator drives the real helper against staged digests before +any gate is measured. It never edits the committed catalog. Instead, +`bun scripts/cso-runtime-promotion.ts evaluation-candidate --staged-root +--output ` converts one staging run's ten `staged-image.json` records +into an evaluation-only catalog. Its revision is `cso-eval--`, it +carries an `evaluation` record instead of `promotion`, and every runtime is an +`evaluation_candidate` whose qualification has `kind: "evaluation"` and no +release-gate field. Then + +```bash +bun run build:cso -- --evaluation-candidate --output +``` + +compiles the helper core and `cso-eval-producer` from a private copy of the +source with that catalog embedded, adds the native launcher and watchdog, and +writes the five-artifact producer unit to a new directory outside the checkout. +Beside the unchanged `.gstack-cso-generation` manifest, it writes +`.gstack-cso-evaluation`, which marks the unit evaluation-only and records the +catalog revision, catalog SHA-256, and core SHA-256. The helper reports +`"evaluationOnly": true` from `--version`, labels its catalog evaluation-only +in runtime readiness, and refuses `provision-images --setup-summary`. +Distribution refuses it at every step. A normal `build:cso` stops when +`lib/cso/runtime-catalog.json` carries a `cso-eval-` revision. Setup fails when +`bin/` holds an `.gstack-cso-evaluation` manifest or a launcher that reports +`evaluationOnly`. Catalog promotion and transitions reject evaluation catalogs, +and the runtime catalog validator rejects an evaluation catalog that carries +`promotion`, release-gate fields, or a revision that does not name its staging +run and source commit. The current reviewed profiles and unpromoted catalog establish this release process; they do not claim published gstack images, passed containment, or diff --git a/docs/cso-scanners.md b/docs/cso-scanners.md index 342ef23ee..f1a904c7d 100644 --- a/docs/cso-scanners.md +++ b/docs/cso-scanners.md @@ -99,7 +99,15 @@ deployments restricted to protected `main`. GitHub otherwise creates a reference missing environment without protection rules. Promotion requires an explicit dispatch from `main` and approval through that configured environment. It opens a pull request containing only the proposed source catalog, so code review remains the final -authorization step. `previousRevision` records the prior compatible catalog for +authorization step. Pushes and pull requests made with the workflow's +`GITHUB_TOKEN` start no workflows, so after pushing the promotion branch the job +dispatches [`free-tests.yml`](../.github/workflows/free-tests.yml) on it +(`actions: write`); that run reports the required `free-tests` check on the +branch head. Opening the PR also requires the repository setting **Allow GitHub +Actions to create and approve pull requests** (Settings → Actions → General → +Workflow permissions); without it `gh pr create` fails after the push and the +dispatch, and a maintainer opens the PR from the pushed branch. +`previousRevision` records the prior compatible catalog for rollback and must still equal the current `main` catalog when promotion runs. Scanner flags follow the primary documentation linked in diff --git a/lib/cso/cli.ts b/lib/cso/cli.ts index e42008414..f27b1db8d 100644 --- a/lib/cso/cli.ts +++ b/lib/cso/cli.ts @@ -61,6 +61,7 @@ import { executable, git, redact, sanitizeForJson, sanitizeHelperForJson } from import { inspectPreparation } from './preparation'; import { assertRuntimeCompatible, + isEvaluationRuntimeCatalog, RUNTIME_CATALOG, selectRuntime, validateRuntimeCatalog, @@ -307,7 +308,7 @@ const SCHEMA = { }, ], security: { - name: 'security assertion', + name: 'security assertion; or send an array of 1..8 assertions with distinct names, all judged in one before/after boot pair', path: '/path', method: 'GET|POST|PUT|PATCH|DELETE', expected: { status: 'fixed status', includes: 'optional', excludes: 'optional' }, @@ -669,7 +670,9 @@ async function start( try { validateRuntimeCatalog(dependencies.runtimeCatalog); selectRuntime(plan.runtimeProfile, platform(), dependencies.runtimeCatalog); - c.evidence.push(`Qualified runtime catalog: ${dependencies.runtimeCatalog.revision}`); + c.evidence.push( + `${isEvaluationRuntimeCatalog(dependencies.runtimeCatalog) ? 'Evaluation-only' : 'Qualified'} runtime catalog: ${dependencies.runtimeCatalog.revision}`, + ); } catch (error: any) { c.gaps.push( error?.message?.startsWith('MISSING_QUALIFIED_RUNTIME') @@ -864,6 +867,8 @@ async function provisionImages(args: string[], dependencies: CsoCliDependencies) const setupSummary = take(args, '--setup-summary'), requestedSeconds = args.includes('--per-image-seconds') ? need(args, '--per-image-seconds') : undefined; if (args.length) throw new CsoError('INVALID_ARGUMENT', `Unknown argument: ${args[0]}`); + if (setupSummary && isEvaluationRuntimeCatalog(dependencies.runtimeCatalog)) + throw new CsoError('INCOMPATIBLE_INPUT', 'Setup never installs an evaluation-only CSO helper build'); if (requestedSeconds !== undefined) catalogImageProvisioningPolicy(0, requestedSeconds); let targetPlatform: 'linux/amd64' | 'linux/arm64'; try { @@ -2740,7 +2745,11 @@ async function main() { return; } if (command === '--version') { - emit({ version: VERSION, abi: ABI }); + emit({ + version: VERSION, + abi: ABI, + ...(isEvaluationRuntimeCatalog(RUNTIME_CATALOG) ? { evaluationOnly: true } : {}), + }); return; } if (command === 'schema') { diff --git a/lib/cso/contracts.ts b/lib/cso/contracts.ts index 52610986f..743537345 100644 --- a/lib/cso/contracts.ts +++ b/lib/cso/contracts.ts @@ -184,7 +184,7 @@ export interface VerificationRequest { port: number; start: Command; legitimate: HttpAssertion[]; - security: HttpAssertion; + security: HttpAssertion | HttpAssertion[]; existingTests: Command[]; fixtures: Record; boundaryFiles: string[]; @@ -678,6 +678,22 @@ function assertion(value: unknown, name: string): HttpAssertion { ...(v.vulnerable === undefined ? {} : { vulnerable: oracle(object(v.vulnerable), `${name}.vulnerable`) }), }; } +export const MAX_SECURITY_ASSERTIONS = 8; +export function securityAssertions(security: VerificationRequest['security']): HttpAssertion[] { + return Array.isArray(security) ? security : [security]; +} +function securityAssertion(value: unknown): VerificationRequest['security'] { + if (!Array.isArray(value)) return assertion(value, 'security'); + if (!value.length || value.length > MAX_SECURITY_ASSERTIONS) + throw new CsoError( + 'INVALID_SCHEMA', + `security must be one assertion or 1..${MAX_SECURITY_ASSERTIONS} assertions`, + ); + const assertions = value.map((x, i) => assertion(x, `security[${i}]`)); + if (new Set(assertions.map((x) => x.name)).size !== assertions.length) + throw new CsoError('INVALID_SCHEMA', 'security assertion names must be distinct'); + return assertions; +} export function validateVerificationRequest(input: unknown): VerificationRequest { const v = object(input, 'verification request'), changes = v.changes, @@ -729,7 +745,7 @@ export function validateVerificationRequest(input: unknown): VerificationRequest legitimate: (Array.isArray(v.legitimate) ? v.legitimate : []).map((x, i) => assertion(x, `legitimate[${i}]`), ), - security: assertion(v.security, 'security'), + security: securityAssertion(v.security), existingTests: (Array.isArray(v.existingTests) ? v.existingTests : []).map((x, i) => validateCommand(x, `existingTests[${i}]`), ), @@ -770,7 +786,7 @@ export function validateVerificationRequest(input: unknown): VerificationRequest throw new CsoError('INVALID_SCHEMA', 'port must be 1024..65535'); if ( !request.legitimate.length || - !request.security.vulnerable || + securityAssertions(request.security).some((x) => !x.vulnerable) || !request.existingTests.length || !request.boundaryFiles.length || !request.testFiles.length @@ -779,17 +795,15 @@ export function validateVerificationRequest(input: unknown): VerificationRequest 'INVALID_SCHEMA', 'Verification needs a legitimate control, distinct before/fixed security oracles, existing tests, immutable test files, and boundary files', ); - const secure = request.security.expected, - vulnerable = request.security.vulnerable; - const mutuallyExclusive = - secure.status !== vulnerable.status || + const mutuallyExclusive = ({ expected: secure, vulnerable }: HttpAssertion) => + secure.status !== vulnerable!.status || (secure.includes !== undefined && - vulnerable.excludes !== undefined && - secure.includes.includes(vulnerable.excludes)) || - (vulnerable.includes !== undefined && + vulnerable!.excludes !== undefined && + secure.includes.includes(vulnerable!.excludes)) || + (vulnerable!.includes !== undefined && secure.excludes !== undefined && - vulnerable.includes.includes(secure.excludes)); - if (!mutuallyExclusive) + vulnerable!.includes.includes(secure.excludes)); + if (!securityAssertions(request.security).every(mutuallyExclusive)) throw new CsoError( 'INVALID_SCHEMA', 'The vulnerable and fixed security oracles must be provably mutually exclusive', diff --git a/lib/cso/runtime-catalog.ts b/lib/cso/runtime-catalog.ts index e1859bcc9..b0ddfde07 100644 --- a/lib/cso/runtime-catalog.ts +++ b/lib/cso/runtime-catalog.ts @@ -2,6 +2,7 @@ import committedCatalog from './runtime-catalog.json'; import type { CsoStack, PreparationPlan } from './preparation'; import { CsoError, canonical, sha256 } from './contracts'; +import { ISOLATION_POLICY_HASH } from './docker'; export const CSO_HELPER_ABI = 3; export type RuntimePlatform = 'linux/amd64' | 'linux/arm64'; @@ -29,12 +30,15 @@ export type RuntimeQualification = RuntimeQualificationProvenance & multiDatabasePassed: true; readinessPassed: true; } + /** Staged and attested, with no release gate measured. Only evaluation catalogs carry it. */ + | { kind: 'evaluation' } ); export interface QualifiedRuntime { id: string; stack: CsoStack | 'postgresql'; platform: RuntimePlatform; - state: 'qualified'; + /** evaluation_candidate appears only in evaluation-only catalogs, never in a distributed helper. */ + state: 'qualified' | 'evaluation_candidate'; image: string; entrypoint: '/opt/cso/entrypoint'; helperAbi: number; @@ -43,6 +47,18 @@ export interface QualifiedRuntime { qualifiedAt: string; qualification: RuntimeQualification; } +/** + * The inputs a qualification was measured under. Changing any of them after + * promotion makes the catalog older than its inputs: requalify or withdraw. + */ +export interface RequalificationTriggers { + helperAbi: number; + /** ISOLATION_POLICY_HASH, which covers the cso-isolation-v1 policy version and every limit. */ + isolationPolicyHash: string; + /** Digest of every source byte compiled or copied into the runtime images (preparation and verifier helpers). */ + preparationSha256: string; + buildInputsRevision: string; +} /** Reviewed build metadata is informative. It never makes an image executable. */ export interface ReviewedRuntimeProfile { id: string; @@ -66,10 +82,28 @@ export interface RuntimeCatalog { evidenceDigest: string; /** Canonical digest of the complete release-gate statements retained externally. */ qualificationEvidenceDigest: string; + /** Trigger inputs measured by the staging run at sourceCommit. */ + requalification: RequalificationTriggers; + }; + /** + * Present only on an evaluation-only catalog (revision cso-eval--): + * staged digests from one protected staging run, built into a private + * evaluation helper. Setup, distribution builds and promotion refuse it. + */ + evaluation?: { + sourceCommit: string; + workflow: string; + /** Canonical digest of the staged runtime matrix. */ + evidenceDigest: string; }; runtimes: QualifiedRuntime[]; } +export const EVALUATION_REVISION_PREFIX = 'cso-eval-'; +export function isEvaluationRuntimeCatalog(catalog: Pick): boolean { + return typeof catalog?.revision === 'string' && catalog.revision.startsWith(EVALUATION_REVISION_PREFIX); +} + const DIGEST = /^sha256:[a-f0-9]{64}$/; const IMAGE = /^(?:[a-z0-9.-]+(?::[0-9]+)?\/)?[a-z0-9][a-z0-9._/-]*@sha256:[a-f0-9]{64}$/; const ID = /^[a-z0-9][a-z0-9._-]{0,100}$/; @@ -120,6 +154,20 @@ function validateRuntimeIdentity(value: { throw new Error('INCOMPATIBLE_PREPARATION_HELPER'); } +export function validRequalificationTriggers(value: unknown, buildRevision: string): boolean { + const triggers = value as RequalificationTriggers; + return ( + !!triggers && + typeof triggers === 'object' && + Object.keys(triggers).sort().join(',') === + 'buildInputsRevision,helperAbi,isolationPolicyHash,preparationSha256' && + triggers.helperAbi === CSO_HELPER_ABI && + /^[a-f0-9]{64}$/.test(triggers.isolationPolicyHash) && + /^[a-f0-9]{64}$/.test(triggers.preparationSha256) && + triggers.buildInputsRevision === buildRevision + ); +} + export function validateRuntimeCatalog(value: unknown): asserts value is RuntimeCatalog { const catalog = value as RuntimeCatalog; if ( @@ -131,6 +179,7 @@ export function validateRuntimeCatalog(value: unknown): asserts value is Runtime !Array.isArray(catalog.runtimes) ) throw new Error('INCOMPATIBLE_RUNTIME_CATALOG'); + const evaluation = isEvaluationRuntimeCatalog(catalog); if ( catalog.previousRevision !== null && (typeof catalog.previousRevision !== 'string' || !BUILD_REVISION.test(catalog.previousRevision)) @@ -170,8 +219,11 @@ export function validateRuntimeCatalog(value: unknown): asserts value is Runtime !QUALIFICATION_WORKFLOW.test(catalog.promotion.workflow) || !DIGEST.test(catalog.promotion.evidenceDigest) || !DIGEST.test(catalog.promotion.qualificationEvidenceDigest) || + !validRequalificationTriggers(catalog.promotion.requalification, catalog.buildRevision) || Object.keys(catalog.promotion).sort().join(',') !== - ['evidenceDigest', 'qualificationEvidenceDigest', 'sourceCommit', 'workflow'].sort().join(',') + ['evidenceDigest', 'qualificationEvidenceDigest', 'requalification', 'sourceCommit', 'workflow'] + .sort() + .join(',') ) { throw new Error('INVALID_RUNTIME_PROMOTION'); } @@ -194,7 +246,7 @@ export function validateRuntimeCatalog(value: unknown): asserts value is Runtime `^ghcr\\.io/garrytan/gstack/cso-staging/${runtime.stack}-${arch}@sha256:[a-f0-9]{64}$`, ); if ( - runtime.state !== 'qualified' || + runtime.state !== (evaluation ? 'evaluation_candidate' : 'qualified') || !IMAGE.test(runtime.image) || !expectedImage.test(runtime.image) || runtime.entrypoint !== '/opt/cso/entrypoint' || @@ -229,7 +281,10 @@ export function validateRuntimeCatalog(value: unknown): asserts value is Runtime 'provenanceDigest', 'verifiedProvenance', ]; - if (['node', 'bun', 'python', 'rails'].includes(runtime.stack)) { + if (evaluation) { + if (qualification.kind !== 'evaluation' || keys.join(',') !== [...common].sort().join(',')) + throw new Error('INVALID_EVALUATION_RUNTIME'); + } else if (['node', 'bun', 'python', 'rails'].includes(runtime.stack)) { if ( qualification.kind !== 'application' || qualification.containmentPassed !== true || @@ -266,6 +321,30 @@ export function validateRuntimeCatalog(value: unknown): asserts value is Runtime if (catalog.runtimes.length > 0) { for (const identity of profileIdentities) if (!runtimeIdentities.has(identity)) throw new Error('INCOMPLETE_QUALIFIED_RUNTIME_MATRIX'); + } + if (evaluation) { + const record = catalog.evaluation; + if ( + !record || + catalog.promotion !== undefined || + catalog.runtimes.length === 0 || + Object.keys(record).sort().join(',') !== 'evidenceDigest,sourceCommit,workflow' || + !/^[a-f0-9]{40}$/.test(record.sourceCommit) || + !QUALIFICATION_WORKFLOW.test(record.workflow) || + catalog.revision !== + `${EVALUATION_REVISION_PREFIX}${record.workflow.slice(record.workflow.lastIndexOf('/') + 1)}-${record.sourceCommit.slice(0, 12)}` || + catalog.runtimes.some( + (runtime) => + runtime.qualification.sourceCommit !== record.sourceCommit || + runtime.qualification.workflow !== record.workflow, + ) || + record.evidenceDigest !== `sha256:${sha256(canonical(catalog.runtimes))}` + ) + throw new Error('INVALID_EVALUATION_RUNTIME_CATALOG'); + return; + } + if (catalog.evaluation !== undefined) throw new Error('INVALID_EVALUATION_RUNTIME_CATALOG'); + if (catalog.runtimes.length > 0) { if (!catalog.promotion) throw new Error('MISSING_RUNTIME_PROMOTION'); if ( catalog.runtimes.some( @@ -326,6 +405,11 @@ export function selectRuntime( catalog: RuntimeCatalog = RUNTIME_CATALOG, ): QualifiedRuntime { validateRuntimeCatalog(catalog); + const recorded = catalog.promotion?.requalification; + if (recorded && recorded.isolationPolicyHash !== ISOLATION_POLICY_HASH) + throw new Error( + 'STALE_RUNTIME_QUALIFICATION: the isolation policy changed after these runtimes were qualified; requalify before target execution.', + ); const matches = catalog.runtimes.filter( (runtime) => runtime.platform === platform && (runtime.id === profile || runtime.stack === profile), ); diff --git a/lib/cso/verifier.ts b/lib/cso/verifier.ts index e14deb84d..f3364b764 100644 --- a/lib/cso/verifier.ts +++ b/lib/cso/verifier.ts @@ -1,13 +1,20 @@ #!/usr/bin/env bun import * as fs from 'node:fs'; import { connect } from 'node:net'; -import { HttpAssertion, VerificationObservation, object } from './contracts'; +import { + HttpAssertion, + MAX_SECURITY_ASSERTIONS, + VerificationObservation, + VerificationRequest, + object, + securityAssertions, +} from './contracts'; interface Config { phase: 'before' | 'after'; port: number; legitimate: HttpAssertion[]; - security: HttpAssertion; + security: VerificationRequest['security']; } function matches(status: number, body: string, oracle: HttpAssertion['expected']): boolean { return ( @@ -66,6 +73,34 @@ async function request(a: HttpAssertion, port: number): Promise<{ status: number clearTimeout(timer); } } +export async function judgeSecurity( + phase: Config['phase'], + security: Config['security'], + port: number, +): Promise<{ + security: VerificationObservation['security']; + members: VerificationObservation['security'][]; +}> { + const members: VerificationObservation['security'][] = []; + for (const assertion of securityAssertions(security)) { + const r = await request(assertion, port), + fixed = matches(r.status, r.body, assertion.expected), + vulnerable = matches(r.status, r.body, assertion.vulnerable!); + members.push( + phase === 'before' + ? vulnerable && !fixed + ? 'intended_failure' + : fixed && !vulnerable + ? 'pass' + : 'inconclusive' + : fixed && !vulnerable + ? 'pass' + : 'inconclusive', + ); + } + const agreed = members.every((member) => member === members[0]) ? members[0] : 'inconclusive'; + return { security: agreed, members }; +} async function ready(port: number): Promise { return await new Promise((resolve) => { const socket = connect({ host: '127.0.0.1', port }), @@ -97,7 +132,8 @@ async function main() { v.port < 1024 || v.port > 65535 || !Array.isArray(v.legitimate) || - !v.security + !v.security || + (Array.isArray(v.security) && (!v.security.length || v.security.length > MAX_SECURITY_ASSERTIONS)) ) throw new Error('invalid verifier policy'); const config = v as Config; @@ -122,20 +158,11 @@ async function main() { }), ) ).every(Boolean); - const r = await request(config.security, config.port), - fixed = matches(r.status, r.body, config.security.expected), - vulnerable = matches(r.status, r.body, config.security.vulnerable!); - security = - config.phase === 'before' - ? vulnerable && !fixed - ? 'intended_failure' - : fixed && !vulnerable - ? 'pass' - : 'inconclusive' - : fixed && !vulnerable - ? 'pass' - : 'inconclusive'; - summary = `boot=true legitimate=${legitimate} security=${security}`; + const judged = await judgeSecurity(config.phase, config.security, config.port); + security = judged.security; + summary = `boot=true legitimate=${legitimate} security=${security}${ + Array.isArray(config.security) ? ` members=${judged.members.join(',')}` : '' + }`; } catch { summary = 'bounded verifier request failed'; } diff --git a/package.json b/package.json index 21e8ad127..79be89e1e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.22", + "version": "1.91.23", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/build-cso.sh b/scripts/build-cso.sh index bc7a692d6..e0d1084d6 100644 --- a/scripts/build-cso.sh +++ b/scripts/build-cso.sh @@ -27,6 +27,7 @@ CSO_NEW_LAUNCHER_INSTALLED=0 CSO_NEW_CORE_INSTALLED=0 CSO_NEW_WATCHDOG_INSTALLED=0 CSO_NEW_GENERATION_INSTALLED=0 +CSO_EVALUATION_UNIT="" cso_checkpoint() { [ "${GSTACK_CSO_BUILD_TESTING:-0}" = 1 ] || return 0 @@ -87,6 +88,7 @@ cso_cleanup() { if ! cso_restore_previous; then status=1; cleanup_stage=0; fi fi if [ "$cleanup_stage" -eq 1 ] && [ -n "$CSO_STAGE" ] && [ ! -f "$CSO_STAGE/.retain-recovery" ]; then rm -rf "$CSO_STAGE"; fi + if [ -n "$CSO_EVALUATION_UNIT" ]; then rm -rf "$CSO_EVALUATION_UNIT"; fi exit "$status" } trap 'cso_cleanup $?' EXIT @@ -143,6 +145,89 @@ cso_sha256() { printf '%s\n' "$digest" } +# Bun's compile-time switches stop configuration discovery in the audited +# working directory before the helper can apply its own policy. +cso_compile_bun() { + "$BUN_CMD" build --compile \ + --no-compile-autoload-dotenv \ + --no-compile-autoload-bunfig \ + --no-compile-autoload-tsconfig \ + --no-compile-autoload-package-json \ + "$1" --outfile "$2" + chmod +x "$2" +} +cso_compile_watchdog() { + "$CSO_CC" -std=c11 -D_POSIX_C_SOURCE=200809L -O2 -Wall -Wextra \ + lib/cso/watchdog.c -o "$1" + chmod +x "$1" +} +cso_compile_posix_launcher() { + CSO_LAUNCHER_FLAGS="" + case "$(uname -s)" in + Linux) CSO_LAUNCHER_FLAGS="-static" ;; + # Local ad-hoc signatures do not distinguish the launcher from another + # ad-hoc dylib. This section makes dyld prune DYLD_* before constructors. + Darwin) CSO_LAUNCHER_FLAGS="-Wl,-sectcreate,__RESTRICT,__restrict,/dev/null" ;; + esac + # shellcheck disable=SC2086 -- the optional platform flags are fixed literals. + "$CSO_CC" -std=c11 -D_POSIX_C_SOURCE=200809L -O2 -Wall -Wextra $CSO_LAUNCHER_FLAGS \ + "-DGSTACK_CSO_CORE_SHA256=\"$2\"" lib/cso/launcher.c -o "$1" +} + +# An evaluation candidate embeds a cso-eval-* runtime catalog in a private +# five-artifact producer unit outside this checkout. It never touches bin/, and +# its .gstack-cso-evaluation manifest marks it evaluation-only for setup. +cso_build_evaluation_candidate() { + [ "$#" -eq 4 ] && [ "$1" = --evaluation-candidate ] && [ "$3" = --output ] || { + echo 'Usage: build-cso.sh --evaluation-candidate --output ' >&2;return 64; } + [ -z "$CSO_EXE" ] || { echo 'CSO evaluation candidates are built only on Linux and macOS.' >&2;return 69; } + command -v "$CSO_CC" >/dev/null 2>&1 || { echo 'CSO build requires a C compiler (cc/clang/gcc) for the trusted launcher and watchdog.' >&2;return 1; } + candidate_catalog=$2 + [ -f "$candidate_catalog" ] && [ ! -L "$candidate_catalog" ] || { echo 'CSO evaluation catalog must be one regular file.' >&2;return 66; } + candidate_parent="$(cd "$(dirname "$4")" 2>/dev/null && pwd -P)" || { echo 'CSO evaluation output parent must be an existing directory.' >&2;return 73; } + candidate_output="$candidate_parent/$(basename "$4")" + if cso_present "$candidate_output";then echo 'CSO evaluation output must be a new directory.' >&2;return 73;fi + case "$candidate_output/" in "$CSO_BUILD_ROOT"/*) echo 'CSO evaluation output must be outside the gstack checkout, where setup and distribution never look.' >&2;return 73;;esac + candidate_revision="$(grep -o '"revision": *"cso-eval-[0-9]\{1,\}-[a-f0-9]\{12\}"' "$candidate_catalog" | head -n 1 | sed 's/.*"\(cso-eval-[^"]*\)"$/\1/' || true)" + [ -n "$candidate_revision" ] || { echo 'CSO evaluation candidates require a cso-eval-- catalog revision.' >&2;return 65; } + + CSO_STAGE="$(mktemp -d "${TMPDIR:-/tmp}/gstack-cso-evaluation.XXXXXX")" + CSO_EVALUATION_UNIT="$(mktemp -d "$candidate_parent/.gstack-cso-evaluation.XXXXXX")" + source_root="$CSO_STAGE/source" + mkdir -m 700 "$source_root" "$source_root/test" + cp -R lib scripts "$source_root/" + cp -R test/helpers "$source_root/test/" + cp "$candidate_catalog" "$source_root/lib/cso/runtime-catalog.json" + (cd "$source_root" && "$BUN_CMD" scripts/cso-runtime-promotion.ts check-evaluation lib/cso/runtime-catalog.json) + (cd "$source_root" && cso_compile_bun lib/cso/cli.ts "$CSO_EVALUATION_UNIT/gstack-cso-core") + (cd "$source_root" && cso_compile_bun scripts/cso-eval-producer.ts "$CSO_EVALUATION_UNIT/cso-eval-producer") + cso_compile_watchdog "$CSO_EVALUATION_UNIT/gstack-cso-watchdog" + if [ "$(uname -s)" = Darwin ]; then + cso_sign_macos_artifact "$CSO_EVALUATION_UNIT/gstack-cso-core" 0 + cso_sign_macos_artifact "$CSO_EVALUATION_UNIT/gstack-cso-watchdog" 0 + fi + candidate_core_sha256="$(cso_sha256 "$CSO_EVALUATION_UNIT/gstack-cso-core")" + cso_compile_posix_launcher "$CSO_EVALUATION_UNIT/gstack-cso-launcher" "$candidate_core_sha256" + chmod +x "$CSO_EVALUATION_UNIT/gstack-cso-launcher" + if [ "$(uname -s)" = Darwin ];then cso_sign_macos_artifact "$CSO_EVALUATION_UNIT/gstack-cso-launcher" 1;fi + printf '%s\n' "$candidate_core_sha256" > "$CSO_EVALUATION_UNIT/.gstack-cso-generation" + printf '{"schemaVersion":1,"evaluationOnly":true,"runtimeCatalogRevision":"%s","runtimeCatalogSha256":"%s","coreSha256":"%s"}\n' \ + "$candidate_revision" "$(cso_sha256 "$source_root/lib/cso/runtime-catalog.json")" "$candidate_core_sha256" \ + > "$CSO_EVALUATION_UNIT/.gstack-cso-evaluation" + for artifact in gstack-cso-core gstack-cso-launcher gstack-cso-watchdog cso-eval-producer;do + cso_valid_artifact "$CSO_EVALUATION_UNIT/$artifact" || { echo "Evaluation $artifact is not one executable regular file." >&2;return 1; } + done + cso_valid_generation "$CSO_EVALUATION_UNIT/.gstack-cso-generation" || { echo 'Evaluation generation manifest is invalid.' >&2;return 1; } + [ "$(cso_sha256 "$CSO_EVALUATION_UNIT/gstack-cso-core")" = "$candidate_core_sha256" ] || { echo 'Evaluation core changed after launcher binding.' >&2;return 1; } + "$CSO_EVALUATION_UNIT/gstack-cso-launcher" --version >/dev/null + chmod 0555 "$CSO_EVALUATION_UNIT"/gstack-cso-core "$CSO_EVALUATION_UNIT"/gstack-cso-launcher "$CSO_EVALUATION_UNIT"/gstack-cso-watchdog "$CSO_EVALUATION_UNIT"/cso-eval-producer + chmod 0444 "$CSO_EVALUATION_UNIT/.gstack-cso-generation" "$CSO_EVALUATION_UNIT/.gstack-cso-evaluation" + chmod 0755 "$CSO_EVALUATION_UNIT" + mv "$CSO_EVALUATION_UNIT" "$candidate_output" + CSO_EVALUATION_UNIT="" + printf '{"output":"%s","revision":"%s","evaluationOnly":true}\n' "$candidate_output" "$candidate_revision" +} + cso_publish_locked() { [ "${GSTACK_CSO_PUBLISH_LOCKED:-}" = 1 ] || { echo 'CSO publication requires the native publisher lock.' >&2;return 73; } [ "$#" -eq 1 ] && [ -d "$1" ] && [ ! -L "$1" ] || { echo 'CSO publication stage is invalid.' >&2;return 69; } @@ -201,6 +286,16 @@ if [ "${1:-}" = __publish_locked ];then cso_publish_locked "$@" exit 0 fi +if [ "${1:-}" = --evaluation-candidate ];then + cso_build_evaluation_candidate "$@" + exit 0 +fi +[ "$#" -eq 0 ] || { echo 'Usage: build-cso.sh [--evaluation-candidate --output ]' >&2;exit 64; } +# Distribution builds embed only the committed catalog, never an evaluation one. +if [ -f lib/cso/runtime-catalog.json ] && grep -q '"revision": *"cso-eval-' lib/cso/runtime-catalog.json;then + echo 'lib/cso/runtime-catalog.json is an evaluation-only catalog; distribution builds refuse it. Restore the committed catalog.' >&2 + exit 65 +fi # This entrypoint can be run directly. Once it may publish a CSO generation, # the previous whole-build completion proof no longer describes all outputs. @@ -226,21 +321,13 @@ if [ -n "$CSO_EXE" ];then : > "$CSO_STAGE/.gstack-cso-generation.lock";fi # These are compile-time switches. Bun otherwise discovers configuration in # the audited working directory before the helper can apply its own policy. -"$BUN_CMD" build --compile \ - --no-compile-autoload-dotenv \ - --no-compile-autoload-bunfig \ - --no-compile-autoload-tsconfig \ - --no-compile-autoload-package-json \ - lib/cso/cli.ts --outfile "$CSO_STAGE_CORE" -chmod +x "$CSO_STAGE_CORE" +cso_compile_bun lib/cso/cli.ts "$CSO_STAGE_CORE" cso_checkpoint core # POSIX process groups are required by the detached watchdog. Windows keeps # comprehensive execution unavailable instead of substituting a weaker timer. if [ -z "$CSO_EXE" ]; then - "$CSO_CC" -std=c11 -D_POSIX_C_SOURCE=200809L -O2 -Wall -Wextra \ - lib/cso/watchdog.c -o "$CSO_STAGE_WATCHDOG" - chmod +x "$CSO_STAGE_WATCHDOG" + cso_compile_watchdog "$CSO_STAGE_WATCHDOG" cso_checkpoint watchdog fi @@ -265,16 +352,7 @@ if [ -n "$CSO_EXE" ];then -CoreSha256 "$CSO_CORE_SHA256" \ -GitExePath "$(cygpath -aw "$CSO_WINDOWS_GIT")" else - CSO_LAUNCHER_FLAGS="" - case "$(uname -s)" in - Linux) CSO_LAUNCHER_FLAGS="-static" ;; - # Local ad-hoc signatures do not distinguish the launcher from another - # ad-hoc dylib. This section makes dyld prune DYLD_* before constructors. - Darwin) CSO_LAUNCHER_FLAGS="-Wl,-sectcreate,__RESTRICT,__restrict,/dev/null" ;; - esac - # shellcheck disable=SC2086 -- the optional platform flags are fixed literals. - "$CSO_CC" -std=c11 -D_POSIX_C_SOURCE=200809L -O2 -Wall -Wextra $CSO_LAUNCHER_FLAGS \ - "-DGSTACK_CSO_CORE_SHA256=\"$CSO_CORE_SHA256\"" lib/cso/launcher.c -o "$CSO_STAGE_LAUNCHER" + cso_compile_posix_launcher "$CSO_STAGE_LAUNCHER" "$CSO_CORE_SHA256" "$CSO_CC" -std=c11 -D_POSIX_C_SOURCE=200809L -O2 -Wall -Wextra lib/cso/publish-lock.c -o "$CSO_STAGE_LOCKER" fi chmod +x "$CSO_STAGE_LAUNCHER" "$CSO_STAGE_LOCKER" diff --git a/scripts/cso-eval-producer.ts b/scripts/cso-eval-producer.ts index 0162f8a35..d16c68dc2 100644 --- a/scripts/cso-eval-producer.ts +++ b/scripts/cso-eval-producer.ts @@ -17,6 +17,8 @@ import { GeminiAdapter, prepareGeminiProducerState, removeGeminiProducerState } import { PRICING } from '../test/helpers/pricing'; import type { ProviderAdapter, RunOpts, RunResult } from '../test/helpers/providers/types'; import { + PRODUCER_PLATFORMS, + producerHostPlatform, producerInputHash, producerArtifactInventoryHash, producerInstallationIdentityHash, @@ -198,7 +200,8 @@ function repositoryIdentity(root: string): string { } function validateCell(cell: ProducerCell): void { - if (!cell || !HEX.test(cell.id) || !cell.caseId || !['node', 'bun', 'python', 'rails'].includes(cell.stack) || !['vulnerable', 'fixed'].includes(cell.variant) || !['v2', 'v3'].includes(cell.version) || !['daily', 'comprehensive'].includes(cell.mode) || ![1, 2, 3].includes(cell.repetition) || !cell.model || !['claude', 'codex', 'gemini'].includes(cell.host) || !Number.isInteger(cell.budgetSeconds) || cell.budgetSeconds <= 60 || cell.budgetSeconds > 3600 || !HEX.test(cell.sourceHash) || !HEX.test(cell.skillHash)) throw new Error('INVALID_PRODUCER_INPUT'); + if (!cell || !HEX.test(cell.id) || !cell.caseId || !['node', 'bun', 'python', 'rails'].includes(cell.stack) || !['vulnerable', 'fixed'].includes(cell.variant) || !['v2', 'v3'].includes(cell.version) || !['daily', 'comprehensive'].includes(cell.mode) || ![1, 2, 3].includes(cell.repetition) || !cell.model || !['claude', 'codex', 'gemini'].includes(cell.host) || !Number.isInteger(cell.budgetSeconds) || cell.budgetSeconds <= 60 || cell.budgetSeconds > 3600 || !HEX.test(cell.sourceHash) || !HEX.test(cell.skillHash) || (cell.platform !== undefined && !PRODUCER_PLATFORMS.includes(cell.platform))) throw new Error('INVALID_PRODUCER_INPUT'); + if (cell.platform !== undefined && cell.platform !== producerHostPlatform()) throw new Error('UNMATCHED_PRODUCER_PLATFORM'); } export interface ProducerHelperBinding { diff --git a/scripts/cso-eval-protocol.ts b/scripts/cso-eval-protocol.ts index d6b71cbc3..827db2e75 100644 --- a/scripts/cso-eval-protocol.ts +++ b/scripts/cso-eval-protocol.ts @@ -5,6 +5,8 @@ export type ProducerVersion = 'v2' | 'v3'; export type ProducerMode = 'daily' | 'comprehensive'; export type ProducerStack = 'node' | 'bun' | 'python' | 'rails'; export type ProducerVariant = 'vulnerable' | 'fixed'; +export type ProducerPlatform = 'linux/amd64' | 'linux/arm64'; +export const PRODUCER_PLATFORMS: readonly ProducerPlatform[] = ['linux/amd64', 'linux/arm64']; export interface ProducerCell { id: string; @@ -19,6 +21,8 @@ export interface ProducerCell { budgetSeconds: number; sourceHash: string; skillHash: string; + /** Release and baseline cells pin the native producer platform; full-profile cells omit it. */ + platform?: ProducerPlatform; } export interface ProducerSourceEntry { @@ -122,6 +126,12 @@ export type ProducerReceiptIndex = Omit & { export const sha256 = (value: string | Buffer): string => createHash('sha256').update(value).digest('hex'); +/** The native platform this producer process runs on, or null when no CSO runtime platform matches it. */ +export function producerHostPlatform(platform: string = process.platform, arch: string = process.arch): ProducerPlatform | null { + if (platform !== 'linux') return null; + return arch === 'x64' ? 'linux/amd64' : arch === 'arm64' ? 'linux/arm64' : null; +} + export function producerInputHash(input: ProducerInput): string { return sha256(JSON.stringify(input)); } diff --git a/scripts/cso-eval.ts b/scripts/cso-eval.ts index 122a850dc..d5aa1516a 100644 --- a/scripts/cso-eval.ts +++ b/scripts/cso-eval.ts @@ -7,8 +7,9 @@ import { dirname, join, resolve } from 'node:path'; import { readBoundedStable } from '../lib/cso/bounded-file'; import { executable } from '../lib/cso/process'; import { atomicWriteSync } from '../lib/fs-atomic'; -import { loadCorpusManifest, materializeCase, sourceFiles, STACKS, type CorpusManifest, type EvalStack, type EvalVariant } from '../test/fixtures/cso-eval/materialize'; +import { loadCorpusManifest, materializeCase, publicCorpus, STACKS, validateCorpusManifest, type CorpusManifest, type EvalCorpus, type EvalStack, type EvalVariant } from '../test/fixtures/cso-eval/materialize'; import { + PRODUCER_PLATFORMS, producerArtifactInventoryHash, producerInputHash, producerInstallationIdentityHash, @@ -20,6 +21,7 @@ import { type ProducerHost, type ProducerInput, type ProducerInstallationIdentity, + type ProducerPlatform, type ProducerProviderIdentity, type ProducerReceipt, type ProducerReceiptIndex, @@ -29,13 +31,53 @@ import { export type EvalVersion = 'v2' | 'v3'; export type EvalMode = 'daily' | 'comprehensive'; export type Outcome = 'passed' | 'failed' | 'blocked' | 'not_attempted'; +/** + * full: matched v2/v3 × 3 repetitions on one producer installation. + * release: v3 only, 1 repetition, each cell pinned to a native platform. + * baseline: v2 comprehensive only, 1 repetition; scored into a reusable EvalBaseline. + */ +export type EvalProfile = 'full' | 'release' | 'baseline'; +export type EvalPlatform = ProducerPlatform; +type EvalCorpusManifest = CorpusManifest; export interface EvalCell extends ProducerCell { id: string; caseId: string; stack: EvalStack; variant: EvalVariant; version: EvalVersion; mode: EvalMode; - repetition: 1 | 2 | 3; model: string; host: ProducerHost; budgetSeconds: number; sourceHash: string; skillHash: string; + repetition: 1 | 2 | 3; model: string; host: ProducerHost; budgetSeconds: number; sourceHash: string; skillHash: string; platform?: EvalPlatform; } -export interface EvalMatrix { +interface EvalMatrixBase { schemaVersion: 1; corpusVersion: string; corpusHash: string; model: string; host: ProducerHost; - budgets: { daily: number; comprehensive: number }; skillHashes: { v2: string; v3: string }; repetitions: 3; cells: EvalCell[]; + budgets: { daily: number; comprehensive: number }; cells: EvalCell[]; +} +export interface FullEvalMatrix extends EvalMatrixBase { profile?: undefined; skillHashes: { v2: string; v3: string }; repetitions: 3 } +export interface ReleaseEvalMatrix extends EvalMatrixBase { + profile: 'release'; skillHashes: { v3: string }; repetitions: 1; + /** Daily cells run on one platform; each comprehensive case is fixed to one platform. */ + platforms: { daily: EvalPlatform; comprehensive: Record }; + /** Pre-registered external v2 baseline, or null when the regression gate stays unmeasured. */ + baselineHash: string | null; +} +export interface BaselineEvalMatrix extends EvalMatrixBase { + profile: 'baseline'; skillHashes: { v2: string }; repetitions: 1; + platforms: { comprehensive: Record }; +} +export type EvalMatrix = FullEvalMatrix | ReleaseEvalMatrix | BaselineEvalMatrix; +interface EvalMatrixCommonOptions { model: string; host: ProducerHost; budgets?: { daily: number; comprehensive: number } } +export interface FullEvalMatrixOptions extends EvalMatrixCommonOptions { profile?: 'full'; skillHashes: { v2: string; v3: string } } +export interface ReleaseEvalMatrixOptions extends EvalMatrixCommonOptions { profile: 'release'; skillHashes: { v3: string }; platforms: ReleaseEvalMatrix['platforms']; baseline?: EvalBaseline } +export interface BaselineEvalMatrixOptions extends EvalMatrixCommonOptions { profile: 'baseline'; skillHashes: { v2: string }; platforms: BaselineEvalMatrix['platforms'] } +export type EvalMatrixOptions = FullEvalMatrixOptions | ReleaseEvalMatrixOptions | BaselineEvalMatrixOptions; +/** + * A scored v2 comprehensive run, reused by release matrices while every bound + * input is unchanged. Release matrices pin its hash before results exist. + */ +export interface EvalBaseline { + schemaVersion: 1; version: 'v2'; + corpusVersion: string; corpusHash: string; model: string; host: ProducerHost; budgetSeconds: number; skillHash: string; + platforms: Record; + matrixHash: string; producerBatchHash: string; modelUsed: string; + provider: { family: ProducerProviderIdentity['family']; policyRevision: string; version: string }; + /** One entry per corpus case, in manifest order: whether the vulnerable cell found the seeded issue. */ + cases: Array<{ caseId: string; found: boolean }>; + baselineHash: string; } export interface PortableSkillFile { path: string; @@ -280,8 +322,8 @@ function initializeFixtureRepository(path: string): void { execFileSync(git, [...safe, 'commit', '--quiet', '-m', 'immutable evaluation fixture'], { cwd: path, env, stdio: 'ignore' }); } -function sourceEntries(caseId: string, variant: EvalVariant): ProducerSourceEntry[] { - return Object.entries(sourceFiles(caseId, variant)).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0).map(([path, contents]) => ({ path, sha256: sha256(contents), bytes: Buffer.byteLength(contents) })); +function sourceEntries(corpus: EvalCorpus, caseId: string, variant: EvalVariant): ProducerSourceEntry[] { + return Object.entries(corpus.sourceFiles(caseId, variant)).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0).map(([path, contents]) => ({ path, sha256: sha256(contents), bytes: Buffer.byteLength(contents) })); } /** @@ -289,10 +331,15 @@ function sourceEntries(caseId: string, variant: EvalVariant): ProducerSourceEntr * evaluator metadata; transfer one job at a time and never give the schedule * or sibling jobs to the producing agent. */ -export function prepareEvalJobs(matrix: EvalMatrix, payloads: { v2: string; v3: string }, destination: string, selectedCellIds?: string[]): PreparedEvalSchedule { - validateMatrix(matrix); - validatePortableSkillPayload(payloads.v2, 'v2'); validatePortableSkillPayload(payloads.v3, 'v3'); - if (sha256(payloads.v2) !== matrix.skillHashes.v2 || sha256(payloads.v3) !== matrix.skillHashes.v3) throw new Error('EVAL_SKILL_HASH_MISMATCH'); +export function prepareEvalJobs(matrix: EvalMatrix, payloads: Partial>, destination: string, selectedCellIds?: string[], corpus: EvalCorpus = publicCorpus()): PreparedEvalSchedule { + validateMatrix(matrix, corpus.manifest); + const skillHashes: Partial> = matrix.skillHashes; + const versions = (['v2', 'v3'] as const).filter(version => skillHashes[version] !== undefined); + if (!payloads || Object.keys(payloads).sort().join(',') !== versions.join(',')) throw new Error('EVAL_SKILL_HASH_MISMATCH'); + for (const version of versions) { + validatePortableSkillPayload(payloads[version]!, version); + if (sha256(payloads[version]!) !== skillHashes[version]) throw new Error('EVAL_SKILL_HASH_MISMATCH'); + } const root = resolve(destination); if (existsSync(root)) throw new Error('EVAL_DESTINATION_EXISTS'); const parent = dirname(root); @@ -308,10 +355,10 @@ export function prepareEvalJobs(matrix: EvalMatrix, payloads: { v2: string; v3: const jobs: PreparedEvalSchedule['jobs'] = []; for (const cell of selected) { const jobRoot = join(jobsRoot, cell.id); mkdirSync(jobRoot, { mode: 0o700 }); - const source = materializeCase(cell.caseId, cell.variant, join(jobRoot, 'source')); + const source = materializeCase(cell.caseId, cell.variant, join(jobRoot, 'source'), corpus); if (source.sourceHash !== cell.sourceHash) throw new Error('CORPUS_INTEGRITY_MISMATCH'); initializeFixtureRepository(source.path); - const input: ProducerInput = { schemaVersion: 1, cell, skill: payloads[cell.version], source: sourceEntries(cell.caseId, cell.variant) }; + const input: ProducerInput = { schemaVersion: 1, cell, skill: payloads[cell.version]!, source: sourceEntries(corpus, cell.caseId, cell.variant) }; const inputHash = producerInputHash(input); safeWriteNew(join(jobRoot, 'producer-input.json'), input); jobs.push({ cellId: cell.id, relativePath: `jobs/${cell.id}`, inputHash }); @@ -386,9 +433,41 @@ function validateReceiptIndex(receipt: ProducerReceiptIndex, cell: EvalCell): vo } const matchedPairKey = (cell: EvalCell): string => [cell.caseId, cell.variant, cell.mode, cell.repetition, cell.model, cell.host, cell.budgetSeconds, cell.sourceHash].join('|'); +const PROFILE_GROUPS: Record> = { + full: [['v2', 'daily'], ['v2', 'comprehensive'], ['v3', 'daily'], ['v3', 'comprehensive']], + release: [['v3', 'daily'], ['v3', 'comprehensive']], + baseline: [['v2', 'comprehensive']], +}; +const profileOf = (matrix: EvalMatrix): EvalProfile => matrix.profile ?? 'full'; +const expectedMatchedPairs = (matrix: EvalMatrix): number => profileOf(matrix) === 'full' ? matrix.cells.length / 2 : 0; -export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEvalSchedule, receipts: ProducerReceipt[]): ProducerBatch { - validateMatrix(matrix); validateSchedule(matrix, schedule); +type IdentityBearing = Pick; +/** + * Each platform has exactly one producer installation and provider binary. + * Platforms differ only in native binaries: the embedded catalogs and the + * reviewed provider policy must be identical across them. + */ +function assertPlatformIdentities(receipts: IdentityBearing[]): void { + const byPlatform = new Map(); + for (const receipt of receipts) { + const key = receipt.cell.platform ?? 'unpinned', list = byPlatform.get(key) ?? []; + list.push(receipt); byPlatform.set(key, list); + } + for (const list of byPlatform.values()) { + if (new Set(list.map(receipt => receipt.installationIdentity.identityHash)).size > 1) throw new Error('UNMATCHED_PRODUCER_INSTALLATIONS'); + if (new Set(list.map(receipt => receipt.providerIdentity.identityHash)).size > 1) throw new Error('UNMATCHED_PRODUCER_PROVIDERS'); + } + const representatives = [...byPlatform.values()].map(list => list[0]); + if (representatives.length < 2) return; + if (new Set(representatives.map(receipt => receipt.installationIdentity.identityHash)).size !== representatives.length) throw new Error('UNMATCHED_PRODUCER_PLATFORMS'); + const catalogs = (receipt: IdentityBearing) => JSON.stringify(receipt.installationIdentity.embeddedCatalogs); + const policy = ({ providerIdentity: { family, policyRevision, version, argsPrefix } }: IdentityBearing) => JSON.stringify({ family, policyRevision, version, argsPrefix }); + if (new Set(representatives.map(catalogs)).size !== 1) throw new Error('UNMATCHED_PRODUCER_INSTALLATIONS'); + if (new Set(representatives.map(policy)).size !== 1) throw new Error('UNMATCHED_PRODUCER_PROVIDERS'); +} + +export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEvalSchedule, receipts: ProducerReceipt[], corpus: EvalCorpusManifest = loadCorpusManifest()): ProducerBatch { + validateMatrix(matrix, corpus); validateSchedule(matrix, schedule); if (!Array.isArray(receipts)) throw new Error('INVALID_PRODUCER_RECEIPTS'); const cells = new Map(matrix.cells.map(cell => [cell.id, cell])); const jobs = new Map(schedule.jobs.map(job => [job.cellId, job])); @@ -398,18 +477,18 @@ export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEv if (!cell || !job || seen.has(cell.id)) throw new Error('UNKNOWN_OR_DUPLICATE_PRODUCER_RECEIPT'); validateReceipt(receipt, cell, job.inputHash); seen.add(cell.id); } - if (new Set(receipts.map(receipt => receipt.installationIdentity.identityHash)).size > 1) throw new Error('UNMATCHED_PRODUCER_INSTALLATIONS'); - if (new Set(receipts.map(receipt => receipt.providerIdentity.identityHash)).size > 1) throw new Error('UNMATCHED_PRODUCER_PROVIDERS'); + assertPlatformIdentities(receipts); const pairs = new Map>>(); - for (const receipt of receipts) { + if (profileOf(matrix) === 'full') for (const receipt of receipts) { const key = matchedPairKey(receipt.cell as EvalCell); const pair = pairs.get(key) ?? {}; pair[receipt.cell.version] = receipt; pairs.set(key, pair); } + else if (new Set(receipts.map(receipt => receipt.modelUsed)).size > 1) throw new Error('UNMATCHED_EFFECTIVE_MODELS: single-version cells used different normalized model identities'); const completePairs = [...pairs.entries()].filter(([, pair]) => pair.v2 && pair.v3) as Array<[string, { v2: ProducerReceipt; v3: ProducerReceipt }]>; const modelMismatches = completePairs.filter(([, pair]) => pair.v2.modelUsed !== pair.v3.modelUsed).map(([pair, value]) => ({ pair: digest(pair), v2: value.v2.modelUsed, v3: value.v3.modelUsed })); if (modelMismatches.length) throw new Error(`UNMATCHED_EFFECTIVE_MODELS: ${modelMismatches.length} matched v2/v3 pair(s) used different normalized model identities`); const groups: ProducerGroupSummary[] = []; - for (const version of ['v2', 'v3'] as const) for (const mode of ['daily', 'comprehensive'] as const) { + for (const [version, mode] of PROFILE_GROUPS[profileOf(matrix)]) { const expected = matrix.cells.filter(cell => cell.version === version && cell.mode === mode); const submitted = receipts.filter(receipt => receipt.cell.version === version && receipt.cell.mode === mode); const tokens = submitted.flatMap(receipt => receipt.usage.inputTokens === null || receipt.usage.outputTokens === null ? [] : [receipt.usage.inputTokens + receipt.usage.outputTokens]); @@ -424,29 +503,99 @@ export function collectProducerReceipts(matrix: EvalMatrix, schedule: PreparedEv const indexes: ProducerReceiptIndex[] = receipts.map(({ output: _output, error, ...receipt }) => ({ ...receipt, ...(error ? { error: { code: error.code } } : {}) })); const base = { schemaVersion: 1 as const, matrixHash: matrixHash(matrix), scheduleHash: digest(JSON.stringify(schedule)), receipts: indexes.sort((left, right) => left.cell.id.localeCompare(right.cell.id)), summary: { scheduled: matrix.cells.length, prepared: schedule.preparedCells, submitted: receipts.length, missing: matrix.cells.length - receipts.length, - matchedPairsExpected: matrix.cells.length / 2, matchedPairsSubmitted: completePairs.length, modelMismatches, groups, + matchedPairsExpected: expectedMatchedPairs(matrix), matchedPairsSubmitted: completePairs.length, modelMismatches, groups, note: 'Costs are pricing-table estimates. First-useful timing is unmeasured because the reused provider adapters return completed runs. Trusted findings and runtime outcomes require separate oracle adjudication.', } }; return { ...base, batchHash: digest(JSON.stringify(base)) }; } -export function createEvalMatrix(options: { model: string; host: ProducerHost; skillHashes: { v2: string; v3: string }; budgets?: { daily: number; comprehensive: number } }, corpus: CorpusManifest = loadCorpusManifest()): EvalMatrix { - if (!options.model?.trim() || !['claude', 'codex', 'gemini'].includes(options.host) || !HEX.test(options.skillHashes.v2) || !HEX.test(options.skillHashes.v3) || options.skillHashes.v2 === options.skillHashes.v3) throw new Error('INVALID_MATCHED_EVAL_INPUT'); - const budgets = options.budgets ?? { daily: 600, comprehensive: 1800 }; - if (![budgets.daily, budgets.comprehensive].every(value => Number.isInteger(value) && value > 60 && value <= 3600)) throw new Error('INVALID_EVAL_BUDGET'); - const cells: EvalCell[] = []; - for (const fixture of corpus.cases) for (const variant of ['vulnerable', 'fixed'] as const) for (const mode of ['daily', 'comprehensive'] as const) for (const repetition of [1, 2, 3] as const) for (const version of ['v2', 'v3'] as const) { - const cell = { caseId: fixture.id, stack: fixture.stack, variant, version, mode, repetition, model: options.model, host: options.host, budgetSeconds: budgets[mode], sourceHash: fixture.filesHash[variant], skillHash: options.skillHashes[version] }; - cells.push({ id: digest(JSON.stringify(cell)), ...cell }); - } - return { schemaVersion: 1, corpusVersion: corpus.version, corpusHash: digest(JSON.stringify(corpus)), model: options.model, host: options.host, budgets, skillHashes: options.skillHashes, repetitions: 3, cells }; +function normalizedPlatforms(corpus: EvalCorpusManifest, value: unknown): Record { + const assignment = value as Record; + if (!assignment || typeof assignment !== 'object' || Array.isArray(assignment) || Object.keys(assignment).length !== corpus.cases.length || + corpus.cases.some(fixture => !PRODUCER_PLATFORMS.includes(assignment[fixture.id]))) throw new Error('INVALID_EVAL_PLATFORMS'); + return Object.fromEntries(corpus.cases.map(fixture => [fixture.id, assignment[fixture.id]])); } -export function validateMatrix(matrix: EvalMatrix, corpus: CorpusManifest = loadCorpusManifest()): void { - const expected = createEvalMatrix({ model: matrix.model, host: matrix.host, skillHashes: matrix.skillHashes, budgets: matrix.budgets }, corpus); +function buildEvalMatrix(options: EvalMatrixOptions, corpus: EvalCorpusManifest, baselineHash: string | null): EvalMatrix { + const profile = options.profile ?? 'full'; + const skillHashes: Partial> = options.skillHashes ?? {}; + const versions = profile === 'full' ? ['v2', 'v3'] : profile === 'release' ? ['v3'] : ['v2']; + if (!options.model?.trim() || !['claude', 'codex', 'gemini'].includes(options.host) || !['full', 'release', 'baseline'].includes(profile) || + Object.keys(skillHashes).sort().join(',') !== versions.join(',') || versions.some(version => !HEX.test(skillHashes[version as EvalVersion] ?? '')) || + (profile === 'full' && skillHashes.v2 === skillHashes.v3)) throw new Error('INVALID_MATCHED_EVAL_INPUT'); + const budgets = options.budgets ?? { daily: 600, comprehensive: 1800 }; + if (![budgets.daily, budgets.comprehensive].every(value => Number.isInteger(value) && value > 60 && value <= 3600)) throw new Error('INVALID_EVAL_BUDGET'); + const corpusHash = digest(JSON.stringify(corpus)); + const cells: EvalCell[] = []; + if (options.profile === undefined || options.profile === 'full') { + for (const fixture of corpus.cases) for (const variant of ['vulnerable', 'fixed'] as const) for (const mode of ['daily', 'comprehensive'] as const) for (const repetition of [1, 2, 3] as const) for (const version of ['v2', 'v3'] as const) { + const cell = { caseId: fixture.id, stack: fixture.stack, variant, version, mode, repetition, model: options.model, host: options.host, budgetSeconds: budgets[mode], sourceHash: fixture.filesHash[variant], skillHash: options.skillHashes[version] }; + cells.push({ id: digest(JSON.stringify(cell)), ...cell }); + } + return { schemaVersion: 1, corpusVersion: corpus.version, corpusHash, model: options.model, host: options.host, budgets, skillHashes: options.skillHashes, repetitions: 3, cells }; + } + const pinned = options as ReleaseEvalMatrixOptions | BaselineEvalMatrixOptions; + const platformKeys = pinned.profile === 'release' ? 'comprehensive,daily' : 'comprehensive'; + if (!pinned.platforms || Object.keys(pinned.platforms).sort().join(',') !== platformKeys) throw new Error('INVALID_EVAL_PLATFORMS'); + const comprehensive = normalizedPlatforms(corpus, pinned.platforms.comprehensive); + const daily = pinned.profile === 'release' ? pinned.platforms.daily : undefined; + if (pinned.profile === 'release' && !PRODUCER_PLATFORMS.includes(daily!)) throw new Error('INVALID_EVAL_PLATFORMS'); + const version: EvalVersion = pinned.profile === 'release' ? 'v3' : 'v2'; + const modes = pinned.profile === 'release' ? ['daily', 'comprehensive'] as const : ['comprehensive'] as const; + for (const fixture of corpus.cases) for (const variant of ['vulnerable', 'fixed'] as const) for (const mode of modes) { + const cell = { caseId: fixture.id, stack: fixture.stack, variant, version, mode, repetition: 1 as const, model: pinned.model, host: pinned.host, budgetSeconds: budgets[mode], sourceHash: fixture.filesHash[variant], skillHash: skillHashes[version]!, platform: mode === 'daily' ? daily! : comprehensive[fixture.id] }; + cells.push({ id: digest(JSON.stringify(cell)), ...cell }); + } + const common = { schemaVersion: 1 as const, corpusVersion: corpus.version, corpusHash, model: pinned.model, host: pinned.host, budgets }; + if (pinned.profile === 'release') { + if (baselineHash !== null && !HEX.test(baselineHash)) throw new Error('UNBOUND_EVAL_BASELINE'); + return { ...common, profile: 'release', skillHashes: { v3: pinned.skillHashes.v3 }, repetitions: 1, platforms: { daily: daily!, comprehensive }, baselineHash, cells }; + } + return { ...common, profile: 'baseline', skillHashes: { v2: pinned.skillHashes.v2 }, repetitions: 1, platforms: { comprehensive }, cells }; +} + +const BASELINE_FIELDS = 'baselineHash,budgetSeconds,cases,corpusHash,corpusVersion,host,matrixHash,model,modelUsed,platforms,producerBatchHash,provider,schemaVersion,skillHash,version'; +/** Check that a baseline was measured under the same corpus, model, host, budget and platform assignment. */ +function assertBaselineBinding(baseline: EvalBaseline, target: { corpus: EvalCorpusManifest; model: string; host: ProducerHost; budgetSeconds: number; platforms: Record }): void { + if (!baseline || typeof baseline !== 'object' || Object.keys(baseline).sort().join(',') !== BASELINE_FIELDS) throw new Error('UNBOUND_EVAL_BASELINE'); + const { baselineHash, ...record } = baseline; + const provider = baseline.provider as unknown as Record; + if (baseline.schemaVersion !== 1 || baseline.version !== 'v2' || !HEX.test(baselineHash) || digest(JSON.stringify(record)) !== baselineHash || + baseline.corpusVersion !== target.corpus.version || baseline.corpusHash !== digest(JSON.stringify(target.corpus)) || + baseline.model !== target.model || baseline.host !== target.host || baseline.budgetSeconds !== target.budgetSeconds || + JSON.stringify(baseline.platforms) !== JSON.stringify(target.platforms) || !HEX.test(baseline.skillHash) || !HEX.test(baseline.matrixHash) || + !HEX.test(baseline.producerBatchHash) || typeof baseline.modelUsed !== 'string' || !baseline.modelUsed || + !provider || typeof provider !== 'object' || Object.keys(provider).sort().join(',') !== 'family,policyRevision,version' || + !['claude', 'gpt', 'gemini'].includes(String(provider.family)) || typeof provider.policyRevision !== 'string' || !provider.policyRevision || + typeof provider.version !== 'string' || !provider.version || !Array.isArray(baseline.cases) || baseline.cases.length !== target.corpus.cases.length || + baseline.cases.some((entry, index) => !entry || Object.keys(entry).sort().join(',') !== 'caseId,found' || entry.caseId !== target.corpus.cases[index].id || typeof entry.found !== 'boolean')) + throw new Error('UNBOUND_EVAL_BASELINE'); +} + +export function createEvalMatrix(options: FullEvalMatrixOptions, corpus?: EvalCorpusManifest): FullEvalMatrix; +export function createEvalMatrix(options: ReleaseEvalMatrixOptions, corpus?: EvalCorpusManifest): ReleaseEvalMatrix; +export function createEvalMatrix(options: BaselineEvalMatrixOptions, corpus?: EvalCorpusManifest): BaselineEvalMatrix; +export function createEvalMatrix(options: EvalMatrixOptions, corpus: EvalCorpusManifest = loadCorpusManifest()): EvalMatrix { + let baselineHash: string | null = null; + if (options.profile === 'release' && options.baseline !== undefined) { + assertBaselineBinding(options.baseline, { corpus, model: options.model, host: options.host, budgetSeconds: (options.budgets ?? { comprehensive: 1800 }).comprehensive, platforms: normalizedPlatforms(corpus, options.platforms?.comprehensive) }); + baselineHash = options.baseline.baselineHash; + } + return buildEvalMatrix(options, corpus, baselineHash); +} + +export function validateMatrix(matrix: EvalMatrix, corpus: EvalCorpusManifest = loadCorpusManifest()): void { + if (!matrix || typeof matrix !== 'object') throw new Error('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX'); + const common = { model: matrix.model, host: matrix.host, budgets: matrix.budgets }; + let expected: EvalMatrix; + try { + expected = matrix.profile === 'release' ? buildEvalMatrix({ ...common, profile: 'release', skillHashes: matrix.skillHashes, platforms: matrix.platforms }, corpus, matrix.baselineHash) + : matrix.profile === 'baseline' ? buildEvalMatrix({ ...common, profile: 'baseline', skillHashes: matrix.skillHashes, platforms: matrix.platforms }, corpus, null) + : buildEvalMatrix({ ...common, skillHashes: matrix.skillHashes }, corpus, null); + } catch { throw new Error('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX'); } if (JSON.stringify(matrix) !== JSON.stringify(expected)) throw new Error('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX'); } -function validateResult(result: EvalResult, cell: EvalCell, corpus: CorpusManifest): void { +function validateResult(result: EvalResult, cell: EvalCell, corpus: EvalCorpusManifest): void { if (result.sourceHash !== cell.sourceHash || result.skillHash !== cell.skillHash || result.model !== cell.model || result.host !== cell.host || result.budgetSeconds !== cell.budgetSeconds) throw new Error('UNMATCHED_EVAL_RESULT'); for (const field of ['setup', 'reproduction', 'repair', 'recheck'] as const) if (!['passed', 'failed', 'blocked', 'not_attempted'].includes(result[field])) throw new Error('INVALID_EVAL_OUTCOME'); for (const field of ['reportPresent', 'reportComplete', 'heldOutAssertionsPassed', 'freshRecheck'] as const) if (typeof result[field] !== 'boolean') throw new Error('INVALID_EVAL_RESULT'); @@ -493,7 +642,7 @@ function quantile(values: number[], fraction: number): number | null { if (!values.length) return null; return [...values].sort((left, right) => left - right)[Math.ceil(values.length * fraction) - 1]; } -function measuredRate(cells: EvalCell[], results: Map, field: 'setup' | 'reproduction' | 'repair' | 'recheck', corpus: CorpusManifest): Rate { +function measuredRate(cells: EvalCell[], results: Map, field: 'setup' | 'reproduction' | 'repair' | 'recheck', corpus: EvalCorpusManifest): Rate { return rate(cells.filter(cell => { const result = results.get(cell.id); if (!result?.reportPresent || !result.reportComplete || result[field] !== 'passed') return false; @@ -511,7 +660,7 @@ function measuredRate(cells: EvalCell[], results: Map, field }).length, cells.length); } -function trustedFindingVerification(cell: EvalCell, result: EvalResult, finding: EvalFinding, stage: 'repair' | 'recheck', corpus: CorpusManifest): boolean { +function trustedFindingVerification(cell: EvalCell, result: EvalResult, finding: EvalFinding, stage: 'repair' | 'recheck', corpus: EvalCorpusManifest): boolean { const verification = finding.trustedVerification; if (!result.reportPresent || !result.reportComplete || cell.version !== 'v3' || cell.mode !== 'comprehensive' || cell.variant !== 'vulnerable' || finding.evidence !== 'supported' || finding.judgment !== 'correct' || finding.matchedCaseId !== cell.caseId || !finding.claimedTested || !verification || @@ -524,8 +673,7 @@ function trustedFindingVerification(cell: EvalCell, result: EvalResult, finding: !!result.recheckEvidenceHash && result.recheckEvidenceHash !== result.oracleEvidenceHash; } -export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualification: ReleaseQualification = { containment: {} }, corpus: CorpusManifest = loadCorpusManifest()) { - validateMatrix(matrix, corpus); +function validatedResults(matrix: EvalMatrix, observations: EvalResult[], corpus: EvalCorpusManifest): Map { if (!Array.isArray(observations)) throw new Error('INVALID_EVAL_RESULTS'); const cells = new Map(matrix.cells.map(cell => [cell.id, cell])); const results = new Map(); @@ -534,18 +682,34 @@ export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualif if (!cell || results.has(result.cellId)) throw new Error('UNKNOWN_OR_DUPLICATE_EVAL_RESULT'); validateResult(result, cell, corpus); results.set(result.cellId, result); } - const eligibleFinding = (cell: EvalCell, finding: EvalFinding) => finding.evidence === 'supported' || (cell.version === 'v2' && finding.evidence === 'legacy_review'); - const found = (cell: EvalCell) => { - const result = results.get(cell.id); - return result?.reportPresent === true && result.reportComplete === true && - result.findings.some(finding => eligibleFinding(cell, finding) && finding.judgment === 'correct'); - }; + return results; +} +const eligibleFinding = (cell: EvalCell, finding: EvalFinding) => finding.evidence === 'supported' || (cell.version === 'v2' && finding.evidence === 'legacy_review'); +const foundSeededIssue = (cell: EvalCell, result: EvalResult | undefined) => result?.reportPresent === true && result.reportComplete === true && + result.findings.some(finding => eligibleFinding(cell, finding) && finding.judgment === 'correct'); +const isHighCritical = (corpus: EvalCorpusManifest, caseId: string) => ['critical', 'high'].includes(corpus.cases.find(fixture => fixture.id === caseId)!.severity); + +/** High/critical recall of a pre-registered external baseline, after proving it binds to this release matrix. */ +function externalBaselineRecall(matrix: ReleaseEvalMatrix, baseline: EvalBaseline, corpus: EvalCorpusManifest): Rate { + if (matrix.baselineHash === null || baseline?.baselineHash !== matrix.baselineHash) throw new Error('UNBOUND_EVAL_BASELINE'); + assertBaselineBinding(baseline, { corpus, model: matrix.model, host: matrix.host, budgetSeconds: matrix.budgets.comprehensive, platforms: matrix.platforms.comprehensive }); + const highCritical = baseline.cases.filter(entry => isHighCritical(corpus, entry.caseId)); + return rate(highCritical.filter(entry => entry.found).length, highCritical.length); +} + +export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualification: ReleaseQualification = { containment: {} }, corpus: EvalCorpusManifest = loadCorpusManifest(), baseline?: EvalBaseline) { + validateMatrix(matrix, corpus); + const profile = profileOf(matrix); + if (profile === 'baseline') throw new Error('BASELINE_PROFILE_NOT_QUALIFIABLE: use createEvalBaseline'); + if (profile !== 'release' && baseline !== undefined) throw new Error('UNBOUND_EVAL_BASELINE'); + const results = validatedResults(matrix, observations, corpus); + const found = (cell: EvalCell) => foundSeededIssue(cell, results.get(cell.id)); const groups: any[] = []; - for (const version of ['v2', 'v3'] as const) for (const mode of ['daily', 'comprehensive'] as const) { + for (const [version, mode] of PROFILE_GROUPS[profile]) { const selected = matrix.cells.filter(cell => cell.version === version && cell.mode === mode); const completed = selected.map(cell => results.get(cell.id)).filter((result): result is EvalResult => !!result); const expected = selected.filter(cell => cell.variant === 'vulnerable'); - const highCritical = expected.filter(cell => ['critical','high'].includes(corpus.cases.find(fixture => fixture.id === cell.caseId)!.severity)); + const highCritical = expected.filter(cell => isHighCritical(corpus, cell.caseId)); let correct = 0, reported = 0, unadjudicated = 0; for (const cell of selected) { const result = results.get(cell.id); @@ -576,7 +740,8 @@ export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualif } const daily = groups.find(group => group.version === 'v3' && group.mode === 'daily'); const comprehensive = groups.find(group => group.version === 'v3' && group.mode === 'comprehensive'); - const baseline = groups.find(group => group.version === 'v2' && group.mode === 'comprehensive'); + const inBatchBaseline = groups.find(group => group.version === 'v2' && group.mode === 'comprehensive'); + const external = matrix.profile === 'release' && baseline !== undefined ? externalBaselineRecall(matrix, baseline, corpus) : null; const perStack = Object.fromEntries(STACKS.map(stack => { const eligible = matrix.cells.filter(cell => cell.version === 'v3' && cell.mode === 'comprehensive' && cell.variant === 'vulnerable' && cell.stack === stack); // A repaired oracle case demonstrates the full find-to-repair workflow only @@ -596,24 +761,27 @@ export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualif mandatoryReports: gate(groups.every(group => group.reports.numerator === group.reports.denominator), observations.length === matrix.cells.length), dailyPrecision95: gate(daily.precision.value === null ? null : daily.precision.value >= 0.95, daily.submitted === daily.cells && !daily.unadjudicated), comprehensiveHighCriticalRecall80: gate(comprehensive.highCriticalRecall.value >= 0.8, comprehensive.submitted === comprehensive.cells && !comprehensive.unadjudicated), - noHighCriticalRecallRegression: gate(comprehensive.highCriticalRecall.value >= baseline.highCriticalRecall.value, comprehensive.submitted === comprehensive.cells && baseline.submitted === baseline.cells && !comprehensive.unadjudicated && !baseline.unadjudicated), + noHighCriticalRecallRegression: profile === 'full' + ? gate(comprehensive.highCriticalRecall.value >= inBatchBaseline.highCriticalRecall.value, comprehensive.submitted === comprehensive.cells && inBatchBaseline.submitted === inBatchBaseline.cells && !comprehensive.unadjudicated && !inBatchBaseline.unadjudicated) + : gate(external === null ? null : comprehensive.highCriticalRecall.value >= external.value!, comprehensive.submitted === comprehensive.cells && !comprehensive.unadjudicated && external !== null), allCoreColdStarts: gate(comprehensive.setup.numerator === comprehensive.setup.denominator, comprehensive.submitted === comprehensive.cells), zeroFalselyTestedRepairs: gate(comprehensive.falseTested === 0 && daily.falseTested === 0, comprehensive.submitted === comprehensive.cells && daily.submitted === daily.cells), heldOutRepairEachStack: gate(Object.values(perStack).every(value => value.correctHeldOutRepairs >= 1), comprehensive.submitted === comprehensive.cells), containmentAndCanaries: containmentValues.includes('failed') ? 'fail' as const : containmentValues.every(value => value === 'passed') ? 'pass' as const : 'unmeasured' as const, }; return { schemaVersion: 1, corpusVersion: corpus.version, model: matrix.model, host: matrix.host, + ...(profile === 'release' ? { profile, baseline: external === null ? null : { baselineHash: baseline!.baselineHash, highCriticalRecall: external } } : {}), status: Object.values(gates).every(value => value === 'pass') ? 'qualified' : observations.length ? 'partial' : 'unmeasured', groups, perStack, gates, notes: ['Missing and setup-blocked supported scenarios remain in recall and workflow denominators.', 'Costs and tokens come only from host-reported usage; wall-clock budgets are not model-spend caps.', 'Oracle judgments and qualification receipts must come from the trusted evaluator, never producing agents.'] }; } -/** Release scoring path: every trusted judgment must bind to a collected producer receipt. */ -export function scoreCollectedEval(matrix: EvalMatrix, batch: ProducerBatch, observations: EvalResult[], qualification: ReleaseQualification = { containment: {} }, corpus: CorpusManifest = loadCorpusManifest()) { - validateMatrix(matrix, corpus); +/** Bind a complete collected batch and every trusted judgment to the matrix's producer receipts. */ +function validateCollectedBatch(matrix: EvalMatrix, batch: ProducerBatch, observations: EvalResult[]): Map { if (!Array.isArray(observations)) throw new Error('INVALID_EVAL_RESULTS'); if (!batch || batch.schemaVersion !== 1 || batch.matrixHash !== matrixHash(matrix) || !HEX.test(batch.scheduleHash) || !HEX.test(batch.batchHash)) throw new Error('INVALID_PRODUCER_BATCH'); const { batchHash, ...withoutHash } = batch; - if (digest(JSON.stringify(withoutHash)) !== batchHash || !Array.isArray(batch.receipts) || batch.receipts.length !== matrix.cells.length || batch.summary.scheduled !== matrix.cells.length || batch.summary.prepared !== matrix.cells.length || batch.summary.submitted !== matrix.cells.length || batch.summary.missing !== 0 || batch.summary.matchedPairsExpected !== matrix.cells.length / 2 || batch.summary.matchedPairsSubmitted !== matrix.cells.length / 2 || batch.summary.modelMismatches.length !== 0) throw new Error('INCOMPLETE_PRODUCER_BATCH'); + const pairs = expectedMatchedPairs(matrix); + if (digest(JSON.stringify(withoutHash)) !== batchHash || !Array.isArray(batch.receipts) || batch.receipts.length !== matrix.cells.length || batch.summary.scheduled !== matrix.cells.length || batch.summary.prepared !== matrix.cells.length || batch.summary.submitted !== matrix.cells.length || batch.summary.missing !== 0 || batch.summary.matchedPairsExpected !== pairs || batch.summary.matchedPairsSubmitted !== pairs || batch.summary.modelMismatches.length !== 0) throw new Error('INCOMPLETE_PRODUCER_BATCH'); const cells = new Map(matrix.cells.map(cell => [cell.id, cell])); const receipts = new Map(); for (const receipt of batch.receipts) { @@ -621,22 +789,67 @@ export function scoreCollectedEval(matrix: EvalMatrix, batch: ProducerBatch, obs if (!cell || receipts.has(cell.id)) throw new Error('INVALID_PRODUCER_BATCH'); validateReceiptIndex(receipt, cell); receipts.set(cell.id, receipt); } - if (new Set(batch.receipts.map(receipt => receipt.installationIdentity.identityHash)).size !== 1) throw new Error('UNMATCHED_PRODUCER_INSTALLATIONS'); - if (new Set(batch.receipts.map(receipt => receipt.providerIdentity.identityHash)).size !== 1) throw new Error('UNMATCHED_PRODUCER_PROVIDERS'); - const actualModels = new Map>>(); - for (const receipt of batch.receipts) { - const key = matchedPairKey(receipt.cell as EvalCell), pair = actualModels.get(key) ?? {}; - pair[receipt.cell.version] = receipt.modelUsed; actualModels.set(key, pair); - } - if (actualModels.size !== matrix.cells.length / 2 || [...actualModels.values()].some(pair => !pair.v2 || !pair.v3 || pair.v2 !== pair.v3)) throw new Error('UNMATCHED_EFFECTIVE_MODELS'); + assertPlatformIdentities(batch.receipts); + if (profileOf(matrix) === 'full') { + const actualModels = new Map>>(); + for (const receipt of batch.receipts) { + const key = matchedPairKey(receipt.cell as EvalCell), pair = actualModels.get(key) ?? {}; + pair[receipt.cell.version] = receipt.modelUsed; actualModels.set(key, pair); + } + if (actualModels.size !== pairs || [...actualModels.values()].some(pair => !pair.v2 || !pair.v3 || pair.v2 !== pair.v3)) throw new Error('UNMATCHED_EFFECTIVE_MODELS'); + } else if (new Set(batch.receipts.map(receipt => receipt.modelUsed)).size !== 1) throw new Error('UNMATCHED_EFFECTIVE_MODELS'); for (const result of observations) { const receipt = receipts.get(result.cellId); if (!receipt || result.producerReceiptHash !== receipt.receiptHash) throw new Error('UNBOUND_EVAL_RESULT'); } - const score = scoreEval(matrix, observations, qualification, corpus); + return receipts; +} + +/** Release scoring path: every trusted judgment must bind to a collected producer receipt. */ +export function scoreCollectedEval(matrix: EvalMatrix, batch: ProducerBatch, observations: EvalResult[], qualification: ReleaseQualification = { containment: {} }, corpus: EvalCorpusManifest = loadCorpusManifest(), baseline?: EvalBaseline) { + validateMatrix(matrix, corpus); + if (profileOf(matrix) === 'baseline') throw new Error('BASELINE_PROFILE_NOT_QUALIFIABLE: use createEvalBaseline'); + validateCollectedBatch(matrix, batch, observations); + if (baseline !== undefined) { + const { modelUsed, providerIdentity: { family, policyRevision, version } } = batch.receipts[0]; + if (baseline?.modelUsed !== modelUsed || JSON.stringify(baseline?.provider) !== JSON.stringify({ family, policyRevision, version })) throw new Error('UNBOUND_EVAL_BASELINE'); + } + const score = scoreEval(matrix, observations, qualification, corpus, baseline); return { ...score, producerBatchHash: batch.batchHash }; } +/** + * Turn a complete, fully adjudicated v2 baseline run into a reusable record. + * Missing reports count as misses; an incomplete or unadjudicated run is refused. + */ +export function createEvalBaseline(matrix: BaselineEvalMatrix, batch: ProducerBatch, observations: EvalResult[], corpus: EvalCorpusManifest = loadCorpusManifest()): EvalBaseline { + if (matrix?.profile !== 'baseline') throw new Error('INVALID_EVAL_BASELINE_MATRIX'); + validateMatrix(matrix, corpus); + validateCollectedBatch(matrix, batch, observations); + const results = validatedResults(matrix, observations, corpus); + if (results.size !== matrix.cells.length) throw new Error('INCOMPLETE_EVAL_BASELINE'); + if ([...results.values()].some(result => result.reportPresent && result.reportComplete && result.findings.some(finding => finding.evidence !== 'hypothesis' && finding.judgment === 'unadjudicated'))) throw new Error('UNADJUDICATED_EVAL_BASELINE'); + const vulnerable = new Map(matrix.cells.filter(cell => cell.variant === 'vulnerable').map(cell => [cell.caseId, cell])); + const { modelUsed, providerIdentity: { family, policyRevision, version } } = batch.receipts[0]; + const record: Omit = { + schemaVersion: 1, version: 'v2', corpusVersion: corpus.version, corpusHash: matrix.corpusHash, model: matrix.model, host: matrix.host, + budgetSeconds: matrix.budgets.comprehensive, skillHash: matrix.skillHashes.v2, platforms: matrix.platforms.comprehensive, + matrixHash: matrixHash(matrix), producerBatchHash: batch.batchHash, modelUsed, provider: { family, policyRevision, version }, + cases: corpus.cases.map(fixture => ({ caseId: fixture.id, found: foundSeededIssue(vulnerable.get(fixture.id)!, results.get(vulnerable.get(fixture.id)!.id)) })), + }; + return { ...record, baselineHash: digest(JSON.stringify(record)) }; +} + +function splitFlags(args: string[], allowed: string[]): { positional: string[]; flags: Record } { + const positional: string[] = [], flags: Record = {}; + for (let index = 0; index < args.length; index++) { + if (!args[index].startsWith('--')) { positional.push(args[index]); continue; } + if (!allowed.includes(args[index]) || !args[index + 1] || flags[args[index]]) throw new Error(`Unsupported or repeated flag: ${args[index]}`); + flags[args[index]] = args[++index]; + } + return { positional, flags }; +} + function cli(args: string[]): void { const [command, ...rest] = args; if (command === 'payload') { @@ -669,25 +882,35 @@ function cli(args: string[]): void { const schedule = prepareEvalJobs(matrix, { v2: readBoundedStable(resolve(rest[2]), PORTABLE_PAYLOAD_LIMIT, 'v2 portable skill payload').toString('utf8'), v3: readBoundedStable(resolve(rest[4]), PORTABLE_PAYLOAD_LIMIT, 'v3 portable skill payload').toString('utf8') }, rest[6]); console.log(JSON.stringify({ scheduled: schedule.scheduledCells, prepared: schedule.preparedCells, output: resolve(rest[6]), paidCalls: 0 })); return; } + const { positional, flags } = splitFlags(rest, ['--corpus-manifest', '--baseline']); + const corpus = flags['--corpus-manifest'] ? validateCorpusManifest(readJsonBounded(flags['--corpus-manifest'], 16 * 1024 * 1024, 'corpus manifest')) : loadCorpusManifest(); if (command === 'collect') { - if (rest.length !== 4) throw new Error('Usage: cso-eval collect '); - const matrix = readJsonBounded(rest[0], 16 * 1024 * 1024, 'evaluation matrix') as EvalMatrix; - const schedule = readJsonBounded(rest[1], 16 * 1024 * 1024, 'evaluation schedule') as PreparedEvalSchedule; - const receiptRoot = resolve(rest[2]), stat = lstatSync(receiptRoot); + if (positional.length !== 4 || flags['--baseline']) throw new Error('Usage: cso-eval collect [--corpus-manifest ]'); + const [matrixPath, schedulePath, receiptDirectory, output] = positional; + const matrix = readJsonBounded(matrixPath, 16 * 1024 * 1024, 'evaluation matrix') as EvalMatrix; + const schedule = readJsonBounded(schedulePath, 16 * 1024 * 1024, 'evaluation schedule') as PreparedEvalSchedule; + const receiptRoot = resolve(receiptDirectory), stat = lstatSync(receiptRoot); if (!stat.isDirectory() || stat.isSymbolicLink() || realpathSync(receiptRoot) !== receiptRoot) throw new Error('UNSAFE_RECEIPT_DIRECTORY'); const receipts: ProducerReceipt[] = []; for (const entry of readdirSync(receiptRoot, { withFileTypes: true })) { if (!entry.isFile() || entry.isSymbolicLink() || !/^[a-f0-9]{64}\.json$/.test(entry.name)) throw new Error('UNSAFE_RECEIPT_DIRECTORY'); receipts.push(readJsonBounded(join(receiptRoot, entry.name), 40 * 1024 * 1024, 'producer receipt')); } - const batch = collectProducerReceipts(matrix, schedule, receipts); - safeWriteNew(resolve(rest[3]), batch); - console.log(JSON.stringify({ output: resolve(rest[3]), submitted: batch.summary.submitted, missing: batch.summary.missing, modelMismatches: batch.summary.modelMismatches.length, paidCalls: 0 })); return; + const batch = collectProducerReceipts(matrix, schedule, receipts, corpus); + safeWriteNew(resolve(output), batch); + console.log(JSON.stringify({ output: resolve(output), submitted: batch.summary.submitted, missing: batch.summary.missing, modelMismatches: batch.summary.modelMismatches.length, paidCalls: 0 })); return; } if (command === 'score') { - if (rest.length < 3 || rest.length > 4) throw new Error('Usage: cso-eval score [qualification.json]'); - console.log(JSON.stringify(scoreCollectedEval(readJsonBounded(rest[0], 16 * 1024 * 1024, 'evaluation matrix'), readJsonBounded(rest[1], 16 * 1024 * 1024, 'producer batch'), readJsonBounded(rest[2], 64 * 1024 * 1024, 'trusted results'), rest[3] ? readJsonBounded(rest[3], 4 * 1024 * 1024, 'qualification') : undefined), null, 2)); return; + if (positional.length < 3 || positional.length > 4) throw new Error('Usage: cso-eval score [qualification.json] [--corpus-manifest ] [--baseline ]'); + const baseline = flags['--baseline'] ? readJsonBounded(flags['--baseline'], 4 * 1024 * 1024, 'evaluation baseline') as EvalBaseline : undefined; + console.log(JSON.stringify(scoreCollectedEval(readJsonBounded(positional[0], 16 * 1024 * 1024, 'evaluation matrix'), readJsonBounded(positional[1], 16 * 1024 * 1024, 'producer batch'), readJsonBounded(positional[2], 64 * 1024 * 1024, 'trusted results'), positional[3] ? readJsonBounded(positional[3], 4 * 1024 * 1024, 'qualification') : undefined, corpus, baseline), null, 2)); return; } - throw new Error('Usage: cso-eval '); + if (command === 'baseline') { + if (positional.length !== 4 || flags['--baseline']) throw new Error('Usage: cso-eval baseline [--corpus-manifest ]'); + const baseline = createEvalBaseline(readJsonBounded(positional[0], 16 * 1024 * 1024, 'evaluation matrix'), readJsonBounded(positional[1], 16 * 1024 * 1024, 'producer batch'), readJsonBounded(positional[2], 64 * 1024 * 1024, 'trusted results'), corpus); + safeWriteNew(resolve(positional[3]), baseline); + console.log(JSON.stringify({ output: resolve(positional[3]), baselineHash: baseline.baselineHash, paidCalls: 0 })); return; + } + throw new Error('Usage: cso-eval '); } if (import.meta.main) { try { cli(process.argv.slice(2)); } catch (error) { console.error(error instanceof Error ? error.message : 'CSO evaluation failed'); process.exitCode = 1; } } diff --git a/scripts/cso-native-evidence.ts b/scripts/cso-native-evidence.ts new file mode 100644 index 000000000..7c3de1b4f --- /dev/null +++ b/scripts/cso-native-evidence.ts @@ -0,0 +1,131 @@ +#!/usr/bin/env bun +/** Derive native runtime gate evidence from the named Docker tests that actually executed and passed. */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { sha256 } from '../lib/cso/contracts'; +import type { ImageBuildRow } from './cso-image-matrix'; +import { requiredChecks } from './cso-runtime-promotion'; + +type Stack = ImageBuildRow['stack']; +export interface TestRef { file: string; name: string } +export interface JUnitCase extends TestRef { status: 'passed' | 'failed' | 'skipped'; assertions: number } + +const MAX_REPORT_BYTES = 4 * 1024 * 1024; +const DOCKER = 'test/cso-docker-integration.test.ts'; +const CONTAINMENT: TestRef[] = [ + { file: DOCKER, name: 'hard fails when local daemon enforcement prerequisites are absent' }, + { file: DOCKER, name: 'rejects image-declared writable volumes before container creation' }, + { file: DOCKER, name: 'shares only loopback while denying egress, privileges, and daemon logs, and reads private source/policy mounts' }, + { file: DOCKER, name: 'machine-wide admission allows only two groups per endpoint' }, +]; +const STAGED: TestRef = { file: DOCKER, name: 'staged runtime executes its trusted verifier and checks every declared tool version' }; +const NODE_LIFECYCLE: TestRef = { + file: 'test/cso-node-lifecycle-integration.test.ts', + name: 'finds, reproduces, bundles, replays, and closes against current source through the command dispatcher', +}; +const COLD_JOURNEY: TestRef = { + file: 'test/cso-stack-cold-integration.test.ts', + name: 'acquires, prepares, boots, controls, and tests the requested staged stack with no target egress', +}; + +/** Each native check key names the tests whose execution proves it for one stack. */ +export function nativeTestRequirements(stack: Stack): Record { + if (stack === 'postgresql') return { + containmentPassed: [...CONTAINMENT, STAGED], + coldStartPassed: [STAGED], + multiDatabasePassed: [STAGED], + readinessPassed: [STAGED], + }; + const journey = stack === 'node' ? NODE_LIFECYCLE : COLD_JOURNEY; + const application = { + containmentPassed: [...CONTAINMENT, STAGED], + coldStartPassed: [journey], + positiveNegativeAssertionsPassed: [STAGED], + acquisitionPublicOnlyPassed: [journey], + offlineLifecyclePassed: [journey], + }; + return stack === 'rails' + ? { ...application, railsSqlitePassed: [COLD_JOURNEY], railsPostgresqlPassed: [COLD_JOURNEY], nativeExtensionsPassed: [COLD_JOURNEY] } + : application; +} + +function decode(value: string): string { + return value.replace(/&(#x[0-9a-fA-F]+|#[0-9]+|amp|lt|gt|quot|apos);/g, (_, entity: string) => { + if (entity.startsWith('#x')) return String.fromCodePoint(Number.parseInt(entity.slice(2), 16)); + if (entity.startsWith('#')) return String.fromCodePoint(Number.parseInt(entity.slice(1), 10)); + return { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }[entity]!; + }); +} + +/** Parse the testcases of Bun's `--reporter=junit` output. */ +export function parseJUnit(xml: string): JUnitCase[] { + if (!/^<\?xml[^>]*\?>\s*]*?)(?:\/>|>([\s\S]*?)<\/testcase>)/g)) { + const attributes = Object.fromEntries([...match[1].matchAll(/([A-Za-z_:][\w:.-]*)="([^"]*)"/g)].map(([, key, value]) => [key, decode(value)])); + const body = match[2] ?? ''; + const status = /<(?:failure|error)\b/.test(body) ? 'failed' : / { + const clean = cases.some(item => item.status === 'passed') && !cases.some(item => item.status === 'failed'); + const passed = (ref: TestRef) => { + const matches = cases.filter(item => item.file === ref.file && item.name === ref.name); + return matches.length === 1 && matches[0].status === 'passed' && matches[0].assertions > 0; + }; + return Object.fromEntries(Object.entries(nativeTestRequirements(stack)) + .map(([key, refs]) => [key, clean && refs.every(passed)])); +} + +/** Native evidence for one staged row; private checks are every required release gate the native job cannot prove. */ +export function nativeEvidence(staged: unknown, stack: Stack, platform: string, report: string): Record { + const record = staged as Record; + if (!record || typeof record !== 'object' || record.state !== 'staged' || record.stack !== stack || record.platform !== platform) { + throw new Error('STAGED_EVIDENCE_MISMATCH'); + } + const required = requiredChecks(stack); + const nativeChecks = deriveNativeChecks(stack, parseJUnit(report)); + const failed = Object.entries(nativeChecks).filter(([key, value]) => !value || !required.includes(key)).map(([key]) => key); + if (failed.length) throw new Error(`NATIVE_GATE_NOT_PASSED: ${failed.sort().join(', ')}`); + return { + ...record, + state: 'native-gates-passed', + nativeChecks, + nativeTestReport: `sha256:${sha256(report)}`, + privateChecks: Object.fromEntries(required.filter(key => !(key in nativeChecks)).map(key => [key, 'pending'])), + qualified: false, + promotion: 'prohibited until authenticated private qualification evidence completes every release gate', + }; +} + +function readFile(file: string): string { + const stat = fs.lstatSync(path.resolve(file)); + if (!stat.isFile() || stat.isSymbolicLink() || stat.size <= 0 || stat.size > MAX_REPORT_BYTES) throw new Error('UNSAFE_NATIVE_EVIDENCE_INPUT'); + return fs.readFileSync(path.resolve(file), 'utf8'); +} + +if (import.meta.main) { + try { + const args = process.argv.slice(2); + const names = ['--stack', '--platform', '--junit', '--staged', '--output']; + if (args.length !== names.length * 2 || names.some((name, index) => args[index * 2] !== name)) { + throw new Error('Usage: cso-native-evidence.ts --stack STACK --platform PLATFORM --junit REPORT.xml --staged staged-image.json --output EVIDENCE.json'); + } + const [stack, platform, junit, staged, output] = names.map((_, index) => args[index * 2 + 1]); + if (!['node', 'bun', 'python', 'rails', 'postgresql'].includes(stack)) throw new Error('INVALID_STACK'); + const evidence = nativeEvidence(JSON.parse(readFile(staged)), stack as Stack, platform, readFile(junit)); + fs.writeFileSync(path.resolve(output), JSON.stringify(evidence, null, 2) + '\n', { flag: 'wx', mode: 0o600 }); + } catch (error) { + process.stderr.write((error instanceof Error ? error.message : 'NATIVE_EVIDENCE_FAILED') + '\n'); + process.exitCode = 1; + } +} diff --git a/scripts/cso-requalification.ts b/scripts/cso-requalification.ts new file mode 100644 index 000000000..5dbf13c42 --- /dev/null +++ b/scripts/cso-requalification.ts @@ -0,0 +1,67 @@ +#!/usr/bin/env bun +/** + * Requalification triggers: the inputs a runtime qualification was measured + * under. A promoted catalog records them; a catalog whose recorded triggers + * differ from the current source is older than its inputs and must be + * requalified (or its runtimes withdrawn) before it ships. + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { canonical, sha256 } from '../lib/cso/contracts'; +import { ISOLATION_POLICY_HASH } from '../lib/cso/docker'; +import { CSO_HELPER_ABI, isEvaluationRuntimeCatalog, type RequalificationTriggers, type RuntimeCatalog } from '../lib/cso/runtime-catalog'; + +const ROOT = path.resolve(import.meta.dir, '..'); +/** Compiled into every runtime image as gstack-cso-verifier and gstack-cso-preparation. */ +export const IMAGE_HELPER_ENTRYPOINTS = ['lib/cso/verifier.ts', 'lib/cso/preparation-container.ts'] as const; +export const IMAGE_CONTEXT = 'lib/cso/images'; +/** Image-context files that describe qualification rather than enter an image; build-inputs.json is its own trigger. */ +const IMAGE_CONTEXT_EXCLUDED = new Set(['README.md', 'qualification.json', 'build-inputs.json']); + +/** Every repository file whose bytes enter a runtime image: the helpers' import closure plus the image build context. */ +export async function imageSourceFiles(root = ROOT): Promise { + const build = await Bun.build({ entrypoints: IMAGE_HELPER_ENTRYPOINTS.map(entry => path.join(root, entry)), target: 'bun', metafile: true }); + if (!build.success || !build.metafile) throw new Error('REQUALIFICATION_SOURCE_CLOSURE_FAILED'); + const closure = Object.keys(build.metafile.inputs).map(input => path.relative(root, path.resolve(root, input)).split(path.sep).join('/')); + if (closure.some(file => file.startsWith('..') || file.includes('node_modules/'))) throw new Error('REQUALIFICATION_SOURCE_OUTSIDE_REPOSITORY'); + const context = fs.readdirSync(path.join(root, IMAGE_CONTEXT), { withFileTypes: true }) + .filter(entry => entry.isFile() && !IMAGE_CONTEXT_EXCLUDED.has(entry.name)) + .map(entry => `${IMAGE_CONTEXT}/${entry.name}`); + return [...new Set([...closure, ...context])].sort(); +} + +export async function requalificationTriggers(root = ROOT): Promise { + const files = await imageSourceFiles(root); + const buildInputs = JSON.parse(fs.readFileSync(path.join(root, IMAGE_CONTEXT, 'build-inputs.json'), 'utf8')); + if (typeof buildInputs?.revision !== 'string' || !buildInputs.revision) throw new Error('INVALID_BUILD_INPUTS_REVISION'); + return { + helperAbi: CSO_HELPER_ABI, + isolationPolicyHash: ISOLATION_POLICY_HASH, + preparationSha256: sha256(canonical(files.map(file => [file, sha256(fs.readFileSync(path.join(root, file)))]))), + buildInputsRevision: buildInputs.revision, + }; +} + +/** Names of the triggers that changed since the catalog's runtimes were qualified; empty when nothing is promoted. */ +export function staleRequalificationTriggers(catalog: RuntimeCatalog, current: RequalificationTriggers): string[] { + if (isEvaluationRuntimeCatalog(catalog) || catalog.runtimes.length === 0) return []; + const recorded = catalog.promotion?.requalification; + if (!recorded) return Object.keys(current).sort(); + return (Object.keys(current) as Array).filter(key => recorded[key] !== current[key]).sort(); +} + +if (import.meta.main) { + try { + const [command, file, ...rest] = process.argv.slice(2); + if (command === 'triggers' && file === undefined) { + process.stdout.write(JSON.stringify(await requalificationTriggers(), null, 2) + '\n'); + } else if (command === 'check' && file && rest.length === 0) { + const stale = staleRequalificationTriggers(JSON.parse(fs.readFileSync(path.resolve(file), 'utf8')), await requalificationTriggers()); + if (stale.length) throw new Error(`STALE_RUNTIME_QUALIFICATION: ${stale.join(', ')} changed since qualification; requalify the runtimes or withdraw them from the catalog`); + process.stdout.write('REQUALIFICATION CURRENT\n'); + } else throw new Error('Usage: cso-requalification.ts triggers | check '); + } catch (error) { + process.stderr.write((error instanceof Error ? error.message : 'REQUALIFICATION_CHECK_FAILED') + '\n'); + process.exitCode = 1; + } +} diff --git a/scripts/cso-runtime-promotion.ts b/scripts/cso-runtime-promotion.ts index e576bd182..4ba5c2e7d 100644 --- a/scripts/cso-runtime-promotion.ts +++ b/scripts/cso-runtime-promotion.ts @@ -7,12 +7,18 @@ import committedCatalog from '../lib/cso/runtime-catalog.json'; import buildInputs from '../lib/cso/images/build-inputs.json'; import { CSO_HELPER_ABI, + EVALUATION_REVISION_PREFIX, + isEvaluationRuntimeCatalog, + validRequalificationTriggers, + type RequalificationTriggers, type QualifiedRuntime, type RuntimeCatalog, type RuntimeQualification, validateRuntimeCatalog, } from '../lib/cso/runtime-catalog'; +import { ISOLATION_POLICY_HASH } from '../lib/cso/docker'; import { imageBuildMatrix, type ImageBuildRow } from './cso-image-matrix'; +import { requalificationTriggers, staleRequalificationTriggers } from './cso-requalification'; const DIGEST = /^sha256:[a-f0-9]{64}$/; const WORKFLOW = /^https:\/\/github\.com\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+\/actions\/runs\/[0-9]+$/; @@ -46,7 +52,7 @@ function versionsKey(value: Record): string { return canonical(value); } -function requiredChecks(stack: ImageBuildRow['stack']): string[] { +export function requiredChecks(stack: ImageBuildRow['stack']): string[] { if (stack === 'postgresql') return [ 'containmentPassed', 'coldStartPassed', 'multiDatabasePassed', 'readinessPassed', 'secretCanaryPassed', 'watchdogCleanupPassed', @@ -112,13 +118,15 @@ function validateStatement(raw: unknown, row: ImageBuildRow): RuntimeQualificati return statement; } -export function catalogPromotionCandidate( - currentValue: unknown, - inputValue: unknown, - evidenceValues: unknown[], -): RuntimeCatalog { +/** The committed (never evaluation-only) catalog and the reviewed build matrix it must describe. */ +function reviewedMatrix(currentValue: unknown, inputValue: unknown): { current: RuntimeCatalog; rows: ImageBuildRow[] } { validateRuntimeCatalog(currentValue); const current = currentValue as RuntimeCatalog; + if (isEvaluationRuntimeCatalog(current)) throw new Error('EVALUATION_CATALOG_NOT_PROMOTABLE'); + return { current, rows: profilesMatchingRows(current, inputValue) }; +} + +function profilesMatchingRows(current: RuntimeCatalog, inputValue: unknown): ImageBuildRow[] { const rows = imageBuildMatrix(inputValue).include; if (!current.profiles || current.buildRevision !== rows[0].inputRevision) throw new Error('CATALOG_BUILD_REVISION_MISMATCH'); const profiles = new Map(current.profiles.map(profile => [profile.id, profile])); @@ -128,17 +136,100 @@ export function catalogPromotionCandidate( if (!profile || profile.stack !== row.stack || profile.platform !== row.platform || versionsKey(profile.versions) !== versionsKey(row.versions)) throw new Error(`CATALOG_PROFILE_MATRIX_MISMATCH: ${row.runtimeId}`); } - if (evidenceValues.length !== rows.length) throw new Error('INCOMPLETE_QUALIFICATION_MATRIX'); + return rows; +} + +/** The candidate build's gate: a valid evaluation-only catalog for the reviewed build inputs. */ +export function validateEvaluationCandidate(catalogValue: unknown, inputValue: unknown): RuntimeCatalog { + validateRuntimeCatalog(catalogValue); + const catalog = catalogValue as RuntimeCatalog; + if (!isEvaluationRuntimeCatalog(catalog)) throw new Error('NOT_AN_EVALUATION_CATALOG'); + const rows = profilesMatchingRows(catalog, inputValue); + for (const row of rows) { + const runtime = catalog.runtimes.find(item => item.id === row.runtimeId); + if (!runtime || !OUTPUT_IMAGE.test(runtime.image) || !runtime.image.includes(`/${row.stack}-${row.arch}@`)) throw new Error(`INVALID_QUALIFIED_IMAGE: ${row.runtimeId}`); + } + return catalog; +} + +function matchRows(rows: ImageBuildRow[], values: unknown[]): Array<[ImageBuildRow, unknown]> { + if (values.length !== rows.length) throw new Error('INCOMPLETE_QUALIFICATION_MATRIX'); const byId = new Map(); - for (const value of evidenceValues) { + for (const value of values) { const id = (value as any)?.runtimeId; if (typeof id !== 'string' || byId.has(id)) throw new Error('DUPLICATE_QUALIFICATION_STATEMENT'); byId.set(id, value); } - const statements = rows.map(row => { + return rows.map(row => { if (!byId.has(row.runtimeId)) throw new Error(`MISSING_QUALIFICATION_STATEMENT: ${row.runtimeId}`); - return validateStatement(byId.get(row.runtimeId), row); + return [row, byId.get(row.runtimeId)]; }); +} + +const STAGED_FIELDS = [ + 'buildRevision', 'helperAbi', 'image', 'platform', 'provenanceDigest', 'qualification', 'runtimeId', + 'sbomDigest', 'schemaVersion', 'sourceCommit', 'stack', 'state', 'verifiedProvenance', 'versions', 'workflow', +].sort(); + +/** + * Build an evaluation-only catalog from one staging run's staged-image.json + * records. It measures no release gate: every runtime is an + * evaluation_candidate, and setup, distribution builds and promotion refuse it. + */ +export function evaluationCandidateCatalog(currentValue: unknown, inputValue: unknown, stagedValues: unknown[], createdAt: string): RuntimeCatalog { + const { current, rows } = reviewedMatrix(currentValue, inputValue); + if (!Number.isFinite(Date.parse(createdAt))) throw new Error('INVALID_EVALUATION_TIME'); + const staged = matchRows(rows, stagedValues).map(([row, value]) => { + const record = value as RuntimeQualificationStatement; + if (!record || typeof record !== 'object' || canonical(Object.keys(record).sort()) !== canonical(STAGED_FIELDS) || + record.schemaVersion !== 1 || record.helperAbi !== CSO_HELPER_ABI || (record.state as string) !== 'staged' || (record as any).verifiedProvenance !== true) throw new Error(`INVALID_STAGED_RUNTIME: ${row.runtimeId}`); + if (record.buildRevision !== row.inputRevision || record.stack !== row.stack || record.platform !== row.platform || + versionsKey(record.versions) !== versionsKey(row.versions)) throw new Error(`QUALIFICATION_BUILD_MISMATCH: ${row.runtimeId}`); + if (!OUTPUT_IMAGE.test(record.image) || !record.image.includes(`/${row.stack}-${row.arch}@`)) throw new Error(`INVALID_QUALIFIED_IMAGE: ${row.runtimeId}`); + if (!/^[a-f0-9]{40}$/.test(record.sourceCommit) || !WORKFLOW.test(record.workflow) || !DIGEST.test(record.sbomDigest) || + !DIGEST.test(record.provenanceDigest)) throw new Error(`MISSING_QUALIFICATION_PROVENANCE: ${row.runtimeId}`); + return record; + }); + if (new Set(staged.map(item => item.sourceCommit)).size !== 1 || new Set(staged.map(item => item.workflow)).size !== 1) throw new Error('SPLIT_STAGING_RUN'); + const { sourceCommit, workflow } = staged[0]; + const runtimes: QualifiedRuntime[] = staged.map(record => ({ + id: record.runtimeId, + stack: record.stack, + platform: record.platform, + state: 'evaluation_candidate', + image: record.image, + entrypoint: '/opt/cso/entrypoint', + helperAbi: CSO_HELPER_ABI, + versions: record.versions, + policyVersion: 'cso-isolation-v1', + qualifiedAt: createdAt, + qualification: { kind: 'evaluation', sourceCommit, workflow, sbomDigest: record.sbomDigest, provenanceDigest: record.provenanceDigest, verifiedProvenance: true }, + })); + const candidate: RuntimeCatalog = { + schemaVersion: 1, + revision: `${EVALUATION_REVISION_PREFIX}${workflow.slice(workflow.lastIndexOf('/') + 1)}-${sourceCommit.slice(0, 12)}`, + previousRevision: current.revision, + helperAbi: CSO_HELPER_ABI, + buildRevision: current.buildRevision, + profiles: current.profiles, + evaluation: { sourceCommit, workflow, evidenceDigest: sha256(canonical(runtimes)) }, + runtimes, + }; + validateRuntimeCatalog(candidate); + return candidate; +} + +export function catalogPromotionCandidate( + currentValue: unknown, + inputValue: unknown, + evidenceValues: unknown[], + /** Measured by the staging run at sourceCommit (scripts/cso-requalification.ts triggers). */ + requalification: RequalificationTriggers, +): RuntimeCatalog { + const { current, rows } = reviewedMatrix(currentValue, inputValue); + if (!validRequalificationTriggers(requalification, current.buildRevision)) throw new Error('INVALID_REQUALIFICATION_TRIGGERS'); + if (requalification.isolationPolicyHash !== ISOLATION_POLICY_HASH) throw new Error('STALE_RUNTIME_QUALIFICATION: isolationPolicyHash'); + const statements = matchRows(rows, evidenceValues).map(([row, value]) => validateStatement(value, row)); const sourceCommits = new Set(statements.map(item => item.sourceCommit)); const workflows = new Set(statements.map(item => item.workflow)); if (sourceCommits.size !== 1 || workflows.size !== 1) throw new Error('SPLIT_QUALIFICATION_RUN'); @@ -167,7 +258,15 @@ export function catalogPromotionCandidate( helperAbi: CSO_HELPER_ABI, buildRevision: current.buildRevision, profiles: current.profiles, - promotion: { sourceCommit, workflow, evidenceDigest, qualificationEvidenceDigest }, + promotion: { + sourceCommit, workflow, evidenceDigest, qualificationEvidenceDigest, + requalification: { + helperAbi: requalification.helperAbi, + isolationPolicyHash: requalification.isolationPolicyHash, + preparationSha256: requalification.preparationSha256, + buildInputsRevision: requalification.buildInputsRevision, + }, + }, runtimes, }; validateRuntimeCatalog(candidate); @@ -180,6 +279,7 @@ export function validateRuntimeCatalogTransition(currentValue: unknown, proposed validateRuntimeCatalog(proposedValue); const current = currentValue as RuntimeCatalog; const proposed = proposedValue as RuntimeCatalog; + if (isEvaluationRuntimeCatalog(current) || isEvaluationRuntimeCatalog(proposed)) throw new Error('EVALUATION_CATALOG_NOT_PROMOTABLE'); if (!proposed.promotion || proposed.runtimes.length === 0) throw new Error('UNQUALIFIED_RUNTIME_CATALOG_PROMOTION'); if (proposed.revision === current.revision || proposed.previousRevision !== current.revision) { throw new Error('RUNTIME_CATALOG_BASE_REVISION_MISMATCH'); @@ -190,7 +290,7 @@ export function validateRuntimeCatalogTransition(currentValue: unknown, proposed } } -function collectEvidence(root: string): unknown[] { +function collectEvidence(root: string, name = 'qualified-runtime.json'): unknown[] { const absolute = path.resolve(root); const rootStat = fs.lstatSync(absolute); if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) throw new Error('UNSAFE_EVIDENCE_ROOT'); @@ -201,7 +301,7 @@ function collectEvidence(root: string): unknown[] { const file = path.join(directory, entry.name); if (entry.isSymbolicLink() || (!entry.isDirectory() && !entry.isFile())) throw new Error('UNSAFE_EVIDENCE_ENTRY'); if (entry.isDirectory()) visit(file, depth + 1); - else if (entry.name === 'qualified-runtime.json') files.push(file); + else if (entry.name === name) files.push(file); if (files.length > MAX_EVIDENCE_FILES) throw new Error('TOO_MANY_QUALIFICATION_STATEMENTS'); } }; @@ -234,7 +334,19 @@ function readCatalogArtifact(file: string): unknown { if (import.meta.main) { try { const args = process.argv.slice(2); - if (args[0] === 'validate-transition') { + if (args[0] === 'check-evaluation') { + if (args.length !== 2) throw new Error('Usage: cso-runtime-promotion.ts check-evaluation '); + const catalog = validateEvaluationCandidate(readCatalogArtifact(args[1]), buildInputs); + process.stdout.write(`EVALUATION-ONLY ${catalog.revision}\n`); + } else if (args[0] === 'evaluation-candidate') { + if (args.length !== 5 || args[1] !== '--staged-root' || args[3] !== '--output') { + throw new Error('Usage: cso-runtime-promotion.ts evaluation-candidate --staged-root --output '); + } + const candidate = evaluationCandidateCatalog(committedCatalog, buildInputs, collectEvidence(args[2], 'staged-image.json'), new Date().toISOString()); + const output = path.resolve(args[4]); + fs.writeFileSync(output, JSON.stringify(candidate, null, 2) + '\n', { flag: 'wx', mode: 0o600 }); + process.stdout.write(`${output}\n`); + } else if (args[0] === 'validate-transition') { if (args.length !== 3) throw new Error('Usage: cso-runtime-promotion.ts validate-transition '); validateRuntimeCatalogTransition(readCatalogArtifact(args[1]), readCatalogArtifact(args[2])); process.stdout.write('VALID TRANSITION\n'); @@ -242,7 +354,11 @@ if (import.meta.main) { if (args.length !== 4 || args[0] !== '--evidence-root' || args[2] !== '--output') { throw new Error('Usage: cso-runtime-promotion.ts --evidence-root --output '); } - const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, collectEvidence(args[1])); + const triggers = collectEvidence(args[1], 'requalification.json'); + if (triggers.length !== 1) throw new Error('MISSING_REQUALIFICATION_TRIGGERS'); + const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, collectEvidence(args[1]), triggers[0] as RequalificationTriggers); + const stale = staleRequalificationTriggers(candidate, await requalificationTriggers()); + if (stale.length) throw new Error(`STALE_RUNTIME_QUALIFICATION: ${stale.join(', ')} changed since the staged commit; re-stage and requalify`); const output = path.resolve(args[3]); fs.writeFileSync(output, JSON.stringify(candidate, null, 2) + '\n', { flag: 'wx', mode: 0o600 }); process.stdout.write(`${output}\n`); diff --git a/scripts/cso-staging-run.ts b/scripts/cso-staging-run.ts new file mode 100644 index 000000000..d22713e86 --- /dev/null +++ b/scripts/cso-staging-run.ts @@ -0,0 +1,124 @@ +#!/usr/bin/env bun +/** Bind a qualification dispatch to the one successful protected-main staging run that signed its images. */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { canonical } from '../lib/cso/contracts'; +import { committedImageBuildMatrix } from './cso-image-matrix'; + +export const STAGING_WORKFLOW_PATH = '.github/workflows/cso-runtime-images.yml'; +export const NATIVE_JOB = 'qualify-native'; +const SLSA_PROVENANCE = 'https://slsa.dev/provenance/v1'; +const REPOSITORY = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; +const INVOCATION = /^https:\/\/github\.com\/([A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+)\/actions\/runs\/([1-9][0-9]*)\/attempts\/[1-9][0-9]*$/; +const MAX_FILE_BYTES = 4 * 1024 * 1024; + +export interface StagingRunEvidence { + repository: string; + runId: string; + sourceCommit: string; + runtimeIds: string[]; + run: unknown; + jobPages: unknown; +} + +/** + * Resolve the staging run from verified SLSA provenance. The predicate's + * invocation and the Fulcio certificate's run URI must agree, and every + * verified statement for every image must name the same run. + */ +export function stagingRunId(repository: string, verifications: unknown[]): string { + if (!REPOSITORY.test(repository) || verifications.length === 0) throw new Error('INVALID_STAGING_PROVENANCE'); + const runs = new Set(); + for (const verification of verifications) { + if (!Array.isArray(verification) || verification.length === 0) throw new Error('INVALID_STAGING_PROVENANCE'); + for (const item of verification) { + const result = (item as any)?.verificationResult; + if (result?.statement?.predicateType !== SLSA_PROVENANCE) throw new Error('INVALID_STAGING_PROVENANCE'); + const invocation = result.statement.predicate?.runDetails?.metadata?.invocationId; + const match = typeof invocation === 'string' ? INVOCATION.exec(invocation) : null; + if (!match || match[1] !== repository || result.signature?.certificate?.runInvocationURI !== invocation) { + throw new Error('STAGING_INVOCATION_MISMATCH'); + } + runs.add(match[2]); + } + } + if (runs.size !== 1) throw new Error('STAGING_RUN_NOT_UNIQUE'); + return [...runs][0]; +} + +/** Every statement in one qualification dispatch names the same staged source commit. */ +export function dispatchSourceCommit(event: unknown): string { + const statements = (event as any)?.client_payload?.statements; + if (!Array.isArray(statements) || statements.length === 0) throw new Error('INVALID_QUALIFICATION_DISPATCH'); + const commits = new Set(statements.map((statement: any) => statement?.sourceCommit)); + const [commit] = [...commits]; + if (commits.size !== 1 || typeof commit !== 'string' || !/^[a-f0-9]{40}$/.test(commit)) throw new Error('STAGING_SOURCE_COMMIT_MISMATCH'); + return commit; +} + +/** The resolved run must be the protected-main staging dispatch at the statements' commit, with every native gate passed. */ +export function verifyStagingRun(evidence: StagingRunEvidence): void { + const { repository, runId, sourceCommit, runtimeIds } = evidence; + const run = evidence.run as any; + if (!run || typeof run !== 'object' || String(run.id) !== runId) throw new Error('STAGING_RUN_ID_MISMATCH'); + if (run.path !== STAGING_WORKFLOW_PATH) throw new Error('STAGING_WORKFLOW_MISMATCH'); + if (run.repository?.full_name !== repository || run.head_repository?.full_name !== repository) throw new Error('STAGING_REPOSITORY_MISMATCH'); + if (run.event !== 'workflow_dispatch' || run.head_branch !== 'main') throw new Error('STAGING_RUN_NOT_PROTECTED_MAIN_DISPATCH'); + if (run.status !== 'completed' || run.conclusion !== 'success') throw new Error('STAGING_RUN_NOT_SUCCESSFUL'); + if (run.head_sha !== sourceCommit) throw new Error('STAGING_SOURCE_COMMIT_MISMATCH'); + + const pages = evidence.jobPages; + if (!Array.isArray(pages) || pages.length === 0) throw new Error('INVALID_STAGING_JOBS'); + const totals = new Set(pages.map((page: any) => page?.total_count)); + if (pages.some((page: any) => !Array.isArray(page?.jobs)) || totals.size !== 1) throw new Error('INVALID_STAGING_JOBS'); + const jobs = pages.flatMap((page: any) => page.jobs); + if (jobs.length !== [...totals][0]) throw new Error('INCOMPLETE_STAGING_JOBS'); + + const native = jobs.filter((job: any) => typeof job?.name === 'string' && job.name.startsWith(NATIVE_JOB)); + const expected = runtimeIds.map(id => `${NATIVE_JOB} ${id}`).sort(); + if (runtimeIds.length === 0 || canonical(native.map((job: any) => job.name).sort()) !== canonical(expected)) { + throw new Error('STAGING_NATIVE_JOBS_MISMATCH'); + } + if (native.some((job: any) => String(job.run_id) !== runId || job.status !== 'completed' || job.conclusion !== 'success')) { + throw new Error('STAGING_NATIVE_GATE_NOT_PASSED'); + } +} + +function readJson(file: string): unknown { + const absolute = path.resolve(file); + const stat = fs.lstatSync(absolute); + if (!stat.isFile() || stat.isSymbolicLink() || stat.size <= 0 || stat.size > MAX_FILE_BYTES) throw new Error('UNSAFE_STAGING_EVIDENCE_FILE'); + try { return JSON.parse(fs.readFileSync(absolute, 'utf8')); } catch { throw new Error('INVALID_STAGING_EVIDENCE_FILE'); } +} + +function option(args: string[], name: string): string { + const index = args.indexOf(name); + const value = index >= 0 ? args[index + 1] : undefined; + if (!value || value.startsWith('--')) throw new Error(`Missing ${name}`); + return value; +} + +if (import.meta.main) { + try { + const [command, ...args] = process.argv.slice(2); + if (command === 'run-id' && args[0] === '--repository' && args.length >= 3) { + process.stdout.write(`${stagingRunId(args[1], args.slice(2).map(readJson))}\n`); + } else if (command === 'verify' && args.length === 10) { + const event = readJson(option(args, '--event')); + verifyStagingRun({ + repository: option(args, '--repository'), + runId: option(args, '--run-id'), + sourceCommit: dispatchSourceCommit(event), + runtimeIds: committedImageBuildMatrix().include.map(row => row.runtimeId), + run: readJson(option(args, '--run')), + jobPages: readJson(option(args, '--jobs')), + }); + process.stdout.write('STAGING RUN VERIFIED\n'); + } else { + throw new Error('Usage: cso-staging-run.ts run-id --repository OWNER/REPO PROVENANCE.json... | verify --repository OWNER/REPO --run-id ID --event EVENT.json --run RUN.json --jobs JOB-PAGES.json'); + } + } catch (error) { + process.stderr.write((error instanceof Error ? error.message : 'STAGING_RUN_VERIFICATION_FAILED') + '\n'); + process.exitCode = 1; + } +} diff --git a/setup b/setup index 70947d524..84c67a3e1 100755 --- a/setup +++ b/setup @@ -1806,6 +1806,12 @@ if [ "$CSO_BUILD_AVAILABLE" -eq 1 ]; then echo "gstack setup failed: CSO build completed without its required native artifact set" >&2 exit 1 fi + # Evaluation-only helpers embed an unqualified cso-eval-* catalog for the + # private evaluator. Setup never installs one, whether marked or copied bare. + if [ -e "$SOURCE_GSTACK_DIR/bin/.gstack-cso-evaluation" ] || "$SOURCE_GSTACK_DIR/bin/gstack-cso-launcher$_EXE" --version 2>/dev/null | grep -q '"evaluationOnly": *true'; then + echo "gstack setup failed: bin/ holds an evaluation-only CSO helper; remove the bin/gstack-cso-* artifacts and re-run setup" >&2 + exit 1 + fi # Runtime/scanner execution never pulls. Setup is the sole automatic # acquisition path: the trusted launcher validates committed catalogs, # rejects remote Docker contexts, and uses an empty Docker config for diff --git a/test/cso-contracts.test.ts b/test/cso-contracts.test.ts index be474ceda..e9513eddd 100644 --- a/test/cso-contracts.test.ts +++ b/test/cso-contracts.test.ts @@ -4,7 +4,7 @@ import * as os from 'node:os'; import * as path from 'node:path'; import * as crypto from 'node:crypto'; import { canonical, completeness, CsoError, fingerprint, importLegacy, renderReport, sha256, snapshotPathHandle, snapshotPathId, validateCoverage, validateFinding, validateVerificationObservation, validateVerificationRequest } from '../lib/cso/contracts'; -import { assertCanonicalStartPlan, canonicalStartPlan, canonicalTestPlan, certify, fileEffect, makeReviewArtifact, patchHash, preparePatchedSource, resolveVerificationRequestPaths, testExecutionPassed, treeHash, validateRepairBundle, validateReviewArtifact, verificationIdentity, verifyRepair } from '../lib/cso/verification'; +import { assertCanonicalStartPlan, canonicalStartPlan, canonicalTestPlan, certify, fileEffect, makeReviewArtifact, patchHash, preparePatchedSource, resolveVerificationRequestPaths, testExecutionPassed, treeHash, validateRepairBundle, validateReviewArtifact, verificationHarnessHash, verificationIdentity, verifyRepair } from '../lib/cso/verification'; import { assertionWitnessPairHash, witnessObservationHash } from '../lib/cso/witness'; import { admit, machinePoolRoot, markSupervised, release } from '../lib/cso/admission'; import { sanitizeHelperForJson } from '../lib/cso/process'; @@ -211,6 +211,50 @@ describe('tested repair certificate gate',()=>{ const drift=structuredClone(bundle);drift.preparation!.after.preparedDependencyHash=h('b');drift.verification.preparationHash=sha256(canonical(drift.preparation));drift.verification.id='';const rebound=verificationIdentity(drift.verification);drift.verification.id=rebound;drift.id=rebound;expect(()=>validateRepairBundle(drift,rebound,before)).toThrow('test toolchain changed'); expect(()=>certify({...params,preparation:undefined})).toThrow('require before/after prepared dependency proofs');expect(()=>certify({...params,preparation:{before:proof(h('9')),after:proof(h('b'))}})).toThrow('toolchain bytes changed'); }); + const securityArray=()=>[ + {name:'tenant isolation',path:'/user?id=2',method:'GET',expected:{status:403},vulnerable:{status:200,includes:'tenant-b'}}, + {name:'tenant isolation by header',path:'/user',method:'GET',headers:{'x-tenant':'b'},expected:{status:403},vulnerable:{status:200,includes:'tenant-b'}}, + {name:'tenant isolation on write',path:'/user/2',method:'PUT',body:'{"name":"x"}',expected:{status:404},vulnerable:{status:204}}, + ]; + test('single-object security keeps its pre-array request, patch, assertion, and harness bytes',()=>{ + const root=tmp();fs.writeFileSync(path.join(root,'test.js'),'pass\n');fs.chmodSync(path.join(root,'test.js'),0o644); + const raw:any=request('vulnerable\n','fixed\n');raw.review.reviewedPatchHash=patchHash(raw);const req=validateVerificationRequest(raw); + expect(Array.isArray(req.security)).toBe(false);expect(canonical(req.security)).toBe(canonical(raw.security)); + expect({patch:patchHash(req),request:sha256(canonical(req)),assertion:sha256(canonical({legitimate:req.legitimate,security:req.security})),harness:verificationHarnessHash(req,root)}).toEqual({ + patch:'cc6ab00ee0160ce484a86130f94a5e6e5f588f3ec975c781c4d077647adff483', + request:'85175d755f1b297eb7d15dc1d9850e1ac453d1d129ceee58213d89cbab0ee59c', + assertion:'7e0f792881bc14137f2aa70b0662a013939749903e49eb507c1ecf48561b0771', + harness:'2a74b61982270fd10c071f0bdd32ae37f04de6e3eca1db6c9225132de29b4309', + }); + }); + test('security accepts a bounded array of distinct assertions that each carry an exclusive vulnerable oracle',()=>{ + const make=(security:unknown)=>{const raw:any=request('vulnerable\n','fixed\n');raw.security=security;raw.review.reviewedPatchHash=patchHash(raw);return raw;}; + const accepted=validateVerificationRequest(make(securityArray()));expect(accepted.security).toEqual(securityArray() as any); + expect(validateVerificationRequest(make(securityArray().slice(0,1))).security).toHaveLength(1); + const full=Array.from({length:8},(_,i)=>({...securityArray()[0],name:`payload ${i}`}));expect(validateVerificationRequest(make(full)).security).toHaveLength(8); + expect(()=>validateVerificationRequest(make([]))).toThrow('1..8 assertions'); + expect(()=>validateVerificationRequest(make([...full,{...full[0],name:'payload 8'}]))).toThrow('1..8 assertions'); + expect(()=>validateVerificationRequest(make([securityArray()[0],{...securityArray()[1],name:'tenant isolation'}]))).toThrow('names must be distinct'); + const missing=securityArray();delete (missing[2] as any).vulnerable;expect(()=>validateVerificationRequest(make(missing))).toThrow('distinct before/fixed security oracles'); + const overlapping=securityArray();overlapping[1].vulnerable={...overlapping[1].expected} as any;expect(()=>validateVerificationRequest(make(overlapping))).toThrow('mutually exclusive'); + const extra:any=securityArray();extra[0].expected.untrusted='yes';expect(()=>validateVerificationRequest(make(extra))).toThrow('Unexpected security[0].expected field'); + expect(()=>validateVerificationRequest(make([null]))).toThrow(); + }); + test('an array security request is observed in exactly one before/after pair and binds every member',async()=>{ + const runDir=tmp(),snapshot=path.join(runDir,'snapshot');fs.mkdirSync(snapshot);fs.writeFileSync(path.join(snapshot,'app.js'),'vulnerable\n');const testBody='test("ok",()=>{})\n';fs.writeFileSync(path.join(snapshot,'app.test.js'),testBody);const packageJson='{"scripts":{"test":"node --test"}}';fs.writeFileSync(path.join(snapshot,'package.json'),packageJson); + const raw:any=request('vulnerable\n','fixed\n');raw.findingId='1abcdefabcdefabcdefabcdefabcdefa';raw.security=securityArray();raw.existingTests=[{executable:'/usr/local/bin/node',args:['--test','--test-reporter=tap','./app.test.js']}];raw.testFiles=['app.test.js'];raw.review.reviewedPatchHash=patchHash(raw); + const manifest:any={version:3,createdAt:'2026-01-01T00:00:00Z',expiresAt:'2026-01-08T00:00:00Z',root:'/repo',headCommit:'a'.repeat(40),originalHash:'b'.repeat(64),executionHash:treeHash(snapshot),entries:[{path:'app.js',originalHash:sha256('vulnerable\n'),executionHash:sha256('vulnerable\n'),bytes:11,mode:0o600},{path:'app.test.js',originalHash:sha256(testBody),executionHash:sha256(testBody),bytes:Buffer.byteLength(testBody),mode:0o600},{path:'package.json',originalHash:sha256(packageJson),executionHash:sha256(packageJson),bytes:packageJson.length,mode:0o600}]}; + const runtime:any={id:'node',stack:'node',image:'runtime@sha256:'+'a'.repeat(64),platform:'linux/amd64'},phases:string[]=[]; + const executor:any={observe:async(_source:string,phase:string,observed:any)=>{phases.push(phase);expect(observed.security).toEqual(securityArray());return{booted:true,legitimate:true,security:phase==='before'?'intended_failure':'pass',existingTests:true,output:'ok',inputHash:''};}}; + let unattested:any;try{await verifyRepair({runId:'run',runDir,manifest,rawRequest:raw,runtime,verifier:runtime,policyHash:'c'.repeat(64),archives:[],executor,persist:false});}catch(error){unattested=error;} + expect(phases).toEqual(['before','after']);expect(unattested).toMatchObject({code:'PREREQUISITE',attempt:{reproduction:'reproduced',repair:'proposed',bundleIssued:false}}); + const root=tmp(),before=path.join(root,'before'),after=path.join(root,'after');fs.mkdirSync(before);fs.writeFileSync(path.join(before,'app.js'),'vulnerable\n');fs.writeFileSync(path.join(before,'test.js'),'pass\n'); + const plain:any=request('vulnerable\n','fixed\n');plain.security=securityArray();plain.review.reviewedPatchHash=patchHash(plain);const req=validateVerificationRequest(plain);preparePatchedSource(before,after,req); + const params:any={runId:'run',manifest:{originalHash:'d'.repeat(64),executionHash:treeHash(before),entries:[]},request:req,runtime:{id:'node',image:'runtime@sha256:'+'a'.repeat(64),platform:'linux/amd64'},verifier:{image:'runtime@sha256:'+'a'.repeat(64)},before:{booted:true,legitimate:true,security:'intended_failure',existingTests:true,output:'before',inputHash:''},after:{booted:true,legitimate:true,security:'pass',existingTests:true,output:'after',inputHash:''},beforeRoot:before,afterRoot:after,policyHash:'c'.repeat(64),auditPolicyHash:'e'.repeat(64),archives:[],testToolchain:'runtime'}; + const bundle=authenticatedBundleFixture(certify(params),params);expect(validateRepairBundle(bundle,bundle.id,before).id).toBe(bundle.id); + const dropped=structuredClone(bundle);(dropped.request.security as any[]).pop();expect(()=>validateRepairBundle(dropped,dropped.id,before)).toThrow('harness'); + expect(certify({...params,after:{...params.after,security:'inconclusive'}}).manifest.result).toBe('inconclusive'); + }); }); function tree(root:string){return sha256(JSON.stringify(fs.readdirSync(root).sort().map(p=>{const file=path.join(root,p),stat=fs.statSync(file);return[p,sha256(fs.readFileSync(file)),stat.mode&0o777];})));} diff --git a/test/cso-distribution.test.ts b/test/cso-distribution.test.ts index 9fe4d6bc0..98168153c 100644 --- a/test/cso-distribution.test.ts +++ b/test/cso-distribution.test.ts @@ -1,10 +1,17 @@ import { afterEach, describe, expect, test } from 'bun:test'; -import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { createHash } from 'node:crypto'; import { tmpdir } from 'node:os'; -import { join, resolve } from 'node:path'; +import { dirname, join, resolve } from 'node:path'; import { spawnSync } from 'node:child_process'; -import { imageBuildMatrix } from '../scripts/cso-image-matrix'; +import committedCatalog from '../lib/cso/runtime-catalog.json'; +import buildInputs from '../lib/cso/images/build-inputs.json'; +import { committedImageBuildMatrix, imageBuildMatrix } from '../scripts/cso-image-matrix'; +import { catalogPromotionCandidate, requiredChecks } from '../scripts/cso-runtime-promotion'; import { runBashScript } from './helpers/bash-script'; +import { requalificationTriggers } from '../scripts/cso-requalification'; + +const CURRENT_TRIGGERS = await requalificationTriggers(); const ROOT = resolve(import.meta.dir, '..'); const temps: string[] = []; @@ -250,12 +257,75 @@ describe('CSO build and distribution wiring', () => { }); }); +const EVALUATION_REF = `sha256:${'e'.repeat(64)}`; +const STAGING_RUN = '777'; +function dispatchedStatements(): any[] { + return committedImageBuildMatrix().include.map(row => ({ + schemaVersion: 1, helperAbi: 3, state: 'qualified', buildRevision: row.inputRevision, runtimeId: row.runtimeId, + stack: row.stack, platform: row.platform, versions: row.versions, sourceCommit: 'a'.repeat(40), + image: `ghcr.io/garrytan/gstack/cso-staging/${row.stack}-${row.arch}@sha256:${Bun.hash(row.runtimeId).toString(16).padStart(64, '0')}`, + checks: Object.fromEntries(requiredChecks(row.stack).map(key => [key, true])), + evaluationRef: EVALUATION_REF, + })); +} +function ingressStep(name: string): string { + const job = (Bun.YAML.parse(readFileSync(join(ROOT, '.github/workflows/cso-runtime-qualification.yml'), 'utf8')) as any).jobs.qualify; + return (job.steps.find((step: any) => step.name === name).run as string).replaceAll('bun run scripts/', `bun run ${quote(ROOT)}/scripts/`); +} +function ingressFixture() { + const dir = mkdtempSync(join(tmpdir(), 'cso-ingress-')); temps.push(dir); + const bin = join(dir, 'bin'), work = join(dir, 'work'), fake = join(dir, 'gh-fixtures'); + for (const path of [bin, work, fake]) mkdirSync(path); + writeFileSync(join(bin, 'gh'), [ + '#!/bin/sh', + 'set -eu', + 'case "$1 $2" in', + ' "attestation verify")', + ' case "$*" in', + ' *https://slsa.dev/provenance/v1*) jq --arg image "$3" \'.[0].verificationResult.statement.subject[0].name = $image\' "$FAKE_GH/provenance.json" ;;', + ' *) jq -n --arg image "$3" \'[{verificationResult:{statement:{predicateType:"https://spdx.dev/Document/v2.3",subject:[{name:$image}]}}}]\' ;;', + ' esac ;;', + ' "api --paginate") cat "$FAKE_GH/jobs.json" ;;', + ' "run download") while [ "$#" -gt 0 ]; do [ "$1" != --dir ] || dir=$2; shift; done; mkdir -p "$dir"; cp "$FAKE_GH/requalification.json" "$dir/" ;;', + ' "api repos/"*) cat "$FAKE_GH/run.json" ;;', + ' *) echo "unexpected gh $*" >&2; exit 2 ;;', + 'esac', + '', + ].join('\n')); + chmodSync(join(bin, 'gh'), 0o755); + const invocation = `https://github.com/garrytan/gstack/actions/runs/${STAGING_RUN}/attempts/1`; + writeFileSync(join(fake, 'provenance.json'), JSON.stringify([{ verificationResult: { + signature: { certificate: { runInvocationURI: invocation } }, + statement: { predicateType: 'https://slsa.dev/provenance/v1', subject: [{ name: '' }], predicate: { runDetails: { metadata: { invocationId: invocation } } } }, + } }])); + const run = (overrides: Record = {}) => writeFileSync(join(fake, 'run.json'), JSON.stringify({ + id: Number(STAGING_RUN), path: '.github/workflows/cso-runtime-images.yml', event: 'workflow_dispatch', head_branch: 'main', + head_sha: 'a'.repeat(40), status: 'completed', conclusion: 'success', + repository: { full_name: 'garrytan/gstack' }, head_repository: { full_name: 'garrytan/gstack' }, ...overrides, + })); + const jobs = (conclusion = 'success') => { + const list = committedImageBuildMatrix().include.map((row, index) => ({ + name: `qualify-native ${row.runtimeId}`, run_id: Number(STAGING_RUN), status: 'completed', conclusion: index === 0 ? conclusion : 'success', + })); + writeFileSync(join(fake, 'jobs.json'), JSON.stringify([{ total_count: list.length, jobs: list }])); + }; + run(); jobs(); + writeFileSync(join(fake, 'requalification.json'), JSON.stringify(CURRENT_TRIGGERS)); + const event = join(dir, 'event.json'); + writeFileSync(event, JSON.stringify({ action: 'cso-runtime-qualified', client_payload: { evaluationRef: EVALUATION_REF, statements: dispatchedStatements() } })); + const env = { ...process.env, PATH: `${bin}:${dirname(process.execPath)}:${process.env.PATH}`, FAKE_GH: fake, GITHUB_EVENT_PATH: event, + GITHUB_REPOSITORY: 'garrytan/gstack', GITHUB_RUN_ID: '555' }; + const step = (name: string) => runBashScript(ingressStep(name), { cwd: work, env, timeout: 60_000 }); + return { work, run, jobs, step }; +} + describe('CSO runtime staging gates', () => { test('runtime workflow shell blocks parse after GitHub expressions are substituted', () => { for (const relative of [ '.github/workflows/cso-runtime-images.yml', '.github/workflows/cso-runtime-qualification.yml', '.github/workflows/cso-runtime-promote.yml', + '.github/workflows/cso-scanner-images.yml', ]) { const raw = readFileSync(join(ROOT, relative), 'utf8'); const workflow = Bun.YAML.parse(raw) as any; @@ -384,13 +454,15 @@ describe('CSO runtime staging gates', () => { expect(workflow.jobs['qualify-native'].needs).toEqual(['reviewed-inputs', 'stage']); expect(raw).toContain('cso-staged-postgresql-${{ matrix.arch }}'); expect(raw).toContain('GSTACK_CSO_TEST_POSTGRES_IMAGE'); - expect(raw).toContain('acquisitionPublicOnlyPassed:true'); - expect(raw).toContain('positiveNegativeAssertionsPassed:true'); - expect(raw).toContain('heldOutRepairPassed:"pending"'); - expect(raw).toContain('railsSqlitePassed:true'); - expect(raw).toContain('railsPostgresqlPassed:true'); - expect(raw).toContain('nativeExtensionsPassed:true'); - expect(raw).toContain('qualified:false'); + const qualify = workflow.jobs['qualify-native']; + expect(qualify.name).toBe('qualify-native ${{ matrix.runtimeId }}'); + expect(raw).not.toMatch(/Passed:\s*true/); + const tested = qualify.steps.findIndex((s: any) => s.run?.includes('bun run test:cso:docker --reporter=junit --reporter-outfile=native-tests.junit.xml')); + const derived = qualify.steps.findIndex((s: any) => s.run?.includes('scripts/cso-native-evidence.ts --stack "$CSO_STACK" --platform "$CSO_PLATFORM"')); + expect(tested).toBeGreaterThanOrEqual(0); + expect(derived).toBe(tested + 1); + expect(qualify.steps[derived].run).toContain('--junit native-tests.junit.xml --staged staged-application/staged-image.json'); + expect(qualify.steps[derived + 1].with.path.split('\n').filter(Boolean)).toEqual(['qualification-evidence.json', 'native-tests.junit.xml']); expect(raw).not.toContain('setup-qemu'); expect(raw).not.toContain('git push'); expect(raw).not.toContain('gh pr create'); @@ -412,7 +484,7 @@ describe('CSO runtime staging gates', () => { expect(Object.keys(workflow.on)).toEqual(['workflow_dispatch']); expect(job.if).toContain("github.ref == 'refs/heads/main'"); expect(job.environment).toBe('cso-runtime-release'); - expect(job.permissions).toMatchObject({ contents: 'write', 'pull-requests': 'write', actions: 'read', packages: 'read', 'id-token': 'write', attestations: 'write' }); + expect(job.permissions).toMatchObject({ contents: 'write', 'pull-requests': 'write', actions: 'write', packages: 'read', 'id-token': 'write', attestations: 'write' }); expect(raw).toContain('.head_branch == "main"'); expect(raw).toContain('.path == ".github/workflows/cso-runtime-qualification.yml"'); expect(raw).toContain('.event == "repository_dispatch"'); @@ -444,15 +516,51 @@ describe('CSO runtime staging gates', () => { for (const step of job.steps) if (step.uses) expect(step.uses).toMatch(/@[a-f0-9]{40}$/); }); + test('catalog promotion PRs dispatch the required free-tests check on the pushed branch', () => { + const freeTests = Bun.YAML.parse(readFileSync(join(ROOT, '.github/workflows/free-tests.yml'), 'utf8')) as any; + expect(Object.keys(freeTests.on)).toContain('workflow_dispatch'); + expect(freeTests.jobs['free-tests'].if).toBe('always()'); + expect(freeTests.jobs['free-tests'].name).toBeUndefined(); + for (const [relative, jobName] of [ + ['.github/workflows/cso-runtime-promote.yml', 'propose'], + ['.github/workflows/cso-scanner-images.yml', 'promote-catalog'], + ]) { + const job = (Bun.YAML.parse(readFileSync(join(ROOT, relative), 'utf8')) as any).jobs[jobName]; + expect(job.permissions.actions, relative).toBe('write'); + const script = job.steps.find((step: any) => step.run?.includes('gh pr create')).run as string; + const pushed = script.indexOf('git push --set-upstream origin "$branch"'); + const dispatched = script.indexOf('gh workflow run free-tests.yml --repo "$GITHUB_REPOSITORY" --ref "$branch"'); + expect(pushed, relative).toBeGreaterThan(0); + expect(dispatched, relative).toBeGreaterThan(pushed); + expect(script.indexOf('gh pr create'), relative).toBeGreaterThan(dispatched); + } + }); + test('private qualification enters only through an actor-restricted protected environment and re-verifies staged attestations', () => { const raw = readFileSync(join(ROOT, '.github/workflows/cso-runtime-qualification.yml'), 'utf8'); const workflow = Bun.YAML.parse(raw) as any; const job = workflow.jobs.qualify; expect(Object.keys(workflow.on)).toEqual(['repository_dispatch']); expect(job.environment).toBe('cso-runtime-release'); - expect(job.permissions).toEqual({ contents: 'read', packages: 'read', attestations: 'read' }); + expect(job.permissions).toEqual({ contents: 'read', packages: 'read', attestations: 'read', actions: 'read' }); + expect(job.needs).toBe('dispatch-summary'); + const summary = workflow.jobs['dispatch-summary']; + expect(summary.permissions).toEqual({}); + expect(summary.environment).toBeUndefined(); + expect(summary.steps.every((step: any) => !step.uses)).toBe(true); expect(raw).toContain('test "$GITHUB_ACTOR" = "$CSO_QUALIFICATION_ACTOR"'); expect(raw).toContain('.client_payload.statements | type == "array" and length == 10'); + expect(raw).not.toContain('^ghcr.io/'); + expect(raw).toContain('^ghcr\\\\.io/garrytan/gstack/cso-staging/'); + const names = job.steps.map((step: any) => step.name); + const attested = names.indexOf('Verify each staged image attestation and normalize qualification statements'); + const staging = names.indexOf('Require one successful protected-main staging run with every native gate passed'); + const contract = names.indexOf('Enforce the complete release contract before retaining evidence'); + expect(attested).toBeGreaterThan(0); + expect(staging).toBe(attested + 1); + expect(contract).toBe(staging + 1); + expect(job.steps[staging].run).toContain('gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/jobs?filter=latest&per_page=100"'); + expect(job.steps.at(-1).with.path).toContain('staging-run-evidence'); expect(raw).toContain('--cert-identity "$signer"'); expect(raw).toContain('--source-digest "$source_commit"'); expect(raw).toContain('--deny-self-hosted-runners'); @@ -489,4 +597,132 @@ describe('CSO runtime staging gates', () => { expect(qualify.steps[loaded].run).toContain('scripts/cso-public-ghcr.ts verify'); expect(qualify.steps[loaded].env.GH_TOKEN).toBe('${{ github.token }}'); }); + + test('the ingress payload gate accepts an optional evaluation reference and summarizes it before approval', () => { + const script = (Bun.YAML.parse(readFileSync(join(ROOT, '.github/workflows/cso-runtime-qualification.yml'), 'utf8')) as any).jobs['dispatch-summary'].steps[0].run; + const dir = mkdtempSync(join(tmpdir(), 'cso-dispatch-summary-')); temps.push(dir); + const event = join(dir, 'event.json'), summary = join(dir, 'summary.md'); + const dispatch = (payload: unknown) => { + writeFileSync(event, JSON.stringify({ action: 'cso-runtime-qualified', client_payload: payload })); + writeFileSync(summary, ''); + const result = runBashScript(script, { cwd: dir, env: { ...process.env, GITHUB_EVENT_PATH: event, GITHUB_STEP_SUMMARY: summary }, timeout: 30_000 }); + return { status: result.status, summary: readFileSync(summary, 'utf8') }; + }; + const statements = dispatchedStatements(); + expect(dispatch({ statements }).summary).toContain('- evaluationRef: `none`'); + const accepted = dispatch({ evaluationRef: EVALUATION_REF, statements }); + expect(accepted.status).toBe(0); + expect(accepted.summary).toContain(`- evaluationRef: \`${EVALUATION_REF}\``); + expect(accepted.summary).toContain(`- sourceCommit: \`${'a'.repeat(40)}\``); + for (const statement of statements) expect(accepted.summary).toContain(`| \`${statement.runtimeId}\` | \`${statement.image}\` |`); + const rejected = (payload: unknown) => expect(dispatch(payload).status).not.toBe(0); + rejected({ evaluationRef: 'sha256:abc', statements }); + rejected({ evaluationRef: EVALUATION_REF.toUpperCase(), statements }); + rejected({ evaluationRef: null, statements }); + rejected({ evaluationRef: EVALUATION_REF, statements, finding: 'held-out detail' }); + rejected({ statements: statements.slice(1) }); + rejected({ statements: statements.map((item, index) => index ? item : { ...item, sourceCommit: 'b'.repeat(40) }) }); + rejected({ statements: statements.map((item, index) => index ? item : { ...item, image: item.image.replace('ghcr.io', 'ghcrxio') }) }); + rejected({ statements: statements.map((item, index) => index ? item : { ...item, runtimeId: 'node`|injected' }) }); + }); + + test('the protected ingress binds verified statements to one successful staging run and never carries the evaluation reference', () => { + const fixture = ingressFixture(); + const normalized = fixture.step('Verify each staged image attestation and normalize qualification statements'); + expect(normalized.status, normalized.stderr).toBe(0); + const staging = fixture.step('Require one successful protected-main staging run with every native gate passed'); + expect(staging.status, staging.stderr).toBe(0); + expect(staging.stdout).toContain('STAGING RUN VERIFIED'); + const contract = fixture.step('Enforce the complete release contract before retaining evidence'); + expect(contract.status, contract.stderr).toBe(0); + + const statements = committedImageBuildMatrix().include.map(row => + JSON.parse(readFileSync(join(fixture.work, 'qualification-evidence', row.runtimeId, 'qualified-runtime.json'), 'utf8'))); + for (const statement of statements) { + expect(statement).not.toHaveProperty('evaluationRef'); + expect(statement.workflow).toBe('https://github.com/garrytan/gstack/actions/runs/555'); + } + expect(JSON.parse(readFileSync(join(fixture.work, 'qualification-evidence', 'requalification.json'), 'utf8'))).toEqual(CURRENT_TRIGGERS); + const candidate = JSON.stringify(catalogPromotionCandidate(committedCatalog, buildInputs, statements, CURRENT_TRIGGERS)); + expect(candidate).not.toContain('evaluationRef'); + expect(candidate).not.toContain('e'.repeat(64)); + + for (const [mutate, error] of [ + [() => fixture.run({ conclusion: 'failure' }), 'STAGING_RUN_NOT_SUCCESSFUL'], + [() => fixture.jobs('failure'), 'STAGING_NATIVE_GATE_NOT_PASSED'], + ] as const) { + fixture.run(); fixture.jobs(); mutate(); + rmSync(join(fixture.work, 'staging-run-evidence'), { recursive: true, force: true }); + const rejected = fixture.step('Require one successful protected-main staging run with every native gate passed'); + expect(rejected.status).not.toBe(0); + expect(rejected.stderr).toContain(error); + } + }); +}); + +describe('CSO evaluation-only candidate builds', () => { + const revision = `cso-eval-123456-${'a'.repeat(12)}`; + const sha = (path: string) => createHash('sha256').update(readFileSync(path)).digest('hex'); + function candidateFixture(catalogRevision = revision) { + const dir = buildFixture(), outside = mkdtempSync(join(tmpdir(), 'gstack-cso-eval-out-')); temps.push(outside); + mkdirSync(join(dir, 'test/helpers'), { recursive: true }); + const catalog = join(outside, 'candidate.json'); + writeFileSync(catalog, JSON.stringify({ schemaVersion: 1, revision: catalogRevision, runtimes: [] }, null, 2) + '\n'); + return { dir, outside, catalog, output: join(outside, 'unit') }; + } + const candidateBuild = (dir: string, catalog: string, output: string, extra = '') => + runBashScript(`${fakeBuildCommand(dir, extra)} --evaluation-candidate ${quote(catalog)} --output ${quote(output)}`, { timeout: 20_000 }); + + test.skipIf(process.platform === 'win32')('builds a marked five-artifact producer unit outside the checkout without touching bin/', () => { + const { dir, outside, catalog, output } = candidateFixture(); + const result = candidateBuild(dir, catalog, output); + expect(result.status).toBe(0); + expect(JSON.parse(result.stdout.trim())).toEqual({ output, revision, evaluationOnly: true }); + expect(readdirSync(output).sort()).toEqual(['.gstack-cso-evaluation', '.gstack-cso-generation', 'cso-eval-producer', 'gstack-cso-core', 'gstack-cso-launcher', 'gstack-cso-watchdog']); + const marker = JSON.parse(readFileSync(join(output, '.gstack-cso-evaluation'), 'utf8')); + expect(marker).toEqual({ schemaVersion: 1, evaluationOnly: true, runtimeCatalogRevision: revision, runtimeCatalogSha256: sha(catalog), coreSha256: sha(join(output, 'gstack-cso-core')) }); + expect(readFileSync(join(output, '.gstack-cso-generation'), 'utf8')).toBe(`${marker.coreSha256}\n`); + expect(readdirSync(join(dir, 'bin'))).toEqual([]); + expect(readdirSync(outside).sort()).toEqual(['candidate.json', 'unit']); + const args = readFileSync(join(dir, 'compiler.log'), 'utf8').trim().split('\n'); + const check = args.indexOf('check-evaluation'); + expect(check).toBeGreaterThan(0); expect(args[check + 1]).toBe('lib/cso/runtime-catalog.json'); + expect(check).toBeLessThan(args.indexOf('lib/cso/cli.ts')); + expect(args).toContain('scripts/cso-eval-producer.ts'); + expect(args.filter(arg => arg === '--no-compile-autoload-package-json')).toHaveLength(2); + }); + + test.skipIf(process.platform === 'win32')('refuses unmarked catalogs, in-checkout or existing outputs, and leaves nothing on a failed check', () => { + const plain = candidateFixture('cso-v3-123456-aaaaaaaaaaaa'); + expect(candidateBuild(plain.dir, plain.catalog, plain.output).status).toBe(65); + const inside = candidateFixture(); + const insideResult = candidateBuild(inside.dir, inside.catalog, join(inside.dir, 'unit')); + expect(insideResult.status).toBe(73); expect(existsSync(join(inside.dir, 'unit'))).toBe(false); + const existing = candidateFixture(); mkdirSync(existing.output); + expect(candidateBuild(existing.dir, existing.catalog, existing.output).status).toBe(73); + const failed = candidateFixture(); + expect(candidateBuild(failed.dir, failed.catalog, failed.output, 'CSO_FAIL_COMPILER_N=1').status).toBe(42); + expect(readdirSync(failed.outside)).toEqual(['candidate.json']); + }); + + test.skipIf(process.platform === 'win32')('distribution builds refuse an evaluation catalog in the source tree', () => { + const dir = buildFixture(); + writeFileSync(join(dir, 'lib/cso/runtime-catalog.json'), JSON.stringify({ revision }, null, 2)); + const result = fakeBuild(dir); + expect(result.status).toBe(65); expect(result.stderr).toContain('evaluation-only catalog'); + expect(existsSync(join(dir, 'compiler.log'))).toBe(false); + }); + + test('the candidate gate rejects the committed release catalog', () => { + const result = spawnSync(process.execPath, ['scripts/cso-runtime-promotion.ts', 'check-evaluation', 'lib/cso/runtime-catalog.json'], { cwd: ROOT, encoding: 'utf8', timeout: 30_000 }); + expect(result.status).toBe(1); expect(result.stderr).toContain('NOT_AN_EVALUATION_CATALOG'); + expect(String((committedCatalog as { revision: string }).revision).startsWith('cso-eval-')).toBe(false); + }); + + test('setup refuses a marked or self-reporting evaluation helper', () => { + const setup = readFileSync(join(ROOT, 'setup'), 'utf8'); + expect(setup).toContain('[ -e "$SOURCE_GSTACK_DIR/bin/.gstack-cso-evaluation" ]'); + expect(setup).toContain(`--version 2>/dev/null | grep -q '"evaluationOnly": *true'`); + expect(setup.indexOf('bin/ holds an evaluation-only CSO helper')).toBeLessThan(setup.indexOf('provision-images --setup-summary')); + }); }); diff --git a/test/cso-docker-integration.test.ts b/test/cso-docker-integration.test.ts index 8cbb370ce..e35735355 100644 --- a/test/cso-docker-integration.test.ts +++ b/test/cso-docker-integration.test.ts @@ -38,9 +38,8 @@ suite('CSO Docker containment integration',()=>{ expect(spawnSync('/usr/bin/docker',['--host',endpoint.uri,'inspect',client],{timeout:30_000}).status).not.toBe(0); }); test('machine-wide admission allows only two groups per endpoint',()=>{const a=admit(endpoint.uri,'a',Date.now()+60_000),b=admit(endpoint.uri,'b',Date.now()+60_000);try{expect(()=>admit(endpoint.uri,'c',Date.now()+60_000)).toThrow('Two reproduction groups');}finally{release(a);release(b);}}); - test('staged runtime executes its trusted verifier and checks every declared tool version', async () => { - const staged = process.env.GSTACK_CSO_TEST_IMAGE; - if (!staged) return; + test.skipIf(!process.env.GSTACK_CSO_TEST_IMAGE)('staged runtime executes its trusted verifier and checks every declared tool version', async () => { + const staged = process.env.GSTACK_CSO_TEST_IMAGE ?? ''; expect(staged).toMatch(/@sha256:[a-f0-9]{64}$/); expect(process.env.GSTACK_CSO_TEST_PLATFORM).toBe(process.arch === 'arm64' ? 'linux/arm64' : 'linux/amd64'); const versions = JSON.parse(process.env.GSTACK_CSO_EXPECTED_VERSIONS || '{}'); diff --git a/test/cso-eval.test.ts b/test/cso-eval.test.ts index 1b887361f..58c389bbc 100644 --- a/test/cso-eval.test.ts +++ b/test/cso-eval.test.ts @@ -4,11 +4,11 @@ import { chmodSync, cpSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readd import { createHash } from 'node:crypto'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; -import { collectProducerReceipts, createEvalMatrix, createPortableSkillPayload, loadPortableSkillPayload, prepareEvalJobs, REQUIRED_CONTAINMENT, scoreCollectedEval, scoreEval, validateMatrix, validatePortableSkillPayload, type EvalCell, type EvalResult, type PreparedEvalSchedule } from '../scripts/cso-eval'; +import { collectProducerReceipts, createEvalBaseline, createEvalMatrix, createPortableSkillPayload, loadPortableSkillPayload, prepareEvalJobs, REQUIRED_CONTAINMENT, scoreCollectedEval, scoreEval, validateMatrix, validatePortableSkillPayload, type EvalBaseline, type EvalCell, type EvalMatrix, type EvalResult, type PreparedEvalSchedule } from '../scripts/cso-eval'; import { PRODUCER_PROVIDER_POLICY, producerFailureMessage, producerInstallationIdentity, resolveProducerHelperBinding, runProducerCell, validateProductionProducerInstallation } from '../scripts/cso-eval-producer'; -import { producerArtifactInventoryHash, producerInstallationIdentityHash, producerProviderIdentityHash, producerReceiptHash, sha256, type ProducerArtifactInventory, type ProducerInstallationIdentity, type ProducerProviderIdentity, type ProducerReceipt } from '../scripts/cso-eval-protocol'; +import { producerArtifactInventoryHash, producerHostPlatform, producerInstallationIdentityHash, producerProviderIdentityHash, producerReceiptHash, sha256, type ProducerArtifactInventory, type ProducerInstallationIdentity, type ProducerProviderIdentity, type ProducerReceipt } from '../scripts/cso-eval-protocol'; import type { Family, ProviderAdapter, RunOpts, RunResult } from './helpers/providers/types'; -import { CORPUS_VERSION, FAMILIES, STACKS, loadCorpusManifest, materializeCase, sourceFiles, sourceHash } from './fixtures/cso-eval/materialize'; +import { CORPUS_VERSION, FAMILIES, STACKS, defineEvalCorpus, loadCorpusManifest, materializeCase, sourceFiles, sourceHash, validateCorpusManifest, type CorpusManifest, type EvalVariant } from './fixtures/cso-eval/materialize'; import { judgeRepair, oracleFor, type PrivateEvidence } from './helpers/cso-eval-oracles'; import { inspectPreparation } from '../lib/cso/preparation'; import { assertRuntimeCompatible, RUNTIME_CATALOG } from '../lib/cso/runtime-catalog'; @@ -41,7 +41,8 @@ function portableSkill(version: 'v2' | 'v3', sectionMarker: string): string { } /** Synthetic accounting records exercise the scorer. These are not measured agent results. */ -function syntheticResult(cell: EvalCell): EvalResult { +const syntheticResult = (cell: EvalCell): EvalResult => syntheticResultFor(cell, corpus); +function syntheticResultFor(cell: EvalCell, manifest: CorpusManifest): EvalResult { const runtime = cell.mode === 'comprehensive'; const positive = cell.variant === 'vulnerable'; const repairEvidenceHash = 'c'.repeat(64), recheckEvidenceHash = 'd'.repeat(64); @@ -50,12 +51,12 @@ function syntheticResult(cell: EvalCell): EvalResult { findings: positive ? [{ id: 'finding-1', evidence: cell.version === 'v2' ? 'legacy_review' : 'supported', claimedTested: runtime && cell.version === 'v3', judgment: 'correct', matchedCaseId: cell.caseId, ...(runtime && cell.version === 'v3' ? { trustedVerification: { repair: 'passed' as const, repairEvidenceHash, recheck: 'passed' as const, recheckEvidenceHash } } : {}) }] : [], setup: runtime ? 'passed' : 'not_attempted', reproduction: runtime && positive ? 'passed' : 'not_attempted', repair: runtime && positive ? 'passed' : 'not_attempted', recheck: runtime && positive ? 'passed' : 'not_attempted', - ...(runtime && positive ? { oracleEvidenceHash: repairEvidenceHash, oracleVersion: CORPUS_VERSION, currentSourceHash: corpus.cases.find(fixture => fixture.id === cell.caseId)!.filesHash.fixed, recheckEvidenceHash } : {}), + ...(runtime && positive ? { oracleEvidenceHash: repairEvidenceHash, oracleVersion: manifest.version, currentSourceHash: manifest.cases.find(fixture => fixture.id === cell.caseId)!.filesHash.fixed, recheckEvidenceHash } : {}), heldOutAssertionsPassed: runtime && positive, freshRecheck: runtime && positive, latencyMs: 1500, firstUsefulResultMs: positive ? 500 : null, }; } const group = (score: ReturnType, version: string, mode: string) => score.groups.find(item => item.version === version && item.mode === mode)!; -function syntheticInstallationIdentity(seed = '1'): ProducerInstallationIdentity { +function syntheticInstallationIdentity(seed = '1', catalogSeed = seed): ProducerInstallationIdentity { const artifact = (offset: number) => ({ sha256: sha256(`${seed}:artifact:${offset}`), bytes: 100 + offset }); const core = artifact(3); const withoutHash: Omit = { @@ -63,8 +64,8 @@ function syntheticInstallationIdentity(seed = '1'): ProducerInstallationIdentity producer: artifact(1), launcher: artifact(2), core, watchdog: artifact(4), generation: { coreSha256: core.sha256, manifest: { sha256: sha256(`${core.sha256}\n`), bytes: 65 } }, embeddedCatalogs: { - runtimeRevision: 'runtime-test', runtimeBuildRevision: 'runtime-build-test', runtimeSha256: sha256(`${seed}:runtime`), - scannerRevision: 'scanner-test', scannerSha256: sha256(`${seed}:scanner`), + runtimeRevision: 'runtime-test', runtimeBuildRevision: 'runtime-build-test', runtimeSha256: sha256(`${catalogSeed}:runtime`), + scannerRevision: 'scanner-test', scannerSha256: sha256(`${catalogSeed}:scanner`), }, }; return { ...withoutHash, identityHash: producerInstallationIdentityHash(withoutHash) }; @@ -80,12 +81,14 @@ function syntheticArtifactInventory(): ProducerArtifactInventory { const base = { schemaVersion: 1 as const, root: 'security/cso' as const, entries: [], totalBytes: 0 }; return { ...base, identityHash: producerArtifactInventoryHash(base) }; } +/** Platform-pinned cells get one native installation and provider binary per platform, sharing catalogs and provider policy. */ function syntheticReceipt(cell: EvalCell): ProducerReceipt { - const withoutHash: Omit = { schemaVersion: 1, cell, inputHash: 'e'.repeat(64), installationIdentity: syntheticInstallationIdentity(), providerIdentity: syntheticProviderIdentity(cell.host === 'codex' ? 'gpt' : cell.host), artifacts: syntheticArtifactInventory(), startedAt: '2026-01-01T00:00:00.000Z', finishedAt: '2026-01-01T00:00:01.000Z', status: 'succeeded', requestedModel: cell.model, modelUsed: `resolved-${cell.model}`, modelIdentitySource: 'provider_reported', durationMs: 1000, firstUsefulResultMs: null, toolCalls: 1, output: 'synthetic producer transcript', outputHash: sha256('synthetic producer transcript'), usage: { inputTokens: 10, outputTokens: 5, cachedTokens: 0, estimatedCostUSD: 0.001 } }; + const seed = cell.platform ?? '1'; + const withoutHash: Omit = { schemaVersion: 1, cell, inputHash: 'e'.repeat(64), installationIdentity: syntheticInstallationIdentity(seed, '1'), providerIdentity: syntheticProviderIdentity(cell.host === 'codex' ? 'gpt' : cell.host, seed), artifacts: syntheticArtifactInventory(), startedAt: '2026-01-01T00:00:00.000Z', finishedAt: '2026-01-01T00:00:01.000Z', status: 'succeeded', requestedModel: cell.model, modelUsed: `resolved-${cell.model}`, modelIdentitySource: 'provider_reported', durationMs: 1000, firstUsefulResultMs: null, toolCalls: 1, output: 'synthetic producer transcript', outputHash: sha256('synthetic producer transcript'), usage: { inputTokens: 10, outputTokens: 5, cachedTokens: 0, estimatedCostUSD: 0.001 } }; return { ...withoutHash, receiptHash: producerReceiptHash(withoutHash) }; } -function syntheticSchedule(): PreparedEvalSchedule { - return { schemaVersion: 1, matrixHash: createHash('sha256').update(JSON.stringify(matrix)).digest('hex'), scheduledCells: matrix.cells.length, preparedCells: matrix.cells.length, jobs: matrix.cells.map(cell => ({ cellId: cell.id, relativePath: `jobs/${cell.id}`, inputHash: 'e'.repeat(64) })) }; +function syntheticSchedule(target: EvalMatrix = matrix): PreparedEvalSchedule { + return { schemaVersion: 1, matrixHash: createHash('sha256').update(JSON.stringify(target)).digest('hex'), scheduledCells: target.cells.length, preparedCells: target.cells.length, jobs: target.cells.map(cell => ({ cellId: cell.id, relativePath: `jobs/${cell.id}`, inputHash: 'e'.repeat(64) })) }; } function rehashReceipt(receipt: ProducerReceipt): ProducerReceipt { const { receiptHash: _receiptHash, ...withoutHash } = receipt; @@ -333,6 +336,131 @@ describe('CSO matched evaluation accounting', () => { }); }); +/** Synthetic stand-in with the private corpus layout. It is not held-out content. */ +const privateFiles = (id: string, variant: EvalVariant): Record => ({ 'README.md': '# Synthetic service\n', 'src/app.txt': `synthetic ${id} ${variant}\n` }); +const privateManifest: CorpusManifest = { schemaVersion: 1, version: 'synthetic-private-1', cases: STACKS.flatMap(stack => ['alpha-family', 'beta-family'].map((family, index) => { + const id = `${stack}-${family}`; + return { id, stack, family, severity: index === 0 ? 'high' as const : 'medium' as const, rootCause: family, location: { path: 'src/app.txt', symbol: 'action' }, + coreColdStart: true as const, heldOut: true as const, filesHash: { vulnerable: sourceHash(privateFiles(id, 'vulnerable')), fixed: sourceHash(privateFiles(id, 'fixed')) } }; +})) }; +const privateCorpus = defineEvalCorpus(privateManifest, privateFiles); +const releasePlatforms = { daily: 'linux/amd64' as const, comprehensive: Object.fromEntries(privateManifest.cases.map((fixture, index) => [fixture.id, index % 2 ? 'linux/arm64' as const : 'linux/amd64' as const])) }; +const releaseOptions = { profile: 'release' as const, model: 'matched-test-model', host: 'codex' as const, skillHashes: { v3: 'b'.repeat(64) }, platforms: releasePlatforms }; +function privateBaseline(mutate: (results: EvalResult[], cells: EvalCell[]) => void = () => {}): EvalBaseline { + const baselineMatrix = createEvalMatrix({ profile: 'baseline', model: 'matched-test-model', host: 'codex', skillHashes: { v2: 'a'.repeat(64) }, platforms: { comprehensive: releasePlatforms.comprehensive } }, privateManifest); + const batch = collectProducerReceipts(baselineMatrix, syntheticSchedule(baselineMatrix), baselineMatrix.cells.map(syntheticReceipt), privateManifest); + const results = baselineMatrix.cells.map(cell => ({ ...syntheticResultFor(cell, privateManifest), producerReceiptHash: batch.receipts.find(receipt => receipt.cell.id === cell.id)!.receiptHash })); + mutate(results, baselineMatrix.cells); + return createEvalBaseline(baselineMatrix, batch, results, privateManifest); +} +function releaseRun(baseline: EvalBaseline | null = privateBaseline()) { + const release = createEvalMatrix({ ...releaseOptions, ...(baseline ? { baseline } : {}) }, privateManifest); + const batch = collectProducerReceipts(release, syntheticSchedule(release), release.cells.map(syntheticReceipt), privateManifest); + const results = release.cells.map(cell => ({ ...syntheticResultFor(cell, privateManifest), producerReceiptHash: batch.receipts.find(receipt => receipt.cell.id === cell.id)!.receiptHash })); + return { release, batch, results }; +} + +describe('CSO release profile on an injected corpus', () => { + test('the full profile matrix is byte-identical to its pre-release-profile form', () => { + expect(createHash('sha256').update(JSON.stringify(matrix)).digest('hex')).toBe('a0f20a127c8c92c73bb2a277f740295c153ccf95901e92cffab34a4553ef976b'); + expect('profile' in matrix).toBe(false); expect(matrix.cells.every(cell => !('platform' in cell))).toBe(true); + }); + test('an injected corpus shares the public layout and binds every generated source', () => { + expect(validateCorpusManifest(structuredClone(corpus))).toEqual(corpus); + const destination = join(root(), 'case'); + expect(materializeCase('rails-beta-family', 'fixed', destination, privateCorpus).sourceHash).toBe(privateManifest.cases.find(item => item.id === 'rails-beta-family')!.filesHash.fixed); + expect(readFileSync(join(destination, 'src/app.txt'), 'utf8')).toBe('synthetic rails-beta-family fixed\n'); + expect(() => defineEvalCorpus({ ...privateManifest, version: CORPUS_VERSION }, privateFiles)).toThrow('INJECTED_CORPUS_REUSES_PUBLIC_VERSION'); + expect(() => defineEvalCorpus(privateManifest, (id, variant) => ({ ...privateFiles(id, variant), extra: 'drift' }))).toThrow('CORPUS_INTEGRITY_MISMATCH'); + expect(() => defineEvalCorpus(privateManifest, () => ({ '../escape.txt': 'x' }))).toThrow('INVALID_CORPUS_SOURCE'); + const unknownStack = structuredClone(privateManifest); (unknownStack.cases[0] as any).stack = 'php'; + expect(() => validateCorpusManifest(unknownStack)).toThrow('INVALID_CORPUS_MANIFEST'); + const extraField = structuredClone(privateManifest); (extraField.cases[0] as any).answer = 'oracle'; + expect(() => validateCorpusManifest(extraField)).toThrow('INVALID_CORPUS_MANIFEST'); + }); + test('release pins v3, one repetition, and one native platform per cell', () => { + const release = createEvalMatrix(releaseOptions, privateManifest); + expect(release.cells).toHaveLength(privateManifest.cases.length * 4); expect(release.repetitions).toBe(1); expect(release.baselineHash).toBeNull(); + expect(release.cells.every(cell => cell.version === 'v3' && cell.repetition === 1)).toBe(true); + expect(release.cells.filter(cell => cell.mode === 'daily').every(cell => cell.platform === 'linux/amd64')).toBe(true); + for (const cell of release.cells.filter(item => item.mode === 'comprehensive')) expect(cell.platform).toBe(releasePlatforms.comprehensive[cell.caseId]); + expect(new Set(release.cells.filter(cell => cell.mode === 'comprehensive').map(cell => cell.platform))).toEqual(new Set(['linux/amd64', 'linux/arm64'])); + validateMatrix(release, privateManifest); + const moved = structuredClone(release); moved.platforms.comprehensive[privateManifest.cases[0].id] = 'linux/arm64'; + expect(() => validateMatrix(moved, privateManifest)).toThrow('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX'); + const { [privateManifest.cases[0].id]: _omitted, ...partial } = releasePlatforms.comprehensive; + expect(() => createEvalMatrix({ ...releaseOptions, platforms: { daily: 'linux/amd64', comprehensive: partial } }, privateManifest)).toThrow('INVALID_EVAL_PLATFORMS'); + expect(() => createEvalMatrix({ ...releaseOptions, platforms: { daily: 'linux/riscv64' as any, comprehensive: releasePlatforms.comprehensive } }, privateManifest)).toThrow('INVALID_EVAL_PLATFORMS'); + expect(() => createEvalMatrix({ ...releaseOptions, skillHashes: { v2: 'a'.repeat(64), v3: 'b'.repeat(64) } as any }, privateManifest)).toThrow('INVALID_MATCHED_EVAL_INPUT'); + }); + test('a two-platform batch collects and scores qualified against a hash-bound v2 baseline', () => { + const { release, batch, results } = releaseRun(); + expect(new Set(batch.receipts.map(receipt => receipt.installationIdentity.identityHash)).size).toBe(2); + expect(batch.summary.groups.map(group => `${group.version}:${group.mode}`)).toEqual(['v3:daily', 'v3:comprehensive']); + const baseline = privateBaseline(); + const score = scoreCollectedEval(release, batch, results, qualification, privateManifest, baseline); + expect(score.status).toBe('qualified'); expect(Object.values(score.gates).every(value => value === 'pass')).toBe(true); + expect(score.profile).toBe('release'); expect(score.baseline).toEqual({ baselineHash: baseline.baselineHash, highCriticalRecall: { numerator: 4, denominator: 4, value: 1 } }); + expect(score.groups.map(item => `${item.version}:${item.mode}`)).toEqual(['v3:daily', 'v3:comprehensive']); + }); + test('a mixed-corpus batch is rejected', () => { + const { release, batch, results } = releaseRun(); + expect(() => scoreCollectedEval(release, batch, results, qualification)).toThrow('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX'); + expect(() => collectProducerReceipts(release, syntheticSchedule(release), release.cells.map(syntheticReceipt))).toThrow('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX'); + const mixed = structuredClone(release); mixed.cells[0] = { ...matrix.cells.find(cell => cell.version === 'v3' && cell.repetition === 1)!, platform: 'linux/amd64' }; + expect(() => validateMatrix(mixed, privateManifest)).toThrow('UNMATCHED_OR_INCOMPLETE_EVAL_MATRIX'); + const publicCell = matrix.cells.find(cell => cell.version === 'v3')!; + expect(() => collectProducerReceipts(release, syntheticSchedule(release), [syntheticReceipt(publicCell)], privateManifest)).toThrow('UNKNOWN_OR_DUPLICATE_PRODUCER_RECEIPT'); + expect(() => scoreCollectedEval(release, syntheticBatch(), results, qualification, privateManifest)).toThrow('INVALID_PRODUCER_BATCH'); + expect(() => scoreEval(release, [syntheticResult(publicCell)], qualification, privateManifest)).toThrow('UNKNOWN_OR_DUPLICATE_EVAL_RESULT'); + }); + test('identity checks hold per platform and require shared catalogs and provider policy across platforms', () => { + const release = createEvalMatrix(releaseOptions, privateManifest), schedule = syntheticSchedule(release); + const amd = release.cells.filter(cell => cell.platform === 'linux/amd64'), arm = release.cells.find(cell => cell.platform === 'linux/arm64')!; + const relabel = (cell: EvalCell, change: (receipt: ProducerReceipt) => void) => { const receipt = syntheticReceipt(cell); change(receipt); return rehashReceipt(receipt); }; + expect(() => collectProducerReceipts(release, schedule, [syntheticReceipt(amd[0]), relabel(amd[1], receipt => { receipt.installationIdentity = syntheticInstallationIdentity('second-amd64', '1'); })], privateManifest)).toThrow('UNMATCHED_PRODUCER_INSTALLATIONS'); + expect(() => collectProducerReceipts(release, schedule, [syntheticReceipt(amd[0]), relabel(amd[1], receipt => { receipt.providerIdentity = syntheticProviderIdentity('gpt', 'second-amd64'); })], privateManifest)).toThrow('UNMATCHED_PRODUCER_PROVIDERS'); + expect(() => collectProducerReceipts(release, schedule, [syntheticReceipt(amd[0]), relabel(arm, receipt => { receipt.installationIdentity = syntheticInstallationIdentity('linux/amd64', '1'); })], privateManifest)).toThrow('UNMATCHED_PRODUCER_PLATFORMS'); + expect(() => collectProducerReceipts(release, schedule, [syntheticReceipt(amd[0]), relabel(arm, receipt => { receipt.installationIdentity = syntheticInstallationIdentity('linux/arm64', 'other-catalog'); })], privateManifest)).toThrow('UNMATCHED_PRODUCER_INSTALLATIONS'); + const otherPolicy = (() => { const { identityHash: _hash, ...base } = syntheticProviderIdentity('gpt', 'linux/arm64'); const changed = { ...base, policyRevision: 'gpt-other-policy' }; return { ...changed, identityHash: producerProviderIdentityHash(changed) }; })(); + expect(() => collectProducerReceipts(release, schedule, [syntheticReceipt(amd[0]), relabel(arm, receipt => { receipt.providerIdentity = otherPolicy; })], privateManifest)).toThrow('UNMATCHED_PRODUCER_PROVIDERS'); + expect(() => collectProducerReceipts(release, schedule, [syntheticReceipt(amd[0]), relabel(amd[1], receipt => { receipt.modelUsed = 'resolved-other-model'; })], privateManifest)).toThrow('UNMATCHED_EFFECTIVE_MODELS'); + }); + test('the external v2 baseline is pre-registered, complete, and bound to the release inputs', () => { + const unpinned = releaseRun(null); + const unmeasured = scoreCollectedEval(unpinned.release, unpinned.batch, unpinned.results, qualification, privateManifest); + expect(unmeasured.gates.noHighCriticalRecallRegression).toBe('unmeasured'); expect(unmeasured.status).toBe('partial'); expect(unmeasured.baseline).toBeNull(); + const baseline = privateBaseline(); + expect(() => scoreCollectedEval(unpinned.release, unpinned.batch, unpinned.results, qualification, privateManifest, baseline)).toThrow('UNBOUND_EVAL_BASELINE'); + const { release, batch, results } = releaseRun(baseline); + const forged = structuredClone(baseline); forged.cases[0].found = false; + expect(() => scoreCollectedEval(release, batch, results, qualification, privateManifest, forged)).toThrow('UNBOUND_EVAL_BASELINE'); + const weaker = privateBaseline(found => { found.forEach(result => { result.findings = []; }); }); + expect(() => scoreCollectedEval(release, batch, results, qualification, privateManifest, weaker)).toThrow('UNBOUND_EVAL_BASELINE'); + expect(() => createEvalMatrix({ ...releaseOptions, model: 'other-model', baseline }, privateManifest)).toThrow('UNBOUND_EVAL_BASELINE'); + expect(() => createEvalMatrix({ ...releaseOptions, budgets: { daily: 600, comprehensive: 1200 }, baseline }, privateManifest)).toThrow('UNBOUND_EVAL_BASELINE'); + const otherProvider = structuredClone(baseline); otherProvider.provider.version = 'gpt-other-version'; + const { baselineHash: _hash, ...record } = otherProvider; otherProvider.baselineHash = createHash('sha256').update(JSON.stringify(record)).digest('hex'); + const rebound = releaseRun(otherProvider); + expect(() => scoreCollectedEval(rebound.release, rebound.batch, rebound.results, qualification, privateManifest, otherProvider)).toThrow('UNBOUND_EVAL_BASELINE'); + const highId = privateManifest.cases.find(item => item.severity === 'high')!.id; + for (let index = 0; index < release.cells.length; index++) { + const cell = release.cells[index]; + if (cell.mode === 'comprehensive' && cell.variant === 'vulnerable' && cell.caseId === highId) results[index].findings = []; + } + const regressed = scoreCollectedEval(release, batch, results, qualification, privateManifest, baseline); + expect(regressed.gates.noHighCriticalRecallRegression).toBe('fail'); expect(regressed.baseline!.highCriticalRecall.value).toBe(1); + }); + test('baseline records refuse incomplete or unadjudicated runs and baseline matrices never qualify', () => { + expect(() => privateBaseline(results => { results.pop(); })).toThrow('INCOMPLETE_EVAL_BASELINE'); + expect(() => privateBaseline((results, cells) => { results[cells.findIndex(cell => cell.variant === 'vulnerable')].findings[0].judgment = 'unadjudicated'; })).toThrow('UNADJUDICATED_EVAL_BASELINE'); + const baselineMatrix = createEvalMatrix({ profile: 'baseline', model: 'matched-test-model', host: 'codex', skillHashes: { v2: 'a'.repeat(64) }, platforms: { comprehensive: releasePlatforms.comprehensive } }, privateManifest); + expect(baselineMatrix.cells.every(cell => cell.version === 'v2' && cell.mode === 'comprehensive')).toBe(true); + expect(() => scoreEval(baselineMatrix, [], qualification, privateManifest)).toThrow('BASELINE_PROFILE_NOT_QUALIFIABLE'); + expect(() => scoreEval(matrix, [], qualification, corpus, privateBaseline())).toThrow('UNBOUND_EVAL_BASELINE'); + }); +}); + describe('CSO matched producer orchestration', () => { const skills = { v2: portableSkill('v2', 'V2_SECTION_ONLY'), v3: portableSkill('v3', 'V3_SECTION_ONLY') }; const producerMatrix = createEvalMatrix({ model: 'exact-eval-model', host: 'codex', skillHashes: { v2: sha256(skills.v2), v3: sha256(skills.v3) } }); @@ -815,6 +943,24 @@ describe('CSO matched producer orchestration', () => { expect(() => collectProducerReceipts(producerMatrix, schedule, [tampered])).toThrow('INVALID_PRODUCER_RECEIPT'); }); + test('a platform-pinned release cell runs only on its native producer platform', async () => { + const release = createEvalMatrix({ ...releaseOptions, skillHashes: { v3: sha256(skills.v3) } }, privateManifest); + const host = producerHostPlatform(), foreign = host === 'linux/amd64' ? 'linux/arm64' : 'linux/amd64'; + const native = release.cells.find(cell => cell.mode === 'comprehensive' && cell.platform === host); + const other = release.cells.find(cell => cell.mode === 'comprehensive' && cell.platform === foreign)!; + const destination = join(root(), 'prepared'); + expect(() => prepareEvalJobs(release, skills, join(root(), 'both'), [other.id], privateCorpus)).toThrow('EVAL_SKILL_HASH_MISMATCH'); + prepareEvalJobs(release, { v3: skills.v3 }, destination, [other.id, ...(native ? [native.id] : [])], privateCorpus); + const receipts = join(root(), 'receipts'); mkdirSync(receipts); + const isolatedOther = isolate(destination, other); + expect(readFileSync(join(isolatedOther.source, 'src/app.txt'), 'utf8')).toBe(`synthetic ${other.caseId} ${other.variant}\n`); + await expect(runProducerCell(isolatedOther.input, join(receipts, `${other.id}.json`), withHelper({ adapter: new FakeAdapter(() => {}), paidExecutionAuthorized: true }))).rejects.toThrow('UNMATCHED_PRODUCER_PLATFORM'); + if (!native) return; + const isolatedNative = isolate(destination, native); + const receipt = await runProducerCell(isolatedNative.input, join(receipts, `${native.id}.json`), withHelper({ adapter: new FakeAdapter(() => {}), paidExecutionAuthorized: true })); + expect(receipt.cell).toEqual(native); + }); + test('seals source read-only and withholds a receipt after a mode/content mutation', async () => { const destination = join(root(), 'prepared'); const cell = selected[0]; diff --git a/test/cso-native-evidence.test.ts b/test/cso-native-evidence.test.ts new file mode 100644 index 000000000..cff83302b --- /dev/null +++ b/test/cso-native-evidence.test.ts @@ -0,0 +1,103 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { deriveNativeChecks, nativeEvidence, nativeTestRequirements, parseJUnit, type JUnitCase } from '../scripts/cso-native-evidence'; +import { requiredChecks } from '../scripts/cso-runtime-promotion'; + +const ROOT = resolve(import.meta.dir, '..'); +const STACKS = ['node', 'bun', 'python', 'rails', 'postgresql'] as const; +const temps: string[] = []; +afterEach(() => { for (const dir of temps.splice(0)) rmSync(dir, { recursive: true, force: true }); }); + +const escape = (value: string) => value.replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>').replaceAll('"', '"'); + +// Mirrors Bun 1.4.0 `bun test --reporter=junit` output: passing cases self-close, skips and failures carry a child element. +function report(cases: JUnitCase[]): string { + const body = cases.map(item => { + const open = ``; + const child = item.status === 'skipped' ? '' : 'AssertionError'; + return ` ${open}>\n ${child}\n `; + }).join('\n'); + const failures = cases.filter(item => item.status === 'failed').length; + return `\n\n \n${body}\n \n\n`; +} + +function passing(stack: typeof STACKS[number]): JUnitCase[] { + const seen = new Map(); + for (const refs of Object.values(nativeTestRequirements(stack))) for (const ref of refs) seen.set(`${ref.file}\0${ref.name}`, { ...ref, status: 'passed', assertions: 3 }); + return [...seen.values()]; +} + +function staged(stack: string, platform = 'linux/amd64') { + return { schemaVersion: 1, state: 'staged', stack, platform, image: `ghcr.io/garrytan/gstack/cso-staging/${stack}-amd64@sha256:${'a'.repeat(64)}` }; +} + +describe('CSO native gate evidence', () => { + test('native check keys are release gates of their stack, and private checks are exactly the remainder', () => { + for (const stack of STACKS) { + const native = Object.keys(nativeTestRequirements(stack)); + expect(native.every(key => requiredChecks(stack).includes(key))).toBe(true); + const evidence = nativeEvidence(staged(stack), stack, 'linux/amd64', report(passing(stack))) as any; + expect(Object.values(evidence.nativeChecks).every(value => value === true)).toBe(true); + expect(Object.keys(evidence.privateChecks).sort()).toEqual(requiredChecks(stack).filter(key => !native.includes(key))); + expect(Object.values(evidence.privateChecks).every(value => value === 'pending')).toBe(true); + expect(evidence).toMatchObject({ state: 'native-gates-passed', qualified: false, stack }); + expect(evidence.nativeTestReport).toMatch(/^sha256:[a-f0-9]{64}$/); + } + const postgres = nativeEvidence(staged('postgresql'), 'postgresql', 'linux/amd64', report(passing('postgresql'))) as any; + expect(Object.keys(postgres.privateChecks).sort()).toEqual(['secretCanaryPassed', 'watchdogCleanupPassed']); + const rails = nativeEvidence(staged('rails'), 'rails', 'linux/amd64', report(passing('rails'))) as any; + expect(Object.keys(rails.privateChecks).sort()).toEqual(['accuracyGatesPassed', 'heldOutRepairPassed', 'secretCanaryPassed', 'watchdogCleanupPassed']); + }); + + test('a skipped, missing, failed, duplicated, or assertion-free named test leaves its checks false', () => { + const named = nativeTestRequirements('node').positiveNegativeAssertionsPassed[0]; + const mutate = (change: (cases: JUnitCase[]) => JUnitCase[]) => deriveNativeChecks('node', parseJUnit(report(change(passing('node'))))); + const skipped = mutate(cases => cases.map(item => item.name === named.name ? { ...item, status: 'skipped' as const, assertions: 0 } : item)); + expect(skipped.positiveNegativeAssertionsPassed).toBe(false); + expect(skipped.containmentPassed).toBe(false); + expect(skipped.coldStartPassed).toBe(true); + expect(mutate(cases => cases.filter(item => item.name !== named.name)).positiveNegativeAssertionsPassed).toBe(false); + expect(mutate(cases => [...cases, cases.find(item => item.name === named.name)!]).positiveNegativeAssertionsPassed).toBe(false); + expect(mutate(cases => cases.map(item => item.name === named.name ? { ...item, assertions: 0 } : item)).positiveNegativeAssertionsPassed).toBe(false); + const unrelatedFailure = mutate(cases => [...cases, { file: 'test/other.test.ts', name: '(unnamed)', status: 'failed', assertions: 0 }]); + expect(Object.values(unrelatedFailure).every(value => value === false)).toBe(true); + expect(() => nativeEvidence(staged('node'), 'node', 'linux/amd64', report(passing('node').map(item => item.name === named.name ? { ...item, status: 'skipped', assertions: 0 } : item)))) + .toThrow('NATIVE_GATE_NOT_PASSED: containmentPassed, positiveNegativeAssertionsPassed'); + }); + + test('a different file with the same test name does not satisfy a gate', () => { + const cases = passing('bun').map(item => item.file === 'test/cso-stack-cold-integration.test.ts' ? { ...item, file: 'test/elsewhere.test.ts' } : item); + const checks = deriveNativeChecks('bun', cases); + expect(checks.coldStartPassed).toBe(false); + expect(checks.acquisitionPublicOnlyPassed).toBe(false); + expect(checks.positiveNegativeAssertionsPassed).toBe(true); + }); + + test('staged evidence must describe the same stack and platform', () => { + expect(() => nativeEvidence(staged('bun'), 'node', 'linux/amd64', report(passing('node')))).toThrow('STAGED_EVIDENCE_MISMATCH'); + expect(() => nativeEvidence(staged('node', 'linux/arm64'), 'node', 'linux/amd64', report(passing('node')))).toThrow('STAGED_EVIDENCE_MISMATCH'); + expect(() => nativeEvidence({ ...staged('node'), state: 'qualified' }, 'node', 'linux/amd64', report(passing('node')))).toThrow('STAGED_EVIDENCE_MISMATCH'); + expect(() => parseJUnit('')).toThrow('INVALID_JUNIT_REPORT'); + expect(() => parseJUnit('\n')).toThrow('EMPTY_JUNIT_REPORT'); + }); + + test('the real Bun JUnit report of the Docker suites names every required test, and skipped suites prove nothing', () => { + const dir = mkdtempSync(join(tmpdir(), 'cso-native-junit-')); temps.push(dir); + const outfile = join(dir, 'native.xml'); + const env = { ...process.env }; + for (const key of ['GSTACK_CSO_DOCKER_TESTS', 'GSTACK_CSO_TEST_IMAGE', 'GSTACK_CSO_TEST_STACK']) delete env[key]; + const run = spawnSync(process.execPath, ['run', 'test:cso:docker', '--reporter=junit', `--reporter-outfile=${outfile}`], { cwd: ROOT, env, encoding: 'utf8', timeout: 120_000 }); + expect(run.status, run.stderr).toBe(0); + const cases = parseJUnit(readFileSync(outfile, 'utf8')); + for (const stack of STACKS) { + for (const refs of Object.values(nativeTestRequirements(stack))) for (const ref of refs) { + expect(cases.filter(item => item.file === ref.file && item.name === ref.name).map(item => item.status)).toEqual(['skipped']); + } + expect(Object.values(deriveNativeChecks(stack, cases)).every(value => value === false)).toBe(true); + } + }); +}); diff --git a/test/cso-requalification.test.ts b/test/cso-requalification.test.ts new file mode 100644 index 000000000..e27e47651 --- /dev/null +++ b/test/cso-requalification.test.ts @@ -0,0 +1,64 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { cpSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import committedCatalog from '../lib/cso/runtime-catalog.json'; +import buildInputs from '../lib/cso/images/build-inputs.json'; +import { ISOLATION_POLICY_HASH } from '../lib/cso/docker'; +import { selectRuntime, validateRuntimeCatalog, type RuntimeCatalog } from '../lib/cso/runtime-catalog'; +import { imageSourceFiles, requalificationTriggers, staleRequalificationTriggers } from '../scripts/cso-requalification'; +import { catalogPromotionCandidate } from '../scripts/cso-runtime-promotion'; +import { completeRuntimeCatalogFixture } from './helpers/cso-runtime-catalog'; + +const ROOT = resolve(import.meta.dir, '..'); +const temps: string[] = []; +afterEach(() => { for (const dir of temps.splice(0)) rmSync(dir, { recursive: true, force: true }); }); +const current = await requalificationTriggers(); + +describe('CSO runtime requalification triggers', () => { + test('the committed catalog is not older than its trigger inputs', () => { + expect(staleRequalificationTriggers(committedCatalog as unknown as RuntimeCatalog, current)).toEqual([]); + const cli = spawnSync(process.execPath, ['scripts/cso-requalification.ts', 'check', 'lib/cso/runtime-catalog.json'], { cwd: ROOT, encoding: 'utf8', timeout: 30_000 }); + expect(cli.status, cli.stderr).toBe(0); expect(cli.stdout).toBe('REQUALIFICATION CURRENT\n'); + }); + + test('triggers cover the helper ABI, isolation policy, every image-embedded source byte, and the build inputs', async () => { + expect(current).toMatchObject({ helperAbi: 3, isolationPolicyHash: ISOLATION_POLICY_HASH, buildInputsRevision: buildInputs.revision }); + const files = await imageSourceFiles(); + for (const file of ['lib/cso/verifier.ts', 'lib/cso/preparation-container.ts', 'lib/cso/contracts.ts', 'lib/cso/images/entrypoint', 'lib/cso/images/rails.Dockerfile']) expect(files).toContain(file); + for (const file of ['lib/cso/images/README.md', 'lib/cso/images/qualification.json', 'lib/cso/images/build-inputs.json', 'lib/cso/cli.ts']) expect(files).not.toContain(file); + const copy = mkdtempSync(join(tmpdir(), 'cso-requalification-')); temps.push(copy); + cpSync(join(ROOT, 'lib'), join(copy, 'lib'), { recursive: true }); + expect(await requalificationTriggers(copy)).toEqual(current); + writeFileSync(join(copy, 'lib/cso/verifier.ts'), '\n', { flag: 'a' }); + expect((await requalificationTriggers(copy)).preparationSha256).not.toBe(current.preparationSha256); + }); + + test('a promoted catalog names each trigger that changed after qualification', () => { + const catalog = completeRuntimeCatalogFixture(); + const recorded = catalog.promotion!.requalification; + expect(staleRequalificationTriggers(catalog, { ...recorded })).toEqual([]); + expect(staleRequalificationTriggers(catalog, { ...recorded, preparationSha256: 'd'.repeat(64), buildInputsRevision: 'next-inputs' })).toEqual(['buildInputsRevision', 'preparationSha256']); + const withdrawn = { ...catalog, runtimes: [], promotion: undefined }; + expect(staleRequalificationTriggers(withdrawn, { ...recorded, preparationSha256: 'd'.repeat(64) })).toEqual([]); + }); + + test('promoted catalogs must record valid triggers, and the helper refuses runtimes qualified under another isolation policy', () => { + const catalog = completeRuntimeCatalogFixture(); + validateRuntimeCatalog(catalog); + expect(selectRuntime('node', 'linux/amd64', catalog).stack).toBe('node'); + const missing = structuredClone(catalog) as any; delete missing.promotion.requalification; + expect(() => validateRuntimeCatalog(missing)).toThrow('INVALID_RUNTIME_PROMOTION'); + const otherBuild = structuredClone(catalog); otherBuild.promotion!.requalification.buildInputsRevision = 'other-build'; + expect(() => validateRuntimeCatalog(otherBuild)).toThrow('INVALID_RUNTIME_PROMOTION'); + const otherPolicy = structuredClone(catalog); otherPolicy.promotion!.requalification.isolationPolicyHash = 'e'.repeat(64); + expect(() => selectRuntime('node', 'linux/amd64', otherPolicy)).toThrow('STALE_RUNTIME_QUALIFICATION'); + }); + + test('promotion records the staged triggers and refuses ones measured under another isolation policy', () => { + expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, [], { ...current, isolationPolicyHash: 'e'.repeat(64) })).toThrow('STALE_RUNTIME_QUALIFICATION'); + expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, [], { ...current, buildInputsRevision: 'other' })).toThrow('INVALID_REQUALIFICATION_TRIGGERS'); + expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, [], { ...current, extra: true } as any)).toThrow('INVALID_REQUALIFICATION_TRIGGERS'); + }); +}); diff --git a/test/cso-runtime-promotion.test.ts b/test/cso-runtime-promotion.test.ts index 2becd10da..5725762aa 100644 --- a/test/cso-runtime-promotion.test.ts +++ b/test/cso-runtime-promotion.test.ts @@ -2,11 +2,13 @@ import { describe, expect, test } from 'bun:test'; import { createHash } from 'node:crypto'; import committedCatalog from '../lib/cso/runtime-catalog.json'; import buildInputs from '../lib/cso/images/build-inputs.json'; -import { validateRuntimeCatalog } from '../lib/cso/runtime-catalog'; +import { isEvaluationRuntimeCatalog, validateRuntimeCatalog } from '../lib/cso/runtime-catalog'; import { imageBuildMatrix, type ImageBuildRow } from '../scripts/cso-image-matrix'; -import { catalogPromotionCandidate, validateRuntimeCatalogTransition, type RuntimeQualificationStatement } from '../scripts/cso-runtime-promotion'; +import { catalogPromotionCandidate, evaluationCandidateCatalog, validateEvaluationCandidate, validateRuntimeCatalogTransition, type RuntimeQualificationStatement } from '../scripts/cso-runtime-promotion'; import { assertVersionOutput, probesForBuildRow } from '../scripts/cso-verify-runtime-base'; +import { requalificationTriggers } from '../scripts/cso-requalification'; +const TRIGGERS = await requalificationTriggers(); const digest = (value: string) => `sha256:${createHash('sha256').update(value).digest('hex')}`; function checks(row: ImageBuildRow): Record { @@ -69,7 +71,7 @@ describe('CSO runtime catalog promotion', () => { }); test('complete same-run evidence generates a validated review candidate bound to its evidence', () => { - const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, statements()); + const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, statements(), TRIGGERS); expect(candidate.previousRevision).toBe(committedCatalog.revision); expect(candidate.runtimes).toHaveLength(10); expect(candidate.promotion).toMatchObject({ @@ -85,30 +87,30 @@ describe('CSO runtime catalog promotion', () => { test('missing release checks, mutable images, and split qualification runs cannot be promoted', () => { const missing = statements(); delete missing[0].checks.heldOutRepairPassed; - expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, missing)).toThrow('MISSING_RELEASE_GATES'); + expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, missing, TRIGGERS)).toThrow('MISSING_RELEASE_GATES'); const mutable = statements(); mutable[0].image = 'ghcr.io/garrytan/gstack/cso-staging/node-amd64:latest'; - expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, mutable)).toThrow('INVALID_QUALIFIED_IMAGE'); + expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, mutable, TRIGGERS)).toThrow('INVALID_QUALIFIED_IMAGE'); const split = statements(); split[0].workflow = 'https://github.com/garrytan/gstack/actions/runs/999999'; - expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, split)).toThrow('SPLIT_QUALIFICATION_RUN'); + expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, split, TRIGGERS)).toThrow('SPLIT_QUALIFICATION_RUN'); const extra = statements() as Array; extra[0].finding = 'must not enter a public release artifact'; - expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, extra)).toThrow('INVALID_QUALIFICATION_STATEMENT_FIELDS'); + expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, extra, TRIGGERS)).toThrow('INVALID_QUALIFICATION_STATEMENT_FIELDS'); }); test('qualification statements must match reviewed versions and every expected platform row', () => { const drift = statements(); drift[0].versions = { ...drift[0].versions, node: '24.5.0' }; - expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, drift)).toThrow('QUALIFICATION_BUILD_MISMATCH'); - expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, statements().slice(1))).toThrow('INCOMPLETE_QUALIFICATION_MATRIX'); + expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, drift, TRIGGERS)).toThrow('QUALIFICATION_BUILD_MISMATCH'); + expect(() => catalogPromotionCandidate(committedCatalog, buildInputs, statements().slice(1), TRIGGERS)).toThrow('INCOMPLETE_QUALIFICATION_MATRIX'); }); test('promotion is a compare-and-swap against the exact reviewed catalog revision', () => { - const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, statements()); + const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, statements(), TRIGGERS); expect(() => validateRuntimeCatalogTransition(committedCatalog, candidate)).not.toThrow(); expect(() => validateRuntimeCatalogTransition( { ...committedCatalog, revision: 'catalog-advanced-concurrently' }, @@ -123,7 +125,7 @@ describe('CSO runtime catalog promotion', () => { }); test('retained promotion evidence rejects a post-generation runtime image substitution', () => { - const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, statements()); + const candidate = catalogPromotionCandidate(committedCatalog, buildInputs, statements(), TRIGGERS); const altered = structuredClone(candidate); altered.runtimes[0].image = altered.runtimes[0].image.replace(/[a-f0-9]{64}$/, 'e'.repeat(64)); expect(altered.promotion!.evidenceDigest).toBe(candidate.promotion!.evidenceDigest); @@ -148,3 +150,53 @@ describe('CSO reviewed base version probes', () => { expect(() => assertVersionOutput('bundler', '2.6.7', 'Bundler version 2.6.7\n')).not.toThrow(); }); }); + +/** staged-image.json records exactly as the stage job writes them. */ +function stagedRecords(): Record[] { + return statements().map(({ checks: _checks, qualifiedAt: _qualifiedAt, state: _state, ...statement }) => ({ + ...statement, state: 'staged', verifiedProvenance: true, qualification: 'pending native and private release gates; no automatic promotion', + })); +} + +describe('CSO evaluation-only runtime catalogs', () => { + test('one staging run yields an evaluation catalog with no measured release gate', () => { + const candidate = evaluationCandidateCatalog(committedCatalog, buildInputs, stagedRecords(), '2026-10-05T00:00:00.000Z'); + expect(candidate.revision).toBe(`cso-eval-123456-${'a'.repeat(12)}`); + expect(isEvaluationRuntimeCatalog(candidate)).toBe(true); expect(isEvaluationRuntimeCatalog(committedCatalog as any)).toBe(false); + expect(candidate.promotion).toBeUndefined(); expect(candidate.evaluation?.workflow).toBe('https://github.com/garrytan/gstack/actions/runs/123456'); + expect(candidate.runtimes.every(runtime => runtime.state === 'evaluation_candidate' && runtime.qualification.kind === 'evaluation')).toBe(true); + expect(JSON.stringify(candidate)).not.toMatch(/Passed/); + expect(validateEvaluationCandidate(candidate, buildInputs)).toBe(candidate); + expect(() => validateEvaluationCandidate(committedCatalog, buildInputs)).toThrow('NOT_AN_EVALUATION_CATALOG'); + }); + test('staging records must come from one run and match the reviewed build rows', () => { + const split = stagedRecords(); split[1].workflow = 'https://github.com/garrytan/gstack/actions/runs/999'; + expect(() => evaluationCandidateCatalog(committedCatalog, buildInputs, split, '2026-10-05T00:00:00.000Z')).toThrow('SPLIT_STAGING_RUN'); + const unstaged = stagedRecords(); unstaged[0].state = 'qualified'; + expect(() => evaluationCandidateCatalog(committedCatalog, buildInputs, unstaged, '2026-10-05T00:00:00.000Z')).toThrow('INVALID_STAGED_RUNTIME'); + const moved = stagedRecords(); moved[0].buildRevision = 'other-build-revision'; + expect(() => evaluationCandidateCatalog(committedCatalog, buildInputs, moved, '2026-10-05T00:00:00.000Z')).toThrow('QUALIFICATION_BUILD_MISMATCH'); + expect(() => evaluationCandidateCatalog(committedCatalog, buildInputs, stagedRecords().slice(1), '2026-10-05T00:00:00.000Z')).toThrow('INCOMPLETE_QUALIFICATION_MATRIX'); + }); + test('evaluation and release catalogs cannot borrow each other\'s shape', () => { + const candidate = evaluationCandidateCatalog(committedCatalog, buildInputs, stagedRecords(), '2026-10-05T00:00:00.000Z'); + const rebound = structuredClone(candidate); rebound.revision = `cso-eval-999-${'a'.repeat(12)}`; + expect(() => validateRuntimeCatalog(rebound)).toThrow('INVALID_EVALUATION_RUNTIME_CATALOG'); + const promoted = structuredClone(candidate) as any; promoted.promotion = { ...promoted.evaluation, qualificationEvidenceDigest: digest('x'), requalification: TRIGGERS }; delete promoted.evaluation; + expect(() => validateRuntimeCatalog(promoted)).toThrow('INVALID_EVALUATION_RUNTIME_CATALOG'); + const disguised = structuredClone(candidate) as any; disguised.revision = `cso-v3-123456-${'a'.repeat(12)}`; + expect(() => validateRuntimeCatalog(disguised)).toThrow('UNQUALIFIED_RUNTIME'); + const gated = structuredClone(candidate) as any; gated.runtimes[0].qualification.heldOutRepairPassed = true; + expect(() => validateRuntimeCatalog(gated)).toThrow('INVALID_EVALUATION_RUNTIME'); + const released = catalogPromotionCandidate(committedCatalog, buildInputs, statements(), TRIGGERS) as any; + released.revision = `cso-eval-123456-${'a'.repeat(12)}`; + expect(() => validateRuntimeCatalog(released)).toThrow('UNQUALIFIED_RUNTIME'); + }); + test('promotion and catalog transitions refuse evaluation catalogs', () => { + const candidate = evaluationCandidateCatalog(committedCatalog, buildInputs, stagedRecords(), '2026-10-05T00:00:00.000Z'); + expect(() => catalogPromotionCandidate(candidate, buildInputs, statements(), TRIGGERS)).toThrow('EVALUATION_CATALOG_NOT_PROMOTABLE'); + expect(() => validateRuntimeCatalogTransition(committedCatalog, candidate)).toThrow('EVALUATION_CATALOG_NOT_PROMOTABLE'); + expect(() => validateRuntimeCatalogTransition(candidate, catalogPromotionCandidate(committedCatalog, buildInputs, statements(), TRIGGERS))).toThrow('EVALUATION_CATALOG_NOT_PROMOTABLE'); + expect(isEvaluationRuntimeCatalog(committedCatalog as any)).toBe(false); + }); +}); diff --git a/test/cso-staging-run.test.ts b/test/cso-staging-run.test.ts new file mode 100644 index 000000000..ce09cf8a6 --- /dev/null +++ b/test/cso-staging-run.test.ts @@ -0,0 +1,164 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { committedImageBuildMatrix } from '../scripts/cso-image-matrix'; +import { dispatchSourceCommit, stagingRunId, verifyStagingRun, type StagingRunEvidence } from '../scripts/cso-staging-run'; + +const ROOT = resolve(import.meta.dir, '..'); +const REPOSITORY = 'garrytan/gstack'; +const RUN_ID = '34980146643'; +const COMMIT = 'a'.repeat(40); +const RUNTIME_IDS = committedImageBuildMatrix().include.map(row => row.runtimeId); +const temps: string[] = []; +afterEach(() => { for (const dir of temps.splice(0)) rmSync(dir, { recursive: true, force: true }); }); + +function invocation(runId = RUN_ID, attempt = 1, repository = REPOSITORY): string { + return `https://github.com/${repository}/actions/runs/${runId}/attempts/${attempt}`; +} + +// Shape of one `gh attestation verify --format json` entry for an actions/attest provenance statement. +function verified(predicateInvocation = invocation(), certificateInvocation = predicateInvocation): unknown[] { + return [{ + attestation: { bundle: {} }, + verificationResult: { + mediaType: 'application/vnd.dev.sigstore.verificationresult+json;version=0.1', + signature: { certificate: { + subjectAlternativeName: `https://github.com/${REPOSITORY}/.github/workflows/cso-runtime-images.yml@refs/heads/main`, + githubWorkflowTrigger: 'workflow_dispatch', + sourceRepositoryDigest: COMMIT, + runInvocationURI: certificateInvocation, + } }, + statement: { + _type: 'https://in-toto.io/Statement/v1', + predicateType: 'https://slsa.dev/provenance/v1', + subject: [{ name: 'ghcr.io/garrytan/gstack/cso-staging/node-amd64', digest: { sha256: 'b'.repeat(64) } }], + predicate: { + buildDefinition: { externalParameters: { workflow: { path: '.github/workflows/cso-runtime-images.yml', ref: 'refs/heads/main' } } }, + runDetails: { builder: { id: 'builder' }, metadata: { invocationId: predicateInvocation } }, + }, + }, + }, + }]; +} + +function run(overrides: Record = {}): Record { + return { + id: Number(RUN_ID), path: '.github/workflows/cso-runtime-images.yml', event: 'workflow_dispatch', + head_branch: 'main', head_sha: COMMIT, status: 'completed', conclusion: 'success', run_attempt: 2, + repository: { full_name: REPOSITORY }, head_repository: { full_name: REPOSITORY }, ...overrides, + }; +} + +function jobs(): Array> { + const job = (name: string) => ({ name, run_id: Number(RUN_ID), status: 'completed', conclusion: 'success' }); + return [ + job('reviewed-inputs'), + { ...job('validate-native'), conclusion: 'skipped' }, + ...RUNTIME_IDS.map(id => job(`stage (${id})`)), + ...RUNTIME_IDS.map(id => job(`qualify-native ${id}`)), + ]; +} + +function pages(list = jobs(), size = 100): unknown[] { + const result: unknown[] = []; + for (let start = 0; start < list.length; start += size) result.push({ total_count: list.length, jobs: list.slice(start, start + size) }); + return result; +} + +function evidence(overrides: Partial = {}): StagingRunEvidence { + return { repository: REPOSITORY, runId: RUN_ID, sourceCommit: COMMIT, runtimeIds: RUNTIME_IDS, run: run(), jobPages: pages(), ...overrides }; +} + +describe('CSO staging run binding', () => { + test('the committed matrix has the ten native rows the ingress requires', () => { + expect(RUNTIME_IDS).toHaveLength(10); + }); + + test('resolves one staging run from every image provenance, across rerun attempts', () => { + const all = RUNTIME_IDS.map((_, index) => verified(invocation(RUN_ID, index === 0 ? 2 : 1))); + expect(stagingRunId(REPOSITORY, all)).toBe(RUN_ID); + }); + + test('rejects statements that resolve to different runs, foreign repositories, or forged invocations', () => { + const split = RUNTIME_IDS.map((_, index) => verified(invocation(index === 3 ? '999' : RUN_ID))); + expect(() => stagingRunId(REPOSITORY, split)).toThrow('STAGING_RUN_NOT_UNIQUE'); + const twoInOne = [[...verified(), ...verified(invocation('999'))]]; + expect(() => stagingRunId(REPOSITORY, twoInOne)).toThrow('STAGING_RUN_NOT_UNIQUE'); + expect(() => stagingRunId(REPOSITORY, [verified(invocation(RUN_ID, 1, 'someone/fork'))])).toThrow('STAGING_INVOCATION_MISMATCH'); + expect(() => stagingRunId(REPOSITORY, [verified(invocation(), invocation('999'))])).toThrow('STAGING_INVOCATION_MISMATCH'); + expect(() => stagingRunId(REPOSITORY, [verified(`${invocation()}/extra`)])).toThrow('STAGING_INVOCATION_MISMATCH'); + const sbom = verified() as any[]; + sbom[0].verificationResult.statement.predicateType = 'https://spdx.dev/Document/v2.3'; + expect(() => stagingRunId(REPOSITORY, [sbom])).toThrow('INVALID_STAGING_PROVENANCE'); + expect(() => stagingRunId(REPOSITORY, [[]])).toThrow('INVALID_STAGING_PROVENANCE'); + expect(() => stagingRunId(REPOSITORY, [])).toThrow('INVALID_STAGING_PROVENANCE'); + }); + + test('accepts the successful protected-main staging dispatch with every native gate passed, across job pages', () => { + expect(() => verifyStagingRun(evidence())).not.toThrow(); + expect(() => verifyStagingRun(evidence({ jobPages: pages(jobs(), 7) }))).not.toThrow(); + }); + + test('rejects a run that is not the successful protected-main staging dispatch at the statements commit', () => { + expect(() => verifyStagingRun(evidence({ run: run({ path: '.github/workflows/cso-runtime-qualification.yml' }) }))).toThrow('STAGING_WORKFLOW_MISMATCH'); + expect(() => verifyStagingRun(evidence({ run: run({ head_branch: 'feature' }) }))).toThrow('STAGING_RUN_NOT_PROTECTED_MAIN_DISPATCH'); + expect(() => verifyStagingRun(evidence({ run: run({ event: 'pull_request' }) }))).toThrow('STAGING_RUN_NOT_PROTECTED_MAIN_DISPATCH'); + expect(() => verifyStagingRun(evidence({ run: run({ conclusion: 'failure' }) }))).toThrow('STAGING_RUN_NOT_SUCCESSFUL'); + expect(() => verifyStagingRun(evidence({ run: run({ status: 'in_progress', conclusion: null }) }))).toThrow('STAGING_RUN_NOT_SUCCESSFUL'); + expect(() => verifyStagingRun(evidence({ run: run({ head_sha: 'c'.repeat(40) }) }))).toThrow('STAGING_SOURCE_COMMIT_MISMATCH'); + expect(() => verifyStagingRun(evidence({ run: run({ id: 1 }) }))).toThrow('STAGING_RUN_ID_MISMATCH'); + expect(() => verifyStagingRun(evidence({ run: run({ head_repository: { full_name: 'someone/fork' } }) }))).toThrow('STAGING_REPOSITORY_MISMATCH'); + }); + + test('rejects a failed, missing, extra, or truncated native gate job', () => { + const failed = jobs(); + failed[failed.length - 1].conclusion = 'failure'; + expect(() => verifyStagingRun(evidence({ jobPages: pages(failed) }))).toThrow('STAGING_NATIVE_GATE_NOT_PASSED'); + const skipped = jobs(); + skipped[skipped.length - 1].conclusion = 'skipped'; + expect(() => verifyStagingRun(evidence({ jobPages: pages(skipped) }))).toThrow('STAGING_NATIVE_GATE_NOT_PASSED'); + const missing = jobs().slice(0, -1); + expect(() => verifyStagingRun(evidence({ jobPages: pages(missing) }))).toThrow('STAGING_NATIVE_JOBS_MISMATCH'); + const extra = [...jobs(), { name: 'qualify-native unexpected', run_id: Number(RUN_ID), status: 'completed', conclusion: 'success' }]; + expect(() => verifyStagingRun(evidence({ jobPages: pages(extra) }))).toThrow('STAGING_NATIVE_JOBS_MISMATCH'); + const truncated = pages(jobs(), 7).slice(0, -1); + expect(() => verifyStagingRun(evidence({ jobPages: truncated }))).toThrow('INCOMPLETE_STAGING_JOBS'); + const otherRun = jobs(); + otherRun[otherRun.length - 1].run_id = 1; + expect(() => verifyStagingRun(evidence({ jobPages: pages(otherRun) }))).toThrow('STAGING_NATIVE_GATE_NOT_PASSED'); + expect(() => verifyStagingRun(evidence({ jobPages: [] }))).toThrow('INVALID_STAGING_JOBS'); + }); + + test('a dispatch must name one exact source commit', () => { + const statements = RUNTIME_IDS.map(runtimeId => ({ runtimeId, sourceCommit: COMMIT })); + expect(dispatchSourceCommit({ client_payload: { statements } })).toBe(COMMIT); + statements[4].sourceCommit = 'c'.repeat(40); + expect(() => dispatchSourceCommit({ client_payload: { statements } })).toThrow('STAGING_SOURCE_COMMIT_MISMATCH'); + expect(() => dispatchSourceCommit({ client_payload: { statements: [{ sourceCommit: 'main' }] } })).toThrow('STAGING_SOURCE_COMMIT_MISMATCH'); + }); + + test('the workflow command line resolves and verifies the run from retained files', () => { + const dir = mkdtempSync(join(tmpdir(), 'cso-staging-run-')); temps.push(dir); + const files = RUNTIME_IDS.map(id => { + const file = join(dir, `${id}.json`); + writeFileSync(file, JSON.stringify(verified())); + return file; + }); + const script = join(ROOT, 'scripts/cso-staging-run.ts'); + const resolved = spawnSync(process.execPath, ['run', script, 'run-id', '--repository', REPOSITORY, ...files], { encoding: 'utf8', timeout: 30_000 }); + expect(resolved.status, resolved.stderr).toBe(0); + expect(resolved.stdout).toBe(`${RUN_ID}\n`); + writeFileSync(join(dir, 'event.json'), JSON.stringify({ client_payload: { statements: RUNTIME_IDS.map(runtimeId => ({ runtimeId, sourceCommit: COMMIT })) } })); + writeFileSync(join(dir, 'run.json'), JSON.stringify(run())); + writeFileSync(join(dir, 'jobs.json'), JSON.stringify(pages())); + const args = ['run', script, 'verify', '--repository', REPOSITORY, '--run-id', RUN_ID, '--event', join(dir, 'event.json'), '--run', join(dir, 'run.json'), '--jobs', join(dir, 'jobs.json')]; + const accepted = spawnSync(process.execPath, args, { encoding: 'utf8', timeout: 30_000 }); + expect(accepted.status, accepted.stderr).toBe(0); + writeFileSync(join(dir, 'run.json'), JSON.stringify(run({ conclusion: 'failure' }))); + const rejected = spawnSync(process.execPath, args, { encoding: 'utf8', timeout: 30_000 }); + expect(rejected.status).toBe(1); + expect(rejected.stderr).toContain('STAGING_RUN_NOT_SUCCESSFUL'); + }); +}); diff --git a/test/cso-verifier.test.ts b/test/cso-verifier.test.ts index 6448441ca..48ab1e1b9 100644 --- a/test/cso-verifier.test.ts +++ b/test/cso-verifier.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from 'bun:test'; -import { boundedResponseBody } from '../lib/cso/verifier'; +import type { HttpAssertion } from '../lib/cso/contracts'; +import { boundedResponseBody, judgeSecurity } from '../lib/cso/verifier'; describe('CSO bounded loopback verifier response reader',()=>{ test('accepts a response exactly at the configured byte limit',async()=>{ @@ -26,3 +27,50 @@ describe('CSO bounded loopback verifier response reader',()=>{ expect(cancelled).toBe(true); }); }); + +describe('CSO verifier security judgement',()=>{ + const payloads:HttpAssertion[]=[ + {name:'id parameter',path:'/user?id=2',method:'GET',expected:{status:403},vulnerable:{status:200,includes:'tenant-b'}}, + {name:'tenant header',path:'/user',method:'GET',headers:{'x-tenant':'b'},expected:{status:403},vulnerable:{status:200,includes:'tenant-b'}}, + {name:'write path',path:'/user/2',method:'PUT',body:'{}',expected:{status:404},vulnerable:{status:204}}, + ]; + const serve=(fixed:Set)=>{ + const seen:string[]=[]; + const server=Bun.serve({port:0,hostname:'127.0.0.1',fetch(req){ + const url=new URL(req.url),key=`${req.method} ${url.pathname}${url.search}`;seen.push(key); + if(req.method==='PUT')return new Response(null,{status:fixed.has(key)?404:204}); + return fixed.has(key)?new Response('denied',{status:403}):new Response('tenant-b secret',{status:200}); + }}); + return {server,seen,port:Number(server.port)}; + }; + const ALL=new Set(['GET /user?id=2','GET /user','PUT /user/2']); + + test('one booted server answers every array member and the before phase needs every vulnerable expectation',async()=>{ + const {server,seen,port}=serve(new Set()); + try{ + expect(await judgeSecurity('before',payloads,port)).toEqual({security:'intended_failure',members:['intended_failure','intended_failure','intended_failure']}); + expect(seen).toEqual(['GET /user?id=2','GET /user','PUT /user/2']); + }finally{server.stop(true);} + }); + + test('the after phase passes only when every array member passes its fixed expectation',async()=>{ + const fixed=serve(ALL); + try{expect(await judgeSecurity('after',payloads,fixed.port)).toEqual({security:'pass',members:['pass','pass','pass']});}finally{fixed.server.stop(true);} + const partial=serve(new Set(['GET /user?id=2','PUT /user/2'])); + try{ + expect(await judgeSecurity('after',payloads,partial.port)).toEqual({security:'inconclusive',members:['pass','inconclusive','pass']}); + expect(await judgeSecurity('before',payloads,partial.port)).toEqual({security:'inconclusive',members:['pass','intended_failure','pass']}); + }finally{partial.server.stop(true);} + const disproved=serve(ALL); + try{expect((await judgeSecurity('before',payloads,disproved.port)).security).toBe('pass');}finally{disproved.server.stop(true);} + }); + + test('a single assertion keeps its original before and after outcomes',async()=>{ + const vulnerable=serve(new Set()),fixed=serve(ALL); + try{ + expect(await judgeSecurity('before',payloads[0],vulnerable.port)).toEqual({security:'intended_failure',members:['intended_failure']}); + expect(await judgeSecurity('after',payloads[0],vulnerable.port)).toEqual({security:'inconclusive',members:['inconclusive']}); + expect(await judgeSecurity('after',payloads[0],fixed.port)).toEqual({security:'pass',members:['pass']}); + }finally{vulnerable.server.stop(true);fixed.server.stop(true);} + }); +}); diff --git a/test/fixtures/cso-eval/README.md b/test/fixtures/cso-eval/README.md index 66067f32b..eca16982b 100644 --- a/test/fixtures/cso-eval/README.md +++ b/test/fixtures/cso-eval/README.md @@ -249,6 +249,47 @@ reports, containment/canary receipts, and enforced oracle separation. Missing evaluations return `unmeasured` or `partial`, never passing release gates. +## Release profile, private corpora, and two architectures + +The matrix above is the `full` profile: matched v2/v3 cells, three repetitions, +and one producer installation. This public corpus is a development set, because +its oracles are public. A private evaluator scores a held-out corpus with the +same layout through the same code: + +- **Injected corpus.** `defineEvalCorpus(manifest, sourceFiles)` accepts a + manifest with this `manifest.json` layout and a deterministic source generator, + checks every variant's source hash, and refuses the public corpus version. + `prepareEvalJobs(..., corpus)` and `materializeCase(..., corpus)` materialize + from it; `createEvalMatrix`, `validateMatrix`, `collectProducerReceipts`, + `scoreCollectedEval`, and `createEvalBaseline` take its manifest. Its oracles + use the `PrivateOracle` shape from `test/helpers/cso-eval-oracles.ts` and stay + in the private evaluator. +- **`release` profile.** `createEvalMatrix({ profile: 'release', skillHashes: { v3 }, + platforms: { daily, comprehensive } })` builds v3-only cells with one repetition. + Daily cells run on one platform; each comprehensive case is fixed to + `linux/amd64` or `linux/arm64`. Every cell carries its `platform`, and the + producer refuses a cell whose platform differs from its host. +- **Per-platform identities.** One batch may hold both architectures. Each + platform must have exactly one producer installation and one provider binary; + platforms must use different installations with identical embedded catalogs + and identical provider family, policy revision, version, and arguments. All + cells must report one effective model. +- **External v2 baseline.** A `baseline` profile matrix holds v2 comprehensive + cells with the release platform assignment. `createEvalBaseline` turns its + complete, fully adjudicated, receipt-bound results into an `EvalBaseline` + record whose `baselineHash` covers the corpus hash, model, host, budget, + platform assignment, v2 payload hash, producer batch, effective model, provider + policy and version, and each case's outcome. A release matrix pins that hash + when it is created; scoring then requires the exact record and the same + effective model and provider policy. Without a pinned baseline the + regression gate stays `unmeasured`. + +A matrix, batch, or result from one corpus never validates against another: +the matrix binds the corpus hash, so a mixed-corpus batch is rejected. The +`collect`, `score`, and `baseline` commands accept `--corpus-manifest `, +and `score` accepts `--baseline `. The `full` profile's matrix bytes and +gates are unchanged. + The free accounting tests use clearly labeled synthetic observations to test the scorer. They are **not measured agent performance**. No paid comparison, fixture cold-start qualification, or containment qualification is claimed by this corpus. diff --git a/test/fixtures/cso-eval/materialize.ts b/test/fixtures/cso-eval/materialize.ts index 53b8be23a..d9c96b994 100644 --- a/test/fixtures/cso-eval/materialize.ts +++ b/test/fixtures/cso-eval/materialize.ts @@ -9,13 +9,22 @@ export const FAMILIES = ['sql-injection', 'command-injection', 'path-traversal', export type EvalStack = typeof STACKS[number]; export type EvalFamily = typeof FAMILIES[number]; export type EvalVariant = 'vulnerable' | 'fixed'; -export interface CorpusCase { - id: string; stack: EvalStack; family: EvalFamily; severity: 'critical' | 'high' | 'medium'; +export interface CorpusCase { + id: string; stack: EvalStack; family: Family; severity: 'critical' | 'high' | 'medium'; rootCause: string; location: { path: string; symbol: string }; coreColdStart: true; heldOut: true; filesHash: { vulnerable: string; fixed: string }; } -export interface CorpusManifest { schemaVersion: 1; version: string; cases: CorpusCase[] } +export interface CorpusManifest { schemaVersion: 1; version: string; cases: CorpusCase[] } +/** + * A corpus with the public layout: a manifest plus a deterministic source + * generator for each case variant. Private evaluators inject their own; the + * default is this public development corpus. + */ +export interface EvalCorpus { + manifest: CorpusManifest; + sourceFiles(id: string, variant: EvalVariant): Record; +} const hash = (input: string) => createHash('sha256').update(input).digest('hex'); export function sourceHash(files: Record): string { return hash(JSON.stringify(Object.keys(files).sort().map(path => [path, hash(files[path])]))); @@ -304,16 +313,67 @@ export function loadCorpusManifest(): CorpusManifest { if (JSON.stringify(manifest) !== JSON.stringify(createManifest())) throw new Error('CORPUS_INTEGRITY_MISMATCH'); return manifest; } -export function materializeCase(id: string, variant: EvalVariant, destination: string): { path: string; sourceHash: string } { - const manifest = loadCorpusManifest(); - const spec = manifest.cases.find(item => item.id === id); +const SLUG = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; +const HEX = /^[a-f0-9]{64}$/; +const SOURCE_PATH = /^[A-Za-z0-9._-]+(?:\/[A-Za-z0-9._-]+)*$/; +const safeSourcePath = (path: string) => SOURCE_PATH.test(path) && path.length <= 256 && path.split('/').every(part => part !== '.' && part !== '..'); + +/** Validate the manifest layout of a public or injected corpus without reading its sources. */ +export function validateCorpusManifest(value: unknown): CorpusManifest { + const manifest = value as CorpusManifest; + if (!manifest || typeof manifest !== 'object' || manifest.schemaVersion !== 1 || typeof manifest.version !== 'string' || !/^[a-z0-9][a-z0-9.-]{0,63}$/.test(manifest.version) || + !Array.isArray(manifest.cases) || manifest.cases.length === 0 || canonicalKeys(manifest) !== 'cases,schemaVersion,version') throw new Error('INVALID_CORPUS_MANIFEST'); + const ids = new Set(); + for (const item of manifest.cases) { + if (!item || canonicalKeys(item) !== 'coreColdStart,family,filesHash,heldOut,id,location,rootCause,severity,stack' || + typeof item.id !== 'string' || !SLUG.test(item.id) || ids.has(item.id) || !STACKS.includes(item.stack) || + typeof item.family !== 'string' || !SLUG.test(item.family) || !['critical', 'high', 'medium'].includes(item.severity) || + typeof item.rootCause !== 'string' || !item.rootCause.trim() || !item.location || typeof item.location.path !== 'string' || !safeSourcePath(item.location.path) || + typeof item.location.symbol !== 'string' || !item.location.symbol.trim() || item.coreColdStart !== true || item.heldOut !== true || + !item.filesHash || !HEX.test(item.filesHash.vulnerable) || !HEX.test(item.filesHash.fixed) || item.filesHash.vulnerable === item.filesHash.fixed) throw new Error('INVALID_CORPUS_MANIFEST'); + ids.add(item.id); + } + return manifest; +} +function canonicalKeys(value: object): string { return Object.keys(value).sort().join(','); } + +/** The public development corpus. Its oracles are public, so it is never held out. */ +export function publicCorpus(): EvalCorpus { + return { manifest: loadCorpusManifest(), sourceFiles }; +} + +/** + * Bind an injected corpus to its manifest: every variant's generated sources + * must hash to the pinned value. An injected corpus cannot reuse the public + * corpus version, so its matrices and results never alias public ones. + */ +export function defineEvalCorpus(manifestValue: unknown, generate: EvalCorpus['sourceFiles']): EvalCorpus { + const manifest = validateCorpusManifest(manifestValue); + if (manifest.version === CORPUS_VERSION) throw new Error('INJECTED_CORPUS_REUSES_PUBLIC_VERSION'); + const read = (id: string, variant: EvalVariant) => { + const files = generate(id, variant); + if (!files || typeof files !== 'object' || !Object.keys(files).length || Object.entries(files).some(([path, contents]) => !safeSourcePath(path) || typeof contents !== 'string')) throw new Error('INVALID_CORPUS_SOURCE'); + return files; + }; + for (const item of manifest.cases) for (const variant of ['vulnerable', 'fixed'] as const) { + if (sourceHash(read(item.id, variant)) !== item.filesHash[variant]) throw new Error('CORPUS_INTEGRITY_MISMATCH'); + } + return { manifest, sourceFiles: (id, variant) => { + if (!manifest.cases.some(item => item.id === id)) throw new Error('UNKNOWN_CORPUS_CASE'); + if (variant !== 'vulnerable' && variant !== 'fixed') throw new Error('INVALID_CORPUS_VARIANT'); + return read(id, variant); + } }; +} + +export function materializeCase(id: string, variant: EvalVariant, destination: string, corpus: EvalCorpus = publicCorpus()): { path: string; sourceHash: string } { + const spec = corpus.manifest.cases.find(item => item.id === id); if (!spec) throw new Error('UNKNOWN_CORPUS_CASE'); const path = resolve(destination); if (existsSync(path)) throw new Error('CORPUS_DESTINATION_EXISTS'); const parent = dirname(path); if (realpathSync(parent) !== parent || !lstatSync(parent).isDirectory()) throw new Error('UNSAFE_CORPUS_DESTINATION'); - const files = sourceFiles(id, variant); - if (sourceHash(files) !== spec.filesHash[variant]) throw new Error('CORPUS_INTEGRITY_MISMATCH'); + const files = corpus.sourceFiles(id, variant); + if (sourceHash(files) !== spec.filesHash[variant] || Object.keys(files).some(file => !safeSourcePath(file))) throw new Error('CORPUS_INTEGRITY_MISMATCH'); mkdirSync(path, { mode: 0o700 }); for (const [file, contents] of Object.entries(files)) { const output = join(path, file); mkdirSync(dirname(output), { recursive: true, mode: 0o700 }); diff --git a/test/helpers/cso-runtime-catalog.ts b/test/helpers/cso-runtime-catalog.ts index 0bcf9e2bd..c7bdd90bf 100644 --- a/test/helpers/cso-runtime-catalog.ts +++ b/test/helpers/cso-runtime-catalog.ts @@ -1,5 +1,6 @@ import type { CsoStack } from '../../lib/cso/preparation'; import { canonical, sha256 } from '../../lib/cso/contracts'; +import { ISOLATION_POLICY_HASH } from '../../lib/cso/docker'; import { CSO_HELPER_ABI, type QualifiedRuntime, @@ -96,6 +97,12 @@ export function completeRuntimeCatalogFixture( workflow: WORKFLOW, evidenceDigest: `sha256:${sha256(canonical(runtimes))}`, qualificationEvidenceDigest: DIGEST, + requalification: { + helperAbi: CSO_HELPER_ABI, + isolationPolicyHash: ISOLATION_POLICY_HASH, + preparationSha256: 'c'.repeat(64), + buildInputsRevision: 'runtime-test-build-v1', + }, }, runtimes, };