From 8243e7213d41cd3d9237ce983e6cabe53f9dc24e Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 2 Oct 2026 18:49:36 +0000 Subject: [PATCH] ci: pin every action to a SHA, Scorecard, checksummed Bun, scoped Codex CLI in the CI image, macOS named-regressions job (#1948, #1706, #1997) --- .github/docker/Dockerfile.ci | 41 +++++- .github/workflows/actionlint.yml | 2 +- .github/workflows/dependency-review.yml | 2 +- .github/workflows/evals-marathon.yml | 23 ++++ .github/workflows/evals-periodic.yml | 46 +++++++ .github/workflows/evals.yml | 23 ++++ .github/workflows/free-tests.yml | 47 ++++++- .github/workflows/make-pdf-gate.yml | 4 +- .github/workflows/quality-gate.yml | 2 +- .github/workflows/scorecard.yml | 52 ++++++++ .github/workflows/skill-docs.yml | 4 +- .github/workflows/version-gate.yml | 4 +- .github/workflows/windows-free-tests.yml | 10 +- .github/workflows/windows-setup-e2e.yml | 6 +- CONTRIBUTING.md | 11 ++ scripts/lib/paid-cases.ts | 59 ++++++++ scripts/test-paid-shards.ts | 3 +- test/bun-version-drift.test.ts | 4 +- test/ci-image-cli-pin.test.ts | 115 ++++++++++++++++ test/codex-ci-access.test.ts | 163 +++++++++++++++++++++++ test/cso-distribution.test.ts | 2 +- test/eng-finding-retry-budget.test.ts | 2 +- test/helpers/periodic-exclude-data.ts | 20 --- test/paid-retry-supervision.test.ts | 4 +- test/skill-e2e-workflow.test.ts | 10 +- test/workflow-action-pins.test.ts | 105 +++++++++++++++ 26 files changed, 707 insertions(+), 57 deletions(-) create mode 100644 .github/workflows/scorecard.yml create mode 100644 test/codex-ci-access.test.ts create mode 100644 test/workflow-action-pins.test.ts diff --git a/.github/docker/Dockerfile.ci b/.github/docker/Dockerfile.ci index 90e6b5d13..97691b28d 100644 --- a/.github/docker/Dockerfile.ci +++ b/.github/docker/Dockerfile.ci @@ -77,13 +77,27 @@ RUN curl --retry 5 --retry-delay 5 --retry-connrefused -fsSL "https://nodejs.org && npm --version # Bun (install to /usr/local so non-root users can access it). -# The version MUST be passed as a positional arg — bun.sh/install ignores a -# BUN_VERSION env var, so the old `| BUN_VERSION=x.y.z bash` form silently -# installed latest on every image rebuild (observed: 1.3.13/1.3.14 drift vs -# the 1.3.10 devs ran locally). +# The exact release archive, verified by SHA-256 before it is unpacked +# (#1706). The old `curl https://bun.sh/install | bash -s bun-vX.Y.Z` ran +# whatever script the network served at build time, and before that the +# `| BUN_VERSION=x.y.z bash` form silently installed latest on every rebuild. +# Hashes come from the release's SHASUMS256.txt. The image builds linux/amd64 +# only (the Node.js tarball above is linux-x64); an architecture without a +# verified archive fails here. test/ci-image-cli-pin.test.ts pins this shape. +ARG BUN_VERSION=1.4.0 +ARG TARGETARCH ENV BUN_INSTALL="/usr/local" -RUN curl --retry 5 --retry-delay 5 --retry-connrefused -fsSL https://bun.sh/install \ - | bash -s "bun-v1.4.0" +RUN case "${TARGETARCH:-amd64}" in \ + amd64) bun_target=linux-x64; bun_sha256=2d03fb5fb83ac8b567aca0a281b2ce1a1a19d488f56c2968d88c3f25e92fe452 ;; \ + *) echo "Dockerfile.ci: no verified Bun ${BUN_VERSION} archive for ${TARGETARCH}" >&2; exit 1 ;; \ + esac \ + && curl --retry 5 --retry-delay 5 --retry-connrefused -fsSL \ + "https://github.com/oven-sh/bun/releases/download/bun-v${BUN_VERSION}/bun-${bun_target}.zip" -o /tmp/bun.zip \ + && echo "${bun_sha256} /tmp/bun.zip" | sha256sum -c - \ + && unzip -q /tmp/bun.zip -d /tmp/bun \ + && install -D -m 0755 "/tmp/bun/bun-${bun_target}/bun" "${BUN_INSTALL}/bin/bun" \ + && rm -rf /tmp/bun /tmp/bun.zip \ + && test "$(bun --version)" = "${BUN_VERSION}" # Claude CLI — pinned to an EXACT version, same discipline as the bun pin # above. The PTY harness (test/helpers/claude-pty-runner.ts) screen-scrapes @@ -102,6 +116,19 @@ RUN curl --retry 5 --retry-delay 5 --retry-connrefused -fsSL https://bun.sh/inst # its SDK-only judge evals, which never start this CLI, were 25% slower too. RUN npm i -g @anthropic-ai/claude-code@2.1.284 +# Codex CLI — exact pin, same discipline as the claude pin above, so the +# Codex eval files execute in CI instead of self-skipping. Installed OFF PATH: +# a skill's outside-voice probe (`command -v codex`) must keep reporting +# not_installed in every other paid case. The paid shard runner puts this +# directory on PATH only for the Codex files (scripts/lib/paid-cases.ts scopeCodexAccess). +# The image carries no credential: the eval workflows run `codex login +# --with-api-key` from the OPENAI_API_KEY secret in same-repo runs only +# (codex 0.160.0 does not read OPENAI_API_KEY from the environment). +ENV GSTACK_CI_CODEX_BIN_DIR=/opt/codex/bin +RUN npm i -g --prefix=/opt/codex @openai/codex@0.160.0 \ + && test -x /opt/codex/bin/codex \ + && ! command -v codex + # Playwright system deps (Chromium) — needed for browse E2E tests RUN npx playwright install-deps chromium @@ -144,7 +171,7 @@ RUN npx playwright install chromium \ && chmod -R a+rX /opt/playwright-browsers # Verify everything works -RUN bun --version && node --version && claude --version && jq --version && gh --version \ +RUN bun --version && node --version && claude --version && /opt/codex/bin/codex --version && jq --version && gh --version \ && python3 --version && command -v file && command -v pdftotext && command -v pdffonts && command -v pdfinfo \ && npx playwright --version \ && fc-match "Liberation Sans" | grep -qi "Liberation" \ diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml index d942f9b21..6a0ef5c4c 100644 --- a/.github/workflows/actionlint.yml +++ b/.github/workflows/actionlint.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubicloud-standard-2 timeout-minutes: 5 steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false # Pull the prebuilt image instead of rhysd/actionlint@v1.7.11 (a Docker diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index ecfee9af5..9a5d23e51 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -23,7 +23,7 @@ jobs: contents: read pull-requests: write steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 with: fail-on-severity: high diff --git a/.github/workflows/evals-marathon.yml b/.github/workflows/evals-marathon.yml index 7d3e52326..ae3a20b2b 100644 --- a/.github/workflows/evals-marathon.yml +++ b/.github/workflows/evals-marathon.yml @@ -172,6 +172,29 @@ jobs: name: marathon-plan path: /tmp/marathon-plan + # The image ships the pinned Codex CLI off PATH and without a + # credential; codex does not read OPENAI_API_KEY from the environment. + # Log it in from the secret (same-repo runs only) under a non-default + # home. The shard runner hands that home and the CLI to the Codex files + # only (scripts/lib/paid-cases.ts scopeCodexAccess); every other paid case sees no + # Codex, as before. Without a key the Codex cases fail; they never pass. + - name: Authenticate the Codex CLI + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: | + set -euo pipefail + codex="${GSTACK_CI_CODEX_BIN_DIR:?}/codex" + export CODEX_HOME="$HOME/.gstack-ci-codex" + mkdir -p "$CODEX_HOME" + echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV" + if [ -z "${OPENAI_API_KEY:-}" ]; then + echo "::warning::OPENAI_API_KEY is unavailable; Codex eval cases cannot authenticate and will fail." + exit 0 + fi + printenv OPENAI_API_KEY | "$codex" login --with-api-key + "$codex" login status >/dev/null 2>&1 + - name: Run marathon slice ${{ matrix.slice }} env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} diff --git a/.github/workflows/evals-periodic.yml b/.github/workflows/evals-periodic.yml index 9713a4600..e1d4d2cd2 100644 --- a/.github/workflows/evals-periodic.yml +++ b/.github/workflows/evals-periodic.yml @@ -206,6 +206,29 @@ jobs: name: paid-plan path: /tmp/paid-plan + # The image ships the pinned Codex CLI off PATH and without a + # credential; codex does not read OPENAI_API_KEY from the environment. + # Log it in from the secret (same-repo runs only) under a non-default + # home. The shard runner hands that home and the CLI to the Codex files + # only (scripts/lib/paid-cases.ts scopeCodexAccess); every other paid case sees no + # Codex, as before. Without a key the Codex cases fail; they never pass. + - name: Authenticate the Codex CLI + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: | + set -euo pipefail + codex="${GSTACK_CI_CODEX_BIN_DIR:?}/codex" + export CODEX_HOME="$HOME/.gstack-ci-codex" + mkdir -p "$CODEX_HOME" + echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV" + if [ -z "${OPENAI_API_KEY:-}" ]; then + echo "::warning::OPENAI_API_KEY is unavailable; Codex eval cases cannot authenticate and will fail." + exit 0 + fi + printenv OPENAI_API_KEY | "$codex" login --with-api-key + "$codex" login status >/dev/null 2>&1 + - name: Run periodic slice ${{ matrix.slice }} env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} @@ -306,6 +329,29 @@ jobs: name: gate-census-plan path: /tmp/gate-census-plan + # The image ships the pinned Codex CLI off PATH and without a + # credential; codex does not read OPENAI_API_KEY from the environment. + # Log it in from the secret (same-repo runs only) under a non-default + # home. The shard runner hands that home and the CLI to the Codex files + # only (scripts/lib/paid-cases.ts scopeCodexAccess); every other paid case sees no + # Codex, as before. Without a key the Codex cases fail; they never pass. + - name: Authenticate the Codex CLI + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: | + set -euo pipefail + codex="${GSTACK_CI_CODEX_BIN_DIR:?}/codex" + export CODEX_HOME="$HOME/.gstack-ci-codex" + mkdir -p "$CODEX_HOME" + echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV" + if [ -z "${OPENAI_API_KEY:-}" ]; then + echo "::warning::OPENAI_API_KEY is unavailable; Codex eval cases cannot authenticate and will fail." + exit 0 + fi + printenv OPENAI_API_KEY | "$codex" login --with-api-key + "$codex" login status >/dev/null 2>&1 + - name: Run gate census slice ${{ matrix.slice }} env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index ee2875132..f1406a491 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -274,6 +274,29 @@ jobs: mkdir -p /tmp/paid-slice-results/receipts if [ -d /tmp/paid-plan/receipts ]; then cp -a /tmp/paid-plan/receipts/. /tmp/paid-slice-results/receipts/; fi + # The image ships the pinned Codex CLI off PATH and without a + # credential; codex does not read OPENAI_API_KEY from the environment. + # Log it in from the secret (same-repo runs only) under a non-default + # home. The shard runner hands that home and the CLI to the Codex files + # only (scripts/lib/paid-cases.ts scopeCodexAccess); every other paid case sees no + # Codex, as before. Without a key the Codex cases fail; they never pass. + - name: Authenticate the Codex CLI + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: | + set -euo pipefail + codex="${GSTACK_CI_CODEX_BIN_DIR:?}/codex" + export CODEX_HOME="$HOME/.gstack-ci-codex" + mkdir -p "$CODEX_HOME" + echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV" + if [ -z "${OPENAI_API_KEY:-}" ]; then + echo "::warning::OPENAI_API_KEY is unavailable; Codex eval cases cannot authenticate and will fail." + exit 0 + fi + printenv OPENAI_API_KEY | "$codex" login --with-api-key + "$codex" login status >/dev/null 2>&1 + - name: Run slice ${{ matrix.slice }} env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} diff --git a/.github/workflows/free-tests.yml b/.github/workflows/free-tests.yml index decf06938..29219ec62 100644 --- a/.github/workflows/free-tests.yml +++ b/.github/workflows/free-tests.yml @@ -91,6 +91,43 @@ jobs: for file in "${files[@]}"; do test -f "$file"; done bun test "${files[@]}" + # Named regressions that only reproduce on macOS's default temp root: + # $TMPDIR sits under /var/folders and /var is a symlink to /private/var, so + # code that compares a realpath with an unresolved path passes on Linux's + # /tmp and fails here (#2982). TMPDIR is deliberately left at the runner's + # default, and the job fails if that layout is not symlinked. + macos-named-regressions: + runs-on: macos-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.4.0 + - name: Install dependencies + run: bun install --frozen-lockfile --ignore-scripts + - name: Build browser prerequisites + run: | + bunx playwright install chromium + bun run build + - name: Configure git identity (tests init temp repos) + run: | + git config --global user.email "free-tests-ci@gstack.test" + git config --global user.name "Free Tests CI" + git config --global init.defaultBranch main + - name: Run the named macOS regressions on the default temp root + run: | + set -euo pipefail + tmp="${TMPDIR:-/tmp}" + resolved="$(realpath "$tmp")" + echo "temp root: $tmp -> $resolved" + test "$resolved" != "${tmp%/}" + files=(test/gstack-design-detect.test.ts browse/test/chromium-profile-isolation.test.ts test/cso-git-hardening.test.ts test/plan-count-file-permission.test.ts test/spec-quality-gate-secret-sink.test.ts test/gstack-memory-ingest.test.ts) + for file in "${files[@]}"; do test -f "$file"; done + bun test "${files[@]}" + cso-windows-launcher: runs-on: windows-latest timeout-minutes: 15 @@ -178,7 +215,7 @@ jobs: name: free-plan path: ${{ runner.temp }} - - uses: actions/cache@v6 + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: ~/.bun/install/cache key: linux-bun-${{ hashFiles('bun.lock') }} @@ -189,7 +226,7 @@ jobs: - name: Install dependencies run: bun install --frozen-lockfile - - uses: actions/cache@v6 + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: ~/.cache/ms-playwright key: linux-playwright-${{ hashFiles('bun.lock') }} @@ -323,15 +360,16 @@ jobs: # gate is merge-blocking without a separate branch-protection migration. free-tests: if: always() - needs: [free-suite, typecheck, cso-macos-launcher, cso-windows-launcher, cso-docker-integration] + needs: [free-suite, typecheck, cso-macos-launcher, macos-named-regressions, cso-windows-launcher, cso-docker-integration] runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - - name: Require the free suite, typecheck, and every CSO platform gate + - name: Require the free suite, typecheck, macOS regressions, and every CSO platform gate env: FREE_SUITE_RESULT: ${{ needs.free-suite.result }} TYPECHECK_RESULT: ${{ needs.typecheck.result }} CSO_MACOS_RESULT: ${{ needs.cso-macos-launcher.result }} + MACOS_REGRESSIONS_RESULT: ${{ needs.macos-named-regressions.result }} CSO_WINDOWS_RESULT: ${{ needs.cso-windows-launcher.result }} CSO_DOCKER_RESULT: ${{ needs.cso-docker-integration.result }} run: | @@ -339,6 +377,7 @@ jobs: test "$FREE_SUITE_RESULT" = success test "$TYPECHECK_RESULT" = success test "$CSO_MACOS_RESULT" = success + test "$MACOS_REGRESSIONS_RESULT" = success test "$CSO_WINDOWS_RESULT" = success test "$CSO_DOCKER_RESULT" = success - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 diff --git a/.github/workflows/make-pdf-gate.yml b/.github/workflows/make-pdf-gate.yml index f9ee442c7..75c5992f1 100644 --- a/.github/workflows/make-pdf-gate.yml +++ b/.github/workflows/make-pdf-gate.yml @@ -49,9 +49,9 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 15 steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: oven-sh/setup-bun@v2 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.4.0 diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index 7b17963e3..50fd9e99a 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -35,7 +35,7 @@ jobs: # take ~12s combined. The one history consumer (the added-lines diff) # fetches its exact base/head SHAs below — an exact-SHA fetch, not a # guessed depth, so long-lived branches and merge queues still resolve. - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.4.0 diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 000000000..982f7cf45 --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,52 @@ +name: OpenSSF Scorecard +# Supply-chain posture report (#1997): pinned actions, token permissions, +# branch protection, dangerous workflow patterns. Results land in the +# Security tab as code-scanning alerts and are published to +# api.scorecard.dev for the badge. +# +# Publishing imposes rules this file must keep: no workflow-level env, +# defaults or write permissions; `id-token: write` only on the job that runs +# ossf/scorecard-action; that job uses only approved actions on a GitHub +# hosted Ubuntu runner, with no env, defaults, containers or `run:` steps. +# Scorecard does not support forks, so the job runs only in this repository. + +on: + push: + branches: [main] + schedule: + - cron: '41 5 * * 2' + branch_protection_rule: + workflow_dispatch: + +permissions: read-all + +jobs: + analysis: + if: github.repository == 'garrytan/gstack' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + actions: read + security-events: write + id-token: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4 + with: + results_file: results.sarif + results_format: sarif + publish_results: true + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: scorecard-sarif + path: results.sarif + retention-days: 5 + + - uses: github/codeql-action/upload-sarif@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + with: + sarif_file: results.sarif diff --git a/.github/workflows/skill-docs.yml b/.github/workflows/skill-docs.yml index 966b878be..3fa8299dc 100644 --- a/.github/workflows/skill-docs.yml +++ b/.github/workflows/skill-docs.yml @@ -25,8 +25,8 @@ jobs: runs-on: ubicloud-standard-2 timeout-minutes: 10 steps: - - uses: actions/checkout@v7 - - uses: oven-sh/setup-bun@v2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.4.0 - run: bun install --frozen-lockfile diff --git a/.github/workflows/version-gate.yml b/.github/workflows/version-gate.yml index fb715c03d..8c72fd505 100644 --- a/.github/workflows/version-gate.yml +++ b/.github/workflows/version-gate.yml @@ -21,13 +21,13 @@ jobs: pull-requests: read steps: - name: Checkout PR head - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - name: Setup Bun - uses: oven-sh/setup-bun@v2 + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.4.0 diff --git a/.github/workflows/windows-free-tests.yml b/.github/workflows/windows-free-tests.yml index 2a1a7ec46..09e3e8e30 100644 --- a/.github/workflows/windows-free-tests.yml +++ b/.github/workflows/windows-free-tests.yml @@ -61,9 +61,9 @@ jobs: timeout-minutes: 15 steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: oven-sh/setup-bun@v2 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.4.0 @@ -74,7 +74,7 @@ jobs: # bun install was 35s of a 55s job, all network. Cache keyed on the # lockfile; bun's install cache lives under ~/.bun/install/cache on # every platform. - - uses: actions/cache@v6 + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: ~/.bun/install/cache key: windows-bun-${{ hashFiles('bun.lock') }} @@ -175,7 +175,7 @@ jobs: # automatically, so no per-file entry is needed here.) - name: Upload full shard logs if: always() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: windows-free-test-shard-logs path: | @@ -186,7 +186,7 @@ jobs: - name: Upload flake ledger if: always() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: flake-ledger-windows path: ${{ runner.temp }}/flake-ledger.jsonl diff --git a/.github/workflows/windows-setup-e2e.yml b/.github/workflows/windows-setup-e2e.yml index 9f41c1d03..906598346 100644 --- a/.github/workflows/windows-setup-e2e.yml +++ b/.github/workflows/windows-setup-e2e.yml @@ -41,15 +41,15 @@ jobs: timeout-minutes: 10 steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: oven-sh/setup-bun@v2 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.4.0 # Same lockfile-keyed install cache as windows-free-tests.yml (install # was 45s of a 64s job, all network). - - uses: actions/cache@v6 + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: ~/.bun/install/cache key: windows-bun-${{ hashFiles('bun.lock') }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 07496edbb..beffbe8bf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -69,6 +69,17 @@ No setup needed. Learnings are logged automatically. View them with `/learn`. This is the best way to contribute: fix gstack while doing your real work, in the project where you actually felt the pain. +**What CI does on a fork PR.** GitHub never gives a fork PR this repository's +secrets, so CI runs everything that needs none: the required `free-tests` check +(the Linux free suite, typecheck, the macOS and Windows gates), Windows Free +Tests, Skill Docs Freshness, Workflow Lint, Version Gate, Quality gate, the +make-pdf gate, and any path-triggered gates your change touches. E2E Evals +still builds the CI image from your `Dockerfile.ci` without publishing it, and +skips its paid eval jobs, which need provider API keys. That skip is expected +and does not block your PR. It is also not a pass: a maintainer runs the paid +evals from a branch in this repository before merging. A first-time +contributor's runs wait for a maintainer to approve them. + ### Session awareness When you have 3+ gstack sessions open simultaneously, every question tells you which project, which branch, and what's happening. No more staring at a question thinking "wait, which window is this?" The format is consistent across all skills. diff --git a/scripts/lib/paid-cases.ts b/scripts/lib/paid-cases.ts index f5f6c46e4..bf186281c 100644 --- a/scripts/lib/paid-cases.ts +++ b/scripts/lib/paid-cases.ts @@ -230,3 +230,62 @@ export function partitionCaseExclusions(keys: string[]): { runnable: string[]; e }); return { runnable, excluded }; } + +/** + * Codex access in the CI eval image, scoped per paid shard. + * + * The image installs the pinned Codex CLI off PATH (GSTACK_CI_CODEX_BIN_DIR) + * and the eval workflows log it in under a non-default home + * (GSTACK_CI_CODEX_HOME). A gate shard never sees either. A non-gate shard + * whose every file exercises Codex gets the CLI on PATH and the home as + * CODEX_HOME. A file that mixes one Codex case with other cases keeps only + * the two variables, and that case opts in on its own session; hermetic + * child environments drop GSTACK_* names, so its sibling sessions see no + * Codex. Every other shard sees neither, so a skill's outside-voice probe + * (`command -v codex`) reports not_installed exactly as it did before Codex + * entered the image. Outside CI neither variable is set and nothing changes. + */ +export const CODEX_CI_ENV = { binDir: 'GSTACK_CI_CODEX_BIN_DIR', home: 'GSTACK_CI_CODEX_HOME' } as const; + +/** Paid files whose every case exercises the real Codex CLI. */ +export const CODEX_CI_FILES: readonly string[] = [ + 'test/codex-e2e.test.ts', + 'test/codex-e2e-sol-scope.test.ts', + 'test/codex-e2e-shared-libs.test.ts', + 'test/codex-e2e-recommendation-substance.test.ts', + 'test/skill-e2e-outside-voice.test.ts', + 'test/skill-e2e-outside-plan-disabled.test.ts', +]; + +/** + * Codex cases inside files whose other cases must not see Codex. Each listed + * case opts in by putting $GSTACK_CI_CODEX_BIN_DIR on its own session PATH + * (inline, so the file's other touchfile keys do not grow a helper). + */ +export const CODEX_CI_CASES: Readonly> = { + 'test/skill-e2e-workflow.test.ts': ['codex-review'], +}; + +export type CodexShardAccess = 'none' | 'case-opt-in' | 'path'; + +/** What one shard, given the test files behind its keys, may see of the CI Codex install in `tier`. */ +export function codexShardAccess(files: readonly string[], tier: string | undefined): CodexShardAccess { + if (tier === 'gate' || files.length === 0) return 'none'; + if (files.every(file => CODEX_CI_FILES.includes(file))) return 'path'; + return files.every(file => file in CODEX_CI_CASES) ? 'case-opt-in' : 'none'; +} + +/** Scope one shard's child environment in place; `files` are the shard keys' test files. */ +export function scopeCodexAccess(env: NodeJS.ProcessEnv, files: readonly string[]): void { + const binDir = env[CODEX_CI_ENV.binDir]; + if (!binDir) return; + const home = env[CODEX_CI_ENV.home]; + const access = codexShardAccess(files, env.EVALS_TIER); + delete env.CODEX_HOME; + if (access === 'case-opt-in') return; + delete env[CODEX_CI_ENV.binDir]; + delete env[CODEX_CI_ENV.home]; + if (access !== 'path') return; + env.PATH = env.PATH ? `${binDir}${path.delimiter}${env.PATH}` : binDir; + if (home) env.CODEX_HOME = home; +} diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts index 9eac2e08b..bc92b31ab 100644 --- a/scripts/test-paid-shards.ts +++ b/scripts/test-paid-shards.ts @@ -105,7 +105,7 @@ import { export { PAID_TEST_GLOBS, isPaidTestFile }; export { PERIODIC_CI_EXCLUDE }; -import { shardFile, shardCaseId, shardTrial, trialShardKey, type CaseTrialPlan, caseTrialPlan, excludedCasesNamePattern, caseTestNamePattern, expandCaseShards, expandTrialShards, partitionCaseExclusions } from './lib/paid-cases'; +import { scopeCodexAccess, shardFile, shardCaseId, shardTrial, trialShardKey, type CaseTrialPlan, caseTrialPlan, excludedCasesNamePattern, caseTestNamePattern, expandCaseShards, expandTrialShards, partitionCaseExclusions } from './lib/paid-cases'; import { retriesForFiles, trialPanelKey, sliceExecutionOrder, buildRunManifest, parseRunManifest, type SliceResult, sliceExitCode, guardTrialRecords, formatSlicePlan, formatCapacityPreflight } from './lib/paid-plan'; import { caseFile, runCaseDiagnosis, formatPanelLine, runPaidReport } from './lib/paid-report'; export * from './lib/paid-cases'; @@ -953,6 +953,7 @@ export async function runPaidShard( const expectedCases = options.expectedCases ?? (caseId !== null ? { [files[0]!]: 1 } : undefined); const baseEnv = { ...(options.env ?? process.env) }; + scopeCodexAccess(baseEnv, files.map(shardFile)); if (options.evalDirBase) { baseEnv.GSTACK_EVAL_DIR = path.join(options.evalDirBase, 'shards', shardSlug(files)); } diff --git a/test/bun-version-drift.test.ts b/test/bun-version-drift.test.ts index 98a80f917..0a0d21578 100644 --- a/test/bun-version-drift.test.ts +++ b/test/bun-version-drift.test.ts @@ -45,10 +45,10 @@ function collectPins(): Pin[] { const dockerfile = fs.readFileSync( path.join(ROOT, '.github', 'docker', 'Dockerfile.ci'), 'utf-8'); - const dockerPin = dockerfile.match(/bash -s ["']?bun-v([\w.]+)["']?/); + const dockerPin = dockerfile.match(/^ARG BUN_VERSION=["']?([\w.]+)["']?$/m); pins.push({ surface: 'Dockerfile.ci', - version: dockerPin ? dockerPin[1] : '', + version: dockerPin ? dockerPin[1] : '', }); const gitlab = fs.readFileSync(path.join(ROOT, '.gitlab-ci.yml'), 'utf-8'); diff --git a/test/ci-image-cli-pin.test.ts b/test/ci-image-cli-pin.test.ts index e07a19cba..a04ce197c 100644 --- a/test/ci-image-cli-pin.test.ts +++ b/test/ci-image-cli-pin.test.ts @@ -14,6 +14,10 @@ * This tripwire fails the free suite when any globally-installed npm package * in Dockerfile.ci lacks an exact `@X.Y.Z` pin. Bumps are deliberate: edit * the pin in a PR and run the PTY gate against the new TUI before merging. + * + * Bun comes from its release archive, verified by SHA-256 for every + * architecture the image builds (#1706): nothing in the image is piped from + * curl into a shell. */ import { describe, expect, test } from 'bun:test'; import * as fs from 'node:fs'; @@ -64,4 +68,115 @@ describe('ci image provider-CLI pins', () => { + 'harness three times before this tripwire existed.', ).toHaveLength(0); }); + + test('the image installs the codex CLI globally so the Codex evals execute in CI', () => { + expect( + specs.some((s) => s.startsWith('@openai/codex@')), + `expected a pinned @openai/codex install in ${path.relative(ROOT, DOCKERFILE)}; found: ${specs.join(', ')}`, + ).toBe(true); + }); +}); + +/** Dockerfile source with `\`-continued lines joined into logical lines. */ +function logicalLines(source: string): string[] { + return source.replace(/\\\n/g, ' ').split('\n'); +} + +/** Logical lines that pipe a download straight into a shell. */ +export function curlPipedToShell(source: string): string[] { + return logicalLines(source).filter((line) => !/^\s*#/.test(line) && + /\b(?:curl|wget)\b/.test(line) && /\|\s*(?:sudo\s+)?(?:ba|z|da)?sh\b/.test(line)); +} + +/** `) bun_target=...; bun_sha256=` arms of the Bun install `case`. */ +export function bunArchiveArms(source: string): Map { + const arms = new Map(); + for (const m of source.matchAll(/^\s*([\w|]+)\)\s*bun_target=[\w-]+;\s*bun_sha256=([0-9a-f]+)\s*;;/gm)) { + for (const arch of m[1].split('|')) arms.set(arch, m[2]); + } + return arms; +} + +/** Docker architectures each build of Dockerfile.ci produces (no `platforms:` = the amd64 runner's). */ +export function builtArchitectures(workflowsDir: string): Set { + const archs = new Set(); + for (const name of fs.readdirSync(workflowsDir)) { + if (!/\.ya?ml$/.test(name)) continue; + const workflow = Bun.YAML.parse(fs.readFileSync(path.join(workflowsDir, name), 'utf-8')) as any; + for (const job of Object.values(workflow?.jobs ?? {})) { + for (const step of job?.steps ?? []) { + if (!String(step?.uses ?? '').startsWith('docker/build-push-action@')) continue; + if (step.with?.file !== '.github/docker/Dockerfile.ci') continue; + const platforms = String(step.with?.platforms ?? 'linux/amd64').split(/[\s,]+/).filter(Boolean); + for (const platform of platforms) archs.add(platform.replace(/^linux\//, '')); + } + } + } + return archs; +} + +describe('ci image Bun install (#1706)', () => { + const source = fs.readFileSync(DOCKERFILE, 'utf-8'); + const bunRun = logicalLines(source).find((line) => line.includes('oven-sh/bun/releases/download/bun-v${BUN_VERSION}/')) ?? ''; + + test('nothing is piped from a download into a shell', () => { + expect(curlPipedToShell(source), 'download the release archive and verify its SHA-256 instead').toEqual([]); + }); + + test('the tripwire recognizes the old installer pipe and leaves non-shell pipes alone', () => { + expect(curlPipedToShell('RUN curl -fsSL https://bun.sh/install \\\n | bash -s "bun-v1.4.0"\n')).toHaveLength(1); + expect(curlPipedToShell('RUN curl -fsSL https://example.test/x.sh | sh\n')).toHaveLength(1); + expect(curlPipedToShell('RUN curl -fsSL https://example.test/key.gpg | gpg --dearmor -o /k.gpg\n')).toEqual([]); + }); + + test('every architecture the image builds has a SHA-256-verified Bun archive', () => { + const built = builtArchitectures(path.join(ROOT, '.github', 'workflows')); + expect([...built], 'scan found no Dockerfile.ci build').toContain('amd64'); + const arms = bunArchiveArms(source); + for (const arch of built) { + expect(arms.get(arch), `no verified Bun archive for ${arch}`).toMatch(/^[0-9a-f]{64}$/); + } + }); + + test('the archive is checked before it is unpacked, and unknown architectures fail closed', () => { + expect(bunRun, 'Bun must come from the GitHub release archive').not.toBe(''); + const download = bunRun.indexOf('releases/download/'); + const verify = bunRun.indexOf('sha256sum -c'); + const unpack = bunRun.indexOf('unzip'); + expect(verify).toBeGreaterThan(download); + expect(unpack).toBeGreaterThan(verify); + expect(bunRun).toMatch(/\*\)(?:(?!;;).)*exit 1\s*;;/); + expect(bunRun).toContain('test "$(bun --version)" = "${BUN_VERSION}"'); + }); +}); + +describe('ci image codex authentication', () => { + const workflowsDir = path.join(ROOT, '.github', 'workflows'); + const lanes: Array<{ lane: string; steps: any[] }> = []; + for (const name of fs.readdirSync(workflowsDir).sort()) { + if (!/\.ya?ml$/.test(name)) continue; + const workflow = Bun.YAML.parse(fs.readFileSync(path.join(workflowsDir, name), 'utf-8')) as any; + for (const [id, job] of Object.entries(workflow?.jobs ?? {})) { + const steps: any[] = job?.steps ?? []; + if (job?.container && steps.some((step) => /test-paid-shards\.ts .*--slice/.test(String(step?.run ?? '')))) { + lanes.push({ lane: `${name}:${id}`, steps }); + } + } + } + + test('every paid lane in the image logs the codex CLI in from the secret before it runs', () => { + expect(lanes.length, 'scan found no paid lanes').toBeGreaterThanOrEqual(4); + for (const { lane, steps } of lanes) { + const login = steps.findIndex((step) => /codex"? login --with-api-key/.test(String(step?.run ?? ''))); + const run = steps.findIndex((step) => /test-paid-shards\.ts .*--slice/.test(String(step?.run ?? ''))); + expect(login, `${lane}: no codex login step`).toBeGreaterThanOrEqual(0); + expect(login, `${lane}: codex login must precede the paid run`).toBeLessThan(run); + const step = steps[login]; + expect(step.env?.OPENAI_API_KEY, `${lane}: key must arrive through env`).toBe('${{ secrets.OPENAI_API_KEY }}'); + expect(String(step.run), `${lane}: never inline a secret into run`).not.toContain('secrets.'); + expect(String(step.run)).toMatch(/codex"? login status/); + expect(String(step.if ?? ''), `${lane}: same-repo runs only`) + .toBe("github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository"); + } + }); }); diff --git a/test/codex-ci-access.test.ts b/test/codex-ci-access.test.ts new file mode 100644 index 000000000..34a0e15c5 --- /dev/null +++ b/test/codex-ci-access.test.ts @@ -0,0 +1,163 @@ +/** + * The CI image's Codex CLI and credential reach only the Codex eval files. + * + * Every other paid case must see Codex exactly as it did before Codex entered + * the image: not on PATH and not logged in, so a skill's outside-voice probe + * (`command -v codex`) reports not_installed and gate verdicts and slice + * budgets do not move. This pins the image (Codex off PATH), the workflows + * (login under a non-default home, never exported as CODEX_HOME), the + * per-shard scoping helper, and the shard runner's use of it. + */ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { CODEX_CI_CASES, CODEX_CI_ENV, CODEX_CI_FILES, buildRunManifest, codexShardAccess, runPaidShards, scopeCodexAccess, shardFile } from '../scripts/test-paid-shards'; +import { buildHermeticEnv } from './helpers/hermetic-env'; +import { E2E_TIERS, E2E_TOUCHFILES } from './helpers/touchfiles'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const BIN = '/opt/codex/bin'; +const HOME = '/home/runner/.gstack-ci-codex'; +const ciEnv = (tier: string): NodeJS.ProcessEnv => ({ PATH: '/usr/local/bin:/usr/bin:/bin', EVALS_TIER: tier, [CODEX_CI_ENV.binDir]: BIN, [CODEX_CI_ENV.home]: HOME }); +const sees = (env: NodeJS.ProcessEnv) => ({ + onPath: (env.PATH ?? '').split(path.delimiter).includes(BIN), + home: env.CODEX_HOME ?? null, + leaked: Object.keys(env).filter(name => name.startsWith('GSTACK_CI_CODEX')), +}); +const planned = (tier: 'gate' | 'periodic') => buildRunManifest({ tier, profile: 'full', sliceCount: 1, evalsAll: true, env: { EVALS_ALL: '1' } }) + .entries.filter(entry => entry.status === 'planned').map(entry => entry.file); + +/** The paid test files a registered E2E case runs in. */ +const caseFiles = (id: string) => (E2E_TOUCHFILES[id] ?? []).filter(dep => /^test\/.*\.test\.ts$/.test(dep)); + +describe('codex CI access scope', () => { + test('the Codex files and opted-in cases hold the approved periodic Codex cases and no gate case', () => { + for (const file of [...CODEX_CI_FILES, ...Object.keys(CODEX_CI_CASES)]) expect(fs.existsSync(path.join(ROOT, file)), file).toBe(true); + const reaches = (id: string) => caseFiles(id).some(file => CODEX_CI_FILES.includes(file) || CODEX_CI_CASES[file]?.includes(id)); + for (const id of ['codex-review', 'codex-review-findings', 'codex-discover-skill', 'outside-voice-codex-to-claude-code', + 'outside-voice-claude-code-to-codex', 'outside-plan-disabled-no-fallback', 'codex-sol-scope-termination']) { + expect(E2E_TIERS[id], id).toBe('periodic'); + expect(reaches(id), `${id} cannot reach the CI Codex`).toBe(true); + } + for (const ids of Object.values(CODEX_CI_CASES)) for (const id of ids) expect(E2E_TIERS[id], id).toBe('periodic'); + const gateInCodexFiles = Object.keys(E2E_TIERS) + .filter(id => E2E_TIERS[id] === 'gate' && caseFiles(id).some(file => CODEX_CI_FILES.includes(file))); + expect(gateInCodexFiles, 'a gate case would run with an authenticated Codex').toEqual([]); + }); + + test('only the opted-in case of a mixed file asks for Codex, and hermetic sessions drop the CI variables', () => { + for (const [file, ids] of Object.entries(CODEX_CI_CASES)) { + const source = fs.readFileSync(path.join(ROOT, file), 'utf8'); + const calls = [...source.matchAll(/process\.env\.GSTACK_CI_CODEX_BIN_DIR\b/g)].map(match => match.index!); + expect(calls, file).toHaveLength(ids.length); + for (const [i, id] of ids.entries()) { + const start = source.indexOf(`IfSelected('${id}'`); + const next = source.slice(start + 1).search(/IfSelected\('/); + const end = next < 0 ? source.length : start + 1 + next; + expect(calls[i]! > start && calls[i]! < end, `${file}: CI Codex read outside the ${id} case`).toBe(true); + } + } + const child = buildHermeticEnv({ ...ciEnv('periodic'), HOME: '/home/runner' }, {}); + expect(Object.keys(child).filter(name => name.startsWith('GSTACK_CI_CODEX'))).toEqual([]); + expect((child.PATH ?? '').split(path.delimiter)).not.toContain(BIN); + }); + + test('no planned gate shard can see the CI Codex CLI or its credential', () => { + const gate = planned('gate'); + expect(gate.length).toBeGreaterThan(20); + for (const key of gate) { + const env = ciEnv('gate'); + scopeCodexAccess(env, [shardFile(key)]); + expect(sees(env), key).toEqual({ onPath: false, home: null, leaked: [] }); + expect(codexShardAccess([shardFile(key)], 'periodic') === 'path', `${key}: a gate shard is a Codex file`).toBe(false); + } + }); + + test('in the periodic census only the Codex files see Codex on PATH', () => { + const periodic = planned('periodic'); + const access = (key: string) => { + const env = ciEnv('periodic'); + scopeCodexAccess(env, [shardFile(key)]); + return { key, ...sees(env) }; + }; + const results = periodic.map(access); + const granted = results.filter(result => result.onPath); + expect([...new Set(granted.map(result => shardFile(result.key)))].sort()).toEqual([...CODEX_CI_FILES].sort()); + for (const result of granted) expect(result.home, result.key).toBe(HOME); + for (const result of results.filter(result => !result.onPath)) { + expect(result.home, result.key).toBeNull(); + expect(result.leaked.length === 0 || shardFile(result.key) in CODEX_CI_CASES, result.key).toBe(true); + } + }); + + test('a mixed shard is denied, case and trial keys follow their file, and local runs are untouched', () => { + expect(codexShardAccess(['test/codex-e2e.test.ts', 'test/skill-e2e-plan.test.ts'], 'periodic')).toBe('none'); + expect(codexShardAccess([shardFile('test/codex-e2e.test.ts#codex-review~t2')], 'periodic')).toBe('path'); + expect(codexShardAccess(['test/codex-e2e.test.ts'], 'gate')).toBe('none'); + expect(codexShardAccess(['test/skill-e2e-workflow.test.ts'], 'gate')).toBe('none'); + expect(codexShardAccess(['test/skill-e2e-workflow.test.ts'], 'periodic')).toBe('case-opt-in'); + expect(codexShardAccess([], 'periodic')).toBe('none'); + const ci = { ...ciEnv('periodic'), CODEX_HOME: '/somewhere/else' }; + scopeCodexAccess(ci, [shardFile('test/skill-e2e-plan.test.ts#codex-offered-eng-review')]); + expect(sees(ci)).toEqual({ onPath: false, home: null, leaked: [] }); + const optIn = { ...ciEnv('periodic'), CODEX_HOME: '/somewhere/else' }; + scopeCodexAccess(optIn, ['test/skill-e2e-workflow.test.ts']); + expect(sees(optIn)).toEqual({ onPath: false, home: null, leaked: [CODEX_CI_ENV.binDir, CODEX_CI_ENV.home] }); + const local: NodeJS.ProcessEnv = { PATH: '/usr/bin', CODEX_HOME: '/home/dev/.codex' }; + scopeCodexAccess(local, ['test/skill-e2e-plan.test.ts']); + expect(local).toEqual({ PATH: '/usr/bin', CODEX_HOME: '/home/dev/.codex' }); + }); + + test.skipIf(process.platform === 'win32')('the paid shard runner gives each child the scoped environment', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codex-ci-access-')); + try { + const bin = path.join(dir, 'bin'), home = path.join(dir, 'home'); + fs.mkdirSync(bin); fs.mkdirSync(home); + fs.writeFileSync(path.join(bin, 'codex'), '#!/bin/sh\nexit 0\n', { mode: 0o755 }); + const shards = [['test/codex-e2e.test.ts'], ['test/skill-e2e-workflow.test.ts'], + ['test/skill-e2e-plan.test.ts#codex-offered-eng-review'], ['test/skill-e2e-outside-voice.test.ts', 'test/skill-e2e-review.test.ts']]; + const probe = 'p=$(command -v codex || echo none); printf "%s|%s|%s" "$p" "${CODEX_HOME:-}" "$(env | grep -c "^GSTACK_CI_CODEX" || true)"'; + const run = async (tier: string) => { + const out = fs.mkdtempSync(path.join(dir, `${tier}-`)); + await runPaidShards(shards, { + jobs: 1, timeoutMs: 30_000, log: () => {}, logDir: dir, + env: { PATH: '/usr/bin:/bin', HOME: dir, EVALS_TIER: tier, GSTACK_CLAUDE_CLI_VERSION: 'test', [CODEX_CI_ENV.binDir]: bin, [CODEX_CI_ENV.home]: home }, + commandFor: files => ({ command: '/bin/sh', args: ['-c', `${probe} > "${path.join(out, String(shards.indexOf(files)))}"`] }), + }); + return shards.map((_, index) => fs.readFileSync(path.join(out, String(index)), 'utf8')); + }; + expect(await run('periodic')).toEqual([`${path.join(bin, 'codex')}|${home}|0`, 'none||2', 'none||0', 'none||0']); + expect(await run('gate')).toEqual(['none||0', 'none||0', 'none||0', 'none||0']); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }, 60_000); +}); + +describe('codex CI access in the image and workflows', () => { + test('the image installs Codex off PATH at the directory it advertises', () => { + const dockerfile = fs.readFileSync(path.join(ROOT, '.github/docker/Dockerfile.ci'), 'utf8'); + expect(dockerfile).toMatch(/^ENV GSTACK_CI_CODEX_BIN_DIR=\/opt\/codex\/bin$/m); + expect(dockerfile).toMatch(/npm i -g --prefix=\/opt\/codex @openai\/codex@\d+\.\d+\.\d+/); + expect(dockerfile).toContain('! command -v codex'); + expect(dockerfile).not.toMatch(/^ENV PATH=.*\/opt\/codex/m); + }); + + test('the login step never makes the credential the default Codex home', () => { + const steps: string[] = []; + for (const name of ['evals.yml', 'evals-periodic.yml', 'evals-marathon.yml']) { + const workflow = Bun.YAML.parse(fs.readFileSync(path.join(ROOT, '.github/workflows', name), 'utf8')) as any; + for (const job of Object.values(workflow.jobs)) { + for (const step of job.steps ?? []) if (/login --with-api-key/.test(String(step.run ?? ''))) steps.push(String(step.run)); + } + } + expect(steps.length).toBeGreaterThanOrEqual(4); + for (const run of steps) { + expect(run).toContain('echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV"'); + expect(run).not.toMatch(/echo "CODEX_HOME=/); + expect(run).not.toContain('$HOME/.codex"'); + expect(run).toContain('"${GSTACK_CI_CODEX_BIN_DIR:?}/codex"'); + } + }); +}); diff --git a/test/cso-distribution.test.ts b/test/cso-distribution.test.ts index 2657e6ab2..9fe4d6bc0 100644 --- a/test/cso-distribution.test.ts +++ b/test/cso-distribution.test.ts @@ -318,7 +318,7 @@ describe('CSO runtime staging gates', () => { expect(gate['continue-on-error']).not.toBe(true); const required = workflow.jobs['free-tests']; expect(required.if).toBe('always()'); - expect(required.needs).toEqual(['free-suite', 'typecheck', 'cso-macos-launcher', 'cso-windows-launcher', 'cso-docker-integration']); + expect(required.needs).toEqual(['free-suite', 'typecheck', 'cso-macos-launcher', 'macos-named-regressions', 'cso-windows-launcher', 'cso-docker-integration']); expect(required.steps[0].run).toContain('test "$CSO_DOCKER_RESULT" = success'); for (const current of Object.values(workflow.jobs) as any[]) for (const step of current.steps) { if (step.uses?.startsWith('oven-sh/setup-bun')) expect(step.uses).toBe('oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6'); diff --git a/test/eng-finding-retry-budget.test.ts b/test/eng-finding-retry-budget.test.ts index f3966fcde..36411c375 100644 --- a/test/eng-finding-retry-budget.test.ts +++ b/test/eng-finding-retry-budget.test.ts @@ -192,7 +192,7 @@ test('current detach supervision covers the live-census floor', () => { expect(floorFor('gate')).toBe(21_725); expect(gateTimeout).toBe(49_320); expect(gateTimeout).toBeGreaterThanOrEqual(floorFor('gate')); - expect(floorFor('periodic')).toBe(33_821); + expect(floorFor('periodic')).toBe(35_711); expect(periodicTimeout).toBeGreaterThanOrEqual(floorFor('periodic')); }); diff --git a/test/helpers/periodic-exclude-data.ts b/test/helpers/periodic-exclude-data.ts index 638c66d18..650a7105b 100644 --- a/test/helpers/periodic-exclude-data.ts +++ b/test/helpers/periodic-exclude-data.ts @@ -15,26 +15,6 @@ * the re-entry mechanism. */ export const PERIODIC_CI_EXCLUDE: Record = { - 'test/codex-e2e.test.ts': { - reason: 'the codex CLI is not installed in the CI image (Dockerfile.ci ships only claude-code); every case self-skips', - tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)', - }, - 'test/codex-e2e-sol-scope.test.ts': { - reason: 'the codex CLI is not installed in the CI image (Dockerfile.ci ships only claude-code); every case self-skips', - tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)', - }, - 'test/codex-e2e-shared-libs.test.ts': { - reason: 'the codex CLI is not installed in the CI image (Dockerfile.ci ships only claude-code); every case self-skips', - tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)', - }, - 'test/codex-e2e-recommendation-substance.test.ts': { - reason: 'the codex CLI is not installed in the CI image (Dockerfile.ci ships only claude-code); every case self-skips', - tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)', - }, - 'test/skill-e2e-outside-voice.test.ts': { - reason: 'needs both the claude and codex CLIs; codex is not in the CI image, so every case self-skips', - tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)', - }, 'test/skill-e2e-aside.test.ts': { reason: 'needs macOS with the Aside app open (asideAvailable()); CI runners are Linux, so every case self-skips', tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)', diff --git a/test/paid-retry-supervision.test.ts b/test/paid-retry-supervision.test.ts index 3b967a507..c3d3d90ec 100644 --- a/test/paid-retry-supervision.test.ts +++ b/test/paid-retry-supervision.test.ts @@ -230,8 +230,8 @@ test('detached PR fallback and release commands cover their actual default worke )) / 1000 * 1.05)); } const detachedReleaseWall = Number(scripts['eval:bg:release'].match(/--timeout (\d+)/)?.[1]) * 1000; - expect(releaseFloors).toEqual([21_725, 33_821]); - expect(releaseFloors.reduce((total, floor) => total + floor, 0)).toBe(55_546); + expect(releaseFloors).toEqual([21_725, 35_711]); + expect(releaseFloors.reduce((total, floor) => total + floor, 0)).toBe(57_436); expect(detachedReleaseWall).toBe(116_700_000); expect(detachedReleaseWall).toBeGreaterThanOrEqual(releaseWall + 120_000); }); diff --git a/test/skill-e2e-workflow.test.ts b/test/skill-e2e-workflow.test.ts index 9ffc14b76..0af2f7193 100644 --- a/test/skill-e2e-workflow.test.ts +++ b/test/skill-e2e-workflow.test.ts @@ -451,8 +451,13 @@ describeIfSelected('Codex skill E2E', ['codex-review'], () => { }); testConcurrentIfSelected('codex-review', async () => { - // Check codex is available — skip if not installed - const codexCheck = spawnSync('which', ['codex'], { stdio: 'pipe', timeout: 3000 }); + // Check codex is available — skip if not installed. In CI only this case + // opts in to the image's off-PATH Codex (scripts/lib/paid-cases.ts scopeCodexAccess). + const ciCodexBin = process.env.GSTACK_CI_CODEX_BIN_DIR, ciCodexHome = process.env.GSTACK_CI_CODEX_HOME; + const codexEnv: Record = ciCodexBin + ? { PATH: `${ciCodexBin}${path.delimiter}${process.env.PATH ?? ''}`, ...(ciCodexHome ? { CODEX_HOME: ciCodexHome } : {}) } + : {}; + const codexCheck = spawnSync('which', ['codex'], { stdio: 'pipe', timeout: 3000, env: { ...process.env, ...codexEnv } }); if (codexCheck.status !== 0) { console.warn('codex CLI not installed — skipping E2E test'); return; @@ -469,6 +474,7 @@ Write the full output (including the GATE verdict) to ${codexDir}/codex-output.m testName: 'codex-review', runId, model: resolveEvalModel('capture'), + env: codexEnv, }); logCost('/codex review', result); diff --git a/test/workflow-action-pins.test.ts b/test/workflow-action-pins.test.ts new file mode 100644 index 000000000..fae27171d --- /dev/null +++ b/test/workflow-action-pins.test.ts @@ -0,0 +1,105 @@ +/** + * Workflow action pins (supply chain, free). + * + * A tag (`@v7`) or branch ref is mutable: whoever controls the action's + * repository can repoint it, and the next CI run executes the new code with + * this repository's token and secrets. Every remote `uses:` in a workflow or + * composite action must name a full 40-character commit SHA (Dependabot's + * `github-actions` ecosystem keeps those current, with the tag in a trailing + * comment), and every `docker://` image must name a sha256 digest. Local + * actions (`./...`) are part of this checkout and need no pin. + */ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +const ROOT = path.resolve(__dirname, '..'); +const GITHUB_DIR = path.join(ROOT, '.github'); + +/** Why a `uses:` reference is not immutable, or null when it is. */ +export function pinProblem(uses: string): string | null { + if (uses.startsWith('./')) return null; + if (uses.startsWith('docker://')) { + return /@sha256:[0-9a-f]{64}$/.test(uses) ? null : 'docker image is not pinned to a sha256 digest'; + } + const at = uses.lastIndexOf('@'); + if (at <= 0) return 'remote action has no ref'; + if (!/^[\w.-]+\/[\w.-]+(?:\/[\w./-]+)?$/.test(uses.slice(0, at))) return 'unrecognized action reference'; + return /^[0-9a-f]{40}$/.test(uses.slice(at + 1)) ? null : 'ref is not a full 40-character commit SHA'; +} + +/** Every `uses:` in a workflow (job steps and reusable-workflow jobs) or composite action. */ +export function usesReferences(document: any): string[] { + const refs: string[] = []; + const steps = (list: any) => { + for (const step of Array.isArray(list) ? list : []) if (typeof step?.uses === 'string') refs.push(step.uses); + }; + for (const job of Object.values(document?.jobs ?? {})) { + if (typeof job?.uses === 'string') refs.push(job.uses); + steps(job?.steps); + } + steps(document?.runs?.steps); + return refs; +} + +function automationFiles(): string[] { + const files: string[] = []; + const workflows = path.join(GITHUB_DIR, 'workflows'); + for (const name of fs.readdirSync(workflows).sort()) { + if (/\.ya?ml$/.test(name)) files.push(path.join(workflows, name)); + } + const actions = path.join(GITHUB_DIR, 'actions'); + for (const name of fs.existsSync(actions) ? fs.readdirSync(actions).sort() : []) { + for (const file of ['action.yml', 'action.yaml']) { + if (fs.existsSync(path.join(actions, name, file))) files.push(path.join(actions, name, file)); + } + } + return files; +} + +describe('workflow action pins', () => { + test('the classifier accepts immutable references and rejects mutable ones', () => { + for (const ok of [ + './.github/actions/restore-deps', + 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1', + 'github/codeql-action/upload-sarif@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2', + 'google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@f4cfcc01edc9c8b756a9b873b7a623ca674da51e', + `docker://alpine@sha256:${'a'.repeat(64)}`, + ]) expect(pinProblem(ok), ok).toBeNull(); + for (const bad of [ + 'actions/checkout@v7', + 'actions/checkout@main', + 'actions/checkout@3d3c42e', + 'actions/checkout', + 'google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@v2.3.8', + 'docker://alpine:3.20', + 'docker://alpine', + ]) expect(pinProblem(bad), bad).not.toBeNull(); + }); + + test('every remote action, reusable workflow and docker image is pinned immutably', () => { + const files = automationFiles(); + const problems: string[] = []; + let scanned = 0; + for (const file of files) { + const source = fs.readFileSync(file, 'utf-8'); + const refs = usesReferences(Bun.YAML.parse(source)); + // A parse that misses `uses:` lines would pass vacuously; count them. + const lines = source.split('\n').filter((line) => /^\s*(?:-\s+)?uses:\s*\S/.test(line)).length; + expect(refs.length, `${path.relative(ROOT, file)}: parsed ${refs.length} of ${lines} uses: lines`).toBe(lines); + scanned += refs.length; + for (const ref of refs) { + const problem = pinProblem(ref); + if (problem) problems.push(`${path.relative(ROOT, file)}: ${ref} (${problem})`); + } + } + expect(files.length).toBeGreaterThan(10); + expect(scanned).toBeGreaterThan(100); + expect(problems, 'pin each action to a full commit SHA with the tag in a trailing comment').toEqual([]); + }); + + test('Dependabot keeps the pinned actions current', () => { + const config = Bun.YAML.parse(fs.readFileSync(path.join(GITHUB_DIR, 'dependabot.yml'), 'utf-8')) as any; + expect((config.updates ?? []).map((update: any) => update['package-ecosystem'])).toContain('github-actions'); + }); +});