mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-04 07:47:27 +01:00
ci: pin every action to a SHA, Scorecard, checksummed Bun, scoped Codex CLI in the CI image, macOS named-regressions job (#1948, #1706, #1997)
This commit is contained in:
garrytan committed
1 parent
7e60945ca3
commit
8243e7213d
26 files changed
+707
-57
No files matched your search
@@ -77,13 +77,27 @@ RUN curl --retry 5 --retry-delay 5 --retry-connrefused -fsSL "https://nodejs.org
|
||||
&& npm --version
|
||||
|
||||
# Bun (install to /usr/local so non-root users can access it).
|
||||
# The version MUST be passed as a positional arg — bun.sh/install ignores a
|
||||
# BUN_VERSION env var, so the old `| BUN_VERSION=x.y.z bash` form silently
|
||||
# installed latest on every image rebuild (observed: 1.3.13/1.3.14 drift vs
|
||||
# the 1.3.10 devs ran locally).
|
||||
# The exact release archive, verified by SHA-256 before it is unpacked
|
||||
# (#1706). The old `curl https://bun.sh/install | bash -s bun-vX.Y.Z` ran
|
||||
# whatever script the network served at build time, and before that the
|
||||
# `| BUN_VERSION=x.y.z bash` form silently installed latest on every rebuild.
|
||||
# Hashes come from the release's SHASUMS256.txt. The image builds linux/amd64
|
||||
# only (the Node.js tarball above is linux-x64); an architecture without a
|
||||
# verified archive fails here. test/ci-image-cli-pin.test.ts pins this shape.
|
||||
ARG BUN_VERSION=1.4.0
|
||||
ARG TARGETARCH
|
||||
ENV BUN_INSTALL="/usr/local"
|
||||
RUN curl --retry 5 --retry-delay 5 --retry-connrefused -fsSL https://bun.sh/install \
|
||||
| bash -s "bun-v1.4.0"
|
||||
RUN case "${TARGETARCH:-amd64}" in \
|
||||
amd64) bun_target=linux-x64; bun_sha256=2d03fb5fb83ac8b567aca0a281b2ce1a1a19d488f56c2968d88c3f25e92fe452 ;; \
|
||||
*) echo "Dockerfile.ci: no verified Bun ${BUN_VERSION} archive for ${TARGETARCH}" >&2; exit 1 ;; \
|
||||
esac \
|
||||
&& curl --retry 5 --retry-delay 5 --retry-connrefused -fsSL \
|
||||
"https://github.com/oven-sh/bun/releases/download/bun-v${BUN_VERSION}/bun-${bun_target}.zip" -o /tmp/bun.zip \
|
||||
&& echo "${bun_sha256} /tmp/bun.zip" | sha256sum -c - \
|
||||
&& unzip -q /tmp/bun.zip -d /tmp/bun \
|
||||
&& install -D -m 0755 "/tmp/bun/bun-${bun_target}/bun" "${BUN_INSTALL}/bin/bun" \
|
||||
&& rm -rf /tmp/bun /tmp/bun.zip \
|
||||
&& test "$(bun --version)" = "${BUN_VERSION}"
|
||||
|
||||
# Claude CLI — pinned to an EXACT version, same discipline as the bun pin
|
||||
# above. The PTY harness (test/helpers/claude-pty-runner.ts) screen-scrapes
|
||||
@@ -102,6 +116,19 @@ RUN curl --retry 5 --retry-delay 5 --retry-connrefused -fsSL https://bun.sh/inst
|
||||
# its SDK-only judge evals, which never start this CLI, were 25% slower too.
|
||||
RUN npm i -g @anthropic-ai/claude-code@2.1.284
|
||||
|
||||
# Codex CLI — exact pin, same discipline as the claude pin above, so the
|
||||
# Codex eval files execute in CI instead of self-skipping. Installed OFF PATH:
|
||||
# a skill's outside-voice probe (`command -v codex`) must keep reporting
|
||||
# not_installed in every other paid case. The paid shard runner puts this
|
||||
# directory on PATH only for the Codex files (scripts/lib/paid-cases.ts scopeCodexAccess).
|
||||
# The image carries no credential: the eval workflows run `codex login
|
||||
# --with-api-key` from the OPENAI_API_KEY secret in same-repo runs only
|
||||
# (codex 0.160.0 does not read OPENAI_API_KEY from the environment).
|
||||
ENV GSTACK_CI_CODEX_BIN_DIR=/opt/codex/bin
|
||||
RUN npm i -g --prefix=/opt/codex @openai/codex@0.160.0 \
|
||||
&& test -x /opt/codex/bin/codex \
|
||||
&& ! command -v codex
|
||||
|
||||
# Playwright system deps (Chromium) — needed for browse E2E tests
|
||||
RUN npx playwright install-deps chromium
|
||||
|
||||
@@ -144,7 +171,7 @@ RUN npx playwright install chromium \
|
||||
&& chmod -R a+rX /opt/playwright-browsers
|
||||
|
||||
# Verify everything works
|
||||
RUN bun --version && node --version && claude --version && jq --version && gh --version \
|
||||
RUN bun --version && node --version && claude --version && /opt/codex/bin/codex --version && jq --version && gh --version \
|
||||
&& python3 --version && command -v file && command -v pdftotext && command -v pdffonts && command -v pdfinfo \
|
||||
&& npx playwright --version \
|
||||
&& fc-match "Liberation Sans" | grep -qi "Liberation" \
|
||||
|
||||
@@ -23,7 +23,7 @@ jobs:
|
||||
runs-on: ubicloud-standard-2
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
# Pull the prebuilt image instead of rhysd/actionlint@v1.7.11 (a Docker
|
||||
|
||||
@@ -23,7 +23,7 @@ jobs:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0
|
||||
with:
|
||||
fail-on-severity: high
|
||||
|
||||
@@ -172,6 +172,29 @@ jobs:
|
||||
name: marathon-plan
|
||||
path: /tmp/marathon-plan
|
||||
|
||||
# The image ships the pinned Codex CLI off PATH and without a
|
||||
# credential; codex does not read OPENAI_API_KEY from the environment.
|
||||
# Log it in from the secret (same-repo runs only) under a non-default
|
||||
# home. The shard runner hands that home and the CLI to the Codex files
|
||||
# only (scripts/lib/paid-cases.ts scopeCodexAccess); every other paid case sees no
|
||||
# Codex, as before. Without a key the Codex cases fail; they never pass.
|
||||
- name: Authenticate the Codex CLI
|
||||
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
codex="${GSTACK_CI_CODEX_BIN_DIR:?}/codex"
|
||||
export CODEX_HOME="$HOME/.gstack-ci-codex"
|
||||
mkdir -p "$CODEX_HOME"
|
||||
echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV"
|
||||
if [ -z "${OPENAI_API_KEY:-}" ]; then
|
||||
echo "::warning::OPENAI_API_KEY is unavailable; Codex eval cases cannot authenticate and will fail."
|
||||
exit 0
|
||||
fi
|
||||
printenv OPENAI_API_KEY | "$codex" login --with-api-key
|
||||
"$codex" login status >/dev/null 2>&1
|
||||
|
||||
- name: Run marathon slice ${{ matrix.slice }}
|
||||
env:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
|
||||
@@ -206,6 +206,29 @@ jobs:
|
||||
name: paid-plan
|
||||
path: /tmp/paid-plan
|
||||
|
||||
# The image ships the pinned Codex CLI off PATH and without a
|
||||
# credential; codex does not read OPENAI_API_KEY from the environment.
|
||||
# Log it in from the secret (same-repo runs only) under a non-default
|
||||
# home. The shard runner hands that home and the CLI to the Codex files
|
||||
# only (scripts/lib/paid-cases.ts scopeCodexAccess); every other paid case sees no
|
||||
# Codex, as before. Without a key the Codex cases fail; they never pass.
|
||||
- name: Authenticate the Codex CLI
|
||||
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
codex="${GSTACK_CI_CODEX_BIN_DIR:?}/codex"
|
||||
export CODEX_HOME="$HOME/.gstack-ci-codex"
|
||||
mkdir -p "$CODEX_HOME"
|
||||
echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV"
|
||||
if [ -z "${OPENAI_API_KEY:-}" ]; then
|
||||
echo "::warning::OPENAI_API_KEY is unavailable; Codex eval cases cannot authenticate and will fail."
|
||||
exit 0
|
||||
fi
|
||||
printenv OPENAI_API_KEY | "$codex" login --with-api-key
|
||||
"$codex" login status >/dev/null 2>&1
|
||||
|
||||
- name: Run periodic slice ${{ matrix.slice }}
|
||||
env:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
@@ -306,6 +329,29 @@ jobs:
|
||||
name: gate-census-plan
|
||||
path: /tmp/gate-census-plan
|
||||
|
||||
# The image ships the pinned Codex CLI off PATH and without a
|
||||
# credential; codex does not read OPENAI_API_KEY from the environment.
|
||||
# Log it in from the secret (same-repo runs only) under a non-default
|
||||
# home. The shard runner hands that home and the CLI to the Codex files
|
||||
# only (scripts/lib/paid-cases.ts scopeCodexAccess); every other paid case sees no
|
||||
# Codex, as before. Without a key the Codex cases fail; they never pass.
|
||||
- name: Authenticate the Codex CLI
|
||||
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
codex="${GSTACK_CI_CODEX_BIN_DIR:?}/codex"
|
||||
export CODEX_HOME="$HOME/.gstack-ci-codex"
|
||||
mkdir -p "$CODEX_HOME"
|
||||
echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV"
|
||||
if [ -z "${OPENAI_API_KEY:-}" ]; then
|
||||
echo "::warning::OPENAI_API_KEY is unavailable; Codex eval cases cannot authenticate and will fail."
|
||||
exit 0
|
||||
fi
|
||||
printenv OPENAI_API_KEY | "$codex" login --with-api-key
|
||||
"$codex" login status >/dev/null 2>&1
|
||||
|
||||
- name: Run gate census slice ${{ matrix.slice }}
|
||||
env:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
|
||||
@@ -274,6 +274,29 @@ jobs:
|
||||
mkdir -p /tmp/paid-slice-results/receipts
|
||||
if [ -d /tmp/paid-plan/receipts ]; then cp -a /tmp/paid-plan/receipts/. /tmp/paid-slice-results/receipts/; fi
|
||||
|
||||
# The image ships the pinned Codex CLI off PATH and without a
|
||||
# credential; codex does not read OPENAI_API_KEY from the environment.
|
||||
# Log it in from the secret (same-repo runs only) under a non-default
|
||||
# home. The shard runner hands that home and the CLI to the Codex files
|
||||
# only (scripts/lib/paid-cases.ts scopeCodexAccess); every other paid case sees no
|
||||
# Codex, as before. Without a key the Codex cases fail; they never pass.
|
||||
- name: Authenticate the Codex CLI
|
||||
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
codex="${GSTACK_CI_CODEX_BIN_DIR:?}/codex"
|
||||
export CODEX_HOME="$HOME/.gstack-ci-codex"
|
||||
mkdir -p "$CODEX_HOME"
|
||||
echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV"
|
||||
if [ -z "${OPENAI_API_KEY:-}" ]; then
|
||||
echo "::warning::OPENAI_API_KEY is unavailable; Codex eval cases cannot authenticate and will fail."
|
||||
exit 0
|
||||
fi
|
||||
printenv OPENAI_API_KEY | "$codex" login --with-api-key
|
||||
"$codex" login status >/dev/null 2>&1
|
||||
|
||||
- name: Run slice ${{ matrix.slice }}
|
||||
env:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
|
||||
@@ -91,6 +91,43 @@ jobs:
|
||||
for file in "${files[@]}"; do test -f "$file"; done
|
||||
bun test "${files[@]}"
|
||||
|
||||
# Named regressions that only reproduce on macOS's default temp root:
|
||||
# $TMPDIR sits under /var/folders and /var is a symlink to /private/var, so
|
||||
# code that compares a realpath with an unresolved path passes on Linux's
|
||||
# /tmp and fails here (#2982). TMPDIR is deliberately left at the runner's
|
||||
# default, and the job fails if that layout is not symlinked.
|
||||
macos-named-regressions:
|
||||
runs-on: macos-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.4.0
|
||||
- name: Install dependencies
|
||||
run: bun install --frozen-lockfile --ignore-scripts
|
||||
- name: Build browser prerequisites
|
||||
run: |
|
||||
bunx playwright install chromium
|
||||
bun run build
|
||||
- name: Configure git identity (tests init temp repos)
|
||||
run: |
|
||||
git config --global user.email "free-tests-ci@gstack.test"
|
||||
git config --global user.name "Free Tests CI"
|
||||
git config --global init.defaultBranch main
|
||||
- name: Run the named macOS regressions on the default temp root
|
||||
run: |
|
||||
set -euo pipefail
|
||||
tmp="${TMPDIR:-/tmp}"
|
||||
resolved="$(realpath "$tmp")"
|
||||
echo "temp root: $tmp -> $resolved"
|
||||
test "$resolved" != "${tmp%/}"
|
||||
files=(test/gstack-design-detect.test.ts browse/test/chromium-profile-isolation.test.ts test/cso-git-hardening.test.ts test/plan-count-file-permission.test.ts test/spec-quality-gate-secret-sink.test.ts test/gstack-memory-ingest.test.ts)
|
||||
for file in "${files[@]}"; do test -f "$file"; done
|
||||
bun test "${files[@]}"
|
||||
|
||||
cso-windows-launcher:
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 15
|
||||
@@ -178,7 +215,7 @@ jobs:
|
||||
name: free-plan
|
||||
path: ${{ runner.temp }}
|
||||
|
||||
- uses: actions/cache@v6
|
||||
- uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
||||
with:
|
||||
path: ~/.bun/install/cache
|
||||
key: linux-bun-${{ hashFiles('bun.lock') }}
|
||||
@@ -189,7 +226,7 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: bun install --frozen-lockfile
|
||||
|
||||
- uses: actions/cache@v6
|
||||
- uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
key: linux-playwright-${{ hashFiles('bun.lock') }}
|
||||
@@ -323,15 +360,16 @@ jobs:
|
||||
# gate is merge-blocking without a separate branch-protection migration.
|
||||
free-tests:
|
||||
if: always()
|
||||
needs: [free-suite, typecheck, cso-macos-launcher, cso-windows-launcher, cso-docker-integration]
|
||||
needs: [free-suite, typecheck, cso-macos-launcher, macos-named-regressions, cso-windows-launcher, cso-docker-integration]
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Require the free suite, typecheck, and every CSO platform gate
|
||||
- name: Require the free suite, typecheck, macOS regressions, and every CSO platform gate
|
||||
env:
|
||||
FREE_SUITE_RESULT: ${{ needs.free-suite.result }}
|
||||
TYPECHECK_RESULT: ${{ needs.typecheck.result }}
|
||||
CSO_MACOS_RESULT: ${{ needs.cso-macos-launcher.result }}
|
||||
MACOS_REGRESSIONS_RESULT: ${{ needs.macos-named-regressions.result }}
|
||||
CSO_WINDOWS_RESULT: ${{ needs.cso-windows-launcher.result }}
|
||||
CSO_DOCKER_RESULT: ${{ needs.cso-docker-integration.result }}
|
||||
run: |
|
||||
@@ -339,6 +377,7 @@ jobs:
|
||||
test "$FREE_SUITE_RESULT" = success
|
||||
test "$TYPECHECK_RESULT" = success
|
||||
test "$CSO_MACOS_RESULT" = success
|
||||
test "$MACOS_REGRESSIONS_RESULT" = success
|
||||
test "$CSO_WINDOWS_RESULT" = success
|
||||
test "$CSO_DOCKER_RESULT" = success
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
@@ -49,9 +49,9 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.4.0
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
# take ~12s combined. The one history consumer (the added-lines diff)
|
||||
# fetches its exact base/head SHAs below — an exact-SHA fetch, not a
|
||||
# guessed depth, so long-lived branches and merge queues still resolve.
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.4.0
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
name: OpenSSF Scorecard
|
||||
# Supply-chain posture report (#1997): pinned actions, token permissions,
|
||||
# branch protection, dangerous workflow patterns. Results land in the
|
||||
# Security tab as code-scanning alerts and are published to
|
||||
# api.scorecard.dev for the badge.
|
||||
#
|
||||
# Publishing imposes rules this file must keep: no workflow-level env,
|
||||
# defaults or write permissions; `id-token: write` only on the job that runs
|
||||
# ossf/scorecard-action; that job uses only approved actions on a GitHub
|
||||
# hosted Ubuntu runner, with no env, defaults, containers or `run:` steps.
|
||||
# Scorecard does not support forks, so the job runs only in this repository.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
schedule:
|
||||
- cron: '41 5 * * 2'
|
||||
branch_protection_rule:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions: read-all
|
||||
|
||||
jobs:
|
||||
analysis:
|
||||
if: github.repository == 'garrytan/gstack'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
security-events: write
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4
|
||||
with:
|
||||
results_file: results.sarif
|
||||
results_format: sarif
|
||||
publish_results: true
|
||||
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: scorecard-sarif
|
||||
path: results.sarif
|
||||
retention-days: 5
|
||||
|
||||
- uses: github/codeql-action/upload-sarif@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
|
||||
with:
|
||||
sarif_file: results.sarif
|
||||
@@ -25,8 +25,8 @@ jobs:
|
||||
runs-on: ubicloud-standard-2
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.4.0
|
||||
- run: bun install --frozen-lockfile
|
||||
|
||||
@@ -21,13 +21,13 @@ jobs:
|
||||
pull-requests: read
|
||||
steps:
|
||||
- name: Checkout PR head
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Setup Bun
|
||||
uses: oven-sh/setup-bun@v2
|
||||
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.4.0
|
||||
|
||||
|
||||
@@ -61,9 +61,9 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.4.0
|
||||
|
||||
@@ -74,7 +74,7 @@ jobs:
|
||||
# bun install was 35s of a 55s job, all network. Cache keyed on the
|
||||
# lockfile; bun's install cache lives under ~/.bun/install/cache on
|
||||
# every platform.
|
||||
- uses: actions/cache@v6
|
||||
- uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
||||
with:
|
||||
path: ~/.bun/install/cache
|
||||
key: windows-bun-${{ hashFiles('bun.lock') }}
|
||||
@@ -175,7 +175,7 @@ jobs:
|
||||
# automatically, so no per-file entry is needed here.)
|
||||
- name: Upload full shard logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: windows-free-test-shard-logs
|
||||
path: |
|
||||
@@ -186,7 +186,7 @@ jobs:
|
||||
|
||||
- name: Upload flake ledger
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: flake-ledger-windows
|
||||
path: ${{ runner.temp }}/flake-ledger.jsonl
|
||||
|
||||
@@ -41,15 +41,15 @@ jobs:
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.4.0
|
||||
|
||||
# Same lockfile-keyed install cache as windows-free-tests.yml (install
|
||||
# was 45s of a 64s job, all network).
|
||||
- uses: actions/cache@v6
|
||||
- uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
||||
with:
|
||||
path: ~/.bun/install/cache
|
||||
key: windows-bun-${{ hashFiles('bun.lock') }}
|
||||
|
||||
@@ -69,6 +69,17 @@ No setup needed. Learnings are logged automatically. View them with `/learn`.
|
||||
This is the best way to contribute: fix gstack while doing your real work, in the
|
||||
project where you actually felt the pain.
|
||||
|
||||
**What CI does on a fork PR.** GitHub never gives a fork PR this repository's
|
||||
secrets, so CI runs everything that needs none: the required `free-tests` check
|
||||
(the Linux free suite, typecheck, the macOS and Windows gates), Windows Free
|
||||
Tests, Skill Docs Freshness, Workflow Lint, Version Gate, Quality gate, the
|
||||
make-pdf gate, and any path-triggered gates your change touches. E2E Evals
|
||||
still builds the CI image from your `Dockerfile.ci` without publishing it, and
|
||||
skips its paid eval jobs, which need provider API keys. That skip is expected
|
||||
and does not block your PR. It is also not a pass: a maintainer runs the paid
|
||||
evals from a branch in this repository before merging. A first-time
|
||||
contributor's runs wait for a maintainer to approve them.
|
||||
|
||||
### Session awareness
|
||||
|
||||
When you have 3+ gstack sessions open simultaneously, every question tells you which project, which branch, and what's happening. No more staring at a question thinking "wait, which window is this?" The format is consistent across all skills.
|
||||
|
||||
@@ -230,3 +230,62 @@ export function partitionCaseExclusions(keys: string[]): { runnable: string[]; e
|
||||
});
|
||||
return { runnable, excluded };
|
||||
}
|
||||
|
||||
/**
|
||||
* Codex access in the CI eval image, scoped per paid shard.
|
||||
*
|
||||
* The image installs the pinned Codex CLI off PATH (GSTACK_CI_CODEX_BIN_DIR)
|
||||
* and the eval workflows log it in under a non-default home
|
||||
* (GSTACK_CI_CODEX_HOME). A gate shard never sees either. A non-gate shard
|
||||
* whose every file exercises Codex gets the CLI on PATH and the home as
|
||||
* CODEX_HOME. A file that mixes one Codex case with other cases keeps only
|
||||
* the two variables, and that case opts in on its own session; hermetic
|
||||
* child environments drop GSTACK_* names, so its sibling sessions see no
|
||||
* Codex. Every other shard sees neither, so a skill's outside-voice probe
|
||||
* (`command -v codex`) reports not_installed exactly as it did before Codex
|
||||
* entered the image. Outside CI neither variable is set and nothing changes.
|
||||
*/
|
||||
export const CODEX_CI_ENV = { binDir: 'GSTACK_CI_CODEX_BIN_DIR', home: 'GSTACK_CI_CODEX_HOME' } as const;
|
||||
|
||||
/** Paid files whose every case exercises the real Codex CLI. */
|
||||
export const CODEX_CI_FILES: readonly string[] = [
|
||||
'test/codex-e2e.test.ts',
|
||||
'test/codex-e2e-sol-scope.test.ts',
|
||||
'test/codex-e2e-shared-libs.test.ts',
|
||||
'test/codex-e2e-recommendation-substance.test.ts',
|
||||
'test/skill-e2e-outside-voice.test.ts',
|
||||
'test/skill-e2e-outside-plan-disabled.test.ts',
|
||||
];
|
||||
|
||||
/**
|
||||
* Codex cases inside files whose other cases must not see Codex. Each listed
|
||||
* case opts in by putting $GSTACK_CI_CODEX_BIN_DIR on its own session PATH
|
||||
* (inline, so the file's other touchfile keys do not grow a helper).
|
||||
*/
|
||||
export const CODEX_CI_CASES: Readonly<Record<string, readonly string[]>> = {
|
||||
'test/skill-e2e-workflow.test.ts': ['codex-review'],
|
||||
};
|
||||
|
||||
export type CodexShardAccess = 'none' | 'case-opt-in' | 'path';
|
||||
|
||||
/** What one shard, given the test files behind its keys, may see of the CI Codex install in `tier`. */
|
||||
export function codexShardAccess(files: readonly string[], tier: string | undefined): CodexShardAccess {
|
||||
if (tier === 'gate' || files.length === 0) return 'none';
|
||||
if (files.every(file => CODEX_CI_FILES.includes(file))) return 'path';
|
||||
return files.every(file => file in CODEX_CI_CASES) ? 'case-opt-in' : 'none';
|
||||
}
|
||||
|
||||
/** Scope one shard's child environment in place; `files` are the shard keys' test files. */
|
||||
export function scopeCodexAccess(env: NodeJS.ProcessEnv, files: readonly string[]): void {
|
||||
const binDir = env[CODEX_CI_ENV.binDir];
|
||||
if (!binDir) return;
|
||||
const home = env[CODEX_CI_ENV.home];
|
||||
const access = codexShardAccess(files, env.EVALS_TIER);
|
||||
delete env.CODEX_HOME;
|
||||
if (access === 'case-opt-in') return;
|
||||
delete env[CODEX_CI_ENV.binDir];
|
||||
delete env[CODEX_CI_ENV.home];
|
||||
if (access !== 'path') return;
|
||||
env.PATH = env.PATH ? `${binDir}${path.delimiter}${env.PATH}` : binDir;
|
||||
if (home) env.CODEX_HOME = home;
|
||||
}
|
||||
@@ -105,7 +105,7 @@ import {
|
||||
export { PAID_TEST_GLOBS, isPaidTestFile };
|
||||
export { PERIODIC_CI_EXCLUDE };
|
||||
|
||||
import { shardFile, shardCaseId, shardTrial, trialShardKey, type CaseTrialPlan, caseTrialPlan, excludedCasesNamePattern, caseTestNamePattern, expandCaseShards, expandTrialShards, partitionCaseExclusions } from './lib/paid-cases';
|
||||
import { scopeCodexAccess, shardFile, shardCaseId, shardTrial, trialShardKey, type CaseTrialPlan, caseTrialPlan, excludedCasesNamePattern, caseTestNamePattern, expandCaseShards, expandTrialShards, partitionCaseExclusions } from './lib/paid-cases';
|
||||
import { retriesForFiles, trialPanelKey, sliceExecutionOrder, buildRunManifest, parseRunManifest, type SliceResult, sliceExitCode, guardTrialRecords, formatSlicePlan, formatCapacityPreflight } from './lib/paid-plan';
|
||||
import { caseFile, runCaseDiagnosis, formatPanelLine, runPaidReport } from './lib/paid-report';
|
||||
export * from './lib/paid-cases';
|
||||
@@ -953,6 +953,7 @@ export async function runPaidShard(
|
||||
const expectedCases = options.expectedCases ?? (caseId !== null ? { [files[0]!]: 1 } : undefined);
|
||||
|
||||
const baseEnv = { ...(options.env ?? process.env) };
|
||||
scopeCodexAccess(baseEnv, files.map(shardFile));
|
||||
if (options.evalDirBase) {
|
||||
baseEnv.GSTACK_EVAL_DIR = path.join(options.evalDirBase, 'shards', shardSlug(files));
|
||||
}
|
||||
|
||||
@@ -45,10 +45,10 @@ function collectPins(): Pin[] {
|
||||
|
||||
const dockerfile = fs.readFileSync(
|
||||
path.join(ROOT, '.github', 'docker', 'Dockerfile.ci'), 'utf-8');
|
||||
const dockerPin = dockerfile.match(/bash -s ["']?bun-v([\w.]+)["']?/);
|
||||
const dockerPin = dockerfile.match(/^ARG BUN_VERSION=["']?([\w.]+)["']?$/m);
|
||||
pins.push({
|
||||
surface: 'Dockerfile.ci',
|
||||
version: dockerPin ? dockerPin[1] : '<no bun-vX.Y.Z positional arg>',
|
||||
version: dockerPin ? dockerPin[1] : '<no ARG BUN_VERSION=X.Y.Z>',
|
||||
});
|
||||
|
||||
const gitlab = fs.readFileSync(path.join(ROOT, '.gitlab-ci.yml'), 'utf-8');
|
||||
|
||||
@@ -14,6 +14,10 @@
|
||||
* This tripwire fails the free suite when any globally-installed npm package
|
||||
* in Dockerfile.ci lacks an exact `@X.Y.Z` pin. Bumps are deliberate: edit
|
||||
* the pin in a PR and run the PTY gate against the new TUI before merging.
|
||||
*
|
||||
* Bun comes from its release archive, verified by SHA-256 for every
|
||||
* architecture the image builds (#1706): nothing in the image is piped from
|
||||
* curl into a shell.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
@@ -64,4 +68,115 @@ describe('ci image provider-CLI pins', () => {
|
||||
+ 'harness three times before this tripwire existed.',
|
||||
).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('the image installs the codex CLI globally so the Codex evals execute in CI', () => {
|
||||
expect(
|
||||
specs.some((s) => s.startsWith('@openai/codex@')),
|
||||
`expected a pinned @openai/codex install in ${path.relative(ROOT, DOCKERFILE)}; found: ${specs.join(', ')}`,
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
/** Dockerfile source with `\`-continued lines joined into logical lines. */
|
||||
function logicalLines(source: string): string[] {
|
||||
return source.replace(/\\\n/g, ' ').split('\n');
|
||||
}
|
||||
|
||||
/** Logical lines that pipe a download straight into a shell. */
|
||||
export function curlPipedToShell(source: string): string[] {
|
||||
return logicalLines(source).filter((line) => !/^\s*#/.test(line) &&
|
||||
/\b(?:curl|wget)\b/.test(line) && /\|\s*(?:sudo\s+)?(?:ba|z|da)?sh\b/.test(line));
|
||||
}
|
||||
|
||||
/** `<arch>) bun_target=...; bun_sha256=<hex>` arms of the Bun install `case`. */
|
||||
export function bunArchiveArms(source: string): Map<string, string> {
|
||||
const arms = new Map<string, string>();
|
||||
for (const m of source.matchAll(/^\s*([\w|]+)\)\s*bun_target=[\w-]+;\s*bun_sha256=([0-9a-f]+)\s*;;/gm)) {
|
||||
for (const arch of m[1].split('|')) arms.set(arch, m[2]);
|
||||
}
|
||||
return arms;
|
||||
}
|
||||
|
||||
/** Docker architectures each build of Dockerfile.ci produces (no `platforms:` = the amd64 runner's). */
|
||||
export function builtArchitectures(workflowsDir: string): Set<string> {
|
||||
const archs = new Set<string>();
|
||||
for (const name of fs.readdirSync(workflowsDir)) {
|
||||
if (!/\.ya?ml$/.test(name)) continue;
|
||||
const workflow = Bun.YAML.parse(fs.readFileSync(path.join(workflowsDir, name), 'utf-8')) as any;
|
||||
for (const job of Object.values<any>(workflow?.jobs ?? {})) {
|
||||
for (const step of job?.steps ?? []) {
|
||||
if (!String(step?.uses ?? '').startsWith('docker/build-push-action@')) continue;
|
||||
if (step.with?.file !== '.github/docker/Dockerfile.ci') continue;
|
||||
const platforms = String(step.with?.platforms ?? 'linux/amd64').split(/[\s,]+/).filter(Boolean);
|
||||
for (const platform of platforms) archs.add(platform.replace(/^linux\//, ''));
|
||||
}
|
||||
}
|
||||
}
|
||||
return archs;
|
||||
}
|
||||
|
||||
describe('ci image Bun install (#1706)', () => {
|
||||
const source = fs.readFileSync(DOCKERFILE, 'utf-8');
|
||||
const bunRun = logicalLines(source).find((line) => line.includes('oven-sh/bun/releases/download/bun-v${BUN_VERSION}/')) ?? '';
|
||||
|
||||
test('nothing is piped from a download into a shell', () => {
|
||||
expect(curlPipedToShell(source), 'download the release archive and verify its SHA-256 instead').toEqual([]);
|
||||
});
|
||||
|
||||
test('the tripwire recognizes the old installer pipe and leaves non-shell pipes alone', () => {
|
||||
expect(curlPipedToShell('RUN curl -fsSL https://bun.sh/install \\\n | bash -s "bun-v1.4.0"\n')).toHaveLength(1);
|
||||
expect(curlPipedToShell('RUN curl -fsSL https://example.test/x.sh | sh\n')).toHaveLength(1);
|
||||
expect(curlPipedToShell('RUN curl -fsSL https://example.test/key.gpg | gpg --dearmor -o /k.gpg\n')).toEqual([]);
|
||||
});
|
||||
|
||||
test('every architecture the image builds has a SHA-256-verified Bun archive', () => {
|
||||
const built = builtArchitectures(path.join(ROOT, '.github', 'workflows'));
|
||||
expect([...built], 'scan found no Dockerfile.ci build').toContain('amd64');
|
||||
const arms = bunArchiveArms(source);
|
||||
for (const arch of built) {
|
||||
expect(arms.get(arch), `no verified Bun archive for ${arch}`).toMatch(/^[0-9a-f]{64}$/);
|
||||
}
|
||||
});
|
||||
|
||||
test('the archive is checked before it is unpacked, and unknown architectures fail closed', () => {
|
||||
expect(bunRun, 'Bun must come from the GitHub release archive').not.toBe('');
|
||||
const download = bunRun.indexOf('releases/download/');
|
||||
const verify = bunRun.indexOf('sha256sum -c');
|
||||
const unpack = bunRun.indexOf('unzip');
|
||||
expect(verify).toBeGreaterThan(download);
|
||||
expect(unpack).toBeGreaterThan(verify);
|
||||
expect(bunRun).toMatch(/\*\)(?:(?!;;).)*exit 1\s*;;/);
|
||||
expect(bunRun).toContain('test "$(bun --version)" = "${BUN_VERSION}"');
|
||||
});
|
||||
});
|
||||
|
||||
describe('ci image codex authentication', () => {
|
||||
const workflowsDir = path.join(ROOT, '.github', 'workflows');
|
||||
const lanes: Array<{ lane: string; steps: any[] }> = [];
|
||||
for (const name of fs.readdirSync(workflowsDir).sort()) {
|
||||
if (!/\.ya?ml$/.test(name)) continue;
|
||||
const workflow = Bun.YAML.parse(fs.readFileSync(path.join(workflowsDir, name), 'utf-8')) as any;
|
||||
for (const [id, job] of Object.entries<any>(workflow?.jobs ?? {})) {
|
||||
const steps: any[] = job?.steps ?? [];
|
||||
if (job?.container && steps.some((step) => /test-paid-shards\.ts .*--slice/.test(String(step?.run ?? '')))) {
|
||||
lanes.push({ lane: `${name}:${id}`, steps });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
test('every paid lane in the image logs the codex CLI in from the secret before it runs', () => {
|
||||
expect(lanes.length, 'scan found no paid lanes').toBeGreaterThanOrEqual(4);
|
||||
for (const { lane, steps } of lanes) {
|
||||
const login = steps.findIndex((step) => /codex"? login --with-api-key/.test(String(step?.run ?? '')));
|
||||
const run = steps.findIndex((step) => /test-paid-shards\.ts .*--slice/.test(String(step?.run ?? '')));
|
||||
expect(login, `${lane}: no codex login step`).toBeGreaterThanOrEqual(0);
|
||||
expect(login, `${lane}: codex login must precede the paid run`).toBeLessThan(run);
|
||||
const step = steps[login];
|
||||
expect(step.env?.OPENAI_API_KEY, `${lane}: key must arrive through env`).toBe('${{ secrets.OPENAI_API_KEY }}');
|
||||
expect(String(step.run), `${lane}: never inline a secret into run`).not.toContain('secrets.');
|
||||
expect(String(step.run)).toMatch(/codex"? login status/);
|
||||
expect(String(step.if ?? ''), `${lane}: same-repo runs only`)
|
||||
.toBe("github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository");
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* The CI image's Codex CLI and credential reach only the Codex eval files.
|
||||
*
|
||||
* Every other paid case must see Codex exactly as it did before Codex entered
|
||||
* the image: not on PATH and not logged in, so a skill's outside-voice probe
|
||||
* (`command -v codex`) reports not_installed and gate verdicts and slice
|
||||
* budgets do not move. This pins the image (Codex off PATH), the workflows
|
||||
* (login under a non-default home, never exported as CODEX_HOME), the
|
||||
* per-shard scoping helper, and the shard runner's use of it.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { CODEX_CI_CASES, CODEX_CI_ENV, CODEX_CI_FILES, buildRunManifest, codexShardAccess, runPaidShards, scopeCodexAccess, shardFile } from '../scripts/test-paid-shards';
|
||||
import { buildHermeticEnv } from './helpers/hermetic-env';
|
||||
import { E2E_TIERS, E2E_TOUCHFILES } from './helpers/touchfiles';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const BIN = '/opt/codex/bin';
|
||||
const HOME = '/home/runner/.gstack-ci-codex';
|
||||
const ciEnv = (tier: string): NodeJS.ProcessEnv => ({ PATH: '/usr/local/bin:/usr/bin:/bin', EVALS_TIER: tier, [CODEX_CI_ENV.binDir]: BIN, [CODEX_CI_ENV.home]: HOME });
|
||||
const sees = (env: NodeJS.ProcessEnv) => ({
|
||||
onPath: (env.PATH ?? '').split(path.delimiter).includes(BIN),
|
||||
home: env.CODEX_HOME ?? null,
|
||||
leaked: Object.keys(env).filter(name => name.startsWith('GSTACK_CI_CODEX')),
|
||||
});
|
||||
const planned = (tier: 'gate' | 'periodic') => buildRunManifest({ tier, profile: 'full', sliceCount: 1, evalsAll: true, env: { EVALS_ALL: '1' } })
|
||||
.entries.filter(entry => entry.status === 'planned').map(entry => entry.file);
|
||||
|
||||
/** The paid test files a registered E2E case runs in. */
|
||||
const caseFiles = (id: string) => (E2E_TOUCHFILES[id] ?? []).filter(dep => /^test\/.*\.test\.ts$/.test(dep));
|
||||
|
||||
describe('codex CI access scope', () => {
|
||||
test('the Codex files and opted-in cases hold the approved periodic Codex cases and no gate case', () => {
|
||||
for (const file of [...CODEX_CI_FILES, ...Object.keys(CODEX_CI_CASES)]) expect(fs.existsSync(path.join(ROOT, file)), file).toBe(true);
|
||||
const reaches = (id: string) => caseFiles(id).some(file => CODEX_CI_FILES.includes(file) || CODEX_CI_CASES[file]?.includes(id));
|
||||
for (const id of ['codex-review', 'codex-review-findings', 'codex-discover-skill', 'outside-voice-codex-to-claude-code',
|
||||
'outside-voice-claude-code-to-codex', 'outside-plan-disabled-no-fallback', 'codex-sol-scope-termination']) {
|
||||
expect(E2E_TIERS[id], id).toBe('periodic');
|
||||
expect(reaches(id), `${id} cannot reach the CI Codex`).toBe(true);
|
||||
}
|
||||
for (const ids of Object.values(CODEX_CI_CASES)) for (const id of ids) expect(E2E_TIERS[id], id).toBe('periodic');
|
||||
const gateInCodexFiles = Object.keys(E2E_TIERS)
|
||||
.filter(id => E2E_TIERS[id] === 'gate' && caseFiles(id).some(file => CODEX_CI_FILES.includes(file)));
|
||||
expect(gateInCodexFiles, 'a gate case would run with an authenticated Codex').toEqual([]);
|
||||
});
|
||||
|
||||
test('only the opted-in case of a mixed file asks for Codex, and hermetic sessions drop the CI variables', () => {
|
||||
for (const [file, ids] of Object.entries(CODEX_CI_CASES)) {
|
||||
const source = fs.readFileSync(path.join(ROOT, file), 'utf8');
|
||||
const calls = [...source.matchAll(/process\.env\.GSTACK_CI_CODEX_BIN_DIR\b/g)].map(match => match.index!);
|
||||
expect(calls, file).toHaveLength(ids.length);
|
||||
for (const [i, id] of ids.entries()) {
|
||||
const start = source.indexOf(`IfSelected('${id}'`);
|
||||
const next = source.slice(start + 1).search(/IfSelected\('/);
|
||||
const end = next < 0 ? source.length : start + 1 + next;
|
||||
expect(calls[i]! > start && calls[i]! < end, `${file}: CI Codex read outside the ${id} case`).toBe(true);
|
||||
}
|
||||
}
|
||||
const child = buildHermeticEnv({ ...ciEnv('periodic'), HOME: '/home/runner' }, {});
|
||||
expect(Object.keys(child).filter(name => name.startsWith('GSTACK_CI_CODEX'))).toEqual([]);
|
||||
expect((child.PATH ?? '').split(path.delimiter)).not.toContain(BIN);
|
||||
});
|
||||
|
||||
test('no planned gate shard can see the CI Codex CLI or its credential', () => {
|
||||
const gate = planned('gate');
|
||||
expect(gate.length).toBeGreaterThan(20);
|
||||
for (const key of gate) {
|
||||
const env = ciEnv('gate');
|
||||
scopeCodexAccess(env, [shardFile(key)]);
|
||||
expect(sees(env), key).toEqual({ onPath: false, home: null, leaked: [] });
|
||||
expect(codexShardAccess([shardFile(key)], 'periodic') === 'path', `${key}: a gate shard is a Codex file`).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('in the periodic census only the Codex files see Codex on PATH', () => {
|
||||
const periodic = planned('periodic');
|
||||
const access = (key: string) => {
|
||||
const env = ciEnv('periodic');
|
||||
scopeCodexAccess(env, [shardFile(key)]);
|
||||
return { key, ...sees(env) };
|
||||
};
|
||||
const results = periodic.map(access);
|
||||
const granted = results.filter(result => result.onPath);
|
||||
expect([...new Set(granted.map(result => shardFile(result.key)))].sort()).toEqual([...CODEX_CI_FILES].sort());
|
||||
for (const result of granted) expect(result.home, result.key).toBe(HOME);
|
||||
for (const result of results.filter(result => !result.onPath)) {
|
||||
expect(result.home, result.key).toBeNull();
|
||||
expect(result.leaked.length === 0 || shardFile(result.key) in CODEX_CI_CASES, result.key).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('a mixed shard is denied, case and trial keys follow their file, and local runs are untouched', () => {
|
||||
expect(codexShardAccess(['test/codex-e2e.test.ts', 'test/skill-e2e-plan.test.ts'], 'periodic')).toBe('none');
|
||||
expect(codexShardAccess([shardFile('test/codex-e2e.test.ts#codex-review~t2')], 'periodic')).toBe('path');
|
||||
expect(codexShardAccess(['test/codex-e2e.test.ts'], 'gate')).toBe('none');
|
||||
expect(codexShardAccess(['test/skill-e2e-workflow.test.ts'], 'gate')).toBe('none');
|
||||
expect(codexShardAccess(['test/skill-e2e-workflow.test.ts'], 'periodic')).toBe('case-opt-in');
|
||||
expect(codexShardAccess([], 'periodic')).toBe('none');
|
||||
const ci = { ...ciEnv('periodic'), CODEX_HOME: '/somewhere/else' };
|
||||
scopeCodexAccess(ci, [shardFile('test/skill-e2e-plan.test.ts#codex-offered-eng-review')]);
|
||||
expect(sees(ci)).toEqual({ onPath: false, home: null, leaked: [] });
|
||||
const optIn = { ...ciEnv('periodic'), CODEX_HOME: '/somewhere/else' };
|
||||
scopeCodexAccess(optIn, ['test/skill-e2e-workflow.test.ts']);
|
||||
expect(sees(optIn)).toEqual({ onPath: false, home: null, leaked: [CODEX_CI_ENV.binDir, CODEX_CI_ENV.home] });
|
||||
const local: NodeJS.ProcessEnv = { PATH: '/usr/bin', CODEX_HOME: '/home/dev/.codex' };
|
||||
scopeCodexAccess(local, ['test/skill-e2e-plan.test.ts']);
|
||||
expect(local).toEqual({ PATH: '/usr/bin', CODEX_HOME: '/home/dev/.codex' });
|
||||
});
|
||||
|
||||
test.skipIf(process.platform === 'win32')('the paid shard runner gives each child the scoped environment', async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codex-ci-access-'));
|
||||
try {
|
||||
const bin = path.join(dir, 'bin'), home = path.join(dir, 'home');
|
||||
fs.mkdirSync(bin); fs.mkdirSync(home);
|
||||
fs.writeFileSync(path.join(bin, 'codex'), '#!/bin/sh\nexit 0\n', { mode: 0o755 });
|
||||
const shards = [['test/codex-e2e.test.ts'], ['test/skill-e2e-workflow.test.ts'],
|
||||
['test/skill-e2e-plan.test.ts#codex-offered-eng-review'], ['test/skill-e2e-outside-voice.test.ts', 'test/skill-e2e-review.test.ts']];
|
||||
const probe = 'p=$(command -v codex || echo none); printf "%s|%s|%s" "$p" "${CODEX_HOME:-}" "$(env | grep -c "^GSTACK_CI_CODEX" || true)"';
|
||||
const run = async (tier: string) => {
|
||||
const out = fs.mkdtempSync(path.join(dir, `${tier}-`));
|
||||
await runPaidShards(shards, {
|
||||
jobs: 1, timeoutMs: 30_000, log: () => {}, logDir: dir,
|
||||
env: { PATH: '/usr/bin:/bin', HOME: dir, EVALS_TIER: tier, GSTACK_CLAUDE_CLI_VERSION: 'test', [CODEX_CI_ENV.binDir]: bin, [CODEX_CI_ENV.home]: home },
|
||||
commandFor: files => ({ command: '/bin/sh', args: ['-c', `${probe} > "${path.join(out, String(shards.indexOf(files)))}"`] }),
|
||||
});
|
||||
return shards.map((_, index) => fs.readFileSync(path.join(out, String(index)), 'utf8'));
|
||||
};
|
||||
expect(await run('periodic')).toEqual([`${path.join(bin, 'codex')}|${home}|0`, 'none||2', 'none||0', 'none||0']);
|
||||
expect(await run('gate')).toEqual(['none||0', 'none||0', 'none||0', 'none||0']);
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
describe('codex CI access in the image and workflows', () => {
|
||||
test('the image installs Codex off PATH at the directory it advertises', () => {
|
||||
const dockerfile = fs.readFileSync(path.join(ROOT, '.github/docker/Dockerfile.ci'), 'utf8');
|
||||
expect(dockerfile).toMatch(/^ENV GSTACK_CI_CODEX_BIN_DIR=\/opt\/codex\/bin$/m);
|
||||
expect(dockerfile).toMatch(/npm i -g --prefix=\/opt\/codex @openai\/codex@\d+\.\d+\.\d+/);
|
||||
expect(dockerfile).toContain('! command -v codex');
|
||||
expect(dockerfile).not.toMatch(/^ENV PATH=.*\/opt\/codex/m);
|
||||
});
|
||||
|
||||
test('the login step never makes the credential the default Codex home', () => {
|
||||
const steps: string[] = [];
|
||||
for (const name of ['evals.yml', 'evals-periodic.yml', 'evals-marathon.yml']) {
|
||||
const workflow = Bun.YAML.parse(fs.readFileSync(path.join(ROOT, '.github/workflows', name), 'utf8')) as any;
|
||||
for (const job of Object.values<any>(workflow.jobs)) {
|
||||
for (const step of job.steps ?? []) if (/login --with-api-key/.test(String(step.run ?? ''))) steps.push(String(step.run));
|
||||
}
|
||||
}
|
||||
expect(steps.length).toBeGreaterThanOrEqual(4);
|
||||
for (const run of steps) {
|
||||
expect(run).toContain('echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV"');
|
||||
expect(run).not.toMatch(/echo "CODEX_HOME=/);
|
||||
expect(run).not.toContain('$HOME/.codex"');
|
||||
expect(run).toContain('"${GSTACK_CI_CODEX_BIN_DIR:?}/codex"');
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -318,7 +318,7 @@ describe('CSO runtime staging gates', () => {
|
||||
expect(gate['continue-on-error']).not.toBe(true);
|
||||
const required = workflow.jobs['free-tests'];
|
||||
expect(required.if).toBe('always()');
|
||||
expect(required.needs).toEqual(['free-suite', 'typecheck', 'cso-macos-launcher', 'cso-windows-launcher', 'cso-docker-integration']);
|
||||
expect(required.needs).toEqual(['free-suite', 'typecheck', 'cso-macos-launcher', 'macos-named-regressions', 'cso-windows-launcher', 'cso-docker-integration']);
|
||||
expect(required.steps[0].run).toContain('test "$CSO_DOCKER_RESULT" = success');
|
||||
for (const current of Object.values(workflow.jobs) as any[]) for (const step of current.steps) {
|
||||
if (step.uses?.startsWith('oven-sh/setup-bun')) expect(step.uses).toBe('oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6');
|
||||
|
||||
@@ -192,7 +192,7 @@ test('current detach supervision covers the live-census floor', () => {
|
||||
expect(floorFor('gate')).toBe(21_725);
|
||||
expect(gateTimeout).toBe(49_320);
|
||||
expect(gateTimeout).toBeGreaterThanOrEqual(floorFor('gate'));
|
||||
expect(floorFor('periodic')).toBe(33_821);
|
||||
expect(floorFor('periodic')).toBe(35_711);
|
||||
expect(periodicTimeout).toBeGreaterThanOrEqual(floorFor('periodic'));
|
||||
});
|
||||
|
||||
|
||||
@@ -15,26 +15,6 @@
|
||||
* the re-entry mechanism.
|
||||
*/
|
||||
export const PERIODIC_CI_EXCLUDE: Record<string, { reason: string; tracking: string }> = {
|
||||
'test/codex-e2e.test.ts': {
|
||||
reason: 'the codex CLI is not installed in the CI image (Dockerfile.ci ships only claude-code); every case self-skips',
|
||||
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
||||
},
|
||||
'test/codex-e2e-sol-scope.test.ts': {
|
||||
reason: 'the codex CLI is not installed in the CI image (Dockerfile.ci ships only claude-code); every case self-skips',
|
||||
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
||||
},
|
||||
'test/codex-e2e-shared-libs.test.ts': {
|
||||
reason: 'the codex CLI is not installed in the CI image (Dockerfile.ci ships only claude-code); every case self-skips',
|
||||
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
||||
},
|
||||
'test/codex-e2e-recommendation-substance.test.ts': {
|
||||
reason: 'the codex CLI is not installed in the CI image (Dockerfile.ci ships only claude-code); every case self-skips',
|
||||
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
||||
},
|
||||
'test/skill-e2e-outside-voice.test.ts': {
|
||||
reason: 'needs both the claude and codex CLIs; codex is not in the CI image, so every case self-skips',
|
||||
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
||||
},
|
||||
'test/skill-e2e-aside.test.ts': {
|
||||
reason: 'needs macOS with the Aside app open (asideAvailable()); CI runners are Linux, so every case self-skips',
|
||||
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
||||
|
||||
@@ -230,8 +230,8 @@ test('detached PR fallback and release commands cover their actual default worke
|
||||
)) / 1000 * 1.05));
|
||||
}
|
||||
const detachedReleaseWall = Number(scripts['eval:bg:release'].match(/--timeout (\d+)/)?.[1]) * 1000;
|
||||
expect(releaseFloors).toEqual([21_725, 33_821]);
|
||||
expect(releaseFloors.reduce((total, floor) => total + floor, 0)).toBe(55_546);
|
||||
expect(releaseFloors).toEqual([21_725, 35_711]);
|
||||
expect(releaseFloors.reduce((total, floor) => total + floor, 0)).toBe(57_436);
|
||||
expect(detachedReleaseWall).toBe(116_700_000);
|
||||
expect(detachedReleaseWall).toBeGreaterThanOrEqual(releaseWall + 120_000);
|
||||
});
|
||||
|
||||
@@ -451,8 +451,13 @@ describeIfSelected('Codex skill E2E', ['codex-review'], () => {
|
||||
});
|
||||
|
||||
testConcurrentIfSelected('codex-review', async () => {
|
||||
// Check codex is available — skip if not installed
|
||||
const codexCheck = spawnSync('which', ['codex'], { stdio: 'pipe', timeout: 3000 });
|
||||
// Check codex is available — skip if not installed. In CI only this case
|
||||
// opts in to the image's off-PATH Codex (scripts/lib/paid-cases.ts scopeCodexAccess).
|
||||
const ciCodexBin = process.env.GSTACK_CI_CODEX_BIN_DIR, ciCodexHome = process.env.GSTACK_CI_CODEX_HOME;
|
||||
const codexEnv: Record<string, string> = ciCodexBin
|
||||
? { PATH: `${ciCodexBin}${path.delimiter}${process.env.PATH ?? ''}`, ...(ciCodexHome ? { CODEX_HOME: ciCodexHome } : {}) }
|
||||
: {};
|
||||
const codexCheck = spawnSync('which', ['codex'], { stdio: 'pipe', timeout: 3000, env: { ...process.env, ...codexEnv } });
|
||||
if (codexCheck.status !== 0) {
|
||||
console.warn('codex CLI not installed — skipping E2E test');
|
||||
return;
|
||||
@@ -469,6 +474,7 @@ Write the full output (including the GATE verdict) to ${codexDir}/codex-output.m
|
||||
testName: 'codex-review',
|
||||
runId,
|
||||
model: resolveEvalModel('capture'),
|
||||
env: codexEnv,
|
||||
});
|
||||
|
||||
logCost('/codex review', result);
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
/**
|
||||
* Workflow action pins (supply chain, free).
|
||||
*
|
||||
* A tag (`@v7`) or branch ref is mutable: whoever controls the action's
|
||||
* repository can repoint it, and the next CI run executes the new code with
|
||||
* this repository's token and secrets. Every remote `uses:` in a workflow or
|
||||
* composite action must name a full 40-character commit SHA (Dependabot's
|
||||
* `github-actions` ecosystem keeps those current, with the tag in a trailing
|
||||
* comment), and every `docker://` image must name a sha256 digest. Local
|
||||
* actions (`./...`) are part of this checkout and need no pin.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
const ROOT = path.resolve(__dirname, '..');
|
||||
const GITHUB_DIR = path.join(ROOT, '.github');
|
||||
|
||||
/** Why a `uses:` reference is not immutable, or null when it is. */
|
||||
export function pinProblem(uses: string): string | null {
|
||||
if (uses.startsWith('./')) return null;
|
||||
if (uses.startsWith('docker://')) {
|
||||
return /@sha256:[0-9a-f]{64}$/.test(uses) ? null : 'docker image is not pinned to a sha256 digest';
|
||||
}
|
||||
const at = uses.lastIndexOf('@');
|
||||
if (at <= 0) return 'remote action has no ref';
|
||||
if (!/^[\w.-]+\/[\w.-]+(?:\/[\w./-]+)?$/.test(uses.slice(0, at))) return 'unrecognized action reference';
|
||||
return /^[0-9a-f]{40}$/.test(uses.slice(at + 1)) ? null : 'ref is not a full 40-character commit SHA';
|
||||
}
|
||||
|
||||
/** Every `uses:` in a workflow (job steps and reusable-workflow jobs) or composite action. */
|
||||
export function usesReferences(document: any): string[] {
|
||||
const refs: string[] = [];
|
||||
const steps = (list: any) => {
|
||||
for (const step of Array.isArray(list) ? list : []) if (typeof step?.uses === 'string') refs.push(step.uses);
|
||||
};
|
||||
for (const job of Object.values<any>(document?.jobs ?? {})) {
|
||||
if (typeof job?.uses === 'string') refs.push(job.uses);
|
||||
steps(job?.steps);
|
||||
}
|
||||
steps(document?.runs?.steps);
|
||||
return refs;
|
||||
}
|
||||
|
||||
function automationFiles(): string[] {
|
||||
const files: string[] = [];
|
||||
const workflows = path.join(GITHUB_DIR, 'workflows');
|
||||
for (const name of fs.readdirSync(workflows).sort()) {
|
||||
if (/\.ya?ml$/.test(name)) files.push(path.join(workflows, name));
|
||||
}
|
||||
const actions = path.join(GITHUB_DIR, 'actions');
|
||||
for (const name of fs.existsSync(actions) ? fs.readdirSync(actions).sort() : []) {
|
||||
for (const file of ['action.yml', 'action.yaml']) {
|
||||
if (fs.existsSync(path.join(actions, name, file))) files.push(path.join(actions, name, file));
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
describe('workflow action pins', () => {
|
||||
test('the classifier accepts immutable references and rejects mutable ones', () => {
|
||||
for (const ok of [
|
||||
'./.github/actions/restore-deps',
|
||||
'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1',
|
||||
'github/codeql-action/upload-sarif@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2',
|
||||
'google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@f4cfcc01edc9c8b756a9b873b7a623ca674da51e',
|
||||
`docker://alpine@sha256:${'a'.repeat(64)}`,
|
||||
]) expect(pinProblem(ok), ok).toBeNull();
|
||||
for (const bad of [
|
||||
'actions/checkout@v7',
|
||||
'actions/checkout@main',
|
||||
'actions/checkout@3d3c42e',
|
||||
'actions/checkout',
|
||||
'google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@v2.3.8',
|
||||
'docker://alpine:3.20',
|
||||
'docker://alpine',
|
||||
]) expect(pinProblem(bad), bad).not.toBeNull();
|
||||
});
|
||||
|
||||
test('every remote action, reusable workflow and docker image is pinned immutably', () => {
|
||||
const files = automationFiles();
|
||||
const problems: string[] = [];
|
||||
let scanned = 0;
|
||||
for (const file of files) {
|
||||
const source = fs.readFileSync(file, 'utf-8');
|
||||
const refs = usesReferences(Bun.YAML.parse(source));
|
||||
// A parse that misses `uses:` lines would pass vacuously; count them.
|
||||
const lines = source.split('\n').filter((line) => /^\s*(?:-\s+)?uses:\s*\S/.test(line)).length;
|
||||
expect(refs.length, `${path.relative(ROOT, file)}: parsed ${refs.length} of ${lines} uses: lines`).toBe(lines);
|
||||
scanned += refs.length;
|
||||
for (const ref of refs) {
|
||||
const problem = pinProblem(ref);
|
||||
if (problem) problems.push(`${path.relative(ROOT, file)}: ${ref} (${problem})`);
|
||||
}
|
||||
}
|
||||
expect(files.length).toBeGreaterThan(10);
|
||||
expect(scanned).toBeGreaterThan(100);
|
||||
expect(problems, 'pin each action to a full commit SHA with the tag in a trailing comment').toEqual([]);
|
||||
});
|
||||
|
||||
test('Dependabot keeps the pinned actions current', () => {
|
||||
const config = Bun.YAML.parse(fs.readFileSync(path.join(GITHUB_DIR, 'dependabot.yml'), 'utf-8')) as any;
|
||||
expect((config.updates ?? []).map((update: any) => update['package-ecosystem'])).toContain('github-actions');
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user