Merge remote-tracking branch 'origin/main' into cso-evaluator-support

# Conflicts:
#	CHANGELOG.md
#	VERSION
#	agents-digest/gstack-AGENTS.md
#	package.json
#	test/cso-docker-integration.test.ts
This commit is contained in:
garrytan committed 2026-10-05 01:33:04 +00:00
commit 2d79967403
288 files changed
+13384 -10107

No files matched your search

+20 -1
View File
@@ -1,5 +1,6 @@
# gstack CI eval runner — pre-baked toolchain + deps
# Rebuild weekly via ci-image.yml, on Dockerfile changes, or on lockfile changes
# Built and pushed by the eval workflows' build-image job whenever its content-hash
# tag (Dockerfile.ci, bun.lock, patches/**) is missing from the registry
FROM ubuntu:24.04
ENV DEBIAN_FRONTEND=noninteractive
@@ -39,6 +40,24 @@ RUN for i in 1 2 3; do \
done \
&& rm -rf /var/lib/apt/lists/*
# Git >= 2.44 for --no-lazy-fetch. bin/gstack-safe-git puts it on every
# /deslop-shared-libs Git read, and noble's git 2.43 exits 129 on it, so every
# paid shared-libs case ran the unsupported-Git fallback (local history
# unavailable) instead of the path it was written for. The Ubuntu Git
# Maintainers PPA ships current Git for noble; the build fails if the flag is
# missing so the image can never silently fall back again.
RUN curl --retry 5 --retry-delay 5 --retry-connrefused -fsSL \
"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xE1DD270288B4E6030699E45FA1715D88E1DF1F24" \
| gpg --dearmor -o /usr/share/keyrings/git-core-ppa.gpg \
&& echo "deb [signed-by=/usr/share/keyrings/git-core-ppa.gpg] https://ppa.launchpadcontent.net/git-core/ppa/ubuntu noble main" \
> /etc/apt/sources.list.d/git-core-ppa.list \
&& for i in 1 2 3; do \
apt-get update && apt-get install -y --no-install-recommends git && break || \
(echo "apt retry $i/3 after failure"; sleep 10); \
done \
&& rm -rf /var/lib/apt/lists/* \
&& git --no-lazy-fetch --version
RUN python3 -m venv /tmp/gstack-ci-venv \
&& /tmp/gstack-ci-venv/bin/python -m pip --version \
&& rm -rf /tmp/gstack-ci-venv
+1 -1
View File
@@ -20,7 +20,7 @@ permissions:
jobs:
actionlint:
runs-on: ubicloud-standard-2
runs-on: ubuntu-24.04
timeout-minutes: 5
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
-85
View File
@@ -1,85 +0,0 @@
name: Build CI Image
on:
# Weekly self-heal (Monday 4am UTC) — deliberately 2h BEFORE
# evals-periodic's 6am cron so the weekly eval run finds the image instead
# of racing a half-pushed tag. With the claude CLI pinned in Dockerfile.ci
# (v1.77+), this cron no longer pulls CLI updates: when the content-hash
# tag already exists it's a ~30s no-op, and it only rebuilds if the tag
# was somehow lost. CLI bumps happen by editing the Dockerfile pin in a PR
# that runs the PTY gate against the new TUI.
schedule:
- cron: '0 4 * * 1'
# Rebuild on Dockerfile or lockfile changes. package.json is deliberately
# NOT a trigger: the tag hash below excludes it (its version field bumps on
# every ship), so a package.json-triggered run rebuilt and re-pushed the
# IDENTICAL tag on every merge to main (~2m26s each for zero content change).
push:
branches: [main]
paths:
- '.github/docker/Dockerfile.ci'
- 'bun.lock'
- 'patches/**'
# Manual trigger
workflow_dispatch:
# Two rapid main pushes must not race pushing the same :latest/:buildcache
# tags; newest wins.
concurrency:
group: ci-image-${{ github.ref }}
cancel-in-progress: true
jobs:
build:
runs-on: ubicloud-standard-8
timeout-minutes: 30
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
# Copy lockfile + package.json into Docker build context
- run: cp package.json bun.lock .github/docker/ && cp -R patches .github/docker/patches
# Same content-hash tag expression as evals.yml / evals-periodic.yml
# (byte-identity pinned by test/ci-image-tag-binding.test.ts). This is
# the tag the eval matrix looks up first — without pushing it here, the
# weekly/main prebuild never warms the cache that matters.
- id: meta
run: echo "tag=ghcr.io/${{ github.repository }}/ci:${{ hashFiles('.github/docker/Dockerfile.ci', 'bun.lock', 'patches/**') }}" >> "$GITHUB_OUTPUT"
- uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
# Skip the ~2.5min build when the content-hash tag already exists
# (mirrors evals.yml's check). The weekly cron still refreshes :latest
# via a full run when the tag is genuinely new.
- name: Check if image exists
id: check
run: |
if docker manifest inspect ${{ steps.meta.outputs.tag }} > /dev/null 2>&1; then
echo "exists=true" >> "$GITHUB_OUTPUT"
else
echo "exists=false" >> "$GITHUB_OUTPUT"
fi
# Registry cache export needs a docker-container builder — the default
# `docker` driver hard-errors on cache-to.
- if: steps.check.outputs.exists == 'false'
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
- if: steps.check.outputs.exists == 'false'
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
with:
context: .github/docker
file: .github/docker/Dockerfile.ci
push: true
cache-from: type=registry,ref=ghcr.io/${{ github.repository }}/ci:buildcache
cache-to: type=registry,ref=ghcr.io/${{ github.repository }}/ci:buildcache,mode=max
tags: |
${{ steps.meta.outputs.tag }}
ghcr.io/${{ github.repository }}/ci:latest
ghcr.io/${{ github.repository }}/ci:${{ github.sha }}
+1 -1
View File
@@ -17,7 +17,7 @@ permissions:
jobs:
dependency-review:
runs-on: ubicloud-standard-2
runs-on: ubuntu-24.04
timeout-minutes: 5
permissions:
contents: read
+12 -6
View File
@@ -12,8 +12,10 @@ on:
- cron: '0 12 * * 6' # Saturday 12:00 UTC, clear of the Monday periodic census
workflow_dispatch:
# One group per ref and event: a branch dispatch never cancels main's
# scheduled marathon, and a manual main dispatch never cancels the schedule.
concurrency:
group: evals-marathon
group: evals-marathon-${{ github.ref }}-${{ github.event_name }}
cancel-in-progress: true
env:
@@ -84,7 +86,7 @@ jobs:
contents: read
outputs:
slices: ${{ steps.matrix.outputs.slices }}
timeout_minutes: ${{ steps.matrix.outputs.timeout_minutes }}
slice_timeouts: ${{ steps.matrix.outputs.slice_timeouts }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
@@ -105,7 +107,7 @@ jobs:
id: matrix
run: |
echo "slices=$(jq -c '[range(1; .sliceCount + 1)]' /tmp/marathon-plan/manifest.json)" >> "$GITHUB_OUTPUT"
echo "timeout_minutes=$(jq -e '.plan.ciTimeoutMinutes' /tmp/marathon-plan/manifest.json)" >> "$GITHUB_OUTPUT"
echo "slice_timeouts=$(jq -ec '[0] + .plan.sliceCiTimeoutMinutes' /tmp/marathon-plan/manifest.json)" >> "$GITHUB_OUTPUT"
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
@@ -118,9 +120,10 @@ jobs:
needs: [build-image, plan-slices]
env:
EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-marathon-${{ matrix.slice }}
# One marathon file per runner; the job timeout is the plan's supervised
# worst case plus 20 minutes setup/upload.
timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.timeout_minutes) }}
# One marathon file per runner; its job ceiling is that file's supervised
# wall plus 20 minutes setup (W2c/ENG-2), indexed by slice ([0] pads the
# array). The executor stops 5 minutes before it so results still upload.
timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.slice_timeouts)[matrix.slice] }}
permissions:
contents: read
packages: read
@@ -136,6 +139,9 @@ jobs:
matrix:
slice: ${{ fromJSON(needs.plan-slices.outputs.slices) }}
steps:
- name: Record the job start for the slice deadline
run: echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
# Full history: files with SELF-derived selection (the LLM-judge
+73 -25
View File
@@ -4,7 +4,7 @@ name: Periodic Evals
# tests can't rot invisibly — the class where the autoplan-dual-voice E2E was
# silently broken for months until a lucky local diff selected it. Engine:
# scripts/test-paid-shards.ts (the same runner local eval:bg:periodic uses):
# one planner manifest packed by recorded durations into as many ~9-minute
# one planner manifest packed by recorded durations into as many ~7-minute
# executors as the work needs (one file, or a tightly packed group, per
# runner; overlays share one final slice), and a FAIL-CLOSED report — a slice
# whose artifact never landed is a failure, not an absence. The matrix size
@@ -15,20 +15,36 @@ name: Periodic Evals
# diff-billed, so without it the full gate census might never execute
# anywhere); the hollow-shard guard (exit 0 + zero executed tests under
# EVALS_ALL fails) makes both lanes census-health checks, not just test runs.
# eval-bg (scripts/eval-bg.ts) finds its dispatch by the nonce in the run name.
run-name: ${{ inputs.nonce && format('Periodic Evals [eval-bg {0}]', inputs.nonce) || 'Periodic Evals' }}
on:
schedule:
- cron: '0 6 * * 1' # Monday 6 AM UTC (ci-image prebuilds at 4 AM)
- cron: '0 6 * * 1' # Monday 6 AM UTC; build-image below builds or reuses the CI image
workflow_dispatch:
inputs:
redispatch_of:
description: 'Run id this run re-dispatches (the one INFRA/INCOMPLETE-only re-dispatch; set by the report job)'
type: string
default: ''
include_gate_census:
description: 'Branch dispatch only: also run the weekly gate census (default: periodic lane only; gate cases validate through an evals.yml dispatch with evals_all)'
type: boolean
default: false
expected_sha:
description: 'eval-bg: the revision the dispatcher tested; the run fails fast when the ref has moved'
type: string
default: ''
nonce:
description: 'eval-bg: dispatch nonce carried in the run name'
type: string
default: ''
# A re-dispatch runs in its own group so it never cancels the run that
# dispatched it; both runs are reported.
# One group per ref AND event: a branch dispatch never cancels main's census,
# and a manual main dispatch never cancels the scheduled one. A re-dispatch
# runs in its own group so it never cancels the run that dispatched it; both
# runs are reported.
concurrency:
group: evals-periodic${{ inputs.redispatch_of && format('-redispatch-{0}', inputs.redispatch_of) || '' }}
group: evals-periodic-${{ github.ref }}-${{ github.event_name }}${{ inputs.redispatch_of && format('-redispatch-{0}', inputs.redispatch_of) || '' }}
cancel-in-progress: true
env:
@@ -99,14 +115,24 @@ jobs:
outputs:
periodic_slices: ${{ steps.periodic-matrix.outputs.slices }}
periodic_codex_slices: ${{ steps.periodic-matrix.outputs.codex_slices }}
periodic_timeout_minutes: ${{ steps.periodic-matrix.outputs.timeout_minutes }}
periodic_slice_timeouts: ${{ steps.periodic-matrix.outputs.slice_timeouts }}
gate_slices: ${{ steps.gate-matrix.outputs.slices }}
gate_timeout_minutes: ${{ steps.gate-matrix.outputs.timeout_minutes }}
gate_slice_timeouts: ${{ steps.gate-matrix.outputs.slice_timeouts }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- name: Verify the dispatched revision
if: inputs.expected_sha != ''
env:
EXPECTED_SHA: ${{ inputs.expected_sha }}
run: |
if [ "$GITHUB_SHA" != "$EXPECTED_SHA" ]; then
echo "::error::eval-bg tested ${EXPECTED_SHA} but ${GITHUB_REF_NAME} is now at ${GITHUB_SHA}; push again and re-run eval:bg, or dispatch without expected_sha"
exit 1
fi
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.4.0
@@ -114,7 +140,7 @@ jobs:
- name: Emit run manifest (ALL periodic tests minus reasoned excludes)
env:
EVALS_ALL: "1"
run: EVALS_TIER=periodic bun --no-install run scripts/test-paid-shards.ts --tier periodic --emit-plan /tmp/paid-plan/manifest.json --slice-budget 540 --jobs 2 --max-parallel 26
run: EVALS_TIER=periodic bun --no-install run scripts/test-paid-shards.ts --tier periodic --emit-plan /tmp/paid-plan/manifest.json --slice-budget 420 --jobs 2 --max-parallel 36
# Codex-only slices run on the host-run eval-codex-slices job; the
# container executor runs every other slice.
@@ -124,7 +150,7 @@ jobs:
{
echo "slices=$(jq -c '[range(1; .sliceCount + 1)] - .plan.codexSlices' /tmp/paid-plan/manifest.json)"
echo "codex_slices=$(jq -c '.plan.codexSlices' /tmp/paid-plan/manifest.json)"
echo "timeout_minutes=$(jq -e '.plan.ciTimeoutMinutes' /tmp/paid-plan/manifest.json)"
echo "slice_timeouts=$(jq -ec '[0] + .plan.sliceCiTimeoutMinutes' /tmp/paid-plan/manifest.json)"
} >> "$GITHUB_OUTPUT"
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
@@ -136,13 +162,13 @@ jobs:
- name: Emit gate census manifest (ALL gate tests)
env:
EVALS_ALL: "1"
run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/gate-census-plan/manifest.json --slice-budget 540 --jobs 2 --skip-judges --max-parallel 16
run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/gate-census-plan/manifest.json --slice-budget 420 --jobs 2 --skip-judges --max-parallel 20
- name: Derive the gate census executor matrix from the plan
id: gate-matrix
run: |
echo "slices=$(jq -c '[range(1; .sliceCount + 1)]' /tmp/gate-census-plan/manifest.json)" >> "$GITHUB_OUTPUT"
echo "timeout_minutes=$(jq -e '.plan.ciTimeoutMinutes' /tmp/gate-census-plan/manifest.json)" >> "$GITHUB_OUTPUT"
echo "slice_timeouts=$(jq -ec '[0] + .plan.sliceCiTimeoutMinutes' /tmp/gate-census-plan/manifest.json)" >> "$GITHUB_OUTPUT"
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
@@ -155,10 +181,12 @@ jobs:
needs: [build-image, plan-slices]
env:
EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-slices-${{ matrix.slice }}
# The planner packs ~9 minutes of recorded work per slice; the job timeout
# is its supervised worst case (every shard at its wall) plus 20 minutes
# setup/upload, computed from the same manifest the slices execute.
timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.periodic_timeout_minutes) }}
# The planner packs ~7 minutes of recorded work per slice; each slice's job
# ceiling is max(2x budget, its longest shard wall, the overlay envelope)
# plus 20 minutes setup (W2c/ENG-2), indexed by slice from the manifest the
# slices execute ([0] pads the array so index = slice). The executor stops
# starting work 5 minutes before the ceiling so its results still upload.
timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.periodic_slice_timeouts)[matrix.slice] }}
permissions:
contents: read
packages: read
@@ -172,10 +200,13 @@ jobs:
fail-fast: false
# Every planned slice starts at once; test/evals-workflow-wiring.test.ts
# fails when the live plan outgrows this cap.
max-parallel: 26
max-parallel: 36
matrix:
slice: ${{ fromJSON(needs.plan-slices.outputs.periodic_slices) }}
steps:
- name: Record the job start for the slice deadline
run: echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
# Full history: files with SELF-derived selection (the LLM-judge
@@ -302,16 +333,19 @@ jobs:
if: ${{ needs.plan-slices.outputs.periodic_codex_slices != '[]' }}
env:
EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-slices-${{ matrix.slice }}
timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.periodic_timeout_minutes) }}
timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.periodic_slice_timeouts)[matrix.slice] }}
permissions:
contents: read
packages: read
strategy:
fail-fast: false
max-parallel: 26
max-parallel: 36
matrix:
slice: ${{ fromJSON(needs.plan-slices.outputs.periodic_codex_slices) }}
steps:
- name: Record the job start for the slice deadline
run: echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
fetch-depth: 0
@@ -350,7 +384,7 @@ jobs:
mkdir -p "$RUNNER_TEMP/eval-home" "$RUNNER_TEMP/paid-slice-results"
docker run --rm --user "$(id -u):$(id -g)" \
--security-opt seccomp=unconfined --security-opt apparmor=unconfined \
-e HOME=/home/runner -e ANTHROPIC_API_KEY -e OPENAI_API_KEY -e GEMINI_API_KEY -e SLICE -e EVALS_JOBS \
-e HOME=/home/runner -e ANTHROPIC_API_KEY -e OPENAI_API_KEY -e GEMINI_API_KEY -e SLICE -e EVALS_JOBS -e GSTACK_SLICE_JOB_STARTED_AT \
-e EVALS_RUN_ID -e EVALS_PROFILE -e EVALS_FRESH -e EVALS_CACHE_PURPOSE -e CI -e GITHUB_ACTIONS \
-e GITHUB_RUN_ID -e GITHUB_RUN_ATTEMPT -e GITHUB_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REPOSITORY \
-e GITHUB_EVENT_NAME -e GITHUB_BASE_REF -e GITHUB_HEAD_REF \
@@ -422,10 +456,14 @@ jobs:
gate-census:
runs-on: ubicloud-standard-8
needs: [build-image, plan-slices]
# A branch dispatch validates the periodic lane only; gate cases on a
# branch go through an evals.yml dispatch with evals_all. Main (schedule
# or dispatch) always runs the census; a branch opts in explicitly.
if: ${{ github.ref == 'refs/heads/main' || inputs.include_gate_census }}
env:
EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-gate-census-${{ matrix.slice }}
# Supervised worst case of the packed plan plus 20 minutes setup/upload.
timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.gate_timeout_minutes) }}
# Per-slice job ceiling from the packed plan (W2c/ENG-2), as for eval-slices.
timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.gate_slice_timeouts)[matrix.slice] }}
permissions:
contents: read
packages: read
@@ -438,10 +476,13 @@ jobs:
strategy:
# Two file workers per slice, each retaining two in-file case workers.
fail-fast: false
max-parallel: 16
max-parallel: 20
matrix:
slice: ${{ fromJSON(needs.plan-slices.outputs.gate_slices) }}
steps:
- name: Record the job start for the slice deadline
run: echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
# Full history: files with SELF-derived selection (the LLM-judge
@@ -585,8 +626,15 @@ jobs:
- name: Reconcile gate census against the manifest (fail-closed)
id: gate-reconcile
if: always()
env:
GATE_CENSUS: ${{ needs.gate-census.result }}
run: |
set +e
if [ "$GATE_CENSUS" = "skipped" ]; then
echo "gate census skipped: branch dispatch without include_gate_census (periodic lane only)" | tee /tmp/gate-report.txt
echo "exit=0" >> "$GITHUB_OUTPUT"
exit 0
fi
EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report /tmp/gate-census-report | tee /tmp/gate-report.txt
echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
@@ -656,7 +704,7 @@ jobs:
# issue (never a new issue per week) with the headline and failure block
# of both lanes, and close it on the next green run.
- name: Upsert tracking issue on failure
if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
if: always() && github.ref == 'refs/heads/main' && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
REDISPATCH: ${{ steps.verdict.outputs.redispatch }}
@@ -708,7 +756,7 @@ jobs:
fi
- name: Close the tracking issue on a green run
if: always() && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success'
if: always() && github.ref == 'refs/heads/main' && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
REDISPATCH_OF: ${{ inputs.redispatch_of }}
@@ -723,7 +771,7 @@ jobs:
fi
- name: Fail the workflow when reconciliation failed
if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || !contains(fromJSON('["success","skipped"]'), needs.gate-census.result))
run: exit 1
# The one INFRA/INCOMPLETE re-dispatch (UC-E1). Its own job so the report
+224 -60
View File
@@ -1,18 +1,35 @@
name: E2E Evals
# eval-bg (scripts/eval-bg.ts) finds its dispatch by the nonce in the run name (ENG-12).
run-name: ${{ inputs.nonce != '' && format('E2E Evals [eval-bg {0}]', inputs.nonce) || '' }}
on:
pull_request:
branches: [main]
workflow_dispatch:
inputs:
evals_all:
description: 'Run ALL gate tests in the sliced lane (bypass diff selection; also arms the hollow-shard guard)'
description: 'Run ALL gate tests in the sliced lane (bypass diff selection; also arms the hollow-shard guard). false = the PR profile on the diff against base_sha/base_ref'
type: boolean
default: true
validation_phase:
description: 'Validation branch phase; run quality before behavior on unchanged inputs'
type: choice
options: [all, quality, cookie-quality, behavior, cookie-behavior]
default: all
base_ref:
description: 'Base branch for diff selection when evals_all is false (default: detected base branch)'
type: string
default: ''
base_sha:
description: 'Exact merge-base commit for diff selection; overrides base_ref'
type: string
default: ''
expected_sha:
description: 'Fail fast unless the checked-out commit is this SHA (scripts/eval-bg.ts sets it)'
type: string
default: ''
nonce:
description: 'Opaque id shown in the run name so the dispatcher can find this run'
type: string
default: ''
pr_receipts:
description: 'Same-repo PR number whose receipt store this run reads read-only (never saves); empty = fresh'
type: string
default: ''
concurrency:
group: evals-${{ github.event.pull_request.number || github.run_id }}
@@ -20,9 +37,10 @@ concurrency:
env:
IMAGE: ghcr.io/${{ github.repository }}/ci
# PRs run changed fast probes; manual runs retain the complete gate census.
EVALS_PROFILE: ${{ github.event_name == 'pull_request' && 'pr' || 'full' }}
EVALS_FRESH: ${{ github.event_name == 'workflow_dispatch' && '1' || '' }}
# PRs and evals_all=false dispatches (eval:bg:pr) run the fast PR profile on the diff; evals_all dispatches run the complete gate census.
EVALS_PROFILE: ${{ (github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && !inputs.evals_all)) && 'pr' || 'full' }}
# Dispatches validate fresh unless they name pr_receipts (read-only reuse, CEO-29/ENG-8); the evals-fresh PR label opts a PR out of reuse (DX-7).
EVALS_FRESH: ${{ ((github.event_name == 'workflow_dispatch' && inputs.pr_receipts == '') || contains(github.event.pull_request.labels.*.name, 'evals-fresh')) && '1' || '' }}
jobs:
# Build Docker image with pre-baked toolchain (cached — only rebuilds on Dockerfile/lockfile change)
@@ -102,6 +120,115 @@ jobs:
docker manifest inspect "$EVAL_IMAGE" > /tmp/eval-runtime-manifest.json
echo "id=$(sha256sum /tmp/eval-runtime-manifest.json | cut -d ' ' -f1)" >> "$GITHUB_OUTPUT"
# ── Push-burst debounce (W1d: CEO-15, ENG-11, DX-7) ──────────────────────
# Only on `synchronize` when another evals.yml run for this PR started in
# the last 15 minutes (a push burst): wait 90 s before planning, so the next
# push's cancel-in-progress lands before any paid slice starts. A first push
# pays nothing. cancel-in-progress does the superseding; the head-SHA check
# only reports "superseded by <sha>" when this run outlives the wait. The PR
# label evals-no-debounce skips the wait.
debounce:
runs-on: ubuntu-24.04
if: github.event_name == 'pull_request' && github.event.action == 'synchronize' && github.actor != 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository && !contains(github.event.pull_request.labels.*.name, 'evals-no-debounce')
timeout-minutes: 5
permissions:
actions: read
pull-requests: read
outputs:
superseded: ${{ steps.wait.outputs.superseded }}
steps:
- name: Wait out a push burst
id: wait
env:
GH_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ github.event.pull_request.number }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
HEAD_BRANCH: ${{ github.event.pull_request.head.ref }}
run: |
since=$(date -u -d '15 minutes ago' +%s)
recent=$(gh api -X GET "repos/$GITHUB_REPOSITORY/actions/workflows/evals.yml/runs" -f event=pull_request \
-f branch="$HEAD_BRANCH" -f per_page=20 \
--jq "[.workflow_runs[] | select(.id < $GITHUB_RUN_ID and any(.pull_requests[]; .number == $PR_NUMBER) and ((.created_at | fromdateiso8601) >= $since))] | length" \
2>/dev/null || echo 0)
if [ "${recent:-0}" -eq 0 ]; then
echo "- Debounce: no other run for this PR in the last 15 min; no wait" >> "$GITHUB_STEP_SUMMARY"
exit 0
fi
echo "- Debounce: ${recent} run(s) for this PR started in the last 15 min; waiting 90 s (label evals-no-debounce skips this)" >> "$GITHUB_STEP_SUMMARY"
sleep 90
head=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER" --jq .head.sha 2>/dev/null || echo "$HEAD_SHA")
if [ "$head" != "$HEAD_SHA" ]; then
echo "superseded=true" >> "$GITHUB_OUTPUT"
echo "- Debounce: superseded by ${head}; this run plans no slices" >> "$GITHUB_STEP_SUMMARY"
fi
# ── Receipt recovery (W1c: CEO-14/34, ENG-4/5/7) ─────────────────────────
# A run cancelled by the next push never reaches slices-report, so its
# finished receipts (passes AND the negatives executors write at the
# execution boundary) never reach the saved store. This job restores the
# PR's store, merges the receipts of this PR's completed cancelled runs
# (oldest first, at most 5, provenance-checked), reconstructs completed
# panel negatives, writes the recovery.json checkpoint and hands the store
# to plan-slices as an artifact. It runs scripts from the BASE ref only and
# alone holds actions: read; plan-slices keeps contents: read. It never
# fails the run: any gh problem degrades to "no recovered receipts" and
# plan-slices turns reuse off, so cases execute unreduced.
recover-receipts:
runs-on: ubuntu-24.04
needs: [debounce]
# Runs after the debounce settles, so a superseded run's cancellation has usually completed.
# A dispatch naming pr_receipts reads that PR's receipts from its runs' artifacts (cache entries are ref-scoped).
if: ${{ !cancelled() && needs.debounce.outputs.superseded != 'true' && github.actor != 'dependabot[bot]' && ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && !contains(github.event.pull_request.labels.*.name, 'evals-fresh')) || (github.event_name == 'workflow_dispatch' && inputs.pr_receipts != '')) }}
timeout-minutes: 5
permissions:
contents: read
actions: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.event.pull_request.base.sha || inputs.base_sha || github.event.repository.default_branch }}
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.4.0
- name: Restore this PR's verified judge and E2E results
id: store
if: github.event_name == 'pull_request'
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
with:
path: /tmp/gstack-eval-input-cache
key: eval-input-v1-${{ github.repository_id }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }}-plan
restore-keys: eval-input-v1-${{ github.repository_id }}-pr-${{ github.event.pull_request.number }}-
- name: Recover receipts of cancelled runs (base-ref code)
env:
GH_TOKEN: ${{ github.token }}
STORE_KEY: ${{ steps.store.outputs.cache-matched-key }}
PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr_receipts }}
HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
HEAD_BRANCH: ${{ github.event.pull_request.head.ref || github.ref_name }}
RECOVERY_MODE: ${{ github.event_name == 'workflow_dispatch' && 'dispatch' || 'pr' }}
run: |
mkdir -p /tmp/gstack-eval-input-cache
if [ ! -f scripts/recover-receipts.ts ]; then
echo "- Receipt recovery unavailable: the base ref has no scripts/recover-receipts.ts; reuse stays off this run" >> "$GITHUB_STEP_SUMMARY"
exit 0
fi
bun --no-install run scripts/recover-receipts.ts collect --repo "$GITHUB_REPOSITORY" --pr "$PR_NUMBER" \
--head-repo "$HEAD_REPO" --branch "$HEAD_BRANCH" --run-id "$GITHUB_RUN_ID" \
--store /tmp/gstack-eval-input-cache --store-key "$STORE_KEY" --mode "$RECOVERY_MODE" --budget-seconds 60 >> "$GITHUB_STEP_SUMMARY" \
|| echo "- Receipt recovery crashed; reuse stays off this run" >> "$GITHUB_STEP_SUMMARY"
- name: Hand the receipt store to plan-slices
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: receipt-store
path: /tmp/gstack-eval-input-cache
if-no-files-found: ignore
retention-days: 7
# ── Sliced lane (the ONLY paid lane; legacy 17-row matrix deleted) ──────────
# One PLANNER computes diff selection + the slice plan ONCE (killing
# per-slice selector divergence); K executors consume the manifest; the
@@ -121,13 +248,15 @@ jobs:
# branch (see CLAUDE.md's garrytan-agents workflow).
plan-slices:
runs-on: ubicloud-standard-8
if: github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
needs: [debounce, recover-receipts]
# !cancelled(): plans even when the debounce or recovery was skipped (first push, dispatch, evals-fresh) or recovery failed (reuse is then off); a superseded run plans nothing.
if: ${{ !cancelled() && needs.debounce.outputs.superseded != 'true' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}
timeout-minutes: 10
permissions:
contents: read
outputs:
slices: ${{ steps.matrix.outputs.slices }}
timeout_minutes: ${{ steps.matrix.outputs.timeout_minutes }}
slice_timeouts: ${{ steps.matrix.outputs.slice_timeouts }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
@@ -140,57 +269,65 @@ jobs:
with:
bun-version: 1.4.0
# Planner-side reuse: restore this PR's newest receipt store (the report
# job saves one merged store per run) and ship ONE filtered set with the
# plan, so every trial of a panel sees the same receipts and a newer FAIL
# blocks any older PASS for the same inputs.
- name: Restore this PR's verified judge and E2E results
if: github.event_name == 'pull_request'
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
# DX-1: a dispatch from scripts/eval-bg.ts names the revision it means
# to test; a different checkout fails here before any paid work.
- name: Verify the dispatched revision
if: github.event_name == 'workflow_dispatch'
env:
EXPECTED_SHA: ${{ inputs.expected_sha }}
NONCE: ${{ inputs.nonce }}
run: |
actual=$(git rev-parse HEAD)
echo "- Tested revision: ${actual} (ref ${GITHUB_REF_NAME}${NONCE:+, nonce ${NONCE}})" >> "$GITHUB_STEP_SUMMARY"
if [ -n "$EXPECTED_SHA" ] && [ "$actual" != "$EXPECTED_SHA" ]; then
echo "::error::evals.yml checked out ${actual} but the dispatch expected ${EXPECTED_SHA}. Push the branch so its head is ${EXPECTED_SHA} and dispatch again (bun run scripts/eval-bg.ts pr), or run locally with --local."
exit 1
fi
# Planner-side reuse: take the PR's receipt store from recover-receipts
# (the restored store plus cancelled runs' receipts) and ship ONE
# filtered set with the plan, so every trial of a panel sees the same
# receipts and a newer FAIL blocks any older PASS for the same inputs.
- name: Download this PR's receipt store
if: github.event_name == 'pull_request' || inputs.pr_receipts != ''
continue-on-error: true
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
name: receipt-store
path: /tmp/gstack-eval-input-cache
key: eval-input-v1-${{ github.repository_id }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }}-plan
restore-keys: eval-input-v1-${{ github.repository_id }}-pr-${{ github.event.pull_request.number }}-
- name: Decide receipt reuse
id: reuse
if: github.event_name == 'pull_request' || inputs.pr_receipts != ''
run: |
mkdir -p /tmp/gstack-eval-input-cache
bun --no-install run scripts/recover-receipts.ts decide /tmp/gstack-eval-input-cache --github-output "$GITHUB_OUTPUT" >> "$GITHUB_STEP_SUMMARY"
- name: Emit run manifest
if: github.event_name != 'workflow_dispatch' || inputs.validation_phase == 'all'
env:
EVALS_ALL: ${{ (github.event_name == 'workflow_dispatch' && inputs.evals_all) && '1' || '' }}
EVALS_CACHE_DIR: ${{ github.event_name == 'pull_request' && '/tmp/gstack-eval-input-cache' || '' }}
run: EVALS_TIER=gate bun --no-install run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/paid-plan/manifest.json --slice-budget 540 --jobs 2 --max-parallel 16
EVALS_CACHE_DIR: ${{ steps.reuse.outputs.reuse == 'on' && '/tmp/gstack-eval-input-cache' || '' }}
# Dispatch diff base (CEO-17): the exact merge-base, else the named base branch, else detection.
EVALS_BASE: ${{ inputs.base_sha || (inputs.base_ref != '' && format('origin/{0}', inputs.base_ref)) || '' }}
run: EVALS_TIER=gate bun --no-install run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/paid-plan/manifest.json --slice-budget 420 --jobs 2 --max-parallel 20
- name: Emit validation-phase manifest
if: github.event_name == 'workflow_dispatch' && inputs.validation_phase != 'all'
env:
VALIDATION_PHASE: ${{ inputs.validation_phase }}
EVALS_ALL: ${{ inputs.evals_all && '1' || '' }}
EVALS_TIER: gate
run: |
bun --no-install -e '
import { mkdirSync, writeFileSync } from "node:fs";
import { buildRunManifest, collectPaidTestFiles, restrictManifestSelection } from "./scripts/test-paid-shards.ts";
const phase = process.env.VALIDATION_PHASE;
if (!["quality", "cookie-quality", "behavior", "cookie-behavior"].includes(phase)) throw new Error("Invalid validation phase");
const cookieBehavior = phase === "cookie-behavior";
const discovered = phase === "cookie-quality" ? ["test/skill-llm-eval.test.ts"]
: cookieBehavior ? ["test/skill-e2e-bws.test.ts", "test/skill-e2e-qa-workflow.test.ts", "test/skill-e2e-design.test.ts", "test/skill-e2e-diagram.test.ts", "test/skill-e2e-deploy.test.ts"]
: collectPaidTestFiles().filter(file => file.startsWith("test/skill-llm-eval") === (phase === "quality"));
const manifest = buildRunManifest({ tier: "gate", profile: "full", sliceBudgetMs: 540000, jobs: 2, evalsAll: !cookieBehavior && process.env.EVALS_ALL === "1", discovered,
...(cookieBehavior ? { changedFiles: ["browse/src/cookie-picker-routes.ts", "browse/src/cookie-import-browser.ts", "browse/src/bun-polyfill.cjs"], env: { ...process.env, EVALS_ALL: "" } } : {}) });
const subset = phase === "cookie-quality" ? { e2e: [], judges: ["setup-browser-cookies/SKILL.md workflow"] }
: cookieBehavior ? { e2e: ["browse-basic", "browse-snapshot", "qa-quick", "qa-only-no-fix", "design-review-detector-shim-dom", "diagram-triplet", "canary-workflow", "benchmark-workflow"], judges: [] } : null;
const restricted = subset ? restrictManifestSelection(manifest, subset, "outside the " + phase + " validation subset") : manifest;
restricted.selectionReason = phase + " validation subset; " + manifest.selectionReason;
mkdirSync("/tmp/paid-plan", { recursive: true });
writeFileSync("/tmp/paid-plan/manifest.json", JSON.stringify(restricted, null, 2) + "\n");
console.log(phase + ": " + restricted.entries.filter(entry => entry.status === "planned").length + " planned shards");
'
# CEO-13: the mode, case counts and, on fallback, every file that
# restored the full gate with the edit that narrows it next time.
- name: Summarize PR coverage
run: bun --no-install run scripts/test-pr-profile.ts summary /tmp/paid-plan/manifest.json >> "$GITHUB_STEP_SUMMARY"
- name: Derive the executor matrix from the plan
id: matrix
run: |
echo "slices=$(jq -c '[range(1; .sliceCount + 1)]' /tmp/paid-plan/manifest.json)" >> "$GITHUB_OUTPUT"
echo "timeout_minutes=$(jq -e '.plan.ciTimeoutMinutes' /tmp/paid-plan/manifest.json)" >> "$GITHUB_OUTPUT"
echo "slice_timeouts=$(jq -ec '[0] + .plan.sliceCiTimeoutMinutes' /tmp/paid-plan/manifest.json)" >> "$GITHUB_OUTPUT"
# The whole store (recovered receipts and the checkpoint included)
# travels to slices-report, which saves it with this run's receipts, so
# a receipt reuse turned off for this run is not lost for the next.
- name: Carry the receipt store to the report
if: github.event_name == 'pull_request'
run: mkdir -p /tmp/paid-plan/store && cp -a /tmp/gstack-eval-input-cache/. /tmp/paid-plan/store/
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
@@ -198,6 +335,7 @@ jobs:
path: |
/tmp/paid-plan/manifest.json
/tmp/paid-plan/receipts
/tmp/paid-plan/store
retention-days: 30
eval-slices:
@@ -209,11 +347,13 @@ jobs:
# (image already published), but a newer push's cancel-in-progress stops
# it instead of letting a superseded run finish its paid slices first.
if: ${{ !cancelled() && needs.build-image.result == 'success' && needs.plan-slices.result == 'success' }}
# The planner packs ~9 minutes of recorded work per slice (EVALS_JOBS=2 x
# The planner packs ~7 minutes of recorded work per slice (EVALS_JOBS=2 x
# EVALS_CONCURRENCY=2 per runner, never the old 40-way per-row fan-out
# that queued claude session STARTUP behind 39 siblings). The job timeout
# is the plan's supervised worst case plus 20 minutes setup/upload.
timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.timeout_minutes) }}
# that queued claude session STARTUP behind 39 siblings). Each slice's job
# timeout is its own ceiling (W2c): max(2x budget, its longest shard wall,
# the overlay envelope) + 20 minutes; the executor stops at the slice
# deadline 5 minutes earlier and reports not_run/hung shards.
timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.slice_timeouts)[matrix.slice] }}
permissions:
contents: read
packages: read
@@ -225,10 +365,13 @@ jobs:
options: --user runner
strategy:
fail-fast: false
max-parallel: 16
max-parallel: 20
matrix:
slice: ${{ fromJSON(needs.plan-slices.outputs.slices) }}
steps:
- name: Record the job start for the slice deadline
run: echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
# Full history: files with SELF-derived selection (the LLM-judge
@@ -309,7 +452,7 @@ jobs:
GSTACK_EVAL_DIR: /tmp/paid-slice-results
EVALS_CACHE_DIR: /tmp/paid-slice-results/receipts
EVALS_CACHE_REPOSITORY: ${{ github.repository }}
EVALS_CACHE_PR: ${{ github.event.pull_request.number }}
EVALS_CACHE_PR: ${{ github.event.pull_request.number || inputs.pr_receipts }}
EVALS_CACHE_RUNTIME_ID: ${{ needs.build-image.outputs.runtime-id }}
run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/paid-plan/manifest.json --slice ${{ matrix.slice }}
@@ -409,6 +552,10 @@ jobs:
# (caught by the ship review army; the wiring test now pins this).
echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
- name: Summarize PR coverage and reuse
if: always()
run: bun --no-install run scripts/test-pr-profile.ts summary /tmp/paid-report/manifest.json /tmp/paid-report/collector-outcomes.json >> "$GITHUB_STEP_SUMMARY"
- name: Stamp trial history series
if: always()
run: |
@@ -416,13 +563,15 @@ jobs:
bun --no-install run scripts/eval-trial-series.ts /tmp/paid-report/trial-outcomes.jsonl
fi
# One merged receipt store per run: the plan's shipped set, every slice's
# new pass receipts, and the report's panel and negative receipts. Saved
# One merged receipt store per run: the carried store (recovered
# receipts and checkpoint included), the plan's shipped set, every
# slice's new pass and negative receipts, and the report's panel and
# negative receipts. Saved
# last, so the next planner restores it as the newest prefix match.
- name: Merge this run's receipts
if: always() && github.event_name == 'pull_request'
run: |
bun --no-install run scripts/e2e-shard-reuse.ts merge /tmp/gstack-eval-input-cache \
bun --no-install run scripts/e2e-shard-reuse.ts merge /tmp/gstack-eval-input-cache /tmp/paid-report/store \
/tmp/paid-report/receipts /tmp/paid-report/report-receipts /tmp/paid-report/paid-slice-*/receipts
- name: Save this PR's verified judge and E2E results
@@ -441,6 +590,7 @@ jobs:
/tmp/report.txt
/tmp/paid-report/collector-outcomes.json
/tmp/paid-report/report-summary.md
/tmp/paid-report/report-receipts
if-no-files-found: ignore
retention-days: 30
@@ -531,7 +681,21 @@ jobs:
FAILURES=$(jq -r '.failures[]' "$VERIFIED")
fi
COVERAGE=$(jq -r '"Profile: \(.profile // "full") / \(.prCoverage.mode // "broad"); selected behaviors: \(.selection.e2e | if . == null then "all" else length end), judges: \(.selection.judges | if . == null then "all" else length end). Deferred to scheduled/release coverage: \(.prCoverage.deferred // [] | length) behaviors and \(.prCoverage.deferredPromptFiles // [] | length) changed prompt files. Deferred checks did not run and receive no PR-pass credit."' /tmp/paid-report/manifest.json) || COVERAGE='Coverage manifest unavailable; no coverage claim.'
COVERAGE=$(jq -r '"Profile: \(.profile // "full") / mode `\(.prCoverage.mode // "broad")`; selected behaviors: \(.selection.e2e | if . == null then "all" else length end), judges: \(.selection.judges | if . == null then "all" else length end). Deferred to scheduled/release coverage: \(.prCoverage.deferred // [] | length) behaviors and \(.prCoverage.deferredPromptFiles // [] | length) changed prompt files. Deferred checks did not run and receive no PR-pass credit."' /tmp/paid-report/manifest.json) || COVERAGE='Coverage manifest unavailable; no coverage claim.'
# CEO-13: on fallback, name every file that restored the full gate and its fix.
FALLBACK=$(jq -r 'select(.prCoverage.mode == "full-fallback") | .prCoverage as $c | ($c.unknownFiles // [])[] as $f
| (($c.unknownFileLabels // []) | map(select(.file == $f)) | first // {label: "unlabeled", fix: "see docs/TESTING_INTERNALS.md#pr-paid-lane-fallback"}) as $l
| "- `\($f)` (\($l.label)): \($l.fix)"' /tmp/paid-report/manifest.json 2>/dev/null | head -40 | sed 's/@/@\xe2\x80\x8b/g') || FALLBACK=''
if [ -n "$FALLBACK" ]; then COVERAGE="${COVERAGE}
Full gate restored by:
${FALLBACK}"; fi
# Dependents mode: unmapped files placed by derived references, with their dependent counts.
DERIVED=$(jq -r '(.prCoverage.derivedFiles // [])[:40][] | "- `\(.file)`: \(.e2e) case(s), \(.judges) judge(s)"' /tmp/paid-report/manifest.json 2>/dev/null | sed 's/@/@\xe2\x80\x8b/g') || DERIVED=''
if [ -n "$DERIVED" ]; then COVERAGE="${COVERAGE}
Derived dependents (scripts/pr-dependencies.ts):
${DERIVED}"; fi
STATUS="✅ PASS"
if [ "${RECONCILE_EXIT:-1}" != "0" ] || [ "$FAILED" -gt 0 ] \
+25 -3
View File
@@ -47,14 +47,21 @@ jobs:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
# Full history: the seed ratchet compares the allowlist with its
# merge-base copy, which a depth-1 checkout cannot read.
fetch-depth: 0
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.4.0
- id: plan
name: Inventory and balance the complete free suite
# Warns in the job summary when more than 5 files are unseeded.
run: |
matrix=$(bun run scripts/test-free-shards.ts --ci-plan "$RUNNER_TEMP/free-plan.json" --shards 20)
echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
- name: Seed growth ratchet (60 s ceiling, shrink-only allowlist)
run: |
GSTACK_FREE_SEED_BASE="$(git merge-base HEAD origin/main)" bun test test/free-seed-ratchet.test.ts
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: free-plan
@@ -176,8 +183,12 @@ jobs:
run: bun install --frozen-lockfile --ignore-scripts
- name: Compile trusted CSO helper and watchdog
run: bun run build:cso
# Only the file that can run here: the node-lifecycle and stack-cold
# files need GSTACK_CSO_TEST_STACK images and run in the CSO runtime
# workflows (test:cso:docker), so in this job they always skipped.
- name: Run CSO Docker integration gate
run: bun run test:cso:docker
run: |
bun test --max-concurrency 1 test/cso-docker-integration.test.ts
env:
GSTACK_CSO_DOCKER_TESTS: "1"
DOCKER_HOST: unix:///var/run/docker.sock
@@ -269,9 +280,11 @@ jobs:
# fonts-noto-color-emoji is the emoji-gate's render font (playwright
# --with-deps usually installs it, but the gate must not depend on a
# transitive package list). Fonts must land BEFORE the first browse
# daemon launch — Chromium snapshots fontconfig at startup.
# daemon launch — Chromium snapshots fontconfig at startup. zsh is not
# in the runner image; without it the bash+zsh portability arms (#2669)
# skip instead of running.
- name: Install Xvfb + X11 utilities + gate tools
run: sudo apt-get install -y --no-install-recommends xvfb x11-utils poppler-utils fonts-noto-color-emoji
run: sudo apt-get install -y --no-install-recommends xvfb x11-utils poppler-utils fonts-noto-color-emoji zsh
- name: Configure git identity (tests init temp repos)
run: |
@@ -403,5 +416,14 @@ jobs:
pattern: free-result-*
merge-multiple: true
path: ${{ runner.temp }}/free-results
# Per-shard ledgers land in their own subdirectories (all are named
# flake-ledger.jsonl). A green run is where a retried flake hides, so the
# summary runs before strict verification.
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
pattern: flake-ledger-*
path: ${{ runner.temp }}/flake-ledgers
- name: Summarize flaky passes absorbed by the serial retry
run: bun run scripts/test-health-report.ts flake-summary "$RUNNER_TEMP/flake-ledgers" >> "$GITHUB_STEP_SUMMARY"
- name: Require exact coverage and complete strict results
run: bun run scripts/test-free-shards.ts --ci-verify "$RUNNER_TEMP/free-plan.json" --results "$RUNNER_TEMP/free-results"
+154
View File
@@ -0,0 +1,154 @@
name: Native Qualification
# Dispatch-only diagnostic campaigns for native browser integrations: owned
# Windows cookie extraction, Windows launch/credential diagnostics, and Dia on
# disposable Apple-silicon Macs. They exercise real browsers and OS services,
# so they are qualification evidence, not PR gates; they used to live in
# windows-free-tests.yml, where their red dispatch runs read as a red PR lane.
#
# One `mode` per dispatch; every job runs only for its own mode:
# gh workflow run native-qualification.yml --ref <branch> -f mode=cookie-native
on:
workflow_dispatch:
inputs:
mode:
description: Which native campaign to run
type: choice
required: true
options:
- cookie-native
- windows-native-diagnostics
- dia-native
- dia-launch-comparison
- dia-gui-readiness
concurrency:
group: native-qualification-${{ github.ref }}-${{ inputs.mode }}
cancel-in-progress: true
permissions:
contents: read
jobs:
cookie-native-qualification:
if: inputs.mode == 'cookie-native'
runs-on: windows-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6
with:
bun-version: 1.4.0
- uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38
with:
node-version: 24.18.0
- name: Install pinned dependencies
run: bun install --frozen-lockfile
- name: Build the qualified Node server inputs
run: bash browse/scripts/build-node-server.sh
shell: bash
- name: Qualify owned native cookie extraction
run: ./.github/scripts/run-cookie-native-qualification.ps1 -OutputRoot "$env:RUNNER_TEMP"
- name: Preserve qualification evidence
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
with:
name: cookie-native-qualification
path: ${{ runner.temp }}/cookie-native-qualification-*/
if-no-files-found: error
windows-native-diagnostics:
if: inputs.mode == 'windows-native-diagnostics'
runs-on: windows-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6
with:
bun-version: 1.4.0
- uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38
with:
node-version: 24.18.0
- name: Configure git identity (required by tests that init temp repos)
run: |
git config --global user.email "windows-ci@gstack.test"
git config --global user.name "Windows CI"
git config --global init.defaultBranch main
shell: bash
- name: Install pinned dependencies
run: bun install --frozen-lockfile
- name: Build server-node.mjs (required by Windows browse path)
run: bash browse/scripts/build-node-server.sh
shell: bash
- name: Install Chromium for the Node worker smoke
run: bunx playwright install chromium
- name: Run focused native launch and credential diagnostics
shell: bash
run: |
set -o pipefail
status=0
bun test browse/test/cookie-import-native-job.test.ts --test-name-pattern 'native Windows launch diagnostics|a locked real Edge profile|real Edge synthetic profile' 2>&1 | tee "$RUNNER_TEMP/gstack-free-test-native-diagnostics.log" || status=1
bun test browse/test/cookie-credential-deadline.test.ts browse/test/cookie-import-node.test.ts browse/test/bun-polyfill.test.ts 2>&1 | tee "$RUNNER_TEMP/gstack-free-test-credential-diagnostics.log" || status=1
exit "$status"
- name: Upload diagnostic logs
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
with:
name: windows-native-diagnostics
path: ${{ runner.temp }}/gstack-free-test-*.log
if-no-files-found: ignore
dia-native-qualification:
if: startsWith(inputs.mode, 'dia-')
runs-on: macos-15
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
runtime: ${{ fromJSON(inputs.mode == 'dia-launch-comparison' && '["bun","node"]' || '["bun"]') }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6
with:
bun-version: 1.4.0
- uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38
if: inputs.mode == 'dia-launch-comparison'
with:
node-version: 24.18.0
architecture: arm64
- name: Install pinned dependencies
if: inputs.mode != 'dia-gui-readiness'
run: bun install --frozen-lockfile
- name: Install the synthetic destination browser
if: inputs.mode != 'dia-gui-readiness'
run: bunx --no-install playwright install chromium
- name: Inspect GUI readiness without browser or Keychain access
if: inputs.mode == 'dia-gui-readiness'
env:
GSTACK_DIA_NATIVE_QUALIFY: '1'
run: bun --no-env-file --no-install --no-macros --config=/dev/null .github/scripts/run-dia-native-qualification.ts --gui-readiness-only
- name: Qualify native Dia discovery, decryption, and import
if: inputs.mode == 'dia-native'
env:
GSTACK_DIA_NATIVE_QUALIFY: '1'
run: bun --no-env-file --no-install --no-macros --config=/dev/null .github/scripts/run-dia-native-qualification.ts
- name: Compare protected native Dia launch without qualification credit
if: inputs.mode == 'dia-launch-comparison'
env:
GSTACK_DIA_NATIVE_QUALIFY: '1'
COMPARISON_RUNTIME: ${{ matrix.runtime }}
run: bun --no-env-file --no-install --no-macros --config=/dev/null .github/scripts/run-dia-native-qualification.ts --launch-comparison "$COMPARISON_RUNTIME"
- name: Preserve only the sanitized qualification receipt
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
with:
name: ${{ inputs.mode == 'dia-gui-readiness' && 'dia-gui-readiness' || inputs.mode == 'dia-launch-comparison' && format('dia-launch-comparison-{0}', matrix.runtime) || 'dia-native-qualification' }}
path: ${{ runner.temp }}/dia-native-qualification.json
if-no-files-found: error
+28
View File
@@ -25,3 +25,31 @@ jobs:
--include-git-root
--recursive
./
# A scheduled red used to notify no one (F11: a three-week unactioned
# backlog). Main runs upsert one tracking issue on a failed scan and close it
# on the next clean scan. Pinned by test/osv-ignore-expiry.test.ts.
report:
needs: scan
if: always() && github.ref == 'refs/heads/main'
runs-on: ubuntu-24.04
timeout-minutes: 5
permissions:
issues: write
steps:
- name: Upsert or close the tracking issue
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SCAN_RESULT: ${{ needs.scan.result }}
run: |
set -euo pipefail
TITLE="OSV scanner: vulnerable dependency needs triage"
RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty')
if [ "$SCAN_RESULT" = "failure" ]; then
BODY="The weekly OSV scan failed: $RUN_URL (findings are in the run's Security tab upload and log). Fix: upgrade the package (package.json overrides), or add a justified [[IgnoredVulns]] entry with an ignoreUntil to .osv-scanner.toml."
if [ -n "$EXISTING" ]; then gh issue comment "$EXISTING" --repo "$GITHUB_REPOSITORY" --body "$BODY";
else gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --body "$BODY"; fi
elif [ "$SCAN_RESULT" = "success" ] && [ -n "$EXISTING" ]; then
gh issue close "$EXISTING" --repo "$GITHUB_REPOSITORY" --comment "Clean OSV scan: $RUN_URL"
fi
@@ -0,0 +1,87 @@
name: Platform qualification
# Opt-in tests with a real product guarantee whose prerequisites no PR or
# weekly lane has: the ML prompt-injection classifier model (~112 MB), the
# pinned gitleaks binary, and a macOS Swift toolchain. In the free suite these
# files self-skip without their opt-in variable; this workflow is the owned
# lane that sets it. It runs on dispatch and once a quarter as the reminder
# that these guarantees are still checked. Each job is a few minutes.
on:
workflow_dispatch:
schedule:
# 08:00 UTC on the first day of January, April, July and October.
- cron: '0 8 1 1,4,7,10 *'
concurrency:
group: platform-qualification-${{ github.ref }}
cancel-in-progress: false
permissions:
contents: read
jobs:
security-ml:
# browse/test/security-bench.test.ts and the L4 cases of
# browse/test/security-live-playwright.test.ts: the TestSavant classifier
# keeps its detection rate on the BrowseSafe-Bench sample and does not
# flag benign product copy.
runs-on: ubuntu-24.04
timeout-minutes: 20
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.4.0
- name: Install dependencies
run: bun install --frozen-lockfile
- name: Install Chromium
run: bunx playwright install --with-deps chromium
- name: Download the classifier model
run: bun -e "const m = await import('./browse/src/security-classifier.ts'); await m.loadTestsavant(msg => console.log(msg)); const s = m.getClassifierStatus(); console.log(JSON.stringify(s)); if (s.testsavant !== 'ok') process.exit(1);"
- name: Run ML security tests
env:
SECURITY_BENCH: "1"
run: bun test browse/test/security-bench.test.ts browse/test/security-live-playwright.test.ts
gitleaks:
# test/gstack-memory-ingest.test.ts and test/gstack-memory-helpers.test.ts:
# memory ingest imports clean pages and blocks recognized secrets through
# the real gitleaks 8.30.1 scanner, not the fake one the free suite uses.
runs-on: ubuntu-24.04
timeout-minutes: 15
env:
GITLEAKS_VERSION: 8.30.1
GITLEAKS_SHA256: 551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.4.0
- name: Install dependencies
run: bun install --frozen-lockfile
- name: Install gitleaks
run: |
mkdir -p "$RUNNER_TEMP/gitleaks"
curl -fsSL -o "$RUNNER_TEMP/gitleaks/gitleaks.tgz" "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz"
echo "${GITLEAKS_SHA256} $RUNNER_TEMP/gitleaks/gitleaks.tgz" | sha256sum -c -
tar -xzf "$RUNNER_TEMP/gitleaks/gitleaks.tgz" -C "$RUNNER_TEMP/gitleaks" gitleaks
"$RUNNER_TEMP/gitleaks/gitleaks" version
- name: Run gitleaks-backed memory tests
run: GSTACK_TEST_GITLEAKS="$RUNNER_TEMP/gitleaks/gitleaks" bun test test/gstack-memory-ingest.test.ts test/gstack-memory-helpers.test.ts
ios-swift-build:
# test/ios-qa-swift-build.test.ts: the DebugBridgeCore package builds, its
# XCTest suite passes, and a Release build contains no DebugBridge symbols.
runs-on: macos-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.4.0
- name: Install dependencies
run: bun install --frozen-lockfile
- name: Run Swift build invariants
env:
GSTACK_TEST_SWIFT: "1"
run: bun test test/ios-qa-swift-build.test.ts
+1 -1
View File
@@ -31,7 +31,7 @@ concurrency:
jobs:
sync:
name: Sync PR title to VERSION
runs-on: ubicloud-standard-2
runs-on: ubuntu-24.04
timeout-minutes: 5
permissions:
contents: read
+11 -8
View File
@@ -1,11 +1,13 @@
# Quality gate (fork port wave 2, adapted from time-attack/gstack GStack 2).
#
# Three generic hygiene checks the repo previously had nowhere in CI:
# Generic hygiene checks:
# 1. Credential scan of the PR diff's ADDED lines through our own
# bin/gstack-redact (HIGH fails the check; MEDIUM is an advisory count —
# there is no human in CI to confirm, so it never fails here).
# 2. bun audit at critical severity.
# 3. ShellCheck (errors only) on the setup/build shell boundary.
# 2. ShellCheck (errors only) on the setup/build shell boundary.
# Dependency advisories are gated by dependency-review.yml (PRs that change
# the lockfile) and the weekly osv-scanner.yml sweep, not here: an unrelated
# PR must not turn red because an advisory was published that morning.
#
# Trigger is `pull_request`, NEVER `pull_request_target`: fork PRs must not
# get secret-bearing contexts. Diff excludes cover the planted-bug fixtures
@@ -23,12 +25,16 @@ permissions:
contents: read
concurrency:
group: quality-gate-${{ github.workflow }}-${{ github.ref }}
# PR-number keyed; pushes to main get their own run_id group so
# back-to-back merges never cancel each other's credential scan.
group: quality-gate-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: true
jobs:
quality:
runs-on: ubicloud-standard-8
# A ~1-minute job: GitHub's free hosted runner (public repo), not a
# billed 8-vCPU Ubicloud machine.
runs-on: ubuntu-24.04
timeout-minutes: 10
steps:
# Shallow checkout: fetch-depth:0 cost 74s of a 92s job for checks that
@@ -76,9 +82,6 @@ jobs:
':(exclude)test/helpers/security-bench*' \
| node .github/scripts/gate-secret-scan.mjs
- name: Gate critical dependency advisories
run: bun audit --audit-level=critical
- name: Install ShellCheck
run: |
sudo apt-get update
+1 -4
View File
@@ -6,9 +6,6 @@ on:
branches: [main]
pull_request:
# Cancel superseded runs for the same branch (matches evals.yml,
# windows-free-tests.yml, etc.). head_ref is set on pull_request; ref_name is
# the fallback for push so a rapid push series doesn't pile up stale runs.
concurrency:
# PR-number keyed (run_id fallback for push/dispatch): a bare branch name
# carries no fork prefix, so same-name branches from two forks would share
@@ -22,7 +19,7 @@ permissions:
jobs:
check-freshness:
runs-on: ubicloud-standard-2
runs-on: ubuntu-24.04
timeout-minutes: 10
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+83
View File
@@ -0,0 +1,83 @@
name: Weekly test health
# The weekly home for checks that need CI history (CEO-18): `bun run
# test:health --enforce` reads the last 7 days of GitHub Actions history and
# fails when a free test file flakes in more than 5% of main free-tests runs
# over a window of at least 20 runs (W7a), or when more than 5 free test files
# are missing from scripts/free-test-durations.json (CEO-10: stale seed).
# Metrics whose inputs are unavailable never fail the run; they are listed in
# the job summary. Runs Monday after the periodic census (06:00 UTC), on a
# free hosted runner, and runs even when the census is cancelled or red.
#
# Tracking issue (DX-9): an enforced failure upserts one issue; a run with
# unavailable inputs keeps it open and lists them; it closes only when every
# enforced check has evidence and passes. Only main runs touch the issue.
# Pinned by test/test-health-workflow.test.ts.
on:
schedule:
- cron: '0 10 * * 1'
workflow_dispatch:
permissions:
contents: read
actions: read
issues: write
concurrency:
group: test-health-${{ github.ref }}
cancel-in-progress: true
jobs:
health:
runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.4.0
- name: Install dependencies
run: bun install --frozen-lockfile
- name: Measure test health
id: health
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set +e
bun run test:health --since-days 7 --enforce --json --summary "$RUNNER_TEMP/health.md" > "$RUNNER_TEMP/health.json"
status=$?
set -e
cat "$RUNNER_TEMP/health.md" >> "$GITHUB_STEP_SUMMARY" || true
action=$(jq -r '.issueAction // "keep-open"' "$RUNNER_TEMP/health.json" 2>/dev/null || echo keep-open)
echo "exit=$status" >> "$GITHUB_OUTPUT"
echo "issue_action=$action" >> "$GITHUB_OUTPUT"
- name: Update the tracking issue
if: always() && github.ref == 'refs/heads/main'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ISSUE_ACTION: ${{ steps.health.outputs.issue_action || 'keep-open' }}
run: |
set -euo pipefail
TITLE="Weekly test health: enforced check needs triage"
RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
BODY_FILE="$RUNNER_TEMP/issue.md"
{ echo "Run: $RUN_URL"; echo; cat "$RUNNER_TEMP/health.md" 2>/dev/null || echo "(no report: the test:health step did not produce one)"; } > "$BODY_FILE"
EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty')
case "$ISSUE_ACTION" in
upsert)
if [ -n "$EXISTING" ]; then gh issue comment "$EXISTING" --repo "$GITHUB_REPOSITORY" --body-file "$BODY_FILE";
else gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --body-file "$BODY_FILE"; fi ;;
close)
if [ -n "$EXISTING" ]; then gh issue close "$EXISTING" --repo "$GITHUB_REPOSITORY" --comment "Every enforced check has evidence and passes: $RUN_URL"; fi ;;
*)
if [ -n "$EXISTING" ]; then gh issue comment "$EXISTING" --repo "$GITHUB_REPOSITORY" --body-file "$BODY_FILE"; fi ;;
esac
- name: Fail when an enforced check failed
if: always() && steps.health.outputs.exit != '0'
run: exit 1
+1 -1
View File
@@ -14,7 +14,7 @@ concurrency:
jobs:
check:
name: Check VERSION is not stale vs queue
runs-on: ubicloud-standard-2
runs-on: ubuntu-24.04
timeout-minutes: 10
permissions:
contents: read
+147 -183
View File
@@ -1,46 +1,32 @@
name: Windows Free Tests
# Curated subset of the free test suite that runs on a paid faster Windows runner.
# The Windows-safe subset of the free suite on GitHub's free `windows-latest`
# runners. scripts/test-free-shards.ts --windows-only is the single source of
# truth for which files run: POSIX-bound tests (bin/ shebang spawns, raw /tmp
# paths, /bin/sh, mode bitmasks, the Bun-driven browse server) are excluded by
# pattern there, with explicit force-include and known-incompatible lists.
# Every run prints the curated count and top exclusion reasons in the job
# summary so a shrinking subset is visible.
#
# Codex's v1.18.0.0 review flagged that the existing evals.yml workflow uses
# a Linux container, so a windows-latest matrix entry there isn't a drop-in.
# This workflow is non-container, runs the curated Windows-safe subset, plus
# targeted resolver tests that exercise the Bun.which-based claude binary
# resolution + the GSTACK_CLAUDE_BIN override path on Windows.
# Same plan/run/verify model as free-tests.yml: one duration-balanced plan of
# the curated files, one shard per Windows job, and an aggregate that requires
# every strict result and exact coverage. The plan and the verify step are
# static (file contents and the git revision), so they run on Linux. Packing
# reads Windows-measured durations (scripts/free-test-durations-windows.json):
# Windows-only files such as the native cookie job cost minutes there and
# nothing on Linux. Refresh it with a `record_durations` dispatch and commit
# the artifact.
#
# Runner: GitHub-hosted free `windows-latest`. The whole rest of CI runs on
# Ubicloud (Linux), but Ubicloud doesn't ship Windows runners and we don't
# want to flip on GitHub's org-level larger-runner billing for just this one
# job. 4 cores, ~60s spin-up, $0. The wave-coverage tests this runs are
# small enough that total job time stays under 2 minutes.
#
# What this DOES NOT do (still out of scope, tracked as follow-up):
# - Run the full free suite on Windows. The 24 tests that hardcode /bin/sh,
# spawn('sh',...), or raw /tmp/ paths are excluded by scripts/test-free-shards.ts
# --windows-only. They need POSIX-bound surfaces to be ported off shell
# primitives before they can run on Windows.
# - Run Playwright/browser-backed tests. Browse server bring-up on Windows is
# a separate concern (PR #1238 windows-pty-bun-pty-fix is in flight).
# Native browser qualification campaigns (Windows cookie extraction, launch
# diagnostics, Dia on macOS) are dispatch-only in native-qualification.yml.
on:
pull_request:
branches: [main]
workflow_dispatch:
inputs:
dia_native_only:
description: Run disposable ARM64 macOS Dia qualification instead of Windows
type: boolean
default: false
native_diagnostics_only:
description: Run Windows launch diagnostics and credential regressions without qualification
type: boolean
default: false
dia_launch_comparison:
description: Compare protected Dia launch under Bun and Node in separate fresh Mac jobs
type: boolean
default: false
dia_gui_readiness:
description: Inspect disposable Mac GUI-session readiness without launching browsers
record_durations:
description: Record per-file Windows durations (artifact) instead of running the lane
type: boolean
default: false
@@ -53,15 +39,46 @@ permissions:
contents: read
jobs:
windows-free-tests:
if: ${{ !inputs.dia_native_only && !inputs.dia_launch_comparison && !inputs.dia_gui_readiness }}
# Ubicloud Windows runner (same provider as the Linux evals workflow).
# To revert: swap to `windows-latest` (GitHub's free 4-core Windows runner).
windows-plan:
if: ${{ !inputs.record_durations }}
runs-on: ubuntu-24.04
timeout-minutes: 5
outputs:
matrix: ${{ steps.plan.outputs.matrix }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.4.0
- id: plan
name: Curate and balance the Windows-safe free suite
# Six shards: ~85 s of Linux-measured work each, ~2.5 min on a
# 4-vCPU Windows runner, so the lane lands near 4 min with setup.
env:
GSTACK_FREE_TEST_DURATIONS: scripts/free-test-durations-windows.json
run: |
matrix=$(bun run scripts/test-free-shards.ts --windows-only --ci-plan "$RUNNER_TEMP/windows-plan.json" --shards 6)
echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: windows-plan
path: ${{ runner.temp }}/windows-plan.json
if-no-files-found: error
windows-free-shard:
needs: windows-plan
runs-on: windows-latest
timeout-minutes: 15
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.windows-plan.outputs.matrix) }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
@@ -71,15 +88,17 @@ jobs:
with:
node-version: 24.18.0
# bun install was 35s of a 55s job, all network. Cache keyed on the
# lockfile; bun's install cache lives under ~/.bun/install/cache on
# every platform.
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
name: windows-plan
path: ${{ runner.temp }}
# bun's install cache lives under ~/.bun/install/cache on every platform.
- uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
with:
path: ~/.bun/install/cache
key: windows-bun-${{ hashFiles('bun.lock') }}
# A lockfile bump starts from the previous cache instead of cold
# (restore alone costs ~26s; without this a bump pays it for nothing).
# A lockfile bump starts from the previous cache instead of cold.
restore-keys: |
windows-bun-
@@ -94,90 +113,49 @@ jobs:
run: bun install --frozen-lockfile
- name: Build server-node.mjs (required by Windows browse path)
# browse/src/cli.ts module-level throws on Windows if server-node.mjs
# browse/src/cli.ts throws at module load on Windows if server-node.mjs
# is missing — Bun can't drive Playwright's Chromium on Windows
# (oven-sh/bun#4253). The bundle must exist for any test that
# transitively loads cli.ts to even import. We build only the
# Node-compatible server bundle here; full `bun run build` would
# also compile every binary which is slow and unnecessary for tests.
# (oven-sh/bun#4253), so any test that transitively loads cli.ts needs
# the Node-compatible server bundle. The full build is unnecessary.
run: bash browse/scripts/build-node-server.sh
shell: bash
- name: Generate host SKILL.md outputs (.agents, .factory)
if: ${{ !inputs.native_diagnostics_only }}
# The golden-file regression tests in test/gen-skill-docs.test.ts read
# .agents/skills/gstack-ship/SKILL.md and .factory/skills/gstack-ship/
# SKILL.md. Both are gitignored — generated on demand by gen:skill-docs.
# On Mac/Linux CI the existing eval workflow regenerates these as part
# of its own pipeline; the windows-free-tests lane doesn't share that
# so it must regenerate explicitly.
# Golden-file tests read gitignored generated host outputs.
run: bun run gen:skill-docs --host all
shell: bash
- name: Install Chromium for the Node worker smoke
run: bunx playwright install chromium
# The Windows job verifies the new portability work this PR delivers,
# not the entire free suite. After v1.20.0.0 ships, full-suite Windows
# parity is a P4 follow-up TODO that depends on porting many tests off
# POSIX-bound surfaces (raw /tmp paths, /bin/bash hardcodes, bash
# shebang spawns, mode-bit assertions, deleted v1.14 sidebar refs, etc).
#
# The curated subset enumeration in scripts/test-free-shards.ts is
# retained for future expansion — `bun run test:windows --list` gives
# contributors a starting point to grow Windows coverage incrementally.
#
# What we verify here is exactly the new code paths v1.20.0.0 ships:
# - bin/gstack-paths state-root resolution (test/gstack-paths.test.ts)
# - browse/src/claude-bin.ts Bun.which wrapper + override + arg-prefix
# resolution including the GSTACK_CLAUDE_BIN=wsl PATHEXT path
# (browse/test/claude-bin.test.ts)
# - scripts/test-free-shards.ts curation logic itself
# (test/test-free-shards.test.ts)
- name: Run curated Windows-safe suite
if: ${{ !inputs.native_diagnostics_only }}
# Replaces the previous hand-listed 13-file subset, which drifted from
# the curation registry it was supposed to sample. The runner's
# --windows-only curation (scripts/test-free-shards.ts) is the single
# source of truth: POSIX-bound tests are excluded by pattern there, so
# growing/pruning Windows coverage is one list, not two. If a test is
# red here because it's genuinely POSIX-bound, add it to the curation
# exclusions — don't resurrect a hand list in this file.
- name: Run Windows-safe shard
env:
GSTACK_FREE_JOBS: '2'
# Same policy as the required Linux lane: process-supervision timing
# tests pass alone on this 4-vCPU runner but can stall under the
# two-shard load, so attributed failures get one serial retry
# (cap 5 files, truncation veto). Every flaky pass is appended to
# the ledger uploaded below, so repeats stay visible and ranked.
# Same policy as the required Linux lane: an attributed failure gets
# one serial retry (cap 5 files, truncation veto) and a clean retry
# is a loud flaky pass, appended to the ledger uploaded below and
# listed in the aggregate job summary.
GSTACK_FREE_RETRY_FLAKY: '1'
GSTACK_FLAKE_LEDGER: ${{ runner.temp }}/flake-ledger.jsonl
TEMP: ${{ runner.temp }}
TMP: ${{ runner.temp }}
run: bun run test:windows
run: bun run test:windows --ci-run "$RUNNER_TEMP/windows-plan.json" --shard ${{ matrix.shard }} --result "$RUNNER_TEMP/windows-results/shard-${{ matrix.shard }}.json"
shell: bash
- name: Run focused native launch and credential diagnostics
if: inputs.native_diagnostics_only
shell: bash
run: |
set -o pipefail
status=0
bun test browse/test/cookie-import-native-job.test.ts --test-name-pattern 'native Windows launch diagnostics|a locked real Edge profile|real Edge synthetic profile' 2>&1 | tee "$RUNNER_TEMP/gstack-free-test-native-diagnostics.log" || status=1
bun test browse/test/cookie-credential-deadline.test.ts browse/test/cookie-import-node.test.ts browse/test/bun-polyfill.test.ts 2>&1 | tee "$RUNNER_TEMP/gstack-free-test-credential-diagnostics.log" || status=1
exit "$status"
- name: Upload strict shard result
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: windows-result-${{ matrix.shard }}
path: ${{ runner.temp }}/windows-results/*.json
if-no-files-found: error
# Same diagnosability contract as free-tests.yml: a red lane must
# carry the WHY (the runner's quiet console names files, not causes).
# (#2561 was written against the old hand-listed subset; its two new
# test files are pure-TS and flow into the --windows-only curation
# automatically, so no per-file entry is needed here.)
# A red lane must carry the WHY: the runner's quiet console names files,
# not causes, so complete shard logs are kept on every run.
- name: Upload full shard logs
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: windows-free-test-shard-logs
name: windows-free-test-shard-logs-${{ matrix.shard }}
path: |
.context/free-test-logs/gstack-free-test-*.log
${{ runner.temp }}/gstack-free-test-*.log
@@ -188,97 +166,83 @@ jobs:
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: flake-ledger-windows
name: flake-ledger-windows-${{ matrix.shard }}
path: ${{ runner.temp }}/flake-ledger.jsonl
if-no-files-found: ignore
retention-days: 90
cookie-native-qualification:
if: github.event_name == 'workflow_dispatch' && !inputs.dia_native_only && !inputs.native_diagnostics_only && !inputs.dia_launch_comparison && !inputs.dia_gui_readiness
runs-on: windows-latest
timeout-minutes: 10
windows-free-tests:
if: ${{ always() && !inputs.record_durations }}
needs: [windows-plan, windows-free-shard]
runs-on: ubuntu-24.04
timeout-minutes: 5
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6
with:
bun-version: 1.4.0
- uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38
with:
node-version: 24.18.0
- name: Install pinned dependencies
run: bun install --frozen-lockfile
- name: Build the qualified Node server inputs
run: bash browse/scripts/build-node-server.sh
shell: bash
- name: Qualify owned native cookie extraction
run: ./.github/scripts/run-cookie-native-qualification.ps1 -OutputRoot "$env:RUNNER_TEMP"
- name: Preserve qualification evidence
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
with:
name: cookie-native-qualification
path: ${{ runner.temp }}/cookie-native-qualification-*/
if-no-files-found: error
dia-native-qualification:
if: github.event_name == 'workflow_dispatch' && (inputs.dia_native_only || inputs.dia_launch_comparison || inputs.dia_gui_readiness)
runs-on: macos-15
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
runtime: ${{ fromJSON(inputs.dia_launch_comparison && !inputs.dia_gui_readiness && '["bun","node"]' || '["bun"]') }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6
with:
bun-version: 1.4.0
- name: Validate GUI readiness selection
if: inputs.dia_gui_readiness
- name: Require the plan and every Windows shard
env:
OTHER_DIA_MODES: ${{ inputs.dia_native_only || inputs.dia_launch_comparison || inputs.native_diagnostics_only }}
PLAN_RESULT: ${{ needs.windows-plan.result }}
SHARD_RESULT: ${{ needs.windows-free-shard.result }}
run: |
bun --no-env-file --no-install --no-macros --config=/dev/null -e '
if (process.env.OTHER_DIA_MODES !== "false") {
console.error("dia_gui_readiness must be selected alone");
process.exit(1);
}
'
set -eu
test "$PLAN_RESULT" = success
test "$SHARD_RESULT" = success
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.4.0
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
name: windows-plan
path: ${{ runner.temp }}
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
pattern: windows-result-*
merge-multiple: true
path: ${{ runner.temp }}/windows-results
- name: Require exact coverage and complete strict results
run: bun run scripts/test-free-shards.ts --windows-only --ci-verify "$RUNNER_TEMP/windows-plan.json" --results "$RUNNER_TEMP/windows-results"
windows-record-durations:
if: ${{ inputs.record_durations }}
runs-on: windows-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.4.0
- uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38
if: inputs.dia_launch_comparison && !inputs.dia_gui_readiness
with:
node-version: 24.18.0
architecture: arm64
- name: Install pinned dependencies
if: ${{ !inputs.dia_gui_readiness }}
run: bun install --frozen-lockfile
- name: Install the synthetic destination browser
if: ${{ !inputs.dia_gui_readiness }}
run: bunx --no-install playwright install chromium
- name: Inspect GUI readiness without browser or Keychain access
if: inputs.dia_gui_readiness
- name: Prepare the same inputs as a Windows shard
shell: bash
run: |
git config --global user.email "windows-ci@gstack.test"
git config --global user.name "Windows CI"
git config --global init.defaultBranch main
bun install --frozen-lockfile
bash browse/scripts/build-node-server.sh
bun run gen:skill-docs --host all
bunx playwright install chromium
# Records into this checkout's default seed path: GSTACK_FREE_TEST_DURATIONS
# would leak into every timed test, and seed-reading tests would then read
# the half-written recording instead of their fixtures.
- name: Time every Windows-safe file alone
shell: bash
env:
GSTACK_DIA_NATIVE_QUALIFY: '1'
run: bun --no-env-file --no-install --no-macros --config=/dev/null .github/scripts/run-dia-native-qualification.ts --gui-readiness-only
- name: Qualify native Dia discovery, decryption, and import
if: ${{ !inputs.dia_launch_comparison && !inputs.dia_gui_readiness }}
env:
GSTACK_DIA_NATIVE_QUALIFY: '1'
run: bun --no-env-file --no-install --no-macros --config=/dev/null .github/scripts/run-dia-native-qualification.ts
- name: Compare protected native Dia launch without qualification credit
if: inputs.dia_launch_comparison && !inputs.dia_gui_readiness
env:
GSTACK_DIA_NATIVE_QUALIFY: '1'
COMPARISON_RUNTIME: ${{ matrix.runtime }}
run: bun --no-env-file --no-install --no-macros --config=/dev/null .github/scripts/run-dia-native-qualification.ts --launch-comparison "$COMPARISON_RUNTIME"
- name: Preserve only the sanitized qualification receipt
TEMP: ${{ runner.temp }}
TMP: ${{ runner.temp }}
run: bun run test:windows --record-durations
- name: Name the recording as the Windows seed
if: always()
shell: bash
run: cp scripts/free-test-durations.json "$RUNNER_TEMP/free-test-durations-windows.json"
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
with:
name: ${{ inputs.dia_gui_readiness && 'dia-gui-readiness' || inputs.dia_launch_comparison && format('dia-launch-comparison-{0}', matrix.runtime) || 'dia-native-qualification' }}
path: ${{ runner.temp }}/dia-native-qualification.json
name: free-test-durations-windows
path: ${{ runner.temp }}/free-test-durations-windows.json
if-no-files-found: error