mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-07 17:28:23 +01:00
Garry Tan
042daeee71
Pin every CI surface to Bun 1.4.0 to avoid extra-stdio finalizers closing reused live sockets. Add an isolated GC/listener regression that fails on Bun 1.3.13, and prevent coordinated rollback to an affected CI runtime. Check renderer cleanup against the render's own staging directory so concurrent renders cannot invalidate the assertion. Make the no-pgrep process-tree walk tolerate disappearing /proc entries, and synchronize its test fixture through child readiness and pipe EOF instead of sleeps. Validation: 9,157 passed, 31 skipped, zero failures across 556 files with retries disabled. Build, all-host generation freshness, and skill checks passed. All three races have failing-before/passing-after regressions.
422 lines
19 KiB
YAML
422 lines
19 KiB
YAML
name: E2E Evals
|
|
on:
|
|
pull_request:
|
|
branches: [main]
|
|
workflow_dispatch:
|
|
inputs:
|
|
evals_all:
|
|
description: 'Run ALL gate tests in the sliced lane (bypass diff selection; also arms the hollow-shard guard)'
|
|
type: boolean
|
|
default: true
|
|
|
|
concurrency:
|
|
group: evals-${{ github.event.pull_request.number || github.run_id }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
IMAGE: ghcr.io/${{ github.repository }}/ci
|
|
|
|
jobs:
|
|
# Build Docker image with pre-baked toolchain (cached — only rebuilds on Dockerfile/lockfile change)
|
|
build-image:
|
|
# Dependabot-triggered pull_request runs get a read-only GITHUB_TOKEN, so
|
|
# a lockfile bump = new hash = failed ghcr push = permanently red check
|
|
# (EV6, fork port wave 2). Skip the build for dependabot; the evals job's
|
|
# needs-chain tolerates it because no eval test selects on a lockfile-only
|
|
# diff — a maintainer's next push rebuilds the image with real perms.
|
|
if: github.actor != 'dependabot[bot]'
|
|
runs-on: ubicloud-standard-8
|
|
timeout-minutes: 15
|
|
permissions:
|
|
contents: read
|
|
packages: write
|
|
outputs:
|
|
image-tag: ${{ steps.meta.outputs.tag }}
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
|
|
- id: meta
|
|
# Key on Dockerfile + lockfile only. package.json is deliberately NOT
|
|
# hashed: its version field changes on every ship (60/60 recent commits),
|
|
# which rebuilt the image each time for a dependency set that only
|
|
# bun.lock determines. A stale baked package.json is harmless — checkout
|
|
# overwrites /workspace and node_modules comes from the lockfile.
|
|
run: echo "tag=${{ env.IMAGE }}:${{ hashFiles('.github/docker/Dockerfile.ci', 'bun.lock', 'patches/**') }}" >> "$GITHUB_OUTPUT"
|
|
|
|
- uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
|
|
with:
|
|
registry: ghcr.io
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Check if image exists
|
|
id: check
|
|
run: |
|
|
if docker manifest inspect ${{ steps.meta.outputs.tag }} > /dev/null 2>&1; then
|
|
echo "exists=true" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "exists=false" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
- if: steps.check.outputs.exists == 'false'
|
|
run: cp package.json bun.lock .github/docker/ && cp -R patches .github/docker/patches
|
|
|
|
# A fork PR's GITHUB_TOKEN only has `packages: read`, so pushing fails.
|
|
# Still BUILD (validates Dockerfile.ci changes), just don't publish. This
|
|
# job intentionally keeps no `if:` so fork PRs still get one real, honest
|
|
# green check here instead of a run where every job is grey.
|
|
# Registry cache export needs a docker-container builder — the default
|
|
# `docker` driver hard-errors on cache-to (first live run of the trio).
|
|
- if: steps.check.outputs.exists == 'false'
|
|
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
|
|
|
|
- if: steps.check.outputs.exists == 'false'
|
|
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
|
|
with:
|
|
context: .github/docker
|
|
file: .github/docker/Dockerfile.ci
|
|
push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
|
|
# Registry layer cache: reads are safe everywhere; the export is gated
|
|
# to same-repo runs because a fork PR's token can't write GHCR.
|
|
cache-from: type=registry,ref=${{ env.IMAGE }}:buildcache
|
|
cache-to: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && format('type=registry,ref={0}:buildcache,mode=max', env.IMAGE) || '' }}
|
|
tags: |
|
|
${{ steps.meta.outputs.tag }}
|
|
${{ env.IMAGE }}:latest
|
|
|
|
# ── Sliced lane (the ONLY paid lane; legacy 17-row matrix deleted) ──────────
|
|
# One PLANNER computes diff selection + the slice plan ONCE (killing
|
|
# per-slice selector divergence); K executors consume the manifest; the
|
|
# report reconciles results against it FAIL-CLOSED (a slice whose artifact
|
|
# never landed is a failure, a planned shard nobody reported is a failure —
|
|
# hollow lanes cannot aggregate green). Engine: scripts/test-paid-shards.ts —
|
|
# the same runner local eval:bg:gate uses, so CI and local share one
|
|
# selection engine, and every gate-tier file is in the census by
|
|
# construction (no hand-enumerated rows to drift). The legacy matrix ran
|
|
# 18 enumerated files for 22.6 min/$21 per PR serialized AHEAD of this
|
|
# lane's 49-file diff-selected census; parity was demonstrated (sliced
|
|
# census ⊇ matrix files) and the matrix deleted — one revert restores it.
|
|
#
|
|
# Fork PRs never receive repository secrets (ANTHROPIC_API_KEY et al), so
|
|
# every API-calling eval fails at SDK auth before a model runs. Skip
|
|
# deterministically; fork work gets real coverage via a trusted base-repo
|
|
# branch (see CLAUDE.md's garrytan-agents workflow).
|
|
plan-slices:
|
|
runs-on: ubicloud-standard-8
|
|
needs: build-image
|
|
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
|
|
timeout-minutes: 10
|
|
permissions:
|
|
contents: read
|
|
packages: read
|
|
container:
|
|
image: ${{ needs.build-image.outputs.image-tag }}
|
|
credentials:
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
options: --user runner
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
# The planner is the ONE place that needs history: diff selection
|
|
# resolves a merge-base. Executors run from the manifest and stay
|
|
# shallow. Selection fails OPEN (run-all) if resolution fails — the
|
|
# documented posture; a planner bug can only run extra work.
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
|
|
- name: Restore deps
|
|
uses: ./.github/actions/restore-deps
|
|
|
|
- name: Emit run manifest
|
|
env:
|
|
EVALS_ALL: ${{ (github.event_name == 'workflow_dispatch' && inputs.evals_all) && '1' || '' }}
|
|
run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/paid-plan/manifest.json --slices 6
|
|
|
|
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: paid-plan
|
|
path: /tmp/paid-plan/manifest.json
|
|
retention-days: 30
|
|
|
|
eval-slices:
|
|
runs-on: ubicloud-standard-8
|
|
needs: [build-image, plan-slices]
|
|
if: always() && needs.plan-slices.result == 'success'
|
|
# Aggregate spawn-concurrency budget: 6 slices x EVALS_JOBS=2 x
|
|
# EVALS_CONCURRENCY=2 = 24 concurrent tests lane-wide (the old matrix's
|
|
# 40-way per row queued claude session STARTUP behind 39 siblings and ate
|
|
# per-test budgets — the documented timeout-flake family). Tune with
|
|
# parity data before raising.
|
|
timeout-minutes: 35
|
|
permissions:
|
|
contents: read
|
|
packages: read
|
|
container:
|
|
image: ${{ needs.build-image.outputs.image-tag }}
|
|
credentials:
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
options: --user runner
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
slice: [1, 2, 3, 4, 5, 6]
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
# Full history: files with SELF-derived selection (the LLM-judge
|
|
# map, routing) walk git at module load, and selection is
|
|
# fail-closed on git errors — a shallow checkout crashed those
|
|
# shards on the lane's first live run ("ambiguous argument
|
|
# 'main...HEAD'"). The manifest still governs WHICH shards run.
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
|
|
- name: Fix bun temp
|
|
uses: ./.github/actions/fix-bun-temp
|
|
|
|
- name: Restore deps
|
|
uses: ./.github/actions/restore-deps
|
|
|
|
- run: bun run build
|
|
|
|
# Any slice can host a PTY smoke, so the seed/registration steps run
|
|
# UNCONDITIONALLY (both are idempotent) — the old matrix keyed them on
|
|
# matrix.suite.name, which a sliced lane cannot do. The register
|
|
# composite carries the fail-fast dangling-symlink/frontmatter
|
|
# verification loop, so a moved skill target fails HERE in seconds,
|
|
# not as a wedged PTY session at the shard wall.
|
|
- name: Seed claude interactive config
|
|
uses: ./.github/actions/seed-claude-config
|
|
with:
|
|
anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
|
|
- name: Register gstack skills for PTY smokes
|
|
uses: ./.github/actions/register-gstack-skills
|
|
|
|
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
|
with:
|
|
name: paid-plan
|
|
path: /tmp/paid-plan
|
|
|
|
- name: Run slice ${{ matrix.slice }}/6
|
|
env:
|
|
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
|
|
PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers
|
|
EVALS_JOBS: "2"
|
|
EVALS_CONCURRENCY: "2"
|
|
GSTACK_EVAL_DIR: /tmp/paid-slice-results
|
|
run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/paid-plan/manifest.json --slice ${{ matrix.slice }}
|
|
|
|
- name: Upload slice results
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: paid-slice-${{ matrix.slice }}
|
|
path: /tmp/paid-slice-results
|
|
retention-days: 90
|
|
|
|
# The spooled per-shard full logs — a red weekly/PR lane three weeks
|
|
# later needs more than a summary line.
|
|
- name: Upload shard logs on failure
|
|
if: failure()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: paid-slice-${{ matrix.slice }}-logs
|
|
# The Fix-bun-temp step points TMPDIR at /home/runner/.cache, so the
|
|
# runner's spool lands THERE, not /tmp — the original /tmp glob
|
|
# uploaded nothing and a red slice's diagnostics were unreachable.
|
|
path: |
|
|
/home/runner/.cache/gstack-paid-shard-*.log
|
|
/tmp/gstack-paid-shard-*.log
|
|
if-no-files-found: ignore
|
|
retention-days: 30
|
|
|
|
slices-report:
|
|
runs-on: ubicloud-standard-2
|
|
needs: [plan-slices, eval-slices]
|
|
# always(): the report must run (and FAIL) when an executor died — a
|
|
# missing slice artifact reading as green is the class this lane kills.
|
|
if: always() && needs.plan-slices.result == 'success'
|
|
timeout-minutes: 5
|
|
# contents:read ONLY — this job executes PR-authored code (bun install
|
|
# lifecycle scripts + the reconcile runner from the PR checkout), so it
|
|
# must never hold a write-scoped token. The PR comment lives in the
|
|
# separate slices-comment job below, which runs NO repo code: a
|
|
# $GITHUB_ENV/BASH_ENV persistence trick is job-scoped, so the split is
|
|
# the trust boundary (codex adversarial finding, 2026-08-31 — the old
|
|
# matrix-era report job had this separation and the consolidation had
|
|
# regressed it).
|
|
permissions:
|
|
contents: read
|
|
outputs:
|
|
reconcile-exit: ${{ steps.reconcile.outputs.exit }}
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
|
with:
|
|
bun-version: 1.4.0
|
|
|
|
- run: bun install --frozen-lockfile
|
|
|
|
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
|
with:
|
|
name: paid-plan
|
|
path: /tmp/paid-report
|
|
|
|
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
|
with:
|
|
pattern: paid-slice-[0-9]*
|
|
path: /tmp/paid-report
|
|
merge-multiple: true
|
|
|
|
- name: Reconcile slices against the manifest (fail-closed)
|
|
id: reconcile
|
|
run: |
|
|
set +e
|
|
EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report /tmp/paid-report | tee /tmp/report.txt
|
|
# PIPESTATUS[0], NOT $?: GitHub's default run-step shell is
|
|
# `bash -e {0}` with NO pipefail, so $? after the pipe is tee's
|
|
# exit (always 0) — the fail-closed gate was silently fail-open
|
|
# (caught by the ship review army; the wiring test now pins this).
|
|
echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Upload reconciliation output for the comment job
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: report-verdict
|
|
path: /tmp/report.txt
|
|
if-no-files-found: ignore
|
|
retention-days: 30
|
|
|
|
- name: Fail the workflow when reconciliation failed
|
|
if: steps.reconcile.outputs.exit != '0'
|
|
run: exit 1
|
|
|
|
# PR comment in its OWN job with the write token and ZERO repo code: no
|
|
# checkout, no bun install — only downloaded artifacts, jq, and gh. See the
|
|
# trust-boundary note on slices-report.
|
|
slices-comment:
|
|
runs-on: ubicloud-standard-2
|
|
needs: slices-report
|
|
if: always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && needs.slices-report.result != 'skipped'
|
|
timeout-minutes: 5
|
|
permissions:
|
|
pull-requests: write
|
|
# The comment upsert calls the REST `/issues/{n}/comments` endpoints
|
|
# (gh api ... issues/comments). With GITHUB_TOKEN those are gated by the
|
|
# `issues` permission, not `pull-requests` (#1802 CI fix).
|
|
issues: write
|
|
steps:
|
|
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
|
with:
|
|
pattern: paid-slice-[0-9]*
|
|
path: /tmp/paid-report
|
|
merge-multiple: true
|
|
|
|
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
|
with:
|
|
name: report-verdict
|
|
path: /tmp/verdict
|
|
continue-on-error: true
|
|
|
|
# Sourced from the slice artifacts' eval-store JSONs. Keeps the
|
|
# "## E2E Evals" marker so the upsert keeps updating the same comment.
|
|
# Runs even when reconciliation failed — a red lane on the PR is the point.
|
|
- name: Post PR comment
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
RECONCILE_EXIT: ${{ needs.slices-report.outputs.reconcile-exit }}
|
|
run: |
|
|
# shellcheck disable=SC2086,SC2059
|
|
RESULTS=$(find /tmp/paid-report -name '*.json' ! -name 'manifest.json' ! -name 'slice-*.json' ! -name '_partial*' 2>/dev/null | sort)
|
|
TOTAL=0; PASSED=0; FAILED=0; FLAKY=0; COST="0"
|
|
SUITE_LINES=""
|
|
for f in $RESULTS; do
|
|
if ! jq -e '.total_tests' "$f" >/dev/null 2>&1; then
|
|
echo "Skipping malformed JSON: $f"
|
|
continue
|
|
fi
|
|
# FINAL-attempt accounting: eval-store keeps EVERY retry attempt
|
|
# as its own record (that's the flake telemetry), so counting raw
|
|
# records marks a pass-on-retry as a failure and inflates totals.
|
|
# Group by test name, judge the LAST record — flaky passes render
|
|
# as the ⚠ line, never as ❌ (WS1 policy: recorded, not blocking).
|
|
# Guarded: a file with total_tests but a null/non-array `tests`
|
|
# passes the -e probe, the group_by then fails, and an empty $T
|
|
# would abort the whole step under bash -e ([ "" -eq 0 ] is an
|
|
# error) — killing the comment on exactly the corrupted-artifact
|
|
# runs where the red evidence matters (claude adversarial).
|
|
STATS=$(jq -r '[.tests | group_by(.name)[] | last] as $final | "\($final | length) \([$final[] | select(.passed)] | length) \([$final[] | select(.passed | not)] | length) \(.flaky_retries // [] | length)"' "$f" 2>/dev/null) || { echo "Skipping malformed tests[] in: $f"; continue; }
|
|
read -r T P F FL <<< "$STATS"
|
|
[ -z "$T" ] && { echo "Skipping malformed tests[] in: $f"; continue; }
|
|
C=$(jq -r '.total_cost_usd // 0' "$f")
|
|
TIER=$(jq -r '.tier // "unknown"' "$f")
|
|
SHARD=$(jq -r '.shard // "-"' "$f")
|
|
[ "$T" -eq 0 ] && continue
|
|
TOTAL=$((TOTAL + T))
|
|
PASSED=$((PASSED + P))
|
|
FAILED=$((FAILED + F))
|
|
FLAKY=$((FLAKY + FL))
|
|
COST=$(echo "$COST + $C" | bc)
|
|
STATUS_ICON="✅"
|
|
[ "$F" -gt 0 ] && STATUS_ICON="❌"
|
|
[ "$F" -eq 0 ] && [ "$FL" -gt 0 ] && STATUS_ICON="✅⚠"
|
|
SUITE_LINES="${SUITE_LINES}| ${TIER}/${SHARD} | ${P}/${T} | ${STATUS_ICON} | \$${C} |\n"
|
|
done
|
|
|
|
STATUS="✅ PASS"
|
|
if [ "${RECONCILE_EXIT:-1}" != "0" ] || [ "$FAILED" -gt 0 ]; then STATUS="❌ FAIL"; fi
|
|
|
|
BODY="## E2E Evals: ${STATUS}
|
|
|
|
**${PASSED}/${TOTAL}** tests passed | **\$${COST}** total cost | reconcile exit: ${RECONCILE_EXIT:-missing}$([ "$FLAKY" -gt 0 ] && printf ' | ⚠ %s flaky pass(es) — recorded, not blocking' "$FLAKY")
|
|
|
|
| Shard | Result | Status | Cost |
|
|
|-------|--------|--------|------|
|
|
$(echo -e "$SUITE_LINES")
|
|
|
|
<details><summary>Fail-closed reconciliation</summary>
|
|
|
|
\`\`\`
|
|
$(tail -c 4000 /tmp/verdict/report.txt 2>/dev/null || echo '(no reconciliation output)')
|
|
\`\`\`
|
|
</details>
|
|
|
|
---
|
|
*Sliced lane: diff-selected gate census via scripts/test-paid-shards.ts (planner → 6 executors → fail-closed report)*"
|
|
|
|
if [ "$FAILED" -gt 0 ]; then
|
|
FAILURES=""
|
|
for f in $RESULTS; do
|
|
if ! jq -e '.failed' "$f" >/dev/null 2>&1; then continue; fi
|
|
FAILS=$(jq -r '[.tests | group_by(.name)[] | last | select(.passed == false)][] | "- ❌ \(.name): \(.exit_reason // "unknown")"' "$f" 2>/dev/null || echo "- ⚠️ parse error")
|
|
FAILURES="${FAILURES}${FAILS}\n"
|
|
done
|
|
BODY="${BODY}
|
|
|
|
### Failures
|
|
$(echo -e "$FAILURES")"
|
|
fi
|
|
|
|
COMMENT_ID=$(gh api repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments \
|
|
--jq '.[] | select(.body | startswith("## E2E Evals")) | .id' | tail -1)
|
|
|
|
if [ -n "$COMMENT_ID" ]; then
|
|
gh api "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" \
|
|
-X PATCH -f body="$BODY"
|
|
else
|
|
# REST, not gh's pr-comment subcommand: this job runs with NO
|
|
# checkout (the token/exec split), and that subcommand resolves
|
|
# the repo FROM git — it dies with "not a git repository" here.
|
|
gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \
|
|
-X POST -f body="$BODY"
|
|
fi
|