mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-09 14:38:59 +02:00
feat: delete the legacy 17-row eval matrix — the sliced lane is the only paid lane
Every PR paid twice: the hand-enumerated matrix (18 test files, 22.6 min, ~$21 API measured on run 33263204465) ran serialized AHEAD of the strictly superior sliced lane via 'needs: evals' — 35.5 min wall and ~2x paid spend for the same diff. 14 of 17 rows carried no tier:, so periodic Opus benchmarks leaked into every PR (the e2e-plan row alone: 12/12 tests, 21.7 min, $7.28 — the wall-clock bound of ALL of CI). Parity receipt (static, pre-deletion): the sliced lane's gate census (49 files, derived from the runner itself) strictly contains all 18 matrix test files, plus 31 files the matrix never ran. Pure deletion — one revert restores it. The PR comment moved into slices-report (same '## E2E Evals' upsert marker, now sourced from slice artifacts + carrying the fail-closed reconciliation verdict). plan-slices loses the needs edge; the dead workflow-level EVALS_TIER env goes with it. test/evals-workflow-matrix.test.ts (and its KNOWN_MATRIX_GAPS / KNOWN_TIER_UNSET burn-down ratchets — retired: the sliced census makes 'every gate file runs' true by construction) is rewritten as test/evals-workflow-wiring.test.ts: matrix stays deleted, planner/executor/ report tier + slice-count agreement, both surviving lanes on the shared register-skills composite with its fail-fast verification loop, PR comment survival. Expected: PR eval wall 35.5 -> ~13 min, per-PR paid spend ~halved. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Fable 5
Claude Fable 5
parent
0f26859bd1
commit
07c2452e6d
+111
-414
@@ -15,7 +15,6 @@ concurrency:
|
||||
|
||||
env:
|
||||
IMAGE: ghcr.io/${{ github.repository }}/ci
|
||||
EVALS_TIER: gate
|
||||
|
||||
jobs:
|
||||
# Build Docker image with pre-baked toolchain (cached — only rebuilds on Dockerfile/lockfile change)
|
||||
@@ -85,424 +84,27 @@ jobs:
|
||||
${{ steps.meta.outputs.tag }}
|
||||
${{ env.IMAGE }}:latest
|
||||
|
||||
# Fork PRs never receive repository secrets (ANTHROPIC_API_KEY et al), so every
|
||||
# API-calling eval fails at SDK auth before a model runs. Skip deterministically
|
||||
# rather than leaving the outcome to Docker-cache luck: a warm cache let these
|
||||
# run and fail, a cold one made build-image fail its push and the shards skip.
|
||||
# Same-repo PRs, pushes, and workflow_dispatch keep full coverage. Fork work
|
||||
# gets real coverage via a trusted base-repo branch.
|
||||
evals:
|
||||
runs-on: ${{ matrix.suite.runner || 'ubicloud-standard-8' }}
|
||||
needs: build-image
|
||||
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
|
||||
# Least privilege for the job that executes PR-authored code with three
|
||||
# provider API keys in env: read-only contents, packages:read for the
|
||||
# container-image pull below. Without this block the job ran on the
|
||||
# repo-default token grant.
|
||||
permissions:
|
||||
contents: read
|
||||
packages: read
|
||||
container:
|
||||
image: ${{ needs.build-image.outputs.image-tag }}
|
||||
credentials:
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
options: --user runner
|
||||
timeout-minutes: ${{ matrix.suite.timeout || 25 }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
suite:
|
||||
- name: llm-judge
|
||||
file: test/skill-llm-eval.test.ts
|
||||
- name: e2e-browse
|
||||
file: test/skill-e2e-bws.test.ts
|
||||
runner: ubicloud-standard-8
|
||||
- name: e2e-plan
|
||||
file: test/skill-e2e-plan.test.ts
|
||||
- name: e2e-deploy
|
||||
file: test/skill-e2e-deploy.test.ts
|
||||
- name: e2e-design
|
||||
file: test/skill-e2e-design.test.ts
|
||||
- name: e2e-qa-bugs
|
||||
file: test/skill-e2e-qa-bugs.test.ts
|
||||
- name: e2e-qa-workflow
|
||||
file: test/skill-e2e-qa-workflow.test.ts
|
||||
- name: e2e-review
|
||||
file: test/skill-e2e-review.test.ts
|
||||
- name: e2e-retro
|
||||
file: test/skill-e2e-retro.test.ts
|
||||
- name: e2e-review-attribution
|
||||
file: test/skill-e2e-review-attribution.test.ts
|
||||
- name: e2e-workflow
|
||||
file: test/skill-e2e-workflow.test.ts
|
||||
# Earned its extra attempt with receipts: document-release is a
|
||||
# long multi-step E2E that timed out on attempt 2 under in-shard
|
||||
# concurrency (PR #2593 round 4) while passing other rounds.
|
||||
retries: 2
|
||||
# Rehomed from the deleted pre-split monolith (its filename never
|
||||
# matched the skill-e2e-* glob, so these gate tests silently never
|
||||
# ran). Both files hold gate-tier tests: review/plan-eng coverage
|
||||
# audits and the /ship failure-ownership triage.
|
||||
- name: e2e-coverage-audit
|
||||
file: test/skill-e2e-coverage-audit.test.ts
|
||||
- name: e2e-triage
|
||||
file: test/skill-e2e-triage.test.ts
|
||||
# ship-docsync is whole-file tier-gated (describeE2ETier('gate') keeps
|
||||
# it out of the periodic shard census), so its row MUST set tier: gate
|
||||
# — without it the self-gate skips every test and the job reports a
|
||||
# hollow green (the same silent-skip class as the rehomed monolith
|
||||
# above, one layer deeper). The Run step exports EVALS_TIER from this
|
||||
# property; rows without it keep EVALS_TIER empty (= unset: every
|
||||
# reader is `=== '<tier>'` or truthiness). Enforced by
|
||||
# test/evals-workflow-matrix.test.ts.
|
||||
- name: e2e-ship-docsync
|
||||
file: test/skill-e2e-ship-docsync.test.ts
|
||||
tier: gate
|
||||
# Consent-gate guardrail for the Third-Party Web Actions contract
|
||||
# (Aside recommended-driver rewrite): hermetic shims, deterministic
|
||||
# grep assertions, gate tier in E2E_TIERS.
|
||||
- name: e2e-third-party-actions
|
||||
file: test/skill-e2e-third-party-actions.test.ts
|
||||
tier: gate
|
||||
- name: e2e-routing
|
||||
file: test/skill-routing-e2e.test.ts
|
||||
# (e2e-codex / e2e-gemini rows deleted: both files are whole-file
|
||||
# periodic-tier, so with no row tier: they ran ZERO tests and
|
||||
# reported green on every PR — ~2 min of runner per PR of pure
|
||||
# false confidence. The periodic lane owns these suites.)
|
||||
# Real-PTY plan-mode smokes. Only the deterministically-reliable ones
|
||||
# are CI-gated: office-hours (asks its mode question first, caught by
|
||||
# the collapsed/bullet prose-AUQ detector) and plan-mode-no-op (no
|
||||
# ask-first dependency). The plan-eng/plan-design plan-mode + floor
|
||||
# smokes are periodic (stochastic ask-first — see touchfiles E2E_TIERS).
|
||||
# Needs the interactive-config seed step below; PTY sessions otherwise
|
||||
# wedge on the fresh-container onboarding/API-key dialog.
|
||||
- name: e2e-pty-plan-smoke
|
||||
file: test/skill-e2e-office-hours-auto-mode.test.ts test/skill-e2e-plan-mode-no-op.test.ts
|
||||
# Both files are whole-file describeE2ETier('gate') — without this
|
||||
# row tier: the job burned ~7 min of setup then skipped every
|
||||
# describe (hollow-green since the files adopted the self-gate).
|
||||
tier: gate
|
||||
timeout: 35
|
||||
# The documented contention-heavy PTY family: ROTATING members
|
||||
# failed attempt 2 in consecutive PR #2593 rounds
|
||||
# (plan-design-review, then plan-eng-review) while the family
|
||||
# passes on branches still running three attempts. Every other
|
||||
# row keeps --retry 1.
|
||||
retries: 2
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
# Don't write the token into .git/config — this job runs
|
||||
# PR-authored code; nothing in it pushes.
|
||||
persist-credentials: false
|
||||
|
||||
# Bun creates root-owned temp dirs during Docker build. GH Actions runs as
|
||||
# runner user with HOME=/github/home. Redirect bun's cache to a writable dir.
|
||||
- name: Fix bun temp
|
||||
run: |
|
||||
mkdir -p /home/runner/.cache/bun
|
||||
{
|
||||
echo "BUN_INSTALL_CACHE_DIR=/home/runner/.cache/bun"
|
||||
echo "BUN_TMPDIR=/home/runner/.cache/bun"
|
||||
echo "TMPDIR=/home/runner/.cache"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
# Restore pre-installed node_modules from Docker image via recursive
|
||||
# copy. Symlink (`ln -s`) breaks bun's module resolution because bun
|
||||
# resolves a file's realpath when walking up to find node_modules/<dep>;
|
||||
# from a symlinked path, realpath escapes the workspace and sibling
|
||||
# deps no longer resolve. Hardlink copy (`cp -al`) fails because /opt
|
||||
# and /workspace are on different overlay-fs layers ("Invalid
|
||||
# cross-device link"). Recursive copy works on every layout. Cost:
|
||||
# ~5s for ~200 packages of small JS files vs ~0s for symlink — still
|
||||
# vastly cheaper than rerunning `bun install` (network + resolution).
|
||||
- name: Restore deps
|
||||
run: |
|
||||
if [ -d /opt/node_modules_cache ] && diff -q /opt/node_modules_cache/.bun.lock bun.lock >/dev/null 2>&1; then
|
||||
cp -r /opt/node_modules_cache node_modules
|
||||
else
|
||||
bun install
|
||||
fi
|
||||
|
||||
- run: bun run build
|
||||
|
||||
# Verify Playwright can launch Chromium (fails fast if sandbox/deps are broken)
|
||||
- name: Verify Chromium
|
||||
if: matrix.suite.name == 'e2e-browse'
|
||||
run: |
|
||||
echo "whoami=$(whoami) HOME=$HOME TMPDIR=${TMPDIR:-unset}"
|
||||
touch /tmp/.bun-test && rm /tmp/.bun-test && echo "/tmp writable"
|
||||
bun -e "import {chromium} from 'playwright';const b=await chromium.launch({args:['--no-sandbox']});console.log('Chromium OK');await b.close()"
|
||||
|
||||
# PTY smokes spawn the interactive `claude` TUI. A fresh container has no
|
||||
# ~/.claude.json, so claude wedges on the onboarding + "use detected
|
||||
# ANTHROPIC_API_KEY?" dialog and the spawned session never reaches the
|
||||
# skill. Seed onboarding-complete + the key approval (mirrors what the
|
||||
# hermetic E2E child env seeds). Scoped to this suite; needs its OWN key
|
||||
# env (the secrets block below is on the Run step only).
|
||||
- name: Seed claude interactive config
|
||||
if: matrix.suite.name == 'e2e-pty-plan-smoke'
|
||||
env:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
run: |
|
||||
node -e '
|
||||
const fs = require("fs"), os = require("os"), path = require("path");
|
||||
const p = path.join(os.homedir(), ".claude.json");
|
||||
const seed = fs.existsSync(p) ? JSON.parse(fs.readFileSync(p, "utf8")) : {};
|
||||
seed.hasCompletedOnboarding = true;
|
||||
const key = process.env.ANTHROPIC_API_KEY || "";
|
||||
if (key) seed.customApiKeyResponses = { approved: [key.slice(-20)], rejected: [] };
|
||||
fs.writeFileSync(p, JSON.stringify(seed, null, 2));
|
||||
console.log("seeded", p);
|
||||
'
|
||||
|
||||
# PTY smokes drive the interactive `claude` TUI and send /office-hours,
|
||||
# /plan-ceo-review, /plan-eng-review, and /plan-design-review. Claude Code
|
||||
# discovers user-scoped skills from $HOME/.claude/skills/<name>/SKILL.md,
|
||||
# but .claude/skills is gitignored, so a fresh CI checkout has NO registry
|
||||
# — claude prints "Unknown command: /plan-ceo-review". Mirror setup's
|
||||
# --no-prefix registry minimally: a gstack root symlink (resolves the
|
||||
# preamble's absolute ~/.claude/skills/gstack/bin/* and
|
||||
# ~/.claude/skills/gstack/<skill>/sections/* paths) plus a per-skill
|
||||
# top-level dir holding SKILL.md (+ sections) symlinks for the four skills
|
||||
# these tests invoke. No ./setup (it builds binaries, launches Chromium,
|
||||
# installs fonts, reads a /dev/tty prompt) and no binary build (SKILL.md +
|
||||
# bin/ + sections/ are committed). $HOME is /github/home here; the spawned
|
||||
# claude inherits it (this runner adds no HOME/CLAUDE_CONFIG_DIR override,
|
||||
# no hermetic mode) and the Seed step already proved claude reads $HOME.
|
||||
#
|
||||
# KEEP THIS STEP even though seedSkills/hermeticSkillsConfigDir() now
|
||||
# registers skills for hermetic PTY children: that registry is SYMLINKS
|
||||
# into the repo checkout, and this container's cross-mount symlinks
|
||||
# defeat the TUI skill scanner (see the note inside the step below) —
|
||||
# the real-file copies here are what the TUI actually reads. HOME is
|
||||
# also not hermeticized, so the absolute ~/.claude/skills/gstack/...
|
||||
# preamble paths resolve through the gstack root symlink this step makes.
|
||||
- name: Register gstack skills for PTY smoke
|
||||
if: matrix.suite.name == 'e2e-pty-plan-smoke'
|
||||
run: |
|
||||
set -eu
|
||||
SKILLS_DIR="$HOME/.claude/skills"
|
||||
REPO="$GITHUB_WORKSPACE" # /__w/gstack/gstack
|
||||
mkdir -p "$SKILLS_DIR"
|
||||
# The gstack root stays a symlink — the preamble's runtime bash resolves
|
||||
# ~/.claude/skills/gstack/bin/* and ~/.claude/skills/gstack/<skill>/sections/*
|
||||
# through it, and bash follows cross-mount symlinks fine.
|
||||
ln -snf "$REPO" "$SKILLS_DIR/gstack"
|
||||
# But the per-skill SKILL.md the TUI DISCOVERS must be a REAL file on the
|
||||
# same mount as $HOME. claude 2.1.187's interactive-TUI skill scanner does
|
||||
# not follow the /github/home -> /__w cross-mount symlink (proven: `claude
|
||||
# -p` discovered the skill — READY — while the TUI rejected /office-hours
|
||||
# as "Unknown command"; a local macOS repro with the identical symlinked
|
||||
# registry recognized it, isolating the failure to the container's
|
||||
# cross-mount symlink). Copy SKILL.md + sections as real files so the TUI
|
||||
# reads them directly.
|
||||
for s in office-hours plan-ceo-review plan-eng-review plan-design-review; do
|
||||
rm -rf "${SKILLS_DIR:?}/$s"
|
||||
mkdir -p "$SKILLS_DIR/$s"
|
||||
cp "$REPO/$s/SKILL.md" "$SKILLS_DIR/$s/SKILL.md"
|
||||
cp -R "$REPO/$s/sections" "$SKILLS_DIR/$s/sections"
|
||||
done
|
||||
# Also register PROJECT-scoped (cwd) skills. claude's interactive TUI
|
||||
# surfaces /slash commands from <cwd>/.claude/skills, and the smokes run
|
||||
# with cwd=$REPO whose .claude/skills is gitignored (absent on a fresh CI
|
||||
# checkout) — the user-dir registration above feeds `claude -p` but the
|
||||
# TUI looks here. No gstack symlink in the project dir: it would point at
|
||||
# its own parent ($REPO). Runtime preamble paths use the user-dir
|
||||
# ~/.claude/skills/gstack symlink above.
|
||||
PROJ_SKILLS="$REPO/.claude/skills"
|
||||
mkdir -p "$PROJ_SKILLS"
|
||||
for s in office-hours plan-ceo-review plan-eng-review plan-design-review; do
|
||||
rm -rf "${PROJ_SKILLS:?}/$s"
|
||||
mkdir -p "$PROJ_SKILLS/$s"
|
||||
cp "$REPO/$s/SKILL.md" "$PROJ_SKILLS/$s/SKILL.md"
|
||||
cp -R "$REPO/$s/sections" "$PROJ_SKILLS/$s/sections"
|
||||
done
|
||||
# Pre-seed every ONE-TIME preamble marker so no PTY child ever takes a
|
||||
# first-run branch mid-test. On a fresh runner these are all missing, so
|
||||
# each smoke's preamble fires feature discovery / telemetry / lake-intro
|
||||
# prompts before the behavior under test — and touching the
|
||||
# feature-discovery marker under ~/.claude/skills/gstack/ trips Claude
|
||||
# Code's sensitive-file permission prompt, stalling the run before the
|
||||
# scope gate renders (the documented intermittent
|
||||
# scope-gate-question-NOT-observed failure: outcome=asked was the
|
||||
# permission dialog, not the gate). Dev machines never hit this because
|
||||
# the operator's markers already exist; CI must seed them explicitly.
|
||||
mkdir -p "$HOME/.gstack"
|
||||
touch "$HOME/.gstack/.activated" \
|
||||
"$HOME/.gstack/.first-loop-tip-shown" \
|
||||
"$HOME/.gstack/.telemetry-prompted" \
|
||||
"$HOME/.gstack/.proactive-prompted" \
|
||||
"$HOME/.gstack/.completeness-intro-seen" \
|
||||
"$HOME/.gstack/.plan-tune-nudge-shown"
|
||||
# These two resolve through the gstack root symlink into $REPO —
|
||||
# untracked scratch in the CI checkout, exactly where the preamble looks.
|
||||
touch "$SKILLS_DIR/gstack/.feature-prompted-continuous-checkpoint" \
|
||||
"$SKILLS_DIR/gstack/.feature-prompted-model-overlay"
|
||||
echo "--- registry under $SKILLS_DIR ---"
|
||||
ls -la "$SKILLS_DIR/gstack" "$SKILLS_DIR/office-hours" "$SKILLS_DIR/plan-ceo-review"
|
||||
# Fail fast if any committed target moved/renamed — a dangling symlink
|
||||
# would otherwise resurface as a silent "Unknown command" + 35-min timeout.
|
||||
for f in \
|
||||
"$SKILLS_DIR/office-hours/SKILL.md" \
|
||||
"$SKILLS_DIR/plan-ceo-review/SKILL.md" \
|
||||
"$SKILLS_DIR/plan-eng-review/SKILL.md" \
|
||||
"$SKILLS_DIR/plan-design-review/SKILL.md" \
|
||||
"$SKILLS_DIR/gstack/bin/gstack-update-check" \
|
||||
"$SKILLS_DIR/gstack/office-hours/sections/design-and-handoff.md" \
|
||||
"$SKILLS_DIR/gstack/plan-ceo-review/sections/review-sections.md" \
|
||||
"$SKILLS_DIR/gstack/plan-eng-review/sections/review-sections.md" \
|
||||
"$SKILLS_DIR/gstack/plan-design-review/sections/review-sections.md"; do
|
||||
if [ ! -e "$f" ]; then
|
||||
echo "ERROR: skill-registry target missing (symlink dangles): $f" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
for s in office-hours plan-ceo-review plan-eng-review plan-design-review; do
|
||||
grep -m1 "^name: $s\$" "$SKILLS_DIR/$s/SKILL.md" >/dev/null \
|
||||
|| { echo "ERROR: $s SKILL.md missing 'name: $s' frontmatter" >&2; exit 1; }
|
||||
done
|
||||
echo "skill registry OK"
|
||||
|
||||
- name: Run ${{ matrix.suite.name }}
|
||||
env:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
|
||||
EVALS_CONCURRENCY: "40"
|
||||
PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers
|
||||
# Per-row tier activation for whole-file-gated suites. Empty when the
|
||||
# row declares no tier — every EVALS_TIER reader treats empty as unset
|
||||
# (`=== '<tier>'` comparisons and the truthiness check in
|
||||
# test/helpers/e2e-helpers.ts:70), so untiered rows are byte-for-byte
|
||||
# unaffected.
|
||||
EVALS_TIER: ${{ matrix.suite.tier || '' }}
|
||||
run: EVALS=1 bun test --retry ${{ matrix.suite.retries || 1 }} --concurrent --max-concurrency 40 ${{ matrix.suite.file }}
|
||||
|
||||
- name: Upload eval results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: eval-${{ matrix.suite.name }}
|
||||
path: ~/.gstack-dev/evals/*.json
|
||||
retention-days: 90
|
||||
|
||||
report:
|
||||
runs-on: ubicloud-standard-2
|
||||
needs: evals
|
||||
if: always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
# The comment upsert below calls the REST `/issues/{n}/comments` endpoints
|
||||
# (gh api ... issues/comments). With GITHUB_TOKEN those are gated by the
|
||||
# `issues` permission, not `pull-requests` — without it the GET returns 401
|
||||
# on every PR that produces eval artifacts (PRs with no artifacts exit
|
||||
# early and never hit it, which is why this stayed hidden). See #1802 CI fix.
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Download all eval artifacts
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: eval-*
|
||||
path: /tmp/eval-results
|
||||
merge-multiple: true
|
||||
|
||||
- name: Post PR comment
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
# shellcheck disable=SC2086,SC2059
|
||||
RESULTS=$(find /tmp/eval-results -name '*.json' 2>/dev/null | sort)
|
||||
if [ -z "$RESULTS" ]; then
|
||||
echo "No eval results found"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
TOTAL=0; PASSED=0; FAILED=0; COST="0"
|
||||
SUITE_LINES=""
|
||||
for f in $RESULTS; do
|
||||
if ! jq -e '.total_tests' "$f" >/dev/null 2>&1; then
|
||||
echo "Skipping malformed JSON: $f"
|
||||
continue
|
||||
fi
|
||||
T=$(jq -r '.total_tests // 0' "$f")
|
||||
P=$(jq -r '.passed // 0' "$f")
|
||||
F=$(jq -r '.failed // 0' "$f")
|
||||
C=$(jq -r '.total_cost_usd // 0' "$f")
|
||||
TIER=$(jq -r '.tier // "unknown"' "$f")
|
||||
[ "$T" -eq 0 ] && continue
|
||||
TOTAL=$((TOTAL + T))
|
||||
PASSED=$((PASSED + P))
|
||||
FAILED=$((FAILED + F))
|
||||
COST=$(echo "$COST + $C" | bc)
|
||||
STATUS_ICON="✅"
|
||||
[ "$F" -gt 0 ] && STATUS_ICON="❌"
|
||||
SUITE_LINES="${SUITE_LINES}| ${TIER} | ${P}/${T} | ${STATUS_ICON} | \$${C} |\n"
|
||||
done
|
||||
|
||||
STATUS="✅ PASS"
|
||||
[ "$FAILED" -gt 0 ] && STATUS="❌ FAIL"
|
||||
|
||||
BODY="## E2E Evals: ${STATUS}
|
||||
|
||||
**${PASSED}/${TOTAL}** tests passed | **\$${COST}** total cost
|
||||
|
||||
| Suite | Result | Status | Cost |
|
||||
|-------|--------|--------|------|
|
||||
$(echo -e "$SUITE_LINES")
|
||||
|
||||
---
|
||||
*ubicloud-standard-8 runners (Docker: pre-baked toolchain + deps) | wall clock ≈ slowest suite*"
|
||||
|
||||
if [ "$FAILED" -gt 0 ]; then
|
||||
FAILURES=""
|
||||
for f in $RESULTS; do
|
||||
if ! jq -e '.failed' "$f" >/dev/null 2>&1; then continue; fi
|
||||
F=$(jq -r '.failed // 0' "$f")
|
||||
[ "$F" -eq 0 ] && continue
|
||||
FAILS=$(jq -r '.tests[] | select(.passed == false) | "- ❌ \(.name): \(.exit_reason // "unknown")"' "$f" 2>/dev/null || echo "- ⚠️ $(basename "$f"): parse error")
|
||||
FAILURES="${FAILURES}${FAILS}\n"
|
||||
done
|
||||
BODY="${BODY}
|
||||
|
||||
### Failures
|
||||
$(echo -e "$FAILURES")"
|
||||
fi
|
||||
|
||||
# Update existing comment or create new one
|
||||
COMMENT_ID=$(gh api repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments \
|
||||
--jq '.[] | select(.body | startswith("## E2E Evals")) | .id' | tail -1)
|
||||
|
||||
if [ -n "$COMMENT_ID" ]; then
|
||||
gh api "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" \
|
||||
-X PATCH -f body="$BODY"
|
||||
else
|
||||
gh pr comment "${{ github.event.pull_request.number }}" --body "$BODY"
|
||||
fi
|
||||
|
||||
# ── Sliced lane (paid-CI re-platform, parity phase) ─────────────────────────
|
||||
# ── Sliced lane (the ONLY paid lane; legacy 17-row matrix deleted) ──────────
|
||||
# One PLANNER computes diff selection + the slice plan ONCE (killing
|
||||
# per-slice selector divergence); K executors consume the manifest; the
|
||||
# report reconciles results against it FAIL-CLOSED (a slice whose artifact
|
||||
# never landed is a failure, a planned shard nobody reported is a failure —
|
||||
# hollow lanes cannot aggregate green). Runs AFTER the matrix (`needs:
|
||||
# evals`) so provider concurrency never doubles while both lanes coexist;
|
||||
# once parity is demonstrated the matrix + its ratchets are deleted and this
|
||||
# lane loses the needs edge. Engine: scripts/test-paid-shards.ts — the same
|
||||
# runner local eval:bg:gate uses, so CI and local share one selection engine.
|
||||
# hollow lanes cannot aggregate green). Engine: scripts/test-paid-shards.ts —
|
||||
# the same runner local eval:bg:gate uses, so CI and local share one
|
||||
# selection engine, and every gate-tier file is in the census by
|
||||
# construction (no hand-enumerated rows to drift). The legacy matrix ran
|
||||
# 18 enumerated files for 22.6 min/$21 per PR serialized AHEAD of this
|
||||
# lane's 49-file diff-selected census; parity was demonstrated (sliced
|
||||
# census ⊇ matrix files) and the matrix deleted — one revert restores it.
|
||||
#
|
||||
# Fork PRs never receive repository secrets (ANTHROPIC_API_KEY et al), so
|
||||
# every API-calling eval fails at SDK auth before a model runs. Skip
|
||||
# deterministically; fork work gets real coverage via a trusted base-repo
|
||||
# branch (see CLAUDE.md's garrytan-agents workflow).
|
||||
plan-slices:
|
||||
runs-on: ubicloud-standard-8
|
||||
needs: [build-image, evals]
|
||||
if: always() && needs.build-image.result == 'success' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
|
||||
needs: build-image
|
||||
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -642,6 +244,11 @@ jobs:
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
# The comment upsert calls the REST `/issues/{n}/comments` endpoints
|
||||
# (gh api ... issues/comments). With GITHUB_TOKEN those are gated by the
|
||||
# `issues` permission, not `pull-requests` (#1802 CI fix).
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
@@ -665,4 +272,94 @@ jobs:
|
||||
merge-multiple: true
|
||||
|
||||
- name: Reconcile slices against the manifest (fail-closed)
|
||||
run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report /tmp/paid-report
|
||||
id: reconcile
|
||||
run: |
|
||||
set +e
|
||||
EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report /tmp/paid-report | tee /tmp/report.txt
|
||||
echo "exit=$?" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# PR comment sourced from the slice artifacts' eval-store JSONs
|
||||
# (shards/<slug>/*.json inside each paid-slice-N artifact). Replaces the
|
||||
# deleted matrix-artifact report job; keeps the "## E2E Evals" marker so
|
||||
# the upsert keeps updating the same comment across the migration. Runs
|
||||
# even when reconciliation failed — a red lane on the PR is the point.
|
||||
- name: Post PR comment
|
||||
if: always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
RECONCILE_EXIT: ${{ steps.reconcile.outputs.exit }}
|
||||
run: |
|
||||
# shellcheck disable=SC2086,SC2059
|
||||
RESULTS=$(find /tmp/paid-report -name '*.json' ! -name 'manifest.json' ! -name 'slice-*.json' ! -name '_partial*' 2>/dev/null | sort)
|
||||
TOTAL=0; PASSED=0; FAILED=0; COST="0"
|
||||
SUITE_LINES=""
|
||||
for f in $RESULTS; do
|
||||
if ! jq -e '.total_tests' "$f" >/dev/null 2>&1; then
|
||||
echo "Skipping malformed JSON: $f"
|
||||
continue
|
||||
fi
|
||||
T=$(jq -r '.total_tests // 0' "$f")
|
||||
P=$(jq -r '.passed // 0' "$f")
|
||||
F=$(jq -r '.failed // 0' "$f")
|
||||
C=$(jq -r '.total_cost_usd // 0' "$f")
|
||||
TIER=$(jq -r '.tier // "unknown"' "$f")
|
||||
SHARD=$(jq -r '.shard // "-"' "$f")
|
||||
[ "$T" -eq 0 ] && continue
|
||||
TOTAL=$((TOTAL + T))
|
||||
PASSED=$((PASSED + P))
|
||||
FAILED=$((FAILED + F))
|
||||
COST=$(echo "$COST + $C" | bc)
|
||||
STATUS_ICON="✅"
|
||||
[ "$F" -gt 0 ] && STATUS_ICON="❌"
|
||||
SUITE_LINES="${SUITE_LINES}| ${TIER}/${SHARD} | ${P}/${T} | ${STATUS_ICON} | \$${C} |\n"
|
||||
done
|
||||
|
||||
STATUS="✅ PASS"
|
||||
if [ "${RECONCILE_EXIT:-1}" != "0" ] || [ "$FAILED" -gt 0 ]; then STATUS="❌ FAIL"; fi
|
||||
|
||||
BODY="## E2E Evals: ${STATUS}
|
||||
|
||||
**${PASSED}/${TOTAL}** tests passed | **\$${COST}** total cost | reconcile exit: ${RECONCILE_EXIT:-missing}
|
||||
|
||||
| Shard | Result | Status | Cost |
|
||||
|-------|--------|--------|------|
|
||||
$(echo -e "$SUITE_LINES")
|
||||
|
||||
<details><summary>Fail-closed reconciliation</summary>
|
||||
|
||||
\`\`\`
|
||||
$(tail -c 4000 /tmp/report.txt 2>/dev/null || echo '(no reconciliation output)')
|
||||
\`\`\`
|
||||
</details>
|
||||
|
||||
---
|
||||
*Sliced lane: diff-selected gate census via scripts/test-paid-shards.ts (planner → 6 executors → fail-closed report)*"
|
||||
|
||||
if [ "$FAILED" -gt 0 ]; then
|
||||
FAILURES=""
|
||||
for f in $RESULTS; do
|
||||
if ! jq -e '.failed' "$f" >/dev/null 2>&1; then continue; fi
|
||||
F=$(jq -r '.failed // 0' "$f")
|
||||
[ "$F" -eq 0 ] && continue
|
||||
FAILS=$(jq -r '.tests[] | select(.passed == false) | "- ❌ \(.name): \(.exit_reason // "unknown")"' "$f" 2>/dev/null || echo "- ⚠️ $(basename "$f"): parse error")
|
||||
FAILURES="${FAILURES}${FAILS}\n"
|
||||
done
|
||||
BODY="${BODY}
|
||||
|
||||
### Failures
|
||||
$(echo -e "$FAILURES")"
|
||||
fi
|
||||
|
||||
COMMENT_ID=$(gh api repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments \
|
||||
--jq '.[] | select(.body | startswith("## E2E Evals")) | .id' | tail -1)
|
||||
|
||||
if [ -n "$COMMENT_ID" ]; then
|
||||
gh api "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" \
|
||||
-X PATCH -f body="$BODY"
|
||||
else
|
||||
gh pr comment "${{ github.event.pull_request.number }}" --body "$BODY"
|
||||
fi
|
||||
|
||||
- name: Fail the workflow when reconciliation failed
|
||||
if: steps.reconcile.outputs.exit != '0'
|
||||
run: exit 1
|
||||
|
||||
@@ -567,14 +567,15 @@ duration-packed free shards, the sharded paid runner as the CI engine
|
||||
coverage contract + gate census, eval-budget timeout tiers, and the
|
||||
coverage fill. Remaining, in rough priority order:
|
||||
|
||||
- **P1 — Delete the legacy evals.yml matrix after parity.** The sliced lane
|
||||
runs alongside the 18-row matrix (`needs: evals`, so provider concurrency
|
||||
never doubles). After 1-2 PR cycles of parity (compare executed-test sets:
|
||||
intersection strict + the 8 KNOWN_MATRIX_GAPS files as expected additions;
|
||||
stochastic outcomes informational), delete the matrix as a PURE-DELETION
|
||||
commit (one revert restores it), drop the `needs: evals` edge, rewrite
|
||||
test/evals-workflow-matrix.test.ts into a runner-wiring pin, and retire
|
||||
KNOWN_MATRIX_GAPS/KNOWN_TIER_UNSET wholesale. Effort S.
|
||||
- **DONE (v1.76 test-infra wave) — Delete the legacy evals.yml matrix after
|
||||
parity.** Deleted as a pure-deletion commit (one revert restores it) after
|
||||
a static parity receipt: sliced gate census (49 files) ⊇ matrix files (18),
|
||||
31 files of extra coverage. `needs: evals` edge dropped, PR comment moved
|
||||
into slices-report, KNOWN_MATRIX_GAPS/KNOWN_TIER_UNSET retired,
|
||||
test/evals-workflow-matrix.test.ts rewritten as
|
||||
test/evals-workflow-wiring.test.ts. The register-skills fail-fast
|
||||
verification loop was ported to the surviving lanes FIRST via the shared
|
||||
.github/actions/register-gstack-skills composite.
|
||||
- **P1 — Maintainer decision: make `slices-report` a required check** once
|
||||
post-migration flake data exists (the Codex outside-voice's "green means
|
||||
green is not delivered while paid stays advisory" point — correct, and
|
||||
|
||||
@@ -76,10 +76,13 @@ slice-result artifacts; `--report <dir>` reconciles them FAIL-CLOSED (a slice
|
||||
whose artifact never landed, or a planned shard nobody reported, is a
|
||||
failure). Under `EVALS_ALL` the hollow-shard guard marks exit-0 shards with
|
||||
ZERO executed tests `passed-empty` (a failure) — census-health, not just
|
||||
test runs. evals.yml runs the sliced gate lane per PR (parity phase:
|
||||
alongside the legacy matrix, `needs:`-sequenced so provider concurrency
|
||||
never doubles; the matrix and its `KNOWN_MATRIX_GAPS`/`KNOWN_TIER_UNSET`
|
||||
ratchets are deleted after demonstrated parity). evals-periodic.yml runs ALL
|
||||
test runs. evals.yml runs the sliced gate lane per PR — the ONLY paid lane
|
||||
since the legacy 17-row matrix (22.6 min/$21 per PR serialized ahead of the
|
||||
slices) was deleted after demonstrated parity; its
|
||||
`KNOWN_MATRIX_GAPS`/`KNOWN_TIER_UNSET` ratchets retired with it and
|
||||
`test/evals-workflow-wiring.test.ts` pins the surviving wiring (slice-count
|
||||
agreement, tier consistency, the shared register-skills composite with its
|
||||
fail-fast verification loop). evals-periodic.yml runs ALL
|
||||
periodic-tier files weekly (the coverage contract) minus the reasoned
|
||||
exclusions in `test/helpers/periodic-exclude-data.ts` (reason + tracking
|
||||
required per entry; removal re-activates the file), plus a weekly
|
||||
|
||||
@@ -42,8 +42,8 @@ const SELF_GATE_RE = /EVALS_TIER\s*===\s*['"](gate|periodic)['"]/g;
|
||||
const HELPER_GATE_RE = /\b(?:describeE2ETier|e2eTierEnabled)\(\s*['"](gate|periodic)['"]/g;
|
||||
|
||||
/**
|
||||
* Ratchet, not amnesty (same contract as KNOWN_MATRIX_GAPS in
|
||||
* test/evals-workflow-matrix.test.ts): skill-e2e files that are named in NO
|
||||
* Ratchet, not amnesty (the contract KNOWN_MATRIX_GAPS pioneered before the
|
||||
* legacy eval matrix and its test retired): skill-e2e files that are named in NO
|
||||
* touchfiles dep list because no E2E map key exists for them. Every entry
|
||||
* carries a one-line reason. Do NOT add new files here — give the test an
|
||||
* E2E map key (touchfiles + tier) and register the file in its dep list.
|
||||
@@ -141,8 +141,8 @@ describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () =>
|
||||
expect(misaligned).toEqual([]);
|
||||
});
|
||||
|
||||
// Ratchet cleanup enforcement (same contract as evals-workflow-matrix's
|
||||
// burn-down test): a KNOWN_UNREGISTERED entry whose file was deleted, or
|
||||
// Ratchet cleanup enforcement (same shrink-only contract as the retired
|
||||
// matrix test's burn-down): a KNOWN_UNREGISTERED entry whose file was deleted, or
|
||||
// whose file is now named in a dep list, is stale — delete the entry so
|
||||
// the set can only shrink.
|
||||
test('KNOWN_UNREGISTERED holds only live, still-unregistered files', () => {
|
||||
|
||||
@@ -1,197 +0,0 @@
|
||||
/**
|
||||
* CI eval-matrix completeness tripwire — kills the silent-skip class where a
|
||||
* gate-tier test exists in the repo but the hand-enumerated matrix in
|
||||
* .github/workflows/evals.yml never runs it, so "gate tier blocks merge" is
|
||||
* quietly false in CI. This has happened before (see the "rehomed from the
|
||||
* deleted pre-split monolith" comment in evals.yml) and was found again on
|
||||
* PR #2700: nine gate-hosting files absent from the matrix, plus matrix rows
|
||||
* whose whole-file tier guards can never fire because the Run step exported
|
||||
* no EVALS_TIER.
|
||||
*
|
||||
* Ratchet, not amnesty: the KNOWN_* lists below enumerate the PRE-EXISTING
|
||||
* gaps with reasons, so no NEW gap can land while the backlog burns down
|
||||
* (same pattern as SCANNER_EXEMPT in egress-receipt-wiring). If you fix a
|
||||
* listed gap (add its matrix row / tier property), this test FAILS until you
|
||||
* remove the entry — stale exemptions are enforced, not decorative.
|
||||
*
|
||||
* Wiring pinned:
|
||||
* - every matrix `file:` path exists on disk (no stale rows),
|
||||
* - every gate-hosting paid file (whole-file gate self-gate, or named in the
|
||||
* dep list of a gate-tier E2E_TOUCHFILES key) appears in the matrix or in
|
||||
* KNOWN_MATRIX_GAPS,
|
||||
* - every matrix file with a whole-file tier guard has a matching row-level
|
||||
* `tier:` property (else the suite self-skips and the job is hollow-green)
|
||||
* or sits in KNOWN_TIER_UNSET.
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { E2E_TOUCHFILES, E2E_TIERS } from './helpers/touchfiles-data';
|
||||
import { isPaidTestFile } from './helpers/paid-test-set';
|
||||
|
||||
const ROOT = path.join(import.meta.dir, '..');
|
||||
const WORKFLOW = path.join(ROOT, '.github', 'workflows', 'evals.yml');
|
||||
|
||||
/**
|
||||
* Pre-existing gate-hosting files with no matrix row (found 2026-08-26,
|
||||
* PR #2700). Adding a row activates real paid runs on every PR — a cost and
|
||||
* flake-surface decision per file, tracked in TODOS.md ("CI gate-lane
|
||||
* hollow-coverage burn-down"). Fix = add a matrix row (plus `tier: gate` when
|
||||
* the file is whole-file gated), then DELETE the entry here.
|
||||
*/
|
||||
const KNOWN_MATRIX_GAPS = new Set([
|
||||
'test/skill-e2e-ask-user-question-format-compliance.test.ts',
|
||||
'test/skill-e2e-hermetic-canary.test.ts',
|
||||
'test/skill-e2e-ios.test.ts',
|
||||
'test/skill-e2e-plan-ceo-finding-floor.test.ts',
|
||||
'test/skill-e2e-plan-ceo-plan-mode.test.ts',
|
||||
'test/skill-e2e-plan-design-with-ui.test.ts',
|
||||
'test/skill-e2e-plan-devex-finding-floor.test.ts',
|
||||
'test/skill-e2e-plan-devex-plan-mode.test.ts',
|
||||
// Exposed by the 2026-08 dep-list self-registration sweep: these eight had
|
||||
// zero gate-key dep-list membership before it, so the census never saw
|
||||
// them as gate-hosting. Their gate tests run in NO CI lane today. The
|
||||
// paid-lane re-platform (test-paid-shards.ts as the CI engine) runs every
|
||||
// gate-tier file by construction and retires this whole ratchet.
|
||||
'test/skill-e2e-cso.test.ts',
|
||||
'test/skill-e2e-diagram.test.ts',
|
||||
'test/skill-e2e-learnings.test.ts',
|
||||
'test/skill-e2e-plan-tune.test.ts',
|
||||
'test/skill-e2e-plan-tune-cathedral.test.ts',
|
||||
'test/skill-e2e-review-army.test.ts',
|
||||
'test/skill-e2e-session-intelligence.test.ts',
|
||||
'test/skill-e2e-skillify.test.ts',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Matrix files whose whole-file tier guard has no matching row `tier:`
|
||||
* property. Burned down to empty 2026-08-29: the vestigial codex/gemini rows
|
||||
* were deleted (periodic-tier files, zero tests per PR) and
|
||||
* e2e-pty-plan-smoke gained its `tier: gate`. The ratchet stays so a future
|
||||
* row/file tier mismatch fails the suite instead of shipping hollow green.
|
||||
*/
|
||||
const KNOWN_TIER_UNSET = new Map<string, string>([]);
|
||||
|
||||
interface MatrixRow {
|
||||
name: string;
|
||||
files: string[];
|
||||
tier?: string;
|
||||
}
|
||||
|
||||
/** Parse the `matrix: suite:` rows (name / file / optional tier) from evals.yml. */
|
||||
function parseMatrixRows(source: string): MatrixRow[] {
|
||||
const rows: MatrixRow[] = [];
|
||||
let current: MatrixRow | null = null;
|
||||
for (const line of source.split('\n')) {
|
||||
const name = line.match(/^\s+- name: (\S+)\s*$/);
|
||||
if (name) {
|
||||
if (current) rows.push(current);
|
||||
current = { name: name[1], files: [] };
|
||||
continue;
|
||||
}
|
||||
if (!current) continue;
|
||||
const file = line.match(/^\s+file: (.+?)\s*$/);
|
||||
if (file) current.files.push(...file[1].trim().split(/\s+/));
|
||||
const tier = line.match(/^\s+tier: (\S+)\s*$/);
|
||||
if (tier) current.tier = tier[1];
|
||||
// `steps:` ends the strategy block — stop before step-level keys leak in.
|
||||
if (/^\s{4}steps:\s*$/.test(line)) break;
|
||||
}
|
||||
if (current) rows.push(current);
|
||||
return rows.filter((r) => r.files.length > 0);
|
||||
}
|
||||
|
||||
const wholeFileTier = (source: string): string | null => {
|
||||
const m =
|
||||
/\b(?:describeE2ETier|e2eTierEnabled)\(\s*['"`](gate|periodic)['"`]/.exec(source) ||
|
||||
/EVALS_TIER\s*===\s*['"`](gate|periodic)['"`]/.exec(source);
|
||||
return m ? m[1] : null;
|
||||
};
|
||||
|
||||
const workflowSource = fs.readFileSync(WORKFLOW, 'utf-8');
|
||||
const rows = parseMatrixRows(workflowSource);
|
||||
const matrixFiles = new Map<string, MatrixRow>();
|
||||
for (const row of rows) for (const f of row.files) matrixFiles.set(f, row);
|
||||
|
||||
const paidFiles = fs
|
||||
.readdirSync(path.join(ROOT, 'test'))
|
||||
.filter((f) => f.endsWith('.test.ts'))
|
||||
.map((f) => `test/${f}`)
|
||||
.filter(isPaidTestFile);
|
||||
|
||||
describe('evals.yml matrix completeness (gate-lane silent-skip tripwire)', () => {
|
||||
test('matrix parse sanity: rows and known suites present', () => {
|
||||
expect(rows.length).toBeGreaterThanOrEqual(15);
|
||||
expect(matrixFiles.has('test/skill-e2e-workflow.test.ts')).toBe(true);
|
||||
expect(matrixFiles.has('test/skill-e2e-ship-docsync.test.ts')).toBe(true);
|
||||
});
|
||||
|
||||
test('every matrix file exists on disk', () => {
|
||||
const missing = [...matrixFiles.keys()].filter(
|
||||
(f) => !fs.existsSync(path.join(ROOT, f))
|
||||
);
|
||||
expect(missing).toEqual([]);
|
||||
});
|
||||
|
||||
test('every gate-hosting paid file is in the matrix (or the documented backlog)', () => {
|
||||
const gaps: string[] = [];
|
||||
for (const file of paidFiles) {
|
||||
const source = fs.readFileSync(path.join(ROOT, file), 'utf-8');
|
||||
const declaresGate = wholeFileTier(source) === 'gate';
|
||||
const inGateDeps = Object.entries(E2E_TOUCHFILES).some(
|
||||
([key, deps]) =>
|
||||
(E2E_TIERS as Record<string, string>)[key] === 'gate' &&
|
||||
(deps as string[]).includes(file)
|
||||
);
|
||||
if (!declaresGate && !inGateDeps) continue;
|
||||
if (matrixFiles.has(file) || KNOWN_MATRIX_GAPS.has(file)) continue;
|
||||
gaps.push(file);
|
||||
}
|
||||
expect(
|
||||
gaps,
|
||||
`Gate-hosting test file(s) missing from the evals.yml matrix — CI will ` +
|
||||
`never run them and "gate tier blocks merge" becomes silently false. ` +
|
||||
`Add a matrix row (with tier: gate when the file is whole-file gated). ` +
|
||||
`Do NOT extend KNOWN_MATRIX_GAPS for new files.`
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
test('matrix rows for whole-file-gated files carry a matching tier property', () => {
|
||||
const mismatches: string[] = [];
|
||||
for (const [file, row] of matrixFiles) {
|
||||
if (!fs.existsSync(path.join(ROOT, file))) continue;
|
||||
const declared = wholeFileTier(fs.readFileSync(path.join(ROOT, file), 'utf-8'));
|
||||
if (!declared) continue;
|
||||
if (row.tier === declared) continue;
|
||||
if (KNOWN_TIER_UNSET.get(file) === declared && row.tier === undefined) continue;
|
||||
mismatches.push(`${file} declares '${declared}' but row '${row.name}' has tier: ${row.tier ?? 'unset'}`);
|
||||
}
|
||||
expect(
|
||||
mismatches,
|
||||
`A whole-file tier guard with no matching row tier means the suite ` +
|
||||
`self-skips and the CI job reports a hollow green. Set tier: <declared> ` +
|
||||
`on the row (the Run step exports it as EVALS_TIER).`
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
test('burn-down lists hold only live gaps (ratchet cleanup enforcement)', () => {
|
||||
const staleGaps = [...KNOWN_MATRIX_GAPS].filter(
|
||||
(f) => matrixFiles.has(f) || !fs.existsSync(path.join(ROOT, f))
|
||||
);
|
||||
expect(
|
||||
staleGaps,
|
||||
'Entry fixed or file removed — delete it from KNOWN_MATRIX_GAPS.'
|
||||
).toEqual([]);
|
||||
const staleTiers = [...KNOWN_TIER_UNSET.entries()].filter(([f, declared]) => {
|
||||
const row = matrixFiles.get(f);
|
||||
if (!row) return true; // row deleted — entry no longer applies
|
||||
if (row.tier === declared) return true; // fixed — entry must go
|
||||
if (!fs.existsSync(path.join(ROOT, f))) return true;
|
||||
return wholeFileTier(fs.readFileSync(path.join(ROOT, f), 'utf-8')) !== declared;
|
||||
});
|
||||
expect(
|
||||
staleTiers.map(([f]) => f),
|
||||
'Entry fixed, row removed, or guard changed — delete it from KNOWN_TIER_UNSET.'
|
||||
).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,131 @@
|
||||
/**
|
||||
* Sliced-lane wiring pins for the paid CI workflows — the successor to
|
||||
* evals-workflow-matrix.test.ts, which enforced completeness of a
|
||||
* hand-enumerated 17-row matrix (and carried KNOWN_MATRIX_GAPS /
|
||||
* KNOWN_TIER_UNSET burn-down ratchets for the files that matrix missed).
|
||||
* The matrix is deleted: the sliced lane's planner derives the gate census
|
||||
* from the runner itself (collectPaidTestFiles + tier selection), so "every
|
||||
* gate-hosting file is in the census" is true BY CONSTRUCTION and the
|
||||
* burn-down ratchets retired with the rows.
|
||||
*
|
||||
* What still needs pinning is the WIRING — the yml plumbing that free tests
|
||||
* are the only guard for:
|
||||
* - the legacy matrix (and its `needs: evals` serialization) stays deleted,
|
||||
* - planner/executor/report all run tier=gate and agree on the slice count,
|
||||
* - both surviving lanes register skills through the SHARED composite that
|
||||
* carries the fail-fast dangling-symlink/frontmatter verification loop
|
||||
* (the sliced + periodic copies had silently dropped it — the loop was
|
||||
* written after a silent "Unknown command" + 35-min-timeout incident),
|
||||
* - the PR comment survives the matrix-report deletion (it moved into
|
||||
* slices-report, keyed on the same "## E2E Evals" upsert marker).
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
const ROOT = path.join(import.meta.dir, '..');
|
||||
const read = (rel: string) => fs.readFileSync(path.join(ROOT, rel), 'utf-8');
|
||||
|
||||
const evalsYml = read('.github/workflows/evals.yml');
|
||||
const periodicYml = read('.github/workflows/evals-periodic.yml');
|
||||
const registerAction = read('.github/actions/register-gstack-skills/action.yml');
|
||||
|
||||
/** Slice count the planner emits (`--slices N`) in a workflow source. */
|
||||
function plannedSlices(source: string): number[] {
|
||||
return [...source.matchAll(/--emit-plan\s+\S+\s+--slices\s+(\d+)/g)].map((m) => Number(m[1]));
|
||||
}
|
||||
|
||||
/** The executor matrix's slice list (`slice: [1, 2, ...]`). */
|
||||
function matrixSlices(source: string): number[][] {
|
||||
return [...source.matchAll(/^\s+slice: \[([\d,\s]+)\]\s*$/gm)].map((m) =>
|
||||
m[1].split(',').map((n) => Number(n.trim())),
|
||||
);
|
||||
}
|
||||
|
||||
describe('evals.yml sliced-lane wiring (post-matrix)', () => {
|
||||
test('the legacy matrix job stays deleted', () => {
|
||||
// Row-enumeration shapes from the deleted matrix. Any reappearance means
|
||||
// someone is re-growing a hand-maintained enumeration next to a lane
|
||||
// whose census is derived — the drift class the deletion killed.
|
||||
expect(evalsYml).not.toMatch(/^\s+suite:\s*$/m);
|
||||
expect(evalsYml).not.toMatch(/^\s+file: test\//m);
|
||||
expect(evalsYml).not.toContain('needs: [build-image, evals]');
|
||||
expect(evalsYml).not.toMatch(/^\s+needs: evals\s*$/m);
|
||||
});
|
||||
|
||||
test('no workflow-level EVALS_TIER env (each command sets its own)', () => {
|
||||
// The workflow-level `EVALS_TIER: gate` was dead config once every
|
||||
// consumer set its own; a resurrected copy would silently leak gate
|
||||
// semantics into steps that must choose explicitly.
|
||||
expect(evalsYml).not.toMatch(/^env:[\s\S]{0,120}^\s+EVALS_TIER:/m);
|
||||
});
|
||||
|
||||
test('planner, executors, and report all run tier=gate on the shared runner', () => {
|
||||
expect(evalsYml).toMatch(/EVALS_TIER=gate bun run scripts\/test-paid-shards\.ts --tier gate --emit-plan/);
|
||||
expect(evalsYml).toMatch(/EVALS_TIER=gate bun run scripts\/test-paid-shards\.ts --tier gate --plan .* --slice /);
|
||||
expect(evalsYml).toMatch(/EVALS_TIER=gate bun run scripts\/test-paid-shards\.ts --tier gate --report /);
|
||||
});
|
||||
|
||||
test('executor matrix slice list matches the planner --slices count', () => {
|
||||
const planned = plannedSlices(evalsYml);
|
||||
const matrices = matrixSlices(evalsYml);
|
||||
expect(planned, 'expected exactly one --emit-plan site in evals.yml').toHaveLength(1);
|
||||
expect(matrices, 'expected exactly one slice matrix in evals.yml').toHaveLength(1);
|
||||
const n = planned[0];
|
||||
expect(matrices[0]).toEqual(Array.from({ length: n }, (_, i) => i + 1));
|
||||
});
|
||||
|
||||
test('the PR comment survived the matrix-report deletion (moved to slices-report)', () => {
|
||||
// Keyed on the upsert marker so the migration keeps updating the SAME
|
||||
// comment; and the job holding it needs the issues permission (#1802).
|
||||
expect(evalsYml).toContain('## E2E Evals');
|
||||
expect(evalsYml).toMatch(/pull-requests: write/);
|
||||
expect(evalsYml).toMatch(/issues: write/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('evals-periodic.yml sliced-lane wiring', () => {
|
||||
test('planner/executor/report tier=periodic and slice counts agree', () => {
|
||||
expect(periodicYml).toMatch(/EVALS_TIER=periodic bun run scripts\/test-paid-shards\.ts --tier periodic --emit-plan/);
|
||||
expect(periodicYml).toMatch(/EVALS_TIER=periodic bun run scripts\/test-paid-shards\.ts --tier periodic --plan .* --slice /);
|
||||
expect(periodicYml).toMatch(/EVALS_TIER=periodic bun run scripts\/test-paid-shards\.ts --tier periodic --report /);
|
||||
const planned = plannedSlices(periodicYml);
|
||||
const matrices = matrixSlices(periodicYml);
|
||||
expect(planned).toHaveLength(1);
|
||||
expect(matrices).toHaveLength(1);
|
||||
expect(matrices[0]).toEqual(Array.from({ length: planned[0] }, (_, i) => i + 1));
|
||||
});
|
||||
});
|
||||
|
||||
describe('shared setup composites (both surviving lanes)', () => {
|
||||
test('both lanes register skills through the shared composite', () => {
|
||||
for (const [name, source] of [['evals.yml', evalsYml], ['evals-periodic.yml', periodicYml]] as const) {
|
||||
expect(source, `${name} must use the register-gstack-skills composite`)
|
||||
.toContain('uses: ./.github/actions/register-gstack-skills');
|
||||
// No inline re-implementation creeping back beside the composite.
|
||||
expect(source, `${name} re-inlines the skill registry instead of using the composite`)
|
||||
.not.toContain('ln -snf "$REPO" "$SKILLS_DIR/gstack"');
|
||||
}
|
||||
});
|
||||
|
||||
test('the register composite carries the fail-fast verification loop', () => {
|
||||
// The loop is the POINT of the composite: a dangling symlink or renamed
|
||||
// committed target fails in seconds with a named path, never as a wedged
|
||||
// PTY session at the shard wall. Pin its load-bearing markers.
|
||||
expect(registerAction).toContain('skill registry OK');
|
||||
expect(registerAction).toContain('skill-registry target missing');
|
||||
expect(registerAction).toContain('gstack root symlink dangles');
|
||||
expect(registerAction).toMatch(/grep -m1 "\^name: \$s\\\$"/);
|
||||
});
|
||||
|
||||
test('seed/deps/temp composites exist and both lanes use them', () => {
|
||||
for (const action of ['seed-claude-config', 'restore-deps', 'fix-bun-temp']) {
|
||||
expect(fs.existsSync(path.join(ROOT, '.github', 'actions', action, 'action.yml')), `missing composite: ${action}`).toBe(true);
|
||||
}
|
||||
for (const [name, source] of [['evals.yml', evalsYml], ['evals-periodic.yml', periodicYml]] as const) {
|
||||
expect(source, `${name} must use seed-claude-config`).toContain('uses: ./.github/actions/seed-claude-config');
|
||||
expect(source, `${name} must use restore-deps`).toContain('uses: ./.github/actions/restore-deps');
|
||||
expect(source, `${name} must use fix-bun-temp`).toContain('uses: ./.github/actions/fix-bun-temp');
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -7,8 +7,9 @@
|
||||
* execute anywhere — forever, silently. Four files were in that state
|
||||
* (codex-e2e-plan-format, codex-e2e-recommendation-substance,
|
||||
* llm-judge-recommendation, carve-section-loading), and the tripwire built
|
||||
* for the adjacent class (test/evals-workflow-matrix.test.ts) couldn't see
|
||||
* them because it filters on isPaidTestFile() FIRST.
|
||||
* for the adjacent class (the since-retired evals-workflow-matrix test;
|
||||
* successor: test/evals-workflow-wiring.test.ts) couldn't see them because
|
||||
* it filtered on isPaidTestFile() FIRST.
|
||||
*
|
||||
* Detection is over source text, so meta-tests and helpers that mention the
|
||||
* gate patterns need reasoned exemptions (same convention as
|
||||
@@ -30,7 +31,7 @@ const SCANNER_EXEMPT = new Map<string, string>([
|
||||
// Meta-tests that quote gate-pattern strings to test classification:
|
||||
['test/helpers/e2e-gate.unit.test.ts', 'free unit test OF the gate predicates (env stubbed)'],
|
||||
['test/paid-shards.test.ts', 'quotes tier-guard strings as classification fixtures'],
|
||||
['test/evals-workflow-matrix.test.ts', 'parses tier guards out of matrix files'],
|
||||
['test/evals-workflow-wiring.test.ts', 'pins the sliced-lane yml wiring (successor to the matrix test)'],
|
||||
['test/e2e-tier-alignment.test.ts', 'parses tier guards to enforce alignment'],
|
||||
['test/paid-orphan-tripwire.test.ts', 'this scanner'],
|
||||
]);
|
||||
|
||||
Reference in New Issue
Block a user