From 07c2452e6dfe347312e8bdf9cc115373393a9c38 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 31 Aug 2026 04:20:40 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20delete=20the=20legacy=2017-row=20eval?= =?UTF-8?q?=20matrix=20=E2=80=94=20the=20sliced=20lane=20is=20the=20only?= =?UTF-8?q?=20paid=20lane?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every PR paid twice: the hand-enumerated matrix (18 test files, 22.6 min, ~$21 API measured on run 33263204465) ran serialized AHEAD of the strictly superior sliced lane via 'needs: evals' — 35.5 min wall and ~2x paid spend for the same diff. 14 of 17 rows carried no tier:, so periodic Opus benchmarks leaked into every PR (the e2e-plan row alone: 12/12 tests, 21.7 min, $7.28 — the wall-clock bound of ALL of CI). Parity receipt (static, pre-deletion): the sliced lane's gate census (49 files, derived from the runner itself) strictly contains all 18 matrix test files, plus 31 files the matrix never ran. Pure deletion — one revert restores it. The PR comment moved into slices-report (same '## E2E Evals' upsert marker, now sourced from slice artifacts + carrying the fail-closed reconciliation verdict). plan-slices loses the needs edge; the dead workflow-level EVALS_TIER env goes with it. test/evals-workflow-matrix.test.ts (and its KNOWN_MATRIX_GAPS / KNOWN_TIER_UNSET burn-down ratchets — retired: the sliced census makes 'every gate file runs' true by construction) is rewritten as test/evals-workflow-wiring.test.ts: matrix stays deleted, planner/executor/ report tier + slice-count agreement, both surviving lanes on the shared register-skills composite with its fail-fast verification loop, PR comment survival. Expected: PR eval wall 35.5 -> ~13 min, per-PR paid spend ~halved. Co-Authored-By: Claude Fable 5 --- .github/workflows/evals.yml | 525 ++++++----------------------- TODOS.md | 17 +- docs/TESTING_INTERNALS.md | 11 +- test/e2e-tier-alignment.test.ts | 8 +- test/evals-workflow-matrix.test.ts | 197 ----------- test/evals-workflow-wiring.test.ts | 131 +++++++ test/paid-orphan-tripwire.test.ts | 7 +- 7 files changed, 266 insertions(+), 630 deletions(-) delete mode 100644 test/evals-workflow-matrix.test.ts create mode 100644 test/evals-workflow-wiring.test.ts diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index c299de6bc..ecacae8c5 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -15,7 +15,6 @@ concurrency: env: IMAGE: ghcr.io/${{ github.repository }}/ci - EVALS_TIER: gate jobs: # Build Docker image with pre-baked toolchain (cached — only rebuilds on Dockerfile/lockfile change) @@ -85,424 +84,27 @@ jobs: ${{ steps.meta.outputs.tag }} ${{ env.IMAGE }}:latest - # Fork PRs never receive repository secrets (ANTHROPIC_API_KEY et al), so every - # API-calling eval fails at SDK auth before a model runs. Skip deterministically - # rather than leaving the outcome to Docker-cache luck: a warm cache let these - # run and fail, a cold one made build-image fail its push and the shards skip. - # Same-repo PRs, pushes, and workflow_dispatch keep full coverage. Fork work - # gets real coverage via a trusted base-repo branch. - evals: - runs-on: ${{ matrix.suite.runner || 'ubicloud-standard-8' }} - needs: build-image - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - # Least privilege for the job that executes PR-authored code with three - # provider API keys in env: read-only contents, packages:read for the - # container-image pull below. Without this block the job ran on the - # repo-default token grant. - permissions: - contents: read - packages: read - container: - image: ${{ needs.build-image.outputs.image-tag }} - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --user runner - timeout-minutes: ${{ matrix.suite.timeout || 25 }} - strategy: - fail-fast: false - matrix: - suite: - - name: llm-judge - file: test/skill-llm-eval.test.ts - - name: e2e-browse - file: test/skill-e2e-bws.test.ts - runner: ubicloud-standard-8 - - name: e2e-plan - file: test/skill-e2e-plan.test.ts - - name: e2e-deploy - file: test/skill-e2e-deploy.test.ts - - name: e2e-design - file: test/skill-e2e-design.test.ts - - name: e2e-qa-bugs - file: test/skill-e2e-qa-bugs.test.ts - - name: e2e-qa-workflow - file: test/skill-e2e-qa-workflow.test.ts - - name: e2e-review - file: test/skill-e2e-review.test.ts - - name: e2e-retro - file: test/skill-e2e-retro.test.ts - - name: e2e-review-attribution - file: test/skill-e2e-review-attribution.test.ts - - name: e2e-workflow - file: test/skill-e2e-workflow.test.ts - # Earned its extra attempt with receipts: document-release is a - # long multi-step E2E that timed out on attempt 2 under in-shard - # concurrency (PR #2593 round 4) while passing other rounds. - retries: 2 - # Rehomed from the deleted pre-split monolith (its filename never - # matched the skill-e2e-* glob, so these gate tests silently never - # ran). Both files hold gate-tier tests: review/plan-eng coverage - # audits and the /ship failure-ownership triage. - - name: e2e-coverage-audit - file: test/skill-e2e-coverage-audit.test.ts - - name: e2e-triage - file: test/skill-e2e-triage.test.ts - # ship-docsync is whole-file tier-gated (describeE2ETier('gate') keeps - # it out of the periodic shard census), so its row MUST set tier: gate - # — without it the self-gate skips every test and the job reports a - # hollow green (the same silent-skip class as the rehomed monolith - # above, one layer deeper). The Run step exports EVALS_TIER from this - # property; rows without it keep EVALS_TIER empty (= unset: every - # reader is `=== ''` or truthiness). Enforced by - # test/evals-workflow-matrix.test.ts. - - name: e2e-ship-docsync - file: test/skill-e2e-ship-docsync.test.ts - tier: gate - # Consent-gate guardrail for the Third-Party Web Actions contract - # (Aside recommended-driver rewrite): hermetic shims, deterministic - # grep assertions, gate tier in E2E_TIERS. - - name: e2e-third-party-actions - file: test/skill-e2e-third-party-actions.test.ts - tier: gate - - name: e2e-routing - file: test/skill-routing-e2e.test.ts - # (e2e-codex / e2e-gemini rows deleted: both files are whole-file - # periodic-tier, so with no row tier: they ran ZERO tests and - # reported green on every PR — ~2 min of runner per PR of pure - # false confidence. The periodic lane owns these suites.) - # Real-PTY plan-mode smokes. Only the deterministically-reliable ones - # are CI-gated: office-hours (asks its mode question first, caught by - # the collapsed/bullet prose-AUQ detector) and plan-mode-no-op (no - # ask-first dependency). The plan-eng/plan-design plan-mode + floor - # smokes are periodic (stochastic ask-first — see touchfiles E2E_TIERS). - # Needs the interactive-config seed step below; PTY sessions otherwise - # wedge on the fresh-container onboarding/API-key dialog. - - name: e2e-pty-plan-smoke - file: test/skill-e2e-office-hours-auto-mode.test.ts test/skill-e2e-plan-mode-no-op.test.ts - # Both files are whole-file describeE2ETier('gate') — without this - # row tier: the job burned ~7 min of setup then skipped every - # describe (hollow-green since the files adopted the self-gate). - tier: gate - timeout: 35 - # The documented contention-heavy PTY family: ROTATING members - # failed attempt 2 in consecutive PR #2593 rounds - # (plan-design-review, then plan-eng-review) while the family - # passes on branches still running three attempts. Every other - # row keeps --retry 1. - retries: 2 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - # Don't write the token into .git/config — this job runs - # PR-authored code; nothing in it pushes. - persist-credentials: false - - # Bun creates root-owned temp dirs during Docker build. GH Actions runs as - # runner user with HOME=/github/home. Redirect bun's cache to a writable dir. - - name: Fix bun temp - run: | - mkdir -p /home/runner/.cache/bun - { - echo "BUN_INSTALL_CACHE_DIR=/home/runner/.cache/bun" - echo "BUN_TMPDIR=/home/runner/.cache/bun" - echo "TMPDIR=/home/runner/.cache" - } >> "$GITHUB_ENV" - - # Restore pre-installed node_modules from Docker image via recursive - # copy. Symlink (`ln -s`) breaks bun's module resolution because bun - # resolves a file's realpath when walking up to find node_modules/; - # from a symlinked path, realpath escapes the workspace and sibling - # deps no longer resolve. Hardlink copy (`cp -al`) fails because /opt - # and /workspace are on different overlay-fs layers ("Invalid - # cross-device link"). Recursive copy works on every layout. Cost: - # ~5s for ~200 packages of small JS files vs ~0s for symlink — still - # vastly cheaper than rerunning `bun install` (network + resolution). - - name: Restore deps - run: | - if [ -d /opt/node_modules_cache ] && diff -q /opt/node_modules_cache/.bun.lock bun.lock >/dev/null 2>&1; then - cp -r /opt/node_modules_cache node_modules - else - bun install - fi - - - run: bun run build - - # Verify Playwright can launch Chromium (fails fast if sandbox/deps are broken) - - name: Verify Chromium - if: matrix.suite.name == 'e2e-browse' - run: | - echo "whoami=$(whoami) HOME=$HOME TMPDIR=${TMPDIR:-unset}" - touch /tmp/.bun-test && rm /tmp/.bun-test && echo "/tmp writable" - bun -e "import {chromium} from 'playwright';const b=await chromium.launch({args:['--no-sandbox']});console.log('Chromium OK');await b.close()" - - # PTY smokes spawn the interactive `claude` TUI. A fresh container has no - # ~/.claude.json, so claude wedges on the onboarding + "use detected - # ANTHROPIC_API_KEY?" dialog and the spawned session never reaches the - # skill. Seed onboarding-complete + the key approval (mirrors what the - # hermetic E2E child env seeds). Scoped to this suite; needs its OWN key - # env (the secrets block below is on the Run step only). - - name: Seed claude interactive config - if: matrix.suite.name == 'e2e-pty-plan-smoke' - env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - run: | - node -e ' - const fs = require("fs"), os = require("os"), path = require("path"); - const p = path.join(os.homedir(), ".claude.json"); - const seed = fs.existsSync(p) ? JSON.parse(fs.readFileSync(p, "utf8")) : {}; - seed.hasCompletedOnboarding = true; - const key = process.env.ANTHROPIC_API_KEY || ""; - if (key) seed.customApiKeyResponses = { approved: [key.slice(-20)], rejected: [] }; - fs.writeFileSync(p, JSON.stringify(seed, null, 2)); - console.log("seeded", p); - ' - - # PTY smokes drive the interactive `claude` TUI and send /office-hours, - # /plan-ceo-review, /plan-eng-review, and /plan-design-review. Claude Code - # discovers user-scoped skills from $HOME/.claude/skills//SKILL.md, - # but .claude/skills is gitignored, so a fresh CI checkout has NO registry - # — claude prints "Unknown command: /plan-ceo-review". Mirror setup's - # --no-prefix registry minimally: a gstack root symlink (resolves the - # preamble's absolute ~/.claude/skills/gstack/bin/* and - # ~/.claude/skills/gstack//sections/* paths) plus a per-skill - # top-level dir holding SKILL.md (+ sections) symlinks for the four skills - # these tests invoke. No ./setup (it builds binaries, launches Chromium, - # installs fonts, reads a /dev/tty prompt) and no binary build (SKILL.md + - # bin/ + sections/ are committed). $HOME is /github/home here; the spawned - # claude inherits it (this runner adds no HOME/CLAUDE_CONFIG_DIR override, - # no hermetic mode) and the Seed step already proved claude reads $HOME. - # - # KEEP THIS STEP even though seedSkills/hermeticSkillsConfigDir() now - # registers skills for hermetic PTY children: that registry is SYMLINKS - # into the repo checkout, and this container's cross-mount symlinks - # defeat the TUI skill scanner (see the note inside the step below) — - # the real-file copies here are what the TUI actually reads. HOME is - # also not hermeticized, so the absolute ~/.claude/skills/gstack/... - # preamble paths resolve through the gstack root symlink this step makes. - - name: Register gstack skills for PTY smoke - if: matrix.suite.name == 'e2e-pty-plan-smoke' - run: | - set -eu - SKILLS_DIR="$HOME/.claude/skills" - REPO="$GITHUB_WORKSPACE" # /__w/gstack/gstack - mkdir -p "$SKILLS_DIR" - # The gstack root stays a symlink — the preamble's runtime bash resolves - # ~/.claude/skills/gstack/bin/* and ~/.claude/skills/gstack//sections/* - # through it, and bash follows cross-mount symlinks fine. - ln -snf "$REPO" "$SKILLS_DIR/gstack" - # But the per-skill SKILL.md the TUI DISCOVERS must be a REAL file on the - # same mount as $HOME. claude 2.1.187's interactive-TUI skill scanner does - # not follow the /github/home -> /__w cross-mount symlink (proven: `claude - # -p` discovered the skill — READY — while the TUI rejected /office-hours - # as "Unknown command"; a local macOS repro with the identical symlinked - # registry recognized it, isolating the failure to the container's - # cross-mount symlink). Copy SKILL.md + sections as real files so the TUI - # reads them directly. - for s in office-hours plan-ceo-review plan-eng-review plan-design-review; do - rm -rf "${SKILLS_DIR:?}/$s" - mkdir -p "$SKILLS_DIR/$s" - cp "$REPO/$s/SKILL.md" "$SKILLS_DIR/$s/SKILL.md" - cp -R "$REPO/$s/sections" "$SKILLS_DIR/$s/sections" - done - # Also register PROJECT-scoped (cwd) skills. claude's interactive TUI - # surfaces /slash commands from /.claude/skills, and the smokes run - # with cwd=$REPO whose .claude/skills is gitignored (absent on a fresh CI - # checkout) — the user-dir registration above feeds `claude -p` but the - # TUI looks here. No gstack symlink in the project dir: it would point at - # its own parent ($REPO). Runtime preamble paths use the user-dir - # ~/.claude/skills/gstack symlink above. - PROJ_SKILLS="$REPO/.claude/skills" - mkdir -p "$PROJ_SKILLS" - for s in office-hours plan-ceo-review plan-eng-review plan-design-review; do - rm -rf "${PROJ_SKILLS:?}/$s" - mkdir -p "$PROJ_SKILLS/$s" - cp "$REPO/$s/SKILL.md" "$PROJ_SKILLS/$s/SKILL.md" - cp -R "$REPO/$s/sections" "$PROJ_SKILLS/$s/sections" - done - # Pre-seed every ONE-TIME preamble marker so no PTY child ever takes a - # first-run branch mid-test. On a fresh runner these are all missing, so - # each smoke's preamble fires feature discovery / telemetry / lake-intro - # prompts before the behavior under test — and touching the - # feature-discovery marker under ~/.claude/skills/gstack/ trips Claude - # Code's sensitive-file permission prompt, stalling the run before the - # scope gate renders (the documented intermittent - # scope-gate-question-NOT-observed failure: outcome=asked was the - # permission dialog, not the gate). Dev machines never hit this because - # the operator's markers already exist; CI must seed them explicitly. - mkdir -p "$HOME/.gstack" - touch "$HOME/.gstack/.activated" \ - "$HOME/.gstack/.first-loop-tip-shown" \ - "$HOME/.gstack/.telemetry-prompted" \ - "$HOME/.gstack/.proactive-prompted" \ - "$HOME/.gstack/.completeness-intro-seen" \ - "$HOME/.gstack/.plan-tune-nudge-shown" - # These two resolve through the gstack root symlink into $REPO — - # untracked scratch in the CI checkout, exactly where the preamble looks. - touch "$SKILLS_DIR/gstack/.feature-prompted-continuous-checkpoint" \ - "$SKILLS_DIR/gstack/.feature-prompted-model-overlay" - echo "--- registry under $SKILLS_DIR ---" - ls -la "$SKILLS_DIR/gstack" "$SKILLS_DIR/office-hours" "$SKILLS_DIR/plan-ceo-review" - # Fail fast if any committed target moved/renamed — a dangling symlink - # would otherwise resurface as a silent "Unknown command" + 35-min timeout. - for f in \ - "$SKILLS_DIR/office-hours/SKILL.md" \ - "$SKILLS_DIR/plan-ceo-review/SKILL.md" \ - "$SKILLS_DIR/plan-eng-review/SKILL.md" \ - "$SKILLS_DIR/plan-design-review/SKILL.md" \ - "$SKILLS_DIR/gstack/bin/gstack-update-check" \ - "$SKILLS_DIR/gstack/office-hours/sections/design-and-handoff.md" \ - "$SKILLS_DIR/gstack/plan-ceo-review/sections/review-sections.md" \ - "$SKILLS_DIR/gstack/plan-eng-review/sections/review-sections.md" \ - "$SKILLS_DIR/gstack/plan-design-review/sections/review-sections.md"; do - if [ ! -e "$f" ]; then - echo "ERROR: skill-registry target missing (symlink dangles): $f" >&2 - exit 1 - fi - done - for s in office-hours plan-ceo-review plan-eng-review plan-design-review; do - grep -m1 "^name: $s\$" "$SKILLS_DIR/$s/SKILL.md" >/dev/null \ - || { echo "ERROR: $s SKILL.md missing 'name: $s' frontmatter" >&2; exit 1; } - done - echo "skill registry OK" - - - name: Run ${{ matrix.suite.name }} - env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} - EVALS_CONCURRENCY: "40" - PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers - # Per-row tier activation for whole-file-gated suites. Empty when the - # row declares no tier — every EVALS_TIER reader treats empty as unset - # (`=== ''` comparisons and the truthiness check in - # test/helpers/e2e-helpers.ts:70), so untiered rows are byte-for-byte - # unaffected. - EVALS_TIER: ${{ matrix.suite.tier || '' }} - run: EVALS=1 bun test --retry ${{ matrix.suite.retries || 1 }} --concurrent --max-concurrency 40 ${{ matrix.suite.file }} - - - name: Upload eval results - if: always() - uses: actions/upload-artifact@v7 - with: - name: eval-${{ matrix.suite.name }} - path: ~/.gstack-dev/evals/*.json - retention-days: 90 - - report: - runs-on: ubicloud-standard-2 - needs: evals - if: always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository - timeout-minutes: 5 - permissions: - contents: read - pull-requests: write - # The comment upsert below calls the REST `/issues/{n}/comments` endpoints - # (gh api ... issues/comments). With GITHUB_TOKEN those are gated by the - # `issues` permission, not `pull-requests` — without it the GET returns 401 - # on every PR that produces eval artifacts (PRs with no artifacts exit - # early and never hit it, which is why this stayed hidden). See #1802 CI fix. - issues: write - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 1 - - - name: Download all eval artifacts - uses: actions/download-artifact@v8 - with: - pattern: eval-* - path: /tmp/eval-results - merge-multiple: true - - - name: Post PR comment - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # shellcheck disable=SC2086,SC2059 - RESULTS=$(find /tmp/eval-results -name '*.json' 2>/dev/null | sort) - if [ -z "$RESULTS" ]; then - echo "No eval results found" - exit 0 - fi - - TOTAL=0; PASSED=0; FAILED=0; COST="0" - SUITE_LINES="" - for f in $RESULTS; do - if ! jq -e '.total_tests' "$f" >/dev/null 2>&1; then - echo "Skipping malformed JSON: $f" - continue - fi - T=$(jq -r '.total_tests // 0' "$f") - P=$(jq -r '.passed // 0' "$f") - F=$(jq -r '.failed // 0' "$f") - C=$(jq -r '.total_cost_usd // 0' "$f") - TIER=$(jq -r '.tier // "unknown"' "$f") - [ "$T" -eq 0 ] && continue - TOTAL=$((TOTAL + T)) - PASSED=$((PASSED + P)) - FAILED=$((FAILED + F)) - COST=$(echo "$COST + $C" | bc) - STATUS_ICON="✅" - [ "$F" -gt 0 ] && STATUS_ICON="❌" - SUITE_LINES="${SUITE_LINES}| ${TIER} | ${P}/${T} | ${STATUS_ICON} | \$${C} |\n" - done - - STATUS="✅ PASS" - [ "$FAILED" -gt 0 ] && STATUS="❌ FAIL" - - BODY="## E2E Evals: ${STATUS} - - **${PASSED}/${TOTAL}** tests passed | **\$${COST}** total cost - - | Suite | Result | Status | Cost | - |-------|--------|--------|------| - $(echo -e "$SUITE_LINES") - - --- - *ubicloud-standard-8 runners (Docker: pre-baked toolchain + deps) | wall clock ≈ slowest suite*" - - if [ "$FAILED" -gt 0 ]; then - FAILURES="" - for f in $RESULTS; do - if ! jq -e '.failed' "$f" >/dev/null 2>&1; then continue; fi - F=$(jq -r '.failed // 0' "$f") - [ "$F" -eq 0 ] && continue - FAILS=$(jq -r '.tests[] | select(.passed == false) | "- ❌ \(.name): \(.exit_reason // "unknown")"' "$f" 2>/dev/null || echo "- ⚠️ $(basename "$f"): parse error") - FAILURES="${FAILURES}${FAILS}\n" - done - BODY="${BODY} - - ### Failures - $(echo -e "$FAILURES")" - fi - - # Update existing comment or create new one - COMMENT_ID=$(gh api repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments \ - --jq '.[] | select(.body | startswith("## E2E Evals")) | .id' | tail -1) - - if [ -n "$COMMENT_ID" ]; then - gh api "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" \ - -X PATCH -f body="$BODY" - else - gh pr comment "${{ github.event.pull_request.number }}" --body "$BODY" - fi - - # ── Sliced lane (paid-CI re-platform, parity phase) ───────────────────────── + # ── Sliced lane (the ONLY paid lane; legacy 17-row matrix deleted) ────────── # One PLANNER computes diff selection + the slice plan ONCE (killing # per-slice selector divergence); K executors consume the manifest; the # report reconciles results against it FAIL-CLOSED (a slice whose artifact # never landed is a failure, a planned shard nobody reported is a failure — - # hollow lanes cannot aggregate green). Runs AFTER the matrix (`needs: - # evals`) so provider concurrency never doubles while both lanes coexist; - # once parity is demonstrated the matrix + its ratchets are deleted and this - # lane loses the needs edge. Engine: scripts/test-paid-shards.ts — the same - # runner local eval:bg:gate uses, so CI and local share one selection engine. + # hollow lanes cannot aggregate green). Engine: scripts/test-paid-shards.ts — + # the same runner local eval:bg:gate uses, so CI and local share one + # selection engine, and every gate-tier file is in the census by + # construction (no hand-enumerated rows to drift). The legacy matrix ran + # 18 enumerated files for 22.6 min/$21 per PR serialized AHEAD of this + # lane's 49-file diff-selected census; parity was demonstrated (sliced + # census ⊇ matrix files) and the matrix deleted — one revert restores it. + # + # Fork PRs never receive repository secrets (ANTHROPIC_API_KEY et al), so + # every API-calling eval fails at SDK auth before a model runs. Skip + # deterministically; fork work gets real coverage via a trusted base-repo + # branch (see CLAUDE.md's garrytan-agents workflow). plan-slices: runs-on: ubicloud-standard-8 - needs: [build-image, evals] - if: always() && needs.build-image.result == 'success' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + needs: build-image + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository timeout-minutes: 10 permissions: contents: read @@ -642,6 +244,11 @@ jobs: timeout-minutes: 5 permissions: contents: read + pull-requests: write + # The comment upsert calls the REST `/issues/{n}/comments` endpoints + # (gh api ... issues/comments). With GITHUB_TOKEN those are gated by the + # `issues` permission, not `pull-requests` (#1802 CI fix). + issues: write steps: - uses: actions/checkout@v7 with: @@ -665,4 +272,94 @@ jobs: merge-multiple: true - name: Reconcile slices against the manifest (fail-closed) - run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report /tmp/paid-report + id: reconcile + run: | + set +e + EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report /tmp/paid-report | tee /tmp/report.txt + echo "exit=$?" >> "$GITHUB_OUTPUT" + + # PR comment sourced from the slice artifacts' eval-store JSONs + # (shards//*.json inside each paid-slice-N artifact). Replaces the + # deleted matrix-artifact report job; keeps the "## E2E Evals" marker so + # the upsert keeps updating the same comment across the migration. Runs + # even when reconciliation failed — a red lane on the PR is the point. + - name: Post PR comment + if: always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RECONCILE_EXIT: ${{ steps.reconcile.outputs.exit }} + run: | + # shellcheck disable=SC2086,SC2059 + RESULTS=$(find /tmp/paid-report -name '*.json' ! -name 'manifest.json' ! -name 'slice-*.json' ! -name '_partial*' 2>/dev/null | sort) + TOTAL=0; PASSED=0; FAILED=0; COST="0" + SUITE_LINES="" + for f in $RESULTS; do + if ! jq -e '.total_tests' "$f" >/dev/null 2>&1; then + echo "Skipping malformed JSON: $f" + continue + fi + T=$(jq -r '.total_tests // 0' "$f") + P=$(jq -r '.passed // 0' "$f") + F=$(jq -r '.failed // 0' "$f") + C=$(jq -r '.total_cost_usd // 0' "$f") + TIER=$(jq -r '.tier // "unknown"' "$f") + SHARD=$(jq -r '.shard // "-"' "$f") + [ "$T" -eq 0 ] && continue + TOTAL=$((TOTAL + T)) + PASSED=$((PASSED + P)) + FAILED=$((FAILED + F)) + COST=$(echo "$COST + $C" | bc) + STATUS_ICON="✅" + [ "$F" -gt 0 ] && STATUS_ICON="❌" + SUITE_LINES="${SUITE_LINES}| ${TIER}/${SHARD} | ${P}/${T} | ${STATUS_ICON} | \$${C} |\n" + done + + STATUS="✅ PASS" + if [ "${RECONCILE_EXIT:-1}" != "0" ] || [ "$FAILED" -gt 0 ]; then STATUS="❌ FAIL"; fi + + BODY="## E2E Evals: ${STATUS} + + **${PASSED}/${TOTAL}** tests passed | **\$${COST}** total cost | reconcile exit: ${RECONCILE_EXIT:-missing} + + | Shard | Result | Status | Cost | + |-------|--------|--------|------| + $(echo -e "$SUITE_LINES") + +
Fail-closed reconciliation + + \`\`\` + $(tail -c 4000 /tmp/report.txt 2>/dev/null || echo '(no reconciliation output)') + \`\`\` +
+ + --- + *Sliced lane: diff-selected gate census via scripts/test-paid-shards.ts (planner → 6 executors → fail-closed report)*" + + if [ "$FAILED" -gt 0 ]; then + FAILURES="" + for f in $RESULTS; do + if ! jq -e '.failed' "$f" >/dev/null 2>&1; then continue; fi + F=$(jq -r '.failed // 0' "$f") + [ "$F" -eq 0 ] && continue + FAILS=$(jq -r '.tests[] | select(.passed == false) | "- ❌ \(.name): \(.exit_reason // "unknown")"' "$f" 2>/dev/null || echo "- ⚠️ $(basename "$f"): parse error") + FAILURES="${FAILURES}${FAILS}\n" + done + BODY="${BODY} + + ### Failures + $(echo -e "$FAILURES")" + fi + + COMMENT_ID=$(gh api repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments \ + --jq '.[] | select(.body | startswith("## E2E Evals")) | .id' | tail -1) + + if [ -n "$COMMENT_ID" ]; then + gh api "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" \ + -X PATCH -f body="$BODY" + else + gh pr comment "${{ github.event.pull_request.number }}" --body "$BODY" + fi + + - name: Fail the workflow when reconciliation failed + if: steps.reconcile.outputs.exit != '0' + run: exit 1 diff --git a/TODOS.md b/TODOS.md index 7be847dde..bc610cc69 100644 --- a/TODOS.md +++ b/TODOS.md @@ -567,14 +567,15 @@ duration-packed free shards, the sharded paid runner as the CI engine coverage contract + gate census, eval-budget timeout tiers, and the coverage fill. Remaining, in rough priority order: -- **P1 — Delete the legacy evals.yml matrix after parity.** The sliced lane - runs alongside the 18-row matrix (`needs: evals`, so provider concurrency - never doubles). After 1-2 PR cycles of parity (compare executed-test sets: - intersection strict + the 8 KNOWN_MATRIX_GAPS files as expected additions; - stochastic outcomes informational), delete the matrix as a PURE-DELETION - commit (one revert restores it), drop the `needs: evals` edge, rewrite - test/evals-workflow-matrix.test.ts into a runner-wiring pin, and retire - KNOWN_MATRIX_GAPS/KNOWN_TIER_UNSET wholesale. Effort S. +- **DONE (v1.76 test-infra wave) — Delete the legacy evals.yml matrix after + parity.** Deleted as a pure-deletion commit (one revert restores it) after + a static parity receipt: sliced gate census (49 files) ⊇ matrix files (18), + 31 files of extra coverage. `needs: evals` edge dropped, PR comment moved + into slices-report, KNOWN_MATRIX_GAPS/KNOWN_TIER_UNSET retired, + test/evals-workflow-matrix.test.ts rewritten as + test/evals-workflow-wiring.test.ts. The register-skills fail-fast + verification loop was ported to the surviving lanes FIRST via the shared + .github/actions/register-gstack-skills composite. - **P1 — Maintainer decision: make `slices-report` a required check** once post-migration flake data exists (the Codex outside-voice's "green means green is not delivered while paid stays advisory" point — correct, and diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index 97d756dff..be803240b 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -76,10 +76,13 @@ slice-result artifacts; `--report ` reconciles them FAIL-CLOSED (a slice whose artifact never landed, or a planned shard nobody reported, is a failure). Under `EVALS_ALL` the hollow-shard guard marks exit-0 shards with ZERO executed tests `passed-empty` (a failure) — census-health, not just -test runs. evals.yml runs the sliced gate lane per PR (parity phase: -alongside the legacy matrix, `needs:`-sequenced so provider concurrency -never doubles; the matrix and its `KNOWN_MATRIX_GAPS`/`KNOWN_TIER_UNSET` -ratchets are deleted after demonstrated parity). evals-periodic.yml runs ALL +test runs. evals.yml runs the sliced gate lane per PR — the ONLY paid lane +since the legacy 17-row matrix (22.6 min/$21 per PR serialized ahead of the +slices) was deleted after demonstrated parity; its +`KNOWN_MATRIX_GAPS`/`KNOWN_TIER_UNSET` ratchets retired with it and +`test/evals-workflow-wiring.test.ts` pins the surviving wiring (slice-count +agreement, tier consistency, the shared register-skills composite with its +fail-fast verification loop). evals-periodic.yml runs ALL periodic-tier files weekly (the coverage contract) minus the reasoned exclusions in `test/helpers/periodic-exclude-data.ts` (reason + tracking required per entry; removal re-activates the file), plus a weekly diff --git a/test/e2e-tier-alignment.test.ts b/test/e2e-tier-alignment.test.ts index ba6bcee10..0a95eb0ef 100644 --- a/test/e2e-tier-alignment.test.ts +++ b/test/e2e-tier-alignment.test.ts @@ -42,8 +42,8 @@ const SELF_GATE_RE = /EVALS_TIER\s*===\s*['"](gate|periodic)['"]/g; const HELPER_GATE_RE = /\b(?:describeE2ETier|e2eTierEnabled)\(\s*['"](gate|periodic)['"]/g; /** - * Ratchet, not amnesty (same contract as KNOWN_MATRIX_GAPS in - * test/evals-workflow-matrix.test.ts): skill-e2e files that are named in NO + * Ratchet, not amnesty (the contract KNOWN_MATRIX_GAPS pioneered before the + * legacy eval matrix and its test retired): skill-e2e files that are named in NO * touchfiles dep list because no E2E map key exists for them. Every entry * carries a one-line reason. Do NOT add new files here — give the test an * E2E map key (touchfiles + tier) and register the file in its dep list. @@ -141,8 +141,8 @@ describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () => expect(misaligned).toEqual([]); }); - // Ratchet cleanup enforcement (same contract as evals-workflow-matrix's - // burn-down test): a KNOWN_UNREGISTERED entry whose file was deleted, or + // Ratchet cleanup enforcement (same shrink-only contract as the retired + // matrix test's burn-down): a KNOWN_UNREGISTERED entry whose file was deleted, or // whose file is now named in a dep list, is stale — delete the entry so // the set can only shrink. test('KNOWN_UNREGISTERED holds only live, still-unregistered files', () => { diff --git a/test/evals-workflow-matrix.test.ts b/test/evals-workflow-matrix.test.ts deleted file mode 100644 index b59353a08..000000000 --- a/test/evals-workflow-matrix.test.ts +++ /dev/null @@ -1,197 +0,0 @@ -/** - * CI eval-matrix completeness tripwire — kills the silent-skip class where a - * gate-tier test exists in the repo but the hand-enumerated matrix in - * .github/workflows/evals.yml never runs it, so "gate tier blocks merge" is - * quietly false in CI. This has happened before (see the "rehomed from the - * deleted pre-split monolith" comment in evals.yml) and was found again on - * PR #2700: nine gate-hosting files absent from the matrix, plus matrix rows - * whose whole-file tier guards can never fire because the Run step exported - * no EVALS_TIER. - * - * Ratchet, not amnesty: the KNOWN_* lists below enumerate the PRE-EXISTING - * gaps with reasons, so no NEW gap can land while the backlog burns down - * (same pattern as SCANNER_EXEMPT in egress-receipt-wiring). If you fix a - * listed gap (add its matrix row / tier property), this test FAILS until you - * remove the entry — stale exemptions are enforced, not decorative. - * - * Wiring pinned: - * - every matrix `file:` path exists on disk (no stale rows), - * - every gate-hosting paid file (whole-file gate self-gate, or named in the - * dep list of a gate-tier E2E_TOUCHFILES key) appears in the matrix or in - * KNOWN_MATRIX_GAPS, - * - every matrix file with a whole-file tier guard has a matching row-level - * `tier:` property (else the suite self-skips and the job is hollow-green) - * or sits in KNOWN_TIER_UNSET. - */ -import { describe, test, expect } from 'bun:test'; -import * as fs from 'fs'; -import * as path from 'path'; -import { E2E_TOUCHFILES, E2E_TIERS } from './helpers/touchfiles-data'; -import { isPaidTestFile } from './helpers/paid-test-set'; - -const ROOT = path.join(import.meta.dir, '..'); -const WORKFLOW = path.join(ROOT, '.github', 'workflows', 'evals.yml'); - -/** - * Pre-existing gate-hosting files with no matrix row (found 2026-08-26, - * PR #2700). Adding a row activates real paid runs on every PR — a cost and - * flake-surface decision per file, tracked in TODOS.md ("CI gate-lane - * hollow-coverage burn-down"). Fix = add a matrix row (plus `tier: gate` when - * the file is whole-file gated), then DELETE the entry here. - */ -const KNOWN_MATRIX_GAPS = new Set([ - 'test/skill-e2e-ask-user-question-format-compliance.test.ts', - 'test/skill-e2e-hermetic-canary.test.ts', - 'test/skill-e2e-ios.test.ts', - 'test/skill-e2e-plan-ceo-finding-floor.test.ts', - 'test/skill-e2e-plan-ceo-plan-mode.test.ts', - 'test/skill-e2e-plan-design-with-ui.test.ts', - 'test/skill-e2e-plan-devex-finding-floor.test.ts', - 'test/skill-e2e-plan-devex-plan-mode.test.ts', - // Exposed by the 2026-08 dep-list self-registration sweep: these eight had - // zero gate-key dep-list membership before it, so the census never saw - // them as gate-hosting. Their gate tests run in NO CI lane today. The - // paid-lane re-platform (test-paid-shards.ts as the CI engine) runs every - // gate-tier file by construction and retires this whole ratchet. - 'test/skill-e2e-cso.test.ts', - 'test/skill-e2e-diagram.test.ts', - 'test/skill-e2e-learnings.test.ts', - 'test/skill-e2e-plan-tune.test.ts', - 'test/skill-e2e-plan-tune-cathedral.test.ts', - 'test/skill-e2e-review-army.test.ts', - 'test/skill-e2e-session-intelligence.test.ts', - 'test/skill-e2e-skillify.test.ts', -]); - -/** - * Matrix files whose whole-file tier guard has no matching row `tier:` - * property. Burned down to empty 2026-08-29: the vestigial codex/gemini rows - * were deleted (periodic-tier files, zero tests per PR) and - * e2e-pty-plan-smoke gained its `tier: gate`. The ratchet stays so a future - * row/file tier mismatch fails the suite instead of shipping hollow green. - */ -const KNOWN_TIER_UNSET = new Map([]); - -interface MatrixRow { - name: string; - files: string[]; - tier?: string; -} - -/** Parse the `matrix: suite:` rows (name / file / optional tier) from evals.yml. */ -function parseMatrixRows(source: string): MatrixRow[] { - const rows: MatrixRow[] = []; - let current: MatrixRow | null = null; - for (const line of source.split('\n')) { - const name = line.match(/^\s+- name: (\S+)\s*$/); - if (name) { - if (current) rows.push(current); - current = { name: name[1], files: [] }; - continue; - } - if (!current) continue; - const file = line.match(/^\s+file: (.+?)\s*$/); - if (file) current.files.push(...file[1].trim().split(/\s+/)); - const tier = line.match(/^\s+tier: (\S+)\s*$/); - if (tier) current.tier = tier[1]; - // `steps:` ends the strategy block — stop before step-level keys leak in. - if (/^\s{4}steps:\s*$/.test(line)) break; - } - if (current) rows.push(current); - return rows.filter((r) => r.files.length > 0); -} - -const wholeFileTier = (source: string): string | null => { - const m = - /\b(?:describeE2ETier|e2eTierEnabled)\(\s*['"`](gate|periodic)['"`]/.exec(source) || - /EVALS_TIER\s*===\s*['"`](gate|periodic)['"`]/.exec(source); - return m ? m[1] : null; -}; - -const workflowSource = fs.readFileSync(WORKFLOW, 'utf-8'); -const rows = parseMatrixRows(workflowSource); -const matrixFiles = new Map(); -for (const row of rows) for (const f of row.files) matrixFiles.set(f, row); - -const paidFiles = fs - .readdirSync(path.join(ROOT, 'test')) - .filter((f) => f.endsWith('.test.ts')) - .map((f) => `test/${f}`) - .filter(isPaidTestFile); - -describe('evals.yml matrix completeness (gate-lane silent-skip tripwire)', () => { - test('matrix parse sanity: rows and known suites present', () => { - expect(rows.length).toBeGreaterThanOrEqual(15); - expect(matrixFiles.has('test/skill-e2e-workflow.test.ts')).toBe(true); - expect(matrixFiles.has('test/skill-e2e-ship-docsync.test.ts')).toBe(true); - }); - - test('every matrix file exists on disk', () => { - const missing = [...matrixFiles.keys()].filter( - (f) => !fs.existsSync(path.join(ROOT, f)) - ); - expect(missing).toEqual([]); - }); - - test('every gate-hosting paid file is in the matrix (or the documented backlog)', () => { - const gaps: string[] = []; - for (const file of paidFiles) { - const source = fs.readFileSync(path.join(ROOT, file), 'utf-8'); - const declaresGate = wholeFileTier(source) === 'gate'; - const inGateDeps = Object.entries(E2E_TOUCHFILES).some( - ([key, deps]) => - (E2E_TIERS as Record)[key] === 'gate' && - (deps as string[]).includes(file) - ); - if (!declaresGate && !inGateDeps) continue; - if (matrixFiles.has(file) || KNOWN_MATRIX_GAPS.has(file)) continue; - gaps.push(file); - } - expect( - gaps, - `Gate-hosting test file(s) missing from the evals.yml matrix — CI will ` + - `never run them and "gate tier blocks merge" becomes silently false. ` + - `Add a matrix row (with tier: gate when the file is whole-file gated). ` + - `Do NOT extend KNOWN_MATRIX_GAPS for new files.` - ).toEqual([]); - }); - - test('matrix rows for whole-file-gated files carry a matching tier property', () => { - const mismatches: string[] = []; - for (const [file, row] of matrixFiles) { - if (!fs.existsSync(path.join(ROOT, file))) continue; - const declared = wholeFileTier(fs.readFileSync(path.join(ROOT, file), 'utf-8')); - if (!declared) continue; - if (row.tier === declared) continue; - if (KNOWN_TIER_UNSET.get(file) === declared && row.tier === undefined) continue; - mismatches.push(`${file} declares '${declared}' but row '${row.name}' has tier: ${row.tier ?? 'unset'}`); - } - expect( - mismatches, - `A whole-file tier guard with no matching row tier means the suite ` + - `self-skips and the CI job reports a hollow green. Set tier: ` + - `on the row (the Run step exports it as EVALS_TIER).` - ).toEqual([]); - }); - - test('burn-down lists hold only live gaps (ratchet cleanup enforcement)', () => { - const staleGaps = [...KNOWN_MATRIX_GAPS].filter( - (f) => matrixFiles.has(f) || !fs.existsSync(path.join(ROOT, f)) - ); - expect( - staleGaps, - 'Entry fixed or file removed — delete it from KNOWN_MATRIX_GAPS.' - ).toEqual([]); - const staleTiers = [...KNOWN_TIER_UNSET.entries()].filter(([f, declared]) => { - const row = matrixFiles.get(f); - if (!row) return true; // row deleted — entry no longer applies - if (row.tier === declared) return true; // fixed — entry must go - if (!fs.existsSync(path.join(ROOT, f))) return true; - return wholeFileTier(fs.readFileSync(path.join(ROOT, f), 'utf-8')) !== declared; - }); - expect( - staleTiers.map(([f]) => f), - 'Entry fixed, row removed, or guard changed — delete it from KNOWN_TIER_UNSET.' - ).toEqual([]); - }); -}); diff --git a/test/evals-workflow-wiring.test.ts b/test/evals-workflow-wiring.test.ts new file mode 100644 index 000000000..b4b2b8b0b --- /dev/null +++ b/test/evals-workflow-wiring.test.ts @@ -0,0 +1,131 @@ +/** + * Sliced-lane wiring pins for the paid CI workflows — the successor to + * evals-workflow-matrix.test.ts, which enforced completeness of a + * hand-enumerated 17-row matrix (and carried KNOWN_MATRIX_GAPS / + * KNOWN_TIER_UNSET burn-down ratchets for the files that matrix missed). + * The matrix is deleted: the sliced lane's planner derives the gate census + * from the runner itself (collectPaidTestFiles + tier selection), so "every + * gate-hosting file is in the census" is true BY CONSTRUCTION and the + * burn-down ratchets retired with the rows. + * + * What still needs pinning is the WIRING — the yml plumbing that free tests + * are the only guard for: + * - the legacy matrix (and its `needs: evals` serialization) stays deleted, + * - planner/executor/report all run tier=gate and agree on the slice count, + * - both surviving lanes register skills through the SHARED composite that + * carries the fail-fast dangling-symlink/frontmatter verification loop + * (the sliced + periodic copies had silently dropped it — the loop was + * written after a silent "Unknown command" + 35-min-timeout incident), + * - the PR comment survives the matrix-report deletion (it moved into + * slices-report, keyed on the same "## E2E Evals" upsert marker). + */ +import { describe, test, expect } from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; + +const ROOT = path.join(import.meta.dir, '..'); +const read = (rel: string) => fs.readFileSync(path.join(ROOT, rel), 'utf-8'); + +const evalsYml = read('.github/workflows/evals.yml'); +const periodicYml = read('.github/workflows/evals-periodic.yml'); +const registerAction = read('.github/actions/register-gstack-skills/action.yml'); + +/** Slice count the planner emits (`--slices N`) in a workflow source. */ +function plannedSlices(source: string): number[] { + return [...source.matchAll(/--emit-plan\s+\S+\s+--slices\s+(\d+)/g)].map((m) => Number(m[1])); +} + +/** The executor matrix's slice list (`slice: [1, 2, ...]`). */ +function matrixSlices(source: string): number[][] { + return [...source.matchAll(/^\s+slice: \[([\d,\s]+)\]\s*$/gm)].map((m) => + m[1].split(',').map((n) => Number(n.trim())), + ); +} + +describe('evals.yml sliced-lane wiring (post-matrix)', () => { + test('the legacy matrix job stays deleted', () => { + // Row-enumeration shapes from the deleted matrix. Any reappearance means + // someone is re-growing a hand-maintained enumeration next to a lane + // whose census is derived — the drift class the deletion killed. + expect(evalsYml).not.toMatch(/^\s+suite:\s*$/m); + expect(evalsYml).not.toMatch(/^\s+file: test\//m); + expect(evalsYml).not.toContain('needs: [build-image, evals]'); + expect(evalsYml).not.toMatch(/^\s+needs: evals\s*$/m); + }); + + test('no workflow-level EVALS_TIER env (each command sets its own)', () => { + // The workflow-level `EVALS_TIER: gate` was dead config once every + // consumer set its own; a resurrected copy would silently leak gate + // semantics into steps that must choose explicitly. + expect(evalsYml).not.toMatch(/^env:[\s\S]{0,120}^\s+EVALS_TIER:/m); + }); + + test('planner, executors, and report all run tier=gate on the shared runner', () => { + expect(evalsYml).toMatch(/EVALS_TIER=gate bun run scripts\/test-paid-shards\.ts --tier gate --emit-plan/); + expect(evalsYml).toMatch(/EVALS_TIER=gate bun run scripts\/test-paid-shards\.ts --tier gate --plan .* --slice /); + expect(evalsYml).toMatch(/EVALS_TIER=gate bun run scripts\/test-paid-shards\.ts --tier gate --report /); + }); + + test('executor matrix slice list matches the planner --slices count', () => { + const planned = plannedSlices(evalsYml); + const matrices = matrixSlices(evalsYml); + expect(planned, 'expected exactly one --emit-plan site in evals.yml').toHaveLength(1); + expect(matrices, 'expected exactly one slice matrix in evals.yml').toHaveLength(1); + const n = planned[0]; + expect(matrices[0]).toEqual(Array.from({ length: n }, (_, i) => i + 1)); + }); + + test('the PR comment survived the matrix-report deletion (moved to slices-report)', () => { + // Keyed on the upsert marker so the migration keeps updating the SAME + // comment; and the job holding it needs the issues permission (#1802). + expect(evalsYml).toContain('## E2E Evals'); + expect(evalsYml).toMatch(/pull-requests: write/); + expect(evalsYml).toMatch(/issues: write/); + }); +}); + +describe('evals-periodic.yml sliced-lane wiring', () => { + test('planner/executor/report tier=periodic and slice counts agree', () => { + expect(periodicYml).toMatch(/EVALS_TIER=periodic bun run scripts\/test-paid-shards\.ts --tier periodic --emit-plan/); + expect(periodicYml).toMatch(/EVALS_TIER=periodic bun run scripts\/test-paid-shards\.ts --tier periodic --plan .* --slice /); + expect(periodicYml).toMatch(/EVALS_TIER=periodic bun run scripts\/test-paid-shards\.ts --tier periodic --report /); + const planned = plannedSlices(periodicYml); + const matrices = matrixSlices(periodicYml); + expect(planned).toHaveLength(1); + expect(matrices).toHaveLength(1); + expect(matrices[0]).toEqual(Array.from({ length: planned[0] }, (_, i) => i + 1)); + }); +}); + +describe('shared setup composites (both surviving lanes)', () => { + test('both lanes register skills through the shared composite', () => { + for (const [name, source] of [['evals.yml', evalsYml], ['evals-periodic.yml', periodicYml]] as const) { + expect(source, `${name} must use the register-gstack-skills composite`) + .toContain('uses: ./.github/actions/register-gstack-skills'); + // No inline re-implementation creeping back beside the composite. + expect(source, `${name} re-inlines the skill registry instead of using the composite`) + .not.toContain('ln -snf "$REPO" "$SKILLS_DIR/gstack"'); + } + }); + + test('the register composite carries the fail-fast verification loop', () => { + // The loop is the POINT of the composite: a dangling symlink or renamed + // committed target fails in seconds with a named path, never as a wedged + // PTY session at the shard wall. Pin its load-bearing markers. + expect(registerAction).toContain('skill registry OK'); + expect(registerAction).toContain('skill-registry target missing'); + expect(registerAction).toContain('gstack root symlink dangles'); + expect(registerAction).toMatch(/grep -m1 "\^name: \$s\\\$"/); + }); + + test('seed/deps/temp composites exist and both lanes use them', () => { + for (const action of ['seed-claude-config', 'restore-deps', 'fix-bun-temp']) { + expect(fs.existsSync(path.join(ROOT, '.github', 'actions', action, 'action.yml')), `missing composite: ${action}`).toBe(true); + } + for (const [name, source] of [['evals.yml', evalsYml], ['evals-periodic.yml', periodicYml]] as const) { + expect(source, `${name} must use seed-claude-config`).toContain('uses: ./.github/actions/seed-claude-config'); + expect(source, `${name} must use restore-deps`).toContain('uses: ./.github/actions/restore-deps'); + expect(source, `${name} must use fix-bun-temp`).toContain('uses: ./.github/actions/fix-bun-temp'); + } + }); +}); diff --git a/test/paid-orphan-tripwire.test.ts b/test/paid-orphan-tripwire.test.ts index b1f0bb1f6..15e76604a 100644 --- a/test/paid-orphan-tripwire.test.ts +++ b/test/paid-orphan-tripwire.test.ts @@ -7,8 +7,9 @@ * execute anywhere — forever, silently. Four files were in that state * (codex-e2e-plan-format, codex-e2e-recommendation-substance, * llm-judge-recommendation, carve-section-loading), and the tripwire built - * for the adjacent class (test/evals-workflow-matrix.test.ts) couldn't see - * them because it filters on isPaidTestFile() FIRST. + * for the adjacent class (the since-retired evals-workflow-matrix test; + * successor: test/evals-workflow-wiring.test.ts) couldn't see them because + * it filtered on isPaidTestFile() FIRST. * * Detection is over source text, so meta-tests and helpers that mention the * gate patterns need reasoned exemptions (same convention as @@ -30,7 +31,7 @@ const SCANNER_EXEMPT = new Map([ // Meta-tests that quote gate-pattern strings to test classification: ['test/helpers/e2e-gate.unit.test.ts', 'free unit test OF the gate predicates (env stubbed)'], ['test/paid-shards.test.ts', 'quotes tier-guard strings as classification fixtures'], - ['test/evals-workflow-matrix.test.ts', 'parses tier guards out of matrix files'], + ['test/evals-workflow-wiring.test.ts', 'pins the sliced-lane yml wiring (successor to the matrix test)'], ['test/e2e-tier-alignment.test.ts', 'parses tier guards to enforce alignment'], ['test/paid-orphan-tripwire.test.ts', 'this scanner'], ]);