gstack/.github/workflows/evals.yml

name: E2E Evals
on:
  pull_request:
    branches: [main]
  workflow_dispatch:
    inputs:
      evals_all:
        description: 'Run ALL gate tests in the sliced lane (bypass diff selection; also arms the hollow-shard guard)'
        type: boolean
        default: true

concurrency:
  group: evals-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: true

env:
  IMAGE: ghcr.io/${{ github.repository }}/ci

jobs:
  # Build Docker image with pre-baked toolchain (cached — only rebuilds on Dockerfile/lockfile change)
  build-image:
    # Dependabot-triggered pull_request runs get a read-only GITHUB_TOKEN, so
    # a lockfile bump = new hash = failed ghcr push = permanently red check
    # (EV6, fork port wave 2). Skip the build for dependabot; the evals job's
    # needs-chain tolerates it because no eval test selects on a lockfile-only
    # diff — a maintainer's next push rebuilds the image with real perms.
    if: github.actor != 'dependabot[bot]'
    runs-on: ubicloud-standard-8
    timeout-minutes: 15
    permissions:
      contents: read
      packages: write
    outputs:
      image-tag: ${{ steps.meta.outputs.tag }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

      - id: meta
        # Key on Dockerfile + lockfile only. package.json is deliberately NOT
        # hashed: its version field changes on every ship (60/60 recent commits),
        # which rebuilt the image each time for a dependency set that only
        # bun.lock determines. A stale baked package.json is harmless — checkout
        # overwrites /workspace and node_modules comes from the lockfile.
        run: echo "tag=${{ env.IMAGE }}:${{ hashFiles('.github/docker/Dockerfile.ci', 'bun.lock', 'patches/**') }}" >> "$GITHUB_OUTPUT"

      - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Check if image exists
        id: check
        run: |
          if docker manifest inspect ${{ steps.meta.outputs.tag }} > /dev/null 2>&1; then
            echo "exists=true" >> "$GITHUB_OUTPUT"
          else
            echo "exists=false" >> "$GITHUB_OUTPUT"
          fi

      - if: steps.check.outputs.exists == 'false'
        run: cp package.json bun.lock .github/docker/ && cp -R patches .github/docker/patches

      # A fork PR's GITHUB_TOKEN only has `packages: read`, so pushing fails.
      # Still BUILD (validates Dockerfile.ci changes), just don't publish. This
      # job intentionally keeps no `if:` so fork PRs still get one real, honest
      # green check here instead of a run where every job is grey.
      # Registry cache export needs a docker-container builder — the default
      # `docker` driver hard-errors on cache-to (first live run of the trio).
      - if: steps.check.outputs.exists == 'false'
        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4

      - if: steps.check.outputs.exists == 'false'
        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
        with:
          context: .github/docker
          file: .github/docker/Dockerfile.ci
          push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
          # Registry layer cache: reads are safe everywhere; the export is gated
          # to same-repo runs because a fork PR's token can't write GHCR.
          cache-from: type=registry,ref=${{ env.IMAGE }}:buildcache
          cache-to: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && format('type=registry,ref={0}:buildcache,mode=max', env.IMAGE) || '' }}
          tags: |
            ${{ steps.meta.outputs.tag }}
            ${{ env.IMAGE }}:latest

  # ── Sliced lane (the ONLY paid lane; legacy 17-row matrix deleted) ──────────
  # One PLANNER computes diff selection + the slice plan ONCE (killing
  # per-slice selector divergence); K executors consume the manifest; the
  # report reconciles results against it FAIL-CLOSED (a slice whose artifact
  # never landed is a failure, a planned shard nobody reported is a failure —
  # hollow lanes cannot aggregate green). Engine: scripts/test-paid-shards.ts —
  # the same runner local eval:bg:gate uses, so CI and local share one
  # selection engine, and every gate-tier file is in the census by
  # construction (no hand-enumerated rows to drift). The legacy matrix ran
  # 18 enumerated files for 22.6 min/$21 per PR serialized AHEAD of this
  # lane's 49-file diff-selected census; parity was demonstrated (sliced
  # census ⊇ matrix files) and the matrix deleted — one revert restores it.
  #
  # Fork PRs never receive repository secrets (ANTHROPIC_API_KEY et al), so
  # every API-calling eval fails at SDK auth before a model runs. Skip
  # deterministically; fork work gets real coverage via a trusted base-repo
  # branch (see CLAUDE.md's garrytan-agents workflow).
  plan-slices:
    runs-on: ubicloud-standard-8
    needs: build-image
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
    timeout-minutes: 10
    permissions:
      contents: read
      packages: read
    container:
      image: ${{ needs.build-image.outputs.image-tag }}
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
      options: --user runner
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          # The planner is the ONE place that needs history: diff selection
          # resolves a merge-base. Executors run from the manifest and stay
          # shallow. Selection fails OPEN (run-all) if resolution fails — the
          # documented posture; a planner bug can only run extra work.
          fetch-depth: 0
          persist-credentials: false

      - name: Restore deps
        uses: ./.github/actions/restore-deps

      - name: Emit run manifest
        env:
          EVALS_ALL: ${{ (github.event_name == 'workflow_dispatch' && inputs.evals_all) && '1' || '' }}
        run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/paid-plan/manifest.json --slices 6

      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: paid-plan
          path: /tmp/paid-plan/manifest.json
          retention-days: 30

  eval-slices:
    runs-on: ubicloud-standard-8
    needs: [build-image, plan-slices]
    if: always() && needs.plan-slices.result == 'success'
    # Aggregate spawn-concurrency budget: 6 slices x EVALS_JOBS=2 x
    # EVALS_CONCURRENCY=2 = 24 concurrent tests lane-wide (the old matrix's
    # 40-way per row queued claude session STARTUP behind 39 siblings and ate
    # per-test budgets — the documented timeout-flake family). Tune with
    # parity data before raising.
    timeout-minutes: 35
    permissions:
      contents: read
      packages: read
    container:
      image: ${{ needs.build-image.outputs.image-tag }}
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
      options: --user runner
    strategy:
      fail-fast: false
      matrix:
        slice: [1, 2, 3, 4, 5, 6]
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          # Full history: files with SELF-derived selection (the LLM-judge
          # map, routing) walk git at module load, and selection is
          # fail-closed on git errors — a shallow checkout crashed those
          # shards on the lane's first live run ("ambiguous argument
          # 'main...HEAD'"). The manifest still governs WHICH shards run.
          fetch-depth: 0
          persist-credentials: false

      - name: Fix bun temp
        uses: ./.github/actions/fix-bun-temp

      - name: Restore deps
        uses: ./.github/actions/restore-deps

      - run: bun run build

      # Any slice can host a PTY smoke, so the seed/registration steps run
      # UNCONDITIONALLY (both are idempotent) — the old matrix keyed them on
      # matrix.suite.name, which a sliced lane cannot do. The register
      # composite carries the fail-fast dangling-symlink/frontmatter
      # verification loop, so a moved skill target fails HERE in seconds,
      # not as a wedged PTY session at the shard wall.
      - name: Seed claude interactive config
        uses: ./.github/actions/seed-claude-config
        with:
          anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}

      - name: Register gstack skills for PTY smokes
        uses: ./.github/actions/register-gstack-skills

      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: paid-plan
          path: /tmp/paid-plan

      - name: Run slice ${{ matrix.slice }}/6
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
          PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers
          EVALS_JOBS: "2"
          EVALS_CONCURRENCY: "2"
          GSTACK_EVAL_DIR: /tmp/paid-slice-results
        run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/paid-plan/manifest.json --slice ${{ matrix.slice }}

      - name: Upload slice results
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: paid-slice-${{ matrix.slice }}
          path: /tmp/paid-slice-results
          retention-days: 90

      # The spooled per-shard full logs — a red weekly/PR lane three weeks
      # later needs more than a summary line.
      - name: Upload shard logs on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: paid-slice-${{ matrix.slice }}-logs
          # The Fix-bun-temp step points TMPDIR at /home/runner/.cache, so the
          # runner's spool lands THERE, not /tmp — the original /tmp glob
          # uploaded nothing and a red slice's diagnostics were unreachable.
          path: |
            /home/runner/.cache/gstack-paid-shard-*.log
            /tmp/gstack-paid-shard-*.log
          if-no-files-found: ignore
          retention-days: 30

  slices-report:
    runs-on: ubicloud-standard-2
    needs: [plan-slices, eval-slices]
    # always(): the report must run (and FAIL) when an executor died — a
    # missing slice artifact reading as green is the class this lane kills.
    if: always() && needs.plan-slices.result == 'success'
    timeout-minutes: 5
    # contents:read ONLY — this job executes PR-authored code (bun install
    # lifecycle scripts + the reconcile runner from the PR checkout), so it
    # must never hold a write-scoped token. The PR comment lives in the
    # separate slices-comment job below, which runs NO repo code: a
    # $GITHUB_ENV/BASH_ENV persistence trick is job-scoped, so the split is
    # the trust boundary (codex adversarial finding, 2026-08-31 — the old
    # matrix-era report job had this separation and the consolidation had
    # regressed it).
    permissions:
      contents: read
    outputs:
      reconcile-exit: ${{ steps.reconcile.outputs.exit }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false

      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
        with:
          bun-version: 1.4.0

      - run: bun install --frozen-lockfile

      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: paid-plan
          path: /tmp/paid-report

      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          pattern: paid-slice-[0-9]*
          path: /tmp/paid-report
          merge-multiple: true

      - name: Reconcile slices against the manifest (fail-closed)
        id: reconcile
        run: |
          set +e
          EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report /tmp/paid-report | tee /tmp/report.txt
          # PIPESTATUS[0], NOT $?: GitHub's default run-step shell is
          # `bash -e {0}` with NO pipefail, so $? after the pipe is tee's
          # exit (always 0) — the fail-closed gate was silently fail-open
          # (caught by the ship review army; the wiring test now pins this).
          echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"

      - name: Upload reconciliation output for the comment job
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: report-verdict
          path: /tmp/report.txt
          if-no-files-found: ignore
          retention-days: 30

      - name: Fail the workflow when reconciliation failed
        if: steps.reconcile.outputs.exit != '0'
        run: exit 1

  # PR comment in its OWN job with the write token and ZERO repo code: no
  # checkout, no bun install — only downloaded artifacts, jq, and gh. See the
  # trust-boundary note on slices-report.
  slices-comment:
    runs-on: ubicloud-standard-2
    needs: slices-report
    if: always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && needs.slices-report.result != 'skipped'
    timeout-minutes: 5
    permissions:
      pull-requests: write
      # The comment upsert calls the REST `/issues/{n}/comments` endpoints
      # (gh api ... issues/comments). With GITHUB_TOKEN those are gated by the
      # `issues` permission, not `pull-requests` (#1802 CI fix).
      issues: write
    steps:
      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          pattern: paid-slice-[0-9]*
          path: /tmp/paid-report
          merge-multiple: true

      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: report-verdict
          path: /tmp/verdict
        continue-on-error: true

      # Sourced from the slice artifacts' eval-store JSONs. Keeps the
      # "## E2E Evals" marker so the upsert keeps updating the same comment.
      # Runs even when reconciliation failed — a red lane on the PR is the point.
      - name: Post PR comment
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          RECONCILE_EXIT: ${{ needs.slices-report.outputs.reconcile-exit }}
        run: |
          # shellcheck disable=SC2086,SC2059
          RESULTS=$(find /tmp/paid-report -name '*.json' ! -name 'manifest.json' ! -name 'slice-*.json' ! -name '_partial*' 2>/dev/null | sort)
          TOTAL=0; PASSED=0; FAILED=0; FLAKY=0; COST="0"
          SUITE_LINES=""
          for f in $RESULTS; do
            if ! jq -e '.total_tests' "$f" >/dev/null 2>&1; then
              echo "Skipping malformed JSON: $f"
              continue
            fi
            # FINAL-attempt accounting: eval-store keeps EVERY retry attempt
            # as its own record (that's the flake telemetry), so counting raw
            # records marks a pass-on-retry as a failure and inflates totals.
            # Group by test name, judge the LAST record — flaky passes render
            # as the ⚠ line, never as ❌ (WS1 policy: recorded, not blocking).
            # Guarded: a file with total_tests but a null/non-array `tests`
            # passes the -e probe, the group_by then fails, and an empty $T
            # would abort the whole step under bash -e ([ "" -eq 0 ] is an
            # error) — killing the comment on exactly the corrupted-artifact
            # runs where the red evidence matters (claude adversarial).
            STATS=$(jq -r '[.tests | group_by(.name)[] | last] as $final | "\($final | length) \([$final[] | select(.passed)] | length) \([$final[] | select(.passed | not)] | length) \(.flaky_retries // [] | length)"' "$f" 2>/dev/null) || { echo "Skipping malformed tests[] in: $f"; continue; }
            read -r T P F FL <<< "$STATS"
            [ -z "$T" ] && { echo "Skipping malformed tests[] in: $f"; continue; }
            C=$(jq -r '.total_cost_usd // 0' "$f")
            TIER=$(jq -r '.tier // "unknown"' "$f")
            SHARD=$(jq -r '.shard // "-"' "$f")
            [ "$T" -eq 0 ] && continue
            TOTAL=$((TOTAL + T))
            PASSED=$((PASSED + P))
            FAILED=$((FAILED + F))
            FLAKY=$((FLAKY + FL))
            COST=$(echo "$COST + $C" | bc)
            STATUS_ICON="✅"
            [ "$F" -gt 0 ] && STATUS_ICON="❌"
            [ "$F" -eq 0 ] && [ "$FL" -gt 0 ] && STATUS_ICON="✅⚠"
            SUITE_LINES="${SUITE_LINES}| ${TIER}/${SHARD} | ${P}/${T} | ${STATUS_ICON} | \$${C} |\n"
          done

          STATUS="✅ PASS"
          if [ "${RECONCILE_EXIT:-1}" != "0" ] || [ "$FAILED" -gt 0 ]; then STATUS="❌ FAIL"; fi

          BODY="## E2E Evals: ${STATUS}

          **${PASSED}/${TOTAL}** tests passed | **\$${COST}** total cost | reconcile exit: ${RECONCILE_EXIT:-missing}$([ "$FLAKY" -gt 0 ] && printf ' | ⚠ %s flaky pass(es) — recorded, not blocking' "$FLAKY")

          | Shard | Result | Status | Cost |
          |-------|--------|--------|------|
          $(echo -e "$SUITE_LINES")

          <details><summary>Fail-closed reconciliation</summary>

          \`\`\`
          $(tail -c 4000 /tmp/verdict/report.txt 2>/dev/null || echo '(no reconciliation output)')
          \`\`\`
          </details>

          ---
          *Sliced lane: diff-selected gate census via scripts/test-paid-shards.ts (planner → 6 executors → fail-closed report)*"

          if [ "$FAILED" -gt 0 ]; then
            FAILURES=""
            for f in $RESULTS; do
              if ! jq -e '.failed' "$f" >/dev/null 2>&1; then continue; fi
              FAILS=$(jq -r '[.tests | group_by(.name)[] | last | select(.passed == false)][] | "- ❌ \(.name): \(.exit_reason // "unknown")"' "$f" 2>/dev/null || echo "- ⚠️ parse error")
              FAILURES="${FAILURES}${FAILS}\n"
            done
            BODY="${BODY}

          ### Failures
          $(echo -e "$FAILURES")"
          fi

          COMMENT_ID=$(gh api repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments \
            --jq '.[] | select(.body | startswith("## E2E Evals")) | .id' | tail -1)

          if [ -n "$COMMENT_ID" ]; then
            gh api "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" \
              -X PATCH -f body="$BODY"
          else
            # REST, not gh's pr-comment subcommand: this job runs with NO
            # checkout (the token/exec split), and that subcommand resolves
            # the repo FROM git — it dies with "not a git repository" here.
            gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \
              -X POST -f body="$BODY"
          fi