gstack/.github/workflows/evals-periodic.yml

name: Periodic Evals
# The weekly coverage contract: EVERY periodic-tier paid test runs (EVALS_ALL,
# minus the reasoned excludes in test/helpers/periodic-exclude-data.ts), so
# tests can't rot invisibly — the class where the autoplan-dual-voice E2E was
# silently broken for months until a lucky local diff selected it. Engine:
# scripts/test-paid-shards.ts (the same runner local eval:bg:periodic uses):
# one planner manifest packed by recorded durations into as many ~7-minute
# executors as the work needs (one file, or a tightly packed group, per
# runner; overlays share one final slice), and a FAIL-CLOSED report — a slice
# whose artifact never landed is a failure, not an absence. The matrix size
# and job timeout come from the plan, so they cannot drift from the census.
# Full end-to-end flows run in the non-blocking marathon lane
# (evals-marathon.yml), never here. The gate-census
# job is the weekly EVALS_ALL backstop for the gate tier (PR lanes are
# diff-billed, so without it the full gate census might never execute
# anywhere); the hollow-shard guard (exit 0 + zero executed tests under
# EVALS_ALL fails) makes both lanes census-health checks, not just test runs.
# eval-bg (scripts/eval-bg.ts) finds its dispatch by the nonce in the run name.
run-name: ${{ inputs.nonce && format('Periodic Evals [eval-bg {0}]', inputs.nonce) || 'Periodic Evals' }}
on:
  schedule:
    - cron: '0 6 * * 1'  # Monday 6 AM UTC; build-image below builds or reuses the CI image
    # Daily, for autoplan-schema-canary alone (latest Claude Code vs the /autoplan
    # guard's payload assumptions); every census job below skips this schedule.
    - cron: '30 7 * * *'
  workflow_dispatch:
    inputs:
      redispatch_of:
        description: 'Run id this run re-dispatches (the one INFRA/INCOMPLETE-only re-dispatch; set by the report job)'
        type: string
        default: ''
      include_gate_census:
        description: 'Branch dispatch only: also run the weekly gate census (default: periodic lane only; gate cases validate through an evals.yml dispatch with evals_all)'
        type: boolean
        default: false
      expected_sha:
        description: 'eval-bg: the revision the dispatcher tested; the run fails fast when the ref has moved'
        type: string
        default: ''
      nonce:
        description: 'eval-bg: dispatch nonce carried in the run name'
        type: string
        default: ''
      schema_canary_only:
        description: 'Run only autoplan-schema-canary (the daily latest-Claude-Code canary), not the census'
        type: boolean
        default: false

# One group per ref AND event: a branch dispatch never cancels main's census,
# and a manual main dispatch never cancels the scheduled one. A re-dispatch
# runs in its own group so it never cancels the run that dispatched it; both
# runs are reported.
concurrency:
  group: evals-periodic-${{ github.ref }}-${{ github.event_name }}${{ inputs.redispatch_of && format('-redispatch-{0}', inputs.redispatch_of) || '' }}${{ (github.event.schedule == '30 7 * * *' || inputs.schema_canary_only) && '-schema-canary' || '' }}
  cancel-in-progress: true

env:
  IMAGE: ghcr.io/${{ github.repository }}/ci
  EVALS_PROFILE: full
  EVALS_FRESH: "1"
  EVALS_CACHE_PURPOSE: periodic

jobs:
  build-image:
    runs-on: ubicloud-standard-8
    timeout-minutes: 15
    permissions:
      contents: read
      packages: write
    outputs:
      image-tag: ${{ steps.meta.outputs.tag }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

      - id: meta
        # Keep in sync with evals.yml — key on Dockerfile + lockfile only
        # (package.json's version field would bust the key on every ship).
        # Byte-identity pinned by test/ci-image-tag-binding.test.ts.
        run: echo "tag=${{ env.IMAGE }}:${{ hashFiles('.github/docker/Dockerfile.ci', 'bun.lock', 'patches/**') }}" >> "$GITHUB_OUTPUT"

      - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Check if image exists
        id: check
        run: |
          if docker manifest inspect ${{ steps.meta.outputs.tag }} > /dev/null 2>&1; then
            echo "exists=true" >> "$GITHUB_OUTPUT"
          else
            echo "exists=false" >> "$GITHUB_OUTPUT"
          fi

      - if: steps.check.outputs.exists == 'false'
        run: cp package.json bun.lock .github/docker/ && cp -R patches .github/docker/patches

      # Registry cache export needs a docker-container builder — the default
      # `docker` driver hard-errors on cache-to.
      - if: steps.check.outputs.exists == 'false'
        uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1

      - if: steps.check.outputs.exists == 'false'
        uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
        with:
          context: .github/docker
          file: .github/docker/Dockerfile.ci
          push: true
          # Cron-triggered in the base repo only, so cache export is always safe here.
          cache-from: type=registry,ref=${{ env.IMAGE }}:buildcache
          cache-to: type=registry,ref=${{ env.IMAGE }}:buildcache,mode=max
          tags: |
            ${{ steps.meta.outputs.tag }}
            ${{ env.IMAGE }}:latest

  plan-slices:
    runs-on: ubicloud-standard-8
    timeout-minutes: 10
    # The daily schedule and a canary-only dispatch run schema-canary alone;
    # every census job needs this one, so they all skip with it.
    if: ${{ github.event.schedule != '30 7 * * *' && !inputs.schema_canary_only }}
    permissions:
      contents: read
    outputs:
      periodic_slices: ${{ steps.periodic-matrix.outputs.slices }}
      periodic_codex_slices: ${{ steps.periodic-matrix.outputs.codex_slices }}
      periodic_slice_timeouts: ${{ steps.periodic-matrix.outputs.slice_timeouts }}
      gate_slices: ${{ steps.gate-matrix.outputs.slices }}
      gate_slice_timeouts: ${{ steps.gate-matrix.outputs.slice_timeouts }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false

      - name: Verify the dispatched revision
        if: inputs.expected_sha != ''
        env:
          EXPECTED_SHA: ${{ inputs.expected_sha }}
        run: |
          if [ "$GITHUB_SHA" != "$EXPECTED_SHA" ]; then
            echo "::error::eval-bg tested ${EXPECTED_SHA} but ${GITHUB_REF_NAME} is now at ${GITHUB_SHA}; push again and re-run eval:bg, or dispatch without expected_sha"
            exit 1
          fi

      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
        with:
          bun-version: 1.4.2

      - name: Emit run manifest (ALL periodic tests minus reasoned excludes)
        env:
          EVALS_ALL: "1"
        run: EVALS_TIER=periodic bun --no-install run scripts/test-paid-shards.ts --tier periodic --emit-plan /tmp/paid-plan/manifest.json --slice-budget 420 --jobs 2 --max-parallel 38

      # Codex-only slices run on the host-run eval-codex-slices job; the
      # container executor runs every other slice.
      - name: Derive the periodic executor matrix from the plan
        id: periodic-matrix
        run: |
          {
            echo "slices=$(jq -c '[range(1; .sliceCount + 1)] - .plan.codexSlices' /tmp/paid-plan/manifest.json)"
            echo "codex_slices=$(jq -c '.plan.codexSlices' /tmp/paid-plan/manifest.json)"
            echo "slice_timeouts=$(jq -ec '[0] + .plan.sliceCiTimeoutMinutes' /tmp/paid-plan/manifest.json)"
          } >> "$GITHUB_OUTPUT"

      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
        with:
          name: paid-plan
          path: /tmp/paid-plan/manifest.json
          retention-days: 30

      - name: Emit gate census manifest (ALL gate tests)
        env:
          EVALS_ALL: "1"
        run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/gate-census-plan/manifest.json --slice-budget 420 --jobs 2 --skip-judges --max-parallel 20

      - name: Derive the gate census executor matrix from the plan
        id: gate-matrix
        run: |
          echo "slices=$(jq -c '[range(1; .sliceCount + 1)]' /tmp/gate-census-plan/manifest.json)" >> "$GITHUB_OUTPUT"
          echo "slice_timeouts=$(jq -ec '[0] + .plan.sliceCiTimeoutMinutes' /tmp/gate-census-plan/manifest.json)" >> "$GITHUB_OUTPUT"

      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
        with:
          name: gate-census-plan
          path: /tmp/gate-census-plan/manifest.json
          retention-days: 30

  # autoplan-schema-canary: installs the latest published Claude Code with
  # install scripts off, audits registry signatures, and checks the Agent and
  # Read hook payloads against the journal with the /autoplan guard's own
  # comparison. Daily, so a schema strip reds within a day of a release; the
  # weekly census also runs it. Read-only token; the key's spend is capped by
  # the case (one haiku session, about $0.02).
  schema-canary:
    runs-on: ubicloud-standard-2
    needs: build-image
    if: ${{ github.event.schedule == '30 7 * * *' || inputs.schema_canary_only }}
    timeout-minutes: 20
    permissions:
      contents: read
      packages: read
    container:
      image: ${{ needs.build-image.outputs.image-tag }}
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
      options: --user runner
    env:
      EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-schema-canary
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false

      - name: Fix bun temp
        uses: ./.github/actions/fix-bun-temp

      - name: Restore deps
        uses: ./.github/actions/restore-deps

      - name: Run autoplan-schema-canary on the latest Claude Code
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          GSTACK_EVAL_DIR: /tmp/schema-canary-results
        run: EVALS_TIER=periodic bun run scripts/test-paid-shards.ts --tier periodic --case autoplan-schema-canary --trials 1

      - name: Upload canary results
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
        with:
          name: schema-canary-a${{ github.run_attempt }}
          path: /tmp/schema-canary-results
          if-no-files-found: ignore
          retention-days: 90

  eval-slices:
    runs-on: ubicloud-standard-8
    needs: [build-image, plan-slices]
    env:
      EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-slices-${{ matrix.slice }}
    # The planner packs ~7 minutes of recorded work per slice; each slice's job
    # ceiling is max(2x budget, its longest shard wall, the overlay envelope)
    # plus 20 minutes setup (W2c/ENG-2), indexed by slice from the manifest the
    # slices execute ([0] pads the array so index = slice). The executor stops
    # starting work 5 minutes before the ceiling so its results still upload.
    timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.periodic_slice_timeouts)[matrix.slice] }}
    permissions:
      contents: read
      packages: read
    container:
      image: ${{ needs.build-image.outputs.image-tag }}
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
      options: --user runner
    strategy:
      fail-fast: false
      # Every planned slice starts at once; test/evals-workflow-wiring.test.ts
      # fails when the live plan outgrows this cap.
      max-parallel: 38
      matrix:
        slice: ${{ fromJSON(needs.plan-slices.outputs.periodic_slices) }}
    steps:
      - name: Record the job start for the slice deadline
        run: echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"

      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          # Full history: files with SELF-derived selection (the LLM-judge
          # map, routing) walk git at module load, and selection is
          # fail-closed on git errors — a shallow checkout crashed those
          # shards on the lane's first live run ("ambiguous argument
          # 'main...HEAD'"). The manifest still governs WHICH shards run.
          fetch-depth: 0
          persist-credentials: false

      - name: Fix bun temp
        uses: ./.github/actions/fix-bun-temp

      - name: Restore deps
        uses: ./.github/actions/restore-deps

      - run: bun run build

      # Any slice can host a PTY test — seed + registration run
      # unconditionally (idempotent; mirrors evals.yml's sliced lane). The
      # register composite carries the fail-fast dangling-symlink/frontmatter
      # verification loop — this lane previously LACKED it, so a moved skill
      # target surfaced as a silent "Unknown command" + wedged PTY session.
      - name: Seed claude interactive config
        uses: ./.github/actions/seed-claude-config
        with:
          anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}

      - name: Register gstack skills for PTY tests
        uses: ./.github/actions/register-gstack-skills

      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
        with:
          name: paid-plan
          path: /tmp/paid-plan

      # The image ships the pinned Codex CLI off PATH and without a
      # credential; codex does not read OPENAI_API_KEY from the environment.
      # Log it in from the secret (same-repo runs only) under a non-default
      # home. The shard runner hands that home and the CLI to the Codex files
      # only (scripts/lib/paid-cases.ts scopeCodexAccess); every other paid case sees no
      # Codex, as before. Without a key the Codex cases fail; they never pass.
      - name: Authenticate the Codex CLI
        shell: bash
        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          set -euo pipefail
          codex="${GSTACK_CI_CODEX_BIN_DIR:?}/codex"
          export CODEX_HOME="$HOME/.gstack-ci-codex"
          mkdir -p "$CODEX_HOME"
          echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV"
          if [ -z "${OPENAI_API_KEY:-}" ]; then
            echo "::warning::OPENAI_API_KEY is unavailable; Codex eval cases cannot authenticate and will fail."
            exit 0
          fi
          printenv OPENAI_API_KEY | "$codex" login --with-api-key
          "$codex" login status >/dev/null 2>&1

      - name: Run periodic slice ${{ matrix.slice }}
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
          PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers
          EVALS_JOBS: "2"
          EVALS_CONCURRENCY: "2"
          GSTACK_EVAL_DIR: /tmp/paid-slice-results
        run: EVALS_TIER=periodic bun run scripts/test-paid-shards.ts --tier periodic --plan /tmp/paid-plan/manifest.json --slice ${{ matrix.slice }}

      - name: Upload slice results
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
        with:
          name: paid-slice-${{ matrix.slice }}-a${{ github.run_attempt }}
          path: /tmp/paid-slice-results
          retention-days: 90

      - name: Upload native capture evidence
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
        with:
          name: native-captures-${{ env.EVALS_RUN_ID }}
          include-hidden-files: true
          path: |
            ~/.gstack/projects/*/e2e-runs
            ~/.gstack/projects/*/evals/qa-callers
            ~/.gstack-dev/e2e-runs
            ~/.gstack-dev/evals/qa-callers
          if-no-files-found: ignore
          retention-days: 90

      # always(), not failure(): a failed behavior trial is a verdict and no
      # longer reds its runner, but its full log is the diagnosis evidence.
      - name: Upload shard logs
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
        with:
          name: paid-logs-slice-${{ matrix.slice }}-a${{ github.run_attempt }}
          include-hidden-files: true
          # The Fix-bun-temp step points TMPDIR at /home/runner/.cache, so the
          # runner's spool lands THERE, not /tmp — the original /tmp glob
          # uploaded nothing and a red slice's diagnostics were unreachable.
          path: |
            /home/runner/.cache/gstack-paid-shard-*.log
            /tmp/gstack-paid-shard-*.log
          if-no-files-found: ignore
          retention-days: 30

  # Codex shards on the host. Codex runs every shell command inside a
  # bubblewrap sandbox, which needs an unprivileged user namespace. A job
  # container cannot get one: Docker's default seccomp and AppArmor profiles
  # block it, and Ubuntu 24.04 hosts set
  # kernel.apparmor_restrict_unprivileged_userns=1, which no container option
  # can change. This job runs on the runner VM itself (ephemeral, one job),
  # lifts that host restriction, and runs the same CI image with both
  # profiles unconfined. Only the planner's Codex slices run here
  # (scripts/lib/paid-plan.ts codexSlicesOf); slice artifacts use the same
  # names as eval-slices, so the report reconciles one manifest.
  eval-codex-slices:
    runs-on: ubicloud-standard-8
    needs: [build-image, plan-slices]
    if: ${{ needs.plan-slices.outputs.periodic_codex_slices != '[]' }}
    env:
      EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-slices-${{ matrix.slice }}
    timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.periodic_slice_timeouts)[matrix.slice] }}
    permissions:
      contents: read
      packages: read
    strategy:
      fail-fast: false
      max-parallel: 38
      matrix:
        slice: ${{ fromJSON(needs.plan-slices.outputs.periodic_codex_slices) }}
    steps:
      - name: Record the job start for the slice deadline
        run: echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"

      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          fetch-depth: 0
          persist-credentials: false

      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
        with:
          name: paid-plan
          path: ${{ runner.temp }}/paid-plan

      - name: Let Codex's bubblewrap sandbox create user namespaces
        run: |
          if [ -e /proc/sys/kernel/apparmor_restrict_unprivileged_userns ]; then
            sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
          fi

      - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      # The same steps as eval-slices (bun temp, deps, build, Codex login,
      # slice run) inside the CI image. The container's home is a fresh
      # runner-temp directory, so logs and captures upload from there.
      - name: Run periodic Codex slice ${{ matrix.slice }}
        shell: bash
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
          EVAL_IMAGE: ${{ needs.build-image.outputs.image-tag }}
          SLICE: ${{ matrix.slice }}
          EVALS_JOBS: "2"
        run: |
          mkdir -p "$RUNNER_TEMP/eval-home" "$RUNNER_TEMP/paid-slice-results"
          docker run --rm --user "$(id -u):$(id -g)" \
            --security-opt seccomp=unconfined --security-opt apparmor=unconfined \
            -e HOME=/home/runner -e ANTHROPIC_API_KEY -e OPENAI_API_KEY -e GEMINI_API_KEY -e SLICE -e EVALS_JOBS -e GSTACK_SLICE_JOB_STARTED_AT \
            -e EVALS_RUN_ID -e EVALS_PROFILE -e EVALS_FRESH -e EVALS_CACHE_PURPOSE -e CI -e GITHUB_ACTIONS \
            -e GITHUB_RUN_ID -e GITHUB_RUN_ATTEMPT -e GITHUB_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REPOSITORY \
            -e GITHUB_EVENT_NAME -e GITHUB_BASE_REF -e GITHUB_HEAD_REF \
            -e PLAYWRIGHT_BROWSERS_PATH=/opt/playwright-browsers -e EVALS_CONCURRENCY=2 -e GSTACK_EVAL_DIR=/tmp/paid-slice-results \
            -v "$GITHUB_WORKSPACE:/workspace/gstack" -w /workspace/gstack \
            -v "$RUNNER_TEMP/eval-home:/home/runner" \
            -v "$RUNNER_TEMP/paid-plan:/tmp/paid-plan:ro" \
            -v "$RUNNER_TEMP/paid-slice-results:/tmp/paid-slice-results" \
            "$EVAL_IMAGE" bash -c '
            set -euo pipefail
            mkdir -p "$HOME/.cache/bun"
            export BUN_INSTALL_CACHE_DIR="$HOME/.cache/bun" BUN_TMPDIR="$HOME/.cache/bun" TMPDIR="$HOME/.cache"
            if diff -q /opt/node_modules_cache/.bun.lock bun.lock >/dev/null 2>&1; then
              rm -rf node_modules && cp -r /opt/node_modules_cache node_modules
            else
              bun install --frozen-lockfile
            fi
            bun run build
            codex="${GSTACK_CI_CODEX_BIN_DIR:?}/codex"
            export CODEX_HOME="$HOME/.gstack-ci-codex"
            mkdir -p "$CODEX_HOME"
            if [ -n "${OPENAI_API_KEY:-}" ]; then
              printenv OPENAI_API_KEY | "$codex" login --with-api-key
              "$codex" login status >/dev/null 2>&1
            else
              echo "::warning::OPENAI_API_KEY is unavailable; Codex eval cases cannot authenticate and will fail."
            fi
            export GSTACK_CI_CODEX_HOME="$CODEX_HOME"
            unset CODEX_HOME
            EVALS_TIER=periodic bun run scripts/test-paid-shards.ts --tier periodic --plan /tmp/paid-plan/manifest.json --slice "$SLICE"
            '

      - name: Upload slice results
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
        with:
          name: paid-slice-${{ matrix.slice }}-a${{ github.run_attempt }}
          path: ${{ runner.temp }}/paid-slice-results
          retention-days: 90

      - name: Upload native capture evidence
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
        with:
          name: native-captures-${{ env.EVALS_RUN_ID }}
          include-hidden-files: true
          path: |
            ${{ runner.temp }}/eval-home/.gstack/projects/*/e2e-runs
            ${{ runner.temp }}/eval-home/.gstack/projects/*/evals/qa-callers
            ${{ runner.temp }}/eval-home/.gstack-dev/e2e-runs
            ${{ runner.temp }}/eval-home/.gstack-dev/evals/qa-callers
          if-no-files-found: ignore
          retention-days: 90

      - name: Upload shard logs
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
        with:
          name: paid-logs-slice-${{ matrix.slice }}-a${{ github.run_attempt }}
          include-hidden-files: true
          path: ${{ runner.temp }}/eval-home/.cache/gstack-paid-shard-*.log
          if-no-files-found: ignore
          retention-days: 30

  # Weekly EVALS_ALL gate-tier census: PR lanes are diff-billed, so without
  # this the full gate census might never execute anywhere and the selector's
  # blind spots rot invisibly. Census health, not selector correctness —
  # selector logic has free synthetic-diff contract tests.
  gate-census:
    runs-on: ubicloud-standard-8
    needs: [build-image, plan-slices]
    # A branch dispatch validates the periodic lane only; gate cases on a
    # branch go through an evals.yml dispatch with evals_all. Main (schedule
    # or dispatch) always runs the census; a branch opts in explicitly.
    if: ${{ github.ref == 'refs/heads/main' || inputs.include_gate_census }}
    env:
      EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-gate-census-${{ matrix.slice }}
    # Per-slice job ceiling from the packed plan (W2c/ENG-2), as for eval-slices.
    timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.gate_slice_timeouts)[matrix.slice] }}
    permissions:
      contents: read
      packages: read
    container:
      image: ${{ needs.build-image.outputs.image-tag }}
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
      options: --user runner
    strategy:
      # Two file workers per slice, each retaining two in-file case workers.
      fail-fast: false
      max-parallel: 20
      matrix:
        slice: ${{ fromJSON(needs.plan-slices.outputs.gate_slices) }}
    steps:
      - name: Record the job start for the slice deadline
        run: echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"

      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          # Full history: files with SELF-derived selection (the LLM-judge
          # map, routing) walk git at module load, and selection is
          # fail-closed on git errors — a shallow checkout crashed those
          # shards on the lane's first live run ("ambiguous argument
          # 'main...HEAD'"). The manifest still governs WHICH shards run.
          fetch-depth: 0
          persist-credentials: false

      - name: Fix bun temp
        uses: ./.github/actions/fix-bun-temp

      - name: Restore deps
        uses: ./.github/actions/restore-deps

      - run: bun run build

      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
        with:
          name: gate-census-plan
          path: /tmp/gate-census-plan

      # The image ships the pinned Codex CLI off PATH and without a
      # credential; codex does not read OPENAI_API_KEY from the environment.
      # Log it in from the secret (same-repo runs only) under a non-default
      # home. The shard runner hands that home and the CLI to the Codex files
      # only (scripts/lib/paid-cases.ts scopeCodexAccess); every other paid case sees no
      # Codex, as before. Without a key the Codex cases fail; they never pass.
      - name: Authenticate the Codex CLI
        shell: bash
        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          set -euo pipefail
          codex="${GSTACK_CI_CODEX_BIN_DIR:?}/codex"
          export CODEX_HOME="$HOME/.gstack-ci-codex"
          mkdir -p "$CODEX_HOME"
          echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV"
          if [ -z "${OPENAI_API_KEY:-}" ]; then
            echo "::warning::OPENAI_API_KEY is unavailable; Codex eval cases cannot authenticate and will fail."
            exit 0
          fi
          printenv OPENAI_API_KEY | "$codex" login --with-api-key
          "$codex" login status >/dev/null 2>&1

      - name: Run gate census slice ${{ matrix.slice }}
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
          PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers
          EVALS_JOBS: "2"
          EVALS_CONCURRENCY: "2"
          GSTACK_EVAL_DIR: /tmp/gate-census-results
        run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/gate-census-plan/manifest.json --slice ${{ matrix.slice }}

      - name: Upload census results
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
        with:
          name: gate-census-${{ matrix.slice }}-a${{ github.run_attempt }}
          path: /tmp/gate-census-results
          retention-days: 90

      - name: Upload native capture evidence
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
        with:
          name: native-captures-${{ env.EVALS_RUN_ID }}
          include-hidden-files: true
          path: |
            ~/.gstack/projects/*/e2e-runs
            ~/.gstack/projects/*/evals/qa-callers
            ~/.gstack-dev/e2e-runs
            ~/.gstack-dev/evals/qa-callers
          if-no-files-found: ignore
          retention-days: 90

  report:
    runs-on: ubicloud-standard-2
    needs: [plan-slices, eval-slices, eval-codex-slices, gate-census]
    # !cancelled(): the report must run (and FAIL) when an executor died — a
    # missing slice artifact reading as green is the class this lane kills —
    # but a cancelled run stops here.
    if: ${{ !cancelled() && needs.plan-slices.result == 'success' }}
    timeout-minutes: 15
    permissions:
      contents: read
      # The notification below upserts (or closes) a tracking issue via
      # `gh issue` — gated by the issues permission.
      issues: write
      # Pass-rate history downloads earlier weekly runs' trial-outcomes.
      actions: read
    outputs:
      redispatch: ${{ steps.verdict.outputs.redispatch }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false

      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
        with:
          bun-version: 1.4.2

      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
        with:
          name: paid-plan
          path: /tmp/paid-report

      # One directory per attempt-scoped slice artifact (no merge): shard
      # records never overwrite each other and the first attempt decides.
      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
        with:
          pattern: paid-slice-*
          path: /tmp/paid-report

      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
        with:
          name: gate-census-plan
          path: /tmp/gate-census-report

      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
        with:
          pattern: gate-census-[0-9]*
          path: /tmp/gate-census-report

      - name: Reconcile slices against the manifest (fail-closed)
        id: reconcile
        if: always()
        run: |
          set +e
          EVALS_TIER=periodic bun --no-install run scripts/test-paid-shards.ts --tier periodic --report /tmp/paid-report | tee /tmp/report.txt
          # PIPESTATUS[0], NOT $?: GitHub's default run-step shell is
          # `bash -e {0}` with NO pipefail, so $? after the pipe is tee's
          # exit (always 0) — the fail-closed gate was silently fail-open
          # (caught by the ship review army; the wiring test now pins this).
          echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"

      - name: Reconcile gate census against the manifest (fail-closed)
        id: gate-reconcile
        if: always()
        env:
          GATE_CENSUS: ${{ needs.gate-census.result }}
        run: |
          set +e
          if [ "$GATE_CENSUS" = "skipped" ]; then
            echo "gate census skipped: branch dispatch without include_gate_census (periodic lane only)" | tee /tmp/gate-report.txt
            echo "exit=0" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report /tmp/gate-census-report | tee /tmp/gate-report.txt
          echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"

      - name: Stamp trial history series
        if: always()
        run: |
          for file in /tmp/paid-report/trial-outcomes.jsonl /tmp/gate-census-report/trial-outcomes.jsonl; do
            if [ -f "$file" ]; then bun --no-install run scripts/eval-trial-series.ts "$file"; fi
          done

      - name: Upload trial outcomes for pass-rate history
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
        with:
          name: trial-outcomes-periodic-a${{ github.run_attempt }}
          path: |
            /tmp/paid-report/trial-outcomes.jsonl
          if-no-files-found: ignore
          retention-days: 90

      - name: Upload gate census trial outcomes for pass-rate history
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
        with:
          name: trial-outcomes-gate-census-a${{ github.run_attempt }}
          path: |
            /tmp/gate-census-report/trial-outcomes.jsonl
          if-no-files-found: ignore
          retention-days: 90

      # Exercised-path map (shadow, non-blocking; docs/TESTING_INTERNALS.md#exercised-path-map):
      # union this census's own recorded commands into the map, report MAP_GAP lines and
      # cases that recorded no commands, and print the refresh command. Never fails the run.
      - name: Refresh the exercised-path map from this census
        if: always()
        continue-on-error: true
        run: |
          mkdir -p /tmp/exercised-map
          dirs=()
          for dir in /tmp/gate-census-report /tmp/paid-report; do [ -d "$dir" ] && dirs+=(--from-dir "$dir"); done
          status=0
          bun --no-install run scripts/exercised-map.ts "${dirs[@]}" \
            --run-id "$GITHUB_RUN_ID" --out /tmp/exercised-map/exercised-commands.json --report /tmp/exercised-map/report.md || status=$?
          if [ "$status" -ne 0 ]; then
            echo "exercised-path map refresh failed with exit $status (non-blocking; no map uploaded)" | tee -a "$GITHUB_STEP_SUMMARY"
          fi
          if [ -f /tmp/exercised-map/report.md ]; then cat /tmp/exercised-map/report.md >> "$GITHUB_STEP_SUMMARY"; fi

      - name: Upload the refreshed exercised-path map
        if: always()
        continue-on-error: true
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
        with:
          name: exercised-map-a${{ github.run_attempt }}
          path: /tmp/exercised-map
          if-no-files-found: ignore
          retention-days: 90

      # Weekly pass-rate gate over the last 10 weekly runs (drift, rule cases
      # behaving like behavior, quarantine exit/expiry/cap). Fails closed when
      # history cannot be fetched.
      - name: Pass-rate history gate
        id: pass-rates
        if: always()
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set +e
          bun run eval:pass-rates --gate --runs 10 --branch "$GITHUB_REF_NAME" > /tmp/pass-rates.txt 2>&1
          echo "exit=$?" >> "$GITHUB_OUTPUT"
          cat /tmp/pass-rates.txt

      # UC-E1 (approved): a run whose every red verdict is machine-classified
      # INFRA or INCOMPLETE may be re-dispatched ONCE as a new run.
      - name: Classify the census verdict
        id: verdict
        if: always()
        env:
          REDISPATCH_OF: ${{ inputs.redispatch_of }}
          PERIODIC_EXIT: ${{ steps.reconcile.outputs.exit }}
          GATE_EXIT: ${{ steps.gate-reconcile.outputs.exit }}
        run: |
          eligible() { # $1 exit, $2 report dir
            [ "$1" = "0" ] && return 0
            jq -e '.version == 2 and .verdict.redispatchEligible == true' "$2/collector-outcomes.json" >/dev/null 2>&1
          }
          if [ -z "$REDISPATCH_OF" ] && { [ "$PERIODIC_EXIT" != "0" ] || [ "$GATE_EXIT" != "0" ]; } \
            && eligible "$PERIODIC_EXIT" /tmp/paid-report && eligible "$GATE_EXIT" /tmp/gate-census-report; then
            echo "redispatch=true" >> "$GITHUB_OUTPUT"
          else
            echo "redispatch=false" >> "$GITHUB_OUTPUT"
          fi

      # A red census writes one report on every ref: the step summary and the
      # census-report artifact. Only main (scheduled weekly, or dispatched on
      # main) upserts ONE tracking issue (never a new issue per week) and
      # closes it on the next green run; a branch-dispatched census never
      # comments on or closes main's issue. Every report-derived line goes
      # through the one published-text sanitizer (scripts/lib/published-text.ts).
      - name: Write the census report
        if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          REDISPATCH: ${{ steps.verdict.outputs.redispatch }}
          REDISPATCH_OF: ${{ inputs.redispatch_of }}
        run: |
          set -euo pipefail
          published() { # $1 fence|line: stdin lines through the published-text sanitizer
            bun --no-install -e 'import { publishedFence, sanitizePublishedText } from "./scripts/lib/published-text";
              const lines = (await Bun.stdin.text()).split("\n").filter(Boolean);
              console.log(process.argv[1] === "line" ? sanitizePublishedText(lines.join(" ")) : publishedFence(lines.length ? lines : ["(none)"]).join("\n"));' "$1"
          }
          RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
          GUIDE_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/${GITHUB_SHA}/docs/evals/census-red.md"
          bun run eval:pass-rates --reds --runs 10 --branch "$GITHUB_REF_NAME" > /tmp/pass-rates-reds.txt 2>&1 \
            || echo "history unavailable: eval:pass-rates --reds exited $?" >> /tmp/pass-rates-reds.txt
          bun run eval:pass-rates --headroom --runs 10 --branch "$GITHUB_REF_NAME" > /tmp/pass-rates-headroom.txt 2>&1 \
            || echo "history unavailable: eval:pass-rates --headroom exited $?" >> /tmp/pass-rates-headroom.txt
          {
            echo "Periodic census report: branch ${GITHUB_REF_NAME} @ ${GITHUB_SHA}, event ${GITHUB_EVENT_NAME}, run ${RUN_URL}"
            echo
            echo "A census went red: triage with ${GUIDE_URL}"
            if [ -n "$REDISPATCH_OF" ]; then echo; echo "This run is the one INFRA/INCOMPLETE re-dispatch of run ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${REDISPATCH_OF}; both runs are reported."; fi
            if [ "$REDISPATCH" = "true" ]; then echo; echo "Every red verdict is machine-classified INFRA/INCOMPLETE: re-dispatching once as a new run (EVAL_POLICY.infraRedispatch). This run stays red and reported."; fi
            echo
            echo "- periodic reconciliation exit: ${{ steps.reconcile.outputs.exit }} (slices job: ${{ needs.eval-slices.result }}, Codex slices job: ${{ needs.eval-codex-slices.result }})"
            echo "- gate census reconciliation exit: ${{ steps.gate-reconcile.outputs.exit }} (census job: ${{ needs.gate-census.result }})"
            echo "- pass-rate history gate exit: ${{ steps.pass-rates.outputs.exit }}"
            echo "- $({ grep -E '^all-green probability' /tmp/pass-rates-reds.txt || echo 'all-green probability: unavailable (see the red ledger)'; } | published line)"
            echo
            echo "### Periodic lane"
            cat /tmp/paid-report/report-summary.md 2>/dev/null || echo "(no periodic report summary)"
            echo
            echo "### Gate census"
            cat /tmp/gate-census-report/report-summary.md 2>/dev/null || echo "(no gate census report summary)"
            echo
            echo "### Pass-rate history (ACTION REQUIRED)"
            { awk '/^ACTION REQUIRED/ { on = 1; print; next } on && /^  \[/ { print; next } { on = 0 } /history unavailable/ { print }' /tmp/pass-rates.txt 2>/dev/null \
              | grep . || echo "(no pass-rate alarms)"; } | head -c 6000 | published fence
            echo
            echo "### Red ledger: bun run eval:pass-rates --reds --runs 10 --branch ${GITHUB_REF_NAME}"
            { head -n 1 /tmp/pass-rates-reds.txt; tail -n +2 /tmp/pass-rates-reds.txt | tail -n 60; } | cut -c1-400 | published fence
            echo
            echo "### Session headroom: bun run eval:pass-rates --headroom --runs 10 --branch ${GITHUB_REF_NAME}"
            head -n 40 /tmp/pass-rates-headroom.txt | cut -c1-400 | published fence
            echo
            echo "<details><summary>Full reconciliation output</summary>"
            echo
            { tail -c 6000 /tmp/report.txt 2>/dev/null || echo "(no reconciliation output)"; } | published fence
            echo
            { tail -c 6000 /tmp/gate-report.txt 2>/dev/null || echo "(no gate census reconciliation output)"; } | published fence
            echo "</details>"
            echo
            echo "Policy: EVAL_POLICY and CASE_QUARANTINE in test/helpers/periodic-exclude-data.ts; history: \`bun run eval:pass-rates\`."
          } > /tmp/census-report.md
          cat /tmp/census-report.md >> "$GITHUB_STEP_SUMMARY"

      - name: Upload the census report
        if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
        with:
          name: census-report-a${{ github.run_attempt }}
          path: /tmp/census-report.md
          if-no-files-found: ignore
          retention-days: 90

      - name: Upsert tracking issue on failure
        if: always() && github.ref == 'refs/heads/main' && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -euo pipefail
          TITLE="Weekly periodic evals: red lane needs triage"
          BODY_FILE=/tmp/census-report.md
          [ -s "$BODY_FILE" ] || echo "Periodic census report missing; run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" > "$BODY_FILE"
          EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty')
          if [ -n "$EXISTING" ]; then
            gh issue comment "$EXISTING" --repo "$GITHUB_REPOSITORY" --body-file "$BODY_FILE"
            echo "commented on #$EXISTING"
          else
            gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --body-file "$BODY_FILE"
          fi

      - name: Close the tracking issue on a green run
        if: always() && github.ref == 'refs/heads/main' && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success'
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          REDISPATCH_OF: ${{ inputs.redispatch_of }}
        run: |
          set -euo pipefail
          TITLE="Weekly periodic evals: red lane needs triage"
          EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty')
          if [ -n "$EXISTING" ]; then
            NOTE="Green weekly run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
            if [ -n "$REDISPATCH_OF" ]; then NOTE="${NOTE} (the INFRA re-dispatch of run ${REDISPATCH_OF}, which stays red and reported)"; fi
            gh issue close "$EXISTING" --repo "$GITHUB_REPOSITORY" --comment "$NOTE"
          fi

      - name: Fail the workflow when reconciliation failed
        if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || !contains(fromJSON('["success","skipped"]'), needs.gate-census.result))
        run: exit 1

  # The one INFRA/INCOMPLETE re-dispatch (UC-E1). Its own job so the report
  # job keeps no actions:write; the new run's concurrency group differs, so it
  # never cancels this run.
  redispatch:
    runs-on: ubicloud-standard-2
    needs: report
    if: ${{ !cancelled() && needs.report.outputs.redispatch == 'true' }}
    timeout-minutes: 5
    permissions:
      actions: write
    steps:
      - name: Re-dispatch the weekly census once
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: gh workflow run evals-periodic.yml --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF_NAME" -f redispatch_of="$GITHUB_RUN_ID"