name: Periodic Evals # The weekly coverage contract: EVERY periodic-tier paid test runs (EVALS_ALL, # minus the reasoned excludes in test/helpers/periodic-exclude-data.ts), so # tests can't rot invisibly — the class where the autoplan-dual-voice E2E was # silently broken for months until a lucky local diff selected it. Engine: # scripts/test-paid-shards.ts (the same runner local eval:bg:periodic uses): # one planner manifest packed by recorded durations into as many ~9-minute # executors as the work needs (one file, or a tightly packed group, per # runner; overlays share one final slice), and a FAIL-CLOSED report — a slice # whose artifact never landed is a failure, not an absence. The matrix size # and job timeout come from the plan, so they cannot drift from the census. # Full end-to-end flows run in the non-blocking marathon lane # (evals-marathon.yml), never here. The gate-census # job is the weekly EVALS_ALL backstop for the gate tier (PR lanes are # diff-billed, so without it the full gate census might never execute # anywhere); the hollow-shard guard (exit 0 + zero executed tests under # EVALS_ALL fails) makes both lanes census-health checks, not just test runs. on: schedule: - cron: '0 6 * * 1' # Monday 6 AM UTC (ci-image prebuilds at 4 AM) workflow_dispatch: inputs: redispatch_of: description: 'Run id this run re-dispatches (the one INFRA/INCOMPLETE-only re-dispatch; set by the report job)' type: string default: '' # A re-dispatch runs in its own group so it never cancels the run that # dispatched it; both runs are reported. concurrency: group: evals-periodic${{ inputs.redispatch_of && format('-redispatch-{0}', inputs.redispatch_of) || '' }} cancel-in-progress: true env: IMAGE: ghcr.io/${{ github.repository }}/ci EVALS_PROFILE: full EVALS_FRESH: "1" EVALS_CACHE_PURPOSE: periodic jobs: build-image: runs-on: ubicloud-standard-8 timeout-minutes: 15 permissions: contents: read packages: write outputs: image-tag: ${{ steps.meta.outputs.tag }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - id: meta # Keep in sync with evals.yml — key on Dockerfile + lockfile only # (package.json's version field would bust the key on every ship). # Byte-identity pinned by test/ci-image-tag-binding.test.ts. run: echo "tag=${{ env.IMAGE }}:${{ hashFiles('.github/docker/Dockerfile.ci', 'bun.lock', 'patches/**') }}" >> "$GITHUB_OUTPUT" - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Check if image exists id: check run: | if docker manifest inspect ${{ steps.meta.outputs.tag }} > /dev/null 2>&1; then echo "exists=true" >> "$GITHUB_OUTPUT" else echo "exists=false" >> "$GITHUB_OUTPUT" fi - if: steps.check.outputs.exists == 'false' run: cp package.json bun.lock .github/docker/ && cp -R patches .github/docker/patches # Registry cache export needs a docker-container builder — the default # `docker` driver hard-errors on cache-to. - if: steps.check.outputs.exists == 'false' uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 - if: steps.check.outputs.exists == 'false' uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 with: context: .github/docker file: .github/docker/Dockerfile.ci push: true # Cron-triggered in the base repo only, so cache export is always safe here. cache-from: type=registry,ref=${{ env.IMAGE }}:buildcache cache-to: type=registry,ref=${{ env.IMAGE }}:buildcache,mode=max tags: | ${{ steps.meta.outputs.tag }} ${{ env.IMAGE }}:latest plan-slices: runs-on: ubicloud-standard-8 timeout-minutes: 10 permissions: contents: read outputs: periodic_slices: ${{ steps.periodic-matrix.outputs.slices }} periodic_timeout_minutes: ${{ steps.periodic-matrix.outputs.timeout_minutes }} gate_slices: ${{ steps.gate-matrix.outputs.slices }} gate_timeout_minutes: ${{ steps.gate-matrix.outputs.timeout_minutes }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.4.0 - name: Emit run manifest (ALL periodic tests minus reasoned excludes) env: EVALS_ALL: "1" run: EVALS_TIER=periodic bun --no-install run scripts/test-paid-shards.ts --tier periodic --emit-plan /tmp/paid-plan/manifest.json --slice-budget 540 --jobs 2 --max-parallel 24 - name: Derive the periodic executor matrix from the plan id: periodic-matrix run: | echo "slices=$(jq -c '[range(1; .sliceCount + 1)]' /tmp/paid-plan/manifest.json)" >> "$GITHUB_OUTPUT" echo "timeout_minutes=$(jq -e '.plan.ciTimeoutMinutes' /tmp/paid-plan/manifest.json)" >> "$GITHUB_OUTPUT" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: paid-plan path: /tmp/paid-plan/manifest.json retention-days: 30 - name: Emit gate census manifest (ALL gate tests) env: EVALS_ALL: "1" run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/gate-census-plan/manifest.json --slice-budget 540 --jobs 2 --skip-judges --max-parallel 16 - name: Derive the gate census executor matrix from the plan id: gate-matrix run: | echo "slices=$(jq -c '[range(1; .sliceCount + 1)]' /tmp/gate-census-plan/manifest.json)" >> "$GITHUB_OUTPUT" echo "timeout_minutes=$(jq -e '.plan.ciTimeoutMinutes' /tmp/gate-census-plan/manifest.json)" >> "$GITHUB_OUTPUT" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: gate-census-plan path: /tmp/gate-census-plan/manifest.json retention-days: 30 eval-slices: runs-on: ubicloud-standard-8 needs: [build-image, plan-slices] env: EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-slices-${{ matrix.slice }} # The planner packs ~9 minutes of recorded work per slice; the job timeout # is its supervised worst case (every shard at its wall) plus 20 minutes # setup/upload, computed from the same manifest the slices execute. timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.periodic_timeout_minutes) }} permissions: contents: read packages: read container: image: ${{ needs.build-image.outputs.image-tag }} credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} options: --user runner strategy: fail-fast: false # Every planned slice starts at once; test/evals-workflow-wiring.test.ts # fails when the live plan outgrows this cap. max-parallel: 24 matrix: slice: ${{ fromJSON(needs.plan-slices.outputs.periodic_slices) }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: # Full history: files with SELF-derived selection (the LLM-judge # map, routing) walk git at module load, and selection is # fail-closed on git errors — a shallow checkout crashed those # shards on the lane's first live run ("ambiguous argument # 'main...HEAD'"). The manifest still governs WHICH shards run. fetch-depth: 0 persist-credentials: false - name: Fix bun temp uses: ./.github/actions/fix-bun-temp - name: Restore deps uses: ./.github/actions/restore-deps - run: bun run build # Any slice can host a PTY test — seed + registration run # unconditionally (idempotent; mirrors evals.yml's sliced lane). The # register composite carries the fail-fast dangling-symlink/frontmatter # verification loop — this lane previously LACKED it, so a moved skill # target surfaced as a silent "Unknown command" + wedged PTY session. - name: Seed claude interactive config uses: ./.github/actions/seed-claude-config with: anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} - name: Register gstack skills for PTY tests uses: ./.github/actions/register-gstack-skills - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: paid-plan path: /tmp/paid-plan - name: Run periodic slice ${{ matrix.slice }} env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers EVALS_JOBS: "2" EVALS_CONCURRENCY: "2" GSTACK_EVAL_DIR: /tmp/paid-slice-results run: EVALS_TIER=periodic bun run scripts/test-paid-shards.ts --tier periodic --plan /tmp/paid-plan/manifest.json --slice ${{ matrix.slice }} - name: Upload slice results if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: paid-slice-${{ matrix.slice }}-a${{ github.run_attempt }} path: /tmp/paid-slice-results retention-days: 90 - name: Upload native capture evidence if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: native-captures-${{ env.EVALS_RUN_ID }} include-hidden-files: true path: | ~/.gstack/projects/*/e2e-runs ~/.gstack/projects/*/evals/qa-callers ~/.gstack-dev/e2e-runs ~/.gstack-dev/evals/qa-callers if-no-files-found: ignore retention-days: 90 # always(), not failure(): a failed behavior trial is a verdict and no # longer reds its runner, but its full log is the diagnosis evidence. - name: Upload shard logs if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: paid-logs-slice-${{ matrix.slice }}-a${{ github.run_attempt }} include-hidden-files: true # The Fix-bun-temp step points TMPDIR at /home/runner/.cache, so the # runner's spool lands THERE, not /tmp — the original /tmp glob # uploaded nothing and a red slice's diagnostics were unreachable. path: | /home/runner/.cache/gstack-paid-shard-*.log /tmp/gstack-paid-shard-*.log if-no-files-found: ignore retention-days: 30 # Weekly EVALS_ALL gate-tier census: PR lanes are diff-billed, so without # this the full gate census might never execute anywhere and the selector's # blind spots rot invisibly. Census health, not selector correctness — # selector logic has free synthetic-diff contract tests. gate-census: runs-on: ubicloud-standard-8 needs: [build-image, plan-slices] env: EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-gate-census-${{ matrix.slice }} # Supervised worst case of the packed plan plus 20 minutes setup/upload. timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.gate_timeout_minutes) }} permissions: contents: read packages: read container: image: ${{ needs.build-image.outputs.image-tag }} credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} options: --user runner strategy: # Two file workers per slice, each retaining two in-file case workers. fail-fast: false max-parallel: 16 matrix: slice: ${{ fromJSON(needs.plan-slices.outputs.gate_slices) }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: # Full history: files with SELF-derived selection (the LLM-judge # map, routing) walk git at module load, and selection is # fail-closed on git errors — a shallow checkout crashed those # shards on the lane's first live run ("ambiguous argument # 'main...HEAD'"). The manifest still governs WHICH shards run. fetch-depth: 0 persist-credentials: false - name: Fix bun temp uses: ./.github/actions/fix-bun-temp - name: Restore deps uses: ./.github/actions/restore-deps - run: bun run build - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: gate-census-plan path: /tmp/gate-census-plan - name: Run gate census slice ${{ matrix.slice }} env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers EVALS_JOBS: "2" EVALS_CONCURRENCY: "2" GSTACK_EVAL_DIR: /tmp/gate-census-results run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/gate-census-plan/manifest.json --slice ${{ matrix.slice }} - name: Upload census results if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: gate-census-${{ matrix.slice }}-a${{ github.run_attempt }} path: /tmp/gate-census-results retention-days: 90 - name: Upload native capture evidence if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: native-captures-${{ env.EVALS_RUN_ID }} include-hidden-files: true path: | ~/.gstack/projects/*/e2e-runs ~/.gstack/projects/*/evals/qa-callers ~/.gstack-dev/e2e-runs ~/.gstack-dev/evals/qa-callers if-no-files-found: ignore retention-days: 90 report: runs-on: ubicloud-standard-2 needs: [plan-slices, eval-slices, gate-census] # !cancelled(): the report must run (and FAIL) when an executor died — a # missing slice artifact reading as green is the class this lane kills — # but a cancelled run stops here. if: ${{ !cancelled() && needs.plan-slices.result == 'success' }} timeout-minutes: 15 permissions: contents: read # The notification below upserts (or closes) a tracking issue via # `gh issue` — gated by the issues permission. issues: write # Pass-rate history downloads earlier weekly runs' trial-outcomes. actions: read outputs: redispatch: ${{ steps.verdict.outputs.redispatch }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.4.0 - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: paid-plan path: /tmp/paid-report # One directory per attempt-scoped slice artifact (no merge): shard # records never overwrite each other and the first attempt decides. - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: pattern: paid-slice-* path: /tmp/paid-report - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: gate-census-plan path: /tmp/gate-census-report - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: pattern: gate-census-[0-9]* path: /tmp/gate-census-report - name: Reconcile slices against the manifest (fail-closed) id: reconcile if: always() run: | set +e EVALS_TIER=periodic bun --no-install run scripts/test-paid-shards.ts --tier periodic --report /tmp/paid-report | tee /tmp/report.txt # PIPESTATUS[0], NOT $?: GitHub's default run-step shell is # `bash -e {0}` with NO pipefail, so $? after the pipe is tee's # exit (always 0) — the fail-closed gate was silently fail-open # (caught by the ship review army; the wiring test now pins this). echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" - name: Reconcile gate census against the manifest (fail-closed) id: gate-reconcile if: always() run: | set +e EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report /tmp/gate-census-report | tee /tmp/gate-report.txt echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" - name: Stamp trial history series if: always() run: | for file in /tmp/paid-report/trial-outcomes.jsonl /tmp/gate-census-report/trial-outcomes.jsonl; do if [ -f "$file" ]; then bun --no-install run scripts/eval-trial-series.ts "$file"; fi done - name: Upload trial outcomes for pass-rate history if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: trial-outcomes-periodic-a${{ github.run_attempt }} path: | /tmp/paid-report/trial-outcomes.jsonl if-no-files-found: ignore retention-days: 90 - name: Upload gate census trial outcomes for pass-rate history if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: trial-outcomes-gate-census-a${{ github.run_attempt }} path: | /tmp/gate-census-report/trial-outcomes.jsonl if-no-files-found: ignore retention-days: 90 # Weekly pass-rate gate over the last 10 weekly runs (drift, rule cases # behaving like behavior, quarantine exit/expiry/cap). Fails closed when # history cannot be fetched. - name: Pass-rate history gate id: pass-rates if: always() env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set +e bun run eval:pass-rates --gate --runs 10 > /tmp/pass-rates.txt 2>&1 echo "exit=$?" >> "$GITHUB_OUTPUT" cat /tmp/pass-rates.txt # UC-E1 (approved): a run whose every red verdict is machine-classified # INFRA or INCOMPLETE may be re-dispatched ONCE as a new run. - name: Classify the census verdict id: verdict if: always() env: REDISPATCH_OF: ${{ inputs.redispatch_of }} PERIODIC_EXIT: ${{ steps.reconcile.outputs.exit }} GATE_EXIT: ${{ steps.gate-reconcile.outputs.exit }} run: | eligible() { # $1 exit, $2 report dir [ "$1" = "0" ] && return 0 jq -e '.version == 2 and .verdict.redispatchEligible == true' "$2/collector-outcomes.json" >/dev/null 2>&1 } if [ -z "$REDISPATCH_OF" ] && { [ "$PERIODIC_EXIT" != "0" ] || [ "$GATE_EXIT" != "0" ]; } \ && eligible "$PERIODIC_EXIT" /tmp/paid-report && eligible "$GATE_EXIT" /tmp/gate-census-report; then echo "redispatch=true" >> "$GITHUB_OUTPUT" else echo "redispatch=false" >> "$GITHUB_OUTPUT" fi # A red weekly lane nobody must action is waste — upsert ONE tracking # issue (never a new issue per week) with the headline and failure block # of both lanes, and close it on the next green run. - name: Upsert tracking issue on failure if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || needs.gate-census.result != 'success') env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REDISPATCH: ${{ steps.verdict.outputs.redispatch }} REDISPATCH_OF: ${{ inputs.redispatch_of }} run: | set -euo pipefail TITLE="Weekly periodic evals: red lane needs triage" BODY_FILE=/tmp/issue-body.md RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" { echo "Automated weekly report — run: ${RUN_URL}" if [ -n "$REDISPATCH_OF" ]; then echo; echo "This run is the one INFRA/INCOMPLETE re-dispatch of run ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${REDISPATCH_OF}; both runs are reported."; fi if [ "$REDISPATCH" = "true" ]; then echo; echo "Every red verdict is machine-classified INFRA/INCOMPLETE: re-dispatching once as a new run (EVAL_POLICY.infraRedispatch). This run stays red and reported."; fi echo echo "- periodic reconciliation exit: ${{ steps.reconcile.outputs.exit }} (slices job: ${{ needs.eval-slices.result }})" echo "- gate census reconciliation exit: ${{ steps.gate-reconcile.outputs.exit }} (census job: ${{ needs.gate-census.result }})" echo "- pass-rate history gate exit: ${{ steps.pass-rates.outputs.exit }}" echo echo "### Periodic lane" cat /tmp/paid-report/report-summary.md 2>/dev/null || echo "(no periodic report summary)" echo echo "### Gate census" cat /tmp/gate-census-report/report-summary.md 2>/dev/null || echo "(no gate census report summary)" echo echo "### Pass-rate history (ACTION REQUIRED)" echo '```' { grep -E 'ACTION REQUIRED|history unavailable' /tmp/pass-rates.txt || echo "(no pass-rate alarms)"; } | sed 's/@/@\xe2\x80\x8b/g' | head -c 6000 echo '```' echo echo "
Full reconciliation output" echo echo '```' tail -c 6000 /tmp/report.txt 2>/dev/null | sed 's/@/@\xe2\x80\x8b/g' || echo "(no reconciliation output)" echo '```' echo echo '```' tail -c 6000 /tmp/gate-report.txt 2>/dev/null | sed 's/@/@\xe2\x80\x8b/g' || echo "(no gate census reconciliation output)" echo '```' echo "
" echo echo "Policy: EVAL_POLICY and CASE_QUARANTINE in test/helpers/periodic-exclude-data.ts; history: \`bun run eval:pass-rates\`." } > "$BODY_FILE" EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty') if [ -n "$EXISTING" ]; then gh issue comment "$EXISTING" --repo "$GITHUB_REPOSITORY" --body-file "$BODY_FILE" echo "commented on #$EXISTING" else gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --body-file "$BODY_FILE" fi - name: Close the tracking issue on a green run if: always() && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.eval-slices.result == 'success' && needs.gate-census.result == 'success' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REDISPATCH_OF: ${{ inputs.redispatch_of }} run: | set -euo pipefail TITLE="Weekly periodic evals: red lane needs triage" EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty') if [ -n "$EXISTING" ]; then NOTE="Green weekly run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" if [ -n "$REDISPATCH_OF" ]; then NOTE="${NOTE} (the INFRA re-dispatch of run ${REDISPATCH_OF}, which stays red and reported)"; fi gh issue close "$EXISTING" --repo "$GITHUB_REPOSITORY" --comment "$NOTE" fi - name: Fail the workflow when reconciliation failed if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || needs.gate-census.result != 'success') run: exit 1 # The one INFRA/INCOMPLETE re-dispatch (UC-E1). Its own job so the report # job keeps no actions:write; the new run's concurrency group differs, so it # never cancels this run. redispatch: runs-on: ubicloud-standard-2 needs: report if: ${{ !cancelled() && needs.report.outputs.redispatch == 'true' }} timeout-minutes: 5 permissions: actions: write steps: - name: Re-dispatch the weekly census once env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: gh workflow run evals-periodic.yml --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF_NAME" -f redispatch_of="$GITHUB_RUN_ID"