diff --git a/.github/workflows/evals-periodic.yml b/.github/workflows/evals-periodic.yml index 19f3efe51..98c68988e 100644 --- a/.github/workflows/evals-periodic.yml +++ b/.github/workflows/evals-periodic.yml @@ -1,7 +1,18 @@ name: Periodic Evals +# The weekly coverage contract: EVERY periodic-tier paid test runs (EVALS_ALL, +# minus the reasoned excludes in test/helpers/periodic-exclude-data.ts), so +# tests can't rot invisibly — the class where the autoplan-dual-voice E2E was +# silently broken for months until a lucky local diff selected it. Engine: +# scripts/test-paid-shards.ts (the same runner local eval:bg:periodic uses): +# one planner manifest, 6 executor slices, and a FAIL-CLOSED report — a slice +# whose artifact never landed is a failure, not an absence. The gate-census +# job is the weekly EVALS_ALL backstop for the gate tier (PR lanes are +# diff-billed, so without it the full gate census might never execute +# anywhere); the hollow-shard guard (exit 0 + zero executed tests under +# EVALS_ALL fails) makes both lanes census-health checks, not just test runs. on: schedule: - - cron: '0 6 * * 1' # Monday 6 AM UTC + - cron: '0 6 * * 1' # Monday 6 AM UTC (ci-image prebuilds at 4 AM) workflow_dispatch: concurrency: @@ -10,12 +21,11 @@ concurrency: env: IMAGE: ghcr.io/${{ github.repository }}/ci - EVALS_TIER: periodic - EVALS_ALL: 1 # Ignore diff — run all periodic tests jobs: build-image: runs-on: ubicloud-standard-8 + timeout-minutes: 15 permissions: contents: read packages: write @@ -27,6 +37,7 @@ jobs: - id: meta # Keep in sync with evals.yml — key on Dockerfile + lockfile only # (package.json's version field would bust the key on every ship). + # Byte-identity pinned by test/ci-image-tag-binding.test.ts. run: echo "tag=${{ env.IMAGE }}:${{ hashFiles('.github/docker/Dockerfile.ci', 'bun.lock', 'patches/**') }}" >> "$GITHUB_OUTPUT" - uses: docker/login-action@v4 @@ -65,11 +76,10 @@ jobs: ${{ steps.meta.outputs.tag }} ${{ env.IMAGE }}:latest - evals: + plan-slices: runs-on: ubicloud-standard-8 needs: build-image - # Least privilege (mirrors evals.yml): read-only contents, packages:read - # for the container-image pull. + timeout-minutes: 10 permissions: contents: read packages: read @@ -79,42 +89,52 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} options: --user runner - timeout-minutes: 25 - strategy: - fail-fast: false - matrix: - suite: - - name: e2e-plan - file: test/skill-e2e-plan.test.ts - - name: e2e-design - file: test/skill-e2e-design.test.ts - - name: e2e-qa-bugs - file: test/skill-e2e-qa-bugs.test.ts - - name: e2e-qa-workflow - file: test/skill-e2e-qa-workflow.test.ts - - name: e2e-review - file: test/skill-e2e-review.test.ts - - name: e2e-retro - file: test/skill-e2e-retro.test.ts - - name: e2e-preamble-ab - file: test/skill-e2e-preamble-script-ab.test.ts - # e2e-review-attribution, e2e-coverage-audit, and e2e-triage are - # gate-only (every test they hold is gate-tier) — deliberately absent - # here; an all-skip shard would just burn a container boot weekly. - - name: e2e-workflow - file: test/skill-e2e-workflow.test.ts - - name: e2e-routing - file: test/skill-routing-e2e.test.ts - - name: e2e-codex - file: test/codex-e2e.test.ts - - name: e2e-codex-sol-scope - file: test/codex-e2e-sol-scope.test.ts - - name: e2e-gemini - file: test/gemini-e2e.test.ts steps: - uses: actions/checkout@v7 with: - fetch-depth: 0 + persist-credentials: false + + - name: Restore deps + run: | + if [ -d /opt/node_modules_cache ] && diff -q /opt/node_modules_cache/.bun.lock bun.lock >/dev/null 2>&1; then + cp -r /opt/node_modules_cache node_modules + else + bun install + fi + + - name: Emit run manifest (ALL periodic tests minus reasoned excludes) + env: + EVALS_ALL: "1" + run: EVALS_TIER=periodic bun run scripts/test-paid-shards.ts --tier periodic --emit-plan /tmp/paid-plan/manifest.json --slices 6 + + - uses: actions/upload-artifact@v7 + with: + name: paid-plan + path: /tmp/paid-plan/manifest.json + retention-days: 30 + + eval-slices: + runs-on: ubicloud-standard-8 + needs: [build-image, plan-slices] + # ~70 shards / 6 slices / EVALS_JOBS=2, 1800s shard wall — worst case is + # bounded by ceil(12/2) x 30min; typical is far under. + timeout-minutes: 200 + permissions: + contents: read + packages: read + container: + image: ${{ needs.build-image.outputs.image-tag }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --user runner + strategy: + fail-fast: false + matrix: + slice: [1, 2, 3, 4, 5, 6] + steps: + - uses: actions/checkout@v7 + with: persist-credentials: false - name: Fix bun temp @@ -126,10 +146,6 @@ jobs: echo "TMPDIR=/home/runner/.cache" } >> "$GITHUB_ENV" - # Recursive copy (cp -r) instead of symlink: bun build resolves a - # file's realpath when looking for sibling deps. See evals.yml for the - # full explanation. cp -al would be faster but /opt and /workspace - # are on different overlay-fs layers, so cross-device hardlink fails. - name: Restore deps run: | if [ -d /opt/node_modules_cache ] && diff -q /opt/node_modules_cache/.bun.lock bun.lock >/dev/null 2>&1; then @@ -140,19 +156,220 @@ jobs: - run: bun run build - - name: Run ${{ matrix.suite.name }} + # Any slice can host a PTY test — seed + registration run + # unconditionally (idempotent; mirrors evals.yml's sliced lane). + - name: Seed claude interactive config + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + run: | + node -e ' + const fs = require("fs"), os = require("os"), path = require("path"); + const p = path.join(os.homedir(), ".claude.json"); + const seed = fs.existsSync(p) ? JSON.parse(fs.readFileSync(p, "utf8")) : {}; + seed.hasCompletedOnboarding = true; + const key = process.env.ANTHROPIC_API_KEY || ""; + if (key) seed.customApiKeyResponses = { approved: [key.slice(-20)], rejected: [] }; + fs.writeFileSync(p, JSON.stringify(seed, null, 2)); + console.log("seeded", p); + ' + + - name: Register gstack skills for PTY tests + run: | + set -eu + SKILLS_DIR="$HOME/.claude/skills" + REPO="$GITHUB_WORKSPACE" + mkdir -p "$SKILLS_DIR" + ln -snf "$REPO" "$SKILLS_DIR/gstack" + for s in office-hours plan-ceo-review plan-eng-review plan-design-review; do + rm -rf "${SKILLS_DIR:?}/$s" + mkdir -p "$SKILLS_DIR/$s" + cp "$REPO/$s/SKILL.md" "$SKILLS_DIR/$s/SKILL.md" + cp -R "$REPO/$s/sections" "$SKILLS_DIR/$s/sections" + done + PROJ_SKILLS="$REPO/.claude/skills" + mkdir -p "$PROJ_SKILLS" + for s in office-hours plan-ceo-review plan-eng-review plan-design-review; do + rm -rf "${PROJ_SKILLS:?}/$s" + mkdir -p "$PROJ_SKILLS/$s" + cp "$REPO/$s/SKILL.md" "$PROJ_SKILLS/$s/SKILL.md" + cp -R "$REPO/$s/sections" "$PROJ_SKILLS/$s/sections" + done + mkdir -p "$HOME/.gstack" + touch "$HOME/.gstack/.activated" \ + "$HOME/.gstack/.first-loop-tip-shown" \ + "$HOME/.gstack/.telemetry-prompted" \ + "$HOME/.gstack/.proactive-prompted" \ + "$HOME/.gstack/.completeness-intro-seen" \ + "$HOME/.gstack/.plan-tune-nudge-shown" + touch "$SKILLS_DIR/gstack/.feature-prompted-continuous-checkpoint" \ + "$SKILLS_DIR/gstack/.feature-prompted-model-overlay" + + - uses: actions/download-artifact@v8 + with: + name: paid-plan + path: /tmp/paid-plan + + - name: Run slice ${{ matrix.slice }}/6 env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} - EVALS_CONCURRENCY: "40" PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers - run: EVALS=1 bun test --retry 1 --concurrent --max-concurrency 40 ${{ matrix.suite.file }} + EVALS_JOBS: "2" + EVALS_CONCURRENCY: "2" + GSTACK_EVAL_DIR: /tmp/paid-slice-results + run: EVALS_TIER=periodic bun run scripts/test-paid-shards.ts --tier periodic --plan /tmp/paid-plan/manifest.json --slice ${{ matrix.slice }} - - name: Upload eval results + - name: Upload slice results if: always() uses: actions/upload-artifact@v7 with: - name: eval-periodic-${{ matrix.suite.name }} - path: ~/.gstack-dev/evals/*.json + name: paid-slice-${{ matrix.slice }} + path: /tmp/paid-slice-results retention-days: 90 + + - name: Upload shard logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: paid-slice-${{ matrix.slice }}-logs + path: /tmp/gstack-paid-shard-*.log + if-no-files-found: ignore + retention-days: 30 + + # Weekly EVALS_ALL gate-tier census: PR lanes are diff-billed, so without + # this the full gate census might never execute anywhere and the selector's + # blind spots rot invisibly. Census health, not selector correctness — + # selector logic has free synthetic-diff contract tests. + gate-census: + runs-on: ubicloud-standard-8 + needs: build-image + timeout-minutes: 300 + permissions: + contents: read + packages: read + container: + image: ${{ needs.build-image.outputs.image-tag }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --user runner + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Fix bun temp + run: | + mkdir -p /home/runner/.cache/bun + { + echo "BUN_INSTALL_CACHE_DIR=/home/runner/.cache/bun" + echo "BUN_TMPDIR=/home/runner/.cache/bun" + echo "TMPDIR=/home/runner/.cache" + } >> "$GITHUB_ENV" + + - name: Restore deps + run: | + if [ -d /opt/node_modules_cache ] && diff -q /opt/node_modules_cache/.bun.lock bun.lock >/dev/null 2>&1; then + cp -r /opt/node_modules_cache node_modules + else + bun install + fi + + - run: bun run build + + - name: Run full gate census + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers + EVALS_ALL: "1" + EVALS_JOBS: "4" + EVALS_CONCURRENCY: "2" + GSTACK_EVAL_DIR: /tmp/gate-census-results + run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate + + - name: Upload census results + if: always() + uses: actions/upload-artifact@v7 + with: + name: gate-census + path: /tmp/gate-census-results + retention-days: 90 + + report: + runs-on: ubicloud-standard-2 + needs: [plan-slices, eval-slices, gate-census] + # always(): the report must run (and FAIL) when an executor died — a + # missing slice artifact reading as green is the class this lane kills. + if: always() && needs.plan-slices.result == 'success' + timeout-minutes: 10 + permissions: + contents: read + # The failure notification below upserts a tracking issue via + # `gh api /issues` — gated by the issues permission. + issues: write + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.13 + + - run: bun install --frozen-lockfile + + - uses: actions/download-artifact@v8 + with: + name: paid-plan + path: /tmp/paid-report + + - uses: actions/download-artifact@v8 + with: + pattern: paid-slice-[0-9]* + path: /tmp/paid-report + merge-multiple: true + + - name: Reconcile slices against the manifest (fail-closed) + id: reconcile + run: | + set +e + EVALS_TIER=periodic bun run scripts/test-paid-shards.ts --tier periodic --report /tmp/paid-report | tee /tmp/report.txt + echo "exit=$?" >> "$GITHUB_OUTPUT" + + # A red weekly lane nobody must action is waste — upsert ONE tracking + # issue (never a new issue per week) with the reconciliation output, so + # failures have an owner-visible artifact with history in one place. + - name: Upsert tracking issue on failure + if: steps.reconcile.outputs.exit != '0' || needs.gate-census.result == 'failure' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + TITLE="Weekly periodic evals: red lane needs triage" + BODY_FILE=/tmp/issue-body.md + { + echo "Automated weekly report — run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + echo + echo "- periodic reconciliation exit: ${{ steps.reconcile.outputs.exit }}" + echo "- gate census job: ${{ needs.gate-census.result }}" + echo + echo '```' + tail -c 6000 /tmp/report.txt 2>/dev/null || echo "(no reconciliation output)" + echo '```' + echo + echo "Exclusion policy: test/helpers/periodic-exclude-data.ts (every entry needs reason + tracking; removal re-activates the file next week)." + } > "$BODY_FILE" + EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty') + if [ -n "$EXISTING" ]; then + gh issue comment "$EXISTING" --repo "$GITHUB_REPOSITORY" --body-file "$BODY_FILE" + echo "commented on #$EXISTING" + else + gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --body-file "$BODY_FILE" + fi + + - name: Fail the workflow when reconciliation failed + if: steps.reconcile.outputs.exit != '0' + run: exit 1 diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts index 20f3afc9c..b27fa22d8 100644 --- a/scripts/test-paid-shards.ts +++ b/scripts/test-paid-shards.ts @@ -63,6 +63,7 @@ import { strictTestExitCode, } from './test-strict-output'; import { PAID_TEST_GLOBS, isPaidTestFile } from '../test/helpers/paid-test-set'; +import { PERIODIC_CI_EXCLUDE } from '../test/helpers/periodic-exclude-data'; import { getProjectEvalDir } from '../test/helpers/eval-store'; import { preflightAnthropicApi } from '../test/helpers/anthropic-preflight'; import { @@ -75,6 +76,7 @@ import { } from '../test/helpers/touchfiles'; export { PAID_TEST_GLOBS, isPaidTestFile }; +export { PERIODIC_CI_EXCLUDE }; const ROOT = path.resolve(import.meta.dir, '..'); @@ -142,7 +144,17 @@ export interface TierSelection { export function selectPaidTestFiles(files: string[], tier: PaidTier, rootDir = ROOT): TierSelection { const selected: string[] = []; const excluded: Array<{ file: string; reason: string }> = []; + // Periodic-lane exclusions (documented-red / manual-hardware files): a + // known-red weekly shard is triage waste locally AND in CI, so the list + // applies to every periodic run, with the reason surfaced per file. + const ciExcluded = (file: string): { reason: string; tracking: string } | undefined => + tier === 'periodic' ? PERIODIC_CI_EXCLUDE[normalizeRelativePath(file)] : undefined; for (const file of files) { + const exclusion = ciExcluded(file); + if (exclusion) { + excluded.push({ file, reason: `excluded: ${exclusion.reason} [${exclusion.tracking}]` }); + continue; + } const source = fs.readFileSync(path.join(rootDir, file), 'utf8'); const classification = classifyPaidTestFile(source, tier); if (classification.included) selected.push(file); diff --git a/test/helpers/periodic-exclude-data.ts b/test/helpers/periodic-exclude-data.ts new file mode 100644 index 000000000..295c177cf --- /dev/null +++ b/test/helpers/periodic-exclude-data.ts @@ -0,0 +1,34 @@ +/** + * Periodic-lane exclusions — LITERALS ONLY (own file, deliberately NOT in + * touchfiles-data.ts: that file is evaluated standalone by map-diff against + * old git versions, and its contract must not grow unrelated exports). + * + * The weekly periodic CI lane runs EVERY periodic-tier file (EVALS_ALL=1) so + * tests can't rot invisibly — the coverage contract. A file lands here only + * when running it weekly is KNOWN waste (documented-red or requires manual + * hardware), and every entry must carry a tracking pointer with a re-entry + * condition, so an exclusion is a decision with an owner, not a place tests + * go to die. Pinned by test/periodic-exclude-policy.test.ts: entries must + * name real files and carry non-empty reason + tracking. + * + * Removing an entry re-activates the file on the next weekly run — that IS + * the re-entry mechanism. + */ +export const PERIODIC_CI_EXCLUDE: Record = { + 'test/skill-e2e-ship-idempotency.test.ts': { + reason: + 'documented-red: the PTY child sits at the Claude Code welcome screen for the full budget ' + + '(readiness/typing race vs CLI 2.1.x); never green since it was born in v1.63', + tracking: 'TODOS.md "periodic tier — three documented-red tests need structural repair" (1 of 3 resolved: sidebar trio already deleted)', + }, + 'test/skill-e2e-brain-privacy-gate.test.ts': { + reason: + 'documented-red: the artifacts-sync stop-gate preconditions do not survive the hermetic env ' + + 'even with per-test HOME/GSTACK_HOME injection; never green anywhere', + tracking: 'TODOS.md "periodic tier — three documented-red tests need structural repair"', + }, + 'test/skill-e2e-ios.test.ts': { + reason: 'requires a live iOS device/simulator toolchain (xcodebuild, devicectl) — manual hardware, not a CI runner capability', + tracking: 'TODOS.md "skill-e2e-ios CI story" (device/runner decision)', + }, +}; diff --git a/test/periodic-exclude-policy.test.ts b/test/periodic-exclude-policy.test.ts new file mode 100644 index 000000000..64af9596c --- /dev/null +++ b/test/periodic-exclude-policy.test.ts @@ -0,0 +1,45 @@ +/** + * The periodic exclude list is a set of DECISIONS, not a place tests go to + * die: every entry names a real file (a deleted/renamed file must drop its + * entry) and carries a non-empty reason + tracking pointer (the re-entry + * condition lives there). The runner surfaces each exclusion per run, and + * removing an entry re-activates the file on the next weekly lane. + */ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import { PERIODIC_CI_EXCLUDE } from './helpers/periodic-exclude-data'; +import { isPaidTestFile } from './helpers/paid-test-set'; +import { selectPaidTestFiles } from '../scripts/test-paid-shards'; + +const ROOT = path.resolve(__dirname, '..'); + +describe('periodic exclude policy', () => { + test('every entry names a real paid file and carries reason + tracking', () => { + const entries = Object.entries(PERIODIC_CI_EXCLUDE); + expect(entries.length).toBeGreaterThan(0); + for (const [file, meta] of entries) { + expect(fs.existsSync(path.join(ROOT, file)), `stale exclude entry: ${file}`).toBe(true); + expect(isPaidTestFile(file), `${file} is not a paid file — exclusion is meaningless`).toBe(true); + expect(meta.reason.length, `${file}: empty reason`).toBeGreaterThan(20); + expect(meta.tracking.length, `${file}: empty tracking pointer`).toBeGreaterThan(5); + } + }); + + test('exclusions apply to the periodic tier only, with the reason surfaced', () => { + const files = Object.keys(PERIODIC_CI_EXCLUDE); + const periodic = selectPaidTestFiles(files, 'periodic'); + expect(periodic.selected).toEqual([]); + for (const { reason } of periodic.excluded) { + expect(reason).toStartWith('excluded: '); + expect(reason).toContain('['); + } + // Gate tier ignores the list (these files are periodic-tier anyway; the + // list must never leak into gate semantics). + const gate = selectPaidTestFiles(files, 'gate'); + for (const { reason } of gate.excluded) { + expect(reason).not.toStartWith('excluded: '); + } + }); +});