feat(ci): weekly periodic lane runs EVERY periodic test + gate census backstop

evals-periodic.yml re-platforms onto the sharded runner: planner
manifest → 6 executor slices → FAIL-CLOSED report. This IS the coverage
contract: all ~70 periodic-tier files weekly (EVALS_ALL=1), killing the
silent-rot class where a hard-coded 9-file matrix left ~57 files
running NOWHERE (the autoplan E2E rotted invisibly for months).

- test/helpers/periodic-exclude-data.ts: reasoned exclusions in their
  OWN literals file (deliberately not touchfiles-data — map-diff
  evaluates old versions of that file standalone). Every entry carries
  reason + tracking with a re-entry condition; the runner surfaces each
  exclusion per run; policy test pins real-file + non-empty fields.
  Initial: ship-idempotency + brain-privacy-gate (documented-red,
  never green) and skill-e2e-ios (manual hardware). The TODOS 'sidebar
  E2E trio' turned out already deleted — only tombstone tests remain.
- gate-census job: weekly EVALS_ALL gate-tier run — PR lanes are
  diff-billed, so without this the full gate census might never execute
  anywhere; with the hollow-shard guard it is a census-health check
  (exit 0 + zero executed tests fails), not just a test run.
- failure notification is a concrete gh issue UPSERT (one tracking
  issue, commented per red week — never issue-per-week spam), with
  issues:write scoped to the report job.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

.github/workflows/evals-periodic.yml

@@ -1,7 +1,18 @@
 name: Periodic Evals
+# The weekly coverage contract: EVERY periodic-tier paid test runs (EVALS_ALL,
+# minus the reasoned excludes in test/helpers/periodic-exclude-data.ts), so
+# tests can't rot invisibly — the class where the autoplan-dual-voice E2E was
+# silently broken for months until a lucky local diff selected it. Engine:
+# scripts/test-paid-shards.ts (the same runner local eval:bg:periodic uses):
+# one planner manifest, 6 executor slices, and a FAIL-CLOSED report — a slice
+# whose artifact never landed is a failure, not an absence. The gate-census
+# job is the weekly EVALS_ALL backstop for the gate tier (PR lanes are
+# diff-billed, so without it the full gate census might never execute
+# anywhere); the hollow-shard guard (exit 0 + zero executed tests under
+# EVALS_ALL fails) makes both lanes census-health checks, not just test runs.
 on:
   schedule:
-    - cron: '0 6 * * 1'  # Monday 6 AM UTC
+    - cron: '0 6 * * 1'  # Monday 6 AM UTC (ci-image prebuilds at 4 AM)
   workflow_dispatch:
 
 concurrency:
@@ -10,12 +21,11 @@ concurrency:
 
 env:
   IMAGE: ghcr.io/${{ github.repository }}/ci
-  EVALS_TIER: periodic
-  EVALS_ALL: 1  # Ignore diff — run all periodic tests
 
 jobs:
   build-image:
     runs-on: ubicloud-standard-8
+    timeout-minutes: 15
     permissions:
       contents: read
       packages: write
@@ -27,6 +37,7 @@ jobs:
       - id: meta
         # Keep in sync with evals.yml — key on Dockerfile + lockfile only
         # (package.json's version field would bust the key on every ship).
+        # Byte-identity pinned by test/ci-image-tag-binding.test.ts.
         run: echo "tag=${{ env.IMAGE }}:${{ hashFiles('.github/docker/Dockerfile.ci', 'bun.lock', 'patches/**') }}" >> "$GITHUB_OUTPUT"
 
       - uses: docker/login-action@v4
@@ -65,11 +76,10 @@ jobs:
             ${{ steps.meta.outputs.tag }}
             ${{ env.IMAGE }}:latest
 
-  evals:
+  plan-slices:
     runs-on: ubicloud-standard-8
     needs: build-image
-    # Least privilege (mirrors evals.yml): read-only contents, packages:read
-    # for the container-image pull.
+    timeout-minutes: 10
     permissions:
       contents: read
       packages: read
@@ -79,42 +89,52 @@ jobs:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
       options: --user runner
-    timeout-minutes: 25
-    strategy:
-      fail-fast: false
-      matrix:
-        suite:
-          - name: e2e-plan
-            file: test/skill-e2e-plan.test.ts
-          - name: e2e-design
-            file: test/skill-e2e-design.test.ts
-          - name: e2e-qa-bugs
-            file: test/skill-e2e-qa-bugs.test.ts
-          - name: e2e-qa-workflow
-            file: test/skill-e2e-qa-workflow.test.ts
-          - name: e2e-review
-            file: test/skill-e2e-review.test.ts
-          - name: e2e-retro
-            file: test/skill-e2e-retro.test.ts
-          - name: e2e-preamble-ab
-            file: test/skill-e2e-preamble-script-ab.test.ts
-          # e2e-review-attribution, e2e-coverage-audit, and e2e-triage are
-          # gate-only (every test they hold is gate-tier) — deliberately absent
-          # here; an all-skip shard would just burn a container boot weekly.
-          - name: e2e-workflow
-            file: test/skill-e2e-workflow.test.ts
-          - name: e2e-routing
-            file: test/skill-routing-e2e.test.ts
-          - name: e2e-codex
-            file: test/codex-e2e.test.ts
-          - name: e2e-codex-sol-scope
-            file: test/codex-e2e-sol-scope.test.ts
-          - name: e2e-gemini
-            file: test/gemini-e2e.test.ts
     steps:
       - uses: actions/checkout@v7
         with:
-          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Restore deps
+        run: |
+          if [ -d /opt/node_modules_cache ] && diff -q /opt/node_modules_cache/.bun.lock bun.lock >/dev/null 2>&1; then
+            cp -r /opt/node_modules_cache node_modules
+          else
+            bun install
+          fi
+
+      - name: Emit run manifest (ALL periodic tests minus reasoned excludes)
+        env:
+          EVALS_ALL: "1"
+        run: EVALS_TIER=periodic bun run scripts/test-paid-shards.ts --tier periodic --emit-plan /tmp/paid-plan/manifest.json --slices 6
+
+      - uses: actions/upload-artifact@v7
+        with:
+          name: paid-plan
+          path: /tmp/paid-plan/manifest.json
+          retention-days: 30
+
+  eval-slices:
+    runs-on: ubicloud-standard-8
+    needs: [build-image, plan-slices]
+    # ~70 shards / 6 slices / EVALS_JOBS=2, 1800s shard wall — worst case is
+    # bounded by ceil(12/2) x 30min; typical is far under.
+    timeout-minutes: 200
+    permissions:
+      contents: read
+      packages: read
+    container:
+      image: ${{ needs.build-image.outputs.image-tag }}
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+      options: --user runner
+    strategy:
+      fail-fast: false
+      matrix:
+        slice: [1, 2, 3, 4, 5, 6]
+    steps:
+      - uses: actions/checkout@v7
+        with:
           persist-credentials: false
 
       - name: Fix bun temp
@@ -126,10 +146,6 @@ jobs:
             echo "TMPDIR=/home/runner/.cache"
           } >> "$GITHUB_ENV"
 
-      # Recursive copy (cp -r) instead of symlink: bun build resolves a
-      # file's realpath when looking for sibling deps. See evals.yml for the
-      # full explanation. cp -al would be faster but /opt and /workspace
-      # are on different overlay-fs layers, so cross-device hardlink fails.
       - name: Restore deps
         run: |
           if [ -d /opt/node_modules_cache ] && diff -q /opt/node_modules_cache/.bun.lock bun.lock >/dev/null 2>&1; then
@@ -140,19 +156,220 @@ jobs:
 
       - run: bun run build
 
-      - name: Run ${{ matrix.suite.name }}
+      # Any slice can host a PTY test — seed + registration run
+      # unconditionally (idempotent; mirrors evals.yml's sliced lane).
+      - name: Seed claude interactive config
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+        run: |
+          node -e '
+            const fs = require("fs"), os = require("os"), path = require("path");
+            const p = path.join(os.homedir(), ".claude.json");
+            const seed = fs.existsSync(p) ? JSON.parse(fs.readFileSync(p, "utf8")) : {};
+            seed.hasCompletedOnboarding = true;
+            const key = process.env.ANTHROPIC_API_KEY || "";
+            if (key) seed.customApiKeyResponses = { approved: [key.slice(-20)], rejected: [] };
+            fs.writeFileSync(p, JSON.stringify(seed, null, 2));
+            console.log("seeded", p);
+          '
+
+      - name: Register gstack skills for PTY tests
+        run: |
+          set -eu
+          SKILLS_DIR="$HOME/.claude/skills"
+          REPO="$GITHUB_WORKSPACE"
+          mkdir -p "$SKILLS_DIR"
+          ln -snf "$REPO" "$SKILLS_DIR/gstack"
+          for s in office-hours plan-ceo-review plan-eng-review plan-design-review; do
+            rm -rf "${SKILLS_DIR:?}/$s"
+            mkdir -p "$SKILLS_DIR/$s"
+            cp "$REPO/$s/SKILL.md" "$SKILLS_DIR/$s/SKILL.md"
+            cp -R "$REPO/$s/sections" "$SKILLS_DIR/$s/sections"
+          done
+          PROJ_SKILLS="$REPO/.claude/skills"
+          mkdir -p "$PROJ_SKILLS"
+          for s in office-hours plan-ceo-review plan-eng-review plan-design-review; do
+            rm -rf "${PROJ_SKILLS:?}/$s"
+            mkdir -p "$PROJ_SKILLS/$s"
+            cp "$REPO/$s/SKILL.md" "$PROJ_SKILLS/$s/SKILL.md"
+            cp -R "$REPO/$s/sections" "$PROJ_SKILLS/$s/sections"
+          done
+          mkdir -p "$HOME/.gstack"
+          touch "$HOME/.gstack/.activated" \
+                "$HOME/.gstack/.first-loop-tip-shown" \
+                "$HOME/.gstack/.telemetry-prompted" \
+                "$HOME/.gstack/.proactive-prompted" \
+                "$HOME/.gstack/.completeness-intro-seen" \
+                "$HOME/.gstack/.plan-tune-nudge-shown"
+          touch "$SKILLS_DIR/gstack/.feature-prompted-continuous-checkpoint" \
+                "$SKILLS_DIR/gstack/.feature-prompted-model-overlay"
+
+      - uses: actions/download-artifact@v8
+        with:
+          name: paid-plan
+          path: /tmp/paid-plan
+
+      - name: Run slice ${{ matrix.slice }}/6
         env:
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
-          EVALS_CONCURRENCY: "40"
           PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers
-        run: EVALS=1 bun test --retry 1 --concurrent --max-concurrency 40 ${{ matrix.suite.file }}
+          EVALS_JOBS: "2"
+          EVALS_CONCURRENCY: "2"
+          GSTACK_EVAL_DIR: /tmp/paid-slice-results
+        run: EVALS_TIER=periodic bun run scripts/test-paid-shards.ts --tier periodic --plan /tmp/paid-plan/manifest.json --slice ${{ matrix.slice }}
 
-      - name: Upload eval results
+      - name: Upload slice results
         if: always()
         uses: actions/upload-artifact@v7
         with:
-          name: eval-periodic-${{ matrix.suite.name }}
-          path: ~/.gstack-dev/evals/*.json
+          name: paid-slice-${{ matrix.slice }}
+          path: /tmp/paid-slice-results
           retention-days: 90
+
+      - name: Upload shard logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: paid-slice-${{ matrix.slice }}-logs
+          path: /tmp/gstack-paid-shard-*.log
+          if-no-files-found: ignore
+          retention-days: 30
+
+  # Weekly EVALS_ALL gate-tier census: PR lanes are diff-billed, so without
+  # this the full gate census might never execute anywhere and the selector's
+  # blind spots rot invisibly. Census health, not selector correctness —
+  # selector logic has free synthetic-diff contract tests.
+  gate-census:
+    runs-on: ubicloud-standard-8
+    needs: build-image
+    timeout-minutes: 300
+    permissions:
+      contents: read
+      packages: read
+    container:
+      image: ${{ needs.build-image.outputs.image-tag }}
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+      options: --user runner
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+
+      - name: Fix bun temp
+        run: |
+          mkdir -p /home/runner/.cache/bun
+          {
+            echo "BUN_INSTALL_CACHE_DIR=/home/runner/.cache/bun"
+            echo "BUN_TMPDIR=/home/runner/.cache/bun"
+            echo "TMPDIR=/home/runner/.cache"
+          } >> "$GITHUB_ENV"
+
+      - name: Restore deps
+        run: |
+          if [ -d /opt/node_modules_cache ] && diff -q /opt/node_modules_cache/.bun.lock bun.lock >/dev/null 2>&1; then
+            cp -r /opt/node_modules_cache node_modules
+          else
+            bun install
+          fi
+
+      - run: bun run build
+
+      - name: Run full gate census
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+          PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers
+          EVALS_ALL: "1"
+          EVALS_JOBS: "4"
+          EVALS_CONCURRENCY: "2"
+          GSTACK_EVAL_DIR: /tmp/gate-census-results
+        run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate
+
+      - name: Upload census results
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: gate-census
+          path: /tmp/gate-census-results
+          retention-days: 90
+
+  report:
+    runs-on: ubicloud-standard-2
+    needs: [plan-slices, eval-slices, gate-census]
+    # always(): the report must run (and FAIL) when an executor died — a
+    # missing slice artifact reading as green is the class this lane kills.
+    if: always() && needs.plan-slices.result == 'success'
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      # The failure notification below upserts a tracking issue via
+      # `gh api /issues` — gated by the issues permission.
+      issues: write
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: 1.3.13
+
+      - run: bun install --frozen-lockfile
+
+      - uses: actions/download-artifact@v8
+        with:
+          name: paid-plan
+          path: /tmp/paid-report
+
+      - uses: actions/download-artifact@v8
+        with:
+          pattern: paid-slice-[0-9]*
+          path: /tmp/paid-report
+          merge-multiple: true
+
+      - name: Reconcile slices against the manifest (fail-closed)
+        id: reconcile
+        run: |
+          set +e
+          EVALS_TIER=periodic bun run scripts/test-paid-shards.ts --tier periodic --report /tmp/paid-report | tee /tmp/report.txt
+          echo "exit=$?" >> "$GITHUB_OUTPUT"
+
+      # A red weekly lane nobody must action is waste — upsert ONE tracking
+      # issue (never a new issue per week) with the reconciliation output, so
+      # failures have an owner-visible artifact with history in one place.
+      - name: Upsert tracking issue on failure
+        if: steps.reconcile.outputs.exit != '0' || needs.gate-census.result == 'failure'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -euo pipefail
+          TITLE="Weekly periodic evals: red lane needs triage"
+          BODY_FILE=/tmp/issue-body.md
+          {
+            echo "Automated weekly report — run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+            echo
+            echo "- periodic reconciliation exit: ${{ steps.reconcile.outputs.exit }}"
+            echo "- gate census job: ${{ needs.gate-census.result }}"
+            echo
+            echo '```'
+            tail -c 6000 /tmp/report.txt 2>/dev/null || echo "(no reconciliation output)"
+            echo '```'
+            echo
+            echo "Exclusion policy: test/helpers/periodic-exclude-data.ts (every entry needs reason + tracking; removal re-activates the file next week)."
+          } > "$BODY_FILE"
+          EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty')
+          if [ -n "$EXISTING" ]; then
+            gh issue comment "$EXISTING" --repo "$GITHUB_REPOSITORY" --body-file "$BODY_FILE"
+            echo "commented on #$EXISTING"
+          else
+            gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --body-file "$BODY_FILE"
+          fi
+
+      - name: Fail the workflow when reconciliation failed
+        if: steps.reconcile.outputs.exit != '0'
+        run: exit 1

scripts/test-paid-shards.ts

@@ -63,6 +63,7 @@ import {
   strictTestExitCode,
 } from './test-strict-output';
 import { PAID_TEST_GLOBS, isPaidTestFile } from '../test/helpers/paid-test-set';
+import { PERIODIC_CI_EXCLUDE } from '../test/helpers/periodic-exclude-data';
 import { getProjectEvalDir } from '../test/helpers/eval-store';
 import { preflightAnthropicApi } from '../test/helpers/anthropic-preflight';
 import {
@@ -75,6 +76,7 @@ import {
 } from '../test/helpers/touchfiles';
 
 export { PAID_TEST_GLOBS, isPaidTestFile };
+export { PERIODIC_CI_EXCLUDE };
 
 const ROOT = path.resolve(import.meta.dir, '..');
 
@@ -142,7 +144,17 @@ export interface TierSelection {
 export function selectPaidTestFiles(files: string[], tier: PaidTier, rootDir = ROOT): TierSelection {
   const selected: string[] = [];
   const excluded: Array<{ file: string; reason: string }> = [];
+  // Periodic-lane exclusions (documented-red / manual-hardware files): a
+  // known-red weekly shard is triage waste locally AND in CI, so the list
+  // applies to every periodic run, with the reason surfaced per file.
+  const ciExcluded = (file: string): { reason: string; tracking: string } | undefined =>
+    tier === 'periodic' ? PERIODIC_CI_EXCLUDE[normalizeRelativePath(file)] : undefined;
   for (const file of files) {
+    const exclusion = ciExcluded(file);
+    if (exclusion) {
+      excluded.push({ file, reason: `excluded: ${exclusion.reason} [${exclusion.tracking}]` });
+      continue;
+    }
     const source = fs.readFileSync(path.join(rootDir, file), 'utf8');
     const classification = classifyPaidTestFile(source, tier);
     if (classification.included) selected.push(file);

test/helpers/periodic-exclude-data.ts

@@ -0,0 +1,34 @@
+/**
+ * Periodic-lane exclusions — LITERALS ONLY (own file, deliberately NOT in
+ * touchfiles-data.ts: that file is evaluated standalone by map-diff against
+ * old git versions, and its contract must not grow unrelated exports).
+ *
+ * The weekly periodic CI lane runs EVERY periodic-tier file (EVALS_ALL=1) so
+ * tests can't rot invisibly — the coverage contract. A file lands here only
+ * when running it weekly is KNOWN waste (documented-red or requires manual
+ * hardware), and every entry must carry a tracking pointer with a re-entry
+ * condition, so an exclusion is a decision with an owner, not a place tests
+ * go to die. Pinned by test/periodic-exclude-policy.test.ts: entries must
+ * name real files and carry non-empty reason + tracking.
+ *
+ * Removing an entry re-activates the file on the next weekly run — that IS
+ * the re-entry mechanism.
+ */
+export const PERIODIC_CI_EXCLUDE: Record<string, { reason: string; tracking: string }> = {
+  'test/skill-e2e-ship-idempotency.test.ts': {
+    reason:
+      'documented-red: the PTY child sits at the Claude Code welcome screen for the full budget '
+      + '(readiness/typing race vs CLI 2.1.x); never green since it was born in v1.63',
+    tracking: 'TODOS.md "periodic tier — three documented-red tests need structural repair" (1 of 3 resolved: sidebar trio already deleted)',
+  },
+  'test/skill-e2e-brain-privacy-gate.test.ts': {
+    reason:
+      'documented-red: the artifacts-sync stop-gate preconditions do not survive the hermetic env '
+      + 'even with per-test HOME/GSTACK_HOME injection; never green anywhere',
+    tracking: 'TODOS.md "periodic tier — three documented-red tests need structural repair"',
+  },
+  'test/skill-e2e-ios.test.ts': {
+    reason: 'requires a live iOS device/simulator toolchain (xcodebuild, devicectl) — manual hardware, not a CI runner capability',
+    tracking: 'TODOS.md "skill-e2e-ios CI story" (device/runner decision)',
+  },
+};

test/periodic-exclude-policy.test.ts

@@ -0,0 +1,45 @@
+/**
+ * The periodic exclude list is a set of DECISIONS, not a place tests go to
+ * die: every entry names a real file (a deleted/renamed file must drop its
+ * entry) and carries a non-empty reason + tracking pointer (the re-entry
+ * condition lives there). The runner surfaces each exclusion per run, and
+ * removing an entry re-activates the file on the next weekly lane.
+ */
+import { describe, expect, test } from 'bun:test';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+import { PERIODIC_CI_EXCLUDE } from './helpers/periodic-exclude-data';
+import { isPaidTestFile } from './helpers/paid-test-set';
+import { selectPaidTestFiles } from '../scripts/test-paid-shards';
+
+const ROOT = path.resolve(__dirname, '..');
+
+describe('periodic exclude policy', () => {
+  test('every entry names a real paid file and carries reason + tracking', () => {
+    const entries = Object.entries(PERIODIC_CI_EXCLUDE);
+    expect(entries.length).toBeGreaterThan(0);
+    for (const [file, meta] of entries) {
+      expect(fs.existsSync(path.join(ROOT, file)), `stale exclude entry: ${file}`).toBe(true);
+      expect(isPaidTestFile(file), `${file} is not a paid file — exclusion is meaningless`).toBe(true);
+      expect(meta.reason.length, `${file}: empty reason`).toBeGreaterThan(20);
+      expect(meta.tracking.length, `${file}: empty tracking pointer`).toBeGreaterThan(5);
+    }
+  });
+
+  test('exclusions apply to the periodic tier only, with the reason surfaced', () => {
+    const files = Object.keys(PERIODIC_CI_EXCLUDE);
+    const periodic = selectPaidTestFiles(files, 'periodic');
+    expect(periodic.selected).toEqual([]);
+    for (const { reason } of periodic.excluded) {
+      expect(reason).toStartWith('excluded: ');
+      expect(reason).toContain('[');
+    }
+    // Gate tier ignores the list (these files are periodic-tier anyway; the
+    // list must never leak into gate semantics).
+    const gate = selectPaidTestFiles(files, 'gate');
+    for (const { reason } of gate.excluded) {
+      expect(reason).not.toStartWith('excluded: ');
+    }
+  });
+});