ci: weekly eval-sweep workflow on main

Mondays 12:00 UTC and on dispatch (k, cap_usd, dry_run): runs
ship-measure sweep in the published CI image by digest with the census's
paid-eval setup, downloads the last 7 days of sweep reports first so the
weekly cap holds across runs, uploads the report and captures, and writes
the job summary. Read-only grants; it never pushes or opens a pull request.
One sweep at a time repository-wide, never cancelled mid-spend.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

.github/actionlint.yaml

@@ -2,3 +2,4 @@ self-hosted-runner:
   labels:
     - ubicloud-standard-2
     - ubicloud-standard-8
+    - ubicloud-standard-16

.github/workflows/eval-sweep.yml

@@ -0,0 +1,204 @@
+name: Eval sweep
+# The weekly off-ship qualification sweep (docs/TESTING_INTERNALS.md#ship-measure-sweep):
+# `ship-measure sweep` ranks the gate cases that cost the gate's all-green
+# probability most (eval:pass-rates history on main) and measures the top K
+# alone at their kind's N on this ref's head, with /ship's runner and
+# measurement bar. Every trial is diagnostic: no verdict, no pass-rate
+# history. The weekly spend cap (ship_measure_sweep_budget_usd, $150) is shared
+# with every sweep report of the last 7 days, downloaded before the sweep runs.
+# This workflow only reads: it never pushes, commits or opens a pull request;
+# an agent fixes from the uploaded report. Pinned by
+# test/eval-sweep-workflow.test.ts.
+on:
+  schedule:
+    - cron: '0 12 * * 1'  # Monday 12:00 UTC, after the 06:00 periodic census
+  workflow_dispatch:
+    inputs:
+      k:
+        description: 'Cases to measure (default: ship_measure_sweep_cases, 5)'
+        type: string
+        default: ''
+      cap_usd:
+        description: 'Weekly spend cap in USD, shared with the last 7 days of sweeps (default: ship_measure_sweep_budget_usd, 150)'
+        type: string
+        default: ''
+      dry_run:
+        description: 'Plan and estimate only ($0)'
+        type: boolean
+        default: false
+
+# One sweep at a time across every ref, never cancelled: two concurrent runs
+# would both read the same week's spend and could exceed the cap together.
+concurrency:
+  group: eval-sweep
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+env:
+  IMAGE: ghcr.io/${{ github.repository }}/ci
+
+jobs:
+  image:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      packages: read
+    outputs:
+      image-ref: ${{ steps.digest.outputs.ref }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
+
+      - id: meta
+        # The same content-hash tag the eval workflows build and push
+        # (test/ci-image-tag-binding.test.ts pins the identical input list).
+        run: echo "tag=${{ env.IMAGE }}:${{ hashFiles('.github/docker/Dockerfile.ci', 'bun.lock', 'patches/**') }}" >> "$GITHUB_OUTPUT"
+
+      - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Run the published image by digest: the digest is part of every
+      # measurement's pooling identity (GSTACK_CI_IMAGE). The sweep never
+      # builds an image; evals.yml and evals-periodic.yml publish it.
+      - name: Resolve the published CI image digest
+        id: digest
+        env:
+          TAG: ${{ steps.meta.outputs.tag }}
+        run: |
+          set -euo pipefail
+          if ! digest=$(docker buildx imagetools inspect "$TAG" --format '{{json .Manifest.Digest}}' | tr -d '"'); then
+            echo "::error::CI image $TAG is not published; run evals-periodic.yml (or an evals.yml PR run) to build it, then rerun the sweep"
+            exit 1
+          fi
+          echo "ref=${IMAGE}@${digest}" >> "$GITHUB_OUTPUT"
+
+  sweep:
+    runs-on: ubicloud-standard-16
+    needs: image
+    timeout-minutes: 240
+    permissions:
+      contents: read
+      actions: read
+      packages: read
+    container:
+      image: ${{ needs.image.outputs.image-ref }}
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+      options: --user runner
+    env:
+      EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-sweep
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          # Full history: selection walks git at module load (evals-periodic.yml).
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Fix bun temp
+        uses: ./.github/actions/fix-bun-temp
+
+      - name: Restore deps
+        uses: ./.github/actions/restore-deps
+
+      - run: bun run build
+
+      - name: Seed claude interactive config
+        uses: ./.github/actions/seed-claude-config
+        with:
+          anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
+
+      - name: Register gstack skills for PTY tests
+        uses: ./.github/actions/register-gstack-skills
+
+      # Same as evals-periodic.yml: the image ships the pinned Codex CLI
+      # without a credential; Codex cases see it only through the shard runner.
+      - name: Authenticate the Codex CLI
+        shell: bash
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: |
+          set -euo pipefail
+          codex="${GSTACK_CI_CODEX_BIN_DIR:?}/codex"
+          export CODEX_HOME="$HOME/.gstack-ci-codex"
+          mkdir -p "$CODEX_HOME"
+          echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV"
+          if [ -z "${OPENAI_API_KEY:-}" ]; then
+            echo "::warning::OPENAI_API_KEY is unavailable; Codex eval cases cannot authenticate and will fail."
+            exit 0
+          fi
+          printenv OPENAI_API_KEY | "$codex" login --with-api-key
+          "$codex" login status >/dev/null 2>&1
+
+      # The weekly cap counts every sweep of the last 7 days. A listing failure
+      # fails the job before any spend; a run without a report spent nothing.
+      - name: Download this week's earlier sweep reports
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PRIOR: /tmp/prior-sweeps
+        run: |
+          set -euo pipefail
+          mkdir -p "$PRIOR"
+          since=$(date -u -d '7 days ago' +%Y-%m-%dT%H:%M:%SZ)
+          ids=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow eval-sweep.yml --limit 100 --json databaseId,createdAt \
+            --jq ".[] | select(.createdAt >= \"$since\") | .databaseId")
+          for id in $ids; do
+            [ "$id" = "$GITHUB_RUN_ID" ] && continue
+            gh run download "$id" --repo "$GITHUB_REPOSITORY" --name ship-measure-sweep-report --dir "$PRIOR/$id" 2>/dev/null \
+              || echo "run $id: no sweep report (nothing charged)"
+          done
+
+      - name: Run the sweep
+        shell: bash
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+          PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers
+          GH_TOKEN: ${{ github.token }}
+          GSTACK_CI_IMAGE: ${{ needs.image.outputs.image-ref }}
+          SWEEP_OUT: /tmp/ship-measure-sweep
+          SWEEP_PRIOR: /tmp/prior-sweeps
+          SWEEP_K: ${{ inputs.k }}
+          SWEEP_CAP_USD: ${{ inputs.cap_usd }}
+          SWEEP_DRY_RUN: ${{ inputs.dry_run }}
+        run: |
+          set -euo pipefail
+          args=(--out "$SWEEP_OUT" --prior "$SWEEP_PRIOR" --jobs 4)
+          if [ -n "${SWEEP_K:-}" ]; then args+=(--k "$SWEEP_K"); fi
+          if [ -n "${SWEEP_CAP_USD:-}" ]; then args+=(--cap-usd "$SWEEP_CAP_USD"); fi
+          if [ "${SWEEP_DRY_RUN:-false}" = "true" ]; then args+=(--dry-run); fi
+          bun run scripts/ship-measure.ts sweep "${args[@]}"
+
+      - name: Upload the sweep report
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: ship-measure-sweep-report
+          path: |
+            /tmp/ship-measure-sweep/sweep-*/sweep-report.json
+            /tmp/ship-measure-sweep/sweep-*/sweep-report.md
+          if-no-files-found: ignore
+          retention-days: 30
+
+      - name: Upload the sweep captures
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: ship-measure-sweep-captures
+          include-hidden-files: true
+          path: /tmp/ship-measure-sweep
+          if-no-files-found: ignore
+          retention-days: 14
+
+      - name: Note a sweep that wrote no report
+        if: failure()
+        run: echo "The sweep failed before writing its report; see the Run the sweep step log." >> "$GITHUB_STEP_SUMMARY"

test/ci-image-tag-binding.test.ts

@@ -13,7 +13,8 @@ import * as fs from 'node:fs';
 import * as path from 'node:path';
 
 const ROOT = path.resolve(__dirname, '..');
-const FILES = ['evals.yml', 'evals-periodic.yml', 'evals-marathon.yml'];
+// eval-sweep.yml only resolves the tag (it never builds), so it must look up the same one.
+const FILES = ['evals.yml', 'evals-periodic.yml', 'evals-marathon.yml', 'eval-sweep.yml'];
 
 function hashFilesCalls(name: string): string[] {
   const source = fs.readFileSync(

test/eval-sweep-workflow.test.ts

@@ -0,0 +1,76 @@
+/**
+ * Weekly off-ship sweep workflow wiring (docs/TESTING_INTERNALS.md#ship-measure-sweep):
+ * weekly after the periodic census plus dispatch, read-only permissions (it
+ * never pushes or opens a pull request), one sweep at a time repository-wide
+ * so two runs never spend the same weekly cap, the CI image by digest with the
+ * census's paid-eval setup, earlier reports downloaded before any spend,
+ * dispatch inputs reaching the shell only through env, and the report uploaded
+ * even when the sweep fails.
+ */
+import { describe, expect, test } from 'bun:test';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+const ROOT = path.resolve(import.meta.dir, '..');
+const source = fs.readFileSync(path.join(ROOT, '.github/workflows/eval-sweep.yml'), 'utf8');
+const workflow = Bun.YAML.parse(source) as any;
+const sweep = workflow.jobs.sweep;
+const step = (name: string) => sweep.steps.find((s: any) => s.name === name);
+
+describe('eval-sweep.yml', () => {
+  test('runs weekly after the 06:00 UTC periodic census, plus dispatch with k, cap and dry-run inputs', () => {
+    expect(Object.keys(workflow.on).sort()).toEqual(['schedule', 'workflow_dispatch']);
+    const [hour] = /cron: '0 (\d+) \* \* 1'/.exec(source)!.slice(1).map(Number);
+    const census = Number(/cron: '0 (\d+) \* \* 1'/.exec(fs.readFileSync(path.join(ROOT, '.github/workflows/evals-periodic.yml'), 'utf8'))![1]);
+    expect(hour!).toBeGreaterThan(census);
+    expect(Object.keys(workflow.on.workflow_dispatch.inputs).sort()).toEqual(['cap_usd', 'dry_run', 'k']);
+  });
+
+  test('never pushes, commits or opens a pull request: read-only grants, no write commands', () => {
+    expect(workflow.permissions).toEqual({ contents: 'read' });
+    for (const job of Object.values<any>(workflow.jobs)) {
+      for (const value of Object.values<string>(job.permissions ?? {})) expect(value).toBe('read');
+    }
+    expect(source).not.toMatch(/git push|git commit|gh pr |gh issue|pull-requests:|contents: write/);
+    for (const s of sweep.steps) if (s.uses?.startsWith('actions/checkout')) expect(s.with['persist-credentials']).toBe(false);
+  });
+
+  test('one sweep at a time across every ref, never cancelled mid-spend', () => {
+    expect(workflow.concurrency).toEqual({ group: 'eval-sweep', 'cancel-in-progress': false });
+  });
+
+  test('runs in the published CI image by digest with the census paid-eval setup', () => {
+    expect(sweep.container.image).toBe('${{ needs.image.outputs.image-ref }}');
+    expect(sweep.container.options).toBe('--user runner');
+    expect(workflow.jobs.image.steps.map((s: any) => s.name ?? s.uses)).toContain('Resolve the published CI image digest');
+    for (const name of ['Seed claude interactive config', 'Register gstack skills for PTY tests', 'Authenticate the Codex CLI']) expect(step(name)).toBeDefined();
+    const run = step('Run the sweep');
+    expect(run.env.ANTHROPIC_API_KEY).toBe('${{ secrets.ANTHROPIC_API_KEY }}');
+    expect(run.env.GSTACK_CI_IMAGE).toBe('${{ needs.image.outputs.image-ref }}');
+  });
+
+  test('downloads the last 7 days of sweep reports before the sweep and passes them as --prior', () => {
+    const names = sweep.steps.map((s: any) => s.name);
+    expect(names.indexOf("Download this week's earlier sweep reports")).toBeLessThan(names.indexOf('Run the sweep'));
+    const download = step("Download this week's earlier sweep reports");
+    expect(download.run).toContain("date -u -d '7 days ago'");
+    expect(download.run).toContain('--name ship-measure-sweep-report');
+    expect(download.run).toContain('set -euo pipefail');
+    expect(step('Run the sweep').run).toContain('--prior "$SWEEP_PRIOR"');
+  });
+
+  test('dispatch inputs reach the shell only through env', () => {
+    const run = step('Run the sweep');
+    expect(run.run).not.toContain('${{');
+    expect(run.env).toMatchObject({ SWEEP_K: '${{ inputs.k }}', SWEEP_CAP_USD: '${{ inputs.cap_usd }}', SWEEP_DRY_RUN: '${{ inputs.dry_run }}' });
+    expect(run.run).toContain('bun run scripts/ship-measure.ts sweep "${args[@]}"');
+  });
+
+  test('uploads the report and captures even when the sweep fails', () => {
+    const report = step('Upload the sweep report');
+    expect(report.if).toBe('always()');
+    expect(report.with.name).toBe('ship-measure-sweep-report');
+    expect(report.with.path).toContain('sweep-report.md');
+    expect(step('Upload the sweep captures').if).toBe('always()');
+  });
+});