From 1d83d0a436d4004793a9780d88a6b8dedf7104a9 Mon Sep 17 00:00:00 2001 From: garrytan Date: Wed, 7 Oct 2026 01:18:17 +0000 Subject: [PATCH] ci: weekly eval-sweep workflow on main Mondays 12:00 UTC and on dispatch (k, cap_usd, dry_run): runs ship-measure sweep in the published CI image by digest with the census's paid-eval setup, downloads the last 7 days of sweep reports first so the weekly cap holds across runs, uploads the report and captures, and writes the job summary. Read-only grants; it never pushes or opens a pull request. One sweep at a time repository-wide, never cancelled mid-spend. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- .github/actionlint.yaml | 1 + .github/workflows/eval-sweep.yml | 204 ++++++++++++++++++++++++++++++ test/ci-image-tag-binding.test.ts | 3 +- test/eval-sweep-workflow.test.ts | 76 +++++++++++ 4 files changed, 283 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/eval-sweep.yml create mode 100644 test/eval-sweep-workflow.test.ts diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index cdd601c83..536ec9cd4 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -2,3 +2,4 @@ self-hosted-runner: labels: - ubicloud-standard-2 - ubicloud-standard-8 + - ubicloud-standard-16 diff --git a/.github/workflows/eval-sweep.yml b/.github/workflows/eval-sweep.yml new file mode 100644 index 000000000..a4e8bd926 --- /dev/null +++ b/.github/workflows/eval-sweep.yml @@ -0,0 +1,204 @@ +name: Eval sweep +# The weekly off-ship qualification sweep (docs/TESTING_INTERNALS.md#ship-measure-sweep): +# `ship-measure sweep` ranks the gate cases that cost the gate's all-green +# probability most (eval:pass-rates history on main) and measures the top K +# alone at their kind's N on this ref's head, with /ship's runner and +# measurement bar. Every trial is diagnostic: no verdict, no pass-rate +# history. The weekly spend cap (ship_measure_sweep_budget_usd, $150) is shared +# with every sweep report of the last 7 days, downloaded before the sweep runs. +# This workflow only reads: it never pushes, commits or opens a pull request; +# an agent fixes from the uploaded report. Pinned by +# test/eval-sweep-workflow.test.ts. +on: + schedule: + - cron: '0 12 * * 1' # Monday 12:00 UTC, after the 06:00 periodic census + workflow_dispatch: + inputs: + k: + description: 'Cases to measure (default: ship_measure_sweep_cases, 5)' + type: string + default: '' + cap_usd: + description: 'Weekly spend cap in USD, shared with the last 7 days of sweeps (default: ship_measure_sweep_budget_usd, 150)' + type: string + default: '' + dry_run: + description: 'Plan and estimate only ($0)' + type: boolean + default: false + +# One sweep at a time across every ref, never cancelled: two concurrent runs +# would both read the same week's spend and could exceed the cap together. +concurrency: + group: eval-sweep + cancel-in-progress: false + +permissions: + contents: read + +env: + IMAGE: ghcr.io/${{ github.repository }}/ci + +jobs: + image: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + packages: read + outputs: + image-ref: ${{ steps.digest.outputs.ref }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - id: meta + # The same content-hash tag the eval workflows build and push + # (test/ci-image-tag-binding.test.ts pins the identical input list). + run: echo "tag=${{ env.IMAGE }}:${{ hashFiles('.github/docker/Dockerfile.ci', 'bun.lock', 'patches/**') }}" >> "$GITHUB_OUTPUT" + + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Run the published image by digest: the digest is part of every + # measurement's pooling identity (GSTACK_CI_IMAGE). The sweep never + # builds an image; evals.yml and evals-periodic.yml publish it. + - name: Resolve the published CI image digest + id: digest + env: + TAG: ${{ steps.meta.outputs.tag }} + run: | + set -euo pipefail + if ! digest=$(docker buildx imagetools inspect "$TAG" --format '{{json .Manifest.Digest}}' | tr -d '"'); then + echo "::error::CI image $TAG is not published; run evals-periodic.yml (or an evals.yml PR run) to build it, then rerun the sweep" + exit 1 + fi + echo "ref=${IMAGE}@${digest}" >> "$GITHUB_OUTPUT" + + sweep: + runs-on: ubicloud-standard-16 + needs: image + timeout-minutes: 240 + permissions: + contents: read + actions: read + packages: read + container: + image: ${{ needs.image.outputs.image-ref }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --user runner + env: + EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-sweep + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + # Full history: selection walks git at module load (evals-periodic.yml). + fetch-depth: 0 + persist-credentials: false + + - name: Fix bun temp + uses: ./.github/actions/fix-bun-temp + + - name: Restore deps + uses: ./.github/actions/restore-deps + + - run: bun run build + + - name: Seed claude interactive config + uses: ./.github/actions/seed-claude-config + with: + anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} + + - name: Register gstack skills for PTY tests + uses: ./.github/actions/register-gstack-skills + + # Same as evals-periodic.yml: the image ships the pinned Codex CLI + # without a credential; Codex cases see it only through the shard runner. + - name: Authenticate the Codex CLI + shell: bash + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: | + set -euo pipefail + codex="${GSTACK_CI_CODEX_BIN_DIR:?}/codex" + export CODEX_HOME="$HOME/.gstack-ci-codex" + mkdir -p "$CODEX_HOME" + echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV" + if [ -z "${OPENAI_API_KEY:-}" ]; then + echo "::warning::OPENAI_API_KEY is unavailable; Codex eval cases cannot authenticate and will fail." + exit 0 + fi + printenv OPENAI_API_KEY | "$codex" login --with-api-key + "$codex" login status >/dev/null 2>&1 + + # The weekly cap counts every sweep of the last 7 days. A listing failure + # fails the job before any spend; a run without a report spent nothing. + - name: Download this week's earlier sweep reports + shell: bash + env: + GH_TOKEN: ${{ github.token }} + PRIOR: /tmp/prior-sweeps + run: | + set -euo pipefail + mkdir -p "$PRIOR" + since=$(date -u -d '7 days ago' +%Y-%m-%dT%H:%M:%SZ) + ids=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow eval-sweep.yml --limit 100 --json databaseId,createdAt \ + --jq ".[] | select(.createdAt >= \"$since\") | .databaseId") + for id in $ids; do + [ "$id" = "$GITHUB_RUN_ID" ] && continue + gh run download "$id" --repo "$GITHUB_REPOSITORY" --name ship-measure-sweep-report --dir "$PRIOR/$id" 2>/dev/null \ + || echo "run $id: no sweep report (nothing charged)" + done + + - name: Run the sweep + shell: bash + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers + GH_TOKEN: ${{ github.token }} + GSTACK_CI_IMAGE: ${{ needs.image.outputs.image-ref }} + SWEEP_OUT: /tmp/ship-measure-sweep + SWEEP_PRIOR: /tmp/prior-sweeps + SWEEP_K: ${{ inputs.k }} + SWEEP_CAP_USD: ${{ inputs.cap_usd }} + SWEEP_DRY_RUN: ${{ inputs.dry_run }} + run: | + set -euo pipefail + args=(--out "$SWEEP_OUT" --prior "$SWEEP_PRIOR" --jobs 4) + if [ -n "${SWEEP_K:-}" ]; then args+=(--k "$SWEEP_K"); fi + if [ -n "${SWEEP_CAP_USD:-}" ]; then args+=(--cap-usd "$SWEEP_CAP_USD"); fi + if [ "${SWEEP_DRY_RUN:-false}" = "true" ]; then args+=(--dry-run); fi + bun run scripts/ship-measure.ts sweep "${args[@]}" + + - name: Upload the sweep report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ship-measure-sweep-report + path: | + /tmp/ship-measure-sweep/sweep-*/sweep-report.json + /tmp/ship-measure-sweep/sweep-*/sweep-report.md + if-no-files-found: ignore + retention-days: 30 + + - name: Upload the sweep captures + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ship-measure-sweep-captures + include-hidden-files: true + path: /tmp/ship-measure-sweep + if-no-files-found: ignore + retention-days: 14 + + - name: Note a sweep that wrote no report + if: failure() + run: echo "The sweep failed before writing its report; see the Run the sweep step log." >> "$GITHUB_STEP_SUMMARY" diff --git a/test/ci-image-tag-binding.test.ts b/test/ci-image-tag-binding.test.ts index 669c8cc37..92941f865 100644 --- a/test/ci-image-tag-binding.test.ts +++ b/test/ci-image-tag-binding.test.ts @@ -13,7 +13,8 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; const ROOT = path.resolve(__dirname, '..'); -const FILES = ['evals.yml', 'evals-periodic.yml', 'evals-marathon.yml']; +// eval-sweep.yml only resolves the tag (it never builds), so it must look up the same one. +const FILES = ['evals.yml', 'evals-periodic.yml', 'evals-marathon.yml', 'eval-sweep.yml']; function hashFilesCalls(name: string): string[] { const source = fs.readFileSync( diff --git a/test/eval-sweep-workflow.test.ts b/test/eval-sweep-workflow.test.ts new file mode 100644 index 000000000..7bfe55572 --- /dev/null +++ b/test/eval-sweep-workflow.test.ts @@ -0,0 +1,76 @@ +/** + * Weekly off-ship sweep workflow wiring (docs/TESTING_INTERNALS.md#ship-measure-sweep): + * weekly after the periodic census plus dispatch, read-only permissions (it + * never pushes or opens a pull request), one sweep at a time repository-wide + * so two runs never spend the same weekly cap, the CI image by digest with the + * census's paid-eval setup, earlier reports downloaded before any spend, + * dispatch inputs reaching the shell only through env, and the report uploaded + * even when the sweep fails. + */ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const source = fs.readFileSync(path.join(ROOT, '.github/workflows/eval-sweep.yml'), 'utf8'); +const workflow = Bun.YAML.parse(source) as any; +const sweep = workflow.jobs.sweep; +const step = (name: string) => sweep.steps.find((s: any) => s.name === name); + +describe('eval-sweep.yml', () => { + test('runs weekly after the 06:00 UTC periodic census, plus dispatch with k, cap and dry-run inputs', () => { + expect(Object.keys(workflow.on).sort()).toEqual(['schedule', 'workflow_dispatch']); + const [hour] = /cron: '0 (\d+) \* \* 1'/.exec(source)!.slice(1).map(Number); + const census = Number(/cron: '0 (\d+) \* \* 1'/.exec(fs.readFileSync(path.join(ROOT, '.github/workflows/evals-periodic.yml'), 'utf8'))![1]); + expect(hour!).toBeGreaterThan(census); + expect(Object.keys(workflow.on.workflow_dispatch.inputs).sort()).toEqual(['cap_usd', 'dry_run', 'k']); + }); + + test('never pushes, commits or opens a pull request: read-only grants, no write commands', () => { + expect(workflow.permissions).toEqual({ contents: 'read' }); + for (const job of Object.values(workflow.jobs)) { + for (const value of Object.values(job.permissions ?? {})) expect(value).toBe('read'); + } + expect(source).not.toMatch(/git push|git commit|gh pr |gh issue|pull-requests:|contents: write/); + for (const s of sweep.steps) if (s.uses?.startsWith('actions/checkout')) expect(s.with['persist-credentials']).toBe(false); + }); + + test('one sweep at a time across every ref, never cancelled mid-spend', () => { + expect(workflow.concurrency).toEqual({ group: 'eval-sweep', 'cancel-in-progress': false }); + }); + + test('runs in the published CI image by digest with the census paid-eval setup', () => { + expect(sweep.container.image).toBe('${{ needs.image.outputs.image-ref }}'); + expect(sweep.container.options).toBe('--user runner'); + expect(workflow.jobs.image.steps.map((s: any) => s.name ?? s.uses)).toContain('Resolve the published CI image digest'); + for (const name of ['Seed claude interactive config', 'Register gstack skills for PTY tests', 'Authenticate the Codex CLI']) expect(step(name)).toBeDefined(); + const run = step('Run the sweep'); + expect(run.env.ANTHROPIC_API_KEY).toBe('${{ secrets.ANTHROPIC_API_KEY }}'); + expect(run.env.GSTACK_CI_IMAGE).toBe('${{ needs.image.outputs.image-ref }}'); + }); + + test('downloads the last 7 days of sweep reports before the sweep and passes them as --prior', () => { + const names = sweep.steps.map((s: any) => s.name); + expect(names.indexOf("Download this week's earlier sweep reports")).toBeLessThan(names.indexOf('Run the sweep')); + const download = step("Download this week's earlier sweep reports"); + expect(download.run).toContain("date -u -d '7 days ago'"); + expect(download.run).toContain('--name ship-measure-sweep-report'); + expect(download.run).toContain('set -euo pipefail'); + expect(step('Run the sweep').run).toContain('--prior "$SWEEP_PRIOR"'); + }); + + test('dispatch inputs reach the shell only through env', () => { + const run = step('Run the sweep'); + expect(run.run).not.toContain('${{'); + expect(run.env).toMatchObject({ SWEEP_K: '${{ inputs.k }}', SWEEP_CAP_USD: '${{ inputs.cap_usd }}', SWEEP_DRY_RUN: '${{ inputs.dry_run }}' }); + expect(run.run).toContain('bun run scripts/ship-measure.ts sweep "${args[@]}"'); + }); + + test('uploads the report and captures even when the sweep fails', () => { + const report = step('Upload the sweep report'); + expect(report.if).toBe('always()'); + expect(report.with.name).toBe('ship-measure-sweep-report'); + expect(report.with.path).toContain('sweep-report.md'); + expect(step('Upload the sweep captures').if).toBe('always()'); + }); +});