diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index cdd601c83..536ec9cd4 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -2,3 +2,4 @@ self-hosted-runner: labels: - ubicloud-standard-2 - ubicloud-standard-8 + - ubicloud-standard-16 diff --git a/.github/workflows/eval-sweep.yml b/.github/workflows/eval-sweep.yml new file mode 100644 index 000000000..a4e8bd926 --- /dev/null +++ b/.github/workflows/eval-sweep.yml @@ -0,0 +1,204 @@ +name: Eval sweep +# The weekly off-ship qualification sweep (docs/TESTING_INTERNALS.md#ship-measure-sweep): +# `ship-measure sweep` ranks the gate cases that cost the gate's all-green +# probability most (eval:pass-rates history on main) and measures the top K +# alone at their kind's N on this ref's head, with /ship's runner and +# measurement bar. Every trial is diagnostic: no verdict, no pass-rate +# history. The weekly spend cap (ship_measure_sweep_budget_usd, $150) is shared +# with every sweep report of the last 7 days, downloaded before the sweep runs. +# This workflow only reads: it never pushes, commits or opens a pull request; +# an agent fixes from the uploaded report. Pinned by +# test/eval-sweep-workflow.test.ts. +on: + schedule: + - cron: '0 12 * * 1' # Monday 12:00 UTC, after the 06:00 periodic census + workflow_dispatch: + inputs: + k: + description: 'Cases to measure (default: ship_measure_sweep_cases, 5)' + type: string + default: '' + cap_usd: + description: 'Weekly spend cap in USD, shared with the last 7 days of sweeps (default: ship_measure_sweep_budget_usd, 150)' + type: string + default: '' + dry_run: + description: 'Plan and estimate only ($0)' + type: boolean + default: false + +# One sweep at a time across every ref, never cancelled: two concurrent runs +# would both read the same week's spend and could exceed the cap together. +concurrency: + group: eval-sweep + cancel-in-progress: false + +permissions: + contents: read + +env: + IMAGE: ghcr.io/${{ github.repository }}/ci + +jobs: + image: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + packages: read + outputs: + image-ref: ${{ steps.digest.outputs.ref }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - id: meta + # The same content-hash tag the eval workflows build and push + # (test/ci-image-tag-binding.test.ts pins the identical input list). + run: echo "tag=${{ env.IMAGE }}:${{ hashFiles('.github/docker/Dockerfile.ci', 'bun.lock', 'patches/**') }}" >> "$GITHUB_OUTPUT" + + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Run the published image by digest: the digest is part of every + # measurement's pooling identity (GSTACK_CI_IMAGE). The sweep never + # builds an image; evals.yml and evals-periodic.yml publish it. + - name: Resolve the published CI image digest + id: digest + env: + TAG: ${{ steps.meta.outputs.tag }} + run: | + set -euo pipefail + if ! digest=$(docker buildx imagetools inspect "$TAG" --format '{{json .Manifest.Digest}}' | tr -d '"'); then + echo "::error::CI image $TAG is not published; run evals-periodic.yml (or an evals.yml PR run) to build it, then rerun the sweep" + exit 1 + fi + echo "ref=${IMAGE}@${digest}" >> "$GITHUB_OUTPUT" + + sweep: + runs-on: ubicloud-standard-16 + needs: image + timeout-minutes: 240 + permissions: + contents: read + actions: read + packages: read + container: + image: ${{ needs.image.outputs.image-ref }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --user runner + env: + EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-sweep + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + # Full history: selection walks git at module load (evals-periodic.yml). + fetch-depth: 0 + persist-credentials: false + + - name: Fix bun temp + uses: ./.github/actions/fix-bun-temp + + - name: Restore deps + uses: ./.github/actions/restore-deps + + - run: bun run build + + - name: Seed claude interactive config + uses: ./.github/actions/seed-claude-config + with: + anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} + + - name: Register gstack skills for PTY tests + uses: ./.github/actions/register-gstack-skills + + # Same as evals-periodic.yml: the image ships the pinned Codex CLI + # without a credential; Codex cases see it only through the shard runner. + - name: Authenticate the Codex CLI + shell: bash + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: | + set -euo pipefail + codex="${GSTACK_CI_CODEX_BIN_DIR:?}/codex" + export CODEX_HOME="$HOME/.gstack-ci-codex" + mkdir -p "$CODEX_HOME" + echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV" + if [ -z "${OPENAI_API_KEY:-}" ]; then + echo "::warning::OPENAI_API_KEY is unavailable; Codex eval cases cannot authenticate and will fail." + exit 0 + fi + printenv OPENAI_API_KEY | "$codex" login --with-api-key + "$codex" login status >/dev/null 2>&1 + + # The weekly cap counts every sweep of the last 7 days. A listing failure + # fails the job before any spend; a run without a report spent nothing. + - name: Download this week's earlier sweep reports + shell: bash + env: + GH_TOKEN: ${{ github.token }} + PRIOR: /tmp/prior-sweeps + run: | + set -euo pipefail + mkdir -p "$PRIOR" + since=$(date -u -d '7 days ago' +%Y-%m-%dT%H:%M:%SZ) + ids=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow eval-sweep.yml --limit 100 --json databaseId,createdAt \ + --jq ".[] | select(.createdAt >= \"$since\") | .databaseId") + for id in $ids; do + [ "$id" = "$GITHUB_RUN_ID" ] && continue + gh run download "$id" --repo "$GITHUB_REPOSITORY" --name ship-measure-sweep-report --dir "$PRIOR/$id" 2>/dev/null \ + || echo "run $id: no sweep report (nothing charged)" + done + + - name: Run the sweep + shell: bash + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers + GH_TOKEN: ${{ github.token }} + GSTACK_CI_IMAGE: ${{ needs.image.outputs.image-ref }} + SWEEP_OUT: /tmp/ship-measure-sweep + SWEEP_PRIOR: /tmp/prior-sweeps + SWEEP_K: ${{ inputs.k }} + SWEEP_CAP_USD: ${{ inputs.cap_usd }} + SWEEP_DRY_RUN: ${{ inputs.dry_run }} + run: | + set -euo pipefail + args=(--out "$SWEEP_OUT" --prior "$SWEEP_PRIOR" --jobs 4) + if [ -n "${SWEEP_K:-}" ]; then args+=(--k "$SWEEP_K"); fi + if [ -n "${SWEEP_CAP_USD:-}" ]; then args+=(--cap-usd "$SWEEP_CAP_USD"); fi + if [ "${SWEEP_DRY_RUN:-false}" = "true" ]; then args+=(--dry-run); fi + bun run scripts/ship-measure.ts sweep "${args[@]}" + + - name: Upload the sweep report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ship-measure-sweep-report + path: | + /tmp/ship-measure-sweep/sweep-*/sweep-report.json + /tmp/ship-measure-sweep/sweep-*/sweep-report.md + if-no-files-found: ignore + retention-days: 30 + + - name: Upload the sweep captures + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ship-measure-sweep-captures + include-hidden-files: true + path: /tmp/ship-measure-sweep + if-no-files-found: ignore + retention-days: 14 + + - name: Note a sweep that wrote no report + if: failure() + run: echo "The sweep failed before writing its report; see the Run the sweep step log." >> "$GITHUB_STEP_SUMMARY" diff --git a/CHANGELOG.md b/CHANGELOG.md index 3225ee7f4..d1c33189e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,55 @@ # Changelog +## [1.91.33.0] - 2026-10-06 + +**A red eval case now gets one clear verdict from its measurement: meets, qualified, extend once, or fix.** +**A weekly sweep measures main's flakiest gate cases before any ship hits them.** + +/ship already measured a red case alone before rerunning the gate, but every 8 out of 10 was "below target", even when both reds were a provider outage. Now the measurement follows one bar. 9/10 meets it. 8/10 meets it as qualified when every red is a proven provider error, judge noise at the threshold, or a model miss with a cited transcript line. 7/10, or an 8/10 that does not qualify, gets exactly one more batch on the identical tree and is decided once on all 20 trials. Anything else, and any timeout, hang, regression or known fixable cause, is a fix round. Separately, a new weekly workflow picks the gate cases that cost the gate's all-green chance the most and measures them on main, with a hard $150-a-week cap. + +### The numbers that matter + +The bar numbers are binomial for a case whose true per-trial pass rate is p (rule kind, 10 trials). The sweep numbers come from `bun run scripts/ship-measure.ts sweep --dry-run` against main's pass-rate history on 2026-10-06. + +| Case with true pass rate p | Clears the bar in one batch | Clears it with one extension (reds unqualified) | Clears it when every red qualifies | +|---|---|---|---| +| 0.95 (healthy) | 91.4% | 95.9% | 99.8% | +| 0.90 | 73.6% | 80.4% | 97.2% | +| 0.80 (flaky) | 37.6% | 40.8% | 75.3% | + +| Sweep, latest gate census | Value | +|---|---| +| Gate verdicts in the census | 106 | +| Predicted all-green probability from history | 30.7% | +| Cases flagged (top 5 by cost to that probability) | 87% to 91% per-trial pass rate each | +| Estimated cost to measure all 5 at 10 trials | about $76, under the $150 weekly cap | + +A healthy 95% case used to be sent to a pointless fix round 8.6% of the time; with one extension that drops to 4.1%, while a flaky 80% case still fails the bar about 60% of the time. + +### What this means for you + +When /ship's measure loop prints needs-classify, read the listed reds and record each one with `ship-measure classify`; it decides for you. A hand-picked "provider" label is refused unless the trial's own failure cause shows a provider or transport error. The weekly sweep report (artifact `ship-measure-sweep-report`, also the job summary) lists each flagged case with its decision, cost and captures, ready for an agent to fix. + +### Itemized changes + +#### Added +- `ship-measure classify`, `decide` and `extend`, and exit codes 5 (needs-classify) and 6 (EXTEND). EXTEND refuses to pool when any input changed (working tree, runner, Claude CLI, model, CI image, Bun, eval policy or harness version) and names the field; there is never a third batch. +- Void batches: more than 30% of trials failing on provider evidence voids a batch, which is redispatched once; both batches are reported. +- `ship-measure sweep` and `.github/workflows/eval-sweep.yml` (Mondays 12:00 UTC and on dispatch with `k`, `cap_usd` and `dry_run`). Config keys `ship_measure_sweep_cases` (5) and `ship_measure_sweep_budget_usd` (150, per 7 days, shared with every sweep that week). The workflow only reads; it never pushes or opens a pull request. + +#### Changed +- Behavior cases are decided on the trial count over 12 (11 meets, 10 qualified, 9 extends; 22 and 20 of 24 pooled) instead of also requiring every panel to pass on its own; a contract violation still sends the case to a fix round at any count. +- The `ship-measure report` table shows each round's decision and batches, and lists qualified results under their own heading. + +#### Fixed +- `ship-measure measure` no longer refuses a case that shares its test file with other cases (for example `ship-exploratory-late-input`): the no-cost selection check before the first paid trial now accepts the per-case trial shard such a case lists. +- `ship-measure measure` runs cases that keep native evidence (functional QA, docs faults) outside CI: each trial gets its own `EVALS_RUN_ID`, where before every trial stopped at "requires EVALS_RUN_ID" without starting a session. +- The `plan-mode-no-op` gate case no longer times out on large pull requests: its CEO review reads a fixed small plan instead of the branch diff. + +#### For contributors +- The bar is one pure module, `scripts/lib/measure-bar.ts`, pinned by `test/ship-measure-bar.test.ts`; the sweep is `scripts/ship-measure-sweep.ts`, pinned by `test/ship-measure-sweep.test.ts` and `test/eval-sweep-workflow.test.ts`. `eval:pass-rates` history loading is now `loadPassRateHistory()` in `scripts/eval-flake-rank.ts`, shared with the sweep. +- Diagnostic trials still never become verdicts and never enter pass-rate history or EVAL_POLICY pooling. See `docs/TESTING_INTERNALS.md#ship-measure` and `#ship-measure-sweep`. + ## [1.91.32.0] - 2026-10-06 **Codex second opinions work on macOS again, and text from a PR or reviewer can no longer run as a shell command.** diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4cdcd74fa..e12d09282 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -201,6 +201,8 @@ the new defaults. | Run the opt-in ML, gitleaks and Swift checks | `gh workflow run platform-qualification.yml --ref ` | Free CI runners | Pushed branch, `gh` with workflow rights | | Look at past local eval runs | `bun run eval:list` / `eval:compare` / `eval:summary` | Free | Local eval history | | See a case's pass rate across recent weekly runs | `bun run eval:pass-rates --case ` | Free | `gh` with repo read access | +| Measure one red case alone against the ship bar (MEETS 9/10) | `bun run scripts/ship-measure.ts measure --case --round baseline` ([bar](docs/TESTING_INTERNALS.md#ship-measure)) | API spend for 10 trials (12 for behavior); asks above $2/trial, stops at $25/case | Same as paid coverage | +| Measure main's flakiest gate cases (the weekly sweep) | `gh workflow run eval-sweep.yml --ref main [-f k=5] [-f cap_usd=150] [-f dry_run=true]`, or locally `bun run scripts/ship-measure.ts sweep --dry-run` ([sweep](docs/TESTING_INTERNALS.md#ship-measure-sweep)) | API spend up to $150 per 7 days (`ship_measure_sweep_budget_usd`); `--dry-run` is free | `gh` with repo read access; CI dispatch needs workflow rights | | See the audit success metrics and weekly health | `bun run test:health [--since-days 7] [--json] [--enforce]` | Free; 5–20 minutes of `gh` reads | `gh` with repo read access | Old command names are listed under [Retired commands](#retired-commands). diff --git a/VERSION b/VERSION index 0086fa199..8438d0e22 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.32.0 +1.91.33.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 69815cc31..0ff5675e6 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.32.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.33.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/bin/gstack-config b/bin/gstack-config index f1aebac24..19e53abec 100755 --- a/bin/gstack-config +++ b/bin/gstack-config @@ -188,15 +188,19 @@ CONFIG_HEADER='# gstack configuration — edit freely, changes take effect on ne # ─── /ship measure-then-fix loop ───────────────────────────────────── # A red eval case is measured alone before the full gate reruns # (scripts/ship-measure.ts; docs/TESTING_INTERNALS.md#ship-measure). -# ship_measure_rule_trials: 10 # rule cases: trials; pass at 90% (9 of 10) -# ship_measure_behavior_panels: 4 # behavior cases: panels of 3; every panel -# # at 2 of 3 and 90% of trials (11 of 12) -# ship_measure_judge_outputs: 10 # judge cases: outputs, each scored by a -# # 3-sample panel; pass at 90% of outputs +# Bar (scripts/lib/measure-bar.ts): MEETS at 9/10 of the trials, MEETS-qualified +# at 8/10 with every red qualifying, EXTEND at 7/10 (one more batch, pooled). +# ship_measure_rule_trials: 10 # rule cases: trials per batch (MEETS 9 of 10) +# ship_measure_behavior_panels: 4 # behavior cases: panels of 3 per batch +# # (12 trials; MEETS 11 of 12) +# ship_measure_judge_outputs: 10 # judge cases: outputs per batch, each scored +# # by its median 3-sample panel (MEETS 9 of 10) # ship_measure_ask_per_trial_usd: 2 # ask first above this estimated cost per trial # ship_measure_budget_usd: 25 # estimated admission budget per red case -# # (baseline, repair rounds and judge scoring) +# # (baseline, extension, redispatch, repair rounds) # ship_measure_max_rounds: 3 # repair rounds before a named-red stop +# ship_measure_sweep_cases: 5 # weekly off-ship sweep: cases measured (K) +# ship_measure_sweep_budget_usd: 150 # weekly off-ship sweep: spend cap per 7 days # ship_rerun_backend: local # local | ubicloud — where free-suite flake reruns # # run; ubicloud also needs UBICLOUD_API_KEY. # # Invalid values are REJECTED (value kept). @@ -271,6 +275,8 @@ lookup_default() { ship_measure_ask_per_trial_usd) echo "2" ;; ship_measure_budget_usd) echo "25" ;; ship_measure_max_rounds) echo "3" ;; + ship_measure_sweep_cases) echo "5" ;; + ship_measure_sweep_budget_usd) echo "150" ;; ship_rerun_backend) echo "local" ;; # local | ubicloud (Ubicloud spend is opt-in) # Unknown key: exit non-zero instead of printing "". The fallback pattern # the preambles use, @@ -675,12 +681,12 @@ case "${1:-}" in # The measure-loop controls bound paid spend: reject a typo instead of # coercing it, so a bad value never changes how much /ship may spend. case "$KEY" in - ship_measure_rule_trials|ship_measure_behavior_panels|ship_measure_judge_outputs|ship_measure_max_rounds) + ship_measure_rule_trials|ship_measure_behavior_panels|ship_measure_judge_outputs|ship_measure_max_rounds|ship_measure_sweep_cases) if ! printf '%s' "$VALUE" | LC_ALL=C grep -qE '^[1-9][0-9]*$'; then echo "Error: $KEY '$VALUE' is not a positive integer. Existing value left unchanged." >&2 exit 1 fi ;; - ship_measure_ask_per_trial_usd|ship_measure_budget_usd) + ship_measure_ask_per_trial_usd|ship_measure_budget_usd|ship_measure_sweep_budget_usd) if ! printf '%s' "$VALUE" | LC_ALL=C grep -qE '^([0-9]+(\.[0-9]+)?|\.[0-9]+)$' || ! printf '%s' "$VALUE" | LC_ALL=C grep -qE '[1-9]'; then echo "Error: $KEY '$VALUE' is not a positive amount in USD (e.g. 25 or 2.50). Existing value left unchanged." >&2 exit 1 @@ -876,7 +882,7 @@ case "${1:-}" in timeline_stop_hook design_detector design_detector_install_prompted memorable_recall \ browse_extension_id ship_measure_rule_trials ship_measure_behavior_panels \ ship_measure_judge_outputs ship_measure_ask_per_trial_usd ship_measure_budget_usd \ - ship_measure_max_rounds ship_rerun_backend; do + ship_measure_max_rounds ship_measure_sweep_cases ship_measure_sweep_budget_usd ship_rerun_backend; do gstack_config_select "$KEY" VALUE="$_gstack_cfg_value" SOURCE="default" @@ -899,7 +905,7 @@ case "${1:-}" in timeline_stop_hook design_detector design_detector_install_prompted memorable_recall \ browse_extension_id ship_measure_rule_trials ship_measure_behavior_panels \ ship_measure_judge_outputs ship_measure_ask_per_trial_usd ship_measure_budget_usd \ - ship_measure_max_rounds ship_rerun_backend; do + ship_measure_max_rounds ship_measure_sweep_cases ship_measure_sweep_budget_usd ship_rerun_backend; do printf ' %-24s %s\n' "$KEY:" "$(lookup_default "$KEY")" done ;; diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index 1480d847b..5c6f96cb9 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -479,25 +479,62 @@ documented single-case command (`--command ' {case}'`, split on whitespace, no shell). Counts per kind come from gstack config (`bin/gstack-config`), so a measurement -is pre-registered before it runs: +is pre-registered before it runs. Each measurement is decided by the +measurement bar (`scripts/lib/measure-bar.ts`, approved 2026-10-06), applied in +this order: -| Kind | Trials (config key, default) | Meets target when | -|---|---|---| -| `rule` | `ship_measure_rule_trials`, 10 | at least 90% of trials pass (9 of 10) | -| `behavior` | `ship_measure_behavior_panels`, 4 panels of 3 (12 trials) | every panel passes at its own 2 of 3 with no contract violation, and at least 90% of trials pass (11 of 12) | -| `judge` | `ship_measure_judge_outputs`, 10 outputs | at least 90% of outputs pass (9 of 10); an output passes when 2 of its 3 judge samples pass | +| Kind | Trials per batch (config key, default) | MEETS | MEETS-qualified | EXTEND, then the pooled decision | +|---|---|---|---|---| +| `rule` | `ship_measure_rule_trials`, 10 | at least 9 of 10 | 8 of 10, every red qualifying | 7 of 10, or 8 that does not qualify: 10 more trials, then 18 of 20 strict or 16 of 20 qualified | +| `behavior` | `ship_measure_behavior_panels`, 4 panels of 3 (12 trials) | at least 11 of 12 | 10 of 12, every red qualifying | 9 of 12, or 10 that does not qualify: 12 more trials, then 22 of 24 strict or 20 of 24 qualified | +| `judge` | `ship_measure_judge_outputs`, 10 outputs | at least 9 of 10 outputs | 8 of 10, every red qualifying | 7 of 10, or 8 that does not qualify: 10 more outputs, then 18 of 20 or 16 of 20 qualified | + +A judge output passes when the median of its 3 samples passes (EVAL_POLICY v3, +so at least 2 of 3). Everything below the EXTEND row is **BELOW**: a fix round. + +- **Qualifying reds** (the only reds MEETS-qualified accepts): `provider`, which + is mechanical: the trial's own `failure_cause` is `api_error`, or + `pre_turn_infra` whose evidence names a 429 or 5xx, a rate limit, an overload + or a connection reset (`ship-measure classify` refuses `provider` otherwise); + `judge-noise`, a judge sample at the threshold; and `model-miss`, which counts + only with a `path:line` citation of the deviation. Never qualifying: a timeout + or hang (`session_timeout`, `observer_timeout` and `provider_stall` are + classified so without review and cannot be reclassified into a qualifying + class), a regression, a contract violation (BELOW at any count), or any red + with a known fixable cause (`fixable`): each goes to a fix round. +- **needs-classify.** A batch below the strict bar but at or above the EXTEND + floor, with a failed trial nobody classified, returns needs-classify (exit 5) + with the trials to classify. `ship-measure classify --trials 2,5 --class + --evidence ` appends one immutable record per trial to the round's + `classifications.jsonl` (a later record supersedes an earlier one) and + re-decides. `unclassified` records "reviewed, no route"; it allows EXTEND but + never qualifies. +- **EXTEND** (exit 6). `ship-measure extend` runs N more trials only when the + round decided EXTEND and only on identical inputs: the measurement's identity + (the working tree's frozen git tree, kind, trials, runner, Claude CLI, model, + CI image `GSTACK_CI_IMAGE`, Bun, EVAL_POLICY and HARNESS_VERSION) must match + field for field, and the refusal names the field that changed. The pooled 2N + is decided once and is never extended again: there is no third batch. +- **Void batches.** A batch where more than 30% of trials fail with affirmative + provider or transport evidence is void (an outage, not a measurement); the + runner redispatches it once, both batches are reported, and a second void + batch stops with a named red. A crash or module-load failure without provider + evidence never voids a batch. `ship_measure_budget_usd` (default 25) is an estimated admission budget per red -case across the baseline, every repair round and judge scoring: before each +case across the baseline, an extension or redispatch, every repair round and +judge scoring: before each batch the runner reserves the estimated cost of every concurrent trial and admits only what fits beside what was already spent, then reconciles actual costs after the batch. When the next trial no longer fits it stops with a named red. A case with no per-trial estimate, or one above `ship_measure_ask_per_trial_usd` (default 2), asks once before any trial runs; with no estimate it then runs one calibration trial alone. `ship_measure_max_rounds` -(default 3) limits repair rounds. `ship-measure report` prints the PR-body table: -each measurement as "observed k/n" (after the named fix for a repair round), -target, estimated and actual spend. A red the user declares infrastructure is +(default 3) limits repair rounds; an extension or a redispatch is not a round. +`ship-measure report` prints the PR-body table: each measurement as "observed +k/n" (after the named fix for a repair round, with every batch), the bar, the +decision, estimated and actual spend, and qualified MEETS under their own +heading. A red the user declares infrastructure is recorded with `ship-measure skip` and shows as `unmeasured`, never as a pass. A case is called fixed only with a named causal change and every failure in the closing measurement explained; the weekly census, not the loop, reports @@ -512,7 +549,9 @@ free random ports, as in the normal suite. `ship_rerun_backend=ubicloud` runs th reruns on one Ubicloud VM through `bun run test:ubicloud --diagnostic`, which also sets the flaky retry off. The seeded paid case `ship-measure-seeded-flake` (periodic, `behavior`) drives /ship through this loop against a stub eval -command that fails trial slots 2, 5 and 9 until its list is sorted. +command that fails trial slots 2, 5 and 9 until its list is sorted. The weekly +[off-ship sweep](#ship-measure-sweep) measures main's flakiest gate cases with +the same runner and bar. **Failure causes and details** (diagnostic only; verdicts unchanged under v1). @@ -770,6 +809,45 @@ against a temp `GSTACK_INSTALL_DIR` / `GSTACK_SKILLS_DIR`, and `freeze/bin/check-freeze.sh` with JSON payloads on stdin (including the `GSTACK_HOME` state-root parity against `bin/gstack-paths`). +### Ship measure sweep + +**Weekly off-ship qualification sweep** (`ship-measure sweep`, +`scripts/ship-measure-sweep.ts`, `.github/workflows/eval-sweep.yml`; approved +2026-10-06). With about 130 gate verdicts, an all-green gate is unlikely even +when every case is healthy (26.8% at 99% per verdict), so flaky cases are +measured on main before any ship hits them. The sweep reads `eval:pass-rates` +history (`evals-periodic.yml` on main plus pooled branch census runs), takes the +latest gate census, and ranks every live, non-quarantined gate case with a +failed trial or red verdict by its cost to the all-green probability +Π(1 − p_i) (the shrunk red-verdict rate from `--reds`; ties go to the lower +per-trial pass rate). It measures the top K (`ship_measure_sweep_cases`, +default 5; `--k`) at their kind's N on the checked-out head with /ship's runner +and bar, in rank order. The report (`sweep-report.md` and `.json` under +`.context/ship-measure/sweep-