From 81b6469d9b9d424cb2984580b4bb77563f95deb8 Mon Sep 17 00:00:00 2001 From: garrytan Date: Mon, 19 Oct 2026 10:41:05 +0000 Subject: [PATCH] chore(evals): T1b CHANGELOG under [Unreleased]; classify evals-main-backstop.yml as free-lane only Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 15 +++++++++++++++ scripts/harness-version.json | 2 +- scripts/test-pr-profile.ts | 1 + 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c8a1988a..88dabde7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## [Unreleased] + +**The evals workflow carries merge trains: a train's own gate publishes its record and the `gstack/full-gate` check, bisects run one executor per prefix, `ready --screen` dispatches a real screen, and a main backstop gates any tree no green train covered.** All of it stays off unless the base revision's `.gstack/ship-policy.json` declares a `train` block. Without one, a PR push and every eval-bg dispatch run today's jobs and steps with today's values. The one addition is a free `route` job that reads the policy with git and jq. gstack's own policy still has no `train` block; turning it on is a separate one-line policy PR. Plan GSTA-49, lane T1b. + +### Added +- **Train gate runs** (`train/` with a train nonce): the full census runs without verdict reuse or reuse binding. The report job writes the train record (`scripts/train-gate.ts train-record`) from this run's plan manifest, one panel per planned case, plus the prefix trees modulo stamps. It uploads the record as the policy's `record_artifact`. A separate `train-check` job, with `checks: write` and no repository code, publishes `full_gate_check` as success only for a complete GREEN census whose nonce and head match the dispatch. A train nonce dispatched on main is refused before any paid work. +- **Bisect inputs `cases` and `prefix_shas`.** Case ids must be in the run's plan, and each prefix must be an ancestor of the train head on `train/`, or the run refuses. A `train-bisect` matrix runs one executor per prefix, checks out and verifies that prefix, and uploads `-bisect-`, the name `train bisect` reads. +- **Dispatched screens** (`screen_pr`, `screen_merge_sha`, `screen_policy`): the run checks out and verifies the pinned merge commit. It waits on the PR head's Free Tests, cancels an older screen of the same PR, and comments on that PR. Verdict reuse (R2a/R2) authenticates a screen by PR number, head SHA, repository and tested base (the merge commit's first parent). It keeps every selection and newer-run veto, and lists pull_request and screen runs filtered by PR. A broad screen never adopts a narrow run. +- **`.github/workflows/evals-main-backstop.yml`** runs on every push to main, one at a time. It does nothing without a `train` block. With one, it waits until no train is merging. It runs the full gate on main only when no green train record lists main's tree modulo stamps, and writes the freeze record `gstack-ship-queue` reads. A five-PR train fires zero backstop gates. +- **The free screening summary:** under a `train` block, a PR push runs readiness and prints the screen selection and an estimated cost instead of starting paid slices. + +### Changed +- **Gate integrity (train block only):** under a `train` block, PR pushes no longer start paid slices; trains, screens and the backstop gate instead. Train and bisect runs never reuse a verdict and never bind one for reuse. No threshold, trial count, case kind or selection changed. +- **Gate integrity (always, GSTA-48 P3-1):** when the readiness job blocks a PR head, a new `readiness-comment` job replaces an existing `## E2E Evals` comment with "NOT RUN on ", so an earlier PASS no longer stands beside a head that never ran. It posts nothing when no such comment exists. A slice killed before its upload step counts as incomplete. + ## [1.91.146.0] - 2026-10-19 **`gstack-ship-queue` can run merge trains: several READY PRs share one full gate, merge one by one only while main's tree matches the prefix that gate tested, and a red train is bisected on its red cases alone.** The surface is off unless the base revision's `.gstack/ship-policy.json` declares a `train` block. Without one, `ready --pr ` prints exactly the block it prints today. gstack's own policy does not turn trains on yet: that waits for the workflow side (the train-nonce record, the `cases` and `prefix_shas` inputs and the dispatched screen) and one dry-run train. Walkthrough: [docs/ship-policy.md#run-a-merge-train](docs/ship-policy.md#run-a-merge-train). Plan GSTA-49, lane T1. diff --git a/scripts/harness-version.json b/scripts/harness-version.json index 50da17512..c104368a1 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -16,7 +16,7 @@ "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6}, "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":6}, "scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":6}, - "scripts/test-pr-profile.ts": {"blob":"2029a89e14f511866cc91bebfee00b4fe9e7cf63","since":6}, + "scripts/test-pr-profile.ts": {"blob":"c03f3298a51ac04a182dc6a3bbaf286644ab59fd","since":6,"nonBehavioral":"classify the new evals-main-backstop.yml workflow as free-lane only: it dispatches evals.yml on main and runs inside no case, so no case's selection on a PR, input or verdict changes"}, "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6}, "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6}, "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6}, diff --git a/scripts/test-pr-profile.ts b/scripts/test-pr-profile.ts index 2029a89e1..c03f3298a 100644 --- a/scripts/test-pr-profile.ts +++ b/scripts/test-pr-profile.ts @@ -273,6 +273,7 @@ export const FREE_ONLY_PR_FILES = [ '.github/workflows/skill-docs.yml', '.github/workflows/test-health.yml', '.github/workflows/eval-sweep.yml', // The weekly off-ship sweep on main; no PR-gate case reads it, and it never gates a PR. + '.github/workflows/evals-main-backstop.yml', // The merge-train backstop on main: it dispatches evals.yml after a push to main; no PR-gate case reads it, and it never gates a PR. '.github/actionlint.yaml', // Runner labels for actionlint only. 'scripts/ship-measure-sweep.ts', // The weekly sweep CLI; it launches paid cases, never runs inside one. '.github/workflows/version-gate.yml',