From 210f998c10c4c70cff60b0067f7cd307134ea8d7 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 18 Oct 2026 18:02:14 +0000 Subject: [PATCH] feat(ship-queue): merge trains with pattern bisect and train-record verify (GSTA-49 T1) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- bin/gstack-doctor-components.sh | 18 + bin/gstack-ship-queue | 28 +- docs/ship-policy.md | 185 +++++++- docs/troubleshooting.md | 192 +++++++++ lib/result-codes-release.ts | 91 ++++ lib/ship-policy.ts | 49 +++ lib/ship-queue-gate.ts | 2 +- lib/ship-queue-train-cli.ts | 108 +++++ lib/ship-queue-train-github.ts | 83 ++++ lib/ship-queue-train-ready.ts | 128 ++++++ lib/ship-queue-train-record.ts | 207 +++++++++ lib/ship-queue-train-verdict.ts | 332 ++++++++++++++ lib/ship-queue-train-verify.ts | 122 ++++++ lib/ship-queue-train.ts | 509 ++++++++++++++++++++++ scripts/eval-bg.ts | 101 +++-- test/fixtures/module-size-ratchet.json | 9 +- test/helpers/ship-queue-train-fixture.ts | 158 +++++++ test/ship-policy-compat.test.ts | 4 +- test/ship-queue-train-logic.test.ts | 205 +++++++++ test/ship-queue-train.test.ts | 524 +++++++++++++++++++++++ 20 files changed, 3021 insertions(+), 34 deletions(-) create mode 100644 lib/ship-queue-train-cli.ts create mode 100644 lib/ship-queue-train-github.ts create mode 100644 lib/ship-queue-train-ready.ts create mode 100644 lib/ship-queue-train-record.ts create mode 100644 lib/ship-queue-train-verdict.ts create mode 100644 lib/ship-queue-train-verify.ts create mode 100644 lib/ship-queue-train.ts create mode 100644 test/helpers/ship-queue-train-fixture.ts create mode 100644 test/ship-queue-train-logic.test.ts create mode 100644 test/ship-queue-train.test.ts diff --git a/bin/gstack-doctor-components.sh b/bin/gstack-doctor-components.sh index eea90fd19..351539bb8 100644 --- a/bin/gstack-doctor-components.sh +++ b/bin/gstack-doctor-components.sh @@ -260,6 +260,24 @@ gstack_result_anchor() { QUEUE_VERIFY_MISMATCH) printf '%s' 'queue-verify-mismatch' ;; QUEUE_GATE_MISSING) printf '%s' 'queue-gate-missing' ;; GATE_AHEAD_ATTEMPT_INVALID) printf '%s' 'gate-ahead-attempt-invalid' ;; + QUEUE_TRAIN_OFF) printf '%s' 'queue-train-off' ;; + QUEUE_TRAIN_UNTRUSTED) printf '%s' 'queue-train-untrusted' ;; + QUEUE_TRAIN_MOVED) printf '%s' 'queue-train-moved' ;; + QUEUE_TRAIN_CONFLICT) printf '%s' 'queue-train-conflict' ;; + QUEUE_TRAIN_FREE_RED) printf '%s' 'queue-train-free-red' ;; + QUEUE_TRAIN_DISPATCH_UNKNOWN) printf '%s' 'queue-train-dispatch-unknown' ;; + QUEUE_TRAIN_WAITING) printf '%s' 'queue-train-waiting' ;; + QUEUE_TRAIN_PENDING) printf '%s' 'queue-train-pending' ;; + QUEUE_TRAIN_INCOMPLETE) printf '%s' 'queue-train-incomplete' ;; + QUEUE_TRAIN_RED) printf '%s' 'queue-train-red' ;; + QUEUE_TRAIN_RED_ON_MAIN) printf '%s' 'queue-train-red-on-main' ;; + QUEUE_TRAIN_UNATTRIBUTED) printf '%s' 'queue-train-unattributed' ;; + QUEUE_TRAIN_BISECT_CAPPED) printf '%s' 'queue-train-bisect-capped' ;; + QUEUE_TRAIN_PREFIX_INVALID) printf '%s' 'queue-train-prefix-invalid' ;; + QUEUE_TRAIN_REBUILD_REFUSED) printf '%s' 'queue-train-rebuild-refused' ;; + QUEUE_TRAIN_TREE_MISMATCH) printf '%s' 'queue-train-tree-mismatch' ;; + QUEUE_TRAIN_FROZEN) printf '%s' 'queue-train-frozen' ;; + READY_SCREEN_CHOICE_REQUIRED) printf '%s' 'ready-screen-choice-required' ;; PREGATE_TIMING_ASSERTION) printf '%s' 'pregate-timing-assertion' ;; APPROVAL_CLASSES_INVALID) printf '%s' 'approval-classes-invalid' ;; APPROVAL_CLASS_FOUND) printf '%s' 'approval-class-found' ;; diff --git a/bin/gstack-ship-queue b/bin/gstack-ship-queue index 6135827bb..f5b1b9304 100755 --- a/bin/gstack-ship-queue +++ b/bin/gstack-ship-queue @@ -11,11 +11,15 @@ * gstack-ship-queue gate-ahead --after --pr [--predecessor-ref ] [--allow-repo-commands] * gstack-ship-queue gate-ahead --record --pr --attempt --exit --log * gstack-ship-queue gate-ahead --list --pr + * gstack-ship-queue train build|status|verdict|bisect|rebuild|prune ... (merge trains; train --help) * * Every subcommand takes --base (default origin/HEAD, else main) and * ends with GSTACK_RESULT: skill=ship-queue status=complete|refused run=. - * gstack-restamp stays the primitive. Nothing here pushes. Logic: - * lib/ship-queue.ts, lib/ship-queue-gate.ts. + * gstack-restamp stays the primitive. Nothing here pushes, except under a + * ship policy with a `train` block: merge trains push train/ branches + * and the refs/notes/gstack-trains records, and dispatch through eval-bg. + * Logic: lib/ship-queue.ts, lib/ship-queue-gate.ts, lib/ship-queue-train*.ts + * (loaded only on the train paths). */ import { spawnSync } from 'node:child_process'; import * as path from 'node:path'; @@ -41,6 +45,9 @@ const HELP = `gstack-ship-queue — the shared merge queue's protocol steps (doc gate-ahead --record --pr --attempt --exit --log bind the gate result to that attempt's gated tree; removes its worktree gate-ahead --list --pr attempts for a PR, newest last (the current one) + train build|status|verdict|bisect|rebuild|prune ... + merge trains, when the base policy declares a train block (train --help; + docs/ship-policy.md#run-a-merge-train); ready then takes --screen|--no-screen --base : base branch (default: origin/HEAD, else main). Nothing here pushes. ${exitTableHelp()}`; @@ -122,7 +129,7 @@ function gateAhead(ctx: QueueContext, args: string[]): number { }); } -function main(argv: string[]): number { +async function main(argv: string[]): Promise { const [cmd, ...args] = argv; if (!cmd || cmd === '--help' || cmd === '-h' || cmd === 'help') { process.stdout.write(HELP + '\n'); return cmd ? EXIT.ok : EXIT.usage; } const probe = spawnSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8', timeout: 15_000 }); @@ -132,11 +139,21 @@ function main(argv: string[]): number { const ctx: QueueContext = { repoRoot, base: resolveBaseBranch(repoRoot, flag(args, '--base')), gh }; const pr = flag(args, '--pr'); if (cmd === 'gate-ahead') return gateAhead(ctx, args); + if (cmd === 'train') { + const train = await import('../lib/ship-queue-train-cli'); + if (!args[0] || args[0].startsWith('-')) { process.stdout.write(train.TRAIN_HELP + '\n'); return args.includes('--help') || args.includes('-h') ? EXIT.ok : EXIT.usage; } + return train.trainMain(ctx, args); + } if (!pr || !/^\d+$/.test(pr)) return usage(`${cmd} needs --pr `); const gateExit = flag(args, '--gate-exit'); if (gateExit !== undefined && !/^-?\d+$/.test(gateExit)) return usage('--gate-exit needs an exit code'); const gate = { gateExit: gateExit === undefined ? undefined : Number(gateExit), gateLog: flag(args, '--gate-log') }; - if (cmd === 'ready') return finish('ready', ready(ctx, { pr, pregate: flag(args, '--pregate'), ...gate })); + if (cmd === 'ready') { + const loaded = loadBasePolicy(ctx); + if (!('message' in loaded) && loaded.policy.train) return (await import('../lib/ship-queue-train-cli')).readyMain(ctx, args, { pr, pregate: flag(args, '--pregate'), ...gate }); + if (['--screen', '--no-screen', '--broad'].some(f => args.includes(f))) process.stderr.write('gstack-ship-queue ready: --screen/--no-screen/--broad ignored: the base policy declares no train block\n'); + return finish('ready', ready(ctx, { pr, pregate: flag(args, '--pregate'), ...gate })); + } if (cmd === 'stamp') { const version = flag(args, '--version'); const expectGated = flag(args, '--expect-gated'); @@ -149,6 +166,7 @@ function main(argv: string[]): number { })); } if (cmd === 'verify') { + if (flag(args, '--train')) return (await import('../lib/ship-queue-train-cli')).verifyMain(ctx, args, pr); const record = flag(args, '--record'); return finish('verify', verify(ctx, { pr, gated: flag(args, '--gated'), record: record ? path.resolve(record) : undefined, receipt: flag(args, '--receipt'), @@ -158,4 +176,4 @@ function main(argv: string[]): number { return usage(`unknown subcommand ${cmd}`); } -process.exitCode = main(process.argv.slice(2)); +process.exitCode = await main(process.argv.slice(2)); diff --git a/docs/ship-policy.md b/docs/ship-policy.md index b2219e50b..4a8df6e33 100644 --- a/docs/ship-policy.md +++ b/docs/ship-policy.md @@ -18,7 +18,7 @@ introduces or changes the policy runs under the reviewed bootstrap path Tools: `gstack-ship-policy init|validate|show-effective|check-history`, `gstack-restamp`, `gstack-tree-receipt`, `gstack-ship-receipt`, -`gstack-ship-queue` ([shared merge queue](#run-a-shared-merge-queue)), +`gstack-ship-queue` ([shared merge queue](#run-a-shared-merge-queue), [merge trains](#run-a-merge-train)), `gstack-evidence bundle|ancestor`. They share the exit table (`0` ok, `1` fail, `2` usage, `3` refused or needs a flag) and end every error line with a code from [troubleshooting.md](troubleshooting.md#queue-aware-ship-policy-restamp-tree-receipt-ship-receipt). @@ -58,6 +58,7 @@ refuses the file (`POLICY_GSTACK_TOO_OLD`). `validate` ends with | `gate_command` | command or `null` | `null` | The repo's full pre-merge gate. `gstack-ship-queue gate-ahead` and `ready` print it for the caller to run (in the background or in CI); gstack never executes it. Needs `min_gstack_version`. | | `full_e2e` | `true` \| `false` | `false` | `true`: every merge needs a passing full-gate result bound to a tree equal to the merging one modulo stamps; `gstack-ship-queue ready`, `stamp` and `verify` refuse without one (`QUEUE_GATE_MISSING`): a gate-ahead record, or the lane's `--gate-exit 0 --gate-log ` from its run on the gated head. Needs `min_gstack_version`. | | `min_gstack_version` | version or `null` | `null` | The oldest gstack that may read the file; see the compatibility rule above. | +| `train` | `{ gate_workflow, free_workflow, full_gate_check, record_artifact, max_size, fill_hours, screen_selection, trial_reservation_usd }` or `null` | `null` | Turns on [merge trains](#run-a-merge-train), read from the base revision only. Every field is required and an unknown field is `POLICY_INVALID`: `max_size` 1 to 6, `fill_hours` above 0 and at most 24, `screen_selection` `gstack-paid` \| `none`, `trial_reservation_usd` above 0 and at most 100. `null` keeps the per-PR queue exactly as it is. Needs `min_gstack_version`. | Containment, checked by `validate` and again at write time: paths and globs are repo-relative with no `..` segment, under 256 characters, at most 64 per list, @@ -254,6 +255,188 @@ prints) only after every lane's gstack is at least that version, the crew's install floor (`GSTACK_MIN_REVISION`). A lane on an older gstack refuses the policy by name rather than reading it differently. + +## Run a merge train + +A merge train gates several READY PRs with one full gate instead of one gate +each. The coordinator merges the pinned, unstamped PR heads in slot order onto +main in a scratch worktree, records each prefix tree (T0 is main, Tk is the +train head), runs the free tests and then one full gate on the train head, and +merges the PRs one at a time, each only while main's tree equals the prefix +the gate covered modulo stamps. A red train is bisected on its red cases only. + +The surface is off unless the base revision's `.gstack/ship-policy.json` +declares a `train` block. Without one, `train …` and `verify --train` refuse +with `QUEUE_TRAIN_OFF`, and `ready --pr ` prints exactly the per-PR READY +block above (`--screen`, `--no-screen` and `--broad` are ignored with a +warning on stderr). A block for gstack itself would read: + +```json +"train": { + "gate_workflow": "evals.yml", "free_workflow": "free-tests.yml", + "full_gate_check": "gstack/full-gate", "record_artifact": "gstack-train-record", + "max_size": 5, "fill_hours": 3, "screen_selection": "gstack-paid", + "trial_reservation_usd": 1.5 +}, +"min_gstack_version": "" +``` + +gstack's own policy does not carry it yet: trains go live for gstack only +after the workflow side (the train-nonce record, the `cases` and +`prefix_shas` inputs, the dispatched screen) has merged and one dry-run train +of two already-merged heads has passed. `gate_workflow` is dispatched with +`evals_all`, `expected_sha` and `nonce` (bisects add `cases` and +`prefix_shas`, screens add `screen_pr`, `screen_merge_sha` and +`screen_policy`). That run publishes `full_gate_check` as a check run on the +train head and the train record as the `record_artifact` artifact only when +its census is complete and green; gstack reads both and never writes a +check. `screen_selection: gstack-paid` computes gstack's paid case selection +for the screen recommendation; `none` leaves the choice to the lane. +`trial_reservation_usd` is what the bisect reserves for a trial without cost +data. + +Every command prints its lines, the coded error line on stderr, then +`NEXT: ` and `GSTACK_RESULT: skill=ship-queue status=… run=`. +Records live under `$GSTACK_STATE_ROOT/ship-queue/` (`trains//train.json` +with one attempt record per dispatch, `ready/` receipts and `freeze/`) and +are mirrored to origin as git notes on `refs/notes/gstack-trains` before any +remote effect, so a lost machine loses nothing: `train status` on a clean +state root rebuilds them. `GSTACK_SHIP_QUEUE_NOW` pins the clock for tests. + +**1. Lane: READY with a screen choice.** Under a train policy READY needs an +explicit choice: + +```bash +gstack-ship-queue ready --pr 42 # prints SCREEN: lines and the recommendation, refuses +gstack-ship-queue ready --pr 42 --no-screen # READY receipt; the PR rides unscreened +gstack-ship-queue ready --pr 42 --screen [--broad] # dispatches one paid screen on the PR's merge SHA +``` + +Without a choice it prints the `SCREEN:` selection (with +`screen_selection: gstack-paid`, the skill's own cases and `recommend +--screen` or `--no-screen`: `--screen` when the screen selects a skill's own +cases) and refuses with `READY_SCREEN_CHOICE_REQUIRED`. The price line says +the dollar figure is pending the cost replay; until it lands, `SCREEN:` shows +case counts and the recommendation only. A lane that already ran focused +local cases says so in the READY block and may pass `--no-screen`. `--screen` +dispatches `gate_workflow` with `screen_pr`, `screen_merge_sha` (the +`refs/pull//merge` SHA) and `screen_policy` (`narrow`, or `broad` with +`--broad`, today's full PR selection). The nonce key carries the purpose and +the policy, so a repeat follows the same run and a broad screen never adopts +a narrow one. The READY block gains `screen: …`, and the receipt is written +locally and to the notes ref. `--dry-run` prints the dispatch without sending +it. + +**2. Coordinator: build.** Name the pins exactly as READY reported them: + +```bash +gstack-ship-queue train build --prs 41@,42@,43@ --dry-run +gstack-ship-queue train build --prs 41@,42@,43@ +``` + +It prints `ORDER:` (the slot order to announce: screened PRs first in READY +order, then unscreened; when `max_size` or more unscreened PRs wait they form +their own train and `HELD:` lists them), one `PREFIX: T +tree=` per prefix and one `SLOT:` per PR. `--dry-run` adds the +`ESTIMATE:` line and writes, pushes and dispatches nothing. Otherwise it +writes `train.json`, pushes `train/`, dispatches `free_workflow` there +and waits (`--free-timeout `, default 2700) for a green +`workflow_dispatch` run on that exact SHA, then dispatches the full gate +through eval-bg's nonce. A train under `max_size` waits until its oldest READY +has waited `fill_hours` (`QUEUE_TRAIN_WAITING`); `--urgent` dispatches a +one-PR train at once. The same pins on the same base return the live train +instead of a second one; `train build --train ` resumes a train from +its record. Refusals, all before any spend: `QUEUE_TRAIN_UNTRUSTED` (a fork +head), `QUEUE_TRAIN_MOVED` (a head moved from its pin), `QUEUE_NOT_READY` (no +READY receipt at that head), `QUEUE_TRAIN_CONFLICT` (the printed +`QUEUE_TRAIN_CONFLICT: train stopped at # (conflicts: ) after +merging #,#; nothing dispatched, no branch pushed; fix: …` line names +the build without that PR) and `QUEUE_TRAIN_FREE_RED` (free tests red, +missing, cancelled or on another SHA). In a merge, version files take the +prefix's side and the CHANGELOG merges by section, since the stamp at merge +rewrites both. + +**3. Coordinator: status and verdict.** + +```bash +gstack-ship-queue train status [--train ] +gstack-ship-queue train verdict --train +``` + +`status` never spends. When a dispatch was attempted but its run never showed +up (`QUEUE_TRAIN_DISPATCH_UNKNOWN`), it lists the workflow's dispatch runs and +either adopts the run carrying the nonce or marks the dispatch `absent`; only +an absent dispatch is sent again, by `train build --train `, under the +same nonce. `verdict` reads the run, its record and the `full_gate_check` +check: `GREEN` only for a complete census bound to the train nonce, head and +tree; `QUEUE_TRAIN_PENDING` while it runs; `QUEUE_TRAIN_INCOMPLETE` for INFRA +or INCOMPLETE panels, unbound, partial or cancelled shards, a slice killed +before upload, or no record (never green); `QUEUE_TRAIN_RED` with the red +cases. + +**4. A red train: bisect, then at most one rebuild.** + +```bash +gstack-ship-queue train bisect --train --spend-cap 5 --dry-run +gstack-ship-queue train bisect --train --spend-cap 5 +gstack-ship-queue train rebuild --train +``` + +The bisect dispatches the red cases (`cases`) on every prefix T0..Tk +(`prefix_shas`), one trial each, after checking every prefix is an ancestor +of `train/` (`QUEUE_TRAIN_PREFIX_INVALID`). It reserves every +case×prefix trial (unknown costs at `trial_reservation_usd`) and launches only +the prefixes that fit the cap (`QUEUE_TRAIN_BISECT_CAPPED`; completed prefixes +are kept and a rerun with a higher cap runs only the rest). Rerun it to read +the results. The pattern decides: `ATTRIBUTED #` only when monotone +(green before, red from that PR on), and `NEXT:` builds the train without it; +red on every prefix with T0 included is `QUEUE_TRAIN_RED_ON_MAIN` (fix forward +on main first); anything else is `QUEUE_TRAIN_UNATTRIBUTED`. An all-green +bisect earns exactly one unchanged rebuild (`train rebuild`, reporting both +runs); a non-monotone pattern earns none and goes to a repair lane. Every +other rebuild is `QUEUE_TRAIN_REBUILD_REFUSED`, and a second red stays red. +An INFRA/INCOMPLETE-only census earns the single re-dispatch +EVAL_POLICY.infraRedispatch allows: `train rebuild --train --infra`. +The rebuild-once rule lives in the train module beside that constant until +EVAL_POLICY records it. Bisect results never turn a train green. + +**5. Coordinator: verify, merge, verify again.** For each slot in order: + +```bash +gstack-ship-queue verify --pr 41 --train # MERGE_CONDITION: sha= +# merge #41 with that expected head; gstack-restamp stamps it at merge +gstack-ship-queue verify --pr 41 --train # VERIFY_TRAIN: #41 merged; main's tree is T1 … +``` + +Before a merge, main's tree must equal the prefix before the PR modulo stamps +and the PR must still be at its pin (stamps aside); after it, main's tree +must equal the prefix after it. Anything else is `QUEUE_TRAIN_TREE_MISMATCH` +(or `QUEUE_TRAIN_MOVED`). When nothing of the train has merged, that is a plain +refusal: build a new train on current main. When earlier slots already +merged, main holds a prefix no gate covered on its own, so verify freezes the +queue: the freeze goes to local state and the notes ref first, then the full +gate is dispatched on main's SHA at once (the backstop), and the +`QUEUE_TRAIN_FROZEN:` line lists the merged PRs, main's tree and the backstop +run. While frozen, ordinary trains and other trains' merges are refused +(`QUEUE_TRAIN_FROZEN`). Admit one repair PR at its exact head: + +```bash +gstack-ship-queue train build --repair --prs 44@ +``` + +It runs the normal free tests, full gate and verification on main plus the +repair. When verify confirms the repair merged, it prints `FREEZE: cleared` +and `NEXT:` builds a train of the PRs still unmerged. + +**6. Clean up.** `gstack-ship-queue train prune [--train ] [--dry-run]` +deletes the `train/` branches of merged, superseded and dropped trains +(`WOULD-DELETE:`, `DELETED:`, `GONE:`). + +**Escape hatches.** Remove the `train` block on the base branch to return +every lane to the per-PR queue at once. `--urgent` ships one PR alone, +`--no-screen` skips the paid screen, `--broad` widens it to today's selection, +and `train build --prs ` drops a PR at any step. + ## What `/ship` reads from it diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index e2719bca0..15094f27e 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -3684,6 +3684,198 @@ and the current one. Record the live attempt, or prepare a new one with `gate-ahead --after ` and run the gate there without editing tracked files. + +### `QUEUE_TRAIN_OFF` + +**Meaning.** A merge-train command (`train …` or `verify --train`) ran +against a base revision whose `.gstack/ship-policy.json` declares no `train` +block, so the train surface is off and the per-PR queue applies. Nothing was +written. + +**Fix.** Use the per-PR queue (`ready --pr `, `stamp`, `verify`), or land +a `train` block with `min_gstack_version` on the base branch first +([Run a merge train](ship-policy.md#run-a-merge-train)). + + +### `QUEUE_TRAIN_UNTRUSTED` + +**Meaning.** A pinned PR's head repository is not the base repository. The +train branch and its dispatches run with repository secrets, so fork work +never rides a train. + +**Fix.** Run the printed `train build --prs `; the fork PR +stays on the trusted-branch path. + + +### `QUEUE_TRAIN_MOVED` + +**Meaning.** A PR's head is no longer the SHA the train pinned (at build, or +past its pin beyond stamps at verify), so the train would not test what +merges. At verify after earlier slots merged, the queue also freezes +(`QUEUE_TRAIN_FROZEN`). + +**Fix.** Ask the lane for a fresh READY on the new head, then build the train +with the new pin (`train build --prs @,…`). + + +### `QUEUE_TRAIN_CONFLICT` + +**Meaning.** Merging a PR onto the prefix before it conflicts outside the +version files and the CHANGELOG. The `QUEUE_TRAIN_CONFLICT:` line names the +PR, the conflicting files and the PRs merged before it. Nothing was +dispatched and no branch was pushed. + +**Fix.** Run the printed `train build --prs `. The PR's +lane merges main (with the earlier PRs once they land) and reports READY +again. + + +### `QUEUE_TRAIN_FREE_RED` + +**Meaning.** The free tests dispatched on `train/` are red, cancelled, +missing (no `workflow_dispatch` run appeared before `--free-timeout`) or ran +on another SHA, so the paid gate was not dispatched. + +**Fix.** Read the free-tests run. A PR-caused red drops that PR (`train build +--prs `); a flaky test goes to its owner before the next +build. + + +### `QUEUE_TRAIN_DISPATCH_UNKNOWN` + +**Meaning.** A train dispatch was attempted, but no run carrying its nonce +appeared, so whether it started is unknown (`gh workflow run` can fail after +GitHub accepted the dispatch). It is never dispatched again blindly. + +**Fix.** Run `gstack-ship-queue train status --train `. It adopts the +run carrying the nonce, or marks the dispatch absent; only then does +`train build --train ` dispatch it again, under the same nonce. + + +### `QUEUE_TRAIN_WAITING` + +**Meaning.** The train has fewer PRs than `train.max_size` and its oldest +READY PR has waited less than `train.fill_hours`, so it does not dispatch +yet. The message prints when it will. + +**Fix.** Add READY PRs, rerun after the printed time, or pass `--urgent` for a +one-PR train. + + +### `QUEUE_TRAIN_PENDING` + +**Meaning.** The train's run (free tests, the full gate or the bisect) has +not finished, so there is no verdict yet. + +**Fix.** Rerun the printed `NEXT:` command later. Nothing is dispatched again +while a run is pending. + + +### `QUEUE_TRAIN_INCOMPLETE` + +**Meaning.** The train gate did not complete its census: INFRA or INCOMPLETE +panels, an unbound, partial or cancelled shard, a slice killed before its +upload, a record bound to another nonce, head or tree, or no published +record. An incomplete gate is never green. + +**Fix.** When every gap is machine-classified INFRA or INCOMPLETE, take the +one re-dispatch EVAL_POLICY.infraRedispatch allows (`train rebuild --train + --infra`) and report both runs. Otherwise diagnose the run. + + +### `QUEUE_TRAIN_RED` + +**Meaning.** The train's full gate is red on the train tree; the message names +the red cases. Its PRs do not merge, and nothing is retried to change the +verdict. + +**Fix.** Bisect the red cases (`train bisect --train --spend-cap +`). After a rebuild's second red, route the cases to a repair lane. + + +### `QUEUE_TRAIN_RED_ON_MAIN` + +**Meaning.** The bisect found the red cases red on every prefix, main (T0) +included: main is broken, not the train's PRs. + +**Fix.** Fix forward on main first (a one-PR train), then rebuild this train +unchanged. + + +### `QUEUE_TRAIN_UNATTRIBUTED` + +**Meaning.** The bisect pattern blames no PR and not main: all green, or not +monotone. One trial per prefix cannot tell a flake from a break. + +**Fix.** All green: the one preregistered rebuild (`train rebuild --train +`), reporting both runs. Any other pattern: a repair lane, then a +changed train once the repair merges. + + +### `QUEUE_TRAIN_BISECT_CAPPED` + +**Meaning.** The bisect's reserved price (every case×prefix trial, trials +without cost data at `train.trial_reservation_usd`) passes `--spend-cap`. +Prefixes that fit were launched, completed prefixes are kept, and the +outcome stays unattributed; it is never green. + +**Fix.** Resume with a higher cap (`train bisect --train --spend-cap +`). Only prefixes without a result run. + + +### `QUEUE_TRAIN_PREFIX_INVALID` + +**Meaning.** A bisect prefix is not an ancestor of the train head on +`train/` (the branch moved), or a red case is not in the gate's plan, +so the bisect inputs are refused before any dispatch. + +**Fix.** Check that `train/` still points at the recorded head +(`train status`). A moved branch needs a new train. + + +### `QUEUE_TRAIN_REBUILD_REFUSED` + +**Meaning.** `train rebuild` is not allowed: there is no bisect record (or, +with `--infra`, no INFRA/INCOMPLETE-only census), the bisect was not all +green, or the train was already rebuilt or re-dispatched once. + +**Fix.** Route the red cases to a repair lane; the train is rebuilt only once +the repair merges (a changed train). + + +### `QUEUE_TRAIN_TREE_MISMATCH` + +**Meaning.** Main's tree is neither the train's prefix before this PR nor the +one after it, modulo stamps, so the merge would not publish what the gate +tested. + +**Fix.** Stop merging. If earlier PRs of the train merged, the queue is now +frozen (`QUEUE_TRAIN_FROZEN`); otherwise build a new train on current main. + + +### `QUEUE_TRAIN_FROZEN` + +**Meaning.** A train stopped partway through publication, so main holds a +prefix no gate covered on its own. The `QUEUE_TRAIN_FROZEN:` line lists the +merged PRs, main's tree and the backstop run dispatched on main's SHA. The +freeze lives in local state and on `refs/notes/gstack-trains`. Ordinary +trains and other trains' merges are refused until a repair lands. + +**Fix.** Admit one repair PR at its exact head (`train build --repair --prs +@`). The freeze clears when verify finds main's tree after that merge +in a green record; `NEXT:` then builds a train of the PRs still unmerged. + + +### `READY_SCREEN_CHOICE_REQUIRED` + +**Meaning.** The base policy turns trains on, so READY needs an explicit +screen choice: `--screen` (pay for a screen on the PR's merge SHA now) or +`--no-screen`. The `SCREEN:` lines show the selection and the recommendation. + +**Fix.** Rerun `gstack-ship-queue ready` with the printed recommendation. A +lane that already ran focused local cases says so in the READY block and may +pass `--no-screen`. + ### `PREGATE_TIMING_ASSERTION` diff --git a/lib/result-codes-release.ts b/lib/result-codes-release.ts index 42ff9b088..96ed5fed2 100644 --- a/lib/result-codes-release.ts +++ b/lib/result-codes-release.ts @@ -46,6 +46,97 @@ export const RELEASE_RESULT_CODES = { summary: 'the gate-ahead attempt is unknown, already recorded, or its worktree no longer holds the gated commit', fix: 'list attempts with gstack-ship-queue gate-ahead --list, then record the live one, or prepare a new attempt', }, + // GSTA-49 lane T1: merge trains in gstack-ship-queue (docs/ship-policy.md#run-a-merge-train). + QUEUE_TRAIN_OFF: { + anchor: 'queue-train-off', + summary: 'the base revision’s .gstack/ship-policy.json declares no `train` block, so the merge-train surface is off and the per-PR queue applies', + fix: 'use the per-PR queue (ready, stamp, verify), or land a `train` block with min_gstack_version on the base branch first', + }, + QUEUE_TRAIN_UNTRUSTED: { + anchor: 'queue-train-untrusted', + summary: 'a PR’s head repository is not the base repository; fork work never runs in a train (the train branch and its dispatch carry repository secrets)', + fix: 'leave the PR out of the train (gstack-ship-queue train build --prs ); fork work stays on the trusted-branch path', + }, + QUEUE_TRAIN_MOVED: { + anchor: 'queue-train-moved', + summary: 'a PR’s head (or the train branch) is no longer the SHA the train pinned, so the train would not test what merges', + fix: 'ask the lane for a fresh READY on the new head, then rebuild the train with the new pin (train build --prs @,...)', + }, + QUEUE_TRAIN_CONFLICT: { + anchor: 'queue-train-conflict', + summary: 'merging a PR onto the previous prefix conflicts outside the version files and the CHANGELOG; nothing was dispatched and no branch pushed', + fix: 'run the printed train build without that PR; the PR’s lane merges main (with the earlier PRs once they land) and reports READY again', + }, + QUEUE_TRAIN_FREE_RED: { + anchor: 'queue-train-free-red', + summary: 'the free tests dispatched on train/ are red, missing, cancelled or ran on another SHA, so the paid gate was not dispatched', + fix: 'read the free-tests run; a PR-caused red drops that PR (train build --prs ), a flaky one goes to its owner before the next build', + }, + QUEUE_TRAIN_DISPATCH_UNKNOWN: { + anchor: 'queue-train-dispatch-unknown', + summary: 'a train dispatch was attempted but its run could not be found by nonce, so whether it started is unknown; it is never re-dispatched blindly', + fix: 'run gstack-ship-queue train status --train : it finds the run by nonce, or marks the dispatch absent so train build --train may dispatch it', + }, + QUEUE_TRAIN_WAITING: { + anchor: 'queue-train-waiting', + summary: 'the train is below its size and its oldest READY PR has waited less than the policy’s fill time, so it does not dispatch yet', + fix: 'add READY PRs, rerun once the printed time has passed, or pass --urgent for a one-PR train', + }, + QUEUE_TRAIN_PENDING: { + anchor: 'queue-train-pending', + summary: 'the train’s run (free tests, the full gate or the bisect) has not finished, so there is no verdict yet', + fix: 'rerun the printed NEXT: command later; nothing is dispatched again while a run is pending', + }, + QUEUE_TRAIN_INCOMPLETE: { + anchor: 'queue-train-incomplete', + summary: 'the train gate did not complete its census (INFRA or INCOMPLETE panels, an unbound, partial or cancelled shard, a slice killed before upload, or no published record), so it is never green', + fix: 'when every gap is machine-classified INFRA or INCOMPLETE, take the one re-dispatch EVAL_POLICY.infraRedispatch allows (train rebuild --train --infra) and report both runs; otherwise diagnose the run', + }, + QUEUE_TRAIN_RED: { + anchor: 'queue-train-red', + summary: 'the train’s full gate is red on the train tree; its PRs do not merge, and nothing is retried to change the verdict', + fix: 'bisect the red cases (train bisect --train --spend-cap ); after a rebuild’s second red, route the cases to a repair lane', + }, + QUEUE_TRAIN_RED_ON_MAIN: { + anchor: 'queue-train-red-on-main', + summary: 'the bisect found the red cases red on every prefix, main (T0) included: main is broken, not the train’s PRs', + fix: 'fix forward on main first (a one-PR train), then rebuild this train unchanged', + }, + QUEUE_TRAIN_UNATTRIBUTED: { + anchor: 'queue-train-unattributed', + summary: 'the bisect pattern blames no PR and not main (all green, or not monotone); one trial cannot tell a flake from a break', + fix: 'all green: the one preregistered rebuild (train rebuild --train ), reporting both runs; any other pattern: a repair lane, then a changed train', + }, + QUEUE_TRAIN_BISECT_CAPPED: { + anchor: 'queue-train-bisect-capped', + summary: 'the bisect’s reserved price (every case×prefix trial, unknown costs at the policy reservation) passes the spend cap; completed prefixes are kept and the outcome stays unattributed', + fix: 'resume with a higher cap (train bisect --train --spend-cap ); only prefixes without a result run', + }, + QUEUE_TRAIN_PREFIX_INVALID: { + anchor: 'queue-train-prefix-invalid', + summary: 'a bisect prefix is not an ancestor of the train head on train/, or a red case is not in the gate’s plan, so the bisect inputs are refused', + fix: 'check that train/ still points at the recorded head (train status); a moved branch needs a new train', + }, + QUEUE_TRAIN_REBUILD_REFUSED: { + anchor: 'queue-train-rebuild-refused', + summary: 'the rebuild is not allowed: the bisect was not all green, the train was already rebuilt (or re-dispatched), or there is no bisect or INFRA record', + fix: 'route the red cases to a repair lane; the train is rebuilt only once the repair merges (a changed train)', + }, + QUEUE_TRAIN_TREE_MISMATCH: { + anchor: 'queue-train-tree-mismatch', + summary: 'main’s tree is neither the train’s prefix before this PR nor after it (modulo stamps), so the merge does not publish what the gate tested', + fix: 'stop merging; if earlier PRs of the train merged, the queue is frozen (QUEUE_TRAIN_FROZEN); otherwise rebuild the train on current main', + }, + QUEUE_TRAIN_FROZEN: { + anchor: 'queue-train-frozen', + summary: 'a train stopped partway through publication, so main holds a prefix no gate covered on its own; ordinary trains are refused until a repair lands', + fix: 'admit one repair PR at its exact head (train build --repair --prs @); the freeze clears when verify finds main’s tree after that merge in a green record', + }, + READY_SCREEN_CHOICE_REQUIRED: { + anchor: 'ready-screen-choice-required', + summary: 'the base policy turns trains on, so READY needs an explicit screen choice: --screen (pay for a diff-relevant screen now) or --no-screen', + fix: 'rerun gstack-ship-queue ready with the printed recommendation (--screen or --no-screen); say in the READY block if focused local cases already ran', + }, // Field list 2 W4 (pregate `timing`). PREGATE_TIMING_ASSERTION: { anchor: 'pregate-timing-assertion', diff --git a/lib/ship-policy.ts b/lib/ship-policy.ts index 8da297f23..95b53c516 100644 --- a/lib/ship-policy.ts +++ b/lib/ship-policy.ts @@ -65,6 +65,28 @@ export interface ShipPolicy { full_e2e: boolean; /** The oldest gstack that may read this policy; a newer key set to a non-default value requires it. */ min_gstack_version: string | null; + /** Merge trains in gstack-ship-queue (docs/ship-policy.md#run-a-merge-train); null keeps the per-PR queue. */ + train: TrainPolicy | null; +} + +/** The `train` block: the gate workflow trains dispatch and how its verdict is read. */ +export interface TrainPolicy { + /** Workflow file dispatched for the full gate and the bisect (it accepts the inputs in TRAIN_DISPATCH_INPUTS). */ + gate_workflow: string; + /** Workflow file dispatched on train/ before the paid gate; its run must be green on the train head. */ + free_workflow: string; + /** The check run the train's own gate run publishes on the train head when its census is complete and green. */ + full_gate_check: string; + /** Artifact name of the train record (and the prefix of each bisect prefix's artifact). */ + record_artifact: string; + /** Most PRs in one train. */ + max_size: number; + /** A train under max_size dispatches once its oldest READY PR has waited this long. */ + fill_hours: number; + /** How `ready` computes the screen's selection: gstack's paid selector, or not at all. */ + screen_selection: 'gstack-paid' | 'none'; + /** Bisect admission price of one case trial that has no recorded cost. */ + trial_reservation_usd: number; } export const DEFAULT_POLICY: ShipPolicy = { @@ -87,6 +109,7 @@ export const DEFAULT_POLICY: ShipPolicy = { gate_command: null, full_e2e: false, min_gstack_version: null, + train: null, }; export type PolicyKey = Exclude; @@ -112,6 +135,7 @@ export const POLICY_EXPLANATIONS: Record = { migrations: '{ dir, id }: the migration directory and a regex with one capture group for the number in a file name; `gstack-pregate regate` reports MIGRATION_COLLISION against the base branch and every open PR.', gate_command: 'The repo’s full pre-merge gate command. gstack-ship-queue gate-ahead and ready print it for the caller to run (in the background or in CI); gstack never executes it. null when the repo has none.', full_e2e: 'true: every merge needs a passing full-gate result (gstack-ship-queue gate-ahead --record, or the lane\'s --gate-exit 0 --gate-log on the gated head) bound to a tree equal to the merging one modulo stamps; ready, stamp and verify refuse without one (QUEUE_GATE_MISSING).', + train: '{ gate_workflow, free_workflow, full_gate_check, record_artifact, max_size, fill_hours, screen_selection, trial_reservation_usd } or null: turns on merge trains in gstack-ship-queue (train build|status|verdict|bisect|rebuild|prune, ready --screen|--no-screen, verify --train). null keeps today\'s per-PR queue unchanged.', min_gstack_version: 'The oldest gstack that may read this file. Any key newer than 1.91.98.0 set to a non-default value requires it at that key’s introducing version (validate prints each); a gstack older than it refuses the file (POLICY_GSTACK_TOO_OLD).', }; @@ -124,11 +148,13 @@ export const POLICY_EXPLANATIONS: Record = { */ export const POLICY_COMPAT_FLOOR = '1.91.98.0'; export const SHIP_QUEUE_KEYS_VERSION = '1.91.108.0'; +export const TRAIN_KEYS_VERSION = '1.91.136.0'; export const POLICY_KEY_INTRODUCED: Record = { bump: '1.91.73.0', queue_mode: '1.91.73.0', release_tool: '1.91.73.0', release_outputs: '1.91.73.0', stamp_paths: '1.91.73.0', changelog: '1.91.73.0', mirror: '1.91.73.0', history: '1.91.73.0', preregistration_shas: '1.91.73.0', stamp_sensitive_gates: '1.91.73.0', pr_count: '1.91.73.0', maintainers: '1.91.73.0', stamp_files: '1.91.98.0', stamp_patterns: '1.91.98.0', restamp_command: '1.91.98.0', migrations: '1.91.98.0', gate_command: SHIP_QUEUE_KEYS_VERSION, full_e2e: SHIP_QUEUE_KEYS_VERSION, min_gstack_version: SHIP_QUEUE_KEYS_VERSION, + train: TRAIN_KEYS_VERSION, }; const newer = (a: string, b: string): boolean => cmpVersion(parseVersion(a)!, parseVersion(b)!) > 0; @@ -241,6 +267,28 @@ function migrationErrors(value: unknown, out: PolicyError[]): void { } catch (e: any) { out.push({ code: 'POLICY_INVALID', path: 'migrations.id', message: `not a valid regex: ${e.message}` }); } } +const TRAIN_FIELDS: Record string | null> = { + gate_workflow: v => (typeof v === 'string' && /^[A-Za-z0-9_.-]{1,64}\.ya?ml$/.test(v) ? null : 'must be a workflow file name such as evals.yml'), + free_workflow: v => (typeof v === 'string' && /^[A-Za-z0-9_.-]{1,64}\.ya?ml$/.test(v) ? null : 'must be a workflow file name such as free-tests.yml'), + full_gate_check: v => (typeof v === 'string' && /^[A-Za-z0-9_./-]{1,64}$/.test(v) ? null : 'must be a check-run name ([A-Za-z0-9_./-]{1,64})'), + record_artifact: v => (typeof v === 'string' && /^[A-Za-z0-9_.-]{1,64}$/.test(v) ? null : 'must be an artifact name ([A-Za-z0-9_.-]{1,64})'), + max_size: v => (Number.isInteger(v) && (v as number) >= 1 && (v as number) <= 6 ? null : 'must be an integer from 1 to 6'), + fill_hours: v => (typeof v === 'number' && v > 0 && v <= 24 ? null : 'must be a number of hours above 0 and at most 24'), + screen_selection: v => (v === 'gstack-paid' || v === 'none' ? null : 'must be gstack-paid | none'), + trial_reservation_usd: v => (typeof v === 'number' && v > 0 && v <= 100 ? null : 'must be dollars above 0 and at most 100'), +}; + +function trainErrors(value: unknown, out: PolicyError[]): void { + const t = value as Record | null; + if (!t || typeof t !== 'object' || Array.isArray(t)) { out.push({ code: 'POLICY_INVALID', path: 'train', message: `must be { ${Object.keys(TRAIN_FIELDS).join(', ')} } or null` }); return; } + for (const k of Object.keys(t)) if (!(k in TRAIN_FIELDS)) out.push({ code: 'POLICY_INVALID', path: `train.${k}`, message: 'unknown key' }); + for (const [k, check] of Object.entries(TRAIN_FIELDS)) { + if (!(k in t)) { out.push({ code: 'POLICY_INVALID', path: `train.${k}`, message: 'missing (every train field is required)' }); continue; } + const why = check(t[k]); + if (why) out.push({ code: 'POLICY_INVALID', path: `train.${k}`, message: `${JSON.stringify(t[k])} ${why}` }); + } +} + /** * Schema and containment check of a parsed policy object. Unknown keys are * errors (a typo must not silently mean "default", and a key from a newer @@ -282,6 +330,7 @@ export function validatePolicy(raw: unknown, opts: { gstackVersion?: string } = else if (t.length > MAX_COMMAND || /[\0\r\n]/.test(t)) errors.push({ code: 'POLICY_CONTAINMENT', path: 'gate_command', message: `must be one line under ${MAX_COMMAND} characters` }); } if ('full_e2e' in obj && typeof obj.full_e2e !== 'boolean') errors.push({ code: 'POLICY_INVALID', path: 'full_e2e', message: 'must be true or false' }); + if ('train' in obj && obj.train !== null) trainErrors(obj.train, errors); if ('stamp_paths' in obj) listErrors('stamp_paths', obj.stamp_paths, errors, globError); for (const key of ['changelog', 'mirror'] as const) { if (!(key in obj) || obj[key] === null) continue; diff --git a/lib/ship-queue-gate.ts b/lib/ship-queue-gate.ts index f8b0242c3..67c6e51cb 100644 --- a/lib/ship-queue-gate.ts +++ b/lib/ship-queue-gate.ts @@ -129,7 +129,7 @@ export function applyStampPatch(repoRoot: string, mergeTree: string, patch: stri }); } -function withIndex(repoRoot: string, tree: string, fn: (env: NodeJS.ProcessEnv) => T): T | { error: string } { +export function withIndex(repoRoot: string, tree: string, fn: (env: NodeJS.ProcessEnv) => T): T | { error: string } { const gitDir = git(repoRoot, ['rev-parse', '--absolute-git-dir']).out; const indexFile = path.join(gitDir, `gstack-ship-queue-${process.pid}-${randomBytes(4).toString('hex')}.index`); const env = { ...process.env, GIT_INDEX_FILE: indexFile }; diff --git a/lib/ship-queue-train-cli.ts b/lib/ship-queue-train-cli.ts new file mode 100644 index 000000000..03c83c780 --- /dev/null +++ b/lib/ship-queue-train-cli.ts @@ -0,0 +1,108 @@ +/** + * Argument parsing and output for the merge-train surface of + * bin/gstack-ship-queue: `train build|status|verdict|bisect|rebuild|prune`, + * `ready --screen|--no-screen [--broad]` and `verify --train`. Every command + * prints its lines, the coded error line on stderr, then `NEXT: ` + * and the canonical `GSTACK_RESULT: skill=ship-queue status= run=`. + * Loaded by the bin only when one of those paths runs. + */ +import { EXIT, formatResult } from './headless-artifacts'; +import { RESULT_CODES } from './result-codes'; +import type { QueueContext } from './ship-queue'; +import { buildTrain, pruneTrains, trainStatus, type TrainOut } from './ship-queue-train'; +import { trainReady } from './ship-queue-train-ready'; +import { trainBisect, trainRebuild, trainVerdict } from './ship-queue-train-verdict'; +import { verifyTrain } from './ship-queue-train-verify'; +import { redactCause } from './ship-queue-train-record'; + +export const TRAIN_HELP = ` train build --prs @[,...] [--urgent | --repair] [--dry-run] [--free-timeout ] + train build --train resume a train (push, free tests, dispatch) from its record + merge pinned unstamped heads in slot order onto main, free tests on + train/, then one full gate through eval-bg's nonce + train status [--train ] never spends: rebuild records from refs/notes/gstack-trains, reconcile + unconfirmed dispatches by nonce + train verdict --train read the gate run and its record: GREEN, RED, INCOMPLETE or PENDING + train bisect --train --spend-cap [--dry-run] + red cases on every prefix T0..Tk, resumable and capped + train rebuild --train [--infra] the one rebuild an all-green bisect earns (or the one INFRA re-dispatch) + train prune [--train ] [--dry-run] + delete train/ branches of merged, superseded or dropped trains + ready --pr --screen [--broad] | --no-screen [--dry-run] + (train policy) READY plus the screen choice, recorded as a receipt + verify --pr --train [--no-fetch] + (train policy) the merge check against a green train record + Merge trains need a "train" block in the base revision's ship policy (docs/ship-policy.md#run-a-merge-train).`; + +function flag(args: string[], name: string): string | undefined { + const i = args.indexOf(name); + return i >= 0 ? args[i + 1] : undefined; +} + +export function printTrain(cmd: string, out: TrainOut): number { + for (const line of out.lines) process.stdout.write(line + '\n'); + if (out.failure) { + const f = out.failure; + process.stderr.write(`gstack-ship-queue ${cmd}: ${f.message}; fix: ${f.fix ?? RESULT_CODES[f.code].fix} (${f.code})\n`); + } + process.stdout.write(`NEXT: ${out.next}\n`); + process.stdout.write(formatResult('ship-queue', out.status, out.run) + '\n'); + return out.exit; +} + +function usage(message: string): number { + process.stderr.write(`gstack-ship-queue: ${message}\n${TRAIN_HELP}\n`); + return EXIT.usage; +} + +async function guarded(cmd: string, fn: () => Promise | TrainOut): Promise { + try { + return printTrain(cmd, await fn()); + } catch (error) { + const cause = redactCause(error instanceof Error ? error.message : String(error)); + process.stderr.write(`gstack-ship-queue ${cmd}: unexpected failure: ${cause}\n`); + process.stdout.write(`NEXT: gstack-ship-queue train status\n`); + process.stdout.write(formatResult('ship-queue', 'interrupted', 'none') + '\n'); + return EXIT.fail; + } +} + +const positive = (s: string | undefined, fallback: number) => (s !== undefined && /^\d+(\.\d+)?$/.test(s) && Number(s) > 0 ? Number(s) : s === undefined ? fallback : NaN); + +export async function trainMain(ctx: QueueContext, args: string[]): Promise { + const [sub, ...rest] = args; + const nonce = flag(rest, '--train'); + const dryRun = rest.includes('--dry-run'); + if (sub === 'build') { + if (!nonce && !flag(rest, '--prs')) return usage('train build needs --prs @[,...] or --train '); + const freeTimeoutSec = positive(flag(rest, '--free-timeout'), 2700); + if (Number.isNaN(freeTimeoutSec)) return usage('--free-timeout needs seconds'); + if (rest.includes('--urgent') && rest.includes('--repair')) return usage('--urgent and --repair are separate one-PR trains; pass one'); + return guarded('train build', () => buildTrain(ctx, { + pins: flag(rest, '--prs'), resume: nonce, dryRun, urgent: rest.includes('--urgent'), repair: rest.includes('--repair'), freeTimeoutSec, + pollMs: Number(process.env.GSTACK_SHIP_QUEUE_POLL_MS) || 30_000, + })); + } + if (sub === 'status') return guarded('train status', () => trainStatus(ctx, { train: nonce })); + if (sub === 'prune') return guarded('train prune', () => pruneTrains(ctx, { dryRun, train: nonce })); + if (!nonce) return usage(`train ${sub ?? ''} needs --train `); + if (sub === 'verdict') return guarded('train verdict', () => trainVerdict(ctx, { train: nonce })); + if (sub === 'rebuild') return guarded('train rebuild', () => trainRebuild(ctx, { train: nonce, infra: rest.includes('--infra') })); + if (sub === 'bisect') { + const cap = positive(flag(rest, '--spend-cap'), NaN); + if (Number.isNaN(cap)) return usage('train bisect needs --spend-cap '); + return guarded('train bisect', () => trainBisect(ctx, { train: nonce, spendCap: cap, dryRun })); + } + return usage(`unknown train subcommand ${sub}`); +} + +export async function readyMain(ctx: QueueContext, args: string[], base: { pr: string; pregate?: string; gateExit?: number; gateLog?: string }): Promise { + const screen = args.includes('--screen'); + const noScreen = args.includes('--no-screen'); + if (screen && noScreen) return usage('pass one of --screen or --no-screen'); + if (args.includes('--broad') && !screen) return usage('--broad selects the screen policy; it needs --screen'); + return guarded('ready', () => trainReady(ctx, { ...base, screen: screen ? 'screen' : noScreen ? 'no-screen' : undefined, broad: args.includes('--broad'), dryRun: args.includes('--dry-run') })); +} + +export async function verifyMain(ctx: QueueContext, args: string[], pr: string): Promise { + return guarded('verify', () => verifyTrain(ctx, { pr, train: flag(args, '--train')!, noFetch: args.includes('--no-fetch') })); +} diff --git a/lib/ship-queue-train-github.ts b/lib/ship-queue-train-github.ts new file mode 100644 index 000000000..82f12820f --- /dev/null +++ b/lib/ship-queue-train-github.ts @@ -0,0 +1,83 @@ +/** + * GitHub reads and writes for merge trains, all through the queue's `gh` + * seam (QueueContext.gh) so tests answer them from a fixture: PR identity, + * workflow runs found by branch or by eval-bg nonce, run artifacts and the + * full-gate check run. Dispatch itself goes through scripts/eval-bg.ts. + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import type { Exec, RunSummary } from '../scripts/eval-bg'; +import type { QueueContext } from './ship-queue'; + +export interface PrInfo { headRefOid: string; isCrossRepository: boolean; state: string; headRefName: string } +export interface RunInfo { databaseId: number; status: string; conclusion: string | null; url: string; headSha: string; event: string; displayTitle: string } +export interface CheckInfo { status: string; conclusion: string | null } + +function json(r: { ok: boolean; out: string }): T | null { + if (!r.ok) return null; + try { return JSON.parse(r.out) as T; } catch { return null; } +} + +/** eval-bg's process seam bound to the queue's gh (only `gh` is ever routed through it). */ +export function execFor(ctx: QueueContext): Exec { + return (command, args) => { + if (command !== 'gh') return { status: null, stdout: '' }; + const r = ctx.gh(args); + return { status: r.ok ? 0 : 1, stdout: r.out }; + }; +} + +export function repoSlug(ctx: QueueContext): string | null { + const r = json<{ nameWithOwner?: string }>(ctx.gh(['repo', 'view', '--json', 'nameWithOwner'])); + return r?.nameWithOwner && /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(r.nameWithOwner) ? r.nameWithOwner : null; +} + +export function prInfo(ctx: QueueContext, pr: string): PrInfo | { error: string } { + const r = ctx.gh(['pr', 'view', pr, '--json', 'headRefOid,isCrossRepository,state,headRefName']); + const p = json>(r); + if (!p?.headRefOid) return { error: `cannot read PR #${pr} through gh (${(r.err ?? r.out).trim().slice(0, 120) || 'no headRefOid'})` }; + return { headRefOid: p.headRefOid, isCrossRepository: p.isCrossRepository !== false, state: p.state ?? 'OPEN', headRefName: p.headRefName ?? '' }; +} + +const RUN_FIELDS = 'databaseId,status,conclusion,url,headSha,event,displayTitle'; + +/** Newest workflow_dispatch run of a workflow on one branch; null when none; 'unreadable' when gh failed. */ +export function newestBranchRun(ctx: QueueContext, slug: string, workflow: string, branch: string): RunInfo | null | 'unreadable' { + const runs = json(ctx.gh(['run', 'list', '--repo', slug, '--workflow', workflow, '--branch', branch, '--event', 'workflow_dispatch', '--limit', '20', '--json', RUN_FIELDS])); + if (!Array.isArray(runs)) return 'unreadable'; + return runs.filter(r => r.event === 'workflow_dispatch').sort((a, b) => b.databaseId - a.databaseId)[0] ?? null; +} + +export function runView(ctx: QueueContext, slug: string, id: number): RunInfo | null { + return json(ctx.gh(['run', 'view', String(id), '--repo', slug, '--json', RUN_FIELDS])); +} + +export function asRun(summary: RunSummary): { id: number; url: string } { + return { id: summary.databaseId, url: summary.url }; +} + +/** One JSON artifact of a run (the first .json file it holds); null when it is missing or unreadable. */ +export function downloadJson(ctx: QueueContext, slug: string, run: number, name: string, scratch: string): unknown { + const dir = path.join(scratch, `${run}-${name}`); + fs.rmSync(dir, { recursive: true, force: true }); + fs.mkdirSync(dir, { recursive: true }); + try { + if (!ctx.gh(['run', 'download', String(run), '--repo', slug, '-n', name, '-D', dir]).ok) return null; + const file = fs.readdirSync(dir).find(f => f.endsWith('.json')); + return file ? JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8')) : null; + } catch { return null; } finally { fs.rmSync(dir, { recursive: true, force: true }); } +} + +/** The named check run on a commit (the newest when several); null when absent or unreadable. */ +export function checkRun(ctx: QueueContext, sha: string, name: string): CheckInfo | null { + const r = json<{ check_runs?: Array<{ name?: string; status?: string; conclusion?: string | null }> }>( + ctx.gh(['api', `repos/{owner}/{repo}/commits/${sha}/check-runs?check_name=${encodeURIComponent(name)}`])); + const hit = r?.check_runs?.find(c => c.name === name); + return hit ? { status: hit.status ?? 'unknown', conclusion: hit.conclusion ?? null } : null; +} + +/** Whether a branch exists on the remote (ls-remote, never a fetch). */ +export function remoteBranchSha(git: (args: string[]) => { ok: boolean; out: string }, remote: string, branch: string): string | null { + const r = git(['ls-remote', remote, `refs/heads/${branch}`]); + return r.ok ? (r.out.split(/\s/)[0] || null) : null; +} diff --git a/lib/ship-queue-train-ready.ts b/lib/ship-queue-train-ready.ts new file mode 100644 index 000000000..12102394e --- /dev/null +++ b/lib/ship-queue-train-ready.ts @@ -0,0 +1,128 @@ +/** + * `ready --pr --screen | --no-screen` under a ship policy with a `train` + * block (docs/ship-policy.md#run-a-merge-train; plan GSTA-49 T1). The + * READY block is today's (lib/ship-queue.ts ready()); this adds the screen + * selection and Garry's default recommendation (screen when the screen + * selects a skill's own cases), requires an explicit choice + * (READY_SCREEN_CHOICE_REQUIRED), dispatches a chosen screen through + * eval-bg's nonce with its purpose and policy in the key, and records the + * choice as a READY receipt (local state and refs/notes/gstack-trains) that + * `train build` requires. The screen's dollar price is left to the T0 cost + * replay (lib/gate-cost.ts) and printed as pending until it lands. + */ +import { randomBytes } from 'node:crypto'; +import { DISPATCH_INPUTS, awaitNonceRun, dedupeKey, dispatchWorkflow, findDuplicateRun, listDispatchRuns, makeNonce, type WorkflowDispatch } from '../scripts/eval-bg'; +import { EXIT } from './headless-artifacts'; +import { ready, type QueueContext } from './ship-queue'; +import { git } from './ship-queue-gate'; +import { execFor, repoSlug } from './ship-queue-train-github'; +import { loadReady, nowIso, persist, readyPath, type ReadyReceipt, type ScreenChoice } from './ship-queue-train-record'; +import { outOk, outPending, outRefused, trainEnv, type TrainEnv, type TrainOut } from './ship-queue-train'; + +const Q = 'gstack-ship-queue'; + +export interface ReadyOptions { pr: string; pregate?: string; gateExit?: number; gateLog?: string; screen?: 'screen' | 'no-screen'; broad: boolean; dryRun: boolean } +export interface ScreenSelection { lines: string[]; recommend: 'screen' | 'no-screen' | null } + +/** Skill directories a diff touches: top-level directories that hold a SKILL.md.tmpl at the head. */ +export function touchedSkills(changed: readonly string[], hasTemplate: (dir: string) => boolean): string[] { + return [...new Set(changed.filter(f => f.includes('/')).map(f => f.split('/')[0]!))].filter(hasTemplate).sort(); +} + +/** The selected cases that are a touched skill's own: their touchfiles name a path inside that skill's directory. */ +export function skillOwnCases(selected: readonly string[], touchfiles: Record, skills: readonly string[]): string[] { + return selected.filter(id => (touchfiles[id] ?? []).some(glob => skills.some(s => glob.startsWith(`${s}/`)))).sort(); +} + +/** gstack's paid selector over a diff (today's PR selection; T2 narrows it), loaded only when the policy asks for it. */ +export async function gstackPaidSelection(repoRoot: string, baseSha: string, changed: string[], hasTemplate: (dir: string) => boolean): Promise { + const { computePaidCaseSelection } = await import('../scripts/lib/paid-select'); + const { E2E_TOUCHFILES, LLM_JUDGE_TOUCHFILES } = await import('../test/helpers/touchfiles'); + const skills = touchedSkills(changed, hasTemplate); + let e2e: string[]; let judges: number; let note = "today's PR selection; T2's narrowed screen has not landed"; + try { + const s = computePaidCaseSelection({ profile: 'pr', rootDir: repoRoot, changedFiles: changed, env: { ...process.env, EVALS_BASE: baseSha, EVALS_ALL: '' } }).selection; + e2e = s.e2e ?? Object.keys(E2E_TOUCHFILES); + judges = s.judges ? s.judges.length : Object.keys(LLM_JUDGE_TOUCHFILES).length; + } catch (error) { + e2e = Object.keys(E2E_TOUCHFILES); + judges = Object.keys(LLM_JUDGE_TOUCHFILES).length; + note = `the PR profile needs full validation (${error instanceof Error ? error.message.slice(0, 160) : String(error)}): the screen is the full gate selection`; + } + const own = skillOwnCases(e2e, E2E_TOUCHFILES, skills); + const recommend = own.length ? 'screen' : 'no-screen'; + return { + recommend, + lines: [ + `SCREEN: selection ${e2e.length} e2e case(s) + ${judges} judge(s) (${note})`, + `SCREEN: skill's own cases ${own.length ? own.slice(0, 12).join(',') + (own.length > 12 ? ` (+${own.length - 12})` : '') : 'none'} → recommend --${recommend}`, + ], + }; +} + +async function screenSelection(env: TrainEnv, head: string, baseSha: string): Promise { + const price = 'SCREEN: price pending the T0 cost replay (lib/gate-cost.ts): cases without cost data are unknown; a red train holds its PRs about an hour'; + if (env.train.screen_selection === 'none') return { recommend: null, lines: ['SCREEN: selection not computed (train.screen_selection is none); no recommendation, the lane decides', price] }; + const changed = git(env.ctx.repoRoot, ['diff', '--name-only', `${baseSha}...${head}`]).out.split('\n').filter(Boolean); + const hasTemplate = (dir: string) => git(env.ctx.repoRoot, ['cat-file', '-e', `${head}:${dir}/SKILL.md.tmpl`]).ok; + const s = await gstackPaidSelection(env.ctx.repoRoot, baseSha, changed, hasTemplate); + return { ...s, lines: [...s.lines, price] }; +} + +/** Dispatch (or adopt) the PR's screen on the base branch: inputs name the PR, its merge SHA and the screen policy. */ +async function dispatchScreen(env: TrainEnv, o: ReadyOptions, mergeSha: string, baseSha: string): Promise<{ choice: ScreenChoice; line: string } | TrainOut> { + const policy = o.broad ? 'broad' : 'narrow'; + const scope = { purpose: 'screen' as const, policy: policy as 'narrow' | 'broad' }; + const nonce = makeNonce('pr', mergeSha, baseSha, randomBytes(3).toString('hex'), scope); + const dispatch: WorkflowDispatch = { workflow: env.train.gate_workflow, inputs: { + [DISPATCH_INPUTS.evalsAll]: 'false', [DISPATCH_INPUTS.expectedSha]: mergeSha, [DISPATCH_INPUTS.nonce]: nonce, [DISPATCH_INPUTS.baseRef]: env.ctx.base, + [DISPATCH_INPUTS.baseSha]: baseSha, [DISPATCH_INPUTS.screenPr]: o.pr, [DISPATCH_INPUTS.screenMergeSha]: mergeSha, [DISPATCH_INPUTS.screenPolicy]: policy, + } }; + const inputs = Object.entries(dispatch.inputs).map(([k, v]) => `${k}=${v}`).join(' '); + if (o.dryRun) return { choice: { choice: 'screen', policy, nonce, run: null, merge_sha: mergeSha }, line: `SCREEN_DISPATCH: ${dispatch.workflow} --ref ${env.ctx.base} ${inputs} (dry run: not dispatched)` }; + const slug = repoSlug(env.ctx); + if (!slug) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: 'gh repo view could not name the repository' }, [], `${Q} ready --pr ${o.pr} --screen`, 'none'); + const exec = execFor(env.ctx); + const existing = findDuplicateRun(listDispatchRuns(dispatch.workflow, exec, slug) ?? [], dedupeKey('pr', mergeSha, baseSha, scope)); + if (existing) return { choice: { choice: 'screen', policy, nonce: /eval-bg (\S+?)\]/.exec(existing.displayTitle)?.[1] ?? nonce, run: existing.databaseId, merge_sha: mergeSha }, line: `SCREEN_DISPATCH: following run ${existing.databaseId} (same merge SHA, base and policy; no duplicate)` }; + const found = dispatchWorkflow(dispatch, env.ctx.base, exec, slug) ? await awaitNonceRun(dispatch.workflow, nonce, exec, slug) : null; + if (!found) return outPending({ code: 'QUEUE_TRAIN_DISPATCH_UNKNOWN', message: `no ${dispatch.workflow} run named ${nonce} appeared after the screen dispatch` }, [], `${Q} ready --pr ${o.pr} --screen${o.broad ? ' --broad' : ''} (adopts the run by its key once it appears)`, 'none'); + return { choice: { choice: 'screen', policy, nonce, run: found.databaseId, merge_sha: mergeSha }, line: `SCREEN_DISPATCH: ${dispatch.workflow} run ${found.databaseId} ${found.url} nonce ${nonce}` }; +} + +export async function trainReady(ctx: QueueContext, o: ReadyOptions): Promise { + const env = trainEnv(ctx); + if ('code' in env) return outRefused(env, [], 'none', 'none'); + const base = ready(ctx, o); + if (!base.ok) { + const out = outRefused(base.failure!, base.lines, `fix the refusal, then ${Q} ready --pr ${o.pr} --screen|--no-screen`, base.run); + return ['QUEUE_VERIFY_MISMATCH', 'QUEUE_STAMP_NOT_STAMP_ONLY'].includes(base.failure!.code) ? { ...out, exit: EXIT.fail } : out; + } + const head = git(ctx.repoRoot, ['rev-parse', 'HEAD']).out; + const baseSha = git(ctx.repoRoot, ['rev-parse', `${env.remote}/${ctx.base}`]).out; + const sel = await screenSelection(env, head, baseSha); + const lines = [...base.lines, ...sel.lines]; + const rec = sel.recommend ?? 'no-screen'; + if (!o.screen) { + return outRefused({ code: 'READY_SCREEN_CHOICE_REQUIRED', message: `the base policy runs merge trains; choose --screen or --no-screen (recommended: --${rec})` }, lines, + `${Q} ready --pr ${o.pr} --${rec}`, 'none'); + } + let choice: ScreenChoice = { choice: 'no-screen', policy: null, nonce: null, run: null, merge_sha: null }; + if (o.screen === 'screen') { + const ls = git(ctx.repoRoot, ['ls-remote', env.remote, `refs/pull/${o.pr}/merge`]); + const mergeSha = ls.ok ? ls.out.split(/\s/)[0] ?? '' : ''; + if (!/^[0-9a-f]{40,64}$/.test(mergeSha)) return outRefused({ code: 'QUEUE_NOT_READY', message: `${env.remote} has no refs/pull/${o.pr}/merge (the PR conflicts with ${ctx.base}, or GitHub has not computed it yet)` }, lines, `merge ${ctx.base} into the branch, push, then ${Q} ready --pr ${o.pr} --screen`, 'none'); + const d = await dispatchScreen(env, o, mergeSha, baseSha); + if ('exit' in d) return { ...d, lines: [...lines, ...d.lines] }; + choice = d.choice; + lines.push(d.line); + } + lines.splice(1, 0, `screen: ${choice.choice}${choice.policy ? ` policy=${choice.policy} nonce=${choice.nonce} run=${choice.run ?? '-'}` : ''}`); + const now = nowIso(); + const readyAt = loadReady(env.repo.key, o.pr, head)?.ready_at ?? now; + const receipt: ReadyReceipt = { schema: 1, kind: 'ready', repo_key: env.repo.key, pr: o.pr, head, base_sha: baseSha, ready_at: readyAt, screen: choice, updated_at: now }; + if (o.dryRun) return outOk([...lines, 'DRY-RUN: no READY receipt written, nothing dispatched'], `${Q} ready --pr ${o.pr} --${o.screen}${o.broad ? ' --broad' : ''}`, 'none'); + const p = persist(ctx.repoRoot, [receipt], { remote: env.remote }); + lines.push(`READY_RECEIPT: ${readyPath(env.repo.key, o.pr, head)} (${p.detail})`); + return outOk(lines, `send the READY block; the coordinator builds the train: ${Q} train build --prs ${o.pr}@${head},...`, readyPath(env.repo.key, o.pr, head)); +} diff --git a/lib/ship-queue-train-record.ts b/lib/ship-queue-train-record.ts new file mode 100644 index 000000000..66c62b150 --- /dev/null +++ b/lib/ship-queue-train-record.ts @@ -0,0 +1,207 @@ +/** + * Merge-train records (docs/ship-policy.md#run-a-merge-train): versioned + * `train.json`, one attempt record per dispatch, the bisect ledger, READY + * receipts and the queue freeze, under `$GSTACK_STATE_ROOT/ship-queue/` + * (lib/state-root.ts resolves the root). Every record is mirrored to origin + * as a git note under `refs/notes/gstack-trains`, each on a blob named for + * the record, so a lost machine rebuilds its state with `train status`. A + * record is written locally before its note, and the note before any remote + * effect that depends on it. + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { redactFindingSpans } from './redact-engine'; +import { git, sha256, writeJsonAtomic } from './ship-queue-gate'; +import { resolveStateRoot } from './state-root'; + +export const TRAIN_SCHEMA = 1; +export const NOTES_REF = 'refs/notes/gstack-trains'; +const NOTES_REMOTE_REF = 'refs/notes/gstack-trains-origin'; + +export type TrainState = + | 'built' | 'pushed' | 'free-pending' | 'free-red' | 'free-green' | 'dispatch-unknown' | 'dispatch-absent' | 'gate-pending' + | 'green' | 'red' | 'incomplete' | 'attributed' | 'red-on-main' | 'unattributed' | 'merging' | 'merged' | 'superseded'; +/** States after which a train never dispatches again; prune deletes their branches. */ +export const TERMINAL_STATES: readonly TrainState[] = ['free-red', 'attributed', 'red-on-main', 'merged', 'superseded']; + +export interface ScreenChoice { choice: 'screen' | 'no-screen'; policy: 'narrow' | 'broad' | null; nonce: string | null; run: number | null; merge_sha: string | null } +export interface TrainSlot { pr: string; head: string; ready_at: string; screen: ScreenChoice } +export interface TrainPrefix { index: number; commit: string; tree: string; pr: string | null } +export interface FreeRun { run: number | null; url: string | null; head_sha: string | null; conclusion: string | null; verdict: 'green' | 'red' | 'missing' | 'stale' | 'cancelled' | 'pending' } +export interface TrainRecord { + schema: 1; kind: 'train'; nonce: string; key: string; repo_key: string; repo: string; base: { ref: string; sha: string }; policy: string; + slots: TrainSlot[]; held: string[]; prefixes: TrainPrefix[]; head: string; tree: string; branch: string; + urgent: boolean; repair: boolean; state: TrainState; free: FreeRun | null; attempts: string[]; + merged: Array<{ pr: string; sha: string; tree: string; at: string }>; superseded_by: string | null; + effects: Array<{ at: string; what: string }>; error: string | null; created_at: string; updated_at: string; +} +export type AttemptPurpose = 'gate' | 'rebuild' | 'infra-redispatch' | 'bisect'; +export type GateOutcome = 'PENDING' | 'GREEN' | 'RED' | 'INCOMPLETE'; +export interface CasePanel { case: string; kind: string; status: string; red_class: string | null; cost_usd: number | null } +export interface AttemptRecord { + schema: 1; kind: 'train-attempt'; train: string; id: string; purpose: AttemptPurpose; nonce: string; workflow: string; ref: string; + inputs: Record; state: 'dispatching' | 'unknown' | 'absent' | 'dispatched' | 'completed'; run: { id: number; url: string } | null; + outcome: GateOutcome | null; detail: string | null; red_cases: string[]; panels: CasePanel[]; infra_eligible: boolean; + cost_usd: number | null; prefixes: number[]; created_at: string; updated_at: string; +} +export type PrefixResult = 'red' | 'green'; +export interface BisectRecord { + schema: 1; kind: 'train-bisect'; train: string; cases: string[]; spend_cap: number; spent_usd: number; + prefixes: Array<{ index: number; sha: string; results: Record; attempts: string[] }>; + outcome: null | { kind: 'attributed' | 'red-on-main' | 'unattributed'; all_green: boolean; prs: string[]; patterns: Record; at: string }; + updated_at: string; +} +export interface FreezeRecord { + schema: 1; kind: 'freeze'; repo_key: string; train: string; merged: string[]; unmerged: Array<{ pr: string; head: string }>; + main: { sha: string; tree: string }; reason: string; backstop: { nonce: string; run: number | null; state: string } | null; + repair: { pr: string; head: string; train: string | null } | null; created_at: string; cleared_at: string | null; cleared_by: string | null; updated_at: string; +} +export interface ReadyReceipt { + schema: 1; kind: 'ready'; repo_key: string; pr: string; head: string; base_sha: string; ready_at: string; screen: ScreenChoice; updated_at: string; +} +export type AnyRecord = TrainRecord | AttemptRecord | BisectRecord | FreezeRecord | ReadyReceipt; + +/** Clock seam for fill-time and record tests (`GSTACK_SHIP_QUEUE_NOW`, ISO-8601). */ +export function nowIso(): string { + const fixed = process.env.GSTACK_SHIP_QUEUE_NOW; + return fixed && !Number.isNaN(Date.parse(fixed)) ? new Date(fixed).toISOString() : new Date().toISOString(); +} + +/** An unexpected failure's cause, with secrets withheld, for the record and the error line. */ +export function redactCause(message: string): string { + const text = message.replace(/\s+/g, ' ').slice(0, 400); + return redactFindingSpans(text) ?? '(cause withheld: it could not be redacted safely)'; +} + +export function queueRoot(env: NodeJS.ProcessEnv = process.env): string { + return path.join(resolveStateRoot(env), 'ship-queue'); +} +export const trainDir = (nonce: string) => path.join(queueRoot(), 'trains', nonce); +export const trainFile = (nonce: string) => path.join(trainDir(nonce), 'train.json'); +const attemptPath = (nonce: string, id: string) => path.join(trainDir(nonce), 'attempts', `${id}.json`); +const bisectPath = (nonce: string) => path.join(trainDir(nonce), 'bisect.json'); +const freezePath = (repoKey: string) => path.join(queueRoot(), 'freeze', `${repoKey}.json`); +export const readyPath = (repoKey: string, pr: string, head: string) => path.join(queueRoot(), 'ready', repoKey, `${pr}-${head}.json`); + +/** The repository's identity: origin's URL, keyed so records of different repositories never mix. */ +export function repoIdentity(repoRoot: string): { key: string; url: string } { + const url = git(repoRoot, ['config', '--get', 'remote.origin.url']).out.trim() || repoRoot; + const norm = url.replace(/\.git$/, '').replace(/^git@github\.com:/, 'https://github.com/').replace(/\/+$/, ''); + return { key: sha256(norm).slice(0, 16), url: norm }; +} + +function readJson(file: string, kind: string): T | null { + try { const r = JSON.parse(fs.readFileSync(file, 'utf8')); return r?.schema === TRAIN_SCHEMA && r.kind === kind ? r as T : null; } catch { return null; } +} + +export const loadTrain = (nonce: string) => (/^[A-Za-z0-9._-]{1,128}$/.test(nonce) ? readJson(trainFile(nonce), 'train') : null); +export const loadAttempt = (nonce: string, id: string) => (/^[A-Za-z0-9._-]{1,64}$/.test(id) ? readJson(attemptPath(nonce, id), 'train-attempt') : null); +export const loadBisect = (nonce: string) => readJson(bisectPath(nonce), 'train-bisect'); +export const loadFreeze = (repoKey: string) => readJson(freezePath(repoKey), 'freeze'); +export const loadReady = (repoKey: string, pr: string, head: string) => readJson(readyPath(repoKey, pr, head), 'ready'); +export const activeFreeze = (repoKey: string): FreezeRecord | null => { const f = loadFreeze(repoKey); return f && !f.cleared_at ? f : null; }; + +export function attemptsOf(train: TrainRecord): AttemptRecord[] { + return train.attempts.map(id => loadAttempt(train.nonce, id)).filter((a): a is AttemptRecord => !!a); +} + +/** Every train of this repository, newest first. */ +export function listTrains(repoKey: string): TrainRecord[] { + const dir = path.join(queueRoot(), 'trains'); + if (!fs.existsSync(dir)) return []; + return fs.readdirSync(dir).map(n => loadTrain(n)).filter((t): t is TrainRecord => !!t && t.repo_key === repoKey) + .sort((a, b) => b.created_at.localeCompare(a.created_at) || b.nonce.localeCompare(a.nonce)); +} + +export function recordPath(r: AnyRecord): string { + if (r.kind === 'train') return trainFile(r.nonce); + if (r.kind === 'train-attempt') return attemptPath(r.train, r.id); + if (r.kind === 'train-bisect') return bisectPath(r.train); + if (r.kind === 'freeze') return freezePath(r.repo_key); + return readyPath(r.repo_key, r.pr, r.head); +} + +function noteName(r: AnyRecord): string { + if (r.kind === 'train') return `gstack-trains:train:${r.nonce}`; + if (r.kind === 'train-attempt') return `gstack-trains:attempt:${r.train}:${r.id}`; + if (r.kind === 'train-bisect') return `gstack-trains:bisect:${r.train}`; + if (r.kind === 'freeze') return `gstack-trains:freeze:${r.repo_key}`; + return `gstack-trains:ready:${r.repo_key}:${r.pr}:${r.head}`; +} + +const NOTES_IDENTITY = { GIT_AUTHOR_NAME: 'gstack-ship-queue', GIT_AUTHOR_EMAIL: 'ship-queue@gstack.local', GIT_COMMITTER_NAME: 'gstack-ship-queue', GIT_COMMITTER_EMAIL: 'ship-queue@gstack.local' }; +const notesEnv = () => ({ ...process.env, ...NOTES_IDENTITY }); + +/** Fetch origin's notes and fold them into the local notes ref (the local side wins a same-record conflict). */ +export function fetchNotes(repoRoot: string, remote = 'origin'): { ok: boolean; detail: string } { + const f = git(repoRoot, ['fetch', '--no-tags', '-q', remote, `+${NOTES_REF}:${NOTES_REMOTE_REF}`]); + if (!f.ok) { + const absent = /couldn't find remote ref|no such ref/i.test(f.err); + return { ok: absent, detail: absent ? `${remote} has no ${NOTES_REF} yet` : `git fetch ${NOTES_REF} failed: ${f.err.slice(0, 160)}` }; + } + const local = git(repoRoot, ['rev-parse', '--verify', '--quiet', NOTES_REF]).out; + const fetched = git(repoRoot, ['rev-parse', '--verify', '--quiet', NOTES_REMOTE_REF]).out; + if (!local) { git(repoRoot, ['update-ref', NOTES_REF, fetched]); return { ok: true, detail: 'notes fetched' }; } + if (local === fetched || git(repoRoot, ['merge-base', '--is-ancestor', fetched, local]).ok) return { ok: true, detail: 'notes current' }; + const m = git(repoRoot, ['notes', `--ref=${NOTES_REF}`, 'merge', '-q', '-s', 'ours', NOTES_REMOTE_REF], notesEnv()); + return m.ok ? { ok: true, detail: 'notes merged' } : { ok: false, detail: `git notes merge failed: ${m.err.slice(0, 160)}` }; +} + +function addNote(repoRoot: string, r: AnyRecord): boolean { + const blob = git(repoRoot, ['hash-object', '-w', '--stdin'], undefined, noteName(r)); + if (!blob.ok) return false; + return git(repoRoot, ['notes', `--ref=${NOTES_REF}`, 'add', '-f', '-F', '-', blob.out], notesEnv(), JSON.stringify(r)).ok; +} + +/** + * Write records locally, then as notes, then push the notes ref (fetch, + * merge and retry on a race). A failed push leaves the local records and + * notes for the next write to carry, and says so. + */ +export function persist(repoRoot: string, records: AnyRecord[], opts: { push?: boolean; remote?: string } = {}): { pushed: boolean; detail: string } { + for (const r of records) writeJsonAtomic(recordPath(r), r); + if (!records.every(r => addNote(repoRoot, r))) return { pushed: false, detail: 'could not write the notes locally' }; + if (opts.push === false) return { pushed: false, detail: 'notes written locally (not pushed)' }; + const remote = opts.remote ?? 'origin'; + let last = ''; + for (let i = 0; i < 3; i++) { + const p = git(repoRoot, ['push', '-q', remote, `${NOTES_REF}:${NOTES_REF}`]); + if (p.ok) return { pushed: true, detail: `notes pushed to ${remote} ${NOTES_REF}` }; + last = p.err.slice(0, 160); + const f = fetchNotes(repoRoot, remote); + if (!f.ok) return { pushed: false, detail: f.detail }; + } + return { pushed: false, detail: `git push ${NOTES_REF} failed: ${last}` }; +} + +function newerOrMissing(r: AnyRecord): boolean { + try { + const cur = JSON.parse(fs.readFileSync(recordPath(r), 'utf8')); + return !(typeof cur.updated_at === 'string' && cur.updated_at >= r.updated_at); + } catch { return true; } +} + +function parseNote(text: string): AnyRecord | null { + try { + const r = JSON.parse(text); + const ids = /^[A-Za-z0-9._-]{1,128}$/; + if (r?.schema !== TRAIN_SCHEMA || typeof r.updated_at !== 'string') return null; + if (r.kind === 'train' && ids.test(r.nonce)) return r; + if ((r.kind === 'train-attempt' && ids.test(r.train) && ids.test(r.id)) || (r.kind === 'train-bisect' && ids.test(r.train))) return r; + if (r.kind === 'freeze' && ids.test(r.repo_key)) return r; + if (r.kind === 'ready' && ids.test(r.repo_key) && /^\d+$/.test(r.pr) && /^[0-9a-f]{40,64}$/.test(r.head)) return r; + } catch {} + return null; +} + +/** Rebuild local records from the notes ref: a note whose record is missing locally, or newer than the local copy, is written. */ +export function rebuildFromNotes(repoRoot: string, remote = 'origin'): { restored: number; read: number; detail: string } { + const f = fetchNotes(repoRoot, remote); + const list = git(repoRoot, ['notes', `--ref=${NOTES_REF}`, 'list']); + if (!list.ok || !list.out.trim()) return { restored: 0, read: 0, detail: f.detail }; + const records = list.out.split('\n').map(l => l.split(' ')[0]!).filter(Boolean) + .map(blob => parseNote(git(repoRoot, ['cat-file', 'blob', blob]).out)).filter((r): r is AnyRecord => !!r); + const fresh = records.filter(newerOrMissing); + for (const r of fresh) writeJsonAtomic(recordPath(r), r); + return { restored: fresh.length, read: records.length, detail: f.detail }; +} diff --git a/lib/ship-queue-train-verdict.ts b/lib/ship-queue-train-verdict.ts new file mode 100644 index 000000000..5ee480ce9 --- /dev/null +++ b/lib/ship-queue-train-verdict.ts @@ -0,0 +1,332 @@ +/** + * Train verdict, bisect and the one preregistered rebuild + * (docs/ship-policy.md#run-a-merge-train; plan GSTA-49 lane T1, P3). + * + * The verdict reads the train's own gate run and the record artifact that run + * publishes (with the `gstack/full-gate` check run) on the train head; it + * only reads, never writes a check. Green needs a complete census from the + * run's own plan manifest. A red train stays red: the bisect runs only its + * red cases, one trial on every prefix T0..Tk, and attributes by pattern. + * Rebuild-once (Garry, 2026-10-18; to be recorded in EVAL_POLICY beside + * infraRedispatch): an all-green bisect earns one unchanged rebuild. + */ +import * as path from 'node:path'; +import { randomBytes } from 'node:crypto'; +import { DISPATCH_INPUTS, listDispatchRuns, makeNonce, runForNonce } from '../scripts/eval-bg'; +import type { QueueContext } from './ship-queue'; +import { git } from './ship-queue-gate'; +import { + checkRun, downloadJson, execFor, asRun, remoteBranchSha, repoSlug, runView, type CheckInfo, type RunInfo, +} from './ship-queue-train-github'; +import { + attemptsOf, loadBisect, nowIso, persist, trainDir, trainFile, + type AttemptRecord, type BisectRecord, type CasePanel, type GateOutcome, type PrefixResult, type TrainRecord, +} from './ship-queue-train-record'; +import { + dispatchAttempt, effect, findTrain, gateDispatch, nextFor, outFailed, outOk, outPending, outRefused, pinList, save, trainEnv, + type TrainEnv, type TrainOut, +} from './ship-queue-train'; + +/** EVAL_POLICY.infraRedispatch (test/helpers/periodic-exclude-data.ts): one re-dispatch of an INFRA/INCOMPLETE-only census. */ +export const INFRA_REDISPATCH = 1; +/** Rebuild-once (Garry, 2026-10-18): an all-green bisect earns exactly one unchanged rebuild; pinned to EVAL_POLICY by its test. */ +export const REBUILD_ONCE = 1; + +const Q = 'gstack-ship-queue'; + +/** The record artifact the train's own gate run publishes (plan T1b), read by verdict, verify and the backstop. */ +export interface TrainGateRecord { + schema: 1; kind: 'train-gate'; nonce: string; head_sha: string; tree: string; verdict: 'GREEN' | 'RED'; + census: { complete: boolean; planned: { rule: number; judge: number }; executed: { rule: number; judge: number }; + unbound: number; partial_shards: number; cancelled_shards: number; killed_before_upload: number }; + panels: CasePanel[]; +} +/** One bisect prefix's artifact: the red cases on that prefix, one trial each. */ +export interface BisectPrefixRecord { schema: 1; kind: 'train-bisect-prefix'; nonce: string; prefix_sha: string; checked_out_sha: string; panels: CasePanel[] } + +export const bisectArtifact = (recordArtifact: string, sha: string) => `${recordArtifact}-bisect-${sha.slice(0, 12)}`; + +export interface GateClass { outcome: GateOutcome; detail: string; red: string[]; panels: CasePanel[]; infraEligible: boolean; cost: number | null } + +/** Every paid-report class mapped to one train outcome; green only for a complete, bound, published census. */ +export function classifyGate(raw: unknown, expect: { nonce: string; head: string; tree: string }, run: Pick, check: CheckInfo | null): GateClass { + const base = { red: [] as string[], panels: [] as CasePanel[], infraEligible: false, cost: null as number | null }; + if (run.status !== 'completed') return { ...base, outcome: 'PENDING', detail: `run is ${run.status}` }; + const rec = raw as TrainGateRecord | null; + if (!rec || rec.schema !== 1 || rec.kind !== 'train-gate' || !Array.isArray(rec.panels) || !rec.census) { + return { ...base, outcome: 'INCOMPLETE', infraEligible: true, detail: `no train record artifact (run concluded ${run.conclusion ?? 'unknown'}; a slice killed before its upload step counts here)` }; + } + const panels = rec.panels; + const known = panels.map(p => p.cost_usd).filter((c): c is number => typeof c === 'number'); + const cost = known.length ? Math.round(known.reduce((a, b) => a + b, 0) * 100) / 100 : null; + const withPanels = { ...base, panels, cost }; + if (rec.nonce !== expect.nonce || rec.head_sha !== expect.head || rec.tree !== expect.tree) { + return { ...withPanels, outcome: 'INCOMPLETE', detail: `the record is bound to nonce ${rec.nonce} head ${String(rec.head_sha).slice(0, 12)} tree ${String(rec.tree).slice(0, 12)}, not this train's attempt` }; + } + const red = panels.filter(p => p.status === 'FAIL' && p.red_class === 'VERDICT').map(p => p.case).sort(); + if (red.length) return { ...withPanels, outcome: 'RED', red, detail: `${red.length} red case(s): ${red.join(', ')}` }; + const c = rec.census; + const gaps = [ + ...(c.planned.rule + c.planned.judge <= 0 ? ['the plan manifest lists no cases'] : []), + ...(c.executed.rule !== c.planned.rule ? [`rule ${c.executed.rule}/${c.planned.rule} executed`] : []), + ...(c.executed.judge !== c.planned.judge ? [`judge ${c.executed.judge}/${c.planned.judge} executed`] : []), + ...(c.unbound ? [`${c.unbound} unbound result`] : []), ...(c.partial_shards ? [`${c.partial_shards} partial shard`] : []), + ...(c.cancelled_shards ? [`${c.cancelled_shards} cancelled shard`] : []), ...(c.killed_before_upload ? [`${c.killed_before_upload} slice killed before upload`] : []), + ...(!c.complete ? ['census marked incomplete'] : []), + ...panels.filter(p => p.status !== 'PASS').map(p => `${p.case} ${p.status}${p.red_class ? ` (${p.red_class})` : ''}`), + ]; + if (gaps.length) return { ...withPanels, outcome: 'INCOMPLETE', infraEligible: !panels.some(p => p.status === 'SKIPPED'), detail: gaps.join('; ') }; + if (run.conclusion !== 'success' || rec.verdict !== 'GREEN') return { ...withPanels, outcome: 'INCOMPLETE', detail: `run concluded ${run.conclusion ?? 'unknown'} with record verdict ${rec.verdict}` }; + if (!check || check.conclusion !== 'success') return { ...withPanels, outcome: 'INCOMPLETE', detail: `the run did not publish a successful check run on the train head (${check ? `${check.status}/${check.conclusion}` : 'missing'})` }; + return { ...withPanels, outcome: 'GREEN', detail: `rule ${c.executed.rule}/${c.planned.rule}, judge ${c.executed.judge}/${c.planned.judge}` }; +} + +const gateAttempts = (t: TrainRecord) => attemptsOf(t).filter(a => a.purpose !== 'bisect'); +const attemptLine = (a: AttemptRecord) => `ATTEMPT: ${a.id} ${a.purpose} run ${a.run?.id ?? '-'} ${a.outcome ?? a.state}${a.red_cases.length ? ` red=${a.red_cases.join(',')}` : ''}${a.cost_usd !== null ? ` cost=$${a.cost_usd.toFixed(2)}` : ''}${a.detail ? ` (${a.detail})` : ''}`; + +function casePrice(env: TrainEnv, red: AttemptRecord, id: string): number { + return red.panels.find(p => p.case === id)?.cost_usd ?? env.train.trial_reservation_usd; +} + +/** The whole-bisect reservation for a red attempt: every red case on every prefix. */ +export function bisectReservation(env: TrainEnv, t: TrainRecord, red: AttemptRecord): number { + const perPrefix = red.red_cases.reduce((sum, id) => sum + casePrice(env, red, id), 0); + return Math.ceil(perPrefix * t.prefixes.length * 100) / 100; +} + +function unknownDispatch(t: TrainRecord, a: AttemptRecord): TrainOut { + return outPending({ code: 'QUEUE_TRAIN_DISPATCH_UNKNOWN', message: `the ${a.purpose} dispatch ${a.id} (nonce ${a.nonce}) is unconfirmed` }, [attemptLine(a)], `${Q} train status --train ${t.nonce}`, trainFile(t.nonce)); +} + +/** `train verdict`: read the current gate attempt's run and record; print every attempt, so a rebuild reports both runs. */ +export function trainVerdict(ctx: QueueContext, o: { train: string }): TrainOut { + const env = trainEnv(ctx); + if ('code' in env) return outRefused(env, [], 'none', 'none'); + const t = findTrain(env, o.train); + if (!t) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.train} here or in refs/notes/gstack-trains` }, [], `${Q} train status`, 'none'); + const run = trainFile(t.nonce); + const a = gateAttempts(t).at(-1); + if (!a) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `train ${t.nonce} has no gate dispatch yet` }, [], nextFor(t), run); + if (a.state === 'dispatching' || a.state === 'unknown') return unknownDispatch(t, a); + if (a.state === 'absent') return outRefused({ code: 'QUEUE_TRAIN_DISPATCH_UNKNOWN', message: `the dispatch ${a.nonce} never started (train status found no run)` }, [attemptLine(a)], `${Q} train build --train ${t.nonce}`, run); + const slug = repoSlug(ctx); + if (!a.outcome || a.outcome === 'PENDING') { + const view = slug && a.run ? runView(ctx, slug, a.run.id) : null; + if (!slug || !view) return outPending({ code: 'QUEUE_TRAIN_PENDING', message: `cannot read run ${a.run?.id} through gh` }, [attemptLine(a)], `${Q} train verdict --train ${t.nonce}`, run); + const record = view.status === 'completed' ? downloadJson(ctx, slug, a.run!.id, env.train.record_artifact, path.join(trainDir(t.nonce), 'scratch')) : null; + const check = view.status === 'completed' ? checkRun(ctx, t.head, env.train.full_gate_check) : null; + const c = classifyGate(record, { nonce: a.nonce, head: t.head, tree: t.tree }, view, check); + Object.assign(a, { outcome: c.outcome, detail: c.detail, red_cases: c.red, panels: c.panels, infra_eligible: c.infraEligible, cost_usd: c.cost, state: view.status === 'completed' ? 'completed' : a.state }); + if (c.outcome !== 'PENDING') { + t.state = c.outcome === 'GREEN' ? 'green' : c.outcome === 'RED' ? (a.purpose === 'rebuild' ? 'unattributed' : 'red') : 'incomplete'; + effect(t, `verdict ${a.id}: ${c.outcome}`); + } + save(env, t, a); + } + const lines = [...gateAttempts(t).map(attemptLine), `VERDICT: ${a.outcome} train ${t.nonce} run ${a.run?.id ?? '-'} head ${t.head.slice(0, 12)} tree ${t.tree.slice(0, 12)}`]; + if (a.outcome === 'PENDING') return outPending({ code: 'QUEUE_TRAIN_PENDING', message: `the gate run ${a.run?.id} has not finished (${a.detail})` }, lines, `${Q} train verdict --train ${t.nonce}`, run); + if (a.outcome === 'GREEN') return outOk(lines, nextFor(t), run); + if (a.outcome === 'RED') { + const next = a.purpose === 'rebuild' + ? `second red stays red: route ${a.red_cases.join(',')} to a repair lane (GStack Minder), then ${Q} train build --prs ` + : `${Q} train bisect --train ${t.nonce} --spend-cap ${bisectReservation(env, t, a).toFixed(2)}`; + return outFailed({ code: 'QUEUE_TRAIN_RED', message: `train ${t.nonce} is red on ${a.red_cases.join(', ')}` }, lines, next, run); + } + const used = gateAttempts(t).filter(x => x.purpose === 'infra-redispatch').length; + const next = a.infra_eligible && used < INFRA_REDISPATCH + ? `${Q} train rebuild --train ${t.nonce} --infra (the single re-dispatch EVAL_POLICY.infraRedispatch allows; report both runs)` + : `diagnose run ${a.run?.url ?? a.run?.id}; the train stays incomplete (GStack Minder routes it)`; + return outFailed({ code: 'QUEUE_TRAIN_INCOMPLETE', message: `train ${t.nonce} gate is incomplete: ${a.detail}` }, lines, next, run); +} + +export interface BisectClass { kind: 'attributed' | 'red-on-main' | 'unattributed'; all_green: boolean; prs: string[]; patterns: Record } + +/** + * Pattern attribution over T0..Tk (one trial each, so only shapes count): + * monotone green-then-red from Ti blames slot i's PR; red everywhere, + * T0 included, is main; all green, or anything else, is unattributed. + */ +export function classifyBisect(cases: string[], results: Array>, slots: Array<{ pr: string }>): BisectClass { + const patterns: Record = {}; + const blamed = new Set(); + let main = false; let other = false; let green = 0; + for (const c of cases) { + const seq = results.map(r => (r[c] === 'red' ? 'R' : 'G')).join(''); + patterns[c] = seq; + const first = seq.indexOf('R'); + if (first < 0) green++; + else if (first === 0 && !seq.includes('G')) main = true; + else if (first > 0 && !seq.slice(first).includes('G')) blamed.add(slots[first - 1]!.pr); + else other = true; + } + if (main) return { kind: 'red-on-main', all_green: false, prs: [], patterns }; + if (other || !blamed.size) return { kind: 'unattributed', all_green: green === cases.length, prs: [], patterns }; + return { kind: 'attributed', all_green: false, prs: [...blamed].sort((a, b) => Number(a) - Number(b)), patterns }; +} + +function prefixResults(raw: unknown, sha: string, nonce: string, cases: string[]): { results: Record; cost: number } | null { + const rec = raw as BisectPrefixRecord | null; + if (!rec || rec.schema !== 1 || rec.kind !== 'train-bisect-prefix' || rec.nonce !== nonce || rec.prefix_sha !== sha || rec.checked_out_sha !== sha || !Array.isArray(rec.panels)) return null; + const results: Record = {}; + let cost = 0; + for (const p of rec.panels) { + if (!cases.includes(p.case)) continue; + cost += typeof p.cost_usd === 'number' ? p.cost_usd : 0; + if (p.status === 'PASS') results[p.case] = 'green'; + else if (p.status === 'FAIL' && p.red_class === 'VERDICT') results[p.case] = 'red'; + } + return { results, cost }; +} + +/** Collect a finished bisect dispatch's prefix artifacts into the ledger; a still-running one returns its run state. */ +function collect(env: TrainEnv, slug: string, t: TrainRecord, b: BisectRecord, a: AttemptRecord): string | null { + const view = a.run ? runView(env.ctx, slug, a.run.id) : null; + if (!view) return 'unreadable through gh'; + if (view.status !== 'completed') return view.status; + let got = 0; + for (const index of a.prefixes) { + const p = b.prefixes[index]!; + const r = prefixResults(downloadJson(env.ctx, slug, a.run!.id, bisectArtifact(env.train.record_artifact, p.sha), path.join(trainDir(t.nonce), 'scratch')), p.sha, a.nonce, b.cases); + if (!r) continue; + Object.assign(p.results, r.results); + b.spent_usd = Math.round((b.spent_usd + r.cost) * 100) / 100; + got++; + } + Object.assign(a, { state: 'completed', detail: `collected ${got} of ${a.prefixes.length} prefix result(s)` }); + return null; +} + +function bisectOutcome(env: TrainEnv, t: TrainRecord, b: BisectRecord): TrainOut { + const c = classifyBisect(b.cases, b.prefixes.map(p => p.results), t.slots); + b.outcome = { ...c, at: nowIso() }; + t.state = c.kind; + effect(t, `bisect: ${c.kind}${c.prs.length ? ' #' + c.prs.join(',#') : ''}`); + b.updated_at = nowIso(); + persist(env.ctx.repoRoot, [b], { remote: env.remote }); + save(env, t); + const run = trainFile(t.nonce); + const lines = [...Object.entries(c.patterns).map(([id, s]) => `PATTERN: ${id} ${s.split('').map((x, i) => `T${i}=${x}`).join(' ')}`), `BISECT: spent $${b.spent_usd.toFixed(2)} of cap $${b.spend_cap.toFixed(2)}`]; + if (c.kind === 'attributed') return outOk([...lines, `BISECT: ATTRIBUTED ${c.prs.map(p => '#' + p).join(',')}`], nextFor(t), run); + if (c.kind === 'red-on-main') return outFailed({ code: 'QUEUE_TRAIN_RED_ON_MAIN', message: `${b.cases.join(', ')} red on every prefix, T0 (main) included` }, [...lines, 'BISECT: RED_ON_MAIN'], nextFor(t), run); + return outFailed({ code: 'QUEUE_TRAIN_UNATTRIBUTED', message: c.all_green ? 'every prefix was green: the red is unattributed; recovery: the one preregistered rebuild' : 'the pattern is not monotone: recovery: a repair lane, then a changed train' }, + [...lines, `BISECT: UNATTRIBUTED (${c.all_green ? 'all green' : 'not monotone'})`], nextFor(t), run); +} + +/** Validate bisect inputs: red cases from the gate's own plan, every prefix an ancestor of train/'s head. */ +function bisectInputs(env: TrainEnv, t: TrainRecord, red: AttemptRecord): string | null { + for (const id of red.red_cases) { + if (!/^[A-Za-z0-9_.:/-]{1,128}$/.test(id) || !red.panels.some(p => p.case === id)) return `red case ${id} is not in the gate's plan`; + } + const ref = `refs/gstack/${t.branch}`; + const f = git(env.ctx.repoRoot, ['fetch', '--no-tags', '-q', env.remote, `+refs/heads/${t.branch}:${ref}`]); + if (!f.ok) return `cannot fetch ${t.branch}: ${f.err.slice(0, 120)}`; + const tip = git(env.ctx.repoRoot, ['rev-parse', ref]).out; + if (tip !== t.head) return `${t.branch} is at ${tip.slice(0, 12)}, not the recorded train head ${t.head.slice(0, 12)}`; + for (const p of t.prefixes) { + if (!git(env.ctx.repoRoot, ['merge-base', '--is-ancestor', p.commit, tip]).ok) return `prefix T${p.index} ${p.commit.slice(0, 12)} is not an ancestor of ${t.branch}`; + } + return null; +} + +/** + * `train bisect`: resumable and spend-capped. Each run reconciles the earlier + * dispatch by nonce, records every prefix result it published, then reserves + * the remaining case×prefix trials at their recorded price (unknown at the + * policy reservation) and launches only the prefixes that fit under the cap. + */ +export async function trainBisect(ctx: QueueContext, o: { train: string; spendCap: number; dryRun: boolean }): Promise { + const env = trainEnv(ctx); + if ('code' in env) return outRefused(env, [], 'none', 'none'); + const t = findTrain(env, o.train); + if (!t) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.train} here or in refs/notes/gstack-trains` }, [], `${Q} train status`, 'none'); + const run = trainFile(t.nonce); + const red = gateAttempts(t).at(-1); + if (!red || red.outcome !== 'RED') return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `bisect needs a red verdict; the train's last gate is ${red?.outcome ?? 'missing'}` }, [], nextFor(t), run); + if (red.purpose === 'rebuild') return outRefused({ code: 'QUEUE_PROOF_MISSING', message: 'a rebuild\'s second red is not bisected again' }, [], nextFor(t), run); + const prior = loadBisect(t.nonce); + if (prior?.outcome) return bisectOutcome(env, t, prior); + const bad = bisectInputs(env, t, red); + if (bad) return outRefused({ code: 'QUEUE_TRAIN_PREFIX_INVALID', message: bad }, [], `${Q} train status --train ${t.nonce}`, run); + const b: BisectRecord = prior ?? { schema: 1, kind: 'train-bisect', train: t.nonce, cases: red.red_cases, spend_cap: o.spendCap, spent_usd: 0, + prefixes: t.prefixes.map(p => ({ index: p.index, sha: p.commit, results: {}, attempts: [] })), outcome: null, updated_at: nowIso() }; + b.spend_cap = o.spendCap; + const slug = repoSlug(ctx); + if (!slug) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: 'gh repo view could not name the repository' }, [], `${Q} train bisect --train ${t.nonce} --spend-cap ${o.spendCap}`, run); + for (const a of attemptsOf(t).filter(x => x.purpose === 'bisect' && x.state !== 'completed' && x.state !== 'absent')) { + if (a.state === 'dispatching' || a.state === 'unknown') { + const hit = runForNonce(listDispatchRuns(a.workflow, execFor(ctx), slug, 100) ?? [], a.nonce); + if (!hit) return unknownDispatch(t, a); + Object.assign(a, { state: 'dispatched', run: asRun(hit) }); + } + const waiting = collect(env, slug, t, b, a); + save(env, t, a); + persist(ctx.repoRoot, [b], { remote: env.remote }); + if (waiting) return outPending({ code: 'QUEUE_TRAIN_PENDING', message: `bisect run ${a.run?.id} is ${waiting}` }, [attemptLine(a)], `${Q} train bisect --train ${t.nonce} --spend-cap ${o.spendCap}`, run); + } + const pending = b.prefixes.filter(p => b.cases.some(c => !p.results[c])); + if (!pending.length) return bisectOutcome(env, t, b); + const perPrefix = b.cases.reduce((sum, id) => sum + casePrice(env, red, id), 0); + const admitted: typeof pending = []; + let reserved = b.spent_usd; + for (const p of pending) if (reserved + perPrefix <= o.spendCap + 1e-9) { admitted.push(p); reserved += perPrefix; } + const higher = Math.ceil((b.spent_usd + perPrefix * pending.length) * 100) / 100; + const lines = [`BISECT: cases ${b.cases.join(',')} prefixes ${pending.map(p => 'T' + p.index).join(',')} need results; $${perPrefix.toFixed(2)} reserved per prefix; spent $${b.spent_usd.toFixed(2)}, cap $${o.spendCap.toFixed(2)}`, + `BISECT: launch ${admitted.map(p => 'T' + p.index).join(',') || '(none)'}${admitted.length < pending.length ? `; ${pending.length - admitted.length} prefix(es) wait for a higher cap` : ''}`]; + if (o.dryRun) return outOk([...lines, 'DRY-RUN: nothing dispatched'], `${Q} train bisect --train ${t.nonce} --spend-cap ${o.spendCap}`, run); + persist(ctx.repoRoot, [b], { remote: env.remote }); + if (!admitted.length) { + return outPending({ code: 'QUEUE_TRAIN_BISECT_CAPPED', message: `the next prefix needs $${perPrefix.toFixed(2)}; $${(o.spendCap - b.spent_usd).toFixed(2)} of the cap is left; the outcome stays unattributed` }, lines, + `${Q} train bisect --train ${t.nonce} --spend-cap ${higher.toFixed(2)}`, run); + } + const nonce = makeNonce('gate', t.head, t.base.sha, randomBytes(3).toString('hex'), { purpose: 'bisect' }); + const a = await dispatchAttempt(env, slug, t, 'bisect', { workflow: env.train.gate_workflow, inputs: { + [DISPATCH_INPUTS.cases]: b.cases.join(','), [DISPATCH_INPUTS.prefixShas]: admitted.map(p => p.sha).join(','), + [DISPATCH_INPUTS.expectedSha]: t.head, [DISPATCH_INPUTS.nonce]: nonce, + } }, admitted.map(p => p.index)); + for (const p of admitted) p.attempts.push(a.id); + b.updated_at = nowIso(); + persist(ctx.repoRoot, [b], { remote: env.remote }); + if (a.state !== 'dispatched') return unknownDispatch(t, a); + lines.push(attemptLine(a)); + if (admitted.length < pending.length) { + return outPending({ code: 'QUEUE_TRAIN_BISECT_CAPPED', message: `${pending.length - admitted.length} prefix(es) are past the cap; completed prefixes are kept and the outcome stays unattributed` }, lines, + `${Q} train bisect --train ${t.nonce} --spend-cap ${higher.toFixed(2)}`, run); + } + return outOk(lines, `${Q} train bisect --train ${t.nonce} --spend-cap ${o.spendCap} (collects run ${a.run?.id} once it finishes)`, run); +} + +/** + * `train rebuild`: the unchanged train dispatched once more. Without + * `--infra`, only after an all-green bisect and only once (rebuild-once); + * with `--infra`, only after an INFRA/INCOMPLETE-only census and at most + * EVAL_POLICY.infraRedispatch times. Both runs stay in the record. + */ +export async function trainRebuild(ctx: QueueContext, o: { train: string; infra: boolean }): Promise { + const env = trainEnv(ctx); + if ('code' in env) return outRefused(env, [], 'none', 'none'); + const t = findTrain(env, o.train); + if (!t) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.train} here or in refs/notes/gstack-trains` }, [], `${Q} train status`, 'none'); + const run = trainFile(t.nonce); + const gates = gateAttempts(t); + const last = gates.at(-1); + const repair = `route ${last?.red_cases.join(',') || 'the red cases'} to a repair lane (GStack Minder); the train is rebuilt only once the repair merges: ${Q} train build --prs ${pinList(t.slots)}`; + const refuse = (reason: string) => outRefused({ code: 'QUEUE_TRAIN_REBUILD_REFUSED', message: `rebuild refused: ${reason}` }, gates.map(attemptLine), repair, run); + if (o.infra) { + if (!last || last.outcome !== 'INCOMPLETE' || !last.infra_eligible) return refuse(`no INFRA/INCOMPLETE-only record (the last gate is ${last?.outcome ?? 'missing'})`); + if (gates.filter(a => a.purpose === 'infra-redispatch').length >= INFRA_REDISPATCH) return refuse('already re-dispatched once (EVAL_POLICY.infraRedispatch)'); + } else { + const b = loadBisect(t.nonce); + if (!b?.outcome) return refuse('no bisect record'); + if (gates.filter(a => a.purpose === 'rebuild').length >= REBUILD_ONCE) return refuse('already rebuilt once'); + if (b.outcome.kind !== 'unattributed' || !b.outcome.all_green) return refuse(`the bisect was not all green (${Object.entries(b.outcome.patterns).map(([c, s]) => `${c}=${s}`).join(' ')})`); + } + const tip = remoteBranchSha(args => git(ctx.repoRoot, args), env.remote, t.branch); + if (tip !== t.head) return outRefused({ code: 'QUEUE_TRAIN_MOVED', message: `${t.branch} is at ${tip?.slice(0, 12) ?? 'nothing'}, not the recorded head ${t.head.slice(0, 12)}` }, [], `${Q} train build --prs ${pinList(t.slots)}`, run); + const slug = repoSlug(ctx); + if (!slug) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: 'gh repo view could not name the repository' }, [], `${Q} train rebuild --train ${t.nonce}${o.infra ? ' --infra' : ''}`, run); + const a = await dispatchAttempt(env, slug, t, o.infra ? 'infra-redispatch' : 'rebuild', gateDispatch(env, t, `${t.nonce}.${gates.length + 1}`)); + t.state = a.state === 'dispatched' ? 'gate-pending' : 'dispatch-unknown'; + save(env, t); + if (a.state !== 'dispatched') return unknownDispatch(t, a); + return outOk([...gateAttempts(t).map(attemptLine), `REBUILD: ${a.purpose} of the unchanged train ${t.nonce} (head ${t.head.slice(0, 12)}); both runs are reported`], nextFor(t), run); +} diff --git a/lib/ship-queue-train-verify.ts b/lib/ship-queue-train-verify.ts new file mode 100644 index 000000000..7261ffa14 --- /dev/null +++ b/lib/ship-queue-train-verify.ts @@ -0,0 +1,122 @@ +/** + * `verify --pr --train `: the coordinator's merge check under + * merge trains (docs/ship-policy.md#run-a-merge-train; plan GSTA-49 T1, P1). + * + * Each merge needs a green train record that includes the PR at its pinned + * head, with main's tree equal to the prefix before the PR (pre-merge: + * MERGE_CONDITION) or after it (post-merge confirmation), modulo stamps + * (lib/tree-receipt.ts). When a train stops partway (a mismatch or a moved + * head after earlier slots merged) the queue freezes: the freeze is written + * locally and to the notes ref first, then the backstop gate is dispatched on + * main's published prefix at once. One repair train is admitted under the + * freeze; verify clears the freeze when main after that merge carries it. + */ +import { randomBytes } from 'node:crypto'; +import { DISPATCH_INPUTS, awaitNonceRun, dispatchWorkflow, makeNonce } from '../scripts/eval-bg'; +import type { QueueContext } from './ship-queue'; +import { git } from './ship-queue-gate'; +import { checkRun, execFor, repoSlug } from './ship-queue-train-github'; +import { activeFreeze, attemptsOf, nowIso, persist, trainFile, type FreezeRecord, type TrainRecord } from './ship-queue-train-record'; +import { + effect, fetchBase, fetchTrainBranch, findTrain, frozenLines, nextFor, outFailed, outOk, outRefused, pinList, save, trainEnv, type TrainEnv, type TrainOut, +} from './ship-queue-train'; +import { buildReceipt } from './tree-receipt'; + +const Q = 'gstack-ship-queue'; + +function sameModuloStamps(env: TrainEnv, gated: string, current: string): { same: boolean; detail: string } { + const r = buildReceipt({ repoRoot: env.ctx.repoRoot, gated, current, policy: env.policy.policy, policyLabel: env.policy.source.label }); + if ('error' in r) return { same: false, detail: r.error }; + return { same: r.tree === 'same-modulo-stamps', detail: r.tree === 'same-modulo-stamps' ? `same modulo stamps (${r.files.length} stamp file(s))` : `differs: ${r.residual.slice(0, 8).join(', ')}` }; +} + +/** + * Freeze the queue: the record goes to local state and the notes ref before + * anything remote, then the backstop gate is dispatched on main's sha. + */ +async function freezeQueue(env: TrainEnv, t: TrainRecord, reason: string, mainSha: string): Promise { + const now = nowIso(); + const f: FreezeRecord = { + schema: 1, kind: 'freeze', repo_key: env.repo.key, train: t.nonce, merged: t.merged.map(m => m.pr), + unmerged: t.slots.filter(s => !t.merged.some(m => m.pr === s.pr)).map(s => ({ pr: s.pr, head: s.head })), + main: { sha: mainSha, tree: git(env.ctx.repoRoot, ['rev-parse', `${mainSha}^{tree}`]).out }, reason, backstop: null, repair: null, + created_at: now, cleared_at: null, cleared_by: null, updated_at: now, + }; + persist(env.ctx.repoRoot, [f], { remote: env.remote }); + effect(t, `queue frozen: ${reason}`); + save(env, t); + const slug = repoSlug(env.ctx); + const nonce = makeNonce('gate', mainSha, mainSha, randomBytes(3).toString('hex'), { purpose: 'backstop' }); + const dispatch = { workflow: env.train.gate_workflow, inputs: { [DISPATCH_INPUTS.evalsAll]: 'true', [DISPATCH_INPUTS.expectedSha]: mainSha, [DISPATCH_INPUTS.nonce]: nonce } }; + const exec = execFor(env.ctx); + const found = slug && dispatchWorkflow(dispatch, env.ctx.base, exec, slug) ? await awaitNonceRun(dispatch.workflow, nonce, exec, slug) : null; + f.backstop = { nonce, run: found?.databaseId ?? null, state: found ? 'dispatched' : 'unconfirmed' }; + f.updated_at = nowIso(); + persist(env.ctx.repoRoot, [f], { remote: env.remote }); + return f; +} + +/** Interrupted publication: freeze when earlier slots merged; otherwise a plain refusal (nothing was published). */ +async function stopped(env: TrainEnv, t: TrainRecord, code: 'QUEUE_TRAIN_TREE_MISMATCH' | 'QUEUE_TRAIN_MOVED', message: string, mainSha: string, lines: string[]): Promise { + const run = trainFile(t.nonce); + if (!t.merged.length) return outFailed({ code, message: `${message}; nothing of train ${t.nonce} merged, so the queue is not frozen` }, lines, `${Q} train build --prs ${pinList(t.slots)} (a new train on current main)`, run); + const f = activeFreeze(env.repo.key) ?? await freezeQueue(env, t, message, mainSha); + return outFailed({ code, message: `${message}; the queue is frozen` }, [...lines, ...frozenLines(f)], `${Q} train build --repair --prs @`, run); +} + +export async function verifyTrain(ctx: QueueContext, o: { pr: string; train: string; noFetch: boolean }): Promise { + const fetchErr = o.noFetch ? null : fetchBase(ctx); + const env = trainEnv(ctx); + if ('code' in env) return outRefused(env, [], 'none', 'none'); + if (fetchErr) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: fetchErr }, [], 'none', 'none'); + const t = findTrain(env, o.train); + if (!t) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.train} here or in refs/notes/gstack-trains` }, [], `${Q} train status`, 'none'); + const run = trainFile(t.nonce); + const i = t.slots.findIndex(s => s.pr === o.pr); + if (i < 0) return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} does not include #${o.pr}` }, [], nextFor(t), run); + const gate = attemptsOf(t).filter(a => a.purpose !== 'bisect').at(-1); + if (gate?.outcome !== 'GREEN' || !['green', 'merging', 'merged'].includes(t.state)) { + return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} has no green record (state ${t.state}, last gate ${gate?.outcome ?? 'none'})` }, [], nextFor(t), run); + } + const check = checkRun(ctx, t.head, env.train.full_gate_check); + if (check?.conclusion !== 'success') return outRefused({ code: 'QUEUE_GATE_MISSING', message: `no successful ${env.train.full_gate_check} check run on the train head ${t.head.slice(0, 12)}` }, [], `${Q} train verdict --train ${t.nonce}`, run); + if (!fetchTrainBranch(env, t)) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `the train's prefix commits are not here and ${t.branch} could not be fetched` }, [], `${Q} train status --train ${t.nonce}`, run); + const main = git(ctx.repoRoot, ['rev-parse', `${env.remote}/${ctx.base}`]).out; + const freeze = activeFreeze(env.repo.key); + if (freeze && freeze.repair?.train !== t.nonce) return outRefused({ code: 'QUEUE_TRAIN_FROZEN', message: `the queue is frozen (${freeze.reason}); only the admitted repair train merges` }, frozenLines(freeze), `${Q} train build --repair --prs @`, run); + const after = t.prefixes[i + 1]!; + const before = t.prefixes[i]!; + const post = sameModuloStamps(env, after.commit, main); + const lines = [`TRAIN: ${t.nonce} slot ${i + 1}/${t.slots.length} #${o.pr} pinned ${t.slots[i]!.head}`]; + if (post.same) return confirmMerged(env, t, o.pr, main, after.index, [...lines, `main: ${main} = T${after.index} (${post.detail})`]); + const pre = sameModuloStamps(env, before.commit, main); + if (!pre.same) { + return stopped(env, t, 'QUEUE_TRAIN_TREE_MISMATCH', `main ${main.slice(0, 12)} is neither T${before.index} nor T${after.index} modulo stamps (vs T${after.index}: ${post.detail})`, main, lines); + } + const headRef = o.noFetch ? '' : (git(ctx.repoRoot, ['fetch', '--no-tags', '-q', env.remote, `+refs/pull/${o.pr}/head:refs/gstack/queue/pr-${o.pr}`]).ok ? `refs/gstack/queue/pr-${o.pr}` : ''); + const head = headRef ? git(ctx.repoRoot, ['rev-parse', headRef]).out : t.slots[i]!.head; + const stampOnly = head === t.slots[i]!.head ? { same: true, detail: 'the pinned head' } : sameModuloStamps(env, t.slots[i]!.head, head); + if (!stampOnly.same) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${o.pr} moved past its pinned head ${t.slots[i]!.head.slice(0, 12)} (${head.slice(0, 12)}: ${stampOnly.detail})`, main, lines); + return outOk([...lines, `main: ${main} = T${before.index} (${pre.detail})`, `head: ${head} (${stampOnly.detail})`, + `VERIFY_TRAIN: #${o.pr} may merge as slot ${i + 1} of train ${t.nonce} (green run ${gate.run?.id ?? '-'}, ${env.train.full_gate_check} success)`, `MERGE_CONDITION: sha=${head}`], + `merge #${o.pr} with expected head ${head.slice(0, 12)} (gstack-restamp stamps at merge), then ${Q} verify --pr ${o.pr} --train ${t.nonce}`, run); +} + +function confirmMerged(env: TrainEnv, t: TrainRecord, pr: string, main: string, index: number, lines: string[]): TrainOut { + const run = trainFile(t.nonce); + if (!t.merged.some(m => m.pr === pr)) t.merged.push({ pr, sha: main, tree: git(env.ctx.repoRoot, ['rev-parse', `${main}^{tree}`]).out, at: nowIso() }); + t.state = t.merged.length === t.slots.length ? 'merged' : 'merging'; + effect(t, `verified #${pr} merged as T${index}`); + save(env, t); + lines.push(`VERIFY_TRAIN: #${pr} merged; main's tree is T${index} of train ${t.nonce} (${t.merged.length}/${t.slots.length} merged)`); + const freeze = activeFreeze(env.repo.key); + if (freeze && freeze.repair?.train === t.nonce && t.state === 'merged') { + freeze.cleared_at = nowIso(); + freeze.cleared_by = `repair #${pr} via train ${t.nonce}; main ${main.slice(0, 12)} carries a green record`; + freeze.updated_at = freeze.cleared_at; + persist(env.ctx.repoRoot, [freeze], { remote: env.remote }); + lines.push(`FREEZE: cleared (${freeze.cleared_by})`); + return outOk(lines, freeze.unmerged.length ? `${Q} train build --prs ${pinList(freeze.unmerged)}` : 'none', run); + } + return outOk(lines, nextFor(t), run); +} diff --git a/lib/ship-queue-train.ts b/lib/ship-queue-train.ts new file mode 100644 index 000000000..69bfd195b --- /dev/null +++ b/lib/ship-queue-train.ts @@ -0,0 +1,509 @@ +/** + * Merge trains for gstack-ship-queue (docs/ship-policy.md#run-a-merge-train; + * plan GSTA-49 lane T1): `train build`, `train status` and `train prune`, + * plus the pieces the verdict, bisect, rebuild and verify modules share. + * The surface is active only when the base revision's ship policy declares a + * `train` block (QUEUE_TRAIN_OFF otherwise). + * + * A train is main plus each READY PR's pinned, unstamped head merged in slot + * order. The merges run in object storage (`git merge-tree --write-tree`, the + * gate-ahead merge rules: version files take the earlier prefix, the + * CHANGELOG merges by section), so each prefix T0..Tk is a commit with a + * fixed identity and date and the same pins always give the same train head. + * Records (lib/ship-queue-train-record.ts) are written before every remote + * effect: the branch push, the free-tests dispatch, the paid dispatch through + * scripts/eval-bg.ts's nonce. + */ +import { randomBytes } from 'node:crypto'; +import { DISPATCH_INPUTS, awaitNonceRun, dispatchWorkflow, listDispatchRuns, makeNonce, runForNonce, type WorkflowDispatch } from '../scripts/eval-bg'; +import { mergeChangelogs, parseChangelog, type ChangelogSection } from './changelog-check'; +import type { ResultStatus } from './headless-artifacts'; +import { versionAtRef, versionFiles } from './restamp'; +import type { LoadedPolicy, ShipPolicy, TrainPolicy } from './ship-policy'; +import { loadBasePolicy, type QueueContext } from './ship-queue'; +import { git, sha256, withIndex, type QueueFailure } from './ship-queue-gate'; +import { asRun, execFor, newestBranchRun, prInfo, remoteBranchSha, repoSlug, runView, checkRun } from './ship-queue-train-github'; +import { + TERMINAL_STATES, activeFreeze, attemptsOf, listTrains, loadBisect, loadReady, loadTrain, nowIso, persist, rebuildFromNotes, repoIdentity, trainFile, + type AttemptPurpose, type AttemptRecord, type TrainRecord, type TrainSlot, +} from './ship-queue-train-record'; + +export interface TrainOut { exit: number; status: ResultStatus; lines: string[]; next: string; failure?: QueueFailure; run: string } +export const outOk = (lines: string[], next: string, run: string): TrainOut => ({ exit: 0, status: 'complete', lines, next, run }); +export const outRefused = (failure: QueueFailure, lines: string[], next: string, run: string): TrainOut => ({ exit: 3, status: 'refused', lines, next, failure, run }); +export const outFailed = (failure: QueueFailure, lines: string[], next: string, run: string): TrainOut => ({ exit: 1, status: 'refused', lines, next, failure, run }); +export const outPending = (failure: QueueFailure, lines: string[], next: string, run: string): TrainOut => ({ exit: 3, status: 'incomplete', lines, next, failure, run }); + +export interface TrainEnv { ctx: QueueContext; policy: LoadedPolicy; train: TrainPolicy; repo: { key: string; url: string }; remote: string } + +const HEX = /^[0-9a-f]{40,64}$/; +const Q = 'gstack-ship-queue'; + +/** The policy gate: the base revision's ship policy must declare a `train` block. */ +export function trainEnv(ctx: QueueContext): TrainEnv | QueueFailure { + const policy = loadBasePolicy(ctx); + if ('message' in policy) return policy; + if (!policy.policy.train) return { code: 'QUEUE_TRAIN_OFF', message: `${policy.source.label} declares no train block; the per-PR queue applies` }; + return { ctx, policy, train: policy.policy.train, repo: repoIdentity(ctx.repoRoot), remote: ctx.remote ?? 'origin' }; +} + +export function fetchBase(ctx: QueueContext): string | null { + const remote = ctx.remote ?? 'origin'; + const f = git(ctx.repoRoot, ['fetch', '--no-tags', '-q', remote, `+refs/heads/${ctx.base}:refs/remotes/${remote}/${ctx.base}`]); + return f.ok ? null : `git fetch ${remote} ${ctx.base} failed: ${f.err.slice(0, 160)}`; +} + +/** Make the train's prefix commits local (another machine built them): fetch train/. */ +export function fetchTrainBranch(env: TrainEnv, train: TrainRecord): string | null { + if (git(env.ctx.repoRoot, ['cat-file', '-e', `${train.head}^{commit}`]).ok) return train.head; + const f = git(env.ctx.repoRoot, ['fetch', '--no-tags', '-q', env.remote, `+refs/heads/${train.branch}:refs/gstack/${train.branch}`]); + return f.ok ? git(env.ctx.repoRoot, ['rev-parse', `refs/gstack/${train.branch}`]).out : null; +} + +/** A train record from local state, or from the notes ref when this machine has none. */ +export function findTrain(env: TrainEnv, nonce: string): TrainRecord | null { + const local = loadTrain(nonce); + if (local) return local; + rebuildFromNotes(env.ctx.repoRoot, env.remote); + return loadTrain(nonce); +} + +export const touch = (r: T): T => { r.updated_at = nowIso(); return r; }; + +export function effect(train: TrainRecord, what: string): void { + train.effects.push({ at: nowIso(), what }); +} + +/** Save the train (and any attempt) locally and to the notes ref; a failed notes push is recorded as an effect, never hidden. */ +export function save(env: TrainEnv, train: TrainRecord, ...extra: AttemptRecord[]): string { + touch(train); + for (const a of extra) touch(a); + const r = persist(env.ctx.repoRoot, [train, ...extra], { remote: env.remote }); + if (!r.pushed) { effect(train, `notes not pushed: ${r.detail}`); persist(env.ctx.repoRoot, [train], { push: false }); } + return r.detail; +} + +export function parsePins(spec: string | undefined): Array<{ pr: string; sha: string }> | string { + if (!spec) return 'needs --prs @[,@...]'; + const pins = spec.split(',').map(s => s.trim()).filter(Boolean).map(s => /^#?(\d+)@([0-9a-f]{7,64})$/.exec(s)); + if (!pins.length || pins.some(m => !m)) return `--prs entries are @ (got ${spec})`; + const out = pins.map(m => ({ pr: m![1]!, sha: m![2]! })); + return new Set(out.map(p => p.pr)).size === out.length ? out : 'a PR appears twice in --prs'; +} + +export const pinList = (slots: Array<{ pr: string; head: string }>) => slots.map(s => `${s.pr}@${s.head}`).join(','); + +/** Slot order: screened PRs first, then unscreened, each in READY order; enough unscreened PRs form their own train. */ +export function orderSlots(slots: TrainSlot[], max: number): { train: TrainSlot[]; held: TrainSlot[]; split: boolean } { + const byReady = (a: TrainSlot, b: TrainSlot) => a.ready_at.localeCompare(b.ready_at) || Number(a.pr) - Number(b.pr); + const screened = slots.filter(s => s.screen.choice === 'screen').sort(byReady); + const unscreened = slots.filter(s => s.screen.choice !== 'screen').sort(byReady); + if (screened.length && unscreened.length >= max) return { train: screened.slice(0, max), held: [...screened.slice(max), ...unscreened], split: true }; + const all = [...screened, ...unscreened]; + return { train: all.slice(0, max), held: all.slice(max), split: false }; +} + +/** When a train under its size may dispatch: its oldest READY plus the fill time; null when it may dispatch now. */ +export function fillDue(slots: TrainSlot[], train: TrainPolicy, now: string, urgent: boolean): string | null { + if (urgent || slots.length >= train.max_size) return null; + const oldest = Math.min(...slots.map(s => Date.parse(s.ready_at))); + const due = oldest + train.fill_hours * 3_600_000; + return Date.parse(now) >= due ? null : new Date(due).toISOString(); +} + +function unionChangelog(ours: string, theirs: string): string { + const a = parseChangelog(ours); + const b = parseChangelog(theirs); + const entry = (s: ChangelogSection) => s.lines.slice(1).join('\n').replace(/\n+$/, ''); + const known = new Set(b.sections.map(entry)); + const header = [...b.header]; + while (header.length && header[header.length - 1] === '') header.pop(); + const body = [...a.sections.filter(s => !known.has(entry(s))), ...b.sections].flatMap(s => { + const lines = [...s.lines]; + while (lines.length && lines[lines.length - 1] === '') lines.pop(); + return [...lines, '']; + }); + return [...header, ...(header.length ? [''] : []), ...body].join('\n').replace(/\n+$/, '\n'); +} + +/** + * One train merge, in object storage: the PR head onto the previous prefix. + * Version files take the prefix's side; the CHANGELOG merges by section (the + * gate-ahead rule, else a union of sections, since each heading is restamped + * at merge). Any other conflict is reported with every conflicting path. + */ +export function mergeStep(repoRoot: string, pred: string, head: string, policy: ShipPolicy): { tree: string } | { conflicts: string[] } { + const r = git(repoRoot, ['merge-tree', '--write-tree', head, pred]); + const lines = r.out.split('\n'); + const tree = lines[0]?.trim() ?? ''; + if (r.status === 0 && HEX.test(tree)) return { tree }; + if (r.status !== 1 || !HEX.test(tree)) return { conflicts: [`(git merge-tree failed: ${(r.err || r.out).slice(0, 120)})`] }; + const stages = new Map>(); + for (const line of lines.slice(1)) { + if (!line.trim()) break; + const m = /^(\d{6}) ([0-9a-f]{40,64}) ([123])\t(.+)$/.exec(line); + if (!m) continue; + const entry = stages.get(m[4]!) ?? new Map(); + entry.set(Number(m[3]), m[2]!); + stages.set(m[4]!, entry); + } + const at = versionAtRef(repoRoot, pred); + const stampFiles = versionFiles(repoRoot, 'rel' in at ? at.rel : 'VERSION'); + const resolved: Array<{ path: string; blob: string }> = []; + const conflicts: string[] = []; + for (const [file, entry] of stages) { + const ours = entry.get(2); + const theirs = entry.get(3); + if (!ours || !theirs) conflicts.push(file); + else if (file === policy.changelog) { + const text = (sha: string) => git(repoRoot, ['cat-file', 'blob', sha]).out + '\n'; + const merged = mergeChangelogs(text(ours), text(theirs)) ?? unionChangelog(text(ours), text(theirs)); + const blob = git(repoRoot, ['hash-object', '-w', '--stdin'], undefined, merged); + if (blob.ok) resolved.push({ path: file, blob: blob.out }); else conflicts.push(file); + } else if (stampFiles.includes(file)) resolved.push({ path: file, blob: theirs }); + else conflicts.push(file); + } + if (conflicts.length) return { conflicts: conflicts.sort() }; + const out = withIndex<{ tree: string } | { error: string }>(repoRoot, tree, index => { + for (const { path: p, blob } of resolved) { + const mode = git(repoRoot, ['ls-tree', tree, '--', p], index).out.split(/\s/)[0] || '100644'; + if (!git(repoRoot, ['update-index', '--add', '--cacheinfo', `${mode},${blob},${p}`], index).ok) return { error: `cannot stage ${p}` }; + } + const w = git(repoRoot, ['write-tree'], index); + return w.ok ? { tree: w.out } : { error: `write-tree failed: ${w.err}` }; + }); + return 'error' in out ? { conflicts: [`(${out.error})`] } : { tree: out.tree }; +} + +const TRAIN_IDENTITY = { GIT_AUTHOR_NAME: 'gstack-ship-queue', GIT_AUTHOR_EMAIL: 'ship-queue@gstack.local', GIT_COMMITTER_NAME: 'gstack-ship-queue', + GIT_COMMITTER_EMAIL: 'ship-queue@gstack.local', GIT_AUTHOR_DATE: '2000-01-01T00:00:00Z', GIT_COMMITTER_DATE: '2000-01-01T00:00:00Z' }; + +/** Build T1..Tk on T0 = base; a conflict stops at the PR that conflicts. */ +export function buildChain(repoRoot: string, baseSha: string, slots: TrainSlot[], policy: ShipPolicy): + { prefixes: TrainRecord['prefixes'] } | { stoppedAt: string; conflicts: string[]; merged: string[] } { + const prefixes: TrainRecord['prefixes'] = [{ index: 0, commit: baseSha, tree: git(repoRoot, ['rev-parse', `${baseSha}^{tree}`]).out, pr: null }]; + for (const [i, slot] of slots.entries()) { + const pred = prefixes[i]!.commit; + const m = mergeStep(repoRoot, pred, slot.head, policy); + if ('conflicts' in m) return { stoppedAt: slot.pr, conflicts: m.conflicts, merged: slots.slice(0, i).map(s => s.pr) }; + const c = git(repoRoot, ['commit-tree', m.tree, '-p', pred, '-p', slot.head, '-m', `train T${i + 1}: merge #${slot.pr} at ${slot.head.slice(0, 12)}`], { ...process.env, ...TRAIN_IDENTITY }); + if (!c.ok) return { stoppedAt: slot.pr, conflicts: [`(commit-tree failed: ${c.err.slice(0, 120)})`], merged: slots.slice(0, i).map(s => s.pr) }; + prefixes.push({ index: i + 1, commit: c.out, tree: m.tree, pr: slot.pr }); + } + return { prefixes }; +} + +/** NEXT for a train in its current state. */ +export function nextFor(t: TrainRecord): string { + const n = t.nonce; + const unmerged = t.slots.filter(s => !t.merged.some(m => m.pr === s.pr)); + switch (t.state) { + case 'built': case 'pushed': case 'free-pending': case 'free-green': case 'dispatch-absent': return `${Q} train build --train ${n}`; + case 'dispatch-unknown': return `${Q} train status --train ${n}`; + case 'gate-pending': return `${Q} train verdict --train ${n}`; + case 'green': case 'merging': return unmerged.length ? `${Q} verify --pr ${unmerged[0]!.pr} --train ${n}` : `${Q} train prune --train ${n}`; + case 'red': return `${Q} train bisect --train ${n} --spend-cap `; + case 'incomplete': return `${Q} train verdict --train ${n}`; + case 'attributed': { + const drop = loadBisect(n)?.outcome?.prs ?? []; + const keep = t.slots.filter(s => !drop.includes(s.pr)); + return keep.length ? `${Q} train build --prs ${pinList(keep)}` : 'none (every PR was dropped)'; + } + case 'red-on-main': return `fix forward on main first, then ${Q} train build --prs ${pinList(t.slots)}`; + case 'unattributed': return loadBisect(n)?.outcome?.all_green && !attemptsOf(t).some(a => a.purpose === 'rebuild') + ? `${Q} train rebuild --train ${n}` + : `route the red cases to a repair lane (GStack Minder); once the repair merges: ${Q} train build --prs ${pinList(t.slots)}`; + default: return `${Q} train prune --train ${n}`; + } +} + +export function attemptId(train: TrainRecord, purpose: AttemptPurpose): string { + const bisect = purpose === 'bisect'; + return `${bisect ? 'b' : 'a'}${train.attempts.filter(id => id.startsWith(bisect ? 'b' : 'a')).length + 1}`; +} + +/** + * Record an attempt as dispatching (locally and in the notes ref), dispatch + * through eval-bg, then find the run by its nonce. A failed or unconfirmed + * dispatch is `unknown`: train status reconciles it, nothing re-dispatches it. + */ +export async function dispatchAttempt(env: TrainEnv, slug: string, train: TrainRecord, purpose: AttemptPurpose, dispatch: WorkflowDispatch, prefixes: number[] = []): Promise { + const now = nowIso(); + const attempt: AttemptRecord = { + schema: 1, kind: 'train-attempt', train: train.nonce, id: attemptId(train, purpose), purpose, nonce: dispatch.inputs[DISPATCH_INPUTS.nonce]!, + workflow: dispatch.workflow, ref: train.branch, inputs: dispatch.inputs, state: 'dispatching', run: null, outcome: null, detail: null, + red_cases: [], panels: [], infra_eligible: false, cost_usd: null, prefixes, created_at: now, updated_at: now, + }; + train.attempts.push(attempt.id); + save(env, train, attempt); + const exec = execFor(env.ctx); + const found = dispatchWorkflow(dispatch, train.branch, exec, slug) ? await awaitNonceRun(dispatch.workflow, attempt.nonce, exec, slug) : null; + attempt.state = found ? 'dispatched' : 'unknown'; + attempt.run = found ? asRun(found) : null; + effect(train, found ? `${purpose} dispatched: ${dispatch.workflow} run ${found.databaseId} (nonce ${attempt.nonce})` : `${purpose} dispatch unconfirmed (nonce ${attempt.nonce})`); + save(env, train, attempt); + return attempt; +} + +export function gateDispatch(env: TrainEnv, train: TrainRecord, nonce: string): WorkflowDispatch { + return { workflow: env.train.gate_workflow, inputs: { [DISPATCH_INPUTS.evalsAll]: 'true', [DISPATCH_INPUTS.expectedSha]: train.head, [DISPATCH_INPUTS.nonce]: nonce } }; +} + +export interface BuildOptions { pins?: string; resume?: string; dryRun: boolean; urgent: boolean; repair: boolean; freeTimeoutSec: number; pollMs: number } + +/** The slots a new train would carry: trusted, unmoved, READY PRs, in slot order. */ +function admitSlots(env: TrainEnv, pins: Array<{ pr: string; sha: string }>): { slots: TrainSlot[]; lines: string[] } | TrainOut { + const { ctx } = env; + const specs = pins.map(p => `+refs/pull/${p.pr}/head:refs/gstack/train/pr-${p.pr}`); + const f = git(ctx.repoRoot, ['fetch', '--no-tags', '-q', env.remote, ...specs]); + if (!f.ok) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `git fetch of the PR heads failed: ${f.err.slice(0, 160)}` }, [], 'none', 'none'); + const slots: TrainSlot[] = []; + for (const p of pins) { + const info = prInfo(ctx, p.pr); + if ('error' in info) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: info.error }, [], 'none', 'none'); + const without = pinList(pins.filter(x => x.pr !== p.pr).map(x => ({ pr: x.pr, head: x.sha }))); + if (info.isCrossRepository) return outRefused({ code: 'QUEUE_TRAIN_UNTRUSTED', message: `PR #${p.pr}'s head repository is not the base repository; fork work stays on the trusted-branch path` }, [], without ? `${Q} train build --prs ${without}` : 'none', 'none'); + const fetched = git(ctx.repoRoot, ['rev-parse', `refs/gstack/train/pr-${p.pr}`]).out; + if (!info.headRefOid.startsWith(p.sha) || fetched !== info.headRefOid) { + return outRefused({ code: 'QUEUE_TRAIN_MOVED', message: `PR #${p.pr} moved: pinned ${p.sha.slice(0, 12)}, head is now ${info.headRefOid.slice(0, 12)}` }, [], `ask #${p.pr}'s lane for READY on ${info.headRefOid.slice(0, 12)}, then ${Q} train build --prs `, 'none'); + } + const ready = loadReady(env.repo.key, p.pr, info.headRefOid); + if (!ready) return outRefused({ code: 'QUEUE_NOT_READY', message: `no READY receipt for #${p.pr} at ${info.headRefOid.slice(0, 12)} (local state or ${'refs/notes/gstack-trains'})` }, [], `the lane runs ${Q} ready --pr ${p.pr} --screen|--no-screen`, 'none'); + slots.push({ pr: p.pr, head: info.headRefOid, ready_at: ready.ready_at, screen: ready.screen }); + } + return { slots, lines: [] }; +} + +/** Start a new train from --prs, or find the live train with the same pins and base. */ +async function newTrain(env: TrainEnv, o: BuildOptions): Promise<{ t: TrainRecord; lines: string[] } | TrainOut> { + const pins = parsePins(o.pins); + if (typeof pins === 'string') return outRefused({ code: 'QUEUE_PROOF_MISSING', message: pins }, [], 'none', 'none'); + if ((o.urgent || o.repair) && pins.length !== 1) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `${o.repair ? '--repair' : '--urgent'} admits exactly one PR` }, [], 'none', 'none'); + rebuildFromNotes(env.ctx.repoRoot, env.remote); + const freeze = activeFreeze(env.repo.key); + if (freeze && !o.repair) return outRefused({ code: 'QUEUE_TRAIN_FROZEN', message: `the queue is frozen since ${freeze.created_at}: train ${freeze.train} stopped after merging ${freeze.merged.map(p => '#' + p).join(',')} (${freeze.reason})` }, + frozenLines(freeze), `${Q} train build --repair --prs @`, 'none'); + const admitted = admitSlots(env, pins); + if ('exit' in admitted) return admitted; + const order = orderSlots(admitted.slots, o.repair ? 1 : env.train.max_size); + const lines = [ + `ORDER: ${order.train.map((s, i) => `${i + 1}:#${s.pr}(${s.screen.choice === 'screen' ? 'screened' : 'unscreened'})`).join(' ')}`, + ...(order.held.length ? [`HELD: ${order.held.map(s => '#' + s.pr).join(',')} (${order.split ? 'unscreened PRs form their own train' : `over the train size ${env.train.max_size}`})`] : []), + ]; + const due = fillDue(order.train, env.train, nowIso(), o.urgent || o.repair); + if (due && !o.dryRun) return outPending({ code: 'QUEUE_TRAIN_WAITING', message: `${order.train.length} of ${env.train.max_size} PRs; the oldest READY dispatches at ${due}` }, lines, `${Q} train build --prs ${pinList(order.train)} (after ${due}, or with more PRs, or --urgent for one)`, 'none'); + const baseSha = git(env.ctx.repoRoot, ['rev-parse', `${env.remote}/${env.ctx.base}`]).out; + const key = sha256(JSON.stringify({ base: baseSha, slots: order.train.map(s => `${s.pr}@${s.head}`) })); + const live = listTrains(env.repo.key).find(t => t.key === key && !TERMINAL_STATES.includes(t.state) && t.state !== 'red' && t.state !== 'unattributed'); + if (live) return { t: Object.assign(live, { held: order.held.map(s => s.pr) }), lines }; + const chain = buildChain(env.ctx.repoRoot, baseSha, order.train, env.policy.policy); + if ('stoppedAt' in chain) { + const without = order.train.filter(s => s.pr !== chain.stoppedAt); + return outFailed({ code: 'QUEUE_TRAIN_CONFLICT', message: `#${chain.stoppedAt} conflicts with the prefix before it` }, [...lines, + `QUEUE_TRAIN_CONFLICT: train ${key.slice(0, 12)} stopped at #${chain.stoppedAt} (conflicts: ${chain.conflicts.join(', ')}) after merging ${chain.merged.map(p => '#' + p).join(',') || '(none)'}; nothing dispatched, no branch pushed; fix: ${Q} train build --prs ${pinList(without)}`], + without.length ? `${Q} train build --prs ${pinList(without)}` : 'none', 'none'); + } + const head = chain.prefixes.at(-1)!; + const nonce = makeNonce('gate', head.commit, baseSha, randomBytes(3).toString('hex'), { purpose: 'train' }); + const now = nowIso(); + return { lines, t: { + schema: 1, kind: 'train', nonce, key, repo_key: env.repo.key, repo: env.repo.url, base: { ref: env.ctx.base, sha: baseSha }, policy: env.policy.source.label, + slots: order.train, held: order.held.map(s => s.pr), prefixes: chain.prefixes, head: head.commit, tree: head.tree, branch: `train/${nonce}`, + urgent: o.urgent, repair: o.repair, state: 'built', free: null, attempts: [], merged: [], superseded_by: null, effects: [], error: null, + created_at: now, updated_at: now, + } }; +} + +export function frozenLines(f: NonNullable>): string[] { + return [ + `QUEUE_TRAIN_FROZEN: train ${f.train} merged ${f.merged.map(p => '#' + p).join(',') || '(none)'}; main ${f.main.sha.slice(0, 12)} tree ${f.main.tree.slice(0, 12)}; backstop ${f.backstop ? `${f.backstop.state} run ${f.backstop.run ?? 'unknown'} (nonce ${f.backstop.nonce})` : 'not dispatched'}`, + `freeze: ${f.reason}; unmerged ${f.unmerged.map(u => '#' + u.pr).join(',') || '(none)'}; repair ${f.repair ? `#${f.repair.pr}@${f.repair.head.slice(0, 12)}` : 'none admitted'}`, + ]; +} + +function describe(t: TrainRecord): string[] { + return [ + `TRAIN: ${t.nonce} state=${t.state} prs=${t.slots.map(s => '#' + s.pr).join(',')} head=${t.head} tree=${t.tree}`, + `base: origin/${t.base.ref}@${t.base.sha}`, + ...t.prefixes.map(p => `PREFIX: T${p.index} ${p.commit} tree=${p.tree}${p.pr ? ` (+#${p.pr})` : ' (main)'}`), + ...t.slots.map((s, i) => `SLOT: ${i + 1} #${s.pr} head=${s.head} ${s.screen.choice === 'screen' ? `screened (${s.screen.policy ?? 'narrow'}${s.screen.run ? `, run ${s.screen.run}` : ''})` : 'unscreened'} ready_at=${s.ready_at}`), + ]; +} + +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + +/** Free tests on train/: dispatched once, then followed until a verdict on the exact train head. */ +async function freePhase(env: TrainEnv, slug: string, t: TrainRecord, o: BuildOptions): Promise { + if (t.free?.verdict === 'green' && t.free.head_sha === t.head) return null; + const run = trainFile(t.nonce); + if (t.state !== 'free-pending') { + if (!env.ctx.gh(['workflow', 'run', env.train.free_workflow, '--repo', slug, '--ref', t.branch]).ok) { + t.free = { run: null, url: null, head_sha: null, conclusion: null, verdict: 'missing' }; + t.state = 'free-red'; + effect(t, `${env.train.free_workflow} dispatch refused by gh`); + save(env, t); + return outFailed({ code: 'QUEUE_TRAIN_FREE_RED', message: `gh could not dispatch ${env.train.free_workflow} on ${t.branch}` }, describe(t), `${Q} train build --prs ${pinList(t.slots)}`, run); + } + t.state = 'free-pending'; + effect(t, `${env.train.free_workflow} dispatched on ${t.branch}`); + save(env, t); + } + const deadline = Date.now() + o.freeTimeoutSec * 1000; + let found = newestBranchRun(env.ctx, slug, env.train.free_workflow, t.branch); + while ((found === null || found === 'unreadable' || found.status !== 'completed') && Date.now() < deadline) { + await sleep(o.pollMs); + found = newestBranchRun(env.ctx, slug, env.train.free_workflow, t.branch); + } + if (found && found !== 'unreadable' && found.status !== 'completed') { + t.free = { run: found.databaseId, url: found.url, head_sha: found.headSha, conclusion: null, verdict: 'pending' }; + save(env, t); + return outPending({ code: 'QUEUE_TRAIN_PENDING', message: `free tests run ${found.databaseId} on ${t.branch} is still ${found.status}` }, describe(t), `${Q} train build --train ${t.nonce}`, run); + } + const r = found && found !== 'unreadable' ? found : null; + const verdict = !r ? 'missing' : r.event !== 'workflow_dispatch' || r.headSha !== t.head ? 'stale' : r.conclusion === 'success' ? 'green' : r.conclusion === 'cancelled' ? 'cancelled' : 'red'; + t.free = { run: r?.databaseId ?? null, url: r?.url ?? null, head_sha: r?.headSha ?? null, conclusion: r?.conclusion ?? null, verdict }; + t.state = verdict === 'green' ? 'free-green' : 'free-red'; + save(env, t); + if (verdict === 'green') return null; + const why = verdict === 'missing' ? `no ${env.train.free_workflow} dispatch run appeared on ${t.branch}` + : verdict === 'stale' ? `run ${r!.databaseId} tested ${(r!.headSha || '?').slice(0, 12)} (${r!.event}), not the train head ${t.head.slice(0, 12)}` + : `run ${r!.databaseId} concluded ${r!.conclusion}`; + return outFailed({ code: 'QUEUE_TRAIN_FREE_RED', message: `free tests ${verdict}: ${why}` }, [...describe(t), `FREE: ${verdict} ${r?.url ?? ''}`.trim()], + `read the free-tests run; drop a PR that caused it: ${Q} train build --prs `, run); +} + +/** Push the branch, gate it with the free tests, then dispatch the paid gate once. */ +async function advance(env: TrainEnv, t: TrainRecord, o: BuildOptions, lines: string[]): Promise { + const run = trainFile(t.nonce); + const slug = repoSlug(env.ctx); + if (!slug) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: 'gh repo view could not name the repository' }, lines, `${Q} train build --train ${t.nonce}`, run); + if (t.state === 'built') { + const p = git(env.ctx.repoRoot, ['push', '-q', env.remote, `${t.head}:refs/heads/${t.branch}`]); + if (!p.ok) { t.error = `branch push failed: ${p.err.slice(0, 160)}`; save(env, t); return outRefused({ code: 'QUEUE_PROOF_MISSING', message: t.error }, lines, `${Q} train build --train ${t.nonce}`, run); } + t.state = 'pushed'; + effect(t, `branch pushed: ${t.branch} at ${t.head.slice(0, 12)}`); + save(env, t); + } + const free = await freePhase(env, slug, t, o); + if (free) return { ...free, lines: [...lines, ...free.lines] }; + if (t.state === 'dispatch-unknown') return outPending({ code: 'QUEUE_TRAIN_DISPATCH_UNKNOWN', message: `the gate dispatch for ${t.nonce} is unconfirmed` }, [...lines, ...describe(t)], `${Q} train status --train ${t.nonce}`, run); + if (t.state === 'free-green' || t.state === 'dispatch-absent') { + const prior = attemptsOf(t).filter(a => a.purpose === 'gate'); + const nonce = prior.at(-1)?.state === 'absent' ? prior.at(-1)!.nonce : t.nonce; + const a = await dispatchAttempt(env, slug, t, 'gate', gateDispatch(env, t, nonce)); + t.state = a.state === 'dispatched' ? 'gate-pending' : 'dispatch-unknown'; + save(env, t); + if (a.state !== 'dispatched') return outPending({ code: 'QUEUE_TRAIN_DISPATCH_UNKNOWN', message: `no ${a.workflow} run named ${a.nonce} appeared after the dispatch` }, [...lines, ...describe(t)], `${Q} train status --train ${t.nonce}`, run); + } + const gate = attemptsOf(t).filter(a => a.purpose !== 'bisect').at(-1); + return outOk([...lines, ...describe(t), `FREE: green run ${t.free?.run ?? '?'} on ${t.head.slice(0, 12)}`, + `GATE: ${gate?.workflow ?? env.train.gate_workflow} run ${gate?.run?.id ?? '?'} ${gate?.run?.url ?? ''} nonce ${gate?.nonce ?? t.nonce}`.trim(), `record: ${run}`], nextFor(t), run); +} + +/** `train build`: admit, order, merge, record, push, free tests, dispatch; `--dry-run` spends nothing and writes nothing. */ +export async function buildTrain(ctx: QueueContext, o: BuildOptions): Promise { + const fetchErr = fetchBase(ctx); + const env = trainEnv(ctx); + if ('code' in env) return outRefused(env, [], 'none', 'none'); + if (fetchErr) return outRefused({ code: 'POLICY_SOURCE_UNAVAILABLE', message: fetchErr }, [], 'none', 'none'); + let t: TrainRecord | TrainOut; + let lines: string[] = []; + if (o.resume) { + t = findTrain(env, o.resume) ?? outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.resume} here or in refs/notes/gstack-trains` }, [], `${Q} train status`, 'none'); + if ('kind' in t && TERMINAL_STATES.includes(t.state)) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `train ${t.nonce} is ${t.state}; it never dispatches again` }, describe(t), nextFor(t), trainFile(t.nonce)); + if ('kind' in t && t.state !== 'free-pending' && t.state !== 'dispatch-absent' && t.state !== 'dispatch-unknown' && t.state !== 'built' && t.state !== 'pushed' && t.state !== 'free-green') { + return outOk(describe(t), nextFor(t), trainFile(t.nonce)); + } + } else { + const made = await newTrain(env, o); + if ('exit' in made) return made; + ({ t, lines } = made); + } + if (!('kind' in t)) return t; + if (o.dryRun) { + return outOk([...lines, ...describe(t), `ESTIMATE: one ${env.train.gate_workflow} full gate (evals_all) on T${t.prefixes.length - 1} plus ${env.train.free_workflow}; dollar figure pending the T0 cost replay (lib/gate-cost.ts)`, + 'DRY-RUN: nothing written, pushed or dispatched'], `${Q} train build --prs ${pinList(t.slots)}`, 'none'); + } + if (t.state === 'built' && !t.attempts.length && !loadTrain(t.nonce)) { + effect(t, 'train.json written'); + save(env, t); + for (const old of listTrains(env.repo.key)) { + if (old.nonce === t.nonce || TERMINAL_STATES.includes(old.state) || old.state === 'green' || old.state === 'merging') continue; + if (!old.slots.some(s => t.slots.some((n: TrainSlot) => n.pr === s.pr))) continue; + old.state = 'superseded'; + old.superseded_by = t.nonce; + save(env, old); + } + if (o.repair) { + const f = activeFreeze(env.repo.key)!; + f.repair = { pr: t.slots[0]!.pr, head: t.slots[0]!.head, train: t.nonce }; + f.updated_at = nowIso(); + persist(env.ctx.repoRoot, [f], { remote: env.remote }); + } + } + return advance(env, t, o, lines); +} + +/** + * `train status`: never spends. Rebuilds local records from the notes ref, + * reconciles each unconfirmed dispatch by its nonce (found, or absent when a + * full listing shows no such run) and reads run and check state. + */ +export function trainStatus(ctx: QueueContext, o: { train?: string }): TrainOut { + const env = trainEnv(ctx); + if ('code' in env) return outRefused(env, [], 'none', 'none'); + const restored = rebuildFromNotes(ctx.repoRoot, env.remote); + const lines = [`NOTES: read ${restored.read} record(s), restored ${restored.restored} (${restored.detail})`]; + const trains = o.train ? [loadTrain(o.train)].filter((t): t is TrainRecord => !!t) : listTrains(env.repo.key).filter(t => !TERMINAL_STATES.includes(t.state)); + if (o.train && !trains.length) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.train} here or in refs/notes/gstack-trains` }, lines, `${Q} train status`, 'none'); + const slug = trains.length ? repoSlug(ctx) : null; + const exec = execFor(ctx); + for (const t of trains) { + lines.push(...describe(t)); + for (const a of attemptsOf(t)) { + if (slug && (a.state === 'dispatching' || a.state === 'unknown')) { + const runs = listDispatchRuns(a.workflow, exec, slug, 100); + const hit = runs ? runForNonce(runs, a.nonce) : null; + if (runs) { + a.state = hit ? 'dispatched' : 'absent'; + a.run = hit ? asRun(hit) : null; + if (t.state === 'dispatch-unknown') t.state = hit ? (a.purpose === 'bisect' ? 'red' : 'gate-pending') : 'dispatch-absent'; + effect(t, `status reconciled ${a.id} (nonce ${a.nonce}): ${hit ? `run ${hit.databaseId}` : 'absent'}`); + save(env, t, a); + } + } + const view = slug && a.run && !a.outcome ? runView(ctx, slug, a.run.id) : null; + lines.push(`ATTEMPT: ${a.id} ${a.purpose} state=${a.state} nonce=${a.nonce} run=${a.run?.id ?? '-'}${view ? ` ${view.status}/${view.conclusion ?? '-'}` : ''} outcome=${a.outcome ?? '-'}`); + } + if (slug && (t.state === 'green' || t.state === 'merging')) { + const c = checkRun(ctx, t.head, env.train.full_gate_check); + lines.push(`CHECK: ${env.train.full_gate_check} on ${t.head.slice(0, 12)}: ${c ? `${c.status}/${c.conclusion ?? '-'}` : 'missing'}`); + } + lines.push(`NEXT(${t.nonce}): ${nextFor(t)}`); + } + if (!trains.length) lines.push('TRAIN: none live'); + const freeze = activeFreeze(env.repo.key); + if (freeze) lines.push(...frozenLines(freeze)); + const single = trains.length === 1 ? trains[0]! : null; + const next = freeze && !freeze.repair ? `${Q} train build --repair --prs @` : single ? nextFor(single) : trains.length ? 'see each NEXT() line' : 'none'; + return outOk(lines, next, single ? trainFile(single.nonce) : 'none'); +} + +/** `train prune`: delete train/ on the remote for every finished (merged, superseded, dropped) train. */ +export function pruneTrains(ctx: QueueContext, o: { dryRun: boolean; train?: string }): TrainOut { + const env = trainEnv(ctx); + if ('code' in env) return outRefused(env, [], 'none', 'none'); + rebuildFromNotes(ctx.repoRoot, env.remote); + const lines: string[] = []; + const rgit = (args: string[]) => git(ctx.repoRoot, args); + for (const t of listTrains(env.repo.key).filter(x => !o.train || x.nonce === o.train)) { + if (!TERMINAL_STATES.includes(t.state)) { lines.push(`KEEP: ${t.branch} (${t.state})`); continue; } + if (!remoteBranchSha(rgit, env.remote, t.branch)) { lines.push(`GONE: ${t.branch} (${t.state})`); continue; } + if (o.dryRun) { lines.push(`WOULD-DELETE: ${t.branch} (${t.state})`); continue; } + const d = git(ctx.repoRoot, ['push', '-q', env.remote, '--delete', `refs/heads/${t.branch}`]); + lines.push(d.ok ? `DELETED: ${t.branch} (${t.state})` : `FAILED: ${t.branch}: ${d.err.slice(0, 120)}`); + if (d.ok) { effect(t, `branch deleted: ${t.branch}`); save(env, t); } + } + if (!lines.length) lines.push('PRUNE: no trains'); + return outOk(lines, o.dryRun ? `${Q} train prune` : 'none', 'none'); +} diff --git a/scripts/eval-bg.ts b/scripts/eval-bg.ts index 44054be12..f198b8e32 100644 --- a/scripts/eval-bg.ts +++ b/scripts/eval-bg.ts @@ -45,7 +45,23 @@ export const UNRECORDED_SHARD_MS = 600_000; /** How long the dispatch follower may hold the lock: the longest CI lane plus queueing. */ export const DISPATCH_FOLLOW_SECONDS = 4 * 60 * 60; -export interface WorkflowDispatch { workflow: 'evals.yml' | 'evals-periodic.yml'; inputs: Record } +export interface WorkflowDispatch { workflow: string; inputs: Record } + +/** + * Every workflow_dispatch input name eval-bg and gstack-ship-queue send, in + * one place. The train inputs (cases, prefix_shas) and the screen inputs + * (screen_pr, screen_merge_sha, screen_policy) are the interface the + * workflow side (plan GSTA-49, lane T1b) adds to evals.yml. + */ +export const DISPATCH_INPUTS = { + evalsAll: 'evals_all', expectedSha: 'expected_sha', nonce: 'nonce', baseRef: 'base_ref', baseSha: 'base_sha', prReceipts: 'pr_receipts', + cases: 'cases', prefixShas: 'prefix_shas', screenPr: 'screen_pr', screenMergeSha: 'screen_merge_sha', screenPolicy: 'screen_policy', +} as const; + +/** What a dispatch is for, beside its lane: the screen policy keeps a broad screen from adopting a narrow run. */ +export type DispatchPurpose = 'screen' | 'train' | 'bisect' | 'backstop'; +export type ScreenPolicy = 'narrow' | 'broad'; +export interface NonceScope { purpose: DispatchPurpose; policy?: ScreenPolicy } interface LaneSpec { /** The package script the local backend runs. */ @@ -65,13 +81,14 @@ export const LANE_SPECS: Record = { /** The workflows a lane dispatches and their inputs (names agreed with evals.yml's workflow_dispatch). */ export function dispatchPlan(lane: EvalBgLane, ids: { sha: string; baseRef: string; baseSha: string; nonce: string; prNumber?: number | null }): WorkflowDispatch[] { + const I = DISPATCH_INPUTS; const evals = (all: boolean): WorkflowDispatch => ({ workflow: 'evals.yml', inputs: { - evals_all: String(all), expected_sha: ids.sha, nonce: ids.nonce, - ...(all ? {} : { base_ref: ids.baseRef, base_sha: ids.baseSha }), + [I.evalsAll]: String(all), [I.expectedSha]: ids.sha, [I.nonce]: ids.nonce, + ...(all ? {} : { [I.baseRef]: ids.baseRef, [I.baseSha]: ids.baseSha }), // CEO-29/ENG-8: a PR lane run on a ref with an open same-repo PR reads that PR's receipts read-only. - ...(!all && ids.prNumber ? { pr_receipts: String(ids.prNumber) } : {}), + ...(!all && ids.prNumber ? { [I.prReceipts]: String(ids.prNumber) } : {}), } }); - const periodic: WorkflowDispatch = { workflow: 'evals-periodic.yml', inputs: { expected_sha: ids.sha, nonce: ids.nonce } }; + const periodic: WorkflowDispatch = { workflow: 'evals-periodic.yml', inputs: { [I.expectedSha]: ids.sha, [I.nonce]: ids.nonce } }; if (lane === 'pr') return [evals(false)]; if (lane === 'gate') return [evals(true)]; if (lane === 'periodic') return [periodic]; @@ -163,12 +180,25 @@ export function exitStatusWord(code: string): 'passed' | 'failed' | 'cancelled' return code === '130' ? 'cancelled' : 'failed'; } -/** Nonce: lane, tested revision and base, plus a random suffix; the dedupe key is everything but the suffix. */ -export function makeNonce(lane: EvalBgLane, sha: string, baseSha: string, random = Math.random().toString(36).slice(2, 8)): string { - return `${dedupeKey(lane, sha, baseSha)}${random}`; +/** + * Nonce: lane, tested revision and base, plus a random suffix; the dedupe key + * is everything but the suffix. A scoped dispatch (a screen, a train, a + * bisect, a backstop) adds its purpose and screen policy to the lane field + * (`pr.screen.broad-…`), so runs of another purpose or policy never match + * its key and the plain lane keys are unchanged. + */ +export function makeNonce(lane: EvalBgLane, sha: string, baseSha: string, random = Math.random().toString(36).slice(2, 8), scope?: NonceScope): string { + return `${dedupeKey(lane, sha, baseSha, scope)}${random}`; } -export function dedupeKey(lane: EvalBgLane, sha: string, baseSha: string): string { - return `${lane}-${sha.slice(0, 12)}-${(baseSha || 'none').slice(0, 12)}-`; +export function dedupeKey(lane: EvalBgLane, sha: string, baseSha: string, scope?: NonceScope): string { + const field = scope ? [lane, scope.purpose, ...(scope.policy ? [scope.policy] : [])].join('.') : lane; + return `${field}-${sha.slice(0, 12)}-${(baseSha || 'none').slice(0, 12)}-`; +} + +/** The scope a nonce was minted with (null for a plain lane nonce). */ +export function nonceScope(nonce: string): NonceScope | null { + const m = /^(?:pr|gate|periodic|release)\.(screen|train|bisect|backstop)(?:\.(narrow|broad))?-[0-9a-f]{1,12}-/.exec(nonce); + return m ? { purpose: m[1] as DispatchPurpose, ...(m[2] ? { policy: m[2] as ScreenPolicy } : {}) } : null; } export interface RunSummary { databaseId: number; displayTitle: string; status: string; conclusion: string | null; url: string; event: string } @@ -186,7 +216,39 @@ export function findDuplicateRun(runs: readonly RunSummary[], key: string): RunS // ─── Process plumbing (git / gh / detach) ──────────────────────────────────── -type Exec = (command: string, args: string[]) => { status: number | null; stdout: string }; +export type Exec = (command: string, args: string[]) => { status: number | null; stdout: string }; + +const RUN_FIELDS = 'databaseId,displayTitle,status,conclusion,url,event'; + +/** Recent workflow_dispatch runs of one workflow; null when gh could not list them (unknown, never "none"). */ +export function listDispatchRuns(workflow: string, run: Exec, repo = CANONICAL_REPO, limit = 30): RunSummary[] | null { + const out = run('gh', ['run', 'list', '--repo', repo, '--workflow', workflow, '--event', 'workflow_dispatch', '--limit', String(limit), '--json', RUN_FIELDS]); + if (out.status !== 0) return null; + try { const parsed = JSON.parse(out.stdout); return Array.isArray(parsed) ? parsed as RunSummary[] : null; } catch { return null; } +} + +/** The dispatch run whose run-name carries exactly this nonce. */ +export function runForNonce(runs: readonly RunSummary[], nonce: string): RunSummary | null { + return runs.find(item => item.event === 'workflow_dispatch' && item.displayTitle.includes(`eval-bg ${nonce}]`)) ?? null; +} + +/** `gh workflow run` with the plan's inputs on `ref`; false when gh refused it. */ +export function dispatchWorkflow(dispatch: WorkflowDispatch, ref: string, run: Exec, repo = CANONICAL_REPO): boolean { + const args = ['workflow', 'run', dispatch.workflow, '--repo', repo, '--ref', ref, + ...Object.entries(dispatch.inputs).flatMap(([name, value]) => ['-f', `${name}=${value}`])]; + return run('gh', args).status === 0; +} + +/** Poll for the run a dispatch started, by its nonce, for up to ten poll intervals; null when none appeared. */ +export async function awaitNonceRun(workflow: string, nonce: string, run: Exec, repo = CANONICAL_REPO): Promise { + const deadline = Date.now() + 10 * pollMs(); + while (Date.now() < deadline) { + await sleep(Math.min(pollMs(), 10_000)); + const found = runForNonce(listDispatchRuns(workflow, run, repo) ?? [], nonce); + if (found) return found; + } + return null; +} const exec: Exec = (command, args) => { const result = spawnSync(command, args, { cwd: ROOT, encoding: 'utf8', timeout: 60_000, maxBuffer: 16 * 1024 * 1024 }); @@ -238,22 +300,11 @@ const pollMs = () => Number(process.env.GSTACK_EVAL_BG_POLL_MS) || 30_000; /** Dispatch one workflow (or adopt a duplicate) and follow it; returns the mapped exit code. */ async function followWorkflow(dispatch: WorkflowDispatch, ref: string, key: string, run: Exec): Promise { - const list = () => { - const out = run('gh', ['run', 'list', '--repo', CANONICAL_REPO, '--workflow', dispatch.workflow, '--event', 'workflow_dispatch', - '--limit', '30', '--json', 'databaseId,displayTitle,status,conclusion,url,event']); - try { return out.status === 0 ? JSON.parse(out.stdout) as RunSummary[] : []; } catch { return []; } - }; - let target = findDuplicateRun(list(), key); + let target = findDuplicateRun(listDispatchRuns(dispatch.workflow, run) ?? [], key); if (target) console.log(`[eval-bg] ${dispatch.workflow}: following existing run ${target.url} (same lane, revision and base; no duplicate dispatch)`); else { - const args = ['workflow', 'run', dispatch.workflow, '--repo', CANONICAL_REPO, '--ref', ref, - ...Object.entries(dispatch.inputs).flatMap(([name, value]) => ['-f', `${name}=${value}`])]; - if (run('gh', args).status !== 0) { console.log(`[eval-bg] ${dispatch.workflow}: gh workflow run failed`); return 2; } - const deadline = Date.now() + 10 * pollMs(); - while (!target && Date.now() < deadline) { - await sleep(Math.min(pollMs(), 10_000)); - target = list().find(item => item.displayTitle.includes(`eval-bg ${dispatch.inputs.nonce}`)) ?? null; - } + if (!dispatchWorkflow(dispatch, ref, run)) { console.log(`[eval-bg] ${dispatch.workflow}: gh workflow run failed`); return 2; } + target = await awaitNonceRun(dispatch.workflow, dispatch.inputs[DISPATCH_INPUTS.nonce]!, run); if (!target) { console.log(`[eval-bg] ${dispatch.workflow}: dispatched, but no run named ${dispatch.inputs.nonce} appeared`); return 2; } } console.log(`[eval-bg] ${dispatch.workflow}: run ${target.databaseId} ${target.url}`); diff --git a/test/fixtures/module-size-ratchet.json b/test/fixtures/module-size-ratchet.json index 6a6555c98..e107d6c6a 100644 --- a/test/fixtures/module-size-ratchet.json +++ b/test/fixtures/module-size-ratchet.json @@ -121,7 +121,14 @@ { "file": "lib/script-invocation.ts", "movedFrom": "(new: the one Windows bash resolver, shebang-bin invocation and wtreeFingerprint, from spawn-bin.ts and gbrain-exec.ts, #3197)" }, { "file": "lib/autoplan-budget.ts", "movedFrom": "(new: round cap, default host estimate, budget_exhausted stop and the budget verb, field list 2 W5)" }, { "file": "lib/autoplan-confirm.ts", "movedFrom": "(new: confirm-round plan delta, revision-qualified correlation and settled findings, field list 2 W5)" }, - { "file": "lib/autoplan-public.ts", "movedFrom": "(new: machine-path-free public run copy and validate --public, field list 2 W6)" } + { "file": "lib/autoplan-public.ts", "movedFrom": "(new: machine-path-free public run copy and validate --public, field list 2 W6)" }, + { "file": "lib/ship-queue-train.ts", "movedFrom": "(new: train build, status and prune, slot order, fill and prefix merges, plan GSTA-49 T1)" }, + { "file": "lib/ship-queue-train-record.ts", "movedFrom": "(new: versioned train, attempt, bisect, freeze and READY records mirrored to refs/notes/gstack-trains, plan GSTA-49 T1)" }, + { "file": "lib/ship-queue-train-github.ts", "movedFrom": "(new: gh reads for train runs, check runs and PR facts, plan GSTA-49 T1)" }, + { "file": "lib/ship-queue-train-verdict.ts", "movedFrom": "(new: train verdict, bisect and rebuild-once, plan GSTA-49 T1)" }, + { "file": "lib/ship-queue-train-verify.ts", "movedFrom": "(new: verify --train, the queue freeze and its backstop, plan GSTA-49 T1)" }, + { "file": "lib/ship-queue-train-ready.ts", "movedFrom": "(new: ready --screen|--no-screen and the READY receipt, plan GSTA-49 T1)" }, + { "file": "lib/ship-queue-train-cli.ts", "movedFrom": "(new: merge-train argument parsing and output, plan GSTA-49 T1)" } ], "residualFiles": [ { "file": "scripts/test-free-shards.ts", "maxLines": 2361, "reason": "free lane policy after the shard engine took spawn, sandbox, logs, seeds and flags" }, diff --git a/test/helpers/ship-queue-train-fixture.ts b/test/helpers/ship-queue-train-fixture.ts new file mode 100644 index 000000000..7c6928771 --- /dev/null +++ b/test/helpers/ship-queue-train-fixture.ts @@ -0,0 +1,158 @@ +/** + * Fixtures for merge trains (lib/ship-queue-train*.ts): a hermetic repo from + * restamp-fixture (bare origin plus working clone under os.tmpdir(), its own + * git identity) whose base policy declares a `train` block, and a stateful + * `gh` on PATH that answers every GitHub call the train surface makes from a + * JSON file: PR views, repo view, workflow dispatches that create runs (with + * a train record, a full-gate check run and bisect prefix artifacts built + * from per-test rules), run lists by workflow, branch and event, run views, + * artifact downloads and check-run reads. Every call is logged, so tests can + * prove what was (and was not) dispatched. + */ +import { createHash } from 'node:crypto'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { TRAIN_KEYS_VERSION } from '../../lib/ship-policy'; +import { BASE_POLICY, makeFixtureRepo, runBin, type FixtureRepo } from './restamp-fixture'; + +export const TRAIN_BLOCK = { + gate_workflow: 'evals.yml', free_workflow: 'free-tests.yml', full_gate_check: 'gstack/full-gate', record_artifact: 'gstack-train-record', + max_size: 5, fill_hours: 3, screen_selection: 'none', trial_reservation_usd: 1.5, +}; +export const TRAIN_POLICY = { ...BASE_POLICY, train: TRAIN_BLOCK, min_gstack_version: TRAIN_KEYS_VERSION }; + +export interface Rule { + fail?: boolean; silent?: boolean; failButCreate?: boolean; status?: string; conclusion?: string | null; event?: string; headSha?: string; + record?: { panels: unknown[]; census?: unknown; verdict?: 'GREEN' | 'RED'; nonce?: string }; + check?: string | null; + bisect?: { prefixes: string[]; patterns: Record; costUsd?: number }; +} +export interface ShimData { + repo: string; origin: string; nextId: number; prs: Record; runs: Array>; + artifacts: Record>; checks: Record>; + ci: Record; rules: Record; artifact: string; log: Array>; +} + +export interface TrainFixture { repo: FixtureRepo; state: string; dataFile: string; env: Record; read: () => ShimData; write: (fn: (d: ShimData) => void) => void } + +export function trainFixture(policy: Record = TRAIN_POLICY, prefix = 'gstack-train-'): TrainFixture { + const repo = makeFixtureRepo({ policy, prefix }); + const dir = path.join(repo.root, 'gh-train-shim'); + fs.mkdirSync(dir, { recursive: true }); + const dataFile = path.join(dir, 'data.json'); + const data: ShimData = { repo: 'fixture/repo', origin: repo.origin, nextId: 100, prs: {}, runs: [], artifacts: {}, checks: {}, ci: {}, rules: {}, artifact: TRAIN_BLOCK.record_artifact, log: [] }; + fs.writeFileSync(dataFile, JSON.stringify(data)); + fs.writeFileSync(path.join(dir, 'shim.js'), shimSource()); + fs.writeFileSync(path.join(dir, 'gh'), `#!/bin/sh\nexec node "${path.join(dir, 'shim.js')}" "${dataFile}" "$@"\n`, { mode: 0o755 }); + const state = path.join(repo.root, 'state'); + const env = { PATH: `${dir}:${process.env.PATH ?? ''}`, GSTACK_STATE_ROOT: state, GSTACK_SHIP_QUEUE_POLL_MS: '20', GSTACK_EVAL_BG_POLL_MS: '20', GSTACK_SHIP_QUEUE_NOW: '2026-10-18T12:00:00Z' }; + const read = () => JSON.parse(fs.readFileSync(dataFile, 'utf8')) as ShimData; + const write = (fn: (d: ShimData) => void) => { const d = read(); fn(d); fs.writeFileSync(dataFile, JSON.stringify(d)); }; + return { repo, state, dataFile, env, read, write }; +} + +/** The repo key lib/ship-queue-train-record.ts derives from origin's URL. */ +export function repoKey(fx: TrainFixture): string { + const norm = fx.repo.origin.replace(/\.git$/, '').replace(/\/+$/, ''); + return createHash('sha256').update(norm).digest('hex').slice(0, 16); +} + +/** A same-repo PR: a branch from origin/main touching `file`, published as refs/pull//head (and /merge), known to the gh shim. */ +export function openPr(fx: TrainFixture, pr: string, opts: { file?: string; text?: string; changelog?: boolean; cross?: boolean; from?: string } = {}): string { + const { repo } = fx; + repo.git('checkout', '-q', opts.from ?? 'origin/main', '-B', `pr-${pr}`); + const file = opts.file ?? `pr-${pr}.txt`; + fs.mkdirSync(path.dirname(path.join(repo.work, file)), { recursive: true }); + fs.writeFileSync(path.join(repo.work, file), opts.text ?? `change from #${pr}\n`); + if (opts.changelog !== false) { + const cl = fs.readFileSync(path.join(repo.work, 'CHANGELOG.md'), 'utf8'); + fs.writeFileSync(path.join(repo.work, 'CHANGELOG.md'), cl.replace('# Changelog\n', `# Changelog\n\n## [Unreleased]\n\n- #${pr} change\n`)); + } + repo.git('add', '-A'); + repo.git('commit', '-q', '-m', `feat #${pr}`); + const head = repo.git('rev-parse', 'HEAD'); + repo.git('push', '-q', '-f', 'origin', `HEAD:refs/pull/${pr}/head`, `HEAD:refs/pull/${pr}/merge`); + fx.write(d => { d.prs[pr] = { headRefOid: head, isCrossRepository: !!opts.cross, state: 'OPEN', headRefName: `pr-${pr}`, url: `https://github.com/fixture/repo/pull/${pr}`, body: '' }; }); + return head; +} + +/** A READY receipt as `ready --screen|--no-screen` writes it (the lane side is tested separately). */ +export function readyReceipt(fx: TrainFixture, pr: string, head: string, opts: { screen?: boolean; at?: string } = {}): void { + const key = repoKey(fx); + const file = path.join(fx.state, 'ship-queue', 'ready', key, `${pr}-${head}.json`); + fs.mkdirSync(path.dirname(file), { recursive: true }); + const at = opts.at ?? '2026-10-18T08:00:00.000Z'; + fs.writeFileSync(file, JSON.stringify({ schema: 1, kind: 'ready', repo_key: key, pr, head, base_sha: fx.repo.git('rev-parse', 'origin/main'), ready_at: at, + screen: { choice: opts.screen ? 'screen' : 'no-screen', policy: opts.screen ? 'narrow' : null, nonce: null, run: null, merge_sha: null }, updated_at: at })); +} + +export const PASS = (id: string, kind = 'rule', cost = 0.7) => ({ case: id, kind, status: 'PASS', red_class: null, cost_usd: cost }); +export const FAIL = (id: string, cls = 'VERDICT', cost = 0.9) => ({ case: id, kind: 'rule', status: 'FAIL', red_class: cls, cost_usd: cost }); +export const COMPLETE = { complete: true, planned: { rule: 2, judge: 1 }, executed: { rule: 2, judge: 1 }, unbound: 0, partial_shards: 0, cancelled_shards: 0, killed_before_upload: 0 }; +export const GREEN_GATE: Rule = { record: { panels: [PASS('c1'), PASS('c2'), PASS('j1', 'judge', 0.1)], census: COMPLETE, verdict: 'GREEN' }, check: 'success' }; +export const RED_GATE: Rule = { conclusion: 'failure', record: { panels: [FAIL('c1'), PASS('c2'), PASS('j1', 'judge', 0.1)], census: COMPLETE, verdict: 'RED' }, check: null }; + +export function queue(fx: TrainFixture, args: string[], extraEnv: Record = {}, cwd = fx.repo.work) { + return runBin(fx.repo, 'gstack-ship-queue', [...args, '--base', 'main'], { ...fx.env, ...extraEnv }, cwd); +} + +export const line = (out: string, prefix: string) => out.split('\n').find(l => l.startsWith(prefix)) ?? ''; +export const nonceOf = (out: string) => /^TRAIN: (\S+)/m.exec(out)?.[1] ?? ''; +export function trainJson(fx: TrainFixture, nonce: string, state = fx.state): any { + return JSON.parse(fs.readFileSync(path.join(state, 'ship-queue', 'trains', nonce, 'train.json'), 'utf8')); +} +export const dispatches = (fx: TrainFixture, workflow?: string) => fx.read().log.filter(l => l.cmd === 'workflow run' && (!workflow || l.workflow === workflow)); + +function shimSource(): string { + return [ + "const fs = require('fs'); const path = require('path'); const cp = require('child_process');", + "const file = process.argv[2]; const a = process.argv.slice(3); const d = JSON.parse(fs.readFileSync(file, 'utf8'));", + "const save = () => fs.writeFileSync(file, JSON.stringify(d));", + "const opt = n => { const i = a.indexOf(n); return i >= 0 ? a[i + 1] : undefined; };", + "const out = v => { console.log(JSON.stringify(v)); save(); process.exit(0); };", + "const fail = m => { console.error(m); save(); process.exit(1); };", + "const rev = r => { const x = cp.spawnSync('git', ['--git-dir', d.origin, 'rev-parse', '--verify', '--quiet', r], { encoding: 'utf8', timeout: 30000 }); return x.status === 0 ? x.stdout.trim() : ''; };", + "d.log.push({ cmd: a.slice(0, 2).join(' '), args: a });", + "if (a[0] === 'repo' && a[1] === 'view') out({ nameWithOwner: d.repo });", + "if (a[0] === 'pr' && a[1] === 'view') { const p = d.prs[a[2]]; if (!p) fail('no pull request found'); out(p); }", + "if (a[0] === 'workflow' && a[1] === 'run') {", + " const w = a[2]; const ref = opt('--ref'); const inputs = {};", + " a.forEach((x, i) => { if (x === '-f') { const kv = a[i + 1]; const k = kv.slice(0, kv.indexOf('=')); inputs[k] = kv.slice(kv.indexOf('=') + 1); } });", + " Object.assign(d.log[d.log.length - 1], { workflow: w, ref, inputs });", + " const rule = d.rules[w] || {};", + " if (rule.fail) fail('HTTP 422: dispatch refused');", + " if (rule.silent) { save(); process.exit(0); }", + " const id = d.nextId++; const head = rule.headSha || inputs.expected_sha || rev('refs/heads/' + ref) || rev(ref);", + " const run = { databaseId: id, workflow: w, headBranch: ref, headSha: head, status: rule.status || 'completed', conclusion: rule.conclusion === undefined ? 'success' : rule.conclusion,", + " url: 'https://example.invalid/runs/' + id, event: rule.event || 'workflow_dispatch', displayTitle: inputs.nonce ? 'E2E Evals [eval-bg ' + inputs.nonce + ']' : w, inputs };", + " d.runs.push(run); d.artifacts[id] = {};", + " if (rule.record && !inputs.prefix_shas) d.artifacts[id][d.artifact] = { schema: 1, kind: 'train-gate', nonce: rule.record.nonce || inputs.nonce, head_sha: head, tree: rev(head + '^{tree}'),", + " verdict: rule.record.verdict || 'GREEN', census: rule.record.census, panels: rule.record.panels };", + " if (rule.check && !inputs.prefix_shas) (d.checks[head] = d.checks[head] || []).push({ name: 'gstack/full-gate', status: 'completed', conclusion: rule.check });", + " if (rule.bisect && inputs.prefix_shas) for (const sha of inputs.prefix_shas.split(',')) {", + " const at = rule.bisect.prefixes.indexOf(sha);", + " d.artifacts[id][d.artifact + '-bisect-' + sha.slice(0, 12)] = { schema: 1, kind: 'train-bisect-prefix', nonce: inputs.nonce, prefix_sha: sha, checked_out_sha: sha,", + " panels: inputs.cases.split(',').map(c => ({ case: c, kind: 'rule', status: rule.bisect.patterns[c][at] === 'R' ? 'FAIL' : 'PASS', red_class: rule.bisect.patterns[c][at] === 'R' ? 'VERDICT' : null, cost_usd: rule.bisect.costUsd ?? 0.5 })) };", + " }", + " if (rule.failButCreate) fail('HTTP 502: gateway timeout');", + " save(); process.exit(0);", + "}", + "if (a[0] === 'run' && a[1] === 'list') {", + " const w = opt('--workflow'); const br = opt('--branch'); const ev = opt('--event');", + " out(d.runs.filter(r => r.workflow === w && (!br || r.headBranch === br) && (!ev || r.event === ev)).sort((x, y) => y.databaseId - x.databaseId).slice(0, Number(opt('--limit') || 20)));", + "}", + "if (a[0] === 'run' && a[1] === 'view') { const r = d.runs.find(x => String(x.databaseId) === a[2]); if (!r) fail('run not found'); out(r); }", + "if (a[0] === 'run' && a[1] === 'download') {", + " const art = (d.artifacts[a[2]] || {})[opt('-n')]; if (!art) fail('no artifact matches');", + " fs.mkdirSync(opt('-D'), { recursive: true }); fs.writeFileSync(path.join(opt('-D'), 'record.json'), JSON.stringify(art)); save(); process.exit(0);", + "}", + "if (a[0] === 'api') {", + " const named = /commits\\/([0-9a-f]+)\\/check-runs\\?check_name=([^&]+)/.exec(a[1] || '');", + " if (named) { const list = (d.checks[named[1]] || []).filter(c => c.name === decodeURIComponent(named[2])); out({ total_count: list.length, check_runs: list }); }", + " const paged = /commits\\/([0-9a-f]+)\\/(check-runs|status)\\?.*page=(\\d+)/.exec(a[1] || '');", + " if (paged) { const list = paged[2] === 'check-runs' ? (d.ci[paged[1]] || []) : []; const key = paged[2] === 'check-runs' ? 'check_runs' : 'statuses';", + " out(paged[3] === '1' ? { total_count: list.length, state: 'success', [key]: list } : { total_count: list.length, [key]: [] }); }", + "}", + "fail('gh shim: unsupported ' + a.join(' '));", + ].join('\n') + '\n'; +} diff --git a/test/ship-policy-compat.test.ts b/test/ship-policy-compat.test.ts index be9efabdb..be5d0fa85 100644 --- a/test/ship-policy-compat.test.ts +++ b/test/ship-policy-compat.test.ts @@ -9,12 +9,14 @@ import { afterEach, describe, expect, test } from 'bun:test'; import { EXIT } from '../lib/headless-artifacts'; import { DEFAULT_POLICY, POLICY_KEY_INTRODUCED, SHIP_QUEUE_KEYS_VERSION, validatePolicy } from '../lib/ship-policy'; +import { cmpVersion, parseVersion } from '../lib/version-source'; import { BASE_POLICY, cleanup, makeFixtureRepo, restamp, runBin, type FixtureRepo } from './helpers/restamp-fixture'; const repos: FixtureRepo[] = []; const fixture = (policy: Record) => { const r = makeFixtureRepo({ policy }); repos.push(r); return r; }; afterEach(() => { while (repos.length) cleanup(repos.pop()!); }); +const newestKeyVersion = Object.values(POLICY_KEY_INTRODUCED).reduce((a, b) => (cmpVersion(parseVersion(b)!, parseVersion(a)!) > 0 ? b : a)); const codes = (raw: unknown, gstackVersion = '1.91.200.0') => validatePolicy(raw, { gstackVersion }).map(e => `${e.code}@${e.path}`); describe('policy compatibility floor', () => { @@ -35,7 +37,7 @@ describe('policy compatibility floor', () => { expect(codes({ gate_command: 'x', min_gstack_version: SHIP_QUEUE_KEYS_VERSION }, '1.91.106.0')).toEqual(['POLICY_GSTACK_TOO_OLD@min_gstack_version']); const future = validatePolicy({ gate_reuse_requires: ['e2e'] }, { gstackVersion: '1.91.200.0' }); expect(future.map(e => `${e.code}@${e.path}`)).toEqual(['POLICY_INVALID@gate_reuse_requires']); - expect(future[0]!.message).toContain(`this gstack reads keys introduced up to ${SHIP_QUEUE_KEYS_VERSION}`); + expect(future[0]!.message).toContain(`this gstack reads keys introduced up to ${newestKeyVersion}`); expect(codes({ min_gstack_version: 'soon' })).toEqual(['POLICY_INVALID@min_gstack_version']); expect(codes({ full_e2e: 'yes', min_gstack_version: SHIP_QUEUE_KEYS_VERSION })).toEqual(['POLICY_INVALID@full_e2e']); }); diff --git a/test/ship-queue-train-logic.test.ts b/test/ship-queue-train-logic.test.ts new file mode 100644 index 000000000..3e2a4969f --- /dev/null +++ b/test/ship-queue-train-logic.test.ts @@ -0,0 +1,205 @@ +/** + * Merge trains (plan GSTA-49 lane T1), the pure decisions: slot order and + * fill, every paid-report class mapped to a train verdict, bisect pattern + * attribution, the rebuild and re-dispatch allowances pinned to EVAL_POLICY, + * the screen recommendation, the `train` policy block and eval-bg's nonce + * key with purpose and screen policy. + */ +import { describe, expect, test } from 'bun:test'; +import { DISPATCH_INPUTS, dedupeKey, dispatchPlan, findDuplicateRun, makeNonce, nonceScope, type RunSummary } from '../scripts/eval-bg'; +import { DEFAULT_POLICY, TRAIN_KEYS_VERSION, validatePolicy } from '../lib/ship-policy'; +import { fillDue, orderSlots, parsePins } from '../lib/ship-queue-train'; +import { gstackPaidSelection, skillOwnCases, touchedSkills } from '../lib/ship-queue-train-ready'; +import type { TrainSlot } from '../lib/ship-queue-train-record'; +import { INFRA_REDISPATCH, REBUILD_ONCE, classifyBisect, classifyGate } from '../lib/ship-queue-train-verdict'; +import { EVAL_POLICY } from './helpers/periodic-exclude-data'; +import { COMPLETE, FAIL, PASS, TRAIN_BLOCK } from './helpers/ship-queue-train-fixture'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const SHA = 'a'.repeat(40); +const slot = (pr: string, screened: boolean, at: string): TrainSlot => ({ pr, head: SHA, ready_at: at, screen: { choice: screened ? 'screen' : 'no-screen', policy: null, nonce: null, run: null, merge_sha: null } }); + +describe('ordering and fill', () => { + test('screened PRs go first in READY order, then unscreened; the order is stable on ties', () => { + const o = orderSlots([slot('4', false, '2026-10-18T01:00:00Z'), slot('3', true, '2026-10-18T03:00:00Z'), slot('2', false, '2026-10-18T00:00:00Z'), slot('1', true, '2026-10-18T02:00:00Z')], 5); + expect(o.train.map(s => s.pr)).toEqual(['1', '3', '2', '4']); + expect(o.held).toEqual([]); + expect(o.split).toBe(false); + }); + + test('when as many unscreened PRs wait as fill a train, they form their own train; over-size PRs are held in order', () => { + const unscreened = ['5', '6', '7'].map((pr, i) => slot(pr, false, `2026-10-18T0${i}:00:00Z`)); + const split = orderSlots([slot('9', true, '2026-10-18T05:00:00Z'), ...unscreened], 3); + expect(split.split).toBe(true); + expect(split.train.map(s => s.pr)).toEqual(['9']); + expect(split.held.map(s => s.pr)).toEqual(['5', '6', '7']); + const over = orderSlots([slot('8', true, '2026-10-18T06:00:00Z'), slot('9', true, '2026-10-18T05:00:00Z'), unscreened[0]!], 2); + expect(over.split).toBe(false); + expect(over.train.map(s => s.pr)).toEqual(['9', '8']); + expect(over.held.map(s => s.pr)).toEqual(['5']); + }); + + test('a train under its size dispatches once its oldest READY has waited fill_hours; full or urgent dispatches at once', () => { + const slots = [slot('1', false, '2026-10-18T09:00:00Z'), slot('2', false, '2026-10-18T10:00:00Z')]; + expect(fillDue(slots, TRAIN_BLOCK as any, '2026-10-18T11:59:00Z', false)).toBe('2026-10-18T12:00:00.000Z'); + expect(fillDue(slots, TRAIN_BLOCK as any, '2026-10-18T12:00:00Z', false)).toBeNull(); + expect(fillDue(slots, TRAIN_BLOCK as any, '2026-10-18T09:30:00Z', true)).toBeNull(); + expect(fillDue(slots, { ...TRAIN_BLOCK, max_size: 2 } as any, '2026-10-18T09:30:00Z', false)).toBeNull(); + }); + + test('pins are @, each PR once', () => { + expect(parsePins(`12@${SHA},#13@abcdef1`)).toEqual([{ pr: '12', sha: SHA }, { pr: '13', sha: 'abcdef1' }]); + expect(parsePins('12')).toMatch(/entries are @/); + expect(parsePins(`12@${SHA},12@${SHA}`)).toBe('a PR appears twice in --prs'); + }); +}); + +describe('verdict class mapping', () => { + const expectIds = { nonce: 'n1', head: SHA, tree: 'b'.repeat(40) }; + const record = (over: Record = {}) => ({ schema: 1, kind: 'train-gate', nonce: 'n1', head_sha: SHA, tree: 'b'.repeat(40), verdict: 'GREEN', census: COMPLETE, panels: [PASS('c1'), PASS('j1', 'judge')], ...over }); + const done: { status: string; conclusion: string | null } = { status: 'completed', conclusion: 'success' }; + const check = { status: 'completed', conclusion: 'success' }; + const classify = (rec: unknown, run = done, chk: typeof check | null = check) => classifyGate(rec, expectIds, run, chk); + + test('GREEN only for a complete, bound census on a successful run that published its check', () => { + const g = classify(record()); + expect(g.outcome).toBe('GREEN'); + expect(g.detail).toBe('rule 2/2, judge 1/1'); + expect(classify(record(), done, null).outcome).toBe('INCOMPLETE'); + expect(classify(record(), done, { status: 'completed', conclusion: 'failure' }).detail).toContain('did not publish a successful check run'); + }); + + test('a run still going is PENDING; no record is INCOMPLETE and eligible for the one INFRA re-dispatch', () => { + expect(classify(record(), { status: 'in_progress', conclusion: null }).outcome).toBe('PENDING'); + const none = classify(null, { status: 'completed', conclusion: 'cancelled' }); + expect([none.outcome, none.infraEligible]).toEqual(['INCOMPLETE', true]); + expect(none.detail).toContain('a slice killed before its upload step'); + }); + + test('a VERDICT FAIL is RED with its cases, even beside census gaps', () => { + const r = classify(record({ verdict: 'RED', panels: [FAIL('c2'), FAIL('c1'), PASS('j1')], census: { ...COMPLETE, cancelled_shards: 1 } }), { status: 'completed', conclusion: 'failure' }); + expect([r.outcome, r.red]).toEqual(['RED', ['c1', 'c2']]); + expect(r.cost).toBe(2.5); + }); + + test('INFRA and INCOMPLETE panels, unbound, partial or cancelled shards and killed slices are INCOMPLETE, never green', () => { + const cases: Array<[Record, boolean, RegExp]> = [ + [{ panels: [FAIL('c1', 'INFRA'), PASS('j1')] }, true, /c1 FAIL \(INFRA\)/], + [{ panels: [{ ...PASS('c1'), status: 'INCOMPLETE', red_class: 'INCOMPLETE' }, PASS('j1')] }, true, /c1 INCOMPLETE/], + [{ panels: [{ ...PASS('c1'), status: 'SKIPPED' }, PASS('j1')] }, false, /c1 SKIPPED/], + [{ census: { ...COMPLETE, unbound: 1 } }, true, /1 unbound result/], + [{ census: { ...COMPLETE, partial_shards: 2 } }, true, /2 partial shard/], + [{ census: { ...COMPLETE, killed_before_upload: 1 } }, true, /slice killed before upload/], + [{ census: { ...COMPLETE, executed: { rule: 1, judge: 1 } } }, true, /rule 1\/2 executed/], + [{ census: { ...COMPLETE, planned: { rule: 0, judge: 0 }, executed: { rule: 0, judge: 0 } } }, true, /lists no cases/], + ]; + for (const [over, eligible, why] of cases) { + const r = classify(record(over)); + expect([r.outcome, r.infraEligible]).toEqual(['INCOMPLETE', eligible]); + expect(r.detail).toMatch(why); + } + }); + + test('a record bound to another nonce, head or tree, or a failed run with a green census, is INCOMPLETE and not eligible', () => { + for (const over of [{ nonce: 'other' }, { head_sha: 'c'.repeat(40) }, { tree: 'd'.repeat(40) }]) { + const r = classify(record(over)); + expect([r.outcome, r.infraEligible]).toEqual(['INCOMPLETE', false]); + expect(r.detail).toContain('not this train'); + } + expect(classify(record(), { status: 'completed', conclusion: 'failure' }).detail).toContain('run concluded failure with record verdict GREEN'); + }); + + test('the one INFRA re-dispatch and the one rebuild are pinned to EVAL_POLICY', () => { + expect(INFRA_REDISPATCH).toBe(EVAL_POLICY.infraRedispatch); + expect(REBUILD_ONCE).toBe(1); + }); +}); + +describe('bisect attribution', () => { + const slots = [{ pr: '11' }, { pr: '12' }, { pr: '13' }]; + const results = (patterns: Record) => [0, 1, 2, 3].map(i => Object.fromEntries(Object.entries(patterns).map(([c, p]) => [c, p[i] === 'R' ? 'red' as const : 'green' as const]))); + + test('monotone green-then-red blames the PR that starts the red, the last PR included', () => { + expect(classifyBisect(['c1'], results({ c1: 'GGRR' }), slots)).toEqual({ kind: 'attributed', all_green: false, prs: ['12'], patterns: { c1: 'GGRR' } }); + expect(classifyBisect(['c1'], results({ c1: 'GGGR' }), slots).prs).toEqual(['13']); + expect(classifyBisect(['c1', 'c2'], results({ c1: 'GRRR', c2: 'GGGR' }), slots).prs).toEqual(['11', '13']); + expect(classifyBisect(['c1', 'c2'], results({ c1: 'GRRR', c2: 'GGGG' }), slots).kind).toBe('attributed'); + }); + + test('red on every prefix, T0 included, is main; all green and non-monotone are unattributed', () => { + expect(classifyBisect(['c1'], results({ c1: 'RRRR' }), slots).kind).toBe('red-on-main'); + expect(classifyBisect(['c1', 'c2'], results({ c1: 'RRRR', c2: 'GGRR' }), slots).kind).toBe('red-on-main'); + expect(classifyBisect(['c1'], results({ c1: 'GGGG' }), slots)).toMatchObject({ kind: 'unattributed', all_green: true, prs: [] }); + expect(classifyBisect(['c1'], results({ c1: 'GRGR' }), slots)).toMatchObject({ kind: 'unattributed', all_green: false }); + expect(classifyBisect(['c1'], results({ c1: 'RGGG' }), slots)).toMatchObject({ kind: 'unattributed', all_green: false }); + expect(classifyBisect(['c1', 'c2'], results({ c1: 'GGRR', c2: 'GRGR' }), slots)).toMatchObject({ kind: 'unattributed', all_green: false }); + }); +}); + +describe('screen recommendation', () => { + test('a skill directory is touched when the diff changes a file under a directory holding SKILL.md.tmpl', () => { + expect(touchedSkills(['review/SKILL.md.tmpl', 'review/x.md', 'lib/a.ts', 'README.md'], d => d === 'review')).toEqual(['review']); + expect(skillOwnCases(['a', 'b', 'c'], { a: ['review/**'], b: ['lib/**'], c: ['review/SKILL.md'] }, ['review'])).toEqual(['a', 'c']); + }); + + test("gstack's selector on gstack's own tree: a skill template change recommends --screen, a docs change --no-screen", async () => { + const has = (dir: string) => fs.existsSync(path.join(ROOT, dir, 'SKILL.md.tmpl')); + const skill = await gstackPaidSelection(ROOT, 'HEAD', ['review/SKILL.md.tmpl'], has); + expect(skill.recommend).toBe('screen'); + expect(skill.lines[0]).toMatch(/^SCREEN: selection \d+ e2e case\(s\) \+ \d+ judge\(s\)/); + expect(skill.lines[1]).toMatch(/^SCREEN: skill's own cases \S+ → recommend --screen$/); + const docs = await gstackPaidSelection(ROOT, 'HEAD', ['docs/ship-policy.md'], has); + expect(docs.recommend).toBe('no-screen'); + }, 60_000); +}); + +describe('the train policy block', () => { + const ok = { train: TRAIN_BLOCK, min_gstack_version: TRAIN_KEYS_VERSION }; + const codes = (raw: unknown) => validatePolicy(raw, { gstackVersion: '1.91.200.0' }).map(e => `${e.code}@${e.path}`); + + test('defaults keep it null; a full block validates with min_gstack_version at its introducing version', () => { + expect(DEFAULT_POLICY.train).toBeNull(); + expect(codes(ok)).toEqual([]); + expect(codes({ train: TRAIN_BLOCK })).toEqual(['POLICY_INVALID@train']); + }); + + test('unknown, missing and out-of-range fields fail closed', () => { + expect(codes({ ...ok, train: { ...TRAIN_BLOCK, colour: 'red' } })).toEqual(['POLICY_INVALID@train.colour']); + const { fill_hours: _drop, ...missing } = TRAIN_BLOCK; + expect(codes({ ...ok, train: missing })).toEqual(['POLICY_INVALID@train.fill_hours']); + expect(codes({ ...ok, train: { ...TRAIN_BLOCK, max_size: 7, gate_workflow: '../evals.yml', screen_selection: 'all' } })) + .toEqual(['POLICY_INVALID@train.gate_workflow', 'POLICY_INVALID@train.max_size', 'POLICY_INVALID@train.screen_selection']); + expect(codes({ ...ok, train: [] })).toEqual(['POLICY_INVALID@train']); + }); +}); + +describe("eval-bg nonce key: purpose and screen policy", () => { + const base = 'c'.repeat(40); + const run = (title: string): RunSummary => ({ databaseId: 1, displayTitle: title, status: 'in_progress', conclusion: null, url: 'u', event: 'workflow_dispatch' }); + + test('a plain lane key is unchanged; a scoped key carries purpose and policy, and parses back', () => { + expect(dedupeKey('pr', SHA, base)).toBe(`pr-${SHA.slice(0, 12)}-${base.slice(0, 12)}-`); + expect(makeNonce('pr', SHA, base, 'zz', { purpose: 'screen', policy: 'broad' })).toBe(`pr.screen.broad-${SHA.slice(0, 12)}-${base.slice(0, 12)}-zz`); + expect(nonceScope(makeNonce('gate', SHA, base, 'q1', { purpose: 'train' }))).toEqual({ purpose: 'train' }); + expect(nonceScope(makeNonce('pr', SHA, base, 'q1', { purpose: 'screen', policy: 'narrow' }))).toEqual({ purpose: 'screen', policy: 'narrow' }); + expect(nonceScope(makeNonce('pr', SHA, base, 'q1'))).toBeNull(); + }); + + test('a broad screen never adopts a narrow run, a screen never adopts a plain PR lane run, and a train never adopts a bisect', () => { + const narrow = makeNonce('pr', SHA, base, 'n1', { purpose: 'screen', policy: 'narrow' }); + const plain = makeNonce('pr', SHA, base, 'p1'); + const bisect = makeNonce('gate', SHA, base, 'b1', { purpose: 'bisect' }); + const runs = [run(`E2E Evals [eval-bg ${narrow}]`), run(`E2E Evals [eval-bg ${plain}]`), run(`E2E Evals [eval-bg ${bisect}]`)]; + expect(findDuplicateRun(runs, dedupeKey('pr', SHA, base, { purpose: 'screen', policy: 'broad' }))).toBeNull(); + expect(findDuplicateRun([runs[1]!], dedupeKey('pr', SHA, base, { purpose: 'screen', policy: 'narrow' }))).toBeNull(); + expect(findDuplicateRun(runs, dedupeKey('pr', SHA, base, { purpose: 'screen', policy: 'narrow' }))?.displayTitle).toContain(narrow); + expect(findDuplicateRun([runs[2]!], dedupeKey('gate', SHA, base, { purpose: 'train' }))).toBeNull(); + }); + + test('every dispatch input name lives in DISPATCH_INPUTS, and the lane plans still send the same inputs', () => { + expect(DISPATCH_INPUTS).toMatchObject({ cases: 'cases', prefixShas: 'prefix_shas', screenPr: 'screen_pr', screenMergeSha: 'screen_merge_sha', screenPolicy: 'screen_policy' }); + expect(dispatchPlan('gate', { sha: SHA, baseRef: 'main', baseSha: base, nonce: 'n' })).toEqual([{ workflow: 'evals.yml', inputs: { evals_all: 'true', expected_sha: SHA, nonce: 'n' } }]); + }); +}); diff --git a/test/ship-queue-train.test.ts b/test/ship-queue-train.test.ts new file mode 100644 index 000000000..061548b44 --- /dev/null +++ b/test/ship-queue-train.test.ts @@ -0,0 +1,524 @@ +/** + * Merge trains through the real bin/gstack-ship-queue (plan GSTA-49 lane T1), + * headless against fixture repos outside the checkout (bare origin, lane and + * coordinator clones, a hermetic git identity) with a stateful gh shim: + * build (happy path, CHANGELOG union, conflict, fork, moved head, READY, + * fill, free tests red/missing/stale/cancelled, dry run, ordering), dispatch + * recovery (partial and absent dispatches), status rebuilt from + * refs/notes/gstack-trains on a clean state root, verdict, bisect + * (monotone, non-monotone, all green, red on main, capped and resumed + * without repeats, prefix ancestry), rebuild-once and the INFRA + * re-dispatch, verify (tree equality, mismatch, freeze, repair and clear), + * prune, ready's screen choice, and the policy gate. + */ +import { afterEach, describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { EXIT } from '../lib/headless-artifacts'; +import { TRAIN_HELP } from '../lib/ship-queue-train-cli'; +import { BASE_POLICY, cleanup, runBin } from './helpers/restamp-fixture'; +import { coordinatorClone } from './helpers/ship-queue-fixture'; +import { + COMPLETE, FAIL, GREEN_GATE, PASS, RED_GATE, TRAIN_BLOCK, TRAIN_POLICY, dispatches, line, nonceOf, openPr, queue, readyReceipt, repoKey, trainFixture, trainJson, + type Rule, type TrainFixture, +} from './helpers/ship-queue-train-fixture'; + +const fixtures: TrainFixture[] = []; +const make = (policy: Record = TRAIN_POLICY) => { const f = trainFixture(policy); fixtures.push(f); return f; }; +afterEach(() => { while (fixtures.length) cleanup(fixtures.pop()!.repo); }); + +const result = (out: string) => line(out, 'GSTACK_RESULT:'); +const next = (out: string) => line(out, 'NEXT:'); +const remoteRefs = (fx: TrainFixture) => fx.repo.git('ls-remote', 'origin'); + +function twoPrTrain(fx: TrainFixture, gate: Rule = GREEN_GATE) { + const h1 = openPr(fx, '1'); + const h2 = openPr(fx, '2'); + readyReceipt(fx, '1', h1); + readyReceipt(fx, '2', h2); + fx.write(d => { d.rules['evals.yml'] = gate; }); + const r = queue(fx, ['train', 'build', '--prs', `1@${h1},2@${h2}`]); + return { h1, h2, r, nonce: nonceOf(r.out) }; +} + +/** A main commit with a prefix's tree plus a VERSION stamp, the way stamp-at-merge publishes it. */ +function publishMain(fx: TrainFixture, prefixCommit: string, version: string, extra?: { file: string; text: string }): string { + const { repo } = fx; + repo.git('checkout', '-q', '-B', 'mainsim', 'origin/main'); + repo.git('rm', '-rq', '.'); + repo.git('checkout', prefixCommit, '--', '.'); + fs.writeFileSync(path.join(repo.work, 'VERSION'), version + '\n'); + if (extra) fs.writeFileSync(path.join(repo.work, extra.file), extra.text); + repo.git('add', '-A'); + repo.git('commit', '-q', '-m', `merge to ${version}`); + repo.git('push', '-q', 'origin', 'HEAD:refs/heads/main'); + repo.git('fetch', '-q', 'origin'); + return repo.git('rev-parse', 'HEAD'); +} + +describe('the train command surface', () => { + test('train with no subcommand (flags only) prints the train help and exits usage; --help exits 0; bad arguments name what is missing', () => { + const fx = make(); + const bare = queue(fx, ['train']); + expect([bare.status, bare.out]).toEqual([EXIT.usage, TRAIN_HELP + '\n']); + expect(queue(fx, ['train', '--help'])).toMatchObject({ status: EXIT.ok, out: TRAIN_HELP + '\n' }); + const noPins = queue(fx, ['train', 'build']); + expect(noPins.status).toBe(EXIT.usage); + expect(noPins.err).toContain('train build needs --prs @[,...] or --train '); + expect(queue(fx, ['train', 'build', '--prs', '1@abc', '--urgent', '--repair']).err).toContain('--urgent and --repair are separate one-PR trains; pass one'); + expect(fs.existsSync(path.join(fx.state, 'ship-queue'))).toBe(false); + }, 60_000); +}); + +describe('the policy gate', () => { + test('without a train block the train surface refuses with QUEUE_TRAIN_OFF and ready prints exactly today\'s block', () => { + const fx = make(BASE_POLICY); + const status = queue(fx, ['train', 'status']); + expect(status.status).toBe(EXIT.refused); + expect(status.err).toContain('declares no train block; the per-PR queue applies'); + expect(status.err).toContain('(QUEUE_TRAIN_OFF)'); + expect(next(status.out)).toBe('NEXT: none'); + expect(result(status.out)).toBe('GSTACK_RESULT: skill=ship-queue status=refused run=none'); + expect(queue(fx, ['verify', '--pr', '1', '--train', 'x']).err).toContain('(QUEUE_TRAIN_OFF)'); + const head = openPr(fx, '1'); + fx.write(d => { d.ci[head] = [{ status: 'completed', conclusion: 'success' }]; }); + const tree = runBin(fx.repo, 'gstack-wtree', []).out.trim(); + const pregate = path.join(fx.repo.root, 'pregate.json'); + fs.writeFileSync(pregate, JSON.stringify({ schema_version: 1, tree, head, verdict: 'pass', checks: [{ id: 'unit', status: 'pass' }] })); + const today = queue(fx, ['ready', '--pr', '1', '--pregate', pregate]); + expect(today.status).toBe(EXIT.ok); + expect(today.out.split('\n')[0]).toBe(`READY #1 ${head} STAMP_VERSION=unstamped`); + expect(today.out).not.toContain('NEXT:'); + expect(today.out).not.toContain('screen'); + expect(result(today.out)).toBe('GSTACK_RESULT: skill=ship-queue status=complete run=.gstack/tmp/pregate.json'); + const flagged = queue(fx, ['ready', '--pr', '1', '--pregate', pregate, '--no-screen']); + expect(flagged.out).toBe(today.out); + expect(flagged.err).toContain('--screen/--no-screen/--broad ignored: the base policy declares no train block'); + expect(fs.existsSync(path.join(fx.state, 'ship-queue'))).toBe(false); + }, 120_000); +}); + +describe('ready under a train policy: the screen choice', () => { + function readyFixture() { + const fx = make(); + const head = openPr(fx, '1'); + fx.write(d => { d.ci[head] = [{ status: 'completed', conclusion: 'success' }]; }); + const tree = runBin(fx.repo, 'gstack-wtree', []).out.trim(); + const pregate = path.join(fx.repo.root, 'pregate.json'); + fs.writeFileSync(pregate, JSON.stringify({ schema_version: 1, tree, head, verdict: 'pass', checks: [{ id: 'unit', status: 'pass' }] })); + return { fx, head, pregate }; + } + + test('no choice refuses with the recommendation; --no-screen records a READY receipt in the block, the state root and the notes ref', () => { + const { fx, head, pregate } = readyFixture(); + const none = queue(fx, ['ready', '--pr', '1', '--pregate', pregate]); + expect(none.status).toBe(EXIT.refused); + expect(none.err).toContain('(READY_SCREEN_CHOICE_REQUIRED)'); + expect(none.out).toContain('SCREEN: selection not computed (train.screen_selection is none)'); + expect(none.out).toContain('SCREEN: price pending the T0 cost replay (lib/gate-cost.ts)'); + expect(next(none.out)).toBe('NEXT: gstack-ship-queue ready --pr 1 --no-screen'); + const ok = queue(fx, ['ready', '--pr', '1', '--pregate', pregate, '--no-screen']); + expect(ok.status).toBe(EXIT.ok); + expect(ok.out.split('\n').slice(0, 2)).toEqual([`READY #1 ${head} STAMP_VERSION=unstamped`, 'screen: no-screen']); + const receipt = path.join(fx.state, 'ship-queue', 'ready', repoKey(fx), `1-${head}.json`); + expect(JSON.parse(fs.readFileSync(receipt, 'utf8'))).toMatchObject({ kind: 'ready', pr: '1', head, ready_at: '2026-10-18T12:00:00.000Z', screen: { choice: 'no-screen' } }); + expect(result(ok.out)).toBe(`GSTACK_RESULT: skill=ship-queue status=complete run=${receipt}`); + expect(remoteRefs(fx)).toContain('refs/notes/gstack-trains'); + expect(dispatches(fx)).toEqual([]); + }, 120_000); + + test('--screen dispatches one screen with the PR, its merge SHA and the policy; a repeat adopts it; a broad screen never adopts the narrow one', () => { + const { fx, head, pregate } = readyFixture(); + const dry = queue(fx, ['ready', '--pr', '1', '--pregate', pregate, '--screen', '--dry-run']); + expect(dry.status).toBe(EXIT.ok); + expect(line(dry.out, 'SCREEN_DISPATCH:')).toMatch(new RegExp(`^SCREEN_DISPATCH: evals\\.yml --ref main evals_all=false expected_sha=${head} nonce=pr\\.screen\\.narrow-\\S+ base_ref=main base_sha=[0-9a-f]{40} screen_pr=1 screen_merge_sha=${head} screen_policy=narrow \\(dry run: not dispatched\\)$`)); + expect(dispatches(fx)).toEqual([]); + fx.write(d => { d.rules['evals.yml'] = { status: 'in_progress', conclusion: null }; }); + const first = queue(fx, ['ready', '--pr', '1', '--pregate', pregate, '--screen']); + expect(first.status).toBe(EXIT.ok); + expect(first.out.split('\n')[1]).toMatch(/^screen: screen policy=narrow nonce=pr\.screen\.narrow-\S+ run=100$/); + const again = queue(fx, ['ready', '--pr', '1', '--pregate', pregate, '--screen']); + expect(again.out).toContain('SCREEN_DISPATCH: following run 100 (same merge SHA, base and policy; no duplicate)'); + const broad = queue(fx, ['ready', '--pr', '1', '--pregate', pregate, '--screen', '--broad']); + expect(broad.status).toBe(EXIT.ok); + expect(dispatches(fx).map(d => (d.inputs as any).screen_policy)).toEqual(['narrow', 'broad']); + expect(queue(fx, ['ready', '--pr', '1', '--broad']).status).toBe(EXIT.usage); + }, 120_000); +}); + +describe('train build', () => { + test('happy path: slot order, prefix trees, CHANGELOG union, records before effects, free tests then one paid dispatch by nonce; resume and identical pins never dispatch again', () => { + const fx = make(); + const { h1, h2, r, nonce } = twoPrTrain(fx); + expect(r.status).toBe(EXIT.ok); + expect(nonce).toMatch(/^gate\.train-[0-9a-f]{12}-[0-9a-f]{12}-[0-9a-f]{6}$/); + const t = trainJson(fx, nonce); + expect(t).toMatchObject({ schema: 1, kind: 'train', state: 'gate-pending', branch: `train/${nonce}`, slots: [{ pr: '1', head: h1 }, { pr: '2', head: h2 }] }); + expect(t.prefixes.map((p: any) => [p.index, p.pr])).toEqual([[0, null], [1, '1'], [2, '2']]); + expect(t.prefixes[0].commit).toBe(fx.repo.git('rev-parse', 'origin/main')); + expect(t.prefixes[2].commit).toBe(t.head); + expect(r.out).toContain(`PREFIX: T2 ${t.head} tree=${t.tree} (+#2)`); + expect(remoteRefs(fx)).toContain(`${t.head}\trefs/heads/train/${nonce}`); + expect(remoteRefs(fx)).toContain('refs/notes/gstack-trains'); + const changelog = fx.repo.git('show', `${t.head}:CHANGELOG.md`); + expect(changelog).toContain('- #1 change'); + expect(changelog).toContain('- #2 change'); + expect(fx.repo.git('show', `${t.head}:pr-1.txt`)).toBe('change from #1'); + const sent = dispatches(fx); + expect(sent.map(d => [d.workflow, d.ref])).toEqual([['free-tests.yml', `train/${nonce}`], ['evals.yml', `train/${nonce}`]]); + expect(sent[1]!.inputs).toEqual({ evals_all: 'true', expected_sha: t.head, nonce }); + expect(t.effects.map((e: any) => e.what)).toEqual(expect.arrayContaining(['train.json written', `branch pushed: train/${nonce} at ${t.head.slice(0, 12)}`])); + expect(next(r.out)).toBe(`NEXT: gstack-ship-queue train verdict --train ${nonce}`); + expect(result(r.out)).toBe(`GSTACK_RESULT: skill=ship-queue status=complete run=${path.join(fx.state, 'ship-queue', 'trains', nonce, 'train.json')}`); + const resume = queue(fx, ['train', 'build', '--train', nonce]); + expect(resume.status).toBe(EXIT.ok); + const same = queue(fx, ['train', 'build', '--prs', `1@${h1},2@${h2}`]); + expect(nonceOf(same.out)).toBe(nonce); + expect(dispatches(fx).length).toBe(2); + }, 180_000); + + test('--dry-run prints order, trees and the estimate, and writes, pushes and dispatches nothing', () => { + const fx = make(); + const h1 = openPr(fx, '1'); + readyReceipt(fx, '1', h1); + const r = queue(fx, ['train', 'build', '--prs', `1@${h1}`, '--dry-run']); + expect(r.status).toBe(EXIT.ok); + expect(r.out).toContain('ORDER: 1:#1(unscreened)'); + expect(r.out).toMatch(/^PREFIX: T1 [0-9a-f]{40} tree=[0-9a-f]{40} \(\+#1\)$/m); + expect(r.out).toContain('ESTIMATE: one evals.yml full gate (evals_all) on T1 plus free-tests.yml; dollar figure pending the T0 cost replay'); + expect(result(r.out)).toBe('GSTACK_RESULT: skill=ship-queue status=complete run=none'); + expect(fs.existsSync(path.join(fx.state, 'ship-queue', 'trains'))).toBe(false); + expect(remoteRefs(fx)).not.toContain('refs/heads/train/'); + expect(dispatches(fx)).toEqual([]); + }, 120_000); + + test('a conflict stops the train at that PR with the exact recovery line; nothing pushed or dispatched', () => { + const fx = make(); + const h1 = openPr(fx, '1', { file: 'same.txt', text: 'one\n', changelog: false }); + const h2 = openPr(fx, '2', { file: 'same.txt', text: 'two\n', changelog: false }); + readyReceipt(fx, '1', h1); + readyReceipt(fx, '2', h2); + const r = queue(fx, ['train', 'build', '--prs', `1@${h1},2@${h2}`]); + expect(r.status).toBe(EXIT.fail); + expect(line(r.out, 'QUEUE_TRAIN_CONFLICT:')).toMatch(new RegExp(`^QUEUE_TRAIN_CONFLICT: train [0-9a-f]{12} stopped at #2 \\(conflicts: same\\.txt\\) after merging #1; nothing dispatched, no branch pushed; fix: gstack-ship-queue train build --prs 1@${h1}$`)); + expect(r.err).toContain('(QUEUE_TRAIN_CONFLICT)'); + expect(next(r.out)).toBe(`NEXT: gstack-ship-queue train build --prs 1@${h1}`); + expect(remoteRefs(fx)).not.toContain('refs/heads/train/'); + expect(dispatches(fx)).toEqual([]); + }, 120_000); + + test('refusals: a fork PR, a moved head, a PR without READY, and a train still filling (until --urgent)', () => { + const fx = make(); + const h1 = openPr(fx, '1'); + const h2 = openPr(fx, '2', { cross: true }); + readyReceipt(fx, '1', h1); + readyReceipt(fx, '2', h2); + const fork = queue(fx, ['train', 'build', '--prs', `1@${h1},2@${h2}`]); + expect(fork.status).toBe(EXIT.refused); + expect(fork.err).toMatch(/PR #2's head repository is not the base repository.*\(QUEUE_TRAIN_UNTRUSTED\)/); + expect(next(fork.out)).toBe(`NEXT: gstack-ship-queue train build --prs 1@${h1}`); + const moved = queue(fx, ['train', 'build', '--prs', `1@${'f'.repeat(40)}`]); + expect(moved.err).toMatch(/PR #1 moved: pinned ffffffffffff, head is now .*\(QUEUE_TRAIN_MOVED\)/); + const h3 = openPr(fx, '3'); + const notReady = queue(fx, ['train', 'build', '--prs', `3@${h3}`]); + expect(notReady.err).toMatch(/no READY receipt for #3 .*\(QUEUE_NOT_READY\)/); + readyReceipt(fx, '3', h3, { at: '2026-10-18T11:00:00.000Z' }); + const filling = queue(fx, ['train', 'build', '--prs', `3@${h3}`]); + expect(filling.status).toBe(EXIT.refused); + expect(filling.err).toMatch(/1 of 5 PRs; the oldest READY dispatches at 2026-10-18T14:00:00\.000Z.*\(QUEUE_TRAIN_WAITING\)/); + expect(result(filling.out)).toBe('GSTACK_RESULT: skill=ship-queue status=incomplete run=none'); + fx.write(d => { d.rules['evals.yml'] = GREEN_GATE; }); + const urgent = queue(fx, ['train', 'build', '--prs', `3@${h3}`, '--urgent']); + expect(urgent.status).toBe(EXIT.ok); + expect(trainJson(fx, nonceOf(urgent.out)).urgent).toBe(true); + expect(dispatches(fx, 'evals.yml').length).toBe(1); + }, 180_000); + + test('free tests red, cancelled, missing or on another SHA refuse with QUEUE_TRAIN_FREE_RED before any paid dispatch', () => { + const fx = make(); + const h1 = openPr(fx, '1'); + readyReceipt(fx, '1', h1); + fx.write(d => { d.rules['evals.yml'] = GREEN_GATE; }); + const variants: Array<[Rule, RegExp, string[]]> = [ + [{ conclusion: 'failure' }, /free tests red: run \d+ concluded failure/, []], + [{ conclusion: 'cancelled' }, /free tests cancelled: run \d+ concluded cancelled/, []], + [{ silent: true }, /free tests missing: no free-tests\.yml dispatch run appeared on train\//, ['--free-timeout', '1']], + [{ headSha: '0'.repeat(40) }, /free tests stale: run \d+ tested 000000000000 \(workflow_dispatch\), not the train head/, []], + ]; + for (const [rule, why, extra] of variants) { + fx.write(d => { d.rules['free-tests.yml'] = rule; }); + const r = queue(fx, ['train', 'build', '--prs', `1@${h1}`, '--urgent', ...extra]); + expect(r.status).toBe(EXIT.fail); + expect(r.err).toMatch(why); + expect(r.err).toContain('(QUEUE_TRAIN_FREE_RED)'); + expect(trainJson(fx, nonceOf(r.out)).state).toBe('free-red'); + } + expect(dispatches(fx, 'evals.yml')).toEqual([]); + expect(dispatches(fx, 'free-tests.yml').length).toBe(4); + }, 180_000); + + test('ordering: screened PRs first in READY order; enough unscreened PRs wait for their own train', () => { + const fx = make({ ...TRAIN_POLICY, train: { ...TRAIN_BLOCK, max_size: 2 } }); + const h = ['1', '2', '3'].map(pr => openPr(fx, pr)); + readyReceipt(fx, '1', h[0]!, { at: '2026-10-18T08:00:00.000Z' }); + readyReceipt(fx, '2', h[1]!, { at: '2026-10-18T07:00:00.000Z' }); + readyReceipt(fx, '3', h[2]!, { at: '2026-10-18T09:00:00.000Z', screen: true }); + const r = queue(fx, ['train', 'build', '--prs', `1@${h[0]},2@${h[1]},3@${h[2]}`, '--dry-run']); + expect(r.status).toBe(EXIT.ok); + expect(line(r.out, 'ORDER:')).toBe('ORDER: 1:#3(screened)'); + expect(line(r.out, 'HELD:')).toBe('HELD: #2,#1 (unscreened PRs form their own train)'); + const wide = make(); + const w = ['1', '2', '3'].map(pr => openPr(wide, pr)); + readyReceipt(wide, '1', w[0]!, { at: '2026-10-18T08:00:00.000Z' }); + readyReceipt(wide, '2', w[1]!, { at: '2026-10-18T07:00:00.000Z' }); + readyReceipt(wide, '3', w[2]!, { at: '2026-10-18T09:00:00.000Z', screen: true }); + const all = queue(wide, ['train', 'build', '--prs', `1@${w[0]},2@${w[1]},3@${w[2]}`, '--dry-run']); + expect(line(all.out, 'ORDER:')).toBe('ORDER: 1:#3(screened) 2:#2(unscreened) 3:#1(unscreened)'); + expect(all.out).toMatch(/^SLOT: 1 #3 head=[0-9a-f]{40} screened \(narrow\) ready_at=2026-10-18T09:00:00\.000Z$/m); + }, 180_000); +}); + +describe('dispatch recovery and status', () => { + test('a partial dispatch (gh failed after GitHub accepted it) is unknown, never re-dispatched, and train status finds the run by nonce', () => { + const fx = make(); + const { r, nonce } = twoPrTrain(fx, { ...GREEN_GATE, failButCreate: true }); + expect(r.status).toBe(EXIT.refused); + expect(r.err).toContain('(QUEUE_TRAIN_DISPATCH_UNKNOWN)'); + expect(next(r.out)).toBe(`NEXT: gstack-ship-queue train status --train ${nonce}`); + expect(trainJson(fx, nonce).state).toBe('dispatch-unknown'); + expect(queue(fx, ['train', 'verdict', '--train', nonce]).err).toContain('(QUEUE_TRAIN_DISPATCH_UNKNOWN)'); + expect(queue(fx, ['train', 'build', '--train', nonce]).err).toContain('(QUEUE_TRAIN_DISPATCH_UNKNOWN)'); + const status = queue(fx, ['train', 'status', '--train', nonce]); + expect(status.status).toBe(EXIT.ok); + expect(status.out).toMatch(new RegExp(`^ATTEMPT: a1 gate state=dispatched nonce=${nonce.replace(/\./g, '\\.')} run=101 `, 'm')); + expect(trainJson(fx, nonce).state).toBe('gate-pending'); + expect(next(status.out)).toBe(`NEXT: gstack-ship-queue train verdict --train ${nonce}`); + expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok); + expect(dispatches(fx, 'evals.yml').length).toBe(1); + }, 180_000); + + test('a dispatch that never started is marked absent by status; only then does build dispatch it again, under the same nonce', () => { + const fx = make(); + const { r, nonce } = twoPrTrain(fx, { silent: true }); + expect(r.err).toContain('(QUEUE_TRAIN_DISPATCH_UNKNOWN)'); + const status = queue(fx, ['train', 'status', '--train', nonce]); + expect(status.out).toContain('ATTEMPT: a1 gate state=absent'); + expect(next(status.out)).toBe(`NEXT: gstack-ship-queue train build --train ${nonce}`); + fx.write(d => { d.rules['evals.yml'] = GREEN_GATE; }); + const again = queue(fx, ['train', 'build', '--train', nonce]); + expect(again.status).toBe(EXIT.ok); + expect(dispatches(fx, 'evals.yml').map(d => (d.inputs as any).nonce)).toEqual([nonce, nonce]); + expect(trainJson(fx, nonce).attempts).toEqual(['a1', 'a2']); + }, 180_000); + + test('train status on a clean state root rebuilds the records from refs/notes/gstack-trains and spends nothing', () => { + const fx = make(); + const { nonce } = twoPrTrain(fx); + expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok); + const before = dispatches(fx).length; + const clone = coordinatorClone(fx.repo); + const fresh = path.join(fx.repo.root, 'fresh-state'); + const s = queue(fx, ['train', 'status', '--train', nonce], { GSTACK_STATE_ROOT: fresh }, clone); + expect(s.status).toBe(EXIT.ok); + expect(s.out).toMatch(/^NOTES: read 2 record\(s\), restored 2 /m); + expect(s.out).toContain(`TRAIN: ${nonce} state=green`); + expect(s.out).toContain('CHECK: gstack/full-gate on'); + expect(trainJson(fx, nonce, fresh)).toEqual(trainJson(fx, nonce)); + expect(dispatches(fx).length).toBe(before); + const all = queue(fx, ['train', 'status'], { GSTACK_STATE_ROOT: fresh }, clone); + expect(all.out).toContain(`NEXT(${nonce}): gstack-ship-queue verify --pr 1 --train ${nonce}`); + }, 180_000); +}); + +describe('verdict, bisect, rebuild', () => { + function redTrain(fx: TrainFixture, pattern: string) { + const t = twoPrTrain(fx, RED_GATE); + const v = queue(fx, ['train', 'verdict', '--train', t.nonce]); + const prefixes = trainJson(fx, t.nonce).prefixes.map((p: any) => p.commit); + fx.write(d => { d.rules['evals.yml'] = { bisect: { prefixes, patterns: { c1: pattern } } }; }); + return { ...t, v, prefixes }; + } + + test('a red train names its cases and a capped bisect; the bisect runs only the red cases on every prefix and attributes a monotone pattern; prune deletes the dropped train\'s branch', () => { + const fx = make(); + const { h1, nonce, v, prefixes } = redTrain(fx, 'GGR'); + expect(v.status).toBe(EXIT.fail); + expect(v.err).toContain(`train ${nonce} is red on c1`); + expect(v.out).toContain('ATTEMPT: a1 gate run 101 RED red=c1 cost=$1.70'); + expect(next(v.out)).toBe(`NEXT: gstack-ship-queue train bisect --train ${nonce} --spend-cap 2.70`); + const dry = queue(fx, ['train', 'bisect', '--train', nonce, '--spend-cap', '2.70', '--dry-run']); + expect(dry.out).toContain('BISECT: cases c1 prefixes T0,T1,T2 need results; $0.90 reserved per prefix; spent $0.00, cap $2.70'); + expect(dry.out).toContain('DRY-RUN: nothing dispatched'); + expect(dispatches(fx, 'evals.yml').length).toBe(1); + const go = queue(fx, ['train', 'bisect', '--train', nonce, '--spend-cap', '2.70']); + expect(go.status).toBe(EXIT.ok); + const sent = dispatches(fx, 'evals.yml').at(-1)!.inputs as any; + expect(sent.cases).toBe('c1'); + expect(sent.prefix_shas).toBe(prefixes.join(',')); + expect(sent.expected_sha).toBe(prefixes[2]); + expect(sent.nonce).toMatch(/^gate\.bisect-/); + const done = queue(fx, ['train', 'bisect', '--train', nonce, '--spend-cap', '2.70']); + expect(done.status).toBe(EXIT.ok); + expect(done.out).toContain('PATTERN: c1 T0=G T1=G T2=R'); + expect(done.out).toContain('BISECT: ATTRIBUTED #2'); + expect(next(done.out)).toBe(`NEXT: gstack-ship-queue train build --prs 1@${h1}`); + expect(trainJson(fx, nonce).state).toBe('attributed'); + expect(queue(fx, ['train', 'prune', '--dry-run']).out).toContain(`WOULD-DELETE: train/${nonce} (attributed)`); + expect(queue(fx, ['train', 'prune']).out).toContain(`DELETED: train/${nonce} (attributed)`); + expect(remoteRefs(fx)).not.toContain(`refs/heads/train/${nonce}`); + expect(queue(fx, ['train', 'prune']).out).toContain(`GONE: train/${nonce}`); + }, 180_000); + + test('red on every prefix, T0 included, is RED_ON_MAIN; a non-monotone pattern is unattributed and earns no rebuild', () => { + const main = make(); + const m = redTrain(main, 'RRR'); + queue(main, ['train', 'bisect', '--train', m.nonce, '--spend-cap', '5']); + const onMain = queue(main, ['train', 'bisect', '--train', m.nonce, '--spend-cap', '5']); + expect(onMain.status).toBe(EXIT.fail); + expect(onMain.err).toContain('(QUEUE_TRAIN_RED_ON_MAIN)'); + expect(next(onMain.out)).toMatch(/^NEXT: fix forward on main first, then gstack-ship-queue train build --prs /); + const fx = make(); + const n = redTrain(fx, 'GRG'); + queue(fx, ['train', 'bisect', '--train', n.nonce, '--spend-cap', '5']); + const mixed = queue(fx, ['train', 'bisect', '--train', n.nonce, '--spend-cap', '5']); + expect(mixed.status).toBe(EXIT.fail); + expect(mixed.err).toContain('(QUEUE_TRAIN_UNATTRIBUTED)'); + expect(mixed.out).toContain('BISECT: UNATTRIBUTED (not monotone)'); + expect(next(mixed.out)).toMatch(/^NEXT: route the red cases to a repair lane \(GStack Minder\); once the repair merges: /); + const refused = queue(fx, ['train', 'rebuild', '--train', n.nonce]); + expect(refused.status).toBe(EXIT.refused); + expect(refused.err).toMatch(/rebuild refused: the bisect was not all green \(c1=GRG\).*\(QUEUE_TRAIN_REBUILD_REFUSED\)/); + }, 240_000); + + test('rebuild-once: refused without a bisect; an all-green bisect earns exactly one unchanged rebuild, both runs reported; a second red stays red', () => { + const fx = make(); + const { nonce } = redTrain(fx, 'GGG'); + expect(queue(fx, ['train', 'rebuild', '--train', nonce]).err).toMatch(/rebuild refused: no bisect record.*\(QUEUE_TRAIN_REBUILD_REFUSED\)/); + queue(fx, ['train', 'bisect', '--train', nonce, '--spend-cap', '5']); + const green = queue(fx, ['train', 'bisect', '--train', nonce, '--spend-cap', '5']); + expect(green.out).toContain('BISECT: UNATTRIBUTED (all green)'); + expect(next(green.out)).toBe(`NEXT: gstack-ship-queue train rebuild --train ${nonce}`); + fx.write(d => { d.rules['evals.yml'] = RED_GATE; }); + const rebuild = queue(fx, ['train', 'rebuild', '--train', nonce]); + expect(rebuild.status).toBe(EXIT.ok); + expect(dispatches(fx, 'evals.yml').at(-1)!.inputs).toMatchObject({ evals_all: 'true', nonce: `${nonce}.2` }); + const second = queue(fx, ['train', 'verdict', '--train', nonce]); + expect(second.status).toBe(EXIT.fail); + expect(second.out).toContain('ATTEMPT: a1 gate run 101 RED red=c1'); + expect(second.out).toMatch(/^ATTEMPT: a2 rebuild run \d+ RED red=c1/m); + expect(next(second.out)).toMatch(/^NEXT: second red stays red: route c1 to a repair lane/); + expect(queue(fx, ['train', 'rebuild', '--train', nonce]).err).toMatch(/rebuild refused: already rebuilt once/); + expect(queue(fx, ['train', 'bisect', '--train', nonce, '--spend-cap', '5']).err).toContain('second red is not bisected again'); + }, 240_000); + + test('a capped bisect launches only what fits, keeps completed prefixes, and resumes without repeating one', () => { + const fx = make(); + const { nonce, prefixes } = redTrain(fx, 'GGR'); + const none = queue(fx, ['train', 'bisect', '--train', nonce, '--spend-cap', '0.5']); + expect(none.status).toBe(EXIT.refused); + expect(none.err).toContain('(QUEUE_TRAIN_BISECT_CAPPED)'); + expect(next(none.out)).toBe(`NEXT: gstack-ship-queue train bisect --train ${nonce} --spend-cap 2.70`); + expect(dispatches(fx, 'evals.yml').length).toBe(1); + const part = queue(fx, ['train', 'bisect', '--train', nonce, '--spend-cap', '1.9']); + expect(part.status).toBe(EXIT.refused); + expect(part.err).toContain('(QUEUE_TRAIN_BISECT_CAPPED)'); + expect(part.out).toContain('BISECT: launch T0,T1; 1 prefix(es) wait for a higher cap'); + expect((dispatches(fx, 'evals.yml').at(-1)!.inputs as any).prefix_shas).toBe(prefixes.slice(0, 2).join(',')); + const resume = queue(fx, ['train', 'bisect', '--train', nonce, '--spend-cap', '5']); + expect(resume.status).toBe(EXIT.ok); + expect((dispatches(fx, 'evals.yml').at(-1)!.inputs as any).prefix_shas).toBe(prefixes[2]); + const done = queue(fx, ['train', 'bisect', '--train', nonce, '--spend-cap', '5']); + expect(done.out).toContain('BISECT: ATTRIBUTED #2'); + expect(done.out).toContain('BISECT: spent $1.50 of cap $5.00'); + expect(dispatches(fx, 'evals.yml').length).toBe(3); + }, 240_000); + + test('a bisect refuses prefixes that are not ancestors of train/', () => { + const fx = make(); + const { nonce } = redTrain(fx, 'GGR'); + fx.repo.git('push', '-q', '-f', 'origin', `origin/main:refs/heads/train/${nonce}`); + const r = queue(fx, ['train', 'bisect', '--train', nonce, '--spend-cap', '5']); + expect(r.status).toBe(EXIT.refused); + expect(r.err).toMatch(/train\/\S+ is at [0-9a-f]{12}, not the recorded train head .*\(QUEUE_TRAIN_PREFIX_INVALID\)/); + expect(dispatches(fx, 'evals.yml').length).toBe(1); + }, 180_000); + + test('an INFRA/INCOMPLETE-only census is never green and earns one re-dispatch, reported beside the first run', () => { + const fx = make(); + const infra: Rule = { conclusion: 'failure', record: { panels: [PASS('c1'), FAIL('c2', 'INFRA'), PASS('j1', 'judge')], census: { ...COMPLETE, killed_before_upload: 1 }, verdict: 'RED' } }; + const { nonce } = twoPrTrain(fx, infra); + const v = queue(fx, ['train', 'verdict', '--train', nonce]); + expect(v.status).toBe(EXIT.fail); + expect(v.err).toMatch(/gate is incomplete: 1 slice killed before upload; c2 FAIL \(INFRA\).*\(QUEUE_TRAIN_INCOMPLETE\)/); + expect(next(v.out)).toBe(`NEXT: gstack-ship-queue train rebuild --train ${nonce} --infra (the single re-dispatch EVAL_POLICY.infraRedispatch allows; report both runs)`); + expect(queue(fx, ['train', 'rebuild', '--train', nonce]).err).toContain('rebuild refused: no bisect record'); + const again = queue(fx, ['train', 'rebuild', '--train', nonce, '--infra']); + expect(again.status).toBe(EXIT.ok); + const second = queue(fx, ['train', 'verdict', '--train', nonce]); + expect(second.out).toMatch(/^ATTEMPT: a2 infra-redispatch run \d+ INCOMPLETE/m); + expect(next(second.out)).toMatch(/^NEXT: diagnose run /); + expect(queue(fx, ['train', 'rebuild', '--train', nonce, '--infra']).err).toMatch(/already re-dispatched once.*\(QUEUE_TRAIN_REBUILD_REFUSED\)/); + }, 180_000); +}); + +describe('verify, freeze, repair and clear', () => { + test('verify accepts each merge only at prefix equality modulo stamps; an interrupted publication freezes the queue until one repair train lands', () => { + const fx = make(); + const { h1, h2, nonce } = twoPrTrain(fx); + expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok); + const t = trainJson(fx, nonce); + const coord = coordinatorClone(fx.repo); + const v = (pr: string, train = nonce) => queue(fx, ['verify', '--pr', pr, '--train', train], {}, coord); + + const pre1 = v('1'); + expect(pre1.status).toBe(EXIT.ok); + expect(pre1.out).toContain(`VERIFY_TRAIN: #1 may merge as slot 1 of train ${nonce}`); + expect(line(pre1.out, 'MERGE_CONDITION:')).toBe(`MERGE_CONDITION: sha=${h1}`); + const early = v('2'); + expect(early.status).toBe(EXIT.fail); + expect(early.err).toMatch(/is neither T1 nor T2 modulo stamps.*nothing of train \S+ merged, so the queue is not frozen.*\(QUEUE_TRAIN_TREE_MISMATCH\)/); + + publishMain(fx, t.prefixes[1].commit, '1.2.4.0'); + const post1 = v('1'); + expect(post1.status).toBe(EXIT.ok); + expect(post1.out).toMatch(/^main: [0-9a-f]{40} = T1 \(same modulo stamps \(1 stamp file\(s\)\)\)$/m); + expect(post1.out).toContain(`VERIFY_TRAIN: #1 merged; main's tree is T1 of train ${nonce} (1/2 merged)`); + expect(next(post1.out)).toBe(`NEXT: gstack-ship-queue verify --pr 2 --train ${nonce}`); + expect(line(v('2').out, 'MERGE_CONDITION:')).toBe(`MERGE_CONDITION: sha=${h2}`); + + const broken = publishMain(fx, t.prefixes[2].commit, '1.2.5.0', { file: 'stray.txt', text: 'not in the train\n' }); + const mismatch = v('2'); + expect(mismatch.status).toBe(EXIT.fail); + expect(mismatch.err).toContain('(QUEUE_TRAIN_TREE_MISMATCH)'); + expect(line(mismatch.out, 'QUEUE_TRAIN_FROZEN:')).toMatch(new RegExp(`^QUEUE_TRAIN_FROZEN: train \\S+ merged #1; main ${broken.slice(0, 12)} tree [0-9a-f]{12}; backstop dispatched run \\d+ \\(nonce gate\\.backstop-`)); + const backstop = dispatches(fx, 'evals.yml').at(-1)!; + expect([backstop.ref, (backstop.inputs as any).expected_sha, (backstop.inputs as any).evals_all]).toEqual(['main', broken, 'true']); + const freezeFile = path.join(fx.state, 'ship-queue', 'freeze', `${repoKey(fx)}.json`); + expect(JSON.parse(fs.readFileSync(freezeFile, 'utf8'))).toMatchObject({ kind: 'freeze', merged: ['1'], unmerged: [{ pr: '2', head: h2 }], cleared_at: null }); + + const h3 = openPr(fx, '3'); + readyReceipt(fx, '3', h3); + const ordinary = queue(fx, ['train', 'build', '--prs', `3@${h3}`, '--urgent']); + expect(ordinary.status).toBe(EXIT.refused); + expect(ordinary.err).toContain('(QUEUE_TRAIN_FROZEN)'); + expect(next(ordinary.out)).toBe('NEXT: gstack-ship-queue train build --repair --prs @'); + const fresh = path.join(fx.repo.root, 'fresh-state'); + expect(queue(fx, ['train', 'status'], { GSTACK_STATE_ROOT: fresh }, coord).out).toMatch(/^QUEUE_TRAIN_FROZEN: train /m); + + const repair = queue(fx, ['train', 'build', '--repair', '--prs', `3@${h3}`]); + expect(repair.status).toBe(EXIT.ok); + const rn = nonceOf(repair.out); + expect(trainJson(fx, rn)).toMatchObject({ repair: true, base: { sha: broken } }); + expect(queue(fx, ['train', 'verdict', '--train', rn]).status).toBe(EXIT.ok); + expect(v('2').err).toContain('(QUEUE_TRAIN_FROZEN)'); + expect(dispatches(fx, 'evals.yml').filter(d => d.ref === 'main').length).toBe(1); + expect(line(v('3', rn).out, 'MERGE_CONDITION:')).toBe(`MERGE_CONDITION: sha=${h3}`); + publishMain(fx, trainJson(fx, rn).prefixes[1].commit, '1.2.6.0'); + const cleared = v('3', rn); + expect(cleared.status).toBe(EXIT.ok); + expect(cleared.out).toMatch(/^FREEZE: cleared \(repair #3 via train \S+; main [0-9a-f]{12} carries a green record\)$/m); + expect(next(cleared.out)).toBe(`NEXT: gstack-ship-queue train build --prs 2@${h2}`); + expect(JSON.parse(fs.readFileSync(freezeFile, 'utf8')).cleared_at).toBe('2026-10-18T12:00:00.000Z'); + expect(queue(fx, ['train', 'status']).out).not.toContain('QUEUE_TRAIN_FROZEN'); + }, 300_000); +});