mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-04 15:57:33 +01:00
feat(ship-queue): merge trains with pattern bisect and train-record verify (GSTA-49 T1)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
25dbe34f42
commit
210f998c10
20 files changed
+3021
-34
No files matched your search
@@ -260,6 +260,24 @@ gstack_result_anchor() {
|
||||
QUEUE_VERIFY_MISMATCH) printf '%s' 'queue-verify-mismatch' ;;
|
||||
QUEUE_GATE_MISSING) printf '%s' 'queue-gate-missing' ;;
|
||||
GATE_AHEAD_ATTEMPT_INVALID) printf '%s' 'gate-ahead-attempt-invalid' ;;
|
||||
QUEUE_TRAIN_OFF) printf '%s' 'queue-train-off' ;;
|
||||
QUEUE_TRAIN_UNTRUSTED) printf '%s' 'queue-train-untrusted' ;;
|
||||
QUEUE_TRAIN_MOVED) printf '%s' 'queue-train-moved' ;;
|
||||
QUEUE_TRAIN_CONFLICT) printf '%s' 'queue-train-conflict' ;;
|
||||
QUEUE_TRAIN_FREE_RED) printf '%s' 'queue-train-free-red' ;;
|
||||
QUEUE_TRAIN_DISPATCH_UNKNOWN) printf '%s' 'queue-train-dispatch-unknown' ;;
|
||||
QUEUE_TRAIN_WAITING) printf '%s' 'queue-train-waiting' ;;
|
||||
QUEUE_TRAIN_PENDING) printf '%s' 'queue-train-pending' ;;
|
||||
QUEUE_TRAIN_INCOMPLETE) printf '%s' 'queue-train-incomplete' ;;
|
||||
QUEUE_TRAIN_RED) printf '%s' 'queue-train-red' ;;
|
||||
QUEUE_TRAIN_RED_ON_MAIN) printf '%s' 'queue-train-red-on-main' ;;
|
||||
QUEUE_TRAIN_UNATTRIBUTED) printf '%s' 'queue-train-unattributed' ;;
|
||||
QUEUE_TRAIN_BISECT_CAPPED) printf '%s' 'queue-train-bisect-capped' ;;
|
||||
QUEUE_TRAIN_PREFIX_INVALID) printf '%s' 'queue-train-prefix-invalid' ;;
|
||||
QUEUE_TRAIN_REBUILD_REFUSED) printf '%s' 'queue-train-rebuild-refused' ;;
|
||||
QUEUE_TRAIN_TREE_MISMATCH) printf '%s' 'queue-train-tree-mismatch' ;;
|
||||
QUEUE_TRAIN_FROZEN) printf '%s' 'queue-train-frozen' ;;
|
||||
READY_SCREEN_CHOICE_REQUIRED) printf '%s' 'ready-screen-choice-required' ;;
|
||||
PREGATE_TIMING_ASSERTION) printf '%s' 'pregate-timing-assertion' ;;
|
||||
APPROVAL_CLASSES_INVALID) printf '%s' 'approval-classes-invalid' ;;
|
||||
APPROVAL_CLASS_FOUND) printf '%s' 'approval-class-found' ;;
|
||||
|
||||
+23
-5
@@ -11,11 +11,15 @@
|
||||
* gstack-ship-queue gate-ahead --after <pr> --pr <n> [--predecessor-ref <ref>] [--allow-repo-commands]
|
||||
* gstack-ship-queue gate-ahead --record --pr <n> --attempt <id> --exit <rc> --log <file>
|
||||
* gstack-ship-queue gate-ahead --list --pr <n>
|
||||
* gstack-ship-queue train build|status|verdict|bisect|rebuild|prune ... (merge trains; train --help)
|
||||
*
|
||||
* Every subcommand takes --base <branch> (default origin/HEAD, else main) and
|
||||
* ends with GSTACK_RESULT: skill=ship-queue status=complete|refused run=<path>.
|
||||
* gstack-restamp stays the primitive. Nothing here pushes. Logic:
|
||||
* lib/ship-queue.ts, lib/ship-queue-gate.ts.
|
||||
* gstack-restamp stays the primitive. Nothing here pushes, except under a
|
||||
* ship policy with a `train` block: merge trains push train/<nonce> branches
|
||||
* and the refs/notes/gstack-trains records, and dispatch through eval-bg.
|
||||
* Logic: lib/ship-queue.ts, lib/ship-queue-gate.ts, lib/ship-queue-train*.ts
|
||||
* (loaded only on the train paths).
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as path from 'node:path';
|
||||
@@ -41,6 +45,9 @@ const HELP = `gstack-ship-queue — the shared merge queue's protocol steps (doc
|
||||
gate-ahead --record --pr <n> --attempt <id> --exit <rc> --log <file>
|
||||
bind the gate result to that attempt's gated tree; removes its worktree
|
||||
gate-ahead --list --pr <n> attempts for a PR, newest last (the current one)
|
||||
train build|status|verdict|bisect|rebuild|prune ...
|
||||
merge trains, when the base policy declares a train block (train --help;
|
||||
docs/ship-policy.md#run-a-merge-train); ready then takes --screen|--no-screen
|
||||
|
||||
--base <branch>: base branch (default: origin/HEAD, else main). Nothing here pushes.
|
||||
${exitTableHelp()}`;
|
||||
@@ -122,7 +129,7 @@ function gateAhead(ctx: QueueContext, args: string[]): number {
|
||||
});
|
||||
}
|
||||
|
||||
function main(argv: string[]): number {
|
||||
async function main(argv: string[]): Promise<number> {
|
||||
const [cmd, ...args] = argv;
|
||||
if (!cmd || cmd === '--help' || cmd === '-h' || cmd === 'help') { process.stdout.write(HELP + '\n'); return cmd ? EXIT.ok : EXIT.usage; }
|
||||
const probe = spawnSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8', timeout: 15_000 });
|
||||
@@ -132,11 +139,21 @@ function main(argv: string[]): number {
|
||||
const ctx: QueueContext = { repoRoot, base: resolveBaseBranch(repoRoot, flag(args, '--base')), gh };
|
||||
const pr = flag(args, '--pr');
|
||||
if (cmd === 'gate-ahead') return gateAhead(ctx, args);
|
||||
if (cmd === 'train') {
|
||||
const train = await import('../lib/ship-queue-train-cli');
|
||||
if (!args[0] || args[0].startsWith('-')) { process.stdout.write(train.TRAIN_HELP + '\n'); return args.includes('--help') || args.includes('-h') ? EXIT.ok : EXIT.usage; }
|
||||
return train.trainMain(ctx, args);
|
||||
}
|
||||
if (!pr || !/^\d+$/.test(pr)) return usage(`${cmd} needs --pr <number>`);
|
||||
const gateExit = flag(args, '--gate-exit');
|
||||
if (gateExit !== undefined && !/^-?\d+$/.test(gateExit)) return usage('--gate-exit needs an exit code');
|
||||
const gate = { gateExit: gateExit === undefined ? undefined : Number(gateExit), gateLog: flag(args, '--gate-log') };
|
||||
if (cmd === 'ready') return finish('ready', ready(ctx, { pr, pregate: flag(args, '--pregate'), ...gate }));
|
||||
if (cmd === 'ready') {
|
||||
const loaded = loadBasePolicy(ctx);
|
||||
if (!('message' in loaded) && loaded.policy.train) return (await import('../lib/ship-queue-train-cli')).readyMain(ctx, args, { pr, pregate: flag(args, '--pregate'), ...gate });
|
||||
if (['--screen', '--no-screen', '--broad'].some(f => args.includes(f))) process.stderr.write('gstack-ship-queue ready: --screen/--no-screen/--broad ignored: the base policy declares no train block\n');
|
||||
return finish('ready', ready(ctx, { pr, pregate: flag(args, '--pregate'), ...gate }));
|
||||
}
|
||||
if (cmd === 'stamp') {
|
||||
const version = flag(args, '--version');
|
||||
const expectGated = flag(args, '--expect-gated');
|
||||
@@ -149,6 +166,7 @@ function main(argv: string[]): number {
|
||||
}));
|
||||
}
|
||||
if (cmd === 'verify') {
|
||||
if (flag(args, '--train')) return (await import('../lib/ship-queue-train-cli')).verifyMain(ctx, args, pr);
|
||||
const record = flag(args, '--record');
|
||||
return finish('verify', verify(ctx, {
|
||||
pr, gated: flag(args, '--gated'), record: record ? path.resolve(record) : undefined, receipt: flag(args, '--receipt'),
|
||||
@@ -158,4 +176,4 @@ function main(argv: string[]): number {
|
||||
return usage(`unknown subcommand ${cmd}`);
|
||||
}
|
||||
|
||||
process.exitCode = main(process.argv.slice(2));
|
||||
process.exitCode = await main(process.argv.slice(2));
|
||||
+184
-1
@@ -18,7 +18,7 @@ introduces or changes the policy runs under the reviewed bootstrap path
|
||||
|
||||
Tools: `gstack-ship-policy init|validate|show-effective|check-history`,
|
||||
`gstack-restamp`, `gstack-tree-receipt`, `gstack-ship-receipt`,
|
||||
`gstack-ship-queue` ([shared merge queue](#run-a-shared-merge-queue)),
|
||||
`gstack-ship-queue` ([shared merge queue](#run-a-shared-merge-queue), [merge trains](#run-a-merge-train)),
|
||||
`gstack-evidence bundle|ancestor`. They share the exit table (`0` ok, `1`
|
||||
fail, `2` usage, `3` refused or needs a flag) and end every error line with a
|
||||
code from [troubleshooting.md](troubleshooting.md#queue-aware-ship-policy-restamp-tree-receipt-ship-receipt).
|
||||
@@ -58,6 +58,7 @@ refuses the file (`POLICY_GSTACK_TOO_OLD`). `validate` ends with
|
||||
| `gate_command` | command or `null` | `null` | The repo's full pre-merge gate. `gstack-ship-queue gate-ahead` and `ready` print it for the caller to run (in the background or in CI); gstack never executes it. Needs `min_gstack_version`. |
|
||||
| `full_e2e` | `true` \| `false` | `false` | `true`: every merge needs a passing full-gate result bound to a tree equal to the merging one modulo stamps; `gstack-ship-queue ready`, `stamp` and `verify` refuse without one (`QUEUE_GATE_MISSING`): a gate-ahead record, or the lane's `--gate-exit 0 --gate-log <file>` from its run on the gated head. Needs `min_gstack_version`. |
|
||||
| `min_gstack_version` | version or `null` | `null` | The oldest gstack that may read the file; see the compatibility rule above. |
|
||||
| `train` | `{ gate_workflow, free_workflow, full_gate_check, record_artifact, max_size, fill_hours, screen_selection, trial_reservation_usd }` or `null` | `null` | Turns on [merge trains](#run-a-merge-train), read from the base revision only. Every field is required and an unknown field is `POLICY_INVALID`: `max_size` 1 to 6, `fill_hours` above 0 and at most 24, `screen_selection` `gstack-paid` \| `none`, `trial_reservation_usd` above 0 and at most 100. `null` keeps the per-PR queue exactly as it is. Needs `min_gstack_version`. |
|
||||
|
||||
Containment, checked by `validate` and again at write time: paths and globs are
|
||||
repo-relative with no `..` segment, under 256 characters, at most 64 per list,
|
||||
@@ -254,6 +255,188 @@ prints) only after every lane's gstack is at least that version, the crew's
|
||||
install floor (`GSTACK_MIN_REVISION`). A lane on an older gstack refuses the
|
||||
policy by name rather than reading it differently.
|
||||
|
||||
<a id="run-a-merge-train"></a>
|
||||
## Run a merge train
|
||||
|
||||
A merge train gates several READY PRs with one full gate instead of one gate
|
||||
each. The coordinator merges the pinned, unstamped PR heads in slot order onto
|
||||
main in a scratch worktree, records each prefix tree (T0 is main, Tk is the
|
||||
train head), runs the free tests and then one full gate on the train head, and
|
||||
merges the PRs one at a time, each only while main's tree equals the prefix
|
||||
the gate covered modulo stamps. A red train is bisected on its red cases only.
|
||||
|
||||
The surface is off unless the base revision's `.gstack/ship-policy.json`
|
||||
declares a `train` block. Without one, `train …` and `verify --train` refuse
|
||||
with `QUEUE_TRAIN_OFF`, and `ready --pr <n>` prints exactly the per-PR READY
|
||||
block above (`--screen`, `--no-screen` and `--broad` are ignored with a
|
||||
warning on stderr). A block for gstack itself would read:
|
||||
|
||||
```json
|
||||
"train": {
|
||||
"gate_workflow": "evals.yml", "free_workflow": "free-tests.yml",
|
||||
"full_gate_check": "gstack/full-gate", "record_artifact": "gstack-train-record",
|
||||
"max_size": 5, "fill_hours": 3, "screen_selection": "gstack-paid",
|
||||
"trial_reservation_usd": 1.5
|
||||
},
|
||||
"min_gstack_version": "<the version that introduced train>"
|
||||
```
|
||||
|
||||
gstack's own policy does not carry it yet: trains go live for gstack only
|
||||
after the workflow side (the train-nonce record, the `cases` and
|
||||
`prefix_shas` inputs, the dispatched screen) has merged and one dry-run train
|
||||
of two already-merged heads has passed. `gate_workflow` is dispatched with
|
||||
`evals_all`, `expected_sha` and `nonce` (bisects add `cases` and
|
||||
`prefix_shas`, screens add `screen_pr`, `screen_merge_sha` and
|
||||
`screen_policy`). That run publishes `full_gate_check` as a check run on the
|
||||
train head and the train record as the `record_artifact` artifact only when
|
||||
its census is complete and green; gstack reads both and never writes a
|
||||
check. `screen_selection: gstack-paid` computes gstack's paid case selection
|
||||
for the screen recommendation; `none` leaves the choice to the lane.
|
||||
`trial_reservation_usd` is what the bisect reserves for a trial without cost
|
||||
data.
|
||||
|
||||
Every command prints its lines, the coded error line on stderr, then
|
||||
`NEXT: <command>` and `GSTACK_RESULT: skill=ship-queue status=… run=<record>`.
|
||||
Records live under `$GSTACK_STATE_ROOT/ship-queue/` (`trains/<nonce>/train.json`
|
||||
with one attempt record per dispatch, `ready/` receipts and `freeze/`) and
|
||||
are mirrored to origin as git notes on `refs/notes/gstack-trains` before any
|
||||
remote effect, so a lost machine loses nothing: `train status` on a clean
|
||||
state root rebuilds them. `GSTACK_SHIP_QUEUE_NOW` pins the clock for tests.
|
||||
|
||||
**1. Lane: READY with a screen choice.** Under a train policy READY needs an
|
||||
explicit choice:
|
||||
|
||||
```bash
|
||||
gstack-ship-queue ready --pr 42 # prints SCREEN: lines and the recommendation, refuses
|
||||
gstack-ship-queue ready --pr 42 --no-screen # READY receipt; the PR rides unscreened
|
||||
gstack-ship-queue ready --pr 42 --screen [--broad] # dispatches one paid screen on the PR's merge SHA
|
||||
```
|
||||
|
||||
Without a choice it prints the `SCREEN:` selection (with
|
||||
`screen_selection: gstack-paid`, the skill's own cases and `recommend
|
||||
--screen` or `--no-screen`: `--screen` when the screen selects a skill's own
|
||||
cases) and refuses with `READY_SCREEN_CHOICE_REQUIRED`. The price line says
|
||||
the dollar figure is pending the cost replay; until it lands, `SCREEN:` shows
|
||||
case counts and the recommendation only. A lane that already ran focused
|
||||
local cases says so in the READY block and may pass `--no-screen`. `--screen`
|
||||
dispatches `gate_workflow` with `screen_pr`, `screen_merge_sha` (the
|
||||
`refs/pull/<n>/merge` SHA) and `screen_policy` (`narrow`, or `broad` with
|
||||
`--broad`, today's full PR selection). The nonce key carries the purpose and
|
||||
the policy, so a repeat follows the same run and a broad screen never adopts
|
||||
a narrow one. The READY block gains `screen: …`, and the receipt is written
|
||||
locally and to the notes ref. `--dry-run` prints the dispatch without sending
|
||||
it.
|
||||
|
||||
**2. Coordinator: build.** Name the pins exactly as READY reported them:
|
||||
|
||||
```bash
|
||||
gstack-ship-queue train build --prs 41@<sha>,42@<sha>,43@<sha> --dry-run
|
||||
gstack-ship-queue train build --prs 41@<sha>,42@<sha>,43@<sha>
|
||||
```
|
||||
|
||||
It prints `ORDER:` (the slot order to announce: screened PRs first in READY
|
||||
order, then unscreened; when `max_size` or more unscreened PRs wait they form
|
||||
their own train and `HELD:` lists them), one `PREFIX: T<i> <commit>
|
||||
tree=<tree>` per prefix and one `SLOT:` per PR. `--dry-run` adds the
|
||||
`ESTIMATE:` line and writes, pushes and dispatches nothing. Otherwise it
|
||||
writes `train.json`, pushes `train/<nonce>`, dispatches `free_workflow` there
|
||||
and waits (`--free-timeout <s>`, default 2700) for a green
|
||||
`workflow_dispatch` run on that exact SHA, then dispatches the full gate
|
||||
through eval-bg's nonce. A train under `max_size` waits until its oldest READY
|
||||
has waited `fill_hours` (`QUEUE_TRAIN_WAITING`); `--urgent` dispatches a
|
||||
one-PR train at once. The same pins on the same base return the live train
|
||||
instead of a second one; `train build --train <nonce>` resumes a train from
|
||||
its record. Refusals, all before any spend: `QUEUE_TRAIN_UNTRUSTED` (a fork
|
||||
head), `QUEUE_TRAIN_MOVED` (a head moved from its pin), `QUEUE_NOT_READY` (no
|
||||
READY receipt at that head), `QUEUE_TRAIN_CONFLICT` (the printed
|
||||
`QUEUE_TRAIN_CONFLICT: train <id> stopped at #<pr> (conflicts: <files>) after
|
||||
merging #<a>,#<b>; nothing dispatched, no branch pushed; fix: …` line names
|
||||
the build without that PR) and `QUEUE_TRAIN_FREE_RED` (free tests red,
|
||||
missing, cancelled or on another SHA). In a merge, version files take the
|
||||
prefix's side and the CHANGELOG merges by section, since the stamp at merge
|
||||
rewrites both.
|
||||
|
||||
**3. Coordinator: status and verdict.**
|
||||
|
||||
```bash
|
||||
gstack-ship-queue train status [--train <nonce>]
|
||||
gstack-ship-queue train verdict --train <nonce>
|
||||
```
|
||||
|
||||
`status` never spends. When a dispatch was attempted but its run never showed
|
||||
up (`QUEUE_TRAIN_DISPATCH_UNKNOWN`), it lists the workflow's dispatch runs and
|
||||
either adopts the run carrying the nonce or marks the dispatch `absent`; only
|
||||
an absent dispatch is sent again, by `train build --train <nonce>`, under the
|
||||
same nonce. `verdict` reads the run, its record and the `full_gate_check`
|
||||
check: `GREEN` only for a complete census bound to the train nonce, head and
|
||||
tree; `QUEUE_TRAIN_PENDING` while it runs; `QUEUE_TRAIN_INCOMPLETE` for INFRA
|
||||
or INCOMPLETE panels, unbound, partial or cancelled shards, a slice killed
|
||||
before upload, or no record (never green); `QUEUE_TRAIN_RED` with the red
|
||||
cases.
|
||||
|
||||
**4. A red train: bisect, then at most one rebuild.**
|
||||
|
||||
```bash
|
||||
gstack-ship-queue train bisect --train <nonce> --spend-cap 5 --dry-run
|
||||
gstack-ship-queue train bisect --train <nonce> --spend-cap 5
|
||||
gstack-ship-queue train rebuild --train <nonce>
|
||||
```
|
||||
|
||||
The bisect dispatches the red cases (`cases`) on every prefix T0..Tk
|
||||
(`prefix_shas`), one trial each, after checking every prefix is an ancestor
|
||||
of `train/<nonce>` (`QUEUE_TRAIN_PREFIX_INVALID`). It reserves every
|
||||
case×prefix trial (unknown costs at `trial_reservation_usd`) and launches only
|
||||
the prefixes that fit the cap (`QUEUE_TRAIN_BISECT_CAPPED`; completed prefixes
|
||||
are kept and a rerun with a higher cap runs only the rest). Rerun it to read
|
||||
the results. The pattern decides: `ATTRIBUTED #<pr>` only when monotone
|
||||
(green before, red from that PR on), and `NEXT:` builds the train without it;
|
||||
red on every prefix with T0 included is `QUEUE_TRAIN_RED_ON_MAIN` (fix forward
|
||||
on main first); anything else is `QUEUE_TRAIN_UNATTRIBUTED`. An all-green
|
||||
bisect earns exactly one unchanged rebuild (`train rebuild`, reporting both
|
||||
runs); a non-monotone pattern earns none and goes to a repair lane. Every
|
||||
other rebuild is `QUEUE_TRAIN_REBUILD_REFUSED`, and a second red stays red.
|
||||
An INFRA/INCOMPLETE-only census earns the single re-dispatch
|
||||
EVAL_POLICY.infraRedispatch allows: `train rebuild --train <nonce> --infra`.
|
||||
The rebuild-once rule lives in the train module beside that constant until
|
||||
EVAL_POLICY records it. Bisect results never turn a train green.
|
||||
|
||||
**5. Coordinator: verify, merge, verify again.** For each slot in order:
|
||||
|
||||
```bash
|
||||
gstack-ship-queue verify --pr 41 --train <nonce> # MERGE_CONDITION: sha=<head>
|
||||
# merge #41 with that expected head; gstack-restamp stamps it at merge
|
||||
gstack-ship-queue verify --pr 41 --train <nonce> # VERIFY_TRAIN: #41 merged; main's tree is T1 …
|
||||
```
|
||||
|
||||
Before a merge, main's tree must equal the prefix before the PR modulo stamps
|
||||
and the PR must still be at its pin (stamps aside); after it, main's tree
|
||||
must equal the prefix after it. Anything else is `QUEUE_TRAIN_TREE_MISMATCH`
|
||||
(or `QUEUE_TRAIN_MOVED`). When nothing of the train has merged, that is a plain
|
||||
refusal: build a new train on current main. When earlier slots already
|
||||
merged, main holds a prefix no gate covered on its own, so verify freezes the
|
||||
queue: the freeze goes to local state and the notes ref first, then the full
|
||||
gate is dispatched on main's SHA at once (the backstop), and the
|
||||
`QUEUE_TRAIN_FROZEN:` line lists the merged PRs, main's tree and the backstop
|
||||
run. While frozen, ordinary trains and other trains' merges are refused
|
||||
(`QUEUE_TRAIN_FROZEN`). Admit one repair PR at its exact head:
|
||||
|
||||
```bash
|
||||
gstack-ship-queue train build --repair --prs 44@<sha>
|
||||
```
|
||||
|
||||
It runs the normal free tests, full gate and verification on main plus the
|
||||
repair. When verify confirms the repair merged, it prints `FREEZE: cleared`
|
||||
and `NEXT:` builds a train of the PRs still unmerged.
|
||||
|
||||
**6. Clean up.** `gstack-ship-queue train prune [--train <nonce>] [--dry-run]`
|
||||
deletes the `train/<nonce>` branches of merged, superseded and dropped trains
|
||||
(`WOULD-DELETE:`, `DELETED:`, `GONE:`).
|
||||
|
||||
**Escape hatches.** Remove the `train` block on the base branch to return
|
||||
every lane to the per-PR queue at once. `--urgent` ships one PR alone,
|
||||
`--no-screen` skips the paid screen, `--broad` widens it to today's selection,
|
||||
and `train build --prs <list without it>` drops a PR at any step.
|
||||
|
||||
<a id="ship"></a>
|
||||
## What `/ship` reads from it
|
||||
|
||||
|
||||
@@ -3684,6 +3684,198 @@ and the current one. Record the live attempt, or prepare a new one with
|
||||
`gate-ahead --after <pr>` and run the gate there without editing tracked
|
||||
files.
|
||||
|
||||
<a id="queue-train-off"></a>
|
||||
### `QUEUE_TRAIN_OFF`
|
||||
|
||||
**Meaning.** A merge-train command (`train …` or `verify --train`) ran
|
||||
against a base revision whose `.gstack/ship-policy.json` declares no `train`
|
||||
block, so the train surface is off and the per-PR queue applies. Nothing was
|
||||
written.
|
||||
|
||||
**Fix.** Use the per-PR queue (`ready --pr <n>`, `stamp`, `verify`), or land
|
||||
a `train` block with `min_gstack_version` on the base branch first
|
||||
([Run a merge train](ship-policy.md#run-a-merge-train)).
|
||||
|
||||
<a id="queue-train-untrusted"></a>
|
||||
### `QUEUE_TRAIN_UNTRUSTED`
|
||||
|
||||
**Meaning.** A pinned PR's head repository is not the base repository. The
|
||||
train branch and its dispatches run with repository secrets, so fork work
|
||||
never rides a train.
|
||||
|
||||
**Fix.** Run the printed `train build --prs <list without it>`; the fork PR
|
||||
stays on the trusted-branch path.
|
||||
|
||||
<a id="queue-train-moved"></a>
|
||||
### `QUEUE_TRAIN_MOVED`
|
||||
|
||||
**Meaning.** A PR's head is no longer the SHA the train pinned (at build, or
|
||||
past its pin beyond stamps at verify), so the train would not test what
|
||||
merges. At verify after earlier slots merged, the queue also freezes
|
||||
(`QUEUE_TRAIN_FROZEN`).
|
||||
|
||||
**Fix.** Ask the lane for a fresh READY on the new head, then build the train
|
||||
with the new pin (`train build --prs <n>@<new sha>,…`).
|
||||
|
||||
<a id="queue-train-conflict"></a>
|
||||
### `QUEUE_TRAIN_CONFLICT`
|
||||
|
||||
**Meaning.** Merging a PR onto the prefix before it conflicts outside the
|
||||
version files and the CHANGELOG. The `QUEUE_TRAIN_CONFLICT:` line names the
|
||||
PR, the conflicting files and the PRs merged before it. Nothing was
|
||||
dispatched and no branch was pushed.
|
||||
|
||||
**Fix.** Run the printed `train build --prs <list without that PR>`. The PR's
|
||||
lane merges main (with the earlier PRs once they land) and reports READY
|
||||
again.
|
||||
|
||||
<a id="queue-train-free-red"></a>
|
||||
### `QUEUE_TRAIN_FREE_RED`
|
||||
|
||||
**Meaning.** The free tests dispatched on `train/<nonce>` are red, cancelled,
|
||||
missing (no `workflow_dispatch` run appeared before `--free-timeout`) or ran
|
||||
on another SHA, so the paid gate was not dispatched.
|
||||
|
||||
**Fix.** Read the free-tests run. A PR-caused red drops that PR (`train build
|
||||
--prs <list without it>`); a flaky test goes to its owner before the next
|
||||
build.
|
||||
|
||||
<a id="queue-train-dispatch-unknown"></a>
|
||||
### `QUEUE_TRAIN_DISPATCH_UNKNOWN`
|
||||
|
||||
**Meaning.** A train dispatch was attempted, but no run carrying its nonce
|
||||
appeared, so whether it started is unknown (`gh workflow run` can fail after
|
||||
GitHub accepted the dispatch). It is never dispatched again blindly.
|
||||
|
||||
**Fix.** Run `gstack-ship-queue train status --train <nonce>`. It adopts the
|
||||
run carrying the nonce, or marks the dispatch absent; only then does
|
||||
`train build --train <nonce>` dispatch it again, under the same nonce.
|
||||
|
||||
<a id="queue-train-waiting"></a>
|
||||
### `QUEUE_TRAIN_WAITING`
|
||||
|
||||
**Meaning.** The train has fewer PRs than `train.max_size` and its oldest
|
||||
READY PR has waited less than `train.fill_hours`, so it does not dispatch
|
||||
yet. The message prints when it will.
|
||||
|
||||
**Fix.** Add READY PRs, rerun after the printed time, or pass `--urgent` for a
|
||||
one-PR train.
|
||||
|
||||
<a id="queue-train-pending"></a>
|
||||
### `QUEUE_TRAIN_PENDING`
|
||||
|
||||
**Meaning.** The train's run (free tests, the full gate or the bisect) has
|
||||
not finished, so there is no verdict yet.
|
||||
|
||||
**Fix.** Rerun the printed `NEXT:` command later. Nothing is dispatched again
|
||||
while a run is pending.
|
||||
|
||||
<a id="queue-train-incomplete"></a>
|
||||
### `QUEUE_TRAIN_INCOMPLETE`
|
||||
|
||||
**Meaning.** The train gate did not complete its census: INFRA or INCOMPLETE
|
||||
panels, an unbound, partial or cancelled shard, a slice killed before its
|
||||
upload, a record bound to another nonce, head or tree, or no published
|
||||
record. An incomplete gate is never green.
|
||||
|
||||
**Fix.** When every gap is machine-classified INFRA or INCOMPLETE, take the
|
||||
one re-dispatch EVAL_POLICY.infraRedispatch allows (`train rebuild --train
|
||||
<nonce> --infra`) and report both runs. Otherwise diagnose the run.
|
||||
|
||||
<a id="queue-train-red"></a>
|
||||
### `QUEUE_TRAIN_RED`
|
||||
|
||||
**Meaning.** The train's full gate is red on the train tree; the message names
|
||||
the red cases. Its PRs do not merge, and nothing is retried to change the
|
||||
verdict.
|
||||
|
||||
**Fix.** Bisect the red cases (`train bisect --train <nonce> --spend-cap
|
||||
<usd>`). After a rebuild's second red, route the cases to a repair lane.
|
||||
|
||||
<a id="queue-train-red-on-main"></a>
|
||||
### `QUEUE_TRAIN_RED_ON_MAIN`
|
||||
|
||||
**Meaning.** The bisect found the red cases red on every prefix, main (T0)
|
||||
included: main is broken, not the train's PRs.
|
||||
|
||||
**Fix.** Fix forward on main first (a one-PR train), then rebuild this train
|
||||
unchanged.
|
||||
|
||||
<a id="queue-train-unattributed"></a>
|
||||
### `QUEUE_TRAIN_UNATTRIBUTED`
|
||||
|
||||
**Meaning.** The bisect pattern blames no PR and not main: all green, or not
|
||||
monotone. One trial per prefix cannot tell a flake from a break.
|
||||
|
||||
**Fix.** All green: the one preregistered rebuild (`train rebuild --train
|
||||
<nonce>`), reporting both runs. Any other pattern: a repair lane, then a
|
||||
changed train once the repair merges.
|
||||
|
||||
<a id="queue-train-bisect-capped"></a>
|
||||
### `QUEUE_TRAIN_BISECT_CAPPED`
|
||||
|
||||
**Meaning.** The bisect's reserved price (every case×prefix trial, trials
|
||||
without cost data at `train.trial_reservation_usd`) passes `--spend-cap`.
|
||||
Prefixes that fit were launched, completed prefixes are kept, and the
|
||||
outcome stays unattributed; it is never green.
|
||||
|
||||
**Fix.** Resume with a higher cap (`train bisect --train <nonce> --spend-cap
|
||||
<usd>`). Only prefixes without a result run.
|
||||
|
||||
<a id="queue-train-prefix-invalid"></a>
|
||||
### `QUEUE_TRAIN_PREFIX_INVALID`
|
||||
|
||||
**Meaning.** A bisect prefix is not an ancestor of the train head on
|
||||
`train/<nonce>` (the branch moved), or a red case is not in the gate's plan,
|
||||
so the bisect inputs are refused before any dispatch.
|
||||
|
||||
**Fix.** Check that `train/<nonce>` still points at the recorded head
|
||||
(`train status`). A moved branch needs a new train.
|
||||
|
||||
<a id="queue-train-rebuild-refused"></a>
|
||||
### `QUEUE_TRAIN_REBUILD_REFUSED`
|
||||
|
||||
**Meaning.** `train rebuild` is not allowed: there is no bisect record (or,
|
||||
with `--infra`, no INFRA/INCOMPLETE-only census), the bisect was not all
|
||||
green, or the train was already rebuilt or re-dispatched once.
|
||||
|
||||
**Fix.** Route the red cases to a repair lane; the train is rebuilt only once
|
||||
the repair merges (a changed train).
|
||||
|
||||
<a id="queue-train-tree-mismatch"></a>
|
||||
### `QUEUE_TRAIN_TREE_MISMATCH`
|
||||
|
||||
**Meaning.** Main's tree is neither the train's prefix before this PR nor the
|
||||
one after it, modulo stamps, so the merge would not publish what the gate
|
||||
tested.
|
||||
|
||||
**Fix.** Stop merging. If earlier PRs of the train merged, the queue is now
|
||||
frozen (`QUEUE_TRAIN_FROZEN`); otherwise build a new train on current main.
|
||||
|
||||
<a id="queue-train-frozen"></a>
|
||||
### `QUEUE_TRAIN_FROZEN`
|
||||
|
||||
**Meaning.** A train stopped partway through publication, so main holds a
|
||||
prefix no gate covered on its own. The `QUEUE_TRAIN_FROZEN:` line lists the
|
||||
merged PRs, main's tree and the backstop run dispatched on main's SHA. The
|
||||
freeze lives in local state and on `refs/notes/gstack-trains`. Ordinary
|
||||
trains and other trains' merges are refused until a repair lands.
|
||||
|
||||
**Fix.** Admit one repair PR at its exact head (`train build --repair --prs
|
||||
<n>@<sha>`). The freeze clears when verify finds main's tree after that merge
|
||||
in a green record; `NEXT:` then builds a train of the PRs still unmerged.
|
||||
|
||||
<a id="ready-screen-choice-required"></a>
|
||||
### `READY_SCREEN_CHOICE_REQUIRED`
|
||||
|
||||
**Meaning.** The base policy turns trains on, so READY needs an explicit
|
||||
screen choice: `--screen` (pay for a screen on the PR's merge SHA now) or
|
||||
`--no-screen`. The `SCREEN:` lines show the selection and the recommendation.
|
||||
|
||||
**Fix.** Rerun `gstack-ship-queue ready` with the printed recommendation. A
|
||||
lane that already ran focused local cases says so in the READY block and may
|
||||
pass `--no-screen`.
|
||||
|
||||
<a id="pregate-timing-assertion"></a>
|
||||
### `PREGATE_TIMING_ASSERTION`
|
||||
|
||||
|
||||
@@ -46,6 +46,97 @@ export const RELEASE_RESULT_CODES = {
|
||||
summary: 'the gate-ahead attempt is unknown, already recorded, or its worktree no longer holds the gated commit',
|
||||
fix: 'list attempts with gstack-ship-queue gate-ahead --list, then record the live one, or prepare a new attempt',
|
||||
},
|
||||
// GSTA-49 lane T1: merge trains in gstack-ship-queue (docs/ship-policy.md#run-a-merge-train).
|
||||
QUEUE_TRAIN_OFF: {
|
||||
anchor: 'queue-train-off',
|
||||
summary: 'the base revision’s .gstack/ship-policy.json declares no `train` block, so the merge-train surface is off and the per-PR queue applies',
|
||||
fix: 'use the per-PR queue (ready, stamp, verify), or land a `train` block with min_gstack_version on the base branch first',
|
||||
},
|
||||
QUEUE_TRAIN_UNTRUSTED: {
|
||||
anchor: 'queue-train-untrusted',
|
||||
summary: 'a PR’s head repository is not the base repository; fork work never runs in a train (the train branch and its dispatch carry repository secrets)',
|
||||
fix: 'leave the PR out of the train (gstack-ship-queue train build --prs <list without it>); fork work stays on the trusted-branch path',
|
||||
},
|
||||
QUEUE_TRAIN_MOVED: {
|
||||
anchor: 'queue-train-moved',
|
||||
summary: 'a PR’s head (or the train branch) is no longer the SHA the train pinned, so the train would not test what merges',
|
||||
fix: 'ask the lane for a fresh READY on the new head, then rebuild the train with the new pin (train build --prs <n>@<new sha>,...)',
|
||||
},
|
||||
QUEUE_TRAIN_CONFLICT: {
|
||||
anchor: 'queue-train-conflict',
|
||||
summary: 'merging a PR onto the previous prefix conflicts outside the version files and the CHANGELOG; nothing was dispatched and no branch pushed',
|
||||
fix: 'run the printed train build without that PR; the PR’s lane merges main (with the earlier PRs once they land) and reports READY again',
|
||||
},
|
||||
QUEUE_TRAIN_FREE_RED: {
|
||||
anchor: 'queue-train-free-red',
|
||||
summary: 'the free tests dispatched on train/<nonce> are red, missing, cancelled or ran on another SHA, so the paid gate was not dispatched',
|
||||
fix: 'read the free-tests run; a PR-caused red drops that PR (train build --prs <list without it>), a flaky one goes to its owner before the next build',
|
||||
},
|
||||
QUEUE_TRAIN_DISPATCH_UNKNOWN: {
|
||||
anchor: 'queue-train-dispatch-unknown',
|
||||
summary: 'a train dispatch was attempted but its run could not be found by nonce, so whether it started is unknown; it is never re-dispatched blindly',
|
||||
fix: 'run gstack-ship-queue train status --train <nonce>: it finds the run by nonce, or marks the dispatch absent so train build --train <nonce> may dispatch it',
|
||||
},
|
||||
QUEUE_TRAIN_WAITING: {
|
||||
anchor: 'queue-train-waiting',
|
||||
summary: 'the train is below its size and its oldest READY PR has waited less than the policy’s fill time, so it does not dispatch yet',
|
||||
fix: 'add READY PRs, rerun once the printed time has passed, or pass --urgent for a one-PR train',
|
||||
},
|
||||
QUEUE_TRAIN_PENDING: {
|
||||
anchor: 'queue-train-pending',
|
||||
summary: 'the train’s run (free tests, the full gate or the bisect) has not finished, so there is no verdict yet',
|
||||
fix: 'rerun the printed NEXT: command later; nothing is dispatched again while a run is pending',
|
||||
},
|
||||
QUEUE_TRAIN_INCOMPLETE: {
|
||||
anchor: 'queue-train-incomplete',
|
||||
summary: 'the train gate did not complete its census (INFRA or INCOMPLETE panels, an unbound, partial or cancelled shard, a slice killed before upload, or no published record), so it is never green',
|
||||
fix: 'when every gap is machine-classified INFRA or INCOMPLETE, take the one re-dispatch EVAL_POLICY.infraRedispatch allows (train rebuild --train <nonce> --infra) and report both runs; otherwise diagnose the run',
|
||||
},
|
||||
QUEUE_TRAIN_RED: {
|
||||
anchor: 'queue-train-red',
|
||||
summary: 'the train’s full gate is red on the train tree; its PRs do not merge, and nothing is retried to change the verdict',
|
||||
fix: 'bisect the red cases (train bisect --train <nonce> --spend-cap <usd>); after a rebuild’s second red, route the cases to a repair lane',
|
||||
},
|
||||
QUEUE_TRAIN_RED_ON_MAIN: {
|
||||
anchor: 'queue-train-red-on-main',
|
||||
summary: 'the bisect found the red cases red on every prefix, main (T0) included: main is broken, not the train’s PRs',
|
||||
fix: 'fix forward on main first (a one-PR train), then rebuild this train unchanged',
|
||||
},
|
||||
QUEUE_TRAIN_UNATTRIBUTED: {
|
||||
anchor: 'queue-train-unattributed',
|
||||
summary: 'the bisect pattern blames no PR and not main (all green, or not monotone); one trial cannot tell a flake from a break',
|
||||
fix: 'all green: the one preregistered rebuild (train rebuild --train <nonce>), reporting both runs; any other pattern: a repair lane, then a changed train',
|
||||
},
|
||||
QUEUE_TRAIN_BISECT_CAPPED: {
|
||||
anchor: 'queue-train-bisect-capped',
|
||||
summary: 'the bisect’s reserved price (every case×prefix trial, unknown costs at the policy reservation) passes the spend cap; completed prefixes are kept and the outcome stays unattributed',
|
||||
fix: 'resume with a higher cap (train bisect --train <nonce> --spend-cap <usd>); only prefixes without a result run',
|
||||
},
|
||||
QUEUE_TRAIN_PREFIX_INVALID: {
|
||||
anchor: 'queue-train-prefix-invalid',
|
||||
summary: 'a bisect prefix is not an ancestor of the train head on train/<nonce>, or a red case is not in the gate’s plan, so the bisect inputs are refused',
|
||||
fix: 'check that train/<nonce> still points at the recorded head (train status); a moved branch needs a new train',
|
||||
},
|
||||
QUEUE_TRAIN_REBUILD_REFUSED: {
|
||||
anchor: 'queue-train-rebuild-refused',
|
||||
summary: 'the rebuild is not allowed: the bisect was not all green, the train was already rebuilt (or re-dispatched), or there is no bisect or INFRA record',
|
||||
fix: 'route the red cases to a repair lane; the train is rebuilt only once the repair merges (a changed train)',
|
||||
},
|
||||
QUEUE_TRAIN_TREE_MISMATCH: {
|
||||
anchor: 'queue-train-tree-mismatch',
|
||||
summary: 'main’s tree is neither the train’s prefix before this PR nor after it (modulo stamps), so the merge does not publish what the gate tested',
|
||||
fix: 'stop merging; if earlier PRs of the train merged, the queue is frozen (QUEUE_TRAIN_FROZEN); otherwise rebuild the train on current main',
|
||||
},
|
||||
QUEUE_TRAIN_FROZEN: {
|
||||
anchor: 'queue-train-frozen',
|
||||
summary: 'a train stopped partway through publication, so main holds a prefix no gate covered on its own; ordinary trains are refused until a repair lands',
|
||||
fix: 'admit one repair PR at its exact head (train build --repair --prs <n>@<sha>); the freeze clears when verify finds main’s tree after that merge in a green record',
|
||||
},
|
||||
READY_SCREEN_CHOICE_REQUIRED: {
|
||||
anchor: 'ready-screen-choice-required',
|
||||
summary: 'the base policy turns trains on, so READY needs an explicit screen choice: --screen (pay for a diff-relevant screen now) or --no-screen',
|
||||
fix: 'rerun gstack-ship-queue ready with the printed recommendation (--screen or --no-screen); say in the READY block if focused local cases already ran',
|
||||
},
|
||||
// Field list 2 W4 (pregate `timing`).
|
||||
PREGATE_TIMING_ASSERTION: {
|
||||
anchor: 'pregate-timing-assertion',
|
||||
|
||||
@@ -65,6 +65,28 @@ export interface ShipPolicy {
|
||||
full_e2e: boolean;
|
||||
/** The oldest gstack that may read this policy; a newer key set to a non-default value requires it. */
|
||||
min_gstack_version: string | null;
|
||||
/** Merge trains in gstack-ship-queue (docs/ship-policy.md#run-a-merge-train); null keeps the per-PR queue. */
|
||||
train: TrainPolicy | null;
|
||||
}
|
||||
|
||||
/** The `train` block: the gate workflow trains dispatch and how its verdict is read. */
|
||||
export interface TrainPolicy {
|
||||
/** Workflow file dispatched for the full gate and the bisect (it accepts the inputs in TRAIN_DISPATCH_INPUTS). */
|
||||
gate_workflow: string;
|
||||
/** Workflow file dispatched on train/<nonce> before the paid gate; its run must be green on the train head. */
|
||||
free_workflow: string;
|
||||
/** The check run the train's own gate run publishes on the train head when its census is complete and green. */
|
||||
full_gate_check: string;
|
||||
/** Artifact name of the train record (and the prefix of each bisect prefix's artifact). */
|
||||
record_artifact: string;
|
||||
/** Most PRs in one train. */
|
||||
max_size: number;
|
||||
/** A train under max_size dispatches once its oldest READY PR has waited this long. */
|
||||
fill_hours: number;
|
||||
/** How `ready` computes the screen's selection: gstack's paid selector, or not at all. */
|
||||
screen_selection: 'gstack-paid' | 'none';
|
||||
/** Bisect admission price of one case trial that has no recorded cost. */
|
||||
trial_reservation_usd: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_POLICY: ShipPolicy = {
|
||||
@@ -87,6 +109,7 @@ export const DEFAULT_POLICY: ShipPolicy = {
|
||||
gate_command: null,
|
||||
full_e2e: false,
|
||||
min_gstack_version: null,
|
||||
train: null,
|
||||
};
|
||||
|
||||
export type PolicyKey = Exclude<keyof ShipPolicy, '$schema'>;
|
||||
@@ -112,6 +135,7 @@ export const POLICY_EXPLANATIONS: Record<PolicyKey, string> = {
|
||||
migrations: '{ dir, id }: the migration directory and a regex with one capture group for the number in a file name; `gstack-pregate regate` reports MIGRATION_COLLISION against the base branch and every open PR.',
|
||||
gate_command: 'The repo’s full pre-merge gate command. gstack-ship-queue gate-ahead and ready print it for the caller to run (in the background or in CI); gstack never executes it. null when the repo has none.',
|
||||
full_e2e: 'true: every merge needs a passing full-gate result (gstack-ship-queue gate-ahead --record, or the lane\'s --gate-exit 0 --gate-log on the gated head) bound to a tree equal to the merging one modulo stamps; ready, stamp and verify refuse without one (QUEUE_GATE_MISSING).',
|
||||
train: '{ gate_workflow, free_workflow, full_gate_check, record_artifact, max_size, fill_hours, screen_selection, trial_reservation_usd } or null: turns on merge trains in gstack-ship-queue (train build|status|verdict|bisect|rebuild|prune, ready --screen|--no-screen, verify --train). null keeps today\'s per-PR queue unchanged.',
|
||||
min_gstack_version: 'The oldest gstack that may read this file. Any key newer than 1.91.98.0 set to a non-default value requires it at that key’s introducing version (validate prints each); a gstack older than it refuses the file (POLICY_GSTACK_TOO_OLD).',
|
||||
};
|
||||
|
||||
@@ -124,11 +148,13 @@ export const POLICY_EXPLANATIONS: Record<PolicyKey, string> = {
|
||||
*/
|
||||
export const POLICY_COMPAT_FLOOR = '1.91.98.0';
|
||||
export const SHIP_QUEUE_KEYS_VERSION = '1.91.108.0';
|
||||
export const TRAIN_KEYS_VERSION = '1.91.136.0';
|
||||
export const POLICY_KEY_INTRODUCED: Record<PolicyKey, string> = {
|
||||
bump: '1.91.73.0', queue_mode: '1.91.73.0', release_tool: '1.91.73.0', release_outputs: '1.91.73.0', stamp_paths: '1.91.73.0',
|
||||
changelog: '1.91.73.0', mirror: '1.91.73.0', history: '1.91.73.0', preregistration_shas: '1.91.73.0', stamp_sensitive_gates: '1.91.73.0',
|
||||
pr_count: '1.91.73.0', maintainers: '1.91.73.0', stamp_files: '1.91.98.0', stamp_patterns: '1.91.98.0', restamp_command: '1.91.98.0',
|
||||
migrations: '1.91.98.0', gate_command: SHIP_QUEUE_KEYS_VERSION, full_e2e: SHIP_QUEUE_KEYS_VERSION, min_gstack_version: SHIP_QUEUE_KEYS_VERSION,
|
||||
train: TRAIN_KEYS_VERSION,
|
||||
};
|
||||
|
||||
const newer = (a: string, b: string): boolean => cmpVersion(parseVersion(a)!, parseVersion(b)!) > 0;
|
||||
@@ -241,6 +267,28 @@ function migrationErrors(value: unknown, out: PolicyError[]): void {
|
||||
} catch (e: any) { out.push({ code: 'POLICY_INVALID', path: 'migrations.id', message: `not a valid regex: ${e.message}` }); }
|
||||
}
|
||||
|
||||
const TRAIN_FIELDS: Record<keyof TrainPolicy, (v: unknown) => string | null> = {
|
||||
gate_workflow: v => (typeof v === 'string' && /^[A-Za-z0-9_.-]{1,64}\.ya?ml$/.test(v) ? null : 'must be a workflow file name such as evals.yml'),
|
||||
free_workflow: v => (typeof v === 'string' && /^[A-Za-z0-9_.-]{1,64}\.ya?ml$/.test(v) ? null : 'must be a workflow file name such as free-tests.yml'),
|
||||
full_gate_check: v => (typeof v === 'string' && /^[A-Za-z0-9_./-]{1,64}$/.test(v) ? null : 'must be a check-run name ([A-Za-z0-9_./-]{1,64})'),
|
||||
record_artifact: v => (typeof v === 'string' && /^[A-Za-z0-9_.-]{1,64}$/.test(v) ? null : 'must be an artifact name ([A-Za-z0-9_.-]{1,64})'),
|
||||
max_size: v => (Number.isInteger(v) && (v as number) >= 1 && (v as number) <= 6 ? null : 'must be an integer from 1 to 6'),
|
||||
fill_hours: v => (typeof v === 'number' && v > 0 && v <= 24 ? null : 'must be a number of hours above 0 and at most 24'),
|
||||
screen_selection: v => (v === 'gstack-paid' || v === 'none' ? null : 'must be gstack-paid | none'),
|
||||
trial_reservation_usd: v => (typeof v === 'number' && v > 0 && v <= 100 ? null : 'must be dollars above 0 and at most 100'),
|
||||
};
|
||||
|
||||
function trainErrors(value: unknown, out: PolicyError[]): void {
|
||||
const t = value as Record<string, unknown> | null;
|
||||
if (!t || typeof t !== 'object' || Array.isArray(t)) { out.push({ code: 'POLICY_INVALID', path: 'train', message: `must be { ${Object.keys(TRAIN_FIELDS).join(', ')} } or null` }); return; }
|
||||
for (const k of Object.keys(t)) if (!(k in TRAIN_FIELDS)) out.push({ code: 'POLICY_INVALID', path: `train.${k}`, message: 'unknown key' });
|
||||
for (const [k, check] of Object.entries(TRAIN_FIELDS)) {
|
||||
if (!(k in t)) { out.push({ code: 'POLICY_INVALID', path: `train.${k}`, message: 'missing (every train field is required)' }); continue; }
|
||||
const why = check(t[k]);
|
||||
if (why) out.push({ code: 'POLICY_INVALID', path: `train.${k}`, message: `${JSON.stringify(t[k])} ${why}` });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Schema and containment check of a parsed policy object. Unknown keys are
|
||||
* errors (a typo must not silently mean "default", and a key from a newer
|
||||
@@ -282,6 +330,7 @@ export function validatePolicy(raw: unknown, opts: { gstackVersion?: string } =
|
||||
else if (t.length > MAX_COMMAND || /[\0\r\n]/.test(t)) errors.push({ code: 'POLICY_CONTAINMENT', path: 'gate_command', message: `must be one line under ${MAX_COMMAND} characters` });
|
||||
}
|
||||
if ('full_e2e' in obj && typeof obj.full_e2e !== 'boolean') errors.push({ code: 'POLICY_INVALID', path: 'full_e2e', message: 'must be true or false' });
|
||||
if ('train' in obj && obj.train !== null) trainErrors(obj.train, errors);
|
||||
if ('stamp_paths' in obj) listErrors('stamp_paths', obj.stamp_paths, errors, globError);
|
||||
for (const key of ['changelog', 'mirror'] as const) {
|
||||
if (!(key in obj) || obj[key] === null) continue;
|
||||
|
||||
@@ -129,7 +129,7 @@ export function applyStampPatch(repoRoot: string, mergeTree: string, patch: stri
|
||||
});
|
||||
}
|
||||
|
||||
function withIndex<T>(repoRoot: string, tree: string, fn: (env: NodeJS.ProcessEnv) => T): T | { error: string } {
|
||||
export function withIndex<T>(repoRoot: string, tree: string, fn: (env: NodeJS.ProcessEnv) => T): T | { error: string } {
|
||||
const gitDir = git(repoRoot, ['rev-parse', '--absolute-git-dir']).out;
|
||||
const indexFile = path.join(gitDir, `gstack-ship-queue-${process.pid}-${randomBytes(4).toString('hex')}.index`);
|
||||
const env = { ...process.env, GIT_INDEX_FILE: indexFile };
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
/**
|
||||
* Argument parsing and output for the merge-train surface of
|
||||
* bin/gstack-ship-queue: `train build|status|verdict|bisect|rebuild|prune`,
|
||||
* `ready --screen|--no-screen [--broad]` and `verify --train`. Every command
|
||||
* prints its lines, the coded error line on stderr, then `NEXT: <command>`
|
||||
* and the canonical `GSTACK_RESULT: skill=ship-queue status=<s> run=<record>`.
|
||||
* Loaded by the bin only when one of those paths runs.
|
||||
*/
|
||||
import { EXIT, formatResult } from './headless-artifacts';
|
||||
import { RESULT_CODES } from './result-codes';
|
||||
import type { QueueContext } from './ship-queue';
|
||||
import { buildTrain, pruneTrains, trainStatus, type TrainOut } from './ship-queue-train';
|
||||
import { trainReady } from './ship-queue-train-ready';
|
||||
import { trainBisect, trainRebuild, trainVerdict } from './ship-queue-train-verdict';
|
||||
import { verifyTrain } from './ship-queue-train-verify';
|
||||
import { redactCause } from './ship-queue-train-record';
|
||||
|
||||
export const TRAIN_HELP = ` train build --prs <n>@<sha>[,...] [--urgent | --repair] [--dry-run] [--free-timeout <s>]
|
||||
train build --train <nonce> resume a train (push, free tests, dispatch) from its record
|
||||
merge pinned unstamped heads in slot order onto main, free tests on
|
||||
train/<nonce>, then one full gate through eval-bg's nonce
|
||||
train status [--train <nonce>] never spends: rebuild records from refs/notes/gstack-trains, reconcile
|
||||
unconfirmed dispatches by nonce
|
||||
train verdict --train <nonce> read the gate run and its record: GREEN, RED, INCOMPLETE or PENDING
|
||||
train bisect --train <nonce> --spend-cap <usd> [--dry-run]
|
||||
red cases on every prefix T0..Tk, resumable and capped
|
||||
train rebuild --train <nonce> [--infra] the one rebuild an all-green bisect earns (or the one INFRA re-dispatch)
|
||||
train prune [--train <nonce>] [--dry-run]
|
||||
delete train/<nonce> branches of merged, superseded or dropped trains
|
||||
ready --pr <n> --screen [--broad] | --no-screen [--dry-run]
|
||||
(train policy) READY plus the screen choice, recorded as a receipt
|
||||
verify --pr <n> --train <nonce> [--no-fetch]
|
||||
(train policy) the merge check against a green train record
|
||||
Merge trains need a "train" block in the base revision's ship policy (docs/ship-policy.md#run-a-merge-train).`;
|
||||
|
||||
function flag(args: string[], name: string): string | undefined {
|
||||
const i = args.indexOf(name);
|
||||
return i >= 0 ? args[i + 1] : undefined;
|
||||
}
|
||||
|
||||
export function printTrain(cmd: string, out: TrainOut): number {
|
||||
for (const line of out.lines) process.stdout.write(line + '\n');
|
||||
if (out.failure) {
|
||||
const f = out.failure;
|
||||
process.stderr.write(`gstack-ship-queue ${cmd}: ${f.message}; fix: ${f.fix ?? RESULT_CODES[f.code].fix} (${f.code})\n`);
|
||||
}
|
||||
process.stdout.write(`NEXT: ${out.next}\n`);
|
||||
process.stdout.write(formatResult('ship-queue', out.status, out.run) + '\n');
|
||||
return out.exit;
|
||||
}
|
||||
|
||||
function usage(message: string): number {
|
||||
process.stderr.write(`gstack-ship-queue: ${message}\n${TRAIN_HELP}\n`);
|
||||
return EXIT.usage;
|
||||
}
|
||||
|
||||
async function guarded(cmd: string, fn: () => Promise<TrainOut> | TrainOut): Promise<number> {
|
||||
try {
|
||||
return printTrain(cmd, await fn());
|
||||
} catch (error) {
|
||||
const cause = redactCause(error instanceof Error ? error.message : String(error));
|
||||
process.stderr.write(`gstack-ship-queue ${cmd}: unexpected failure: ${cause}\n`);
|
||||
process.stdout.write(`NEXT: gstack-ship-queue train status\n`);
|
||||
process.stdout.write(formatResult('ship-queue', 'interrupted', 'none') + '\n');
|
||||
return EXIT.fail;
|
||||
}
|
||||
}
|
||||
|
||||
const positive = (s: string | undefined, fallback: number) => (s !== undefined && /^\d+(\.\d+)?$/.test(s) && Number(s) > 0 ? Number(s) : s === undefined ? fallback : NaN);
|
||||
|
||||
export async function trainMain(ctx: QueueContext, args: string[]): Promise<number> {
|
||||
const [sub, ...rest] = args;
|
||||
const nonce = flag(rest, '--train');
|
||||
const dryRun = rest.includes('--dry-run');
|
||||
if (sub === 'build') {
|
||||
if (!nonce && !flag(rest, '--prs')) return usage('train build needs --prs <n>@<sha>[,...] or --train <nonce>');
|
||||
const freeTimeoutSec = positive(flag(rest, '--free-timeout'), 2700);
|
||||
if (Number.isNaN(freeTimeoutSec)) return usage('--free-timeout needs seconds');
|
||||
if (rest.includes('--urgent') && rest.includes('--repair')) return usage('--urgent and --repair are separate one-PR trains; pass one');
|
||||
return guarded('train build', () => buildTrain(ctx, {
|
||||
pins: flag(rest, '--prs'), resume: nonce, dryRun, urgent: rest.includes('--urgent'), repair: rest.includes('--repair'), freeTimeoutSec,
|
||||
pollMs: Number(process.env.GSTACK_SHIP_QUEUE_POLL_MS) || 30_000,
|
||||
}));
|
||||
}
|
||||
if (sub === 'status') return guarded('train status', () => trainStatus(ctx, { train: nonce }));
|
||||
if (sub === 'prune') return guarded('train prune', () => pruneTrains(ctx, { dryRun, train: nonce }));
|
||||
if (!nonce) return usage(`train ${sub ?? '<subcommand>'} needs --train <nonce>`);
|
||||
if (sub === 'verdict') return guarded('train verdict', () => trainVerdict(ctx, { train: nonce }));
|
||||
if (sub === 'rebuild') return guarded('train rebuild', () => trainRebuild(ctx, { train: nonce, infra: rest.includes('--infra') }));
|
||||
if (sub === 'bisect') {
|
||||
const cap = positive(flag(rest, '--spend-cap'), NaN);
|
||||
if (Number.isNaN(cap)) return usage('train bisect needs --spend-cap <usd>');
|
||||
return guarded('train bisect', () => trainBisect(ctx, { train: nonce, spendCap: cap, dryRun }));
|
||||
}
|
||||
return usage(`unknown train subcommand ${sub}`);
|
||||
}
|
||||
|
||||
export async function readyMain(ctx: QueueContext, args: string[], base: { pr: string; pregate?: string; gateExit?: number; gateLog?: string }): Promise<number> {
|
||||
const screen = args.includes('--screen');
|
||||
const noScreen = args.includes('--no-screen');
|
||||
if (screen && noScreen) return usage('pass one of --screen or --no-screen');
|
||||
if (args.includes('--broad') && !screen) return usage('--broad selects the screen policy; it needs --screen');
|
||||
return guarded('ready', () => trainReady(ctx, { ...base, screen: screen ? 'screen' : noScreen ? 'no-screen' : undefined, broad: args.includes('--broad'), dryRun: args.includes('--dry-run') }));
|
||||
}
|
||||
|
||||
export async function verifyMain(ctx: QueueContext, args: string[], pr: string): Promise<number> {
|
||||
return guarded('verify', () => verifyTrain(ctx, { pr, train: flag(args, '--train')!, noFetch: args.includes('--no-fetch') }));
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
/**
|
||||
* GitHub reads and writes for merge trains, all through the queue's `gh`
|
||||
* seam (QueueContext.gh) so tests answer them from a fixture: PR identity,
|
||||
* workflow runs found by branch or by eval-bg nonce, run artifacts and the
|
||||
* full-gate check run. Dispatch itself goes through scripts/eval-bg.ts.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import type { Exec, RunSummary } from '../scripts/eval-bg';
|
||||
import type { QueueContext } from './ship-queue';
|
||||
|
||||
export interface PrInfo { headRefOid: string; isCrossRepository: boolean; state: string; headRefName: string }
|
||||
export interface RunInfo { databaseId: number; status: string; conclusion: string | null; url: string; headSha: string; event: string; displayTitle: string }
|
||||
export interface CheckInfo { status: string; conclusion: string | null }
|
||||
|
||||
function json<T>(r: { ok: boolean; out: string }): T | null {
|
||||
if (!r.ok) return null;
|
||||
try { return JSON.parse(r.out) as T; } catch { return null; }
|
||||
}
|
||||
|
||||
/** eval-bg's process seam bound to the queue's gh (only `gh` is ever routed through it). */
|
||||
export function execFor(ctx: QueueContext): Exec {
|
||||
return (command, args) => {
|
||||
if (command !== 'gh') return { status: null, stdout: '' };
|
||||
const r = ctx.gh(args);
|
||||
return { status: r.ok ? 0 : 1, stdout: r.out };
|
||||
};
|
||||
}
|
||||
|
||||
export function repoSlug(ctx: QueueContext): string | null {
|
||||
const r = json<{ nameWithOwner?: string }>(ctx.gh(['repo', 'view', '--json', 'nameWithOwner']));
|
||||
return r?.nameWithOwner && /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(r.nameWithOwner) ? r.nameWithOwner : null;
|
||||
}
|
||||
|
||||
export function prInfo(ctx: QueueContext, pr: string): PrInfo | { error: string } {
|
||||
const r = ctx.gh(['pr', 'view', pr, '--json', 'headRefOid,isCrossRepository,state,headRefName']);
|
||||
const p = json<Partial<PrInfo>>(r);
|
||||
if (!p?.headRefOid) return { error: `cannot read PR #${pr} through gh (${(r.err ?? r.out).trim().slice(0, 120) || 'no headRefOid'})` };
|
||||
return { headRefOid: p.headRefOid, isCrossRepository: p.isCrossRepository !== false, state: p.state ?? 'OPEN', headRefName: p.headRefName ?? '' };
|
||||
}
|
||||
|
||||
const RUN_FIELDS = 'databaseId,status,conclusion,url,headSha,event,displayTitle';
|
||||
|
||||
/** Newest workflow_dispatch run of a workflow on one branch; null when none; 'unreadable' when gh failed. */
|
||||
export function newestBranchRun(ctx: QueueContext, slug: string, workflow: string, branch: string): RunInfo | null | 'unreadable' {
|
||||
const runs = json<RunInfo[]>(ctx.gh(['run', 'list', '--repo', slug, '--workflow', workflow, '--branch', branch, '--event', 'workflow_dispatch', '--limit', '20', '--json', RUN_FIELDS]));
|
||||
if (!Array.isArray(runs)) return 'unreadable';
|
||||
return runs.filter(r => r.event === 'workflow_dispatch').sort((a, b) => b.databaseId - a.databaseId)[0] ?? null;
|
||||
}
|
||||
|
||||
export function runView(ctx: QueueContext, slug: string, id: number): RunInfo | null {
|
||||
return json<RunInfo>(ctx.gh(['run', 'view', String(id), '--repo', slug, '--json', RUN_FIELDS]));
|
||||
}
|
||||
|
||||
export function asRun(summary: RunSummary): { id: number; url: string } {
|
||||
return { id: summary.databaseId, url: summary.url };
|
||||
}
|
||||
|
||||
/** One JSON artifact of a run (the first .json file it holds); null when it is missing or unreadable. */
|
||||
export function downloadJson(ctx: QueueContext, slug: string, run: number, name: string, scratch: string): unknown {
|
||||
const dir = path.join(scratch, `${run}-${name}`);
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
try {
|
||||
if (!ctx.gh(['run', 'download', String(run), '--repo', slug, '-n', name, '-D', dir]).ok) return null;
|
||||
const file = fs.readdirSync(dir).find(f => f.endsWith('.json'));
|
||||
return file ? JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8')) : null;
|
||||
} catch { return null; } finally { fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
}
|
||||
|
||||
/** The named check run on a commit (the newest when several); null when absent or unreadable. */
|
||||
export function checkRun(ctx: QueueContext, sha: string, name: string): CheckInfo | null {
|
||||
const r = json<{ check_runs?: Array<{ name?: string; status?: string; conclusion?: string | null }> }>(
|
||||
ctx.gh(['api', `repos/{owner}/{repo}/commits/${sha}/check-runs?check_name=${encodeURIComponent(name)}`]));
|
||||
const hit = r?.check_runs?.find(c => c.name === name);
|
||||
return hit ? { status: hit.status ?? 'unknown', conclusion: hit.conclusion ?? null } : null;
|
||||
}
|
||||
|
||||
/** Whether a branch exists on the remote (ls-remote, never a fetch). */
|
||||
export function remoteBranchSha(git: (args: string[]) => { ok: boolean; out: string }, remote: string, branch: string): string | null {
|
||||
const r = git(['ls-remote', remote, `refs/heads/${branch}`]);
|
||||
return r.ok ? (r.out.split(/\s/)[0] || null) : null;
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
/**
|
||||
* `ready --pr <n> --screen | --no-screen` under a ship policy with a `train`
|
||||
* block (docs/ship-policy.md#run-a-merge-train; plan GSTA-49 T1). The
|
||||
* READY block is today's (lib/ship-queue.ts ready()); this adds the screen
|
||||
* selection and Garry's default recommendation (screen when the screen
|
||||
* selects a skill's own cases), requires an explicit choice
|
||||
* (READY_SCREEN_CHOICE_REQUIRED), dispatches a chosen screen through
|
||||
* eval-bg's nonce with its purpose and policy in the key, and records the
|
||||
* choice as a READY receipt (local state and refs/notes/gstack-trains) that
|
||||
* `train build` requires. The screen's dollar price is left to the T0 cost
|
||||
* replay (lib/gate-cost.ts) and printed as pending until it lands.
|
||||
*/
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import { DISPATCH_INPUTS, awaitNonceRun, dedupeKey, dispatchWorkflow, findDuplicateRun, listDispatchRuns, makeNonce, type WorkflowDispatch } from '../scripts/eval-bg';
|
||||
import { EXIT } from './headless-artifacts';
|
||||
import { ready, type QueueContext } from './ship-queue';
|
||||
import { git } from './ship-queue-gate';
|
||||
import { execFor, repoSlug } from './ship-queue-train-github';
|
||||
import { loadReady, nowIso, persist, readyPath, type ReadyReceipt, type ScreenChoice } from './ship-queue-train-record';
|
||||
import { outOk, outPending, outRefused, trainEnv, type TrainEnv, type TrainOut } from './ship-queue-train';
|
||||
|
||||
const Q = 'gstack-ship-queue';
|
||||
|
||||
export interface ReadyOptions { pr: string; pregate?: string; gateExit?: number; gateLog?: string; screen?: 'screen' | 'no-screen'; broad: boolean; dryRun: boolean }
|
||||
export interface ScreenSelection { lines: string[]; recommend: 'screen' | 'no-screen' | null }
|
||||
|
||||
/** Skill directories a diff touches: top-level directories that hold a SKILL.md.tmpl at the head. */
|
||||
export function touchedSkills(changed: readonly string[], hasTemplate: (dir: string) => boolean): string[] {
|
||||
return [...new Set(changed.filter(f => f.includes('/')).map(f => f.split('/')[0]!))].filter(hasTemplate).sort();
|
||||
}
|
||||
|
||||
/** The selected cases that are a touched skill's own: their touchfiles name a path inside that skill's directory. */
|
||||
export function skillOwnCases(selected: readonly string[], touchfiles: Record<string, string[]>, skills: readonly string[]): string[] {
|
||||
return selected.filter(id => (touchfiles[id] ?? []).some(glob => skills.some(s => glob.startsWith(`${s}/`)))).sort();
|
||||
}
|
||||
|
||||
/** gstack's paid selector over a diff (today's PR selection; T2 narrows it), loaded only when the policy asks for it. */
|
||||
export async function gstackPaidSelection(repoRoot: string, baseSha: string, changed: string[], hasTemplate: (dir: string) => boolean): Promise<ScreenSelection> {
|
||||
const { computePaidCaseSelection } = await import('../scripts/lib/paid-select');
|
||||
const { E2E_TOUCHFILES, LLM_JUDGE_TOUCHFILES } = await import('../test/helpers/touchfiles');
|
||||
const skills = touchedSkills(changed, hasTemplate);
|
||||
let e2e: string[]; let judges: number; let note = "today's PR selection; T2's narrowed screen has not landed";
|
||||
try {
|
||||
const s = computePaidCaseSelection({ profile: 'pr', rootDir: repoRoot, changedFiles: changed, env: { ...process.env, EVALS_BASE: baseSha, EVALS_ALL: '' } }).selection;
|
||||
e2e = s.e2e ?? Object.keys(E2E_TOUCHFILES);
|
||||
judges = s.judges ? s.judges.length : Object.keys(LLM_JUDGE_TOUCHFILES).length;
|
||||
} catch (error) {
|
||||
e2e = Object.keys(E2E_TOUCHFILES);
|
||||
judges = Object.keys(LLM_JUDGE_TOUCHFILES).length;
|
||||
note = `the PR profile needs full validation (${error instanceof Error ? error.message.slice(0, 160) : String(error)}): the screen is the full gate selection`;
|
||||
}
|
||||
const own = skillOwnCases(e2e, E2E_TOUCHFILES, skills);
|
||||
const recommend = own.length ? 'screen' : 'no-screen';
|
||||
return {
|
||||
recommend,
|
||||
lines: [
|
||||
`SCREEN: selection ${e2e.length} e2e case(s) + ${judges} judge(s) (${note})`,
|
||||
`SCREEN: skill's own cases ${own.length ? own.slice(0, 12).join(',') + (own.length > 12 ? ` (+${own.length - 12})` : '') : 'none'} → recommend --${recommend}`,
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
async function screenSelection(env: TrainEnv, head: string, baseSha: string): Promise<ScreenSelection> {
|
||||
const price = 'SCREEN: price pending the T0 cost replay (lib/gate-cost.ts): cases without cost data are unknown; a red train holds its PRs about an hour';
|
||||
if (env.train.screen_selection === 'none') return { recommend: null, lines: ['SCREEN: selection not computed (train.screen_selection is none); no recommendation, the lane decides', price] };
|
||||
const changed = git(env.ctx.repoRoot, ['diff', '--name-only', `${baseSha}...${head}`]).out.split('\n').filter(Boolean);
|
||||
const hasTemplate = (dir: string) => git(env.ctx.repoRoot, ['cat-file', '-e', `${head}:${dir}/SKILL.md.tmpl`]).ok;
|
||||
const s = await gstackPaidSelection(env.ctx.repoRoot, baseSha, changed, hasTemplate);
|
||||
return { ...s, lines: [...s.lines, price] };
|
||||
}
|
||||
|
||||
/** Dispatch (or adopt) the PR's screen on the base branch: inputs name the PR, its merge SHA and the screen policy. */
|
||||
async function dispatchScreen(env: TrainEnv, o: ReadyOptions, mergeSha: string, baseSha: string): Promise<{ choice: ScreenChoice; line: string } | TrainOut> {
|
||||
const policy = o.broad ? 'broad' : 'narrow';
|
||||
const scope = { purpose: 'screen' as const, policy: policy as 'narrow' | 'broad' };
|
||||
const nonce = makeNonce('pr', mergeSha, baseSha, randomBytes(3).toString('hex'), scope);
|
||||
const dispatch: WorkflowDispatch = { workflow: env.train.gate_workflow, inputs: {
|
||||
[DISPATCH_INPUTS.evalsAll]: 'false', [DISPATCH_INPUTS.expectedSha]: mergeSha, [DISPATCH_INPUTS.nonce]: nonce, [DISPATCH_INPUTS.baseRef]: env.ctx.base,
|
||||
[DISPATCH_INPUTS.baseSha]: baseSha, [DISPATCH_INPUTS.screenPr]: o.pr, [DISPATCH_INPUTS.screenMergeSha]: mergeSha, [DISPATCH_INPUTS.screenPolicy]: policy,
|
||||
} };
|
||||
const inputs = Object.entries(dispatch.inputs).map(([k, v]) => `${k}=${v}`).join(' ');
|
||||
if (o.dryRun) return { choice: { choice: 'screen', policy, nonce, run: null, merge_sha: mergeSha }, line: `SCREEN_DISPATCH: ${dispatch.workflow} --ref ${env.ctx.base} ${inputs} (dry run: not dispatched)` };
|
||||
const slug = repoSlug(env.ctx);
|
||||
if (!slug) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: 'gh repo view could not name the repository' }, [], `${Q} ready --pr ${o.pr} --screen`, 'none');
|
||||
const exec = execFor(env.ctx);
|
||||
const existing = findDuplicateRun(listDispatchRuns(dispatch.workflow, exec, slug) ?? [], dedupeKey('pr', mergeSha, baseSha, scope));
|
||||
if (existing) return { choice: { choice: 'screen', policy, nonce: /eval-bg (\S+?)\]/.exec(existing.displayTitle)?.[1] ?? nonce, run: existing.databaseId, merge_sha: mergeSha }, line: `SCREEN_DISPATCH: following run ${existing.databaseId} (same merge SHA, base and policy; no duplicate)` };
|
||||
const found = dispatchWorkflow(dispatch, env.ctx.base, exec, slug) ? await awaitNonceRun(dispatch.workflow, nonce, exec, slug) : null;
|
||||
if (!found) return outPending({ code: 'QUEUE_TRAIN_DISPATCH_UNKNOWN', message: `no ${dispatch.workflow} run named ${nonce} appeared after the screen dispatch` }, [], `${Q} ready --pr ${o.pr} --screen${o.broad ? ' --broad' : ''} (adopts the run by its key once it appears)`, 'none');
|
||||
return { choice: { choice: 'screen', policy, nonce, run: found.databaseId, merge_sha: mergeSha }, line: `SCREEN_DISPATCH: ${dispatch.workflow} run ${found.databaseId} ${found.url} nonce ${nonce}` };
|
||||
}
|
||||
|
||||
export async function trainReady(ctx: QueueContext, o: ReadyOptions): Promise<TrainOut> {
|
||||
const env = trainEnv(ctx);
|
||||
if ('code' in env) return outRefused(env, [], 'none', 'none');
|
||||
const base = ready(ctx, o);
|
||||
if (!base.ok) {
|
||||
const out = outRefused(base.failure!, base.lines, `fix the refusal, then ${Q} ready --pr ${o.pr} --screen|--no-screen`, base.run);
|
||||
return ['QUEUE_VERIFY_MISMATCH', 'QUEUE_STAMP_NOT_STAMP_ONLY'].includes(base.failure!.code) ? { ...out, exit: EXIT.fail } : out;
|
||||
}
|
||||
const head = git(ctx.repoRoot, ['rev-parse', 'HEAD']).out;
|
||||
const baseSha = git(ctx.repoRoot, ['rev-parse', `${env.remote}/${ctx.base}`]).out;
|
||||
const sel = await screenSelection(env, head, baseSha);
|
||||
const lines = [...base.lines, ...sel.lines];
|
||||
const rec = sel.recommend ?? 'no-screen';
|
||||
if (!o.screen) {
|
||||
return outRefused({ code: 'READY_SCREEN_CHOICE_REQUIRED', message: `the base policy runs merge trains; choose --screen or --no-screen (recommended: --${rec})` }, lines,
|
||||
`${Q} ready --pr ${o.pr} --${rec}`, 'none');
|
||||
}
|
||||
let choice: ScreenChoice = { choice: 'no-screen', policy: null, nonce: null, run: null, merge_sha: null };
|
||||
if (o.screen === 'screen') {
|
||||
const ls = git(ctx.repoRoot, ['ls-remote', env.remote, `refs/pull/${o.pr}/merge`]);
|
||||
const mergeSha = ls.ok ? ls.out.split(/\s/)[0] ?? '' : '';
|
||||
if (!/^[0-9a-f]{40,64}$/.test(mergeSha)) return outRefused({ code: 'QUEUE_NOT_READY', message: `${env.remote} has no refs/pull/${o.pr}/merge (the PR conflicts with ${ctx.base}, or GitHub has not computed it yet)` }, lines, `merge ${ctx.base} into the branch, push, then ${Q} ready --pr ${o.pr} --screen`, 'none');
|
||||
const d = await dispatchScreen(env, o, mergeSha, baseSha);
|
||||
if ('exit' in d) return { ...d, lines: [...lines, ...d.lines] };
|
||||
choice = d.choice;
|
||||
lines.push(d.line);
|
||||
}
|
||||
lines.splice(1, 0, `screen: ${choice.choice}${choice.policy ? ` policy=${choice.policy} nonce=${choice.nonce} run=${choice.run ?? '-'}` : ''}`);
|
||||
const now = nowIso();
|
||||
const readyAt = loadReady(env.repo.key, o.pr, head)?.ready_at ?? now;
|
||||
const receipt: ReadyReceipt = { schema: 1, kind: 'ready', repo_key: env.repo.key, pr: o.pr, head, base_sha: baseSha, ready_at: readyAt, screen: choice, updated_at: now };
|
||||
if (o.dryRun) return outOk([...lines, 'DRY-RUN: no READY receipt written, nothing dispatched'], `${Q} ready --pr ${o.pr} --${o.screen}${o.broad ? ' --broad' : ''}`, 'none');
|
||||
const p = persist(ctx.repoRoot, [receipt], { remote: env.remote });
|
||||
lines.push(`READY_RECEIPT: ${readyPath(env.repo.key, o.pr, head)} (${p.detail})`);
|
||||
return outOk(lines, `send the READY block; the coordinator builds the train: ${Q} train build --prs ${o.pr}@${head},...`, readyPath(env.repo.key, o.pr, head));
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
/**
|
||||
* Merge-train records (docs/ship-policy.md#run-a-merge-train): versioned
|
||||
* `train.json`, one attempt record per dispatch, the bisect ledger, READY
|
||||
* receipts and the queue freeze, under `$GSTACK_STATE_ROOT/ship-queue/`
|
||||
* (lib/state-root.ts resolves the root). Every record is mirrored to origin
|
||||
* as a git note under `refs/notes/gstack-trains`, each on a blob named for
|
||||
* the record, so a lost machine rebuilds its state with `train status`. A
|
||||
* record is written locally before its note, and the note before any remote
|
||||
* effect that depends on it.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { redactFindingSpans } from './redact-engine';
|
||||
import { git, sha256, writeJsonAtomic } from './ship-queue-gate';
|
||||
import { resolveStateRoot } from './state-root';
|
||||
|
||||
export const TRAIN_SCHEMA = 1;
|
||||
export const NOTES_REF = 'refs/notes/gstack-trains';
|
||||
const NOTES_REMOTE_REF = 'refs/notes/gstack-trains-origin';
|
||||
|
||||
export type TrainState =
|
||||
| 'built' | 'pushed' | 'free-pending' | 'free-red' | 'free-green' | 'dispatch-unknown' | 'dispatch-absent' | 'gate-pending'
|
||||
| 'green' | 'red' | 'incomplete' | 'attributed' | 'red-on-main' | 'unattributed' | 'merging' | 'merged' | 'superseded';
|
||||
/** States after which a train never dispatches again; prune deletes their branches. */
|
||||
export const TERMINAL_STATES: readonly TrainState[] = ['free-red', 'attributed', 'red-on-main', 'merged', 'superseded'];
|
||||
|
||||
export interface ScreenChoice { choice: 'screen' | 'no-screen'; policy: 'narrow' | 'broad' | null; nonce: string | null; run: number | null; merge_sha: string | null }
|
||||
export interface TrainSlot { pr: string; head: string; ready_at: string; screen: ScreenChoice }
|
||||
export interface TrainPrefix { index: number; commit: string; tree: string; pr: string | null }
|
||||
export interface FreeRun { run: number | null; url: string | null; head_sha: string | null; conclusion: string | null; verdict: 'green' | 'red' | 'missing' | 'stale' | 'cancelled' | 'pending' }
|
||||
export interface TrainRecord {
|
||||
schema: 1; kind: 'train'; nonce: string; key: string; repo_key: string; repo: string; base: { ref: string; sha: string }; policy: string;
|
||||
slots: TrainSlot[]; held: string[]; prefixes: TrainPrefix[]; head: string; tree: string; branch: string;
|
||||
urgent: boolean; repair: boolean; state: TrainState; free: FreeRun | null; attempts: string[];
|
||||
merged: Array<{ pr: string; sha: string; tree: string; at: string }>; superseded_by: string | null;
|
||||
effects: Array<{ at: string; what: string }>; error: string | null; created_at: string; updated_at: string;
|
||||
}
|
||||
export type AttemptPurpose = 'gate' | 'rebuild' | 'infra-redispatch' | 'bisect';
|
||||
export type GateOutcome = 'PENDING' | 'GREEN' | 'RED' | 'INCOMPLETE';
|
||||
export interface CasePanel { case: string; kind: string; status: string; red_class: string | null; cost_usd: number | null }
|
||||
export interface AttemptRecord {
|
||||
schema: 1; kind: 'train-attempt'; train: string; id: string; purpose: AttemptPurpose; nonce: string; workflow: string; ref: string;
|
||||
inputs: Record<string, string>; state: 'dispatching' | 'unknown' | 'absent' | 'dispatched' | 'completed'; run: { id: number; url: string } | null;
|
||||
outcome: GateOutcome | null; detail: string | null; red_cases: string[]; panels: CasePanel[]; infra_eligible: boolean;
|
||||
cost_usd: number | null; prefixes: number[]; created_at: string; updated_at: string;
|
||||
}
|
||||
export type PrefixResult = 'red' | 'green';
|
||||
export interface BisectRecord {
|
||||
schema: 1; kind: 'train-bisect'; train: string; cases: string[]; spend_cap: number; spent_usd: number;
|
||||
prefixes: Array<{ index: number; sha: string; results: Record<string, PrefixResult>; attempts: string[] }>;
|
||||
outcome: null | { kind: 'attributed' | 'red-on-main' | 'unattributed'; all_green: boolean; prs: string[]; patterns: Record<string, string>; at: string };
|
||||
updated_at: string;
|
||||
}
|
||||
export interface FreezeRecord {
|
||||
schema: 1; kind: 'freeze'; repo_key: string; train: string; merged: string[]; unmerged: Array<{ pr: string; head: string }>;
|
||||
main: { sha: string; tree: string }; reason: string; backstop: { nonce: string; run: number | null; state: string } | null;
|
||||
repair: { pr: string; head: string; train: string | null } | null; created_at: string; cleared_at: string | null; cleared_by: string | null; updated_at: string;
|
||||
}
|
||||
export interface ReadyReceipt {
|
||||
schema: 1; kind: 'ready'; repo_key: string; pr: string; head: string; base_sha: string; ready_at: string; screen: ScreenChoice; updated_at: string;
|
||||
}
|
||||
export type AnyRecord = TrainRecord | AttemptRecord | BisectRecord | FreezeRecord | ReadyReceipt;
|
||||
|
||||
/** Clock seam for fill-time and record tests (`GSTACK_SHIP_QUEUE_NOW`, ISO-8601). */
|
||||
export function nowIso(): string {
|
||||
const fixed = process.env.GSTACK_SHIP_QUEUE_NOW;
|
||||
return fixed && !Number.isNaN(Date.parse(fixed)) ? new Date(fixed).toISOString() : new Date().toISOString();
|
||||
}
|
||||
|
||||
/** An unexpected failure's cause, with secrets withheld, for the record and the error line. */
|
||||
export function redactCause(message: string): string {
|
||||
const text = message.replace(/\s+/g, ' ').slice(0, 400);
|
||||
return redactFindingSpans(text) ?? '(cause withheld: it could not be redacted safely)';
|
||||
}
|
||||
|
||||
export function queueRoot(env: NodeJS.ProcessEnv = process.env): string {
|
||||
return path.join(resolveStateRoot(env), 'ship-queue');
|
||||
}
|
||||
export const trainDir = (nonce: string) => path.join(queueRoot(), 'trains', nonce);
|
||||
export const trainFile = (nonce: string) => path.join(trainDir(nonce), 'train.json');
|
||||
const attemptPath = (nonce: string, id: string) => path.join(trainDir(nonce), 'attempts', `${id}.json`);
|
||||
const bisectPath = (nonce: string) => path.join(trainDir(nonce), 'bisect.json');
|
||||
const freezePath = (repoKey: string) => path.join(queueRoot(), 'freeze', `${repoKey}.json`);
|
||||
export const readyPath = (repoKey: string, pr: string, head: string) => path.join(queueRoot(), 'ready', repoKey, `${pr}-${head}.json`);
|
||||
|
||||
/** The repository's identity: origin's URL, keyed so records of different repositories never mix. */
|
||||
export function repoIdentity(repoRoot: string): { key: string; url: string } {
|
||||
const url = git(repoRoot, ['config', '--get', 'remote.origin.url']).out.trim() || repoRoot;
|
||||
const norm = url.replace(/\.git$/, '').replace(/^git@github\.com:/, 'https://github.com/').replace(/\/+$/, '');
|
||||
return { key: sha256(norm).slice(0, 16), url: norm };
|
||||
}
|
||||
|
||||
function readJson<T>(file: string, kind: string): T | null {
|
||||
try { const r = JSON.parse(fs.readFileSync(file, 'utf8')); return r?.schema === TRAIN_SCHEMA && r.kind === kind ? r as T : null; } catch { return null; }
|
||||
}
|
||||
|
||||
export const loadTrain = (nonce: string) => (/^[A-Za-z0-9._-]{1,128}$/.test(nonce) ? readJson<TrainRecord>(trainFile(nonce), 'train') : null);
|
||||
export const loadAttempt = (nonce: string, id: string) => (/^[A-Za-z0-9._-]{1,64}$/.test(id) ? readJson<AttemptRecord>(attemptPath(nonce, id), 'train-attempt') : null);
|
||||
export const loadBisect = (nonce: string) => readJson<BisectRecord>(bisectPath(nonce), 'train-bisect');
|
||||
export const loadFreeze = (repoKey: string) => readJson<FreezeRecord>(freezePath(repoKey), 'freeze');
|
||||
export const loadReady = (repoKey: string, pr: string, head: string) => readJson<ReadyReceipt>(readyPath(repoKey, pr, head), 'ready');
|
||||
export const activeFreeze = (repoKey: string): FreezeRecord | null => { const f = loadFreeze(repoKey); return f && !f.cleared_at ? f : null; };
|
||||
|
||||
export function attemptsOf(train: TrainRecord): AttemptRecord[] {
|
||||
return train.attempts.map(id => loadAttempt(train.nonce, id)).filter((a): a is AttemptRecord => !!a);
|
||||
}
|
||||
|
||||
/** Every train of this repository, newest first. */
|
||||
export function listTrains(repoKey: string): TrainRecord[] {
|
||||
const dir = path.join(queueRoot(), 'trains');
|
||||
if (!fs.existsSync(dir)) return [];
|
||||
return fs.readdirSync(dir).map(n => loadTrain(n)).filter((t): t is TrainRecord => !!t && t.repo_key === repoKey)
|
||||
.sort((a, b) => b.created_at.localeCompare(a.created_at) || b.nonce.localeCompare(a.nonce));
|
||||
}
|
||||
|
||||
export function recordPath(r: AnyRecord): string {
|
||||
if (r.kind === 'train') return trainFile(r.nonce);
|
||||
if (r.kind === 'train-attempt') return attemptPath(r.train, r.id);
|
||||
if (r.kind === 'train-bisect') return bisectPath(r.train);
|
||||
if (r.kind === 'freeze') return freezePath(r.repo_key);
|
||||
return readyPath(r.repo_key, r.pr, r.head);
|
||||
}
|
||||
|
||||
function noteName(r: AnyRecord): string {
|
||||
if (r.kind === 'train') return `gstack-trains:train:${r.nonce}`;
|
||||
if (r.kind === 'train-attempt') return `gstack-trains:attempt:${r.train}:${r.id}`;
|
||||
if (r.kind === 'train-bisect') return `gstack-trains:bisect:${r.train}`;
|
||||
if (r.kind === 'freeze') return `gstack-trains:freeze:${r.repo_key}`;
|
||||
return `gstack-trains:ready:${r.repo_key}:${r.pr}:${r.head}`;
|
||||
}
|
||||
|
||||
const NOTES_IDENTITY = { GIT_AUTHOR_NAME: 'gstack-ship-queue', GIT_AUTHOR_EMAIL: 'ship-queue@gstack.local', GIT_COMMITTER_NAME: 'gstack-ship-queue', GIT_COMMITTER_EMAIL: 'ship-queue@gstack.local' };
|
||||
const notesEnv = () => ({ ...process.env, ...NOTES_IDENTITY });
|
||||
|
||||
/** Fetch origin's notes and fold them into the local notes ref (the local side wins a same-record conflict). */
|
||||
export function fetchNotes(repoRoot: string, remote = 'origin'): { ok: boolean; detail: string } {
|
||||
const f = git(repoRoot, ['fetch', '--no-tags', '-q', remote, `+${NOTES_REF}:${NOTES_REMOTE_REF}`]);
|
||||
if (!f.ok) {
|
||||
const absent = /couldn't find remote ref|no such ref/i.test(f.err);
|
||||
return { ok: absent, detail: absent ? `${remote} has no ${NOTES_REF} yet` : `git fetch ${NOTES_REF} failed: ${f.err.slice(0, 160)}` };
|
||||
}
|
||||
const local = git(repoRoot, ['rev-parse', '--verify', '--quiet', NOTES_REF]).out;
|
||||
const fetched = git(repoRoot, ['rev-parse', '--verify', '--quiet', NOTES_REMOTE_REF]).out;
|
||||
if (!local) { git(repoRoot, ['update-ref', NOTES_REF, fetched]); return { ok: true, detail: 'notes fetched' }; }
|
||||
if (local === fetched || git(repoRoot, ['merge-base', '--is-ancestor', fetched, local]).ok) return { ok: true, detail: 'notes current' };
|
||||
const m = git(repoRoot, ['notes', `--ref=${NOTES_REF}`, 'merge', '-q', '-s', 'ours', NOTES_REMOTE_REF], notesEnv());
|
||||
return m.ok ? { ok: true, detail: 'notes merged' } : { ok: false, detail: `git notes merge failed: ${m.err.slice(0, 160)}` };
|
||||
}
|
||||
|
||||
function addNote(repoRoot: string, r: AnyRecord): boolean {
|
||||
const blob = git(repoRoot, ['hash-object', '-w', '--stdin'], undefined, noteName(r));
|
||||
if (!blob.ok) return false;
|
||||
return git(repoRoot, ['notes', `--ref=${NOTES_REF}`, 'add', '-f', '-F', '-', blob.out], notesEnv(), JSON.stringify(r)).ok;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write records locally, then as notes, then push the notes ref (fetch,
|
||||
* merge and retry on a race). A failed push leaves the local records and
|
||||
* notes for the next write to carry, and says so.
|
||||
*/
|
||||
export function persist(repoRoot: string, records: AnyRecord[], opts: { push?: boolean; remote?: string } = {}): { pushed: boolean; detail: string } {
|
||||
for (const r of records) writeJsonAtomic(recordPath(r), r);
|
||||
if (!records.every(r => addNote(repoRoot, r))) return { pushed: false, detail: 'could not write the notes locally' };
|
||||
if (opts.push === false) return { pushed: false, detail: 'notes written locally (not pushed)' };
|
||||
const remote = opts.remote ?? 'origin';
|
||||
let last = '';
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const p = git(repoRoot, ['push', '-q', remote, `${NOTES_REF}:${NOTES_REF}`]);
|
||||
if (p.ok) return { pushed: true, detail: `notes pushed to ${remote} ${NOTES_REF}` };
|
||||
last = p.err.slice(0, 160);
|
||||
const f = fetchNotes(repoRoot, remote);
|
||||
if (!f.ok) return { pushed: false, detail: f.detail };
|
||||
}
|
||||
return { pushed: false, detail: `git push ${NOTES_REF} failed: ${last}` };
|
||||
}
|
||||
|
||||
function newerOrMissing(r: AnyRecord): boolean {
|
||||
try {
|
||||
const cur = JSON.parse(fs.readFileSync(recordPath(r), 'utf8'));
|
||||
return !(typeof cur.updated_at === 'string' && cur.updated_at >= r.updated_at);
|
||||
} catch { return true; }
|
||||
}
|
||||
|
||||
function parseNote(text: string): AnyRecord | null {
|
||||
try {
|
||||
const r = JSON.parse(text);
|
||||
const ids = /^[A-Za-z0-9._-]{1,128}$/;
|
||||
if (r?.schema !== TRAIN_SCHEMA || typeof r.updated_at !== 'string') return null;
|
||||
if (r.kind === 'train' && ids.test(r.nonce)) return r;
|
||||
if ((r.kind === 'train-attempt' && ids.test(r.train) && ids.test(r.id)) || (r.kind === 'train-bisect' && ids.test(r.train))) return r;
|
||||
if (r.kind === 'freeze' && ids.test(r.repo_key)) return r;
|
||||
if (r.kind === 'ready' && ids.test(r.repo_key) && /^\d+$/.test(r.pr) && /^[0-9a-f]{40,64}$/.test(r.head)) return r;
|
||||
} catch {}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Rebuild local records from the notes ref: a note whose record is missing locally, or newer than the local copy, is written. */
|
||||
export function rebuildFromNotes(repoRoot: string, remote = 'origin'): { restored: number; read: number; detail: string } {
|
||||
const f = fetchNotes(repoRoot, remote);
|
||||
const list = git(repoRoot, ['notes', `--ref=${NOTES_REF}`, 'list']);
|
||||
if (!list.ok || !list.out.trim()) return { restored: 0, read: 0, detail: f.detail };
|
||||
const records = list.out.split('\n').map(l => l.split(' ')[0]!).filter(Boolean)
|
||||
.map(blob => parseNote(git(repoRoot, ['cat-file', 'blob', blob]).out)).filter((r): r is AnyRecord => !!r);
|
||||
const fresh = records.filter(newerOrMissing);
|
||||
for (const r of fresh) writeJsonAtomic(recordPath(r), r);
|
||||
return { restored: fresh.length, read: records.length, detail: f.detail };
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
/**
|
||||
* Train verdict, bisect and the one preregistered rebuild
|
||||
* (docs/ship-policy.md#run-a-merge-train; plan GSTA-49 lane T1, P3).
|
||||
*
|
||||
* The verdict reads the train's own gate run and the record artifact that run
|
||||
* publishes (with the `gstack/full-gate` check run) on the train head; it
|
||||
* only reads, never writes a check. Green needs a complete census from the
|
||||
* run's own plan manifest. A red train stays red: the bisect runs only its
|
||||
* red cases, one trial on every prefix T0..Tk, and attributes by pattern.
|
||||
* Rebuild-once (Garry, 2026-10-18; to be recorded in EVAL_POLICY beside
|
||||
* infraRedispatch): an all-green bisect earns one unchanged rebuild.
|
||||
*/
|
||||
import * as path from 'node:path';
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import { DISPATCH_INPUTS, listDispatchRuns, makeNonce, runForNonce } from '../scripts/eval-bg';
|
||||
import type { QueueContext } from './ship-queue';
|
||||
import { git } from './ship-queue-gate';
|
||||
import {
|
||||
checkRun, downloadJson, execFor, asRun, remoteBranchSha, repoSlug, runView, type CheckInfo, type RunInfo,
|
||||
} from './ship-queue-train-github';
|
||||
import {
|
||||
attemptsOf, loadBisect, nowIso, persist, trainDir, trainFile,
|
||||
type AttemptRecord, type BisectRecord, type CasePanel, type GateOutcome, type PrefixResult, type TrainRecord,
|
||||
} from './ship-queue-train-record';
|
||||
import {
|
||||
dispatchAttempt, effect, findTrain, gateDispatch, nextFor, outFailed, outOk, outPending, outRefused, pinList, save, trainEnv,
|
||||
type TrainEnv, type TrainOut,
|
||||
} from './ship-queue-train';
|
||||
|
||||
/** EVAL_POLICY.infraRedispatch (test/helpers/periodic-exclude-data.ts): one re-dispatch of an INFRA/INCOMPLETE-only census. */
|
||||
export const INFRA_REDISPATCH = 1;
|
||||
/** Rebuild-once (Garry, 2026-10-18): an all-green bisect earns exactly one unchanged rebuild; pinned to EVAL_POLICY by its test. */
|
||||
export const REBUILD_ONCE = 1;
|
||||
|
||||
const Q = 'gstack-ship-queue';
|
||||
|
||||
/** The record artifact the train's own gate run publishes (plan T1b), read by verdict, verify and the backstop. */
|
||||
export interface TrainGateRecord {
|
||||
schema: 1; kind: 'train-gate'; nonce: string; head_sha: string; tree: string; verdict: 'GREEN' | 'RED';
|
||||
census: { complete: boolean; planned: { rule: number; judge: number }; executed: { rule: number; judge: number };
|
||||
unbound: number; partial_shards: number; cancelled_shards: number; killed_before_upload: number };
|
||||
panels: CasePanel[];
|
||||
}
|
||||
/** One bisect prefix's artifact: the red cases on that prefix, one trial each. */
|
||||
export interface BisectPrefixRecord { schema: 1; kind: 'train-bisect-prefix'; nonce: string; prefix_sha: string; checked_out_sha: string; panels: CasePanel[] }
|
||||
|
||||
export const bisectArtifact = (recordArtifact: string, sha: string) => `${recordArtifact}-bisect-${sha.slice(0, 12)}`;
|
||||
|
||||
export interface GateClass { outcome: GateOutcome; detail: string; red: string[]; panels: CasePanel[]; infraEligible: boolean; cost: number | null }
|
||||
|
||||
/** Every paid-report class mapped to one train outcome; green only for a complete, bound, published census. */
|
||||
export function classifyGate(raw: unknown, expect: { nonce: string; head: string; tree: string }, run: Pick<RunInfo, 'status' | 'conclusion'>, check: CheckInfo | null): GateClass {
|
||||
const base = { red: [] as string[], panels: [] as CasePanel[], infraEligible: false, cost: null as number | null };
|
||||
if (run.status !== 'completed') return { ...base, outcome: 'PENDING', detail: `run is ${run.status}` };
|
||||
const rec = raw as TrainGateRecord | null;
|
||||
if (!rec || rec.schema !== 1 || rec.kind !== 'train-gate' || !Array.isArray(rec.panels) || !rec.census) {
|
||||
return { ...base, outcome: 'INCOMPLETE', infraEligible: true, detail: `no train record artifact (run concluded ${run.conclusion ?? 'unknown'}; a slice killed before its upload step counts here)` };
|
||||
}
|
||||
const panels = rec.panels;
|
||||
const known = panels.map(p => p.cost_usd).filter((c): c is number => typeof c === 'number');
|
||||
const cost = known.length ? Math.round(known.reduce((a, b) => a + b, 0) * 100) / 100 : null;
|
||||
const withPanels = { ...base, panels, cost };
|
||||
if (rec.nonce !== expect.nonce || rec.head_sha !== expect.head || rec.tree !== expect.tree) {
|
||||
return { ...withPanels, outcome: 'INCOMPLETE', detail: `the record is bound to nonce ${rec.nonce} head ${String(rec.head_sha).slice(0, 12)} tree ${String(rec.tree).slice(0, 12)}, not this train's attempt` };
|
||||
}
|
||||
const red = panels.filter(p => p.status === 'FAIL' && p.red_class === 'VERDICT').map(p => p.case).sort();
|
||||
if (red.length) return { ...withPanels, outcome: 'RED', red, detail: `${red.length} red case(s): ${red.join(', ')}` };
|
||||
const c = rec.census;
|
||||
const gaps = [
|
||||
...(c.planned.rule + c.planned.judge <= 0 ? ['the plan manifest lists no cases'] : []),
|
||||
...(c.executed.rule !== c.planned.rule ? [`rule ${c.executed.rule}/${c.planned.rule} executed`] : []),
|
||||
...(c.executed.judge !== c.planned.judge ? [`judge ${c.executed.judge}/${c.planned.judge} executed`] : []),
|
||||
...(c.unbound ? [`${c.unbound} unbound result`] : []), ...(c.partial_shards ? [`${c.partial_shards} partial shard`] : []),
|
||||
...(c.cancelled_shards ? [`${c.cancelled_shards} cancelled shard`] : []), ...(c.killed_before_upload ? [`${c.killed_before_upload} slice killed before upload`] : []),
|
||||
...(!c.complete ? ['census marked incomplete'] : []),
|
||||
...panels.filter(p => p.status !== 'PASS').map(p => `${p.case} ${p.status}${p.red_class ? ` (${p.red_class})` : ''}`),
|
||||
];
|
||||
if (gaps.length) return { ...withPanels, outcome: 'INCOMPLETE', infraEligible: !panels.some(p => p.status === 'SKIPPED'), detail: gaps.join('; ') };
|
||||
if (run.conclusion !== 'success' || rec.verdict !== 'GREEN') return { ...withPanels, outcome: 'INCOMPLETE', detail: `run concluded ${run.conclusion ?? 'unknown'} with record verdict ${rec.verdict}` };
|
||||
if (!check || check.conclusion !== 'success') return { ...withPanels, outcome: 'INCOMPLETE', detail: `the run did not publish a successful check run on the train head (${check ? `${check.status}/${check.conclusion}` : 'missing'})` };
|
||||
return { ...withPanels, outcome: 'GREEN', detail: `rule ${c.executed.rule}/${c.planned.rule}, judge ${c.executed.judge}/${c.planned.judge}` };
|
||||
}
|
||||
|
||||
const gateAttempts = (t: TrainRecord) => attemptsOf(t).filter(a => a.purpose !== 'bisect');
|
||||
const attemptLine = (a: AttemptRecord) => `ATTEMPT: ${a.id} ${a.purpose} run ${a.run?.id ?? '-'} ${a.outcome ?? a.state}${a.red_cases.length ? ` red=${a.red_cases.join(',')}` : ''}${a.cost_usd !== null ? ` cost=$${a.cost_usd.toFixed(2)}` : ''}${a.detail ? ` (${a.detail})` : ''}`;
|
||||
|
||||
function casePrice(env: TrainEnv, red: AttemptRecord, id: string): number {
|
||||
return red.panels.find(p => p.case === id)?.cost_usd ?? env.train.trial_reservation_usd;
|
||||
}
|
||||
|
||||
/** The whole-bisect reservation for a red attempt: every red case on every prefix. */
|
||||
export function bisectReservation(env: TrainEnv, t: TrainRecord, red: AttemptRecord): number {
|
||||
const perPrefix = red.red_cases.reduce((sum, id) => sum + casePrice(env, red, id), 0);
|
||||
return Math.ceil(perPrefix * t.prefixes.length * 100) / 100;
|
||||
}
|
||||
|
||||
function unknownDispatch(t: TrainRecord, a: AttemptRecord): TrainOut {
|
||||
return outPending({ code: 'QUEUE_TRAIN_DISPATCH_UNKNOWN', message: `the ${a.purpose} dispatch ${a.id} (nonce ${a.nonce}) is unconfirmed` }, [attemptLine(a)], `${Q} train status --train ${t.nonce}`, trainFile(t.nonce));
|
||||
}
|
||||
|
||||
/** `train verdict`: read the current gate attempt's run and record; print every attempt, so a rebuild reports both runs. */
|
||||
export function trainVerdict(ctx: QueueContext, o: { train: string }): TrainOut {
|
||||
const env = trainEnv(ctx);
|
||||
if ('code' in env) return outRefused(env, [], 'none', 'none');
|
||||
const t = findTrain(env, o.train);
|
||||
if (!t) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.train} here or in refs/notes/gstack-trains` }, [], `${Q} train status`, 'none');
|
||||
const run = trainFile(t.nonce);
|
||||
const a = gateAttempts(t).at(-1);
|
||||
if (!a) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `train ${t.nonce} has no gate dispatch yet` }, [], nextFor(t), run);
|
||||
if (a.state === 'dispatching' || a.state === 'unknown') return unknownDispatch(t, a);
|
||||
if (a.state === 'absent') return outRefused({ code: 'QUEUE_TRAIN_DISPATCH_UNKNOWN', message: `the dispatch ${a.nonce} never started (train status found no run)` }, [attemptLine(a)], `${Q} train build --train ${t.nonce}`, run);
|
||||
const slug = repoSlug(ctx);
|
||||
if (!a.outcome || a.outcome === 'PENDING') {
|
||||
const view = slug && a.run ? runView(ctx, slug, a.run.id) : null;
|
||||
if (!slug || !view) return outPending({ code: 'QUEUE_TRAIN_PENDING', message: `cannot read run ${a.run?.id} through gh` }, [attemptLine(a)], `${Q} train verdict --train ${t.nonce}`, run);
|
||||
const record = view.status === 'completed' ? downloadJson(ctx, slug, a.run!.id, env.train.record_artifact, path.join(trainDir(t.nonce), 'scratch')) : null;
|
||||
const check = view.status === 'completed' ? checkRun(ctx, t.head, env.train.full_gate_check) : null;
|
||||
const c = classifyGate(record, { nonce: a.nonce, head: t.head, tree: t.tree }, view, check);
|
||||
Object.assign(a, { outcome: c.outcome, detail: c.detail, red_cases: c.red, panels: c.panels, infra_eligible: c.infraEligible, cost_usd: c.cost, state: view.status === 'completed' ? 'completed' : a.state });
|
||||
if (c.outcome !== 'PENDING') {
|
||||
t.state = c.outcome === 'GREEN' ? 'green' : c.outcome === 'RED' ? (a.purpose === 'rebuild' ? 'unattributed' : 'red') : 'incomplete';
|
||||
effect(t, `verdict ${a.id}: ${c.outcome}`);
|
||||
}
|
||||
save(env, t, a);
|
||||
}
|
||||
const lines = [...gateAttempts(t).map(attemptLine), `VERDICT: ${a.outcome} train ${t.nonce} run ${a.run?.id ?? '-'} head ${t.head.slice(0, 12)} tree ${t.tree.slice(0, 12)}`];
|
||||
if (a.outcome === 'PENDING') return outPending({ code: 'QUEUE_TRAIN_PENDING', message: `the gate run ${a.run?.id} has not finished (${a.detail})` }, lines, `${Q} train verdict --train ${t.nonce}`, run);
|
||||
if (a.outcome === 'GREEN') return outOk(lines, nextFor(t), run);
|
||||
if (a.outcome === 'RED') {
|
||||
const next = a.purpose === 'rebuild'
|
||||
? `second red stays red: route ${a.red_cases.join(',')} to a repair lane (GStack Minder), then ${Q} train build --prs <pins without the PRs held>`
|
||||
: `${Q} train bisect --train ${t.nonce} --spend-cap ${bisectReservation(env, t, a).toFixed(2)}`;
|
||||
return outFailed({ code: 'QUEUE_TRAIN_RED', message: `train ${t.nonce} is red on ${a.red_cases.join(', ')}` }, lines, next, run);
|
||||
}
|
||||
const used = gateAttempts(t).filter(x => x.purpose === 'infra-redispatch').length;
|
||||
const next = a.infra_eligible && used < INFRA_REDISPATCH
|
||||
? `${Q} train rebuild --train ${t.nonce} --infra (the single re-dispatch EVAL_POLICY.infraRedispatch allows; report both runs)`
|
||||
: `diagnose run ${a.run?.url ?? a.run?.id}; the train stays incomplete (GStack Minder routes it)`;
|
||||
return outFailed({ code: 'QUEUE_TRAIN_INCOMPLETE', message: `train ${t.nonce} gate is incomplete: ${a.detail}` }, lines, next, run);
|
||||
}
|
||||
|
||||
export interface BisectClass { kind: 'attributed' | 'red-on-main' | 'unattributed'; all_green: boolean; prs: string[]; patterns: Record<string, string> }
|
||||
|
||||
/**
|
||||
* Pattern attribution over T0..Tk (one trial each, so only shapes count):
|
||||
* monotone green-then-red from Ti blames slot i's PR; red everywhere,
|
||||
* T0 included, is main; all green, or anything else, is unattributed.
|
||||
*/
|
||||
export function classifyBisect(cases: string[], results: Array<Record<string, PrefixResult>>, slots: Array<{ pr: string }>): BisectClass {
|
||||
const patterns: Record<string, string> = {};
|
||||
const blamed = new Set<string>();
|
||||
let main = false; let other = false; let green = 0;
|
||||
for (const c of cases) {
|
||||
const seq = results.map(r => (r[c] === 'red' ? 'R' : 'G')).join('');
|
||||
patterns[c] = seq;
|
||||
const first = seq.indexOf('R');
|
||||
if (first < 0) green++;
|
||||
else if (first === 0 && !seq.includes('G')) main = true;
|
||||
else if (first > 0 && !seq.slice(first).includes('G')) blamed.add(slots[first - 1]!.pr);
|
||||
else other = true;
|
||||
}
|
||||
if (main) return { kind: 'red-on-main', all_green: false, prs: [], patterns };
|
||||
if (other || !blamed.size) return { kind: 'unattributed', all_green: green === cases.length, prs: [], patterns };
|
||||
return { kind: 'attributed', all_green: false, prs: [...blamed].sort((a, b) => Number(a) - Number(b)), patterns };
|
||||
}
|
||||
|
||||
function prefixResults(raw: unknown, sha: string, nonce: string, cases: string[]): { results: Record<string, PrefixResult>; cost: number } | null {
|
||||
const rec = raw as BisectPrefixRecord | null;
|
||||
if (!rec || rec.schema !== 1 || rec.kind !== 'train-bisect-prefix' || rec.nonce !== nonce || rec.prefix_sha !== sha || rec.checked_out_sha !== sha || !Array.isArray(rec.panels)) return null;
|
||||
const results: Record<string, PrefixResult> = {};
|
||||
let cost = 0;
|
||||
for (const p of rec.panels) {
|
||||
if (!cases.includes(p.case)) continue;
|
||||
cost += typeof p.cost_usd === 'number' ? p.cost_usd : 0;
|
||||
if (p.status === 'PASS') results[p.case] = 'green';
|
||||
else if (p.status === 'FAIL' && p.red_class === 'VERDICT') results[p.case] = 'red';
|
||||
}
|
||||
return { results, cost };
|
||||
}
|
||||
|
||||
/** Collect a finished bisect dispatch's prefix artifacts into the ledger; a still-running one returns its run state. */
|
||||
function collect(env: TrainEnv, slug: string, t: TrainRecord, b: BisectRecord, a: AttemptRecord): string | null {
|
||||
const view = a.run ? runView(env.ctx, slug, a.run.id) : null;
|
||||
if (!view) return 'unreadable through gh';
|
||||
if (view.status !== 'completed') return view.status;
|
||||
let got = 0;
|
||||
for (const index of a.prefixes) {
|
||||
const p = b.prefixes[index]!;
|
||||
const r = prefixResults(downloadJson(env.ctx, slug, a.run!.id, bisectArtifact(env.train.record_artifact, p.sha), path.join(trainDir(t.nonce), 'scratch')), p.sha, a.nonce, b.cases);
|
||||
if (!r) continue;
|
||||
Object.assign(p.results, r.results);
|
||||
b.spent_usd = Math.round((b.spent_usd + r.cost) * 100) / 100;
|
||||
got++;
|
||||
}
|
||||
Object.assign(a, { state: 'completed', detail: `collected ${got} of ${a.prefixes.length} prefix result(s)` });
|
||||
return null;
|
||||
}
|
||||
|
||||
function bisectOutcome(env: TrainEnv, t: TrainRecord, b: BisectRecord): TrainOut {
|
||||
const c = classifyBisect(b.cases, b.prefixes.map(p => p.results), t.slots);
|
||||
b.outcome = { ...c, at: nowIso() };
|
||||
t.state = c.kind;
|
||||
effect(t, `bisect: ${c.kind}${c.prs.length ? ' #' + c.prs.join(',#') : ''}`);
|
||||
b.updated_at = nowIso();
|
||||
persist(env.ctx.repoRoot, [b], { remote: env.remote });
|
||||
save(env, t);
|
||||
const run = trainFile(t.nonce);
|
||||
const lines = [...Object.entries(c.patterns).map(([id, s]) => `PATTERN: ${id} ${s.split('').map((x, i) => `T${i}=${x}`).join(' ')}`), `BISECT: spent $${b.spent_usd.toFixed(2)} of cap $${b.spend_cap.toFixed(2)}`];
|
||||
if (c.kind === 'attributed') return outOk([...lines, `BISECT: ATTRIBUTED ${c.prs.map(p => '#' + p).join(',')}`], nextFor(t), run);
|
||||
if (c.kind === 'red-on-main') return outFailed({ code: 'QUEUE_TRAIN_RED_ON_MAIN', message: `${b.cases.join(', ')} red on every prefix, T0 (main) included` }, [...lines, 'BISECT: RED_ON_MAIN'], nextFor(t), run);
|
||||
return outFailed({ code: 'QUEUE_TRAIN_UNATTRIBUTED', message: c.all_green ? 'every prefix was green: the red is unattributed; recovery: the one preregistered rebuild' : 'the pattern is not monotone: recovery: a repair lane, then a changed train' },
|
||||
[...lines, `BISECT: UNATTRIBUTED (${c.all_green ? 'all green' : 'not monotone'})`], nextFor(t), run);
|
||||
}
|
||||
|
||||
/** Validate bisect inputs: red cases from the gate's own plan, every prefix an ancestor of train/<nonce>'s head. */
|
||||
function bisectInputs(env: TrainEnv, t: TrainRecord, red: AttemptRecord): string | null {
|
||||
for (const id of red.red_cases) {
|
||||
if (!/^[A-Za-z0-9_.:/-]{1,128}$/.test(id) || !red.panels.some(p => p.case === id)) return `red case ${id} is not in the gate's plan`;
|
||||
}
|
||||
const ref = `refs/gstack/${t.branch}`;
|
||||
const f = git(env.ctx.repoRoot, ['fetch', '--no-tags', '-q', env.remote, `+refs/heads/${t.branch}:${ref}`]);
|
||||
if (!f.ok) return `cannot fetch ${t.branch}: ${f.err.slice(0, 120)}`;
|
||||
const tip = git(env.ctx.repoRoot, ['rev-parse', ref]).out;
|
||||
if (tip !== t.head) return `${t.branch} is at ${tip.slice(0, 12)}, not the recorded train head ${t.head.slice(0, 12)}`;
|
||||
for (const p of t.prefixes) {
|
||||
if (!git(env.ctx.repoRoot, ['merge-base', '--is-ancestor', p.commit, tip]).ok) return `prefix T${p.index} ${p.commit.slice(0, 12)} is not an ancestor of ${t.branch}`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* `train bisect`: resumable and spend-capped. Each run reconciles the earlier
|
||||
* dispatch by nonce, records every prefix result it published, then reserves
|
||||
* the remaining case×prefix trials at their recorded price (unknown at the
|
||||
* policy reservation) and launches only the prefixes that fit under the cap.
|
||||
*/
|
||||
export async function trainBisect(ctx: QueueContext, o: { train: string; spendCap: number; dryRun: boolean }): Promise<TrainOut> {
|
||||
const env = trainEnv(ctx);
|
||||
if ('code' in env) return outRefused(env, [], 'none', 'none');
|
||||
const t = findTrain(env, o.train);
|
||||
if (!t) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.train} here or in refs/notes/gstack-trains` }, [], `${Q} train status`, 'none');
|
||||
const run = trainFile(t.nonce);
|
||||
const red = gateAttempts(t).at(-1);
|
||||
if (!red || red.outcome !== 'RED') return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `bisect needs a red verdict; the train's last gate is ${red?.outcome ?? 'missing'}` }, [], nextFor(t), run);
|
||||
if (red.purpose === 'rebuild') return outRefused({ code: 'QUEUE_PROOF_MISSING', message: 'a rebuild\'s second red is not bisected again' }, [], nextFor(t), run);
|
||||
const prior = loadBisect(t.nonce);
|
||||
if (prior?.outcome) return bisectOutcome(env, t, prior);
|
||||
const bad = bisectInputs(env, t, red);
|
||||
if (bad) return outRefused({ code: 'QUEUE_TRAIN_PREFIX_INVALID', message: bad }, [], `${Q} train status --train ${t.nonce}`, run);
|
||||
const b: BisectRecord = prior ?? { schema: 1, kind: 'train-bisect', train: t.nonce, cases: red.red_cases, spend_cap: o.spendCap, spent_usd: 0,
|
||||
prefixes: t.prefixes.map(p => ({ index: p.index, sha: p.commit, results: {}, attempts: [] })), outcome: null, updated_at: nowIso() };
|
||||
b.spend_cap = o.spendCap;
|
||||
const slug = repoSlug(ctx);
|
||||
if (!slug) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: 'gh repo view could not name the repository' }, [], `${Q} train bisect --train ${t.nonce} --spend-cap ${o.spendCap}`, run);
|
||||
for (const a of attemptsOf(t).filter(x => x.purpose === 'bisect' && x.state !== 'completed' && x.state !== 'absent')) {
|
||||
if (a.state === 'dispatching' || a.state === 'unknown') {
|
||||
const hit = runForNonce(listDispatchRuns(a.workflow, execFor(ctx), slug, 100) ?? [], a.nonce);
|
||||
if (!hit) return unknownDispatch(t, a);
|
||||
Object.assign(a, { state: 'dispatched', run: asRun(hit) });
|
||||
}
|
||||
const waiting = collect(env, slug, t, b, a);
|
||||
save(env, t, a);
|
||||
persist(ctx.repoRoot, [b], { remote: env.remote });
|
||||
if (waiting) return outPending({ code: 'QUEUE_TRAIN_PENDING', message: `bisect run ${a.run?.id} is ${waiting}` }, [attemptLine(a)], `${Q} train bisect --train ${t.nonce} --spend-cap ${o.spendCap}`, run);
|
||||
}
|
||||
const pending = b.prefixes.filter(p => b.cases.some(c => !p.results[c]));
|
||||
if (!pending.length) return bisectOutcome(env, t, b);
|
||||
const perPrefix = b.cases.reduce((sum, id) => sum + casePrice(env, red, id), 0);
|
||||
const admitted: typeof pending = [];
|
||||
let reserved = b.spent_usd;
|
||||
for (const p of pending) if (reserved + perPrefix <= o.spendCap + 1e-9) { admitted.push(p); reserved += perPrefix; }
|
||||
const higher = Math.ceil((b.spent_usd + perPrefix * pending.length) * 100) / 100;
|
||||
const lines = [`BISECT: cases ${b.cases.join(',')} prefixes ${pending.map(p => 'T' + p.index).join(',')} need results; $${perPrefix.toFixed(2)} reserved per prefix; spent $${b.spent_usd.toFixed(2)}, cap $${o.spendCap.toFixed(2)}`,
|
||||
`BISECT: launch ${admitted.map(p => 'T' + p.index).join(',') || '(none)'}${admitted.length < pending.length ? `; ${pending.length - admitted.length} prefix(es) wait for a higher cap` : ''}`];
|
||||
if (o.dryRun) return outOk([...lines, 'DRY-RUN: nothing dispatched'], `${Q} train bisect --train ${t.nonce} --spend-cap ${o.spendCap}`, run);
|
||||
persist(ctx.repoRoot, [b], { remote: env.remote });
|
||||
if (!admitted.length) {
|
||||
return outPending({ code: 'QUEUE_TRAIN_BISECT_CAPPED', message: `the next prefix needs $${perPrefix.toFixed(2)}; $${(o.spendCap - b.spent_usd).toFixed(2)} of the cap is left; the outcome stays unattributed` }, lines,
|
||||
`${Q} train bisect --train ${t.nonce} --spend-cap ${higher.toFixed(2)}`, run);
|
||||
}
|
||||
const nonce = makeNonce('gate', t.head, t.base.sha, randomBytes(3).toString('hex'), { purpose: 'bisect' });
|
||||
const a = await dispatchAttempt(env, slug, t, 'bisect', { workflow: env.train.gate_workflow, inputs: {
|
||||
[DISPATCH_INPUTS.cases]: b.cases.join(','), [DISPATCH_INPUTS.prefixShas]: admitted.map(p => p.sha).join(','),
|
||||
[DISPATCH_INPUTS.expectedSha]: t.head, [DISPATCH_INPUTS.nonce]: nonce,
|
||||
} }, admitted.map(p => p.index));
|
||||
for (const p of admitted) p.attempts.push(a.id);
|
||||
b.updated_at = nowIso();
|
||||
persist(ctx.repoRoot, [b], { remote: env.remote });
|
||||
if (a.state !== 'dispatched') return unknownDispatch(t, a);
|
||||
lines.push(attemptLine(a));
|
||||
if (admitted.length < pending.length) {
|
||||
return outPending({ code: 'QUEUE_TRAIN_BISECT_CAPPED', message: `${pending.length - admitted.length} prefix(es) are past the cap; completed prefixes are kept and the outcome stays unattributed` }, lines,
|
||||
`${Q} train bisect --train ${t.nonce} --spend-cap ${higher.toFixed(2)}`, run);
|
||||
}
|
||||
return outOk(lines, `${Q} train bisect --train ${t.nonce} --spend-cap ${o.spendCap} (collects run ${a.run?.id} once it finishes)`, run);
|
||||
}
|
||||
|
||||
/**
|
||||
* `train rebuild`: the unchanged train dispatched once more. Without
|
||||
* `--infra`, only after an all-green bisect and only once (rebuild-once);
|
||||
* with `--infra`, only after an INFRA/INCOMPLETE-only census and at most
|
||||
* EVAL_POLICY.infraRedispatch times. Both runs stay in the record.
|
||||
*/
|
||||
export async function trainRebuild(ctx: QueueContext, o: { train: string; infra: boolean }): Promise<TrainOut> {
|
||||
const env = trainEnv(ctx);
|
||||
if ('code' in env) return outRefused(env, [], 'none', 'none');
|
||||
const t = findTrain(env, o.train);
|
||||
if (!t) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.train} here or in refs/notes/gstack-trains` }, [], `${Q} train status`, 'none');
|
||||
const run = trainFile(t.nonce);
|
||||
const gates = gateAttempts(t);
|
||||
const last = gates.at(-1);
|
||||
const repair = `route ${last?.red_cases.join(',') || 'the red cases'} to a repair lane (GStack Minder); the train is rebuilt only once the repair merges: ${Q} train build --prs ${pinList(t.slots)}`;
|
||||
const refuse = (reason: string) => outRefused({ code: 'QUEUE_TRAIN_REBUILD_REFUSED', message: `rebuild refused: ${reason}` }, gates.map(attemptLine), repair, run);
|
||||
if (o.infra) {
|
||||
if (!last || last.outcome !== 'INCOMPLETE' || !last.infra_eligible) return refuse(`no INFRA/INCOMPLETE-only record (the last gate is ${last?.outcome ?? 'missing'})`);
|
||||
if (gates.filter(a => a.purpose === 'infra-redispatch').length >= INFRA_REDISPATCH) return refuse('already re-dispatched once (EVAL_POLICY.infraRedispatch)');
|
||||
} else {
|
||||
const b = loadBisect(t.nonce);
|
||||
if (!b?.outcome) return refuse('no bisect record');
|
||||
if (gates.filter(a => a.purpose === 'rebuild').length >= REBUILD_ONCE) return refuse('already rebuilt once');
|
||||
if (b.outcome.kind !== 'unattributed' || !b.outcome.all_green) return refuse(`the bisect was not all green (${Object.entries(b.outcome.patterns).map(([c, s]) => `${c}=${s}`).join(' ')})`);
|
||||
}
|
||||
const tip = remoteBranchSha(args => git(ctx.repoRoot, args), env.remote, t.branch);
|
||||
if (tip !== t.head) return outRefused({ code: 'QUEUE_TRAIN_MOVED', message: `${t.branch} is at ${tip?.slice(0, 12) ?? 'nothing'}, not the recorded head ${t.head.slice(0, 12)}` }, [], `${Q} train build --prs ${pinList(t.slots)}`, run);
|
||||
const slug = repoSlug(ctx);
|
||||
if (!slug) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: 'gh repo view could not name the repository' }, [], `${Q} train rebuild --train ${t.nonce}${o.infra ? ' --infra' : ''}`, run);
|
||||
const a = await dispatchAttempt(env, slug, t, o.infra ? 'infra-redispatch' : 'rebuild', gateDispatch(env, t, `${t.nonce}.${gates.length + 1}`));
|
||||
t.state = a.state === 'dispatched' ? 'gate-pending' : 'dispatch-unknown';
|
||||
save(env, t);
|
||||
if (a.state !== 'dispatched') return unknownDispatch(t, a);
|
||||
return outOk([...gateAttempts(t).map(attemptLine), `REBUILD: ${a.purpose} of the unchanged train ${t.nonce} (head ${t.head.slice(0, 12)}); both runs are reported`], nextFor(t), run);
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
/**
|
||||
* `verify --pr <n> --train <nonce>`: the coordinator's merge check under
|
||||
* merge trains (docs/ship-policy.md#run-a-merge-train; plan GSTA-49 T1, P1).
|
||||
*
|
||||
* Each merge needs a green train record that includes the PR at its pinned
|
||||
* head, with main's tree equal to the prefix before the PR (pre-merge:
|
||||
* MERGE_CONDITION) or after it (post-merge confirmation), modulo stamps
|
||||
* (lib/tree-receipt.ts). When a train stops partway (a mismatch or a moved
|
||||
* head after earlier slots merged) the queue freezes: the freeze is written
|
||||
* locally and to the notes ref first, then the backstop gate is dispatched on
|
||||
* main's published prefix at once. One repair train is admitted under the
|
||||
* freeze; verify clears the freeze when main after that merge carries it.
|
||||
*/
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import { DISPATCH_INPUTS, awaitNonceRun, dispatchWorkflow, makeNonce } from '../scripts/eval-bg';
|
||||
import type { QueueContext } from './ship-queue';
|
||||
import { git } from './ship-queue-gate';
|
||||
import { checkRun, execFor, repoSlug } from './ship-queue-train-github';
|
||||
import { activeFreeze, attemptsOf, nowIso, persist, trainFile, type FreezeRecord, type TrainRecord } from './ship-queue-train-record';
|
||||
import {
|
||||
effect, fetchBase, fetchTrainBranch, findTrain, frozenLines, nextFor, outFailed, outOk, outRefused, pinList, save, trainEnv, type TrainEnv, type TrainOut,
|
||||
} from './ship-queue-train';
|
||||
import { buildReceipt } from './tree-receipt';
|
||||
|
||||
const Q = 'gstack-ship-queue';
|
||||
|
||||
function sameModuloStamps(env: TrainEnv, gated: string, current: string): { same: boolean; detail: string } {
|
||||
const r = buildReceipt({ repoRoot: env.ctx.repoRoot, gated, current, policy: env.policy.policy, policyLabel: env.policy.source.label });
|
||||
if ('error' in r) return { same: false, detail: r.error };
|
||||
return { same: r.tree === 'same-modulo-stamps', detail: r.tree === 'same-modulo-stamps' ? `same modulo stamps (${r.files.length} stamp file(s))` : `differs: ${r.residual.slice(0, 8).join(', ')}` };
|
||||
}
|
||||
|
||||
/**
|
||||
* Freeze the queue: the record goes to local state and the notes ref before
|
||||
* anything remote, then the backstop gate is dispatched on main's sha.
|
||||
*/
|
||||
async function freezeQueue(env: TrainEnv, t: TrainRecord, reason: string, mainSha: string): Promise<FreezeRecord> {
|
||||
const now = nowIso();
|
||||
const f: FreezeRecord = {
|
||||
schema: 1, kind: 'freeze', repo_key: env.repo.key, train: t.nonce, merged: t.merged.map(m => m.pr),
|
||||
unmerged: t.slots.filter(s => !t.merged.some(m => m.pr === s.pr)).map(s => ({ pr: s.pr, head: s.head })),
|
||||
main: { sha: mainSha, tree: git(env.ctx.repoRoot, ['rev-parse', `${mainSha}^{tree}`]).out }, reason, backstop: null, repair: null,
|
||||
created_at: now, cleared_at: null, cleared_by: null, updated_at: now,
|
||||
};
|
||||
persist(env.ctx.repoRoot, [f], { remote: env.remote });
|
||||
effect(t, `queue frozen: ${reason}`);
|
||||
save(env, t);
|
||||
const slug = repoSlug(env.ctx);
|
||||
const nonce = makeNonce('gate', mainSha, mainSha, randomBytes(3).toString('hex'), { purpose: 'backstop' });
|
||||
const dispatch = { workflow: env.train.gate_workflow, inputs: { [DISPATCH_INPUTS.evalsAll]: 'true', [DISPATCH_INPUTS.expectedSha]: mainSha, [DISPATCH_INPUTS.nonce]: nonce } };
|
||||
const exec = execFor(env.ctx);
|
||||
const found = slug && dispatchWorkflow(dispatch, env.ctx.base, exec, slug) ? await awaitNonceRun(dispatch.workflow, nonce, exec, slug) : null;
|
||||
f.backstop = { nonce, run: found?.databaseId ?? null, state: found ? 'dispatched' : 'unconfirmed' };
|
||||
f.updated_at = nowIso();
|
||||
persist(env.ctx.repoRoot, [f], { remote: env.remote });
|
||||
return f;
|
||||
}
|
||||
|
||||
/** Interrupted publication: freeze when earlier slots merged; otherwise a plain refusal (nothing was published). */
|
||||
async function stopped(env: TrainEnv, t: TrainRecord, code: 'QUEUE_TRAIN_TREE_MISMATCH' | 'QUEUE_TRAIN_MOVED', message: string, mainSha: string, lines: string[]): Promise<TrainOut> {
|
||||
const run = trainFile(t.nonce);
|
||||
if (!t.merged.length) return outFailed({ code, message: `${message}; nothing of train ${t.nonce} merged, so the queue is not frozen` }, lines, `${Q} train build --prs ${pinList(t.slots)} (a new train on current main)`, run);
|
||||
const f = activeFreeze(env.repo.key) ?? await freezeQueue(env, t, message, mainSha);
|
||||
return outFailed({ code, message: `${message}; the queue is frozen` }, [...lines, ...frozenLines(f)], `${Q} train build --repair --prs <repair-pr>@<sha>`, run);
|
||||
}
|
||||
|
||||
export async function verifyTrain(ctx: QueueContext, o: { pr: string; train: string; noFetch: boolean }): Promise<TrainOut> {
|
||||
const fetchErr = o.noFetch ? null : fetchBase(ctx);
|
||||
const env = trainEnv(ctx);
|
||||
if ('code' in env) return outRefused(env, [], 'none', 'none');
|
||||
if (fetchErr) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: fetchErr }, [], 'none', 'none');
|
||||
const t = findTrain(env, o.train);
|
||||
if (!t) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.train} here or in refs/notes/gstack-trains` }, [], `${Q} train status`, 'none');
|
||||
const run = trainFile(t.nonce);
|
||||
const i = t.slots.findIndex(s => s.pr === o.pr);
|
||||
if (i < 0) return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} does not include #${o.pr}` }, [], nextFor(t), run);
|
||||
const gate = attemptsOf(t).filter(a => a.purpose !== 'bisect').at(-1);
|
||||
if (gate?.outcome !== 'GREEN' || !['green', 'merging', 'merged'].includes(t.state)) {
|
||||
return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} has no green record (state ${t.state}, last gate ${gate?.outcome ?? 'none'})` }, [], nextFor(t), run);
|
||||
}
|
||||
const check = checkRun(ctx, t.head, env.train.full_gate_check);
|
||||
if (check?.conclusion !== 'success') return outRefused({ code: 'QUEUE_GATE_MISSING', message: `no successful ${env.train.full_gate_check} check run on the train head ${t.head.slice(0, 12)}` }, [], `${Q} train verdict --train ${t.nonce}`, run);
|
||||
if (!fetchTrainBranch(env, t)) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `the train's prefix commits are not here and ${t.branch} could not be fetched` }, [], `${Q} train status --train ${t.nonce}`, run);
|
||||
const main = git(ctx.repoRoot, ['rev-parse', `${env.remote}/${ctx.base}`]).out;
|
||||
const freeze = activeFreeze(env.repo.key);
|
||||
if (freeze && freeze.repair?.train !== t.nonce) return outRefused({ code: 'QUEUE_TRAIN_FROZEN', message: `the queue is frozen (${freeze.reason}); only the admitted repair train merges` }, frozenLines(freeze), `${Q} train build --repair --prs <repair-pr>@<sha>`, run);
|
||||
const after = t.prefixes[i + 1]!;
|
||||
const before = t.prefixes[i]!;
|
||||
const post = sameModuloStamps(env, after.commit, main);
|
||||
const lines = [`TRAIN: ${t.nonce} slot ${i + 1}/${t.slots.length} #${o.pr} pinned ${t.slots[i]!.head}`];
|
||||
if (post.same) return confirmMerged(env, t, o.pr, main, after.index, [...lines, `main: ${main} = T${after.index} (${post.detail})`]);
|
||||
const pre = sameModuloStamps(env, before.commit, main);
|
||||
if (!pre.same) {
|
||||
return stopped(env, t, 'QUEUE_TRAIN_TREE_MISMATCH', `main ${main.slice(0, 12)} is neither T${before.index} nor T${after.index} modulo stamps (vs T${after.index}: ${post.detail})`, main, lines);
|
||||
}
|
||||
const headRef = o.noFetch ? '' : (git(ctx.repoRoot, ['fetch', '--no-tags', '-q', env.remote, `+refs/pull/${o.pr}/head:refs/gstack/queue/pr-${o.pr}`]).ok ? `refs/gstack/queue/pr-${o.pr}` : '');
|
||||
const head = headRef ? git(ctx.repoRoot, ['rev-parse', headRef]).out : t.slots[i]!.head;
|
||||
const stampOnly = head === t.slots[i]!.head ? { same: true, detail: 'the pinned head' } : sameModuloStamps(env, t.slots[i]!.head, head);
|
||||
if (!stampOnly.same) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${o.pr} moved past its pinned head ${t.slots[i]!.head.slice(0, 12)} (${head.slice(0, 12)}: ${stampOnly.detail})`, main, lines);
|
||||
return outOk([...lines, `main: ${main} = T${before.index} (${pre.detail})`, `head: ${head} (${stampOnly.detail})`,
|
||||
`VERIFY_TRAIN: #${o.pr} may merge as slot ${i + 1} of train ${t.nonce} (green run ${gate.run?.id ?? '-'}, ${env.train.full_gate_check} success)`, `MERGE_CONDITION: sha=${head}`],
|
||||
`merge #${o.pr} with expected head ${head.slice(0, 12)} (gstack-restamp stamps at merge), then ${Q} verify --pr ${o.pr} --train ${t.nonce}`, run);
|
||||
}
|
||||
|
||||
function confirmMerged(env: TrainEnv, t: TrainRecord, pr: string, main: string, index: number, lines: string[]): TrainOut {
|
||||
const run = trainFile(t.nonce);
|
||||
if (!t.merged.some(m => m.pr === pr)) t.merged.push({ pr, sha: main, tree: git(env.ctx.repoRoot, ['rev-parse', `${main}^{tree}`]).out, at: nowIso() });
|
||||
t.state = t.merged.length === t.slots.length ? 'merged' : 'merging';
|
||||
effect(t, `verified #${pr} merged as T${index}`);
|
||||
save(env, t);
|
||||
lines.push(`VERIFY_TRAIN: #${pr} merged; main's tree is T${index} of train ${t.nonce} (${t.merged.length}/${t.slots.length} merged)`);
|
||||
const freeze = activeFreeze(env.repo.key);
|
||||
if (freeze && freeze.repair?.train === t.nonce && t.state === 'merged') {
|
||||
freeze.cleared_at = nowIso();
|
||||
freeze.cleared_by = `repair #${pr} via train ${t.nonce}; main ${main.slice(0, 12)} carries a green record`;
|
||||
freeze.updated_at = freeze.cleared_at;
|
||||
persist(env.ctx.repoRoot, [freeze], { remote: env.remote });
|
||||
lines.push(`FREEZE: cleared (${freeze.cleared_by})`);
|
||||
return outOk(lines, freeze.unmerged.length ? `${Q} train build --prs ${pinList(freeze.unmerged)}` : 'none', run);
|
||||
}
|
||||
return outOk(lines, nextFor(t), run);
|
||||
}
|
||||
@@ -0,0 +1,509 @@
|
||||
/**
|
||||
* Merge trains for gstack-ship-queue (docs/ship-policy.md#run-a-merge-train;
|
||||
* plan GSTA-49 lane T1): `train build`, `train status` and `train prune`,
|
||||
* plus the pieces the verdict, bisect, rebuild and verify modules share.
|
||||
* The surface is active only when the base revision's ship policy declares a
|
||||
* `train` block (QUEUE_TRAIN_OFF otherwise).
|
||||
*
|
||||
* A train is main plus each READY PR's pinned, unstamped head merged in slot
|
||||
* order. The merges run in object storage (`git merge-tree --write-tree`, the
|
||||
* gate-ahead merge rules: version files take the earlier prefix, the
|
||||
* CHANGELOG merges by section), so each prefix T0..Tk is a commit with a
|
||||
* fixed identity and date and the same pins always give the same train head.
|
||||
* Records (lib/ship-queue-train-record.ts) are written before every remote
|
||||
* effect: the branch push, the free-tests dispatch, the paid dispatch through
|
||||
* scripts/eval-bg.ts's nonce.
|
||||
*/
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import { DISPATCH_INPUTS, awaitNonceRun, dispatchWorkflow, listDispatchRuns, makeNonce, runForNonce, type WorkflowDispatch } from '../scripts/eval-bg';
|
||||
import { mergeChangelogs, parseChangelog, type ChangelogSection } from './changelog-check';
|
||||
import type { ResultStatus } from './headless-artifacts';
|
||||
import { versionAtRef, versionFiles } from './restamp';
|
||||
import type { LoadedPolicy, ShipPolicy, TrainPolicy } from './ship-policy';
|
||||
import { loadBasePolicy, type QueueContext } from './ship-queue';
|
||||
import { git, sha256, withIndex, type QueueFailure } from './ship-queue-gate';
|
||||
import { asRun, execFor, newestBranchRun, prInfo, remoteBranchSha, repoSlug, runView, checkRun } from './ship-queue-train-github';
|
||||
import {
|
||||
TERMINAL_STATES, activeFreeze, attemptsOf, listTrains, loadBisect, loadReady, loadTrain, nowIso, persist, rebuildFromNotes, repoIdentity, trainFile,
|
||||
type AttemptPurpose, type AttemptRecord, type TrainRecord, type TrainSlot,
|
||||
} from './ship-queue-train-record';
|
||||
|
||||
export interface TrainOut { exit: number; status: ResultStatus; lines: string[]; next: string; failure?: QueueFailure; run: string }
|
||||
export const outOk = (lines: string[], next: string, run: string): TrainOut => ({ exit: 0, status: 'complete', lines, next, run });
|
||||
export const outRefused = (failure: QueueFailure, lines: string[], next: string, run: string): TrainOut => ({ exit: 3, status: 'refused', lines, next, failure, run });
|
||||
export const outFailed = (failure: QueueFailure, lines: string[], next: string, run: string): TrainOut => ({ exit: 1, status: 'refused', lines, next, failure, run });
|
||||
export const outPending = (failure: QueueFailure, lines: string[], next: string, run: string): TrainOut => ({ exit: 3, status: 'incomplete', lines, next, failure, run });
|
||||
|
||||
export interface TrainEnv { ctx: QueueContext; policy: LoadedPolicy; train: TrainPolicy; repo: { key: string; url: string }; remote: string }
|
||||
|
||||
const HEX = /^[0-9a-f]{40,64}$/;
|
||||
const Q = 'gstack-ship-queue';
|
||||
|
||||
/** The policy gate: the base revision's ship policy must declare a `train` block. */
|
||||
export function trainEnv(ctx: QueueContext): TrainEnv | QueueFailure {
|
||||
const policy = loadBasePolicy(ctx);
|
||||
if ('message' in policy) return policy;
|
||||
if (!policy.policy.train) return { code: 'QUEUE_TRAIN_OFF', message: `${policy.source.label} declares no train block; the per-PR queue applies` };
|
||||
return { ctx, policy, train: policy.policy.train, repo: repoIdentity(ctx.repoRoot), remote: ctx.remote ?? 'origin' };
|
||||
}
|
||||
|
||||
export function fetchBase(ctx: QueueContext): string | null {
|
||||
const remote = ctx.remote ?? 'origin';
|
||||
const f = git(ctx.repoRoot, ['fetch', '--no-tags', '-q', remote, `+refs/heads/${ctx.base}:refs/remotes/${remote}/${ctx.base}`]);
|
||||
return f.ok ? null : `git fetch ${remote} ${ctx.base} failed: ${f.err.slice(0, 160)}`;
|
||||
}
|
||||
|
||||
/** Make the train's prefix commits local (another machine built them): fetch train/<nonce>. */
|
||||
export function fetchTrainBranch(env: TrainEnv, train: TrainRecord): string | null {
|
||||
if (git(env.ctx.repoRoot, ['cat-file', '-e', `${train.head}^{commit}`]).ok) return train.head;
|
||||
const f = git(env.ctx.repoRoot, ['fetch', '--no-tags', '-q', env.remote, `+refs/heads/${train.branch}:refs/gstack/${train.branch}`]);
|
||||
return f.ok ? git(env.ctx.repoRoot, ['rev-parse', `refs/gstack/${train.branch}`]).out : null;
|
||||
}
|
||||
|
||||
/** A train record from local state, or from the notes ref when this machine has none. */
|
||||
export function findTrain(env: TrainEnv, nonce: string): TrainRecord | null {
|
||||
const local = loadTrain(nonce);
|
||||
if (local) return local;
|
||||
rebuildFromNotes(env.ctx.repoRoot, env.remote);
|
||||
return loadTrain(nonce);
|
||||
}
|
||||
|
||||
export const touch = <T extends { updated_at: string }>(r: T): T => { r.updated_at = nowIso(); return r; };
|
||||
|
||||
export function effect(train: TrainRecord, what: string): void {
|
||||
train.effects.push({ at: nowIso(), what });
|
||||
}
|
||||
|
||||
/** Save the train (and any attempt) locally and to the notes ref; a failed notes push is recorded as an effect, never hidden. */
|
||||
export function save(env: TrainEnv, train: TrainRecord, ...extra: AttemptRecord[]): string {
|
||||
touch(train);
|
||||
for (const a of extra) touch(a);
|
||||
const r = persist(env.ctx.repoRoot, [train, ...extra], { remote: env.remote });
|
||||
if (!r.pushed) { effect(train, `notes not pushed: ${r.detail}`); persist(env.ctx.repoRoot, [train], { push: false }); }
|
||||
return r.detail;
|
||||
}
|
||||
|
||||
export function parsePins(spec: string | undefined): Array<{ pr: string; sha: string }> | string {
|
||||
if (!spec) return 'needs --prs <n>@<sha>[,<n>@<sha>...]';
|
||||
const pins = spec.split(',').map(s => s.trim()).filter(Boolean).map(s => /^#?(\d+)@([0-9a-f]{7,64})$/.exec(s));
|
||||
if (!pins.length || pins.some(m => !m)) return `--prs entries are <pr>@<sha> (got ${spec})`;
|
||||
const out = pins.map(m => ({ pr: m![1]!, sha: m![2]! }));
|
||||
return new Set(out.map(p => p.pr)).size === out.length ? out : 'a PR appears twice in --prs';
|
||||
}
|
||||
|
||||
export const pinList = (slots: Array<{ pr: string; head: string }>) => slots.map(s => `${s.pr}@${s.head}`).join(',');
|
||||
|
||||
/** Slot order: screened PRs first, then unscreened, each in READY order; enough unscreened PRs form their own train. */
|
||||
export function orderSlots(slots: TrainSlot[], max: number): { train: TrainSlot[]; held: TrainSlot[]; split: boolean } {
|
||||
const byReady = (a: TrainSlot, b: TrainSlot) => a.ready_at.localeCompare(b.ready_at) || Number(a.pr) - Number(b.pr);
|
||||
const screened = slots.filter(s => s.screen.choice === 'screen').sort(byReady);
|
||||
const unscreened = slots.filter(s => s.screen.choice !== 'screen').sort(byReady);
|
||||
if (screened.length && unscreened.length >= max) return { train: screened.slice(0, max), held: [...screened.slice(max), ...unscreened], split: true };
|
||||
const all = [...screened, ...unscreened];
|
||||
return { train: all.slice(0, max), held: all.slice(max), split: false };
|
||||
}
|
||||
|
||||
/** When a train under its size may dispatch: its oldest READY plus the fill time; null when it may dispatch now. */
|
||||
export function fillDue(slots: TrainSlot[], train: TrainPolicy, now: string, urgent: boolean): string | null {
|
||||
if (urgent || slots.length >= train.max_size) return null;
|
||||
const oldest = Math.min(...slots.map(s => Date.parse(s.ready_at)));
|
||||
const due = oldest + train.fill_hours * 3_600_000;
|
||||
return Date.parse(now) >= due ? null : new Date(due).toISOString();
|
||||
}
|
||||
|
||||
function unionChangelog(ours: string, theirs: string): string {
|
||||
const a = parseChangelog(ours);
|
||||
const b = parseChangelog(theirs);
|
||||
const entry = (s: ChangelogSection) => s.lines.slice(1).join('\n').replace(/\n+$/, '');
|
||||
const known = new Set(b.sections.map(entry));
|
||||
const header = [...b.header];
|
||||
while (header.length && header[header.length - 1] === '') header.pop();
|
||||
const body = [...a.sections.filter(s => !known.has(entry(s))), ...b.sections].flatMap(s => {
|
||||
const lines = [...s.lines];
|
||||
while (lines.length && lines[lines.length - 1] === '') lines.pop();
|
||||
return [...lines, ''];
|
||||
});
|
||||
return [...header, ...(header.length ? [''] : []), ...body].join('\n').replace(/\n+$/, '\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* One train merge, in object storage: the PR head onto the previous prefix.
|
||||
* Version files take the prefix's side; the CHANGELOG merges by section (the
|
||||
* gate-ahead rule, else a union of sections, since each heading is restamped
|
||||
* at merge). Any other conflict is reported with every conflicting path.
|
||||
*/
|
||||
export function mergeStep(repoRoot: string, pred: string, head: string, policy: ShipPolicy): { tree: string } | { conflicts: string[] } {
|
||||
const r = git(repoRoot, ['merge-tree', '--write-tree', head, pred]);
|
||||
const lines = r.out.split('\n');
|
||||
const tree = lines[0]?.trim() ?? '';
|
||||
if (r.status === 0 && HEX.test(tree)) return { tree };
|
||||
if (r.status !== 1 || !HEX.test(tree)) return { conflicts: [`(git merge-tree failed: ${(r.err || r.out).slice(0, 120)})`] };
|
||||
const stages = new Map<string, Map<number, string>>();
|
||||
for (const line of lines.slice(1)) {
|
||||
if (!line.trim()) break;
|
||||
const m = /^(\d{6}) ([0-9a-f]{40,64}) ([123])\t(.+)$/.exec(line);
|
||||
if (!m) continue;
|
||||
const entry = stages.get(m[4]!) ?? new Map<number, string>();
|
||||
entry.set(Number(m[3]), m[2]!);
|
||||
stages.set(m[4]!, entry);
|
||||
}
|
||||
const at = versionAtRef(repoRoot, pred);
|
||||
const stampFiles = versionFiles(repoRoot, 'rel' in at ? at.rel : 'VERSION');
|
||||
const resolved: Array<{ path: string; blob: string }> = [];
|
||||
const conflicts: string[] = [];
|
||||
for (const [file, entry] of stages) {
|
||||
const ours = entry.get(2);
|
||||
const theirs = entry.get(3);
|
||||
if (!ours || !theirs) conflicts.push(file);
|
||||
else if (file === policy.changelog) {
|
||||
const text = (sha: string) => git(repoRoot, ['cat-file', 'blob', sha]).out + '\n';
|
||||
const merged = mergeChangelogs(text(ours), text(theirs)) ?? unionChangelog(text(ours), text(theirs));
|
||||
const blob = git(repoRoot, ['hash-object', '-w', '--stdin'], undefined, merged);
|
||||
if (blob.ok) resolved.push({ path: file, blob: blob.out }); else conflicts.push(file);
|
||||
} else if (stampFiles.includes(file)) resolved.push({ path: file, blob: theirs });
|
||||
else conflicts.push(file);
|
||||
}
|
||||
if (conflicts.length) return { conflicts: conflicts.sort() };
|
||||
const out = withIndex<{ tree: string } | { error: string }>(repoRoot, tree, index => {
|
||||
for (const { path: p, blob } of resolved) {
|
||||
const mode = git(repoRoot, ['ls-tree', tree, '--', p], index).out.split(/\s/)[0] || '100644';
|
||||
if (!git(repoRoot, ['update-index', '--add', '--cacheinfo', `${mode},${blob},${p}`], index).ok) return { error: `cannot stage ${p}` };
|
||||
}
|
||||
const w = git(repoRoot, ['write-tree'], index);
|
||||
return w.ok ? { tree: w.out } : { error: `write-tree failed: ${w.err}` };
|
||||
});
|
||||
return 'error' in out ? { conflicts: [`(${out.error})`] } : { tree: out.tree };
|
||||
}
|
||||
|
||||
const TRAIN_IDENTITY = { GIT_AUTHOR_NAME: 'gstack-ship-queue', GIT_AUTHOR_EMAIL: 'ship-queue@gstack.local', GIT_COMMITTER_NAME: 'gstack-ship-queue',
|
||||
GIT_COMMITTER_EMAIL: 'ship-queue@gstack.local', GIT_AUTHOR_DATE: '2000-01-01T00:00:00Z', GIT_COMMITTER_DATE: '2000-01-01T00:00:00Z' };
|
||||
|
||||
/** Build T1..Tk on T0 = base; a conflict stops at the PR that conflicts. */
|
||||
export function buildChain(repoRoot: string, baseSha: string, slots: TrainSlot[], policy: ShipPolicy):
|
||||
{ prefixes: TrainRecord['prefixes'] } | { stoppedAt: string; conflicts: string[]; merged: string[] } {
|
||||
const prefixes: TrainRecord['prefixes'] = [{ index: 0, commit: baseSha, tree: git(repoRoot, ['rev-parse', `${baseSha}^{tree}`]).out, pr: null }];
|
||||
for (const [i, slot] of slots.entries()) {
|
||||
const pred = prefixes[i]!.commit;
|
||||
const m = mergeStep(repoRoot, pred, slot.head, policy);
|
||||
if ('conflicts' in m) return { stoppedAt: slot.pr, conflicts: m.conflicts, merged: slots.slice(0, i).map(s => s.pr) };
|
||||
const c = git(repoRoot, ['commit-tree', m.tree, '-p', pred, '-p', slot.head, '-m', `train T${i + 1}: merge #${slot.pr} at ${slot.head.slice(0, 12)}`], { ...process.env, ...TRAIN_IDENTITY });
|
||||
if (!c.ok) return { stoppedAt: slot.pr, conflicts: [`(commit-tree failed: ${c.err.slice(0, 120)})`], merged: slots.slice(0, i).map(s => s.pr) };
|
||||
prefixes.push({ index: i + 1, commit: c.out, tree: m.tree, pr: slot.pr });
|
||||
}
|
||||
return { prefixes };
|
||||
}
|
||||
|
||||
/** NEXT for a train in its current state. */
|
||||
export function nextFor(t: TrainRecord): string {
|
||||
const n = t.nonce;
|
||||
const unmerged = t.slots.filter(s => !t.merged.some(m => m.pr === s.pr));
|
||||
switch (t.state) {
|
||||
case 'built': case 'pushed': case 'free-pending': case 'free-green': case 'dispatch-absent': return `${Q} train build --train ${n}`;
|
||||
case 'dispatch-unknown': return `${Q} train status --train ${n}`;
|
||||
case 'gate-pending': return `${Q} train verdict --train ${n}`;
|
||||
case 'green': case 'merging': return unmerged.length ? `${Q} verify --pr ${unmerged[0]!.pr} --train ${n}` : `${Q} train prune --train ${n}`;
|
||||
case 'red': return `${Q} train bisect --train ${n} --spend-cap <usd>`;
|
||||
case 'incomplete': return `${Q} train verdict --train ${n}`;
|
||||
case 'attributed': {
|
||||
const drop = loadBisect(n)?.outcome?.prs ?? [];
|
||||
const keep = t.slots.filter(s => !drop.includes(s.pr));
|
||||
return keep.length ? `${Q} train build --prs ${pinList(keep)}` : 'none (every PR was dropped)';
|
||||
}
|
||||
case 'red-on-main': return `fix forward on main first, then ${Q} train build --prs ${pinList(t.slots)}`;
|
||||
case 'unattributed': return loadBisect(n)?.outcome?.all_green && !attemptsOf(t).some(a => a.purpose === 'rebuild')
|
||||
? `${Q} train rebuild --train ${n}`
|
||||
: `route the red cases to a repair lane (GStack Minder); once the repair merges: ${Q} train build --prs ${pinList(t.slots)}`;
|
||||
default: return `${Q} train prune --train ${n}`;
|
||||
}
|
||||
}
|
||||
|
||||
export function attemptId(train: TrainRecord, purpose: AttemptPurpose): string {
|
||||
const bisect = purpose === 'bisect';
|
||||
return `${bisect ? 'b' : 'a'}${train.attempts.filter(id => id.startsWith(bisect ? 'b' : 'a')).length + 1}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record an attempt as dispatching (locally and in the notes ref), dispatch
|
||||
* through eval-bg, then find the run by its nonce. A failed or unconfirmed
|
||||
* dispatch is `unknown`: train status reconciles it, nothing re-dispatches it.
|
||||
*/
|
||||
export async function dispatchAttempt(env: TrainEnv, slug: string, train: TrainRecord, purpose: AttemptPurpose, dispatch: WorkflowDispatch, prefixes: number[] = []): Promise<AttemptRecord> {
|
||||
const now = nowIso();
|
||||
const attempt: AttemptRecord = {
|
||||
schema: 1, kind: 'train-attempt', train: train.nonce, id: attemptId(train, purpose), purpose, nonce: dispatch.inputs[DISPATCH_INPUTS.nonce]!,
|
||||
workflow: dispatch.workflow, ref: train.branch, inputs: dispatch.inputs, state: 'dispatching', run: null, outcome: null, detail: null,
|
||||
red_cases: [], panels: [], infra_eligible: false, cost_usd: null, prefixes, created_at: now, updated_at: now,
|
||||
};
|
||||
train.attempts.push(attempt.id);
|
||||
save(env, train, attempt);
|
||||
const exec = execFor(env.ctx);
|
||||
const found = dispatchWorkflow(dispatch, train.branch, exec, slug) ? await awaitNonceRun(dispatch.workflow, attempt.nonce, exec, slug) : null;
|
||||
attempt.state = found ? 'dispatched' : 'unknown';
|
||||
attempt.run = found ? asRun(found) : null;
|
||||
effect(train, found ? `${purpose} dispatched: ${dispatch.workflow} run ${found.databaseId} (nonce ${attempt.nonce})` : `${purpose} dispatch unconfirmed (nonce ${attempt.nonce})`);
|
||||
save(env, train, attempt);
|
||||
return attempt;
|
||||
}
|
||||
|
||||
export function gateDispatch(env: TrainEnv, train: TrainRecord, nonce: string): WorkflowDispatch {
|
||||
return { workflow: env.train.gate_workflow, inputs: { [DISPATCH_INPUTS.evalsAll]: 'true', [DISPATCH_INPUTS.expectedSha]: train.head, [DISPATCH_INPUTS.nonce]: nonce } };
|
||||
}
|
||||
|
||||
export interface BuildOptions { pins?: string; resume?: string; dryRun: boolean; urgent: boolean; repair: boolean; freeTimeoutSec: number; pollMs: number }
|
||||
|
||||
/** The slots a new train would carry: trusted, unmoved, READY PRs, in slot order. */
|
||||
function admitSlots(env: TrainEnv, pins: Array<{ pr: string; sha: string }>): { slots: TrainSlot[]; lines: string[] } | TrainOut {
|
||||
const { ctx } = env;
|
||||
const specs = pins.map(p => `+refs/pull/${p.pr}/head:refs/gstack/train/pr-${p.pr}`);
|
||||
const f = git(ctx.repoRoot, ['fetch', '--no-tags', '-q', env.remote, ...specs]);
|
||||
if (!f.ok) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `git fetch of the PR heads failed: ${f.err.slice(0, 160)}` }, [], 'none', 'none');
|
||||
const slots: TrainSlot[] = [];
|
||||
for (const p of pins) {
|
||||
const info = prInfo(ctx, p.pr);
|
||||
if ('error' in info) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: info.error }, [], 'none', 'none');
|
||||
const without = pinList(pins.filter(x => x.pr !== p.pr).map(x => ({ pr: x.pr, head: x.sha })));
|
||||
if (info.isCrossRepository) return outRefused({ code: 'QUEUE_TRAIN_UNTRUSTED', message: `PR #${p.pr}'s head repository is not the base repository; fork work stays on the trusted-branch path` }, [], without ? `${Q} train build --prs ${without}` : 'none', 'none');
|
||||
const fetched = git(ctx.repoRoot, ['rev-parse', `refs/gstack/train/pr-${p.pr}`]).out;
|
||||
if (!info.headRefOid.startsWith(p.sha) || fetched !== info.headRefOid) {
|
||||
return outRefused({ code: 'QUEUE_TRAIN_MOVED', message: `PR #${p.pr} moved: pinned ${p.sha.slice(0, 12)}, head is now ${info.headRefOid.slice(0, 12)}` }, [], `ask #${p.pr}'s lane for READY on ${info.headRefOid.slice(0, 12)}, then ${Q} train build --prs <pins>`, 'none');
|
||||
}
|
||||
const ready = loadReady(env.repo.key, p.pr, info.headRefOid);
|
||||
if (!ready) return outRefused({ code: 'QUEUE_NOT_READY', message: `no READY receipt for #${p.pr} at ${info.headRefOid.slice(0, 12)} (local state or ${'refs/notes/gstack-trains'})` }, [], `the lane runs ${Q} ready --pr ${p.pr} --screen|--no-screen`, 'none');
|
||||
slots.push({ pr: p.pr, head: info.headRefOid, ready_at: ready.ready_at, screen: ready.screen });
|
||||
}
|
||||
return { slots, lines: [] };
|
||||
}
|
||||
|
||||
/** Start a new train from --prs, or find the live train with the same pins and base. */
|
||||
async function newTrain(env: TrainEnv, o: BuildOptions): Promise<{ t: TrainRecord; lines: string[] } | TrainOut> {
|
||||
const pins = parsePins(o.pins);
|
||||
if (typeof pins === 'string') return outRefused({ code: 'QUEUE_PROOF_MISSING', message: pins }, [], 'none', 'none');
|
||||
if ((o.urgent || o.repair) && pins.length !== 1) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `${o.repair ? '--repair' : '--urgent'} admits exactly one PR` }, [], 'none', 'none');
|
||||
rebuildFromNotes(env.ctx.repoRoot, env.remote);
|
||||
const freeze = activeFreeze(env.repo.key);
|
||||
if (freeze && !o.repair) return outRefused({ code: 'QUEUE_TRAIN_FROZEN', message: `the queue is frozen since ${freeze.created_at}: train ${freeze.train} stopped after merging ${freeze.merged.map(p => '#' + p).join(',')} (${freeze.reason})` },
|
||||
frozenLines(freeze), `${Q} train build --repair --prs <repair-pr>@<sha>`, 'none');
|
||||
const admitted = admitSlots(env, pins);
|
||||
if ('exit' in admitted) return admitted;
|
||||
const order = orderSlots(admitted.slots, o.repair ? 1 : env.train.max_size);
|
||||
const lines = [
|
||||
`ORDER: ${order.train.map((s, i) => `${i + 1}:#${s.pr}(${s.screen.choice === 'screen' ? 'screened' : 'unscreened'})`).join(' ')}`,
|
||||
...(order.held.length ? [`HELD: ${order.held.map(s => '#' + s.pr).join(',')} (${order.split ? 'unscreened PRs form their own train' : `over the train size ${env.train.max_size}`})`] : []),
|
||||
];
|
||||
const due = fillDue(order.train, env.train, nowIso(), o.urgent || o.repair);
|
||||
if (due && !o.dryRun) return outPending({ code: 'QUEUE_TRAIN_WAITING', message: `${order.train.length} of ${env.train.max_size} PRs; the oldest READY dispatches at ${due}` }, lines, `${Q} train build --prs ${pinList(order.train)} (after ${due}, or with more PRs, or --urgent for one)`, 'none');
|
||||
const baseSha = git(env.ctx.repoRoot, ['rev-parse', `${env.remote}/${env.ctx.base}`]).out;
|
||||
const key = sha256(JSON.stringify({ base: baseSha, slots: order.train.map(s => `${s.pr}@${s.head}`) }));
|
||||
const live = listTrains(env.repo.key).find(t => t.key === key && !TERMINAL_STATES.includes(t.state) && t.state !== 'red' && t.state !== 'unattributed');
|
||||
if (live) return { t: Object.assign(live, { held: order.held.map(s => s.pr) }), lines };
|
||||
const chain = buildChain(env.ctx.repoRoot, baseSha, order.train, env.policy.policy);
|
||||
if ('stoppedAt' in chain) {
|
||||
const without = order.train.filter(s => s.pr !== chain.stoppedAt);
|
||||
return outFailed({ code: 'QUEUE_TRAIN_CONFLICT', message: `#${chain.stoppedAt} conflicts with the prefix before it` }, [...lines,
|
||||
`QUEUE_TRAIN_CONFLICT: train ${key.slice(0, 12)} stopped at #${chain.stoppedAt} (conflicts: ${chain.conflicts.join(', ')}) after merging ${chain.merged.map(p => '#' + p).join(',') || '(none)'}; nothing dispatched, no branch pushed; fix: ${Q} train build --prs ${pinList(without)}`],
|
||||
without.length ? `${Q} train build --prs ${pinList(without)}` : 'none', 'none');
|
||||
}
|
||||
const head = chain.prefixes.at(-1)!;
|
||||
const nonce = makeNonce('gate', head.commit, baseSha, randomBytes(3).toString('hex'), { purpose: 'train' });
|
||||
const now = nowIso();
|
||||
return { lines, t: {
|
||||
schema: 1, kind: 'train', nonce, key, repo_key: env.repo.key, repo: env.repo.url, base: { ref: env.ctx.base, sha: baseSha }, policy: env.policy.source.label,
|
||||
slots: order.train, held: order.held.map(s => s.pr), prefixes: chain.prefixes, head: head.commit, tree: head.tree, branch: `train/${nonce}`,
|
||||
urgent: o.urgent, repair: o.repair, state: 'built', free: null, attempts: [], merged: [], superseded_by: null, effects: [], error: null,
|
||||
created_at: now, updated_at: now,
|
||||
} };
|
||||
}
|
||||
|
||||
export function frozenLines(f: NonNullable<ReturnType<typeof activeFreeze>>): string[] {
|
||||
return [
|
||||
`QUEUE_TRAIN_FROZEN: train ${f.train} merged ${f.merged.map(p => '#' + p).join(',') || '(none)'}; main ${f.main.sha.slice(0, 12)} tree ${f.main.tree.slice(0, 12)}; backstop ${f.backstop ? `${f.backstop.state} run ${f.backstop.run ?? 'unknown'} (nonce ${f.backstop.nonce})` : 'not dispatched'}`,
|
||||
`freeze: ${f.reason}; unmerged ${f.unmerged.map(u => '#' + u.pr).join(',') || '(none)'}; repair ${f.repair ? `#${f.repair.pr}@${f.repair.head.slice(0, 12)}` : 'none admitted'}`,
|
||||
];
|
||||
}
|
||||
|
||||
function describe(t: TrainRecord): string[] {
|
||||
return [
|
||||
`TRAIN: ${t.nonce} state=${t.state} prs=${t.slots.map(s => '#' + s.pr).join(',')} head=${t.head} tree=${t.tree}`,
|
||||
`base: origin/${t.base.ref}@${t.base.sha}`,
|
||||
...t.prefixes.map(p => `PREFIX: T${p.index} ${p.commit} tree=${p.tree}${p.pr ? ` (+#${p.pr})` : ' (main)'}`),
|
||||
...t.slots.map((s, i) => `SLOT: ${i + 1} #${s.pr} head=${s.head} ${s.screen.choice === 'screen' ? `screened (${s.screen.policy ?? 'narrow'}${s.screen.run ? `, run ${s.screen.run}` : ''})` : 'unscreened'} ready_at=${s.ready_at}`),
|
||||
];
|
||||
}
|
||||
|
||||
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
|
||||
|
||||
/** Free tests on train/<nonce>: dispatched once, then followed until a verdict on the exact train head. */
|
||||
async function freePhase(env: TrainEnv, slug: string, t: TrainRecord, o: BuildOptions): Promise<TrainOut | null> {
|
||||
if (t.free?.verdict === 'green' && t.free.head_sha === t.head) return null;
|
||||
const run = trainFile(t.nonce);
|
||||
if (t.state !== 'free-pending') {
|
||||
if (!env.ctx.gh(['workflow', 'run', env.train.free_workflow, '--repo', slug, '--ref', t.branch]).ok) {
|
||||
t.free = { run: null, url: null, head_sha: null, conclusion: null, verdict: 'missing' };
|
||||
t.state = 'free-red';
|
||||
effect(t, `${env.train.free_workflow} dispatch refused by gh`);
|
||||
save(env, t);
|
||||
return outFailed({ code: 'QUEUE_TRAIN_FREE_RED', message: `gh could not dispatch ${env.train.free_workflow} on ${t.branch}` }, describe(t), `${Q} train build --prs ${pinList(t.slots)}`, run);
|
||||
}
|
||||
t.state = 'free-pending';
|
||||
effect(t, `${env.train.free_workflow} dispatched on ${t.branch}`);
|
||||
save(env, t);
|
||||
}
|
||||
const deadline = Date.now() + o.freeTimeoutSec * 1000;
|
||||
let found = newestBranchRun(env.ctx, slug, env.train.free_workflow, t.branch);
|
||||
while ((found === null || found === 'unreadable' || found.status !== 'completed') && Date.now() < deadline) {
|
||||
await sleep(o.pollMs);
|
||||
found = newestBranchRun(env.ctx, slug, env.train.free_workflow, t.branch);
|
||||
}
|
||||
if (found && found !== 'unreadable' && found.status !== 'completed') {
|
||||
t.free = { run: found.databaseId, url: found.url, head_sha: found.headSha, conclusion: null, verdict: 'pending' };
|
||||
save(env, t);
|
||||
return outPending({ code: 'QUEUE_TRAIN_PENDING', message: `free tests run ${found.databaseId} on ${t.branch} is still ${found.status}` }, describe(t), `${Q} train build --train ${t.nonce}`, run);
|
||||
}
|
||||
const r = found && found !== 'unreadable' ? found : null;
|
||||
const verdict = !r ? 'missing' : r.event !== 'workflow_dispatch' || r.headSha !== t.head ? 'stale' : r.conclusion === 'success' ? 'green' : r.conclusion === 'cancelled' ? 'cancelled' : 'red';
|
||||
t.free = { run: r?.databaseId ?? null, url: r?.url ?? null, head_sha: r?.headSha ?? null, conclusion: r?.conclusion ?? null, verdict };
|
||||
t.state = verdict === 'green' ? 'free-green' : 'free-red';
|
||||
save(env, t);
|
||||
if (verdict === 'green') return null;
|
||||
const why = verdict === 'missing' ? `no ${env.train.free_workflow} dispatch run appeared on ${t.branch}`
|
||||
: verdict === 'stale' ? `run ${r!.databaseId} tested ${(r!.headSha || '?').slice(0, 12)} (${r!.event}), not the train head ${t.head.slice(0, 12)}`
|
||||
: `run ${r!.databaseId} concluded ${r!.conclusion}`;
|
||||
return outFailed({ code: 'QUEUE_TRAIN_FREE_RED', message: `free tests ${verdict}: ${why}` }, [...describe(t), `FREE: ${verdict} ${r?.url ?? ''}`.trim()],
|
||||
`read the free-tests run; drop a PR that caused it: ${Q} train build --prs <pins without it>`, run);
|
||||
}
|
||||
|
||||
/** Push the branch, gate it with the free tests, then dispatch the paid gate once. */
|
||||
async function advance(env: TrainEnv, t: TrainRecord, o: BuildOptions, lines: string[]): Promise<TrainOut> {
|
||||
const run = trainFile(t.nonce);
|
||||
const slug = repoSlug(env.ctx);
|
||||
if (!slug) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: 'gh repo view could not name the repository' }, lines, `${Q} train build --train ${t.nonce}`, run);
|
||||
if (t.state === 'built') {
|
||||
const p = git(env.ctx.repoRoot, ['push', '-q', env.remote, `${t.head}:refs/heads/${t.branch}`]);
|
||||
if (!p.ok) { t.error = `branch push failed: ${p.err.slice(0, 160)}`; save(env, t); return outRefused({ code: 'QUEUE_PROOF_MISSING', message: t.error }, lines, `${Q} train build --train ${t.nonce}`, run); }
|
||||
t.state = 'pushed';
|
||||
effect(t, `branch pushed: ${t.branch} at ${t.head.slice(0, 12)}`);
|
||||
save(env, t);
|
||||
}
|
||||
const free = await freePhase(env, slug, t, o);
|
||||
if (free) return { ...free, lines: [...lines, ...free.lines] };
|
||||
if (t.state === 'dispatch-unknown') return outPending({ code: 'QUEUE_TRAIN_DISPATCH_UNKNOWN', message: `the gate dispatch for ${t.nonce} is unconfirmed` }, [...lines, ...describe(t)], `${Q} train status --train ${t.nonce}`, run);
|
||||
if (t.state === 'free-green' || t.state === 'dispatch-absent') {
|
||||
const prior = attemptsOf(t).filter(a => a.purpose === 'gate');
|
||||
const nonce = prior.at(-1)?.state === 'absent' ? prior.at(-1)!.nonce : t.nonce;
|
||||
const a = await dispatchAttempt(env, slug, t, 'gate', gateDispatch(env, t, nonce));
|
||||
t.state = a.state === 'dispatched' ? 'gate-pending' : 'dispatch-unknown';
|
||||
save(env, t);
|
||||
if (a.state !== 'dispatched') return outPending({ code: 'QUEUE_TRAIN_DISPATCH_UNKNOWN', message: `no ${a.workflow} run named ${a.nonce} appeared after the dispatch` }, [...lines, ...describe(t)], `${Q} train status --train ${t.nonce}`, run);
|
||||
}
|
||||
const gate = attemptsOf(t).filter(a => a.purpose !== 'bisect').at(-1);
|
||||
return outOk([...lines, ...describe(t), `FREE: green run ${t.free?.run ?? '?'} on ${t.head.slice(0, 12)}`,
|
||||
`GATE: ${gate?.workflow ?? env.train.gate_workflow} run ${gate?.run?.id ?? '?'} ${gate?.run?.url ?? ''} nonce ${gate?.nonce ?? t.nonce}`.trim(), `record: ${run}`], nextFor(t), run);
|
||||
}
|
||||
|
||||
/** `train build`: admit, order, merge, record, push, free tests, dispatch; `--dry-run` spends nothing and writes nothing. */
|
||||
export async function buildTrain(ctx: QueueContext, o: BuildOptions): Promise<TrainOut> {
|
||||
const fetchErr = fetchBase(ctx);
|
||||
const env = trainEnv(ctx);
|
||||
if ('code' in env) return outRefused(env, [], 'none', 'none');
|
||||
if (fetchErr) return outRefused({ code: 'POLICY_SOURCE_UNAVAILABLE', message: fetchErr }, [], 'none', 'none');
|
||||
let t: TrainRecord | TrainOut;
|
||||
let lines: string[] = [];
|
||||
if (o.resume) {
|
||||
t = findTrain(env, o.resume) ?? outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.resume} here or in refs/notes/gstack-trains` }, [], `${Q} train status`, 'none');
|
||||
if ('kind' in t && TERMINAL_STATES.includes(t.state)) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `train ${t.nonce} is ${t.state}; it never dispatches again` }, describe(t), nextFor(t), trainFile(t.nonce));
|
||||
if ('kind' in t && t.state !== 'free-pending' && t.state !== 'dispatch-absent' && t.state !== 'dispatch-unknown' && t.state !== 'built' && t.state !== 'pushed' && t.state !== 'free-green') {
|
||||
return outOk(describe(t), nextFor(t), trainFile(t.nonce));
|
||||
}
|
||||
} else {
|
||||
const made = await newTrain(env, o);
|
||||
if ('exit' in made) return made;
|
||||
({ t, lines } = made);
|
||||
}
|
||||
if (!('kind' in t)) return t;
|
||||
if (o.dryRun) {
|
||||
return outOk([...lines, ...describe(t), `ESTIMATE: one ${env.train.gate_workflow} full gate (evals_all) on T${t.prefixes.length - 1} plus ${env.train.free_workflow}; dollar figure pending the T0 cost replay (lib/gate-cost.ts)`,
|
||||
'DRY-RUN: nothing written, pushed or dispatched'], `${Q} train build --prs ${pinList(t.slots)}`, 'none');
|
||||
}
|
||||
if (t.state === 'built' && !t.attempts.length && !loadTrain(t.nonce)) {
|
||||
effect(t, 'train.json written');
|
||||
save(env, t);
|
||||
for (const old of listTrains(env.repo.key)) {
|
||||
if (old.nonce === t.nonce || TERMINAL_STATES.includes(old.state) || old.state === 'green' || old.state === 'merging') continue;
|
||||
if (!old.slots.some(s => t.slots.some((n: TrainSlot) => n.pr === s.pr))) continue;
|
||||
old.state = 'superseded';
|
||||
old.superseded_by = t.nonce;
|
||||
save(env, old);
|
||||
}
|
||||
if (o.repair) {
|
||||
const f = activeFreeze(env.repo.key)!;
|
||||
f.repair = { pr: t.slots[0]!.pr, head: t.slots[0]!.head, train: t.nonce };
|
||||
f.updated_at = nowIso();
|
||||
persist(env.ctx.repoRoot, [f], { remote: env.remote });
|
||||
}
|
||||
}
|
||||
return advance(env, t, o, lines);
|
||||
}
|
||||
|
||||
/**
|
||||
* `train status`: never spends. Rebuilds local records from the notes ref,
|
||||
* reconciles each unconfirmed dispatch by its nonce (found, or absent when a
|
||||
* full listing shows no such run) and reads run and check state.
|
||||
*/
|
||||
export function trainStatus(ctx: QueueContext, o: { train?: string }): TrainOut {
|
||||
const env = trainEnv(ctx);
|
||||
if ('code' in env) return outRefused(env, [], 'none', 'none');
|
||||
const restored = rebuildFromNotes(ctx.repoRoot, env.remote);
|
||||
const lines = [`NOTES: read ${restored.read} record(s), restored ${restored.restored} (${restored.detail})`];
|
||||
const trains = o.train ? [loadTrain(o.train)].filter((t): t is TrainRecord => !!t) : listTrains(env.repo.key).filter(t => !TERMINAL_STATES.includes(t.state));
|
||||
if (o.train && !trains.length) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.train} here or in refs/notes/gstack-trains` }, lines, `${Q} train status`, 'none');
|
||||
const slug = trains.length ? repoSlug(ctx) : null;
|
||||
const exec = execFor(ctx);
|
||||
for (const t of trains) {
|
||||
lines.push(...describe(t));
|
||||
for (const a of attemptsOf(t)) {
|
||||
if (slug && (a.state === 'dispatching' || a.state === 'unknown')) {
|
||||
const runs = listDispatchRuns(a.workflow, exec, slug, 100);
|
||||
const hit = runs ? runForNonce(runs, a.nonce) : null;
|
||||
if (runs) {
|
||||
a.state = hit ? 'dispatched' : 'absent';
|
||||
a.run = hit ? asRun(hit) : null;
|
||||
if (t.state === 'dispatch-unknown') t.state = hit ? (a.purpose === 'bisect' ? 'red' : 'gate-pending') : 'dispatch-absent';
|
||||
effect(t, `status reconciled ${a.id} (nonce ${a.nonce}): ${hit ? `run ${hit.databaseId}` : 'absent'}`);
|
||||
save(env, t, a);
|
||||
}
|
||||
}
|
||||
const view = slug && a.run && !a.outcome ? runView(ctx, slug, a.run.id) : null;
|
||||
lines.push(`ATTEMPT: ${a.id} ${a.purpose} state=${a.state} nonce=${a.nonce} run=${a.run?.id ?? '-'}${view ? ` ${view.status}/${view.conclusion ?? '-'}` : ''} outcome=${a.outcome ?? '-'}`);
|
||||
}
|
||||
if (slug && (t.state === 'green' || t.state === 'merging')) {
|
||||
const c = checkRun(ctx, t.head, env.train.full_gate_check);
|
||||
lines.push(`CHECK: ${env.train.full_gate_check} on ${t.head.slice(0, 12)}: ${c ? `${c.status}/${c.conclusion ?? '-'}` : 'missing'}`);
|
||||
}
|
||||
lines.push(`NEXT(${t.nonce}): ${nextFor(t)}`);
|
||||
}
|
||||
if (!trains.length) lines.push('TRAIN: none live');
|
||||
const freeze = activeFreeze(env.repo.key);
|
||||
if (freeze) lines.push(...frozenLines(freeze));
|
||||
const single = trains.length === 1 ? trains[0]! : null;
|
||||
const next = freeze && !freeze.repair ? `${Q} train build --repair --prs <repair-pr>@<sha>` : single ? nextFor(single) : trains.length ? 'see each NEXT(<nonce>) line' : 'none';
|
||||
return outOk(lines, next, single ? trainFile(single.nonce) : 'none');
|
||||
}
|
||||
|
||||
/** `train prune`: delete train/<nonce> on the remote for every finished (merged, superseded, dropped) train. */
|
||||
export function pruneTrains(ctx: QueueContext, o: { dryRun: boolean; train?: string }): TrainOut {
|
||||
const env = trainEnv(ctx);
|
||||
if ('code' in env) return outRefused(env, [], 'none', 'none');
|
||||
rebuildFromNotes(ctx.repoRoot, env.remote);
|
||||
const lines: string[] = [];
|
||||
const rgit = (args: string[]) => git(ctx.repoRoot, args);
|
||||
for (const t of listTrains(env.repo.key).filter(x => !o.train || x.nonce === o.train)) {
|
||||
if (!TERMINAL_STATES.includes(t.state)) { lines.push(`KEEP: ${t.branch} (${t.state})`); continue; }
|
||||
if (!remoteBranchSha(rgit, env.remote, t.branch)) { lines.push(`GONE: ${t.branch} (${t.state})`); continue; }
|
||||
if (o.dryRun) { lines.push(`WOULD-DELETE: ${t.branch} (${t.state})`); continue; }
|
||||
const d = git(ctx.repoRoot, ['push', '-q', env.remote, '--delete', `refs/heads/${t.branch}`]);
|
||||
lines.push(d.ok ? `DELETED: ${t.branch} (${t.state})` : `FAILED: ${t.branch}: ${d.err.slice(0, 120)}`);
|
||||
if (d.ok) { effect(t, `branch deleted: ${t.branch}`); save(env, t); }
|
||||
}
|
||||
if (!lines.length) lines.push('PRUNE: no trains');
|
||||
return outOk(lines, o.dryRun ? `${Q} train prune` : 'none', 'none');
|
||||
}
|
||||
+76
-25
@@ -45,7 +45,23 @@ export const UNRECORDED_SHARD_MS = 600_000;
|
||||
/** How long the dispatch follower may hold the lock: the longest CI lane plus queueing. */
|
||||
export const DISPATCH_FOLLOW_SECONDS = 4 * 60 * 60;
|
||||
|
||||
export interface WorkflowDispatch { workflow: 'evals.yml' | 'evals-periodic.yml'; inputs: Record<string, string> }
|
||||
export interface WorkflowDispatch { workflow: string; inputs: Record<string, string> }
|
||||
|
||||
/**
|
||||
* Every workflow_dispatch input name eval-bg and gstack-ship-queue send, in
|
||||
* one place. The train inputs (cases, prefix_shas) and the screen inputs
|
||||
* (screen_pr, screen_merge_sha, screen_policy) are the interface the
|
||||
* workflow side (plan GSTA-49, lane T1b) adds to evals.yml.
|
||||
*/
|
||||
export const DISPATCH_INPUTS = {
|
||||
evalsAll: 'evals_all', expectedSha: 'expected_sha', nonce: 'nonce', baseRef: 'base_ref', baseSha: 'base_sha', prReceipts: 'pr_receipts',
|
||||
cases: 'cases', prefixShas: 'prefix_shas', screenPr: 'screen_pr', screenMergeSha: 'screen_merge_sha', screenPolicy: 'screen_policy',
|
||||
} as const;
|
||||
|
||||
/** What a dispatch is for, beside its lane: the screen policy keeps a broad screen from adopting a narrow run. */
|
||||
export type DispatchPurpose = 'screen' | 'train' | 'bisect' | 'backstop';
|
||||
export type ScreenPolicy = 'narrow' | 'broad';
|
||||
export interface NonceScope { purpose: DispatchPurpose; policy?: ScreenPolicy }
|
||||
|
||||
interface LaneSpec {
|
||||
/** The package script the local backend runs. */
|
||||
@@ -65,13 +81,14 @@ export const LANE_SPECS: Record<EvalBgLane, LaneSpec> = {
|
||||
|
||||
/** The workflows a lane dispatches and their inputs (names agreed with evals.yml's workflow_dispatch). */
|
||||
export function dispatchPlan(lane: EvalBgLane, ids: { sha: string; baseRef: string; baseSha: string; nonce: string; prNumber?: number | null }): WorkflowDispatch[] {
|
||||
const I = DISPATCH_INPUTS;
|
||||
const evals = (all: boolean): WorkflowDispatch => ({ workflow: 'evals.yml', inputs: {
|
||||
evals_all: String(all), expected_sha: ids.sha, nonce: ids.nonce,
|
||||
...(all ? {} : { base_ref: ids.baseRef, base_sha: ids.baseSha }),
|
||||
[I.evalsAll]: String(all), [I.expectedSha]: ids.sha, [I.nonce]: ids.nonce,
|
||||
...(all ? {} : { [I.baseRef]: ids.baseRef, [I.baseSha]: ids.baseSha }),
|
||||
// CEO-29/ENG-8: a PR lane run on a ref with an open same-repo PR reads that PR's receipts read-only.
|
||||
...(!all && ids.prNumber ? { pr_receipts: String(ids.prNumber) } : {}),
|
||||
...(!all && ids.prNumber ? { [I.prReceipts]: String(ids.prNumber) } : {}),
|
||||
} });
|
||||
const periodic: WorkflowDispatch = { workflow: 'evals-periodic.yml', inputs: { expected_sha: ids.sha, nonce: ids.nonce } };
|
||||
const periodic: WorkflowDispatch = { workflow: 'evals-periodic.yml', inputs: { [I.expectedSha]: ids.sha, [I.nonce]: ids.nonce } };
|
||||
if (lane === 'pr') return [evals(false)];
|
||||
if (lane === 'gate') return [evals(true)];
|
||||
if (lane === 'periodic') return [periodic];
|
||||
@@ -163,12 +180,25 @@ export function exitStatusWord(code: string): 'passed' | 'failed' | 'cancelled'
|
||||
return code === '130' ? 'cancelled' : 'failed';
|
||||
}
|
||||
|
||||
/** Nonce: lane, tested revision and base, plus a random suffix; the dedupe key is everything but the suffix. */
|
||||
export function makeNonce(lane: EvalBgLane, sha: string, baseSha: string, random = Math.random().toString(36).slice(2, 8)): string {
|
||||
return `${dedupeKey(lane, sha, baseSha)}${random}`;
|
||||
/**
|
||||
* Nonce: lane, tested revision and base, plus a random suffix; the dedupe key
|
||||
* is everything but the suffix. A scoped dispatch (a screen, a train, a
|
||||
* bisect, a backstop) adds its purpose and screen policy to the lane field
|
||||
* (`pr.screen.broad-…`), so runs of another purpose or policy never match
|
||||
* its key and the plain lane keys are unchanged.
|
||||
*/
|
||||
export function makeNonce(lane: EvalBgLane, sha: string, baseSha: string, random = Math.random().toString(36).slice(2, 8), scope?: NonceScope): string {
|
||||
return `${dedupeKey(lane, sha, baseSha, scope)}${random}`;
|
||||
}
|
||||
export function dedupeKey(lane: EvalBgLane, sha: string, baseSha: string): string {
|
||||
return `${lane}-${sha.slice(0, 12)}-${(baseSha || 'none').slice(0, 12)}-`;
|
||||
export function dedupeKey(lane: EvalBgLane, sha: string, baseSha: string, scope?: NonceScope): string {
|
||||
const field = scope ? [lane, scope.purpose, ...(scope.policy ? [scope.policy] : [])].join('.') : lane;
|
||||
return `${field}-${sha.slice(0, 12)}-${(baseSha || 'none').slice(0, 12)}-`;
|
||||
}
|
||||
|
||||
/** The scope a nonce was minted with (null for a plain lane nonce). */
|
||||
export function nonceScope(nonce: string): NonceScope | null {
|
||||
const m = /^(?:pr|gate|periodic|release)\.(screen|train|bisect|backstop)(?:\.(narrow|broad))?-[0-9a-f]{1,12}-/.exec(nonce);
|
||||
return m ? { purpose: m[1] as DispatchPurpose, ...(m[2] ? { policy: m[2] as ScreenPolicy } : {}) } : null;
|
||||
}
|
||||
|
||||
export interface RunSummary { databaseId: number; displayTitle: string; status: string; conclusion: string | null; url: string; event: string }
|
||||
@@ -186,7 +216,39 @@ export function findDuplicateRun(runs: readonly RunSummary[], key: string): RunS
|
||||
|
||||
// ─── Process plumbing (git / gh / detach) ────────────────────────────────────
|
||||
|
||||
type Exec = (command: string, args: string[]) => { status: number | null; stdout: string };
|
||||
export type Exec = (command: string, args: string[]) => { status: number | null; stdout: string };
|
||||
|
||||
const RUN_FIELDS = 'databaseId,displayTitle,status,conclusion,url,event';
|
||||
|
||||
/** Recent workflow_dispatch runs of one workflow; null when gh could not list them (unknown, never "none"). */
|
||||
export function listDispatchRuns(workflow: string, run: Exec, repo = CANONICAL_REPO, limit = 30): RunSummary[] | null {
|
||||
const out = run('gh', ['run', 'list', '--repo', repo, '--workflow', workflow, '--event', 'workflow_dispatch', '--limit', String(limit), '--json', RUN_FIELDS]);
|
||||
if (out.status !== 0) return null;
|
||||
try { const parsed = JSON.parse(out.stdout); return Array.isArray(parsed) ? parsed as RunSummary[] : null; } catch { return null; }
|
||||
}
|
||||
|
||||
/** The dispatch run whose run-name carries exactly this nonce. */
|
||||
export function runForNonce(runs: readonly RunSummary[], nonce: string): RunSummary | null {
|
||||
return runs.find(item => item.event === 'workflow_dispatch' && item.displayTitle.includes(`eval-bg ${nonce}]`)) ?? null;
|
||||
}
|
||||
|
||||
/** `gh workflow run` with the plan's inputs on `ref`; false when gh refused it. */
|
||||
export function dispatchWorkflow(dispatch: WorkflowDispatch, ref: string, run: Exec, repo = CANONICAL_REPO): boolean {
|
||||
const args = ['workflow', 'run', dispatch.workflow, '--repo', repo, '--ref', ref,
|
||||
...Object.entries(dispatch.inputs).flatMap(([name, value]) => ['-f', `${name}=${value}`])];
|
||||
return run('gh', args).status === 0;
|
||||
}
|
||||
|
||||
/** Poll for the run a dispatch started, by its nonce, for up to ten poll intervals; null when none appeared. */
|
||||
export async function awaitNonceRun(workflow: string, nonce: string, run: Exec, repo = CANONICAL_REPO): Promise<RunSummary | null> {
|
||||
const deadline = Date.now() + 10 * pollMs();
|
||||
while (Date.now() < deadline) {
|
||||
await sleep(Math.min(pollMs(), 10_000));
|
||||
const found = runForNonce(listDispatchRuns(workflow, run, repo) ?? [], nonce);
|
||||
if (found) return found;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const exec: Exec = (command, args) => {
|
||||
const result = spawnSync(command, args, { cwd: ROOT, encoding: 'utf8', timeout: 60_000, maxBuffer: 16 * 1024 * 1024 });
|
||||
@@ -238,22 +300,11 @@ const pollMs = () => Number(process.env.GSTACK_EVAL_BG_POLL_MS) || 30_000;
|
||||
|
||||
/** Dispatch one workflow (or adopt a duplicate) and follow it; returns the mapped exit code. */
|
||||
async function followWorkflow(dispatch: WorkflowDispatch, ref: string, key: string, run: Exec): Promise<number> {
|
||||
const list = () => {
|
||||
const out = run('gh', ['run', 'list', '--repo', CANONICAL_REPO, '--workflow', dispatch.workflow, '--event', 'workflow_dispatch',
|
||||
'--limit', '30', '--json', 'databaseId,displayTitle,status,conclusion,url,event']);
|
||||
try { return out.status === 0 ? JSON.parse(out.stdout) as RunSummary[] : []; } catch { return []; }
|
||||
};
|
||||
let target = findDuplicateRun(list(), key);
|
||||
let target = findDuplicateRun(listDispatchRuns(dispatch.workflow, run) ?? [], key);
|
||||
if (target) console.log(`[eval-bg] ${dispatch.workflow}: following existing run ${target.url} (same lane, revision and base; no duplicate dispatch)`);
|
||||
else {
|
||||
const args = ['workflow', 'run', dispatch.workflow, '--repo', CANONICAL_REPO, '--ref', ref,
|
||||
...Object.entries(dispatch.inputs).flatMap(([name, value]) => ['-f', `${name}=${value}`])];
|
||||
if (run('gh', args).status !== 0) { console.log(`[eval-bg] ${dispatch.workflow}: gh workflow run failed`); return 2; }
|
||||
const deadline = Date.now() + 10 * pollMs();
|
||||
while (!target && Date.now() < deadline) {
|
||||
await sleep(Math.min(pollMs(), 10_000));
|
||||
target = list().find(item => item.displayTitle.includes(`eval-bg ${dispatch.inputs.nonce}`)) ?? null;
|
||||
}
|
||||
if (!dispatchWorkflow(dispatch, ref, run)) { console.log(`[eval-bg] ${dispatch.workflow}: gh workflow run failed`); return 2; }
|
||||
target = await awaitNonceRun(dispatch.workflow, dispatch.inputs[DISPATCH_INPUTS.nonce]!, run);
|
||||
if (!target) { console.log(`[eval-bg] ${dispatch.workflow}: dispatched, but no run named ${dispatch.inputs.nonce} appeared`); return 2; }
|
||||
}
|
||||
console.log(`[eval-bg] ${dispatch.workflow}: run ${target.databaseId} ${target.url}`);
|
||||
|
||||
+8
-1
@@ -121,7 +121,14 @@
|
||||
{ "file": "lib/script-invocation.ts", "movedFrom": "(new: the one Windows bash resolver, shebang-bin invocation and wtreeFingerprint, from spawn-bin.ts and gbrain-exec.ts, #3197)" },
|
||||
{ "file": "lib/autoplan-budget.ts", "movedFrom": "(new: round cap, default host estimate, budget_exhausted stop and the budget verb, field list 2 W5)" },
|
||||
{ "file": "lib/autoplan-confirm.ts", "movedFrom": "(new: confirm-round plan delta, revision-qualified correlation and settled findings, field list 2 W5)" },
|
||||
{ "file": "lib/autoplan-public.ts", "movedFrom": "(new: machine-path-free public run copy and validate --public, field list 2 W6)" }
|
||||
{ "file": "lib/autoplan-public.ts", "movedFrom": "(new: machine-path-free public run copy and validate --public, field list 2 W6)" },
|
||||
{ "file": "lib/ship-queue-train.ts", "movedFrom": "(new: train build, status and prune, slot order, fill and prefix merges, plan GSTA-49 T1)" },
|
||||
{ "file": "lib/ship-queue-train-record.ts", "movedFrom": "(new: versioned train, attempt, bisect, freeze and READY records mirrored to refs/notes/gstack-trains, plan GSTA-49 T1)" },
|
||||
{ "file": "lib/ship-queue-train-github.ts", "movedFrom": "(new: gh reads for train runs, check runs and PR facts, plan GSTA-49 T1)" },
|
||||
{ "file": "lib/ship-queue-train-verdict.ts", "movedFrom": "(new: train verdict, bisect and rebuild-once, plan GSTA-49 T1)" },
|
||||
{ "file": "lib/ship-queue-train-verify.ts", "movedFrom": "(new: verify --train, the queue freeze and its backstop, plan GSTA-49 T1)" },
|
||||
{ "file": "lib/ship-queue-train-ready.ts", "movedFrom": "(new: ready --screen|--no-screen and the READY receipt, plan GSTA-49 T1)" },
|
||||
{ "file": "lib/ship-queue-train-cli.ts", "movedFrom": "(new: merge-train argument parsing and output, plan GSTA-49 T1)" }
|
||||
],
|
||||
"residualFiles": [
|
||||
{ "file": "scripts/test-free-shards.ts", "maxLines": 2361, "reason": "free lane policy after the shard engine took spawn, sandbox, logs, seeds and flags" },
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
/**
|
||||
* Fixtures for merge trains (lib/ship-queue-train*.ts): a hermetic repo from
|
||||
* restamp-fixture (bare origin plus working clone under os.tmpdir(), its own
|
||||
* git identity) whose base policy declares a `train` block, and a stateful
|
||||
* `gh` on PATH that answers every GitHub call the train surface makes from a
|
||||
* JSON file: PR views, repo view, workflow dispatches that create runs (with
|
||||
* a train record, a full-gate check run and bisect prefix artifacts built
|
||||
* from per-test rules), run lists by workflow, branch and event, run views,
|
||||
* artifact downloads and check-run reads. Every call is logged, so tests can
|
||||
* prove what was (and was not) dispatched.
|
||||
*/
|
||||
import { createHash } from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { TRAIN_KEYS_VERSION } from '../../lib/ship-policy';
|
||||
import { BASE_POLICY, makeFixtureRepo, runBin, type FixtureRepo } from './restamp-fixture';
|
||||
|
||||
export const TRAIN_BLOCK = {
|
||||
gate_workflow: 'evals.yml', free_workflow: 'free-tests.yml', full_gate_check: 'gstack/full-gate', record_artifact: 'gstack-train-record',
|
||||
max_size: 5, fill_hours: 3, screen_selection: 'none', trial_reservation_usd: 1.5,
|
||||
};
|
||||
export const TRAIN_POLICY = { ...BASE_POLICY, train: TRAIN_BLOCK, min_gstack_version: TRAIN_KEYS_VERSION };
|
||||
|
||||
export interface Rule {
|
||||
fail?: boolean; silent?: boolean; failButCreate?: boolean; status?: string; conclusion?: string | null; event?: string; headSha?: string;
|
||||
record?: { panels: unknown[]; census?: unknown; verdict?: 'GREEN' | 'RED'; nonce?: string };
|
||||
check?: string | null;
|
||||
bisect?: { prefixes: string[]; patterns: Record<string, string>; costUsd?: number };
|
||||
}
|
||||
export interface ShimData {
|
||||
repo: string; origin: string; nextId: number; prs: Record<string, unknown>; runs: Array<Record<string, unknown>>;
|
||||
artifacts: Record<string, Record<string, unknown>>; checks: Record<string, Array<{ name: string; status: string; conclusion: string | null }>>;
|
||||
ci: Record<string, unknown[]>; rules: Record<string, Rule>; artifact: string; log: Array<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
export interface TrainFixture { repo: FixtureRepo; state: string; dataFile: string; env: Record<string, string>; read: () => ShimData; write: (fn: (d: ShimData) => void) => void }
|
||||
|
||||
export function trainFixture(policy: Record<string, unknown> = TRAIN_POLICY, prefix = 'gstack-train-'): TrainFixture {
|
||||
const repo = makeFixtureRepo({ policy, prefix });
|
||||
const dir = path.join(repo.root, 'gh-train-shim');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const dataFile = path.join(dir, 'data.json');
|
||||
const data: ShimData = { repo: 'fixture/repo', origin: repo.origin, nextId: 100, prs: {}, runs: [], artifacts: {}, checks: {}, ci: {}, rules: {}, artifact: TRAIN_BLOCK.record_artifact, log: [] };
|
||||
fs.writeFileSync(dataFile, JSON.stringify(data));
|
||||
fs.writeFileSync(path.join(dir, 'shim.js'), shimSource());
|
||||
fs.writeFileSync(path.join(dir, 'gh'), `#!/bin/sh\nexec node "${path.join(dir, 'shim.js')}" "${dataFile}" "$@"\n`, { mode: 0o755 });
|
||||
const state = path.join(repo.root, 'state');
|
||||
const env = { PATH: `${dir}:${process.env.PATH ?? ''}`, GSTACK_STATE_ROOT: state, GSTACK_SHIP_QUEUE_POLL_MS: '20', GSTACK_EVAL_BG_POLL_MS: '20', GSTACK_SHIP_QUEUE_NOW: '2026-10-18T12:00:00Z' };
|
||||
const read = () => JSON.parse(fs.readFileSync(dataFile, 'utf8')) as ShimData;
|
||||
const write = (fn: (d: ShimData) => void) => { const d = read(); fn(d); fs.writeFileSync(dataFile, JSON.stringify(d)); };
|
||||
return { repo, state, dataFile, env, read, write };
|
||||
}
|
||||
|
||||
/** The repo key lib/ship-queue-train-record.ts derives from origin's URL. */
|
||||
export function repoKey(fx: TrainFixture): string {
|
||||
const norm = fx.repo.origin.replace(/\.git$/, '').replace(/\/+$/, '');
|
||||
return createHash('sha256').update(norm).digest('hex').slice(0, 16);
|
||||
}
|
||||
|
||||
/** A same-repo PR: a branch from origin/main touching `file`, published as refs/pull/<n>/head (and /merge), known to the gh shim. */
|
||||
export function openPr(fx: TrainFixture, pr: string, opts: { file?: string; text?: string; changelog?: boolean; cross?: boolean; from?: string } = {}): string {
|
||||
const { repo } = fx;
|
||||
repo.git('checkout', '-q', opts.from ?? 'origin/main', '-B', `pr-${pr}`);
|
||||
const file = opts.file ?? `pr-${pr}.txt`;
|
||||
fs.mkdirSync(path.dirname(path.join(repo.work, file)), { recursive: true });
|
||||
fs.writeFileSync(path.join(repo.work, file), opts.text ?? `change from #${pr}\n`);
|
||||
if (opts.changelog !== false) {
|
||||
const cl = fs.readFileSync(path.join(repo.work, 'CHANGELOG.md'), 'utf8');
|
||||
fs.writeFileSync(path.join(repo.work, 'CHANGELOG.md'), cl.replace('# Changelog\n', `# Changelog\n\n## [Unreleased]\n\n- #${pr} change\n`));
|
||||
}
|
||||
repo.git('add', '-A');
|
||||
repo.git('commit', '-q', '-m', `feat #${pr}`);
|
||||
const head = repo.git('rev-parse', 'HEAD');
|
||||
repo.git('push', '-q', '-f', 'origin', `HEAD:refs/pull/${pr}/head`, `HEAD:refs/pull/${pr}/merge`);
|
||||
fx.write(d => { d.prs[pr] = { headRefOid: head, isCrossRepository: !!opts.cross, state: 'OPEN', headRefName: `pr-${pr}`, url: `https://github.com/fixture/repo/pull/${pr}`, body: '' }; });
|
||||
return head;
|
||||
}
|
||||
|
||||
/** A READY receipt as `ready --screen|--no-screen` writes it (the lane side is tested separately). */
|
||||
export function readyReceipt(fx: TrainFixture, pr: string, head: string, opts: { screen?: boolean; at?: string } = {}): void {
|
||||
const key = repoKey(fx);
|
||||
const file = path.join(fx.state, 'ship-queue', 'ready', key, `${pr}-${head}.json`);
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
const at = opts.at ?? '2026-10-18T08:00:00.000Z';
|
||||
fs.writeFileSync(file, JSON.stringify({ schema: 1, kind: 'ready', repo_key: key, pr, head, base_sha: fx.repo.git('rev-parse', 'origin/main'), ready_at: at,
|
||||
screen: { choice: opts.screen ? 'screen' : 'no-screen', policy: opts.screen ? 'narrow' : null, nonce: null, run: null, merge_sha: null }, updated_at: at }));
|
||||
}
|
||||
|
||||
export const PASS = (id: string, kind = 'rule', cost = 0.7) => ({ case: id, kind, status: 'PASS', red_class: null, cost_usd: cost });
|
||||
export const FAIL = (id: string, cls = 'VERDICT', cost = 0.9) => ({ case: id, kind: 'rule', status: 'FAIL', red_class: cls, cost_usd: cost });
|
||||
export const COMPLETE = { complete: true, planned: { rule: 2, judge: 1 }, executed: { rule: 2, judge: 1 }, unbound: 0, partial_shards: 0, cancelled_shards: 0, killed_before_upload: 0 };
|
||||
export const GREEN_GATE: Rule = { record: { panels: [PASS('c1'), PASS('c2'), PASS('j1', 'judge', 0.1)], census: COMPLETE, verdict: 'GREEN' }, check: 'success' };
|
||||
export const RED_GATE: Rule = { conclusion: 'failure', record: { panels: [FAIL('c1'), PASS('c2'), PASS('j1', 'judge', 0.1)], census: COMPLETE, verdict: 'RED' }, check: null };
|
||||
|
||||
export function queue(fx: TrainFixture, args: string[], extraEnv: Record<string, string> = {}, cwd = fx.repo.work) {
|
||||
return runBin(fx.repo, 'gstack-ship-queue', [...args, '--base', 'main'], { ...fx.env, ...extraEnv }, cwd);
|
||||
}
|
||||
|
||||
export const line = (out: string, prefix: string) => out.split('\n').find(l => l.startsWith(prefix)) ?? '';
|
||||
export const nonceOf = (out: string) => /^TRAIN: (\S+)/m.exec(out)?.[1] ?? '';
|
||||
export function trainJson(fx: TrainFixture, nonce: string, state = fx.state): any {
|
||||
return JSON.parse(fs.readFileSync(path.join(state, 'ship-queue', 'trains', nonce, 'train.json'), 'utf8'));
|
||||
}
|
||||
export const dispatches = (fx: TrainFixture, workflow?: string) => fx.read().log.filter(l => l.cmd === 'workflow run' && (!workflow || l.workflow === workflow));
|
||||
|
||||
function shimSource(): string {
|
||||
return [
|
||||
"const fs = require('fs'); const path = require('path'); const cp = require('child_process');",
|
||||
"const file = process.argv[2]; const a = process.argv.slice(3); const d = JSON.parse(fs.readFileSync(file, 'utf8'));",
|
||||
"const save = () => fs.writeFileSync(file, JSON.stringify(d));",
|
||||
"const opt = n => { const i = a.indexOf(n); return i >= 0 ? a[i + 1] : undefined; };",
|
||||
"const out = v => { console.log(JSON.stringify(v)); save(); process.exit(0); };",
|
||||
"const fail = m => { console.error(m); save(); process.exit(1); };",
|
||||
"const rev = r => { const x = cp.spawnSync('git', ['--git-dir', d.origin, 'rev-parse', '--verify', '--quiet', r], { encoding: 'utf8', timeout: 30000 }); return x.status === 0 ? x.stdout.trim() : ''; };",
|
||||
"d.log.push({ cmd: a.slice(0, 2).join(' '), args: a });",
|
||||
"if (a[0] === 'repo' && a[1] === 'view') out({ nameWithOwner: d.repo });",
|
||||
"if (a[0] === 'pr' && a[1] === 'view') { const p = d.prs[a[2]]; if (!p) fail('no pull request found'); out(p); }",
|
||||
"if (a[0] === 'workflow' && a[1] === 'run') {",
|
||||
" const w = a[2]; const ref = opt('--ref'); const inputs = {};",
|
||||
" a.forEach((x, i) => { if (x === '-f') { const kv = a[i + 1]; const k = kv.slice(0, kv.indexOf('=')); inputs[k] = kv.slice(kv.indexOf('=') + 1); } });",
|
||||
" Object.assign(d.log[d.log.length - 1], { workflow: w, ref, inputs });",
|
||||
" const rule = d.rules[w] || {};",
|
||||
" if (rule.fail) fail('HTTP 422: dispatch refused');",
|
||||
" if (rule.silent) { save(); process.exit(0); }",
|
||||
" const id = d.nextId++; const head = rule.headSha || inputs.expected_sha || rev('refs/heads/' + ref) || rev(ref);",
|
||||
" const run = { databaseId: id, workflow: w, headBranch: ref, headSha: head, status: rule.status || 'completed', conclusion: rule.conclusion === undefined ? 'success' : rule.conclusion,",
|
||||
" url: 'https://example.invalid/runs/' + id, event: rule.event || 'workflow_dispatch', displayTitle: inputs.nonce ? 'E2E Evals [eval-bg ' + inputs.nonce + ']' : w, inputs };",
|
||||
" d.runs.push(run); d.artifacts[id] = {};",
|
||||
" if (rule.record && !inputs.prefix_shas) d.artifacts[id][d.artifact] = { schema: 1, kind: 'train-gate', nonce: rule.record.nonce || inputs.nonce, head_sha: head, tree: rev(head + '^{tree}'),",
|
||||
" verdict: rule.record.verdict || 'GREEN', census: rule.record.census, panels: rule.record.panels };",
|
||||
" if (rule.check && !inputs.prefix_shas) (d.checks[head] = d.checks[head] || []).push({ name: 'gstack/full-gate', status: 'completed', conclusion: rule.check });",
|
||||
" if (rule.bisect && inputs.prefix_shas) for (const sha of inputs.prefix_shas.split(',')) {",
|
||||
" const at = rule.bisect.prefixes.indexOf(sha);",
|
||||
" d.artifacts[id][d.artifact + '-bisect-' + sha.slice(0, 12)] = { schema: 1, kind: 'train-bisect-prefix', nonce: inputs.nonce, prefix_sha: sha, checked_out_sha: sha,",
|
||||
" panels: inputs.cases.split(',').map(c => ({ case: c, kind: 'rule', status: rule.bisect.patterns[c][at] === 'R' ? 'FAIL' : 'PASS', red_class: rule.bisect.patterns[c][at] === 'R' ? 'VERDICT' : null, cost_usd: rule.bisect.costUsd ?? 0.5 })) };",
|
||||
" }",
|
||||
" if (rule.failButCreate) fail('HTTP 502: gateway timeout');",
|
||||
" save(); process.exit(0);",
|
||||
"}",
|
||||
"if (a[0] === 'run' && a[1] === 'list') {",
|
||||
" const w = opt('--workflow'); const br = opt('--branch'); const ev = opt('--event');",
|
||||
" out(d.runs.filter(r => r.workflow === w && (!br || r.headBranch === br) && (!ev || r.event === ev)).sort((x, y) => y.databaseId - x.databaseId).slice(0, Number(opt('--limit') || 20)));",
|
||||
"}",
|
||||
"if (a[0] === 'run' && a[1] === 'view') { const r = d.runs.find(x => String(x.databaseId) === a[2]); if (!r) fail('run not found'); out(r); }",
|
||||
"if (a[0] === 'run' && a[1] === 'download') {",
|
||||
" const art = (d.artifacts[a[2]] || {})[opt('-n')]; if (!art) fail('no artifact matches');",
|
||||
" fs.mkdirSync(opt('-D'), { recursive: true }); fs.writeFileSync(path.join(opt('-D'), 'record.json'), JSON.stringify(art)); save(); process.exit(0);",
|
||||
"}",
|
||||
"if (a[0] === 'api') {",
|
||||
" const named = /commits\\/([0-9a-f]+)\\/check-runs\\?check_name=([^&]+)/.exec(a[1] || '');",
|
||||
" if (named) { const list = (d.checks[named[1]] || []).filter(c => c.name === decodeURIComponent(named[2])); out({ total_count: list.length, check_runs: list }); }",
|
||||
" const paged = /commits\\/([0-9a-f]+)\\/(check-runs|status)\\?.*page=(\\d+)/.exec(a[1] || '');",
|
||||
" if (paged) { const list = paged[2] === 'check-runs' ? (d.ci[paged[1]] || []) : []; const key = paged[2] === 'check-runs' ? 'check_runs' : 'statuses';",
|
||||
" out(paged[3] === '1' ? { total_count: list.length, state: 'success', [key]: list } : { total_count: list.length, [key]: [] }); }",
|
||||
"}",
|
||||
"fail('gh shim: unsupported ' + a.join(' '));",
|
||||
].join('\n') + '\n';
|
||||
}
|
||||
@@ -9,12 +9,14 @@
|
||||
import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import { EXIT } from '../lib/headless-artifacts';
|
||||
import { DEFAULT_POLICY, POLICY_KEY_INTRODUCED, SHIP_QUEUE_KEYS_VERSION, validatePolicy } from '../lib/ship-policy';
|
||||
import { cmpVersion, parseVersion } from '../lib/version-source';
|
||||
import { BASE_POLICY, cleanup, makeFixtureRepo, restamp, runBin, type FixtureRepo } from './helpers/restamp-fixture';
|
||||
|
||||
const repos: FixtureRepo[] = [];
|
||||
const fixture = (policy: Record<string, unknown>) => { const r = makeFixtureRepo({ policy }); repos.push(r); return r; };
|
||||
afterEach(() => { while (repos.length) cleanup(repos.pop()!); });
|
||||
|
||||
const newestKeyVersion = Object.values(POLICY_KEY_INTRODUCED).reduce((a, b) => (cmpVersion(parseVersion(b)!, parseVersion(a)!) > 0 ? b : a));
|
||||
const codes = (raw: unknown, gstackVersion = '1.91.200.0') => validatePolicy(raw, { gstackVersion }).map(e => `${e.code}@${e.path}`);
|
||||
|
||||
describe('policy compatibility floor', () => {
|
||||
@@ -35,7 +37,7 @@ describe('policy compatibility floor', () => {
|
||||
expect(codes({ gate_command: 'x', min_gstack_version: SHIP_QUEUE_KEYS_VERSION }, '1.91.106.0')).toEqual(['POLICY_GSTACK_TOO_OLD@min_gstack_version']);
|
||||
const future = validatePolicy({ gate_reuse_requires: ['e2e'] }, { gstackVersion: '1.91.200.0' });
|
||||
expect(future.map(e => `${e.code}@${e.path}`)).toEqual(['POLICY_INVALID@gate_reuse_requires']);
|
||||
expect(future[0]!.message).toContain(`this gstack reads keys introduced up to ${SHIP_QUEUE_KEYS_VERSION}`);
|
||||
expect(future[0]!.message).toContain(`this gstack reads keys introduced up to ${newestKeyVersion}`);
|
||||
expect(codes({ min_gstack_version: 'soon' })).toEqual(['POLICY_INVALID@min_gstack_version']);
|
||||
expect(codes({ full_e2e: 'yes', min_gstack_version: SHIP_QUEUE_KEYS_VERSION })).toEqual(['POLICY_INVALID@full_e2e']);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
/**
|
||||
* Merge trains (plan GSTA-49 lane T1), the pure decisions: slot order and
|
||||
* fill, every paid-report class mapped to a train verdict, bisect pattern
|
||||
* attribution, the rebuild and re-dispatch allowances pinned to EVAL_POLICY,
|
||||
* the screen recommendation, the `train` policy block and eval-bg's nonce
|
||||
* key with purpose and screen policy.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { DISPATCH_INPUTS, dedupeKey, dispatchPlan, findDuplicateRun, makeNonce, nonceScope, type RunSummary } from '../scripts/eval-bg';
|
||||
import { DEFAULT_POLICY, TRAIN_KEYS_VERSION, validatePolicy } from '../lib/ship-policy';
|
||||
import { fillDue, orderSlots, parsePins } from '../lib/ship-queue-train';
|
||||
import { gstackPaidSelection, skillOwnCases, touchedSkills } from '../lib/ship-queue-train-ready';
|
||||
import type { TrainSlot } from '../lib/ship-queue-train-record';
|
||||
import { INFRA_REDISPATCH, REBUILD_ONCE, classifyBisect, classifyGate } from '../lib/ship-queue-train-verdict';
|
||||
import { EVAL_POLICY } from './helpers/periodic-exclude-data';
|
||||
import { COMPLETE, FAIL, PASS, TRAIN_BLOCK } from './helpers/ship-queue-train-fixture';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const SHA = 'a'.repeat(40);
|
||||
const slot = (pr: string, screened: boolean, at: string): TrainSlot => ({ pr, head: SHA, ready_at: at, screen: { choice: screened ? 'screen' : 'no-screen', policy: null, nonce: null, run: null, merge_sha: null } });
|
||||
|
||||
describe('ordering and fill', () => {
|
||||
test('screened PRs go first in READY order, then unscreened; the order is stable on ties', () => {
|
||||
const o = orderSlots([slot('4', false, '2026-10-18T01:00:00Z'), slot('3', true, '2026-10-18T03:00:00Z'), slot('2', false, '2026-10-18T00:00:00Z'), slot('1', true, '2026-10-18T02:00:00Z')], 5);
|
||||
expect(o.train.map(s => s.pr)).toEqual(['1', '3', '2', '4']);
|
||||
expect(o.held).toEqual([]);
|
||||
expect(o.split).toBe(false);
|
||||
});
|
||||
|
||||
test('when as many unscreened PRs wait as fill a train, they form their own train; over-size PRs are held in order', () => {
|
||||
const unscreened = ['5', '6', '7'].map((pr, i) => slot(pr, false, `2026-10-18T0${i}:00:00Z`));
|
||||
const split = orderSlots([slot('9', true, '2026-10-18T05:00:00Z'), ...unscreened], 3);
|
||||
expect(split.split).toBe(true);
|
||||
expect(split.train.map(s => s.pr)).toEqual(['9']);
|
||||
expect(split.held.map(s => s.pr)).toEqual(['5', '6', '7']);
|
||||
const over = orderSlots([slot('8', true, '2026-10-18T06:00:00Z'), slot('9', true, '2026-10-18T05:00:00Z'), unscreened[0]!], 2);
|
||||
expect(over.split).toBe(false);
|
||||
expect(over.train.map(s => s.pr)).toEqual(['9', '8']);
|
||||
expect(over.held.map(s => s.pr)).toEqual(['5']);
|
||||
});
|
||||
|
||||
test('a train under its size dispatches once its oldest READY has waited fill_hours; full or urgent dispatches at once', () => {
|
||||
const slots = [slot('1', false, '2026-10-18T09:00:00Z'), slot('2', false, '2026-10-18T10:00:00Z')];
|
||||
expect(fillDue(slots, TRAIN_BLOCK as any, '2026-10-18T11:59:00Z', false)).toBe('2026-10-18T12:00:00.000Z');
|
||||
expect(fillDue(slots, TRAIN_BLOCK as any, '2026-10-18T12:00:00Z', false)).toBeNull();
|
||||
expect(fillDue(slots, TRAIN_BLOCK as any, '2026-10-18T09:30:00Z', true)).toBeNull();
|
||||
expect(fillDue(slots, { ...TRAIN_BLOCK, max_size: 2 } as any, '2026-10-18T09:30:00Z', false)).toBeNull();
|
||||
});
|
||||
|
||||
test('pins are <pr>@<sha>, each PR once', () => {
|
||||
expect(parsePins(`12@${SHA},#13@abcdef1`)).toEqual([{ pr: '12', sha: SHA }, { pr: '13', sha: 'abcdef1' }]);
|
||||
expect(parsePins('12')).toMatch(/entries are <pr>@<sha>/);
|
||||
expect(parsePins(`12@${SHA},12@${SHA}`)).toBe('a PR appears twice in --prs');
|
||||
});
|
||||
});
|
||||
|
||||
describe('verdict class mapping', () => {
|
||||
const expectIds = { nonce: 'n1', head: SHA, tree: 'b'.repeat(40) };
|
||||
const record = (over: Record<string, unknown> = {}) => ({ schema: 1, kind: 'train-gate', nonce: 'n1', head_sha: SHA, tree: 'b'.repeat(40), verdict: 'GREEN', census: COMPLETE, panels: [PASS('c1'), PASS('j1', 'judge')], ...over });
|
||||
const done: { status: string; conclusion: string | null } = { status: 'completed', conclusion: 'success' };
|
||||
const check = { status: 'completed', conclusion: 'success' };
|
||||
const classify = (rec: unknown, run = done, chk: typeof check | null = check) => classifyGate(rec, expectIds, run, chk);
|
||||
|
||||
test('GREEN only for a complete, bound census on a successful run that published its check', () => {
|
||||
const g = classify(record());
|
||||
expect(g.outcome).toBe('GREEN');
|
||||
expect(g.detail).toBe('rule 2/2, judge 1/1');
|
||||
expect(classify(record(), done, null).outcome).toBe('INCOMPLETE');
|
||||
expect(classify(record(), done, { status: 'completed', conclusion: 'failure' }).detail).toContain('did not publish a successful check run');
|
||||
});
|
||||
|
||||
test('a run still going is PENDING; no record is INCOMPLETE and eligible for the one INFRA re-dispatch', () => {
|
||||
expect(classify(record(), { status: 'in_progress', conclusion: null }).outcome).toBe('PENDING');
|
||||
const none = classify(null, { status: 'completed', conclusion: 'cancelled' });
|
||||
expect([none.outcome, none.infraEligible]).toEqual(['INCOMPLETE', true]);
|
||||
expect(none.detail).toContain('a slice killed before its upload step');
|
||||
});
|
||||
|
||||
test('a VERDICT FAIL is RED with its cases, even beside census gaps', () => {
|
||||
const r = classify(record({ verdict: 'RED', panels: [FAIL('c2'), FAIL('c1'), PASS('j1')], census: { ...COMPLETE, cancelled_shards: 1 } }), { status: 'completed', conclusion: 'failure' });
|
||||
expect([r.outcome, r.red]).toEqual(['RED', ['c1', 'c2']]);
|
||||
expect(r.cost).toBe(2.5);
|
||||
});
|
||||
|
||||
test('INFRA and INCOMPLETE panels, unbound, partial or cancelled shards and killed slices are INCOMPLETE, never green', () => {
|
||||
const cases: Array<[Record<string, unknown>, boolean, RegExp]> = [
|
||||
[{ panels: [FAIL('c1', 'INFRA'), PASS('j1')] }, true, /c1 FAIL \(INFRA\)/],
|
||||
[{ panels: [{ ...PASS('c1'), status: 'INCOMPLETE', red_class: 'INCOMPLETE' }, PASS('j1')] }, true, /c1 INCOMPLETE/],
|
||||
[{ panels: [{ ...PASS('c1'), status: 'SKIPPED' }, PASS('j1')] }, false, /c1 SKIPPED/],
|
||||
[{ census: { ...COMPLETE, unbound: 1 } }, true, /1 unbound result/],
|
||||
[{ census: { ...COMPLETE, partial_shards: 2 } }, true, /2 partial shard/],
|
||||
[{ census: { ...COMPLETE, killed_before_upload: 1 } }, true, /slice killed before upload/],
|
||||
[{ census: { ...COMPLETE, executed: { rule: 1, judge: 1 } } }, true, /rule 1\/2 executed/],
|
||||
[{ census: { ...COMPLETE, planned: { rule: 0, judge: 0 }, executed: { rule: 0, judge: 0 } } }, true, /lists no cases/],
|
||||
];
|
||||
for (const [over, eligible, why] of cases) {
|
||||
const r = classify(record(over));
|
||||
expect([r.outcome, r.infraEligible]).toEqual(['INCOMPLETE', eligible]);
|
||||
expect(r.detail).toMatch(why);
|
||||
}
|
||||
});
|
||||
|
||||
test('a record bound to another nonce, head or tree, or a failed run with a green census, is INCOMPLETE and not eligible', () => {
|
||||
for (const over of [{ nonce: 'other' }, { head_sha: 'c'.repeat(40) }, { tree: 'd'.repeat(40) }]) {
|
||||
const r = classify(record(over));
|
||||
expect([r.outcome, r.infraEligible]).toEqual(['INCOMPLETE', false]);
|
||||
expect(r.detail).toContain('not this train');
|
||||
}
|
||||
expect(classify(record(), { status: 'completed', conclusion: 'failure' }).detail).toContain('run concluded failure with record verdict GREEN');
|
||||
});
|
||||
|
||||
test('the one INFRA re-dispatch and the one rebuild are pinned to EVAL_POLICY', () => {
|
||||
expect(INFRA_REDISPATCH).toBe(EVAL_POLICY.infraRedispatch);
|
||||
expect(REBUILD_ONCE).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('bisect attribution', () => {
|
||||
const slots = [{ pr: '11' }, { pr: '12' }, { pr: '13' }];
|
||||
const results = (patterns: Record<string, string>) => [0, 1, 2, 3].map(i => Object.fromEntries(Object.entries(patterns).map(([c, p]) => [c, p[i] === 'R' ? 'red' as const : 'green' as const])));
|
||||
|
||||
test('monotone green-then-red blames the PR that starts the red, the last PR included', () => {
|
||||
expect(classifyBisect(['c1'], results({ c1: 'GGRR' }), slots)).toEqual({ kind: 'attributed', all_green: false, prs: ['12'], patterns: { c1: 'GGRR' } });
|
||||
expect(classifyBisect(['c1'], results({ c1: 'GGGR' }), slots).prs).toEqual(['13']);
|
||||
expect(classifyBisect(['c1', 'c2'], results({ c1: 'GRRR', c2: 'GGGR' }), slots).prs).toEqual(['11', '13']);
|
||||
expect(classifyBisect(['c1', 'c2'], results({ c1: 'GRRR', c2: 'GGGG' }), slots).kind).toBe('attributed');
|
||||
});
|
||||
|
||||
test('red on every prefix, T0 included, is main; all green and non-monotone are unattributed', () => {
|
||||
expect(classifyBisect(['c1'], results({ c1: 'RRRR' }), slots).kind).toBe('red-on-main');
|
||||
expect(classifyBisect(['c1', 'c2'], results({ c1: 'RRRR', c2: 'GGRR' }), slots).kind).toBe('red-on-main');
|
||||
expect(classifyBisect(['c1'], results({ c1: 'GGGG' }), slots)).toMatchObject({ kind: 'unattributed', all_green: true, prs: [] });
|
||||
expect(classifyBisect(['c1'], results({ c1: 'GRGR' }), slots)).toMatchObject({ kind: 'unattributed', all_green: false });
|
||||
expect(classifyBisect(['c1'], results({ c1: 'RGGG' }), slots)).toMatchObject({ kind: 'unattributed', all_green: false });
|
||||
expect(classifyBisect(['c1', 'c2'], results({ c1: 'GGRR', c2: 'GRGR' }), slots)).toMatchObject({ kind: 'unattributed', all_green: false });
|
||||
});
|
||||
});
|
||||
|
||||
describe('screen recommendation', () => {
|
||||
test('a skill directory is touched when the diff changes a file under a directory holding SKILL.md.tmpl', () => {
|
||||
expect(touchedSkills(['review/SKILL.md.tmpl', 'review/x.md', 'lib/a.ts', 'README.md'], d => d === 'review')).toEqual(['review']);
|
||||
expect(skillOwnCases(['a', 'b', 'c'], { a: ['review/**'], b: ['lib/**'], c: ['review/SKILL.md'] }, ['review'])).toEqual(['a', 'c']);
|
||||
});
|
||||
|
||||
test("gstack's selector on gstack's own tree: a skill template change recommends --screen, a docs change --no-screen", async () => {
|
||||
const has = (dir: string) => fs.existsSync(path.join(ROOT, dir, 'SKILL.md.tmpl'));
|
||||
const skill = await gstackPaidSelection(ROOT, 'HEAD', ['review/SKILL.md.tmpl'], has);
|
||||
expect(skill.recommend).toBe('screen');
|
||||
expect(skill.lines[0]).toMatch(/^SCREEN: selection \d+ e2e case\(s\) \+ \d+ judge\(s\)/);
|
||||
expect(skill.lines[1]).toMatch(/^SCREEN: skill's own cases \S+ → recommend --screen$/);
|
||||
const docs = await gstackPaidSelection(ROOT, 'HEAD', ['docs/ship-policy.md'], has);
|
||||
expect(docs.recommend).toBe('no-screen');
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
describe('the train policy block', () => {
|
||||
const ok = { train: TRAIN_BLOCK, min_gstack_version: TRAIN_KEYS_VERSION };
|
||||
const codes = (raw: unknown) => validatePolicy(raw, { gstackVersion: '1.91.200.0' }).map(e => `${e.code}@${e.path}`);
|
||||
|
||||
test('defaults keep it null; a full block validates with min_gstack_version at its introducing version', () => {
|
||||
expect(DEFAULT_POLICY.train).toBeNull();
|
||||
expect(codes(ok)).toEqual([]);
|
||||
expect(codes({ train: TRAIN_BLOCK })).toEqual(['POLICY_INVALID@train']);
|
||||
});
|
||||
|
||||
test('unknown, missing and out-of-range fields fail closed', () => {
|
||||
expect(codes({ ...ok, train: { ...TRAIN_BLOCK, colour: 'red' } })).toEqual(['POLICY_INVALID@train.colour']);
|
||||
const { fill_hours: _drop, ...missing } = TRAIN_BLOCK;
|
||||
expect(codes({ ...ok, train: missing })).toEqual(['POLICY_INVALID@train.fill_hours']);
|
||||
expect(codes({ ...ok, train: { ...TRAIN_BLOCK, max_size: 7, gate_workflow: '../evals.yml', screen_selection: 'all' } }))
|
||||
.toEqual(['POLICY_INVALID@train.gate_workflow', 'POLICY_INVALID@train.max_size', 'POLICY_INVALID@train.screen_selection']);
|
||||
expect(codes({ ...ok, train: [] })).toEqual(['POLICY_INVALID@train']);
|
||||
});
|
||||
});
|
||||
|
||||
describe("eval-bg nonce key: purpose and screen policy", () => {
|
||||
const base = 'c'.repeat(40);
|
||||
const run = (title: string): RunSummary => ({ databaseId: 1, displayTitle: title, status: 'in_progress', conclusion: null, url: 'u', event: 'workflow_dispatch' });
|
||||
|
||||
test('a plain lane key is unchanged; a scoped key carries purpose and policy, and parses back', () => {
|
||||
expect(dedupeKey('pr', SHA, base)).toBe(`pr-${SHA.slice(0, 12)}-${base.slice(0, 12)}-`);
|
||||
expect(makeNonce('pr', SHA, base, 'zz', { purpose: 'screen', policy: 'broad' })).toBe(`pr.screen.broad-${SHA.slice(0, 12)}-${base.slice(0, 12)}-zz`);
|
||||
expect(nonceScope(makeNonce('gate', SHA, base, 'q1', { purpose: 'train' }))).toEqual({ purpose: 'train' });
|
||||
expect(nonceScope(makeNonce('pr', SHA, base, 'q1', { purpose: 'screen', policy: 'narrow' }))).toEqual({ purpose: 'screen', policy: 'narrow' });
|
||||
expect(nonceScope(makeNonce('pr', SHA, base, 'q1'))).toBeNull();
|
||||
});
|
||||
|
||||
test('a broad screen never adopts a narrow run, a screen never adopts a plain PR lane run, and a train never adopts a bisect', () => {
|
||||
const narrow = makeNonce('pr', SHA, base, 'n1', { purpose: 'screen', policy: 'narrow' });
|
||||
const plain = makeNonce('pr', SHA, base, 'p1');
|
||||
const bisect = makeNonce('gate', SHA, base, 'b1', { purpose: 'bisect' });
|
||||
const runs = [run(`E2E Evals [eval-bg ${narrow}]`), run(`E2E Evals [eval-bg ${plain}]`), run(`E2E Evals [eval-bg ${bisect}]`)];
|
||||
expect(findDuplicateRun(runs, dedupeKey('pr', SHA, base, { purpose: 'screen', policy: 'broad' }))).toBeNull();
|
||||
expect(findDuplicateRun([runs[1]!], dedupeKey('pr', SHA, base, { purpose: 'screen', policy: 'narrow' }))).toBeNull();
|
||||
expect(findDuplicateRun(runs, dedupeKey('pr', SHA, base, { purpose: 'screen', policy: 'narrow' }))?.displayTitle).toContain(narrow);
|
||||
expect(findDuplicateRun([runs[2]!], dedupeKey('gate', SHA, base, { purpose: 'train' }))).toBeNull();
|
||||
});
|
||||
|
||||
test('every dispatch input name lives in DISPATCH_INPUTS, and the lane plans still send the same inputs', () => {
|
||||
expect(DISPATCH_INPUTS).toMatchObject({ cases: 'cases', prefixShas: 'prefix_shas', screenPr: 'screen_pr', screenMergeSha: 'screen_merge_sha', screenPolicy: 'screen_policy' });
|
||||
expect(dispatchPlan('gate', { sha: SHA, baseRef: 'main', baseSha: base, nonce: 'n' })).toEqual([{ workflow: 'evals.yml', inputs: { evals_all: 'true', expected_sha: SHA, nonce: 'n' } }]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,524 @@
|
||||
/**
|
||||
* Merge trains through the real bin/gstack-ship-queue (plan GSTA-49 lane T1),
|
||||
* headless against fixture repos outside the checkout (bare origin, lane and
|
||||
* coordinator clones, a hermetic git identity) with a stateful gh shim:
|
||||
* build (happy path, CHANGELOG union, conflict, fork, moved head, READY,
|
||||
* fill, free tests red/missing/stale/cancelled, dry run, ordering), dispatch
|
||||
* recovery (partial and absent dispatches), status rebuilt from
|
||||
* refs/notes/gstack-trains on a clean state root, verdict, bisect
|
||||
* (monotone, non-monotone, all green, red on main, capped and resumed
|
||||
* without repeats, prefix ancestry), rebuild-once and the INFRA
|
||||
* re-dispatch, verify (tree equality, mismatch, freeze, repair and clear),
|
||||
* prune, ready's screen choice, and the policy gate.
|
||||
*/
|
||||
import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { EXIT } from '../lib/headless-artifacts';
|
||||
import { TRAIN_HELP } from '../lib/ship-queue-train-cli';
|
||||
import { BASE_POLICY, cleanup, runBin } from './helpers/restamp-fixture';
|
||||
import { coordinatorClone } from './helpers/ship-queue-fixture';
|
||||
import {
|
||||
COMPLETE, FAIL, GREEN_GATE, PASS, RED_GATE, TRAIN_BLOCK, TRAIN_POLICY, dispatches, line, nonceOf, openPr, queue, readyReceipt, repoKey, trainFixture, trainJson,
|
||||
type Rule, type TrainFixture,
|
||||
} from './helpers/ship-queue-train-fixture';
|
||||
|
||||
const fixtures: TrainFixture[] = [];
|
||||
const make = (policy: Record<string, unknown> = TRAIN_POLICY) => { const f = trainFixture(policy); fixtures.push(f); return f; };
|
||||
afterEach(() => { while (fixtures.length) cleanup(fixtures.pop()!.repo); });
|
||||
|
||||
const result = (out: string) => line(out, 'GSTACK_RESULT:');
|
||||
const next = (out: string) => line(out, 'NEXT:');
|
||||
const remoteRefs = (fx: TrainFixture) => fx.repo.git('ls-remote', 'origin');
|
||||
|
||||
function twoPrTrain(fx: TrainFixture, gate: Rule = GREEN_GATE) {
|
||||
const h1 = openPr(fx, '1');
|
||||
const h2 = openPr(fx, '2');
|
||||
readyReceipt(fx, '1', h1);
|
||||
readyReceipt(fx, '2', h2);
|
||||
fx.write(d => { d.rules['evals.yml'] = gate; });
|
||||
const r = queue(fx, ['train', 'build', '--prs', `1@${h1},2@${h2}`]);
|
||||
return { h1, h2, r, nonce: nonceOf(r.out) };
|
||||
}
|
||||
|
||||
/** A main commit with a prefix's tree plus a VERSION stamp, the way stamp-at-merge publishes it. */
|
||||
function publishMain(fx: TrainFixture, prefixCommit: string, version: string, extra?: { file: string; text: string }): string {
|
||||
const { repo } = fx;
|
||||
repo.git('checkout', '-q', '-B', 'mainsim', 'origin/main');
|
||||
repo.git('rm', '-rq', '.');
|
||||
repo.git('checkout', prefixCommit, '--', '.');
|
||||
fs.writeFileSync(path.join(repo.work, 'VERSION'), version + '\n');
|
||||
if (extra) fs.writeFileSync(path.join(repo.work, extra.file), extra.text);
|
||||
repo.git('add', '-A');
|
||||
repo.git('commit', '-q', '-m', `merge to ${version}`);
|
||||
repo.git('push', '-q', 'origin', 'HEAD:refs/heads/main');
|
||||
repo.git('fetch', '-q', 'origin');
|
||||
return repo.git('rev-parse', 'HEAD');
|
||||
}
|
||||
|
||||
describe('the train command surface', () => {
|
||||
test('train with no subcommand (flags only) prints the train help and exits usage; --help exits 0; bad arguments name what is missing', () => {
|
||||
const fx = make();
|
||||
const bare = queue(fx, ['train']);
|
||||
expect([bare.status, bare.out]).toEqual([EXIT.usage, TRAIN_HELP + '\n']);
|
||||
expect(queue(fx, ['train', '--help'])).toMatchObject({ status: EXIT.ok, out: TRAIN_HELP + '\n' });
|
||||
const noPins = queue(fx, ['train', 'build']);
|
||||
expect(noPins.status).toBe(EXIT.usage);
|
||||
expect(noPins.err).toContain('train build needs --prs <n>@<sha>[,...] or --train <nonce>');
|
||||
expect(queue(fx, ['train', 'build', '--prs', '1@abc', '--urgent', '--repair']).err).toContain('--urgent and --repair are separate one-PR trains; pass one');
|
||||
expect(fs.existsSync(path.join(fx.state, 'ship-queue'))).toBe(false);
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
describe('the policy gate', () => {
|
||||
test('without a train block the train surface refuses with QUEUE_TRAIN_OFF and ready prints exactly today\'s block', () => {
|
||||
const fx = make(BASE_POLICY);
|
||||
const status = queue(fx, ['train', 'status']);
|
||||
expect(status.status).toBe(EXIT.refused);
|
||||
expect(status.err).toContain('declares no train block; the per-PR queue applies');
|
||||
expect(status.err).toContain('(QUEUE_TRAIN_OFF)');
|
||||
expect(next(status.out)).toBe('NEXT: none');
|
||||
expect(result(status.out)).toBe('GSTACK_RESULT: skill=ship-queue status=refused run=none');
|
||||
expect(queue(fx, ['verify', '--pr', '1', '--train', 'x']).err).toContain('(QUEUE_TRAIN_OFF)');
|
||||
const head = openPr(fx, '1');
|
||||
fx.write(d => { d.ci[head] = [{ status: 'completed', conclusion: 'success' }]; });
|
||||
const tree = runBin(fx.repo, 'gstack-wtree', []).out.trim();
|
||||
const pregate = path.join(fx.repo.root, 'pregate.json');
|
||||
fs.writeFileSync(pregate, JSON.stringify({ schema_version: 1, tree, head, verdict: 'pass', checks: [{ id: 'unit', status: 'pass' }] }));
|
||||
const today = queue(fx, ['ready', '--pr', '1', '--pregate', pregate]);
|
||||
expect(today.status).toBe(EXIT.ok);
|
||||
expect(today.out.split('\n')[0]).toBe(`READY #1 ${head} STAMP_VERSION=unstamped`);
|
||||
expect(today.out).not.toContain('NEXT:');
|
||||
expect(today.out).not.toContain('screen');
|
||||
expect(result(today.out)).toBe('GSTACK_RESULT: skill=ship-queue status=complete run=.gstack/tmp/pregate.json');
|
||||
const flagged = queue(fx, ['ready', '--pr', '1', '--pregate', pregate, '--no-screen']);
|
||||
expect(flagged.out).toBe(today.out);
|
||||
expect(flagged.err).toContain('--screen/--no-screen/--broad ignored: the base policy declares no train block');
|
||||
expect(fs.existsSync(path.join(fx.state, 'ship-queue'))).toBe(false);
|
||||
}, 120_000);
|
||||
});
|
||||
|
||||
describe('ready under a train policy: the screen choice', () => {
|
||||
function readyFixture() {
|
||||
const fx = make();
|
||||
const head = openPr(fx, '1');
|
||||
fx.write(d => { d.ci[head] = [{ status: 'completed', conclusion: 'success' }]; });
|
||||
const tree = runBin(fx.repo, 'gstack-wtree', []).out.trim();
|
||||
const pregate = path.join(fx.repo.root, 'pregate.json');
|
||||
fs.writeFileSync(pregate, JSON.stringify({ schema_version: 1, tree, head, verdict: 'pass', checks: [{ id: 'unit', status: 'pass' }] }));
|
||||
return { fx, head, pregate };
|
||||
}
|
||||
|
||||
test('no choice refuses with the recommendation; --no-screen records a READY receipt in the block, the state root and the notes ref', () => {
|
||||
const { fx, head, pregate } = readyFixture();
|
||||
const none = queue(fx, ['ready', '--pr', '1', '--pregate', pregate]);
|
||||
expect(none.status).toBe(EXIT.refused);
|
||||
expect(none.err).toContain('(READY_SCREEN_CHOICE_REQUIRED)');
|
||||
expect(none.out).toContain('SCREEN: selection not computed (train.screen_selection is none)');
|
||||
expect(none.out).toContain('SCREEN: price pending the T0 cost replay (lib/gate-cost.ts)');
|
||||
expect(next(none.out)).toBe('NEXT: gstack-ship-queue ready --pr 1 --no-screen');
|
||||
const ok = queue(fx, ['ready', '--pr', '1', '--pregate', pregate, '--no-screen']);
|
||||
expect(ok.status).toBe(EXIT.ok);
|
||||
expect(ok.out.split('\n').slice(0, 2)).toEqual([`READY #1 ${head} STAMP_VERSION=unstamped`, 'screen: no-screen']);
|
||||
const receipt = path.join(fx.state, 'ship-queue', 'ready', repoKey(fx), `1-${head}.json`);
|
||||
expect(JSON.parse(fs.readFileSync(receipt, 'utf8'))).toMatchObject({ kind: 'ready', pr: '1', head, ready_at: '2026-10-18T12:00:00.000Z', screen: { choice: 'no-screen' } });
|
||||
expect(result(ok.out)).toBe(`GSTACK_RESULT: skill=ship-queue status=complete run=${receipt}`);
|
||||
expect(remoteRefs(fx)).toContain('refs/notes/gstack-trains');
|
||||
expect(dispatches(fx)).toEqual([]);
|
||||
}, 120_000);
|
||||
|
||||
test('--screen dispatches one screen with the PR, its merge SHA and the policy; a repeat adopts it; a broad screen never adopts the narrow one', () => {
|
||||
const { fx, head, pregate } = readyFixture();
|
||||
const dry = queue(fx, ['ready', '--pr', '1', '--pregate', pregate, '--screen', '--dry-run']);
|
||||
expect(dry.status).toBe(EXIT.ok);
|
||||
expect(line(dry.out, 'SCREEN_DISPATCH:')).toMatch(new RegExp(`^SCREEN_DISPATCH: evals\\.yml --ref main evals_all=false expected_sha=${head} nonce=pr\\.screen\\.narrow-\\S+ base_ref=main base_sha=[0-9a-f]{40} screen_pr=1 screen_merge_sha=${head} screen_policy=narrow \\(dry run: not dispatched\\)$`));
|
||||
expect(dispatches(fx)).toEqual([]);
|
||||
fx.write(d => { d.rules['evals.yml'] = { status: 'in_progress', conclusion: null }; });
|
||||
const first = queue(fx, ['ready', '--pr', '1', '--pregate', pregate, '--screen']);
|
||||
expect(first.status).toBe(EXIT.ok);
|
||||
expect(first.out.split('\n')[1]).toMatch(/^screen: screen policy=narrow nonce=pr\.screen\.narrow-\S+ run=100$/);
|
||||
const again = queue(fx, ['ready', '--pr', '1', '--pregate', pregate, '--screen']);
|
||||
expect(again.out).toContain('SCREEN_DISPATCH: following run 100 (same merge SHA, base and policy; no duplicate)');
|
||||
const broad = queue(fx, ['ready', '--pr', '1', '--pregate', pregate, '--screen', '--broad']);
|
||||
expect(broad.status).toBe(EXIT.ok);
|
||||
expect(dispatches(fx).map(d => (d.inputs as any).screen_policy)).toEqual(['narrow', 'broad']);
|
||||
expect(queue(fx, ['ready', '--pr', '1', '--broad']).status).toBe(EXIT.usage);
|
||||
}, 120_000);
|
||||
});
|
||||
|
||||
describe('train build', () => {
|
||||
test('happy path: slot order, prefix trees, CHANGELOG union, records before effects, free tests then one paid dispatch by nonce; resume and identical pins never dispatch again', () => {
|
||||
const fx = make();
|
||||
const { h1, h2, r, nonce } = twoPrTrain(fx);
|
||||
expect(r.status).toBe(EXIT.ok);
|
||||
expect(nonce).toMatch(/^gate\.train-[0-9a-f]{12}-[0-9a-f]{12}-[0-9a-f]{6}$/);
|
||||
const t = trainJson(fx, nonce);
|
||||
expect(t).toMatchObject({ schema: 1, kind: 'train', state: 'gate-pending', branch: `train/${nonce}`, slots: [{ pr: '1', head: h1 }, { pr: '2', head: h2 }] });
|
||||
expect(t.prefixes.map((p: any) => [p.index, p.pr])).toEqual([[0, null], [1, '1'], [2, '2']]);
|
||||
expect(t.prefixes[0].commit).toBe(fx.repo.git('rev-parse', 'origin/main'));
|
||||
expect(t.prefixes[2].commit).toBe(t.head);
|
||||
expect(r.out).toContain(`PREFIX: T2 ${t.head} tree=${t.tree} (+#2)`);
|
||||
expect(remoteRefs(fx)).toContain(`${t.head}\trefs/heads/train/${nonce}`);
|
||||
expect(remoteRefs(fx)).toContain('refs/notes/gstack-trains');
|
||||
const changelog = fx.repo.git('show', `${t.head}:CHANGELOG.md`);
|
||||
expect(changelog).toContain('- #1 change');
|
||||
expect(changelog).toContain('- #2 change');
|
||||
expect(fx.repo.git('show', `${t.head}:pr-1.txt`)).toBe('change from #1');
|
||||
const sent = dispatches(fx);
|
||||
expect(sent.map(d => [d.workflow, d.ref])).toEqual([['free-tests.yml', `train/${nonce}`], ['evals.yml', `train/${nonce}`]]);
|
||||
expect(sent[1]!.inputs).toEqual({ evals_all: 'true', expected_sha: t.head, nonce });
|
||||
expect(t.effects.map((e: any) => e.what)).toEqual(expect.arrayContaining(['train.json written', `branch pushed: train/${nonce} at ${t.head.slice(0, 12)}`]));
|
||||
expect(next(r.out)).toBe(`NEXT: gstack-ship-queue train verdict --train ${nonce}`);
|
||||
expect(result(r.out)).toBe(`GSTACK_RESULT: skill=ship-queue status=complete run=${path.join(fx.state, 'ship-queue', 'trains', nonce, 'train.json')}`);
|
||||
const resume = queue(fx, ['train', 'build', '--train', nonce]);
|
||||
expect(resume.status).toBe(EXIT.ok);
|
||||
const same = queue(fx, ['train', 'build', '--prs', `1@${h1},2@${h2}`]);
|
||||
expect(nonceOf(same.out)).toBe(nonce);
|
||||
expect(dispatches(fx).length).toBe(2);
|
||||
}, 180_000);
|
||||
|
||||
test('--dry-run prints order, trees and the estimate, and writes, pushes and dispatches nothing', () => {
|
||||
const fx = make();
|
||||
const h1 = openPr(fx, '1');
|
||||
readyReceipt(fx, '1', h1);
|
||||
const r = queue(fx, ['train', 'build', '--prs', `1@${h1}`, '--dry-run']);
|
||||
expect(r.status).toBe(EXIT.ok);
|
||||
expect(r.out).toContain('ORDER: 1:#1(unscreened)');
|
||||
expect(r.out).toMatch(/^PREFIX: T1 [0-9a-f]{40} tree=[0-9a-f]{40} \(\+#1\)$/m);
|
||||
expect(r.out).toContain('ESTIMATE: one evals.yml full gate (evals_all) on T1 plus free-tests.yml; dollar figure pending the T0 cost replay');
|
||||
expect(result(r.out)).toBe('GSTACK_RESULT: skill=ship-queue status=complete run=none');
|
||||
expect(fs.existsSync(path.join(fx.state, 'ship-queue', 'trains'))).toBe(false);
|
||||
expect(remoteRefs(fx)).not.toContain('refs/heads/train/');
|
||||
expect(dispatches(fx)).toEqual([]);
|
||||
}, 120_000);
|
||||
|
||||
test('a conflict stops the train at that PR with the exact recovery line; nothing pushed or dispatched', () => {
|
||||
const fx = make();
|
||||
const h1 = openPr(fx, '1', { file: 'same.txt', text: 'one\n', changelog: false });
|
||||
const h2 = openPr(fx, '2', { file: 'same.txt', text: 'two\n', changelog: false });
|
||||
readyReceipt(fx, '1', h1);
|
||||
readyReceipt(fx, '2', h2);
|
||||
const r = queue(fx, ['train', 'build', '--prs', `1@${h1},2@${h2}`]);
|
||||
expect(r.status).toBe(EXIT.fail);
|
||||
expect(line(r.out, 'QUEUE_TRAIN_CONFLICT:')).toMatch(new RegExp(`^QUEUE_TRAIN_CONFLICT: train [0-9a-f]{12} stopped at #2 \\(conflicts: same\\.txt\\) after merging #1; nothing dispatched, no branch pushed; fix: gstack-ship-queue train build --prs 1@${h1}$`));
|
||||
expect(r.err).toContain('(QUEUE_TRAIN_CONFLICT)');
|
||||
expect(next(r.out)).toBe(`NEXT: gstack-ship-queue train build --prs 1@${h1}`);
|
||||
expect(remoteRefs(fx)).not.toContain('refs/heads/train/');
|
||||
expect(dispatches(fx)).toEqual([]);
|
||||
}, 120_000);
|
||||
|
||||
test('refusals: a fork PR, a moved head, a PR without READY, and a train still filling (until --urgent)', () => {
|
||||
const fx = make();
|
||||
const h1 = openPr(fx, '1');
|
||||
const h2 = openPr(fx, '2', { cross: true });
|
||||
readyReceipt(fx, '1', h1);
|
||||
readyReceipt(fx, '2', h2);
|
||||
const fork = queue(fx, ['train', 'build', '--prs', `1@${h1},2@${h2}`]);
|
||||
expect(fork.status).toBe(EXIT.refused);
|
||||
expect(fork.err).toMatch(/PR #2's head repository is not the base repository.*\(QUEUE_TRAIN_UNTRUSTED\)/);
|
||||
expect(next(fork.out)).toBe(`NEXT: gstack-ship-queue train build --prs 1@${h1}`);
|
||||
const moved = queue(fx, ['train', 'build', '--prs', `1@${'f'.repeat(40)}`]);
|
||||
expect(moved.err).toMatch(/PR #1 moved: pinned ffffffffffff, head is now .*\(QUEUE_TRAIN_MOVED\)/);
|
||||
const h3 = openPr(fx, '3');
|
||||
const notReady = queue(fx, ['train', 'build', '--prs', `3@${h3}`]);
|
||||
expect(notReady.err).toMatch(/no READY receipt for #3 .*\(QUEUE_NOT_READY\)/);
|
||||
readyReceipt(fx, '3', h3, { at: '2026-10-18T11:00:00.000Z' });
|
||||
const filling = queue(fx, ['train', 'build', '--prs', `3@${h3}`]);
|
||||
expect(filling.status).toBe(EXIT.refused);
|
||||
expect(filling.err).toMatch(/1 of 5 PRs; the oldest READY dispatches at 2026-10-18T14:00:00\.000Z.*\(QUEUE_TRAIN_WAITING\)/);
|
||||
expect(result(filling.out)).toBe('GSTACK_RESULT: skill=ship-queue status=incomplete run=none');
|
||||
fx.write(d => { d.rules['evals.yml'] = GREEN_GATE; });
|
||||
const urgent = queue(fx, ['train', 'build', '--prs', `3@${h3}`, '--urgent']);
|
||||
expect(urgent.status).toBe(EXIT.ok);
|
||||
expect(trainJson(fx, nonceOf(urgent.out)).urgent).toBe(true);
|
||||
expect(dispatches(fx, 'evals.yml').length).toBe(1);
|
||||
}, 180_000);
|
||||
|
||||
test('free tests red, cancelled, missing or on another SHA refuse with QUEUE_TRAIN_FREE_RED before any paid dispatch', () => {
|
||||
const fx = make();
|
||||
const h1 = openPr(fx, '1');
|
||||
readyReceipt(fx, '1', h1);
|
||||
fx.write(d => { d.rules['evals.yml'] = GREEN_GATE; });
|
||||
const variants: Array<[Rule, RegExp, string[]]> = [
|
||||
[{ conclusion: 'failure' }, /free tests red: run \d+ concluded failure/, []],
|
||||
[{ conclusion: 'cancelled' }, /free tests cancelled: run \d+ concluded cancelled/, []],
|
||||
[{ silent: true }, /free tests missing: no free-tests\.yml dispatch run appeared on train\//, ['--free-timeout', '1']],
|
||||
[{ headSha: '0'.repeat(40) }, /free tests stale: run \d+ tested 000000000000 \(workflow_dispatch\), not the train head/, []],
|
||||
];
|
||||
for (const [rule, why, extra] of variants) {
|
||||
fx.write(d => { d.rules['free-tests.yml'] = rule; });
|
||||
const r = queue(fx, ['train', 'build', '--prs', `1@${h1}`, '--urgent', ...extra]);
|
||||
expect(r.status).toBe(EXIT.fail);
|
||||
expect(r.err).toMatch(why);
|
||||
expect(r.err).toContain('(QUEUE_TRAIN_FREE_RED)');
|
||||
expect(trainJson(fx, nonceOf(r.out)).state).toBe('free-red');
|
||||
}
|
||||
expect(dispatches(fx, 'evals.yml')).toEqual([]);
|
||||
expect(dispatches(fx, 'free-tests.yml').length).toBe(4);
|
||||
}, 180_000);
|
||||
|
||||
test('ordering: screened PRs first in READY order; enough unscreened PRs wait for their own train', () => {
|
||||
const fx = make({ ...TRAIN_POLICY, train: { ...TRAIN_BLOCK, max_size: 2 } });
|
||||
const h = ['1', '2', '3'].map(pr => openPr(fx, pr));
|
||||
readyReceipt(fx, '1', h[0]!, { at: '2026-10-18T08:00:00.000Z' });
|
||||
readyReceipt(fx, '2', h[1]!, { at: '2026-10-18T07:00:00.000Z' });
|
||||
readyReceipt(fx, '3', h[2]!, { at: '2026-10-18T09:00:00.000Z', screen: true });
|
||||
const r = queue(fx, ['train', 'build', '--prs', `1@${h[0]},2@${h[1]},3@${h[2]}`, '--dry-run']);
|
||||
expect(r.status).toBe(EXIT.ok);
|
||||
expect(line(r.out, 'ORDER:')).toBe('ORDER: 1:#3(screened)');
|
||||
expect(line(r.out, 'HELD:')).toBe('HELD: #2,#1 (unscreened PRs form their own train)');
|
||||
const wide = make();
|
||||
const w = ['1', '2', '3'].map(pr => openPr(wide, pr));
|
||||
readyReceipt(wide, '1', w[0]!, { at: '2026-10-18T08:00:00.000Z' });
|
||||
readyReceipt(wide, '2', w[1]!, { at: '2026-10-18T07:00:00.000Z' });
|
||||
readyReceipt(wide, '3', w[2]!, { at: '2026-10-18T09:00:00.000Z', screen: true });
|
||||
const all = queue(wide, ['train', 'build', '--prs', `1@${w[0]},2@${w[1]},3@${w[2]}`, '--dry-run']);
|
||||
expect(line(all.out, 'ORDER:')).toBe('ORDER: 1:#3(screened) 2:#2(unscreened) 3:#1(unscreened)');
|
||||
expect(all.out).toMatch(/^SLOT: 1 #3 head=[0-9a-f]{40} screened \(narrow\) ready_at=2026-10-18T09:00:00\.000Z$/m);
|
||||
}, 180_000);
|
||||
});
|
||||
|
||||
describe('dispatch recovery and status', () => {
|
||||
test('a partial dispatch (gh failed after GitHub accepted it) is unknown, never re-dispatched, and train status finds the run by nonce', () => {
|
||||
const fx = make();
|
||||
const { r, nonce } = twoPrTrain(fx, { ...GREEN_GATE, failButCreate: true });
|
||||
expect(r.status).toBe(EXIT.refused);
|
||||
expect(r.err).toContain('(QUEUE_TRAIN_DISPATCH_UNKNOWN)');
|
||||
expect(next(r.out)).toBe(`NEXT: gstack-ship-queue train status --train ${nonce}`);
|
||||
expect(trainJson(fx, nonce).state).toBe('dispatch-unknown');
|
||||
expect(queue(fx, ['train', 'verdict', '--train', nonce]).err).toContain('(QUEUE_TRAIN_DISPATCH_UNKNOWN)');
|
||||
expect(queue(fx, ['train', 'build', '--train', nonce]).err).toContain('(QUEUE_TRAIN_DISPATCH_UNKNOWN)');
|
||||
const status = queue(fx, ['train', 'status', '--train', nonce]);
|
||||
expect(status.status).toBe(EXIT.ok);
|
||||
expect(status.out).toMatch(new RegExp(`^ATTEMPT: a1 gate state=dispatched nonce=${nonce.replace(/\./g, '\\.')} run=101 `, 'm'));
|
||||
expect(trainJson(fx, nonce).state).toBe('gate-pending');
|
||||
expect(next(status.out)).toBe(`NEXT: gstack-ship-queue train verdict --train ${nonce}`);
|
||||
expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok);
|
||||
expect(dispatches(fx, 'evals.yml').length).toBe(1);
|
||||
}, 180_000);
|
||||
|
||||
test('a dispatch that never started is marked absent by status; only then does build dispatch it again, under the same nonce', () => {
|
||||
const fx = make();
|
||||
const { r, nonce } = twoPrTrain(fx, { silent: true });
|
||||
expect(r.err).toContain('(QUEUE_TRAIN_DISPATCH_UNKNOWN)');
|
||||
const status = queue(fx, ['train', 'status', '--train', nonce]);
|
||||
expect(status.out).toContain('ATTEMPT: a1 gate state=absent');
|
||||
expect(next(status.out)).toBe(`NEXT: gstack-ship-queue train build --train ${nonce}`);
|
||||
fx.write(d => { d.rules['evals.yml'] = GREEN_GATE; });
|
||||
const again = queue(fx, ['train', 'build', '--train', nonce]);
|
||||
expect(again.status).toBe(EXIT.ok);
|
||||
expect(dispatches(fx, 'evals.yml').map(d => (d.inputs as any).nonce)).toEqual([nonce, nonce]);
|
||||
expect(trainJson(fx, nonce).attempts).toEqual(['a1', 'a2']);
|
||||
}, 180_000);
|
||||
|
||||
test('train status on a clean state root rebuilds the records from refs/notes/gstack-trains and spends nothing', () => {
|
||||
const fx = make();
|
||||
const { nonce } = twoPrTrain(fx);
|
||||
expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok);
|
||||
const before = dispatches(fx).length;
|
||||
const clone = coordinatorClone(fx.repo);
|
||||
const fresh = path.join(fx.repo.root, 'fresh-state');
|
||||
const s = queue(fx, ['train', 'status', '--train', nonce], { GSTACK_STATE_ROOT: fresh }, clone);
|
||||
expect(s.status).toBe(EXIT.ok);
|
||||
expect(s.out).toMatch(/^NOTES: read 2 record\(s\), restored 2 /m);
|
||||
expect(s.out).toContain(`TRAIN: ${nonce} state=green`);
|
||||
expect(s.out).toContain('CHECK: gstack/full-gate on');
|
||||
expect(trainJson(fx, nonce, fresh)).toEqual(trainJson(fx, nonce));
|
||||
expect(dispatches(fx).length).toBe(before);
|
||||
const all = queue(fx, ['train', 'status'], { GSTACK_STATE_ROOT: fresh }, clone);
|
||||
expect(all.out).toContain(`NEXT(${nonce}): gstack-ship-queue verify --pr 1 --train ${nonce}`);
|
||||
}, 180_000);
|
||||
});
|
||||
|
||||
describe('verdict, bisect, rebuild', () => {
|
||||
function redTrain(fx: TrainFixture, pattern: string) {
|
||||
const t = twoPrTrain(fx, RED_GATE);
|
||||
const v = queue(fx, ['train', 'verdict', '--train', t.nonce]);
|
||||
const prefixes = trainJson(fx, t.nonce).prefixes.map((p: any) => p.commit);
|
||||
fx.write(d => { d.rules['evals.yml'] = { bisect: { prefixes, patterns: { c1: pattern } } }; });
|
||||
return { ...t, v, prefixes };
|
||||
}
|
||||
|
||||
test('a red train names its cases and a capped bisect; the bisect runs only the red cases on every prefix and attributes a monotone pattern; prune deletes the dropped train\'s branch', () => {
|
||||
const fx = make();
|
||||
const { h1, nonce, v, prefixes } = redTrain(fx, 'GGR');
|
||||
expect(v.status).toBe(EXIT.fail);
|
||||
expect(v.err).toContain(`train ${nonce} is red on c1`);
|
||||
expect(v.out).toContain('ATTEMPT: a1 gate run 101 RED red=c1 cost=$1.70');
|
||||
expect(next(v.out)).toBe(`NEXT: gstack-ship-queue train bisect --train ${nonce} --spend-cap 2.70`);
|
||||
const dry = queue(fx, ['train', 'bisect', '--train', nonce, '--spend-cap', '2.70', '--dry-run']);
|
||||
expect(dry.out).toContain('BISECT: cases c1 prefixes T0,T1,T2 need results; $0.90 reserved per prefix; spent $0.00, cap $2.70');
|
||||
expect(dry.out).toContain('DRY-RUN: nothing dispatched');
|
||||
expect(dispatches(fx, 'evals.yml').length).toBe(1);
|
||||
const go = queue(fx, ['train', 'bisect', '--train', nonce, '--spend-cap', '2.70']);
|
||||
expect(go.status).toBe(EXIT.ok);
|
||||
const sent = dispatches(fx, 'evals.yml').at(-1)!.inputs as any;
|
||||
expect(sent.cases).toBe('c1');
|
||||
expect(sent.prefix_shas).toBe(prefixes.join(','));
|
||||
expect(sent.expected_sha).toBe(prefixes[2]);
|
||||
expect(sent.nonce).toMatch(/^gate\.bisect-/);
|
||||
const done = queue(fx, ['train', 'bisect', '--train', nonce, '--spend-cap', '2.70']);
|
||||
expect(done.status).toBe(EXIT.ok);
|
||||
expect(done.out).toContain('PATTERN: c1 T0=G T1=G T2=R');
|
||||
expect(done.out).toContain('BISECT: ATTRIBUTED #2');
|
||||
expect(next(done.out)).toBe(`NEXT: gstack-ship-queue train build --prs 1@${h1}`);
|
||||
expect(trainJson(fx, nonce).state).toBe('attributed');
|
||||
expect(queue(fx, ['train', 'prune', '--dry-run']).out).toContain(`WOULD-DELETE: train/${nonce} (attributed)`);
|
||||
expect(queue(fx, ['train', 'prune']).out).toContain(`DELETED: train/${nonce} (attributed)`);
|
||||
expect(remoteRefs(fx)).not.toContain(`refs/heads/train/${nonce}`);
|
||||
expect(queue(fx, ['train', 'prune']).out).toContain(`GONE: train/${nonce}`);
|
||||
}, 180_000);
|
||||
|
||||
test('red on every prefix, T0 included, is RED_ON_MAIN; a non-monotone pattern is unattributed and earns no rebuild', () => {
|
||||
const main = make();
|
||||
const m = redTrain(main, 'RRR');
|
||||
queue(main, ['train', 'bisect', '--train', m.nonce, '--spend-cap', '5']);
|
||||
const onMain = queue(main, ['train', 'bisect', '--train', m.nonce, '--spend-cap', '5']);
|
||||
expect(onMain.status).toBe(EXIT.fail);
|
||||
expect(onMain.err).toContain('(QUEUE_TRAIN_RED_ON_MAIN)');
|
||||
expect(next(onMain.out)).toMatch(/^NEXT: fix forward on main first, then gstack-ship-queue train build --prs /);
|
||||
const fx = make();
|
||||
const n = redTrain(fx, 'GRG');
|
||||
queue(fx, ['train', 'bisect', '--train', n.nonce, '--spend-cap', '5']);
|
||||
const mixed = queue(fx, ['train', 'bisect', '--train', n.nonce, '--spend-cap', '5']);
|
||||
expect(mixed.status).toBe(EXIT.fail);
|
||||
expect(mixed.err).toContain('(QUEUE_TRAIN_UNATTRIBUTED)');
|
||||
expect(mixed.out).toContain('BISECT: UNATTRIBUTED (not monotone)');
|
||||
expect(next(mixed.out)).toMatch(/^NEXT: route the red cases to a repair lane \(GStack Minder\); once the repair merges: /);
|
||||
const refused = queue(fx, ['train', 'rebuild', '--train', n.nonce]);
|
||||
expect(refused.status).toBe(EXIT.refused);
|
||||
expect(refused.err).toMatch(/rebuild refused: the bisect was not all green \(c1=GRG\).*\(QUEUE_TRAIN_REBUILD_REFUSED\)/);
|
||||
}, 240_000);
|
||||
|
||||
test('rebuild-once: refused without a bisect; an all-green bisect earns exactly one unchanged rebuild, both runs reported; a second red stays red', () => {
|
||||
const fx = make();
|
||||
const { nonce } = redTrain(fx, 'GGG');
|
||||
expect(queue(fx, ['train', 'rebuild', '--train', nonce]).err).toMatch(/rebuild refused: no bisect record.*\(QUEUE_TRAIN_REBUILD_REFUSED\)/);
|
||||
queue(fx, ['train', 'bisect', '--train', nonce, '--spend-cap', '5']);
|
||||
const green = queue(fx, ['train', 'bisect', '--train', nonce, '--spend-cap', '5']);
|
||||
expect(green.out).toContain('BISECT: UNATTRIBUTED (all green)');
|
||||
expect(next(green.out)).toBe(`NEXT: gstack-ship-queue train rebuild --train ${nonce}`);
|
||||
fx.write(d => { d.rules['evals.yml'] = RED_GATE; });
|
||||
const rebuild = queue(fx, ['train', 'rebuild', '--train', nonce]);
|
||||
expect(rebuild.status).toBe(EXIT.ok);
|
||||
expect(dispatches(fx, 'evals.yml').at(-1)!.inputs).toMatchObject({ evals_all: 'true', nonce: `${nonce}.2` });
|
||||
const second = queue(fx, ['train', 'verdict', '--train', nonce]);
|
||||
expect(second.status).toBe(EXIT.fail);
|
||||
expect(second.out).toContain('ATTEMPT: a1 gate run 101 RED red=c1');
|
||||
expect(second.out).toMatch(/^ATTEMPT: a2 rebuild run \d+ RED red=c1/m);
|
||||
expect(next(second.out)).toMatch(/^NEXT: second red stays red: route c1 to a repair lane/);
|
||||
expect(queue(fx, ['train', 'rebuild', '--train', nonce]).err).toMatch(/rebuild refused: already rebuilt once/);
|
||||
expect(queue(fx, ['train', 'bisect', '--train', nonce, '--spend-cap', '5']).err).toContain('second red is not bisected again');
|
||||
}, 240_000);
|
||||
|
||||
test('a capped bisect launches only what fits, keeps completed prefixes, and resumes without repeating one', () => {
|
||||
const fx = make();
|
||||
const { nonce, prefixes } = redTrain(fx, 'GGR');
|
||||
const none = queue(fx, ['train', 'bisect', '--train', nonce, '--spend-cap', '0.5']);
|
||||
expect(none.status).toBe(EXIT.refused);
|
||||
expect(none.err).toContain('(QUEUE_TRAIN_BISECT_CAPPED)');
|
||||
expect(next(none.out)).toBe(`NEXT: gstack-ship-queue train bisect --train ${nonce} --spend-cap 2.70`);
|
||||
expect(dispatches(fx, 'evals.yml').length).toBe(1);
|
||||
const part = queue(fx, ['train', 'bisect', '--train', nonce, '--spend-cap', '1.9']);
|
||||
expect(part.status).toBe(EXIT.refused);
|
||||
expect(part.err).toContain('(QUEUE_TRAIN_BISECT_CAPPED)');
|
||||
expect(part.out).toContain('BISECT: launch T0,T1; 1 prefix(es) wait for a higher cap');
|
||||
expect((dispatches(fx, 'evals.yml').at(-1)!.inputs as any).prefix_shas).toBe(prefixes.slice(0, 2).join(','));
|
||||
const resume = queue(fx, ['train', 'bisect', '--train', nonce, '--spend-cap', '5']);
|
||||
expect(resume.status).toBe(EXIT.ok);
|
||||
expect((dispatches(fx, 'evals.yml').at(-1)!.inputs as any).prefix_shas).toBe(prefixes[2]);
|
||||
const done = queue(fx, ['train', 'bisect', '--train', nonce, '--spend-cap', '5']);
|
||||
expect(done.out).toContain('BISECT: ATTRIBUTED #2');
|
||||
expect(done.out).toContain('BISECT: spent $1.50 of cap $5.00');
|
||||
expect(dispatches(fx, 'evals.yml').length).toBe(3);
|
||||
}, 240_000);
|
||||
|
||||
test('a bisect refuses prefixes that are not ancestors of train/<nonce>', () => {
|
||||
const fx = make();
|
||||
const { nonce } = redTrain(fx, 'GGR');
|
||||
fx.repo.git('push', '-q', '-f', 'origin', `origin/main:refs/heads/train/${nonce}`);
|
||||
const r = queue(fx, ['train', 'bisect', '--train', nonce, '--spend-cap', '5']);
|
||||
expect(r.status).toBe(EXIT.refused);
|
||||
expect(r.err).toMatch(/train\/\S+ is at [0-9a-f]{12}, not the recorded train head .*\(QUEUE_TRAIN_PREFIX_INVALID\)/);
|
||||
expect(dispatches(fx, 'evals.yml').length).toBe(1);
|
||||
}, 180_000);
|
||||
|
||||
test('an INFRA/INCOMPLETE-only census is never green and earns one re-dispatch, reported beside the first run', () => {
|
||||
const fx = make();
|
||||
const infra: Rule = { conclusion: 'failure', record: { panels: [PASS('c1'), FAIL('c2', 'INFRA'), PASS('j1', 'judge')], census: { ...COMPLETE, killed_before_upload: 1 }, verdict: 'RED' } };
|
||||
const { nonce } = twoPrTrain(fx, infra);
|
||||
const v = queue(fx, ['train', 'verdict', '--train', nonce]);
|
||||
expect(v.status).toBe(EXIT.fail);
|
||||
expect(v.err).toMatch(/gate is incomplete: 1 slice killed before upload; c2 FAIL \(INFRA\).*\(QUEUE_TRAIN_INCOMPLETE\)/);
|
||||
expect(next(v.out)).toBe(`NEXT: gstack-ship-queue train rebuild --train ${nonce} --infra (the single re-dispatch EVAL_POLICY.infraRedispatch allows; report both runs)`);
|
||||
expect(queue(fx, ['train', 'rebuild', '--train', nonce]).err).toContain('rebuild refused: no bisect record');
|
||||
const again = queue(fx, ['train', 'rebuild', '--train', nonce, '--infra']);
|
||||
expect(again.status).toBe(EXIT.ok);
|
||||
const second = queue(fx, ['train', 'verdict', '--train', nonce]);
|
||||
expect(second.out).toMatch(/^ATTEMPT: a2 infra-redispatch run \d+ INCOMPLETE/m);
|
||||
expect(next(second.out)).toMatch(/^NEXT: diagnose run /);
|
||||
expect(queue(fx, ['train', 'rebuild', '--train', nonce, '--infra']).err).toMatch(/already re-dispatched once.*\(QUEUE_TRAIN_REBUILD_REFUSED\)/);
|
||||
}, 180_000);
|
||||
});
|
||||
|
||||
describe('verify, freeze, repair and clear', () => {
|
||||
test('verify accepts each merge only at prefix equality modulo stamps; an interrupted publication freezes the queue until one repair train lands', () => {
|
||||
const fx = make();
|
||||
const { h1, h2, nonce } = twoPrTrain(fx);
|
||||
expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok);
|
||||
const t = trainJson(fx, nonce);
|
||||
const coord = coordinatorClone(fx.repo);
|
||||
const v = (pr: string, train = nonce) => queue(fx, ['verify', '--pr', pr, '--train', train], {}, coord);
|
||||
|
||||
const pre1 = v('1');
|
||||
expect(pre1.status).toBe(EXIT.ok);
|
||||
expect(pre1.out).toContain(`VERIFY_TRAIN: #1 may merge as slot 1 of train ${nonce}`);
|
||||
expect(line(pre1.out, 'MERGE_CONDITION:')).toBe(`MERGE_CONDITION: sha=${h1}`);
|
||||
const early = v('2');
|
||||
expect(early.status).toBe(EXIT.fail);
|
||||
expect(early.err).toMatch(/is neither T1 nor T2 modulo stamps.*nothing of train \S+ merged, so the queue is not frozen.*\(QUEUE_TRAIN_TREE_MISMATCH\)/);
|
||||
|
||||
publishMain(fx, t.prefixes[1].commit, '1.2.4.0');
|
||||
const post1 = v('1');
|
||||
expect(post1.status).toBe(EXIT.ok);
|
||||
expect(post1.out).toMatch(/^main: [0-9a-f]{40} = T1 \(same modulo stamps \(1 stamp file\(s\)\)\)$/m);
|
||||
expect(post1.out).toContain(`VERIFY_TRAIN: #1 merged; main's tree is T1 of train ${nonce} (1/2 merged)`);
|
||||
expect(next(post1.out)).toBe(`NEXT: gstack-ship-queue verify --pr 2 --train ${nonce}`);
|
||||
expect(line(v('2').out, 'MERGE_CONDITION:')).toBe(`MERGE_CONDITION: sha=${h2}`);
|
||||
|
||||
const broken = publishMain(fx, t.prefixes[2].commit, '1.2.5.0', { file: 'stray.txt', text: 'not in the train\n' });
|
||||
const mismatch = v('2');
|
||||
expect(mismatch.status).toBe(EXIT.fail);
|
||||
expect(mismatch.err).toContain('(QUEUE_TRAIN_TREE_MISMATCH)');
|
||||
expect(line(mismatch.out, 'QUEUE_TRAIN_FROZEN:')).toMatch(new RegExp(`^QUEUE_TRAIN_FROZEN: train \\S+ merged #1; main ${broken.slice(0, 12)} tree [0-9a-f]{12}; backstop dispatched run \\d+ \\(nonce gate\\.backstop-`));
|
||||
const backstop = dispatches(fx, 'evals.yml').at(-1)!;
|
||||
expect([backstop.ref, (backstop.inputs as any).expected_sha, (backstop.inputs as any).evals_all]).toEqual(['main', broken, 'true']);
|
||||
const freezeFile = path.join(fx.state, 'ship-queue', 'freeze', `${repoKey(fx)}.json`);
|
||||
expect(JSON.parse(fs.readFileSync(freezeFile, 'utf8'))).toMatchObject({ kind: 'freeze', merged: ['1'], unmerged: [{ pr: '2', head: h2 }], cleared_at: null });
|
||||
|
||||
const h3 = openPr(fx, '3');
|
||||
readyReceipt(fx, '3', h3);
|
||||
const ordinary = queue(fx, ['train', 'build', '--prs', `3@${h3}`, '--urgent']);
|
||||
expect(ordinary.status).toBe(EXIT.refused);
|
||||
expect(ordinary.err).toContain('(QUEUE_TRAIN_FROZEN)');
|
||||
expect(next(ordinary.out)).toBe('NEXT: gstack-ship-queue train build --repair --prs <repair-pr>@<sha>');
|
||||
const fresh = path.join(fx.repo.root, 'fresh-state');
|
||||
expect(queue(fx, ['train', 'status'], { GSTACK_STATE_ROOT: fresh }, coord).out).toMatch(/^QUEUE_TRAIN_FROZEN: train /m);
|
||||
|
||||
const repair = queue(fx, ['train', 'build', '--repair', '--prs', `3@${h3}`]);
|
||||
expect(repair.status).toBe(EXIT.ok);
|
||||
const rn = nonceOf(repair.out);
|
||||
expect(trainJson(fx, rn)).toMatchObject({ repair: true, base: { sha: broken } });
|
||||
expect(queue(fx, ['train', 'verdict', '--train', rn]).status).toBe(EXIT.ok);
|
||||
expect(v('2').err).toContain('(QUEUE_TRAIN_FROZEN)');
|
||||
expect(dispatches(fx, 'evals.yml').filter(d => d.ref === 'main').length).toBe(1);
|
||||
expect(line(v('3', rn).out, 'MERGE_CONDITION:')).toBe(`MERGE_CONDITION: sha=${h3}`);
|
||||
publishMain(fx, trainJson(fx, rn).prefixes[1].commit, '1.2.6.0');
|
||||
const cleared = v('3', rn);
|
||||
expect(cleared.status).toBe(EXIT.ok);
|
||||
expect(cleared.out).toMatch(/^FREEZE: cleared \(repair #3 via train \S+; main [0-9a-f]{12} carries a green record\)$/m);
|
||||
expect(next(cleared.out)).toBe(`NEXT: gstack-ship-queue train build --prs 2@${h2}`);
|
||||
expect(JSON.parse(fs.readFileSync(freezeFile, 'utf8')).cleared_at).toBe('2026-10-18T12:00:00.000Z');
|
||||
expect(queue(fx, ['train', 'status']).out).not.toContain('QUEUE_TRAIN_FROZEN');
|
||||
}, 300_000);
|
||||
});
|
||||
Reference in new issue
Block a user