mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-01 06:24:41 +01:00
v1.91.146.0 feat(ship-queue): merge trains with pattern bisect and train-record verify (GSTA-49 T1) (#3272)
* feat(ship-queue): merge trains with pattern bisect and train-record verify (GSTA-49 T1) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test(windows): curate ship-queue-train.test.ts out of the Windows subset (gh shim not resolved on windows-latest) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * v1.91.145.0 feat(ship-queue): merge trains with pattern bisect and train-record verify (GSTA-49 T1) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * v1.91.146.0 restamp: #3295 takes v1.91.145.0 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
Garry Tanandcapy-ai[bot] authored and GitHub committed
1 parent
a09b3b8532
commit
6a4ab69dcd
25 files changed
+3047
-36
No files matched your search
@@ -46,6 +46,97 @@ export const RELEASE_RESULT_CODES = {
|
||||
summary: 'the gate-ahead attempt is unknown, already recorded, or its worktree no longer holds the gated commit',
|
||||
fix: 'list attempts with gstack-ship-queue gate-ahead --list, then record the live one, or prepare a new attempt',
|
||||
},
|
||||
// GSTA-49 lane T1: merge trains in gstack-ship-queue (docs/ship-policy.md#run-a-merge-train).
|
||||
QUEUE_TRAIN_OFF: {
|
||||
anchor: 'queue-train-off',
|
||||
summary: 'the base revision’s .gstack/ship-policy.json declares no `train` block, so the merge-train surface is off and the per-PR queue applies',
|
||||
fix: 'use the per-PR queue (ready, stamp, verify), or land a `train` block with min_gstack_version on the base branch first',
|
||||
},
|
||||
QUEUE_TRAIN_UNTRUSTED: {
|
||||
anchor: 'queue-train-untrusted',
|
||||
summary: 'a PR’s head repository is not the base repository; fork work never runs in a train (the train branch and its dispatch carry repository secrets)',
|
||||
fix: 'leave the PR out of the train (gstack-ship-queue train build --prs <list without it>); fork work stays on the trusted-branch path',
|
||||
},
|
||||
QUEUE_TRAIN_MOVED: {
|
||||
anchor: 'queue-train-moved',
|
||||
summary: 'a PR’s head (or the train branch) is no longer the SHA the train pinned, so the train would not test what merges',
|
||||
fix: 'ask the lane for a fresh READY on the new head, then rebuild the train with the new pin (train build --prs <n>@<new sha>,...)',
|
||||
},
|
||||
QUEUE_TRAIN_CONFLICT: {
|
||||
anchor: 'queue-train-conflict',
|
||||
summary: 'merging a PR onto the previous prefix conflicts outside the version files and the CHANGELOG; nothing was dispatched and no branch pushed',
|
||||
fix: 'run the printed train build without that PR; the PR’s lane merges main (with the earlier PRs once they land) and reports READY again',
|
||||
},
|
||||
QUEUE_TRAIN_FREE_RED: {
|
||||
anchor: 'queue-train-free-red',
|
||||
summary: 'the free tests dispatched on train/<nonce> are red, missing, cancelled or ran on another SHA, so the paid gate was not dispatched',
|
||||
fix: 'read the free-tests run; a PR-caused red drops that PR (train build --prs <list without it>), a flaky one goes to its owner before the next build',
|
||||
},
|
||||
QUEUE_TRAIN_DISPATCH_UNKNOWN: {
|
||||
anchor: 'queue-train-dispatch-unknown',
|
||||
summary: 'a train dispatch was attempted but its run could not be found by nonce, so whether it started is unknown; it is never re-dispatched blindly',
|
||||
fix: 'run gstack-ship-queue train status --train <nonce>: it finds the run by nonce, or marks the dispatch absent so train build --train <nonce> may dispatch it',
|
||||
},
|
||||
QUEUE_TRAIN_WAITING: {
|
||||
anchor: 'queue-train-waiting',
|
||||
summary: 'the train is below its size and its oldest READY PR has waited less than the policy’s fill time, so it does not dispatch yet',
|
||||
fix: 'add READY PRs, rerun once the printed time has passed, or pass --urgent for a one-PR train',
|
||||
},
|
||||
QUEUE_TRAIN_PENDING: {
|
||||
anchor: 'queue-train-pending',
|
||||
summary: 'the train’s run (free tests, the full gate or the bisect) has not finished, so there is no verdict yet',
|
||||
fix: 'rerun the printed NEXT: command later; nothing is dispatched again while a run is pending',
|
||||
},
|
||||
QUEUE_TRAIN_INCOMPLETE: {
|
||||
anchor: 'queue-train-incomplete',
|
||||
summary: 'the train gate did not complete its census (INFRA or INCOMPLETE panels, an unbound, partial or cancelled shard, a slice killed before upload, or no published record), so it is never green',
|
||||
fix: 'when every gap is machine-classified INFRA or INCOMPLETE, take the one re-dispatch EVAL_POLICY.infraRedispatch allows (train rebuild --train <nonce> --infra) and report both runs; otherwise diagnose the run',
|
||||
},
|
||||
QUEUE_TRAIN_RED: {
|
||||
anchor: 'queue-train-red',
|
||||
summary: 'the train’s full gate is red on the train tree; its PRs do not merge, and nothing is retried to change the verdict',
|
||||
fix: 'bisect the red cases (train bisect --train <nonce> --spend-cap <usd>); after a rebuild’s second red, route the cases to a repair lane',
|
||||
},
|
||||
QUEUE_TRAIN_RED_ON_MAIN: {
|
||||
anchor: 'queue-train-red-on-main',
|
||||
summary: 'the bisect found the red cases red on every prefix, main (T0) included: main is broken, not the train’s PRs',
|
||||
fix: 'fix forward on main first (a one-PR train), then rebuild this train unchanged',
|
||||
},
|
||||
QUEUE_TRAIN_UNATTRIBUTED: {
|
||||
anchor: 'queue-train-unattributed',
|
||||
summary: 'the bisect pattern blames no PR and not main (all green, or not monotone); one trial cannot tell a flake from a break',
|
||||
fix: 'all green: the one preregistered rebuild (train rebuild --train <nonce>), reporting both runs; any other pattern: a repair lane, then a changed train',
|
||||
},
|
||||
QUEUE_TRAIN_BISECT_CAPPED: {
|
||||
anchor: 'queue-train-bisect-capped',
|
||||
summary: 'the bisect’s reserved price (every case×prefix trial, unknown costs at the policy reservation) passes the spend cap; completed prefixes are kept and the outcome stays unattributed',
|
||||
fix: 'resume with a higher cap (train bisect --train <nonce> --spend-cap <usd>); only prefixes without a result run',
|
||||
},
|
||||
QUEUE_TRAIN_PREFIX_INVALID: {
|
||||
anchor: 'queue-train-prefix-invalid',
|
||||
summary: 'a bisect prefix is not an ancestor of the train head on train/<nonce>, or a red case is not in the gate’s plan, so the bisect inputs are refused',
|
||||
fix: 'check that train/<nonce> still points at the recorded head (train status); a moved branch needs a new train',
|
||||
},
|
||||
QUEUE_TRAIN_REBUILD_REFUSED: {
|
||||
anchor: 'queue-train-rebuild-refused',
|
||||
summary: 'the rebuild is not allowed: the bisect was not all green, the train was already rebuilt (or re-dispatched), or there is no bisect or INFRA record',
|
||||
fix: 'route the red cases to a repair lane; the train is rebuilt only once the repair merges (a changed train)',
|
||||
},
|
||||
QUEUE_TRAIN_TREE_MISMATCH: {
|
||||
anchor: 'queue-train-tree-mismatch',
|
||||
summary: 'main’s tree is neither the train’s prefix before this PR nor after it (modulo stamps), so the merge does not publish what the gate tested',
|
||||
fix: 'stop merging; if earlier PRs of the train merged, the queue is frozen (QUEUE_TRAIN_FROZEN); otherwise rebuild the train on current main',
|
||||
},
|
||||
QUEUE_TRAIN_FROZEN: {
|
||||
anchor: 'queue-train-frozen',
|
||||
summary: 'a train stopped partway through publication, so main holds a prefix no gate covered on its own; ordinary trains are refused until a repair lands',
|
||||
fix: 'admit one repair PR at its exact head (train build --repair --prs <n>@<sha>); the freeze clears when verify finds main’s tree after that merge in a green record',
|
||||
},
|
||||
READY_SCREEN_CHOICE_REQUIRED: {
|
||||
anchor: 'ready-screen-choice-required',
|
||||
summary: 'the base policy turns trains on, so READY needs an explicit screen choice: --screen (pay for a diff-relevant screen now) or --no-screen',
|
||||
fix: 'rerun gstack-ship-queue ready with the printed recommendation (--screen or --no-screen); say in the READY block if focused local cases already ran',
|
||||
},
|
||||
// Field list 2 W4 (pregate `timing`).
|
||||
PREGATE_TIMING_ASSERTION: {
|
||||
anchor: 'pregate-timing-assertion',
|
||||
|
||||
@@ -65,6 +65,28 @@ export interface ShipPolicy {
|
||||
full_e2e: boolean;
|
||||
/** The oldest gstack that may read this policy; a newer key set to a non-default value requires it. */
|
||||
min_gstack_version: string | null;
|
||||
/** Merge trains in gstack-ship-queue (docs/ship-policy.md#run-a-merge-train); null keeps the per-PR queue. */
|
||||
train: TrainPolicy | null;
|
||||
}
|
||||
|
||||
/** The `train` block: the gate workflow trains dispatch and how its verdict is read. */
|
||||
export interface TrainPolicy {
|
||||
/** Workflow file dispatched for the full gate and the bisect (it accepts the inputs in TRAIN_DISPATCH_INPUTS). */
|
||||
gate_workflow: string;
|
||||
/** Workflow file dispatched on train/<nonce> before the paid gate; its run must be green on the train head. */
|
||||
free_workflow: string;
|
||||
/** The check run the train's own gate run publishes on the train head when its census is complete and green. */
|
||||
full_gate_check: string;
|
||||
/** Artifact name of the train record (and the prefix of each bisect prefix's artifact). */
|
||||
record_artifact: string;
|
||||
/** Most PRs in one train. */
|
||||
max_size: number;
|
||||
/** A train under max_size dispatches once its oldest READY PR has waited this long. */
|
||||
fill_hours: number;
|
||||
/** How `ready` computes the screen's selection: gstack's paid selector, or not at all. */
|
||||
screen_selection: 'gstack-paid' | 'none';
|
||||
/** Bisect admission price of one case trial that has no recorded cost. */
|
||||
trial_reservation_usd: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_POLICY: ShipPolicy = {
|
||||
@@ -87,6 +109,7 @@ export const DEFAULT_POLICY: ShipPolicy = {
|
||||
gate_command: null,
|
||||
full_e2e: false,
|
||||
min_gstack_version: null,
|
||||
train: null,
|
||||
};
|
||||
|
||||
export type PolicyKey = Exclude<keyof ShipPolicy, '$schema'>;
|
||||
@@ -112,6 +135,7 @@ export const POLICY_EXPLANATIONS: Record<PolicyKey, string> = {
|
||||
migrations: '{ dir, id }: the migration directory and a regex with one capture group for the number in a file name; `gstack-pregate regate` reports MIGRATION_COLLISION against the base branch and every open PR.',
|
||||
gate_command: 'The repo’s full pre-merge gate command. gstack-ship-queue gate-ahead and ready print it for the caller to run (in the background or in CI); gstack never executes it. null when the repo has none.',
|
||||
full_e2e: 'true: every merge needs a passing full-gate result (gstack-ship-queue gate-ahead --record, or the lane\'s --gate-exit 0 --gate-log on the gated head) bound to a tree equal to the merging one modulo stamps; ready, stamp and verify refuse without one (QUEUE_GATE_MISSING).',
|
||||
train: '{ gate_workflow, free_workflow, full_gate_check, record_artifact, max_size, fill_hours, screen_selection, trial_reservation_usd } or null: turns on merge trains in gstack-ship-queue (train build|status|verdict|bisect|rebuild|prune, ready --screen|--no-screen, verify --train). null keeps today\'s per-PR queue unchanged.',
|
||||
min_gstack_version: 'The oldest gstack that may read this file. Any key newer than 1.91.98.0 set to a non-default value requires it at that key’s introducing version (validate prints each); a gstack older than it refuses the file (POLICY_GSTACK_TOO_OLD).',
|
||||
};
|
||||
|
||||
@@ -124,11 +148,13 @@ export const POLICY_EXPLANATIONS: Record<PolicyKey, string> = {
|
||||
*/
|
||||
export const POLICY_COMPAT_FLOOR = '1.91.98.0';
|
||||
export const SHIP_QUEUE_KEYS_VERSION = '1.91.108.0';
|
||||
export const TRAIN_KEYS_VERSION = '1.91.146.0';
|
||||
export const POLICY_KEY_INTRODUCED: Record<PolicyKey, string> = {
|
||||
bump: '1.91.73.0', queue_mode: '1.91.73.0', release_tool: '1.91.73.0', release_outputs: '1.91.73.0', stamp_paths: '1.91.73.0',
|
||||
changelog: '1.91.73.0', mirror: '1.91.73.0', history: '1.91.73.0', preregistration_shas: '1.91.73.0', stamp_sensitive_gates: '1.91.73.0',
|
||||
pr_count: '1.91.73.0', maintainers: '1.91.73.0', stamp_files: '1.91.98.0', stamp_patterns: '1.91.98.0', restamp_command: '1.91.98.0',
|
||||
migrations: '1.91.98.0', gate_command: SHIP_QUEUE_KEYS_VERSION, full_e2e: SHIP_QUEUE_KEYS_VERSION, min_gstack_version: SHIP_QUEUE_KEYS_VERSION,
|
||||
train: TRAIN_KEYS_VERSION,
|
||||
};
|
||||
|
||||
const newer = (a: string, b: string): boolean => cmpVersion(parseVersion(a)!, parseVersion(b)!) > 0;
|
||||
@@ -241,6 +267,28 @@ function migrationErrors(value: unknown, out: PolicyError[]): void {
|
||||
} catch (e: any) { out.push({ code: 'POLICY_INVALID', path: 'migrations.id', message: `not a valid regex: ${e.message}` }); }
|
||||
}
|
||||
|
||||
const TRAIN_FIELDS: Record<keyof TrainPolicy, (v: unknown) => string | null> = {
|
||||
gate_workflow: v => (typeof v === 'string' && /^[A-Za-z0-9_.-]{1,64}\.ya?ml$/.test(v) ? null : 'must be a workflow file name such as evals.yml'),
|
||||
free_workflow: v => (typeof v === 'string' && /^[A-Za-z0-9_.-]{1,64}\.ya?ml$/.test(v) ? null : 'must be a workflow file name such as free-tests.yml'),
|
||||
full_gate_check: v => (typeof v === 'string' && /^[A-Za-z0-9_./-]{1,64}$/.test(v) ? null : 'must be a check-run name ([A-Za-z0-9_./-]{1,64})'),
|
||||
record_artifact: v => (typeof v === 'string' && /^[A-Za-z0-9_.-]{1,64}$/.test(v) ? null : 'must be an artifact name ([A-Za-z0-9_.-]{1,64})'),
|
||||
max_size: v => (Number.isInteger(v) && (v as number) >= 1 && (v as number) <= 6 ? null : 'must be an integer from 1 to 6'),
|
||||
fill_hours: v => (typeof v === 'number' && v > 0 && v <= 24 ? null : 'must be a number of hours above 0 and at most 24'),
|
||||
screen_selection: v => (v === 'gstack-paid' || v === 'none' ? null : 'must be gstack-paid | none'),
|
||||
trial_reservation_usd: v => (typeof v === 'number' && v > 0 && v <= 100 ? null : 'must be dollars above 0 and at most 100'),
|
||||
};
|
||||
|
||||
function trainErrors(value: unknown, out: PolicyError[]): void {
|
||||
const t = value as Record<string, unknown> | null;
|
||||
if (!t || typeof t !== 'object' || Array.isArray(t)) { out.push({ code: 'POLICY_INVALID', path: 'train', message: `must be { ${Object.keys(TRAIN_FIELDS).join(', ')} } or null` }); return; }
|
||||
for (const k of Object.keys(t)) if (!(k in TRAIN_FIELDS)) out.push({ code: 'POLICY_INVALID', path: `train.${k}`, message: 'unknown key' });
|
||||
for (const [k, check] of Object.entries(TRAIN_FIELDS)) {
|
||||
if (!(k in t)) { out.push({ code: 'POLICY_INVALID', path: `train.${k}`, message: 'missing (every train field is required)' }); continue; }
|
||||
const why = check(t[k]);
|
||||
if (why) out.push({ code: 'POLICY_INVALID', path: `train.${k}`, message: `${JSON.stringify(t[k])} ${why}` });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Schema and containment check of a parsed policy object. Unknown keys are
|
||||
* errors (a typo must not silently mean "default", and a key from a newer
|
||||
@@ -282,6 +330,7 @@ export function validatePolicy(raw: unknown, opts: { gstackVersion?: string } =
|
||||
else if (t.length > MAX_COMMAND || /[\0\r\n]/.test(t)) errors.push({ code: 'POLICY_CONTAINMENT', path: 'gate_command', message: `must be one line under ${MAX_COMMAND} characters` });
|
||||
}
|
||||
if ('full_e2e' in obj && typeof obj.full_e2e !== 'boolean') errors.push({ code: 'POLICY_INVALID', path: 'full_e2e', message: 'must be true or false' });
|
||||
if ('train' in obj && obj.train !== null) trainErrors(obj.train, errors);
|
||||
if ('stamp_paths' in obj) listErrors('stamp_paths', obj.stamp_paths, errors, globError);
|
||||
for (const key of ['changelog', 'mirror'] as const) {
|
||||
if (!(key in obj) || obj[key] === null) continue;
|
||||
|
||||
@@ -129,7 +129,7 @@ export function applyStampPatch(repoRoot: string, mergeTree: string, patch: stri
|
||||
});
|
||||
}
|
||||
|
||||
function withIndex<T>(repoRoot: string, tree: string, fn: (env: NodeJS.ProcessEnv) => T): T | { error: string } {
|
||||
export function withIndex<T>(repoRoot: string, tree: string, fn: (env: NodeJS.ProcessEnv) => T): T | { error: string } {
|
||||
const gitDir = git(repoRoot, ['rev-parse', '--absolute-git-dir']).out;
|
||||
const indexFile = path.join(gitDir, `gstack-ship-queue-${process.pid}-${randomBytes(4).toString('hex')}.index`);
|
||||
const env = { ...process.env, GIT_INDEX_FILE: indexFile };
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
/**
|
||||
* Argument parsing and output for the merge-train surface of
|
||||
* bin/gstack-ship-queue: `train build|status|verdict|bisect|rebuild|prune`,
|
||||
* `ready --screen|--no-screen [--broad]` and `verify --train`. Every command
|
||||
* prints its lines, the coded error line on stderr, then `NEXT: <command>`
|
||||
* and the canonical `GSTACK_RESULT: skill=ship-queue status=<s> run=<record>`.
|
||||
* Loaded by the bin only when one of those paths runs.
|
||||
*/
|
||||
import { EXIT, formatResult } from './headless-artifacts';
|
||||
import { RESULT_CODES } from './result-codes';
|
||||
import type { QueueContext } from './ship-queue';
|
||||
import { buildTrain, pruneTrains, trainStatus, type TrainOut } from './ship-queue-train';
|
||||
import { trainReady } from './ship-queue-train-ready';
|
||||
import { trainBisect, trainRebuild, trainVerdict } from './ship-queue-train-verdict';
|
||||
import { verifyTrain } from './ship-queue-train-verify';
|
||||
import { redactCause } from './ship-queue-train-record';
|
||||
|
||||
export const TRAIN_HELP = ` train build --prs <n>@<sha>[,...] [--urgent | --repair] [--dry-run] [--free-timeout <s>]
|
||||
train build --train <nonce> resume a train (push, free tests, dispatch) from its record
|
||||
merge pinned unstamped heads in slot order onto main, free tests on
|
||||
train/<nonce>, then one full gate through eval-bg's nonce
|
||||
train status [--train <nonce>] never spends: rebuild records from refs/notes/gstack-trains, reconcile
|
||||
unconfirmed dispatches by nonce
|
||||
train verdict --train <nonce> read the gate run and its record: GREEN, RED, INCOMPLETE or PENDING
|
||||
train bisect --train <nonce> --spend-cap <usd> [--dry-run]
|
||||
red cases on every prefix T0..Tk, resumable and capped
|
||||
train rebuild --train <nonce> [--infra] the one rebuild an all-green bisect earns (or the one INFRA re-dispatch)
|
||||
train prune [--train <nonce>] [--dry-run]
|
||||
delete train/<nonce> branches of merged, superseded or dropped trains
|
||||
ready --pr <n> --screen [--broad] | --no-screen [--dry-run]
|
||||
(train policy) READY plus the screen choice, recorded as a receipt
|
||||
verify --pr <n> --train <nonce> [--no-fetch]
|
||||
(train policy) the merge check against a green train record
|
||||
Merge trains need a "train" block in the base revision's ship policy (docs/ship-policy.md#run-a-merge-train).`;
|
||||
|
||||
function flag(args: string[], name: string): string | undefined {
|
||||
const i = args.indexOf(name);
|
||||
return i >= 0 ? args[i + 1] : undefined;
|
||||
}
|
||||
|
||||
export function printTrain(cmd: string, out: TrainOut): number {
|
||||
for (const line of out.lines) process.stdout.write(line + '\n');
|
||||
if (out.failure) {
|
||||
const f = out.failure;
|
||||
process.stderr.write(`gstack-ship-queue ${cmd}: ${f.message}; fix: ${f.fix ?? RESULT_CODES[f.code].fix} (${f.code})\n`);
|
||||
}
|
||||
process.stdout.write(`NEXT: ${out.next}\n`);
|
||||
process.stdout.write(formatResult('ship-queue', out.status, out.run) + '\n');
|
||||
return out.exit;
|
||||
}
|
||||
|
||||
function usage(message: string): number {
|
||||
process.stderr.write(`gstack-ship-queue: ${message}\n${TRAIN_HELP}\n`);
|
||||
return EXIT.usage;
|
||||
}
|
||||
|
||||
async function guarded(cmd: string, fn: () => Promise<TrainOut> | TrainOut): Promise<number> {
|
||||
try {
|
||||
return printTrain(cmd, await fn());
|
||||
} catch (error) {
|
||||
const cause = redactCause(error instanceof Error ? error.message : String(error));
|
||||
process.stderr.write(`gstack-ship-queue ${cmd}: unexpected failure: ${cause}\n`);
|
||||
process.stdout.write(`NEXT: gstack-ship-queue train status\n`);
|
||||
process.stdout.write(formatResult('ship-queue', 'interrupted', 'none') + '\n');
|
||||
return EXIT.fail;
|
||||
}
|
||||
}
|
||||
|
||||
const positive = (s: string | undefined, fallback: number) => (s !== undefined && /^\d+(\.\d+)?$/.test(s) && Number(s) > 0 ? Number(s) : s === undefined ? fallback : NaN);
|
||||
|
||||
export async function trainMain(ctx: QueueContext, args: string[]): Promise<number> {
|
||||
const [sub, ...rest] = args;
|
||||
const nonce = flag(rest, '--train');
|
||||
const dryRun = rest.includes('--dry-run');
|
||||
if (sub === 'build') {
|
||||
if (!nonce && !flag(rest, '--prs')) return usage('train build needs --prs <n>@<sha>[,...] or --train <nonce>');
|
||||
const freeTimeoutSec = positive(flag(rest, '--free-timeout'), 2700);
|
||||
if (Number.isNaN(freeTimeoutSec)) return usage('--free-timeout needs seconds');
|
||||
if (rest.includes('--urgent') && rest.includes('--repair')) return usage('--urgent and --repair are separate one-PR trains; pass one');
|
||||
return guarded('train build', () => buildTrain(ctx, {
|
||||
pins: flag(rest, '--prs'), resume: nonce, dryRun, urgent: rest.includes('--urgent'), repair: rest.includes('--repair'), freeTimeoutSec,
|
||||
pollMs: Number(process.env.GSTACK_SHIP_QUEUE_POLL_MS) || 30_000,
|
||||
}));
|
||||
}
|
||||
if (sub === 'status') return guarded('train status', () => trainStatus(ctx, { train: nonce }));
|
||||
if (sub === 'prune') return guarded('train prune', () => pruneTrains(ctx, { dryRun, train: nonce }));
|
||||
if (!nonce) return usage(`train ${sub ?? '<subcommand>'} needs --train <nonce>`);
|
||||
if (sub === 'verdict') return guarded('train verdict', () => trainVerdict(ctx, { train: nonce }));
|
||||
if (sub === 'rebuild') return guarded('train rebuild', () => trainRebuild(ctx, { train: nonce, infra: rest.includes('--infra') }));
|
||||
if (sub === 'bisect') {
|
||||
const cap = positive(flag(rest, '--spend-cap'), NaN);
|
||||
if (Number.isNaN(cap)) return usage('train bisect needs --spend-cap <usd>');
|
||||
return guarded('train bisect', () => trainBisect(ctx, { train: nonce, spendCap: cap, dryRun }));
|
||||
}
|
||||
return usage(`unknown train subcommand ${sub}`);
|
||||
}
|
||||
|
||||
export async function readyMain(ctx: QueueContext, args: string[], base: { pr: string; pregate?: string; gateExit?: number; gateLog?: string }): Promise<number> {
|
||||
const screen = args.includes('--screen');
|
||||
const noScreen = args.includes('--no-screen');
|
||||
if (screen && noScreen) return usage('pass one of --screen or --no-screen');
|
||||
if (args.includes('--broad') && !screen) return usage('--broad selects the screen policy; it needs --screen');
|
||||
return guarded('ready', () => trainReady(ctx, { ...base, screen: screen ? 'screen' : noScreen ? 'no-screen' : undefined, broad: args.includes('--broad'), dryRun: args.includes('--dry-run') }));
|
||||
}
|
||||
|
||||
export async function verifyMain(ctx: QueueContext, args: string[], pr: string): Promise<number> {
|
||||
return guarded('verify', () => verifyTrain(ctx, { pr, train: flag(args, '--train')!, noFetch: args.includes('--no-fetch') }));
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
/**
|
||||
* GitHub reads and writes for merge trains, all through the queue's `gh`
|
||||
* seam (QueueContext.gh) so tests answer them from a fixture: PR identity,
|
||||
* workflow runs found by branch or by eval-bg nonce, run artifacts and the
|
||||
* full-gate check run. Dispatch itself goes through scripts/eval-bg.ts.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import type { Exec, RunSummary } from '../scripts/eval-bg';
|
||||
import type { QueueContext } from './ship-queue';
|
||||
|
||||
export interface PrInfo { headRefOid: string; isCrossRepository: boolean; state: string; headRefName: string }
|
||||
export interface RunInfo { databaseId: number; status: string; conclusion: string | null; url: string; headSha: string; event: string; displayTitle: string }
|
||||
export interface CheckInfo { status: string; conclusion: string | null }
|
||||
|
||||
function json<T>(r: { ok: boolean; out: string }): T | null {
|
||||
if (!r.ok) return null;
|
||||
try { return JSON.parse(r.out) as T; } catch { return null; }
|
||||
}
|
||||
|
||||
/** eval-bg's process seam bound to the queue's gh (only `gh` is ever routed through it). */
|
||||
export function execFor(ctx: QueueContext): Exec {
|
||||
return (command, args) => {
|
||||
if (command !== 'gh') return { status: null, stdout: '' };
|
||||
const r = ctx.gh(args);
|
||||
return { status: r.ok ? 0 : 1, stdout: r.out };
|
||||
};
|
||||
}
|
||||
|
||||
export function repoSlug(ctx: QueueContext): string | null {
|
||||
const r = json<{ nameWithOwner?: string }>(ctx.gh(['repo', 'view', '--json', 'nameWithOwner']));
|
||||
return r?.nameWithOwner && /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(r.nameWithOwner) ? r.nameWithOwner : null;
|
||||
}
|
||||
|
||||
export function prInfo(ctx: QueueContext, pr: string): PrInfo | { error: string } {
|
||||
const r = ctx.gh(['pr', 'view', pr, '--json', 'headRefOid,isCrossRepository,state,headRefName']);
|
||||
const p = json<Partial<PrInfo>>(r);
|
||||
if (!p?.headRefOid) return { error: `cannot read PR #${pr} through gh (${(r.err ?? r.out).trim().slice(0, 120) || 'no headRefOid'})` };
|
||||
return { headRefOid: p.headRefOid, isCrossRepository: p.isCrossRepository !== false, state: p.state ?? 'OPEN', headRefName: p.headRefName ?? '' };
|
||||
}
|
||||
|
||||
const RUN_FIELDS = 'databaseId,status,conclusion,url,headSha,event,displayTitle';
|
||||
|
||||
/** Newest workflow_dispatch run of a workflow on one branch; null when none; 'unreadable' when gh failed. */
|
||||
export function newestBranchRun(ctx: QueueContext, slug: string, workflow: string, branch: string): RunInfo | null | 'unreadable' {
|
||||
const runs = json<RunInfo[]>(ctx.gh(['run', 'list', '--repo', slug, '--workflow', workflow, '--branch', branch, '--event', 'workflow_dispatch', '--limit', '20', '--json', RUN_FIELDS]));
|
||||
if (!Array.isArray(runs)) return 'unreadable';
|
||||
return runs.filter(r => r.event === 'workflow_dispatch').sort((a, b) => b.databaseId - a.databaseId)[0] ?? null;
|
||||
}
|
||||
|
||||
export function runView(ctx: QueueContext, slug: string, id: number): RunInfo | null {
|
||||
return json<RunInfo>(ctx.gh(['run', 'view', String(id), '--repo', slug, '--json', RUN_FIELDS]));
|
||||
}
|
||||
|
||||
export function asRun(summary: RunSummary): { id: number; url: string } {
|
||||
return { id: summary.databaseId, url: summary.url };
|
||||
}
|
||||
|
||||
/** One JSON artifact of a run (the first .json file it holds); null when it is missing or unreadable. */
|
||||
export function downloadJson(ctx: QueueContext, slug: string, run: number, name: string, scratch: string): unknown {
|
||||
const dir = path.join(scratch, `${run}-${name}`);
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
try {
|
||||
if (!ctx.gh(['run', 'download', String(run), '--repo', slug, '-n', name, '-D', dir]).ok) return null;
|
||||
const file = fs.readdirSync(dir).find(f => f.endsWith('.json'));
|
||||
return file ? JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8')) : null;
|
||||
} catch { return null; } finally { fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
}
|
||||
|
||||
/** The named check run on a commit (the newest when several); null when absent or unreadable. */
|
||||
export function checkRun(ctx: QueueContext, sha: string, name: string): CheckInfo | null {
|
||||
const r = json<{ check_runs?: Array<{ name?: string; status?: string; conclusion?: string | null }> }>(
|
||||
ctx.gh(['api', `repos/{owner}/{repo}/commits/${sha}/check-runs?check_name=${encodeURIComponent(name)}`]));
|
||||
const hit = r?.check_runs?.find(c => c.name === name);
|
||||
return hit ? { status: hit.status ?? 'unknown', conclusion: hit.conclusion ?? null } : null;
|
||||
}
|
||||
|
||||
/** Whether a branch exists on the remote (ls-remote, never a fetch). */
|
||||
export function remoteBranchSha(git: (args: string[]) => { ok: boolean; out: string }, remote: string, branch: string): string | null {
|
||||
const r = git(['ls-remote', remote, `refs/heads/${branch}`]);
|
||||
return r.ok ? (r.out.split(/\s/)[0] || null) : null;
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
/**
|
||||
* `ready --pr <n> --screen | --no-screen` under a ship policy with a `train`
|
||||
* block (docs/ship-policy.md#run-a-merge-train; plan GSTA-49 T1). The
|
||||
* READY block is today's (lib/ship-queue.ts ready()); this adds the screen
|
||||
* selection and Garry's default recommendation (screen when the screen
|
||||
* selects a skill's own cases), requires an explicit choice
|
||||
* (READY_SCREEN_CHOICE_REQUIRED), dispatches a chosen screen through
|
||||
* eval-bg's nonce with its purpose and policy in the key, and records the
|
||||
* choice as a READY receipt (local state and refs/notes/gstack-trains) that
|
||||
* `train build` requires. The screen's dollar price is left to the T0 cost
|
||||
* replay (lib/gate-cost.ts) and printed as pending until it lands.
|
||||
*/
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import { DISPATCH_INPUTS, awaitNonceRun, dedupeKey, dispatchWorkflow, findDuplicateRun, listDispatchRuns, makeNonce, type WorkflowDispatch } from '../scripts/eval-bg';
|
||||
import { EXIT } from './headless-artifacts';
|
||||
import { ready, type QueueContext } from './ship-queue';
|
||||
import { git } from './ship-queue-gate';
|
||||
import { execFor, repoSlug } from './ship-queue-train-github';
|
||||
import { loadReady, nowIso, persist, readyPath, type ReadyReceipt, type ScreenChoice } from './ship-queue-train-record';
|
||||
import { outOk, outPending, outRefused, trainEnv, type TrainEnv, type TrainOut } from './ship-queue-train';
|
||||
|
||||
const Q = 'gstack-ship-queue';
|
||||
|
||||
export interface ReadyOptions { pr: string; pregate?: string; gateExit?: number; gateLog?: string; screen?: 'screen' | 'no-screen'; broad: boolean; dryRun: boolean }
|
||||
export interface ScreenSelection { lines: string[]; recommend: 'screen' | 'no-screen' | null }
|
||||
|
||||
/** Skill directories a diff touches: top-level directories that hold a SKILL.md.tmpl at the head. */
|
||||
export function touchedSkills(changed: readonly string[], hasTemplate: (dir: string) => boolean): string[] {
|
||||
return [...new Set(changed.filter(f => f.includes('/')).map(f => f.split('/')[0]!))].filter(hasTemplate).sort();
|
||||
}
|
||||
|
||||
/** The selected cases that are a touched skill's own: their touchfiles name a path inside that skill's directory. */
|
||||
export function skillOwnCases(selected: readonly string[], touchfiles: Record<string, string[]>, skills: readonly string[]): string[] {
|
||||
return selected.filter(id => (touchfiles[id] ?? []).some(glob => skills.some(s => glob.startsWith(`${s}/`)))).sort();
|
||||
}
|
||||
|
||||
/** gstack's paid selector over a diff (today's PR selection; T2 narrows it), loaded only when the policy asks for it. */
|
||||
export async function gstackPaidSelection(repoRoot: string, baseSha: string, changed: string[], hasTemplate: (dir: string) => boolean): Promise<ScreenSelection> {
|
||||
const { computePaidCaseSelection } = await import('../scripts/lib/paid-select');
|
||||
const { E2E_TOUCHFILES, LLM_JUDGE_TOUCHFILES } = await import('../test/helpers/touchfiles');
|
||||
const skills = touchedSkills(changed, hasTemplate);
|
||||
let e2e: string[]; let judges: number; let note = "today's PR selection; T2's narrowed screen has not landed";
|
||||
try {
|
||||
const s = computePaidCaseSelection({ profile: 'pr', rootDir: repoRoot, changedFiles: changed, env: { ...process.env, EVALS_BASE: baseSha, EVALS_ALL: '' } }).selection;
|
||||
e2e = s.e2e ?? Object.keys(E2E_TOUCHFILES);
|
||||
judges = s.judges ? s.judges.length : Object.keys(LLM_JUDGE_TOUCHFILES).length;
|
||||
} catch (error) {
|
||||
e2e = Object.keys(E2E_TOUCHFILES);
|
||||
judges = Object.keys(LLM_JUDGE_TOUCHFILES).length;
|
||||
note = `the PR profile needs full validation (${error instanceof Error ? error.message.slice(0, 160) : String(error)}): the screen is the full gate selection`;
|
||||
}
|
||||
const own = skillOwnCases(e2e, E2E_TOUCHFILES, skills);
|
||||
const recommend = own.length ? 'screen' : 'no-screen';
|
||||
return {
|
||||
recommend,
|
||||
lines: [
|
||||
`SCREEN: selection ${e2e.length} e2e case(s) + ${judges} judge(s) (${note})`,
|
||||
`SCREEN: skill's own cases ${own.length ? own.slice(0, 12).join(',') + (own.length > 12 ? ` (+${own.length - 12})` : '') : 'none'} → recommend --${recommend}`,
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
async function screenSelection(env: TrainEnv, head: string, baseSha: string): Promise<ScreenSelection> {
|
||||
const price = 'SCREEN: price pending the T0 cost replay (lib/gate-cost.ts): cases without cost data are unknown; a red train holds its PRs about an hour';
|
||||
if (env.train.screen_selection === 'none') return { recommend: null, lines: ['SCREEN: selection not computed (train.screen_selection is none); no recommendation, the lane decides', price] };
|
||||
const changed = git(env.ctx.repoRoot, ['diff', '--name-only', `${baseSha}...${head}`]).out.split('\n').filter(Boolean);
|
||||
const hasTemplate = (dir: string) => git(env.ctx.repoRoot, ['cat-file', '-e', `${head}:${dir}/SKILL.md.tmpl`]).ok;
|
||||
const s = await gstackPaidSelection(env.ctx.repoRoot, baseSha, changed, hasTemplate);
|
||||
return { ...s, lines: [...s.lines, price] };
|
||||
}
|
||||
|
||||
/** Dispatch (or adopt) the PR's screen on the base branch: inputs name the PR, its merge SHA and the screen policy. */
|
||||
async function dispatchScreen(env: TrainEnv, o: ReadyOptions, mergeSha: string, baseSha: string): Promise<{ choice: ScreenChoice; line: string } | TrainOut> {
|
||||
const policy = o.broad ? 'broad' : 'narrow';
|
||||
const scope = { purpose: 'screen' as const, policy: policy as 'narrow' | 'broad' };
|
||||
const nonce = makeNonce('pr', mergeSha, baseSha, randomBytes(3).toString('hex'), scope);
|
||||
const dispatch: WorkflowDispatch = { workflow: env.train.gate_workflow, inputs: {
|
||||
[DISPATCH_INPUTS.evalsAll]: 'false', [DISPATCH_INPUTS.expectedSha]: mergeSha, [DISPATCH_INPUTS.nonce]: nonce, [DISPATCH_INPUTS.baseRef]: env.ctx.base,
|
||||
[DISPATCH_INPUTS.baseSha]: baseSha, [DISPATCH_INPUTS.screenPr]: o.pr, [DISPATCH_INPUTS.screenMergeSha]: mergeSha, [DISPATCH_INPUTS.screenPolicy]: policy,
|
||||
} };
|
||||
const inputs = Object.entries(dispatch.inputs).map(([k, v]) => `${k}=${v}`).join(' ');
|
||||
if (o.dryRun) return { choice: { choice: 'screen', policy, nonce, run: null, merge_sha: mergeSha }, line: `SCREEN_DISPATCH: ${dispatch.workflow} --ref ${env.ctx.base} ${inputs} (dry run: not dispatched)` };
|
||||
const slug = repoSlug(env.ctx);
|
||||
if (!slug) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: 'gh repo view could not name the repository' }, [], `${Q} ready --pr ${o.pr} --screen`, 'none');
|
||||
const exec = execFor(env.ctx);
|
||||
const existing = findDuplicateRun(listDispatchRuns(dispatch.workflow, exec, slug) ?? [], dedupeKey('pr', mergeSha, baseSha, scope));
|
||||
if (existing) return { choice: { choice: 'screen', policy, nonce: /eval-bg (\S+?)\]/.exec(existing.displayTitle)?.[1] ?? nonce, run: existing.databaseId, merge_sha: mergeSha }, line: `SCREEN_DISPATCH: following run ${existing.databaseId} (same merge SHA, base and policy; no duplicate)` };
|
||||
const found = dispatchWorkflow(dispatch, env.ctx.base, exec, slug) ? await awaitNonceRun(dispatch.workflow, nonce, exec, slug) : null;
|
||||
if (!found) return outPending({ code: 'QUEUE_TRAIN_DISPATCH_UNKNOWN', message: `no ${dispatch.workflow} run named ${nonce} appeared after the screen dispatch` }, [], `${Q} ready --pr ${o.pr} --screen${o.broad ? ' --broad' : ''} (adopts the run by its key once it appears)`, 'none');
|
||||
return { choice: { choice: 'screen', policy, nonce, run: found.databaseId, merge_sha: mergeSha }, line: `SCREEN_DISPATCH: ${dispatch.workflow} run ${found.databaseId} ${found.url} nonce ${nonce}` };
|
||||
}
|
||||
|
||||
export async function trainReady(ctx: QueueContext, o: ReadyOptions): Promise<TrainOut> {
|
||||
const env = trainEnv(ctx);
|
||||
if ('code' in env) return outRefused(env, [], 'none', 'none');
|
||||
const base = ready(ctx, o);
|
||||
if (!base.ok) {
|
||||
const out = outRefused(base.failure!, base.lines, `fix the refusal, then ${Q} ready --pr ${o.pr} --screen|--no-screen`, base.run);
|
||||
return ['QUEUE_VERIFY_MISMATCH', 'QUEUE_STAMP_NOT_STAMP_ONLY'].includes(base.failure!.code) ? { ...out, exit: EXIT.fail } : out;
|
||||
}
|
||||
const head = git(ctx.repoRoot, ['rev-parse', 'HEAD']).out;
|
||||
const baseSha = git(ctx.repoRoot, ['rev-parse', `${env.remote}/${ctx.base}`]).out;
|
||||
const sel = await screenSelection(env, head, baseSha);
|
||||
const lines = [...base.lines, ...sel.lines];
|
||||
const rec = sel.recommend ?? 'no-screen';
|
||||
if (!o.screen) {
|
||||
return outRefused({ code: 'READY_SCREEN_CHOICE_REQUIRED', message: `the base policy runs merge trains; choose --screen or --no-screen (recommended: --${rec})` }, lines,
|
||||
`${Q} ready --pr ${o.pr} --${rec}`, 'none');
|
||||
}
|
||||
let choice: ScreenChoice = { choice: 'no-screen', policy: null, nonce: null, run: null, merge_sha: null };
|
||||
if (o.screen === 'screen') {
|
||||
const ls = git(ctx.repoRoot, ['ls-remote', env.remote, `refs/pull/${o.pr}/merge`]);
|
||||
const mergeSha = ls.ok ? ls.out.split(/\s/)[0] ?? '' : '';
|
||||
if (!/^[0-9a-f]{40,64}$/.test(mergeSha)) return outRefused({ code: 'QUEUE_NOT_READY', message: `${env.remote} has no refs/pull/${o.pr}/merge (the PR conflicts with ${ctx.base}, or GitHub has not computed it yet)` }, lines, `merge ${ctx.base} into the branch, push, then ${Q} ready --pr ${o.pr} --screen`, 'none');
|
||||
const d = await dispatchScreen(env, o, mergeSha, baseSha);
|
||||
if ('exit' in d) return { ...d, lines: [...lines, ...d.lines] };
|
||||
choice = d.choice;
|
||||
lines.push(d.line);
|
||||
}
|
||||
lines.splice(1, 0, `screen: ${choice.choice}${choice.policy ? ` policy=${choice.policy} nonce=${choice.nonce} run=${choice.run ?? '-'}` : ''}`);
|
||||
const now = nowIso();
|
||||
const readyAt = loadReady(env.repo.key, o.pr, head)?.ready_at ?? now;
|
||||
const receipt: ReadyReceipt = { schema: 1, kind: 'ready', repo_key: env.repo.key, pr: o.pr, head, base_sha: baseSha, ready_at: readyAt, screen: choice, updated_at: now };
|
||||
if (o.dryRun) return outOk([...lines, 'DRY-RUN: no READY receipt written, nothing dispatched'], `${Q} ready --pr ${o.pr} --${o.screen}${o.broad ? ' --broad' : ''}`, 'none');
|
||||
const p = persist(ctx.repoRoot, [receipt], { remote: env.remote });
|
||||
lines.push(`READY_RECEIPT: ${readyPath(env.repo.key, o.pr, head)} (${p.detail})`);
|
||||
return outOk(lines, `send the READY block; the coordinator builds the train: ${Q} train build --prs ${o.pr}@${head},...`, readyPath(env.repo.key, o.pr, head));
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
/**
|
||||
* Merge-train records (docs/ship-policy.md#run-a-merge-train): versioned
|
||||
* `train.json`, one attempt record per dispatch, the bisect ledger, READY
|
||||
* receipts and the queue freeze, under `$GSTACK_STATE_ROOT/ship-queue/`
|
||||
* (lib/state-root.ts resolves the root). Every record is mirrored to origin
|
||||
* as a git note under `refs/notes/gstack-trains`, each on a blob named for
|
||||
* the record, so a lost machine rebuilds its state with `train status`. A
|
||||
* record is written locally before its note, and the note before any remote
|
||||
* effect that depends on it.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { redactFindingSpans } from './redact-engine';
|
||||
import { git, sha256, writeJsonAtomic } from './ship-queue-gate';
|
||||
import { resolveStateRoot } from './state-root';
|
||||
|
||||
export const TRAIN_SCHEMA = 1;
|
||||
export const NOTES_REF = 'refs/notes/gstack-trains';
|
||||
const NOTES_REMOTE_REF = 'refs/notes/gstack-trains-origin';
|
||||
|
||||
export type TrainState =
|
||||
| 'built' | 'pushed' | 'free-pending' | 'free-red' | 'free-green' | 'dispatch-unknown' | 'dispatch-absent' | 'gate-pending'
|
||||
| 'green' | 'red' | 'incomplete' | 'attributed' | 'red-on-main' | 'unattributed' | 'merging' | 'merged' | 'superseded';
|
||||
/** States after which a train never dispatches again; prune deletes their branches. */
|
||||
export const TERMINAL_STATES: readonly TrainState[] = ['free-red', 'attributed', 'red-on-main', 'merged', 'superseded'];
|
||||
|
||||
export interface ScreenChoice { choice: 'screen' | 'no-screen'; policy: 'narrow' | 'broad' | null; nonce: string | null; run: number | null; merge_sha: string | null }
|
||||
export interface TrainSlot { pr: string; head: string; ready_at: string; screen: ScreenChoice }
|
||||
export interface TrainPrefix { index: number; commit: string; tree: string; pr: string | null }
|
||||
export interface FreeRun { run: number | null; url: string | null; head_sha: string | null; conclusion: string | null; verdict: 'green' | 'red' | 'missing' | 'stale' | 'cancelled' | 'pending' }
|
||||
export interface TrainRecord {
|
||||
schema: 1; kind: 'train'; nonce: string; key: string; repo_key: string; repo: string; base: { ref: string; sha: string }; policy: string;
|
||||
slots: TrainSlot[]; held: string[]; prefixes: TrainPrefix[]; head: string; tree: string; branch: string;
|
||||
urgent: boolean; repair: boolean; state: TrainState; free: FreeRun | null; attempts: string[];
|
||||
merged: Array<{ pr: string; sha: string; tree: string; at: string }>; superseded_by: string | null;
|
||||
effects: Array<{ at: string; what: string }>; error: string | null; created_at: string; updated_at: string;
|
||||
}
|
||||
export type AttemptPurpose = 'gate' | 'rebuild' | 'infra-redispatch' | 'bisect';
|
||||
export type GateOutcome = 'PENDING' | 'GREEN' | 'RED' | 'INCOMPLETE';
|
||||
export interface CasePanel { case: string; kind: string; status: string; red_class: string | null; cost_usd: number | null }
|
||||
export interface AttemptRecord {
|
||||
schema: 1; kind: 'train-attempt'; train: string; id: string; purpose: AttemptPurpose; nonce: string; workflow: string; ref: string;
|
||||
inputs: Record<string, string>; state: 'dispatching' | 'unknown' | 'absent' | 'dispatched' | 'completed'; run: { id: number; url: string } | null;
|
||||
outcome: GateOutcome | null; detail: string | null; red_cases: string[]; panels: CasePanel[]; infra_eligible: boolean;
|
||||
cost_usd: number | null; prefixes: number[]; created_at: string; updated_at: string;
|
||||
}
|
||||
export type PrefixResult = 'red' | 'green';
|
||||
export interface BisectRecord {
|
||||
schema: 1; kind: 'train-bisect'; train: string; cases: string[]; spend_cap: number; spent_usd: number;
|
||||
prefixes: Array<{ index: number; sha: string; results: Record<string, PrefixResult>; attempts: string[] }>;
|
||||
outcome: null | { kind: 'attributed' | 'red-on-main' | 'unattributed'; all_green: boolean; prs: string[]; patterns: Record<string, string>; at: string };
|
||||
updated_at: string;
|
||||
}
|
||||
export interface FreezeRecord {
|
||||
schema: 1; kind: 'freeze'; repo_key: string; train: string; merged: string[]; unmerged: Array<{ pr: string; head: string }>;
|
||||
main: { sha: string; tree: string }; reason: string; backstop: { nonce: string; run: number | null; state: string } | null;
|
||||
repair: { pr: string; head: string; train: string | null } | null; created_at: string; cleared_at: string | null; cleared_by: string | null; updated_at: string;
|
||||
}
|
||||
export interface ReadyReceipt {
|
||||
schema: 1; kind: 'ready'; repo_key: string; pr: string; head: string; base_sha: string; ready_at: string; screen: ScreenChoice; updated_at: string;
|
||||
}
|
||||
export type AnyRecord = TrainRecord | AttemptRecord | BisectRecord | FreezeRecord | ReadyReceipt;
|
||||
|
||||
/** Clock seam for fill-time and record tests (`GSTACK_SHIP_QUEUE_NOW`, ISO-8601). */
|
||||
export function nowIso(): string {
|
||||
const fixed = process.env.GSTACK_SHIP_QUEUE_NOW;
|
||||
return fixed && !Number.isNaN(Date.parse(fixed)) ? new Date(fixed).toISOString() : new Date().toISOString();
|
||||
}
|
||||
|
||||
/** An unexpected failure's cause, with secrets withheld, for the record and the error line. */
|
||||
export function redactCause(message: string): string {
|
||||
const text = message.replace(/\s+/g, ' ').slice(0, 400);
|
||||
return redactFindingSpans(text) ?? '(cause withheld: it could not be redacted safely)';
|
||||
}
|
||||
|
||||
export function queueRoot(env: NodeJS.ProcessEnv = process.env): string {
|
||||
return path.join(resolveStateRoot(env), 'ship-queue');
|
||||
}
|
||||
export const trainDir = (nonce: string) => path.join(queueRoot(), 'trains', nonce);
|
||||
export const trainFile = (nonce: string) => path.join(trainDir(nonce), 'train.json');
|
||||
const attemptPath = (nonce: string, id: string) => path.join(trainDir(nonce), 'attempts', `${id}.json`);
|
||||
const bisectPath = (nonce: string) => path.join(trainDir(nonce), 'bisect.json');
|
||||
const freezePath = (repoKey: string) => path.join(queueRoot(), 'freeze', `${repoKey}.json`);
|
||||
export const readyPath = (repoKey: string, pr: string, head: string) => path.join(queueRoot(), 'ready', repoKey, `${pr}-${head}.json`);
|
||||
|
||||
/** The repository's identity: origin's URL, keyed so records of different repositories never mix. */
|
||||
export function repoIdentity(repoRoot: string): { key: string; url: string } {
|
||||
const url = git(repoRoot, ['config', '--get', 'remote.origin.url']).out.trim() || repoRoot;
|
||||
const norm = url.replace(/\.git$/, '').replace(/^git@github\.com:/, 'https://github.com/').replace(/\/+$/, '');
|
||||
return { key: sha256(norm).slice(0, 16), url: norm };
|
||||
}
|
||||
|
||||
function readJson<T>(file: string, kind: string): T | null {
|
||||
try { const r = JSON.parse(fs.readFileSync(file, 'utf8')); return r?.schema === TRAIN_SCHEMA && r.kind === kind ? r as T : null; } catch { return null; }
|
||||
}
|
||||
|
||||
export const loadTrain = (nonce: string) => (/^[A-Za-z0-9._-]{1,128}$/.test(nonce) ? readJson<TrainRecord>(trainFile(nonce), 'train') : null);
|
||||
export const loadAttempt = (nonce: string, id: string) => (/^[A-Za-z0-9._-]{1,64}$/.test(id) ? readJson<AttemptRecord>(attemptPath(nonce, id), 'train-attempt') : null);
|
||||
export const loadBisect = (nonce: string) => readJson<BisectRecord>(bisectPath(nonce), 'train-bisect');
|
||||
export const loadFreeze = (repoKey: string) => readJson<FreezeRecord>(freezePath(repoKey), 'freeze');
|
||||
export const loadReady = (repoKey: string, pr: string, head: string) => readJson<ReadyReceipt>(readyPath(repoKey, pr, head), 'ready');
|
||||
export const activeFreeze = (repoKey: string): FreezeRecord | null => { const f = loadFreeze(repoKey); return f && !f.cleared_at ? f : null; };
|
||||
|
||||
export function attemptsOf(train: TrainRecord): AttemptRecord[] {
|
||||
return train.attempts.map(id => loadAttempt(train.nonce, id)).filter((a): a is AttemptRecord => !!a);
|
||||
}
|
||||
|
||||
/** Every train of this repository, newest first. */
|
||||
export function listTrains(repoKey: string): TrainRecord[] {
|
||||
const dir = path.join(queueRoot(), 'trains');
|
||||
if (!fs.existsSync(dir)) return [];
|
||||
return fs.readdirSync(dir).map(n => loadTrain(n)).filter((t): t is TrainRecord => !!t && t.repo_key === repoKey)
|
||||
.sort((a, b) => b.created_at.localeCompare(a.created_at) || b.nonce.localeCompare(a.nonce));
|
||||
}
|
||||
|
||||
export function recordPath(r: AnyRecord): string {
|
||||
if (r.kind === 'train') return trainFile(r.nonce);
|
||||
if (r.kind === 'train-attempt') return attemptPath(r.train, r.id);
|
||||
if (r.kind === 'train-bisect') return bisectPath(r.train);
|
||||
if (r.kind === 'freeze') return freezePath(r.repo_key);
|
||||
return readyPath(r.repo_key, r.pr, r.head);
|
||||
}
|
||||
|
||||
function noteName(r: AnyRecord): string {
|
||||
if (r.kind === 'train') return `gstack-trains:train:${r.nonce}`;
|
||||
if (r.kind === 'train-attempt') return `gstack-trains:attempt:${r.train}:${r.id}`;
|
||||
if (r.kind === 'train-bisect') return `gstack-trains:bisect:${r.train}`;
|
||||
if (r.kind === 'freeze') return `gstack-trains:freeze:${r.repo_key}`;
|
||||
return `gstack-trains:ready:${r.repo_key}:${r.pr}:${r.head}`;
|
||||
}
|
||||
|
||||
const NOTES_IDENTITY = { GIT_AUTHOR_NAME: 'gstack-ship-queue', GIT_AUTHOR_EMAIL: 'ship-queue@gstack.local', GIT_COMMITTER_NAME: 'gstack-ship-queue', GIT_COMMITTER_EMAIL: 'ship-queue@gstack.local' };
|
||||
const notesEnv = () => ({ ...process.env, ...NOTES_IDENTITY });
|
||||
|
||||
/** Fetch origin's notes and fold them into the local notes ref (the local side wins a same-record conflict). */
|
||||
export function fetchNotes(repoRoot: string, remote = 'origin'): { ok: boolean; detail: string } {
|
||||
const f = git(repoRoot, ['fetch', '--no-tags', '-q', remote, `+${NOTES_REF}:${NOTES_REMOTE_REF}`]);
|
||||
if (!f.ok) {
|
||||
const absent = /couldn't find remote ref|no such ref/i.test(f.err);
|
||||
return { ok: absent, detail: absent ? `${remote} has no ${NOTES_REF} yet` : `git fetch ${NOTES_REF} failed: ${f.err.slice(0, 160)}` };
|
||||
}
|
||||
const local = git(repoRoot, ['rev-parse', '--verify', '--quiet', NOTES_REF]).out;
|
||||
const fetched = git(repoRoot, ['rev-parse', '--verify', '--quiet', NOTES_REMOTE_REF]).out;
|
||||
if (!local) { git(repoRoot, ['update-ref', NOTES_REF, fetched]); return { ok: true, detail: 'notes fetched' }; }
|
||||
if (local === fetched || git(repoRoot, ['merge-base', '--is-ancestor', fetched, local]).ok) return { ok: true, detail: 'notes current' };
|
||||
const m = git(repoRoot, ['notes', `--ref=${NOTES_REF}`, 'merge', '-q', '-s', 'ours', NOTES_REMOTE_REF], notesEnv());
|
||||
return m.ok ? { ok: true, detail: 'notes merged' } : { ok: false, detail: `git notes merge failed: ${m.err.slice(0, 160)}` };
|
||||
}
|
||||
|
||||
function addNote(repoRoot: string, r: AnyRecord): boolean {
|
||||
const blob = git(repoRoot, ['hash-object', '-w', '--stdin'], undefined, noteName(r));
|
||||
if (!blob.ok) return false;
|
||||
return git(repoRoot, ['notes', `--ref=${NOTES_REF}`, 'add', '-f', '-F', '-', blob.out], notesEnv(), JSON.stringify(r)).ok;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write records locally, then as notes, then push the notes ref (fetch,
|
||||
* merge and retry on a race). A failed push leaves the local records and
|
||||
* notes for the next write to carry, and says so.
|
||||
*/
|
||||
export function persist(repoRoot: string, records: AnyRecord[], opts: { push?: boolean; remote?: string } = {}): { pushed: boolean; detail: string } {
|
||||
for (const r of records) writeJsonAtomic(recordPath(r), r);
|
||||
if (!records.every(r => addNote(repoRoot, r))) return { pushed: false, detail: 'could not write the notes locally' };
|
||||
if (opts.push === false) return { pushed: false, detail: 'notes written locally (not pushed)' };
|
||||
const remote = opts.remote ?? 'origin';
|
||||
let last = '';
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const p = git(repoRoot, ['push', '-q', remote, `${NOTES_REF}:${NOTES_REF}`]);
|
||||
if (p.ok) return { pushed: true, detail: `notes pushed to ${remote} ${NOTES_REF}` };
|
||||
last = p.err.slice(0, 160);
|
||||
const f = fetchNotes(repoRoot, remote);
|
||||
if (!f.ok) return { pushed: false, detail: f.detail };
|
||||
}
|
||||
return { pushed: false, detail: `git push ${NOTES_REF} failed: ${last}` };
|
||||
}
|
||||
|
||||
function newerOrMissing(r: AnyRecord): boolean {
|
||||
try {
|
||||
const cur = JSON.parse(fs.readFileSync(recordPath(r), 'utf8'));
|
||||
return !(typeof cur.updated_at === 'string' && cur.updated_at >= r.updated_at);
|
||||
} catch { return true; }
|
||||
}
|
||||
|
||||
function parseNote(text: string): AnyRecord | null {
|
||||
try {
|
||||
const r = JSON.parse(text);
|
||||
const ids = /^[A-Za-z0-9._-]{1,128}$/;
|
||||
if (r?.schema !== TRAIN_SCHEMA || typeof r.updated_at !== 'string') return null;
|
||||
if (r.kind === 'train' && ids.test(r.nonce)) return r;
|
||||
if ((r.kind === 'train-attempt' && ids.test(r.train) && ids.test(r.id)) || (r.kind === 'train-bisect' && ids.test(r.train))) return r;
|
||||
if (r.kind === 'freeze' && ids.test(r.repo_key)) return r;
|
||||
if (r.kind === 'ready' && ids.test(r.repo_key) && /^\d+$/.test(r.pr) && /^[0-9a-f]{40,64}$/.test(r.head)) return r;
|
||||
} catch {}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Rebuild local records from the notes ref: a note whose record is missing locally, or newer than the local copy, is written. */
|
||||
export function rebuildFromNotes(repoRoot: string, remote = 'origin'): { restored: number; read: number; detail: string } {
|
||||
const f = fetchNotes(repoRoot, remote);
|
||||
const list = git(repoRoot, ['notes', `--ref=${NOTES_REF}`, 'list']);
|
||||
if (!list.ok || !list.out.trim()) return { restored: 0, read: 0, detail: f.detail };
|
||||
const records = list.out.split('\n').map(l => l.split(' ')[0]!).filter(Boolean)
|
||||
.map(blob => parseNote(git(repoRoot, ['cat-file', 'blob', blob]).out)).filter((r): r is AnyRecord => !!r);
|
||||
const fresh = records.filter(newerOrMissing);
|
||||
for (const r of fresh) writeJsonAtomic(recordPath(r), r);
|
||||
return { restored: fresh.length, read: records.length, detail: f.detail };
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
/**
|
||||
* Train verdict, bisect and the one preregistered rebuild
|
||||
* (docs/ship-policy.md#run-a-merge-train; plan GSTA-49 lane T1, P3).
|
||||
*
|
||||
* The verdict reads the train's own gate run and the record artifact that run
|
||||
* publishes (with the `gstack/full-gate` check run) on the train head; it
|
||||
* only reads, never writes a check. Green needs a complete census from the
|
||||
* run's own plan manifest. A red train stays red: the bisect runs only its
|
||||
* red cases, one trial on every prefix T0..Tk, and attributes by pattern.
|
||||
* Rebuild-once (Garry, 2026-10-18; to be recorded in EVAL_POLICY beside
|
||||
* infraRedispatch): an all-green bisect earns one unchanged rebuild.
|
||||
*/
|
||||
import * as path from 'node:path';
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import { DISPATCH_INPUTS, listDispatchRuns, makeNonce, runForNonce } from '../scripts/eval-bg';
|
||||
import type { QueueContext } from './ship-queue';
|
||||
import { git } from './ship-queue-gate';
|
||||
import {
|
||||
checkRun, downloadJson, execFor, asRun, remoteBranchSha, repoSlug, runView, type CheckInfo, type RunInfo,
|
||||
} from './ship-queue-train-github';
|
||||
import {
|
||||
attemptsOf, loadBisect, nowIso, persist, trainDir, trainFile,
|
||||
type AttemptRecord, type BisectRecord, type CasePanel, type GateOutcome, type PrefixResult, type TrainRecord,
|
||||
} from './ship-queue-train-record';
|
||||
import {
|
||||
dispatchAttempt, effect, findTrain, gateDispatch, nextFor, outFailed, outOk, outPending, outRefused, pinList, save, trainEnv,
|
||||
type TrainEnv, type TrainOut,
|
||||
} from './ship-queue-train';
|
||||
|
||||
/** EVAL_POLICY.infraRedispatch (test/helpers/periodic-exclude-data.ts): one re-dispatch of an INFRA/INCOMPLETE-only census. */
|
||||
export const INFRA_REDISPATCH = 1;
|
||||
/** Rebuild-once (Garry, 2026-10-18): an all-green bisect earns exactly one unchanged rebuild; pinned to EVAL_POLICY by its test. */
|
||||
export const REBUILD_ONCE = 1;
|
||||
|
||||
const Q = 'gstack-ship-queue';
|
||||
|
||||
/** The record artifact the train's own gate run publishes (plan T1b), read by verdict, verify and the backstop. */
|
||||
export interface TrainGateRecord {
|
||||
schema: 1; kind: 'train-gate'; nonce: string; head_sha: string; tree: string; verdict: 'GREEN' | 'RED';
|
||||
census: { complete: boolean; planned: { rule: number; judge: number }; executed: { rule: number; judge: number };
|
||||
unbound: number; partial_shards: number; cancelled_shards: number; killed_before_upload: number };
|
||||
panels: CasePanel[];
|
||||
}
|
||||
/** One bisect prefix's artifact: the red cases on that prefix, one trial each. */
|
||||
export interface BisectPrefixRecord { schema: 1; kind: 'train-bisect-prefix'; nonce: string; prefix_sha: string; checked_out_sha: string; panels: CasePanel[] }
|
||||
|
||||
export const bisectArtifact = (recordArtifact: string, sha: string) => `${recordArtifact}-bisect-${sha.slice(0, 12)}`;
|
||||
|
||||
export interface GateClass { outcome: GateOutcome; detail: string; red: string[]; panels: CasePanel[]; infraEligible: boolean; cost: number | null }
|
||||
|
||||
/** Every paid-report class mapped to one train outcome; green only for a complete, bound, published census. */
|
||||
export function classifyGate(raw: unknown, expect: { nonce: string; head: string; tree: string }, run: Pick<RunInfo, 'status' | 'conclusion'>, check: CheckInfo | null): GateClass {
|
||||
const base = { red: [] as string[], panels: [] as CasePanel[], infraEligible: false, cost: null as number | null };
|
||||
if (run.status !== 'completed') return { ...base, outcome: 'PENDING', detail: `run is ${run.status}` };
|
||||
const rec = raw as TrainGateRecord | null;
|
||||
if (!rec || rec.schema !== 1 || rec.kind !== 'train-gate' || !Array.isArray(rec.panels) || !rec.census) {
|
||||
return { ...base, outcome: 'INCOMPLETE', infraEligible: true, detail: `no train record artifact (run concluded ${run.conclusion ?? 'unknown'}; a slice killed before its upload step counts here)` };
|
||||
}
|
||||
const panels = rec.panels;
|
||||
const known = panels.map(p => p.cost_usd).filter((c): c is number => typeof c === 'number');
|
||||
const cost = known.length ? Math.round(known.reduce((a, b) => a + b, 0) * 100) / 100 : null;
|
||||
const withPanels = { ...base, panels, cost };
|
||||
if (rec.nonce !== expect.nonce || rec.head_sha !== expect.head || rec.tree !== expect.tree) {
|
||||
return { ...withPanels, outcome: 'INCOMPLETE', detail: `the record is bound to nonce ${rec.nonce} head ${String(rec.head_sha).slice(0, 12)} tree ${String(rec.tree).slice(0, 12)}, not this train's attempt` };
|
||||
}
|
||||
const red = panels.filter(p => p.status === 'FAIL' && p.red_class === 'VERDICT').map(p => p.case).sort();
|
||||
if (red.length) return { ...withPanels, outcome: 'RED', red, detail: `${red.length} red case(s): ${red.join(', ')}` };
|
||||
const c = rec.census;
|
||||
const gaps = [
|
||||
...(c.planned.rule + c.planned.judge <= 0 ? ['the plan manifest lists no cases'] : []),
|
||||
...(c.executed.rule !== c.planned.rule ? [`rule ${c.executed.rule}/${c.planned.rule} executed`] : []),
|
||||
...(c.executed.judge !== c.planned.judge ? [`judge ${c.executed.judge}/${c.planned.judge} executed`] : []),
|
||||
...(c.unbound ? [`${c.unbound} unbound result`] : []), ...(c.partial_shards ? [`${c.partial_shards} partial shard`] : []),
|
||||
...(c.cancelled_shards ? [`${c.cancelled_shards} cancelled shard`] : []), ...(c.killed_before_upload ? [`${c.killed_before_upload} slice killed before upload`] : []),
|
||||
...(!c.complete ? ['census marked incomplete'] : []),
|
||||
...panels.filter(p => p.status !== 'PASS').map(p => `${p.case} ${p.status}${p.red_class ? ` (${p.red_class})` : ''}`),
|
||||
];
|
||||
if (gaps.length) return { ...withPanels, outcome: 'INCOMPLETE', infraEligible: !panels.some(p => p.status === 'SKIPPED'), detail: gaps.join('; ') };
|
||||
if (run.conclusion !== 'success' || rec.verdict !== 'GREEN') return { ...withPanels, outcome: 'INCOMPLETE', detail: `run concluded ${run.conclusion ?? 'unknown'} with record verdict ${rec.verdict}` };
|
||||
if (!check || check.conclusion !== 'success') return { ...withPanels, outcome: 'INCOMPLETE', detail: `the run did not publish a successful check run on the train head (${check ? `${check.status}/${check.conclusion}` : 'missing'})` };
|
||||
return { ...withPanels, outcome: 'GREEN', detail: `rule ${c.executed.rule}/${c.planned.rule}, judge ${c.executed.judge}/${c.planned.judge}` };
|
||||
}
|
||||
|
||||
const gateAttempts = (t: TrainRecord) => attemptsOf(t).filter(a => a.purpose !== 'bisect');
|
||||
const attemptLine = (a: AttemptRecord) => `ATTEMPT: ${a.id} ${a.purpose} run ${a.run?.id ?? '-'} ${a.outcome ?? a.state}${a.red_cases.length ? ` red=${a.red_cases.join(',')}` : ''}${a.cost_usd !== null ? ` cost=$${a.cost_usd.toFixed(2)}` : ''}${a.detail ? ` (${a.detail})` : ''}`;
|
||||
|
||||
function casePrice(env: TrainEnv, red: AttemptRecord, id: string): number {
|
||||
return red.panels.find(p => p.case === id)?.cost_usd ?? env.train.trial_reservation_usd;
|
||||
}
|
||||
|
||||
/** The whole-bisect reservation for a red attempt: every red case on every prefix. */
|
||||
export function bisectReservation(env: TrainEnv, t: TrainRecord, red: AttemptRecord): number {
|
||||
const perPrefix = red.red_cases.reduce((sum, id) => sum + casePrice(env, red, id), 0);
|
||||
return Math.ceil(perPrefix * t.prefixes.length * 100) / 100;
|
||||
}
|
||||
|
||||
function unknownDispatch(t: TrainRecord, a: AttemptRecord): TrainOut {
|
||||
return outPending({ code: 'QUEUE_TRAIN_DISPATCH_UNKNOWN', message: `the ${a.purpose} dispatch ${a.id} (nonce ${a.nonce}) is unconfirmed` }, [attemptLine(a)], `${Q} train status --train ${t.nonce}`, trainFile(t.nonce));
|
||||
}
|
||||
|
||||
/** `train verdict`: read the current gate attempt's run and record; print every attempt, so a rebuild reports both runs. */
|
||||
export function trainVerdict(ctx: QueueContext, o: { train: string }): TrainOut {
|
||||
const env = trainEnv(ctx);
|
||||
if ('code' in env) return outRefused(env, [], 'none', 'none');
|
||||
const t = findTrain(env, o.train);
|
||||
if (!t) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.train} here or in refs/notes/gstack-trains` }, [], `${Q} train status`, 'none');
|
||||
const run = trainFile(t.nonce);
|
||||
const a = gateAttempts(t).at(-1);
|
||||
if (!a) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `train ${t.nonce} has no gate dispatch yet` }, [], nextFor(t), run);
|
||||
if (a.state === 'dispatching' || a.state === 'unknown') return unknownDispatch(t, a);
|
||||
if (a.state === 'absent') return outRefused({ code: 'QUEUE_TRAIN_DISPATCH_UNKNOWN', message: `the dispatch ${a.nonce} never started (train status found no run)` }, [attemptLine(a)], `${Q} train build --train ${t.nonce}`, run);
|
||||
const slug = repoSlug(ctx);
|
||||
if (!a.outcome || a.outcome === 'PENDING') {
|
||||
const view = slug && a.run ? runView(ctx, slug, a.run.id) : null;
|
||||
if (!slug || !view) return outPending({ code: 'QUEUE_TRAIN_PENDING', message: `cannot read run ${a.run?.id} through gh` }, [attemptLine(a)], `${Q} train verdict --train ${t.nonce}`, run);
|
||||
const record = view.status === 'completed' ? downloadJson(ctx, slug, a.run!.id, env.train.record_artifact, path.join(trainDir(t.nonce), 'scratch')) : null;
|
||||
const check = view.status === 'completed' ? checkRun(ctx, t.head, env.train.full_gate_check) : null;
|
||||
const c = classifyGate(record, { nonce: a.nonce, head: t.head, tree: t.tree }, view, check);
|
||||
Object.assign(a, { outcome: c.outcome, detail: c.detail, red_cases: c.red, panels: c.panels, infra_eligible: c.infraEligible, cost_usd: c.cost, state: view.status === 'completed' ? 'completed' : a.state });
|
||||
if (c.outcome !== 'PENDING') {
|
||||
t.state = c.outcome === 'GREEN' ? 'green' : c.outcome === 'RED' ? (a.purpose === 'rebuild' ? 'unattributed' : 'red') : 'incomplete';
|
||||
effect(t, `verdict ${a.id}: ${c.outcome}`);
|
||||
}
|
||||
save(env, t, a);
|
||||
}
|
||||
const lines = [...gateAttempts(t).map(attemptLine), `VERDICT: ${a.outcome} train ${t.nonce} run ${a.run?.id ?? '-'} head ${t.head.slice(0, 12)} tree ${t.tree.slice(0, 12)}`];
|
||||
if (a.outcome === 'PENDING') return outPending({ code: 'QUEUE_TRAIN_PENDING', message: `the gate run ${a.run?.id} has not finished (${a.detail})` }, lines, `${Q} train verdict --train ${t.nonce}`, run);
|
||||
if (a.outcome === 'GREEN') return outOk(lines, nextFor(t), run);
|
||||
if (a.outcome === 'RED') {
|
||||
const next = a.purpose === 'rebuild'
|
||||
? `second red stays red: route ${a.red_cases.join(',')} to a repair lane (GStack Minder), then ${Q} train build --prs <pins without the PRs held>`
|
||||
: `${Q} train bisect --train ${t.nonce} --spend-cap ${bisectReservation(env, t, a).toFixed(2)}`;
|
||||
return outFailed({ code: 'QUEUE_TRAIN_RED', message: `train ${t.nonce} is red on ${a.red_cases.join(', ')}` }, lines, next, run);
|
||||
}
|
||||
const used = gateAttempts(t).filter(x => x.purpose === 'infra-redispatch').length;
|
||||
const next = a.infra_eligible && used < INFRA_REDISPATCH
|
||||
? `${Q} train rebuild --train ${t.nonce} --infra (the single re-dispatch EVAL_POLICY.infraRedispatch allows; report both runs)`
|
||||
: `diagnose run ${a.run?.url ?? a.run?.id}; the train stays incomplete (GStack Minder routes it)`;
|
||||
return outFailed({ code: 'QUEUE_TRAIN_INCOMPLETE', message: `train ${t.nonce} gate is incomplete: ${a.detail}` }, lines, next, run);
|
||||
}
|
||||
|
||||
export interface BisectClass { kind: 'attributed' | 'red-on-main' | 'unattributed'; all_green: boolean; prs: string[]; patterns: Record<string, string> }
|
||||
|
||||
/**
|
||||
* Pattern attribution over T0..Tk (one trial each, so only shapes count):
|
||||
* monotone green-then-red from Ti blames slot i's PR; red everywhere,
|
||||
* T0 included, is main; all green, or anything else, is unattributed.
|
||||
*/
|
||||
export function classifyBisect(cases: string[], results: Array<Record<string, PrefixResult>>, slots: Array<{ pr: string }>): BisectClass {
|
||||
const patterns: Record<string, string> = {};
|
||||
const blamed = new Set<string>();
|
||||
let main = false; let other = false; let green = 0;
|
||||
for (const c of cases) {
|
||||
const seq = results.map(r => (r[c] === 'red' ? 'R' : 'G')).join('');
|
||||
patterns[c] = seq;
|
||||
const first = seq.indexOf('R');
|
||||
if (first < 0) green++;
|
||||
else if (first === 0 && !seq.includes('G')) main = true;
|
||||
else if (first > 0 && !seq.slice(first).includes('G')) blamed.add(slots[first - 1]!.pr);
|
||||
else other = true;
|
||||
}
|
||||
if (main) return { kind: 'red-on-main', all_green: false, prs: [], patterns };
|
||||
if (other || !blamed.size) return { kind: 'unattributed', all_green: green === cases.length, prs: [], patterns };
|
||||
return { kind: 'attributed', all_green: false, prs: [...blamed].sort((a, b) => Number(a) - Number(b)), patterns };
|
||||
}
|
||||
|
||||
function prefixResults(raw: unknown, sha: string, nonce: string, cases: string[]): { results: Record<string, PrefixResult>; cost: number } | null {
|
||||
const rec = raw as BisectPrefixRecord | null;
|
||||
if (!rec || rec.schema !== 1 || rec.kind !== 'train-bisect-prefix' || rec.nonce !== nonce || rec.prefix_sha !== sha || rec.checked_out_sha !== sha || !Array.isArray(rec.panels)) return null;
|
||||
const results: Record<string, PrefixResult> = {};
|
||||
let cost = 0;
|
||||
for (const p of rec.panels) {
|
||||
if (!cases.includes(p.case)) continue;
|
||||
cost += typeof p.cost_usd === 'number' ? p.cost_usd : 0;
|
||||
if (p.status === 'PASS') results[p.case] = 'green';
|
||||
else if (p.status === 'FAIL' && p.red_class === 'VERDICT') results[p.case] = 'red';
|
||||
}
|
||||
return { results, cost };
|
||||
}
|
||||
|
||||
/** Collect a finished bisect dispatch's prefix artifacts into the ledger; a still-running one returns its run state. */
|
||||
function collect(env: TrainEnv, slug: string, t: TrainRecord, b: BisectRecord, a: AttemptRecord): string | null {
|
||||
const view = a.run ? runView(env.ctx, slug, a.run.id) : null;
|
||||
if (!view) return 'unreadable through gh';
|
||||
if (view.status !== 'completed') return view.status;
|
||||
let got = 0;
|
||||
for (const index of a.prefixes) {
|
||||
const p = b.prefixes[index]!;
|
||||
const r = prefixResults(downloadJson(env.ctx, slug, a.run!.id, bisectArtifact(env.train.record_artifact, p.sha), path.join(trainDir(t.nonce), 'scratch')), p.sha, a.nonce, b.cases);
|
||||
if (!r) continue;
|
||||
Object.assign(p.results, r.results);
|
||||
b.spent_usd = Math.round((b.spent_usd + r.cost) * 100) / 100;
|
||||
got++;
|
||||
}
|
||||
Object.assign(a, { state: 'completed', detail: `collected ${got} of ${a.prefixes.length} prefix result(s)` });
|
||||
return null;
|
||||
}
|
||||
|
||||
function bisectOutcome(env: TrainEnv, t: TrainRecord, b: BisectRecord): TrainOut {
|
||||
const c = classifyBisect(b.cases, b.prefixes.map(p => p.results), t.slots);
|
||||
b.outcome = { ...c, at: nowIso() };
|
||||
t.state = c.kind;
|
||||
effect(t, `bisect: ${c.kind}${c.prs.length ? ' #' + c.prs.join(',#') : ''}`);
|
||||
b.updated_at = nowIso();
|
||||
persist(env.ctx.repoRoot, [b], { remote: env.remote });
|
||||
save(env, t);
|
||||
const run = trainFile(t.nonce);
|
||||
const lines = [...Object.entries(c.patterns).map(([id, s]) => `PATTERN: ${id} ${s.split('').map((x, i) => `T${i}=${x}`).join(' ')}`), `BISECT: spent $${b.spent_usd.toFixed(2)} of cap $${b.spend_cap.toFixed(2)}`];
|
||||
if (c.kind === 'attributed') return outOk([...lines, `BISECT: ATTRIBUTED ${c.prs.map(p => '#' + p).join(',')}`], nextFor(t), run);
|
||||
if (c.kind === 'red-on-main') return outFailed({ code: 'QUEUE_TRAIN_RED_ON_MAIN', message: `${b.cases.join(', ')} red on every prefix, T0 (main) included` }, [...lines, 'BISECT: RED_ON_MAIN'], nextFor(t), run);
|
||||
return outFailed({ code: 'QUEUE_TRAIN_UNATTRIBUTED', message: c.all_green ? 'every prefix was green: the red is unattributed; recovery: the one preregistered rebuild' : 'the pattern is not monotone: recovery: a repair lane, then a changed train' },
|
||||
[...lines, `BISECT: UNATTRIBUTED (${c.all_green ? 'all green' : 'not monotone'})`], nextFor(t), run);
|
||||
}
|
||||
|
||||
/** Validate bisect inputs: red cases from the gate's own plan, every prefix an ancestor of train/<nonce>'s head. */
|
||||
function bisectInputs(env: TrainEnv, t: TrainRecord, red: AttemptRecord): string | null {
|
||||
for (const id of red.red_cases) {
|
||||
if (!/^[A-Za-z0-9_.:/-]{1,128}$/.test(id) || !red.panels.some(p => p.case === id)) return `red case ${id} is not in the gate's plan`;
|
||||
}
|
||||
const ref = `refs/gstack/${t.branch}`;
|
||||
const f = git(env.ctx.repoRoot, ['fetch', '--no-tags', '-q', env.remote, `+refs/heads/${t.branch}:${ref}`]);
|
||||
if (!f.ok) return `cannot fetch ${t.branch}: ${f.err.slice(0, 120)}`;
|
||||
const tip = git(env.ctx.repoRoot, ['rev-parse', ref]).out;
|
||||
if (tip !== t.head) return `${t.branch} is at ${tip.slice(0, 12)}, not the recorded train head ${t.head.slice(0, 12)}`;
|
||||
for (const p of t.prefixes) {
|
||||
if (!git(env.ctx.repoRoot, ['merge-base', '--is-ancestor', p.commit, tip]).ok) return `prefix T${p.index} ${p.commit.slice(0, 12)} is not an ancestor of ${t.branch}`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* `train bisect`: resumable and spend-capped. Each run reconciles the earlier
|
||||
* dispatch by nonce, records every prefix result it published, then reserves
|
||||
* the remaining case×prefix trials at their recorded price (unknown at the
|
||||
* policy reservation) and launches only the prefixes that fit under the cap.
|
||||
*/
|
||||
export async function trainBisect(ctx: QueueContext, o: { train: string; spendCap: number; dryRun: boolean }): Promise<TrainOut> {
|
||||
const env = trainEnv(ctx);
|
||||
if ('code' in env) return outRefused(env, [], 'none', 'none');
|
||||
const t = findTrain(env, o.train);
|
||||
if (!t) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.train} here or in refs/notes/gstack-trains` }, [], `${Q} train status`, 'none');
|
||||
const run = trainFile(t.nonce);
|
||||
const red = gateAttempts(t).at(-1);
|
||||
if (!red || red.outcome !== 'RED') return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `bisect needs a red verdict; the train's last gate is ${red?.outcome ?? 'missing'}` }, [], nextFor(t), run);
|
||||
if (red.purpose === 'rebuild') return outRefused({ code: 'QUEUE_PROOF_MISSING', message: 'a rebuild\'s second red is not bisected again' }, [], nextFor(t), run);
|
||||
const prior = loadBisect(t.nonce);
|
||||
if (prior?.outcome) return bisectOutcome(env, t, prior);
|
||||
const bad = bisectInputs(env, t, red);
|
||||
if (bad) return outRefused({ code: 'QUEUE_TRAIN_PREFIX_INVALID', message: bad }, [], `${Q} train status --train ${t.nonce}`, run);
|
||||
const b: BisectRecord = prior ?? { schema: 1, kind: 'train-bisect', train: t.nonce, cases: red.red_cases, spend_cap: o.spendCap, spent_usd: 0,
|
||||
prefixes: t.prefixes.map(p => ({ index: p.index, sha: p.commit, results: {}, attempts: [] })), outcome: null, updated_at: nowIso() };
|
||||
b.spend_cap = o.spendCap;
|
||||
const slug = repoSlug(ctx);
|
||||
if (!slug) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: 'gh repo view could not name the repository' }, [], `${Q} train bisect --train ${t.nonce} --spend-cap ${o.spendCap}`, run);
|
||||
for (const a of attemptsOf(t).filter(x => x.purpose === 'bisect' && x.state !== 'completed' && x.state !== 'absent')) {
|
||||
if (a.state === 'dispatching' || a.state === 'unknown') {
|
||||
const hit = runForNonce(listDispatchRuns(a.workflow, execFor(ctx), slug, 100) ?? [], a.nonce);
|
||||
if (!hit) return unknownDispatch(t, a);
|
||||
Object.assign(a, { state: 'dispatched', run: asRun(hit) });
|
||||
}
|
||||
const waiting = collect(env, slug, t, b, a);
|
||||
save(env, t, a);
|
||||
persist(ctx.repoRoot, [b], { remote: env.remote });
|
||||
if (waiting) return outPending({ code: 'QUEUE_TRAIN_PENDING', message: `bisect run ${a.run?.id} is ${waiting}` }, [attemptLine(a)], `${Q} train bisect --train ${t.nonce} --spend-cap ${o.spendCap}`, run);
|
||||
}
|
||||
const pending = b.prefixes.filter(p => b.cases.some(c => !p.results[c]));
|
||||
if (!pending.length) return bisectOutcome(env, t, b);
|
||||
const perPrefix = b.cases.reduce((sum, id) => sum + casePrice(env, red, id), 0);
|
||||
const admitted: typeof pending = [];
|
||||
let reserved = b.spent_usd;
|
||||
for (const p of pending) if (reserved + perPrefix <= o.spendCap + 1e-9) { admitted.push(p); reserved += perPrefix; }
|
||||
const higher = Math.ceil((b.spent_usd + perPrefix * pending.length) * 100) / 100;
|
||||
const lines = [`BISECT: cases ${b.cases.join(',')} prefixes ${pending.map(p => 'T' + p.index).join(',')} need results; $${perPrefix.toFixed(2)} reserved per prefix; spent $${b.spent_usd.toFixed(2)}, cap $${o.spendCap.toFixed(2)}`,
|
||||
`BISECT: launch ${admitted.map(p => 'T' + p.index).join(',') || '(none)'}${admitted.length < pending.length ? `; ${pending.length - admitted.length} prefix(es) wait for a higher cap` : ''}`];
|
||||
if (o.dryRun) return outOk([...lines, 'DRY-RUN: nothing dispatched'], `${Q} train bisect --train ${t.nonce} --spend-cap ${o.spendCap}`, run);
|
||||
persist(ctx.repoRoot, [b], { remote: env.remote });
|
||||
if (!admitted.length) {
|
||||
return outPending({ code: 'QUEUE_TRAIN_BISECT_CAPPED', message: `the next prefix needs $${perPrefix.toFixed(2)}; $${(o.spendCap - b.spent_usd).toFixed(2)} of the cap is left; the outcome stays unattributed` }, lines,
|
||||
`${Q} train bisect --train ${t.nonce} --spend-cap ${higher.toFixed(2)}`, run);
|
||||
}
|
||||
const nonce = makeNonce('gate', t.head, t.base.sha, randomBytes(3).toString('hex'), { purpose: 'bisect' });
|
||||
const a = await dispatchAttempt(env, slug, t, 'bisect', { workflow: env.train.gate_workflow, inputs: {
|
||||
[DISPATCH_INPUTS.cases]: b.cases.join(','), [DISPATCH_INPUTS.prefixShas]: admitted.map(p => p.sha).join(','),
|
||||
[DISPATCH_INPUTS.expectedSha]: t.head, [DISPATCH_INPUTS.nonce]: nonce,
|
||||
} }, admitted.map(p => p.index));
|
||||
for (const p of admitted) p.attempts.push(a.id);
|
||||
b.updated_at = nowIso();
|
||||
persist(ctx.repoRoot, [b], { remote: env.remote });
|
||||
if (a.state !== 'dispatched') return unknownDispatch(t, a);
|
||||
lines.push(attemptLine(a));
|
||||
if (admitted.length < pending.length) {
|
||||
return outPending({ code: 'QUEUE_TRAIN_BISECT_CAPPED', message: `${pending.length - admitted.length} prefix(es) are past the cap; completed prefixes are kept and the outcome stays unattributed` }, lines,
|
||||
`${Q} train bisect --train ${t.nonce} --spend-cap ${higher.toFixed(2)}`, run);
|
||||
}
|
||||
return outOk(lines, `${Q} train bisect --train ${t.nonce} --spend-cap ${o.spendCap} (collects run ${a.run?.id} once it finishes)`, run);
|
||||
}
|
||||
|
||||
/**
|
||||
* `train rebuild`: the unchanged train dispatched once more. Without
|
||||
* `--infra`, only after an all-green bisect and only once (rebuild-once);
|
||||
* with `--infra`, only after an INFRA/INCOMPLETE-only census and at most
|
||||
* EVAL_POLICY.infraRedispatch times. Both runs stay in the record.
|
||||
*/
|
||||
export async function trainRebuild(ctx: QueueContext, o: { train: string; infra: boolean }): Promise<TrainOut> {
|
||||
const env = trainEnv(ctx);
|
||||
if ('code' in env) return outRefused(env, [], 'none', 'none');
|
||||
const t = findTrain(env, o.train);
|
||||
if (!t) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.train} here or in refs/notes/gstack-trains` }, [], `${Q} train status`, 'none');
|
||||
const run = trainFile(t.nonce);
|
||||
const gates = gateAttempts(t);
|
||||
const last = gates.at(-1);
|
||||
const repair = `route ${last?.red_cases.join(',') || 'the red cases'} to a repair lane (GStack Minder); the train is rebuilt only once the repair merges: ${Q} train build --prs ${pinList(t.slots)}`;
|
||||
const refuse = (reason: string) => outRefused({ code: 'QUEUE_TRAIN_REBUILD_REFUSED', message: `rebuild refused: ${reason}` }, gates.map(attemptLine), repair, run);
|
||||
if (o.infra) {
|
||||
if (!last || last.outcome !== 'INCOMPLETE' || !last.infra_eligible) return refuse(`no INFRA/INCOMPLETE-only record (the last gate is ${last?.outcome ?? 'missing'})`);
|
||||
if (gates.filter(a => a.purpose === 'infra-redispatch').length >= INFRA_REDISPATCH) return refuse('already re-dispatched once (EVAL_POLICY.infraRedispatch)');
|
||||
} else {
|
||||
const b = loadBisect(t.nonce);
|
||||
if (!b?.outcome) return refuse('no bisect record');
|
||||
if (gates.filter(a => a.purpose === 'rebuild').length >= REBUILD_ONCE) return refuse('already rebuilt once');
|
||||
if (b.outcome.kind !== 'unattributed' || !b.outcome.all_green) return refuse(`the bisect was not all green (${Object.entries(b.outcome.patterns).map(([c, s]) => `${c}=${s}`).join(' ')})`);
|
||||
}
|
||||
const tip = remoteBranchSha(args => git(ctx.repoRoot, args), env.remote, t.branch);
|
||||
if (tip !== t.head) return outRefused({ code: 'QUEUE_TRAIN_MOVED', message: `${t.branch} is at ${tip?.slice(0, 12) ?? 'nothing'}, not the recorded head ${t.head.slice(0, 12)}` }, [], `${Q} train build --prs ${pinList(t.slots)}`, run);
|
||||
const slug = repoSlug(ctx);
|
||||
if (!slug) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: 'gh repo view could not name the repository' }, [], `${Q} train rebuild --train ${t.nonce}${o.infra ? ' --infra' : ''}`, run);
|
||||
const a = await dispatchAttempt(env, slug, t, o.infra ? 'infra-redispatch' : 'rebuild', gateDispatch(env, t, `${t.nonce}.${gates.length + 1}`));
|
||||
t.state = a.state === 'dispatched' ? 'gate-pending' : 'dispatch-unknown';
|
||||
save(env, t);
|
||||
if (a.state !== 'dispatched') return unknownDispatch(t, a);
|
||||
return outOk([...gateAttempts(t).map(attemptLine), `REBUILD: ${a.purpose} of the unchanged train ${t.nonce} (head ${t.head.slice(0, 12)}); both runs are reported`], nextFor(t), run);
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
/**
|
||||
* `verify --pr <n> --train <nonce>`: the coordinator's merge check under
|
||||
* merge trains (docs/ship-policy.md#run-a-merge-train; plan GSTA-49 T1, P1).
|
||||
*
|
||||
* Each merge needs a green train record that includes the PR at its pinned
|
||||
* head, with main's tree equal to the prefix before the PR (pre-merge:
|
||||
* MERGE_CONDITION) or after it (post-merge confirmation), modulo stamps
|
||||
* (lib/tree-receipt.ts). When a train stops partway (a mismatch or a moved
|
||||
* head after earlier slots merged) the queue freezes: the freeze is written
|
||||
* locally and to the notes ref first, then the backstop gate is dispatched on
|
||||
* main's published prefix at once. One repair train is admitted under the
|
||||
* freeze; verify clears the freeze when main after that merge carries it.
|
||||
*/
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import { DISPATCH_INPUTS, awaitNonceRun, dispatchWorkflow, makeNonce } from '../scripts/eval-bg';
|
||||
import type { QueueContext } from './ship-queue';
|
||||
import { git } from './ship-queue-gate';
|
||||
import { checkRun, execFor, repoSlug } from './ship-queue-train-github';
|
||||
import { activeFreeze, attemptsOf, nowIso, persist, trainFile, type FreezeRecord, type TrainRecord } from './ship-queue-train-record';
|
||||
import {
|
||||
effect, fetchBase, fetchTrainBranch, findTrain, frozenLines, nextFor, outFailed, outOk, outRefused, pinList, save, trainEnv, type TrainEnv, type TrainOut,
|
||||
} from './ship-queue-train';
|
||||
import { buildReceipt } from './tree-receipt';
|
||||
|
||||
const Q = 'gstack-ship-queue';
|
||||
|
||||
function sameModuloStamps(env: TrainEnv, gated: string, current: string): { same: boolean; detail: string } {
|
||||
const r = buildReceipt({ repoRoot: env.ctx.repoRoot, gated, current, policy: env.policy.policy, policyLabel: env.policy.source.label });
|
||||
if ('error' in r) return { same: false, detail: r.error };
|
||||
return { same: r.tree === 'same-modulo-stamps', detail: r.tree === 'same-modulo-stamps' ? `same modulo stamps (${r.files.length} stamp file(s))` : `differs: ${r.residual.slice(0, 8).join(', ')}` };
|
||||
}
|
||||
|
||||
/**
|
||||
* Freeze the queue: the record goes to local state and the notes ref before
|
||||
* anything remote, then the backstop gate is dispatched on main's sha.
|
||||
*/
|
||||
async function freezeQueue(env: TrainEnv, t: TrainRecord, reason: string, mainSha: string): Promise<FreezeRecord> {
|
||||
const now = nowIso();
|
||||
const f: FreezeRecord = {
|
||||
schema: 1, kind: 'freeze', repo_key: env.repo.key, train: t.nonce, merged: t.merged.map(m => m.pr),
|
||||
unmerged: t.slots.filter(s => !t.merged.some(m => m.pr === s.pr)).map(s => ({ pr: s.pr, head: s.head })),
|
||||
main: { sha: mainSha, tree: git(env.ctx.repoRoot, ['rev-parse', `${mainSha}^{tree}`]).out }, reason, backstop: null, repair: null,
|
||||
created_at: now, cleared_at: null, cleared_by: null, updated_at: now,
|
||||
};
|
||||
persist(env.ctx.repoRoot, [f], { remote: env.remote });
|
||||
effect(t, `queue frozen: ${reason}`);
|
||||
save(env, t);
|
||||
const slug = repoSlug(env.ctx);
|
||||
const nonce = makeNonce('gate', mainSha, mainSha, randomBytes(3).toString('hex'), { purpose: 'backstop' });
|
||||
const dispatch = { workflow: env.train.gate_workflow, inputs: { [DISPATCH_INPUTS.evalsAll]: 'true', [DISPATCH_INPUTS.expectedSha]: mainSha, [DISPATCH_INPUTS.nonce]: nonce } };
|
||||
const exec = execFor(env.ctx);
|
||||
const found = slug && dispatchWorkflow(dispatch, env.ctx.base, exec, slug) ? await awaitNonceRun(dispatch.workflow, nonce, exec, slug) : null;
|
||||
f.backstop = { nonce, run: found?.databaseId ?? null, state: found ? 'dispatched' : 'unconfirmed' };
|
||||
f.updated_at = nowIso();
|
||||
persist(env.ctx.repoRoot, [f], { remote: env.remote });
|
||||
return f;
|
||||
}
|
||||
|
||||
/** Interrupted publication: freeze when earlier slots merged; otherwise a plain refusal (nothing was published). */
|
||||
async function stopped(env: TrainEnv, t: TrainRecord, code: 'QUEUE_TRAIN_TREE_MISMATCH' | 'QUEUE_TRAIN_MOVED', message: string, mainSha: string, lines: string[]): Promise<TrainOut> {
|
||||
const run = trainFile(t.nonce);
|
||||
if (!t.merged.length) return outFailed({ code, message: `${message}; nothing of train ${t.nonce} merged, so the queue is not frozen` }, lines, `${Q} train build --prs ${pinList(t.slots)} (a new train on current main)`, run);
|
||||
const f = activeFreeze(env.repo.key) ?? await freezeQueue(env, t, message, mainSha);
|
||||
return outFailed({ code, message: `${message}; the queue is frozen` }, [...lines, ...frozenLines(f)], `${Q} train build --repair --prs <repair-pr>@<sha>`, run);
|
||||
}
|
||||
|
||||
export async function verifyTrain(ctx: QueueContext, o: { pr: string; train: string; noFetch: boolean }): Promise<TrainOut> {
|
||||
const fetchErr = o.noFetch ? null : fetchBase(ctx);
|
||||
const env = trainEnv(ctx);
|
||||
if ('code' in env) return outRefused(env, [], 'none', 'none');
|
||||
if (fetchErr) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: fetchErr }, [], 'none', 'none');
|
||||
const t = findTrain(env, o.train);
|
||||
if (!t) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.train} here or in refs/notes/gstack-trains` }, [], `${Q} train status`, 'none');
|
||||
const run = trainFile(t.nonce);
|
||||
const i = t.slots.findIndex(s => s.pr === o.pr);
|
||||
if (i < 0) return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} does not include #${o.pr}` }, [], nextFor(t), run);
|
||||
const gate = attemptsOf(t).filter(a => a.purpose !== 'bisect').at(-1);
|
||||
if (gate?.outcome !== 'GREEN' || !['green', 'merging', 'merged'].includes(t.state)) {
|
||||
return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} has no green record (state ${t.state}, last gate ${gate?.outcome ?? 'none'})` }, [], nextFor(t), run);
|
||||
}
|
||||
const check = checkRun(ctx, t.head, env.train.full_gate_check);
|
||||
if (check?.conclusion !== 'success') return outRefused({ code: 'QUEUE_GATE_MISSING', message: `no successful ${env.train.full_gate_check} check run on the train head ${t.head.slice(0, 12)}` }, [], `${Q} train verdict --train ${t.nonce}`, run);
|
||||
if (!fetchTrainBranch(env, t)) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `the train's prefix commits are not here and ${t.branch} could not be fetched` }, [], `${Q} train status --train ${t.nonce}`, run);
|
||||
const main = git(ctx.repoRoot, ['rev-parse', `${env.remote}/${ctx.base}`]).out;
|
||||
const freeze = activeFreeze(env.repo.key);
|
||||
if (freeze && freeze.repair?.train !== t.nonce) return outRefused({ code: 'QUEUE_TRAIN_FROZEN', message: `the queue is frozen (${freeze.reason}); only the admitted repair train merges` }, frozenLines(freeze), `${Q} train build --repair --prs <repair-pr>@<sha>`, run);
|
||||
const after = t.prefixes[i + 1]!;
|
||||
const before = t.prefixes[i]!;
|
||||
const post = sameModuloStamps(env, after.commit, main);
|
||||
const lines = [`TRAIN: ${t.nonce} slot ${i + 1}/${t.slots.length} #${o.pr} pinned ${t.slots[i]!.head}`];
|
||||
if (post.same) return confirmMerged(env, t, o.pr, main, after.index, [...lines, `main: ${main} = T${after.index} (${post.detail})`]);
|
||||
const pre = sameModuloStamps(env, before.commit, main);
|
||||
if (!pre.same) {
|
||||
return stopped(env, t, 'QUEUE_TRAIN_TREE_MISMATCH', `main ${main.slice(0, 12)} is neither T${before.index} nor T${after.index} modulo stamps (vs T${after.index}: ${post.detail})`, main, lines);
|
||||
}
|
||||
const headRef = o.noFetch ? '' : (git(ctx.repoRoot, ['fetch', '--no-tags', '-q', env.remote, `+refs/pull/${o.pr}/head:refs/gstack/queue/pr-${o.pr}`]).ok ? `refs/gstack/queue/pr-${o.pr}` : '');
|
||||
const head = headRef ? git(ctx.repoRoot, ['rev-parse', headRef]).out : t.slots[i]!.head;
|
||||
const stampOnly = head === t.slots[i]!.head ? { same: true, detail: 'the pinned head' } : sameModuloStamps(env, t.slots[i]!.head, head);
|
||||
if (!stampOnly.same) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${o.pr} moved past its pinned head ${t.slots[i]!.head.slice(0, 12)} (${head.slice(0, 12)}: ${stampOnly.detail})`, main, lines);
|
||||
return outOk([...lines, `main: ${main} = T${before.index} (${pre.detail})`, `head: ${head} (${stampOnly.detail})`,
|
||||
`VERIFY_TRAIN: #${o.pr} may merge as slot ${i + 1} of train ${t.nonce} (green run ${gate.run?.id ?? '-'}, ${env.train.full_gate_check} success)`, `MERGE_CONDITION: sha=${head}`],
|
||||
`merge #${o.pr} with expected head ${head.slice(0, 12)} (gstack-restamp stamps at merge), then ${Q} verify --pr ${o.pr} --train ${t.nonce}`, run);
|
||||
}
|
||||
|
||||
function confirmMerged(env: TrainEnv, t: TrainRecord, pr: string, main: string, index: number, lines: string[]): TrainOut {
|
||||
const run = trainFile(t.nonce);
|
||||
if (!t.merged.some(m => m.pr === pr)) t.merged.push({ pr, sha: main, tree: git(env.ctx.repoRoot, ['rev-parse', `${main}^{tree}`]).out, at: nowIso() });
|
||||
t.state = t.merged.length === t.slots.length ? 'merged' : 'merging';
|
||||
effect(t, `verified #${pr} merged as T${index}`);
|
||||
save(env, t);
|
||||
lines.push(`VERIFY_TRAIN: #${pr} merged; main's tree is T${index} of train ${t.nonce} (${t.merged.length}/${t.slots.length} merged)`);
|
||||
const freeze = activeFreeze(env.repo.key);
|
||||
if (freeze && freeze.repair?.train === t.nonce && t.state === 'merged') {
|
||||
freeze.cleared_at = nowIso();
|
||||
freeze.cleared_by = `repair #${pr} via train ${t.nonce}; main ${main.slice(0, 12)} carries a green record`;
|
||||
freeze.updated_at = freeze.cleared_at;
|
||||
persist(env.ctx.repoRoot, [freeze], { remote: env.remote });
|
||||
lines.push(`FREEZE: cleared (${freeze.cleared_by})`);
|
||||
return outOk(lines, freeze.unmerged.length ? `${Q} train build --prs ${pinList(freeze.unmerged)}` : 'none', run);
|
||||
}
|
||||
return outOk(lines, nextFor(t), run);
|
||||
}
|
||||
@@ -0,0 +1,509 @@
|
||||
/**
|
||||
* Merge trains for gstack-ship-queue (docs/ship-policy.md#run-a-merge-train;
|
||||
* plan GSTA-49 lane T1): `train build`, `train status` and `train prune`,
|
||||
* plus the pieces the verdict, bisect, rebuild and verify modules share.
|
||||
* The surface is active only when the base revision's ship policy declares a
|
||||
* `train` block (QUEUE_TRAIN_OFF otherwise).
|
||||
*
|
||||
* A train is main plus each READY PR's pinned, unstamped head merged in slot
|
||||
* order. The merges run in object storage (`git merge-tree --write-tree`, the
|
||||
* gate-ahead merge rules: version files take the earlier prefix, the
|
||||
* CHANGELOG merges by section), so each prefix T0..Tk is a commit with a
|
||||
* fixed identity and date and the same pins always give the same train head.
|
||||
* Records (lib/ship-queue-train-record.ts) are written before every remote
|
||||
* effect: the branch push, the free-tests dispatch, the paid dispatch through
|
||||
* scripts/eval-bg.ts's nonce.
|
||||
*/
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import { DISPATCH_INPUTS, awaitNonceRun, dispatchWorkflow, listDispatchRuns, makeNonce, runForNonce, type WorkflowDispatch } from '../scripts/eval-bg';
|
||||
import { mergeChangelogs, parseChangelog, type ChangelogSection } from './changelog-check';
|
||||
import type { ResultStatus } from './headless-artifacts';
|
||||
import { versionAtRef, versionFiles } from './restamp';
|
||||
import type { LoadedPolicy, ShipPolicy, TrainPolicy } from './ship-policy';
|
||||
import { loadBasePolicy, type QueueContext } from './ship-queue';
|
||||
import { git, sha256, withIndex, type QueueFailure } from './ship-queue-gate';
|
||||
import { asRun, execFor, newestBranchRun, prInfo, remoteBranchSha, repoSlug, runView, checkRun } from './ship-queue-train-github';
|
||||
import {
|
||||
TERMINAL_STATES, activeFreeze, attemptsOf, listTrains, loadBisect, loadReady, loadTrain, nowIso, persist, rebuildFromNotes, repoIdentity, trainFile,
|
||||
type AttemptPurpose, type AttemptRecord, type TrainRecord, type TrainSlot,
|
||||
} from './ship-queue-train-record';
|
||||
|
||||
export interface TrainOut { exit: number; status: ResultStatus; lines: string[]; next: string; failure?: QueueFailure; run: string }
|
||||
export const outOk = (lines: string[], next: string, run: string): TrainOut => ({ exit: 0, status: 'complete', lines, next, run });
|
||||
export const outRefused = (failure: QueueFailure, lines: string[], next: string, run: string): TrainOut => ({ exit: 3, status: 'refused', lines, next, failure, run });
|
||||
export const outFailed = (failure: QueueFailure, lines: string[], next: string, run: string): TrainOut => ({ exit: 1, status: 'refused', lines, next, failure, run });
|
||||
export const outPending = (failure: QueueFailure, lines: string[], next: string, run: string): TrainOut => ({ exit: 3, status: 'incomplete', lines, next, failure, run });
|
||||
|
||||
export interface TrainEnv { ctx: QueueContext; policy: LoadedPolicy; train: TrainPolicy; repo: { key: string; url: string }; remote: string }
|
||||
|
||||
const HEX = /^[0-9a-f]{40,64}$/;
|
||||
const Q = 'gstack-ship-queue';
|
||||
|
||||
/** The policy gate: the base revision's ship policy must declare a `train` block. */
|
||||
export function trainEnv(ctx: QueueContext): TrainEnv | QueueFailure {
|
||||
const policy = loadBasePolicy(ctx);
|
||||
if ('message' in policy) return policy;
|
||||
if (!policy.policy.train) return { code: 'QUEUE_TRAIN_OFF', message: `${policy.source.label} declares no train block; the per-PR queue applies` };
|
||||
return { ctx, policy, train: policy.policy.train, repo: repoIdentity(ctx.repoRoot), remote: ctx.remote ?? 'origin' };
|
||||
}
|
||||
|
||||
export function fetchBase(ctx: QueueContext): string | null {
|
||||
const remote = ctx.remote ?? 'origin';
|
||||
const f = git(ctx.repoRoot, ['fetch', '--no-tags', '-q', remote, `+refs/heads/${ctx.base}:refs/remotes/${remote}/${ctx.base}`]);
|
||||
return f.ok ? null : `git fetch ${remote} ${ctx.base} failed: ${f.err.slice(0, 160)}`;
|
||||
}
|
||||
|
||||
/** Make the train's prefix commits local (another machine built them): fetch train/<nonce>. */
|
||||
export function fetchTrainBranch(env: TrainEnv, train: TrainRecord): string | null {
|
||||
if (git(env.ctx.repoRoot, ['cat-file', '-e', `${train.head}^{commit}`]).ok) return train.head;
|
||||
const f = git(env.ctx.repoRoot, ['fetch', '--no-tags', '-q', env.remote, `+refs/heads/${train.branch}:refs/gstack/${train.branch}`]);
|
||||
return f.ok ? git(env.ctx.repoRoot, ['rev-parse', `refs/gstack/${train.branch}`]).out : null;
|
||||
}
|
||||
|
||||
/** A train record from local state, or from the notes ref when this machine has none. */
|
||||
export function findTrain(env: TrainEnv, nonce: string): TrainRecord | null {
|
||||
const local = loadTrain(nonce);
|
||||
if (local) return local;
|
||||
rebuildFromNotes(env.ctx.repoRoot, env.remote);
|
||||
return loadTrain(nonce);
|
||||
}
|
||||
|
||||
export const touch = <T extends { updated_at: string }>(r: T): T => { r.updated_at = nowIso(); return r; };
|
||||
|
||||
export function effect(train: TrainRecord, what: string): void {
|
||||
train.effects.push({ at: nowIso(), what });
|
||||
}
|
||||
|
||||
/** Save the train (and any attempt) locally and to the notes ref; a failed notes push is recorded as an effect, never hidden. */
|
||||
export function save(env: TrainEnv, train: TrainRecord, ...extra: AttemptRecord[]): string {
|
||||
touch(train);
|
||||
for (const a of extra) touch(a);
|
||||
const r = persist(env.ctx.repoRoot, [train, ...extra], { remote: env.remote });
|
||||
if (!r.pushed) { effect(train, `notes not pushed: ${r.detail}`); persist(env.ctx.repoRoot, [train], { push: false }); }
|
||||
return r.detail;
|
||||
}
|
||||
|
||||
export function parsePins(spec: string | undefined): Array<{ pr: string; sha: string }> | string {
|
||||
if (!spec) return 'needs --prs <n>@<sha>[,<n>@<sha>...]';
|
||||
const pins = spec.split(',').map(s => s.trim()).filter(Boolean).map(s => /^#?(\d+)@([0-9a-f]{7,64})$/.exec(s));
|
||||
if (!pins.length || pins.some(m => !m)) return `--prs entries are <pr>@<sha> (got ${spec})`;
|
||||
const out = pins.map(m => ({ pr: m![1]!, sha: m![2]! }));
|
||||
return new Set(out.map(p => p.pr)).size === out.length ? out : 'a PR appears twice in --prs';
|
||||
}
|
||||
|
||||
export const pinList = (slots: Array<{ pr: string; head: string }>) => slots.map(s => `${s.pr}@${s.head}`).join(',');
|
||||
|
||||
/** Slot order: screened PRs first, then unscreened, each in READY order; enough unscreened PRs form their own train. */
|
||||
export function orderSlots(slots: TrainSlot[], max: number): { train: TrainSlot[]; held: TrainSlot[]; split: boolean } {
|
||||
const byReady = (a: TrainSlot, b: TrainSlot) => a.ready_at.localeCompare(b.ready_at) || Number(a.pr) - Number(b.pr);
|
||||
const screened = slots.filter(s => s.screen.choice === 'screen').sort(byReady);
|
||||
const unscreened = slots.filter(s => s.screen.choice !== 'screen').sort(byReady);
|
||||
if (screened.length && unscreened.length >= max) return { train: screened.slice(0, max), held: [...screened.slice(max), ...unscreened], split: true };
|
||||
const all = [...screened, ...unscreened];
|
||||
return { train: all.slice(0, max), held: all.slice(max), split: false };
|
||||
}
|
||||
|
||||
/** When a train under its size may dispatch: its oldest READY plus the fill time; null when it may dispatch now. */
|
||||
export function fillDue(slots: TrainSlot[], train: TrainPolicy, now: string, urgent: boolean): string | null {
|
||||
if (urgent || slots.length >= train.max_size) return null;
|
||||
const oldest = Math.min(...slots.map(s => Date.parse(s.ready_at)));
|
||||
const due = oldest + train.fill_hours * 3_600_000;
|
||||
return Date.parse(now) >= due ? null : new Date(due).toISOString();
|
||||
}
|
||||
|
||||
function unionChangelog(ours: string, theirs: string): string {
|
||||
const a = parseChangelog(ours);
|
||||
const b = parseChangelog(theirs);
|
||||
const entry = (s: ChangelogSection) => s.lines.slice(1).join('\n').replace(/\n+$/, '');
|
||||
const known = new Set(b.sections.map(entry));
|
||||
const header = [...b.header];
|
||||
while (header.length && header[header.length - 1] === '') header.pop();
|
||||
const body = [...a.sections.filter(s => !known.has(entry(s))), ...b.sections].flatMap(s => {
|
||||
const lines = [...s.lines];
|
||||
while (lines.length && lines[lines.length - 1] === '') lines.pop();
|
||||
return [...lines, ''];
|
||||
});
|
||||
return [...header, ...(header.length ? [''] : []), ...body].join('\n').replace(/\n+$/, '\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* One train merge, in object storage: the PR head onto the previous prefix.
|
||||
* Version files take the prefix's side; the CHANGELOG merges by section (the
|
||||
* gate-ahead rule, else a union of sections, since each heading is restamped
|
||||
* at merge). Any other conflict is reported with every conflicting path.
|
||||
*/
|
||||
export function mergeStep(repoRoot: string, pred: string, head: string, policy: ShipPolicy): { tree: string } | { conflicts: string[] } {
|
||||
const r = git(repoRoot, ['merge-tree', '--write-tree', head, pred]);
|
||||
const lines = r.out.split('\n');
|
||||
const tree = lines[0]?.trim() ?? '';
|
||||
if (r.status === 0 && HEX.test(tree)) return { tree };
|
||||
if (r.status !== 1 || !HEX.test(tree)) return { conflicts: [`(git merge-tree failed: ${(r.err || r.out).slice(0, 120)})`] };
|
||||
const stages = new Map<string, Map<number, string>>();
|
||||
for (const line of lines.slice(1)) {
|
||||
if (!line.trim()) break;
|
||||
const m = /^(\d{6}) ([0-9a-f]{40,64}) ([123])\t(.+)$/.exec(line);
|
||||
if (!m) continue;
|
||||
const entry = stages.get(m[4]!) ?? new Map<number, string>();
|
||||
entry.set(Number(m[3]), m[2]!);
|
||||
stages.set(m[4]!, entry);
|
||||
}
|
||||
const at = versionAtRef(repoRoot, pred);
|
||||
const stampFiles = versionFiles(repoRoot, 'rel' in at ? at.rel : 'VERSION');
|
||||
const resolved: Array<{ path: string; blob: string }> = [];
|
||||
const conflicts: string[] = [];
|
||||
for (const [file, entry] of stages) {
|
||||
const ours = entry.get(2);
|
||||
const theirs = entry.get(3);
|
||||
if (!ours || !theirs) conflicts.push(file);
|
||||
else if (file === policy.changelog) {
|
||||
const text = (sha: string) => git(repoRoot, ['cat-file', 'blob', sha]).out + '\n';
|
||||
const merged = mergeChangelogs(text(ours), text(theirs)) ?? unionChangelog(text(ours), text(theirs));
|
||||
const blob = git(repoRoot, ['hash-object', '-w', '--stdin'], undefined, merged);
|
||||
if (blob.ok) resolved.push({ path: file, blob: blob.out }); else conflicts.push(file);
|
||||
} else if (stampFiles.includes(file)) resolved.push({ path: file, blob: theirs });
|
||||
else conflicts.push(file);
|
||||
}
|
||||
if (conflicts.length) return { conflicts: conflicts.sort() };
|
||||
const out = withIndex<{ tree: string } | { error: string }>(repoRoot, tree, index => {
|
||||
for (const { path: p, blob } of resolved) {
|
||||
const mode = git(repoRoot, ['ls-tree', tree, '--', p], index).out.split(/\s/)[0] || '100644';
|
||||
if (!git(repoRoot, ['update-index', '--add', '--cacheinfo', `${mode},${blob},${p}`], index).ok) return { error: `cannot stage ${p}` };
|
||||
}
|
||||
const w = git(repoRoot, ['write-tree'], index);
|
||||
return w.ok ? { tree: w.out } : { error: `write-tree failed: ${w.err}` };
|
||||
});
|
||||
return 'error' in out ? { conflicts: [`(${out.error})`] } : { tree: out.tree };
|
||||
}
|
||||
|
||||
const TRAIN_IDENTITY = { GIT_AUTHOR_NAME: 'gstack-ship-queue', GIT_AUTHOR_EMAIL: 'ship-queue@gstack.local', GIT_COMMITTER_NAME: 'gstack-ship-queue',
|
||||
GIT_COMMITTER_EMAIL: 'ship-queue@gstack.local', GIT_AUTHOR_DATE: '2000-01-01T00:00:00Z', GIT_COMMITTER_DATE: '2000-01-01T00:00:00Z' };
|
||||
|
||||
/** Build T1..Tk on T0 = base; a conflict stops at the PR that conflicts. */
|
||||
export function buildChain(repoRoot: string, baseSha: string, slots: TrainSlot[], policy: ShipPolicy):
|
||||
{ prefixes: TrainRecord['prefixes'] } | { stoppedAt: string; conflicts: string[]; merged: string[] } {
|
||||
const prefixes: TrainRecord['prefixes'] = [{ index: 0, commit: baseSha, tree: git(repoRoot, ['rev-parse', `${baseSha}^{tree}`]).out, pr: null }];
|
||||
for (const [i, slot] of slots.entries()) {
|
||||
const pred = prefixes[i]!.commit;
|
||||
const m = mergeStep(repoRoot, pred, slot.head, policy);
|
||||
if ('conflicts' in m) return { stoppedAt: slot.pr, conflicts: m.conflicts, merged: slots.slice(0, i).map(s => s.pr) };
|
||||
const c = git(repoRoot, ['commit-tree', m.tree, '-p', pred, '-p', slot.head, '-m', `train T${i + 1}: merge #${slot.pr} at ${slot.head.slice(0, 12)}`], { ...process.env, ...TRAIN_IDENTITY });
|
||||
if (!c.ok) return { stoppedAt: slot.pr, conflicts: [`(commit-tree failed: ${c.err.slice(0, 120)})`], merged: slots.slice(0, i).map(s => s.pr) };
|
||||
prefixes.push({ index: i + 1, commit: c.out, tree: m.tree, pr: slot.pr });
|
||||
}
|
||||
return { prefixes };
|
||||
}
|
||||
|
||||
/** NEXT for a train in its current state. */
|
||||
export function nextFor(t: TrainRecord): string {
|
||||
const n = t.nonce;
|
||||
const unmerged = t.slots.filter(s => !t.merged.some(m => m.pr === s.pr));
|
||||
switch (t.state) {
|
||||
case 'built': case 'pushed': case 'free-pending': case 'free-green': case 'dispatch-absent': return `${Q} train build --train ${n}`;
|
||||
case 'dispatch-unknown': return `${Q} train status --train ${n}`;
|
||||
case 'gate-pending': return `${Q} train verdict --train ${n}`;
|
||||
case 'green': case 'merging': return unmerged.length ? `${Q} verify --pr ${unmerged[0]!.pr} --train ${n}` : `${Q} train prune --train ${n}`;
|
||||
case 'red': return `${Q} train bisect --train ${n} --spend-cap <usd>`;
|
||||
case 'incomplete': return `${Q} train verdict --train ${n}`;
|
||||
case 'attributed': {
|
||||
const drop = loadBisect(n)?.outcome?.prs ?? [];
|
||||
const keep = t.slots.filter(s => !drop.includes(s.pr));
|
||||
return keep.length ? `${Q} train build --prs ${pinList(keep)}` : 'none (every PR was dropped)';
|
||||
}
|
||||
case 'red-on-main': return `fix forward on main first, then ${Q} train build --prs ${pinList(t.slots)}`;
|
||||
case 'unattributed': return loadBisect(n)?.outcome?.all_green && !attemptsOf(t).some(a => a.purpose === 'rebuild')
|
||||
? `${Q} train rebuild --train ${n}`
|
||||
: `route the red cases to a repair lane (GStack Minder); once the repair merges: ${Q} train build --prs ${pinList(t.slots)}`;
|
||||
default: return `${Q} train prune --train ${n}`;
|
||||
}
|
||||
}
|
||||
|
||||
export function attemptId(train: TrainRecord, purpose: AttemptPurpose): string {
|
||||
const bisect = purpose === 'bisect';
|
||||
return `${bisect ? 'b' : 'a'}${train.attempts.filter(id => id.startsWith(bisect ? 'b' : 'a')).length + 1}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record an attempt as dispatching (locally and in the notes ref), dispatch
|
||||
* through eval-bg, then find the run by its nonce. A failed or unconfirmed
|
||||
* dispatch is `unknown`: train status reconciles it, nothing re-dispatches it.
|
||||
*/
|
||||
export async function dispatchAttempt(env: TrainEnv, slug: string, train: TrainRecord, purpose: AttemptPurpose, dispatch: WorkflowDispatch, prefixes: number[] = []): Promise<AttemptRecord> {
|
||||
const now = nowIso();
|
||||
const attempt: AttemptRecord = {
|
||||
schema: 1, kind: 'train-attempt', train: train.nonce, id: attemptId(train, purpose), purpose, nonce: dispatch.inputs[DISPATCH_INPUTS.nonce]!,
|
||||
workflow: dispatch.workflow, ref: train.branch, inputs: dispatch.inputs, state: 'dispatching', run: null, outcome: null, detail: null,
|
||||
red_cases: [], panels: [], infra_eligible: false, cost_usd: null, prefixes, created_at: now, updated_at: now,
|
||||
};
|
||||
train.attempts.push(attempt.id);
|
||||
save(env, train, attempt);
|
||||
const exec = execFor(env.ctx);
|
||||
const found = dispatchWorkflow(dispatch, train.branch, exec, slug) ? await awaitNonceRun(dispatch.workflow, attempt.nonce, exec, slug) : null;
|
||||
attempt.state = found ? 'dispatched' : 'unknown';
|
||||
attempt.run = found ? asRun(found) : null;
|
||||
effect(train, found ? `${purpose} dispatched: ${dispatch.workflow} run ${found.databaseId} (nonce ${attempt.nonce})` : `${purpose} dispatch unconfirmed (nonce ${attempt.nonce})`);
|
||||
save(env, train, attempt);
|
||||
return attempt;
|
||||
}
|
||||
|
||||
export function gateDispatch(env: TrainEnv, train: TrainRecord, nonce: string): WorkflowDispatch {
|
||||
return { workflow: env.train.gate_workflow, inputs: { [DISPATCH_INPUTS.evalsAll]: 'true', [DISPATCH_INPUTS.expectedSha]: train.head, [DISPATCH_INPUTS.nonce]: nonce } };
|
||||
}
|
||||
|
||||
export interface BuildOptions { pins?: string; resume?: string; dryRun: boolean; urgent: boolean; repair: boolean; freeTimeoutSec: number; pollMs: number }
|
||||
|
||||
/** The slots a new train would carry: trusted, unmoved, READY PRs, in slot order. */
|
||||
function admitSlots(env: TrainEnv, pins: Array<{ pr: string; sha: string }>): { slots: TrainSlot[]; lines: string[] } | TrainOut {
|
||||
const { ctx } = env;
|
||||
const specs = pins.map(p => `+refs/pull/${p.pr}/head:refs/gstack/train/pr-${p.pr}`);
|
||||
const f = git(ctx.repoRoot, ['fetch', '--no-tags', '-q', env.remote, ...specs]);
|
||||
if (!f.ok) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `git fetch of the PR heads failed: ${f.err.slice(0, 160)}` }, [], 'none', 'none');
|
||||
const slots: TrainSlot[] = [];
|
||||
for (const p of pins) {
|
||||
const info = prInfo(ctx, p.pr);
|
||||
if ('error' in info) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: info.error }, [], 'none', 'none');
|
||||
const without = pinList(pins.filter(x => x.pr !== p.pr).map(x => ({ pr: x.pr, head: x.sha })));
|
||||
if (info.isCrossRepository) return outRefused({ code: 'QUEUE_TRAIN_UNTRUSTED', message: `PR #${p.pr}'s head repository is not the base repository; fork work stays on the trusted-branch path` }, [], without ? `${Q} train build --prs ${without}` : 'none', 'none');
|
||||
const fetched = git(ctx.repoRoot, ['rev-parse', `refs/gstack/train/pr-${p.pr}`]).out;
|
||||
if (!info.headRefOid.startsWith(p.sha) || fetched !== info.headRefOid) {
|
||||
return outRefused({ code: 'QUEUE_TRAIN_MOVED', message: `PR #${p.pr} moved: pinned ${p.sha.slice(0, 12)}, head is now ${info.headRefOid.slice(0, 12)}` }, [], `ask #${p.pr}'s lane for READY on ${info.headRefOid.slice(0, 12)}, then ${Q} train build --prs <pins>`, 'none');
|
||||
}
|
||||
const ready = loadReady(env.repo.key, p.pr, info.headRefOid);
|
||||
if (!ready) return outRefused({ code: 'QUEUE_NOT_READY', message: `no READY receipt for #${p.pr} at ${info.headRefOid.slice(0, 12)} (local state or ${'refs/notes/gstack-trains'})` }, [], `the lane runs ${Q} ready --pr ${p.pr} --screen|--no-screen`, 'none');
|
||||
slots.push({ pr: p.pr, head: info.headRefOid, ready_at: ready.ready_at, screen: ready.screen });
|
||||
}
|
||||
return { slots, lines: [] };
|
||||
}
|
||||
|
||||
/** Start a new train from --prs, or find the live train with the same pins and base. */
|
||||
async function newTrain(env: TrainEnv, o: BuildOptions): Promise<{ t: TrainRecord; lines: string[] } | TrainOut> {
|
||||
const pins = parsePins(o.pins);
|
||||
if (typeof pins === 'string') return outRefused({ code: 'QUEUE_PROOF_MISSING', message: pins }, [], 'none', 'none');
|
||||
if ((o.urgent || o.repair) && pins.length !== 1) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `${o.repair ? '--repair' : '--urgent'} admits exactly one PR` }, [], 'none', 'none');
|
||||
rebuildFromNotes(env.ctx.repoRoot, env.remote);
|
||||
const freeze = activeFreeze(env.repo.key);
|
||||
if (freeze && !o.repair) return outRefused({ code: 'QUEUE_TRAIN_FROZEN', message: `the queue is frozen since ${freeze.created_at}: train ${freeze.train} stopped after merging ${freeze.merged.map(p => '#' + p).join(',')} (${freeze.reason})` },
|
||||
frozenLines(freeze), `${Q} train build --repair --prs <repair-pr>@<sha>`, 'none');
|
||||
const admitted = admitSlots(env, pins);
|
||||
if ('exit' in admitted) return admitted;
|
||||
const order = orderSlots(admitted.slots, o.repair ? 1 : env.train.max_size);
|
||||
const lines = [
|
||||
`ORDER: ${order.train.map((s, i) => `${i + 1}:#${s.pr}(${s.screen.choice === 'screen' ? 'screened' : 'unscreened'})`).join(' ')}`,
|
||||
...(order.held.length ? [`HELD: ${order.held.map(s => '#' + s.pr).join(',')} (${order.split ? 'unscreened PRs form their own train' : `over the train size ${env.train.max_size}`})`] : []),
|
||||
];
|
||||
const due = fillDue(order.train, env.train, nowIso(), o.urgent || o.repair);
|
||||
if (due && !o.dryRun) return outPending({ code: 'QUEUE_TRAIN_WAITING', message: `${order.train.length} of ${env.train.max_size} PRs; the oldest READY dispatches at ${due}` }, lines, `${Q} train build --prs ${pinList(order.train)} (after ${due}, or with more PRs, or --urgent for one)`, 'none');
|
||||
const baseSha = git(env.ctx.repoRoot, ['rev-parse', `${env.remote}/${env.ctx.base}`]).out;
|
||||
const key = sha256(JSON.stringify({ base: baseSha, slots: order.train.map(s => `${s.pr}@${s.head}`) }));
|
||||
const live = listTrains(env.repo.key).find(t => t.key === key && !TERMINAL_STATES.includes(t.state) && t.state !== 'red' && t.state !== 'unattributed');
|
||||
if (live) return { t: Object.assign(live, { held: order.held.map(s => s.pr) }), lines };
|
||||
const chain = buildChain(env.ctx.repoRoot, baseSha, order.train, env.policy.policy);
|
||||
if ('stoppedAt' in chain) {
|
||||
const without = order.train.filter(s => s.pr !== chain.stoppedAt);
|
||||
return outFailed({ code: 'QUEUE_TRAIN_CONFLICT', message: `#${chain.stoppedAt} conflicts with the prefix before it` }, [...lines,
|
||||
`QUEUE_TRAIN_CONFLICT: train ${key.slice(0, 12)} stopped at #${chain.stoppedAt} (conflicts: ${chain.conflicts.join(', ')}) after merging ${chain.merged.map(p => '#' + p).join(',') || '(none)'}; nothing dispatched, no branch pushed; fix: ${Q} train build --prs ${pinList(without)}`],
|
||||
without.length ? `${Q} train build --prs ${pinList(without)}` : 'none', 'none');
|
||||
}
|
||||
const head = chain.prefixes.at(-1)!;
|
||||
const nonce = makeNonce('gate', head.commit, baseSha, randomBytes(3).toString('hex'), { purpose: 'train' });
|
||||
const now = nowIso();
|
||||
return { lines, t: {
|
||||
schema: 1, kind: 'train', nonce, key, repo_key: env.repo.key, repo: env.repo.url, base: { ref: env.ctx.base, sha: baseSha }, policy: env.policy.source.label,
|
||||
slots: order.train, held: order.held.map(s => s.pr), prefixes: chain.prefixes, head: head.commit, tree: head.tree, branch: `train/${nonce}`,
|
||||
urgent: o.urgent, repair: o.repair, state: 'built', free: null, attempts: [], merged: [], superseded_by: null, effects: [], error: null,
|
||||
created_at: now, updated_at: now,
|
||||
} };
|
||||
}
|
||||
|
||||
export function frozenLines(f: NonNullable<ReturnType<typeof activeFreeze>>): string[] {
|
||||
return [
|
||||
`QUEUE_TRAIN_FROZEN: train ${f.train} merged ${f.merged.map(p => '#' + p).join(',') || '(none)'}; main ${f.main.sha.slice(0, 12)} tree ${f.main.tree.slice(0, 12)}; backstop ${f.backstop ? `${f.backstop.state} run ${f.backstop.run ?? 'unknown'} (nonce ${f.backstop.nonce})` : 'not dispatched'}`,
|
||||
`freeze: ${f.reason}; unmerged ${f.unmerged.map(u => '#' + u.pr).join(',') || '(none)'}; repair ${f.repair ? `#${f.repair.pr}@${f.repair.head.slice(0, 12)}` : 'none admitted'}`,
|
||||
];
|
||||
}
|
||||
|
||||
function describe(t: TrainRecord): string[] {
|
||||
return [
|
||||
`TRAIN: ${t.nonce} state=${t.state} prs=${t.slots.map(s => '#' + s.pr).join(',')} head=${t.head} tree=${t.tree}`,
|
||||
`base: origin/${t.base.ref}@${t.base.sha}`,
|
||||
...t.prefixes.map(p => `PREFIX: T${p.index} ${p.commit} tree=${p.tree}${p.pr ? ` (+#${p.pr})` : ' (main)'}`),
|
||||
...t.slots.map((s, i) => `SLOT: ${i + 1} #${s.pr} head=${s.head} ${s.screen.choice === 'screen' ? `screened (${s.screen.policy ?? 'narrow'}${s.screen.run ? `, run ${s.screen.run}` : ''})` : 'unscreened'} ready_at=${s.ready_at}`),
|
||||
];
|
||||
}
|
||||
|
||||
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
|
||||
|
||||
/** Free tests on train/<nonce>: dispatched once, then followed until a verdict on the exact train head. */
|
||||
async function freePhase(env: TrainEnv, slug: string, t: TrainRecord, o: BuildOptions): Promise<TrainOut | null> {
|
||||
if (t.free?.verdict === 'green' && t.free.head_sha === t.head) return null;
|
||||
const run = trainFile(t.nonce);
|
||||
if (t.state !== 'free-pending') {
|
||||
if (!env.ctx.gh(['workflow', 'run', env.train.free_workflow, '--repo', slug, '--ref', t.branch]).ok) {
|
||||
t.free = { run: null, url: null, head_sha: null, conclusion: null, verdict: 'missing' };
|
||||
t.state = 'free-red';
|
||||
effect(t, `${env.train.free_workflow} dispatch refused by gh`);
|
||||
save(env, t);
|
||||
return outFailed({ code: 'QUEUE_TRAIN_FREE_RED', message: `gh could not dispatch ${env.train.free_workflow} on ${t.branch}` }, describe(t), `${Q} train build --prs ${pinList(t.slots)}`, run);
|
||||
}
|
||||
t.state = 'free-pending';
|
||||
effect(t, `${env.train.free_workflow} dispatched on ${t.branch}`);
|
||||
save(env, t);
|
||||
}
|
||||
const deadline = Date.now() + o.freeTimeoutSec * 1000;
|
||||
let found = newestBranchRun(env.ctx, slug, env.train.free_workflow, t.branch);
|
||||
while ((found === null || found === 'unreadable' || found.status !== 'completed') && Date.now() < deadline) {
|
||||
await sleep(o.pollMs);
|
||||
found = newestBranchRun(env.ctx, slug, env.train.free_workflow, t.branch);
|
||||
}
|
||||
if (found && found !== 'unreadable' && found.status !== 'completed') {
|
||||
t.free = { run: found.databaseId, url: found.url, head_sha: found.headSha, conclusion: null, verdict: 'pending' };
|
||||
save(env, t);
|
||||
return outPending({ code: 'QUEUE_TRAIN_PENDING', message: `free tests run ${found.databaseId} on ${t.branch} is still ${found.status}` }, describe(t), `${Q} train build --train ${t.nonce}`, run);
|
||||
}
|
||||
const r = found && found !== 'unreadable' ? found : null;
|
||||
const verdict = !r ? 'missing' : r.event !== 'workflow_dispatch' || r.headSha !== t.head ? 'stale' : r.conclusion === 'success' ? 'green' : r.conclusion === 'cancelled' ? 'cancelled' : 'red';
|
||||
t.free = { run: r?.databaseId ?? null, url: r?.url ?? null, head_sha: r?.headSha ?? null, conclusion: r?.conclusion ?? null, verdict };
|
||||
t.state = verdict === 'green' ? 'free-green' : 'free-red';
|
||||
save(env, t);
|
||||
if (verdict === 'green') return null;
|
||||
const why = verdict === 'missing' ? `no ${env.train.free_workflow} dispatch run appeared on ${t.branch}`
|
||||
: verdict === 'stale' ? `run ${r!.databaseId} tested ${(r!.headSha || '?').slice(0, 12)} (${r!.event}), not the train head ${t.head.slice(0, 12)}`
|
||||
: `run ${r!.databaseId} concluded ${r!.conclusion}`;
|
||||
return outFailed({ code: 'QUEUE_TRAIN_FREE_RED', message: `free tests ${verdict}: ${why}` }, [...describe(t), `FREE: ${verdict} ${r?.url ?? ''}`.trim()],
|
||||
`read the free-tests run; drop a PR that caused it: ${Q} train build --prs <pins without it>`, run);
|
||||
}
|
||||
|
||||
/** Push the branch, gate it with the free tests, then dispatch the paid gate once. */
|
||||
async function advance(env: TrainEnv, t: TrainRecord, o: BuildOptions, lines: string[]): Promise<TrainOut> {
|
||||
const run = trainFile(t.nonce);
|
||||
const slug = repoSlug(env.ctx);
|
||||
if (!slug) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: 'gh repo view could not name the repository' }, lines, `${Q} train build --train ${t.nonce}`, run);
|
||||
if (t.state === 'built') {
|
||||
const p = git(env.ctx.repoRoot, ['push', '-q', env.remote, `${t.head}:refs/heads/${t.branch}`]);
|
||||
if (!p.ok) { t.error = `branch push failed: ${p.err.slice(0, 160)}`; save(env, t); return outRefused({ code: 'QUEUE_PROOF_MISSING', message: t.error }, lines, `${Q} train build --train ${t.nonce}`, run); }
|
||||
t.state = 'pushed';
|
||||
effect(t, `branch pushed: ${t.branch} at ${t.head.slice(0, 12)}`);
|
||||
save(env, t);
|
||||
}
|
||||
const free = await freePhase(env, slug, t, o);
|
||||
if (free) return { ...free, lines: [...lines, ...free.lines] };
|
||||
if (t.state === 'dispatch-unknown') return outPending({ code: 'QUEUE_TRAIN_DISPATCH_UNKNOWN', message: `the gate dispatch for ${t.nonce} is unconfirmed` }, [...lines, ...describe(t)], `${Q} train status --train ${t.nonce}`, run);
|
||||
if (t.state === 'free-green' || t.state === 'dispatch-absent') {
|
||||
const prior = attemptsOf(t).filter(a => a.purpose === 'gate');
|
||||
const nonce = prior.at(-1)?.state === 'absent' ? prior.at(-1)!.nonce : t.nonce;
|
||||
const a = await dispatchAttempt(env, slug, t, 'gate', gateDispatch(env, t, nonce));
|
||||
t.state = a.state === 'dispatched' ? 'gate-pending' : 'dispatch-unknown';
|
||||
save(env, t);
|
||||
if (a.state !== 'dispatched') return outPending({ code: 'QUEUE_TRAIN_DISPATCH_UNKNOWN', message: `no ${a.workflow} run named ${a.nonce} appeared after the dispatch` }, [...lines, ...describe(t)], `${Q} train status --train ${t.nonce}`, run);
|
||||
}
|
||||
const gate = attemptsOf(t).filter(a => a.purpose !== 'bisect').at(-1);
|
||||
return outOk([...lines, ...describe(t), `FREE: green run ${t.free?.run ?? '?'} on ${t.head.slice(0, 12)}`,
|
||||
`GATE: ${gate?.workflow ?? env.train.gate_workflow} run ${gate?.run?.id ?? '?'} ${gate?.run?.url ?? ''} nonce ${gate?.nonce ?? t.nonce}`.trim(), `record: ${run}`], nextFor(t), run);
|
||||
}
|
||||
|
||||
/** `train build`: admit, order, merge, record, push, free tests, dispatch; `--dry-run` spends nothing and writes nothing. */
|
||||
export async function buildTrain(ctx: QueueContext, o: BuildOptions): Promise<TrainOut> {
|
||||
const fetchErr = fetchBase(ctx);
|
||||
const env = trainEnv(ctx);
|
||||
if ('code' in env) return outRefused(env, [], 'none', 'none');
|
||||
if (fetchErr) return outRefused({ code: 'POLICY_SOURCE_UNAVAILABLE', message: fetchErr }, [], 'none', 'none');
|
||||
let t: TrainRecord | TrainOut;
|
||||
let lines: string[] = [];
|
||||
if (o.resume) {
|
||||
t = findTrain(env, o.resume) ?? outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.resume} here or in refs/notes/gstack-trains` }, [], `${Q} train status`, 'none');
|
||||
if ('kind' in t && TERMINAL_STATES.includes(t.state)) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `train ${t.nonce} is ${t.state}; it never dispatches again` }, describe(t), nextFor(t), trainFile(t.nonce));
|
||||
if ('kind' in t && t.state !== 'free-pending' && t.state !== 'dispatch-absent' && t.state !== 'dispatch-unknown' && t.state !== 'built' && t.state !== 'pushed' && t.state !== 'free-green') {
|
||||
return outOk(describe(t), nextFor(t), trainFile(t.nonce));
|
||||
}
|
||||
} else {
|
||||
const made = await newTrain(env, o);
|
||||
if ('exit' in made) return made;
|
||||
({ t, lines } = made);
|
||||
}
|
||||
if (!('kind' in t)) return t;
|
||||
if (o.dryRun) {
|
||||
return outOk([...lines, ...describe(t), `ESTIMATE: one ${env.train.gate_workflow} full gate (evals_all) on T${t.prefixes.length - 1} plus ${env.train.free_workflow}; dollar figure pending the T0 cost replay (lib/gate-cost.ts)`,
|
||||
'DRY-RUN: nothing written, pushed or dispatched'], `${Q} train build --prs ${pinList(t.slots)}`, 'none');
|
||||
}
|
||||
if (t.state === 'built' && !t.attempts.length && !loadTrain(t.nonce)) {
|
||||
effect(t, 'train.json written');
|
||||
save(env, t);
|
||||
for (const old of listTrains(env.repo.key)) {
|
||||
if (old.nonce === t.nonce || TERMINAL_STATES.includes(old.state) || old.state === 'green' || old.state === 'merging') continue;
|
||||
if (!old.slots.some(s => t.slots.some((n: TrainSlot) => n.pr === s.pr))) continue;
|
||||
old.state = 'superseded';
|
||||
old.superseded_by = t.nonce;
|
||||
save(env, old);
|
||||
}
|
||||
if (o.repair) {
|
||||
const f = activeFreeze(env.repo.key)!;
|
||||
f.repair = { pr: t.slots[0]!.pr, head: t.slots[0]!.head, train: t.nonce };
|
||||
f.updated_at = nowIso();
|
||||
persist(env.ctx.repoRoot, [f], { remote: env.remote });
|
||||
}
|
||||
}
|
||||
return advance(env, t, o, lines);
|
||||
}
|
||||
|
||||
/**
|
||||
* `train status`: never spends. Rebuilds local records from the notes ref,
|
||||
* reconciles each unconfirmed dispatch by its nonce (found, or absent when a
|
||||
* full listing shows no such run) and reads run and check state.
|
||||
*/
|
||||
export function trainStatus(ctx: QueueContext, o: { train?: string }): TrainOut {
|
||||
const env = trainEnv(ctx);
|
||||
if ('code' in env) return outRefused(env, [], 'none', 'none');
|
||||
const restored = rebuildFromNotes(ctx.repoRoot, env.remote);
|
||||
const lines = [`NOTES: read ${restored.read} record(s), restored ${restored.restored} (${restored.detail})`];
|
||||
const trains = o.train ? [loadTrain(o.train)].filter((t): t is TrainRecord => !!t) : listTrains(env.repo.key).filter(t => !TERMINAL_STATES.includes(t.state));
|
||||
if (o.train && !trains.length) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.train} here or in refs/notes/gstack-trains` }, lines, `${Q} train status`, 'none');
|
||||
const slug = trains.length ? repoSlug(ctx) : null;
|
||||
const exec = execFor(ctx);
|
||||
for (const t of trains) {
|
||||
lines.push(...describe(t));
|
||||
for (const a of attemptsOf(t)) {
|
||||
if (slug && (a.state === 'dispatching' || a.state === 'unknown')) {
|
||||
const runs = listDispatchRuns(a.workflow, exec, slug, 100);
|
||||
const hit = runs ? runForNonce(runs, a.nonce) : null;
|
||||
if (runs) {
|
||||
a.state = hit ? 'dispatched' : 'absent';
|
||||
a.run = hit ? asRun(hit) : null;
|
||||
if (t.state === 'dispatch-unknown') t.state = hit ? (a.purpose === 'bisect' ? 'red' : 'gate-pending') : 'dispatch-absent';
|
||||
effect(t, `status reconciled ${a.id} (nonce ${a.nonce}): ${hit ? `run ${hit.databaseId}` : 'absent'}`);
|
||||
save(env, t, a);
|
||||
}
|
||||
}
|
||||
const view = slug && a.run && !a.outcome ? runView(ctx, slug, a.run.id) : null;
|
||||
lines.push(`ATTEMPT: ${a.id} ${a.purpose} state=${a.state} nonce=${a.nonce} run=${a.run?.id ?? '-'}${view ? ` ${view.status}/${view.conclusion ?? '-'}` : ''} outcome=${a.outcome ?? '-'}`);
|
||||
}
|
||||
if (slug && (t.state === 'green' || t.state === 'merging')) {
|
||||
const c = checkRun(ctx, t.head, env.train.full_gate_check);
|
||||
lines.push(`CHECK: ${env.train.full_gate_check} on ${t.head.slice(0, 12)}: ${c ? `${c.status}/${c.conclusion ?? '-'}` : 'missing'}`);
|
||||
}
|
||||
lines.push(`NEXT(${t.nonce}): ${nextFor(t)}`);
|
||||
}
|
||||
if (!trains.length) lines.push('TRAIN: none live');
|
||||
const freeze = activeFreeze(env.repo.key);
|
||||
if (freeze) lines.push(...frozenLines(freeze));
|
||||
const single = trains.length === 1 ? trains[0]! : null;
|
||||
const next = freeze && !freeze.repair ? `${Q} train build --repair --prs <repair-pr>@<sha>` : single ? nextFor(single) : trains.length ? 'see each NEXT(<nonce>) line' : 'none';
|
||||
return outOk(lines, next, single ? trainFile(single.nonce) : 'none');
|
||||
}
|
||||
|
||||
/** `train prune`: delete train/<nonce> on the remote for every finished (merged, superseded, dropped) train. */
|
||||
export function pruneTrains(ctx: QueueContext, o: { dryRun: boolean; train?: string }): TrainOut {
|
||||
const env = trainEnv(ctx);
|
||||
if ('code' in env) return outRefused(env, [], 'none', 'none');
|
||||
rebuildFromNotes(ctx.repoRoot, env.remote);
|
||||
const lines: string[] = [];
|
||||
const rgit = (args: string[]) => git(ctx.repoRoot, args);
|
||||
for (const t of listTrains(env.repo.key).filter(x => !o.train || x.nonce === o.train)) {
|
||||
if (!TERMINAL_STATES.includes(t.state)) { lines.push(`KEEP: ${t.branch} (${t.state})`); continue; }
|
||||
if (!remoteBranchSha(rgit, env.remote, t.branch)) { lines.push(`GONE: ${t.branch} (${t.state})`); continue; }
|
||||
if (o.dryRun) { lines.push(`WOULD-DELETE: ${t.branch} (${t.state})`); continue; }
|
||||
const d = git(ctx.repoRoot, ['push', '-q', env.remote, '--delete', `refs/heads/${t.branch}`]);
|
||||
lines.push(d.ok ? `DELETED: ${t.branch} (${t.state})` : `FAILED: ${t.branch}: ${d.err.slice(0, 120)}`);
|
||||
if (d.ok) { effect(t, `branch deleted: ${t.branch}`); save(env, t); }
|
||||
}
|
||||
if (!lines.length) lines.push('PRUNE: no trains');
|
||||
return outOk(lines, o.dryRun ? `${Q} train prune` : 'none', 'none');
|
||||
}
|
||||
Reference in new issue
Block a user