/** * `verify --pr --train `: the coordinator's merge check under * merge trains (docs/ship-policy.md#run-a-merge-train; plan GSTA-49 T1, P1). * * Each merge needs a green train record that includes the PR at its pinned * head, or at a head whose tree is the slot's prefix after it modulo stamps * (from slot 2 on, the lane merges main into its pin so GitHub can merge it), * with main's tree equal to the prefix before the PR (pre-merge: * MERGE_CONDITION) or after it (post-merge confirmation), modulo stamps * (lib/tree-receipt.ts). When a train stops partway (a mismatch or a moved * head after earlier slots merged) the queue freezes: the freeze is written * locally and to the notes ref first, then the backstop gate is dispatched on * main's published prefix at once. One repair train is admitted under the * freeze; verify clears the freeze when main after that merge carries it. */ import { randomBytes } from 'node:crypto'; import { DISPATCH_INPUTS, awaitNonceRun, dispatchWorkflow, makeNonce } from '../scripts/eval-bg'; import type { QueueContext } from './ship-queue'; import { git } from './ship-queue-gate'; import { bindReceipt } from './ship-receipt'; import { checkRun, execFor, prInfo, repoSlug } from './ship-queue-train-github'; import { activeFreeze, attemptsOf, listTrains, nowIso, persist, rebuildFromNotes, trainFile, type FreezeRecord, type TrainRecord } from './ship-queue-train-record'; import { effect, fetchBase, fetchTrainBranch, findTrain, frozenLines, nextFor, outFailed, outOk, outRefused, pinList, prState, prsArgs, save, trainEnv, type TrainEnv, type TrainOut, } from './ship-queue-train'; import { mergeChangelogs, ownEntryAboveMain, parseChangelog } from './changelog-check'; import { buildReceipt } from './tree-receipt'; const Q = 'gstack-ship-queue'; function sameModuloStamps(env: TrainEnv, gated: string, current: string): { same: boolean; detail: string } { const r = buildReceipt({ repoRoot: env.ctx.repoRoot, gated, current, policy: env.policy.policy, policyLabel: env.policy.source.label }); if ('error' in r) return { same: false, detail: r.error }; return { same: r.tree === 'same-modulo-stamps', detail: r.tree === 'same-modulo-stamps' ? `same modulo stamps (${r.files.length} stamp file(s))` : `differs: ${r.residual.slice(0, 8).join(', ')}` }; } /** * The one CHANGELOG rule of a slot, applied before and after the merge: * `current`'s CHANGELOG must be `mainBefore`'s with the PR's own entry above it * (ownEntryAboveMain), the own lines being those the pin's CHANGELOG diff from * its fork point added, or those `mergeChangelogs` itself writes for the pin * against that main (a released parent's carried corrections, GSTA-96). Pre-merge * `current` is the lane's head and `mainBefore` is main; post-merge `current` is * main and `mainBefore` its first parent, the main the pre-merge verify saw. * Null when it holds, else why not. */ function changelogIsMainPlusOwn(env: TrainEnv, t: TrainRecord, i: number, current: string, mainBefore: string): string | null { const cl = env.policy.policy.changelog!; const root = env.ctx.repoRoot; const pin = t.slots[i]!.head; const forkPoint = git(root, ['merge-base', pin, t.prefixes[0]!.commit]).out; const diff = git(root, ['diff', '--unified=0', '--no-color', '--no-ext-diff', forkPoint, pin, '--', cl]); const show = (rev: string) => git(root, ['show', `${rev}:${cl}`]); const [currentCl, mainCl, pinCl, forkCl] = [show(current), show(mainBefore), show(pin), show(forkPoint)]; if (!forkPoint || !diff.ok || !currentCl.ok || !mainCl.ok) return `cannot read ${cl} at the pin, ${current.slice(0, 12)} or main ${mainBefore.slice(0, 12)}`; const authored = new Set(diff.out.split('\n').filter(l => l.startsWith('+') && !l.startsWith('+++')).map(l => l.slice(1))); const tool = pinCl.ok ? mergeChangelogs(pinCl.out + '\n', mainCl.out + '\n', forkCl.ok ? forkCl.out + '\n' : undefined) : null; if (tool) { const mainKeys = new Set(parseChangelog(mainCl.out).sections.map(s => s.version ?? 'Unreleased')); for (const s of parseChangelog(tool).sections.filter(s => !mainKeys.has(s.version ?? 'Unreleased'))) for (const l of s.lines.slice(1)) authored.add(l); } return ownEntryAboveMain(currentCl.out, mainCl.out, authored); } /** * A slot head with main merged in (from slot 2 on the lane merges main into * its pin so GitHub can merge it): it must contain main, and its tree must be * the slot's prefix T modulo stamps. The CHANGELOG is judged against main * instead of T's mechanical merge (changelogIsMainPlusOwn), so a hand * resolve is accepted when it is main's CHANGELOG plus the PR's own entry; * any other difference is refused. */ function mergedIn(env: TrainEnv, t: TrainRecord, i: number, head: string, main: string): { same: boolean; detail: string } { const after = t.prefixes[i + 1]!; const root = env.ctx.repoRoot; if (!git(root, ['merge-base', '--is-ancestor', main, head]).ok) return { same: false, detail: `does not contain main ${main.slice(0, 12)}` }; const r = buildReceipt({ repoRoot: root, gated: after.commit, current: head, policy: env.policy.policy, policyLabel: env.policy.source.label }); if ('error' in r) return { same: false, detail: r.error }; const stamps = `${r.files.filter(f => f.verdict === 'stamp').length} stamp file(s)`; if (r.tree === 'same-modulo-stamps') return { same: true, detail: `T${after.index} with main merged in, same modulo stamps (${stamps})` }; const cl = env.policy.policy.changelog; const beyond = r.residual.filter(f => f !== cl); if (beyond.length || !cl) return { same: false, detail: `differs from T${after.index}: ${r.residual.slice(0, 8).join(', ')}` }; const why = changelogIsMainPlusOwn(env, t, i, head, main); if (why) return { same: false, detail: `differs from T${after.index} in ${cl}, which is not main's plus #${t.slots[i]!.pr}'s own entry: ${why}` }; return { same: true, detail: `T${after.index} with main merged in, same modulo stamps (${stamps}) except ${cl}, resolved as main's ${cl} plus #${t.slots[i]!.pr}'s own entry` }; } /** * Post-merge, the same rule the pre-merge verify applied: main is T modulo * stamps, or differs from it in the CHANGELOG alone while being its own first * parent (the main the pre-merge verify saw, itself T modulo stamps) plus * the PR's own entry. A merge the pre-merge verify accepted therefore confirms * here with the same CHANGELOG (train 24, 2026-10-27: #3453's accepted head * froze the queue post-merge because T2's one [Unreleased] had become two * released sections). */ function published(env: TrainEnv, t: TrainRecord, i: number, main: string): { same: boolean; detail: string } { const after = t.prefixes[i + 1]!; const post = sameModuloStamps(env, after.commit, main); if (post.same) return post; const cl = env.policy.policy.changelog; const r = buildReceipt({ repoRoot: env.ctx.repoRoot, gated: after.commit, current: main, policy: env.policy.policy, policyLabel: env.policy.source.label }); if ('error' in r || !cl || r.residual.some(f => f !== cl)) return post; const parent = git(env.ctx.repoRoot, ['rev-parse', '--verify', '-q', `${main}^`]).out; if (!parent || !sameModuloStamps(env, t.prefixes[i]!.commit, parent).same) return { same: false, detail: `${post.detail}; main's parent ${parent.slice(0, 12) || '(none)'} is not T${t.prefixes[i]!.index} modulo stamps` }; const why = changelogIsMainPlusOwn(env, t, i, main, parent); if (why) return { same: false, detail: `${post.detail}, which is not main ${parent.slice(0, 12)}'s plus #${t.slots[i]!.pr}'s own entry: ${why}` }; return { same: true, detail: `same modulo stamps (${r.files.filter(f => f.verdict === 'stamp').length} stamp file(s)) except ${cl}, main ${parent.slice(0, 12)}'s ${cl} plus #${t.slots[i]!.pr}'s own entry` }; } /** * Freeze the queue: the record goes to local state and the notes ref before * anything remote, then the backstop gate is dispatched on main's sha. * `states` is the one GitHub read of every unverified slot, made by the * caller: a slot that read MERGED is merged, verified tree or not, and a slot * that could not be read is left out of `unmerged`, recorded under * `unreadable` and named in the reason and in the NEXT after the repair (the * operator seats it by hand once gh answers), so the rebuild after the repair * never seats a PR that may already be in. */ async function freezeQueue(env: TrainEnv, t: TrainRecord, reason: string, mainSha: string, states: ReadonlyMap): Promise { const now = nowIso(); const unverified = t.slots.filter(s => !t.merged.some(m => m.pr === s.pr)); const landed = unverified.filter(s => states.get(s.pr) === 'MERGED').map(s => s.pr); const unreadable = unverified.filter(s => states.get(s.pr) === 'unreadable').map(s => s.pr); const prs = (list: string[]) => list.map(p => '#' + p).join(', '); const f: FreezeRecord = { schema: 1, kind: 'freeze', repo_key: env.repo.key, train: t.nonce, merged: [...t.merged.map(m => m.pr), ...landed], unmerged: unverified.filter(s => !landed.includes(s.pr) && !unreadable.includes(s.pr)).map(s => ({ pr: s.pr, head: s.head })), unreadable: unverified.filter(s => unreadable.includes(s.pr)).map(s => ({ pr: s.pr, head: s.head })), main: { sha: mainSha, tree: git(env.ctx.repoRoot, ['rev-parse', `${mainSha}^{tree}`]).out }, reason: [reason, landed.length ? `${prs(landed)} merged on GitHub without a verified tree` : '', unreadable.length ? `${prs(unreadable)} unreadable on GitHub, left out of unmerged; seat it in a train by hand once gh pr view answers` : ''].filter(Boolean).join('; '), backstop: null, repair: null, created_at: now, cleared_at: null, cleared_by: null, updated_at: now, }; persist(env.ctx.repoRoot, [f], { remote: env.remote }); effect(t, `queue frozen: ${reason}`); save(env, t); const slug = repoSlug(env.ctx); const nonce = makeNonce('gate', mainSha, mainSha, randomBytes(3).toString('hex'), { purpose: 'backstop' }); const dispatch = { workflow: env.train.gate_workflow, inputs: { [DISPATCH_INPUTS.evalsAll]: 'true', [DISPATCH_INPUTS.expectedSha]: mainSha, [DISPATCH_INPUTS.nonce]: nonce } }; const exec = execFor(env.ctx); const found = slug && dispatchWorkflow(dispatch, env.ctx.base, exec, slug) ? await awaitNonceRun(dispatch.workflow, nonce, exec, slug) : null; f.backstop = { nonce, run: found?.databaseId ?? null, state: found ? 'dispatched' : 'unconfirmed' }; f.updated_at = nowIso(); persist(env.ctx.repoRoot, [f], { remote: env.remote }); return f; } /** * Where another train's verify confirmed main's exact sha as one of its merged * prefixes, read from local records after the notes ref (a machine that never * ran that train still sees it). Null when none did. */ function verifiedElsewhere(env: TrainEnv, t: TrainRecord, mainSha: string): string | null { rebuildFromNotes(env.ctx.repoRoot, env.remote); for (const other of listTrains(env.repo.keys)) { if (other.nonce === t.nonce) continue; const m = other.merged.find(x => x.sha === mainSha); if (m) return `T${other.prefixes.find(p => p.pr === m.pr)?.index ?? '?'} of train ${other.nonce} (#${m.pr}, verified at ${m.at})`; } return null; } /** * Interrupted publication: freeze when a slot merged, verified or merged on * GitHub without a verified tree (slot 1's own squash failing the post-merge * check leaves main unverified with nothing in `merged` yet); otherwise a * plain refusal (nothing was published). A GitHub state that cannot be read * is neither: the verify refuses and asks to be rerun, never "nothing merged". */ async function stopped(env: TrainEnv, t: TrainRecord, code: 'QUEUE_TRAIN_TREE_MISMATCH' | 'QUEUE_TRAIN_MOVED', message: string, mainSha: string, lines: string[], fetched: boolean): Promise { const run = trainFile(t.nonce); // One GitHub read per unverified slot decides both the freeze and its record. const states = new Map(t.slots.filter(s => !t.merged.some(m => m.pr === s.pr)).map(s => [s.pr, prState(env.ctx, s.pr)] as const)); const landed = t.merged.length > 0 || [...states.values()].includes('MERGED'); const unreadable = landed ? [] : [...states].filter(([, state]) => state === 'unreadable').map(([pr]) => `#${pr}`); if (unreadable.length) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `${message}; whether ${unreadable.join(', ')} merged on GitHub could not be read, so whether the queue must freeze is unknown` }, lines, `${Q} verify --pr ${t.slots[0]!.pr} --train ${t.nonce} (once gh pr view answers)`, run); if (!landed) return outFailed({ code, message: `${message}; nothing of train ${t.nonce} merged, so the queue is not frozen` }, lines, `${Q} train build ${prsArgs(t.slots, t.explicit_order)} (a new train on current main)`, run); const later = fetched && t.slots.every(s => t.merged.some(m => m.pr === s.pr) || states.get(s.pr) === 'MERGED') ? verifiedElsewhere(env, t, mainSha) : null; if (later) { const held = activeFreeze(env.repo.keys); return outRefused({ code: 'QUEUE_TRAIN_STALE', message: `${message}; every slot of train ${t.nonce} merged on GitHub and main ${mainSha.slice(0, 12)} is ${later}, so this record is stale and ${held ? `freezes nothing (the queue is already frozen by train ${held.train})` : 'the queue is not frozen'}` }, held ? [...lines, ...frozenLines(held)] : lines, `${Q} train prune --stale --train ${t.nonce}`, run); } const f = activeFreeze(env.repo.keys) ?? await freezeQueue(env, t, message, mainSha, states); return outFailed({ code, message: `${message}; the queue is frozen` }, [...lines, ...frozenLines(f)], `${Q} train build --repair --prs @`, run); } export async function verifyTrain(ctx: QueueContext, o: { pr: string; train: string; noFetch: boolean }): Promise { const fetchErr = o.noFetch ? null : fetchBase(ctx); const env = trainEnv(ctx); if ('code' in env) return outRefused(env, [], 'none', 'none'); if (fetchErr) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: fetchErr }, [], 'none', 'none'); const t = findTrain(env, o.train); if (!t) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.train} here or in refs/notes/gstack-trains` }, [], `${Q} train status`, 'none'); const run = trainFile(t.nonce); const i = t.slots.findIndex(s => s.pr === o.pr); if (i < 0) return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} does not include #${o.pr}` }, [], nextFor(t), run); const gate = attemptsOf(t).filter(a => a.purpose !== 'bisect').at(-1); if (gate?.outcome !== 'GREEN' || !['green', 'merging', 'merged'].includes(t.state)) { return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} has no green record (state ${t.state}, last gate ${gate?.outcome ?? 'none'})` }, [], nextFor(t), run); } const check = checkRun(ctx, t.head, env.train.full_gate_check); if (check?.conclusion !== 'success') return outRefused({ code: 'QUEUE_GATE_MISSING', message: `no successful ${env.train.full_gate_check} check run on the train head ${t.head.slice(0, 12)}` }, [], `${Q} train verdict --train ${t.nonce}`, run); if (!fetchTrainBranch(env, t)) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `the train's prefix commits are not here and ${t.branch} could not be fetched` }, [], `${Q} train status --train ${t.nonce}`, run); const main = git(ctx.repoRoot, ['rev-parse', `${env.remote}/${ctx.base}`]).out; const freeze = activeFreeze(env.repo.keys); if (freeze && freeze.repair?.train !== t.nonce) return outRefused({ code: 'QUEUE_TRAIN_FROZEN', message: `the queue is frozen (${freeze.reason}); only the admitted repair train merges` }, frozenLines(freeze), `${Q} train build --repair --prs @`, run); const after = t.prefixes[i + 1]!; const before = t.prefixes[i]!; const post = published(env, t, i, main); const lines = [`TRAIN: ${t.nonce} slot ${i + 1}/${t.slots.length} #${o.pr} pinned ${t.slots[i]!.head}`]; if (post.same) return confirmMerged(env, t, o.pr, main, after.index, [...lines, `main: ${main} = T${after.index} (${post.detail})`]); const pre = sameModuloStamps(env, before.commit, main); if (!pre.same) { return stopped(env, t, 'QUEUE_TRAIN_TREE_MISMATCH', `main ${main.slice(0, 12)} is neither T${before.index} nor T${after.index} modulo stamps (vs T${after.index}: ${post.detail})`, main, lines, !o.noFetch); } const headRef = o.noFetch ? '' : (git(ctx.repoRoot, ['fetch', '--no-tags', '-q', env.remote, `+refs/pull/${o.pr}/head:refs/gstack/queue/pr-${o.pr}`]).ok ? `refs/gstack/queue/pr-${o.pr}` : ''); const head = headRef ? git(ctx.repoRoot, ['rev-parse', headRef]).out : t.slots[i]!.head; const pinned = head === t.slots[i]!.head ? { same: true, detail: 'the pinned head' } : sameModuloStamps(env, t.slots[i]!.head, head); const stampOnly = pinned.same ? pinned : mergedIn(env, t, i, head, main); if (!stampOnly.same) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${o.pr} moved past its pinned head ${t.slots[i]!.head.slice(0, 12)} and is not T${after.index} modulo stamps (${head.slice(0, 12)}: ${stampOnly.detail}; against the pin it ${pinned.detail})`, main, lines, !o.noFetch); const info = prInfo(ctx, o.pr); if ('error' in info) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: info.error }, lines, `${Q} verify --pr ${o.pr} --train ${t.nonce}`, run); const shipped = bindReceipt(info.body, o.pr, [t.slots[i]!.head, head]); if (!shipped.ok) return outRefused({ code: shipped.code, message: shipped.message }, [...lines, shipped.line], `#${o.pr}'s lane runs /ship and READYs again; then ${Q} train build --prs `, run); return outOk([...lines, `main: ${main} = T${before.index} (${pre.detail})`, `head: ${head} (${stampOnly.detail})`, `${shipped.line} (binds ${shipped.head === t.slots[i]!.head ? 'the pinned head' : 'the current head'})`, `VERIFY_TRAIN: #${o.pr} may merge as slot ${i + 1} of train ${t.nonce} (green run ${gate.run?.id ?? '-'}, ${env.train.full_gate_check} success)`, `MERGE_CONDITION: sha=${head}`], `merge #${o.pr} with expected head ${head.slice(0, 12)} (gstack-restamp stamps at merge), then ${Q} verify --pr ${o.pr} --train ${t.nonce}`, run); } function confirmMerged(env: TrainEnv, t: TrainRecord, pr: string, main: string, index: number, lines: string[]): TrainOut { const run = trainFile(t.nonce); if (!t.merged.some(m => m.pr === pr)) t.merged.push({ pr, sha: main, tree: git(env.ctx.repoRoot, ['rev-parse', `${main}^{tree}`]).out, at: nowIso() }); t.state = t.merged.length === t.slots.length ? 'merged' : 'merging'; effect(t, `verified #${pr} merged as T${index}`); save(env, t); lines.push(`VERIFY_TRAIN: #${pr} merged; main's tree is T${index} of train ${t.nonce} (${t.merged.length}/${t.slots.length} merged)`); const freeze = activeFreeze(env.repo.keys); if (freeze && freeze.repair?.train === t.nonce && t.state === 'merged') { freeze.cleared_at = nowIso(); freeze.cleared_by = `repair #${pr} via train ${t.nonce}; main ${main.slice(0, 12)} carries a green record`; freeze.updated_at = freeze.cleared_at; persist(env.ctx.repoRoot, [freeze], { remote: env.remote }); lines.push(`FREEZE: cleared (${freeze.cleared_by})`); const rebuild = freeze.unmerged.length ? `${Q} train build --prs ${pinList(freeze.unmerged)}` : ''; const unread = freeze.unreadable ?? []; const check = unread.length ? `${unread.map(u => `gh pr view ${u.pr} --json state,headRefOid`).join(' && ')} (unreadable at the freeze: ${unread.map(u => `#${u.pr}@${u.head.slice(0, 12)}`).join(', ')}); seat each that is OPEN in a train` : ''; return outOk(lines, [rebuild, check].filter(Boolean).join('; then ') || 'none', run); } return outOk(lines, nextFor(t), run); }