From ead4a7c2737f26854b93b69eaffa78d61a498ece Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 04:44:43 +0000 Subject: [PATCH] feat(ship-queue): train build stamps S_i under fragments; train land publishes S_i exactly; backstop covers exact trees (PR B) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- lib/ship-queue-land.ts | 2 +- lib/ship-queue-train-cli.ts | 13 +++ lib/ship-queue-train-land.ts | 91 ++++++++++++++++++++ lib/ship-queue-train-record.ts | 4 + lib/ship-queue-train-stamp.ts | 6 +- lib/ship-queue-train-verify.ts | 5 +- lib/ship-queue-train.ts | 11 ++- scripts/train-backstop.ts | 16 ++-- scripts/train-gate.ts | 13 +-- test/ship-queue-train-land.test.ts | 132 +++++++++++++++++++++++++++++ test/train-backstop.test.ts | 47 ++++++++++ 11 files changed, 321 insertions(+), 19 deletions(-) create mode 100644 lib/ship-queue-train-land.ts create mode 100644 test/ship-queue-train-land.test.ts diff --git a/lib/ship-queue-land.ts b/lib/ship-queue-land.ts index 63e04aad8..6d8142292 100644 --- a/lib/ship-queue-land.ts +++ b/lib/ship-queue-land.ts @@ -216,7 +216,7 @@ function prInfo(ctx: QueueContext, pr: string): PrInfo | QueueFailure { } /** Poll the stamp commit's checks (the same read as the pin's) until they are green, one is red, or the time runs out. */ -function waitChecks(ctx: QueueContext, sha: string, minutes: number): void { +export function waitChecks(ctx: QueueContext, sha: string, minutes: number): void { const deadline = Date.now() + minutes * 60_000; for (let ci = ciAtHead(ctx, sha); !ci.green && !ci.line.startsWith('red') && Date.now() < deadline; ci = ciAtHead(ctx, sha)) { Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 30_000); diff --git a/lib/ship-queue-train-cli.ts b/lib/ship-queue-train-cli.ts index 01fb58477..4e4111b35 100644 --- a/lib/ship-queue-train-cli.ts +++ b/lib/ship-queue-train-cli.ts @@ -19,6 +19,8 @@ import { trainForget } from './ship-queue-train-forget'; import { redactCause } from './ship-queue-train-record'; export const TRAIN_HELP = ` train build --prs @[,...] [--order ,...] [--urgent | --repair] [--dry-run [--stack-tests [--stack-tests-command ""]]] [--free-timeout ] + [--allow-repo-commands]: changelog_fragments trains stamp S_1..S_k with the + base policy's restamp_command or release_tool --order: explicit slot order, every PR in --prs once (default: screened first, then READY order) train build --train resume a train (push, free tests, dispatch) from its record @@ -27,6 +29,9 @@ export const TRAIN_HELP = ` train build --prs @[,...] [--order ,...] train status [--train ] never spends: rebuild records from refs/notes/gstack-trains, reconcile unconfirmed dispatches by nonce train verdict --train read the gate run and its record: GREEN, RED, INCOMPLETE or PENDING + train land --train --pr [--no-wait] [--timeout ] + (changelog_fragments trains) publish slot n's stamped prefix S_i: commit its tree on + [pin, main], push under the pin's lease, wait for checks, squash, confirm main is S_i train bisect --train --spend-cap [--dry-run] red cases on every prefix T0..Tk, resumable and capped train rebuild --train [--infra] the one rebuild an all-green bisect earns (or the one INFRA re-dispatch) @@ -105,6 +110,7 @@ export async function trainMain(ctx: QueueContext, args: string[]): Promise buildTrain(ctx, { pins: flag(rest, '--prs'), order: flag(rest, '--order'), resume: nonce, dryRun, urgent: rest.includes('--urgent'), repair: rest.includes('--repair'), freeTimeoutSec, pollMs: Number(process.env.GSTACK_SHIP_QUEUE_POLL_MS) || 30_000, stackTests: stackWords?.length ? stackWords : rest.includes('--stack-tests') ? true : undefined, + allowRepoCommands: rest.includes('--allow-repo-commands'), })); } if (sub === 'status') return guarded('train status', () => trainStatus(ctx, { train: nonce })); @@ -123,6 +129,13 @@ export async function trainMain(ctx: QueueContext, args: string[]): Promise trainWatch(ctx, { train: nonce, timeoutSec, pollMs: Number(process.env.GSTACK_SHIP_QUEUE_POLL_MS) || 30_000, freeTimeoutSec })); } if (sub === 'verdict') return guarded('train verdict', () => trainVerdict(ctx, { train: nonce })); + if (sub === 'land') { + const pr = flag(rest, '--pr'); + if (!pr || !/^\d+$/.test(pr)) return usage('train land needs --pr '); + const timeout = positive(flag(rest, '--timeout'), 60); + if (Number.isNaN(timeout)) return usage('--timeout needs minutes'); + return guarded('train land', async () => (await import('./ship-queue-train-land')).trainLand(ctx, { train: nonce, pr, noWait: rest.includes('--no-wait'), timeoutMin: timeout })); + } if (sub === 'rebuild') return guarded('train rebuild', () => trainRebuild(ctx, { train: nonce, infra: rest.includes('--infra') })); if (sub === 'bisect') { const cap = positive(flag(rest, '--spend-cap'), NaN); diff --git a/lib/ship-queue-train-land.ts b/lib/ship-queue-train-land.ts new file mode 100644 index 000000000..46bf53677 --- /dev/null +++ b/lib/ship-queue-train-land.ts @@ -0,0 +1,91 @@ +/** + * `train land --train --pr ` (docs/ship-policy.md#train-land, GSTA-154 PR B): the merge step for + * one slot of a stamped train (changelog_fragments). The train built S_1..S_k and gated S_k; slot i + * publishes S_i exactly. With main's tree equal to S_(i-1)'s (slot 1: the train base), land commits S_i's + * tree with parents [pin_i, main], labels the PR evals-no-debounce (a tool-made head gets no later push, + * so the quiet window protects nothing), pushes under --force-with-lease=:, waits for the + * PR's checks on that commit (CI runs live at the slot until GSTA-156's reuse is on main), merges with + * --match-head-commit and confirms main's tree is S_i. Any other main is the freeze path + * (QUEUE_TRAIN_TREE_MISMATCH), as for verify. Journaled beside the train record; a rerun resumes. + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { ciAtHead, type QueueContext } from './ship-queue'; +import { git, writeJsonAtomic } from './ship-queue-gate'; +import { waitChecks } from './ship-queue-land'; +import { bindReceipt } from './ship-receipt'; +import { checkRun, prInfo } from './ship-queue-train-github'; +import { activeFreeze, attemptsOf, trainFile } from './ship-queue-train-record'; +import { fetchBase, fetchTrainBranch, findTrain, frozenLines, nextFor, outFailed, outOk, outPending, outRefused, trainEnv, type TrainOut } from './ship-queue-train'; +import { confirmMerged, stopped } from './ship-queue-train-verify'; + +const Q = 'gstack-ship-queue'; +interface SlotJournal { schema_version: 1; pr: string; pin: string; stamp_commit: string; base: string } + +export async function trainLand(ctx: QueueContext, o: { train: string; pr: string; noWait?: boolean; timeoutMin?: number }): Promise { + const fetchErr = fetchBase(ctx); + const env = trainEnv(ctx); + if ('code' in env) return outRefused(env, [], 'none', 'none'); + if (fetchErr) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: fetchErr }, [], 'none', 'none'); + const t = findTrain(env, o.train); + if (!t) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.train} here or in refs/notes/gstack-trains` }, [], `${Q} train status`, 'none'); + const run = trainFile(t.nonce); + const i = t.slots.findIndex(s => s.pr === o.pr); + if (i < 0) return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} does not include #${o.pr}` }, [], nextFor(t), run); + if (!t.stamped) return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} was built without stamped prefixes; merge it with the slot stamp and ${Q} verify` }, [], `${Q} verify --pr ${o.pr} --train ${t.nonce}`, run); + const gate = attemptsOf(t).filter(a => a.purpose !== 'bisect').at(-1); + if (gate?.outcome !== 'GREEN' || !['green', 'merging', 'merged'].includes(t.state)) return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} has no green record (state ${t.state}, last gate ${gate?.outcome ?? 'none'})` }, [], nextFor(t), run); + const check = checkRun(ctx, t.head, env.train.full_gate_check); + if (check?.conclusion !== 'success') return outRefused({ code: 'QUEUE_GATE_MISSING', message: `no successful ${env.train.full_gate_check} check run on the train head ${t.head.slice(0, 12)}` }, [], `${Q} train verdict --train ${t.nonce}`, run); + if (!fetchTrainBranch(env, t)) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `the train's prefix commits are not here and ${t.branch} could not be fetched` }, [], `${Q} train status --train ${t.nonce}`, run); + const freeze = activeFreeze(env.repo.keys); + if (freeze && freeze.repair?.train !== t.nonce) return outRefused({ code: 'QUEUE_TRAIN_FROZEN', message: `the queue is frozen (${freeze.reason}); only the admitted repair train merges` }, frozenLines(freeze), `${Q} train build --repair --prs @`, run); + + const root = ctx.repoRoot; + const s = t.stamped[i]!; + const pred = i === 0 ? { index: 0, tree: t.prefixes[0]!.tree } : t.stamped[i - 1]!; + const pin = t.slots[i]!.head; + const mainNow = () => { fetchBase(ctx); const sha = git(root, ['rev-parse', `${env.remote}/${ctx.base}`]).out; return { sha, tree: git(root, ['rev-parse', `${sha}^{tree}`]).out }; }; + const lines = [`TRAIN_LAND: ${t.nonce} slot ${i + 1}/${t.slots.length} #${o.pr} pin ${pin} -> S${s.index} v${s.version} tree ${s.tree}`]; + let main = mainNow(); + if (main.tree === s.tree) return confirmMerged(env, t, o.pr, main.sha, s.index, [...lines, `main: ${main.sha} is S${s.index} exactly`]); + if (main.tree !== pred.tree) return stopped(env, t, 'QUEUE_TRAIN_TREE_MISMATCH', `main ${main.sha.slice(0, 12)} is neither S${pred.index} nor S${s.index} exactly`, main.sha, lines, true); + + const file = path.join(path.dirname(run), `land-${o.pr}.json`); + let j: SlotJournal | null = null; + try { j = JSON.parse(fs.readFileSync(file, 'utf8')) as SlotJournal; } catch { j = null; } + const info = prInfo(ctx, o.pr); + if ('error' in info) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: info.error }, lines, `${Q} train land --train ${t.nonce} --pr ${o.pr}`, run); + if (info.isCrossRepository) return outRefused({ code: 'QUEUE_TRAIN_UNTRUSTED', message: `#${o.pr} is a fork head; train land pushes only crew branches` }, lines, 'none', run); + if (!j || j.base !== main.sha) { + if (info.headRefOid !== pin && info.headRefOid !== j?.stamp_commit) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${o.pr}'s head is ${info.headRefOid.slice(0, 12)}, not its pin ${pin.slice(0, 12)}`, main.sha, lines, true); + const shipped = bindReceipt(info.body, o.pr, [pin]); + lines.push(shipped.line); + if (!shipped.ok) return outRefused({ code: shipped.code, message: shipped.message }, lines, `#${o.pr}'s lane runs /ship at its pin and READYs again`, run); + const commit = git(root, ['commit-tree', s.tree, '-p', pin, '-p', main.sha, '-m', `v${s.version} stamp: train ${t.nonce} S${s.index} (#${o.pr})`]).out; + const label = ctx.gh(['pr', 'edit', o.pr, '--add-label', 'evals-no-debounce']); + lines.push(`LABEL: evals-no-debounce ${label.ok ? 'added' : `not added (${(label.err ?? label.out).trim().slice(0, 80)}); the 10-minute quiet window applies`}`); + const lease = j?.stamp_commit && info.headRefOid === j.stamp_commit ? j.stamp_commit : pin; + const push = git(root, ['push', '-q', `--force-with-lease=refs/heads/${info.headRefName}:${lease}`, env.remote, `${commit}:refs/heads/${info.headRefName}`]); + if (!push.ok) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${o.pr}'s branch moved off ${lease.slice(0, 12)} before the stamp push; train land never overwrites it`, main.sha, lines, true); + j = { schema_version: 1, pr: o.pr, pin, stamp_commit: commit, base: main.sha }; + fs.mkdirSync(path.dirname(file), { recursive: true }); + writeJsonAtomic(file, j); + lines.push(`PUSHED: ${commit} (tree S${s.index}, parents [pin, main ${main.sha.slice(0, 12)}]) to ${info.headRefName}`); + const next = `gstack-pr-wait ${o.pr} --until green && ${Q} train land --train ${t.nonce} --pr ${o.pr}`; + if (o.noWait) return outPending({ code: 'QUEUE_NOT_READY', message: `checks run on the stamp commit ${commit.slice(0, 12)}` }, lines, next, run); + waitChecks(ctx, commit, o.timeoutMin ?? 60); + } + const head = prInfo(ctx, o.pr); + if ('error' in head || head.headRefOid !== j.stamp_commit) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${o.pr}'s head is not the stamp commit ${j.stamp_commit.slice(0, 12)}`, main.sha, lines, true); + const ci = ciAtHead(ctx, j.stamp_commit); + lines.push(`CI: ${ci.line}`); + if (!ci.green) return outPending({ code: 'QUEUE_NOT_READY', message: `checks on the stamp commit ${j.stamp_commit.slice(0, 12)} are not green` }, lines, `gstack-pr-wait ${o.pr} --until green && ${Q} train land --train ${t.nonce} --pr ${o.pr}`, run); + main = mainNow(); + if (main.sha !== j.base) return stopped(env, t, 'QUEUE_TRAIN_TREE_MISMATCH', `main moved to ${main.sha.slice(0, 12)} during the slot's checks (stamped on ${j.base.slice(0, 12)})`, main.sha, lines, true); + const m = ctx.gh(['pr', 'merge', o.pr, '--squash', '--match-head-commit', j.stamp_commit]); + if (!m.ok) return outFailed({ code: 'QUEUE_NOT_READY', message: `gh pr merge refused: ${(m.err ?? m.out).trim().slice(0, 200)}` }, lines, `${Q} train land --train ${t.nonce} --pr ${o.pr}`, run); + main = mainNow(); + if (main.tree !== s.tree) return stopped(env, t, 'QUEUE_TRAIN_TREE_MISMATCH', `after the merge main ${main.sha.slice(0, 12)} is not S${s.index} exactly`, main.sha, lines, true); + return confirmMerged(env, t, o.pr, main.sha, s.index, [...lines, `MERGED: #${o.pr} as ${main.sha}; main's tree is S${s.index} exactly`]); +} diff --git a/lib/ship-queue-train-record.ts b/lib/ship-queue-train-record.ts index 9b65c7619..bef44522c 100644 --- a/lib/ship-queue-train-record.ts +++ b/lib/ship-queue-train-record.ts @@ -34,12 +34,16 @@ export interface ScreenChoice { } export interface TrainSlot { pr: string; head: string; ready_at: string; screen: ScreenChoice } export interface TrainPrefix { index: number; commit: string; tree: string; pr: string | null } +/** A stamped prefix S_i (changelog_fragments trains, lib/ship-queue-train-stamp.ts): what slot i publishes, exactly. */ +export interface StampedPrefix { index: number; commit: string; tree: string; pr: string; version: string; date: string } export interface FreeRun { run: number | null; url: string | null; head_sha: string | null; conclusion: string | null; verdict: 'green' | 'red' | 'missing' | 'stale' | 'cancelled' | 'pending' } export interface TrainRecord { schema: 1; kind: 'train'; nonce: string; key: string; repo_key: string; repo: string; base: { ref: string; sha: string }; policy: string; slots: TrainSlot[]; held: string[]; prefixes: TrainPrefix[]; head: string; tree: string; branch: string; urgent: boolean; repair: boolean; explicit_order?: boolean; state: TrainState; free: FreeRun | null; attempts: string[]; merged: Array<{ pr: string; sha: string; tree: string; at: string }>; superseded_by: string | null; + /** S_1..S_k when the base policy declares changelog_fragments; head and tree are then S_k's. Absent on plain trains. */ + stamped?: StampedPrefix[]; effects: Array<{ at: string; what: string }>; error: string | null; created_at: string; updated_at: string; } export type AttemptPurpose = 'gate' | 'rebuild' | 'infra-redispatch' | 'bisect'; diff --git a/lib/ship-queue-train-stamp.ts b/lib/ship-queue-train-stamp.ts index 04030db1a..16554bd6f 100644 --- a/lib/ship-queue-train-stamp.ts +++ b/lib/ship-queue-train-stamp.ts @@ -17,10 +17,10 @@ import { restamp, versionAtRef, versionFiles } from './restamp'; import { matchesAny, type ShipPolicy } from './ship-policy'; import { git, type QueueFailure } from './ship-queue-gate'; import { mergeCommit } from './ship-queue-train'; -import type { TrainPrefix, TrainSlot } from './ship-queue-train-record'; +import type { StampedPrefix, TrainPrefix, TrainSlot } from './ship-queue-train-record'; import { buildReceipt } from './tree-receipt'; -export interface StampedPrefix { index: number; commit: string; tree: string; pr: string; version: string; date: string } +export type { StampedPrefix }; export interface StampChainOptions { repoRoot: string; base: string; policy: ShipPolicy; policyLabel: string; baseSha: string; slots: TrainSlot[]; plain: TrainPrefix[]; allowRepoCommands: boolean; now: Date; @@ -86,7 +86,7 @@ export function stampChain(o: StampChainOptions): { stamped: StampedPrefix[]; li git(o.repoRoot, ['worktree', 'remove', '--force', wt]); fs.rmSync(wt, { recursive: true, force: true }); } - const c = git(o.repoRoot, ['commit-tree', tree, '-p', pred, '-p', slot.head, '-m', `train S${i + 1}: #${slot.pr} v${version}`], { ...process.env, ...IDENTITY }); + const c = git(o.repoRoot, ['commit-tree', tree, '-p', pred, '-p', slot.head, '-m', `train S${i + 1}: merge #${slot.pr} at ${slot.head.slice(0, 12)} v${version}`], { ...process.env, ...IDENTITY }); if (!c.ok) return { code: 'QUEUE_TRAIN_STAMP_MISMATCH', message: `commit-tree failed for S${i + 1}: ${c.err.slice(0, 120)}` }; const plain = o.plain[i + 1]; const why = plain ? stampMismatch(o, m.commit, c.out, plain.tree) : `no plain prefix T${i + 1}`; diff --git a/lib/ship-queue-train-verify.ts b/lib/ship-queue-train-verify.ts index 4d75b33ce..927656c00 100644 --- a/lib/ship-queue-train-verify.ts +++ b/lib/ship-queue-train-verify.ts @@ -170,7 +170,7 @@ function verifiedElsewhere(env: TrainEnv, t: TrainRecord, mainSha: string): stri * plain refusal (nothing was published). A GitHub state that cannot be read * is neither: the verify refuses and asks to be rerun, never "nothing merged". */ -async function stopped(env: TrainEnv, t: TrainRecord, code: 'QUEUE_TRAIN_TREE_MISMATCH' | 'QUEUE_TRAIN_MOVED', message: string, mainSha: string, lines: string[], fetched: boolean): Promise { +export async function stopped(env: TrainEnv, t: TrainRecord, code: 'QUEUE_TRAIN_TREE_MISMATCH' | 'QUEUE_TRAIN_MOVED', message: string, mainSha: string, lines: string[], fetched: boolean): Promise { const run = trainFile(t.nonce); // One GitHub read per unverified slot decides both the freeze and its record. const states = new Map(t.slots.filter(s => !t.merged.some(m => m.pr === s.pr)).map(s => [s.pr, prState(env.ctx, s.pr)] as const)); @@ -197,6 +197,7 @@ export async function verifyTrain(ctx: QueueContext, o: { pr: string; train: str const run = trainFile(t.nonce); const i = t.slots.findIndex(s => s.pr === o.pr); if (i < 0) return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} does not include #${o.pr}` }, [], nextFor(t), run); + if (t.stamped) return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} has stamped prefixes; its slots publish S_i exactly through train land, never a hand stamp checked modulo stamps` }, [], `${Q} train land --train ${t.nonce} --pr ${o.pr}`, run); const gate = attemptsOf(t).filter(a => a.purpose !== 'bisect').at(-1); if (gate?.outcome !== 'GREEN' || !['green', 'merging', 'merged'].includes(t.state)) { return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} has no green record (state ${t.state}, last gate ${gate?.outcome ?? 'none'})` }, [], nextFor(t), run); @@ -230,7 +231,7 @@ export async function verifyTrain(ctx: QueueContext, o: { pr: string; train: str `merge #${o.pr} with expected head ${head.slice(0, 12)} (gstack-restamp stamps at merge), then ${Q} verify --pr ${o.pr} --train ${t.nonce}`, run); } -function confirmMerged(env: TrainEnv, t: TrainRecord, pr: string, main: string, index: number, lines: string[]): TrainOut { +export function confirmMerged(env: TrainEnv, t: TrainRecord, pr: string, main: string, index: number, lines: string[]): TrainOut { const run = trainFile(t.nonce); if (!t.merged.some(m => m.pr === pr)) t.merged.push({ pr, sha: main, tree: git(env.ctx.repoRoot, ['rev-parse', `${main}^{tree}`]).out, at: nowIso() }); t.state = t.merged.length === t.slots.length ? 'merged' : 'merging'; diff --git a/lib/ship-queue-train.ts b/lib/ship-queue-train.ts index 5485f9b20..a55a36c2b 100644 --- a/lib/ship-queue-train.ts +++ b/lib/ship-queue-train.ts @@ -26,6 +26,7 @@ import { bindShippedReceipt } from './ship-receipt-depth'; import { git, sha256, withIndex, type QueueFailure } from './ship-queue-gate'; import { recordVerdict, screenVerdict, verdictLines } from './ship-queue-screen'; import { stackTestLine, stackTests } from './ship-queue-stack-tests'; +import { stampChain } from './ship-queue-train-stamp'; import { asRun, execFor, newestBranchRun, prInfo, remoteBranchSha, repoSlug, runView, checkRun } from './ship-queue-train-github'; import { TERMINAL_STATES, activeFreeze, attemptsOf, listTrains, loadBisect, loadReady, loadTrain, nowIso, persist, rebuildFromNotes, repoIdentity, trainFile, @@ -321,6 +322,8 @@ export interface BuildOptions { pins?: string; order?: string; resume?: string; dryRun: boolean; urgent: boolean; repair: boolean; freeTimeoutSec: number; pollMs: number; /** `--dry-run --stack-tests`: run each PR's new test files on its integrated prefix (lib/ship-queue-stack-tests.ts); the command's words when `--stack-tests-command` names one. */ stackTests?: string[] | true; + /** Stamping S_i runs the base policy's restamp_command or release_tool (changelog_fragments trains). */ + allowRepoCommands?: boolean; } /** `--order ,,...`: every pinned PR exactly once (a leading # allowed). */ @@ -412,13 +415,17 @@ async function newTrain(env: TrainEnv, o: BuildOptions): Promise<{ t: TrainRecor if (due && !o.dryRun) return outPending({ code: 'QUEUE_TRAIN_WAITING', message: `${order.train.length} of ${env.train.max_size} PRs; the oldest READY dispatches at ${due}` }, lines, `${Q} train build ${prsArgs(order.train, !!explicit)} (after ${due}, or with more PRs, or --urgent for one)`, 'none'); const live = listTrains(env.repo.keys).find(t => t.key === key && !TERMINAL_STATES.includes(t.state) && t.state !== 'red' && t.state !== 'unattributed'); if (live) return { t: Object.assign(live, { held: order.held.map(s => s.pr) }), lines }; - const head = chain.prefixes.at(-1)!; + const policy = env.policy.policy; + const stamped = policy.changelog_fragments ? stampChain({ repoRoot: env.ctx.repoRoot, base: env.ctx.base, policy, policyLabel: env.policy.source.label, baseSha, slots: order.train, plain: chain.prefixes, allowRepoCommands: !!o.allowRepoCommands, now: new Date(nowIso()) }) : null; + if (stamped && 'code' in stamped) return outFailed(stamped, lines, `${Q} train build ${prsArgs(order.train, !!explicit)} --allow-repo-commands (after the named slot is fixed or dropped)`, 'none'); + if (stamped) lines.push(...stamped.lines); + const head = stamped ? stamped.stamped.at(-1)! : chain.prefixes.at(-1)!; const nonce = makeNonce('gate', head.commit, baseSha, randomBytes(3).toString('hex'), { purpose: 'train' }); const now = nowIso(); return { lines, t: { schema: 1, kind: 'train', nonce, key, repo_key: env.repo.key, repo: env.repo.url, base: { ref: env.ctx.base, sha: baseSha }, policy: env.policy.source.label, slots: order.train, held: order.held.map(s => s.pr), prefixes: chain.prefixes, head: head.commit, tree: head.tree, branch: `train/${nonce}`, - urgent: o.urgent, repair: o.repair, ...(explicit ? { explicit_order: true } : {}), state: 'built', free: null, attempts: [], merged: [], superseded_by: null, effects: [], error: null, + urgent: o.urgent, repair: o.repair, ...(explicit ? { explicit_order: true } : {}), ...(stamped ? { stamped: stamped.stamped } : {}), state: 'built', free: null, attempts: [], merged: [], superseded_by: null, effects: [], error: null, created_at: now, updated_at: now, } }; } diff --git a/scripts/train-backstop.ts b/scripts/train-backstop.ts index 935353550..5f7a0db1a 100644 --- a/scripts/train-backstop.ts +++ b/scripts/train-backstop.ts @@ -53,8 +53,11 @@ function noteRecords(git: Git, ki }); } -/** The newest green-train record whose prefix trees include this digest, with its run; null when none. */ -export function coveringRecord(gh: GhRunner, repo: string, train: TrainPolicy, digest: string, lines: string[]): { run: Run; record: TrainRecord; index: number } | null { +/** + * The newest green-train record with a prefix covering main, with its run; null when none. A plain prefix T_i + * covers main's tree modulo stamps (its digest); a stamped prefix S_i covers only main's exact tree. + */ +export function coveringRecord(gh: GhRunner, repo: string, train: TrainPolicy, main: { digest: string | null; tree: string }, lines: string[]): { run: Run; record: TrainRecord; index: number; exact: boolean } | null { const runs = json<{ workflow_runs?: Run[] }>(gh(['api', '-X', 'GET', `repos/${repo}/actions/workflows/${train.gate_workflow}/runs`, '-f', 'event=workflow_dispatch', '-f', 'status=success', '-f', 'per_page=50'], GH_MS)); if (!runs) { lines.push('cannot list the train gate runs; the backstop gates main'); return null; } const trains = (runs.workflow_runs ?? []).map(run => ({ run, nonce: /\[eval-bg ([A-Za-z0-9._-]+)\]/.exec(run.display_title ?? '')?.[1] ?? '' })) @@ -69,7 +72,7 @@ export function coveringRecord(gh: GhRunner, repo: string, train: TrainPolicy, d try { record = file ? JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8')) as TrainRecord : null; } catch { record = null; } if (!record || record.kind !== 'train-gate' || record.verdict !== 'GREEN' || record.census?.complete !== true || record.nonce !== nonce || record.head_sha !== run.head_sha || !Array.isArray(record.prefixes)) continue; - const hit = record.prefixes.find(p => p.stamp_free_tree === digest); + const hit = record.prefixes.find(p => p.exact ? p.tree === main.tree : main.digest !== null && p.stamp_free_tree === main.digest); if (!hit) continue; const checks = json<{ check_runs?: Array<{ name?: string; conclusion?: string | null }> }>( gh(['api', `repos/${repo}/commits/${run.head_sha}/check-runs?check_name=${encodeURIComponent(train.full_gate_check)}`], GH_MS)); @@ -77,7 +80,7 @@ export function coveringRecord(gh: GhRunner, repo: string, train: TrainPolicy, d lines.push(`train ${nonce} (run ${run.id}) lists this tree but its ${train.full_gate_check} check run is not a success`); continue; } - return { run, record, index: hit.index }; + return { run, record, index: hit.index, exact: !!hit.exact }; } return null; } finally { @@ -150,9 +153,10 @@ export function runBackstop(o: BackstopOptions): { action: BackstopAction; lines } const digest = stampFreeTree(git, o.sha, o.repoRoot, loaded.policy); if (typeof digest !== 'string') { lines.push(`BACKSTOP: no tree modulo stamps (${digest.error}); gating main`); } - const covered = typeof digest === 'string' ? coveringRecord(gh, o.repo, train, digest, lines) : null; + const tree = git(['rev-parse', `${o.sha}^{tree}`]).out; + const covered = coveringRecord(gh, o.repo, train, { digest: typeof digest === 'string' ? digest : null, tree }, lines); if (covered) { - lines.push(`BACKSTOP: covered: main ${o.sha.slice(0, 12)} is T${covered.index} of green train ${covered.record.nonce} (run ${covered.run.id}, tree modulo stamps ${String(digest).slice(0, 12)}); no gate`); + lines.push(`BACKSTOP: covered: main ${o.sha.slice(0, 12)} is ${covered.exact ? `S${covered.index}` : `T${covered.index}`} of green train ${covered.record.nonce} (run ${covered.run.id}, ${covered.exact ? `exact tree ${tree.slice(0, 12)}` : `tree modulo stamps ${String(digest).slice(0, 12)}`}); no gate`); return done('covered'); } const owned = noteRecords(git, 'freeze', repoKey).find(f => !f.cleared_at && f.main.sha === o.sha && f.backstop); diff --git a/scripts/train-gate.ts b/scripts/train-gate.ts index b5b433feb..1613dfe2b 100644 --- a/scripts/train-gate.ts +++ b/scripts/train-gate.ts @@ -49,7 +49,7 @@ export const MAX_BISECT_CASES = 40; /** A full gate's recorded cost (MEASUREMENT.md: $90.93 to $94.70; lib/gate-cost.ts uses the same scale). */ export const FULL_GATE_USD = 93.14; /** T1's prefix commits: `train T: merge # at ` (lib/ship-queue-train.ts buildChain). */ -const PREFIX_SUBJECT = /^train T(\d+): merge #(\d+) at [0-9a-f]{12}$/; +const PREFIX_SUBJECT = /^train (?:T(\d+): merge #(\d+) at [0-9a-f]{12}|S(\d+): merge #(\d+) at [0-9a-f]{12} v[0-9.]+)$/; export type Git = (args: string[]) => { ok: boolean; out: string }; export const gitIn = (cwd: string): Git => (args) => { @@ -185,9 +185,10 @@ export function stampFreeTree(git: Git, commit: string, repoRoot: string, policy return createHash('sha256').update(lines.sort().join('\n')).digest('hex'); } -export interface TrainPrefixTree { index: number; commit: string; tree: string; stamp_free_tree: string } +/** `exact`: a stamped prefix S_i (changelog_fragments trains), which covers only its exact tree. */ +export interface TrainPrefixTree { index: number; commit: string; tree: string; stamp_free_tree: string; exact?: true } -/** T1..Tk from the train head, walking first parents over T1's prefix commits. */ +/** T1..Tk (or stamped S1..Sk, never mixed) from the train head, walking first parents over the prefix commits. */ export function trainPrefixes(git: Git, head: string, repoRoot: string, policy: ShipPolicy): TrainPrefixTree[] | { error: string } { const log = git(['log', '--first-parent', `--max-count=${MAX_PREFIXES}`, '--format=%H%x00%P%x00%s', head]); if (!log.ok) return { error: `cannot walk ${head.slice(0, 12)}'s first parents (${log.out.split('\n')[0]})` }; @@ -196,11 +197,13 @@ export function trainPrefixes(git: Git, head: string, repoRoot: string, policy: const [commit, parents, subject] = line.split('\0') as [string, string, string]; const m = PREFIX_SUBJECT.exec(subject ?? ''); if (!m || parents.split(' ').length !== 2) break; - const index = Number(m[1]); + const index = Number(m[1] ?? m[3]); + const exact = m[3] !== undefined; + if (found.length && exact !== !!found[0]!.exact) return { error: `${commit.slice(0, 12)} mixes stamped and plain prefixes` }; if (found.length && index !== found.at(-1)!.index - 1) return { error: `prefix T${index} follows T${found.at(-1)!.index} at ${commit.slice(0, 12)}` }; const digest = stampFreeTree(git, commit, repoRoot, policy); if (typeof digest !== 'string') return digest; - found.push({ index, commit, tree: git(['rev-parse', `${commit}^{tree}`]).out, stamp_free_tree: digest }); + found.push({ index, commit, tree: git(['rev-parse', `${commit}^{tree}`]).out, stamp_free_tree: digest, ...(exact ? { exact: true as const } : {}) }); } if (!found.length || found.at(-1)!.index !== 1) return { error: `${head.slice(0, 12)} is not a train head (no T1..Tk prefix chain)` }; return found.reverse(); diff --git a/test/ship-queue-train-land.test.ts b/test/ship-queue-train-land.test.ts new file mode 100644 index 000000000..e8f2d0b36 --- /dev/null +++ b/test/ship-queue-train-land.test.ts @@ -0,0 +1,132 @@ +/** + * Stamped trains (changelog_fragments; GSTA-154 PR B): train build makes S_1..S_k beside T_1..T_k and gates + * S_k; train land publishes each slot's S_i exactly (a tool-made stamp commit on [pin, main], pushed under + * the pin's lease, labelled evals-no-debounce), confirms main is S_i, and freezes on any other main. + * Fixture repos with a bare origin and the train gh shim, wrapped for pr edit, pr merge and the PR's checks. + */ +import { afterEach, describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { EXIT } from '../lib/headless-artifacts'; +import { cleanup } from './helpers/restamp-fixture'; +import { + GREEN_GATE, TRAIN_POLICY, line, nonceOf, openPr, queue, readyReceipt, trainFixture, trainJson, type TrainFixture, +} from './helpers/ship-queue-train-fixture'; + +const POLICY = { ...TRAIN_POLICY, release_tool: null, release_outputs: [], changelog_fragments: { dir: 'changelog.d' }, min_gstack_version: '1.92.0.0' }; +const fixtures: TrainFixture[] = []; +afterEach(() => { while (fixtures.length) cleanup(fixtures.pop()!.repo); }); + +/** pr view reads the head from origin's branch; pr edit and pr merge act on origin; any commit without recorded CI reads green. */ +function wrap(fx: TrainFixture): void { + const dir = path.join(fx.repo.root, 'gh-land-wrap'); + fs.mkdirSync(dir, { recursive: true }); + const shim = path.join(fx.repo.root, 'gh-train-shim', 'shim.js'); + fs.writeFileSync(path.join(dir, 'gh'), [ + '#!/usr/bin/env node', + "const fs=require('fs');const cp=require('child_process');const D=process.env.LAND_WRAP_DATA;const a=process.argv.slice(2);", + "const d=JSON.parse(fs.readFileSync(D,'utf8'));const save=()=>fs.writeFileSync(D,JSON.stringify(d));", + "const g=(...x)=>cp.execFileSync('git',['--git-dir',d.origin,...x],{encoding:'utf8'}).trim();", + "const rev=r=>{try{return g('rev-parse','--verify','--quiet',r)}catch{return ''}};", + "if(a[0]==='pr'&&a[1]==='view'&&d.prs[a[2]]){const p=d.prs[a[2]];const h=rev('refs/heads/'+p.headRefName);if(h)p.headRefOid=h;save();console.log(JSON.stringify(p));process.exit(0)}", + "if(a[0]==='pr'&&a[1]==='edit'){(d.labels=d.labels||{})[a[2]]=a[a.indexOf('--add-label')+1];save();process.exit(0)}", + "if(a[0]==='pr'&&a[1]==='merge'){const p=d.prs[a[2]];const want=a[a.indexOf('--match-head-commit')+1];if(rev('refs/heads/'+p.headRefName)!==want){console.error('Head branch was modified');process.exit(1)}", + " const before=rev('main');let tree=rev(want+'^{tree}');", + " if(d.outsideOnMerge){const blob=cp.execFileSync('git',['--git-dir',d.origin,'hash-object','-w','--stdin'],{input:'outside\\n',encoding:'utf8'}).trim();", + " const ls=g('ls-tree',before+'^{tree}')+'\\n100644 blob '+blob+'\\toutside.txt';const ot=cp.execFileSync('git',['--git-dir',d.origin,'mktree'],{input:ls+'\\n',encoding:'utf8'}).trim();", + " g('update-ref','refs/heads/main',g('commit-tree',ot,'-p',before,'-m','outside'));tree=g('merge-tree','--write-tree','--merge-base='+before,rev('main'),want).split('\\n')[0];d.outsideOnMerge=false}", + " g('update-ref','refs/heads/main',g('commit-tree',tree,'-p',rev('main'),'-m','squash #'+a[2]));p.state='MERGED';save();process.exit(0)}", + "const paged=/commits\\/([0-9a-f]+)\\/(check-runs|status)\\?.*page=(\\d+)/.exec(a[1]||'');", + "if(a[0]==='api'&&paged&&!(d.ci||{})[paged[1]]&&!((d.statuses||{})[paged[1]])){const list=paged[2]==='check-runs'?[{name:'free-tests',status:'completed',conclusion:'success',id:1,completed_at:'2026-10-26T00:00:00Z'}]:[];const key=paged[2]==='check-runs'?'check_runs':'statuses';console.log(JSON.stringify(paged[3]==='1'?{total_count:list.length,state:'success',[key]:list}:{total_count:list.length,[key]:[]}));process.exit(0)}", + `const r=cp.spawnSync('node',[${JSON.stringify(shim)},D,...a],{stdio:'inherit'});process.exit(r.status??1);`, + '', + ].join('\n'), { mode: 0o755 }); + fx.env.PATH = `${dir}:${fx.env.PATH}`; + (fx.env as Record).LAND_WRAP_DATA = fx.dataFile; +} + +function fragmentPr(fx: TrainFixture, pr: string, fragment = `- #${pr} change\n`): string { + const head = openPr(fx, pr, { file: `changelog.d/pr-${pr}.md`, text: fragment, changelog: false }); + fx.repo.git('push', '-q', 'origin', `${head}:refs/heads/pr-${pr}`); + readyReceipt(fx, pr, head); + return head; +} +function stampedTrain() { + const fx = trainFixture(POLICY, 'gstack-train-land-'); fixtures.push(fx); wrap(fx); + const h1 = fragmentPr(fx, '1'); + const h2 = fragmentPr(fx, '2', '---\nbump: minor\n---\n- #2 change\n'); + fx.write(d => { d.rules['evals.yml'] = GREEN_GATE; }); + const built = queue(fx, ['train', 'build', '--prs', `1@${h1},2@${h2}`]); + const nonce = nonceOf(built.out); + return { fx, h1, h2, built, nonce }; +} +const land = (fx: TrainFixture, nonce: string, pr: string) => queue(fx, ['train', 'land', '--train', nonce, '--pr', pr, '--no-wait']); +const originMain = (fx: TrainFixture) => fx.repo.git('--git-dir', fx.repo.origin, 'rev-parse', 'main'); + +describe('stamped trains: build and land', () => { + test('build records S1, S2 and gates S2; land publishes each slot exactly, with the label, and records the merge', () => { + const { fx, h1, h2, built, nonce } = stampedTrain(); + expect(built.status, built.err).toBe(EXIT.ok); + expect(built.out).toMatch(/^S1: #1 v1\.2\.4\.0 [0-9a-f]{12} /m); + expect(built.out).toMatch(/^S2: #2 v1\.3\.0\.0 [0-9a-f]{12} /m); + const t = trainJson(fx, nonce); + expect(t.stamped.map((s: any) => [s.pr, s.version])).toEqual([['1', '1.2.4.0'], ['2', '1.3.0.0']]); + expect(t.head).toBe(t.stamped[1].commit); + expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok); + + const verify = queue(fx, ['verify', '--pr', '1', '--train', nonce]); + expect(verify.err).toMatch(/has stamped prefixes.*\(QUEUE_GATE_MISSING\)/); + expect(line(verify.out, 'NEXT:')).toBe(`NEXT: gstack-ship-queue train land --train ${nonce} --pr 1`); + const early = land(fx, nonce, '2'); + expect(early.err).toMatch(/is neither S1 nor S2 exactly.*not frozen.*\(QUEUE_TRAIN_TREE_MISMATCH\)/); + const pushed = land(fx, nonce, '1'); + expect(pushed.out, pushed.err).toContain('LABEL: evals-no-debounce added'); + const stamp = /^PUSHED: ([0-9a-f]{40})/m.exec(pushed.out)![1]!; + expect(fx.repo.git('--git-dir', fx.repo.origin, 'rev-parse', 'refs/heads/pr-1')).toBe(stamp); + expect(fx.repo.git('--git-dir', fx.repo.origin, 'rev-list', '--parents', '-n', '1', stamp).split(' ').slice(1)).toEqual([h1, originMain(fx)]); + expect(fx.repo.git('--git-dir', fx.repo.origin, 'rev-parse', stamp + '^{tree}')).toBe(t.stamped[0].tree); + const merged = land(fx, nonce, '1'); + expect(merged.status, merged.err).toBe(EXIT.ok); + expect(merged.out).toContain("main's tree is S1 exactly"); + expect(fx.repo.git('--git-dir', fx.repo.origin, 'rev-parse', 'main^{tree}')).toBe(t.stamped[0].tree); + land(fx, nonce, '2'); + const second = land(fx, nonce, '2'); + expect(second.status, second.err).toBe(EXIT.ok); + expect(fx.repo.git('--git-dir', fx.repo.origin, 'show', 'main:VERSION')).toBe('1.3.0.0'); + expect(trainJson(fx, nonce)).toMatchObject({ state: 'merged' }); + expect(fx.read().prs['2']).toMatchObject({ state: 'MERGED' }); + expect((fx.read() as any).labels).toEqual({ 1: 'evals-no-debounce', 2: 'evals-no-debounce' }); + void h2; + }, 300_000); + + test('a foreign push to the PR branch is never overwritten; a merge outside the train freezes the queue', () => { + const { fx, nonce } = stampedTrain(); + expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok); + fx.repo.git('push', '-q', '-f', 'origin', 'origin/main:refs/heads/pr-1'); + const moved = land(fx, nonce, '1'); + expect(moved.err).toContain('(QUEUE_TRAIN_MOVED)'); + expect(fx.repo.git('--git-dir', fx.repo.origin, 'rev-parse', 'refs/heads/pr-1')).toBe(fx.repo.git('--git-dir', fx.repo.origin, 'rev-parse', 'main')); + + const second = stampedTrain(); + expect(queue(second.fx, ['train', 'verdict', '--train', second.nonce]).status).toBe(EXIT.ok); + land(second.fx, second.nonce, '1'); + expect(land(second.fx, second.nonce, '1').status).toBe(EXIT.ok); + land(second.fx, second.nonce, '2'); + second.fx.write(d => { (d as any).outsideOnMerge = true; }); + const drift = land(second.fx, second.nonce, '2'); + expect(drift.err).toContain('(QUEUE_TRAIN_TREE_MISMATCH)'); + expect(line(drift.out, 'QUEUE_TRAIN_FROZEN:')).toMatch(/^QUEUE_TRAIN_FROZEN: train \S+ merged #1/); + }, 300_000); + + test('a plain train refuses train land and keeps the slot-stamp path', () => { + const fx = trainFixture(TRAIN_POLICY, 'gstack-train-land-'); fixtures.push(fx); wrap(fx); + const h1 = openPr(fx, '1'); + readyReceipt(fx, '1', h1); + fx.write(d => { d.rules['evals.yml'] = GREEN_GATE; }); + const nonce = nonceOf(queue(fx, ['train', 'build', '--prs', `1@${h1}`, '--urgent']).out); + expect(trainJson(fx, nonce).stamped).toBeUndefined(); + expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok); + const r = land(fx, nonce, '1'); + expect(r.err).toMatch(/was built without stamped prefixes.*\(QUEUE_GATE_MISSING\)/); + }, 180_000); +}); diff --git a/test/train-backstop.test.ts b/test/train-backstop.test.ts index b71244faf..ecdf6aeed 100644 --- a/test/train-backstop.test.ts +++ b/test/train-backstop.test.ts @@ -223,3 +223,50 @@ describe('backstop: debounce while a train merges', () => { expect(dispatches()).toEqual([]); }); }); + +describe('backstop: a stamped train (changelog_fragments) covers only exact trees', () => { + test('main equal to S1 is covered as S1; the same tree with another VERSION is gated', () => { + const fx = trainFixture(); + const { repo, main0, prs } = fx; + let parent = main0; + const stamped = prs.slice(0, 2).map((head, i) => { + repo.git('checkout', '-q', '-f', '--detach', parent); + repo.git('merge', '-q', '--no-ff', '--no-commit', head); + fs.writeFileSync(path.join(repo.work, 'VERSION'), `1.2.${4 + i}.0\n`); + repo.git('add', '-A'); + parent = repo.git('commit-tree', repo.git('write-tree'), '-p', parent, '-p', head, '-m', `train S${i + 1}: merge #${101 + i} at ${head.slice(0, 12)} v1.2.${4 + i}.0`); + repo.git('merge', '--abort'); + return parent; + }); + repo.git('checkout', '-q', '-f', 'main'); + const prefixes = trainPrefixes(gitIn(repo.work), parent, repo.work, loadPolicyFromHead(repo.work).policy); + if (!Array.isArray(prefixes)) throw new Error(prefixes.error); + expect(prefixes.map(p => [p.index, p.exact])).toEqual([[1, true], [2, true]]); + const record = { ...fx.record, head_sha: parent, tree: repo.git('rev-parse', `${parent}^{tree}`), prefixes }; + const push = (tree: string) => { + const sha = repo.git('commit-tree', tree, '-p', repo.git('rev-parse', 'main'), '-m', 'squash'); + repo.git('update-ref', 'refs/heads/main', sha); + repo.git('push', '-q', 'origin', 'main'); + return sha; + }; + const s1 = push(repo.git('rev-parse', `${stamped[0]}^{tree}`)); + const covered = backstop({ ...fx, record }, s1, fakeGh({ ...fx, record }, { record }).gh); + expect(covered.action, covered.lines.join('\n')).toBe('covered'); + expect(covered.lines.at(-1)).toContain(`is S1 of green train ${TRAIN_NONCE} (run 900, exact tree`); + repo.git('checkout', '-q', '-f', '--detach', stamped[0]!); + fs.writeFileSync(path.join(repo.work, 'VERSION'), '1.2.9.0\n'); + repo.git('add', '-A'); + const moduloStamps = push(repo.git('write-tree')); + repo.git('checkout', '-q', '-f', 'main'); + const gated = backstop({ ...fx, record }, moduloStamps, fakeGh({ ...fx, record }, { record, gate: 'success' }).gh); + expect(gated.lines.join('\n')).toContain("no green-train record lists main's tree"); + expect(gated.action).not.toBe('covered'); + }); + + test('a chain that mixes stamped and plain prefixes is not a train head', () => { + const fx = trainFixture(); + const { repo, train, prs } = fx; + const s = repo.git('commit-tree', repo.git('rev-parse', `${train[0]}^{tree}`), '-p', train[0]!, '-p', prs[1]!, '-m', `train S2: merge #102 at ${prs[1]!.slice(0, 12)} v1.2.5.0`); + expect(trainPrefixes(gitIn(repo.work), s, repo.work, loadPolicyFromHead(repo.work).policy)).toEqual({ error: `${train[0]!.slice(0, 12)} mixes stamped and plain prefixes` }); + }); +});