feat(ship-queue): train build stamps S_i under fragments; train land publishes S_i exactly; backstop covers exact trees (PR B)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-29 08:52:02 +00:00
1 parent dc309c793e
commit ead4a7c273
11 files changed
+321 -19

No files matched your search

+1 -1
View File
@@ -216,7 +216,7 @@ function prInfo(ctx: QueueContext, pr: string): PrInfo | QueueFailure {
}
/** Poll the stamp commit's checks (the same read as the pin's) until they are green, one is red, or the time runs out. */
function waitChecks(ctx: QueueContext, sha: string, minutes: number): void {
export function waitChecks(ctx: QueueContext, sha: string, minutes: number): void {
const deadline = Date.now() + minutes * 60_000;
for (let ci = ciAtHead(ctx, sha); !ci.green && !ci.line.startsWith('red') && Date.now() < deadline; ci = ciAtHead(ctx, sha)) {
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 30_000);
+13
View File
@@ -19,6 +19,8 @@ import { trainForget } from './ship-queue-train-forget';
import { redactCause } from './ship-queue-train-record';
export const TRAIN_HELP = ` train build --prs <n>@<sha>[,...] [--order <n>,...] [--urgent | --repair] [--dry-run [--stack-tests [--stack-tests-command "<cmd>"]]] [--free-timeout <s>]
[--allow-repo-commands]: changelog_fragments trains stamp S_1..S_k with the
base policy's restamp_command or release_tool
--order: explicit slot order, every PR in --prs once (default: screened
first, then READY order)
train build --train <nonce> resume a train (push, free tests, dispatch) from its record
@@ -27,6 +29,9 @@ export const TRAIN_HELP = ` train build --prs <n>@<sha>[,...] [--order <n>,...]
train status [--train <nonce>] never spends: rebuild records from refs/notes/gstack-trains, reconcile
unconfirmed dispatches by nonce
train verdict --train <nonce> read the gate run and its record: GREEN, RED, INCOMPLETE or PENDING
train land --train <nonce> --pr <n> [--no-wait] [--timeout <min>]
(changelog_fragments trains) publish slot n's stamped prefix S_i: commit its tree on
[pin, main], push under the pin's lease, wait for checks, squash, confirm main is S_i
train bisect --train <nonce> --spend-cap <usd> [--dry-run]
red cases on every prefix T0..Tk, resumable and capped
train rebuild --train <nonce> [--infra] the one rebuild an all-green bisect earns (or the one INFRA re-dispatch)
@@ -105,6 +110,7 @@ export async function trainMain(ctx: QueueContext, args: string[]): Promise<numb
return guarded('train build', () => buildTrain(ctx, {
pins: flag(rest, '--prs'), order: flag(rest, '--order'), resume: nonce, dryRun, urgent: rest.includes('--urgent'), repair: rest.includes('--repair'), freeTimeoutSec,
pollMs: Number(process.env.GSTACK_SHIP_QUEUE_POLL_MS) || 30_000, stackTests: stackWords?.length ? stackWords : rest.includes('--stack-tests') ? true : undefined,
allowRepoCommands: rest.includes('--allow-repo-commands'),
}));
}
if (sub === 'status') return guarded('train status', () => trainStatus(ctx, { train: nonce }));
@@ -123,6 +129,13 @@ export async function trainMain(ctx: QueueContext, args: string[]): Promise<numb
return guarded('train watch', () => trainWatch(ctx, { train: nonce, timeoutSec, pollMs: Number(process.env.GSTACK_SHIP_QUEUE_POLL_MS) || 30_000, freeTimeoutSec }));
}
if (sub === 'verdict') return guarded('train verdict', () => trainVerdict(ctx, { train: nonce }));
if (sub === 'land') {
const pr = flag(rest, '--pr');
if (!pr || !/^\d+$/.test(pr)) return usage('train land needs --pr <number>');
const timeout = positive(flag(rest, '--timeout'), 60);
if (Number.isNaN(timeout)) return usage('--timeout needs minutes');
return guarded('train land', async () => (await import('./ship-queue-train-land')).trainLand(ctx, { train: nonce, pr, noWait: rest.includes('--no-wait'), timeoutMin: timeout }));
}
if (sub === 'rebuild') return guarded('train rebuild', () => trainRebuild(ctx, { train: nonce, infra: rest.includes('--infra') }));
if (sub === 'bisect') {
const cap = positive(flag(rest, '--spend-cap'), NaN);
+91
View File
@@ -0,0 +1,91 @@
/**
* `train land --train <nonce> --pr <n>` (docs/ship-policy.md#train-land, GSTA-154 PR B): the merge step for
* one slot of a stamped train (changelog_fragments). The train built S_1..S_k and gated S_k; slot i
* publishes S_i exactly. With main's tree equal to S_(i-1)'s (slot 1: the train base), land commits S_i's
* tree with parents [pin_i, main], labels the PR evals-no-debounce (a tool-made head gets no later push,
* so the quiet window protects nothing), pushes under --force-with-lease=<branch>:<pin>, waits for the
* PR's checks on that commit (CI runs live at the slot until GSTA-156's reuse is on main), merges with
* --match-head-commit and confirms main's tree is S_i. Any other main is the freeze path
* (QUEUE_TRAIN_TREE_MISMATCH), as for verify. Journaled beside the train record; a rerun resumes.
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import { ciAtHead, type QueueContext } from './ship-queue';
import { git, writeJsonAtomic } from './ship-queue-gate';
import { waitChecks } from './ship-queue-land';
import { bindReceipt } from './ship-receipt';
import { checkRun, prInfo } from './ship-queue-train-github';
import { activeFreeze, attemptsOf, trainFile } from './ship-queue-train-record';
import { fetchBase, fetchTrainBranch, findTrain, frozenLines, nextFor, outFailed, outOk, outPending, outRefused, trainEnv, type TrainOut } from './ship-queue-train';
import { confirmMerged, stopped } from './ship-queue-train-verify';
const Q = 'gstack-ship-queue';
interface SlotJournal { schema_version: 1; pr: string; pin: string; stamp_commit: string; base: string }
export async function trainLand(ctx: QueueContext, o: { train: string; pr: string; noWait?: boolean; timeoutMin?: number }): Promise<TrainOut> {
const fetchErr = fetchBase(ctx);
const env = trainEnv(ctx);
if ('code' in env) return outRefused(env, [], 'none', 'none');
if (fetchErr) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: fetchErr }, [], 'none', 'none');
const t = findTrain(env, o.train);
if (!t) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `no train record ${o.train} here or in refs/notes/gstack-trains` }, [], `${Q} train status`, 'none');
const run = trainFile(t.nonce);
const i = t.slots.findIndex(s => s.pr === o.pr);
if (i < 0) return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} does not include #${o.pr}` }, [], nextFor(t), run);
if (!t.stamped) return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} was built without stamped prefixes; merge it with the slot stamp and ${Q} verify` }, [], `${Q} verify --pr ${o.pr} --train ${t.nonce}`, run);
const gate = attemptsOf(t).filter(a => a.purpose !== 'bisect').at(-1);
if (gate?.outcome !== 'GREEN' || !['green', 'merging', 'merged'].includes(t.state)) return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} has no green record (state ${t.state}, last gate ${gate?.outcome ?? 'none'})` }, [], nextFor(t), run);
const check = checkRun(ctx, t.head, env.train.full_gate_check);
if (check?.conclusion !== 'success') return outRefused({ code: 'QUEUE_GATE_MISSING', message: `no successful ${env.train.full_gate_check} check run on the train head ${t.head.slice(0, 12)}` }, [], `${Q} train verdict --train ${t.nonce}`, run);
if (!fetchTrainBranch(env, t)) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `the train's prefix commits are not here and ${t.branch} could not be fetched` }, [], `${Q} train status --train ${t.nonce}`, run);
const freeze = activeFreeze(env.repo.keys);
if (freeze && freeze.repair?.train !== t.nonce) return outRefused({ code: 'QUEUE_TRAIN_FROZEN', message: `the queue is frozen (${freeze.reason}); only the admitted repair train merges` }, frozenLines(freeze), `${Q} train build --repair --prs <repair-pr>@<sha>`, run);
const root = ctx.repoRoot;
const s = t.stamped[i]!;
const pred = i === 0 ? { index: 0, tree: t.prefixes[0]!.tree } : t.stamped[i - 1]!;
const pin = t.slots[i]!.head;
const mainNow = () => { fetchBase(ctx); const sha = git(root, ['rev-parse', `${env.remote}/${ctx.base}`]).out; return { sha, tree: git(root, ['rev-parse', `${sha}^{tree}`]).out }; };
const lines = [`TRAIN_LAND: ${t.nonce} slot ${i + 1}/${t.slots.length} #${o.pr} pin ${pin} -> S${s.index} v${s.version} tree ${s.tree}`];
let main = mainNow();
if (main.tree === s.tree) return confirmMerged(env, t, o.pr, main.sha, s.index, [...lines, `main: ${main.sha} is S${s.index} exactly`]);
if (main.tree !== pred.tree) return stopped(env, t, 'QUEUE_TRAIN_TREE_MISMATCH', `main ${main.sha.slice(0, 12)} is neither S${pred.index} nor S${s.index} exactly`, main.sha, lines, true);
const file = path.join(path.dirname(run), `land-${o.pr}.json`);
let j: SlotJournal | null = null;
try { j = JSON.parse(fs.readFileSync(file, 'utf8')) as SlotJournal; } catch { j = null; }
const info = prInfo(ctx, o.pr);
if ('error' in info) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: info.error }, lines, `${Q} train land --train ${t.nonce} --pr ${o.pr}`, run);
if (info.isCrossRepository) return outRefused({ code: 'QUEUE_TRAIN_UNTRUSTED', message: `#${o.pr} is a fork head; train land pushes only crew branches` }, lines, 'none', run);
if (!j || j.base !== main.sha) {
if (info.headRefOid !== pin && info.headRefOid !== j?.stamp_commit) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${o.pr}'s head is ${info.headRefOid.slice(0, 12)}, not its pin ${pin.slice(0, 12)}`, main.sha, lines, true);
const shipped = bindReceipt(info.body, o.pr, [pin]);
lines.push(shipped.line);
if (!shipped.ok) return outRefused({ code: shipped.code, message: shipped.message }, lines, `#${o.pr}'s lane runs /ship at its pin and READYs again`, run);
const commit = git(root, ['commit-tree', s.tree, '-p', pin, '-p', main.sha, '-m', `v${s.version} stamp: train ${t.nonce} S${s.index} (#${o.pr})`]).out;
const label = ctx.gh(['pr', 'edit', o.pr, '--add-label', 'evals-no-debounce']);
lines.push(`LABEL: evals-no-debounce ${label.ok ? 'added' : `not added (${(label.err ?? label.out).trim().slice(0, 80)}); the 10-minute quiet window applies`}`);
const lease = j?.stamp_commit && info.headRefOid === j.stamp_commit ? j.stamp_commit : pin;
const push = git(root, ['push', '-q', `--force-with-lease=refs/heads/${info.headRefName}:${lease}`, env.remote, `${commit}:refs/heads/${info.headRefName}`]);
if (!push.ok) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${o.pr}'s branch moved off ${lease.slice(0, 12)} before the stamp push; train land never overwrites it`, main.sha, lines, true);
j = { schema_version: 1, pr: o.pr, pin, stamp_commit: commit, base: main.sha };
fs.mkdirSync(path.dirname(file), { recursive: true });
writeJsonAtomic(file, j);
lines.push(`PUSHED: ${commit} (tree S${s.index}, parents [pin, main ${main.sha.slice(0, 12)}]) to ${info.headRefName}`);
const next = `gstack-pr-wait ${o.pr} --until green && ${Q} train land --train ${t.nonce} --pr ${o.pr}`;
if (o.noWait) return outPending({ code: 'QUEUE_NOT_READY', message: `checks run on the stamp commit ${commit.slice(0, 12)}` }, lines, next, run);
waitChecks(ctx, commit, o.timeoutMin ?? 60);
}
const head = prInfo(ctx, o.pr);
if ('error' in head || head.headRefOid !== j.stamp_commit) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${o.pr}'s head is not the stamp commit ${j.stamp_commit.slice(0, 12)}`, main.sha, lines, true);
const ci = ciAtHead(ctx, j.stamp_commit);
lines.push(`CI: ${ci.line}`);
if (!ci.green) return outPending({ code: 'QUEUE_NOT_READY', message: `checks on the stamp commit ${j.stamp_commit.slice(0, 12)} are not green` }, lines, `gstack-pr-wait ${o.pr} --until green && ${Q} train land --train ${t.nonce} --pr ${o.pr}`, run);
main = mainNow();
if (main.sha !== j.base) return stopped(env, t, 'QUEUE_TRAIN_TREE_MISMATCH', `main moved to ${main.sha.slice(0, 12)} during the slot's checks (stamped on ${j.base.slice(0, 12)})`, main.sha, lines, true);
const m = ctx.gh(['pr', 'merge', o.pr, '--squash', '--match-head-commit', j.stamp_commit]);
if (!m.ok) return outFailed({ code: 'QUEUE_NOT_READY', message: `gh pr merge refused: ${(m.err ?? m.out).trim().slice(0, 200)}` }, lines, `${Q} train land --train ${t.nonce} --pr ${o.pr}`, run);
main = mainNow();
if (main.tree !== s.tree) return stopped(env, t, 'QUEUE_TRAIN_TREE_MISMATCH', `after the merge main ${main.sha.slice(0, 12)} is not S${s.index} exactly`, main.sha, lines, true);
return confirmMerged(env, t, o.pr, main.sha, s.index, [...lines, `MERGED: #${o.pr} as ${main.sha}; main's tree is S${s.index} exactly`]);
}
+4
View File
@@ -34,12 +34,16 @@ export interface ScreenChoice {
}
export interface TrainSlot { pr: string; head: string; ready_at: string; screen: ScreenChoice }
export interface TrainPrefix { index: number; commit: string; tree: string; pr: string | null }
/** A stamped prefix S_i (changelog_fragments trains, lib/ship-queue-train-stamp.ts): what slot i publishes, exactly. */
export interface StampedPrefix { index: number; commit: string; tree: string; pr: string; version: string; date: string }
export interface FreeRun { run: number | null; url: string | null; head_sha: string | null; conclusion: string | null; verdict: 'green' | 'red' | 'missing' | 'stale' | 'cancelled' | 'pending' }
export interface TrainRecord {
schema: 1; kind: 'train'; nonce: string; key: string; repo_key: string; repo: string; base: { ref: string; sha: string }; policy: string;
slots: TrainSlot[]; held: string[]; prefixes: TrainPrefix[]; head: string; tree: string; branch: string;
urgent: boolean; repair: boolean; explicit_order?: boolean; state: TrainState; free: FreeRun | null; attempts: string[];
merged: Array<{ pr: string; sha: string; tree: string; at: string }>; superseded_by: string | null;
/** S_1..S_k when the base policy declares changelog_fragments; head and tree are then S_k's. Absent on plain trains. */
stamped?: StampedPrefix[];
effects: Array<{ at: string; what: string }>; error: string | null; created_at: string; updated_at: string;
}
export type AttemptPurpose = 'gate' | 'rebuild' | 'infra-redispatch' | 'bisect';
+3 -3
View File
@@ -17,10 +17,10 @@ import { restamp, versionAtRef, versionFiles } from './restamp';
import { matchesAny, type ShipPolicy } from './ship-policy';
import { git, type QueueFailure } from './ship-queue-gate';
import { mergeCommit } from './ship-queue-train';
import type { TrainPrefix, TrainSlot } from './ship-queue-train-record';
import type { StampedPrefix, TrainPrefix, TrainSlot } from './ship-queue-train-record';
import { buildReceipt } from './tree-receipt';
export interface StampedPrefix { index: number; commit: string; tree: string; pr: string; version: string; date: string }
export type { StampedPrefix };
export interface StampChainOptions {
repoRoot: string; base: string; policy: ShipPolicy; policyLabel: string; baseSha: string;
slots: TrainSlot[]; plain: TrainPrefix[]; allowRepoCommands: boolean; now: Date;
@@ -86,7 +86,7 @@ export function stampChain(o: StampChainOptions): { stamped: StampedPrefix[]; li
git(o.repoRoot, ['worktree', 'remove', '--force', wt]);
fs.rmSync(wt, { recursive: true, force: true });
}
const c = git(o.repoRoot, ['commit-tree', tree, '-p', pred, '-p', slot.head, '-m', `train S${i + 1}: #${slot.pr} v${version}`], { ...process.env, ...IDENTITY });
const c = git(o.repoRoot, ['commit-tree', tree, '-p', pred, '-p', slot.head, '-m', `train S${i + 1}: merge #${slot.pr} at ${slot.head.slice(0, 12)} v${version}`], { ...process.env, ...IDENTITY });
if (!c.ok) return { code: 'QUEUE_TRAIN_STAMP_MISMATCH', message: `commit-tree failed for S${i + 1}: ${c.err.slice(0, 120)}` };
const plain = o.plain[i + 1];
const why = plain ? stampMismatch(o, m.commit, c.out, plain.tree) : `no plain prefix T${i + 1}`;
+3 -2
View File
@@ -170,7 +170,7 @@ function verifiedElsewhere(env: TrainEnv, t: TrainRecord, mainSha: string): stri
* plain refusal (nothing was published). A GitHub state that cannot be read
* is neither: the verify refuses and asks to be rerun, never "nothing merged".
*/
async function stopped(env: TrainEnv, t: TrainRecord, code: 'QUEUE_TRAIN_TREE_MISMATCH' | 'QUEUE_TRAIN_MOVED', message: string, mainSha: string, lines: string[], fetched: boolean): Promise<TrainOut> {
export async function stopped(env: TrainEnv, t: TrainRecord, code: 'QUEUE_TRAIN_TREE_MISMATCH' | 'QUEUE_TRAIN_MOVED', message: string, mainSha: string, lines: string[], fetched: boolean): Promise<TrainOut> {
const run = trainFile(t.nonce);
// One GitHub read per unverified slot decides both the freeze and its record.
const states = new Map(t.slots.filter(s => !t.merged.some(m => m.pr === s.pr)).map(s => [s.pr, prState(env.ctx, s.pr)] as const));
@@ -197,6 +197,7 @@ export async function verifyTrain(ctx: QueueContext, o: { pr: string; train: str
const run = trainFile(t.nonce);
const i = t.slots.findIndex(s => s.pr === o.pr);
if (i < 0) return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} does not include #${o.pr}` }, [], nextFor(t), run);
if (t.stamped) return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} has stamped prefixes; its slots publish S_i exactly through train land, never a hand stamp checked modulo stamps` }, [], `${Q} train land --train ${t.nonce} --pr ${o.pr}`, run);
const gate = attemptsOf(t).filter(a => a.purpose !== 'bisect').at(-1);
if (gate?.outcome !== 'GREEN' || !['green', 'merging', 'merged'].includes(t.state)) {
return outRefused({ code: 'QUEUE_GATE_MISSING', message: `train ${t.nonce} has no green record (state ${t.state}, last gate ${gate?.outcome ?? 'none'})` }, [], nextFor(t), run);
@@ -230,7 +231,7 @@ export async function verifyTrain(ctx: QueueContext, o: { pr: string; train: str
`merge #${o.pr} with expected head ${head.slice(0, 12)} (gstack-restamp stamps at merge), then ${Q} verify --pr ${o.pr} --train ${t.nonce}`, run);
}
function confirmMerged(env: TrainEnv, t: TrainRecord, pr: string, main: string, index: number, lines: string[]): TrainOut {
export function confirmMerged(env: TrainEnv, t: TrainRecord, pr: string, main: string, index: number, lines: string[]): TrainOut {
const run = trainFile(t.nonce);
if (!t.merged.some(m => m.pr === pr)) t.merged.push({ pr, sha: main, tree: git(env.ctx.repoRoot, ['rev-parse', `${main}^{tree}`]).out, at: nowIso() });
t.state = t.merged.length === t.slots.length ? 'merged' : 'merging';
+9 -2
View File
@@ -26,6 +26,7 @@ import { bindShippedReceipt } from './ship-receipt-depth';
import { git, sha256, withIndex, type QueueFailure } from './ship-queue-gate';
import { recordVerdict, screenVerdict, verdictLines } from './ship-queue-screen';
import { stackTestLine, stackTests } from './ship-queue-stack-tests';
import { stampChain } from './ship-queue-train-stamp';
import { asRun, execFor, newestBranchRun, prInfo, remoteBranchSha, repoSlug, runView, checkRun } from './ship-queue-train-github';
import {
TERMINAL_STATES, activeFreeze, attemptsOf, listTrains, loadBisect, loadReady, loadTrain, nowIso, persist, rebuildFromNotes, repoIdentity, trainFile,
@@ -321,6 +322,8 @@ export interface BuildOptions {
pins?: string; order?: string; resume?: string; dryRun: boolean; urgent: boolean; repair: boolean; freeTimeoutSec: number; pollMs: number;
/** `--dry-run --stack-tests`: run each PR's new test files on its integrated prefix (lib/ship-queue-stack-tests.ts); the command's words when `--stack-tests-command` names one. */
stackTests?: string[] | true;
/** Stamping S_i runs the base policy's restamp_command or release_tool (changelog_fragments trains). */
allowRepoCommands?: boolean;
}
/** `--order <pr>,<pr>,...`: every pinned PR exactly once (a leading # allowed). */
@@ -412,13 +415,17 @@ async function newTrain(env: TrainEnv, o: BuildOptions): Promise<{ t: TrainRecor
if (due && !o.dryRun) return outPending({ code: 'QUEUE_TRAIN_WAITING', message: `${order.train.length} of ${env.train.max_size} PRs; the oldest READY dispatches at ${due}` }, lines, `${Q} train build ${prsArgs(order.train, !!explicit)} (after ${due}, or with more PRs, or --urgent for one)`, 'none');
const live = listTrains(env.repo.keys).find(t => t.key === key && !TERMINAL_STATES.includes(t.state) && t.state !== 'red' && t.state !== 'unattributed');
if (live) return { t: Object.assign(live, { held: order.held.map(s => s.pr) }), lines };
const head = chain.prefixes.at(-1)!;
const policy = env.policy.policy;
const stamped = policy.changelog_fragments ? stampChain({ repoRoot: env.ctx.repoRoot, base: env.ctx.base, policy, policyLabel: env.policy.source.label, baseSha, slots: order.train, plain: chain.prefixes, allowRepoCommands: !!o.allowRepoCommands, now: new Date(nowIso()) }) : null;
if (stamped && 'code' in stamped) return outFailed(stamped, lines, `${Q} train build ${prsArgs(order.train, !!explicit)} --allow-repo-commands (after the named slot is fixed or dropped)`, 'none');
if (stamped) lines.push(...stamped.lines);
const head = stamped ? stamped.stamped.at(-1)! : chain.prefixes.at(-1)!;
const nonce = makeNonce('gate', head.commit, baseSha, randomBytes(3).toString('hex'), { purpose: 'train' });
const now = nowIso();
return { lines, t: {
schema: 1, kind: 'train', nonce, key, repo_key: env.repo.key, repo: env.repo.url, base: { ref: env.ctx.base, sha: baseSha }, policy: env.policy.source.label,
slots: order.train, held: order.held.map(s => s.pr), prefixes: chain.prefixes, head: head.commit, tree: head.tree, branch: `train/${nonce}`,
urgent: o.urgent, repair: o.repair, ...(explicit ? { explicit_order: true } : {}), state: 'built', free: null, attempts: [], merged: [], superseded_by: null, effects: [], error: null,
urgent: o.urgent, repair: o.repair, ...(explicit ? { explicit_order: true } : {}), ...(stamped ? { stamped: stamped.stamped } : {}), state: 'built', free: null, attempts: [], merged: [], superseded_by: null, effects: [], error: null,
created_at: now, updated_at: now,
} };
}
+10 -6
View File
@@ -53,8 +53,11 @@ function noteRecords<T extends { kind: string; repo_key?: string }>(git: Git, ki
});
}
/** The newest green-train record whose prefix trees include this digest, with its run; null when none. */
export function coveringRecord(gh: GhRunner, repo: string, train: TrainPolicy, digest: string, lines: string[]): { run: Run; record: TrainRecord; index: number } | null {
/**
* The newest green-train record with a prefix covering main, with its run; null when none. A plain prefix T_i
* covers main's tree modulo stamps (its digest); a stamped prefix S_i covers only main's exact tree.
*/
export function coveringRecord(gh: GhRunner, repo: string, train: TrainPolicy, main: { digest: string | null; tree: string }, lines: string[]): { run: Run; record: TrainRecord; index: number; exact: boolean } | null {
const runs = json<{ workflow_runs?: Run[] }>(gh(['api', '-X', 'GET', `repos/${repo}/actions/workflows/${train.gate_workflow}/runs`, '-f', 'event=workflow_dispatch', '-f', 'status=success', '-f', 'per_page=50'], GH_MS));
if (!runs) { lines.push('cannot list the train gate runs; the backstop gates main'); return null; }
const trains = (runs.workflow_runs ?? []).map(run => ({ run, nonce: /\[eval-bg ([A-Za-z0-9._-]+)\]/.exec(run.display_title ?? '')?.[1] ?? '' }))
@@ -69,7 +72,7 @@ export function coveringRecord(gh: GhRunner, repo: string, train: TrainPolicy, d
try { record = file ? JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8')) as TrainRecord : null; } catch { record = null; }
if (!record || record.kind !== 'train-gate' || record.verdict !== 'GREEN' || record.census?.complete !== true || record.nonce !== nonce
|| record.head_sha !== run.head_sha || !Array.isArray(record.prefixes)) continue;
const hit = record.prefixes.find(p => p.stamp_free_tree === digest);
const hit = record.prefixes.find(p => p.exact ? p.tree === main.tree : main.digest !== null && p.stamp_free_tree === main.digest);
if (!hit) continue;
const checks = json<{ check_runs?: Array<{ name?: string; conclusion?: string | null }> }>(
gh(['api', `repos/${repo}/commits/${run.head_sha}/check-runs?check_name=${encodeURIComponent(train.full_gate_check)}`], GH_MS));
@@ -77,7 +80,7 @@ export function coveringRecord(gh: GhRunner, repo: string, train: TrainPolicy, d
lines.push(`train ${nonce} (run ${run.id}) lists this tree but its ${train.full_gate_check} check run is not a success`);
continue;
}
return { run, record, index: hit.index };
return { run, record, index: hit.index, exact: !!hit.exact };
}
return null;
} finally {
@@ -150,9 +153,10 @@ export function runBackstop(o: BackstopOptions): { action: BackstopAction; lines
}
const digest = stampFreeTree(git, o.sha, o.repoRoot, loaded.policy);
if (typeof digest !== 'string') { lines.push(`BACKSTOP: no tree modulo stamps (${digest.error}); gating main`); }
const covered = typeof digest === 'string' ? coveringRecord(gh, o.repo, train, digest, lines) : null;
const tree = git(['rev-parse', `${o.sha}^{tree}`]).out;
const covered = coveringRecord(gh, o.repo, train, { digest: typeof digest === 'string' ? digest : null, tree }, lines);
if (covered) {
lines.push(`BACKSTOP: covered: main ${o.sha.slice(0, 12)} is T${covered.index} of green train ${covered.record.nonce} (run ${covered.run.id}, tree modulo stamps ${String(digest).slice(0, 12)}); no gate`);
lines.push(`BACKSTOP: covered: main ${o.sha.slice(0, 12)} is ${covered.exact ? `S${covered.index}` : `T${covered.index}`} of green train ${covered.record.nonce} (run ${covered.run.id}, ${covered.exact ? `exact tree ${tree.slice(0, 12)}` : `tree modulo stamps ${String(digest).slice(0, 12)}`}); no gate`);
return done('covered');
}
const owned = noteRecords<FreezeRecord>(git, 'freeze', repoKey).find(f => !f.cleared_at && f.main.sha === o.sha && f.backstop);
+8 -5
View File
@@ -49,7 +49,7 @@ export const MAX_BISECT_CASES = 40;
/** A full gate's recorded cost (MEASUREMENT.md: $90.93 to $94.70; lib/gate-cost.ts uses the same scale). */
export const FULL_GATE_USD = 93.14;
/** T1's prefix commits: `train T<i>: merge #<pr> at <sha12>` (lib/ship-queue-train.ts buildChain). */
const PREFIX_SUBJECT = /^train T(\d+): merge #(\d+) at [0-9a-f]{12}$/;
const PREFIX_SUBJECT = /^train (?:T(\d+): merge #(\d+) at [0-9a-f]{12}|S(\d+): merge #(\d+) at [0-9a-f]{12} v[0-9.]+)$/;
export type Git = (args: string[]) => { ok: boolean; out: string };
export const gitIn = (cwd: string): Git => (args) => {
@@ -185,9 +185,10 @@ export function stampFreeTree(git: Git, commit: string, repoRoot: string, policy
return createHash('sha256').update(lines.sort().join('\n')).digest('hex');
}
export interface TrainPrefixTree { index: number; commit: string; tree: string; stamp_free_tree: string }
/** `exact`: a stamped prefix S_i (changelog_fragments trains), which covers only its exact tree. */
export interface TrainPrefixTree { index: number; commit: string; tree: string; stamp_free_tree: string; exact?: true }
/** T1..Tk from the train head, walking first parents over T1's prefix commits. */
/** T1..Tk (or stamped S1..Sk, never mixed) from the train head, walking first parents over the prefix commits. */
export function trainPrefixes(git: Git, head: string, repoRoot: string, policy: ShipPolicy): TrainPrefixTree[] | { error: string } {
const log = git(['log', '--first-parent', `--max-count=${MAX_PREFIXES}`, '--format=%H%x00%P%x00%s', head]);
if (!log.ok) return { error: `cannot walk ${head.slice(0, 12)}'s first parents (${log.out.split('\n')[0]})` };
@@ -196,11 +197,13 @@ export function trainPrefixes(git: Git, head: string, repoRoot: string, policy:
const [commit, parents, subject] = line.split('\0') as [string, string, string];
const m = PREFIX_SUBJECT.exec(subject ?? '');
if (!m || parents.split(' ').length !== 2) break;
const index = Number(m[1]);
const index = Number(m[1] ?? m[3]);
const exact = m[3] !== undefined;
if (found.length && exact !== !!found[0]!.exact) return { error: `${commit.slice(0, 12)} mixes stamped and plain prefixes` };
if (found.length && index !== found.at(-1)!.index - 1) return { error: `prefix T${index} follows T${found.at(-1)!.index} at ${commit.slice(0, 12)}` };
const digest = stampFreeTree(git, commit, repoRoot, policy);
if (typeof digest !== 'string') return digest;
found.push({ index, commit, tree: git(['rev-parse', `${commit}^{tree}`]).out, stamp_free_tree: digest });
found.push({ index, commit, tree: git(['rev-parse', `${commit}^{tree}`]).out, stamp_free_tree: digest, ...(exact ? { exact: true as const } : {}) });
}
if (!found.length || found.at(-1)!.index !== 1) return { error: `${head.slice(0, 12)} is not a train head (no T1..Tk prefix chain)` };
return found.reverse();
+132
View File
@@ -0,0 +1,132 @@
/**
* Stamped trains (changelog_fragments; GSTA-154 PR B): train build makes S_1..S_k beside T_1..T_k and gates
* S_k; train land publishes each slot's S_i exactly (a tool-made stamp commit on [pin, main], pushed under
* the pin's lease, labelled evals-no-debounce), confirms main is S_i, and freezes on any other main.
* Fixture repos with a bare origin and the train gh shim, wrapped for pr edit, pr merge and the PR's checks.
*/
import { afterEach, describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { EXIT } from '../lib/headless-artifacts';
import { cleanup } from './helpers/restamp-fixture';
import {
GREEN_GATE, TRAIN_POLICY, line, nonceOf, openPr, queue, readyReceipt, trainFixture, trainJson, type TrainFixture,
} from './helpers/ship-queue-train-fixture';
const POLICY = { ...TRAIN_POLICY, release_tool: null, release_outputs: [], changelog_fragments: { dir: 'changelog.d' }, min_gstack_version: '1.92.0.0' };
const fixtures: TrainFixture[] = [];
afterEach(() => { while (fixtures.length) cleanup(fixtures.pop()!.repo); });
/** pr view reads the head from origin's branch; pr edit and pr merge act on origin; any commit without recorded CI reads green. */
function wrap(fx: TrainFixture): void {
const dir = path.join(fx.repo.root, 'gh-land-wrap');
fs.mkdirSync(dir, { recursive: true });
const shim = path.join(fx.repo.root, 'gh-train-shim', 'shim.js');
fs.writeFileSync(path.join(dir, 'gh'), [
'#!/usr/bin/env node',
"const fs=require('fs');const cp=require('child_process');const D=process.env.LAND_WRAP_DATA;const a=process.argv.slice(2);",
"const d=JSON.parse(fs.readFileSync(D,'utf8'));const save=()=>fs.writeFileSync(D,JSON.stringify(d));",
"const g=(...x)=>cp.execFileSync('git',['--git-dir',d.origin,...x],{encoding:'utf8'}).trim();",
"const rev=r=>{try{return g('rev-parse','--verify','--quiet',r)}catch{return ''}};",
"if(a[0]==='pr'&&a[1]==='view'&&d.prs[a[2]]){const p=d.prs[a[2]];const h=rev('refs/heads/'+p.headRefName);if(h)p.headRefOid=h;save();console.log(JSON.stringify(p));process.exit(0)}",
"if(a[0]==='pr'&&a[1]==='edit'){(d.labels=d.labels||{})[a[2]]=a[a.indexOf('--add-label')+1];save();process.exit(0)}",
"if(a[0]==='pr'&&a[1]==='merge'){const p=d.prs[a[2]];const want=a[a.indexOf('--match-head-commit')+1];if(rev('refs/heads/'+p.headRefName)!==want){console.error('Head branch was modified');process.exit(1)}",
" const before=rev('main');let tree=rev(want+'^{tree}');",
" if(d.outsideOnMerge){const blob=cp.execFileSync('git',['--git-dir',d.origin,'hash-object','-w','--stdin'],{input:'outside\\n',encoding:'utf8'}).trim();",
" const ls=g('ls-tree',before+'^{tree}')+'\\n100644 blob '+blob+'\\toutside.txt';const ot=cp.execFileSync('git',['--git-dir',d.origin,'mktree'],{input:ls+'\\n',encoding:'utf8'}).trim();",
" g('update-ref','refs/heads/main',g('commit-tree',ot,'-p',before,'-m','outside'));tree=g('merge-tree','--write-tree','--merge-base='+before,rev('main'),want).split('\\n')[0];d.outsideOnMerge=false}",
" g('update-ref','refs/heads/main',g('commit-tree',tree,'-p',rev('main'),'-m','squash #'+a[2]));p.state='MERGED';save();process.exit(0)}",
"const paged=/commits\\/([0-9a-f]+)\\/(check-runs|status)\\?.*page=(\\d+)/.exec(a[1]||'');",
"if(a[0]==='api'&&paged&&!(d.ci||{})[paged[1]]&&!((d.statuses||{})[paged[1]])){const list=paged[2]==='check-runs'?[{name:'free-tests',status:'completed',conclusion:'success',id:1,completed_at:'2026-10-26T00:00:00Z'}]:[];const key=paged[2]==='check-runs'?'check_runs':'statuses';console.log(JSON.stringify(paged[3]==='1'?{total_count:list.length,state:'success',[key]:list}:{total_count:list.length,[key]:[]}));process.exit(0)}",
`const r=cp.spawnSync('node',[${JSON.stringify(shim)},D,...a],{stdio:'inherit'});process.exit(r.status??1);`,
'',
].join('\n'), { mode: 0o755 });
fx.env.PATH = `${dir}:${fx.env.PATH}`;
(fx.env as Record<string, string>).LAND_WRAP_DATA = fx.dataFile;
}
function fragmentPr(fx: TrainFixture, pr: string, fragment = `- #${pr} change\n`): string {
const head = openPr(fx, pr, { file: `changelog.d/pr-${pr}.md`, text: fragment, changelog: false });
fx.repo.git('push', '-q', 'origin', `${head}:refs/heads/pr-${pr}`);
readyReceipt(fx, pr, head);
return head;
}
function stampedTrain() {
const fx = trainFixture(POLICY, 'gstack-train-land-'); fixtures.push(fx); wrap(fx);
const h1 = fragmentPr(fx, '1');
const h2 = fragmentPr(fx, '2', '---\nbump: minor\n---\n- #2 change\n');
fx.write(d => { d.rules['evals.yml'] = GREEN_GATE; });
const built = queue(fx, ['train', 'build', '--prs', `1@${h1},2@${h2}`]);
const nonce = nonceOf(built.out);
return { fx, h1, h2, built, nonce };
}
const land = (fx: TrainFixture, nonce: string, pr: string) => queue(fx, ['train', 'land', '--train', nonce, '--pr', pr, '--no-wait']);
const originMain = (fx: TrainFixture) => fx.repo.git('--git-dir', fx.repo.origin, 'rev-parse', 'main');
describe('stamped trains: build and land', () => {
test('build records S1, S2 and gates S2; land publishes each slot exactly, with the label, and records the merge', () => {
const { fx, h1, h2, built, nonce } = stampedTrain();
expect(built.status, built.err).toBe(EXIT.ok);
expect(built.out).toMatch(/^S1: #1 v1\.2\.4\.0 [0-9a-f]{12} /m);
expect(built.out).toMatch(/^S2: #2 v1\.3\.0\.0 [0-9a-f]{12} /m);
const t = trainJson(fx, nonce);
expect(t.stamped.map((s: any) => [s.pr, s.version])).toEqual([['1', '1.2.4.0'], ['2', '1.3.0.0']]);
expect(t.head).toBe(t.stamped[1].commit);
expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok);
const verify = queue(fx, ['verify', '--pr', '1', '--train', nonce]);
expect(verify.err).toMatch(/has stamped prefixes.*\(QUEUE_GATE_MISSING\)/);
expect(line(verify.out, 'NEXT:')).toBe(`NEXT: gstack-ship-queue train land --train ${nonce} --pr 1`);
const early = land(fx, nonce, '2');
expect(early.err).toMatch(/is neither S1 nor S2 exactly.*not frozen.*\(QUEUE_TRAIN_TREE_MISMATCH\)/);
const pushed = land(fx, nonce, '1');
expect(pushed.out, pushed.err).toContain('LABEL: evals-no-debounce added');
const stamp = /^PUSHED: ([0-9a-f]{40})/m.exec(pushed.out)![1]!;
expect(fx.repo.git('--git-dir', fx.repo.origin, 'rev-parse', 'refs/heads/pr-1')).toBe(stamp);
expect(fx.repo.git('--git-dir', fx.repo.origin, 'rev-list', '--parents', '-n', '1', stamp).split(' ').slice(1)).toEqual([h1, originMain(fx)]);
expect(fx.repo.git('--git-dir', fx.repo.origin, 'rev-parse', stamp + '^{tree}')).toBe(t.stamped[0].tree);
const merged = land(fx, nonce, '1');
expect(merged.status, merged.err).toBe(EXIT.ok);
expect(merged.out).toContain("main's tree is S1 exactly");
expect(fx.repo.git('--git-dir', fx.repo.origin, 'rev-parse', 'main^{tree}')).toBe(t.stamped[0].tree);
land(fx, nonce, '2');
const second = land(fx, nonce, '2');
expect(second.status, second.err).toBe(EXIT.ok);
expect(fx.repo.git('--git-dir', fx.repo.origin, 'show', 'main:VERSION')).toBe('1.3.0.0');
expect(trainJson(fx, nonce)).toMatchObject({ state: 'merged' });
expect(fx.read().prs['2']).toMatchObject({ state: 'MERGED' });
expect((fx.read() as any).labels).toEqual({ 1: 'evals-no-debounce', 2: 'evals-no-debounce' });
void h2;
}, 300_000);
test('a foreign push to the PR branch is never overwritten; a merge outside the train freezes the queue', () => {
const { fx, nonce } = stampedTrain();
expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok);
fx.repo.git('push', '-q', '-f', 'origin', 'origin/main:refs/heads/pr-1');
const moved = land(fx, nonce, '1');
expect(moved.err).toContain('(QUEUE_TRAIN_MOVED)');
expect(fx.repo.git('--git-dir', fx.repo.origin, 'rev-parse', 'refs/heads/pr-1')).toBe(fx.repo.git('--git-dir', fx.repo.origin, 'rev-parse', 'main'));
const second = stampedTrain();
expect(queue(second.fx, ['train', 'verdict', '--train', second.nonce]).status).toBe(EXIT.ok);
land(second.fx, second.nonce, '1');
expect(land(second.fx, second.nonce, '1').status).toBe(EXIT.ok);
land(second.fx, second.nonce, '2');
second.fx.write(d => { (d as any).outsideOnMerge = true; });
const drift = land(second.fx, second.nonce, '2');
expect(drift.err).toContain('(QUEUE_TRAIN_TREE_MISMATCH)');
expect(line(drift.out, 'QUEUE_TRAIN_FROZEN:')).toMatch(/^QUEUE_TRAIN_FROZEN: train \S+ merged #1/);
}, 300_000);
test('a plain train refuses train land and keeps the slot-stamp path', () => {
const fx = trainFixture(TRAIN_POLICY, 'gstack-train-land-'); fixtures.push(fx); wrap(fx);
const h1 = openPr(fx, '1');
readyReceipt(fx, '1', h1);
fx.write(d => { d.rules['evals.yml'] = GREEN_GATE; });
const nonce = nonceOf(queue(fx, ['train', 'build', '--prs', `1@${h1}`, '--urgent']).out);
expect(trainJson(fx, nonce).stamped).toBeUndefined();
expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok);
const r = land(fx, nonce, '1');
expect(r.err).toMatch(/was built without stamped prefixes.*\(QUEUE_GATE_MISSING\)/);
}, 180_000);
});
+47
View File
@@ -223,3 +223,50 @@ describe('backstop: debounce while a train merges', () => {
expect(dispatches()).toEqual([]);
});
});
describe('backstop: a stamped train (changelog_fragments) covers only exact trees', () => {
test('main equal to S1 is covered as S1; the same tree with another VERSION is gated', () => {
const fx = trainFixture();
const { repo, main0, prs } = fx;
let parent = main0;
const stamped = prs.slice(0, 2).map((head, i) => {
repo.git('checkout', '-q', '-f', '--detach', parent);
repo.git('merge', '-q', '--no-ff', '--no-commit', head);
fs.writeFileSync(path.join(repo.work, 'VERSION'), `1.2.${4 + i}.0\n`);
repo.git('add', '-A');
parent = repo.git('commit-tree', repo.git('write-tree'), '-p', parent, '-p', head, '-m', `train S${i + 1}: merge #${101 + i} at ${head.slice(0, 12)} v1.2.${4 + i}.0`);
repo.git('merge', '--abort');
return parent;
});
repo.git('checkout', '-q', '-f', 'main');
const prefixes = trainPrefixes(gitIn(repo.work), parent, repo.work, loadPolicyFromHead(repo.work).policy);
if (!Array.isArray(prefixes)) throw new Error(prefixes.error);
expect(prefixes.map(p => [p.index, p.exact])).toEqual([[1, true], [2, true]]);
const record = { ...fx.record, head_sha: parent, tree: repo.git('rev-parse', `${parent}^{tree}`), prefixes };
const push = (tree: string) => {
const sha = repo.git('commit-tree', tree, '-p', repo.git('rev-parse', 'main'), '-m', 'squash');
repo.git('update-ref', 'refs/heads/main', sha);
repo.git('push', '-q', 'origin', 'main');
return sha;
};
const s1 = push(repo.git('rev-parse', `${stamped[0]}^{tree}`));
const covered = backstop({ ...fx, record }, s1, fakeGh({ ...fx, record }, { record }).gh);
expect(covered.action, covered.lines.join('\n')).toBe('covered');
expect(covered.lines.at(-1)).toContain(`is S1 of green train ${TRAIN_NONCE} (run 900, exact tree`);
repo.git('checkout', '-q', '-f', '--detach', stamped[0]!);
fs.writeFileSync(path.join(repo.work, 'VERSION'), '1.2.9.0\n');
repo.git('add', '-A');
const moduloStamps = push(repo.git('write-tree'));
repo.git('checkout', '-q', '-f', 'main');
const gated = backstop({ ...fx, record }, moduloStamps, fakeGh({ ...fx, record }, { record, gate: 'success' }).gh);
expect(gated.lines.join('\n')).toContain("no green-train record lists main's tree");
expect(gated.action).not.toBe('covered');
});
test('a chain that mixes stamped and plain prefixes is not a train head', () => {
const fx = trainFixture();
const { repo, train, prs } = fx;
const s = repo.git('commit-tree', repo.git('rev-parse', `${train[0]}^{tree}`), '-p', train[0]!, '-p', prs[1]!, '-m', `train S2: merge #102 at ${prs[1]!.slice(0, 12)} v1.2.5.0`);
expect(trainPrefixes(gitIn(repo.work), s, repo.work, loadPolicyFromHead(repo.work).policy)).toEqual({ error: `${train[0]!.slice(0, 12)} mixes stamped and plain prefixes` });
});
});