From e25997515cc071077313c6e2131ebc895af1bb3f Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 1 Nov 2026 19:23:35 -0800 Subject: [PATCH] v1.92.54.0 ship-queue: train land takes the shared land lock; verify proves the stamp; restamp stages its writes (#3599, #3531) (#3737) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(ship): pause /ship on a red base instead of failing (gstack-base-red, conditional receipts) (GSTA-157 PR 1) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: base-red shell runner launches bash through runBashSync Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: base-red coverage additions (unreproduced head, unresolved head, readPause, wait unconfigured) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: base-red pause in unattended endings; BASE_RED_UNCONFIGURED covers an invalid base policy Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: base-red coverage pass 2 (write --base-red, summary line, classify usage before fetch, unresolved-head detail) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * feat(ship-queue): merge trains for any repo — train.gates (workflows/command), every prefix gated, generalized train land (direct/update, merge method), base-red trains, train init/doctor, watch --land (GSTA-157 PR 2) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes (train gates: dispatch bound, lease, rebase, post-merge checks, output tail, caps) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: train land refuses the next slot until the previous slot's post-merge checks are green Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: ship-policy, troubleshooting and captain for train gates review fixes Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore: changelog fragment for merge trains for any repo Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: keep ship-queue-train-generic out of the Windows subset (gh shim resolves gh.exe there) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(ship-queue): train land takes the shared land lock (train-scoped, freeze-owned), verify --train proves the stamp and needs GitHub MERGED, restamp stages its writes (#3599, #3531) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes (train keeps the land lock through a mid-train refusal; freeze takeover reads the notes ref; a refused lock push is not reported as a taken lock) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: a freeze lock is taken over only on a freeze record cleared after it (pre-landing review pass 2) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: a stale restamp staging marker is dropped, not a failure; a failed dry run refuses before the real run; settleLock names the real lock-write failure (Step 11 review) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(ship-queue): land takes over a cleared freeze lock; deleted lock ref reads absent; unread verify head refuses QUEUE_PROOF_MISSING Step 11 adversarial findings on PR #3737: a freeze lock whose release push failed blocked every standalone land after its freeze cleared (freezeLockCleared shared by land and train land takeover); readLock kept a cached ref deleted on origin; verify --train --no-fetch judged the unstamped pin; the slot call now restores the worktree for an unrequested fifth file. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore(changelog): name the fragment after the branch slug Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(captain): the slot restamp stages the stamp files it wrote Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(ship-queue): verify --train refuses QUEUE_PROOF_MISSING on an unread head Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 16 + VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- bin/gstack-doctor-components.sh | 3 + bin/gstack-restamp | 81 ++++- bin/gstack-ship-queue | 6 +- captain/templates/slot-call.md | 10 +- docs/captain.md | 5 +- docs/ship-policy.md | 56 +++- docs/troubleshooting.md | 63 +++- lib/result-codes-release.ts | 20 +- lib/ship-queue-land-lock.ts | 137 +++++++++ lib/ship-queue-land.ts | 85 ++---- lib/ship-queue-train-cli.ts | 4 +- lib/ship-queue-train-gate.ts | 4 +- lib/ship-queue-train-land.ts | 81 ++++- lib/ship-queue-train-record.ts | 15 +- lib/ship-queue-train-verify.ts | 51 +++- package.json | 2 +- scripts/lib/windows-curation.ts | 4 + test/fixtures/module-size-ratchet.json | 1 + test/gstack-restamp-queue.test.ts | 101 ++++++- test/helpers/ship-queue-train-cases.ts | 37 ++- test/helpers/ship-queue-train-fixture.ts | 35 +++ test/ship-queue-land-lock.test.ts | 368 +++++++++++++++++++++++ test/ship-queue-train-freeze.test.ts | 10 +- test/ship-queue-train-land.test.ts | 43 +-- test/ship-queue-train-verify.test.ts | 168 ++++++++++- 28 files changed, 1248 insertions(+), 162 deletions(-) create mode 100644 lib/ship-queue-land-lock.ts create mode 100644 test/ship-queue-land-lock.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index e75ec6b6b..b70bafdea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## [1.92.54.0] - 2026-11-02 + +**`train land` and a single-PR `land` can no longer publish onto main at the same time, and a slot that forgot its version bump can no longer merge (#3599, #3531; GSTA-157).** Before, `train land` took no lock, so it and `land` could both read the same main and publish a combined tree neither tested. And train 33's `verify --train` said "may merge" for #3527's head after a bare commit dropped the stamp that restamp had written but not staged, so #3527 shipped under #3526's version. + +### Fixed +- **`train land` takes the shared land lock** (`refs/notes/gstack-land`, now in lib/ship-queue-land-lock.ts) as `train slot (#)`. Whichever of `land` and `train land` starts second refuses with `QUEUE_LAND_LOCKED`, naming the holder. Once a slot has merged, the train keeps the lock in phase `between` until its last slot lands, through a refusal on the next slot too, so no single land slips in between slots and sets off a freeze. A stop that freezes the queue leaves the lock in phase `freeze`, which only the admitted repair train takes over while the freeze lasts; once the freeze is cleared, the next `land` or `train land` takes over a freeze lock whose release push failed. A lock ref deleted on origin now reads as no lock, not as this clone's cached copy. +- **`--break-lock` prints the holder** (`LOCK_HOLDER:`), keeps land's staleness, mid-push and drift rules, never breaks a `freeze` lock, waits 10 minutes on a `between` lock, and refuses a live train run's lock unless `--break-train-lock` is also given. `train land` accepts both flags. +- **`verify --train` checks the slot stamp itself**: the head's VERSION must be above main's and its CHANGELOG headed with that version directly above main's top release, or it refuses with the new `QUEUE_TRAIN_UNSTAMPED`, naming the rule and both versions. It prints `STAMP_VERSION:` from the head. +- **`verify --train` calls a PR merged only when GitHub reports it MERGED.** An open release-only PR (#3530's shape) now gets the pre-merge check, and any other open or unreadable PR refuses with `QUEUE_PROOF_MISSING` without freezing the queue. So does a verify that could not read the PR's current head (`--no-fetch` or a failed fetch), instead of judging the unstamped pin. +- **`gstack-restamp` stages the files it writes** and prints `STAGED:`. It refuses before writing when a stamp file holds unstaged edits of your own (`RESTAMP_UNSTAGED_EDITS`); when staging fails, it says `RESTAMP_NOT_STAGED` and the next run stages the files. +- Two comments in lib/ship-queue-train-gate.ts now say "one spend cap per command gate across every prefix of the train", which is what the code does. + +### Tests +- `test/ship-queue-land-lock.test.ts`: two writers racing one lock sha, the four lock read states, owner kinds, the `--break-lock` rules, and both lock directions, the between phase, refusal and resume, and the freeze lock's repair takeover through `bin/gstack-ship-queue`. +- `test/ship-queue-train.test.ts`: the #3527 and #3530 shapes, the CHANGELOG rule, a claim-at-open pin, and no false freeze. `test/gstack-restamp-queue.test.ts`: staging after a no-commit merge, the unstaged-edits refusal, and a staging failure and its rerun. + ## [1.92.53.0] - 2026-11-02 **`reap` and `up` now destroy only your own Ubicloud VMs (GST-573).** Before, every `up` destroyed any VM in the project whose TTL had ended, whoever created it, which broke the rule that cloud deletes stay with the owner who made the resource. Now reap destroys only the caller's own expired VMs and its own old TTL-less ones, and only counts other owners' expired VMs in its log (`reap: left N expired VMs of other owners; only their owners reap them`). diff --git a/VERSION b/VERSION index 222a2a9f5..07152b9e4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.92.53.0 +1.92.54.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 716c0c598..2d83ab766 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.92.53.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.92.54.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/bin/gstack-doctor-components.sh b/bin/gstack-doctor-components.sh index 33f77842f..5bda51ce5 100644 --- a/bin/gstack-doctor-components.sh +++ b/bin/gstack-doctor-components.sh @@ -338,6 +338,9 @@ gstack_result_anchor() { RECEIPT_GIT_TOO_OLD) printf '%s' 'receipt-git-too-old' ;; CHANGELOG_FRAGMENT_INVALID) printf '%s' 'changelog-fragment-invalid' ;; QUEUE_LAND_LOCKED) printf '%s' 'queue-land-locked' ;; + QUEUE_TRAIN_UNSTAMPED) printf '%s' 'queue-train-unstamped' ;; + RESTAMP_NOT_STAGED) printf '%s' 'restamp-not-staged' ;; + RESTAMP_UNSTAGED_EDITS) printf '%s' 'restamp-unstaged-edits' ;; QUEUE_LAND_CONFLICT) printf '%s' 'queue-land-conflict' ;; QUEUE_LAND_DRIFT) printf '%s' 'queue-land-drift' ;; QUEUE_LAND_BASE_BUSY) printf '%s' 'queue-land-base-busy' ;; diff --git a/bin/gstack-restamp b/bin/gstack-restamp index b4922171b..ef6e44c7b 100755 --- a/bin/gstack-restamp +++ b/bin/gstack-restamp @@ -10,11 +10,18 @@ * Runs gstack-version-bump write, the policy's release_tool, the CHANGELOG * re-heading and the stamp_paths rewrites in an isolated staging worktree, * shows the diff limited to stamp files, and writes the branch only after - * every stage validated (journal: .gstack/tmp/restamp-journal.json). - * Idempotent; --dry-run writes nothing. Workflow command: ends with the + * every stage validated (journal: .gstack/tmp/restamp-journal.json), then + * stages what it wrote (`git add -A -- `, STAGED:), so a bare commit after + * a --no-commit base merge cannot drop the stamp (#3531). A stamp file holding + * unstaged edits of the caller's own refuses before anything is written + * (RESTAMP_UNSTAGED_EDITS); a staging failure leaves the files written and + * .gstack/tmp/restamp-staging.json, which the noop rerun stages, or drops when a + * file changed since (RESTAMP_NOT_STAGED). Idempotent; --dry-run writes nothing. Workflow command: ends with the * GSTACK_RESULT line. Logic: lib/restamp.ts; policy: lib/ship-policy.ts. */ import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; import { EXIT, exitTableHelp, formatResult } from '../lib/headless-artifacts'; import { GATE_AHEAD_REL, JOURNAL_REL, restamp, type RestampResult } from '../lib/restamp'; import { RESULT_CODES, type ResultCodeName } from '../lib/result-codes'; @@ -39,7 +46,7 @@ const HELP = `gstack-restamp — stamp the version, release tool, CHANGELOG and --dry-run print the full diff, write nothing --json JSON only on stdout -Prints STAMP_VERSION: , the stage table, the stamp diff and +Prints STAMP_VERSION: , the stage table, the stamp diff, STAGED: and GSTACK_RESULT: skill=restamp status=complete|refused|incomplete run=${JOURNAL_REL}. ${exitTableHelp()}`; @@ -48,6 +55,49 @@ function flag(args: string[], name: string): string | undefined { return i >= 0 ? args[i + 1] : undefined; } +const STAGING_REL = '.gstack/tmp/restamp-staging.json'; +interface Pending { files: string[]; blobs: Record } +interface Staging { staged?: string[]; dropped?: string[]; failure?: string } + +const gitIn = (root: string, args: string[]) => spawnSync('git', args, { cwd: root, encoding: 'utf8', timeout: 60_000 }); +/** Each file's blob sha in the worktree (one git call), null for a file that does not exist. */ +function blobsOf(root: string, files: string[]): Record { + const present = files.filter(f => fs.existsSync(path.join(root, f))); + const shas = present.length ? gitIn(root, ['hash-object', '--', ...present]).stdout.trim().split('\n') : []; + return Object.fromEntries(files.map(f => [f, present.includes(f) ? shas[present.indexOf(f)] ?? '' : null])); +} +function readPending(root: string): Pending | null { + try { return JSON.parse(fs.readFileSync(path.join(root, STAGING_REL), 'utf8')) as Pending; } catch { return null; } +} + +/** Files with unstaged edits that are the caller's own (not bytes a pending restamp wrote and could not stage). */ +function ownUnstaged(root: string): string[] { + const pending = readPending(root); + const unstaged = gitIn(root, ['diff', '--name-only', '-z']).stdout.split('\0').filter(Boolean); + const now = pending ? blobsOf(root, unstaged.filter(f => f in pending.blobs)) : {}; + return unstaged.filter(f => !(pending && f in now && pending.blobs[f] === now[f])); +} + +/** + * Stage what restamp wrote (`written`), or what a pending marker says is still unstaged; a marker file whose bytes + * changed since (a hand commit, another branch, a base merge) is stale for that file and dropped, never a failure. + * Null when there is nothing to stage. + */ +function stage(root: string, written: string[] | null): Staging | null { + const marker = path.join(root, STAGING_REL); + const pending = written ? { files: written, blobs: blobsOf(root, written) } : readPending(root); + if (!pending?.files.length) return null; + if (written) { fs.mkdirSync(path.dirname(marker), { recursive: true }); fs.writeFileSync(marker, JSON.stringify(pending, null, 2) + '\n'); } + const now = written ? pending.blobs : blobsOf(root, pending.files); + const dropped = pending.files.filter(f => now[f] !== pending.blobs[f]); + const files = pending.files.filter(f => !dropped.includes(f)); + if (!files.length) { fs.rmSync(marker, { force: true }); return { staged: [], dropped }; } + const add = gitIn(root, ['add', '-A', '--', ...files]); + if (add.status !== 0) return { failure: `git add of ${files.join(' ')} failed (${(add.stderr || add.stdout).trim().split('\n')[0]}); the stamp stays written and ${STAGING_REL} names it` }; + fs.rmSync(marker, { force: true }); + return { staged: gitIn(root, ['diff', '--cached', '--name-only', '-z', '--', ...files]).stdout.split('\0').filter(Boolean), dropped }; +} + function main(argv: string[]): number { if (argv.includes('--help') || argv.includes('-h')) { process.stdout.write(HELP + '\n'); return EXIT.ok; } const json = argv.includes('--json'); @@ -65,23 +115,32 @@ function main(argv: string[]): number { const after = flag(argv, '--after'); const base = resolveBaseBranch(repoRoot, flag(argv, '--base')); - const result = restamp({ + const options = { repoRoot, base, version: flag(argv, '--version'), next: argv.includes('--next'), bump, queueMode, after: after ? { pr: after, ref: flag(argv, '--predecessor-ref') } : undefined, pr: flag(argv, '--pr'), expectBase: flag(argv, '--expect-base'), was: flag(argv, '--was'), expectOrder: flag(argv, '--expect-order')?.split(',').map(s => s.trim()).filter(Boolean), policyFrom, allowRepoCommands: argv.includes('--allow-repo-commands'), dryRun: argv.includes('--dry-run'), failAfter: process.env.GSTACK_RESTAMP_FAIL_AFTER, - }); - return report(result, json); + }; + const writes = !after && !options.dryRun; + const own = writes ? ownUnstaged(repoRoot) : []; + const dry = own.length ? restamp({ ...options, dryRun: true }) : null; + if (dry && (dry.status === 'refused' || dry.status === 'failed')) return report(dry, json); + const clash = (dry?.files ?? []).filter(f => own.includes(f)); + if (clash.length) return report({ status: 'refused', code: 'RESTAMP_UNSTAGED_EDITS', message: `${clash.join(', ')} ${clash.length === 1 ? 'holds' : 'hold'} unstaged edits of your own, which staging the stamp would sweep into the stamp commit; nothing written`, stages: [] }, json); + const result = restamp(options); + const staging = writes && (result.status === 'stamped' || result.status === 'noop') ? stage(repoRoot, result.status === 'stamped' ? result.files ?? [] : null) : null; + return report(result, json, staging); } -function report(r: RestampResult, json: boolean): number { - const status = r.status === 'refused' ? 'refused' : r.status === 'failed' ? 'incomplete' : 'complete'; - const code = r.status === 'refused' ? EXIT.refused : r.status === 'failed' ? EXIT.fail : EXIT.ok; +function report(r: RestampResult, json: boolean, staging: Staging | null = null): number { + const status = r.status === 'refused' ? 'refused' : r.status === 'failed' || staging?.failure ? 'incomplete' : 'complete'; + const code = r.status === 'refused' ? EXIT.refused : r.status === 'failed' || staging?.failure ? EXIT.fail : EXIT.ok; if (json) { - process.stdout.write(JSON.stringify({ ...r, result: status }) + '\n'); + process.stdout.write(JSON.stringify({ ...r, ...(staging ?? {}), ...(staging?.failure && !r.code ? { code: 'RESTAMP_NOT_STAGED' } : {}), result: status }) + '\n'); return code; } + if (staging?.failure) process.stderr.write(`gstack-restamp: ${staging.failure}; fix: ${RESULT_CODES.RESTAMP_NOT_STAGED.fix} (RESTAMP_NOT_STAGED)\n`); if (r.code) { const row = RESULT_CODES[r.code as ResultCodeName]; process.stderr.write(`gstack-restamp: ${r.message}; fix: ${r.fix ?? row.fix} (${r.code})\n`); @@ -95,6 +154,8 @@ function report(r: RestampResult, json: boolean): number { if (r.gateAhead) process.stdout.write(`GATE_AHEAD: head=${r.gateAhead.gated_head} tree=${r.gateAhead.gated_tree} predecessor=#${r.gateAhead.predecessor_pr}@${r.gateAhead.predecessor_head.slice(0, 12)} worktree=${r.gateAhead.worktree} evidence_branch=${r.gateAhead.evidence_branch_key}\n`); if (r.diff) process.stdout.write(r.diff + (r.diff.endsWith('\n') ? '' : '\n')); process.stdout.write(`FILES: ${(r.files ?? []).join(' ') || '(none)'}\n`); + if (staging?.staged) process.stdout.write(`STAGED: ${staging.staged.join(' ') || '(none)'}\n`); + if (staging?.dropped?.length) process.stdout.write(`NOTE: ${staging.dropped.join(' ')} changed since an earlier restamp wrote ${staging.dropped.length === 1 ? 'it' : 'them'}; not staged, and that run's staging marker is dropped\n`); if (r.mergeCondition) process.stdout.write(`MERGE_CONDITION: sha=${r.mergeCondition}\n`); if (r.status === 'gate-ahead' && r.gateAhead) process.stdout.write(`next: run the verification gate in ${r.gateAhead.worktree}, then \`gstack-evidence bundle --out \` there; the branch is untouched. At this PR's turn rerun \`gstack-restamp --next\` here and read \`gstack-tree-receipt --from ${GATE_AHEAD_REL}\` before deciding what to rerun.\n`); else if ((r.status === 'stamped' || r.status === 'noop') && r.queueMode === 'stamp-at-merge') process.stdout.write(`next: NEW_VERSION=${r.newVersion}; commit and push only the stamp diff above (FILES), with no edit of your own: the train's verify accepts the CHANGELOG only as the base's plus this PR's entry re-headed; merge with the expected head ${r.mergeCondition ?? 'sha'} and recompute on a stale queue.\n`); diff --git a/bin/gstack-ship-queue b/bin/gstack-ship-queue index f72bd0e8f..738acc9c2 100755 --- a/bin/gstack-ship-queue +++ b/bin/gstack-ship-queue @@ -14,7 +14,7 @@ * gstack-ship-queue gate-ahead --list --pr * gstack-ship-queue verify --pr --screen (train policy: the head's screen verdict from its run) * gstack-ship-queue verify --post-merge [--timeout ] (after a merge: main's push workflows at that commit until they settle) - * gstack-ship-queue land --pr [--dry-run] [--no-wait] [--break-lock] (the merge step; lib/ship-queue-land.ts) + * gstack-ship-queue land --pr [--dry-run] [--no-wait] [--break-lock [--break-train-lock]] (the merge step; lib/ship-queue-land.ts) * gstack-ship-queue land --pr --repair (after QUEUE_LAND_DRIFT: open the outside merge's release PR) * gstack-ship-queue land --census * gstack-ship-queue train build|status|verdict|bisect|rebuild|watch|prune ... (merge trains; train --help) @@ -57,7 +57,7 @@ const HELP = `gstack-ship-queue — the shared merge queue's protocol steps (doc verify --pr --screen (train policy) the PR head's screen verdict read from its run (SCREEN_VERDICT) verify --post-merge [--timeout ] wait for every check run at a merged commit to settle: POST_MERGE: green|red|pending and one CHECK: line per workflow; resumable, default timeout 2700 s - land --pr [--dry-run] [--no-wait] [--timeout ] [--break-lock] [--allow-repo-commands] + land --pr [--dry-run] [--no-wait] [--timeout ] [--break-lock [--break-train-lock]] [--allow-repo-commands] the merge step: lock, merge the base, stamp (fragments folded), push the stamp commit, wait for its checks, squash with --match-head-commit, confirm the base tree; resumable land --pr --repair after QUEUE_LAND_DRIFT: open the PR that gives the merge outside land its own @@ -172,7 +172,7 @@ async function main(argv: string[]): Promise { if (args.includes('--census')) return finish('land', l.census(ctx)); if (!pr || !/^\d+$/.test(pr)) return usage('land needs --pr (or --census)'); const timeout = flag(args, '--timeout'); - return finish('land', l.land(ctx, { pr, repair: args.includes('--repair'), dryRun: args.includes('--dry-run'), noWait: args.includes('--no-wait'), breakLock: args.includes('--break-lock'), allowRepoCommands: args.includes('--allow-repo-commands'), waitTimeoutMin: timeout && /^\d+$/.test(timeout) ? Number(timeout) : undefined })); + return finish('land', l.land(ctx, { pr, repair: args.includes('--repair'), dryRun: args.includes('--dry-run'), noWait: args.includes('--no-wait'), breakLock: args.includes('--break-lock'), breakTrainLock: args.includes('--break-train-lock'), allowRepoCommands: args.includes('--allow-repo-commands'), waitTimeoutMin: timeout && /^\d+$/.test(timeout) ? Number(timeout) : undefined })); } if (!pr || !/^\d+$/.test(pr)) return usage(`${cmd} needs --pr `); const gateExit = flag(args, '--gate-exit'); diff --git a/captain/templates/slot-call.md b/captain/templates/slot-call.md index f1908884a..36d70f1db 100644 --- a/captain/templates/slot-call.md +++ b/captain/templates/slot-call.md @@ -12,10 +12,12 @@ Run: gstack-restamp --next --base --pr --expect-base --expect-o Slot 2 and later (and slot 1 when BASE moved): merge BASE into your pin first (expect a CHANGELOG-only conflict; resolve it with mergeChangelogs from lib/changelog-check.ts, the one-liner in docs/ship-policy.md's merge-train section, never by hand; grep for conflict -markers before you stamp). Commit only the BASE merge and the stamp files this call names: - (plus the file carrying a `*_KEY_VERSION` your PR introduced, on your own -line only, when this call names it), never a fifth: a file the restamp lists that this call -does not name is not a stamp, leave it out and say so. +markers before you stamp). The restamp stages what it writes (STAGED:); commit what it +staged, which must be only the BASE merge and the stamp files this call names: +(plus the file carrying a `*_KEY_VERSION` your PR introduced, on your own line only, when this +call names it), never a fifth: a file the call does not name is not a stamp, `git restore +--staged --worktree` it and say so. verify reads the head's own STAMP_VERSION and refuses an unstamped +head (QUEUE_TRAIN_UNSTAMPED). A pin stacked on a PR that merged earlier conflicts beyond the CHANGELOG where the parent's squash touched your lines: take your pin's bytes there (verify checks the tree against T); mergeChangelogs carries your edits to the parent's released entry as Gate integrity diff --git a/docs/captain.md b/docs/captain.md index 3b0ee7d45..cdc83da98 100644 --- a/docs/captain.md +++ b/docs/captain.md @@ -317,8 +317,9 @@ stamp on the bare pin leaves the PR conflicting on the CHANGELOG. The lane merge into its pin first (expect a CHANGELOG-only conflict; resolve it with `mergeChangelogs` from `lib/changelog-check.ts`, the `bun -e` one-liner in [ship-policy.md](ship-policy.md#run-a-merge-train), never by hand, and grep for conflict -markers before stamping), then restamps, then commits only the merge and the stamp -files. In practice slot 1 needs the same merge whenever the base moved after the pin was +markers before stamping), then restamps (the restamp stages the stamp files it wrote, +so a plain `git commit` after a `--no-commit` merge keeps them), then commits only the +merge and the stamp files. In practice slot 1 needs the same merge whenever the base moved after the pin was taken. A stamp commit with one extra byte freezes the queue for every lane behind it. diff --git a/docs/ship-policy.md b/docs/ship-policy.md index 0d3fc4493..26f3ef71e 100644 --- a/docs/ship-policy.md +++ b/docs/ship-policy.md @@ -401,6 +401,28 @@ release paths (version files, CHANGELOG, fragments, `release_outputs`, train head and branch are S_k, and the one full gate runs on S_k, the exact tree the last merge publishes. +**One land lock.** `train land` takes the same repository land lock as +[`land`](#land), `refs/notes/gstack-land`, as `train slot (#)`, +before its first read of main, so a standalone land and a train slot never +both publish onto one main: whichever starts second refuses with +`QUEUE_LAND_LOCKED`, naming the holder. Once a slot has merged, the train keeps +the lock in phase `between`, which only the same train's next slot takes over, +until its last landable slot lands; a refusal before a later slot's push (a +failed `gh` or `git fetch` read, say) keeps it too, so no standalone land slips +in while the train waits on a rerun. Before any slot merged, a refusal before +the push releases it. After a push, a failure keeps the lock in its phase and a +rerun of the same slot takes it back. A stop that freezes the queue +leaves the lock in phase `freeze`, which `--break-lock` never breaks: only the +repair train the freeze admits takes it over, and it is released when that +train clears the freeze. `--break-lock` prints the holder (`LOCK_HOLDER:`), +keeps land's staleness, mid-push and drift rules, refuses a `between` lock +younger than 10 minutes, and refuses a live train run's lock unless +`--break-train-lock` is also given. `train watch --land` goes through `train +land`, so it stops with `NEXT:` on `QUEUE_LAND_LOCKED` like any refusal. A train +`train land` refuses (one built without stamped prefixes or a gate) still merges +through `verify --train` and `gh pr merge`, which neither take nor check this +lock. + Slots merge with `train land --train --pr `, never a slot call: 1. The record must be the gated head's own chain. Walking first parents from @@ -851,8 +873,8 @@ git fetch origin main git merge --no-ff --no-commit origin/main # expect a CHANGELOG.md conflict only bun --no-env-file --config=/dev/null -e "import { mergeChangelogsWithNotes } from './lib/changelog-check'; import { spawnSync } from 'node:child_process'; import { writeFileSync } from 'node:fs'; const s = (n) => spawnSync('git', ['show', ':' + n + ':CHANGELOG.md'], { encoding: 'utf8', maxBuffer: 1 << 26, timeout: 30000 }).stdout; const m = mergeChangelogsWithNotes(s(2), s(3), s(1)); for (const n of m.notes) console.error('CHANGELOG: carried as a Gate integrity correction: ' + n); if (!m.text) { console.error('CHANGELOG needs a hand merge'); process.exit(1); } writeFileSync('CHANGELOG.md', m.text);" git add CHANGELOG.md && git commit --no-edit -gstack-restamp --next --base main --pr 42 --expect-base
--expect-order --allow-repo-commands -git add CHANGELOG.md VERSION package.json && git commit -m "v stamp" && git push +gstack-restamp --next --base main --pr 42 --expect-base
--expect-order --allow-repo-commands # stages what it writes (STAGED:) +git commit -m "v stamp" && git push ``` A conflict in any file other than CHANGELOG.md means the merge isn't the train's prefix. Stop and report it. When the helper says the CHANGELOG needs a hand merge, take main's CHANGELOG unchanged and put your own entry above it. `verify` accepts the resulting head because it contains main and its tree equals T modulo stamps, with its CHANGELOG judged as main's plus lines your pin added, however the conflict was resolved; the `head:` line says "T with main merged in". @@ -865,8 +887,12 @@ introduced). A line the base added since the pin's fork point is the base's, whether the base merge is committed or still staged (`stage stamp_paths` says "N line(s) the base added since the fork point are the base's"; train 24, 2026-10-27, moved main's `DOCS_CAPS_KEY_VERSION` in both slots before this -rule). A file the restamp lists that the slot call did not name is not a stamp: -leave it out and report it. The tree receipt refuses such a move after the fact +rule). Restamp stages what it writes and prints `STAGED:`; commit what restamp +staged. A file the slot call does not name is not a stamp: `git restore --staged +--worktree` it and report it. A stamp file holding unstaged edits of your own refuses before +anything is written (`RESTAMP_UNSTAGED_EDITS`), and a staging failure leaves the +files written for the noop rerun to stage (`RESTAMP_NOT_STAGED`), so a bare +commit after a `--no-commit` base merge can no longer drop the stamp (#3527). The tree receipt refuses such a move after the fact only while the moved version is older than the base's at the build, so the restamp, not the receipt, is what keeps the base's key in place. @@ -937,7 +963,18 @@ saw, itself the prefix before modulo stamps) plus the PR's own entry under the same rule. One CHANGELOG rule applies before and after, so a head the pre-merge verify accepted confirms after its squash merge with the same CHANGELOG, and a stacked child whose one `[Unreleased]` became two released sections is not a -mismatch (train 24, 2026-10-27, froze on exactly that). Anything else is +mismatch (train 24, 2026-10-27, froze on exactly that). Before +`MERGE_CONDITION`, the head must also carry its own stamp: its `VERSION` above +main's, and its CHANGELOG headed with that version directly above main's top +release, with no `[Unreleased]` section main does not have; otherwise +`QUEUE_TRAIN_UNSTAMPED` names the rule and both versions (train 33 merged #3527 +under #3526's version without it). verify prints `STAMP_VERSION:` from the head +itself. A post-merge confirmation needs GitHub to report the PR MERGED: an open +PR whose stamp-only diff already matches main (a release-only PR, #3530) takes +the pre-merge path, and any other open or unreadable PR refuses +`QUEUE_PROOF_MISSING` without freezing, as does a verify that could not read +the PR's current head (`--no-fetch` or a failed fetch) instead of judging the +unstamped pin. Anything else is `QUEUE_TRAIN_TREE_MISMATCH` (or `QUEUE_TRAIN_MOVED`). When nothing of the train has merged, that is a plain refusal, and a new train is built on current main. When earlier slots already merged, main holds a prefix no gate covered on its own, @@ -1629,9 +1666,12 @@ command: 2. **Lock.** Take the repository land lock, `refs/notes/gstack-land` (its own ref, never the trains notes ref), by exact lease. It records the PR, holder, phase and journal. A held lock is `QUEUE_LAND_LOCKED` naming the holder; - `--break-lock` takes over a crashed holder's lock and is refused while that - holder entered a push or a merge less than 10 minutes ago, and for a drift - lock. A refusal before the push releases the lock. + `--break-lock` prints the holder (`LOCK_HOLDER:`), takes over a crashed + holder's lock and is refused while that holder entered a push or a merge + less than 10 minutes ago, and for a drift lock. `train land` shares this + lock ([one land lock](#train-land)); a train slot's lock needs + `--break-train-lock` as well while its train still lands, and a `freeze` + lock is never broken. A refusal before the push releases the lock. 3. **Merge and stamp.** In a scratch worktree, merge the base into the pin. Conflicts in release fields only (the version file and `release_outputs` take the base's side, the CHANGELOG merges by section, a manifest or npm diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 378a8b708..159dda2f4 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -4604,7 +4604,10 @@ train still building, running or merging, `train watch` on an unknown record, unreadable run), `verify --pr --train ` when main's tree mismatches and whether a slot merged on GitHub cannot be read (so whether the queue must freeze is unknown; it never answers "nothing merged" from an unreadable -state), `train land` when the base or the train branch cannot be fetched or +state), `verify --pr --train ` when GitHub does not report the PR +MERGED and it is not a release-only PR whose stamp-only diff already matches +main, or when it could not read the PR's current head (`--no-fetch` or a failed +fetch), without freezing the queue, `train land` when the base or the train branch cannot be fetched or the record's T0 and S_1..S_k are not the gated head's first-parent chain (each S_i's second parent slot i's pin), `train land` when main's tree is the slot's S_i but GitHub does not report the PR as MERGED (a record whose slot @@ -4710,14 +4713,60 @@ the name already landed. Then rerun. ### `QUEUE_LAND_LOCKED` -**Meaning.** Another `gstack-ship-queue land` holds the repository land lock -(`refs/notes/gstack-land`), or took it from under this run, so this one would -race it for the next version. The line names the holder, its PR and phase. +**Meaning.** Another `gstack-ship-queue land`, or a `train land` slot, holds the +repository land lock (`refs/notes/gstack-land`), or took it from under this run, +so this one would race it onto main. The line names the holder: `#` for a +land, `train slot (#)` for a train, with its holder and phase. A +train keeps the lock in phase `between` from one merged slot to the next, so no +standalone land slips in between slots. **Fix.** Wait for the holder to finish. If it crashed, rerun with -`--break-lock`; that is refused while the holder entered a push or a merge -less than 10 minutes ago. A lock in phase `drift` is released by landing its -repair PR ([QUEUE_LAND_DRIFT](#queue-land-drift)). +`--break-lock`, which prints the holder (`LOCK_HOLDER:`). That is refused while +the holder entered a push or a merge less than 10 minutes ago, while a train is +between slots for less than 10 minutes, and for a live train run unless you also +pass `--break-train-lock`. A lock in phase `drift` is released by landing its +repair PR ([QUEUE_LAND_DRIFT](#queue-land-drift)); a lock in phase `freeze` is +released by the verify that clears the freeze +([QUEUE_TRAIN_FROZEN](#queue-train-frozen)). If that release push failed, the +next `land` or `train land` takes the lock over once the freeze record it fetches +shows the freeze cleared after the lock was written; `--break-lock` never does. + + +### `QUEUE_TRAIN_UNSTAMPED` + +**Meaning.** `verify --pr --train ` found the slot head it would let +merge without its own stamp: the head's `VERSION` is not above main's, or its +CHANGELOG's top section is not headed with that version directly above main's +top release (or keeps an `[Unreleased]` section main does not have). The line +names the rule, the head's version and main's. Train 33 merged #3527 this way +(a no-commit base merge, a restamp whose files were never staged, a bare +commit) and shipped it under #3526's version. Nothing was merged. + +**Fix.** On the lane, run `gstack-restamp --next` on the head, commit what +restamp staged, push, and rerun the verify. + + +### `RESTAMP_NOT_STAGED` + +**Meaning.** `gstack-restamp` wrote the stamp but `git add` failed, so a bare +`git commit` would leave the stamp out. The files stay written, and +`.gstack/tmp/restamp-staging.json` records what is still to stage. + +**Fix.** Clear what blocked `git add` (a stale `.git/index.lock`, for +example) and rerun `gstack-restamp`; the rerun is a noop that stages the files +it wrote. When any of them changed since restamp wrote it (you committed by +hand, switched branches or merged the base), the marker is stale: the noop +rerun drops it and stages nothing, and a stamp still owed is restamped as usual. + + +### `RESTAMP_UNSTAGED_EDITS` + +**Meaning.** A file `gstack-restamp` would stamp holds unstaged edits of yours. +Restamp stages what it writes, so those edits would ride along in the stamp +commit. Nothing was written. + +**Fix.** Stage (`git add`) or stash your own edits to the named files, then +rerun `gstack-restamp`. ### `QUEUE_LAND_CONFLICT` diff --git a/lib/result-codes-release.ts b/lib/result-codes-release.ts index 9c6496756..3797aeae2 100644 --- a/lib/result-codes-release.ts +++ b/lib/result-codes-release.ts @@ -418,8 +418,24 @@ export const RELEASE_RESULT_CODES = { }, QUEUE_LAND_LOCKED: { anchor: 'queue-land-locked', - summary: 'another gstack-ship-queue land holds the repository land lock (refs/notes/gstack-land), so this one would race it for the next version', - fix: 'wait for the holder named in the line to finish; if it crashed, gstack-ship-queue land --pr --break-lock (refused while its journal shows a push in flight)', + summary: 'another gstack-ship-queue land or train land slot holds the repository land lock (refs/notes/gstack-land), so this one would race it onto main', + fix: 'wait for the holder named in the line to finish; if it crashed, rerun with --break-lock (refused while it is mid-push or mid-merge, between train slots less than 10 minutes, and for a live train run without --break-train-lock)', + }, + // #3531: verify --train's slot stamp proof and gstack-restamp's staging of what it writes. + QUEUE_TRAIN_UNSTAMPED: { + anchor: 'queue-train-unstamped', + summary: 'the slot head verify --train would let merge is not stamped above main: its VERSION is not above main\'s, or its CHANGELOG is not headed with that version directly above main\'s top release', + fix: 'on the lane, run gstack-restamp --next on the head, commit what restamp staged, push, then rerun gstack-ship-queue verify --pr --train ', + }, + RESTAMP_NOT_STAGED: { + anchor: 'restamp-not-staged', + summary: 'gstack-restamp wrote the stamp but could not stage it, so a bare git commit would leave the stamp out', + fix: 'clear what blocked git add (an index.lock), then rerun gstack-restamp: the noop rerun stages the files it wrote, or drops the marker when they changed since', + }, + RESTAMP_UNSTAGED_EDITS: { + anchor: 'restamp-unstaged-edits', + summary: 'a file gstack-restamp would stamp holds unstaged edits of the caller, which staging the stamp would sweep into the stamp commit; nothing was written', + fix: 'stage (git add) or stash your own edits to the files named, then rerun gstack-restamp', }, QUEUE_LAND_CONFLICT: { anchor: 'queue-land-conflict', diff --git a/lib/ship-queue-land-lock.ts b/lib/ship-queue-land-lock.ts new file mode 100644 index 000000000..9751e3754 --- /dev/null +++ b/lib/ship-queue-land-lock.ts @@ -0,0 +1,137 @@ +/** + * The repository land lock (docs/ship-policy.md#land, #3599): one ref, refs/notes/gstack-land, whose commit + * holds lock.json, shared by `gstack-ship-queue land` (owner `{ pr }`) and `train land` (owner + * `{ train, slot, pr }`), so a standalone land and a train slot never both publish onto one main. Every write + * is an exact lease on the sha read, so two writers racing one sha leave one winner. A lock is resumed only by + * the same owner kind, identity and journal; anything else is a takeover the caller allows (a drift repair, a + * train's next slot, the admitted repair train) or a --break-lock under the staleness, mid-push, drift, freeze + * and live-train rules. + */ +import { git, type QueueFailure } from './ship-queue-gate'; + +export const LAND_LOCK_REF = 'refs/notes/gstack-land'; +export const PUSH_STALE_MS = 10 * 60_000; +export type LandLockPhase = 'stamping' | 'pushing' | 'waiting' | 'merging' | 'drift' | 'between' | 'freeze'; +export interface LandLock { pr: string; holder: string; started_at: string; phase: LandLockPhase; journal: string; phase_at?: string; repair_pr?: string; drift_of?: string; train?: string; slot?: number } +export type LockOwner = { pr: string; train?: undefined; slot?: undefined } | { pr: string; train: string; slot: number }; +export type LockRead = { state: 'absent'; sha: '' } | { state: 'released'; sha: string } | { state: 'held'; sha: string; lock: LandLock } | { state: 'unreadable'; sha: string; detail: string }; +type Released = { released: true; by: string; at: string }; + +const isSha = (s: string | undefined) => /^[0-9a-f]{40,64}$/.test(s ?? ''); +/** How a lock names its owner in every message: `#`, or `train slot (#)`. */ +export const lockLabel = (lock: LandLock): string => (lock.train ? `train ${lock.train} slot ${lock.slot} (#${lock.pr})` : `#${lock.pr}`); +/** The same owner kind, identity and journal: a resume of this run's journal, never just the same PR. */ +export const ownsLock = (lock: LandLock, owner: LockOwner, journal: string): boolean => + lock.pr === owner.pr && (lock.train ?? null) === (owner.train ?? null) && (lock.slot ?? null) === (owner.slot ?? null) && lock.journal === journal; + +export function readLock(root: string, remote = 'origin'): LockRead { + const fetched = git(root, ['fetch', '-q', remote, `+${LAND_LOCK_REF}:${LAND_LOCK_REF}`]); + if (!fetched.ok) { + const listed = git(root, ['ls-remote', remote, LAND_LOCK_REF]); + if (listed.ok && !listed.out.trim()) git(root, ['update-ref', '-d', LAND_LOCK_REF]); + } + const sha = git(root, ['rev-parse', '--verify', '--quiet', LAND_LOCK_REF]).out; + if (!sha) return { state: 'absent', sha: '' }; + const body = git(root, ['show', `${sha}:lock.json`]); + if (!body.ok) return { state: 'unreadable', sha, detail: 'no lock.json' }; + let value: Partial & { released?: boolean }; + try { value = JSON.parse(body.out); } catch { return { state: 'unreadable', sha, detail: 'lock.json is not JSON' }; } + if (value?.released === true) return { state: 'released', sha }; + if (!/^\d+$/.test(String(value?.pr)) || typeof value.phase !== 'string' || typeof value.holder !== 'string') return { state: 'unreadable', sha, detail: 'lock.json names no PR, holder and phase' }; + return { state: 'held', sha, lock: value as LandLock }; +} + +export function writeLock(root: string, expected: string, value: LandLock | Released, remote = 'origin'): { ok: boolean; sha: string } { + const blob = git(root, ['hash-object', '-w', '--stdin'], undefined, JSON.stringify(value, null, 2) + '\n').out; + const tree = git(root, ['mktree'], undefined, `100644 blob ${blob}\tlock.json\n`).out; + const message = 'released' in value ? `land lock released by ${value.by}` : `land lock ${value.train ? `train ${value.train} slot ${value.slot} ` : ''}#${value.pr} ${value.phase}`; + const commit = git(root, ['commit-tree', tree, ...(expected ? ['-p', expected] : []), '-m', message]).out; + if (![blob, tree, commit].every(isSha)) return { ok: false, sha: '' }; + const push = git(root, ['push', '-q', `--force-with-lease=${LAND_LOCK_REF}:${expected}`, remote, `${commit}:${LAND_LOCK_REF}`]); + if (push.ok) git(root, ['update-ref', LAND_LOCK_REF, commit]); + return { ok: push.ok, sha: commit }; +} + +export interface HeldLandLock { + /** The lock as this run last wrote it. */ + readonly lock: LandLock; + /** This owner's lock as it was before this run took it back (a resume), else null. */ + readonly resumed: LandLock | null; + /** Move to a phase under the exact lease; a failure is another lander's lock (QUEUE_LAND_LOCKED) or a refused push (QUEUE_NOT_READY). */ + set(phase: LandLockPhase, extra?: Partial): QueueFailure | null; + release(): boolean; +} +export interface AcquireOptions { + root: string; remote?: string; holder: string; owner: LockOwner; journal: string; now: () => Date; + /** The first phase, given this owner's own lock when this run resumes it. */ + phase: (own: LandLock | null) => LandLockPhase; + breakLock?: boolean; breakTrainLock?: boolean; + /** A takeover the caller allows (a drift repair, a train's next slot, the admitted repair train): its LOCK: line, else null. */ + takeover?: (lock: LandLock) => string | null; + /** Whether a train still lands (its record is not terminal); unknown counts as live. */ + liveTrain?: (nonce: string) => boolean; + /** Fields every write of this run carries (land's drift_of), given this owner's own lock when resumed. */ + extra?: (own: LandLock | null) => Partial; +} + +/** Why --break-lock may not break a foreign lock, or null when it may. */ +function unbreakable(lock: LandLock, o: AcquireOptions, age: number): QueueFailure | null { + const label = lockLabel(lock); + if (lock.phase === 'drift' || lock.drift_of) return { code: 'QUEUE_LAND_DRIFT', message: `#${lock.drift_of ?? lock.pr} holds a drift lock; --break-lock never breaks it, only its repair PR's land takes it over`, fix: `gstack-ship-queue land --pr ${lock.drift_of ?? lock.pr} --repair` }; + if (lock.phase === 'freeze') return { code: 'QUEUE_TRAIN_FROZEN', message: `${label} holds the land lock for a frozen queue; --break-lock never breaks it, only the repair train the freeze admits takes it over` }; + if (lock.train && (o.liveTrain?.(lock.train) ?? true) && !o.breakTrainLock) return { code: 'QUEUE_LAND_LOCKED', message: `${label} is a live train run's lock (phase ${lock.phase}); --break-lock refused without --break-train-lock` }; + if ((lock.phase === 'pushing' || lock.phase === 'merging') && age < PUSH_STALE_MS) return { code: 'QUEUE_LAND_LOCKED', message: `${label}'s holder is mid-${lock.phase === 'pushing' ? 'push' : 'merge'} (${Math.round(age / 1000)} s ago); --break-lock refused` }; + if (lock.phase === 'between' && age < PUSH_STALE_MS) return { code: 'QUEUE_LAND_LOCKED', message: `${label} is between slots (${Math.round(age / 1000)} s ago); --break-lock refused until it is ${PUSH_STALE_MS / 60_000} minutes old` }; + return null; +} + +/** + * Take the land lock for `owner`: resume this owner's own lock, take over one the caller allows, or (with + * --break-lock, which always names the holder) break a foreign one the rules permit; then write the first + * phase under the exact lease on the sha read. + */ +export function acquireLandLock(o: AcquireOptions): { held: HeldLandLock; lines: string[] } | { failure: QueueFailure; lines: string[] } { + const remote = o.remote ?? 'origin'; + const lines: string[] = []; + const read = readLock(o.root, remote); + let sha = read.sha; + let current: LandLock | null = read.state === 'held' ? read.lock : null; + const own = current && ownsLock(current, o.owner, o.journal) ? current : null; + if (read.state === 'unreadable') { + if (!o.breakLock) return { failure: { code: 'QUEUE_LAND_LOCKED', message: `the land lock at ${read.sha.slice(0, 12)} cannot be read (${read.detail}); --break-lock replaces it` }, lines }; + lines.push(`LOCK: broke an unreadable lock at ${read.sha.slice(0, 12)} (${read.detail})`); + } else if (current && !own) { + const allowed = o.takeover?.(current) ?? null; + if (allowed) lines.push(allowed); + else { + const age = o.now().getTime() - Date.parse(current.phase_at ?? current.started_at); + if (!o.breakLock) return { failure: { code: 'QUEUE_LAND_LOCKED', message: `${lockLabel(current)} holds the land lock (${current.holder}, phase ${current.phase}, since ${current.started_at})` }, lines }; + lines.push(`LOCK_HOLDER: ${lockLabel(current)} holder=${current.holder} phase=${current.phase} since=${current.started_at}`); + const refusal = unbreakable(current, o, age); + if (refusal) return { failure: refusal, lines }; + lines.push(`LOCK: broke ${lockLabel(current)}'s lock (${current.holder}, phase ${current.phase})`); + } + } + const startedAt = own?.started_at ?? o.now().toISOString(); + const always = o.extra?.(own) ?? {}; + const set = (phase: LandLockPhase, extra: Partial = {}): QueueFailure | null => { + const next: LandLock = { pr: o.owner.pr, ...(o.owner.train ? { train: o.owner.train, slot: o.owner.slot } : {}), holder: o.holder, started_at: startedAt, phase, journal: o.journal, phase_at: o.now().toISOString(), ...always, ...extra }; + const w = writeLock(o.root, sha, next, remote); + if (!w.ok) { + return readLock(o.root, remote).sha === sha + ? { code: 'QUEUE_NOT_READY', message: `the land lock push to ${remote} failed and the lock there is unchanged; nothing was pushed` } + : { code: 'QUEUE_LAND_LOCKED', message: 'the land lock moved under this run (another lander took it)' }; + } + sha = w.sha; current = next; + return null; + }; + const failed = set(o.phase(own)); + if (failed) return { failure: failed, lines }; + const held: HeldLandLock = { + get lock() { return current!; }, + resumed: own, + set, + release: () => writeLock(o.root, sha, { released: true, by: o.holder, at: o.now().toISOString() }, remote).ok, + }; + return { held, lines }; +} diff --git a/lib/ship-queue-land.ts b/lib/ship-queue-land.ts index ee71af2f6..dd8d22d31 100644 --- a/lib/ship-queue-land.ts +++ b/lib/ship-queue-land.ts @@ -17,15 +17,15 @@ import * as path from 'node:path'; import { mergeChangelogs, parseChangelog } from './changelog-check'; import { fragmentSlug, readFragments } from './changelog-fragments'; import { restamp, versionAtRef } from './restamp'; +import { acquireLandLock, lockLabel, readLock, writeLock, type LandLockPhase } from './ship-queue-land-lock'; import { matchesAny, type ShipPolicy } from './ship-policy'; import { ciAtHead, loadBasePolicy, type QueueContext, type StepOutput } from './ship-queue'; import { QUEUE_DIR_REL, git, writeJsonAtomic, type QueueFailure } from './ship-queue-gate'; import { bindShippedReceipt } from './ship-receipt-depth'; -import { loadReady, rebuildFromNotes, repoIdentity } from './ship-queue-train-record'; +import { TERMINAL_STATES, freezeLockCleared, loadReady, loadTrain, rebuildFromNotes, repoIdentity } from './ship-queue-train-record'; -export const LAND_LOCK_REF = 'refs/notes/gstack-land'; +export { LAND_LOCK_REF } from './ship-queue-land-lock'; export const MAX_RESTAMPS = 2; -const PUSH_STALE_MS = 10 * 60_000; const MANIFESTS = ['package.json', 'package-lock.json', 'npm-shrinkwrap.json']; const isSha = (s: string | undefined) => /^[0-9a-f]{40,64}$/.test(s ?? ''); /** The first path segment of `rel` inside `wt` that is a symlink, or null: land writes only inside the checkout. */ @@ -49,35 +49,13 @@ export interface LandJournal { drift_main?: string; repair_pr?: string; phase: 'locked' | 'stamped' | 'pushed' | 'merged' | 'landed' | 'drift'; } -export interface LandLock { pr: string; holder: string; started_at: string; phase: 'stamping' | 'pushing' | 'waiting' | 'merging' | 'drift'; journal: string; phase_at?: string; repair_pr?: string; drift_of?: string } -export interface LandOptions { pr: string; repair?: boolean; dryRun?: boolean; noWait?: boolean; breakLock?: boolean; allowRepoCommands?: boolean; waitTimeoutMin?: number; now?: () => Date; holder?: string } +export interface LandOptions { pr: string; repair?: boolean; dryRun?: boolean; noWait?: boolean; breakLock?: boolean; breakTrainLock?: boolean; allowRepoCommands?: boolean; waitTimeoutMin?: number; now?: () => Date; holder?: string } const fail = (lines: string[], failure: QueueFailure, run: string): StepOutput => ({ ok: false, lines, failure, run }); export const landJournalRel = (pr: string) => `${QUEUE_DIR_REL}/land/${pr}.json`; function readJson(file: string): T | null { try { return JSON.parse(fs.readFileSync(file, 'utf8')) as T; } catch { return null; } } -/** The remote land lock: a ref whose commit holds lock.json, or a released marker. Updated only by exact lease. */ -function readLock(root: string): { sha: string; lock: LandLock | null } { - git(root, ['fetch', '-q', 'origin', `+${LAND_LOCK_REF}:${LAND_LOCK_REF}`]); - const sha = git(root, ['rev-parse', '--verify', '--quiet', LAND_LOCK_REF]).out; - if (!sha) return { sha: '', lock: null }; - const body = git(root, ['show', `${sha}:lock.json`]); - let lock: LandLock | { released: true } | null = null; - try { lock = body.ok ? JSON.parse(body.out) : null; } catch { lock = null; } - return { sha, lock: lock && 'pr' in lock && /^\d+$/.test(String(lock.pr)) ? lock : null }; -} - -function writeLock(root: string, expected: string, value: LandLock | { released: true; by: string; at: string }): { ok: boolean; sha: string } { - const blob = git(root, ['hash-object', '-w', '--stdin'], undefined, JSON.stringify(value, null, 2) + '\n').out; - const tree = git(root, ['mktree'], undefined, `100644 blob ${blob}\tlock.json\n`).out; - const commit = git(root, ['commit-tree', tree, ...(expected ? ['-p', expected] : []), '-m', 'released' in value ? `land lock released by ${value.by}` : `land lock #${value.pr} ${value.phase}`]).out; - if (![blob, tree, commit].every(isSha)) return { ok: false, sha: '' }; - const push = git(root, ['push', '-q', `--force-with-lease=${LAND_LOCK_REF}:${expected}`, 'origin', `${commit}:${LAND_LOCK_REF}`]); - if (push.ok) git(root, ['update-ref', LAND_LOCK_REF, commit]); - return { ok: push.ok, sha: commit }; -} - /** * Release-version conflicts only: the version file and release outputs take the base's side, the * CHANGELOG merges by section (mergeChangelogs), and a manifest or npm lockfile is re-merged with @@ -328,7 +306,7 @@ function repair(ctx: QueueContext, policy: ShipPolicy, j: LandJournal | null, fi if (!n) return fail(lines, { code: 'QUEUE_NOT_READY', message: `gh pr create failed: ${(pr.err ?? pr.out).trim().slice(0, 160)}` }, run); j.repair_pr = n; writeJsonAtomic(file, j); const held = readLock(root); - if (held.lock?.pr === j.pr && held.lock.phase === 'drift') writeLock(root, held.sha, { ...held.lock, repair_pr: n }); + if (held.state === 'held' && !held.lock.train && held.lock.pr === j.pr && held.lock.phase === 'drift') writeLock(root, held.sha, { ...held.lock, repair_pr: n }); return { ok: true, run, lines: [...lines, `REPAIR_PR: #${n} ${branch} ${pushed}`, `NEXT: run /ship on #${n}, then gstack-ship-queue land --pr ${n} (it takes over #${j.pr}'s drift lock and releases it)`] }; } finally { git(root, ['worktree', 'remove', '--force', wt]); @@ -379,28 +357,31 @@ export function land(ctx: QueueContext, o: LandOptions): StepOutput { return { ok: true, run, lines: [...lines, ...s.lines, `STAMP_VERSION: ${s.version}`, `STAMP_TREE: ${s.tree}`, 'DRY-RUN: nothing locked, pushed or merged'] }; } - let { sha: lockSha, lock } = readLock(root); - const drifted = lock && lock.pr !== o.pr && lock.phase === 'drift' ? lock.pr : null; - const repairOf = drifted ? readJson(path.join(root, landJournalRel(drifted))) : null; - const driftOf = drifted && (lock!.repair_pr === o.pr || repairOf?.repair_pr === o.pr) ? drifted : lock?.pr === o.pr ? lock.drift_of : undefined; - if (drifted && driftOf) lines.push(`LOCK: taking over #${drifted}'s drift lock for its repair`); - else if (lock && lock.pr !== o.pr) { - const age = now().getTime() - Date.parse(lock.phase_at ?? lock.started_at); - if (!o.breakLock) return fail(lines, { code: 'QUEUE_LAND_LOCKED', message: `#${lock.pr} holds the land lock (${lock.holder}, phase ${lock.phase}, since ${lock.started_at})` }, run); - if (lock.phase === 'drift' || lock.drift_of) return fail(lines, { code: 'QUEUE_LAND_DRIFT', message: `#${lock.drift_of ?? lock.pr} holds a drift lock; --break-lock never breaks it, only its repair PR's land takes it over`, fix: `gstack-ship-queue land --pr ${lock.drift_of ?? lock.pr} --repair` }, run); - if ((lock.phase === 'pushing' || lock.phase === 'merging') && age < PUSH_STALE_MS) return fail(lines, { code: 'QUEUE_LAND_LOCKED', message: `#${lock.pr}'s holder is mid-${lock.phase === 'pushing' ? 'push' : 'merge'} (${Math.round(age / 1000)} s ago); --break-lock refused` }, run); - lines.push(`LOCK: broke #${lock.pr}'s lock (${lock.holder}, phase ${lock.phase})`); - } - const setLock = (phase: LandLock['phase']): QueueFailure | null => { - const next: LandLock = { pr: o.pr, holder, started_at: lock?.pr === o.pr ? lock.started_at : now().toISOString(), phase, journal: run, phase_at: now().toISOString(), ...(driftOf ? { drift_of: driftOf } : {}) }; - const w = writeLock(root, lockSha, next); - if (!w.ok) return { code: 'QUEUE_LAND_LOCKED', message: 'the land lock moved under this run (another lander took it)' }; - lockSha = w.sha; lock = next; - return null; - }; + let driftOf: string | undefined; + const got = acquireLandLock({ + root, holder, owner: { pr: o.pr }, journal: run, now, breakLock: o.breakLock, breakTrainLock: o.breakTrainLock, + takeover: held => { + if (held.phase === 'freeze') { + const cleared = freezeLockCleared(root, ctx.remote ?? 'origin', keys, held); + return cleared ? `LOCK: taking over ${lockLabel(held)}'s freeze lock; that freeze was cleared at ${cleared}` : null; + } + if (held.train || held.phase !== 'drift' || (held.repair_pr !== o.pr && readJson(path.join(root, landJournalRel(held.pr)))?.repair_pr !== o.pr)) return null; + driftOf = held.pr; + return `LOCK: taking over #${held.pr}'s drift lock for its repair`; + }, + liveTrain: nonce => { + if (!loadTrain(nonce)) rebuildFromNotes(root, ctx.remote ?? 'origin'); + const t = loadTrain(nonce); + return !t || !TERMINAL_STATES.includes(t.state); + }, + extra: own => { driftOf ??= own?.drift_of; return driftOf ? { drift_of: driftOf } : {}; }, + phase: () => (j?.phase === 'pushed' ? 'waiting' : j?.phase === 'merged' ? 'merging' : 'stamping'), + }); + lines.push(...got.lines); + if ('failure' in got) return fail(lines, got.failure, run); + const lock = got.held; + const setLock = (phase: LandLockPhase) => lock.set(phase); const save = () => writeJsonAtomic(file, j); - const failed = setLock(j?.phase === 'pushed' ? 'waiting' : j?.phase === 'merged' ? 'merging' : 'stamping'); - if (failed) return fail(lines, failed, run); if (!j) { j = { schema_version: 1, pr: o.pr, branch: info.headRefName, reviewed_pin: pin, last_pushed_head: recovered ? info.headRefOid : null, stamp_commit: null, stamp_tree: null, base_sha: null, version: null, date: null, restamps: 0, phase: 'locked' }; save(); } for (;;) { @@ -409,7 +390,7 @@ export function land(ctx: QueueContext, o: LandOptions): StepOutput { if (j.phase === 'pushed' && j.base_sha !== baseNow) { if (j.restamps >= MAX_RESTAMPS) { j.restamps = 0; save(); - if (!driftOf) writeLock(root, lockSha, { released: true, by: holder, at: now().toISOString() }); + if (!driftOf) lock.release(); return fail(lines, { code: 'QUEUE_LAND_BASE_BUSY', message: `origin/${ctx.base} moved during CI ${MAX_RESTAMPS + 1} times; the lock is released and a later run restamps from the pin` }, run); } lines.push(`BASE_MOVED: ${j.base_sha?.slice(0, 12)} -> ${baseNow.slice(0, 12)}; restamping`); @@ -419,7 +400,7 @@ export function land(ctx: QueueContext, o: LandOptions): StepOutput { const s = buildStamp(ctx, policy, { pin, baseSha: baseNow, pr: o.pr, title: info.title, allowRepoCommands: !!o.allowRepoCommands, now: j.date ? new Date(`${j.date}T12:00:00Z`) : now() }); if ('code' in s) { if (driftOf) setLock('stamping'); - else writeLock(root, lockSha, { released: true, by: holder, at: now().toISOString() }); + else lock.release(); return fail([...lines], s, run); } lines.push(...s.lines); @@ -471,8 +452,8 @@ export function land(ctx: QueueContext, o: LandOptions): StepOutput { j.phase = 'landed'; save(); const driftJournal = driftOf ? readJson(path.join(root, landJournalRel(driftOf))) : null; if (driftOf && driftJournal) writeJsonAtomic(path.join(root, landJournalRel(driftOf)), { ...driftJournal, phase: 'landed' }); - const rel = writeLock(root, lockSha, { released: true, by: holder, at: now().toISOString() }); - return { ok: true, run, lines: [...lines, `MERGED: #${o.pr} as ${main} (${j.version})`, `LANDED: origin/${ctx.base} tree ${tree} equals the stamped tree`, `LOCK: ${rel.ok ? 'released' : 'release push failed (the next land takes it over)'}`] }; + const released = lock.release(); + return { ok: true, run, lines: [...lines, `MERGED: #${o.pr} as ${main} (${j.version})`, `LANDED: origin/${ctx.base} tree ${tree} equals the stamped tree`, `LOCK: ${released ? 'released' : 'release push failed (the next land takes it over)'}`] }; } if (j.phase === 'drift') return fail(lines, { code: 'QUEUE_LAND_DRIFT', message: 'a previous land of this PR found drift', fix: `gstack-ship-queue land --pr ${o.pr} --repair` }, run); } diff --git a/lib/ship-queue-train-cli.ts b/lib/ship-queue-train-cli.ts index a4ce4e5a6..dcbd2431d 100644 --- a/lib/ship-queue-train-cli.ts +++ b/lib/ship-queue-train-cli.ts @@ -36,7 +36,7 @@ export const TRAIN_HELP = ` train build --prs @[,...] [--order ,...] train status [--train ] never spends: rebuild records from refs/notes/gstack-trains, reconcile unconfirmed dispatches by nonce train verdict --train read the gate run and its record: GREEN, RED, INCOMPLETE or PENDING - train land --train --pr [--no-wait] [--timeout ] + train land --train --pr [--no-wait] [--timeout ] [--break-lock [--break-train-lock]] publish slot n's prefix (stamped S_i, else gated T_i): update mode commits its tree on [pin, main], pushes under the pin's lease, waits for checks and merges; direct mode (trains with gates, no strict protection, a green pin) runs gh pr merge @@ -156,7 +156,7 @@ export async function trainMain(ctx: QueueContext, args: string[]): Promise'); const timeout = positive(flag(rest, '--timeout'), 60); if (Number.isNaN(timeout)) return usage('--timeout needs minutes'); - return guarded('train land', async () => (await import('./ship-queue-train-land')).trainLand(ctx, { train: nonce, pr, noWait: rest.includes('--no-wait'), timeoutMin: timeout })); + return guarded('train land', async () => (await import('./ship-queue-train-land')).trainLand(ctx, { train: nonce, pr, noWait: rest.includes('--no-wait'), timeoutMin: timeout, breakLock: rest.includes('--break-lock'), breakTrainLock: rest.includes('--break-train-lock') })); } if (sub === 'rebuild') return guarded('train rebuild', () => trainRebuild(ctx, { train: nonce, infra: rest.includes('--infra') })); if (sub === 'bisect') { diff --git a/lib/ship-queue-train-gate.ts b/lib/ship-queue-train-gate.ts index 6b7e3056d..a7867cb1d 100644 --- a/lib/ship-queue-train-gate.ts +++ b/lib/ship-queue-train-gate.ts @@ -11,7 +11,7 @@ * a second red is real; both attempts' jobs are kept. * - `command`: a clean detached checkout of the prefix head runs on a Ubicloud VM through * scripts/ubicloud/ubi-runner.sh, forwarding only `pass_env`; the VM's `git write-tree` must equal the prefix - * tree and exit 0 is GREEN. One spend cap covers every run of the train. + * tree and exit 0 is GREEN. One spend cap per command gate covers that gate's runs across every prefix of the train. * * The verdict is per prefix; the train is GREEN when every prefix is, and otherwise the first red prefix after a * green one names the culprit (classifyPrefixes). Slot 1's READY may declare FIXES_BASE_RED: T0 is then allowed @@ -366,7 +366,7 @@ async function settleCommand(env: TrainEnv, t: TrainRecord, a: AttemptRecord, g: lines.push(`COMMAND: gate ${a.gate} on T${p.index} ${r.outcome} (${r.detail})`); } -/** Run every command unit that has no finished attempt, all prefixes in parallel, under the train's one spend cap. */ +/** Run every command unit that has no finished attempt, all prefixes in parallel, under one spend cap per command gate across every prefix of the train. */ async function runCommands(env: TrainEnv, t: TrainRecord, units: GateUnit[], lines: string[]): Promise { const commands = units.filter(u => u.workflow === 'command'); if (!commands.length) return null; diff --git a/lib/ship-queue-train-land.ts b/lib/ship-queue-train-land.ts index 0ed0c86a4..1a7d42c85 100644 --- a/lib/ship-queue-train-land.ts +++ b/lib/ship-queue-train-land.ts @@ -17,16 +17,24 @@ * Either way main's tree must be the prefix after the merge; any other main is the freeze path * (QUEUE_TRAIN_TREE_MISMATCH), as for verify. Fork heads are refused. Journaled beside the train record; a rerun * resumes: open at the pin, open at the update commit, or merged (confirmed through main's tree). + * + * Train land takes the shared land lock (lib/ship-queue-land-lock.ts, #3599) as `train slot (#pr)` + * before its first main read, so no standalone land publishes beside a slot. After a slot merges with landable + * slots left, the lock stays in a train-owned `between` phase that the same train's next slot takes over; a stop + * that freezes the queue leaves it in a `freeze` phase only the admitted repair train takes over. The release is + * decided once, at the trainLand boundary (settleLock). */ import * as fs from 'node:fs'; +import * as os from 'node:os'; import * as path from 'node:path'; import { ciAtHead, type QueueContext } from './ship-queue'; -import { git, writeJsonAtomic } from './ship-queue-gate'; +import { git, writeJsonAtomic, type QueueFailure } from './ship-queue-gate'; +import { acquireLandLock, lockLabel, type HeldLandLock, type LandLock, type LandLockPhase } from './ship-queue-land-lock'; import { waitChecks } from './ship-queue-land'; import { verifyPostMerge } from './ship-queue-post-merge'; import { bindTrainReceipt } from './ship-queue-train-gate'; import { checkRun, prInfo, type PrInfo } from './ship-queue-train-github'; -import { activeFreeze, attemptsOf, nowIso, trainFile, type LandingRecord, type TrainRecord } from './ship-queue-train-record'; +import { TERMINAL_STATES, activeFreeze, attemptsOf, freezeLockCleared, nowIso, trainFile, type LandingRecord, type TrainRecord } from './ship-queue-train-record'; import { fetchBase, fetchTrainBranch, findTrain, frozenLines, nextFor, outFailed, outPending, outRefused, save, trainEnv, type TrainEnv, type TrainOut } from './ship-queue-train'; import { confirmMerged, stopped } from './ship-queue-train-verify'; @@ -36,7 +44,10 @@ const STAMP_IDENTITY = { GIT_AUTHOR_NAME: 'gstack-ship-queue', GIT_AUTHOR_EMAIL: const isSha = (s: string) => /^[0-9a-f]{40,64}$/.test(s); interface SlotJournal { schema_version: 1; pr: string; pin: string; stamp_commit: string; base: string } interface Link { index: number; commit: string; tree: string; pin: string } -interface Slot { env: TrainEnv; t: TrainRecord; i: number; pin: string; target: Link & { version?: string }; pred: Link; run: string; rerun: string; lines: string[] } +interface Slot { env: TrainEnv; t: TrainRecord; i: number; pin: string; target: Link & { version?: string }; pred: Link; run: string; rerun: string; lines: string[]; lock: { held: HeldLandLock | null; effect: boolean } } + +/** Move this slot's land lock to a phase; a failure means another lander took it, so nothing remote may follow. */ +const lockPhase = (s: Slot, phase: LandLockPhase): QueueFailure | null => s.lock.held?.set(phase) ?? null; /** The checks train land requires on the landing commit: changelog_fragments.required_checks, else train.land.required_checks. */ export const landRequiredChecks = (env: TrainEnv): string[] => env.policy.policy.changelog_fragments?.required_checks ?? env.train.land?.required_checks ?? []; @@ -131,6 +142,9 @@ async function mergeAt(ctx: QueueContext, s: Slot, head: string, base: string): const target = prInfo(ctx, pr); if ('error' in target) return outPending({ code: 'QUEUE_PROOF_MISSING', message: target.error }, s.lines, s.rerun, s.run); if (target.baseRefName && target.baseRefName !== ctx.base) return retargeted(s, target.baseRefName, ctx.base); + const moved = lockPhase(s, 'merging'); + if (moved) return outFailed(moved, s.lines, s.rerun, s.run); + s.lock.effect = true; const m = ctx.gh(['pr', 'merge', pr, `--${method}`, '--match-head-commit', head]); if (!m.ok) return outFailed({ code: 'QUEUE_NOT_READY', message: `gh pr merge refused: ${(m.err ?? m.out).trim().slice(0, 200)}` }, s.lines, s.rerun, s.run); main = mainNow(ctx, s.env); @@ -189,6 +203,8 @@ async function landUpdate(ctx: QueueContext, s: Slot, info: PrInfo, base: string writeJsonAtomic(file, j); recordLanding(s, 'update', commit, base); const pushed = info.headRefOid !== commit; + const pushing = lockPhase(s, 'pushing'); + if (pushing) return outFailed(pushing, s.lines, s.rerun, s.run); if (pushed) { const push = git(root, ['push', '-q', `--force-with-lease=refs/heads/${info.headRefName}:${info.headRefOid}`, env.remote, `${commit}:refs/heads/${info.headRefName}`]); if (!push.ok) { @@ -198,6 +214,9 @@ async function landUpdate(ctx: QueueContext, s: Slot, info: PrInfo, base: string if (!remote.ok || now !== commit) return outFailed({ code: 'QUEUE_NOT_READY', message: `the ${what} push failed and ${info.headRefName} still reads ${remote.ok ? info.headRefOid.slice(0, 12) : 'unknown'}: ${push.err.split('\n')[0]}` }, s.lines, s.rerun, s.run); } } + s.lock.effect = true; + const waiting = lockPhase(s, 'waiting'); + if (waiting) return outFailed(waiting, s.lines, s.rerun, s.run); s.lines.push(`${pushed ? 'PUSHED' : 'RESUMED'}: ${commit} (tree ${label(s)}, parents [pin, main ${base.slice(0, 12)}]) ${pushed ? 'to' : 'is already the head of'} ${info.headRefName}`); if (o.noWait && pushed) return outPending({ code: 'QUEUE_NOT_READY', message: `checks run on the ${what} commit ${commit.slice(0, 12)}` }, s.lines, `gstack-pr-wait ${pr} --until green && ${s.rerun}`, s.run); if (!o.noWait) waitChecks(ctx, commit, o.timeoutMin ?? 60); @@ -220,7 +239,43 @@ async function landUpdate(ctx: QueueContext, s: Slot, info: PrInfo, base: string const retargeted = (s: Slot, b: string, base: string) => outRefused({ code: 'QUEUE_NOT_READY', message: `#${s.t.slots[s.i]!.pr} targets ${b}, not ${base}; train land merges only PRs on the base the train built` }, s.lines, `retarget #${s.t.slots[s.i]!.pr} to ${base}, then ${s.rerun}`, s.run); -export async function trainLand(ctx: QueueContext, o: { train: string; pr: string; noWait?: boolean; timeoutMin?: number }): Promise { +interface TrainLandOptions { train: string; pr: string; noWait?: boolean; timeoutMin?: number; breakLock?: boolean; breakTrainLock?: boolean } + +/** + * The takeovers train land allows: its own train's between-slots lock, the freeze lock when the freeze admits this + * train as its repair (or, with the notes ref fetched, a freeze record cleared after the lock was written: absence + * is never proof), and a lock whose train record is terminal. + */ +function trainTakeover(env: TrainEnv, t: TrainRecord, i: number, lock: LandLock): string | null { + if (lock.train === t.nonce && lock.phase === 'between') return `LOCK: train ${t.nonce} slot ${i + 1} takes over its train's between-slots lock (from slot ${lock.slot})`; + if (lock.phase === 'freeze') { + const cleared = freezeLockCleared(env.ctx.repoRoot, env.remote, env.repo.keys, lock); + if (activeFreeze(env.repo.keys)?.repair?.train === t.nonce) return `LOCK: repair train ${t.nonce} takes over the freeze lock of ${lockLabel(lock)}`; + return cleared ? `LOCK: train ${t.nonce} takes over ${lockLabel(lock)}'s freeze lock; that freeze was cleared at ${cleared}` : null; + } + if (!lock.train || lock.train === t.nonce) return null; + const other = findTrain(env, lock.train); + return other && TERMINAL_STATES.includes(other.state) ? `LOCK: train ${t.nonce} takes over ${lockLabel(lock)}'s lock; that train is ${other.state}` : null; +} + +/** + * The one release point: a freeze keeps the lock in `freeze`; once a slot merged, a train still merging keeps it + * `between` (a merged slot, a pending wait or a refusal before this slot's push); a failure after a remote effect + * keeps its phase for the resume; anything else releases it. + */ +function settleLock(s: Slot, out: TrainOut): TrainOut { + const held = s.lock.held!; + const midway = s.t.state === 'merging' && s.t.merged.length > 0; + const keep = (phase: LandLockPhase, line: string) => { const failed = held.set(phase); return failed ? `LOCK: ${phase} could not be written (${failed.message})` : line; }; + let line: string; + if (activeFreeze(s.env.repo.keys)) line = keep('freeze', 'LOCK: freeze (kept for the repair train the freeze admits; --break-lock never breaks it)'); + else if (midway && (out.exit === 0 || !s.lock.effect)) line = keep('between', `LOCK: between (train ${s.t.nonce} keeps the land lock for its next slot)`); + else if (out.exit !== 0 && s.lock.effect) line = `LOCK: kept in phase ${held.lock.phase}; ${s.rerun} resumes it`; + else line = held.release() ? 'LOCK: released' : 'LOCK: release push failed (the next lander takes it over)'; + return { ...out, lines: [...out.lines, line] }; +} + +export async function trainLand(ctx: QueueContext, o: TrainLandOptions): Promise { const fetchErr = fetchBase(ctx); const env = trainEnv(ctx); if ('code' in env) return outRefused(env, [], 'none', 'none'); @@ -241,8 +296,24 @@ export async function trainLand(ctx: QueueContext, o: { train: string; pr: strin if (!chain) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `the record's T0 and ${t.stamped ? 'S' : 'T'}1..${t.stamped ? 'S' : 'T'}${k} (with each slot's pin as second parent) are not the first-parent chain of the gated head ${t.head.slice(0, 12)}` }, [], `${Q} train status --train ${t.nonce}`, run); const pin = t.slots[i]!.head; const rerun = `${Q} train land --train ${t.nonce} --pr ${o.pr}`; - const s: Slot = { env, t, i, pin, target: { ...chain[i + 1]!, version: t.stamped?.[i]?.version }, pred: chain[i]!, run, rerun, lines: [] }; + const s: Slot = { env, t, i, pin, target: { ...chain[i + 1]!, version: t.stamped?.[i]?.version }, pred: chain[i]!, run, rerun, lines: [], lock: { held: null, effect: false } }; s.lines.push(`TRAIN_LAND: ${t.nonce} slot ${i + 1}/${k} #${o.pr} pin ${pin} -> ${label(s)}${s.target.version ? ` v${s.target.version}` : ''} tree ${s.target.tree}`); + const got = acquireLandLock({ + root: ctx.repoRoot, remote: env.remote, holder: `${os.hostname()}:${process.pid}`, owner: { train: t.nonce, slot: i + 1, pr: o.pr }, journal: `train/${t.nonce}/slot-${i + 1}`, + now: () => new Date(), breakLock: o.breakLock, breakTrainLock: o.breakTrainLock, + phase: own => (own && ['pushing', 'waiting', 'merging'].includes(own.phase) ? own.phase : 'stamping'), + takeover: lock => trainTakeover(env, t, i, lock), + liveTrain: nonce => { const other = findTrain(env, nonce); return !other || !TERMINAL_STATES.includes(other.state); }, + }); + s.lines.push(...got.lines); + if ('failure' in got) return outRefused(got.failure, s.lines, `wait for the holder named above, then ${rerun}`, run); + s.lock = { held: got.held, effect: !!got.held.resumed && ['pushing', 'waiting', 'merging'].includes(got.held.resumed.phase) }; + return settleLock(s, await landSlot(ctx, s, chain, o)); +} + +/** Train land under the lock: main read, confirmation, post-merge checks of the previous slot, then direct or update. */ +async function landSlot(ctx: QueueContext, s: Slot, chain: Link[], o: TrainLandOptions): Promise { + const { env, t, i, pin, run, rerun } = s; const main = mainNow(ctx, env); if ('error' in main) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: main.error }, s.lines, rerun, run); if (main.tree === s.target.tree) return confirmed(s, main.sha, `main: ${main.sha} is ${label(s)} exactly`); diff --git a/lib/ship-queue-train-record.ts b/lib/ship-queue-train-record.ts index a35811f99..5d8f7df97 100644 --- a/lib/ship-queue-train-record.ts +++ b/lib/ship-queue-train-record.ts @@ -187,6 +187,15 @@ const firstOf = (k: RepoKeys, read: (key: string) => T | null): T | null => { export const loadFreeze = (repoKeys: RepoKeys) => firstOf(repoKeys, key => readJson(freezePath(key), 'freeze')); export const loadReady = (repoKeys: RepoKeys, pr: string, head: string) => firstOf(repoKeys, key => readJson(readyPath(key, pr, head), 'ready')); export const activeFreeze = (repoKeys: RepoKeys): FreezeRecord | null => firstOf(repoKeys, key => { const f = readJson(freezePath(key), 'freeze'); return f && !f.cleared_at ? f : null; }); +/** + * When a freeze-phase land lock's freeze is over, the cleared_at that proves it: the notes ref fetched, no freeze + * active, and the last freeze cleared at or after the lock was written. Absence of a record is never proof. + */ +export function freezeLockCleared(repoRoot: string, remote: string, repoKeys: RepoKeys, lock: { phase_at?: string; started_at: string }): string | null { + if (!rebuildFromNotes(repoRoot, remote).fetched || activeFreeze(repoKeys)) return null; + const last = loadFreeze(repoKeys); + return last?.cleared_at && Date.parse(last.cleared_at) >= Date.parse(lock.phase_at ?? lock.started_at) ? last.cleared_at : null; +} export const loadWave = (nonce: string) => (/^[A-Za-z0-9._-]{1,128}$/.test(nonce) ? readJson(wavePath(nonce), 'wave') : null); /** Every wave of this repository, newest first. */ @@ -325,13 +334,13 @@ export function forgetNotes(repoRoot: string, names: string[], remote = 'origin' } /** Rebuild local records from the notes ref: a note whose record is missing locally, or newer than the local copy, is written. */ -export function rebuildFromNotes(repoRoot: string, remote = 'origin'): { restored: number; read: number; detail: string } { +export function rebuildFromNotes(repoRoot: string, remote = 'origin'): { restored: number; read: number; detail: string; fetched: boolean } { const f = fetchNotes(repoRoot, remote); const list = git(repoRoot, ['notes', `--ref=${NOTES_REF}`, 'list']); - if (!list.ok || !list.out.trim()) return { restored: 0, read: 0, detail: f.detail }; + if (!list.ok || !list.out.trim()) return { restored: 0, read: 0, detail: f.detail, fetched: f.ok }; const records = list.out.split('\n').map(l => l.split(' ')[0]!).filter(Boolean) .map(blob => parseNote(git(repoRoot, ['cat-file', 'blob', blob]).out)).filter((r): r is AnyRecord => !!r); const fresh = records.filter(newerOrMissing); for (const r of fresh) writeJsonAtomic(recordPath(r), r); - return { restored: fresh.length, read: records.length, detail: f.detail }; + return { restored: fresh.length, read: records.length, detail: f.detail, fetched: f.ok }; } diff --git a/lib/ship-queue-train-verify.ts b/lib/ship-queue-train-verify.ts index a67bc7d22..5c7981e00 100644 --- a/lib/ship-queue-train-verify.ts +++ b/lib/ship-queue-train-verify.ts @@ -6,8 +6,9 @@ * head, or at a head whose tree is the slot's prefix after it modulo stamps * (from slot 2 on, the lane merges main into its pin so GitHub can merge it), * with main's tree equal to the prefix before the PR (pre-merge: - * MERGE_CONDITION) or after it (post-merge confirmation), modulo stamps - * (lib/tree-receipt.ts). When a train stops partway (a mismatch or a moved + * MERGE_CONDITION, once the head carries its own stamp above main's version, + * #3531) or after it (post-merge confirmation, once GitHub reports the PR + * MERGED), modulo stamps (lib/tree-receipt.ts). When a train stops partway (a mismatch or a moved * head after earlier slots merged) the queue freezes: the freeze is written * locally and to the notes ref first, then the backstop gate is dispatched on * main's published prefix at once. One repair train is admitted under the @@ -25,6 +26,9 @@ import { } from './ship-queue-train'; import { mergeChangelogs, ownEntryAboveMain, parseChangelog } from './changelog-check'; import { buildReceipt } from './tree-receipt'; +import { versionAtRef } from './restamp'; +import { cmpVersion, parseVersion } from './version-source'; +import { readLock, writeLock } from './ship-queue-land-lock'; import { genericBackstop } from './ship-queue-train-gate'; const Q = 'gstack-ship-queue'; @@ -110,6 +114,31 @@ function published(env: TrainEnv, t: TrainRecord, i: number, main: string): { sa return { same: true, detail: `same modulo stamps (${r.files.filter(f => f.verdict === 'stamp').length} stamp file(s)) except ${cl}, main ${parent.slice(0, 12)}'s ${cl} plus #${t.slots[i]!.pr}'s own entry` }; } +/** + * The slot stamp a hand-stamped head must carry before it may merge (#3531, train 33 shipped #3527 under #3526's + * version): its VERSION above main's and, with a CHANGELOG, its top section headed with that version directly above + * main's top release, with no [Unreleased] section when main has none. The failing rule is named. + */ +function stampProof(env: TrainEnv, head: string, main: string): { version: string } | { rule: string; detail: string } { + const root = env.ctx.repoRoot; + const hv = versionAtRef(root, head); + const mv = versionAtRef(root, main); + if ('error' in hv) return { rule: 'version', detail: `the head's version cannot be read (${hv.error})` }; + if ('error' in mv) return { rule: 'version', detail: `main's version cannot be read (${mv.error})` }; + const [h, m] = [parseVersion(hv.version), parseVersion(mv.version)]; + if (!h || !m || cmpVersion(h, m) <= 0) return { rule: 'version', detail: `the head's ${hv.rel} ${hv.version} is not above main's ${mv.version}` }; + const cl = env.policy.policy.changelog; + if (!cl) return { version: hv.version }; + const sections = (rev: string) => parseChangelog(git(root, ['show', `${rev}:${cl}`]).out).sections; + const [ours, theirs] = [sections(head), sections(main)]; + const mainTop = theirs.find(s => !s.unreleased)?.version ?? null; + if (ours[0]?.version !== hv.version) return { rule: 'changelog', detail: `${cl}'s top section at the head is [${ours[0] ? ours[0].version ?? 'Unreleased' : 'none'}], not [${hv.version}]` }; + const below = ours.slice(1).find(s => !s.unreleased)?.version ?? null; + if (below !== mainTop) return { rule: 'changelog', detail: `${cl}'s release below [${hv.version}] at the head is [${below ?? 'none'}], not main's top release [${mainTop ?? 'none'}]` }; + if (!theirs.some(s => s.unreleased) && ours.some(s => s.unreleased)) return { rule: 'changelog', detail: `${cl} at the head keeps an [Unreleased] section that main does not have` }; + return { version: hv.version }; +} + /** * Freeze the queue: the record goes to local state and the notes ref before * anything remote, then the backstop gate is dispatched on main's sha. @@ -225,8 +254,13 @@ export async function verifyTrain(ctx: QueueContext, o: { pr: string; train: str const before = t.prefixes[i]!; const post = published(env, t, i, main); const lines = [`TRAIN: ${t.nonce} slot ${i + 1}/${t.slots.length} #${o.pr} pinned ${t.slots[i]!.head}`]; - if (post.same) return confirmMerged(env, t, o.pr, main, after.index, [...lines, `main: ${main} = T${after.index} (${post.detail})`]); const pre = sameModuloStamps(env, before.commit, main); + if (post.same) { + const state = prState(ctx, o.pr); + if (state === 'MERGED') return confirmMerged(env, t, o.pr, main, after.index, [...lines, `main: ${main} = T${after.index} (${post.detail})`]); + if (state !== 'OPEN' || !pre.same) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `main ${main.slice(0, 12)} is T${after.index} modulo stamps (${post.detail}), but GitHub reports #${o.pr} as ${state}, not MERGED; nothing is confirmed and the queue is not frozen` }, lines, `gh pr view ${o.pr} --json state,mergeCommit; rerun once GitHub reports #${o.pr} MERGED: ${Q} verify --pr ${o.pr} --train ${t.nonce}`, run); + lines.push(`main: also T${after.index} modulo stamps, but #${o.pr} is OPEN, so this is the pre-merge check (a stamp-only PR)`); + } if (!pre.same) { return stopped(env, t, 'QUEUE_TRAIN_TREE_MISMATCH', `main ${main.slice(0, 12)} is neither T${before.index} nor T${after.index} modulo stamps (vs T${after.index}: ${post.detail})`, main, lines, !o.noFetch); } @@ -239,7 +273,10 @@ export async function verifyTrain(ctx: QueueContext, o: { pr: string; train: str if ('error' in info) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: info.error }, lines, `${Q} verify --pr ${o.pr} --train ${t.nonce}`, run); const shipped = bindReceipt(info.body, o.pr, [t.slots[i]!.head, head]); if (!shipped.ok) return outRefused({ code: shipped.code, message: shipped.message }, [...lines, shipped.line], `#${o.pr}'s lane runs /ship and READYs again; then ${Q} train build --prs `, run); - return outOk([...lines, `main: ${main} = T${before.index} (${pre.detail})`, `head: ${head} (${stampOnly.detail})`, `${shipped.line} (binds ${shipped.head === t.slots[i]!.head ? 'the pinned head' : 'the current head'})`, + if (!headRef) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `#${o.pr}'s current head was not read (${o.noFetch ? '--no-fetch' : 'the head fetch failed'}), and the pin it falls back to is unstamped by design, so the stamp cannot be proved` }, lines, `${Q} verify --pr ${o.pr} --train ${t.nonce}`, run); + const stamp = stampProof(env, head, main); + if ('rule' in stamp) return outRefused({ code: 'QUEUE_TRAIN_UNSTAMPED', message: `#${o.pr}'s head ${head.slice(0, 12)} is not stamped above main: ${stamp.rule}: ${stamp.detail}` }, lines, `#${o.pr}'s lane runs gstack-restamp --next on its head, commits what restamp staged, pushes, then ${Q} verify --pr ${o.pr} --train ${t.nonce}`, run); + return outOk([...lines, `main: ${main} = T${before.index} (${pre.detail})`, `head: ${head} (${stampOnly.detail})`, `${shipped.line} (binds ${shipped.head === t.slots[i]!.head ? 'the pinned head' : 'the current head'})`, `STAMP_VERSION: ${stamp.version}`, `VERIFY_TRAIN: #${o.pr} may merge as slot ${i + 1} of train ${t.nonce} (green run ${gate.run?.id ?? '-'}, ${env.train.full_gate_check} success)`, `MERGE_CONDITION: sha=${head}`], `merge #${o.pr} with expected head ${head.slice(0, 12)} (gstack-restamp stamps at merge), then ${Q} verify --pr ${o.pr} --train ${t.nonce}`, run); } @@ -262,6 +299,12 @@ export function confirmMerged(env: TrainEnv, t: TrainRecord, pr: string, main: s freeze.updated_at = freeze.cleared_at; persist(env.ctx.repoRoot, [freeze], { remote: env.remote }); lines.push(`FREEZE: cleared (${freeze.cleared_by})`); + const held = readLock(env.ctx.repoRoot, env.remote); + if (held.state === 'held' && held.lock.phase === 'freeze') { + lines.push(writeLock(env.ctx.repoRoot, held.sha, { released: true, by: `verify #${pr} via train ${t.nonce}`, at: nowIso() }, env.remote).ok + ? 'LOCK: released the freeze lock' + : 'LOCK: the freeze lock could not be released; the next land or train land takes it over once it reads this cleared freeze'); + } const rebuild = freeze.unmerged.length ? `${Q} train build --prs ${pinList(freeze.unmerged)}` : ''; const unread = freeze.unreadable ?? []; const check = unread.length ? `${unread.map(u => `gh pr view ${u.pr} --json state,headRefOid`).join(' && ')} (unreadable at the freeze: ${unread.map(u => `#${u.pr}@${u.head.slice(0, 12)}`).join(', ')}); seat each that is OPEN in a train` : ''; diff --git a/package.json b/package.json index 5ef1cf28b..8a0708e4e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.92.53", + "version": "1.92.54", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/lib/windows-curation.ts b/scripts/lib/windows-curation.ts index 1f3aa5474..99b720fe7 100644 --- a/scripts/lib/windows-curation.ts +++ b/scripts/lib/windows-curation.ts @@ -315,6 +315,10 @@ export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }> file: 'test/ship-queue-train-land.test.ts', reason: 'drives train land against test/helpers/ship-queue-train-fixture.ts plus an extensionless node gh wrapper on a colon-joined PATH; on windows-latest spawnSync(\'gh\') resolves the runner\'s gh.exe instead, as for test/ship-queue-train.test.ts and test/ship-queue-land.test.ts (windows-free-shard 6, run 40824965988)', }, + { + file: 'test/ship-queue-land-lock.test.ts', + reason: 'drives land and train land against test/helpers/ship-queue-train-fixture.ts plus an extensionless node gh wrapper on a colon-joined PATH; on windows-latest spawnSync(\'gh\') resolves the runner\'s gh.exe instead, as for test/ship-queue-train-land.test.ts and test/ship-queue-land.test.ts (windows-free-shard 6, run 40824965988)', + }, { file: 'test/ship-queue-train.test.ts', reason: 'drives bin/gstack-ship-queue against the stateful #!/bin/sh gh shim in test/helpers/ship-queue-train-fixture.ts; on windows-latest spawnSync(\'gh\') resolves the runner\'s gh.exe instead of the shim (windows-free-shard run 39152702056)', diff --git a/test/fixtures/module-size-ratchet.json b/test/fixtures/module-size-ratchet.json index ad2cf1081..74e49bbc7 100644 --- a/test/fixtures/module-size-ratchet.json +++ b/test/fixtures/module-size-ratchet.json @@ -174,6 +174,7 @@ { "file": "lib/ship-queue-train-verify.ts", "movedFrom": "(new: verify --train, the queue freeze and its backstop, plan GSTA-49 T1)" }, { "file": "lib/ship-queue-train-ready.ts", "movedFrom": "(new: ready --screen|--no-screen and the READY receipt, plan GSTA-49 T1)" }, { "file": "lib/ship-queue-train-cli.ts", "movedFrom": "(new: merge-train argument parsing and output, plan GSTA-49 T1)" }, + { "file": "lib/ship-queue-land-lock.ts", "movedFrom": "lib/ship-queue-land.ts (the land lock, shared with train land, #3599)" }, { "file": "lib/gate-cost.ts", "movedFrom": "(new: paid gate cost by model and token class, per-case costs, selection replays and the merge-train model, GSTA-49 T0)" }, { "file": "scripts/gate-cost-replay.ts", "movedFrom": "(new: CLI behind lib/gate-cost.ts: cached report artifacts, merged-PR replay and the train model table, GSTA-49 T0)" }, { "file": "lib/gate-cost-history.ts", "movedFrom": "(new: red attribution, red-head screen replay, READY times and the READY-arrival train replay, GSTA-49 T0b)" }, diff --git a/test/gstack-restamp-queue.test.ts b/test/gstack-restamp-queue.test.ts index 253ef75b8..3f599fe7b 100644 --- a/test/gstack-restamp-queue.test.ts +++ b/test/gstack-restamp-queue.test.ts @@ -8,6 +8,7 @@ * version-source shapes (package-manifest pin, monorepo pin, absent source). */ import { afterEach, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; import * as fs from 'node:fs'; import * as path from 'node:path'; import { EXIT } from '../lib/headless-artifacts'; @@ -163,7 +164,7 @@ describe('three PRs, reordered twice, zero manual edits', () => { expect(changed.status).toBe(EXIT.fail); expect(changed.out).toMatch(/^TREE: changed \(pr-b\.txt\)$/m); expect(changed.out).toMatch(/^gate-reuse: not-eligible \(tree changed: pr-b\.txt\) — fix: .* \(GATE_REUSE_NOT_ELIGIBLE\)$/m); - }); + }, 30_000); test('--dry-run prints the full diff and writes nothing; --was overrides the mention source', () => { const repo = fixture(); @@ -191,7 +192,7 @@ describe('three PRs, reordered twice, zero manual edits', () => { const stamped = restamp(repo, ['--next']); expect(stamped.status).toBe(EXIT.ok); expect(stamped.out).toMatch(/^next: NEW_VERSION=1\.2\.4\.0; record the release decision, then write the CHANGELOG entry text only/m); - }); + }, 30_000); // Value: protects=a bump never repo's --next never invents a MICRO stamp: without --bump or --version it refuses and writes nothing, and an explicit level stamps as before; fails_when=restamp falls back to micro for a repo whose policy names no level; // why_new=GST-380 added bump never; seam=gstack-ship-policy, gstack-version-bump write; no_claim=the claim queue read @@ -454,7 +455,7 @@ describe('gate-ahead (--after), predecessor movement and queue staleness', () => const viaGh = runBin(repo, 'gstack-tree-receipt', ['--from', path.join(repo.work, '.gstack/tmp/gate-ahead.json'), '--predecessor-pr', '1', '--bundle', bundle, '--base', 'main', '--json'], ghShim(repo, { '1': { headRefOid: repo.git('rev-parse', 'pr-a') } })); expect(JSON.parse(viaGh.out).predecessor.moved).toBe(true); expect(JSON.parse(viaGh.out).code).toBe('RESTAMP_PREDECESSOR_MOVED'); - }); + }, 30_000); test('a gate-ahead merge that conflicts outside the stamp files is RESTAMP_MERGE_CONFLICT with nothing written', () => { const repo = fixture(); @@ -587,3 +588,97 @@ describe('version-source shapes', () => { expect(fs.existsSync(path.join(repo.work, 'VERSION'))).toBe(false); }); }); + +describe('gstack-restamp stages what it writes (#3531)', () => { + /** Branch b forks from main; branch a stamps and merges first, so b's base merge conflicts only on the CHANGELOG. */ + function behindMain(): FixtureRepo { + const repo = fixture(); + makePr(repo, 'pr-b'); + makePr(repo, 'pr-a'); + expect(restamp(repo, ['--next']).status).toBe(EXIT.ok); + repo.git('add', '-A'); + repo.git('commit', '-q', '-m', 'stamp pr-a'); + mergeToMain(repo, 'pr-a'); + repo.git('fetch', '-q', 'origin'); + repo.git('checkout', '-q', 'pr-b'); + return repo; + } + const cached = (repo: FixtureRepo) => repo.git('diff', '--cached', '--name-only').split('\n').filter(Boolean); + const unstaged = (repo: FixtureRepo) => repo.git('diff', '--name-only').split('\n').filter(Boolean); + const filesOf = (out: string) => (/^FILES: (.*)$/m.exec(out)?.[1] ?? '').split(' ').filter(Boolean); + + test('after --next on a staged no-commit base merge, every FILES entry is staged, STAGED: names them, and a bare commit carries the stamp', () => { + const repo = behindMain(); + const merge = spawnSync('git', ['merge', '--no-ff', '--no-commit', '-q', 'origin/main'], { cwd: repo.work, encoding: 'utf8', timeout: 30_000, env: { ...process.env, ...repo.env } }); + if (merge.status !== 0) { + fs.writeFileSync(path.join(repo.work, 'CHANGELOG.md'), mergeChangelogs(repo.git('show', ':2:CHANGELOG.md') + '\n', repo.git('show', ':3:CHANGELOG.md') + '\n')!); + repo.git('add', 'CHANGELOG.md'); + } + const r = restamp(repo, ['--next']); + expect(r.status, r.err).toBe(EXIT.ok); + const files = filesOf(r.out); + expect(files).toEqual(expect.arrayContaining(['VERSION', 'CHANGELOG.md'])); + expect(cached(repo)).toEqual(expect.arrayContaining(files)); + expect(unstaged(repo).filter(f => files.includes(f))).toEqual([]); + const staged = /^STAGED: (.*)$/m.exec(r.out)?.[1]?.split(' ') ?? []; + expect(staged).toEqual(expect.arrayContaining(files)); + repo.git('commit', '-q', '--no-edit'); + expect(repo.git('show', 'HEAD:VERSION')).toBe('1.2.5.0'); + expect(repo.git('show', 'HEAD:CHANGELOG.md')).toMatch(/^# Changelog\n\n## \[1\.2\.5\.0\]/); + expect(fs.existsSync(path.join(repo.work, '.gstack/tmp/restamp-staging.json'))).toBe(false); + }); + + test('a stamp file with unstaged edits of its own refuses before writing (RESTAMP_UNSTAGED_EDITS); nothing written or staged', () => { + const repo = fixture(); + makePr(repo, 'pr-c'); + fs.appendFileSync(path.join(repo.work, 'CHANGELOG.md'), '\n- an edit the caller has not staged\n'); + const r = restamp(repo, ['--next']); + expect(r.status).toBe(EXIT.refused); + expect(r.err).toMatch(/CHANGELOG\.md holds? unstaged edits of your own.*nothing written.*\(RESTAMP_UNSTAGED_EDITS\)/); + expect(readFile(repo, 'VERSION')).toBe('1.2.3.0\n'); + expect(cached(repo)).toEqual([]); + }); + + test('a staging failure prints RESTAMP_NOT_STAGED with the files written and a marker; the noop rerun stages them', () => { + const repo = fixture(); + makePr(repo, 'pr-d'); + const lock = path.join(repo.work, '.git', 'index.lock'); + fs.writeFileSync(lock, ''); + const r = restamp(repo, ['--next']); + expect(r.status).toBe(EXIT.fail); + expect(r.err).toContain('(RESTAMP_NOT_STAGED)'); + expect(r.out).not.toMatch(/^STAGED:/m); + expect(readFile(repo, 'VERSION')).toBe('1.2.4.0\n'); + expect(cached(repo)).toEqual([]); + expect(fs.existsSync(path.join(repo.work, '.gstack/tmp/restamp-staging.json'))).toBe(true); + fs.rmSync(lock, { force: true }); + const again = restamp(repo, ['--next']); + expect(again.status, again.err).toBe(EXIT.ok); + expect(again.out).toMatch(/^RESTAMP: .*\(noop;/m); + expect(cached(repo)).toEqual(expect.arrayContaining(['VERSION', 'CHANGELOG.md'])); + expect(again.out).toMatch(/^STAGED: .*VERSION/m); + expect(fs.existsSync(path.join(repo.work, '.gstack/tmp/restamp-staging.json'))).toBe(false); + }); + + // fails_when=a staging marker left by a failure the lane then cleared by hand makes every later noop restamp exit 1 with RESTAMP_NOT_STAGED, whose fix (rerun) can never clear it (outside voice, PR #3737); why_new=the test above reruns on the same unchanged files; seam=.git/index.lock; no_claim=how the lane cleared the failure + test('a stale staging marker (its files changed since: a hand commit, then another stamped branch) is dropped by the noop rerun instead of failing it', () => { + const repo = fixture(); + makePr(repo, 'pr-a'); + expect(restamp(repo, ['--next']).status).toBe(EXIT.ok); + repo.git('commit', '-q', '-m', 'stamp pr-a'); + makePr(repo, 'pr-d'); + const lock = path.join(repo.work, '.git', 'index.lock'); + fs.writeFileSync(lock, ''); + expect(restamp(repo, ['--next']).err).toContain('(RESTAMP_NOT_STAGED)'); + fs.rmSync(lock, { force: true }); + repo.git('add', '-A'); + repo.git('commit', '-q', '-m', 'stamp pr-d by hand'); + repo.git('checkout', '-q', 'pr-a'); + const noop = restamp(repo, ['--next']); + expect(noop.status, noop.err).toBe(EXIT.ok); + expect(noop.err).not.toContain('RESTAMP_NOT_STAGED'); + expect(noop.out).toMatch(/^NOTE: .*CHANGELOG\.md.* changed since an earlier restamp wrote (it|them); not staged/m); + expect(fs.existsSync(path.join(repo.work, '.gstack/tmp/restamp-staging.json'))).toBe(false); + expect(restamp(repo, ['--next']).status).toBe(EXIT.ok); + }); +}); diff --git a/test/helpers/ship-queue-train-cases.ts b/test/helpers/ship-queue-train-cases.ts index 5e265932b..3ec17945b 100644 --- a/test/helpers/ship-queue-train-cases.ts +++ b/test/helpers/ship-queue-train-cases.ts @@ -1,7 +1,12 @@ /** Train setups shared by the test/ship-queue-train*.test.ts files (moved from test/ship-queue-train.test.ts, GSTA-151). */ +import { expect } from 'bun:test'; +import { spawnSync } from 'node:child_process'; import * as fs from 'node:fs'; import * as path from 'node:path'; -import { GREEN_GATE, nonceOf, openPr, queue, readyReceipt, type Rule, type TrainFixture } from './ship-queue-train-fixture'; +import { mergeChangelogs } from '../../lib/changelog-check'; +import { EXIT } from '../../lib/headless-artifacts'; +import { runBin } from './restamp-fixture'; +import { GREEN_GATE, line, nonceOf, openPr, queue, readyReceipt, type Rule, type TrainFixture } from './ship-queue-train-fixture'; export function twoPrTrain(fx: TrainFixture, gate: Rule = GREEN_GATE) { const h1 = openPr(fx, '1'); @@ -27,3 +32,33 @@ export function publishMain(fx: TrainFixture, prefixCommit: string, version: str repo.git('fetch', '-q', 'origin'); return repo.git('rev-parse', 'HEAD'); } + +/** The lane's slot stamp on its pin (main merged in first from slot 2 on): gstack-restamp --next, then the stamp files committed and pushed as the PR head. */ +export function laneStamp(fx: TrainFixture, pr: string, pin: string, o: { mergeMain?: boolean; dropStamp?: boolean } = {}): string { + const { repo } = fx; + repo.git('reset', '-q', '--hard'); + repo.git('fetch', '-q', 'origin'); + repo.git('checkout', '-q', '-B', `lane${pr}`, pin); + if (o.mergeMain) { + const m = spawnSync('git', ['merge', '--no-ff', '--no-commit', '-q', 'origin/main'], { cwd: repo.work, encoding: 'utf8', timeout: 30_000, env: { ...process.env, ...repo.env } }); + if (m.status !== 0) { + const side = (n: number) => repo.git('show', `:${n}:CHANGELOG.md`) + '\n'; + fs.writeFileSync(path.join(repo.work, 'CHANGELOG.md'), mergeChangelogs(side(2), side(3), side(1))!); + repo.git('add', 'CHANGELOG.md'); + } + } + const index = repo.git('write-tree'); + const stamp = runBin(repo, 'gstack-restamp', ['--next', '--base', 'main', '--allow-repo-commands'], fx.env); + expect(stamp.status, stamp.err).toBe(EXIT.ok); + const files = line(stamp.out, 'FILES:').slice('FILES: '.length).split(' ').filter(Boolean); + if (o.dropStamp) repo.git('read-tree', index); + else repo.git('add', '-A', '--', ...files); + repo.git('commit', '-q', '--no-edit', '-m', `slot stamp #${pr}`); + if (o.dropStamp) repo.git('checkout', '-q', '--', '.'); + const head = repo.git('rev-parse', 'HEAD'); + repo.git('push', '-q', '-f', 'origin', `${head}:refs/pull/${pr}/head`); + return head; +} + +/** GitHub reports the PR MERGED, which verify --train needs before it confirms a merge (#3531). */ +export const markMerged = (fx: TrainFixture, pr: string) => fx.write(d => { (d.prs[pr] as any).state = 'MERGED'; }); diff --git a/test/helpers/ship-queue-train-fixture.ts b/test/helpers/ship-queue-train-fixture.ts index c48fed515..17e3f822b 100644 --- a/test/helpers/ship-queue-train-fixture.ts +++ b/test/helpers/ship-queue-train-fixture.ts @@ -147,6 +147,41 @@ export function trainJson(fx: TrainFixture, nonce: string, state = fx.state): an } export const dispatches = (fx: TrainFixture, workflow?: string) => fx.read().log.filter(l => l.cmd === 'workflow run' && (!workflow || l.workflow === workflow)); +/** For train land and land: pr view reads the head from origin's branch; pr edit and pr merge act on origin; any commit without recorded CI reads green. */ +export function wrapLand(fx: TrainFixture): void { + const dir = path.join(fx.repo.root, 'gh-land-wrap'); + fs.mkdirSync(dir, { recursive: true }); + const shim = path.join(fx.repo.root, 'gh-train-shim', 'shim.js'); + fs.writeFileSync(path.join(dir, 'gh'), [ + '#!/usr/bin/env node', + "const fs=require('fs');const cp=require('child_process');const D=process.env.LAND_WRAP_DATA;const a=process.argv.slice(2);", + "const d=JSON.parse(fs.readFileSync(D,'utf8'));const save=()=>fs.writeFileSync(D,JSON.stringify(d));", + "const g=(...x)=>cp.execFileSync('git',['--git-dir',d.origin,...x],{encoding:'utf8',timeout:30000}).trim();", + "const rev=r=>{try{return g('rev-parse','--verify','--quiet',r)}catch{return ''}};", + "if(a[0]==='pr'&&a[1]==='view'&&d.prs[a[2]]){const p=d.prs[a[2]];const h=rev('refs/heads/'+p.headRefName);if(h)p.headRefOid=h;save();console.log(JSON.stringify(p));process.exit(0)}", + "if(a[0]==='pr'&&a[1]==='edit'){(d.labels=d.labels||{})[a[2]]=a[a.indexOf('--add-label')+1];save();process.exit(0)}", + "if(a[0]==='pr'&&a[1]==='merge'){const p=d.prs[a[2]];const want=a[a.indexOf('--match-head-commit')+1];if(rev('refs/heads/'+p.headRefName)!==want){console.error('Head branch was modified');process.exit(1)}", + " const before=rev('main');let tree=rev(want+'^{tree}');", + " if(d.outsideOnMerge){const blob=cp.execFileSync('git',['--git-dir',d.origin,'hash-object','-w','--stdin'],{input:'outside\\n',encoding:'utf8',timeout:30000}).trim();", + " const ls=g('ls-tree',before+'^{tree}')+'\\n100644 blob '+blob+'\\toutside.txt';const ot=cp.execFileSync('git',['--git-dir',d.origin,'mktree'],{input:ls+'\\n',encoding:'utf8',timeout:30000}).trim();", + " g('update-ref','refs/heads/main',g('commit-tree',ot,'-p',before,'-m','outside'));tree=g('merge-tree','--write-tree','--merge-base='+before,rev('main'),want).split('\\n')[0];d.outsideOnMerge=false}", + " g('update-ref','refs/heads/main',g('commit-tree',tree,'-p',rev('main'),'-m','squash #'+a[2]));p.state='MERGED';save();process.exit(0)}", + "const paged=/commits\\/([0-9a-f]+)\\/(check-runs|status)\\?.*page=(\\d+)/.exec(a[1]||'');", + "if(a[0]==='api'&&paged&&!(d.ci||{})[paged[1]]&&!((d.statuses||{})[paged[1]])){const list=paged[2]==='check-runs'?[{name:'free-tests',status:'completed',conclusion:'success',id:1,completed_at:'2026-10-26T00:00:00Z'}]:[];const key=paged[2]==='check-runs'?'check_runs':'statuses';console.log(JSON.stringify(paged[3]==='1'?{total_count:list.length,state:'success',[key]:list}:{total_count:list.length,[key]:[]}));process.exit(0)}", + `const r=cp.spawnSync('node',[${JSON.stringify(shim)},D,...a],{stdio:'inherit',timeout:120000});process.exit(r.status??1);`, + '', + ].join('\n'), { mode: 0o755 }); + fx.env.PATH = `${dir}:${fx.env.PATH}`; + (fx.env as Record).LAND_WRAP_DATA = fx.dataFile; +} + +/** A changelog_fragments PR: a fragment under changelog.d/, pushed as pr-, with a READY receipt. */ +export function fragmentPr(fx: TrainFixture, pr: string, fragment = `- #${pr} change\n`): string { + const head = openPr(fx, pr, { file: `changelog.d/pr-${pr}.md`, text: fragment, changelog: false }); + fx.repo.git('push', '-q', 'origin', `${head}:refs/heads/pr-${pr}`); + readyReceipt(fx, pr, head); + return head; +} function shimSource(): string { return [ "const fs = require('fs'); const path = require('path'); const cp = require('child_process');", diff --git a/test/ship-queue-land-lock.test.ts b/test/ship-queue-land-lock.test.ts new file mode 100644 index 000000000..6afefbfce --- /dev/null +++ b/test/ship-queue-land-lock.test.ts @@ -0,0 +1,368 @@ +/** + * The shared land lock (lib/ship-queue-land-lock.ts, #3599): `land` and `train land` take the same + * refs/notes/gstack-land lock, so a standalone land and a train slot can never both publish onto one main. + * Module cases run against a bare origin and two clones (the race, the read states, owner kinds, the + * --break-lock rules); command cases drive bin/gstack-ship-queue against the train fixture and its gh + * shims (both directions, the train-scoped between phase, refusal and resume, the freeze-owned lock and the + * repair train's takeover). + */ +import { afterEach, describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { EXIT } from '../lib/headless-artifacts'; +import { acquireLandLock, LAND_LOCK_REF, readLock, writeLock, type LandLock, type LockOwner } from '../lib/ship-queue-land-lock'; +import { BASE_POLICY, cleanup, makeFixtureRepo, type FixtureRepo } from './helpers/restamp-fixture'; +import { coordinatorClone } from './helpers/ship-queue-fixture'; +import { + GREEN_GATE, TRAIN_POLICY, fragmentPr, nonceOf, queue, repoKey, trainFixture, trainJson, wrapLand, type TrainFixture, +} from './helpers/ship-queue-train-fixture'; + +const repos: FixtureRepo[] = []; +const fixtures: TrainFixture[] = []; +afterEach(() => { + while (repos.length) cleanup(repos.pop()!); + while (fixtures.length) cleanup(fixtures.pop()!.repo); +}); + +const T0 = Date.parse('2026-10-31T12:00:00Z'); +const at = (min: number) => () => new Date(T0 + min * 60_000); + +/** Run fn with the fixture's git identity in process.env (the lock writes commits in-process). */ +function withRepoEnv(repo: FixtureRepo, fn: () => T): T { + const saved = Object.fromEntries(Object.keys(repo.env).map(k => [k, process.env[k]])); + Object.assign(process.env, repo.env); + try { return fn(); } finally { + for (const [k, v] of Object.entries(saved)) { if (v === undefined) delete process.env[k]; else process.env[k] = v; } + } +} +function bareRepo(): FixtureRepo { const r = makeFixtureRepo({ policy: BASE_POLICY, prefix: 'gstack-land-lock-' }); repos.push(r); return r; } +const standalone = (pr: string): LockOwner => ({ pr }); +const slotOf = (train: string, slot: number, pr: string): LockOwner => ({ train, slot, pr }); +function take(root: string, owner: LockOwner, o: { journal?: string; now?: () => Date; breakLock?: boolean; breakTrainLock?: boolean; liveTrain?: (nonce: string) => boolean; phase?: LandLock['phase'] } = {}) { + return acquireLandLock({ + root, holder: 'test:1', owner, journal: o.journal ?? ('train' in owner && owner.train ? `train/${owner.train}/slot-${owner.slot}` : `land/${owner.pr}`), + phase: () => o.phase ?? 'stamping', now: o.now ?? at(0), breakLock: o.breakLock, breakTrainLock: o.breakTrainLock, liveTrain: o.liveTrain, + }); +} +const failureOf = (r: ReturnType) => ('failure' in r ? r.failure : null); + +describe('the lock module', () => { + test('two writers racing the same lock sha leave one winner', () => { + const repo = bareRepo(); + const other = coordinatorClone(repo); + withRepoEnv(repo, () => { + const lock = (pr: string): LandLock => ({ pr, holder: `h${pr}`, started_at: new Date(T0).toISOString(), phase: 'stamping', journal: `land/${pr}` }); + expect(readLock(repo.work).state).toBe('absent'); + const first = writeLock(repo.work, '', lock('1')); + const second = writeLock(other, '', lock('2')); + expect([first.ok, second.ok]).toEqual([true, false]); + const seen = readLock(other); + expect(seen.state === 'held' && seen.lock.pr).toBe('1'); + const a = writeLock(repo.work, seen.sha, lock('3')); + const b = writeLock(other, seen.sha, lock('4')); + expect([a.ok, b.ok]).toEqual([true, false]); + const final = readLock(other); + expect(final.state === 'held' && final.lock.pr).toBe('3'); + }); + }, 60_000); + + test('absent, released, held and unreadable are distinct; an unreadable lock refuses, and --break-lock replaces it', () => { + const repo = bareRepo(); + withRepoEnv(repo, () => { + const root = repo.work; + expect(readLock(root)).toMatchObject({ state: 'absent' }); + const got = take(root, standalone('5')); + expect(failureOf(got)).toBeNull(); + expect(readLock(root)).toMatchObject({ state: 'held', lock: { pr: '5', phase: 'stamping' } }); + if ('held' in got) expect(got.held.release()).toBe(true); + expect(readLock(root)).toMatchObject({ state: 'released' }); + const junk = repo.git('commit-tree', '4b825dc642cb6eb9a060e54bf8d69288fbee4904', '-m', 'not a lock'); + repo.git('push', '-q', '-f', 'origin', `${junk}:${LAND_LOCK_REF}`); + expect(readLock(root)).toMatchObject({ state: 'unreadable' }); + const refused = take(root, standalone('6')); + expect(failureOf(refused)).toMatchObject({ code: 'QUEUE_LAND_LOCKED' }); + expect(failureOf(refused)!.message).toContain('cannot be read'); + const broke = take(root, standalone('6'), { breakLock: true }); + expect(failureOf(broke)).toBeNull(); + expect(broke.lines.join('\n')).toContain(`LOCK: broke an unreadable lock at ${junk.slice(0, 12)}`); + expect(readLock(root)).toMatchObject({ state: 'held', lock: { pr: '6' } }); + }); + }, 60_000); + + test('ownership compares the owner kind and journal: the same PR across commands and across trains refuses; a resume of the same journal and a sequential acquisition succeed', () => { + const repo = bareRepo(); + withRepoEnv(repo, () => { + const root = repo.work; + const mine = take(root, standalone('7'), { phase: 'waiting' }); + expect(failureOf(mine)).toBeNull(); + const asSlot = take(root, slotOf('trainA', 1, '7')); + expect(failureOf(asSlot)).toMatchObject({ code: 'QUEUE_LAND_LOCKED' }); + expect(failureOf(asSlot)!.message).toMatch(/^#7 holds the land lock \(test:1, phase waiting, since /); + const resumed = take(root, standalone('7'), { now: at(3) }); + expect(failureOf(resumed)).toBeNull(); + expect(readLock(root)).toMatchObject({ state: 'held', lock: { pr: '7', started_at: new Date(T0).toISOString() } }); + if ('held' in resumed) resumed.held.release(); + + const trainA = take(root, slotOf('trainA', 1, '7'), { phase: 'pushing' }); + expect(failureOf(trainA)).toBeNull(); + expect(readLock(root)).toMatchObject({ state: 'held', lock: { train: 'trainA', slot: 1, pr: '7' } }); + const trainB = take(root, slotOf('trainB', 1, '7')); + expect(failureOf(trainB)!.message).toMatch(/^train trainA slot 1 \(#7\) holds the land lock/); + expect(failureOf(take(root, standalone('7')))!.message).toMatch(/^train trainA slot 1 \(#7\) holds the land lock/); + expect(failureOf(take(root, slotOf('trainA', 2, '8')))!.message).toMatch(/^train trainA slot 1 \(#7\) holds the land lock/); + const same = take(root, slotOf('trainA', 1, '7')); + expect(failureOf(same)).toBeNull(); + if ('held' in same) same.held.release(); + expect(failureOf(take(root, standalone('8')))).toBeNull(); + }); + }, 60_000); + + test('--break-lock prints the holder, refuses a live train run\'s lock without --break-train-lock, and keeps the staleness, mid-push, between and freeze rules', () => { + const repo = bareRepo(); + withRepoEnv(repo, () => { + const root = repo.work; + const hold = (owner: LockOwner, phase: LandLock['phase']) => { + const r = readLock(root); + if (r.state === 'held') writeLock(root, r.sha, { released: true, by: 'test', at: new Date(T0).toISOString() }); + expect(failureOf(take(root, owner, { phase }))).toBeNull(); + }; + hold(slotOf('trainA', 2, '9'), 'waiting'); + const live = take(root, standalone('3'), { breakLock: true, now: at(1) }); + expect(failureOf(live)).toMatchObject({ code: 'QUEUE_LAND_LOCKED' }); + expect(failureOf(live)!.message).toContain('--break-train-lock'); + expect(live.lines).toContain(`LOCK_HOLDER: train trainA slot 2 (#9) holder=test:1 phase=waiting since=${new Date(T0).toISOString()}`); + const ended = take(root, standalone('3'), { breakLock: true, now: at(1), liveTrain: () => false }); + expect(failureOf(ended)).toBeNull(); + expect(ended.lines).toContain("LOCK: broke train trainA slot 2 (#9)'s lock (test:1, phase waiting)"); + + hold(slotOf('trainA', 2, '9'), 'waiting'); + const forced = take(root, standalone('3'), { breakLock: true, breakTrainLock: true, now: at(1) }); + expect(failureOf(forced)).toBeNull(); + + hold(slotOf('trainA', 1, '9'), 'between'); + const young = take(root, standalone('3'), { breakLock: true, breakTrainLock: true, now: at(1) }); + expect(failureOf(young)!.message).toMatch(/train trainA slot 1 \(#9\) is between slots \(60 s ago\); --break-lock refused/); + expect(failureOf(take(root, standalone('3'), { breakLock: true, breakTrainLock: true, now: at(11) }))).toBeNull(); + + hold(slotOf('trainA', 1, '9'), 'pushing'); + expect(failureOf(take(root, standalone('3'), { breakLock: true, breakTrainLock: true, now: at(1) }))!.message).toMatch(/train trainA slot 1 \(#9\)'s holder is mid-push \(60 s ago\); --break-lock refused/); + + hold(slotOf('trainA', 1, '9'), 'freeze'); + const frozen = take(root, standalone('3'), { breakLock: true, breakTrainLock: true, now: at(60) }); + expect(failureOf(frozen)).toMatchObject({ code: 'QUEUE_TRAIN_FROZEN' }); + expect(failureOf(take(root, slotOf('trainB', 1, '4'), { breakLock: true, breakTrainLock: true, now: at(60) }))).toMatchObject({ code: 'QUEUE_TRAIN_FROZEN' }); + + hold(standalone('5'), 'waiting'); + const plain = take(root, standalone('3'), { breakLock: true, now: at(1) }); + expect(plain.lines).toContain(`LOCK_HOLDER: #5 holder=test:1 phase=waiting since=${new Date(T0).toISOString()}`); + expect(plain.lines).toContain("LOCK: broke #5's lock (test:1, phase waiting)"); + }); + }, 60_000); +}); + +const POLICY = { ...TRAIN_POLICY, release_tool: null, release_outputs: [], changelog_fragments: { dir: 'changelog.d' }, min_gstack_version: '1.91.336.0' }; +function stampedTrain(prs = ['1', '2']) { + const fx = trainFixture(POLICY, 'gstack-land-lock-'); fixtures.push(fx); wrapLand(fx); + const heads = prs.map(pr => fragmentPr(fx, pr)); + fx.write(d => { d.rules['evals.yml'] = GREEN_GATE; }); + const nonce = nonceOf(queue(fx, ['train', 'build', '--prs', prs.map((pr, i) => `${pr}@${heads[i]}`).join(',')]).out); + expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok); + return { fx, heads, nonce }; +} +const trainLand = (fx: TrainFixture, nonce: string, pr: string, extra: string[] = []) => queue(fx, ['train', 'land', '--train', nonce, '--pr', pr, '--no-wait', ...extra]); +const land = (fx: TrainFixture, pr: string, extra: string[] = []) => queue(fx, ['land', '--pr', pr, '--no-wait', '--allow-repo-commands', ...extra]); +const originRev = (fx: TrainFixture, ref: string) => fx.repo.git('--git-dir', fx.repo.origin, 'rev-parse', '--verify', '-q', ref); +/** The lock as origin holds it: the parsed lock.json, or null when origin has no lock ref. */ +function originLock(fx: TrainFixture): any { + const sha = (() => { try { return originRev(fx, LAND_LOCK_REF); } catch { return ''; } })(); + return sha ? JSON.parse(fx.repo.git('--git-dir', fx.repo.origin, 'show', `${sha}:lock.json`)) : null; +} +const freezeFile = (fx: TrainFixture) => path.join(fx.state, 'ship-queue', 'freeze'); + +describe('land and train land share the lock', () => { + test('a train slot holding the lock refuses a standalone land (no takeover, no freeze; --break-lock names the train and wants --break-train-lock); the train keeps it between slots and releases it after the last', () => { + const { fx, heads, nonce } = stampedTrain(); + const pin3 = fragmentPr(fx, '3'); + const pushed = trainLand(fx, nonce, '1'); + expect(pushed.out, pushed.err).toMatch(/^PUSHED: [0-9a-f]{40}/m); + expect(originLock(fx)).toMatchObject({ train: nonce, slot: 1, pr: '1', phase: 'waiting' }); + const lockSha = originRev(fx, LAND_LOCK_REF); + + const blocked = land(fx, '3'); + expect(blocked.status).toBe(EXIT.refused); + expect(blocked.err).toMatch(new RegExp(`train ${nonce} slot 1 \\(#1\\) holds the land lock .*phase waiting.*\\(QUEUE_LAND_LOCKED\\)`)); + const broke = land(fx, '3', ['--break-lock']); + expect(broke.status).toBe(EXIT.refused); + expect(broke.err).toContain('--break-train-lock'); + expect(broke.out).toMatch(new RegExp(`^LOCK_HOLDER: train ${nonce} slot 1 \\(#1\\) holder=\\S+ phase=waiting since=`, 'm')); + expect(originRev(fx, LAND_LOCK_REF)).toBe(lockSha); + expect(originRev(fx, 'refs/heads/pr-3')).toBe(pin3); + expect(fs.existsSync(freezeFile(fx))).toBe(false); + + const merged = trainLand(fx, nonce, '1'); + expect(merged.status, merged.err).toBe(EXIT.ok); + expect(originLock(fx)).toMatchObject({ train: nonce, slot: 1, pr: '1', phase: 'between' }); + expect(land(fx, '3').err).toMatch(new RegExp(`train ${nonce} slot 1 \\(#1\\) holds the land lock .*phase between.*\\(QUEUE_LAND_LOCKED\\)`)); + expect(originRev(fx, 'refs/heads/pr-3')).toBe(pin3); + + const second = trainLand(fx, nonce, '2'); + expect(second.out, second.err).toMatch(/^LOCK: train \S+ slot 2 takes over its train's between-slots lock/m); + expect(originLock(fx)).toMatchObject({ train: nonce, slot: 2, pr: '2', phase: 'waiting' }); + expect(trainLand(fx, nonce, '2').status).toBe(EXIT.ok); + expect(originLock(fx)).toMatchObject({ released: true }); + expect(trainJson(fx, nonce).state).toBe('merged'); + expect(land(fx, '3').out).toMatch(/^PUSHED: [0-9a-f]{40}/m); + void heads; + }, 300_000); + + // fails_when=settleLock releases the lock on a refusal partway through a train, so one transient gh or fetch failure on slot 2 lets a standalone land publish onto S1 and the rerun freezes the queue (red team, PR #3737 pass 1); why_new=the test above refuses slot 1 only, before anything merged; seam=the fixture's gh pr view; no_claim=the freeze itself + test('a refusal partway through a train (slot 1 merged, slot 2 cannot read its PR) keeps the lock between slots, so a standalone land still refuses', () => { + const { fx, nonce } = stampedTrain(); + fragmentPr(fx, '3'); + trainLand(fx, nonce, '1'); + expect(trainLand(fx, nonce, '1').status).toBe(EXIT.ok); + const row = fx.read().prs['2']; + fx.write(d => { delete d.prs['2']; }); + const refused = trainLand(fx, nonce, '2'); + expect(refused.err).toContain('(QUEUE_PROOF_MISSING)'); + expect(refused.out).toMatch(/^LOCK: between /m); + expect(originLock(fx)).toMatchObject({ train: nonce, phase: 'between' }); + expect(land(fx, '3').err).toMatch(/holds the land lock .*phase between.*\(QUEUE_LAND_LOCKED\)/); + fx.write(d => { d.prs['2'] = row; }); + expect(trainLand(fx, nonce, '2').out).toMatch(/^PUSHED: [0-9a-f]{40}/m); + }, 300_000); + + /** Another train's lock, written to origin as that train's run would have left it. */ + function seedLock(fx: TrainFixture, lock: Partial & { train: string; phase: LandLock['phase'] }): void { + withRepoEnv(fx.repo, () => { + const cur = readLock(fx.repo.work); + expect(writeLock(fx.repo.work, cur.sha, { pr: '9', slot: 1, holder: 'old:1', started_at: '2026-10-31T12:00:00.000Z', phase_at: '2026-10-31T12:00:00.000Z', journal: `train/${lock.train}/slot-1`, ...lock } as LandLock).ok).toBe(true); + }); + } + /** A freeze record for this repository, cleared at `cleared`, written to the state root as the clearing verify leaves it. */ + function clearedFreeze(fx: TrainFixture, train: string, cleared: string): void { + const file = path.join(freezeFile(fx), `${repoKey(fx)}.json`); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify({ schema: 1, kind: 'freeze', repo_key: repoKey(fx), train, merged: ['9'], unmerged: [], main: { sha: '0'.repeat(40), tree: '0'.repeat(40) }, reason: 'test', backstop: null, repair: null, created_at: '2026-10-31T11:00:00.000Z', cleared_at: cleared, cleared_by: 'test', updated_at: cleared })); + } + + // fails_when=trainTakeover reads "no active freeze record here" as "the freeze is cleared", so a train whose notes fetch failed, or whose freezer never pushed the record, takes the freeze lock and lands onto a frozen main (red team, PR #3737 pass 2); why_new=the freeze test above only exercises the admitted repair train's takeover; seam=a lock and a freeze record written straight to origin and the state root; no_claim=how the record went missing + test('another train\'s freeze lock is taken over only on a freeze record cleared after the lock: no record, or one cleared before it, refuses', () => { + const { fx, nonce } = stampedTrain(['1']); + const old = 'gate.train-old'; + seedLock(fx, { train: old, phase: 'freeze' }); + const noRecord = trainLand(fx, nonce, '1'); + expect(noRecord.status).toBe(EXIT.refused); + expect(noRecord.err).toMatch(/train gate\.train-old slot 1 \(#9\) holds the land lock .*phase freeze.*\(QUEUE_LAND_LOCKED\)/); + expect(noRecord.out).not.toMatch(/^PUSHED:/m); + expect(originLock(fx)).toMatchObject({ train: old, phase: 'freeze' }); + + clearedFreeze(fx, old, '2026-10-31T11:30:00.000Z'); + expect(trainLand(fx, nonce, '1').err).toContain('(QUEUE_LAND_LOCKED)'); + clearedFreeze(fx, old, '2026-10-31T12:30:00.000Z'); + const taken = trainLand(fx, nonce, '1'); + expect(taken.out, taken.err).toContain(`LOCK: train ${nonce} takes over train ${old} slot 1 (#9)'s freeze lock; that freeze was cleared at 2026-10-31T12:30:00.000Z`); + expect(taken.out).toMatch(/^PUSHED: [0-9a-f]{40}/m); + }, 300_000); + + // fails_when=a freeze lock whose release failed after its freeze cleared refuses every standalone land for good, or a lock ref deleted on origin still reads held from this clone's cached copy (adversarial review, PR #3737 Step 11); why_new=the tests above take the freeze lock over only from train land; seam=a lock and a freeze record written straight to origin and the state root; no_claim=why the release failed + test('a standalone land takes over a freeze lock once a fetched freeze record shows it cleared after the lock, and a lock deleted on origin reads as absent', () => { + const { fx } = stampedTrain(['1']); + fragmentPr(fx, '3'); + seedLock(fx, { train: 'gate.train-old', phase: 'freeze' }); + const frozen = land(fx, '3'); + expect(frozen.status).toBe(EXIT.refused); + expect(frozen.err).toMatch(/train gate\.train-old slot 1 \(#9\) holds the land lock .*phase freeze.*\(QUEUE_LAND_LOCKED\)/); + clearedFreeze(fx, 'gate.train-old', '2026-10-31T12:30:00.000Z'); + const taken = land(fx, '3'); + expect(taken.out, taken.err).toContain("LOCK: taking over train gate.train-old slot 1 (#9)'s freeze lock; that freeze was cleared at 2026-10-31T12:30:00.000Z"); + expect(taken.out).toMatch(/^PUSHED: [0-9a-f]{40}/m); + + const { fx: other } = stampedTrain(['1']); + fragmentPr(other, '3'); + seedLock(other, { train: 'gate.train-old', phase: 'waiting' }); + expect(land(other, '3').err).toContain('(QUEUE_LAND_LOCKED)'); + other.repo.git('--git-dir', other.repo.origin, 'update-ref', '-d', LAND_LOCK_REF); + expect(land(other, '3').out).toMatch(/^PUSHED: [0-9a-f]{40}/m); + }, 300_000); + + // fails_when=trainTakeover treats a live train as terminal (two landers publish onto one main) or never takes over a finished train's lock (a stale lock blocks every later train); why_new=no other test seats a foreign train's lock; seam=a second train record written into the state root; no_claim=how that train ended + test('another train\'s lock is taken over once that train\'s record is terminal, and refused while it is still merging', () => { + const { fx, nonce } = stampedTrain(['1']); + const other = `${nonce}-other`; + const put = (state: string) => { + const dir = path.join(fx.state, 'ship-queue', 'trains', other); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'train.json'), JSON.stringify({ ...trainJson(fx, nonce), nonce: other, state })); + }; + put('merging'); + seedLock(fx, { train: other, phase: 'waiting' }); + expect(trainLand(fx, nonce, '1').err).toMatch(/holds the land lock .*phase waiting.*\(QUEUE_LAND_LOCKED\)/); + put('merged'); + const taken = trainLand(fx, nonce, '1'); + expect(taken.out, taken.err).toContain(`takes over train ${other} slot 1 (#9)'s lock; that train is merged`); + expect(taken.out).toMatch(/^PUSHED: [0-9a-f]{40}/m); + }, 300_000); + + test('a standalone land holding the lock refuses train land; a refusal before any push releases the train\'s lock', () => { + const { fx, heads, nonce } = stampedTrain(); + fragmentPr(fx, '3'); + expect(land(fx, '3').out).toMatch(/^PUSHED: /m); + const blocked = trainLand(fx, nonce, '1'); + expect(blocked.status).toBe(EXIT.refused); + expect(blocked.err).toMatch(/#3 holds the land lock .*phase waiting.*\(QUEUE_LAND_LOCKED\)/); + expect(blocked.out).not.toMatch(/^PUSHED:/m); + expect(originRev(fx, 'refs/heads/pr-1')).toBe(heads[0]); + + expect(land(fx, '3').out).toContain('LOCK: released'); + const stale = trainLand(fx, nonce, '1'); + expect(stale.err).toMatch(/nothing of train \S+ merged, so the queue is not frozen.*\(QUEUE_TRAIN_TREE_MISMATCH\)/); + expect(originLock(fx)).toMatchObject({ released: true }); + }, 300_000); + + test('after a push the train keeps its lock through a failed wait and a resume takes it back', () => { + const { fx, nonce } = stampedTrain(['1']); + const stamp = /^PUSHED: ([0-9a-f]{40})/m.exec(trainLand(fx, nonce, '1').out)![1]!; + fx.write(d => { d.ci[stamp] = [{ name: 'free-tests', status: 'completed', conclusion: 'failure' }] as any; }); + const red = trainLand(fx, nonce, '1'); + expect(red.status).not.toBe(EXIT.ok); + expect(originLock(fx)).toMatchObject({ train: nonce, slot: 1, pr: '1', phase: 'waiting' }); + fx.write(d => { delete d.ci[stamp]; }); + const resumed = trainLand(fx, nonce, '1'); + expect(resumed.status, resumed.err).toBe(EXIT.ok); + expect(originLock(fx)).toMatchObject({ released: true }); + }, 300_000); + + test('a stop that freezes leaves the lock in a freeze phase that --break-lock never breaks; only the admitted repair train takes it over, and it is released once the freeze clears', () => { + const { fx, nonce } = stampedTrain(); + trainLand(fx, nonce, '1'); + expect(trainLand(fx, nonce, '1').status).toBe(EXIT.ok); + trainLand(fx, nonce, '2'); + fx.write(d => { (d as any).outsideOnMerge = true; }); + const drift = trainLand(fx, nonce, '2'); + expect(drift.err).toContain('(QUEUE_TRAIN_TREE_MISMATCH)'); + expect(drift.out).toMatch(/^QUEUE_TRAIN_FROZEN: /m); + expect(originLock(fx)).toMatchObject({ train: nonce, phase: 'freeze' }); + const lockSha = originRev(fx, LAND_LOCK_REF); + + fragmentPr(fx, '3'); + const refused = land(fx, '3', ['--break-lock', '--break-train-lock']); + expect(refused.status).toBe(EXIT.refused); + expect(refused.err).toContain('(QUEUE_TRAIN_FROZEN)'); + expect(originRev(fx, LAND_LOCK_REF)).toBe(lockSha); + + const h4 = fragmentPr(fx, '4'); + const repair = queue(fx, ['train', 'build', '--repair', '--prs', `4@${h4}`]); + expect(repair.status, repair.err).toBe(EXIT.ok); + const rn = nonceOf(repair.out); + expect(queue(fx, ['train', 'verdict', '--train', rn]).status).toBe(EXIT.ok); + const taken = trainLand(fx, rn, '4'); + expect(taken.out, taken.err).toMatch(new RegExp(`^LOCK: repair train ${rn} takes over the freeze lock of train ${nonce}`, 'm')); + expect(originLock(fx)).toMatchObject({ train: rn, slot: 1, pr: '4', phase: 'freeze' }); + expect(land(fx, '3', ['--break-lock', '--break-train-lock']).err).toContain('(QUEUE_TRAIN_FROZEN)'); + const cleared = trainLand(fx, rn, '4'); + expect(cleared.status, cleared.err).toBe(EXIT.ok); + expect(cleared.out).toMatch(/^FREEZE: cleared /m); + expect(originLock(fx)).toMatchObject({ released: true }); + }, 600_000); +}); diff --git a/test/ship-queue-train-freeze.test.ts b/test/ship-queue-train-freeze.test.ts index 13c09b2d4..a5602c003 100644 --- a/test/ship-queue-train-freeze.test.ts +++ b/test/ship-queue-train-freeze.test.ts @@ -5,7 +5,7 @@ import * as path from 'node:path'; import { EXIT } from '../lib/headless-artifacts'; import { cleanup } from './helpers/restamp-fixture'; import { coordinatorClone } from './helpers/ship-queue-fixture'; -import { publishMain, twoPrTrain } from './helpers/ship-queue-train-cases'; +import { laneStamp, markMerged, publishMain, twoPrTrain } from './helpers/ship-queue-train-cases'; import { TRAIN_POLICY, dispatches, line, nonceOf, openPr, queue, readyReceipt, repoKey, trainFixture, trainJson, type TrainFixture, @@ -28,8 +28,10 @@ describe('verify, freeze, repair and clear', () => { const v = (pr: string) => queue(fx, ['verify', '--pr', pr, '--train', nonce], {}, coord); const released = (prefix: string, version: string) => fx.repo.git('show', `${prefix}:CHANGELOG.md`).replace('## [Unreleased]', `## [${version}] - 2026-10-18`) + '\n'; const main1 = publishMain(fx, t.prefixes[1].commit, '1.2.4.0', { file: 'CHANGELOG.md', text: released(t.prefixes[1].commit, '1.2.4.0') }); + markMerged(fx, '1'); expect(v('1').status).toBe(EXIT.ok); - expect(line(v('2').out, 'MERGE_CONDITION:')).toBe(`MERGE_CONDITION: sha=${h2}`); + const s2 = laneStamp(fx, '2', h2, { mergeMain: true }); + expect(line(v('2').out, 'MERGE_CONDITION:')).toBe(`MERGE_CONDITION: sha=${s2}`); // Main's CHANGELOG plus #2's stamped entry, with one line the pin never wrote. const smuggled = fx.repo.git('show', `${main1}:CHANGELOG.md`).replace('# Changelog\n', '# Changelog\n\n## [1.2.5.0] - 2026-10-18\n\n- #2 change\n- added after the gate\n') + '\n'; const broken = publishMain(fx, t.prefixes[2].commit, '1.2.5.0', { file: 'CHANGELOG.md', text: smuggled }); @@ -111,6 +113,7 @@ describe('verify, freeze, repair and clear', () => { const rn = nonceOf(queue(fx, ['train', 'build', '--repair', '--prs', `3@${h3}`]).out); expect(queue(fx, ['train', 'verdict', '--train', rn]).status).toBe(EXIT.ok); publishMain(fx, trainJson(fx, rn).prefixes[1].commit, '1.2.6.0'); + markMerged(fx, '3'); const cleared = queue(fx, ['verify', '--pr', '3', '--train', rn], {}, coord); expect(cleared.out).toMatch(/^FREEZE: cleared /m); expect(next(cleared.out)).toBe(`NEXT: gh pr view 2 --json state,headRefOid (unreadable at the freeze: #2@${h2.slice(0, 12)}); seat each that is OPEN in a train`); @@ -125,6 +128,7 @@ describe('verify, freeze, repair and clear', () => { const coord = coordinatorClone(fx.repo); const v = (pr: string) => queue(fx, ['verify', '--pr', pr, '--train', nonce], {}, coord); publishMain(fx, t.prefixes[1].commit, '1.2.4.0'); + markMerged(fx, '1'); expect(v('1').status).toBe(EXIT.ok); const later = publishMain(fx, t.prefixes[2].commit, '1.2.5.0', { file: 'stray.txt', text: 'a later train\n' }); fx.write(d => { (d.prs['2'] as any).state = 'MERGED'; }); @@ -157,6 +161,7 @@ describe('verify, freeze, repair and clear', () => { const coord = coordinatorClone(fx.repo); const v = (pr: string) => queue(fx, ['verify', '--pr', pr, '--train', nonce], {}, coord); publishMain(fx, t.prefixes[1].commit, '1.2.4.0'); + markMerged(fx, '1'); expect(v('1').status).toBe(EXIT.ok); const later = publishMain(fx, t.prefixes[2].commit, '1.2.5.0', { file: 'stray.txt', text: 'a later train\n' }); const tree = fx.repo.git('rev-parse', `${later}^{tree}`); @@ -181,6 +186,7 @@ describe('verify, freeze, repair and clear', () => { const v = (pr: string) => queue(fx, ['verify', '--pr', pr, '--train', nonce], {}, coord); const released = fx.repo.git('show', `${t.prefixes[1].commit}:CHANGELOG.md`).replace('## [Unreleased]', '## [1.2.4.0] - 2026-10-18') + '\n'; const main1 = publishMain(fx, t.prefixes[1].commit, '1.2.4.0', { file: 'CHANGELOG.md', text: released }); + markMerged(fx, '1'); expect(v('1').status).toBe(EXIT.ok); const foreign = publishMain(fx, t.prefixes[1].commit, '1.2.4.1', { file: 'stray.txt', text: 'landed between the two slots\n' }); const text = fx.repo.git('show', `${main1}:CHANGELOG.md`).replace('# Changelog\n', '# Changelog\n\n## [1.2.5.0] - 2026-10-18\n\n- #2 change\n- added after the gate\n') + '\n'; diff --git a/test/ship-queue-train-land.test.ts b/test/ship-queue-train-land.test.ts index 64a046ee7..9ac7ac65a 100644 --- a/test/ship-queue-train-land.test.ts +++ b/test/ship-queue-train-land.test.ts @@ -10,47 +10,13 @@ import * as path from 'node:path'; import { EXIT } from '../lib/headless-artifacts'; import { cleanup } from './helpers/restamp-fixture'; import { - GREEN_GATE, RED_GATE, TRAIN_POLICY, line, nonceOf, openPr, queue, readyReceipt, shippedBody, trainFixture, trainJson, type TrainFixture, + GREEN_GATE, RED_GATE, TRAIN_POLICY, fragmentPr, line, nonceOf, openPr, queue, readyReceipt, shippedBody, trainFixture, trainJson, wrapLand as wrap, type TrainFixture, } from './helpers/ship-queue-train-fixture'; const POLICY = { ...TRAIN_POLICY, release_tool: null, release_outputs: [], changelog_fragments: { dir: 'changelog.d' }, min_gstack_version: '1.91.336.0' }; const fixtures: TrainFixture[] = []; afterEach(() => { while (fixtures.length) cleanup(fixtures.pop()!.repo); }); -/** pr view reads the head from origin's branch; pr edit and pr merge act on origin; any commit without recorded CI reads green. */ -function wrap(fx: TrainFixture): void { - const dir = path.join(fx.repo.root, 'gh-land-wrap'); - fs.mkdirSync(dir, { recursive: true }); - const shim = path.join(fx.repo.root, 'gh-train-shim', 'shim.js'); - fs.writeFileSync(path.join(dir, 'gh'), [ - '#!/usr/bin/env node', - "const fs=require('fs');const cp=require('child_process');const D=process.env.LAND_WRAP_DATA;const a=process.argv.slice(2);", - "const d=JSON.parse(fs.readFileSync(D,'utf8'));const save=()=>fs.writeFileSync(D,JSON.stringify(d));", - "const g=(...x)=>cp.execFileSync('git',['--git-dir',d.origin,...x],{encoding:'utf8',timeout:30000}).trim();", - "const rev=r=>{try{return g('rev-parse','--verify','--quiet',r)}catch{return ''}};", - "if(a[0]==='pr'&&a[1]==='view'&&d.prs[a[2]]){const p=d.prs[a[2]];const h=rev('refs/heads/'+p.headRefName);if(h)p.headRefOid=h;save();console.log(JSON.stringify(p));process.exit(0)}", - "if(a[0]==='pr'&&a[1]==='edit'){(d.labels=d.labels||{})[a[2]]=a[a.indexOf('--add-label')+1];save();process.exit(0)}", - "if(a[0]==='pr'&&a[1]==='merge'){const p=d.prs[a[2]];const want=a[a.indexOf('--match-head-commit')+1];if(rev('refs/heads/'+p.headRefName)!==want){console.error('Head branch was modified');process.exit(1)}", - " const before=rev('main');let tree=rev(want+'^{tree}');", - " if(d.outsideOnMerge){const blob=cp.execFileSync('git',['--git-dir',d.origin,'hash-object','-w','--stdin'],{input:'outside\\n',encoding:'utf8',timeout:30000}).trim();", - " const ls=g('ls-tree',before+'^{tree}')+'\\n100644 blob '+blob+'\\toutside.txt';const ot=cp.execFileSync('git',['--git-dir',d.origin,'mktree'],{input:ls+'\\n',encoding:'utf8',timeout:30000}).trim();", - " g('update-ref','refs/heads/main',g('commit-tree',ot,'-p',before,'-m','outside'));tree=g('merge-tree','--write-tree','--merge-base='+before,rev('main'),want).split('\\n')[0];d.outsideOnMerge=false}", - " g('update-ref','refs/heads/main',g('commit-tree',tree,'-p',rev('main'),'-m','squash #'+a[2]));p.state='MERGED';save();process.exit(0)}", - "const paged=/commits\\/([0-9a-f]+)\\/(check-runs|status)\\?.*page=(\\d+)/.exec(a[1]||'');", - "if(a[0]==='api'&&paged&&!(d.ci||{})[paged[1]]&&!((d.statuses||{})[paged[1]])){const list=paged[2]==='check-runs'?[{name:'free-tests',status:'completed',conclusion:'success',id:1,completed_at:'2026-10-26T00:00:00Z'}]:[];const key=paged[2]==='check-runs'?'check_runs':'statuses';console.log(JSON.stringify(paged[3]==='1'?{total_count:list.length,state:'success',[key]:list}:{total_count:list.length,[key]:[]}));process.exit(0)}", - `const r=cp.spawnSync('node',[${JSON.stringify(shim)},D,...a],{stdio:'inherit',timeout:120000});process.exit(r.status??1);`, - '', - ].join('\n'), { mode: 0o755 }); - fx.env.PATH = `${dir}:${fx.env.PATH}`; - (fx.env as Record).LAND_WRAP_DATA = fx.dataFile; -} - -function fragmentPr(fx: TrainFixture, pr: string, fragment = `- #${pr} change\n`): string { - const head = openPr(fx, pr, { file: `changelog.d/pr-${pr}.md`, text: fragment, changelog: false }); - fx.repo.git('push', '-q', 'origin', `${head}:refs/heads/pr-${pr}`); - readyReceipt(fx, pr, head); - return head; -} function stampedTrain(gate = GREEN_GATE, policy: Record = POLICY) { const fx = trainFixture(policy, 'gstack-train-land-'); fixtures.push(fx); wrap(fx); const h1 = fragmentPr(fx, '1'); @@ -190,10 +156,15 @@ describe('stamped trains: build and land', () => { }, 300_000); test('a failed stamp push with the branch still at the pin fails retryably instead of reporting a move', () => { - const { fx, nonce } = stampedTrain(); + const { fx, h1, nonce } = stampedTrain(); expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok); const hook = path.join(fx.repo.origin, 'hooks', 'pre-receive'); fs.writeFileSync(hook, '#!/bin/sh\nexit 1\n', { mode: 0o755 }); + const refused = land(fx, nonce, '1'); + expect(refused.err).toMatch(/the land lock push to origin failed and the lock there is unchanged; nothing was pushed.*\(QUEUE_NOT_READY\)/); + expect(refused.err).not.toContain('another lander'); + expect(fx.repo.git('--git-dir', fx.repo.origin, 'rev-parse', 'refs/heads/pr-1')).toBe(h1); + fs.writeFileSync(hook, '#!/bin/sh\nwhile read old new ref; do [ "$ref" = refs/heads/pr-1 ] && exit 1; done\nexit 0\n', { mode: 0o755 }); const failed = land(fx, nonce, '1'); expect(failed.err).toMatch(/the stamp push failed and pr-1 still reads.*\(QUEUE_NOT_READY\)/); expect(failed.err).not.toContain('QUEUE_TRAIN_MOVED'); diff --git a/test/ship-queue-train-verify.test.ts b/test/ship-queue-train-verify.test.ts index 5c8551d0b..67ebcf224 100644 --- a/test/ship-queue-train-verify.test.ts +++ b/test/ship-queue-train-verify.test.ts @@ -6,7 +6,7 @@ import { mergeChangelogs, parseChangelog } from '../lib/changelog-check'; import { EXIT } from '../lib/headless-artifacts'; import { cleanup } from './helpers/restamp-fixture'; import { coordinatorClone } from './helpers/ship-queue-fixture'; -import { publishMain, twoPrTrain } from './helpers/ship-queue-train-cases'; +import { laneStamp, markMerged, publishMain, twoPrTrain } from './helpers/ship-queue-train-cases'; import { GREEN_GATE, TRAIN_POLICY, dispatches, line, nonceOf, openPr, queue, readyReceipt, repoKey, setPrBody, shippedBody, trainFixture, trainJson, type TrainFixture, @@ -27,21 +27,24 @@ describe('verify, freeze, repair and clear', () => { const coord = coordinatorClone(fx.repo); const v = (pr: string, train = nonce) => queue(fx, ['verify', '--pr', pr, '--train', train], {}, coord); + const s1 = laneStamp(fx, '1', h1); const pre1 = v('1'); expect(pre1.status).toBe(EXIT.ok); expect(pre1.out).toContain(`VERIFY_TRAIN: #1 may merge as slot 1 of train ${nonce}`); - expect(line(pre1.out, 'MERGE_CONDITION:')).toBe(`MERGE_CONDITION: sha=${h1}`); + expect(line(pre1.out, 'MERGE_CONDITION:')).toBe(`MERGE_CONDITION: sha=${s1}`); const early = v('2'); expect(early.status).toBe(EXIT.fail); expect(early.err).toMatch(/is neither T1 nor T2 modulo stamps.*nothing of train \S+ merged, so the queue is not frozen.*\(QUEUE_TRAIN_TREE_MISMATCH\)/); - publishMain(fx, t.prefixes[1].commit, '1.2.4.0'); + publishMain(fx, t.prefixes[1].commit, '1.2.4.0', { file: 'CHANGELOG.md', text: fx.repo.git('show', `${s1}:CHANGELOG.md`) + '\n' }); + markMerged(fx, '1'); const post1 = v('1'); expect(post1.status).toBe(EXIT.ok); - expect(post1.out).toMatch(/^main: [0-9a-f]{40} = T1 \(same modulo stamps \(1 stamp file\(s\)\)\)$/m); + expect(post1.out).toMatch(/^main: [0-9a-f]{40} = T1 \(same modulo stamps \(2 stamp file\(s\)\)\)$/m); expect(post1.out).toContain(`VERIFY_TRAIN: #1 merged; main's tree is T1 of train ${nonce} (1/2 merged)`); expect(next(post1.out)).toBe(`NEXT: gstack-ship-queue verify --pr 2 --train ${nonce}`); - expect(line(v('2').out, 'MERGE_CONDITION:')).toBe(`MERGE_CONDITION: sha=${h2}`); + const s2 = laneStamp(fx, '2', h2, { mergeMain: true }); + expect(line(v('2').out, 'MERGE_CONDITION:')).toBe(`MERGE_CONDITION: sha=${s2}`); const broken = publishMain(fx, t.prefixes[2].commit, '1.2.5.0', { file: 'stray.txt', text: 'not in the train\n' }); const mismatch = v('2'); @@ -69,8 +72,16 @@ describe('verify, freeze, repair and clear', () => { expect(queue(fx, ['train', 'verdict', '--train', rn]).status).toBe(EXIT.ok); expect(v('2').err).toContain('(QUEUE_TRAIN_FROZEN)'); expect(dispatches(fx, 'evals.yml').filter(d => d.ref === 'main').length).toBe(1); - expect(line(v('3', rn).out, 'MERGE_CONDITION:')).toBe(`MERGE_CONDITION: sha=${h3}`); + // The broken main still carries T2's [Unreleased] section, so #3's lane stamps its own entry by hand. + fx.repo.git('checkout', '-q', '-B', 'lane3', h3); + fs.writeFileSync(path.join(fx.repo.work, 'VERSION'), '1.2.6.0\n'); + fs.writeFileSync(path.join(fx.repo.work, 'CHANGELOG.md'), fx.repo.git('show', `${h3}:CHANGELOG.md`).replace('## [Unreleased]', '## [1.2.6.0] - 2026-10-18') + '\n'); + fx.repo.git('commit', '-qam', 'slot stamp #3'); + const s3 = fx.repo.git('rev-parse', 'HEAD'); + fx.repo.git('push', '-q', '-f', 'origin', `${s3}:refs/pull/3/head`); + expect(line(v('3', rn).out, 'MERGE_CONDITION:')).toBe(`MERGE_CONDITION: sha=${s3}`); publishMain(fx, trainJson(fx, rn).prefixes[1].commit, '1.2.6.0'); + markMerged(fx, '3'); const cleared = v('3', rn); expect(cleared.status).toBe(EXIT.ok); expect(cleared.out).toMatch(/^FREEZE: cleared \(repair #3 via train \S+; main [0-9a-f]{12} carries a green record\)$/m); @@ -85,14 +96,10 @@ describe('verify, freeze, repair and clear', () => { const coord = coordinatorClone(fx.repo); const v = () => queue(fx, ['verify', '--pr', '1', '--train', nonce], {}, coord); const { repo } = fx; - repo.git('checkout', '-q', '-B', 'lane1', h1); - fs.writeFileSync(path.join(repo.work, 'VERSION'), '1.2.4.0\n'); - repo.git('commit', '-qam', 'v1.2.4.0 stamp'); - const stamped = repo.git('rev-parse', 'HEAD'); - repo.git('push', '-q', '-f', 'origin', `${stamped}:refs/pull/1/head`); + const stamped = laneStamp(fx, '1', h1); const ok = v(); expect(ok.status).toBe(EXIT.ok); - expect(ok.out).toContain(`head: ${stamped} (same modulo stamps (1 stamp file(s)))`); + expect(ok.out).toContain(`head: ${stamped} (same modulo stamps (5 stamp file(s)))`); expect(ok.out).toContain(`SHIP_RECEIPT: pr=1 head=${h1.slice(0, 12)} version=- base=main (binds the pinned head)`); expect(line(ok.out, 'MERGE_CONDITION:')).toBe(`MERGE_CONDITION: sha=${stamped}`); setPrBody(fx, '1', shippedBody('1', 'b'.repeat(40))); @@ -133,6 +140,7 @@ describe('verify, freeze, repair and clear', () => { const coord = coordinatorClone(fx.repo); const v = (pr: string) => queue(fx, ['verify', '--pr', pr, '--train', nonce], {}, coord); const main1 = publishMain(fx, t.prefixes[1].commit, '1.2.4.0'); + markMerged(fx, '1'); expect(v('1').status).toBe(EXIT.ok); const { repo } = fx; @@ -142,6 +150,7 @@ describe('verify, freeze, repair and clear', () => { repo.git('rm', '-rq', '.'); repo.git('checkout', t.prefixes[2].commit, '--', '.'); fs.writeFileSync(path.join(repo.work, 'VERSION'), '1.2.5.0\n'); + fs.writeFileSync(path.join(repo.work, 'CHANGELOG.md'), repo.git('show', `${t.prefixes[2].commit}:CHANGELOG.md`).replace('## [Unreleased]', '## [1.2.5.0] - 2026-10-18') + '\n'); if (extra) fs.writeFileSync(path.join(repo.work, extra.file), extra.text); repo.git('add', '-A'); const tree = repo.git('write-tree'); @@ -224,8 +233,10 @@ describe('verify, freeze, repair and clear', () => { const v = (pr: string) => queue(fx, ['verify', '--pr', pr, '--train', nonce], {}, coord); // Slot 1: #1 squash-merges and is released as 1.2.4.0 (its [Unreleased] re-headed). - expect(line(v('1').out, 'MERGE_CONDITION:')).toBe(`MERGE_CONDITION: sha=${h1}`); + const s1 = laneStamp(fx, '1', h1); + expect(line(v('1').out, 'MERGE_CONDITION:')).toBe(`MERGE_CONDITION: sha=${s1}`); const main1 = publishMain(fx, t.prefixes[1].commit, '1.2.4.0', { file: 'CHANGELOG.md', text: repo.git('show', `${t.prefixes[1].commit}:CHANGELOG.md`).replace('## [Unreleased]', '## [1.2.4.0] - 2026-10-18') + '\n' }); + markMerged(fx, '1'); expect(v('1').out).toContain(`VERIFY_TRAIN: #1 merged; main's tree is T1 of train ${nonce} (1/2 merged)`); // Slot 2: the lane merges main into its pin; mergeChangelogs carries the child past its released parent. @@ -249,6 +260,7 @@ describe('verify, freeze, repair and clear', () => { // The squash merge publishes exactly that tree; the post-merge verify confirms it instead of freezing the queue. const main2 = publishMain(fx, laneHead, '1.2.5.0'); expect(repo.git('rev-parse', `${main2}^{tree}`)).toBe(repo.git('rev-parse', `${laneHead}^{tree}`)); + markMerged(fx, '2'); const post = v('2'); expect([post.status, post.err]).toEqual([EXIT.ok, '']); expect(line(post.out, 'main:')).toBe(`main: ${main2} = T2 (same modulo stamps (1 stamp file(s)) except CHANGELOG.md, main ${main1.slice(0, 12)}'s CHANGELOG.md plus #2's own entry)`); @@ -256,3 +268,133 @@ describe('verify, freeze, repair and clear', () => { expect(queue(fx, ['train', 'status']).out).not.toContain('QUEUE_TRAIN_FROZEN'); }, 300_000); }); + +describe('verify --train proves the slot stamp and a GitHub merge (#3531)', () => { + /** A main commit that carries an unstamped [Unreleased] entry, the way a bare commit after a no-commit merge shipped #3527. */ + function unstampedMain(fx: TrainFixture, entry: string): string { + const { repo } = fx; + repo.git('fetch', '-q', 'origin'); + repo.git('checkout', '-q', '-B', 'mainsim', 'origin/main'); + fs.writeFileSync(path.join(repo.work, 'CHANGELOG.md'), fs.readFileSync(path.join(repo.work, 'CHANGELOG.md'), 'utf8').replace('# Changelog\n', `# Changelog\n\n## [Unreleased]\n\n${entry}\n`)); + repo.git('commit', '-qam', 'merged without its stamp'); + repo.git('push', '-q', 'origin', 'HEAD:refs/heads/main'); + repo.git('fetch', '-q', 'origin'); + return repo.git('rev-parse', 'HEAD'); + } + const freezeFile = (fx: TrainFixture) => path.join(fx.state, 'ship-queue', 'freeze', `${repoKey(fx)}.json`); + + test('#3527: an unstamped slot head (the bare pin, or a no-commit base merge committed without the restamp it wrote) refuses QUEUE_TRAIN_UNSTAMPED naming both versions; the stamped head passes with STAMP_VERSION from the head', () => { + const fx = make(); + const { h1, h2, nonce } = twoPrTrain(fx); + expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok); + const t = trainJson(fx, nonce); + const coord = coordinatorClone(fx.repo); + const v = (pr: string) => queue(fx, ['verify', '--pr', pr, '--train', nonce], {}, coord); + + const bare = v('1'); + expect(bare.status).toBe(EXIT.refused); + expect(bare.err).toMatch(/#1's head [0-9a-f]{12} is not stamped above main: version: the head's VERSION 1\.2\.3\.0 is not above main's 1\.2\.3\.0.*\(QUEUE_TRAIN_UNSTAMPED\)/); + expect(bare.out).not.toContain('MERGE_CONDITION:'); + expect(next(bare.out)).toMatch(/^NEXT: #1's lane runs gstack-restamp --next on its head, commits what restamp staged/); + + const s1 = laneStamp(fx, '1', h1); + const ok = v('1'); + expect(ok.status, ok.err).toBe(EXIT.ok); + expect(ok.out).toContain('STAMP_VERSION: 1.2.4.0'); + expect(line(ok.out, 'MERGE_CONDITION:')).toBe(`MERGE_CONDITION: sha=${s1}`); + const unread = queue(fx, ['verify', '--pr', '1', '--train', nonce, '--no-fetch'], {}, coord); + expect(unread.status).toBe(EXIT.refused); + expect(unread.err).toMatch(/#1's current head was not read \(--no-fetch\).*\(QUEUE_PROOF_MISSING\)/); + expect(unread.out).not.toContain('MERGE_CONDITION:'); + publishMain(fx, t.prefixes[1].commit, '1.2.4.0', { file: 'CHANGELOG.md', text: fx.repo.git('show', `${s1}:CHANGELOG.md`) + '\n' }); + markMerged(fx, '1'); + expect(v('1').out).toContain(`VERIFY_TRAIN: #1 merged; main's tree is T1 of train ${nonce} (1/2 merged)`); + + laneStamp(fx, '2', h2, { mergeMain: true, dropStamp: true }); + const dropped = v('2'); + expect(dropped.status, dropped.err).toBe(EXIT.refused); + expect(dropped.err).toMatch(/#2's head [0-9a-f]{12} is not stamped above main: version: the head's VERSION 1\.2\.4\.0 is not above main's 1\.2\.4\.0.*\(QUEUE_TRAIN_UNSTAMPED\)/); + expect(dropped.out).not.toContain('MERGE_CONDITION:'); + expect(fs.existsSync(freezeFile(fx))).toBe(false); + const s2 = laneStamp(fx, '2', h2, { mergeMain: true }); + const stamped = v('2'); + expect(stamped.status, stamped.err).toBe(EXIT.ok); + expect(stamped.out).toContain('STAMP_VERSION: 1.2.5.0'); + expect(line(stamped.out, 'MERGE_CONDITION:')).toBe(`MERGE_CONDITION: sha=${s2}`); + }, 300_000); + + test('the CHANGELOG rule is named: a head whose VERSION moved but whose entry is still [Unreleased] refuses', () => { + const fx = make(); + const { h1, nonce } = twoPrTrain(fx); + expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok); + const { repo } = fx; + repo.git('checkout', '-q', '-B', 'lane1', h1); + fs.writeFileSync(path.join(repo.work, 'VERSION'), '1.2.4.0\n'); + repo.git('commit', '-qam', 'VERSION only'); + repo.git('push', '-q', '-f', 'origin', `${repo.git('rev-parse', 'HEAD')}:refs/pull/1/head`); + const r = queue(fx, ['verify', '--pr', '1', '--train', nonce], {}, coordinatorClone(fx.repo)); + expect(r.status).toBe(EXIT.refused); + expect(r.err).toMatch(/is not stamped above main: changelog: CHANGELOG\.md's top section at the head is \[Unreleased\], not \[1\.2\.4\.0\].*\(QUEUE_TRAIN_UNSTAMPED\)/); + }, 300_000); + + test('claim-at-open: a pin that carries its own stamp verifies at head == pin', () => { + const fx = make({ ...TRAIN_POLICY, queue_mode: 'claim-at-open' }); + const h1 = openPr(fx, '1', { file: 'VERSION', text: '1.2.4.0\n', changelog: cl => cl.replace('# Changelog\n', '# Changelog\n\n## [1.2.4.0] - 2026-10-18\n\n- #1 change\n') }); + readyReceipt(fx, '1', h1); + fx.write(d => { d.rules['evals.yml'] = GREEN_GATE; }); + const nonce = nonceOf(queue(fx, ['train', 'build', '--prs', `1@${h1}`, '--urgent']).out); + expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok); + const r = queue(fx, ['verify', '--pr', '1', '--train', nonce], {}, coordinatorClone(fx.repo)); + expect(r.status, r.err).toBe(EXIT.ok); + expect(r.out).toContain('STAMP_VERSION: 1.2.4.0'); + expect(line(r.out, 'MERGE_CONDITION:')).toBe(`MERGE_CONDITION: sha=${h1}`); + }, 300_000); + + test('#3530: an open release-only PR whose stamp-only diff makes main equal its prefix modulo stamps is not reported merged; it takes the pre-merge path', () => { + const fx = make(); + unstampedMain(fx, '- #3527 change'); + const h5 = openPr(fx, '5', { file: 'VERSION', text: '1.2.4.0\n', changelog: cl => cl.replace('## [Unreleased]', '## [1.2.4.0] - 2026-10-18') }); + readyReceipt(fx, '5', h5); + fx.write(d => { d.rules['evals.yml'] = GREEN_GATE; }); + const nonce = nonceOf(queue(fx, ['train', 'build', '--prs', `5@${h5}`, '--urgent']).out); + expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok); + const r = queue(fx, ['verify', '--pr', '5', '--train', nonce], {}, coordinatorClone(fx.repo)); + expect(r.out).not.toMatch(/^VERIFY_TRAIN: #5 merged/m); + expect(r.status, r.err).toBe(EXIT.ok); + expect(r.out).toContain(`VERIFY_TRAIN: #5 may merge as slot 1 of train ${nonce}`); + expect(line(r.out, 'MERGE_CONDITION:')).toBe(`MERGE_CONDITION: sha=${h5}`); + expect(trainJson(fx, nonce).merged).toEqual([]); + }, 300_000); + + test('no false freeze: main at the next prefix while GitHub reports the PR OPEN, or cannot be read, refuses QUEUE_PROOF_MISSING and never freezes; MERGED confirms', () => { + const fx = make(); + const { h1, nonce } = twoPrTrain(fx); + expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok); + const t = trainJson(fx, nonce); + const coord = coordinatorClone(fx.repo); + const v = (pr: string) => queue(fx, ['verify', '--pr', pr, '--train', nonce], {}, coord); + laneStamp(fx, '1', h1); + expect(v('1').status).toBe(EXIT.ok); + publishMain(fx, t.prefixes[1].commit, '1.2.4.0'); + markMerged(fx, '1'); + expect(v('1').status).toBe(EXIT.ok); + publishMain(fx, t.prefixes[2].commit, '1.2.5.0'); + + const open = v('2'); + expect(open.status).toBe(EXIT.refused); + expect(open.err).toMatch(/GitHub reports #2 as OPEN, not MERGED.*\(QUEUE_PROOF_MISSING\)/); + expect(open.out).not.toMatch(/^VERIFY_TRAIN: #2 merged/m); + expect(open.out).not.toMatch(/^QUEUE_TRAIN_FROZEN:/m); + expect(next(open.out)).toContain('rerun once GitHub reports #2 MERGED'); + const saved = fx.read().prs['2']; + fx.write(d => { delete d.prs['2']; }); + const unread = v('2'); + expect(unread.err).toContain('(QUEUE_PROOF_MISSING)'); + expect(unread.out).not.toMatch(/^QUEUE_TRAIN_FROZEN:/m); + expect(fs.existsSync(freezeFile(fx))).toBe(false); + fx.write(d => { d.prs['2'] = { ...(saved as object), state: 'MERGED' }; }); + const done = v('2'); + expect(done.status, done.err).toBe(EXIT.ok); + expect(done.out).toContain(`VERIFY_TRAIN: #2 merged; main's tree is T2 of train ${nonce} (2/2 merged)`); + }, 300_000); +});