mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-05 16:34:00 +01:00
fix(ship-queue): under merge: direct, land --pin --repair admits one repair while frozen and a green backstop on its main clears the freeze (GST-1354) (#4001)
* fix(ship-queue): under merge: direct, land --pin --repair admits one repair while frozen and a green backstop on its main clears the freeze (GST-1354) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(ship-queue): direct freeze review fixes: unjudged gates keep the repair slot, a repair merged at another head gives way, leftover train freezes, notes re-read after red cases (GST-1354) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(ship-queue): a standing direct freeze names the backstop rerun that gates main again; pin the leftover train freeze green clear (GST-1354) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(ship-queue): backstop freeze rewrites by compare-and-swap, an active freeze always gates, land repairs only backstop freezes and never a release (GST-1354) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test(ship-queue): the backstop helper clears GITHUB_RUN_ID; rewriteFreeze fails on a local notes write failure (GST-1354) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore: name the changelog fragment after the branch (GST-1354) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(ship-queue): the backstop takes its run id from the CLI, not the environment, so its tests hold under Actions (GST-1354) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
Garry Tanandcapy-ai[bot] authored and GitHub committed
1 parent
b244e6d11a
commit
4debdba226
10 files changed
+469
-19
No files matched your search
@@ -14,7 +14,7 @@
|
||||
* gstack-ship-queue gate-ahead --list --pr <n>
|
||||
* gstack-ship-queue verify --pr <n> --screen (train policy: the head's screen verdict from its run)
|
||||
* gstack-ship-queue verify --post-merge <sha> [--timeout <s>] (after a merge: main's push workflows at that commit until they settle)
|
||||
* gstack-ship-queue land --pr <n> --pin <sha> [--dry-run] [--allow-repo-commands] [--break-lock] (merge: direct; lib/ship-queue-land-direct.ts)
|
||||
* gstack-ship-queue land --pr <n> --pin <sha> [--repair] [--dry-run] [--allow-repo-commands] [--break-lock] (merge: direct; lib/ship-queue-land-direct.ts)
|
||||
* gstack-ship-queue land --pr <n> [--dry-run] [--no-wait] [--break-lock [--break-train-lock]] (the merge step; lib/ship-queue-land.ts)
|
||||
* gstack-ship-queue land --pr <n> --repair (after QUEUE_LAND_DRIFT: open the outside merge's release PR)
|
||||
* gstack-ship-queue land --census
|
||||
@@ -58,11 +58,12 @@ const HELP = `gstack-ship-queue — the shared merge queue's protocol steps (doc
|
||||
verify --pr <n> --screen (train policy) the PR head's screen verdict read from its run (SCREEN_VERDICT)
|
||||
verify --post-merge <sha> [--timeout <s>] wait for every check run at a merged commit to settle: POST_MERGE: green|red|pending
|
||||
and one CHECK: line per workflow; resumable, default timeout 2700 s
|
||||
land --pr <n> --pin <sha> [--dry-run] [--allow-repo-commands] [--break-lock]
|
||||
land --pr <n> --pin <sha> [--repair] [--dry-run] [--allow-repo-commands] [--break-lock]
|
||||
merge: direct: merge the PR at its own green head (receipt and READY at the pin, no stamp
|
||||
files, checks green, mergeable clean), verify the tree, write a land note; a stale base
|
||||
runs land.merged_checks, repo commands, only with --allow-repo-commands (never --dry-run);
|
||||
it holds the land lock from the re-fetch to the merge (--break-lock: a stale one)
|
||||
it holds the land lock from the re-fetch to the merge (--break-lock: a stale one);
|
||||
--repair: while frozen, land the one repair PR the freeze admits (recorded under the lock)
|
||||
land --pr <n> [--dry-run] [--no-wait] [--timeout <min>] [--break-lock [--break-train-lock]] [--allow-repo-commands]
|
||||
the merge step: lock, merge the base, stamp (fragments folded), push the stamp commit,
|
||||
wait for its checks, squash with --match-head-commit, confirm the base tree; resumable
|
||||
@@ -182,7 +183,7 @@ async function main(argv: string[]): Promise<number> {
|
||||
if (args.includes('--pin')) {
|
||||
const pin = flag(args, '--pin') ?? '';
|
||||
if (!/^[0-9a-f]{40,64}$/.test(pin)) return usage('--pin needs the full head sha the READY names');
|
||||
return finish('land', (await import('../lib/ship-queue-land-direct')).landDirect(ctx, { pr, pin, dryRun: args.includes('--dry-run'), allowRepoCommands: args.includes('--allow-repo-commands'), breakLock: args.includes('--break-lock') }));
|
||||
return finish('land', (await import('../lib/ship-queue-land-direct')).landDirect(ctx, { pr, pin, dryRun: args.includes('--dry-run'), allowRepoCommands: args.includes('--allow-repo-commands'), breakLock: args.includes('--break-lock'), repair: args.includes('--repair') }));
|
||||
}
|
||||
const timeout = flag(args, '--timeout');
|
||||
return finish('land', l.land(ctx, { pr, repair: args.includes('--repair'), dryRun: args.includes('--dry-run'), noWait: args.includes('--no-wait'), breakLock: args.includes('--break-lock'), breakTrainLock: args.includes('--break-train-lock'), allowRepoCommands: args.includes('--allow-repo-commands'), waitTimeoutMin: timeout && /^\d+$/.test(timeout) ? Number(timeout) : undefined }));
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
### Fixed
|
||||
|
||||
- **Under `merge: direct`, a red main backstop no longer freezes the queue for good.** The backstop wrote a freeze that `land --pin` and `release-cut` refused under, but the only code that cleared a freeze was a repair train's verify, and `train build --repair` is off when trains are off. A later green backstop didn't clear it either, so the freeze from run 41817566885 (#3992) blocked every land and the release cut. Now `gstack-ship-queue land --pr <n> --pin <sha> --repair` lands exactly one repair PR while frozen: it runs every normal pin check, records the repair on the freeze under the land lock, and leaves the freeze active. The main backstop clears the freeze only when its gate is green on a main at or after the freeze's main that contains that repair's merge, and it writes `cleared_by: "repair #<n>; backstop <run> green on <sha>"`. A red gate keeps the freeze and rewrites its reason; when that main contains the repair, the slot opens for the next repair. A land without `--repair` stays refused while frozen, and nothing clears a freeze without a green gate.
|
||||
+53
-2
@@ -1001,7 +1001,8 @@ unstamped pin. Anything else is
|
||||
`QUEUE_TRAIN_TREE_MISMATCH` (or `QUEUE_TRAIN_MOVED`). When nothing of the train
|
||||
has merged, that is a plain refusal, and a new train is built on current main. When
|
||||
earlier slots already merged, main holds a prefix no gate covered on its own,
|
||||
so verify freezes the queue. The freeze goes to local state and the notes ref
|
||||
so verify freezes the queue (under `merge: direct`, where only the main
|
||||
backstop freezes, see [direct freeze](#direct-freeze)). The freeze goes to local state and the notes ref
|
||||
first, then the full gate is dispatched on main's SHA at once (the backstop),
|
||||
and the `QUEUE_TRAIN_FROZEN:` line lists the merged PRs, main's tree and the
|
||||
backstop run. A slot GitHub reports MERGED counts as merged in that record,
|
||||
@@ -2025,7 +2026,7 @@ release-file rows):
|
||||
| the PR head is the pin | [`QUEUE_LAND_PIN_MOVED`](troubleshooting.md#queue-land-pin-moved) |
|
||||
| the PR body's /ship receipt is bound to the pin | `QUEUE_RECEIPT_MISSING`, `QUEUE_RECEIPT_STALE` |
|
||||
| a READY note from `ready` exists at the pin | `QUEUE_NOT_READY` |
|
||||
| no freeze is active and no land lock is held | `QUEUE_TRAIN_FROZEN`, `QUEUE_LAND_LOCKED` |
|
||||
| no freeze is active (with `--repair`: one is, and its repair slot is free or already this pin; see [direct freeze](#direct-freeze)) and no land lock is held | [`QUEUE_TRAIN_FROZEN`](troubleshooting.md#queue-train-frozen), `QUEUE_NOT_READY`, `QUEUE_LAND_LOCKED` |
|
||||
| VERSION, the `package.json` version, the CHANGELOG and `.gstack/version-path` are unchanged since the merge base | [`QUEUE_LAND_STAMPED`](troubleshooting.md#queue-land-stamped) |
|
||||
| every check at the pin is green with none pending, and each `changelog_fragments.required_checks` entry passed there | [`QUEUE_LAND_CHECKS`](troubleshooting.md#queue-land-checks) |
|
||||
| GitHub reports the PR `MERGEABLE` and `CLEAN` | [`QUEUE_LAND_NOT_CLEAN`](troubleshooting.md#queue-land-not-clean) |
|
||||
@@ -2079,6 +2080,56 @@ in place of the merged checks, and merges nothing; on a PR that already
|
||||
merged it verifies nothing. Under `merge: direct`, `train build` refuses and
|
||||
names this command.
|
||||
|
||||
<a id="direct-freeze"></a>
|
||||
#### The freeze under `merge: direct`: `land --pin --repair`
|
||||
|
||||
While main's policy keeps a `train` block, the main backstop gates every push
|
||||
to main, and a red gate writes a freeze record (`train backstop`) to
|
||||
`refs/notes/gstack-trains`. `land --pin` and `release-cut` refuse while it is
|
||||
active (`QUEUE_TRAIN_FROZEN`), and `train build --repair` is off with the
|
||||
trains. So the repair goes through land:
|
||||
|
||||
1. `gstack-ship-queue land --pr <n> --pin <sha> --repair` admits exactly one PR.
|
||||
It runs every check in the table above. Then, under the land lock and after
|
||||
fetching the notes ref, it records `repair: { pr, head, train: null }` on the
|
||||
freeze, prints `REPAIR: #<n>@<sha> admitted under the land lock`, and lands
|
||||
the PR. The freeze stays active. Without an active freeze, `--repair`
|
||||
refuses with `QUEUE_NOT_READY`; a land without `--repair` stays refused.
|
||||
A train's freeze and a `release/<version>` head are refused with
|
||||
`QUEUE_TRAIN_FROZEN`: only a backstop freeze takes a land repair, and a
|
||||
release never repairs main.
|
||||
2. A second `--repair` at another pin is refused while the admitted repair
|
||||
has merged on GitHub at its recorded pin (or GitHub can't say). One that
|
||||
never merged there, such as a land refused after the record was written or
|
||||
a PR merged by hand at another head, gives way to the next. A repair
|
||||
train's slot (`train` set) is never taken.
|
||||
3. The backstop on the repair's main clears the freeze only on evidence: its
|
||||
gate is green on a commit at or after the freeze's main (by ancestry) that
|
||||
contains the merge the repair's `land` note names, and main is still that
|
||||
commit. While a freeze is active, a green-train record never stands in for
|
||||
the gate. It writes `cleared_at`
|
||||
and `cleared_by: "repair #<n>; backstop <run> green on <sha>"` and prints
|
||||
`BACKSTOP: freeze cleared (…)`. A green gate without that evidence prints
|
||||
[`BACKSTOP: green, but the freeze stands: <why>`](troubleshooting.md#backstop-freeze-stands).
|
||||
No green gate, no clear.
|
||||
4. A red gate keeps the freeze and rewrites its reason with the new run
|
||||
([`BACKSTOP: red, and the queue is already frozen`](troubleshooting.md#backstop-red-frozen)).
|
||||
When that main contains the admitted repair's merge and the gate concluded
|
||||
`failure`, the repair is spent: the slot is freed for the next `--repair`. A
|
||||
gate that never judged it (no run, cancelled, past the deadline) keeps the
|
||||
slot and says so in the reason. Main doesn't move while frozen, so both
|
||||
lines name `gh run rerun <backstop run id>` to gate the same commit again
|
||||
once the cause is fixed. Both steps rewrite the freeze by compare-and-swap
|
||||
after the gate finishes: they start from origin's notes, push without
|
||||
merging, and start over when the push is rejected. So a repair recorded
|
||||
while a gate ran is never overwritten, and a freeze the notes fetch can't
|
||||
read is never rewritten (`failed`).
|
||||
|
||||
A tooling fix merged by hand while frozen (`gh pr merge --squash
|
||||
--match-head-commit`) clears nothing: its backstop, green or red, finds no
|
||||
admitted repair. Train freezes keep their own path: `train build --repair`,
|
||||
cleared by that train's verify.
|
||||
|
||||
<a id="release-stamp"></a>
|
||||
### Versions at release: the release PR (`merge: direct`)
|
||||
|
||||
|
||||
@@ -5638,6 +5638,51 @@ trains and other trains' merges are refused until a repair lands.
|
||||
<n>@<sha>`). The freeze clears when verify finds main's tree after that merge
|
||||
in a green record; `NEXT:` then builds a train of the PRs still unmerged.
|
||||
|
||||
Under `merge: direct` trains are off, and a red main backstop (`train
|
||||
backstop` in the line) freezes `land --pin` the same way. Land the one repair
|
||||
with `gstack-ship-queue land --pr <n> --pin <sha> --repair`. It runs every
|
||||
normal pin check, then records the repair on the freeze under the land lock
|
||||
(`REPAIR: #<n>@<sha> admitted under the land lock`) and lands it; the freeze
|
||||
stays active. The next main backstop clears it when its gate is green on that
|
||||
main (`BACKSTOP: freeze cleared (repair #<n>; backstop <run> green on <sha>)`).
|
||||
A second `--repair` is refused while the admitted one has merged at its
|
||||
recorded pin and no backstop has judged it; one that never merged there gives
|
||||
way. See
|
||||
[direct freeze](ship-policy.md#direct-freeze).
|
||||
|
||||
<a id="backstop-freeze-stands"></a>
|
||||
### `BACKSTOP: green, but the freeze stands`
|
||||
|
||||
**Meaning.** The main backstop gated main green, but the active freeze
|
||||
lacks the evidence to clear: no repair was admitted, the admitted repair has no
|
||||
`land` note (it was merged outside `land --pin --repair`), main doesn't contain
|
||||
that merge, main isn't at or after the freeze's main, or main moved past the
|
||||
gated commit (that push's backstop judges it). A train's freeze clears through
|
||||
its repair train's verify instead.
|
||||
|
||||
**Fix.** Land the repair with `gstack-ship-queue land --pr <n> --pin <sha>
|
||||
--repair`. A repair that merged at its recorded pin but has no `land` note (the
|
||||
post-merge tree check could not finish) gets the note when you rerun that
|
||||
command, which goes straight to the tree check. Main doesn't move while
|
||||
frozen, and the backstop runs only on a push, so then gate the same commit
|
||||
again with the command the line names: `gh run rerun <backstop run id> -R
|
||||
<owner/repo>`. Nothing else clears the freeze.
|
||||
|
||||
<a id="backstop-red-frozen"></a>
|
||||
### `BACKSTOP: red, and the queue is already frozen`
|
||||
|
||||
**Meaning.** A red gate on main never writes a second freeze. On a backstop
|
||||
freeze it rewrites the reason with this red run and keeps `frozen since main
|
||||
<sha>`. When main contains the admitted repair's merge and the gate concluded
|
||||
`failure`, the reason adds `repair #<n> did not turn main green` and the repair
|
||||
slot is freed. A gate that never judged it (no run, cancelled, past the
|
||||
deadline) adds `repair #<n> was not judged` and keeps the slot: gate that main
|
||||
again with `gh run rerun <backstop run id> -R <owner/repo>`, as the reason
|
||||
says.
|
||||
|
||||
**Fix.** Land the next repair (revert or fix-forward) with `land --pin
|
||||
--repair`.
|
||||
|
||||
<a id="queue-wave-conflict"></a>
|
||||
### `QUEUE_WAVE_CONFLICT`
|
||||
|
||||
|
||||
@@ -179,8 +179,8 @@ export const RELEASE_RESULT_CODES = {
|
||||
},
|
||||
QUEUE_TRAIN_FROZEN: {
|
||||
anchor: 'queue-train-frozen',
|
||||
summary: 'a train stopped partway through publication, so main holds a prefix no gate covered on its own; ordinary trains are refused until a repair lands',
|
||||
fix: 'admit one repair PR at its exact head (train build --repair --prs <n>@<sha>); the freeze clears when verify finds main’s tree after that merge in a green record',
|
||||
summary: 'a train stopped partway through publication, or the main backstop was red, so main holds a tree no gate passed; ordinary trains and lands are refused until a repair lands',
|
||||
fix: 'admit one repair PR at its exact head (train build --repair --prs <n>@<sha>; under merge: direct, land --pr <n> --pin <sha> --repair); the freeze clears when verify finds main’s tree after that merge in a green record, or, under merge: direct, when the main backstop is green on a main that contains it',
|
||||
},
|
||||
// Wave PRs (Garry R46, GST-574; docs/ship-policy.md#wave-prs): 3-6 same-area PRs in one train slot.
|
||||
QUEUE_WAVE_CONFLICT: {
|
||||
|
||||
@@ -21,11 +21,11 @@ import { ciAtHead, loadBasePolicy, type QueueContext, type StepOutput } from './
|
||||
import { git, type QueueFailure } from './ship-queue-gate';
|
||||
import { acquireLandLock, lockLabel, ownsLock, readLock, type HeldLandLock, type LandLock } from './ship-queue-land-lock';
|
||||
import * as os from 'node:os';
|
||||
import { activeFreeze, loadReady, persist, rebuildFromNotes, repoIdentity } from './ship-queue-train-record';
|
||||
import { activeFreeze, loadReady, persist, rebuildFromNotes, repoIdentity, type FreezeRecord } from './ship-queue-train-record';
|
||||
import { bindShippedReceipt } from './ship-receipt-depth';
|
||||
|
||||
interface DirectPr { state: string; baseRefName?: string; headRefName: string; headRefOid: string; isCrossRepository: boolean; body: string; mergeable?: string; mergeStateStatus?: string; mergeCommit?: { oid: string } | null }
|
||||
export interface DirectOptions { pr: string; pin: string; dryRun?: boolean; allowRepoCommands?: boolean; breakLock?: boolean; now?: () => Date }
|
||||
export interface DirectOptions { pr: string; pin: string; dryRun?: boolean; allowRepoCommands?: boolean; breakLock?: boolean; repair?: boolean; now?: () => Date }
|
||||
|
||||
const isSha = (s: string | undefined) => /^[0-9a-f]{40,64}$/.test(s ?? '');
|
||||
|
||||
@@ -34,6 +34,24 @@ function view(ctx: QueueContext, pr: string): DirectPr | QueueFailure {
|
||||
try { return JSON.parse(r.out) as DirectPr; } catch { return { code: 'QUEUE_NOT_READY', message: `gh pr view ${pr} failed: ${(r.err ?? r.out).trim().slice(0, 160)}` }; }
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether `land --pin --repair` may record #pr@pin as the freeze's one repair (docs/ship-policy.md#direct-freeze): null
|
||||
* admits. A repair already recorded at another pin gives way once GitHub reports it did not merge at that pin; one that merged there
|
||||
* holds the slot until the main backstop clears the freeze or a red backstop on a main containing it frees the slot.
|
||||
*/
|
||||
function repairRefusal(ctx: QueueContext, freeze: FreezeRecord | null, pr: string, pin: string): QueueFailure | null {
|
||||
if (!freeze) return { code: 'QUEUE_NOT_READY', message: 'no freeze is active; --repair lands only the one repair PR a freeze admits', fix: `land it without --repair: gstack-ship-queue land --pr ${pr} --pin ${pin}` };
|
||||
if (freeze.train !== 'backstop') return { code: 'QUEUE_TRAIN_FROZEN', message: `train ${freeze.train} froze the queue (${freeze.reason}); land --pin --repair admits a repair only into a main backstop freeze`, fix: 'land the repair through train build --repair under merge: train; its verify clears the freeze' };
|
||||
const held = freeze.repair;
|
||||
if (!held || (held.pr === pr && held.head === pin && !held.train)) return null;
|
||||
const admitted = `the freeze admitted repair #${held.pr}@${held.head.slice(0, 12)}`;
|
||||
if (held.train) return { code: 'QUEUE_TRAIN_FROZEN', message: `repair train ${held.train} holds the freeze (${freeze.reason})`, fix: 'land the repair through that train; its verify clears the freeze' };
|
||||
const other = view(ctx, held.pr);
|
||||
if ('code' in other) return { code: 'QUEUE_TRAIN_FROZEN', message: `${admitted}, and whether it merged cannot be read: ${other.message}`, fix: `rerun land once gh pr view ${held.pr} answers` };
|
||||
if (other.state === 'MERGED' && other.headRefOid === held.head) return { code: 'QUEUE_TRAIN_FROZEN', message: `${admitted}, which merged; the queue stays frozen until the main backstop is green on a main that contains it, and a red backstop there frees the slot for the next repair`, fix: 'wait for the main backstop run on that merge (docs/ship-policy.md#direct-freeze)' };
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Release files the pin changed since its merge base: the version file, the manifest version and the CHANGELOG; an unreadable version is an error, never a pass. */
|
||||
export function stampedFiles(root: string, policy: ShipPolicy, mergeBase: string, pin: string): string[] | { error: string } {
|
||||
const out: string[] = [];
|
||||
@@ -145,7 +163,9 @@ export function landDirect(ctx: QueueContext, o: DirectOptions): StepOutput {
|
||||
if (!loadReady(keys, o.pr, o.pin)) return fail(lines, { code: 'QUEUE_NOT_READY', message: `no READY note for #${o.pr} at the pin ${o.pin.slice(0, 12)}`, fix: `the lane runs gstack-ship-queue ready --pr ${o.pr} --no-screen at the pin` });
|
||||
}
|
||||
const freeze = activeFreeze(keys);
|
||||
if (freeze) return fail(lines, { code: 'QUEUE_TRAIN_FROZEN', message: `main is frozen by train ${freeze.train} (${freeze.reason})` });
|
||||
if (freeze && !o.repair) return fail(lines, { code: 'QUEUE_TRAIN_FROZEN', message: `main is frozen by train ${freeze.train} (${freeze.reason})`, fix: 'only the one repair PR lands: gstack-ship-queue land --pr <repair-pr> --pin <sha> --repair; the freeze clears when the main backstop is green on a main that contains it (docs/ship-policy.md#direct-freeze)' });
|
||||
const refused = !o.repair ? null : release ? { code: 'QUEUE_TRAIN_FROZEN' as const, message: `${info.headRefName} is a release PR, which never repairs main`, fix: 'land the revert or fix-forward with --repair; release once the backstop clears the freeze' } : repairRefusal(ctx, freeze, o.pr, o.pin);
|
||||
if (refused) return fail(lines, refused);
|
||||
const lock = o.dryRun ? readLock(root, remote) : null;
|
||||
if (lock?.state === 'held') return fail(lines, { code: 'QUEUE_LAND_LOCKED', message: `${lockLabel(lock.lock)} holds the land lock; a direct land would race it onto main` });
|
||||
|
||||
@@ -194,6 +214,19 @@ export function landDirect(ctx: QueueContext, o: DirectOptions): StepOutput {
|
||||
}
|
||||
}
|
||||
if (o.dryRun) return { ok: true, run, lines: [...lines, `DRY-RUN: nothing merged (would run gh pr merge ${o.pr} --squash --match-head-commit ${o.pin})`] };
|
||||
if (o.repair) {
|
||||
if (!rebuildFromNotes(root, remote).fetched) return fail(lines, { code: 'QUEUE_NOT_READY', message: `cannot fetch refs/notes/gstack-trains under the land lock, so the freeze's repair slot cannot be read; nothing merged`, fix: 'fix the fetch, then rerun land' });
|
||||
const frozen = activeFreeze(keys);
|
||||
const taken = repairRefusal(ctx, frozen, o.pr, o.pin);
|
||||
if (taken) return fail(lines, taken);
|
||||
if (frozen!.repair?.pr !== o.pr || frozen!.repair.head !== o.pin) {
|
||||
frozen!.repair = { pr: o.pr, head: o.pin, train: null };
|
||||
frozen!.updated_at = now().toISOString();
|
||||
const saved = persist(root, [frozen!], { remote });
|
||||
if (!saved.pushed) return fail(lines, { code: 'QUEUE_NOT_READY', message: `cannot record #${o.pr} as the freeze's repair: ${saved.detail}; nothing merged`, fix: 'fix the notes push, then rerun land' });
|
||||
}
|
||||
lines.push(`REPAIR: #${o.pr}@${o.pin.slice(0, 12)} admitted under the land lock; the freeze stays active until the main backstop is green on a main that contains this merge`);
|
||||
}
|
||||
|
||||
const method = policy.history === 'merge-only' ? '--merge' : '--squash';
|
||||
const lost = held!.set('merging');
|
||||
|
||||
@@ -16,7 +16,7 @@ import { resolveStateRoot } from './state-root';
|
||||
|
||||
export const TRAIN_SCHEMA = 1;
|
||||
export const NOTES_REF = 'refs/notes/gstack-trains';
|
||||
const NOTES_REMOTE_REF = 'refs/notes/gstack-trains-origin';
|
||||
export const NOTES_REMOTE_REF = 'refs/notes/gstack-trains-origin';
|
||||
|
||||
export type TrainState =
|
||||
| 'built' | 'pushed' | 'free-pending' | 'free-red' | 'free-green' | 'dispatch-unknown' | 'dispatch-absent' | 'gate-pending'
|
||||
|
||||
@@ -22,7 +22,7 @@ import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { loadPolicyFromHead, type TrainPolicy } from '../lib/ship-policy';
|
||||
import { NOTES_REF, fetchNotes, nowIso, persist, repoIdentity, type FreezeRecord } from '../lib/ship-queue-train-record';
|
||||
import { NOTES_REF, NOTES_REMOTE_REF, fetchNotes, nowIso, persist, repoIdentity, type FreezeRecord } from '../lib/ship-queue-train-record';
|
||||
import { E2E_TOUCHFILES, LLM_JUDGE_TOUCHFILES } from '../test/helpers/touchfiles-data';
|
||||
import { DISPATCH_INPUTS, makeNonce, nonceScope } from './eval-bg';
|
||||
import { ghFailureReason, runGh, type GhRunner } from './recover-receipts';
|
||||
@@ -36,6 +36,8 @@ export interface BackstopOptions {
|
||||
repo: string; branch: string; sha: string; repoRoot: string;
|
||||
gh?: GhRunner; git?: Git; now?: () => number; sleep?: (ms: number) => void;
|
||||
settleMs?: number; pollMs?: number; gateDeadlineMs?: number; random?: string;
|
||||
/** This workflow run's id (GITHUB_RUN_ID), named in the rerun command a standing freeze prints. */
|
||||
runId?: string;
|
||||
}
|
||||
interface Run { id: number; status: string; conclusion: string | null; head_sha: string; display_title?: string; html_url?: string }
|
||||
|
||||
@@ -154,7 +156,9 @@ export function runBackstop(o: BackstopOptions): { action: BackstopAction; lines
|
||||
const digest = stampFreeTree(git, o.sha, o.repoRoot, loaded.policy);
|
||||
if (typeof digest !== 'string') { lines.push(`BACKSTOP: no tree modulo stamps (${digest.error}); gating main`); }
|
||||
const tree = git(['rev-parse', `${o.sha}^{tree}`]).out;
|
||||
const covered = coveringRecord(gh, o.repo, train, { digest: typeof digest === 'string' ? digest : null, tree }, lines);
|
||||
const frozen = noteRecords<FreezeRecord>(git, 'freeze', repoKey).some(f => !f.cleared_at);
|
||||
if (frozen) lines.push('BACKSTOP: the queue is frozen, so only a gate on main can clear it; no green-train record stands in');
|
||||
const covered = frozen ? null : coveringRecord(gh, o.repo, train, { digest: typeof digest === 'string' ? digest : null, tree }, lines);
|
||||
if (covered) {
|
||||
lines.push(`BACKSTOP: covered: main ${o.sha.slice(0, 12)} is ${covered.exact ? `S${covered.index}` : `T${covered.index}`} of green train ${covered.record.nonce} (run ${covered.run.id}, ${covered.exact ? `exact tree ${tree.slice(0, 12)}` : `tree modulo stamps ${String(digest).slice(0, 12)}`}); no gate`);
|
||||
return done('covered');
|
||||
@@ -167,17 +171,90 @@ export function runBackstop(o: BackstopOptions): { action: BackstopAction; lines
|
||||
lines.push(`BACKSTOP: no green-train record lists main's tree modulo stamps${typeof digest === 'string' ? ` ${digest.slice(0, 12)}` : ''}; gating main`);
|
||||
const nonce = makeNonce('gate', o.sha, o.sha, o.random ?? randomBytes(3).toString('hex'), { purpose: 'backstop' });
|
||||
const run = gateMain({ repo: o.repo, branch: o.branch, sha: o.sha, gh, train, now, sleep, pollMs, deadline: now() + (o.gateDeadlineMs ?? 270 * 60_000), nonce }, lines);
|
||||
if (run?.status === 'completed' && run.conclusion === 'success') { lines.push(`BACKSTOP: gated: run ${run.id} is green on main ${o.sha.slice(0, 12)}`); return done('gated'); }
|
||||
if (run?.status === 'completed' && run.conclusion === 'success') { lines.push(`BACKSTOP: gated: run ${run.id} is green on main ${o.sha.slice(0, 12)}`); return clearOnGreen(o, { git, repoKey, run }, lines); }
|
||||
return freeze(o, { gh, git, repoKey, nonce, run }, lines);
|
||||
}
|
||||
|
||||
/** Write the freeze record (local state, then refs/notes/gstack-trains) unless a freeze is already active. */
|
||||
/** Main does not move while frozen, so rerunning this backstop run gates the same commit again (the workflow has no other trigger than a push). */
|
||||
const rerunCommand = (o: BackstopOptions) => `gh run rerun ${o.runId ?? '<this backstop run>'} -R ${o.repo}`;
|
||||
|
||||
/** The land note of a merge: direct freeze's admitted repair (gstack-ship-queue land --pin --repair), when main contains its merge. */
|
||||
function repairIn(o: BackstopOptions, git: Git, repoKey: string, f: FreezeRecord): { merge_sha: string } | string {
|
||||
const r = f.repair;
|
||||
if (!r) return 'no repair was admitted (gstack-ship-queue land --pr <repair-pr> --pin <sha> --repair)';
|
||||
if (r.train) return `repair train ${r.train} holds it, and that train's verify clears it`;
|
||||
const landed = noteRecords<{ kind: string; repo_key: string; pr: string; pin: string; merge_sha: string }>(git, 'land', repoKey).find(l => l.pr === r.pr && l.pin === r.head);
|
||||
if (!landed || !SHA.test(landed.merge_sha ?? '')) return `repair #${r.pr} has no land note at ${r.head.slice(0, 12)}, so land never verified its merge`;
|
||||
if (!git(['merge-base', '--is-ancestor', landed.merge_sha, o.sha]).ok) return `main ${o.sha.slice(0, 12)} does not contain repair #${r.pr}'s merge ${landed.merge_sha.slice(0, 12)}`;
|
||||
return landed;
|
||||
}
|
||||
|
||||
/** One compare-and-swap rewrite of the active freeze: none active, `change` declined (null), pushed, or failed. */
|
||||
type Rewrite = { state: 'none' } | { state: 'unchanged' | 'pushed'; record: FreezeRecord } | { state: 'failed'; detail: string };
|
||||
|
||||
/**
|
||||
* Rewrite the active freeze on origin with a compare-and-swap: start from origin's notes (unpushed local notes are
|
||||
* dropped), apply `change`, and push without merging. A rejected push starts over from origin's new notes, so a repair
|
||||
* that `land --pin --repair` recorded meanwhile is never overwritten by this run's stale copy.
|
||||
*/
|
||||
function rewriteFreeze(git: Git, repoRoot: string, repoKey: string, change: (f: FreezeRecord) => FreezeRecord | null): Rewrite {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const f = git(['fetch', '--no-tags', '-q', 'origin', `+${NOTES_REF}:${NOTES_REMOTE_REF}`]);
|
||||
if (!f.ok) return /couldn't find remote ref|no such ref/i.test(f.out) ? { state: 'none' } : { state: 'failed', detail: `git fetch ${NOTES_REF} failed: ${f.out.slice(0, 160)}` };
|
||||
if (!git(['update-ref', NOTES_REF, NOTES_REMOTE_REF]).ok) return { state: 'failed', detail: `cannot reset ${NOTES_REF} to origin's notes` };
|
||||
const active = noteRecords<FreezeRecord>(git, 'freeze', repoKey).find(r => !r.cleared_at);
|
||||
if (!active) return { state: 'none' };
|
||||
const next = change(structuredClone(active));
|
||||
if (!next) return { state: 'unchanged', record: active };
|
||||
const written = persist(repoRoot, [next], { push: false });
|
||||
if (written.detail !== 'notes written locally (not pushed)') return { state: 'failed', detail: written.detail };
|
||||
if (git(['push', '-q', 'origin', `${NOTES_REF}:${NOTES_REF}`]).ok) return { state: 'pushed', record: next };
|
||||
}
|
||||
return { state: 'failed', detail: `origin's ${NOTES_REF} moved under three pushes` };
|
||||
}
|
||||
|
||||
/**
|
||||
* A green gate clears the active backstop freeze only on evidence: main is still this commit, at or after the freeze's
|
||||
* main, and contains the merge of the repair the freeze admitted (docs/ship-policy.md#direct-freeze).
|
||||
*/
|
||||
function clearOnGreen(o: BackstopOptions, c: { git: Git; repoKey: string; run: Run }, lines: string[]): { action: BackstopAction; lines: string[] } {
|
||||
const tip = c.git(['ls-remote', 'origin', `refs/heads/${o.branch}`]).out.split(/\s/)[0] ?? '';
|
||||
let why = '';
|
||||
const out = rewriteFreeze(c.git, o.repoRoot, c.repoKey, f => {
|
||||
const repair = f.train !== 'backstop' ? `train ${f.train} froze the queue, and its repair train's verify clears it`
|
||||
: tip !== o.sha ? `${o.branch} moved to ${tip.slice(0, 12) || 'an unreadable tip'}; that push's backstop judges it`
|
||||
: !c.git(['merge-base', '--is-ancestor', f.main.sha, o.sha]).ok ? `main ${o.sha.slice(0, 12)} is not at or after the freeze main ${f.main.sha.slice(0, 12)}`
|
||||
: repairIn(o, c.git, c.repoKey, f);
|
||||
if (typeof repair === 'string') { why = repair; return null; }
|
||||
const at = nowIso();
|
||||
return { ...f, cleared_at: at, cleared_by: `repair #${f.repair!.pr}; backstop ${c.run.id} green on ${o.sha}`, updated_at: at };
|
||||
});
|
||||
if (out.state === 'none') return { action: 'gated', lines };
|
||||
if (out.state === 'failed') { lines.push(`BACKSTOP: green, but the freeze stands: ${out.detail}; gate this main again with ${rerunCommand(o)} (docs/troubleshooting.md#backstop-freeze-stands)`); return { action: 'failed', lines }; }
|
||||
if (out.state === 'unchanged') { lines.push(`BACKSTOP: green, but the freeze stands: ${why}; once that is fixed, gate this main again with ${rerunCommand(o)} (docs/troubleshooting.md#backstop-freeze-stands)`); return { action: 'gated', lines }; }
|
||||
lines.push(`BACKSTOP: freeze cleared (${out.record.cleared_by})`);
|
||||
return { action: 'gated', lines };
|
||||
}
|
||||
|
||||
/** Write the freeze record (local state, then refs/notes/gstack-trains); an active backstop freeze is rewritten, never replaced. */
|
||||
function freeze(o: BackstopOptions, c: { gh: GhRunner; git: Git; repoKey: string; nonce: string; run: Run | null }, lines: string[]): { action: BackstopAction; lines: string[] } {
|
||||
const active = noteRecords<FreezeRecord>(c.git, 'freeze', c.repoKey).find(f => !f.cleared_at);
|
||||
if (active) { lines.push(`BACKSTOP: red, and the queue is already frozen (${active.reason}); the existing freeze stands`); return { action: 'frozen', lines }; }
|
||||
const red = c.run?.status === 'completed' ? redCases(c.gh, o.repo, c.run) : [];
|
||||
const runText = c.run ? `run ${c.run.id} (${c.run.status}/${c.run.conclusion ?? '-'})` : 'no run';
|
||||
const cases = red.length ? red.map(r => `${r.id} (inputs ${r.inputs.slice(0, 4).join(', ') || 'none declared'})`).join('; ') : 'no red case recorded';
|
||||
const judged = c.run?.status === 'completed' && c.run.conclusion === 'failure';
|
||||
const out = rewriteFreeze(c.git, o.repoRoot, c.repoKey, f => {
|
||||
if (f.train !== 'backstop') return null;
|
||||
const repair = repairIn(o, c.git, c.repoKey, f);
|
||||
const verdict = typeof repair === 'string' ? '' : judged
|
||||
? `; repair #${f.repair!.pr} did not turn main green, so its admission is spent: land the next repair with land --pin --repair`
|
||||
: `; repair #${f.repair!.pr} was not judged (${runText}), so its admission stands: gate this main again with ${rerunCommand(o)}`;
|
||||
const issue = (f as { issue?: string | null }).issue;
|
||||
return { ...f, reason: `backstop red on main ${o.sha.slice(0, 12)}: ${cases}; ${runText}${verdict}; frozen since main ${f.main.sha.slice(0, 12)}; fix: land the revert or fix-forward${issue ? ` (${issue})` : ''}`,
|
||||
repair: verdict && judged ? null : f.repair, updated_at: nowIso() };
|
||||
});
|
||||
if (out.state === 'failed') { lines.push(`BACKSTOP: red, and the freeze record could not be read or written (${out.detail}); nothing written`); return { action: 'failed', lines }; }
|
||||
if (out.state === 'unchanged') { lines.push(`BACKSTOP: red, and the queue is already frozen (${out.record.reason}); the existing freeze stands`); return { action: 'frozen', lines }; }
|
||||
if (out.state === 'pushed') { lines.push(`BACKSTOP: red, and the queue is already frozen; the freeze stands with its reason updated: ${out.record.reason}`); return { action: 'frozen', lines }; }
|
||||
const body = [`The main backstop gate is not green on ${o.sha}: ${runText}.`, '', `Red cases: ${cases}.`, '', 'gstack-ship-queue refuses ordinary trains until a repair train merges (docs/ship-policy.md#run-a-merge-train). Land the revert or fix-forward.'].join('\n');
|
||||
const issue = c.gh(['issue', 'create', '-R', o.repo, '--title', `Main backstop red on ${o.sha.slice(0, 12)}`, '--body', body], GH_MS);
|
||||
const issueUrl = issue.status === 0 ? issue.stdout.trim().split('\n').at(-1) ?? null : null;
|
||||
@@ -206,7 +283,7 @@ if (import.meta.main) {
|
||||
const minutes = (name: string, fallback: number) => Number(flag(name) ?? fallback) * 60_000;
|
||||
const [repo, branch, sha] = [flag('repo'), flag('branch'), flag('sha')];
|
||||
if (command !== 'run' || !repo || !branch || !sha) { console.error(USAGE); process.exit(2); }
|
||||
const { action, lines } = runBackstop({ repo: repo!, branch: branch!, sha: sha!, repoRoot: process.cwd(), settleMs: minutes('settle-minutes', 60),
|
||||
const { action, lines } = runBackstop({ repo: repo!, branch: branch!, sha: sha!, repoRoot: process.cwd(), runId: process.env.GITHUB_RUN_ID, settleMs: minutes('settle-minutes', 60),
|
||||
pollMs: Number(flag('poll-seconds') ?? 60) * 1000, gateDeadlineMs: minutes('gate-deadline-minutes', 270) });
|
||||
const summary = flag('summary');
|
||||
for (const line of lines) {
|
||||
|
||||
@@ -20,6 +20,11 @@ import { bumpVersion } from '../lib/version-gate-train';
|
||||
import { openRelease } from '../lib/release-stamp';
|
||||
import { runMergedChecks } from '../lib/land-merged-checks';
|
||||
import { readLock, writeLock } from '../lib/ship-queue-land-lock';
|
||||
import { runBackstop } from '../scripts/train-backstop';
|
||||
import { readTrainNotes } from '../scripts/release-cut';
|
||||
import { gitIn } from '../scripts/train-gate';
|
||||
import type { GhRunner } from '../scripts/recover-receipts';
|
||||
import { TRAIN_BLOCK } from './helpers/ship-queue-train-fixture';
|
||||
|
||||
const REQUIRED = ['free-tests', 'Check VERSION is not stale vs queue'];
|
||||
const POLICY = { ...BASE_POLICY, release_tool: null, release_outputs: [], changelog_fragments: { dir: 'changelog.d', required_checks: REQUIRED }, merge: 'direct', min_gstack_version: MERGE_KEY_VERSION };
|
||||
@@ -729,3 +734,225 @@ describe('land --pin: the release PR (docs/ship-policy.md#release-stamp)', () =>
|
||||
expect(remote(fx, 'main')).toBe(moved);
|
||||
}, 120_000);
|
||||
});
|
||||
|
||||
/** A backstop freeze on origin's notes (what scripts/train-backstop.ts writes on a red gate), written from another state root. */
|
||||
function freezeMain(fx: Fx, over: Record<string, unknown> = {}): string {
|
||||
const main = remote(fx, 'main');
|
||||
const prior = process.env.GSTACK_STATE_ROOT;
|
||||
process.env.GSTACK_STATE_ROOT = path.join(fx.repo.root, 'freeze-state');
|
||||
try {
|
||||
const at = new Date().toISOString();
|
||||
persist(fx.repo.work, [{ schema: 1, kind: 'freeze', repo_key: repoIdentity(fx.repo.work).key, train: 'backstop', merged: [], unmerged: [], main: { sha: main, tree: fx.repo.git('rev-parse', main + '^{tree}') },
|
||||
reason: `backstop red on main ${main.slice(0, 12)}: no red case recorded; run 41817566885 (completed/failure)`, backstop: { nonce: 'gate.backstop-x', run: 41817566885, state: 'completed/failure' },
|
||||
repair: null, created_at: at, cleared_at: null, cleared_by: null, updated_at: at, ...over } as any]);
|
||||
} finally { if (prior === undefined) delete process.env.GSTACK_STATE_ROOT; else process.env.GSTACK_STATE_ROOT = prior; }
|
||||
return main;
|
||||
}
|
||||
const originRecords = (fx: Fx, kind: string): any[] => {
|
||||
const notes = (...a: string[]) => fx.repo.git('--git-dir', fx.repo.origin, 'notes', '--ref=refs/notes/gstack-trains', ...a);
|
||||
let list = '';
|
||||
try { list = notes('list'); } catch { return []; }
|
||||
return list.split('\n').filter(Boolean).map(l => JSON.parse(notes('show', l.split(' ')[1]!))).filter(r => r.kind === kind);
|
||||
};
|
||||
const freezeOf = (fx: Fx) => { const all = originRecords(fx, 'freeze'); expect(all).toHaveLength(1); return all[0]; };
|
||||
|
||||
/**
|
||||
* scripts/train-backstop.ts on main's head from its own clone (the CI runner's checkout) with a scripted gh: no green
|
||||
* train covers it, and the gate run concludes as asked; `during` runs while the gate runs (a land meanwhile).
|
||||
*/
|
||||
function backstopOn(fx: Fx, gate: 'success' | 'failure' | 'cancelled', run: number, during?: () => void) {
|
||||
const sha = remote(fx, 'main');
|
||||
const runner = path.join(fx.repo.root, 'runner');
|
||||
const rg = (...a: string[]) => { const r = spawnSync('git', ['-C', runner, ...a], { encoding: 'utf8', timeout: 30_000 }); if (r.status !== 0) throw new Error(`git ${a.join(' ')}: ${r.stderr}`); };
|
||||
if (!fs.existsSync(runner)) { const c = spawnSync('git', ['clone', '-q', fx.repo.origin, runner], { encoding: 'utf8', timeout: 30_000 }); if (c.status !== 0) throw new Error(c.stderr); }
|
||||
rg('fetch', '-q', 'origin');
|
||||
rg('checkout', '-q', '--detach', sha);
|
||||
let dispatched = '';
|
||||
const ok = (body: unknown) => ({ status: 0, stdout: typeof body === 'string' ? body : JSON.stringify(body), stderr: '' });
|
||||
const gh: GhRunner = (args) => {
|
||||
const joined = args.join(' ');
|
||||
if (args[0] === 'api' && joined.includes('/actions/workflows/') && joined.includes('status=success')) return ok({ workflow_runs: [] });
|
||||
if (args[0] === 'workflow' && args[1] === 'run') { dispatched = args.find(a => a.startsWith('nonce='))!.slice('nonce='.length); return ok(''); }
|
||||
if (args[0] === 'api' && joined.includes('/actions/workflows/')) return ok({ workflow_runs: dispatched ? [{ id: run, status: 'queued', conclusion: null, head_sha: sha, display_title: `E2E Evals [eval-bg ${dispatched}]` }] : [] });
|
||||
if (args[0] === 'api' && joined.endsWith(`actions/runs/${run}`)) { during?.(); during = undefined; return ok({ id: run, status: 'completed', conclusion: gate, head_sha: sha, html_url: `https://example.invalid/runs/${run}` }); }
|
||||
if (args[0] === 'run' && args[1] === 'download') { const dir = args[args.indexOf('-D') + 1]!; fs.mkdirSync(dir, { recursive: true }); fs.writeFileSync(path.join(dir, 'trial-outcomes.jsonl'), JSON.stringify({ case: 'red-case', attempt: 1, outcome: 'failed' }) + '\n'); return ok(''); }
|
||||
if (args[0] === 'issue' && args[1] === 'create') return ok('https://github.com/fixture/repo/issues/3992\n');
|
||||
return { status: 1, stdout: '', stderr: `unexpected gh ${joined}` };
|
||||
};
|
||||
const prior = process.env.GSTACK_STATE_ROOT;
|
||||
process.env.GSTACK_STATE_ROOT = path.join(fx.repo.root, 'backstop-state');
|
||||
try { return { sha, ...runBackstop({ repo: 'fixture/repo', branch: 'main', sha, repoRoot: runner, gh, sleep: () => {}, pollMs: 1, random: 'abc123' }) }; }
|
||||
finally { if (prior === undefined) delete process.env.GSTACK_STATE_ROOT; else process.env.GSTACK_STATE_ROOT = prior; }
|
||||
}
|
||||
/** gstack's own shape: merge: direct with the train block still set, so the backstop gates every push to main. */
|
||||
const DIRECT_TRAIN = { ...POLICY, train: TRAIN_BLOCK };
|
||||
|
||||
describe('land --pin --repair: the one way through a backstop freeze under merge: direct (GST-1354)', () => {
|
||||
test('a freeze refuses a plain land and names --repair; --repair with no freeze refuses; nothing merges', () => {
|
||||
const fx = setup();
|
||||
const pin8 = openPr(fx, '8');
|
||||
const early = land(fx, '8', pin8, ['--repair']);
|
||||
expect(early.status).toBe(EXIT.refused);
|
||||
expect(early.err).toContain('no freeze is active');
|
||||
expect(early.err).toContain('(QUEUE_NOT_READY)');
|
||||
freezeMain(fx);
|
||||
const pin7 = openPr(fx, '7');
|
||||
const plain = land(fx, '7', pin7);
|
||||
expect(plain.status).toBe(EXIT.refused);
|
||||
expect(plain.err).toContain('(QUEUE_TRAIN_FROZEN)');
|
||||
expect(plain.err).toContain(`land --pr <repair-pr> --pin <sha> --repair`);
|
||||
expect(fx.read().mergeArgs).toBeUndefined();
|
||||
expect(freezeOf(fx)).toMatchObject({ repair: null, cleared_at: null });
|
||||
}, 90_000);
|
||||
|
||||
test('--repair admits exactly one PR: the normal pin checks run first, the repair is recorded under the land lock, it lands, and the freeze stays active; a second repair and a plain land stay refused', () => {
|
||||
const fx = setup();
|
||||
freezeMain(fx);
|
||||
const pin7 = openPr(fx, '7');
|
||||
const pin8 = openPr(fx, '8');
|
||||
fx.write(d => { d.checks[pin7] = { 'free-tests': 'failure', 'Check VERSION is not stale vs queue': 'success' }; });
|
||||
const red = land(fx, '7', pin7, ['--repair']);
|
||||
expect(red.err).toContain('(QUEUE_LAND_CHECKS)');
|
||||
expect(freezeOf(fx).repair).toBeNull();
|
||||
fx.write(d => { delete d.checks[pin7]; });
|
||||
const r = land(fx, '7', pin7, ['--repair']);
|
||||
expect(r.err).toBe('');
|
||||
expect(r.status).toBe(EXIT.ok);
|
||||
expect(r.out).toContain(`REPAIR: #7@${pin7.slice(0, 12)} admitted under the land lock`);
|
||||
expect(fx.read().mergeArgs).toEqual(['7', '--squash', '--match-head-commit', pin7]);
|
||||
expect(fx.read().lockAtMerge).toMatchObject({ pr: '7', journal: 'land-pin:' + pin7 });
|
||||
expect(freezeOf(fx)).toMatchObject({ train: 'backstop', repair: { pr: '7', head: pin7, train: null }, cleared_at: null, cleared_by: null });
|
||||
expect(landNotes(fx)).toEqual([expect.objectContaining({ pr: '7', pin: pin7, merge_sha: remote(fx, 'main') })]);
|
||||
const second = land(fx, '8', pin8, ['--repair']);
|
||||
expect(second.err).toContain('(QUEUE_TRAIN_FROZEN)');
|
||||
expect(second.err).toContain(`repair #7@${pin7.slice(0, 12)}`);
|
||||
expect(land(fx, '8', pin8).err).toContain('(QUEUE_TRAIN_FROZEN)');
|
||||
expect(fx.read().mergeArgs).toEqual(['7', '--squash', '--match-head-commit', pin7]);
|
||||
}, 120_000);
|
||||
|
||||
test('a recorded repair that never merged at its pin gives way to the next --repair; a repair train holding the freeze does not', () => {
|
||||
const fx = setup();
|
||||
const stale = 'a'.repeat(40);
|
||||
fx.write(d => { d.prs['5'] = { state: 'MERGED', baseRefName: 'main', headRefName: 'pr-5', headRefOid: 'c'.repeat(40), isCrossRepository: false, body: '' }; });
|
||||
freezeMain(fx, { repair: { pr: '5', head: stale, train: null } });
|
||||
const pin7 = openPr(fx, '7');
|
||||
const r = land(fx, '7', pin7, ['--repair']);
|
||||
expect(r.err).toBe('');
|
||||
expect(freezeOf(fx).repair).toEqual({ pr: '7', head: pin7, train: null });
|
||||
const fy = setup();
|
||||
freezeMain(fy, { repair: { pr: '5', head: stale, train: 'gate.train-x' } });
|
||||
const pin9 = openPr(fy, '9');
|
||||
const t = land(fy, '9', pin9, ['--repair']);
|
||||
expect(t.err).toContain('repair train gate.train-x');
|
||||
expect(t.err).toContain('(QUEUE_TRAIN_FROZEN)');
|
||||
expect(fy.read().mergeArgs).toBeUndefined();
|
||||
}, 120_000);
|
||||
|
||||
test("the Captain's sequence: red backstop freezes; a hand merge's backstop, green or red, clears nothing; land --pin --repair lands the repair; the green backstop on that main clears the freeze, and release-cut and land run again", () => {
|
||||
const fx = setup(DIRECT_TRAIN);
|
||||
const red0 = backstopOn(fx, 'failure', 41817566885);
|
||||
expect(red0.action).toBe('frozen');
|
||||
const m0 = red0.sha;
|
||||
expect(freezeOf(fx)).toMatchObject({ train: 'backstop', main: { sha: m0 }, cleared_at: null, repair: null });
|
||||
expect(readTrainNotes(gitIn(fx.repo.work)).freeze).toContain('queue frozen since train backstop');
|
||||
moveMain(fx, 'tooling-fix.txt', 'the hand-merged tooling fix\n');
|
||||
const green1 = backstopOn(fx, 'success', 41817566887);
|
||||
expect(green1.action).toBe('gated');
|
||||
expect(green1.lines.join('\n')).toContain('the freeze stands: no repair was admitted');
|
||||
expect(freezeOf(fx)).toMatchObject({ main: { sha: m0 }, cleared_at: null, repair: null });
|
||||
const pin = openPr(fx, '3996');
|
||||
let landed: ReturnType<typeof land> | undefined;
|
||||
const red1 = backstopOn(fx, 'failure', 41817566886, () => { landed = land(fx, '3996', pin, ['--repair']); });
|
||||
expect(landed!.err).toBe('');
|
||||
expect(landed!.status).toBe(EXIT.ok);
|
||||
expect(red1.action).toBe('frozen');
|
||||
expect(red1.lines.join('\n')).toContain('the freeze stands with its reason updated');
|
||||
expect(freezeOf(fx)).toMatchObject({ main: { sha: m0 }, cleared_at: null, repair: { pr: '3996', head: pin, train: null } });
|
||||
expect(freezeOf(fx).reason).toContain(`backstop red on main ${red1.sha.slice(0, 12)}`);
|
||||
expect(freezeOf(fx).reason).not.toContain('did not turn main green');
|
||||
const green2 = backstopOn(fx, 'success', 41817566888);
|
||||
expect(green2.action).toBe('gated');
|
||||
const cleared = freezeOf(fx);
|
||||
expect(cleared.cleared_by).toBe(`repair #3996; backstop 41817566888 green on ${green2.sha}`);
|
||||
expect(Date.parse(cleared.cleared_at)).not.toBeNaN();
|
||||
expect(green2.lines).toContain(`BACKSTOP: freeze cleared (${cleared.cleared_by})`);
|
||||
expect(readTrainNotes(gitIn(fx.repo.work)).freeze).toBeNull();
|
||||
const pin11 = openPr(fx, '11');
|
||||
const after = land(fx, '11', pin11);
|
||||
expect(after.err).toBe('');
|
||||
expect(after.status).toBe(EXIT.ok);
|
||||
expect(land(fx, '11', pin11, ['--repair']).status).toBe(EXIT.ok);
|
||||
}, 180_000);
|
||||
|
||||
test('a red backstop on a main that contains the repair keeps the freeze, updates its reason and frees the slot for the next repair, whose green backstop clears it; a cancelled gate judges nothing and keeps the slot', () => {
|
||||
const fx = setup(DIRECT_TRAIN);
|
||||
expect(backstopOn(fx, 'failure', 501).action).toBe('frozen');
|
||||
const pin7 = openPr(fx, '7');
|
||||
expect(land(fx, '7', pin7, ['--repair']).status).toBe(EXIT.ok);
|
||||
const unjudged = backstopOn(fx, 'cancelled', 5015);
|
||||
expect(unjudged.action).toBe('frozen');
|
||||
expect(freezeOf(fx)).toMatchObject({ cleared_at: null, repair: { pr: '7', head: pin7, train: null } });
|
||||
expect(freezeOf(fx).reason).toContain('repair #7 was not judged');
|
||||
const red = backstopOn(fx, 'failure', 502);
|
||||
expect(red.action).toBe('frozen');
|
||||
const kept = freezeOf(fx);
|
||||
expect(kept).toMatchObject({ cleared_at: null, repair: null });
|
||||
expect(kept.reason).toContain(`backstop red on main ${red.sha.slice(0, 12)}`);
|
||||
expect(kept.reason).toContain('repair #7 did not turn main green');
|
||||
const pin8 = openPr(fx, '8');
|
||||
expect(land(fx, '8', pin8, ['--repair']).status).toBe(EXIT.ok);
|
||||
const green = backstopOn(fx, 'success', 503);
|
||||
expect(freezeOf(fx).cleared_by).toBe(`repair #8; backstop 503 green on ${green.sha}`);
|
||||
}, 180_000);
|
||||
|
||||
test('no clear without the evidence: a green gate on a main that is not at or after the freeze main, or with the repair\'s land note missing, leaves the freeze active; a train freeze takes no land repair and no backstop clear', () => {
|
||||
const fx = setup(DIRECT_TRAIN);
|
||||
const orphan = fx.repo.git('commit-tree', fx.repo.git('rev-parse', 'origin/main^{tree}'), '-m', 'a main this history never had');
|
||||
freezeMain(fx, { main: { sha: orphan, tree: fx.repo.git('rev-parse', orphan + '^{tree}') } });
|
||||
const pin7 = openPr(fx, '7');
|
||||
expect(land(fx, '7', pin7, ['--repair']).status).toBe(EXIT.ok);
|
||||
const green = backstopOn(fx, 'success', 601);
|
||||
expect(green.action).toBe('gated');
|
||||
expect(green.lines.join('\n')).toContain(`is not at or after the freeze main ${orphan.slice(0, 12)}`);
|
||||
expect(freezeOf(fx).cleared_at).toBeNull();
|
||||
const fy = setup(DIRECT_TRAIN);
|
||||
freezeMain(fy, { repair: { pr: '9', head: 'b'.repeat(40), train: null } });
|
||||
moveMain(fy, 'hand.txt', 'merged without land\n');
|
||||
const g2 = backstopOn(fy, 'success', 602);
|
||||
expect(g2.lines.join('\n')).toContain('repair #9 has no land note');
|
||||
expect(g2.lines.join('\n')).toContain('gate this main again with gh run rerun <this backstop run> -R fixture/repo');
|
||||
expect(freezeOf(fy).cleared_at).toBeNull();
|
||||
const fz = setup(DIRECT_TRAIN);
|
||||
freezeMain(fz, { train: 'gate.train-old', backstop: null });
|
||||
const pin3 = openPr(fz, '3');
|
||||
const t3 = land(fz, '3', pin3, ['--repair']);
|
||||
expect(t3.err).toContain('admits a repair only into a main backstop freeze');
|
||||
expect(t3.err).toContain('(QUEUE_TRAIN_FROZEN)');
|
||||
expect(fz.read().mergeArgs).toBeUndefined();
|
||||
moveMain(fz, 'hand.txt', 'merged without land\n');
|
||||
const g3 = backstopOn(fz, 'success', 604);
|
||||
expect(g3.lines.join('\n')).toContain("train gate.train-old froze the queue, and its repair train's verify clears it");
|
||||
expect(freezeOf(fz)).toMatchObject({ train: 'gate.train-old', cleared_at: null });
|
||||
}, 180_000);
|
||||
});
|
||||
|
||||
describe('the backstop rewrites a freeze by compare-and-swap (GST-1354)', () => {
|
||||
test("a runner whose local notes hold a stale copy of the freeze never overwrites the repair land recorded on origin", () => {
|
||||
const fx = setup(DIRECT_TRAIN);
|
||||
expect(backstopOn(fx, 'failure', 701).action).toBe('frozen');
|
||||
const stale = freezeOf(fx);
|
||||
const runner = path.join(fx.repo.root, 'runner');
|
||||
const prior = process.env.GSTACK_STATE_ROOT;
|
||||
process.env.GSTACK_STATE_ROOT = path.join(fx.repo.root, 'backstop-state');
|
||||
try { persist(runner, [{ ...stale, reason: 'a stale local rewrite', updated_at: new Date(Date.now() + 60_000).toISOString() }], { push: false }); }
|
||||
finally { if (prior === undefined) delete process.env.GSTACK_STATE_ROOT; else process.env.GSTACK_STATE_ROOT = prior; }
|
||||
const pin = openPr(fx, '7');
|
||||
expect(land(fx, '7', pin, ['--repair']).status).toBe(EXIT.ok);
|
||||
const red = backstopOn(fx, 'cancelled', 702);
|
||||
expect(red.action).toBe('frozen');
|
||||
expect(freezeOf(fx)).toMatchObject({ repair: { pr: '7', head: pin, train: null }, cleared_at: null });
|
||||
expect(freezeOf(fx).reason).toContain('repair #7 was not judged');
|
||||
const green = backstopOn(fx, 'success', 703);
|
||||
expect(freezeOf(fx).cleared_by).toBe(`repair #7; backstop 703 green on ${green.sha}`);
|
||||
}, 180_000);
|
||||
});
|
||||
@@ -167,6 +167,19 @@ describe("backstop: a train's merges are covered by the train's own green record
|
||||
expect(dispatches()).toEqual([]);
|
||||
});
|
||||
|
||||
test('while a freeze is active, a covering green-train record does not stand in for the gate (GST-1354)', () => {
|
||||
const fx = trainFixture();
|
||||
const { gh, dispatches } = fakeGh(fx, { gate: 'success' });
|
||||
const main = squashMerge(fx.repo, 1);
|
||||
const at = new Date().toISOString();
|
||||
persist(fx.repo.work, [{ schema: 1, kind: 'freeze', repo_key: repoIdentity(fx.repo.work).key, train: 'backstop', merged: [], unmerged: [], main: { sha: fx.main0, tree: fx.repo.git('rev-parse', fx.main0 + '^{tree}') },
|
||||
reason: 'backstop red', backstop: { nonce: 'gate.backstop-x', run: 1, state: 'completed/failure' }, repair: null, created_at: at, cleared_at: null, cleared_by: null, updated_at: at } as any]);
|
||||
const out = backstop(fx, main, gh);
|
||||
expect(out.action, out.lines.join('\n')).toBe('gated');
|
||||
expect(out.lines).toContain('BACKSTOP: the queue is frozen, so only a gate on main can clear it; no green-train record stands in');
|
||||
expect(dispatches()).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('a superseded push hands main to the newer push without a gh call', () => {
|
||||
const fx = trainFixture();
|
||||
const first = squashMerge(fx.repo, 1);
|
||||
|
||||
Reference in new issue
Block a user