diff --git a/lib/ship-queue-train-land.ts b/lib/ship-queue-train-land.ts index ef193cdf1..6156132be 100644 --- a/lib/ship-queue-train-land.ts +++ b/lib/ship-queue-train-land.ts @@ -56,13 +56,16 @@ export async function trainLand(ctx: QueueContext, o: { train: string; pr: strin }; const rerun = `${Q} train land --train ${t.nonce} --pr ${o.pr}`; const lines = [`TRAIN_LAND: ${t.nonce} slot ${i + 1}/${t.slots.length} #${o.pr} pin ${pin} -> S${s.index} v${s.version} tree ${s.tree}`]; - if (git(root, ['rev-parse', `${t.head}~${t.stamped.length - s.index}`]).out !== s.commit || git(root, ['rev-parse', `${s.commit}^{tree}`]).out !== s.tree) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `the record's S${s.index} is not the gated head ${t.head.slice(0, 12)}'s prefix ${s.index} with tree ${s.tree.slice(0, 12)}` }, lines, `${Q} train status --train ${t.nonce}`, run); + if (t.stamped.length !== t.slots.length || s.index !== i + 1 || git(root, ['rev-parse', `${t.head}~${t.slots.length - i - 1}`]).out !== s.commit || git(root, ['rev-parse', `${s.commit}^{tree}`]).out !== s.tree) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `the record's S${i + 1} is not the gated head ${t.head.slice(0, 12)}'s prefix ${i + 1} with tree ${s.tree.slice(0, 12)}` }, lines, `${Q} train status --train ${t.nonce}`, run); let main = mainNow(); if ('error' in main) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: main.error }, lines, rerun, run); if (main.tree === s.tree) return confirmMerged(env, t, o.pr, main.sha, s.index, [...lines, `main: ${main.sha} is S${s.index} exactly`]); const mainTree = main.tree; - const earlier = t.stamped.slice(0, Math.max(i - 1, 0)).find(p => p.tree === mainTree); - if (earlier) return outRefused({ code: 'QUEUE_TRAIN_TREE_MISMATCH', message: `main is S${earlier.index}; land #${t.stamped[earlier.index]!.pr} (slot ${earlier.index + 1}) first` }, lines, `${Q} train land --train ${t.nonce} --pr ${t.stamped[earlier.index]!.pr}`, run); + const chain = [{ index: 0, tree: t.prefixes[0]!.tree }, ...t.stamped]; + const earlier = chain.slice(0, i).find(p => p.tree === mainTree); + if (earlier) return outRefused({ code: 'QUEUE_TRAIN_TREE_MISMATCH', message: `main is ${earlier.index ? `S${earlier.index}` : 'the train base'}; land #${t.slots[earlier.index]!.pr} (slot ${earlier.index + 1}) first` }, lines, `${Q} train land --train ${t.nonce} --pr ${t.slots[earlier.index]!.pr}`, run); + const later = chain.slice(i + 2).find(p => p.tree === mainTree); + if (later) return confirmMerged(env, t, o.pr, main.sha, s.index, [...lines, `main: ${main.sha} is S${later.index}, past S${s.index}`]); if (main.tree !== pred.tree) return stopped(env, t, 'QUEUE_TRAIN_TREE_MISMATCH', `main ${main.sha.slice(0, 12)} is neither S${pred.index} nor S${s.index} exactly`, main.sha, lines, true); const file = path.join(path.dirname(run), `land-${o.pr}.json`); @@ -88,7 +91,12 @@ export async function trainLand(ctx: QueueContext, o: { train: string; pr: strin const pushed = info.headRefOid !== commit; if (pushed) { const push = git(root, ['push', '-q', `--force-with-lease=refs/heads/${info.headRefName}:${info.headRefOid}`, env.remote, `${commit}:refs/heads/${info.headRefName}`]); - if (!push.ok) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${o.pr}'s branch moved off ${info.headRefOid.slice(0, 12)} before the stamp push; train land never overwrites it`, main.sha, lines, true); + if (!push.ok) { + const remote = git(root, ['ls-remote', env.remote, `refs/heads/${info.headRefName}`]); + const now = remote.ok ? remote.out.split(/\s/)[0] ?? '' : ''; + if (remote.ok && now !== info.headRefOid) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${o.pr}'s branch moved off ${info.headRefOid.slice(0, 12)} before the stamp push; train land never overwrites it`, main.sha, lines, true); + return outFailed({ code: 'QUEUE_NOT_READY', message: `the stamp push failed and ${info.headRefName} still reads ${remote.ok ? info.headRefOid.slice(0, 12) : 'unknown'}: ${push.err.split('\n')[0]}` }, lines, rerun, run); + } } lines.push(`${pushed ? 'PUSHED' : 'RESUMED'}: ${commit} (tree S${s.index}, parents [pin, main ${main.sha.slice(0, 12)}]) ${pushed ? 'to' : 'is already the head of'} ${info.headRefName}`); const next = `gstack-pr-wait ${o.pr} --until green && ${Q} train land --train ${t.nonce} --pr ${o.pr}`; @@ -96,7 +104,8 @@ export async function trainLand(ctx: QueueContext, o: { train: string; pr: strin if (!o.noWait) waitChecks(ctx, commit, o.timeoutMin ?? 60); } const head = prInfo(ctx, o.pr); - if ('error' in head || head.headRefOid !== j.stamp_commit) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${o.pr}'s head is not the stamp commit ${j.stamp_commit.slice(0, 12)}`, main.sha, lines, true); + if ('error' in head) return outPending({ code: 'QUEUE_PROOF_MISSING', message: head.error }, lines, rerun, run); + if (head.headRefOid !== j.stamp_commit) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${o.pr}'s head is not the stamp commit ${j.stamp_commit.slice(0, 12)}`, main.sha, lines, true); const movedBase = retargeted(head.baseRefName); if (movedBase) return movedBase; const ci = ciAtHead(ctx, j.stamp_commit); diff --git a/test/ship-queue-train-land.test.ts b/test/ship-queue-train-land.test.ts index 82033eaee..3602629c0 100644 --- a/test/ship-queue-train-land.test.ts +++ b/test/ship-queue-train-land.test.ts @@ -78,7 +78,7 @@ describe('stamped trains: build and land', () => { expect(verify.err).toMatch(/has stamped prefixes.*\(QUEUE_GATE_MISSING\)/); expect(line(verify.out, 'NEXT:')).toBe(`NEXT: gstack-ship-queue train land --train ${nonce} --pr 1`); const early = land(fx, nonce, '2'); - expect(early.err).toMatch(/is neither S1 nor S2 exactly.*not frozen.*\(QUEUE_TRAIN_TREE_MISMATCH\)/); + expect(early.err).toMatch(/main is the train base; land #1 \(slot 1\) first.*\(QUEUE_TRAIN_TREE_MISMATCH\)/); const pushed = land(fx, nonce, '1'); expect(pushed.out, pushed.err).toContain('LABEL: evals-no-debounce added'); const stamp = /^PUSHED: ([0-9a-f]{40})/m.exec(pushed.out)![1]!; @@ -123,12 +123,21 @@ describe('stamped trains: build and land', () => { expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok); const file = path.join(fx.state, 'ship-queue', 'trains', nonce, 'train.json'); const t = trainJson(fx, nonce); - t.stamped[0] = { ...t.stamped[0], commit: h1, tree: fx.repo.git('rev-parse', h1 + '^{tree}') }; - fs.writeFileSync(file, JSON.stringify(t)); - const r = land(fx, nonce, '1'); - expect(r.status).toBe(EXIT.refused); - expect(r.err).toMatch(/is not the gated head \S+'s prefix 1 with tree.*\(QUEUE_PROOF_MISSING\)/); - expect(r.out).not.toMatch(/^PUSHED:/m); + const base = trainJson(fx, nonce).prefixes[0]; + const forged = [ + { ...t.stamped[0], commit: h1, tree: fx.repo.git('rev-parse', h1 + '^{tree}') }, + { ...t.stamped[0], tree: 'f'.repeat(40) }, + { ...t.stamped[1] }, + { ...t.stamped[0], index: 0, commit: base.commit, tree: base.tree }, + ]; + for (const entry of forged) { + fs.writeFileSync(file, JSON.stringify({ ...t, stamped: [entry, t.stamped[1]] })); + const r = land(fx, nonce, '1'); + expect(r.status).toBe(EXIT.refused); + expect(r.err).toMatch(/is not the gated head \S+'s prefix 1 with tree.*\(QUEUE_PROOF_MISSING\)/); + expect(r.out).not.toMatch(/^PUSHED:/m); + } + expect(fx.read().prs['1']).not.toMatchObject({ state: 'MERGED' }); expect(fx.repo.git('--git-dir', fx.repo.origin, 'rev-parse', 'refs/heads/pr-1')).toBe(h1); }, 180_000); @@ -156,7 +165,7 @@ describe('stamped trains: build and land', () => { expect(land(fx, nonce, '1').status).toBe(EXIT.ok); }, 300_000); - test('a slot landed out of order after an earlier slot merged refuses without freezing and names the next slot', () => { + test('a slot landed out of order after an earlier slot merged refuses without freezing and names the next slot; a rerun of a landed slot confirms', () => { const fx = trainFixture(POLICY, 'gstack-train-land-'); fixtures.push(fx); wrap(fx); const heads = ['1', '2', '3'].map(pr => fragmentPr(fx, pr)); fx.write(d => { d.rules['evals.yml'] = GREEN_GATE; }); @@ -168,6 +177,24 @@ describe('stamped trains: build and land', () => { expect(early.err).toMatch(/main is S1; land #2 \(slot 2\) first.*\(QUEUE_TRAIN_TREE_MISMATCH\)/); expect(early.out).not.toMatch(/^QUEUE_TRAIN_FROZEN:/m); expect(line(early.out, 'NEXT:')).toBe(`NEXT: gstack-ship-queue train land --train ${nonce} --pr 2`); + land(fx, nonce, '2'); + expect(land(fx, nonce, '2').status).toBe(EXIT.ok); + const rerun = land(fx, nonce, '1'); + expect(rerun.status, rerun.err).toBe(EXIT.ok); + expect(rerun.out).toContain('is S2, past S1'); + expect(rerun.out).not.toMatch(/^QUEUE_TRAIN_FROZEN:/m); + }, 300_000); + + test('a failed stamp push with the branch still at the pin fails retryably instead of reporting a move', () => { + const { fx, nonce } = stampedTrain(); + expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok); + const hook = path.join(fx.repo.origin, 'hooks', 'pre-receive'); + fs.writeFileSync(hook, '#!/bin/sh\nexit 1\n', { mode: 0o755 }); + const failed = land(fx, nonce, '1'); + expect(failed.err).toMatch(/the stamp push failed and pr-1 still reads.*\(QUEUE_NOT_READY\)/); + expect(failed.err).not.toContain('QUEUE_TRAIN_MOVED'); + fs.rmSync(hook); + expect(land(fx, nonce, '1').out).toMatch(/^PUSHED: [0-9a-f]{40}/m); }, 300_000); test('a PR retargeted off the train base refuses before any push', () => {