fix(train-land): pin S_i to slot position; a failed push or unreadable PR never reads as a move; reruns of landed slots confirm; T0 in the out-of-order refusal

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

lib/ship-queue-train-land.ts

@@ -56,13 +56,16 @@ export async function trainLand(ctx: QueueContext, o: { train: string; pr: strin
   };
   const rerun = `${Q} train land --train ${t.nonce} --pr ${o.pr}`;
   const lines = [`TRAIN_LAND: ${t.nonce} slot ${i + 1}/${t.slots.length} #${o.pr} pin ${pin} -> S${s.index} v${s.version} tree ${s.tree}`];
-  if (git(root, ['rev-parse', `${t.head}~${t.stamped.length - s.index}`]).out !== s.commit || git(root, ['rev-parse', `${s.commit}^{tree}`]).out !== s.tree) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `the record's S${s.index} is not the gated head ${t.head.slice(0, 12)}'s prefix ${s.index} with tree ${s.tree.slice(0, 12)}` }, lines, `${Q} train status --train ${t.nonce}`, run);
+  if (t.stamped.length !== t.slots.length || s.index !== i + 1 || git(root, ['rev-parse', `${t.head}~${t.slots.length - i - 1}`]).out !== s.commit || git(root, ['rev-parse', `${s.commit}^{tree}`]).out !== s.tree) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: `the record's S${i + 1} is not the gated head ${t.head.slice(0, 12)}'s prefix ${i + 1} with tree ${s.tree.slice(0, 12)}` }, lines, `${Q} train status --train ${t.nonce}`, run);
   let main = mainNow();
   if ('error' in main) return outRefused({ code: 'QUEUE_PROOF_MISSING', message: main.error }, lines, rerun, run);
   if (main.tree === s.tree) return confirmMerged(env, t, o.pr, main.sha, s.index, [...lines, `main: ${main.sha} is S${s.index} exactly`]);
   const mainTree = main.tree;
-  const earlier = t.stamped.slice(0, Math.max(i - 1, 0)).find(p => p.tree === mainTree);
-  if (earlier) return outRefused({ code: 'QUEUE_TRAIN_TREE_MISMATCH', message: `main is S${earlier.index}; land #${t.stamped[earlier.index]!.pr} (slot ${earlier.index + 1}) first` }, lines, `${Q} train land --train ${t.nonce} --pr ${t.stamped[earlier.index]!.pr}`, run);
+  const chain = [{ index: 0, tree: t.prefixes[0]!.tree }, ...t.stamped];
+  const earlier = chain.slice(0, i).find(p => p.tree === mainTree);
+  if (earlier) return outRefused({ code: 'QUEUE_TRAIN_TREE_MISMATCH', message: `main is ${earlier.index ? `S${earlier.index}` : 'the train base'}; land #${t.slots[earlier.index]!.pr} (slot ${earlier.index + 1}) first` }, lines, `${Q} train land --train ${t.nonce} --pr ${t.slots[earlier.index]!.pr}`, run);
+  const later = chain.slice(i + 2).find(p => p.tree === mainTree);
+  if (later) return confirmMerged(env, t, o.pr, main.sha, s.index, [...lines, `main: ${main.sha} is S${later.index}, past S${s.index}`]);
   if (main.tree !== pred.tree) return stopped(env, t, 'QUEUE_TRAIN_TREE_MISMATCH', `main ${main.sha.slice(0, 12)} is neither S${pred.index} nor S${s.index} exactly`, main.sha, lines, true);
 
   const file = path.join(path.dirname(run), `land-${o.pr}.json`);
@@ -88,7 +91,12 @@ export async function trainLand(ctx: QueueContext, o: { train: string; pr: strin
     const pushed = info.headRefOid !== commit;
     if (pushed) {
       const push = git(root, ['push', '-q', `--force-with-lease=refs/heads/${info.headRefName}:${info.headRefOid}`, env.remote, `${commit}:refs/heads/${info.headRefName}`]);
-      if (!push.ok) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${o.pr}'s branch moved off ${info.headRefOid.slice(0, 12)} before the stamp push; train land never overwrites it`, main.sha, lines, true);
+      if (!push.ok) {
+        const remote = git(root, ['ls-remote', env.remote, `refs/heads/${info.headRefName}`]);
+        const now = remote.ok ? remote.out.split(/\s/)[0] ?? '' : '';
+        if (remote.ok && now !== info.headRefOid) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${o.pr}'s branch moved off ${info.headRefOid.slice(0, 12)} before the stamp push; train land never overwrites it`, main.sha, lines, true);
+        return outFailed({ code: 'QUEUE_NOT_READY', message: `the stamp push failed and ${info.headRefName} still reads ${remote.ok ? info.headRefOid.slice(0, 12) : 'unknown'}: ${push.err.split('\n')[0]}` }, lines, rerun, run);
+      }
     }
     lines.push(`${pushed ? 'PUSHED' : 'RESUMED'}: ${commit} (tree S${s.index}, parents [pin, main ${main.sha.slice(0, 12)}]) ${pushed ? 'to' : 'is already the head of'} ${info.headRefName}`);
     const next = `gstack-pr-wait ${o.pr} --until green && ${Q} train land --train ${t.nonce} --pr ${o.pr}`;
@@ -96,7 +104,8 @@ export async function trainLand(ctx: QueueContext, o: { train: string; pr: strin
     if (!o.noWait) waitChecks(ctx, commit, o.timeoutMin ?? 60);
   }
   const head = prInfo(ctx, o.pr);
-  if ('error' in head || head.headRefOid !== j.stamp_commit) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${o.pr}'s head is not the stamp commit ${j.stamp_commit.slice(0, 12)}`, main.sha, lines, true);
+  if ('error' in head) return outPending({ code: 'QUEUE_PROOF_MISSING', message: head.error }, lines, rerun, run);
+  if (head.headRefOid !== j.stamp_commit) return stopped(env, t, 'QUEUE_TRAIN_MOVED', `#${o.pr}'s head is not the stamp commit ${j.stamp_commit.slice(0, 12)}`, main.sha, lines, true);
   const movedBase = retargeted(head.baseRefName);
   if (movedBase) return movedBase;
   const ci = ciAtHead(ctx, j.stamp_commit);

test/ship-queue-train-land.test.ts

@@ -78,7 +78,7 @@ describe('stamped trains: build and land', () => {
     expect(verify.err).toMatch(/has stamped prefixes.*\(QUEUE_GATE_MISSING\)/);
     expect(line(verify.out, 'NEXT:')).toBe(`NEXT: gstack-ship-queue train land --train ${nonce} --pr 1`);
     const early = land(fx, nonce, '2');
-    expect(early.err).toMatch(/is neither S1 nor S2 exactly.*not frozen.*\(QUEUE_TRAIN_TREE_MISMATCH\)/);
+    expect(early.err).toMatch(/main is the train base; land #1 \(slot 1\) first.*\(QUEUE_TRAIN_TREE_MISMATCH\)/);
     const pushed = land(fx, nonce, '1');
     expect(pushed.out, pushed.err).toContain('LABEL: evals-no-debounce added');
     const stamp = /^PUSHED: ([0-9a-f]{40})/m.exec(pushed.out)![1]!;
@@ -123,12 +123,21 @@ describe('stamped trains: build and land', () => {
     expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok);
     const file = path.join(fx.state, 'ship-queue', 'trains', nonce, 'train.json');
     const t = trainJson(fx, nonce);
-    t.stamped[0] = { ...t.stamped[0], commit: h1, tree: fx.repo.git('rev-parse', h1 + '^{tree}') };
-    fs.writeFileSync(file, JSON.stringify(t));
-    const r = land(fx, nonce, '1');
-    expect(r.status).toBe(EXIT.refused);
-    expect(r.err).toMatch(/is not the gated head \S+'s prefix 1 with tree.*\(QUEUE_PROOF_MISSING\)/);
-    expect(r.out).not.toMatch(/^PUSHED:/m);
+    const base = trainJson(fx, nonce).prefixes[0];
+    const forged = [
+      { ...t.stamped[0], commit: h1, tree: fx.repo.git('rev-parse', h1 + '^{tree}') },
+      { ...t.stamped[0], tree: 'f'.repeat(40) },
+      { ...t.stamped[1] },
+      { ...t.stamped[0], index: 0, commit: base.commit, tree: base.tree },
+    ];
+    for (const entry of forged) {
+      fs.writeFileSync(file, JSON.stringify({ ...t, stamped: [entry, t.stamped[1]] }));
+      const r = land(fx, nonce, '1');
+      expect(r.status).toBe(EXIT.refused);
+      expect(r.err).toMatch(/is not the gated head \S+'s prefix 1 with tree.*\(QUEUE_PROOF_MISSING\)/);
+      expect(r.out).not.toMatch(/^PUSHED:/m);
+    }
+    expect(fx.read().prs['1']).not.toMatchObject({ state: 'MERGED' });
     expect(fx.repo.git('--git-dir', fx.repo.origin, 'rev-parse', 'refs/heads/pr-1')).toBe(h1);
   }, 180_000);
 
@@ -156,7 +165,7 @@ describe('stamped trains: build and land', () => {
     expect(land(fx, nonce, '1').status).toBe(EXIT.ok);
   }, 300_000);
 
-  test('a slot landed out of order after an earlier slot merged refuses without freezing and names the next slot', () => {
+  test('a slot landed out of order after an earlier slot merged refuses without freezing and names the next slot; a rerun of a landed slot confirms', () => {
     const fx = trainFixture(POLICY, 'gstack-train-land-'); fixtures.push(fx); wrap(fx);
     const heads = ['1', '2', '3'].map(pr => fragmentPr(fx, pr));
     fx.write(d => { d.rules['evals.yml'] = GREEN_GATE; });
@@ -168,6 +177,24 @@ describe('stamped trains: build and land', () => {
     expect(early.err).toMatch(/main is S1; land #2 \(slot 2\) first.*\(QUEUE_TRAIN_TREE_MISMATCH\)/);
     expect(early.out).not.toMatch(/^QUEUE_TRAIN_FROZEN:/m);
     expect(line(early.out, 'NEXT:')).toBe(`NEXT: gstack-ship-queue train land --train ${nonce} --pr 2`);
+    land(fx, nonce, '2');
+    expect(land(fx, nonce, '2').status).toBe(EXIT.ok);
+    const rerun = land(fx, nonce, '1');
+    expect(rerun.status, rerun.err).toBe(EXIT.ok);
+    expect(rerun.out).toContain('is S2, past S1');
+    expect(rerun.out).not.toMatch(/^QUEUE_TRAIN_FROZEN:/m);
+  }, 300_000);
+
+  test('a failed stamp push with the branch still at the pin fails retryably instead of reporting a move', () => {
+    const { fx, nonce } = stampedTrain();
+    expect(queue(fx, ['train', 'verdict', '--train', nonce]).status).toBe(EXIT.ok);
+    const hook = path.join(fx.repo.origin, 'hooks', 'pre-receive');
+    fs.writeFileSync(hook, '#!/bin/sh\nexit 1\n', { mode: 0o755 });
+    const failed = land(fx, nonce, '1');
+    expect(failed.err).toMatch(/the stamp push failed and pr-1 still reads.*\(QUEUE_NOT_READY\)/);
+    expect(failed.err).not.toContain('QUEUE_TRAIN_MOVED');
+    fs.rmSync(hook);
+    expect(land(fx, nonce, '1').out).toMatch(/^PUSHED: [0-9a-f]{40}/m);
   }, 300_000);
 
   test('a PR retargeted off the train base refuses before any push', () => {