test: land-direct runs runMergedChecks in a child Bun so recovery shares a process (GST-1613) (#4068)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

changelog.d/capy-gst-1613-land-recovery-isolation.md

@@ -0,0 +1,3 @@
+### Fixed
+
+- **`test/ship-queue-land-direct.test.ts` and `test/ubicloud-runner-recovery.test.ts` pass in one bun process.** The land file's merged-checks case called `runMergedChecks` (lib/land-merged-checks.ts) inside the test runner. On Linux that call sets `PR_SET_CHILD_SUBREAPER` and clears `PR_SET_DUMPABLE` on its caller and never undoes them, so the runner adopted every later orphan and never reaped it. The fake VM's detached phases and the sink pusher's `sleep` stayed zombies that `tail --pid` and `kill -0` count as alive, so recovery's run cases hung at the 90 s timeout (Free Tests 42207455885, shard 10). The case now runs `runMergedChecks` in a child Bun. The fake VM fails at once with a named error when the runner is a subreaper, recovery fails on any zombie child it leaves, and a new regression runs the pair in one process. (GST-1613)

test/helpers/process-reaping.ts

@@ -0,0 +1,29 @@
+/**
+ * GST-1613: a Linux test runner that is a child subreaper (prctl PR_SET_CHILD_SUBREAPER) adopts every orphan its
+ * tests leave and never reaps it, so a detached process that exits stays a zombie that kill -0 and tail --pid count
+ * as alive. lib/land-merged-checks.ts sets the flag on its caller, so tests run it in a child Bun.
+ */
+import { dlopen, FFIType, ptr } from 'bun:ffi';
+import * as fs from 'node:fs';
+
+const openLibc = () => dlopen('libc.so.6', { prctl: { args: [FFIType.i32, FFIType.ptr, FFIType.u64, FFIType.u64, FFIType.u64], returns: FFIType.i32 } }).symbols;
+let libc: ReturnType<typeof openLibc> | undefined;
+
+/** PR_GET_CHILD_SUBREAPER for this process; false off Linux. */
+export function isChildSubreaper(): boolean {
+  if (process.platform !== 'linux') return false;
+  libc ??= openLibc();
+  const flag = new Int32Array(1);
+  return libc.prctl(37, ptr(flag), 0, 0, 0) === 0 && flag[0] !== 0;
+}
+
+/** This process's zombie children as `<pid> <comm>`; [] off Linux. */
+export function zombieChildren(): string[] {
+  if (process.platform !== 'linux') return [];
+  return fs.readdirSync('/proc').filter(d => /^\d+$/.test(d)).flatMap(d => {
+    let stat = '';
+    try { stat = fs.readFileSync(`/proc/${d}/stat`, 'utf8'); } catch { return []; }
+    const [state, ppid] = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
+    return state === 'Z' && Number(ppid) === process.pid ? [`${d} ${stat.slice(stat.indexOf('(') + 1, stat.lastIndexOf(')'))}`] : [];
+  });
+}

test/helpers/ubi-runner-fake-vm.ts

@@ -5,6 +5,7 @@
 import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join, resolve } from 'node:path';
+import { isChildSubreaper } from './process-reaping';
 
 export const ROOT = resolve(import.meta.dir, '../..');
 export const DIR = join(ROOT, 'scripts/ubicloud');
@@ -31,6 +32,7 @@ export const read = (rel: string) => readFileSync(join(ROOT, rel), 'utf8');
  * make-<target>.sh from the fixture root. No network, no VM.
  */
 export function fakeVm(opts: { listed?: string; refuse?: string } = {}) {
+  if (isChildSubreaper()) throw new Error('this bun process is a child subreaper (an earlier test called prctl PR_SET_CHILD_SUBREAPER in-process, as lib/land-merged-checks.ts runMergedChecks does), so the fake VM\'s detached phases would stay unreaped zombies and run cases would hang (GST-1613)');
   const root = mkdtempSync(join(tmpdir(), 'ubi-run-'));
   const bin = join(root, 'bin'), state = join(root, 'state'), log = join(root, 'calls.log'), sshLog = join(root, 'ssh.log'), record = join(root, 'vms.tsv');
   const vm = join(root, 'vm'), vmbin = join(root, 'vmbin'), srcDir = join(root, 'src'), checkpoints = join(root, 'checkpoints');

test/land-direct-recovery-isolation.test.ts

@@ -0,0 +1,19 @@
+/**
+ * GST-1613: test/ship-queue-land-direct.test.ts and test/ubicloud-runner-recovery.test.ts in one bun process, as a
+ * free shard can pack them (shard 10, run 42207455885). The land file ran lib/land-merged-checks.ts runMergedChecks in
+ * the runner, whose prctl(PR_SET_CHILD_SUBREAPER) made the runner adopt every later orphan without reaping it, so the
+ * fake VM's detached phases stayed zombies that tail --pid and kill -0 count as alive and recovery's run cases hung at
+ * 90 s. The child selects the merged-checks case and one run case; recovery's afterAll fails on any zombie child.
+ */
+import { expect, test } from 'bun:test';
+import { spawnSync } from 'node:child_process';
+import * as path from 'node:path';
+
+test.skipIf(process.platform === 'win32')('the land-direct merged-checks case and a recovery run case pass in one process with no zombie left', () => {
+  const r = spawnSync(process.execPath, ['test', 'test/ship-queue-land-direct.test.ts', 'test/ubicloud-runner-recovery.test.ts', '-t', 'runMergedChecks: HEAD is|run keeps the presigned URL'],
+    { cwd: path.resolve(import.meta.dir, '..'), encoding: 'utf8', timeout: 150_000 });
+  const out = r.stdout + r.stderr;
+  expect(r.status, out).toBe(0);
+  expect(out.match(/^\(pass\) .*(runMergedChecks: HEAD is|run keeps the presigned URL)/gm) ?? [], out).toHaveLength(2);
+  expect(out, out).not.toMatch(/^\(fail\)/m);
+}, 180_000);

test/ship-queue-land-direct.test.ts

@@ -18,7 +18,8 @@ import { persist, repoIdentity } from '../lib/ship-queue-train-record';
 import { rewriteStamps, tentativeMentions } from '../lib/restamp';
 import { bumpVersion } from '../lib/version-gate-train';
 import { openRelease } from '../lib/release-stamp';
-import { runMergedChecks } from '../lib/land-merged-checks';
+import type { MergedRun } from '../lib/land-merged-checks';
+import { bunIsolationArgs } from '../lib/bun-isolation';
 import { readLock, writeLock } from '../lib/ship-queue-land-lock';
 import { flakyGit } from './helpers/flaky-git';
 import { runBackstop } from '../scripts/train-backstop';
@@ -113,6 +114,13 @@ const landNotes = (fx: Fx) => {
 };
 /** Whether a clone holds a commit. */
 const spawnCheck = (dir: string, sha: string) => spawnSync('git', ['-C', dir, 'cat-file', '-e', `${sha}^{commit}`], { timeout: 30_000 }).status === 0;
+/** runMergedChecks in a child Bun: on Linux it makes its caller a non-dumpable child subreaper for good, and a test runner that is one never reaps the orphans later test files leave, which hung test/ubicloud-runner-recovery.test.ts in the same process (GST-1613). */
+function runMergedChecks(root: string, base: string, pin: string, checks: string[], ms?: number): MergedRun {
+  const lib = JSON.stringify(path.join(import.meta.dir, '..', 'lib', 'land-merged-checks.ts'));
+  const r = spawnSync(process.execPath, [...bunIsolationArgs(), '-e', `const { runMergedChecks } = await import(${lib}); const [root, base, pin, checks, ms] = JSON.parse(await Bun.stdin.text()); console.log(JSON.stringify(runMergedChecks(root, base, pin, checks, ms ?? undefined)));`], { input: JSON.stringify([root, base, pin, checks, ms ?? null]), encoding: 'utf8', timeout: 60_000 });
+  if (r.status !== 0) throw new Error(`runMergedChecks child exited ${r.status ?? r.signal}: ${r.stderr}`);
+  return JSON.parse(r.stdout) as MergedRun;
+}
 /** Commit a file straight onto origin/main (another PR that landed first). */
 function moveMain(fx: Fx, file: string, text: string): string {
   const { repo } = fx;

test/test-of-test-ratchet.test.ts

@@ -132,6 +132,7 @@ const BASELINE = [
   'test/ios-qa-stateserver-hardening.test.ts',
   'test/ios-qa-swift-build.test.ts',
   'test/jargon-list.test.ts',
+  'test/land-direct-recovery-isolation.test.ts', // GST-1613: runs test/ship-queue-land-direct.test.ts and test/ubicloud-runner-recovery.test.ts in one bun process; the product code is what those two files run
   'test/llm-judge-frontier.test.ts',
   'test/llm-judge-stream.test.ts', // landed in v1.91.7.0 before this ratchet: harness owner for its functional-QA/docsync paid evals
   'test/memory-cache-injection.test.ts',

test/ubicloud-runner-recovery.test.ts

@@ -3,10 +3,11 @@
  * caller's sink, and `recover` rebuilds a VM's local state on another controller
  * from the encrypted bundle `up` wrote. Offline: the fake API and fake VM only.
  */
-import { afterEach, describe, expect, test } from 'bun:test';
+import { afterAll, afterEach, describe, expect, test } from 'bun:test';
 import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { creates, fakeVm, gitSrc, run, until, vmDirs, type FakeVm } from './helpers/ubi-runner-fake-vm';
+import { zombieChildren } from './helpers/process-reaping';
 
 const fixtures: FakeVm[] = [];
 const make = () => { const f = fakeVm(); fixtures.push(f); return f; };
@@ -21,6 +22,10 @@ const stopSink = (f: FakeVm) => {
 afterEach(() => {
   for (const f of fixtures.splice(0)) { stopSink(f); rmSync(f.root, { recursive: true, force: true }); }
 });
+afterAll(async () => {
+  await until(() => zombieChildren().length === 0, 2_000).catch(() => {});
+  expect(zombieChildren()).toEqual([]);
+}, 10_000);
 const out = (r: { stdout: Buffer; stderr: Buffer }) => r.stdout.toString() + r.stderr.toString();
 const tarList = (tgz: string) => Bun.spawnSync(['tar', '-tzf', tgz], { timeout: 10_000 }).stdout.toString();
 const heartbeat = (tgz: string) => {