diff --git a/changelog.d/capy-gst-1613-land-recovery-isolation.md b/changelog.d/capy-gst-1613-land-recovery-isolation.md new file mode 100644 index 000000000..667cb06dc --- /dev/null +++ b/changelog.d/capy-gst-1613-land-recovery-isolation.md @@ -0,0 +1,3 @@ +### Fixed + +- **`test/ship-queue-land-direct.test.ts` and `test/ubicloud-runner-recovery.test.ts` pass in one bun process.** The land file's merged-checks case called `runMergedChecks` (lib/land-merged-checks.ts) inside the test runner. On Linux that call sets `PR_SET_CHILD_SUBREAPER` and clears `PR_SET_DUMPABLE` on its caller and never undoes them, so the runner adopted every later orphan and never reaped it. The fake VM's detached phases and the sink pusher's `sleep` stayed zombies that `tail --pid` and `kill -0` count as alive, so recovery's run cases hung at the 90 s timeout (Free Tests 42207455885, shard 10). The case now runs `runMergedChecks` in a child Bun. The fake VM fails at once with a named error when the runner is a subreaper, recovery fails on any zombie child it leaves, and a new regression runs the pair in one process. (GST-1613) diff --git a/test/helpers/process-reaping.ts b/test/helpers/process-reaping.ts new file mode 100644 index 000000000..8bfb7d200 --- /dev/null +++ b/test/helpers/process-reaping.ts @@ -0,0 +1,29 @@ +/** + * GST-1613: a Linux test runner that is a child subreaper (prctl PR_SET_CHILD_SUBREAPER) adopts every orphan its + * tests leave and never reaps it, so a detached process that exits stays a zombie that kill -0 and tail --pid count + * as alive. lib/land-merged-checks.ts sets the flag on its caller, so tests run it in a child Bun. + */ +import { dlopen, FFIType, ptr } from 'bun:ffi'; +import * as fs from 'node:fs'; + +const openLibc = () => dlopen('libc.so.6', { prctl: { args: [FFIType.i32, FFIType.ptr, FFIType.u64, FFIType.u64, FFIType.u64], returns: FFIType.i32 } }).symbols; +let libc: ReturnType | undefined; + +/** PR_GET_CHILD_SUBREAPER for this process; false off Linux. */ +export function isChildSubreaper(): boolean { + if (process.platform !== 'linux') return false; + libc ??= openLibc(); + const flag = new Int32Array(1); + return libc.prctl(37, ptr(flag), 0, 0, 0) === 0 && flag[0] !== 0; +} + +/** This process's zombie children as ` `; [] off Linux. */ +export function zombieChildren(): string[] { + if (process.platform !== 'linux') return []; + return fs.readdirSync('/proc').filter(d => /^\d+$/.test(d)).flatMap(d => { + let stat = ''; + try { stat = fs.readFileSync(`/proc/${d}/stat`, 'utf8'); } catch { return []; } + const [state, ppid] = stat.slice(stat.lastIndexOf(')') + 2).split(' '); + return state === 'Z' && Number(ppid) === process.pid ? [`${d} ${stat.slice(stat.indexOf('(') + 1, stat.lastIndexOf(')'))}`] : []; + }); +} diff --git a/test/helpers/ubi-runner-fake-vm.ts b/test/helpers/ubi-runner-fake-vm.ts index b46f8313c..7954e2c53 100644 --- a/test/helpers/ubi-runner-fake-vm.ts +++ b/test/helpers/ubi-runner-fake-vm.ts @@ -5,6 +5,7 @@ import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; +import { isChildSubreaper } from './process-reaping'; export const ROOT = resolve(import.meta.dir, '../..'); export const DIR = join(ROOT, 'scripts/ubicloud'); @@ -31,6 +32,7 @@ export const read = (rel: string) => readFileSync(join(ROOT, rel), 'utf8'); * make-.sh from the fixture root. No network, no VM. */ export function fakeVm(opts: { listed?: string; refuse?: string } = {}) { + if (isChildSubreaper()) throw new Error('this bun process is a child subreaper (an earlier test called prctl PR_SET_CHILD_SUBREAPER in-process, as lib/land-merged-checks.ts runMergedChecks does), so the fake VM\'s detached phases would stay unreaped zombies and run cases would hang (GST-1613)'); const root = mkdtempSync(join(tmpdir(), 'ubi-run-')); const bin = join(root, 'bin'), state = join(root, 'state'), log = join(root, 'calls.log'), sshLog = join(root, 'ssh.log'), record = join(root, 'vms.tsv'); const vm = join(root, 'vm'), vmbin = join(root, 'vmbin'), srcDir = join(root, 'src'), checkpoints = join(root, 'checkpoints'); diff --git a/test/land-direct-recovery-isolation.test.ts b/test/land-direct-recovery-isolation.test.ts new file mode 100644 index 000000000..2ee61c5f8 --- /dev/null +++ b/test/land-direct-recovery-isolation.test.ts @@ -0,0 +1,19 @@ +/** + * GST-1613: test/ship-queue-land-direct.test.ts and test/ubicloud-runner-recovery.test.ts in one bun process, as a + * free shard can pack them (shard 10, run 42207455885). The land file ran lib/land-merged-checks.ts runMergedChecks in + * the runner, whose prctl(PR_SET_CHILD_SUBREAPER) made the runner adopt every later orphan without reaping it, so the + * fake VM's detached phases stayed zombies that tail --pid and kill -0 count as alive and recovery's run cases hung at + * 90 s. The child selects the merged-checks case and one run case; recovery's afterAll fails on any zombie child. + */ +import { expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as path from 'node:path'; + +test.skipIf(process.platform === 'win32')('the land-direct merged-checks case and a recovery run case pass in one process with no zombie left', () => { + const r = spawnSync(process.execPath, ['test', 'test/ship-queue-land-direct.test.ts', 'test/ubicloud-runner-recovery.test.ts', '-t', 'runMergedChecks: HEAD is|run keeps the presigned URL'], + { cwd: path.resolve(import.meta.dir, '..'), encoding: 'utf8', timeout: 150_000 }); + const out = r.stdout + r.stderr; + expect(r.status, out).toBe(0); + expect(out.match(/^\(pass\) .*(runMergedChecks: HEAD is|run keeps the presigned URL)/gm) ?? [], out).toHaveLength(2); + expect(out, out).not.toMatch(/^\(fail\)/m); +}, 180_000); diff --git a/test/ship-queue-land-direct.test.ts b/test/ship-queue-land-direct.test.ts index aef9fa465..835afece2 100644 --- a/test/ship-queue-land-direct.test.ts +++ b/test/ship-queue-land-direct.test.ts @@ -18,7 +18,8 @@ import { persist, repoIdentity } from '../lib/ship-queue-train-record'; import { rewriteStamps, tentativeMentions } from '../lib/restamp'; import { bumpVersion } from '../lib/version-gate-train'; import { openRelease } from '../lib/release-stamp'; -import { runMergedChecks } from '../lib/land-merged-checks'; +import type { MergedRun } from '../lib/land-merged-checks'; +import { bunIsolationArgs } from '../lib/bun-isolation'; import { readLock, writeLock } from '../lib/ship-queue-land-lock'; import { flakyGit } from './helpers/flaky-git'; import { runBackstop } from '../scripts/train-backstop'; @@ -113,6 +114,13 @@ const landNotes = (fx: Fx) => { }; /** Whether a clone holds a commit. */ const spawnCheck = (dir: string, sha: string) => spawnSync('git', ['-C', dir, 'cat-file', '-e', `${sha}^{commit}`], { timeout: 30_000 }).status === 0; +/** runMergedChecks in a child Bun: on Linux it makes its caller a non-dumpable child subreaper for good, and a test runner that is one never reaps the orphans later test files leave, which hung test/ubicloud-runner-recovery.test.ts in the same process (GST-1613). */ +function runMergedChecks(root: string, base: string, pin: string, checks: string[], ms?: number): MergedRun { + const lib = JSON.stringify(path.join(import.meta.dir, '..', 'lib', 'land-merged-checks.ts')); + const r = spawnSync(process.execPath, [...bunIsolationArgs(), '-e', `const { runMergedChecks } = await import(${lib}); const [root, base, pin, checks, ms] = JSON.parse(await Bun.stdin.text()); console.log(JSON.stringify(runMergedChecks(root, base, pin, checks, ms ?? undefined)));`], { input: JSON.stringify([root, base, pin, checks, ms ?? null]), encoding: 'utf8', timeout: 60_000 }); + if (r.status !== 0) throw new Error(`runMergedChecks child exited ${r.status ?? r.signal}: ${r.stderr}`); + return JSON.parse(r.stdout) as MergedRun; +} /** Commit a file straight onto origin/main (another PR that landed first). */ function moveMain(fx: Fx, file: string, text: string): string { const { repo } = fx; diff --git a/test/test-of-test-ratchet.test.ts b/test/test-of-test-ratchet.test.ts index 5d6a40821..0df256000 100644 --- a/test/test-of-test-ratchet.test.ts +++ b/test/test-of-test-ratchet.test.ts @@ -132,6 +132,7 @@ const BASELINE = [ 'test/ios-qa-stateserver-hardening.test.ts', 'test/ios-qa-swift-build.test.ts', 'test/jargon-list.test.ts', + 'test/land-direct-recovery-isolation.test.ts', // GST-1613: runs test/ship-queue-land-direct.test.ts and test/ubicloud-runner-recovery.test.ts in one bun process; the product code is what those two files run 'test/llm-judge-frontier.test.ts', 'test/llm-judge-stream.test.ts', // landed in v1.91.7.0 before this ratchet: harness owner for its functional-QA/docsync paid evals 'test/memory-cache-injection.test.ts', diff --git a/test/ubicloud-runner-recovery.test.ts b/test/ubicloud-runner-recovery.test.ts index af2a29f6b..b401ce7c5 100644 --- a/test/ubicloud-runner-recovery.test.ts +++ b/test/ubicloud-runner-recovery.test.ts @@ -3,10 +3,11 @@ * caller's sink, and `recover` rebuilds a VM's local state on another controller * from the encrypted bundle `up` wrote. Offline: the fake API and fake VM only. */ -import { afterEach, describe, expect, test } from 'bun:test'; +import { afterAll, afterEach, describe, expect, test } from 'bun:test'; import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { creates, fakeVm, gitSrc, run, until, vmDirs, type FakeVm } from './helpers/ubi-runner-fake-vm'; +import { zombieChildren } from './helpers/process-reaping'; const fixtures: FakeVm[] = []; const make = () => { const f = fakeVm(); fixtures.push(f); return f; }; @@ -21,6 +22,10 @@ const stopSink = (f: FakeVm) => { afterEach(() => { for (const f of fixtures.splice(0)) { stopSink(f); rmSync(f.root, { recursive: true, force: true }); } }); +afterAll(async () => { + await until(() => zombieChildren().length === 0, 2_000).catch(() => {}); + expect(zombieChildren()).toEqual([]); +}, 10_000); const out = (r: { stdout: Buffer; stderr: Buffer }) => r.stdout.toString() + r.stderr.toString(); const tarList = (tgz: string) => Bun.spawnSync(['tar', '-tzf', tgz], { timeout: 10_000 }).stdout.toString(); const heartbeat = (tgz: string) => {