From 4f5bd2d9cbac8941963f636d9df8a546e89ac359 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 31 Oct 2026 18:32:34 +0000 Subject: [PATCH] test(free): split test/ubicloud-runner.test.ts over the 60 s seed ceiling; keep setup-codex-scope allowlisted The reseed at the main merge (free-durations-record run 41226501374, 221320e) put test/ubicloud-runner.test.ts, which main added after this branch's first reseed, at 127 s and test/setup-codex-scope.test.ts back at 66 s. The fake-API cases move into four parts by describe topic with their shared fake VM in test/helpers/ubi-runner-fake-vm.ts (46 tests before and after). The allowlist keeps main's setup-codex-scope entry. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- scripts/free-test-seed-allowlist.json | 3 +- scripts/lib/windows-curation.ts | 4 + test/helpers/ubi-runner-fake-vm.ts | 119 ++++ test/test-of-test-ratchet.test.ts | 2 + test/ubicloud-runner-reattach.test.ts | 157 +++++ test/ubicloud-runner-run.test.ts | 139 ++++ test/ubicloud-runner-tiers.test.ts | 182 ++++++ test/ubicloud-runner-verify-quota.test.ts | 184 ++++++ test/ubicloud-runner.test.ts | 732 +--------------------- 9 files changed, 790 insertions(+), 732 deletions(-) create mode 100644 test/helpers/ubi-runner-fake-vm.ts create mode 100644 test/ubicloud-runner-reattach.test.ts create mode 100644 test/ubicloud-runner-run.test.ts create mode 100644 test/ubicloud-runner-tiers.test.ts create mode 100644 test/ubicloud-runner-verify-quota.test.ts diff --git a/scripts/free-test-seed-allowlist.json b/scripts/free-test-seed-allowlist.json index a2d4233e8..c2492267e 100644 --- a/scripts/free-test-seed-allowlist.json +++ b/scripts/free-test-seed-allowlist.json @@ -3,6 +3,7 @@ {"file": "test/host-conformance.test.ts", "reason": "real ./setup per host and layout; the time is setup's own skill-linking loop, not fixture copies (shared fixture landed in W6c)"}, {"file": "test/shared-libs-stage-actor.test.ts", "reason": "re-runs the paid shared-libs runner body against git fixture repos per test; owned by the paid-harness value-bar review"}, {"file": "test/setup-install-registry.test.ts", "reason": "82s in the 2026-10-15 recording (58s on 2026-10-04): 11 cases each run the real ./setup, gstack-relink, gstack-config and gstack-uninstall against throwaway checkouts", "todo": "split by describe block so the packer can balance it, then rerun bun run test:ubicloud --record-durations (TODOS.md, fix wave 2 follow-ups)"}, - {"file": "test/brain-sync.test.ts", "reason": "89s in the 2026-10-19 weekly recording (58s on 2026-10-15): #3254 added the case-collision suite, and every case drains a real gstack-brain-sync against a local bare remote", "todo": "split by describe block so the packer can balance it, then rerun bun run test:ubicloud --record-durations (TODOS.md, fix wave 2 follow-ups)"} + {"file": "test/brain-sync.test.ts", "reason": "89s in the 2026-10-19 weekly recording (58s on 2026-10-15): #3254 added the case-collision suite, and every case drains a real gstack-brain-sync against a local bare remote", "todo": "split by describe block so the packer can balance it, then rerun bun run test:ubicloud --record-durations (TODOS.md, fix wave 2 follow-ups)"}, + {"file": "test/setup-codex-scope.test.ts", "reason": "64s in the 2026-10-19 weekly recording (59s on 2026-10-15): each case runs the real ./setup twice against a Codex install fixture", "todo": "split by describe block so the packer can balance it, then rerun bun run test:ubicloud --record-durations (TODOS.md, fix wave 2 follow-ups)"} ] } diff --git a/scripts/lib/windows-curation.ts b/scripts/lib/windows-curation.ts index 0510a41cc..e103665d9 100644 --- a/scripts/lib/windows-curation.ts +++ b/scripts/lib/windows-curation.ts @@ -327,6 +327,10 @@ export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }> file: 'test/ship-queue-train-verify.test.ts', reason: 'split from test/ship-queue-train.test.ts (GSTA-151): drives bin/gstack-ship-queue against the same #!/bin/sh gh shim in test/helpers/ship-queue-train-fixture.ts', }, + { + file: 'test/ubicloud-runner-tiers.test.ts', + reason: 'split from test/ubicloud-runner.test.ts, excluded as a whole: runs scripts/ubicloud/ubi-runner.sh through bash with the #!/usr/bin/env bash curl, ssh and make shims test/helpers/ubi-runner-fake-vm.ts puts on a colon-joined PATH, like test/ubicloud-runner-run.test.ts', + }, { file: 'test/ship-queue-train-freeze.test.ts', reason: 'split from test/ship-queue-train.test.ts (GSTA-151): drives bin/gstack-ship-queue against the same #!/bin/sh gh shim in test/helpers/ship-queue-train-fixture.ts', diff --git a/test/helpers/ubi-runner-fake-vm.ts b/test/helpers/ubi-runner-fake-vm.ts new file mode 100644 index 000000000..20a6a8514 --- /dev/null +++ b/test/helpers/ubi-runner-fake-vm.ts @@ -0,0 +1,119 @@ +/** + * Fake Ubicloud API and fake VM for scripts/ubicloud/ubi-runner.sh, shared by the + * test/ubicloud-runner-*.test.ts parts split from test/ubicloud-runner.test.ts (GSTA-151). + */ +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +export const ROOT = resolve(import.meta.dir, '../..'); +export const DIR = join(ROOT, 'scripts/ubicloud'); +export const read = (rel: string) => readFileSync(join(ROOT, rel), 'utf8'); + +/** + * Fake Ubicloud API that can create: create adds the name to the listed file + * (or answers the quota refusal in refuse with HTTP 400), show is running at + * 192.0.2.1 while listed, destroy unlists. The fake VM is the directory vm/: + * the fake ssh records every remote command in ssh.log, keeps copies of + * ~/.ubirun-env and ~/.ubirun-setup.sh under the fixture root, serves pulls + * from remote-fs/, and runs every other command there for real (HOME=vm/, + * vmbin/ first on PATH), so uploads unpack, detached phases write their pid, + * output and exit files, tail --pid follows them and pkill ends them. The stub + * make in vmbin/ echoes `ran: make ARGS`, or runs command-mode or + * make-.sh from the fixture root. No network, no VM. + */ +export function fakeVm(opts: { listed?: string; refuse?: string } = {}) { + const root = mkdtempSync(join(tmpdir(), 'ubi-run-')); + const bin = join(root, 'bin'), state = join(root, 'state'), log = join(root, 'calls.log'), sshLog = join(root, 'ssh.log'), record = join(root, 'vms.tsv'); + const vm = join(root, 'vm'), vmbin = join(root, 'vmbin'), srcDir = join(root, 'src'), checkpoints = join(root, 'checkpoints'); + mkdirSync(vm); mkdirSync(vmbin); mkdirSync(srcDir); + writeFileSync(join(srcDir, 'README'), 'fixture checkout\n'); + mkdirSync(bin); mkdirSync(state, { recursive: true }); + writeFileSync(join(root, 'listed'), opts.listed ?? ''); + writeFileSync(join(bin, 'curl'), '#!/usr/bin/env bash\n' + + 'out=""; while [ $# -gt 0 ]; do [ "$1" = -o ] && out=$2; shift; done\n' + + 'body=$(cat); echo "$body" >> ' + JSON.stringify(log) + '\n' + + 'listed=' + JSON.stringify(join(root, 'listed')) + '\n' + + 'loc=$(printf \'%s\' "$body" | sed -n \'s/.*"vm", *"\\([^/"]*\\)\\/[^"]*".*/\\1/p\')\n' + + 'name=$(printf \'%s\' "$body" | sed -n \'s/.*"vm", *"[^/"]*\\/\\([^"]*\\)".*/\\1/p\')\n' + + 'case "$body" in\n' + + ' *\'"create"\'*) if [ -n ' + JSON.stringify(opts.refuse ?? '') + ' ]; then printf \'%s\' ' + JSON.stringify(opts.refuse ?? '') + ' > "$out"; printf 400; else echo "$loc $name" >> "$listed"; : > "$out"; printf 200; fi ;;\n' + + ' *destroy*) : > "$out"; if [ -f "$listed.destroy-fails" ]; then printf 500; else grep -v " $name$" "$listed" > "$listed.tmp"; mv "$listed.tmp" "$listed"; printf 200; fi ;;\n' + + ' *\'"list"\'*) if [ -f "$listed.list-fails" ]; then echo down > "$out"; printf 503; else cat "$listed" > "$out"; printf 200; fi ;;\n' + + ' *\'"show"\'*) [ -z "${FAKE_SHOW_DELAY:-}" ] || sleep "$FAKE_SHOW_DELAY"; if grep -q " $name$" "$listed"; then printf \'state: running\\nsize: standard-16\\nip4: 192.0.2.1\\n\' > "$out"; printf 200; else : > "$out"; printf 404; fi ;;\n' + + ' *) : > "$out"; printf 200 ;;\n' + + 'esac\n'); + writeFileSync(join(bin, 'ssh'), '#!/usr/bin/env bash\n' + + 'root=' + JSON.stringify(root) + '; vm="$root/vm"\n' + + 'cmd=${*: -1}\n' + + 'printf \'%s\\n\' "$cmd" >> "$root/ssh.log"\n' + + 'export HOME="$vm" PATH="$root/vmbin:$PATH"\n' + + 'cd "$vm" || exit 255\n' + + 'case "$cmd" in\n' + + ' *pkill*) echo "$cmd" >> "$root/cancel.log"; exec bash -c "$cmd" ;;\n' + + ' *captured*) cat > "$root/captured" ;;\n' + + ' *"cat > ~/.ubirun-env"*) tee "$root/ubirun-env" > "$vm/.ubirun-env" ;;\n' + + ' *"cat > ~/.ubirun-setup.sh"*) tee "$root/ubirun-setup.sh" > "$vm/.ubirun-setup.sh" ;;\n' + + ' *"tar -czf -"*) base=${cmd##* }; tar -czf - -C "$root/remote-fs" "$base" ;;\n' + + ' *"ls -d -- "*) : ;;\n' + + ' *) exec bash -c "$cmd" ;;\n' + + 'esac\n'); + writeFileSync(join(vmbin, 'make'), '#!/usr/bin/env bash\n' + + 'root=' + JSON.stringify(root) + '\n' + + 'if [ -f "$root/command-mode" ]; then exec bash -c "$(cat "$root/command-mode")"; fi\n' + + 'echo "ran: make $*"\n' + + '[ ! -f "$root/make-$1.sh" ] || exec bash "$root/make-$1.sh"\n'); + chmodSync(join(bin, 'curl'), 0o755); chmodSync(join(bin, 'ssh'), 0o755); chmodSync(join(vmbin, 'make'), 0o755); + const env: Record = { ...(process.env as Record), PATH: `${bin}:${process.env.PATH}`, UBICLOUD_API_KEY: 'offline', UBI_RUNNER_STATE: state, UBI_POLL_SECONDS: '1', UBI_VM_RECORD: record, UBI_OWNER: 'thrd1', UBI_HEARTBEAT: '0', UBI_CHECKPOINT_DIR: checkpoints }; + const run = (args: string[], extra: Record = {}, stdin?: string) => + Bun.spawnSync(['bash', join(DIR, 'ubi-runner.sh'), ...args], { env: { ...env, ...extra }, stdin: stdin === undefined ? undefined : Buffer.from(stdin), timeout: 90_000 }); + const spawn = (args: string[], extra: Record = {}) => + Bun.spawn(['bash', join(DIR, 'ubi-runner.sh'), ...args], { env: { ...env, ...extra }, stdin: 'ignore', stdout: 'pipe', stderr: 'pipe', timeout: 120_000 }); + return { + root, state, record, run, spawn, vm, src: srcDir, checkpoints, + calls: () => (existsSync(log) ? readFileSync(log, 'utf8') : ''), + ssh: () => (existsSync(sshLog) ? readFileSync(sshLog, 'utf8') : ''), + listed: () => readFileSync(join(root, 'listed'), 'utf8'), + }; +} + +export const src = (f: { root: string }, cfg: unknown) => { + const dir = join(f.root, 'repo'); + mkdirSync(join(dir, '.gstack'), { recursive: true }); + mkdirSync(join(dir, 'scripts'), { recursive: true }); + writeFileSync(join(dir, 'scripts/boot.sh'), 'echo boot\n'); + writeFileSync(join(dir, '.gstack/ubicloud.json'), JSON.stringify(cfg)); + return dir; +}; + +export const run = (f: ReturnType, args: string[], extra: Record = {}, stdin?: string) => f.run(args, extra, stdin); + +// GSTA-158 P3: detached phases, reattach and tier checkpoints. +export const until = async (cond: () => boolean, ms: number) => { + const deadline = Date.now() + ms; + while (!cond()) { if (Date.now() > deadline) throw new Error('condition not met in time'); await Bun.sleep(100); } +}; +export const vmDirs = (f: ReturnType) => (existsSync(f.state) ? readdirSync(f.state).filter(d => d.startsWith('ubirun-')) : []); +export const phaseStarted = (f: ReturnType, phase: string) => vmDirs(f).some(d => existsSync(join(f.state, d, 'phase.' + phase + '.start'))); +export const creates = (f: ReturnType) => f.calls().split('\n').filter(l => l.includes('"create"')).length; +export const killSession = async (f: ReturnType, phase: string) => { + const pidFile = join(f.vm, '.ubirun-' + phase + '.pid'); + await until(() => existsSync(pidFile) && readFileSync(pidFile, 'utf8').trim() !== '', 10_000); + const pid = Number(readFileSync(pidFile, 'utf8').trim()); + try { process.kill(-pid, 'SIGKILL'); } catch { /* already gone */ } + await until(() => { try { process.kill(pid, 0); return false; } catch { return true; } }, 10_000); +}; +export const gitSrc = (f: ReturnType, cfg?: unknown) => { + const dir = cfg === undefined ? join(f.root, 'repo') : src(f, cfg); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, 'README'), 'fixture repo\n'); + for (const args of [['init', '-q', '-b', 'main'], ['add', '-A'], ['commit', '-qm', 'fixture']]) { + const r = Bun.spawnSync(['git', '-c', 'user.email=t@t', '-c', 'user.name=t', ...args], { cwd: dir, timeout: 30_000 }); + if (r.exitCode !== 0) throw new Error('git ' + args.join(' ') + ': ' + r.stderr.toString()); + } + return dir; +}; +export const TIERS = { setup: 'scripts/boot.sh', tiers: [{ name: 't1', command: 'make t1' }, { name: 't2', command: 'make t2' }, { name: 't3', command: 'make t3' }] }; +export const bootCounts = (f: ReturnType, dir: string) => writeFileSync(join(dir, 'scripts/boot.sh'), 'echo boot >> ~/setup-count\n'); +export const verifyLine = (out: string) => out.split('\n').find(l => l.startsWith('UBI_VERIFY: ')) ?? ''; +export type FakeVm = ReturnType; diff --git a/test/test-of-test-ratchet.test.ts b/test/test-of-test-ratchet.test.ts index 32a9b210f..cad251bda 100644 --- a/test/test-of-test-ratchet.test.ts +++ b/test/test-of-test-ratchet.test.ts @@ -238,6 +238,8 @@ const BASELINE = [ 'test/tasks-section-jq.test.ts', 'test/test-free-shards-capture.test.ts', 'test/timeline.test.ts', + 'test/ubicloud-runner-reattach.test.ts', // GSTA-151: split from test/ubicloud-runner.test.ts for shard time; runs scripts/ubicloud/ubi-runner.sh through test/helpers/ubi-runner-fake-vm.ts + 'test/ubicloud-runner-tiers.test.ts', // GSTA-151: split from test/ubicloud-runner.test.ts for shard time; runs scripts/ubicloud/ubi-runner.sh through test/helpers/ubi-runner-fake-vm.ts 'test/workflow-judge-cache.test.ts', // slices test/skill-llm-eval.test.ts source (recorded 2026-10 when the ratchet learned paid-source slicing) 'test/workflow-judge-input.test.ts', // slices test/skill-llm-eval.test.ts source (recorded 2026-10 when the ratchet learned paid-source slicing) ]; diff --git a/test/ubicloud-runner-reattach.test.ts b/test/ubicloud-runner-reattach.test.ts new file mode 100644 index 000000000..1f9de77a9 --- /dev/null +++ b/test/ubicloud-runner-reattach.test.ts @@ -0,0 +1,157 @@ +/** + * ubi-runner detached phases, reattach, signals and the run key (GSTA-158 P3), against the fake Ubicloud API and fake ssh in + * test/helpers/ubi-runner-fake-vm.ts. Split from test/ubicloud-runner.test.ts (GSTA-151). + */ +import { describe, expect, test } from 'bun:test'; +import { existsSync, lstatSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { DIR, fakeVm, src, run, until, vmDirs, phaseStarted, creates, killSession, gitSrc } from './helpers/ubi-runner-fake-vm'; + +describe('ubi-runner run, verify and quota against a fake API and a fake ssh', () => { + test('run: a SIGKILLed runner leaves its VM; the same command reattaches, skips the finished setup and reports the command\'s exit', async () => { + const f = fakeVm(); + try { + const repo = gitSrc(f); + const setup = join(f.root, 'setup.sh'); writeFileSync(setup, 'echo setup >> ~/setup-count\n'); + writeFileSync(join(f.root, 'make-test.sh'), 'echo begin; sleep 4; echo end; exit 3\n'); + const args = ['run', '--src', repo, '--setup', setup, '--', 'make test']; + const a = f.spawn(args); + await until(() => vmDirs(f).some(d => existsSync(join(f.state, d, 'command.out')) && readFileSync(join(f.state, d, 'command.out'), 'utf8').includes('begin')), 30_000); + process.kill(a.pid, 'SIGKILL'); + await a.exited; + const name = vmDirs(f)[0]!; + expect(f.listed()).toContain(name); + const b = f.run(args); + const err = b.stderr.toString(); + expect(b.exitCode, err).toBe(3); + expect(err).toMatch(new RegExp('ubi-runner: reattached to ' + name + ' created_at=\\d{10} key=[0-9a-f]{12}\n')); + expect(b.stdout.toString().split('\n').filter(l => l === 'begin' || l === 'end')).toEqual(['begin', 'end']); + expect(err).toMatch(/command exited 3 after \d+s/); + expect(creates(f)).toBe(1); + expect(readFileSync(join(f.vm, 'setup-count'), 'utf8')).toBe('setup\n'); + expect(f.listed()).toBe(''); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 90_000); + + test('run: a second live runner of the same command refuses with UBI_ATTACH_BUSY and creates nothing', async () => { + const f = fakeVm(); + try { + const repo = gitSrc(f); + writeFileSync(join(f.root, 'make-test.sh'), 'sleep 4; echo done\n'); + const args = ['run', '--src', repo, '--', 'make test']; + const a = f.spawn(args); + await until(() => phaseStarted(f, 'command'), 30_000); + const b = f.run(args); + expect(b.exitCode).toBe(1); + expect(b.stderr.toString()).toMatch(new RegExp('UBI_ATTACH_BUSY: vm=' + vmDirs(f)[0] + ' key=[0-9a-f]{12}: runner pid ' + a.pid + ' ')); + expect(await a.exited).toBe(0); + expect(await new Response(a.stdout).text()).toContain('done'); + expect(creates(f)).toBe(1); + expect(f.listed()).toBe(''); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 90_000); + + test('run: HUP once the command is live detaches (UBI_DETACHED, exit 129, VM kept, no pass value on disk); a rerun reattaches and finishes', async () => { + const f = fakeVm(); + try { + const repo = gitSrc(f); + writeFileSync(join(f.root, 'make-test.sh'), 'sleep 3; echo finished\n'); + const args = ['run', '--src', repo, '--pass', 'UBI_TEST_TOKEN', '--', 'make test']; + const a = f.spawn(args, { UBI_TEST_TOKEN: 'hup-secret-value' }); + await until(() => phaseStarted(f, 'command'), 30_000); + await Bun.sleep(500); + process.kill(a.pid, 'SIGHUP'); + expect(await a.exited).toBe(129); + const name = vmDirs(f)[0]!; + const err = await new Response(a.stderr).text(); + expect(err).toMatch(new RegExp('UBI_DETACHED: vm=' + name + ' key=[0-9a-f]{12} reattach: rerun the same command')); + expect(f.listed()).toContain(name); + expect(f.calls()).not.toContain('destroy'); + for (const file of readdirSync(join(f.state, name))) { + const p = join(f.state, name, file); + if (lstatSync(p).isFile()) expect(readFileSync(p, 'utf8')).not.toContain('hup-secret-value'); + } + const b = f.run(args, { UBI_TEST_TOKEN: 'hup-secret-value' }); + expect(b.exitCode, b.stderr.toString()).toBe(0); + expect(b.stderr.toString()).toContain('reattached to ' + name); + expect(b.stdout.toString()).toContain('finished'); + expect(creates(f)).toBe(1); + expect(f.listed()).toBe(''); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 90_000); + + test('run: INT and TERM during a live command still tear the VM down', async () => { + for (const sig of ['SIGTERM', 'SIGINT'] as const) { + const f = fakeVm(); + try { + const repo = gitSrc(f); + writeFileSync(join(f.root, 'make-test.sh'), 'sleep 3\n'); + const a = f.spawn(['run', '--src', repo, '--', 'make test']); + await until(() => phaseStarted(f, 'command'), 30_000); + await Bun.sleep(500); + process.kill(a.pid, sig); + expect(await a.exited).toBe(130); + expect(await new Response(a.stderr).text()).toMatch(/destroyed eu-central-h1\/ubirun-thrd1-/); + expect(f.listed()).toBe(''); + } finally { rmSync(f.root, { recursive: true, force: true }); } + } + }, 90_000); + + test('run: a remote session gone without an exit status exits 125 with UBI_ATTACH_LOST and the VM name', async () => { + const f = fakeVm(); + try { + const repo = gitSrc(f); + writeFileSync(join(f.root, 'make-test.sh'), 'sleep 30\n'); + const args = ['run', '--src', repo, '--', 'make test']; + const a = f.spawn(args); + await until(() => existsSync(join(f.vm, '.ubirun-command.pid')) && readFileSync(join(f.vm, '.ubirun-command.pid'), 'utf8').trim() !== '', 30_000); + process.kill(a.pid, 'SIGKILL'); + await a.exited; + await killSession(f, 'command'); + const name = vmDirs(f)[0]!; + const b = f.run(args); + expect(b.exitCode).toBe(125); + expect(b.stderr.toString()).toContain('UBI_ATTACH_LOST: vm=' + name + ' phase=command'); + expect(f.listed()).toBe(''); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 90_000); + + test('run: the run key covers tree, command, setup content, env, pass value, image, origin/HEAD and the runner itself; a pass value never appears in it', () => { + const f = fakeVm(); + try { + const repo = gitSrc(f); + const setup = join(f.root, 'setup.sh'); writeFileSync(setup, 'echo one\n'); + const runner = join(f.root, 'runner-copy.sh'); writeFileSync(runner, readFileSync(join(DIR, 'ubi-runner.sh'), 'utf8')); + const key = (opts: { args?: string[]; env?: Record; cmd?: string; script?: string } = {}) => { + const r = Bun.spawnSync(['bash', opts.script ?? join(DIR, 'ubi-runner.sh'), 'run', '--probe', '--src', repo, '--setup', setup, '--env', 'MODE=a', '--pass', 'UBI_TEST_TOKEN', ...(opts.args ?? []), '--', opts.cmd ?? 'make test'], + { env: { ...process.env, PATH: join(f.root, 'bin') + ':' + process.env.PATH, UBICLOUD_API_KEY: 'offline', UBI_RUNNER_STATE: f.state, UBI_OWNER: 'thrd1', UBI_TEST_TOKEN: 'v1', ...(opts.env ?? {}) }, timeout: 30_000 }); + const out = r.stdout.toString() + r.stderr.toString(); + expect(out).not.toContain('v1'); + const m = /^UBI_ATTACH_INFO: none key=([0-9a-f]{12})$/m.exec(out); + if (!m) throw new Error(out); + return m[1]!; + }; + const base = key(); + expect(key()).toBe(base); + const variants: Record = {}; + variants.command = key({ cmd: 'make check' }); + variants.env = key({ args: ['--env', 'MODE=b'] }); + variants.pass = key({ env: { UBI_TEST_TOKEN: 'v2' } }); + variants.image = key({ args: ['-b', 'ubuntu-jammy'] }); + variants.size = key({ args: ['-s', 'standard-8'] }); + writeFileSync(runner, readFileSync(runner, 'utf8') + '# edited\n'); + variants.runner = key({ script: runner }); + writeFileSync(setup, 'echo two\n'); + variants.setup = key(); + writeFileSync(setup, 'echo one\n'); + expect(key()).toBe(base); + Bun.spawnSync(['git', 'update-ref', 'refs/remotes/origin/HEAD', 'HEAD'], { cwd: repo, timeout: 10_000 }); + variants.base = key(); + writeFileSync(join(repo, 'new-file.txt'), 'untracked source\n'); + variants.tree = key(); + for (const [what, k] of Object.entries(variants)) expect({ what, same: k === base }).toEqual({ what, same: false }); + expect(new Set(Object.values(variants)).size).toBe(Object.keys(variants).length); + expect(f.calls()).not.toContain('"create"'); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 60_000); +}); diff --git a/test/ubicloud-runner-run.test.ts b/test/ubicloud-runner-run.test.ts new file mode 100644 index 000000000..81890df77 --- /dev/null +++ b/test/ubicloud-runner-run.test.ts @@ -0,0 +1,139 @@ +/** + * ubi-runner run: watchdog, timeouts, teardown accounting, ssh dispatch and --pass, against the fake Ubicloud API and fake ssh in + * test/helpers/ubi-runner-fake-vm.ts. Split from test/ubicloud-runner.test.ts (GSTA-151). + */ +import { describe, expect, test } from 'bun:test'; +import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { read, fakeVm, src, run } from './helpers/ubi-runner-fake-vm'; + +describe('ubi-runner run, verify and quota against a fake API and a fake ssh', () => { + test('run: a hung setup is killed by the watchdog, exits 124, never reaches the command, and the VM is destroyed', () => { + const f = fakeVm(); + try { + const setup = join(f.root, 'setup.sh'); writeFileSync(setup, 'echo "setup start"; sleep 20\n'); + const r = run(f, ['run', '--src', f.src, '--setup', setup, '--', 'make test'], { UBI_IDLE_TIMEOUT: '2' }); + const err = r.stderr.toString(); + expect(r.exitCode, err).toBe(124); + expect(err).toMatch(/watchdog: setup printed nothing for \d+s \(UBI_IDLE_TIMEOUT=2\)/); + expect(err).toContain('setup failed (exit 124)'); + expect(f.ssh()).not.toContain('bash -lc'); + expect(err).toContain('setup start'); + // The setup runs detached in its own session: its leader's pid, both streams and the exit status are files on the VM. + expect(f.ssh()).toMatch(/setsid -f bash -c 'echo \$\$ >~\/\.ubirun-setup\.pid\.tmp && mv ~\/\.ubirun-setup\.pid\.tmp ~\/\.ubirun-setup\.pid; \( cd work\/[^ ]+ && \. ~\/\.ubirun-env && bash -l ~\/\.ubirun-setup\.sh \) >~\/\.ubirun-setup\.out 2>~\/\.ubirun-setup\.err <\/dev\/null; echo \$\? >~\/\.ubirun-setup\.exit\.tmp && mv ~\/\.ubirun-setup\.exit\.tmp ~\/\.ubirun-setup\.exit' <\/dev\/null/m); + expect(readFileSync(join(f.root, 'cancel.log'), 'utf8')).toMatch(/sid=\$\(cat ~\/\.ubirun-setup\.pid 2>\/dev\/null\); rm -f ~\/\.ubirun-setup\.pid; \[ -z "\$sid" \] \|\| \{ pkill -TERM -s "\$sid"; sleep 2; pkill -KILL -s "\$sid"; \}/); + expect(existsSync(join(f.vm, '.ubirun-setup.exit'))).toBe(false); + expect(f.calls()).toContain('destroy'); + expect(err).toMatch(/destroyed eu-central-h1\/ubirun-thrd1-\d{10}-[0-9a-f]{8}-t3/); + expect(f.listed()).toBe(''); + expect(readFileSync(f.record, 'utf8')).toBe(''); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 60_000); + + test('run: UBI_RUN_TIMEOUT kills a command that keeps printing, with exit 124 and its reason', () => { + const f = fakeVm(); + try { + writeFileSync(join(f.root, 'command-mode'), 'for i in 1 2 3 4 5 6 7 8 9 10; do echo "tick $i"; sleep 1; done'); + const r = run(f, ['run', '--src', f.src, '--', 'make test'], { UBI_RUN_TIMEOUT: '3' }); + expect(r.exitCode).toBe(124); + expect(r.stderr.toString()).toMatch(/watchdog: command ran \d+s \(UBI_RUN_TIMEOUT=3\); killing it/); + expect(r.stderr.toString()).toContain('command exited 124 after'); + expect(f.calls()).toContain('destroy'); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 60_000); + + test('run: ends with UBICLOUD: left=0 for its own VM; a same-owner sibling is counted as information and does not fail the run', () => { + const f = fakeVm({ listed: 'eu-central-h1 ubirun-thrd1-9999999999-sib1-t3\n' }); + try { + const leftFile = join(f.root, 'left'); + const r = f.run(['run', '--src', f.src, '--', 'true'], { UBI_REAP: '0', UBI_LEFT_FILE: leftFile }); + const err = r.stderr.toString(); + expect(r.exitCode, err).toBe(0); + expect(err).toMatch(/UBICLOUD: left=0 owner=thrd1 vm=ubirun-thrd1-\d{10}-[0-9a-f]{8}-t3 others=1\n/); + expect(f.listed()).toBe('eu-central-h1 ubirun-thrd1-9999999999-sib1-t3\n'); + // Value: protects=UBI_LEFT_FILE carries the same leftover line pregate-remote.sh parses, apart from the command's output; fails_when=the file is not written or its format drifts from pregate-remote.sh's parser; why_new=the pregate stub hand-copies the format; seam=none; no_claim=real VM teardown (fake API and ssh) + expect(readFileSync(leftFile, 'utf8')).toMatch(/^UBICLOUD: left=0 owner=thrd1 vm=ubirun-thrd1-\d{10}-[0-9a-f]{8}-t3 others=1\n$/); + const parser = read('scripts/ubicloud/pregate-remote.sh').match(/left=\$\(sed -n '([^']+)' "\$log\.left"/)![1]; + expect(Bun.spawnSync(['sed', '-n', parser, leftFile], { timeout: 30_000 }).stdout.toString()).toBe('0\n'); + // Value: protects=run --keep reports its live VM as UBICLOUD: left=1 ... kept=1 and never a clean left=0; fails_when=the kept line is dropped or --keep still prints the teardown line; why_new=no test runs run --keep; seam=none; no_claim=real VM lifetime (fake API and ssh) + const kept = f.run(['run', '--src', f.src, '--keep', '--', 'true'], { UBI_REAP: '0' }); + const keptErr = kept.stderr.toString(); + expect(kept.exitCode, keptErr).toBe(0); + expect(keptErr).toMatch(/UBICLOUD: left=1 owner=thrd1 vm=ubirun-thrd1-\d{10}-[0-9a-f]{8}-t3 kept=1\n/); + expect(keptErr).not.toContain('left=0'); + expect(f.listed()).toMatch(/ubirun-thrd1-\d{10}-[0-9a-f]{8}-t3\n/); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 30_000); + + test('run: a passing command whose teardown fails exits 1 with UBICLOUD: left=1 and the down command; an unreadable VM list fails closed', () => { + const f = fakeVm(); + try { + writeFileSync(join(f.root, 'listed.destroy-fails'), ''); + const leaked = f.run(['run', '--src', f.src, '--', 'true'], { UBI_REAP: '0', UBI_DOWN_TIMEOUT: '3' }); + const err = leaked.stderr.toString(); + expect(leaked.exitCode, err).toBe(1); + expect(err).toMatch(/UBICLOUD: left=1 owner=thrd1 vm=(ubirun-thrd1-\d{10}-[0-9a-f]{8}-t3) others=0\n[^]*VM cleanup not confirmed for \1 \(left=1\); destroy it with: \S+ down \1/); + expect(f.listed()).toContain('ubirun-thrd1-'); + // Value: protects=a failing command whose VM also leaks keeps its own exit status, not 1; fails_when=run_teardown exits 1 whenever left is not 0; why_new=the leak case above only runs a passing command; seam=none; no_claim=real destroy failure modes (fake API returns 500) + writeFileSync(join(f.root, 'command-mode'), 'exit 7'); + const failedLeak = f.run(['run', '--src', f.src, '--', 'make test'], { UBI_REAP: '0', UBI_DOWN_TIMEOUT: '3' }); + expect(failedLeak.exitCode, failedLeak.stderr.toString()).toBe(7); + expect(failedLeak.stderr.toString()).toMatch(/UBICLOUD: left=1 owner=thrd1 vm=ubirun-thrd1-\d{10}-[0-9a-f]{8}-t3 others=1\n/); + const blind = fakeVm(); + try { + writeFileSync(join(blind.root, 'bin/curl.real'), readFileSync(join(blind.root, 'bin/curl'))); + chmodSync(join(blind.root, 'bin/curl.real'), 0o755); + writeFileSync(join(blind.root, 'bin/curl'), '#!/usr/bin/env bash\nif [ -f ' + JSON.stringify(join(blind.root, 'armed')) + ' ]; then touch ' + JSON.stringify(join(blind.root, 'listed.list-fails')) + '; fi\nexec ' + JSON.stringify(join(blind.root, 'bin/curl.real')) + ' "$@"\n'); + chmodSync(join(blind.root, 'bin/curl'), 0o755); + writeFileSync(join(blind.root, 'command-mode'), 'touch ' + JSON.stringify(join(blind.root, 'armed'))); + const r = blind.run(['run', '--src', blind.src, '--', 'make test'], { UBI_REAP: '0' }); + expect(r.exitCode, r.stderr.toString()).toBe(1); + expect(r.stderr.toString()).toMatch(/UBICLOUD: left=unknown owner=thrd1 vm=ubirun-thrd1-\d{10}-[0-9a-f]{8}-t3 others=unknown\n/); + } finally { rmSync(blind.root, { recursive: true, force: true }); } + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 60_000); + + test('run: the fake ssh dispatches on the remote command, so a checkout whose name holds -O still syncs and runs (GSTA-139)', () => { + const f = fakeVm(); + try { + const checkout = join(f.root, 'ubi-run-O1x2y3'); + mkdirSync(checkout); + const r = run(f, ['run', '--src', checkout, '--', 'make test']); + expect(r.exitCode, r.stderr.toString()).toBe(0); + expect(r.stdout.toString()).toContain('ran: '); + expect(f.ssh()).toContain('tar -xzf - -C work/ubi-run-O1x2y3'); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 60_000); + + test('run: --pass forwards a named local variable only; UBICLOUD_* and non-names are refused before any API call', () => { + const f = fakeVm(); + try { + const ok = run(f, ['run', '--src', f.src, '--pass', 'UBI_TEST_TOKEN', '--env', 'CI=1', '--', 'make test'], { UBI_TEST_TOKEN: 'sekret value' }); + expect(ok.exitCode).toBe(0); + expect(ok.stderr.toString()).toContain('command exited 0 after'); + const envFile = readFileSync(join(f.root, 'ubirun-env'), 'utf8'); + expect(envFile).toContain('export UBI_TEST_TOKEN=sekret\\ value'); + expect(envFile).toContain('export CI=1'); + expect(envFile).not.toContain('UBICLOUD'); + expect(ok.stdout.toString() + ok.stderr.toString()).not.toContain('sekret'); + for (const bad of ['UBICLOUD_API_KEY', 'UBICLOUD_API_TOKEN', 'X;Y', 'HOME/..']) { + const before = f.calls(); + const r = run(f, ['run', '--src', f.src, '--pass', bad, '--', 'true']); + expect(r.exitCode).toBe(1); + expect(r.stderr.toString()).toMatch(bad.startsWith('UBICLOUD_') ? /never forwards its own Ubicloud credentials/ : /not a variable name/); + expect(f.calls()).toBe(before); + } + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 60_000); + + test('ssh NAME COMMAND hands the caller’s stdin to the remote command', () => { + const f = fakeVm({ listed: 'eu-central-h1 vm-a\n' }); + try { + mkdirSync(join(f.state, 'vm-a'), { recursive: true }); + writeFileSync(join(f.state, 'vm-a/env'), 'NAME=vm-a\nLOCATION=eu-central-h1\nSIZE=standard-16\nIP=192.0.2.1\nCREATED=yes\nCREATE_AT=0\nCREATE_PID=1\n'); + const r = run(f, ['ssh', 'vm-a', 'cat > captured'], { UBI_IDLE_TIMEOUT: '0' }, 'payload line\n'); + expect(r.exitCode).toBe(0); + expect(readFileSync(join(f.root, 'captured'), 'utf8')).toBe('payload line\n'); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 30_000); +}); diff --git a/test/ubicloud-runner-tiers.test.ts b/test/ubicloud-runner-tiers.test.ts new file mode 100644 index 000000000..17019e59f --- /dev/null +++ b/test/ubicloud-runner-tiers.test.ts @@ -0,0 +1,182 @@ +/** + * ubi-runner verify tiers and their checkpoints (GSTA-158 P3), against the fake Ubicloud API and fake ssh in + * test/helpers/ubi-runner-fake-vm.ts. Split from test/ubicloud-runner.test.ts (GSTA-151). + */ +import { describe, expect, test } from 'bun:test'; +import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { DIR, fakeVm, src, run, until, vmDirs, creates, killSession, gitSrc, TIERS, bootCounts, verifyLine } from './helpers/ubi-runner-fake-vm'; + +describe('ubi-runner run, verify and quota against a fake API and a fake ssh', () => { + test('verify tiers: config refuses command with tiers, bad and duplicate tier names, before any API call', () => { + const f = fakeVm(); + try { + const cases: Array<[unknown, RegExp]> = [ + [{ command: 'make', tiers: [{ name: 'a', command: 'x' }] }, /UBI_CONFIG_TIERS_AND_COMMAND/], + [{ tiers: [{ name: 'Unit', command: 'x' }] }, /UBI_CONFIG_TIER_NAME: tier name 'Unit' must match/], + [{ tiers: [{ name: '-x', command: 'x' }] }, /UBI_CONFIG_TIER_NAME: tier name '-x'/], + [{ tiers: [{ name: 'a'.repeat(33), command: 'x' }] }, /UBI_CONFIG_TIER_NAME/], + [{ tiers: [{ name: 'a.b', command: 'x' }] }, /UBI_CONFIG_TIER_NAME/], + [{ tiers: [{ name: 'a', command: 'x' }, { name: 'a', command: 'y' }] }, /UBI_CONFIG_TIER_NAME: tier name 'a' appears twice/], + [{ tiers: [{ name: 'a' }] }, /UBI_CONFIG_TIER_NAME: tier 'a' needs a command/], + [{ tiers: [] }, /UBI_CONFIG_TIER_NAME: tiers must be a non-empty list/], + ]; + for (const [cfg, want] of cases) { + const dir = src(f, cfg); + const r = run(f, ['verify', '--src', dir]); + expect(r.exitCode).toBe(1); + expect(r.stderr.toString()).toMatch(want); + } + expect(f.calls()).toBe(''); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 30_000); + + test('verify tiers: a run killed during tier 3 with its VM gone resumes on a new VM: setup reruns, t1 and t2 are reused, t3 reruns', async () => { + const f = fakeVm(); + try { + const dir = gitSrc(f, TIERS); + bootCounts(f, dir); + Bun.spawnSync(['git', '-c', 'user.email=t@t', '-c', 'user.name=t', 'commit', '-qam', 'boot'], { cwd: dir, timeout: 10_000 }); + writeFileSync(join(f.root, 'make-t3.sh'), 'touch ' + JSON.stringify(join(f.root, 't3-started')) + '; sleep 30\n'); + const a = f.spawn(['verify', '--src', dir]); + await until(() => existsSync(join(f.root, 't3-started')), 60_000); + process.kill(a.pid, 'SIGKILL'); + await a.exited; + await killSession(f, 'tier-t3'); + const first = vmDirs(f)[0]!; + writeFileSync(join(f.root, 'listed'), ''); + writeFileSync(join(f.root, 'make-t3.sh'), 'echo t3 ok\n'); + const b = f.run(['verify', '--src', dir]); + const err = b.stderr.toString(), out = b.stdout.toString(); + expect(b.exitCode, err).toBe(0); + expect(err).toMatch(/UBI_VERIFY_RESUME: tier=t1 reuse t1\.\d{10}\.\d+\.json \(\d+s\)\n/); + expect(err).toMatch(/UBI_VERIFY_RESUME: tier=t2 reuse t2\.\d{10}\.\d+\.json \(\d+s\)\n/); + expect(err).toMatch(/UBI_VERIFY_RESUME: tier=t3 run: interrupted \(only a start record, t3\.\d{10}\.\d+\.json\)\n/); + expect(err).toContain('run record for ' + first + ': the VM is gone; starting a new one'); + expect(err).not.toContain('reattached to'); + expect(creates(f)).toBe(2); + expect(readFileSync(join(f.vm, 'setup-count'), 'utf8')).toBe('boot\nboot\n'); + expect(out).not.toContain('ran: make t1'); + expect(out).toContain('t3 ok'); + expect(verifyLine(out)).toMatch(/^UBI_VERIFY: tree=[0-9a-f]{12} key=[0-9a-f]{12} tiers=3 passed=3 reused=t1,t2 failed=none prior_failures=none saved_s=\d+$/); + expect(f.listed()).toBe(''); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 120_000); + + test('verify tiers: a fail record blocks a rerun until --rerun-failed (then prior_failures); passes are reused unless stale, tampered, --no-resume or the tree changed', () => { + const f = fakeVm(); + try { + const dir = gitSrc(f, TIERS); + writeFileSync(join(f.root, 'make-t2.sh'), 'echo t2 broke; exit 7\n'); + const first = run(f, ['verify', '--src', dir]); + expect(first.exitCode, first.stderr.toString()).toBe(7); + expect(first.stdout.toString()).not.toContain('ran: make t3'); + expect(verifyLine(first.stdout.toString())).toMatch(/ tiers=3 passed=1 reused=none failed=t2 prior_failures=none saved_s=0$/); + expect(creates(f)).toBe(1); + + const blocked = run(f, ['verify', '--src', dir]); + expect(blocked.exitCode).toBe(7); + expect(blocked.stderr.toString()).toMatch(/UBI_VERIFY_RESUME: tier=t2 fail t2\.\d{10}\.\d+\.json \(\d+s, exit 7\): not rerun without --rerun-failed/); + expect(verifyLine(blocked.stdout.toString())).toMatch(/ failed=t2 prior_failures=none /); + expect(creates(f)).toBe(1); + + writeFileSync(join(f.root, 'make-t2.sh'), 'echo t2 fixed\n'); + const rerun = run(f, ['verify', '--src', dir, '--rerun-failed']); + expect(rerun.exitCode, rerun.stderr.toString()).toBe(0); + expect(rerun.stderr.toString()).toMatch(/UBI_VERIFY_RESUME: tier=t1 reuse /); + expect(rerun.stderr.toString()).toMatch(/UBI_VERIFY_RESUME: tier=t2 run: prior fail t2\.\d{10}\.\d+\.json \(exit 7; --rerun-failed\)/); + expect(verifyLine(rerun.stdout.toString())).toMatch(/ tiers=3 passed=3 reused=t1 failed=none prior_failures=t2:7 saved_s=\d+$/); + expect(creates(f)).toBe(2); + + const all = run(f, ['verify', '--src', dir]); + expect(all.exitCode).toBe(0); + expect(verifyLine(all.stdout.toString())).toMatch(/ passed=3 reused=t1,t2,t3 failed=none prior_failures=none saved_s=\d+$/); + expect(all.stderr.toString()).toContain('every tier passed under this key; no VM needed'); + expect(creates(f)).toBe(2); + + const probe = (args: string[] = [], extra: Record = {}) => run(f, ['verify', '--src', dir, '--probe', ...args], extra).stderr.toString(); + expect(probe(['--no-resume'])).toMatch(/tier=t1 run: --no-resume\n[^]*tier=t2 run: --no-resume\n[^]*tier=t3 run: --no-resume\n/); + expect(probe([], { UBI_CHECKPOINT_MAX_AGE: '0' })).toMatch(/tier=t1 run: pass t1\.\d{10}\.\d+\.json is \d+s old \(UBI_CHECKPOINT_MAX_AGE=0\)/); + const keyDir = join(f.checkpoints, readdirSync(f.checkpoints)[0]!); + const t1Log = readdirSync(keyDir).filter(n => n.startsWith('t1.') && n.endsWith('.log')).sort().at(-1)!; + writeFileSync(join(keyDir, t1Log), 'tampered\n'); + const tampered = probe(); + expect(tampered).toMatch(/tier=t1 run: pass t1\.\d{10}\.\d+\.json: its log does not re-hash/); + expect(tampered).toMatch(/tier=t2 reuse /); + writeFileSync(join(dir, 'new-source.ts'), 'export {}\n'); + const moved = probe(); + for (const t of ['t1', 't2', 't3']) expect(moved).toContain('UBI_VERIFY_RESUME: tier=' + t + ' run: no record'); + expect(creates(f)).toBe(2); + // Records are write-once JSON; every one carries its generation. + const gens = readdirSync(keyDir).filter(n => n.endsWith('.json')).map(n => JSON.parse(readFileSync(join(keyDir, n), 'utf8')).generation as number).sort((x, y) => x - y); + expect(gens).toEqual(gens.map((_, i) => i + 1)); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 120_000); + + test('verify tiers: a reattach whose VM TTL no longer covers the run left plus UBI_REATTACH_MARGIN starts a fresh VM, destroys the old one and reuses the passed tiers', async () => { + const f = fakeVm(); + try { + const dir = gitSrc(f, TIERS); + bootCounts(f, dir); + Bun.spawnSync(['git', '-c', 'user.email=t@t', '-c', 'user.name=t', 'commit', '-qam', 'boot'], { cwd: dir, timeout: 10_000 }); + writeFileSync(join(f.root, 'make-t3.sh'), 'touch ' + JSON.stringify(join(f.root, 't3-started')) + '; sleep 30\n'); + const env = { UBI_REAP: '0', UBI_TTL_HOURS: '1' }; + const a = f.spawn(['verify', '--src', dir], env); + await until(() => existsSync(join(f.root, 't3-started')), 60_000); + process.kill(a.pid, 'SIGKILL'); + await a.exited; + const first = vmDirs(f)[0]!; + expect(first).toMatch(/-t1$/); + const probe = (extra: Record) => run(f, ['verify', '--src', dir, '--probe'], { ...env, ...extra }).stdout.toString(); + expect(probe({})).toMatch(new RegExp('UBI_ATTACH_INFO: vm=' + first + ' created_at=\\d{10} ')); + // A 1 h TTL leaves about 3600 s: t3's 1200 s run timeout plus a 2700 s margin does not fit. + const short = { UBI_RUN_TIMEOUT: '1200', UBI_REATTACH_MARGIN: '2700' }; + expect(probe(short)).toMatch(new RegExp('UBI_ATTACH_INFO: none key=[0-9a-f]{12} ttl_short vm=' + first + ' ttl_left=\\d+s needs=\\d+s \\(run still to do \\d+s \\+ UBI_REATTACH_MARGIN=2700s\\)')); + expect(creates(f)).toBe(1); + writeFileSync(join(f.root, 'make-t3.sh'), 'echo t3 ok\n'); + const b = run(f, ['verify', '--src', dir], { ...env, ...short }); + const err = b.stderr.toString(); + expect(b.exitCode, err).toBe(0); + expect(err).toMatch(new RegExp('UBI_REATTACH_TTL_SHORT: vm=' + first + ' key=[0-9a-f]{12} ttl_left=\\d+s needs=\\d+s \\(run still to do \\d+s \\+ UBI_REATTACH_MARGIN=2700s\\): starting a fresh VM \\(tier checkpoints carry over\\) and destroying ' + first)); + expect(err).not.toContain('reattached to'); + expect(err).toMatch(/UBI_VERIFY_RESUME: tier=t1 reuse /); + expect(err).toMatch(/UBI_VERIFY_RESUME: tier=t2 reuse /); + expect(verifyLine(b.stdout.toString())).toMatch(/ tiers=3 passed=3 reused=t1,t2 failed=none /); + expect(creates(f)).toBe(2); + expect(f.listed()).toBe(''); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 120_000); + + test('verify tiers: a writer whose start record is no longer the newest publishes nothing (stale writer)', () => { + const py = /\nCKPT_PY='\n([^']*)'\n/.exec(readFileSync(join(DIR, 'ubi-runner.sh'), 'utf8'))![1]!; + const root = mkdtempSync(join(tmpdir(), 'ubi-ckpt-')); + try { + const d = join(root, 'key'); + const log = join(root, 'tier.log'); writeFileSync(log, 'ok\n'); + const publish = (status: string, startgen: string, pid: string) => Bun.spawnSync(['python3', '-c', py, 'publish', d, 't1', status, 'vm', '100', '200', '0', status === 'start' ? '' : log, startgen, 'k', 'tree', pid], { timeout: 30_000 }); + expect(publish('start', '0', '1').stdout.toString().trim()).toBe('1'); + expect(publish('start', '0', '2').stdout.toString().trim()).toBe('2'); + const stale = publish('pass', '1', '1'); + expect(stale.exitCode).toBe(2); + expect(stale.stderr.toString()).toContain('stale writer: its start record is generation 1, the newest for t1 is 2'); + expect(readdirSync(d).filter(n => n.endsWith('.json')).length).toBe(2); + expect(publish('pass', '2', '2').stdout.toString().trim()).toBe('3'); + } finally { rmSync(root, { recursive: true, force: true }); } + }); + + test('verify tiers: a tier that edits the source gets no pass record (UBI_VERIFY_TREE_MOVED) and reruns next time', () => { + const f = fakeVm(); + try { + const dir = gitSrc(f, { tiers: [{ name: 'mover', command: 'make mover' }, { name: 'clean', command: 'make clean' }] }); + writeFileSync(join(f.root, 'make-mover.sh'), 'echo edited > moved.txt\n'); + const r = run(f, ['verify', '--src', dir]); + expect(r.exitCode, r.stderr.toString()).toBe(0); + expect(r.stderr.toString()).toMatch(/UBI_VERIFY_TREE_MOVED: tier=mover before=[0-9a-f]{12} after=[0-9a-f]{12}: a tier must leave the source tree as it found it; no pass record/); + expect(verifyLine(r.stdout.toString())).toMatch(/ tiers=2 passed=2 reused=none failed=none /); + const next = run(f, ['verify', '--src', dir, '--probe']).stderr.toString(); + expect(next).toMatch(/tier=mover run: interrupted \(only a start record/); + expect(next).toMatch(/tier=clean reuse /); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 90_000); +}); diff --git a/test/ubicloud-runner-verify-quota.test.ts b/test/ubicloud-runner-verify-quota.test.ts new file mode 100644 index 000000000..92966c925 --- /dev/null +++ b/test/ubicloud-runner-verify-quota.test.ts @@ -0,0 +1,184 @@ +/** + * ubi-runner verify pass names and pull containment, quota cache and wait, reap, against the fake Ubicloud API and fake ssh in + * test/helpers/ubi-runner-fake-vm.ts. Split from test/ubicloud-runner.test.ts (GSTA-151). + */ +import { describe, expect, test } from 'bun:test'; +import { chmodSync, existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, rmSync, symlinkSync, utimesSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { read, fakeVm, src, run } from './helpers/ubi-runner-fake-vm'; + +describe('ubi-runner run, verify and quota against a fake API and a fake ssh', () => { + test('verify forwards a configured pass name only when the command line grants it, and never UBICLOUD_*', () => { + const f = fakeVm(); + try { + const dir = src(f, { setup: 'scripts/boot.sh', command: 'make verify', pass: ['UBI_TEST_TOKEN'] }); + const refused = run(f, ['verify', '--src', dir], { UBI_TEST_TOKEN: 'x' }); + expect(refused.exitCode).toBe(1); + expect(refused.stderr.toString()).toContain('asks to forward local UBI_TEST_TOKEN; grant each on this command line with --pass NAME, or every name it lists with --trust-config'); + expect(f.calls()).toBe(''); + const granted = run(f, ['verify', '--src', dir, '--pass', 'UBI_TEST_TOKEN'], { UBI_TEST_TOKEN: 'granted value' }); + expect(granted.exitCode).toBe(0); + expect(readFileSync(join(f.root, 'ubirun-env'), 'utf8')).toContain('export UBI_TEST_TOKEN=granted\\ value'); + expect(f.ssh()).toContain('bash -lc make\\ verify'); + rmSync(join(f.root, 'ubirun-env')); + const trusted = run(f, ['verify', '--src', dir, '--trust-config'], { UBI_TEST_TOKEN: 'trusted value' }); + expect(trusted.exitCode).toBe(0); + expect(readFileSync(join(f.root, 'ubirun-env'), 'utf8')).toContain('export UBI_TEST_TOKEN=trusted\\ value'); + rmSync(join(f.root, 'ubirun-env')); + const asValue = run(f, ['verify', '--src', dir, '--dest', '--trust-config', '--keep', '--dest', 'work/safe'], { UBI_TEST_TOKEN: 'x' }); + expect(asValue.exitCode).toBe(1); + expect(asValue.stderr.toString()).toContain('asks to forward local UBI_TEST_TOKEN'); + expect(existsSync(join(f.root, 'ubirun-env'))).toBe(false); + expect(run(f, ['verify', '--src', dir, '--pass']).stderr.toString()).toContain('verify: --pass needs a value'); + expect(run(f, ['verify', '--src', dir, '--bogus']).stderr.toString()).toContain('verify: unknown option --bogus'); + writeFileSync(join(dir, '.gstack/ubicloud.json'), JSON.stringify({ command: 'make verify', pass: ['UBICLOUD_API_KEY'] })); + const before = f.calls(); + const key = run(f, ['verify', '--src', dir, '--trust-config']); + expect(key.exitCode).toBe(1); + expect(key.stderr.toString()).toContain('never forwards its own Ubicloud credentials'); + expect(f.calls()).toBe(before); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 90_000); + + test('verify keeps a configured setup script and pull destination inside --src: absolute, ~ and .. paths are refused before any API call', () => { + const f = fakeVm(); + try { + const dir = src(f, {}); + const outside = join(f.root, 'outside.sh'); writeFileSync(outside, 'echo outside\n'); + const cases: Array<[unknown, RegExp]> = [ + [{ setup: outside, command: 'x' }, /setup '[^']*outside\.sh' must be a relative path inside the checkout/], + [{ setup: '../outside.sh', command: 'x' }, /setup '\.\.\/outside\.sh' must be a relative path inside the checkout/], + [{ setup: 'scripts/../../outside.sh', command: 'x' }, /must be a relative path inside the checkout/], + [{ setup: '~/outside.sh', command: 'x' }, /must be a relative path inside the checkout/], + [{ command: 'x', pull: ['work/logs:/etc/cron.d'] }, /pull destination '\/etc\/cron\.d' must be a relative path inside the checkout/], + [{ command: 'x', pull: ['work/logs:../stolen'] }, /pull destination '\.\.\/stolen' must be a relative path inside the checkout/], + [{ command: 'x', pull: ['no-colon'] }, /pull entry 'no-colon' must be REMOTE_GLOB:LOCAL_DIR/], + ]; + for (const [cfg, want] of cases) { + writeFileSync(join(dir, '.gstack/ubicloud.json'), JSON.stringify(cfg)); + const r = run(f, ['verify', '--src', dir]); + expect(r.exitCode).toBe(1); + expect(r.stderr.toString()).toMatch(want); + } + expect(f.calls()).toBe(''); + writeFileSync(join(dir, '.gstack/ubicloud.json'), JSON.stringify({ setup: 'scripts/boot.sh', command: 'make verify', pull: ['work/repo/logs:.context/ubicloud'] })); + const ok = run(f, ['verify', '--src', dir]); + expect(ok.exitCode).toBe(0); + expect(readFileSync(join(f.root, 'ubirun-setup.sh'), 'utf8')).toBe('echo boot\n'); + expect(f.ssh()).toContain('ls -d -- logs'); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 90_000); + + test('verify rechecks every pull destination at pull time: a symlink an earlier pull planted cannot redirect a later pull outside --src', () => { + const f = fakeVm(); + try { + const dir = src(f, { command: 'make verify', pull: ['work/repo/escape:.', 'work/repo/artifacts:escape/new/sub'] }); + const victim = join(f.root, 'victim'); mkdirSync(victim); + mkdirSync(join(f.root, 'remote-fs/artifacts'), { recursive: true }); + writeFileSync(join(f.root, 'remote-fs/artifacts/authorized_keys'), 'ssh-ed25519 AAAA attacker\n'); + symlinkSync(victim, join(f.root, 'remote-fs/escape')); + const r = run(f, ['verify', '--src', dir]); + expect(r.exitCode).toBe(1); + expect(r.stderr.toString()).toMatch(/pull refused: .*\/repo\/escape\/new\/sub resolves outside .*\/repo/); + expect(lstatSync(join(dir, 'escape')).isSymbolicLink()).toBe(true); + expect(readdirSync(victim)).toEqual([]); + expect(f.calls()).toContain('destroy'); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 60_000); + + test('quota: a refusal caches the maximum, the cache drives the preflight for a day and is dropped after; UBI_QUOTA_WAIT waits before refusing', () => { + const refuse = fakeVm({ refuse: 'Validation failed for following fields: Requested vCPU count: 16, currently used vCPU count: 32, maximum allowed vCPU count: 32' }); + try { + const r = run(refuse, ['up']); + expect(r.exitCode).toBe(1); + expect(r.stderr.toString()).toContain('create failed: vCPU quota exhausted'); + expect(readFileSync(join(refuse.state, 'quota-max'), 'utf8').trim()).toBe('32'); + expect(refuse.listed()).toBe(''); + } finally { rmSync(refuse.root, { recursive: true, force: true }); } + + const f = fakeVm({ listed: 'eu-central-h1 ubirun-peer-9999999999-cccc\n' }); + try { + writeFileSync(join(f.state, 'quota-max'), '31\n'); + const cached = run(f, ['up']); + expect(cached.exitCode).toBe(1); + expect(cached.stderr.toString()).toMatch(/already use 16 of 31 \(ceiling from the last refusal, cached in .*quota-max;/); + expect(f.calls()).not.toContain('"create"'); + + const started = Date.now(); + const waited = run(f, ['up'], { UBI_QUOTA_WAIT: '3' }); + expect(waited.exitCode).toBe(1); + expect(waited.stderr.toString()).toMatch(/quota: 16 of 31 vCPUs in use and 16 needed; waiting up to 3s more for room/); + expect(waited.stderr.toString()).toContain('create refused before any request: vCPU quota exhausted'); + expect(Date.now() - started).toBeGreaterThanOrEqual(3000); + expect(f.calls()).not.toContain('"create"'); + + utimesSync(join(f.state, 'quota-max'), new Date(Date.now() - 2 * 86_400_000), new Date(Date.now() - 2 * 86_400_000)); + const stale = run(f, ['up']); + expect(stale.exitCode).toBe(0); + expect(stale.stderr.toString()).not.toContain('quota preflight refused'); + expect(f.calls()).toContain('"create"'); + expect(existsSync(join(f.state, 'quota-max'))).toBe(false); + const name = stale.stdout.toString().trim(); + expect(name).toMatch(/^ubirun-thrd1-\d{10}-[0-9a-f]{8}-t3$/); + expect(run(f, ['down', name]).exitCode).toBe(0); + expect(f.listed()).toBe('eu-central-h1 ubirun-peer-9999999999-cccc\n'); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 90_000); + + // Value: protects=the quota wait budget counts seconds slept, not usage reads: a slow usage read keeps the printed budget, a wait above 30 s counts down in 30 s steps, and the wait releases when room appears; fails_when=the budget goes back to a wall-clock deadline taken before the first usage read (the first log line shrinks to 1s), the step cap or the decrement is lost, or room found mid-wait still refuses; why_new=the WAIT=3 case above passes under both the old deadline and the new countdown because its fake usage read is instant and it never sees a second step; seam=none (the fixture's bin/ is already first on PATH, so sleep and curl are stubbed there); no_claim=real elapsed time of the long wait (the 70 s case stubs sleep) + test('quota wait: a slow usage read does not shave the printed budget; a long wait counts down in 30s steps and releases when room appears', () => { + const slow = fakeVm({ listed: 'eu-central-h1 ubirun-peer-9999999999-cccc\n' }); + try { + writeFileSync(join(slow.state, 'quota-max'), '31\n'); + const r = run(slow, ['up'], { UBI_QUOTA_WAIT: '2', FAKE_SHOW_DELAY: '1.2' }); + const err = r.stderr.toString(); + expect(r.exitCode, err).toBe(1); + expect(err).toMatch(/quota: 16 of 31 vCPUs in use and 16 needed; waiting up to 2s more for room/); + expect(err).toContain('create refused before any request: vCPU quota exhausted'); + expect(slow.calls()).not.toContain('"create"'); + } finally { rmSync(slow.root, { recursive: true, force: true }); } + + const f = fakeVm({ listed: 'eu-central-h1 ubirun-peer-9999999999-cccc\n' }); + try { + writeFileSync(join(f.state, 'quota-max'), '31\n'); + const sleepLog = join(f.root, 'sleep.log'); + writeFileSync(join(f.root, 'bin/sleep'), '#!/usr/bin/env bash\necho "$1" >> ' + JSON.stringify(sleepLog) + '\n[ "$(wc -l < ' + JSON.stringify(sleepLog) + ')" -ne 2 ] || : > ' + JSON.stringify(join(f.root, 'listed')) + '\n'); + chmodSync(join(f.root, 'bin/sleep'), 0o755); + const r = run(f, ['up'], { UBI_QUOTA_WAIT: '70' }); + const err = r.stderr.toString(); + expect(r.exitCode, err).toBe(0); + expect(err).toMatch(/waiting up to 70s more for room/); + expect(err).toMatch(/waiting up to 40s more for room/); + expect(err).not.toContain('quota preflight refused'); + expect(readFileSync(sleepLog, 'utf8').split('\n').slice(0, 2)).toEqual(['30', '30']); + expect(f.calls()).toContain('"create"'); + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 90_000); + + test('reap reports a failed list and a failed destroy as failures; UBI_TTL_HOURS rejects leading zeros and more than three digits', () => { + const f = fakeVm({ listed: 'eu-central-h1 ubirun-other-1000000000-bbbb-t1\n' }); + try { + writeFileSync(join(f.root, 'bin/curl'), '#!/usr/bin/env bash\nout=""; while [ $# -gt 0 ]; do [ "$1" = -o ] && out=$2; shift; done\nbody=$(cat)\ncase "$body" in\n *\'"list"\'*) echo "eu-central-h1 ubirun-other-1000000000-bbbb-t1" > "$out"; printf 200 ;;\n *\'"show"\'*) printf \'state: running\\nsize: standard-16\\nip4: 192.0.2.1\\n\' > "$out"; printf 200 ;;\n *) echo "boom" > "$out"; printf 500 ;;\nesac\n'); + const destroyFails = run(f, ['reap']); + expect(destroyFails.exitCode).toBe(1); + expect(destroyFails.stderr.toString()).toContain('failed to reap eu-central-h1/ubirun-other-1000000000-bbbb-t1'); + expect(destroyFails.stderr.toString()).toContain('reap: 1 of 1 expired VMs could not be destroyed'); + writeFileSync(join(f.root, 'bin/curl'), '#!/usr/bin/env bash\nout=""; while [ $# -gt 0 ]; do [ "$1" = -o ] && out=$2; shift; done\ncat >/dev/null; echo "down" > "$out"; printf 503\n'); + const listFails = run(f, ['reap']); + expect(listFails.exitCode).toBe(1); + expect(listFails.stderr.toString()).toContain("reap: could not list the project's VMs; nothing destroyed"); + for (const ttl of ['0', '08', '1000', 'abc']) { + const r = run(f, ['up'], { UBI_TTL_HOURS: ttl }); + expect(r.exitCode).toBe(1); + expect(r.stderr.toString()).toContain('UBI_TTL_HOURS must be a whole number of hours from 1 to 999'); + } + // Value: protects=an old UBI_TTL_HOURS=0 or bad UBI_LEGACY_HOURS never blocks commands that create nothing (down, list, owner); fails_when=the TTL checks move back to startup; why_new=the checks were global and blocked cleanup; seam=none; no_claim=real down against the API (owner stands in for every read-only command) + expect(run(f, ['owner'], { UBI_TTL_HOURS: '0', UBI_LEGACY_HOURS: '0' }).exitCode).toBe(0); + for (const legacy of ['0', '04', 'x']) { + const r = run(f, ['reap'], { UBI_LEGACY_HOURS: legacy }); + expect(r.exitCode).toBe(1); + expect(r.stderr.toString()).toContain('UBI_LEGACY_HOURS must be a whole number of hours from 1 to 9999'); + } + } finally { rmSync(f.root, { recursive: true, force: true }); } + }, 30_000); +}); diff --git a/test/ubicloud-runner.test.ts b/test/ubicloud-runner.test.ts index fd6580884..ae075c25c 100644 --- a/test/ubicloud-runner.test.ts +++ b/test/ubicloud-runner.test.ts @@ -6,7 +6,7 @@ * call when the token is absent. */ import { describe, expect, test } from 'bun:test'; -import { chmodSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, utimesSync, writeFileSync } from 'node:fs'; +import { chmodSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; @@ -384,733 +384,3 @@ test('pack ships a linked worktree with a self-contained .git on the checked-out expect(runner).toContain('$HOME/.capy/work/ubi-runner/vms.tsv'); }); }); - -describe('ubi-runner run, verify and quota against a fake API and a fake ssh', () => { - /** - * Fake Ubicloud API that can create: create adds the name to the listed file - * (or answers the quota refusal in refuse with HTTP 400), show is running at - * 192.0.2.1 while listed, destroy unlists. The fake VM is the directory vm/: - * the fake ssh records every remote command in ssh.log, keeps copies of - * ~/.ubirun-env and ~/.ubirun-setup.sh under the fixture root, serves pulls - * from remote-fs/, and runs every other command there for real (HOME=vm/, - * vmbin/ first on PATH), so uploads unpack, detached phases write their pid, - * output and exit files, tail --pid follows them and pkill ends them. The stub - * make in vmbin/ echoes `ran: make ARGS`, or runs command-mode or - * make-.sh from the fixture root. No network, no VM. - */ - function fakeVm(opts: { listed?: string; refuse?: string } = {}) { - const root = mkdtempSync(join(tmpdir(), 'ubi-run-')); - const bin = join(root, 'bin'), state = join(root, 'state'), log = join(root, 'calls.log'), sshLog = join(root, 'ssh.log'), record = join(root, 'vms.tsv'); - const vm = join(root, 'vm'), vmbin = join(root, 'vmbin'), srcDir = join(root, 'src'), checkpoints = join(root, 'checkpoints'); - mkdirSync(vm); mkdirSync(vmbin); mkdirSync(srcDir); - writeFileSync(join(srcDir, 'README'), 'fixture checkout\n'); - mkdirSync(bin); mkdirSync(state, { recursive: true }); - writeFileSync(join(root, 'listed'), opts.listed ?? ''); - writeFileSync(join(bin, 'curl'), '#!/usr/bin/env bash\n' + - 'out=""; while [ $# -gt 0 ]; do [ "$1" = -o ] && out=$2; shift; done\n' + - 'body=$(cat); echo "$body" >> ' + JSON.stringify(log) + '\n' + - 'listed=' + JSON.stringify(join(root, 'listed')) + '\n' + - 'loc=$(printf \'%s\' "$body" | sed -n \'s/.*"vm", *"\\([^/"]*\\)\\/[^"]*".*/\\1/p\')\n' + - 'name=$(printf \'%s\' "$body" | sed -n \'s/.*"vm", *"[^/"]*\\/\\([^"]*\\)".*/\\1/p\')\n' + - 'case "$body" in\n' + - ' *\'"create"\'*) if [ -n ' + JSON.stringify(opts.refuse ?? '') + ' ]; then printf \'%s\' ' + JSON.stringify(opts.refuse ?? '') + ' > "$out"; printf 400; else echo "$loc $name" >> "$listed"; : > "$out"; printf 200; fi ;;\n' + - ' *destroy*) : > "$out"; if [ -f "$listed.destroy-fails" ]; then printf 500; else grep -v " $name$" "$listed" > "$listed.tmp"; mv "$listed.tmp" "$listed"; printf 200; fi ;;\n' + - ' *\'"list"\'*) if [ -f "$listed.list-fails" ]; then echo down > "$out"; printf 503; else cat "$listed" > "$out"; printf 200; fi ;;\n' + - ' *\'"show"\'*) [ -z "${FAKE_SHOW_DELAY:-}" ] || sleep "$FAKE_SHOW_DELAY"; if grep -q " $name$" "$listed"; then printf \'state: running\\nsize: standard-16\\nip4: 192.0.2.1\\n\' > "$out"; printf 200; else : > "$out"; printf 404; fi ;;\n' + - ' *) : > "$out"; printf 200 ;;\n' + - 'esac\n'); - writeFileSync(join(bin, 'ssh'), '#!/usr/bin/env bash\n' + - 'root=' + JSON.stringify(root) + '; vm="$root/vm"\n' + - 'cmd=${*: -1}\n' + - 'printf \'%s\\n\' "$cmd" >> "$root/ssh.log"\n' + - 'export HOME="$vm" PATH="$root/vmbin:$PATH"\n' + - 'cd "$vm" || exit 255\n' + - 'case "$cmd" in\n' + - ' *pkill*) echo "$cmd" >> "$root/cancel.log"; exec bash -c "$cmd" ;;\n' + - ' *captured*) cat > "$root/captured" ;;\n' + - ' *"cat > ~/.ubirun-env"*) tee "$root/ubirun-env" > "$vm/.ubirun-env" ;;\n' + - ' *"cat > ~/.ubirun-setup.sh"*) tee "$root/ubirun-setup.sh" > "$vm/.ubirun-setup.sh" ;;\n' + - ' *"tar -czf -"*) base=${cmd##* }; tar -czf - -C "$root/remote-fs" "$base" ;;\n' + - ' *"ls -d -- "*) : ;;\n' + - ' *) exec bash -c "$cmd" ;;\n' + - 'esac\n'); - writeFileSync(join(vmbin, 'make'), '#!/usr/bin/env bash\n' + - 'root=' + JSON.stringify(root) + '\n' + - 'if [ -f "$root/command-mode" ]; then exec bash -c "$(cat "$root/command-mode")"; fi\n' + - 'echo "ran: make $*"\n' + - '[ ! -f "$root/make-$1.sh" ] || exec bash "$root/make-$1.sh"\n'); - chmodSync(join(bin, 'curl'), 0o755); chmodSync(join(bin, 'ssh'), 0o755); chmodSync(join(vmbin, 'make'), 0o755); - const env: Record = { ...(process.env as Record), PATH: `${bin}:${process.env.PATH}`, UBICLOUD_API_KEY: 'offline', UBI_RUNNER_STATE: state, UBI_POLL_SECONDS: '1', UBI_VM_RECORD: record, UBI_OWNER: 'thrd1', UBI_HEARTBEAT: '0', UBI_CHECKPOINT_DIR: checkpoints }; - const run = (args: string[], extra: Record = {}, stdin?: string) => - Bun.spawnSync(['bash', join(DIR, 'ubi-runner.sh'), ...args], { env: { ...env, ...extra }, stdin: stdin === undefined ? undefined : Buffer.from(stdin), timeout: 90_000 }); - const spawn = (args: string[], extra: Record = {}) => - Bun.spawn(['bash', join(DIR, 'ubi-runner.sh'), ...args], { env: { ...env, ...extra }, stdin: 'ignore', stdout: 'pipe', stderr: 'pipe', timeout: 120_000 }); - return { - root, state, record, run, spawn, vm, src: srcDir, checkpoints, - calls: () => (existsSync(log) ? readFileSync(log, 'utf8') : ''), - ssh: () => (existsSync(sshLog) ? readFileSync(sshLog, 'utf8') : ''), - listed: () => readFileSync(join(root, 'listed'), 'utf8'), - }; - } - - const src = (f: { root: string }, cfg: unknown) => { - const dir = join(f.root, 'repo'); - mkdirSync(join(dir, '.gstack'), { recursive: true }); - mkdirSync(join(dir, 'scripts'), { recursive: true }); - writeFileSync(join(dir, 'scripts/boot.sh'), 'echo boot\n'); - writeFileSync(join(dir, '.gstack/ubicloud.json'), JSON.stringify(cfg)); - return dir; - }; - - const run = (f: ReturnType, args: string[], extra: Record = {}, stdin?: string) => f.run(args, extra, stdin); - - test('run: a hung setup is killed by the watchdog, exits 124, never reaches the command, and the VM is destroyed', () => { - const f = fakeVm(); - try { - const setup = join(f.root, 'setup.sh'); writeFileSync(setup, 'echo "setup start"; sleep 20\n'); - const r = run(f, ['run', '--src', f.src, '--setup', setup, '--', 'make test'], { UBI_IDLE_TIMEOUT: '2' }); - const err = r.stderr.toString(); - expect(r.exitCode, err).toBe(124); - expect(err).toMatch(/watchdog: setup printed nothing for \d+s \(UBI_IDLE_TIMEOUT=2\)/); - expect(err).toContain('setup failed (exit 124)'); - expect(f.ssh()).not.toContain('bash -lc'); - expect(err).toContain('setup start'); - // The setup runs detached in its own session: its leader's pid, both streams and the exit status are files on the VM. - expect(f.ssh()).toMatch(/setsid -f bash -c 'echo \$\$ >~\/\.ubirun-setup\.pid\.tmp && mv ~\/\.ubirun-setup\.pid\.tmp ~\/\.ubirun-setup\.pid; \( cd work\/[^ ]+ && \. ~\/\.ubirun-env && bash -l ~\/\.ubirun-setup\.sh \) >~\/\.ubirun-setup\.out 2>~\/\.ubirun-setup\.err <\/dev\/null; echo \$\? >~\/\.ubirun-setup\.exit\.tmp && mv ~\/\.ubirun-setup\.exit\.tmp ~\/\.ubirun-setup\.exit' <\/dev\/null/m); - expect(readFileSync(join(f.root, 'cancel.log'), 'utf8')).toMatch(/sid=\$\(cat ~\/\.ubirun-setup\.pid 2>\/dev\/null\); rm -f ~\/\.ubirun-setup\.pid; \[ -z "\$sid" \] \|\| \{ pkill -TERM -s "\$sid"; sleep 2; pkill -KILL -s "\$sid"; \}/); - expect(existsSync(join(f.vm, '.ubirun-setup.exit'))).toBe(false); - expect(f.calls()).toContain('destroy'); - expect(err).toMatch(/destroyed eu-central-h1\/ubirun-thrd1-\d{10}-[0-9a-f]{8}-t3/); - expect(f.listed()).toBe(''); - expect(readFileSync(f.record, 'utf8')).toBe(''); - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 60_000); - - test('run: UBI_RUN_TIMEOUT kills a command that keeps printing, with exit 124 and its reason', () => { - const f = fakeVm(); - try { - writeFileSync(join(f.root, 'command-mode'), 'for i in 1 2 3 4 5 6 7 8 9 10; do echo "tick $i"; sleep 1; done'); - const r = run(f, ['run', '--src', f.src, '--', 'make test'], { UBI_RUN_TIMEOUT: '3' }); - expect(r.exitCode).toBe(124); - expect(r.stderr.toString()).toMatch(/watchdog: command ran \d+s \(UBI_RUN_TIMEOUT=3\); killing it/); - expect(r.stderr.toString()).toContain('command exited 124 after'); - expect(f.calls()).toContain('destroy'); - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 60_000); - - test('run: ends with UBICLOUD: left=0 for its own VM; a same-owner sibling is counted as information and does not fail the run', () => { - const f = fakeVm({ listed: 'eu-central-h1 ubirun-thrd1-9999999999-sib1-t3\n' }); - try { - const leftFile = join(f.root, 'left'); - const r = f.run(['run', '--src', f.src, '--', 'true'], { UBI_REAP: '0', UBI_LEFT_FILE: leftFile }); - const err = r.stderr.toString(); - expect(r.exitCode, err).toBe(0); - expect(err).toMatch(/UBICLOUD: left=0 owner=thrd1 vm=ubirun-thrd1-\d{10}-[0-9a-f]{8}-t3 others=1\n/); - expect(f.listed()).toBe('eu-central-h1 ubirun-thrd1-9999999999-sib1-t3\n'); - // Value: protects=UBI_LEFT_FILE carries the same leftover line pregate-remote.sh parses, apart from the command's output; fails_when=the file is not written or its format drifts from pregate-remote.sh's parser; why_new=the pregate stub hand-copies the format; seam=none; no_claim=real VM teardown (fake API and ssh) - expect(readFileSync(leftFile, 'utf8')).toMatch(/^UBICLOUD: left=0 owner=thrd1 vm=ubirun-thrd1-\d{10}-[0-9a-f]{8}-t3 others=1\n$/); - const parser = read('scripts/ubicloud/pregate-remote.sh').match(/left=\$\(sed -n '([^']+)' "\$log\.left"/)![1]; - expect(Bun.spawnSync(['sed', '-n', parser, leftFile], { timeout: 30_000 }).stdout.toString()).toBe('0\n'); - // Value: protects=run --keep reports its live VM as UBICLOUD: left=1 ... kept=1 and never a clean left=0; fails_when=the kept line is dropped or --keep still prints the teardown line; why_new=no test runs run --keep; seam=none; no_claim=real VM lifetime (fake API and ssh) - const kept = f.run(['run', '--src', f.src, '--keep', '--', 'true'], { UBI_REAP: '0' }); - const keptErr = kept.stderr.toString(); - expect(kept.exitCode, keptErr).toBe(0); - expect(keptErr).toMatch(/UBICLOUD: left=1 owner=thrd1 vm=ubirun-thrd1-\d{10}-[0-9a-f]{8}-t3 kept=1\n/); - expect(keptErr).not.toContain('left=0'); - expect(f.listed()).toMatch(/ubirun-thrd1-\d{10}-[0-9a-f]{8}-t3\n/); - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 30_000); - - test('run: a passing command whose teardown fails exits 1 with UBICLOUD: left=1 and the down command; an unreadable VM list fails closed', () => { - const f = fakeVm(); - try { - writeFileSync(join(f.root, 'listed.destroy-fails'), ''); - const leaked = f.run(['run', '--src', f.src, '--', 'true'], { UBI_REAP: '0', UBI_DOWN_TIMEOUT: '3' }); - const err = leaked.stderr.toString(); - expect(leaked.exitCode, err).toBe(1); - expect(err).toMatch(/UBICLOUD: left=1 owner=thrd1 vm=(ubirun-thrd1-\d{10}-[0-9a-f]{8}-t3) others=0\n[^]*VM cleanup not confirmed for \1 \(left=1\); destroy it with: \S+ down \1/); - expect(f.listed()).toContain('ubirun-thrd1-'); - // Value: protects=a failing command whose VM also leaks keeps its own exit status, not 1; fails_when=run_teardown exits 1 whenever left is not 0; why_new=the leak case above only runs a passing command; seam=none; no_claim=real destroy failure modes (fake API returns 500) - writeFileSync(join(f.root, 'command-mode'), 'exit 7'); - const failedLeak = f.run(['run', '--src', f.src, '--', 'make test'], { UBI_REAP: '0', UBI_DOWN_TIMEOUT: '3' }); - expect(failedLeak.exitCode, failedLeak.stderr.toString()).toBe(7); - expect(failedLeak.stderr.toString()).toMatch(/UBICLOUD: left=1 owner=thrd1 vm=ubirun-thrd1-\d{10}-[0-9a-f]{8}-t3 others=1\n/); - const blind = fakeVm(); - try { - writeFileSync(join(blind.root, 'bin/curl.real'), readFileSync(join(blind.root, 'bin/curl'))); - chmodSync(join(blind.root, 'bin/curl.real'), 0o755); - writeFileSync(join(blind.root, 'bin/curl'), '#!/usr/bin/env bash\nif [ -f ' + JSON.stringify(join(blind.root, 'armed')) + ' ]; then touch ' + JSON.stringify(join(blind.root, 'listed.list-fails')) + '; fi\nexec ' + JSON.stringify(join(blind.root, 'bin/curl.real')) + ' "$@"\n'); - chmodSync(join(blind.root, 'bin/curl'), 0o755); - writeFileSync(join(blind.root, 'command-mode'), 'touch ' + JSON.stringify(join(blind.root, 'armed'))); - const r = blind.run(['run', '--src', blind.src, '--', 'make test'], { UBI_REAP: '0' }); - expect(r.exitCode, r.stderr.toString()).toBe(1); - expect(r.stderr.toString()).toMatch(/UBICLOUD: left=unknown owner=thrd1 vm=ubirun-thrd1-\d{10}-[0-9a-f]{8}-t3 others=unknown\n/); - } finally { rmSync(blind.root, { recursive: true, force: true }); } - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 60_000); - - test('run: the fake ssh dispatches on the remote command, so a checkout whose name holds -O still syncs and runs (GSTA-139)', () => { - const f = fakeVm(); - try { - const checkout = join(f.root, 'ubi-run-O1x2y3'); - mkdirSync(checkout); - const r = run(f, ['run', '--src', checkout, '--', 'make test']); - expect(r.exitCode, r.stderr.toString()).toBe(0); - expect(r.stdout.toString()).toContain('ran: '); - expect(f.ssh()).toContain('tar -xzf - -C work/ubi-run-O1x2y3'); - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 60_000); - - test('run: --pass forwards a named local variable only; UBICLOUD_* and non-names are refused before any API call', () => { - const f = fakeVm(); - try { - const ok = run(f, ['run', '--src', f.src, '--pass', 'UBI_TEST_TOKEN', '--env', 'CI=1', '--', 'make test'], { UBI_TEST_TOKEN: 'sekret value' }); - expect(ok.exitCode).toBe(0); - expect(ok.stderr.toString()).toContain('command exited 0 after'); - const envFile = readFileSync(join(f.root, 'ubirun-env'), 'utf8'); - expect(envFile).toContain('export UBI_TEST_TOKEN=sekret\\ value'); - expect(envFile).toContain('export CI=1'); - expect(envFile).not.toContain('UBICLOUD'); - expect(ok.stdout.toString() + ok.stderr.toString()).not.toContain('sekret'); - for (const bad of ['UBICLOUD_API_KEY', 'UBICLOUD_API_TOKEN', 'X;Y', 'HOME/..']) { - const before = f.calls(); - const r = run(f, ['run', '--src', f.src, '--pass', bad, '--', 'true']); - expect(r.exitCode).toBe(1); - expect(r.stderr.toString()).toMatch(bad.startsWith('UBICLOUD_') ? /never forwards its own Ubicloud credentials/ : /not a variable name/); - expect(f.calls()).toBe(before); - } - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 60_000); - - test('ssh NAME COMMAND hands the caller’s stdin to the remote command', () => { - const f = fakeVm({ listed: 'eu-central-h1 vm-a\n' }); - try { - mkdirSync(join(f.state, 'vm-a'), { recursive: true }); - writeFileSync(join(f.state, 'vm-a/env'), 'NAME=vm-a\nLOCATION=eu-central-h1\nSIZE=standard-16\nIP=192.0.2.1\nCREATED=yes\nCREATE_AT=0\nCREATE_PID=1\n'); - const r = run(f, ['ssh', 'vm-a', 'cat > captured'], { UBI_IDLE_TIMEOUT: '0' }, 'payload line\n'); - expect(r.exitCode).toBe(0); - expect(readFileSync(join(f.root, 'captured'), 'utf8')).toBe('payload line\n'); - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 30_000); - - test('verify forwards a configured pass name only when the command line grants it, and never UBICLOUD_*', () => { - const f = fakeVm(); - try { - const dir = src(f, { setup: 'scripts/boot.sh', command: 'make verify', pass: ['UBI_TEST_TOKEN'] }); - const refused = run(f, ['verify', '--src', dir], { UBI_TEST_TOKEN: 'x' }); - expect(refused.exitCode).toBe(1); - expect(refused.stderr.toString()).toContain('asks to forward local UBI_TEST_TOKEN; grant each on this command line with --pass NAME, or every name it lists with --trust-config'); - expect(f.calls()).toBe(''); - const granted = run(f, ['verify', '--src', dir, '--pass', 'UBI_TEST_TOKEN'], { UBI_TEST_TOKEN: 'granted value' }); - expect(granted.exitCode).toBe(0); - expect(readFileSync(join(f.root, 'ubirun-env'), 'utf8')).toContain('export UBI_TEST_TOKEN=granted\\ value'); - expect(f.ssh()).toContain('bash -lc make\\ verify'); - rmSync(join(f.root, 'ubirun-env')); - const trusted = run(f, ['verify', '--src', dir, '--trust-config'], { UBI_TEST_TOKEN: 'trusted value' }); - expect(trusted.exitCode).toBe(0); - expect(readFileSync(join(f.root, 'ubirun-env'), 'utf8')).toContain('export UBI_TEST_TOKEN=trusted\\ value'); - rmSync(join(f.root, 'ubirun-env')); - const asValue = run(f, ['verify', '--src', dir, '--dest', '--trust-config', '--keep', '--dest', 'work/safe'], { UBI_TEST_TOKEN: 'x' }); - expect(asValue.exitCode).toBe(1); - expect(asValue.stderr.toString()).toContain('asks to forward local UBI_TEST_TOKEN'); - expect(existsSync(join(f.root, 'ubirun-env'))).toBe(false); - expect(run(f, ['verify', '--src', dir, '--pass']).stderr.toString()).toContain('verify: --pass needs a value'); - expect(run(f, ['verify', '--src', dir, '--bogus']).stderr.toString()).toContain('verify: unknown option --bogus'); - writeFileSync(join(dir, '.gstack/ubicloud.json'), JSON.stringify({ command: 'make verify', pass: ['UBICLOUD_API_KEY'] })); - const before = f.calls(); - const key = run(f, ['verify', '--src', dir, '--trust-config']); - expect(key.exitCode).toBe(1); - expect(key.stderr.toString()).toContain('never forwards its own Ubicloud credentials'); - expect(f.calls()).toBe(before); - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 90_000); - - test('verify keeps a configured setup script and pull destination inside --src: absolute, ~ and .. paths are refused before any API call', () => { - const f = fakeVm(); - try { - const dir = src(f, {}); - const outside = join(f.root, 'outside.sh'); writeFileSync(outside, 'echo outside\n'); - const cases: Array<[unknown, RegExp]> = [ - [{ setup: outside, command: 'x' }, /setup '[^']*outside\.sh' must be a relative path inside the checkout/], - [{ setup: '../outside.sh', command: 'x' }, /setup '\.\.\/outside\.sh' must be a relative path inside the checkout/], - [{ setup: 'scripts/../../outside.sh', command: 'x' }, /must be a relative path inside the checkout/], - [{ setup: '~/outside.sh', command: 'x' }, /must be a relative path inside the checkout/], - [{ command: 'x', pull: ['work/logs:/etc/cron.d'] }, /pull destination '\/etc\/cron\.d' must be a relative path inside the checkout/], - [{ command: 'x', pull: ['work/logs:../stolen'] }, /pull destination '\.\.\/stolen' must be a relative path inside the checkout/], - [{ command: 'x', pull: ['no-colon'] }, /pull entry 'no-colon' must be REMOTE_GLOB:LOCAL_DIR/], - ]; - for (const [cfg, want] of cases) { - writeFileSync(join(dir, '.gstack/ubicloud.json'), JSON.stringify(cfg)); - const r = run(f, ['verify', '--src', dir]); - expect(r.exitCode).toBe(1); - expect(r.stderr.toString()).toMatch(want); - } - expect(f.calls()).toBe(''); - writeFileSync(join(dir, '.gstack/ubicloud.json'), JSON.stringify({ setup: 'scripts/boot.sh', command: 'make verify', pull: ['work/repo/logs:.context/ubicloud'] })); - const ok = run(f, ['verify', '--src', dir]); - expect(ok.exitCode).toBe(0); - expect(readFileSync(join(f.root, 'ubirun-setup.sh'), 'utf8')).toBe('echo boot\n'); - expect(f.ssh()).toContain('ls -d -- logs'); - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 90_000); - - test('verify rechecks every pull destination at pull time: a symlink an earlier pull planted cannot redirect a later pull outside --src', () => { - const f = fakeVm(); - try { - const dir = src(f, { command: 'make verify', pull: ['work/repo/escape:.', 'work/repo/artifacts:escape/new/sub'] }); - const victim = join(f.root, 'victim'); mkdirSync(victim); - mkdirSync(join(f.root, 'remote-fs/artifacts'), { recursive: true }); - writeFileSync(join(f.root, 'remote-fs/artifacts/authorized_keys'), 'ssh-ed25519 AAAA attacker\n'); - symlinkSync(victim, join(f.root, 'remote-fs/escape')); - const r = run(f, ['verify', '--src', dir]); - expect(r.exitCode).toBe(1); - expect(r.stderr.toString()).toMatch(/pull refused: .*\/repo\/escape\/new\/sub resolves outside .*\/repo/); - expect(lstatSync(join(dir, 'escape')).isSymbolicLink()).toBe(true); - expect(readdirSync(victim)).toEqual([]); - expect(f.calls()).toContain('destroy'); - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 60_000); - - test('quota: a refusal caches the maximum, the cache drives the preflight for a day and is dropped after; UBI_QUOTA_WAIT waits before refusing', () => { - const refuse = fakeVm({ refuse: 'Validation failed for following fields: Requested vCPU count: 16, currently used vCPU count: 32, maximum allowed vCPU count: 32' }); - try { - const r = run(refuse, ['up']); - expect(r.exitCode).toBe(1); - expect(r.stderr.toString()).toContain('create failed: vCPU quota exhausted'); - expect(readFileSync(join(refuse.state, 'quota-max'), 'utf8').trim()).toBe('32'); - expect(refuse.listed()).toBe(''); - } finally { rmSync(refuse.root, { recursive: true, force: true }); } - - const f = fakeVm({ listed: 'eu-central-h1 ubirun-peer-9999999999-cccc\n' }); - try { - writeFileSync(join(f.state, 'quota-max'), '31\n'); - const cached = run(f, ['up']); - expect(cached.exitCode).toBe(1); - expect(cached.stderr.toString()).toMatch(/already use 16 of 31 \(ceiling from the last refusal, cached in .*quota-max;/); - expect(f.calls()).not.toContain('"create"'); - - const started = Date.now(); - const waited = run(f, ['up'], { UBI_QUOTA_WAIT: '3' }); - expect(waited.exitCode).toBe(1); - expect(waited.stderr.toString()).toMatch(/quota: 16 of 31 vCPUs in use and 16 needed; waiting up to 3s more for room/); - expect(waited.stderr.toString()).toContain('create refused before any request: vCPU quota exhausted'); - expect(Date.now() - started).toBeGreaterThanOrEqual(3000); - expect(f.calls()).not.toContain('"create"'); - - utimesSync(join(f.state, 'quota-max'), new Date(Date.now() - 2 * 86_400_000), new Date(Date.now() - 2 * 86_400_000)); - const stale = run(f, ['up']); - expect(stale.exitCode).toBe(0); - expect(stale.stderr.toString()).not.toContain('quota preflight refused'); - expect(f.calls()).toContain('"create"'); - expect(existsSync(join(f.state, 'quota-max'))).toBe(false); - const name = stale.stdout.toString().trim(); - expect(name).toMatch(/^ubirun-thrd1-\d{10}-[0-9a-f]{8}-t3$/); - expect(run(f, ['down', name]).exitCode).toBe(0); - expect(f.listed()).toBe('eu-central-h1 ubirun-peer-9999999999-cccc\n'); - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 90_000); - - // Value: protects=the quota wait budget counts seconds slept, not usage reads: a slow usage read keeps the printed budget, a wait above 30 s counts down in 30 s steps, and the wait releases when room appears; fails_when=the budget goes back to a wall-clock deadline taken before the first usage read (the first log line shrinks to 1s), the step cap or the decrement is lost, or room found mid-wait still refuses; why_new=the WAIT=3 case above passes under both the old deadline and the new countdown because its fake usage read is instant and it never sees a second step; seam=none (the fixture's bin/ is already first on PATH, so sleep and curl are stubbed there); no_claim=real elapsed time of the long wait (the 70 s case stubs sleep) - test('quota wait: a slow usage read does not shave the printed budget; a long wait counts down in 30s steps and releases when room appears', () => { - const slow = fakeVm({ listed: 'eu-central-h1 ubirun-peer-9999999999-cccc\n' }); - try { - writeFileSync(join(slow.state, 'quota-max'), '31\n'); - const r = run(slow, ['up'], { UBI_QUOTA_WAIT: '2', FAKE_SHOW_DELAY: '1.2' }); - const err = r.stderr.toString(); - expect(r.exitCode, err).toBe(1); - expect(err).toMatch(/quota: 16 of 31 vCPUs in use and 16 needed; waiting up to 2s more for room/); - expect(err).toContain('create refused before any request: vCPU quota exhausted'); - expect(slow.calls()).not.toContain('"create"'); - } finally { rmSync(slow.root, { recursive: true, force: true }); } - - const f = fakeVm({ listed: 'eu-central-h1 ubirun-peer-9999999999-cccc\n' }); - try { - writeFileSync(join(f.state, 'quota-max'), '31\n'); - const sleepLog = join(f.root, 'sleep.log'); - writeFileSync(join(f.root, 'bin/sleep'), '#!/usr/bin/env bash\necho "$1" >> ' + JSON.stringify(sleepLog) + '\n[ "$(wc -l < ' + JSON.stringify(sleepLog) + ')" -ne 2 ] || : > ' + JSON.stringify(join(f.root, 'listed')) + '\n'); - chmodSync(join(f.root, 'bin/sleep'), 0o755); - const r = run(f, ['up'], { UBI_QUOTA_WAIT: '70' }); - const err = r.stderr.toString(); - expect(r.exitCode, err).toBe(0); - expect(err).toMatch(/waiting up to 70s more for room/); - expect(err).toMatch(/waiting up to 40s more for room/); - expect(err).not.toContain('quota preflight refused'); - expect(readFileSync(sleepLog, 'utf8').split('\n').slice(0, 2)).toEqual(['30', '30']); - expect(f.calls()).toContain('"create"'); - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 90_000); - - test('reap reports a failed list and a failed destroy as failures; UBI_TTL_HOURS rejects leading zeros and more than three digits', () => { - const f = fakeVm({ listed: 'eu-central-h1 ubirun-other-1000000000-bbbb-t1\n' }); - try { - writeFileSync(join(f.root, 'bin/curl'), '#!/usr/bin/env bash\nout=""; while [ $# -gt 0 ]; do [ "$1" = -o ] && out=$2; shift; done\nbody=$(cat)\ncase "$body" in\n *\'"list"\'*) echo "eu-central-h1 ubirun-other-1000000000-bbbb-t1" > "$out"; printf 200 ;;\n *\'"show"\'*) printf \'state: running\\nsize: standard-16\\nip4: 192.0.2.1\\n\' > "$out"; printf 200 ;;\n *) echo "boom" > "$out"; printf 500 ;;\nesac\n'); - const destroyFails = run(f, ['reap']); - expect(destroyFails.exitCode).toBe(1); - expect(destroyFails.stderr.toString()).toContain('failed to reap eu-central-h1/ubirun-other-1000000000-bbbb-t1'); - expect(destroyFails.stderr.toString()).toContain('reap: 1 of 1 expired VMs could not be destroyed'); - writeFileSync(join(f.root, 'bin/curl'), '#!/usr/bin/env bash\nout=""; while [ $# -gt 0 ]; do [ "$1" = -o ] && out=$2; shift; done\ncat >/dev/null; echo "down" > "$out"; printf 503\n'); - const listFails = run(f, ['reap']); - expect(listFails.exitCode).toBe(1); - expect(listFails.stderr.toString()).toContain("reap: could not list the project's VMs; nothing destroyed"); - for (const ttl of ['0', '08', '1000', 'abc']) { - const r = run(f, ['up'], { UBI_TTL_HOURS: ttl }); - expect(r.exitCode).toBe(1); - expect(r.stderr.toString()).toContain('UBI_TTL_HOURS must be a whole number of hours from 1 to 999'); - } - // Value: protects=an old UBI_TTL_HOURS=0 or bad UBI_LEGACY_HOURS never blocks commands that create nothing (down, list, owner); fails_when=the TTL checks move back to startup; why_new=the checks were global and blocked cleanup; seam=none; no_claim=real down against the API (owner stands in for every read-only command) - expect(run(f, ['owner'], { UBI_TTL_HOURS: '0', UBI_LEGACY_HOURS: '0' }).exitCode).toBe(0); - for (const legacy of ['0', '04', 'x']) { - const r = run(f, ['reap'], { UBI_LEGACY_HOURS: legacy }); - expect(r.exitCode).toBe(1); - expect(r.stderr.toString()).toContain('UBI_LEGACY_HOURS must be a whole number of hours from 1 to 9999'); - } - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 30_000); - - // GSTA-158 P3: detached phases, reattach and tier checkpoints. - const until = async (cond: () => boolean, ms: number) => { - const deadline = Date.now() + ms; - while (!cond()) { if (Date.now() > deadline) throw new Error('condition not met in time'); await Bun.sleep(100); } - }; - const vmDirs = (f: ReturnType) => (existsSync(f.state) ? readdirSync(f.state).filter(d => d.startsWith('ubirun-')) : []); - const phaseStarted = (f: ReturnType, phase: string) => vmDirs(f).some(d => existsSync(join(f.state, d, 'phase.' + phase + '.start'))); - const creates = (f: ReturnType) => f.calls().split('\n').filter(l => l.includes('"create"')).length; - const killSession = async (f: ReturnType, phase: string) => { - const pidFile = join(f.vm, '.ubirun-' + phase + '.pid'); - await until(() => existsSync(pidFile) && readFileSync(pidFile, 'utf8').trim() !== '', 10_000); - const pid = Number(readFileSync(pidFile, 'utf8').trim()); - try { process.kill(-pid, 'SIGKILL'); } catch { /* already gone */ } - await until(() => { try { process.kill(pid, 0); return false; } catch { return true; } }, 10_000); - }; - const gitSrc = (f: ReturnType, cfg?: unknown) => { - const dir = cfg === undefined ? join(f.root, 'repo') : src(f, cfg); - mkdirSync(dir, { recursive: true }); - writeFileSync(join(dir, 'README'), 'fixture repo\n'); - for (const args of [['init', '-q', '-b', 'main'], ['add', '-A'], ['commit', '-qm', 'fixture']]) { - const r = Bun.spawnSync(['git', '-c', 'user.email=t@t', '-c', 'user.name=t', ...args], { cwd: dir, timeout: 30_000 }); - if (r.exitCode !== 0) throw new Error('git ' + args.join(' ') + ': ' + r.stderr.toString()); - } - return dir; - }; - const TIERS = { setup: 'scripts/boot.sh', tiers: [{ name: 't1', command: 'make t1' }, { name: 't2', command: 'make t2' }, { name: 't3', command: 'make t3' }] }; - const bootCounts = (f: ReturnType, dir: string) => writeFileSync(join(dir, 'scripts/boot.sh'), 'echo boot >> ~/setup-count\n'); - const verifyLine = (out: string) => out.split('\n').find(l => l.startsWith('UBI_VERIFY: ')) ?? ''; - - test('run: a SIGKILLed runner leaves its VM; the same command reattaches, skips the finished setup and reports the command\'s exit', async () => { - const f = fakeVm(); - try { - const repo = gitSrc(f); - const setup = join(f.root, 'setup.sh'); writeFileSync(setup, 'echo setup >> ~/setup-count\n'); - writeFileSync(join(f.root, 'make-test.sh'), 'echo begin; sleep 4; echo end; exit 3\n'); - const args = ['run', '--src', repo, '--setup', setup, '--', 'make test']; - const a = f.spawn(args); - await until(() => vmDirs(f).some(d => existsSync(join(f.state, d, 'command.out')) && readFileSync(join(f.state, d, 'command.out'), 'utf8').includes('begin')), 30_000); - process.kill(a.pid, 'SIGKILL'); - await a.exited; - const name = vmDirs(f)[0]!; - expect(f.listed()).toContain(name); - const b = f.run(args); - const err = b.stderr.toString(); - expect(b.exitCode, err).toBe(3); - expect(err).toMatch(new RegExp('ubi-runner: reattached to ' + name + ' created_at=\\d{10} key=[0-9a-f]{12}\n')); - expect(b.stdout.toString().split('\n').filter(l => l === 'begin' || l === 'end')).toEqual(['begin', 'end']); - expect(err).toMatch(/command exited 3 after \d+s/); - expect(creates(f)).toBe(1); - expect(readFileSync(join(f.vm, 'setup-count'), 'utf8')).toBe('setup\n'); - expect(f.listed()).toBe(''); - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 90_000); - - test('run: a second live runner of the same command refuses with UBI_ATTACH_BUSY and creates nothing', async () => { - const f = fakeVm(); - try { - const repo = gitSrc(f); - writeFileSync(join(f.root, 'make-test.sh'), 'sleep 4; echo done\n'); - const args = ['run', '--src', repo, '--', 'make test']; - const a = f.spawn(args); - await until(() => phaseStarted(f, 'command'), 30_000); - const b = f.run(args); - expect(b.exitCode).toBe(1); - expect(b.stderr.toString()).toMatch(new RegExp('UBI_ATTACH_BUSY: vm=' + vmDirs(f)[0] + ' key=[0-9a-f]{12}: runner pid ' + a.pid + ' ')); - expect(await a.exited).toBe(0); - expect(await new Response(a.stdout).text()).toContain('done'); - expect(creates(f)).toBe(1); - expect(f.listed()).toBe(''); - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 90_000); - - test('run: HUP once the command is live detaches (UBI_DETACHED, exit 129, VM kept, no pass value on disk); a rerun reattaches and finishes', async () => { - const f = fakeVm(); - try { - const repo = gitSrc(f); - writeFileSync(join(f.root, 'make-test.sh'), 'sleep 3; echo finished\n'); - const args = ['run', '--src', repo, '--pass', 'UBI_TEST_TOKEN', '--', 'make test']; - const a = f.spawn(args, { UBI_TEST_TOKEN: 'hup-secret-value' }); - await until(() => phaseStarted(f, 'command'), 30_000); - await Bun.sleep(500); - process.kill(a.pid, 'SIGHUP'); - expect(await a.exited).toBe(129); - const name = vmDirs(f)[0]!; - const err = await new Response(a.stderr).text(); - expect(err).toMatch(new RegExp('UBI_DETACHED: vm=' + name + ' key=[0-9a-f]{12} reattach: rerun the same command')); - expect(f.listed()).toContain(name); - expect(f.calls()).not.toContain('destroy'); - for (const file of readdirSync(join(f.state, name))) { - const p = join(f.state, name, file); - if (lstatSync(p).isFile()) expect(readFileSync(p, 'utf8')).not.toContain('hup-secret-value'); - } - const b = f.run(args, { UBI_TEST_TOKEN: 'hup-secret-value' }); - expect(b.exitCode, b.stderr.toString()).toBe(0); - expect(b.stderr.toString()).toContain('reattached to ' + name); - expect(b.stdout.toString()).toContain('finished'); - expect(creates(f)).toBe(1); - expect(f.listed()).toBe(''); - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 90_000); - - test('run: INT and TERM during a live command still tear the VM down', async () => { - for (const sig of ['SIGTERM', 'SIGINT'] as const) { - const f = fakeVm(); - try { - const repo = gitSrc(f); - writeFileSync(join(f.root, 'make-test.sh'), 'sleep 3\n'); - const a = f.spawn(['run', '--src', repo, '--', 'make test']); - await until(() => phaseStarted(f, 'command'), 30_000); - await Bun.sleep(500); - process.kill(a.pid, sig); - expect(await a.exited).toBe(130); - expect(await new Response(a.stderr).text()).toMatch(/destroyed eu-central-h1\/ubirun-thrd1-/); - expect(f.listed()).toBe(''); - } finally { rmSync(f.root, { recursive: true, force: true }); } - } - }, 90_000); - - test('run: a remote session gone without an exit status exits 125 with UBI_ATTACH_LOST and the VM name', async () => { - const f = fakeVm(); - try { - const repo = gitSrc(f); - writeFileSync(join(f.root, 'make-test.sh'), 'sleep 30\n'); - const args = ['run', '--src', repo, '--', 'make test']; - const a = f.spawn(args); - await until(() => existsSync(join(f.vm, '.ubirun-command.pid')) && readFileSync(join(f.vm, '.ubirun-command.pid'), 'utf8').trim() !== '', 30_000); - process.kill(a.pid, 'SIGKILL'); - await a.exited; - await killSession(f, 'command'); - const name = vmDirs(f)[0]!; - const b = f.run(args); - expect(b.exitCode).toBe(125); - expect(b.stderr.toString()).toContain('UBI_ATTACH_LOST: vm=' + name + ' phase=command'); - expect(f.listed()).toBe(''); - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 90_000); - - test('run: the run key covers tree, command, setup content, env, pass value, image, origin/HEAD and the runner itself; a pass value never appears in it', () => { - const f = fakeVm(); - try { - const repo = gitSrc(f); - const setup = join(f.root, 'setup.sh'); writeFileSync(setup, 'echo one\n'); - const runner = join(f.root, 'runner-copy.sh'); writeFileSync(runner, readFileSync(join(DIR, 'ubi-runner.sh'), 'utf8')); - const key = (opts: { args?: string[]; env?: Record; cmd?: string; script?: string } = {}) => { - const r = Bun.spawnSync(['bash', opts.script ?? join(DIR, 'ubi-runner.sh'), 'run', '--probe', '--src', repo, '--setup', setup, '--env', 'MODE=a', '--pass', 'UBI_TEST_TOKEN', ...(opts.args ?? []), '--', opts.cmd ?? 'make test'], - { env: { ...process.env, PATH: join(f.root, 'bin') + ':' + process.env.PATH, UBICLOUD_API_KEY: 'offline', UBI_RUNNER_STATE: f.state, UBI_OWNER: 'thrd1', UBI_TEST_TOKEN: 'v1', ...(opts.env ?? {}) }, timeout: 30_000 }); - const out = r.stdout.toString() + r.stderr.toString(); - expect(out).not.toContain('v1'); - const m = /^UBI_ATTACH_INFO: none key=([0-9a-f]{12})$/m.exec(out); - if (!m) throw new Error(out); - return m[1]!; - }; - const base = key(); - expect(key()).toBe(base); - const variants: Record = {}; - variants.command = key({ cmd: 'make check' }); - variants.env = key({ args: ['--env', 'MODE=b'] }); - variants.pass = key({ env: { UBI_TEST_TOKEN: 'v2' } }); - variants.image = key({ args: ['-b', 'ubuntu-jammy'] }); - variants.size = key({ args: ['-s', 'standard-8'] }); - writeFileSync(runner, readFileSync(runner, 'utf8') + '# edited\n'); - variants.runner = key({ script: runner }); - writeFileSync(setup, 'echo two\n'); - variants.setup = key(); - writeFileSync(setup, 'echo one\n'); - expect(key()).toBe(base); - Bun.spawnSync(['git', 'update-ref', 'refs/remotes/origin/HEAD', 'HEAD'], { cwd: repo, timeout: 10_000 }); - variants.base = key(); - writeFileSync(join(repo, 'new-file.txt'), 'untracked source\n'); - variants.tree = key(); - for (const [what, k] of Object.entries(variants)) expect({ what, same: k === base }).toEqual({ what, same: false }); - expect(new Set(Object.values(variants)).size).toBe(Object.keys(variants).length); - expect(f.calls()).not.toContain('"create"'); - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 60_000); - - test('verify tiers: config refuses command with tiers, bad and duplicate tier names, before any API call', () => { - const f = fakeVm(); - try { - const cases: Array<[unknown, RegExp]> = [ - [{ command: 'make', tiers: [{ name: 'a', command: 'x' }] }, /UBI_CONFIG_TIERS_AND_COMMAND/], - [{ tiers: [{ name: 'Unit', command: 'x' }] }, /UBI_CONFIG_TIER_NAME: tier name 'Unit' must match/], - [{ tiers: [{ name: '-x', command: 'x' }] }, /UBI_CONFIG_TIER_NAME: tier name '-x'/], - [{ tiers: [{ name: 'a'.repeat(33), command: 'x' }] }, /UBI_CONFIG_TIER_NAME/], - [{ tiers: [{ name: 'a.b', command: 'x' }] }, /UBI_CONFIG_TIER_NAME/], - [{ tiers: [{ name: 'a', command: 'x' }, { name: 'a', command: 'y' }] }, /UBI_CONFIG_TIER_NAME: tier name 'a' appears twice/], - [{ tiers: [{ name: 'a' }] }, /UBI_CONFIG_TIER_NAME: tier 'a' needs a command/], - [{ tiers: [] }, /UBI_CONFIG_TIER_NAME: tiers must be a non-empty list/], - ]; - for (const [cfg, want] of cases) { - const dir = src(f, cfg); - const r = run(f, ['verify', '--src', dir]); - expect(r.exitCode).toBe(1); - expect(r.stderr.toString()).toMatch(want); - } - expect(f.calls()).toBe(''); - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 30_000); - - test('verify tiers: a run killed during tier 3 with its VM gone resumes on a new VM: setup reruns, t1 and t2 are reused, t3 reruns', async () => { - const f = fakeVm(); - try { - const dir = gitSrc(f, TIERS); - bootCounts(f, dir); - Bun.spawnSync(['git', '-c', 'user.email=t@t', '-c', 'user.name=t', 'commit', '-qam', 'boot'], { cwd: dir, timeout: 10_000 }); - writeFileSync(join(f.root, 'make-t3.sh'), 'touch ' + JSON.stringify(join(f.root, 't3-started')) + '; sleep 30\n'); - const a = f.spawn(['verify', '--src', dir]); - await until(() => existsSync(join(f.root, 't3-started')), 60_000); - process.kill(a.pid, 'SIGKILL'); - await a.exited; - await killSession(f, 'tier-t3'); - const first = vmDirs(f)[0]!; - writeFileSync(join(f.root, 'listed'), ''); - writeFileSync(join(f.root, 'make-t3.sh'), 'echo t3 ok\n'); - const b = f.run(['verify', '--src', dir]); - const err = b.stderr.toString(), out = b.stdout.toString(); - expect(b.exitCode, err).toBe(0); - expect(err).toMatch(/UBI_VERIFY_RESUME: tier=t1 reuse t1\.\d{10}\.\d+\.json \(\d+s\)\n/); - expect(err).toMatch(/UBI_VERIFY_RESUME: tier=t2 reuse t2\.\d{10}\.\d+\.json \(\d+s\)\n/); - expect(err).toMatch(/UBI_VERIFY_RESUME: tier=t3 run: interrupted \(only a start record, t3\.\d{10}\.\d+\.json\)\n/); - expect(err).toContain('run record for ' + first + ': the VM is gone; starting a new one'); - expect(err).not.toContain('reattached to'); - expect(creates(f)).toBe(2); - expect(readFileSync(join(f.vm, 'setup-count'), 'utf8')).toBe('boot\nboot\n'); - expect(out).not.toContain('ran: make t1'); - expect(out).toContain('t3 ok'); - expect(verifyLine(out)).toMatch(/^UBI_VERIFY: tree=[0-9a-f]{12} key=[0-9a-f]{12} tiers=3 passed=3 reused=t1,t2 failed=none prior_failures=none saved_s=\d+$/); - expect(f.listed()).toBe(''); - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 120_000); - - test('verify tiers: a fail record blocks a rerun until --rerun-failed (then prior_failures); passes are reused unless stale, tampered, --no-resume or the tree changed', () => { - const f = fakeVm(); - try { - const dir = gitSrc(f, TIERS); - writeFileSync(join(f.root, 'make-t2.sh'), 'echo t2 broke; exit 7\n'); - const first = run(f, ['verify', '--src', dir]); - expect(first.exitCode, first.stderr.toString()).toBe(7); - expect(first.stdout.toString()).not.toContain('ran: make t3'); - expect(verifyLine(first.stdout.toString())).toMatch(/ tiers=3 passed=1 reused=none failed=t2 prior_failures=none saved_s=0$/); - expect(creates(f)).toBe(1); - - const blocked = run(f, ['verify', '--src', dir]); - expect(blocked.exitCode).toBe(7); - expect(blocked.stderr.toString()).toMatch(/UBI_VERIFY_RESUME: tier=t2 fail t2\.\d{10}\.\d+\.json \(\d+s, exit 7\): not rerun without --rerun-failed/); - expect(verifyLine(blocked.stdout.toString())).toMatch(/ failed=t2 prior_failures=none /); - expect(creates(f)).toBe(1); - - writeFileSync(join(f.root, 'make-t2.sh'), 'echo t2 fixed\n'); - const rerun = run(f, ['verify', '--src', dir, '--rerun-failed']); - expect(rerun.exitCode, rerun.stderr.toString()).toBe(0); - expect(rerun.stderr.toString()).toMatch(/UBI_VERIFY_RESUME: tier=t1 reuse /); - expect(rerun.stderr.toString()).toMatch(/UBI_VERIFY_RESUME: tier=t2 run: prior fail t2\.\d{10}\.\d+\.json \(exit 7; --rerun-failed\)/); - expect(verifyLine(rerun.stdout.toString())).toMatch(/ tiers=3 passed=3 reused=t1 failed=none prior_failures=t2:7 saved_s=\d+$/); - expect(creates(f)).toBe(2); - - const all = run(f, ['verify', '--src', dir]); - expect(all.exitCode).toBe(0); - expect(verifyLine(all.stdout.toString())).toMatch(/ passed=3 reused=t1,t2,t3 failed=none prior_failures=none saved_s=\d+$/); - expect(all.stderr.toString()).toContain('every tier passed under this key; no VM needed'); - expect(creates(f)).toBe(2); - - const probe = (args: string[] = [], extra: Record = {}) => run(f, ['verify', '--src', dir, '--probe', ...args], extra).stderr.toString(); - expect(probe(['--no-resume'])).toMatch(/tier=t1 run: --no-resume\n[^]*tier=t2 run: --no-resume\n[^]*tier=t3 run: --no-resume\n/); - expect(probe([], { UBI_CHECKPOINT_MAX_AGE: '0' })).toMatch(/tier=t1 run: pass t1\.\d{10}\.\d+\.json is \d+s old \(UBI_CHECKPOINT_MAX_AGE=0\)/); - const keyDir = join(f.checkpoints, readdirSync(f.checkpoints)[0]!); - const t1Log = readdirSync(keyDir).filter(n => n.startsWith('t1.') && n.endsWith('.log')).sort().at(-1)!; - writeFileSync(join(keyDir, t1Log), 'tampered\n'); - const tampered = probe(); - expect(tampered).toMatch(/tier=t1 run: pass t1\.\d{10}\.\d+\.json: its log does not re-hash/); - expect(tampered).toMatch(/tier=t2 reuse /); - writeFileSync(join(dir, 'new-source.ts'), 'export {}\n'); - const moved = probe(); - for (const t of ['t1', 't2', 't3']) expect(moved).toContain('UBI_VERIFY_RESUME: tier=' + t + ' run: no record'); - expect(creates(f)).toBe(2); - // Records are write-once JSON; every one carries its generation. - const gens = readdirSync(keyDir).filter(n => n.endsWith('.json')).map(n => JSON.parse(readFileSync(join(keyDir, n), 'utf8')).generation as number).sort((x, y) => x - y); - expect(gens).toEqual(gens.map((_, i) => i + 1)); - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 120_000); - - test('verify tiers: a reattach whose VM TTL no longer covers the run left plus UBI_REATTACH_MARGIN starts a fresh VM, destroys the old one and reuses the passed tiers', async () => { - const f = fakeVm(); - try { - const dir = gitSrc(f, TIERS); - bootCounts(f, dir); - Bun.spawnSync(['git', '-c', 'user.email=t@t', '-c', 'user.name=t', 'commit', '-qam', 'boot'], { cwd: dir, timeout: 10_000 }); - writeFileSync(join(f.root, 'make-t3.sh'), 'touch ' + JSON.stringify(join(f.root, 't3-started')) + '; sleep 30\n'); - const env = { UBI_REAP: '0', UBI_TTL_HOURS: '1' }; - const a = f.spawn(['verify', '--src', dir], env); - await until(() => existsSync(join(f.root, 't3-started')), 60_000); - process.kill(a.pid, 'SIGKILL'); - await a.exited; - const first = vmDirs(f)[0]!; - expect(first).toMatch(/-t1$/); - const probe = (extra: Record) => run(f, ['verify', '--src', dir, '--probe'], { ...env, ...extra }).stdout.toString(); - expect(probe({})).toMatch(new RegExp('UBI_ATTACH_INFO: vm=' + first + ' created_at=\\d{10} ')); - // A 1 h TTL leaves about 3600 s: t3's 1200 s run timeout plus a 2700 s margin does not fit. - const short = { UBI_RUN_TIMEOUT: '1200', UBI_REATTACH_MARGIN: '2700' }; - expect(probe(short)).toMatch(new RegExp('UBI_ATTACH_INFO: none key=[0-9a-f]{12} ttl_short vm=' + first + ' ttl_left=\\d+s needs=\\d+s \\(run still to do \\d+s \\+ UBI_REATTACH_MARGIN=2700s\\)')); - expect(creates(f)).toBe(1); - writeFileSync(join(f.root, 'make-t3.sh'), 'echo t3 ok\n'); - const b = run(f, ['verify', '--src', dir], { ...env, ...short }); - const err = b.stderr.toString(); - expect(b.exitCode, err).toBe(0); - expect(err).toMatch(new RegExp('UBI_REATTACH_TTL_SHORT: vm=' + first + ' key=[0-9a-f]{12} ttl_left=\\d+s needs=\\d+s \\(run still to do \\d+s \\+ UBI_REATTACH_MARGIN=2700s\\): starting a fresh VM \\(tier checkpoints carry over\\) and destroying ' + first)); - expect(err).not.toContain('reattached to'); - expect(err).toMatch(/UBI_VERIFY_RESUME: tier=t1 reuse /); - expect(err).toMatch(/UBI_VERIFY_RESUME: tier=t2 reuse /); - expect(verifyLine(b.stdout.toString())).toMatch(/ tiers=3 passed=3 reused=t1,t2 failed=none /); - expect(creates(f)).toBe(2); - expect(f.listed()).toBe(''); - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 120_000); - - test('verify tiers: a writer whose start record is no longer the newest publishes nothing (stale writer)', () => { - const py = /\nCKPT_PY='\n([^']*)'\n/.exec(readFileSync(join(DIR, 'ubi-runner.sh'), 'utf8'))![1]!; - const root = mkdtempSync(join(tmpdir(), 'ubi-ckpt-')); - try { - const d = join(root, 'key'); - const log = join(root, 'tier.log'); writeFileSync(log, 'ok\n'); - const publish = (status: string, startgen: string, pid: string) => Bun.spawnSync(['python3', '-c', py, 'publish', d, 't1', status, 'vm', '100', '200', '0', status === 'start' ? '' : log, startgen, 'k', 'tree', pid], { timeout: 30_000 }); - expect(publish('start', '0', '1').stdout.toString().trim()).toBe('1'); - expect(publish('start', '0', '2').stdout.toString().trim()).toBe('2'); - const stale = publish('pass', '1', '1'); - expect(stale.exitCode).toBe(2); - expect(stale.stderr.toString()).toContain('stale writer: its start record is generation 1, the newest for t1 is 2'); - expect(readdirSync(d).filter(n => n.endsWith('.json')).length).toBe(2); - expect(publish('pass', '2', '2').stdout.toString().trim()).toBe('3'); - } finally { rmSync(root, { recursive: true, force: true }); } - }); - - test('verify tiers: a tier that edits the source gets no pass record (UBI_VERIFY_TREE_MOVED) and reruns next time', () => { - const f = fakeVm(); - try { - const dir = gitSrc(f, { tiers: [{ name: 'mover', command: 'make mover' }, { name: 'clean', command: 'make clean' }] }); - writeFileSync(join(f.root, 'make-mover.sh'), 'echo edited > moved.txt\n'); - const r = run(f, ['verify', '--src', dir]); - expect(r.exitCode, r.stderr.toString()).toBe(0); - expect(r.stderr.toString()).toMatch(/UBI_VERIFY_TREE_MOVED: tier=mover before=[0-9a-f]{12} after=[0-9a-f]{12}: a tier must leave the source tree as it found it; no pass record/); - expect(verifyLine(r.stdout.toString())).toMatch(/ tiers=2 passed=2 reused=none failed=none /); - const next = run(f, ['verify', '--src', dir, '--probe']).stderr.toString(); - expect(next).toMatch(/tier=mover run: interrupted \(only a start record/); - expect(next).toMatch(/tier=clean reuse /); - } finally { rmSync(f.root, { recursive: true, force: true }); } - }, 90_000); -});