From e79209c47e42585737692a2f422252eb16538203 Mon Sep 17 00:00:00 2001 From: garrytan Date: Mon, 26 Oct 2026 18:56:34 +0000 Subject: [PATCH] ubicloud: the remote session file is per runner invocation and setsid is optional on the image Two watched commands on one VM wrote the same ~/.ubirun-.sid, so a cancel could end the other's session; the file now carries the runner's pid and is removed when the command ends. An image without setsid runs the command through plain bash instead of failing with exit 127. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 2 +- scripts/ubicloud/ubi-runner.sh | 25 ++++++++++++++----------- test/ubicloud-runner.test.ts | 6 +++--- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1493b0331..4f56e1cb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ - **Quota preflight**: `up` adds the size's vCPUs to the project's running VMs and refuses before the create request when that exceeds the known ceiling (`UBI_QUOTA_VCPUS`, or the maximum the last API refusal printed, remembered in the state root); `UBI_QUOTA_WAIT=` polls for room instead of refusing. The per-owner `usage` table is printed either way. - **Watchdog and heartbeat**: `run`, `verify` and `ssh NAME COMMAND` stream the remote output (both streams kept in the VM's state dir) through a watchdog: a heartbeat every `UBI_HEARTBEAT` seconds (300), and a command silent for `UBI_IDLE_TIMEOUT` seconds (1800) or older than `UBI_RUN_TIMEOUT` (0, unlimited) is killed in its own session with exit 124 and its last 40 lines on stderr. `run` also prints the VM's TTL and its `down` recovery command as soon as the VM is ready. - **`verify`**: runs the repository's committed `.gstack/ubicloud.json` (setup, command, size, location, storage, image, env, pass, pull; flags and a trailing `-- COMMAND` override it) and refuses without the file or a command, so every lane runs one definition instead of its own wrapper. The file is repository content and gets no reach beyond the checkout: `setup` and each `pull` destination must be relative paths that resolve inside `--src` (absolute, `~` and `..` paths are refused before any API call), and a name under `pass` is forwarded only when the command line grants it (`--pass NAME`, or every listed name with `--trust-config`). `--pass` never forwards `UBICLOUD_*`, from the file or the command line. -- **Watched commands keep their stdin and exit codes**: `ssh NAME COMMAND` hands the caller's stdin to the remote command (`printf payload | gstack-ubi ssh NAME 'cat > file'` wrote an empty file once the watchdog backgrounded the client), a command that finished while the watchdog read the clock is never reported as a timeout, a hung setup script exits 124 like a hung command instead of 1, Ctrl-C on a watched command takes the ssh session down with it, and when the watchdog fires the runner also ends the command's session on the VM (`setsid` there, leader pid in `~/.ubirun-.sid`, `pkill -s` bounded to 20 s), since killing the local client alone leaves a silent remote command running until it next writes. +- **Watched commands keep their stdin and exit codes**: `ssh NAME COMMAND` hands the caller's stdin to the remote command (`printf payload | gstack-ubi ssh NAME 'cat > file'` wrote an empty file once the watchdog backgrounded the client), a command that finished while the watchdog read the clock is never reported as a timeout, a hung setup script exits 124 like a hung command instead of 1, Ctrl-C on a watched command takes the ssh session down with it, and when the watchdog fires the runner also ends the command's session on the VM (`setsid` there, leader pid in `~/.ubirun--.sid`, `pkill -s` bounded to 20 s), since killing the local client alone leaves a silent remote command running until it next writes. - **Quota ceiling and reaper outcomes**: the maximum a refusal cached is trusted for a day, then the API's own answer decides again; the refusal line says where the ceiling came from and names `UBI_QUOTA_VCPUS`, and its `vCPU quota exhausted` wording is the one `pregate-remote.sh` retries once on. `reap` returns 1 when the list fails (nothing destroyed) or an expired VM could not be destroyed, instead of 0 either way; a TTL suffix is read in base 10 (`UBI_TTL_HOURS` takes 0 to 999 with no leading zero). ### Tests diff --git a/scripts/ubicloud/ubi-runner.sh b/scripts/ubicloud/ubi-runner.sh index a9e861b01..25e71360c 100755 --- a/scripts/ubicloud/ubi-runner.sh +++ b/scripts/ubicloud/ubi-runner.sh @@ -182,13 +182,13 @@ mtime() { stat -c %Y "$1" 2>/dev/null || stat -f %m "$1"; } # sq STRING: STRING as one single-quoted shell word. sq() { printf "'%s'" "$(printf %s "$1" | sed "s/'/'\\\\''/g")"; } -# remote_cancel NAME PHASE: end the session a watched PHASE started on the VM -# (its leader pid is in ~/.ubirun-PHASE.sid there), bounded to 20 seconds. -# Killing the local ssh client alone leaves a silent remote command running -# until it next writes to the closed channel. +# remote_cancel NAME SIDFILE: end the session a watched command started on the +# VM (its leader pid is in SIDFILE there), bounded to 20 seconds. Killing the +# local ssh client alone leaves a silent remote command running until it next +# writes to the closed channel. remote_cancel() { - local name=$1 phase=$2 cpid i=0 - remote "$name" "sid=\$(cat ~/.ubirun-$phase.sid 2>/dev/null); [ -z \"\$sid\" ] || { pkill -TERM -s \"\$sid\"; sleep 2; pkill -KILL -s \"\$sid\"; }; true" >/dev/null 2>&1 /dev/null); [ -z \"\$sid\" ] || { pkill -TERM -s \"\$sid\"; sleep 2; pkill -KILL -s \"\$sid\"; }; true" >/dev/null 2>&1 /dev/null && [ "$i" -lt 20 ]; do sleep 1; i=$(( i + 1 )); done kill "$cpid" 2>/dev/null || true @@ -201,14 +201,17 @@ remote_cancel() { # RUN_TIMEOUT seconds in total: exit 124, reason and the last 40 lines on # stderr. The ssh runs in its own session (python3's setsid) so the watchdog # kills it and anything it started, not just the shell that launched it; on -# the VM the command runs in its own session too, whose leader pid is written -# to ~/.ubirun-PHASE.sid so remote_cancel can end it when the client is killed. +# the VM the command runs in its own session too (setsid where the image has +# it), whose leader pid is written to ~/.ubirun-PHASE-.sid so +# remote_cancel can end it when the client is killed. watched() { - local name=$1 phase=$2 cmd=$3 out pid rc=0 start now last_beat idle elapsed why="" old_traps + local name=$1 phase=$2 cmd=$3 out pid rc=0 start now last_beat idle elapsed why="" old_traps inner sidf + sidf="~/.ubirun-$phase-$$.sid" load "$name" out="$(state_dir "$name")/$phase.log" : >"$out" - cmd="setsid -w bash -c $(sq "printf %s \$\$ >~/.ubirun-$phase.sid; $cmd")" + inner=$(sq "printf %s \$\$ >$sidf; $cmd") + cmd="if command -v setsid >/dev/null 2>&1; then setsid -w bash -c $inner; else bash -c $inner; fi; rc=\$?; rm -f $sidf; exit \$rc" # The caller's stdin is handed to the background ssh explicitly (a background # job in a non-interactive shell would otherwise read /dev/null), so piped # input reaches the remote command as it did before the watchdog. @@ -244,7 +247,7 @@ watched() { kill -TERM -- "-$pid" 2>/dev/null || kill "$pid" 2>/dev/null || true wait "$pid" 2>/dev/null || true trap - INT TERM; eval "$old_traps" - remote_cancel "$name" "$phase" + remote_cancel "$name" "$sidf" return 124 fi if [ "$HEARTBEAT" -gt 0 ] && [ $(( now - last_beat )) -ge "$HEARTBEAT" ]; then diff --git a/test/ubicloud-runner.test.ts b/test/ubicloud-runner.test.ts index d53f70eb0..2243b7fc2 100644 --- a/test/ubicloud-runner.test.ts +++ b/test/ubicloud-runner.test.ts @@ -252,7 +252,7 @@ esac expect(err).toMatch(/watchdog: ssh printed nothing for \d+s \(UBI_IDLE_TIMEOUT=2\); killing it. Last output:/); expect(err.indexOf('line one')).toBeGreaterThan(err.indexOf('Last output:')); expect(readFileSync(join(f.root, 'state/vm-a/ssh.log'), 'utf8')).toContain('line one'); - expect(readFileSync(join(f.root, 'cancel.log'), 'utf8')).toMatch(/sid=\$\(cat ~\/\.ubirun-ssh\.sid .*pkill -TERM -s/); + expect(readFileSync(join(f.root, 'cancel.log'), 'utf8')).toMatch(/sid=\$\(cat ~\/\.ubirun-ssh-\d+\.sid .*pkill -TERM -s/); writeFileSync(ssh, '#!/usr/bin/env bash\nfor i in 1 2 3 4 5 6; do echo "tick $i"; sleep 0.5; done\n'); const beating = Bun.spawnSync(['bash', join(DIR, 'ubi-runner.sh'), 'ssh', 'vm-a', 'true'], { env: { ...env, UBI_IDLE_TIMEOUT: '0', UBI_HEARTBEAT: '1' }, timeout: 60_000 }); expect(beating.exitCode).toBe(0); @@ -371,8 +371,8 @@ describe('ubi-runner run, verify and quota against a fake API and a fake ssh', ( expect(err).toMatch(/watchdog: setup printed nothing for \d+s \(UBI_IDLE_TIMEOUT=2\)/); expect(err).toContain('setup failed (exit 124)'); expect(f.ssh()).not.toContain('bash -lc'); - expect(readFileSync(join(f.root, 'cancel.log'), 'utf8')).toContain('cat ~/.ubirun-setup.sid'); - expect(f.ssh()).toMatch(/setsid -w bash -c 'printf %s \$\$ >~\/\.ubirun-setup\.sid; cd work\/[^ ]+ && \. ~\/\.ubirun-env && bash -l ~\/\.ubirun-setup\.sh'/); + expect(readFileSync(join(f.root, 'cancel.log'), 'utf8')).toMatch(/cat ~\/\.ubirun-setup-\d+\.sid/); + expect(f.ssh()).toMatch(/setsid -w bash -c 'printf %s \$\$ >~\/\.ubirun-setup-\d+\.sid; cd work\/[^ ]+ && \. ~\/\.ubirun-env && bash -l ~\/\.ubirun-setup\.sh'; else bash -c '[^']*'; fi; rc=\$\?; rm -f ~\/\.ubirun-setup-\d+\.sid; exit \$rc/); expect(f.calls()).toContain('destroy'); expect(err).toMatch(/destroyed eu-central-h1\/ubirun-thrd1-\d{10}-[0-9a-f]{8}-t4/); expect(f.listed()).toBe('');