diff --git a/CHANGELOG.md b/CHANGELOG.md index 1493b0331..4f56e1cb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ - **Quota preflight**: `up` adds the size's vCPUs to the project's running VMs and refuses before the create request when that exceeds the known ceiling (`UBI_QUOTA_VCPUS`, or the maximum the last API refusal printed, remembered in the state root); `UBI_QUOTA_WAIT=` polls for room instead of refusing. The per-owner `usage` table is printed either way. - **Watchdog and heartbeat**: `run`, `verify` and `ssh NAME COMMAND` stream the remote output (both streams kept in the VM's state dir) through a watchdog: a heartbeat every `UBI_HEARTBEAT` seconds (300), and a command silent for `UBI_IDLE_TIMEOUT` seconds (1800) or older than `UBI_RUN_TIMEOUT` (0, unlimited) is killed in its own session with exit 124 and its last 40 lines on stderr. `run` also prints the VM's TTL and its `down` recovery command as soon as the VM is ready. - **`verify`**: runs the repository's committed `.gstack/ubicloud.json` (setup, command, size, location, storage, image, env, pass, pull; flags and a trailing `-- COMMAND` override it) and refuses without the file or a command, so every lane runs one definition instead of its own wrapper. The file is repository content and gets no reach beyond the checkout: `setup` and each `pull` destination must be relative paths that resolve inside `--src` (absolute, `~` and `..` paths are refused before any API call), and a name under `pass` is forwarded only when the command line grants it (`--pass NAME`, or every listed name with `--trust-config`). `--pass` never forwards `UBICLOUD_*`, from the file or the command line. -- **Watched commands keep their stdin and exit codes**: `ssh NAME COMMAND` hands the caller's stdin to the remote command (`printf payload | gstack-ubi ssh NAME 'cat > file'` wrote an empty file once the watchdog backgrounded the client), a command that finished while the watchdog read the clock is never reported as a timeout, a hung setup script exits 124 like a hung command instead of 1, Ctrl-C on a watched command takes the ssh session down with it, and when the watchdog fires the runner also ends the command's session on the VM (`setsid` there, leader pid in `~/.ubirun-.sid`, `pkill -s` bounded to 20 s), since killing the local client alone leaves a silent remote command running until it next writes. +- **Watched commands keep their stdin and exit codes**: `ssh NAME COMMAND` hands the caller's stdin to the remote command (`printf payload | gstack-ubi ssh NAME 'cat > file'` wrote an empty file once the watchdog backgrounded the client), a command that finished while the watchdog read the clock is never reported as a timeout, a hung setup script exits 124 like a hung command instead of 1, Ctrl-C on a watched command takes the ssh session down with it, and when the watchdog fires the runner also ends the command's session on the VM (`setsid` there, leader pid in `~/.ubirun--.sid`, `pkill -s` bounded to 20 s), since killing the local client alone leaves a silent remote command running until it next writes. - **Quota ceiling and reaper outcomes**: the maximum a refusal cached is trusted for a day, then the API's own answer decides again; the refusal line says where the ceiling came from and names `UBI_QUOTA_VCPUS`, and its `vCPU quota exhausted` wording is the one `pregate-remote.sh` retries once on. `reap` returns 1 when the list fails (nothing destroyed) or an expired VM could not be destroyed, instead of 0 either way; a TTL suffix is read in base 10 (`UBI_TTL_HOURS` takes 0 to 999 with no leading zero). ### Tests diff --git a/scripts/ubicloud/ubi-runner.sh b/scripts/ubicloud/ubi-runner.sh index a9e861b01..25e71360c 100755 --- a/scripts/ubicloud/ubi-runner.sh +++ b/scripts/ubicloud/ubi-runner.sh @@ -182,13 +182,13 @@ mtime() { stat -c %Y "$1" 2>/dev/null || stat -f %m "$1"; } # sq STRING: STRING as one single-quoted shell word. sq() { printf "'%s'" "$(printf %s "$1" | sed "s/'/'\\\\''/g")"; } -# remote_cancel NAME PHASE: end the session a watched PHASE started on the VM -# (its leader pid is in ~/.ubirun-PHASE.sid there), bounded to 20 seconds. -# Killing the local ssh client alone leaves a silent remote command running -# until it next writes to the closed channel. +# remote_cancel NAME SIDFILE: end the session a watched command started on the +# VM (its leader pid is in SIDFILE there), bounded to 20 seconds. Killing the +# local ssh client alone leaves a silent remote command running until it next +# writes to the closed channel. remote_cancel() { - local name=$1 phase=$2 cpid i=0 - remote "$name" "sid=\$(cat ~/.ubirun-$phase.sid 2>/dev/null); [ -z \"\$sid\" ] || { pkill -TERM -s \"\$sid\"; sleep 2; pkill -KILL -s \"\$sid\"; }; true" >/dev/null 2>&1 /dev/null); [ -z \"\$sid\" ] || { pkill -TERM -s \"\$sid\"; sleep 2; pkill -KILL -s \"\$sid\"; }; true" >/dev/null 2>&1 /dev/null && [ "$i" -lt 20 ]; do sleep 1; i=$(( i + 1 )); done kill "$cpid" 2>/dev/null || true @@ -201,14 +201,17 @@ remote_cancel() { # RUN_TIMEOUT seconds in total: exit 124, reason and the last 40 lines on # stderr. The ssh runs in its own session (python3's setsid) so the watchdog # kills it and anything it started, not just the shell that launched it; on -# the VM the command runs in its own session too, whose leader pid is written -# to ~/.ubirun-PHASE.sid so remote_cancel can end it when the client is killed. +# the VM the command runs in its own session too (setsid where the image has +# it), whose leader pid is written to ~/.ubirun-PHASE-.sid so +# remote_cancel can end it when the client is killed. watched() { - local name=$1 phase=$2 cmd=$3 out pid rc=0 start now last_beat idle elapsed why="" old_traps + local name=$1 phase=$2 cmd=$3 out pid rc=0 start now last_beat idle elapsed why="" old_traps inner sidf + sidf="~/.ubirun-$phase-$$.sid" load "$name" out="$(state_dir "$name")/$phase.log" : >"$out" - cmd="setsid -w bash -c $(sq "printf %s \$\$ >~/.ubirun-$phase.sid; $cmd")" + inner=$(sq "printf %s \$\$ >$sidf; $cmd") + cmd="if command -v setsid >/dev/null 2>&1; then setsid -w bash -c $inner; else bash -c $inner; fi; rc=\$?; rm -f $sidf; exit \$rc" # The caller's stdin is handed to the background ssh explicitly (a background # job in a non-interactive shell would otherwise read /dev/null), so piped # input reaches the remote command as it did before the watchdog. @@ -244,7 +247,7 @@ watched() { kill -TERM -- "-$pid" 2>/dev/null || kill "$pid" 2>/dev/null || true wait "$pid" 2>/dev/null || true trap - INT TERM; eval "$old_traps" - remote_cancel "$name" "$phase" + remote_cancel "$name" "$sidf" return 124 fi if [ "$HEARTBEAT" -gt 0 ] && [ $(( now - last_beat )) -ge "$HEARTBEAT" ]; then diff --git a/test/ubicloud-runner.test.ts b/test/ubicloud-runner.test.ts index d53f70eb0..2243b7fc2 100644 --- a/test/ubicloud-runner.test.ts +++ b/test/ubicloud-runner.test.ts @@ -252,7 +252,7 @@ esac expect(err).toMatch(/watchdog: ssh printed nothing for \d+s \(UBI_IDLE_TIMEOUT=2\); killing it. Last output:/); expect(err.indexOf('line one')).toBeGreaterThan(err.indexOf('Last output:')); expect(readFileSync(join(f.root, 'state/vm-a/ssh.log'), 'utf8')).toContain('line one'); - expect(readFileSync(join(f.root, 'cancel.log'), 'utf8')).toMatch(/sid=\$\(cat ~\/\.ubirun-ssh\.sid .*pkill -TERM -s/); + expect(readFileSync(join(f.root, 'cancel.log'), 'utf8')).toMatch(/sid=\$\(cat ~\/\.ubirun-ssh-\d+\.sid .*pkill -TERM -s/); writeFileSync(ssh, '#!/usr/bin/env bash\nfor i in 1 2 3 4 5 6; do echo "tick $i"; sleep 0.5; done\n'); const beating = Bun.spawnSync(['bash', join(DIR, 'ubi-runner.sh'), 'ssh', 'vm-a', 'true'], { env: { ...env, UBI_IDLE_TIMEOUT: '0', UBI_HEARTBEAT: '1' }, timeout: 60_000 }); expect(beating.exitCode).toBe(0); @@ -371,8 +371,8 @@ describe('ubi-runner run, verify and quota against a fake API and a fake ssh', ( expect(err).toMatch(/watchdog: setup printed nothing for \d+s \(UBI_IDLE_TIMEOUT=2\)/); expect(err).toContain('setup failed (exit 124)'); expect(f.ssh()).not.toContain('bash -lc'); - expect(readFileSync(join(f.root, 'cancel.log'), 'utf8')).toContain('cat ~/.ubirun-setup.sid'); - expect(f.ssh()).toMatch(/setsid -w bash -c 'printf %s \$\$ >~\/\.ubirun-setup\.sid; cd work\/[^ ]+ && \. ~\/\.ubirun-env && bash -l ~\/\.ubirun-setup\.sh'/); + expect(readFileSync(join(f.root, 'cancel.log'), 'utf8')).toMatch(/cat ~\/\.ubirun-setup-\d+\.sid/); + expect(f.ssh()).toMatch(/setsid -w bash -c 'printf %s \$\$ >~\/\.ubirun-setup-\d+\.sid; cd work\/[^ ]+ && \. ~\/\.ubirun-env && bash -l ~\/\.ubirun-setup\.sh'; else bash -c '[^']*'; fi; rc=\$\?; rm -f ~\/\.ubirun-setup-\d+\.sid; exit \$rc/); expect(f.calls()).toContain('destroy'); expect(err).toMatch(/destroyed eu-central-h1\/ubirun-thrd1-\d{10}-[0-9a-f]{8}-t4/); expect(f.listed()).toBe('');