diff --git a/docs/pregate.md b/docs/pregate.md index c9c36c8ba..70667b328 100644 --- a/docs/pregate.md +++ b/docs/pregate.md @@ -615,11 +615,16 @@ prints the VM's name and the exact `bash scripts/ubicloud/ubi-runner.sh down ` line to destroy it. A run killed before its teardown line leaves its VM running. A machine restart -kills a background pregate this way. Don't destroy the VM: rerun the same -`gstack-pregate` command on the same tree. It reattaches to that VM -(`reattaching to VM `), finishes the run, and bills from the VM's -original create. Use the `down` line only when you are giving up on the run -or the tree has changed. A reattach whose billed minutes leave the cap no room +kills a background pregate this way. Don't destroy the VM first: rerun the same +`gstack-pregate` command before anything it depends on changes. The run key +covers the tree, the planned file set, the commit `origin/HEAD` points at (a +`git fetch` that moves it counts), `UBI_LOCATION`, the size, the environment +and the gstack version. When the rerun prints `reattaching to VM ` with +the name from the create line, it finishes the run on that VM and bills from +the VM's original create. When it creates a new VM instead (the key changed, +or the restart came before the runner recorded the VM), destroy the old one +with the `down` line from its create line. Use the same line when you give up +on the run. A reattach whose billed minutes leave the cap no room refuses (`PREGATE_REMOTE_CAP_SPENT`) and prints both choices: destroy the VM, or rerun with a higher `--spend-cap`. The whole script is parsed before it runs, so editing it during a run changes nothing. `--timeout 2700` gives a diff --git a/scripts/ubicloud/pregate-remote.sh b/scripts/ubicloud/pregate-remote.sh index 06299ddee..4ac5fe5c1 100755 --- a/scripts/ubicloud/pregate-remote.sh +++ b/scripts/ubicloud/pregate-remote.sh @@ -248,7 +248,7 @@ while :; do rm -f "$log.pid" "$log.timedout" "$log.left" t0=$(date +%s) vm_name="ubirun-$(bash "$here/ubi-runner.sh" owner)-$(date +%s)-$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')-t$(( (stop_after + 900 + 3599) / 3600 ))" - [ -n "$prior_vm" ] || say "VM $vm_name; if this run is killed before its teardown line, rerun the same command to reattach; to give up on it, destroy it with: bash scripts/ubicloud/ubi-runner.sh down $vm_name" + [ -n "$prior_vm" ] || say "VM $vm_name; if this run is killed before its teardown line, rerun the same command to reattach; if the rerun does not reattach to it, or you give up, destroy it with: bash scripts/ubicloud/ubi-runner.sh down $vm_name" UBI_LEFT_FILE="$log.left" python3 -c 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \ bash -c "$vm_session" pregate-remote-vm "$stop_after" "$log" "$stream" "$here/ubi-runner.sh" "${args[@]}" -n "$vm_name" & vmrun=$! diff --git a/test/ubicloud-pregate-remote.test.ts b/test/ubicloud-pregate-remote.test.ts index af53836c4..82709c95c 100644 --- a/test/ubicloud-pregate-remote.test.ts +++ b/test/ubicloud-pregate-remote.test.ts @@ -272,7 +272,7 @@ describe('pregate-remote.sh keyed path (stub ubi-runner)', () => { expect(create.exit, create.out).toBe(4); expect(create.out).toContain('PREGATE_REMOTE_INFRA: infrastructure failure (ubi-runner exit 1): ubi-runner: create failed'); expect(create.out).toMatch(/VM: eu-central-h1\/(ubirun-stub-\d+-[0-9a-f]{8}-t1); ubi-runner: gone: eu-central-h1\/\1 \(never existed or already destroyed\)/); - expect(create.out).toMatch(/VM (ubirun-stub-\d+-[0-9a-f]{8}-t1); if this run is killed before its teardown line, rerun the same command to reattach; to give up on it, destroy it with: bash scripts\/ubicloud\/ubi-runner\.sh down \1\n/); + expect(create.out).toMatch(/VM (ubirun-stub-\d+-[0-9a-f]{8}-t1); if this run is killed before its teardown line, rerun the same command to reattach; if the rerun does not reattach to it, or you give up, destroy it with: bash scripts\/ubicloud\/ubi-runner\.sh down \1\n/); expect(create.out).toMatch(/log: \S+\/\.context\/pregate-remote\/\S+\/run\.log/); expect(create.out).toContain('spent=0.00 (0 billed min on standard-16)'); const setup = gate(fixture(), ['test/alpha.test.ts'], { ...KEY, STUB_SETUP_FAIL: '1' });