From 33d83412c6cfd6d9239217e8a3aa1669c2bcbaf2 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Tue, 3 Nov 2026 17:40:51 -0800 Subject: [PATCH] docs(pregate): after a restart, rerun the remote pregate to reattach (GST-1028) (#3886) * docs(pregate): after a machine restart, rerun the same remote pregate so it reattaches (GST-1028) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test(ship-driver): pin the restart rerun line (GST-1028) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes (name the full reattach key; destroy the old VM when a rerun does not reattach) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(changelog): GST-1028 pregate reattach fragment Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(pregate): the run key hashes the runner script and setup, not a gstack version Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(changelog): GST-1028 fragment matches the merged driver text Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- changelog.d/gst-1028-pregate-reattach-docs.md | 9 +++++++++ docs/pregate.md | 17 +++++++++++++++-- scripts/ubicloud/pregate-remote.sh | 2 +- ship/DRIVER.md | 2 +- ship/DRIVER.md.tmpl | 2 +- test/ship-driver.test.ts | 4 ++-- test/ubicloud-pregate-remote.test.ts | 2 +- 7 files changed, 30 insertions(+), 8 deletions(-) create mode 100644 changelog.d/gst-1028-pregate-reattach-docs.md diff --git a/changelog.d/gst-1028-pregate-reattach-docs.md b/changelog.d/gst-1028-pregate-reattach-docs.md new file mode 100644 index 000000000..ddb69bdef --- /dev/null +++ b/changelog.d/gst-1028-pregate-reattach-docs.md @@ -0,0 +1,9 @@ +**A remote pregate killed by a machine restart is rerun, not destroyed (GST-1028 b).** When a lane's machine restarted, its background pregate died but its Ubicloud VM kept running, and the line the pregate printed at create time told lanes to destroy it. Lanes did, and paid for a fresh VM, even though rerunning the same command reattaches to the live VM and finishes the run. + +### Changed +- **The create-time line says to rerun first.** `pregate-remote.sh` now prints "rerun the same command to reattach; if the rerun does not reattach to it, or you give up, destroy it with: …". +- **`docs/pregate.md` explains reattach.** It lists what the run key covers (the tree, the planned files, the commit `origin/HEAD` points at, `UBI_LOCATION`, size, environment, and the runner script and setup it uploads), so a lane knows when a rerun will find the old VM. It also says to destroy the old VM when the rerun creates a new one instead. +- **The /ship driver says it too.** `ship/DRIVER.md` tells a lane to rerun the same command after a restart, and stays under its 12,288-byte cap. + +### Tests +- `test/ubicloud-pregate-remote.test.ts` pins the new create-time line, and `test/ship-driver.test.ts` pins the driver's restart line. diff --git a/docs/pregate.md b/docs/pregate.md index 5ca21a0b9..8520efd60 100644 --- a/docs/pregate.md +++ b/docs/pregate.md @@ -651,8 +651,21 @@ the VM to stop at whichever comes first, that point or the cap's wall (at most 45 minutes, minus 3 for teardown). A deadline that leaves the VM under 300 s refuses before any VM (`PREGATE_REMOTE_TIMEOUT_TOO_SHORT`). At create time it prints the VM's name and the exact -`bash scripts/ubicloud/ubi-runner.sh down ` line to destroy it if the -run is killed before its teardown line. The whole script is parsed before it +`bash scripts/ubicloud/ubi-runner.sh down ` line to destroy it. + +A run killed before its teardown line leaves its VM running. A machine restart +kills a background pregate this way. Don't destroy the VM first: rerun the same +`gstack-pregate` command before anything it depends on changes. The run key +covers the tree, the planned file set, the commit `origin/HEAD` points at (a +`git fetch` that moves it counts), `UBI_LOCATION`, the size, the environment, +and the runner script and setup it uploads. When the rerun prints `reattaching to VM ` with +the name from the create line, it finishes the run on that VM and bills from +the VM's original create. When it creates a new VM instead (the key changed, +or the restart came before the runner recorded the VM), destroy the old one +with the `down` line from its create line. Use the same line when you give up +on the run. A reattach whose billed minutes leave the cap no room +refuses (`PREGATE_REMOTE_CAP_SPENT`) and prints both choices: destroy the VM, +or rerun with a higher `--spend-cap`. The whole script is parsed before it runs, so editing it during a run changes nothing. `--timeout 2700` gives a whole-tier run room inside the 45-minute wall: diff --git a/scripts/ubicloud/pregate-remote.sh b/scripts/ubicloud/pregate-remote.sh index c7eb3f90f..4ac5fe5c1 100755 --- a/scripts/ubicloud/pregate-remote.sh +++ b/scripts/ubicloud/pregate-remote.sh @@ -248,7 +248,7 @@ while :; do rm -f "$log.pid" "$log.timedout" "$log.left" t0=$(date +%s) vm_name="ubirun-$(bash "$here/ubi-runner.sh" owner)-$(date +%s)-$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')-t$(( (stop_after + 900 + 3599) / 3600 ))" - [ -n "$prior_vm" ] || say "VM $vm_name; if this run is killed before its teardown line, destroy it with: bash scripts/ubicloud/ubi-runner.sh down $vm_name" + [ -n "$prior_vm" ] || say "VM $vm_name; if this run is killed before its teardown line, rerun the same command to reattach; if the rerun does not reattach to it, or you give up, destroy it with: bash scripts/ubicloud/ubi-runner.sh down $vm_name" UBI_LEFT_FILE="$log.left" python3 -c 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \ bash -c "$vm_session" pregate-remote-vm "$stop_after" "$log" "$stream" "$here/ubi-runner.sh" "${args[@]}" -n "$vm_name" & vmrun=$! diff --git a/ship/DRIVER.md b/ship/DRIVER.md index 177d38f13..66e2c0963 100644 --- a/ship/DRIVER.md +++ b/ship/DRIVER.md @@ -37,7 +37,7 @@ A dispatching program takes `BEFORE` (R and C) as readers start and passes `--dispatched-at --dispatched-credits --readers ` with the reading after the last settles. **Waits.** No call waits past 200 s: a program returns what still runs 200 s after its first line; the next -call resumes the poll. Lanes, pregate, `finish` run in background. Never end the turn while +call resumes the poll. Lanes, pregate, `finish` run in background; after a restart, rerun the same command. Never end the turn while readers run (each wakes it). A small waiting child saves nothing. Past 100 minutes, stop. Returns: ≤32 KB per program; split longer reads at headings, each byte once, before first use: phase programs end with `gstack-ship-parts next --run --phase --room <32000 - its output>`; on diff --git a/ship/DRIVER.md.tmpl b/ship/DRIVER.md.tmpl index 31c260211..f3f9594fc 100644 --- a/ship/DRIVER.md.tmpl +++ b/ship/DRIVER.md.tmpl @@ -35,7 +35,7 @@ A dispatching program takes `BEFORE` (R and C) as readers start and passes `--dispatched-at --dispatched-credits --readers ` with the reading after the last settles. **Waits.** No call waits past 200 s: a program returns what still runs 200 s after its first line; the next -call resumes the poll. Lanes, pregate, `finish` run in background. Never end the turn while +call resumes the poll. Lanes, pregate, `finish` run in background; after a restart, rerun the same command. Never end the turn while readers run (each wakes it). A small waiting child saves nothing. Past 100 minutes, stop. Returns: ≤32 KB per program; split longer reads at headings, each byte once, before first use: phase programs end with `gstack-ship-parts next --run --phase --room <32000 - its output>`; on diff --git a/test/ship-driver.test.ts b/test/ship-driver.test.ts index 6b00c15c5..749abe859 100644 --- a/test/ship-driver.test.ts +++ b/test/ship-driver.test.ts @@ -177,11 +177,11 @@ describe('variant G: the call plan, observe budget, waits and audit readers (GST expect(doc).toContain('gstack-ship-budget close --reading R --credits C'); }); - // Value: protects=no call waits past the 5-minute prompt cache and no turn ends while readers run: a program returns 200 s after its first line, the next call resumes the poll, long commands run in background, and Capy waits on operations only when no reader runs, with the choice of a ping over a waiting child stated where the parent and Capy read it; fails_when=a phase waits in one program until slow work ends (cache probe: a call after a 7-minute gap cost ~2,820 credits against ~70), the parent ends its turn on `wait` while readers run (GSTA-189: 23 of 25 waits were one wake per finished reader), the cadence drifts back toward the cache lifetime (240 s counted from the poll loop, or 270 s), or the reason is dropped; + // Value: protects=no call waits past the 5-minute prompt cache and no turn ends while readers run: a program returns 200 s after its first line, the next call resumes the poll, long commands run in background, and Capy waits on operations only when no reader runs, with the choice of a ping over a waiting child stated where the parent and Capy read it; fails_when=a phase waits in one program until slow work ends (cache probe: a call after a 7-minute gap cost ~2,820 credits against ~70), the parent ends its turn on `wait` while readers run (GSTA-189: 23 of 25 waits were one wake per finished reader), the cadence drifts back toward the cache lifetime (240 s counted from the poll loop, or 270 s), or the reason is dropped, or a lane is no longer told to rerun the same command after a machine restart (GST-1028: lanes destroyed pregate VMs a rerun would have reattached); // why_new=GSTA-167 calls plan C1; seam=none; no_claim=the host's cache lifetime or wake policy test('waits return 200 s after a program starts and never end the turn while readers run, in the driver and the Capy skill', () => { const waits = flat(doc.slice(doc.indexOf('**Waits.**'), doc.indexOf('\n\n', doc.indexOf('**Waits.**')))); - for (const s of ['No call waits past 200 s', '200 s after its first line', 'the next call resumes the poll', 'run in background', 'Never end the turn while readers run', + for (const s of ['No call waits past 200 s', '200 s after its first line', 'the next call resumes the poll', 'run in background', 'after a restart, rerun the same command', 'Never end the turn while readers run', 'A small waiting child saves nothing', 'Past 100 minutes']) expect(waits).toContain(s); const capy = flat(read('capy/skill/gstack/SKILL.md')); for (const s of ['never let 5 minutes pass between calls', 'While readers run, never end the turn on `wait`', '`duration_seconds: 200`', 'one `tools.subagent.list` per ~20 s tick', diff --git a/test/ubicloud-pregate-remote.test.ts b/test/ubicloud-pregate-remote.test.ts index 79afd1296..82709c95c 100644 --- a/test/ubicloud-pregate-remote.test.ts +++ b/test/ubicloud-pregate-remote.test.ts @@ -272,7 +272,7 @@ describe('pregate-remote.sh keyed path (stub ubi-runner)', () => { expect(create.exit, create.out).toBe(4); expect(create.out).toContain('PREGATE_REMOTE_INFRA: infrastructure failure (ubi-runner exit 1): ubi-runner: create failed'); expect(create.out).toMatch(/VM: eu-central-h1\/(ubirun-stub-\d+-[0-9a-f]{8}-t1); ubi-runner: gone: eu-central-h1\/\1 \(never existed or already destroyed\)/); - expect(create.out).toMatch(/VM (ubirun-stub-\d+-[0-9a-f]{8}-t1); if this run is killed before its teardown line, destroy it with: bash scripts\/ubicloud\/ubi-runner\.sh down \1\n/); + expect(create.out).toMatch(/VM (ubirun-stub-\d+-[0-9a-f]{8}-t1); if this run is killed before its teardown line, rerun the same command to reattach; if the rerun does not reattach to it, or you give up, destroy it with: bash scripts\/ubicloud\/ubi-runner\.sh down \1\n/); expect(create.out).toMatch(/log: \S+\/\.context\/pregate-remote\/\S+\/run\.log/); expect(create.out).toContain('spent=0.00 (0 billed min on standard-16)'); const setup = gate(fixture(), ['test/alpha.test.ts'], { ...KEY, STUB_SETUP_FAIL: '1' });