diff --git a/docs/pregate.md b/docs/pregate.md index ca55368c9..33bf9b813 100644 --- a/docs/pregate.md +++ b/docs/pregate.md @@ -251,10 +251,20 @@ it runs the same files locally in parallel shards selection (skill-e2e and the other paid globs in `test/helpers/paid-test-set.ts`) are named and left to the paid PR gate; a selection of only paid files reports `tests=0`, which never clears the obligation. The command is the same either way -(`--remote` always needs a cap; the keyless path spends nothing). `--timeout -2700` keeps pregate's default 600 s deadline from cutting off a slow VM run -inside its 45-minute wall (the VM's teardown runs in its own session, so a -pregate timeout never strands it): +(`--remote` always needs a cap; the keyless path spends nothing). + +pregate kills the gate at its `--timeout` (default 600 s) with a 5 s grace, so +the script finishes before that deadline: it reads the deadline from +`PREGATE_TIMEOUT_S`, else from the `gstack-pregate` command it runs under (its +`--timeout`, else 600), reserves 120 s for teardown plus 15 s margin, and tells +the VM to stop at whichever comes first, that point or the cap's wall (at most +45 minutes, minus 3 for teardown). A deadline that leaves the VM under 300 s +refuses before any VM (`PREGATE_REMOTE_TIMEOUT_TOO_SHORT`). At create time it +prints the VM's name and the exact +`bash scripts/ubicloud/ubi-runner.sh down ` line to destroy it if the +run is killed before its teardown line. The whole script is parsed before it +runs, so editing it during a run changes nothing. `--timeout 2700` gives a +whole-tier run room inside the 45-minute wall: ```bash gstack-pregate --stage tests --remote --spend-cap 2 --timeout 2700 --allow-repo-commands @@ -262,9 +272,9 @@ gstack-pregate --stage tests --remote --spend-cap 2 --timeout 2700 --allow-repo- Measured (October 2026): keyed, all 1,280 free files on standard-16 took 404 s from start to teardown (create 24 s, setup 72 s, tests 250 s) for $0.06. -Keyless on 4 cores, 98 files took 103 s, and a 798-file pregate selection took -3,550 s in an earlier wave, so keyless is practical for small selections only. A large selection on a small -machine needs a key, or a `--timeout` longer than the run. Keyless also runs in +Keyless on 4 cores, 98 files took 103 s, and GSTA-35's 798-file selection took +3,550 s, so keyless is practical for small selections only. A large selection +on a small machine needs a key, or a `--timeout` longer than the run. Keyless also runs in your environment, not CI's: a file that needs what setup-free-suite.sh installs (Playwright's Chromium, Xvfb) fails locally until you install it. @@ -280,7 +290,8 @@ Every refusal line starts with a reason code: | `PREGATE_REMOTE_PAID_ONLY` | 0 | every selected file is a paid test file; `tests=0` cannot clear | | `PREGATE_REMOTE_CAP_TOO_LOW` | 5 | the cap cannot buy a 20-minute run at the size's price; nothing created | | `PREGATE_REMOTE_NO_PRICE` | 5 | `UBI_SIZE` has no list price in the script | -| `PREGATE_REMOTE_INFRA` | 4 | create, setup, SSH or the wall timeout failed; the VM, its teardown line and `spent=` follow | +| `PREGATE_REMOTE_TIMEOUT_TOO_SHORT` | 5 | pregate's deadline leaves the VM under 300 s after the 135 s teardown reserve; nothing created | +| `PREGATE_REMOTE_INFRA` | 4 | create, setup, SSH or the stop deadline failed; the VM, its teardown line and `spent=` follow | | `PREGATE_REMOTE_INTERRUPTED` | 130 | TERM or INT; ubi-runner tears the VM down on its own | diff --git a/scripts/ubicloud/pregate-remote.sh b/scripts/ubicloud/pregate-remote.sh index 18045bded..ab47f9ea2 100755 --- a/scripts/ubicloud/pregate-remote.sh +++ b/scripts/ubicloud/pregate-remote.sh @@ -29,14 +29,17 @@ # # Exit codes: 0 pass (or a paid-only selection, tests=0); 1 test failures (or # a rejected file list); 2 missing input; 3 tree or plan mismatch (or the tree -# moved during the run); 4 infrastructure failure (create, setup, SSH, wall -# timeout); 5 the spend cap refuses the run. Every refusal line starts with a +# moved during the run); 4 infrastructure failure (create, setup, SSH, stop +# deadline); 5 the spend cap or pregate's deadline refuses the run. Every refusal line starts with a # PREGATE_REMOTE_* reason code (docs/pregate.md#remote-gate). Nothing is # uploaded or spent before the local tree check. set -euo pipefail # Keep heredoc bodies on temp files, not the pipe window (test/heredoc-pipe-deadlock.test.ts). BASH_COMPAT=50 +# The whole script is one braced block, so bash parses it before running and an +# edit to this file during a run cannot change what the run executes. +{ here=$(cd "$(dirname "$0")" && pwd) root=$(cd "$here/../.." && pwd) rerun='gstack-pregate --stage tests --remote --spend-cap --allow-repo-commands' @@ -48,6 +51,7 @@ stop() { local code=$1 reason=$2; shift 2; say "$reason: $1"; shift; [ $# -eq 0 [ -n "${PREGATE_TREE:-}" ] || stop 2 PREGATE_REMOTE_USAGE "PREGATE_TREE is not set; run: $rerun" [ -f "$PREGATE_FILES_FILE" ] || stop 2 PREGATE_REMOTE_USAGE "PREGATE_FILES_FILE=$PREGATE_FILES_FILE does not exist; run: $rerun" +started=$(date +%s) wtree() { (cd "$root" && bin/gstack-wtree); } tree=$(wtree) || stop 3 PREGATE_REMOTE_TREE_MISMATCH "bin/gstack-wtree could not fingerprint $root" "nothing uploaded, nothing spent" "rerun: $rerun" if [ "$tree" != "$PREGATE_TREE" ]; then @@ -112,7 +116,35 @@ usd() { awk -v m="$1" -v p="$ppm" 'BEGIN { s = m * p; c = int(s * 100); if (c < if [ "$wall_min" -lt "$min_min" ]; then stop 5 PREGATE_REMOTE_CAP_TOO_LOW "worst case \$$(usd "$min_min") ($min_min min x \$$ppm/min on $size) exceeds the spend cap \$$cap" "nothing created, nothing spent" "rerun with --spend-cap $(usd "$min_min") or more" fi -say "$count files on one $size VM; wall timeout $wall_min min, worst case \$$(usd "$wall_min") under the cap \$$cap" +# The enclosing gstack-pregate kills this gate at its --timeout (default 600 s) +# with a 5 s grace, so the VM must be told to stop early enough for teardown to +# be confirmed first: 120 s reserved for teardown plus 15 s margin. The deadline +# comes from PREGATE_TIMEOUT_S, else from the gstack-pregate process this gate +# runs under (its --timeout, else 600); run by hand there is none. +outer_s=${PREGATE_TIMEOUT_S:-} outer_from=PREGATE_TIMEOUT_S +if [ -z "$outer_s" ]; then + outer_from="" pid=$PPID + for _ in 1 2 3 4 5 6 7 8; do + { [ -n "$pid" ] && [ "$pid" -gt 1 ]; } || break + cmdline=$(ps -o args= -p "$pid" 2>/dev/null) || break + case $cmdline in + *gstack-pregate*) + outer_s=$(sed -n 's/.*--timeout[ =]\([0-9][0-9]*\).*/\1/p' <<<"$cmdline") + outer_s=${outer_s:-600} outer_from="gstack-pregate --timeout" + break ;; + esac + pid=$(ps -o ppid= -p "$pid" 2>/dev/null | tr -d ' ') + done +fi +stop_after=$(( (wall_min - 3) * 60 )) stop_why="the cap's $wall_min min wall minus 3 min for teardown" +if [ -n "$outer_s" ]; then + outer_stop=$(( outer_s - ($(date +%s) - started) - 135 )) + if [ "$outer_stop" -lt 300 ]; then + stop 5 PREGATE_REMOTE_TIMEOUT_TOO_SHORT "$outer_from ${outer_s}s leaves ${outer_stop}s for the VM after 120 s reserved for teardown and 15 s margin (at least 300 s needed)" "nothing created, nothing spent" "rerun with --timeout 2700: $rerun" + fi + if [ "$outer_stop" -lt "$stop_after" ]; then stop_after=$outer_stop stop_why="$outer_from ${outer_s}s minus 120 s teardown reserve and 15 s margin"; fi +fi +say "$count files on one $size VM; worst case \$$(usd "$wall_min") under the cap \$$cap; the VM is told to stop after ${stop_after}s ($stop_why)" digest=$(tr -d '\r' <"$files" | grep . | LC_ALL=C sort | sha256sum | cut -d' ' -f1) prelude="$logs/vm-setup.sh" @@ -171,7 +203,6 @@ exit "$rc" ' stream="$logs/stream.log" at() { grep -m1 -E "^[0-9]+ $1" "$log" | cut -d' ' -f1 || true; } -started=$(date +%s) ( while sleep 60 /dev/null 2>&1; do say "still running ($(( $(date +%s) - started )) s)"; done ) & heartbeat=$! vmrun="" tailer="" @@ -185,8 +216,10 @@ while :; do : >"$stream" rm -f "$log.pid" "$log.timedout" t0=$(date +%s) + vm_name="ubirun-$(bash "$here/ubi-runner.sh" owner)-$(date +%s)-$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')" + say "VM $vm_name; if this run is killed before its teardown line, destroy it with: bash scripts/ubicloud/ubi-runner.sh down $vm_name" python3 -c 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \ - bash -c "$vm_session" pregate-remote-vm "$(( (wall_min - 3) * 60 ))" "$log" "$stream" "$here/ubi-runner.sh" "${args[@]}" & + bash -c "$vm_session" pregate-remote-vm "$stop_after" "$log" "$stream" "$here/ubi-runner.sh" "${args[@]}" -n "$vm_name" & vmrun=$! tail -n +1 -f "$stream" & tailer=$! @@ -239,6 +272,7 @@ fi vm=$(sed -n 's/^[0-9]* ubi-runner: creating \([^ ]*\) .*/\1/p' "$log" | tail -1) down=$(grep -E 'ubi-runner: (destroyed|gone:|WARNING|FAILED to destroy)' "$log" | tail -1 | cut -d' ' -f2- || true) cause=$(grep -E 'ubi-runner: ' "$log" | grep -vE 'ubi-runner: (destroyed|gone:|destroy requested|waiting for an in-flight)' | tail -1 | cut -d' ' -f2- || true) -[ "$rc" -ne 124 ] || cause="wall timeout of $(( wall_min - 3 )) min reached (cap \$$cap); the VM was torn down" +[ "$rc" -ne 124 ] || cause="the VM was told to stop after ${stop_after}s ($stop_why) and torn down" stop 4 PREGATE_REMOTE_INFRA "infrastructure failure (ubi-runner exit $rc): ${cause:-no ubi-runner message}" \ "VM: ${vm:-none created}${down:+; $down}" "log: $log" "spent=$spent ($billed billed min on $size)" +} diff --git a/test/ubicloud-pregate-remote.test.ts b/test/ubicloud-pregate-remote.test.ts index 28c6ab52f..2a4a66a9c 100644 --- a/test/ubicloud-pregate-remote.test.ts +++ b/test/ubicloud-pregate-remote.test.ts @@ -39,13 +39,15 @@ const UBI_RUNNER_STUB = `#!/usr/bin/env bash # sleeps before the command; TERM tears down as ubi-runner's trap does. # Every teardown is appended to $STUB_CALLS.down. set -uo pipefail +[ "$1" != owner ] || { echo stub; exit 0; } [ "$1" = run ] || exit 2 shift printf '%s\\n' "$*" >>"$STUB_CALLS" -src=. setup="" size="" envs=() +src=. setup="" size="" name=unnamed envs=() while [ $# -gt 0 ]; do case $1 in -s) size=$2; shift 2 ;; + -n) name=$2; shift 2 ;; --src) src=$2; shift 2 ;; --setup) setup=$2; shift 2 ;; --env) envs+=("$2"); shift 2 ;; @@ -56,7 +58,6 @@ while [ $# -gt 0 ]; do done cmd=$* log() { echo "ubi-runner: $*" >&2; } -name=ubirun-stub-1700000000-abcd log "creating eu-central-h1/$name ($size)" calls=$(wc -l <"$STUB_CALLS") fail=\${STUB_FAIL:-} @@ -181,7 +182,7 @@ describe('pregate-remote.sh keyed path (stub ubi-runner)', () => { expect(r.exit, r.out).toBe(0); expect(r.report).toEqual({ tree: f.tree, tests: 2, failed: 0, spent: '0.01' }); expect(remoteReportProblem(r.report!, f.tree)).toBeNull(); - expect(r.out).toContain('2 files on one standard-16 VM; wall timeout 45 min, worst case $0.38 under the cap $2'); + expect(r.out).toContain("2 files on one standard-16 VM; worst case $0.38 under the cap $2; the VM is told to stop after 2520s (the cap's 45 min wall minus 3 min for teardown)"); expect(r.out).toMatch(/phases: create \d+s, setup \d+s, run \d+s, teardown \d+s; total \d+s \(target 600s\)/); expect(r.out.trim().split('\n').at(-1)).toMatch(/^PREGATE_REMOTE: /); expect(r.calls).toHaveLength(1); @@ -230,13 +231,14 @@ describe('pregate-remote.sh keyed path (stub ubi-runner)', () => { const create = gate(fixture(), ['test/alpha.test.ts'], { ...KEY, STUB_FAIL: 'create' }); expect(create.exit, create.out).toBe(4); expect(create.out).toContain('PREGATE_REMOTE_INFRA: infrastructure failure (ubi-runner exit 1): ubi-runner: create failed'); - expect(create.out).toContain('VM: eu-central-h1/ubirun-stub-1700000000-abcd; ubi-runner: gone: eu-central-h1/ubirun-stub-1700000000-abcd (never existed or already destroyed)'); + expect(create.out).toMatch(/VM: eu-central-h1\/(ubirun-stub-\d+-[0-9a-f]{8}); ubi-runner: gone: eu-central-h1\/\1 \(never existed or already destroyed\)/); + expect(create.out).toMatch(/VM (ubirun-stub-\d+-[0-9a-f]{8}); if this run is killed before its teardown line, destroy it with: bash scripts\/ubicloud\/ubi-runner\.sh down \1\n/); expect(create.out).toMatch(/log: \S+\/\.context\/pregate-remote\/\S+\/run\.log/); expect(create.out).toContain('spent=0.00 (0 billed min on standard-16)'); const setup = gate(fixture(), ['test/alpha.test.ts'], { ...KEY, STUB_SETUP_FAIL: '1' }); expect(setup.exit, setup.out).toBe(4); expect(setup.out).toContain('ubi-runner: setup failed'); - expect(setup.out).toContain('; ubi-runner: destroyed eu-central-h1/ubirun-stub-1700000000-abcd'); + expect(setup.out).toMatch(/; ubi-runner: destroyed eu-central-h1\/ubirun-stub-\d+-[0-9a-f]{8}/); expect(setup.out).toContain('spent=0.01 (1 billed min on standard-16)'); const ssh = gate(fixture(), ['test/alpha.test.ts'], { ...KEY, STUB_FAIL: 'ssh' }); expect(ssh.exit, ssh.out).toBe(4); @@ -270,13 +272,13 @@ describe('pregate-remote.sh keyed path (stub ubi-runner)', () => { expect([edited, listed, moved].every(r => r.report === null)).toBe(true); }, 90_000); - test('killing the gate\'s process group (pregate\'s timeout) never cuts teardown short; TERM to the gate tears the VM down', async () => { + test('killing the gate\'s process group (pregate\'s timeout) never cuts teardown short; TERM under an outer deadline tears the VM down; an edit to the script mid-run changes nothing', async () => { const run = (f: Fixture) => { const list = path.join(path.dirname(f.repo), 'files.txt'); fs.writeFileSync(list, 'test/alpha.test.ts\n'); return Bun.spawn(['bash', 'scripts/ubicloud/pregate-remote.sh'], { cwd: f.repo, detached: true, stdout: 'pipe', stderr: 'pipe', env: { ...process.env, ...KEY, UBI_SIZE: '', GSTACK_FREE_JOBS: '2', STUB_CALLS: f.calls, STUB_VM_HOME: f.vm, STUB_BIN: f.bin, STUB_HANG: '4', - PREGATE_FILES_FILE: list, PREGATE_TREE: f.tree, PREGATE_SPEND_CAP_USD: '2' } as Record }); + PREGATE_FILES_FILE: list, PREGATE_TREE: f.tree, PREGATE_SPEND_CAP_USD: '2', PREGATE_TIMEOUT_S: '900' } as Record }); }; const until = async (cond: () => boolean, ms: number) => { const deadline = Date.now() + ms; @@ -289,7 +291,7 @@ describe('pregate-remote.sh keyed path (stub ubi-runner)', () => { process.kill(-a.pid, 'SIGKILL'); await a.exited; await until(() => fs.existsSync(killed.calls + '.down'), 30_000); - expect(fs.readFileSync(killed.calls + '.down', 'utf8')).toBe('exit ubirun-stub-1700000000-abcd\n'); + expect(fs.readFileSync(killed.calls + '.down', 'utf8')).toMatch(/^exit ubirun-stub-\d+-[0-9a-f]{8}\n$/); const termed = fixture(); const b = run(termed); await until(() => vmLog(termed).includes('ubi-runner: running on'), 30_000); @@ -297,9 +299,38 @@ describe('pregate-remote.sh keyed path (stub ubi-runner)', () => { expect(await b.exited).toBe(130); expect(await new Response(b.stdout).text()).toContain('interrupted: ubi-runner is tearing the VM down on its own'); await until(() => fs.existsSync(termed.calls + '.down'), 10_000); - expect(fs.readFileSync(termed.calls + '.down', 'utf8')).toBe('term ubirun-stub-1700000000-abcd\n'); + expect(fs.readFileSync(termed.calls + '.down', 'utf8')).toMatch(/^term ubirun-stub-\d+-[0-9a-f]{8}\n$/); + const edited = fixture(); + const c = run(edited); + await until(() => vmLog(edited).includes('ubi-runner: running on'), 30_000); + const script = path.join(edited.repo, 'scripts/ubicloud/pregate-remote.sh'); + fs.writeFileSync(script, '#!/usr/bin/env bash\n# an edit that shifts every byte offset\n' + fs.readFileSync(script, 'utf8').split('\n').slice(1).join('\n')); + expect(await c.exited).toBe(0); + expect((await new Response(c.stdout).text()).trim().split('\n').at(-1)).toMatch(new RegExp(`^PREGATE_REMOTE: tree=${edited.tree} tests=1 failed=0 spent=`)); }, 90_000); + test('an enclosing pregate deadline shorter than the wall sets when the VM is told to stop; one too short refuses before any VM', () => { + const f = fixture(); + const viaEnv = gate(f, ['test/alpha.test.ts'], { ...KEY, PREGATE_TIMEOUT_S: '900' }); + expect(viaEnv.exit, viaEnv.out).toBe(0); + expect(viaEnv.out).toMatch(/the VM is told to stop after (76[0-5])s \(PREGATE_TIMEOUT_S 900s minus 120 s teardown reserve and 15 s margin\)/); + const short = gate(f, ['test/alpha.test.ts'], { ...KEY, PREGATE_TIMEOUT_S: '400' }); + expect(short.exit).toBe(5); + expect(short.out).toMatch(/PREGATE_REMOTE_TIMEOUT_TOO_SHORT: PREGATE_TIMEOUT_S 400s leaves 26\ds for the VM after 120 s reserved for teardown and 15 s margin \(at least 300 s needed\)/); + expect(short.out).toContain('nothing created, nothing spent'); + expect(fs.readFileSync(f.calls, 'utf8').trim().split('\n')).toHaveLength(1); + // Run under a process named gstack-pregate, the deadline is read from its --timeout (600 s when it passes none). + const shim = path.join(path.dirname(f.repo), 'gstack-pregate'); + fs.writeFileSync(shim, '#!/usr/bin/env bash\nexec bash scripts/ubicloud/pregate-remote.sh\n', { mode: 0o755 }); + const run = (args: string[]) => { + const r = Bun.spawnSync(['bash', '-c', '"$0" "$@"; exit $?', shim, ...args], { cwd: f.repo, timeout: 90_000, env: { ...process.env, ...KEY, UBI_SIZE: '', GSTACK_FREE_JOBS: '2', + STUB_CALLS: f.calls, STUB_VM_HOME: f.vm, STUB_BIN: f.bin, PREGATE_FILES_FILE: path.join(path.dirname(f.repo), 'files.txt'), PREGATE_TREE: f.tree, PREGATE_SPEND_CAP_USD: '2' } as Record }); + return r.stdout.toString() + r.stderr.toString(); + }; + expect(run(['--stage', 'tests', '--remote', '--timeout', '1200'])).toMatch(/told to stop after 106[0-5]s \(gstack-pregate --timeout 1200s minus/); + expect(run(['--stage', 'tests', '--remote'])).toMatch(/told to stop after 46[0-5]s \(gstack-pregate --timeout 600s minus/); + }, 120_000); + test('the VM environment is bun run test:ubicloud\'s (scripts/ubicloud/test-free.sh), so the two cannot drift', () => { const script = fs.readFileSync(path.join(ROOT, 'scripts/ubicloud/pregate-remote.sh'), 'utf8'); const wrapper = fs.readFileSync(path.join(ROOT, 'scripts/ubicloud/test-free.sh'), 'utf8');