From 5567c493b6638f318cfbea4c7dda16d04f63c286 Mon Sep 17 00:00:00 2001
From: garrytan <garrytan@gmail.com>
Date: Sat, 17 Oct 2026 19:31:41 +0000
Subject: [PATCH] feat(pregate): declare scripts/ubicloud/pregate-remote.sh as
 gstack's remote gate (keyed Ubicloud VM, keyless local shards)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
---
 .gstack/pregate.json                 |   1 +
 scripts/ubicloud/pregate-remote.sh   | 224 +++++++++++++++++
 test/ubicloud-pregate-remote.test.ts | 349 +++++++++++++++++++++++++++
 3 files changed, 574 insertions(+)
 create mode 100755 scripts/ubicloud/pregate-remote.sh
 create mode 100644 test/ubicloud-pregate-remote.test.ts

diff --git a/.gstack/pregate.json b/.gstack/pregate.json
index 69573040e..f9cb2832c 100644
--- a/.gstack/pregate.json
+++ b/.gstack/pregate.json
@@ -1,6 +1,7 @@
 {
   "$schema": "https://github.com/garrytan/gstack/docs/pregate.md#pregate-v1",
   "runner": "bun test {files}",
+  "remote_gate": "bash scripts/ubicloud/pregate-remote.sh",
   "dependencies": {
     ".github/workflows/**": ["test/pregate-selection.test.ts", "test/free-tests-workflow-wiring.test.ts"],
     "scripts/gen-ship-goldens.ts": ["test/host-config.test.ts"],
diff --git a/scripts/ubicloud/pregate-remote.sh b/scripts/ubicloud/pregate-remote.sh
new file mode 100755
index 000000000..a49c44a4e
--- /dev/null
+++ b/scripts/ubicloud/pregate-remote.sh
@@ -0,0 +1,224 @@
+#!/usr/bin/env bash
+# pregate-remote.sh — gstack's declared remote_gate (.gstack/pregate.json).
+#
+# gstack-pregate runs this under 8 cores for --stage tests --remote:
+#
+#   gstack-pregate --stage tests --remote --spend-cap 2 --allow-repo-commands
+#
+# with PREGATE_FILES_FILE (the selected free test files, one per line),
+# PREGATE_TREE (the working-tree fingerprint being tested) and
+# PREGATE_SPEND_CAP_USD. It prints the contract line pregate reads
+# (docs/pregate.md#remote-gate):
+#
+#   PREGATE_REMOTE: tree=<tree tested> tests=<n> failed=<n> spent=<usd>
+#
+# With UBICLOUD_API_KEY set it runs the files on one Ubicloud VM with the
+# environment of bun run test:ubicloud (scripts/ubicloud/test-free.sh:
+# setup-free-suite.sh, xvfb-run, GSTACK_EXPECT_BINARIES=1, the free retry
+# policy), through test-free-shards.ts --files-from. The wall timeout is the
+# whole minutes the cap buys at the size's list price (UBI_SIZE, default
+# standard-16), at most 45; under 20 (the shortest useful run) it refuses
+# before the VM is created. On the VM it recomputes bin/gstack-wtree
+# after unpack (the tree reported) and checks that the planned set's digest
+# equals the list's before any test runs. Setup rebuilds committed build
+# output exactly as the CI free lane does (lib/diagram-render/dist is not
+# byte-reproducible there); the script prints what setup rewrote, and the
+# tests must leave the post-setup tree unchanged. spent= is billed minutes (create to confirmed
+# teardown) times the size's price. Without a key it runs the same files
+# locally in parallel shards and reports spent=0.
+#
+# Exit codes: 0 pass; 1 test failures (or a rejected file list); 2 missing
+# input; 3 tree or plan mismatch (or the tree moved during the run); 4
+# infrastructure failure (create, setup, SSH, wall timeout); 5 the spend cap
+# refuses the run. Nothing is uploaded or spent before the local tree check.
+set -euo pipefail
+# Keep heredoc bodies on temp files, not the pipe window (test/heredoc-pipe-deadlock.test.ts).
+BASH_COMPAT=50
+
+here=$(cd "$(dirname "$0")" && pwd)
+root=$(cd "$here/../.." && pwd)
+rerun='gstack-pregate --stage tests --remote --spend-cap <usd> --allow-repo-commands'
+
+say() { printf 'pregate-remote: %s\n' "$@"; }
+stop() { local code=$1; shift; say "$@"; exit "$code"; }
+
+[ -n "${PREGATE_FILES_FILE:-}" ] || stop 2 "PREGATE_FILES_FILE is not set; run: $rerun"
+[ -n "${PREGATE_TREE:-}" ] || stop 2 "PREGATE_TREE is not set; run: $rerun"
+[ -f "$PREGATE_FILES_FILE" ] || stop 2 "PREGATE_FILES_FILE=$PREGATE_FILES_FILE does not exist; run: $rerun"
+
+wtree() { (cd "$root" && bin/gstack-wtree); }
+tree=$(wtree) || stop 3 "bin/gstack-wtree could not fingerprint $root" "nothing uploaded, nothing spent" "rerun: $rerun"
+if [ "$tree" != "$PREGATE_TREE" ]; then
+  stop 3 "the working tree is ${tree:0:12}, not the planned tree ${PREGATE_TREE:0:12}" "nothing uploaded, nothing spent" "rerun: $rerun"
+fi
+
+count=$(grep -c . "$PREGATE_FILES_FILE" || true)
+logs="$root/.context/pregate-remote/$(date -u +%Y%m%dT%H%M%SZ)-$$"
+mkdir -p "$logs"
+log="$logs/run.log"
+
+result_line() { grep '^\[test:free\] FILES-FROM RESULT: ' "$1" | tail -1 || true; }
+field() { sed -n "s/.* $1=\([0-9][0-9]*\).*/\1/p" <<<"$2"; }
+
+if [ -z "${UBICLOUD_API_KEY:-${UBICLOUD_API_TOKEN:-}}" ]; then
+  say "no UBICLOUD_API_KEY: running $count files locally in parallel shards"
+  set +e
+  (cd "$root" && bun run scripts/test-free-shards.ts --files-from "$PREGATE_FILES_FILE") 2>&1 | tee "$log"
+  rc=${PIPESTATUS[0]}
+  set -e
+  after=$(wtree) || after=unavailable
+  if [ "$after" != "$tree" ]; then
+    (cd "$root" && git diff --name-status "$tree" "$after" 2>&1 | head -20 | sed 's/^/  moved: /') || true
+    stop 3 "the working tree moved during the run (${tree:0:12} before, ${after:0:12} after); no result reported" "log: $log" "rerun on a tree nothing else edits: $rerun"
+  fi
+  result=$(result_line "$log")
+  [ -n "$result" ] || stop "$(( rc == 0 ? 1 : rc ))" "test-free-shards exited $rc without a FILES-FROM RESULT line" "log: $log"
+  say "log: $log"
+  echo "PREGATE_REMOTE: tree=$tree tests=$(field tests "$result") failed=$(field failed "$result") spent=0"
+  exit "$rc"
+fi
+
+size=${UBI_SIZE:-standard-16}
+case $size in
+  standard-2) ppm=0.00104 ;; standard-4) ppm=0.00208 ;; standard-8) ppm=0.00416 ;;
+  standard-16) ppm=0.00831 ;; standard-30) ppm=0.01559 ;; standard-60) ppm=0.03117 ;;
+  *) stop 5 "no price for UBI_SIZE=$size (priced: standard-2, -4, -8, -16, -30, -60; Ubicloud list price per minute)" "nothing created, nothing spent" ;;
+esac
+cap=${PREGATE_SPEND_CAP_USD:-}
+[[ "$cap" =~ ^[0-9]+(\.[0-9]+)?$ ]] || stop 2 "PREGATE_SPEND_CAP_USD is not a dollar amount ('$cap'); run: $rerun"
+max_min=45 min_min=20
+wall_min=$(awk -v c="$cap" -v p="$ppm" -v m="$max_min" 'BEGIN { w = int(c / p); print (w < m ? w : m) }')
+usd() { awk -v m="$1" -v p="$ppm" 'BEGIN { s = m * p; c = int(s * 100); if (c < s * 100) c++; printf "%.2f", c / 100 }'; }
+if [ "$wall_min" -lt "$min_min" ]; then
+  stop 5 "worst case \$$(usd "$min_min") ($min_min min x \$$ppm/min on $size) exceeds the spend cap \$$cap" "nothing created, nothing spent" "rerun with --spend-cap $(usd "$min_min") or more"
+fi
+say "$count files on one $size VM; wall timeout $wall_min min, worst case \$$(usd "$wall_min") under the cap \$$cap"
+
+digest=$(tr -d '\r' <"$PREGATE_FILES_FILE" | grep . | LC_ALL=C sort | sha256sum | cut -d' ' -f1)
+prelude="$logs/vm-setup.sh"
+{
+  printf '%s\n' 'set -euo pipefail' 'BASH_COMPAT=50' \
+    'command -v git >/dev/null || { sudo apt-get update -qq && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git >/dev/null; }' \
+    'vm_tree=$(bin/gstack-wtree) || vm_tree=unavailable' \
+    "if [ \"\$vm_tree\" != $tree ]; then echo \"PREGATE_VM_TREE_MISMATCH vm=\$vm_tree\"; exit 3; fi" \
+    'echo "PREGATE_VM_TREE $vm_tree"' \
+    "cat > ~/.pregate-files.txt <<'PREGATE_FILES'"
+  tr -d '\r' <"$PREGATE_FILES_FILE" | grep .
+  printf '%s\n' 'PREGATE_FILES' "cat > ~/.pregate-run.sh <<'PREGATE_RUN'" \
+    'planned=$(bun run scripts/test-free-shards.ts --files-from ~/.pregate-files.txt --dry-run | sed -n "s/^\[test:free\] files-from .*: [0-9]* files, sha256=\([0-9a-f]*\)$/\1/p")' \
+    "if [ \"\$planned\" != $digest ]; then echo \"PREGATE_VM_PLAN_MISMATCH planned=\${planned:-none} expected=$digest\"; exit 3; fi" \
+    'xvfb-run -a bun run test:free --files-from ~/.pregate-files.txt' \
+    'rc=$?' \
+    'after=$(bin/gstack-wtree || echo unavailable)' \
+    'built=$(cat ~/.pregate-built-tree)' \
+    'echo "PREGATE_VM_TREE_AFTER $after"' \
+    '[ "$after" = "$built" ] || git diff --name-status "$built" "$after" 2>&1 | head -20 | sed "s/^/PREGATE_VM_MOVED /"' \
+    'exit $rc' \
+    'PREGATE_RUN' \
+    'bash scripts/ubicloud/setup-free-suite.sh' \
+    'built=$(bin/gstack-wtree || echo unavailable)' \
+    'echo "$built" > ~/.pregate-built-tree' \
+    'echo "PREGATE_VM_TREE_BUILT $built"' \
+    '[ "$built" = "$vm_tree" ] || git diff --name-status "$vm_tree" "$built" 2>&1 | head -20 | sed "s/^/PREGATE_VM_REBUILT /"'
+} >"$prelude"
+
+args=(run -s "$size" --src "$root" --setup "$prelude"
+  --env GSTACK_EXPECT_BINARIES=1 --env GSTACK_FREE_RETRY_FLAKY=1
+  --env GSTACK_FLAKE_LEDGER=/tmp/gstack-free-test-flake-ledger.jsonl
+  --pull "/tmp/gstack-free-test-*:$logs"
+  --pull "work/$(basename "$root")/.context/free-test-logs:$logs")
+[ -z "${GSTACK_FREE_JOBS:-}" ] || args+=(--env "GSTACK_FREE_JOBS=$GSTACK_FREE_JOBS")
+
+# The VM run gets its own session (python3, which ubi-runner already needs, for
+# setsid on Linux and macOS): pregate's --timeout and stall kill signal this
+# script's process group and then SIGKILL it, which must never cut ubi-runner's
+# teardown short. Output goes to files (a dead reader would SIGPIPE ubi-runner),
+# stamped for the phase timings; this script streams it. A watchdog sends
+# ubi-runner TERM at the cap-derived wall timeout, and so does a TERM or INT
+# to this script; ubi-runner then destroys the VM.
+vm_session='
+exec </dev/null > >(exec >/dev/null 2>&1; trap "" INT TERM HUP; while IFS= read -r line; do printf "%s %s\n" "$(date +%s)" "$line" >>"$2"; printf "%s\n" "$line" >>"$3"; done) 2>&1
+bash "$4" "${@:5}" -- "bash ~/.pregate-run.sh" &
+runner=$!
+echo "$runner" >"$2.pid"
+( sleep "$1"; touch "$2.timedout"; kill -TERM "$runner" ) </dev/null >/dev/null 2>&1 &
+watchdog=$!
+wait "$runner"
+rc=$?
+kill "$watchdog" 2>/dev/null
+[ ! -f "$2.timedout" ] || rc=124
+exit "$rc"
+'
+stream="$logs/stream.log"
+at() { grep -m1 -E "^[0-9]+ $1" "$log" | cut -d' ' -f1 || true; }
+started=$(date +%s)
+( while sleep 60 </dev/null >/dev/null 2>&1; do say "still running ($(( $(date +%s) - started )) s)"; done ) &
+heartbeat=$!
+vmrun="" tailer=""
+trap 'kill "$heartbeat" $tailer 2>/dev/null || true' EXIT
+trap '[ -z "$vmrun" ] || kill -TERM "$(cat "$log.pid" 2>/dev/null)" 2>/dev/null; stop 130 "interrupted: ubi-runner is tearing the VM down on its own (log: $log)"' INT TERM
+
+billed=0 attempt=0
+while :; do
+  attempt=$(( attempt + 1 ))
+  : >"$log"
+  : >"$stream"
+  rm -f "$log.pid" "$log.timedout"
+  t0=$(date +%s)
+  python3 -c 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \
+    bash -c "$vm_session" pregate-remote-vm "$(( (wall_min - 3) * 60 ))" "$log" "$stream" "$here/ubi-runner.sh" "${args[@]}" &
+  vmrun=$!
+  tail -n +1 -f "$stream" &
+  tailer=$!
+  set +e
+  wait "$vmrun"
+  rc=$?
+  set -e
+  vmrun=""
+  sleep 1
+  kill "$tailer" 2>/dev/null || true
+  tailer=""
+  t_end=$(date +%s)
+  t_create=$(at 'ubi-runner: creating ')
+  if [ -n "$t_create" ] && ! grep -q 'ubi-runner: create failed' "$log"; then minutes=$(( (t_end - t_create + 59) / 60 )); billed=$(( billed + (minutes > 0 ? minutes : 1) )); fi
+  if [ "$rc" -ne 0 ] && [ "$attempt" -eq 1 ] && grep -q 'vCPU quota exhausted' "$log"; then
+    say "vCPU quota refused the VM; one retry in ${PREGATE_REMOTE_QUOTA_WAIT_S:-60} s"
+    cp "$log" "$logs/attempt-1.log"
+    sleep "${PREGATE_REMOTE_QUOTA_WAIT_S:-60}"
+    continue
+  fi
+  break
+done
+spent=$(usd "$billed")
+
+t_ready=$(at 'ubi-runner: ready: ')
+t_setup=$(at 'ubi-runner: running setup')
+t_run=$(at 'ubi-runner: running on ')
+t_exit=$(at 'ubi-runner: command exited')
+span() { if [ -n "$1" ] && [ -n "$2" ]; then echo "$(( $2 - $1 ))s"; else echo "-"; fi; }
+say "phases: create $(span "$t0" "$t_ready"), setup $(span "$t_setup" "$t_run"), run $(span "$t_run" "$t_exit"), teardown $(span "$t_exit" "$t_end"); total $(( t_end - started ))s (target 600s)"
+say "log: $log"
+
+vm_tree=$(sed -n 's/^[0-9]* PREGATE_VM_TREE \([0-9a-f]*\)$/\1/p' "$log" | tail -1)
+mismatch=$(grep -m1 -oE 'PREGATE_VM_(TREE|PLAN)_MISMATCH .*' "$log" || true)
+if [ -n "$mismatch" ]; then
+  stop 3 "the VM refused before running tests: $mismatch (expected tree ${tree:0:12})" "no tests ran; spent=$spent ($billed billed min on $size)" "rerun: $rerun"
+fi
+result=$(result_line <(cut -d' ' -f2- "$log"))
+if [ -n "$result" ] && [ "$rc" -ne 124 ]; then
+  after=$(sed -n 's/^[0-9]* PREGATE_VM_TREE_AFTER \(.*\)$/\1/p' "$log" | tail -1)
+  built=$(sed -n 's/^[0-9]* PREGATE_VM_TREE_BUILT \(.*\)$/\1/p' "$log" | tail -1)
+  sed -n 's/^[0-9]* PREGATE_VM_REBUILT /  setup rebuilt (committed build output, as the CI free lane rebuilds it): /p' "$log"
+  if [ "$after" != "$built" ]; then
+    sed -n 's/^[0-9]* PREGATE_VM_MOVED /  moved: /p' "$log"
+    stop 3 "the VM's tree moved during the run (${built:0:12} after setup, ${after:0:12} after tests); no result reported" "spent=$spent ($billed billed min on $size)" "rerun: $rerun"
+  fi
+  echo "PREGATE_REMOTE: tree=$vm_tree tests=$(field tests "$result") failed=$(field failed "$result") spent=$spent"
+  exit $(( rc == 0 ? 0 : 1 ))
+fi
+vm=$(sed -n 's/^[0-9]* ubi-runner: creating \([^ ]*\) .*/\1/p' "$log" | tail -1)
+down=$(grep -E 'ubi-runner: (destroyed|gone:|WARNING|FAILED to destroy)' "$log" | tail -1 | cut -d' ' -f2- || true)
+cause=$(grep -E 'ubi-runner: ' "$log" | grep -vE 'ubi-runner: (destroyed|gone:|destroy requested|waiting for an in-flight)' | tail -1 | cut -d' ' -f2- || true)
+[ "$rc" -ne 124 ] || cause="wall timeout of $(( wall_min - 3 )) min reached (cap \$$cap); the VM was torn down"
+stop 4 "infrastructure failure (ubi-runner exit $rc): ${cause:-no ubi-runner message}" \
+  "VM: ${vm:-none created}${down:+; $down}" "log: $log" "spent=$spent ($billed billed min on $size)"
diff --git a/test/ubicloud-pregate-remote.test.ts b/test/ubicloud-pregate-remote.test.ts
new file mode 100644
index 000000000..0136b5e1f
--- /dev/null
+++ b/test/ubicloud-pregate-remote.test.ts
@@ -0,0 +1,349 @@
+/**
+ * W2-A3 (fix wave 3): scripts/ubicloud/pregate-remote.sh, gstack's declared
+ * remote_gate. Each case runs the real script in a fixture checkout whose
+ * scripts/ubicloud/ubi-runner.sh is a stub that "creates" a local VM
+ * directory, so the keyed path runs its real VM-side setup prelude, the real
+ * test-free-shards.ts --files-from and the real bin/gstack-wtree, offline and
+ * at no cost. Reports are judged through the exported parseRemoteReport /
+ * remoteReportProblem, exactly as gstack-pregate --remote judges them.
+ */
+import { afterAll, describe, expect, test } from 'bun:test';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { parseConfig } from '../lib/pregate/config';
+import { parseRemoteReport, remoteReportProblem } from '../lib/pregate/checks/lanes';
+
+const ROOT = path.resolve(import.meta.dir, '..');
+const SCRATCH = fs.mkdtempSync(path.join(os.tmpdir(), 'pregate-remote-gate-'));
+afterAll(() => fs.rmSync(SCRATCH, { recursive: true, force: true }));
+
+const COPIED = ['scripts/ubicloud/pregate-remote.sh', 'bin/gstack-wtree', 'bin/gstack-path-spelling.sh',
+  'scripts/test-free-shards.ts', 'scripts/test-strict-output.ts', 'scripts/lib/shard-engine.ts', 'scripts/lib/free-home-guard.ts',
+  'scripts/lib/free-ci-health.ts', 'scripts/lib/windows-curation.ts', 'lib/state-root.ts', 'test/helpers/paid-test-set.ts',
+  'test/helpers/touchfiles.ts', 'test/helpers/touchfiles-data.ts', 'test/helpers/test-selection.ts'];
+const TESTS: Record<string, string> = {
+  'test/alpha.test.ts': "import { expect, test } from 'bun:test';\ntest('alpha', () => { expect(1).toBe(1); });\n",
+  'test/beta.test.ts': "import { expect, test } from 'bun:test';\ntest('beta', () => { expect(2).toBe(2); });\n",
+  'test/fails.test.ts': "import { expect, test } from 'bun:test';\ntest('fails', () => { expect(1).toBe(2); });\n",
+  'test/empty.test.ts': "import 'bun:test';\n",
+  'test/mover.test.ts': "import { test } from 'bun:test';\nimport * as fs from 'node:fs';\nimport * as path from 'node:path';\ntest('moves the tree', () => { fs.writeFileSync(path.join(import.meta.dir, '..', 'moved.txt'), 'x'); });\n",
+};
+
+/** Stub of scripts/ubicloud/ubi-runner.sh (written into each fixture as that file). */
+const UBI_RUNNER_STUB = `#!/usr/bin/env bash
+# Stub of scripts/ubicloud/ubi-runner.sh for test/ubicloud-pregate-remote.test.ts:
+# "run" copies the checkout (tracked + untracked-unignored + .git) into a local
+# VM directory, runs setup and the command there, and prints ubi-runner's log
+# lines. STUB_FAIL picks a failure: create, quota, quota-once, ssh. STUB_HANG
+# sleeps before the command; TERM tears down as ubi-runner's trap does.
+# Every teardown is appended to $STUB_CALLS.down.
+set -uo pipefail
+[ "$1" = run ] || exit 2
+shift
+printf '%s\\n' "$*" >>"$STUB_CALLS"
+src=. setup="" size="" envs=()
+while [ $# -gt 0 ]; do
+  case $1 in
+    -s) size=$2; shift 2 ;;
+    --src) src=$2; shift 2 ;;
+    --setup) setup=$2; shift 2 ;;
+    --env) envs+=("$2"); shift 2 ;;
+    --pull) shift 2 ;;
+    --) shift; break ;;
+    *) echo "stub: unknown option $1" >&2; exit 2 ;;
+  esac
+done
+cmd=$*
+log() { echo "ubi-runner: $*" >&2; }
+name=ubirun-stub-1700000000-abcd
+log "creating eu-central-h1/$name ($size)"
+calls=$(wc -l <"$STUB_CALLS")
+fail=\${STUB_FAIL:-}
+if [ "$fail" = quota ] || { [ "$fail" = quota-once ] && [ "$calls" -eq 1 ]; }; then
+  log "quota refused eu-central-h1/$name ($size): it needs 16 vCPUs and the project already uses 64 of 64"
+  log "create failed: vCPU quota exhausted; wait for running VMs to finish, or use fewer or smaller VMs (ci:ubicloud --vms N)"
+  log "gone: eu-central-h1/$name (never existed or already destroyed)"
+  exit 1
+fi
+if [ "$fail" = create ]; then
+  echo "Validation failed for following fields: size" >&2
+  log "create failed"
+  log "gone: eu-central-h1/$name (never existed or already destroyed)"
+  exit 1
+fi
+log "ready: $name (192.0.2.1)"
+trap 'log "destroyed eu-central-h1/$name"; echo "term $name" >>"$STUB_CALLS.down"; exit 130' TERM
+vm=$STUB_VM_HOME
+dest="$vm/work/$(basename "$src")"
+rm -rf "$dest" && mkdir -p "$dest"
+(cd "$src" && { git ls-files -z -co --exclude-standard; printf '.git\\0'; } | tar --null -T - -cf -) | tar -xf - -C "$dest"
+[ -z "\${STUB_VM_EDIT:-}" ] || echo "// edited on the VM" >>"$dest/$STUB_VM_EDIT"
+for e in \${envs[@]+"\${envs[@]}"}; do printf 'export %s\\n' "$e"; done >"$vm/.ubirun-env"
+log "synced $src -> $name:$dest"
+if [ -n "$setup" ]; then
+  cp "$setup" "$vm/.ubirun-setup.sh"
+  log "running setup $(basename "$setup")"
+  if ! (cd "$dest" && HOME="$vm" bash -c '. ~/.ubirun-env && bash ~/.ubirun-setup.sh'); then
+    log "setup failed"
+    log "destroyed eu-central-h1/$name"
+    exit 1
+  fi
+fi
+[ -z "\${STUB_VM_LIST_ADD:-}" ] || echo "$STUB_VM_LIST_ADD" >>"$vm/.pregate-files.txt"
+log "running on $name: $cmd"
+if [ -n "\${STUB_HANG:-}" ]; then sleep "$STUB_HANG" & wait $!; fi
+rc=0
+if [ "$fail" = ssh ]; then
+  rc=255
+else
+  (cd "$dest" && HOME="$vm" PATH="$STUB_BIN:$PATH" bash -c ". ~/.ubirun-env && $cmd") || rc=$?
+fi
+log "command exited $rc after 1s"
+log "destroyed eu-central-h1/$name"
+echo "exit $name" >>"$STUB_CALLS.down"
+exit "$rc"
+`;
+
+interface Fixture { repo: string; vm: string; calls: string; bin: string; tree: string }
+
+let counter = 0;
+function fixture(): Fixture {
+  const base = path.join(SCRATCH, String(++counter));
+  const repo = path.join(base, 'gstack');
+  const vm = path.join(base, 'vm');
+  const bin = path.join(base, 'bin');
+  for (const dir of [repo, vm, bin]) fs.mkdirSync(dir, { recursive: true });
+  const put = (rel: string, body: string, mode = 0o644) => {
+    fs.mkdirSync(path.dirname(path.join(repo, rel)), { recursive: true });
+    fs.writeFileSync(path.join(repo, rel), body, { mode });
+  };
+  for (const rel of COPIED) put(rel, fs.readFileSync(path.join(ROOT, rel), 'utf8'), fs.statSync(path.join(ROOT, rel)).mode & 0o777);
+  for (const [rel, body] of Object.entries(TESTS)) put(rel, body);
+  put('scripts/ubicloud/ubi-runner.sh', UBI_RUNNER_STUB, 0o755);
+  put('scripts/ubicloud/setup-free-suite.sh', '#!/usr/bin/env bash\necho "setup-free-suite (stub)"\n[ -z "${STUB_SETUP_FAIL:-}" ] || exit 1\n[ -z "${STUB_SETUP_REBUILD:-}" ] || echo rebuilt >>dist/bundle.txt\n', 0o755);
+  put('dist/bundle.txt', 'committed build output\n');
+  put('.gitignore', '.context/\n');
+  put('package.json', JSON.stringify({ scripts: { 'test:free': 'bun run scripts/test-free-shards.ts' } }) + '\n');
+  fs.writeFileSync(path.join(bin, 'xvfb-run'), '#!/bin/sh\n[ "$1" = -a ] && shift\nexec "$@"\n', { mode: 0o755 });
+  const git = (...args: string[]) => {
+    const r = Bun.spawnSync(['git', '-c', 'user.email=t@t', '-c', 'user.name=t', ...args], { cwd: repo, timeout: 30_000 });
+    if (r.exitCode !== 0) throw new Error(`git ${args.join(' ')}: ${r.stderr.toString()}`);
+    return r.stdout.toString().trim();
+  };
+  git('init', '-q', '-b', 'main');
+  git('add', '-A');
+  git('commit', '-qm', 'fixture');
+  const tree = Bun.spawnSync(['bin/gstack-wtree'], { cwd: repo, timeout: 30_000 }).stdout.toString().trim();
+  return { repo, vm, calls: path.join(base, 'calls.log'), bin, tree };
+}
+
+function gate(f: Fixture, files: string[] | null, env: Record<string, string | undefined> = {}) {
+  const list = path.join(path.dirname(f.repo), 'files.txt');
+  if (files) fs.writeFileSync(list, files.join('\n') + '\n');
+  const base: Record<string, string | undefined> = { ...process.env, UBICLOUD_API_KEY: undefined, UBICLOUD_API_TOKEN: undefined, UBI_SIZE: undefined,
+    GSTACK_FREE_JOBS: '2', GSTACK_FLAKE_LEDGER: path.join(path.dirname(f.repo), 'flakes.jsonl'), STUB_CALLS: f.calls, STUB_VM_HOME: f.vm, STUB_BIN: f.bin };
+  if (files) Object.assign(base, { PREGATE_FILES_FILE: list, PREGATE_TREE: f.tree, PREGATE_SPEND_CAP_USD: '2' });
+  const merged = Object.fromEntries(Object.entries({ ...base, ...env }).filter(([, v]) => v !== undefined)) as Record<string, string>;
+  const r = Bun.spawnSync(['bash', 'scripts/ubicloud/pregate-remote.sh'], { cwd: f.repo, env: merged, timeout: 90_000 });
+  const out = r.stdout.toString() + r.stderr.toString();
+  return { exit: r.exitCode, out, report: parseRemoteReport(out), calls: fs.existsSync(f.calls) ? fs.readFileSync(f.calls, 'utf8').trim().split('\n') : [] };
+}
+const KEY = { UBICLOUD_API_KEY: 'offline-stub' };
+
+describe('pregate-remote.sh inputs and the local tree check', () => {
+  test('run bare it exits 2 before any work, naming the pregate command', () => {
+    const f = fixture();
+    const r = gate(f, null, KEY);
+    expect(r.exit).toBe(2);
+    expect(r.out).toContain('PREGATE_FILES_FILE is not set; run: gstack-pregate --stage tests --remote --spend-cap <usd> --allow-repo-commands');
+    expect(r.calls).toEqual([]);
+    expect(fs.existsSync(path.join(f.repo, '.context'))).toBe(false);
+  }, 60_000);
+
+  test('a working tree that is not PREGATE_TREE exits 3 before any upload, with both prefixes and the rerun command', () => {
+    const f = fixture();
+    const r = gate(f, ['test/alpha.test.ts'], { ...KEY, PREGATE_TREE: 'f'.repeat(40) });
+    expect(r.exit).toBe(3);
+    expect(r.out).toContain(`the working tree is ${f.tree.slice(0, 12)}, not the planned tree ffffffffffff`);
+    expect(r.out).toContain('nothing uploaded, nothing spent');
+    expect(r.out).toContain('rerun: gstack-pregate --stage tests --remote --spend-cap <usd> --allow-repo-commands');
+    expect(r.calls).toEqual([]);
+    expect(r.report).toBeNull();
+  }, 60_000);
+});
+
+describe('pregate-remote.sh keyed path (stub ubi-runner)', () => {
+  test('a passing run prints the VM tree, the counts and billed spend; pregate clears it', () => {
+    const f = fixture();
+    const r = gate(f, ['test/alpha.test.ts', 'test/beta.test.ts'], KEY);
+    expect(r.exit, r.out).toBe(0);
+    expect(r.report).toEqual({ tree: f.tree, tests: 2, failed: 0, spent: '0.01' });
+    expect(remoteReportProblem(r.report!, f.tree)).toBeNull();
+    expect(r.out).toContain('2 files on one standard-16 VM; wall timeout 45 min, worst case $0.38 under the cap $2');
+    expect(r.out).toMatch(/phases: create \d+s, setup \d+s, run \d+s, teardown \d+s; total \d+s \(target 600s\)/);
+    expect(r.out.trim().split('\n').at(-1)).toMatch(/^PREGATE_REMOTE: /);
+    expect(r.calls).toHaveLength(1);
+    for (const arg of ['-s standard-16 ', '--env GSTACK_EXPECT_BINARIES=1', '--env GSTACK_FREE_RETRY_FLAKY=1', '--env GSTACK_FREE_JOBS=2', '-- bash ~/.pregate-run.sh']) expect(r.calls[0]).toContain(arg);
+    expect(fs.readFileSync(path.join(f.vm, '.pregate-files.txt'), 'utf8')).toBe('test/alpha.test.ts\ntest/beta.test.ts\n');
+  }, 90_000);
+
+  test('setup rebuilding committed build output (as CI does) is printed; the reported tree stays the uploaded one', () => {
+    const f = fixture();
+    const r = gate(f, ['test/alpha.test.ts'], { ...KEY, STUB_SETUP_REBUILD: '1' });
+    expect(r.exit, r.out).toBe(0);
+    expect(r.out).toContain('  setup rebuilt (committed build output, as the CI free lane rebuilds it): M\tdist/bundle.txt');
+    expect(r.report).toEqual({ tree: f.tree, tests: 1, failed: 0, spent: '0.01' });
+  }, 90_000);
+
+  test('a failing test reports failed>0 and exits 1; a file with zero tests reports tests=0; neither clears', () => {
+    const failing = gate(fixture(), ['test/alpha.test.ts', 'test/fails.test.ts'], KEY);
+    expect(failing.exit, failing.out).toBe(1);
+    expect(failing.report).toMatchObject({ tests: 2, failed: 1 });
+    expect(remoteReportProblem(failing.report!, failing.report!.tree)).toBe('the remote gate reported 1 failed of 2');
+    const empty = gate(fixture(), ['test/empty.test.ts'], KEY);
+    expect(empty.report, empty.out).toMatchObject({ tests: 0, failed: 0 });
+    expect(remoteReportProblem(empty.report!, empty.report!.tree)).toBe('the remote gate reported 0 tests');
+  }, 90_000);
+
+  test('the cap refuses before any VM is created, at the price of the size actually used', () => {
+    const f = fixture();
+    const low = gate(f, ['test/alpha.test.ts'], { ...KEY, PREGATE_SPEND_CAP_USD: '0.1' });
+    expect(low.exit).toBe(5);
+    expect(low.out).toContain('worst case $0.17 (20 min x $0.00831/min on standard-16) exceeds the spend cap $0.1');
+    expect(low.out).toContain('nothing created, nothing spent');
+    expect(low.out).toContain('rerun with --spend-cap 0.17 or more');
+    const big = gate(f, ['test/alpha.test.ts'], { ...KEY, PREGATE_SPEND_CAP_USD: '0.5', UBI_SIZE: 'standard-60' });
+    expect(big.exit).toBe(5);
+    expect(big.out).toContain('worst case $0.63 (20 min x $0.03117/min on standard-60) exceeds the spend cap $0.5');
+    const unpriced = gate(f, ['test/alpha.test.ts'], { ...KEY, UBI_SIZE: 'burstable-1' });
+    expect(unpriced.exit).toBe(5);
+    expect(unpriced.out).toContain('no price for UBI_SIZE=burstable-1');
+    const noCap = gate(f, ['test/alpha.test.ts'], { ...KEY, PREGATE_SPEND_CAP_USD: '' });
+    expect(noCap.exit).toBe(2);
+    expect([low, big, unpriced, noCap].every(r => r.report === null)).toBe(true);
+    expect(fs.existsSync(f.calls)).toBe(false);
+  }, 90_000);
+
+  test('create, setup and SSH failures exit 4 with the ubi-runner message, the VM, its teardown line, the log and spent=', () => {
+    const create = gate(fixture(), ['test/alpha.test.ts'], { ...KEY, STUB_FAIL: 'create' });
+    expect(create.exit, create.out).toBe(4);
+    expect(create.out).toContain('infrastructure failure (ubi-runner exit 1): ubi-runner: create failed');
+    expect(create.out).toContain('VM: eu-central-h1/ubirun-stub-1700000000-abcd; ubi-runner: gone: eu-central-h1/ubirun-stub-1700000000-abcd (never existed or already destroyed)');
+    expect(create.out).toMatch(/log: \S+\/\.context\/pregate-remote\/\S+\/run\.log/);
+    expect(create.out).toContain('spent=0.00 (0 billed min on standard-16)');
+    const setup = gate(fixture(), ['test/alpha.test.ts'], { ...KEY, STUB_SETUP_FAIL: '1' });
+    expect(setup.exit, setup.out).toBe(4);
+    expect(setup.out).toContain('ubi-runner: setup failed');
+    expect(setup.out).toContain('; ubi-runner: destroyed eu-central-h1/ubirun-stub-1700000000-abcd');
+    expect(setup.out).toContain('spent=0.01 (1 billed min on standard-16)');
+    const ssh = gate(fixture(), ['test/alpha.test.ts'], { ...KEY, STUB_FAIL: 'ssh' });
+    expect(ssh.exit, ssh.out).toBe(4);
+    expect(ssh.out).toContain('infrastructure failure (ubi-runner exit 255): ubi-runner: command exited 255 after 1s');
+    expect([create, setup, ssh].every(r => r.report === null)).toBe(true);
+  }, 90_000);
+
+  test('a quota refusal is retried once, then the run stops', () => {
+    const once = gate(fixture(), ['test/alpha.test.ts'], { ...KEY, STUB_FAIL: 'quota-once', PREGATE_REMOTE_QUOTA_WAIT_S: '0' });
+    expect(once.exit, once.out).toBe(0);
+    expect(once.out).toContain('vCPU quota refused the VM; one retry in 0 s');
+    expect(once.calls).toHaveLength(2);
+    const twice = gate(fixture(), ['test/alpha.test.ts'], { ...KEY, STUB_FAIL: 'quota', PREGATE_REMOTE_QUOTA_WAIT_S: '0' });
+    expect(twice.exit, twice.out).toBe(4);
+    expect(twice.calls).toHaveLength(2);
+    expect(twice.out).toContain('create failed: vCPU quota exhausted');
+  }, 90_000);
+
+  test('the VM refuses a tree that differs after unpack, a planned set that differs from the list, and a tree that moved during the run', () => {
+    const edited = gate(fixture(), ['test/alpha.test.ts'], { ...KEY, STUB_VM_EDIT: 'test/alpha.test.ts' });
+    expect(edited.exit, edited.out).toBe(3);
+    expect(edited.out).toContain('the VM refused before running tests: PREGATE_VM_TREE_MISMATCH vm=');
+    expect(edited.out).toContain('no tests ran; spent=0.01');
+    const listed = gate(fixture(), ['test/alpha.test.ts'], { ...KEY, STUB_VM_LIST_ADD: 'test/beta.test.ts' });
+    expect(listed.exit, listed.out).toBe(3);
+    expect(listed.out).toContain('PREGATE_VM_PLAN_MISMATCH planned=');
+    const moved = gate(fixture(), ['test/alpha.test.ts', 'test/mover.test.ts'], KEY);
+    expect(moved.exit, moved.out).toBe(3);
+    expect(moved.out).toContain("the VM's tree moved during the run");
+    expect(moved.out).toContain('  moved: A\tmoved.txt');
+    expect([edited, listed, moved].every(r => r.report === null)).toBe(true);
+  }, 90_000);
+
+  test('killing the gate\'s process group (pregate\'s timeout) never cuts teardown short; TERM to the gate tears the VM down', async () => {
+    const run = (f: Fixture) => {
+      const list = path.join(path.dirname(f.repo), 'files.txt');
+      fs.writeFileSync(list, 'test/alpha.test.ts\n');
+      return Bun.spawn(['bash', 'scripts/ubicloud/pregate-remote.sh'], { cwd: f.repo, detached: true, stdout: 'pipe', stderr: 'pipe',
+        env: { ...process.env, ...KEY, UBI_SIZE: '', GSTACK_FREE_JOBS: '2', STUB_CALLS: f.calls, STUB_VM_HOME: f.vm, STUB_BIN: f.bin, STUB_HANG: '4',
+          PREGATE_FILES_FILE: list, PREGATE_TREE: f.tree, PREGATE_SPEND_CAP_USD: '2' } as Record<string, string> });
+    };
+    const until = async (cond: () => boolean, ms: number) => {
+      const deadline = Date.now() + ms;
+      while (!cond()) { if (Date.now() > deadline) throw new Error('condition not met in time'); await Bun.sleep(50); }
+    };
+    const vmLog = (f: Fixture) => { try { return fs.readFileSync(fs.readdirSync(path.join(f.repo, '.context/pregate-remote')).map(d => path.join(f.repo, '.context/pregate-remote', d, 'run.log'))[0]!, 'utf8'); } catch { return ''; } };
+    const killed = fixture();
+    const a = run(killed);
+    await until(() => vmLog(killed).includes('ubi-runner: running on'), 30_000);
+    process.kill(-a.pid, 'SIGKILL');
+    await a.exited;
+    await until(() => fs.existsSync(killed.calls + '.down'), 30_000);
+    expect(fs.readFileSync(killed.calls + '.down', 'utf8')).toBe('exit ubirun-stub-1700000000-abcd\n');
+    const termed = fixture();
+    const b = run(termed);
+    await until(() => vmLog(termed).includes('ubi-runner: running on'), 30_000);
+    process.kill(b.pid, 'SIGTERM');
+    expect(await b.exited).toBe(130);
+    expect(await new Response(b.stdout).text()).toContain('interrupted: ubi-runner is tearing the VM down on its own');
+    await until(() => fs.existsSync(termed.calls + '.down'), 10_000);
+    expect(fs.readFileSync(termed.calls + '.down', 'utf8')).toBe('term ubirun-stub-1700000000-abcd\n');
+  }, 90_000);
+
+  test('the VM environment is bun run test:ubicloud\'s (scripts/ubicloud/test-free.sh), so the two cannot drift', () => {
+    const script = fs.readFileSync(path.join(ROOT, 'scripts/ubicloud/pregate-remote.sh'), 'utf8');
+    const wrapper = fs.readFileSync(path.join(ROOT, 'scripts/ubicloud/test-free.sh'), 'utf8');
+    const flags = [...wrapper.matchAll(/--(?:env|pull) "?[^\s"]+(?: [^\s"]+)?/g)].map(m => m[0]).filter(flag => !flag.includes('GSTACK_FREE_JOBS'));
+    expect(flags.length).toBeGreaterThanOrEqual(5);
+    for (const flag of flags) expect(script).toContain(flag);
+    expect(wrapper).toContain('--setup "$here/setup-free-suite.sh"');
+    expect(script).toContain("'bash scripts/ubicloud/setup-free-suite.sh'");
+    expect(wrapper).toContain('xvfb-run -a bun run test:free');
+    expect(script).toContain("'xvfb-run -a bun run test:free --files-from ~/.pregate-files.txt'");
+  });
+});
+
+describe('pregate-remote.sh keyless path', () => {
+  test('without a key it runs the list locally in parallel shards and reports the tree it tested with spent=0', () => {
+    const f = fixture();
+    const r = gate(f, ['test/alpha.test.ts', 'test/beta.test.ts']);
+    expect(r.exit, r.out).toBe(0);
+    expect(r.out).toContain('no UBICLOUD_API_KEY: running 2 files locally in parallel shards');
+    expect(r.report).toEqual({ tree: f.tree, tests: 2, failed: 0, spent: '0' });
+    expect(remoteReportProblem(r.report!, f.tree)).toBeNull();
+    expect(fs.existsSync(f.calls)).toBe(false);
+  }, 90_000);
+
+  test('a tree that moves during the local run fails without a report', () => {
+    const r = gate(fixture(), ['test/alpha.test.ts', 'test/mover.test.ts']);
+    expect(r.exit, r.out).toBe(3);
+    expect(r.out).toContain('the working tree moved during the run');
+    expect(r.out).toContain('  moved: A\tmoved.txt');
+    expect(r.report).toBeNull();
+  }, 90_000);
+
+  test('a list test-free-shards rejects exits 1 without a report', () => {
+    const r = gate(fixture(), ['test/alpha.test.ts', 'test/missing.test.ts']);
+    expect(r.exit, r.out).toBe(1);
+    expect(r.out).toContain('1 path(s) not in the free planned set: test/missing.test.ts');
+    expect(r.out).toContain('without a FILES-FROM RESULT line');
+    expect(r.report).toBeNull();
+  }, 90_000);
+});
+
+describe('.gstack/pregate.json declares the remote gate', () => {
+  test('remote_gate is this script, and the config still validates', () => {
+    const { config, errors } = parseConfig(fs.readFileSync(path.join(ROOT, '.gstack/pregate.json'), 'utf8'));
+    expect(errors).toEqual([]);
+    expect(config.remote_gate).toBe('bash scripts/ubicloud/pregate-remote.sh');
+  });
+});