mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-11 11:10:13 +01:00
feat(pregate): declare scripts/ubicloud/pregate-remote.sh as gstack's remote gate (keyed Ubicloud VM, keyless local shards)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed
1 parent
79058a8fff
commit
5567c493b6
3 files changed
+574
No files matched your search
@@ -1,6 +1,7 @@
|
||||
{
|
||||
"$schema": "https://github.com/garrytan/gstack/docs/pregate.md#pregate-v1",
|
||||
"runner": "bun test {files}",
|
||||
"remote_gate": "bash scripts/ubicloud/pregate-remote.sh",
|
||||
"dependencies": {
|
||||
".github/workflows/**": ["test/pregate-selection.test.ts", "test/free-tests-workflow-wiring.test.ts"],
|
||||
"scripts/gen-ship-goldens.ts": ["test/host-config.test.ts"],
|
||||
|
||||
Executable
+224
@@ -0,0 +1,224 @@
|
||||
#!/usr/bin/env bash
|
||||
# pregate-remote.sh — gstack's declared remote_gate (.gstack/pregate.json).
|
||||
#
|
||||
# gstack-pregate runs this under 8 cores for --stage tests --remote:
|
||||
#
|
||||
# gstack-pregate --stage tests --remote --spend-cap 2 --allow-repo-commands
|
||||
#
|
||||
# with PREGATE_FILES_FILE (the selected free test files, one per line),
|
||||
# PREGATE_TREE (the working-tree fingerprint being tested) and
|
||||
# PREGATE_SPEND_CAP_USD. It prints the contract line pregate reads
|
||||
# (docs/pregate.md#remote-gate):
|
||||
#
|
||||
# PREGATE_REMOTE: tree=<tree tested> tests=<n> failed=<n> spent=<usd>
|
||||
#
|
||||
# With UBICLOUD_API_KEY set it runs the files on one Ubicloud VM with the
|
||||
# environment of bun run test:ubicloud (scripts/ubicloud/test-free.sh:
|
||||
# setup-free-suite.sh, xvfb-run, GSTACK_EXPECT_BINARIES=1, the free retry
|
||||
# policy), through test-free-shards.ts --files-from. The wall timeout is the
|
||||
# whole minutes the cap buys at the size's list price (UBI_SIZE, default
|
||||
# standard-16), at most 45; under 20 (the shortest useful run) it refuses
|
||||
# before the VM is created. On the VM it recomputes bin/gstack-wtree
|
||||
# after unpack (the tree reported) and checks that the planned set's digest
|
||||
# equals the list's before any test runs. Setup rebuilds committed build
|
||||
# output exactly as the CI free lane does (lib/diagram-render/dist is not
|
||||
# byte-reproducible there); the script prints what setup rewrote, and the
|
||||
# tests must leave the post-setup tree unchanged. spent= is billed minutes (create to confirmed
|
||||
# teardown) times the size's price. Without a key it runs the same files
|
||||
# locally in parallel shards and reports spent=0.
|
||||
#
|
||||
# Exit codes: 0 pass; 1 test failures (or a rejected file list); 2 missing
|
||||
# input; 3 tree or plan mismatch (or the tree moved during the run); 4
|
||||
# infrastructure failure (create, setup, SSH, wall timeout); 5 the spend cap
|
||||
# refuses the run. Nothing is uploaded or spent before the local tree check.
|
||||
set -euo pipefail
|
||||
# Keep heredoc bodies on temp files, not the pipe window (test/heredoc-pipe-deadlock.test.ts).
|
||||
BASH_COMPAT=50
|
||||
|
||||
here=$(cd "$(dirname "$0")" && pwd)
|
||||
root=$(cd "$here/../.." && pwd)
|
||||
rerun='gstack-pregate --stage tests --remote --spend-cap <usd> --allow-repo-commands'
|
||||
|
||||
say() { printf 'pregate-remote: %s\n' "$@"; }
|
||||
stop() { local code=$1; shift; say "$@"; exit "$code"; }
|
||||
|
||||
[ -n "${PREGATE_FILES_FILE:-}" ] || stop 2 "PREGATE_FILES_FILE is not set; run: $rerun"
|
||||
[ -n "${PREGATE_TREE:-}" ] || stop 2 "PREGATE_TREE is not set; run: $rerun"
|
||||
[ -f "$PREGATE_FILES_FILE" ] || stop 2 "PREGATE_FILES_FILE=$PREGATE_FILES_FILE does not exist; run: $rerun"
|
||||
|
||||
wtree() { (cd "$root" && bin/gstack-wtree); }
|
||||
tree=$(wtree) || stop 3 "bin/gstack-wtree could not fingerprint $root" "nothing uploaded, nothing spent" "rerun: $rerun"
|
||||
if [ "$tree" != "$PREGATE_TREE" ]; then
|
||||
stop 3 "the working tree is ${tree:0:12}, not the planned tree ${PREGATE_TREE:0:12}" "nothing uploaded, nothing spent" "rerun: $rerun"
|
||||
fi
|
||||
|
||||
count=$(grep -c . "$PREGATE_FILES_FILE" || true)
|
||||
logs="$root/.context/pregate-remote/$(date -u +%Y%m%dT%H%M%SZ)-$$"
|
||||
mkdir -p "$logs"
|
||||
log="$logs/run.log"
|
||||
|
||||
result_line() { grep '^\[test:free\] FILES-FROM RESULT: ' "$1" | tail -1 || true; }
|
||||
field() { sed -n "s/.* $1=\([0-9][0-9]*\).*/\1/p" <<<"$2"; }
|
||||
|
||||
if [ -z "${UBICLOUD_API_KEY:-${UBICLOUD_API_TOKEN:-}}" ]; then
|
||||
say "no UBICLOUD_API_KEY: running $count files locally in parallel shards"
|
||||
set +e
|
||||
(cd "$root" && bun run scripts/test-free-shards.ts --files-from "$PREGATE_FILES_FILE") 2>&1 | tee "$log"
|
||||
rc=${PIPESTATUS[0]}
|
||||
set -e
|
||||
after=$(wtree) || after=unavailable
|
||||
if [ "$after" != "$tree" ]; then
|
||||
(cd "$root" && git diff --name-status "$tree" "$after" 2>&1 | head -20 | sed 's/^/ moved: /') || true
|
||||
stop 3 "the working tree moved during the run (${tree:0:12} before, ${after:0:12} after); no result reported" "log: $log" "rerun on a tree nothing else edits: $rerun"
|
||||
fi
|
||||
result=$(result_line "$log")
|
||||
[ -n "$result" ] || stop "$(( rc == 0 ? 1 : rc ))" "test-free-shards exited $rc without a FILES-FROM RESULT line" "log: $log"
|
||||
say "log: $log"
|
||||
echo "PREGATE_REMOTE: tree=$tree tests=$(field tests "$result") failed=$(field failed "$result") spent=0"
|
||||
exit "$rc"
|
||||
fi
|
||||
|
||||
size=${UBI_SIZE:-standard-16}
|
||||
case $size in
|
||||
standard-2) ppm=0.00104 ;; standard-4) ppm=0.00208 ;; standard-8) ppm=0.00416 ;;
|
||||
standard-16) ppm=0.00831 ;; standard-30) ppm=0.01559 ;; standard-60) ppm=0.03117 ;;
|
||||
*) stop 5 "no price for UBI_SIZE=$size (priced: standard-2, -4, -8, -16, -30, -60; Ubicloud list price per minute)" "nothing created, nothing spent" ;;
|
||||
esac
|
||||
cap=${PREGATE_SPEND_CAP_USD:-}
|
||||
[[ "$cap" =~ ^[0-9]+(\.[0-9]+)?$ ]] || stop 2 "PREGATE_SPEND_CAP_USD is not a dollar amount ('$cap'); run: $rerun"
|
||||
max_min=45 min_min=20
|
||||
wall_min=$(awk -v c="$cap" -v p="$ppm" -v m="$max_min" 'BEGIN { w = int(c / p); print (w < m ? w : m) }')
|
||||
usd() { awk -v m="$1" -v p="$ppm" 'BEGIN { s = m * p; c = int(s * 100); if (c < s * 100) c++; printf "%.2f", c / 100 }'; }
|
||||
if [ "$wall_min" -lt "$min_min" ]; then
|
||||
stop 5 "worst case \$$(usd "$min_min") ($min_min min x \$$ppm/min on $size) exceeds the spend cap \$$cap" "nothing created, nothing spent" "rerun with --spend-cap $(usd "$min_min") or more"
|
||||
fi
|
||||
say "$count files on one $size VM; wall timeout $wall_min min, worst case \$$(usd "$wall_min") under the cap \$$cap"
|
||||
|
||||
digest=$(tr -d '\r' <"$PREGATE_FILES_FILE" | grep . | LC_ALL=C sort | sha256sum | cut -d' ' -f1)
|
||||
prelude="$logs/vm-setup.sh"
|
||||
{
|
||||
printf '%s\n' 'set -euo pipefail' 'BASH_COMPAT=50' \
|
||||
'command -v git >/dev/null || { sudo apt-get update -qq && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git >/dev/null; }' \
|
||||
'vm_tree=$(bin/gstack-wtree) || vm_tree=unavailable' \
|
||||
"if [ \"\$vm_tree\" != $tree ]; then echo \"PREGATE_VM_TREE_MISMATCH vm=\$vm_tree\"; exit 3; fi" \
|
||||
'echo "PREGATE_VM_TREE $vm_tree"' \
|
||||
"cat > ~/.pregate-files.txt <<'PREGATE_FILES'"
|
||||
tr -d '\r' <"$PREGATE_FILES_FILE" | grep .
|
||||
printf '%s\n' 'PREGATE_FILES' "cat > ~/.pregate-run.sh <<'PREGATE_RUN'" \
|
||||
'planned=$(bun run scripts/test-free-shards.ts --files-from ~/.pregate-files.txt --dry-run | sed -n "s/^\[test:free\] files-from .*: [0-9]* files, sha256=\([0-9a-f]*\)$/\1/p")' \
|
||||
"if [ \"\$planned\" != $digest ]; then echo \"PREGATE_VM_PLAN_MISMATCH planned=\${planned:-none} expected=$digest\"; exit 3; fi" \
|
||||
'xvfb-run -a bun run test:free --files-from ~/.pregate-files.txt' \
|
||||
'rc=$?' \
|
||||
'after=$(bin/gstack-wtree || echo unavailable)' \
|
||||
'built=$(cat ~/.pregate-built-tree)' \
|
||||
'echo "PREGATE_VM_TREE_AFTER $after"' \
|
||||
'[ "$after" = "$built" ] || git diff --name-status "$built" "$after" 2>&1 | head -20 | sed "s/^/PREGATE_VM_MOVED /"' \
|
||||
'exit $rc' \
|
||||
'PREGATE_RUN' \
|
||||
'bash scripts/ubicloud/setup-free-suite.sh' \
|
||||
'built=$(bin/gstack-wtree || echo unavailable)' \
|
||||
'echo "$built" > ~/.pregate-built-tree' \
|
||||
'echo "PREGATE_VM_TREE_BUILT $built"' \
|
||||
'[ "$built" = "$vm_tree" ] || git diff --name-status "$vm_tree" "$built" 2>&1 | head -20 | sed "s/^/PREGATE_VM_REBUILT /"'
|
||||
} >"$prelude"
|
||||
|
||||
args=(run -s "$size" --src "$root" --setup "$prelude"
|
||||
--env GSTACK_EXPECT_BINARIES=1 --env GSTACK_FREE_RETRY_FLAKY=1
|
||||
--env GSTACK_FLAKE_LEDGER=/tmp/gstack-free-test-flake-ledger.jsonl
|
||||
--pull "/tmp/gstack-free-test-*:$logs"
|
||||
--pull "work/$(basename "$root")/.context/free-test-logs:$logs")
|
||||
[ -z "${GSTACK_FREE_JOBS:-}" ] || args+=(--env "GSTACK_FREE_JOBS=$GSTACK_FREE_JOBS")
|
||||
|
||||
# The VM run gets its own session (python3, which ubi-runner already needs, for
|
||||
# setsid on Linux and macOS): pregate's --timeout and stall kill signal this
|
||||
# script's process group and then SIGKILL it, which must never cut ubi-runner's
|
||||
# teardown short. Output goes to files (a dead reader would SIGPIPE ubi-runner),
|
||||
# stamped for the phase timings; this script streams it. A watchdog sends
|
||||
# ubi-runner TERM at the cap-derived wall timeout, and so does a TERM or INT
|
||||
# to this script; ubi-runner then destroys the VM.
|
||||
vm_session='
|
||||
exec </dev/null > >(exec >/dev/null 2>&1; trap "" INT TERM HUP; while IFS= read -r line; do printf "%s %s\n" "$(date +%s)" "$line" >>"$2"; printf "%s\n" "$line" >>"$3"; done) 2>&1
|
||||
bash "$4" "${@:5}" -- "bash ~/.pregate-run.sh" &
|
||||
runner=$!
|
||||
echo "$runner" >"$2.pid"
|
||||
( sleep "$1"; touch "$2.timedout"; kill -TERM "$runner" ) </dev/null >/dev/null 2>&1 &
|
||||
watchdog=$!
|
||||
wait "$runner"
|
||||
rc=$?
|
||||
kill "$watchdog" 2>/dev/null
|
||||
[ ! -f "$2.timedout" ] || rc=124
|
||||
exit "$rc"
|
||||
'
|
||||
stream="$logs/stream.log"
|
||||
at() { grep -m1 -E "^[0-9]+ $1" "$log" | cut -d' ' -f1 || true; }
|
||||
started=$(date +%s)
|
||||
( while sleep 60 </dev/null >/dev/null 2>&1; do say "still running ($(( $(date +%s) - started )) s)"; done ) &
|
||||
heartbeat=$!
|
||||
vmrun="" tailer=""
|
||||
trap 'kill "$heartbeat" $tailer 2>/dev/null || true' EXIT
|
||||
trap '[ -z "$vmrun" ] || kill -TERM "$(cat "$log.pid" 2>/dev/null)" 2>/dev/null; stop 130 "interrupted: ubi-runner is tearing the VM down on its own (log: $log)"' INT TERM
|
||||
|
||||
billed=0 attempt=0
|
||||
while :; do
|
||||
attempt=$(( attempt + 1 ))
|
||||
: >"$log"
|
||||
: >"$stream"
|
||||
rm -f "$log.pid" "$log.timedout"
|
||||
t0=$(date +%s)
|
||||
python3 -c 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \
|
||||
bash -c "$vm_session" pregate-remote-vm "$(( (wall_min - 3) * 60 ))" "$log" "$stream" "$here/ubi-runner.sh" "${args[@]}" &
|
||||
vmrun=$!
|
||||
tail -n +1 -f "$stream" &
|
||||
tailer=$!
|
||||
set +e
|
||||
wait "$vmrun"
|
||||
rc=$?
|
||||
set -e
|
||||
vmrun=""
|
||||
sleep 1
|
||||
kill "$tailer" 2>/dev/null || true
|
||||
tailer=""
|
||||
t_end=$(date +%s)
|
||||
t_create=$(at 'ubi-runner: creating ')
|
||||
if [ -n "$t_create" ] && ! grep -q 'ubi-runner: create failed' "$log"; then minutes=$(( (t_end - t_create + 59) / 60 )); billed=$(( billed + (minutes > 0 ? minutes : 1) )); fi
|
||||
if [ "$rc" -ne 0 ] && [ "$attempt" -eq 1 ] && grep -q 'vCPU quota exhausted' "$log"; then
|
||||
say "vCPU quota refused the VM; one retry in ${PREGATE_REMOTE_QUOTA_WAIT_S:-60} s"
|
||||
cp "$log" "$logs/attempt-1.log"
|
||||
sleep "${PREGATE_REMOTE_QUOTA_WAIT_S:-60}"
|
||||
continue
|
||||
fi
|
||||
break
|
||||
done
|
||||
spent=$(usd "$billed")
|
||||
|
||||
t_ready=$(at 'ubi-runner: ready: ')
|
||||
t_setup=$(at 'ubi-runner: running setup')
|
||||
t_run=$(at 'ubi-runner: running on ')
|
||||
t_exit=$(at 'ubi-runner: command exited')
|
||||
span() { if [ -n "$1" ] && [ -n "$2" ]; then echo "$(( $2 - $1 ))s"; else echo "-"; fi; }
|
||||
say "phases: create $(span "$t0" "$t_ready"), setup $(span "$t_setup" "$t_run"), run $(span "$t_run" "$t_exit"), teardown $(span "$t_exit" "$t_end"); total $(( t_end - started ))s (target 600s)"
|
||||
say "log: $log"
|
||||
|
||||
vm_tree=$(sed -n 's/^[0-9]* PREGATE_VM_TREE \([0-9a-f]*\)$/\1/p' "$log" | tail -1)
|
||||
mismatch=$(grep -m1 -oE 'PREGATE_VM_(TREE|PLAN)_MISMATCH .*' "$log" || true)
|
||||
if [ -n "$mismatch" ]; then
|
||||
stop 3 "the VM refused before running tests: $mismatch (expected tree ${tree:0:12})" "no tests ran; spent=$spent ($billed billed min on $size)" "rerun: $rerun"
|
||||
fi
|
||||
result=$(result_line <(cut -d' ' -f2- "$log"))
|
||||
if [ -n "$result" ] && [ "$rc" -ne 124 ]; then
|
||||
after=$(sed -n 's/^[0-9]* PREGATE_VM_TREE_AFTER \(.*\)$/\1/p' "$log" | tail -1)
|
||||
built=$(sed -n 's/^[0-9]* PREGATE_VM_TREE_BUILT \(.*\)$/\1/p' "$log" | tail -1)
|
||||
sed -n 's/^[0-9]* PREGATE_VM_REBUILT / setup rebuilt (committed build output, as the CI free lane rebuilds it): /p' "$log"
|
||||
if [ "$after" != "$built" ]; then
|
||||
sed -n 's/^[0-9]* PREGATE_VM_MOVED / moved: /p' "$log"
|
||||
stop 3 "the VM's tree moved during the run (${built:0:12} after setup, ${after:0:12} after tests); no result reported" "spent=$spent ($billed billed min on $size)" "rerun: $rerun"
|
||||
fi
|
||||
echo "PREGATE_REMOTE: tree=$vm_tree tests=$(field tests "$result") failed=$(field failed "$result") spent=$spent"
|
||||
exit $(( rc == 0 ? 0 : 1 ))
|
||||
fi
|
||||
vm=$(sed -n 's/^[0-9]* ubi-runner: creating \([^ ]*\) .*/\1/p' "$log" | tail -1)
|
||||
down=$(grep -E 'ubi-runner: (destroyed|gone:|WARNING|FAILED to destroy)' "$log" | tail -1 | cut -d' ' -f2- || true)
|
||||
cause=$(grep -E 'ubi-runner: ' "$log" | grep -vE 'ubi-runner: (destroyed|gone:|destroy requested|waiting for an in-flight)' | tail -1 | cut -d' ' -f2- || true)
|
||||
[ "$rc" -ne 124 ] || cause="wall timeout of $(( wall_min - 3 )) min reached (cap \$$cap); the VM was torn down"
|
||||
stop 4 "infrastructure failure (ubi-runner exit $rc): ${cause:-no ubi-runner message}" \
|
||||
"VM: ${vm:-none created}${down:+; $down}" "log: $log" "spent=$spent ($billed billed min on $size)"
|
||||
@@ -0,0 +1,349 @@
|
||||
/**
|
||||
* W2-A3 (fix wave 3): scripts/ubicloud/pregate-remote.sh, gstack's declared
|
||||
* remote_gate. Each case runs the real script in a fixture checkout whose
|
||||
* scripts/ubicloud/ubi-runner.sh is a stub that "creates" a local VM
|
||||
* directory, so the keyed path runs its real VM-side setup prelude, the real
|
||||
* test-free-shards.ts --files-from and the real bin/gstack-wtree, offline and
|
||||
* at no cost. Reports are judged through the exported parseRemoteReport /
|
||||
* remoteReportProblem, exactly as gstack-pregate --remote judges them.
|
||||
*/
|
||||
import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { parseConfig } from '../lib/pregate/config';
|
||||
import { parseRemoteReport, remoteReportProblem } from '../lib/pregate/checks/lanes';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const SCRATCH = fs.mkdtempSync(path.join(os.tmpdir(), 'pregate-remote-gate-'));
|
||||
afterAll(() => fs.rmSync(SCRATCH, { recursive: true, force: true }));
|
||||
|
||||
const COPIED = ['scripts/ubicloud/pregate-remote.sh', 'bin/gstack-wtree', 'bin/gstack-path-spelling.sh',
|
||||
'scripts/test-free-shards.ts', 'scripts/test-strict-output.ts', 'scripts/lib/shard-engine.ts', 'scripts/lib/free-home-guard.ts',
|
||||
'scripts/lib/free-ci-health.ts', 'scripts/lib/windows-curation.ts', 'lib/state-root.ts', 'test/helpers/paid-test-set.ts',
|
||||
'test/helpers/touchfiles.ts', 'test/helpers/touchfiles-data.ts', 'test/helpers/test-selection.ts'];
|
||||
const TESTS: Record<string, string> = {
|
||||
'test/alpha.test.ts': "import { expect, test } from 'bun:test';\ntest('alpha', () => { expect(1).toBe(1); });\n",
|
||||
'test/beta.test.ts': "import { expect, test } from 'bun:test';\ntest('beta', () => { expect(2).toBe(2); });\n",
|
||||
'test/fails.test.ts': "import { expect, test } from 'bun:test';\ntest('fails', () => { expect(1).toBe(2); });\n",
|
||||
'test/empty.test.ts': "import 'bun:test';\n",
|
||||
'test/mover.test.ts': "import { test } from 'bun:test';\nimport * as fs from 'node:fs';\nimport * as path from 'node:path';\ntest('moves the tree', () => { fs.writeFileSync(path.join(import.meta.dir, '..', 'moved.txt'), 'x'); });\n",
|
||||
};
|
||||
|
||||
/** Stub of scripts/ubicloud/ubi-runner.sh (written into each fixture as that file). */
|
||||
const UBI_RUNNER_STUB = `#!/usr/bin/env bash
|
||||
# Stub of scripts/ubicloud/ubi-runner.sh for test/ubicloud-pregate-remote.test.ts:
|
||||
# "run" copies the checkout (tracked + untracked-unignored + .git) into a local
|
||||
# VM directory, runs setup and the command there, and prints ubi-runner's log
|
||||
# lines. STUB_FAIL picks a failure: create, quota, quota-once, ssh. STUB_HANG
|
||||
# sleeps before the command; TERM tears down as ubi-runner's trap does.
|
||||
# Every teardown is appended to $STUB_CALLS.down.
|
||||
set -uo pipefail
|
||||
[ "$1" = run ] || exit 2
|
||||
shift
|
||||
printf '%s\\n' "$*" >>"$STUB_CALLS"
|
||||
src=. setup="" size="" envs=()
|
||||
while [ $# -gt 0 ]; do
|
||||
case $1 in
|
||||
-s) size=$2; shift 2 ;;
|
||||
--src) src=$2; shift 2 ;;
|
||||
--setup) setup=$2; shift 2 ;;
|
||||
--env) envs+=("$2"); shift 2 ;;
|
||||
--pull) shift 2 ;;
|
||||
--) shift; break ;;
|
||||
*) echo "stub: unknown option $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
cmd=$*
|
||||
log() { echo "ubi-runner: $*" >&2; }
|
||||
name=ubirun-stub-1700000000-abcd
|
||||
log "creating eu-central-h1/$name ($size)"
|
||||
calls=$(wc -l <"$STUB_CALLS")
|
||||
fail=\${STUB_FAIL:-}
|
||||
if [ "$fail" = quota ] || { [ "$fail" = quota-once ] && [ "$calls" -eq 1 ]; }; then
|
||||
log "quota refused eu-central-h1/$name ($size): it needs 16 vCPUs and the project already uses 64 of 64"
|
||||
log "create failed: vCPU quota exhausted; wait for running VMs to finish, or use fewer or smaller VMs (ci:ubicloud --vms N)"
|
||||
log "gone: eu-central-h1/$name (never existed or already destroyed)"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$fail" = create ]; then
|
||||
echo "Validation failed for following fields: size" >&2
|
||||
log "create failed"
|
||||
log "gone: eu-central-h1/$name (never existed or already destroyed)"
|
||||
exit 1
|
||||
fi
|
||||
log "ready: $name (192.0.2.1)"
|
||||
trap 'log "destroyed eu-central-h1/$name"; echo "term $name" >>"$STUB_CALLS.down"; exit 130' TERM
|
||||
vm=$STUB_VM_HOME
|
||||
dest="$vm/work/$(basename "$src")"
|
||||
rm -rf "$dest" && mkdir -p "$dest"
|
||||
(cd "$src" && { git ls-files -z -co --exclude-standard; printf '.git\\0'; } | tar --null -T - -cf -) | tar -xf - -C "$dest"
|
||||
[ -z "\${STUB_VM_EDIT:-}" ] || echo "// edited on the VM" >>"$dest/$STUB_VM_EDIT"
|
||||
for e in \${envs[@]+"\${envs[@]}"}; do printf 'export %s\\n' "$e"; done >"$vm/.ubirun-env"
|
||||
log "synced $src -> $name:$dest"
|
||||
if [ -n "$setup" ]; then
|
||||
cp "$setup" "$vm/.ubirun-setup.sh"
|
||||
log "running setup $(basename "$setup")"
|
||||
if ! (cd "$dest" && HOME="$vm" bash -c '. ~/.ubirun-env && bash ~/.ubirun-setup.sh'); then
|
||||
log "setup failed"
|
||||
log "destroyed eu-central-h1/$name"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
[ -z "\${STUB_VM_LIST_ADD:-}" ] || echo "$STUB_VM_LIST_ADD" >>"$vm/.pregate-files.txt"
|
||||
log "running on $name: $cmd"
|
||||
if [ -n "\${STUB_HANG:-}" ]; then sleep "$STUB_HANG" & wait $!; fi
|
||||
rc=0
|
||||
if [ "$fail" = ssh ]; then
|
||||
rc=255
|
||||
else
|
||||
(cd "$dest" && HOME="$vm" PATH="$STUB_BIN:$PATH" bash -c ". ~/.ubirun-env && $cmd") || rc=$?
|
||||
fi
|
||||
log "command exited $rc after 1s"
|
||||
log "destroyed eu-central-h1/$name"
|
||||
echo "exit $name" >>"$STUB_CALLS.down"
|
||||
exit "$rc"
|
||||
`;
|
||||
|
||||
interface Fixture { repo: string; vm: string; calls: string; bin: string; tree: string }
|
||||
|
||||
let counter = 0;
|
||||
function fixture(): Fixture {
|
||||
const base = path.join(SCRATCH, String(++counter));
|
||||
const repo = path.join(base, 'gstack');
|
||||
const vm = path.join(base, 'vm');
|
||||
const bin = path.join(base, 'bin');
|
||||
for (const dir of [repo, vm, bin]) fs.mkdirSync(dir, { recursive: true });
|
||||
const put = (rel: string, body: string, mode = 0o644) => {
|
||||
fs.mkdirSync(path.dirname(path.join(repo, rel)), { recursive: true });
|
||||
fs.writeFileSync(path.join(repo, rel), body, { mode });
|
||||
};
|
||||
for (const rel of COPIED) put(rel, fs.readFileSync(path.join(ROOT, rel), 'utf8'), fs.statSync(path.join(ROOT, rel)).mode & 0o777);
|
||||
for (const [rel, body] of Object.entries(TESTS)) put(rel, body);
|
||||
put('scripts/ubicloud/ubi-runner.sh', UBI_RUNNER_STUB, 0o755);
|
||||
put('scripts/ubicloud/setup-free-suite.sh', '#!/usr/bin/env bash\necho "setup-free-suite (stub)"\n[ -z "${STUB_SETUP_FAIL:-}" ] || exit 1\n[ -z "${STUB_SETUP_REBUILD:-}" ] || echo rebuilt >>dist/bundle.txt\n', 0o755);
|
||||
put('dist/bundle.txt', 'committed build output\n');
|
||||
put('.gitignore', '.context/\n');
|
||||
put('package.json', JSON.stringify({ scripts: { 'test:free': 'bun run scripts/test-free-shards.ts' } }) + '\n');
|
||||
fs.writeFileSync(path.join(bin, 'xvfb-run'), '#!/bin/sh\n[ "$1" = -a ] && shift\nexec "$@"\n', { mode: 0o755 });
|
||||
const git = (...args: string[]) => {
|
||||
const r = Bun.spawnSync(['git', '-c', 'user.email=t@t', '-c', 'user.name=t', ...args], { cwd: repo, timeout: 30_000 });
|
||||
if (r.exitCode !== 0) throw new Error(`git ${args.join(' ')}: ${r.stderr.toString()}`);
|
||||
return r.stdout.toString().trim();
|
||||
};
|
||||
git('init', '-q', '-b', 'main');
|
||||
git('add', '-A');
|
||||
git('commit', '-qm', 'fixture');
|
||||
const tree = Bun.spawnSync(['bin/gstack-wtree'], { cwd: repo, timeout: 30_000 }).stdout.toString().trim();
|
||||
return { repo, vm, calls: path.join(base, 'calls.log'), bin, tree };
|
||||
}
|
||||
|
||||
function gate(f: Fixture, files: string[] | null, env: Record<string, string | undefined> = {}) {
|
||||
const list = path.join(path.dirname(f.repo), 'files.txt');
|
||||
if (files) fs.writeFileSync(list, files.join('\n') + '\n');
|
||||
const base: Record<string, string | undefined> = { ...process.env, UBICLOUD_API_KEY: undefined, UBICLOUD_API_TOKEN: undefined, UBI_SIZE: undefined,
|
||||
GSTACK_FREE_JOBS: '2', GSTACK_FLAKE_LEDGER: path.join(path.dirname(f.repo), 'flakes.jsonl'), STUB_CALLS: f.calls, STUB_VM_HOME: f.vm, STUB_BIN: f.bin };
|
||||
if (files) Object.assign(base, { PREGATE_FILES_FILE: list, PREGATE_TREE: f.tree, PREGATE_SPEND_CAP_USD: '2' });
|
||||
const merged = Object.fromEntries(Object.entries({ ...base, ...env }).filter(([, v]) => v !== undefined)) as Record<string, string>;
|
||||
const r = Bun.spawnSync(['bash', 'scripts/ubicloud/pregate-remote.sh'], { cwd: f.repo, env: merged, timeout: 90_000 });
|
||||
const out = r.stdout.toString() + r.stderr.toString();
|
||||
return { exit: r.exitCode, out, report: parseRemoteReport(out), calls: fs.existsSync(f.calls) ? fs.readFileSync(f.calls, 'utf8').trim().split('\n') : [] };
|
||||
}
|
||||
const KEY = { UBICLOUD_API_KEY: 'offline-stub' };
|
||||
|
||||
describe('pregate-remote.sh inputs and the local tree check', () => {
|
||||
test('run bare it exits 2 before any work, naming the pregate command', () => {
|
||||
const f = fixture();
|
||||
const r = gate(f, null, KEY);
|
||||
expect(r.exit).toBe(2);
|
||||
expect(r.out).toContain('PREGATE_FILES_FILE is not set; run: gstack-pregate --stage tests --remote --spend-cap <usd> --allow-repo-commands');
|
||||
expect(r.calls).toEqual([]);
|
||||
expect(fs.existsSync(path.join(f.repo, '.context'))).toBe(false);
|
||||
}, 60_000);
|
||||
|
||||
test('a working tree that is not PREGATE_TREE exits 3 before any upload, with both prefixes and the rerun command', () => {
|
||||
const f = fixture();
|
||||
const r = gate(f, ['test/alpha.test.ts'], { ...KEY, PREGATE_TREE: 'f'.repeat(40) });
|
||||
expect(r.exit).toBe(3);
|
||||
expect(r.out).toContain(`the working tree is ${f.tree.slice(0, 12)}, not the planned tree ffffffffffff`);
|
||||
expect(r.out).toContain('nothing uploaded, nothing spent');
|
||||
expect(r.out).toContain('rerun: gstack-pregate --stage tests --remote --spend-cap <usd> --allow-repo-commands');
|
||||
expect(r.calls).toEqual([]);
|
||||
expect(r.report).toBeNull();
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
describe('pregate-remote.sh keyed path (stub ubi-runner)', () => {
|
||||
test('a passing run prints the VM tree, the counts and billed spend; pregate clears it', () => {
|
||||
const f = fixture();
|
||||
const r = gate(f, ['test/alpha.test.ts', 'test/beta.test.ts'], KEY);
|
||||
expect(r.exit, r.out).toBe(0);
|
||||
expect(r.report).toEqual({ tree: f.tree, tests: 2, failed: 0, spent: '0.01' });
|
||||
expect(remoteReportProblem(r.report!, f.tree)).toBeNull();
|
||||
expect(r.out).toContain('2 files on one standard-16 VM; wall timeout 45 min, worst case $0.38 under the cap $2');
|
||||
expect(r.out).toMatch(/phases: create \d+s, setup \d+s, run \d+s, teardown \d+s; total \d+s \(target 600s\)/);
|
||||
expect(r.out.trim().split('\n').at(-1)).toMatch(/^PREGATE_REMOTE: /);
|
||||
expect(r.calls).toHaveLength(1);
|
||||
for (const arg of ['-s standard-16 ', '--env GSTACK_EXPECT_BINARIES=1', '--env GSTACK_FREE_RETRY_FLAKY=1', '--env GSTACK_FREE_JOBS=2', '-- bash ~/.pregate-run.sh']) expect(r.calls[0]).toContain(arg);
|
||||
expect(fs.readFileSync(path.join(f.vm, '.pregate-files.txt'), 'utf8')).toBe('test/alpha.test.ts\ntest/beta.test.ts\n');
|
||||
}, 90_000);
|
||||
|
||||
test('setup rebuilding committed build output (as CI does) is printed; the reported tree stays the uploaded one', () => {
|
||||
const f = fixture();
|
||||
const r = gate(f, ['test/alpha.test.ts'], { ...KEY, STUB_SETUP_REBUILD: '1' });
|
||||
expect(r.exit, r.out).toBe(0);
|
||||
expect(r.out).toContain(' setup rebuilt (committed build output, as the CI free lane rebuilds it): M\tdist/bundle.txt');
|
||||
expect(r.report).toEqual({ tree: f.tree, tests: 1, failed: 0, spent: '0.01' });
|
||||
}, 90_000);
|
||||
|
||||
test('a failing test reports failed>0 and exits 1; a file with zero tests reports tests=0; neither clears', () => {
|
||||
const failing = gate(fixture(), ['test/alpha.test.ts', 'test/fails.test.ts'], KEY);
|
||||
expect(failing.exit, failing.out).toBe(1);
|
||||
expect(failing.report).toMatchObject({ tests: 2, failed: 1 });
|
||||
expect(remoteReportProblem(failing.report!, failing.report!.tree)).toBe('the remote gate reported 1 failed of 2');
|
||||
const empty = gate(fixture(), ['test/empty.test.ts'], KEY);
|
||||
expect(empty.report, empty.out).toMatchObject({ tests: 0, failed: 0 });
|
||||
expect(remoteReportProblem(empty.report!, empty.report!.tree)).toBe('the remote gate reported 0 tests');
|
||||
}, 90_000);
|
||||
|
||||
test('the cap refuses before any VM is created, at the price of the size actually used', () => {
|
||||
const f = fixture();
|
||||
const low = gate(f, ['test/alpha.test.ts'], { ...KEY, PREGATE_SPEND_CAP_USD: '0.1' });
|
||||
expect(low.exit).toBe(5);
|
||||
expect(low.out).toContain('worst case $0.17 (20 min x $0.00831/min on standard-16) exceeds the spend cap $0.1');
|
||||
expect(low.out).toContain('nothing created, nothing spent');
|
||||
expect(low.out).toContain('rerun with --spend-cap 0.17 or more');
|
||||
const big = gate(f, ['test/alpha.test.ts'], { ...KEY, PREGATE_SPEND_CAP_USD: '0.5', UBI_SIZE: 'standard-60' });
|
||||
expect(big.exit).toBe(5);
|
||||
expect(big.out).toContain('worst case $0.63 (20 min x $0.03117/min on standard-60) exceeds the spend cap $0.5');
|
||||
const unpriced = gate(f, ['test/alpha.test.ts'], { ...KEY, UBI_SIZE: 'burstable-1' });
|
||||
expect(unpriced.exit).toBe(5);
|
||||
expect(unpriced.out).toContain('no price for UBI_SIZE=burstable-1');
|
||||
const noCap = gate(f, ['test/alpha.test.ts'], { ...KEY, PREGATE_SPEND_CAP_USD: '' });
|
||||
expect(noCap.exit).toBe(2);
|
||||
expect([low, big, unpriced, noCap].every(r => r.report === null)).toBe(true);
|
||||
expect(fs.existsSync(f.calls)).toBe(false);
|
||||
}, 90_000);
|
||||
|
||||
test('create, setup and SSH failures exit 4 with the ubi-runner message, the VM, its teardown line, the log and spent=', () => {
|
||||
const create = gate(fixture(), ['test/alpha.test.ts'], { ...KEY, STUB_FAIL: 'create' });
|
||||
expect(create.exit, create.out).toBe(4);
|
||||
expect(create.out).toContain('infrastructure failure (ubi-runner exit 1): ubi-runner: create failed');
|
||||
expect(create.out).toContain('VM: eu-central-h1/ubirun-stub-1700000000-abcd; ubi-runner: gone: eu-central-h1/ubirun-stub-1700000000-abcd (never existed or already destroyed)');
|
||||
expect(create.out).toMatch(/log: \S+\/\.context\/pregate-remote\/\S+\/run\.log/);
|
||||
expect(create.out).toContain('spent=0.00 (0 billed min on standard-16)');
|
||||
const setup = gate(fixture(), ['test/alpha.test.ts'], { ...KEY, STUB_SETUP_FAIL: '1' });
|
||||
expect(setup.exit, setup.out).toBe(4);
|
||||
expect(setup.out).toContain('ubi-runner: setup failed');
|
||||
expect(setup.out).toContain('; ubi-runner: destroyed eu-central-h1/ubirun-stub-1700000000-abcd');
|
||||
expect(setup.out).toContain('spent=0.01 (1 billed min on standard-16)');
|
||||
const ssh = gate(fixture(), ['test/alpha.test.ts'], { ...KEY, STUB_FAIL: 'ssh' });
|
||||
expect(ssh.exit, ssh.out).toBe(4);
|
||||
expect(ssh.out).toContain('infrastructure failure (ubi-runner exit 255): ubi-runner: command exited 255 after 1s');
|
||||
expect([create, setup, ssh].every(r => r.report === null)).toBe(true);
|
||||
}, 90_000);
|
||||
|
||||
test('a quota refusal is retried once, then the run stops', () => {
|
||||
const once = gate(fixture(), ['test/alpha.test.ts'], { ...KEY, STUB_FAIL: 'quota-once', PREGATE_REMOTE_QUOTA_WAIT_S: '0' });
|
||||
expect(once.exit, once.out).toBe(0);
|
||||
expect(once.out).toContain('vCPU quota refused the VM; one retry in 0 s');
|
||||
expect(once.calls).toHaveLength(2);
|
||||
const twice = gate(fixture(), ['test/alpha.test.ts'], { ...KEY, STUB_FAIL: 'quota', PREGATE_REMOTE_QUOTA_WAIT_S: '0' });
|
||||
expect(twice.exit, twice.out).toBe(4);
|
||||
expect(twice.calls).toHaveLength(2);
|
||||
expect(twice.out).toContain('create failed: vCPU quota exhausted');
|
||||
}, 90_000);
|
||||
|
||||
test('the VM refuses a tree that differs after unpack, a planned set that differs from the list, and a tree that moved during the run', () => {
|
||||
const edited = gate(fixture(), ['test/alpha.test.ts'], { ...KEY, STUB_VM_EDIT: 'test/alpha.test.ts' });
|
||||
expect(edited.exit, edited.out).toBe(3);
|
||||
expect(edited.out).toContain('the VM refused before running tests: PREGATE_VM_TREE_MISMATCH vm=');
|
||||
expect(edited.out).toContain('no tests ran; spent=0.01');
|
||||
const listed = gate(fixture(), ['test/alpha.test.ts'], { ...KEY, STUB_VM_LIST_ADD: 'test/beta.test.ts' });
|
||||
expect(listed.exit, listed.out).toBe(3);
|
||||
expect(listed.out).toContain('PREGATE_VM_PLAN_MISMATCH planned=');
|
||||
const moved = gate(fixture(), ['test/alpha.test.ts', 'test/mover.test.ts'], KEY);
|
||||
expect(moved.exit, moved.out).toBe(3);
|
||||
expect(moved.out).toContain("the VM's tree moved during the run");
|
||||
expect(moved.out).toContain(' moved: A\tmoved.txt');
|
||||
expect([edited, listed, moved].every(r => r.report === null)).toBe(true);
|
||||
}, 90_000);
|
||||
|
||||
test('killing the gate\'s process group (pregate\'s timeout) never cuts teardown short; TERM to the gate tears the VM down', async () => {
|
||||
const run = (f: Fixture) => {
|
||||
const list = path.join(path.dirname(f.repo), 'files.txt');
|
||||
fs.writeFileSync(list, 'test/alpha.test.ts\n');
|
||||
return Bun.spawn(['bash', 'scripts/ubicloud/pregate-remote.sh'], { cwd: f.repo, detached: true, stdout: 'pipe', stderr: 'pipe',
|
||||
env: { ...process.env, ...KEY, UBI_SIZE: '', GSTACK_FREE_JOBS: '2', STUB_CALLS: f.calls, STUB_VM_HOME: f.vm, STUB_BIN: f.bin, STUB_HANG: '4',
|
||||
PREGATE_FILES_FILE: list, PREGATE_TREE: f.tree, PREGATE_SPEND_CAP_USD: '2' } as Record<string, string> });
|
||||
};
|
||||
const until = async (cond: () => boolean, ms: number) => {
|
||||
const deadline = Date.now() + ms;
|
||||
while (!cond()) { if (Date.now() > deadline) throw new Error('condition not met in time'); await Bun.sleep(50); }
|
||||
};
|
||||
const vmLog = (f: Fixture) => { try { return fs.readFileSync(fs.readdirSync(path.join(f.repo, '.context/pregate-remote')).map(d => path.join(f.repo, '.context/pregate-remote', d, 'run.log'))[0]!, 'utf8'); } catch { return ''; } };
|
||||
const killed = fixture();
|
||||
const a = run(killed);
|
||||
await until(() => vmLog(killed).includes('ubi-runner: running on'), 30_000);
|
||||
process.kill(-a.pid, 'SIGKILL');
|
||||
await a.exited;
|
||||
await until(() => fs.existsSync(killed.calls + '.down'), 30_000);
|
||||
expect(fs.readFileSync(killed.calls + '.down', 'utf8')).toBe('exit ubirun-stub-1700000000-abcd\n');
|
||||
const termed = fixture();
|
||||
const b = run(termed);
|
||||
await until(() => vmLog(termed).includes('ubi-runner: running on'), 30_000);
|
||||
process.kill(b.pid, 'SIGTERM');
|
||||
expect(await b.exited).toBe(130);
|
||||
expect(await new Response(b.stdout).text()).toContain('interrupted: ubi-runner is tearing the VM down on its own');
|
||||
await until(() => fs.existsSync(termed.calls + '.down'), 10_000);
|
||||
expect(fs.readFileSync(termed.calls + '.down', 'utf8')).toBe('term ubirun-stub-1700000000-abcd\n');
|
||||
}, 90_000);
|
||||
|
||||
test('the VM environment is bun run test:ubicloud\'s (scripts/ubicloud/test-free.sh), so the two cannot drift', () => {
|
||||
const script = fs.readFileSync(path.join(ROOT, 'scripts/ubicloud/pregate-remote.sh'), 'utf8');
|
||||
const wrapper = fs.readFileSync(path.join(ROOT, 'scripts/ubicloud/test-free.sh'), 'utf8');
|
||||
const flags = [...wrapper.matchAll(/--(?:env|pull) "?[^\s"]+(?: [^\s"]+)?/g)].map(m => m[0]).filter(flag => !flag.includes('GSTACK_FREE_JOBS'));
|
||||
expect(flags.length).toBeGreaterThanOrEqual(5);
|
||||
for (const flag of flags) expect(script).toContain(flag);
|
||||
expect(wrapper).toContain('--setup "$here/setup-free-suite.sh"');
|
||||
expect(script).toContain("'bash scripts/ubicloud/setup-free-suite.sh'");
|
||||
expect(wrapper).toContain('xvfb-run -a bun run test:free');
|
||||
expect(script).toContain("'xvfb-run -a bun run test:free --files-from ~/.pregate-files.txt'");
|
||||
});
|
||||
});
|
||||
|
||||
describe('pregate-remote.sh keyless path', () => {
|
||||
test('without a key it runs the list locally in parallel shards and reports the tree it tested with spent=0', () => {
|
||||
const f = fixture();
|
||||
const r = gate(f, ['test/alpha.test.ts', 'test/beta.test.ts']);
|
||||
expect(r.exit, r.out).toBe(0);
|
||||
expect(r.out).toContain('no UBICLOUD_API_KEY: running 2 files locally in parallel shards');
|
||||
expect(r.report).toEqual({ tree: f.tree, tests: 2, failed: 0, spent: '0' });
|
||||
expect(remoteReportProblem(r.report!, f.tree)).toBeNull();
|
||||
expect(fs.existsSync(f.calls)).toBe(false);
|
||||
}, 90_000);
|
||||
|
||||
test('a tree that moves during the local run fails without a report', () => {
|
||||
const r = gate(fixture(), ['test/alpha.test.ts', 'test/mover.test.ts']);
|
||||
expect(r.exit, r.out).toBe(3);
|
||||
expect(r.out).toContain('the working tree moved during the run');
|
||||
expect(r.out).toContain(' moved: A\tmoved.txt');
|
||||
expect(r.report).toBeNull();
|
||||
}, 90_000);
|
||||
|
||||
test('a list test-free-shards rejects exits 1 without a report', () => {
|
||||
const r = gate(fixture(), ['test/alpha.test.ts', 'test/missing.test.ts']);
|
||||
expect(r.exit, r.out).toBe(1);
|
||||
expect(r.out).toContain('1 path(s) not in the free planned set: test/missing.test.ts');
|
||||
expect(r.out).toContain('without a FILES-FROM RESULT line');
|
||||
expect(r.report).toBeNull();
|
||||
}, 90_000);
|
||||
});
|
||||
|
||||
describe('.gstack/pregate.json declares the remote gate', () => {
|
||||
test('remote_gate is this script, and the config still validates', () => {
|
||||
const { config, errors } = parseConfig(fs.readFileSync(path.join(ROOT, '.gstack/pregate.json'), 'utf8'));
|
||||
expect(errors).toEqual([]);
|
||||
expect(config.remote_gate).toBe('bash scripts/ubicloud/pregate-remote.sh');
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user