diff --git a/changelog.d/gst-1441-ubi-runner-money-path.md b/changelog.d/gst-1441-ubi-runner-money-path.md new file mode 100644 index 000000000..927c63df8 --- /dev/null +++ b/changelog.d/gst-1441-ubi-runner-money-path.md @@ -0,0 +1 @@ +- **The Ubicloud runner no longer reads an API error as success.** A create Ubicloud refuses (a 409 name collision, any 4xx) now disarms teardown, so `up` and `run` never destroy a VM someone else holds under that name. A 5xx or unanswered create is looked up by name before anything is decided: a VM that landed anyway is adopted, and one nobody can confirm either way keeps its record and recovery bundle and exits `UBI_CREATE_UNKNOWN`. `down` counts a VM as gone only on a readable answer, so a failed or garbled VM list exits non-zero and keeps the bundle. `pull` exits non-zero when ssh fails, and `run` fails when one of its pulls does. `gc` exits non-zero when a stale VM could not be destroyed, a run record whose VM the API cannot check refuses with `UBI_ATTACH_UNKNOWN` instead of starting a second VM, and the API key reaches curl as a header on a pipe, out of `ps`. (GST-1441) diff --git a/scripts/ubicloud/ubi-runner.sh b/scripts/ubicloud/ubi-runner.sh index fcac57beb..c46664041 100755 --- a/scripts/ubicloud/ubi-runner.sh +++ b/scripts/ubicloud/ubi-runner.sh @@ -35,7 +35,15 @@ # confirmed gone or provably never existed, waiting out a create that may still # be in flight. An interrupted `up` or `run` (EXIT, INT, TERM, QUIT, and HUP # until a remote phase is live) runs that `down`; further signals are ignored -# until it finishes. A create refused +# until it finishes. Teardown is armed only once this call's create request is +# sent, and disarmed when Ubicloud refuses it (HTTP 4xx, a 409 name collision +# included), so it never destroys a VM this call did not create. A 5xx or no +# answer is not a refusal: `up` looks the VM up by name for UBI_CREATE_GRACE +# seconds, and when that cannot settle it either, it keeps the record and the +# recovery bundle and exits UBI_CREATE_UNKNOWN. `down` counts a VM as gone only +# on a readable answer: a failed or unreadable VM list or show is unknown, so +# down exits non-zero and keeps the recovery bundle. The API key reaches curl +# as a header file on a pipe, never in its argv. A create refused # for vCPU quota prints the project's usage per owner. # # The owner-age sweep is off by default (the TTL reap below always runs). With UBI_GC_HOURS set to a positive @@ -225,7 +233,7 @@ api() { shift code=$(python3 -c 'import json,sys; print(json.dumps({"argv": sys.argv[1:]}))' "$@" \ | curl -sS -o "$out" -w '%{http_code}' --max-time "$API_TIMEOUT" -X POST \ - -H "Authorization: Bearer $UBICLOUD_API_KEY" \ + -H @<(printf 'Authorization: Bearer %s\n' "$UBICLOUD_API_KEY") \ -H 'Accept: text/plain' -H 'Content-Type: application/json' \ -H 'X-Ubi-Version: 1.0.0' --data @- "$API/cli") || true echo "${code:-000}" @@ -552,6 +560,8 @@ lease_busy() { # vm_alive NAME: the recorded VM still answers show. One confirmed gone is # cleaned up (state dir and scratch record), so its record is never tried again. +# Returns 2 when the API gives no answer either way: the caller must not start a +# second VM beside one that may still be running. vm_alive() { local n=$1 out code ( load "$n"; [ "$CREATED" = yes ] && [ -n "$IP" ] ) 2>/dev/null || return 1 @@ -561,7 +571,7 @@ vm_alive() { case $code in 200) return 0 ;; 404) log "run record for $n: the VM is gone; starting a new one"; cmd_down "$n" >/dev/null 2>&1 || true; return 1 ;; - *) return 1 ;; + *) log "run record for $n: could not check the VM (HTTP $code)"; return 2 ;; esac } @@ -603,7 +613,8 @@ find_record() { [ -f "$f" ] || continue grep -qx "RUN_KEY=$RUN_KEY" "$f" || continue n=$(basename "$(dirname "$f")") - if vm_alive "$n"; then echo "$n"; return 0; fi + vm_alive "$n" && { echo "$n"; return 0; } + [ $? -ne 2 ] || return 2 done } @@ -775,8 +786,13 @@ run_tiers() { # did not name; expires is the epoch the name's -t TTL ends at, or "-"; # created is the name's epoch, or "-". vm_rows() { - local loc name owner created expires - cli vm list -N -f location,name | while read -r loc name; do + local loc name owner created expires list + list=$(cli vm list -N -f location,name) || return 1 + if grep -Evq '^[[:blank:]]*([a-z0-9-]+[[:blank:]]+[a-z0-9-]+)?[[:blank:]]*$' <<<"$list"; then + log "the VM list answer is unreadable (a line is not 'location name')" + return 1 + fi + while read -r loc name; do [ -n "$name" ] || continue owner=. created="" expires=- if [[ "$name" =~ ^${PREFIX}-([a-z][a-z0-9]{0,11})-([0-9]{10})- ]]; then @@ -788,7 +804,7 @@ vm_rows() { expires=$(( 10#$created + 10#${BASH_REMATCH[1]} * 3600 )) fi echo "$loc $name $owner $expires ${created:--}" - done + done <<<"$list" } # reap: destroy the caller's own VMs whose TTL ended more than REAP_GRACE @@ -823,18 +839,20 @@ cmd_reap() { } cmd_gc() { - local hours=${1:-$GC_HOURS} now loc name owner + local hours=${1:-$GC_HOURS} now rows loc name owner failed=0 [[ "$hours" =~ ^[0-9]+$ ]] || die "gc: HOURS must be a whole number (or set UBI_GC_HOURS)" if [ "$hours" -eq 0 ]; then log "gc: no sweep (give HOURS > 0 or set UBI_GC_HOURS); nothing listed, nothing destroyed"; return 0; fi now=$(date +%s) - vm_rows | while read -r loc name owner _; do + rows=$(vm_rows) || { log "gc: could not list the project's VMs; nothing destroyed"; return 1; } + while read -r loc name owner _; do [ "$owner" = "$OWNER" ] || continue [[ "$name" =~ -([0-9]{10})- ]] || continue if [ $(( (now - BASH_REMATCH[1]) / 3600 )) -ge "$hours" ]; then log "destroying stale $loc/$name (owner $OWNER, older than ${hours}h)" - cmd_down "$name" -l "$loc" || log "failed to destroy $loc/$name" + cmd_down "$name" -l "$loc" || { failed=$(( failed + 1 )); log "failed to destroy $loc/$name"; } fi - done + done <<<"$rows" + [ "$failed" -eq 0 ] || { log "gc: $failed stale VMs could not be destroyed"; return 1; } } cmd_list() { @@ -1296,25 +1314,63 @@ cmd_up() { [ -z "$storage" ] || args+=(-S "$storage") args+=("$(cat "$dir/key.pub")") create_at=$(date +%s) + log "creating $location/$name ($size)" + # Signals are held from before the create request until its answer is + # recorded. Teardown is armed (UP_NAME) only once the request is on its way + # and disarmed when Ubicloud refuses it (HTTP 4xx): a refused create made + # nothing, and a VM that answers to the name (a 409) is not this call's. + local interrupted="" created code err refusal need used max + trap 'interrupted=1' INT TERM HUP QUIT write_state "$name" "$location" "$size" "" unknown "$create_at" bundle_save "$name" record_vm "$name" "$location" UP_NAME=$name - log "creating $location/$name ($size)" - # Hold signals until the create answer is recorded, so teardown knows - # whether the VM exists instead of waiting out UBI_CREATE_GRACE. - local interrupted="" created=yes refusal need used max - trap 'interrupted=1' INT TERM HUP QUIT - cli "${args[@]}" >/dev/null 2>"$dir/create.err" || created=no - write_state "$name" "$location" "$size" "" "$created" "$create_at" - [ "$created" = no ] || bundle_save "$name" + code=$(api "$dir/create.err" "${args[@]}") + case $code in + 200) created=yes ;; + 4[0-9][0-9]) created=no ;; + *) created=unknown ;; + esac + if [ "$created" = no ]; then + UP_NAME="" + err=$(cat "$dir/create.err") + rm -rf "$dir" + forget_vm "$name" + [ -z "$RECOVERY_DIR" ] || rm -f "$RECOVERY_DIR/$OWNER/$name.bundle" + else + write_state "$name" "$location" "$size" "" "$created" "$create_at" + [ "$created" = unknown ] || bundle_save "$name" + fi trap 'exit 130' INT TERM HUP QUIT [ -z "$interrupted" ] || exit 130 + if [ "$created" = unknown ]; then + # A 5xx or no answer (000) is not a refusal: the VM may exist. Look it up + # by name until it shows or UBI_CREATE_GRACE has passed. + log "create of $location/$name answered HTTP $code; looking it up by name before deciding" + local seen_code + while :; do + seen_code=$(api "$dir/lookup.out" vm "$location/$name" show) + [ "$seen_code" != 200 ] || break + [ $(( $(date +%s) - create_at )) -lt "$CREATE_GRACE" ] || break + sleep "$POLL" + done + rm -f "$dir/lookup.out" + case $seen_code in + 200) created=yes ;; + 404) created=no ;; + esac + write_state "$name" "$location" "$size" "" "$created" "$create_at" + case $created in + yes) bundle_save "$name"; log "create of $location/$name went through; continuing" ;; + no) cat "$dir/create.err" >&2; die "create failed (HTTP $code): $location/$name was still absent ${CREATE_GRACE}s after the request" ;; + unknown) die "UBI_CREATE_UNKNOWN: could not tell whether $location/$name exists (create answered HTTP $code, its lookup HTTP $seen_code); its record and recovery bundle stay, teardown destroys it if it shows up, and this owner's reap destroys it once its -t TTL ends; check with: $0 down $name" ;; + esac + fi if [ "$created" = no ]; then - refusal=$(sed -n 's/.*Requested vCPU count: \([0-9]*\), currently used vCPU count: \([0-9]*\), maximum allowed vCPU count: \([0-9]*\).*/\1 \2 \3/p' "$dir/create.err") + refusal=$(sed -n 's/.*Requested vCPU count: \([0-9]*\), currently used vCPU count: \([0-9]*\), maximum allowed vCPU count: \([0-9]*\).*/\1 \2 \3/p' <<<"$err") if [ -z "$refusal" ]; then - cat "$dir/create.err" >&2 - die "create failed" + printf '%s\n' "$err" >&2 + die "create failed (HTTP $code); nothing was created" fi read -r need used max <<<"$refusal" echo "$max" >"$STATE_ROOT/quota-max" @@ -1380,14 +1436,20 @@ cmd_down() { fi fi - local deadline=$(( $(date +%s) + DOWN_TIMEOUT )) seen=0 asked=0 out code now + local deadline=$(( $(date +%s) + DOWN_TIMEOUT )) seen=0 asked=0 out code now rows unlisted=0 out=$(mktemp) while :; do now=$(date +%s) if [ -z "$loc" ]; then - loc=$(vm_rows 2>/dev/null | awk -v n="$name" '$2==n {print $1}') || loc="" - code=404 - [ -z "$loc" ] || code=$(api "$out" vm "$loc/$name" show) + if rows=$(vm_rows 2>/dev/null); then + loc=$(awk -v n="$name" '$2==n {print $1}' <<<"$rows") + code=404 + [ -z "$loc" ] || code=$(api "$out" vm "$loc/$name" show) + else + code=unlisted + [ "$unlisted" = 1 ] || log "could not read the project's VM list; $name is not counted as gone" + unlisted=1 + fi else code=$(api "$out" vm "$loc/$name" show) fi @@ -1541,12 +1603,14 @@ cmd_sync() { # Relative remote paths start at /home/ubi. A glob that matches nothing (for # example an optional flake ledger) is reported and skipped, not a failure. cmd_pull() { - local name=$1 from=$2 to=$3 dir + local name=$1 from=$2 to=$3 dir rc=0 dir=$(printf %q "$(dirname "$from")") - if ! remote "$name" "cd $dir 2>/dev/null && ls -d -- $(basename "$from") >/dev/null 2>&1"; then - log "pull: nothing matches $from" - return 0 - fi + remote "$name" "cd $dir 2>/dev/null || exit 3; ls -d -- $(basename "$from") >/dev/null 2>&1 || exit 3" || rc=$? + case $rc in + 0) ;; + 3) log "pull: nothing matches $from"; return 0 ;; + *) log "pull failed: could not check $from on $name (exit $rc)"; return 1 ;; + esac mkdir -p "$to" remote "$name" "cd $dir && tar -czf - $(basename "$from")" | tar -xzf - -C "$to" } @@ -1602,13 +1666,14 @@ PULL_ROOT="" run_teardown() { local rc=$1 rows left others trap "" INT TERM HUP QUIT - cmd_down "$RUN_VM" || log "WARNING: failed to destroy $RUN_VM; run: $0 down $RUN_VM" + [ -z "$UP_NAME" ] || cmd_down "$RUN_VM" || log "WARNING: failed to destroy $RUN_VM; run: $0 down $RUN_VM" if rows=$(vm_rows 2>/dev/null); then left=$(awk -v n="$RUN_VM" '$2==n' <<<"$rows" | grep -c . || true) others=$(awk -v n="$RUN_VM" -v o="$OWNER" '$3==o && $2!=n' <<<"$rows" | grep -c . || true) else left=unknown others=unknown fi + [ -n "$UP_NAME" ] || left=0 leftover "left=$left owner=$OWNER vm=$RUN_VM others=$others" if [ "$left" = 0 ]; then [[ "$SINK_FLUSH_FAILED" != *" $RUN_VM "* ]] || [ "$rc" -ne 0 ] || exit 1 @@ -1691,10 +1756,10 @@ cmd_run() { local attach="" if [ -n "$FORCE_ATTACH" ]; then - vm_alive "$FORCE_ATTACH" || die "attach: $FORCE_ATTACH no longer exists; rerun the original command to start fresh" + vm_alive "$FORCE_ATTACH" || { [ $? -ne 2 ] || die "UBI_ATTACH_UNKNOWN: could not check whether $FORCE_ATTACH still exists; nothing created, rerun"; die "attach: $FORCE_ATTACH no longer exists; rerun the original command to start fresh"; } attach=$FORCE_ATTACH else - attach=$(find_record) + attach=$(find_record) || die "UBI_ATTACH_UNKNOWN: a run record for this key names a VM the Ubicloud API could not check; nothing created (a new VM would leave that one billing), rerun" fi if [ "$probe" = 1 ]; then if [ -z "$attach" ]; then echo "UBI_ATTACH_INFO: none key=${RUN_KEY:0:12}" @@ -1750,6 +1815,7 @@ cmd_run() { trap 'exit 130' INT TERM QUIT trap on_hup HUP if [ -n "$attach" ]; then + UP_NAME=$name load "$name" log "reattached to $name created_at=$CREATE_AT key=${RUN_KEY:0:12}" if [ -n "$sink" ]; then @@ -1821,7 +1887,7 @@ cmd_run() { [ "$rc" -ne 0 ] || rc=1 continue fi - cmd_pull "$name" "${p%%:*}" "${p#*:}" || log "pull failed: $p" + cmd_pull "$name" "${p%%:*}" "${p#*:}" || { log "pull failed: $p"; [ "$rc" -ne 0 ] || rc=1; } done [ ${#TIER_NAMES[@]} -eq 0 ] || verify_line return "$rc" diff --git a/test/helpers/ubi-runner-fake-vm.ts b/test/helpers/ubi-runner-fake-vm.ts index 53db8146a..8899670bf 100644 --- a/test/helpers/ubi-runner-fake-vm.ts +++ b/test/helpers/ubi-runner-fake-vm.ts @@ -13,10 +13,14 @@ export const read = (rel: string) => readFileSync(join(ROOT, rel), 'utf8'); /** * Fake Ubicloud API that can create: create adds the name to the listed file * (or answers the quota refusal in refuse with HTTP 400), show is running at - * 192.0.2.1 while listed, destroy unlists. The fake VM is the directory vm/: + * 192.0.2.1 while listed, destroy unlists. Flag files beside listed change one + * answer: listed.create-status answers every create with that HTTP status, and + * with listed.create-lands (a count N) the VM still lands, listed from the Nth + * show on; listed.list-fails, listed.show-fails and listed.destroy-fails answer + * 503 or 500; listed.list-body is the list's 200 body. The fake VM is the directory vm/: * the fake ssh records every remote command in ssh.log, keeps copies of * ~/.ubirun-env and ~/.ubirun-setup.sh under the fixture root, serves pulls - * from remote-fs/, and runs every other command there for real (HOME=vm/, + * from remote-fs/ (a pull's existence check matches only what remote-fs/ holds), and runs every other command there for real (HOME=vm/, * vmbin/ first on PATH), so uploads unpack, detached phases write their pid, * output and exit files, tail --pid follows them and pkill ends them. The stub * make in vmbin/ echoes `ran: make ARGS`, or runs command-mode or @@ -37,10 +41,10 @@ export function fakeVm(opts: { listed?: string; refuse?: string } = {}) { 'loc=$(printf \'%s\' "$body" | sed -n \'s/.*"vm", *"\\([^/"]*\\)\\/[^"]*".*/\\1/p\')\n' + 'name=$(printf \'%s\' "$body" | sed -n \'s/.*"vm", *"[^/"]*\\/\\([^"]*\\)".*/\\1/p\')\n' + 'case "$body" in\n' + - ' *\'"create"\'*) if [ -n ' + JSON.stringify(opts.refuse ?? '') + ' ]; then printf \'%s\' ' + JSON.stringify(opts.refuse ?? '') + ' > "$out"; printf 400; else echo "$loc $name" >> "$listed"; : > "$out"; printf 200; fi ;;\n' + + ' *\'"create"\'*) if [ -f "$listed.create-status" ]; then : > "$out"; [ ! -f "$listed.create-lands" ] || echo "$loc $name" > "$listed.pending"; cat "$listed.create-status"; elif [ -n ' + JSON.stringify(opts.refuse ?? '') + ' ]; then printf \'%s\' ' + JSON.stringify(opts.refuse ?? '') + ' > "$out"; printf 400; else echo "$loc $name" >> "$listed"; : > "$out"; printf 200; fi ;;\n' + ' *destroy*) : > "$out"; if [ -f "$listed.destroy-fails" ]; then printf 500; else grep -v " $name$" "$listed" > "$listed.tmp"; mv "$listed.tmp" "$listed"; printf 200; fi ;;\n' + - ' *\'"list"\'*) if [ -f "$listed.list-fails" ]; then echo down > "$out"; printf 503; else cat "$listed" "$listed.ghost" > "$out" 2>/dev/null; printf 200; fi ;;\n' + - ' *\'"show"\'*) [ -z "${FAKE_SHOW_DELAY:-}" ] || sleep "$FAKE_SHOW_DELAY"; if [ -f "$listed.show-fails" ]; then echo down > "$out"; printf 503; elif grep -q " $name$" "$listed"; then printf \'state: running\\nsize: standard-16\\nip4: 192.0.2.1\\n\' > "$out"; printf 200; else : > "$out"; printf 404; fi ;;\n' + + ' *\'"list"\'*) if [ -f "$listed.list-fails" ]; then echo down > "$out"; printf 503; elif [ -f "$listed.list-body" ]; then cat "$listed.list-body" > "$out"; printf 200; else cat "$listed" "$listed.ghost" > "$out" 2>/dev/null; printf 200; fi ;;\n' + + ' *\'"show"\'*) [ -z "${FAKE_SHOW_DELAY:-}" ] || sleep "$FAKE_SHOW_DELAY"; if [ -f "$listed.pending" ]; then n=$(( $(cat "$listed.shows" 2>/dev/null || echo 0) + 1 )); echo "$n" > "$listed.shows"; if [ "$n" -ge "$(cat "$listed.create-lands")" ]; then cat "$listed.pending" >> "$listed"; rm -f "$listed.pending"; fi; fi; if [ -f "$listed.show-fails" ]; then echo down > "$out"; printf 503; elif grep -q " $name$" "$listed"; then printf \'state: running\\nsize: standard-16\\nip4: 192.0.2.1\\n\' > "$out"; printf 200; else : > "$out"; printf 404; fi ;;\n' + ' *) : > "$out"; printf 200 ;;\n' + 'esac\n'); writeFileSync(join(bin, 'ssh'), '#!/usr/bin/env bash\n' + @@ -55,7 +59,7 @@ export function fakeVm(opts: { listed?: string; refuse?: string } = {}) { ' *"cat > ~/.ubirun-env"*) tee "$root/ubirun-env" > "$vm/.ubirun-env" ;;\n' + ' *"cat > ~/.ubirun-setup.sh"*) tee "$root/ubirun-setup.sh" > "$vm/.ubirun-setup.sh" ;;\n' + ' *"tar -czf -"*) base=${cmd##* }; tar -czf - -C "$root/remote-fs" "$base" ;;\n' + - ' *"ls -d -- "*) : ;;\n' + + ' *"ls -d -- "*) g=${cmd#*ls -d -- }; g=${g%% *}; cd "$root/remote-fs" 2>/dev/null && ls -d -- $g >/dev/null 2>&1 || exit 3 ;;\n' + ' *) exec bash -c "$cmd" ;;\n' + 'esac\n'); writeFileSync(join(vmbin, 'make'), '#!/usr/bin/env bash\n' + diff --git a/test/ubicloud-runner-recovery.test.ts b/test/ubicloud-runner-recovery.test.ts index a6c21d174..6e4e8ece9 100644 --- a/test/ubicloud-runner-recovery.test.ts +++ b/test/ubicloud-runner-recovery.test.ts @@ -314,3 +314,110 @@ describe('ubi-runner recovery bundle (GST-1220 b)', () => { expect(await a.exited).toBe(0); }, 90_000); }); + +describe('ubi-runner money path: an API error is never read as success (GST-1441)', () => { + const rd = (f: FakeVm) => join(f.root, 'rd'); + const bundle = (f: FakeVm, name: string) => join(rd(f), 'thrd1', name + '.bundle'); + const recorded = (f: FakeVm, name: string) => existsSync(f.record) && readFileSync(f.record, 'utf8').includes(name); + const flag = (f: FakeVm, file: string, body = '') => writeFileSync(join(f.root, 'listed.' + file), body); + const NAME = 'ubirun-thrd1-1000000000-aaaaaaaa-t3'; + + // Value: protects=a refused create never arms teardown, so up and run never destroy a VM another caller owns under the same name; fails_when=teardown runs down on a name whose create Ubicloud refused (409); why_new=every fake create succeeded or answered the quota 400 on a name nobody held; seam=none; no_claim=the real API's 409 body + test('a refused create (409, the name is taken) disarms teardown: the VM holding the name is never destroyed', () => { + const f = make(); + writeFileSync(join(f.root, 'listed'), 'eu-central-h1 ' + NAME + '\n'); + flag(f, 'create-status', '409'); + const repo = gitSrc(f); + for (const args of [['up', '-n', NAME, '-l', 'eu-central-h1'], ['run', '-n', NAME, '-l', 'eu-central-h1', '--src', repo, '--', 'true']]) { + const r = run(f, args, { UBI_RECOVERY_DIR: rd(f), UBI_REAP: '0' }); + expect(r.exitCode, out(r)).not.toBe(0); + expect(out(r)).toContain('create failed (HTTP 409); nothing was created'); + expect(f.calls()).not.toContain('"destroy"'); + expect(f.listed()).toContain(NAME); + expect(existsSync(join(f.state, NAME))).toBe(false); + expect(recorded(f, NAME)).toBe(false); + expect(existsSync(bundle(f, NAME))).toBe(false); + if (args[0] === 'run') expect(out(r)).toContain('UBICLOUD: left=0 owner=thrd1 vm=' + NAME); + } + }, 60_000); + + // Value: protects=down settles a VM as gone only on a readable list, so a caller settling a reserve on down's exit never leaves a live VM billing; fails_when=a failed or unreadable list counts as gone again and the bundle is deleted; why_new=down was only tested with a working list; seam=none; no_claim=the real API's error bodies + test('down with no local state: a failed or unreadable VM list is unknown, never gone; exit non-zero and the bundle stays', () => { + const f = make(); + writeFileSync(join(f.root, 'listed'), 'eu-central-h1 ' + NAME + '\n'); + mkdirSync(join(rd(f), 'thrd1'), { recursive: true }); + writeFileSync(bundle(f, NAME), 'sealed'); + for (const [file, body] of [['list-fails', ''], ['list-body', 'upstream error\n']]) { + flag(f, file, body); + const r = run(f, ['down', NAME], { UBI_RECOVERY_DIR: rd(f), UBI_DOWN_TIMEOUT: '2' }); + expect(r.exitCode, out(r)).not.toBe(0); + expect(out(r)).not.toContain('gone: '); + expect(existsSync(bundle(f, NAME))).toBe(true); + expect(f.listed()).toContain(NAME); + rmSync(join(f.root, 'listed.' + file)); + } + const ok = run(f, ['down', NAME], { UBI_RECOVERY_DIR: rd(f) }); + expect(ok.exitCode, out(ok)).toBe(0); + expect(existsSync(bundle(f, NAME))).toBe(false); + }, 60_000); + + // Value: protects=a 5xx or timed-out create is looked up by name, so a VM Ubicloud created anyway is adopted instead of orphaned until its TTL; fails_when=a 502/503/000 create is treated as a refusal and the first 404 settles it as gone; why_new=no fake create ever failed and landed; seam=none; no_claim=how late a real 5xx create appears + test('a 5xx or unanswered create that lands anyway is found by name and adopted, not orphaned', () => { + for (const status of ['503', '000']) { + const f = make(); + flag(f, 'create-status', status); + flag(f, 'create-lands', '2'); + const r = run(f, ['up'], { UBI_RECOVERY_DIR: rd(f), UBI_REAP: '0', UBI_CREATE_GRACE: '20' }); + expect(r.exitCode, out(r)).toBe(0); + const name = r.stdout.toString().trim(); + expect(out(r)).toContain('answered HTTP ' + status + '; looking it up by name'); + expect(f.listed()).toContain(name); + expect(recorded(f, name)).toBe(true); + expect(existsSync(bundle(f, name))).toBe(true); + expect(readFileSync(join(f.state, name, 'env'), 'utf8')).toContain('CREATED=yes'); + } + }, 60_000); + + // Value: protects=a create nobody can confirm either way keeps its record and bundle for the reaper and recovery; fails_when=an unconfirmable 5xx create is settled as refused and its bundle dropped; why_new=no fake create ever failed; seam=none; no_claim=real outage shapes + test('a 5xx create the lookup cannot settle is UBI_CREATE_UNKNOWN: record and bundle kept, exit non-zero, nothing destroyed', () => { + const f = make(); + flag(f, 'create-status', '502'); + flag(f, 'show-fails'); + const r = run(f, ['up'], { UBI_RECOVERY_DIR: rd(f), UBI_REAP: '0', UBI_CREATE_GRACE: '2', UBI_DOWN_TIMEOUT: '2' }); + expect(r.exitCode, out(r)).not.toBe(0); + expect(out(r)).toContain('UBI_CREATE_UNKNOWN'); + const name = vmDirs(f)[0]!; + expect(readFileSync(join(f.state, name, 'env'), 'utf8')).toContain('CREATED=unknown'); + expect(recorded(f, name)).toBe(true); + expect(existsSync(bundle(f, name))).toBe(true); + expect(f.calls()).not.toContain('"destroy"'); + }, 60_000); + + // Value: protects=gc reports a stale VM it could not destroy; fails_when=gc exits 0 after a failed destroy; why_new=gc's destroy failure was only logged; seam=none; no_claim=real destroy failures + test('gc exits non-zero when a stale VM of this owner could not be destroyed', () => { + const f = make(); + writeFileSync(join(f.root, 'listed'), 'eu-central-h1 ' + NAME + '\n'); + flag(f, 'destroy-fails'); + const r = run(f, ['gc', '1'], { UBI_DOWN_TIMEOUT: '2' }); + expect(r.exitCode, out(r)).not.toBe(0); + expect(out(r)).toContain('gc: 1 stale VMs could not be destroyed'); + }, 60_000); + + // Value: protects=a run record whose VM the API cannot check is never abandoned for a second VM; fails_when=an unanswered show reads as "VM gone" and run creates another VM beside the live one; why_new=reattach was only tested with a working show; seam=none; no_claim=real API outages + test('run: a recorded VM the API cannot check refuses with UBI_ATTACH_UNKNOWN instead of creating a second VM', async () => { + const f = make(); + const repo = gitSrc(f); + writeFileSync(join(f.root, 'make-test.sh'), 'echo begin; sleep 3\n'); + const args = ['run', '--src', repo, '--', 'make test']; + const a = f.spawn(args, { UBI_REAP: '0' }); + await until(() => vmDirs(f).some(d => existsSync(join(f.state, d, 'command.out')) && readFileSync(join(f.state, d, 'command.out'), 'utf8').includes('begin')), 30_000); + process.kill(a.pid, 'SIGKILL'); + await a.exited; + flag(f, 'show-fails'); + const b = run(f, args, { UBI_REAP: '0' }); + expect(b.exitCode, out(b)).not.toBe(0); + expect(out(b)).toContain('UBI_ATTACH_UNKNOWN'); + expect(creates(f)).toBe(1); + }, 90_000); +}); + diff --git a/test/ubicloud-runner.test.ts b/test/ubicloud-runner.test.ts index bd9fe4844..11f6872a6 100644 --- a/test/ubicloud-runner.test.ts +++ b/test/ubicloud-runner.test.ts @@ -86,6 +86,62 @@ describe('ubicloud free-suite runner', () => { } }); + // Value: protects=a pull that cannot reach the VM fails, so a caller never keeps a stale copy as fresh; fails_when=an ssh failure in the existence check reads as "nothing matches" and exits 0 again; why_new=only a reachable VM was tested; seam=none; no_claim=real ssh error codes beyond 255 + test('pull exits non-zero when ssh fails, and only a reachable VM with no match counts as nothing to pull (GST-1441)', () => { + const root = mkdtempSync(join(tmpdir(), 'ubi-pull-')); + try { + const bin = join(root, 'bin'), state = join(root, 'state'); + mkdirSync(join(state, 'fake-vm'), { recursive: true }); + writeFileSync(join(state, 'fake-vm/env'), 'IP=192.0.2.1\n'); + mkdirSync(bin); + writeFileSync(join(bin, 'ssh'), '#!/usr/bin/env bash\necho "ssh: connect to host 192.0.2.1 port 22: Connection timed out" >&2\nexit 255\n'); + chmodSync(join(bin, 'ssh'), 0o755); + const env = { ...process.env, PATH: `${bin}:${process.env.PATH}`, UBICLOUD_API_KEY: 'offline', UBI_RUNNER_STATE: state }; + const r = Bun.spawnSync(['bash', join(DIR, 'ubi-runner.sh'), 'pull', 'fake-vm', 'work/results', join(root, 'local')], { env, timeout: 10_000 }); + expect(r.exitCode).not.toBe(0); + expect(r.stderr.toString()).toContain('pull failed: could not check work/results on fake-vm (exit 255)'); + expect(r.stderr.toString()).not.toContain('nothing matches'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + // Value: protects=the Ubicloud key never appears in curl's argv (visible to every local user in ps) and the pinned curl still sends it; fails_when=the key goes back on the command line, or -H @file stops delivering it; why_new=the key's argv exposure was never checked; seam=none; no_claim=curl builds older than 7.55, which lack -H @file + test('the API key reaches curl as a header file, never in its argv, and real curl still sends it (GST-1441)', async () => { + const root = mkdtempSync(join(tmpdir(), 'ubi-key-')); + const key = 'sekrit-ubi-key-4711'; + try { + const bin = join(root, 'bin'), argv = join(root, 'argv.log'), hdr = join(root, 'hdr.log'); + mkdirSync(bin); + writeFileSync(join(bin, 'curl'), `#!/usr/bin/env bash +printf '%s\\n' "$*" >> ${JSON.stringify(argv)} +out=""; prev="" +for a in "$@"; do + [ "$prev" = -o ] && out=$a + [ "$prev" = -H ] && case $a in @*) cat "\${a#@}" >> ${JSON.stringify(hdr)} ;; esac + prev=$a +done +cat >/dev/null; printf '' > "$out"; printf 200 +`); + chmodSync(join(bin, 'curl'), 0o755); + const base = { ...process.env, UBICLOUD_API_KEY: key, UBI_RUNNER_STATE: join(root, 'state') }; + const faked = Bun.spawnSync(['bash', join(DIR, 'ubi-runner.sh'), 'cli', 'vm', 'list'], { env: { ...base, PATH: `${bin}:${process.env.PATH}` }, timeout: 10_000 }); + expect(faked.exitCode, faked.stderr.toString()).toBe(0); + expect(readFileSync(argv, 'utf8')).not.toContain(key); + expect(readFileSync(hdr, 'utf8')).toBe(`Authorization: Bearer ${key}\n`); + let seen = ''; + const server = Bun.serve({ port: 0, hostname: '127.0.0.1', fetch(req) { seen = req.headers.get('authorization') ?? ''; return new Response('eu-central-h1 vm-a\n'); } }); + try { + const real = Bun.spawn(['bash', join(DIR, 'ubi-runner.sh'), 'cli', 'vm', 'list'], { env: { ...base, UBICLOUD_API_URL: `http://127.0.0.1:${server.port}` }, stdout: 'pipe', stderr: 'pipe', timeout: 20_000 }); + expect(await real.exited).toBe(0); + expect(await new Response(real.stdout).text()).toBe('eu-central-h1 vm-a\n'); + expect(seen).toBe(`Bearer ${key}`); + } finally { server.stop(true); } + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + test('refuses to start without UBICLOUD_API_KEY, before any network call', () => { const env = { ...process.env, UBICLOUD_API_URL: 'http://127.0.0.1:9' } as Record; delete env.UBICLOUD_API_KEY; @@ -461,7 +517,7 @@ test('pack ships a linked worktree with a self-contained .git on the checked-out test('up records the VM in the scratch file before the create request is sent', () => { const runner = read('scripts/ubicloud/ubi-runner.sh'); - const create = runner.indexOf('cli "${args[@]}" >/dev/null 2>"$dir/create.err"'); + const create = runner.indexOf('code=$(api "$dir/create.err" "${args[@]}")'); expect(create).toBeGreaterThan(0); expect(runner.indexOf('record_vm "$name" "$location"')).toBeLessThan(create); expect(runner).toContain('$HOME/.capy/work/ubi-runner/vms.tsv');