fix(ubi-runner): an API error is never read as success on the money path (GST-1441)

A refused create disarms teardown, a 5xx/unanswered create is looked up by
name, down counts a VM gone only on a readable list, pull fails on ssh
failure, gc reports failed destroys, an unanswerable run record refuses
instead of starting a second VM, and the API key leaves curl's argv.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

changelog.d/gst-1441-ubi-runner-money-path.md

@@ -0,0 +1 @@
+- **The Ubicloud runner no longer reads an API error as success.** A create Ubicloud refuses (a 409 name collision, any 4xx) now disarms teardown, so `up` and `run` never destroy a VM someone else holds under that name. A 5xx or unanswered create is looked up by name before anything is decided: a VM that landed anyway is adopted, and one nobody can confirm either way keeps its record and recovery bundle and exits `UBI_CREATE_UNKNOWN`. `down` counts a VM as gone only on a readable answer, so a failed or garbled VM list exits non-zero and keeps the bundle. `pull` exits non-zero when ssh fails, and `run` fails when one of its pulls does. `gc` exits non-zero when a stale VM could not be destroyed, a run record whose VM the API cannot check refuses with `UBI_ATTACH_UNKNOWN` instead of starting a second VM, and the API key reaches curl as a header on a pipe, out of `ps`. (GST-1441)

scripts/ubicloud/ubi-runner.sh

@@ -35,7 +35,15 @@
 # confirmed gone or provably never existed, waiting out a create that may still
 # be in flight. An interrupted `up` or `run` (EXIT, INT, TERM, QUIT, and HUP
 # until a remote phase is live) runs that `down`; further signals are ignored
-# until it finishes. A create refused
+# until it finishes. Teardown is armed only once this call's create request is
+# sent, and disarmed when Ubicloud refuses it (HTTP 4xx, a 409 name collision
+# included), so it never destroys a VM this call did not create. A 5xx or no
+# answer is not a refusal: `up` looks the VM up by name for UBI_CREATE_GRACE
+# seconds, and when that cannot settle it either, it keeps the record and the
+# recovery bundle and exits UBI_CREATE_UNKNOWN. `down` counts a VM as gone only
+# on a readable answer: a failed or unreadable VM list or show is unknown, so
+# down exits non-zero and keeps the recovery bundle. The API key reaches curl
+# as a header file on a pipe, never in its argv. A create refused
 # for vCPU quota prints the project's usage per owner.
 #
 # The owner-age sweep is off by default (the TTL reap below always runs). With UBI_GC_HOURS set to a positive
@@ -225,7 +233,7 @@ api() {
   shift
   code=$(python3 -c 'import json,sys; print(json.dumps({"argv": sys.argv[1:]}))' "$@" \
     | curl -sS -o "$out" -w '%{http_code}' --max-time "$API_TIMEOUT" -X POST \
-        -H "Authorization: Bearer $UBICLOUD_API_KEY" \
+        -H @<(printf 'Authorization: Bearer %s\n' "$UBICLOUD_API_KEY") \
         -H 'Accept: text/plain' -H 'Content-Type: application/json' \
         -H 'X-Ubi-Version: 1.0.0' --data @- "$API/cli") || true
   echo "${code:-000}"
@@ -552,6 +560,8 @@ lease_busy() {
 
 # vm_alive NAME: the recorded VM still answers show. One confirmed gone is
 # cleaned up (state dir and scratch record), so its record is never tried again.
+# Returns 2 when the API gives no answer either way: the caller must not start a
+# second VM beside one that may still be running.
 vm_alive() {
   local n=$1 out code
   ( load "$n"; [ "$CREATED" = yes ] && [ -n "$IP" ] ) 2>/dev/null || return 1
@@ -561,7 +571,7 @@ vm_alive() {
   case $code in
     200) return 0 ;;
     404) log "run record for $n: the VM is gone; starting a new one"; cmd_down "$n" >/dev/null 2>&1 || true; return 1 ;;
-    *) return 1 ;;
+    *) log "run record for $n: could not check the VM (HTTP $code)"; return 2 ;;
   esac
 }
 
@@ -603,7 +613,8 @@ find_record() {
     [ -f "$f" ] || continue
     grep -qx "RUN_KEY=$RUN_KEY" "$f" || continue
     n=$(basename "$(dirname "$f")")
-    if vm_alive "$n"; then echo "$n"; return 0; fi
+    vm_alive "$n" && { echo "$n"; return 0; }
+    [ $? -ne 2 ] || return 2
   done
 }
 
@@ -775,8 +786,13 @@ run_tiers() {
 # did not name; expires is the epoch the name's -t<hours> TTL ends at, or "-";
 # created is the name's epoch, or "-".
 vm_rows() {
-  local loc name owner created expires
+  local loc name owner created expires list
-  cli vm list -N -f location,name | while read -r loc name; do
+  list=$(cli vm list -N -f location,name) || return 1
+  if grep -Evq '^[[:blank:]]*([a-z0-9-]+[[:blank:]]+[a-z0-9-]+)?[[:blank:]]*$' <<<"$list"; then
+    log "the VM list answer is unreadable (a line is not 'location name')"
+    return 1
+  fi
+  while read -r loc name; do
     [ -n "$name" ] || continue
     owner=. created="" expires=-
     if [[ "$name" =~ ^${PREFIX}-([a-z][a-z0-9]{0,11})-([0-9]{10})- ]]; then
@@ -788,7 +804,7 @@ vm_rows() {
       expires=$(( 10#$created + 10#${BASH_REMATCH[1]} * 3600 ))
     fi
     echo "$loc $name $owner $expires ${created:--}"
-  done
+  done <<<"$list"
 }
 
 # reap: destroy the caller's own VMs whose TTL ended more than REAP_GRACE
@@ -823,18 +839,20 @@ cmd_reap() {
 }
 
 cmd_gc() {
-  local hours=${1:-$GC_HOURS} now loc name owner
+  local hours=${1:-$GC_HOURS} now rows loc name owner failed=0
   [[ "$hours" =~ ^[0-9]+$ ]] || die "gc: HOURS must be a whole number (or set UBI_GC_HOURS)"
   if [ "$hours" -eq 0 ]; then log "gc: no sweep (give HOURS > 0 or set UBI_GC_HOURS); nothing listed, nothing destroyed"; return 0; fi
   now=$(date +%s)
-  vm_rows | while read -r loc name owner _; do
+  rows=$(vm_rows) || { log "gc: could not list the project's VMs; nothing destroyed"; return 1; }
+  while read -r loc name owner _; do
     [ "$owner" = "$OWNER" ] || continue
     [[ "$name" =~ -([0-9]{10})- ]] || continue
     if [ $(( (now - BASH_REMATCH[1]) / 3600 )) -ge "$hours" ]; then
       log "destroying stale $loc/$name (owner $OWNER, older than ${hours}h)"
-      cmd_down "$name" -l "$loc" || log "failed to destroy $loc/$name"
+      cmd_down "$name" -l "$loc" || { failed=$(( failed + 1 )); log "failed to destroy $loc/$name"; }
     fi
-  done
+  done <<<"$rows"
+  [ "$failed" -eq 0 ] || { log "gc: $failed stale VMs could not be destroyed"; return 1; }
 }
 
 cmd_list() {
@@ -1296,25 +1314,63 @@ cmd_up() {
   [ -z "$storage" ] || args+=(-S "$storage")
   args+=("$(cat "$dir/key.pub")")
   create_at=$(date +%s)
+  log "creating $location/$name ($size)"
+  # Signals are held from before the create request until its answer is
+  # recorded. Teardown is armed (UP_NAME) only once the request is on its way
+  # and disarmed when Ubicloud refuses it (HTTP 4xx): a refused create made
+  # nothing, and a VM that answers to the name (a 409) is not this call's.
+  local interrupted="" created code err refusal need used max
+  trap 'interrupted=1' INT TERM HUP QUIT
   write_state "$name" "$location" "$size" "" unknown "$create_at"
   bundle_save "$name"
   record_vm "$name" "$location"
   UP_NAME=$name
-  log "creating $location/$name ($size)"
+  code=$(api "$dir/create.err" "${args[@]}")
-  # Hold signals until the create answer is recorded, so teardown knows
+  case $code in
-  # whether the VM exists instead of waiting out UBI_CREATE_GRACE.
+    200) created=yes ;;
-  local interrupted="" created=yes refusal need used max
+    4[0-9][0-9]) created=no ;;
-  trap 'interrupted=1' INT TERM HUP QUIT
+    *) created=unknown ;;
-  cli "${args[@]}" >/dev/null 2>"$dir/create.err" || created=no
+  esac
-  write_state "$name" "$location" "$size" "" "$created" "$create_at"
+  if [ "$created" = no ]; then
-  [ "$created" = no ] || bundle_save "$name"
+    UP_NAME=""
+    err=$(cat "$dir/create.err")
+    rm -rf "$dir"
+    forget_vm "$name"
+    [ -z "$RECOVERY_DIR" ] || rm -f "$RECOVERY_DIR/$OWNER/$name.bundle"
+  else
+    write_state "$name" "$location" "$size" "" "$created" "$create_at"
+    [ "$created" = unknown ] || bundle_save "$name"
+  fi
   trap 'exit 130' INT TERM HUP QUIT
   [ -z "$interrupted" ] || exit 130
+  if [ "$created" = unknown ]; then
+    # A 5xx or no answer (000) is not a refusal: the VM may exist. Look it up
+    # by name until it shows or UBI_CREATE_GRACE has passed.
+    log "create of $location/$name answered HTTP $code; looking it up by name before deciding"
+    local seen_code
+    while :; do
+      seen_code=$(api "$dir/lookup.out" vm "$location/$name" show)
+      [ "$seen_code" != 200 ] || break
+      [ $(( $(date +%s) - create_at )) -lt "$CREATE_GRACE" ] || break
+      sleep "$POLL"
+    done
+    rm -f "$dir/lookup.out"
+    case $seen_code in
+      200) created=yes ;;
+      404) created=no ;;
+    esac
+    write_state "$name" "$location" "$size" "" "$created" "$create_at"
+    case $created in
+      yes) bundle_save "$name"; log "create of $location/$name went through; continuing" ;;
+      no) cat "$dir/create.err" >&2; die "create failed (HTTP $code): $location/$name was still absent ${CREATE_GRACE}s after the request" ;;
+      unknown) die "UBI_CREATE_UNKNOWN: could not tell whether $location/$name exists (create answered HTTP $code, its lookup HTTP $seen_code); its record and recovery bundle stay, teardown destroys it if it shows up, and this owner's reap destroys it once its -t<hours> TTL ends; check with: $0 down $name" ;;
+    esac
+  fi
   if [ "$created" = no ]; then
-    refusal=$(sed -n 's/.*Requested vCPU count: \([0-9]*\), currently used vCPU count: \([0-9]*\), maximum allowed vCPU count: \([0-9]*\).*/\1 \2 \3/p' "$dir/create.err")
+    refusal=$(sed -n 's/.*Requested vCPU count: \([0-9]*\), currently used vCPU count: \([0-9]*\), maximum allowed vCPU count: \([0-9]*\).*/\1 \2 \3/p' <<<"$err")
     if [ -z "$refusal" ]; then
-      cat "$dir/create.err" >&2
+      printf '%s\n' "$err" >&2
-      die "create failed"
+      die "create failed (HTTP $code); nothing was created"
     fi
     read -r need used max <<<"$refusal"
     echo "$max" >"$STATE_ROOT/quota-max"
@@ -1380,14 +1436,20 @@ cmd_down() {
     fi
   fi
 
-  local deadline=$(( $(date +%s) + DOWN_TIMEOUT )) seen=0 asked=0 out code now
+  local deadline=$(( $(date +%s) + DOWN_TIMEOUT )) seen=0 asked=0 out code now rows unlisted=0
   out=$(mktemp)
   while :; do
     now=$(date +%s)
     if [ -z "$loc" ]; then
-      loc=$(vm_rows 2>/dev/null | awk -v n="$name" '$2==n {print $1}') || loc=""
+      if rows=$(vm_rows 2>/dev/null); then
-      code=404
+        loc=$(awk -v n="$name" '$2==n {print $1}' <<<"$rows")
-      [ -z "$loc" ] || code=$(api "$out" vm "$loc/$name" show)
+        code=404
+        [ -z "$loc" ] || code=$(api "$out" vm "$loc/$name" show)
+      else
+        code=unlisted
+        [ "$unlisted" = 1 ] || log "could not read the project's VM list; $name is not counted as gone"
+        unlisted=1
+      fi
     else
       code=$(api "$out" vm "$loc/$name" show)
     fi
@@ -1541,12 +1603,14 @@ cmd_sync() {
 # Relative remote paths start at /home/ubi. A glob that matches nothing (for
 # example an optional flake ledger) is reported and skipped, not a failure.
 cmd_pull() {
-  local name=$1 from=$2 to=$3 dir
+  local name=$1 from=$2 to=$3 dir rc=0
   dir=$(printf %q "$(dirname "$from")")
-  if ! remote "$name" "cd $dir 2>/dev/null && ls -d -- $(basename "$from") >/dev/null 2>&1"; then
+  remote "$name" "cd $dir 2>/dev/null || exit 3; ls -d -- $(basename "$from") >/dev/null 2>&1 || exit 3" || rc=$?
-    log "pull: nothing matches $from"
+  case $rc in
-    return 0
+    0) ;;
-  fi
+    3) log "pull: nothing matches $from"; return 0 ;;
+    *) log "pull failed: could not check $from on $name (exit $rc)"; return 1 ;;
+  esac
   mkdir -p "$to"
   remote "$name" "cd $dir && tar -czf - $(basename "$from")" | tar -xzf - -C "$to"
 }
@@ -1602,13 +1666,14 @@ PULL_ROOT=""
 run_teardown() {
   local rc=$1 rows left others
   trap "" INT TERM HUP QUIT
-  cmd_down "$RUN_VM" || log "WARNING: failed to destroy $RUN_VM; run: $0 down $RUN_VM"
+  [ -z "$UP_NAME" ] || cmd_down "$RUN_VM" || log "WARNING: failed to destroy $RUN_VM; run: $0 down $RUN_VM"
   if rows=$(vm_rows 2>/dev/null); then
     left=$(awk -v n="$RUN_VM" '$2==n' <<<"$rows" | grep -c . || true)
     others=$(awk -v n="$RUN_VM" -v o="$OWNER" '$3==o && $2!=n' <<<"$rows" | grep -c . || true)
   else
     left=unknown others=unknown
   fi
+  [ -n "$UP_NAME" ] || left=0
   leftover "left=$left owner=$OWNER vm=$RUN_VM others=$others"
   if [ "$left" = 0 ]; then
     [[ "$SINK_FLUSH_FAILED" != *" $RUN_VM "* ]] || [ "$rc" -ne 0 ] || exit 1
@@ -1691,10 +1756,10 @@ cmd_run() {
 
   local attach=""
   if [ -n "$FORCE_ATTACH" ]; then
-    vm_alive "$FORCE_ATTACH" || die "attach: $FORCE_ATTACH no longer exists; rerun the original command to start fresh"
+    vm_alive "$FORCE_ATTACH" || { [ $? -ne 2 ] || die "UBI_ATTACH_UNKNOWN: could not check whether $FORCE_ATTACH still exists; nothing created, rerun"; die "attach: $FORCE_ATTACH no longer exists; rerun the original command to start fresh"; }
     attach=$FORCE_ATTACH
   else
-    attach=$(find_record)
+    attach=$(find_record) || die "UBI_ATTACH_UNKNOWN: a run record for this key names a VM the Ubicloud API could not check; nothing created (a new VM would leave that one billing), rerun"
   fi
   if [ "$probe" = 1 ]; then
     if [ -z "$attach" ]; then echo "UBI_ATTACH_INFO: none key=${RUN_KEY:0:12}"
@@ -1750,6 +1815,7 @@ cmd_run() {
   trap 'exit 130' INT TERM QUIT
   trap on_hup HUP
   if [ -n "$attach" ]; then
+    UP_NAME=$name
     load "$name"
     log "reattached to $name created_at=$CREATE_AT key=${RUN_KEY:0:12}"
     if [ -n "$sink" ]; then
@@ -1821,7 +1887,7 @@ cmd_run() {
       [ "$rc" -ne 0 ] || rc=1
       continue
     fi
-    cmd_pull "$name" "${p%%:*}" "${p#*:}" || log "pull failed: $p"
+    cmd_pull "$name" "${p%%:*}" "${p#*:}" || { log "pull failed: $p"; [ "$rc" -ne 0 ] || rc=1; }
   done
   [ ${#TIER_NAMES[@]} -eq 0 ] || verify_line
   return "$rc"

test/helpers/ubi-runner-fake-vm.ts

@@ -13,10 +13,14 @@ export const read = (rel: string) => readFileSync(join(ROOT, rel), 'utf8');
 /**
  * Fake Ubicloud API that can create: create adds the name to the listed file
  * (or answers the quota refusal in refuse with HTTP 400), show is running at
- * 192.0.2.1 while listed, destroy unlists. The fake VM is the directory vm/:
+ * 192.0.2.1 while listed, destroy unlists. Flag files beside listed change one
+ * answer: listed.create-status answers every create with that HTTP status, and
+ * with listed.create-lands (a count N) the VM still lands, listed from the Nth
+ * show on; listed.list-fails, listed.show-fails and listed.destroy-fails answer
+ * 503 or 500; listed.list-body is the list's 200 body. The fake VM is the directory vm/:
  * the fake ssh records every remote command in ssh.log, keeps copies of
  * ~/.ubirun-env and ~/.ubirun-setup.sh under the fixture root, serves pulls
- * from remote-fs/, and runs every other command there for real (HOME=vm/,
+ * from remote-fs/ (a pull's existence check matches only what remote-fs/ holds), and runs every other command there for real (HOME=vm/,
  * vmbin/ first on PATH), so uploads unpack, detached phases write their pid,
  * output and exit files, tail --pid follows them and pkill ends them. The stub
  * make in vmbin/ echoes `ran: make ARGS`, or runs command-mode or
@@ -37,10 +41,10 @@ export function fakeVm(opts: { listed?: string; refuse?: string } = {}) {
     'loc=$(printf \'%s\' "$body" | sed -n \'s/.*"vm", *"\\([^/"]*\\)\\/[^"]*".*/\\1/p\')\n' +
     'name=$(printf \'%s\' "$body" | sed -n \'s/.*"vm", *"[^/"]*\\/\\([^"]*\\)".*/\\1/p\')\n' +
     'case "$body" in\n' +
-    '  *\'"create"\'*) if [ -n ' + JSON.stringify(opts.refuse ?? '') + ' ]; then printf \'%s\' ' + JSON.stringify(opts.refuse ?? '') + ' > "$out"; printf 400; else echo "$loc $name" >> "$listed"; : > "$out"; printf 200; fi ;;\n' +
+    '  *\'"create"\'*) if [ -f "$listed.create-status" ]; then : > "$out"; [ ! -f "$listed.create-lands" ] || echo "$loc $name" > "$listed.pending"; cat "$listed.create-status"; elif [ -n ' + JSON.stringify(opts.refuse ?? '') + ' ]; then printf \'%s\' ' + JSON.stringify(opts.refuse ?? '') + ' > "$out"; printf 400; else echo "$loc $name" >> "$listed"; : > "$out"; printf 200; fi ;;\n' +
     '  *destroy*) : > "$out"; if [ -f "$listed.destroy-fails" ]; then printf 500; else grep -v " $name$" "$listed" > "$listed.tmp"; mv "$listed.tmp" "$listed"; printf 200; fi ;;\n' +
-    '  *\'"list"\'*) if [ -f "$listed.list-fails" ]; then echo down > "$out"; printf 503; else cat "$listed" "$listed.ghost" > "$out" 2>/dev/null; printf 200; fi ;;\n' +
+    '  *\'"list"\'*) if [ -f "$listed.list-fails" ]; then echo down > "$out"; printf 503; elif [ -f "$listed.list-body" ]; then cat "$listed.list-body" > "$out"; printf 200; else cat "$listed" "$listed.ghost" > "$out" 2>/dev/null; printf 200; fi ;;\n' +
-    '  *\'"show"\'*) [ -z "${FAKE_SHOW_DELAY:-}" ] || sleep "$FAKE_SHOW_DELAY"; if [ -f "$listed.show-fails" ]; then echo down > "$out"; printf 503; elif grep -q " $name$" "$listed"; then printf \'state: running\\nsize: standard-16\\nip4: 192.0.2.1\\n\' > "$out"; printf 200; else : > "$out"; printf 404; fi ;;\n' +
+    '  *\'"show"\'*) [ -z "${FAKE_SHOW_DELAY:-}" ] || sleep "$FAKE_SHOW_DELAY"; if [ -f "$listed.pending" ]; then n=$(( $(cat "$listed.shows" 2>/dev/null || echo 0) + 1 )); echo "$n" > "$listed.shows"; if [ "$n" -ge "$(cat "$listed.create-lands")" ]; then cat "$listed.pending" >> "$listed"; rm -f "$listed.pending"; fi; fi; if [ -f "$listed.show-fails" ]; then echo down > "$out"; printf 503; elif grep -q " $name$" "$listed"; then printf \'state: running\\nsize: standard-16\\nip4: 192.0.2.1\\n\' > "$out"; printf 200; else : > "$out"; printf 404; fi ;;\n' +
     '  *) : > "$out"; printf 200 ;;\n' +
     'esac\n');
   writeFileSync(join(bin, 'ssh'), '#!/usr/bin/env bash\n' +
@@ -55,7 +59,7 @@ export function fakeVm(opts: { listed?: string; refuse?: string } = {}) {
     '  *"cat > ~/.ubirun-env"*) tee "$root/ubirun-env" > "$vm/.ubirun-env" ;;\n' +
     '  *"cat > ~/.ubirun-setup.sh"*) tee "$root/ubirun-setup.sh" > "$vm/.ubirun-setup.sh" ;;\n' +
     '  *"tar -czf -"*) base=${cmd##* }; tar -czf - -C "$root/remote-fs" "$base" ;;\n' +
-    '  *"ls -d -- "*) : ;;\n' +
+    '  *"ls -d -- "*) g=${cmd#*ls -d -- }; g=${g%% *}; cd "$root/remote-fs" 2>/dev/null && ls -d -- $g >/dev/null 2>&1 || exit 3 ;;\n' +
     '  *) exec bash -c "$cmd" ;;\n' +
     'esac\n');
   writeFileSync(join(vmbin, 'make'), '#!/usr/bin/env bash\n' +

test/ubicloud-runner-recovery.test.ts

@@ -314,3 +314,110 @@ describe('ubi-runner recovery bundle (GST-1220 b)', () => {
     expect(await a.exited).toBe(0);
   }, 90_000);
 });
+
+describe('ubi-runner money path: an API error is never read as success (GST-1441)', () => {
+  const rd = (f: FakeVm) => join(f.root, 'rd');
+  const bundle = (f: FakeVm, name: string) => join(rd(f), 'thrd1', name + '.bundle');
+  const recorded = (f: FakeVm, name: string) => existsSync(f.record) && readFileSync(f.record, 'utf8').includes(name);
+  const flag = (f: FakeVm, file: string, body = '') => writeFileSync(join(f.root, 'listed.' + file), body);
+  const NAME = 'ubirun-thrd1-1000000000-aaaaaaaa-t3';
+
+  // Value: protects=a refused create never arms teardown, so up and run never destroy a VM another caller owns under the same name; fails_when=teardown runs down on a name whose create Ubicloud refused (409); why_new=every fake create succeeded or answered the quota 400 on a name nobody held; seam=none; no_claim=the real API's 409 body
+  test('a refused create (409, the name is taken) disarms teardown: the VM holding the name is never destroyed', () => {
+    const f = make();
+    writeFileSync(join(f.root, 'listed'), 'eu-central-h1 ' + NAME + '\n');
+    flag(f, 'create-status', '409');
+    const repo = gitSrc(f);
+    for (const args of [['up', '-n', NAME, '-l', 'eu-central-h1'], ['run', '-n', NAME, '-l', 'eu-central-h1', '--src', repo, '--', 'true']]) {
+      const r = run(f, args, { UBI_RECOVERY_DIR: rd(f), UBI_REAP: '0' });
+      expect(r.exitCode, out(r)).not.toBe(0);
+      expect(out(r)).toContain('create failed (HTTP 409); nothing was created');
+      expect(f.calls()).not.toContain('"destroy"');
+      expect(f.listed()).toContain(NAME);
+      expect(existsSync(join(f.state, NAME))).toBe(false);
+      expect(recorded(f, NAME)).toBe(false);
+      expect(existsSync(bundle(f, NAME))).toBe(false);
+      if (args[0] === 'run') expect(out(r)).toContain('UBICLOUD: left=0 owner=thrd1 vm=' + NAME);
+    }
+  }, 60_000);
+
+  // Value: protects=down settles a VM as gone only on a readable list, so a caller settling a reserve on down's exit never leaves a live VM billing; fails_when=a failed or unreadable list counts as gone again and the bundle is deleted; why_new=down was only tested with a working list; seam=none; no_claim=the real API's error bodies
+  test('down with no local state: a failed or unreadable VM list is unknown, never gone; exit non-zero and the bundle stays', () => {
+    const f = make();
+    writeFileSync(join(f.root, 'listed'), 'eu-central-h1 ' + NAME + '\n');
+    mkdirSync(join(rd(f), 'thrd1'), { recursive: true });
+    writeFileSync(bundle(f, NAME), 'sealed');
+    for (const [file, body] of [['list-fails', ''], ['list-body', '<html>upstream error</html>\n']]) {
+      flag(f, file, body);
+      const r = run(f, ['down', NAME], { UBI_RECOVERY_DIR: rd(f), UBI_DOWN_TIMEOUT: '2' });
+      expect(r.exitCode, out(r)).not.toBe(0);
+      expect(out(r)).not.toContain('gone: ');
+      expect(existsSync(bundle(f, NAME))).toBe(true);
+      expect(f.listed()).toContain(NAME);
+      rmSync(join(f.root, 'listed.' + file));
+    }
+    const ok = run(f, ['down', NAME], { UBI_RECOVERY_DIR: rd(f) });
+    expect(ok.exitCode, out(ok)).toBe(0);
+    expect(existsSync(bundle(f, NAME))).toBe(false);
+  }, 60_000);
+
+  // Value: protects=a 5xx or timed-out create is looked up by name, so a VM Ubicloud created anyway is adopted instead of orphaned until its TTL; fails_when=a 502/503/000 create is treated as a refusal and the first 404 settles it as gone; why_new=no fake create ever failed and landed; seam=none; no_claim=how late a real 5xx create appears
+  test('a 5xx or unanswered create that lands anyway is found by name and adopted, not orphaned', () => {
+    for (const status of ['503', '000']) {
+      const f = make();
+      flag(f, 'create-status', status);
+      flag(f, 'create-lands', '2');
+      const r = run(f, ['up'], { UBI_RECOVERY_DIR: rd(f), UBI_REAP: '0', UBI_CREATE_GRACE: '20' });
+      expect(r.exitCode, out(r)).toBe(0);
+      const name = r.stdout.toString().trim();
+      expect(out(r)).toContain('answered HTTP ' + status + '; looking it up by name');
+      expect(f.listed()).toContain(name);
+      expect(recorded(f, name)).toBe(true);
+      expect(existsSync(bundle(f, name))).toBe(true);
+      expect(readFileSync(join(f.state, name, 'env'), 'utf8')).toContain('CREATED=yes');
+    }
+  }, 60_000);
+
+  // Value: protects=a create nobody can confirm either way keeps its record and bundle for the reaper and recovery; fails_when=an unconfirmable 5xx create is settled as refused and its bundle dropped; why_new=no fake create ever failed; seam=none; no_claim=real outage shapes
+  test('a 5xx create the lookup cannot settle is UBI_CREATE_UNKNOWN: record and bundle kept, exit non-zero, nothing destroyed', () => {
+    const f = make();
+    flag(f, 'create-status', '502');
+    flag(f, 'show-fails');
+    const r = run(f, ['up'], { UBI_RECOVERY_DIR: rd(f), UBI_REAP: '0', UBI_CREATE_GRACE: '2', UBI_DOWN_TIMEOUT: '2' });
+    expect(r.exitCode, out(r)).not.toBe(0);
+    expect(out(r)).toContain('UBI_CREATE_UNKNOWN');
+    const name = vmDirs(f)[0]!;
+    expect(readFileSync(join(f.state, name, 'env'), 'utf8')).toContain('CREATED=unknown');
+    expect(recorded(f, name)).toBe(true);
+    expect(existsSync(bundle(f, name))).toBe(true);
+    expect(f.calls()).not.toContain('"destroy"');
+  }, 60_000);
+
+  // Value: protects=gc reports a stale VM it could not destroy; fails_when=gc exits 0 after a failed destroy; why_new=gc's destroy failure was only logged; seam=none; no_claim=real destroy failures
+  test('gc exits non-zero when a stale VM of this owner could not be destroyed', () => {
+    const f = make();
+    writeFileSync(join(f.root, 'listed'), 'eu-central-h1 ' + NAME + '\n');
+    flag(f, 'destroy-fails');
+    const r = run(f, ['gc', '1'], { UBI_DOWN_TIMEOUT: '2' });
+    expect(r.exitCode, out(r)).not.toBe(0);
+    expect(out(r)).toContain('gc: 1 stale VMs could not be destroyed');
+  }, 60_000);
+
+  // Value: protects=a run record whose VM the API cannot check is never abandoned for a second VM; fails_when=an unanswered show reads as "VM gone" and run creates another VM beside the live one; why_new=reattach was only tested with a working show; seam=none; no_claim=real API outages
+  test('run: a recorded VM the API cannot check refuses with UBI_ATTACH_UNKNOWN instead of creating a second VM', async () => {
+    const f = make();
+    const repo = gitSrc(f);
+    writeFileSync(join(f.root, 'make-test.sh'), 'echo begin; sleep 3\n');
+    const args = ['run', '--src', repo, '--', 'make test'];
+    const a = f.spawn(args, { UBI_REAP: '0' });
+    await until(() => vmDirs(f).some(d => existsSync(join(f.state, d, 'command.out')) && readFileSync(join(f.state, d, 'command.out'), 'utf8').includes('begin')), 30_000);
+    process.kill(a.pid, 'SIGKILL');
+    await a.exited;
+    flag(f, 'show-fails');
+    const b = run(f, args, { UBI_REAP: '0' });
+    expect(b.exitCode, out(b)).not.toBe(0);
+    expect(out(b)).toContain('UBI_ATTACH_UNKNOWN');
+    expect(creates(f)).toBe(1);
+  }, 90_000);
+});
+

test/ubicloud-runner.test.ts

@@ -86,6 +86,62 @@ describe('ubicloud free-suite runner', () => {
     }
   });
 
+  // Value: protects=a pull that cannot reach the VM fails, so a caller never keeps a stale copy as fresh; fails_when=an ssh failure in the existence check reads as "nothing matches" and exits 0 again; why_new=only a reachable VM was tested; seam=none; no_claim=real ssh error codes beyond 255
+  test('pull exits non-zero when ssh fails, and only a reachable VM with no match counts as nothing to pull (GST-1441)', () => {
+    const root = mkdtempSync(join(tmpdir(), 'ubi-pull-'));
+    try {
+      const bin = join(root, 'bin'), state = join(root, 'state');
+      mkdirSync(join(state, 'fake-vm'), { recursive: true });
+      writeFileSync(join(state, 'fake-vm/env'), 'IP=192.0.2.1\n');
+      mkdirSync(bin);
+      writeFileSync(join(bin, 'ssh'), '#!/usr/bin/env bash\necho "ssh: connect to host 192.0.2.1 port 22: Connection timed out" >&2\nexit 255\n');
+      chmodSync(join(bin, 'ssh'), 0o755);
+      const env = { ...process.env, PATH: `${bin}:${process.env.PATH}`, UBICLOUD_API_KEY: 'offline', UBI_RUNNER_STATE: state };
+      const r = Bun.spawnSync(['bash', join(DIR, 'ubi-runner.sh'), 'pull', 'fake-vm', 'work/results', join(root, 'local')], { env, timeout: 10_000 });
+      expect(r.exitCode).not.toBe(0);
+      expect(r.stderr.toString()).toContain('pull failed: could not check work/results on fake-vm (exit 255)');
+      expect(r.stderr.toString()).not.toContain('nothing matches');
+    } finally {
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+
+  // Value: protects=the Ubicloud key never appears in curl's argv (visible to every local user in ps) and the pinned curl still sends it; fails_when=the key goes back on the command line, or -H @file stops delivering it; why_new=the key's argv exposure was never checked; seam=none; no_claim=curl builds older than 7.55, which lack -H @file
+  test('the API key reaches curl as a header file, never in its argv, and real curl still sends it (GST-1441)', async () => {
+    const root = mkdtempSync(join(tmpdir(), 'ubi-key-'));
+    const key = 'sekrit-ubi-key-4711';
+    try {
+      const bin = join(root, 'bin'), argv = join(root, 'argv.log'), hdr = join(root, 'hdr.log');
+      mkdirSync(bin);
+      writeFileSync(join(bin, 'curl'), `#!/usr/bin/env bash
+printf '%s\\n' "$*" >> ${JSON.stringify(argv)}
+out=""; prev=""
+for a in "$@"; do
+  [ "$prev" = -o ] && out=$a
+  [ "$prev" = -H ] && case $a in @*) cat "\${a#@}" >> ${JSON.stringify(hdr)} ;; esac
+  prev=$a
+done
+cat >/dev/null; printf '' > "$out"; printf 200
+`);
+      chmodSync(join(bin, 'curl'), 0o755);
+      const base = { ...process.env, UBICLOUD_API_KEY: key, UBI_RUNNER_STATE: join(root, 'state') };
+      const faked = Bun.spawnSync(['bash', join(DIR, 'ubi-runner.sh'), 'cli', 'vm', 'list'], { env: { ...base, PATH: `${bin}:${process.env.PATH}` }, timeout: 10_000 });
+      expect(faked.exitCode, faked.stderr.toString()).toBe(0);
+      expect(readFileSync(argv, 'utf8')).not.toContain(key);
+      expect(readFileSync(hdr, 'utf8')).toBe(`Authorization: Bearer ${key}\n`);
+      let seen = '';
+      const server = Bun.serve({ port: 0, hostname: '127.0.0.1', fetch(req) { seen = req.headers.get('authorization') ?? ''; return new Response('eu-central-h1 vm-a\n'); } });
+      try {
+        const real = Bun.spawn(['bash', join(DIR, 'ubi-runner.sh'), 'cli', 'vm', 'list'], { env: { ...base, UBICLOUD_API_URL: `http://127.0.0.1:${server.port}` }, stdout: 'pipe', stderr: 'pipe', timeout: 20_000 });
+        expect(await real.exited).toBe(0);
+        expect(await new Response(real.stdout).text()).toBe('eu-central-h1 vm-a\n');
+        expect(seen).toBe(`Bearer ${key}`);
+      } finally { server.stop(true); }
+    } finally {
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+
   test('refuses to start without UBICLOUD_API_KEY, before any network call', () => {
     const env = { ...process.env, UBICLOUD_API_URL: 'http://127.0.0.1:9' } as Record<string, string | undefined>;
     delete env.UBICLOUD_API_KEY;
@@ -461,7 +517,7 @@ test('pack ships a linked worktree with a self-contained .git on the checked-out
 
   test('up records the VM in the scratch file before the create request is sent', () => {
     const runner = read('scripts/ubicloud/ubi-runner.sh');
-    const create = runner.indexOf('cli "${args[@]}" >/dev/null 2>"$dir/create.err"');
+    const create = runner.indexOf('code=$(api "$dir/create.err" "${args[@]}")');
     expect(create).toBeGreaterThan(0);
     expect(runner.indexOf('record_vm "$name" "$location"')).toBeLessThan(create);
     expect(runner).toContain('$HOME/.capy/work/ubi-runner/vms.tsv');