mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-14 20:37:14 +01:00
v1.92.53.0 fix(ubicloud): reap destroys only the caller's own VMs (GST-573) (#3756)
* fix(ubicloud): pack and usage on macOS, empty projects (#3680, #3679, #3678) pack builds its archive in two tar passes plus gzip, which bsdtar and GNU tar both accept, with COPYFILE_DISABLE=1 so Apple's tar adds no ._* files. vm_rows skips the blank row Ubicloud returns for a project with no VMs. usage parenthesizes its ternary for the one-true-awk and exits 1 on an unreadable list or size; the quota preflight logs that and defers to the API. Co-authored-by: Jared <182125291+yolo-jared@users.noreply.github.com> Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes (usage skips a VM destroyed since the list; spawn timeout) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes (quota wait retries an unreadable usage) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: changelog fragment for the ubicloud macOS and empty-project fixes Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: TESTING_INTERNALS notes the quota preflight's unreadable-usage path Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(ubicloud): reap destroys only the caller's own VMs (GST-573) reap, which every up runs first, destroyed any VM in the project whose TTL had ended, whatever owner created it. That broke the owner-scoped cloud-deletes rule. It now destroys only the caller's own expired and old TTL-less VMs and logs a count of other owners' expired VMs. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(ubicloud): trim the owner-scoped reap prose back under the context budget Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: TESTING_INTERNALS says the TTL reaper covers only this owner's VMs Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(ubicloud): restore the expiry and reap conditions in the trimmed reap prose Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: Jared <182125291+yolo-jared@users.noreply.github.com> Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
12 files changed
+91
-56
No files matched your search
@@ -44,12 +44,13 @@
|
||||
#
|
||||
# TTL and reaper: a name this script generates ends in -t<hours> (UBI_TTL_HOURS,
|
||||
# 1-999, default 3), the creator's own declaration of how long the VM may live.
|
||||
# `reap` destroys every VM in the project whose TTL expired more than ten
|
||||
# minutes ago, whichever owner made it, and this owner's own ubirun-<owner>-*
|
||||
# names with no TTL once they are UBI_LEGACY_HOURS old (default 4; another
|
||||
# owner's or an untagged TTL-less VM is never reaped); every `up` runs it first (UBI_REAP=0 skips it), so a runner
|
||||
# killed by a sleeping machine leaves nothing behind for longer than its TTL.
|
||||
# Names outside the ubirun- prefix are never reaped.
|
||||
# `reap` destroys only this owner's VMs: its ubirun-<owner>-* names whose TTL
|
||||
# expired more than ten minutes ago, and those with no TTL once they are
|
||||
# UBI_LEGACY_HOURS old (default 4). Another owner's VM, an untagged one and
|
||||
# names outside the ubirun- prefix are never reaped, expired or not; reap only
|
||||
# counts other owners' expired VMs in its log (cloud deletes stay owner-scoped,
|
||||
# GST-573). Every `up` runs it first (UBI_REAP=0 skips it), so a lane killed by
|
||||
# a sleeping machine has its leftovers destroyed by its own next `up`.
|
||||
#
|
||||
# Leftover line: `run` and `verify` end with
|
||||
# UBICLOUD: left=<0|1|unknown> owner=<owner> vm=<name> others=<n>
|
||||
@@ -754,23 +755,26 @@ vm_rows() {
|
||||
done
|
||||
}
|
||||
|
||||
# reap: destroy every VM whose name carries a TTL that ended more than
|
||||
# REAP_GRACE seconds ago, whichever owner created it (the TTL is that owner's
|
||||
# own declaration), and the caller's own TTL-less ubirun-<UBI_OWNER>-* names at
|
||||
# least LEGACY_HOURS old. Another owner's TTL-less VM, an untagged one, and names
|
||||
# outside the prefix are never touched: only a TTL lets reap judge another lane's VM.
|
||||
# reap: destroy the caller's own VMs whose TTL ended more than REAP_GRACE
|
||||
# seconds ago, and its TTL-less ubirun-<UBI_OWNER>-* names at least LEGACY_HOURS
|
||||
# old. Every other VM is never touched, whatever its TTL says: a delete stays
|
||||
# with the owner that created the VM. Other owners' expired VMs are only counted.
|
||||
cmd_reap() {
|
||||
local now rows loc name owner expires created n=0 failed=0
|
||||
local now rows loc name owner expires created n=0 failed=0 others=0
|
||||
[[ "$LEGACY_HOURS" =~ ^[1-9][0-9]{0,3}$ ]] || die "UBI_LEGACY_HOURS must be a whole number of hours from 1 to 9999 with no leading zero (how old a TTL-less ubirun-* VM must be before reap destroys it)"
|
||||
now=$(date +%s)
|
||||
rows=$(vm_rows) || { log "reap: could not list the project's VMs; nothing destroyed"; return 1; }
|
||||
while read -r loc name owner expires created; do
|
||||
[ -n "$name" ] || continue
|
||||
if [ "$owner" != "$OWNER" ]; then
|
||||
[ "$expires" = - ] || [ "$now" -lt $(( expires + REAP_GRACE )) ] || others=$(( others + 1 ))
|
||||
continue
|
||||
fi
|
||||
if [ "$expires" != - ]; then
|
||||
[ "$now" -ge $(( expires + REAP_GRACE )) ] || continue
|
||||
log "reaping expired $loc/$name (owner $owner; its TTL ended $(( (now - expires) / 60 )) min ago)"
|
||||
else
|
||||
[ -n "${UBI_OWNER:-}" ] && [ "$owner" = "$OWNER" ] && [ "$created" != - ] || continue
|
||||
[ -n "${UBI_OWNER:-}" ] && [ "$created" != - ] || continue
|
||||
[ $(( now - 10#$created )) -ge $(( LEGACY_HOURS * 3600 )) ] || continue
|
||||
log "reaping TTL-less $loc/$name (owner $owner; $(( (now - 10#$created) / 3600 ))h old, legacy sweep at ${LEGACY_HOURS}h)"
|
||||
fi
|
||||
@@ -778,6 +782,7 @@ cmd_reap() {
|
||||
cmd_down "$name" -l "$loc" || { failed=$(( failed + 1 )); log "failed to reap $loc/$name"; }
|
||||
done <<<"$rows"
|
||||
[ "$n" -gt 0 ] || log "reap: no expired VMs"
|
||||
[ "$others" -eq 0 ] || log "reap: left $others expired VMs of other owners; only their owners reap them"
|
||||
[ "$failed" -eq 0 ] || { log "reap: $failed of $n expired VMs could not be destroyed"; return 1; }
|
||||
}
|
||||
|
||||
@@ -1363,7 +1368,7 @@ cmd_run() {
|
||||
trap 'exit 130' INT TERM QUIT
|
||||
trap on_hup HUP
|
||||
if [[ "$name" =~ -t([0-9]+)$ ]]; then
|
||||
log "$name carries a ${BASH_REMATCH[1]}h TTL (any later run of this script reaps it); if this process dies first: $0 down $name"
|
||||
log "$name carries a ${BASH_REMATCH[1]}h TTL (this owner's next up or reap destroys it once it expires); if this process dies first: $0 down $name"
|
||||
elif [[ "$name" =~ ^$PREFIX-$OWNER-[0-9]{10}- ]]; then
|
||||
log "$name has no TTL (reap destroys it once it is ${LEGACY_HOURS}h old); if this process dies before its teardown line, destroy it with: $0 down $name"
|
||||
else
|
||||
@@ -1579,8 +1584,8 @@ usage: ubi-runner.sh <command> [args]
|
||||
owner print this caller's owner tag ($OWNER)
|
||||
records VMs this machine created and has not confirmed gone ($VM_RECORD)
|
||||
gc HOURS destroy this owner's VMs older than HOURS (never other owners')
|
||||
reap destroy every VM whose -t<hours> TTL has ended (any owner) and this
|
||||
owner's TTL-less ubirun-* VMs older than UBI_LEGACY_HOURS
|
||||
reap destroy this owner's VMs whose -t<hours> TTL has ended and its
|
||||
TTL-less ubirun-* VMs older than UBI_LEGACY_HOURS (never other owners')
|
||||
cli ARGS... raw Ubicloud CLI passthrough (e.g. cli vm list)
|
||||
|
||||
signals: INT and TERM tear the VM down. HUP, once a remote phase is live, detaches
|
||||
|
||||
Reference in new issue
Block a user