mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 22:14:25 +01:00
v1.91.79.0 feat(eval-plan): /eval-plan tier 1, lane-check, evidence verify, coordinator contract, owner brief, contributor-PR mode, banned terms (multi-agent wave PR E) (#3123)
* v1.91.79.0 feat(eval-plan): /eval-plan tier 1, lane-check, evidence verify, coordinator contract, owner brief, contributor-PR mode, banned terms (multi-agent wave PR E) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(spec,review): fail closed when the coordinator contract cannot be read; restore the omitted-Army sentence The spec --execute spawn captures `gstack-autoplan contract` before piping the archive, so a lane never starts without the contract; the C9 fixture installs its gstack-like checkout (bin, lib, a contract stub) at every host's global root. Review Step 5.8 keeps the pinned 'without claiming specialist coverage' wording. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
ab79e55119
commit
5d647455f2
87 files changed
+4137
-762
No files matched your search
@@ -59,6 +59,10 @@ export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }>
|
||||
{ file: 'test/gstack-regen.test.ts', reason: 'spawns bin/gstack-regen through test/helpers/pregate-fixture.ts runBin (shebang via CreateProcess) and POSIX fixture regen commands' },
|
||||
{ file: 'test/pregate-invalidation.test.ts', reason: 'spawns bin/gstack-pregate and bin/gstack-ship-receipt through test/helpers/pregate-fixture.ts runBin (shebang via CreateProcess) and the pipefail lane runner' },
|
||||
{ file: 'test/ship-issue-intent.test.ts', reason: 'spawns bin/gstack-issue-links and bin/gstack-ship-receipt through test/helpers/pregate-fixture.ts runBin (shebang via CreateProcess) with a `gh` shell shim on PATH' },
|
||||
// PR E2: lane ownership and evidence verification run bin/gstack-lane-check and bin/gstack-evidence
|
||||
// through test/helpers/restamp-fixture.ts runBin (shebang via CreateProcess); the lib paths are
|
||||
// exercised in the same file, so there is no Windows-safe split to keep.
|
||||
{ file: 'test/lane-coordination.test.ts', reason: 'spawns bin/gstack-lane-check and bin/gstack-evidence through test/helpers/restamp-fixture.ts runBin (shebang via CreateProcess) and reruns POSIX lane commands' },
|
||||
{
|
||||
file: 'test/ship-measure-seeded-fixture.test.ts',
|
||||
reason: 'drives a POSIX shell stub project (`./evals.sh` with a shebang) that CreateProcess cannot exec; the measure runner itself is covered by ship-measure.test.ts with in-process fake runners',
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
/**
|
||||
* {{COORDINATOR_CONTRACT}} — the coordinator block every lane prompt inherits
|
||||
* word for word (plan E2). Source: lib/coordinator-contract.ts, shared with
|
||||
* bin/gstack-autoplan's prompt writer so the rendered skill text and the
|
||||
* printed lane prompts never drift.
|
||||
*/
|
||||
import type { ResolverFn } from './types';
|
||||
import { COORDINATOR_CONTRACT } from '../../lib/coordinator-contract';
|
||||
|
||||
export const generateCoordinatorContract: ResolverFn = () => COORDINATOR_CONTRACT;
|
||||
@@ -51,6 +51,7 @@ import { generateTestValueBar, generateTestValueMessage } from './test-value';
|
||||
import { generateImplementerDispatchRules } from './dispatch-rules';
|
||||
import { generatePlanRealityRows } from './plan-reality';
|
||||
import { generateSidewaysSweep, generateFlakeFixNumber } from './sideways';
|
||||
import { generateCoordinatorContract } from './coordinator-contract';
|
||||
import { generateQAScope, generateQAExploratory, generateQAFunctional, generateQAResource, generateQAReview, generateQAReviewPreflight, generateQAMethodReads } from './qa';
|
||||
|
||||
export const RESOLVERS: Record<string, ResolverFn> = {
|
||||
@@ -174,6 +175,7 @@ export const RESOLVERS: Record<string, ResolverFn> = {
|
||||
BRAIN_WRITE_BACK: generateBrainWriteBack,
|
||||
TASKS_SECTION_EMIT: generateTasksSectionEmit,
|
||||
TASKS_SECTION_AGGREGATE: generateTasksSectionAggregate,
|
||||
COORDINATOR_CONTRACT: generateCoordinatorContract,
|
||||
SECTION,
|
||||
SECTION_INDEX,
|
||||
};
|
||||
+317
-68
@@ -3,16 +3,41 @@
|
||||
#
|
||||
# UBICLOUD_API_KEY=... scripts/ubicloud/ubi-runner.sh run --setup S.sh -- 'make test'
|
||||
#
|
||||
# UBICLOUD_API_TOKEN is accepted as an alias for UBICLOUD_API_KEY. SSH
|
||||
# connections are multiplexed per VM, and `pack` / `unpack` upload one prebuilt
|
||||
# checkout tarball to many VMs. This is the repo copy the user-scope Capy
|
||||
# `ubicloud` skill and gbrain's scripts/ubicloud/ubi-runner.sh are kept in sync
|
||||
# with (plan E2); `bun run test:ubicloud` drives it through test-free.sh.
|
||||
#
|
||||
# Scratch record: every VM this machine creates is appended to a VM-id file
|
||||
# (UBI_VM_RECORD; default ~/.capy/work/ubi-runner/vms.tsv when ~/.capy/work
|
||||
# exists, else $STATE_ROOT/vms.tsv) as `name location owner created_at pid`,
|
||||
# and removed once `down` confirms it gone. A resumed Capy thread reads that
|
||||
# file to find VMs an earlier turn left behind (docs/TESTING_INTERNALS.md,
|
||||
# "Resuming after a wake").
|
||||
#
|
||||
# Creates a VM (standard-16 by default), streams the checkout to it (tracked +
|
||||
# untracked-unignored files + .git, so uncommitted edits are included), runs an
|
||||
# optional setup script and the command, copies requested artifacts back, and
|
||||
# destroys the VM on every exit path. Exits with the command's status.
|
||||
# Needs only bash, curl, python3, ssh, ssh-keygen, and tar locally, so it works
|
||||
# from dev boxes, containers, and cloud sandboxes. VMs are named
|
||||
# ubirun-<epoch>-<hex>. Every exit path destroys this client's VM. Nothing
|
||||
# sweeps stale VMs by default: the project quota is shared with other agents'
|
||||
# runners, and an age-only sweep destroyed their long runs too. Set
|
||||
# UBI_GC_HOURS to a positive number, or run `gc HOURS`, to opt in.
|
||||
# from dev boxes, containers, and cloud sandboxes.
|
||||
#
|
||||
# Ownership: VMs are named ubirun-<owner>-<epoch>-<suffix>. <owner> is UBI_OWNER
|
||||
# (lowercased, letters and digits only, at most 12 characters, starting with a
|
||||
# letter) or a random per-machine id kept in $STATE_ROOT/owner-id. Ubicloud names
|
||||
# allow 63 characters of [a-z0-9-], so a full name stays under 40.
|
||||
#
|
||||
# Teardown: `up` records each VM's state (location, create status) before it
|
||||
# sends the create request, and `down` destroys and polls until the VM is
|
||||
# confirmed gone or provably never existed, waiting out a create that may still
|
||||
# be in flight. An interrupted `up` or `run` (EXIT, INT, TERM, HUP, QUIT) runs
|
||||
# that `down`; further signals are ignored until it finishes. A create refused
|
||||
# for vCPU quota prints the project's usage per owner.
|
||||
#
|
||||
# Stale-VM sweeps are off by default. With UBI_GC_HOURS set to a positive
|
||||
# number, every `up` first destroys this owner's VMs older than that; `gc HOURS`
|
||||
# runs the same sweep on demand. Neither ever touches another owner's VMs.
|
||||
set -euo pipefail
|
||||
# Keep heredoc bodies on temp files, not the pipe window (test/heredoc-pipe-deadlock.test.ts).
|
||||
BASH_COMPAT=50
|
||||
@@ -22,7 +47,14 @@ STATE_ROOT="${UBI_RUNNER_STATE:-${XDG_STATE_HOME:-$HOME/.local/state}/ubi-runner
|
||||
DEFAULT_SIZE="${UBI_SIZE:-standard-16}"
|
||||
DEFAULT_LOCATION="${UBI_LOCATION:-eu-central-h1}"
|
||||
GC_HOURS="${UBI_GC_HOURS:-0}"
|
||||
API_TIMEOUT="${UBI_API_TIMEOUT:-120}"
|
||||
CREATE_GRACE="${UBI_CREATE_GRACE:-180}"
|
||||
DOWN_TIMEOUT="${UBI_DOWN_TIMEOUT:-900}"
|
||||
POLL="${UBI_POLL_SECONDS:-5}"
|
||||
PREFIX="ubirun"
|
||||
if [ -n "${UBI_VM_RECORD:-}" ]; then VM_RECORD="$UBI_VM_RECORD"
|
||||
elif [ -d "$HOME/.capy/work" ]; then VM_RECORD="$HOME/.capy/work/ubi-runner/vms.tsv"
|
||||
else VM_RECORD="$STATE_ROOT/vms.tsv"; fi
|
||||
|
||||
die() { echo "ubi-runner: $*" >&2; exit 1; }
|
||||
log() { echo "ubi-runner: $*" >&2; }
|
||||
@@ -30,16 +62,49 @@ log() { echo "ubi-runner: $*" >&2; }
|
||||
for bin in curl python3 ssh ssh-keygen tar; do
|
||||
command -v "$bin" >/dev/null || die "$bin is required"
|
||||
done
|
||||
[ -n "${UBICLOUD_API_KEY:-}" ] || die "UBICLOUD_API_KEY is not set (create a token under your Ubicloud project's Tokens page)"
|
||||
UBICLOUD_API_KEY="${UBICLOUD_API_KEY:-${UBICLOUD_API_TOKEN:-}}"
|
||||
[[ "$GC_HOURS" =~ ^[0-9]+$ ]] || die "UBI_GC_HOURS must be a whole number of hours (unset or 0 disables the sweep)"
|
||||
|
||||
owner_id() {
|
||||
local raw=${UBI_OWNER:-} file="$STATE_ROOT/owner-id" tmp
|
||||
if [ -z "$raw" ]; then
|
||||
if [ ! -s "$file" ]; then
|
||||
mkdir -p "$STATE_ROOT"
|
||||
tmp="$file.$$"
|
||||
echo "m$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')" >"$tmp"
|
||||
mv -n "$tmp" "$file"
|
||||
rm -f "$tmp"
|
||||
fi
|
||||
raw=$(cat "$file")
|
||||
fi
|
||||
raw=$(printf %s "$raw" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9')
|
||||
[ -n "$raw" ] || die "UBI_OWNER must contain letters or digits"
|
||||
[[ "$raw" =~ ^[a-z] ]] || raw="u$raw"
|
||||
echo "${raw:0:12}"
|
||||
}
|
||||
OWNER=$(owner_id)
|
||||
|
||||
# api OUT ARGS...: send one Ubicloud CLI command, write the body to OUT, and
|
||||
# print the HTTP status (000 when no response arrived).
|
||||
api() {
|
||||
local out=$1 code
|
||||
shift
|
||||
code=$(python3 -c 'import json,sys; print(json.dumps({"argv": sys.argv[1:]}))' "$@" \
|
||||
| curl -sS -o "$out" -w '%{http_code}' --max-time "$API_TIMEOUT" -X POST \
|
||||
-H "Authorization: Bearer $UBICLOUD_API_KEY" \
|
||||
-H 'Accept: text/plain' -H 'Content-Type: application/json' \
|
||||
-H 'X-Ubi-Version: 1.0.0' --data @- "$API/cli") || true
|
||||
echo "${code:-000}"
|
||||
}
|
||||
|
||||
cli() {
|
||||
local out code
|
||||
out=$(mktemp)
|
||||
code=$(python3 -c 'import json,sys; print(json.dumps({"argv": sys.argv[1:]}))' "$@" \
|
||||
| curl -sS -o "$out" -w '%{http_code}' -X POST \
|
||||
-H "Authorization: Bearer $UBICLOUD_API_KEY" \
|
||||
-H 'Accept: text/plain' -H 'Content-Type: application/json' \
|
||||
-H 'X-Ubi-Version: 1.0.0' --data @- "$API/cli") || { rm -f "$out"; die "request failed: ubi $*"; }
|
||||
code=$(api "$out" "$@")
|
||||
if [ "$code" = 000 ]; then
|
||||
rm -f "$out"
|
||||
die "request failed: ubi $*"
|
||||
fi
|
||||
if [ "$code" != 200 ]; then
|
||||
cat "$out" >&2
|
||||
rm -f "$out"
|
||||
@@ -62,7 +127,8 @@ load() {
|
||||
|
||||
ssh_opts() {
|
||||
echo -i "$KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
||||
-o LogLevel=ERROR -o ServerAliveInterval=30 -o ServerAliveCountMax=6 -o ConnectTimeout=10
|
||||
-o LogLevel=ERROR -o ServerAliveInterval=30 -o ServerAliveCountMax=6 -o ConnectTimeout=10 \
|
||||
-o ControlMaster=auto -o "ControlPath=$(dirname "$KEY")/ctl" -o ControlPersist=300
|
||||
}
|
||||
|
||||
# Ubuntu's default umask (002) leaves new directories group-writable, which
|
||||
@@ -74,25 +140,97 @@ remote() {
|
||||
ssh $(ssh_opts) "ubi@$IP" "umask 022; $*"
|
||||
}
|
||||
|
||||
cmd_gc() {
|
||||
local hours=${1:-$GC_HOURS} now loc name ts
|
||||
[ "$hours" -gt 0 ] || return 0
|
||||
now=$(date +%s)
|
||||
# vm_rows: "location name owner" for every VM in the project; owner is
|
||||
# "-" for untagged ubirun-<epoch>-* names and "." for VMs this runner did not name.
|
||||
vm_rows() {
|
||||
cli vm list -N -f location,name | while read -r loc name; do
|
||||
[[ "$name" =~ ^${PREFIX}-([0-9]{10})- ]] || continue
|
||||
ts=${BASH_REMATCH[1]}
|
||||
if [ $(( (now - ts) / 3600 )) -ge "$hours" ]; then
|
||||
log "destroying stale $loc/$name (older than ${hours}h)"
|
||||
cli vm "$loc/$name" destroy -f >/dev/null || log "failed to destroy $loc/$name"
|
||||
rm -rf "$(state_dir "$name")"
|
||||
if [[ "$name" =~ ^${PREFIX}-([a-z][a-z0-9]{0,11})-[0-9]{10}- ]]; then
|
||||
echo "$loc $name ${BASH_REMATCH[1]}"
|
||||
elif [[ "$name" =~ ^${PREFIX}-[0-9]{10}- ]]; then
|
||||
echo "$loc $name -"
|
||||
else
|
||||
echo "$loc $name ."
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
write_state() {
|
||||
printf 'NAME=%q\nLOCATION=%q\nSIZE=%q\nIP=%q\n' "$1" "$2" "$3" "$4" >"$(state_dir "$1")/env"
|
||||
cmd_gc() {
|
||||
local hours=${1:-$GC_HOURS} now loc name owner
|
||||
[[ "$hours" =~ ^[0-9]+$ ]] || die "gc: HOURS must be a whole number (or set UBI_GC_HOURS)"
|
||||
if [ "$hours" -eq 0 ]; then log "gc: no sweep (give HOURS > 0 or set UBI_GC_HOURS); nothing listed, nothing destroyed"; return 0; fi
|
||||
now=$(date +%s)
|
||||
vm_rows | while read -r loc name owner; do
|
||||
[ "$owner" = "$OWNER" ] || continue
|
||||
[[ "$name" =~ -([0-9]{10})- ]] || continue
|
||||
if [ $(( (now - BASH_REMATCH[1]) / 3600 )) -ge "$hours" ]; then
|
||||
log "destroying stale $loc/$name (owner $OWNER, older than ${hours}h)"
|
||||
cmd_down "$name" -l "$loc" || log "failed to destroy $loc/$name"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
cmd_list() {
|
||||
case ${1:-} in
|
||||
"") cli vm list ;;
|
||||
--mine) vm_rows | awk -v o="$OWNER" '$3==o {print $1 " " $2}' ;;
|
||||
*) die "list: unknown option $1 (use --mine)" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# usage: VMs and vCPUs per owner, from each VM's size (standard-16 = 16 vCPUs).
|
||||
cmd_usage() {
|
||||
local tmp loc name owner i=0
|
||||
tmp=$(mktemp -d)
|
||||
while read -r loc name owner; do
|
||||
i=$(( i + 1 ))
|
||||
cli vm "$loc/$name" show 2>/dev/null | sed -n "s/^size: /$owner /p" >"$tmp/$i" &
|
||||
[ $(( i % 8 )) -ne 0 ] || wait
|
||||
done < <(vm_rows)
|
||||
wait
|
||||
cat "$tmp"/* 2>/dev/null | awk -v me="$OWNER" '
|
||||
{ n = $2; sub(/^.*-/, "", n); o = $1 == "-" ? "(untagged)" : $1 == "." ? "(other)" : $1
|
||||
vms[o]++; cpu[o] += n; total += n; count++ }
|
||||
END {
|
||||
printf "%-14s %4s %6s\n", "OWNER", "VMS", "VCPUS"
|
||||
for (o in cpu) printf "%-14s %4d %6d%s\n", o, vms[o], cpu[o], o == me ? " (you)" : "" | "sort -k3,3nr"
|
||||
close("sort -k3,3nr")
|
||||
printf "%-14s %4d %6d\n", "total", count, total
|
||||
}'
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
write_state() {
|
||||
printf 'NAME=%q\nLOCATION=%q\nSIZE=%q\nIP=%q\nCREATED=%q\nCREATE_AT=%q\nCREATE_PID=%q\n' \
|
||||
"$1" "$2" "$3" "$4" "$5" "$6" "$$" >"$(state_dir "$1")/env"
|
||||
}
|
||||
|
||||
# record_vm NAME LOCATION: append to the scratch VM-id file (record-before-create).
|
||||
record_vm() {
|
||||
mkdir -p "$(dirname "$VM_RECORD")" 2>/dev/null || return 0
|
||||
printf '%s\t%s\t%s\t%s\t%s\n' "$1" "$2" "$OWNER" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$$" >>"$VM_RECORD"
|
||||
}
|
||||
|
||||
# forget_vm NAME: drop the VM from the scratch file once it is confirmed gone.
|
||||
forget_vm() {
|
||||
[ -f "$VM_RECORD" ] || return 0
|
||||
local tmp="$VM_RECORD.$$"
|
||||
awk -F'\t' -v n="$1" '$1!=n' "$VM_RECORD" >"$tmp" && mv "$tmp" "$VM_RECORD"
|
||||
}
|
||||
|
||||
# records: print the scratch file (VMs this machine created and has not confirmed gone).
|
||||
cmd_records() {
|
||||
echo "record: $VM_RECORD"
|
||||
[ -s "$VM_RECORD" ] || { echo "(no VMs recorded)"; return 0; }
|
||||
printf '%-40s %-14s %-12s %-20s %s\n' NAME LOCATION OWNER CREATED_AT PID
|
||||
awk -F'\t' '{ printf "%-40s %-14s %-12s %-20s %s\n", $1, $2, $3, $4, $5 }' "$VM_RECORD"
|
||||
}
|
||||
|
||||
new_name() { echo "$PREFIX-$OWNER-$(date +%s)-$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')"; }
|
||||
|
||||
valid_name() { [[ "$1" =~ ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$ ]]; }
|
||||
|
||||
UP_NAME=""
|
||||
UP_READY=""
|
||||
cmd_up() {
|
||||
local size=$DEFAULT_SIZE location=$DEFAULT_LOCATION name="" storage="" image=ubuntu-noble
|
||||
while [ $# -gt 0 ]; do
|
||||
@@ -105,10 +243,13 @@ cmd_up() {
|
||||
*) die "up: unknown option $1" ;;
|
||||
esac
|
||||
done
|
||||
[ -n "$name" ] || name="$PREFIX-$(date +%s)-$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')"
|
||||
cmd_gc "$GC_HOURS" || log "stale-VM cleanup failed; continuing"
|
||||
[ -n "$name" ] || name=$(new_name)
|
||||
valid_name "$name" || die "invalid VM name '$name' (Ubicloud allows 1-63 of a-z, 0-9 and '-', starting and ending with a letter or digit)"
|
||||
if [ "$GC_HOURS" -gt 0 ]; then
|
||||
cmd_gc "$GC_HOURS" || log "stale-VM cleanup failed; continuing"
|
||||
fi
|
||||
|
||||
local dir
|
||||
local dir create_at
|
||||
dir=$(state_dir "$name")
|
||||
mkdir -p "$dir"
|
||||
chmod 700 "$dir"
|
||||
@@ -117,9 +258,31 @@ cmd_up() {
|
||||
local args=(vm "$location/$name" create -s "$size" -b "$image")
|
||||
[ -z "$storage" ] || args+=(-S "$storage")
|
||||
args+=("$(cat "$dir/key.pub")")
|
||||
create_at=$(date +%s)
|
||||
write_state "$name" "$location" "$size" "" unknown "$create_at"
|
||||
record_vm "$name" "$location"
|
||||
UP_NAME=$name
|
||||
log "creating $location/$name ($size)"
|
||||
cli "${args[@]}" >/dev/null || { rm -rf "$dir"; die "create failed"; }
|
||||
write_state "$name" "$location" "$size" ""
|
||||
# Hold signals until the create answer is recorded, so teardown knows
|
||||
# whether the VM exists instead of waiting out UBI_CREATE_GRACE.
|
||||
local interrupted="" created=yes refusal need used max
|
||||
trap 'interrupted=1' INT TERM HUP QUIT
|
||||
cli "${args[@]}" >/dev/null 2>"$dir/create.err" || created=no
|
||||
write_state "$name" "$location" "$size" "" "$created" "$create_at"
|
||||
trap 'exit 130' INT TERM HUP QUIT
|
||||
[ -z "$interrupted" ] || exit 130
|
||||
if [ "$created" = no ]; then
|
||||
refusal=$(sed -n 's/.*Requested vCPU count: \([0-9]*\), currently used vCPU count: \([0-9]*\), maximum allowed vCPU count: \([0-9]*\).*/\1 \2 \3/p' "$dir/create.err")
|
||||
if [ -z "$refusal" ]; then
|
||||
cat "$dir/create.err" >&2
|
||||
die "create failed"
|
||||
fi
|
||||
read -r need used max <<<"$refusal"
|
||||
log "quota refused $location/$name ($size): it needs $need vCPUs and the project already uses $used of $max"
|
||||
log "VMs in the project by owner (the rest of the used count is other usage, such as managed GitHub runners):"
|
||||
cmd_usage >&2 || true
|
||||
die "create failed: vCPU quota exhausted; wait for running VMs to finish, or use fewer or smaller VMs (ci:ubicloud --vms N)"
|
||||
fi
|
||||
|
||||
local deadline=$(( $(date +%s) + 600 )) show state ip
|
||||
while :; do
|
||||
@@ -127,54 +290,99 @@ cmd_up() {
|
||||
state=$(sed -n 's/^state: //p' <<<"$show")
|
||||
ip=$(sed -n 's/^ip4: //p' <<<"$show")
|
||||
[ "$state" = running ] && [ -n "$ip" ] && break
|
||||
[ "$(date +%s)" -lt "$deadline" ] || { cmd_down "$name"; die "VM did not reach running in 10 minutes (last state: ${state:-unknown})"; }
|
||||
sleep 5
|
||||
[ "$(date +%s)" -lt "$deadline" ] || die "VM did not reach running in 10 minutes (last state: ${state:-unknown})"
|
||||
sleep "$POLL"
|
||||
done
|
||||
write_state "$name" "$location" "$size" "$ip"
|
||||
write_state "$name" "$location" "$size" "$ip" yes "$create_at"
|
||||
|
||||
load "$name"
|
||||
deadline=$(( $(date +%s) + 300 ))
|
||||
# shellcheck disable=SC2046
|
||||
until ssh $(ssh_opts) "ubi@$IP" true 2>/dev/null; do
|
||||
[ "$(date +%s)" -lt "$deadline" ] || { cmd_down "$name"; die "SSH did not come up on $IP"; }
|
||||
sleep 5
|
||||
[ "$(date +%s)" -lt "$deadline" ] || die "SSH did not come up on $IP"
|
||||
sleep "$POLL"
|
||||
done
|
||||
remote "$name" 'cloud-init status --wait >/dev/null 2>&1 || true'
|
||||
UP_READY=1
|
||||
log "ready: $name ($IP)"
|
||||
echo "$name"
|
||||
}
|
||||
|
||||
# down NAME [-l LOCATION]: destroy NAME and return 0 only once it is confirmed
|
||||
# gone or never existed. While this machine's create request for NAME may
|
||||
# still be in flight (its `up` process is alive, or the create was sent less
|
||||
# than UBI_CREATE_GRACE seconds ago with no answer), "not found" is not final.
|
||||
cmd_down() {
|
||||
local name=$1 dir loc
|
||||
local name=${1:-} loc="" dir
|
||||
[ -n "$name" ] || die "down: missing NAME"
|
||||
shift
|
||||
while [ $# -gt 0 ]; do
|
||||
case $1 in
|
||||
-l|--location) loc=$2; shift 2 ;;
|
||||
*) die "down: unknown option $1" ;;
|
||||
esac
|
||||
done
|
||||
valid_name "$name" || die "invalid VM name '$name'"
|
||||
dir=$(state_dir "$name")
|
||||
local CREATED=none CREATE_AT=0 CREATE_PID="" IP="" LOCATION="" KEY=""
|
||||
if [ -f "$dir/env" ]; then
|
||||
load "$name"
|
||||
loc=$LOCATION
|
||||
else
|
||||
loc=$(cli vm list -N -f location,name | awk -v n="$name" '$2==n {print $1}')
|
||||
[ -n "$loc" ] || die "VM '$name' not found"
|
||||
if [ -S "$dir/ctl" ] && [ -n "$IP" ]; then
|
||||
# shellcheck disable=SC2046
|
||||
ssh $(ssh_opts) -O exit "ubi@$IP" >/dev/null 2>&1 || true
|
||||
fi
|
||||
fi
|
||||
# A failed destroy, or a VM still listed afterwards, is a failure: callers
|
||||
# report success only when the VM is really gone.
|
||||
if ! cli vm "$loc/$name" destroy -f >/dev/null; then
|
||||
log "FAILED to destroy $loc/$name; retry: $0 down $name"
|
||||
return 1
|
||||
fi
|
||||
local i
|
||||
for i in $(seq 1 30); do
|
||||
cli vm list -N -f location,name | awk -v n="$name" '$2==n {found=1} END {exit !found}' || { log "destroyed $loc/$name"; rm -rf "$dir"; return 0; }
|
||||
sleep 2
|
||||
|
||||
local deadline=$(( $(date +%s) + DOWN_TIMEOUT )) seen=0 asked=0 out code now
|
||||
out=$(mktemp)
|
||||
while :; do
|
||||
now=$(date +%s)
|
||||
if [ -z "$loc" ]; then
|
||||
loc=$(vm_rows 2>/dev/null | awk -v n="$name" '$2==n {print $1}') || loc=""
|
||||
code=404
|
||||
[ -z "$loc" ] || code=$(api "$out" vm "$loc/$name" show)
|
||||
else
|
||||
code=$(api "$out" vm "$loc/$name" show)
|
||||
fi
|
||||
if [ "$code" = 200 ]; then
|
||||
seen=1
|
||||
if [ $(( now - asked )) -ge 60 ]; then
|
||||
code=$(api "$out" vm "$loc/$name" destroy -f)
|
||||
if [ "$code" != 200 ]; then
|
||||
cat "$out" >&2
|
||||
rm -f "$out"
|
||||
log "FAILED to destroy $loc/$name (HTTP $code); retry: $0 down $name"
|
||||
return 1
|
||||
fi
|
||||
asked=$now
|
||||
log "destroy requested: $loc/$name"
|
||||
fi
|
||||
elif [ "$code" = 404 ]; then
|
||||
if [ "$seen" = 1 ] || [ "$CREATED" != unknown ] || { [ $(( now - CREATE_AT )) -ge "$CREATE_GRACE" ] \
|
||||
&& { [ "$CREATE_PID" = "$$" ] || ! kill -0 "$CREATE_PID" 2>/dev/null; }; }; then
|
||||
rm -f "$out"
|
||||
rm -rf "$dir"
|
||||
forget_vm "$name"
|
||||
if [ "$seen" = 1 ]; then log "destroyed ${loc:-?}/$name"; else log "gone: ${loc:-?}/$name (never existed or already destroyed)"; fi
|
||||
return 0
|
||||
fi
|
||||
log "waiting for an in-flight create of $name to resolve"
|
||||
fi
|
||||
if [ "$now" -ge "$deadline" ]; then
|
||||
rm -f "$out"
|
||||
log "WARNING: could not confirm ${loc:-?}/$name is gone after ${DOWN_TIMEOUT}s; rerun: $0 down $name"
|
||||
return 1
|
||||
fi
|
||||
sleep "$POLL"
|
||||
done
|
||||
log "FAILED: $loc/$name is still listed after destroy; retry: $0 down $name"
|
||||
return 1
|
||||
}
|
||||
|
||||
cmd_sync() {
|
||||
local name=$1 src=${2:-.} dest=${3:-}
|
||||
src=$(cd "$src" && pwd)
|
||||
[ -n "$dest" ] || dest="work/$(basename "$src")"
|
||||
local qdest
|
||||
qdest=$(printf %q "$dest")
|
||||
# pack SRC: write a gzipped checkout tarball to stdout (tracked + untracked
|
||||
# files that are not ignored, plus .git; a non-git directory is taken whole).
|
||||
cmd_pack() {
|
||||
local src
|
||||
src=$(cd "${1:-.}" && pwd)
|
||||
if git -C "$src" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
|
||||
(
|
||||
cd "$src"
|
||||
@@ -183,10 +391,24 @@ cmd_sync() {
|
||||
if [ -e "$f" ] || [ -L "$f" ]; then printf '%s\0' "$f"; fi
|
||||
done \
|
||||
| tar --null -T - -czf -
|
||||
) | remote "$name" "mkdir -p $qdest && tar -xzf - -C $qdest"
|
||||
)
|
||||
else
|
||||
tar -C "$src" -czf - . | remote "$name" "mkdir -p $qdest && tar -xzf - -C $qdest"
|
||||
tar -C "$src" -czf - .
|
||||
fi
|
||||
}
|
||||
|
||||
# unpack NAME DEST: extract a tarball read from stdin into DEST on the VM.
|
||||
cmd_unpack() {
|
||||
local qdest
|
||||
qdest=$(printf %q "$2")
|
||||
remote "$1" "mkdir -p $qdest && tar -xzf - -C $qdest"
|
||||
}
|
||||
|
||||
cmd_sync() {
|
||||
local name=$1 src=${2:-.} dest=${3:-}
|
||||
src=$(cd "$src" && pwd)
|
||||
[ -n "$dest" ] || dest="work/$(basename "$src")"
|
||||
cmd_pack "$src" | cmd_unpack "$name" "$dest"
|
||||
log "synced $src -> $name:$dest"
|
||||
}
|
||||
|
||||
@@ -225,13 +447,20 @@ cmd_run() {
|
||||
src=$(cd "$src" && pwd)
|
||||
[ -n "$dest" ] || dest="work/$(basename "$src")"
|
||||
|
||||
RUN_VM=$(cmd_up ${up_args[@]+"${up_args[@]}"})
|
||||
local name=$RUN_VM
|
||||
local i name=""
|
||||
for (( i = 0; i < ${#up_args[@]}; i += 2 )); do
|
||||
case ${up_args[i]} in -n|--name) name=${up_args[i+1]} ;; esac
|
||||
done
|
||||
[ -n "$name" ] || { name=$(new_name); up_args+=(-n "$name"); }
|
||||
RUN_VM=$name
|
||||
# Armed before the create request: an interrupted `up` still destroys its VM.
|
||||
# A second signal must not cut teardown short, so the EXIT handler ignores them.
|
||||
trap 'trap "" INT TERM HUP QUIT; cmd_down "$RUN_VM" || log "WARNING: failed to destroy $RUN_VM; run: $0 down $RUN_VM"' EXIT
|
||||
trap 'exit 130' INT TERM HUP QUIT
|
||||
cmd_up "${up_args[@]}" >/dev/null
|
||||
if [ "$keep" = 1 ]; then
|
||||
trap - EXIT
|
||||
log "--keep: leaving $name running; destroy with: $0 down $name"
|
||||
else
|
||||
trap 'cmd_down "$RUN_VM" || log "WARNING: failed to destroy $RUN_VM; run: $0 down $RUN_VM"' EXIT
|
||||
trap 'exit 130' INT TERM
|
||||
fi
|
||||
|
||||
local e
|
||||
@@ -271,29 +500,49 @@ usage: ubi-runner.sh <command> [args]
|
||||
create a VM and wait for SSH; prints its name
|
||||
ssh NAME [COMMAND] shell or login-shell command on the VM (user ubi, passwordless sudo)
|
||||
sync NAME [SRC] [DEST] stream a checkout (tracked + untracked-unignored + .git)
|
||||
pack [SRC] write that checkout tarball to stdout
|
||||
unpack NAME DEST extract a tarball from stdin into DEST on the VM
|
||||
pull NAME REMOTE_GLOB LOCAL_DIR
|
||||
copy matching remote entries into LOCAL_DIR
|
||||
down NAME destroy the VM
|
||||
list list all VMs in the project
|
||||
gc HOURS destroy $PREFIX-* VMs older than HOURS (every owner's; off by default)
|
||||
down NAME [-l LOC] destroy the VM; returns once it is confirmed gone or never existed
|
||||
list [--mine] list all VMs in the project, or only this owner's
|
||||
usage VMs and vCPUs per owner across the project
|
||||
owner print this caller's owner tag ($OWNER)
|
||||
records VMs this machine created and has not confirmed gone ($VM_RECORD)
|
||||
gc HOURS destroy this owner's VMs older than HOURS (never other owners')
|
||||
cli ARGS... raw Ubicloud CLI passthrough (e.g. cli vm list)
|
||||
|
||||
VMs are named $PREFIX-<owner>-<epoch>-<suffix>; set UBI_OWNER (e.g. your thread
|
||||
code) to tag them, otherwise a per-machine id is used.
|
||||
defaults: size=$DEFAULT_SIZE location=$DEFAULT_LOCATION (env UBI_SIZE, UBI_LOCATION)
|
||||
env: UBI_GC_HOURS (unset/0: no sweep on up), UBI_CREATE_GRACE (${CREATE_GRACE}s),
|
||||
UBI_DOWN_TIMEOUT (${DOWN_TIMEOUT}s), UBI_API_TIMEOUT (${API_TIMEOUT}s), UBI_VM_RECORD (scratch VM-id file)
|
||||
EOF
|
||||
}
|
||||
|
||||
cmd=${1:-help}
|
||||
[ $# -eq 0 ] || shift
|
||||
case $cmd in
|
||||
owner|pack|records|help|-h|--help) ;;
|
||||
*) [ -n "${UBICLOUD_API_KEY:-}" ] || die "UBICLOUD_API_KEY is not set (create a token under your Ubicloud project's Tokens page)" ;;
|
||||
esac
|
||||
case $cmd in
|
||||
run) cmd_run "$@" ;;
|
||||
up) cmd_up "$@" ;;
|
||||
up) trap 'trap "" INT TERM HUP QUIT; [ -z "$UP_NAME" ] || [ -n "$UP_READY" ] || cmd_down "$UP_NAME" || log "WARNING: failed to destroy $UP_NAME; run: $0 down $UP_NAME"' EXIT
|
||||
trap 'exit 130' INT TERM HUP QUIT
|
||||
cmd_up "$@" ;;
|
||||
ssh) name=$1; shift; load "$name"
|
||||
# shellcheck disable=SC2046
|
||||
if [ $# -eq 0 ]; then exec ssh -t $(ssh_opts) "ubi@$IP"; else remote "$name" "bash -lc $(printf %q "$*")"; fi ;;
|
||||
sync) cmd_sync "$@" ;;
|
||||
pack) cmd_pack "$@" ;;
|
||||
unpack) cmd_unpack "$@" ;;
|
||||
pull) cmd_pull "$@" ;;
|
||||
down) cmd_down "$@" ;;
|
||||
list) cli vm list ;;
|
||||
list) cmd_list "$@" ;;
|
||||
usage) cmd_usage ;;
|
||||
owner) echo "$OWNER" ;;
|
||||
records) cmd_records ;;
|
||||
gc) cmd_gc "$@" ;;
|
||||
cli) cli "$@" ;;
|
||||
help|-h|--help) usage ;;
|
||||
|
||||
Reference in new issue
Block a user