mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-04 07:47:27 +01:00
fix: gstack-skill-end releases its run on the workflow lease, so a finished /ship stops blocking gstack-upgrade (GST-1205)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed
1 parent
c197a139b7
commit
03ada799e8
8 files changed
+100
-22
No files matched your search
+38
-8
@@ -8,8 +8,10 @@
|
||||
# shares them.
|
||||
#
|
||||
# Usage:
|
||||
# gstack-lease touch [--root <checkout>] [--pid <p> | --from <pid>]
|
||||
# write <git dir>/gstack-leases/<pid> holding the pid's start identity;
|
||||
# gstack-lease touch [--root <checkout>] [--pid <p> | --from <pid>] [--run <id>]
|
||||
# write <git dir>/gstack-leases/<pid> holding the pid's start identity
|
||||
# and the open run <id> (gstack-skill-start's SESSION_ID); a lease with
|
||||
# no open run (written without --run, or by an older gstack) never idles;
|
||||
# pid defaults to the harness pid behind this call
|
||||
# (bin/gstack-harness-pid.sh, the same derivation the session marker
|
||||
# uses), or behind --from (a caller whose own parent is short-lived
|
||||
@@ -24,8 +26,11 @@
|
||||
# --count-shared-harness (bin/gstack-upgrade only) still prints an
|
||||
# ancestor that is a shared harness (capy-server: every session on the
|
||||
# machine runs under it) while its lease is under 120 minutes old
|
||||
# gstack-lease release --pid <p> --run <id> [--root <checkout>]
|
||||
# close run <id> on <pid>'s lease (gstack-skill-end); once every run it
|
||||
# recorded is closed the lease is idle and live skips it (GST-1205)
|
||||
# gstack-lease list [--root <checkout>]
|
||||
# print `<pid> <age>m alive|dead|stale` for every lease
|
||||
# print `<pid> <age>m alive|idle|dead|stale` for every lease
|
||||
# A pid alone is not a holder: pids are reused after a reboot or a machine
|
||||
# wake. A lease records its process's start identity (Linux: boot id and
|
||||
# /proc start time; elsewhere `ps -o lstart`), and a lease whose pid now
|
||||
@@ -47,12 +52,13 @@ BIN="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
}
|
||||
|
||||
VERB="${1:-}"; [ $# -gt 0 ] && shift
|
||||
ROOT="" PID="" EXCLUDE="" FROM="" SHARED=0
|
||||
ROOT="" PID="" EXCLUDE="" FROM="" SHARED=0 RUN=""
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--root) ROOT="${2:-}"; shift ;;
|
||||
--pid) PID="${2:-}"; shift ;;
|
||||
--from) FROM="${2:-}"; shift ;;
|
||||
--run) RUN="${2:-}"; shift ;;
|
||||
--exclude-ancestry-of) EXCLUDE="${2:-}"; shift ;;
|
||||
--count-shared-harness) SHARED=1 ;;
|
||||
*) echo "gstack-lease: unknown argument '$1'" >&2; exit 2 ;;
|
||||
@@ -60,8 +66,8 @@ while [ $# -gt 0 ]; do
|
||||
shift
|
||||
done
|
||||
case "$VERB" in
|
||||
touch|live|list) ;;
|
||||
-h|--help|'') sed -n '2,28p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||
touch|live|list|release) ;;
|
||||
-h|--help|'') sed -n '2,33p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||
*) echo "gstack-lease: unknown verb '$VERB'" >&2; exit 2 ;;
|
||||
esac
|
||||
[ -n "$ROOT" ] || ROOT=$(gstack_source_root "$BIN")
|
||||
@@ -264,7 +270,13 @@ case "$VERB" in
|
||||
exit 3
|
||||
}
|
||||
_tmp="$DIR/.$PID.$$"
|
||||
identity "$PID" > "$_tmp" && mv -f "$_tmp" "$DIR/$PID" || { rm -f "$_tmp"; unlock; exit 2; }
|
||||
_id=$(identity "$PID") _runs=""
|
||||
[ "$(head -1 "$DIR/$PID" 2>/dev/null)" = "$_id" ] && _runs=$(sed -n '2,$p' "$DIR/$PID" 2>/dev/null | grep '^run ' | grep -vxF "run ${RUN:--}")
|
||||
{
|
||||
printf '%s\n' "$_id"
|
||||
[ -n "$_runs" ] && printf '%s\n' "$_runs"
|
||||
[ -z "$RUN" ] || printf 'run %s\n' "$RUN"
|
||||
} > "$_tmp" && mv -f "$_tmp" "$DIR/$PID" || { rm -f "$_tmp"; unlock; exit 2; }
|
||||
find "$DIR" -maxdepth 1 -mmin +120 -type f ! -name '.lock*' -exec rm -f {} + 2>/dev/null
|
||||
for _c in "$DIR"/.lock*.claim.*; do
|
||||
[ -f "$_c" ] || continue
|
||||
@@ -274,12 +286,29 @@ case "$VERB" in
|
||||
rm -rf "$DIR"/.lock*.gone.* 2>/dev/null
|
||||
unlock
|
||||
exit 0 ;;
|
||||
release)
|
||||
case "$PID" in ''|*[!0-9]*) echo "gstack-lease: release needs --pid <number> and --run <id>" >&2; exit 2 ;; esac
|
||||
[ -n "$RUN" ] && [ "$RUN" != - ] || { echo "gstack-lease: release needs --pid <number> and --run <id>" >&2; exit 2; }
|
||||
[ -f "$DIR/$PID" ] || exit 0
|
||||
lock || exit 3
|
||||
f="$DIR/$PID" _tmp="$DIR/.$PID.$$"
|
||||
if grep -qxF "run $RUN" "$f" 2>/dev/null; then
|
||||
{
|
||||
head -1 "$f"
|
||||
sed -n '2,$p' "$f" | grep -vxF "run $RUN"
|
||||
sed -n '2,$p' "$f" | grep -vxF "run $RUN" | grep -q '^run ' || echo idle
|
||||
} > "$_tmp" && touch -r "$f" "$_tmp" && mv -f "$_tmp" "$f" || { rm -f "$_tmp"; unlock; exit 2; }
|
||||
fi
|
||||
unlock
|
||||
exit 0 ;;
|
||||
list)
|
||||
for f in "$DIR"/*; do
|
||||
[ -f "$f" ] || continue
|
||||
p=${f##*/}
|
||||
_age=$(age_min "$f")
|
||||
echo "$p ${_age}m $(state "$p" "$(head -1 "$f" 2>/dev/null)" "$_age")"
|
||||
_st=$(state "$p" "$(head -1 "$f" 2>/dev/null)" "$_age")
|
||||
[ "$_st" = alive ] && grep -qx idle "$f" 2>/dev/null && _st=idle
|
||||
echo "$p ${_age}m $_st"
|
||||
done
|
||||
exit 0 ;;
|
||||
esac
|
||||
@@ -296,6 +325,7 @@ done
|
||||
FOUND=1
|
||||
for f in "$DIR"/*; do
|
||||
[ -f "$f" ] || continue
|
||||
grep -qx idle "$f" 2>/dev/null && continue
|
||||
p=${f##*/}
|
||||
_age=$(age_min "$f")
|
||||
case "$ANCESTRY" in *" $p "*) [ "$SHARED" -eq 1 ] && [ "$_age" -lt 120 ] && gstack_shared_harness "$p" || continue ;; esac
|
||||
|
||||
@@ -57,6 +57,9 @@ _TEL_END=$(date +%s)
|
||||
case "$TEL_START" in ''|*[!0-9]*) TEL_START="$_TEL_END" ;; esac
|
||||
_TEL_DUR=$(( _TEL_END - TEL_START ))
|
||||
[ -n "$SESSION_ID" ] && rm -f "$_GH/analytics/.pending-$SESSION_ID" 2>/dev/null || true
|
||||
# Release the workflow lease run gstack-skill-start opened (SESSION_ID starts
|
||||
# with its harness pid), so a finished run stops blocking gstack-upgrade (GST-1205).
|
||||
[ -n "$SESSION_ID" ] && "$_BIN/gstack-lease" release --pid "${SESSION_ID%%-*}" --run "$SESSION_ID" >/dev/null 2>&1 || true
|
||||
|
||||
# Record completion before the final drain, including its durable enqueue.
|
||||
# The normal sync gates still decide publication: full may sync timelines,
|
||||
|
||||
@@ -127,13 +127,6 @@ fi
|
||||
mkdir -p "$_GH/sessions" 2>/dev/null || true
|
||||
touch "$_GH/sessions/$PARENT_PID" 2>/dev/null || true
|
||||
find "$_GH/sessions" -mmin +120 -type f -exec rm {} + 2>/dev/null || true
|
||||
# The install-scoped twin (bin/gstack-lease): what bin/gstack-upgrade reads
|
||||
# before moving a checkout other sessions share. Fail-open, but a lease the
|
||||
# held lock kept from being written is reported.
|
||||
"$_BIN/gstack-lease" touch --pid "$PARENT_PID" >/dev/null 2>&1 || {
|
||||
[ $? -eq 3 ] && echo "gstack-lease: the lease directory stayed locked by a live process; this session holds no workflow lease, so bin/gstack-upgrade cannot see it" >&2
|
||||
true
|
||||
}
|
||||
_SESSIONS=$(find "$_GH/sessions" -mmin -120 -type f 2>/dev/null | wc -l | tr -d ' ')
|
||||
|
||||
_PROACTIVE=$("$_BIN/gstack-config" get proactive 2>/dev/null || echo "core")
|
||||
@@ -244,6 +237,14 @@ _TEL_START=$(date +%s)
|
||||
# the plain form when urandom is unavailable.
|
||||
_SID_RAND=$(od -An -N4 -tx4 /dev/urandom 2>/dev/null | tr -d ' \n' || true)
|
||||
_SESSION_ID="$PARENT_PID-$_TEL_START${_SID_RAND:+-$_SID_RAND}"
|
||||
# The install-scoped twin (bin/gstack-lease): what bin/gstack-upgrade reads
|
||||
# before moving a checkout other sessions share. This run holds it until
|
||||
# gstack-skill-end releases its SESSION_ID (GST-1205). Fail-open, but a lease
|
||||
# the held lock kept from being written is reported.
|
||||
"$_BIN/gstack-lease" touch --pid "$PARENT_PID" --run "$_SESSION_ID" >/dev/null 2>&1 || {
|
||||
[ $? -eq 3 ] && echo "gstack-lease: the lease directory stayed locked by a live process; this session holds no workflow lease, so bin/gstack-upgrade cannot see it" >&2
|
||||
true
|
||||
}
|
||||
echo "TELEMETRY: ${_TEL:-off}"
|
||||
echo "TEL_PROMPTED: $_TEL_PROMPTED"
|
||||
echo "SESSION_ID: $_SESSION_ID"
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
### Fixed
|
||||
|
||||
- **A finished /ship no longer blocks `gstack-upgrade` for two hours (GST-1205).** Each `gstack-skill-start` writes a workflow lease that `gstack-upgrade` checks before moving a shared checkout. Nothing released it when the workflow ended, and on Capy every session's lease names the machine's one `capy-server`. So after a /ship finished, its own lease kept refusing the upgrade with UPGRADE_BUSY until it was 120 minutes old, unless you set `GSTACK_UPGRADE_FORCE=1`. Now `gstack-skill-start` records its `SESSION_ID` as an open run on the lease, and `gstack-skill-end` closes it with the new `gstack-lease release`. Once every run on a lease is closed, the lease is idle and blocks nothing (`gstack-lease list` shows `idle`). A run another session still has open keeps blocking, and so does a lease with no recorded run (an older gstack, or a touch without `--run`). A run that never reaches `gstack-skill-end` still ages out after 120 minutes.
|
||||
@@ -187,7 +187,7 @@ run from inside a workflow (/ship's docs step running `install.sh --for
|
||||
document-release`, #3571) prints `SELF_UPGRADE: skipped (a gstack run in this
|
||||
session holds a workflow lease on this installation: ... (UPGRADE_BUSY))`,
|
||||
leaves the checkout where the run started and stays ok with `behind=`. Each
|
||||
`gstack-skill-start` renews the lease. On Capy the leased pid is the machine's
|
||||
`gstack-skill-start` renews the lease and records its `SESSION_ID` as an open run; `gstack-skill-end` closes that run, and a lease whose runs are all closed is idle and blocks nothing, so a finished /ship no longer holds the upgrade back (GST-1205). A lease with no recorded run (written without `--run`, or by an older gstack) counts as before. On Capy the leased pid is the machine's
|
||||
long-lived `capy-server`, so the deferral ends when the lease turns 120
|
||||
minutes old, not when the process exits. Every session on the machine runs
|
||||
under that same `capy-server`, so `bin/gstack-upgrade` run by hand also
|
||||
@@ -231,7 +231,7 @@ identities, a taker stalled over 2 minutes between its `mkdir` and its owner
|
||||
rename can be broken and then publish into its successor's lock. While a live process
|
||||
holds the lock, or it cannot be taken at all, `touch` exits 3 without writing
|
||||
(`gstack-skill-start` reports that the session holds no lease) and a reclaim
|
||||
keeps the lease. `bin/gstack-lease list` shows `alive|dead|stale`, and
|
||||
keeps the lease. `bin/gstack-lease list` shows `alive|idle|dead|stale`, and
|
||||
`GSTACK_UPGRADE_FORCE=1` overrides.
|
||||
|
||||
## What the installer does
|
||||
|
||||
@@ -1705,10 +1705,12 @@ session holds a workflow lease on this installation: pid <p> (<age>m
|
||||
<checkout>/bin/gstack-upgrade once the lease is 120 minutes old, or
|
||||
GSTACK_UPGRADE_FORCE=1 <checkout>/bin/gstack-upgrade when no other session uses
|
||||
this checkout (UPGRADE_BUSY))` and leaves the
|
||||
checkout where the run started. Each `gstack-skill-start` renews the lease.
|
||||
checkout where the run started. Each `gstack-skill-start` renews the lease
|
||||
and opens a run that its `gstack-skill-end` closes; a lease whose runs are all
|
||||
closed is idle and never counts (GST-1205).
|
||||
|
||||
**Fix.** Wait for those sessions to finish (`bin/gstack-lease list` shows
|
||||
every lease with its age and `alive|dead|stale`; a holder is named as
|
||||
every lease with its age and `alive|idle|dead|stale`; a holder is named as
|
||||
`pid <p> (<age>m <command>)`), then re-run `bin/gstack-upgrade`. For the
|
||||
session's own run, or a shared-harness lease, re-run once the lease is 120
|
||||
minutes old. When you know no other session uses the checkout, `GSTACK_UPGRADE_FORCE=1 bin/gstack-upgrade`
|
||||
|
||||
@@ -91,7 +91,7 @@ describe('gstack-lease', () => {
|
||||
const r = spawnSync('bash', ['-n', LEASE], { encoding: 'utf8', timeout: 30_000 });
|
||||
expect(r.status, r.stderr).toBe(0);
|
||||
const help = spawnSync('bash', [LEASE, '--help'], { encoding: 'utf8', timeout: 30_000 });
|
||||
expect(help.stdout).toContain('print `<pid> <age>m alive|dead|stale` for every lease');
|
||||
expect(help.stdout).toContain('print `<pid> <age>m alive|idle|dead|stale` for every lease');
|
||||
});
|
||||
|
||||
test('touch writes <git dir>/gstack-leases/<pid> for --pid and prunes leases older than 120 minutes', () => {
|
||||
@@ -716,6 +716,30 @@ describe('gstack-lease', () => {
|
||||
}
|
||||
});
|
||||
|
||||
// GST-1205: gstack-skill-end releases the run gstack-skill-start opened, so a finished run stops holding the lease.
|
||||
test('gstack-skill-start opens a run on the lease and gstack-skill-end with its SESSION_ID releases it; another holder still counts', () => {
|
||||
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-lease-home-'));
|
||||
const state = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-lease-state-'));
|
||||
dirs.push(home, state);
|
||||
fs.writeFileSync(path.join(state, 'config.yaml'), 'update_check: false\n');
|
||||
const leaseDir = path.join(state, 'leases');
|
||||
const env = { PATH: process.env.PATH!, HOME: home, GSTACK_HOME: state, GSTACK_LEASE_DIR: leaseDir };
|
||||
const holder = sleeper();
|
||||
const start = spawnSync(START, ['--skill', 'ship', '--parent-pid', String(holder)], { encoding: 'utf8', cwd: home, env, timeout: 30_000 });
|
||||
expect(start.status, start.stdout + start.stderr).toBe(0);
|
||||
const sid = start.stdout.match(/^SESSION_ID: (\S+)$/m)?.[1] ?? '';
|
||||
expect(sid.startsWith(`${holder}-`), sid).toBe(true);
|
||||
expect(run(['live'], env).status).toBe(0);
|
||||
const end = spawnSync(path.join(REPO, 'bin/gstack-skill-end'), ['--skill', 'ship', '--outcome', 'success', '--session-id', sid, '--tel-start', '0'], { encoding: 'utf8', cwd: home, env, timeout: 30_000 });
|
||||
expect(end.status, end.stdout + end.stderr).toBe(0);
|
||||
const after = run(['live'], env);
|
||||
expect(after.status, after.stdout + after.stderr).toBe(1);
|
||||
expect(run(['list'], env).stdout).toMatch(new RegExp(`^${holder} \\d+m idle$`, 'm'));
|
||||
const other = sleeper();
|
||||
expect(run(['touch', '--pid', String(other)], env).status).toBe(0);
|
||||
expect(run(['live'], env).stdout).toMatch(new RegExp(`^${other} \\d+m sleep$`, 'm'));
|
||||
});
|
||||
|
||||
test('gstack-skill-start reports a lease the held lock kept from being written, and still starts', () => {
|
||||
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-lease-home-'));
|
||||
const state = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-lease-state-'));
|
||||
|
||||
@@ -216,14 +216,14 @@ describe('gstack-upgrade', () => {
|
||||
// every session's lease names capy-server and the caller's own-ancestry
|
||||
// exclusion used to drop a sibling's fresh lease: a hand-run upgrade moved the checkout.
|
||||
describe('a lease held by a shared harness (capy-server) ancestor', () => {
|
||||
const underHarness = (f: ReturnType<typeof fixture>, harness: string, ageMinutes: number, extra: Record<string, string> = {}) => {
|
||||
const underHarness = (f: ReturnType<typeof fixture>, harness: string, ageMinutes: number, extra: Record<string, string> = {}, lease = 'bash "$1" touch --root "$2" --pid $$ || exit 9') => {
|
||||
const bin = path.join(f.host.base, 'harness');
|
||||
fs.mkdirSync(bin, { recursive: true });
|
||||
const exe = path.join(bin, harness);
|
||||
if (!fs.existsSync(exe)) fs.copyFileSync(fs.realpathSync(spawnSync('bash', ['-c', 'command -v bash'], { encoding: 'utf8', timeout: 30_000 }).stdout.trim()), exe);
|
||||
fs.chmodSync(exe, 0o755);
|
||||
const script = [
|
||||
'bash "$1" touch --root "$2" --pid $$ || exit 9',
|
||||
lease,
|
||||
'[ "$4" -gt 0 ] && "$5" -e "const t = Date.now() / 1000 - Number(process.argv[2]) * 60; require(\'fs\').utimesSync(process.argv[1], t, t)" "${GSTACK_LEASE_DIR:-$(git -C "$2" rev-parse --absolute-git-dir)/gstack-leases}/$$" "$4"',
|
||||
'bash "$3"',
|
||||
'echo "exit=$?"',
|
||||
@@ -242,6 +242,21 @@ describe('gstack-upgrade', () => {
|
||||
expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b);
|
||||
});
|
||||
|
||||
// GST-1205: a finished /ship run's lease used to block the upgrade for 120 minutes.
|
||||
test('a lease whose runs all finished (gstack-lease release) no longer blocks; a sibling run still open on the same harness does', () => {
|
||||
const touch = (id: string) => `bash "$1" touch --root "$2" --pid $$ --run "$$-${id}"`;
|
||||
const release = (id: string) => `bash "$1" release --root "$2" --pid $$ --run "$$-${id}"`;
|
||||
const f = fixture();
|
||||
const open = underHarness(f, 'capy-server', 0, {}, `${touch('1-a')} && ${touch('2-b')} && ${release('1-a')} || exit 9`);
|
||||
expect(open.stdout, open.stdout + open.stderr).toContain('exit=3');
|
||||
expect(open.stderr).toContain('(UPGRADE_BUSY)');
|
||||
expect(f.git(['rev-parse', 'HEAD'])).toBe(f.a);
|
||||
const finished = underHarness(f, 'capy-server', 0, {}, `${touch('1-a')} && ${release('1-a')} || exit 9`);
|
||||
expect(finished.stdout, finished.stdout + finished.stderr).toContain('exit=0');
|
||||
expect(finished.stderr).not.toContain('UPGRADE_BUSY');
|
||||
expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b);
|
||||
});
|
||||
|
||||
test('once that lease is 120 minutes old the upgrade proceeds', () => {
|
||||
const f = fixture();
|
||||
const r = underHarness(f, 'capy-server', 121);
|
||||
|
||||
Reference in new issue
Block a user