diff --git a/bin/gstack-lease b/bin/gstack-lease index a04c50c20..30b0f1d86 100755 --- a/bin/gstack-lease +++ b/bin/gstack-lease @@ -65,23 +65,48 @@ identity() { # PID — the process's start identity, empty when unknown if [ -r "/proc/$1/stat" ] && [ -r /proc/sys/kernel/random/boot_id ]; then _st=$(cat "/proc/$1/stat" 2>/dev/null) || return 0 _st=${_st##*) } - printf 'linux %s %s\n' "$(cat /proc/sys/kernel/random/boot_id)" "$(printf '%s\n' "$_st" | awk '{print $20}')" + _id="linux $(cat /proc/sys/kernel/random/boot_id 2>/dev/null) $(printf '%s\n' "$_st" | awk '{print $20}')" else - _ls=$(ps -o lstart= -p "$1" 2>/dev/null | sed 's/^ *//; s/ *$//') - [ -n "$_ls" ] && printf 'ps %s\n' "$_ls" + _id="ps $(LC_ALL=C TZ=UTC0 ps -o lstart= -p "$1" 2>/dev/null | sed 's/^ *//; s/ *$//')" fi + valid_identity "$_id" && printf '%s\n' "$_id" return 0 } -# state FILE PID — alive | dead | stale (the pid now belongs to another process) +# valid_identity ID — a well-formed identity from either backend +valid_identity() { + case "$1" in + "linux "*) printf '%s\n' "$1" | grep -Eq '^linux [0-9a-f-]{36} [0-9]+$' ;; + "ps "?*) case "$1" in "ps "*[!\ ]*) return 0 ;; esac; return 1 ;; + *) return 1 ;; + esac +} +# state FILE PID — alive | dead | stale. Stale needs proof: two well-formed +# identities from the same backend that differ. An identity that cannot be read +# now, a malformed record or a backend change keeps the lease (never reclaim a +# live session on missing evidence). state() { alive "$2" || { echo dead; return; } _rec=$(head -1 "$1" 2>/dev/null) - if [ -n "$_rec" ]; then - [ "$_rec" = "$(identity "$2")" ] && echo alive || echo stale - elif [ "$(age_min "$1")" -ge 120 ]; then echo stale + if [ -z "$_rec" ]; then + [ "$(age_min "$1")" -ge 120 ] && echo stale || echo alive + return + fi + _now=$(identity "$2") + if valid_identity "$_rec" && [ -n "$_now" ] && [ "${_rec%% *}" = "${_now%% *}" ] && [ "$_rec" != "$_now" ]; then echo stale else echo alive fi } +# reclaim FILE RECORD — remove a stale lease without losing a concurrent renewal: +# rename it aside atomically, and put it back (hard link: only if no newer lease +# was written meanwhile) when what was renamed is not the record judged stale. +reclaim() { + _aside="${1%/*}/.reclaim.${1##*/}.$$" + mv -f "$1" "$_aside" 2>/dev/null || return 1 + if [ "$(head -1 "$_aside" 2>/dev/null)" = "$2" ]; then rm -f "$_aside"; return 0; fi + ln "$_aside" "$1" 2>/dev/null + rm -f "$_aside" + return 1 +} age_min() { # FILE — minutes since its mtime _m=$(stat -c %Y "$1" 2>/dev/null || stat -f %m "$1" 2>/dev/null || echo 0) echo $(( ( $(date +%s) - _m ) / 60 )) @@ -91,7 +116,8 @@ case "$VERB" in touch) [ -n "$PID" ] || PID=$(gstack_harness_pid "${FROM:-$PPID}") mkdir -p "$DIR" || exit 2 - identity "$PID" > "$DIR/$PID" || exit 2 + _tmp="$DIR/.$PID.$$" + identity "$PID" > "$_tmp" && mv -f "$_tmp" "$DIR/$PID" || { rm -f "$_tmp"; exit 2; } find "$DIR" -mmin +120 -type f -exec rm -f {} + 2>/dev/null exit 0 ;; list) @@ -120,8 +146,11 @@ for f in "$DIR"/*; do case "$(state "$f" "$p")" in dead) continue ;; stale) - echo "gstack-lease: reclaimed stale lease $p ($(age_min "$f")m; recorded '$(head -1 "$f")', pid $p is now $(ps -o comm= -p "$p" 2>/dev/null | head -1) '$(identity "$p")')" >&2 - rm -f "$f"; continue ;; + _rec=$(head -1 "$f" 2>/dev/null) _age=$(age_min "$f") + if reclaim "$f" "$_rec"; then + echo "gstack-lease: reclaimed stale lease $p (${_age}m; recorded '$_rec', pid $p is now $(ps -o comm= -p "$p" 2>/dev/null | head -1) '$(identity "$p")')" >&2 + continue + fi ;; esac echo "$p $(age_min "$f")m $(ps -o comm= -p "$p" 2>/dev/null | head -1 | sed 's|.*/||')" FOUND=0 diff --git a/docs/capy-install.md b/docs/capy-install.md index 2377a68ee..ce96b23f9 100644 --- a/docs/capy-install.md +++ b/docs/capy-install.md @@ -86,7 +86,11 @@ while a lease held by a live process outside its own chain exists, naming each holder as `pid

(m )`. A lease whose pid now belongs to another process, or an identity-less lease from an older gstack past 120 minutes, is stale: `bin/gstack-lease live` reclaims it (`gstack-lease: reclaimed stale -lease …`). `bin/gstack-lease list` shows `alive|dead|stale`, and +lease …`). Stale needs proof: two well-formed identities from the same backend +that differ (`ps` is read under `LC_ALL=C TZ=UTC0`). An identity that cannot be +read, a malformed record or a backend change keeps the lease. Reclaiming renames +the lease aside first and restores it when a renewal landed meanwhile, and +`touch` publishes by rename, so a reader never sees a half-written record. `bin/gstack-lease list` shows `alive|dead|stale`, and `GSTACK_UPGRADE_FORCE=1` overrides. ## What the installer does diff --git a/test/capy-skill-wrapper.test.ts b/test/capy-skill-wrapper.test.ts index 88422021e..711973ad0 100644 --- a/test/capy-skill-wrapper.test.ts +++ b/test/capy-skill-wrapper.test.ts @@ -362,7 +362,8 @@ describe('capy skill: the session hands the installer GSTACK_CAPY_BEHIND', () => expect(busy.last).toMatch(new RegExp(`behind=1 to=1\\.0\\.2 fix=.+ blocked=live gstack sessions hold this installation: pid ${holder.pid} \\(\\d+m sleep\\); fix: .+GSTACK_UPGRADE_FORCE=1.+ \\(UPGRADE_BUSY\\)$`)); expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b); const file = path.join(leaseDir, String(holder.pid)); - fs.writeFileSync(file, fs.readFileSync(file, 'utf8').trim().replace(/\S+$/, 'x1') + '\n'); + const recorded = fs.readFileSync(file, 'utf8').trim(); + fs.writeFileSync(file, (recorded.startsWith('linux ') ? recorded.replace(/\d+$/, (n) => String(Number(n) + 1)) : 'ps Thu Jan 1 00:00:00 1970') + '\n'); const reclaimed = f.run(['--for', 'autoplan'], { GSTACK_LEASE_DIR: leaseDir }); expect(reclaimed.stdout).toMatch(/^SELF_UPGRADE: done 1\.0\.1 -> 1\.0\.2$/m); expect(f.git(['rev-parse', 'HEAD'])).toBe(f.c); diff --git a/test/gstack-lease.test.ts b/test/gstack-lease.test.ts index 4cfbd437e..a408751a1 100644 --- a/test/gstack-lease.test.ts +++ b/test/gstack-lease.test.ts @@ -54,6 +54,18 @@ function deadPid(): number { return r.pid!; } +/** The same backend's identity for a different process start (what a reused pid shows). */ +function otherStart(recorded: string): string { + return recorded.startsWith('linux ') ? recorded.replace(/\d+$/, (n) => String(Number(n) + 1)) : 'ps Thu Jan 1 00:00:00 1970'; +} + +/** A BASH_ENV file whose functions shadow commands inside the lease script (fault injection). */ +function bashEnv(dir: string, body: string): string { + const file = path.join(dir, 'bash-env.sh'); + fs.writeFileSync(file, body); + return file; +} + describe('gstack-lease', () => { test('bash -n passes and the bin is executable', () => { expect(fs.existsSync(LEASE), LEASE).toBe(true); @@ -102,7 +114,7 @@ describe('gstack-lease', () => { expect(run(['touch', '--root', dir, '--pid', String(reused)]).status).toBe(0); const recorded = fs.readFileSync(path.join(leases, String(reused)), 'utf8').trim(); expect(recorded).toMatch(/^(linux [0-9a-f-]+ \d+|ps .+)$/); - fs.writeFileSync(path.join(leases, String(reused)), recorded.replace(/\S+$/, 'x1') + '\n'); + fs.writeFileSync(path.join(leases, String(reused)), otherStart(recorded) + '\n'); expect(run(['list', '--root', dir]).stdout).toMatch(new RegExp(`^${reused} \\d+m stale$`, 'm')); const live = run(['live', '--root', dir]); expect(live.status, live.stdout + live.stderr).toBe(1); @@ -111,6 +123,51 @@ describe('gstack-lease', () => { expect(fs.existsSync(path.join(leases, String(reused)))).toBe(false); }); + test('missing evidence never reclaims a live lease: an unreadable current identity, a malformed record or another backend keeps it', () => { + const { dir, leases } = checkout(); + const holder = sleeper(); + const file = path.join(leases, String(holder)); + expect(run(['touch', '--root', dir, '--pid', String(holder)]).status).toBe(0); + const recorded = fs.readFileSync(file, 'utf8').trim(); + const blind = bashEnv(dir, 'cat() { case "$1" in /proc/*/stat) return 1 ;; esac; command cat "$@"; }\nps() { case "$*" in *lstart*) return 1 ;; esac; command ps "$@"; }\n'); + const unreadable = run(['live', '--root', dir], { BASH_ENV: blind }); + expect(unreadable.status, unreadable.stdout + unreadable.stderr).toBe(0); + expect(unreadable.stderr).not.toContain('reclaimed'); + for (const record of ['linux not-a-boot-id 12', 'linux ' + '0'.repeat(36) + ' x1', recorded.startsWith('linux ') ? 'ps Thu Jan 1 00:00:00 1970' : 'linux ' + '0'.repeat(8) + '-0000-0000-0000-' + '0'.repeat(12) + ' 1']) { + fs.writeFileSync(file, record + '\n'); + const r = run(['live', '--root', dir]); + expect(r.status, record + ': ' + r.stderr).toBe(0); + expect(fs.existsSync(file), record).toBe(true); + } + }); + + test('the recorded identity does not depend on the caller\'s timezone or locale', () => { + const { dir } = checkout(); + const holder = sleeper(); + expect(run(['touch', '--root', dir, '--pid', String(holder)], { TZ: 'UTC', LC_ALL: 'C' }).status).toBe(0); + const r = run(['live', '--root', dir], { TZ: 'Pacific/Kiritimati', LC_ALL: 'C.UTF-8' }); + expect(r.status, r.stdout + r.stderr).toBe(0); + expect(r.stderr).not.toContain('reclaimed'); + }); + + test('a renewal that lands while a stale lease is being reclaimed survives (rename aside, then restore)', () => { + const { dir, leases } = checkout(); + const holder = sleeper(); + const file = path.join(leases, String(holder)); + expect(run(['touch', '--root', dir, '--pid', String(holder)]).status).toBe(0); + const fresh = fs.readFileSync(file, 'utf8'); + fs.writeFileSync(file, otherStart(fresh.trim()) + '\n'); + const renewed = path.join(dir, 'renewed'); + fs.writeFileSync(renewed, fresh); + // The first rename-aside of the lease is preceded by the holder's renewal, as if touch ran between the stale verdict and the rename. + const racing = bashEnv(dir, `mv() { case "$*" in *".reclaim."*) command cp "${renewed}" "${file}" ;; esac; command mv "$@"; }\n`); + const r = run(['live', '--root', dir], { BASH_ENV: racing }); + expect(r.status, r.stdout + r.stderr).toBe(0); + expect(r.stderr).not.toContain('reclaimed'); + expect(fs.readFileSync(file, 'utf8')).toBe(fresh); + expect(fs.readdirSync(leases).filter((n) => n.startsWith('.'))).toEqual([]); + }); + test('a lease without a recorded identity (older gstack) counts while fresh and is stale past the 120-minute window', () => { const { dir, leases } = checkout(); const holder = sleeper();