fix(lease): stale needs proof (unreadable or malformed identities keep the lease), ps identity is timezone-independent, and reclaim never deletes a concurrent renewal

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

bin/gstack-lease

@@ -65,23 +65,48 @@ identity() { # PID — the process's start identity, empty when unknown
   if [ -r "/proc/$1/stat" ] && [ -r /proc/sys/kernel/random/boot_id ]; then
     _st=$(cat "/proc/$1/stat" 2>/dev/null) || return 0
     _st=${_st##*) }
-    printf 'linux %s %s\n' "$(cat /proc/sys/kernel/random/boot_id)" "$(printf '%s\n' "$_st" | awk '{print $20}')"
+    _id="linux $(cat /proc/sys/kernel/random/boot_id 2>/dev/null) $(printf '%s\n' "$_st" | awk '{print $20}')"
   else
-    _ls=$(ps -o lstart= -p "$1" 2>/dev/null | sed 's/^ *//; s/ *$//')
-    [ -n "$_ls" ] && printf 'ps %s\n' "$_ls"
+    _id="ps $(LC_ALL=C TZ=UTC0 ps -o lstart= -p "$1" 2>/dev/null | sed 's/^ *//; s/ *$//')"
   fi
+  valid_identity "$_id" && printf '%s\n' "$_id"
   return 0
 }
-# state FILE PID — alive | dead | stale (the pid now belongs to another process)
+# valid_identity ID — a well-formed identity from either backend
+valid_identity() {
+  case "$1" in
+    "linux "*) printf '%s\n' "$1" | grep -Eq '^linux [0-9a-f-]{36} [0-9]+$' ;;
+    "ps "?*) case "$1" in "ps "*[!\ ]*) return 0 ;; esac; return 1 ;;
+    *) return 1 ;;
+  esac
+}
+# state FILE PID — alive | dead | stale. Stale needs proof: two well-formed
+# identities from the same backend that differ. An identity that cannot be read
+# now, a malformed record or a backend change keeps the lease (never reclaim a
+# live session on missing evidence).
 state() {
   alive "$2" || { echo dead; return; }
   _rec=$(head -1 "$1" 2>/dev/null)
-  if [ -n "$_rec" ]; then
-    [ "$_rec" = "$(identity "$2")" ] && echo alive || echo stale
-  elif [ "$(age_min "$1")" -ge 120 ]; then echo stale
+  if [ -z "$_rec" ]; then
+    [ "$(age_min "$1")" -ge 120 ] && echo stale || echo alive
+    return
+  fi
+  _now=$(identity "$2")
+  if valid_identity "$_rec" && [ -n "$_now" ] && [ "${_rec%% *}" = "${_now%% *}" ] && [ "$_rec" != "$_now" ]; then echo stale
   else echo alive
   fi
 }
+# reclaim FILE RECORD — remove a stale lease without losing a concurrent renewal:
+# rename it aside atomically, and put it back (hard link: only if no newer lease
+# was written meanwhile) when what was renamed is not the record judged stale.
+reclaim() {
+  _aside="${1%/*}/.reclaim.${1##*/}.$$"
+  mv -f "$1" "$_aside" 2>/dev/null || return 1
+  if [ "$(head -1 "$_aside" 2>/dev/null)" = "$2" ]; then rm -f "$_aside"; return 0; fi
+  ln "$_aside" "$1" 2>/dev/null
+  rm -f "$_aside"
+  return 1
+}
 age_min() { # FILE — minutes since its mtime
   _m=$(stat -c %Y "$1" 2>/dev/null || stat -f %m "$1" 2>/dev/null || echo 0)
   echo $(( ( $(date +%s) - _m ) / 60 ))
@@ -91,7 +116,8 @@ case "$VERB" in
   touch)
     [ -n "$PID" ] || PID=$(gstack_harness_pid "${FROM:-$PPID}")
     mkdir -p "$DIR" || exit 2
-    identity "$PID" > "$DIR/$PID" || exit 2
+    _tmp="$DIR/.$PID.$$"
+    identity "$PID" > "$_tmp" && mv -f "$_tmp" "$DIR/$PID" || { rm -f "$_tmp"; exit 2; }
     find "$DIR" -mmin +120 -type f -exec rm -f {} + 2>/dev/null
     exit 0 ;;
   list)
@@ -120,8 +146,11 @@ for f in "$DIR"/*; do
   case "$(state "$f" "$p")" in
     dead) continue ;;
     stale)
-      echo "gstack-lease: reclaimed stale lease $p ($(age_min "$f")m; recorded '$(head -1 "$f")', pid $p is now $(ps -o comm= -p "$p" 2>/dev/null | head -1) '$(identity "$p")')" >&2
-      rm -f "$f"; continue ;;
+      _rec=$(head -1 "$f" 2>/dev/null) _age=$(age_min "$f")
+      if reclaim "$f" "$_rec"; then
+        echo "gstack-lease: reclaimed stale lease $p (${_age}m; recorded '$_rec', pid $p is now $(ps -o comm= -p "$p" 2>/dev/null | head -1) '$(identity "$p")')" >&2
+        continue
+      fi ;;
   esac
   echo "$p $(age_min "$f")m $(ps -o comm= -p "$p" 2>/dev/null | head -1 | sed 's|.*/||')"
   FOUND=0

docs/capy-install.md

@@ -86,7 +86,11 @@ while a lease held by a live process outside its own chain exists, naming each
 holder as `pid <p> (<age>m <command>)`. A lease whose pid now belongs to another
 process, or an identity-less lease from an older gstack past 120 minutes, is
 stale: `bin/gstack-lease live` reclaims it (`gstack-lease: reclaimed stale
-lease …`). `bin/gstack-lease list` shows `alive|dead|stale`, and
+lease …`). Stale needs proof: two well-formed identities from the same backend
+that differ (`ps` is read under `LC_ALL=C TZ=UTC0`). An identity that cannot be
+read, a malformed record or a backend change keeps the lease. Reclaiming renames
+the lease aside first and restores it when a renewal landed meanwhile, and
+`touch` publishes by rename, so a reader never sees a half-written record. `bin/gstack-lease list` shows `alive|dead|stale`, and
 `GSTACK_UPGRADE_FORCE=1` overrides.
 
 ## What the installer does

test/capy-skill-wrapper.test.ts

@@ -362,7 +362,8 @@ describe('capy skill: the session hands the installer GSTACK_CAPY_BEHIND', () =>
       expect(busy.last).toMatch(new RegExp(`behind=1 to=1\\.0\\.2 fix=.+ blocked=live gstack sessions hold this installation: pid ${holder.pid} \\(\\d+m sleep\\); fix: .+GSTACK_UPGRADE_FORCE=1.+ \\(UPGRADE_BUSY\\)$`));
       expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b);
       const file = path.join(leaseDir, String(holder.pid));
-      fs.writeFileSync(file, fs.readFileSync(file, 'utf8').trim().replace(/\S+$/, 'x1') + '\n');
+      const recorded = fs.readFileSync(file, 'utf8').trim();
+      fs.writeFileSync(file, (recorded.startsWith('linux ') ? recorded.replace(/\d+$/, (n) => String(Number(n) + 1)) : 'ps Thu Jan  1 00:00:00 1970') + '\n');
       const reclaimed = f.run(['--for', 'autoplan'], { GSTACK_LEASE_DIR: leaseDir });
       expect(reclaimed.stdout).toMatch(/^SELF_UPGRADE: done 1\.0\.1 -> 1\.0\.2$/m);
       expect(f.git(['rev-parse', 'HEAD'])).toBe(f.c);

test/gstack-lease.test.ts

@@ -54,6 +54,18 @@ function deadPid(): number {
   return r.pid!;
 }
 
+/** The same backend's identity for a different process start (what a reused pid shows). */
+function otherStart(recorded: string): string {
+  return recorded.startsWith('linux ') ? recorded.replace(/\d+$/, (n) => String(Number(n) + 1)) : 'ps Thu Jan  1 00:00:00 1970';
+}
+
+/** A BASH_ENV file whose functions shadow commands inside the lease script (fault injection). */
+function bashEnv(dir: string, body: string): string {
+  const file = path.join(dir, 'bash-env.sh');
+  fs.writeFileSync(file, body);
+  return file;
+}
+
 describe('gstack-lease', () => {
   test('bash -n passes and the bin is executable', () => {
     expect(fs.existsSync(LEASE), LEASE).toBe(true);
@@ -102,7 +114,7 @@ describe('gstack-lease', () => {
     expect(run(['touch', '--root', dir, '--pid', String(reused)]).status).toBe(0);
     const recorded = fs.readFileSync(path.join(leases, String(reused)), 'utf8').trim();
     expect(recorded).toMatch(/^(linux [0-9a-f-]+ \d+|ps .+)$/);
-    fs.writeFileSync(path.join(leases, String(reused)), recorded.replace(/\S+$/, 'x1') + '\n');
+    fs.writeFileSync(path.join(leases, String(reused)), otherStart(recorded) + '\n');
     expect(run(['list', '--root', dir]).stdout).toMatch(new RegExp(`^${reused} \\d+m stale$`, 'm'));
     const live = run(['live', '--root', dir]);
     expect(live.status, live.stdout + live.stderr).toBe(1);
@@ -111,6 +123,51 @@ describe('gstack-lease', () => {
     expect(fs.existsSync(path.join(leases, String(reused)))).toBe(false);
   });
 
+  test('missing evidence never reclaims a live lease: an unreadable current identity, a malformed record or another backend keeps it', () => {
+    const { dir, leases } = checkout();
+    const holder = sleeper();
+    const file = path.join(leases, String(holder));
+    expect(run(['touch', '--root', dir, '--pid', String(holder)]).status).toBe(0);
+    const recorded = fs.readFileSync(file, 'utf8').trim();
+    const blind = bashEnv(dir, 'cat() { case "$1" in /proc/*/stat) return 1 ;; esac; command cat "$@"; }\nps() { case "$*" in *lstart*) return 1 ;; esac; command ps "$@"; }\n');
+    const unreadable = run(['live', '--root', dir], { BASH_ENV: blind });
+    expect(unreadable.status, unreadable.stdout + unreadable.stderr).toBe(0);
+    expect(unreadable.stderr).not.toContain('reclaimed');
+    for (const record of ['linux not-a-boot-id 12', 'linux ' + '0'.repeat(36) + ' x1', recorded.startsWith('linux ') ? 'ps Thu Jan  1 00:00:00 1970' : 'linux ' + '0'.repeat(8) + '-0000-0000-0000-' + '0'.repeat(12) + ' 1']) {
+      fs.writeFileSync(file, record + '\n');
+      const r = run(['live', '--root', dir]);
+      expect(r.status, record + ': ' + r.stderr).toBe(0);
+      expect(fs.existsSync(file), record).toBe(true);
+    }
+  });
+
+  test('the recorded identity does not depend on the caller\'s timezone or locale', () => {
+    const { dir } = checkout();
+    const holder = sleeper();
+    expect(run(['touch', '--root', dir, '--pid', String(holder)], { TZ: 'UTC', LC_ALL: 'C' }).status).toBe(0);
+    const r = run(['live', '--root', dir], { TZ: 'Pacific/Kiritimati', LC_ALL: 'C.UTF-8' });
+    expect(r.status, r.stdout + r.stderr).toBe(0);
+    expect(r.stderr).not.toContain('reclaimed');
+  });
+
+  test('a renewal that lands while a stale lease is being reclaimed survives (rename aside, then restore)', () => {
+    const { dir, leases } = checkout();
+    const holder = sleeper();
+    const file = path.join(leases, String(holder));
+    expect(run(['touch', '--root', dir, '--pid', String(holder)]).status).toBe(0);
+    const fresh = fs.readFileSync(file, 'utf8');
+    fs.writeFileSync(file, otherStart(fresh.trim()) + '\n');
+    const renewed = path.join(dir, 'renewed');
+    fs.writeFileSync(renewed, fresh);
+    // The first rename-aside of the lease is preceded by the holder's renewal, as if touch ran between the stale verdict and the rename.
+    const racing = bashEnv(dir, `mv() { case "$*" in *".reclaim."*) command cp "${renewed}" "${file}" ;; esac; command mv "$@"; }\n`);
+    const r = run(['live', '--root', dir], { BASH_ENV: racing });
+    expect(r.status, r.stdout + r.stderr).toBe(0);
+    expect(r.stderr).not.toContain('reclaimed');
+    expect(fs.readFileSync(file, 'utf8')).toBe(fresh);
+    expect(fs.readdirSync(leases).filter((n) => n.startsWith('.'))).toEqual([]);
+  });
+
   test('a lease without a recorded identity (older gstack) counts while fresh and is stale past the 120-minute window', () => {
     const { dir, leases } = checkout();
     const holder = sleeper();