mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-21 07:33:14 +01:00
fix(gstack-lease): take the lease lock with mkdir, not ln -s, so Git Bash on Windows stops waiting out 300 tries (#3330)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
c23d7d48fe
commit
03acc162d1
5 files changed
+302
-48
No files matched your search
@@ -0,0 +1,86 @@
|
||||
/**
|
||||
* #3330: gstack-lease took its lock with `ln -s <holder token> .lock`. Git
|
||||
* for Windows' default symlink mode (MSYS unset) cannot link to a target that
|
||||
* does not exist, so the lock was never taken and every skill start waited out
|
||||
* 300 tries (one report: 499 s) and still wrote no lease. The lock is now a
|
||||
* directory (mkdir) with an owner file. test/gstack-lease.test.ts pins the full
|
||||
* contract but is POSIX-bound (mode bits, sleep children), so this file is the
|
||||
* one the curated windows-free-tests lane runs: it launches the bin through
|
||||
* lib/script-invocation.ts, which is Git Bash on windows-latest.
|
||||
*/
|
||||
import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { BIN_DIR, runBashSync, runScriptSync } from '../lib/script-invocation';
|
||||
|
||||
const LEASE = path.join(BIN_DIR, 'gstack-lease');
|
||||
const fwd = (p: string) => p.replace(/\\/g, '/');
|
||||
const dirs: string[] = [];
|
||||
afterEach(() => {
|
||||
for (const d of dirs.splice(0)) fs.rmSync(d, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// No inherited lease dir from the test preload, and Git Bash's default symlink mode (no winsymlinks).
|
||||
const { GSTACK_LEASE_DIR: _preload, MSYS: _msys, CYGWIN: _cygwin, GSTACK_LEASE_LOCK_TRIES: _tries, ...baseEnv } = process.env as Record<string, string>;
|
||||
|
||||
function checkout() {
|
||||
const dir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-lease-gb-')));
|
||||
dirs.push(dir);
|
||||
const r = spawnSync('git', ['init', '-q', dir], { encoding: 'utf8', timeout: 30_000 });
|
||||
if (r.status !== 0) throw new Error(r.stderr);
|
||||
return { dir, leases: path.join(dir, '.git', 'gstack-leases') };
|
||||
}
|
||||
|
||||
/** A BASH_ENV file in which `ln -s` behaves as Git Bash's default: linking to a missing target fails. */
|
||||
function gitBashLn(dir: string): string {
|
||||
const file = path.join(dir, 'git-bash-ln.sh');
|
||||
fs.writeFileSync(file, 'ln() { if [ "$1" = -s ] && [ ! -e "$2" ]; then echo "ln: failed to create symbolic link \'$3\': No such file or directory" >&2; return 1; fi; command ln "$@"; }\n');
|
||||
return file;
|
||||
}
|
||||
|
||||
describe('gstack-lease under Git Bash semantics (#3330)', () => {
|
||||
test('touch takes the lock and writes the lease at once with the default tries, leaving no lock behind', () => {
|
||||
const { dir, leases } = checkout();
|
||||
const started = Date.now();
|
||||
const r = runScriptSync(LEASE, ['touch', '--root', fwd(dir), '--pid', '4242'], { encoding: 'utf8', env: baseEnv, timeout: 120_000 });
|
||||
expect(r.status, `${r.error ?? ''}${r.stdout}${r.stderr}`).toBe(0);
|
||||
expect(Date.now() - started).toBeLessThan(25_000);
|
||||
expect(fs.readdirSync(leases)).toEqual(['4242']);
|
||||
});
|
||||
|
||||
test('a host whose ln -s cannot link to a missing target still takes the lock (negative control: the shim really fails ln -s)', () => {
|
||||
const { dir, leases } = checkout();
|
||||
const shim = gitBashLn(dir);
|
||||
const probe = runBashSync('ln -s "1 2" "$PROBE"', { encoding: 'utf8', env: { ...baseEnv, BASH_ENV: fwd(shim), PROBE: fwd(path.join(dir, 'probe')) }, timeout: 30_000 });
|
||||
expect(probe.status, `${probe.error ?? ''}${probe.stderr}`).not.toBe(0);
|
||||
expect(probe.stderr).toContain('failed to create symbolic link');
|
||||
const r = runScriptSync(LEASE, ['touch', '--root', fwd(dir), '--pid', '4243'], { encoding: 'utf8', env: { ...baseEnv, BASH_ENV: fwd(shim), GSTACK_LEASE_LOCK_TRIES: '3' }, timeout: 120_000 });
|
||||
expect(r.status, `${r.error ?? ''}${r.stdout}${r.stderr}`).toBe(0);
|
||||
expect(fs.readdirSync(leases)).toEqual(['4243']);
|
||||
});
|
||||
|
||||
test('a lock held by a live Git Bash process is kept (touch exits 3 naming it); a dead holder\'s lock is broken', () => {
|
||||
const { dir, leases } = checkout();
|
||||
const script = [
|
||||
'set -u',
|
||||
'mkdir -p "$LEASES/.lock" && printf "%s \\n" "$$" > "$LEASES/.lock/owner"',
|
||||
'GSTACK_LEASE_LOCK_TRIES=2 bash "$LEASE" touch --root "$ROOT" --pid 4244; echo "held=$? holder=$$"',
|
||||
'( exit 0 ) & dead=$!; wait "$dead"',
|
||||
'printf "%s \\n" "$dead" > "$LEASES/.lock/owner"',
|
||||
'GSTACK_LEASE_LOCK_TRIES=2 bash "$LEASE" touch --root "$ROOT" --pid 4245; echo "dead=$?"',
|
||||
].join('\n');
|
||||
const r = runBashSync(script, {
|
||||
encoding: 'utf8', timeout: 120_000,
|
||||
env: { ...baseEnv, LEASE: fwd(LEASE), ROOT: fwd(dir), LEASES: fwd(leases) },
|
||||
});
|
||||
const out = `${r.error ?? ''}${r.stdout}${r.stderr}`;
|
||||
expect(r.status, out).toBe(0);
|
||||
const holder = /held=(\d+) holder=(\d+)/.exec(r.stdout);
|
||||
expect(holder?.[1], out).toBe('3');
|
||||
expect(r.stderr).toContain(`locked by live process ${holder?.[2]}`);
|
||||
expect(r.stdout, out).toContain('dead=0');
|
||||
expect(fs.readdirSync(leases).sort()).toEqual(['4245']);
|
||||
});
|
||||
});
|
||||
+105
-17
@@ -59,6 +59,19 @@ function otherStart(recorded: string): string {
|
||||
return recorded.startsWith('linux ') ? recorded.replace(/\d+$/, (n) => String(Number(n) + 1)) : 'ps Thu Jan 1 00:00:00 1970';
|
||||
}
|
||||
|
||||
/** A directory lock as gstack-lease takes it: <path>/owner holds the holder's token. */
|
||||
function plantLock(lockPath: string, token: string) {
|
||||
fs.mkdirSync(lockPath, { recursive: true });
|
||||
fs.writeFileSync(path.join(lockPath, 'owner'), token + '\n');
|
||||
}
|
||||
const ownerOf = (lockPath: string) => fs.readFileSync(path.join(lockPath, 'owner'), 'utf8').trim();
|
||||
|
||||
/** Each lock layout a holder can leave: the directory lock, and the symlink lock an older gstack took. */
|
||||
const LAYOUTS: Array<[string, (lockPath: string, token: string) => void]> = [
|
||||
['directory lock', plantLock],
|
||||
['legacy symlink lock', (lockPath, token) => fs.symlinkSync(token, lockPath)],
|
||||
];
|
||||
|
||||
/** A BASH_ENV file whose functions shadow commands inside the lease script (fault injection). */
|
||||
function bashEnv(dir: string, body: string): string {
|
||||
const file = path.join(dir, 'bash-env.sh');
|
||||
@@ -160,7 +173,7 @@ describe('gstack-lease', () => {
|
||||
const renewed = path.join(dir, 'renewed');
|
||||
fs.writeFileSync(renewed, fresh);
|
||||
// The holder renews just before reclaim takes the lock, as if touch ran after the stale verdict.
|
||||
const racing = bashEnv(dir, `ln() { case "$*" in *"/.lock") command cp "${renewed}" "${file}" ;; esac; command ln "$@"; }\n`);
|
||||
const racing = bashEnv(dir, `mkdir() { case "$*" in *"/.lock") command cp "${renewed}" "${file}" ;; esac; command mkdir "$@"; }\n`);
|
||||
const r = run(['live', '--root', dir], { BASH_ENV: racing });
|
||||
expect(r.status, r.stdout + r.stderr).toBe(0);
|
||||
expect(r.stderr).not.toContain('reclaimed');
|
||||
@@ -168,23 +181,24 @@ describe('gstack-lease', () => {
|
||||
expect(fs.readdirSync(leases).filter((n) => n.startsWith('.'))).toEqual([]);
|
||||
});
|
||||
|
||||
test('a lock whose holder died, or whose pid now belongs to another process, is broken at once', () => {
|
||||
test.each(LAYOUTS)('a %s whose holder died, or whose pid now belongs to another process, is broken at once', (_layout, plant) => {
|
||||
const { dir, leases } = checkout();
|
||||
const holder = sleeper();
|
||||
fs.mkdirSync(leases, { recursive: true });
|
||||
fs.symlinkSync(`${deadPid()} `, path.join(leases, '.lock'));
|
||||
plant(path.join(leases, '.lock'), `${deadPid()} `);
|
||||
const r = run(['touch', '--root', dir, '--pid', String(holder)], { GSTACK_LEASE_LOCK_TRIES: '3' });
|
||||
expect(r.status, r.stdout + r.stderr).toBe(0);
|
||||
expect(fs.existsSync(path.join(leases, '.lock'))).toBe(false);
|
||||
const identity = fs.readFileSync(path.join(leases, String(holder)), 'utf8').trim();
|
||||
expect(identity).toMatch(/^(linux|ps) /);
|
||||
fs.symlinkSync(`${holder} ${otherStart(identity)}`, path.join(leases, '.lock'));
|
||||
plant(path.join(leases, '.lock'), `${holder} ${otherStart(identity)}`);
|
||||
const again = run(['touch', '--root', dir, '--pid', String(holder)], { GSTACK_LEASE_LOCK_TRIES: '3' });
|
||||
expect(again.status, again.stdout + again.stderr).toBe(0);
|
||||
expect(fs.existsSync(path.join(leases, '.lock'))).toBe(false);
|
||||
expect(fs.readdirSync(leases).filter((n) => n.startsWith('.'))).toEqual([]);
|
||||
});
|
||||
|
||||
test('a live holder\'s lock is never broken, however old: touch exits 3 without writing and a reclaim keeps the lease', () => {
|
||||
test.each(LAYOUTS)('a live holder\'s %s is never broken, however old: touch exits 3 without writing and a reclaim keeps the lease', (layout, plant) => {
|
||||
const { dir, leases } = checkout();
|
||||
const owner = sleeper();
|
||||
const holder = sleeper();
|
||||
@@ -192,9 +206,10 @@ describe('gstack-lease', () => {
|
||||
expect(run(['touch', '--root', dir, '--pid', String(holder)]).status).toBe(0);
|
||||
const ownerToken = `${owner} ${fs.readFileSync(path.join(leases, String(owner)), 'utf8').trim()}`;
|
||||
const lockPath = path.join(leases, '.lock');
|
||||
fs.symlinkSync(ownerToken, lockPath);
|
||||
plant(lockPath, ownerToken);
|
||||
const old = new Date(Date.now() - 5 * 60_000);
|
||||
fs.lutimesSync(lockPath, old, old);
|
||||
if (layout === 'directory lock') fs.utimesSync(path.join(lockPath, 'owner'), old, old);
|
||||
const holderFile = path.join(leases, String(holder));
|
||||
const stale = otherStart(fs.readFileSync(holderFile, 'utf8').trim()) + '\n';
|
||||
fs.writeFileSync(holderFile, stale);
|
||||
@@ -207,7 +222,7 @@ describe('gstack-lease', () => {
|
||||
expect(l.stderr).not.toContain('reclaimed');
|
||||
expect(l.stdout).toContain(`${holder} `);
|
||||
expect(fs.existsSync(holderFile)).toBe(true);
|
||||
expect(fs.readlinkSync(lockPath)).toBe(ownerToken);
|
||||
expect(layout === 'directory lock' ? ownerOf(lockPath) : fs.readlinkSync(lockPath)).toBe(ownerToken);
|
||||
});
|
||||
|
||||
test('two breakers of one dead lock: the second rereads under .lock.break and leaves the lock the first one took', () => {
|
||||
@@ -217,21 +232,21 @@ describe('gstack-lease', () => {
|
||||
expect(run(['touch', '--root', dir, '--pid', String(owner)]).status).toBe(0);
|
||||
const ownerToken = `${owner} ${fs.readFileSync(path.join(leases, String(owner)), 'utf8').trim()}`;
|
||||
const lockPath = path.join(leases, '.lock');
|
||||
fs.symlinkSync(`${deadPid()} `, lockPath);
|
||||
plantLock(lockPath, `${deadPid()} `);
|
||||
// Between this breaker's dead verdict and its .lock.break, another breaker removes the dead lock and takes it.
|
||||
const first = bashEnv(dir, `ln() { case "$*" in *"/.lock.break") command rm -f "${lockPath}"; command ln -s "${ownerToken}" "${lockPath}" ;; esac; command ln "$@"; }\n`);
|
||||
const first = bashEnv(dir, `mkdir() { case "$*" in *"/.lock.break") command rm -rf "${lockPath}"; command mkdir "${lockPath}"; printf '%s\\n' "${ownerToken}" > "${lockPath}/owner" ;; esac; command mkdir "$@"; }\n`);
|
||||
const t = run(['touch', '--root', dir, '--pid', String(holder)], { BASH_ENV: first, GSTACK_LEASE_LOCK_TRIES: '3' });
|
||||
expect(t.status, t.stdout + t.stderr).toBe(3);
|
||||
expect(fs.readlinkSync(lockPath)).toBe(ownerToken);
|
||||
expect(fs.existsSync(path.join(leases, '.lock.break'))).toBe(false);
|
||||
expect(ownerOf(lockPath)).toBe(ownerToken);
|
||||
expect(fs.readdirSync(leases).filter((n) => n.startsWith('.')).sort()).toEqual(['.lock']);
|
||||
});
|
||||
|
||||
test('a .lock.break left by a dead breaker is cleared and the dead lock is still broken', () => {
|
||||
test.each(LAYOUTS)('a .lock.break (%s) left by a dead breaker is cleared and the dead lock is still broken', (_layout, plant) => {
|
||||
const { dir, leases } = checkout();
|
||||
const holder = sleeper();
|
||||
fs.mkdirSync(leases, { recursive: true });
|
||||
fs.symlinkSync(`${deadPid()} `, path.join(leases, '.lock'));
|
||||
fs.symlinkSync(`${deadPid()} `, path.join(leases, '.lock.break'));
|
||||
plant(path.join(leases, '.lock'), `${deadPid()} `);
|
||||
plant(path.join(leases, '.lock.break'), `${deadPid()} `);
|
||||
const r = run(['touch', '--root', dir, '--pid', String(holder)], { GSTACK_LEASE_LOCK_TRIES: '5' });
|
||||
expect(r.status, r.stdout + r.stderr).toBe(0);
|
||||
expect(fs.existsSync(path.join(leases, String(holder)))).toBe(true);
|
||||
@@ -249,10 +264,83 @@ describe('gstack-lease', () => {
|
||||
fs.writeFileSync(holderFile, otherStart(fs.readFileSync(holderFile, 'utf8').trim()) + '\n');
|
||||
const lockPath = path.join(leases, '.lock');
|
||||
// While reclaim deletes the stale lease under its lock, the lock changes hands.
|
||||
const swap = bashEnv(dir, `rm() { if [ "$*" = "-f ${holderFile}" ]; then command rm -f "${lockPath}"; command ln -s "${ownerToken}" "${lockPath}"; fi; command rm "$@"; }\n`);
|
||||
const swap = bashEnv(dir, `rm() { if [ "$*" = "-f ${holderFile}" ]; then command rm -rf "${lockPath}"; command mkdir "${lockPath}"; printf '%s\\n' "${ownerToken}" > "${lockPath}/owner"; fi; command rm "$@"; }\n`);
|
||||
const r = run(['live', '--root', dir, '--exclude-ancestry-of', String(owner)], { BASH_ENV: swap });
|
||||
expect(r.stderr).toContain(`reclaimed stale lease ${holder}`);
|
||||
expect(fs.readlinkSync(lockPath)).toBe(ownerToken);
|
||||
expect(ownerOf(lockPath)).toBe(ownerToken);
|
||||
});
|
||||
|
||||
test('a lock held by a running gstack-lease excludes a second taker, which names it; both leases land once it is released', async () => {
|
||||
const { dir, leases } = checkout();
|
||||
const [first, second] = [sleeper(), sleeper()];
|
||||
// The first touch holds the lock across its prune (find runs under the lock).
|
||||
const slow = bashEnv(dir, 'find() { sleep 3; command find "$@"; }\n');
|
||||
const holder = spawn('bash', [LEASE, 'touch', '--root', dir, '--pid', String(first)], { env: { ...baseEnv, BASH_ENV: slow }, stdio: 'ignore' });
|
||||
children.push(holder);
|
||||
const exited = new Promise<number | null>((resolve) => holder.on('exit', resolve));
|
||||
const owner = path.join(leases, '.lock', 'owner');
|
||||
for (let i = 0; i < 100 && !fs.existsSync(owner); i++) await Bun.sleep(20);
|
||||
expect(fs.readFileSync(owner, 'utf8')).toMatch(new RegExp(`^${holder.pid} `));
|
||||
const blocked = run(['touch', '--root', dir, '--pid', String(second)], { GSTACK_LEASE_LOCK_TRIES: '3' });
|
||||
expect(blocked.status, blocked.stdout + blocked.stderr).toBe(3);
|
||||
expect(blocked.stderr).toContain(`locked by live process ${holder.pid};`);
|
||||
expect(await exited).toBe(0);
|
||||
expect(run(['touch', '--root', dir, '--pid', String(second)]).status).toBe(0);
|
||||
expect(fs.readdirSync(leases).sort()).toEqual([String(first), String(second)].sort());
|
||||
});
|
||||
|
||||
test('parallel touches all take the lock in turn and leave no lock, claim or temp file behind', async () => {
|
||||
const { dir, leases } = checkout();
|
||||
const pids = Array.from({ length: 16 }, (_, i) => 4300 + i);
|
||||
const codes = await Promise.all(pids.map((p) => new Promise<number | null>((resolve) => {
|
||||
const c = spawn('bash', [LEASE, 'touch', '--root', dir, '--pid', String(p)], { env: baseEnv, stdio: 'ignore' });
|
||||
children.push(c);
|
||||
c.on('exit', resolve);
|
||||
})));
|
||||
expect(codes).toEqual(pids.map(() => 0));
|
||||
expect(fs.readdirSync(leases).sort()).toEqual(pids.map(String).sort());
|
||||
});
|
||||
|
||||
// #3330: a taker killed between its mkdir and its owner rename leaves an ownerless lock.
|
||||
test('an ownerless lock is broken once every claimant is gone, and kept while a live claimant may still own it', () => {
|
||||
const { dir, leases } = checkout();
|
||||
const holder = sleeper();
|
||||
const lockPath = path.join(leases, '.lock');
|
||||
fs.mkdirSync(lockPath, { recursive: true });
|
||||
fs.writeFileSync(path.join(leases, `.lock.claim.${holder}`), `${holder} \n`);
|
||||
const kept = run(['touch', '--root', dir, '--pid', String(holder)], { GSTACK_LEASE_LOCK_TRIES: '3' });
|
||||
expect(kept.status, kept.stdout + kept.stderr).toBe(3);
|
||||
expect(kept.stderr).toContain('cannot take the lease directory lock');
|
||||
expect(fs.existsSync(path.join(leases, String(holder)))).toBe(false);
|
||||
expect(fs.readdirSync(lockPath)).toEqual([]);
|
||||
|
||||
const dead = deadPid();
|
||||
fs.renameSync(path.join(leases, `.lock.claim.${holder}`), path.join(leases, `.lock.claim.${dead}`));
|
||||
fs.writeFileSync(path.join(leases, `.lock.claim.${dead}`), `${dead} \n`);
|
||||
fs.writeFileSync(path.join(leases, '.lock.claim.1'), '');
|
||||
const broken = run(['touch', '--root', dir, '--pid', String(holder)], { GSTACK_LEASE_LOCK_TRIES: '3' });
|
||||
expect(broken.status, broken.stdout + broken.stderr).toBe(0);
|
||||
expect(fs.existsSync(path.join(leases, String(holder)))).toBe(true);
|
||||
expect(fs.existsSync(lockPath)).toBe(false);
|
||||
|
||||
fs.rmSync(path.join(leases, `.lock.claim.${dead}`));
|
||||
fs.mkdirSync(lockPath);
|
||||
const unclaimed = run(['touch', '--root', dir, '--pid', String(holder)], { GSTACK_LEASE_LOCK_TRIES: '3' });
|
||||
expect(unclaimed.status, unclaimed.stdout + unclaimed.stderr).toBe(0);
|
||||
expect(fs.existsSync(lockPath)).toBe(false);
|
||||
});
|
||||
|
||||
test('a lock that cannot be taken while nobody holds it fails at once instead of waiting out the tries', () => {
|
||||
const { dir, leases } = checkout();
|
||||
const holder = sleeper();
|
||||
const broken = bashEnv(dir, 'mkdir() { case "$*" in *"/.lock") return 1 ;; esac; command mkdir "$@"; }\n');
|
||||
const started = Date.now();
|
||||
const r = run(['touch', '--root', dir, '--pid', String(holder)], { BASH_ENV: broken });
|
||||
expect(r.status, r.stdout + r.stderr).toBe(3);
|
||||
expect(Date.now() - started).toBeLessThan(10_000);
|
||||
expect(r.stderr).toContain('cannot take the lease directory lock');
|
||||
expect(r.stderr).not.toContain('locked by live process');
|
||||
expect(fs.readdirSync(leases)).toEqual([]);
|
||||
});
|
||||
|
||||
test('a lease without a recorded identity (older gstack) counts while fresh and is stale past the 120-minute window', () => {
|
||||
@@ -333,7 +421,7 @@ describe('gstack-lease', () => {
|
||||
fs.writeFileSync(path.join(state, 'config.yaml'), 'update_check: false\n');
|
||||
const leaseDir = path.join(state, 'leases');
|
||||
fs.mkdirSync(leaseDir);
|
||||
fs.symlinkSync(`${sleeper()} `, path.join(leaseDir, '.lock'));
|
||||
plantLock(path.join(leaseDir, '.lock'), `${sleeper()} `);
|
||||
const r = spawnSync(START, ['--skill', 'testskill', '--parent-pid', '424244'], {
|
||||
encoding: 'utf8', cwd: home, env: { PATH: process.env.PATH!, HOME: home, GSTACK_HOME: state, GSTACK_LEASE_DIR: leaseDir, GSTACK_LEASE_LOCK_TRIES: '2' }, timeout: 30_000,
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user