mirror of
https://github.com/garrytan/gstack.git
synced 2026-07-05 23:57:53 +02:00
f42d7bac6d
terminal-agent could die independently of the server — SIGKILL from the OS
OOM killer, an uncaught exception under PTY churn, an external `pkill` from
a sibling debugging session. Pre-v1.44 the sidebar would observe the broken
connection and stay broken until the user reloaded the sidebar. Now a 60s
ticker checks the recorded agent PID and respawns via the shared
spawnTerminalAgent helper when dead.
Identity-based liveness (T4 from the eng review):
* Uses readAgentRecord + isProcessAlive (signal 0 probe), not a name match.
* Slow-but-alive agents intentionally fall through — respawning around a
living agent would create split-brain (two agents writing the port
file, tokens diverging between them, mystery upgrade 401s).
* Pairs with the v1.44 generation counter in /internal/* loopback calls:
if a stale agent does come back to life mid-cycle, its X-Browse-Gen
no longer matches and the parent's calls 409 cleanly.
Crash-loop guard:
* 3 respawn attempts inside a rolling 60s window → stop trying. A daemon
up for a week with one crash a day shouldn't trip the guard.
* On trip: one-line error to console (`respawn guard tripped`) and the
watchdog goes dormant. Manual restart via the sidebar Restart button
is the explicit signal to re-arm (added in Commit 2 of the larger PR).
Shared spawn path (refactor):
* New spawnTerminalAgent(opts) in terminal-agent-control.ts handles:
prior-PID cleanup → spawn → record stash. Both the CLI cold-start path
in cli.ts and the new server.ts watchdog route through it. Removes the
copy-paste between them; future env wiring lands in one place.
Gated on cfg.ownsTerminalAgent — embedders that pre-launch their own PTY
server (gbrowser phoenix overlay) still own the full lifecycle.
GSTACK_AGENT_WATCHDOG_TICK_MS env knob compresses the 60s tick for e2e
tests without 60s waits per assertion.
Tests:
* browse/test/terminal-agent-watchdog.test.ts — 7 static-grep tripwires
for the load-bearing invariants (ownsTerminalAgent gate, PID-based
liveness, crash-loop guard with window pruning, shutdown cleanup,
CLI cold-start uses the same helper, env knob exists).
* Live process-kill tests belong in the e2e tier; cheaper invariants
here catch refactor regressions in ~1ms each.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
92 lines
4.4 KiB
TypeScript
92 lines
4.4 KiB
TypeScript
import { describe, test, expect } from 'bun:test';
|
|
import * as fs from 'fs';
|
|
import * as path from 'path';
|
|
|
|
// v1.44 terminal-agent watchdog — static-grep invariants.
|
|
//
|
|
// The watchdog respawns terminal-agent when its PID dies. Live process-tree
|
|
// tests would require spawning, killing, and observing across two real Bun
|
|
// processes — slow and flaky in the free tier. These tripwires defend the
|
|
// load-bearing properties: identity-based liveness check (not name match),
|
|
// crash-loop guard, gated on ownsTerminalAgent, and cleared on shutdown.
|
|
|
|
const SERVER_TS = path.resolve(new URL(import.meta.url).pathname, '..', '..', 'src', 'server.ts');
|
|
const CONTROL_TS = path.resolve(new URL(import.meta.url).pathname, '..', '..', 'src', 'terminal-agent-control.ts');
|
|
|
|
describe('terminal-agent watchdog (v1.44+)', () => {
|
|
test('1. spawnTerminalAgent helper exists with PID return type', () => {
|
|
const src = fs.readFileSync(CONTROL_TS, 'utf-8');
|
|
expect(src).toMatch(/export function spawnTerminalAgent\(/);
|
|
// Must clean up prior PID before spawning (no zombies).
|
|
expect(src).toContain('readAgentRecord(stateDir)');
|
|
expect(src).toContain('killAgentByRecord(prior');
|
|
expect(src).toContain('clearAgentRecord(stateDir)');
|
|
});
|
|
|
|
test('2. watchdog is gated on ownsTerminalAgent', () => {
|
|
const src = fs.readFileSync(SERVER_TS, 'utf-8');
|
|
// Match the comment + the guard. The guard MUST be a positive check;
|
|
// an inverted check would respawn for embedders and trample their PTY.
|
|
const block = sliceBetween(src, '─── Terminal-Agent Watchdog', 'Factory-scoped validateAuth');
|
|
expect(block).toMatch(/if \(ownsTerminalAgent\)/);
|
|
expect(block).toContain('agentWatchdogInterval = setInterval');
|
|
});
|
|
|
|
test('3. watchdog uses PID liveness, not process name probe', () => {
|
|
const src = fs.readFileSync(SERVER_TS, 'utf-8');
|
|
const block = sliceBetween(src, '─── Terminal-Agent Watchdog', 'Factory-scoped validateAuth');
|
|
// The whole point of the v1.44 watchdog over v1.43- pkill teardown:
|
|
// identity-based liveness. Slow-but-alive agents must NOT trigger
|
|
// respawn (split-brain defense).
|
|
expect(block).toContain('readAgentRecord(stateDir)');
|
|
expect(block).toContain('isProcessAlive(record.pid)');
|
|
// Negative: no executable name-based process lookup. Allow the strings
|
|
// to appear in prose comments (the watchdog doc explains what it
|
|
// replaces), reject only actual invocations.
|
|
expect(block).not.toMatch(/spawnSync\s*\(\s*['"]pkill/);
|
|
expect(block).not.toMatch(/Bun\.spawn\s*\(\s*\[\s*['"]pgrep/);
|
|
});
|
|
|
|
test('4. crash-loop guard with rolling window', () => {
|
|
const src = fs.readFileSync(SERVER_TS, 'utf-8');
|
|
const block = sliceBetween(src, '─── Terminal-Agent Watchdog', 'Factory-scoped validateAuth');
|
|
expect(block).toContain('RESPAWN_GUARD_WINDOW_MS = 60_000');
|
|
expect(block).toContain('RESPAWN_GUARD_MAX = 3');
|
|
expect(block).toContain('respawnHistory');
|
|
expect(block).toContain('agentRespawnGuardTripped');
|
|
// Window pruning: old entries must be evicted before counting toward
|
|
// the limit. Otherwise a daemon up for a week with one crash a day
|
|
// would eventually trip the guard.
|
|
expect(block).toMatch(/respawnHistory\.shift\(\)/);
|
|
});
|
|
|
|
test('5. watchdog interval is cleared on shutdown', () => {
|
|
const src = fs.readFileSync(SERVER_TS, 'utf-8');
|
|
expect(src).toContain('if (agentWatchdogInterval) clearInterval(agentWatchdogInterval)');
|
|
});
|
|
|
|
test('6. tick interval is env-overridable for tests', () => {
|
|
const src = fs.readFileSync(SERVER_TS, 'utf-8');
|
|
expect(src).toContain('GSTACK_AGENT_WATCHDOG_TICK_MS');
|
|
});
|
|
|
|
test('7. CLI cold-start path uses the same spawnTerminalAgent helper', () => {
|
|
const cli = fs.readFileSync(
|
|
path.resolve(new URL(import.meta.url).pathname, '..', '..', 'src', 'cli.ts'),
|
|
'utf-8',
|
|
);
|
|
// Otherwise the CLI and watchdog could drift on spawn env/cwd, and
|
|
// teardown invariants tested against one would silently miss the other.
|
|
expect(cli).toContain('spawnTerminalAgent({');
|
|
expect(cli).toContain("from './terminal-agent-control'");
|
|
});
|
|
});
|
|
|
|
function sliceBetween(source: string, start: string, end: string): string {
|
|
const i = source.indexOf(start);
|
|
if (i === -1) throw new Error(`marker not found: ${start}`);
|
|
const j = source.indexOf(end, i + start.length);
|
|
if (j === -1) throw new Error(`end marker not found: ${end}`);
|
|
return source.slice(i, j);
|
|
}
|