mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-27 15:11:47 +02:00
v1.88.1.0 fix: harden credential boundaries and owned state (#2942)
* fix(settings): preserve symlinked settings targets
Resolve the selected target for locking, mutation, backup, and rollback; refuse target changes and preserve private modes. Addresses #2830.
* fix(redact): bind masking to original detected spans
Inspired by #2929's anchored-span diagnosis; independently implemented using normalization offsets. Addresses #2930 and the relocation portion of #2912 without changing detection sensitivity.
* fix(evals): exclude operator credentials from prefix admission
Adapts the credential-suffix screen proposed in #2636, with real launched-child regression coverage and deliberate provider-auth exceptions.
* fix(artifacts): retain custom allowlist rules on reinitialization
Preserve the exact user-owned suffix and publish only a successfully assembled replacement. Independently implements the repair reported in #2907.
* test(cso): verify exact masked reads and unmaskable payload refusal
* fix(cso): preserve exact filesystem identities through lease recovery
Preserve 64-bit device/inode identity and nanosecond race checks. Add native NTFS lifecycle coverage for #2927; retain ambiguous legacy-state refusal without claiming Windows PID-reuse recovery is resolved.
* fix(redact): bind pre-push scans to destination and preserve seam context
Uses #2935 (bd07318) as source evidence for push-target range and slice-overlap defects. Independently implemented; no cherry-pick or release metadata adoption.
* test(ci): gate native agent ownership and settings links on macOS
* fix(browse): bind agent lifetimes and cleanup to owned generations
Uses #2931 by Chris Hutton / Claude Fable 5.1 as attributed design input; independently implemented without broad sweeps or copied code. Keep uncertain children and locks rather than deleting foreign state.
* test(ci): include concurrent shutdown controls in the native macOS gate
* v1.88.1.0 fix: harden credential boundaries and owned state
* fix(redact): preserve target provenance and scan boundary semantics
* test(artifacts): read managed rules from atomic allowlist assembly
* fix: preserve native exit observations and fixture prerequisites
* fix: preserve UTF-16 offsets through redaction normalization
This commit is contained in:
Garry Tan
+18
-2
@@ -527,6 +527,7 @@ async function startServer(extraEnv?: Record<string, string>): Promise<ServerSta
|
||||
// Parse as int so stray whitespace ("0\n") still opts out — matches the
|
||||
// server's own parseInt at server.ts:760.
|
||||
const parentPid = parseInt(process.env.BROWSE_PARENT_PID || '', 10) === 0 ? '0' : String(process.pid);
|
||||
let spawnedServer: { pid: number; startTime: string } | null = null;
|
||||
|
||||
if (IS_WINDOWS && NODE_SERVER_SCRIPT) {
|
||||
// Windows: Bun.spawn() + proc.unref() doesn't truly detach on Windows —
|
||||
@@ -561,12 +562,14 @@ async function startServer(extraEnv?: Record<string, string>): Promise<ServerSta
|
||||
// (PPID=1, STAT=Ss) and survives the spawning shell's exit. Mirrors
|
||||
// the Windows path's rationale — same root cause, different OS API.
|
||||
const daemonLogFd = openDaemonLogSink();
|
||||
nodeSpawn('bun', ['run', SERVER_SCRIPT], {
|
||||
const child = nodeSpawn('bun', ['run', SERVER_SCRIPT], {
|
||||
detached: true,
|
||||
windowsHide: true,
|
||||
stdio: ['ignore', daemonLogFd, daemonLogFd],
|
||||
env: { ...process.env, BROWSE_STATE_FILE: config.stateFile, BROWSE_PARENT_PID: parentPid, ...extraEnv },
|
||||
}).unref();
|
||||
});
|
||||
child.unref();
|
||||
if (child.pid) spawnedServer = { pid: child.pid, startTime: readPidStartTime(child.pid) };
|
||||
}
|
||||
|
||||
// Wait for server to become healthy.
|
||||
@@ -592,6 +595,17 @@ async function startServer(extraEnv?: Record<string, string>): Promise<ServerSta
|
||||
return lateState;
|
||||
}
|
||||
|
||||
if (spawnedServer?.startTime) {
|
||||
const { pid, startTime } = spawnedServer;
|
||||
const stillOurs = () => readPidStartTime(pid) === startTime && readPidCmdline(pid).split(/\s+/).includes(SERVER_SCRIPT);
|
||||
if (stillOurs()) {
|
||||
safeKill(pid, 'SIGTERM');
|
||||
const deadline = Date.now() + 500;
|
||||
while (Date.now() < deadline && stillOurs()) await Bun.sleep(50);
|
||||
if (stillOurs()) safeKill(pid, 'SIGKILL');
|
||||
}
|
||||
}
|
||||
|
||||
// Server didn't start in time — check the on-disk startup error log.
|
||||
// Both platforms now spawn with stdio: 'ignore', so the server writes
|
||||
// errors to disk for the CLI to read (see server.ts start().catch).
|
||||
@@ -1664,6 +1678,7 @@ Refs: After 'snapshot', use @e1, @e2... as selectors:
|
||||
const newPid = spawnTerminalAgent({
|
||||
stateFile: config.stateFile,
|
||||
serverPort: newState.port,
|
||||
ownerPid: newState.pid,
|
||||
cwd: config.projectDir,
|
||||
});
|
||||
if (newPid) {
|
||||
@@ -1756,6 +1771,7 @@ Refs: After 'snapshot', use @e1, @e2... as selectors:
|
||||
spawnTerminalAgent({
|
||||
stateFile: config.stateFile,
|
||||
serverPort: respawned.port,
|
||||
ownerPid: respawned.pid,
|
||||
cwd: config.projectDir,
|
||||
});
|
||||
} catch (err: any) {
|
||||
|
||||
+106
-25
@@ -51,7 +51,7 @@ import { safeUnlink, safeUnlinkQuiet, safeKill } from './error-handling';
|
||||
import {
|
||||
findAvailablePort, formatExplicitPortUnavailableError, formatRandomPortUnavailableError,
|
||||
} from './port-allocator';
|
||||
import { readAgentRecord, killAgentByRecord, agentRecordPath, spawnTerminalAgent } from './terminal-agent-control';
|
||||
import { acquireAgentStateLock, readAgentRecord, clearAgentRecord, isOurAgent, isAgentRecordLive, isAgentRecordGone, stopAgentByRecord, spawnTerminalAgent } from './terminal-agent-control';
|
||||
import { isProcessAlive } from './error-handling';
|
||||
import { sanitizeBody, stripLoneSurrogateEscapes, stripLoneSurrogates, sanitizeReplacer } from './sanitize';
|
||||
import { startSocksBridge, testUpstream, type BridgeHandle } from './socks-bridge';
|
||||
@@ -75,6 +75,18 @@ import * as net from 'net';
|
||||
import * as path from 'path';
|
||||
import * as crypto from 'crypto';
|
||||
|
||||
const SERVER_INSTANCE_ID = crypto.randomUUID();
|
||||
|
||||
function removeOwnedDaemonStateQuiet(): void {
|
||||
try {
|
||||
const release = acquireAgentStateLock(path.dirname(config.stateFile), 0);
|
||||
try {
|
||||
const state = JSON.parse(fs.readFileSync(config.stateFile, 'utf8'));
|
||||
if (state.pid === process.pid && state.instanceId === SERVER_INSTANCE_ID) safeUnlinkQuiet(config.stateFile);
|
||||
} finally { release(); }
|
||||
} catch {}
|
||||
}
|
||||
|
||||
// ─── Unicode Sanitization ───────────────────────────────────────
|
||||
// Unpaired UTF-16 surrogate halves (\uD800–\uDFFF) in page DOM text, OCR
|
||||
// output, and other CDP-sourced strings are rejected by JSON consumers
|
||||
@@ -466,11 +478,15 @@ async function startTunnel(opts: {
|
||||
console.log(`[browse] Tunnel listener bound on 127.0.0.1:${tunnelPort}, ngrok → ${tunnelUrl}`);
|
||||
|
||||
// Update state file
|
||||
const stateContent = JSON.parse(fs.readFileSync(config.stateFile, 'utf-8'));
|
||||
stateContent.tunnel = { url: tunnelUrl, domain: domain || null, startedAt: new Date().toISOString() };
|
||||
const tmpState = tmpStatePath();
|
||||
fs.writeFileSync(tmpState, JSON.stringify(stateContent, null, 2), { mode: 0o600 });
|
||||
fs.renameSync(tmpState, config.stateFile);
|
||||
const releaseStateLock = acquireAgentStateLock(config.stateDir);
|
||||
try {
|
||||
const stateContent = JSON.parse(fs.readFileSync(config.stateFile, 'utf-8'));
|
||||
if (stateContent.pid !== process.pid || stateContent.instanceId !== SERVER_INSTANCE_ID) throw new Error('daemon state was replaced');
|
||||
stateContent.tunnel = { url: tunnelUrl, domain: domain || null, startedAt: new Date().toISOString() };
|
||||
const tmpState = tmpStatePath();
|
||||
fs.writeFileSync(tmpState, JSON.stringify(stateContent, null, 2), { mode: 0o600 });
|
||||
fs.renameSync(tmpState, config.stateFile);
|
||||
} finally { releaseStateLock(); }
|
||||
|
||||
return { ok: true, url: tunnelUrl! };
|
||||
} catch (err: any) {
|
||||
@@ -735,6 +751,7 @@ const idleCheckInterval = setInterval(idleCheckTick, 60_000);
|
||||
// Production code must never import this — see `idle timer + onDisconnect
|
||||
// dual-instance fix` describe block for usage.
|
||||
export const __testInternals__ = {
|
||||
serverInstanceId: SERVER_INSTANCE_ID,
|
||||
idleCheckTick,
|
||||
// Watchdog seams (watchdog.test.ts): drive the 15s poll against an
|
||||
// arbitrary (dead) PID, trigger the handoff-promotion suppression exactly
|
||||
@@ -1432,9 +1449,7 @@ if (import.meta.main) {
|
||||
// Windows: taskkill /F bypasses SIGTERM, but 'exit' fires for some shutdown paths.
|
||||
// Defense-in-depth — primary cleanup is the CLI's stale-state detection via health check.
|
||||
if (process.platform === 'win32') {
|
||||
process.on('exit', () => {
|
||||
safeUnlinkQuiet(config.stateFile);
|
||||
});
|
||||
process.on('exit', removeOwnedDaemonStateQuiet);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1452,6 +1467,7 @@ function emergencyCleanup() {
|
||||
if (fs.existsSync(config.stateFile)) {
|
||||
const raw = fs.readFileSync(config.stateFile, 'utf-8');
|
||||
const state = JSON.parse(raw);
|
||||
if (state.pid !== process.pid || state.instanceId !== SERVER_INSTANCE_ID) return;
|
||||
if (state.xvfbPid && state.xvfbStartTime) {
|
||||
// Lazy import — emergencyCleanup may run on platforms where
|
||||
// ./xvfb's Linux-specific helpers fail to load. Best effort.
|
||||
@@ -1472,7 +1488,7 @@ function emergencyCleanup() {
|
||||
if (activeBrowserManager.getConnectionMode() === 'headed' || process.env.BROWSE_HEADED === '1') {
|
||||
cleanSingletonLocks(resolveChromiumProfile());
|
||||
}
|
||||
safeUnlinkQuiet(config.stateFile);
|
||||
removeOwnedDaemonStateQuiet();
|
||||
}
|
||||
// Same import.meta.main gate as SIGINT/SIGTERM — embedders register their
|
||||
// own crash handlers.
|
||||
@@ -1582,6 +1598,7 @@ export function buildFetchHandler(cfg: ServerConfig): ServerHandle {
|
||||
AGENT_WATCHDOG_TICK_MS * (RESPAWN_GUARD_MAX + 2),
|
||||
);
|
||||
let agentRespawnGuardTripped = false;
|
||||
let consecutiveSpawnFailures = 0;
|
||||
|
||||
if (ownsTerminalAgent) {
|
||||
agentWatchdogInterval = setInterval(() => {
|
||||
@@ -1594,7 +1611,7 @@ export function buildFetchHandler(cfg: ServerConfig): ServerHandle {
|
||||
// intentionally fall through here — split-brain is worse than
|
||||
// unresponsiveness, and slow recovery is handled by the user via
|
||||
// restart.
|
||||
if (record && isProcessAlive(record.pid)) return;
|
||||
if (record && !isAgentRecordGone(record)) return;
|
||||
// Either no record (never spawned, or cleaned up after crash) or
|
||||
// PID is dead. Try to respawn.
|
||||
const now = Date.now();
|
||||
@@ -1617,13 +1634,20 @@ export function buildFetchHandler(cfg: ServerConfig): ServerHandle {
|
||||
cwd: cfg.config.projectDir,
|
||||
});
|
||||
if (pid) {
|
||||
consecutiveSpawnFailures = 0;
|
||||
console.log(`[browse] terminal-agent respawned by watchdog (PID: ${pid})`);
|
||||
} else {
|
||||
consecutiveSpawnFailures++;
|
||||
console.warn('[browse] terminal-agent respawn skipped — script not found on disk');
|
||||
}
|
||||
} catch (err: any) {
|
||||
consecutiveSpawnFailures++;
|
||||
console.warn('[browse] terminal-agent respawn failed:', err?.message || err);
|
||||
}
|
||||
if (consecutiveSpawnFailures >= RESPAWN_GUARD_MAX) {
|
||||
agentRespawnGuardTripped = true;
|
||||
console.error('[browse] terminal-agent respawn guard tripped after repeated failed starts — manual restart required');
|
||||
}
|
||||
}, AGENT_WATCHDOG_TICK_MS);
|
||||
// Detach the watchdog timer from Node's event-loop ref count so a
|
||||
// healthy idle process can still exit cleanly if everything else is
|
||||
@@ -1654,23 +1678,44 @@ export function buildFetchHandler(cfg: ServerConfig): ServerHandle {
|
||||
// State and terminal files belong to this instance, including embedders
|
||||
// whose cfg differs from the standalone daemon's module-level config.
|
||||
const config = cfg.config;
|
||||
const stateOwner = (() => {
|
||||
try { return JSON.parse(fs.readFileSync(config.stateFile, 'utf8')); } catch { return null; }
|
||||
})();
|
||||
const foreignState = Number.isSafeInteger(stateOwner?.pid) && stateOwner.pid > 0
|
||||
&& (stateOwner.pid !== process.pid || (stateOwner.instanceId && stateOwner.instanceId !== SERVER_INSTANCE_ID));
|
||||
|
||||
console.log('[browse] Shutting down...');
|
||||
if (ownsTerminalAgent) {
|
||||
if (ownsTerminalAgent && !foreignState) {
|
||||
// Identity-based kill (v1.44+). Replaces the v1.43- `pkill -f
|
||||
// terminal-agent\.ts` regex teardown which matched sibling gstack
|
||||
// sessions on the same host. Only the PID recorded in
|
||||
// `<stateDir>/terminal-agent-pid` by THIS daemon's agent is signaled.
|
||||
try {
|
||||
const stateDir = path.dirname(config.stateFile);
|
||||
const record = readAgentRecord(stateDir);
|
||||
if (record) killAgentByRecord(record, 'SIGTERM');
|
||||
const releaseAgentLock = acquireAgentStateLock(stateDir);
|
||||
try {
|
||||
let currentState: { pid?: number; instanceId?: string } | null = null;
|
||||
try { currentState = JSON.parse(fs.readFileSync(config.stateFile, 'utf8')); } catch {}
|
||||
if (currentState?.pid && (currentState.pid !== process.pid
|
||||
|| (currentState.instanceId && currentState.instanceId !== SERVER_INSTANCE_ID))) {
|
||||
console.warn('[browse] terminal-agent state now belongs to a successor; retaining its files');
|
||||
} else {
|
||||
const record = readAgentRecord(stateDir);
|
||||
const agentStopped = !record || (record.pid !== 0
|
||||
&& (!isAgentRecordLive(record) || (isOurAgent(record, process.pid) && stopAgentByRecord(record))));
|
||||
const current = readAgentRecord(stateDir);
|
||||
if (agentStopped && (!record || (current?.pid === record.pid && current.gen === record.gen))) {
|
||||
safeUnlinkQuiet(path.join(stateDir, 'terminal-port'));
|
||||
safeUnlinkQuiet(path.join(stateDir, 'terminal-internal-token'));
|
||||
if (record) clearAgentRecord(stateDir, record);
|
||||
} else if (!agentStopped) {
|
||||
console.warn('[browse] terminal-agent identity or exit could not be confirmed; retaining its record');
|
||||
}
|
||||
}
|
||||
} finally { releaseAgentLock(); }
|
||||
} catch (err: any) {
|
||||
console.warn('[browse] Failed to kill terminal-agent:', err.message);
|
||||
console.warn('[browse] Failed to stop terminal-agent; retaining its state:', err.message);
|
||||
}
|
||||
safeUnlinkQuiet(path.join(path.dirname(config.stateFile), 'terminal-port'));
|
||||
safeUnlinkQuiet(path.join(path.dirname(config.stateFile), 'terminal-internal-token'));
|
||||
safeUnlinkQuiet(agentRecordPath(path.dirname(config.stateFile)));
|
||||
}
|
||||
try { detachSession(); } catch (err: any) {
|
||||
console.warn('[browse] Failed to detach CDP session:', err.message);
|
||||
@@ -1712,7 +1757,17 @@ export function buildFetchHandler(cfg: ServerConfig): ServerHandle {
|
||||
if (cfgBrowserManager.getConnectionMode() === 'headed') {
|
||||
cleanSingletonLocks(resolveChromiumProfile());
|
||||
}
|
||||
safeUnlinkQuiet(config.stateFile);
|
||||
if (!foreignState) {
|
||||
try {
|
||||
const releaseStateLock = acquireAgentStateLock(path.dirname(config.stateFile));
|
||||
try {
|
||||
const currentState = JSON.parse(fs.readFileSync(config.stateFile, 'utf8'));
|
||||
if (currentState.pid === process.pid && currentState.instanceId === SERVER_INSTANCE_ID) safeUnlinkQuiet(config.stateFile);
|
||||
} finally { releaseStateLock(); }
|
||||
} catch (err: any) {
|
||||
if (fs.existsSync(config.stateFile)) console.warn('[browse] Daemon state cleanup could not confirm ownership:', err?.message || err);
|
||||
}
|
||||
}
|
||||
process.exit(exitCode);
|
||||
}
|
||||
|
||||
@@ -3182,6 +3237,7 @@ export async function start() {
|
||||
// Write state file (atomic: write .tmp then rename)
|
||||
const state: Record<string, unknown> = {
|
||||
pid: process.pid,
|
||||
instanceId: SERVER_INSTANCE_ID,
|
||||
port,
|
||||
token: envCfg.authToken,
|
||||
startedAt: new Date().toISOString(),
|
||||
@@ -3206,7 +3262,28 @@ export async function start() {
|
||||
};
|
||||
const tmpFile = tmpStatePath();
|
||||
fs.writeFileSync(tmpFile, JSON.stringify(state, null, 2), { mode: 0o600 });
|
||||
fs.renameSync(tmpFile, config.stateFile);
|
||||
try {
|
||||
const releaseStateLock = acquireAgentStateLock(config.stateDir);
|
||||
try { fs.renameSync(tmpFile, config.stateFile); } finally { releaseStateLock(); }
|
||||
} catch (err) {
|
||||
safeUnlinkQuiet(tmpFile);
|
||||
throw err;
|
||||
}
|
||||
|
||||
const stateWatchMs = parseInt(process.env.GSTACK_STATE_WATCH_MS || '60000', 10);
|
||||
if (stateWatchMs > 0) {
|
||||
let missed = 0;
|
||||
const stateWatch = setInterval(() => {
|
||||
let owner: { pid?: number; instanceId?: string } | null = null;
|
||||
try { owner = JSON.parse(fs.readFileSync(config.stateFile, 'utf8')); } catch {}
|
||||
if (owner?.pid === process.pid && owner.instanceId === SERVER_INSTANCE_ID) { missed = 0; return; }
|
||||
if (++missed < 2) return;
|
||||
clearInterval(stateWatch);
|
||||
console.warn('[browse] daemon state is no longer reachable; shutting down this instance');
|
||||
handle.shutdown();
|
||||
}, stateWatchMs);
|
||||
(stateWatch as any).unref?.();
|
||||
}
|
||||
|
||||
browserManager.serverPort = port;
|
||||
|
||||
@@ -3338,11 +3415,15 @@ export async function start() {
|
||||
tunnelActive = true;
|
||||
const tunnelPort = boundTunnel.port;
|
||||
console.log(`[browse] Tunnel listener bound (local-only test mode) on 127.0.0.1:${tunnelPort}`);
|
||||
const stateContent = JSON.parse(fs.readFileSync(config.stateFile, 'utf-8'));
|
||||
stateContent.tunnelLocalPort = tunnelPort;
|
||||
const tmpState = tmpStatePath();
|
||||
fs.writeFileSync(tmpState, JSON.stringify(stateContent, null, 2), { mode: 0o600 });
|
||||
fs.renameSync(tmpState, config.stateFile);
|
||||
const releaseStateLock = acquireAgentStateLock(config.stateDir);
|
||||
try {
|
||||
const stateContent = JSON.parse(fs.readFileSync(config.stateFile, 'utf-8'));
|
||||
if (stateContent.pid !== process.pid || stateContent.instanceId !== SERVER_INSTANCE_ID) throw new Error('daemon state was replaced');
|
||||
stateContent.tunnelLocalPort = tunnelPort;
|
||||
const tmpState = tmpStatePath();
|
||||
fs.writeFileSync(tmpState, JSON.stringify(stateContent, null, 2), { mode: 0o600 });
|
||||
fs.renameSync(tmpState, config.stateFile);
|
||||
} finally { releaseStateLock(); }
|
||||
} catch (err: any) {
|
||||
console.error(`[browse] BROWSE_TUNNEL_LOCAL_ONLY=1 listener bind failed: ${err.message}`);
|
||||
}
|
||||
|
||||
@@ -16,9 +16,57 @@
|
||||
*/
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { safeUnlink, safeKill, isProcessAlive } from './error-handling';
|
||||
import * as crypto from 'crypto';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { safeUnlink, isProcessAlive } from './error-handling';
|
||||
import { restrictFilePermissions, mkdirSecure } from './file-permissions';
|
||||
import { atomicWriteSync } from '../../lib/fs-atomic';
|
||||
import { readPidCmdline, readPidStartTime } from './xvfb';
|
||||
|
||||
function agentProcessInfo(pid: number): { startTime: string; commandLine: string } {
|
||||
if (!Number.isSafeInteger(pid) || pid <= 0) return { startTime: '', commandLine: '' };
|
||||
if (process.platform !== 'win32') return { startTime: readPidStartTime(pid), commandLine: readPidCmdline(pid) };
|
||||
const script = `(Get-CimInstance Win32_Process -Filter 'ProcessId = ${pid}' | Select-Object CreationDate,CommandLine | ConvertTo-Json -Compress)`;
|
||||
try {
|
||||
const result = spawnSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', script], { encoding: 'utf8', windowsHide: true, timeout: 2000 });
|
||||
if (result.status !== 0 || !result.stdout) return { startTime: '', commandLine: '' };
|
||||
const processInfo = JSON.parse(result.stdout);
|
||||
return { startTime: processInfo?.CreationDate || '', commandLine: processInfo?.CommandLine || '' };
|
||||
} catch { return { startTime: '', commandLine: '' }; }
|
||||
}
|
||||
|
||||
export function readAgentStartTime(pid: number): string {
|
||||
return agentProcessInfo(pid).startTime;
|
||||
}
|
||||
|
||||
const pendingAgentExits = new Set<any>();
|
||||
|
||||
export function acquireAgentStateLock(stateDir: string, waitMs = 5000): () => void {
|
||||
mkdirSecure(stateDir);
|
||||
const lockPath = path.join(stateDir, 'terminal-agent-pid.lock');
|
||||
const deadline = Date.now() + waitMs;
|
||||
let fd: number;
|
||||
while (true) {
|
||||
try {
|
||||
fd = fs.openSync(lockPath, 'wx', 0o600);
|
||||
break;
|
||||
} catch (err: any) {
|
||||
if (err?.code !== 'EEXIST' || Date.now() >= deadline) {
|
||||
throw new Error(`terminal-agent state lock unavailable at ${lockPath}: ${err?.code || err}; inspect the owning process before manual recovery`);
|
||||
}
|
||||
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 25);
|
||||
}
|
||||
}
|
||||
const owned = fs.fstatSync(fd, { bigint: true });
|
||||
return () => {
|
||||
try {
|
||||
const current = fs.statSync(lockPath, { bigint: true });
|
||||
if (current.dev === owned.dev && current.ino === owned.ino) fs.unlinkSync(lockPath);
|
||||
} catch (err: any) {
|
||||
if (err?.code !== 'ENOENT') throw err;
|
||||
} finally { fs.closeSync(fd); }
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Locate the terminal-agent script on disk. In dev (cli.ts running via
|
||||
@@ -41,11 +89,8 @@ export function resolveTerminalAgentScript(searchHints: { metaDir?: string; exec
|
||||
}
|
||||
|
||||
/**
|
||||
* Spawn a fresh terminal-agent as a detached child. Handles the standard
|
||||
* three steps: kill any prior agent recorded at `<stateDir>/terminal-agent-pid`,
|
||||
* clear the stale record, then `Bun.spawn(['bun', 'run', script], ...)` with
|
||||
* env wiring. Returns the PID of the new agent on success, null when the
|
||||
* agent script can't be located.
|
||||
* Spawn an owned terminal-agent. A prior record is retained until its exact
|
||||
* process exits, and the new generation is recorded before it may bind.
|
||||
*
|
||||
* Used by both the CLI cold-start path (cli.ts) and the v1.44 watchdog in
|
||||
* server.ts. Centralizing here removes a copy-paste between them and means
|
||||
@@ -62,31 +107,82 @@ export function spawnTerminalAgent(opts: {
|
||||
/** Override script lookup for tests. */
|
||||
scriptPath?: string;
|
||||
}): number | null {
|
||||
if (!Number.isSafeInteger(opts.ownerPid) || opts.ownerPid <= 0) throw new Error('terminal-agent requires a daemon owner PID');
|
||||
const stateDir = path.dirname(opts.stateFile);
|
||||
const prior = readAgentRecord(stateDir);
|
||||
if (prior) {
|
||||
killAgentByRecord(prior, 'SIGTERM');
|
||||
clearAgentRecord(stateDir);
|
||||
}
|
||||
const script = opts.scriptPath || resolveTerminalAgentScript();
|
||||
if (!script || !fs.existsSync(script)) return null;
|
||||
const proc = (Bun as any).spawn(['bun', 'run', script], {
|
||||
cwd: opts.cwd || process.cwd(),
|
||||
env: {
|
||||
...process.env,
|
||||
BROWSE_STATE_FILE: opts.stateFile,
|
||||
BROWSE_SERVER_PORT: String(opts.serverPort),
|
||||
BROWSE_OWNER_PID: String(opts.ownerPid),
|
||||
...(opts.extraEnv || {}),
|
||||
},
|
||||
stdio: ['ignore', 'ignore', 'ignore'],
|
||||
// Explicit for the Node fallback path (dist/bun-polyfill.cjs), where the
|
||||
// host default is the opposite of Bun's. A visible console window on every
|
||||
// watchdog respawn is the symptom when this is missing.
|
||||
windowsHide: true,
|
||||
});
|
||||
proc.unref?.();
|
||||
return proc.pid ?? null;
|
||||
const release = acquireAgentStateLock(stateDir);
|
||||
try {
|
||||
const prior = readAgentRecord(stateDir);
|
||||
if (prior) {
|
||||
if (prior.pid === 0) {
|
||||
console.warn('[browse] terminal-agent startup or failed exit remains unconfirmed; retaining its reservation');
|
||||
return null;
|
||||
}
|
||||
if (isAgentRecordLive(prior) && !stopAgentByRecord(prior)) {
|
||||
console.warn(`[browse] terminal-agent PID ${prior.pid} is still running or its identity cannot be confirmed; refusing a second agent`);
|
||||
return null;
|
||||
}
|
||||
clearAgentRecord(stateDir, prior);
|
||||
safeUnlink(path.join(stateDir, 'terminal-port'));
|
||||
safeUnlink(path.join(stateDir, 'terminal-internal-token'));
|
||||
}
|
||||
const ownerStartTime = readAgentStartTime(opts.ownerPid);
|
||||
if (!ownerStartTime) throw new Error('terminal-agent owner identity is unavailable');
|
||||
const gen = crypto.randomBytes(16).toString('base64url');
|
||||
const reservation: AgentRecord = { pid: 0, gen, startedAt: Date.now(), ownerPid: opts.ownerPid, ownerStartTime };
|
||||
writeAgentRecord(stateDir, reservation);
|
||||
let proc: any;
|
||||
try {
|
||||
proc = (Bun as any).spawn(['bun', 'run', script, `--agent-gen=${gen}`], {
|
||||
cwd: opts.cwd || process.cwd(),
|
||||
env: {
|
||||
...process.env,
|
||||
...(opts.extraEnv || {}),
|
||||
BROWSE_STATE_FILE: opts.stateFile,
|
||||
BROWSE_SERVER_PORT: String(opts.serverPort),
|
||||
BROWSE_OWNER_PID: String(opts.ownerPid),
|
||||
BROWSE_OWNER_START_TIME: ownerStartTime,
|
||||
BROWSE_AGENT_GEN: gen,
|
||||
},
|
||||
stdio: ['ignore', 'ignore', 'ignore'],
|
||||
windowsHide: true,
|
||||
});
|
||||
} catch (err) {
|
||||
clearAgentRecord(stateDir, reservation);
|
||||
throw err;
|
||||
}
|
||||
const retainUntilExit = () => {
|
||||
pendingAgentExits.add(proc);
|
||||
proc.exited?.then(() => {
|
||||
try {
|
||||
const releasePending = acquireAgentStateLock(stateDir);
|
||||
try { clearAgentRecord(stateDir, reservation); } finally { releasePending(); }
|
||||
} catch (err) { console.warn('[browse] terminal-agent pending exit cleanup failed:', err); }
|
||||
pendingAgentExits.delete(proc);
|
||||
}, (err: unknown) => console.warn('[browse] terminal-agent exit remains unconfirmed:', err));
|
||||
};
|
||||
const pid = proc.pid;
|
||||
const startTime = pid ? readAgentStartTime(pid) : '';
|
||||
if (!pid || !startTime) {
|
||||
try { proc.kill('SIGTERM'); } catch {}
|
||||
retainUntilExit();
|
||||
throw new Error('terminal-agent process identity is unavailable');
|
||||
}
|
||||
const record: AgentRecord = { pid, gen, startedAt: Date.now(), startTime, ownerPid: opts.ownerPid, ownerStartTime };
|
||||
try {
|
||||
writeAgentRecord(stateDir, record);
|
||||
} catch (err) {
|
||||
if (!stopAgentByRecord(record)) {
|
||||
retainUntilExit();
|
||||
throw new Error(`terminal-agent record update failed and child ${pid} exit is unconfirmed: ${err}`);
|
||||
}
|
||||
clearAgentRecord(stateDir, reservation);
|
||||
throw err;
|
||||
}
|
||||
proc.unref?.();
|
||||
return pid;
|
||||
} finally { release(); }
|
||||
}
|
||||
|
||||
export interface AgentRecord {
|
||||
@@ -95,6 +191,9 @@ export interface AgentRecord {
|
||||
gen: string;
|
||||
/** ms since epoch. Reserved for future PID-reuse guards. */
|
||||
startedAt: number;
|
||||
startTime?: string;
|
||||
ownerPid?: number;
|
||||
ownerStartTime?: string;
|
||||
}
|
||||
|
||||
export function agentRecordPath(stateDir: string): string {
|
||||
@@ -124,27 +223,79 @@ export function writeAgentRecord(stateDir: string, record: AgentRecord): void {
|
||||
restrictFilePermissions(target);
|
||||
}
|
||||
|
||||
export function clearAgentRecord(stateDir: string): void {
|
||||
export function clearAgentRecord(stateDir: string, expected?: AgentRecord): void {
|
||||
if (expected) {
|
||||
const current = readAgentRecord(stateDir);
|
||||
if (!current || current.pid !== expected.pid || current.gen !== expected.gen) return;
|
||||
}
|
||||
safeUnlink(agentRecordPath(stateDir));
|
||||
}
|
||||
|
||||
export function isAgentRecordLive(record: AgentRecord): boolean {
|
||||
return Number.isSafeInteger(record.pid) && record.pid > 0 && isProcessAlive(record.pid);
|
||||
}
|
||||
|
||||
function agentStatus(record: AgentRecord, ownerPid?: number): 'owned' | 'gone' | 'unknown' {
|
||||
if (!isAgentRecordLive(record)) return 'gone';
|
||||
if (!record.startTime || !record.ownerPid || !record.ownerStartTime) return 'unknown';
|
||||
if (ownerPid !== undefined && (record.ownerPid !== ownerPid || record.ownerStartTime !== readAgentStartTime(ownerPid))) return 'unknown';
|
||||
const actual = agentProcessInfo(record.pid);
|
||||
if (!actual.startTime) return isAgentRecordLive(record) ? 'unknown' : 'gone';
|
||||
if (actual.startTime !== record.startTime) return 'gone';
|
||||
try {
|
||||
let state: string | undefined;
|
||||
if (process.platform === 'linux') {
|
||||
state = fs.readFileSync(`/proc/${record.pid}/stat`, 'utf8').match(/^\d+ \(.*\) ([A-Z])/u)?.[1];
|
||||
} else if (process.platform === 'darwin') {
|
||||
const result = spawnSync('ps', ['-p', String(record.pid), '-o', 'stat='], { encoding: 'utf8', windowsHide: true, timeout: 2000 });
|
||||
if (result.status === 0) state = result.stdout?.trim()?.[0];
|
||||
}
|
||||
if (state === 'Z') return 'gone';
|
||||
} catch {}
|
||||
if (!isAgentRecordLive(record)) return 'gone';
|
||||
return actual.commandLine.split(/\s+/).some(arg => arg.replace(/^['"]|['"]$/g, '') === `--agent-gen=${record.gen}`)
|
||||
? 'owned' : 'unknown';
|
||||
}
|
||||
|
||||
export function isOurAgent(record: AgentRecord, ownerPid?: number): boolean {
|
||||
return agentStatus(record, ownerPid) === 'owned';
|
||||
}
|
||||
|
||||
export function isAgentRecordGone(record: AgentRecord): boolean {
|
||||
return agentStatus(record) === 'gone';
|
||||
}
|
||||
|
||||
/**
|
||||
* Kill the agent identified by `record`. Signal defaults to SIGTERM (give
|
||||
* the agent a chance to run its own SIGTERM cleanup). Returns true if a
|
||||
* signal was actually sent to a live PID; false if the PID was already
|
||||
* dead (no-op). Never throws — ESRCH is swallowed by safeKill.
|
||||
*
|
||||
* Validates liveness BEFORE signaling so a PID-reuse race (the recorded
|
||||
* PID was reaped and a brand-new unrelated process now holds it) can't
|
||||
* cause us to kill the wrong process. This is a best-effort defense:
|
||||
* Linux/macOS don't expose process-start-time cheaply, and the gap
|
||||
* between record-write and watchdog-tick is small (60s max).
|
||||
* signal reached an exact-generation live process, false otherwise.
|
||||
*/
|
||||
export function killAgentByRecord(
|
||||
record: AgentRecord,
|
||||
signal: NodeJS.Signals = 'SIGTERM',
|
||||
): boolean {
|
||||
if (!isProcessAlive(record.pid)) return false;
|
||||
safeKill(record.pid, signal);
|
||||
return true;
|
||||
if (!isOurAgent(record)) return false;
|
||||
try { process.kill(record.pid, signal); return true; } catch { return false; }
|
||||
}
|
||||
|
||||
export function stopAgentByRecord(record: AgentRecord, graceMs = 1000): boolean {
|
||||
const initial = agentStatus(record);
|
||||
if (initial === 'gone') return true;
|
||||
if (initial !== 'owned') return false;
|
||||
const waitForExit = (ms: number) => {
|
||||
const deadline = Date.now() + ms;
|
||||
while (Date.now() < deadline) {
|
||||
const status = agentStatus(record);
|
||||
if (status === 'gone') return true;
|
||||
if (status === 'unknown') return false;
|
||||
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 25);
|
||||
}
|
||||
return agentStatus(record) === 'gone';
|
||||
};
|
||||
if (!killAgentByRecord(record, 'SIGTERM')) return agentStatus(record) === 'gone';
|
||||
if (waitForExit(graceMs)) return true;
|
||||
const afterGrace = agentStatus(record);
|
||||
if (afterGrace !== 'owned') return afterGrace === 'gone';
|
||||
if (!killAgentByRecord(record, 'SIGKILL')) return agentStatus(record) === 'gone';
|
||||
return waitForExit(graceMs);
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ import * as crypto from 'crypto';
|
||||
import { writeSecureFile, restrictFilePermissions, mkdirSecure } from './file-permissions';
|
||||
import { atomicWriteSync, atomicWriteQuiet } from '../../lib/fs-atomic';
|
||||
import { safeUnlink } from './error-handling';
|
||||
import { writeAgentRecord, clearAgentRecord } from './terminal-agent-control';
|
||||
import { writeAgentRecord, readAgentRecord, clearAgentRecord, readAgentStartTime, acquireAgentStateLock } from './terminal-agent-control';
|
||||
import { findAvailablePort } from './port-allocator';
|
||||
import { extractPtyCookie } from './pty-session-cookie';
|
||||
import {
|
||||
@@ -38,6 +38,7 @@ const STATE_FILE = process.env.BROWSE_STATE_FILE || path.join(process.env.HOME |
|
||||
const PORT_FILE = path.join(path.dirname(STATE_FILE), 'terminal-port');
|
||||
const BROWSE_SERVER_PORT = parseInt(process.env.BROWSE_SERVER_PORT || '0', 10);
|
||||
const BROWSE_OWNER_PID = parseInt(process.env.BROWSE_OWNER_PID || '0', 10);
|
||||
const BROWSE_OWNER_START_TIME = process.env.BROWSE_OWNER_START_TIME || (BROWSE_OWNER_PID > 0 ? readAgentStartTime(BROWSE_OWNER_PID) : '');
|
||||
const OWNER_WATCHDOG_MS = parseInt(
|
||||
process.env.GSTACK_TERMINAL_OWNER_WATCHDOG_MS || '15000',
|
||||
10,
|
||||
@@ -51,7 +52,7 @@ const INTERNAL_TOKEN = crypto.randomBytes(32).toString('base64url'); // shared w
|
||||
* header means "legacy caller" and is accepted (backward compat); a
|
||||
* present-but-mismatched header returns 409 stale generation.
|
||||
*/
|
||||
const CURRENT_GEN = crypto.randomBytes(16).toString('base64url');
|
||||
const CURRENT_GEN = process.env.BROWSE_AGENT_GEN || crypto.randomBytes(16).toString('base64url');
|
||||
|
||||
// In-memory attach-token registry. Parent posts /internal/grant after
|
||||
// /pty-session; we validate WS upgrades against this map.
|
||||
@@ -1004,6 +1005,25 @@ function readBrowseToken(): string {
|
||||
|
||||
// Boot.
|
||||
async function main() {
|
||||
const dir = path.dirname(PORT_FILE);
|
||||
if (process.env.BROWSE_AGENT_GEN) {
|
||||
const deadline = Date.now() + 2000;
|
||||
while (Date.now() < deadline) {
|
||||
const pending = readAgentRecord(dir);
|
||||
if (pending?.gen === CURRENT_GEN && pending.pid === process.pid) break;
|
||||
if (pending && pending.gen !== CURRENT_GEN) throw new Error('terminal-agent startup record was replaced');
|
||||
await Bun.sleep(25);
|
||||
}
|
||||
const recorded = readAgentRecord(dir);
|
||||
if (recorded?.pid !== process.pid || recorded.ownerPid !== BROWSE_OWNER_PID || recorded.ownerStartTime !== BROWSE_OWNER_START_TIME) {
|
||||
throw new Error('terminal-agent startup record was not confirmed');
|
||||
}
|
||||
}
|
||||
const pauseFile = process.env.NODE_ENV === 'test' ? process.env.GSTACK_TERMINAL_TEST_PUBLISH_BARRIER : undefined;
|
||||
if (pauseFile) {
|
||||
fs.writeFileSync(`${pauseFile}.ready`, 'ready');
|
||||
while (!fs.existsSync(pauseFile)) await Bun.sleep(10);
|
||||
}
|
||||
writeClaudeAvailable();
|
||||
// #2314: allocate from the shared fixed scan range, then bind. Probe-then-
|
||||
// bind has a TOCTOU window — a concurrent process can take the port between
|
||||
@@ -1032,17 +1052,21 @@ async function main() {
|
||||
|
||||
// Write port file atomically so the parent server can pick it up.
|
||||
// Throws on failure — a boot without a discoverable port file is broken.
|
||||
const dir = path.dirname(PORT_FILE);
|
||||
try { mkdirSecure(dir); } catch {}
|
||||
atomicWriteSync(PORT_FILE, String(port), { mode: 0o600 });
|
||||
restrictFilePermissions(PORT_FILE); // Windows ACL hardening
|
||||
|
||||
// Write identity-based agent record (pid + per-boot gen). Replaces the
|
||||
// v1.43- `pkill -f terminal-agent\.ts` regex teardown that could kill
|
||||
// sibling gstack sessions. Callers (cli.ts spawn site, server.ts
|
||||
// shutdown, the v1.44 watchdog) now route through killAgentByRecord in
|
||||
// terminal-agent-control.ts.
|
||||
writeAgentRecord(dir, { pid: process.pid, gen: CURRENT_GEN, startedAt: Date.now() });
|
||||
const releasePublication = acquireAgentStateLock(dir);
|
||||
let record;
|
||||
try {
|
||||
const current = readAgentRecord(dir);
|
||||
if (current && current.pid !== process.pid && current.pid > 0) throw new Error('terminal-agent record was replaced before bind');
|
||||
record = process.env.BROWSE_AGENT_GEN ? current : {
|
||||
pid: process.pid, gen: CURRENT_GEN, startedAt: Date.now(), startTime: readAgentStartTime(process.pid),
|
||||
ownerPid: BROWSE_OWNER_PID, ownerStartTime: BROWSE_OWNER_START_TIME,
|
||||
};
|
||||
if (!record || record.pid !== process.pid || record.gen !== CURRENT_GEN) throw new Error('terminal-agent record was replaced before bind');
|
||||
if (!process.env.BROWSE_AGENT_GEN) writeAgentRecord(dir, record);
|
||||
writeSecureFile(INTERNAL_TOKEN_FILE, INTERNAL_TOKEN);
|
||||
atomicWriteSync(PORT_FILE, String(port), { mode: 0o600 });
|
||||
restrictFilePermissions(PORT_FILE);
|
||||
} finally { releasePublication(); }
|
||||
|
||||
// Hand the parent the internal token so it can call /internal/grant.
|
||||
// Parent learns INTERNAL_TOKEN via env (TERMINAL_AGENT_INTERNAL_TOKEN below).
|
||||
@@ -1055,9 +1079,16 @@ async function main() {
|
||||
const cleanup = () => {
|
||||
if (cleaningUp) return;
|
||||
cleaningUp = true;
|
||||
safeUnlink(PORT_FILE);
|
||||
safeUnlink(INTERNAL_TOKEN_FILE);
|
||||
clearAgentRecord(dir);
|
||||
try {
|
||||
const releaseCleanup = acquireAgentStateLock(dir, 25);
|
||||
try {
|
||||
if (readAgentRecord(dir)?.gen === CURRENT_GEN) {
|
||||
safeUnlink(PORT_FILE);
|
||||
safeUnlink(INTERNAL_TOKEN_FILE);
|
||||
clearAgentRecord(dir, record);
|
||||
}
|
||||
} finally { releaseCleanup(); }
|
||||
} catch {}
|
||||
process.exit(0);
|
||||
};
|
||||
process.on('SIGTERM', cleanup);
|
||||
@@ -1070,11 +1101,8 @@ async function main() {
|
||||
// the same cleanup path as an intentional shutdown when it disappears.
|
||||
if (BROWSE_OWNER_PID > 0) {
|
||||
const ownerWatchdog = setInterval(() => {
|
||||
try {
|
||||
process.kill(BROWSE_OWNER_PID, 0);
|
||||
} catch {
|
||||
cleanup();
|
||||
}
|
||||
if (!BROWSE_OWNER_START_TIME || readAgentStartTime(BROWSE_OWNER_PID) !== BROWSE_OWNER_START_TIME
|
||||
|| readAgentRecord(dir)?.gen !== CURRENT_GEN) cleanup();
|
||||
}, OWNER_WATCHDOG_MS);
|
||||
(ownerWatchdog as any)?.unref?.();
|
||||
}
|
||||
@@ -1087,10 +1115,6 @@ async function main() {
|
||||
// In practice, the agent generates INTERNAL_TOKEN once at boot and writes it
|
||||
// to a state file the parent reads. This avoids env-passing races. See main().
|
||||
const INTERNAL_TOKEN_FILE = path.join(path.dirname(STATE_FILE), 'terminal-internal-token');
|
||||
try {
|
||||
mkdirSecure(path.dirname(INTERNAL_TOKEN_FILE));
|
||||
writeSecureFile(INTERNAL_TOKEN_FILE, INTERNAL_TOKEN);
|
||||
} catch {}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(`[terminal-agent] boot failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { describe, test, expect, spyOn } from 'bun:test';
|
||||
import * as childProcess from 'node:child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
@@ -103,34 +104,54 @@ describe('process liveness probe (Windows terminal-agent leak)', () => {
|
||||
expect(offenders).toEqual([]);
|
||||
});
|
||||
|
||||
test('5. spawnTerminalAgent passes windowsHide so no console is shown', () => {
|
||||
test('5. spawnTerminalAgent passes windowsHide so no console is shown', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-hide-'));
|
||||
const script = path.join(tmpDir, 'fake-agent.ts');
|
||||
fs.writeFileSync(script, '// no-op\n');
|
||||
const origSpawn = (Bun as any).spawn;
|
||||
const originalProbe = Bun.spawnSync;
|
||||
const originalWindowsProbe = childProcess.spawnSync;
|
||||
const probe = spyOn(Bun, 'spawnSync').mockImplementation(((command: string[], options: any) => {
|
||||
if (command[0] === 'ps' && command[2] === String(process.pid)) {
|
||||
return { exitCode: 0, stdout: Buffer.from('fixture-owner-start'), stderr: Buffer.alloc(0) };
|
||||
}
|
||||
return originalProbe(command, options);
|
||||
}) as typeof Bun.spawnSync);
|
||||
const windowsProbe = spyOn(childProcess, 'spawnSync').mockImplementation(((command: string, args: string[], options: any) => {
|
||||
if (command === 'powershell.exe') {
|
||||
const owner = args.join(' ').includes(`ProcessId = ${process.pid}'`);
|
||||
return { status: 0, stdout: JSON.stringify(owner ? { CreationDate: 'fixture-owner-start', CommandLine: 'test-owner' } : null), stderr: '' };
|
||||
}
|
||||
return originalWindowsProbe(command, args, options);
|
||||
}) as typeof childProcess.spawnSync);
|
||||
const exited = Promise.resolve(0);
|
||||
let captured: any = null;
|
||||
(Bun as any).spawn = (_cmd: any, opts: any) => {
|
||||
captured = opts;
|
||||
return { pid: 4242, unref() {} };
|
||||
return { pid: 2147483647, exited, kill() {}, unref() {} };
|
||||
};
|
||||
try {
|
||||
const pid = spawnTerminalAgent({
|
||||
expect(() => spawnTerminalAgent({
|
||||
stateFile: path.join(tmpDir, 'state.json'),
|
||||
serverPort: 12345,
|
||||
ownerPid: process.pid,
|
||||
cwd: tmpDir,
|
||||
scriptPath: script,
|
||||
});
|
||||
expect(pid).toBe(4242);
|
||||
})).toThrow('terminal-agent process identity is unavailable');
|
||||
expect(captured).not.toBeNull();
|
||||
expect(captured.windowsHide).toBe(true);
|
||||
// Owner-PID lifetime tie (#2019): the agent polls this and exits when
|
||||
// its owning browse server dies, so it can't be adopted by PID 1.
|
||||
expect(captured.env.BROWSE_OWNER_PID).toBe(String(process.pid));
|
||||
expect(captured.env.BROWSE_OWNER_START_TIME).toBe('fixture-owner-start');
|
||||
// Detached background daemon — must not inherit a terminal either.
|
||||
expect(captured.stdio).toEqual(['ignore', 'ignore', 'ignore']);
|
||||
await exited;
|
||||
expect(fs.existsSync(path.join(tmpDir, 'terminal-agent-pid'))).toBe(false);
|
||||
} finally {
|
||||
(Bun as any).spawn = origSpawn;
|
||||
windowsProbe.mockRestore();
|
||||
probe.mockRestore();
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { describe, test, expect, beforeEach, beforeAll, afterAll } from 'bun:test';
|
||||
import { describe, test, expect, beforeEach, beforeAll, afterAll, spyOn } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
@@ -219,4 +219,83 @@ describe('buildFetchHandler ownsTerminalAgent gate', () => {
|
||||
// match cannot be satisfied by the JSDoc reference earlier in the file.
|
||||
expect(source).toMatch(/ownsTerminalAgent:\s*true,\s*\/\/\s*CLI spawns terminal-agent\.ts/);
|
||||
});
|
||||
|
||||
test('5. shutdown cannot remove a successor published after its current-record read', async () => {
|
||||
writeSentinels();
|
||||
const ready = path.join(fixtureDir, 'competitor-ready');
|
||||
const script = path.join(fixtureDir, 'competitor.ts');
|
||||
fs.writeFileSync(script, `
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { acquireAgentStateLock } from ${JSON.stringify(path.resolve(import.meta.dir, '../src/terminal-agent-control.ts'))};
|
||||
const stateDir = process.argv[2];
|
||||
fs.writeFileSync(${JSON.stringify(ready)}, 'ready');
|
||||
const release = acquireAgentStateLock(stateDir);
|
||||
try {
|
||||
fs.writeFileSync(path.join(stateDir, 'terminal-port'), 'successor-port');
|
||||
fs.writeFileSync(path.join(stateDir, 'terminal-internal-token'), 'synthetic-successor-token');
|
||||
fs.writeFileSync(path.join(stateDir, 'terminal-agent-pid'), JSON.stringify({ pid: process.pid, gen: 'successor', startedAt: Date.now() }));
|
||||
} finally { release(); }
|
||||
`);
|
||||
const originalRead = fs.readFileSync;
|
||||
let recordReads = 0;
|
||||
let actor: ReturnType<typeof Bun.spawn> | undefined;
|
||||
const reader = spyOn(fs, 'readFileSync').mockImplementation(((file: fs.PathOrFileDescriptor, options?: any) => {
|
||||
const result = originalRead(file as any, options);
|
||||
if (String(file) === AGENT_RECORD_FILE && ++recordReads === 2) {
|
||||
actor = Bun.spawn([process.execPath, script, stateDir], { stdio: ['ignore', 'ignore', 'ignore'] });
|
||||
const deadline = Date.now() + 3000;
|
||||
while (!fs.existsSync(ready) && Date.now() < deadline) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10);
|
||||
if (!fs.existsSync(ready)) throw new Error('competitor never reached publication');
|
||||
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 150);
|
||||
}
|
||||
return result;
|
||||
}) as typeof fs.readFileSync);
|
||||
try {
|
||||
const handle = buildFetchHandler(makeMinimalConfig({ ownsTerminalAgent: true }));
|
||||
await withStubs(async () => runShutdown(handle));
|
||||
expect(recordReads).toBeGreaterThanOrEqual(2);
|
||||
expect(actor).toBeDefined();
|
||||
expect(await Promise.race([actor!.exited.then(() => true), Bun.sleep(5000).then(() => false)])).toBe(true);
|
||||
expect(readIfExists(PORT_FILE)).toBe('successor-port');
|
||||
expect(readIfExists(TOKEN_FILE)).toBe('synthetic-successor-token');
|
||||
expect(JSON.parse(readIfExists(AGENT_RECORD_FILE)!)).toMatchObject({ gen: 'successor' });
|
||||
} finally {
|
||||
reader.mockRestore();
|
||||
try { actor?.kill('SIGKILL'); } catch {}
|
||||
fs.rmSync(ready, { force: true });
|
||||
fs.rmSync(script, { force: true });
|
||||
}
|
||||
}, 15000);
|
||||
|
||||
test('6. unavailable state lock retains agent files rather than guessing ownership', async () => {
|
||||
writeSentinels();
|
||||
const originalOpen = fs.openSync;
|
||||
const opened = spyOn(fs, 'openSync').mockImplementation(((file: fs.PathLike, flags: string | number, mode?: number) => {
|
||||
if (String(file) === path.join(stateDir, 'terminal-agent-pid.lock')) {
|
||||
throw Object.assign(new Error('synthetic lock denial'), { code: 'EACCES' });
|
||||
}
|
||||
return originalOpen(file, flags as any, mode);
|
||||
}) as typeof fs.openSync);
|
||||
try {
|
||||
const handle = buildFetchHandler(makeMinimalConfig({ ownsTerminalAgent: true }));
|
||||
await withStubs(async () => runShutdown(handle));
|
||||
expect(readIfExists(PORT_FILE)).toBe(SENTINEL_PORT);
|
||||
expect(readIfExists(TOKEN_FILE)).toBe(SENTINEL_TOKEN);
|
||||
expect(readIfExists(AGENT_RECORD_FILE)).not.toBeNull();
|
||||
} finally { opened.mockRestore(); }
|
||||
});
|
||||
|
||||
test('7. late state takeover is not removed after browser close', async () => {
|
||||
fs.mkdirSync(stateDir, { recursive: true });
|
||||
fs.writeFileSync(fixtureConfig.stateFile, JSON.stringify({ pid: process.pid }));
|
||||
const successor = { pid: process.pid, instanceId: 'synthetic-late-successor' };
|
||||
const browserManager = new BrowserManager();
|
||||
browserManager.close = async () => { fs.writeFileSync(fixtureConfig.stateFile, JSON.stringify(successor)); };
|
||||
try {
|
||||
const handle = buildFetchHandler(makeMinimalConfig({ browserManager, ownsTerminalAgent: false }));
|
||||
await withStubs(async () => runShutdown(handle));
|
||||
expect(JSON.parse(fs.readFileSync(fixtureConfig.stateFile, 'utf8'))).toEqual(successor);
|
||||
} finally { fs.rmSync(fixtureConfig.stateFile, { force: true }); }
|
||||
});
|
||||
});
|
||||
|
||||
@@ -246,11 +246,11 @@ describe('buildFetchHandler factory contract', () => {
|
||||
fs.mkdirSync(path.dirname(globalState), { recursive: true });
|
||||
fs.mkdirSync(path.dirname(instanceState), { recursive: true });
|
||||
fs.writeFileSync(globalState, 'unrelated daemon state');
|
||||
fs.writeFileSync(instanceState, 'owned instance state');
|
||||
const script = `
|
||||
import fs from 'node:fs';
|
||||
import { buildFetchHandler } from ${JSON.stringify(path.resolve(__dirname, '../src/server.ts'))};
|
||||
import { buildFetchHandler, __testInternals__ } from ${JSON.stringify(path.resolve(__dirname, '../src/server.ts'))};
|
||||
import { resolveConfig } from ${JSON.stringify(path.resolve(__dirname, '../src/config.ts'))};
|
||||
fs.writeFileSync(${JSON.stringify(instanceState)}, JSON.stringify({ pid: process.pid, instanceId: __testInternals__.serverInstanceId }));
|
||||
const handle = buildFetchHandler({
|
||||
authToken: 'factory-shutdown-ownership-test', browsePort: 34567,
|
||||
config: resolveConfig({ BROWSE_STATE_FILE: ${JSON.stringify(instanceState)} }),
|
||||
|
||||
@@ -235,7 +235,8 @@ describe('cli.ts: sidebar-agent is no longer spawned', () => {
|
||||
'utf-8',
|
||||
);
|
||||
expect(CONTROL_SRC).toContain('terminal-agent.ts');
|
||||
expect(CONTROL_SRC).toMatch(/\.spawn\(\['bun',\s*'run',\s*script\]/);
|
||||
expect(CONTROL_SRC).toMatch(/\.spawn\(\['bun',\s*'run',\s*script,\s*`--agent-gen=\$\{gen\}`\]/);
|
||||
expect(CONTROL_SRC).toContain('BROWSE_OWNER_PID: String(opts.ownerPid)');
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -967,9 +967,10 @@ describe('shutdown cleanup (server.ts)', () => {
|
||||
// by browse/test/terminal-agent-pid-identity.test.ts).
|
||||
const shutdownFn = serverSrc.slice(
|
||||
serverSrc.indexOf('async function shutdown('),
|
||||
serverSrc.indexOf('async function shutdown(') + 1200,
|
||||
serverSrc.indexOf('try { detachSession()', serverSrc.indexOf('async function shutdown(')),
|
||||
);
|
||||
expect(shutdownFn).toContain('killAgentByRecord');
|
||||
expect(shutdownFn).toContain('stopAgentByRecord');
|
||||
expect(shutdownFn).toContain('isOurAgent(record, process.pid)');
|
||||
expect(shutdownFn).toContain('readAgentRecord');
|
||||
// No pkill CALL — the word may appear in the explanatory comment, so
|
||||
// match invocation shapes only. The repo-wide reintroduction tripwire
|
||||
@@ -994,4 +995,3 @@ describe('cookie import button (sidebar)', () => {
|
||||
expect(js).toContain('cookie-picker');
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,363 @@
|
||||
import { afterEach, describe, expect, spyOn, test } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import {
|
||||
acquireAgentStateLock, agentRecordPath, clearAgentRecord, isOurAgent, killAgentByRecord, readAgentRecord,
|
||||
readAgentStartTime, spawnTerminalAgent, stopAgentByRecord, type AgentRecord,
|
||||
writeAgentRecord,
|
||||
} from '../src/terminal-agent-control';
|
||||
|
||||
const sourceDir = path.join(import.meta.dir, '..', 'src');
|
||||
const dirs: string[] = [];
|
||||
const pids: number[] = [];
|
||||
const dir = () => {
|
||||
const value = fs.mkdtempSync(path.join(os.tmpdir(), 'g4-'));
|
||||
dirs.push(value);
|
||||
return value;
|
||||
};
|
||||
const waitFor = async (check: () => boolean, timeout = 3000) => {
|
||||
const deadline = Date.now() + timeout;
|
||||
while (Date.now() < deadline) {
|
||||
if (check()) return true;
|
||||
await Bun.sleep(25);
|
||||
}
|
||||
return check();
|
||||
};
|
||||
const spawn = (stateDir: string, ownerPid = process.pid) => {
|
||||
const pid = spawnTerminalAgent({ stateFile: path.join(stateDir, 'browse.json'), serverPort: 0, ownerPid,
|
||||
extraEnv: { GSTACK_TERMINAL_OWNER_WATCHDOG_MS: '25' } });
|
||||
if (pid) pids.push(pid);
|
||||
return pid;
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
for (const pid of pids.splice(0)) {
|
||||
const record = dirs.map(readAgentRecord).find(value => value?.pid === pid);
|
||||
if (record) stopAgentByRecord(record, 200);
|
||||
}
|
||||
for (const stateDir of dirs.splice(0)) fs.rmSync(stateDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('terminal-agent owned lifecycle regression', () => {
|
||||
for (const field of ['dev', 'ino'] as const) {
|
||||
test(`lock release preserves a replacement with an adjacent 64-bit ${field}`, () => {
|
||||
const stateDir = dir();
|
||||
const lockPath = path.join(stateDir, 'terminal-agent-pid.lock');
|
||||
const identity = 1n << 63n;
|
||||
expect(Number(identity)).toBe(Number(identity + 1n));
|
||||
const originalFstat = fs.fstatSync;
|
||||
const originalStat = fs.statSync;
|
||||
const descriptor = spyOn(fs, 'fstatSync').mockImplementation(((fd: number, options?: any) => {
|
||||
const stat = originalFstat(fd, options);
|
||||
return Object.assign(stat, { [field]: options?.bigint ? identity : Number(identity) });
|
||||
}) as typeof fs.fstatSync);
|
||||
const pathname = spyOn(fs, 'statSync').mockImplementation(((file: fs.PathLike, options?: any) => {
|
||||
const stat = originalStat(file, options);
|
||||
return String(file) === lockPath
|
||||
? Object.assign(stat, { [field]: options?.bigint ? identity + 1n : Number(identity + 1n) })
|
||||
: stat;
|
||||
}) as typeof fs.statSync);
|
||||
try {
|
||||
acquireAgentStateLock(stateDir)();
|
||||
expect(fs.existsSync(lockPath)).toBe(true);
|
||||
} finally {
|
||||
descriptor.mockRestore();
|
||||
pathname.mockRestore();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test('connect and supervisor pass the persistent daemon as owner', () => {
|
||||
const cli = fs.readFileSync(path.join(sourceDir, 'cli.ts'), 'utf8');
|
||||
const connect = cli.slice(cli.indexOf('// Auto-start terminal agent'), cli.indexOf('// ─── Outer Supervisor'));
|
||||
const supervisor = cli.slice(cli.indexOf('// ─── Outer Supervisor'), cli.indexOf('// ─── Headed Disconnect'));
|
||||
expect(connect).toMatch(/spawnTerminalAgent\(\{[^}]*ownerPid:\s*newState\.pid/s);
|
||||
expect(supervisor).toMatch(/spawnTerminalAgent\(\{[^}]*ownerPid:\s*respawned\.pid/s);
|
||||
});
|
||||
|
||||
test('owned agent starts, is replaced only after exit, and leaves a live sibling alone', async () => {
|
||||
const firstDir = dir();
|
||||
const siblingDir = dir();
|
||||
const first = spawn(firstDir);
|
||||
const sibling = spawn(siblingDir);
|
||||
expect(first).toBeGreaterThan(0);
|
||||
expect(sibling).toBeGreaterThan(0);
|
||||
expect(await waitFor(() => fs.existsSync(path.join(firstDir, 'terminal-port')))).toBe(true);
|
||||
const firstRecord = readAgentRecord(firstDir)!;
|
||||
const siblingRecord = readAgentRecord(siblingDir)!;
|
||||
expect(isOurAgent(firstRecord, process.pid)).toBe(true);
|
||||
const replacement = spawn(firstDir);
|
||||
expect(replacement).toBeGreaterThan(0);
|
||||
expect(replacement).not.toBe(first);
|
||||
expect(isOurAgent(firstRecord)).toBe(false);
|
||||
expect(isOurAgent(siblingRecord)).toBe(true);
|
||||
expect(readAgentRecord(firstDir)?.pid).toBe(replacement);
|
||||
});
|
||||
|
||||
test('child waits for its PID to replace the pre-spawn reservation', async () => {
|
||||
const stateDir = dir();
|
||||
const originalSpawn = Bun.spawn;
|
||||
(Bun as any).spawn = (...args: Parameters<typeof Bun.spawn>) => {
|
||||
const child = originalSpawn(...args);
|
||||
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 800);
|
||||
return child;
|
||||
};
|
||||
try {
|
||||
const pid = spawn(stateDir);
|
||||
expect(pid).toBeGreaterThan(0);
|
||||
expect(await waitFor(() => fs.existsSync(path.join(stateDir, 'terminal-port')))).toBe(true);
|
||||
expect(readAgentRecord(stateDir)?.pid).toBe(pid);
|
||||
} finally { (Bun as any).spawn = originalSpawn; }
|
||||
});
|
||||
|
||||
test('failed signals retain the live record and prevent a duplicate spawn', () => {
|
||||
const stateDir = dir();
|
||||
const first = spawn(stateDir)!;
|
||||
const record = readAgentRecord(stateDir)!;
|
||||
const original = process.kill;
|
||||
(process as any).kill = ((pid: number, signal: NodeJS.Signals | number) => {
|
||||
if (pid === first && signal !== 0) throw Object.assign(new Error('denied'), { code: 'EPERM' });
|
||||
return original(pid, signal);
|
||||
}) as typeof process.kill;
|
||||
try {
|
||||
expect(spawn(stateDir)).toBeNull();
|
||||
expect(readAgentRecord(stateDir)).toEqual(record);
|
||||
} finally {
|
||||
(process as any).kill = original;
|
||||
}
|
||||
});
|
||||
|
||||
test('a transient identity lookup failure after a signal is not confirmed exit', () => {
|
||||
const stateDir = dir();
|
||||
const first = spawn(stateDir)!;
|
||||
const record = readAgentRecord(stateDir)!;
|
||||
const originalKill = process.kill;
|
||||
const originalSpawnSync = Bun.spawnSync;
|
||||
let obscured = false;
|
||||
(process as any).kill = ((pid: number, signal: NodeJS.Signals | number) => {
|
||||
if (pid === first && signal !== 0) { obscured = true; return true; }
|
||||
return originalKill(pid, signal);
|
||||
}) as typeof process.kill;
|
||||
(Bun as any).spawnSync = (...args: Parameters<typeof Bun.spawnSync>) => {
|
||||
const command = args[0] as string[];
|
||||
if (obscured && command[0] === 'ps' && command[2] === String(first)) {
|
||||
return { exitCode: 1, stdout: Buffer.alloc(0), stderr: Buffer.alloc(0) };
|
||||
}
|
||||
return originalSpawnSync(...args);
|
||||
};
|
||||
try {
|
||||
expect(spawn(stateDir)).toBeNull();
|
||||
expect(readAgentRecord(stateDir)).toEqual(record);
|
||||
} finally {
|
||||
(process as any).kill = originalKill;
|
||||
(Bun as any).spawnSync = originalSpawnSync;
|
||||
}
|
||||
});
|
||||
|
||||
test('PID reuse and foreign records are never signaled', () => {
|
||||
const stateDir = dir();
|
||||
const forged: AgentRecord = {
|
||||
pid: process.pid, gen: 'synthetic-foreign-generation', startedAt: Date.now(),
|
||||
startTime: readAgentStartTime(process.pid), ownerPid: process.pid, ownerStartTime: readAgentStartTime(process.pid),
|
||||
};
|
||||
fs.writeFileSync(agentRecordPath(stateDir), JSON.stringify(forged));
|
||||
expect(isOurAgent(forged)).toBe(false);
|
||||
expect(killAgentByRecord(forged, 'SIGTERM')).toBe(false);
|
||||
expect(spawn(stateDir)).toBeNull();
|
||||
expect(readAgentRecord(stateDir)).toEqual(forged);
|
||||
clearAgentRecord(stateDir, { ...forged, gen: 'different' });
|
||||
expect(readAgentRecord(stateDir)).toEqual(forged);
|
||||
});
|
||||
|
||||
test('unwritable record path rejects before spawning any agent', () => {
|
||||
const stateDir = dir();
|
||||
const blocker = path.join(stateDir, 'blocker');
|
||||
fs.writeFileSync(blocker, 'block');
|
||||
const originalSpawn = Bun.spawn;
|
||||
let child: ReturnType<typeof Bun.spawn> | undefined;
|
||||
(Bun as any).spawn = (...args: Parameters<typeof Bun.spawn>) => {
|
||||
child = originalSpawn(...args);
|
||||
return child;
|
||||
};
|
||||
try {
|
||||
expect(() => spawnTerminalAgent({ stateFile: path.join(blocker, 'browse.json'), serverPort: 0, ownerPid: process.pid }))
|
||||
.toThrow();
|
||||
expect(child).toBeUndefined();
|
||||
expect(fs.readdirSync(stateDir)).toEqual(['blocker']);
|
||||
} finally {
|
||||
(Bun as any).spawn = originalSpawn;
|
||||
try { child?.kill('SIGKILL'); } catch {}
|
||||
}
|
||||
});
|
||||
|
||||
test('a leftover exclusive lock refuses recovery without stealing ownership', () => {
|
||||
const stateDir = dir();
|
||||
const lock = path.join(stateDir, 'terminal-agent-pid.lock');
|
||||
fs.writeFileSync(lock, '');
|
||||
expect(() => acquireAgentStateLock(stateDir, 0)).toThrow('state lock unavailable');
|
||||
expect(fs.existsSync(lock)).toBe(true);
|
||||
expect(readAgentRecord(stateDir)).toBeNull();
|
||||
});
|
||||
|
||||
test('record update failure after spawn confirms child exit before dropping its handle', async () => {
|
||||
const stateDir = dir();
|
||||
const originalSpawn = Bun.spawn;
|
||||
const originalRename = fs.renameSync;
|
||||
let child: ReturnType<typeof Bun.spawn> | undefined;
|
||||
let writes = 0;
|
||||
const rename = spyOn(fs, 'renameSync').mockImplementation(((from: fs.PathLike, to: fs.PathLike) => {
|
||||
if (String(to) === agentRecordPath(stateDir) && ++writes === 2) {
|
||||
throw Object.assign(new Error('synthetic state write failure'), { code: 'EIO' });
|
||||
}
|
||||
return originalRename(from, to);
|
||||
}) as typeof fs.renameSync);
|
||||
(Bun as any).spawn = (...args: Parameters<typeof Bun.spawn>) => {
|
||||
child = originalSpawn(...args);
|
||||
return child;
|
||||
};
|
||||
try {
|
||||
expect(() => spawn(stateDir)).toThrow('synthetic state write failure');
|
||||
expect(writes).toBe(2);
|
||||
expect(child).toBeDefined();
|
||||
expect(await Promise.race([child!.exited.then(() => true), Bun.sleep(3000).then(() => false)])).toBe(true);
|
||||
expect(readAgentRecord(stateDir)).toBeNull();
|
||||
} finally {
|
||||
rename.mockRestore();
|
||||
(Bun as any).spawn = originalSpawn;
|
||||
try { child?.kill('SIGKILL'); } catch {}
|
||||
}
|
||||
});
|
||||
|
||||
(process.platform === 'win32' ? test.skip : test)('unconfirmed post-write child keeps its reservation until it exits', async () => {
|
||||
const stateDir = dir();
|
||||
const originalSpawn = Bun.spawn;
|
||||
const originalRename = fs.renameSync;
|
||||
const originalKill = process.kill;
|
||||
let child: ReturnType<typeof Bun.spawn> | undefined;
|
||||
let writes = 0;
|
||||
let deniedSignals = 0;
|
||||
const rename = spyOn(fs, 'renameSync').mockImplementation(((from: fs.PathLike, to: fs.PathLike) => {
|
||||
if (String(to) === agentRecordPath(stateDir) && ++writes === 2) throw new Error('synthetic update refusal');
|
||||
return originalRename(from, to);
|
||||
}) as typeof fs.renameSync);
|
||||
(Bun as any).spawn = (...args: Parameters<typeof Bun.spawn>) => {
|
||||
child = originalSpawn(...args);
|
||||
originalKill(child.pid, 'SIGSTOP');
|
||||
return child;
|
||||
};
|
||||
(process as any).kill = ((pid: number, signal: NodeJS.Signals | number) => {
|
||||
if (child && pid === child.pid && signal !== 0) {
|
||||
deniedSignals++;
|
||||
throw Object.assign(new Error('signal denied'), { code: 'EPERM' });
|
||||
}
|
||||
return originalKill(pid, signal);
|
||||
}) as typeof process.kill;
|
||||
try {
|
||||
expect(() => spawn(stateDir)).toThrow('exit is unconfirmed');
|
||||
expect(deniedSignals).toBeGreaterThan(0);
|
||||
expect(readAgentRecord(stateDir)?.pid).toBe(0);
|
||||
expect(spawn(stateDir)).toBeNull();
|
||||
expect(readAgentRecord(stateDir)?.pid).toBe(0);
|
||||
originalKill(child!.pid, 'SIGCONT');
|
||||
expect(await Promise.race([child!.exited.then(() => true), Bun.sleep(4000).then(() => false)])).toBe(true);
|
||||
expect(await waitFor(() => readAgentRecord(stateDir) === null)).toBe(true);
|
||||
} finally {
|
||||
(process as any).kill = originalKill;
|
||||
(Bun as any).spawn = originalSpawn;
|
||||
rename.mockRestore();
|
||||
if (child) try { originalKill(child.pid, 'SIGCONT'); } catch {}
|
||||
try { child?.kill('SIGKILL'); } catch {}
|
||||
}
|
||||
}, 6000);
|
||||
|
||||
test('owner death and record takeover shut down the old generation without deleting its successor', async () => {
|
||||
const stateDir = dir();
|
||||
const owner = Bun.spawn([process.execPath, '-e', 'process.stdin.resume()'], { stdio: ['pipe', 'ignore', 'ignore'] });
|
||||
try {
|
||||
const pid = spawn(stateDir, owner.pid)!;
|
||||
expect(await waitFor(() => fs.existsSync(path.join(stateDir, 'terminal-port')))).toBe(true);
|
||||
const record = readAgentRecord(stateDir)!;
|
||||
owner.kill('SIGTERM');
|
||||
await owner.exited;
|
||||
expect(await waitFor(() => !isOurAgent(record), 5000)).toBe(true);
|
||||
expect(readAgentRecord(stateDir)).toBeNull();
|
||||
} finally { try { owner.kill('SIGKILL'); } catch {} }
|
||||
|
||||
const first = spawn(stateDir)!;
|
||||
expect(await waitFor(() => fs.existsSync(path.join(stateDir, 'terminal-port')))).toBe(true);
|
||||
const original = readAgentRecord(stateDir)!;
|
||||
const successor = { ...original, pid: 2147483646, gen: 'synthetic-successor' };
|
||||
fs.writeFileSync(agentRecordPath(stateDir), JSON.stringify(successor));
|
||||
expect(await waitFor(() => !isOurAgent(original), 5000)).toBe(true);
|
||||
expect(readAgentRecord(stateDir)).toEqual(successor);
|
||||
expect(first).toBeGreaterThan(0);
|
||||
}, 12000);
|
||||
|
||||
test('losing concurrent startup cannot publish over the winning generation', async () => {
|
||||
const stateDir = dir();
|
||||
const stateFile = path.join(stateDir, 'browse.json');
|
||||
const barrier = path.join(stateDir, 'go');
|
||||
const ownerStartTime = readAgentStartTime(process.pid);
|
||||
const rawAgent = (gen: string, paused: boolean) => Bun.spawn(['bun', 'run', path.join(sourceDir, 'terminal-agent.ts'), `--agent-gen=${gen}`], {
|
||||
env: { ...process.env, BROWSE_STATE_FILE: stateFile, BROWSE_OWNER_PID: String(process.pid),
|
||||
BROWSE_OWNER_START_TIME: ownerStartTime, BROWSE_AGENT_GEN: gen, NODE_ENV: 'test',
|
||||
GSTACK_TERMINAL_OWNER_WATCHDOG_MS: '25',
|
||||
...(paused ? { GSTACK_TERMINAL_TEST_PUBLISH_BARRIER: barrier } : {}) },
|
||||
stdio: ['ignore', 'ignore', 'ignore'],
|
||||
});
|
||||
const old = rawAgent('synthetic-old-generation', true);
|
||||
let winner: ReturnType<typeof Bun.spawn> | undefined;
|
||||
try {
|
||||
writeAgentRecord(stateDir, { pid: old.pid, gen: 'synthetic-old-generation', startedAt: Date.now(),
|
||||
startTime: readAgentStartTime(old.pid), ownerPid: process.pid, ownerStartTime });
|
||||
expect(await waitFor(() => fs.existsSync(`${barrier}.ready`))).toBe(true);
|
||||
winner = rawAgent('synthetic-new-generation', false);
|
||||
writeAgentRecord(stateDir, { pid: winner.pid, gen: 'synthetic-new-generation', startedAt: Date.now(),
|
||||
startTime: readAgentStartTime(winner.pid), ownerPid: process.pid, ownerStartTime });
|
||||
expect(await waitFor(() => fs.existsSync(path.join(stateDir, 'terminal-port')))).toBe(true);
|
||||
const port = fs.readFileSync(path.join(stateDir, 'terminal-port'), 'utf8');
|
||||
const token = fs.readFileSync(path.join(stateDir, 'terminal-internal-token'), 'utf8');
|
||||
fs.writeFileSync(barrier, 'continue');
|
||||
expect(await Promise.race([old.exited.then(() => true), Bun.sleep(3000).then(() => false)])).toBe(true);
|
||||
expect(readAgentRecord(stateDir)?.gen).toBe('synthetic-new-generation');
|
||||
expect(fs.readFileSync(path.join(stateDir, 'terminal-port'), 'utf8')).toBe(port);
|
||||
expect(fs.readFileSync(path.join(stateDir, 'terminal-internal-token'), 'utf8')).toBe(token);
|
||||
} finally {
|
||||
fs.writeFileSync(barrier, 'continue');
|
||||
try { old.kill('SIGKILL'); } catch {}
|
||||
try { winner?.kill('SIGKILL'); } catch {}
|
||||
await old.exited;
|
||||
if (winner) await winner.exited;
|
||||
}
|
||||
}, 10000);
|
||||
|
||||
test('daemon respawns after agent crash, then exits without deleting a successor state', async () => {
|
||||
const stateDir = dir();
|
||||
const stateFile = path.join(stateDir, 'browse.json');
|
||||
const daemon = Bun.spawn(['bun', 'run', path.join(sourceDir, 'server.ts')], {
|
||||
env: { ...process.env, BROWSE_STATE_FILE: stateFile, BROWSE_HEADLESS_SKIP: '1', BROWSE_PARENT_PID: '0',
|
||||
GSTACK_AGENT_WATCHDOG_TICK_MS: '50', GSTACK_STATE_WATCH_MS: '50' },
|
||||
stdio: ['ignore', 'ignore', 'ignore'],
|
||||
});
|
||||
try {
|
||||
expect(await waitFor(() => fs.existsSync(stateFile))).toBe(true);
|
||||
expect(await waitFor(() => {
|
||||
const record = readAgentRecord(stateDir);
|
||||
return !!record && record.pid > 0 && isOurAgent(record, daemon.pid);
|
||||
}, 5000)).toBe(true);
|
||||
const old = readAgentRecord(stateDir)!;
|
||||
expect(old.ownerPid).toBe(daemon.pid);
|
||||
expect(isOurAgent(old, daemon.pid)).toBe(true);
|
||||
expect(killAgentByRecord(old, 'SIGKILL')).toBe(true);
|
||||
expect(await waitFor(() => !!readAgentRecord(stateDir) && readAgentRecord(stateDir)!.gen !== old.gen, 5000)).toBe(true);
|
||||
const successor = { ...JSON.parse(fs.readFileSync(stateFile, 'utf8')), pid: process.pid, instanceId: 'synthetic-successor' };
|
||||
fs.writeFileSync(stateFile, JSON.stringify(successor));
|
||||
expect(await waitFor(() => daemon.exitCode !== null, 5000)).toBe(true);
|
||||
expect(JSON.parse(fs.readFileSync(stateFile, 'utf8'))).toEqual(successor);
|
||||
} finally {
|
||||
try { daemon.kill('SIGKILL'); } catch {}
|
||||
await daemon.exited;
|
||||
}
|
||||
}, 15000);
|
||||
});
|
||||
@@ -0,0 +1,66 @@
|
||||
import { describe, expect, spyOn, test } from 'bun:test';
|
||||
import * as childProcess from 'node:child_process';
|
||||
import { isAgentRecordGone, isOurAgent, stopAgentByRecord } from '../src/terminal-agent-control';
|
||||
|
||||
describe('terminal-agent native exit observations', () => {
|
||||
const pid = 2147483645;
|
||||
const startTime = 'Tue Sep 22 23:39:21 2026';
|
||||
const gen = 'test-observed-generation';
|
||||
const ownedCommand = `bun run terminal-agent.ts --agent-gen=${gen}`;
|
||||
const cases = [
|
||||
{ name: 'Darwin zombie retains nonempty command text', command: '(bun)', state: 'Z', gone: true },
|
||||
{ name: 'zombie retains its generation argument', command: ownedCommand, state: 'Z', gone: true },
|
||||
{ name: 'process exits during start-time lookup', command: '', state: '', missingStart: true, reap: true, gone: true },
|
||||
{ name: 'process exits during command lookup', command: '', state: '', reap: true, gone: true },
|
||||
{ name: 'live process has a failed start-time lookup', command: '', state: 'S', missingStart: true, gone: false },
|
||||
{ name: 'live process has a failed command lookup', command: '', state: 'S', gone: false },
|
||||
{ name: 'live foreign generation is not ours', command: 'bun unrelated.ts', state: 'S', gone: false },
|
||||
{ name: 'live owned generation remains ours', command: ownedCommand, state: 'S', gone: false, owned: true },
|
||||
{ name: 'failed state probe cannot certify a zombie', command: '', state: 'Z', stateStatus: 1, gone: false },
|
||||
{ name: 'process exits before the owned signal', command: ownedCommand, state: 'S', gone: false, owned: true, reapOnSignal: true },
|
||||
];
|
||||
|
||||
for (const scenario of cases) {
|
||||
test(scenario.name, () => {
|
||||
const platform = Object.getOwnPropertyDescriptor(process, 'platform')!;
|
||||
let live = true;
|
||||
const signals: unknown[] = [];
|
||||
const kill = spyOn(process, 'kill').mockImplementation(((target: number, signal: unknown) => {
|
||||
expect(target).toBe(pid);
|
||||
if (signal !== 0) {
|
||||
signals.push(signal);
|
||||
if (scenario.reapOnSignal) { live = false; throw Object.assign(new Error('gone'), { code: 'ESRCH' }); }
|
||||
throw new Error('unexpected signal');
|
||||
}
|
||||
if (!live) throw Object.assign(new Error('gone'), { code: 'ESRCH' });
|
||||
return true;
|
||||
}) as typeof process.kill);
|
||||
const probe = spyOn(Bun, 'spawnSync').mockImplementation(((command: string[]) => {
|
||||
expect(command.slice(0, 3)).toEqual(['ps', '-p', String(pid)]);
|
||||
const start = command[4] === 'lstart=';
|
||||
if (scenario.reap && (!start || scenario.missingStart)) live = false;
|
||||
return {
|
||||
exitCode: start && scenario.missingStart ? 1 : 0,
|
||||
stdout: Buffer.from(start ? scenario.missingStart ? '' : startTime : scenario.command),
|
||||
stderr: Buffer.alloc(0),
|
||||
};
|
||||
}) as typeof Bun.spawnSync);
|
||||
const state = spyOn(childProcess, 'spawnSync').mockReturnValue({
|
||||
status: scenario.stateStatus ?? 0, stdout: scenario.state, stderr: '',
|
||||
} as any);
|
||||
Object.defineProperty(process, 'platform', { ...platform, value: 'darwin' });
|
||||
try {
|
||||
const record = { pid, gen, startTime, startedAt: 0, ownerPid: pid, ownerStartTime: startTime };
|
||||
expect(isAgentRecordGone(record)).toBe(scenario.gone);
|
||||
expect(isOurAgent(record)).toBe(scenario.owned ?? false);
|
||||
if (scenario.gone || scenario.reapOnSignal) expect(stopAgentByRecord(record, 0)).toBe(true);
|
||||
expect(signals).toEqual(scenario.reapOnSignal ? ['SIGTERM'] : []);
|
||||
} finally {
|
||||
Object.defineProperty(process, 'platform', platform);
|
||||
state.mockRestore();
|
||||
probe.mockRestore();
|
||||
kill.mockRestore();
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
@@ -19,8 +19,8 @@ describe('terminal-agent watchdog (v1.44+)', () => {
|
||||
expect(src).toMatch(/export function spawnTerminalAgent\(/);
|
||||
// Must clean up prior PID before spawning (no zombies).
|
||||
expect(src).toContain('readAgentRecord(stateDir)');
|
||||
expect(src).toContain('killAgentByRecord(prior');
|
||||
expect(src).toContain('clearAgentRecord(stateDir)');
|
||||
expect(src).toContain('stopAgentByRecord(prior)');
|
||||
expect(src).toContain('clearAgentRecord(stateDir, prior)');
|
||||
});
|
||||
|
||||
test('2. watchdog is gated on ownsTerminalAgent', () => {
|
||||
@@ -39,7 +39,11 @@ describe('terminal-agent watchdog (v1.44+)', () => {
|
||||
// identity-based liveness. Slow-but-alive agents must NOT trigger
|
||||
// respawn (split-brain defense).
|
||||
expect(block).toContain('readAgentRecord(stateDir)');
|
||||
expect(block).toContain('isProcessAlive(record.pid)');
|
||||
expect(block).toContain('isAgentRecordGone(record)');
|
||||
const control = fs.readFileSync(CONTROL_TS, 'utf-8');
|
||||
expect(control).toContain('if (result.status === 0) state = result.stdout?.trim()?.[0];');
|
||||
expect(control).toContain("if (state === 'Z') return 'gone'");
|
||||
expect(control.indexOf("if (state === 'Z') return 'gone'")).toBeLessThan(control.indexOf('return actual.commandLine.split'));
|
||||
// Negative: no executable name-based process lookup. Allow the strings
|
||||
// to appear in prose comments (the watchdog doc explains what it
|
||||
// replaces), reject only actual invocations.
|
||||
|
||||
Reference in New Issue
Block a user