Files
gstack/browse/src/terminal-agent-control.ts
T
Garry Tanandcapy-ai[bot] fb30ed1047 v1.91.297.0 fix(browse): read Windows process start times in-process so the terminal agent boots without PowerShell (GSTA-139 flaky pass) (#3545)
* fix(browse): read Windows process start times in-process; drop PowerShell from two flaky test paths (GSTA-139 S5)

The terminal agent's identity probe launched powershell.exe Get-CimInstance
with a 2 s timeout on every boot (twice when started standalone). On loaded
Windows runners a launch takes ~0.94 s at p50 and up to 10.8 s, so the agent
missed the watchdog test's 5 s window and a stalled launch returned an empty
identity. Under Bun the start time now comes from GetProcessTimes in the
same /Date(ms)/ form PowerShell's CreationDate serialises to, so records,
the Node daemon (which keeps the PowerShell reader) and earlier releases
agree. The kill-time command-line check still uses PowerShell.

path-security-realpath builds its 8.3 alias with GetShortPathNameW instead
of a PowerShell launch that stalled for 30 s and was reported as '8.3
disabled'. cookie-import-native-job prints the Restart Manager receipt when
its owner assertion fails.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(browse): scope test 5's PowerShell assertion to the owner; identify a win32 agent by generation when readers disagree

Review of d4f0ea3c (specialist + outside voice, confirmed by Windows Free
Tests 40699973746 shard 3): test 5's fake child pid cannot be opened, so its
start time falls back to the PowerShell query; only the owner must skip it.
Adversarial note: the Bun reader and the Node daemon's PowerShell reader can
disagree for a process created in a DST-ambiguous hour; on win32 a live
process carrying the record's --agent-gen is still the agent, so it is
stopped, never treated as gone beside a second agent.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(browse): a win32 start-time mismatch with an unreadable command line is unknown, not gone

Fix-check (adversarial reviewer): when the Bun reader and a Node-written
record disagree and the PowerShell command-line query also fails, the agent
was read as gone and a second agent could start beside it. Base treated a
failed query as unknown; this restores that. CHANGELOG [Unreleased] entry.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(browse): keep agentStatus's zombie check before the generation return (terminal-agent-watchdog source pin)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
2026-10-28 19:23:18 +00:00

430 lines
19 KiB
TypeScript

/**
* terminal-agent process-control primitives shared by cli.ts spawn site,
* server.ts shutdown teardown, and the v1.44 watchdog/respawn loop.
*
* Why this exists: pre-v1.44 used `pkill -f terminal-agent\.ts`, which
* matches any process whose argv contains the string and would kill
* sibling gstack sessions on the same host. The agent now writes a
* structured `terminal-agent-pid` record (`{pid, gen, startedAt}`) and
* every kill site routes through `killAgentByRecord` here — identity-based,
* no regex.
*
* The `gen` field is a per-boot generation counter. Loopback /internal/*
* calls from the parent server include `X-Browse-Gen` so a slow agent that
* the watchdog respawned around can't accidentally service a stale grant
* from the old generation.
*/
import * as fs from 'fs';
import * as path from 'path';
import * as crypto from 'crypto';
import { spawnSync } from 'child_process';
import { safeUnlink, isProcessAlive } from './error-handling';
import { restrictFilePermissions, mkdirSecure } from './file-permissions';
import { atomicWriteSync } from '../../lib/fs-atomic';
import { readPidCmdline, readPidStartTime } from './xvfb';
import { emptyBunConfig, findWindowsBunExe, type BunExeLookup } from './bun-child-config';
const FILETIME_UNIX_EPOCH_MS = 11_644_473_600_000n;
let win32Kernel: any;
function windowsProcessQuery(pid: number): { startTime: string; commandLine: string } {
const script = `(Get-CimInstance Win32_Process -Filter 'ProcessId = ${pid}' | Select-Object CreationDate,CommandLine | ConvertTo-Json -Compress)`;
try {
const result = spawnSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', script], { encoding: 'utf8', windowsHide: true, timeout: 2000 });
if (result.status !== 0 || !result.stdout) return { startTime: '', commandLine: '' };
const processInfo = JSON.parse(result.stdout);
return { startTime: processInfo?.CreationDate || '', commandLine: processInfo?.CommandLine || '' };
} catch { return { startTime: '', commandLine: '' }; }
}
export const win32CreationTimeSource = {
read(pid: number): string | undefined {
if (!process.versions.bun) return;
try {
const { dlopen, FFIType, ptr } = require('bun:ffi');
win32Kernel ??= dlopen('kernel32.dll', {
OpenProcess: { args: [FFIType.u32, FFIType.i32, FFIType.u32], returns: FFIType.u64 },
GetProcessTimes: { args: [FFIType.u64, FFIType.ptr, FFIType.ptr, FFIType.ptr, FFIType.ptr], returns: FFIType.i32 },
CloseHandle: { args: [FFIType.u64], returns: FFIType.i32 },
}).symbols;
const handle = win32Kernel.OpenProcess(0x1000, 0, pid);
if (!handle) return;
try {
const times = new BigUint64Array(4);
if (!win32Kernel.GetProcessTimes(handle, ptr(times, 0), ptr(times, 8), ptr(times, 16), ptr(times, 24)) || times[0] === 0n) return;
return `/Date(${times[0] / 10_000n - FILETIME_UNIX_EPOCH_MS})/`;
} finally {
win32Kernel.CloseHandle(handle);
}
} catch { return; }
},
};
function agentProcessInfo(pid: number): { startTime: string; commandLine: string } {
if (!Number.isSafeInteger(pid) || pid <= 0) return { startTime: '', commandLine: '' };
if (process.platform !== 'win32') return { startTime: readPidStartTime(pid), commandLine: readPidCmdline(pid) };
const query = windowsProcessQuery(pid);
return { startTime: win32CreationTimeSource.read(pid) ?? query.startTime, commandLine: query.commandLine };
}
export function readAgentStartTime(pid: number): string {
if (!Number.isSafeInteger(pid) || pid <= 0) return '';
if (process.platform !== 'win32') return readPidStartTime(pid);
return win32CreationTimeSource.read(pid) ?? windowsProcessQuery(pid).startTime;
}
const pendingAgentExits = new Set<any>();
function reclaimPublicationLock(stateDir: string, lockPath: string): boolean {
try {
const inode = fs.lstatSync(lockPath, { bigint: true });
if (!inode.isFile() || inode.size === 0n || inode.size > 4096n) return false;
const contents = fs.readFileSync(lockPath, 'utf8');
const lock = JSON.parse(contents);
const record = readAgentRecord(stateDir);
if (lock?.kind !== 'agent-publication-v1' || !record
|| !Number.isSafeInteger(record.pid) || record.pid <= 0
|| typeof record.gen !== 'string' || !record.gen
|| typeof record.startTime !== 'string' || !record.startTime
|| record.ownerPid !== process.pid
|| typeof record.ownerStartTime !== 'string' || !record.ownerStartTime) return false;
const fields = ['pid', 'gen', 'startTime', 'ownerPid', 'ownerStartTime'] as const;
if (fields.some(field => lock[field] !== record[field])
|| readAgentStartTime(process.pid) !== record.ownerStartTime) return false;
let present = true;
try { process.kill(record.pid, 0); }
catch (err: any) {
if (err?.code !== 'ESRCH') return false;
present = false;
}
if (present) {
if (readAgentStartTime(record.pid) !== record.startTime) return false;
if (process.platform === 'linux') {
const state = fs.readFileSync(`/proc/${record.pid}/stat`, 'utf8').match(/^\d+ \(.*\) ([A-Z])/u)?.[1];
if (state !== 'Z') return false;
} else if (process.platform === 'darwin') {
const result = spawnSync('ps', ['-p', String(record.pid), '-o', 'stat='], { encoding: 'utf8', windowsHide: true, timeout: 2000 });
if (result.status !== 0 || result.stdout?.trim()?.[0] !== 'Z') return false;
} else return false;
}
const currentRecord = readAgentRecord(stateDir);
if (!currentRecord || fields.some(field => currentRecord[field] !== record[field])) return false;
if (present && readAgentStartTime(record.pid) !== record.startTime) return false;
if (fs.readFileSync(lockPath, 'utf8') !== contents) return false;
const current = fs.lstatSync(lockPath, { bigint: true });
if (!current.isFile() || current.dev !== inode.dev || current.ino !== inode.ino) return false;
fs.unlinkSync(lockPath);
return true;
} catch { return false; }
}
export function acquireAgentStateLock(stateDir: string, waitMs = 5000, publicationGen?: string): () => void {
mkdirSecure(stateDir);
const lockPath = path.join(stateDir, 'terminal-agent-pid.lock');
let publication: string | undefined;
if (publicationGen !== undefined) {
const record = readAgentRecord(stateDir);
if (!record || record.pid !== process.pid || record.gen !== publicationGen
|| !record.ownerPid || !isOurAgent(record, record.ownerPid)) {
throw new Error('terminal-agent publication lock identity is unavailable');
}
publication = JSON.stringify({ kind: 'agent-publication-v1', pid: record.pid, gen: record.gen,
startTime: record.startTime, ownerPid: record.ownerPid, ownerStartTime: record.ownerStartTime });
}
const deadline = Date.now() + waitMs;
let reclaimed = false;
let fd: number;
while (true) {
try {
if (publication !== undefined) {
atomicWriteSync(lockPath, publication, { mode: 0o600, noReplace: true });
fd = fs.openSync(lockPath, 'r');
} else {
fd = fs.openSync(lockPath, 'wx', 0o600);
}
break;
} catch (err: any) {
if (err?.code === 'EEXIST' && !reclaimed && reclaimPublicationLock(stateDir, lockPath)) {
reclaimed = true;
continue;
}
if (err?.code !== 'EEXIST' || Date.now() >= deadline) {
throw new Error(`terminal-agent state lock unavailable at ${lockPath}: ${err?.code || err}; inspect the owning process before manual recovery`);
}
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 25);
}
}
const owned = fs.fstatSync(fd, { bigint: true });
return () => {
try {
const current = fs.statSync(lockPath, { bigint: true });
if (current.dev === owned.dev && current.ino === owned.ino) fs.unlinkSync(lockPath);
} catch (err: any) {
if (err?.code !== 'ENOENT') throw err;
} finally { fs.closeSync(fd); }
};
}
/**
* Locate the terminal-agent script on disk. In dev (cli.ts running via
* `bun run`), it lives next to this file in browse/src. In a compiled
* binary, Bun's --compile bakes the source into the executable and
* exposes it relative to process.execPath. Either path must work or
* the agent can't be spawned at all.
*/
/**
* Flags for every `bun run` child the daemon spawns. The child runs in the
* user's project, and Bun would otherwise load that project's .env (which can
* set GSTACK_CHROMIUM_NO_SANDBOX) and bunfig.toml (whose preload runs code).
* Built per spawn so the empty config is re-checked each time.
*/
export function bunChildFlags(): string[] {
return ['--no-env-file', `--config=${emptyBunConfig()}`];
}
/**
* The bun executable to spawn the agent with. On Windows a bare 'bun' under
* the Node polyfill resolves (via cross-spawn) to npm's bun.cmd shim, run as
* `cmd.exe /c bun ...`. That breaks two ways: windowsHide only hides cmd.exe,
* and the detached cmd.exe has no console, so bun.exe allocates a fresh
* visible one (a window that steals focus on every spawn); and proc.pid is
* cmd.exe's, so the agent's startup check (record.pid === process.pid) never
* matches and it exits after 2s, which the 60s watchdog then respawns.
* Spawning bun.exe directly fixes both. Falls back to 'bun' when no .exe is
* found, which is the old behavior.
*/
export function resolveBunExecutable(opts: BunExeLookup & { platform?: string } = {}): string {
if ((opts.platform ?? process.platform) !== 'win32') return 'bun';
return findWindowsBunExe(opts) ?? 'bun';
}
export function resolveTerminalAgentScript(searchHints: { metaDir?: string; execPath?: string } = {}): string | null {
const meta = searchHints.metaDir || __dirname;
const exec = searchHints.execPath || process.execPath;
const candidates = [
path.resolve(meta, 'terminal-agent.ts'),
path.resolve(path.dirname(exec), '..', 'src', 'terminal-agent.ts'),
];
for (const c of candidates) {
if (fs.existsSync(c)) return c;
}
return null;
}
/**
* Spawn an owned terminal-agent. A prior record is retained until its exact
* process exits, and the new generation is recorded before it may bind.
*
* Used by both the CLI cold-start path (cli.ts) and the v1.44 watchdog in
* server.ts. Centralizing here removes a copy-paste between them and means
* spawn-env additions (BROWSE_OWNER_PID being the first) land in one place.
*/
export function spawnTerminalAgent(opts: {
stateFile: string;
serverPort: number;
/** PID of the browse server that owns this agent. */
ownerPid: number;
cwd?: string;
/** Optional extra env vars to add to the agent's process env. */
extraEnv?: Record<string, string>;
/** Override script lookup for tests. */
scriptPath?: string;
}): number | null {
if (!Number.isSafeInteger(opts.ownerPid) || opts.ownerPid <= 0) throw new Error('terminal-agent requires a daemon owner PID');
const stateDir = path.dirname(opts.stateFile);
const script = opts.scriptPath || resolveTerminalAgentScript();
if (!script || !fs.existsSync(script)) return null;
const release = acquireAgentStateLock(stateDir);
try {
const prior = readAgentRecord(stateDir);
if (prior) {
if (prior.pid === 0) {
console.warn('[browse] terminal-agent startup or failed exit remains unconfirmed; retaining its reservation');
return null;
}
if (isAgentRecordLive(prior) && !stopAgentByRecord(prior)) {
console.warn(`[browse] terminal-agent PID ${prior.pid} is still running or its identity cannot be confirmed; refusing a second agent`);
return null;
}
clearAgentRecord(stateDir, prior);
safeUnlink(path.join(stateDir, 'terminal-port'));
safeUnlink(path.join(stateDir, 'terminal-internal-token'));
}
const ownerStartTime = readAgentStartTime(opts.ownerPid);
if (!ownerStartTime) throw new Error('terminal-agent owner identity is unavailable');
const gen = crypto.randomBytes(16).toString('base64url');
const reservation: AgentRecord = { pid: 0, gen, startedAt: Date.now(), ownerPid: opts.ownerPid, ownerStartTime };
writeAgentRecord(stateDir, reservation);
let proc: any;
try {
proc = (Bun as any).spawn([resolveBunExecutable(), 'run', ...bunChildFlags(), script, `--agent-gen=${gen}`], {
cwd: opts.cwd || process.cwd(),
env: {
...process.env,
...(opts.extraEnv || {}),
BROWSE_STATE_FILE: opts.stateFile,
BROWSE_SERVER_PORT: String(opts.serverPort),
BROWSE_OWNER_PID: String(opts.ownerPid),
BROWSE_OWNER_START_TIME: ownerStartTime,
BROWSE_AGENT_GEN: gen,
},
stdio: ['ignore', 'ignore', 'ignore'],
windowsHide: true,
// A non-detached Windows child is killed with its parent's job, so an
// agent started by `browse connect` died when the CLI exited (#2637).
// The owner watchdog (BROWSE_OWNER_PID) still ends it with the daemon.
detached: process.platform === 'win32',
});
} catch (err) {
clearAgentRecord(stateDir, reservation);
throw err;
}
const retainUntilExit = () => {
pendingAgentExits.add(proc);
proc.exited?.then(() => {
try {
const releasePending = acquireAgentStateLock(stateDir);
try { clearAgentRecord(stateDir, reservation); } finally { releasePending(); }
} catch (err) { console.warn('[browse] terminal-agent pending exit cleanup failed:', err); }
pendingAgentExits.delete(proc);
}, (err: unknown) => console.warn('[browse] terminal-agent exit remains unconfirmed:', err));
};
const pid = proc.pid;
const startTime = pid ? readAgentStartTime(pid) : '';
if (!pid || !startTime) {
try { proc.kill('SIGTERM'); } catch {}
retainUntilExit();
throw new Error('terminal-agent process identity is unavailable');
}
const record: AgentRecord = { pid, gen, startedAt: Date.now(), startTime, ownerPid: opts.ownerPid, ownerStartTime };
try {
writeAgentRecord(stateDir, record);
} catch (err) {
if (!stopAgentByRecord(record)) {
retainUntilExit();
throw new Error(`terminal-agent record update failed and child ${pid} exit is unconfirmed: ${err}`);
}
clearAgentRecord(stateDir, reservation);
throw err;
}
proc.unref?.();
return pid;
} finally { release(); }
}
export interface AgentRecord {
pid: number;
/** Random per-boot identifier. Loopback /internal/* sees X-Browse-Gen: <gen>. */
gen: string;
/** ms since epoch. Reserved for future PID-reuse guards. */
startedAt: number;
startTime?: string;
ownerPid?: number;
ownerStartTime?: string;
}
export function agentRecordPath(stateDir: string): string {
return path.join(stateDir, 'terminal-agent-pid');
}
/** Read the current record. Returns null on missing/malformed file. */
export function readAgentRecord(stateDir: string): AgentRecord | null {
try {
const raw = fs.readFileSync(agentRecordPath(stateDir), 'utf-8');
const j = JSON.parse(raw);
if (typeof j?.pid === 'number' && typeof j?.gen === 'string' && typeof j?.startedAt === 'number') {
return j as AgentRecord;
}
return null;
} catch {
return null;
}
}
/** Atomic write (throws on failure — boot must not proceed on a bad record). */
export function writeAgentRecord(stateDir: string, record: AgentRecord): void {
try { mkdirSecure(stateDir); } catch {}
const target = agentRecordPath(stateDir);
atomicWriteSync(target, JSON.stringify(record), { mode: 0o600 });
// Windows ACL hardening (POSIX chmod is redundant with mode above).
restrictFilePermissions(target);
}
export function clearAgentRecord(stateDir: string, expected?: AgentRecord): void {
if (expected) {
const current = readAgentRecord(stateDir);
if (!current || current.pid !== expected.pid || current.gen !== expected.gen) return;
}
safeUnlink(agentRecordPath(stateDir));
}
export function isAgentRecordLive(record: AgentRecord): boolean {
return Number.isSafeInteger(record.pid) && record.pid > 0 && isProcessAlive(record.pid);
}
function agentStatus(record: AgentRecord, ownerPid?: number): 'owned' | 'gone' | 'unknown' {
if (!isAgentRecordLive(record)) return 'gone';
if (!record.startTime || !record.ownerPid || !record.ownerStartTime) return 'unknown';
if (ownerPid !== undefined && (record.ownerPid !== ownerPid || record.ownerStartTime !== readAgentStartTime(ownerPid))) return 'unknown';
const actual = agentProcessInfo(record.pid);
if (!actual.startTime) return isAgentRecordLive(record) ? 'unknown' : 'gone';
if (actual.startTime !== record.startTime && process.platform !== 'win32') return 'gone';
if (actual.startTime !== record.startTime && !actual.commandLine) return 'unknown';
try {
let state: string | undefined;
if (process.platform === 'linux') {
state = fs.readFileSync(`/proc/${record.pid}/stat`, 'utf8').match(/^\d+ \(.*\) ([A-Z])/u)?.[1];
} else if (process.platform === 'darwin') {
const result = spawnSync('ps', ['-p', String(record.pid), '-o', 'stat='], { encoding: 'utf8', windowsHide: true, timeout: 2000 });
if (result.status === 0) state = result.stdout?.trim()?.[0];
}
if (state === 'Z') return 'gone';
} catch {}
if (!isAgentRecordLive(record)) return 'gone';
return actual.commandLine.split(/\s+/).some(arg => arg.replace(/^['"]|['"]$/g, '') === `--agent-gen=${record.gen}`)
? 'owned' : actual.startTime === record.startTime ? 'unknown' : 'gone';
}
export function isOurAgent(record: AgentRecord, ownerPid?: number): boolean {
return agentStatus(record, ownerPid) === 'owned';
}
export function isAgentRecordGone(record: AgentRecord): boolean {
return agentStatus(record) === 'gone';
}
/**
* Kill the agent identified by `record`. Signal defaults to SIGTERM (give
* the agent a chance to run its own SIGTERM cleanup). Returns true if a
* signal reached an exact-generation live process, false otherwise.
*/
export function killAgentByRecord(
record: AgentRecord,
signal: NodeJS.Signals = 'SIGTERM',
): boolean {
if (!isOurAgent(record)) return false;
try { process.kill(record.pid, signal); return true; } catch { return false; }
}
export function stopAgentByRecord(record: AgentRecord, graceMs = 1000): boolean {
const initial = agentStatus(record);
if (initial === 'gone') return true;
if (initial !== 'owned') return false;
const waitForExit = (ms: number) => {
const deadline = Date.now() + ms;
while (Date.now() < deadline) {
const status = agentStatus(record);
if (status === 'gone') return true;
if (status === 'unknown') return false;
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 25);
}
return agentStatus(record) === 'gone';
};
if (!killAgentByRecord(record, 'SIGTERM')) return agentStatus(record) === 'gone';
if (waitForExit(graceMs)) return true;
const afterGrace = agentStatus(record);
if (afterGrace !== 'owned') return afterGrace === 'gone';
if (!killAgentByRecord(record, 'SIGKILL')) return agentStatus(record) === 'gone';
return waitForExit(graceMs);
}