mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-03 01:46:55 +02:00
fix(browse): make connect --supervise actually respawn a crashed server
The supervisor respawned with a block-scoped env that no longer existed, so every attempt threw and the loop gave up after five tries. The headed env is now one pure helper used by connect and respawn, the loop is an injectable runHeadedSupervisor with behavioral tests, failures name the daemon log and relaunch command, and connect's usage advertises --supervise.
This commit is contained in:
garrytan committed
1 parent
efdf62414a
commit
03e5911dba
5 files changed
+242
-75
No files matched your search
+123
-69
@@ -130,7 +130,7 @@ interface ServerState {
|
||||
configHash?: string;
|
||||
/** Xvfb child PID for cleanup on disconnect. */
|
||||
xvfbPid?: number;
|
||||
xvfbStartTime?: number;
|
||||
xvfbStartTime?: string;
|
||||
xvfbDisplay?: string;
|
||||
/** Launched-Chromium identity for post-stop reaping (#2709). */
|
||||
chromiumPid?: number;
|
||||
@@ -423,6 +423,102 @@ export function buildRestartEnv(
|
||||
return env;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the env for the headed `$B connect` server. Used by the initial
|
||||
* connect and by the opt-in supervisor's respawn, so a respawned server keeps
|
||||
* the same port, watchdog setting, proxy and config hash. Pure + exported for tests.
|
||||
*/
|
||||
export function buildHeadedServerEnv(
|
||||
globalFlags: Pick<GlobalFlags, 'proxyUrl' | 'configHash'>,
|
||||
): Record<string, string> {
|
||||
return {
|
||||
BROWSE_HEADED: '1',
|
||||
// Use a well-known port so the Chrome extension auto-connects.
|
||||
BROWSE_PORT: '34567',
|
||||
// Disable parent-process watchdog: the user controls the headed browser
|
||||
// window lifecycle. The CLI exits immediately after connect, so watching
|
||||
// it would kill the server ~15s later. Cleanup happens via browser
|
||||
// disconnect event or $B disconnect.
|
||||
BROWSE_PARENT_PID: '0',
|
||||
// Apply --proxy from this invocation if present. Without this,
|
||||
// `browse --proxy <url> connect` would launch headed Chromium
|
||||
// bypassing the SOCKS bridge entirely.
|
||||
...(globalFlags.proxyUrl ? { BROWSE_PROXY_URL: globalFlags.proxyUrl } : {}),
|
||||
...(globalFlags.configHash ? { BROWSE_CONFIG_HASH: globalFlags.configHash } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export const SUPERVISOR_GUARD_WINDOW_MS = 5 * 60_000;
|
||||
export const SUPERVISOR_GUARD_MAX = 5;
|
||||
|
||||
export interface HeadedSupervisorDeps {
|
||||
env: Record<string, string>;
|
||||
tickMs: number;
|
||||
backoffMs: number[];
|
||||
daemonLog: string;
|
||||
readState: () => { pid?: number } | null;
|
||||
isProcessAlive: (pid: number) => boolean;
|
||||
startServer: (env: Record<string, string>) => Promise<{ pid: number; port: number }>;
|
||||
spawnTerminalAgent: (server: { pid: number; port: number }) => void;
|
||||
sleep: (ms: number) => Promise<void>;
|
||||
now: () => number;
|
||||
isExiting: () => boolean;
|
||||
log: (line: string) => void;
|
||||
warn: (line: string) => void;
|
||||
error: (line: string) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* The opt-in `$B connect --supervise` loop: poll the server PID every tick and
|
||||
* respawn it with the connect env when it dies. Five respawns inside the
|
||||
* rolling five-minute window give up. Returns 'stopped' when a signal asked it
|
||||
* to exit and 'gave_up' when the crash-loop guard tripped.
|
||||
*/
|
||||
export async function runHeadedSupervisor(deps: HeadedSupervisorDeps): Promise<'stopped' | 'gave_up'> {
|
||||
const respawns: number[] = [];
|
||||
while (!deps.isExiting()) {
|
||||
await deps.sleep(deps.tickMs);
|
||||
if (deps.isExiting()) break;
|
||||
const state = deps.readState();
|
||||
if (state?.pid && deps.isProcessAlive(state.pid)) continue;
|
||||
// Server died. Prune rolling window and check guard.
|
||||
const now = deps.now();
|
||||
while (respawns.length && now - respawns[0] > SUPERVISOR_GUARD_WINDOW_MS) {
|
||||
respawns.shift();
|
||||
}
|
||||
if (respawns.length >= SUPERVISOR_GUARD_MAX) {
|
||||
deps.error(
|
||||
`[browse] Supervisor: ${SUPERVISOR_GUARD_MAX} server crashes in ${SUPERVISOR_GUARD_WINDOW_MS / 1000}s, giving up. ` +
|
||||
`Crash reasons: ${deps.daemonLog}. Relaunch: $B connect --supervise`,
|
||||
);
|
||||
return 'gave_up';
|
||||
}
|
||||
const attempt = respawns.length;
|
||||
respawns.push(now);
|
||||
const backoff = deps.backoffMs[Math.min(attempt, deps.backoffMs.length - 1)] ?? 30_000;
|
||||
deps.warn(`[browse] Supervisor: server PID gone — respawning in ${backoff}ms (attempt ${attempt + 1}/${SUPERVISOR_GUARD_MAX})...`);
|
||||
await deps.sleep(backoff);
|
||||
if (deps.isExiting()) break;
|
||||
let respawned: { pid: number; port: number };
|
||||
try {
|
||||
respawned = await deps.startServer(deps.env);
|
||||
} catch (err: any) {
|
||||
// Let the next tick try again — the crash-loop guard already
|
||||
// bounded the retries via the rolling window.
|
||||
deps.error(`[browse] Supervisor: server respawn failed: ${err?.message || err}. Daemon log: ${deps.daemonLog}`);
|
||||
continue;
|
||||
}
|
||||
deps.log(`[browse] Supervisor: server respawned (PID ${respawned.pid}, port ${respawned.port}).`);
|
||||
// Re-spawn the terminal-agent too; same env wiring as the initial connect.
|
||||
try {
|
||||
deps.spawnTerminalAgent(respawned);
|
||||
} catch (err: any) {
|
||||
deps.warn(`[browse] Supervisor: terminal-agent respawn failed: ${err?.message || err}`);
|
||||
}
|
||||
}
|
||||
return 'stopped';
|
||||
}
|
||||
|
||||
/** macOS only: pull the headed Chromium window to the user's current Space.
|
||||
* "Google Chrome for Testing" frequently opens behind the active window or on
|
||||
* another Space — the first thing users read as "I can't see the browser"
|
||||
@@ -1640,22 +1736,7 @@ Refs: After 'snapshot', use @e1, @e2... as selectors:
|
||||
console.log('Launching headed Chromium with extension + terminal agent...');
|
||||
try {
|
||||
// Start server in headed mode with extension auto-loaded
|
||||
// Use a well-known port so the Chrome extension auto-connects
|
||||
const serverEnv: Record<string, string> = {
|
||||
BROWSE_HEADED: '1',
|
||||
BROWSE_PORT: '34567',
|
||||
// Disable parent-process watchdog: the user controls the headed browser
|
||||
// window lifecycle. The CLI exits immediately after connect, so watching
|
||||
// it would kill the server ~15s later. Cleanup happens via browser
|
||||
// disconnect event or $B disconnect.
|
||||
BROWSE_PARENT_PID: '0',
|
||||
// Apply --proxy from this invocation if present. Without this,
|
||||
// `browse --proxy <url> connect` would launch headed Chromium
|
||||
// bypassing the SOCKS bridge entirely.
|
||||
...(globalFlags.proxyUrl ? { BROWSE_PROXY_URL: globalFlags.proxyUrl } : {}),
|
||||
...(globalFlags.configHash ? { BROWSE_CONFIG_HASH: globalFlags.configHash } : {}),
|
||||
};
|
||||
const newState = await startServer(serverEnv);
|
||||
const newState = await startServer(buildHeadedServerEnv(globalFlags));
|
||||
|
||||
// Print connected status
|
||||
const resp = await fetch(`http://127.0.0.1:${newState.port}/command`, {
|
||||
@@ -1737,58 +1818,31 @@ Refs: After 'snapshot', use @e1, @e2... as selectors:
|
||||
process.on('SIGINT', () => teardownAndExit('SIGINT'));
|
||||
process.on('SIGTERM', () => teardownAndExit('SIGTERM'));
|
||||
|
||||
const SUPERVISOR_TICK_MS = parseInt(
|
||||
process.env.GSTACK_SUPERVISOR_TICK_MS || '30000',
|
||||
10,
|
||||
);
|
||||
const SUPERVISOR_GUARD_WINDOW_MS = 5 * 60_000;
|
||||
const SUPERVISOR_GUARD_MAX = 5;
|
||||
const SUPERVISOR_BACKOFF_MS = (process.env.GSTACK_SUPERVISOR_BACKOFF || '1000,2000,4000,8000,30000')
|
||||
.split(',').map(s => parseInt(s.trim(), 10)).filter(n => Number.isFinite(n));
|
||||
const respawns: number[] = [];
|
||||
|
||||
while (!supervisorExiting) {
|
||||
await new Promise(resolve => setTimeout(resolve, SUPERVISOR_TICK_MS));
|
||||
if (supervisorExiting) break;
|
||||
const state = readState();
|
||||
if (state?.pid && isProcessAlive(state.pid)) continue;
|
||||
// Server died. Prune rolling window and check guard.
|
||||
const now = Date.now();
|
||||
while (respawns.length && now - respawns[0] > SUPERVISOR_GUARD_WINDOW_MS) {
|
||||
respawns.shift();
|
||||
}
|
||||
if (respawns.length >= SUPERVISOR_GUARD_MAX) {
|
||||
console.error(
|
||||
`[browse] Supervisor: ${SUPERVISOR_GUARD_MAX} crashes in ${SUPERVISOR_GUARD_WINDOW_MS / 1000}s — giving up.`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
const attempt = respawns.length;
|
||||
respawns.push(now);
|
||||
const backoff = SUPERVISOR_BACKOFF_MS[Math.min(attempt, SUPERVISOR_BACKOFF_MS.length - 1)] ?? 30_000;
|
||||
console.warn(`[browse] Supervisor: server PID gone — respawning in ${backoff}ms (attempt ${attempt + 1}/${SUPERVISOR_GUARD_MAX})...`);
|
||||
await new Promise(resolve => setTimeout(resolve, backoff));
|
||||
if (supervisorExiting) break;
|
||||
try {
|
||||
const respawned = await startServer(serverEnv);
|
||||
console.log(`[browse] Supervisor: server respawned (PID ${respawned.pid}, port ${respawned.port}).`);
|
||||
// Re-spawn the terminal-agent too; same env wiring as the initial connect.
|
||||
try {
|
||||
spawnTerminalAgent({
|
||||
stateFile: config.stateFile,
|
||||
serverPort: respawned.port,
|
||||
ownerPid: respawned.pid,
|
||||
cwd: config.projectDir,
|
||||
});
|
||||
} catch (err: any) {
|
||||
console.warn(`[browse] Supervisor: terminal-agent respawn failed: ${err?.message || err}`);
|
||||
}
|
||||
} catch (err: any) {
|
||||
console.error(`[browse] Supervisor: server respawn failed: ${err?.message || err}`);
|
||||
// Let the next tick try again — the crash-loop guard already
|
||||
// bounded the retries via the rolling window.
|
||||
}
|
||||
}
|
||||
const outcome = await runHeadedSupervisor({
|
||||
env: buildHeadedServerEnv(globalFlags),
|
||||
tickMs: parseInt(process.env.GSTACK_SUPERVISOR_TICK_MS || '30000', 10),
|
||||
backoffMs: (process.env.GSTACK_SUPERVISOR_BACKOFF || '1000,2000,4000,8000,30000')
|
||||
.split(',').map(s => parseInt(s.trim(), 10)).filter(n => Number.isFinite(n)),
|
||||
daemonLog: daemonLogPath(),
|
||||
readState,
|
||||
isProcessAlive,
|
||||
startServer,
|
||||
spawnTerminalAgent: (respawned) => {
|
||||
spawnTerminalAgent({
|
||||
stateFile: config.stateFile,
|
||||
serverPort: respawned.port,
|
||||
ownerPid: respawned.pid,
|
||||
cwd: config.projectDir,
|
||||
});
|
||||
},
|
||||
sleep: (ms) => new Promise(resolve => setTimeout(resolve, ms)),
|
||||
now: Date.now,
|
||||
isExiting: () => supervisorExiting,
|
||||
log: (line) => console.log(line),
|
||||
warn: (line) => console.warn(line),
|
||||
error: (line) => console.error(line),
|
||||
});
|
||||
if (outcome === 'gave_up') process.exit(1);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
|
||||
@@ -161,7 +161,7 @@ export const COMMAND_DESCRIPTIONS: Record<string, { category: string; descriptio
|
||||
'handoff': { category: 'Server', description: 'Open visible Chrome at current page for user takeover', usage: 'handoff [message]' },
|
||||
'resume': { category: 'Server', description: 'Re-snapshot after user takeover, return control to AI', usage: 'resume' },
|
||||
// Headed mode
|
||||
'connect': { category: 'Server', description: 'Launch headed Chromium with Chrome extension', usage: 'connect' },
|
||||
'connect': { category: 'Server', description: 'Launch headed Chromium with Chrome extension; --supervise keeps the CLI attached and respawns a crashed server', usage: 'connect [--supervise]' },
|
||||
'disconnect': { category: 'Server', description: 'Disconnect headed browser, return to headless mode' },
|
||||
'focus': { category: 'Server', description: 'Bring headed browser window to foreground (macOS)', usage: 'focus [@ref]' },
|
||||
// Inbox
|
||||
|
||||
Reference in new issue
Block a user