test: free tests never write the real HOME (home-write tripwire), no orphaned browse daemons, symlinked-TMPDIR (macOS) fixes, gbrain-on-PATH isolation (#2895, #2816, #2829, #2841, #2982, #2911, #2909, #2908, #2910)

This commit is contained in:
garrytan committed 2026-10-02 20:09:36 +00:00
1 parent aefcb1bd4e
commit 615c0dcaa6
38 files changed
+447 -65

No files matched your search

+2 -4
View File
@@ -21,8 +21,6 @@ const GSTACK_DEV_DIR = path.join(os.homedir(), '.gstack-dev');
// getProjectEvalDir() (or GSTACK_EVAL_DIR), so watching the legacy global
// path missed it whenever slug detection succeeded — i.e. the normal case.
const HEARTBEAT_PATH = path.join(GSTACK_DEV_DIR, 'e2e-live.json');
const EVAL_DIR = process.env.GSTACK_EVAL_DIR || getProjectEvalDir();
const PARTIAL_PATH = path.join(EVAL_DIR, '_partial-e2e*.json');
const STALE_THRESHOLD_SEC = 600; // 10 minutes
export interface HeartbeatData {
@@ -54,7 +52,7 @@ function readJSON<T>(filePath: string): T | null {
}
/** Read the legacy collector and suite collectors in this shard, never finals. */
export function readPartialResults(evalDir = EVAL_DIR): PartialData | null {
export function readPartialResults(evalDir = process.env.GSTACK_EVAL_DIR || getProjectEvalDir()): PartialData | null {
let names: string[];
try { names = fs.readdirSync(evalDir).sort(); } catch { return null; }
const partials = names.filter(name => /^_partial-e2e(?:-[a-z0-9]+(?:-[a-z0-9]+)*)?\.json$/.test(name))
@@ -98,7 +96,7 @@ export function renderDashboard(heartbeat: HeartbeatData | null, partial: Partia
lines.push('E2E Watch — No active run detected');
lines.push('');
lines.push(`Heartbeat: ${HEARTBEAT_PATH} (not found)`);
lines.push(`Partial: ${PARTIAL_PATH} (not found)`);
lines.push(`Partial: ${path.join(process.env.GSTACK_EVAL_DIR || getProjectEvalDir(), '_partial-e2e*.json')} (not found)`);
lines.push('');
lines.push('Start a run with: EVALS=1 bun test test/skill-e2e-*.test.ts');
return lines.join('\n');
+102
View File
@@ -0,0 +1,102 @@
/**
* Free-suite home-write tripwire (#2895).
*
* Snapshots path + mtime + size of the gstack install and config surfaces under
* the real home before and after each free shard, and names the shard's files
* when one of those entries changed. It watches only these surfaces; it is not
* a universal write-containment check. Live agent-session logs that a
* concurrently running host keeps writing are excluded, and directory mtimes
* are ignored, so only file, link and directory-set changes count. Concurrent
* shards whose windows overlap a change all report it.
*/
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
export const FREE_HOME_SURFACES = ['.gstack', '.claude', '.codex', '.agents', '.config/gstack'] as const;
export const FREE_HOME_VOLATILE = [
'.claude/projects', '.claude/todos', '.claude/shell-snapshots', '.claude/statsig', '.claude/ide',
'.claude/session-env', '.claude/file-history', '.claude/debug', '.claude/telemetry', '.claude/history.jsonl',
'.claude/.credentials.json', '.codex/sessions', '.codex/archived_sessions', '.codex/log', '.codex/history.jsonl',
'.codex/auth.json',
] as const;
const MAX_DEPTH = 4;
const MAX_REPORTED = 10;
export type FreeHomeSnapshot = Map<string, string>;
export function freeHomeDir(env: NodeJS.ProcessEnv = process.env): string {
return env.HOME || env.USERPROFILE || os.homedir();
}
export function snapshotFreeHome(home: string): FreeHomeSnapshot {
const entries: FreeHomeSnapshot = new Map();
const volatile = new Set<string>(FREE_HOME_VOLATILE);
const visit = (relative: string, depth: number) => {
if (volatile.has(relative)) return;
const absolute = path.join(home, relative);
let stat: fs.BigIntStats;
try {
stat = fs.lstatSync(absolute, { bigint: true });
} catch (error) {
if (['ENOENT', 'ENOTDIR'].includes((error as NodeJS.ErrnoException).code ?? '')) return;
throw error;
}
if (stat.isSymbolicLink()) {
entries.set(relative, `link:${fs.readlinkSync(absolute)}:${stat.mtimeNs}`);
return;
}
if (!stat.isDirectory()) {
entries.set(relative, `file:${stat.size}:${stat.mtimeNs}`);
return;
}
entries.set(relative, 'dir');
if (depth >= MAX_DEPTH) return;
let names: string[];
try {
names = fs.readdirSync(absolute);
} catch (error) {
const code = (error as NodeJS.ErrnoException).code ?? '';
if (['EACCES', 'EPERM'].includes(code)) entries.set(relative, 'dir:unreadable');
if (['ENOENT', 'ENOTDIR', 'EACCES', 'EPERM'].includes(code)) return;
throw error;
}
for (const name of names.sort()) visit(path.join(relative, name), depth + 1);
};
for (const surface of FREE_HOME_SURFACES) visit(path.normalize(surface), 0);
return entries;
}
export function diffFreeHome(before: FreeHomeSnapshot, after: FreeHomeSnapshot): string[] {
const changed = new Set<string>();
for (const [entry, value] of before) if (after.get(entry) !== value) changed.add(entry);
for (const entry of after.keys()) if (!before.has(entry)) changed.add(entry);
return [...changed].sort();
}
export function formatFreeHomeChange(changed: string[], files: string[]): string {
const shown = changed.slice(0, MAX_REPORTED).map(entry => `~/${entry.split(path.sep).join('/')}`);
const more = changed.length > MAX_REPORTED ? ` (+${changed.length - MAX_REPORTED} more)` : '';
return `real home changed while this shard ran (watches ${FREE_HOME_SURFACES.map(s => `~/${s}`).join(', ')} only): `
+ `${shown.join(', ')}${more}. Give the writer a private HOME/GSTACK_HOME. Shard files: ${files.join(', ')}`;
}
/** Take the baseline now; verify() returns null or the failure to report. */
export function guardFreeHome(files: string[], env: NodeJS.ProcessEnv = process.env): { verify(): string | null } {
const home = freeHomeDir(env);
const unreadable = (error: unknown) => `real home surfaces could not be snapshotted (${(error as NodeJS.ErrnoException).code ?? 'error'}); shard files: ${files.join(', ')}`;
let before: FreeHomeSnapshot | undefined;
let baselineError: string | undefined;
try { before = snapshotFreeHome(home); } catch (error) { baselineError = unreadable(error); }
return {
verify() {
if (!before) return baselineError!;
try {
const changed = diffFreeHome(before, snapshotFreeHome(home));
return changed.length ? formatFreeHomeChange(changed, files) : null;
} catch (error) { return unreadable(error); }
},
};
}
+7 -7
View File
@@ -86,6 +86,7 @@ import { StringDecoder } from 'node:string_decoder';
import { createHash, randomUUID } from 'node:crypto';
import { isPaidTestFile } from '../test/helpers/paid-test-set';
import { resolveStateRoot } from '../lib/state-root';
import { guardFreeHome } from './lib/free-home-guard';
import {
BunTestOutputClassifier,
createShardSandbox,
@@ -107,7 +108,6 @@ import {
type LanePolicy,
type ShardChildResult,
} from './lib/shard-engine';
export { normalizeRelativePath } from './lib/shard-engine';
/**
@@ -354,6 +354,7 @@ export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }>
// pattern hit is a false positive — the point of these files is Windows
// coverage, so auto-excluding them defeats the regression tests they carry.
const KNOWN_WINDOWS_SAFE: Array<{ file: string; reason: string }> = [
{ file: 'test/ship-hook-windows-paths.test.ts', reason: 'runs bin/ helpers through explicit bash and Bun argv with forward-slash paths; never executes a shebang; the path-spelling simulation is skipIf win32' },
{
file: 'test/state-root-parity.test.ts',
reason: 'runs the bash twin and lib/state-root.ts over an env table with PATH empty; no shebang execution, raw-string comparison is platform-neutral',
@@ -384,8 +385,7 @@ const KNOWN_WINDOWS_SAFE: Array<{ file: string; reason: string }> = [
},
{
file: 'test/claude-code-runner.test.ts',
// The bin/ path is launched through process.execPath (Bun), never as a
// shebang executable. Keep taskkill tree supervision in the Windows lane.
// The bin/ path is launched through process.execPath (Bun), never as a shebang; keep taskkill supervision in the Windows lane.
reason: 'invokes the runner via Bun argv; fake CLI and timeout descendant assertions cover native Windows taskkill',
},
{
@@ -1810,7 +1810,7 @@ function explainFreeVerdict(label: string, status: FreeShardStatus, facts: {
summary: ReturnType<BunTestOutputClassifier['end']>; expectedFiles: number; wallTimeoutMs: number;
}): void {
const { summary, exitCode } = facts;
if (facts.cleanupError) console.error(`${label} browser cleanup failed: ${facts.cleanupError}; retained ${facts.stateDir}`);
if (facts.cleanupError) console.error(`${label} shard cleanup or home containment failed: ${facts.cleanupError}; retained ${facts.stateDir}`);
if (status === 'timed-out') {
console.error(
`${label} exceeded the ${Math.round(facts.wallTimeoutMs / 1000)}s wall-clock deadline — `
@@ -1832,7 +1832,7 @@ function explainFreeVerdict(label: string, status: FreeShardStatus, facts: {
function logFreeRecovery(log: (line: string) => void, outcome: FreeShardOutcome, facts: {
cleanupError: string | null; logWriteFailed: boolean; captureIncomplete: boolean; rootDir: string;
}): void {
const problem = facts.cleanupError ? 'Owned-process cleanup is unconfirmed; inspect the retained state before another run.'
const problem = facts.cleanupError ? 'Owned-process cleanup or home containment is unconfirmed; inspect the reported paths and retained state before another run.'
: facts.logWriteFailed ? 'The evidence log could not be retained; repair the log destination before another run.'
: facts.captureIncomplete ? 'Evidence capture is incomplete; repair the stream or early exit before another run.'
: outcome.status === 'timed-out' ? 'Execution exceeded its deadline; inspect the last completed step before changing code or rerunning.'
@@ -1952,8 +1952,8 @@ export async function runFreeShard(
child = await runShardChild({
command, args, cwd: rootDir, env, timeoutMs: wallTimeoutMs,
attach: () => {
const browser = trackShardBrowser(stateDir, env);
return { signal: (force) => browser.signal(force), settle: async () => { cleanupError = await browser.settle(); } };
const browser = trackShardBrowser(stateDir, env), home = guardFreeHome(files, env);
return { signal: (force) => browser.signal(force), settle: async () => { cleanupError = await browser.settle() ?? home.verify(); } };
},
hookStreams: (spawned) => [consumeStream(spawned.stdout, 'stdout'), consumeStream(spawned.stderr, 'stderr')],
});