mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-09 18:28:15 +01:00
test: free tests never write the real HOME (home-write tripwire), no orphaned browse daemons, symlinked-TMPDIR (macOS) fixes, gbrain-on-PATH isolation (#2895, #2816, #2829, #2841, #2982, #2911, #2909, #2908, #2910)
This commit is contained in:
1 parent
aefcb1bd4e
commit
615c0dcaa6
38 files changed
+447
-65
No files matched your search
@@ -21,8 +21,6 @@ const GSTACK_DEV_DIR = path.join(os.homedir(), '.gstack-dev');
|
||||
// getProjectEvalDir() (or GSTACK_EVAL_DIR), so watching the legacy global
|
||||
// path missed it whenever slug detection succeeded — i.e. the normal case.
|
||||
const HEARTBEAT_PATH = path.join(GSTACK_DEV_DIR, 'e2e-live.json');
|
||||
const EVAL_DIR = process.env.GSTACK_EVAL_DIR || getProjectEvalDir();
|
||||
const PARTIAL_PATH = path.join(EVAL_DIR, '_partial-e2e*.json');
|
||||
const STALE_THRESHOLD_SEC = 600; // 10 minutes
|
||||
|
||||
export interface HeartbeatData {
|
||||
@@ -54,7 +52,7 @@ function readJSON<T>(filePath: string): T | null {
|
||||
}
|
||||
|
||||
/** Read the legacy collector and suite collectors in this shard, never finals. */
|
||||
export function readPartialResults(evalDir = EVAL_DIR): PartialData | null {
|
||||
export function readPartialResults(evalDir = process.env.GSTACK_EVAL_DIR || getProjectEvalDir()): PartialData | null {
|
||||
let names: string[];
|
||||
try { names = fs.readdirSync(evalDir).sort(); } catch { return null; }
|
||||
const partials = names.filter(name => /^_partial-e2e(?:-[a-z0-9]+(?:-[a-z0-9]+)*)?\.json$/.test(name))
|
||||
@@ -98,7 +96,7 @@ export function renderDashboard(heartbeat: HeartbeatData | null, partial: Partia
|
||||
lines.push('E2E Watch — No active run detected');
|
||||
lines.push('');
|
||||
lines.push(`Heartbeat: ${HEARTBEAT_PATH} (not found)`);
|
||||
lines.push(`Partial: ${PARTIAL_PATH} (not found)`);
|
||||
lines.push(`Partial: ${path.join(process.env.GSTACK_EVAL_DIR || getProjectEvalDir(), '_partial-e2e*.json')} (not found)`);
|
||||
lines.push('');
|
||||
lines.push('Start a run with: EVALS=1 bun test test/skill-e2e-*.test.ts');
|
||||
return lines.join('\n');
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
/**
|
||||
* Free-suite home-write tripwire (#2895).
|
||||
*
|
||||
* Snapshots path + mtime + size of the gstack install and config surfaces under
|
||||
* the real home before and after each free shard, and names the shard's files
|
||||
* when one of those entries changed. It watches only these surfaces; it is not
|
||||
* a universal write-containment check. Live agent-session logs that a
|
||||
* concurrently running host keeps writing are excluded, and directory mtimes
|
||||
* are ignored, so only file, link and directory-set changes count. Concurrent
|
||||
* shards whose windows overlap a change all report it.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
|
||||
export const FREE_HOME_SURFACES = ['.gstack', '.claude', '.codex', '.agents', '.config/gstack'] as const;
|
||||
|
||||
export const FREE_HOME_VOLATILE = [
|
||||
'.claude/projects', '.claude/todos', '.claude/shell-snapshots', '.claude/statsig', '.claude/ide',
|
||||
'.claude/session-env', '.claude/file-history', '.claude/debug', '.claude/telemetry', '.claude/history.jsonl',
|
||||
'.claude/.credentials.json', '.codex/sessions', '.codex/archived_sessions', '.codex/log', '.codex/history.jsonl',
|
||||
'.codex/auth.json',
|
||||
] as const;
|
||||
|
||||
const MAX_DEPTH = 4;
|
||||
const MAX_REPORTED = 10;
|
||||
|
||||
export type FreeHomeSnapshot = Map<string, string>;
|
||||
|
||||
export function freeHomeDir(env: NodeJS.ProcessEnv = process.env): string {
|
||||
return env.HOME || env.USERPROFILE || os.homedir();
|
||||
}
|
||||
|
||||
export function snapshotFreeHome(home: string): FreeHomeSnapshot {
|
||||
const entries: FreeHomeSnapshot = new Map();
|
||||
const volatile = new Set<string>(FREE_HOME_VOLATILE);
|
||||
const visit = (relative: string, depth: number) => {
|
||||
if (volatile.has(relative)) return;
|
||||
const absolute = path.join(home, relative);
|
||||
let stat: fs.BigIntStats;
|
||||
try {
|
||||
stat = fs.lstatSync(absolute, { bigint: true });
|
||||
} catch (error) {
|
||||
if (['ENOENT', 'ENOTDIR'].includes((error as NodeJS.ErrnoException).code ?? '')) return;
|
||||
throw error;
|
||||
}
|
||||
if (stat.isSymbolicLink()) {
|
||||
entries.set(relative, `link:${fs.readlinkSync(absolute)}:${stat.mtimeNs}`);
|
||||
return;
|
||||
}
|
||||
if (!stat.isDirectory()) {
|
||||
entries.set(relative, `file:${stat.size}:${stat.mtimeNs}`);
|
||||
return;
|
||||
}
|
||||
entries.set(relative, 'dir');
|
||||
if (depth >= MAX_DEPTH) return;
|
||||
let names: string[];
|
||||
try {
|
||||
names = fs.readdirSync(absolute);
|
||||
} catch (error) {
|
||||
const code = (error as NodeJS.ErrnoException).code ?? '';
|
||||
if (['EACCES', 'EPERM'].includes(code)) entries.set(relative, 'dir:unreadable');
|
||||
if (['ENOENT', 'ENOTDIR', 'EACCES', 'EPERM'].includes(code)) return;
|
||||
throw error;
|
||||
}
|
||||
for (const name of names.sort()) visit(path.join(relative, name), depth + 1);
|
||||
};
|
||||
for (const surface of FREE_HOME_SURFACES) visit(path.normalize(surface), 0);
|
||||
return entries;
|
||||
}
|
||||
|
||||
export function diffFreeHome(before: FreeHomeSnapshot, after: FreeHomeSnapshot): string[] {
|
||||
const changed = new Set<string>();
|
||||
for (const [entry, value] of before) if (after.get(entry) !== value) changed.add(entry);
|
||||
for (const entry of after.keys()) if (!before.has(entry)) changed.add(entry);
|
||||
return [...changed].sort();
|
||||
}
|
||||
|
||||
export function formatFreeHomeChange(changed: string[], files: string[]): string {
|
||||
const shown = changed.slice(0, MAX_REPORTED).map(entry => `~/${entry.split(path.sep).join('/')}`);
|
||||
const more = changed.length > MAX_REPORTED ? ` (+${changed.length - MAX_REPORTED} more)` : '';
|
||||
return `real home changed while this shard ran (watches ${FREE_HOME_SURFACES.map(s => `~/${s}`).join(', ')} only): `
|
||||
+ `${shown.join(', ')}${more}. Give the writer a private HOME/GSTACK_HOME. Shard files: ${files.join(', ')}`;
|
||||
}
|
||||
|
||||
/** Take the baseline now; verify() returns null or the failure to report. */
|
||||
export function guardFreeHome(files: string[], env: NodeJS.ProcessEnv = process.env): { verify(): string | null } {
|
||||
const home = freeHomeDir(env);
|
||||
const unreadable = (error: unknown) => `real home surfaces could not be snapshotted (${(error as NodeJS.ErrnoException).code ?? 'error'}); shard files: ${files.join(', ')}`;
|
||||
let before: FreeHomeSnapshot | undefined;
|
||||
let baselineError: string | undefined;
|
||||
try { before = snapshotFreeHome(home); } catch (error) { baselineError = unreadable(error); }
|
||||
return {
|
||||
verify() {
|
||||
if (!before) return baselineError!;
|
||||
try {
|
||||
const changed = diffFreeHome(before, snapshotFreeHome(home));
|
||||
return changed.length ? formatFreeHomeChange(changed, files) : null;
|
||||
} catch (error) { return unreadable(error); }
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -86,6 +86,7 @@ import { StringDecoder } from 'node:string_decoder';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { isPaidTestFile } from '../test/helpers/paid-test-set';
|
||||
import { resolveStateRoot } from '../lib/state-root';
|
||||
import { guardFreeHome } from './lib/free-home-guard';
|
||||
import {
|
||||
BunTestOutputClassifier,
|
||||
createShardSandbox,
|
||||
@@ -107,7 +108,6 @@ import {
|
||||
type LanePolicy,
|
||||
type ShardChildResult,
|
||||
} from './lib/shard-engine';
|
||||
|
||||
export { normalizeRelativePath } from './lib/shard-engine';
|
||||
|
||||
/**
|
||||
@@ -354,6 +354,7 @@ export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }>
|
||||
// pattern hit is a false positive — the point of these files is Windows
|
||||
// coverage, so auto-excluding them defeats the regression tests they carry.
|
||||
const KNOWN_WINDOWS_SAFE: Array<{ file: string; reason: string }> = [
|
||||
{ file: 'test/ship-hook-windows-paths.test.ts', reason: 'runs bin/ helpers through explicit bash and Bun argv with forward-slash paths; never executes a shebang; the path-spelling simulation is skipIf win32' },
|
||||
{
|
||||
file: 'test/state-root-parity.test.ts',
|
||||
reason: 'runs the bash twin and lib/state-root.ts over an env table with PATH empty; no shebang execution, raw-string comparison is platform-neutral',
|
||||
@@ -384,8 +385,7 @@ const KNOWN_WINDOWS_SAFE: Array<{ file: string; reason: string }> = [
|
||||
},
|
||||
{
|
||||
file: 'test/claude-code-runner.test.ts',
|
||||
// The bin/ path is launched through process.execPath (Bun), never as a
|
||||
// shebang executable. Keep taskkill tree supervision in the Windows lane.
|
||||
// The bin/ path is launched through process.execPath (Bun), never as a shebang; keep taskkill supervision in the Windows lane.
|
||||
reason: 'invokes the runner via Bun argv; fake CLI and timeout descendant assertions cover native Windows taskkill',
|
||||
},
|
||||
{
|
||||
@@ -1810,7 +1810,7 @@ function explainFreeVerdict(label: string, status: FreeShardStatus, facts: {
|
||||
summary: ReturnType<BunTestOutputClassifier['end']>; expectedFiles: number; wallTimeoutMs: number;
|
||||
}): void {
|
||||
const { summary, exitCode } = facts;
|
||||
if (facts.cleanupError) console.error(`${label} browser cleanup failed: ${facts.cleanupError}; retained ${facts.stateDir}`);
|
||||
if (facts.cleanupError) console.error(`${label} shard cleanup or home containment failed: ${facts.cleanupError}; retained ${facts.stateDir}`);
|
||||
if (status === 'timed-out') {
|
||||
console.error(
|
||||
`${label} exceeded the ${Math.round(facts.wallTimeoutMs / 1000)}s wall-clock deadline — `
|
||||
@@ -1832,7 +1832,7 @@ function explainFreeVerdict(label: string, status: FreeShardStatus, facts: {
|
||||
function logFreeRecovery(log: (line: string) => void, outcome: FreeShardOutcome, facts: {
|
||||
cleanupError: string | null; logWriteFailed: boolean; captureIncomplete: boolean; rootDir: string;
|
||||
}): void {
|
||||
const problem = facts.cleanupError ? 'Owned-process cleanup is unconfirmed; inspect the retained state before another run.'
|
||||
const problem = facts.cleanupError ? 'Owned-process cleanup or home containment is unconfirmed; inspect the reported paths and retained state before another run.'
|
||||
: facts.logWriteFailed ? 'The evidence log could not be retained; repair the log destination before another run.'
|
||||
: facts.captureIncomplete ? 'Evidence capture is incomplete; repair the stream or early exit before another run.'
|
||||
: outcome.status === 'timed-out' ? 'Execution exceeded its deadline; inspect the last completed step before changing code or rerunning.'
|
||||
@@ -1952,8 +1952,8 @@ export async function runFreeShard(
|
||||
child = await runShardChild({
|
||||
command, args, cwd: rootDir, env, timeoutMs: wallTimeoutMs,
|
||||
attach: () => {
|
||||
const browser = trackShardBrowser(stateDir, env);
|
||||
return { signal: (force) => browser.signal(force), settle: async () => { cleanupError = await browser.settle(); } };
|
||||
const browser = trackShardBrowser(stateDir, env), home = guardFreeHome(files, env);
|
||||
return { signal: (force) => browser.signal(force), settle: async () => { cleanupError = await browser.settle() ?? home.verify(); } };
|
||||
},
|
||||
hookStreams: (spawned) => [consumeStream(spawned.stdout, 'stdout'), consumeStream(spawned.stderr, 'stderr')],
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user