mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-05 02:47:17 +02:00
v1.91.7.0 feat: add functional QA and pre-publication docs checks (#2983)
* feat: add surface-aware exploratory QA and ship documentation gates * test: preserve delegated QA setup authority after main integration * fix(qa): clarify exploration order and preserve report artifacts * test(qa): follow the shared setup reference directly * refactor(ship): make verification and recovery routes explicit * test(ship): align evidence and review guards with explicit routes * fix(workflows): clarify ship recovery and functional QA evidence * fix(workflows): clarify approval recovery and full QA coverage * refactor(workflows): order review transactions and clarify ship state * fix(ship): clarify final verification and fail closed at publication * fix(evals): attribute native atomic documentation writes * fix(ship): clarify recovery and documentation lifecycle guidance * fix(test): preserve observed native placeholder styling in CI * fix(codex): report watchdog timeouts without a process-exit race * Checkpoint functional QA implementation and workflow validation repairs * Fix documentation and shared-review fixture contracts * docs: clarify judge reuse and evaluation supervision * test: align review evidence and selected case contracts * test: verify append-only documentation checkpoints and recovery * fix: qualify QA workflows and CI validation repairs * fix: launch shared-libs fixture scripts on Windows * fix: qualify QA deadlines, fixture isolation, and shard cleanup * fix: preserve qualified QA and cancellation repairs * fix: enforce functional fixture authority and share strict event decoding * fix: retain free-test evidence and explain recovery * fix: reject malformed native evidence after decoder consolidation * test: use reliable capture for telemetry privacy filters * test: refresh measured quick coverage and document validation costs * Fix native fixture receipts and preserve VM validation evidence * Align negative judge controls with upstream clarity policy * Fix report-only QA preparation and public evidence handling * Clarify QA-only preparation and current-report preservation * Stream Ship quality judgments with an explicit 64k response contract * Validate compact judge reasoning locally with supported wire schema * Align functional QA fixture instructions with evidence acceptance * Bind native browser diagnostics to execution evidence and align review verdicts * Preserve native diagnostic line boundaries * Serialize functional QA evidence from native captures * Keep large QA evidence fixture payload out of Windows argv
This commit is contained in:
Garry Tan authored and GitHub committed
1 parent
65bfb0ce49
commit
dcaea52800
333 files changed
+41755
-7357
No files matched your search
+475
-49
@@ -51,7 +51,7 @@
|
||||
* on the windows-latest CI job.
|
||||
*
|
||||
* Output contract (v1.66): the full child stream ALWAYS lands in a per-run
|
||||
* log file under os.tmpdir() (path printed once at start and again in the
|
||||
* private log under .context/free-test-logs (path printed at start and in the
|
||||
* epilogue). The console is quiet by default — only the runner's own
|
||||
* [test:free] lines, `(fail)` result lines, bun error/crash markers
|
||||
* (`error:`, `panic:`, `crashed`, `Unhandled error`), and the terminal
|
||||
@@ -83,7 +83,7 @@ import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { spawn, spawnSync } from 'child_process';
|
||||
import { StringDecoder } from 'node:string_decoder';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { isPaidTestFile } from '../test/helpers/paid-test-set';
|
||||
import {
|
||||
BunTestOutputClassifier,
|
||||
@@ -167,6 +167,22 @@ const WINDOWS_FRAGILE_PATTERNS: Array<{ pattern: RegExp; reason: string }> = [
|
||||
// when possible; this list is for environment-/runtime-specific tests where
|
||||
// the failure mode is structural rather than detectable via source-file scan.
|
||||
export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }> = [
|
||||
{
|
||||
file: 'test/qa-evidence-producer.test.ts',
|
||||
reason: 'executes the registered Linux native actor and its inotify observer; portable capture and Windows job behavior are covered by qa-evidence.test.ts',
|
||||
},
|
||||
{
|
||||
file: 'test/qa-functional-fixture.test.ts',
|
||||
reason: 'executes graceful POSIX signal cancellation; Bun on Windows uses TerminateProcess and cannot run the fixture SIGTERM cleanup handler',
|
||||
},
|
||||
{
|
||||
file: 'test/qa-functional-observer-atomic.test.ts',
|
||||
reason: 'exercises real Linux inotify inode and directory watches through libc.so.6; Windows has no equivalent kernel interface',
|
||||
},
|
||||
{
|
||||
file: 'test/docsync-report-interface.test.ts',
|
||||
reason: 'executes registered native documentation callbacks with their real Linux inotify write observer before the model boundary',
|
||||
},
|
||||
{
|
||||
file: 'test/setup-gbrain-fixture.test.ts',
|
||||
reason: 'the fixture invokes real POSIX detector/verifier helpers through executable shebang wrappers',
|
||||
@@ -313,6 +329,26 @@ export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }>
|
||||
// pattern hit is a false positive — the point of these files is Windows
|
||||
// coverage, so auto-excluding them defeats the regression tests they carry.
|
||||
const KNOWN_WINDOWS_SAFE: Array<{ file: string; reason: string }> = [
|
||||
{
|
||||
file: 'test/qa-evidence.test.ts',
|
||||
reason: 'invokes the production helper through Bun argv and exercises native Windows job cleanup, private file captures and backpressured receipt output',
|
||||
},
|
||||
{
|
||||
file: 'test/qa-evidence-selection.test.ts',
|
||||
reason: 'bin/ strings are literal dependency and Windows-selection assertions; no native actor or shebang command is launched',
|
||||
},
|
||||
{
|
||||
file: 'test/qa-deadline.test.ts',
|
||||
reason: 'launches the guard through Bun argv; mode assertions and POSIX signal cases are platform-gated, while Windows job cleanup must execute natively',
|
||||
},
|
||||
{
|
||||
file: 'test/qa-deadline-selection.test.ts',
|
||||
reason: 'bin/ strings are dependency-selection inputs; this suite never launches a shebang executable',
|
||||
},
|
||||
{
|
||||
file: 'test/shared-libs-source-reads.test.ts',
|
||||
reason: 'bin/ literal is a mocked launch assertion; actual worktree fingerprinting explicitly invokes Bash on Windows',
|
||||
},
|
||||
{
|
||||
file: 'test/claude-code-windows-job.test.ts',
|
||||
reason: 'invokes Bun directly; verifies Windows job containment at the standalone CLI boundary',
|
||||
@@ -484,24 +520,16 @@ export const WORKER_HOSTILE: Record<string, string> = {
|
||||
};
|
||||
|
||||
/**
|
||||
* TREE-SERIAL files: run in ONE serial shard AFTER the parallel shards.
|
||||
* EMPTY since the 2026-08 dissolution — kept as a mechanism, not a museum:
|
||||
* a test that must regenerate shared repo artifacts IN PLACE (and cannot
|
||||
* render into an out-dir instead) earns an entry here with a reason, and
|
||||
* the runner will serialize it again.
|
||||
*
|
||||
* How it emptied: gen-skill-docs gained a main() guard (imports stopped
|
||||
* regenerating 71 files at load) and --out-dir grew to every host, so all
|
||||
* eight mutators now render into mkdtemps — the live tree is never written
|
||||
* by the suite (pinned by gen-skill-docs-import-purity + each migrated
|
||||
* file's own porcelain/mtime assertions). With zero mutators, the four
|
||||
* ratchet READERS (parity caps, size budgets, carve parity/ordering) get a
|
||||
* quiet tree by construction in any shard, so they rejoined the parallel
|
||||
* phase — the ~35-40s serial tail on every full-suite run is gone.
|
||||
* Exclusive host-state fixtures: run in ONE serial shard AFTER the parallel
|
||||
* shards. The public name is retained for callers of the original tree-write
|
||||
* classification. Entries need a concrete shared-state hazard that fixture
|
||||
* directories cannot isolate, such as host-wide procfs visibility.
|
||||
* Keys are pinned against the live file census by test-free-shards.test.ts —
|
||||
* a renamed file fails the suite instead of silently dropping serialization.
|
||||
*/
|
||||
export const TREE_MUTATING: Record<string, string> = {};
|
||||
export const TREE_MUTATING: Record<string, string> = {
|
||||
'test/bootstrap-retention.test.ts': 'Creates nondumpable same-UID processes visible to every host procfs census; must not overlap other native-retention fixtures.',
|
||||
};
|
||||
|
||||
export function isFreeTestFile(relativePath: string): boolean {
|
||||
const normalized = normalizeRelativePath(relativePath);
|
||||
@@ -733,10 +761,10 @@ const planDigest = (plan: Omit<FreeCiPlan, 'id'>): string =>
|
||||
/** One immutable plan is shared by isolated CI machines; never repack per job. */
|
||||
export function createFreeCiPlan(files: string[], count: number, durations: Record<string, number>, revision: string): FreeCiPlan {
|
||||
const readers = files.filter(file => !(file in TREE_MUTATING));
|
||||
const mutators = files.filter(file => file in TREE_MUTATING).sort();
|
||||
const exclusive = files.filter(file => file in TREE_MUTATING).sort();
|
||||
const packed = packShardsByDuration(readers, count, durations);
|
||||
const shards = packed.shards.map((files, index) => ({ shard: index + 1, files, predictedMs: packed.predictedMs[index] }));
|
||||
if (mutators.length) shards.push({ shard: shards.length + 1, files: mutators, predictedMs: mutators.reduce((ms, file) => ms + (durations[file] ?? 0), 0) });
|
||||
if (exclusive.length) shards.push({ shard: shards.length + 1, files: exclusive, predictedMs: exclusive.reduce((ms, file) => ms + (durations[file] ?? 0), 0) });
|
||||
const body = { version: 1 as const, revision, shards };
|
||||
return { ...body, id: planDigest(body) };
|
||||
}
|
||||
@@ -811,6 +839,8 @@ export const QUICK_CORE = [
|
||||
'test/strict-output.test.ts', 'test/gen-skill-docs.test.ts',
|
||||
'test/skill-check-driver.test.ts', 'test/ceo-native-ledger-replay.test.ts',
|
||||
'test/skill-ceo-section-ordering.test.ts',
|
||||
'test/qa-functional-observer.test.ts', 'test/qa-checkpoint-evidence.test.ts',
|
||||
'test/test-free-shards-capture.test.ts',
|
||||
];
|
||||
|
||||
export function selectQuickFreeFiles(files: string[], durations: Record<string, number>): string[] {
|
||||
@@ -1363,7 +1393,7 @@ export interface RunFreeShardOptions {
|
||||
* Runner-owned [test:free] lines go through `log`, not this sink.
|
||||
*/
|
||||
consoleWrite?: (text: string) => void;
|
||||
/** Per-run full-stream log path (tests inject). Default: a timestamped file under os.tmpdir(). */
|
||||
/** Per-run full-stream log path (tests inject). Default: a private retained file under .context/free-test-logs. */
|
||||
logFilePath?: string;
|
||||
log?: (line: string) => void;
|
||||
}
|
||||
@@ -1374,6 +1404,370 @@ const EPILOGUE_WORD: Record<FreeShardStatus, string> = {
|
||||
'timed-out': 'timed-out',
|
||||
};
|
||||
|
||||
function trackShardBrowser(stateDir: string, env: NodeJS.ProcessEnv) {
|
||||
class BrowserCleanupError extends Error {}
|
||||
class CaptureStopped extends Error {}
|
||||
type Identity = { pid: number; parent: number; start: string; daemon: number; root: boolean };
|
||||
type Capture = { abort: AbortController; deadline: number; probes: Set<Promise<unknown>>; records?: [string, string] };
|
||||
const identities = new Map<number, Identity>();
|
||||
const nativeStarts = new Map<string, string>();
|
||||
const interrupted = new Set<string>();
|
||||
const errors = new Set<string>();
|
||||
const stateFile = env.BROWSE_STATE_FILE!;
|
||||
let stopping = false;
|
||||
let ready = true;
|
||||
let closed = false;
|
||||
let forced = false;
|
||||
let cancellation = false;
|
||||
let deadline = Infinity;
|
||||
let forceAt = Infinity;
|
||||
let active: Capture | null = null;
|
||||
let pending: Promise<void> | null = null;
|
||||
let observed = false;
|
||||
let alive = true;
|
||||
|
||||
const check = (capture: Capture) => {
|
||||
if (closed || capture.abort.signal.aborted || active !== capture) throw new CaptureStopped();
|
||||
if (performance.now() >= Math.min(capture.deadline, deadline)) throw new BrowserCleanupError('browser ownership deadline exceeded');
|
||||
};
|
||||
const probe = async (capture: Capture, command: string, args: string[], timeout: number) => {
|
||||
check(capture);
|
||||
const remaining = Math.min(timeout, capture.deadline - performance.now() - 100, deadline - performance.now() - 100);
|
||||
if (remaining <= 0) throw new BrowserCleanupError('browser ownership deadline exceeded');
|
||||
const task = new Promise<{ status: number | null; stdout: string }>((resolve, reject) => {
|
||||
const child = spawn(command, args, { detached: true, stdio: ['ignore', 'pipe', 'ignore'], windowsHide: true });
|
||||
let output = '';
|
||||
let failed = false;
|
||||
let done = false;
|
||||
let reaper: ReturnType<typeof setTimeout> | undefined;
|
||||
const finish = (status: number | null) => {
|
||||
if (done) return;
|
||||
done = true;
|
||||
clearTimeout(timer);
|
||||
clearTimeout(reaper);
|
||||
capture.abort.signal.removeEventListener('abort', stop);
|
||||
child.stdout?.destroy();
|
||||
child.unref();
|
||||
if (failed) reject(new BrowserCleanupError('browser identity probe did not complete'));
|
||||
else resolve({ status, stdout: output });
|
||||
};
|
||||
const stop = () => {
|
||||
if (done || failed) return;
|
||||
failed = true;
|
||||
killProcessGroup(child, 'SIGKILL');
|
||||
reaper = setTimeout(() => finish(null), 100);
|
||||
};
|
||||
const timer = setTimeout(stop, Math.max(1, remaining));
|
||||
capture.abort.signal.addEventListener('abort', stop, { once: true });
|
||||
child.once('error', () => { failed = true; finish(null); });
|
||||
child.once('close', finish);
|
||||
child.stdout?.on('data', chunk => {
|
||||
output += chunk.toString();
|
||||
if (output.length > 65536) stop();
|
||||
});
|
||||
});
|
||||
capture.probes.add(task);
|
||||
try {
|
||||
const result = await task;
|
||||
check(capture);
|
||||
return result;
|
||||
} finally { capture.probes.delete(task); }
|
||||
};
|
||||
|
||||
const failure = (error: unknown) => {
|
||||
if (error instanceof CaptureStopped) return;
|
||||
const code = (error as NodeJS.ErrnoException)?.code;
|
||||
const file = (error as NodeJS.ErrnoException & { path?: string })?.path;
|
||||
const pid = typeof file === 'string' ? /^\/proc\/(\d+)\//.exec(file)?.[1] : undefined;
|
||||
if (pid && identities.has(Number(pid)) && ['EACCES', 'EPERM', 'ENOENT', 'ESRCH'].includes(code ?? '')) return;
|
||||
errors.add(error instanceof BrowserCleanupError ? error.message : 'browser ownership unavailable');
|
||||
};
|
||||
|
||||
const inspectLinux = (pid: number): Omit<Identity, 'daemon' | 'root'> | null => {
|
||||
try {
|
||||
const raw = fs.readFileSync(`/proc/${pid}/stat`, 'utf8');
|
||||
const fields = raw.slice(raw.lastIndexOf(') ') + 2).trim().split(/\s+/);
|
||||
if (!/^\d+$/.test(fields[19] ?? '')) throw new BrowserCleanupError('process start identity unavailable');
|
||||
return fields[0] === 'Z' || fields[0] === 'X' ? null
|
||||
: { pid, parent: Number(fields[1]), start: fields[19] };
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT' || (error as NodeJS.ErrnoException).code === 'ESRCH') return null;
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
const inspect = async (capture: Capture, pid: number): Promise<Omit<Identity, 'daemon' | 'root'> | null> => {
|
||||
check(capture);
|
||||
if (!Number.isSafeInteger(pid) || pid <= 1) throw new BrowserCleanupError('invalid process identity');
|
||||
if (process.platform === 'linux') return inspectLinux(pid);
|
||||
const result = await probe(capture, 'ps', ['-p', String(pid), '-o', 'ppid=,stat=,lstart='], 500);
|
||||
if (result.status === 1 && !result.stdout.trim()) return null;
|
||||
if (result.status !== 0) throw new BrowserCleanupError('process identity unavailable');
|
||||
const fields = result.stdout.trim().split(/\s+/);
|
||||
return fields[1]?.startsWith('Z') ? null : { pid, parent: Number(fields[0]), start: fields.slice(2).join(' ') };
|
||||
};
|
||||
const required = [`BROWSE_STATE_FILE=${stateFile}`, `GSTACK_FREE_SHARD_ID=${env.GSTACK_FREE_SHARD_ID}`];
|
||||
const boundLinux = (pid: number) => {
|
||||
try {
|
||||
const values = fs.readFileSync(`/proc/${pid}/environ`, 'utf8').split('\0');
|
||||
return required.every(value => values.includes(value));
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT' || (error as NodeJS.ErrnoException).code === 'ESRCH') return false;
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
const bound = async (capture: Capture, pid: number): Promise<boolean> => {
|
||||
check(capture);
|
||||
if (process.platform === 'linux') return boundLinux(pid);
|
||||
const [command, environment] = await Promise.all([
|
||||
probe(capture, 'ps', ['-ww', '-p', String(pid), '-o', 'command='], 500),
|
||||
probe(capture, 'ps', ['eww', '-p', String(pid), '-o', 'command='], 500),
|
||||
]);
|
||||
if (command.status !== 0 || environment.status !== 0) return false;
|
||||
const prefix = command.stdout.trim();
|
||||
if (!prefix || !environment.stdout.trim().startsWith(prefix + ' ')) return false;
|
||||
const values = ' ' + environment.stdout.trim().slice(prefix.length).trim() + ' ';
|
||||
return required.every(value => values.includes(' ' + value + ' '));
|
||||
};
|
||||
const live = async (capture: Capture, identity: Identity): Promise<boolean> => {
|
||||
const current = await inspect(capture, identity.pid);
|
||||
check(capture);
|
||||
if (!current) return false;
|
||||
if (!current.start || current.start !== identity.start) {
|
||||
errors.add('captured process identity was replaced');
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
const record = (file: string): any => {
|
||||
if (!fs.existsSync(file)) return null;
|
||||
const info = fs.lstatSync(file);
|
||||
if (!info.isFile() || info.size > 65536) throw new BrowserCleanupError('unsafe browser state record');
|
||||
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
};
|
||||
const nativeStart = async (capture: Capture, identity: Identity): Promise<string> => {
|
||||
check(capture);
|
||||
const key = `${identity.pid}:${identity.start}`;
|
||||
const recorded = nativeStarts.get(key);
|
||||
if (recorded) return recorded;
|
||||
if (!await live(capture, identity)) throw new BrowserCleanupError('process exited before its native identity was captured');
|
||||
const result = await probe(capture, 'ps', ['-p', String(identity.pid), '-o', 'lstart='], 2000);
|
||||
const value = result.status === 0 ? result.stdout.trim().replace(/\s+/g, ' ') : '';
|
||||
if (!value || !await live(capture, identity)) throw new BrowserCleanupError('native process identity unavailable');
|
||||
check(capture);
|
||||
nativeStarts.set(key, value);
|
||||
return value;
|
||||
};
|
||||
const remember = (capture: Capture, identity: Identity) => {
|
||||
check(capture);
|
||||
const previous = identities.get(identity.pid);
|
||||
if (previous && previous.start !== identity.start) throw new BrowserCleanupError('captured process identity was replaced');
|
||||
identities.set(identity.pid, identity);
|
||||
};
|
||||
const descendants = async (capture: Capture, parent: Identity, visited: Set<number>): Promise<void> => {
|
||||
check(capture);
|
||||
if (visited.has(parent.pid)) return;
|
||||
visited.add(parent.pid);
|
||||
if (visited.size > 256) throw new BrowserCleanupError('owned browser process limit exceeded');
|
||||
if (!await live(capture, parent)) return;
|
||||
let children: number[];
|
||||
if (process.platform === 'linux') {
|
||||
try {
|
||||
children = fs.readFileSync(`/proc/${parent.pid}/task/${parent.pid}/children`, 'utf8').trim().split(/\s+/).filter(Boolean).map(Number);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return;
|
||||
throw error;
|
||||
}
|
||||
} else {
|
||||
const result = await probe(capture, 'pgrep', ['-P', String(parent.pid)], 500);
|
||||
if (result.status !== 0 && result.status !== 1) throw new BrowserCleanupError('owned child identities unavailable');
|
||||
children = result.stdout.trim().split(/\s+/).filter(Boolean).map(Number);
|
||||
}
|
||||
for (const pid of children) {
|
||||
const child = await inspect(capture, pid);
|
||||
if (!child || child.parent !== parent.pid || !await live(capture, parent)) continue;
|
||||
const identity = { ...child, daemon: parent.daemon, root: false };
|
||||
remember(capture, identity);
|
||||
await descendants(capture, identity, visited);
|
||||
}
|
||||
};
|
||||
const capture = async (operation: Capture) => {
|
||||
check(operation);
|
||||
ready = true;
|
||||
if (process.platform === 'win32') return;
|
||||
try {
|
||||
if (fs.realpathSync(stateDir) !== stateDir) throw new BrowserCleanupError('shard directory was replaced');
|
||||
const directory = path.dirname(stateFile);
|
||||
if (fs.existsSync(directory) && (!fs.lstatSync(directory).isDirectory()
|
||||
|| fs.lstatSync(directory).isSymbolicLink())) throw new BrowserCleanupError('browser directory was replaced');
|
||||
const state = record(stateFile);
|
||||
if (state?.pid !== undefined) {
|
||||
const current = await inspect(operation, state.pid);
|
||||
if (current) {
|
||||
if (!await bound(operation, current.pid)) {
|
||||
if (!await inspect(operation, current.pid)) return;
|
||||
throw new BrowserCleanupError('daemon is not bound to this shard');
|
||||
}
|
||||
remember(operation, { ...current, daemon: current.pid, root: true });
|
||||
} else if (!identities.has(state.pid)) {
|
||||
throw new BrowserCleanupError('daemon exited before ownership was captured');
|
||||
}
|
||||
}
|
||||
for (const identity of identities.values()) if (identity.root) await descendants(operation, identity, new Set());
|
||||
const validateChild = async (pid: unknown, start: unknown, daemon: unknown) => {
|
||||
check(operation);
|
||||
if (!Number.isSafeInteger(pid) || (pid as number) <= 1) throw new BrowserCleanupError('invalid browser child identity');
|
||||
const identity = identities.get(pid as number);
|
||||
if (!identity || identity.daemon !== daemon || identity.root) throw new BrowserCleanupError('browser child ownership is unconfirmed');
|
||||
if (await live(operation, identity) && (typeof start !== 'string' || !start
|
||||
|| await nativeStart(operation, identity) !== start.replace(/\s+/g, ' '))) throw new BrowserCleanupError('browser child identity was replaced');
|
||||
};
|
||||
const agent = record(path.join(directory, 'terminal-agent-pid'));
|
||||
const validations: Promise<unknown>[] = [];
|
||||
if (agent) {
|
||||
const daemon = identities.get(agent.ownerPid);
|
||||
if (!daemon?.root || (state?.pid !== undefined && agent.ownerPid !== state.pid)) throw new BrowserCleanupError('terminal owner is unconfirmed');
|
||||
validations.push(nativeStart(operation, daemon).then(start => {
|
||||
if (start !== agent.ownerStartTime?.replace(/\s+/g, ' ')) throw new BrowserCleanupError('terminal owner is unconfirmed');
|
||||
}));
|
||||
if (agent.pid === 0) ready = false;
|
||||
else validations.push(validateChild(agent.pid, agent.startTime, agent.ownerPid));
|
||||
}
|
||||
if (state?.chromiumPid !== undefined) validations.push(validateChild(state.chromiumPid, state.chromiumStartTime, state.pid));
|
||||
const results = await Promise.allSettled(validations);
|
||||
check(operation);
|
||||
for (const result of results) if (result.status === 'rejected') throw result.reason;
|
||||
operation.records = [JSON.stringify(state), JSON.stringify(agent)];
|
||||
} catch (error) {
|
||||
check(operation);
|
||||
ready = false;
|
||||
failure(error);
|
||||
}
|
||||
};
|
||||
const signalOwned = async (operation: Capture, force: boolean) => {
|
||||
for (const identity of identities.values()) {
|
||||
if (!force && (!identity.root || !ready || errors.size > 0)) continue;
|
||||
const key = `${identity.pid}:${identity.start}`;
|
||||
if (!force && interrupted.has(key)) continue;
|
||||
try {
|
||||
if (!await live(operation, identity)) continue;
|
||||
if (identity.root && !await bound(operation, identity.pid)) {
|
||||
if (!await live(operation, identity)) continue;
|
||||
throw new BrowserCleanupError('daemon environment changed before termination');
|
||||
}
|
||||
if (!await live(operation, identity)) continue;
|
||||
check(operation);
|
||||
if (!force && (!operation.records || JSON.stringify(record(stateFile)) !== operation.records[0]
|
||||
|| JSON.stringify(record(path.join(path.dirname(stateFile), 'terminal-agent-pid'))) !== operation.records[1])) {
|
||||
ready = false;
|
||||
continue;
|
||||
}
|
||||
process.kill(identity.pid, force ? 'SIGKILL' : 'SIGINT');
|
||||
if (!force) interrupted.add(key);
|
||||
} catch (error) {
|
||||
check(operation);
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ESRCH') failure(error);
|
||||
}
|
||||
}
|
||||
};
|
||||
const forceLinux = () => {
|
||||
if (process.platform !== 'linux') return;
|
||||
for (const identity of identities.values()) {
|
||||
try {
|
||||
const current = inspectLinux(identity.pid);
|
||||
if (!current) continue;
|
||||
if (current.start !== identity.start) throw new BrowserCleanupError('captured process identity was replaced');
|
||||
if (identity.root && !boundLinux(identity.pid)) {
|
||||
if (!inspectLinux(identity.pid)) continue;
|
||||
throw new BrowserCleanupError('daemon environment changed before termination');
|
||||
}
|
||||
if (inspectLinux(identity.pid)?.start === identity.start) process.kill(identity.pid, 'SIGKILL');
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ESRCH') failure(error);
|
||||
}
|
||||
}
|
||||
};
|
||||
const enqueue = () => {
|
||||
if (closed || pending || process.platform === 'win32') return;
|
||||
const operation: Capture = { abort: new AbortController(), deadline: Math.min(performance.now() + 10000, deadline, forced ? Infinity : forceAt), probes: new Set() };
|
||||
active = operation;
|
||||
pending = (async () => {
|
||||
try {
|
||||
if (!forced) await capture(operation);
|
||||
if (stopping) {
|
||||
await signalOwned(operation, forced);
|
||||
let stillAlive = false;
|
||||
for (const identity of identities.values()) if (await live(operation, identity)) stillAlive = true;
|
||||
check(operation);
|
||||
alive = stillAlive;
|
||||
observed = true;
|
||||
}
|
||||
} catch (error) {
|
||||
if (!closed && !operation.abort.signal.aborted) failure(error);
|
||||
} finally {
|
||||
operation.abort.abort();
|
||||
await Promise.allSettled([...operation.probes]);
|
||||
if (active === operation) active = null;
|
||||
}
|
||||
})().finally(() => { pending = null; });
|
||||
};
|
||||
const signal = (force: boolean) => {
|
||||
if (closed) return;
|
||||
if (!cancellation) {
|
||||
cancellation = true;
|
||||
stopping = true;
|
||||
deadline = Math.min(deadline, performance.now() + 5500);
|
||||
forceAt = Math.min(forceAt, performance.now() + 5000);
|
||||
active?.abort.abort();
|
||||
}
|
||||
if (force) {
|
||||
forced = true;
|
||||
active?.abort.abort();
|
||||
forceLinux();
|
||||
}
|
||||
if (pending) void pending.then(enqueue);
|
||||
else enqueue();
|
||||
};
|
||||
const timer = setInterval(enqueue, process.platform === 'darwin' ? 1000 : 250);
|
||||
timer.unref();
|
||||
return {
|
||||
signal,
|
||||
async settle(): Promise<string | null> {
|
||||
clearInterval(timer);
|
||||
if (process.platform === 'win32') { closed = true; return null; }
|
||||
stopping = true;
|
||||
deadline = Math.min(deadline, performance.now() + 10000);
|
||||
forceAt = Math.min(forceAt, performance.now() + 5000);
|
||||
active?.abort.abort();
|
||||
try {
|
||||
while (true) {
|
||||
if (performance.now() >= deadline) {
|
||||
errors.add('owned browser settlement deadline exceeded');
|
||||
break;
|
||||
}
|
||||
if (!forced && performance.now() >= forceAt) {
|
||||
forced = true;
|
||||
active?.abort.abort();
|
||||
forceLinux();
|
||||
}
|
||||
if (pending) await pending;
|
||||
enqueue();
|
||||
if (pending) await pending;
|
||||
if (observed && !alive) break;
|
||||
await new Promise(resolve => setTimeout(resolve, Math.min(50, Math.max(0, deadline - performance.now()))));
|
||||
}
|
||||
} catch {
|
||||
errors.add('owned browser settlement could not be verified');
|
||||
} finally {
|
||||
closed = true;
|
||||
clearInterval(timer);
|
||||
active?.abort.abort();
|
||||
if (pending) await pending;
|
||||
}
|
||||
return errors.size ? [...errors].join('; ') : null;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** One line per shard, printed after the run: `[test:free] shard i/N: M files, XXs, pass|fail|timed-out`. */
|
||||
function shardEpilogue(outcome: FreeShardOutcome, totalShards: number): string {
|
||||
return `[test:free] shard ${outcome.shard}/${totalShards}: ${outcome.files.length} files, `
|
||||
@@ -1429,8 +1823,8 @@ export async function runFreeShard(
|
||||
// Full-stream capture: EVERY child byte lands here, whatever the console
|
||||
// shows. Printed once at start so a wedged or noisy run is inspectable
|
||||
// without a re-run.
|
||||
const logPath = options.logFilePath ?? nextDefaultLogPath();
|
||||
const logStream = fs.createWriteStream(logPath);
|
||||
const logPath = options.logFilePath ?? nextDefaultLogPath(rootDir);
|
||||
const logStream = fs.createWriteStream(logPath, { mode: 0o600 });
|
||||
let logWriteFailed = false;
|
||||
logStream.on('error', (err) => {
|
||||
if (logWriteFailed) return;
|
||||
@@ -1444,7 +1838,7 @@ export async function runFreeShard(
|
||||
: { command: process.execPath, args: buildShardArgs(files, { parallel: options.parallel, rootDir }) };
|
||||
|
||||
const env = { ...(options.env ?? process.env) };
|
||||
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-free-shard-'));
|
||||
const stateDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-free-shard-')));
|
||||
const childTmp = path.join(stateDir, 'tmp');
|
||||
fs.mkdirSync(childTmp);
|
||||
env.TMPDIR = childTmp;
|
||||
@@ -1455,6 +1849,7 @@ export async function runFreeShard(
|
||||
// daemons from prior runs can then replace or remove each other's state.
|
||||
// Override inherited state too; the shard owns this directory's cleanup.
|
||||
env.BROWSE_STATE_FILE = path.join(stateDir, '.gstack', 'browse.json');
|
||||
env.GSTACK_FREE_SHARD_ID = randomUUID();
|
||||
// Per-shard Chromium profile (same isolation idea as TMPDIR): nine test
|
||||
// files launch in-process persistent contexts or daemons that default to
|
||||
// the SHARED ~/.gstack/chromium-profile, and two concurrent shards on one
|
||||
@@ -1476,10 +1871,12 @@ export async function runFreeShard(
|
||||
windowsHide: true,
|
||||
});
|
||||
const groupPid = child.pid ?? null;
|
||||
const browser = trackShardBrowser(stateDir, env);
|
||||
// Group-kill on parent SIGINT/SIGTERM too, not just on timeout.
|
||||
const forwarding = installChildSignalForwarding({
|
||||
kill: (signal?: NodeJS.Signals | number) => {
|
||||
killProcessGroup(child, (signal as NodeJS.Signals) ?? 'SIGTERM');
|
||||
browser.signal(signal === 'SIGKILL');
|
||||
return true;
|
||||
},
|
||||
});
|
||||
@@ -1541,6 +1938,7 @@ export async function runFreeShard(
|
||||
}, wallTimeoutMs);
|
||||
|
||||
let exitCode: number | null = null;
|
||||
let cleanupError: string | null = null;
|
||||
try {
|
||||
const streams = [consumeStream(child.stdout, 'stdout'), consumeStream(child.stderr, 'stderr')];
|
||||
exitCode = await new Promise<number | null>((resolve, reject) => {
|
||||
@@ -1550,9 +1948,9 @@ export async function runFreeShard(
|
||||
await Promise.all(streams);
|
||||
} finally {
|
||||
clearTimeout(killTimer);
|
||||
forwarding.dispose();
|
||||
// Reap survivors of this shard even on the clean path.
|
||||
killProcessGroup(child, 'SIGKILL');
|
||||
cleanupError = await browser.settle();
|
||||
reporter.end();
|
||||
for (const [origin, error] of captureFailures) {
|
||||
const diagnostic = `${label} ${origin} capture incomplete: ${error.message} `
|
||||
@@ -1562,24 +1960,28 @@ export async function runFreeShard(
|
||||
}
|
||||
await new Promise<void>((resolve) => logStream.end(() => resolve()));
|
||||
try {
|
||||
fs.rmSync(stateDir, { recursive: true, force: true });
|
||||
if (!cleanupError) fs.rmSync(stateDir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Best-effort cleanup of a throwaway temp dir — a locked file on
|
||||
// Windows must not turn a real verdict into an exception.
|
||||
if (process.platform !== 'win32') cleanupError = 'could not remove the owned shard directory';
|
||||
}
|
||||
forwarding.dispose();
|
||||
}
|
||||
|
||||
const summary = classifier.end();
|
||||
const status: FreeShardStatus = timedOut
|
||||
? 'timed-out'
|
||||
: captureFailures.size === 0 && strictTestExitCode(exitCode ?? 1, summary, files.length) === 0 ? 'passed' : 'failed';
|
||||
: !cleanupError && !logWriteFailed && captureFailures.size === 0 && strictTestExitCode(exitCode ?? 1, summary, files.length) === 0 ? 'passed' : 'failed';
|
||||
|
||||
if (cleanupError) console.error(`${label} browser cleanup failed: ${cleanupError}; retained ${stateDir}`);
|
||||
|
||||
if (status === 'timed-out') {
|
||||
console.error(
|
||||
`${label} exceeded the ${Math.round(wallTimeoutMs / 1000)}s wall-clock deadline — `
|
||||
+ 'killed the process group. Reporting as TIMED-OUT (distinct from failed).',
|
||||
);
|
||||
} else if (status === 'failed' && captureFailures.size === 0 && (exitCode ?? 1) === 0) {
|
||||
} else if (status === 'failed' && !cleanupError && !logWriteFailed && captureFailures.size === 0 && (exitCode ?? 1) === 0) {
|
||||
const reason = summary.failedTests > 0 || summary.unhandledBetweenTests > 0
|
||||
? `reported ${summary.failedTests} failing test(s) and ${summary.unhandledBetweenTests} unhandled error(s) between tests`
|
||||
: summary.terminalFileCounts.length === 0
|
||||
@@ -1600,6 +2002,8 @@ export async function runFreeShard(
|
||||
+ report.unreportedFailures
|
||||
+ report.unhandledErrors.length
|
||||
+ captureFailures.size
|
||||
+ (logWriteFailed ? 1 : 0)
|
||||
+ (cleanupError ? 1 : 0)
|
||||
+ (report.sawTerminalSummary ? 0 : 1);
|
||||
const outcome: FreeShardOutcome = {
|
||||
shard: shardNumber, files, status, exitCode, elapsedMs: Date.now() - startedAt, groupPid, failingFiles, unattributedFailures,
|
||||
@@ -1607,16 +2011,38 @@ export async function runFreeShard(
|
||||
};
|
||||
log(shardEpilogue(outcome, totalShards));
|
||||
for (const line of buildRunEpilogue(status, report, outcome.elapsedMs, logPath)) log(line);
|
||||
if (status !== 'passed') {
|
||||
const problem = cleanupError ? 'Owned-process cleanup is unconfirmed; inspect the retained state before another run.'
|
||||
: logWriteFailed ? 'The evidence log could not be retained; repair the log destination before another run.'
|
||||
: captureFailures.size || !report.sawTerminalSummary ? 'Evidence capture is incomplete; repair the stream or early exit before another run.'
|
||||
: status === 'timed-out' ? 'Execution exceeded its deadline; inspect the last completed step before changing code or rerunning.'
|
||||
: 'A test or module failed; the root cause is not established. Inspect the full log and repair the cause first.';
|
||||
log(`[test:free] Recovery: ${problem} See docs/TESTING_INTERNALS.md.`);
|
||||
const focused = failingFiles.filter(file => files.includes(file) && fs.existsSync(path.resolve(rootDir, file)));
|
||||
if (!unattributedFailures && focused.length) {
|
||||
log(`[test:free] After repair, focused check: bun test ${focused.map(file => `'${file.replaceAll("'", "'\\''")}'`).join(' ')}`);
|
||||
} else {
|
||||
log('[test:free] No complete narrower failure scope is available; do not treat a subset rerun as complete coverage.');
|
||||
}
|
||||
}
|
||||
return outcome;
|
||||
}
|
||||
|
||||
let logPathSequence = 0;
|
||||
|
||||
/** Timestamped per-run log file under os.tmpdir(); pid+sequence defeat same-ms collisions. */
|
||||
function nextDefaultLogPath(): string {
|
||||
/** Timestamped retained log file; pid+sequence defeat same-ms collisions. */
|
||||
function nextDefaultLogPath(rootDir: string): string {
|
||||
let directory = fs.realpathSync(rootDir);
|
||||
for (const part of ['.context', 'free-test-logs']) {
|
||||
directory = path.join(directory, part);
|
||||
const existing = fs.lstatSync(directory, { throwIfNoEntry: false });
|
||||
if (existing && (!existing.isDirectory() || existing.isSymbolicLink())) throw new Error('Free-test log directory must not traverse links');
|
||||
if (!existing) fs.mkdirSync(directory, { mode: 0o700 });
|
||||
}
|
||||
fs.chmodSync(directory, 0o700);
|
||||
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
logPathSequence += 1;
|
||||
return path.join(os.tmpdir(), `gstack-free-test-${stamp}-${process.pid}-${logPathSequence}.log`);
|
||||
return path.join(directory, `gstack-free-test-${stamp}-${process.pid}-${logPathSequence}.log`);
|
||||
}
|
||||
|
||||
function exitCodeFor(status: FreeShardStatus): number {
|
||||
@@ -1651,11 +2077,11 @@ async function recordFreeTestDurations(files: string[], jobs: number): Promise<n
|
||||
if (outcome.status !== 'passed') failed.push(file);
|
||||
}
|
||||
};
|
||||
// As in full-suite mode, finish readers before any checkout-mutating tests.
|
||||
const mutators = files.filter(file => file in TREE_MUTATING);
|
||||
// As in full-suite mode, finish parallel work before exclusive host-state fixtures.
|
||||
const exclusive = files.filter(file => file in TREE_MUTATING);
|
||||
files = files.filter(file => !(file in TREE_MUTATING));
|
||||
await Promise.all(Array.from({ length: Math.max(1, jobs) }, () => worker()));
|
||||
files = mutators;
|
||||
files = exclusive;
|
||||
cursor = 0;
|
||||
await worker();
|
||||
if (Object.keys(durations).length !== expectedCount) {
|
||||
@@ -1841,7 +2267,7 @@ async function main(): Promise<number> {
|
||||
console.log(
|
||||
`\nWould run ${files.length} files across ${shards.length} shards (${occupied} occupied). `
|
||||
+ 'Without --shard, the full suite runs as N concurrent shard processes '
|
||||
+ '(plus a serial tree-mutating shard) instead.',
|
||||
+ '(plus an exclusive host-state shard) instead.',
|
||||
);
|
||||
for (const line of formatShardSummary(shards)) console.log(line);
|
||||
return 0;
|
||||
@@ -1874,18 +2300,18 @@ async function main(): Promise<number> {
|
||||
// wedge only ever costs its own shard. WORKER_HOSTILE files are moot in
|
||||
// process shards (no workers) and fold back into normal assignment.
|
||||
const jobs = fullSuiteJobs();
|
||||
// Phase split: tree-mutating tests run AFTER the parallel shards, in one
|
||||
// serial shard, so no concurrent shard ever reads a half-regenerated tree.
|
||||
const mutators = files.filter((f) => f in TREE_MUTATING);
|
||||
// Phase split: exclusive host-state fixtures run AFTER the parallel shards,
|
||||
// so their shared process or filesystem state cannot interfere with readers.
|
||||
const exclusive = files.filter((f) => f in TREE_MUTATING);
|
||||
const readers = files.filter((f) => !(f in TREE_MUTATING));
|
||||
const durations = loadFreeTestDurations();
|
||||
if (durations) warnUnseededFreeFiles(files, durations);
|
||||
const packed = durations ? packShardsByDuration(readers, jobs, durations) : null;
|
||||
const shards = packed ? packed.shards : assignFilesToShards(readers, jobs);
|
||||
const totalShards = jobs + (mutators.length > 0 ? 1 : 0);
|
||||
const totalShards = jobs + (exclusive.length > 0 ? 1 : 0);
|
||||
console.log(`[test:free] full suite: ${readers.length} files across ${jobs} shard processes`
|
||||
+ (packed ? ' (duration-packed)' : '')
|
||||
+ (mutators.length > 0 ? `, then ${mutators.length} tree-mutating file(s) serially` : ''));
|
||||
+ (exclusive.length > 0 ? `, then ${exclusive.length} exclusive host-state file(s) serially` : ''));
|
||||
if (packed) {
|
||||
// One line per shard so a packing regression is diagnosable from any log.
|
||||
packed.predictedMs.forEach((ms, i) => {
|
||||
@@ -1906,33 +2332,33 @@ async function main(): Promise<number> {
|
||||
})),
|
||||
);
|
||||
let worst = Math.max(...outcomes.map((o) => exitCodeFor(o.status)));
|
||||
// Cancellation stops the run: don't launch the serial tree-mutating shard
|
||||
// Cancellation stops the run: don't launch the exclusive host-state shard
|
||||
// after a SIGINT/SIGTERM already killed the parallel phase.
|
||||
if (mutators.length > 0 && !isTerminationRequested()) {
|
||||
const mutatorOutcome = await runFreeShard(mutators, totalShards, totalShards, {
|
||||
wallTimeoutMs: shardTimeout(mutators.length),
|
||||
if (exclusive.length > 0 && !isTerminationRequested()) {
|
||||
const exclusiveOutcome = await runFreeShard(exclusive, totalShards, totalShards, {
|
||||
wallTimeoutMs: shardTimeout(exclusive.length),
|
||||
verbose: options.verbose,
|
||||
});
|
||||
worst = Math.max(worst, exitCodeFor(mutatorOutcome.status));
|
||||
if (mutatorOutcome.status !== 'passed') {
|
||||
// Mutator safety rests on each test restoring default state itself; a
|
||||
worst = Math.max(worst, exitCodeFor(exclusiveOutcome.status));
|
||||
if (exclusiveOutcome.status !== 'passed') {
|
||||
// Fixture safety rests on each test restoring default state itself; a
|
||||
// SIGKILL at the wall deadline (or a mid-regeneration crash) defeats
|
||||
// that by construction. Say so, loudly, before someone commits
|
||||
// regenerated SKILL.md / .agents artifacts by accident.
|
||||
const dirty = spawnSyncGitStatusGenerated();
|
||||
if (dirty.length > 0) {
|
||||
console.error('[test:free] ⚠ tree-mutating shard did not finish cleanly — generated artifacts may be mid-regeneration:');
|
||||
console.error('[test:free] ⚠ exclusive host-state shard did not finish cleanly — generated artifacts are dirty:');
|
||||
for (const line of dirty.slice(0, 20)) console.error(`[test:free] ${line}`);
|
||||
console.error('[test:free] restore with: bun run gen:skill-docs (or git checkout -- <paths>)');
|
||||
}
|
||||
}
|
||||
outcomes.push(mutatorOutcome);
|
||||
outcomes.push(exclusiveOutcome);
|
||||
}
|
||||
|
||||
return (await retryFailedFreeFiles(outcomes, totalShards, options)).exitCode;
|
||||
}
|
||||
|
||||
/** Dirty generated artifacts (SKILL.md / host outputs) after a failed mutator shard. */
|
||||
/** Dirty generated artifacts (SKILL.md / host outputs) after a failed exclusive shard. */
|
||||
function spawnSyncGitStatusGenerated(): string[] {
|
||||
const result = spawnSync('git', ['status', '--porcelain'], { cwd: ROOT, encoding: 'utf8' });
|
||||
if (result.status !== 0 || !result.stdout) return [];
|
||||
|
||||
Reference in new issue
Block a user