From 66c0e5f02e8a094f396e746ab4f098137fb4f8e5 Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 2 Oct 2026 21:00:04 +0000 Subject: [PATCH] =?UTF-8?q?fix:=20full-suite=20repairs=20=E2=80=94=20priva?= =?UTF-8?q?te=20state=20roots=20for=207=20real-HOME=20writers,=20truthful?= =?UTF-8?q?=20home-guard=20attribution=20with=20--attribute-home,=20gstack?= =?UTF-8?q?-paths=20shim=20in=20ship-hook=20fixtures,=20Copilot=20in=20tea?= =?UTF-8?q?m-init=20probes,=20test-of-test=20detector,=20cookie=20review?= =?UTF-8?q?=20fixture=20pinned=20to=20the=20approved=20excerpt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/gstack-team-init | 4 +- browse/test/extension-token.test.ts | 3 + browse/test/server-factory.test.ts | 3 + .../test/server-route-auth-blackbox.test.ts | 3 + browse/test/server-route-table.test.ts | 3 + docs/TESTING_INTERNALS.md | 19 +++++ ios-qa/daemon/test/auth-mint.test.ts | 3 + scripts/lib/free-home-guard.ts | 75 +++++++++++++++++-- scripts/lib/shard-engine.ts | 16 ++++ scripts/test-free-shards.ts | 60 +++++++-------- test/fixtures/shard-cli-parity/cases.ts | 2 + test/fixtures/shard-cli-parity/results.json | 47 +++++++++++- test/free-home-guard.test.ts | 43 ++++++++++- test/helpers/manual-judge-review-fixture.ts | 16 +++- test/helpers/ship-hook-actor.ts | 1 + test/security-dashboard-fallback.test.ts | 1 + test/ship-hook-refresh.test.ts | 1 + test/skill-validation.test.ts | 14 ++-- test/spec-template-invariants.test.ts | 5 +- test/test-of-test-ratchet.test.ts | 74 +----------------- 20 files changed, 270 insertions(+), 123 deletions(-) diff --git a/bin/gstack-team-init b/bin/gstack-team-init index d4e368a2c..eba34c66d 100755 --- a/bin/gstack-team-init +++ b/bin/gstack-team-init @@ -81,7 +81,7 @@ else ```bash _GS="" -for _D in "${GSTACK_ROOT:-}" "$HOME/.claude/skills/gstack" "$HOME/.codex/skills/gstack" "$HOME/.factory/skills/gstack" "$HOME/.kiro/skills/gstack" "$HOME/.config/opencode/skills/gstack" "$HOME/.slate/skills/gstack" "$HOME/.cursor/skills/gstack" "$HOME/.openclaw/skills/gstack" "$HOME/.hermes/skills/gstack" "$HOME/.gbrain/skills/gstack" "$HOME/.gstack/repos/gstack"; do +for _D in "${GSTACK_ROOT:-}" "$HOME/.claude/skills/gstack" "$HOME/.codex/skills/gstack" "$HOME/.factory/skills/gstack" "$HOME/.kiro/skills/gstack" "$HOME/.config/opencode/skills/gstack" "$HOME/.slate/skills/gstack" "$HOME/.cursor/skills/gstack" "$HOME/.openclaw/skills/gstack" "$HOME/.hermes/skills/gstack" "$HOME/.gbrain/skills/gstack" "$HOME/.copilot/skills/gstack" "$HOME/.gstack/repos/gstack"; do [ -z "$_GS" ] && [ -n "$_D" ] && [ -d "$_D/bin" ] && _GS="$_D" done [ -n "$_GS" ] && echo "GSTACK_OK: $_GS" || echo "GSTACK_MISSING" @@ -135,7 +135,7 @@ if [ "$MODE" = "required" ]; then # repo location. Block only when NONE exist (#2500 — hardcoding # ~/.claude/skills/gstack false-blocked Codex-host and migrated-repo installs). _GSTACK_ROOT="" -for _D in "${GSTACK_ROOT:-}" "$HOME/.claude/skills/gstack" "$HOME/.codex/skills/gstack" "$HOME/.factory/skills/gstack" "$HOME/.kiro/skills/gstack" "$HOME/.config/opencode/skills/gstack" "$HOME/.slate/skills/gstack" "$HOME/.cursor/skills/gstack" "$HOME/.openclaw/skills/gstack" "$HOME/.hermes/skills/gstack" "$HOME/.gbrain/skills/gstack" "$HOME/.gstack/repos/gstack"; do +for _D in "${GSTACK_ROOT:-}" "$HOME/.claude/skills/gstack" "$HOME/.codex/skills/gstack" "$HOME/.factory/skills/gstack" "$HOME/.kiro/skills/gstack" "$HOME/.config/opencode/skills/gstack" "$HOME/.slate/skills/gstack" "$HOME/.cursor/skills/gstack" "$HOME/.openclaw/skills/gstack" "$HOME/.hermes/skills/gstack" "$HOME/.gbrain/skills/gstack" "$HOME/.copilot/skills/gstack" "$HOME/.gstack/repos/gstack"; do [ -z "$_GSTACK_ROOT" ] && [ -n "$_D" ] && [ -d "$_D/bin" ] && _GSTACK_ROOT="$_D" done diff --git a/browse/test/extension-token.test.ts b/browse/test/extension-token.test.ts index dca3d243d..cc66d6d80 100644 --- a/browse/test/extension-token.test.ts +++ b/browse/test/extension-token.test.ts @@ -29,6 +29,9 @@ import { import { __resetRegistry } from '../src/token-registry'; import { BrowserManager } from '../src/browser-manager'; import { resolveConfig } from '../src/config'; +import { usePrivateStateRoot } from '../../test/helpers/private-state-root'; + +usePrivateStateRoot(); const PINNED_ORIGIN = `chrome-extension://${GSTACK_EXTENSION_ID}`; const fixtureDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-extension-token-'))); diff --git a/browse/test/server-factory.test.ts b/browse/test/server-factory.test.ts index b214ad46e..166e9e0e4 100644 --- a/browse/test/server-factory.test.ts +++ b/browse/test/server-factory.test.ts @@ -15,6 +15,9 @@ import * as crypto from 'crypto'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; +import { usePrivateStateRoot } from '../../test/helpers/private-state-root'; + +usePrivateStateRoot(); const fixtureDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-server-factory-')); const fixtureConfig = resolveConfig({ BROWSE_STATE_FILE: path.join(fixtureDir, 'state/browse.json') }); diff --git a/browse/test/server-route-auth-blackbox.test.ts b/browse/test/server-route-auth-blackbox.test.ts index a1b0e5fa7..b9a1c9930 100644 --- a/browse/test/server-route-auth-blackbox.test.ts +++ b/browse/test/server-route-auth-blackbox.test.ts @@ -23,6 +23,9 @@ import { __resetRegistry, __resetConnectRateLimit, createToken } from '../src/to import { mintSseSessionToken, SSE_COOKIE_NAME } from '../src/sse-session-cookie'; import { BrowserManager } from '../src/browser-manager'; import { resolveConfig } from '../src/config'; +import { usePrivateStateRoot } from '../../test/helpers/private-state-root'; + +usePrivateStateRoot(); const fixtureDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-route-blackbox-')); const fixtureConfig = resolveConfig({ BROWSE_STATE_FILE: path.join(fixtureDir, 'state/browse.json') }); diff --git a/browse/test/server-route-table.test.ts b/browse/test/server-route-table.test.ts index 0fbbc80fa..a959a0750 100644 --- a/browse/test/server-route-table.test.ts +++ b/browse/test/server-route-table.test.ts @@ -27,6 +27,9 @@ import { buildFetchHandler, __testInternals__ } from '../src/server'; import { __resetRegistry } from '../src/token-registry'; import { BrowserManager } from '../src/browser-manager'; import { resolveConfig } from '../src/config'; +import { usePrivateStateRoot } from '../../test/helpers/private-state-root'; + +usePrivateStateRoot(); type Cred = 'none' | 'wrong' | 'root' | 'scoped' | 'sse-cookie' | 'extension-origin'; const CREDS: Cred[] = ['none', 'wrong', 'root', 'scoped', 'sse-cookie', 'extension-origin']; diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index 4ffe05a4e..a30e1653b 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -716,6 +716,25 @@ resolves (`browserAvailable()` — Aside, or the browse binary CI builds with `bun run build:gates`) and skip only when neither exists; the fallback engine's own tests run everywhere. +### Real-home tripwire + +No free test may write the developer's real home. `scripts/lib/free-home-guard.ts` +snapshots `~/.gstack`, `~/.claude`, `~/.codex`, `~/.agents` and `~/.config/gstack` +(live session logs excluded) and fails the run when an entry changes. + +- A shard that runs alone (`--shard`, CI, the exclusive host-state shard, the + flaky retry) owns its window, so the failure names its files. +- Full-suite shards run concurrently on one HOME and cannot tell whose write a + change was. The runner guards that whole phase once and names no shard. +- `bun run scripts/test-free-shards.ts --attribute-home` names the writer: it + runs every file alone with a private HOME (browser cache and git identity + still come from the real home) and reports each file that wrote a watched + surface. The retained shard directory keeps the written files as evidence. + +Fix a writer with `usePrivateStateRoot()` (`test/helpers/private-state-root.ts`) +or a child HOME/GSTACK_HOME, and resolve product state paths at write time, +never at import. + ## Running evals as an agent: detach Moved verbatim from CLAUDE.md (#2096 size limit). The short rule stays in diff --git a/ios-qa/daemon/test/auth-mint.test.ts b/ios-qa/daemon/test/auth-mint.test.ts index 844c6eff9..bfa0076e7 100644 --- a/ios-qa/daemon/test/auth-mint.test.ts +++ b/ios-qa/daemon/test/auth-mint.test.ts @@ -8,6 +8,9 @@ import { join } from 'path'; import { mintForCaller } from '../src/auth-mint'; import { SessionTokenStore } from '../src/session-tokens'; import { grantIdentity } from '../src/allowlist'; +import { usePrivateStateRoot } from '../../../test/helpers/private-state-root'; + +usePrivateStateRoot(); let tmpDir: string; let listPath: string; diff --git a/scripts/lib/free-home-guard.ts b/scripts/lib/free-home-guard.ts index 59373e9ba..981a85336 100644 --- a/scripts/lib/free-home-guard.ts +++ b/scripts/lib/free-home-guard.ts @@ -6,12 +6,17 @@ * when one of those entries changed. It watches only these surfaces; it is not * a universal write-containment check. Live agent-session logs that a * concurrently running host keeps writing are excluded, and directory mtimes - * are ignored, so only file, link and directory-set changes count. Concurrent - * shards whose windows overlap a change all report it. + * are ignored, so only file, link and directory-set changes count. + * + * A shard that runs alone owns its window, so its guard names its files. + * Concurrent shards share one HOME and cannot tell whose write a change was, so + * the runner guards that phase once and names no file; `--attribute-home` + * then runs every file alone in a private HOME, where any write is that file's. */ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; +import { forEachFileAlone } from './shard-engine'; export const FREE_HOME_SURFACES = ['.gstack', '.claude', '.codex', '.agents', '.config/gstack'] as const; @@ -76,15 +81,28 @@ export function diffFreeHome(before: FreeHomeSnapshot, after: FreeHomeSnapshot): return [...changed].sort(); } -export function formatFreeHomeChange(changed: string[], files: string[]): string { +/** Who owns the snapshotted window: one shard, a concurrent phase, or one file in a private HOME. */ +export type FreeHomeScope = { kind: 'shard' } | { kind: 'concurrent'; shards: number } | { kind: 'private' }; + +export function formatFreeHomeChange(changed: string[], files: string[], scope: FreeHomeScope = { kind: 'shard' }): string { const shown = changed.slice(0, MAX_REPORTED).map(entry => `~/${entry.split(path.sep).join('/')}`); const more = changed.length > MAX_REPORTED ? ` (+${changed.length - MAX_REPORTED} more)` : ''; - return `real home changed while this shard ran (watches ${FREE_HOME_SURFACES.map(s => `~/${s}`).join(', ')} only): ` - + `${shown.join(', ')}${more}. Give the writer a private HOME/GSTACK_HOME. Shard files: ${files.join(', ')}`; + const watched = `(watches ${FREE_HOME_SURFACES.map(s => `~/${s}`).join(', ')} only): ${shown.join(', ')}${more}.`; + const fix = 'Give the writer a private HOME/GSTACK_HOME.'; + if (scope.kind === 'concurrent') { + return `real home changed while ${scope.shards} shards ran concurrently ${watched} They share one HOME, so no shard or file ` + + `is named. Find the writer with \`bun run scripts/test-free-shards.ts --attribute-home\` (each file alone in a private HOME). ${fix}`; + } + if (scope.kind === 'private') return `${files.join(', ')} wrote its private HOME ${watched} With the real HOME it writes there. ${fix}`; + return `real home changed while this shard ran ${watched} ${fix} Shard files: ${files.join(', ')}`; } +export interface FreeHomeGuard { verify(): string | null } +/** How runFreeShard guards one shard; it may redirect `env` before the child starts. */ +export type FreeHomeGuardFactory = (files: string[], env: NodeJS.ProcessEnv, stateDir: string) => FreeHomeGuard; + /** Take the baseline now; verify() returns null or the failure to report. */ -export function guardFreeHome(files: string[], env: NodeJS.ProcessEnv = process.env): { verify(): string | null } { +export function guardFreeHome(files: string[], env: NodeJS.ProcessEnv = process.env, scope: FreeHomeScope = { kind: 'shard' }): FreeHomeGuard { const home = freeHomeDir(env); const unreadable = (error: unknown) => `real home surfaces could not be snapshotted (${(error as NodeJS.ErrnoException).code ?? 'error'}); shard files: ${files.join(', ')}`; let before: FreeHomeSnapshot | undefined; @@ -95,8 +113,51 @@ export function guardFreeHome(files: string[], env: NodeJS.ProcessEnv = process. if (!before) return baselineError!; try { const changed = diffFreeHome(before, snapshotFreeHome(home)); - return changed.length ? formatFreeHomeChange(changed, files) : null; + return changed.length ? formatFreeHomeChange(changed, files, scope) : null; } catch (error) { return unreadable(error); } }, }; } + +/** For a shard inside a concurrently guarded phase: the phase guard reports, this one never does. */ +export const sharedFreeHome = (): FreeHomeGuard => ({ verify: () => null }); + +/** + * Redirect one shard's HOME to `/home` and guard that instead. The + * browser cache and git identity still come from the real home, so tests run + * as they would there; only home writes are diverted. + */ +export function privateFreeHome(files: string[], env: NodeJS.ProcessEnv, stateDir: string): FreeHomeGuard { + const real = freeHomeDir(env); + const home = path.join(stateDir, 'home'); + fs.mkdirSync(home, { recursive: true, mode: 0o700 }); + env.PLAYWRIGHT_BROWSERS_PATH ??= path.join(real, process.platform === 'darwin' ? 'Library/Caches' : '.cache', 'ms-playwright'); + if (!env.GIT_CONFIG_GLOBAL && fs.existsSync(path.join(real, '.gitconfig'))) env.GIT_CONFIG_GLOBAL = path.join(real, '.gitconfig'); + env.HOME = home; + if (env.USERPROFILE !== undefined) env.USERPROFILE = home; + return guardFreeHome(files, env, { kind: 'private' }); +} + +/** + * `--attribute-home`: run each reader file alone with a private HOME, `jobs` + * at a time, then each exclusive host-state file alone, and name every file + * that wrote a watched surface. Exit 1 when any did. + */ +export async function attributeFreeHomeWriters( + readers: string[], exclusive: string[], jobs: number, + runAlone: (file: string, index: number, homeGuard: FreeHomeGuardFactory) => Promise, +): Promise { + const writers: string[] = []; + console.log(`[test:free] attributing home writes: ${readers.length + exclusive.length} files, each alone in a private HOME, ${jobs} at a time`); + await forEachFileAlone(readers, exclusive, jobs, async (file, index) => { + let change = null as string | null; + await runAlone(file, index, (files, env, stateDir) => { + const guard = privateFreeHome(files, env, stateDir); + return { verify: () => (change = guard.verify()) }; + }); + if (change) writers.push(change); + }); + for (const writer of writers.sort()) console.error(`[test:free] ✗ ${writer}`); + console.log(`[test:free] ${writers.length ? `${writers.length} file(s) wrote home surfaces` : 'no file wrote a watched home surface'}`); + return writers.length ? 1 : 0; +} diff --git a/scripts/lib/shard-engine.ts b/scripts/lib/shard-engine.ts index 341d8a3b6..adaf35dc0 100644 --- a/scripts/lib/shard-engine.ts +++ b/scripts/lib/shard-engine.ts @@ -595,6 +595,22 @@ export async function removeShardSandbox(stateDir: string): Promise { catch { /* a locked file must not turn a real verdict into an exception */ } } +/** Run once per file, alone: readers `jobs` at a time, then exclusive files one by one. Stops on termination. */ +export async function forEachFileAlone( + readers: string[], exclusive: string[], jobs: number, run: (file: string, index: number) => Promise, +): Promise { + let cursor = 0; + const worker = async (files: string[], offset: number) => { + while (cursor < files.length && !isTerminationRequested()) { + const index = cursor++; + await run(files[index], offset + index); + } + }; + await Promise.all(Array.from({ length: Math.max(1, jobs) }, () => worker(readers, 0))); + cursor = 0; + await worker(exclusive, readers.length); +} + let shardLogSequence = 0; /** Timestamped log path; pid + sequence defeat same-millisecond collisions. */ diff --git a/scripts/test-free-shards.ts b/scripts/test-free-shards.ts index 0763b5ea2..aa28ff511 100755 --- a/scripts/test-free-shards.ts +++ b/scripts/test-free-shards.ts @@ -73,6 +73,7 @@ * bun run scripts/test-free-shards.ts --wall-timeout 600 # override the kill deadline * bun run scripts/test-free-shards.ts --verbose # forward the full child stream * bun run scripts/test-free-shards.ts --quick # explicit fast subset, not acceptance + * bun run scripts/test-free-shards.ts --attribute-home # name real-home writers: each file alone, private HOME * bun run scripts/test-free-shards.ts --ci-plan plan.json --shards 20 * bun run scripts/test-free-shards.ts --ci-run plan.json --shard 1 --result result.json * bun run scripts/test-free-shards.ts --ci-verify plan.json --results results/ @@ -86,11 +87,12 @@ import { StringDecoder } from 'node:string_decoder'; import { createHash, randomUUID } from 'node:crypto'; import { isPaidTestFile } from '../test/helpers/paid-test-set'; import { resolveStateRoot } from '../lib/state-root'; -import { guardFreeHome } from './lib/free-home-guard'; +import { attributeFreeHomeWriters, guardFreeHome, sharedFreeHome, type FreeHomeGuardFactory } from './lib/free-home-guard'; import { BunTestOutputClassifier, createShardSandbox, exactTestFileSelectors, + forEachFileAlone, isTerminationRequested, killProcessGroup, BunFailureSummaryParser, @@ -902,6 +904,7 @@ type CliOptions = { /** True when --wall-timeout was passed explicitly; full-suite mode only auto-scales the default. */ wallTimeoutExplicit: boolean; quick: boolean; + attributeHome: boolean; ciPlan: string | null; ciRun: string | null; ciVerify: string | null; @@ -920,6 +923,7 @@ export function parseCliOptions(argv: string[]): CliOptions { let wallTimeoutMs = DEFAULT_WALL_TIMEOUT_MS; let wallTimeoutExplicit = false; let quick = false; + let attributeHome = false; const paths: Record<'ciPlan' | 'ciRun' | 'ciVerify' | 'result' | 'results', string | null> = { ciPlan: null, ciRun: null, ciVerify: null, result: null, results: null, }; @@ -936,6 +940,7 @@ export function parseCliOptions(argv: string[]): CliOptions { '--windows-only': () => { windowsOnly = true; }, '--verbose': () => { verbose = true; }, '--quick': () => { quick = true; }, + '--attribute-home': () => { attributeHome = true; }, '--ci-plan': pathFlag('--ci-plan', 'ciPlan'), '--ci-run': pathFlag('--ci-run', 'ciRun'), '--ci-verify': pathFlag('--ci-verify', 'ciVerify'), @@ -960,11 +965,11 @@ export function parseCliOptions(argv: string[]): CliOptions { }); const ciModes = [paths.ciPlan, paths.ciRun, paths.ciVerify].filter(Boolean).length; - if (ciModes > 1 || (ciModes && (quick || listOnly || dryRun || recordDurations || windowsOnly))) throw new Error('CI modes cannot be combined with other selection modes'); + if (ciModes > 1 || (ciModes && (quick || listOnly || dryRun || recordDurations || windowsOnly || attributeHome))) throw new Error('CI modes cannot be combined with other selection modes'); if (paths.ciRun && (shardIndex === null || !paths.result)) throw new Error('--ci-run requires --shard and --result'); if (paths.ciVerify && !paths.results) throw new Error('--ci-verify requires --results'); if (quick && (recordDurations || windowsOnly || shardIndex !== null)) throw new Error('--quick cannot change recording, Windows or shard selection'); - return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, ...paths }; + return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths }; } function formatShardSummary(shards: string[][]): string[] { @@ -1394,6 +1399,8 @@ export interface RunFreeShardOptions { /** Per-run full-stream log path (tests inject). Default: a private retained file under .context/free-test-logs. */ logFilePath?: string; log?: (line: string) => void; + /** Home tripwire for this shard. Default: guard the real home and name this shard's files. */ + homeGuard?: FreeHomeGuardFactory; } const EPILOGUE_WORD: Record = { @@ -1915,6 +1922,7 @@ export async function runFreeShard( // shards and prior daemons replace each other's state; override inherited state. env.BROWSE_STATE_FILE = path.join(stateDir, '.gstack', 'browse.json'); env.GSTACK_FREE_SHARD_ID = randomUUID(); + const home = options.homeGuard ? options.homeGuard(files, env, stateDir) : guardFreeHome(files, env); const startedAt = Date.now(); const classifier = new BunTestOutputClassifier(); @@ -1952,7 +1960,7 @@ export async function runFreeShard( child = await runShardChild({ command, args, cwd: rootDir, env, timeoutMs: wallTimeoutMs, attach: () => { - const browser = trackShardBrowser(stateDir, env), home = guardFreeHome(files, env); + const browser = trackShardBrowser(stateDir, env); return { signal: (force) => browser.signal(force), settle: async () => { cleanupError = await browser.settle() ?? home.verify(); } }; }, hookStreams: (spawned) => [consumeStream(spawned.stdout, 'stdout'), consumeStream(spawned.stderr, 'stderr')], @@ -2053,30 +2061,18 @@ async function recordFreeTestDurations(files: string[], jobs: number): Promise = {}; const failed: string[] = []; const expectedCount = files.length; - let cursor = 0; console.log(`[test:free] recording per-file durations: ${files.length} files across ${jobs} workers`); - const worker = async (): Promise => { - for (;;) { - if (isTerminationRequested()) return; - const index = cursor; - cursor += 1; - if (index >= files.length) return; - const file = files[index]; - const outcome = await runFreeShard([file], index + 1, files.length, { - wallTimeoutMs: wallTimeoutForShard(1), - quiet: true, - }); - durations[normalizeRelativePath(file)] = outcome.elapsedMs; - if (outcome.status !== 'passed') failed.push(file); - } - }; + const phaseHome = guardFreeHome(files, process.env, { kind: 'concurrent', shards: jobs }); // As in full-suite mode, finish parallel work before exclusive host-state fixtures. - const exclusive = files.filter(file => file in TREE_MUTATING); - files = files.filter(file => !(file in TREE_MUTATING)); - await Promise.all(Array.from({ length: Math.max(1, jobs) }, () => worker())); - files = exclusive; - cursor = 0; - await worker(); + await forEachFileAlone(files.filter(file => !(file in TREE_MUTATING)), files.filter(file => file in TREE_MUTATING), jobs, async (file, index) => { + const outcome = await runFreeShard([file], index + 1, files.length, { + wallTimeoutMs: wallTimeoutForShard(1), quiet: true, homeGuard: sharedFreeHome, + }); + durations[normalizeRelativePath(file)] = outcome.elapsedMs; + if (outcome.status !== 'passed') failed.push(file); + }); + const homeChange = phaseHome.verify(); + if (homeChange) console.error(`[test:free] ${homeChange}`); if (Object.keys(durations).length !== expectedCount) { throw new Error('Duration recording was interrupted; the seed was not replaced.'); } @@ -2085,7 +2081,7 @@ async function recordFreeTestDurations(files: string[], jobs: number): Promise 0) { + if (failed.length > 0 || homeChange) { // Failures still recorded (a red file's duration is still a real cost), // but surfaced loudly — recording from a broken tree deserves a look. console.error(`[test:free] WARNING: ${failed.length} file(s) failed while recording:`); @@ -2245,6 +2241,8 @@ async function main(): Promise { if (options.recordDurations) { return recordFreeTestDurations(files, fullSuiteJobs()); } + if (options.attributeHome) return attributeFreeHomeWriters(files.filter(file => !(file in TREE_MUTATING)), files.filter(file => file in TREE_MUTATING), + fullSuiteJobs(), (file, index, homeGuard) => runFreeShard([file], index + 1, files.length, { wallTimeoutMs: wallTimeoutForShard(1), quiet: true, log: () => {}, homeGuard })); if (options.dryRun) { const shards = assignFilesToShards(files, options.shardCount); @@ -2305,8 +2303,10 @@ async function main(): Promise { } const shardTimeout = (fileCount: number): number => options.wallTimeoutExplicit ? options.wallTimeoutMs : wallTimeoutForShard(fileCount, options.wallTimeoutMs); + const phaseHome = guardFreeHome(readers, process.env, { kind: 'concurrent', shards: shards.length }); const outcomes = await Promise.all( shards.map((shardFiles, index) => runFreeShard(shardFiles, index + 1, totalShards, { + homeGuard: sharedFreeHome, // Packed shards get duration-aware walls: LPT decouples file count from // cost BY DESIGN, so the 5s/file heuristic would undersize a shard // holding few expensive files. @@ -2316,7 +2316,8 @@ async function main(): Promise { verbose: options.verbose, })), ); - let worst = Math.max(...outcomes.map((o) => exitCodeFor(o.status))); + const homeChange = phaseHome.verify(); + if (homeChange) console.error(`[test:free] ${homeChange}`); // Cancellation stops the run: don't launch the exclusive host-state shard // after a SIGINT/SIGTERM already killed the parallel phase. if (exclusive.length > 0 && !isTerminationRequested()) { @@ -2324,7 +2325,6 @@ async function main(): Promise { wallTimeoutMs: shardTimeout(exclusive.length), verbose: options.verbose, }); - worst = Math.max(worst, exitCodeFor(exclusiveOutcome.status)); if (exclusiveOutcome.status !== 'passed') { // Fixture safety rests on each test restoring default state itself; a // SIGKILL at the wall deadline (or a mid-regeneration crash) defeats @@ -2340,7 +2340,7 @@ async function main(): Promise { outcomes.push(exclusiveOutcome); } - return (await retryFailedFreeFiles(outcomes, totalShards, options)).exitCode; + return Math.max(homeChange ? 1 : 0, (await retryFailedFreeFiles(outcomes, totalShards, options)).exitCode); } /** Dirty generated artifacts (SKILL.md / host outputs) after a failed exclusive shard. */ diff --git a/test/fixtures/shard-cli-parity/cases.ts b/test/fixtures/shard-cli-parity/cases.ts index 75b2a4db0..b7184d0e2 100644 --- a/test/fixtures/shard-cli-parity/cases.ts +++ b/test/fixtures/shard-cli-parity/cases.ts @@ -27,6 +27,8 @@ export const FREE_CASES: string[][] = [ ['--ci-verify', 'plan.json'], ['--quick', '--shard', '1'], ['--quick', '--windows-only'], + ['--attribute-home'], + ['--ci-plan', 'a.json', '--attribute-home'], ]; export const PAID_CASES: Array<{ argv: string[]; env?: Record }> = [ diff --git a/test/fixtures/shard-cli-parity/results.json b/test/fixtures/shard-cli-parity/results.json index b2e315866..b24a4fa80 100644 --- a/test/fixtures/shard-cli-parity/results.json +++ b/test/fixtures/shard-cli-parity/results.json @@ -1,5 +1,5 @@ { - "recordedFrom": "96764e80 (pre-engine base runners); paid lane re-recorded at v1.91.12.0 for the marathon tier and the --slice-budget, --max-parallel, --case and --trials flags (PR 2999)", + "recordedFrom": "96764e80 (pre-engine base runners); paid lane re-recorded at v1.91.12.0 for the marathon tier and the --slice-budget, --max-parallel, --case and --trials flags (PR 2999); free lane re-recorded for the --attribute-home flag (PR 3014)", "free": [ { "argv": [], @@ -15,6 +15,7 @@ "wallTimeoutMs": 360000, "wallTimeoutExplicit": false, "quick": false, + "attributeHome": false, "ciPlan": null, "ciRun": null, "ciVerify": null, @@ -41,6 +42,7 @@ "wallTimeoutMs": 360000, "wallTimeoutExplicit": false, "quick": false, + "attributeHome": false, "ciPlan": null, "ciRun": null, "ciVerify": null, @@ -66,6 +68,7 @@ "wallTimeoutMs": 360000, "wallTimeoutExplicit": false, "quick": false, + "attributeHome": false, "ciPlan": null, "ciRun": null, "ciVerify": null, @@ -90,6 +93,7 @@ "wallTimeoutMs": 360000, "wallTimeoutExplicit": false, "quick": false, + "attributeHome": false, "ciPlan": null, "ciRun": null, "ciVerify": null, @@ -114,6 +118,7 @@ "wallTimeoutMs": 360000, "wallTimeoutExplicit": false, "quick": true, + "attributeHome": false, "ciPlan": null, "ciRun": null, "ciVerify": null, @@ -141,6 +146,7 @@ "wallTimeoutMs": 360000, "wallTimeoutExplicit": false, "quick": false, + "attributeHome": false, "ciPlan": null, "ciRun": null, "ciVerify": null, @@ -166,6 +172,7 @@ "wallTimeoutMs": 600000, "wallTimeoutExplicit": true, "quick": false, + "attributeHome": false, "ciPlan": null, "ciRun": null, "ciVerify": null, @@ -193,6 +200,7 @@ "wallTimeoutMs": 360000, "wallTimeoutExplicit": false, "quick": false, + "attributeHome": false, "ciPlan": "plan.json", "ciRun": null, "ciVerify": null, @@ -222,6 +230,7 @@ "wallTimeoutMs": 360000, "wallTimeoutExplicit": false, "quick": false, + "attributeHome": false, "ciPlan": null, "ciRun": "plan.json", "ciVerify": null, @@ -249,6 +258,7 @@ "wallTimeoutMs": 360000, "wallTimeoutExplicit": false, "quick": false, + "attributeHome": false, "ciPlan": null, "ciRun": null, "ciVerify": "plan.json", @@ -381,6 +391,41 @@ "result": { "error": "--quick cannot change recording, Windows or shard selection" } + }, + { + "argv": [ + "--attribute-home" + ], + "result": { + "ok": { + "dryRun": false, + "listOnly": false, + "recordDurations": false, + "windowsOnly": false, + "verbose": false, + "shardCount": 20, + "shardIndex": null, + "wallTimeoutMs": 360000, + "wallTimeoutExplicit": false, + "quick": false, + "attributeHome": true, + "ciPlan": null, + "ciRun": null, + "ciVerify": null, + "result": null, + "results": null + } + } + }, + { + "argv": [ + "--ci-plan", + "a.json", + "--attribute-home" + ], + "result": { + "error": "CI modes cannot be combined with other selection modes" + } } ], "paid": [ diff --git a/test/free-home-guard.test.ts b/test/free-home-guard.test.ts index a1e026cde..d01c8c979 100644 --- a/test/free-home-guard.test.ts +++ b/test/free-home-guard.test.ts @@ -2,7 +2,7 @@ import { afterEach, describe, expect, spyOn, test } from 'bun:test'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; -import { diffFreeHome, guardFreeHome, snapshotFreeHome } from '../scripts/lib/free-home-guard'; +import { attributeFreeHomeWriters, diffFreeHome, guardFreeHome, snapshotFreeHome } from '../scripts/lib/free-home-guard'; import { runFreeShard } from '../scripts/test-free-shards'; const homes: string[] = []; @@ -78,6 +78,47 @@ describe('free-suite home-write tripwire (#2895)', () => { expect(failure).toContain('only'); }); + test('a concurrent phase guard names no shard and points at attribution', () => { + const home = fakeHome(); + const guard = guardFreeHome(['test/a.test.ts', 'test/b.test.ts'], { HOME: home }, { kind: 'concurrent', shards: 3 }); + fs.writeFileSync(path.join(home, '.gstack/security/egress.jsonl'), '{}\n'); + const failure = guard.verify(); + expect(failure).toContain('~/.gstack/security/egress.jsonl'); + expect(failure).toContain('while 3 shards ran concurrently'); + expect(failure).toContain('--attribute-home'); + expect(failure).not.toContain('test/a.test.ts'); + }); + + test.skipIf(process.platform === 'win32')('attribution runs each file alone in a private HOME and names only the writer', async () => { + const home = fakeHome(); + const before = snapshotFreeHome(home); + const writer = `require('fs').mkdirSync(require('path').join(process.env.HOME, '.gstack', 'security'), { recursive: true });` + + `require('fs').writeFileSync(require('path').join(process.env.HOME, '.gstack', 'security', 'attempts.jsonl'), '{}\\n');` + + `console.log('Ran 1 tests across 1 files. [1.00ms]')`; + const clean = `console.log('Ran 1 tests across 1 files. [1.00ms]')`; + const errors: string[] = []; + const spy = spyOn(console, 'error').mockImplementation((...args: unknown[]) => { errors.push(args.join(' ')); }); + const logSpy = spyOn(console, 'log').mockImplementation(() => {}); + try { + const exitCode = await attributeFreeHomeWriters(['test/clean.test.ts', 'test/writer.test.ts'], [], 2, (file, index, homeGuard) => + runFreeShard([file], index + 1, 2, { env: { ...process.env, HOME: home }, quiet: true, log: () => {}, homeGuard, + logFilePath: path.join(home, `${index}.log`), + commandFor: () => ({ command: process.execPath, args: ['-e', file.includes('writer') ? writer : clean] }) })); + expect(exitCode).toBe(1); + const verdicts = errors.filter(line => line.startsWith('[test:free] ✗ ')); + expect(verdicts).toHaveLength(1); + expect(verdicts[0]).toContain('test/writer.test.ts wrote its private HOME'); + expect(verdicts[0]).toContain('~/.gstack/security/attempts.jsonl'); + expect(diffFreeHome(before, snapshotFreeHome(home))).toEqual([]); + } finally { + spy.mockRestore(); + logSpy.mockRestore(); + for (const retained of errors.flatMap(line => /retained (\S+)$/.exec(line)?.[1] ?? [])) { + if (/^gstack-free-shard-/.test(path.basename(retained))) fs.rmSync(retained, { recursive: true, force: true }); + } + } + }); + test.skipIf(process.platform === 'win32')('a free shard that writes the real home fails and names its files', async () => { const home = fakeHome(); const writer = `require('fs').writeFileSync(require('path').join(process.env.HOME, '.gstack', '.last-setup-version'), '7.7.7\\n');` diff --git a/test/helpers/manual-judge-review-fixture.ts b/test/helpers/manual-judge-review-fixture.ts index 538fa61c7..148c357df 100644 --- a/test/helpers/manual-judge-review-fixture.ts +++ b/test/helpers/manual-judge-review-fixture.ts @@ -14,14 +14,26 @@ const LATER_BROWSER_EDITS: Array<[string, string]> = [ [' Opera and Opera GX are Windows-only and read from `%APPDATA%\\Opera Software\\Opera Stable` or `Opera GX Stable`, in `Default` or `Profile N` directories; legacy root-level layouts, Opera side profiles and portable or relocated installs are not detected. Opera has no native extraction, so its App-Bound cookies (if any) need manual sign-in.', ''], ]; +// The judged SKILL.md excerpt follows the generated preamble, so later preamble +// edits move its line range without changing its bytes. The approved request +// named lines from 153; the parallel-waves one-line skill start (#2763) removed +// three preamble lines. A raw SKILL.md gets those lines back, and a built prompt, +// which has no preamble, gets the approved start line. +const APPROVED_ENTRYPOINT_START = 153; +const LATER_PREAMBLE_EDITS: Array<[string, string]> = [ + ['~/.claude/skills/gstack/bin/gstack-skill-start --skill "setup-browser-cookies" --model "claude"\n', '_SS="$HOME/.claude/skills/gstack/bin/gstack-skill-start"\n[ -x "$_SS" ] || _SS=".claude/skills/gstack/bin/gstack-skill-start"\n"$_SS" --skill "setup-browser-cookies" --model "claude" --parent-pid "$PPID" \\\n || echo "SKILL_START: unavailable — stale install; run ./setup or /gstack-upgrade (preamble degraded, continue the user\'s task)"\n'], +]; + export function approvedCookieWorkflowSource(source: string): string { let removedLines = 0; let historical = source .replace('sha256sum < "$tmpfile" | awk \'{print $(1)}\'', 'sha256sum "$tmpfile" | awk \'{print $1}\'') .replace('shasum -a 256 < "$tmpfile" | awk \'{print $(1)}\'', 'shasum -a 256 "$tmpfile" | awk \'{print $1}\'') .replace(LATER_BROWSER_BLOCK, block => { removedLines = block.split('\n').length - 1; return ''; }); - for (const [later, earlier] of LATER_BROWSER_EDITS) historical = historical.replace(later, earlier); - return historical.replace(/(--- BEGIN FILE "BROWSER\.md" \(lines \d+-)(\d+)(; section\) ---)/, (_, head, end, tail) => `${head}${Number(end) - removedLines}${tail}`); + for (const [later, earlier] of [...LATER_BROWSER_EDITS, ...LATER_PREAMBLE_EDITS]) historical = historical.replace(later, earlier); + return historical + .replace(/(--- BEGIN FILE "BROWSER\.md" \(lines \d+-)(\d+)(; section\) ---)/, (_, head, end, tail) => `${head}${Number(end) - removedLines}${tail}`) + .replace(/(--- BEGIN FILE "setup-browser-cookies\/SKILL\.md" \(lines )(\d+)-(\d+)(; entrypoint\) ---)/, (_, head, start, end, tail) => `${head}${APPROVED_ENTRYPOINT_START}-${APPROVED_ENTRYPOINT_START + Number(end) - Number(start)}${tail}`); } export function manualReviewFixture(root = resolve(import.meta.dir, '../..')): EvalTestEntry { diff --git a/test/helpers/ship-hook-actor.ts b/test/helpers/ship-hook-actor.ts index 54f268f04..3dc6b1284 100644 --- a/test/helpers/ship-hook-actor.ts +++ b/test/helpers/ship-hook-actor.ts @@ -50,6 +50,7 @@ export function createShipHookFixture(id: ShipHookCase, root = fs.mkdtempSync(pa write(path.join(state, 'config.yaml'), 'redact_prepush_hook: true\n'); write(path.join(installed, 'gstack-config'), `#!/bin/sh\nprintf 'CONFIG:%s\\n' "$*" >> ${quote(receipts)}\nexec bash ${quote(path.join(ROOT, 'bin/gstack-config'))} "$@"\n`, true); write(path.join(installed, 'gstack-redact'), `#!/bin/sh\nprintf 'INSTALL:%s\\n' "$*" >> ${quote(receipts)}\nexec ${quote(process.execPath)} ${quote(path.join(ROOT, 'bin/gstack-redact'))} "$@"\n`, true); + write(path.join(installed, 'gstack-paths'), `#!/bin/sh\nexec bash ${quote(path.join(ROOT, 'bin/gstack-paths'))} "$@"\n`, true); const workflow = readWorkflowExcerpt('ship/SKILL.md', '**Credential pre-push guard', '**Idempotency check'); write(path.join(repo, 'workflow.md'), workflow); const guard = workflow.match(/```bash\n([\s\S]*?)```/)![1].trim(); diff --git a/test/security-dashboard-fallback.test.ts b/test/security-dashboard-fallback.test.ts index 3c07ab4de..fce1936ca 100644 --- a/test/security-dashboard-fallback.test.ts +++ b/test/security-dashboard-fallback.test.ts @@ -114,6 +114,7 @@ function run( ...process.env, PATH: pathEnv, GSTACK_DIR: ROOT, + GSTACK_HOME: join(tmp, "state"), GSTACK_SUPABASE_URL: "https://stub.supabase.test", GSTACK_SUPABASE_ANON_KEY: "stub-key", STUB_CURL_MODE: opts.mode, diff --git a/test/ship-hook-refresh.test.ts b/test/ship-hook-refresh.test.ts index ff55c5736..5aa99956a 100644 --- a/test/ship-hook-refresh.test.ts +++ b/test/ship-hook-refresh.test.ts @@ -43,6 +43,7 @@ beforeEach(() => { mkdirSync(bin, { recursive: true }); writeFixture(join(bin, 'gstack-config'), `#!/bin/sh\nexec bash "${join(ROOT, 'bin/gstack-config')}" "$@"\n`); writeFixture(join(bin, 'gstack-redact'), `#!/bin/sh\nexec "${process.execPath}" "${join(ROOT, 'bin/gstack-redact')}" "$@"\n`); + writeFixture(join(bin, 'gstack-paths'), `#!/bin/sh\nexec bash "${join(ROOT, 'bin/gstack-paths')}" "$@"\n`); expect(shell('~/.claude/skills/gstack/bin/gstack-config set redact_prepush_hook true').status).toBe(0); }); diff --git a/test/skill-validation.test.ts b/test/skill-validation.test.ts index a1ad3c603..563e216a5 100644 --- a/test/skill-validation.test.ts +++ b/test/skill-validation.test.ts @@ -5,6 +5,7 @@ import { SNAPSHOT_FLAGS } from '../browse/src/snapshot'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; +import { usePrivateStateRoot } from './helpers/private-state-root'; const ROOT = path.resolve(import.meta.dir, '..'); @@ -1096,6 +1097,7 @@ describe('CEO review mode validation', () => { describe('gstack-slug', () => { const SLUG_BIN = path.join(ROOT, 'bin', 'gstack-slug'); + usePrivateStateRoot(); test('binary exists and is executable', () => { expect(fs.existsSync(SLUG_BIN)).toBe(true); @@ -1104,7 +1106,7 @@ describe('gstack-slug', () => { }); test('outputs SLUG and BRANCH lines in a git repo', () => { - const result = Bun.spawnSync([SLUG_BIN], { cwd: ROOT, stdout: 'pipe', stderr: 'pipe', timeout: 30_000 }); + const result = Bun.spawnSync([SLUG_BIN], { cwd: ROOT, env: { ...process.env }, stdout: 'pipe', stderr: 'pipe', timeout: 30_000 }); expect(result.exitCode).toBe(0); const output = result.stdout.toString(); expect(output).toContain('SLUG='); @@ -1112,21 +1114,21 @@ describe('gstack-slug', () => { }); test('SLUG does not contain forward slashes', () => { - const result = Bun.spawnSync([SLUG_BIN], { cwd: ROOT, stdout: 'pipe', stderr: 'pipe', timeout: 30_000 }); + const result = Bun.spawnSync([SLUG_BIN], { cwd: ROOT, env: { ...process.env }, stdout: 'pipe', stderr: 'pipe', timeout: 30_000 }); const slug = result.stdout.toString().match(/SLUG=(.*)/)?.[1] ?? ''; expect(slug).not.toContain('/'); expect(slug.length).toBeGreaterThan(0); }); test('BRANCH does not contain forward slashes', () => { - const result = Bun.spawnSync([SLUG_BIN], { cwd: ROOT, stdout: 'pipe', stderr: 'pipe', timeout: 30_000 }); + const result = Bun.spawnSync([SLUG_BIN], { cwd: ROOT, env: { ...process.env }, stdout: 'pipe', stderr: 'pipe', timeout: 30_000 }); const branch = result.stdout.toString().match(/BRANCH=(.*)/)?.[1] ?? ''; expect(branch).not.toContain('/'); expect(branch.length).toBeGreaterThan(0); }); test('output is eval-compatible (KEY=VALUE format)', () => { - const result = Bun.spawnSync([SLUG_BIN], { cwd: ROOT, stdout: 'pipe', stderr: 'pipe', timeout: 30_000 }); + const result = Bun.spawnSync([SLUG_BIN], { cwd: ROOT, env: { ...process.env }, stdout: 'pipe', stderr: 'pipe', timeout: 30_000 }); const lines = result.stdout.toString().trim().split('\n'); expect(lines.length).toBe(2); expect(lines[0]).toMatch(/^SLUG=.+/); @@ -1134,7 +1136,7 @@ describe('gstack-slug', () => { }); test('output values contain only safe characters (no shell metacharacters)', () => { - const result = Bun.spawnSync([SLUG_BIN], { cwd: ROOT, stdout: 'pipe', stderr: 'pipe', timeout: 30_000 }); + const result = Bun.spawnSync([SLUG_BIN], { cwd: ROOT, env: { ...process.env }, stdout: 'pipe', stderr: 'pipe', timeout: 30_000 }); const slug = result.stdout.toString().match(/SLUG=(.*)/)?.[1] ?? ''; const branch = result.stdout.toString().match(/BRANCH=(.*)/)?.[1] ?? ''; // Only alphanumeric, dot, dash, underscore are allowed (#133) @@ -1144,7 +1146,7 @@ describe('gstack-slug', () => { test('eval sets variables under bash with set -euo pipefail', () => { const result = Bun.spawnSync( ['bash', '-c', 'set -euo pipefail; eval "$(./bin/gstack-slug 2>/dev/null)"; echo "SLUG=$SLUG"; echo "BRANCH=$BRANCH"'], - { cwd: ROOT, stdout: 'pipe', stderr: 'pipe', timeout: 30_000 } + { cwd: ROOT, env: { ...process.env }, stdout: 'pipe', stderr: 'pipe', timeout: 30_000 } ); expect(result.exitCode).toBe(0); const output = result.stdout.toString(); diff --git a/test/spec-template-invariants.test.ts b/test/spec-template-invariants.test.ts index f135a77f3..eb1222d9a 100644 --- a/test/spec-template-invariants.test.ts +++ b/test/spec-template-invariants.test.ts @@ -247,8 +247,9 @@ describe('/spec --no-gate keeps redacting', () => { }); describe('/spec archive (carved: gate-and-file section)', () => { - test('uses eval $(gstack-paths) not hardcoded ~/.gstack/', () => { - expect(SEC_TMPL).toMatch(/eval "\$\(.+gstack-paths\)"/); + test('resolves the state root with gstack-paths --get, not hardcoded ~/.gstack/', () => { + expect(SEC_TMPL).toContain('GSTACK_STATE_ROOT=$(~/.claude/skills/gstack/bin/gstack-paths --get GSTACK_STATE_ROOT); : "${GSTACK_STATE_ROOT:?gstack-paths failed; reinstall with ./setup or /gstack-upgrade}"'); + expect(SEC_TMPL).not.toMatch(/eval "\$\(.+gstack-paths\)"/); expect(SEC_TMPL).toMatch(/\$GSTACK_STATE_ROOT\/projects\/\$SLUG\/specs/); // No hardcoded ~/.gstack/projects path anywhere in /spec: expect(TMPL_UNION).not.toMatch(/~\/\.gstack\/projects\/\$SLUG\/specs/); diff --git a/test/test-of-test-ratchet.test.ts b/test/test-of-test-ratchet.test.ts index 7a63e088f..111b67986 100644 --- a/test/test-of-test-ratchet.test.ts +++ b/test/test-of-test-ratchet.test.ts @@ -18,9 +18,7 @@ const ROOT = path.resolve(import.meta.dir, '..'); /** Recorded after workstream D. Add a path only with a one-line reason comment, reviewed in the diff. */ const BASELINE = [ 'test/anthropic-preflight.test.ts', - 'test/artifacts-allowlist-decisions.test.ts', 'test/artifacts-init-migration.test.ts', - 'test/auto-decide-fixture.test.ts', 'test/autoplan-artifact-recorder.test.ts', 'test/autoplan-artifact-windows-argv.test.ts', 'test/autoplan-edit-digests-al.test.ts', @@ -29,15 +27,12 @@ const BASELINE = [ 'test/autoplan-pending-question.test.ts', 'test/autoplan-phase-observer.test.ts', 'test/autoplan-public-narration.test.ts', - 'test/benchmark-cli.test.ts', 'test/bootstrap-retention-shard.test.ts', // landed in v1.91.7.0 before this ratchet: harness owner for its functional-QA/docsync paid evals 'test/bootstrap-retention.test.ts', // landed in v1.91.7.0 before this ratchet: harness owner for its functional-QA/docsync paid evals 'test/bootstrap-session-lifecycle.test.ts', // landed in v1.91.7.0 before this ratchet: harness owner for its functional-QA/docsync paid evals - 'test/brain-sync-windows-paths.test.ts', 'test/build-script-shell-compat.test.ts', 'test/builder-profile.test.ts', 'test/bun-subprocess-fd-lifetime.test.ts', - 'test/bun-version-drift.test.ts', 'test/carve-guard-completeness.test.ts', 'test/carve-plan-fixture.test.ts', 'test/carve-section-ordering.test.ts', @@ -51,20 +46,14 @@ const BASELINE = [ 'test/ceo-mode-pending-submit.test.ts', 'test/ceo-mode-posture-native.test.ts', 'test/ceo-mode-prerequisite.test.ts', - 'test/ceo-mode-routing-fixture.test.ts', - 'test/ceo-plan-mode-fixture.test.ts', 'test/ceo-posture-packet.test.ts', 'test/ceo-section-loading-fixture.test.ts', 'test/ceo-split-collection.test.ts', 'test/ceo-split-question-policy.test.ts', 'test/changed-files-union.test.ts', - 'test/ci-image-cli-pin.test.ts', - 'test/ci-image-tag-binding.test.ts', 'test/claude-provider-keychain.test.ts', - 'test/code-intelligence-cli.test.ts', 'test/codex-carve-fixture.test.ts', 'test/codex-eval-recording.test.ts', - 'test/codex-model-probe.test.ts', 'test/codex-resume-flag-semantics.test.ts', 'test/cso-ntfs-fixture.test.ts', 'test/cso-windows-build-contract.test.ts', @@ -75,11 +64,7 @@ const BASELINE = [ 'test/design-html-section-completion.test.ts', 'test/devex-peer-comparison-calibration.test.ts', 'test/diagram-render-drift.test.ts', - 'test/diff-scope.test.ts', 'test/disabled-dated-record-at.test.ts', - 'test/distill-apply.test.ts', - 'test/distill-free-text.test.ts', - 'test/docs-config-keys.test.ts', 'test/docsync-atomic-writes.test.ts', // landed in v1.91.7.0 before this ratchet: harness owner for its functional-QA/docsync paid evals 'test/docsync-command-grammar.test.ts', // landed in v1.91.7.0 before this ratchet: harness owner for its functional-QA/docsync paid evals 'test/docsync-nested-writes.test.ts', // landed in v1.91.7.0 before this ratchet: harness owner for its functional-QA/docsync paid evals @@ -97,40 +82,11 @@ const BASELINE = [ 'test/eng-semantic-terminal.test.ts', 'test/eng-test-plan-edit-approval.test.ts', 'test/eval-list-cli.test.ts', - 'test/evidence.test.ts', - 'test/explain-level-config.test.ts', 'test/extension-pty-inject-invariant.test.ts', - 'test/founder-resources-optout.test.ts', - 'test/free-tests-workflow-wiring.test.ts', - 'test/gbrain-lib-validate-varname.test.ts', - 'test/gbrain-lib-verify.test.ts', - 'test/gbrain-refresh-install-render.test.ts', - 'test/gbrain-repo-policy.test.ts', 'test/gbrain-source-worktree-advance.test.ts', - 'test/gbrain-sync-skip.test.ts', 'test/gbrain-sync-voyage-code-3-integration.test.ts', - 'test/gstack-artifacts-url.test.ts', - 'test/gstack-codex-session-import.test.ts', - 'test/gstack-config-cross-project.test.ts', - 'test/gstack-config-defaults.test.ts', - 'test/gstack-config-key-locale.test.ts', - 'test/gstack-config-memorable-key.test.ts', - 'test/gstack-config-redact-keys.test.ts', - 'test/gstack-detach.test.ts', - 'test/gstack-developer-profile.test.ts', - 'test/gstack-gbrain-mcp-verify.test.ts', 'test/gstack-gbrain-source-wireup.test.ts', 'test/gstack-home-module-scope.test.ts', - 'test/gstack-learnings-search.test.ts', - 'test/gstack-question-log.test.ts', - 'test/gstack-question-preference.test.ts', - 'test/gstack-redact-cli.test.ts', - 'test/gstack-repo-mode.test.ts', - 'test/gstack-session-kind.test.ts', - 'test/gstack-slug-parity.test.ts', - 'test/gstack-slug-sanitize.test.ts', - 'test/gstack-state-root-override.test.ts', - 'test/gstack-team-init-hook-schema.test.ts', 'test/gstack-upgrade-migration-v1_17_0_0.test.ts', 'test/gstack-upgrade-migration-v1_37_0_0.test.ts', 'test/gstack-upgrade-migration-v1_40_0_0.test.ts', @@ -139,7 +95,6 @@ const BASELINE = [ 'test/helpers/budget-override.test.ts', 'test/helpers/capture-parity-baseline.test.ts', 'test/helpers/claude-pty-runner.auq.unit.test.ts', // W4: moved from claude-pty-runner.unit.test.ts (exempt via its design SKILL.md.tmpl read), split along the pty/ module seams - 'test/helpers/claude-pty-runner.classify.unit.test.ts', // W4: moved from claude-pty-runner.unit.test.ts (exempt via its design SKILL.md.tmpl read), split along the pty/ module seams 'test/helpers/claude-pty-runner.launch.unit.test.ts', // W4: moved from claude-pty-runner.unit.test.ts (exempt via its design SKILL.md.tmpl read), split along the pty/ module seams 'test/helpers/claude-pty-runner.plan-native.unit.test.ts', // W4: moved from claude-pty-runner.unit.test.ts (exempt via its design SKILL.md.tmpl read), split along the pty/ module seams 'test/helpers/claude-pty-runner.runners.unit.test.ts', // W4 fake-driver regressions: the PTY harness is itself the subject (plan-mode paid evals stand on it) @@ -152,22 +107,16 @@ const BASELINE = [ 'test/helpers/session-runner.test.ts', 'test/helpers/sync-command-capture.test.ts', 'test/hooks-windows-paths.test.ts', - 'test/hostile-path-writers.test.ts', 'test/ios-debug-bridge-release-guard.test.ts', 'test/ios-qa-stateserver-hardening.test.ts', 'test/ios-qa-swift-build.test.ts', 'test/jargon-list.test.ts', - 'test/jsonl-merge.test.ts', - 'test/learnings-injection.test.ts', 'test/llm-judge-frontier.test.ts', 'test/llm-judge-stream.test.ts', // landed in v1.91.7.0 before this ratchet: harness owner for its functional-QA/docsync paid evals 'test/memory-cache-injection.test.ts', - 'test/memory-ingest-include-gitignored.test.ts', - 'test/migrations-v1.27.0.0.test.ts', 'test/module-size-ratchet.test.ts', // ratchet (c) source scanner (W2): measures product modules as text; its counter is test tooling 'test/native-auto-decide-pty.test.ts', 'test/no-suicide-exit.test.ts', - 'test/osv-config-wiring.test.ts', 'test/outside-voice-fixture.test.ts', 'test/overlay-measurement.test.ts', 'test/overlay-recording-order.test.ts', @@ -200,19 +149,14 @@ const BASELINE = [ 'test/plan-create-combined-permission.test.ts', 'test/plan-create-permission.test.ts', 'test/plan-create-prepublication.test.ts', - 'test/plan-design-floor-fixture.test.ts', 'test/plan-edit-cropped-permission.test.ts', 'test/plan-floor-dx-actor.test.ts', 'test/plan-floor-review.test.ts', - 'test/plan-floor-target.test.ts', 'test/plan-mode-evidence.test.ts', 'test/plan-pending-question-pty.test.ts', 'test/plan-seed-submission.test.ts', 'test/plan-skill-read-permission.test.ts', 'test/plan-skill-webfetch-permission.test.ts', - 'test/plan-tune-cathedral.test.ts', - 'test/pr-title-rewrite.test.ts', - 'test/pr-title-sync-workflow-safety.test.ts', 'test/pty-askuserquestion-single-line.test.ts', 'test/pty-numbered-option-indent-native.test.ts', 'test/pty-option-selection.test.ts', @@ -229,12 +173,9 @@ const BASELINE = [ 'test/qa-functional-fixture.test.ts', // landed in v1.91.7.0 before this ratchet: harness owner for its functional-QA/docsync paid evals 'test/qa-functional-observer-atomic.test.ts', // landed in v1.91.7.0 before this ratchet: harness owner for its functional-QA/docsync paid evals 'test/qa-functional-observer.test.ts', // landed in v1.91.7.0 before this ratchet: harness owner for its functional-QA/docsync paid evals - 'test/qa-only-capability.test.ts', 'test/qa-supervision-selection.test.ts', // landed in v1.91.7.0 before this ratchet: harness owner for its functional-QA/docsync paid evals 'test/question-log-hook.test.ts', 'test/readme-throughput.test.ts', - 'test/redact-prepush-rebase-force-push.test.ts', - 'test/redact-prepush-scan-range.test.ts', 'test/review-army-budget.test.ts', 'test/review-consensus-lifecycle.test.ts', 'test/review-count-markdown.test.ts', @@ -246,29 +187,17 @@ const BASELINE = [ 'test/setup-browser-hint.test.ts', 'test/setup-bun-cmd-and-pipe-bugs.test.ts', 'test/setup-conductor-worktree.test.ts', - 'test/setup-plan-tune-hooks-noninteractive.test.ts', 'test/setup-playwright-platform.test.ts', 'test/setup-sections-linking.test.ts', - 'test/setup-timeline-hook-gate.test.ts', 'test/shared-libs-cancellation.test.ts', // landed in v1.91.7.0 before this ratchet: harness owner for its functional-QA/docsync paid evals 'test/shared-libs-plan-actor.test.ts', - 'test/shared-libs-revalidation-prompt.test.ts', - 'test/shared-libs-source-reads.test.ts', - 'test/ship-coverage-audit-af.test.ts', - 'test/ship-pr-liveness-policy.test.ts', 'test/ship-section-fixture.test.ts', 'test/skill-budget-regression.test.ts', 'test/skill-parser.test.ts', 'test/spawnsync-timeout-tripwire.test.ts', 'test/tasks-section-jq.test.ts', - 'test/taste-engine.test.ts', - 'test/telemetry-repo-strip.test.ts', 'test/test-free-shards-capture.test.ts', 'test/timeline.test.ts', - 'test/ubicloud-runner.test.ts', - 'test/update-check-crash-sentinel.test.ts', - 'test/verify-gate.test.ts', - 'test/workflow-concurrency.test.ts', ]; function freeTests(): string[] { @@ -280,7 +209,8 @@ function testsOnlyTestCode(file: string): { counted: boolean; helper?: string } const source = fs.readFileSync(path.join(ROOT, file), 'utf8'); const targets = directSpecifiers(source).map(specifier => resolveRepoSpecifier(ROOT, file, specifier)).filter(Boolean) as string[]; if (targets.some(target => !target.startsWith('test/') && !target.startsWith('node_modules/'))) return { counted: false }; - if (/(['"`])[^'"`\n]*(?:\bbin\/|SKILL\.md|\.tmpl)[^'"`\n]*\1/.test(source)) return { counted: false }; + if (/(['"`])[^'"`\n]*(?:\bbin\/|SKILL\.md|CLAUDE\.md|\.tmpl)[^'"`\n]*\1/.test(source)) return { counted: false }; + if (/(['"`])(?:\.\/)?\.github\b[^'"`\n]*\1|(['"`])bin\2\s*,\s*(['"`])[\w.-]+\3/.test(source)) return { counted: false }; return { counted: true, helper: targets.find(target => target.startsWith('test/helpers/') && !target.endsWith('.test.ts')) }; }