diff --git a/scripts/pr-dependencies.ts b/scripts/pr-dependencies.ts index a6dc94e3a..077e01f71 100644 --- a/scripts/pr-dependencies.ts +++ b/scripts/pr-dependencies.ts @@ -90,6 +90,8 @@ export const SELECTION_DATA_MODULES: readonly string[] = [ 'scripts/build.sh', // The reason-code table: its `fix:` prose names the command a person reruns, not a file the module consumes. 'lib/result-codes.ts', 'lib/result-codes-release.ts', + // The exercised-path map: observed commands per case, read only by exercisedConsumers() (shadow data; a change selects no paid case). + 'test/helpers/exercised-commands.json', ]; const DEFAULT_JUDGE_OWNER = 'test/skill-llm-eval.test.ts'; @@ -326,3 +328,295 @@ export function derivedDependencies(maps: DependencyMaps, root: string): Derived return cache.get(key)!; } +// ─── Exercised-path consumer map (W2-A2, shadow mode) ────────────────────── + +/** The committed per-case observed commands (generated by scripts/exercised-map.ts; union-only refreshes). */ +export const EXERCISED_MAP_PATH = 'test/helpers/exercised-commands.json'; + +export interface ExercisedCase { + /** Recorded trials (run, case) the commands were taken from. */ + observations: number; + /** Of those, trials that recorded at least one command. */ + withCommands: number; + /** Bins and compiled outfiles the case invoked (binsInCommand over every recorded command). */ + commands: string[]; + /** Mean known trial cost per run, for the shadow cost estimate. */ + costUsd?: number; + costRuns?: number; +} + +export interface ExercisedMap { + schema: 1; + generatedAt: string; + observedThrough: string; + sourceRuns: string[]; + cases: Record; +} + +export function isExercisedMap(value: unknown): value is ExercisedMap { + const map = value as ExercisedMap; + return !!map && map.schema === 1 && typeof map.generatedAt === 'string' && typeof map.observedThrough === 'string' + && Array.isArray(map.sourceRuns) && map.sourceRuns.every(run => typeof run === 'string') + && !!map.cases && typeof map.cases === 'object' && Object.values(map.cases).every(entry => !!entry + && Number.isSafeInteger(entry.observations) && Number.isSafeInteger(entry.withCommands) + && Array.isArray(entry.commands) && entry.commands.every(command => typeof command === 'string')); +} + +/** Parse an exercised map's text; throws on malformed input so callers fail closed. */ +export function parseExercisedMap(text: string): ExercisedMap { + const parsed = JSON.parse(text); + if (!isExercisedMap(parsed)) throw new Error(`${EXERCISED_MAP_PATH}: not a schema-1 exercised map`); + return parsed; +} + +export interface ExercisedConsumers { + /** Repository file -> case ids whose exercised closure contains it. */ + e2e: Map>; + /** One case's exercised closure: file -> how it is reached (touchfile, import, skill bash via , observed , declared via , host). */ + closure(caseId: string): Map; + /** Bins (repo paths) and compiled outfiles one recorded command invokes. */ + binsInCommand(command: string): string[]; + /** The tracked files a bin or compiled outfile runs. */ + commandFiles(command: string): string[]; + /** The observed map the consumers were built with. */ + map: ExercisedMap | null; +} + +const SHELL_SHEBANG = /^#![^\n]*\b(?:ba|z|da)?sh\b/; +const MESSAGE_CALL = /^(?:echo|printf|say|warn|die|fail|err|log|info|note|_dc_row|row)\b/; +const CMD_START = /(?:^|[;|&({\x60]|\$\(|\b(?:then|do|else|exec|command|nohup|bash|sh|bun|node|timeout\s+\S+|env(?:\s+\w+=\S*)*|if|while|until|!|time))\s*"?$/; +const CMD_PREFIX = /(?:"?(?:\$\{?\w+\}?|\$\(dirname\s+"?\$0"?\)|~|\.)?(?:\/(?:[\w.@-]+\/)*|(?:[\w.@-]+\/)+))?"?$/; +const INTERPRETER = /\b(?:bash|sh|zsh|bun|node|eval|source|python3?)\b/; + +/** Text between `$(`…`)` and backticks in a span: substitutions run even inside a message. */ +function substitutions(span: string): string[] { + const out: string[] = []; + for (let i = 0; i < span.length; i++) { + if (span[i] === '\\') { i++; continue; } + if (span[i] === '\x60') { const end = span.indexOf('\x60', i + 1); if (end < 0) break; out.push(`$(${span.slice(i + 1, end)})`); i = end; continue; } + if (span[i] !== '$' || span[i + 1] !== '(') continue; + let depth = 0, j = i + 1; + for (; j < span.length; j++) { if (span[j] === '(') depth++; else if (span[j] === ')' && --depth === 0) break; } + out.push(span.slice(i, j + 1)); + i = j; + } + return out; +} + +/** Message calls (echo, printf, die …) are prose; their command substitutions still execute and are kept. */ +function stripMessages(text: string): string { + let out = '', i = 0; + while (i < text.length) { + const prev = i === 0 ? '\n' : text[i - 1]!; + const call = /[\s;|&(\x60{]/.test(prev) ? MESSAGE_CALL.exec(text.slice(i, i + 8)) : null; + if (!call) { out += text[i]; i++; continue; } + let j = i + call[0].length, quote: string | null = null, depth = 0; + while (j < text.length) { + const c = text[j]!; + if (c === '\\') { j += 2; continue; } + if (quote) { if (c === quote) quote = null; else if (quote === '"' && c === '$' && text[j + 1] === '(') { depth++; j++; } else if (c === ')' && depth) depth--; j++; continue; } + if (c === '"' || c === "'") { quote = c; j++; continue; } + if (c === '$' && text[j + 1] === '(') { depth++; j += 2; continue; } + if (c === ')' && depth) { depth--; j++; continue; } + if (c === '\n' || c === ';' || c === '|' || c === '&' || c === ')') break; + j++; + } + out += ` ${substitutions(text.slice(i, j)).join('; ')} `; + i = j; + } + return out; +} + +/** + * Executable shell text: comments, case patterns and message calls are + * prose. A heredoc an interpreter reads stays whole; an expanding (unquoted) + * heredoc keeps its command substitutions; a quoted one fed to anything else + * is dropped. + */ +export function executableShell(text: string): string { + return stripMessages(text + .replace(/^(.*?)<<-?\s*(['"]?)(\w+)\2([^\n]*)\n([\s\S]*?)\n\s*\3\s*$/gm, (whole, head: string, quote: string, _word: string, rest: string, body: string) => + INTERPRETER.test(head + rest) ? whole : quote ? head : [head, ...substitutions(body)].join('\n')) + .split('\n').filter(line => !/^\s*#(?!!)/.test(line)) + .map(line => line.replace(/(^\s*|\bin\s+|;;\s*)[\w*."'-]+(?:\s*\|\s*[\w*."'-]+)*\)(?=\s|$)/g, '$1').replace(/\s#\s.*$/, '')) + .join('\n')); +} + +function commandPosition(before: string): boolean { + const head = before.replace(CMD_PREFIX, ''); + return CMD_START.test(head) || (/\/$/.test(before) && /\b\w+="?$/.test(head)); +} + +/** + * Consumers by exercised paths: a case consumes its touchfile matches, the + * closure of its test file, the executable shell of its loaded skill + * (template and generated SKILL.md) and of every bin, lib and script that + * shell runs, and the bins its recorded trials invoked (`observed`). The + * closure stops at `stopAt` (the full-gate files) and the global set, + * which stay full. + */ +export function exercisedConsumers(maps: DependencyMaps, root: string, options: { observed: ExercisedMap | null; stopAt: readonly string[] }): ExercisedConsumers { + const base = derivedDependencies(maps, root); + const tracked = [...base.tracked]; + const texts = new Map(); + const read = (file: string): string => { + if (!texts.has(file)) { + let text = ''; + try { if (base.tracked.has(file) && fs.statSync(path.join(root, file)).size < 3_000_000) text = fs.readFileSync(path.join(root, file), 'utf8'); } catch { /* unreadable */ } + texts.set(file, text); + } + return texts.get(file)!; + }; + const isShell = (file: string) => !isCodeFile(file, read(file)) && (/\.sh$/.test(file) || file === 'setup' || SHELL_SHEBANG.test(read(file))); + const dirs = new Set(tracked.flatMap(file => file.split('/').slice(0, -1).map((_, i, parts) => parts.slice(0, i + 1).join('/')))); + const expandRef = (ref: string): string[] => { + const clean = ref.replace(/[.]+$/, ''); + if (base.tracked.has(clean)) return [clean]; + return dirs.has(clean) ? tracked.filter(file => file.startsWith(`${clean}/`)) : []; + }; + const expanded = new Map(); + const expand = (pattern: string): string[] => { + if (!pattern.includes('*')) return base.tracked.has(pattern) ? [pattern] : []; + if (!expanded.has(pattern)) expanded.set(pattern, tracked.filter(file => matchGlob(file, pattern))); + return expanded.get(pattern)!; + }; + const binByName = new Map(); + for (const file of tracked.filter(file => file.startsWith('bin/') || file.startsWith('browse/bin/'))) { + binByName.set(path.posix.basename(file), file); + binByName.set(path.posix.basename(file).replace(/\.(?:ts|sh|js)$/, ''), file); + } + const binaries = [...read('scripts/build.sh').matchAll(/--compile\b[^\n]*?\s([\w./-]+\.ts)\s+--outfile\s+([\w./-]+)/g)] + .map(match => ({ outfile: match[2]!, tree: match[1]!.startsWith('bin/') ? [match[1]!] : tracked.filter(file => matchGlob(file, `${path.posix.dirname(match[1]!)}/**`)) })); + const outfileHit = (outfile: string, text: string) => text.includes(outfile) + || (outfile === 'browse/dist/browse' && /\$B\b|\$\{B\}|find-browse/.test(text)) || (outfile === 'design/dist/design' && /\$D\b|\$\{D\}/.test(text)); + let scripts: Record = {}; + try { scripts = JSON.parse(read('package.json')).scripts ?? {}; } catch { /* no package scripts */ } + + const shellEdges = (text: string, from: string, out: Set) => { + for (const match of text.matchAll(/gstack-[a-z0-9][a-z0-9-]*[a-z0-9](?:\.(?:sh|ts))?\b/g)) { + const start = text.lastIndexOf('\n', match.index! - 1) + 1; + const bin = binByName.get(match[0]); + if (bin && commandPosition(text.slice(start, match.index))) out.add(bin); + } + for (const match of text.matchAll(/(?:bin|lib|scripts|browse|design|make-pdf)\/[\w.@-]+(?:\/[\w.@-]+)*/g)) { + const before = text.slice(text.lastIndexOf('\n', match.index! - 1) + 1, match.index); + if (/[\w.@-]$/.test(before)) continue; + if (commandPosition(before) || /\bbun\s+(?:run\s+)?"?(?:\$\{?\w+\}?\/)?$/.test(before)) for (const file of expandRef(match[0])) out.add(file); + } + for (const match of text.matchAll(/(?:bin|lib|scripts|browse|design|make-pdf|hosts|capy|test\/fixtures|test\/helpers)\/[\w.@-]+(?:\/[\w.@-]+)*\.(?:json|jsonl|env|txt|ya?ml|toml|csv|tsv)\b/g)) for (const file of expandRef(match[0])) out.add(file); + for (const match of text.matchAll(/(?:^|[\s;&|({])(?:\.|source)\s+([^;|&\n]+)/gm)) { + for (const ref of match[1]!.matchAll(/(?:bin|lib|scripts)\/[\w.@-]+(?:\/[\w.@-]+)*\.sh\b/g)) for (const file of expandRef(ref[0])) out.add(file); + const name = [...match[1]!.matchAll(/([\w.-]+\.sh|gstack-[\w.-]+)/g)].map(found => found[1]!).at(-1); + if (!name) continue; + const sibling = path.posix.join(path.posix.dirname(from), name); + if (base.tracked.has(sibling)) out.add(sibling); + else if (binByName.has(name)) out.add(binByName.get(name)!); + } + if (/(?:^|[\s;&|(])"?(?:\.|\$\{?\w+\}?)\/setup\b/m.test(text) && base.tracked.has('setup')) out.add('setup'); + for (const { outfile, tree } of binaries) if (outfileHit(outfile, text)) for (const file of tree) out.add(file); + for (const match of text.matchAll(PACKAGE_RUN)) { + const command = scripts[match[1]!]; + if (command) for (const ref of command.matchAll(REPO_PATH)) for (const file of expandRef(ref[1]!)) out.add(file); + } + }; + const edges = new Map(); + const execRefs = (file: string): string[] => { + if (edges.has(file)) return edges.get(file)!; + const raw = read(file); + const out = new Set(); + if (raw && !(SELECTION_DATA_MODULES as readonly string[]).includes(file)) { + if (isCodeFile(file, raw)) { + try { for (const specifier of scanCodeImports(file, raw)) { const hit = resolveSpecifier(root, specifier, file); if (hit) out.add(hit); } } + catch (error) { base.scanErrors.set(file, String(error)); } + const code = raw.replace(/^#![^\n]*/, '').replace(/\/\*[\s\S]*?\*\//g, '').replace(/^[ \t]*\/\/.*$/gm, ''); + for (const match of code.matchAll(/['"\x60](?:\$\{[^}]+\}\/)?((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)['"\x60]/g)) for (const hit of expandRef(match[1]!)) out.add(hit); + for (const match of code.matchAll(JOINED_PATH)) for (const hit of expandRef(`${match[1]}/${match[2]}`)) out.add(hit); + if (/join\([^)\n]*['"]setup['"]\s*\)/.test(code) && base.tracked.has('setup')) out.add('setup'); + for (const { outfile, tree } of binaries) if (code.includes(`'${outfile}'`) || code.includes(`"${outfile}"`)) for (const hit of tree) out.add(hit); + } else if (isShell(file)) { + shellEdges(executableShell(raw), file, out); + } else if (/\.(?:md|tmpl)$/.test(file)) { + shellEdges(executableShell([...raw.matchAll(/\x60\x60\x60(?:bash|sh|shell)\n([\s\S]*?)\x60\x60\x60/g)].map(match => match[1]!).join('\n')), file, out); + } + } + out.delete(file); + edges.set(file, [...out]); + return edges.get(file)!; + }; + const stopped = new Map(); + const stops = (file: string): boolean => { + if (!stopped.has(file)) stopped.set(file, base.global.has(file) || options.stopAt.some(pattern => matchGlob(file, pattern))); + return stopped.get(file)!; + }; + const binsInCommand = (command: string): string[] => { + const out = new Set(); + const pty = command.startsWith('PTY:'); + for (const match of command.matchAll(/gstack-[a-z0-9-]*[a-z0-9](?:\.(?:sh|ts))?/g)) { + let token = match[0]; + if (pty) while (token.length > 7 && !binByName.has(token)) { const cut = token.lastIndexOf('-'); token = cut > 6 ? token.slice(0, cut) : ''; } + const at = match.index!; + if (command[at - 1] === '/' && at >= 2 && /[\s"'\x60(]/.test(command[at - 2]!)) continue; + const bin = binByName.get(token); + if (bin) out.add(bin); + } + for (const match of command.matchAll(/browse\/bin\/([a-z-]+)/g)) { const bin = binByName.get(match[1]!); if (bin) out.add(bin); } + for (const { outfile } of binaries) if (outfileHit(outfile, command) || (outfile === 'make-pdf/dist/pdf' && command.includes(outfile))) out.add(outfile); + return [...out]; + }; + const commandFiles = (command: string) => base.tracked.has(command) ? [command] : binaries.find(binary => binary.outfile === command)?.tree ?? []; + + const hostIds = tracked.filter(file => /^hosts\/[\w-]+\.ts$/.test(file) && file !== 'hosts/index.ts').map(file => path.posix.basename(file, '.ts')); + const closures = new Map>(); + const caseClosure = (id: string): Map => { + if (closures.has(id)) return closures.get(id)!; + const patterns = maps.e2eTouchfiles[id] ?? []; + const owners = patterns.filter(pattern => OWNER.test(pattern) && isPaidTestFile(pattern)); + const declared = patterns.flatMap(expand); + const generated = declared.filter(file => file.endsWith('.tmpl')).map(file => file.replace(/\.tmpl$/, '')).filter(file => base.tracked.has(file)); + const touchfiles = new Set([...declared, ...generated]); + const skillDoc = (file: string) => /\.(?:md|tmpl)$/.test(file); + const seeds: Array<[string, string]> = [ + ...owners.map(file => [file, 'import'] as [string, string]), + ...[...declared, ...generated].map(file => [file, skillDoc(file) ? 'skill' : `declared via ${file}`] as [string, string]), + ...(options.observed?.cases[id]?.commands ?? []).flatMap(command => commandFiles(command).map(file => [file, `observed ${command}`] as [string, string])), + ]; + const ownerText = owners.map(read).join('\n'); + for (const host of hostIds) { + if (new RegExp(`['"\x60]${host}['"\x60]|--host[ =]${host}\\b`).test(ownerText)) for (const file of [`hosts/${host}.ts`, ...expand(`hosts/${host}/**`)]) seeds.push([file, 'host']); + } + const reached = new Map(); + const queue = seeds.filter(([file]) => base.tracked.has(file)); + for (let head = 0; head < queue.length; head++) { + const [file, label] = queue[head]!; + if (reached.has(file)) continue; + reached.set(file, touchfiles.has(file) ? 'touchfile' : label); + if (stops(file)) continue; + for (const next of execRefs(file)) { + if (reached.has(next) || (/\.test\.tsx?$/.test(next) && isPaidTestFile(next))) continue; + queue.push([next, label === 'skill' ? `skill bash via ${next}` : label]); + } + } + closures.set(id, reached); + return reached; + }; + const e2e = new Map>(); + for (const id of Object.keys(maps.e2eTouchfiles)) { + for (const file of caseClosure(id).keys()) { + if (!e2e.has(file)) e2e.set(file, new Set()); + e2e.get(file)!.add(id); + } + } + return { e2e, closure: caseClosure, binsInCommand, commandFiles, map: options.observed }; +} + +const exercisedCache = new Map(); + +/** The exercised consumers of the checkout at `root` with its committed map, computed once per process; throws when the map is malformed. */ +export function checkoutExercisedConsumers(maps: DependencyMaps, root: string, stopAt: readonly string[]): ExercisedConsumers { + const key = `${root}\0${Object.keys(maps.e2eTouchfiles).length}`; + if (!exercisedCache.has(key)) { + const observed = parseExercisedMap(fs.readFileSync(path.join(root, EXERCISED_MAP_PATH), 'utf8')); + exercisedCache.set(key, exercisedConsumers(maps, root, { observed, stopAt })); + } + return exercisedCache.get(key)!; +} diff --git a/scripts/test-pr-profile.ts b/scripts/test-pr-profile.ts index 52c325e18..21e9fed3b 100644 --- a/scripts/test-pr-profile.ts +++ b/scripts/test-pr-profile.ts @@ -5,7 +5,9 @@ import { E2E_TOUCHFILES, E2E_TIERS, GLOBAL_TOUCHFILES, LLM_JUDGE_TOUCHFILES } fr import { matchGlob, TOUCHFILES_DATA_PATH } from '../test/helpers/test-selection'; import { isPaidTestFile } from '../test/helpers/paid-test-set'; import { FREE_FIXTURES } from '../test/helpers/free-fixtures-data'; -import { derivedDependencies, type DerivedDependencies } from './pr-dependencies'; +import { + checkoutExercisedConsumers, derivedDependencies, EXERCISED_MAP_PATH, type DerivedDependencies, type ExercisedConsumers, +} from './pr-dependencies'; /** Existing short behavioral probes; intersect with changed-input selection. */ export const PR_PROFILE_CASE_IDS = [ @@ -157,6 +159,27 @@ export interface PrProfileSelection { missingCoverage: string[]; needsFullValidation: boolean; reasons: string[]; + /** What the exercised-path map would select (shadow mode: never changes e2e or judges). */ + shadow?: ShadowSelection; +} + +/** The exercised-path map's selection beside the real one (docs/TESTING_INTERNALS.md#exercised-path-map). */ +export interface ShadowSelection { + /** Gate cases the map would select. */ + e2e: string[]; + /** Selected gate cases the map would not select. */ + wouldDrop: string[]; + /** Gate cases the map would select that today's selection does not. */ + wouldAdd: string[]; + /** Per changed file with exercised consumers: how many gate cases it reaches, and through which edge. */ + provenance: Array<{ file: string; summary: string }>; + /** Mean known cost of the dropped cases minus the added ones (the map's per-case costUsd; an unpriced case counts $0): a lower bound. */ + predictedSavingUsd: number; + unpricedCases: number; + /** Share of observed case runs whose trials carry no known cost (PTY trials); their spend is missing from every dollar figure. */ + unknownCostShare: number; + /** Why no shadow could be computed (the real selection is unaffected). */ + error?: string; } /** Reject stale profile registrations and deferred cases with no scheduled home. */ @@ -296,6 +319,7 @@ export function knownNonBehaviorFile(file: string): boolean { } export const FALLBACK_FIX_ANCHOR = 'docs/TESTING_INTERNALS.md#pr-paid-lane-fallback'; +export const EXERCISED_MAP_ANCHOR = 'docs/TESTING_INTERNALS.md#exercised-path-map'; /** Why one changed file restored the full gate, and the edit that narrows it next time (DX-6). */ export function unknownFileLabel(file: string): { label: string; fix: string } { @@ -347,6 +371,8 @@ export function selectPrProfile(options: { readSource?: (file: string) => string; /** Derived references; default: derived from the checkout for the real maps, none for fixture maps. */ derived?: DerivedDependencies | null; + /** Exercised-path consumers for the shadow block; default: the checkout's committed map for the real maps, none for fixture maps. */ + exercised?: ExercisedConsumers | null; }): PrProfileSelection { const maps = options.maps ?? PR_PROFILE_MAPS; const profile = options.profile ?? PR_PROFILE_CASE_IDS; @@ -360,7 +386,7 @@ export function selectPrProfile(options: { ...Object.values(maps.e2eTouchfiles).flat(), ...Object.values(maps.judgeTouchfiles).flat(), ...maps.globalTouchfiles, ])]; - const unmapped = files.filter(file => file !== TOUCHFILES_DATA_PATH + const unmapped = files.filter(file => file !== TOUCHFILES_DATA_PATH && file !== EXERCISED_MAP_PATH && !depends(file, dependencyPatterns) && !knownNonBehaviorFile(file)); // Unmapped files are placed by derived references where the derivation covers them; the rest restore the full gate. const derived = options.derived !== undefined ? options.derived : maps === PR_PROFILE_MAPS ? derivedDependencies(maps, ROOT) : null; @@ -437,25 +463,80 @@ export function selectPrProfile(options: { if (sharedInputs.length) reasons.push(`Shared runtime/build inputs restore every gate case and judge: ${sharedInputs.join(', ')}`); if (dependentsMode) reasons.push(`Derived references select every dependent gate case: ${derivedFiles.map(file => `${file} (${consumers.get(file)!.e2e.size} cases)`).join(', ')}`); else if (!fallback) reasons.push('Changed-input selection intersected with the fast PR profile; selected judges retained'); + if (files.includes(EXERCISED_MAP_PATH)) reasons.push(`${EXERCISED_MAP_PATH} is shadow-only data (it changes no selection yet), so it selects no paid case (${EXERCISED_MAP_ANCHOR})`); if (derivedGlobal.length) reasons.push(`Paid-runner inputs (global touchfile import closure) select every case: ${derivedGlobal.join(', ')}`); if (noConsumerFiles.length) reasons.push(`No paid case's reference closure reaches (or, deleted, still references): ${noConsumerFiles.join(', ')}`); if (directCases.length) reasons.push(`Edited paid test files select their own gate cases: ${directCases.join(', ')}`); if (deferred.length) reasons.push(`${deferred.length} selected behaviors remain scheduled/release coverage, not PR passes`); if (deferredPromptFiles.length) reasons.push(`No quick live coverage; known broad prompt checks deferred: ${deferredPromptFiles.join(', ')}`); if (missingCoverage.length) reasons.push(`Full validation required for prompts without a relevant PR check: ${missingCoverage.join(', ')}`); + const exercised = options.exercised !== undefined ? options.exercised : maps === PR_PROFILE_MAPS ? checkoutExercised() : null; + const shadow = exercised instanceof Error ? emptyShadow(e2e, exercised.message) + : exercised ? shadowSelection({ ...options, maps, profile }, { e2e, fallback, derived, exercised, files }) : undefined; return { mode: fallback ? 'full-fallback' : dependentsMode ? 'dependents' : 'pr', e2e, judges, deferred, unknownFiles, unknownFileLabels: unknownFiles.map(file => ({ file, ...(deletedLabels.get(file) ?? unknownFileLabel(file)) })), directCases, derivedFiles: derivedFiles.map(file => ({ file, e2e: consumers.get(file)!.e2e.size, judges: consumers.get(file)!.judges.size })), noConsumerFiles, - deferredPromptFiles, missingCoverage, needsFullValidation: missingCoverage.length > 0, reasons, + deferredPromptFiles, missingCoverage, needsFullValidation: missingCoverage.length > 0, reasons, ...(shadow ? { shadow } : {}), }; } +function checkoutExercised(): ExercisedConsumers | Error { + try { return checkoutExercisedConsumers(PR_PROFILE_MAPS, ROOT, PR_FULL_GATE_FILES); } + catch (error) { return error instanceof Error ? error : new Error(String(error)); } +} + +function emptyShadow(e2e: string[], error: string): ShadowSelection { + return { e2e, wouldDrop: [], wouldAdd: [], provenance: [], predictedSavingUsd: 0, unpricedCases: 0, unknownCostShare: 0, error }; +} + +/** + * Shadow selection (variant cu of the measured prototype): the same policy + * with the exercised map in place of the derived closure, unioned with the + * exercised consumers of every changed file that is not global or full-gate. + * A full-gate fallback stays full. + */ +function shadowSelection(options: Parameters[0] & { maps: PrProfileMaps }, + today: { e2e: string[]; fallback: boolean; derived: DerivedDependencies | null; exercised: ExercisedConsumers; files: string[] }): ShadowSelection { + const { maps } = options; + const { exercised } = today; + const observedCases = Object.values(exercised.map?.cases ?? {}); + const observations = observedCases.reduce((sum, entry) => sum + entry.observations, 0); + const unknownCostShare = observations ? Math.round(1000 * (1 - observedCases.reduce((sum, entry) => sum + (entry.costRuns ?? 0), 0) / observations)) / 1000 : 0; + if (today.fallback) return { e2e: today.e2e, wouldDrop: [], wouldAdd: [], provenance: [], predictedSavingUsd: 0, unpricedCases: 0, unknownCostShare }; + const gate = (ids: Iterable) => [...ids].filter(id => maps.tiers[id] === 'gate'); + const reach = new Map(); + const inner = selectPrProfile({ ...options, derived: today.derived ? { ...today.derived, e2e: exercised.e2e } : null, exercised: null }); + const sources = [...new Set([...today.files, ...Object.values(options.sourceAliases ?? {})])]; + const shadowSet = new Set(inner.e2e); + for (const file of sources) { + if (today.derived?.global.has(file) || matches(file, PR_FULL_GATE_FILES) || knownNonBehaviorFile(file)) continue; + const via = !today.derived || today.derived.tracked.has(file) ? [file] : today.derived.referencers(file); + const ids = gate(new Set(via.flatMap(source => [...exercised.e2e.get(source) ?? []]))); + if (ids.length) reach.set(file, ids); + for (const id of ids) shadowSet.add(id); + } + const e2e = [...shadowSet].sort(); + const wouldDrop = today.e2e.filter(id => !shadowSet.has(id)); + const wouldAdd = e2e.filter(id => !today.e2e.includes(id)); + const cost = (ids: string[]) => ids.reduce((sum, id) => sum + (exercised.map?.cases[id]?.costUsd ?? 0), 0); + const provenance = [...reach].map(([file, ids]) => { + const groups = new Map(); + for (const id of ids) { + const label = exercised.closure(id).get(file) ?? 'referencer'; + groups.set(label, (groups.get(label) ?? 0) + 1); + } + return { file, summary: `selects ${ids.length} case(s): ${[...groups].sort((a, b) => b[1] - a[1]).slice(0, 6).map(([label, n]) => `${label} (${n})`).join(', ')}` }; + }); + return { e2e, wouldDrop, wouldAdd, provenance, predictedSavingUsd: Math.round((cost(wouldDrop) - cost(wouldAdd)) * 100) / 100, + unpricedCases: [...wouldDrop, ...wouldAdd].filter(id => exercised.map?.cases[id]?.costUsd === undefined).length, unknownCostShare }; +} + /** The manifest fields the coverage summary reads (a PaidRunManifest subset). */ export interface CoverageSummaryInput { profile?: string; selection?: { e2e: string[] | null; judges: string[] | null }; - prCoverage?: Pick & Partial>; + prCoverage?: Pick & Partial>; } /** @@ -476,6 +557,9 @@ export function formatPrCoverageSummary(manifest: CoverageSummaryInput, totals?: lines.push(`- Derived dependents (scripts/pr-dependencies.ts) of ${coverage.derivedFiles.length} unmapped file(s):`); for (const item of coverage.derivedFiles.slice(0, 40)) lines.push(` - \`${item.file}\`: ${item.e2e} case(s), ${item.judges} judge(s)`); } + const shadow = coverage?.shadow; + if (shadow) lines.push(shadow.error ? `- Shadow exercised-path map: unavailable (${shadow.error}); selection unaffected` + : `- Shadow exercised-path map (not active): would select ${shadow.e2e.length} gate case(s), dropping ${shadow.wouldDrop.length} and adding ${shadow.wouldAdd.length}; predicted saving $${shadow.predictedSavingUsd.toFixed(2)}${shadow.unpricedCases ? ` (${shadow.unpricedCases} unpriced)` : ''}, a lower bound: known cost excludes the ${Math.round(100 * (shadow.unknownCostShare ?? 0))}% of observed case runs with unknown cost (PTY) (${EXERCISED_MAP_ANCHOR})`); if (coverage?.noConsumerFiles?.length) lines.push(`- No paid case reaches: ${coverage.noConsumerFiles.slice(0, 40).map(file => `\`${file}\``).join(', ')}`); if (coverage?.mode === 'full-fallback' && coverage.unknownFiles.length) { lines.push(`- Full gate restored by ${coverage.unknownFiles.length} file(s) (fix: ${FALLBACK_FIX_ANCHOR}):`); diff --git a/test/helpers/exercised-commands.json b/test/helpers/exercised-commands.json new file mode 100644 index 000000000..e2647ee8a --- /dev/null +++ b/test/helpers/exercised-commands.json @@ -0,0 +1,1905 @@ +{ + "schema": 1, + "generatedAt": "2026-10-17T19:29:23.672Z", + "observedThrough": "2026-10-08T06:50:33Z", + "sourceRuns": [ + "36205327552", + "36220963292", + "36278514291", + "36302678692", + "36315459968", + "36318630639", + "36391800575", + "36465804057", + "36468274063", + "36481155145", + "36481295477", + "36485436963", + "36491692039", + "36493452542", + "36493616568", + "36497566037", + "36505065023", + "36516246523", + "36521598255", + "36526785991", + "36537184166", + "36539603295", + "36551562564", + "36579289566", + "36589496678", + "36615456309", + "36620926989", + "36636909400", + "36637107473", + "36637661593", + "36638638110", + "36640582052", + "36640919692", + "36641031019", + "36641547412", + "36641801658", + "36641824710", + "36670216491", + "36682017584", + "36683045674", + "36685469932", + "36709485593", + "36711269409", + "36711836168", + "36713227509", + "36713549527", + "36714493404", + "36717552105", + "36717834406", + "36718311034", + "36718368006", + "36718436969", + "36718514685", + "36719119326", + "36720872161", + "36722276798", + "36725680777", + "36730053861", + "36730220242", + "36740979126", + "36741522675", + "36743390793", + "36745049965", + "36753939836", + "36755432181", + "36758397400", + "36760073893", + "36761671799", + "36762284181", + "36764370405", + "36768263333", + "36768400454", + "36770061511", + "36774149885", + "36775002132", + "36775932413", + "36776096754", + "36777335875", + "36781170544", + "36783214499", + "36783749239", + "36786871824", + "36787264898", + "36790850100", + "36793107556", + "36794197606", + "36794871032", + "36797095286", + "36797191931", + "36797673165", + "36798542003", + "36823310593", + "36834605576", + "36897642796", + "36898539571", + "36899642230", + "36903605989", + "36905008393", + "36907899270", + "36914699033", + "36914734297", + "36917041210", + "36920579887", + "36922422698", + "36965719520", + "37024904147", + "37025330680", + "37026118550", + "37033755821", + "37060128357", + "37060776291", + "37061202449", + "37062583804", + "37063836999", + "37064151421", + "37066055701", + "37070126890", + "37073429415", + "37077373640", + "37080729942", + "37082906717", + "37087797888", + "37090604134", + "37090925543", + "37091093956", + "37091743395", + "37092497208", + "37092677055", + "37093937027", + "37094035231", + "37097119454", + "37097590162", + "37098585955", + "37099423845", + "37100121821", + "37111582659", + "37116024111", + "37143137677", + "37146354490", + "37152219461", + "37153488058", + "37157517739", + "37158193827", + "37158662432", + "37160557715", + "37161546382", + "37162342436", + "37162847235", + "37163124276", + "37163226077", + "37163388183", + "37163438154", + "37163862654", + "37163935292", + "37164023690", + "37164663577", + "37164967731", + "37165022930", + "37166243626", + "37166586458", + "37168384066", + "37168819307", + "37170258467", + "37170610789", + "37172266374", + "37172907071", + "37173609560", + "37176237665", + "37176835584", + "37177847549", + "37178146922", + "37178720668", + "37179168735", + "37181717177", + "37182208158", + "37184467123", + "37186091144", + "37186918046", + "37188014112", + "37188668421", + "37189974078", + "37192833320", + "37194511147", + "37196784022", + "37196811165", + "37196900729", + "37197708438", + "37199579960", + "37199589136", + "37228201907", + "37228564701", + "37230295724", + "37230639146", + "37230870756", + "37232374839", + "37233114987", + "37234681021", + "37234978002", + "37235771700", + "37237194905", + "37238469830", + "37239016858", + "37241493377", + "37242691913", + "37246213438", + "37246668414", + "37248168494", + "37248388404", + "37248650879", + "37250221023", + "37250545914", + "37250645386", + "37251015146", + "37251229631", + "37251358164", + "37251850439", + "37252022510", + "37252831019", + "37253679118", + "37254021038", + "37256076805", + "37256265663", + "37265273012", + "37266174921", + "37266371726", + "37278830943", + "37296674175", + "37324177668", + "37324423488", + "37327038121", + "37346036236", + "37346541110", + "37347305215", + "37349332448", + "37374409928", + "37375445479", + "37375586954", + "37376593071", + "37389946212", + "37394378126", + "37395241262", + "37397309034", + "37398521095", + "37413464119", + "37415996926", + "37416279318", + "37494809494", + "37504870257", + "37506300031", + "37507007764", + "37510319705", + "37512150975", + "37523820462", + "37540020229", + "37540144327", + "37541010145", + "37542732748", + "37543978473", + "37545094812", + "37545716546", + "37546366036", + "37559982334", + "37560420246", + "37564096977", + "37565786884", + "37640916778", + "37691104551", + "37691665506", + "37693401496", + "37694473528", + "37701124596", + "37707903246", + "37718098351", + "37728544941", + "37731666423", + "37734578981", + "37734678850", + "37734795948", + "37738679055", + "37739140797", + "37739365674", + "37739711591", + "37739845694", + "37739864688", + "37739923087", + "37740005194", + "37741231927", + "37741305367", + "37741434771", + "37742517444", + "37742917771", + "37743060710", + "37743622252", + "37744583272", + "37745729841", + "37747272797", + "37747282656", + "37749419367", + "37761125544", + "37762901988", + "37762951968", + "37766639989", + "37799081635", + "37801128025", + "37803230995", + "37803446927", + "37805183773", + "37810177756", + "37811045289", + "37812094929", + "37813893652", + "37816281826", + "37834873799", + "37887208552", + "37887677058", + "37888258971", + "37892285758", + "37892504905", + "37897476192", + "38009577313", + "38009751458", + "38077369614", + "38087167729", + "38087206509", + "38087752336", + "38089176598", + "38093707846", + "38093940617", + "38094956965", + "38095294863", + "38096225501", + "38097202777", + "38097643686", + "38097811247", + "38097941308", + "38098277242", + "38098894928", + "38099588461", + "38100216011", + "38100768977", + "38101113898", + "38101502677", + "38101616162", + "38102454452", + "38102657574", + "38103692513", + "38105628107", + "38106695235", + "38114482325", + "38114743865", + "38114753174", + "38115320223", + "38115388121", + "38115472535", + "38116231959", + "38130211171", + "38146252958", + "38149926651", + "38150794867", + "38151498909", + "38151698313", + "38152603302", + "38152625837", + "38152692645", + "38152760329", + "38153103642", + "38153201884", + "38153296624", + "38153330406", + "38153440113", + "38153468775", + "38153478324", + "38153524782", + "38153545779", + "38154139862", + "38154283169", + "38154299190", + "38154371839", + "38154587215", + "38155175678", + "38155221280", + "38156016740", + "38159532295", + "38159643806", + "38160879452", + "38161561304", + "38162103937", + "38165705267", + "38168912246", + "38186078591", + "38189716570", + "38190450074", + "38191930614", + "38210650474", + "38254994238", + "38292011452", + "38295240201", + "38304815896", + "38320765570", + "38335637302", + "38335683298", + "38336135754", + "38336400566", + "38336529808", + "38337974933", + "38339027571", + "38344574992", + "38345572852", + "38416346427", + "38416389781", + "38418037093", + "38421455413", + "38428252213", + "38439240685", + "38478095918", + "38478947158", + "38508804072", + "38512515424", + "38514180564", + "38518769031", + "38531754196", + "38604265113", + "38605448921", + "38624056266", + "38635247772", + "38637567300", + "38669096043", + "38669488941", + "38671593723", + "38673080757", + "38674274388", + "38725836286", + "38785477428", + "38800593092", + "38815098576", + "38817374062", + "38824819744", + "38827690901", + "38829062754", + "38829515637", + "38829994639", + "38832498886", + "38833122675", + "38835263541", + "38842124064", + "38864846669", + "38964584946", + "38964801793", + "38964995514", + "38965078377", + "38965221701", + "38965260163", + "38969798938", + "38969924945", + "38972920715", + "38973329985", + "38973338025", + "38973420671", + "38974669950", + "38975010169", + "38975560167", + "38975789267", + "38976253774", + "38976351570", + "38978531340", + "38978533724", + "38978745945", + "38979194128", + "38979925666", + "38980005793", + "38980008914", + "38980089002", + "38980452190", + "38980914399", + "38981388043", + "38981492840", + "38981523623", + "38981831429", + "38982033023", + "38982079721", + "38982609798", + "38982836813", + "38983306684", + "38983355048", + "38983422226", + "38983471330", + "38983723250", + "38983767702", + "38984904318", + "38984910364", + "38984977679", + "38985082997", + "38985111066", + "38985644097", + "38985800954", + "38986120559", + "38986222182", + "38986505512", + "38987526600", + "38987672759", + "38987773038", + "38987895421", + "38988823602", + "38988898707", + "38988927989", + "38989956807", + "38990111230", + "38990141270", + "38991706282", + "38991832658", + "38992787872", + "38992860354", + "38993076767", + "38994816769", + "38996100960", + "38996605130", + "38996899290", + "38997410828", + "38998005661", + "38999746202", + "39000951833", + "39001049596", + "39001107574", + "39001765726", + "39001816434", + "39002175316", + "39002269281", + "39003503204", + "39003571667", + "39004466422", + "39004497896", + "39005025976", + "39005664700", + "39006896632", + "39009803881", + "39010388987", + "39010883961", + "39011101113", + "39017702380", + "39044530662", + "39051148306" + ], + "cases": { + "auq-format-gate": { + "observations": 162, + "withCommands": 0, + "commands": [], + "costUsd": 0, + "costRuns": 110 + }, + "autoplan-guard-pty": { + "observations": 7, + "withCommands": 0, + "commands": [], + "costUsd": 0, + "costRuns": 89 + }, + "benchmark-workflow": { + "observations": 179, + "withCommands": 179, + "commands": [ + "bin/gstack-ci-gate", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-decision-search", + "bin/gstack-diff-scope", + "bin/gstack-evidence", + "bin/gstack-issue-guard", + "bin/gstack-learnings-log", + "bin/gstack-next-version", + "bin/gstack-paths", + "bin/gstack-question-log", + "bin/gstack-question-preference", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-wtree", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "browse-basic": { + "observations": 158, + "withCommands": 158, + "commands": [ + "bin/gstack-learnings-log", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 106 + }, + "browse-snapshot": { + "observations": 158, + "withCommands": 158, + "commands": [ + "bin/gstack-learnings-log", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 106 + }, + "canary-workflow": { + "observations": 182, + "withCommands": 182, + "commands": [ + "bin/gstack-ci-gate", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-decision-search", + "bin/gstack-diff-scope", + "bin/gstack-evidence", + "bin/gstack-issue-guard", + "bin/gstack-learnings-log", + "bin/gstack-next-version", + "bin/gstack-paths", + "bin/gstack-question-log", + "bin/gstack-question-preference", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-wtree", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "context-recovery-artifacts": { + "observations": 166, + "withCommands": 161, + "commands": [ + "bin/gstack-context-recovery", + "bin/gstack-decision-search", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-remote-identity.sh", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-state-root.sh", + "bin/gstack-timeline-log", + "bin/gstack-timeline-read", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "context-restore-loads-latest": { + "observations": 167, + "withCommands": 162, + "commands": [ + "bin/gstack-context-recovery", + "bin/gstack-decision-search", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-remote-identity.sh", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-state-root.sh", + "bin/gstack-timeline-log", + "bin/gstack-timeline-read", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 105 + }, + "context-save-writes-file": { + "observations": 166, + "withCommands": 161, + "commands": [ + "bin/gstack-context-recovery", + "bin/gstack-decision-search", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-remote-identity.sh", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-state-root.sh", + "bin/gstack-timeline-log", + "bin/gstack-timeline-read", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "cso-diff-mode": { + "observations": 157, + "withCommands": 157, + "commands": [ + "design/dist/design" + ], + "costUsd": 1.5371, + "costRuns": 91 + }, + "design-review-detector-shim": { + "observations": 174, + "withCommands": 174, + "commands": [ + "bin/gstack-design-detect.ts", + "bin/gstack-redact", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 92 + }, + "design-review-detector-shim-dom": { + "observations": 170, + "withCommands": 170, + "commands": [ + "bin/gstack-design-detect.ts", + "bin/gstack-redact", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "design-review-plugin-handoff": { + "observations": 162, + "withCommands": 162, + "commands": [ + "bin/gstack-config", + "bin/gstack-design-detect.ts", + "bin/gstack-learnings-search", + "bin/gstack-redact", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 91 + }, + "diagram-triplet": { + "observations": 141, + "withCommands": 0, + "commands": [], + "costUsd": 0, + "costRuns": 106 + }, + "docsync-spawned": { + "observations": 181, + "withCommands": 180, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 1.2246, + "costRuns": 107 + }, + "document-release": { + "observations": 155, + "withCommands": 155, + "commands": [ + "bin/gstack-bun-version.sh", + "bin/gstack-codex-probe", + "bin/gstack-config", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-decision-search", + "bin/gstack-learnings-log", + "bin/gstack-next-version", + "bin/gstack-paths", + "bin/gstack-review-log", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-update-check", + "bin/gstack-upgrade", + "bin/gstack-version-bump", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 91 + }, + "gstack-upgrade-happy-path": { + "observations": 157, + "withCommands": 156, + "commands": [ + "bin/gstack-bun-version.sh", + "bin/gstack-config", + "bin/gstack-paths", + "bin/gstack-update-check", + "bin/gstack-upgrade" + ], + "costUsd": 0, + "costRuns": 117 + }, + "hermetic-canary": { + "observations": 167, + "withCommands": 167, + "commands": [], + "costUsd": 0.0127, + "costRuns": 101 + }, + "hermetic-sentinel": { + "observations": 167, + "withCommands": 167, + "commands": [], + "costUsd": 0.013, + "costRuns": 101 + }, + "investigate-owned-abort": { + "observations": 178, + "withCommands": 178, + "commands": [], + "costUsd": 0.2212, + "costRuns": 105 + }, + "investigate-owned-completion": { + "observations": 164, + "withCommands": 164, + "commands": [], + "costUsd": 0.2184, + "costRuns": 105 + }, + "investigate-owned-ending-error": { + "observations": 178, + "withCommands": 178, + "commands": [], + "costUsd": 0.2694, + "costRuns": 105 + }, + "land-and-deploy-first-run": { + "observations": 179, + "withCommands": 179, + "commands": [ + "bin/gstack-ci-gate", + "bin/gstack-config", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-decision-search", + "bin/gstack-diff-scope", + "bin/gstack-evidence", + "bin/gstack-issue-guard", + "bin/gstack-learnings-log", + "bin/gstack-next-version", + "bin/gstack-paths", + "bin/gstack-question-log", + "bin/gstack-question-preference", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-wtree", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "land-and-deploy-review-gate": { + "observations": 175, + "withCommands": 175, + "commands": [ + "bin/gstack-ci-gate", + "bin/gstack-config", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-decision-search", + "bin/gstack-diff-scope", + "bin/gstack-evidence", + "bin/gstack-issue-guard", + "bin/gstack-learnings-log", + "bin/gstack-next-version", + "bin/gstack-paths", + "bin/gstack-question-log", + "bin/gstack-question-preference", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-wtree", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 92 + }, + "land-and-deploy-workflow": { + "observations": 180, + "withCommands": 180, + "commands": [ + "bin/gstack-ci-gate", + "bin/gstack-config", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-decision-search", + "bin/gstack-diff-scope", + "bin/gstack-evidence", + "bin/gstack-issue-guard", + "bin/gstack-learnings-log", + "bin/gstack-next-version", + "bin/gstack-paths", + "bin/gstack-question-log", + "bin/gstack-question-preference", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-wtree", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 92 + }, + "learnings-show": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-context-recovery", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug" + ], + "costUsd": 0, + "costRuns": 105 + }, + "office-hours-auto-mode": { + "observations": 181, + "withCommands": 0, + "commands": [], + "costUsd": 0, + "costRuns": 108 + }, + "operational-learning": { + "observations": 155, + "withCommands": 155, + "commands": [ + "bin/gstack-learnings-log", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 91 + }, + "plan-ceo-finding-floor": { + "observations": 199, + "withCommands": 197, + "commands": [ + "bin/gstack-ceo-mode-handoff", + "bin/gstack-config", + "bin/gstack-context-recovery", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-start", + "bin/gstack-slug", + "browse/bin/remote-slug" + ], + "costUsd": 0, + "costRuns": 92 + }, + "plan-ceo-review": { + "observations": 59, + "withCommands": 59, + "commands": [] + }, + "plan-ceo-review-plan-mode": { + "observations": 166, + "withCommands": 166, + "commands": [ + "bin/gstack-context-recovery", + "bin/gstack-skill-start", + "bin/gstack-slug", + "browse/bin/remote-slug" + ], + "costUsd": 0, + "costRuns": 109 + }, + "plan-design-review-no-ui-scope": { + "observations": 166, + "withCommands": 166, + "commands": [ + "bin/gstack-design-detect.ts", + "bin/gstack-learnings-log", + "bin/gstack-redact", + "bin/gstack-skill-end", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 106 + }, + "plan-design-with-ui-scope": { + "observations": 183, + "withCommands": 183, + "commands": [ + "bin/gstack-config", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-start", + "bin/gstack-slug", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "plan-devex-finding-floor": { + "observations": 180, + "withCommands": 178, + "commands": [ + "bin/gstack-config", + "bin/gstack-context-recovery", + "bin/gstack-question-log", + "bin/gstack-review-log", + "bin/gstack-skill-start", + "bin/gstack-slug", + "browse/bin/remote-slug", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "plan-devex-review-plan-mode": { + "observations": 161, + "withCommands": 0, + "commands": [], + "costUsd": 0, + "costRuns": 90 + }, + "plan-eng-coverage-audit": { + "observations": 179, + "withCommands": 178, + "commands": [], + "costUsd": 0.4321, + "costRuns": 111 + }, + "plan-mode-no-op": { + "observations": 180, + "withCommands": 0, + "commands": [], + "costUsd": 0, + "costRuns": 93 + }, + "plan-review-report": { + "observations": 183, + "withCommands": 100, + "commands": [ + "bin/gstack-codex-probe", + "bin/gstack-config", + "bin/gstack-office-hours-review", + "bin/gstack-review-log" + ], + "costUsd": 0, + "costRuns": 111 + }, + "plan-tune-inspect": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-config", + "bin/gstack-developer-profile", + "bin/gstack-paths", + "bin/gstack-question-log", + "bin/gstack-question-preference", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-state-root.sh" + ], + "costUsd": 0, + "costRuns": 105 + }, + "qa-bootstrap": { + "observations": 174, + "withCommands": 173, + "commands": [ + "bin/gstack-config", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-slug", + "browse/bin/find-browse", + "browse/bin/remote-slug", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 108 + }, + "qa-functional-cli-fix": { + "observations": 159, + "withCommands": 158, + "commands": [ + "bin/gstack-qa-evidence" + ], + "costUsd": 1.3402, + "costRuns": 106 + }, + "qa-functional-cli-report": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-qa-evidence" + ], + "costUsd": 1.1367, + "costRuns": 106 + }, + "qa-functional-webhook-fix": { + "observations": 159, + "withCommands": 158, + "commands": [ + "bin/gstack-qa-evidence" + ], + "costUsd": 1.4296, + "costRuns": 106 + }, + "qa-functional-webhook-report": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-qa-evidence" + ], + "costUsd": 1.3103, + "costRuns": 106 + }, + "qa-only-no-fix": { + "observations": 174, + "withCommands": 174, + "commands": [ + "bin/gstack-config", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-slug", + "browse/bin/find-browse", + "browse/bin/remote-slug", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 92 + }, + "qa-quick": { + "observations": 174, + "withCommands": 174, + "commands": [ + "bin/gstack-config", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-slug", + "browse/bin/find-browse", + "browse/bin/remote-slug", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 92 + }, + "retro": { + "observations": 134, + "withCommands": 134, + "commands": [ + "bin/gstack-config", + "bin/gstack-decision-search", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-retro-metrics", + "bin/gstack-slug", + "bin/gstack-state-root.sh" + ] + }, + "retro-base-branch": { + "observations": 159, + "withCommands": 25, + "commands": [ + "bin/gstack-config", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-retro-metrics", + "bin/gstack-slug" + ], + "costUsd": 0, + "costRuns": 94 + }, + "review-army-delivery-audit": { + "observations": 177, + "withCommands": 177, + "commands": [ + "bin/gstack-diff-scope", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-review-log", + "bin/gstack-specialist-stats" + ], + "costUsd": 0, + "costRuns": 95 + }, + "review-army-json-findings": { + "observations": 160, + "withCommands": 160, + "commands": [ + "bin/gstack-diff-scope", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-review-log", + "bin/gstack-specialist-stats" + ], + "costUsd": 0, + "costRuns": 95 + }, + "review-army-migration-safety": { + "observations": 164, + "withCommands": 164, + "commands": [ + "bin/gstack-diff-scope", + "bin/gstack-egress-lib.sh", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-review-log", + "bin/gstack-specialist-stats", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 95 + }, + "review-army-perf-n-plus-one": { + "observations": 165, + "withCommands": 165, + "commands": [ + "bin/gstack-decision-search", + "bin/gstack-diff-scope", + "bin/gstack-egress-lib.sh", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-review-log", + "bin/gstack-specialist-stats", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 95 + }, + "review-army-quality-score": { + "observations": 160, + "withCommands": 160, + "commands": [ + "bin/gstack-diff-scope", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-review-log", + "bin/gstack-specialist-stats" + ], + "costUsd": 0.3515, + "costRuns": 95 + }, + "review-base-branch": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-diff-scope", + "bin/gstack-review-read" + ], + "costUsd": 0, + "costRuns": 91 + }, + "review-coverage-audit": { + "observations": 178, + "withCommands": 167, + "commands": [], + "costUsd": 0.4079, + "costRuns": 105 + }, + "review-dashboard-via": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-diff-scope", + "bin/gstack-review-read" + ], + "costUsd": 0, + "costRuns": 93 + }, + "review-enum-completeness": { + "observations": 166, + "withCommands": 166, + "commands": [ + "bin/gstack-decision-search", + "bin/gstack-review-log" + ], + "costUsd": 0, + "costRuns": 91 + }, + "review-exploratory-small-cli": { + "observations": 170, + "withCommands": 169, + "commands": [ + "bin/gstack-codex-probe", + "bin/gstack-config", + "bin/gstack-diff-scope", + "bin/gstack-learnings-search", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-specialist-stats" + ], + "costUsd": 1.9457, + "costRuns": 115 + }, + "review-gate-integrity": { + "observations": 0, + "withCommands": 0, + "commands": [], + "costUsd": 0, + "costRuns": 81 + }, + "review-sql-injection": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-decision-search", + "bin/gstack-review-log" + ], + "costUsd": 0, + "costRuns": 105 + }, + "review-test-value": { + "observations": 169, + "withCommands": 169, + "commands": [], + "costUsd": 0.6768, + "costRuns": 105 + }, + "safety-codex-consult-embed": { + "observations": 68, + "withCommands": 68, + "commands": [ + "bin/gstack-codex-probe", + "bin/gstack-paths" + ], + "costUsd": 0.9964, + "costRuns": 91 + }, + "safety-pair-agent-block": { + "observations": 68, + "withCommands": 68, + "commands": [ + "bin/gstack-config", + "browse/dist/browse" + ], + "costUsd": 0.3055, + "costRuns": 91 + }, + "safety-ship-stale-evidence": { + "observations": 67, + "withCommands": 67, + "commands": [ + "bin/gstack-ci-gate", + "bin/gstack-evidence", + "bin/gstack-slug", + "bin/gstack-wtree" + ], + "costUsd": 0.5915, + "costRuns": 91 + }, + "setup-deploy-workflow": { + "observations": 181, + "withCommands": 181, + "commands": [ + "bin/gstack-ci-gate", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-decision-search", + "bin/gstack-diff-scope", + "bin/gstack-evidence", + "bin/gstack-issue-guard", + "bin/gstack-learnings-log", + "bin/gstack-next-version", + "bin/gstack-paths", + "bin/gstack-question-log", + "bin/gstack-question-preference", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-wtree", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 106 + }, + "shared-libs-read-only": { + "observations": 166, + "withCommands": 166, + "commands": [ + "bin/gstack-safe-git", + "browse/dist/browse" + ], + "costUsd": 0.786, + "costRuns": 92 + }, + "shared-libs-review-index-flags": { + "observations": 177, + "withCommands": 176, + "commands": [ + "bin/gstack-review-log", + "bin/gstack-review-read" + ], + "costUsd": 2.4926, + "costRuns": 91 + }, + "shared-libs-review-lifecycle": { + "observations": 174, + "withCommands": 173, + "commands": [ + "bin/gstack-egress-lib.sh", + "bin/gstack-review-log", + "bin/gstack-review-read" + ], + "costUsd": 2.4643, + "costRuns": 92 + }, + "shared-libs-review-path-eligibility": { + "observations": 171, + "withCommands": 170, + "commands": [ + "bin/gstack-review-log", + "bin/gstack-review-read" + ], + "costUsd": 3.9227, + "costRuns": 91 + }, + "shared-libs-review-prior-coverage": { + "observations": 164, + "withCommands": 164, + "commands": [ + "bin/gstack-review-log", + "bin/gstack-review-read" + ], + "costUsd": 2.517, + "costRuns": 91 + }, + "shared-libs-review-revalidation": { + "observations": 174, + "withCommands": 171, + "commands": [ + "bin/gstack-review-log", + "bin/gstack-review-read" + ], + "costUsd": 4.8033, + "costRuns": 92 + }, + "shared-libs-unsupported-git": { + "observations": 159, + "withCommands": 159, + "commands": [ + "bin/gstack-safe-git", + "browse/dist/browse" + ], + "costUsd": 0.5855, + "costRuns": 92 + }, + "ship-base-branch": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-diff-scope", + "bin/gstack-review-read" + ], + "costUsd": 0, + "costRuns": 92 + }, + "ship-coverage-audit": { + "observations": 157, + "withCommands": 156, + "commands": [ + "bin/gstack-bun-version.sh", + "bin/gstack-config", + "bin/gstack-paths", + "bin/gstack-slug", + "bin/gstack-update-check", + "bin/gstack-upgrade" + ], + "costUsd": 0.7746, + "costRuns": 107 + }, + "ship-coverage-value": { + "observations": 169, + "withCommands": 168, + "commands": [ + "bin/gstack-paths", + "bin/gstack-slug" + ], + "costUsd": 0.9375, + "costRuns": 107 + }, + "ship-docsync-completion": { + "observations": 152, + "withCommands": 151, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-session-kind", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 3.5392, + "costRuns": 110 + }, + "ship-docsync-current": { + "observations": 153, + "withCommands": 152, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-session-kind", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 3.2887, + "costRuns": 110 + }, + "ship-docsync-failure": { + "observations": 156, + "withCommands": 155, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 2.028, + "costRuns": 110 + }, + "ship-docsync-late-result": { + "observations": 170, + "withCommands": 168, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 0.9185, + "costRuns": 109 + }, + "ship-docsync-launch-failure": { + "observations": 154, + "withCommands": 153, + "commands": [ + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 0.4608, + "costRuns": 110 + }, + "ship-docsync-missing-asset": { + "observations": 148, + "withCommands": 147, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 1.3017, + "costRuns": 109 + }, + "ship-docsync-missing-marker": { + "observations": 175, + "withCommands": 174, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 0.5063, + "costRuns": 110 + }, + "ship-docsync-recovery": { + "observations": 160, + "withCommands": 159, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 1.0055, + "costRuns": 110 + }, + "ship-docsync-stale-after": { + "observations": 172, + "withCommands": 171, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 1.1196, + "costRuns": 110 + }, + "ship-docsync-stale-before": { + "observations": 165, + "withCommands": 164, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 1.1591, + "costRuns": 110 + }, + "ship-docsync-store": { + "observations": 154, + "withCommands": 153, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-session-kind", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 3.5563, + "costRuns": 110 + }, + "ship-docsync-timeout-unsettled": { + "observations": 167, + "withCommands": 166, + "commands": [ + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 0.5324, + "costRuns": 110 + }, + "ship-exploratory-late-input": { + "observations": 168, + "withCommands": 167, + "commands": [ + "bin/gstack-config", + "bin/gstack-diff-scope", + "bin/gstack-learnings-search", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-specialist-stats" + ], + "costUsd": 2.1399, + "costRuns": 115 + }, + "ship-exploratory-plan-checks": { + "observations": 167, + "withCommands": 166, + "commands": [ + "bin/gstack-config", + "bin/gstack-diff-scope", + "bin/gstack-learnings-search", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-specialist-stats" + ], + "costUsd": 1.8717, + "costRuns": 115 + }, + "ship-exploratory-small-cli": { + "observations": 171, + "withCommands": 170, + "commands": [ + "bin/gstack-config", + "bin/gstack-diff-scope", + "bin/gstack-learnings-search", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-specialist-stats" + ], + "costUsd": 1.8332, + "costRuns": 115 + }, + "ship-exploratory-unavailable": { + "observations": 172, + "withCommands": 171, + "commands": [ + "bin/gstack-config", + "bin/gstack-diff-scope", + "bin/gstack-learnings-search", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-specialist-stats" + ], + "costUsd": 1.837, + "costRuns": 115 + }, + "ship-local-hook-preservation": { + "observations": 161, + "withCommands": 161, + "commands": [ + "bin/gstack-config", + "bin/gstack-paths", + "bin/gstack-redact" + ], + "costUsd": 0.307, + "costRuns": 107 + }, + "ship-local-workflow": { + "observations": 157, + "withCommands": 156, + "commands": [ + "bin/gstack-bun-version.sh", + "bin/gstack-config", + "bin/gstack-paths", + "bin/gstack-update-check", + "bin/gstack-upgrade" + ], + "costUsd": 0, + "costRuns": 107 + }, + "ship-managed-hook-refresh": { + "observations": 160, + "withCommands": 160, + "commands": [ + "bin/gstack-config", + "bin/gstack-paths", + "bin/gstack-redact" + ], + "costUsd": 0.2737, + "costRuns": 107 + }, + "ship-skipped-queued-finding": { + "observations": 154, + "withCommands": 153, + "commands": [], + "costUsd": 0.6029, + "costRuns": 92 + }, + "ship-triage": { + "observations": 180, + "withCommands": 180, + "commands": [], + "costUsd": 0, + "costRuns": 92 + }, + "ship-unmanaged-hook-consent": { + "observations": 161, + "withCommands": 161, + "commands": [ + "bin/gstack-config", + "bin/gstack-paths", + "bin/gstack-redact" + ], + "costUsd": 0.3538, + "costRuns": 107 + }, + "skillify-approval-reject": { + "observations": 186, + "withCommands": 175, + "commands": [ + "bin/gstack-learnings-log", + "bin/gstack-state-root.sh", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "skillify-happy-path": { + "observations": 186, + "withCommands": 175, + "commands": [ + "bin/gstack-learnings-log", + "bin/gstack-state-root.sh", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "skillify-provenance-refusal": { + "observations": 190, + "withCommands": 175, + "commands": [ + "bin/gstack-learnings-log", + "bin/gstack-state-root.sh", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 108 + }, + "skillmd-no-local-binary": { + "observations": 155, + "withCommands": 155, + "commands": [ + "bin/gstack-learnings-log", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 91 + }, + "skillmd-outside-git": { + "observations": 155, + "withCommands": 155, + "commands": [ + "bin/gstack-learnings-log", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 91 + }, + "skillmd-setup-discovery": { + "observations": 156, + "withCommands": 156, + "commands": [ + "bin/gstack-learnings-log", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 117 + }, + "test-audit-report-only": { + "observations": 169, + "withCommands": 154, + "commands": [], + "costUsd": 0.9871, + "costRuns": 106 + }, + "timeline-event-flow": { + "observations": 166, + "withCommands": 161, + "commands": [ + "bin/gstack-context-recovery", + "bin/gstack-decision-search", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-remote-identity.sh", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-state-root.sh", + "bin/gstack-timeline-log", + "bin/gstack-timeline-read", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 90 + }, + "tpa-absent-darwin": { + "observations": 184, + "withCommands": 180, + "commands": [], + "costUsd": 0.2257, + "costRuns": 91 + }, + "tpa-absent-linux": { + "observations": 185, + "withCommands": 185, + "commands": [], + "costUsd": 0.2243, + "costRuns": 106 + }, + "tpa-apple-ban": { + "observations": 184, + "withCommands": 149, + "commands": [], + "costUsd": 0.3062, + "costRuns": 91 + }, + "tpa-broken": { + "observations": 184, + "withCommands": 181, + "commands": [], + "costUsd": 0.2265, + "costRuns": 91 + }, + "tpa-present": { + "observations": 185, + "withCommands": 185, + "commands": [], + "costUsd": 0.2325, + "costRuns": 106 + } + } +} diff --git a/test/helpers/exercised-prototype.ts b/test/helpers/exercised-prototype.ts new file mode 100644 index 000000000..0d7dd801a --- /dev/null +++ b/test/helpers/exercised-prototype.ts @@ -0,0 +1,149 @@ +/** + * Frozen parity oracle for the exercised-path map (W2-A2): the measured + * prototype (GSTA-45 investigation, exercised.ts at 521fd42, REPORT-2) with + * only its module-level root and base derivation turned into parameters, its + * path-prefix regex rewritten to an equivalent non-backtracking form, its + * import scan switched to A1's scanCodeImports (the prototype ran on the + * shebang-fixed base; A1 also fixed the TSX loader misreading .ts generics), + * and the per-case build of its replay (variant cu) appended. Do not edit its + * edge rules: test/exercised-map.test.ts pins that the shipped + * exercisedConsumers() equals or is a reviewed superset of this map. + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { derivedDependencies, scanCodeImports, SELECTION_DATA_MODULES, type DependencyMaps, type ExercisedMap } from '../../scripts/pr-dependencies'; +import { matchGlob } from './test-selection'; +import { isPaidTestFile } from './paid-test-set'; + +export function prototypeConsumers(maps: DependencyMaps, ROOT: string, observedMap: ExercisedMap | null, PR_FULL_GATE_FILES: readonly string[]): Map> { + const base = derivedDependencies(maps, ROOT); + const tracked: string[] = [...base.tracked]; + const trackedSet = base.tracked as Set; + const cache = new Map(); + const read = (f: string) => { if (!cache.has(f)) { let t = ''; try { if (trackedSet.has(f) && fs.statSync(path.join(ROOT, f)).size < 3_000_000) t = fs.readFileSync(path.join(ROOT, f), 'utf8'); } catch {} cache.set(f, t); } return cache.get(f)!; }; + const CODE = /\.(?:[cm]?[jt]s|tsx|jsx)$/; + const isBunShebang = (f: string) => /^#!.*\b(?:bun|node)\b/.test(read(f).slice(0, 200)); + const isCode = (f: string) => CODE.test(f) || isBunShebang(f); + const isShell = (f: string) => !isCode(f) && (/\.sh$/.test(f) || f === 'setup' || /^#!.*\b(?:ba|z|da)?sh\b/.test(read(f).slice(0, 200))); + const resolveSpec = (spec: string, from: string) => { + if (!spec.startsWith('.')) return null; + const b = path.posix.normalize(path.posix.join(path.posix.dirname(from), spec)); + for (const c of [b, b + '.ts', b + '.tsx', b + '.js', b + '.mjs', b + '/index.ts', b + '/index.js']) if (trackedSet.has(c)) return c; + return null; + }; + const dirs = new Set(tracked.flatMap(f => f.split('/').slice(0, -1).map((_, i, p) => p.slice(0, i + 1).join('/')))); + const REPO_PATH = /(?:^|[^\w.@-])((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|test\/helpers|test\/fixtures|capy)\/[\w.@-]+(?:\/[\w.@-]+)*)/g; + const JOINED = /['"\x60](bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents)['"\x60]\s*,\s*['"\x60]([\w.@-]+)['"\x60]/g; + const binByName = new Map(); + for (const f of tracked) if (f.startsWith('bin/') || f.startsWith('browse/bin/')) { const b = path.posix.basename(f); binByName.set(b, f); binByName.set(b.replace(/\.(ts|sh|js)$/, ''), f); } + const BINARIES = [...read('scripts/build.sh').matchAll(/--compile\b[^\n]*?\s([\w./-]+\.ts)\s+--outfile\s+([\w./-]+)/g)].map(m => ({ outfile: m[2]!, entry: m[1]!, tree: m[1]!.startsWith('bin/') ? [m[1]!] : tracked.filter(f => matchGlob(f, path.posix.dirname(m[1]!) + '/**')) })); + let pkgScripts: Record = {}; try { pkgScripts = JSON.parse(read('package.json')).scripts ?? {}; } catch {} + const expandRef = (ref: string) => { ref = ref.replace(/[.]+$/, ''); if (trackedSet.has(ref)) return [ref]; if (dirs.has(ref)) return tracked.filter(f => f.startsWith(ref + '/')); return []; }; + const stripCode = (t: string) => t.replace(/\/\*[\s\S]*?\*\//g, '').replace(/^[ \t]*\/\/.*$/gm, ''); + const stripMessages = (t: string) => { + let out = '', i = 0; + while (i < t.length) { + const m = /^(?:echo|printf|say|warn|die|fail|err|log|info|note|_dc_row|row)\b/.exec(t.slice(i, i + 8)); + const prev = i === 0 ? '\n' : t[i - 1]!; + if (m && /[\s;|&(\x60{]/.test(prev)) { + let j = i + m[0].length, q: string | null = null; + while (j < t.length) { const c = t[j]!; if (q) { if (c === '\\') { j += 2; continue; } if (c === q) q = null; j++; continue; } if (c === '"' || c === "'") { q = c; j++; continue; } if (c === '\n' || c === ';' || c === '|' || c === '&' || c === ')') break; j++; } + out += ' '; i = j; continue; + } + out += t[i]; i++; + } + return out; + }; + const stripShell = (t: string) => stripMessages(t + .replace(/<<-?\s*['"]?(\w+)['"]?[^\n]*\n[\s\S]*?\n\s*\1\s*$/gm, '') + .split('\n').filter(l => !/^\s*#(?!!)/.test(l)) + .map(l => l.replace(/(^\s*|\bin\s+|;;\s*)[\w*."'-]+(?:\s*\|\s*[\w*."'-]+)*\)(?=\s|$)/g, '$1').replace(/\s#\s.*$/, '')) + .join('\n')); + const CMD_START = /(?:^|[;|&({\x60]|\$\(|\b(?:then|do|else|exec|command|nohup|bash|sh|bun|node|timeout\s+\S+|env(?:\s+\w+=\S*)*|if|while|until|!|time))\s*"?$/; + // Changed from the prototype: its `(?:\/?[\w.@-]+)*\/` path prefix backtracks exponentially; this is the same language (words separated by single slashes, ending in a slash) in an unambiguous form. + const PREFIX = /(?:"?(?:\$\{?\w+\}?|\$\(dirname\s+"?\$0"?\)|~|\.)?(?:\/(?:[\w.@-]+\/)*|(?:[\w.@-]+\/)+))?"?$/; + const cmdOk = (before: string) => { const b = before.replace(PREFIX, ''); return CMD_START.test(b) || (/\/$/.test(before) && /\b\w+="?$/.test(b)); }; + const shellExec = (text: string, from: string, out: Set) => { + for (const m of text.matchAll(/gstack-[a-z0-9][a-z0-9-]*[a-z0-9](?:\.(?:sh|ts))?\b/g)) { + const i = m.index!; const ls = text.lastIndexOf('\n', i - 1) + 1; + if (!cmdOk(text.slice(ls, i))) continue; + const b = binByName.get(m[0]); if (b) out.add(b); + } + for (const m of text.matchAll(/(?:bin|lib|scripts|browse|design|make-pdf)\/[\w.@-]+(?:\/[\w.@-]+)*/g)) { + const i = m.index!; const ls = text.lastIndexOf('\n', i - 1) + 1; const before = text.slice(ls, i); + if (/[\w.@-]$/.test(before)) continue; + if (!cmdOk(before) && !/\bbun\s+(?:run\s+)?"?(?:\$\{?\w+\}?\/)?$/.test(before)) continue; + for (const f of expandRef(m[0])) out.add(f); + } + for (const m of text.matchAll(/(?:bin|lib|scripts|browse|design|make-pdf|hosts|capy|test\/fixtures|test\/helpers)\/[\w.@-]+(?:\/[\w.@-]+)*\.(?:json|jsonl|env|txt|ya?ml|toml|csv|tsv)\b/g)) for (const f of expandRef(m[0])) out.add(f); + for (const m of text.matchAll(/(?:^|[\s;&|({])(?:\.|source)\s+([^;|&\n]+)/gm)) { const names = [...m[1]!.matchAll(/([\w.-]+\.sh|gstack-[\w.-]+)/g)].map(x => x[1]!); if (!names.length) continue; const b = names[names.length - 1]!; const sib = path.posix.join(path.posix.dirname(from), b); if (trackedSet.has(sib)) out.add(sib); else if (binByName.get(b)) out.add(binByName.get(b)!); } + for (const bin of BINARIES) if (text.includes(bin.outfile) || (bin.outfile === 'browse/dist/browse' && /\$B\b|find-browse/.test(text)) || (bin.outfile === 'design/dist/design' && /\$D\b/.test(text))) for (const f of bin.tree) out.add(f); + for (const m of text.matchAll(/\b(?:bun|npm) run ([a-z][\w:.-]*)/g)) { const c = pkgScripts[m[1]!]; if (c) for (const r of c.matchAll(REPO_PATH)) for (const f of expandRef(r[1]!)) out.add(f); } + }; + const bashBlocks = (md: string) => [...md.matchAll(/\x60\x60\x60(?:bash|sh|shell)\n([\s\S]*?)\x60\x60\x60/g)].map(m => m[1]!).join('\n'); + const edges = new Map(); + const execRefs = (f: string): string[] => { + if (edges.has(f)) return edges.get(f)!; + const out = new Set(); + const raw = read(f); + if ((SELECTION_DATA_MODULES as readonly string[]).includes(f) || !raw) { edges.set(f, []); return []; } + if (isCode(f)) { + try { for (const spec of scanCodeImports(f, raw)) { const h = resolveSpec(spec, f); if (h) out.add(h); } } catch {} + const code = stripCode(raw.replace(/^#!.*/, '')); + for (const m of code.matchAll(/['"\x60](?:\$\{[^}]+\}\/)?((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)['"\x60]/g)) for (const x of expandRef(m[1]!)) out.add(x); + for (const m of code.matchAll(JOINED)) for (const x of expandRef(m[1] + '/' + m[2])) out.add(x); + for (const bin of BINARIES) if (code.includes("'" + bin.outfile + "'") || code.includes('"' + bin.outfile + '"')) for (const x of bin.tree) out.add(x); + } else if (isShell(f)) { + shellExec(stripShell(raw), f, out); + } else if (/\.(md|tmpl)$/.test(f)) { + shellExec(stripShell(bashBlocks(raw)), f, out); + } + out.delete(f); + const list = [...out]; edges.set(f, list); return list; + }; + const isFullGate = (f: string) => PR_FULL_GATE_FILES.some((p: string) => matchGlob(f, p)); + const closure = (entries: Iterable) => { + const seen = new Set(); const q = [...entries].filter(f => trackedSet.has(f)); + while (q.length) { const f = q.pop()!; if (seen.has(f)) continue; seen.add(f); if (base.global.has(f) || isFullGate(f)) continue; for (const n of execRefs(f)) if (!seen.has(n) && !(/\.test\.tsx?$/.test(n) && isPaidTestFile(n))) q.push(n); } + return seen; + }; + const expand = (p: string) => p.includes('*') ? tracked.filter(f => matchGlob(f, p)) : trackedSet.has(p) ? [p] : []; + const OWNER = /^test\/[^/]+\.test\.ts$/; + function caseEntries(patterns: string[], defaultOwner: string | null) { + const owners = patterns.filter(p => OWNER.test(p) && isPaidTestFile(p)); + const files = [...(owners.length ? owners : defaultOwner ? [defaultOwner] : []), ...patterns.flatMap(expand)]; + const gen = files.filter(f => f.endsWith('.tmpl')).map(f => f.replace(/\.tmpl$/, '')).filter(f => trackedSet.has(f)); + return [...files, ...gen]; + } + function binsInCommand(cmd: string): string[] { + const out = new Set(); + const pty = cmd.startsWith('PTY:'); + for (const m of cmd.matchAll(/gstack-[a-z0-9-]*[a-z0-9](?:\.(?:sh|ts))?/g)) { + let tok = m[0]!; + if (pty) { while (tok.length > 7 && !binByName.has(tok)) { const k = tok.lastIndexOf('-'); tok = k > 6 ? tok.slice(0, k) : ''; } } + const i = m.index!; if (cmd[i - 1] === '/' && i >= 2 && /[\s"'\x60(]/.test(cmd[i - 2]!)) continue; + const b = binByName.get(tok); if (b) out.add(b); + } + for (const m of cmd.matchAll(/browse\/bin\/([a-z-]+)/g)) { const b = binByName.get(m[1]!); if (b) out.add(b); } + for (const bin of BINARIES) { + const hit = cmd.includes(bin.outfile) || (bin.outfile === 'browse/dist/browse' && /\$B\b|\$\{B\}|find-browse/.test(cmd)) || (bin.outfile === 'design/dist/design' && /\$D\b|\$\{D\}/.test(cmd)) || (bin.outfile === 'make-pdf/dist/pdf' && /make-pdf\/dist\/pdf/.test(cmd)); + if (hit) out.add(bin.outfile); + } + return [...out]; + } + const binaryTree = (outfile: string) => BINARIES.find(b => b.outfile === outfile)?.tree ?? []; + + const observed = observedMap?.cases ?? {}; + const hostIds = tracked.filter((f: string) => /^hosts\/[\w-]+\.ts$/.test(f) && f !== 'hosts/index.ts').map((f: string) => path.basename(f, '.ts')); + const byFile = new Map>(); + for (const [id, patterns] of Object.entries(maps.e2eTouchfiles)) { + const entries = caseEntries(patterns, null); + for (const b of observed[id]?.commands ?? []) { if (trackedSet.has(b)) entries.push(b); else entries.push(...binaryTree(b)); } + const files = closure(entries); + const ownerText = entries.filter((f: string) => /^test\/[^/]+\.test\.ts$/.test(f)).map(read).join('\n'); + for (const h of hostIds) if (new RegExp('[\'"\x60]' + h + '[\'"\x60]|--host[ =]' + h + '\\b').test(ownerText)) for (const f of closure(['hosts/' + h + '.ts', ...expand('hosts/' + h + '/**')])) files.add(f); + for (const f of files) { if (!byFile.has(f)) byFile.set(f, new Set()); byFile.get(f)!.add(id); } + } + void binsInCommand; void stripShell; void execRefs; + return byFile; +}