diff --git a/.github/workflows/evals-periodic.yml b/.github/workflows/evals-periodic.yml index 45aeae1ce..da9f8dde6 100644 --- a/.github/workflows/evals-periodic.yml +++ b/.github/workflows/evals-periodic.yml @@ -723,6 +723,34 @@ jobs: if-no-files-found: ignore retention-days: 90 + # Exercised-path map (shadow, non-blocking; docs/TESTING_INTERNALS.md#exercised-path-map): + # union this census's own recorded commands into the map, report MAP_GAP lines and + # cases that recorded no commands, and print the refresh command. Never fails the run. + - name: Refresh the exercised-path map from this census + if: always() + continue-on-error: true + run: | + mkdir -p /tmp/exercised-map + dirs=() + for dir in /tmp/gate-census-report /tmp/paid-report; do [ -d "$dir" ] && dirs+=(--from-dir "$dir"); done + status=0 + bun --no-install run scripts/exercised-map.ts "${dirs[@]}" \ + --run-id "$GITHUB_RUN_ID" --out /tmp/exercised-map/exercised-commands.json --report /tmp/exercised-map/report.md || status=$? + if [ "$status" -ne 0 ]; then + echo "exercised-path map refresh failed with exit $status (non-blocking; no map uploaded)" | tee -a "$GITHUB_STEP_SUMMARY" + fi + if [ -f /tmp/exercised-map/report.md ]; then cat /tmp/exercised-map/report.md >> "$GITHUB_STEP_SUMMARY"; fi + + - name: Upload the refreshed exercised-path map + if: always() + continue-on-error: true + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: exercised-map-a${{ github.run_attempt }} + path: /tmp/exercised-map + if-no-files-found: ignore + retention-days: 90 + # Weekly pass-rate gate over the last 10 weekly runs (drift, rule cases # behaving like behavior, quarantine exit/expiry/cap). Fails closed when # history cannot be fetched. diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ecfedfec..796a13f7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## [1.91.130.0] - 2026-10-18 + +**Every PR plan now shows what a tighter, exercised-path case map would run, without changing what actually runs.** Today a change to a bin or lib file reaches almost every gate case, because selection follows every mention of the file. The exercised-path map counts only what a case really executes: its touchfiles, its test's imports, the shell its skill runs, and the bins its recorded trials invoked. It runs in shadow mode. Replayed over every PR run that had a gate red between 2026-10-07 and 2026-10-17 (37 runs, 47 failed gate cases), it keeps all 47: `bun run test:exercised-map --replay`. + +### Added +- **A shadow line in each PR plan's coverage summary.** It names the gate cases the map would select, drop and add, with per-file provenance and a predicted saving. The saving is a lower bound, because about 39% of observed case runs (the PTY trials) have no known cost. +- **`bun run test:health` reports the shadow saving and shadow misses per week.** A red on a case the shadow would drop counts as a miss unless the case passed later at the same head SHA, which lists it as a flaky drop. +- **MAP_GAP lines in the paid report.** A command a case ran outside its map is printed as `MAP_GAP `, marked non-blocking for this PR, and widens the map at the next refresh. It never affects a verdict. +- **`bun run test:exercised-map`** refreshes the committed map (`test/helpers/exercised-commands.json`) from a census run (`--from-run `) or from local artifacts (`--from-dir`). Refreshes only add commands. Unavailable or partial input writes nothing and exits with a stable code: 2 usage, 3 unavailable, 4 incomplete. + +### For contributors +- The weekly census unions its own recorded commands into the map and uploads it as `exercised-map-a`. +- A PR that changes only the map selects no paid case while the map is in shadow mode. +- docs/TESTING_INTERNALS.md#exercised-path-map documents the edge classes, the refresh contract and the activation bar. + ## [1.91.129.0] - 2026-10-18 **PR paid-lane selection reads every script it depends on, and fails closed when it can't.** Selection derives which paid cases a changed file reaches by scanning imports. It skipped extensionless `bun`/`node` scripts, read 15 `.ts` files with the TSX loader, and treated a scan error as "no consumers", which under-selects. Across all 2,223 code files in the repo, scans now finish with 0 errors, and no (file, case) pair that main selected is lost. diff --git a/VERSION b/VERSION index de898c0c6..97bde5585 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.129.0 +1.91.130.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index fbec08ef6..895f1b42e 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.129.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.130.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index 7b7e7956a..6e8f0408a 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -1054,6 +1054,78 @@ free-only list. Editing a paid test file runs its own gate cases on the PR when their Bun test is named by the case id; the file's other cases stay deferred with a reason that says to name the test by its case id. +### Exercised-path map + +The derived closure (above) reaches almost every gate case from most bin/ and +lib/ files, because it follows prose, string and doc-comment mentions. The +exercised-path map is a tighter consumer set that runs in **shadow mode**: it +is computed and reported on every PR plan, and it never changes which cases a +PR runs. Activating it is a later decision that needs PTY command capture for +the cases that record no commands, two weeks of shadow data with zero shadow +misses, and Garry's call on the measured saving. + +A case's exercised consumers (`exercisedConsumers` in +`scripts/pr-dependencies.ts`) are the union of its touchfile matches, the +closure of its test file and helpers, the executable shell of its loaded skill +(template and generated SKILL.md) and of every bin, lib and script that shell +runs, and the bins its recorded trials invoked. Shell edges are command-position +`gstack-*` names and repo paths, `.`/`source`, `./setup`, quoted path +literals, `path.join` pairs, data files, `bun run` scripts and compiled outfiles +(`browse/dist/browse`, `$B`, `design/dist/design`, `$D`, `make-pdf/dist/pdf`). +Comments, echo/printf/die messages, case patterns, bare `VAR=gstack-x` strings +and slash-command skill names are not edges; command substitutions are kept, an +expanding heredoc keeps its substitutions and a heredoc an interpreter reads +stays whole. The closure stops at `PR_FULL_GATE_FILES` and the global set, which +stay full. `test/exercised-map.test.ts` pins that this function equals or is a +reviewed superset of the measured prototype (`test/helpers/exercised-prototype.ts`). + +The observed half is `test/helpers/exercised-commands.json` (generated; per case +the bins and compiled outfiles its recorded trials ran, observation counts, the +source run ids and a mean known cost per run). Refreshes are union-only: an +observation never removes a command and a missing or empty artifact never erases +one; removing a command is a reviewed edit. While the map is in shadow mode it +changes no selection, so a PR that changes only the map selects no paid case. + +Dollar figures are lower bounds. Known cost excludes PTY trials, whose cost is +unknown: about 39% of observed case runs in the committed map. The shadow block +and `test:health` both say so. + +- **Shadow block:** `prCoverage.shadow` in the paid-plan manifest lists the gate + cases the map would select, would drop and would add, a provenance line per + changed file ("selects N case(s): touchfile (k), import (k), skill bash via + bin/gstack-x (k), observed bin/gstack-y (k)"), the predicted saving and the + unknown-cost share. The coverage summary prints one line of it. +- **Shadow misses (`bun run test:health`, metric `pr-shadow-map`):** a red case + (more executed failed than passed trials in the run) that the shadow would + drop is an activation miss unless the case passed in a later run of the same + branch at the same head SHA. That later pass marks it a flaky drop, which is + listed but not counted. The D3 activation bar counts activation misses only. +- **MAP_GAP:** the paid report runs the same bin matcher over each executed + case's recorded commands and prints, and writes to `map-gaps.txt`, + `MAP_GAP : non-blocking for this PR; the next census refresh + widens the map (docs/TESTING_INTERNALS.md#exercised-path-map)` for a command + outside the case's static-plus-observed closure. It is its own result: never a + case verdict, a lane problem or an INFRA re-dispatch input. +- **Census refresh:** the weekly census unions its own recorded commands into the + map (no REST artifact downloads), uploads it as `exercised-map-a`, + reports MAP_GAP lines and the cases that recorded no commands, and prints the + refresh command. Run `bun run test:exercised-map --from-run `, or + `--from-dir ` on a locally downloaded `exercised-map-a` artifact + when the download hits HTTP 403, and open the change as an ordinary PR. + `bun run test:exercised-map --check` validates the committed file. +- **Refresh contract:** a refresh whose input is unavailable or partial writes + nothing, keeps the committed union and exits with a stable code that names + what was missing: `2` usage; `3` input unavailable (a missing directory, or a + failed or HTTP 403 download); `4` input incomplete (no `shards/`, no + `trial-outcomes.jsonl`, an executed case without a shard artifact, or a missing + or malformed `exercised-commands.json`). The census step reports a failed + refresh in its job summary and uploads no map. +- **Replay receipt:** `bun run test:exercised-map --replay` replays the recorded + red PR runs in `test/fixtures/exercised-map-replay.json` (every executed failed + gate trial in 225 `evals.yml` pull_request runs, 2026-10-07 to 2026-10-17) + through today's selector and its shadow, and exits 1 on any MISS: a failed + gate case the shadow would not run. `test/exercised-map.test.ts` pins zero. + ### PR receipts, recovery and debounce Passes and failures of single-trial shards are written as receipts at the diff --git a/package.json b/package.json index 5bb38fc1d..1a72f3fc1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.129", + "version": "1.91.130", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", @@ -50,6 +50,7 @@ "eval:bg:gate": "bun run scripts/eval-bg.ts gate", "eval:bg:periodic": "bun run scripts/eval-bg.ts periodic", "test:health": "bun run scripts/test-health-report.ts", + "test:exercised-map": "bun run scripts/exercised-map.ts", "eval:list": "bun run scripts/eval-list.ts", "eval:compare": "bun run scripts/eval-compare.ts", "eval:summary": "bun run scripts/eval-summary.ts", diff --git a/scripts/exercised-map.ts b/scripts/exercised-map.ts new file mode 100644 index 000000000..126fa8b03 --- /dev/null +++ b/scripts/exercised-map.ts @@ -0,0 +1,384 @@ +#!/usr/bin/env bun +/** + * test:exercised-map — maintain test/helpers/exercised-commands.json, the + * observed half of the exercised-path consumer map (scripts/pr-dependencies.ts + * exercisedConsumers; shadow mode, docs/TESTING_INTERNALS.md#exercised-path-map). + * + * Refreshes are union-only: an observation adds commands, never removes one. + * A refresh whose input is unavailable or partial writes nothing, keeps the + * committed union and exits non-zero with a stable code naming what was + * missing: 2 usage, 3 input unavailable (missing directory, failed or HTTP 403 + * download), 4 input incomplete (no shards, no trial outcomes, an executed case + * without a shard artifact, a missing or malformed exercised-commands.json). + * Removing a command is a reviewed edit of the JSON file. + * + * Usage: + * bun run test:exercised-map --from-run [--out ] + * download that census run's exercised-map artifact and union it into the committed map + * bun run test:exercised-map --from-dir [--out ] + * the same from a locally downloaded artifact (the fallback when the download hits HTTP 403) + * bun run test:exercised-map --from-dir [--from-dir ...] --run-id [--out ] [--report ] + * read paid-slice / gate-census report directories (shards//...: transcripts, junit.xml, + * PTY terminal.visible.log, trial-outcomes.jsonl) and union their commands into the map + * bun run test:exercised-map --check + * validate the committed map + * bun run test:exercised-map --replay [] + * replay recorded red PR runs (default test/fixtures/exercised-map-replay.json) through + * today's selector and its shadow; a MISS is a failed gate case the shadow would not run + */ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { + checkoutExercisedConsumers, EXERCISED_MAP_PATH, parseExercisedMap, type ExercisedCase, type ExercisedConsumers, type ExercisedMap, +} from './pr-dependencies'; +import { caseOwnerFile, EXERCISED_MAP_ANCHOR, PR_FULL_GATE_FILES, PR_PROFILE_MAPS } from './test-pr-profile'; +import { CASE_TEST_NAMES } from './lib/paid-cases'; +import { computePaidCaseSelection } from './test-paid-shards'; + +const ROOT = path.resolve(import.meta.dir, '..'); +export const EXERCISED_MAP_ARTIFACT = 'exercised-map'; + +/** Commands one shard directory recorded: per Bun test name, and shard-level (no test name). */ +export interface ShardCommands { shard: string; executed: string[]; perTest: Record; shardLevel: string[] } + +function walk(dir: string): string[] { + return fs.readdirSync(dir, { withFileTypes: true }).flatMap(entry => entry.isDirectory() ? walk(path.join(dir, entry.name)) : [path.join(dir, entry.name)]); +} + +function toolCommands(node: unknown, out: string[]): void { + if (Array.isArray(node)) { for (const item of node) toolCommands(item, out); return; } + if (!node || typeof node !== 'object') return; + const record = node as Record; + if (record.type === 'tool_use' && typeof record.input?.command === 'string') out.push(record.input.command); + for (const value of Object.values(record)) if (value && typeof value === 'object') toolCommands(value, out); +} + +/** Every `shards//` directory under `dir`, with the Bash commands its transcripts, JSONL logs and PTY screens recorded. */ +export function readShardCommands(dir: string): ShardCommands[] { + if (!fs.existsSync(dir)) return []; + const shardDirs = walk(dir).map(file => path.relative(dir, file).split(path.sep)).flatMap(parts => { + const at = parts.lastIndexOf('shards'); + return at >= 0 && parts.length > at + 2 ? [path.join(dir, ...parts.slice(0, at + 2))] : []; + }); + return [...new Set(shardDirs)].sort().map(shardDir => { + const record: ShardCommands = { shard: path.basename(shardDir), executed: [], perTest: {}, shardLevel: [] }; + for (const file of walk(shardDir)) { + const base = path.basename(file); + try { + if (base === 'junit.xml') { + for (const match of fs.readFileSync(file, 'utf8').matchAll(/]*?)(\/>|>([\s\S]*?)<\/testcase>)/g)) { + const name = /\bname="([^"]*)"/.exec(match[1]!)?.[1]; + if (name && !/ test?.transcript)) { + for (const test of parsed.tests) { const out: string[] = []; toolCommands(test.transcript ?? [], out); if (out.length) (record.perTest[test.name] ??= []).push(...out); } + } else toolCommands(parsed, record.shardLevel); + } else if (base.endsWith('.jsonl')) { + for (const line of fs.readFileSync(file, 'utf8').split('\n')) if (line.includes('tool_use')) { try { toolCommands(JSON.parse(line), record.shardLevel); } catch { /* partial line */ } } + } else if (base === 'terminal.visible.log') { + for (const match of fs.readFileSync(file, 'utf8').matchAll(/Bash\(([^\r\n]{0,400})/g)) record.shardLevel.push(`PTY:${match[1]}`); + } + } catch { /* unreadable evidence file: its commands are simply not observed */ } + } + record.shardLevel = [...new Set(record.shardLevel)]; + for (const name of Object.keys(record.perTest)) record.perTest[name] = [...new Set(record.perTest[name])]; + return record; + }); +} + +const testNameCase = (name: string, ids: string[]): string | null => + ids.includes(name) ? name : Object.keys(CASE_TEST_NAMES).find(id => CASE_TEST_NAMES[id] === name && ids.includes(id)) ?? ids.find(id => name.includes(id)) ?? null; + +/** + * Raw commands per case. A case shard (`--`) is that case; a file + * shard's per-test commands go to the case its test name names, and its + * file-level commands to every case of the file that ran (by JUnit name, else + * all of them): conservative over-attribution, as in the measured prototype. + */ +export function attributeCommands(shards: ShardCommands[], maps = PR_PROFILE_MAPS): Map { + const ids = Object.keys(maps.e2eTouchfiles); + const byFile = new Map(); + for (const id of ids) { const owner = caseOwnerFile(id, maps); if (owner) byFile.set(owner, [...byFile.get(owner) ?? [], id]); } + const out = new Map(); + const add = (id: string, commands: string[]) => out.set(id, [...out.get(id) ?? [], ...commands]); + for (const shard of shards) { + const [stem, caseKey] = shard.shard.split('--'); + const caseId = caseKey?.replace(/\.t\d+$/, ''); + const owned = byFile.get(`test/${stem}.test.ts`) ?? []; + const ran = owned.filter(id => shard.executed.some(name => testNameCase(name, [id]) === id)); + const targets = caseId && ids.includes(caseId) ? [caseId] : ran.length ? ran : owned; + for (const id of targets) add(id, shard.shardLevel); + for (const [name, commands] of Object.entries(shard.perTest)) { + const id = testNameCase(name, targets) ?? testNameCase(name, ids); + for (const target of id ? [id] : targets) add(target, commands); + } + } + return out; +} + +/** Known trial cost per case summed over the trial-outcomes.jsonl files under `dir` (executed trials only). */ +export function readCaseCosts(dir: string): Map { + const out = new Map(); + if (!fs.existsSync(dir)) return out; + for (const file of walk(dir).filter(file => path.basename(file) === 'trial-outcomes.jsonl')) { + for (const line of fs.readFileSync(file, 'utf8').split('\n').filter(Boolean)) { + try { + const record = JSON.parse(line); + if (record.execution === 'executed' && record.cost_known !== false && typeof record.case === 'string') out.set(record.case, (out.get(record.case) ?? 0) + (Number(record.cost_usd) || 0)); + } catch { /* malformed line */ } + } + } + return out; +} + +/** One run's observations as a map: the bins each attributed case invoked, and its known cost. */ +export function observeRun(commandsByCase: Map, ex: Pick, run: { runId: string; observedAt: string; costs?: Map }): ExercisedMap { + const cases: Record = {}; + for (const [id, commands] of commandsByCase) { + cases[id] = { observations: 1, withCommands: commands.length ? 1 : 0, commands: [...new Set(commands.flatMap(command => ex.binsInCommand(command)))].sort() }; + } + for (const [id, cost] of run.costs ?? []) { + cases[id] ??= { observations: 1, withCommands: 0, commands: [] }; + Object.assign(cases[id], { costUsd: Math.round(cost * 10000) / 10000, costRuns: 1 }); + } + return { schema: 1, generatedAt: run.observedAt, observedThrough: run.observedAt, sourceRuns: [run.runId], cases }; +} + +/** + * Union-only merge: commands are unioned, counts add for runs the base has + * not seen, costs average by run count, and nothing the base holds is ever + * removed (an absent or empty case in `fresh` changes nothing). + */ +export function mergeExercisedMap(base: ExercisedMap, fresh: ExercisedMap, generatedAt = new Date().toISOString()): ExercisedMap { + const counts = fresh.sourceRuns.some(run => !base.sourceRuns.includes(run)); + const cases: Record = structuredClone(base.cases); + for (const [id, entry] of Object.entries(fresh.cases)) { + const old = cases[id]; + const merged: ExercisedCase = { + observations: (old?.observations ?? 0) + (counts ? entry.observations : 0), + withCommands: (old?.withCommands ?? 0) + (counts ? entry.withCommands : 0), + commands: [...new Set([...old?.commands ?? [], ...entry.commands])].sort(), + }; + const oldRuns = old?.costRuns ?? 0, freshRuns = counts && entry.costUsd !== undefined ? entry.costRuns ?? 1 : 0; + if (freshRuns) Object.assign(merged, { costUsd: Math.round(((old?.costUsd ?? 0) * oldRuns + entry.costUsd! * freshRuns) / (oldRuns + freshRuns) * 10000) / 10000, costRuns: oldRuns + freshRuns }); + else if (old?.costUsd !== undefined) Object.assign(merged, { costUsd: old.costUsd, costRuns: oldRuns }); + cases[id] = merged; + } + return { + schema: 1, generatedAt, + observedThrough: [base.observedThrough, fresh.observedThrough].sort().at(-1)!, + sourceRuns: [...new Set([...base.sourceRuns, ...fresh.sourceRuns])].sort(), + cases: Object.fromEntries(Object.entries(cases).sort(([a], [b]) => a.localeCompare(b))), + }; +} + +export function formatExercisedMap(map: ExercisedMap): string { + return `${JSON.stringify(map, null, 2)}\n`; +} + +/** `MAP_GAP `: a recorded command whose files all lie outside the case's static-plus-observed exercised closure. */ +export function mapGaps(commandsByCase: Map, ex: ExercisedConsumers): Array<{ id: string; command: string }> { + const gaps: Array<{ id: string; command: string }> = []; + for (const [id, commands] of [...commandsByCase].sort(([a], [b]) => a.localeCompare(b))) { + const closure = ex.closure(id); + for (const command of [...new Set(commands.flatMap(raw => ex.binsInCommand(raw)))].sort()) { + const files = ex.commandFiles(command); + if (files.length && !files.some(file => closure.has(file))) gaps.push({ id, command }); + } + } + return gaps; +} + +/** Executed cases that recorded no command although their loaded skill's shell runs bins: blind to gap detection. */ +export function blindCases(commandsByCase: Map, ex: ExercisedConsumers): string[] { + return [...commandsByCase].filter(([id, commands]) => commands.length === 0 + && [...ex.closure(id)].some(([file, label]) => file.startsWith('bin/') && label.startsWith('skill bash via'))).map(([id]) => id).sort(); +} + +/** Stable exit codes of test:exercised-map (docs/TESTING_INTERNALS.md#exercised-path-map). */ +export const REFRESH_EXIT = { usage: 2, unavailable: 3, incomplete: 4 } as const; + +/** A refresh that cannot be trusted: nothing is written and the committed union stays as it was. */ +export class RefreshError extends Error { + constructor(readonly exitCode: typeof REFRESH_EXIT.unavailable | typeof REFRESH_EXIT.incomplete, message: string) { super(message); } +} + +/** One MAP_GAP line, saying who it is for and what follows from it. */ +export function formatMapGap(gap: { id: string; command: string }): string { + return `MAP_GAP ${gap.id} ${gap.command}: non-blocking for this PR; the next census refresh widens the map (${EXERCISED_MAP_ANCHOR})`; +} + +function writeAtomically(file: string, text: string): void { + fs.mkdirSync(path.dirname(path.resolve(file)), { recursive: true }); + const temp = `${file}.${process.pid}.tmp`; + fs.writeFileSync(temp, text); + fs.renameSync(temp, file); +} + +/** + * Census refresh from local report directories (paid-slice artifacts under + * shards/, plus trial-outcomes.jsonl). Every executed case must have shard + * evidence; a missing directory, no shards, no trial outcomes or an executed + * case without a shard throws before anything is written. + */ +export function refreshFromReportDirs(dirs: string[], runId: string, base: ExercisedMap, ex: ExercisedConsumers, observedAt = new Date().toISOString(), maps = PR_PROFILE_MAPS) { + const missing = dirs.filter(dir => !fs.existsSync(dir)); + if (missing.length) throw new RefreshError(REFRESH_EXIT.unavailable, `report director${missing.length > 1 ? 'ies' : 'y'} not found: ${missing.join(', ')}`); + const shards = dirs.flatMap(readShardCommands); + if (shards.length === 0) throw new RefreshError(REFRESH_EXIT.incomplete, `no shards// directories under ${dirs.join(', ')}; download every paid-slice-* artifact of the run first`); + const outcomes = dirs.flatMap(dir => walk(dir).filter(file => path.basename(file) === 'trial-outcomes.jsonl')); + if (outcomes.length === 0) throw new RefreshError(REFRESH_EXIT.incomplete, `no trial-outcomes.jsonl under ${dirs.join(', ')}, so the executed cases cannot be checked against the shard evidence`); + const commands = attributeCommands(shards, maps); + const executed = new Set(outcomes.flatMap(file => fs.readFileSync(file, 'utf8').split('\n').flatMap(line => { + try { const record = JSON.parse(line); return record.execution === 'executed' && typeof record.case === 'string' ? [record.case as string] : []; } catch { return []; } + }))); + const uncovered = [...executed].filter(id => Object.hasOwn(maps.e2eTouchfiles, id) && !commands.has(id)).sort(); + if (uncovered.length) throw new RefreshError(REFRESH_EXIT.incomplete, `${uncovered.length} executed case(s) have no shard artifact: ${uncovered.slice(0, 10).join(', ')}${uncovered.length > 10 ? ' ...' : ''}`); + const costs = new Map(); + for (const dir of dirs) for (const [id, cost] of readCaseCosts(dir)) costs.set(id, (costs.get(id) ?? 0) + cost); + const merged = mergeExercisedMap(base, observeRun(commands, ex, { runId, observedAt, costs }), observedAt); + return { merged, commands }; +} + +function census(dirs: string[], runId: string, out: string, reportPath: string | undefined): void { + const ex = checkoutExercisedConsumers(PR_PROFILE_MAPS, ROOT, PR_FULL_GATE_FILES); + const base = ex.map!; + const { merged, commands } = refreshFromReportDirs(dirs, runId, base, ex); + writeAtomically(out, formatExercisedMap(merged)); + const added = Object.entries(merged.cases).flatMap(([id, entry]) => entry.commands.filter(command => !base.cases[id]?.commands.includes(command)).map(command => `${id} ${command}`)); + const gaps = mapGaps(commands, ex); + const blind = blindCases(commands, ex); + const lines = [ + '## Exercised-path map (shadow, non-blocking)', '', + `- ${commands.size} executed case(s), ${[...commands.values()].reduce((sum, list) => sum + list.length, 0)} recorded command(s); ${added.length} new observed edge(s).`, + `- Refresh: \`bun run test:exercised-map --from-run ${runId}\`, then open the change as an ordinary PR (shadow-only data: it selects no paid case).`, + `- MAP_GAP (a command outside its case's static-plus-observed closure): ${gaps.length}`, + ...gaps.map(gap => ` - ${formatMapGap(gap)}`), + `- Cases that recorded no commands although their skill runs bins (PTY capture is the activation prerequisite): ${blind.length ? blind.join(', ') : 'none'}`, + ...added.slice(0, 50).map(edge => ` - new: ${edge}`), + `- Docs: ${EXERCISED_MAP_ANCHOR}`, + ]; + console.log(lines.join('\n')); + if (reportPath) fs.writeFileSync(reportPath, `${lines.join('\n')}\n`); +} + +/** Union every exercised-commands.json of a downloaded exercised-map artifact into `base`; any missing or malformed file throws. */ +export function mergeArtifactDir(dir: string, base: ExercisedMap, generatedAt = new Date().toISOString()): ExercisedMap { + const files = fs.existsSync(dir) ? walk(dir).filter(file => path.basename(file) === 'exercised-commands.json') : []; + if (files.length === 0) throw new RefreshError(REFRESH_EXIT.incomplete, `no exercised-commands.json in ${dir} (the census refresh step failed or its ${EXERCISED_MAP_ARTIFACT}-a artifact is partial)`); + let map = base; + for (const file of files.sort()) { + let fresh: ExercisedMap; + try { fresh = parseExercisedMap(fs.readFileSync(file, 'utf8')); } + catch (error) { throw new RefreshError(REFRESH_EXIT.incomplete, `${path.relative(dir, file)} is not a complete exercised map: ${error instanceof Error ? error.message : String(error)}`); } + map = mergeExercisedMap(map, fresh, generatedAt); + } + return map; +} + +export type ArtifactDownload = (runId: string, dir: string) => { status: number | null; stderr: string }; + +const ghDownload: ArtifactDownload = (runId, dir) => { + const result = spawnSync('gh', ['run', 'download', runId, '--pattern', `${EXERCISED_MAP_ARTIFACT}-*`, '--dir', dir], { encoding: 'utf8', timeout: 120_000 }); + return { status: result.status, stderr: result.stderr || result.error?.message || '' }; +}; + +/** Download a census run's exercised-map artifact and union it into `target`; on any failure nothing is written. */ +export function refreshFromRun(runId: string, target: string, download: ArtifactDownload = ghDownload): ExercisedMap { + if (!/^\d+$/.test(runId)) throw new RefreshError(REFRESH_EXIT.unavailable, `--from-run needs a numeric GitHub run id, got ${runId}`); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'exercised-map-')); + try { + const result = download(runId, dir); + if (result.status !== 0) { + const reason = result.stderr.trim(); + const hint = /\b403\b/.test(reason) + ? `HTTP 403 (the installation's artifact download limit); download the ${EXERCISED_MAP_ARTIFACT}-a artifact from the run page and pass its directory with --from-dir` + : `check \`gh auth status\` and that run ${runId} uploaded an ${EXERCISED_MAP_ARTIFACT}-a artifact`; + throw new RefreshError(REFRESH_EXIT.unavailable, `gh run download ${runId} failed: ${reason || `exit ${result.status}`}; ${hint}`); + } + const map = mergeArtifactDir(dir, parseExercisedMap(fs.readFileSync(target, 'utf8'))); + writeAtomically(target, formatExercisedMap(map)); + return map; + } finally { fs.rmSync(dir, { recursive: true, force: true }); } +} + +export const REPLAY_FIXTURE = 'test/fixtures/exercised-map-replay.json'; + +export interface ReplayRun { runId: number; pr: number; packageVersionOnly: boolean; failedGateCases: string[]; changedFiles: string[] } + +/** One recorded red run under today's selection and its shadow: which failed gate cases each would run. */ +export interface ShadowReplayRow { runId: number; pr: number; mode: string; today: number; shadow: number; savingUsd: number; caughtToday: string[]; misses: string[]; retired: string[] } + +export function replayShadow(runs: readonly ReplayRun[]): ShadowReplayRow[] { + const gateIds = Object.keys(PR_PROFILE_MAPS.tiers).filter(id => PR_PROFILE_MAPS.tiers[id] === 'gate'); + return runs.map(run => { + const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: run.changedFiles, packageVersionOnly: run.packageVersionOnly }); + const today = new Set(result.selection.e2e ?? gateIds); + const shadow = result.coverage?.shadow; + if (shadow?.error) throw new Error(`run ${run.runId}: shadow unavailable (${shadow.error})`); + const shadowSet = new Set(shadow?.e2e ?? today); + const live = run.failedGateCases.filter(id => PR_PROFILE_MAPS.tiers[id] === 'gate'); + return { runId: run.runId, pr: run.pr, mode: result.coverage?.mode ?? 'full', today: today.size, shadow: shadowSet.size, savingUsd: shadow?.predictedSavingUsd ?? 0, + caughtToday: live.filter(id => today.has(id)), misses: live.filter(id => !shadowSet.has(id)), retired: run.failedGateCases.filter(id => !live.includes(id)) }; + }); +} + +export function formatShadowReplay(rows: readonly ShadowReplayRow[]): string[] { + const pairs = (list: readonly ShadowReplayRow[]) => list.reduce((sum, row) => sum + new Set([...row.caughtToday, ...row.misses]).size, 0); + const misses = rows.flatMap(row => row.misses.map(id => `${row.runId} #${row.pr} ${id}`)); + const modes = new Map(); + for (const row of rows) modes.set(row.mode, (modes.get(row.mode) ?? 0) + 1); + const mean = (pick: (row: ShadowReplayRow) => number) => rows.length ? (rows.reduce((sum, row) => sum + pick(row), 0) / rows.length).toFixed(1) : 'n/a'; + return [ + `Exercised-path shadow replay over ${rows.length} recorded red PR run(s) on this checkout`, + `- failed gate (run, case) pairs: ${pairs(rows)}, ${pairs(rows.filter(row => row.mode !== 'full-fallback'))} of them in runs that do not fall back to the full gate (${rows.reduce((sum, row) => sum + row.retired.length, 0)} more on cases no longer in the gate)`, + `- today's selection runs ${rows.reduce((sum, row) => sum + row.caughtToday.length, 0)} of them; modes ${[...modes].map(([mode, n]) => `${mode} ${n}`).join(', ')}`, + `- mean gate cases: today ${mean(row => row.today)}, shadow ${mean(row => row.shadow)}; predicted shadow saving over these runs $${rows.reduce((sum, row) => sum + row.savingUsd, 0).toFixed(2)}`, + `- MISSES (failed, not in the shadow selection): ${misses.length}`, + ...misses.map(miss => ` - ${miss}`), + ]; +} + +const USAGE = 'usage: bun run test:exercised-map --from-run [--out ] | --from-dir ... [--run-id ] [--out ] [--report ] | --check | --replay []'; + +if (import.meta.main) { + const argv = process.argv.slice(2); + const values = (flag: string) => argv.flatMap((arg, index) => arg === flag && argv[index + 1] ? [argv[index + 1]!] : []); + const out = values('--out')[0] ?? path.join(ROOT, EXERCISED_MAP_PATH); + try { + if (argv.includes('--check')) { + const map = parseExercisedMap(fs.readFileSync(path.join(ROOT, EXERCISED_MAP_PATH), 'utf8')); + console.log(`${EXERCISED_MAP_PATH}: ${Object.keys(map.cases).length} cases, ${map.sourceRuns.length} source runs, observed through ${map.observedThrough}`); + } else if (argv.includes('--replay')) { + const fixture = JSON.parse(fs.readFileSync(path.resolve(ROOT, values('--replay')[0] ?? REPLAY_FIXTURE), 'utf8')) as { runs: ReplayRun[] }; + const rows = replayShadow(fixture.runs); + console.log(formatShadowReplay(rows).join('\n')); + if (rows.some(row => row.misses.length)) process.exit(1); + } else if (values('--from-run').length) { + const runId = values('--from-run')[0]!; + const map = refreshFromRun(runId, out); + console.log(`union-merged run ${runId}'s ${EXERCISED_MAP_ARTIFACT} artifact into ${path.relative(ROOT, out)} (${Object.keys(map.cases).length} cases, observed through ${map.observedThrough}); review the diff and open it as a PR`); + } else if (values('--from-dir').length) { + const dirs = values('--from-dir'); + if (dirs.length === 1 && fs.existsSync(dirs[0]!) && readShardCommands(dirs[0]!).length === 0 && walk(dirs[0]!).some(file => path.basename(file) === 'exercised-commands.json')) { + const map = mergeArtifactDir(dirs[0]!, parseExercisedMap(fs.readFileSync(out, 'utf8'))); + writeAtomically(out, formatExercisedMap(map)); + console.log(`union-merged the ${EXERCISED_MAP_ARTIFACT} artifact in ${dirs[0]} into ${path.relative(ROOT, out)}; review the diff and open it as a PR`); + } else { + const runId = values('--run-id')[0]; + if (!runId) { console.error(`test:exercised-map: --from-dir with report directories needs --run-id (the run whose artifacts these are)\n${USAGE}`); process.exit(REFRESH_EXIT.usage); } + census(dirs, runId, out, values('--report')[0]); + } + } else { + console.error(USAGE); + process.exit(REFRESH_EXIT.usage); + } + } catch (error) { + console.error(`test:exercised-map: ${error instanceof Error ? error.message : String(error)}${error instanceof RefreshError ? '; nothing was written, the committed union is unchanged' : ''}`); + process.exit(error instanceof RefreshError ? error.exitCode : 1); + } +} diff --git a/scripts/harness-version.json b/scripts/harness-version.json index f37b3d4c3..a306a10bc 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -12,11 +12,11 @@ "files": { "scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":5}, "scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":5}, - "scripts/lib/paid-report.ts": {"blob":"3fda96a8b1f7f3d55797934847353f254cc4d054","since":5}, + "scripts/lib/paid-report.ts": {"blob":"b66c41237611ccecaaa7bd6a4d7b7aac8dc548cd","since":5,"nonBehavioral":"A2 exercised-path map: test-pr-profile.ts adds a shadow block and classifies the shadow-only map file as selecting no case (selection only); paid-report.ts writes non-blocking MAP_GAP lines after the verdict inputs, inside try/catch, never a verdict or lane problem"}, "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":5}, "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":5}, "scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":5}, - "scripts/test-pr-profile.ts": {"blob":"64cd7acfc0f2a4f53472bbdc8a50216984230c5a","since":5,"nonBehavioral":"PR selection only (A1 merged with #3253's free-only entry): shebang and per-extension import scanning, fail-closed scan errors, free-only repository-config files, capy/ derivable, the full-gate fallback label; it changes which cases a PR runs, never how any case is judged"}, + "scripts/test-pr-profile.ts": {"blob":"2029a89e14f511866cc91bebfee00b4fe9e7cf63","since":5,"nonBehavioral":"A2 exercised-path map: test-pr-profile.ts adds a shadow block and classifies the shadow-only map file as selecting no case (selection only); paid-report.ts writes non-blocking MAP_GAP lines after the verdict inputs, inside try/catch, never a verdict or lane problem"}, "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":5}, "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":5}, "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":5}, diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index 3fda96a8b..b66c41237 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -22,6 +22,9 @@ import { type ManifestEntry, PAID_TEST_DURATIONS_FILE, type PaidRunManifest, typ import { DEFAULT_JOBS, type PaidCaseSelection, type PaidTier, ROOT, type ShardTrialRecord } from './paid-types'; import { collectPaidTestFiles, isAllSkippedPass, shardSlug, paidSelectionEnv } from './paid-select'; import type { RunShardsOptions } from '../test-paid-shards'; +import { attributeCommands, formatMapGap, mapGaps, readShardCommands } from '../exercised-map'; +import { checkoutExercisedConsumers } from '../pr-dependencies'; +import { PR_FULL_GATE_FILES, PR_PROFILE_MAPS } from '../test-pr-profile'; // ─── Local diagnosis: one case through the CI panel runner (A9) ──────────── @@ -482,6 +485,30 @@ export function infraOnly(problems: readonly string[]): boolean { return problems.length > 0 && problems.every(problem => INFRA_PROBLEMS.some(re => re.test(problem))); } +/** + * Shadow gap detection (docs/TESTING_INTERNALS.md#exercised-path-map): a + * command an executed case ran outside its static-plus-observed exercised + * closure prints a MAP_GAP line (formatMapGap: non-blocking for this PR, the + * next census refresh widens the map) and lands in map-gaps.txt. It is its own + * result: never a case verdict, a lane problem or a re-dispatch input. + */ +export function reportMapGaps(reportDir: string, executedCases: ReadonlySet, rootDir = ROOT): string[] { + const out = path.join(reportDir, 'map-gaps.txt'); + try { + const commands = new Map([...attributeCommands(readShardCommands(reportDir))].filter(([id, list]) => executedCases.has(id) && list.length > 0)); + const lines = commands.size === 0 ? [] : mapGaps(commands, checkoutExercisedConsumers(PR_PROFILE_MAPS, rootDir, PR_FULL_GATE_FILES)).map(formatMapGap); + fs.writeFileSync(out, lines.length ? `${lines.join('\n')}\n` : ''); + console.log(`[test:paid] exercised-path map (shadow, non-blocking): ${lines.length} MAP_GAP line(s) over ${commands.size} executed case(s) with recorded commands`); + for (const line of lines) console.log(` ${line}`); + return lines; + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + fs.writeFileSync(out, `MAP_GAP unavailable: ${reason}\n`); + console.log(`[test:paid] exercised-path map (shadow, non-blocking): MAP_GAP unavailable: ${reason}`); + return []; + } +} + /** * Report mode: reconcile slice artifacts against the manifest (fail-closed), * compute every panel verdict with panelVerdict(), write collector-outcomes @@ -743,6 +770,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo ...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}) })) })), // evals.yml wraps these in a fixed fence for the PR comment. failures: failureLines.map(line => sanitizeFixedFenceLine(line)) }, null, 2) + '\n'); + reportMapGaps(reportDir, new Set(executed.map(record => record.case)), rootDir); if (verdict.problems.length) { console.error(`[test:paid] report: ${verdict.problems.length} problem(s):`); for (const problem of verdict.problems) console.error(` ✗ ${problem}`); diff --git a/scripts/pr-dependencies.ts b/scripts/pr-dependencies.ts index a6dc94e3a..077e01f71 100644 --- a/scripts/pr-dependencies.ts +++ b/scripts/pr-dependencies.ts @@ -90,6 +90,8 @@ export const SELECTION_DATA_MODULES: readonly string[] = [ 'scripts/build.sh', // The reason-code table: its `fix:` prose names the command a person reruns, not a file the module consumes. 'lib/result-codes.ts', 'lib/result-codes-release.ts', + // The exercised-path map: observed commands per case, read only by exercisedConsumers() (shadow data; a change selects no paid case). + 'test/helpers/exercised-commands.json', ]; const DEFAULT_JUDGE_OWNER = 'test/skill-llm-eval.test.ts'; @@ -326,3 +328,295 @@ export function derivedDependencies(maps: DependencyMaps, root: string): Derived return cache.get(key)!; } +// ─── Exercised-path consumer map (W2-A2, shadow mode) ────────────────────── + +/** The committed per-case observed commands (generated by scripts/exercised-map.ts; union-only refreshes). */ +export const EXERCISED_MAP_PATH = 'test/helpers/exercised-commands.json'; + +export interface ExercisedCase { + /** Recorded trials (run, case) the commands were taken from. */ + observations: number; + /** Of those, trials that recorded at least one command. */ + withCommands: number; + /** Bins and compiled outfiles the case invoked (binsInCommand over every recorded command). */ + commands: string[]; + /** Mean known trial cost per run, for the shadow cost estimate. */ + costUsd?: number; + costRuns?: number; +} + +export interface ExercisedMap { + schema: 1; + generatedAt: string; + observedThrough: string; + sourceRuns: string[]; + cases: Record; +} + +export function isExercisedMap(value: unknown): value is ExercisedMap { + const map = value as ExercisedMap; + return !!map && map.schema === 1 && typeof map.generatedAt === 'string' && typeof map.observedThrough === 'string' + && Array.isArray(map.sourceRuns) && map.sourceRuns.every(run => typeof run === 'string') + && !!map.cases && typeof map.cases === 'object' && Object.values(map.cases).every(entry => !!entry + && Number.isSafeInteger(entry.observations) && Number.isSafeInteger(entry.withCommands) + && Array.isArray(entry.commands) && entry.commands.every(command => typeof command === 'string')); +} + +/** Parse an exercised map's text; throws on malformed input so callers fail closed. */ +export function parseExercisedMap(text: string): ExercisedMap { + const parsed = JSON.parse(text); + if (!isExercisedMap(parsed)) throw new Error(`${EXERCISED_MAP_PATH}: not a schema-1 exercised map`); + return parsed; +} + +export interface ExercisedConsumers { + /** Repository file -> case ids whose exercised closure contains it. */ + e2e: Map>; + /** One case's exercised closure: file -> how it is reached (touchfile, import, skill bash via , observed , declared via , host). */ + closure(caseId: string): Map; + /** Bins (repo paths) and compiled outfiles one recorded command invokes. */ + binsInCommand(command: string): string[]; + /** The tracked files a bin or compiled outfile runs. */ + commandFiles(command: string): string[]; + /** The observed map the consumers were built with. */ + map: ExercisedMap | null; +} + +const SHELL_SHEBANG = /^#![^\n]*\b(?:ba|z|da)?sh\b/; +const MESSAGE_CALL = /^(?:echo|printf|say|warn|die|fail|err|log|info|note|_dc_row|row)\b/; +const CMD_START = /(?:^|[;|&({\x60]|\$\(|\b(?:then|do|else|exec|command|nohup|bash|sh|bun|node|timeout\s+\S+|env(?:\s+\w+=\S*)*|if|while|until|!|time))\s*"?$/; +const CMD_PREFIX = /(?:"?(?:\$\{?\w+\}?|\$\(dirname\s+"?\$0"?\)|~|\.)?(?:\/(?:[\w.@-]+\/)*|(?:[\w.@-]+\/)+))?"?$/; +const INTERPRETER = /\b(?:bash|sh|zsh|bun|node|eval|source|python3?)\b/; + +/** Text between `$(`…`)` and backticks in a span: substitutions run even inside a message. */ +function substitutions(span: string): string[] { + const out: string[] = []; + for (let i = 0; i < span.length; i++) { + if (span[i] === '\\') { i++; continue; } + if (span[i] === '\x60') { const end = span.indexOf('\x60', i + 1); if (end < 0) break; out.push(`$(${span.slice(i + 1, end)})`); i = end; continue; } + if (span[i] !== '$' || span[i + 1] !== '(') continue; + let depth = 0, j = i + 1; + for (; j < span.length; j++) { if (span[j] === '(') depth++; else if (span[j] === ')' && --depth === 0) break; } + out.push(span.slice(i, j + 1)); + i = j; + } + return out; +} + +/** Message calls (echo, printf, die …) are prose; their command substitutions still execute and are kept. */ +function stripMessages(text: string): string { + let out = '', i = 0; + while (i < text.length) { + const prev = i === 0 ? '\n' : text[i - 1]!; + const call = /[\s;|&(\x60{]/.test(prev) ? MESSAGE_CALL.exec(text.slice(i, i + 8)) : null; + if (!call) { out += text[i]; i++; continue; } + let j = i + call[0].length, quote: string | null = null, depth = 0; + while (j < text.length) { + const c = text[j]!; + if (c === '\\') { j += 2; continue; } + if (quote) { if (c === quote) quote = null; else if (quote === '"' && c === '$' && text[j + 1] === '(') { depth++; j++; } else if (c === ')' && depth) depth--; j++; continue; } + if (c === '"' || c === "'") { quote = c; j++; continue; } + if (c === '$' && text[j + 1] === '(') { depth++; j += 2; continue; } + if (c === ')' && depth) { depth--; j++; continue; } + if (c === '\n' || c === ';' || c === '|' || c === '&' || c === ')') break; + j++; + } + out += ` ${substitutions(text.slice(i, j)).join('; ')} `; + i = j; + } + return out; +} + +/** + * Executable shell text: comments, case patterns and message calls are + * prose. A heredoc an interpreter reads stays whole; an expanding (unquoted) + * heredoc keeps its command substitutions; a quoted one fed to anything else + * is dropped. + */ +export function executableShell(text: string): string { + return stripMessages(text + .replace(/^(.*?)<<-?\s*(['"]?)(\w+)\2([^\n]*)\n([\s\S]*?)\n\s*\3\s*$/gm, (whole, head: string, quote: string, _word: string, rest: string, body: string) => + INTERPRETER.test(head + rest) ? whole : quote ? head : [head, ...substitutions(body)].join('\n')) + .split('\n').filter(line => !/^\s*#(?!!)/.test(line)) + .map(line => line.replace(/(^\s*|\bin\s+|;;\s*)[\w*."'-]+(?:\s*\|\s*[\w*."'-]+)*\)(?=\s|$)/g, '$1').replace(/\s#\s.*$/, '')) + .join('\n')); +} + +function commandPosition(before: string): boolean { + const head = before.replace(CMD_PREFIX, ''); + return CMD_START.test(head) || (/\/$/.test(before) && /\b\w+="?$/.test(head)); +} + +/** + * Consumers by exercised paths: a case consumes its touchfile matches, the + * closure of its test file, the executable shell of its loaded skill + * (template and generated SKILL.md) and of every bin, lib and script that + * shell runs, and the bins its recorded trials invoked (`observed`). The + * closure stops at `stopAt` (the full-gate files) and the global set, + * which stay full. + */ +export function exercisedConsumers(maps: DependencyMaps, root: string, options: { observed: ExercisedMap | null; stopAt: readonly string[] }): ExercisedConsumers { + const base = derivedDependencies(maps, root); + const tracked = [...base.tracked]; + const texts = new Map(); + const read = (file: string): string => { + if (!texts.has(file)) { + let text = ''; + try { if (base.tracked.has(file) && fs.statSync(path.join(root, file)).size < 3_000_000) text = fs.readFileSync(path.join(root, file), 'utf8'); } catch { /* unreadable */ } + texts.set(file, text); + } + return texts.get(file)!; + }; + const isShell = (file: string) => !isCodeFile(file, read(file)) && (/\.sh$/.test(file) || file === 'setup' || SHELL_SHEBANG.test(read(file))); + const dirs = new Set(tracked.flatMap(file => file.split('/').slice(0, -1).map((_, i, parts) => parts.slice(0, i + 1).join('/')))); + const expandRef = (ref: string): string[] => { + const clean = ref.replace(/[.]+$/, ''); + if (base.tracked.has(clean)) return [clean]; + return dirs.has(clean) ? tracked.filter(file => file.startsWith(`${clean}/`)) : []; + }; + const expanded = new Map(); + const expand = (pattern: string): string[] => { + if (!pattern.includes('*')) return base.tracked.has(pattern) ? [pattern] : []; + if (!expanded.has(pattern)) expanded.set(pattern, tracked.filter(file => matchGlob(file, pattern))); + return expanded.get(pattern)!; + }; + const binByName = new Map(); + for (const file of tracked.filter(file => file.startsWith('bin/') || file.startsWith('browse/bin/'))) { + binByName.set(path.posix.basename(file), file); + binByName.set(path.posix.basename(file).replace(/\.(?:ts|sh|js)$/, ''), file); + } + const binaries = [...read('scripts/build.sh').matchAll(/--compile\b[^\n]*?\s([\w./-]+\.ts)\s+--outfile\s+([\w./-]+)/g)] + .map(match => ({ outfile: match[2]!, tree: match[1]!.startsWith('bin/') ? [match[1]!] : tracked.filter(file => matchGlob(file, `${path.posix.dirname(match[1]!)}/**`)) })); + const outfileHit = (outfile: string, text: string) => text.includes(outfile) + || (outfile === 'browse/dist/browse' && /\$B\b|\$\{B\}|find-browse/.test(text)) || (outfile === 'design/dist/design' && /\$D\b|\$\{D\}/.test(text)); + let scripts: Record = {}; + try { scripts = JSON.parse(read('package.json')).scripts ?? {}; } catch { /* no package scripts */ } + + const shellEdges = (text: string, from: string, out: Set) => { + for (const match of text.matchAll(/gstack-[a-z0-9][a-z0-9-]*[a-z0-9](?:\.(?:sh|ts))?\b/g)) { + const start = text.lastIndexOf('\n', match.index! - 1) + 1; + const bin = binByName.get(match[0]); + if (bin && commandPosition(text.slice(start, match.index))) out.add(bin); + } + for (const match of text.matchAll(/(?:bin|lib|scripts|browse|design|make-pdf)\/[\w.@-]+(?:\/[\w.@-]+)*/g)) { + const before = text.slice(text.lastIndexOf('\n', match.index! - 1) + 1, match.index); + if (/[\w.@-]$/.test(before)) continue; + if (commandPosition(before) || /\bbun\s+(?:run\s+)?"?(?:\$\{?\w+\}?\/)?$/.test(before)) for (const file of expandRef(match[0])) out.add(file); + } + for (const match of text.matchAll(/(?:bin|lib|scripts|browse|design|make-pdf|hosts|capy|test\/fixtures|test\/helpers)\/[\w.@-]+(?:\/[\w.@-]+)*\.(?:json|jsonl|env|txt|ya?ml|toml|csv|tsv)\b/g)) for (const file of expandRef(match[0])) out.add(file); + for (const match of text.matchAll(/(?:^|[\s;&|({])(?:\.|source)\s+([^;|&\n]+)/gm)) { + for (const ref of match[1]!.matchAll(/(?:bin|lib|scripts)\/[\w.@-]+(?:\/[\w.@-]+)*\.sh\b/g)) for (const file of expandRef(ref[0])) out.add(file); + const name = [...match[1]!.matchAll(/([\w.-]+\.sh|gstack-[\w.-]+)/g)].map(found => found[1]!).at(-1); + if (!name) continue; + const sibling = path.posix.join(path.posix.dirname(from), name); + if (base.tracked.has(sibling)) out.add(sibling); + else if (binByName.has(name)) out.add(binByName.get(name)!); + } + if (/(?:^|[\s;&|(])"?(?:\.|\$\{?\w+\}?)\/setup\b/m.test(text) && base.tracked.has('setup')) out.add('setup'); + for (const { outfile, tree } of binaries) if (outfileHit(outfile, text)) for (const file of tree) out.add(file); + for (const match of text.matchAll(PACKAGE_RUN)) { + const command = scripts[match[1]!]; + if (command) for (const ref of command.matchAll(REPO_PATH)) for (const file of expandRef(ref[1]!)) out.add(file); + } + }; + const edges = new Map(); + const execRefs = (file: string): string[] => { + if (edges.has(file)) return edges.get(file)!; + const raw = read(file); + const out = new Set(); + if (raw && !(SELECTION_DATA_MODULES as readonly string[]).includes(file)) { + if (isCodeFile(file, raw)) { + try { for (const specifier of scanCodeImports(file, raw)) { const hit = resolveSpecifier(root, specifier, file); if (hit) out.add(hit); } } + catch (error) { base.scanErrors.set(file, String(error)); } + const code = raw.replace(/^#![^\n]*/, '').replace(/\/\*[\s\S]*?\*\//g, '').replace(/^[ \t]*\/\/.*$/gm, ''); + for (const match of code.matchAll(/['"\x60](?:\$\{[^}]+\}\/)?((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)['"\x60]/g)) for (const hit of expandRef(match[1]!)) out.add(hit); + for (const match of code.matchAll(JOINED_PATH)) for (const hit of expandRef(`${match[1]}/${match[2]}`)) out.add(hit); + if (/join\([^)\n]*['"]setup['"]\s*\)/.test(code) && base.tracked.has('setup')) out.add('setup'); + for (const { outfile, tree } of binaries) if (code.includes(`'${outfile}'`) || code.includes(`"${outfile}"`)) for (const hit of tree) out.add(hit); + } else if (isShell(file)) { + shellEdges(executableShell(raw), file, out); + } else if (/\.(?:md|tmpl)$/.test(file)) { + shellEdges(executableShell([...raw.matchAll(/\x60\x60\x60(?:bash|sh|shell)\n([\s\S]*?)\x60\x60\x60/g)].map(match => match[1]!).join('\n')), file, out); + } + } + out.delete(file); + edges.set(file, [...out]); + return edges.get(file)!; + }; + const stopped = new Map(); + const stops = (file: string): boolean => { + if (!stopped.has(file)) stopped.set(file, base.global.has(file) || options.stopAt.some(pattern => matchGlob(file, pattern))); + return stopped.get(file)!; + }; + const binsInCommand = (command: string): string[] => { + const out = new Set(); + const pty = command.startsWith('PTY:'); + for (const match of command.matchAll(/gstack-[a-z0-9-]*[a-z0-9](?:\.(?:sh|ts))?/g)) { + let token = match[0]; + if (pty) while (token.length > 7 && !binByName.has(token)) { const cut = token.lastIndexOf('-'); token = cut > 6 ? token.slice(0, cut) : ''; } + const at = match.index!; + if (command[at - 1] === '/' && at >= 2 && /[\s"'\x60(]/.test(command[at - 2]!)) continue; + const bin = binByName.get(token); + if (bin) out.add(bin); + } + for (const match of command.matchAll(/browse\/bin\/([a-z-]+)/g)) { const bin = binByName.get(match[1]!); if (bin) out.add(bin); } + for (const { outfile } of binaries) if (outfileHit(outfile, command) || (outfile === 'make-pdf/dist/pdf' && command.includes(outfile))) out.add(outfile); + return [...out]; + }; + const commandFiles = (command: string) => base.tracked.has(command) ? [command] : binaries.find(binary => binary.outfile === command)?.tree ?? []; + + const hostIds = tracked.filter(file => /^hosts\/[\w-]+\.ts$/.test(file) && file !== 'hosts/index.ts').map(file => path.posix.basename(file, '.ts')); + const closures = new Map>(); + const caseClosure = (id: string): Map => { + if (closures.has(id)) return closures.get(id)!; + const patterns = maps.e2eTouchfiles[id] ?? []; + const owners = patterns.filter(pattern => OWNER.test(pattern) && isPaidTestFile(pattern)); + const declared = patterns.flatMap(expand); + const generated = declared.filter(file => file.endsWith('.tmpl')).map(file => file.replace(/\.tmpl$/, '')).filter(file => base.tracked.has(file)); + const touchfiles = new Set([...declared, ...generated]); + const skillDoc = (file: string) => /\.(?:md|tmpl)$/.test(file); + const seeds: Array<[string, string]> = [ + ...owners.map(file => [file, 'import'] as [string, string]), + ...[...declared, ...generated].map(file => [file, skillDoc(file) ? 'skill' : `declared via ${file}`] as [string, string]), + ...(options.observed?.cases[id]?.commands ?? []).flatMap(command => commandFiles(command).map(file => [file, `observed ${command}`] as [string, string])), + ]; + const ownerText = owners.map(read).join('\n'); + for (const host of hostIds) { + if (new RegExp(`['"\x60]${host}['"\x60]|--host[ =]${host}\\b`).test(ownerText)) for (const file of [`hosts/${host}.ts`, ...expand(`hosts/${host}/**`)]) seeds.push([file, 'host']); + } + const reached = new Map(); + const queue = seeds.filter(([file]) => base.tracked.has(file)); + for (let head = 0; head < queue.length; head++) { + const [file, label] = queue[head]!; + if (reached.has(file)) continue; + reached.set(file, touchfiles.has(file) ? 'touchfile' : label); + if (stops(file)) continue; + for (const next of execRefs(file)) { + if (reached.has(next) || (/\.test\.tsx?$/.test(next) && isPaidTestFile(next))) continue; + queue.push([next, label === 'skill' ? `skill bash via ${next}` : label]); + } + } + closures.set(id, reached); + return reached; + }; + const e2e = new Map>(); + for (const id of Object.keys(maps.e2eTouchfiles)) { + for (const file of caseClosure(id).keys()) { + if (!e2e.has(file)) e2e.set(file, new Set()); + e2e.get(file)!.add(id); + } + } + return { e2e, closure: caseClosure, binsInCommand, commandFiles, map: options.observed }; +} + +const exercisedCache = new Map(); + +/** The exercised consumers of the checkout at `root` with its committed map, computed once per process; throws when the map is malformed. */ +export function checkoutExercisedConsumers(maps: DependencyMaps, root: string, stopAt: readonly string[]): ExercisedConsumers { + const key = `${root}\0${Object.keys(maps.e2eTouchfiles).length}`; + if (!exercisedCache.has(key)) { + const observed = parseExercisedMap(fs.readFileSync(path.join(root, EXERCISED_MAP_PATH), 'utf8')); + exercisedCache.set(key, exercisedConsumers(maps, root, { observed, stopAt })); + } + return exercisedCache.get(key)!; +} diff --git a/scripts/test-health-report.ts b/scripts/test-health-report.ts index cf11e4f3e..bccc11639 100644 --- a/scripts/test-health-report.ts +++ b/scripts/test-health-report.ts @@ -228,6 +228,30 @@ export function collectHealth(inputs: HealthInputs): HealthReport { detail: modes.map(mode => { const list = entries.filter(entry => (modeOf.get(entry.run.id) ?? 'no manifest') === mode); return `${mode}: ${mean(list)} (${list.length} runs)`; }) }; })); + metrics.push(metric('pr-shadow-map', 'Exercised-path map shadow (not active)', 'over evals.yml pull_request runs whose paid-plan manifest carries prCoverage.shadow: the predicted saving (map per-case known cost of dropped minus added cases; a lower bound, since known cost excludes PTY trials with unknown cost) and shadow drops of red cases (more executed failed than passed trials in the run, on a case the shadow would drop). A drop counts toward the D3 activation bar as a miss unless the case passed in a later run of the same branch at the same head SHA, which marks it a flaky drop. Scaled to 7 days; docs/TESTING_INTERNALS.md#exercised-path-map', () => { + const shadowed = need(prManifests).filter(({ manifest }) => manifest.prCoverage?.shadow && !manifest.prCoverage.shadow.error); + if (shadowed.length === 0) throw new Error('no paid-plan manifest with a prCoverage.shadow block in the window'); + const records = prRecords instanceof Error ? [] : prRecords; + const executed = (trials: TrialRecord[], id: string, outcome: string) => trials.filter(record => record.case === id && record.execution === 'executed' && record.outcome === outcome).length; + const saving = shadowed.reduce((sum, { manifest }) => sum + (Number(manifest.prCoverage.shadow.predictedSavingUsd) || 0), 0); + const shares = shadowed.map(({ manifest }) => Number(manifest.prCoverage.shadow.unknownCostShare)).filter(share => Number.isFinite(share) && share > 0); + const misses: string[] = []; + const flaky: string[] = []; + for (const { run, manifest } of shadowed) { + const trials = records.find(entry => entry.run.id === run.id)?.trials ?? []; + for (const id of (manifest.prCoverage.shadow.wouldDrop ?? []) as string[]) { + if (executed(trials, id, 'failed') <= executed(trials, id, 'passed')) continue; + const laterPass = records.find(entry => entry.run.branch === run.branch && entry.run.sha === run.sha + && (entry.run.createdAt > run.createdAt || (entry.run.id === run.id && entry.run.attempt > run.attempt)) && executed(entry.trials, id, 'passed') > 0); + if (laterPass) flaky.push(`${run.id}:${id} (passed in run ${laterPass.run.id})`); else misses.push(`${run.id}:${id}`); + } + } + const week = 7 / sinceDays; + const bound = shares.length ? `a lower bound: known cost excludes the ${Math.round(100 * Math.max(...shares))}% of observed case runs with unknown cost (PTY)` : 'a lower bound: known cost excludes PTY trials with unknown cost'; + return { value: `$${(saving * week).toFixed(2)} predicted saving/week (${bound}); ${(misses.length * week).toFixed(1)} activation miss(es)/week, ${flaky.length} flaky drop(s) not counted (${shadowed.length} runs)`, + detail: [...misses.slice(0, 20).map(miss => `shadow miss: ${miss}`), ...flaky.slice(0, 20).map(drop => `flaky drop: ${drop}`)] }; + })); + metrics.push(metric('pr-reused-trials', 'Reused PR trial records', 'trial-outcomes records with execution = reused across non-cancelled PR runs', () => { const all = need(prRecords).flatMap(entry => entry.trials); const reused = all.filter(record => record.execution === 'reused').length; diff --git a/scripts/test-pr-profile.ts b/scripts/test-pr-profile.ts index 64cd7acfc..2029a89e1 100644 --- a/scripts/test-pr-profile.ts +++ b/scripts/test-pr-profile.ts @@ -5,7 +5,9 @@ import { E2E_TOUCHFILES, E2E_TIERS, GLOBAL_TOUCHFILES, LLM_JUDGE_TOUCHFILES } fr import { matchGlob, TOUCHFILES_DATA_PATH } from '../test/helpers/test-selection'; import { isPaidTestFile } from '../test/helpers/paid-test-set'; import { FREE_FIXTURES } from '../test/helpers/free-fixtures-data'; -import { derivedDependencies, type DerivedDependencies } from './pr-dependencies'; +import { + checkoutExercisedConsumers, derivedDependencies, EXERCISED_MAP_PATH, type DerivedDependencies, type ExercisedConsumers, +} from './pr-dependencies'; /** Existing short behavioral probes; intersect with changed-input selection. */ export const PR_PROFILE_CASE_IDS = [ @@ -157,6 +159,27 @@ export interface PrProfileSelection { missingCoverage: string[]; needsFullValidation: boolean; reasons: string[]; + /** What the exercised-path map would select (shadow mode: never changes e2e or judges). */ + shadow?: ShadowSelection; +} + +/** The exercised-path map's selection beside the real one (docs/TESTING_INTERNALS.md#exercised-path-map). */ +export interface ShadowSelection { + /** Gate cases the map would select. */ + e2e: string[]; + /** Selected gate cases the map would not select. */ + wouldDrop: string[]; + /** Gate cases the map would select that today's selection does not. */ + wouldAdd: string[]; + /** Per changed file with exercised consumers: how many gate cases it reaches, and through which edge. */ + provenance: Array<{ file: string; summary: string }>; + /** Mean known cost of the dropped cases minus the added ones (the map's per-case costUsd; an unpriced case counts $0): a lower bound. */ + predictedSavingUsd: number; + unpricedCases: number; + /** Share of observed case runs whose trials carry no known cost (PTY trials); their spend is missing from every dollar figure. */ + unknownCostShare: number; + /** Why no shadow could be computed (the real selection is unaffected). */ + error?: string; } /** Reject stale profile registrations and deferred cases with no scheduled home. */ @@ -297,6 +320,7 @@ export function knownNonBehaviorFile(file: string): boolean { } export const FALLBACK_FIX_ANCHOR = 'docs/TESTING_INTERNALS.md#pr-paid-lane-fallback'; +export const EXERCISED_MAP_ANCHOR = 'docs/TESTING_INTERNALS.md#exercised-path-map'; /** Why one changed file restored the full gate, and the edit that narrows it next time (DX-6). */ export function unknownFileLabel(file: string): { label: string; fix: string } { @@ -348,6 +372,8 @@ export function selectPrProfile(options: { readSource?: (file: string) => string; /** Derived references; default: derived from the checkout for the real maps, none for fixture maps. */ derived?: DerivedDependencies | null; + /** Exercised-path consumers for the shadow block; default: the checkout's committed map for the real maps, none for fixture maps. */ + exercised?: ExercisedConsumers | null; }): PrProfileSelection { const maps = options.maps ?? PR_PROFILE_MAPS; const profile = options.profile ?? PR_PROFILE_CASE_IDS; @@ -361,7 +387,7 @@ export function selectPrProfile(options: { ...Object.values(maps.e2eTouchfiles).flat(), ...Object.values(maps.judgeTouchfiles).flat(), ...maps.globalTouchfiles, ])]; - const unmapped = files.filter(file => file !== TOUCHFILES_DATA_PATH + const unmapped = files.filter(file => file !== TOUCHFILES_DATA_PATH && file !== EXERCISED_MAP_PATH && !depends(file, dependencyPatterns) && !knownNonBehaviorFile(file)); // Unmapped files are placed by derived references where the derivation covers them; the rest restore the full gate. const derived = options.derived !== undefined ? options.derived : maps === PR_PROFILE_MAPS ? derivedDependencies(maps, ROOT) : null; @@ -438,25 +464,80 @@ export function selectPrProfile(options: { if (sharedInputs.length) reasons.push(`Shared runtime/build inputs restore every gate case and judge: ${sharedInputs.join(', ')}`); if (dependentsMode) reasons.push(`Derived references select every dependent gate case: ${derivedFiles.map(file => `${file} (${consumers.get(file)!.e2e.size} cases)`).join(', ')}`); else if (!fallback) reasons.push('Changed-input selection intersected with the fast PR profile; selected judges retained'); + if (files.includes(EXERCISED_MAP_PATH)) reasons.push(`${EXERCISED_MAP_PATH} is shadow-only data (it changes no selection yet), so it selects no paid case (${EXERCISED_MAP_ANCHOR})`); if (derivedGlobal.length) reasons.push(`Paid-runner inputs (global touchfile import closure) select every case: ${derivedGlobal.join(', ')}`); if (noConsumerFiles.length) reasons.push(`No paid case's reference closure reaches (or, deleted, still references): ${noConsumerFiles.join(', ')}`); if (directCases.length) reasons.push(`Edited paid test files select their own gate cases: ${directCases.join(', ')}`); if (deferred.length) reasons.push(`${deferred.length} selected behaviors remain scheduled/release coverage, not PR passes`); if (deferredPromptFiles.length) reasons.push(`No quick live coverage; known broad prompt checks deferred: ${deferredPromptFiles.join(', ')}`); if (missingCoverage.length) reasons.push(`Full validation required for prompts without a relevant PR check: ${missingCoverage.join(', ')}`); + const exercised = options.exercised !== undefined ? options.exercised : maps === PR_PROFILE_MAPS ? checkoutExercised() : null; + const shadow = exercised instanceof Error ? emptyShadow(e2e, exercised.message) + : exercised ? shadowSelection({ ...options, maps, profile }, { e2e, fallback, derived, exercised, files }) : undefined; return { mode: fallback ? 'full-fallback' : dependentsMode ? 'dependents' : 'pr', e2e, judges, deferred, unknownFiles, unknownFileLabels: unknownFiles.map(file => ({ file, ...(deletedLabels.get(file) ?? unknownFileLabel(file)) })), directCases, derivedFiles: derivedFiles.map(file => ({ file, e2e: consumers.get(file)!.e2e.size, judges: consumers.get(file)!.judges.size })), noConsumerFiles, - deferredPromptFiles, missingCoverage, needsFullValidation: missingCoverage.length > 0, reasons, + deferredPromptFiles, missingCoverage, needsFullValidation: missingCoverage.length > 0, reasons, ...(shadow ? { shadow } : {}), }; } +function checkoutExercised(): ExercisedConsumers | Error { + try { return checkoutExercisedConsumers(PR_PROFILE_MAPS, ROOT, PR_FULL_GATE_FILES); } + catch (error) { return error instanceof Error ? error : new Error(String(error)); } +} + +function emptyShadow(e2e: string[], error: string): ShadowSelection { + return { e2e, wouldDrop: [], wouldAdd: [], provenance: [], predictedSavingUsd: 0, unpricedCases: 0, unknownCostShare: 0, error }; +} + +/** + * Shadow selection (variant cu of the measured prototype): the same policy + * with the exercised map in place of the derived closure, unioned with the + * exercised consumers of every changed file that is not global or full-gate. + * A full-gate fallback stays full. + */ +function shadowSelection(options: Parameters[0] & { maps: PrProfileMaps }, + today: { e2e: string[]; fallback: boolean; derived: DerivedDependencies | null; exercised: ExercisedConsumers; files: string[] }): ShadowSelection { + const { maps } = options; + const { exercised } = today; + const observedCases = Object.values(exercised.map?.cases ?? {}); + const observations = observedCases.reduce((sum, entry) => sum + entry.observations, 0); + const unknownCostShare = observations ? Math.round(1000 * (1 - observedCases.reduce((sum, entry) => sum + (entry.costRuns ?? 0), 0) / observations)) / 1000 : 0; + if (today.fallback) return { e2e: today.e2e, wouldDrop: [], wouldAdd: [], provenance: [], predictedSavingUsd: 0, unpricedCases: 0, unknownCostShare }; + const gate = (ids: Iterable) => [...ids].filter(id => maps.tiers[id] === 'gate'); + const reach = new Map(); + const inner = selectPrProfile({ ...options, derived: today.derived ? { ...today.derived, e2e: exercised.e2e } : null, exercised: null }); + const sources = [...new Set([...today.files, ...Object.values(options.sourceAliases ?? {})])]; + const shadowSet = new Set(inner.e2e); + for (const file of sources) { + if (today.derived?.global.has(file) || matches(file, PR_FULL_GATE_FILES) || knownNonBehaviorFile(file)) continue; + const via = !today.derived || today.derived.tracked.has(file) ? [file] : today.derived.referencers(file); + const ids = gate(new Set(via.flatMap(source => [...exercised.e2e.get(source) ?? []]))); + if (ids.length) reach.set(file, ids); + for (const id of ids) shadowSet.add(id); + } + const e2e = [...shadowSet].sort(); + const wouldDrop = today.e2e.filter(id => !shadowSet.has(id)); + const wouldAdd = e2e.filter(id => !today.e2e.includes(id)); + const cost = (ids: string[]) => ids.reduce((sum, id) => sum + (exercised.map?.cases[id]?.costUsd ?? 0), 0); + const provenance = [...reach].map(([file, ids]) => { + const groups = new Map(); + for (const id of ids) { + const label = exercised.closure(id).get(file) ?? 'referencer'; + groups.set(label, (groups.get(label) ?? 0) + 1); + } + return { file, summary: `selects ${ids.length} case(s): ${[...groups].sort((a, b) => b[1] - a[1]).slice(0, 6).map(([label, n]) => `${label} (${n})`).join(', ')}` }; + }); + return { e2e, wouldDrop, wouldAdd, provenance, predictedSavingUsd: Math.round((cost(wouldDrop) - cost(wouldAdd)) * 100) / 100, + unpricedCases: [...wouldDrop, ...wouldAdd].filter(id => exercised.map?.cases[id]?.costUsd === undefined).length, unknownCostShare }; +} + /** The manifest fields the coverage summary reads (a PaidRunManifest subset). */ export interface CoverageSummaryInput { profile?: string; selection?: { e2e: string[] | null; judges: string[] | null }; - prCoverage?: Pick & Partial>; + prCoverage?: Pick & Partial>; } /** @@ -477,6 +558,9 @@ export function formatPrCoverageSummary(manifest: CoverageSummaryInput, totals?: lines.push(`- Derived dependents (scripts/pr-dependencies.ts) of ${coverage.derivedFiles.length} unmapped file(s):`); for (const item of coverage.derivedFiles.slice(0, 40)) lines.push(` - \`${item.file}\`: ${item.e2e} case(s), ${item.judges} judge(s)`); } + const shadow = coverage?.shadow; + if (shadow) lines.push(shadow.error ? `- Shadow exercised-path map: unavailable (${shadow.error}); selection unaffected` + : `- Shadow exercised-path map (not active): would select ${shadow.e2e.length} gate case(s), dropping ${shadow.wouldDrop.length} and adding ${shadow.wouldAdd.length}; predicted saving $${shadow.predictedSavingUsd.toFixed(2)}${shadow.unpricedCases ? ` (${shadow.unpricedCases} unpriced)` : ''}, a lower bound: known cost excludes the ${Math.round(100 * (shadow.unknownCostShare ?? 0))}% of observed case runs with unknown cost (PTY) (${EXERCISED_MAP_ANCHOR})`); if (coverage?.noConsumerFiles?.length) lines.push(`- No paid case reaches: ${coverage.noConsumerFiles.slice(0, 40).map(file => `\`${file}\``).join(', ')}`); if (coverage?.mode === 'full-fallback' && coverage.unknownFiles.length) { lines.push(`- Full gate restored by ${coverage.unknownFiles.length} file(s) (fix: ${FALLBACK_FIX_ANCHOR}):`); diff --git a/test/exercised-map.test.ts b/test/exercised-map.test.ts new file mode 100644 index 000000000..9a3c5165a --- /dev/null +++ b/test/exercised-map.test.ts @@ -0,0 +1,357 @@ +/** + * Exercised-path consumer map (W2-A2, shadow mode): edge classes on a + * synthetic tree, the five variant-c miss shapes as negative controls, + * union-only refreshes, the refresh contract on unavailable or partial input, + * shadow-only map data selecting nothing, shadow output that never changes the + * real selection, MAP_GAP detection, and parity with the + * measured prototype (docs/TESTING_INTERNALS.md#exercised-path-map). + */ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { + deriveDependencies, EXERCISED_MAP_PATH, exercisedConsumers, parseExercisedMap, type ExercisedMap, +} from '../scripts/pr-dependencies'; +import { + formatPrCoverageSummary, PR_FULL_GATE_FILES, PR_PROFILE_MAPS, selectPrProfile, +} from '../scripts/test-pr-profile'; +import { + attributeCommands, blindCases, formatExercisedMap, formatMapGap, formatShadowReplay, mapGaps, mergeExercisedMap, observeRun, readCaseCosts, readShardCommands, + REFRESH_EXIT, RefreshError, refreshFromReportDirs, refreshFromRun, REPLAY_FIXTURE, replayShadow, type ArtifactDownload, type ReplayRun, +} from '../scripts/exercised-map'; +import { reportMapGaps } from '../scripts/lib/paid-report'; +import { computePaidCaseSelection } from '../scripts/test-paid-shards'; +import { prototypeConsumers } from './helpers/exercised-prototype'; + +const ROOT = path.resolve(import.meta.dir, '..'); + +function fixtureRepo(files: Record): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'exercised-map-')); + for (const [file, text] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); + fs.writeFileSync(path.join(root, file), text); + } + for (const args of [['init', '-q'], ['add', '-A']]) { + const result = spawnSync('git', args, { cwd: root, encoding: 'utf8', timeout: 10_000 }); + if (result.status !== 0) throw new Error(`git ${args[0]} failed: ${result.stderr}`); + } + return root; +} + +const map = (cases: ExercisedMap['cases'], sourceRuns = ['1'], observedThrough = '2026-10-01T00:00:00Z'): ExercisedMap => + ({ schema: 1, generatedAt: observedThrough, observedThrough, sourceRuns, cases }); + +const FENCE = '```'; +const skill = [ + '# alpha', '', 'Prose naming gstack-prose and /gstack-slash is not run.', '', FENCE + 'bash', + 'gstack-start --quiet', + 'echo "see gstack-echoed for details"', + 'echo "now $(gstack-subst --print)"', + '# gstack-commented', + 'case "$x" in gstack-casepat) true ;; esac', + 'VAR=gstack-bare', + '"$ROOT/bin/gstack-pathed" run', + '. "$ROOT/lib/sourced.sh"', + "cat <<'EOF'", 'gstack-quoted-heredoc', 'EOF', + 'cat < [`bin/gstack-${name}`, '#!/bin/sh\n'])), + 'test/skill-e2e-beta.test.ts': '', 'test/skill-e2e-gamma.test.ts': '', 'test/skill-e2e-delta.test.ts': '', + 'gamma/SKILL.md.tmpl': `# gamma\n\n${FENCE}bash\ngstack-config get proactive\n${FENCE}\n`, + 'scripts/build.sh': 'bun build --compile browse/src/cli.ts --outfile browse/dist/browse\n', 'browse/src/cli.ts': "import './page';\n", 'browse/src/page.ts': '', + 'scripts/thing.ts': '', 'scripts/resolvers/index.ts': '', 'setup': '#!/usr/bin/env bash\n', + 'package.json': JSON.stringify({ scripts: { 'build-thing': 'bun run scripts/thing.ts' } }), +}; +const maps = { + e2eTouchfiles: { + alpha: ['test/skill-e2e-alpha.test.ts', 'alpha/**'], beta: ['test/skill-e2e-beta.test.ts'], + gamma: ['test/skill-e2e-gamma.test.ts', 'gamma/**'], delta: ['test/skill-e2e-delta.test.ts', 'bin/gstack-config', 'lib/deep.ts'], + }, + judgeTouchfiles: {}, globalTouchfiles: [] as string[], +}; +const profileMaps = { ...maps, tiers: { alpha: 'gate', beta: 'gate', gamma: 'gate', delta: 'gate' } as const }; +const observed = map({ beta: { observations: 3, withCommands: 2, commands: ['bin/gstack-observed'], costUsd: 2, costRuns: 3 }, + alpha: { observations: 4, withCommands: 4, commands: [], costUsd: 5, costRuns: 4 }, gamma: { observations: 2, withCommands: 0, commands: [] } }); + +describe('exercised consumers on a synthetic tree', () => { + const root = fixtureRepo(tree); + const ex = exercisedConsumers(maps, root, { observed, stopAt: ['setup'] }); + const alpha = ex.closure('alpha'); + + test('each kept edge class reaches its file, with its provenance', () => { + expect(alpha.get('alpha/SKILL.md.tmpl')).toBe('touchfile'); + expect(alpha.get('bin/gstack-start')).toBe('skill bash via bin/gstack-start'); + expect(alpha.get('lib/deep.ts')).toBe('skill bash via bin/gstack-start'); + expect(alpha.get('lib/joined.ts')).toBe('import'); + expect(alpha.get('lib/quoted.ts')).toBe('import'); + for (const file of ['bin/gstack-subst', 'bin/gstack-pathed', 'lib/sourced.sh', 'bin/gstack-expanding', 'bin/gstack-interp', 'browse/src/page.ts', 'scripts/thing.ts', 'setup']) { + expect(alpha.has(file), file).toBe(true); + } + expect(ex.closure('beta').get('bin/gstack-observed')).toBe('observed bin/gstack-observed'); + }); + + test('messages, comments, case patterns, bare assignments, quoted and expanded heredoc prose, prose and slash commands are not edges', () => { + for (const name of ['echoed', 'commented', 'casepat', 'bare', 'quoted-heredoc', 'heredoc-prose', 'prose', 'slash']) { + expect(alpha.has(`bin/gstack-${name}`), name).toBe(false); + } + expect(ex.e2e.get('lib/unreached.ts')).toBeUndefined(); + }); + + test('the closure stops at a full-gate file and at the global set', () => { + const stopped = exercisedConsumers({ ...maps, globalTouchfiles: ['bin/gstack-start'] }, fixtureRepo(tree), { observed, stopAt: ['setup'] }); + expect(stopped.closure('alpha').has('bin/gstack-start')).toBe(true); + expect(stopped.closure('alpha').has('lib/deep.ts')).toBe(false); + }); + + test('bins in recorded commands: paths, bare names, PTY lines and compiled outfiles', () => { + expect(ex.binsInCommand('gstack-start --x && "$B" goto y')).toEqual(['bin/gstack-start', 'browse/dist/browse']); + expect(ex.binsInCommand('PTY:gstack-config-get proactive')).toEqual(['bin/gstack-config']); + expect(ex.binsInCommand('"$ROOT/bin/gstack-start" --x')).toEqual(['bin/gstack-start']); + expect(ex.binsInCommand('run /gstack-start next')).toEqual([]); + expect(ex.commandFiles('browse/dist/browse').sort()).toEqual(['browse/src/cli.ts', 'browse/src/page.ts']); + }); + + test('the variant-c miss shapes stay selected in the shadow (negative controls)', () => { + const shadow = (changedFiles: string[], selectedE2E: string[]) => selectPrProfile({ + maps: profileMaps, profile: ['alpha', 'beta', 'gamma', 'delta'], selectedE2E, selectedJudges: [], changedFiles, + derived: deriveDependencies(maps, root), exercised: ex, + }).shadow!; + // A preamble hub a touchfile maps only for another case (skill-start/end x3; gstack-config for the PTY case with no observations). + expect(shadow(['bin/gstack-config'], ['delta']).e2e).toEqual(['delta', 'gamma']); + expect(observed.cases.gamma!.withCommands).toBe(0); + // A lib reached only through a bin the skill runs (redact-patterns via gstack-redact). + expect(shadow(['lib/deep.ts'], ['delta']).e2e).toEqual(['alpha', 'delta']); + expect(shadow(['lib/deep.ts'], ['delta']).provenance).toEqual([{ file: 'lib/deep.ts', summary: 'selects 2 case(s): skill bash via bin/gstack-start (1), touchfile (1)' }]); + }); +}); + +describe('the five recorded variant-c misses on this checkout', () => { + const shadowOf = (file: string) => computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: [file], packageVersionOnly: true }).coverage!.shadow!; + + test('skill-start reds stay in the shadow of a gstack-skill-start change', () => { + const shadow = shadowOf('bin/gstack-skill-start'); + expect(shadow.error).toBeUndefined(); + for (const id of ['qa-functional-webhook-fix', 'qa-functional-cli-report', 'ship-exploratory-plan-checks']) expect(shadow.e2e, id).toContain(id); + }, 60_000); + + test('gstack-config reaches the PTY case that records no commands; redact-patterns reaches ship-docsync-store', () => { + expect(shadowOf('bin/gstack-config').e2e).toContain('plan-mode-no-op'); + const committed = parseExercisedMap(fs.readFileSync(path.join(ROOT, EXERCISED_MAP_PATH), 'utf8')); + const ex = exercisedConsumers(PR_PROFILE_MAPS, ROOT, { observed: committed, stopAt: PR_FULL_GATE_FILES }); + expect(ex.closure('plan-mode-no-op').get('bin/gstack-config')).toStartWith('skill bash via'); + expect(ex.closure('ship-docsync-store').has('lib/redact-patterns.ts')).toBe(true); + }, 60_000); +}); + +describe('shadow output never changes the real selection', () => { + test('mode, cases, judges and reasons equal the selection without a shadow', () => { + for (const files of [['bin/gstack-config'], ['design/src/evolve.ts'], ['review/SKILL.md.tmpl'], ['lib/fs-utils.ts', 'docs/x.md'], ['ETHOS.md']]) { + const withShadow = selectPrProfile({ selectedE2E: [], selectedJudges: [], changedFiles: files }); + const without = selectPrProfile({ selectedE2E: [], selectedJudges: [], changedFiles: files, exercised: null }); + expect(withShadow.shadow, files.join()).toBeDefined(); + const { shadow, ...rest } = withShadow; + expect(rest, files.join()).toEqual(without); + if (withShadow.mode === 'full-fallback') expect(shadow!.e2e).toEqual(withShadow.e2e); + } + }, 60_000); + + test('the coverage summary prints one shadow line; a broken map is reported, not fatal', () => { + const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: ['design/src/evolve.ts'], packageVersionOnly: true }); + const lines = formatPrCoverageSummary({ profile: 'pr', selection: result.selection, prCoverage: result.coverage }); + expect(lines.filter(line => line.includes('Shadow exercised-path map (not active)'))).toHaveLength(1); + const broken = selectPrProfile({ selectedE2E: [], selectedJudges: [], changedFiles: ['design/src/evolve.ts'], exercised: null }); + expect(formatPrCoverageSummary({ prCoverage: { ...broken, shadow: { e2e: broken.e2e, wouldDrop: [], wouldAdd: [], provenance: [], predictedSavingUsd: 0, unpricedCases: 0, unknownCostShare: 0, error: 'bad map' } } }) + .some(line => line.includes('unavailable (bad map); selection unaffected'))).toBe(true); + }); +}); + +describe('union-only refreshes', () => { + const base = map({ a: { observations: 2, withCommands: 1, commands: ['bin/x'], costUsd: 4, costRuns: 2 } }, ['1']); + + test('disjoint branches merge to the same union in either order, and nothing is removed', () => { + const left = map({ a: { observations: 1, withCommands: 1, commands: ['bin/y'] } }, ['2'], '2026-10-02T00:00:00Z'); + const right = map({ b: { observations: 1, withCommands: 1, commands: ['bin/z'], costUsd: 1, costRuns: 1 } }, ['3'], '2026-10-03T00:00:00Z'); + const one = mergeExercisedMap(mergeExercisedMap(base, left, 't'), right, 't'); + const two = mergeExercisedMap(mergeExercisedMap(base, right, 't'), left, 't'); + expect(one).toEqual(two); + expect(one.cases.a!.commands).toEqual(['bin/x', 'bin/y']); + expect(one.cases.b!.commands).toEqual(['bin/z']); + expect(one.sourceRuns).toEqual(['1', '2', '3']); + expect(one.observedThrough).toBe('2026-10-03T00:00:00Z'); + expect(one.cases.a!.costUsd).toBe(4); + }); + + test('empty or missing artifacts never erase consumers, and a run already merged is not counted twice', () => { + expect(mergeExercisedMap(base, map({}, ['9']), 't').cases).toEqual(base.cases); + expect(mergeExercisedMap(base, map({ a: { observations: 1, withCommands: 0, commands: [] } }, ['9']), 't').cases.a!.commands).toEqual(['bin/x']); + expect(mergeExercisedMap(base, map({ a: { observations: 5, withCommands: 5, commands: ['bin/x'], costUsd: 9, costRuns: 1 } }, ['1']), 't').cases.a) + .toEqual(base.cases.a!); + const costed = mergeExercisedMap(base, map({ a: { observations: 1, withCommands: 1, commands: [], costUsd: 7, costRuns: 1 } }, ['4']), 't'); + expect(costed.cases.a).toMatchObject({ observations: 3, costUsd: 5, costRuns: 3 }); + expect(parseExercisedMap(formatExercisedMap(costed))).toEqual(costed); + }); +}); + +describe('shadow-only map data', () => { + test('a map-only diff selects no paid case and says why', () => { + const selection = selectPrProfile({ selectedE2E: [], selectedJudges: [], changedFiles: [EXERCISED_MAP_PATH] }); + expect(selection).toMatchObject({ mode: 'pr', e2e: [], judges: [], unknownFiles: [], derivedFiles: [], noConsumerFiles: [] }); + expect(selection.shadow!.e2e).toEqual([]); + expect(selection.reasons).toContain(`${EXERCISED_MAP_PATH} is shadow-only data (it changes no selection yet), so it selects no paid case (docs/TESTING_INTERNALS.md#exercised-path-map)`); + const plan = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: [EXERCISED_MAP_PATH, 'package.json'], packageVersionOnly: true }); + expect(plan.selection.e2e).toEqual([]); + expect(plan.selection.judges).toEqual([]); + }, 60_000); +}); + +describe('refresh contract: unavailable or partial input writes nothing', () => { + const root = fixtureRepo(tree); + const ex = exercisedConsumers(maps, root, { observed, stopAt: ['setup'] }); + const report = (files: Record) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'exercised-refresh-')); + for (const [file, text] of Object.entries(files)) { fs.mkdirSync(path.dirname(path.join(dir, file)), { recursive: true }); fs.writeFileSync(path.join(dir, file), text); } + return dir; + }; + const outcome = (id: string) => JSON.stringify({ case: id, execution: 'executed', cost_usd: 1 }); + const shard = { 'paid-slice-1-a1/shards/skill-e2e-beta/junit.xml': '' }; + const code = (run: () => unknown) => { try { run(); return 0; } catch (error) { return error instanceof RefreshError ? error.exitCode : -1; } }; + + test('report directories: missing is unavailable; no shards, no trial outcomes or an executed case without a shard is incomplete', () => { + expect(code(() => refreshFromReportDirs([path.join(os.tmpdir(), 'no-such-census-dir')], '1', observed, ex, 't', profileMaps))).toBe(REFRESH_EXIT.unavailable); + expect(code(() => refreshFromReportDirs([report({ 'trial-outcomes.jsonl': outcome('beta') })], '1', observed, ex, 't', profileMaps))).toBe(REFRESH_EXIT.incomplete); + expect(code(() => refreshFromReportDirs([report(shard)], '1', observed, ex, 't', profileMaps))).toBe(REFRESH_EXIT.incomplete); + expect(() => refreshFromReportDirs([report({ ...shard, 'trial-outcomes.jsonl': `${outcome('beta')}\n${outcome('alpha')}` })], '1', observed, ex, 't', profileMaps)) + .toThrow('1 executed case(s) have no shard artifact: alpha'); + const { merged } = refreshFromReportDirs([report({ ...shard, 'trial-outcomes.jsonl': outcome('beta') })], '9', observed, ex, 't', profileMaps); + expect(merged.sourceRuns).toEqual(['1', '9']); + expect(merged.cases.beta!.observations).toBe(4); + }); + + test('--from-run: a 403, a missing map or a malformed map leaves the target byte-identical; a complete artifact is unioned', () => { + const target = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'exercised-target-')), 'exercised-commands.json'); + fs.writeFileSync(target, formatExercisedMap(observed)); + const before = fs.readFileSync(target, 'utf8'); + const artifact = (files: Record): ArtifactDownload => (_runId, dir) => { + for (const [file, text] of Object.entries(files)) { fs.mkdirSync(path.dirname(path.join(dir, file)), { recursive: true }); fs.writeFileSync(path.join(dir, file), text); } + return { status: 0, stderr: '' }; + }; + let forbidden: unknown; + try { refreshFromRun('42', target, () => ({ status: 1, stderr: 'HTTP 403: Resource not accessible by integration' })); } catch (error) { forbidden = error; } + expect(forbidden).toBeInstanceOf(RefreshError); + expect((forbidden as RefreshError).exitCode).toBe(REFRESH_EXIT.unavailable); + expect((forbidden as Error).message).toContain('pass its directory with --from-dir'); + expect(code(() => refreshFromRun('42', target, artifact({ 'exercised-map-a1/report.md': 'x' })))).toBe(REFRESH_EXIT.incomplete); + expect(code(() => refreshFromRun('42', target, artifact({ 'exercised-map-a1/exercised-commands.json': '{"schema":1,"cases":' })))).toBe(REFRESH_EXIT.incomplete); + expect(code(() => refreshFromRun('42', target, artifact({ 'exercised-map-a1/exercised-commands.json': formatExercisedMap(map({}, ['5'])), 'exercised-map-a2/exercised-commands.json': '{}' })))).toBe(REFRESH_EXIT.incomplete); + expect(code(() => refreshFromRun('not-a-run', target))).toBe(REFRESH_EXIT.unavailable); + expect(fs.readFileSync(target, 'utf8')).toBe(before); + const fresh = map({ beta: { observations: 1, withCommands: 1, commands: ['bin/gstack-new'] } }, ['5'], '2026-10-09T00:00:00Z'); + refreshFromRun('5', target, artifact({ 'exercised-map-a1/exercised-commands.json': formatExercisedMap(fresh) })); + const after = parseExercisedMap(fs.readFileSync(target, 'utf8')); + expect(after.cases.beta!.commands).toEqual(['bin/gstack-new', 'bin/gstack-observed']); + expect(after.sourceRuns).toEqual(['1', '5']); + }); + + test('the CLI exits 2 on usage and 3 on a missing report directory, writing nothing', () => { + const out = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'exercised-cli-')), 'map.json'); + const cli = (args: string[]) => spawnSync('bun', ['run', path.join(ROOT, 'scripts/exercised-map.ts'), ...args], { cwd: ROOT, encoding: 'utf8', timeout: 60_000 }); + expect(cli([]).status).toBe(REFRESH_EXIT.usage); + const missing = cli(['--from-dir', path.join(os.tmpdir(), 'no-such-census-dir'), '--run-id', '1', '--out', out]); + expect(missing.status).toBe(REFRESH_EXIT.unavailable); + expect(missing.stderr).toContain('nothing was written, the committed union is unchanged'); + expect(fs.existsSync(out)).toBe(false); + }, 90_000); +}); + +describe('recorded commands and MAP_GAP', () => { + const root = fixtureRepo(tree); + const ex = exercisedConsumers(maps, root, { observed, stopAt: ['setup'] }); + const report = fs.mkdtempSync(path.join(os.tmpdir(), 'exercised-report-')); + const write = (file: string, text: string) => { fs.mkdirSync(path.dirname(path.join(report, file)), { recursive: true }); fs.writeFileSync(path.join(report, file), text); }; + write('paid-slice-1-a1/shards/skill-e2e-alpha--alpha.t1/result.json', JSON.stringify({ tests: [{ name: 'alpha', transcript: [{ type: 'tool_use', input: { command: 'gstack-start && $B goto x' } }] }] })); + write('paid-slice-1-a1/shards/skill-e2e-beta/junit.xml', ''); + write('paid-slice-1-a1/shards/skill-e2e-beta/session.jsonl', `${JSON.stringify({ type: 'tool_use', input: { command: 'bin/gstack-gapper --x' } })}\n`); + write('paid-slice-1-a1/shards/skill-e2e-gamma/junit.xml', ''); + write('paid-slice-1-a1/trial-outcomes.jsonl', [{ case: 'alpha', execution: 'executed', cost_usd: 1.5 }, { case: 'alpha', execution: 'executed', cost_usd: 0, cost_known: false }, { case: 'beta', execution: 'reused', cost_usd: 3 }].map(record => JSON.stringify(record)).join('\n')); + + test('commands are read per shard and attributed to their cases', () => { + const commands = attributeCommands(readShardCommands(report), profileMaps); + expect(commands.get('alpha')).toEqual(['gstack-start && $B goto x']); + expect(commands.get('beta')).toEqual(['bin/gstack-gapper --x']); + expect(commands.get('gamma')).toEqual([]); + expect(observeRun(commands, ex, { runId: '7', observedAt: 't', costs: readCaseCosts(report) }).cases.alpha) + .toEqual({ observations: 1, withCommands: 1, commands: ['bin/gstack-start', 'browse/dist/browse'], costUsd: 1.5, costRuns: 1 }); + }); + + test('a command outside the static-plus-observed closure is a MAP_GAP; a skill-running case with no commands is blind', () => { + const commands = attributeCommands(readShardCommands(report), profileMaps); + expect(mapGaps(commands, ex)).toEqual([{ id: 'beta', command: 'bin/gstack-gapper' }]); + expect(blindCases(commands, ex)).toEqual(['gamma']); + }); + + test('the paid report writes MAP_GAP lines without failing, and an unreadable checkout degrades to unavailable', () => { + const live = fs.mkdtempSync(path.join(os.tmpdir(), 'exercised-live-')); + const committed = parseExercisedMap(fs.readFileSync(path.join(ROOT, EXERCISED_MAP_PATH), 'utf8')); + const ex = exercisedConsumers(PR_PROFILE_MAPS, ROOT, { observed: committed, stopAt: PR_FULL_GATE_FILES }); + const outside = fs.readdirSync(path.join(ROOT, 'bin')).map(name => `bin/${name}`).find(file => /^bin\/gstack-[a-z-]+$/.test(file) && !ex.closure('ship-docsync-store').has(file))!; + fs.mkdirSync(path.join(live, 'paid-slice-2-a1/shards/skill-e2e-ship-docsync--ship-docsync-store.t1'), { recursive: true }); + fs.writeFileSync(path.join(live, 'paid-slice-2-a1/shards/skill-e2e-ship-docsync--ship-docsync-store.t1/r.json'), + JSON.stringify({ tests: [{ name: 'ship-docsync-store', transcript: [{ type: 'tool_use', input: { command: `${path.basename(outside)} --help` } }] }] })); + const line = `MAP_GAP ship-docsync-store ${outside}: non-blocking for this PR; the next census refresh widens the map (docs/TESTING_INTERNALS.md#exercised-path-map)`; + expect(formatMapGap({ id: 'ship-docsync-store', command: outside })).toBe(line); + expect(reportMapGaps(live, new Set(['ship-docsync-store']), ROOT)).toEqual([line]); + expect(fs.readFileSync(path.join(live, 'map-gaps.txt'), 'utf8')).toBe(`${line}\n`); + expect(reportMapGaps(live, new Set(['ship-docsync-store']), path.join(live, 'no-checkout'))).toEqual([]); + expect(fs.readFileSync(path.join(live, 'map-gaps.txt'), 'utf8')).toStartWith('MAP_GAP unavailable:'); + }, 60_000); +}); + +describe('parity with the measured prototype', () => { + /** Files the shipped function reaches beyond the prototype, each with the reviewed edge class behind it. */ + const REVIEWED_SUPERSET: Record = { + setup: './setup, $ROOT/setup and join(..., "setup") exec edges (plan A2-2); setup is a full-gate file, so the closure stops there', + }; + + test('the shipped map equals or is a reviewed superset of the prototype map on this checkout', () => { + const committed = parseExercisedMap(fs.readFileSync(path.join(ROOT, EXERCISED_MAP_PATH), 'utf8')); + const prototype = prototypeConsumers(PR_PROFILE_MAPS, ROOT, committed, PR_FULL_GATE_FILES); + const shipped = exercisedConsumers(PR_PROFILE_MAPS, ROOT, { observed: committed, stopAt: PR_FULL_GATE_FILES }); + const missing = [...prototype].flatMap(([file, ids]) => [...ids].filter(id => !shipped.e2e.get(file)?.has(id)).map(id => `${file} ${id}`)); + expect(missing).toEqual([]); + const unreviewed = [...new Set([...shipped.e2e].filter(([file, ids]) => [...ids].some(id => !prototype.get(file)?.has(id))).map(([file]) => file))] + .filter(file => !Object.hasOwn(REVIEWED_SUPERSET, file)); + expect(unreviewed.map(file => `${file}: reached beyond the prototype; review the edge class and add it to REVIEWED_SUPERSET with its reason`)).toEqual([]); + }, 60_000); +}); + +describe('replay of the recorded red PR runs', () => { + test('every recorded failed gate case stays in the shadow selection (the PR receipt: bun run test:exercised-map --replay)', () => { + const fixture = JSON.parse(fs.readFileSync(path.join(ROOT, REPLAY_FIXTURE), 'utf8')) as { failedGateCases: number; runs: ReplayRun[] }; + expect(fixture.runs.reduce((sum, run) => sum + run.failedGateCases.length, 0)).toBe(fixture.failedGateCases); + const rows = replayShadow(fixture.runs); + expect(rows.flatMap(row => row.misses.map(id => `${row.runId}:${id}`))).toEqual([]); + expect(formatShadowReplay(rows).at(-1)).toBe('- MISSES (failed, not in the shadow selection): 0'); + }, 120_000); + + test('a red the shadow drops is reported as a miss', () => { + const [row] = replayShadow([{ runId: 1, pr: 1, packageVersionOnly: true, failedGateCases: ['plan-mode-no-op'], changedFiles: ['docs/designs/x.md'] }]); + expect(row!.misses).toEqual(['plan-mode-no-op']); + expect(formatShadowReplay([row!]).slice(-2)).toEqual(['- MISSES (failed, not in the shadow selection): 1', ' - 1 #1 plan-mode-no-op']); + }); +}); diff --git a/test/fixtures/exercised-map-replay.json b/test/fixtures/exercised-map-replay.json new file mode 100644 index 000000000..0023f6b3d --- /dev/null +++ b/test/fixtures/exercised-map-replay.json @@ -0,0 +1,44 @@ +{ + "schema": 1, + "source": "evals.yml pull_request runs 2026-10-07..2026-10-17 (225 runs): every run with an executed failed gate trial, deduplicated per (run, case); changed files are the three-dot compare of the merge-base with the run head", + "failedGateCases": 47, + "runs": [ + {"runId":37560420246,"pr":3059,"headSha":"d6250a058af971dd697036acd6e05f7740e8e210","packageVersionOnly":true,"failedGateCases":["plan-mode-no-op"],"changedFiles":[".github/actionlint.yaml",".github/workflows/eval-sweep.yml","CHANGELOG.md","CONTRIBUTING.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-config","docs/TESTING_INTERNALS.md","package.json","scripts/eval-flake-rank.ts","scripts/harness-version.json","scripts/lib/measure-bar.ts","scripts/ship-measure-sweep.ts","scripts/ship-measure.ts","scripts/test-pr-profile.ts","ship/sections/measure.md","ship/sections/measure.md.tmpl","test/ci-image-tag-binding.test.ts","test/eval-sweep-workflow.test.ts","test/fixtures/module-size-ratchet.json","test/helpers/carve-guards.ts","test/helpers/touchfiles-data.ts","test/ship-measure-bar.test.ts","test/ship-measure-loop-skill.test.ts","test/ship-measure-seeded-fixture.test.ts","test/ship-measure-sweep.test.ts","test/ship-measure.test.ts"]}, + {"runId":37564096977,"pr":3059,"headSha":"36c0cf1579a2f823783c2ecee5c509e872b0c20a","packageVersionOnly":true,"failedGateCases":["qa-functional-cli-report"],"changedFiles":[".github/actionlint.yaml",".github/workflows/eval-sweep.yml","CHANGELOG.md","CONTRIBUTING.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-config","docs/TESTING_INTERNALS.md","package.json","scripts/eval-flake-rank.ts","scripts/harness-version.json","scripts/lib/measure-bar.ts","scripts/ship-measure-sweep.ts","scripts/ship-measure.ts","scripts/test-pr-profile.ts","ship/sections/measure.md","ship/sections/measure.md.tmpl","test/ci-image-tag-binding.test.ts","test/eval-sweep-workflow.test.ts","test/fixtures/module-size-ratchet.json","test/helpers/carve-guards.ts","test/helpers/touchfiles-data.ts","test/ship-measure-bar.test.ts","test/ship-measure-loop-skill.test.ts","test/ship-measure-seeded-fixture.test.ts","test/ship-measure-sweep.test.ts","test/ship-measure.test.ts","test/skill-e2e-plan-mode-no-op.test.ts"]}, + {"runId":37691665506,"pr":3069,"headSha":"068a3099baf660e995c693b5625b4b3e4546b7a8","packageVersionOnly":true,"failedGateCases":["shared-libs-read-only"],"changedFiles":[".github/ISSUE_TEMPLATE/bug_report.md",".github/docker/Dockerfile.ci",".github/workflows/evals-periodic.yml",".github/workflows/free-tests.yml",".github/workflows/measure-journal-read.yml","CHANGELOG.md","README.md","SKILL.md","TODOS.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/SKILL.md","autoplan/bin/guard-journal.ts","autoplan/bin/guard-log.ts","autoplan/bin/guard-reasons.ts","autoplan/bin/phase-publication-hook.ts","autoplan/sections/ceo-phase.md","autoplan/sections/ceo-phase.md.tmpl","autoplan/sections/design-phase.md","autoplan/sections/design-phase.md.tmpl","autoplan/sections/dx-phase.md","autoplan/sections/dx-phase.md.tmpl","autoplan/sections/eng-phase.md","autoplan/sections/eng-phase.md.tmpl","autoplan/sections/phase-close.md","autoplan/sections/phase-close.md.tmpl","benchmark-models/SKILL.md","benchmark/SKILL.md","bin/gstack-artifacts-init","bin/gstack-brain-lock.sh","bin/gstack-brain-restore","bin/gstack-brain-sync","bin/gstack-codex-probe","bin/gstack-codex-status.sh","bin/gstack-config","bin/gstack-doctor","bin/gstack-gbrain-source-wireup","bin/gstack-greptile-early","bin/gstack-post","bin/gstack-pr-title-rewrite.sh","bin/gstack-question-preference","bin/gstack-redact-prepush","bin/gstack-skill-start","bin/gstack-slug","browse/SKILL.md","browse/test/extension-sender-auth.test.ts","canary/SKILL.md","codex/SKILL.md","codex/SKILL.md.tmpl","codex/sections/challenge-mode.md","codex/sections/challenge-mode.md.tmpl","codex/sections/consult-mode.md","codex/sections/consult-mode.md.tmpl","codex/sections/review-mode.md","codex/sections/review-mode.md.tmpl","context-restore/SKILL.md","context-restore/SKILL.md.tmpl","context-save/SKILL.md","context-save/SKILL.md.tmpl","design-consultation/SKILL.md","design-consultation/sections/proposal-and-preview.md","design-html/SKILL.md","design-review/SKILL.md","design-shotgun/SKILL.md","devex-review/SKILL.md","diagram/SKILL.md","docs/autoplan-guard-troubleshooting.md","docs/designs/FOLLOWUP_WAVE_2026_10_07.md","docs/gbrain-sync-errors.md","docs/gbrain-sync.md","docs/gbrain-write-surfaces.md","docs/troubleshooting.md","document-generate/SKILL.md","document-release/SKILL.md","document-release/sections/release-body.md","health/SKILL.md","investigate/SKILL.md","ios-clean/SKILL.md","ios-design-review/SKILL.md","ios-fix/SKILL.md","ios-qa/SKILL.md","ios-sync/SKILL.md","land-and-deploy/SKILL.md","landing-report/SKILL.md","learn/SKILL.md","lib/claude-journal-records.ts","lib/claude-owned-journal.ts","lib/claude-public-transcript.ts","lib/greptile-early.ts","lib/gstack-post.ts","lib/outside-review-result.ts","make-pdf/SKILL.md","office-hours/SKILL.md","office-hours/sections/design-and-handoff.md","open-gstack-browser/SKILL.md","package.json","pair-agent/SKILL.md","plan-ceo-review/SKILL.md","plan-ceo-review/sections/review-sections.md","plan-design-review/SKILL.md","plan-devex-review/SKILL.md","plan-devex-review/sections/review-sections.md","plan-eng-review/SKILL.md","plan-eng-review/sections/review-sections.md","plan-tune/SKILL.md","plan-tune/SKILL.md.tmpl","qa-only/SKILL.md","qa/SKILL.md","qa/sections/browser-setup.md","retro/SKILL.md","review/SKILL.md","review/greptile-triage.md","review/sections/adversarial.md","review/sections/review-army.md","scrape/SKILL.md","scripts/harness-version.json","scripts/lib/windows-curation.ts","scripts/measure-journal-read.ts","scripts/resolvers/aside.ts","scripts/resolvers/constants.ts","scripts/resolvers/design.ts","scripts/resolvers/gbrain.ts","scripts/resolvers/index.ts","scripts/resolvers/outside-voice-steps.ts","scripts/resolvers/outside-voice.ts","scripts/resolvers/preamble/generate-brain-sync-block.ts","scripts/resolvers/question-tuning.ts","scripts/resolvers/redact-doc.ts","scripts/resolvers/review-army.ts","scripts/resolvers/spec-review.ts","scripts/test-pr-profile.ts","scripts/typecheck-test-baseline.json","setup","setup-browser-cookies/SKILL.md","setup-deploy/SKILL.md","setup-gbrain/SKILL.md","ship/SKILL.md","ship/SKILL.md.tmpl","ship/sections/adversarial.md","ship/sections/documentation.md","ship/sections/greptile-early.md","ship/sections/greptile-early.md.tmpl","ship/sections/greptile.md","ship/sections/greptile.md.tmpl","ship/sections/manifest.json","ship/sections/pr-body.md","ship/sections/pr-body.md.tmpl","ship/sections/review-army.md","skillify/SKILL.md","spec/SKILL.md","spec/sections/gate-and-file.md","spec/sections/gate-and-file.md.tmpl","sync-gbrain/SKILL.md","test-audit/SKILL.md","test/aside-driver.test.ts","test/autoplan-bounded-journal.test.ts","test/autoplan-dual-voice-evidence.test.ts","test/autoplan-guard-log.test.ts","test/autoplan-guard-reasons.test.ts","test/autoplan-long-session-observer.test.ts","test/autoplan-publication-guard.test.ts","test/autoplan-publication-hook.test.ts","test/autoplan-publication-payload.test.ts","test/autoplan-schema-canary-compare.test.ts","test/binding-template-drift.test.ts","test/brain-sync.test.ts","test/ceo-mode-preference-al.test.ts","test/codex-hardening.test.ts","test/codex-invocation-sites.test.ts","test/codex-model-probe.test.ts","test/codex-probe-subcommands.test.ts","test/codex-skill-isolation.test.ts","test/codex-web-search-flag.test.ts","test/context-identity.test.ts","test/design-consultation-contract.test.ts","test/eng-finding-retry-budget.test.ts","test/fixtures/claude-agent-payload-2.1.292.json","test/fixtures/golden/claude-ship-SKILL.md","test/fixtures/golden/codex-ship-SKILL.md","test/fixtures/golden/factory-ship-SKILL.md","test/fixtures/module-size-ratchet.json","test/fixtures/outside-review-severity/ceo-codex-shape.txt","test/fixtures/outside-review-severity/dx-codex-shape.txt","test/fixtures/outside-review-severity/eng-codex-tagged.txt","test/fixtures/outside-review-severity/prose-negative.txt","test/fixtures/outside-review-severity/prose-unlabeled.txt","test/fixtures/outside-review-severity/table-shape.txt","test/gate-tail-lint.test.ts","test/gen-skill-docs.test.ts","test/generated-bash-placeholders.test.ts","test/greptile-early-cli.test.ts","test/greptile-early.test.ts","test/gstack-doctor.test.ts","test/gstack-post-cli.test.ts","test/gstack-post.test.ts","test/gstack-question-preference.test.ts","test/gstack-skill-start.test.ts","test/helpers/autoplan-dual-voice-evidence.ts","test/helpers/autoplan-guard-fixture.ts","test/helpers/autoplan-guard-pty.ts","test/helpers/carve-guards.ts","test/helpers/codex-probe-sourcing.ts","test/helpers/free-fixtures-data.ts","test/helpers/journal-padding.ts","test/helpers/native-auto-decide.ts","test/helpers/placeholder-allowlist.ts","test/helpers/schema-canary.ts","test/helpers/touchfiles-data.ts","test/outside-review-result.test.ts","test/outside-review-severity.test.ts","test/outside-voice-invocation.test.ts","test/outside-voice-preflight.test.ts","test/outside-voice-receipt.test.ts","test/pr-title-rewrite.test.ts","test/redact-doc-resolver.test.ts","test/redact-prepush-emails.test.ts","test/regression-issue3024-zsh-sourced-helpers.test.ts","test/regression-pr1169-mktemp-fallbacks.test.ts","test/resolvers-gbrain-save-results.test.ts","test/run-in-background-guidance.test.ts","test/ship-control-flow.test.ts","test/ship-greptile-early.test.ts","test/ship-publication-gates.test.ts","test/ship-reentry-gates.test.ts","test/ship-template-redaction.test.ts","test/ship-version-sync.test.ts","test/ship-workflow-clarity.test.ts","test/skill-e2e-autoplan-guard-pty.test.ts","test/skill-e2e-autoplan-long-session.test.ts","test/skill-e2e-autoplan-schema-canary.test.ts","test/skill-validation.test.ts","test/spec-template-invariants.test.ts"]}, + {"runId":37694473528,"pr":3069,"headSha":"c2d1aa1976a1403a0a6f7846a748080f61c8c726","packageVersionOnly":true,"failedGateCases":["qa-quick","shared-libs-read-only","shared-libs-unsupported-git"],"changedFiles":[".github/ISSUE_TEMPLATE/bug_report.md",".github/docker/Dockerfile.ci",".github/workflows/evals-periodic.yml",".github/workflows/free-tests.yml",".github/workflows/measure-journal-read.yml","CHANGELOG.md","README.md","SKILL.md","TODOS.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/SKILL.md","autoplan/bin/guard-journal.ts","autoplan/bin/guard-log.ts","autoplan/bin/guard-reasons.ts","autoplan/bin/phase-publication-hook.ts","autoplan/sections/ceo-phase.md","autoplan/sections/ceo-phase.md.tmpl","autoplan/sections/design-phase.md","autoplan/sections/design-phase.md.tmpl","autoplan/sections/dx-phase.md","autoplan/sections/dx-phase.md.tmpl","autoplan/sections/eng-phase.md","autoplan/sections/eng-phase.md.tmpl","autoplan/sections/phase-close.md","autoplan/sections/phase-close.md.tmpl","benchmark-models/SKILL.md","benchmark/SKILL.md","bin/gstack-artifacts-init","bin/gstack-brain-lock.sh","bin/gstack-brain-restore","bin/gstack-brain-sync","bin/gstack-codex-probe","bin/gstack-codex-status.sh","bin/gstack-config","bin/gstack-doctor","bin/gstack-gbrain-source-wireup","bin/gstack-greptile-early","bin/gstack-post","bin/gstack-pr-title-rewrite.sh","bin/gstack-question-preference","bin/gstack-redact-prepush","bin/gstack-skill-start","bin/gstack-slug","browse/SKILL.md","browse/test/extension-sender-auth.test.ts","canary/SKILL.md","codex/SKILL.md","codex/SKILL.md.tmpl","codex/sections/challenge-mode.md","codex/sections/challenge-mode.md.tmpl","codex/sections/consult-mode.md","codex/sections/consult-mode.md.tmpl","codex/sections/review-mode.md","codex/sections/review-mode.md.tmpl","context-restore/SKILL.md","context-restore/SKILL.md.tmpl","context-save/SKILL.md","context-save/SKILL.md.tmpl","design-consultation/SKILL.md","design-consultation/sections/proposal-and-preview.md","design-html/SKILL.md","design-review/SKILL.md","design-shotgun/SKILL.md","devex-review/SKILL.md","diagram/SKILL.md","docs/autoplan-guard-troubleshooting.md","docs/designs/FOLLOWUP_WAVE_2026_10_07.md","docs/gbrain-sync-errors.md","docs/gbrain-sync.md","docs/gbrain-write-surfaces.md","docs/troubleshooting.md","document-generate/SKILL.md","document-release/SKILL.md","document-release/sections/release-body.md","health/SKILL.md","investigate/SKILL.md","ios-clean/SKILL.md","ios-design-review/SKILL.md","ios-fix/SKILL.md","ios-qa/SKILL.md","ios-sync/SKILL.md","land-and-deploy/SKILL.md","landing-report/SKILL.md","learn/SKILL.md","lib/claude-journal-records.ts","lib/claude-owned-journal.ts","lib/claude-public-transcript.ts","lib/greptile-early.ts","lib/gstack-post.ts","lib/outside-review-result.ts","make-pdf/SKILL.md","office-hours/SKILL.md","office-hours/sections/design-and-handoff.md","open-gstack-browser/SKILL.md","package.json","pair-agent/SKILL.md","plan-ceo-review/SKILL.md","plan-ceo-review/sections/review-sections.md","plan-design-review/SKILL.md","plan-devex-review/SKILL.md","plan-devex-review/sections/review-sections.md","plan-eng-review/SKILL.md","plan-eng-review/sections/review-sections.md","plan-tune/SKILL.md","plan-tune/SKILL.md.tmpl","qa-only/SKILL.md","qa/SKILL.md","qa/sections/browser-setup.md","retro/SKILL.md","review/SKILL.md","review/greptile-triage.md","review/sections/adversarial.md","review/sections/review-army.md","scrape/SKILL.md","scripts/harness-version.json","scripts/lib/windows-curation.ts","scripts/measure-journal-read.ts","scripts/resolvers/aside.ts","scripts/resolvers/constants.ts","scripts/resolvers/design.ts","scripts/resolvers/gbrain.ts","scripts/resolvers/index.ts","scripts/resolvers/outside-voice-steps.ts","scripts/resolvers/outside-voice.ts","scripts/resolvers/preamble/generate-brain-sync-block.ts","scripts/resolvers/question-tuning.ts","scripts/resolvers/redact-doc.ts","scripts/resolvers/review-army.ts","scripts/resolvers/spec-review.ts","scripts/test-pr-profile.ts","scripts/typecheck-test-baseline.json","setup","setup-browser-cookies/SKILL.md","setup-deploy/SKILL.md","setup-gbrain/SKILL.md","ship/SKILL.md","ship/SKILL.md.tmpl","ship/sections/adversarial.md","ship/sections/documentation.md","ship/sections/greptile-early.md","ship/sections/greptile-early.md.tmpl","ship/sections/greptile.md","ship/sections/greptile.md.tmpl","ship/sections/manifest.json","ship/sections/pr-body.md","ship/sections/pr-body.md.tmpl","ship/sections/review-army.md","skillify/SKILL.md","spec/SKILL.md","spec/sections/gate-and-file.md","spec/sections/gate-and-file.md.tmpl","sync-gbrain/SKILL.md","test-audit/SKILL.md","test/aside-driver.test.ts","test/autoplan-bounded-journal.test.ts","test/autoplan-dual-voice-evidence.test.ts","test/autoplan-guard-log.test.ts","test/autoplan-guard-reasons.test.ts","test/autoplan-long-session-observer.test.ts","test/autoplan-publication-guard.test.ts","test/autoplan-publication-hook.test.ts","test/autoplan-publication-payload.test.ts","test/autoplan-schema-canary-compare.test.ts","test/binding-template-drift.test.ts","test/brain-sync.test.ts","test/ceo-mode-preference-al.test.ts","test/codex-hardening.test.ts","test/codex-invocation-sites.test.ts","test/codex-model-probe.test.ts","test/codex-probe-subcommands.test.ts","test/codex-skill-isolation.test.ts","test/codex-web-search-flag.test.ts","test/context-identity.test.ts","test/design-consultation-contract.test.ts","test/eng-finding-retry-budget.test.ts","test/fixtures/claude-agent-payload-2.1.292.json","test/fixtures/golden/claude-ship-SKILL.md","test/fixtures/golden/codex-ship-SKILL.md","test/fixtures/golden/factory-ship-SKILL.md","test/fixtures/module-size-ratchet.json","test/fixtures/outside-review-severity/ceo-codex-shape.txt","test/fixtures/outside-review-severity/dx-codex-shape.txt","test/fixtures/outside-review-severity/eng-codex-tagged.txt","test/fixtures/outside-review-severity/prose-negative.txt","test/fixtures/outside-review-severity/prose-unlabeled.txt","test/fixtures/outside-review-severity/table-shape.txt","test/fixtures/workflow-required-steps.json","test/gate-tail-lint.test.ts","test/gen-skill-docs.test.ts","test/generated-bash-placeholders.test.ts","test/greptile-early-cli.test.ts","test/greptile-early.test.ts","test/gstack-doctor.test.ts","test/gstack-post-cli.test.ts","test/gstack-post.test.ts","test/gstack-question-preference.test.ts","test/gstack-skill-start.test.ts","test/helpers/autoplan-dual-voice-evidence.ts","test/helpers/autoplan-guard-fixture.ts","test/helpers/autoplan-guard-pty.ts","test/helpers/carve-guards.ts","test/helpers/codex-probe-sourcing.ts","test/helpers/free-fixtures-data.ts","test/helpers/journal-padding.ts","test/helpers/manual-judge-review-fixture.ts","test/helpers/native-auto-decide.ts","test/helpers/placeholder-allowlist.ts","test/helpers/schema-canary.ts","test/helpers/touchfiles-data.ts","test/outside-review-result.test.ts","test/outside-review-severity.test.ts","test/outside-voice-invocation.test.ts","test/outside-voice-preflight.test.ts","test/outside-voice-receipt.test.ts","test/pr-title-rewrite.test.ts","test/qa-supervision-selection.test.ts","test/redact-doc-resolver.test.ts","test/redact-prepush-emails.test.ts","test/regression-issue3024-zsh-sourced-helpers.test.ts","test/regression-pr1169-mktemp-fallbacks.test.ts","test/resolvers-gbrain-save-results.test.ts","test/run-in-background-guidance.test.ts","test/ship-control-flow.test.ts","test/ship-document-release-dispatch.test.ts","test/ship-greptile-early.test.ts","test/ship-publication-gates.test.ts","test/ship-reentry-gates.test.ts","test/ship-template-redaction.test.ts","test/ship-version-sync.test.ts","test/ship-workflow-clarity.test.ts","test/skill-e2e-autoplan-guard-pty.test.ts","test/skill-e2e-autoplan-long-session.test.ts","test/skill-e2e-autoplan-schema-canary.test.ts","test/skill-validation.test.ts","test/spec-template-invariants.test.ts","test/workflow-excerpt.test.ts"]}, + {"runId":37731666423,"pr":3073,"headSha":"872e1542e42d40b3a95356d5284c6e4a3553c31f","packageVersionOnly":true,"failedGateCases":["plan-ceo-finding-floor","ship-docsync-missing-asset"],"changedFiles":[".github/workflows/model-policy-freshness.yml","CHANGELOG.md","CONTRIBUTING.md","README.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/SKILL.md","autoplan/SKILL.md.tmpl","autoplan/sections/ceo-phase.md","autoplan/sections/design-phase.md","autoplan/sections/dx-phase.md","autoplan/sections/eng-phase.md","bin/gstack-codex-probe","bin/gstack-config","bin/gstack-models","claude-code/SKILL.md.tmpl","codex/SKILL.md","codex/SKILL.md.tmpl","codex/sections/challenge-mode.md","codex/sections/challenge-mode.md.tmpl","codex/sections/consult-mode.md","codex/sections/consult-mode.md.tmpl","codex/sections/review-mode.md","codex/sections/review-mode.md.tmpl","docs/model-policy.md","docs/skills.md","docs/troubleshooting.md","lib/claude-code-migration.ts","lib/claude-code.ts","lib/model-catalog.ts","lib/model-policy-cli.ts","lib/model-policy-freshness-github.ts","lib/model-policy-freshness-parser-worker.ts","lib/model-policy-freshness-publication.ts","lib/model-policy-freshness-sources.ts","lib/model-policy-freshness.ts","lib/model-policy-notice.ts","lib/model-policy.ts","package.json","plan-ceo-review/sections/review-sections.md","plan-design-review/SKILL.md","plan-devex-review/sections/review-sections.md","plan-eng-review/sections/review-sections.md","plan-eng-review/sections/review-sections.md.tmpl","review/SKILL.md","scripts/harness-version.json","scripts/model-policy-freshness.ts","scripts/preflight-codex-overlap.ts","scripts/resolve-codex-generation-model.ts","scripts/resolvers/constants.ts","scripts/resolvers/design.ts","scripts/resolvers/index.ts","scripts/resolvers/outside-voice-steps.ts","scripts/resolvers/outside-voice.ts","scripts/resolvers/plan-review.ts","scripts/resolvers/qa.ts","scripts/test-pr-profile.ts","ship/sections/review-army.md","spec/SKILL.md","spec/SKILL.md.tmpl","spec/sections/gate-and-file.md","spec/sections/gate-and-file.md.tmpl","test/claude-code-migration.test.ts","test/claude-code-skill.test.ts","test/codex-hardening.test.ts","test/codex-probe-cache.test.ts","test/codex-under-codex-detection.test.ts","test/fixtures/model-policy-freshness.ts","test/fixtures/model-policy-freshness/anthropic-lifecycle.md","test/fixtures/model-policy-freshness/anthropic-models.md","test/fixtures/model-policy-freshness/openai-lifecycle.md","test/fixtures/model-policy-freshness/openai-models.md","test/fixtures/qa-caller-read-probe-mtb1007a.json","test/helpers/free-fixtures-data.ts","test/helpers/pty/launch.ts","test/helpers/qa-callers-fixture.ts","test/model-catalog-isolation.test.ts","test/model-policy-cli.test.ts","test/model-policy-freshness.test.ts","test/model-policy-workflow.test.ts","test/model-policy.test.ts","test/model-role-invocation.test.ts","test/outside-voice-invocation.test.ts","test/outside-voice-preflight.test.ts","test/plan-review-cases.test.ts","test/pty-output-wake.test.ts","test/qa-exploratory-callers.test.ts","test/qa-probe-gates.test.ts","test/run-in-background-guidance.test.ts","test/setup-claude-code-migration.test.ts","test/team-mode.test.ts"]}, + {"runId":37739845694,"pr":3077,"headSha":"e2e725ad3ddefac67b57f91c2a000848de56b752","packageVersionOnly":true,"failedGateCases":["ship-docsync-missing-asset"],"changedFiles":["CHANGELOG.md","TODOS.md","VERSION","agents-digest/gstack-AGENTS.md","careful/SKILL.md","careful/SKILL.md.tmpl","careful/bin/check-careful.sh","careful/bin/hook-extract.sh","docs/TESTING_INTERNALS.md","docs/skills.md","freeze/SKILL.md","freeze/SKILL.md.tmpl","freeze/bin/check-freeze.sh","guard/SKILL.md","guard/SKILL.md.tmpl","investigate/SKILL.md","investigate/SKILL.md.tmpl","package.json","scripts/harness-version.json","scripts/lib/windows-curation.ts","scripts/test-pr-profile.ts","test/fixtures/context-budget.json","test/hook-scripts.test.ts"]}, + {"runId":37743622252,"pr":3079,"headSha":"4637fae03baad83603ab7d554f4137c4702b4bd4","packageVersionOnly":true,"failedGateCases":["qa-quick"],"changedFiles":[".github/workflows/free-tests.yml",".github/workflows/make-pdf-gate.yml",".github/workflows/windows-setup-e2e.yml",".gitignore","CHANGELOG.md","README.md","TODOS.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-doctor","bin/gstack-global-discover.ts","bin/gstack-launch-probe.sh","browse/src/cli.ts","browse/src/find-browse.ts","browse/test/sidebar-ux.test.ts","docs/troubleshooting.md","extension/sidepanel-terminal.js","extension/sidepanel.css","make-pdf/src/cli.ts","package.json","scripts/build-cso.sh","scripts/build.sh","setup","test/binary-version-probe.test.ts","test/build-cso-nonfatal.test.ts","test/build-cso-result-record.test.ts","test/cso-windows-launcher.test.ts","test/gstack-doctor.test.ts","test/gstack-launch-probe.test.ts","test/setup-cso-degrade.test.ts","test/setup-cso-windows-probe.test.ts","test/setup-windows-launch-hint.test.ts"]}, + {"runId":37762901988,"pr":3079,"headSha":"b29f02250967df02f0e02dc4c8310155771cdf41","packageVersionOnly":true,"failedGateCases":["ship-exploratory-unavailable"],"changedFiles":[".github/workflows/free-tests.yml",".github/workflows/make-pdf-gate.yml",".github/workflows/windows-setup-e2e.yml",".gitignore","CHANGELOG.md","README.md","TODOS.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-doctor","bin/gstack-global-discover.ts","bin/gstack-launch-probe.sh","browse/src/cli.ts","browse/src/find-browse.ts","browse/test/sidebar-ux.test.ts","docs/troubleshooting.md","extension/sidepanel-terminal.js","extension/sidepanel.css","make-pdf/src/cli.ts","package.json","scripts/build-cso.sh","scripts/build.sh","setup","test/binary-version-probe.test.ts","test/build-cso-nonfatal.test.ts","test/build-cso-result-record.test.ts","test/cso-windows-launcher.test.ts","test/gstack-doctor.test.ts","test/gstack-launch-probe.test.ts","test/setup-cso-degrade.test.ts","test/setup-cso-windows-probe.test.ts","test/setup-windows-launch-hint.test.ts"]}, + {"runId":38087752336,"pr":3114,"headSha":"3e51fb06ba5c9c8943e7211ba32fb93fcccfe402","packageVersionOnly":false,"failedGateCases":["qa-functional-cli-report"],"changedFiles":["AGENTS.md","ETHOS.md","README.md","SKILL.md","autoplan/SKILL.md","benchmark-models/SKILL.md","benchmark/SKILL.md","bin/gstack-evidence","bin/gstack-gate-diff","bin/gstack-review-log","browse/SKILL.md","canary/SKILL.md","codex/SKILL.md","context-restore/SKILL.md","context-save/SKILL.md","design-consultation/SKILL.md","design-html/SKILL.md","design-review/SKILL.md","design-shotgun/SKILL.md","devex-review/SKILL.md","diagram/SKILL.md","docs/designs/HONEST_WORK_GATE_INTEGRITY.md","docs/designs/gate-diff-calibration.md","document-generate/SKILL.md","document-release/SKILL.md","health/SKILL.md","investigate/SKILL.md","ios-clean/SKILL.md","ios-design-review/SKILL.md","ios-fix/SKILL.md","ios-qa/SKILL.md","ios-sync/SKILL.md","land-and-deploy/SKILL.md","landing-report/SKILL.md","learn/SKILL.md","lib/gate-diff/classify.ts","lib/gate-diff/git.ts","lib/gate-diff/index.ts","lib/gate-diff/render.ts","lib/gate-diff/tags.ts","lib/gate-diff/types.ts","lib/review-evidence.ts","make-pdf/SKILL.md","office-hours/SKILL.md","open-gstack-browser/SKILL.md","pair-agent/SKILL.md","plan-ceo-review/SKILL.md","plan-ceo-review/sections/review-sections.md","plan-ceo-review/sections/review-sections.md.tmpl","plan-design-review/SKILL.md","plan-devex-review/SKILL.md","plan-eng-review/SKILL.md","plan-eng-review/sections/review-sections.md","plan-tune/SKILL.md","qa-only/SKILL.md","qa/SKILL.md","retro/SKILL.md","review/SKILL.md","review/SKILL.md.tmpl","review/checklist.md","review/gate-integrity.md","review/specialists/testing.md","scrape/SKILL.md","scripts/gate-diff-calibrate.ts","scripts/harness-version.json","scripts/question-registry.ts","scripts/resolvers/dispatch-rules.ts","scripts/resolvers/gate-scan.ts","scripts/resolvers/index.ts","scripts/resolvers/plan-gates.ts","scripts/resolvers/preamble/generate-completion-status.ts","scripts/resolvers/preamble/generate-evidence-directive.ts","scripts/resolvers/test-value.ts","scripts/resolvers/testing.ts","scripts/test-pr-profile.ts","setup-browser-cookies/SKILL.md","setup-deploy/SKILL.md","setup-gbrain/SKILL.md","ship/SKILL.md","ship/SKILL.md.tmpl","ship/sections/plan-completion.md","ship/sections/plan-completion.md.tmpl","ship/sections/pr-body.md","ship/sections/pr-body.md.tmpl","ship/sections/review-army.md","ship/sections/review-army.md.tmpl","ship/sections/test-coverage.md","ship/sections/test-coverage.md.tmpl","ship/sections/tests.md","ship/sections/tests.md.tmpl","skillify/SKILL.md","spec/SKILL.md","spec/SKILL.md.tmpl","spec/sections/gate-and-file.md","spec/sections/gate-and-file.md.tmpl","sync-gbrain/SKILL.md","test-audit/SKILL.md","test/dispatch-rules.test.ts","test/eng-finding-retry-budget.test.ts","test/env-host-runtime-assets.test.ts","test/evidence.test.ts","test/fixtures/context-budget.json","test/fixtures/gate-diff/adv-forged-measured.base.patch","test/fixtures/gate-diff/adv-forged-measured.patch","test/fixtures/gate-diff/adv-golden-beside-unrelated.base.patch","test/fixtures/gate-diff/adv-golden-beside-unrelated.patch","test/fixtures/gate-diff/adv-kotlin-ignore.base.patch","test/fixtures/gate-diff/adv-kotlin-ignore.patch","test/fixtures/gate-diff/control-added-test.base.patch","test/fixtures/gate-diff/control-added-test.patch","test/fixtures/gate-diff/control-rename.base.patch","test/fixtures/gate-diff/control-rename.patch","test/fixtures/gate-diff/control-snapshot-with-owner.base.patch","test/fixtures/gate-diff/control-snapshot-with-owner.patch","test/fixtures/gate-diff/control-spec-skill-dir.base.patch","test/fixtures/gate-diff/control-spec-skill-dir.patch","test/fixtures/gate-diff/control-test-deleted-with-source.base.patch","test/fixtures/gate-diff/control-test-deleted-with-source.patch","test/fixtures/gate-diff/package-json-scripts.base.patch","test/fixtures/gate-diff/package-json-scripts.patch","test/fixtures/gate-diff/rh1-removed.base.patch","test/fixtures/gate-diff/rh1-removed.patch","test/fixtures/gate-diff/rh1-skip.base.patch","test/fixtures/gate-diff/rh1-skip.patch","test/fixtures/gate-diff/rh12-env.base.patch","test/fixtures/gate-diff/rh12-env.patch","test/fixtures/gate-diff/rh13-pragma.base.patch","test/fixtures/gate-diff/rh13-pragma.patch","test/fixtures/gate-diff/rh14-bypass.base.patch","test/fixtures/gate-diff/rh14-bypass.patch","test/fixtures/gate-diff/rh15-bound.base.patch","test/fixtures/gate-diff/rh15-bound.patch","test/fixtures/gate-diff/rh15-timeout.base.patch","test/fixtures/gate-diff/rh15-timeout.patch","test/fixtures/gate-diff/rh16-swallow.base.patch","test/fixtures/gate-diff/rh16-swallow.patch","test/fixtures/gate-diff/rh3-golden.base.patch","test/fixtures/gate-diff/rh3-golden.patch","test/fixtures/gate-diff/rh3-snapshot-no-owner.base.patch","test/fixtures/gate-diff/rh3-snapshot-no-owner.patch","test/fixtures/gate-diff/rh4-placeholder.base.patch","test/fixtures/gate-diff/rh4-placeholder.patch","test/fixtures/golden/agy-ship-SKILL.md","test/fixtures/golden/claude-ship-SKILL.md","test/fixtures/golden/codex-ship-SKILL.md","test/fixtures/golden/factory-ship-SKILL.md","test/fixtures/module-size-ratchet.json","test/fixtures/workflow-required-steps.json","test/gate-diff-cli.test.ts","test/gate-diff.test.ts","test/gate-integrity-templates.test.ts","test/helpers/carve-guards.ts","test/helpers/gate-diff-repo.ts","test/helpers/placeholder-allowlist.ts","test/helpers/touchfiles-data.ts","test/honesty-kernel.test.ts","test/outside-voice-routing.test.ts","test/plan-tune.test.ts","test/review-start-evidence.test.ts","test/review-workflow-clarity.test.ts","test/ship-greptile-early.test.ts","test/ship-plan-completion-invariants.test.ts","test/skill-e2e-gate-integrity.test.ts","test/test-value-bar.test.ts"]}, + {"runId":38095294863,"pr":3116,"headSha":"82828943447b168adbbdc173c2095fcf33ba6f53","packageVersionOnly":true,"failedGateCases":["ship-exploratory-plan-checks"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","benchmark/SKILL.md","bin/gstack-browser-ensure","bin/gstack-bun-version.sh","bin/gstack-capy-install","bin/gstack-doctor","bin/gstack-doctor-check.sh","bin/gstack-doctor-components.sh","bin/gstack-runtime-pins.ts","browse/SKILL.md","canary/SKILL.md","design-consultation/SKILL.md","design-review/SKILL.md","devex-review/SKILL.md","docs/capy-install.md","docs/state-root.md","docs/troubleshooting.md","land-and-deploy/SKILL.md","lib/doctor-components.ts","lib/result-codes.ts","lib/runtime-pins.ts","open-gstack-browser/SKILL.md","package.json","pair-agent/SKILL.md","qa/sections/browser-setup.md","scrape/SKILL.md","scripts/gen-doctor-components.ts","scripts/gen-skill-docs.ts","scripts/resolvers/browse.ts","scripts/typecheck-test-baseline.json","setup","setup-browser-cookies/SKILL.md","test/bun-version-drift.test.ts","test/doctor-components.test.ts","test/egress-receipt-wiring.test.ts","test/fixtures/module-size-ratchet.json","test/gen-skill-docs-checks.test.ts","test/gen-skill-docs-prune-stale.test.ts","test/gstack-browser-ensure.test.ts","test/gstack-capy-install.test.ts","test/gstack-doctor.test.ts","test/helpers/manual-judge-review-fixture.ts","test/host-conformance.test.ts","test/runtime-pins.test.ts","test/troubleshooting-anchors.test.ts"]}, + {"runId":38097941308,"pr":3117,"headSha":"d0febdf79b1608f3de47f3c1ec099f7829f709f5","packageVersionOnly":true,"failedGateCases":["qa-functional-cli-report"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/SKILL.md","autoplan/SKILL.md.tmpl","autoplan/sections/manifest.json","autoplan/sections/unattended.md","autoplan/sections/unattended.md.tmpl","bin/gstack-autoplan","bin/gstack-codex-login","bin/gstack-doctor-components.sh","bin/gstack-office-hours-review","bin/gstack-outside-voice","docs/troubleshooting.md","docs/unattended.md","lib/autoplan-export.ts","lib/autoplan-gate.ts","lib/autoplan-prompts.ts","lib/autoplan-reconcile.ts","lib/autoplan-run.ts","lib/autoplan-state.ts","lib/headless-artifacts.ts","lib/office-hours-review.ts","lib/outside-voice-api.ts","lib/outside-voice-runner.ts","lib/result-codes.ts","lib/spec-review-cap.ts","lib/spend-ledger.ts","office-hours/sections/design-and-handoff.md","package.json","plan-ceo-review/SKILL.md","plan-ceo-review/sections/checklist.md","plan-ceo-review/sections/checklist.md.tmpl","plan-ceo-review/sections/manifest.json","plan-design-review/sections/checklist.md","plan-design-review/sections/checklist.md.tmpl","plan-design-review/sections/manifest.json","plan-devex-review/sections/checklist.md","plan-devex-review/sections/checklist.md.tmpl","plan-devex-review/sections/manifest.json","plan-eng-review/sections/checklist.md","plan-eng-review/sections/checklist.md.tmpl","plan-eng-review/sections/manifest.json","scripts/resolvers/sections.ts","scripts/resolvers/spec-review.ts","test/autoplan-answer-invalidation.test.ts","test/autoplan-run-concurrency.test.ts","test/autoplan-run-recovery.test.ts","test/egress-receipt-wiring.test.ts","test/fixtures/module-size-ratchet.json","test/fixtures/multi-agent-wave/fixture-reviewer.ts","test/helpers/autoplan-run-fixture.ts","test/helpers/free-fixtures-data.ts","test/helpers/office-hours-completion.ts","test/helpers/workflow-judge-input.ts","test/office-hours-completion.test.ts","test/office-hours-review.test.ts","test/outside-runner-contract.test.ts","test/review-checklists.test.ts","test/spend-ledger.test.ts","test/workflow-judge-input.test.ts"]}, + {"runId":38101502677,"pr":3119,"headSha":"c9a1b4c273a2479dd62d10ba1477d0fe5c6c5824","packageVersionOnly":true,"failedGateCases":["land-and-deploy-review-gate","qa-functional-cli-fix"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/SKILL.md","autoplan/SKILL.md.tmpl","autoplan/sections/ceo-phase.md","autoplan/sections/ceo-phase.md.tmpl","autoplan/sections/eng-phase.md","autoplan/sections/eng-phase.md.tmpl","benchmark/SKILL.md","benchmark/SKILL.md.tmpl","bin/gstack-doctor-components.sh","bin/gstack-plan-reality","cso/SKILL.md","cso/SKILL.md.tmpl","docs/troubleshooting.md","investigate/SKILL.md","investigate/SKILL.md.tmpl","investigate/sections/backlog.md","investigate/sections/backlog.md.tmpl","investigate/sections/eval-bisect.md","investigate/sections/eval-bisect.md.tmpl","investigate/sections/flake.md","investigate/sections/flake.md.tmpl","investigate/sections/manifest.json","investigate/sections/polluter.md","investigate/sections/polluter.md.tmpl","investigate/sections/race.md","investigate/sections/race.md.tmpl","lib/plan-reality.ts","lib/result-codes.ts","package.json","plan-ceo-review/SKILL.md","plan-ceo-review/SKILL.md.tmpl","plan-ceo-review/sections/checklist.md","plan-ceo-review/sections/checklist.md.tmpl","plan-ceo-review/sections/review-sections.md","plan-ceo-review/sections/review-sections.md.tmpl","plan-eng-review/sections/checklist.md","plan-eng-review/sections/checklist.md.tmpl","plan-eng-review/sections/review-sections.md","plan-eng-review/sections/review-sections.md.tmpl","review/SKILL.md","review/SKILL.md.tmpl","review/checklist.md","review/sections/plan-completion.md","scripts/resolvers/index.ts","scripts/resolvers/plan-gates.ts","scripts/resolvers/plan-reality.ts","scripts/resolvers/sideways.ts","test/carve-section-loading.test.ts","test/fixtures/module-size-ratchet.json","test/fixtures/multi-agent-wave/INCIDENTS.md","test/fixtures/multi-agent-wave/reality-rows.json","test/helpers/carve-guards.ts","test/helpers/free-fixtures-data.ts","test/helpers/parity-harness.ts","test/helpers/touchfiles-data.ts","test/investigate-recipes.test.ts","test/plan-reality.test.ts","test/qa-supervision-selection.test.ts","test/sideways-review.test.ts","test/skill-fixture.test.ts"]}, + {"runId":38101616162,"pr":3121,"headSha":"819ef8e083162096a5259625b6cc74465d3889df","packageVersionOnly":true,"failedGateCases":["design-review-plugin-handoff"],"changedFiles":[".gitignore",".gstack/generated.json",".gstack/pregate.json","CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-doctor-components.sh","bin/gstack-issue-links","bin/gstack-pregate","bin/gstack-regen","bin/gstack-ship-receipt","docs/pregate.md","docs/troubleshooting.md","lib/headless-artifacts.ts","lib/issue-links.ts","lib/pregate/checks/lanes.ts","lib/pregate/checks/literals.ts","lib/pregate/checks/regen.ts","lib/pregate/checks/secrets.ts","lib/pregate/checks/strays.ts","lib/pregate/config.ts","lib/pregate/context.ts","lib/pregate/import-graph.ts","lib/pregate/index.ts","lib/pregate/lane-selection.ts","lib/pregate/types.ts","lib/pregate/workflows.ts","lib/regen.ts","lib/result-codes.ts","lib/runtime-pins.ts","lib/ship-receipt.ts","package.json","scripts/gen-ship-goldens.ts","ship/sections/pr-body.md","ship/sections/pr-body.md.tmpl","ship/sections/tests.md","ship/sections/tests.md.tmpl","test/fixtures/module-size-ratchet.json","test/fixtures/multi-agent-wave/INCIDENTS.md","test/gstack-regen.test.ts","test/helpers/pregate-fixture.ts","test/pregate-invalidation.test.ts","test/pregate-selection.test.ts","test/ship-issue-intent.test.ts"]}, + {"runId":38336400566,"pr":3166,"headSha":"8bf348ddee3a31bed6e914f0cd95e49bf12f3d8e","packageVersionOnly":true,"failedGateCases":["ship-docsync-launch-failure"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-doctor-components.sh","bin/gstack-eval-plan","bin/gstack-pregate","bin/gstack-redact-prepush","docs/designs/GSTACK_FIELD_WAVE_2026_10_12.md","docs/eval-calls.md","docs/pregate.md","docs/ship-policy.md","docs/troubleshooting.md","docs/unattended.md","eval-plan/SKILL.md","eval-plan/SKILL.md.tmpl","lib/coordinator-contract.ts","lib/eval-plan.ts","lib/pregate/checks/guards.ts","lib/pregate/checks/lanes.ts","lib/pregate/config.ts","lib/pregate/index.ts","lib/pregate/regate.ts","lib/pregate/runtime.ts","lib/pricing.ts","lib/restamp.ts","lib/result-codes.ts","lib/sealed-guard.ts","lib/ship-policy.ts","lib/tree-receipt.ts","package.json","scripts/lib/windows-curation.ts","scripts/pr-dependencies.ts","spec/sections/gate-and-file.md","test/benchmark-runner.test.ts","test/fixtures/module-size-ratchet.json","test/helpers/capy-host.ts","test/helpers/placeholder-allowlist.ts","test/lane-coordination.test.ts","test/pr-dependencies.test.ts","test/pregate-guards.test.ts","test/pregate-invalidation.test.ts","test/pregate-regate.test.ts","test/pregate-selection.test.ts","test/pricing-eval-plan.test.ts","test/restamp-command-boundary.test.ts","test/sealed-guard.test.ts","test/stamp-templates.test.ts"]}, + {"runId":38337974933,"pr":3164,"headSha":"f6997e51f4351963fc198bf389872a92ee5b95c1","packageVersionOnly":true,"failedGateCases":["tpa-apple-ban"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-bun-version.sh","bin/gstack-capy-install","bin/gstack-capy-session","bin/gstack-doctor","bin/gstack-doctor-check.sh","bin/gstack-doctor-components.sh","bin/gstack-install-verdict.sh","bin/gstack-skill-end","bin/gstack-skill-start","bin/gstack-upgrade","capy/skill/gstack/SKILL.md","capy/skill/gstack/scripts/install.sh","capy/support.env","docs/CHANGELOG_STYLE.md","docs/capy-install.md","docs/designs/GSTACK_FIELD_WAVE_2026_10_12.md","docs/troubleshooting.md","docs/unattended.md","lib/doctor-components.ts","lib/install-verdict.ts","lib/result-codes.ts","package.json","scripts/harness-version.json","scripts/test-pr-profile.ts","test/capy-skill-wrapper.test.ts","test/fixtures/context-budget.json","test/gstack-capy-install.test.ts","test/gstack-doctor.test.ts","test/gstack-skill-start.test.ts","test/gstack-upgrade-bin.test.ts","test/helpers/capy-host.ts","test/install-verdict.test.ts","test/test-pr-profile.test.ts"]}, + {"runId":38345572852,"pr":3164,"headSha":"2dc7afe70762ddefd69c5ebbf15eea4cc3669fa4","packageVersionOnly":true,"failedGateCases":["qa-functional-webhook-report"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-bun-version.sh","bin/gstack-capy-install","bin/gstack-capy-session","bin/gstack-doctor","bin/gstack-doctor-check.sh","bin/gstack-doctor-components.sh","bin/gstack-install-verdict.sh","bin/gstack-skill-end","bin/gstack-skill-start","bin/gstack-upgrade","capy/skill/gstack/SKILL.md","capy/skill/gstack/scripts/install.sh","capy/support.env","docs/CHANGELOG_STYLE.md","docs/capy-install.md","docs/designs/GSTACK_FIELD_WAVE_2026_10_12.md","docs/troubleshooting.md","docs/unattended.md","lib/doctor-components.ts","lib/install-verdict.ts","lib/result-codes.ts","package.json","scripts/harness-version.json","scripts/test-pr-profile.ts","test/capy-skill-wrapper.test.ts","test/fixtures/context-budget.json","test/gstack-capy-install.test.ts","test/gstack-doctor.test.ts","test/gstack-skill-start.test.ts","test/gstack-upgrade-bin.test.ts","test/helpers/capy-host.ts","test/install-verdict.test.ts","test/test-pr-profile.test.ts"]}, + {"runId":38418037093,"pr":3170,"headSha":"44d380c494816ea915db718c7e3e98889e8f3256","packageVersionOnly":true,"failedGateCases":["shared-libs-read-only"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/DRIVER.md","autoplan/DRIVER.md.tmpl","autoplan/SKILL.md","autoplan/SKILL.md.tmpl","bin/gstack-artifact","bin/gstack-autoplan","bin/gstack-autoplan-snapshot.ts","bin/gstack-doctor-components.sh","bin/gstack-plan-reality","docs/troubleshooting.md","docs/unattended.md","lib/autoplan-admission.ts","lib/autoplan-gate.ts","lib/autoplan-migrate.ts","lib/autoplan-plan-text.ts","lib/autoplan-prompts.ts","lib/autoplan-reconcile.ts","lib/autoplan-revise.ts","lib/autoplan-run.ts","lib/autoplan-scope.ts","lib/autoplan-state.ts","lib/gate-list.ts","lib/headless-artifacts.ts","lib/plan-reality.ts","lib/result-codes.ts","lib/source-root.ts","package.json","scripts/gen-skill-docs.ts","scripts/resolvers/autoplan-driver.ts","scripts/resolvers/index.ts","test/autoplan-admission.test.ts","test/autoplan-answer-invalidation.test.ts","test/autoplan-capy-e2e.test.ts","test/autoplan-driver.test.ts","test/autoplan-gate-judgment.test.ts","test/autoplan-record.test.ts","test/autoplan-revise.test.ts","test/autoplan-run-concurrency.test.ts","test/autoplan-scope.test.ts","test/fixtures/module-size-ratchet.json","test/gen-skill-docs-checks.test.ts","test/helpers/autoplan-capy.ts","test/plan-reality.test.ts","test/review-checklists.test.ts"]}, + {"runId":38428252213,"pr":3171,"headSha":"e2b08d308cd72f6d9bfbfcf514f0ece0891da153","packageVersionOnly":true,"failedGateCases":["land-and-deploy-workflow","ship-docsync-stale-before","ship-exploratory-plan-checks"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-doctor-check.sh","capy/skill/gstack/SKILL.md","docs/troubleshooting.md","lib/result-codes.ts","package.json","test/gstack-doctor.test.ts"]}, + {"runId":38478947158,"pr":3173,"headSha":"b6389e0631c99449be8f6ac8e9466f26b3daa611","packageVersionOnly":true,"failedGateCases":["ship-exploratory-plan-checks"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/DRIVER.md","autoplan/DRIVER.md.tmpl","bin/gstack-autoplan","bin/gstack-capy-session","bin/gstack-doctor-components.sh","bin/gstack-gbrain-sync.ts","bin/gstack-harness-pid.sh","bin/gstack-lease","bin/gstack-skill-start","bin/gstack-upgrade","capy/skill/gstack/SKILL.md","capy/support.env","docs/capy-install.md","docs/designs/FIX_WAVE_2026_10_12.md","docs/troubleshooting.md","docs/unattended.md","lib/autoplan-admission.ts","lib/autoplan-export.ts","lib/autoplan-gate.ts","lib/autoplan-prompts.ts","lib/autoplan-run.ts","lib/gate-list.ts","lib/headless-artifacts.ts","lib/owner-brief.ts","lib/restamp.ts","lib/result-codes.ts","package.json","scripts/harness-version.json","test-setup.ts","test/auto-decide-fixture.test.ts","test/autoplan-answer-invalidation.test.ts","test/autoplan-capy-e2e.test.ts","test/autoplan-gate-judgment.test.ts","test/autoplan-preconfigured-onboarding-ar.test.ts","test/autoplan-residuals.test.ts","test/autoplan-revise.test.ts","test/capy-skill-wrapper.test.ts","test/ceo-finding-fixture.test.ts","test/gstack-lease.test.ts","test/gstack-skill-start.test.ts","test/gstack-upgrade-bin.test.ts","test/helpers/capy-host.ts","test/helpers/hermetic-env.ts"]}, + {"runId":38829994639,"pr":3209,"headSha":"2961edad7f56c982d78c97a85c87015bab29b155","packageVersionOnly":false,"failedGateCases":["qa-functional-cli-report"],"changedFiles":[".github/dependabot.yml",".github/workflows/arm-setup-smoke.yml",".github/workflows/cso-runtime-images.yml",".github/workflows/cso-runtime-promote.yml",".github/workflows/cso-runtime-qualification.yml",".github/workflows/cso-scanner-images.yml",".github/workflows/eval-sweep.yml",".github/workflows/evals-marathon.yml",".github/workflows/evals-periodic.yml",".github/workflows/evals.yml",".github/workflows/model-policy-freshness.yml",".github/workflows/native-qualification.yml",".github/workflows/scorecard.yml",".github/workflows/version-gate.yml",".github/workflows/windows-free-tests.yml",".github/workflows/windows-pregate-e2e.yml",".github/workflows/windows-setup-e2e.yml",".osv-scanner.toml","CHANGELOG.md","README.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-browser-ensure","bin/gstack-bun-version.sh","bin/gstack-ceo-plan-find","bin/gstack-claude-root.sh","bin/gstack-config","bin/gstack-doctor-components.sh","bin/gstack-evidence","bin/gstack-ff-update","bin/gstack-gate-diff","bin/gstack-gbrain-sync.ts","bin/gstack-memorable","bin/gstack-sanitize-echo.sh","bin/gstack-session-update","bin/gstack-skill-end","bin/gstack-timeline-log","browse/src/browser-manager.ts","browse/src/meta-commands.ts","browse/src/server.ts","browse/src/terminal-agent.ts","browse/test/tab-external-close-dispatch.test.ts","browse/test/tab-external-close.test.ts","browse/test/terminal-agent-signal-cleanup.test.ts","bun.lock","docs/autoplan-guard-troubleshooting.md","docs/designs/FIX_WAVE_2026_10_15.md","docs/howto-ios-testing-with-gstack.md","docs/troubleshooting.md","gstack-upgrade/SKILL.md","gstack-upgrade/SKILL.md.tmpl","hosts/claude/hooks/spawn-bin.ts","ios-qa/daemon/src/devicectl.ts","ios-qa/daemon/src/tunnel-bootstrap.ts","ios-qa/daemon/test/devicectl-launch-env.test.ts","lib/claude-journal-records.ts","lib/cso/docker.ts","lib/cso/process.ts","lib/cso/scanner-executor.ts","lib/cso/watchdog.c","lib/gate-diff/git.ts","lib/gbrain-exec.ts","lib/pregate/checks/secrets.ts","lib/pregate/import-graph.ts","lib/pregate/lane-selection.ts","lib/redact-engine.ts","lib/regen.ts","lib/restamp.ts","lib/result-codes.ts","lib/sanitize-echo.ts","lib/script-invocation.ts","package.json","plan-ceo-review/sections/review-sections.md","plan-devex-review/sections/review-sections.md","plan-eng-review/SKILL.md","plan-eng-review/SKILL.md.tmpl","plan-eng-review/sections/review-sections.md","scripts/harness-version.json","scripts/resolvers/outside-voice-steps.ts","scripts/test-pr-profile.ts","setup","setup.ps1","test/autoplan-compaction-preserved-segment.test.ts","test/bun-version-shells.test.ts","test/ceo-plan-find-cross-branch.test.ts","test/cso-docker-argv0.test.ts","test/cso-json-redaction.test.ts","test/cso-scanner-release.test.ts","test/dependency-security.test.ts","test/design-doc-data-guard.test.ts","test/fixtures/module-size-ratchet.json","test/fixtures/session-update-heartbeat.sh","test/fixtures/windows-bash-resolution.json","test/gbrain-source-gitignore.test.ts","test/gbrain-spawn-windows-shell.test.ts","test/gbrain-sync-no-bash-message.test.ts","test/gstack-browser-ensure.test.ts","test/gstack-ceo-plan-find.test.ts","test/gstack-ff-update.test.ts","test/gstack-memorable.test.ts","test/gstack-upgrade-fresh-shell.test.ts","test/held-install-recovery.test.ts","test/helpers/auq-sdk-capture.ts","test/helpers/carve-guards.ts","test/helpers/free-fixtures-data.ts","test/helpers/touchfiles-data.ts","test/model-policy-workflow.test.ts","test/osv-ignore-expiry.test.ts","test/outside-voice-native-deadline.test.ts","test/plan-tune-cathedral.test.ts","test/pregate-import-graph-paths.test.ts","test/pregate-manifest-paths.test.ts","test/readme-skill-lists.test.ts","test/redact-prepush-rebase-force-push.test.ts","test/redact-prepush-scan-range.test.ts","test/run-in-background-guidance.test.ts","test/sanitize-echo.test.ts","test/script-invocation-tripwire.test.ts","test/script-invocation.test.ts","test/section-capture-native-tools.test.ts","test/session-update-autostash.test.ts","test/session-update-heartbeat.test.ts","test/session-update-held-banner.test.ts","test/session-update-hook-hold.test.ts","test/session-update-stages.test.ts","test/setup-claude-code-migration.test.ts","test/setup-cursor-legacy-skills.test.ts","test/setup-hook-canonical-paths.test.ts","test/setup-hook-syntax-gate.test.ts","test/setup-playwright-best-effort.test.ts","test/setup-ps1.test.ts","test/ship-measure-sweep.test.ts","test/ship-measure.test.ts","test/skill-end-completion-durability.test.ts","test/state-root-config-merge.test.ts","test/team-mode.test.ts","test/test-value-bar.test.ts","test/timeline-stop-hook-slug.test.ts","test/upgrade-setup-recovery.test.ts","test/upgrade-template-pins.test.ts","test/workflow-action-pins.test.ts","test/wtree-consumers.test.ts"]}, + {"runId":38969924945,"pr":3225,"headSha":"09fe44fe435e7ccf6291f688d4883bcdcb5464da","packageVersionOnly":false,"failedGateCases":["qa-only-no-fix","ship-docsync-store"],"changedFiles":["benchmark/SKILL.md","bin/gstack-upgrade","browse/SKILL.md","canary/SKILL.md","design-consultation/SKILL.md","design-review/SKILL.md","devex-review/SKILL.md","land-and-deploy/SKILL.md","lib/redact-patterns.ts","plan-design-review/sections/review-sections.md","plan-design-review/sections/review-sections.md.tmpl","qa/sections/browser-setup.md","scrape/SKILL.md","scripts/resolvers/aside.ts","test/aside-driver.test.ts","test/cso-git-hardening.test.ts","test/gstack-upgrade-path-spelling.test.ts","test/redact-engine.test.ts","test/review-entry-and-design-clarity-au.test.ts"]}, + {"runId":38973329985,"pr":3225,"headSha":"c130a85ad516ff288a2d0840dfb3180b07b83eca","packageVersionOnly":false,"failedGateCases":["ship-exploratory-small-cli"],"changedFiles":["benchmark/SKILL.md","bin/gstack-upgrade","browse/SKILL.md","canary/SKILL.md","design-consultation/SKILL.md","design-review/SKILL.md","devex-review/SKILL.md","land-and-deploy/SKILL.md","lib/pregate/checks/secrets.ts","lib/redact-patterns.ts","plan-design-review/sections/review-sections.md","plan-design-review/sections/review-sections.md.tmpl","plan-devex-review/sections/review-sections.md","plan-devex-review/sections/review-sections.md.tmpl","qa/sections/browser-setup.md","scrape/SKILL.md","scripts/resolvers/aside.ts","test/aside-driver.test.ts","test/cso-git-hardening.test.ts","test/gstack-upgrade-path-spelling.test.ts","test/pregate-secrets-source-context.test.ts","test/redact-engine.test.ts","test/review-entry-and-design-clarity-au.test.ts"]}, + {"runId":38974669950,"pr":3190,"headSha":"03968f0a7c3f54d4181a20c37ee77594de26974f","packageVersionOnly":true,"failedGateCases":["qa-functional-webhook-fix"],"changedFiles":[".github/workflows/free-tests.yml","CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/DRIVER.md","autoplan/DRIVER.md.tmpl","autoplan/sections/ceo-phase.md","autoplan/sections/eng-phase.md","bin/gstack-autoplan","bin/gstack-bun-version.sh","bin/gstack-capy-install","bin/gstack-capy-session","bin/gstack-doctor-check.sh","bin/gstack-doctor-components.sh","bin/gstack-gate","bin/gstack-install-verdict.sh","bin/gstack-issue-links","bin/gstack-plan-reality","bin/gstack-pregate","bin/gstack-ship-receipt","bin/gstack-upgrade","capy/skill/gstack/SKILL.md","capy/skill/gstack/scripts/install.sh","docs/capy-install.md","docs/pregate.md","docs/troubleshooting.md","docs/unattended.md","lib/autoplan-checkpoint.ts","lib/autoplan-export.ts","lib/autoplan-gate.ts","lib/autoplan-plan-text.ts","lib/autoplan-prompts.ts","lib/autoplan-reconcile.ts","lib/autoplan-revise.ts","lib/autoplan-run.ts","lib/autoplan-state.ts","lib/doctor-components.ts","lib/gate-list.ts","lib/headless-artifacts.ts","lib/install-verdict.ts","lib/issue-links.ts","lib/plan-reality.ts","lib/pregate/checks/guards.ts","lib/pregate/checks/lanes.ts","lib/pregate/config.ts","lib/pregate/context.ts","lib/pregate/index.ts","lib/pregate/runtime.ts","lib/pregate/supervise.ts","lib/pregate/types.ts","lib/result-codes.ts","lib/runtime-pins.ts","lib/ship-receipt.ts","package.json","scripts/lib/windows-curation.ts","scripts/resolvers/autoplan-driver.ts","scripts/resolvers/plan-reality.ts","scripts/ubicloud/ubi-runner.sh","setup","ship/SKILL.md","ship/SKILL.md.tmpl","ship/sections/handoff.md","ship/sections/handoff.md.tmpl","ship/sections/pr-body.md","ship/sections/pr-body.md.tmpl","ship/sections/tests.md","ship/sections/tests.md.tmpl","test/autoplan-answer-invalidation.test.ts","test/autoplan-capy-e2e.test.ts","test/autoplan-checkpoint.test.ts","test/autoplan-driver.test.ts","test/autoplan-gate-artifacts.test.ts","test/autoplan-gate-judgment.test.ts","test/autoplan-pairing.test.ts","test/autoplan-record.test.ts","test/autoplan-residuals.test.ts","test/autoplan-revise.test.ts","test/autoplan-run-concurrency.test.ts","test/autoplan-run-recovery.test.ts","test/bun-resolve.test.ts","test/bun-version-drift.test.ts","test/capy-skill-wrapper.test.ts","test/cso-distribution.test.ts","test/fixtures/autoplan-pairing/run-20261014-pairs.json","test/fixtures/context-budget.json","test/fixtures/golden/agy-ship-SKILL.md","test/fixtures/golden/claude-ship-SKILL.md","test/fixtures/golden/codex-ship-SKILL.md","test/fixtures/golden/factory-ship-SKILL.md","test/gate-list.test.ts","test/gstack-capy-install.test.ts","test/gstack-doctor.test.ts","test/gstack-upgrade-bin.test.ts","test/helpers/autoplan-capy.ts","test/helpers/autoplan-run-fixture.ts","test/helpers/carve-guards.ts","test/helpers/free-fixtures-data.ts","test/helpers/pregate-fixture.ts","test/host-conformance.test.ts","test/install-verdict.test.ts","test/plan-reality.test.ts","test/pregate-guards.test.ts","test/pregate-remote.test.ts","test/pregate-supervise.test.ts","test/redact-prepush-hook.test.ts","test/runtime-pins.test.ts","test/script-invocation-tripwire.test.ts","test/setup-hook-syntax-gate.test.ts","test/setup-playwright-best-effort.test.ts","test/ship-issue-intent.test.ts","test/ubicloud-runner.test.ts"]}, + {"runId":38976253774,"pr":3190,"headSha":"7d68dedae91ffadf494f4cd9a1ab5c253f4b73d6","packageVersionOnly":true,"failedGateCases":["autoplan-guard-pty"],"changedFiles":[".github/workflows/free-tests.yml","CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/DRIVER.md","autoplan/DRIVER.md.tmpl","autoplan/sections/ceo-phase.md","autoplan/sections/eng-phase.md","bin/gstack-autoplan","bin/gstack-bun-version.sh","bin/gstack-capy-install","bin/gstack-capy-session","bin/gstack-doctor-check.sh","bin/gstack-doctor-components.sh","bin/gstack-gate","bin/gstack-install-verdict.sh","bin/gstack-issue-links","bin/gstack-plan-reality","bin/gstack-pregate","bin/gstack-ship-receipt","bin/gstack-upgrade","capy/skill/gstack/SKILL.md","capy/skill/gstack/scripts/install.sh","docs/capy-install.md","docs/pregate.md","docs/troubleshooting.md","docs/unattended.md","lib/autoplan-checkpoint.ts","lib/autoplan-export.ts","lib/autoplan-gate.ts","lib/autoplan-plan-text.ts","lib/autoplan-prompts.ts","lib/autoplan-reconcile.ts","lib/autoplan-revise.ts","lib/autoplan-run.ts","lib/autoplan-state.ts","lib/doctor-components.ts","lib/gate-list.ts","lib/headless-artifacts.ts","lib/install-verdict.ts","lib/issue-links.ts","lib/plan-reality.ts","lib/pregate/checks/guards.ts","lib/pregate/checks/lanes.ts","lib/pregate/config.ts","lib/pregate/context.ts","lib/pregate/index.ts","lib/pregate/runtime.ts","lib/pregate/supervise.ts","lib/pregate/types.ts","lib/result-codes.ts","lib/runtime-pins.ts","lib/ship-receipt.ts","package.json","scripts/lib/windows-curation.ts","scripts/resolvers/autoplan-driver.ts","scripts/resolvers/plan-reality.ts","scripts/ubicloud/ubi-runner.sh","setup","ship/SKILL.md","ship/SKILL.md.tmpl","ship/sections/handoff.md","ship/sections/handoff.md.tmpl","ship/sections/pr-body.md","ship/sections/pr-body.md.tmpl","ship/sections/tests.md","ship/sections/tests.md.tmpl","test/autoplan-answer-invalidation.test.ts","test/autoplan-capy-e2e.test.ts","test/autoplan-checkpoint.test.ts","test/autoplan-driver.test.ts","test/autoplan-gate-artifacts.test.ts","test/autoplan-gate-judgment.test.ts","test/autoplan-pairing.test.ts","test/autoplan-record.test.ts","test/autoplan-residuals.test.ts","test/autoplan-revise.test.ts","test/autoplan-run-concurrency.test.ts","test/autoplan-run-recovery.test.ts","test/bun-resolve.test.ts","test/bun-version-drift.test.ts","test/capy-skill-wrapper.test.ts","test/cso-distribution.test.ts","test/fixtures/autoplan-pairing/run-20261014-pairs.json","test/fixtures/context-budget.json","test/fixtures/golden/agy-ship-SKILL.md","test/fixtures/golden/claude-ship-SKILL.md","test/fixtures/golden/codex-ship-SKILL.md","test/fixtures/golden/factory-ship-SKILL.md","test/gate-list.test.ts","test/gstack-capy-install.test.ts","test/gstack-doctor.test.ts","test/gstack-upgrade-bin.test.ts","test/helpers/autoplan-capy.ts","test/helpers/autoplan-run-fixture.ts","test/helpers/carve-guards.ts","test/helpers/free-fixtures-data.ts","test/helpers/pregate-fixture.ts","test/helpers/qa-functional-fixture.ts","test/host-conformance.test.ts","test/install-verdict.test.ts","test/plan-reality.test.ts","test/pregate-guards.test.ts","test/pregate-remote.test.ts","test/pregate-supervise.test.ts","test/redact-prepush-hook.test.ts","test/runtime-pins.test.ts","test/script-invocation-tripwire.test.ts","test/setup-hook-syntax-gate.test.ts","test/setup-playwright-best-effort.test.ts","test/ship-issue-intent.test.ts","test/ubicloud-runner.test.ts"]}, + {"runId":38980005793,"pr":3225,"headSha":"3f4a98ecac3fc434f8aeef61bc9def17080718d9","packageVersionOnly":false,"failedGateCases":["design-review-detector-shim-dom","qa-quick"],"changedFiles":["benchmark/SKILL.md","bin/gstack-upgrade","browse/SKILL.md","canary/SKILL.md","design-consultation/SKILL.md","design-review/SKILL.md","devex-review/SKILL.md","land-and-deploy/SKILL.md","lib/pregate/checks/secrets.ts","lib/redact-patterns.ts","plan-design-review/sections/review-sections.md","plan-design-review/sections/review-sections.md.tmpl","plan-devex-review/sections/review-sections.md","plan-devex-review/sections/review-sections.md.tmpl","qa/sections/browser-setup.md","scrape/SKILL.md","scripts/resolvers/aside.ts","test/aside-driver.test.ts","test/cso-git-hardening.test.ts","test/gstack-upgrade-path-spelling.test.ts","test/pregate-secrets-source-context.test.ts","test/redact-engine.test.ts","test/review-entry-and-design-clarity-au.test.ts"]}, + {"runId":38981523623,"pr":3231,"headSha":"3ba8b460645b972f91042bb73e6a7a07eb4b05ab","packageVersionOnly":false,"failedGateCases":["ship-exploratory-plan-checks"],"changedFiles":["bin/gstack-skill-end","bin/gstack-skill-start","test/detector-corpus-ship-docsync-launch-failure.test.ts","test/docsync-command-grammar.test.ts","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail-derived-current-claim.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail-derived-prose-claim.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38829994639-t1-pass.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38972920715-t1-pass.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/inventory.json","test/fixtures/docsync-store-38969924945-bash.json","test/gstack-skill-start.test.ts","test/helpers/docsync-fault-eval.ts","test/helpers/docsync-observer.ts","test/helpers/free-fixtures-data.ts","test/helpers/qa-browser-deadline-evidence.ts","test/qa-browser-deadline-evidence.test.ts","test/test-of-test-ratchet.test.ts"]}, + {"runId":38983306684,"pr":3231,"headSha":"84c29e9ee3325c19626edff677dc1ff2ff4f21d9","packageVersionOnly":false,"failedGateCases":["qa-functional-cli-report","ship-docsync-missing-asset"],"changedFiles":["bin/gstack-skill-end","bin/gstack-skill-start","review/sections/review-army.md","scripts/resolvers/review-army.ts","ship/sections/review-army.md","test/detector-corpus-ship-docsync-launch-failure.test.ts","test/docsync-command-grammar.test.ts","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail-derived-current-claim.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail-derived-prose-claim.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38829994639-t1-pass.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38972920715-t1-pass.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/inventory.json","test/fixtures/docsync-store-38969924945-bash.json","test/gstack-skill-start.test.ts","test/helpers/docsync-fault-eval.ts","test/helpers/docsync-observer.ts","test/helpers/free-fixtures-data.ts","test/helpers/qa-browser-deadline-evidence.ts","test/qa-browser-deadline-evidence.test.ts","test/review-workflow-clarity.test.ts","test/test-of-test-ratchet.test.ts"]}, + {"runId":38984977679,"pr":3226,"headSha":"122638084111bcea20fc3f62e4a8cd286cbaa437","packageVersionOnly":false,"failedGateCases":["ship-docsync-failure"],"changedFiles":["README.md","autoplan/SKILL.md","autoplan/SKILL.md.tmpl","autoplan/sections/dx-phase.md","autoplan/sections/dx-phase.md.tmpl","bin/gstack-autoplan-snapshot.ts","bin/gstack-doctor-components.sh","bin/gstack-error-capture","devex-review/SKILL.md","devex-review/SKILL.md.tmpl","docs/agent-ready-errors.md","docs/skills.md","docs/troubleshooting.md","lib/autoplan-prompts.ts","lib/dx-error-capture.ts","lib/dx-error-contract.ts","lib/qa-deadline.ts","lib/result-codes.ts","plan-devex-review/SKILL.md","plan-devex-review/SKILL.md.tmpl","plan-devex-review/dx-hall-of-fame.md","plan-devex-review/sections/checklist.md","plan-devex-review/sections/checklist.md.tmpl","plan-devex-review/sections/review-sections.md","plan-devex-review/sections/review-sections.md.tmpl","scripts/resolvers/dx.ts","scripts/resolvers/index.ts","test/dx-error-capture.test.ts","test/dx-error-contract-grader.test.ts","test/dx-error-contract.test.ts","test/eng-finding-retry-budget.test.ts","test/fixtures/dx-error-contract-case.ts","test/helpers/touchfiles-data.ts","test/skill-e2e-dx-error-contract.test.ts"]}, + {"runId":38986120559,"pr":3231,"headSha":"6fd20bef0fe9387cf13dee88cb144e7b3914575b","packageVersionOnly":true,"failedGateCases":["qa-functional-webhook-fix"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-skill-end","bin/gstack-skill-start","package.json","review/sections/review-army.md","scripts/resolvers/review-army.ts","ship/sections/review-army.md","test/detector-corpus-ship-docsync-launch-failure.test.ts","test/docsync-command-grammar.test.ts","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail-derived-current-claim.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail-derived-prose-claim.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38829994639-t1-pass.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38972920715-t1-pass.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/inventory.json","test/fixtures/docsync-store-38969924945-bash.json","test/gstack-skill-start.test.ts","test/helpers/docsync-fault-eval.ts","test/helpers/docsync-observer.ts","test/helpers/free-fixtures-data.ts","test/helpers/qa-browser-deadline-evidence.ts","test/qa-browser-deadline-evidence.test.ts","test/review-workflow-clarity.test.ts","test/test-of-test-ratchet.test.ts"]}, + {"runId":38988927989,"pr":3232,"headSha":"641916c650257b7fb5719ec45bef70b00066874e","packageVersionOnly":false,"failedGateCases":["tpa-absent-linux"],"changedFiles":["bin/gstack-capy-install","bin/gstack-capy-session","bin/gstack-lease","bin/gstack-skill-start","capy/skill/gstack/SKILL.md","docs/capy-install.md","docs/troubleshooting.md","test/capy-skill-wrapper.test.ts","test/gstack-capy-install.test.ts","test/gstack-lease.test.ts","test/gstack-upgrade-bin.test.ts"]}, + {"runId":38990111230,"pr":3226,"headSha":"6c07d85c82d17bda81860cd0d78616de3dc14cc2","packageVersionOnly":false,"failedGateCases":["ship-docsync-stale-after"],"changedFiles":["README.md","autoplan/SKILL.md","autoplan/SKILL.md.tmpl","autoplan/sections/dx-phase.md","autoplan/sections/dx-phase.md.tmpl","bin/gstack-autoplan-snapshot.ts","bin/gstack-doctor-components.sh","bin/gstack-error-capture","devex-review/SKILL.md","devex-review/SKILL.md.tmpl","docs/agent-ready-errors.md","docs/skills.md","docs/troubleshooting.md","lib/autoplan-prompts.ts","lib/dx-error-capture.ts","lib/dx-error-contract.ts","lib/qa-deadline.ts","lib/result-codes.ts","plan-devex-review/SKILL.md","plan-devex-review/SKILL.md.tmpl","plan-devex-review/dx-hall-of-fame.md","plan-devex-review/sections/checklist.md","plan-devex-review/sections/checklist.md.tmpl","plan-devex-review/sections/review-sections.md","plan-devex-review/sections/review-sections.md.tmpl","scripts/resolvers/dx.ts","scripts/resolvers/index.ts","test/dx-error-capture.test.ts","test/dx-error-contract-grader.test.ts","test/dx-error-contract.test.ts","test/eng-finding-retry-budget.test.ts","test/fixtures/dx-error-contract-case.ts","test/helpers/touchfiles-data.ts","test/skill-e2e-dx-error-contract.test.ts"]}, + {"runId":38993076767,"pr":3234,"headSha":"b40d8418f9435804a4803981dcb01c7288d87e73","packageVersionOnly":false,"failedGateCases":["ship-docsync-store","ship-exploratory-small-cli"],"changedFiles":["autoplan/DRIVER.md","autoplan/DRIVER.md.tmpl","bin/gstack-artifact","bin/gstack-autoplan","bin/gstack-capy-install","bin/gstack-doctor-check.sh","bin/gstack-doctor-components.sh","bin/gstack-eval-plan","bin/gstack-install-verdict.sh","capy/skill/gstack/SKILL.md","capy/skill/gstack/scripts/install.sh","docs/capy-install.md","docs/designs/field-list-2-2026-10-16/PLAN.md","docs/designs/field-list-2-2026-10-16/ROUND2-OBLIGATIONS.md","docs/eval-models.md","docs/troubleshooting.md","docs/unattended.md","eval-plan/SKILL.md","eval-plan/SKILL.md.tmpl","lib/autoplan-budget.ts","lib/autoplan-checkpoint.ts","lib/autoplan-confirm.ts","lib/autoplan-export.ts","lib/autoplan-gate.ts","lib/autoplan-prompts.ts","lib/autoplan-public.ts","lib/autoplan-reconcile.ts","lib/autoplan-revise.ts","lib/autoplan-run.ts","lib/autoplan-state.ts","lib/coordinator-contract.ts","lib/doctor-components.ts","lib/eval-models.ts","lib/eval-plan.ts","lib/gate-list.ts","lib/headless-artifacts.ts","lib/install-verdict.ts","lib/result-codes.ts","lib/spend-ledger.ts","scripts/resolvers/autoplan-driver.ts","ship/sections/handoff.md","ship/sections/handoff.md.tmpl","spec/sections/gate-and-file.md","test/autoplan-budget.test.ts","test/autoplan-checkpoint.test.ts","test/autoplan-durability.test.ts","test/autoplan-public-export.test.ts","test/capy-review-voice.test.ts","test/capy-skill-wrapper.test.ts","test/doctor-components.test.ts","test/eval-models.test.ts","test/eval-plan-prereg-diagnostics.test.ts","test/fixtures/context-budget.json","test/fixtures/module-size-ratchet.json","test/gstack-capy-install.test.ts","test/gstack-doctor.test.ts"]}, + {"runId":39001107574,"pr":3236,"headSha":"79554ccaba38b47a06d55f43dc8fdbb922b738b9","packageVersionOnly":true,"failedGateCases":["qa-functional-cli-report"],"changedFiles":[".github/workflows/windows-pregate-e2e.yml","CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-doctor","bin/gstack-doctor-components.sh","bin/gstack-pregate","bin/gstack-windows-bash.ts","docs/ADDING_A_HOST.md","docs/troubleshooting.md","lib/pregate/checks/lanes.ts","lib/pregate/context.ts","lib/pregate/import-graph.ts","lib/pregate/index.ts","lib/regen.ts","lib/result-codes-release.ts","lib/script-invocation.ts","package.json","scripts/lib/windows-curation.ts","scripts/resolvers/preamble.ts","scripts/resolvers/preamble/generate-completion-status.ts","test/fixtures/golden/agy-ship-SKILL.md","test/fixtures/golden/codex-ship-SKILL.md","test/fixtures/golden/factory-ship-SKILL.md","test/fixtures/skill-md-size-allowlist.json","test/gen-skill-docs.test.ts","test/gstack-doctor.test.ts","test/helpers/free-fixtures-data.ts","test/helpers/pregate-fixture.ts","test/helpers/restamp-fixture.ts","test/pregate-bats-nested.test.ts","test/pregate-wtree-refusal.test.ts","test/review-entry-and-design-clarity-au.test.ts","test/script-invocation-tripwire.test.ts","test/script-invocation.test.ts","test/skill-md-size-guard.test.ts"]}, + {"runId":39002269281,"pr":3239,"headSha":"8e878d9ee9f06a196ff04ccfb0a4d836c13cb1aa","packageVersionOnly":false,"failedGateCases":["ship-exploratory-plan-checks"],"changedFiles":[".github/dependabot.yml",".github/workflows/version-gate.yml",".osv-scanner.toml","TODOS.md","bun.lock","lib/diagram-render/bun.lock","lib/diagram-render/package.json","make-pdf/src/image-size.ts","make-pdf/src/orchestrator.ts","make-pdf/test/docx-image-size.test.ts","make-pdf/test/fixtures/docx-images/doc.md","make-pdf/test/fixtures/docx-images/document.golden.xml","make-pdf/test/fixtures/docx-images/square.gif","make-pdf/test/fixtures/docx-images/tall.jpg","make-pdf/test/fixtures/docx-images/wide.png","package.json","scripts/compare-pr-version.ts","scripts/typecheck-test-baseline.json","test/ceo-finding-fixture.test.ts","test/compare-pr-version.test.ts","test/osv-config-wiring.test.ts","test/osv-ignore-expiry.test.ts","test/workflow-action-pins.test.ts"]}, + {"runId":39004466422,"pr":3239,"headSha":"65ca746bda533a348a66032a6541647b43339416","packageVersionOnly":false,"failedGateCases":["plan-mode-no-op"],"changedFiles":[".github/dependabot.yml",".github/workflows/version-gate.yml",".osv-scanner.toml","TODOS.md","bun.lock","lib/diagram-render/bun.lock","lib/diagram-render/package.json","make-pdf/src/docx-images.ts","make-pdf/src/html-to-vdom.d.ts","make-pdf/src/orchestrator.ts","make-pdf/test/docx-image-size.test.ts","make-pdf/test/fixtures/docx-images/doc.md","make-pdf/test/fixtures/docx-images/document.golden.xml","make-pdf/test/fixtures/docx-images/square.gif","make-pdf/test/fixtures/docx-images/tall.jpg","make-pdf/test/fixtures/docx-images/wide.png","package.json","scripts/compare-pr-version.ts","scripts/typecheck-test-baseline.json","test/ceo-finding-fixture.test.ts","test/compare-pr-version.test.ts","test/osv-config-wiring.test.ts","test/osv-ignore-expiry.test.ts","test/workflow-action-pins.test.ts","tsconfig.test.json"]}, + {"runId":39006896632,"pr":3239,"headSha":"3449a9cd5fd1ed51fd9242bde3a5636ae8d97b77","packageVersionOnly":false,"failedGateCases":["qa-functional-webhook-report"],"changedFiles":[".github/dependabot.yml",".github/workflows/version-gate.yml",".osv-scanner.toml","CHANGELOG.md","TODOS.md","VERSION","agents-digest/gstack-AGENTS.md","bun.lock","lib/diagram-render/bun.lock","lib/diagram-render/package.json","make-pdf/src/docx-images.ts","make-pdf/src/html-to-vdom.d.ts","make-pdf/src/orchestrator.ts","make-pdf/test/docx-image-size.test.ts","make-pdf/test/fixtures/docx-images/doc.md","make-pdf/test/fixtures/docx-images/document.golden.xml","make-pdf/test/fixtures/docx-images/square.gif","make-pdf/test/fixtures/docx-images/tall.jpg","make-pdf/test/fixtures/docx-images/wide.png","package.json","scripts/compare-pr-version.ts","scripts/typecheck-test-baseline.json","test/ceo-finding-fixture.test.ts","test/compare-pr-version.test.ts","test/osv-config-wiring.test.ts","test/osv-ignore-expiry.test.ts","test/workflow-action-pins.test.ts","tsconfig.test.json"]}, + {"runId":39011101113,"pr":3239,"headSha":"599d3f044952ad9ca5f92e6e4f78d342278b44e9","packageVersionOnly":false,"failedGateCases":["tpa-absent-linux"],"changedFiles":[".github/dependabot.yml",".github/workflows/version-gate.yml",".osv-scanner.toml","CHANGELOG.md","TODOS.md","VERSION","agents-digest/gstack-AGENTS.md","bun.lock","lib/diagram-render/bun.lock","lib/diagram-render/package.json","make-pdf/src/docx-images.ts","make-pdf/src/html-to-vdom.d.ts","make-pdf/src/orchestrator.ts","make-pdf/test/docx-image-size.test.ts","make-pdf/test/fixtures/docx-images/doc.md","make-pdf/test/fixtures/docx-images/document.golden.xml","make-pdf/test/fixtures/docx-images/square.gif","make-pdf/test/fixtures/docx-images/tall.jpg","make-pdf/test/fixtures/docx-images/wide.png","package.json","scripts/compare-pr-version.ts","scripts/typecheck-test-baseline.json","test/ceo-finding-fixture.test.ts","test/compare-pr-version.test.ts","test/osv-config-wiring.test.ts","test/osv-ignore-expiry.test.ts","test/workflow-action-pins.test.ts","tsconfig.test.json"]} + ] +} diff --git a/test/helpers/exercised-commands.json b/test/helpers/exercised-commands.json new file mode 100644 index 000000000..e2647ee8a --- /dev/null +++ b/test/helpers/exercised-commands.json @@ -0,0 +1,1905 @@ +{ + "schema": 1, + "generatedAt": "2026-10-17T19:29:23.672Z", + "observedThrough": "2026-10-08T06:50:33Z", + "sourceRuns": [ + "36205327552", + "36220963292", + "36278514291", + "36302678692", + "36315459968", + "36318630639", + "36391800575", + "36465804057", + "36468274063", + "36481155145", + "36481295477", + "36485436963", + "36491692039", + "36493452542", + "36493616568", + "36497566037", + "36505065023", + "36516246523", + "36521598255", + "36526785991", + "36537184166", + "36539603295", + "36551562564", + "36579289566", + "36589496678", + "36615456309", + "36620926989", + "36636909400", + "36637107473", + "36637661593", + "36638638110", + "36640582052", + "36640919692", + "36641031019", + "36641547412", + "36641801658", + "36641824710", + "36670216491", + "36682017584", + "36683045674", + "36685469932", + "36709485593", + "36711269409", + "36711836168", + "36713227509", + "36713549527", + "36714493404", + "36717552105", + "36717834406", + "36718311034", + "36718368006", + "36718436969", + "36718514685", + "36719119326", + "36720872161", + "36722276798", + "36725680777", + "36730053861", + "36730220242", + "36740979126", + "36741522675", + "36743390793", + "36745049965", + "36753939836", + "36755432181", + "36758397400", + "36760073893", + "36761671799", + "36762284181", + "36764370405", + "36768263333", + "36768400454", + "36770061511", + "36774149885", + "36775002132", + "36775932413", + "36776096754", + "36777335875", + "36781170544", + "36783214499", + "36783749239", + "36786871824", + "36787264898", + "36790850100", + "36793107556", + "36794197606", + "36794871032", + "36797095286", + "36797191931", + "36797673165", + "36798542003", + "36823310593", + "36834605576", + "36897642796", + "36898539571", + "36899642230", + "36903605989", + "36905008393", + "36907899270", + "36914699033", + "36914734297", + "36917041210", + "36920579887", + "36922422698", + "36965719520", + "37024904147", + "37025330680", + "37026118550", + "37033755821", + "37060128357", + "37060776291", + "37061202449", + "37062583804", + "37063836999", + "37064151421", + "37066055701", + "37070126890", + "37073429415", + "37077373640", + "37080729942", + "37082906717", + "37087797888", + "37090604134", + "37090925543", + "37091093956", + "37091743395", + "37092497208", + "37092677055", + "37093937027", + "37094035231", + "37097119454", + "37097590162", + "37098585955", + "37099423845", + "37100121821", + "37111582659", + "37116024111", + "37143137677", + "37146354490", + "37152219461", + "37153488058", + "37157517739", + "37158193827", + "37158662432", + "37160557715", + "37161546382", + "37162342436", + "37162847235", + "37163124276", + "37163226077", + "37163388183", + "37163438154", + "37163862654", + "37163935292", + "37164023690", + "37164663577", + "37164967731", + "37165022930", + "37166243626", + "37166586458", + "37168384066", + "37168819307", + "37170258467", + "37170610789", + "37172266374", + "37172907071", + "37173609560", + "37176237665", + "37176835584", + "37177847549", + "37178146922", + "37178720668", + "37179168735", + "37181717177", + "37182208158", + "37184467123", + "37186091144", + "37186918046", + "37188014112", + "37188668421", + "37189974078", + "37192833320", + "37194511147", + "37196784022", + "37196811165", + "37196900729", + "37197708438", + "37199579960", + "37199589136", + "37228201907", + "37228564701", + "37230295724", + "37230639146", + "37230870756", + "37232374839", + "37233114987", + "37234681021", + "37234978002", + "37235771700", + "37237194905", + "37238469830", + "37239016858", + "37241493377", + "37242691913", + "37246213438", + "37246668414", + "37248168494", + "37248388404", + "37248650879", + "37250221023", + "37250545914", + "37250645386", + "37251015146", + "37251229631", + "37251358164", + "37251850439", + "37252022510", + "37252831019", + "37253679118", + "37254021038", + "37256076805", + "37256265663", + "37265273012", + "37266174921", + "37266371726", + "37278830943", + "37296674175", + "37324177668", + "37324423488", + "37327038121", + "37346036236", + "37346541110", + "37347305215", + "37349332448", + "37374409928", + "37375445479", + "37375586954", + "37376593071", + "37389946212", + "37394378126", + "37395241262", + "37397309034", + "37398521095", + "37413464119", + "37415996926", + "37416279318", + "37494809494", + "37504870257", + "37506300031", + "37507007764", + "37510319705", + "37512150975", + "37523820462", + "37540020229", + "37540144327", + "37541010145", + "37542732748", + "37543978473", + "37545094812", + "37545716546", + "37546366036", + "37559982334", + "37560420246", + "37564096977", + "37565786884", + "37640916778", + "37691104551", + "37691665506", + "37693401496", + "37694473528", + "37701124596", + "37707903246", + "37718098351", + "37728544941", + "37731666423", + "37734578981", + "37734678850", + "37734795948", + "37738679055", + "37739140797", + "37739365674", + "37739711591", + "37739845694", + "37739864688", + "37739923087", + "37740005194", + "37741231927", + "37741305367", + "37741434771", + "37742517444", + "37742917771", + "37743060710", + "37743622252", + "37744583272", + "37745729841", + "37747272797", + "37747282656", + "37749419367", + "37761125544", + "37762901988", + "37762951968", + "37766639989", + "37799081635", + "37801128025", + "37803230995", + "37803446927", + "37805183773", + "37810177756", + "37811045289", + "37812094929", + "37813893652", + "37816281826", + "37834873799", + "37887208552", + "37887677058", + "37888258971", + "37892285758", + "37892504905", + "37897476192", + "38009577313", + "38009751458", + "38077369614", + "38087167729", + "38087206509", + "38087752336", + "38089176598", + "38093707846", + "38093940617", + "38094956965", + "38095294863", + "38096225501", + "38097202777", + "38097643686", + "38097811247", + "38097941308", + "38098277242", + "38098894928", + "38099588461", + "38100216011", + "38100768977", + "38101113898", + "38101502677", + "38101616162", + "38102454452", + "38102657574", + "38103692513", + "38105628107", + "38106695235", + "38114482325", + "38114743865", + "38114753174", + "38115320223", + "38115388121", + "38115472535", + "38116231959", + "38130211171", + "38146252958", + "38149926651", + "38150794867", + "38151498909", + "38151698313", + "38152603302", + "38152625837", + "38152692645", + "38152760329", + "38153103642", + "38153201884", + "38153296624", + "38153330406", + "38153440113", + "38153468775", + "38153478324", + "38153524782", + "38153545779", + "38154139862", + "38154283169", + "38154299190", + "38154371839", + "38154587215", + "38155175678", + "38155221280", + "38156016740", + "38159532295", + "38159643806", + "38160879452", + "38161561304", + "38162103937", + "38165705267", + "38168912246", + "38186078591", + "38189716570", + "38190450074", + "38191930614", + "38210650474", + "38254994238", + "38292011452", + "38295240201", + "38304815896", + "38320765570", + "38335637302", + "38335683298", + "38336135754", + "38336400566", + "38336529808", + "38337974933", + "38339027571", + "38344574992", + "38345572852", + "38416346427", + "38416389781", + "38418037093", + "38421455413", + "38428252213", + "38439240685", + "38478095918", + "38478947158", + "38508804072", + "38512515424", + "38514180564", + "38518769031", + "38531754196", + "38604265113", + "38605448921", + "38624056266", + "38635247772", + "38637567300", + "38669096043", + "38669488941", + "38671593723", + "38673080757", + "38674274388", + "38725836286", + "38785477428", + "38800593092", + "38815098576", + "38817374062", + "38824819744", + "38827690901", + "38829062754", + "38829515637", + "38829994639", + "38832498886", + "38833122675", + "38835263541", + "38842124064", + "38864846669", + "38964584946", + "38964801793", + "38964995514", + "38965078377", + "38965221701", + "38965260163", + "38969798938", + "38969924945", + "38972920715", + "38973329985", + "38973338025", + "38973420671", + "38974669950", + "38975010169", + "38975560167", + "38975789267", + "38976253774", + "38976351570", + "38978531340", + "38978533724", + "38978745945", + "38979194128", + "38979925666", + "38980005793", + "38980008914", + "38980089002", + "38980452190", + "38980914399", + "38981388043", + "38981492840", + "38981523623", + "38981831429", + "38982033023", + "38982079721", + "38982609798", + "38982836813", + "38983306684", + "38983355048", + "38983422226", + "38983471330", + "38983723250", + "38983767702", + "38984904318", + "38984910364", + "38984977679", + "38985082997", + "38985111066", + "38985644097", + "38985800954", + "38986120559", + "38986222182", + "38986505512", + "38987526600", + "38987672759", + "38987773038", + "38987895421", + "38988823602", + "38988898707", + "38988927989", + "38989956807", + "38990111230", + "38990141270", + "38991706282", + "38991832658", + "38992787872", + "38992860354", + "38993076767", + "38994816769", + "38996100960", + "38996605130", + "38996899290", + "38997410828", + "38998005661", + "38999746202", + "39000951833", + "39001049596", + "39001107574", + "39001765726", + "39001816434", + "39002175316", + "39002269281", + "39003503204", + "39003571667", + "39004466422", + "39004497896", + "39005025976", + "39005664700", + "39006896632", + "39009803881", + "39010388987", + "39010883961", + "39011101113", + "39017702380", + "39044530662", + "39051148306" + ], + "cases": { + "auq-format-gate": { + "observations": 162, + "withCommands": 0, + "commands": [], + "costUsd": 0, + "costRuns": 110 + }, + "autoplan-guard-pty": { + "observations": 7, + "withCommands": 0, + "commands": [], + "costUsd": 0, + "costRuns": 89 + }, + "benchmark-workflow": { + "observations": 179, + "withCommands": 179, + "commands": [ + "bin/gstack-ci-gate", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-decision-search", + "bin/gstack-diff-scope", + "bin/gstack-evidence", + "bin/gstack-issue-guard", + "bin/gstack-learnings-log", + "bin/gstack-next-version", + "bin/gstack-paths", + "bin/gstack-question-log", + "bin/gstack-question-preference", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-wtree", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "browse-basic": { + "observations": 158, + "withCommands": 158, + "commands": [ + "bin/gstack-learnings-log", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 106 + }, + "browse-snapshot": { + "observations": 158, + "withCommands": 158, + "commands": [ + "bin/gstack-learnings-log", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 106 + }, + "canary-workflow": { + "observations": 182, + "withCommands": 182, + "commands": [ + "bin/gstack-ci-gate", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-decision-search", + "bin/gstack-diff-scope", + "bin/gstack-evidence", + "bin/gstack-issue-guard", + "bin/gstack-learnings-log", + "bin/gstack-next-version", + "bin/gstack-paths", + "bin/gstack-question-log", + "bin/gstack-question-preference", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-wtree", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "context-recovery-artifacts": { + "observations": 166, + "withCommands": 161, + "commands": [ + "bin/gstack-context-recovery", + "bin/gstack-decision-search", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-remote-identity.sh", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-state-root.sh", + "bin/gstack-timeline-log", + "bin/gstack-timeline-read", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "context-restore-loads-latest": { + "observations": 167, + "withCommands": 162, + "commands": [ + "bin/gstack-context-recovery", + "bin/gstack-decision-search", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-remote-identity.sh", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-state-root.sh", + "bin/gstack-timeline-log", + "bin/gstack-timeline-read", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 105 + }, + "context-save-writes-file": { + "observations": 166, + "withCommands": 161, + "commands": [ + "bin/gstack-context-recovery", + "bin/gstack-decision-search", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-remote-identity.sh", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-state-root.sh", + "bin/gstack-timeline-log", + "bin/gstack-timeline-read", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "cso-diff-mode": { + "observations": 157, + "withCommands": 157, + "commands": [ + "design/dist/design" + ], + "costUsd": 1.5371, + "costRuns": 91 + }, + "design-review-detector-shim": { + "observations": 174, + "withCommands": 174, + "commands": [ + "bin/gstack-design-detect.ts", + "bin/gstack-redact", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 92 + }, + "design-review-detector-shim-dom": { + "observations": 170, + "withCommands": 170, + "commands": [ + "bin/gstack-design-detect.ts", + "bin/gstack-redact", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "design-review-plugin-handoff": { + "observations": 162, + "withCommands": 162, + "commands": [ + "bin/gstack-config", + "bin/gstack-design-detect.ts", + "bin/gstack-learnings-search", + "bin/gstack-redact", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 91 + }, + "diagram-triplet": { + "observations": 141, + "withCommands": 0, + "commands": [], + "costUsd": 0, + "costRuns": 106 + }, + "docsync-spawned": { + "observations": 181, + "withCommands": 180, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 1.2246, + "costRuns": 107 + }, + "document-release": { + "observations": 155, + "withCommands": 155, + "commands": [ + "bin/gstack-bun-version.sh", + "bin/gstack-codex-probe", + "bin/gstack-config", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-decision-search", + "bin/gstack-learnings-log", + "bin/gstack-next-version", + "bin/gstack-paths", + "bin/gstack-review-log", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-update-check", + "bin/gstack-upgrade", + "bin/gstack-version-bump", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 91 + }, + "gstack-upgrade-happy-path": { + "observations": 157, + "withCommands": 156, + "commands": [ + "bin/gstack-bun-version.sh", + "bin/gstack-config", + "bin/gstack-paths", + "bin/gstack-update-check", + "bin/gstack-upgrade" + ], + "costUsd": 0, + "costRuns": 117 + }, + "hermetic-canary": { + "observations": 167, + "withCommands": 167, + "commands": [], + "costUsd": 0.0127, + "costRuns": 101 + }, + "hermetic-sentinel": { + "observations": 167, + "withCommands": 167, + "commands": [], + "costUsd": 0.013, + "costRuns": 101 + }, + "investigate-owned-abort": { + "observations": 178, + "withCommands": 178, + "commands": [], + "costUsd": 0.2212, + "costRuns": 105 + }, + "investigate-owned-completion": { + "observations": 164, + "withCommands": 164, + "commands": [], + "costUsd": 0.2184, + "costRuns": 105 + }, + "investigate-owned-ending-error": { + "observations": 178, + "withCommands": 178, + "commands": [], + "costUsd": 0.2694, + "costRuns": 105 + }, + "land-and-deploy-first-run": { + "observations": 179, + "withCommands": 179, + "commands": [ + "bin/gstack-ci-gate", + "bin/gstack-config", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-decision-search", + "bin/gstack-diff-scope", + "bin/gstack-evidence", + "bin/gstack-issue-guard", + "bin/gstack-learnings-log", + "bin/gstack-next-version", + "bin/gstack-paths", + "bin/gstack-question-log", + "bin/gstack-question-preference", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-wtree", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "land-and-deploy-review-gate": { + "observations": 175, + "withCommands": 175, + "commands": [ + "bin/gstack-ci-gate", + "bin/gstack-config", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-decision-search", + "bin/gstack-diff-scope", + "bin/gstack-evidence", + "bin/gstack-issue-guard", + "bin/gstack-learnings-log", + "bin/gstack-next-version", + "bin/gstack-paths", + "bin/gstack-question-log", + "bin/gstack-question-preference", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-wtree", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 92 + }, + "land-and-deploy-workflow": { + "observations": 180, + "withCommands": 180, + "commands": [ + "bin/gstack-ci-gate", + "bin/gstack-config", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-decision-search", + "bin/gstack-diff-scope", + "bin/gstack-evidence", + "bin/gstack-issue-guard", + "bin/gstack-learnings-log", + "bin/gstack-next-version", + "bin/gstack-paths", + "bin/gstack-question-log", + "bin/gstack-question-preference", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-wtree", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 92 + }, + "learnings-show": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-context-recovery", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug" + ], + "costUsd": 0, + "costRuns": 105 + }, + "office-hours-auto-mode": { + "observations": 181, + "withCommands": 0, + "commands": [], + "costUsd": 0, + "costRuns": 108 + }, + "operational-learning": { + "observations": 155, + "withCommands": 155, + "commands": [ + "bin/gstack-learnings-log", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 91 + }, + "plan-ceo-finding-floor": { + "observations": 199, + "withCommands": 197, + "commands": [ + "bin/gstack-ceo-mode-handoff", + "bin/gstack-config", + "bin/gstack-context-recovery", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-start", + "bin/gstack-slug", + "browse/bin/remote-slug" + ], + "costUsd": 0, + "costRuns": 92 + }, + "plan-ceo-review": { + "observations": 59, + "withCommands": 59, + "commands": [] + }, + "plan-ceo-review-plan-mode": { + "observations": 166, + "withCommands": 166, + "commands": [ + "bin/gstack-context-recovery", + "bin/gstack-skill-start", + "bin/gstack-slug", + "browse/bin/remote-slug" + ], + "costUsd": 0, + "costRuns": 109 + }, + "plan-design-review-no-ui-scope": { + "observations": 166, + "withCommands": 166, + "commands": [ + "bin/gstack-design-detect.ts", + "bin/gstack-learnings-log", + "bin/gstack-redact", + "bin/gstack-skill-end", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 106 + }, + "plan-design-with-ui-scope": { + "observations": 183, + "withCommands": 183, + "commands": [ + "bin/gstack-config", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-start", + "bin/gstack-slug", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "plan-devex-finding-floor": { + "observations": 180, + "withCommands": 178, + "commands": [ + "bin/gstack-config", + "bin/gstack-context-recovery", + "bin/gstack-question-log", + "bin/gstack-review-log", + "bin/gstack-skill-start", + "bin/gstack-slug", + "browse/bin/remote-slug", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "plan-devex-review-plan-mode": { + "observations": 161, + "withCommands": 0, + "commands": [], + "costUsd": 0, + "costRuns": 90 + }, + "plan-eng-coverage-audit": { + "observations": 179, + "withCommands": 178, + "commands": [], + "costUsd": 0.4321, + "costRuns": 111 + }, + "plan-mode-no-op": { + "observations": 180, + "withCommands": 0, + "commands": [], + "costUsd": 0, + "costRuns": 93 + }, + "plan-review-report": { + "observations": 183, + "withCommands": 100, + "commands": [ + "bin/gstack-codex-probe", + "bin/gstack-config", + "bin/gstack-office-hours-review", + "bin/gstack-review-log" + ], + "costUsd": 0, + "costRuns": 111 + }, + "plan-tune-inspect": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-config", + "bin/gstack-developer-profile", + "bin/gstack-paths", + "bin/gstack-question-log", + "bin/gstack-question-preference", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-state-root.sh" + ], + "costUsd": 0, + "costRuns": 105 + }, + "qa-bootstrap": { + "observations": 174, + "withCommands": 173, + "commands": [ + "bin/gstack-config", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-slug", + "browse/bin/find-browse", + "browse/bin/remote-slug", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 108 + }, + "qa-functional-cli-fix": { + "observations": 159, + "withCommands": 158, + "commands": [ + "bin/gstack-qa-evidence" + ], + "costUsd": 1.3402, + "costRuns": 106 + }, + "qa-functional-cli-report": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-qa-evidence" + ], + "costUsd": 1.1367, + "costRuns": 106 + }, + "qa-functional-webhook-fix": { + "observations": 159, + "withCommands": 158, + "commands": [ + "bin/gstack-qa-evidence" + ], + "costUsd": 1.4296, + "costRuns": 106 + }, + "qa-functional-webhook-report": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-qa-evidence" + ], + "costUsd": 1.3103, + "costRuns": 106 + }, + "qa-only-no-fix": { + "observations": 174, + "withCommands": 174, + "commands": [ + "bin/gstack-config", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-slug", + "browse/bin/find-browse", + "browse/bin/remote-slug", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 92 + }, + "qa-quick": { + "observations": 174, + "withCommands": 174, + "commands": [ + "bin/gstack-config", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-slug", + "browse/bin/find-browse", + "browse/bin/remote-slug", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 92 + }, + "retro": { + "observations": 134, + "withCommands": 134, + "commands": [ + "bin/gstack-config", + "bin/gstack-decision-search", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-retro-metrics", + "bin/gstack-slug", + "bin/gstack-state-root.sh" + ] + }, + "retro-base-branch": { + "observations": 159, + "withCommands": 25, + "commands": [ + "bin/gstack-config", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-retro-metrics", + "bin/gstack-slug" + ], + "costUsd": 0, + "costRuns": 94 + }, + "review-army-delivery-audit": { + "observations": 177, + "withCommands": 177, + "commands": [ + "bin/gstack-diff-scope", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-review-log", + "bin/gstack-specialist-stats" + ], + "costUsd": 0, + "costRuns": 95 + }, + "review-army-json-findings": { + "observations": 160, + "withCommands": 160, + "commands": [ + "bin/gstack-diff-scope", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-review-log", + "bin/gstack-specialist-stats" + ], + "costUsd": 0, + "costRuns": 95 + }, + "review-army-migration-safety": { + "observations": 164, + "withCommands": 164, + "commands": [ + "bin/gstack-diff-scope", + "bin/gstack-egress-lib.sh", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-review-log", + "bin/gstack-specialist-stats", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 95 + }, + "review-army-perf-n-plus-one": { + "observations": 165, + "withCommands": 165, + "commands": [ + "bin/gstack-decision-search", + "bin/gstack-diff-scope", + "bin/gstack-egress-lib.sh", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-review-log", + "bin/gstack-specialist-stats", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 95 + }, + "review-army-quality-score": { + "observations": 160, + "withCommands": 160, + "commands": [ + "bin/gstack-diff-scope", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-review-log", + "bin/gstack-specialist-stats" + ], + "costUsd": 0.3515, + "costRuns": 95 + }, + "review-base-branch": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-diff-scope", + "bin/gstack-review-read" + ], + "costUsd": 0, + "costRuns": 91 + }, + "review-coverage-audit": { + "observations": 178, + "withCommands": 167, + "commands": [], + "costUsd": 0.4079, + "costRuns": 105 + }, + "review-dashboard-via": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-diff-scope", + "bin/gstack-review-read" + ], + "costUsd": 0, + "costRuns": 93 + }, + "review-enum-completeness": { + "observations": 166, + "withCommands": 166, + "commands": [ + "bin/gstack-decision-search", + "bin/gstack-review-log" + ], + "costUsd": 0, + "costRuns": 91 + }, + "review-exploratory-small-cli": { + "observations": 170, + "withCommands": 169, + "commands": [ + "bin/gstack-codex-probe", + "bin/gstack-config", + "bin/gstack-diff-scope", + "bin/gstack-learnings-search", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-specialist-stats" + ], + "costUsd": 1.9457, + "costRuns": 115 + }, + "review-gate-integrity": { + "observations": 0, + "withCommands": 0, + "commands": [], + "costUsd": 0, + "costRuns": 81 + }, + "review-sql-injection": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-decision-search", + "bin/gstack-review-log" + ], + "costUsd": 0, + "costRuns": 105 + }, + "review-test-value": { + "observations": 169, + "withCommands": 169, + "commands": [], + "costUsd": 0.6768, + "costRuns": 105 + }, + "safety-codex-consult-embed": { + "observations": 68, + "withCommands": 68, + "commands": [ + "bin/gstack-codex-probe", + "bin/gstack-paths" + ], + "costUsd": 0.9964, + "costRuns": 91 + }, + "safety-pair-agent-block": { + "observations": 68, + "withCommands": 68, + "commands": [ + "bin/gstack-config", + "browse/dist/browse" + ], + "costUsd": 0.3055, + "costRuns": 91 + }, + "safety-ship-stale-evidence": { + "observations": 67, + "withCommands": 67, + "commands": [ + "bin/gstack-ci-gate", + "bin/gstack-evidence", + "bin/gstack-slug", + "bin/gstack-wtree" + ], + "costUsd": 0.5915, + "costRuns": 91 + }, + "setup-deploy-workflow": { + "observations": 181, + "withCommands": 181, + "commands": [ + "bin/gstack-ci-gate", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-decision-search", + "bin/gstack-diff-scope", + "bin/gstack-evidence", + "bin/gstack-issue-guard", + "bin/gstack-learnings-log", + "bin/gstack-next-version", + "bin/gstack-paths", + "bin/gstack-question-log", + "bin/gstack-question-preference", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-wtree", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 106 + }, + "shared-libs-read-only": { + "observations": 166, + "withCommands": 166, + "commands": [ + "bin/gstack-safe-git", + "browse/dist/browse" + ], + "costUsd": 0.786, + "costRuns": 92 + }, + "shared-libs-review-index-flags": { + "observations": 177, + "withCommands": 176, + "commands": [ + "bin/gstack-review-log", + "bin/gstack-review-read" + ], + "costUsd": 2.4926, + "costRuns": 91 + }, + "shared-libs-review-lifecycle": { + "observations": 174, + "withCommands": 173, + "commands": [ + "bin/gstack-egress-lib.sh", + "bin/gstack-review-log", + "bin/gstack-review-read" + ], + "costUsd": 2.4643, + "costRuns": 92 + }, + "shared-libs-review-path-eligibility": { + "observations": 171, + "withCommands": 170, + "commands": [ + "bin/gstack-review-log", + "bin/gstack-review-read" + ], + "costUsd": 3.9227, + "costRuns": 91 + }, + "shared-libs-review-prior-coverage": { + "observations": 164, + "withCommands": 164, + "commands": [ + "bin/gstack-review-log", + "bin/gstack-review-read" + ], + "costUsd": 2.517, + "costRuns": 91 + }, + "shared-libs-review-revalidation": { + "observations": 174, + "withCommands": 171, + "commands": [ + "bin/gstack-review-log", + "bin/gstack-review-read" + ], + "costUsd": 4.8033, + "costRuns": 92 + }, + "shared-libs-unsupported-git": { + "observations": 159, + "withCommands": 159, + "commands": [ + "bin/gstack-safe-git", + "browse/dist/browse" + ], + "costUsd": 0.5855, + "costRuns": 92 + }, + "ship-base-branch": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-diff-scope", + "bin/gstack-review-read" + ], + "costUsd": 0, + "costRuns": 92 + }, + "ship-coverage-audit": { + "observations": 157, + "withCommands": 156, + "commands": [ + "bin/gstack-bun-version.sh", + "bin/gstack-config", + "bin/gstack-paths", + "bin/gstack-slug", + "bin/gstack-update-check", + "bin/gstack-upgrade" + ], + "costUsd": 0.7746, + "costRuns": 107 + }, + "ship-coverage-value": { + "observations": 169, + "withCommands": 168, + "commands": [ + "bin/gstack-paths", + "bin/gstack-slug" + ], + "costUsd": 0.9375, + "costRuns": 107 + }, + "ship-docsync-completion": { + "observations": 152, + "withCommands": 151, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-session-kind", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 3.5392, + "costRuns": 110 + }, + "ship-docsync-current": { + "observations": 153, + "withCommands": 152, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-session-kind", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 3.2887, + "costRuns": 110 + }, + "ship-docsync-failure": { + "observations": 156, + "withCommands": 155, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 2.028, + "costRuns": 110 + }, + "ship-docsync-late-result": { + "observations": 170, + "withCommands": 168, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 0.9185, + "costRuns": 109 + }, + "ship-docsync-launch-failure": { + "observations": 154, + "withCommands": 153, + "commands": [ + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 0.4608, + "costRuns": 110 + }, + "ship-docsync-missing-asset": { + "observations": 148, + "withCommands": 147, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 1.3017, + "costRuns": 109 + }, + "ship-docsync-missing-marker": { + "observations": 175, + "withCommands": 174, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 0.5063, + "costRuns": 110 + }, + "ship-docsync-recovery": { + "observations": 160, + "withCommands": 159, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 1.0055, + "costRuns": 110 + }, + "ship-docsync-stale-after": { + "observations": 172, + "withCommands": 171, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 1.1196, + "costRuns": 110 + }, + "ship-docsync-stale-before": { + "observations": 165, + "withCommands": 164, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 1.1591, + "costRuns": 110 + }, + "ship-docsync-store": { + "observations": 154, + "withCommands": 153, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-session-kind", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 3.5563, + "costRuns": 110 + }, + "ship-docsync-timeout-unsettled": { + "observations": 167, + "withCommands": 166, + "commands": [ + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 0.5324, + "costRuns": 110 + }, + "ship-exploratory-late-input": { + "observations": 168, + "withCommands": 167, + "commands": [ + "bin/gstack-config", + "bin/gstack-diff-scope", + "bin/gstack-learnings-search", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-specialist-stats" + ], + "costUsd": 2.1399, + "costRuns": 115 + }, + "ship-exploratory-plan-checks": { + "observations": 167, + "withCommands": 166, + "commands": [ + "bin/gstack-config", + "bin/gstack-diff-scope", + "bin/gstack-learnings-search", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-specialist-stats" + ], + "costUsd": 1.8717, + "costRuns": 115 + }, + "ship-exploratory-small-cli": { + "observations": 171, + "withCommands": 170, + "commands": [ + "bin/gstack-config", + "bin/gstack-diff-scope", + "bin/gstack-learnings-search", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-specialist-stats" + ], + "costUsd": 1.8332, + "costRuns": 115 + }, + "ship-exploratory-unavailable": { + "observations": 172, + "withCommands": 171, + "commands": [ + "bin/gstack-config", + "bin/gstack-diff-scope", + "bin/gstack-learnings-search", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-specialist-stats" + ], + "costUsd": 1.837, + "costRuns": 115 + }, + "ship-local-hook-preservation": { + "observations": 161, + "withCommands": 161, + "commands": [ + "bin/gstack-config", + "bin/gstack-paths", + "bin/gstack-redact" + ], + "costUsd": 0.307, + "costRuns": 107 + }, + "ship-local-workflow": { + "observations": 157, + "withCommands": 156, + "commands": [ + "bin/gstack-bun-version.sh", + "bin/gstack-config", + "bin/gstack-paths", + "bin/gstack-update-check", + "bin/gstack-upgrade" + ], + "costUsd": 0, + "costRuns": 107 + }, + "ship-managed-hook-refresh": { + "observations": 160, + "withCommands": 160, + "commands": [ + "bin/gstack-config", + "bin/gstack-paths", + "bin/gstack-redact" + ], + "costUsd": 0.2737, + "costRuns": 107 + }, + "ship-skipped-queued-finding": { + "observations": 154, + "withCommands": 153, + "commands": [], + "costUsd": 0.6029, + "costRuns": 92 + }, + "ship-triage": { + "observations": 180, + "withCommands": 180, + "commands": [], + "costUsd": 0, + "costRuns": 92 + }, + "ship-unmanaged-hook-consent": { + "observations": 161, + "withCommands": 161, + "commands": [ + "bin/gstack-config", + "bin/gstack-paths", + "bin/gstack-redact" + ], + "costUsd": 0.3538, + "costRuns": 107 + }, + "skillify-approval-reject": { + "observations": 186, + "withCommands": 175, + "commands": [ + "bin/gstack-learnings-log", + "bin/gstack-state-root.sh", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "skillify-happy-path": { + "observations": 186, + "withCommands": 175, + "commands": [ + "bin/gstack-learnings-log", + "bin/gstack-state-root.sh", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "skillify-provenance-refusal": { + "observations": 190, + "withCommands": 175, + "commands": [ + "bin/gstack-learnings-log", + "bin/gstack-state-root.sh", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 108 + }, + "skillmd-no-local-binary": { + "observations": 155, + "withCommands": 155, + "commands": [ + "bin/gstack-learnings-log", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 91 + }, + "skillmd-outside-git": { + "observations": 155, + "withCommands": 155, + "commands": [ + "bin/gstack-learnings-log", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 91 + }, + "skillmd-setup-discovery": { + "observations": 156, + "withCommands": 156, + "commands": [ + "bin/gstack-learnings-log", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 117 + }, + "test-audit-report-only": { + "observations": 169, + "withCommands": 154, + "commands": [], + "costUsd": 0.9871, + "costRuns": 106 + }, + "timeline-event-flow": { + "observations": 166, + "withCommands": 161, + "commands": [ + "bin/gstack-context-recovery", + "bin/gstack-decision-search", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-remote-identity.sh", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-state-root.sh", + "bin/gstack-timeline-log", + "bin/gstack-timeline-read", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 90 + }, + "tpa-absent-darwin": { + "observations": 184, + "withCommands": 180, + "commands": [], + "costUsd": 0.2257, + "costRuns": 91 + }, + "tpa-absent-linux": { + "observations": 185, + "withCommands": 185, + "commands": [], + "costUsd": 0.2243, + "costRuns": 106 + }, + "tpa-apple-ban": { + "observations": 184, + "withCommands": 149, + "commands": [], + "costUsd": 0.3062, + "costRuns": 91 + }, + "tpa-broken": { + "observations": 184, + "withCommands": 181, + "commands": [], + "costUsd": 0.2265, + "costRuns": 91 + }, + "tpa-present": { + "observations": 185, + "withCommands": 185, + "commands": [], + "costUsd": 0.2325, + "costRuns": 106 + } + } +} diff --git a/test/helpers/exercised-prototype.ts b/test/helpers/exercised-prototype.ts new file mode 100644 index 000000000..0d7dd801a --- /dev/null +++ b/test/helpers/exercised-prototype.ts @@ -0,0 +1,149 @@ +/** + * Frozen parity oracle for the exercised-path map (W2-A2): the measured + * prototype (GSTA-45 investigation, exercised.ts at 521fd42, REPORT-2) with + * only its module-level root and base derivation turned into parameters, its + * path-prefix regex rewritten to an equivalent non-backtracking form, its + * import scan switched to A1's scanCodeImports (the prototype ran on the + * shebang-fixed base; A1 also fixed the TSX loader misreading .ts generics), + * and the per-case build of its replay (variant cu) appended. Do not edit its + * edge rules: test/exercised-map.test.ts pins that the shipped + * exercisedConsumers() equals or is a reviewed superset of this map. + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { derivedDependencies, scanCodeImports, SELECTION_DATA_MODULES, type DependencyMaps, type ExercisedMap } from '../../scripts/pr-dependencies'; +import { matchGlob } from './test-selection'; +import { isPaidTestFile } from './paid-test-set'; + +export function prototypeConsumers(maps: DependencyMaps, ROOT: string, observedMap: ExercisedMap | null, PR_FULL_GATE_FILES: readonly string[]): Map> { + const base = derivedDependencies(maps, ROOT); + const tracked: string[] = [...base.tracked]; + const trackedSet = base.tracked as Set; + const cache = new Map(); + const read = (f: string) => { if (!cache.has(f)) { let t = ''; try { if (trackedSet.has(f) && fs.statSync(path.join(ROOT, f)).size < 3_000_000) t = fs.readFileSync(path.join(ROOT, f), 'utf8'); } catch {} cache.set(f, t); } return cache.get(f)!; }; + const CODE = /\.(?:[cm]?[jt]s|tsx|jsx)$/; + const isBunShebang = (f: string) => /^#!.*\b(?:bun|node)\b/.test(read(f).slice(0, 200)); + const isCode = (f: string) => CODE.test(f) || isBunShebang(f); + const isShell = (f: string) => !isCode(f) && (/\.sh$/.test(f) || f === 'setup' || /^#!.*\b(?:ba|z|da)?sh\b/.test(read(f).slice(0, 200))); + const resolveSpec = (spec: string, from: string) => { + if (!spec.startsWith('.')) return null; + const b = path.posix.normalize(path.posix.join(path.posix.dirname(from), spec)); + for (const c of [b, b + '.ts', b + '.tsx', b + '.js', b + '.mjs', b + '/index.ts', b + '/index.js']) if (trackedSet.has(c)) return c; + return null; + }; + const dirs = new Set(tracked.flatMap(f => f.split('/').slice(0, -1).map((_, i, p) => p.slice(0, i + 1).join('/')))); + const REPO_PATH = /(?:^|[^\w.@-])((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|test\/helpers|test\/fixtures|capy)\/[\w.@-]+(?:\/[\w.@-]+)*)/g; + const JOINED = /['"\x60](bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents)['"\x60]\s*,\s*['"\x60]([\w.@-]+)['"\x60]/g; + const binByName = new Map(); + for (const f of tracked) if (f.startsWith('bin/') || f.startsWith('browse/bin/')) { const b = path.posix.basename(f); binByName.set(b, f); binByName.set(b.replace(/\.(ts|sh|js)$/, ''), f); } + const BINARIES = [...read('scripts/build.sh').matchAll(/--compile\b[^\n]*?\s([\w./-]+\.ts)\s+--outfile\s+([\w./-]+)/g)].map(m => ({ outfile: m[2]!, entry: m[1]!, tree: m[1]!.startsWith('bin/') ? [m[1]!] : tracked.filter(f => matchGlob(f, path.posix.dirname(m[1]!) + '/**')) })); + let pkgScripts: Record = {}; try { pkgScripts = JSON.parse(read('package.json')).scripts ?? {}; } catch {} + const expandRef = (ref: string) => { ref = ref.replace(/[.]+$/, ''); if (trackedSet.has(ref)) return [ref]; if (dirs.has(ref)) return tracked.filter(f => f.startsWith(ref + '/')); return []; }; + const stripCode = (t: string) => t.replace(/\/\*[\s\S]*?\*\//g, '').replace(/^[ \t]*\/\/.*$/gm, ''); + const stripMessages = (t: string) => { + let out = '', i = 0; + while (i < t.length) { + const m = /^(?:echo|printf|say|warn|die|fail|err|log|info|note|_dc_row|row)\b/.exec(t.slice(i, i + 8)); + const prev = i === 0 ? '\n' : t[i - 1]!; + if (m && /[\s;|&(\x60{]/.test(prev)) { + let j = i + m[0].length, q: string | null = null; + while (j < t.length) { const c = t[j]!; if (q) { if (c === '\\') { j += 2; continue; } if (c === q) q = null; j++; continue; } if (c === '"' || c === "'") { q = c; j++; continue; } if (c === '\n' || c === ';' || c === '|' || c === '&' || c === ')') break; j++; } + out += ' '; i = j; continue; + } + out += t[i]; i++; + } + return out; + }; + const stripShell = (t: string) => stripMessages(t + .replace(/<<-?\s*['"]?(\w+)['"]?[^\n]*\n[\s\S]*?\n\s*\1\s*$/gm, '') + .split('\n').filter(l => !/^\s*#(?!!)/.test(l)) + .map(l => l.replace(/(^\s*|\bin\s+|;;\s*)[\w*."'-]+(?:\s*\|\s*[\w*."'-]+)*\)(?=\s|$)/g, '$1').replace(/\s#\s.*$/, '')) + .join('\n')); + const CMD_START = /(?:^|[;|&({\x60]|\$\(|\b(?:then|do|else|exec|command|nohup|bash|sh|bun|node|timeout\s+\S+|env(?:\s+\w+=\S*)*|if|while|until|!|time))\s*"?$/; + // Changed from the prototype: its `(?:\/?[\w.@-]+)*\/` path prefix backtracks exponentially; this is the same language (words separated by single slashes, ending in a slash) in an unambiguous form. + const PREFIX = /(?:"?(?:\$\{?\w+\}?|\$\(dirname\s+"?\$0"?\)|~|\.)?(?:\/(?:[\w.@-]+\/)*|(?:[\w.@-]+\/)+))?"?$/; + const cmdOk = (before: string) => { const b = before.replace(PREFIX, ''); return CMD_START.test(b) || (/\/$/.test(before) && /\b\w+="?$/.test(b)); }; + const shellExec = (text: string, from: string, out: Set) => { + for (const m of text.matchAll(/gstack-[a-z0-9][a-z0-9-]*[a-z0-9](?:\.(?:sh|ts))?\b/g)) { + const i = m.index!; const ls = text.lastIndexOf('\n', i - 1) + 1; + if (!cmdOk(text.slice(ls, i))) continue; + const b = binByName.get(m[0]); if (b) out.add(b); + } + for (const m of text.matchAll(/(?:bin|lib|scripts|browse|design|make-pdf)\/[\w.@-]+(?:\/[\w.@-]+)*/g)) { + const i = m.index!; const ls = text.lastIndexOf('\n', i - 1) + 1; const before = text.slice(ls, i); + if (/[\w.@-]$/.test(before)) continue; + if (!cmdOk(before) && !/\bbun\s+(?:run\s+)?"?(?:\$\{?\w+\}?\/)?$/.test(before)) continue; + for (const f of expandRef(m[0])) out.add(f); + } + for (const m of text.matchAll(/(?:bin|lib|scripts|browse|design|make-pdf|hosts|capy|test\/fixtures|test\/helpers)\/[\w.@-]+(?:\/[\w.@-]+)*\.(?:json|jsonl|env|txt|ya?ml|toml|csv|tsv)\b/g)) for (const f of expandRef(m[0])) out.add(f); + for (const m of text.matchAll(/(?:^|[\s;&|({])(?:\.|source)\s+([^;|&\n]+)/gm)) { const names = [...m[1]!.matchAll(/([\w.-]+\.sh|gstack-[\w.-]+)/g)].map(x => x[1]!); if (!names.length) continue; const b = names[names.length - 1]!; const sib = path.posix.join(path.posix.dirname(from), b); if (trackedSet.has(sib)) out.add(sib); else if (binByName.get(b)) out.add(binByName.get(b)!); } + for (const bin of BINARIES) if (text.includes(bin.outfile) || (bin.outfile === 'browse/dist/browse' && /\$B\b|find-browse/.test(text)) || (bin.outfile === 'design/dist/design' && /\$D\b/.test(text))) for (const f of bin.tree) out.add(f); + for (const m of text.matchAll(/\b(?:bun|npm) run ([a-z][\w:.-]*)/g)) { const c = pkgScripts[m[1]!]; if (c) for (const r of c.matchAll(REPO_PATH)) for (const f of expandRef(r[1]!)) out.add(f); } + }; + const bashBlocks = (md: string) => [...md.matchAll(/\x60\x60\x60(?:bash|sh|shell)\n([\s\S]*?)\x60\x60\x60/g)].map(m => m[1]!).join('\n'); + const edges = new Map(); + const execRefs = (f: string): string[] => { + if (edges.has(f)) return edges.get(f)!; + const out = new Set(); + const raw = read(f); + if ((SELECTION_DATA_MODULES as readonly string[]).includes(f) || !raw) { edges.set(f, []); return []; } + if (isCode(f)) { + try { for (const spec of scanCodeImports(f, raw)) { const h = resolveSpec(spec, f); if (h) out.add(h); } } catch {} + const code = stripCode(raw.replace(/^#!.*/, '')); + for (const m of code.matchAll(/['"\x60](?:\$\{[^}]+\}\/)?((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)['"\x60]/g)) for (const x of expandRef(m[1]!)) out.add(x); + for (const m of code.matchAll(JOINED)) for (const x of expandRef(m[1] + '/' + m[2])) out.add(x); + for (const bin of BINARIES) if (code.includes("'" + bin.outfile + "'") || code.includes('"' + bin.outfile + '"')) for (const x of bin.tree) out.add(x); + } else if (isShell(f)) { + shellExec(stripShell(raw), f, out); + } else if (/\.(md|tmpl)$/.test(f)) { + shellExec(stripShell(bashBlocks(raw)), f, out); + } + out.delete(f); + const list = [...out]; edges.set(f, list); return list; + }; + const isFullGate = (f: string) => PR_FULL_GATE_FILES.some((p: string) => matchGlob(f, p)); + const closure = (entries: Iterable) => { + const seen = new Set(); const q = [...entries].filter(f => trackedSet.has(f)); + while (q.length) { const f = q.pop()!; if (seen.has(f)) continue; seen.add(f); if (base.global.has(f) || isFullGate(f)) continue; for (const n of execRefs(f)) if (!seen.has(n) && !(/\.test\.tsx?$/.test(n) && isPaidTestFile(n))) q.push(n); } + return seen; + }; + const expand = (p: string) => p.includes('*') ? tracked.filter(f => matchGlob(f, p)) : trackedSet.has(p) ? [p] : []; + const OWNER = /^test\/[^/]+\.test\.ts$/; + function caseEntries(patterns: string[], defaultOwner: string | null) { + const owners = patterns.filter(p => OWNER.test(p) && isPaidTestFile(p)); + const files = [...(owners.length ? owners : defaultOwner ? [defaultOwner] : []), ...patterns.flatMap(expand)]; + const gen = files.filter(f => f.endsWith('.tmpl')).map(f => f.replace(/\.tmpl$/, '')).filter(f => trackedSet.has(f)); + return [...files, ...gen]; + } + function binsInCommand(cmd: string): string[] { + const out = new Set(); + const pty = cmd.startsWith('PTY:'); + for (const m of cmd.matchAll(/gstack-[a-z0-9-]*[a-z0-9](?:\.(?:sh|ts))?/g)) { + let tok = m[0]!; + if (pty) { while (tok.length > 7 && !binByName.has(tok)) { const k = tok.lastIndexOf('-'); tok = k > 6 ? tok.slice(0, k) : ''; } } + const i = m.index!; if (cmd[i - 1] === '/' && i >= 2 && /[\s"'\x60(]/.test(cmd[i - 2]!)) continue; + const b = binByName.get(tok); if (b) out.add(b); + } + for (const m of cmd.matchAll(/browse\/bin\/([a-z-]+)/g)) { const b = binByName.get(m[1]!); if (b) out.add(b); } + for (const bin of BINARIES) { + const hit = cmd.includes(bin.outfile) || (bin.outfile === 'browse/dist/browse' && /\$B\b|\$\{B\}|find-browse/.test(cmd)) || (bin.outfile === 'design/dist/design' && /\$D\b|\$\{D\}/.test(cmd)) || (bin.outfile === 'make-pdf/dist/pdf' && /make-pdf\/dist\/pdf/.test(cmd)); + if (hit) out.add(bin.outfile); + } + return [...out]; + } + const binaryTree = (outfile: string) => BINARIES.find(b => b.outfile === outfile)?.tree ?? []; + + const observed = observedMap?.cases ?? {}; + const hostIds = tracked.filter((f: string) => /^hosts\/[\w-]+\.ts$/.test(f) && f !== 'hosts/index.ts').map((f: string) => path.basename(f, '.ts')); + const byFile = new Map>(); + for (const [id, patterns] of Object.entries(maps.e2eTouchfiles)) { + const entries = caseEntries(patterns, null); + for (const b of observed[id]?.commands ?? []) { if (trackedSet.has(b)) entries.push(b); else entries.push(...binaryTree(b)); } + const files = closure(entries); + const ownerText = entries.filter((f: string) => /^test\/[^/]+\.test\.ts$/.test(f)).map(read).join('\n'); + for (const h of hostIds) if (new RegExp('[\'"\x60]' + h + '[\'"\x60]|--host[ =]' + h + '\\b').test(ownerText)) for (const f of closure(['hosts/' + h + '.ts', ...expand('hosts/' + h + '/**')])) files.add(f); + for (const f of files) { if (!byFile.has(f)) byFile.set(f, new Set()); byFile.get(f)!.add(id); } + } + void binsInCommand; void stripShell; void execRefs; + return byFile; +} diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index e76d37780..3cb515285 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -116,6 +116,7 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/eng-test-plan-edit-cli.js': ['test/eng-test-plan-edit-approval.test.ts'], 'test/fixtures/eng-test-plan-edit-dacc.json': ['test/eng-test-plan-edit-approval.test.ts'], 'test/fixtures/eval-history/wave-reds-8-censuses.json': ['test/eval-history.test.ts'], + 'test/fixtures/exercised-map-replay.json': ['scripts/exercised-map.ts'], 'test/fixtures/exit-propagation/**': ['test/exit-propagation.test.ts'], 'test/fixtures/free-shard-capture.ts': ['test/test-free-shards-capture.test.ts'], 'test/fixtures/golden/**': ['test/host-config.test.ts', 'test/ship-review-loop.test.ts'], diff --git a/test/test-health-report.test.ts b/test/test-health-report.test.ts index 38a2e31f4..0d654dba8 100644 --- a/test/test-health-report.test.ts +++ b/test/test-health-report.test.ts @@ -191,7 +191,8 @@ describe('collectHealth over recorded fixture data', () => { [/runs\/61\/artifacts/, () => ({ artifacts: [] })], [/workflows\/windows-free-tests\.yml\/runs/, () => ({ workflow_runs: windowsRuns })], ], { - 11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'full-fallback', e2e: ['gate-a', 'gate-b'], judges: ['judge-a'] } }) }, + 11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'full-fallback', e2e: ['gate-a', 'gate-b'], judges: ['judge-a'], + shadow: { e2e: ['gate-a', 'gate-b'], wouldDrop: [], wouldAdd: [], provenance: [], predictedSavingUsd: 0, unpricedCases: 0 } } }) }, 12: { 'trial-outcomes.jsonl': prTrials }, 51: { 'trial-outcomes.jsonl': census }, 61: { 'flake-ledger.jsonl': ledger('test/a.test.ts') }, @@ -207,6 +208,7 @@ describe('collectHealth over recorded fixture data', () => { expect(value('pr-spend')).toMatchObject({ value: '$2.50 recorded', detail: ['per merged PR: $2.50 (1 merged)', 'per validated revision: $2.50 (1 revisions)'] }); expect(value('pr-cases-per-run')).toMatchObject({ value: '2.0 gate cases, 1.0 judges per run (1 runs)', detail: ['full-fallback: 2.0 gate cases, 1.0 judges (1 runs)'] }); expect(value('pr-known-cost-per-run')).toMatchObject({ value: '$2.50 per run (1 runs; 0 executed trial(s) with unknown cost)', detail: ['full-fallback: $2.50 (1 runs)'] }); + expect(value('pr-shadow-map')).toMatchObject({ value: '$0.00 predicted saving/week (a lower bound: known cost excludes PTY trials with unknown cost); 0.0 activation miss(es)/week, 0 flaky drop(s) not counted (1 runs)', detail: [] }); expect(report.headline).toBe('Cost per merged PR: $2.50 (1 merged PRs, $2.50 recorded)'); expect(formatHealth(report).split('\n')[1]).toBe('Cost per merged PR: $2.50 (1 merged PRs, $2.50 recorded)'); expect(value('cancelled-slice-minutes').value).toBe('30 min/week (1 cancelled run(s))'); @@ -247,6 +249,38 @@ describe('collectHealth over recorded fixture data', () => { expect(report.headline).toBe('Cost per merged PR: unavailable (0 merged PRs)'); }); + test('the shadow map metric counts red drops as activation misses unless the case passed later at the same head SHA, and labels dollars a lower bound', () => { + const gate = (caseId: string, outcome: 'passed' | 'failed', extra: Partial = {}) => trialRecord(caseId, outcome, { tier: 'gate', lane: 'gate/pr', ...extra }); + const first = formatTrialOutcomes([gate('gate-a', 'failed', { cost_usd: 3 }), gate('gate-b', 'failed'), gate('gate-c', 'failed'), + gate('gate-d', 'failed', { execution: 'reused', cost_usd: 0 }), gate('gate-e', 'passed')]); + const rerun = formatTrialOutcomes([gate('gate-b', 'passed'), gate('gate-a', 'failed')]); + const shadow = (wouldDrop: string[], predictedSavingUsd: number, unknownCostShare = 0.391) => + ({ e2e: [], wouldDrop, wouldAdd: [], provenance: [], predictedSavingUsd, unpricedCases: 0, unknownCostShare }); + const client = fakeClient([ + [/workflows\/evals\.yml\/runs/, () => ({ workflow_runs: [ + run(1, { event: 'pull_request', head_branch: 'a', head_sha: 'same', created_at: day(3) }), + run(2, { event: 'pull_request', head_branch: 'a', head_sha: 'same', created_at: day(2) }), + run(3, { event: 'pull_request', head_branch: 'c' }), + run(4, { event: 'pull_request', head_branch: 'd' })] })], + [/runs\/1\/artifacts/, () => ({ artifacts: [{ id: 11, name: 'paid-plan', size_in_bytes: 10 }, { id: 12, name: 'trial-outcomes-pr-a1', size_in_bytes: 10 }] })], + [/runs\/2\/artifacts/, () => ({ artifacts: [{ id: 21, name: 'paid-plan', size_in_bytes: 10 }, { id: 22, name: 'trial-outcomes-pr-a1', size_in_bytes: 10 }] })], + [/runs\/3\/artifacts/, () => ({ artifacts: [{ id: 31, name: 'paid-plan', size_in_bytes: 10 }] })], + [/runs\/4\/artifacts/, () => ({ artifacts: [{ id: 41, name: 'paid-plan', size_in_bytes: 10 }] })], + [/pulls\?state=closed/, () => []], + ], { + 11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'dependents', e2e: [], judges: [], shadow: shadow(['gate-a', 'gate-b', 'gate-d', 'gate-e'], 4.5) } }) }, + 12: { 'trial-outcomes.jsonl': first }, + 21: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'pr', e2e: [], judges: [], shadow: shadow([], -1.5) } }) }, + 22: { 'trial-outcomes.jsonl': rerun }, + 31: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'pr', e2e: [], judges: [], shadow: { ...shadow([], 99), error: 'bad map' } } }) }, + 41: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'pr', e2e: [], judges: [] } }) }, + }); + const report = collectHealth({ client, repo: 'o/r', now: NOW, sinceDays: 7, cacheDir: tmp('health-cache-'), root: freeRoot(3, 0) }); + expect(report.metrics.find(entry => entry.id === 'pr-shadow-map')).toMatchObject({ + value: '$3.00 predicted saving/week (a lower bound: known cost excludes the 39% of observed case runs with unknown cost (PTY)); 1.0 activation miss(es)/week, 1 flaky drop(s) not counted (2 runs)', + detail: ['shadow miss: 1:gate-a', 'flaky drop: 1:gate-b (passed in run 2)'] }); + }); + test('the Windows wall counts only successful pull_request runs and says so when there are none', () => { const wall = (windowsRuns: unknown[]) => collectHealth({ client: fixture(windowsRuns), repo: 'o/r', now: NOW, sinceDays: 7, cacheDir: tmp('health-cache-'), root: freeRoot(3, 0) }) .metrics.find(entry => entry.id === 'free-wall')!;