diff --git a/CHANGELOG.md b/CHANGELOG.md index 2db7e8018..0ecfedfec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## [1.91.129.0] - 2026-10-18 + +**PR paid-lane selection reads every script it depends on, and fails closed when it can't.** Selection derives which paid cases a changed file reaches by scanning imports. It skipped extensionless `bun`/`node` scripts, read 15 `.ts` files with the TSX loader, and treated a scan error as "no consumers", which under-selects. Across all 2,223 code files in the repo, scans now finish with 0 errors, and no (file, case) pair that main selected is lost. + +### Fixed +- **Shebang scripts and `.ts` files are scanned correctly.** A leading `#!` line is stripped before scanning, extensionless `bun`/`node` scripts count as code, and the loader is chosen by file extension. 41 files that reached no paid case now reach the cases that use them. +- **A failed import scan restores the full gate** with the label `import scan failed` and the file to fix, instead of silently selecting nothing. +- **Repository config no longer looks like an unknown dependency.** `.gstack/ship-policy.example.json`, `.github/dependabot.yml`, `.osv-scanner.toml`, `setup.ps1`, `.gitlab-ci.yml` and `.github/ISSUE_TEMPLATE/**` are audited free-only files; `capy/` paths are derived like other source; a full-gate file is labelled `full-gate input`. + +### Added +- **`bun run test:health` reports cost per merged PR first**, plus gate cases per PR run and known cost per PR run, split by selection mode. + +### For contributors +- The selection replay (`scripts/replay-pr-selection.ts`) honours the same free-only exemptions as the selector when it counts misses. +- docs/TESTING_INTERNALS.md#pr-paid-lane-fallback describes the fallback classes as they now behave, and records why the lockfile stays a full-gate input. + ## [1.91.128.0] - 2026-10-18 **The two free-suite duration seeds now refresh themselves weekly.** The shard packers read a Linux and a Windows duration seed, and both went stale as test files came and went. They were refreshed by hand twice this week. A new weekly workflow records both seeds and opens one seed-refresh PR when either seed's file set changes. That PR goes through the normal review. diff --git a/VERSION b/VERSION index d6ddc385a..de898c0c6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.128.0 +1.91.129.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 2e080b314..fbec08ef6 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.128.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.129.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index 2befad2fe..7b7e7956a 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -1009,17 +1009,44 @@ select nothing: docs, any non-paid `*.test.ts(x)`, `FREE_ONLY_PR_FILES` (the free and paid duration seeds, `scripts/ubicloud/**`, every workflow that runs no paid evals, and free-only tools), and the fixtures in `FREE_FIXTURES` (`test/helpers/free-fixtures-data.ts`, each fixture with its free consumers). -`PAID_WORKFLOW_FILES` (`evals.yml`, `evals-periodic.yml`, `evals-marathon.yml`) -and any other unknown file restore the full gate (`prCoverage.mode = -full-fallback`). The job summary and PR comment list each file that caused a -fallback with one of three labels: +Unmapped files under `DERIVABLE_PREFIXES` (bin/, lib/, scripts/, browse/, +design/, make-pdf/, hosts/, extension/, model-overlays/, agents/, test/helpers/, +capy/) are not unknown: `scripts/pr-dependencies.ts` derives the cases whose +reference closure reaches them (`prCoverage.mode = dependents`), or reports that +no paid case reaches them. The derivation scans the imports of every code file, +including `#!`-shebang `.ts` files and extensionless Bun or Node scripts; if any +import scan fails, every unmapped file restores the full gate (label **import +scan failed**). `PR_FULL_GATE_FILES`, `PAID_WORKFLOW_FILES` (`evals.yml`, +`evals-periodic.yml`, `evals-marathon.yml`) and any other unknown file restore +the full gate (`prCoverage.mode = full-fallback`). The job summary and PR comment +list each unmapped file that caused a fallback with one of these labels: - **needs touchfile entry**: a paid test, helper or fixture no case maps; add it to the owning case in `E2E_TOUCHFILES` (`test/helpers/touchfiles-data.ts`). - **add to FREE_ONLY_PR_FILES**: a file only free lanes read; add it to the list in `scripts/test-pr-profile.ts` (or a fixture to `FREE_FIXTURES`). -- **real unknown dependency**: shared code (bin/, lib/, resolvers, hosts, setup) - whose dependents are not mapped; the full gate is the correct result. +- **full-gate input**: a `PR_FULL_GATE_FILES` entry every case consumes; the + full gate is intended and no mapping narrows it. +- **deleted but still referenced** and **import scan failed**: fix the named + reference or parse error. +- **real unknown dependency**: a file outside the derivable directories that no + list classifies (for example `ETHOS.md`); map it, exempt it, or derive its + directory. + +The fallback rate is a label, not the bill: a dependents run whose derived +closure reaches every gate case costs about what a full-fallback run does. +`bun run test:health` leads with the cost per merged PR and reports +`pr-cases-per-run` (mean selected gate cases and judges per PR run, per mode) +and `pr-known-cost-per-run` (mean known trial cost per PR run, per mode) beside +`pr-fallback-rate`. + +`package.json` (beyond a version-only change) and `bun.lock` stay full-gate +inputs. Over the 225 PR runs from 2026-10-07 to 2026-10-17, no run fell back on +them alone, dependabot PRs never reach the paid lane (`evals.yml` skips +`github.actor == 'dependabot[bot]'`), and every runtime package's importer +(`test/helpers/agent-sdk-runner.ts`, `test/helpers/llm-judge.ts`, `browse/src/*`, +`make-pdf/src/*`) reaches every gate case or is in the global set, so narrowing +them would select the same cases. `test/free-fixtures.test.ts` holds the inventory: each `test/fixtures/**` path belongs to a paid touchfile or `FREE_FIXTURES`, and each workflow to the paid or diff --git a/package.json b/package.json index 5bc904faf..5bb38fc1d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.128", + "version": "1.91.129", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/harness-version.json b/scripts/harness-version.json index ee40de983..f37b3d4c3 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -16,7 +16,7 @@ "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":5}, "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":5}, "scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":5}, - "scripts/test-pr-profile.ts": {"blob":"67d3e163dd443bfc5e381586e28f7e83ed59125a","since":5,"nonBehavioral":"Classifies the new free-durations-record.yml workflow as free-lane only; no paid case reads it, so no verdict can change"}, + "scripts/test-pr-profile.ts": {"blob":"64cd7acfc0f2a4f53472bbdc8a50216984230c5a","since":5,"nonBehavioral":"PR selection only (A1 merged with #3253's free-only entry): shebang and per-extension import scanning, fail-closed scan errors, free-only repository-config files, capy/ derivable, the full-gate fallback label; it changes which cases a PR runs, never how any case is judged"}, "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":5}, "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":5}, "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":5}, diff --git a/scripts/pr-dependencies.ts b/scripts/pr-dependencies.ts index fbf1ef43d..a6dc94e3a 100644 --- a/scripts/pr-dependencies.ts +++ b/scripts/pr-dependencies.ts @@ -49,6 +49,12 @@ export interface DerivedDependencies { tracked: Set; /** The reference chain from an E2E case's declared inputs to `file` (for fix text and audits), else null. */ trace(caseId: string, file: string): string[] | null; + /** + * Files whose import scan failed for a reason other than a leading `#!` + * line (their imports are unknown). PR selection restores the full gate + * while this is non-empty; it fills as the derivation reads files. + */ + scanErrors: Map; /** * Tracked files whose reference text still names `missing`, a path absent * from the tree (deleted in the diff): its path or extensionless import @@ -60,8 +66,10 @@ export interface DerivedDependencies { } const CODE = /\.(?:[cm]?[jt]s|tsx|jsx)$/; +/** An extensionless script Bun or Node runs: its imports are code, not shell. */ +const CODE_SHEBANG = /^#![^\n]*\b(?:bun|node)\b/; const TEXT = /\.(?:[cm]?[jt]s|tsx|jsx|sh|md|tmpl|json|ya?ml|toml|txt|py)$/; -const REPO_PATH = /(?:^|[^\w.@-])((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)/g; +const REPO_PATH = /(?:^|[^\w.@-])((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|capy|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)/g; /** `path.join(ROOT, 'lib', 'x.ts')`: a top-level directory and its entry as separate string literals. */ const JOINED_PATH = /['"`](bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents)['"`]\s*,\s*['"`]([\w.@-]+)['"`]/g; const BIN_NAME = /\bgstack-[a-z0-9][a-z0-9-]*[a-z0-9]\b/g; @@ -85,7 +93,26 @@ export const SELECTION_DATA_MODULES: readonly string[] = [ ]; const DEFAULT_JUDGE_OWNER = 'test/skill-llm-eval.test.ts'; -const transpiler = new Bun.Transpiler({ loader: 'tsx' }); +const transpilers = { + ts: new Bun.Transpiler({ loader: 'ts' }), tsx: new Bun.Transpiler({ loader: 'tsx' }), + js: new Bun.Transpiler({ loader: 'js' }), jsx: new Bun.Transpiler({ loader: 'jsx' }), +}; + +/** Code by extension, or an extensionless file whose shebang runs Bun or Node. */ +export function isCodeFile(file: string, text: string): boolean { + return CODE.test(file) || (!/\.[^/]+$/.test(path.posix.basename(file)) && CODE_SHEBANG.test(text)); +} + +/** + * Relative import specifiers of a code file. A leading `#!` line is blanked + * first (`scanImports` rejects it), and each extension gets its own loader + * (the TSX loader misreads `.ts` generics and casts). Any other parse error + * throws, so callers fail closed. + */ +export function scanCodeImports(file: string, text: string): string[] { + const loader = /\.tsx$/.test(file) ? 'tsx' : /\.jsx$/.test(file) ? 'jsx' : /\.[cm]?js$/.test(file) ? 'js' : 'ts'; + return transpilers[loader].scanImports(text.replace(/^#![^\n]*/, '')).map(item => item.path); +} /** * Text a reference scan reads: whole-line comments are prose about other @@ -168,15 +195,17 @@ export function deriveDependencies(maps: DependencyMaps, root: string): DerivedD }; const dirs = new Set(tracked.flatMap(file => file.split('/').slice(0, -1).map((_, i, parts) => parts.slice(0, i + 1).join('/')))); + const scanErrors = new Map(); + const importsOf = (file: string): string[] => { + try { return scanCodeImports(file, read(file)); } catch (error) { scanErrors.set(file, String(error)); return []; } + }; const edges = new Map(); const refs = (file: string): string[] => { if (edges.has(file)) return edges.get(file)!; const raw = read(file); const text = (SELECTION_DATA_MODULES as readonly string[]).includes(file) ? '' : referenceText(file, raw); const out = new Set(); - if (CODE.test(file)) { - try { for (const item of transpiler.scanImports(raw)) { const hit = resolveSpecifier(root, item.path, file); if (hit) out.add(hit); } } catch { /* literal scan below still applies */ } - } + if (isCodeFile(file, raw)) for (const specifier of importsOf(file)) { const hit = resolveSpecifier(root, specifier, file); if (hit) out.add(hit); } for (const match of text.matchAll(REPO_PATH)) { const ref = match[1]!.replace(/[.]+$/, ''); if (trackedSet.has(ref)) out.add(ref); @@ -238,19 +267,17 @@ export function deriveDependencies(maps: DependencyMaps, root: string): DerivedD const file = pending.pop()!; if (global.has(file)) continue; global.add(file); - if (!CODE.test(file)) continue; - try { for (const item of transpiler.scanImports(read(file))) { const hit = resolveSpecifier(root, item.path, file); if (hit && !global.has(hit)) pending.push(hit); } } catch { /* unparseable: its own entry stays global */ } + if (!isCodeFile(file, read(file))) continue; + for (const specifier of importsOf(file)) { const hit = resolveSpecifier(root, specifier, file); if (hit && !global.has(hit)) pending.push(hit); } } const importTargets = new Map(); const targetsOf = (file: string): string[] => { if (!importTargets.has(file)) { const targets: string[] = []; - if (CODE.test(file) && !(SELECTION_DATA_MODULES as readonly string[]).includes(file)) { - try { - for (const item of transpiler.scanImports(read(file))) { - if (item.path.startsWith('.')) targets.push(path.posix.normalize(path.posix.join(path.posix.dirname(file), item.path))); - } - } catch { /* literal mentions below still apply */ } + if (isCodeFile(file, read(file)) && !(SELECTION_DATA_MODULES as readonly string[]).includes(file)) { + for (const specifier of importsOf(file)) { + if (specifier.startsWith('.')) targets.push(path.posix.normalize(path.posix.join(path.posix.dirname(file), specifier))); + } } importTargets.set(file, targets); } @@ -287,7 +314,7 @@ export function deriveDependencies(maps: DependencyMaps, root: string): DerivedD while (parents.has(chain[0]!)) chain.unshift(parents.get(chain[0]!)!); return chain; }; - return { e2e, judges: consumers(maps.judgeTouchfiles, DEFAULT_JUDGE_OWNER), everyCase, global, tracked: trackedSet, trace, referencers }; + return { e2e, judges: consumers(maps.judgeTouchfiles, DEFAULT_JUDGE_OWNER), everyCase, global, tracked: trackedSet, trace, referencers, scanErrors }; } const cache = new Map(); diff --git a/scripts/replay-pr-selection.ts b/scripts/replay-pr-selection.ts index 71e79d5de..3e6ed6903 100644 --- a/scripts/replay-pr-selection.ts +++ b/scripts/replay-pr-selection.ts @@ -26,7 +26,7 @@ import { packageChangeOnlyVersion } from './test-pr-profile'; import { E2E_TIERS, E2E_TOUCHFILES, GLOBAL_TOUCHFILES } from '../test/helpers/touchfiles-data'; import { matchGlob } from '../test/helpers/test-selection'; import { derivedDependencies } from './pr-dependencies'; -import { PR_PROFILE_MAPS } from './test-pr-profile'; +import { knownNonBehaviorFile, PR_PROFILE_MAPS } from './test-pr-profile'; export interface ReplayInput { runId: number; @@ -59,7 +59,9 @@ export interface ReplayRow { /** * Whether a case depends on any changed file under today's dependency graph: * its touchfiles, the globals, and the derived reference closure - * (scripts/pr-dependencies.ts, its global runner closure included). + * (scripts/pr-dependencies.ts, its global runner closure included). Like + * the selector, a touchfile match wins over the audited free-only exemptions, + * and an exempt file reaches no case through the derived closure. */ export function caseTouchesDiff(id: string, changedFiles: readonly string[]): boolean { const patterns = [...(E2E_TOUCHFILES[id] ?? []), ...GLOBAL_TOUCHFILES]; @@ -67,7 +69,7 @@ export function caseTouchesDiff(id: string, changedFiles: readonly string[]): bo // A path absent from today's tree reaches a case through the live files that still reference it. const via = (file: string) => derived.tracked.has(file) ? [file] : derived.referencers(file); return changedFiles.some(file => patterns.some(pattern => matchGlob(file, pattern)) - || via(file).some(source => derived.global.has(source) || !!derived.e2e.get(source)?.has(id))); + || (!knownNonBehaviorFile(file) && via(file).some(source => derived.global.has(source) || !!derived.e2e.get(source)?.has(id)))); } export function replayRun(input: ReplayInput, oldSelection: readonly string[]): ReplayRow { diff --git a/scripts/test-health-report.ts b/scripts/test-health-report.ts index e4dc5da46..cf11e4f3e 100644 --- a/scripts/test-health-report.ts +++ b/scripts/test-health-report.ts @@ -33,7 +33,7 @@ export const COMPLEXITY_PATHS = ['scripts', 'test/helpers', '.github/workflows'] export interface Metric { id: string; title: string; rule: string; value?: string; detail?: string[]; unavailable?: string } export interface Check { id: string; title: string; status: 'pass' | 'fail' | 'unavailable'; detail: string; fix: string } export type IssueAction = 'upsert' | 'keep-open' | 'close'; -export interface HealthReport { repo: string; sinceDays: number; generatedAt: string; metrics: Metric[]; checks: Check[]; issueAction: IssueAction } +export interface HealthReport { repo: string; sinceDays: number; generatedAt: string; headline?: string; metrics: Metric[]; checks: Check[]; issueAction: IssueAction } export interface HealthInputs { client: GhClient; @@ -188,14 +188,17 @@ export function collectHealth(inputs: HealthInputs): HealthReport { const need = (value: T | Error): T => { if (value instanceof Error) throw value; return value; }; const prRecords = attempt(() => need(prRuns).filter(run => run.conclusion !== 'cancelled') .map(run => ({ run, trials: records(artifactDirs(run, name => name.startsWith('trial-outcomes'))) }))); + const prManifests = attempt(() => need(prRuns).flatMap(run => { + const file = artifactDirs(run, name => name === 'paid-plan').map(dir => path.join(dir, 'manifest.json')).find(candidate => fs.existsSync(candidate)); + return file ? [{ run, manifest: readJson(file) }] : []; + })); const metrics: Metric[] = []; + let costPerMergedPr: string | undefined; - metrics.push(metric('pr-fallback-rate', 'PR full-fallback rate', 'share of evals.yml pull_request runs whose paid-plan manifest has prCoverage.mode = full-fallback', () => { + metrics.push(metric('pr-fallback-rate', 'PR full-fallback rate', 'share of evals.yml pull_request runs whose paid-plan manifest has prCoverage.mode = full-fallback (a label, not spend: dependents runs select about as much)', () => { const modes = new Map(); - for (const run of need(prRuns)) { - const manifest = artifactDirs(run, name => name === 'paid-plan').flatMap(dir => [path.join(dir, 'manifest.json')]).find(file => fs.existsSync(file)); - if (!manifest) continue; - const mode = String(readJson(manifest).prCoverage?.mode ?? 'none'); + for (const { manifest } of need(prManifests)) { + const mode = String(manifest.prCoverage?.mode ?? 'none'); modes.set(mode, (modes.get(mode) ?? 0) + 1); } const total = [...modes.values()].reduce((a, b) => a + b, 0); @@ -204,6 +207,27 @@ export function collectHealth(inputs: HealthInputs): HealthReport { detail: [...modes].map(([mode, count]) => `${mode}: ${count}`) }; })); + metrics.push(metric('pr-cases-per-run', 'Selected gate cases and judges per PR run', 'mean length of prCoverage.e2e (gate cases) and prCoverage.judges in the paid-plan manifests of evals.yml pull_request runs, overall and per mode', () => { + const planned = need(prManifests).filter(({ manifest }) => Array.isArray(manifest.prCoverage?.e2e) && Array.isArray(manifest.prCoverage?.judges)); + if (planned.length === 0) throw new Error('no paid-plan manifest with prCoverage.e2e and prCoverage.judges in the window'); + const mean = (entries: typeof planned) => `${(entries.reduce((sum, { manifest }) => sum + manifest.prCoverage.e2e.length, 0) / entries.length).toFixed(1)} gate cases, ${(entries.reduce((sum, { manifest }) => sum + manifest.prCoverage.judges.length, 0) / entries.length).toFixed(1)} judges`; + const modes = [...new Set(planned.map(({ manifest }) => String(manifest.prCoverage.mode)))].sort(); + return { value: `${mean(planned)} per run (${planned.length} runs)`, + detail: modes.map(mode => { const entries = planned.filter(({ manifest }) => String(manifest.prCoverage.mode) === mode); return `${mode}: ${mean(entries)} (${entries.length} runs)`; }) }; + })); + + metrics.push(metric('pr-known-cost-per-run', 'Known cost per PR run', 'mean over non-cancelled evals.yml pull_request runs of executed trial cost_usd with a known cost (a run with no trial records counts $0), overall and per manifest mode; unknown-cost trials (PTY) are counted, not priced', () => { + const entries = need(prRecords); + if (entries.length === 0) throw new Error('no non-cancelled PR runs in the window'); + const known = (trials: TrialRecord[]) => trials.filter(record => record.execution === 'executed' && record.cost_known !== false).reduce((sum, record) => sum + record.cost_usd, 0); + const unknown = entries.flatMap(entry => entry.trials).filter(record => record.execution === 'executed' && record.cost_known === false).length; + const modeOf = new Map((prManifests instanceof Error ? [] : prManifests).map(({ run, manifest }) => [run.id, String(manifest.prCoverage?.mode ?? 'none')])); + const mean = (list: typeof entries) => `$${(list.reduce((sum, entry) => sum + known(entry.trials), 0) / list.length).toFixed(2)}`; + const modes = [...new Set(entries.map(entry => modeOf.get(entry.run.id) ?? 'no manifest'))].sort(); + return { value: `${mean(entries)} per run (${entries.length} runs; ${unknown} executed trial(s) with unknown cost)`, + detail: modes.map(mode => { const list = entries.filter(entry => (modeOf.get(entry.run.id) ?? 'no manifest') === mode); return `${mode}: ${mean(list)} (${list.length} runs)`; }) }; + })); + metrics.push(metric('pr-reused-trials', 'Reused PR trial records', 'trial-outcomes records with execution = reused across non-cancelled PR runs', () => { const all = need(prRecords).flatMap(entry => entry.trials); const reused = all.filter(record => record.execution === 'reused').length; @@ -216,6 +240,7 @@ export function collectHealth(inputs: HealthInputs): HealthReport { .reduce((sum, record) => sum + record.cost_usd, 0); const revisions = new Set(entries.filter(entry => entry.run.conclusion === 'success' || entry.run.conclusion === 'failure').map(entry => entry.run.sha)).size; const merged = listMergedPulls(client, repo, since); + if (merged) costPerMergedPr = `$${(spend / merged).toFixed(2)} (${merged} merged PRs, $${spend.toFixed(2)} recorded)`; return { value: `$${spend.toFixed(2)} recorded`, detail: [ `per merged PR: ${merged ? `$${(spend / merged).toFixed(2)} (${merged} merged)` : 'n/a (0 merged)'}`, `per validated revision: ${revisions ? `$${(spend / revisions).toFixed(2)} (${revisions} revisions)` : 'n/a'}`] }; @@ -267,7 +292,8 @@ export function collectHealth(inputs: HealthInputs): HealthReport { })); const checks = [flakeCheck(flakes), unseededCheck(root)]; - return { repo, sinceDays, generatedAt: new Date(now).toISOString(), metrics, checks, issueAction: issueActionFor(checks) }; + const headline = `Cost per merged PR: ${costPerMergedPr ?? `unavailable (${metrics.find(entry => entry.id === 'pr-spend')?.unavailable ?? '0 merged PRs'})`}`; + return { repo, sinceDays, generatedAt: new Date(now).toISOString(), headline, metrics, checks, issueAction: issueActionFor(checks) }; } function listMergedPulls(client: GhClient, repo: string, since: string): number { @@ -340,7 +366,7 @@ function censusMetrics(inputs: HealthInputs, artifactDirs: (run: CiRun, match: ( // --- Rendering --- export function formatHealth(report: HealthReport): string { - const lines = [`test:health ${report.repo}, last ${report.sinceDays} day(s), ${report.generatedAt}`]; + const lines = [`test:health ${report.repo}, last ${report.sinceDays} day(s), ${report.generatedAt}`, ...(report.headline ? [report.headline] : [])]; for (const entry of report.metrics) { lines.push(`- ${entry.title}: ${entry.unavailable ? `unavailable: ${entry.unavailable}` : entry.value}`); lines.push(` rule: ${entry.rule}`); diff --git a/scripts/test-pr-profile.ts b/scripts/test-pr-profile.ts index 67d3e163d..64cd7acfc 100644 --- a/scripts/test-pr-profile.ts +++ b/scripts/test-pr-profile.ts @@ -216,6 +216,12 @@ export const FREE_ONLY_PR_FILES = [ '.gstack/generated.json', // gstack's own regen registry: gstack-regen/gstack-pregate read it from origin/ of the repo they run in; paid cases run in fixture repos. '.gstack/pregate.json', // gstack's own pre-gate policy: same trust boundary, same reason. '.gstack/ship-policy.json', // gstack's own ship policy: gstack-ship-policy/gstack-restamp read it from origin/ of the repo they run in; same reason. + '.gstack/ship-policy.example.json', // The example beside it: only .gitignore and docs/ship-policy.md name it. + '.github/dependabot.yml', // Dependabot's own config: read by GitHub, and by free test/compare-pr-version.test.ts; dependabot runs skip the paid lane. + '.osv-scanner.toml', // Read by .github/workflows/osv-scanner.yml and the free OSV tests (test/osv-*.test.ts). + 'setup.ps1', // The Windows installer: run only by .github/workflows/windows-setup-e2e.yml; the paid lane is Linux. + '.gitlab-ci.yml', // gstack's own GitLab config: skills probe the fixture repo's .gitlab-ci.yml, never this one. + '.github/ISSUE_TEMPLATE/**', // Issue templates: read by GitHub and free test/gstack-doctor.test.ts. 'scripts/retired-command.ts', // One-release stubs for retired package scripts; no paid case imports it. 'careful/bin/check-careful.sh', // /careful PreToolUse hook: covered by test/hook-scripts.test.ts; no paid fixture copies or runs it. // Reporting and launch tools: they read CI history or start a lane, never run inside a paid case. @@ -274,9 +280,9 @@ export const PR_FULL_GATE_FILES = [ export const PERIODIC_ONLY_PR_FILES = ['.github/workflows/evals-periodic.yml', '.github/workflows/evals-marathon.yml'] as const; /** Directories the reference derivation covers: an unmapped file there with no consumer is consumed by no paid case. */ -export const DERIVABLE_PREFIXES = ['bin/', 'lib/', 'scripts/', 'browse/', 'design/', 'make-pdf/', 'hosts/', 'extension/', 'model-overlays/', 'agents/', 'test/helpers/'] as const; +export const DERIVABLE_PREFIXES = ['bin/', 'lib/', 'scripts/', 'browse/', 'design/', 'make-pdf/', 'hosts/', 'extension/', 'model-overlays/', 'agents/', 'test/helpers/', 'capy/'] as const; -function knownNonBehaviorFile(file: string): boolean { +export function knownNonBehaviorFile(file: string): boolean { // A mapped dependency still wins over these exemptions. New helper, // runtime, dependency, or paid-workflow files are deliberately not exempted. return /^(?:docs\/|(?:README|CONTRIBUTING|ARCHITECTURE|CHANGELOG|TODOS)\.md$|VERSION$)/.test(file) @@ -294,6 +300,9 @@ export const FALLBACK_FIX_ANCHOR = 'docs/TESTING_INTERNALS.md#pr-paid-lane-fallb /** Why one changed file restored the full gate, and the edit that narrows it next time (DX-6). */ export function unknownFileLabel(file: string): { label: string; fix: string } { + if (matches(file, PR_FULL_GATE_FILES)) { + return { label: 'full-gate input', fix: `${file} is in PR_FULL_GATE_FILES (scripts/test-pr-profile.ts): every paid case consumes it, so the full gate is intended and no mapping narrows it` }; + } if (isPaidTestFile(file) || file.startsWith('test/helpers/')) { return { label: 'needs touchfile entry', fix: `register ${file} under the cases that consume it in test/helpers/touchfiles-data.ts` }; } @@ -381,6 +390,12 @@ export function selectPrProfile(options: { if (found.e2e.size + found.judges.size > 0) consumers.set(file, found); else noConsumerFiles.push(file); } + // A failed import scan leaves some closure unknown: every unmapped file restores the full gate (fail closed). + const scanFailed = derived && unmapped.length ? [...derived.scanErrors.keys()].sort() : []; + if (scanFailed.length) { + consumers.clear(); derivedGlobal.length = 0; noConsumerFiles.length = 0; + for (const file of unmapped) deletedLabels.set(file, { label: 'import scan failed', fix: `scripts/pr-dependencies.ts could not scan the imports of ${scanFailed.slice(0, 3).join(', ')}${scanFailed.length > 3 ? ` and ${scanFailed.length - 3} more` : ''}; fix the parse error so ${file}'s consumers can be derived` }); + } const derivedFiles = [...consumers.keys()]; const unknownFiles = unmapped.filter(file => !derivedGlobal.includes(file) && !consumers.has(file) && !noConsumerFiles.includes(file)); const sharedInputs = files.filter(file => depends(file, PR_FULL_GATE_FILES)); diff --git a/test/free-fixtures.test.ts b/test/free-fixtures.test.ts index c9cb1d933..4feeb8c37 100644 --- a/test/free-fixtures.test.ts +++ b/test/free-fixtures.test.ts @@ -85,7 +85,7 @@ describe('workflow inventory (CEO-20)', () => { test('the inventory rule also reads .yaml files', () => { expect(unknownFileLabel('.github/workflows/new-check.yaml').label).toBe('add to FREE_ONLY_PR_FILES'); - expect(unknownFileLabel('.github/workflows/evals.yml').label).toBe('real unknown dependency'); + expect(unknownFileLabel('.github/workflows/evals.yml').label).toBe('full-gate input'); }); }); @@ -111,7 +111,7 @@ describe('PR selection over the W1a single-file fallbacks', () => { const result = select(['.github/workflows/evals.yml']); expect(result.mode).toBe('full-fallback'); expect(result.unknownFileLabels).toEqual([{ file: '.github/workflows/evals.yml', ...unknownFileLabel('.github/workflows/evals.yml') }]); - expect(result.reasons[0]).toContain('.github/workflows/evals.yml (real unknown dependency)'); + expect(result.reasons[0]).toContain('.github/workflows/evals.yml (full-gate input)'); }); test('a mapped free fixture is exempt; a paid-registered fixture selects its cases; an unmapped one falls back', () => { diff --git a/test/paid-free-boundary.test.ts b/test/paid-free-boundary.test.ts index 402ac6917..dd4ea3eb9 100644 --- a/test/paid-free-boundary.test.ts +++ b/test/paid-free-boundary.test.ts @@ -114,7 +114,7 @@ describe('paid/free dependency boundary', () => { expect(result.selection).toEqual({ e2e: [], judges: [] }); } // Tracked files outside every mapping and outside the derivable directories restore the full gate. - for (const file of ['ETHOS.md', 'SECURITY.md', 'conductor.json', '.osv-scanner.toml']) { + for (const file of ['ETHOS.md', 'SECURITY.md', 'conductor.json', 'NOTICE.md']) { const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: [FREE_ONLY_PR_FILES[0], file] }); expect(result.coverage?.mode, file).toBe('full-fallback'); expect(result.coverage?.unknownFiles).toContain(file); diff --git a/test/pr-dependencies.test.ts b/test/pr-dependencies.test.ts index 2470405e9..a3ddcf73d 100644 --- a/test/pr-dependencies.test.ts +++ b/test/pr-dependencies.test.ts @@ -96,6 +96,93 @@ describe('reference derivation on a fixture tree', () => { }); }); +describe('shebang files and import scan failures (A1-1)', () => { + const tree = (extra: Record = {}) => fixtureRepo({ + 'test/skill-e2e-alpha.test.ts': "import '../lib/entry';\n", 'test/skill-e2e-beta.test.ts': '', + 'lib/entry.ts': '#!/usr/bin/env bun\nimport { y } from \'./typed\';\nexport const x = y;\n', + 'lib/typed.ts': 'export const y = (v: T) => v as unknown as T;\nimport \'./typed-dep\';\n', 'lib/typed-dep.ts': '', + 'alpha/SKILL.md.tmpl': 'Run `gstack-sb`.\n', 'bin/gstack-sb': '#!/usr/bin/env -S bun run\nimport { z } from \'../lib/sb-dep\';\n', 'lib/sb-dep.ts': '', + 'bin/gstack-sh': '#!/bin/sh\nimport ../lib/sh-only.ts\n', 'lib/sh-only.ts': '', + 'scripts/build.sh': '', 'scripts/resolvers/index.ts': '', 'package.json': '{}', ...extra, + }); + const maps = { e2eTouchfiles: { alpha: ['test/skill-e2e-alpha.test.ts', 'alpha/**'], beta: ['test/skill-e2e-beta.test.ts'] }, judgeTouchfiles: {}, globalTouchfiles: [] as string[] }; + const profileMaps = { ...maps, tiers: { alpha: 'gate', beta: 'gate' } as const }; + const pick = (derived: ReturnType, changedFiles: string[]) => + selectPrProfile({ maps: profileMaps, profile: [], selectedE2E: [], selectedJudges: [], changedFiles, derived }); + + test('a shebang .ts file, a bun-shebang extensionless bin and a .ts generic arrow are all import-scanned', () => { + const derived = deriveDependencies(maps, tree()); + expect([...derived.e2e.get('lib/typed.ts') ?? []]).toEqual(['alpha']); + expect([...derived.e2e.get('lib/typed-dep.ts') ?? []]).toEqual(['alpha']); + expect([...derived.e2e.get('lib/sb-dep.ts') ?? []]).toEqual(['alpha']); + expect(derived.e2e.get('lib/sh-only.ts')).toBeUndefined(); + expect(derived.scanErrors.size).toBe(0); + expect(pick(derived, ['lib/sb-dep.ts'])).toMatchObject({ mode: 'dependents', e2e: ['alpha'] }); + }); + + test('lib/fs-utils.ts selects the cases of bin/gstack-redact-prepush, which imports it', () => { + const bin = gate.filter(id => real.e2e.get('bin/gstack-redact-prepush')?.has(id)); + expect(bin.length).toBeGreaterThan(0); + const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: ['lib/fs-utils.ts'], packageVersionOnly: true }); + expect(result.coverage?.mode).toBe('dependents'); + for (const id of bin) expect(result.selection.e2e, id).toContain(id); + expect(real.scanErrors.size).toBe(0); + expect(real.global.has('test/helpers/bootstrap-retention.ts')).toBe(true); + }); + + test('any other scan error fails closed at every site: case closures, the global walk and deleted-file referencers', () => { + const broken = "import { from './nowhere';\n"; + const inClosure = deriveDependencies(maps, tree({ 'lib/typed-dep.ts': broken })); + expect([...inClosure.scanErrors.keys()]).toEqual(['lib/typed-dep.ts']); + const closed = pick(inClosure, ['lib/sb-dep.ts']); + expect(closed).toMatchObject({ mode: 'full-fallback', e2e: ['alpha', 'beta'], unknownFiles: ['lib/sb-dep.ts'] }); + expect(closed.unknownFileLabels[0]).toMatchObject({ label: 'import scan failed' }); + expect(closed.unknownFileLabels[0]!.fix).toContain('lib/typed-dep.ts'); + + const inGlobal = deriveDependencies({ ...maps, globalTouchfiles: ['lib/runner.ts'] }, tree({ 'lib/runner.ts': broken })); + expect([...inGlobal.scanErrors.keys()]).toEqual(['lib/runner.ts']); + expect(pick(inGlobal, ['lib/sb-dep.ts']).mode).toBe('full-fallback'); + + const unreached = deriveDependencies(maps, tree({ 'lib/orphan.ts': broken })); + expect(unreached.scanErrors.size).toBe(0); + expect(pick(unreached, ['lib/gone.ts'])).toMatchObject({ mode: 'full-fallback', unknownFiles: ['lib/gone.ts'] }); + expect([...unreached.scanErrors.keys()]).toEqual(['lib/orphan.ts']); + }); +}); + +describe('safe classification (A1-2)', () => { + test('capy/support.env derives to the bins that read it instead of restoring the full gate', () => { + for (const bin of ['bin/gstack-install-verdict.sh', 'bin/gstack-doctor-check.sh']) { + for (const id of real.e2e.get(bin) ?? []) expect(real.e2e.get('capy/support.env')?.has(id), `${bin} ${id}`).toBe(true); + } + const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: ['capy/support.env'], packageVersionOnly: true }); + expect(result.coverage?.unknownFiles).toEqual([]); + expect(result.coverage?.mode).toBe('dependents'); + expect(DERIVABLE_PREFIXES).toContain('capy/'); + }); + + test('repository config read only by free lanes or GitHub stays in the PR profile', () => { + const files = ['.github/dependabot.yml', '.osv-scanner.toml', 'setup.ps1', '.gitlab-ci.yml', '.github/ISSUE_TEMPLATE/bug_report.md', '.gstack/ship-policy.example.json']; + for (const file of files) expect(real.tracked.has(file), file).toBe(true); + const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: files, packageVersionOnly: true }); + expect(result.coverage?.mode).toBe('pr'); + expect(result.selection).toEqual({ e2e: [], judges: [] }); + }); + + test('a full-gate file is never labelled a real unknown dependency', () => { + for (const file of ['.github/workflows/evals.yml', 'test-setup.ts', '.gitattributes']) { + const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: [file], packageVersionOnly: true }); + expect(result.coverage?.mode, file).toBe('full-fallback'); + for (const label of result.coverage!.unknownFileLabels) { + expect(label.label, file).toBe('full-gate input'); + expect(label.fix).toContain('PR_FULL_GATE_FILES'); + } + expect(unknownFileLabel(file).label).toBe('full-gate input'); + } + expect(unknownFileLabel('ETHOS.md').label).toBe('real unknown dependency'); + }); +}); + describe('derived dependencies of this checkout', () => { test('a bin script a skill template invokes selects that skill\'s cases', () => { const consumers = real.e2e.get('bin/gstack-next-version') ?? new Set(); diff --git a/test/test-health-report.test.ts b/test/test-health-report.test.ts index 75970cef3..38a2e31f4 100644 --- a/test/test-health-report.test.ts +++ b/test/test-health-report.test.ts @@ -191,7 +191,7 @@ describe('collectHealth over recorded fixture data', () => { [/runs\/61\/artifacts/, () => ({ artifacts: [] })], [/workflows\/windows-free-tests\.yml\/runs/, () => ({ workflow_runs: windowsRuns })], ], { - 11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'full-fallback' } }) }, + 11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'full-fallback', e2e: ['gate-a', 'gate-b'], judges: ['judge-a'] } }) }, 12: { 'trial-outcomes.jsonl': prTrials }, 51: { 'trial-outcomes.jsonl': census }, 61: { 'flake-ledger.jsonl': ledger('test/a.test.ts') }, @@ -205,6 +205,10 @@ describe('collectHealth over recorded fixture data', () => { expect(value('net-complexity').unavailable).toContain('git grep HEAD failed'); expect(value('pr-reused-trials').value).toBe('1 of 2 trial records'); expect(value('pr-spend')).toMatchObject({ value: '$2.50 recorded', detail: ['per merged PR: $2.50 (1 merged)', 'per validated revision: $2.50 (1 revisions)'] }); + expect(value('pr-cases-per-run')).toMatchObject({ value: '2.0 gate cases, 1.0 judges per run (1 runs)', detail: ['full-fallback: 2.0 gate cases, 1.0 judges (1 runs)'] }); + expect(value('pr-known-cost-per-run')).toMatchObject({ value: '$2.50 per run (1 runs; 0 executed trial(s) with unknown cost)', detail: ['full-fallback: $2.50 (1 runs)'] }); + expect(report.headline).toBe('Cost per merged PR: $2.50 (1 merged PRs, $2.50 recorded)'); + expect(formatHealth(report).split('\n')[1]).toBe('Cost per merged PR: $2.50 (1 merged PRs, $2.50 recorded)'); expect(value('cancelled-slice-minutes').value).toBe('30 min/week (1 cancelled run(s))'); expect(value('push-to-verdict').value).toBe('p50 10.0 min, p95 10.0 min (1 runs)'); expect(value('census-periodic').value).toBe('1.00 expected failed trials/run (1/3 over 1 run(s)); P(red) 33.3% in force, 33.3% single-trial'); @@ -223,6 +227,26 @@ describe('collectHealth over recorded fixture data', () => { expect(formatHealth(report)).toContain('rule: share of evals.yml pull_request runs'); }); + test('known cost per PR run prices only known-cost executed trials and counts the unknown ones', () => { + const trials = formatTrialOutcomes([trialRecord('gate-a', 'passed', { tier: 'gate', lane: 'gate/pr', cost_usd: 4 }), + trialRecord('gate-pty', 'passed', { tier: 'gate', lane: 'gate/pr', cost_usd: 0, cost_known: false })]); + const client = fakeClient([ + [/workflows\/evals\.yml\/runs/, () => ({ workflow_runs: [run(1, { event: 'pull_request', head_branch: 'a' }), run(2, { event: 'pull_request', head_branch: 'b' })] })], + [/runs\/1\/artifacts/, () => ({ artifacts: [{ id: 11, name: 'paid-plan', size_in_bytes: 10 }, { id: 12, name: 'trial-outcomes-pr-a1', size_in_bytes: 10 }] })], + [/runs\/2\/artifacts/, () => ({ artifacts: [{ id: 21, name: 'paid-plan', size_in_bytes: 10 }] })], + [/pulls\?state=closed/, () => []], + ], { + 11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'dependents', e2e: ['gate-a', 'gate-pty', 'gate-c'], judges: ['j'] } }) }, + 12: { 'trial-outcomes.jsonl': trials }, + 21: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'pr', e2e: [], judges: [] } }) }, + }); + const report = collectHealth({ client, repo: 'o/r', now: NOW, sinceDays: 7, cacheDir: tmp('health-cache-'), root: freeRoot(3, 0) }); + const value = (id: string) => report.metrics.find(entry => entry.id === id)!; + expect(value('pr-known-cost-per-run')).toMatchObject({ value: '$2.00 per run (2 runs; 1 executed trial(s) with unknown cost)', detail: ['dependents: $4.00 (1 runs)', 'pr: $0.00 (1 runs)'] }); + expect(value('pr-cases-per-run')).toMatchObject({ value: '1.5 gate cases, 0.5 judges per run (2 runs)', detail: ['dependents: 3.0 gate cases, 1.0 judges (1 runs)', 'pr: 0.0 gate cases, 0.0 judges (1 runs)'] }); + expect(report.headline).toBe('Cost per merged PR: unavailable (0 merged PRs)'); + }); + test('the Windows wall counts only successful pull_request runs and says so when there are none', () => { const wall = (windowsRuns: unknown[]) => collectHealth({ client: fixture(windowsRuns), repo: 'o/r', now: NOW, sinceDays: 7, cacheDir: tmp('health-cache-'), root: freeRoot(3, 0) }) .metrics.find(entry => entry.id === 'free-wall')!;