diff --git a/changelog.d/capy-paid-screens-narrow-screens.md b/changelog.d/capy-paid-screens-narrow-screens.md new file mode 100644 index 000000000..33c0cfdd7 --- /dev/null +++ b/changelog.d/capy-paid-screens-narrow-screens.md @@ -0,0 +1 @@ +- **Narrow paid screens no longer run every gate case when a PR edits `setup`.** `setup` was listed as an input every paid case consumes, so any PR touching it (#4045, #4047) screened all 113 gate cases for about $37. No paid CI run executes `./setup`, and the touchfile map already names the cases that test it, so a narrow screen now treats it like any other file, and those two diffs select their 5 cases. `pull_request` runs, broad screens and `--profile full` keep the full gate for it. `modules/loadouts.json` is now placed through its reader, `scripts/modules.ts`, instead of restoring the full gate as an unknown file. That reader reaches every gate case, so a narrow screen leaves a loadouts-only edit to the train gate's full census. (GST-1575) diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index 38b4078de..76ff503ad 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -1200,7 +1200,9 @@ Unmapped files under `DERIVABLE_PREFIXES` (bin/, lib/, scripts/, browse/, design/, make-pdf/, hosts/, extension/, model-overlays/, agents/, test/helpers/, capy/) are not unknown: `scripts/pr-dependencies.ts` derives the cases whose reference closure reaches them (`prCoverage.mode = dependents`), or reports that -no paid case reaches them. The derivation scans the imports of every code file, +no paid case reaches them. References to `modules/` data count too, so +`modules/loadouts.json` takes the cases that reach its reader, +`scripts/modules.ts`. The derivation scans the imports of every code file, including `#!`-shebang `.ts` files and extensionless Bun or Node scripts; if any import scan fails, every unmapped file restores the full gate (label **import scan failed**). `PR_FULL_GATE_FILES`, `PAID_WORKFLOW_FILES` (`evals.yml`, @@ -1226,7 +1228,9 @@ derived file whose closure reaches every gate case; the plan lists it as left to the train gate's full census, and that census runs those cases. `pull_request` runs, broad screens, `EVALS_SCREEN_POLICY=broad` and `--profile full` keep every derived dependent. Full-gate inputs and unknown files -still restore the full gate on a narrow screen. The policy is in +still restore the full gate on a narrow screen, except `setup`: no paid run +executes it, so a narrow screen derives it like any other file +(`NARROW_DERIVED_FULL_GATE_FILES`); every other policy keeps its full gate. The policy is in [ship-policy.md](ship-policy.md#run-a-merge-train). The fallback rate is a label, not the bill: a dependents run whose derived diff --git a/docs/ship-policy.md b/docs/ship-policy.md index 2746ffbf0..fe029a4ed 100644 --- a/docs/ship-policy.md +++ b/docs/ship-policy.md @@ -765,7 +765,8 @@ narrower selects, along with its derived judges: the plan's reason names it, and the train gate's full census runs those cases. A `pull_request` run, the merge gate when the base has no `train` block, keeps the broad selection. Full-gate files, shared runtime and paid-runner inputs, and files the derivation cannot place still run the full gate on a -narrow screen. The nonce key carries the purpose and +narrow screen, except `setup`, which no paid run executes: a narrow screen +derives it like any other file. The nonce key carries the purpose and the policy, so a repeat follows the same run and a broad screen never adopts a narrow one. The READY block gains `screen: …`, and the receipt is written locally and to the notes ref. `--dry-run` prints the dispatch without sending diff --git a/scripts/harness-pins/scripts/test-pr-profile.ts.json b/scripts/harness-pins/scripts/test-pr-profile.ts.json index 9c3a13d46..e7f54e435 100644 --- a/scripts/harness-pins/scripts/test-pr-profile.ts.json +++ b/scripts/harness-pins/scripts/test-pr-profile.ts.json @@ -1 +1 @@ -{"blob":"900a5ec1eb151f99f93a07b3b8f3c4b2c047e4b1","since":6,"nonBehavioral":"scripts/test-pr-profile.ts only adds qm/** to FREE_ONLY_PR_FILES; no paid case reads qm/, so no case's selection or verdict can change (GST-1492)"} +{"blob":"812d84b94328409328020a842696e1a34b22b095","since":6,"nonBehavioral":"narrow screens derive setup and loadouts.json is derived; case selection only, no verdict can change (GST-1575)"} diff --git a/scripts/pr-dependencies.ts b/scripts/pr-dependencies.ts index ad17b40c2..d498f1c40 100644 --- a/scripts/pr-dependencies.ts +++ b/scripts/pr-dependencies.ts @@ -69,7 +69,7 @@ const CODE = /\.(?:[cm]?[jt]s|tsx|jsx)$/; /** An extensionless script Bun or Node runs: its imports are code, not shell. */ const CODE_SHEBANG = /^#![^\n]*\b(?:bun|node)\b/; const TEXT = /\.(?:[cm]?[jt]s|tsx|jsx|sh|md|tmpl|json|ya?ml|toml|txt|py)$/; -const REPO_PATH = /(?:^|[^\w.@-])((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|capy|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)/g; +const REPO_PATH = /(?:^|[^\w.@-])((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|capy|modules|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)/g; /** `path.join(ROOT, 'lib', 'x.ts')`: a top-level directory and its entry as separate string literals. */ const JOINED_PATH = /['"`](bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents)['"`]\s*,\s*['"`]([\w.@-]+)['"`]/g; const BIN_NAME = /\bgstack-[a-z0-9][a-z0-9-]*[a-z0-9]\b/g; diff --git a/scripts/test-pr-profile.ts b/scripts/test-pr-profile.ts index 900a5ec1e..812d84b94 100644 --- a/scripts/test-pr-profile.ts +++ b/scripts/test-pr-profile.ts @@ -328,6 +328,13 @@ export const PR_FULL_GATE_FILES = [ 'test/helpers/periodic-exclude-data.ts', ] as const; +/** + * Full-gate inputs a narrow screen derives like any other file. No paid run executes ./setup (evals.yml only + * builds) and scripts/pr-dependencies.ts maps it to the cases that invoke it; pull_request runs, broad screens + * and --profile full still restore the full gate for it, and the train gate's full census runs every case. + */ +export const NARROW_DERIVED_FULL_GATE_FILES = ['setup'] as const; + /** Periodic- and marathon-lane workflows: the PR gate never runs a case under them; their branch dispatch validates them. */ export const PERIODIC_ONLY_PR_FILES = ['.github/workflows/evals-periodic.yml', '.github/workflows/evals-marathon.yml'] as const; @@ -463,7 +470,9 @@ export function selectPrProfile(options: { ? [...consumers.keys()].filter(file => derived?.tracked.has(file) && gateIds.every(id => consumers.get(file)!.e2e.has(id))) : []; const derivedFiles = [...consumers.keys()].filter(file => !hubFiles.includes(file)); const unknownFiles = unmapped.filter(file => !derivedGlobal.includes(file) && !consumers.has(file) && !noConsumerFiles.includes(file)); - const sharedInputs = files.filter(file => depends(file, PR_FULL_GATE_FILES)); + const fullGateFiles = options.screenPolicy === 'narrow' + ? PR_FULL_GATE_FILES.filter(file => !(NARROW_DERIVED_FULL_GATE_FILES as readonly string[]).includes(file)) : PR_FULL_GATE_FILES; + const sharedInputs = files.filter(file => depends(file, fullGateFiles)); const fallback = unknownFiles.length > 0 || sharedInputs.length > 0; const dependentsMode = !fallback && derivedFiles.length > 0; const derivedE2E = derivedFiles.flatMap(file => [...consumers.get(file)!.e2e]); diff --git a/test/pr-shared-input-selection.test.ts b/test/pr-shared-input-selection.test.ts index 901928b3a..c01eff941 100644 --- a/test/pr-shared-input-selection.test.ts +++ b/test/pr-shared-input-selection.test.ts @@ -84,3 +84,80 @@ test('cumulative shared, native and prompt edits retain every gate case and judg expect(result.coverage?.deferred.map(({ id }) => id).sort()).toEqual(deferredIds); expect(result.coverage?.needsFullValidation).toBe(false); }); + +// GST-1575: no paid run executes ./setup (evals.yml only builds), so a narrow screen derives it like any +// other file. Diffs of #4045 (9e809ab) and #4047 (4743132), whose narrow screens (runs 42093259982 and +// 42103721891) fell back to all gate cases on setup alone. +const setupHookCases = ['gstack-upgrade-happy-path', 'ship-local-hook-preservation', 'ship-managed-hook-refresh', + 'ship-unmanaged-hook-consent', 'skillmd-setup-discovery']; +const setupScreens: Record = { + '#4045': ['bin/gstack-config', 'bin/gstack-settings-hook', 'bin/gstack-skill-gates', 'bin/gstack-skill-route', + 'changelog.d/capy-gst-1461-investigate-gate.md', 'docs/skill-gates.md', 'docs/skill-routing-and-contracts.md', + 'hosts/claude/hooks/investigate-gate-post', 'hosts/claude/hooks/investigate-gate-post-failure', + 'hosts/claude/hooks/investigate-gate-pre', 'hosts/claude/hooks/investigate-gate-stop', + 'hosts/claude/hooks/investigate-gate.ts', 'lib/investigate-gate.ts', 'lib/skill-contract.ts', 'lib/skill-gates.ts', + 'lib/skill-route.ts', 'scripts/skill-gates-smoke.ts', 'setup', 'test/investigate-gate.test.ts', + 'test/setup-hook-syntax-gate.test.ts', 'test/setup-skill-hooks-gate.test.ts', + 'test/skill-gates-advisory-bytes.test.ts', 'test/skill-gates.test.ts'], + '#4047': ['bin/gstack-config', 'bin/gstack-settings-hook', 'bin/gstack-skill-gates', 'bin/gstack-skill-route', + 'changelog.d/capy-gst-1461-ceo-review-gate.md', 'docs/skill-gates.md', 'docs/skill-routing-and-contracts.md', + 'hosts/claude/hooks/ceo-review-gate-stop', 'hosts/claude/hooks/ceo-review-gate-subagent-stop', + 'hosts/claude/hooks/ceo-review-gate.ts', 'hosts/claude/hooks/investigate-gate.ts', + 'hosts/claude/hooks/skill-contract-stop-hook.ts', 'hosts/claude/hooks/spawn-bin.ts', 'lib/ceo-review-gate.ts', + 'lib/investigate-gate.ts', 'lib/skill-contract.ts', 'lib/skill-gates.ts', 'lib/skill-route.ts', + 'scripts/skill-gates-smoke.ts', 'setup', 'test/ceo-review-gate.test.ts', 'test/investigate-gate.test.ts', + 'test/setup-hook-syntax-gate.test.ts', 'test/setup-skill-hooks-gate.test.ts', + 'test/skill-gates-advisory-bytes.test.ts', 'test/skill-gates.test.ts'], +}; +const screen = (policy: 'narrow' | 'broad', changedFiles: string[]) => computePaidCaseSelection({ + profile: 'pr', env: { EVALS_BASE: 'fixture-base', EVALS_SCREEN_POLICY: policy }, changedFiles, packageVersionOnly: false }); + +test.each(Object.entries(setupScreens))('a narrow screen of %s derives setup and selects its five cases', (_pr, files) => { + const result = screen('narrow', files); + expect(result.coverage?.mode).not.toBe('full-fallback'); + expect(result.selection.e2e).toEqual(setupHookCases); + expect(result.coverage?.reasons.join('\n')).not.toContain('Shared runtime/build inputs'); +}); + +test('a narrow screen of setup alone runs only the gate cases that invoke it', () => { + const result = screen('narrow', ['setup']); + expect(result.coverage?.mode).not.toBe('full-fallback'); + expect(result.selection.e2e).toEqual(['gstack-upgrade-happy-path', 'skillmd-setup-discovery']); +}); + +test.each(Object.entries(setupScreens))('%s keeps the full gate on a broad screen, and --profile full ignores the screen policy', (_pr, files) => { + for (const changed of [files, ['setup']]) { + const broad = screen('broad', changed); + expect(broad.coverage?.mode).toBe('full-fallback'); + expect(broad.selection.e2e).toEqual(gateIds); + expect(broad.selection.judges).toEqual(judgeIds); + expect(broad.coverage?.reasons).toContain('Shared runtime/build inputs restore every gate case and judge: setup'); + const full = (policy: string, all = '') => computePaidCaseSelection({ profile: 'full', + env: { EVALS_BASE: 'fixture-base', EVALS_SCREEN_POLICY: policy, EVALS_ALL: all }, changedFiles: changed, packageVersionOnly: false }); + expect(full('narrow')).toEqual(full('broad')); + expect(full('narrow', '1')).toEqual({ selection: { e2e: null, judges: null }, reason: 'run-all' }); + } +}); + +test('narrow screens still restore the full gate for every other shared input', () => { + for (const file of sharedInputs) { + const result = screen('narrow', [file]); + expect(result.coverage?.mode, file).toBe('full-fallback'); + expect(result.selection.e2e, file).toEqual(gateIds); + } +}); + + +// GST-1575: modules/loadouts.json is read through scripts/modules.ts (LOADOUTS_FILE), whose static closure +// reaches every gate case; no paid case passes --loadout. Derived like that reader instead of an unknown file. +test('modules/loadouts.json is derived through its reader instead of restoring the full gate', () => { + const narrow = screen('narrow', ['modules/loadouts.json']); + expect(narrow.coverage?.mode).toBe('pr'); + expect(narrow.selection).toEqual({ e2e: [], judges: [] }); + expect(narrow.coverage?.hubFiles).toEqual(['modules/loadouts.json']); + const broad = screen('broad', ['modules/loadouts.json']); + expect(broad.coverage?.mode).toBe('dependents'); + expect(broad.selection.e2e).toEqual(gateIds); + expect(broad.coverage?.unknownFiles).toEqual([]); +}); +