mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-10 02:30:39 +01:00
* Add a check that keeps tracker codes and internal jargon out of PR titles and branch names gstack-plain-title checks the readable front of a title, branch name, PR opening or changelog entry against the repository's .gstack/plain-titles.json; one trailing issue code stays allowed. A Plain title workflow runs it on every new pull request, and /ship's push refuses a never-pushed branch whose name fails it. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Move the PR body's machine-read lines into a closing Agent notes section The ship receipt, its runs block and the Plan line now live in one collapsed section at the end of the body, and every reader looks there first while still accepting the old receipt-at-top layout. gstack-post refuses to post a title the plain-title check refuses, and land uses the checked PR title as main's commit title. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Teach /ship to write plain titles, PR openings, commit messages and changelog entries The README now lists what gstack is for, so a PR opening can name the goal it serves. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Keep tests, size budgets and generated files in step with plain titles and Agent notes Copy lists that install result-code modules now include the plain-title codes, the ship goldens and size ledger are refreshed, the new workflow is classified, and the pinned prose tests read the new title and body layout. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Record the test-profile list change as non-behavioral for the shared test harness Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Keep text people add below the PR notes, and fix three smaller review findings The notes section now ends at its own closing tag, so a collapsible added after it survives the next post. The opening check splits the body the same way the notes reader does. Wave PR titles no longer name the area, which the plain-title check could refuse. The Capy line in the PR-body steps now puts Approvals in the notes. Review-Finding: p1/specialist-batched#1 reader=testing Review-Finding: p1/red-team#1 reader=red-team Review-Finding: p1/red-team#2 reader=red-team Review-Finding: p1/red-team#3 reader=red-team Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Mask credentials in refused-title lines and keep older PRs' plan lines readable A refused title's printed match is masked by the secret scanner first. The Plan line reader falls back to the whole body when the Agent notes section has none, so a body written before this change keeps its plan when its first receipt lands. The notes section also ends at its own closing tag when its opener is split or marked open, or when a collapsible inside it is left unclosed. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Print a fixed placeholder when a credential in a refused title cannot be masked Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Keep a plain-title policy edit from scheduling every paid eval Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Record the paid-eval list change as non-behavioral for the shared test harness Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Recheck the PR title on every edit so a base change cannot turn a refused title green Review-Finding: s11/targeted#1 reader=targeted Review-Finding: s11/adversarial-native#1 reader=adversarial-native Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
405 lines
25 KiB
TypeScript
405 lines
25 KiB
TypeScript
/**
|
|
* Derived PR-lane dependencies (scripts/pr-dependencies.ts) and the
|
|
* fail-closed full-gate list (PR_FULL_GATE_FILES): unmapped files select the
|
|
* gate cases whose reference closure reaches them, underivable files and
|
|
* global inputs restore the full gate, and every fallback names its fix.
|
|
*/
|
|
import { describe, expect, test } from 'bun:test';
|
|
import * as fs from 'node:fs';
|
|
import * as os from 'node:os';
|
|
import * as path from 'node:path';
|
|
import { spawnSync } from 'node:child_process';
|
|
import { deriveDependencies, derivedDependencies, referenceText } from '../scripts/pr-dependencies';
|
|
import {
|
|
DERIVABLE_PREFIXES, PERIODIC_ONLY_PR_FILES, PR_FULL_GATE_FILES, PR_PROFILE_MAPS, formatPrCoverageSummary, selectPrProfile, unknownFileLabel,
|
|
} from '../scripts/test-pr-profile';
|
|
import { buildRunManifest, computePaidCaseSelection } from '../scripts/test-paid-shards';
|
|
import { E2E_TIERS, LLM_JUDGE_TOUCHFILES } from './helpers/touchfiles-data';
|
|
import { matchGlob } from './helpers/test-selection';
|
|
|
|
const ROOT = path.resolve(import.meta.dir, '..');
|
|
const real = derivedDependencies(PR_PROFILE_MAPS, ROOT);
|
|
const gate = Object.keys(E2E_TIERS).filter(id => E2E_TIERS[id] === 'gate');
|
|
const FIX = 'docs/TESTING_INTERNALS.md#pr-paid-lane-fallback';
|
|
|
|
function fixtureRepo(files: Record<string, string>): string {
|
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'pr-deps-'));
|
|
for (const [file, text] of Object.entries(files)) {
|
|
fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true });
|
|
fs.writeFileSync(path.join(root, file), text);
|
|
}
|
|
for (const args of [['init', '-q'], ['add', '-A']]) {
|
|
const result = spawnSync('git', args, { cwd: root, encoding: 'utf8', timeout: 10_000 });
|
|
if (result.status !== 0) throw new Error(`git ${args[0]} failed: ${result.stderr}`);
|
|
}
|
|
return root;
|
|
}
|
|
|
|
describe('reference derivation on a fixture tree', () => {
|
|
const root = fixtureRepo({
|
|
'test/skill-e2e-alpha.test.ts': "import { help } from './helpers/alpha-helper';\nimport { other } from './skill-e2e-beta.test';\n",
|
|
'test/skill-e2e-beta.test.ts': "export const other = 'lib/beta-only.ts';\n",
|
|
'test/helpers/alpha-helper.ts': "import { x } from '../../lib/alpha';\n// prose about lib/commented.ts is not a reference\nexport const help = x;\n",
|
|
'test/helpers/touchfiles-data.ts': "export const MAP = { a: ['lib/listed-only.ts'] };\n",
|
|
'test/helpers/joined.ts': "export const p = path.join(ROOT, 'lib', 'joined.ts');\n", 'lib/joined.ts': '',
|
|
'lib/alpha.ts': "import './../test/helpers/joined';\nexport const x = 1;\n", 'lib/beta-only.ts': '', 'lib/commented.ts': '', 'lib/listed-only.ts': '', 'lib/bin-lib.ts': '', 'lib/unused.ts': '',
|
|
'alpha/SKILL.md.tmpl': '{{GREETING}}\nRun `gstack-tool` then open design/dist/design.\n',
|
|
'bin/gstack-tool': "#!/bin/sh\n# mentions lib/commented.ts only in a comment\nbun run \"$ROOT/lib/bin-lib.ts\"\n",
|
|
'scripts/build.sh': 'bun build --compile design/src/cli.ts --outfile design/dist/design\n',
|
|
'design/src/cli.ts': "import './render';\n", 'design/src/render.ts': '',
|
|
'scripts/resolvers/index.ts': "import { greet } from './greeting';\nexport const RESOLVERS = {\n GREETING: greet,\n};\n",
|
|
'scripts/resolvers/greeting.ts': "import { tone } from './tone';\nexport const greet = () => tone;\n", 'scripts/resolvers/tone.ts': 'export const tone = 1;\n',
|
|
'hosts/acme.ts': '', 'hosts/zeta.ts': '',
|
|
'package.json': '{"scripts":{}}\n',
|
|
});
|
|
const maps = { e2eTouchfiles: { alpha: ['test/skill-e2e-alpha.test.ts', 'alpha/**'], beta: ['test/skill-e2e-beta.test.ts'] }, judgeTouchfiles: {}, globalTouchfiles: [] };
|
|
const derived = deriveDependencies(maps, root);
|
|
const consumers = (file: string) => [...derived.e2e.get(file) ?? []].sort();
|
|
|
|
test('imports, bin invocations, compiled binaries and template placeholders all reach their files', () => {
|
|
expect(consumers('lib/alpha.ts')).toEqual(['alpha']);
|
|
expect(consumers('lib/joined.ts')).toEqual(['alpha']);
|
|
expect(consumers('bin/gstack-tool')).toEqual(['alpha']);
|
|
expect(consumers('lib/bin-lib.ts')).toEqual(['alpha']);
|
|
expect(consumers('design/src/render.ts')).toEqual(['alpha']);
|
|
expect(consumers('scripts/resolvers/tone.ts')).toEqual(['alpha']);
|
|
expect(derived.trace('alpha', 'scripts/resolvers/tone.ts')).toEqual(['alpha/SKILL.md.tmpl', 'scripts/resolvers/greeting.ts', 'scripts/resolvers/tone.ts']);
|
|
});
|
|
|
|
test('comments, selection data and other paid test files are not references', () => {
|
|
expect(consumers('lib/commented.ts')).toEqual([]);
|
|
expect(consumers('lib/listed-only.ts')).toEqual([]);
|
|
expect(consumers('lib/beta-only.ts')).toEqual(['beta']);
|
|
expect(consumers('lib/unused.ts')).toEqual([]);
|
|
expect(referenceText('a.ts', ' // lib/x.ts\nconst y = "lib/z.ts"; // trailing lib/w.ts\n')).toContain('lib/z.ts');
|
|
expect(referenceText('a.ts', ' // lib/x.ts\n')).not.toContain('lib/x.ts');
|
|
expect(referenceText('bin/x', '# lib/x.ts\necho lib/y.ts\n')).toBe('\necho lib/y.ts\n');
|
|
expect(referenceText('data.json', '{"a":"lib/x.ts"}')).toBe('');
|
|
});
|
|
|
|
test('trace names the shortest chain: a template that invokes a bin directly wins over a long import path to it', () => {
|
|
const repo = fixtureRepo({
|
|
'test/skill-e2e-gamma.test.ts': '', 'gamma/SKILL.md.tmpl': 'Run bin/gstack-target.\n', 'gamma/sections/a.md.tmpl': 'See lib/one.ts.\n',
|
|
'lib/one.ts': "import './two';\n", 'lib/two.ts': "import './three';\n", 'lib/three.ts': "export const x = 'bin/gstack-target';\n",
|
|
'bin/gstack-target': '#!/bin/sh\n', 'package.json': '{}',
|
|
});
|
|
const traced = deriveDependencies({ e2eTouchfiles: { gamma: ['test/skill-e2e-gamma.test.ts', 'gamma/**'] }, judgeTouchfiles: {}, globalTouchfiles: [] }, repo);
|
|
expect(traced.trace('gamma', 'bin/gstack-target')).toEqual(['gamma/SKILL.md.tmpl', 'bin/gstack-target']);
|
|
expect(traced.trace('gamma', 'lib/three.ts')).toEqual(['gamma/sections/a.md.tmpl', 'lib/one.ts', 'lib/two.ts', 'lib/three.ts']);
|
|
});
|
|
|
|
test('a host config feeds only cases whose test names the host', () => {
|
|
const named = deriveDependencies({ ...maps, e2eTouchfiles: { ...maps.e2eTouchfiles, beta: ['test/skill-e2e-beta.test.ts'] } },
|
|
fixtureRepo({ 'test/skill-e2e-beta.test.ts': "run({ host: 'acme' });\n", 'hosts/acme.ts': '', 'hosts/zeta.ts': '', 'scripts/build.sh': '', 'scripts/resolvers/index.ts': '', 'package.json': '{}' }));
|
|
expect([...named.e2e.get('hosts/acme.ts') ?? []]).toEqual(['beta']);
|
|
expect(named.e2e.get('hosts/zeta.ts')).toBeUndefined();
|
|
});
|
|
});
|
|
|
|
describe('shebang files and import scan failures (A1-1)', () => {
|
|
const tree = (extra: Record<string, string> = {}) => fixtureRepo({
|
|
'test/skill-e2e-alpha.test.ts': "import '../lib/entry';\n", 'test/skill-e2e-beta.test.ts': '',
|
|
'lib/entry.ts': '#!/usr/bin/env bun\nimport { y } from \'./typed\';\nexport const x = y;\n',
|
|
'lib/typed.ts': 'export const y = <T,>(v: T) => v as unknown as T;\nimport \'./typed-dep\';\n', 'lib/typed-dep.ts': '',
|
|
'alpha/SKILL.md.tmpl': 'Run `gstack-sb`.\n', 'bin/gstack-sb': '#!/usr/bin/env -S bun run\nimport { z } from \'../lib/sb-dep\';\n', 'lib/sb-dep.ts': '',
|
|
'bin/gstack-sh': '#!/bin/sh\nimport ../lib/sh-only.ts\n', 'lib/sh-only.ts': '',
|
|
'scripts/build.sh': '', 'scripts/resolvers/index.ts': '', 'package.json': '{}', ...extra,
|
|
});
|
|
const maps = { e2eTouchfiles: { alpha: ['test/skill-e2e-alpha.test.ts', 'alpha/**'], beta: ['test/skill-e2e-beta.test.ts'] }, judgeTouchfiles: {}, globalTouchfiles: [] as string[] };
|
|
const profileMaps = { ...maps, tiers: { alpha: 'gate', beta: 'gate' } as const };
|
|
const pick = (derived: ReturnType<typeof deriveDependencies>, changedFiles: string[]) =>
|
|
selectPrProfile({ maps: profileMaps, profile: [], selectedE2E: [], selectedJudges: [], changedFiles, derived });
|
|
|
|
test('a shebang .ts file, a bun-shebang extensionless bin and a .ts generic arrow are all import-scanned', () => {
|
|
const derived = deriveDependencies(maps, tree());
|
|
expect([...derived.e2e.get('lib/typed.ts') ?? []]).toEqual(['alpha']);
|
|
expect([...derived.e2e.get('lib/typed-dep.ts') ?? []]).toEqual(['alpha']);
|
|
expect([...derived.e2e.get('lib/sb-dep.ts') ?? []]).toEqual(['alpha']);
|
|
expect(derived.e2e.get('lib/sh-only.ts')).toBeUndefined();
|
|
expect(derived.scanErrors.size).toBe(0);
|
|
expect(pick(derived, ['lib/sb-dep.ts'])).toMatchObject({ mode: 'dependents', e2e: ['alpha'] });
|
|
});
|
|
|
|
test('lib/fs-utils.ts selects the cases of bin/gstack-redact-prepush, which imports it', () => {
|
|
const bin = gate.filter(id => real.e2e.get('bin/gstack-redact-prepush')?.has(id));
|
|
expect(bin.length).toBeGreaterThan(0);
|
|
const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: ['lib/fs-utils.ts'], packageVersionOnly: true });
|
|
expect(result.coverage?.mode).toBe('dependents');
|
|
for (const id of bin) expect(result.selection.e2e, id).toContain(id);
|
|
expect(real.scanErrors.size).toBe(0);
|
|
expect(real.global.has('test/helpers/bootstrap-retention.ts')).toBe(true);
|
|
});
|
|
|
|
test('any other scan error fails closed at every site: case closures, the global walk and deleted-file referencers', () => {
|
|
const broken = "import { from './nowhere';\n";
|
|
const inClosure = deriveDependencies(maps, tree({ 'lib/typed-dep.ts': broken }));
|
|
expect([...inClosure.scanErrors.keys()]).toEqual(['lib/typed-dep.ts']);
|
|
const closed = pick(inClosure, ['lib/sb-dep.ts']);
|
|
expect(closed).toMatchObject({ mode: 'full-fallback', e2e: ['alpha', 'beta'], unknownFiles: ['lib/sb-dep.ts'] });
|
|
expect(closed.unknownFileLabels[0]).toMatchObject({ label: 'import scan failed' });
|
|
expect(closed.unknownFileLabels[0]!.fix).toContain('lib/typed-dep.ts');
|
|
|
|
const inGlobal = deriveDependencies({ ...maps, globalTouchfiles: ['lib/runner.ts'] }, tree({ 'lib/runner.ts': broken }));
|
|
expect([...inGlobal.scanErrors.keys()]).toEqual(['lib/runner.ts']);
|
|
expect(pick(inGlobal, ['lib/sb-dep.ts']).mode).toBe('full-fallback');
|
|
|
|
const unreached = deriveDependencies(maps, tree({ 'lib/orphan.ts': broken }));
|
|
expect(unreached.scanErrors.size).toBe(0);
|
|
expect(pick(unreached, ['lib/gone.ts'])).toMatchObject({ mode: 'full-fallback', unknownFiles: ['lib/gone.ts'] });
|
|
expect([...unreached.scanErrors.keys()]).toEqual(['lib/orphan.ts']);
|
|
});
|
|
});
|
|
|
|
describe('safe classification (A1-2)', () => {
|
|
test('capy/support.env derives to the bins that read it instead of restoring the full gate', () => {
|
|
for (const bin of ['bin/gstack-install-verdict.sh', 'bin/gstack-doctor-check.sh']) {
|
|
for (const id of real.e2e.get(bin) ?? []) expect(real.e2e.get('capy/support.env')?.has(id), `${bin} ${id}`).toBe(true);
|
|
}
|
|
const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: ['capy/support.env'], packageVersionOnly: true });
|
|
expect(result.coverage?.unknownFiles).toEqual([]);
|
|
expect(result.coverage?.mode).toBe('dependents');
|
|
expect(DERIVABLE_PREFIXES).toContain('capy/');
|
|
});
|
|
|
|
test('repository config read only by free lanes or GitHub stays in the PR profile', () => {
|
|
const files = ['.github/dependabot.yml', '.osv-scanner.toml', 'setup.ps1', '.gitlab-ci.yml', '.github/ISSUE_TEMPLATE/bug_report.md', '.gstack/ship-policy.example.json', '.gstack/plain-titles.json'];
|
|
for (const file of files) expect(real.tracked.has(file), file).toBe(true);
|
|
const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: files, packageVersionOnly: true });
|
|
expect(result.coverage?.mode).toBe('pr');
|
|
expect(result.selection).toEqual({ e2e: [], judges: [] });
|
|
});
|
|
|
|
test('a full-gate file is never labelled a real unknown dependency', () => {
|
|
for (const file of ['.github/workflows/evals.yml', 'test-setup.ts', '.gitattributes']) {
|
|
const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: [file], packageVersionOnly: true });
|
|
expect(result.coverage?.mode, file).toBe('full-fallback');
|
|
for (const label of result.coverage!.unknownFileLabels) {
|
|
expect(label.label, file).toBe('full-gate input');
|
|
expect(label.fix).toContain('PR_FULL_GATE_FILES');
|
|
}
|
|
expect(unknownFileLabel(file).label).toBe('full-gate input');
|
|
}
|
|
expect(unknownFileLabel('SECURITY.md').label).toBe('real unknown dependency');
|
|
});
|
|
});
|
|
|
|
describe('derived dependencies of this checkout', () => {
|
|
test('a bin script a skill template invokes selects that skill\'s cases', () => {
|
|
const consumers = real.e2e.get('bin/gstack-next-version') ?? new Set();
|
|
expect(consumers.has('review-sql-injection')).toBe(true);
|
|
expect(real.trace('review-sql-injection', 'bin/gstack-next-version')?.[0]).toBe('review/SKILL.md.tmpl');
|
|
});
|
|
|
|
test('a compiled design source reaches exactly the cases whose closure names the design binary', () => {
|
|
const consumers = [...real.e2e.get('design/src/evolve.ts') ?? []];
|
|
expect(consumers.length).toBeGreaterThan(0);
|
|
expect(consumers.length).toBeLessThan(Object.keys(E2E_TIERS).length);
|
|
for (const id of consumers.slice(0, 5)) expect(real.trace(id, 'design/src/evolve.ts')?.some(file => file === 'design/src/cli.ts' || file.startsWith('design/src/'))).toBe(true);
|
|
});
|
|
|
|
test('the paid runner\'s import closure is global, and only that', () => {
|
|
for (const file of ['scripts/test-paid-shards.ts', 'scripts/e2e-shard-reuse.ts', 'test/helpers/session-runner.ts', 'test/helpers/eval-store.ts', 'scripts/lib/shard-engine.ts']) {
|
|
expect(real.global.has(file), file).toBe(true);
|
|
}
|
|
expect(real.global.has('bin/gstack-next-version')).toBe(false);
|
|
expect(real.global.size).toBeLessThan(120);
|
|
});
|
|
});
|
|
|
|
describe('PR selection with derived dependencies', () => {
|
|
const select = (changedFiles: string[]) => computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles, packageVersionOnly: true });
|
|
|
|
test('an unmapped file with derived consumers runs every dependent gate case and nothing else', () => {
|
|
const result = select(['design/src/evolve.ts']);
|
|
expect(result.coverage?.mode).toBe('dependents');
|
|
const expected = gate.filter(id => real.e2e.get('design/src/evolve.ts')?.has(id)).sort();
|
|
expect(result.selection.e2e).toEqual(expected);
|
|
expect(result.coverage?.derivedFiles).toEqual([{ file: 'design/src/evolve.ts', e2e: real.e2e.get('design/src/evolve.ts')!.size, judges: real.judges.get('design/src/evolve.ts')?.size ?? 0 }]);
|
|
const manifest = buildRunManifest({ tier: 'gate', profile: 'pr', sliceCount: 2, evalsAll: false, env: {}, changedFiles: ['design/src/evolve.ts'] });
|
|
expect(manifest.prCoverage?.mode).toBe('dependents');
|
|
expect(manifest.entries.some(entry => entry.status === 'planned')).toBe(true);
|
|
});
|
|
|
|
test('a tracked file no paid case reaches is exempt; periodic-only workflows never run the PR gate', () => {
|
|
const result = select(['scripts/typecheck-test.ts', ...PERIODIC_ONLY_PR_FILES]);
|
|
expect(result.coverage?.mode).toBe('pr');
|
|
expect(result.coverage?.noConsumerFiles).toEqual(['scripts/typecheck-test.ts']);
|
|
expect(result.selection).toEqual({ e2e: [], judges: [] });
|
|
});
|
|
|
|
test('a paid-runner import selects every case, as a global touchfile does', () => {
|
|
const viaClosure = select(['scripts/e2e-shard-reuse.ts']);
|
|
const viaGlobal = select(['scripts/test-paid-shards.ts']);
|
|
expect(viaClosure.coverage?.mode).toBe('pr');
|
|
expect(viaClosure.selection).toEqual(viaGlobal.selection);
|
|
});
|
|
|
|
test('derivation is skipped for fixture maps unless passed explicitly', () => {
|
|
const result = selectPrProfile({ maps: { ...PR_PROFILE_MAPS, e2eTouchfiles: { ...PR_PROFILE_MAPS.e2eTouchfiles } }, selectedE2E: [], selectedJudges: [], changedFiles: ['design/src/evolve.ts'] });
|
|
expect(result.mode).toBe('full-fallback');
|
|
});
|
|
});
|
|
|
|
describe('the fail-closed full-gate list', () => {
|
|
const tracked = [...real.tracked];
|
|
|
|
test('every entry matches a tracked file', () => {
|
|
const stale = PR_FULL_GATE_FILES.filter(pattern => !tracked.some(file => matchGlob(file, pattern)));
|
|
expect(stale.map(pattern => `${pattern}: matches no tracked file. Fix: delete it from PR_FULL_GATE_FILES in scripts/test-pr-profile.ts (${FIX})`)).toEqual([]);
|
|
});
|
|
|
|
test('every entry restores the full gate even when a touchfile or a derived reference also names it', () => {
|
|
for (const pattern of PR_FULL_GATE_FILES) {
|
|
const file = tracked.find(candidate => matchGlob(candidate, pattern))!;
|
|
const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: [file], packageVersionOnly: false });
|
|
expect({ file, mode: result.coverage?.mode }).toEqual({ file, mode: 'full-fallback' });
|
|
expect(result.selection.e2e).toEqual(gate.sort());
|
|
}
|
|
});
|
|
|
|
test('the setup, preamble, eval policy and PR workflow inputs are on it', () => {
|
|
for (const file of ['setup', 'scripts/resolvers/preamble.ts', 'test/helpers/periodic-exclude-data.ts', '.github/workflows/evals.yml', 'bun.lock']) {
|
|
expect(PR_FULL_GATE_FILES.some(pattern => matchGlob(file, pattern)), file).toBe(true);
|
|
}
|
|
});
|
|
|
|
test('a file the derivation cannot place restores the full gate with its fix', () => {
|
|
for (const file of ['SECURITY.md', 'conductor.json']) {
|
|
const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: [file], packageVersionOnly: true });
|
|
expect(result.coverage?.mode, file).toBe('full-fallback');
|
|
const label = result.coverage!.unknownFileLabels[0]!;
|
|
expect(label.file).toBe(file);
|
|
expect(label.fix).toContain(file);
|
|
expect(label.fix).toContain('touchfiles-data.ts');
|
|
expect(label.fix).toContain('DERIVABLE_PREFIXES');
|
|
}
|
|
expect(DERIVABLE_PREFIXES).not.toContain('test/fixtures/');
|
|
});
|
|
|
|
});
|
|
|
|
describe('deleted files (paths absent from the head tree)', () => {
|
|
const without = (file: string) => ({ ...real, tracked: new Set([...real.tracked].filter(other => other !== file)) });
|
|
const pick = (file: string, derived = without(file)) => selectPrProfile({ selectedE2E: [], selectedJudges: [], changedFiles: [file], derived });
|
|
|
|
test('a deleted file nothing references and no touchfile matches has no dependents', () => {
|
|
const result = pick('scripts/retired-tool-nobody-calls.ts', real);
|
|
expect(result.mode).toBe('pr');
|
|
expect(result.noConsumerFiles).toEqual(['scripts/retired-tool-nobody-calls.ts']);
|
|
expect(result.e2e).toEqual([]);
|
|
});
|
|
|
|
test('a deleted file a touchfile matches selects that touchfile\'s cases', () => {
|
|
const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: ['browse/src/retired-module.ts'], packageVersionOnly: true });
|
|
expect(result.coverage?.unknownFiles).toEqual([]);
|
|
expect(result.coverage?.mode).toBe('pr');
|
|
expect(result.selection.e2e).toContain('browse-basic');
|
|
});
|
|
|
|
test('a deleted file live code still references selects the referencing files\' dependents', () => {
|
|
const result = pick('bin/gstack-next-version');
|
|
expect(result.mode).toBe('dependents');
|
|
expect(result.e2e).toContain('review-sql-injection');
|
|
expect(result.derivedFiles[0]?.file).toBe('bin/gstack-next-version');
|
|
expect(real.referencers('bin/gstack-next-version')).toContain('review/SKILL.md.tmpl');
|
|
});
|
|
|
|
test('a file the PR adds is read from the checkout under review, not from the selector\'s own tree (an older install)', () => {
|
|
const added = 'hosts/claude/hooks/zz-added-hook.ts';
|
|
expect(fs.existsSync(path.join(ROOT, added))).toBe(false);
|
|
expect(real.referencers(added)).toContain('setup');
|
|
const checkout = fixtureRepo({ setup: 'SETUP_HOOK=hosts/claude/hooks/zz-added-hook\n', [added]: 'export {};\n' });
|
|
try {
|
|
const result = computePaidCaseSelection({ profile: 'pr', rootDir: checkout, env: { EVALS_BASE: 'HEAD' }, changedFiles: [added], packageVersionOnly: true });
|
|
expect(result.coverage!.unknownFileLabels.filter(l => l.file === added)).toEqual([]);
|
|
expect(result.reason).not.toContain('deleted but still referenced');
|
|
} finally {
|
|
fs.rmSync(checkout, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test('a deleted file a full-gate input still references, or a deleted full-gate entry, restores the full gate', () => {
|
|
expect(real.referencers('bin/gstack-install-registry.sh')).toContain('setup');
|
|
const referenced = pick('bin/gstack-install-registry.sh');
|
|
expect(referenced.mode).toBe('full-fallback');
|
|
expect(referenced.unknownFileLabels[0]).toMatchObject({ file: 'bin/gstack-install-registry.sh', label: 'deleted but still referenced' });
|
|
expect(referenced.unknownFileLabels[0]!.fix).toContain('setup');
|
|
expect(pick('bunfig.toml').mode).toBe('full-fallback');
|
|
});
|
|
|
|
test('comments and data files never count as references to a deleted path', () => {
|
|
const root = fixtureRepo({
|
|
'test/skill-e2e-alpha.test.ts': "// lib/gone.ts used to live here\nimport './helpers/h';\n",
|
|
'test/helpers/h.ts': "export const data = 'x';\n", 'data/list.json': '["lib/gone.ts"]',
|
|
'scripts/build.sh': '', 'scripts/resolvers/index.ts': '', 'package.json': '{}',
|
|
});
|
|
const derived = deriveDependencies({ e2eTouchfiles: { alpha: ['test/skill-e2e-alpha.test.ts'] }, judgeTouchfiles: {}, globalTouchfiles: [] }, root);
|
|
expect(derived.referencers('lib/gone.ts')).toEqual([]);
|
|
const relative = fixtureRepo({ 'test/skill-e2e-alpha.test.ts': "import './helpers/gone';\n", 'scripts/build.sh': '', 'scripts/resolvers/index.ts': '', 'package.json': '{}' });
|
|
expect(deriveDependencies({ e2eTouchfiles: { alpha: ['test/skill-e2e-alpha.test.ts'] }, judgeTouchfiles: {}, globalTouchfiles: [] }, relative)
|
|
.referencers('test/helpers/gone.ts')).toEqual(['test/skill-e2e-alpha.test.ts']);
|
|
});
|
|
});
|
|
|
|
describe('narrow screens leave hub files to the train gate', () => {
|
|
const plan = (changedFiles: string[], policy = 'narrow', profile: 'pr' | 'full' = 'pr') =>
|
|
computePaidCaseSelection({ profile, env: { EVALS_SCREEN_POLICY: policy }, changedFiles, packageVersionOnly: true });
|
|
const hub = 'lib/restamp.ts';
|
|
|
|
test('a hub file reaches every gate case only through the derived closure', () => {
|
|
expect(gate.every(id => real.e2e.get(hub)?.has(id))).toBe(true);
|
|
});
|
|
|
|
test('a hub-only diff selects no case or judge on a narrow screen and names the hub file', () => {
|
|
const result = plan([hub]);
|
|
expect(result.coverage?.hubFiles).toEqual([hub]);
|
|
expect(result.coverage?.mode).toBe('pr');
|
|
expect(result.selection).toEqual({ e2e: [], judges: [] });
|
|
expect(plan([hub], 'broad').selection.judges).toEqual(Object.keys(LLM_JUDGE_TOUCHFILES).sort());
|
|
expect(formatPrCoverageSummary({ profile: 'pr', selection: result.selection, prCoverage: result.coverage }).join('\n'))
|
|
.toContain(`Left to the train gate's full census (narrow screen): \`${hub}\``);
|
|
expect(result.reason).toContain(`train gate's full census runs them: ${hub}`);
|
|
});
|
|
|
|
test('a broad screen, an unset policy and the full profile keep every dependent of a hub file', () => {
|
|
expect(computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: [hub], packageVersionOnly: true }).selection.e2e).toEqual(gate.sort());
|
|
const broad = plan([hub], 'broad');
|
|
expect(broad.coverage?.mode).toBe('dependents');
|
|
expect(broad.coverage?.hubFiles).toEqual([]);
|
|
expect(broad.selection.e2e).toEqual(gate.sort());
|
|
expect(plan([hub], 'narrow', 'full')).toEqual(plan([hub], 'broad', 'full'));
|
|
});
|
|
|
|
test('a hub file beside a shared runtime input, an unknown file or a full-gate file still runs the full gate', () => {
|
|
for (const other of ['scripts/resolvers/preamble/generate-voice-directive.ts', 'SECURITY.md', 'bun.lock']) {
|
|
expect({ other, e2e: plan([hub, other]).selection.e2e }).toEqual({ other, e2e: gate.sort() });
|
|
}
|
|
});
|
|
|
|
test('a hub file beside a direct edit keeps that edit\'s own cases', () => {
|
|
const direct = plan(['browse/src/retired-module.ts']).selection.e2e;
|
|
expect(direct).toContain('browse-basic');
|
|
expect(plan([hub, 'browse/src/retired-module.ts']).selection.e2e).toEqual(direct);
|
|
});
|
|
|
|
test('a deleted file that still reaches every gate case is not a hub: a narrow screen keeps its dependents', () => {
|
|
const without = { ...real, tracked: new Set([...real.tracked].filter(file => file !== hub)) };
|
|
const result = selectPrProfile({ selectedE2E: [], selectedJudges: [], changedFiles: [hub], derived: without, screenPolicy: 'narrow' });
|
|
expect(result.hubFiles).toEqual([]);
|
|
expect(result.mode).toBe('dependents');
|
|
expect(result.e2e).toEqual(gate.sort());
|
|
});
|
|
|
|
test('only dispatched screens default to narrow in the PR workflow; a pull_request run plans broad', () => {
|
|
const workflow = fs.readFileSync(path.join(ROOT, '.github', 'workflows', 'evals.yml'), 'utf8');
|
|
expect(workflow).toContain("EVALS_SCREEN_POLICY: ${{ inputs.screen_policy || (github.event_name == 'workflow_dispatch' && 'narrow') || 'broad' }}");
|
|
expect(workflow).not.toContain("EVALS_SCREEN_POLICY: ${{ inputs.screen_policy || 'narrow' }}");
|
|
});
|
|
|
|
test('size-ledger baselines select only the cases that read them, none in the gate', () => {
|
|
const result = plan(['test/size-ledger/ship/base.json']);
|
|
expect(result.coverage?.unknownFiles).toEqual([]);
|
|
expect(result.selection.e2e).toEqual([]);
|
|
});
|
|
});
|