mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-04 07:47:27 +01:00
test: fold the moved-code touchfile golden into touchfiles.test.ts; fix integration fixture closure and caps
This commit is contained in:
garrytan committed
1 parent
7a50eb250e
commit
0778c70b91
5 files changed
+100
-108
No files matched your search
+1
-1
@@ -33,7 +33,7 @@
|
||||
{ "file": "test/helpers/pty/session.ts", "movedFrom": "test/helpers/claude-pty-runner.ts" }
|
||||
],
|
||||
"residualFiles": [
|
||||
{ "file": "scripts/test-free-shards.ts", "maxLines": 2356, "reason": "free lane policy after the shard engine took spawn, sandbox, logs, seeds and flags" },
|
||||
{ "file": "scripts/test-free-shards.ts", "maxLines": 2361, "reason": "free lane policy after the shard engine took spawn, sandbox, logs, seeds and flags" },
|
||||
{ "file": "scripts/test-paid-shards.ts", "maxLines": 1921, "reason": "paid lane policy after the shard engine took sandbox, logs, seeds, verdicts and flags" },
|
||||
{ "file": "browse/src/server.ts", "maxLines": 2224, "reason": "server lifecycle, tunnel filter and fetch entry after the route table took dispatch, auth and handlers" },
|
||||
{ "file": "test/helpers/claude-pty-runner.ts", "maxLines": 30, "reason": "re-export barrel for the 117 importers after the split into test/helpers/pty/*" }
|
||||
|
||||
@@ -1659,8 +1659,9 @@ describe('SPEC_REVIEW_LOOP resolver', () => {
|
||||
expect(report).toContain('failed mkdir or append stops the review');
|
||||
expect(report.replace(/\s+/g, ' ')).toContain('Recording the **0H spec-review metrics** is required when writing is permitted, even if the reviewer failed');
|
||||
expect(report.replace(/\s+/g, ' ')).toContain('If the reviewer fails, report that limit and continue after recording the outcome; if a required save fails, stop before claiming completion');
|
||||
expect(report).toContain('mkdir -p ~/.gstack/analytics || exit 1');
|
||||
expect(report).toContain('>> ~/.gstack/analytics/spec-review.jsonl || exit 1');
|
||||
expect(report).toContain('eval "$(~/.claude/skills/gstack/bin/gstack-paths)"; : "${GSTACK_STATE_ROOT:?gstack-paths failed; reinstall with ./setup or /gstack-upgrade}"');
|
||||
expect(report).toContain('mkdir -p "$GSTACK_STATE_ROOT/analytics" || exit 1');
|
||||
expect(report).toContain('>> "$GSTACK_STATE_ROOT/analytics/spec-review.jsonl" || exit 1');
|
||||
expect(report).not.toContain('Your doc survived');
|
||||
});
|
||||
|
||||
|
||||
@@ -467,7 +467,7 @@ describe('test-free-shards: exclusive host-state phase', () => {
|
||||
let child: ReturnType<typeof Bun.spawn> | undefined;
|
||||
let watchdog: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
for (const file of ['scripts/test-free-shards.ts', 'scripts/test-strict-output.ts', 'scripts/lib/shard-engine.ts',
|
||||
for (const file of ['scripts/test-free-shards.ts', 'scripts/test-strict-output.ts', 'scripts/lib/shard-engine.ts', 'lib/state-root.ts',
|
||||
'test/helpers/paid-test-set.ts', 'test/helpers/touchfiles.ts', 'test/helpers/touchfiles-data.ts', 'test/helpers/test-selection.ts']) {
|
||||
const target = path.join(directory, file);
|
||||
fs.mkdirSync(path.dirname(target), { recursive: true });
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
/**
|
||||
* Touchfile coverage for moved code.
|
||||
*
|
||||
* Each JSON file in test/fixtures/touchfile-moved-code/ records, before a
|
||||
* refactor moved code, the paid evals that touching each source file selected
|
||||
* ("global" when the source was a global touchfile). Touching any module the
|
||||
* code moved into must select a superset, so a move never silently drops eval
|
||||
* coverage. A module key ending in "/" covers every .ts file under it.
|
||||
*/
|
||||
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { E2E_TOUCHFILES, GLOBAL_TOUCHFILES, LLM_JUDGE_TOUCHFILES, selectTests } from './helpers/touchfiles';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const GOLDEN_DIR = 'test/fixtures/touchfile-moved-code';
|
||||
const LANES = ['e2e', 'llmJudge'] as const;
|
||||
|
||||
type Lane = typeof LANES[number];
|
||||
type Selection = 'global' | string[];
|
||||
type Maps = Record<Lane, Record<string, string[]>>;
|
||||
interface Golden {
|
||||
workstream: string;
|
||||
recordedAt: string;
|
||||
sources: Record<string, Record<Lane, Selection>>;
|
||||
modules: Record<string, string[]>;
|
||||
}
|
||||
|
||||
const MAPS: Maps = { e2e: E2E_TOUCHFILES, llmJudge: LLM_JUDGE_TOUCHFILES };
|
||||
|
||||
function expandModule(key: string): string[] {
|
||||
if (!key.endsWith('/')) return [key];
|
||||
return fs.readdirSync(path.join(ROOT, key), { recursive: true })
|
||||
.map(String).filter(file => file.endsWith('.ts')).map(file => key + file.replace(/\\/g, '/'));
|
||||
}
|
||||
|
||||
export function selectionGaps(golden: Golden, maps: Maps, globals: string[], modulesFor = expandModule): string[] {
|
||||
const gaps: string[] = [];
|
||||
for (const [key, sources] of Object.entries(golden.modules)) {
|
||||
for (const module of modulesFor(key)) {
|
||||
for (const lane of LANES) {
|
||||
const result = selectTests([module], maps[lane], globals);
|
||||
if (result.reason.startsWith('global')) continue;
|
||||
const selected = new Set(result.selected);
|
||||
for (const source of sources) {
|
||||
const recorded = golden.sources[source]?.[lane];
|
||||
if (recorded === undefined) { gaps.push(`${module} no recorded ${lane} selection for source ${source}`); continue; }
|
||||
const missing = recorded === 'global' ? ['(every eval: source was a global touchfile)'] : recorded.filter(name => !selected.has(name));
|
||||
if (missing.length) gaps.push(`${module} ${lane} misses ${missing.join(', ')} (selected by ${source} before the move)`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return gaps;
|
||||
}
|
||||
|
||||
function formatGaps(gaps: string[], file: string): string {
|
||||
return [
|
||||
`Touchfile move coverage: ${gaps.length} gap(s):`,
|
||||
...gaps.map(gap => ` ${gap}`),
|
||||
'Rule: code moved out of a file keeps that file\'s paid-eval selection, so an edit to the new module still runs the evals the old file ran.',
|
||||
'Fix: add the new module (or a directory glob such as \'test/helpers/pty/**\') to every touchfile entry that lists its source file in test/helpers/touchfiles-data.ts (E2E, LLM judge, or GLOBAL_TOUCHFILES).',
|
||||
`Golden: ${GOLDEN_DIR}/${file}; re-record a source's selection only for a deliberate, reviewed touchfile change.`,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
const goldens = fs.readdirSync(path.join(ROOT, GOLDEN_DIR)).filter(file => file.endsWith('.json')).sort();
|
||||
|
||||
describe('touchfile coverage for moved code', () => {
|
||||
test('at least one golden is recorded', () => {
|
||||
expect(goldens.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test.each(goldens)('%s: every moved module exists and selects a superset of its sources', file => {
|
||||
const golden = JSON.parse(fs.readFileSync(path.join(ROOT, GOLDEN_DIR, file), 'utf8')) as Golden;
|
||||
for (const key of Object.keys(golden.modules)) {
|
||||
const modules = expandModule(key);
|
||||
expect(modules.length, `${key} in ${GOLDEN_DIR}/${file} names no files`).toBeGreaterThan(0);
|
||||
for (const module of modules) expect(fs.existsSync(path.join(ROOT, module)), `${module} is listed in ${GOLDEN_DIR}/${file} but does not exist`).toBe(true);
|
||||
}
|
||||
const gaps = selectionGaps(golden, MAPS, GLOBAL_TOUCHFILES);
|
||||
if (gaps.length) throw new Error(formatGaps(gaps, file));
|
||||
});
|
||||
|
||||
test('a planted module missing one source entry, or a dropped global source, is reported with the fix', () => {
|
||||
const golden: Golden = {
|
||||
workstream: 'planted', recordedAt: 'test',
|
||||
sources: {
|
||||
'src/old.ts': { e2e: ['eval-a', 'eval-b'], llmJudge: [] },
|
||||
'src/global.ts': { e2e: 'global', llmJudge: [] },
|
||||
},
|
||||
modules: { 'src/new/': ['src/old.ts'], 'src/moved-global.ts': ['src/global.ts'] },
|
||||
};
|
||||
const maps: Maps = { e2e: { 'eval-a': ['src/old.ts', 'src/new/**'], 'eval-b': ['src/old.ts'] }, llmJudge: {} };
|
||||
const gaps = selectionGaps(golden, maps, ['src/global.ts'], key => key.endsWith('/') ? ['src/new/module.ts'] : [key]);
|
||||
expect(gaps).toEqual([
|
||||
'src/new/module.ts e2e misses eval-b (selected by src/old.ts before the move)',
|
||||
'src/moved-global.ts e2e misses (every eval: source was a global touchfile) (selected by src/global.ts before the move)',
|
||||
]);
|
||||
expect(formatGaps(gaps, 'planted.json')).toContain('Fix: add the new module');
|
||||
expect(formatGaps(gaps, 'planted.json')).toContain(`Golden: ${GOLDEN_DIR}/planted.json`);
|
||||
});
|
||||
});
|
||||
@@ -717,3 +717,98 @@ describe('derived touchfile closure', () => {
|
||||
expect(paid.filter(file => !keyed(file))).toEqual(Object.keys(KEYLESS_PAID).sort());
|
||||
});
|
||||
});
|
||||
|
||||
// Touchfile coverage for moved code. Each JSON file in
|
||||
// test/fixtures/touchfile-moved-code/ records, before a refactor moved code,
|
||||
// the paid evals that touching each source file selected ("global" when the
|
||||
// source was a global touchfile). Touching any module the code moved into must
|
||||
// select a superset, so a move never silently drops eval coverage. A module
|
||||
// key ending in "/" covers every .ts file under it.
|
||||
const GOLDEN_DIR = 'test/fixtures/touchfile-moved-code';
|
||||
const LANES = ['e2e', 'llmJudge'] as const;
|
||||
|
||||
type Lane = typeof LANES[number];
|
||||
type Selection = 'global' | string[];
|
||||
type Maps = Record<Lane, Record<string, string[]>>;
|
||||
interface Golden {
|
||||
workstream: string;
|
||||
recordedAt: string;
|
||||
sources: Record<string, Record<Lane, Selection>>;
|
||||
modules: Record<string, string[]>;
|
||||
}
|
||||
|
||||
const MAPS: Maps = { e2e: E2E_TOUCHFILES, llmJudge: LLM_JUDGE_TOUCHFILES };
|
||||
|
||||
function expandModule(key: string): string[] {
|
||||
if (!key.endsWith('/')) return [key];
|
||||
return fs.readdirSync(path.join(ROOT, key), { recursive: true })
|
||||
.map(String).filter(file => file.endsWith('.ts')).map(file => key + file.replace(/\\/g, '/'));
|
||||
}
|
||||
|
||||
function selectionGaps(golden: Golden, maps: Maps, globals: string[], modulesFor = expandModule): string[] {
|
||||
const gaps: string[] = [];
|
||||
for (const [key, sources] of Object.entries(golden.modules)) {
|
||||
for (const module of modulesFor(key)) {
|
||||
for (const lane of LANES) {
|
||||
const result = selectTests([module], maps[lane], globals);
|
||||
if (result.reason.startsWith('global')) continue;
|
||||
const selected = new Set(result.selected);
|
||||
for (const source of sources) {
|
||||
const recorded = golden.sources[source]?.[lane];
|
||||
if (recorded === undefined) { gaps.push(`${module} no recorded ${lane} selection for source ${source}`); continue; }
|
||||
const missing = recorded === 'global' ? ['(every eval: source was a global touchfile)'] : recorded.filter(name => !selected.has(name));
|
||||
if (missing.length) gaps.push(`${module} ${lane} misses ${missing.join(', ')} (selected by ${source} before the move)`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return gaps;
|
||||
}
|
||||
|
||||
function formatGaps(gaps: string[], file: string): string {
|
||||
return [
|
||||
`Touchfile move coverage: ${gaps.length} gap(s):`,
|
||||
...gaps.map(gap => ` ${gap}`),
|
||||
'Rule: code moved out of a file keeps that file\'s paid-eval selection, so an edit to the new module still runs the evals the old file ran.',
|
||||
'Fix: add the new module (or a directory glob such as \'test/helpers/pty/**\') to every touchfile entry that lists its source file in test/helpers/touchfiles-data.ts (E2E, LLM judge, or GLOBAL_TOUCHFILES).',
|
||||
`Golden: ${GOLDEN_DIR}/${file}; re-record a source's selection only for a deliberate, reviewed touchfile change.`,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
const goldens = fs.readdirSync(path.join(ROOT, GOLDEN_DIR)).filter(file => file.endsWith('.json')).sort();
|
||||
|
||||
describe('touchfile coverage for moved code', () => {
|
||||
test('at least one golden is recorded', () => {
|
||||
expect(goldens.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test.each(goldens)('%s: every moved module exists and selects a superset of its sources', file => {
|
||||
const golden = JSON.parse(fs.readFileSync(path.join(ROOT, GOLDEN_DIR, file), 'utf8')) as Golden;
|
||||
for (const key of Object.keys(golden.modules)) {
|
||||
const modules = expandModule(key);
|
||||
expect(modules.length, `${key} in ${GOLDEN_DIR}/${file} names no files`).toBeGreaterThan(0);
|
||||
for (const module of modules) expect(fs.existsSync(path.join(ROOT, module)), `${module} is listed in ${GOLDEN_DIR}/${file} but does not exist`).toBe(true);
|
||||
}
|
||||
const gaps = selectionGaps(golden, MAPS, GLOBAL_TOUCHFILES);
|
||||
if (gaps.length) throw new Error(formatGaps(gaps, file));
|
||||
});
|
||||
|
||||
test('a planted module missing one source entry, or a dropped global source, is reported with the fix', () => {
|
||||
const golden: Golden = {
|
||||
workstream: 'planted', recordedAt: 'test',
|
||||
sources: {
|
||||
'src/old.ts': { e2e: ['eval-a', 'eval-b'], llmJudge: [] },
|
||||
'src/global.ts': { e2e: 'global', llmJudge: [] },
|
||||
},
|
||||
modules: { 'src/new/': ['src/old.ts'], 'src/moved-global.ts': ['src/global.ts'] },
|
||||
};
|
||||
const maps: Maps = { e2e: { 'eval-a': ['src/old.ts', 'src/new/**'], 'eval-b': ['src/old.ts'] }, llmJudge: {} };
|
||||
const gaps = selectionGaps(golden, maps, ['src/global.ts'], key => key.endsWith('/') ? ['src/new/module.ts'] : [key]);
|
||||
expect(gaps).toEqual([
|
||||
'src/new/module.ts e2e misses eval-b (selected by src/old.ts before the move)',
|
||||
'src/moved-global.ts e2e misses (every eval: source was a global touchfile) (selected by src/global.ts before the move)',
|
||||
]);
|
||||
expect(formatGaps(gaps, 'planted.json')).toContain('Fix: add the new module');
|
||||
expect(formatGaps(gaps, 'planted.json')).toContain(`Golden: ${GOLDEN_DIR}/planted.json`);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user