test: derive paid touchfiles from each eval's static closure (E)

touchfiles.test.ts now checks, per key, that the paid file's static
test/helpers and test/fixtures closure (plus fixture paths it names in string
literals) is covered, and names the file, path, chain and key to fix when it is
not. Free *.test.ts files are no longer touchfiles, so editing a free replay
test stops selecting paid evals: 950 entries removed, 653 real closure paths
added. The hand-copied inventories go: periodic-fixture-selection,
fake-impeccable-touchfiles and 45 per-file selection examples. Selection for
the sample edits (plan-eng-review template, claude-pty-runner,
plan-count-fixture, gstack-config) loses no case under either profile.
CONTRIBUTING documents the rule and its lower bound.
This commit is contained in:
garrytan committed 2026-09-29 07:00:28 +00:00
1 parent 689ef3094b
commit 7cbb5e04e4
53 files changed
+627 -1873

No files matched your search

-8
View File
@@ -1,6 +1,5 @@
import { describe, expect, test } from 'bun:test';
import { ENG_REVIEW_EXCERPT, readWorkflowExcerpt } from './helpers/workflow-excerpt';
import { LLM_JUDGE_TOUCHFILES, selectTests } from './helpers/touchfiles';
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
@@ -54,13 +53,6 @@ function expectOutsideReviewControlFlow(text: string, promptHeading: string): vo
}
describe('workflow judge excerpts', () => {
test('helper changes select all dependent workflow judges', () => {
const selected = selectTests(['test/helpers/workflow-excerpt.ts'], LLM_JUDGE_TOUCHFILES, []).selected;
expect(selected).toHaveLength(14);
expect(selected).toContain('ship/SKILL.md workflow');
expect(selected).toContain('plan-design-review/SKILL.md passes');
});
test('expands ship sections in execution order, not alphabetical order', () => {
const text = readWorkflowExcerpt('ship/SKILL.md', '# Ship:', '## Important Rules');
const headings = ['## Step 3:', '## Step 4:', '## Step 7:', '## Step 8:', '## Step 9:', '## Step 10:', '## Step 11:', '## Step 12:', '## Step 13:', '## Step 14:'];