CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-05-10 23:14:56 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: granular touchfiles + 2-tier E2E test system (gate/periodic)

Browse Source 
- Shrink GLOBAL_TOUCHFILES from 9 to 3 (only truly global deps)
- Move scoped deps (gen-skill-docs, llm-judge, test-server, worktree,
  codex/gemini session runners) into individual test entries
- Add E2E_TIERS map classifying each test as gate or periodic
- Replace EVALS_FAST with EVALS_TIER env var (gate/periodic)
- Add tier validation test (E2E_TIERS keys must match E2E_TOUCHFILES)
- CI runs only gate tests; periodic tests run weekly via cron
- Add evals-periodic.yml workflow (Monday 6 AM UTC + manual)
- Remove allow_failure flags (gate tests should be reliable)
- Add test:gate and test:periodic scripts, remove test:e2e:fast
This commit is contained in:
Garry Tan
2026-03-24 08:15:53 -07:00
parent 3501f5dd03
commit c4aa31c988
7 changed files with 340 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/touchfiles.test.ts
+44 -4
View File
@@ -13,6 +13,7 @@ import {
selectTests,
detectBaseBranch,
E2E_TOUCHFILES,
E2E_TIERS,
LLM_JUDGE_TOUCHFILES,
GLOBAL_TOUCHFILES,
} from './helpers/touchfiles';
@@ -91,10 +92,19 @@ describe('selectTests', () => {
expect(result.reason).toContain('global');
});
test('gen-skill-docs.ts is a global touchfile', () => {
test('gen-skill-docs.ts is a scoped touchfile, not global', () => {
const result = selectTests(['scripts/gen-skill-docs.ts'], E2E_TOUCHFILES);
expect(result.selected.length).toBe(Object.keys(E2E_TOUCHFILES).length);
expect(result.reason).toContain('global');
// Should select tests that list gen-skill-docs.ts in their touchfiles, not ALL tests
expect(result.selected.length).toBeGreaterThan(0);
expect(result.selected.length).toBeLessThan(Object.keys(E2E_TOUCHFILES).length);
expect(result.reason).toBe('diff');
// Should include tests that depend on gen-skill-docs.ts
expect(result.selected).toContain('skillmd-setup-discovery');
expect(result.selected).toContain('contributor-mode');
expect(result.selected).toContain('journey-ideation');
// Should NOT include tests that don't depend on it
expect(result.selected).not.toContain('retro');
expect(result.selected).not.toContain('cso-full-audit');
});
test('unrelated file selects nothing', () => {
@@ -143,7 +153,7 @@ describe('selectTests', () => {
});
test('global touchfiles work for LLM-judge tests too', () => {
const result = selectTests(['scripts/gen-skill-docs.ts'], LLM_JUDGE_TOUCHFILES);
const result = selectTests(['test/helpers/session-runner.ts'], LLM_JUDGE_TOUCHFILES);
expect(result.selected.length).toBe(Object.keys(LLM_JUDGE_TOUCHFILES).length);
});
});
@@ -233,6 +243,36 @@ describe('TOUCHFILES completeness', () => {
}
});
test('E2E_TIERS covers exactly the same tests as E2E_TOUCHFILES', () => {
const touchfileKeys = new Set(Object.keys(E2E_TOUCHFILES));
const tierKeys = new Set(Object.keys(E2E_TIERS));
const missingFromTiers = [...touchfileKeys].filter(k => !tierKeys.has(k));
const extraInTiers = [...tierKeys].filter(k => !touchfileKeys.has(k));
if (missingFromTiers.length > 0) {
throw new Error(
`E2E tests missing TIER entries: ${missingFromTiers.join(', ')}\n` +
`Add these to E2E_TIERS in test/helpers/touchfiles.ts`,
);
}
if (extraInTiers.length > 0) {
throw new Error(
`E2E_TIERS has extra entries not in E2E_TOUCHFILES: ${extraInTiers.join(', ')}\n` +
`Remove these from E2E_TIERS or add to E2E_TOUCHFILES`,
);
}
});
test('E2E_TIERS only contains valid tier values', () => {
const validTiers = ['gate', 'periodic'];
for (const [name, tier] of Object.entries(E2E_TIERS)) {
if (!validTiers.includes(tier)) {
throw new Error(`E2E_TIERS['${name}'] has invalid tier '${tier}'. Valid: ${validTiers.join(', ')}`);
}
}
});
test('every LLM-judge test has a TOUCHFILES entry', () => {
const llmContent = fs.readFileSync(
path.join(ROOT, 'test', 'skill-llm-eval.test.ts'),