CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-09-01 02:40:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(evals): every E2E key's dep list names its own declaring test file

Browse Source 
129-of-177 keys omitted their own test file, so editing only a test's
prompt or assertions selected NOTHING — the changed test never ran on
the change that changed it. 135 keys self-registered (110 E2E + 25
LLM-judge), resolved by strict declaration evidence (testName:/
testIfSelected/judge call sites), with skill-name false positives
excluded.

e2e-tier-alignment's warn-only branch for unregistered files is now a
hard failure with a 4-entry KNOWN_UNREGISTERED ratchet (template-
literal testNames, fail-open-safe) + a burn-down test so the set only
shrinks. Selection sanity: a one-file diff on skill-e2e-qa-workflow now
selects its 4 tests (was 0); skill-llm-eval 0 → 25.

Known follow-ups (filed): 15 E2E + 2 judge PHANTOM keys select tests
that exist nowhere; codex-e2e-plan-format's testIfSelected names have
no map keys (run-all only).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-29 05:06:57 +00:00
co-authored by Claude Fable 5
parent 95b779f7cd
commit 2ec4dcf105
2 changed files with 203 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/e2e-tier-alignment.test.ts
+68 -9
View File
@@ -11,9 +11,15 @@
* Mapping rule (test filenames do NOT map mechanically to tier keys): for each
* `test/skill-e2e-*.test.ts` with an EVALS_TIER self-gate, search the
* E2E_TOUCHFILES / LLM_JUDGE_TOUCHFILES dep lists for the exact file path. If
* found under key K, the file's self-gate tier must equal E2E_TIERS[K]. Files
* not named in any dep list are REPORTED as unmapped (a nudge to add them to
* their eval's dep list), never silently skipped.
* found under key K, the file's self-gate tier must equal E2E_TIERS[K].
*
* Self-registration is a HARD invariant (the dep-list sweep): every
* skill-e2e file must be named in at least one touchfiles dep list, so that
* editing only the test's prompt/assertions diff-selects the test itself.
* Before the sweep, 129 of ~177 E2E keys did not list their own declaring
* file — a changed test never re-ran on its own change. Files that genuinely
* cannot be mapped (no E2E map key exists for them) sit in KNOWN_UNREGISTERED
* below; that set is a ratchet, it only shrinks.
*/
import { describe, test, expect } from 'bun:test';
@@ -35,6 +41,26 @@ const SELF_GATE_RE = /EVALS_TIER\s*===\s*['"](gate|periodic)['"]/g;
// the declared tier exactly like the raw predicate's tier literal did.
const HELPER_GATE_RE = /\b(?:describeE2ETier|e2eTierEnabled)\(\s*['"](gate|periodic)['"]/g;
/**
* Ratchet, not amnesty (same contract as KNOWN_MATRIX_GAPS in
* test/evals-workflow-matrix.test.ts): skill-e2e files that are named in NO
* touchfiles dep list because no E2E map key exists for them. Every entry
* carries a one-line reason. Do NOT add new files here — give the test an
* E2E map key (touchfiles + tier) and register the file in its dep list.
* A stale entry (file deleted, or file now registered) FAILS the suite —
* delete it. Target: empty set.
*/
const KNOWN_UNREGISTERED = new Set([
// Standalone periodic self-gated probe; template-literal testNames (auq-consistency-${i}), no E2E map key — fail-open-safe, runs on every periodic sweep.
'test/skill-e2e-auq-consistency.test.ts',
// Standalone periodic self-gated matrix; template-literal testNames (auq-matrix-${m.skill}), no E2E map key — fail-open-safe, runs on every periodic sweep.
'test/skill-e2e-auq-matrix.test.ts',
// Standalone periodic self-gated A/B probe; template-literal testNames (auq-ab-${label}), no E2E map key — fail-open-safe, runs on every periodic sweep.
'test/skill-e2e-auq-verbose-vs-carved-ab.test.ts',
// bin-script pipeline test (spawns bun scripts, no model spend) that lives under the skill-e2e-* glob; no E2E map key exists for it.
'test/skill-e2e-memory-pipeline.test.ts',
]);
describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () => {
const testFiles = readdirSync(TEST_DIR)
.filter((f) => f.startsWith('skill-e2e-') && f.endsWith('.test.ts'))
@@ -44,14 +70,28 @@ describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () =>
test('every self-gated test file named in a dep list matches its declared tier', () => {
const misaligned: string[] = [];
const unregistered: string[] = [];
const reported: string[] = [];
for (const file of testFiles) {
const content = readFileSync(path.join(TEST_DIR, file), 'utf-8');
const repoPath = `test/${file}`;
// HARD self-registration invariant, independent of self-gate shape:
// a skill-e2e file named in no dep list means editing the test itself
// selects nothing — the changed test never re-runs on its own change.
const owningKeys = Object.keys(allDeps).filter((k) => allDeps[k].includes(repoPath));
if (owningKeys.length === 0 && !KNOWN_UNREGISTERED.has(repoPath)) {
unregistered.push(
`${repoPath}: not named in any touchfiles dep list — editing this test file would `
+ 'never diff-select it. Add the file path to its E2E map key\'s dep list in '
+ 'test/helpers/touchfiles-data.ts (do NOT extend KNOWN_UNREGISTERED for new files).',
);
}
const tiers = new Set<string>();
for (const m of content.matchAll(SELF_GATE_RE)) tiers.add(m[1]);
for (const m of content.matchAll(HELPER_GATE_RE)) tiers.add(m[1]);
const repoPath = `test/${file}`;
if (tiers.size === 0) {
// Every skill-e2e file is expected to self-gate; zero matches means
// either a genuinely ungated file or a gate shape the regex can't
@@ -65,8 +105,9 @@ describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () =>
}
const selfTier = [...tiers][0];
const owningKeys = Object.keys(allDeps).filter((k) => allDeps[k].includes(repoPath));
if (owningKeys.length === 0) {
// Only KNOWN_UNREGISTERED files reach here (anything else already
// hard-failed above) — keep the visible nudge.
reported.push(`${repoPath} (self-gates '${selfTier}'): not named in any touchfiles dep list`);
continue;
}
@@ -86,18 +127,36 @@ describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () =>
}
}
// Reported, not asserted: coverage holes the invariant can see but not
// arbitrate. Add the test file to its eval's dep list (or a tier entry
// for the key) to bring it under the invariant.
// Reported, not asserted: tier-observability holes the invariant can see
// but not arbitrate (map-driven files legitimately have no whole-file
// self-gate; ratcheted files stay visible). Registration itself is
// asserted below.
if (reported.length > 0) {
console.warn(
`[tier-alignment] ${reported.length} file(s) outside the invariant:\n ` + reported.join('\n '),
`[tier-alignment] ${reported.length} file(s) outside the tier invariant:\n ` + reported.join('\n '),
);
}
expect(unregistered).toEqual([]);
expect(misaligned).toEqual([]);
});
// Ratchet cleanup enforcement (same contract as evals-workflow-matrix's
// burn-down test): a KNOWN_UNREGISTERED entry whose file was deleted, or
// whose file is now named in a dep list, is stale — delete the entry so
// the set can only shrink.
test('KNOWN_UNREGISTERED holds only live, still-unregistered files', () => {
const stale = [...KNOWN_UNREGISTERED].filter((repoPath) => {
const file = repoPath.replace(/^test\//, '');
if (!testFiles.includes(file)) return true; // file gone
return Object.keys(allDeps).some((k) => allDeps[k].includes(repoPath)); // now registered
});
expect(
stale,
'Entry registered in a dep list or file removed — delete it from KNOWN_UNREGISTERED.',
).toEqual([]);
});
// HARD invariant (C6): the paid sharded runner skips a skill-e2e shard when
// none of the file's MAPPED test names (E2E map keys quoted in its source,
// union E2E map keys whose dep list registers the file) are diff-selected.