CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-09-12 07:59:02 +02:00
Code Issues Packages Projects Releases Wiki Activity

test(evals): harden no-op asserts, close tier-invariant fail-open holes, pin gate question strings

Browse Source 
- no-op regression: gate-must-ask is now UNCONDITIONAL for eng/design (the
  outcome==='asked' conditional let a silent-bypass plan_ready run sail
  through); eng/design cases force --disallowedTools so the pinned prose
  shape is contractual rather than hoping native AUQ renders match; the
  named-target case uses trackTokens for consumption and lists
  wrote_findings_before_asking in its diagnostic throw branch.
- tier-alignment invariant: both quote styles matched; zero-self-gate,
  mixed-tier, and owning-keys-without-E2E_TIERS-entries are all REPORTED
  instead of silently skipped (the fail-open holes three reviewers found).
- drift-guard: the generated gate menus must carry the exact question/option
  strings the PTY question detector anchors on — free CI fails before the
  paid smokes can go vacuous on a menu reword.
- touchfiles: corrected the no-op cost note for CI concurrency + retry
  semantics.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-12 09:09:48 -07:00
co-authored by Claude Fable 5
parent 4e61233021
commit 3ddbee5952
4 changed files with 76 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/e2e-tier-alignment.test.ts
+30 -12
View File
@@ -22,7 +22,10 @@ import * as path from 'path';
import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from './helpers/touchfiles';
const TEST_DIR = import.meta.dir;
const SELF_GATE_RE = /EVALS_TIER\s*===\s*'(gate|periodic)'/g;
// Both quote styles — a mechanical refactor to double quotes must not
// silently drop a file from the invariant (fail-open is the defect class
// this test exists to kill).
const SELF_GATE_RE = /EVALS_TIER\s*===\s*['"](gate|periodic)['"]/g;
describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () => {
const testFiles = readdirSync(TEST_DIR)
@@ -33,24 +36,40 @@ describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () =>
test('every self-gated test file named in a dep list matches its declared tier', () => {
const misaligned: string[] = [];
const unmapped: string[] = [];
const reported: string[] = [];
for (const file of testFiles) {
const content = readFileSync(path.join(TEST_DIR, file), 'utf-8');
const tiers = new Set<string>();
for (const m of content.matchAll(SELF_GATE_RE)) tiers.add(m[1]);
if (tiers.size !== 1) continue; // no self-gate, or mixed-tier file — out of scope
const repoPath = `test/${file}`;
if (tiers.size === 0) {
// Every skill-e2e file is expected to self-gate; zero matches means
// either a genuinely ungated file or a gate shape the regex can't
// see — both worth a visible report, never a silent skip.
reported.push(`${repoPath}: no detectable EVALS_TIER self-gate`);
continue;
}
if (tiers.size > 1) {
reported.push(`${repoPath}: mixed-tier self-gates (${[...tiers].join(', ')}) — not tier-checked`);
continue;
}
const selfTier = [...tiers][0];
const repoPath = `test/${file}`;
const owningKeys = Object.keys(allDeps).filter((k) => allDeps[k].includes(repoPath));
if (owningKeys.length === 0) {
unmapped.push(`${repoPath} (self-gates '${selfTier}')`);
reported.push(`${repoPath} (self-gates '${selfTier}'): not named in any touchfiles dep list`);
continue;
}
for (const k of owningKeys) {
const declared = E2E_TIERS[k];
if (declared && declared !== selfTier) {
if (!declared) {
// A dep-list key with no E2E_TIERS entry (e.g. an LLM-judge key)
// can't tier-check this file — report instead of silently passing.
reported.push(`${repoPath}: matched key '${k}' which has no E2E_TIERS entry`);
continue;
}
if (declared !== selfTier) {
misaligned.push(
`${repoPath}: self-gates on '${selfTier}' but E2E_TIERS['${k}'] declares '${declared}' — the declaration is inert`,
);
@@ -58,13 +77,12 @@ describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () =>
}
}
// Reported, not asserted: these files' evals can't be tier-checked until
// their dep lists name the test file. Add `test/<file>` to the eval's
// touchfiles entry to bring them under the invariant.
if (unmapped.length > 0) {
// Reported, not asserted: coverage holes the invariant can see but not
// arbitrate. Add the test file to its eval's dep list (or a tier entry
// for the key) to bring it under the invariant.
if (reported.length > 0) {
console.warn(
`[tier-alignment] ${unmapped.length} self-gated test file(s) not named in any touchfiles dep list:\n ` +
unmapped.join('\n '),
`[tier-alignment] ${reported.length} file(s) outside the invariant:\n ` + reported.join('\n '),
);
}