test(evals): harden no-op asserts, close tier-invariant fail-open holes, pin gate question strings

- no-op regression: gate-must-ask is now UNCONDITIONAL for eng/design (the
  outcome==='asked' conditional let a silent-bypass plan_ready run sail
  through); eng/design cases force --disallowedTools so the pinned prose
  shape is contractual rather than hoping native AUQ renders match; the
  named-target case uses trackTokens for consumption and lists
  wrote_findings_before_asking in its diagnostic throw branch.
- tier-alignment invariant: both quote styles matched; zero-self-gate,
  mixed-tier, and owning-keys-without-E2E_TIERS-entries are all REPORTED
  instead of silently skipped (the fail-open holes three reviewers found).
- drift-guard: the generated gate menus must carry the exact question/option
  strings the PTY question detector anchors on — free CI fails before the
  paid smokes can go vacuous on a menu reword.
- touchfiles: corrected the no-op cost note for CI concurrency + retry
  semantics.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-12 09:09:48 -07:00
co-authored by Claude Fable 5
parent 4e61233021
commit 3ddbee5952
4 changed files with 76 additions and 32 deletions
+13 -1
View File
@@ -3256,7 +3256,7 @@ describe('scope-gate exceptions drift-guard', () => {
// instead of drifting. The real fix (shared {{SCOPE_GATE}} resolver) is a
// filed TODO — this guard is the stopgap that makes the duplication safe.
const START_MARKER = '**Exceptions — check in this order, BEFORE asking:**';
const END_MARKER = 'nothing changes: this gate is a hard STOP.';
const END_MARKER = 'in any mode — it is a hard STOP.';
function extractExceptionsBlock(skill: string): string {
const md = fs.readFileSync(path.join(ROOT, skill, 'SKILL.md'), 'utf-8');
@@ -3290,6 +3290,18 @@ describe('scope-gate exceptions drift-guard', () => {
);
}
});
test('gate menu carries the question strings the PTY question detector pins', () => {
// isScopeGateQuestionVisible (claude-pty-runner.ts) anchors on the
// question text + option A's body. If the menu is reworded without
// updating the detector, the paid smokes' must-stay-false assertions go
// vacuous — this free pin fails first.
for (const skill of ['plan-eng-review', 'plan-design-review']) {
const md = fs.readFileSync(path.join(ROOT, skill, 'SKILL.md'), 'utf-8');
expect(md, `${skill}: gate question text`).toContain('What should I review?');
expect(md, `${skill}: option A body text`).toContain('The current branch diff');
}
});
});
describe('GSTACK REVIEW REPORT mandatory unresolved-decisions status', () => {