mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-13 08:29:04 +02:00
Garry TanandClaude Fable 5
07c2452e6d
Every PR paid twice: the hand-enumerated matrix (18 test files, 22.6 min, ~$21 API measured on run 33263204465) ran serialized AHEAD of the strictly superior sliced lane via 'needs: evals' — 35.5 min wall and ~2x paid spend for the same diff. 14 of 17 rows carried no tier:, so periodic Opus benchmarks leaked into every PR (the e2e-plan row alone: 12/12 tests, 21.7 min, $7.28 — the wall-clock bound of ALL of CI). Parity receipt (static, pre-deletion): the sliced lane's gate census (49 files, derived from the runner itself) strictly contains all 18 matrix test files, plus 31 files the matrix never ran. Pure deletion — one revert restores it. The PR comment moved into slices-report (same '## E2E Evals' upsert marker, now sourced from slice artifacts + carrying the fail-closed reconciliation verdict). plan-slices loses the needs edge; the dead workflow-level EVALS_TIER env goes with it. test/evals-workflow-matrix.test.ts (and its KNOWN_MATRIX_GAPS / KNOWN_TIER_UNSET burn-down ratchets — retired: the sliced census makes 'every gate file runs' true by construction) is rewritten as test/evals-workflow-wiring.test.ts: matrix stays deleted, planner/executor/ report tier + slice-count agreement, both surviving lanes on the shared register-skills composite with its fail-fast verification loop, PR comment survival. Expected: PR eval wall 35.5 -> ~13 min, per-PR paid spend ~halved. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
82 lines
3.5 KiB
TypeScript
82 lines
3.5 KiB
TypeScript
/**
|
|
* No paid-gated test file may sit outside PAID_TEST_GLOBS.
|
|
*
|
|
* The orphan class this kills (found 2026-08): a file whose source gates on
|
|
* EVALS/tier (so the free suite loads it as describe.skip) but whose NAME
|
|
* doesn't match the paid globs (so no paid lane ever selects it) can never
|
|
* execute anywhere — forever, silently. Four files were in that state
|
|
* (codex-e2e-plan-format, codex-e2e-recommendation-substance,
|
|
* llm-judge-recommendation, carve-section-loading), and the tripwire built
|
|
* for the adjacent class (the since-retired evals-workflow-matrix test;
|
|
* successor: test/evals-workflow-wiring.test.ts) couldn't see them because
|
|
* it filtered on isPaidTestFile() FIRST.
|
|
*
|
|
* Detection is over source text, so meta-tests and helpers that mention the
|
|
* gate patterns need reasoned exemptions (same convention as
|
|
* test/egress-receipt-wiring.test.ts's SCANNER_EXEMPT).
|
|
*/
|
|
import { describe, expect, test } from 'bun:test';
|
|
import { spawnSync } from 'node:child_process';
|
|
import * as fs from 'node:fs';
|
|
import * as path from 'node:path';
|
|
|
|
import { isPaidTestFile } from './helpers/paid-test-set';
|
|
|
|
const ROOT = path.resolve(__dirname, '..');
|
|
|
|
/** Files that legitimately mention gate patterns without being paid tests. */
|
|
const SCANNER_EXEMPT = new Map<string, string>([
|
|
// The gate helpers themselves and their free unit tests:
|
|
['test/helpers/e2e-gate.ts', 'defines the gate predicates'],
|
|
// Meta-tests that quote gate-pattern strings to test classification:
|
|
['test/helpers/e2e-gate.unit.test.ts', 'free unit test OF the gate predicates (env stubbed)'],
|
|
['test/paid-shards.test.ts', 'quotes tier-guard strings as classification fixtures'],
|
|
['test/evals-workflow-wiring.test.ts', 'pins the sliced-lane yml wiring (successor to the matrix test)'],
|
|
['test/e2e-tier-alignment.test.ts', 'parses tier guards to enforce alignment'],
|
|
['test/paid-orphan-tripwire.test.ts', 'this scanner'],
|
|
]);
|
|
|
|
/**
|
|
* Source shapes that mean "this file self-gates on the paid env":
|
|
* the shared helpers, or a direct EVALS/EVALS_TIER env read.
|
|
*/
|
|
const GATE_PATTERNS = [
|
|
/\bdescribeE2ETier\s*\(/,
|
|
/\be2eTierEnabled\s*\(/,
|
|
/process\.env\.EVALS\b/,
|
|
];
|
|
|
|
function trackedTestFiles(): string[] {
|
|
const out = spawnSync('git', ['ls-files', '*.test.ts'], { cwd: ROOT, encoding: 'utf-8' });
|
|
if (out.status !== 0) throw new Error(`git ls-files failed: ${out.stderr}`);
|
|
return out.stdout.split('\n').filter(Boolean);
|
|
}
|
|
|
|
describe('paid orphan tripwire', () => {
|
|
test('every EVALS/tier-gated test file is inside PAID_TEST_GLOBS (or exempt with a reason)', () => {
|
|
const files = trackedTestFiles();
|
|
expect(files.length).toBeGreaterThan(100); // scan-rot guard
|
|
|
|
const orphans: string[] = [];
|
|
for (const rel of files) {
|
|
if (isPaidTestFile(rel)) continue;
|
|
if (SCANNER_EXEMPT.has(rel)) continue;
|
|
const source = fs.readFileSync(path.join(ROOT, rel), 'utf-8');
|
|
const hit = GATE_PATTERNS.find((p) => p.test(source));
|
|
if (hit) orphans.push(`${rel} (matches ${hit})`);
|
|
}
|
|
expect(orphans,
|
|
'paid-gated test files OUTSIDE the paid globs can never run in any lane. '
|
|
+ 'Fix: extend PAID_TEST_GLOBS in test/helpers/paid-test-set.ts (and mirror '
|
|
+ 'package.json), or add a reasoned SCANNER_EXEMPT entry if the file only '
|
|
+ `mentions the patterns:\n${orphans.join('\n')}`,
|
|
).toEqual([]);
|
|
});
|
|
|
|
test('exemption entries stay real (stale entries must be deleted)', () => {
|
|
for (const [rel] of SCANNER_EXEMPT) {
|
|
expect(fs.existsSync(path.join(ROOT, rel)), `stale SCANNER_EXEMPT entry: ${rel}`).toBe(true);
|
|
}
|
|
});
|
|
});
|