CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-09-09 22:48:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

refactor(evals): single shared paid-test-set module

Browse Source 
test/helpers/paid-test-set.ts is now the one definition of which test
files are paid (the exact globs package.json's test:gate expands).
scripts/test-free-shards.ts derives its free/paid exclusion from it
instead of a private regex list, dropping the dead
browse/test/security-review-fullstack.test.ts pattern (file no longer
exists). The sharded paid runner derives its enumeration from the same
module, so a file added to one list can no longer silently miss the
other.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
(cherry picked from commit a7f36479a6a1f3656452370f5883371f3cb65623)
This commit is contained in:
Garry Tan
2026-08-12 15:31:47 -07:00
parent 281227262d
commit ea1c2851e5
2 changed files with 27 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/test-free-shards.ts
+2 -12
View File
@@ -27,22 +27,12 @@
import * as fs from 'fs';
import * as path from 'path';
import { spawnSync } from 'child_process';
import { isPaidTestFile } from '../test/helpers/paid-test-set';
const ROOT = path.resolve(import.meta.dir, '..');
const TEST_ROOTS = ['browse/test', 'test', 'make-pdf/test'] as const;
const TEST_FILE_REGEX = /\.test\.(?:[cm]?[jt]s|tsx|jsx)$/;
// Tests that require API spend, external services, or e2e harnesses.
// These are filtered out before any sharding or curation.
const PAID_EVAL_TESTS = [
/^browse\/test\/security-review-fullstack\.test\.ts$/,
/^test\/skill-e2e-.*\.test\.ts$/,
/^test\/skill-llm-eval\.test\.ts$/,
/^test\/skill-routing-e2e\.test\.ts$/,
/^test\/codex-e2e\.test\.ts$/,
/^test\/gemini-e2e\.test\.ts$/,
] as const;
// POSIX-only patterns that indicate a test will fail on windows-latest no
// matter how the runner shards. Codex's v1.18.0.0 review flagged the first
// three as concrete examples in the existing free suite (test/ship-version-sync.test.ts:72,
@@ -118,7 +108,7 @@ export function normalizeRelativePath(filePath: string): string {
export function isFreeTestFile(relativePath: string): boolean {
const normalized = normalizeRelativePath(relativePath);
if (!TEST_FILE_REGEX.test(normalized)) return false;
return !PAID_EVAL_TESTS.some(pattern => pattern.test(normalized));
return !isPaidTestFile(normalized);
}
/**