CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-09-01 02:40:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(evals): activate the 4 paid test files that could never run anywhere

Browse Source 
carve-section-loading, codex-e2e-plan-format,
codex-e2e-recommendation-substance, and llm-judge-recommendation gated
on EVALS/tier (free suite loads them as describe.skip) but their names
fell outside PAID_TEST_GLOBS, so no paid lane ever selected them — net
execution zero, forever. The existing matrix tripwire filtered on
isPaidTestFile() first, so it was blind to exactly this class (the same
bug that hid the pre-split monolith's gate tests for ~8 releases).

- PAID_TEST_GLOBS: codex-e2e* + skill-llm-eval* wildcards (replacing
  exact names) + llm-judge-recommendation + carve-section-loading;
  package.json's six test-script glob lists mirrored
- codex-e2e-plan-format gains the explicit periodic tier gate its
  siblings carry (external-service rule) — without it the sharded
  runner's no-guard default would spawn Codex in the gate tier per PR
- eval:bg:periodic --timeout 32400→37800: the census growth pushed the
  periodic worst case to 35910s; the old value had 270s of headroom
  BEFORE this change and would now kill healthy runs mid-flight
- new test/paid-orphan-tripwire.test.ts: any EVALS/tier-gated test file
  outside the globs fails the free suite (reasoned SCANNER_EXEMPT for
  the gate helpers + meta-tests) — the class-killer
- paid-shards pins updated: the four orphans now assert INSIDE the
  census

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-29 04:52:47 +00:00
co-authored by Claude Fable 5
parent 442a46f989
commit 72a5246aae
5 changed files with 113 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package.json
+7 -7
View File
@@ -22,12 +22,12 @@
"test": "bun run scripts/test-free-shards.ts && (bun run slop:diff 2>/dev/null || true)",
"test:free": "bun run scripts/test-free-shards.ts",
"test:windows": "bun run scripts/test-free-shards.ts --windows-only",
"test:evals": "EVALS=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/codex-e2e-sol-scope.test.ts test/gemini-e2e.test.ts",
"test:evals:all": "EVALS=1 EVALS_ALL=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/codex-e2e-sol-scope.test.ts test/gemini-e2e.test.ts",
"test:e2e": "EVALS=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/codex-e2e-sol-scope.test.ts test/gemini-e2e.test.ts",
"test:e2e:all": "EVALS=1 EVALS_ALL=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/codex-e2e-sol-scope.test.ts test/gemini-e2e.test.ts",
"test:gate": "EVALS=1 EVALS_TIER=gate bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/codex-e2e-sol-scope.test.ts test/gemini-e2e.test.ts",
"test:periodic": "EVALS=1 EVALS_TIER=periodic EVALS_ALL=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/codex-e2e-sol-scope.test.ts test/gemini-e2e.test.ts",
"test:evals": "EVALS=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval*.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e*.test.ts test/gemini-e2e.test.ts test/llm-judge-recommendation.test.ts test/carve-section-loading.test.ts",
"test:evals:all": "EVALS=1 EVALS_ALL=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval*.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e*.test.ts test/gemini-e2e.test.ts test/llm-judge-recommendation.test.ts test/carve-section-loading.test.ts",
"test:e2e": "EVALS=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e*.test.ts test/gemini-e2e.test.ts test/carve-section-loading.test.ts",
"test:e2e:all": "EVALS=1 EVALS_ALL=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e*.test.ts test/gemini-e2e.test.ts test/carve-section-loading.test.ts",
"test:gate": "EVALS=1 EVALS_TIER=gate bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval*.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e*.test.ts test/gemini-e2e.test.ts test/llm-judge-recommendation.test.ts test/carve-section-loading.test.ts",
"test:periodic": "EVALS=1 EVALS_TIER=periodic EVALS_ALL=1 bun test --retry 1 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval*.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e*.test.ts test/gemini-e2e.test.ts test/llm-judge-recommendation.test.ts test/carve-section-loading.test.ts",
"test:gate:sharded": "bun run scripts/test-paid-shards.ts --tier gate",
"test:periodic:sharded": "EVALS_ALL=1 bun run scripts/test-paid-shards.ts --tier periodic",
"test:codex": "EVALS=1 bun test test/codex-e2e.test.ts test/codex-e2e-sol-scope.test.ts",
@@ -40,7 +40,7 @@
"eval:bg": "bin/gstack-detach --label evals --lock gstack-evals --timeout 5400 -- bun run test:evals",
"eval:bg:all": "bin/gstack-detach --label evals-all --lock gstack-evals --timeout 7200 -- bun run test:evals:all",
"eval:bg:gate": "bin/gstack-detach --label evals-gate --lock gstack-evals --timeout 25200 -- bun run test:gate:sharded",
"eval:bg:periodic": "bin/gstack-detach --label evals-periodic --lock gstack-evals --timeout 32400 -- bun run test:periodic:sharded",
"eval:bg:periodic": "bin/gstack-detach --label evals-periodic --lock gstack-evals --timeout 37800 -- bun run test:periodic:sharded",
"eval:list": "bun run scripts/eval-list.ts",
"eval:compare": "bun run scripts/eval-compare.ts",
"eval:summary": "bun run scripts/eval-summary.ts",