CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-09-12 07:59:02 +02:00
Code Issues Packages Projects Releases Wiki Activity

refactor(evals): arm-benchmark selftest runs FREE on every PR

Browse Source 
The selftest lived inside the paid skill-e2e-* file, so fixture-integrity
and plumbing pins executed weekly at best — a broken fixture would ship past
every gating check and be discovered when the periodic run burned money on a
dead instrument. Harness extracted to test/helpers/arm-benchmark-harness.ts,
selftest to test/arm-benchmark-selftest.test.ts (free suite). Touchfiles:
harness added to the three benchmark dep lists; the auq-repetition-cut-ab
tier comment now states the MANUAL re-run obligation honestly (periodic runs
force EVALS_ALL, so dep lists cannot auto-trigger it).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-29 05:42:57 +00:00
co-authored by Claude Fable 5
parent 760554d0ac
commit 2ee61ab455
4 changed files with 445 additions and 407 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/helpers/touchfiles-data.ts
+4 -1
View File
@@ -454,6 +454,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
'scripts/resolvers/preamble/generate-voice-directive.ts',
'test/fixtures/arm-benchmark/**',
'test/helpers/llm-judge.ts',
'test/helpers/arm-benchmark-harness.ts',
'test/skill-e2e-arm-benchmark.test.ts',
'ship/SKILL.md',
],
@@ -462,6 +463,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
'scripts/resolvers/preamble/generate-voice-directive.ts',
'test/fixtures/arm-benchmark/**',
'test/helpers/llm-judge.ts',
'test/helpers/arm-benchmark-harness.ts',
'test/skill-e2e-arm-benchmark.test.ts',
'ship/SKILL.md',
],
@@ -470,6 +472,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
'scripts/resolvers/preamble/generate-voice-directive.ts',
'test/fixtures/arm-benchmark/**',
'test/helpers/llm-judge.ts',
'test/helpers/arm-benchmark-harness.ts',
'test/skill-e2e-arm-benchmark.test.ts',
'ship/SKILL.md',
],
@@ -581,7 +584,7 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
// gate: cheap, deterministic, run on every PR
// periodic: long-running or expensive (>$3/run), run weekly
'preamble-script-ab': 'periodic', // Phase 1-3 A/B: script vs inline preamble; demoted post-Phase-3 (OV7)
'auq-repetition-cut-ab': 'periodic', // AUQ repetition-cut NOT-WORSE gate (passed pre-landing; re-runs on AUQ format changes)
'auq-repetition-cut-ab': 'periodic', // AUQ repetition-cut NOT-WORSE gate (passed pre-landing). Periodic runs force EVALS_ALL, so the dep list cannot auto-trigger it — an AUQ format edit carries a MANUAL re-run obligation (bun test test/skill-e2e-auq-repetition-cut-ab.test.ts with EVALS=1 EVALS_TIER=periodic)
'auq-format-gate': 'gate', // ~$0.50/run, SDK capture, single skill probe
'plan-ceo-mode-routing': 'periodic', // ~$3/run, deep navigation through 8-12 prior AskUserQuestions
'plan-design-with-ui-scope': 'gate', // ~$0.80/run