CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-08-31 18:30:39 +02:00
Code Issues Packages Projects Releases Wiki Activity

refactor(test): mechanical sweep — 298 paid-test timeouts onto eval-budget tiers

Browse Source 
69 files, both shapes (trailing bun-test budgets and runner
timeout/timeoutMs options), ROUND-UP ONLY so nothing that passed can
start failing: 75 → JUDGE_MS, 137 → CAPTURE_MS, 74 → CAPTURE_LONG_MS,
9 → PTY_MS, 3 → PTY_LONG_MS. Raw >=60s literal count in the paid scope:
395 → 97, of which 51 are non-timeout noise (fixture dates, run IDs)
and 46 are enumerated justified holds (comment-carrying calibrated
budgets, poll-loop constants, utility spawn waits, and the seven
physical-ceiling 1_500_000 sites). The eval-budgets policy ratchet
keeps the residue from regrowing.

Known collapse: where an inner runner budget and its enclosing test
budget now share a tier, the old stagger is gone — an overrun surfaces
as a bun test timeout instead of a graceful runner timeout
(diagnosability trade, not a correctness one).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-29 05:25:06 +00:00
co-authored by Claude Fable 5
parent 74ae357e0f
commit 6841183c35
69 changed files with 367 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/skill-e2e-plan-prosons.test.ts
+9 -8
View File
@@ -27,6 +27,7 @@
* cases will land as follow-up PRs per skill.
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId,
@@ -161,7 +162,7 @@ ${captureInstruction(outFile)}
After writing the file, stop.`,
workingDirectory: planDir,
maxTurns: 10,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'plan-review-prosons-format',
runId,
model: 'claude-opus-4-7',
@@ -191,7 +192,7 @@ After writing the file, stop.`,
// (recommended) label on one option
expect(captured).toMatch(RECOMMENDED_LABEL_RE);
}, 300_000);
}, CAPTURE_MS);
});
// --- Case 2: Hard-stop escape NEGATIVE (CT2) ---
@@ -220,7 +221,7 @@ ${captureInstruction(outFile)}
After writing the file, stop.`,
workingDirectory: planDir,
maxTurns: 10,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'plan-review-prosons-hardstop-neg',
runId,
model: 'claude-opus-4-7',
@@ -241,7 +242,7 @@ After writing the file, stop.`,
// Must have real pros and cons (≥2 ✅ + ≥1 ❌ per option)
expect(countChars(captured, '✅')).toBeGreaterThanOrEqual(4);
expect(countChars(captured, '❌')).toBeGreaterThanOrEqual(2);
}, 300_000);
}, CAPTURE_MS);
});
// --- Case 3: Neutral-posture NEGATIVE (CT2) ---
@@ -270,7 +271,7 @@ ${captureInstruction(outFile)}
After writing the file, stop.`,
workingDirectory: planDir,
maxTurns: 10,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'plan-review-prosons-neutral-neg',
runId,
model: 'claude-opus-4-7',
@@ -292,7 +293,7 @@ After writing the file, stop.`,
expect(captured).toMatch(RECOMMENDED_LABEL_RE);
// Recommendation line must contain "because" (concrete reason, not "no preference")
expect(captured).toMatch(/[Rr]ecommendation:.*because/);
}, 300_000);
}, CAPTURE_MS);
});
// --- Case 4: Hard-stop POSITIVE (escape allowed when legitimately one-sided) ---
@@ -321,7 +322,7 @@ ${captureInstruction(outFile)}
After writing the file, stop.`,
workingDirectory: planDir,
maxTurns: 10,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'plan-ceo-review-prosons-cadence',
runId,
model: 'claude-opus-4-7',
@@ -344,7 +345,7 @@ After writing the file, stop.`,
const hasEscape = HARD_STOP_ESCAPE_RE.test(captured);
const hasProsAndCons = countChars(captured, '✅') >= 1 && countChars(captured, '❌') >= 1;
expect(hasEscape || hasProsAndCons).toBe(true);
}, 300_000);
}, CAPTURE_MS);
});
afterAll(async () => {