refactor(test): mechanical sweep — 298 paid-test timeouts onto eval-budget tiers

69 files, both shapes (trailing bun-test budgets and runner
timeout/timeoutMs options), ROUND-UP ONLY so nothing that passed can
start failing: 75 → JUDGE_MS, 137 → CAPTURE_MS, 74 → CAPTURE_LONG_MS,
9 → PTY_MS, 3 → PTY_LONG_MS. Raw >=60s literal count in the paid scope:
395 → 97, of which 51 are non-timeout noise (fixture dates, run IDs)
and 46 are enumerated justified holds (comment-carrying calibrated
budgets, poll-loop constants, utility spawn waits, and the seven
physical-ceiling 1_500_000 sites). The eval-budgets policy ratchet
keeps the residue from regrowing.

Known collapse: where an inner runner budget and its enclosing test
budget now share a tier, the old stagger is gone — an overrun surfaces
as a bun test timeout instead of a graceful runner timeout
(diagnosability trade, not a correctness one).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-29 05:25:06 +00:00
co-authored by Claude Fable 5
parent 74ae357e0f
commit 6841183c35
69 changed files with 367 additions and 298 deletions
+17 -16
View File
@@ -12,6 +12,7 @@
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId, evalsEnabled,
@@ -163,7 +164,7 @@ describeIfSelected('Context Skills E2E (live-fire)', [
env: { GSTACK_HOME: gstackHome },
maxTurns: 12,
allowedTools: ['Skill', 'Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'context-save-routing',
runId,
});
@@ -185,7 +186,7 @@ describeIfSelected('Context Skills E2E (live-fire)', [
expect(routedToContextSave).toBe(true);
expect(files.length).toBeGreaterThan(0);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 180_000);
}, CAPTURE_MS);
// ── 2. Round-trip: save then restore in the same session ─────────────
testConcurrentIfSelected('context-save-then-restore-roundtrip', async () => {
@@ -205,7 +206,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 25,
allowedTools: ['Skill', 'Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'context-save-then-restore-roundtrip',
runId,
});
@@ -232,7 +233,7 @@ Do NOT use AskUserQuestion.`,
expect(files.length).toBeGreaterThan(0);
expect(restoreMentionsTitle).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 240_000);
}, CAPTURE_MS);
// ── 3. /context-restore <fragment> loads the matching save ───────────
testConcurrentIfSelected('context-restore-fragment-match', async () => {
@@ -255,7 +256,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 10,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'context-restore-fragment-match',
runId,
});
@@ -279,7 +280,7 @@ Do NOT use AskUserQuestion.`,
expect(loadedPayments).toBe(true);
expect(didNotLoadOthers).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 180_000);
}, CAPTURE_MS);
// ── 4. /context-restore with zero saves → graceful empty-state ───────
testConcurrentIfSelected('context-restore-empty-state', async () => {
@@ -294,7 +295,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 8,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 90_000,
timeout: JUDGE_MS,
testName: 'context-restore-empty-state',
runId,
});
@@ -319,7 +320,7 @@ Do NOT use AskUserQuestion.`,
expect(routedToRestore).toBe(true);
expect(gracefulMessage).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 150_000);
}, CAPTURE_MS);
// ── 5. /context-restore list redirects to /context-save list ─────────
testConcurrentIfSelected('context-restore-list-delegates', async () => {
@@ -334,7 +335,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 8,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 90_000,
timeout: JUDGE_MS,
testName: 'context-restore-list-delegates',
runId,
});
@@ -357,7 +358,7 @@ Do NOT use AskUserQuestion.`,
expect(routedToRestore).toBe(true);
expect(mentionsSaveList).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 150_000);
}, CAPTURE_MS);
// ── 6. Legacy compat: pre-rename save files still load ───────────────
testConcurrentIfSelected('context-restore-legacy-compat', async () => {
@@ -381,7 +382,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 8,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'context-restore-legacy-compat',
runId,
});
@@ -414,7 +415,7 @@ Do NOT use AskUserQuestion.`,
expect(routedToRestore).toBe(true);
expect(loadedLegacy).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 180_000);
}, CAPTURE_MS);
// ── 7. /context-save list: default filters to current branch ─────────
testConcurrentIfSelected('context-save-list-current-branch', async () => {
@@ -437,7 +438,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 10,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'context-save-list-current-branch',
runId,
});
@@ -472,7 +473,7 @@ Do NOT use AskUserQuestion.`,
expect(hidesAlpha).toBe(true);
expect(hidesBeta).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 180_000);
}, CAPTURE_MS);
// ── 8. /context-save list --all: shows every branch ──────────────────
testConcurrentIfSelected('context-save-list-all-branches', async () => {
@@ -494,7 +495,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 10,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'context-save-list-all-branches',
runId,
});
@@ -520,5 +521,5 @@ Do NOT use AskUserQuestion.`,
expect(routed).toBe(true);
expect(filesShown).toBe(3);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 180_000);
}, CAPTURE_MS);
});