refactor(test): mechanical sweep — 298 paid-test timeouts onto eval-budget tiers

69 files, both shapes (trailing bun-test budgets and runner
timeout/timeoutMs options), ROUND-UP ONLY so nothing that passed can
start failing: 75 → JUDGE_MS, 137 → CAPTURE_MS, 74 → CAPTURE_LONG_MS,
9 → PTY_MS, 3 → PTY_LONG_MS. Raw >=60s literal count in the paid scope:
395 → 97, of which 51 are non-timeout noise (fixture dates, run IDs)
and 46 are enumerated justified holds (comment-carrying calibrated
budgets, poll-loop constants, utility spawn waits, and the seven
physical-ceiling 1_500_000 sites). The eval-budgets policy ratchet
keeps the residue from regrowing.

Known collapse: where an inner runner budget and its enclosing test
budget now share a tier, the old stagger is gone — an overrun surfaces
as a bun test timeout instead of a graceful runner timeout
(diagnosability trade, not a correctness one).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-29 05:25:06 +00:00
co-authored by Claude Fable 5
parent 74ae357e0f
commit 6841183c35
69 changed files with 367 additions and 298 deletions
+13 -12
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, browseBin, runId, evalsEnabled,
@@ -67,7 +68,7 @@ Do NOT use AskUserQuestion. Do NOT run gh or fly commands.`,
workingDirectory: landDir,
maxTurns: 20,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'land-and-deploy-workflow',
runId,
});
@@ -85,7 +86,7 @@ Do NOT use AskUserQuestion. Do NOT run gh or fly commands.`,
const reportDir = path.join(landDir, '.gstack', 'deploy-reports');
expect(fs.existsSync(reportDir)).toBe(true);
}, 180_000);
}, CAPTURE_MS);
});
// --- Land-and-Deploy First-Run E2E ---
@@ -148,7 +149,7 @@ Just demonstrate the first-run dry-run output.`,
workingDirectory: firstRunDir,
maxTurns: 20,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'land-and-deploy-first-run',
runId,
});
@@ -167,7 +168,7 @@ Just demonstrate the first-run dry-run output.`,
const reportContent = fs.readFileSync(path.join(reportDir, reportFiles[0]), 'utf-8');
const hasPlatform = reportContent.toLowerCase().includes('fly') || reportContent.toLowerCase().includes('first-run-app');
expect(hasPlatform).toBe(true);
}, 180_000);
}, CAPTURE_MS);
});
// --- Land-and-Deploy Review Gate E2E ---
@@ -226,7 +227,7 @@ Show what the readiness gate output would look like.`,
workingDirectory: reviewDir,
maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'land-and-deploy-review-gate',
runId,
});
@@ -246,7 +247,7 @@ Show what the readiness gate output would look like.`,
const hasReviewMention = reportContent.toLowerCase().includes('review') ||
reportContent.toLowerCase().includes('not run');
expect(hasReviewMention).toBe(true);
}, 180_000);
}, CAPTURE_MS);
});
// --- Canary skill E2E ---
@@ -294,7 +295,7 @@ Just create the directory structure and report files showing the correct schema.
workingDirectory: canaryDir,
maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'canary-workflow',
runId,
});
@@ -307,7 +308,7 @@ Just create the directory structure and report files showing the correct schema.
const reportDir = path.join(canaryDir, '.gstack', 'canary-reports');
const files = fs.readdirSync(reportDir, { recursive: true }) as string[];
expect(files.length).toBeGreaterThan(0);
}, 180_000);
}, CAPTURE_MS);
});
// --- Benchmark skill E2E ---
@@ -357,7 +358,7 @@ Just create the files showing the correct schema and report format.`,
workingDirectory: benchDir,
maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'benchmark-workflow',
runId,
});
@@ -372,7 +373,7 @@ Just create the files showing the correct schema and report format.`,
const files = fs.readdirSync(baselineDir);
expect(files.length).toBeGreaterThan(0);
}
}, 180_000);
}, CAPTURE_MS);
});
// --- Setup-Deploy skill E2E ---
@@ -418,7 +419,7 @@ Just detect the platform and write the config.`,
workingDirectory: setupDir,
maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'setup-deploy-workflow',
runId,
});
@@ -434,7 +435,7 @@ Just detect the platform and write the config.`,
expect(content.toLowerCase()).toContain('fly');
expect(content).toContain('my-cool-app');
expect(content).toContain('Deploy Configuration');
}, 180_000);
}, CAPTURE_MS);
});
// Module-level afterAll — finalize eval collector after all tests complete