mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-10 06:58:59 +02:00
Garry TanandClaude Fable 5
3e674e4c01
spawnSync/execSync/Bun.spawnSync BLOCK the main thread, so bun's in-process per-test timeout can never fire while one waits — a hung child (stdin read, network probe, dead daemon) wedges the whole shard until the runner's external wall-clock SIGKILL. This exact class reached main: free-tests run 33262077256, test/gstack-memory-ingest.test.ts (normally 2.3s) held shard 2 at the 360s wall while its five siblings finished in ~65s. Mechanical sweep in two waves (12 + 4 fan-out agents, every edit verified against its call site): default timeout: 30_000 (matches the free runner's per-test budget), 120_000 for genuinely slow ops (installs, builds, playwright, provider CLIs), helper wrappers fixed ONCE where call sites route through them. Sites that only LOOK like calls (string fixtures, grep needles, comments) were skipped with reasons — the enforcement commit that follows marks them exempt. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
63 lines
2.7 KiB
TypeScript
63 lines
2.7 KiB
TypeScript
/**
|
|
* Two invariants over paid-test timeout policy:
|
|
*
|
|
* 1. FIT: every tier in test/helpers/eval-budgets.ts executes inside the
|
|
* sharded runner's wall with real overhead (bun startup + module load +
|
|
* reporting). A budget the wall kills first is fiction — the failure
|
|
* surfaces as a shard 'timed-out' (no bun summary, no per-test message)
|
|
* instead of a clean test timeout. This is the structural fix for the
|
|
* seven 1,700s-inside-a-1,500s-job literals found in the 2026-08 audit.
|
|
*
|
|
* 2. RATCHET: raw numeric timeout literals in paid test files only shrink.
|
|
* New tests use the tiers; a literal is legal only with justification,
|
|
* and the count is pinned so sprawl can't regrow.
|
|
*/
|
|
import { describe, expect, test } from 'bun:test';
|
|
import { spawnSync } from 'node:child_process';
|
|
import * as fs from 'node:fs';
|
|
import * as path from 'node:path';
|
|
|
|
import { ALL_TIERS, PTY_LONG_MS } from './helpers/eval-budgets';
|
|
import { isPaidTestFile } from './helpers/paid-test-set';
|
|
import { DEFAULT_SHARD_TIMEOUT_MS } from '../scripts/test-paid-shards';
|
|
|
|
const ROOT = path.resolve(__dirname, '..');
|
|
|
|
/** Wall overhead reserve: bun startup, module load, retry bookkeeping. */
|
|
const WALL_OVERHEAD_MS = 120_000;
|
|
|
|
describe('eval budget tiers', () => {
|
|
test('every tier fits inside the shard wall minus overhead', () => {
|
|
for (const [name, ms] of Object.entries(ALL_TIERS)) {
|
|
expect(ms, `${name} exceeds the shard wall minus overhead`)
|
|
.toBeLessThanOrEqual(DEFAULT_SHARD_TIMEOUT_MS - WALL_OVERHEAD_MS);
|
|
}
|
|
});
|
|
|
|
test('tiers are ordered and the ceiling is PTY_LONG', () => {
|
|
const values = Object.values(ALL_TIERS);
|
|
expect([...values].sort((a, b) => a - b)).toEqual(values);
|
|
expect(Math.max(...values)).toBe(PTY_LONG_MS);
|
|
});
|
|
|
|
test('no paid-test timeout literal exceeds the ceiling tier', () => {
|
|
const out = spawnSync('git', ['ls-files', 'test/*.test.ts'], { cwd: ROOT, encoding: 'utf-8', timeout: 30_000 });
|
|
const files = out.stdout.split('\n').filter((f) => f && isPaidTestFile(f));
|
|
expect(files.length).toBeGreaterThan(50); // scan-rot guard
|
|
|
|
const offenders: string[] = [];
|
|
for (const rel of files) {
|
|
const source = fs.readFileSync(path.join(ROOT, rel), 'utf-8');
|
|
// Trailing test-timeout args: `}, 1_234_000);` / `}, 300000);`
|
|
for (const m of source.matchAll(/\}\s*,\s*(\d[\d_]*)\s*(?:\/\*[^*]*\*\/\s*)?\)/g)) {
|
|
const ms = Number(m[1].replaceAll('_', ''));
|
|
if (ms > PTY_LONG_MS * 1.25) offenders.push(`${rel}: ${m[1]}`);
|
|
}
|
|
}
|
|
expect(offenders,
|
|
`paid-test timeouts above the PTY_LONG ceiling (x1.25 slack) are fiction ` +
|
|
`against the ${DEFAULT_SHARD_TIMEOUT_MS / 1000}s shard wall — split the test instead:\n${offenders.join('\n')}`,
|
|
).toEqual([]);
|
|
});
|
|
});
|