mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-02 17:40:02 +02:00
garrytan
6415690a18
- C0/C1: the five never-green evals (skill-e2e-autoplan-chain and
skill-e2e-plan-{ceo,eng,design,devex}-finding-count) failed on harness and
budget, never on skill behavior; delete them, their touchfile/tier ids,
AUTOPLAN_CHAIN_BUDGET and the dedicated eighth periodic slice (--slices 7).
- C2: delete the helper groups whose only paid consumers were those files
(11 modules), trim claude-pty-runner and eng-seeded-coverage to the paid
closure, and delete the free replay tests whose assertions exercised only
that dead code (89 files, 135 orphaned fixtures). Blocks that used dead code
only as input for a live subject keep their assertions: the multiSelect
default moved to plan-review-decisions, runner PTY tests use inline caller
policies, and the timer-safe budget checks moved to eng-finding-retry-budget.
- The eight production-touching files stay except ceo-current-decision-record
(its template read only feeds the retired counter).
- CARVE_GUARDS.autoplan is behavioral 'none'; TODOS records the lost chain
and per-finding cadence coverage with their re-entry tests.
70 lines
3.2 KiB
TypeScript
70 lines
3.2 KiB
TypeScript
/**
|
|
* Two invariants over paid-test timeout policy:
|
|
*
|
|
* 1. FIT: every tier in test/helpers/eval-budgets.ts executes inside the
|
|
* sharded runner's wall with real overhead (bun startup + module load +
|
|
* reporting). A budget the wall kills first is fiction — the failure
|
|
* surfaces as a shard 'timed-out' (no bun summary, no per-test message)
|
|
* instead of a clean test timeout. This is the structural fix for the
|
|
* seven 1,700s-inside-a-1,500s-job literals found in the 2026-08 audit.
|
|
*
|
|
* 2. RATCHET: raw numeric timeout literals in paid test files only shrink.
|
|
* New tests use the tiers; a literal is legal only with justification,
|
|
* and the count is pinned so sprawl can't regrow.
|
|
*/
|
|
import { describe, expect, test } from 'bun:test';
|
|
import { spawnSync } from 'node:child_process';
|
|
import * as fs from 'node:fs';
|
|
import * as path from 'node:path';
|
|
|
|
import { ALL_TIERS, PTY_LONG_MS, assertPaidTestBudget } from './helpers/eval-budgets';
|
|
import { isPaidTestFile } from './helpers/paid-test-set';
|
|
import { DEFAULT_SHARD_TIMEOUT_MS } from '../scripts/test-paid-shards';
|
|
|
|
const ROOT = path.resolve(__dirname, '..');
|
|
|
|
/** Wall overhead reserve: bun startup, module load, retry bookkeeping. */
|
|
const WALL_OVERHEAD_MS = 120_000;
|
|
|
|
describe('eval budget tiers', () => {
|
|
test('every tier fits inside the shard wall minus overhead', () => {
|
|
for (const [name, ms] of Object.entries(ALL_TIERS)) {
|
|
expect(ms, `${name} exceeds the shard wall minus overhead`)
|
|
.toBeLessThanOrEqual(DEFAULT_SHARD_TIMEOUT_MS - WALL_OVERHEAD_MS);
|
|
}
|
|
});
|
|
|
|
test('tiers are ordered and the ceiling is PTY_LONG', () => {
|
|
const values = Object.values(ALL_TIERS);
|
|
expect([...values].sort((a, b) => a - b)).toEqual(values);
|
|
expect(Math.max(...values)).toBe(PTY_LONG_MS);
|
|
});
|
|
|
|
test('deploy workflow sessions use capture budgets, not single-call judge budgets', () => {
|
|
const source = fs.readFileSync(path.join(ROOT, 'test/skill-e2e-deploy.test.ts'), 'utf8');
|
|
expect(source).not.toContain('JUDGE_MS');
|
|
expect([...source.matchAll(/timeout:\s*CAPTURE_MS/g)]).toHaveLength(6);
|
|
expect([...source.matchAll(/\},\s*CAPTURE_LONG_MS\);/g)]).toHaveLength(6);
|
|
});
|
|
|
|
test('paid timeouts above the ordinary ceiling are rejected', () => {
|
|
const out = spawnSync('git', ['ls-files', 'test/*.test.ts'], { cwd: ROOT, encoding: 'utf-8', timeout: 30_000 });
|
|
const files = out.stdout.split('\n').filter((f) => f && isPaidTestFile(f));
|
|
expect(files.length).toBeGreaterThan(50); // scan-rot guard
|
|
|
|
const offenders: string[] = [];
|
|
for (const rel of files) {
|
|
const source = fs.readFileSync(path.join(ROOT, rel), 'utf-8');
|
|
// Trailing test-timeout args: `}, 1_234_000);` / `}, 300000);`
|
|
for (const m of source.matchAll(/\}\s*,\s*(\d[\d_]*)\s*(?:\/\*[^*]*\*\/\s*)?\)/g)) {
|
|
const ms = Number(m[1].replaceAll('_', ''));
|
|
try { assertPaidTestBudget(rel, ms); } catch { offenders.push(`${rel}: ${m[1]}`); }
|
|
}
|
|
}
|
|
expect(offenders,
|
|
`paid-test timeouts above the PTY_LONG ceiling (x1.25 slack) are fiction ` +
|
|
`against the ${DEFAULT_SHARD_TIMEOUT_MS / 1000}s ordinary wall:\n${offenders.join('\n')}`,
|
|
).toEqual([]);
|
|
});
|
|
});
|