mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-11 05:28:54 +02:00
test: retire the finding-count cluster and trim its helpers (C)
- C0/C1: the five never-green evals (skill-e2e-autoplan-chain and
skill-e2e-plan-{ceo,eng,design,devex}-finding-count) failed on harness and
budget, never on skill behavior; delete them, their touchfile/tier ids,
AUTOPLAN_CHAIN_BUDGET and the dedicated eighth periodic slice (--slices 7).
- C2: delete the helper groups whose only paid consumers were those files
(11 modules), trim claude-pty-runner and eng-seeded-coverage to the paid
closure, and delete the free replay tests whose assertions exercised only
that dead code (89 files, 135 orphaned fixtures). Blocks that used dead code
only as input for a live subject keep their assertions: the multiSelect
default moved to plan-review-decisions, runner PTY tests use inline caller
policies, and the timer-safe budget checks moved to eng-finding-retry-budget.
- The eight production-touching files stay except ceo-current-decision-record
(its template read only feeds the retired counter).
- CARVE_GUARDS.autoplan is behavioral 'none'; TODOS records the lost chain
and per-finding cadence coverage with their re-entry tests.
This commit is contained in:
1 parent
53e7f3212f
commit
6415690a18
317 files changed
+315
-54111
No files matched your search
@@ -1,6 +1,6 @@
|
||||
import { expect, test } from 'bun:test';
|
||||
import { resolvePaidShardBudget, retriesForFiles, planPaidShards, parseRunManifest, verifySliceResults, runPaidShard, buildRunManifest, paidShardWallUpperBoundMs, collectPaidTestFiles, selectPaidTestFiles, isOverlayTestFile, OVERLAY_MAX_ACTIVE_SHARDS, DEFAULT_SHARD_TIMEOUT_MS, DEFAULT_JOBS } from '../scripts/test-paid-shards';
|
||||
import { FINDING_RETRY_BUDGETS, ALL_TIERS, AUTOPLAN_CHAIN_BUDGET } from './helpers/eval-budgets';
|
||||
import { FINDING_RETRY_BUDGETS, ALL_TIERS, SHARD_RESERVE_MS } from './helpers/eval-budgets';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
@@ -10,7 +10,7 @@ for (const budget of FINDING_RETRY_BUDGETS) {
|
||||
expect(budget.testMs).toBe(1_500_000);
|
||||
expect(budget.retries).toBe(1);
|
||||
expect(retriesForFiles([budget.file])).toBe(budget.retries);
|
||||
expect(budget.shardReserveMs).toBe(AUTOPLAN_CHAIN_BUDGET.shardReserveMs);
|
||||
expect(budget.shardReserveMs).toBe(SHARD_RESERVE_MS);
|
||||
expect(budget.shardMs).toBe(budget.cases * budget.testMs * (budget.retries + 1) + budget.shardReserveMs);
|
||||
expect(resolvePaidShardBudget([budget.file])).toEqual({ timeoutMs: budget.shardMs, source: 'registered', policyId: budget.id });
|
||||
const source = fs.readFileSync(path.join(import.meta.dir, '..', budget.file), 'utf8');
|
||||
@@ -20,12 +20,6 @@ for (const budget of FINDING_RETRY_BUDGETS) {
|
||||
expect([...source.matchAll(/const deadlineAt = Date\.now\(\) \+ 1_500_000;/g)]).toHaveLength(budget.cases);
|
||||
expect([...source.matchAll(/timeoutMs:\s*deadlineAt - Date\.now\(\)/g)]).toHaveLength(budget.cases);
|
||||
expect(source).toContain("floor: FLOOR, kind: 'scope', deadlineAt");
|
||||
} else if (budget.file === 'test/skill-e2e-plan-eng-finding-count.test.ts') {
|
||||
// Its terminal assessment shares the original allowance with the actor.
|
||||
expect([...source.matchAll(/const startedAt = Date\.now\(\);/g)]).toHaveLength(budget.cases);
|
||||
expect([...source.matchAll(/const deadlineAt = startedAt \+ 1_500_000;/g)]).toHaveLength(budget.cases);
|
||||
expect([...source.matchAll(/timeoutMs:\s*deadlineAt - Date\.now\(\)/g)]).toHaveLength(budget.cases);
|
||||
expect(source).toContain('deadlineAt: Math.min(input.deadlineAt, deadlineAt)');
|
||||
} else {
|
||||
expect([...source.matchAll(/timeoutMs:\s*1_500_000\b/g)]).toHaveLength(budget.cases);
|
||||
}
|
||||
@@ -86,15 +80,18 @@ for (const budget of FINDING_RETRY_BUDGETS) {
|
||||
});
|
||||
}
|
||||
|
||||
test('ordinary tiers and Autoplan allocations remain unchanged', () => {
|
||||
test('ordinary tiers and registered allocations remain unchanged', () => {
|
||||
expect(ALL_TIERS).toEqual({ JUDGE_MS: 120000, CAPTURE_MS: 300000, CAPTURE_LONG_MS: 600000, PTY_MS: 900000, PTY_LONG_MS: 1200000 });
|
||||
expect(resolvePaidShardBudget(['test/other.test.ts'])).toEqual({ timeoutMs: 1800000, source: 'default', policyId: null });
|
||||
expect(resolvePaidShardBudget([AUTOPLAN_CHAIN_BUDGET.file])).toEqual({ timeoutMs: AUTOPLAN_CHAIN_BUDGET.shardMs, source: 'registered', policyId: AUTOPLAN_CHAIN_BUDGET.id });
|
||||
expect(new Set(FINDING_RETRY_BUDGETS.map(b => b.file)).size).toBe(6);
|
||||
expect(resolvePaidShardBudget(['test/other.test.ts'], 12_000)).toEqual({ timeoutMs: 12_000, source: 'explicit', policyId: null });
|
||||
for (const value of [NaN, Infinity, -1, 0, 1.5, 2_147_483_648]) {
|
||||
expect(() => resolvePaidShardBudget(['test/other.test.ts'], value)).toThrow('timer-safe');
|
||||
}
|
||||
expect(new Set(FINDING_RETRY_BUDGETS.map(b => b.file)).size).toBe(2);
|
||||
});
|
||||
|
||||
test('actual shard launcher honors the explicit saved planner limit without a provider', async () => {
|
||||
const budget = FINDING_RETRY_BUDGETS.find(b => b.file.includes('plan-eng-finding-count'))!;
|
||||
const budget = FINDING_RETRY_BUDGETS.find(b => b.file.includes('plan-eng-multi-finding-batching'))!;
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'finding-retry-wall-'));
|
||||
try {
|
||||
const outcome = await runPaidShard([budget.file], 1, 1, { rootDir: dir, logDir: dir, jobs: 2,
|
||||
@@ -114,11 +111,11 @@ const periodicSliceCount = Number(periodicPlanStep.run.match(/--slices\s+(\d+)/)
|
||||
const periodicRunStep = periodicJob.steps.find((step: any) => step.run?.includes('--plan /tmp/paid-plan/manifest.json'));
|
||||
const periodicWorkers = Number(periodicRunStep.env.EVALS_JOBS);
|
||||
const livePlan = (discovered?: string[]) => buildRunManifest({ tier: 'periodic', sliceCount: periodicSliceCount,
|
||||
evalsAll: true, dedicatedAutoplanSlice: true, env: { EVALS_ALL: '1' }, discovered });
|
||||
evalsAll: true, env: { EVALS_ALL: '1' }, discovered });
|
||||
|
||||
test('live periodic census fits the declared CI wall including setup', () => {
|
||||
const m = livePlan();
|
||||
expect(periodicPlanStep.run).toContain('--autoplan-slice');
|
||||
expect(periodicPlanStep.run).not.toContain('--autoplan-slice');
|
||||
expect(periodicJob.strategy.matrix.slice).toEqual(Array.from({ length: periodicSliceCount }, (_, index) => index + 1));
|
||||
expect(periodicWorkers).toBe(2);
|
||||
const walls = Array.from({ length: periodicSliceCount }, (_, index) => {
|
||||
@@ -127,16 +124,15 @@ test('live periodic census fits the declared CI wall including setup', () => {
|
||||
return paidShardWallUpperBoundMs(files, workers);
|
||||
});
|
||||
expect(Math.max(...walls) + 20 * 60_000).toBeLessThanOrEqual(periodicJob['timeout-minutes'] * 60_000);
|
||||
expect(m.entries.filter(e => e.status === 'planned')).toHaveLength(82);
|
||||
const overlays = m.entries.filter(e => e.status === 'planned' && e.slice === periodicSliceCount - 1);
|
||||
expect(m.entries.filter(e => e.status === 'planned')).toHaveLength(77);
|
||||
const overlays = m.entries.filter(e => e.status === 'planned' && e.slice === periodicSliceCount);
|
||||
expect(overlays).toHaveLength(4);
|
||||
expect(overlays.every(e => isOverlayTestFile(e.file))).toBe(true);
|
||||
expect(m.entries.filter(e => e.status === 'planned' && e.slice === periodicSliceCount).map(e => e.file)).toEqual([AUTOPLAN_CHAIN_BUDGET.file]);
|
||||
});
|
||||
|
||||
test('registered allocation is deterministic and preserves every discovered file', () => {
|
||||
const files = collectPaidTestFiles();
|
||||
expect(files).toHaveLength(105);
|
||||
expect(files).toHaveLength(100);
|
||||
const m = livePlan(files);
|
||||
expect(livePlan([...files].reverse())).toEqual(m);
|
||||
expect(m.entries.map(e => e.file).sort()).toEqual([...files].sort());
|
||||
@@ -157,7 +153,7 @@ test('ordinary-only manifests retain round-robin allocation', () => {
|
||||
|
||||
test('explicit allocation keeps its timer across load scheduling', () => {
|
||||
const m = buildRunManifest({ tier: 'periodic', sliceCount: 7, evalsAll: true,
|
||||
dedicatedAutoplanSlice: true, timeoutMs: 2_000_000, env: { EVALS_ALL: '1' } });
|
||||
timeoutMs: 2_000_000, env: { EVALS_ALL: '1' } });
|
||||
for (const entry of m.entries.filter(e => e.budget)) {
|
||||
expect(entry.budget).toEqual(resolvePaidShardBudget([entry.file], 2_000_000));
|
||||
}
|
||||
@@ -175,13 +171,13 @@ test('current detach supervision covers the live-census floor', () => {
|
||||
const floor = Math.ceil((Math.ceil(files.length / DEFAULT_JOBS) * DEFAULT_SHARD_TIMEOUT_MS + excess) / 1000 * 1.05);
|
||||
const pkg = JSON.parse(fs.readFileSync(path.join(import.meta.dir, '../package.json'), 'utf8'));
|
||||
const configured = Number(pkg.scripts['eval:bg:periodic'].match(/--timeout\s+(\d+)/)[1]);
|
||||
expect(floor).toBe(57330);
|
||||
expect(floor).toBe(37800);
|
||||
expect(configured).toBeGreaterThanOrEqual(floor);
|
||||
expect(pkg.scripts['eval:bg:gate']).toContain('--timeout 36000');
|
||||
});
|
||||
|
||||
for (const jobs of [1, 2, 3]) test(`FIFO bound covers partial durations with ${jobs} workers`, () => {
|
||||
const long = FINDING_RETRY_BUDGETS.find(b => b.cases === 2)!.file;
|
||||
const long = FINDING_RETRY_BUDGETS[0]!.file;
|
||||
for (const files of [[], ['test/a.test.ts'], [long, 'test/a.test.ts', 'test/b.test.ts'],
|
||||
['test/a.test.ts', 'test/b.test.ts', long, 'test/c.test.ts', 'test/d.test.ts'],
|
||||
[long, FINDING_RETRY_BUDGETS[1]!.file, 'test/a.test.ts', 'test/b.test.ts']]) {
|
||||
|
||||
Reference in new issue
Block a user