mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-02 17:40:02 +02:00
garrytan
b1f5bc0032
Paid evals never retry (approved 2026-09-29): delete SHORT_CASE_RETRY_FILES and retriesWithinCaseCap, drop the retry fields from the registered wall rows (walls now cover one run plus reserve), make retriesForFiles return 0, pass --retry 0 explicitly, and drop --retry 1 from the package.json paid scripts. Add the eval:pass-rates alias. Tests that pinned the old retry allowance are updated as a policy change; review-finalization-budget now proves late-result recording under the production zero-retry arguments.
45 lines
2.9 KiB
TypeScript
45 lines
2.9 KiB
TypeScript
import { describe, expect, test } from 'bun:test';
|
|
import * as fs from 'node:fs';
|
|
import * as path from 'node:path';
|
|
import { CARVE_GUARDS } from './helpers/carve-guards';
|
|
import { isPaidTestFile } from './helpers/paid-test-set';
|
|
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
|
import { buildRunManifest, DEFAULT_SHARD_TIMEOUT_MS, retriesForFiles, selectPaidTestFiles } from '../scripts/test-paid-shards';
|
|
|
|
describe('carved-skill cases each get a complete paid process budget', () => {
|
|
const files = fs.readdirSync(import.meta.dir).filter(name => /^carve-section-loading-.*\.test\.ts$/.test(name));
|
|
test('every generic registry entry has exactly one periodic wrapper and no external entry does', () => {
|
|
const covered = files.flatMap(file => {
|
|
const source = fs.readFileSync(path.join(import.meta.dir, file), 'utf8');
|
|
expect(source).toContain("describeE2ETier('periodic')");
|
|
const calls = [...source.matchAll(/registerCarveSectionCase\('([^']+)'\)/g)];
|
|
expect(calls).toHaveLength(1);
|
|
expect(isPaidTestFile('test/' + file)).toBe(true);
|
|
return calls.map(match => match[1]);
|
|
});
|
|
expect(covered.sort()).toEqual(Object.values(CARVE_GUARDS).filter(guard => guard.behavioral === 'plan' || guard.behavioral === 'prompt').map(guard => guard.skill).sort());
|
|
expect(new Set(covered).size).toBe(covered.length);
|
|
expect(selectPaidTestFiles(files.map(file => 'test/' + file), 'periodic').selected).toHaveLength(files.length);
|
|
expect(selectPaidTestFiles(files.map(file => 'test/' + file), 'gate').selected).toHaveLength(0);
|
|
});
|
|
test('every case run plus teardown fits even with within-shard concurrency one', () => {
|
|
for (const file of files) {
|
|
const attempts = retriesForFiles(['test/' + file]) + 1;
|
|
expect(CAPTURE_LONG_MS * attempts + 10_000).toBeLessThan(DEFAULT_SHARD_TIMEOUT_MS);
|
|
}
|
|
});
|
|
test('explicit skill scope selects one process and records why the others are excluded', () => {
|
|
const discovered = files.map(file => 'test/' + file);
|
|
const env = { GSTACK_CARVE_SKILL: ' review ', EVALS_ALL: '1' };
|
|
const root = path.resolve(import.meta.dir, '..');
|
|
const result = selectPaidTestFiles(discovered, 'periodic', root, env);
|
|
expect(result.selected).toEqual(['test/carve-section-loading-review.test.ts']);
|
|
expect(result.excluded).toHaveLength(discovered.length - 1);
|
|
expect(result.excluded.every(entry => entry.reason.includes('GSTACK_CARVE_SKILL=review'))).toBe(true);
|
|
const manifest = buildRunManifest({ tier: 'periodic', sliceCount: 2, evalsAll: true, discovered, env, rootDir: root });
|
|
expect(manifest.entries.filter(entry => entry.status === 'planned').map(entry => entry.file)).toEqual(result.selected);
|
|
expect(manifest.entries.filter(entry => entry.status === 'excluded')).toHaveLength(result.excluded.length);
|
|
expect(() => selectPaidTestFiles(discovered, 'periodic', root, { GSTACK_CARVE_SKILL: 'typo' })).toThrow('no generic section-loading wrapper');
|
|
});
|
|
});
|