mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-04 18:36:54 +02:00
feat(evals): ~12-minute blocking paid lanes and a non-blocking marathon lane
- Planner budget mode (--slice-budget S --jobs J): recorded per-tier wall times pack into as many ~9-minute executors as the work needs; the plan records per-slice estimates and the CI job timeout (supervised worst case + 20 min). evals.yml and evals-periodic.yml derive matrix size and timeout-minutes from it; max-parallel covers every slice at once. - Case shards: plan/design/review-army/shared-libs(-paths) run one registered case per process (<file>#<case id>, exact name pattern, exactly one case). - Retry rule: a timed-out attempt is a verdict. Only files whose every case budget is CAPTURE tier or shorter keep one retry; walls shrink to match. - Marathon tier: positive selection, excluded from gate/periodic planners, run by the new evals-marathon.yml (weekly + dispatch, fresh, own report). - PR-lane E2E reuse of verified first-attempt passes on identical inputs; the report rejects reuse outside the fast PR profile. - Duration seed from census run 36385945043, per tier and per case shard.
This commit is contained in:
1 parent
acb7bc02b1
commit
0023d011a3
18 files changed
+1773
-450
No files matched your search
@@ -24,9 +24,10 @@ import {
|
||||
DEFAULT_JOBS,
|
||||
DEFAULT_SHARD_TIMEOUT_MS,
|
||||
resolvePaidShardBudget,
|
||||
expandCaseShards,
|
||||
type PaidTier,
|
||||
} from '../scripts/test-paid-shards';
|
||||
import { FINDING_RETRY_BUDGETS } from './helpers/eval-budgets';
|
||||
import { FILE_RETRY_BUDGETS } from './helpers/eval-budgets';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
// 5% margin over the theoretical bound: detach setup, lock wait, aggregation.
|
||||
@@ -57,7 +58,7 @@ describe('eval:bg detach timeouts cover the sharded runner worst case', () => {
|
||||
['periodic', 'eval:bg:periodic'],
|
||||
] as Array<[PaidTier, string]>) {
|
||||
test(`${script} covers ordinary ${tier} waves plus registered excess x ${MARGIN}`, () => {
|
||||
const files = selectPaidTestFiles(collectPaidTestFiles(), tier).selected;
|
||||
const files = expandCaseShards(selectPaidTestFiles(collectPaidTestFiles(), tier).selected, tier);
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
const floor = Math.ceil(worstCaseSeconds(files) * MARGIN);
|
||||
const configured = detachTimeoutSeconds(script);
|
||||
@@ -78,7 +79,7 @@ describe('eval:bg detach timeouts cover the sharded runner worst case', () => {
|
||||
const pkg = JSON.parse(fs.readFileSync(path.join(ROOT, 'package.json'), 'utf8'));
|
||||
const jobs = Number(pkg.scripts['test:pr'].match(/EVALS_JOBS=\$\{EVALS_JOBS:-(\d+)\}/)?.[1]);
|
||||
expect(jobs).toBe(2);
|
||||
const files = selectPaidTestFiles(collectPaidTestFiles(), 'gate').selected;
|
||||
const files = expandCaseShards(selectPaidTestFiles(collectPaidTestFiles(), 'gate').selected, 'gate');
|
||||
const floor = Math.ceil(worstCaseSeconds(files, jobs) * MARGIN);
|
||||
expect(detachTimeoutSeconds('eval:bg:pr')).toBeGreaterThanOrEqual(floor);
|
||||
});
|
||||
@@ -86,7 +87,7 @@ describe('eval:bg detach timeouts cover the sharded runner worst case', () => {
|
||||
test('eval:bg:release covers both complete tiers and their existing margins', () => {
|
||||
const files = collectPaidTestFiles();
|
||||
const floor = (['gate', 'periodic'] as const).reduce((sum, tier) =>
|
||||
sum + Math.ceil(worstCaseSeconds(selectPaidTestFiles(files, tier).selected) * MARGIN), 0);
|
||||
sum + Math.ceil(worstCaseSeconds(expandCaseShards(selectPaidTestFiles(files, tier).selected, tier)) * MARGIN), 0);
|
||||
expect(detachTimeoutSeconds('eval:bg:release')).toBeGreaterThanOrEqual(floor);
|
||||
});
|
||||
});
|
||||
@@ -94,7 +95,9 @@ describe('eval:bg detach timeouts cover the sharded runner worst case', () => {
|
||||
// One long job and one ordinary job can run side by side; the long job still
|
||||
// needs its whole wall, regardless of the number of ordinary workers.
|
||||
test('a heterogeneous pair rejects the old uniform-wall floor', () => {
|
||||
const pair = [FINDING_RETRY_BUDGETS[0]!.file, 'test/skill-e2e-other.test.ts'];
|
||||
// The longest registered wall (single-attempt finding files now fit the ordinary wall).
|
||||
const longest = [...FILE_RETRY_BUDGETS].sort((a, b) => b.shardMs - a.shardMs)[0]!;
|
||||
const pair = [longest.file, 'test/skill-e2e-other.test.ts'];
|
||||
const actualLongest = Math.max(...pair.map(file => resolvePaidShardBudget([file]).timeoutMs)) / 1000;
|
||||
expect(worstCaseSeconds(pair, 2)).toBe(actualLongest);
|
||||
expect(worstCaseSeconds(pair, 2)).toBeGreaterThan(DEFAULT_SHARD_TIMEOUT_MS / 1000);
|
||||
|
||||
Reference in new issue
Block a user