mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-04 10:26:52 +02:00
feat(evals): ~12-minute blocking paid lanes and a non-blocking marathon lane
- Planner budget mode (--slice-budget S --jobs J): recorded per-tier wall times pack into as many ~9-minute executors as the work needs; the plan records per-slice estimates and the CI job timeout (supervised worst case + 20 min). evals.yml and evals-periodic.yml derive matrix size and timeout-minutes from it; max-parallel covers every slice at once. - Case shards: plan/design/review-army/shared-libs(-paths) run one registered case per process (<file>#<case id>, exact name pattern, exactly one case). - Retry rule: a timed-out attempt is a verdict. Only files whose every case budget is CAPTURE tier or shorter keep one retry; walls shrink to match. - Marathon tier: positive selection, excluded from gate/periodic planners, run by the new evals-marathon.yml (weekly + dispatch, fresh, own report). - PR-lane E2E reuse of verified first-attempt passes on identical inputs; the report rejects reuse outside the fast PR profile. - Duration seed from census run 36385945043, per tier and per case shard.
This commit is contained in:
1 parent
acb7bc02b1
commit
0023d011a3
18 files changed
+1773
-450
No files matched your search
@@ -30,8 +30,10 @@ test('the actual CI cookie repair planner executes only eight dependent cases wi
|
||||
expect(manifest.evalsAll).toBe(false);
|
||||
expect(manifest.selection).toEqual({ e2e: ['browse-basic', 'browse-snapshot', 'qa-quick', 'qa-only-no-fix', 'design-review-detector-shim-dom', 'diagram-triplet', 'canary-workflow', 'benchmark-workflow'], judges: [] });
|
||||
expect(manifest.entries.filter(entry => entry.status === 'planned').map(entry => entry.file).sort()).toEqual([
|
||||
'test/skill-e2e-bws.test.ts', 'test/skill-e2e-deploy.test.ts', 'test/skill-e2e-design.test.ts', 'test/skill-e2e-diagram.test.ts', 'test/skill-e2e-qa-workflow.test.ts',
|
||||
'test/skill-e2e-bws.test.ts', 'test/skill-e2e-deploy.test.ts', 'test/skill-e2e-design.test.ts#design-review-detector-shim-dom', 'test/skill-e2e-diagram.test.ts', 'test/skill-e2e-qa-workflow.test.ts',
|
||||
]);
|
||||
// The case-sharded design file runs only its one selected cookie case.
|
||||
expect(manifest.entries.filter(entry => entry.file.startsWith('test/skill-e2e-design.test.ts#') && entry.status === 'skipped-by-diff').length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('the existing quality and behavior phases retain their complete separate shard census', () => {
|
||||
@@ -42,7 +44,10 @@ test('the existing quality and behavior phases retain their complete separate sh
|
||||
expect(quality.evalsAll).toBe(true);
|
||||
expect(behavior.evalsAll).toBe(true);
|
||||
expect(qualityFiles).toHaveLength(1);
|
||||
expect(behaviorFiles).toHaveLength(45);
|
||||
// 44 files (first-task-scaffold registers no gate case, so the gate lane
|
||||
// skips it); the five case-sharded files contribute one shard per gate case.
|
||||
expect(new Set(behaviorFiles.map(file => file.split('#')[0])).size).toBe(44);
|
||||
expect(behaviorFiles).toHaveLength(62);
|
||||
expect(behaviorFiles).toEqual(expect.arrayContaining([
|
||||
'test/skill-e2e-qa-callers.test.ts',
|
||||
'test/skill-e2e-qa-functional-fix.test.ts',
|
||||
|
||||
Reference in new issue
Block a user