mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-10 10:40:32 +01:00
test(evals): keep the review judge at default effort; only lift its output cap
Truncation was the defect: thinking and the answer share max_tokens, and the ~64k-token review bundle once spent all 8192. The registration keeps default effort and response format and gets ship's 64k streaming cap; the guard test pins exactly that.
This commit is contained in:
1 parent
e36c82374e
commit
78d65fbcb9
2 files changed
+6
-7
No files matched your search
@@ -669,12 +669,11 @@ describeIfSelected('Other skill evals', [
|
||||
judgeContext: 'a pre-landing review with bounded exploratory QA',
|
||||
judgeGoal: 'how to review and explore changed behavior even for small diffs without a plan or server, preserve report-only discovery and the test_stub ASK gate, handle incomplete probes honestly, and rerun affected evidence after approved repairs',
|
||||
agentCapability: 'frontier',
|
||||
// This ~64k-token bundle is the largest workflow input. At default effort, thinking reached 6.9k output
|
||||
// tokens and 88 s in 24 measured samples, and a CI sample spent all 8192 tokens without an answer.
|
||||
// Medium kept 24 of 24 under 5k tokens and 63 s with the same gating scores; the budget is no cap.
|
||||
// This ~64k-token bundle is the largest workflow input. Thinking and the answer share max_tokens:
|
||||
// at default effort, 24 measured samples used up to 6.9k output tokens and a CI sample spent all
|
||||
// 8192 without an answer. The ship-sized streaming cap removes truncation; effort stays default.
|
||||
maxTokens: 65_536,
|
||||
stream: true,
|
||||
effort: 'medium',
|
||||
});
|
||||
}, WORKFLOW_JUDGE_TEST_MS);
|
||||
|
||||
|
||||
@@ -399,13 +399,13 @@ test('Ship sends its authorized 64k cap and compact response contract through th
|
||||
expect(options.maxTokens).toBe(65_536);
|
||||
expect(options.stream).toBe(true);
|
||||
expect(options.effort).toBe('medium');
|
||||
// Only measured registrations bound judge effort: ship and the ~64k-token review bundle.
|
||||
expect(source.match(/^\s+effort: '/gm)).toHaveLength(2);
|
||||
expect(source.match(/^\s+effort: '/gm)).toHaveLength(1);
|
||||
// The ~64k-token review bundle gets only ship's output cap and streaming; its effort and response format stay default.
|
||||
const review = source.match(/testIfSelected\('review\/SKILL\.md workflow',[\s\S]*?await runWorkflowJudge\(\{([\s\S]*?)\n \}\);/);
|
||||
expect(review).not.toBeNull();
|
||||
const reviewOptions = new Function('QA_DISCOVERY_REFERENCES', `return ({${review![1]}});`)(QA_DISCOVERY_REFERENCES);
|
||||
expect([reviewOptions.maxTokens, reviewOptions.stream, reviewOptions.effort, reviewOptions.schemaTransport, reviewOptions.compactReasoning])
|
||||
.toEqual([65_536, true, 'medium', undefined, undefined]);
|
||||
.toEqual([65_536, true, undefined, undefined, undefined]);
|
||||
expect(WORKFLOW_JUDGE_RESPONSE_SCHEMA.properties.reasoning).not.toHaveProperty('pattern');
|
||||
expect(WORKFLOW_JUDGE_RESPONSE_SCHEMA.properties.reasoning).not.toHaveProperty('maxLength');
|
||||
expect(source.match(/compactReasoning: true/g)).toHaveLength(1);
|
||||
|
||||
Reference in new issue
Block a user