mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-04 07:47:27 +01:00
test(llm-judge): ask frontier workflow judges for 120-word reasoning under the unchanged 150-word check
Told "under 150 words", the ship judge's reasoning landed at 130-156 words (3 of 18 probe samples at 152-156), so the structured-response check failed about one panel in three independent of effort. The prompt's frontier block and the response schema description now say under 120 words; the validator still rejects 150 words or more. The changed prompt bytes reach only the two frontier judges: ship/SKILL.md workflow (prompt and schema) and review/SKILL.md workflow (prompt).
This commit is contained in:
1 parent
a5b66d5cb4
commit
852f999214
2 files changed
+14
-4
No files matched your search
@@ -25,6 +25,8 @@ export const QA_DISCOVERY_REFERENCES = [
|
||||
];
|
||||
|
||||
export const WORKFLOW_JUDGE_REASONING_WORD_LIMIT = 150;
|
||||
/** The instructed length sits below the enforced limit: judges asked for <150 landed at 130-156 words. */
|
||||
export const WORKFLOW_JUDGE_REASONING_WORD_TARGET = 120;
|
||||
|
||||
export const WORKFLOW_JUDGE_RESPONSE_SCHEMA = {
|
||||
type: 'object',
|
||||
@@ -33,7 +35,7 @@ export const WORKFLOW_JUDGE_RESPONSE_SCHEMA = {
|
||||
completeness: { type: 'integer', enum: [1, 2, 3, 4, 5] },
|
||||
actionability: { type: 'integer', enum: [1, 2, 3, 4, 5] },
|
||||
reasoning: { type: 'string',
|
||||
description: `Under ${WORKFLOW_JUDGE_REASONING_WORD_LIMIT} words with at most two decisive examples, evaluating the complete supplied workflow.` },
|
||||
description: `Under ${WORKFLOW_JUDGE_REASONING_WORD_TARGET} words with at most two decisive examples, evaluating the complete supplied workflow.` },
|
||||
},
|
||||
required: ['clarity', 'completeness', 'actionability', 'reasoning'],
|
||||
additionalProperties: false,
|
||||
@@ -61,7 +63,7 @@ Clarity 4 means the target agent can determine the next permitted action on each
|
||||
5 additionally means those paths are easy to locate and understand.
|
||||
Score clarity 3 or lower when execution still requires guessing because of
|
||||
conflicting order, undefined decisions, unclear authority or missing input/output handling.
|
||||
Evaluate the whole workflow, but keep the JSON reasoning under 150 words with at most two decisive examples.
|
||||
Evaluate the whole workflow, but keep the JSON reasoning under ${WORKFLOW_JUDGE_REASONING_WORD_TARGET} words with at most two decisive examples.
|
||||
For a clarity defect, cite the specific file/step and explain the competing actions or missing decision.
|
||||
Keep completeness and actionability independent: reader capability does not supply missing requirements.` : ''}
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSyn
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join, resolve } from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { readWorkflowJudgeInput, buildWorkflowJudgePrompt, QA_DISCOVERY_REFERENCES } from './helpers/workflow-judge-input';
|
||||
import { readWorkflowJudgeInput, buildWorkflowJudgePrompt, QA_DISCOVERY_REFERENCES, WORKFLOW_JUDGE_RESPONSE_SCHEMA, WORKFLOW_JUDGE_REASONING_WORD_LIMIT } from './helpers/workflow-judge-input';
|
||||
import { ENG_REVIEW_EXCERPT } from './helpers/workflow-excerpt';
|
||||
|
||||
const ROOT = resolve(import.meta.dir, '..');
|
||||
@@ -41,12 +41,20 @@ test.each(['ship', 'review'])('%s clarity targets frontier readers without excus
|
||||
test('frontier calibration bounds reporting without reducing the evaluated source bundle', () => {
|
||||
const input = { files: [], text: 'Entire source bundle remains present.' };
|
||||
const prompt = buildWorkflowJudgePrompt({ judgeContext: 'a workflow', judgeGoal: 'how to finish', agentCapability: 'frontier' }, input);
|
||||
expect(prompt).toContain('Evaluate the whole workflow, but keep the JSON reasoning under 150 words with at most two decisive examples');
|
||||
expect(prompt).toContain('Evaluate the whole workflow, but keep the JSON reasoning under 120 words with at most two decisive examples');
|
||||
expect(prompt).toContain('For a clarity defect, cite the specific file/step and explain the competing actions or missing decision');
|
||||
expect(prompt).not.toContain('For each clarity defect');
|
||||
expect(prompt.endsWith(input.text)).toBe(true);
|
||||
});
|
||||
|
||||
test('judges are asked for 120 words while the enforced reasoning limit stays below 150', () => {
|
||||
const prompt = buildWorkflowJudgePrompt({ judgeContext: 'a workflow', judgeGoal: 'how to finish', agentCapability: 'frontier' }, { files: [], text: '' });
|
||||
expect(prompt).toContain('keep the JSON reasoning under 120 words');
|
||||
expect(prompt).not.toContain('150 words');
|
||||
expect(WORKFLOW_JUDGE_RESPONSE_SCHEMA.properties.reasoning.description).toStartWith('Under 120 words');
|
||||
expect(WORKFLOW_JUDGE_REASONING_WORD_LIMIT).toBe(150);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const root of scratchRoots.splice(0)) rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user