diff --git a/test/helpers/workflow-judge-input.ts b/test/helpers/workflow-judge-input.ts index 8c81ab4ce..36873687b 100644 --- a/test/helpers/workflow-judge-input.ts +++ b/test/helpers/workflow-judge-input.ts @@ -25,6 +25,8 @@ export const QA_DISCOVERY_REFERENCES = [ ]; export const WORKFLOW_JUDGE_REASONING_WORD_LIMIT = 150; +/** The instructed length sits below the enforced limit: judges asked for <150 landed at 130-156 words. */ +export const WORKFLOW_JUDGE_REASONING_WORD_TARGET = 120; export const WORKFLOW_JUDGE_RESPONSE_SCHEMA = { type: 'object', @@ -33,7 +35,7 @@ export const WORKFLOW_JUDGE_RESPONSE_SCHEMA = { completeness: { type: 'integer', enum: [1, 2, 3, 4, 5] }, actionability: { type: 'integer', enum: [1, 2, 3, 4, 5] }, reasoning: { type: 'string', - description: `Under ${WORKFLOW_JUDGE_REASONING_WORD_LIMIT} words with at most two decisive examples, evaluating the complete supplied workflow.` }, + description: `Under ${WORKFLOW_JUDGE_REASONING_WORD_TARGET} words with at most two decisive examples, evaluating the complete supplied workflow.` }, }, required: ['clarity', 'completeness', 'actionability', 'reasoning'], additionalProperties: false, @@ -61,7 +63,7 @@ Clarity 4 means the target agent can determine the next permitted action on each 5 additionally means those paths are easy to locate and understand. Score clarity 3 or lower when execution still requires guessing because of conflicting order, undefined decisions, unclear authority or missing input/output handling. -Evaluate the whole workflow, but keep the JSON reasoning under 150 words with at most two decisive examples. +Evaluate the whole workflow, but keep the JSON reasoning under ${WORKFLOW_JUDGE_REASONING_WORD_TARGET} words with at most two decisive examples. For a clarity defect, cite the specific file/step and explain the competing actions or missing decision. Keep completeness and actionability independent: reader capability does not supply missing requirements.` : ''} diff --git a/test/workflow-judge-input.test.ts b/test/workflow-judge-input.test.ts index 1f3a9a248..47ff4b3a7 100644 --- a/test/workflow-judge-input.test.ts +++ b/test/workflow-judge-input.test.ts @@ -4,7 +4,7 @@ import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSyn import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { createHash } from 'node:crypto'; -import { readWorkflowJudgeInput, buildWorkflowJudgePrompt, QA_DISCOVERY_REFERENCES } from './helpers/workflow-judge-input'; +import { readWorkflowJudgeInput, buildWorkflowJudgePrompt, QA_DISCOVERY_REFERENCES, WORKFLOW_JUDGE_RESPONSE_SCHEMA, WORKFLOW_JUDGE_REASONING_WORD_LIMIT } from './helpers/workflow-judge-input'; import { ENG_REVIEW_EXCERPT } from './helpers/workflow-excerpt'; const ROOT = resolve(import.meta.dir, '..'); @@ -41,12 +41,20 @@ test.each(['ship', 'review'])('%s clarity targets frontier readers without excus test('frontier calibration bounds reporting without reducing the evaluated source bundle', () => { const input = { files: [], text: 'Entire source bundle remains present.' }; const prompt = buildWorkflowJudgePrompt({ judgeContext: 'a workflow', judgeGoal: 'how to finish', agentCapability: 'frontier' }, input); - expect(prompt).toContain('Evaluate the whole workflow, but keep the JSON reasoning under 150 words with at most two decisive examples'); + expect(prompt).toContain('Evaluate the whole workflow, but keep the JSON reasoning under 120 words with at most two decisive examples'); expect(prompt).toContain('For a clarity defect, cite the specific file/step and explain the competing actions or missing decision'); expect(prompt).not.toContain('For each clarity defect'); expect(prompt.endsWith(input.text)).toBe(true); }); +test('judges are asked for 120 words while the enforced reasoning limit stays below 150', () => { + const prompt = buildWorkflowJudgePrompt({ judgeContext: 'a workflow', judgeGoal: 'how to finish', agentCapability: 'frontier' }, { files: [], text: '' }); + expect(prompt).toContain('keep the JSON reasoning under 120 words'); + expect(prompt).not.toContain('150 words'); + expect(WORKFLOW_JUDGE_RESPONSE_SCHEMA.properties.reasoning.description).toStartWith('Under 120 words'); + expect(WORKFLOW_JUDGE_REASONING_WORD_LIMIT).toBe(150); +}); + afterEach(() => { for (const root of scratchRoots.splice(0)) rmSync(root, { recursive: true, force: true }); });