v1.91.310.0 fix(flaky): the cookie workflow judge sends the response schema, released on GStack Minder's ruling (GSTA-139 S3) (#3581)

* fix(flaky): cookie workflow judge sends the response schema, released on GStack Minder's 2026-10-29 ruling

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test: register the train-42 replies fixture; keep a run URL and a low-entropy fake key out of the secrets scan

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

test/cookie-workflow-judge-input.test.ts

@@ -1,4 +1,4 @@
-import { afterEach, describe, expect, test } from 'bun:test';
+import { afterEach, describe, expect, spyOn, test } from 'bun:test';
 import { createHash } from 'node:crypto';
 import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
@@ -11,7 +11,8 @@ import { selectTests } from './helpers/test-selection';
 import { E2E_TOUCHFILES, LLM_JUDGE_TOUCHFILES } from './helpers/touchfiles-data';
 import { selectPrProfile } from '../scripts/test-pr-profile';
 import { JUDGE_MS } from './helpers/eval-budgets';
-import { JudgeRefusalError, DEFAULT_JUDGE_MAX_TOKENS, judgePanel, judgePanelMean, judgePanelMedian, judgePanelReasoning, JUDGE_SCORE_DIMENSIONS, JUDGE_PANEL_SAMPLES } from './helpers/llm-judge';
+import { Messages } from '@anthropic-ai/sdk/resources/messages';
+import { callJudge, type CallJudgeOptions, JudgeRefusalError, DEFAULT_JUDGE_MAX_TOKENS, judgePanel, judgePanelMean, judgePanelMedian, judgePanelReasoning, JUDGE_SCORE_DIMENSIONS, JUDGE_PANEL_SAMPLES } from './helpers/llm-judge';
 import { COOKIE_MANUAL_REVIEW_FILE, getCookieWorkflowManualReview, isManualReviewEntry } from './helpers/cookie-workflow-manual-review';
 
 const ROOT = resolve(import.meta.dir, '..');
@@ -51,6 +52,7 @@ const passingScore: FixtureScore = { ...COOKIE_WORKFLOW_JUDGE.thresholds, reason
 
 function actualCookieCallback(root: string, overrides: {
   judge?: () => Promise<FixtureScore>;
+  callJudge?: (prompt: string, model: string | undefined, options: CallJudgeOptions) => Promise<FixtureScore>;
   clock?: () => number;
   budget?: number;
   setTimer?: typeof setTimeout;
@@ -74,7 +76,11 @@ function actualCookieCallback(root: string, overrides: {
     (_suite: string, names: string[], run: () => void) => { expect(names).toEqual([NAME]); run(); },
     (name: string, run: () => Promise<void>, budget: number) => { expect(name).toBe(NAME); expect(budget).toBe(JUDGE_MS + 10_000); callback = run; },
     root, buildCookieWorkflowJudgeInput, (_kind: string, explicit?: string) => explicit ?? 'fixture-model',
-    async (prompt: string, model: string | undefined, options: { signal: AbortSignal }) => { requests.push({ prompt, model, signal: options.signal }); return overrides.judge ? overrides.judge() : passingScore; },
+    async (prompt: string, model: string | undefined, options: CallJudgeOptions & { signal: AbortSignal }) => {
+      requests.push({ prompt, model, signal: options.signal });
+      if (overrides.callJudge) return overrides.callJudge(prompt, model, options);
+      return overrides.judge ? overrides.judge() : passingScore;
+    },
     COOKIE_WORKFLOW_JUDGE, overrides.budget ?? JUDGE_MS, JUDGE_MS + 10_000, 5_000,
     { addTest: (record: EvalTestEntry) => records.push(record) }, expect, { log() {} },
     readWorkflowJudgeInput, buildWorkflowJudgePrompt,
@@ -269,6 +275,46 @@ describe('cookie workflow judge input', () => {
     expect(selectTests(['README.md'], LLM_JUDGE_TOUCHFILES).selected).not.toContain(NAME);
   });
 
+  test('the registered cookie judge sends the response schema, so a free-text reply like train 42\'s is not what gets parsed', async () => {
+    const { replies } = JSON.parse(readFileSync(join(ROOT, 'test/fixtures/cookie-judge-train42-replies.json'), 'utf8')) as {
+      replies: Array<{ id: string; model: string; stop_reason: string; text: string }> };
+    expect(replies.map(reply => reply.id)).toEqual(['msg_011CgVnRtVj5vyLvb56m6KJH', 'msg_011CgVne5GStZ6f9pRoCvgWE']);
+    const original = process.env.ANTHROPIC_API_KEY;
+    process.env.ANTHROPIC_API_KEY = 'fake';
+    const create = spyOn(Messages.prototype, 'create');
+    try {
+      for (const reply of replies) {
+        expect(() => JSON.parse(reply.text)).toThrow();
+        const marker = '"reasoning": "';
+        const judgment: FixtureScore = {
+          clarity: Number(/"clarity": (\d)/.exec(reply.text)![1]),
+          completeness: Number(/"completeness": (\d)/.exec(reply.text)![1]),
+          actionability: Number(/"actionability": (\d)/.exec(reply.text)![1]),
+          reasoning: reply.text.slice(reply.text.indexOf(marker) + marker.length, reply.text.lastIndexOf('"}')),
+        };
+        expect(judgment.reasoning).toContain('"one --domain per run"');
+        create.mockReset();
+        create.mockResolvedValue({ id: reply.id, model: reply.model, stop_reason: 'end_turn',
+          content: [{ type: 'text', text: JSON.stringify(judgment) }], usage: { input_tokens: 1, output_tokens: 1 } } as never);
+        const root = fixture();
+        const h = actualCookieCallback(root, { callJudge: (prompt, model, options) => callJudge<FixtureScore>(prompt, model, options) });
+        await h.run();
+        expect(create).toHaveBeenCalledTimes(JUDGE_PANEL_SAMPLES);
+        for (const [request, options] of create.mock.calls as unknown as Array<[Record<string, unknown>, { signal: AbortSignal }]>) {
+          expect(request).toEqual({ model: COOKIE_WORKFLOW_JUDGE.model, max_tokens: DEFAULT_JUDGE_MAX_TOKENS,
+            output_config: { format: { type: 'json_schema', schema: WORKFLOW_JUDGE_RESPONSE_SCHEMA } },
+            messages: [{ role: 'user', content: buildCookieWorkflowJudgeInput(root).prompt }] });
+          expect(options.signal).toBeInstanceOf(AbortSignal);
+        }
+        expect(h.records[0]).toMatchObject({ name: NAME, passed: true, execution: 'executed',
+          judge_scores: { clarity: judgment.clarity, completeness: judgment.completeness, actionability: judgment.actionability } });
+      }
+    } finally {
+      create.mockRestore();
+      if (original === undefined) delete process.env.ANTHROPIC_API_KEY; else process.env.ANTHROPIC_API_KEY = original;
+    }
+  });
+
   test('the managed callback sends and records exact custom input without borrowing a normal cache identity', async () => {
     const root = fixture();
     const input = buildCookieWorkflowJudgeInput(root);

test/fixtures/cookie-judge-train42-replies.json

@@ -0,0 +1,17 @@
+{
+  "source": "train 42 gate run 40809051510 (paid-logs-slice-5-a1, rerun-logs-1-a1): llm-judge-response-parse-error events",
+  "replies": [
+    {
+      "id": "msg_011CgVnRtVj5vyLvb56m6KJH",
+      "model": "claude-sonnet-5-5",
+      "stop_reason": "end_turn",
+      "text": "{\"clarity\": 4, \"completeness\": 4, \"actionability\": 4, \"reasoning\": \"The workflow is well ordered: check Aside, collect decisions up front, prepare the browser, import, and report. Consent for storage reset and --all is explicit, and the guidance separates copied cookies from verified sign-in. Error tables, platform limits, and the privacy rules are clear. Weaknesses: (1) the picker-versus-direct choice is underspecified, because the picker is used when browser, profile or domains are open and direct import when they are named, but the target-URL, `--all`, and verify-auth flows under each are not fully mapped. (2) The `--all` handling is thin. SKILL.md does not give direct-import syntax for it and does not say whether `$B goto` is still needed. (3) Verification setup has an ordering wrinkle: the env vars must be set before the daemon starts, while the Aside and `$B status` checks come first, and the instructions only partly say when to stop and restart the daemon. (4) A few edge cases are not defined, such as what to do when the reset or verification fails, how to count partial results, and how to proceed after the failure-reason handoff. (5) The multi-domain instruction is \"one --domain per run\", but it is unclear whether to repeat runs or how to aggregate the receipts. (6) The \"Windows App-Bound Encryption\" recovery depends on a display, with no alternative if none is available. These gaps are minor, and the workflow is mostly executable.\"}"
+    },
+    {
+      "id": "msg_011CgVne5GStZ6f9pRoCvgWE",
+      "model": "claude-sonnet-5-5",
+      "stop_reason": "end_turn",
+      "text": "{\"clarity\": 4, \"completeness\": 4, \"actionability\": 4, \"reasoning\": \"The workflow is well ordered: probe Aside, collect all decisions up front, set up the browser, import, then report three honest lines. Consent boundaries for verification, storage clearing and --all are explicit, the sign-in-versus-cookie-copy distinction is stated repeatedly, and the error and receipt tables give recovery paths. Gaps: (1) Step 2 asks for decisions before `$B` exists, yet the picker-versus-direct choice is tied to browser, profile and domains rather than to the user's wording, and the step 3.1 env vars come before the daemon starts. (2) The conditional step 3.3 (Mode: cdp) and the `$B stop` guidance could be cleaner. (3) The direct-import syntax for `--all` is not shown, and the \"one --domain per run\" rule is stated without saying what to do for several domains (loop, with the tab open on each). (4) The verification-missing return path says to return to step 3 but doesn't say how to handle the already-running daemon beyond `$B stop`. (5) Handling of partial and failed receipts, and the handoff for non-Windows cases, is thin. (6) How the agent learns the profile directory when the user only knows a display name is left to the picker. (7) The root-selection logic with \"never a generated host stub\" is a bit dense. These are minor; the workflow is mostly executable.\"}"
+    }
+  ]
+}

test/fixtures/judge-calibration/README.md

@@ -51,7 +51,11 @@ reservation; its 14 cases now send `WORKFLOW_JUDGE_RESPONSE_SCHEMA` without the
 compact-reasoning validator (still only on `ship`). The frontier (`review`) configuration has no corpus and stays on today's request. A $0.12 two-call pilot sized
 output tokens before the manifest was priced. Total W2 spend: $90.47 of $115.
 
-`cookie-workflow` (GSTA-91, separately funded) is held and stays on today's request. Its one
+`cookie-workflow` (GSTA-91, separately funded) was held, and the hold was released on
+GStack Minder's ruling of 2026-10-29: "GStack Minder, 2026-10-29: release the GSTA-91 hold; a07 is
+matched by old-b; free-text parse errors cost train 42" (`cookie-workflow/release.json`). The
+cookie workflow judge now sends `WORKFLOW_JUDGE_RESPONSE_SCHEMA`, with its thresholds, model and
+samples unchanged; the comparison below stays recorded as held. Its one
 new false pass, a07, scored 4/4/4 on all three new samples and on all three `old-b` samples;
 `old-a`'s a07 panel is incomplete because one old sample returned invalid JSON (the defect the
 schema transport removes), so the comparison counts it against the new request only. Both

test/fixtures/judge-calibration/cookie-workflow/release.json

@@ -0,0 +1,16 @@
+{
+  "config": "cookie-workflow",
+  "released": true,
+  "date": "2026-10-29",
+  "by": "GStack Minder",
+  "ruling": "GStack Minder, 2026-10-29: release the GSTA-91 hold; a07 is matched by old-b; free-text parse errors cost train 42",
+  "scope": "cookie workflow judge transport only",
+  "recorded_result": "runs/comparison-1.result.json",
+  "train_red": {
+    "gate_run": "https://github.com/garrytan/gstack/actions/runs/40809051510",
+    "responses": [
+      "msg_011CgVnRtVj5vyLvb56m6KJH",
+      "msg_011CgVne5GStZ6f9pRoCvgWE"
+    ]
+  }
+}

test/helpers/free-fixtures-data.ts

@@ -62,6 +62,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
   'test/fixtures/context-bill/**': ['test/context-bill.test.ts', 'test/context-budget-ratchet.test.ts'],
   'test/fixtures/context-budget.json': ['test/helpers/capture-context-budget.ts', 'test/test-value-bar.test.ts'],
   'test/fixtures/context-restore-fragment-match-captures.json': ['test/context-restore-fragment-verdict.test.ts'],
+  'test/fixtures/cookie-judge-train42-replies.json': ['test/cookie-workflow-judge-input.test.ts'],
   'test/fixtures/coverage-audit-cd-chain.json': ['test/coverage-audit-evidence.test.ts'],
   'test/fixtures/coverage-audit-parallel-column.json': ['test/coverage-audit-evidence.test.ts'],
   'test/fixtures/coverage-audit-separator-collision.json': ['test/coverage-audit-evidence.test.ts'],

test/judge-calibration.test.ts

@@ -321,17 +321,21 @@ describe('landed schemas match the recorded calibration', () => {
     expect(recorded.complete).toBe(true);
     const cookie = result('cookie-workflow');
     expect(cookie.complete).toBe(true);
+    expect(cookie.landing.lands).toBe(false);
+    const release = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures', 'judge-calibration', 'cookie-workflow', 'release.json'), 'utf8'));
+    expect(release).toMatchObject({ config: 'cookie-workflow', released: true, by: 'GStack Minder', scope: 'cookie workflow judge transport only',
+      ruling: 'GStack Minder, 2026-10-29: release the GSTA-91 hold; a07 is matched by old-b; free-text parse errors cost train 42' });
     const registrations = [...evalSource.matchAll(/await runWorkflowJudge\(\{([\s\S]*?)\n    \}\);/g)].map(match => match[1]!);
     expect(registrations.length).toBeGreaterThan(14);
     for (const body of registrations) {
       const testName = /testName: '([^']+)'/.exec(body)![1];
       const defaultConfig = !/^\s+(?:agentCapability|model|readInput|thresholds|maxTokens|stream|effort):|\.\.\.COOKIE_WORKFLOW_JUDGE/m.test(body);
       const expected = testName === 'ship/SKILL.md workflow' || (defaultConfig && recorded.landing.lands)
-        || (testName === 'setup-browser-cookies/SKILL.md workflow' && cookie.landing.lands);
+        || testName === 'setup-browser-cookies/SKILL.md workflow';
       expect(body.includes('schemaTransport: true'), testName).toBe(expected);
       expect(body.includes('compactReasoning: true'), testName).toBe(testName === 'ship/SKILL.md workflow');
     }
-    expect(registrations.filter(body => body.includes('schemaTransport: true'))).toHaveLength(cookie.landing.lands ? 16 : 15);
+    expect(registrations.filter(body => body.includes('schemaTransport: true'))).toHaveLength(16);
     const cookieBody = registrations.find(body => body.includes("testName: 'setup-browser-cookies/SKILL.md workflow'"))!;
     expect(cookieBody).toContain('...COOKIE_WORKFLOW_JUDGE');
     expect(cookieBody).not.toMatch(/^\s+(?:agentCapability|model|thresholds|maxTokens|stream|effort):/m);

test/skill-llm-eval.test.ts

@@ -792,6 +792,7 @@ describeIfSelected('Cookie setup workflow quality', ['setup-browser-cookies/SKIL
       startMarker: '# Setup Browser Cookies',
       endMarker: null,
       ...COOKIE_WORKFLOW_JUDGE,
+      schemaTransport: true,
       readInput: () => buildCookieWorkflowJudgeInput(ROOT),
     });
   }, WORKFLOW_JUDGE_TEST_MS);