From 6bfc352b325041e4674f2344723f63e3021b840e Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 2 Oct 2026 23:40:55 +0000 Subject: [PATCH 01/35] fix(browse): sidebar tab hint names real tabs.json fields and how $B resolves (s11 H1, s11 M8, s11 M9) --- browse/src/terminal-agent.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/browse/src/terminal-agent.ts b/browse/src/terminal-agent.ts index c5417bd38..88e89a719 100644 --- a/browse/src/terminal-agent.ts +++ b/browse/src/terminal-agent.ts @@ -301,11 +301,12 @@ function buildTabAwarenessHint(stateDir: string): string { 'You are running inside the gstack browser sidebar with live access to the user\'s browser tabs.', '', 'Tab state files (kept fresh automatically by the extension):', - ` ${tabsFile} — all open tabs (id, url, title, active, pinned)`, - ` ${activeFile} — the currently active tab`, - 'Read these any time the user asks about "tabs", "the current page", or anything multi-tab. Do NOT shell out to $B tabs just to learn what\'s open — read the file.', + ` ${tabsFile} — all open tabs: tabs[] of {tabId, url, title, active, pinned, windowId}`, + ` ${activeFile} — the currently active tab: {tabId, url, title}`, + 'Read these any time the user asks about "tabs", "the current page", or anything multi-tab. They\'re kept current, so read them instead of running $B tabs when you only need to know what\'s open.', '', 'Tab manipulation commands (via $B):', + ' ($B is the gstack browse CLI. This shell does not set it, so resolve it the way the /browse skill does: the project\'s .claude/skills/gstack/browse/dist/browse if present, else ~/.claude/skills/gstack/browse/dist/browse.)', ' $B tab — switch to a tab', ' $B newtab [url] — open a new tab', ' $B closetab [id] — close a tab (current if no id)', From 7a80bab143ec4f1bef408148c015ec74f4cfd16c Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 2 Oct 2026 23:42:46 +0000 Subject: [PATCH 02/35] test: anchor plan-design excerpts on the question-procedure heading, not '## CRITICAL RULE' (W7a judge-anchor move; workflow-excerpt.test.ts:109,296) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The end marker now accepts a RegExp. ASK_QUESTIONS_HEADING matches the heading with or without its 'CRITICAL RULE — ' prefix, so the excerpt is byte-identical on the current templates and survives the s04 F9 / s05 L3 rename. --- test/helpers/touchfiles-data.ts | 16 ++++++++-------- test/helpers/workflow-excerpt.ts | 14 ++++++++++++-- test/helpers/workflow-judge-cache.ts | 2 +- test/helpers/workflow-judge-input.ts | 5 +++-- test/workflow-excerpt.test.ts | 8 ++++---- 5 files changed, 28 insertions(+), 17 deletions(-) diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 0fd0e3558..9a88765e0 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -137,7 +137,7 @@ export const E2E_TOUCHFILES: Record = { 'test/helpers/office-hours-attempt.ts'], // Office Hours - 'office-hours-spec-review': [ 'office-hours/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', + 'office-hours-spec-review': [ 'office-hours/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', 'bin/gstack-office-hours-review', 'lib/office-hours-review.ts', 'lib/fs-atomic.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/skill-fixture.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/workflow-judge-input.ts'], 'office-hours-forcing-energy': [ 'office-hours/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-office-hours.test.ts', 'test/helpers/office-hours-attempt.ts' @@ -147,26 +147,26 @@ export const E2E_TOUCHFILES: Record = { ], // Plan reviews - 'plan-ceo-review': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts', + 'plan-ceo-review': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], - 'plan-ceo-review-selective': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts', + 'plan-ceo-review-selective': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], - 'plan-ceo-review-benefits': [ 'plan-ceo-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', + 'plan-ceo-review-benefits': [ 'plan-ceo-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], - 'plan-ceo-review-expansion-energy': [ 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan.test.ts', + 'plan-ceo-review-expansion-energy': [ 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], 'plan-eng-review': [ 'scripts/resolvers/learnings.ts', - "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts', + "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts", 'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], 'plan-eng-review-artifact': [ 'scripts/resolvers/learnings.ts', - "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts', + "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts", 'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], 'plan-review-report': [ @@ -175,7 +175,7 @@ export const E2E_TOUCHFILES: Record = { 'test/fixtures/plan-review-report-public.json', 'scripts/resolvers/learnings.ts', - "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', + "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts", 'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], diff --git a/test/helpers/workflow-excerpt.ts b/test/helpers/workflow-excerpt.ts index 4ed2a48e1..838fbbed3 100644 --- a/test/helpers/workflow-excerpt.ts +++ b/test/helpers/workflow-excerpt.ts @@ -10,12 +10,22 @@ export const ENG_REVIEW_EXCERPT = { endMarker: '## Section self-check (before you finish)', } as const; +// The question-procedure heading in the plan-review sections, with or without +// its older "CRITICAL RULE — " prefix. +export const ASK_QUESTIONS_HEADING = /^## [^\n]*How to ask questions[^\n]*$/m; + +export function markerIndex(text: string, marker: string | RegExp, from: number): number { + if (typeof marker === 'string') return text.indexOf(marker, from); + const at = text.slice(from).search(marker); + return at < 0 ? -1 : from + at; +} + // Same generated two-line pointer consumed by setup-gbrain-fixture.ts. const STOP_POINTER = /^> \*\*STOP\.\*\* Before [^\n]*sections\/([a-z0-9-]+\.md)[^\n]*\n> in full\.[^\n]*/gm; /** Expand on-demand sections where the agent reads them, then take the requested excerpt. */ -export function readWorkflowExcerpt(skillPath: string, startMarker: string, endMarker: string | null): string { +export function readWorkflowExcerpt(skillPath: string, startMarker: string, endMarker: string | RegExp | null): string { const secDir = path.join(ROOT, path.dirname(skillPath), 'sections'); const content = fs.readFileSync(path.join(ROOT, skillPath), 'utf-8').replace(STOP_POINTER, (_pointer, file: string) => { const body = fs.readFileSync(path.join(secDir, file), 'utf-8') @@ -25,7 +35,7 @@ export function readWorkflowExcerpt(skillPath: string, startMarker: string, endM }); const start = content.indexOf(startMarker); if (start < 0) throw new Error(`Start marker not found in ${skillPath}: "${startMarker}"`); - const end = endMarker ? content.indexOf(endMarker, start) : content.length; + const end = endMarker ? markerIndex(content, endMarker, start) : content.length; if (end < 0) throw new Error(`End marker not found in ${skillPath}: "${endMarker}"`); return content.slice(start, end); } diff --git a/test/helpers/workflow-judge-cache.ts b/test/helpers/workflow-judge-cache.ts index 272fdcd2d..f1bd4024e 100644 --- a/test/helpers/workflow-judge-cache.ts +++ b/test/helpers/workflow-judge-cache.ts @@ -11,7 +11,7 @@ import { buildEvalInputIdentity, lookupEvalInputCache, sourceDependencyClosure, type Thresholds = { clarity: number; completeness: number; actionability: number }; export interface WorkflowCacheOptions { - root: string; testName: string; skillPath: string; startMarker: string; endMarker: string | null; + root: string; testName: string; skillPath: string; startMarker: string; endMarker: string | RegExp | null; judgeContext: string; judgeGoal: string; model?: string; thresholds: Thresholds; prompt: string; attempt: number; references?: readonly string[]; agentCapability?: 'frontier'; diff --git a/test/helpers/workflow-judge-input.ts b/test/helpers/workflow-judge-input.ts index 36873687b..f661022f1 100644 --- a/test/helpers/workflow-judge-input.ts +++ b/test/helpers/workflow-judge-input.ts @@ -1,6 +1,7 @@ /** Preserve source-file boundaries when a workflow judge reads carved skills. */ import * as fs from 'node:fs'; import * as path from 'node:path'; +import { markerIndex } from './workflow-excerpt'; export interface WorkflowJudgeFile { path: string; @@ -84,7 +85,7 @@ export function readWorkflowJudgeInput(opts: { root: string; skillPath: string; startMarker: string; - endMarker: string | null; + endMarker: string | RegExp | null; references?: readonly string[]; }): WorkflowJudgeInput { const sources = [{ @@ -115,7 +116,7 @@ export function readWorkflowJudgeInput(opts: { const union = allSources.map(file => file.content).join('\n'); const start = union.indexOf(opts.startMarker); if (start < 0) throw new Error(`Start marker not found in ${opts.skillPath}: "${opts.startMarker}"`); - const end = opts.endMarker === null ? union.length : union.indexOf(opts.endMarker, start); + const end = opts.endMarker === null ? union.length : markerIndex(union, opts.endMarker, start); if (end < 0) throw new Error(`End marker not found in ${opts.skillPath}: "${opts.endMarker}"`); const files: WorkflowJudgeFile[] = []; diff --git a/test/workflow-excerpt.test.ts b/test/workflow-excerpt.test.ts index 50d0090aa..0f93be40d 100644 --- a/test/workflow-excerpt.test.ts +++ b/test/workflow-excerpt.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'bun:test'; -import { ENG_REVIEW_EXCERPT, readWorkflowExcerpt } from './helpers/workflow-excerpt'; +import { ASK_QUESTIONS_HEADING, ENG_REVIEW_EXCERPT, readWorkflowExcerpt } from './helpers/workflow-excerpt'; import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs'; import { tmpdir } from 'os'; import { join } from 'path'; @@ -106,9 +106,9 @@ describe('workflow judge excerpts', () => { }); test('a sliced section is not appended again with its generated header', () => { - const text = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', '## CRITICAL RULE'); + const text = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', ASK_QUESTIONS_HEADING); expect(text.match(/## Review Sections/g)).toHaveLength(1); - expect(text).not.toContain('## CRITICAL RULE'); + expect(text).not.toMatch(ASK_QUESTIONS_HEADING); expect(text).not.toContain('AUTO-GENERATED'); }); @@ -293,7 +293,7 @@ console.log(JSON.stringify({calls, results})); expect(pendingDecision).toContain("**STOP until the actual answer arrives.**"); expect(pendingDecision.replace(/\s+/g, ' ')).toContain("Do not apply a remedy, make another call, start the next section or call ExitPlanMode while the choice awaits an answer"); expect(eng.replace(/\s+/g, ' ')).toContain("Use a scoped Edit to save this record and only the authorized working-plan amendments. Leave other choices unchanged"); - const design = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', '## CRITICAL RULE'); + const design = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', ASK_QUESTIONS_HEADING); expect(design).toContain('wait for approval, then edit the plan and re-rate'); const pass4 = design.slice(design.indexOf('### Pass 4:'), design.indexOf('### Pass 5:')); expect(pass4.match(/^### /gm)).toHaveLength(1); From c4d133128c33ad7e11629a5d6988735204cbd9bd Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 2 Oct 2026 23:42:52 +0000 Subject: [PATCH 03/35] test: plan-design judge-input case anchors on the question-procedure heading (W7a judge-anchor move; workflow-judge-input.test.ts:366,373) --- test/workflow-judge-input.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/workflow-judge-input.test.ts b/test/workflow-judge-input.test.ts index 47ff4b3a7..4b47d50da 100644 --- a/test/workflow-judge-input.test.ts +++ b/test/workflow-judge-input.test.ts @@ -5,7 +5,7 @@ import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { createHash } from 'node:crypto'; import { readWorkflowJudgeInput, buildWorkflowJudgePrompt, QA_DISCOVERY_REFERENCES, WORKFLOW_JUDGE_RESPONSE_SCHEMA, WORKFLOW_JUDGE_REASONING_WORD_LIMIT } from './helpers/workflow-judge-input'; -import { ENG_REVIEW_EXCERPT } from './helpers/workflow-excerpt'; +import { ASK_QUESTIONS_HEADING, ENG_REVIEW_EXCERPT } from './helpers/workflow-excerpt'; const ROOT = resolve(import.meta.dir, '..'); const scratchRoots: string[] = []; @@ -363,14 +363,14 @@ describe('workflow judge file bundle', () => { test('generated plan-design passes retain their full section without duplicating Pass 1', () => { const input = readWorkflowJudgeInput({ - root: ROOT, skillPath: 'plan-design-review/SKILL.md', startMarker: '## Review Sections', endMarker: '## CRITICAL RULE', + root: ROOT, skillPath: 'plan-design-review/SKILL.md', startMarker: '## Review Sections', endMarker: ASK_QUESTIONS_HEADING, }); expect(input.files.filter(file => file.kind === 'entrypoint')).toHaveLength(0); expect(input.files.map(file => file.path)).toEqual(sectionPaths('plan-design-review')); const section = input.files.find(file => file.path === 'plan-design-review/sections/review-sections.md'); expect(section?.content).toBe(readFileSync(join(ROOT, 'plan-design-review/sections/review-sections.md'), 'utf8')); expect(section?.content).toStartWith('\n/gm, '').trim(); From 99720e744ee7810f0ba1da2cd78db44ee8315ea8 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 00:23:29 +0000 Subject: [PATCH 26/35] test: CEO plan-mode failure message names the current rule, not the '## CRITICAL RULE' heading (t13 F20) --- test/skill-e2e-plan-ceo-plan-mode.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/skill-e2e-plan-ceo-plan-mode.test.ts b/test/skill-e2e-plan-ceo-plan-mode.test.ts index 39f5f37ec..e3d27cd84 100644 --- a/test/skill-e2e-plan-ceo-plan-mode.test.ts +++ b/test/skill-e2e-plan-ceo-plan-mode.test.ts @@ -99,8 +99,7 @@ describeE2E('plan-ceo-review plan-mode smoke (gate)', () => { `plan-ceo-review smoke FAILED: outcome=${obs.outcome}\n` + `${diagnosis}\n` + `Expected 'asked'. See plan-ceo-review/SKILL.md.tmpl: the Step 0 STOP rules ` + - `and the "One issue = one AskUserQuestion call" rule under "CRITICAL RULE — ` + - `How to ask questions".\n` + + `and the "One decision unit = one AskUserQuestion call" rule in its question-procedure section.\n` + `summary: ${obs.summary}\n` + `elapsed: ${obs.elapsedMs}ms\n` + `--- evidence (last 2KB visible) ---\n${obs.evidence}`, From 0725d99d99c3e331da9dbc7c556591ef1706e16f Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 00:25:09 +0000 Subject: [PATCH 27/35] test: browse reference judge gates on its stated floors; the baseline comparison is recorded, not a ratchet (W7d browse judge baseline ratchet; t08 F7) expect(regressions).toEqual([]) compared a noisy three-sample mean against eval-baselines.json (clarity 4), silently raising the stated clarity floor of 3. The comparison stays in judge_reasoning; pass and assertions use BROWSE_JUDGE_FLOORS (clarity 3, completeness 4, actionability 4). Judge prompt and schema are unchanged. Stored panel means: three known-good (two the ratchet rejected) pass; three below-floor panels fail. --- test/browse-judge-floors.test.ts | 13 +++++++++++++ test/fixtures/browse-judge/panel-means.json | 13 +++++++++++++ test/helpers/workflow-judge-cache.ts | 12 ++++++++++++ test/skill-llm-eval.test.ts | 15 +++++++-------- 4 files changed, 45 insertions(+), 8 deletions(-) create mode 100644 test/browse-judge-floors.test.ts create mode 100644 test/fixtures/browse-judge/panel-means.json diff --git a/test/browse-judge-floors.test.ts b/test/browse-judge-floors.test.ts new file mode 100644 index 000000000..53a1a160f --- /dev/null +++ b/test/browse-judge-floors.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; +import { browseJudgeFloorsMet } from './helpers/workflow-judge-cache'; + +type Scores = { clarity: number; completeness: number; actionability: number }; +const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/browse-judge/panel-means.json'), 'utf8')) as + { known_good: Record; known_bad: Record }; + +describe('browse judge floors', () => { + test.each(Object.entries(stored.known_good))('passes %s', (_name, scores) => expect(browseJudgeFloorsMet(scores)).toBe(true)); + test.each(Object.entries(stored.known_bad))('fails %s', (_name, scores) => expect(browseJudgeFloorsMet(scores)).toBe(false)); +}); diff --git a/test/fixtures/browse-judge/panel-means.json b/test/fixtures/browse-judge/panel-means.json new file mode 100644 index 000000000..ad4864e77 --- /dev/null +++ b/test/fixtures/browse-judge/panel-means.json @@ -0,0 +1,13 @@ +{ + "_comment": "Browse reference judge panel means against baseline {clarity:4, completeness:4, actionability:4}. known_good passes the floors (the old ratchet failed the first two); known_bad fails them.", + "known_good": { + "clarity-dip-to-3.67": { "clarity": 3.67, "completeness": 4, "actionability": 4 }, + "clarity-at-floor": { "clarity": 3, "completeness": 4.33, "actionability": 4 }, + "at-baseline": { "clarity": 4, "completeness": 4, "actionability": 4 } + }, + "known_bad": { + "clarity-below-floor": { "clarity": 2.67, "completeness": 4, "actionability": 4 }, + "completeness-below-floor": { "clarity": 4, "completeness": 3.67, "actionability": 4 }, + "actionability-below-floor": { "clarity": 4, "completeness": 4, "actionability": 3.33 } + } +} diff --git a/test/helpers/workflow-judge-cache.ts b/test/helpers/workflow-judge-cache.ts index f1bd4024e..3a6ad000f 100644 --- a/test/helpers/workflow-judge-cache.ts +++ b/test/helpers/workflow-judge-cache.ts @@ -135,3 +135,15 @@ export function prepareWorkflowJudgeCache(opts: WorkflowCacheOptions): { }, }; } + +/** + * Pass floors for the browse reference judge panel mean. The stored baseline + * is recorded for comparison only: a three-sample mean is too noisy for a + * no-dip ratchet, and gating on it silently raised the clarity floor to 4. + */ +export const BROWSE_JUDGE_FLOORS = { clarity: 3, completeness: 4, actionability: 4 } as const; + +export function browseJudgeFloorsMet(scores: Record): boolean { + return (Object.keys(BROWSE_JUDGE_FLOORS) as Array) + .every(dim => scores[dim] >= BROWSE_JUDGE_FLOORS[dim]); +} diff --git a/test/skill-llm-eval.test.ts b/test/skill-llm-eval.test.ts index cb6465e0d..40a91a410 100644 --- a/test/skill-llm-eval.test.ts +++ b/test/skill-llm-eval.test.ts @@ -18,7 +18,7 @@ import { callJudge, judge, JudgeRefusalError, DEFAULT_JUDGE_MAX_TOKENS, judgePan import { ASK_QUESTIONS_HEADING, ENG_REVIEW_EXCERPT } from './helpers/workflow-excerpt'; import type { JudgeScore } from './helpers/llm-judge'; import { readWorkflowJudgeInput, buildWorkflowJudgePrompt, QA_DISCOVERY_REFERENCES, WORKFLOW_JUDGE_RESPONSE_SCHEMA, type WorkflowJudgeInput } from './helpers/workflow-judge-input'; -import { prepareWorkflowJudgeCache, validWorkflowJudgeScore } from './helpers/workflow-judge-cache'; +import { BROWSE_JUDGE_FLOORS, browseJudgeFloorsMet, prepareWorkflowJudgeCache, validWorkflowJudgeScore } from './helpers/workflow-judge-cache'; import { buildCookieWorkflowJudgeInput, COOKIE_WORKFLOW_JUDGE } from './helpers/cookie-workflow-judge-input'; import { getCookieWorkflowManualReview, type ManualJudgeReview } from './helpers/cookie-workflow-manual-review'; import { resolveEvalModel } from '../lib/eval-model'; @@ -96,8 +96,8 @@ describeIfSelected('LLM-as-judge quality evals', [ const t0 = Date.now(); // Browse carve: snapshot flags + the full command list are the whole // generated section file; one judge grades the union. Scores are also - // pinned against test/fixtures/eval-baselines.json (UPDATE_BASELINES=1 - // rewrites the pin). + // compared with test/fixtures/eval-baselines.json and the difference is + // recorded (UPDATE_BASELINES=1 rewrites it); only the floors gate. const section = sliceBrowseSection('## Snapshot Flags'); const samples = await judgePanel(() => judge('browse skill reference (flags + commands)', section)); @@ -119,17 +119,16 @@ describeIfSelected('LLM-as-judge quality evals', [ name: 'browse/SKILL.md reference', suite: 'LLM-as-judge quality evals', tier: 'llm-judge', - passed: scores.clarity >= 3 && scores.completeness >= 4 && scores.actionability >= 4 && regressions.length === 0, + passed: browseJudgeFloorsMet(scores), duration_ms: Date.now() - t0, cost_usd: 0.02 * samples.length, judge_scores: { clarity: scores.clarity, completeness: scores.completeness, actionability: scores.actionability }, judge_reasoning: regressions.length ? `${judgePanelReasoning(samples)} | ${regressions.join('; ')}` : judgePanelReasoning(samples), }); - expect(scores.clarity).toBeGreaterThanOrEqual(3); - expect(scores.completeness).toBeGreaterThanOrEqual(4); - expect(scores.actionability).toBeGreaterThanOrEqual(4); - expect(regressions).toEqual([]); + expect(scores.clarity).toBeGreaterThanOrEqual(BROWSE_JUDGE_FLOORS.clarity); + expect(scores.completeness).toBeGreaterThanOrEqual(BROWSE_JUDGE_FLOORS.completeness); + expect(scores.actionability).toBeGreaterThanOrEqual(BROWSE_JUDGE_FLOORS.actionability); }, JUDGE_MS); testIfSelected('setup block', async () => { From f43fe0de13f29908c5eab69e26639757d18c074d Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 00:28:22 +0000 Subject: [PATCH 28/35] test: eval-kinds drops the exact 22 behavior / 25 judge census (W7d exact count in eval-kinds; t12 F6) The kind contract is already asserted: every live case has one kind, kinds are rule/behavior/judge, every LLM judge is judge-kind, every behavior case has a literal registration and a BEHAVIOR_WHY, and the named contract-shaped cases stay rule. The census count only broke on adding or retiring a case. --- test/eval-kinds.test.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/eval-kinds.test.ts b/test/eval-kinds.test.ts index ea6480f6c..d385ac044 100644 --- a/test/eval-kinds.test.ts +++ b/test/eval-kinds.test.ts @@ -58,9 +58,7 @@ describe('E2E_KINDS registry', () => { } }); - test('the classification is the reviewed one: rule by default, 22 behavior, 25 judge', () => { - const counts = Object.values(E2E_KINDS).reduce>((acc, kind) => ({ ...acc, [kind]: (acc[kind] ?? 0) + 1 }), {}); - expect(counts).toEqual({ rule: liveIds.length - 22 - 25, behavior: 22, judge: 25 }); + test('contract-shaped cases stay rule-kind', () => { // Contract-shaped cases stay rule: ask-before-decide, plan-mode no-writes, // mandated steps, secrets, and the batching floor never ride a majority. for (const id of ['plan-ceo-mode-routing', 'plan-eng-multi-finding-batching', 'plan-design-review-plan-mode', From d1e3eadff8efe7c6cf33f73862ccf8c7816ab998 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 00:31:02 +0000 Subject: [PATCH 29/35] test: plan-eng artifact case binds to the hermetic state root and requires the artifact; report case drops its shouted requirement (W7g plan-artifact fixture binding; W7g skill-e2e-plan.test.ts:479,720) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - plan-eng-review-artifact: actor, observer and cleanup use one isolated GSTACK_HOME (the hermetic run's, or a seeded temporary one when hermetic mode is off) and the slug gstack-slug resolves for the fixture, instead of os.homedir()/.gstack/projects/test-project. A guard refuses a state root under the real home, and cleanup removes only the fixture's project directory. The 'IMPORTANT … you MUST write the test-plan artifact' line is replaced by the environment fact (state root and slug). The soft warning becomes an assertion: exactly one new *eng-review-test-plan-*.md about the reviewed dashboard change. - plan-review-report: removes 'CRITICAL REQUIREMENT … MOST IMPORTANT deliverable' and the restated report contract; keeps 'plan.md is the plan file'. The existing assertions (report present, last, rows, unresolved status) now measure the skill's own report prose. --- test/skill-e2e-plan.test.ts | 87 +++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 46 deletions(-) diff --git a/test/skill-e2e-plan.test.ts b/test/skill-e2e-plan.test.ts index d9aa25f84..cb91b20f8 100644 --- a/test/skill-e2e-plan.test.ts +++ b/test/skill-e2e-plan.test.ts @@ -1,6 +1,7 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; +import { getHermeticDirs, isHermeticEnabled, seedHermeticGstackHome } from './helpers/hermetic-env'; import { EvalCollector } from './helpers/eval-store'; import { OFFICE_HOURS_BUN_GRACE_MS, runRecordedOfficeHoursAttempt } from './helpers/office-hours-attempt'; import { @@ -379,6 +380,9 @@ Focus on architecture, code quality, tests, and performance sections.`, describeIfSelected('Plan-Eng-Review Test-Plan Artifact E2E', ['plan-eng-review-artifact'], () => { let planDir: string; + let gstackHome: string; + let ownsGstackHome = false; + let slug: string; let projectDir: string; beforeAll(() => { @@ -438,35 +442,38 @@ export function main() { return Dashboard(); } // Set up remote-slug shim and browse shims (plan-eng-review uses remote-slug for artifact path) setupBrowseShims(planDir); - // Create project directory for artifacts - projectDir = path.join(os.homedir(), '.gstack', 'projects', 'test-project'); - fs.mkdirSync(projectDir, { recursive: true }); - - // Clean up stale test-plan files from previous runs - try { - const staleFiles = fs.readdirSync(projectDir).filter(f => f.includes('test-plan')); - for (const f of staleFiles) { - fs.unlinkSync(path.join(projectDir, f)); - } - } catch {} + // The actor, this observer and cleanup share one isolated state root: the + // hermetic run's GSTACK_HOME (or a seeded temporary one when hermetic mode + // is off), and the slug gstack-slug resolves for this fixture. Nothing is + // read or deleted under the operator's real home. + if (isHermeticEnabled()) gstackHome = getHermeticDirs().gstackHome; + else { + gstackHome = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-plan-artifact-home-')); + seedHermeticGstackHome(gstackHome); + ownsGstackHome = true; + } + const slugOut = spawnSync(path.join(ROOT, 'bin', 'gstack-slug'), [], { + cwd: planDir, stdio: 'pipe', timeout: 5000, env: { ...process.env, GSTACK_HOME: gstackHome }, + }).stdout.toString(); + slug = /^SLUG=(.+)$/m.exec(slugOut)?.[1]?.trim() ?? ''; + if (!slug) throw new Error(`plan-eng-review-artifact: gstack-slug resolved no slug: ${slugOut}`); + projectDir = path.join(gstackHome, 'projects', slug); + const realHome = path.resolve(os.homedir()); + if (projectDir === realHome || projectDir.startsWith(realHome + path.sep) && !projectDir.startsWith(path.resolve(os.tmpdir()) + path.sep)) { + throw new Error(`plan-eng-review-artifact: state root ${projectDir} is under the real home directory`); + } }); afterAll(() => { try { fs.rmSync(planDir, { recursive: true, force: true }); } catch {} - // Clean up test-plan artifacts (but not the project dir itself) - try { - const files = fs.readdirSync(projectDir); - for (const f of files) { - if (f.includes('test-plan')) { - fs.unlinkSync(path.join(projectDir, f)); - } - } - } catch {} + // Remove only this fixture's project directory inside the isolated state root. + try { fs.rmSync(ownsGstackHome ? gstackHome : projectDir, { recursive: true, force: true }); } catch {} }); testConcurrentIfSelected('plan-eng-review-artifact', async () => { - // Count existing test-plan files before - const beforeFiles = fs.readdirSync(projectDir).filter(f => f.includes('test-plan')); + const testPlans = () => fs.existsSync(projectDir) + ? fs.readdirSync(projectDir).filter(f => /eng-review-test-plan-.*\.md$/.test(f)) : []; + const beforeFiles = testPlans(); const result = await runSkillTest({ prompt: `Read plan-eng-review/SKILL.md for the review workflow. @@ -476,10 +483,11 @@ Read plan.md — that's the plan to review. This is a standalone plan with sourc Proceed directly to the full review. Skip any AskUserQuestion calls — this is non-interactive. -IMPORTANT: After your review, you MUST write the test-plan artifact as described in the "Test Plan Artifact" section of SKILL.md. The remote-slug shim is at ${planDir}/browse/bin/remote-slug. +In this fixture the gstack state root is GSTACK_HOME=${gstackHome} and the project slug is ${slug}. Write your review to ${planDir}/review-output.md`, workingDirectory: planDir, + env: { GSTACK_HOME: gstackHome }, maxTurns: 25, allowedTools: ['Bash', 'Read', 'Write', 'Glob', 'Grep'], timeout: CAPTURE_LONG_MS, @@ -489,30 +497,19 @@ Write your review to ${planDir}/review-output.md`, }); logCost('/plan-eng-review artifact', result); + // The QA test plan is a required discovery artifact of the review: exactly + // one new file under this fixture's project directory, about this plan. + const newFiles = testPlans().filter(f => !beforeFiles.includes(f)); + const content = newFiles.length === 1 ? fs.readFileSync(path.join(projectDir, newFiles[0]!), 'utf-8') : ''; + const aboutPlan = /dashboard|fetchStats|\/api\/stats/i.test(content); + console.log(`Test-plan artifacts in ${projectDir}: ${newFiles.length} new${newFiles.length ? ` (${newFiles[0]}, ${content.length} chars)` : ''}`); recordE2E(evalCollector, '/plan-eng-review test-plan artifact', 'Plan-Eng-Review Test-Plan Artifact E2E', result, { - passed: ['success', 'error_max_turns'].includes(result.exitReason), + passed: ['success', 'error_max_turns'].includes(result.exitReason) && newFiles.length === 1 && aboutPlan, }); expect(['success', 'error_max_turns']).toContain(result.exitReason); - - // Verify test-plan artifact was written - const afterFiles = fs.readdirSync(projectDir).filter(f => f.includes('test-plan')); - const newFiles = afterFiles.filter(f => !beforeFiles.includes(f)); - console.log(`Test-plan artifacts: ${beforeFiles.length} before, ${afterFiles.length} after, ${newFiles.length} new`); - - if (newFiles.length > 0) { - const content = fs.readFileSync(path.join(projectDir, newFiles[0]), 'utf-8'); - console.log(`Test-plan artifact (${newFiles[0]}): ${content.length} chars`); - expect(content.length).toBeGreaterThan(50); - } else { - console.warn('No test-plan artifact found — agent may not have followed artifact instructions'); - } - - // Soft assertion: we expect an artifact but agent compliance is not guaranteed. - // Log rather than fail — the test-plan artifact is a bonus output, not the core test. - if (newFiles.length === 0) { - console.warn('SOFT FAIL: No test-plan artifact written — agent did not follow artifact instructions'); - } + expect(newFiles, `expected one new eng-review test plan in ${projectDir}`).toHaveLength(1); + expect(aboutPlan, 'the test plan covers the reviewed dashboard change').toBe(true); }, CAPTURE_LONG_MS); }); @@ -717,9 +714,7 @@ Read plan.md — that's the plan to review. This is a standalone plan document, Proceed directly to the full review. Skip any AskUserQuestion calls — this is non-interactive. Skip the preamble bash block, lake intro, telemetry, and contributor mode sections. -CRITICAL REQUIREMENT: plan.md IS the plan file for this review session. After completing your review, you MUST write a "## GSTACK REVIEW REPORT" section to the END of plan.md, exactly as described in the "Plan File Review Report" section of plan-eng-review/sections/review-sections.md. Use that canonical table, with all five review rows and honest not-run entries when review history is unavailable. The report MUST end with the mandatory unresolved-decisions status as its final line — the exact unbolded line NO UNRESOLVED DECISIONS when nothing is open, or a "**UNRESOLVED DECISIONS:**" block of bullets when items remain. Nothing may follow it. Use the Edit tool to append to plan.md — do NOT overwrite the existing plan content. - -This review report at the bottom of the plan is the MOST IMPORTANT deliverable of this test.`, +plan.md is the plan file for this review session.`, workingDirectory: planDir, maxTurns: 20, timeout: CAPTURE_LONG_MS, From 903c46f3d435bf8c707e8ad2d2bbf71c33be95fd Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 00:32:00 +0000 Subject: [PATCH 30/35] test: session-runner-tools checks harness prompt properties, not a byte-for-byte copy (W7e session-runner-tools.test.ts:544; t09 F7, F8) The ship capture prompt test asserts its purpose (no CEO-only writing guidance) plus the functional rules: one allowed skill path, AskUserQuestion unavailable, the section Read rule, no git/gh/mutation, and the REPORT.md path. The two 'MUST actually Read' pins match the Read rule case-insensitively. Negative control: dropping the no-mutation rule from the harness prompt turns the test red. --- test/session-runner-tools.test.ts | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/test/session-runner-tools.test.ts b/test/session-runner-tools.test.ts index 580899ee5..395e93d97 100644 --- a/test/session-runner-tools.test.ts +++ b/test/session-runner-tools.test.ts @@ -431,7 +431,7 @@ describe.skipIf(process.platform === 'win32')('session-runner explicit tool avai expect(child.prompt).toContain('Preserve every finding and original requirement, required decision fields and comparisons, exact approvals and verification; every diagram must retain its specified format.'); expect(child.prompt).toContain('unless needed to specify an accepted change'); expect(child.prompt).toContain('Complete every required artifact and verification before returning'); - expect(child.prompt).toContain('MUST actually Read that sections/ file with the Read tool BEFORE doing the work it covers'); + expect(child.prompt).toMatch(/read that sections\/ file with the Read tool before doing the work it covers/i); expect(child.prompt).toContain('report outside coverage as disabled'); expect(child.prompt).toContain('After all required writes are complete'); expect(child.prompt).toContain('with Write/Edit at the workflow checkpoints below'); @@ -527,7 +527,7 @@ describe.skipIf(process.platform === 'win32')('session-runner explicit tool avai expect(child.prompt).toContain('do not hide it or claim approval when no offered alternative meets these constraints'); expect(child.prompt).not.toContain("silently pick the skill's recommended option"); expect(child.prompt).toContain('Every required section, finding, approval and output still has to be completed'); - expect(child.prompt).toContain('MUST actually Read that sections/ file with the Read tool BEFORE doing the work it covers'); + expect(child.prompt).toMatch(/read that sections\/ file with the Read tool before doing the work it covers/i); expect(child.prompt).toContain('After all required writes are complete'); expect(flagValue(child.args, '--tools')).toBe('Read,Grep,Glob,Write,Edit,Agent,Bash'); expect(fs.readFileSync(path.join(dir, 'PLAN.md'), 'utf8')).toBe(fixtures['PLAN.md']); @@ -548,20 +548,19 @@ describe.skipIf(process.platform === 'win32')('session-runner explicit tool avai planDir: dir, skillName: 'ship', scenario, testName: 'ship-report-writing', timeout: 5_000, }); const skillPath = path.join(dir, 'ship', 'SKILL.md'); - expect(observed().prompt).toBe(`You are running an automated skill-execution test. No human is present, so AskUserQuestion is unavailable. The ONLY skill file you may read is this absolute path: ${skillPath}. Do NOT Glob/find/search for any other SKILL.md anywhere — especially nothing under ~/.claude or /Users. - -Read ${skillPath} and EXECUTE its workflow for this scenario: - -${scenario} - -Rules for this run: -- Skip system-audit, environment-setup, telemetry, and unrelated codebase exploration. Read the supplied plan's referenced fixture files when its review requires them. -- At any decision point that would call AskUserQuestion, silently pick the skill's recommended option and continue. Do NOT stop to ask. -- This skill's body has been carved into on-demand sections/. When the skill gives a STOP-Read directive (for example "Read \`.../sections/\` and execute it in full"), you MUST actually Read that sections/ file with the Read tool BEFORE doing the work it covers. Do not work from memory. -- Resolve installed-root paths for section and companion Markdown files under ${dir}, where this fixture's skill package is copied. -- Do NOT run git, gh, commit, push, or any mutating command. -- When the workflow is complete, write the skill's final output (the full review report / ship plan, including any required report table) to ${path.join(dir, 'REPORT.md')}. -- After all required writes are complete, return a brief completion message and STOP. Do not reproduce the full report in the final response.`); + const prompt = observed().prompt; + // CEO-only writing guidance stays out of another skill's capture prompt. + for (const ceoOnly of ['all 11 sections', 'Preserve original requirements and accepted plan amendments', + 'Cross-reference saved IDs', 'report outside coverage as disabled', 'with Write/Edit at the workflow checkpoints']) { + expect(prompt).not.toContain(ceoOnly); + } + // The functional harness rules, checked by meaning rather than bytes. + expect(prompt).toContain(scenario); + expect(prompt.match(/\/[^\s]*SKILL\.md/g)?.every(file => file === skillPath)).toBe(true); + expect(prompt).toMatch(/AskUserQuestion is unavailable/i); + expect(prompt).toMatch(/read that sections\/ file with the Read tool before/i); + expect(prompt).toMatch(/do not run git, gh, commit, push, or any mutating command/i); + expect(prompt).toContain(path.join(dir, 'REPORT.md')); }); }); From 89627151767ea6638936b02593d5a206f4bf4772 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 00:33:37 +0000 Subject: [PATCH 31/35] test: section-capture native tools checks harness prompt properties, not a copied golden (W7e section-capture-native-tools.test.ts:467; t04 F4) Asserts the absolute skill path is the only SKILL.md named, the scenario, AskUserQuestion unavailable, the section Read rule (case-insensitive, also at :308), no git/gh/mutation, and the REPORT.md path. argv/tool-list checks are unchanged. Negative control: dropping the no-mutation rule turns it red. --- test/section-capture-native-tools.test.ts | 29 ++++++++++------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/test/section-capture-native-tools.test.ts b/test/section-capture-native-tools.test.ts index 8f3e48c0e..1ec7b464b 100644 --- a/test/section-capture-native-tools.test.ts +++ b/test/section-capture-native-tools.test.ts @@ -303,9 +303,9 @@ console.log(JSON.stringify({plain:plain.exitReason,literal:literal.exitReason,se expect(added).toBeGreaterThan(-1); expect(args(1)[added + 1]).toBe(literal); expect(args(1).filter((_, i) => i !== added && i !== added + 1)).toEqual(args(0)); const sectionArgs = args(2); - const instruction = launches[2].prompt.split('\n').find((line: string) => line.includes('with the Read tool BEFORE')); + const instruction = launches[2].prompt.split('\n').find((line: string) => /with the Read tool before/i.test(line)); expect(sectionArgs).not.toContain('--append-system-prompt'); - expect(instruction).toContain('you MUST actually Read that sections/ file with the Read tool BEFORE doing the work it covers'); + expect(instruction).toMatch(/read that sections\/ file with the Read tool before doing the work it covers/i); expect(instruction).not.toContain('actual.md'); expect(instruction).not.toContain(dir); expect(sectionArgs.slice(sectionArgs.indexOf('--allowed-tools') + 1, sectionArgs.indexOf('--allowed-tools') + 8)) .toEqual(['Read', 'Grep', 'Glob', 'Write', 'Edit', 'Agent', 'Bash']); @@ -464,21 +464,16 @@ console.log(JSON.stringify(observed));`); expect(request).not.toHaveProperty('appendSystemPrompt'); const skill=['fixture','ship','office-hours'][Math.floor((i-4)/2)]!; const skillPath=path.join(dir,skill,'SKILL.md'); - const expected=`You are running an automated skill-execution test. No human is present, so AskUserQuestion is unavailable. The ONLY skill file you may read is this absolute path: ${skillPath}. Do NOT Glob/find/search for any other SKILL.md anywhere — especially nothing under ~/.claude or /Users. - -Read ${skillPath} and EXECUTE its workflow for this scenario: - -Complete the supplied scenario. - -Rules for this run: -- Skip system-audit, environment-setup, telemetry, and unrelated codebase exploration. Read the supplied plan's referenced fixture files when its review requires them. -- At any decision point that would call AskUserQuestion, silently pick the skill's recommended option and continue. Do NOT stop to ask. -- This skill's body has been carved into on-demand sections/. When the skill gives a STOP-Read directive (for example "Read \`.../sections/\` and execute it in full"), you MUST actually Read that sections/ file with the Read tool BEFORE doing the work it covers. Do not work from memory. -- Resolve installed-root paths for section and companion Markdown files under ${dir}, where this fixture's skill package is copied. -- Do NOT run git, gh, commit, push, or any mutating command. -- When the workflow is complete, write the skill's final output (the full review report / ship plan, including any required report table) to ${path.join(dir,'REPORT.md')}. -- After all required writes are complete, return a brief completion message and STOP. Do not reproduce the full report in the final response.`; - expect(request.prompt).toBe(expected); + // Harness properties, not its wording: the one allowed skill path, the + // scenario, AskUserQuestion fallback, section Read rule, no mutation, + // and the report destination. + expect(request.prompt.match(/\/[^\s]*SKILL\.md/g)?.every((file: string)=>file===skillPath)).toBe(true); + expect(request.prompt).toContain(`Read ${skillPath} and EXECUTE its workflow`); + expect(request.prompt).toContain('Complete the supplied scenario.'); + expect(request.prompt).toMatch(/AskUserQuestion is unavailable/i); + expect(request.prompt).toMatch(/read that sections\/ file with the Read tool before/i); + expect(request.prompt).toMatch(/do not run git, gh, commit, push, or any mutating command/i); + expect(request.prompt).toContain(path.join(dir,'REPORT.md')); } } }finally{if(child.exitCode===null)child.kill();await child.exited;fs.rmSync(dir,{recursive:true,force:true});} From 744e5a2392744eaa7e8c405ee3c5f149b153146f Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 00:33:54 +0000 Subject: [PATCH 32/35] test: office-hours review-preservation judge test keeps its schema and untrusted-data checks, not its wording (W7e office-hours-completion.test.ts:416-429; t09 F10) Drops 13 phrase pins on the judge prompt. Kept: both inputs carried as JSON, the response schema (wire format), and the separate untrusted-DATA test. The validator behavior tests already use stub judges. Negative control: changing the schema string in the prompt builder turns the test red. --- test/office-hours-completion.test.ts | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/test/office-hours-completion.test.ts b/test/office-hours-completion.test.ts index 05c8fc09d..e16685d7c 100644 --- a/test/office-hours-completion.test.ts +++ b/test/office-hours-completion.test.ts @@ -413,20 +413,8 @@ describe('office-hours reviewer finding preservation', () => { calls++; expect(prompt).toContain(JSON.stringify(original)); expect(prompt).toContain(JSON.stringify(review!.concerns)); - expect(prompt).toContain('Do not require verbatim wording or judge by issue count alone'); - expect(prompt).toContain('optional alternative remedies may be omitted'); - expect(prompt).toContain('only actual defects, never successful mappings'); - expect(prompt).toContain('Keep reasoning brief'); - expect(prompt).toContain('Complete EVERY applicable audit even if another already fails'); - expect(prompt).toContain('a coverage failure cannot skip metrics'); - expect(prompt).toContain('a metrics failure cannot skip convergence'); - expect(prompt).toContain('one brief conclusion per applicable audit'); - expect(prompt).toContain('APPROVED records user approval and is compatible with CONCERNS_RECORDED'); + // The response schema is the wire contract; the judge's wording is not. expect(prompt).toContain('{"reasoning":string,"missing":string[],"unsupported":string[],"complete":boolean}'); - expect(prompt).toContain('finalize missing and unsupported, and emit complete last'); - expect(prompt).toContain('at most 150 words total'); - expect(prompt).toContain('Coverage and the remaining inventory come from the FINAL reviewer verdict only'); - expect(prompt).toContain('lack of a confirmed prior fix does not invent a missing final-verdict finding'); return JSON.stringify(accepted); }); expect(calls).toBe(1); From a31c0a7ece7d592ab7b69775530eb253f10f8bb4 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 00:34:22 +0000 Subject: [PATCH 33/35] test: docsync lifecycle interface drops pins on its own helper prompt (W7e docsync-lifecycle-interface.test.ts; t02 #17) docsNativeInterface is the test helper's own prompt, so its sentence pins only checked that the helper says what it says. Kept: owned skill path, the two extracted lifecycle commands (exact), the scripts-list interpolation, the repo path and Git read examples, and every docsCommandAllowed / docsToolFailures behavior check that enforces the interface. --- test/docsync-lifecycle-interface.test.ts | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/test/docsync-lifecycle-interface.test.ts b/test/docsync-lifecycle-interface.test.ts index 1b2687da0..08cc41278 100644 --- a/test/docsync-lifecycle-interface.test.ts +++ b/test/docsync-lifecycle-interface.test.ts @@ -17,9 +17,6 @@ test('fixture lifecycle guidance exposes owned skill paths and literal commands' const guidance = docsNativeInterface(fixture); const skills = fixture.skills.split(path.sep).join('/'); expect(guidance).toContain(`${skills}/document-release/SKILL.md`); - expect(guidance).toContain('instead of copying the generated shell wrappers'); - expect(guidance).toContain("satisfy the skill's start/end lifecycle requirements here"); - expect(guidance).toContain('applies to parent and every child; include this interface in child prompts'); const [start, end] = lifecycleCommands(); expect(lifecycleCommands()).toHaveLength(2); expect(start).toBe(`GSTACK_SESSION_KIND=spawned ${skills}/bin/gstack-skill-start --skill document-release --model claude`); @@ -33,11 +30,6 @@ test('fixture lifecycle guidance exposes owned skill paths and literal commands' }); test.each(['success', 'error', 'abort', 'unknown'])('same-session literal end accepts outcome %s without wrappers', outcome => { - const guidance = docsNativeInterface(fixture); - expect(guidance).toContain('actual literal values echoed by that same start call'); - expect(guidance).toContain('Never execute the placeholders or reuse values from another session'); - expect(guidance).toContain('If start fails or SESSION_KIND is not spawned, report the blocker'); - expect(guidance).toContain('do not suppress an error or claim completion if it failed'); const end = lifecycleCommands()[1].replace('OUTCOME', outcome).replace('SESSION_ID_VALUE', '123-1790299423-control').replace('TEL_START_VALUE', '1790299423'); expect(docsCommandAllowed(end, fixture)).toBe(true); const result = { toolCalls: [{ tool: 'Bash', input: { command: end }, output: '' }] } as SkillTestResult; @@ -49,7 +41,6 @@ test('historical misplaced spawned prefix remains rejected while corrected pream const misplaced = `GSTACK_SESSION_KIND=spawned ${original}`; expect(docsCommandAllowed(misplaced, fixture)).toBe(false); expect(docsCommandAllowed(spawned, fixture)).toBe(true); - expect(docsNativeInterface(fixture)).toContain('prefix belongs directly on the helper invocation, not on a preceding assignment'); }); test.each([ @@ -95,21 +86,17 @@ test.each([ }); test('lifecycle instructions neither widen command authority nor grant mutation approvals', () => { - const guidance = docsNativeInterface(fixture); - expect(guidance).toContain('The only additional scripts are none'); - expect(guidance).toContain('do not grant any additional scripts, write paths or risk approvals'); - expect(guidance).toContain('do not rewrite installed skills, config, actor state or scripts'); + expect(docsNativeInterface(fixture)).toContain('The only additional scripts are none'); + expect(docsNativeInterface(fixture, ['/owned/publish.ts'])).toContain('The only additional scripts are /owned/publish.ts'); for (const suffix of [' 2>/dev/null', ' || true', ' && git status', '\ntrue', ' $EXTRA']) { expect(docsCommandAllowed(lifecycleCommands()[0] + suffix, fixture)).toBe(false); } expect(docsCommandAllowed('bun arbitrary.ts', fixture)).toBe(false); }); -test('Git guidance uses the existing working directory without authorizing global options', () => { +test('Git guidance uses the existing working directory; --git-dir and --work-tree stay unauthorized', () => { const guidance = docsNativeInterface(fixture); - expect(guidance).toContain(`working directory for parent and child Bash calls is already ${fixture.repo}`); - expect(guidance).toContain('literal git subcommand must immediately follow git'); - expect(guidance).toContain('does not make git -C an allowed command'); + expect(guidance).toContain(fixture.repo); for (const command of ['git status', 'git diff --cached', 'git merge-base main HEAD', 'git rev-parse HEAD']) { expect(guidance).toContain(command); expect(docsCommandAllowed(command, fixture)).toBe(true); From c2d3182f24d163901591257cb45c112352938167 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 00:38:00 +0000 Subject: [PATCH 34/35] test: shared-libs revalidation prompt keeps its machine-shaped checks, not ~69 phrase pins (W7e shared-libs-revalidation-prompt; t11 #3, #8; t05 F17) reviewRevalidationPrompt and reviewPrompt are test-owned harness prompts (non-test callers: 0); the paid shared-libs E2Es measure whether they work. Kept: the three exact receipt commands, base-prompt prefix and resumed suffix identity, branch selection (actor / default / resumed coverage), the state path and turn-budget interpolations, read-record-before-diff and first-response-before-start ordering, no embedded token, no supplied Skip answer, reusable:true before reusable:false, the captured R59 stage boundaries in the exclusion list, asset roots and interfaces, and the stale step-6 reference ban. The review template's Step 4.7/4.8 completion rule is matched by step number on one line, not by sentence. Negative control: reordering the diff-read instruction turns the test red. --- test/shared-libs-revalidation-prompt.test.ts | 93 +++++--------------- 1 file changed, 20 insertions(+), 73 deletions(-) diff --git a/test/shared-libs-revalidation-prompt.test.ts b/test/shared-libs-revalidation-prompt.test.ts index c2ad0f2d9..2ce0b016c 100644 --- a/test/shared-libs-revalidation-prompt.test.ts +++ b/test/shared-libs-revalidation-prompt.test.ts @@ -71,11 +71,13 @@ function pathCaptureAdapter(capture: (...args: any[]) => Promise) { } describe('bounded shared-code revalidation prompt', () => { + // The harness prompt's wording is not a contract: the paid shared-libs E2Es + // measure whether it works. These checks keep its machine-shaped parts: + // exact receipt commands, interpolated paths and budgets, branch selection, + // prefix/suffix identity, ordering and the absence of a supplied token. test('the actor scope replaces the captured R59 exploratory stage instead of adding another prerequisite', () => { const prompt = reviewPrompt(f, instructions, input, { actorCommand: 'cat /fixture/stage-output.json' }); - expect(prompt).toContain('replaces the entire Step 4.7 QA and Step 4.8 native adversarial stages'); - expect(prompt).toContain("Step 4's early QA selection/method-loading prerequisites"); - expect(prompt).toContain("Step 5.8's QA report requirement"); + expect(prompt).toContain('cat /fixture/stage-output.json'); const excluded = prompt.slice(prompt.indexOf('Do not perform QA scope/method asset loads'), prompt.indexOf('Existing tests')); for (const packet of stageScope.outside_component) { expect(packet.result.tool_use_id).toBe(packet.call.id); @@ -85,26 +87,15 @@ describe('bounded shared-code revalidation prompt', () => { } expect(stageScope.post_fix_verification.call.input.command).toContain('bun test test/retry-after.test.ts'); expect(stageScope.post_fix_verification.result.content).toContain('worker===lib true route===lib true'); - expect(prompt).toContain('Existing tests and caller/import checks needed to verify your source fixes still run'); - expect(prompt).toContain('do not restart exploratory QA or require QA artifacts'); - for (const retained of ['core/checklist', 'source/identity/snapshot checks', 'Fix-First decisions', 'approved source edits', - 're-review with a new REVIEW_START', 'zero-edit convergence', 'final persistence', 'Missing, failed, stale or wrong-state results require noncompletion', - 'no actual native coverage credit', 'Separate genuine QA/native evaluations remain required']) expect(prompt).toContain(retained); for (const other of [reviewPrompt(f, instructions, input), reviewRevalidationPrompt(f, instructions, input), reviewRevalidationPrompt(f, instructions, input, { input: '/fixture/resumed.json', checkCommand: 'check-prerequisites' })]) { expect(other).not.toContain('Do not perform QA scope/method asset loads'); } }); - test('edit-capable replay declares fresh actor invocations instead of refreshing settled input', () => { + test('edit-capable replay declares its actor instead of the fixed revalidation coverage', () => { const prompt = reviewPrompt(f, instructions, input, { actorCommand: 'cat /fixture/stage-output.json' }); - expect(prompt).toContain('explicitly declared SYNTHETIC prerequisite actor'); - for (const rule of ['each review pass', 'NEW synthetic result', 'exact current state and tool-use ID', - 'All prior receipts are preserved', 'Source-changing cycles invalidate earlier results', - 'invoke the actor again on the new zero-edit pass', "Never refresh an old receipt's hashes", - 'Missing, failed, stale or wrong-state results require noncompletion', 'cannot complete core/checklist review', - 'only a current settled:true actor result from the final pass supplies the replaced Step 4.7 QA and Step 4.8 native adversarial prerequisites', - 'a reporting label, not a missing stage', 'no actual native coverage credit']) expect(prompt).toContain(rule); + expect(prompt).toContain('cat /fixture/stage-output.json'); expect(prompt).not.toContain('Required reviewer coverage for this scoped replay'); expect(prompt).not.toContain('Do not edit target source'); }); @@ -113,19 +104,15 @@ describe('bounded shared-code revalidation prompt', () => { const resumed = { input: '/isolated/synthetic-prerequisites.json', checkCommand: 'fixture-prerequisite-check' }; const original = reviewRevalidationPrompt(f, instructions, input); const prompt = reviewRevalidationPrompt(f, instructions, input, resumed); - expect(original).toContain('Required reviewer coverage for this scoped replay is the core/checklist review plus the supplied completed maintainability result.'); + expect(original).toContain('Required reviewer coverage for this scoped replay'); expect(prompt).not.toContain('Required reviewer coverage for this scoped replay'); - expect(prompt).toContain('SYNTHETIC settled Step 4.7 QA and Step 4.8 native adversarial'); - for (const requirement of ['not evidence that this model executed those stages', 'never actual native coverage credit', - 'Missing, failed, blocked, malformed or stale prerequisites require noncompletion', 'unchanged COMPLETED and CONVERGED rules', - 'Any source, branch, base, index or configuration change invalidates', 'do not regenerate them', - 'A finding that requires edits blocks this bounded replay', resumed.input, resumed.checkCommand, - "replace Step 4's early QA selection and method-loading prerequisites", 'read no QA scope or method assets']) expect(prompt).toContain(requirement); - expect(original).not.toContain('read no QA scope or method assets'); + expect(prompt).toContain(resumed.input); + expect(prompt).toContain(resumed.checkCommand); expect(prompt.slice(prompt.indexOf('Revalidation fixture execution contract:'))) .toBe(original.slice(original.indexOf('Revalidation fixture execution contract:'))); + // The review completion rule names both prerequisite stages by step number. const production = fs.readFileSync(path.join(SHARED_LIBS_ROOT, 'review/SKILL.md.tmpl'), 'utf8'); - expect(production).toContain('Step 4.8 adversarial pass finish, and every required Step 4.7 probe passes.'); + expect(production.split('\n').some(line => /Step 4\.8\b/.test(line) && /Step 4\.7\b/.test(line))).toBe(true); }); test('adds execution guidance after the complete shared prompt without supplying an answer or token', () => { @@ -135,64 +122,32 @@ describe('bounded shared-code revalidation prompt', () => { const contract = prompt.slice(base.length); expect(contract).toContain(`${SHARED_INTERACTIVE_MAX_TURNS} assistant turns`); expect(contract).toContain(path.join(f.state, 'projects/fixture-shared-libs/.review-starts/.json')); - expect(contract).toContain('token actually returned by --start'); - expect(contract).toContain('separate, successful Read tool call or a single cat command'); - expect(contract).toContain('Verify its repo, branch, working tree and start time'); - expect(contract).toContain('Do not combine the record read with --start, the diff or other diagnostic commands'); - expect(contract).toContain('if the read fails, retry it before proceeding'); - expect(contract).toContain('Do not read the diff until step 2 verifies the start record'); + expect(contract.indexOf('Read that token\'s record')).toBeGreaterThan(-1); expect(contract.indexOf('Read that token\'s record')).toBeLessThan(contract.indexOf('Then read the diff in a subsequent call')); expect(contract).not.toMatch(/[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}/); - expect(prompt).toContain(`${SHARED_LIBS_ROOT}/review/sections/shared-code-reuse.md (Step 5.0 requires it for the supplied prior Skip)`); + expect(prompt).toContain(`${SHARED_LIBS_ROOT}/review/sections/shared-code-reuse.md`); expect(prompt.indexOf('Your first response holds')).toBeLessThan(prompt.indexOf("gstack-review-log' --start review")); - expect(prompt).toContain('Keep the final review summary to at most twelve lines'); - expect(base).not.toContain('Keep the final review summary'); - expect(contract).toContain('Batch independent required source reads'); - expect(contract).toContain('Preserve every required evidence check and dependency'); - expect(contract).toContain('complete, untruncated read-back'); - expect(contract).toContain('same tool invocation'); - expect(contract).toContain('then return the final review summary'); - expect(contract).toContain('Failed persistence or verification remains a failure'); - expect(contract).toContain("Late source changes still require the workflow's normal re-review"); expect(contract).not.toContain('choose Skip'); }); - test('declares isolated receipt commands, direct reads and a separate supplied-finding disposition', () => { + test('declares isolated receipt commands', () => { const prompt = reviewRevalidationPrompt(f, instructions, input); const commands = [...prompt.matchAll(/```bash\n([\s\S]*?)\n```/g)].map(match => match[1]); expect(commands).toHaveLength(3); expect(commands[0]).toBe(`'${SHARED_LIBS_ROOT}/bin/gstack-review-log' --start review`); expect(commands[1]).toBe(`'${SHARED_LIBS_ROOT}/bin/gstack-review-log' --check-shared-libs REVIEW_START <<'GSTACK_REVALIDATION_FINDING'\nCURRENT_FINDING_JSON\nGSTACK_REVALIDATION_FINDING`); expect(commands[2]).toBe(`'${SHARED_LIBS_ROOT}/bin/gstack-review-log' 'FINAL_REVIEW_JSON' --finish REVIEW_START && '${SHARED_LIBS_ROOT}/bin/gstack-review-read'`); - for (const requirement of ['sole command', 'only the token', 'only one JSON value', 'literal path operands', - 'even when its body appeared in the diff', 'No path-variable loops', 'before the checker', - 'metadata in earlier calls', 'No preliminary commands', 'materially revised proposal is a separate finding', - 'unsupported or unfinished supplied finding stays blocked', 'without its actual explicit decision']) { - expect(prompt).toContain(requirement); - } }); - test('step 5 branches on the checker result: true suppresses without a new decision, false requires a fresh one', () => { + test('step 5 branches on the checker result, reusable:true before reusable:false', () => { const prompt = reviewRevalidationPrompt(f, instructions, input); const step5 = prompt.slice(prompt.indexOf('5. Act on the checker result'), prompt.indexOf('6. Complete final evidence')); - expect(step5).toContain('reusable:true'); - expect(step5).toContain('the prior Skip carries forward'); - expect(step5).toContain('Ask no new decision question'); - expect(step5).toContain("exclude this advisory from the current pass's findings"); - expect(step5).toContain('Do not re-persist it as a current finding'); - expect(step5).toContain('reusable:false'); - expect(step5).toContain('Perform a fresh authored-source review and make an actual new decision'); - expect(step5).toContain('materially revised proposal is a separate finding'); - expect(step5).toContain('without its actual explicit decision'); - expect(step5).toContain('unsupported or unfinished supplied finding stays blocked'); - expect(step5).toContain('reusable:true whose independent authored/current-source verification does not hold'); - expect(step5).toContain('regardless of the checker result'); - expect(step5).toContain('never force a new Skip on invalid evidence'); + expect(step5.indexOf('reusable:true')).toBeGreaterThan(-1); expect(step5.indexOf('reusable:true')).toBeLessThan(step5.indexOf('reusable:false')); expect(prompt).not.toContain("Record the supplied finding's disposition under its own"); }); - test('the shared review prompt maps exact trusted asset roots and documented interfaces to avoid discovery', () => { + test('the shared review prompt maps exact trusted asset roots and documented interfaces', () => { const prompt = reviewPrompt(f, instructions, input); expect(prompt).toContain(`${SHARED_LIBS_ROOT}/review/checklist.md`); expect(prompt).toContain(`${SHARED_LIBS_ROOT}/review/sections/`); @@ -202,21 +157,13 @@ describe('bounded shared-code revalidation prompt', () => { expect(prompt).toContain(f.bin); for (const iface of ['gstack-review-log --start review', '--check-shared-libs REVIEW_START', '--finish REVIEW_START', 'gstack-review-read']) expect(prompt).toContain(iface); - for (const forbidden of ['do not rediscover it', 'enumerate the bin/lib/review/qa roots', - 'probe --help', 'read the fixture request logs']) expect(prompt).toContain(forbidden); - expect(prompt).toContain('batch independent reads'); - expect(prompt).toContain('keep receipt-ordered commands separate'); - expect(prompt).toContain('capture the start token before reading the diff'); - expect(prompt).toContain('run --start, the checker and any declared stage-actor invocation each as its own sole command'); - expect(prompt).toContain('The only combined receipt call is the final persistence'); - expect(prompt).toContain('Still inspect the target repository source'); }); - test('common guidance states the finish+read-back receipt contract directly, not by a dangling step 6 reference', () => { + test('common guidance states the finish+read-back receipt directly, not by a dangling step 6 reference', () => { const lifecycle = reviewPrompt(f, instructions, input, { actorCommand: 'bun /fx/stage-actor.ts run' }); const revalidation = reviewRevalidationPrompt(f, instructions, input); for (const prompt of [lifecycle, revalidation]) { - expect(prompt).toContain('The only combined receipt call is the final persistence'); + expect(prompt).toContain('--finish REVIEW_START'); expect(prompt).not.toContain('exactly as step 6 shows'); } const commands = [...revalidation.matchAll(/```bash\n([\s\S]*?)\n```/g)].map(match => match[1]); From e28ef17040be942335fb72c67184f46bb2891d6a Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 00:38:58 +0000 Subject: [PATCH 35/35] test: plan-design SDK fixture pins its harness prompt once, by ordered properties (W7e plan-design-sdk-fixture.test.ts:91-120; t01 F13) The prompt was pinned three ways (sha256, ~20 sentences, an order check). Kept: the skill and lazy-section paths, the plan path, the ordered read -> review -> edit -> Write -> read-back -> summary steps, and the no-browsing rule (case-insensitive). Dropped the hash and sentence pins. Negative control: removing the read-back step turns it red. --- test/plan-design-sdk-fixture.test.ts | 31 ++++++---------------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/test/plan-design-sdk-fixture.test.ts b/test/plan-design-sdk-fixture.test.ts index f4ae51f99..c42e04e72 100644 --- a/test/plan-design-sdk-fixture.test.ts +++ b/test/plan-design-sdk-fixture.test.ts @@ -87,37 +87,18 @@ async function exercise(mode: 'success' | 'max-turns' | 'first-timeout' | 'secon expect(opts.publicStreamDiagnostics).toBe(true); expect(opts.signal).toBeInstanceOf(AbortSignal); sdkSignal = opts.signal; expect(opts.signal.aborted).toBe(false); - // Bind the complete actual compact-delivery prompt, not selected snippets. - expect(new Bun.CryptoHasher('sha256').update(opts.prompt).digest('hex')) - .toBe('021c9f2b0522fc55bc265a976356678a048f3c17d27b63669674c805a485b28e'); expect(opts.testName).toBe(id); expect(opts.maxTurns).toBe(15); expect(opts.timeout).toBe(CAPTURE_MS); for (const key of ['model', 'tools', 'allowedTools', 'appendSystemPrompt', 'env']) expect(opts).not.toHaveProperty(key); - expect(opts.prompt).toContain('Review the plan in ./plan.md'); - expect(opts.prompt).toContain('Review all 7 design passes'); - expect(opts.prompt).toContain('0-10 and explain what would make it a 10'); - expect(opts.prompt).toContain('preserve the unresolved-decisions pass'); - expect(opts.prompt).toContain('interaction state table, empty states, responsive behavior'); - expect(opts.prompt).toContain('full required review report'); - expect(opts.prompt).toContain('Write before publishing a completed walkthrough'); - expect(opts.prompt).toContain('Read plan.md back to verify the saved changes'); - expect(opts.prompt).toContain('Then return a brief, concrete summary'); - expect(opts.prompt).toContain('natively Read plan-design-review/SKILL.md'); - expect(opts.prompt).toContain('plan-design-review/sections/review-sections.md (the one lazy section this review requires)'); - expect(opts.prompt).toContain('including the report, to about 10,000 characters; this is a drafting target, not a check, so do not count characters or trim after saving'); - expect(opts.prompt).toContain('in at most ten lines'); - expect(opts.prompt).toContain('execute every required pass and lazy-section Read'); - expect(opts.prompt).toContain('Retain all required report fields, design decisions, diagrams, ratings, and explanations'); - expect(opts.prompt).toContain('use the canonical tables and decision IDs'); - expect(opts.prompt).toContain('Specify each design requirement once'); - expect(opts.prompt).toContain('instead of repeating that specification'); - expect(opts.prompt).toContain('concise score rationales and 10/10 explanations'); + // The harness prompt's properties, in order; its wording is free. + expect(opts.prompt).toContain('plan-design-review/SKILL.md'); + expect(opts.prompt).toContain('plan-design-review/sections/review-sections.md'); + expect(opts.prompt).toContain('./plan.md'); const ordered = ['Read every lazy section', 'Review all 7 design passes', - 'EDIT plan.md', 'Keep the saved review compact', - 'Persist that complete plan and review with Write', 'Read plan.md back', + 'EDIT plan.md', 'Persist that complete plan and review with Write', 'Read plan.md back', 'Then return a brief, concrete summary'].map(text => opts.prompt.indexOf(text)); expect(ordered.every(index => index >= 0)).toBe(true); expect(ordered).toEqual([...ordered].sort((a, b) => a - b)); - expect(opts.prompt).toContain('Do NOT try to browse any URLs'); + expect(opts.prompt).toMatch(/do not try to browse any URLs/i); inputVerified = true; launched = true; if (mode === 'attempt-deadline') return await new Promise((_resolve, reject) => { opts.signal.addEventListener('abort', () => reject(opts.signal.reason), { once: true });