diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 0fd0e3558..9a88765e0 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -137,7 +137,7 @@ export const E2E_TOUCHFILES: Record = { 'test/helpers/office-hours-attempt.ts'], // Office Hours - 'office-hours-spec-review': [ 'office-hours/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', + 'office-hours-spec-review': [ 'office-hours/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', 'bin/gstack-office-hours-review', 'lib/office-hours-review.ts', 'lib/fs-atomic.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/skill-fixture.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/workflow-judge-input.ts'], 'office-hours-forcing-energy': [ 'office-hours/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-office-hours.test.ts', 'test/helpers/office-hours-attempt.ts' @@ -147,26 +147,26 @@ export const E2E_TOUCHFILES: Record = { ], // Plan reviews - 'plan-ceo-review': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts', + 'plan-ceo-review': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], - 'plan-ceo-review-selective': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts', + 'plan-ceo-review-selective': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], - 'plan-ceo-review-benefits': [ 'plan-ceo-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', + 'plan-ceo-review-benefits': [ 'plan-ceo-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], - 'plan-ceo-review-expansion-energy': [ 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan.test.ts', + 'plan-ceo-review-expansion-energy': [ 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], 'plan-eng-review': [ 'scripts/resolvers/learnings.ts', - "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts', + "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts", 'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], 'plan-eng-review-artifact': [ 'scripts/resolvers/learnings.ts', - "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts', + "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts", 'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], 'plan-review-report': [ @@ -175,7 +175,7 @@ export const E2E_TOUCHFILES: Record = { 'test/fixtures/plan-review-report-public.json', 'scripts/resolvers/learnings.ts', - "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', + "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts", 'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], diff --git a/test/helpers/workflow-excerpt.ts b/test/helpers/workflow-excerpt.ts index 4ed2a48e1..838fbbed3 100644 --- a/test/helpers/workflow-excerpt.ts +++ b/test/helpers/workflow-excerpt.ts @@ -10,12 +10,22 @@ export const ENG_REVIEW_EXCERPT = { endMarker: '## Section self-check (before you finish)', } as const; +// The question-procedure heading in the plan-review sections, with or without +// its older "CRITICAL RULE — " prefix. +export const ASK_QUESTIONS_HEADING = /^## [^\n]*How to ask questions[^\n]*$/m; + +export function markerIndex(text: string, marker: string | RegExp, from: number): number { + if (typeof marker === 'string') return text.indexOf(marker, from); + const at = text.slice(from).search(marker); + return at < 0 ? -1 : from + at; +} + // Same generated two-line pointer consumed by setup-gbrain-fixture.ts. const STOP_POINTER = /^> \*\*STOP\.\*\* Before [^\n]*sections\/([a-z0-9-]+\.md)[^\n]*\n> in full\.[^\n]*/gm; /** Expand on-demand sections where the agent reads them, then take the requested excerpt. */ -export function readWorkflowExcerpt(skillPath: string, startMarker: string, endMarker: string | null): string { +export function readWorkflowExcerpt(skillPath: string, startMarker: string, endMarker: string | RegExp | null): string { const secDir = path.join(ROOT, path.dirname(skillPath), 'sections'); const content = fs.readFileSync(path.join(ROOT, skillPath), 'utf-8').replace(STOP_POINTER, (_pointer, file: string) => { const body = fs.readFileSync(path.join(secDir, file), 'utf-8') @@ -25,7 +35,7 @@ export function readWorkflowExcerpt(skillPath: string, startMarker: string, endM }); const start = content.indexOf(startMarker); if (start < 0) throw new Error(`Start marker not found in ${skillPath}: "${startMarker}"`); - const end = endMarker ? content.indexOf(endMarker, start) : content.length; + const end = endMarker ? markerIndex(content, endMarker, start) : content.length; if (end < 0) throw new Error(`End marker not found in ${skillPath}: "${endMarker}"`); return content.slice(start, end); } diff --git a/test/helpers/workflow-judge-cache.ts b/test/helpers/workflow-judge-cache.ts index 272fdcd2d..f1bd4024e 100644 --- a/test/helpers/workflow-judge-cache.ts +++ b/test/helpers/workflow-judge-cache.ts @@ -11,7 +11,7 @@ import { buildEvalInputIdentity, lookupEvalInputCache, sourceDependencyClosure, type Thresholds = { clarity: number; completeness: number; actionability: number }; export interface WorkflowCacheOptions { - root: string; testName: string; skillPath: string; startMarker: string; endMarker: string | null; + root: string; testName: string; skillPath: string; startMarker: string; endMarker: string | RegExp | null; judgeContext: string; judgeGoal: string; model?: string; thresholds: Thresholds; prompt: string; attempt: number; references?: readonly string[]; agentCapability?: 'frontier'; diff --git a/test/helpers/workflow-judge-input.ts b/test/helpers/workflow-judge-input.ts index 36873687b..f661022f1 100644 --- a/test/helpers/workflow-judge-input.ts +++ b/test/helpers/workflow-judge-input.ts @@ -1,6 +1,7 @@ /** Preserve source-file boundaries when a workflow judge reads carved skills. */ import * as fs from 'node:fs'; import * as path from 'node:path'; +import { markerIndex } from './workflow-excerpt'; export interface WorkflowJudgeFile { path: string; @@ -84,7 +85,7 @@ export function readWorkflowJudgeInput(opts: { root: string; skillPath: string; startMarker: string; - endMarker: string | null; + endMarker: string | RegExp | null; references?: readonly string[]; }): WorkflowJudgeInput { const sources = [{ @@ -115,7 +116,7 @@ export function readWorkflowJudgeInput(opts: { const union = allSources.map(file => file.content).join('\n'); const start = union.indexOf(opts.startMarker); if (start < 0) throw new Error(`Start marker not found in ${opts.skillPath}: "${opts.startMarker}"`); - const end = opts.endMarker === null ? union.length : union.indexOf(opts.endMarker, start); + const end = opts.endMarker === null ? union.length : markerIndex(union, opts.endMarker, start); if (end < 0) throw new Error(`End marker not found in ${opts.skillPath}: "${opts.endMarker}"`); const files: WorkflowJudgeFile[] = []; diff --git a/test/workflow-excerpt.test.ts b/test/workflow-excerpt.test.ts index 50d0090aa..0f93be40d 100644 --- a/test/workflow-excerpt.test.ts +++ b/test/workflow-excerpt.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'bun:test'; -import { ENG_REVIEW_EXCERPT, readWorkflowExcerpt } from './helpers/workflow-excerpt'; +import { ASK_QUESTIONS_HEADING, ENG_REVIEW_EXCERPT, readWorkflowExcerpt } from './helpers/workflow-excerpt'; import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs'; import { tmpdir } from 'os'; import { join } from 'path'; @@ -106,9 +106,9 @@ describe('workflow judge excerpts', () => { }); test('a sliced section is not appended again with its generated header', () => { - const text = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', '## CRITICAL RULE'); + const text = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', ASK_QUESTIONS_HEADING); expect(text.match(/## Review Sections/g)).toHaveLength(1); - expect(text).not.toContain('## CRITICAL RULE'); + expect(text).not.toMatch(ASK_QUESTIONS_HEADING); expect(text).not.toContain('AUTO-GENERATED'); }); @@ -293,7 +293,7 @@ console.log(JSON.stringify({calls, results})); expect(pendingDecision).toContain("**STOP until the actual answer arrives.**"); expect(pendingDecision.replace(/\s+/g, ' ')).toContain("Do not apply a remedy, make another call, start the next section or call ExitPlanMode while the choice awaits an answer"); expect(eng.replace(/\s+/g, ' ')).toContain("Use a scoped Edit to save this record and only the authorized working-plan amendments. Leave other choices unchanged"); - const design = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', '## CRITICAL RULE'); + const design = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', ASK_QUESTIONS_HEADING); expect(design).toContain('wait for approval, then edit the plan and re-rate'); const pass4 = design.slice(design.indexOf('### Pass 4:'), design.indexOf('### Pass 5:')); expect(pass4.match(/^### /gm)).toHaveLength(1);