test: anchor plan-design excerpts on the question-procedure heading, not '## CRITICAL RULE' (W7a judge-anchor move; workflow-excerpt.test.ts:109,296)

The end marker now accepts a RegExp. ASK_QUESTIONS_HEADING matches the heading
with or without its 'CRITICAL RULE — ' prefix, so the excerpt is byte-identical
on the current templates and survives the s04 F9 / s05 L3 rename.
This commit is contained in:
garrytan committed 2026-10-03 00:16:37 +00:00
1 parent 6bfc352b32
commit 7a80bab143
5 files changed
+28 -17

No files matched your search

+8 -8
View File
@@ -137,7 +137,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
'test/helpers/office-hours-attempt.ts'],
// Office Hours
'office-hours-spec-review': [ 'office-hours/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts',
'office-hours-spec-review': [ 'office-hours/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts',
'bin/gstack-office-hours-review', 'lib/office-hours-review.ts', 'lib/fs-atomic.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/skill-fixture.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/workflow-judge-input.ts'],
'office-hours-forcing-energy': [ 'office-hours/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-office-hours.test.ts',
'test/helpers/office-hours-attempt.ts'
@@ -147,26 +147,26 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
],
// Plan reviews
'plan-ceo-review': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts',
'plan-ceo-review': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts',
'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
'plan-ceo-review-selective': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts',
'plan-ceo-review-selective': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts',
'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
'plan-ceo-review-benefits': [ 'plan-ceo-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts',
'plan-ceo-review-benefits': [ 'plan-ceo-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts',
'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
'plan-ceo-review-expansion-energy': [ 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan.test.ts',
'plan-ceo-review-expansion-energy': [ 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts',
'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
'plan-eng-review': [
'scripts/resolvers/learnings.ts',
"test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts',
"test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts',
"scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts",
'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
'plan-eng-review-artifact': [
'scripts/resolvers/learnings.ts',
"test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts',
"test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts',
"scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts",
'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
'plan-review-report': [
@@ -175,7 +175,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
'test/fixtures/plan-review-report-public.json',
'scripts/resolvers/learnings.ts',
"test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts',
"test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts',
"scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts",
'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
+12 -2
View File
@@ -10,12 +10,22 @@ export const ENG_REVIEW_EXCERPT = {
endMarker: '## Section self-check (before you finish)',
} as const;
// The question-procedure heading in the plan-review sections, with or without
// its older "CRITICAL RULE — " prefix.
export const ASK_QUESTIONS_HEADING = /^## [^\n]*How to ask questions[^\n]*$/m;
export function markerIndex(text: string, marker: string | RegExp, from: number): number {
if (typeof marker === 'string') return text.indexOf(marker, from);
const at = text.slice(from).search(marker);
return at < 0 ? -1 : from + at;
}
// Same generated two-line pointer consumed by setup-gbrain-fixture.ts.
const STOP_POINTER =
/^> \*\*STOP\.\*\* Before [^\n]*sections\/([a-z0-9-]+\.md)[^\n]*\n> in full\.[^\n]*/gm;
/** Expand on-demand sections where the agent reads them, then take the requested excerpt. */
export function readWorkflowExcerpt(skillPath: string, startMarker: string, endMarker: string | null): string {
export function readWorkflowExcerpt(skillPath: string, startMarker: string, endMarker: string | RegExp | null): string {
const secDir = path.join(ROOT, path.dirname(skillPath), 'sections');
const content = fs.readFileSync(path.join(ROOT, skillPath), 'utf-8').replace(STOP_POINTER, (_pointer, file: string) => {
const body = fs.readFileSync(path.join(secDir, file), 'utf-8')
@@ -25,7 +35,7 @@ export function readWorkflowExcerpt(skillPath: string, startMarker: string, endM
});
const start = content.indexOf(startMarker);
if (start < 0) throw new Error(`Start marker not found in ${skillPath}: "${startMarker}"`);
const end = endMarker ? content.indexOf(endMarker, start) : content.length;
const end = endMarker ? markerIndex(content, endMarker, start) : content.length;
if (end < 0) throw new Error(`End marker not found in ${skillPath}: "${endMarker}"`);
return content.slice(start, end);
}
+1 -1
View File
@@ -11,7 +11,7 @@ import { buildEvalInputIdentity, lookupEvalInputCache, sourceDependencyClosure,
type Thresholds = { clarity: number; completeness: number; actionability: number };
export interface WorkflowCacheOptions {
root: string; testName: string; skillPath: string; startMarker: string; endMarker: string | null;
root: string; testName: string; skillPath: string; startMarker: string; endMarker: string | RegExp | null;
judgeContext: string; judgeGoal: string; model?: string; thresholds: Thresholds; prompt: string; attempt: number;
references?: readonly string[];
agentCapability?: 'frontier';
+3 -2
View File
@@ -1,6 +1,7 @@
/** Preserve source-file boundaries when a workflow judge reads carved skills. */
import * as fs from 'node:fs';
import * as path from 'node:path';
import { markerIndex } from './workflow-excerpt';
export interface WorkflowJudgeFile {
path: string;
@@ -84,7 +85,7 @@ export function readWorkflowJudgeInput(opts: {
root: string;
skillPath: string;
startMarker: string;
endMarker: string | null;
endMarker: string | RegExp | null;
references?: readonly string[];
}): WorkflowJudgeInput {
const sources = [{
@@ -115,7 +116,7 @@ export function readWorkflowJudgeInput(opts: {
const union = allSources.map(file => file.content).join('\n');
const start = union.indexOf(opts.startMarker);
if (start < 0) throw new Error(`Start marker not found in ${opts.skillPath}: "${opts.startMarker}"`);
const end = opts.endMarker === null ? union.length : union.indexOf(opts.endMarker, start);
const end = opts.endMarker === null ? union.length : markerIndex(union, opts.endMarker, start);
if (end < 0) throw new Error(`End marker not found in ${opts.skillPath}: "${opts.endMarker}"`);
const files: WorkflowJudgeFile[] = [];
+4 -4
View File
@@ -1,5 +1,5 @@
import { describe, expect, test } from 'bun:test';
import { ENG_REVIEW_EXCERPT, readWorkflowExcerpt } from './helpers/workflow-excerpt';
import { ASK_QUESTIONS_HEADING, ENG_REVIEW_EXCERPT, readWorkflowExcerpt } from './helpers/workflow-excerpt';
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
@@ -106,9 +106,9 @@ describe('workflow judge excerpts', () => {
});
test('a sliced section is not appended again with its generated header', () => {
const text = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', '## CRITICAL RULE');
const text = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', ASK_QUESTIONS_HEADING);
expect(text.match(/## Review Sections/g)).toHaveLength(1);
expect(text).not.toContain('## CRITICAL RULE');
expect(text).not.toMatch(ASK_QUESTIONS_HEADING);
expect(text).not.toContain('AUTO-GENERATED');
});
@@ -293,7 +293,7 @@ console.log(JSON.stringify({calls, results}));
expect(pendingDecision).toContain("**STOP until the actual answer arrives.**");
expect(pendingDecision.replace(/\s+/g, ' ')).toContain("Do not apply a remedy, make another call, start the next section or call ExitPlanMode while the choice awaits an answer");
expect(eng.replace(/\s+/g, ' ')).toContain("Use a scoped Edit to save this record and only the authorized working-plan amendments. Leave other choices unchanged");
const design = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', '## CRITICAL RULE');
const design = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', ASK_QUESTIONS_HEADING);
expect(design).toContain('wait for approval, then edit the plan and re-rate');
const pass4 = design.slice(design.indexOf('### Pass 4:'), design.indexOf('### Pass 5:'));
expect(pass4.match(/^### /gm)).toHaveLength(1);