mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-02 06:54:50 +01:00
test: anchor plan-design excerpts on the question-procedure heading, not '## CRITICAL RULE' (W7a judge-anchor move; workflow-excerpt.test.ts:109,296)
The end marker now accepts a RegExp. ASK_QUESTIONS_HEADING matches the heading with or without its 'CRITICAL RULE — ' prefix, so the excerpt is byte-identical on the current templates and survives the s04 F9 / s05 L3 rename.
This commit is contained in:
1 parent
6bfc352b32
commit
7a80bab143
5 files changed
+28
-17
No files matched your search
@@ -137,7 +137,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'test/helpers/office-hours-attempt.ts'],
|
||||
|
||||
// Office Hours
|
||||
'office-hours-spec-review': [ 'office-hours/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts',
|
||||
'office-hours-spec-review': [ 'office-hours/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts',
|
||||
'bin/gstack-office-hours-review', 'lib/office-hours-review.ts', 'lib/fs-atomic.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/skill-fixture.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/workflow-judge-input.ts'],
|
||||
'office-hours-forcing-energy': [ 'office-hours/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-office-hours.test.ts',
|
||||
'test/helpers/office-hours-attempt.ts'
|
||||
@@ -147,26 +147,26 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
],
|
||||
|
||||
// Plan reviews
|
||||
'plan-ceo-review': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts',
|
||||
'plan-ceo-review': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts',
|
||||
'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
|
||||
'plan-ceo-review-selective': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts',
|
||||
'plan-ceo-review-selective': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts',
|
||||
'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
|
||||
'plan-ceo-review-benefits': [ 'plan-ceo-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts',
|
||||
'plan-ceo-review-benefits': [ 'plan-ceo-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts',
|
||||
'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
|
||||
'plan-ceo-review-expansion-energy': [ 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan.test.ts',
|
||||
'plan-ceo-review-expansion-energy': [ 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts',
|
||||
'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
|
||||
'plan-eng-review': [
|
||||
|
||||
'scripts/resolvers/learnings.ts',
|
||||
|
||||
"test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts',
|
||||
"test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts',
|
||||
"scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts",
|
||||
'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
|
||||
'plan-eng-review-artifact': [
|
||||
|
||||
'scripts/resolvers/learnings.ts',
|
||||
|
||||
"test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts',
|
||||
"test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts',
|
||||
"scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts",
|
||||
'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
|
||||
'plan-review-report': [
|
||||
@@ -175,7 +175,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'test/fixtures/plan-review-report-public.json',
|
||||
'scripts/resolvers/learnings.ts',
|
||||
|
||||
"test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts',
|
||||
"test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts',
|
||||
"scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts",
|
||||
'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
|
||||
|
||||
|
||||
@@ -10,12 +10,22 @@ export const ENG_REVIEW_EXCERPT = {
|
||||
endMarker: '## Section self-check (before you finish)',
|
||||
} as const;
|
||||
|
||||
// The question-procedure heading in the plan-review sections, with or without
|
||||
// its older "CRITICAL RULE — " prefix.
|
||||
export const ASK_QUESTIONS_HEADING = /^## [^\n]*How to ask questions[^\n]*$/m;
|
||||
|
||||
export function markerIndex(text: string, marker: string | RegExp, from: number): number {
|
||||
if (typeof marker === 'string') return text.indexOf(marker, from);
|
||||
const at = text.slice(from).search(marker);
|
||||
return at < 0 ? -1 : from + at;
|
||||
}
|
||||
|
||||
// Same generated two-line pointer consumed by setup-gbrain-fixture.ts.
|
||||
const STOP_POINTER =
|
||||
/^> \*\*STOP\.\*\* Before [^\n]*sections\/([a-z0-9-]+\.md)[^\n]*\n> in full\.[^\n]*/gm;
|
||||
|
||||
/** Expand on-demand sections where the agent reads them, then take the requested excerpt. */
|
||||
export function readWorkflowExcerpt(skillPath: string, startMarker: string, endMarker: string | null): string {
|
||||
export function readWorkflowExcerpt(skillPath: string, startMarker: string, endMarker: string | RegExp | null): string {
|
||||
const secDir = path.join(ROOT, path.dirname(skillPath), 'sections');
|
||||
const content = fs.readFileSync(path.join(ROOT, skillPath), 'utf-8').replace(STOP_POINTER, (_pointer, file: string) => {
|
||||
const body = fs.readFileSync(path.join(secDir, file), 'utf-8')
|
||||
@@ -25,7 +35,7 @@ export function readWorkflowExcerpt(skillPath: string, startMarker: string, endM
|
||||
});
|
||||
const start = content.indexOf(startMarker);
|
||||
if (start < 0) throw new Error(`Start marker not found in ${skillPath}: "${startMarker}"`);
|
||||
const end = endMarker ? content.indexOf(endMarker, start) : content.length;
|
||||
const end = endMarker ? markerIndex(content, endMarker, start) : content.length;
|
||||
if (end < 0) throw new Error(`End marker not found in ${skillPath}: "${endMarker}"`);
|
||||
return content.slice(start, end);
|
||||
}
|
||||
@@ -11,7 +11,7 @@ import { buildEvalInputIdentity, lookupEvalInputCache, sourceDependencyClosure,
|
||||
|
||||
type Thresholds = { clarity: number; completeness: number; actionability: number };
|
||||
export interface WorkflowCacheOptions {
|
||||
root: string; testName: string; skillPath: string; startMarker: string; endMarker: string | null;
|
||||
root: string; testName: string; skillPath: string; startMarker: string; endMarker: string | RegExp | null;
|
||||
judgeContext: string; judgeGoal: string; model?: string; thresholds: Thresholds; prompt: string; attempt: number;
|
||||
references?: readonly string[];
|
||||
agentCapability?: 'frontier';
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
/** Preserve source-file boundaries when a workflow judge reads carved skills. */
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { markerIndex } from './workflow-excerpt';
|
||||
|
||||
export interface WorkflowJudgeFile {
|
||||
path: string;
|
||||
@@ -84,7 +85,7 @@ export function readWorkflowJudgeInput(opts: {
|
||||
root: string;
|
||||
skillPath: string;
|
||||
startMarker: string;
|
||||
endMarker: string | null;
|
||||
endMarker: string | RegExp | null;
|
||||
references?: readonly string[];
|
||||
}): WorkflowJudgeInput {
|
||||
const sources = [{
|
||||
@@ -115,7 +116,7 @@ export function readWorkflowJudgeInput(opts: {
|
||||
const union = allSources.map(file => file.content).join('\n');
|
||||
const start = union.indexOf(opts.startMarker);
|
||||
if (start < 0) throw new Error(`Start marker not found in ${opts.skillPath}: "${opts.startMarker}"`);
|
||||
const end = opts.endMarker === null ? union.length : union.indexOf(opts.endMarker, start);
|
||||
const end = opts.endMarker === null ? union.length : markerIndex(union, opts.endMarker, start);
|
||||
if (end < 0) throw new Error(`End marker not found in ${opts.skillPath}: "${opts.endMarker}"`);
|
||||
|
||||
const files: WorkflowJudgeFile[] = [];
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { ENG_REVIEW_EXCERPT, readWorkflowExcerpt } from './helpers/workflow-excerpt';
|
||||
import { ASK_QUESTIONS_HEADING, ENG_REVIEW_EXCERPT, readWorkflowExcerpt } from './helpers/workflow-excerpt';
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
@@ -106,9 +106,9 @@ describe('workflow judge excerpts', () => {
|
||||
});
|
||||
|
||||
test('a sliced section is not appended again with its generated header', () => {
|
||||
const text = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', '## CRITICAL RULE');
|
||||
const text = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', ASK_QUESTIONS_HEADING);
|
||||
expect(text.match(/## Review Sections/g)).toHaveLength(1);
|
||||
expect(text).not.toContain('## CRITICAL RULE');
|
||||
expect(text).not.toMatch(ASK_QUESTIONS_HEADING);
|
||||
expect(text).not.toContain('AUTO-GENERATED');
|
||||
});
|
||||
|
||||
@@ -293,7 +293,7 @@ console.log(JSON.stringify({calls, results}));
|
||||
expect(pendingDecision).toContain("**STOP until the actual answer arrives.**");
|
||||
expect(pendingDecision.replace(/\s+/g, ' ')).toContain("Do not apply a remedy, make another call, start the next section or call ExitPlanMode while the choice awaits an answer");
|
||||
expect(eng.replace(/\s+/g, ' ')).toContain("Use a scoped Edit to save this record and only the authorized working-plan amendments. Leave other choices unchanged");
|
||||
const design = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', '## CRITICAL RULE');
|
||||
const design = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', ASK_QUESTIONS_HEADING);
|
||||
expect(design).toContain('wait for approval, then edit the plan and re-rate');
|
||||
const pass4 = design.slice(design.indexOf('### Pass 4:'), design.indexOf('### Pass 5:'));
|
||||
expect(pass4.match(/^### /gm)).toHaveLength(1);
|
||||
|
||||
Reference in new issue
Block a user