D2: reword the format-approach question prompt

Both plan-ceo-review-format-approach refusals in census 37195203538
(Fable 5.1 model_refusal_no_fallback, category reasoning_extraction:
"duplicating model outputs") were anchored on the session prompt, whose
capture instruction asked for the "verbatim" question and "the exact
prose you would have shown" in a "format-capture test". The approach
case now asks for the same question in the same file as the step's
output, without that wording. The other three plan-format cases keep
their unchanged prompt (no refusals in 72 trials).
This commit is contained in:
garrytan committed 2026-10-04 17:37:55 +00:00
1 parent 4dd99f8379
commit cbcccf9354
2 files changed
+35 -3

No files matched your search

+21
View File
@@ -0,0 +1,21 @@
import { expect, test } from 'bun:test';
import * as fs from 'fs';
import * as path from 'path';
// D2: census 37195203538 refused plan-ceo-review-format-approach twice
// (reasoning_extraction, "duplicating model outputs") on its session prompt.
const source = fs.readFileSync(path.join(import.meta.dir, 'skill-e2e-plan-format.test.ts'), 'utf8');
const start = source.indexOf("testConcurrentIfSelected('plan-ceo-review-format-approach'");
const block = source.slice(start, source.indexOf('validate:', start));
const instruction = /function approachQuestionInstruction\(outFile: string\): string \{\n\s*return `([^`]+)`;/.exec(source)?.[1] ?? '';
test('the approach-menu prompt delivers its question without output-harvesting wording', () => {
expect(start).toBeGreaterThan(-1);
expect(block).toContain('${approachQuestionInstruction(outFile)}');
expect(block).not.toContain('captureInstruction(');
const prompt = block + instruction;
for (const phrase of [/verbatim/i, /paraphrase/i, /exact prose/i, /format-capture/i]) expect(prompt).not.toMatch(phrase);
expect(instruction).toContain('${outFile}');
expect(instruction).toContain('do not call any tool to ask the user');
expect(instruction).toContain('every option, and the Recommendation line');
});
+14 -3
View File
@@ -104,12 +104,23 @@ function setupPlanDir(tmpPrefix: string, skillName: 'plan-ceo-review' | 'plan-en
return planDir;
}
// The capture instruction passed to every case. Tells the agent to dump
// AskUserQuestion content to a file instead of calling a tool.
// The capture instruction passed to every case except the approach menu (D2).
// Tells the agent to dump AskUserQuestion content to a file instead of calling a tool.
function captureInstruction(outFile: string): string {
return `Write the verbatim text of every AskUserQuestion you would have made to ${outFile} (one question per session, full text including options and recommendation line). Do NOT call any tool to ask the user. Do NOT paraphrase — include the exact prose you would have shown. This is a format-capture test, not an interactive session.`;
}
/**
* D2: both approach-menu refusals in census 37195203538 (Fable 5.1,
* `reasoning_extraction`, "duplicating model outputs") were anchored on the
* session prompt. This wording delivers the same question to the file as the
* step's output, without the "verbatim ... exact prose ... format-capture"
* phrasing.
*/
function approachQuestionInstruction(outFile: string): string {
return `This session has no interactive question tool, so do not call any tool to ask the user. Write the one AskUserQuestion this step asks to ${outFile} instead: the question text, every option, and the Recommendation line, formatted as the skill specifies.`;
}
// --- Case 1: plan-ceo-review mode selection (kind-differentiated) ---
describeIfSelected('Plan Format — CEO Mode Selection', ['plan-ceo-review-format-mode'], () => {
@@ -207,7 +218,7 @@ Read plan.md — that's the plan to review. This is a standalone plan document,
Proceed to Alternatives (the implementation approach menu). This is where the skill generates 2-3 approaches (minimal viable vs ideal architecture) and presents them via AskUserQuestion. These options differ in coverage (complete vs shortcut), so Completeness: N/10 applies.
${captureInstruction(outFile)}
${approachQuestionInstruction(outFile)}
After writing the file, stop. Do not continue the review.`,
workingDirectory: planDir,