test(plan-format): mode menu delivers its question without output-harvesting wording (GSTA-96)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-27 06:14:30 +00:00
1 parent 5aa7765c9f
commit e1e2ce1a8e
2 files changed
+19 -15

No files matched your search

+15 -12
View File
@@ -4,18 +4,21 @@ import * as path from 'path';
// D2: census 37195203538 refused plan-ceo-review-format-approach twice
// (reasoning_extraction, "duplicating model outputs") on its session prompt.
// GSTA-96: the mode menu refused at the same capture Write twice in three trials.
const source = fs.readFileSync(path.join(import.meta.dir, 'skill-e2e-plan-format.test.ts'), 'utf8');
const start = source.indexOf("testConcurrentIfSelected('plan-ceo-review-format-approach'");
const block = source.slice(start, source.indexOf('validate:', start));
const instruction = /function approachQuestionInstruction\(outFile: string\): string \{\n\s*return `([^`]+)`;/.exec(source)?.[1] ?? '';
test('the approach-menu prompt delivers its question without output-harvesting wording', () => {
expect(start).toBeGreaterThan(-1);
expect(block).toContain('${approachQuestionInstruction(outFile)}');
expect(block).not.toContain('captureInstruction(');
const prompt = block + instruction;
for (const phrase of [/verbatim/i, /paraphrase/i, /exact prose/i, /format-capture/i]) expect(prompt).not.toMatch(phrase);
expect(instruction).toContain('${outFile}');
expect(instruction).toContain('do not call any tool to ask the user');
expect(instruction).toContain('every option, and the Recommendation line');
});
for (const caseId of ['plan-ceo-review-format-approach', 'plan-ceo-review-format-mode']) {
test(`${caseId} delivers its question without output-harvesting wording`, () => {
const start = source.indexOf(`testConcurrentIfSelected('${caseId}'`);
const block = source.slice(start, source.indexOf('validate:', start));
expect(start).toBeGreaterThan(-1);
expect(block).toContain('${approachQuestionInstruction(outFile)}');
expect(block).not.toContain('captureInstruction(');
const prompt = block + instruction;
for (const phrase of [/verbatim/i, /paraphrase/i, /exact prose/i, /format-capture/i]) expect(prompt).not.toMatch(phrase);
expect(instruction).toContain('${outFile}');
expect(instruction).toContain('do not call any tool to ask the user');
expect(instruction).toContain('every option, and the Recommendation line');
});
}
+4 -3
View File
@@ -107,7 +107,7 @@ function setupPlanDir(tmpPrefix: string, skillName: 'plan-ceo-review' | 'plan-en
return planDir;
}
// The capture instruction passed to every case except the approach menu (D2).
// The capture instruction passed to every case except the mode and approach menus (D2).
// Tells the agent to dump AskUserQuestion content to a file instead of calling a tool.
function captureInstruction(outFile: string): string {
return `Write the verbatim text of every AskUserQuestion you would have made to ${outFile} (one question per session, full text including options and recommendation line). Do NOT call any tool to ask the user. Do NOT paraphrase — include the exact prose you would have shown. This is a format-capture test, not an interactive session.`;
@@ -118,7 +118,8 @@ function captureInstruction(outFile: string): string {
* `reasoning_extraction`, "duplicating model outputs") were anchored on the
* session prompt. This wording delivers the same question to the file as the
* step's output, without the "verbatim ... exact prose ... format-capture"
* phrasing.
* phrasing. The mode menu uses it too: both of its reds on the GSTA-96
* fixture repair were refusals at the capture Write (`reasoning_extraction`).
*/
function approachQuestionInstruction(outFile: string): string {
return `This session has no interactive question tool, so do not call any tool to ask the user. Write the one AskUserQuestion this step asks to ${outFile} instead: the question text, every option, and the Recommendation line, formatted as the skill specifies.`;
@@ -153,7 +154,7 @@ Read plan.md — that's the plan to review. This is a standalone plan document,
Proceed to Mode Selection. This is where the skill presents 4 mode options (SCOPE EXPANSION, SELECTIVE EXPANSION, HOLD SCOPE, SCOPE REDUCTION) to the user via AskUserQuestion. These options differ in kind (review posture), not in coverage.
${captureInstruction(outFile)}
${approachQuestionInstruction(outFile)}
After writing the file, stop. Do not continue the review.`,
workingDirectory: planDir,