mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 22:14:25 +01:00
D2: reword the format-approach question prompt
Both plan-ceo-review-format-approach refusals in census 37195203538 (Fable 5.1 model_refusal_no_fallback, category reasoning_extraction: "duplicating model outputs") were anchored on the session prompt, whose capture instruction asked for the "verbatim" question and "the exact prose you would have shown" in a "format-capture test". The approach case now asks for the same question in the same file as the step's output, without that wording. The other three plan-format cases keep their unchanged prompt (no refusals in 72 trials).
This commit is contained in:
1 parent
4dd99f8379
commit
cbcccf9354
2 files changed
+35
-3
No files matched your search
@@ -0,0 +1,21 @@
|
||||
import { expect, test } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
// D2: census 37195203538 refused plan-ceo-review-format-approach twice
|
||||
// (reasoning_extraction, "duplicating model outputs") on its session prompt.
|
||||
const source = fs.readFileSync(path.join(import.meta.dir, 'skill-e2e-plan-format.test.ts'), 'utf8');
|
||||
const start = source.indexOf("testConcurrentIfSelected('plan-ceo-review-format-approach'");
|
||||
const block = source.slice(start, source.indexOf('validate:', start));
|
||||
const instruction = /function approachQuestionInstruction\(outFile: string\): string \{\n\s*return `([^`]+)`;/.exec(source)?.[1] ?? '';
|
||||
|
||||
test('the approach-menu prompt delivers its question without output-harvesting wording', () => {
|
||||
expect(start).toBeGreaterThan(-1);
|
||||
expect(block).toContain('${approachQuestionInstruction(outFile)}');
|
||||
expect(block).not.toContain('captureInstruction(');
|
||||
const prompt = block + instruction;
|
||||
for (const phrase of [/verbatim/i, /paraphrase/i, /exact prose/i, /format-capture/i]) expect(prompt).not.toMatch(phrase);
|
||||
expect(instruction).toContain('${outFile}');
|
||||
expect(instruction).toContain('do not call any tool to ask the user');
|
||||
expect(instruction).toContain('every option, and the Recommendation line');
|
||||
});
|
||||
@@ -104,12 +104,23 @@ function setupPlanDir(tmpPrefix: string, skillName: 'plan-ceo-review' | 'plan-en
|
||||
return planDir;
|
||||
}
|
||||
|
||||
// The capture instruction passed to every case. Tells the agent to dump
|
||||
// AskUserQuestion content to a file instead of calling a tool.
|
||||
// The capture instruction passed to every case except the approach menu (D2).
|
||||
// Tells the agent to dump AskUserQuestion content to a file instead of calling a tool.
|
||||
function captureInstruction(outFile: string): string {
|
||||
return `Write the verbatim text of every AskUserQuestion you would have made to ${outFile} (one question per session, full text including options and recommendation line). Do NOT call any tool to ask the user. Do NOT paraphrase — include the exact prose you would have shown. This is a format-capture test, not an interactive session.`;
|
||||
}
|
||||
|
||||
/**
|
||||
* D2: both approach-menu refusals in census 37195203538 (Fable 5.1,
|
||||
* `reasoning_extraction`, "duplicating model outputs") were anchored on the
|
||||
* session prompt. This wording delivers the same question to the file as the
|
||||
* step's output, without the "verbatim ... exact prose ... format-capture"
|
||||
* phrasing.
|
||||
*/
|
||||
function approachQuestionInstruction(outFile: string): string {
|
||||
return `This session has no interactive question tool, so do not call any tool to ask the user. Write the one AskUserQuestion this step asks to ${outFile} instead: the question text, every option, and the Recommendation line, formatted as the skill specifies.`;
|
||||
}
|
||||
|
||||
// --- Case 1: plan-ceo-review mode selection (kind-differentiated) ---
|
||||
|
||||
describeIfSelected('Plan Format — CEO Mode Selection', ['plan-ceo-review-format-mode'], () => {
|
||||
@@ -207,7 +218,7 @@ Read plan.md — that's the plan to review. This is a standalone plan document,
|
||||
|
||||
Proceed to Alternatives (the implementation approach menu). This is where the skill generates 2-3 approaches (minimal viable vs ideal architecture) and presents them via AskUserQuestion. These options differ in coverage (complete vs shortcut), so Completeness: N/10 applies.
|
||||
|
||||
${captureInstruction(outFile)}
|
||||
${approachQuestionInstruction(outFile)}
|
||||
|
||||
After writing the file, stop. Do not continue the review.`,
|
||||
workingDirectory: planDir,
|
||||
|
||||
Reference in new issue
Block a user