mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-05 16:34:00 +01:00
test(plan-format): mode menu delivers its question without output-harvesting wording (GSTA-96)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
5aa7765c9f
commit
e1e2ce1a8e
2 files changed
+19
-15
No files matched your search
@@ -4,18 +4,21 @@ import * as path from 'path';
|
||||
|
||||
// D2: census 37195203538 refused plan-ceo-review-format-approach twice
|
||||
// (reasoning_extraction, "duplicating model outputs") on its session prompt.
|
||||
// GSTA-96: the mode menu refused at the same capture Write twice in three trials.
|
||||
const source = fs.readFileSync(path.join(import.meta.dir, 'skill-e2e-plan-format.test.ts'), 'utf8');
|
||||
const start = source.indexOf("testConcurrentIfSelected('plan-ceo-review-format-approach'");
|
||||
const block = source.slice(start, source.indexOf('validate:', start));
|
||||
const instruction = /function approachQuestionInstruction\(outFile: string\): string \{\n\s*return `([^`]+)`;/.exec(source)?.[1] ?? '';
|
||||
|
||||
test('the approach-menu prompt delivers its question without output-harvesting wording', () => {
|
||||
expect(start).toBeGreaterThan(-1);
|
||||
expect(block).toContain('${approachQuestionInstruction(outFile)}');
|
||||
expect(block).not.toContain('captureInstruction(');
|
||||
const prompt = block + instruction;
|
||||
for (const phrase of [/verbatim/i, /paraphrase/i, /exact prose/i, /format-capture/i]) expect(prompt).not.toMatch(phrase);
|
||||
expect(instruction).toContain('${outFile}');
|
||||
expect(instruction).toContain('do not call any tool to ask the user');
|
||||
expect(instruction).toContain('every option, and the Recommendation line');
|
||||
});
|
||||
for (const caseId of ['plan-ceo-review-format-approach', 'plan-ceo-review-format-mode']) {
|
||||
test(`${caseId} delivers its question without output-harvesting wording`, () => {
|
||||
const start = source.indexOf(`testConcurrentIfSelected('${caseId}'`);
|
||||
const block = source.slice(start, source.indexOf('validate:', start));
|
||||
expect(start).toBeGreaterThan(-1);
|
||||
expect(block).toContain('${approachQuestionInstruction(outFile)}');
|
||||
expect(block).not.toContain('captureInstruction(');
|
||||
const prompt = block + instruction;
|
||||
for (const phrase of [/verbatim/i, /paraphrase/i, /exact prose/i, /format-capture/i]) expect(prompt).not.toMatch(phrase);
|
||||
expect(instruction).toContain('${outFile}');
|
||||
expect(instruction).toContain('do not call any tool to ask the user');
|
||||
expect(instruction).toContain('every option, and the Recommendation line');
|
||||
});
|
||||
}
|
||||
@@ -107,7 +107,7 @@ function setupPlanDir(tmpPrefix: string, skillName: 'plan-ceo-review' | 'plan-en
|
||||
return planDir;
|
||||
}
|
||||
|
||||
// The capture instruction passed to every case except the approach menu (D2).
|
||||
// The capture instruction passed to every case except the mode and approach menus (D2).
|
||||
// Tells the agent to dump AskUserQuestion content to a file instead of calling a tool.
|
||||
function captureInstruction(outFile: string): string {
|
||||
return `Write the verbatim text of every AskUserQuestion you would have made to ${outFile} (one question per session, full text including options and recommendation line). Do NOT call any tool to ask the user. Do NOT paraphrase — include the exact prose you would have shown. This is a format-capture test, not an interactive session.`;
|
||||
@@ -118,7 +118,8 @@ function captureInstruction(outFile: string): string {
|
||||
* `reasoning_extraction`, "duplicating model outputs") were anchored on the
|
||||
* session prompt. This wording delivers the same question to the file as the
|
||||
* step's output, without the "verbatim ... exact prose ... format-capture"
|
||||
* phrasing.
|
||||
* phrasing. The mode menu uses it too: both of its reds on the GSTA-96
|
||||
* fixture repair were refusals at the capture Write (`reasoning_extraction`).
|
||||
*/
|
||||
function approachQuestionInstruction(outFile: string): string {
|
||||
return `This session has no interactive question tool, so do not call any tool to ask the user. Write the one AskUserQuestion this step asks to ${outFile} instead: the question text, every option, and the Recommendation line, formatted as the skill specifies.`;
|
||||
@@ -153,7 +154,7 @@ Read plan.md — that's the plan to review. This is a standalone plan document,
|
||||
|
||||
Proceed to Mode Selection. This is where the skill presents 4 mode options (SCOPE EXPANSION, SELECTIVE EXPANSION, HOLD SCOPE, SCOPE REDUCTION) to the user via AskUserQuestion. These options differ in kind (review posture), not in coverage.
|
||||
|
||||
${captureInstruction(outFile)}
|
||||
${approachQuestionInstruction(outFile)}
|
||||
|
||||
After writing the file, stop. Do not continue the review.`,
|
||||
workingDirectory: planDir,
|
||||
|
||||
Reference in new issue
Block a user