mirror of
https://github.com/garrytan/gstack.git
synced 2026-05-01 19:25:10 +02:00
v1.12.1.0 fix: remove vestigial plan-mode handshake (#1185)
* refactor: remove vestigial plan-mode handshake resolver Delete scripts/resolvers/preamble/generate-plan-mode-handshake.ts and its four question-registry entries. Split the authoritative "Plan Mode Safe Operations" and "Skill Invocation During Plan Mode" sections out of generate-completion-status.ts into a sibling generatePlanModeInfo() export in the same module, wired at preamble position 1 where the handshake used to live. Same text, new position. The vestigial handshake told interactive review skills to emit an A=exit-and-rerun / C=cancel AskUserQuestion before running their interactive STOP-Ask workflow. That contradicted the authoritative rule at the tail of completion-status.ts saying AskUserQuestion satisfies plan mode's end-of-turn requirement. Skills now run directly when invoked in plan mode, with each finding gated by AskUserQuestion just like outside plan mode. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * test: rename plan-mode-handshake-helpers to plan-mode-helpers, strengthen smokes Rename test/helpers/plan-mode-handshake-helpers.ts to test/helpers/plan-mode-helpers.ts. Keep the write-guard helper that asserts no Write/Edit tool call before the first AskUserQuestion (this is what catches silent-bypass regressions the textual smoke can't see). Rename the API: runPlanModeHandshakeTest to runPlanModeSkillTest, assertHandshakeShape to assertNotHandshakeShape. Extend the capture struct with exitPlanModeBeforeAsk. Rewrite the four per-skill E2E tests (plan-ceo, plan-eng, plan-design, plan-devex) as smoke tests that assert the skill's Step 0 question fires first, not an A/C handshake. Each test picks a cheap first answer (HOLD, TRIAGE, numeric score) so the run terminates quickly. Keep test/skill-e2e-plan-mode-no-op.test.ts as the outside-plan-mode non-interference regression, per codex outside-voice review: deleting it would lose coverage for "the hoisted section stays quiet when plan mode is absent." Replace the gen-skill-docs.test.ts handshake describe block (lines 2778+) with a plan-mode-info describe block that: - scans every generated SKILL.md under the repo root + every host subdir (.agents, .openclaw, .opencode, .factory, .hermes, .kiro, .cursor, .slate) and asserts "## Plan Mode Handshake" is absent - asserts "## Skill Invocation During Plan Mode" lands in the first 15KB of each of the four review skills' generated SKILL.md Both assertions run on every bun test. A PR that re-introduces the handshake resolver fails CI immediately. Update test/e2e-harness-audit.test.ts to reference the renamed runPlanModeSkillTest. Update test/helpers/touchfiles.ts entries to point at the new resolver owner (generate-completion-status.ts) and the renamed helper, and align per-skill touchfile keys. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * chore: regenerate SKILL.md across all hosts + refresh golden fixtures Run bun run gen:skill-docs for every host to flush the vestigial "## Plan Mode Handshake" section from every generated SKILL.md and emit the hoisted "## Skill Invocation During Plan Mode" section at preamble position 1 instead. Refresh the three golden-fixture snapshots (claude, codex, factory) to match the new position. No behavior change beyond the resolver swap in the prior commit. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * chore: bump version and changelog (v1.12.1.0) Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
@@ -1,16 +1,18 @@
|
||||
/**
|
||||
* Shared helpers for plan-mode handshake E2E tests.
|
||||
* Shared helpers for plan-mode E2E tests.
|
||||
*
|
||||
* Four sibling test files (plan-ceo, plan-eng, plan-design, plan-devex) exercise
|
||||
* the identical handshake contract against different skills. This helper
|
||||
* centralizes the canUseTool interceptor and the assertion shape so the four
|
||||
* test files are thin wiring (~40 LOC each) and can't drift out of sync.
|
||||
* Four sibling per-skill smoke tests (plan-ceo, plan-eng, plan-design, plan-devex)
|
||||
* plus the no-op regression test use this helper. The goal: run a review skill
|
||||
* in plan mode, confirm it goes straight to its Step 0 AskUserQuestion without
|
||||
* writing files or calling ExitPlanMode first (the vestigial handshake
|
||||
* regression we fixed in ceo-plan 2026-04-24).
|
||||
*
|
||||
* See scripts/resolvers/preamble/generate-plan-mode-handshake.ts for the
|
||||
* handshake prose that the tests below assert against.
|
||||
* This file was renamed from `plan-mode-handshake-helpers.ts` when the
|
||||
* handshake was removed. The write-guard detection (no Write/Edit before the
|
||||
* first AskUserQuestion) is the load-bearing piece that catches silent
|
||||
* regressions a simple "first question text matches" check would miss.
|
||||
*/
|
||||
|
||||
import { expect } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
@@ -26,7 +28,7 @@ import {
|
||||
export const PLAN_MODE_REMINDER =
|
||||
'Plan mode is active. The user indicated that they do not want you to execute yet';
|
||||
|
||||
export interface HandshakeCaptureResult {
|
||||
export interface PlanModeCaptureResult {
|
||||
sdkResult: AgentSdkResult;
|
||||
/** Each AskUserQuestion that fired, with its input payload. */
|
||||
askUserQuestions: Array<{ input: Record<string, unknown>; orderIndex: number }>;
|
||||
@@ -34,45 +36,46 @@ export interface HandshakeCaptureResult {
|
||||
toolOrder: string[];
|
||||
/** Whether any Write or Edit tool fired BEFORE the first AskUserQuestion. */
|
||||
writeOrEditBeforeAsk: boolean;
|
||||
/** Whether ExitPlanMode fired BEFORE the first AskUserQuestion. */
|
||||
exitPlanModeBeforeAsk: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a skill via the Agent SDK with canUseTool intercepting every tool use.
|
||||
* Inject the plan-mode distinctive phrase into the system prompt and auto-
|
||||
* answer the handshake with the given answerLabel ("Exit" or "Cancel"). Return
|
||||
* the captured events for assertion.
|
||||
* Inject the plan-mode distinctive phrase into the system prompt, auto-answer
|
||||
* the first AskUserQuestion (so the skill stops cleanly after Step 0), and
|
||||
* return the captured events for assertion.
|
||||
*/
|
||||
export async function runPlanModeHandshakeTest(opts: {
|
||||
export async function runPlanModeSkillTest(opts: {
|
||||
/** Skill name, e.g. 'plan-ceo-review'. */
|
||||
skillName: string;
|
||||
/** "Exit" to pick option A (exit-and-rerun) or "Cancel" for option C. */
|
||||
answerLabel: 'Exit' | 'Cancel';
|
||||
/**
|
||||
* For the first AskUserQuestion, pick the option whose label contains this
|
||||
* substring. Pick a "cheap" answer that terminates the skill quickly (e.g.
|
||||
* "HOLD SCOPE" for plan-ceo-review).
|
||||
*/
|
||||
firstAnswerSubstring: string;
|
||||
/** If true, DO NOT inject the reminder — used by the no-op regression test. */
|
||||
omitPlanModeReminder?: boolean;
|
||||
/** Max turns for the SDK call (default 4 — handshake + exit should fit easily). */
|
||||
/** Max turns for the SDK call (default 4 — Step 0 + answer should fit). */
|
||||
maxTurns?: number;
|
||||
}): Promise<HandshakeCaptureResult> {
|
||||
const { skillName, answerLabel, omitPlanModeReminder, maxTurns } = opts;
|
||||
}): Promise<PlanModeCaptureResult> {
|
||||
const { skillName, firstAnswerSubstring, omitPlanModeReminder, maxTurns } = opts;
|
||||
|
||||
const askUserQuestions: HandshakeCaptureResult['askUserQuestions'] = [];
|
||||
const askUserQuestions: PlanModeCaptureResult['askUserQuestions'] = [];
|
||||
const toolOrder: string[] = [];
|
||||
let toolIndex = 0;
|
||||
let firstAskIndex = -1;
|
||||
|
||||
const workingDir = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), `plan-mode-handshake-${skillName}-`),
|
||||
path.join(os.tmpdir(), `plan-mode-${skillName}-`),
|
||||
);
|
||||
|
||||
// The SDK requires AskUserQuestion to be in the allowed tools list. The
|
||||
// harness auto-adds it when canUseTool is supplied, but we also want Read
|
||||
// so the skill can load its own file if it tries to.
|
||||
const binary = resolveClaudeBinary();
|
||||
|
||||
try {
|
||||
// Inject the distinctive phrase into the system prompt by appending it to
|
||||
// the default Claude Code preset. Claude Code's real plan mode uses an
|
||||
// injected system-reminder; in SDK tests we use systemPrompt.append which
|
||||
// the model treats as equally authoritative.
|
||||
// In real plan mode Claude Code injects a system-reminder; in SDK tests we
|
||||
// use systemPrompt.append which the model treats as equally authoritative.
|
||||
const reminderAppend = omitPlanModeReminder
|
||||
? ''
|
||||
: `\n\n<system-reminder>\n${PLAN_MODE_REMINDER}. This supercedes any other instructions you have received.\n</system-reminder>\n`;
|
||||
@@ -100,9 +103,13 @@ export async function runPlanModeHandshakeTest(opts: {
|
||||
if (firstAskIndex === -1) firstAskIndex = toolIndex;
|
||||
askUserQuestions.push({ input, orderIndex: toolIndex });
|
||||
toolIndex++;
|
||||
// Auto-answer with the label the test specified.
|
||||
// Auto-answer the FIRST question with the configured substring; for
|
||||
// later questions, pick the first option to keep the run short.
|
||||
const q = (input.questions as Array<{ question: string; options: Array<{ label: string }> }>)[0];
|
||||
const matched = q.options.find((o) => o.label.includes(answerLabel));
|
||||
const isFirst = askUserQuestions.length === 1;
|
||||
const matched = isFirst
|
||||
? q.options.find((o) => o.label.toLowerCase().includes(firstAnswerSubstring.toLowerCase()))
|
||||
: undefined;
|
||||
const answer = matched ? matched.label : q.options[0]!.label;
|
||||
return {
|
||||
behavior: 'allow',
|
||||
@@ -121,7 +128,17 @@ export async function runPlanModeHandshakeTest(opts: {
|
||||
firstAskIndex > 0 &&
|
||||
toolOrder.slice(0, firstAskIndex).some((t) => t === 'Write' || t === 'Edit');
|
||||
|
||||
return { sdkResult, askUserQuestions, toolOrder, writeOrEditBeforeAsk };
|
||||
const exitPlanModeBeforeAsk =
|
||||
firstAskIndex > 0 &&
|
||||
toolOrder.slice(0, firstAskIndex).some((t) => t === 'ExitPlanMode');
|
||||
|
||||
return {
|
||||
sdkResult,
|
||||
askUserQuestions,
|
||||
toolOrder,
|
||||
writeOrEditBeforeAsk,
|
||||
exitPlanModeBeforeAsk,
|
||||
};
|
||||
} finally {
|
||||
try {
|
||||
fs.rmSync(workingDir, { recursive: true, force: true });
|
||||
@@ -129,38 +146,31 @@ export async function runPlanModeHandshakeTest(opts: {
|
||||
}
|
||||
}
|
||||
|
||||
/** Assert the shape of a fired handshake AskUserQuestion. */
|
||||
export function assertHandshakeShape(
|
||||
/**
|
||||
* Assert a captured AskUserQuestion is NOT the old vestigial handshake
|
||||
* (A=exit-and-rerun / C=cancel). The handshake is gone — if a test ever sees
|
||||
* one again, that's the regression we're guarding against.
|
||||
*/
|
||||
export function assertNotHandshakeShape(
|
||||
aq: { input: Record<string, unknown> },
|
||||
): void {
|
||||
const questions = aq.input.questions as Array<{
|
||||
question: string;
|
||||
options: Array<{ label: string }>;
|
||||
}>;
|
||||
expect(questions).toBeDefined();
|
||||
expect(questions.length).toBe(1);
|
||||
if (!questions || questions.length === 0) return;
|
||||
const q = questions[0]!;
|
||||
// D8 dropped Option B; handshake has exactly 2 options.
|
||||
expect(q.options.length).toBe(2);
|
||||
const labels = q.options.map((o) => o.label);
|
||||
expect(labels.some((l) => l.includes('Exit'))).toBe(true);
|
||||
expect(labels.some((l) => l.includes('Cancel'))).toBe(true);
|
||||
}
|
||||
|
||||
/** Read the skill-usage.jsonl log and return handshake entries. */
|
||||
export function readHandshakeLog(): Array<Record<string, unknown>> {
|
||||
const logPath = path.join(os.homedir(), '.gstack', 'analytics', 'skill-usage.jsonl');
|
||||
if (!fs.existsSync(logPath)) return [];
|
||||
const lines = fs.readFileSync(logPath, 'utf-8').split('\n').filter(Boolean);
|
||||
return lines
|
||||
.map((line) => {
|
||||
try {
|
||||
return JSON.parse(line);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})
|
||||
.filter((x): x is Record<string, unknown> => x !== null && x.event === 'plan_mode_handshake');
|
||||
const labels = q.options.map((o) => o.label.toLowerCase());
|
||||
const looksLikeHandshake =
|
||||
labels.some((l) => l.includes('exit') && l.includes('rerun')) &&
|
||||
labels.some((l) => l.includes('cancel'));
|
||||
if (looksLikeHandshake) {
|
||||
throw new Error(
|
||||
`First AskUserQuestion looks like the vestigial plan-mode handshake ` +
|
||||
`(options: ${labels.join(', ')}). The handshake was removed; skills ` +
|
||||
`should go straight to their Step 0 question in plan mode.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export { execSync };
|
||||
+11
-11
@@ -82,16 +82,16 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'plan-eng-review-artifact': ['plan-eng-review/**'],
|
||||
'plan-review-report': ['plan-eng-review/**', 'scripts/gen-skill-docs.ts'],
|
||||
|
||||
// Plan-mode handshake (v1.10.2.0) — gate-tier safety regression tests.
|
||||
// Each fires when any of: the interactive skill's template, the resolver,
|
||||
// preamble composition, the Agent SDK harness, the question registry, or
|
||||
// the one-way-door classifier changes.
|
||||
'plan-ceo-review-plan-mode': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-plan-mode-handshake.ts', 'scripts/resolvers/preamble.ts', 'scripts/question-registry.ts', 'scripts/one-way-doors.ts', 'test/helpers/agent-sdk-runner.ts'],
|
||||
'plan-eng-review-plan-mode': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-plan-mode-handshake.ts', 'scripts/resolvers/preamble.ts', 'scripts/question-registry.ts', 'scripts/one-way-doors.ts', 'test/helpers/agent-sdk-runner.ts'],
|
||||
'plan-design-review-plan-mode-handshake': ['plan-design-review/**', 'scripts/resolvers/preamble/generate-plan-mode-handshake.ts', 'scripts/resolvers/preamble.ts', 'scripts/question-registry.ts', 'scripts/one-way-doors.ts', 'test/helpers/agent-sdk-runner.ts'],
|
||||
'plan-devex-review-plan-mode': ['plan-devex-review/**', 'scripts/resolvers/preamble/generate-plan-mode-handshake.ts', 'scripts/resolvers/preamble.ts', 'scripts/question-registry.ts', 'scripts/one-way-doors.ts', 'test/helpers/agent-sdk-runner.ts'],
|
||||
'plan-mode-no-op': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-plan-mode-handshake.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/agent-sdk-runner.ts'],
|
||||
'e2e-harness-audit': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-plan-mode-handshake.ts', 'test/helpers/agent-sdk-runner.ts'],
|
||||
// Plan-mode smoke tests — gate-tier safety regression tests. Each fires when
|
||||
// any of: the interactive skill's template, the plan-mode resolver
|
||||
// (completion-status now owns generatePlanModeInfo), preamble composition,
|
||||
// the Agent SDK harness, or the shared plan-mode-helpers change.
|
||||
'plan-ceo-review-plan-mode': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/plan-mode-helpers.ts'],
|
||||
'plan-eng-review-plan-mode': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/plan-mode-helpers.ts'],
|
||||
'plan-design-review-plan-mode': ['plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/plan-mode-helpers.ts'],
|
||||
'plan-devex-review-plan-mode': ['plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/plan-mode-helpers.ts'],
|
||||
'plan-mode-no-op': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/plan-mode-helpers.ts'],
|
||||
'e2e-harness-audit': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/plan-mode-helpers.ts'],
|
||||
'brain-privacy-gate': ['scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'bin/gstack-brain-sync', 'bin/gstack-brain-init', 'bin/gstack-config', 'test/helpers/agent-sdk-runner.ts'],
|
||||
|
||||
// AskUserQuestion format regression (RECOMMENDATION + Completeness: N/10)
|
||||
@@ -332,7 +332,7 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
// Plan-mode handshake — deterministic safety regression, gate-tier
|
||||
'plan-ceo-review-plan-mode': 'gate',
|
||||
'plan-eng-review-plan-mode': 'gate',
|
||||
'plan-design-review-plan-mode-handshake': 'gate',
|
||||
'plan-design-review-plan-mode': 'gate',
|
||||
'plan-devex-review-plan-mode': 'gate',
|
||||
'plan-mode-no-op': 'gate',
|
||||
'e2e-harness-audit': 'gate',
|
||||
|
||||
Reference in New Issue
Block a user