From 178ca72c1717bb4d7221014e63cb4d049276b2b7 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Tue, 8 Sep 2026 06:31:15 +0000 Subject: [PATCH] test: stabilize review fixtures and Claude eval startup Preserve source boundaries in workflow judge inputs, isolate CEO mode plans, and wait for interactive trust input readiness. Keep startup failure evidence and retain existing models, budgets, and assertions. Co-Authored-By: OpenAI Codex --- test/helpers/claude-pty-runner.ts | 43 ++-- test/helpers/pty-trust-dialog.ts | 37 ++++ test/helpers/touchfiles-data.ts | 72 +++---- test/helpers/workflow-judge-input.ts | 86 ++++++++ test/plan-count-fixture.test.ts | 6 + test/pty-trust-dialog.test.ts | 140 +++++++++++++ test/skill-e2e-plan-ceo-mode-routing.test.ts | 40 +++- test/skill-llm-eval.test.ts | 53 +---- test/workflow-judge-input.test.ts | 203 +++++++++++++++++++ 9 files changed, 583 insertions(+), 97 deletions(-) create mode 100644 test/helpers/pty-trust-dialog.ts create mode 100644 test/helpers/workflow-judge-input.ts create mode 100644 test/pty-trust-dialog.test.ts create mode 100644 test/workflow-judge-input.test.ts diff --git a/test/helpers/claude-pty-runner.ts b/test/helpers/claude-pty-runner.ts index 825b01bbf..ba56825ed 100644 --- a/test/helpers/claude-pty-runner.ts +++ b/test/helpers/claude-pty-runner.ts @@ -25,8 +25,10 @@ import { resolveEvalModel } from '../../lib/eval-model'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; +import { stripVTControlCharacters } from 'node:util'; import { hermeticChildEnv, hermeticSkillsConfigDir, isHermeticEnabled } from './hermetic-env'; import { createPlanCountFixture } from './plan-count-fixture'; +import { trustDialogInput } from './pty-trust-dialog'; /** Strip ANSI escapes for pattern-matching against visible text. */ export function stripAnsi(s: string): string { @@ -151,10 +153,9 @@ export interface ClaudePtySession { close(): Promise; } -/** Detect the workspace-trust dialog rendering. */ +/** Detect a complete, recognized workspace-trust menu. */ export function isTrustDialogVisible(visible: string): boolean { - // Phrase Claude Code prints. Stable across versions in this branch's range. - return visible.includes('trust this folder'); + return trustDialogInput(visible) !== null; } /** @@ -1283,8 +1284,8 @@ export const devexStep0Boundary: Step0BoundaryPredicate = (fp) => * handle. Caller is responsible for `await session.close()` to release the * subprocess and any timers. * - * Auto-handles the workspace-trust dialog (presses "1\r" if it appears - * during the boot window). Tests should NOT have to handle it themselves. + * Auto-handles the workspace-trust dialog by selecting its explicit + * affirmative option. Tests should NOT have to handle it themselves. */ export async function launchClaudePty( opts: ClaudePtyOptions = {}, @@ -1367,17 +1368,26 @@ export async function launchClaudePty( }, timeoutMs); // Auto-handle the workspace-trust dialog. Runs once during the boot - // window; idempotent (only fires if the phrase is still on screen). + // window, after both choices and the selected cursor are visible. Newer + // unnumbered menus default to "No, exit", so "1\r" would reject trust. + // The first paint can precede the input handler's readiness. Let startup + // settle, then deliver navigation and confirmation as separate events. let trustHandled = false; + let trustVisibleAt: number | undefined; + const trustInputTimers: ReturnType[] = []; const trustWatcher = setInterval(() => { if (trustHandled || exited) return; - const visible = stripAnsi(buffer); - if (isTrustDialogVisible(visible)) { + const input = trustDialogInput(buffer); + if (input !== null) { + trustVisibleAt ??= Date.now(); + if (Date.now() - trustVisibleAt < 1_500) return; trustHandled = true; - try { - proc.terminal?.write?.('1\r'); - } catch { - /* ignore */ + const keys = input.match(/\x1b\[[AB]|\r/g) ?? []; + for (const [i, key] of keys.entries()) { + trustInputTimers.push(setTimeout(() => { + if (exited) return; + try { proc.terminal?.write?.(key); } catch { /* ignore */ } + }, i * 500)); } } }, 200); @@ -1463,6 +1473,7 @@ export async function launchClaudePty( clearTimeout(wallTimer); clearTimeout(trustWatcherStop); clearInterval(trustWatcher); + for (const timer of trustInputTimers) clearTimeout(timer); if (exited) return; try { proc.kill?.('SIGINT'); @@ -2032,10 +2043,16 @@ export async function runPlanSkillCounting(opts: { summary: string, visible: string, ): PlanSkillCountObservation { + const clean = (text: string) => stripVTControlCharacters(text) + .replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, ''); + const failed = outcome === 'exited' || outcome === 'timeout'; return { outcome, summary, - evidence: visible.slice(-3000), + evidence: failed + ? `exitCode=${session.exitCode()}\n--- post-command evidence (last 3KB) ---\n${clean(visible).slice(-3000)}` + + `\n--- full-session evidence, including startup (last 6KB) ---\n${clean(session.visibleText()).slice(-6000)}` + : visible.slice(-3000), elapsedMs: Date.now() - startedAt, fingerprints, step0Count, diff --git a/test/helpers/pty-trust-dialog.ts b/test/helpers/pty-trust-dialog.ts new file mode 100644 index 000000000..f578e236a --- /dev/null +++ b/test/helpers/pty-trust-dialog.ts @@ -0,0 +1,37 @@ +import { stripVTControlCharacters } from 'node:util'; + +/** Select the explicit affirmative entry in a fully rendered startup menu. */ +export function trustDialogInput(visible: string): string | null { + const clean = stripVTControlCharacters(visible); + const headers = [...clean.matchAll(/Accessing\s*workspace:|Do\s*you\s*trust\s*(?:the\s*files\s*in\s*)?this\s*folder\?|Trust\s*this\s*folder\?/gi)]; + const header = headers.at(-1); + if (!header) return null; + const dialog = clean.slice(header.index); + + type Option = { selected: boolean; affirmative: boolean }; + const options: Option[] = []; + for (const line of dialog.split(/[\r\n]+/)) { + const match = line.replace(/\s+/g, '').match( + /^(❯)?(?:\d+[.)])?(Yes(?:,Itrustthisfolder|,andalwaysallowaccessto.+)?|No(?:,exit)?)(?:\(Recommended\))?$/i, + ); + if (match) options.push({ selected: !!match[1], affirmative: /^Yes/i.test(match[2]!) }); + } + + // Cursor-positioning escapes can collapse the two current menu rows into + // one logical line. Their complete labels still identify both choices. + if (options.length < 2) { + options.length = 0; + for (const match of dialog.replace(/\s+/g, '').matchAll(/(❯)?(?:\d+[.)])?(Yes,Itrustthisfolder|No,exit)/gi)) { + options.push({ selected: !!match[1], affirmative: /^Yes/i.test(match[2]!) }); + } + } + + const target = options.findIndex((option) => option.affirmative); + const selected = options.findIndex((option) => option.selected); + // Wait for both choices and a single cursor. An incomplete or unfamiliar + // frame must never turn a guessed default into an implicit rejection. + if (target < 0 || selected < 0 || !options.some((option) => !option.affirmative) || + options.filter((option) => option.selected).length !== 1) return null; + const distance = target - selected; + return (distance < 0 ? '\x1b[A' : '\x1b[B').repeat(Math.abs(distance)) + '\r'; +} diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index cf2dce00d..32fc74e18 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -100,17 +100,17 @@ export const E2E_TOUCHFILES: Record = { // include question-tuning.ts and generate-ask-user-format.ts because the // AUTO_DECIDE preamble injection lives there and changes can flip the // regression test outcome between 'asked' and 'auto_decided'. - 'plan-ceo-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-plan-mode.test.ts'], - 'plan-eng-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-eng-plan-mode.test.ts'], - 'plan-design-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts', 'test/skill-e2e-design.test.ts'], - 'plan-devex-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-devex-plan-mode.test.ts'], + 'plan-ceo-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-ceo-plan-mode.test.ts'], + 'plan-eng-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-eng-plan-mode.test.ts'], + 'plan-design-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts', 'test/skill-e2e-design.test.ts'], + 'plan-devex-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-devex-plan-mode.test.ts'], // Covers ceo (preamble misfire) + eng/design (scope-gate bypass must not // fire outside plan mode) + the named-target exception case. 4 PTY runs; // in CI these run CONCURRENT with the rest of the pty-plan-smoke suite // (--max-concurrency + --retry 1), so worst-case cost is ~2x a single // pass of each, sharing the API budget with sibling tests — not the // sequential ~+10min a local read suggests. - 'plan-mode-no-op': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-mode-no-op.test.ts'], + 'plan-mode-no-op': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-mode-no-op.test.ts'], // v1.21+ AskUserQuestion-blocked regression tests — Conductor launches // claude with `--disallowedTools AskUserQuestion --permission-mode default` @@ -120,7 +120,7 @@ export const E2E_TOUCHFILES: Record = { // INSIDE the existing 4 plan-X-review-plan-mode test files (covered // transitively by the entries above). Two new standalone files exist for // skills with no prior plan-mode test: - 'office-hours-auto-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'office-hours/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-office-hours-auto-mode.test.ts'], + 'office-hours-auto-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'office-hours/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-office-hours-auto-mode.test.ts'], 'office-hours-phase4-fork': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'office-hours/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/question-tuning.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-office-hours-phase4.test.ts'], 'llm-judge-recommendation': ['codex/**', 'test/helpers/llm-judge.ts', 'test/llm-judge-recommendation.test.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'codex/SKILL.md.tmpl', 'scripts/resolvers/review.ts'], // v1.21+ AUTO_DECIDE preserve eval (periodic). Verifies the Tool resolution @@ -128,20 +128,20 @@ export const E2E_TOUCHFILES: Record = { // written a never-ask preference, AUQ should still auto-decide rather than // surfacing the question. Touches the question-tuning + preference // infrastructure plus the resolvers that own the AUTO_DECIDE preamble. - 'auto-decide-preserved': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-session-kind', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'plan-ceo-review/**', 'bin/gstack-question-preference', 'bin/gstack-config', 'bin/gstack-slug', 'hosts/claude/hooks/question-preference-hook.ts', 'hosts/claude/hooks/spawned-directive.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-auto-decide-preserved.test.ts'], + 'auto-decide-preserved': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-session-kind', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'plan-ceo-review/**', 'bin/gstack-question-preference', 'bin/gstack-config', 'bin/gstack-slug', 'hosts/claude/hooks/question-preference-hook.ts', 'hosts/claude/hooks/spawned-directive.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-auto-decide-preserved.test.ts'], // Conductor → prose decision brief (Conductor signal makes prose the default; // the PreToolUse hook denies the flaky tool). Touches the resolver that owns // the Conductor rule, the preamble signal, the hook, and the detection helper. - 'conductor-prose': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-session-kind', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble.ts', 'plan-eng-review/**', 'hosts/claude/hooks/question-preference-hook.ts', 'hosts/claude/hooks/spawned-directive.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-conductor-prose.test.ts'], + 'conductor-prose': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-session-kind', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble.ts', 'plan-eng-review/**', 'hosts/claude/hooks/question-preference-hook.ts', 'hosts/claude/hooks/spawned-directive.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-conductor-prose.test.ts'], // Real-PTY E2E batch (#6 new tests on the harness). // Each one tests behavior the SDK harness can't observe (rendered TTY, // numbered-option lists, multi-phase ordering, idempotency state echo). 'auq-format-gate': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-ask-user-question-format-compliance.test.ts'], - 'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/ceo-mode-option.ts', 'test/ceo-mode-option.test.ts', 'test/skill-e2e-plan-ceo-mode-routing.test.ts'], - 'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-with-ui.test.ts'], - 'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-ship-idempotency.test.ts'], + 'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/ceo-mode-option.ts', 'test/ceo-mode-option.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-ceo-mode-routing.test.ts'], + 'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-design-with-ui.test.ts'], + 'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-ship-idempotency.test.ts'], 'tpa-present': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'], 'tpa-absent-linux': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'], 'tpa-broken': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'], @@ -154,34 +154,34 @@ export const E2E_TOUCHFILES: Record = { // the selector sets GSTACK_CARVE_SKILL= to scope cost to the changed // skill (D-CODEX A). Touching the registry/helper or sections.ts runs all. 'carve-section-loading': ['design-html/**', 'design-shotgun/**', 'qa/**', 'browse/**', 'retro/**', 'autoplan/**', 'spec/**', 'setup-gbrain/**', 'review/**', 'codex/**', 'land-and-deploy/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/resolvers/redact-doc.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/carve-section-loading.test.ts'], - 'autoplan-chain-pty': ['autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-autoplan-chain.test.ts'], + 'autoplan-chain-pty': ['autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-autoplan-chain.test.ts'], // Per-finding AskUserQuestion count + review-report-at-bottom assertion. // Each test drives its skill end-to-end; touchfiles include preamble + // completion-status resolvers because they affect question cadence and // terminal output (the regression surface this test catches). - 'plan-ceo-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-ceo-finding-count.test.ts'], - 'plan-eng-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-eng-finding-count.test.ts'], - 'plan-design-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-design-finding-count.test.ts'], - 'plan-devex-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts'], + 'plan-ceo-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-ceo-finding-count.test.ts'], + 'plan-eng-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-eng-finding-count.test.ts'], + 'plan-design-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-design-finding-count.test.ts'], + 'plan-devex-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts'], // Gate-tier reviewCount-floor counterparts. Catch the May 2026 transcript // bug (model wrote a plan-mode plan and ExitPlanMode'd without firing any // review-phase AskUserQuestion). Uses runPlanSkillFloorCheck — minimal // "did agent fire ANY AUQ?" observer that exits early on first non-permission // numbered-option render. ~1-3 min typical wall time per test, ~$2-6 total. - 'plan-eng-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-finding-floor.test.ts'], - 'plan-ceo-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-finding-floor.test.ts'], - 'plan-design-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-design-finding-floor.test.ts'], - 'plan-devex-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-devex-finding-floor.test.ts'], + 'plan-eng-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-finding-floor.test.ts'], + 'plan-ceo-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-finding-floor.test.ts'], + 'plan-design-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-design-finding-floor.test.ts'], + 'plan-devex-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-devex-finding-floor.test.ts'], // Multi-finding batching regression — periodic tier complement to the // gate-tier finding-floor. Catches the May 2026 transcript shape where // a model fires one AUQ then batches the rest into a "## Decisions to // confirm" plan write. runPlanSkillFloorCheck cannot detect that shape // (it exits on first AUQ); runPlanSkillCounting can. - 'plan-eng-multi-finding-batching': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts'], - 'plan-ceo-split-overflow': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts'], + 'plan-eng-multi-finding-batching': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts'], + 'plan-ceo-split-overflow': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts'], 'brain-privacy-gate': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'bin/gstack-brain-sync', 'bin/gstack-artifacts-init', 'bin/gstack-config', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-brain-privacy-gate.test.ts'], // /setup-gbrain Path 4 (Remote MCP) — happy + bad-token end-to-end via @@ -850,30 +850,30 @@ export const LLM_JUDGE_TOUCHFILES: Record = { 'baseline score pinning': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'test/fixtures/eval-baselines.json', 'test/skill-llm-eval.test.ts'], // Ship & Release - 'ship/SKILL.md workflow': ['ship/SKILL.md', 'ship/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], - 'document-release/SKILL.md workflow': ['document-release/SKILL.md', 'document-release/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'ship/SKILL.md workflow': ['ship/SKILL.md', 'ship/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], + 'document-release/SKILL.md workflow': ['document-release/SKILL.md', 'document-release/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], // Plan Reviews - 'plan-ceo-review/SKILL.md modes': ['plan-ceo-review/SKILL.md', 'plan-ceo-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], - 'plan-eng-review/SKILL.md sections': ['plan-eng-review/SKILL.md', 'plan-eng-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'plan-ceo-review/SKILL.md modes': ['plan-ceo-review/SKILL.md', 'plan-ceo-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], + 'plan-eng-review/SKILL.md sections': ['plan-eng-review/SKILL.md', 'plan-eng-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], // /spec authored-spec quality (paid LLM-judge — periodic-tier). - 'plan-design-review/SKILL.md passes': ['plan-design-review/SKILL.md', 'plan-design-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'plan-design-review/SKILL.md passes': ['plan-design-review/SKILL.md', 'plan-design-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], // Design skills - 'design-review/SKILL.md fix loop': ['design-review/SKILL.md', 'design-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], - 'design-consultation/SKILL.md research': ['design-consultation/SKILL.md', 'design-consultation/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'design-review/SKILL.md fix loop': ['design-review/SKILL.md', 'design-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], + 'design-consultation/SKILL.md research': ['design-consultation/SKILL.md', 'design-consultation/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], // Deploy skills - 'land-and-deploy/SKILL.md workflow': ['land-and-deploy/SKILL.md', 'land-and-deploy/SKILL.md.tmpl', 'land-and-deploy/sections/**', 'test/skill-llm-eval.test.ts'], - 'canary/SKILL.md monitoring loop': ['canary/SKILL.md', 'canary/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], - 'benchmark/SKILL.md perf collection': ['benchmark/SKILL.md', 'benchmark/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], - 'setup-deploy/SKILL.md platform setup': ['setup-deploy/SKILL.md', 'setup-deploy/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'land-and-deploy/SKILL.md workflow': ['land-and-deploy/SKILL.md', 'land-and-deploy/SKILL.md.tmpl', 'land-and-deploy/sections/**', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], + 'canary/SKILL.md monitoring loop': ['canary/SKILL.md', 'canary/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], + 'benchmark/SKILL.md perf collection': ['benchmark/SKILL.md', 'benchmark/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], + 'setup-deploy/SKILL.md platform setup': ['setup-deploy/SKILL.md', 'setup-deploy/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], // Other skills - 'retro/SKILL.md instructions': ['retro/sections/**', 'retro/SKILL.md', 'retro/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], - 'qa-only/SKILL.md workflow': ['qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], - 'gstack-upgrade/SKILL.md upgrade flow': ['gstack-upgrade/SKILL.md', 'gstack-upgrade/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'retro/SKILL.md instructions': ['retro/sections/**', 'retro/SKILL.md', 'retro/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], + 'qa-only/SKILL.md workflow': ['qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], + 'gstack-upgrade/SKILL.md upgrade flow': ['gstack-upgrade/SKILL.md', 'gstack-upgrade/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], // Voice directive 'voice directive tone': ['scripts/resolvers/preamble.ts', 'review/SKILL.md', 'review/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-llm-eval.test.ts'], diff --git a/test/helpers/workflow-judge-input.ts b/test/helpers/workflow-judge-input.ts new file mode 100644 index 000000000..135ba81af --- /dev/null +++ b/test/helpers/workflow-judge-input.ts @@ -0,0 +1,86 @@ +/** Preserve source-file boundaries when a workflow judge reads carved skills. */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +export interface WorkflowJudgeFile { + path: string; + kind: 'entrypoint' | 'section'; + content: string; + startLine: number; + endLine: number; +} + +export interface WorkflowJudgeInput { + files: WorkflowJudgeFile[]; + text: string; +} + +export function readWorkflowJudgeInput(opts: { + root: string; + skillPath: string; + startMarker: string; + endMarker: string | null; +}): WorkflowJudgeInput { + const sources = [{ + path: opts.skillPath, + kind: 'entrypoint' as const, + content: fs.readFileSync(path.join(opts.root, opts.skillPath), 'utf8'), + }]; + const sectionDir = path.join(path.dirname(opts.skillPath), 'sections'); + const sectionRoot = path.join(opts.root, sectionDir); + const sections = fs.existsSync(sectionRoot) + ? fs.readdirSync(sectionRoot).sort().filter(name => name.endsWith('.md')) + .map(name => ({ + path: path.join(sectionDir, name), + kind: 'section' as const, + content: fs.readFileSync(path.join(sectionRoot, name), 'utf8'), + })) + : []; + const allSources = [...sources, ...sections]; + + // Preserve the existing marker window, including markers that moved into + // section files. Offsets identify the source of each slice; prose prefixes + // cannot reliably identify a file after its generated header was sliced off. + const union = allSources.map(file => file.content).join('\n'); + const start = union.indexOf(opts.startMarker); + if (start < 0) throw new Error(`Start marker not found in ${opts.skillPath}: "${opts.startMarker}"`); + const end = opts.endMarker === null ? union.length : union.indexOf(opts.endMarker, start); + if (end < 0) throw new Error(`End marker not found in ${opts.skillPath}: "${opts.endMarker}"`); + + const files: WorkflowJudgeFile[] = []; + let offset = 0; + for (const file of allSources) { + // Every section was already supplied in full by the old judge input. Keep + // that coverage, but include each file once even when the marker window + // also covers part of it. Only the entrypoint retains the requested slice. + const from = file.kind === 'section' ? 0 : Math.max(0, start - offset); + const to = file.kind === 'section' ? file.content.length : Math.min(file.content.length, end - offset); + if (from < to) { + files.push({ + ...file, + path: file.path.split(path.sep).join('/'), + content: file.content.slice(from, to), + startLine: file.content.slice(0, from).split('\n').length, + endLine: file.content.slice(0, to - 1).split('\n').length, + }); + } + offset += file.content.length + 1; + } + + const context = [ + 'The material below is a bundle of source-file excerpts, with each original file and line range labeled.', + 'SKILL.md is the entry point; the labeled ranges identify which excerpts are supplied.', + ...(sections.length > 0 ? [ + 'The section files remain separate on disk and are read at the points and conditions specified by the skill\'s Read directives.', + 'They are supplied here as on-demand references; their order in this bundle is not execution order.', + ] : []), + ].join('\n'); + return { + files, + text: context + '\n\n' + files.map(file => [ + `--- BEGIN FILE ${JSON.stringify(file.path)} (lines ${file.startLine}-${file.endLine}; ${file.kind}) ---`, + file.content, + `--- END FILE ${JSON.stringify(file.path)} ---`, + ].join('\n')).join('\n\n'), + }; +} diff --git a/test/plan-count-fixture.test.ts b/test/plan-count-fixture.test.ts index 61e80199e..95a6e0183 100644 --- a/test/plan-count-fixture.test.ts +++ b/test/plan-count-fixture.test.ts @@ -124,6 +124,7 @@ record({ sections: fs.existsSync(path.join(skillDir, 'sections', 'review-sections.md')) ? fs.readFileSync(path.join(skillDir, 'sections', 'review-sections.md'), 'utf8') : null, }); +if (process.env.FIXTURE_MODE === 'exit') process.stdout.write('\x1b[?25lSTARTUP_DIAGNOSTIC fixture CLI booted\x1b[?25h\n'); process.stdin.setRawMode?.(true); let firstInput = true; let completion; @@ -201,6 +202,11 @@ await Bun.write(${JSON.stringify(resultPath)}, JSON.stringify(results)); expect(() => process.kill(startup.pid, 0)).toThrow(); const result = results.find((result) => result.name === item.name); expect(result.observation.outcome).toBe(item.mode === 'exit' ? 'exited' : 'completion_summary'); + if (item.mode === 'exit') { + expect(result.observation.evidence).toContain('exitCode=7'); + expect(result.observation.evidence).toContain('STARTUP_DIAGNOSTIC fixture CLI booted'); + expect(result.observation.evidence).not.toContain('\x1b'); + } if (item.mode === 'complete') expect(events.at(-1).type).toBe('closed'); cwds.add(startup.cwd); } diff --git a/test/pty-trust-dialog.test.ts b/test/pty-trust-dialog.test.ts new file mode 100644 index 000000000..7be74a22d --- /dev/null +++ b/test/pty-trust-dialog.test.ts @@ -0,0 +1,140 @@ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { trustDialogInput } from './helpers/pty-trust-dialog'; + +const OLD = 'Do you trust the files in this folder?\n❯ 1. Yes, I trust this folder\n 2. No, exit\nEnter to confirm'; +const CURRENT = 'Accessingworkspace:\r\n/tmp/fixture\r\nQuicksafetycheck:Isthisaprojectyoucreatedoroneyoutrust?\r\n❯No,exit\r\nYes,Itrustthisfolder\r\nEntertoconfirm·Esctocancel'; + +describe('startup trust selection', () => { + test('confirms the explicit affirmative choice in the old numbered menu', () => { + expect(trustDialogInput(OLD)).toBe('\r'); + expect(trustDialogInput('Do you trust the files in this folder?\n 1. Yes\n 2. Yes, and always allow access to /tmp/fixture\n❯ 3. No, exit')) + .toBe('\x1b[A\x1b[A\r'); + }); + + test('moves to affirmative when the current unnumbered menu defaults to exit', () => { + expect(trustDialogInput(CURRENT)).toBe('\x1b[B\r'); + expect(trustDialogInput(CURRENT.replace('❯No,exit', 'No,exit').replace('Yes,Itrust', '❯Yes,Itrust'))).toBe('\r'); + }); + + test('handles ANSI cursor visibility and rows collapsed by cursor positioning', () => { + expect(trustDialogInput('\x1b[?25l' + CURRENT + '\x1b[?25h')).toBe('\x1b[B\r'); + expect(trustDialogInput('Accessing workspace:\n❯No,\x1b[1Cexit\x1b[1BYes,I\x1b[1Ctrustthisfolder')) + .toBe('\x1b[B\r'); + }); + + test('waits for complete choices and a known selected option', () => { + expect(trustDialogInput('Accessing workspace:\n❯ No, exit')).toBeNull(); + expect(trustDialogInput('Accessing workspace:\n❯ Yes, I trust this folder')).toBeNull(); + expect(trustDialogInput(CURRENT.replace('❯', ''))).toBeNull(); + expect(trustDialogInput('Accessing workspace:\n❯ No, exit\n❯ Yes, I trust this folder')).toBeNull(); + expect(trustDialogInput('Do you trust the files in this folder?\n❯ Continue\nCancel')).toBeNull(); + }); + + test('ignores ordinary questions and uses the latest full dialog redraw', () => { + expect(trustDialogInput('Should we enable a setting?\n❯ 1. Yes\n2. No')).toBeNull(); + expect(trustDialogInput('The text says "trust this folder" but no menu is shown.')).toBeNull(); + expect(trustDialogInput(OLD + '\n' + CURRENT)).toBe('\x1b[B\r'); + }); + + test.skipIf(process.platform === 'win32')('the real PTY driver accepts both menu orders, including a partial current render', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-trust-replay-')); + const fake = path.join(dir, 'fake-claude'); + const worker = path.join(dir, 'worker.ts'); + const resultFile = path.join(dir, 'result.json'); + const cases = [ + { name: 'old', menu: OLD, selected: 0, affirmative: 0, expected: '\r' }, + { name: 'current', menu: CURRENT, selected: 0, affirmative: 1, expected: '\x1b[B\r' }, + { name: 'partial', menu: CURRENT, selected: 0, affirmative: 1, expected: '\x1b[B\r' }, + ].map((item) => ({ ...item, cwd: path.join(dir, item.name), record: path.join(dir, item.name + '.jsonl') })); + for (const item of cases) fs.mkdirSync(item.cwd); + fs.writeFileSync(fake, `#!${process.execPath}\n` + String.raw` +import * as fs from 'node:fs'; +const item = JSON.parse(process.env.TRUST_REPLAY); +const record = (event) => fs.appendFileSync(item.record, JSON.stringify(event) + '\n'); +record({type:'startup', pid:process.pid, cwd:process.cwd(), config:process.env.CLAUDE_CONFIG_DIR}); +process.stdin.setRawMode?.(true); +let selected = item.selected; +let menuComplete = false; +let inputReadyAt = Infinity; +let pending = ''; +process.stdin.on('data', (data) => { + const inputReady = Date.now() >= inputReadyAt; + record({type:'input', data:data.toString(), menuComplete, inputReady}); + // Current Claude startup can paint the menu before its input listener is + // ready. Early navigation is lost; an early Enter retains the exit default. + if (!inputReady) return; + pending += data.toString(); + while (pending.length) { + if (pending.startsWith('\x1b[A')) { setTimeout(() => { selected -= 1; }, 100); pending = pending.slice(3); } + else if (pending.startsWith('\x1b[B')) { setTimeout(() => { selected += 1; }, 100); pending = pending.slice(3); } + else if (pending[0] === '\x1b' && pending.length < 3) break; + else { + const key = pending[0]; pending = pending.slice(1); + if (key === '\r') { + if (!menuComplete || selected !== item.affirmative) process.exit(1); + process.stdout.write('\nTRUST_ACCEPTED\n'); + } + } + } +}); +if (item.name === 'partial') { + process.stdout.write('Accessingworkspace:\r\n❯No,exit\r\n'); + setTimeout(() => { menuComplete = true; inputReadyAt = Date.now() + 900; process.stdout.write('Yes,Itrustthisfolder\r\nEntertoconfirm'); }, 650); +} else { + menuComplete = true; + inputReadyAt = Date.now() + 900; + process.stdout.write(item.menu); +} +process.on('SIGINT', () => process.exit(0)); +process.stdin.resume(); +`); + fs.chmodSync(fake, 0o755); + const runner = pathToFileURL(path.resolve(import.meta.dir, 'helpers/claude-pty-runner.ts')).href; + fs.writeFileSync(worker, ` +import { launchClaudePty } from ${JSON.stringify(runner)}; +const cases = ${JSON.stringify(cases)}; +const results = await Promise.all(cases.map(async item => { + const session = await launchClaudePty({cwd:item.cwd, timeoutMs:8000, env:{TRUST_REPLAY:JSON.stringify(item)}}); + try { await session.waitFor('TRUST_ACCEPTED', {timeoutMs:5000}); return item.name; } + finally { await session.close(); } +})); +await Bun.write(${JSON.stringify(resultFile)}, JSON.stringify(results)); +`); + const child = Bun.spawn([process.execPath, worker], { + env: { ...process.env, BROWSE_TERMINAL_BINARY: fake, EVALS_HERMETIC: '1' }, + stdout: 'pipe', stderr: 'pipe', + }); + const killer = setTimeout(() => child.kill('SIGKILL'), 12_000); + try { + const [exitCode, stdout, stderr] = await Promise.all([ + child.exited, new Response(child.stdout).text(), new Response(child.stderr).text(), + ]); + expect(exitCode, stdout + stderr).toBe(0); + expect(JSON.parse(fs.readFileSync(resultFile, 'utf8'))).toEqual(cases.map((item) => item.name)); + for (const item of cases) { + const events = fs.readFileSync(item.record, 'utf8').trim().split('\n').map((line) => JSON.parse(line)); + expect(events[0].cwd).toBe(fs.realpathSync(item.cwd)); + expect(events[0].config).not.toBe(path.join(os.homedir(), '.claude')); + const inputs = events.filter((event) => event.type === 'input'); + expect(inputs.map((event) => event.data).join('')).toBe(item.expected); + expect(inputs.every((event) => event.menuComplete)).toBe(true); + expect(inputs.every((event) => event.inputReady)).toBe(true); + if (item.affirmative !== item.selected) expect(inputs.map((event) => event.data)).toEqual(['\x1b[B', '\r']); + expect(() => process.kill(events[0].pid, 0)).toThrow(); + } + } finally { + clearTimeout(killer); + child.kill('SIGKILL'); + for (const item of cases) { + if (!fs.existsSync(item.record)) continue; + const first = JSON.parse(fs.readFileSync(item.record, 'utf8').split('\n')[0]!); + try { process.kill(first.pid, 'SIGKILL'); } catch { /* already reaped */ } + } + fs.rmSync(dir, { recursive: true, force: true }); + } + }, 15_000); +}); diff --git a/test/skill-e2e-plan-ceo-mode-routing.test.ts b/test/skill-e2e-plan-ceo-mode-routing.test.ts index a27c61007..a46bb5dad 100644 --- a/test/skill-e2e-plan-ceo-mode-routing.test.ts +++ b/test/skill-e2e-plan-ceo-mode-routing.test.ts @@ -44,6 +44,7 @@ import { type ClaudePtySession, } from './helpers/claude-pty-runner'; import { findCeoModeOption } from './helpers/ceo-mode-option'; +import { createPlanCountFixture } from './helpers/plan-count-fixture'; const describeE2E = describeE2ETier('periodic'); @@ -58,6 +59,26 @@ const CASES: ModeCase[] = [ { mode: 'SCOPE EXPANSION', postureRe: /\b(expansion|10x|delight|dream|cathedral|opt[\s-]?in)\b/i }, ]; +// Both cases review the same plan, available before the slash command starts. +// The checkout supplying skills must not become the implicit review target. +const PLAN = [ + '# Plan: Add saved project views', + '', + '## Goal', + 'Team members repeatedly recreate filters on a project task list. Let each', + 'member save a named combination of filters and sort order and reopen it later.', + '', + '## Approach', + '- Add a saved_views table scoped to the project and member.', + '- Provide authenticated create, list, update, and delete endpoints.', + '- Add a view picker and a save action beside the existing task filters.', + '- Keep existing task access rules when applying a saved view.', + '', + '## Validation', + '- Test persistence, project access, and reopening a view after task changes.', + '- Measure whether members reuse saved views during a two-week pilot.', +].join('\n'); + /** * Navigate prior AskUserQuestions by picking option 1 until we hit an AskUserQuestion whose * options match one of the 4 mode names. Returns the option index @@ -144,12 +165,15 @@ describeE2E('/plan-ceo-review mode routing (gate)', () => { test( `mode "${c.mode}" routes to its distinctive posture`, async () => { - const session = await launchClaudePty({ - permissionMode: 'plan', - timeoutMs: CAPTURE_LONG_MS, - seedSkills: true, - }); + const fixture = createPlanCountFixture(PLAN); + let session: ClaudePtySession | undefined; try { + session = await launchClaudePty({ + cwd: fixture.cwd, + permissionMode: 'plan', + timeoutMs: CAPTURE_LONG_MS, + seedSkills: true, + }); await Bun.sleep(8000); const since = session.mark(); session.send('/plan-ceo-review\r'); @@ -199,7 +223,11 @@ describeE2E('/plan-ceo-review mode routing (gate)', () => { ); } } finally { - await session.close(); + try { + await session?.close(); + } finally { + fixture.cleanup(); + } } }, CAPTURE_LONG_MS, diff --git a/test/skill-llm-eval.test.ts b/test/skill-llm-eval.test.ts index 7f7a448de..6b9eb472d 100644 --- a/test/skill-llm-eval.test.ts +++ b/test/skill-llm-eval.test.ts @@ -17,6 +17,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { callJudge, judge } from './helpers/llm-judge'; import type { JudgeScore } from './helpers/llm-judge'; +import { readWorkflowJudgeInput } from './helpers/workflow-judge-input'; import { LLM_JUDGE_TOUCHFILES } from './helpers/touchfiles'; // Runs when EVALS=1 is set (requires ANTHROPIC_API_KEY in env) — the EVALS // gate lives in the shared describeIfSelected. Selection machinery is shared @@ -586,48 +587,16 @@ async function runWorkflowJudge(opts: { const defaults = { clarity: 4, completeness: 3, actionability: 4 }; const thresholds = { ...defaults, ...opts.thresholds }; - // Read the skeleton + sections UNION so carved skills (v2 plan T9) still - // expose markers that moved into sections/*.md (e.g. plan-eng's "## Review - // Sections" + "## CRITICAL RULE", plan-design's 7 passes). Without this the - // slice markers vanish from the skeleton and the judge scores empty content. - let content = fs.readFileSync(path.join(ROOT, opts.skillPath), 'utf-8'); - const secDir = path.join(ROOT, path.dirname(opts.skillPath), 'sections'); - const sectionBodies: string[] = []; - if (fs.existsSync(secDir)) { - for (const f of fs.readdirSync(secDir).sort()) { - if (f.endsWith('.md') && !f.endsWith('.md.tmpl')) { - const body = fs.readFileSync(path.join(secDir, f), 'utf-8'); - sectionBodies.push(body); - content += '\n' + body; - } - } - } - const startIdx = content.indexOf(opts.startMarker); - if (startIdx === -1) throw new Error(`Start marker not found in ${opts.skillPath}: "${opts.startMarker}"`); - - let section: string; - if (opts.endMarker) { - const endIdx = content.indexOf(opts.endMarker, startIdx); - if (endIdx === -1) throw new Error(`End marker not found in ${opts.skillPath}: "${opts.endMarker}"`); - section = content.slice(startIdx, endIdx); - } else { - section = content.slice(startIdx); - } - - // Two carve shapes exist. plan-eng/plan-design moved the MARKERS into the - // section files, so the slice above already reaches the carved content. - // document-release instead keeps its markers in the skeleton and carves the - // workflow BODY (Steps 2-9 → sections/release-body.md) AFTER the endMarker, - // so the marker slice drops it. Re-append any carved section the window - // excluded, so the judge always sees the full workflow the agent executes. - for (const body of sectionBodies) { - const head = body.trim().slice(0, 120); - if (head && !section.includes(head)) section += '\n' + body; - } + const input = readWorkflowJudgeInput({ + root: ROOT, + skillPath: opts.skillPath, + startMarker: opts.startMarker, + endMarker: opts.endMarker, + }); const scores = await callJudge(`You are evaluating the quality of ${opts.judgeContext} for an AI coding agent. -The agent reads this document to learn ${opts.judgeGoal}. It references external tools and files +The agent reads these source files to learn ${opts.judgeGoal}. They reference external tools and files that are documented separately — do NOT penalize for missing external definitions. Rate on three dimensions (1-5 scale): @@ -638,9 +607,9 @@ Rate on three dimensions (1-5 scale): Respond with ONLY valid JSON: {"clarity": N, "completeness": N, "actionability": N, "reasoning": "brief explanation"} -Here is the document to evaluate: +Here is the source-file bundle to evaluate: -${section}`); +${input.text}`); console.log(`${opts.testName} scores:`, JSON.stringify(scores, null, 2)); @@ -667,7 +636,7 @@ describeIfSelected('Ship & Release skill evals', ['ship/SKILL.md workflow', 'doc testName: 'ship/SKILL.md workflow', suite: 'Ship & Release skill evals', skillPath: 'ship/SKILL.md', - startMarker: '# Ship:', + startMarker: '## Step 0: Detect platform and base branch', endMarker: '## Important Rules', judgeContext: 'a ship/release workflow document', judgeGoal: 'how to create a PR: merge base branch, run tests, review diff, bump version, update changelog, push, and open PR', diff --git a/test/workflow-judge-input.test.ts b/test/workflow-judge-input.test.ts new file mode 100644 index 000000000..7880ae32c --- /dev/null +++ b/test/workflow-judge-input.test.ts @@ -0,0 +1,203 @@ +/** Free regression coverage for the file bundle sent to workflow judges. */ +import { afterEach, describe, expect, test } from 'bun:test'; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { readWorkflowJudgeInput } from './helpers/workflow-judge-input'; + +const ROOT = resolve(import.meta.dir, '..'); +const scratchRoots: string[] = []; + +afterEach(() => { + for (const root of scratchRoots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function fixture(files: Record): string { + const root = mkdtempSync(join(tmpdir(), 'gstack-workflow-judge-')); + scratchRoots.push(root); + for (const [path, content] of Object.entries(files)) { + mkdirSync(dirname(join(root, path)), { recursive: true }); + writeFileSync(join(root, path), content); + } + return root; +} + +function occurrences(text: string, needle: string): number { + return text.split(needle).length - 1; +} + +function sectionPaths(skill: string): string[] { + return readdirSync(join(ROOT, skill, 'sections')) + .filter(name => name.endsWith('.md')) + .sort() + .map(name => `${skill}/sections/${name}`); +} + +describe('workflow judge file bundle', () => { + test('preserves the exact entrypoint excerpt and each complete section in sorted named files', () => { + const entrypoint = 'excluded preamble\n## Begin\nRead sections/z-last.md when directed.\n## End\nexcluded epilogue'; + const sections = { + 'example/sections/z-last.md': 'Z section prefix\nZ section suffix', + 'example/sections/a-first.md': 'A section prefix\nA section suffix', + }; + const root = fixture({ + 'example/SKILL.md': entrypoint, + ...sections, + 'example/sections/ignored.md.tmpl': 'DO NOT EVALUATE TEMPLATE', + 'example/sections/manifest.json': '{"ignored": true}', + }); + const input = readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Begin', endMarker: '## End' }); + + expect(input.files).toEqual([ + { path: 'example/SKILL.md', kind: 'entrypoint', content: '## Begin\nRead sections/z-last.md when directed.\n', startLine: 2, endLine: 3 }, + { path: 'example/sections/a-first.md', kind: 'section', content: sections['example/sections/a-first.md'], startLine: 1, endLine: 2 }, + { path: 'example/sections/z-last.md', kind: 'section', content: sections['example/sections/z-last.md'], startLine: 1, endLine: 2 }, + ]); + expect(input.text).not.toContain('excluded preamble'); + expect(input.text).not.toContain('excluded epilogue'); + expect(input.text).not.toContain('DO NOT EVALUATE TEMPLATE'); + expect(input.text).not.toContain('manifest.json'); + for (const file of input.files) { + expect(occurrences(input.text, file.content)).toBe(1); + const begin = input.text.split('\n').filter(line => line.includes('BEGIN FILE') && line.includes(file.path)); + const end = input.text.split('\n').filter(line => line.includes('END FILE') && line.includes(file.path)); + expect(begin).toHaveLength(1); + expect(end).toHaveLength(1); + expect(begin[0]).toContain(`${file.startLine}-${file.endLine}`); + } + }); + + test('describes lazy file loading without presenting bundle order as execution order', () => { + const root = fixture({ + 'example/SKILL.md': '## Begin\nSTOP and read sections/step.md.\n## End', + 'example/sections/step.md': 'Run the separately loaded step.', + }); + const { text } = readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Begin', endMarker: '## End' }); + const header = text.slice(0, text.indexOf('BEGIN FILE')); + expect(header).toMatch(/bundle/i); + expect(header).toMatch(/file/i); + expect(header).toMatch(/separate|separately/i); + expect(header).toMatch(/lazy|on-demand/i); + expect(header).toMatch(/directives|instructions/i); + expect(header).toMatch(/(?:not|isn't|does not)[^.\n]*execution order/i); + expect(text).toContain('STOP and read sections/step.md.'); + }); + + test('retains section prelude and suffix exactly once when both markers are inside a section', () => { + // Generated comments made the old 120-character prefix heuristic append + // this whole file after its partial slice, duplicating every review pass. + const section = [ + '', + '', + 'section context before the selected marker', + '## Review Sections', + '### Pass 1: Information Architecture', + 'Evaluate the first pass.', + '## CRITICAL RULE', + 'section context after the selected marker', + ].join('\n'); + const root = fixture({ + 'example/SKILL.md': 'Entrypoint preamble.\nSTOP and read sections/review-sections.md.', + 'example/sections/review-sections.md': section, + }); + const input = readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Review Sections', endMarker: '## CRITICAL RULE' }); + + expect(input.files).toEqual([ + { path: 'example/sections/review-sections.md', kind: 'section', content: section, startLine: 1, endLine: 8 }, + ]); + expect(occurrences(input.text, '### Pass 1: Information Architecture')).toBe(1); + expect(occurrences(input.text, 'section context before the selected marker')).toBe(1); + expect(occurrences(input.text, 'section context after the selected marker')).toBe(1); + expect(occurrences(input.text, section)).toBe(1); + expect(input.text).not.toContain('Entrypoint preamble.'); + }); + + test('handles marker windows spanning files without losing section prefixes or suffixes', () => { + const root = fixture({ + 'example/SKILL.md': 'omitted\n## Begin\nEntrypoint tail', + 'example/sections/a.md': 'First section prefix\nFirst section body\nFirst section suffix', + 'example/sections/b.md': 'Second section prefix\n## End\nSecond section suffix', + }); + const input = readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Begin', endMarker: '## End' }); + + expect(input.files[0]).toEqual({ + path: 'example/SKILL.md', kind: 'entrypoint', content: '## Begin\nEntrypoint tail', startLine: 2, endLine: 3, + }); + expect(input.files.slice(1).map(file => file.content)).toEqual([ + 'First section prefix\nFirst section body\nFirst section suffix', + 'Second section prefix\n## End\nSecond section suffix', + ]); + expect(occurrences(input.text, 'Second section prefix')).toBe(1); + expect(occurrences(input.text, 'Second section suffix')).toBe(1); + }); + + test('keeps separate files even when their generated preludes and contents match', () => { + const body = '\nShared instruction'; + const root = fixture({ + 'example/SKILL.md': '## Begin\nRead both sections.\n## End', + 'example/sections/a.md': body, + 'example/sections/b.md': body, + }); + const input = readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Begin', endMarker: '## End' }); + expect(input.files.filter(file => file.kind === 'section').map(file => file.path)).toEqual([ + 'example/sections/a.md', 'example/sections/b.md', + ]); + expect(occurrences(input.text, body)).toBe(2); + }); + + test('supports an uncarved workflow and an open-ended slice', () => { + const root = fixture({ 'example/SKILL.md': 'omitted\n## Begin\nRetain this final line' }); + const input = readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Begin', endMarker: null }); + expect(input.files).toEqual([ + { path: 'example/SKILL.md', kind: 'entrypoint', content: '## Begin\nRetain this final line', startLine: 2, endLine: 3 }, + ]); + }); + + test('rejects missing start markers, missing end markers, and end markers preceding the start', () => { + const root = fixture({ 'example/SKILL.md': '## Earlier\n## Begin\nBody' }); + expect(() => readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Missing', endMarker: null })).toThrow(/Start marker not found/); + expect(() => readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Begin', endMarker: '## Missing' })).toThrow(/End marker not found/); + expect(() => readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Begin', endMarker: '## Earlier' })).toThrow(/End marker not found/); + }); + + test('generated ship includes base-branch initialization and every lazy section once', () => { + const skillPath = 'ship/SKILL.md'; + const source = readFileSync(join(ROOT, skillPath), 'utf8'); + // Read the paid caller's actual slice so changing its marker back to the + // title cannot silently drop initialization while this helper test passes. + const caller = readFileSync(join(ROOT, 'test/skill-llm-eval.test.ts'), 'utf8'); + const markers = caller.match(/skillPath: 'ship\/SKILL\.md',\s+startMarker: '([^']+)',\s+endMarker: '([^']+)'/); + expect(markers).not.toBeNull(); + const [, startMarker, endMarker] = markers!; + expect(startMarker).toBe('## Step 0: Detect platform and base branch'); + const input = readWorkflowJudgeInput({ root: ROOT, skillPath, startMarker, endMarker }); + const entrypoint = input.files.find(file => file.kind === 'entrypoint'); + expect(entrypoint?.content).toBe(source.slice(source.indexOf(startMarker), source.indexOf(endMarker, source.indexOf(startMarker)))); + expect(entrypoint?.content).toContain('git remote get-url origin'); + expect(entrypoint?.content).toContain('gh pr view --json baseRefName'); + expect(entrypoint?.content).toContain('Print the detected base branch name.'); + expect(occurrences(input.text, startMarker)).toBe(1); + const sections = input.files.filter(file => file.kind === 'section'); + expect(sections.map(file => file.path)).toEqual(sectionPaths('ship')); + for (const file of sections) { + const source = readFileSync(join(ROOT, file.path), 'utf8'); + expect(file.content).toBe(source); + expect(occurrences(input.text, source)).toBe(1); + expect(file.startLine).toBe(1); + } + }); + + test('generated plan-design passes retain their full section without duplicating Pass 1', () => { + const input = readWorkflowJudgeInput({ + root: ROOT, skillPath: 'plan-design-review/SKILL.md', startMarker: '## Review Sections', endMarker: '## CRITICAL RULE', + }); + expect(input.files.filter(file => file.kind === 'entrypoint')).toHaveLength(0); + expect(input.files.map(file => file.path)).toEqual(sectionPaths('plan-design-review')); + const section = input.files.find(file => file.path === 'plan-design-review/sections/review-sections.md'); + expect(section?.content).toBe(readFileSync(join(ROOT, 'plan-design-review/sections/review-sections.md'), 'utf8')); + expect(section?.content).toStartWith('