diff --git a/test/helpers/claude-pty-runner.ts b/test/helpers/claude-pty-runner.ts index 825b01bbf..ba56825ed 100644 --- a/test/helpers/claude-pty-runner.ts +++ b/test/helpers/claude-pty-runner.ts @@ -25,8 +25,10 @@ import { resolveEvalModel } from '../../lib/eval-model'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; +import { stripVTControlCharacters } from 'node:util'; import { hermeticChildEnv, hermeticSkillsConfigDir, isHermeticEnabled } from './hermetic-env'; import { createPlanCountFixture } from './plan-count-fixture'; +import { trustDialogInput } from './pty-trust-dialog'; /** Strip ANSI escapes for pattern-matching against visible text. */ export function stripAnsi(s: string): string { @@ -151,10 +153,9 @@ export interface ClaudePtySession { close(): Promise; } -/** Detect the workspace-trust dialog rendering. */ +/** Detect a complete, recognized workspace-trust menu. */ export function isTrustDialogVisible(visible: string): boolean { - // Phrase Claude Code prints. Stable across versions in this branch's range. - return visible.includes('trust this folder'); + return trustDialogInput(visible) !== null; } /** @@ -1283,8 +1284,8 @@ export const devexStep0Boundary: Step0BoundaryPredicate = (fp) => * handle. Caller is responsible for `await session.close()` to release the * subprocess and any timers. * - * Auto-handles the workspace-trust dialog (presses "1\r" if it appears - * during the boot window). Tests should NOT have to handle it themselves. + * Auto-handles the workspace-trust dialog by selecting its explicit + * affirmative option. Tests should NOT have to handle it themselves. */ export async function launchClaudePty( opts: ClaudePtyOptions = {}, @@ -1367,17 +1368,26 @@ export async function launchClaudePty( }, timeoutMs); // Auto-handle the workspace-trust dialog. Runs once during the boot - // window; idempotent (only fires if the phrase is still on screen). + // window, after both choices and the selected cursor are visible. Newer + // unnumbered menus default to "No, exit", so "1\r" would reject trust. + // The first paint can precede the input handler's readiness. Let startup + // settle, then deliver navigation and confirmation as separate events. let trustHandled = false; + let trustVisibleAt: number | undefined; + const trustInputTimers: ReturnType[] = []; const trustWatcher = setInterval(() => { if (trustHandled || exited) return; - const visible = stripAnsi(buffer); - if (isTrustDialogVisible(visible)) { + const input = trustDialogInput(buffer); + if (input !== null) { + trustVisibleAt ??= Date.now(); + if (Date.now() - trustVisibleAt < 1_500) return; trustHandled = true; - try { - proc.terminal?.write?.('1\r'); - } catch { - /* ignore */ + const keys = input.match(/\x1b\[[AB]|\r/g) ?? []; + for (const [i, key] of keys.entries()) { + trustInputTimers.push(setTimeout(() => { + if (exited) return; + try { proc.terminal?.write?.(key); } catch { /* ignore */ } + }, i * 500)); } } }, 200); @@ -1463,6 +1473,7 @@ export async function launchClaudePty( clearTimeout(wallTimer); clearTimeout(trustWatcherStop); clearInterval(trustWatcher); + for (const timer of trustInputTimers) clearTimeout(timer); if (exited) return; try { proc.kill?.('SIGINT'); @@ -2032,10 +2043,16 @@ export async function runPlanSkillCounting(opts: { summary: string, visible: string, ): PlanSkillCountObservation { + const clean = (text: string) => stripVTControlCharacters(text) + .replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, ''); + const failed = outcome === 'exited' || outcome === 'timeout'; return { outcome, summary, - evidence: visible.slice(-3000), + evidence: failed + ? `exitCode=${session.exitCode()}\n--- post-command evidence (last 3KB) ---\n${clean(visible).slice(-3000)}` + + `\n--- full-session evidence, including startup (last 6KB) ---\n${clean(session.visibleText()).slice(-6000)}` + : visible.slice(-3000), elapsedMs: Date.now() - startedAt, fingerprints, step0Count, diff --git a/test/helpers/pty-trust-dialog.ts b/test/helpers/pty-trust-dialog.ts new file mode 100644 index 000000000..f578e236a --- /dev/null +++ b/test/helpers/pty-trust-dialog.ts @@ -0,0 +1,37 @@ +import { stripVTControlCharacters } from 'node:util'; + +/** Select the explicit affirmative entry in a fully rendered startup menu. */ +export function trustDialogInput(visible: string): string | null { + const clean = stripVTControlCharacters(visible); + const headers = [...clean.matchAll(/Accessing\s*workspace:|Do\s*you\s*trust\s*(?:the\s*files\s*in\s*)?this\s*folder\?|Trust\s*this\s*folder\?/gi)]; + const header = headers.at(-1); + if (!header) return null; + const dialog = clean.slice(header.index); + + type Option = { selected: boolean; affirmative: boolean }; + const options: Option[] = []; + for (const line of dialog.split(/[\r\n]+/)) { + const match = line.replace(/\s+/g, '').match( + /^(❯)?(?:\d+[.)])?(Yes(?:,Itrustthisfolder|,andalwaysallowaccessto.+)?|No(?:,exit)?)(?:\(Recommended\))?$/i, + ); + if (match) options.push({ selected: !!match[1], affirmative: /^Yes/i.test(match[2]!) }); + } + + // Cursor-positioning escapes can collapse the two current menu rows into + // one logical line. Their complete labels still identify both choices. + if (options.length < 2) { + options.length = 0; + for (const match of dialog.replace(/\s+/g, '').matchAll(/(❯)?(?:\d+[.)])?(Yes,Itrustthisfolder|No,exit)/gi)) { + options.push({ selected: !!match[1], affirmative: /^Yes/i.test(match[2]!) }); + } + } + + const target = options.findIndex((option) => option.affirmative); + const selected = options.findIndex((option) => option.selected); + // Wait for both choices and a single cursor. An incomplete or unfamiliar + // frame must never turn a guessed default into an implicit rejection. + if (target < 0 || selected < 0 || !options.some((option) => !option.affirmative) || + options.filter((option) => option.selected).length !== 1) return null; + const distance = target - selected; + return (distance < 0 ? '\x1b[A' : '\x1b[B').repeat(Math.abs(distance)) + '\r'; +} diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index cf2dce00d..32fc74e18 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -100,17 +100,17 @@ export const E2E_TOUCHFILES: Record = { // include question-tuning.ts and generate-ask-user-format.ts because the // AUTO_DECIDE preamble injection lives there and changes can flip the // regression test outcome between 'asked' and 'auto_decided'. - 'plan-ceo-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-plan-mode.test.ts'], - 'plan-eng-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-eng-plan-mode.test.ts'], - 'plan-design-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts', 'test/skill-e2e-design.test.ts'], - 'plan-devex-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-devex-plan-mode.test.ts'], + 'plan-ceo-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-ceo-plan-mode.test.ts'], + 'plan-eng-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-eng-plan-mode.test.ts'], + 'plan-design-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts', 'test/skill-e2e-design.test.ts'], + 'plan-devex-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-devex-plan-mode.test.ts'], // Covers ceo (preamble misfire) + eng/design (scope-gate bypass must not // fire outside plan mode) + the named-target exception case. 4 PTY runs; // in CI these run CONCURRENT with the rest of the pty-plan-smoke suite // (--max-concurrency + --retry 1), so worst-case cost is ~2x a single // pass of each, sharing the API budget with sibling tests — not the // sequential ~+10min a local read suggests. - 'plan-mode-no-op': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-mode-no-op.test.ts'], + 'plan-mode-no-op': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-mode-no-op.test.ts'], // v1.21+ AskUserQuestion-blocked regression tests — Conductor launches // claude with `--disallowedTools AskUserQuestion --permission-mode default` @@ -120,7 +120,7 @@ export const E2E_TOUCHFILES: Record = { // INSIDE the existing 4 plan-X-review-plan-mode test files (covered // transitively by the entries above). Two new standalone files exist for // skills with no prior plan-mode test: - 'office-hours-auto-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'office-hours/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-office-hours-auto-mode.test.ts'], + 'office-hours-auto-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'office-hours/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-office-hours-auto-mode.test.ts'], 'office-hours-phase4-fork': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'office-hours/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/question-tuning.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-office-hours-phase4.test.ts'], 'llm-judge-recommendation': ['codex/**', 'test/helpers/llm-judge.ts', 'test/llm-judge-recommendation.test.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'codex/SKILL.md.tmpl', 'scripts/resolvers/review.ts'], // v1.21+ AUTO_DECIDE preserve eval (periodic). Verifies the Tool resolution @@ -128,20 +128,20 @@ export const E2E_TOUCHFILES: Record = { // written a never-ask preference, AUQ should still auto-decide rather than // surfacing the question. Touches the question-tuning + preference // infrastructure plus the resolvers that own the AUTO_DECIDE preamble. - 'auto-decide-preserved': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-session-kind', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'plan-ceo-review/**', 'bin/gstack-question-preference', 'bin/gstack-config', 'bin/gstack-slug', 'hosts/claude/hooks/question-preference-hook.ts', 'hosts/claude/hooks/spawned-directive.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-auto-decide-preserved.test.ts'], + 'auto-decide-preserved': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-session-kind', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'plan-ceo-review/**', 'bin/gstack-question-preference', 'bin/gstack-config', 'bin/gstack-slug', 'hosts/claude/hooks/question-preference-hook.ts', 'hosts/claude/hooks/spawned-directive.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-auto-decide-preserved.test.ts'], // Conductor → prose decision brief (Conductor signal makes prose the default; // the PreToolUse hook denies the flaky tool). Touches the resolver that owns // the Conductor rule, the preamble signal, the hook, and the detection helper. - 'conductor-prose': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-session-kind', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble.ts', 'plan-eng-review/**', 'hosts/claude/hooks/question-preference-hook.ts', 'hosts/claude/hooks/spawned-directive.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-conductor-prose.test.ts'], + 'conductor-prose': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-session-kind', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble.ts', 'plan-eng-review/**', 'hosts/claude/hooks/question-preference-hook.ts', 'hosts/claude/hooks/spawned-directive.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-conductor-prose.test.ts'], // Real-PTY E2E batch (#6 new tests on the harness). // Each one tests behavior the SDK harness can't observe (rendered TTY, // numbered-option lists, multi-phase ordering, idempotency state echo). 'auq-format-gate': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-ask-user-question-format-compliance.test.ts'], - 'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/ceo-mode-option.ts', 'test/ceo-mode-option.test.ts', 'test/skill-e2e-plan-ceo-mode-routing.test.ts'], - 'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-with-ui.test.ts'], - 'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-ship-idempotency.test.ts'], + 'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/ceo-mode-option.ts', 'test/ceo-mode-option.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-ceo-mode-routing.test.ts'], + 'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-design-with-ui.test.ts'], + 'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-ship-idempotency.test.ts'], 'tpa-present': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'], 'tpa-absent-linux': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'], 'tpa-broken': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'], @@ -154,34 +154,34 @@ export const E2E_TOUCHFILES: Record = { // the selector sets GSTACK_CARVE_SKILL= to scope cost to the changed // skill (D-CODEX A). Touching the registry/helper or sections.ts runs all. 'carve-section-loading': ['design-html/**', 'design-shotgun/**', 'qa/**', 'browse/**', 'retro/**', 'autoplan/**', 'spec/**', 'setup-gbrain/**', 'review/**', 'codex/**', 'land-and-deploy/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/resolvers/redact-doc.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/carve-section-loading.test.ts'], - 'autoplan-chain-pty': ['autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-autoplan-chain.test.ts'], + 'autoplan-chain-pty': ['autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-autoplan-chain.test.ts'], // Per-finding AskUserQuestion count + review-report-at-bottom assertion. // Each test drives its skill end-to-end; touchfiles include preamble + // completion-status resolvers because they affect question cadence and // terminal output (the regression surface this test catches). - 'plan-ceo-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-ceo-finding-count.test.ts'], - 'plan-eng-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-eng-finding-count.test.ts'], - 'plan-design-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-design-finding-count.test.ts'], - 'plan-devex-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts'], + 'plan-ceo-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-ceo-finding-count.test.ts'], + 'plan-eng-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-eng-finding-count.test.ts'], + 'plan-design-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-design-finding-count.test.ts'], + 'plan-devex-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts'], // Gate-tier reviewCount-floor counterparts. Catch the May 2026 transcript // bug (model wrote a plan-mode plan and ExitPlanMode'd without firing any // review-phase AskUserQuestion). Uses runPlanSkillFloorCheck — minimal // "did agent fire ANY AUQ?" observer that exits early on first non-permission // numbered-option render. ~1-3 min typical wall time per test, ~$2-6 total. - 'plan-eng-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-finding-floor.test.ts'], - 'plan-ceo-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-finding-floor.test.ts'], - 'plan-design-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-design-finding-floor.test.ts'], - 'plan-devex-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-devex-finding-floor.test.ts'], + 'plan-eng-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-finding-floor.test.ts'], + 'plan-ceo-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-finding-floor.test.ts'], + 'plan-design-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-design-finding-floor.test.ts'], + 'plan-devex-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-devex-finding-floor.test.ts'], // Multi-finding batching regression — periodic tier complement to the // gate-tier finding-floor. Catches the May 2026 transcript shape where // a model fires one AUQ then batches the rest into a "## Decisions to // confirm" plan write. runPlanSkillFloorCheck cannot detect that shape // (it exits on first AUQ); runPlanSkillCounting can. - 'plan-eng-multi-finding-batching': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts'], - 'plan-ceo-split-overflow': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts'], + 'plan-eng-multi-finding-batching': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts'], + 'plan-ceo-split-overflow': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts'], 'brain-privacy-gate': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'bin/gstack-brain-sync', 'bin/gstack-artifacts-init', 'bin/gstack-config', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-brain-privacy-gate.test.ts'], // /setup-gbrain Path 4 (Remote MCP) — happy + bad-token end-to-end via @@ -850,30 +850,30 @@ export const LLM_JUDGE_TOUCHFILES: Record = { 'baseline score pinning': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'test/fixtures/eval-baselines.json', 'test/skill-llm-eval.test.ts'], // Ship & Release - 'ship/SKILL.md workflow': ['ship/SKILL.md', 'ship/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], - 'document-release/SKILL.md workflow': ['document-release/SKILL.md', 'document-release/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'ship/SKILL.md workflow': ['ship/SKILL.md', 'ship/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], + 'document-release/SKILL.md workflow': ['document-release/SKILL.md', 'document-release/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], // Plan Reviews - 'plan-ceo-review/SKILL.md modes': ['plan-ceo-review/SKILL.md', 'plan-ceo-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], - 'plan-eng-review/SKILL.md sections': ['plan-eng-review/SKILL.md', 'plan-eng-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'plan-ceo-review/SKILL.md modes': ['plan-ceo-review/SKILL.md', 'plan-ceo-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], + 'plan-eng-review/SKILL.md sections': ['plan-eng-review/SKILL.md', 'plan-eng-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], // /spec authored-spec quality (paid LLM-judge — periodic-tier). - 'plan-design-review/SKILL.md passes': ['plan-design-review/SKILL.md', 'plan-design-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'plan-design-review/SKILL.md passes': ['plan-design-review/SKILL.md', 'plan-design-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], // Design skills - 'design-review/SKILL.md fix loop': ['design-review/SKILL.md', 'design-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], - 'design-consultation/SKILL.md research': ['design-consultation/SKILL.md', 'design-consultation/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'design-review/SKILL.md fix loop': ['design-review/SKILL.md', 'design-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], + 'design-consultation/SKILL.md research': ['design-consultation/SKILL.md', 'design-consultation/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], // Deploy skills - 'land-and-deploy/SKILL.md workflow': ['land-and-deploy/SKILL.md', 'land-and-deploy/SKILL.md.tmpl', 'land-and-deploy/sections/**', 'test/skill-llm-eval.test.ts'], - 'canary/SKILL.md monitoring loop': ['canary/SKILL.md', 'canary/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], - 'benchmark/SKILL.md perf collection': ['benchmark/SKILL.md', 'benchmark/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], - 'setup-deploy/SKILL.md platform setup': ['setup-deploy/SKILL.md', 'setup-deploy/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'land-and-deploy/SKILL.md workflow': ['land-and-deploy/SKILL.md', 'land-and-deploy/SKILL.md.tmpl', 'land-and-deploy/sections/**', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], + 'canary/SKILL.md monitoring loop': ['canary/SKILL.md', 'canary/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], + 'benchmark/SKILL.md perf collection': ['benchmark/SKILL.md', 'benchmark/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], + 'setup-deploy/SKILL.md platform setup': ['setup-deploy/SKILL.md', 'setup-deploy/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], // Other skills - 'retro/SKILL.md instructions': ['retro/sections/**', 'retro/SKILL.md', 'retro/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], - 'qa-only/SKILL.md workflow': ['qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], - 'gstack-upgrade/SKILL.md upgrade flow': ['gstack-upgrade/SKILL.md', 'gstack-upgrade/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'], + 'retro/SKILL.md instructions': ['retro/sections/**', 'retro/SKILL.md', 'retro/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], + 'qa-only/SKILL.md workflow': ['qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], + 'gstack-upgrade/SKILL.md upgrade flow': ['gstack-upgrade/SKILL.md', 'gstack-upgrade/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/workflow-judge-input.test.ts'], // Voice directive 'voice directive tone': ['scripts/resolvers/preamble.ts', 'review/SKILL.md', 'review/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-llm-eval.test.ts'], diff --git a/test/helpers/workflow-judge-input.ts b/test/helpers/workflow-judge-input.ts new file mode 100644 index 000000000..135ba81af --- /dev/null +++ b/test/helpers/workflow-judge-input.ts @@ -0,0 +1,86 @@ +/** Preserve source-file boundaries when a workflow judge reads carved skills. */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +export interface WorkflowJudgeFile { + path: string; + kind: 'entrypoint' | 'section'; + content: string; + startLine: number; + endLine: number; +} + +export interface WorkflowJudgeInput { + files: WorkflowJudgeFile[]; + text: string; +} + +export function readWorkflowJudgeInput(opts: { + root: string; + skillPath: string; + startMarker: string; + endMarker: string | null; +}): WorkflowJudgeInput { + const sources = [{ + path: opts.skillPath, + kind: 'entrypoint' as const, + content: fs.readFileSync(path.join(opts.root, opts.skillPath), 'utf8'), + }]; + const sectionDir = path.join(path.dirname(opts.skillPath), 'sections'); + const sectionRoot = path.join(opts.root, sectionDir); + const sections = fs.existsSync(sectionRoot) + ? fs.readdirSync(sectionRoot).sort().filter(name => name.endsWith('.md')) + .map(name => ({ + path: path.join(sectionDir, name), + kind: 'section' as const, + content: fs.readFileSync(path.join(sectionRoot, name), 'utf8'), + })) + : []; + const allSources = [...sources, ...sections]; + + // Preserve the existing marker window, including markers that moved into + // section files. Offsets identify the source of each slice; prose prefixes + // cannot reliably identify a file after its generated header was sliced off. + const union = allSources.map(file => file.content).join('\n'); + const start = union.indexOf(opts.startMarker); + if (start < 0) throw new Error(`Start marker not found in ${opts.skillPath}: "${opts.startMarker}"`); + const end = opts.endMarker === null ? union.length : union.indexOf(opts.endMarker, start); + if (end < 0) throw new Error(`End marker not found in ${opts.skillPath}: "${opts.endMarker}"`); + + const files: WorkflowJudgeFile[] = []; + let offset = 0; + for (const file of allSources) { + // Every section was already supplied in full by the old judge input. Keep + // that coverage, but include each file once even when the marker window + // also covers part of it. Only the entrypoint retains the requested slice. + const from = file.kind === 'section' ? 0 : Math.max(0, start - offset); + const to = file.kind === 'section' ? file.content.length : Math.min(file.content.length, end - offset); + if (from < to) { + files.push({ + ...file, + path: file.path.split(path.sep).join('/'), + content: file.content.slice(from, to), + startLine: file.content.slice(0, from).split('\n').length, + endLine: file.content.slice(0, to - 1).split('\n').length, + }); + } + offset += file.content.length + 1; + } + + const context = [ + 'The material below is a bundle of source-file excerpts, with each original file and line range labeled.', + 'SKILL.md is the entry point; the labeled ranges identify which excerpts are supplied.', + ...(sections.length > 0 ? [ + 'The section files remain separate on disk and are read at the points and conditions specified by the skill\'s Read directives.', + 'They are supplied here as on-demand references; their order in this bundle is not execution order.', + ] : []), + ].join('\n'); + return { + files, + text: context + '\n\n' + files.map(file => [ + `--- BEGIN FILE ${JSON.stringify(file.path)} (lines ${file.startLine}-${file.endLine}; ${file.kind}) ---`, + file.content, + `--- END FILE ${JSON.stringify(file.path)} ---`, + ].join('\n')).join('\n\n'), + }; +} diff --git a/test/plan-count-fixture.test.ts b/test/plan-count-fixture.test.ts index 61e80199e..95a6e0183 100644 --- a/test/plan-count-fixture.test.ts +++ b/test/plan-count-fixture.test.ts @@ -124,6 +124,7 @@ record({ sections: fs.existsSync(path.join(skillDir, 'sections', 'review-sections.md')) ? fs.readFileSync(path.join(skillDir, 'sections', 'review-sections.md'), 'utf8') : null, }); +if (process.env.FIXTURE_MODE === 'exit') process.stdout.write('\x1b[?25lSTARTUP_DIAGNOSTIC fixture CLI booted\x1b[?25h\n'); process.stdin.setRawMode?.(true); let firstInput = true; let completion; @@ -201,6 +202,11 @@ await Bun.write(${JSON.stringify(resultPath)}, JSON.stringify(results)); expect(() => process.kill(startup.pid, 0)).toThrow(); const result = results.find((result) => result.name === item.name); expect(result.observation.outcome).toBe(item.mode === 'exit' ? 'exited' : 'completion_summary'); + if (item.mode === 'exit') { + expect(result.observation.evidence).toContain('exitCode=7'); + expect(result.observation.evidence).toContain('STARTUP_DIAGNOSTIC fixture CLI booted'); + expect(result.observation.evidence).not.toContain('\x1b'); + } if (item.mode === 'complete') expect(events.at(-1).type).toBe('closed'); cwds.add(startup.cwd); } diff --git a/test/pty-trust-dialog.test.ts b/test/pty-trust-dialog.test.ts new file mode 100644 index 000000000..7be74a22d --- /dev/null +++ b/test/pty-trust-dialog.test.ts @@ -0,0 +1,140 @@ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { trustDialogInput } from './helpers/pty-trust-dialog'; + +const OLD = 'Do you trust the files in this folder?\n❯ 1. Yes, I trust this folder\n 2. No, exit\nEnter to confirm'; +const CURRENT = 'Accessingworkspace:\r\n/tmp/fixture\r\nQuicksafetycheck:Isthisaprojectyoucreatedoroneyoutrust?\r\n❯No,exit\r\nYes,Itrustthisfolder\r\nEntertoconfirm·Esctocancel'; + +describe('startup trust selection', () => { + test('confirms the explicit affirmative choice in the old numbered menu', () => { + expect(trustDialogInput(OLD)).toBe('\r'); + expect(trustDialogInput('Do you trust the files in this folder?\n 1. Yes\n 2. Yes, and always allow access to /tmp/fixture\n❯ 3. No, exit')) + .toBe('\x1b[A\x1b[A\r'); + }); + + test('moves to affirmative when the current unnumbered menu defaults to exit', () => { + expect(trustDialogInput(CURRENT)).toBe('\x1b[B\r'); + expect(trustDialogInput(CURRENT.replace('❯No,exit', 'No,exit').replace('Yes,Itrust', '❯Yes,Itrust'))).toBe('\r'); + }); + + test('handles ANSI cursor visibility and rows collapsed by cursor positioning', () => { + expect(trustDialogInput('\x1b[?25l' + CURRENT + '\x1b[?25h')).toBe('\x1b[B\r'); + expect(trustDialogInput('Accessing workspace:\n❯No,\x1b[1Cexit\x1b[1BYes,I\x1b[1Ctrustthisfolder')) + .toBe('\x1b[B\r'); + }); + + test('waits for complete choices and a known selected option', () => { + expect(trustDialogInput('Accessing workspace:\n❯ No, exit')).toBeNull(); + expect(trustDialogInput('Accessing workspace:\n❯ Yes, I trust this folder')).toBeNull(); + expect(trustDialogInput(CURRENT.replace('❯', ''))).toBeNull(); + expect(trustDialogInput('Accessing workspace:\n❯ No, exit\n❯ Yes, I trust this folder')).toBeNull(); + expect(trustDialogInput('Do you trust the files in this folder?\n❯ Continue\nCancel')).toBeNull(); + }); + + test('ignores ordinary questions and uses the latest full dialog redraw', () => { + expect(trustDialogInput('Should we enable a setting?\n❯ 1. Yes\n2. No')).toBeNull(); + expect(trustDialogInput('The text says "trust this folder" but no menu is shown.')).toBeNull(); + expect(trustDialogInput(OLD + '\n' + CURRENT)).toBe('\x1b[B\r'); + }); + + test.skipIf(process.platform === 'win32')('the real PTY driver accepts both menu orders, including a partial current render', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-trust-replay-')); + const fake = path.join(dir, 'fake-claude'); + const worker = path.join(dir, 'worker.ts'); + const resultFile = path.join(dir, 'result.json'); + const cases = [ + { name: 'old', menu: OLD, selected: 0, affirmative: 0, expected: '\r' }, + { name: 'current', menu: CURRENT, selected: 0, affirmative: 1, expected: '\x1b[B\r' }, + { name: 'partial', menu: CURRENT, selected: 0, affirmative: 1, expected: '\x1b[B\r' }, + ].map((item) => ({ ...item, cwd: path.join(dir, item.name), record: path.join(dir, item.name + '.jsonl') })); + for (const item of cases) fs.mkdirSync(item.cwd); + fs.writeFileSync(fake, `#!${process.execPath}\n` + String.raw` +import * as fs from 'node:fs'; +const item = JSON.parse(process.env.TRUST_REPLAY); +const record = (event) => fs.appendFileSync(item.record, JSON.stringify(event) + '\n'); +record({type:'startup', pid:process.pid, cwd:process.cwd(), config:process.env.CLAUDE_CONFIG_DIR}); +process.stdin.setRawMode?.(true); +let selected = item.selected; +let menuComplete = false; +let inputReadyAt = Infinity; +let pending = ''; +process.stdin.on('data', (data) => { + const inputReady = Date.now() >= inputReadyAt; + record({type:'input', data:data.toString(), menuComplete, inputReady}); + // Current Claude startup can paint the menu before its input listener is + // ready. Early navigation is lost; an early Enter retains the exit default. + if (!inputReady) return; + pending += data.toString(); + while (pending.length) { + if (pending.startsWith('\x1b[A')) { setTimeout(() => { selected -= 1; }, 100); pending = pending.slice(3); } + else if (pending.startsWith('\x1b[B')) { setTimeout(() => { selected += 1; }, 100); pending = pending.slice(3); } + else if (pending[0] === '\x1b' && pending.length < 3) break; + else { + const key = pending[0]; pending = pending.slice(1); + if (key === '\r') { + if (!menuComplete || selected !== item.affirmative) process.exit(1); + process.stdout.write('\nTRUST_ACCEPTED\n'); + } + } + } +}); +if (item.name === 'partial') { + process.stdout.write('Accessingworkspace:\r\n❯No,exit\r\n'); + setTimeout(() => { menuComplete = true; inputReadyAt = Date.now() + 900; process.stdout.write('Yes,Itrustthisfolder\r\nEntertoconfirm'); }, 650); +} else { + menuComplete = true; + inputReadyAt = Date.now() + 900; + process.stdout.write(item.menu); +} +process.on('SIGINT', () => process.exit(0)); +process.stdin.resume(); +`); + fs.chmodSync(fake, 0o755); + const runner = pathToFileURL(path.resolve(import.meta.dir, 'helpers/claude-pty-runner.ts')).href; + fs.writeFileSync(worker, ` +import { launchClaudePty } from ${JSON.stringify(runner)}; +const cases = ${JSON.stringify(cases)}; +const results = await Promise.all(cases.map(async item => { + const session = await launchClaudePty({cwd:item.cwd, timeoutMs:8000, env:{TRUST_REPLAY:JSON.stringify(item)}}); + try { await session.waitFor('TRUST_ACCEPTED', {timeoutMs:5000}); return item.name; } + finally { await session.close(); } +})); +await Bun.write(${JSON.stringify(resultFile)}, JSON.stringify(results)); +`); + const child = Bun.spawn([process.execPath, worker], { + env: { ...process.env, BROWSE_TERMINAL_BINARY: fake, EVALS_HERMETIC: '1' }, + stdout: 'pipe', stderr: 'pipe', + }); + const killer = setTimeout(() => child.kill('SIGKILL'), 12_000); + try { + const [exitCode, stdout, stderr] = await Promise.all([ + child.exited, new Response(child.stdout).text(), new Response(child.stderr).text(), + ]); + expect(exitCode, stdout + stderr).toBe(0); + expect(JSON.parse(fs.readFileSync(resultFile, 'utf8'))).toEqual(cases.map((item) => item.name)); + for (const item of cases) { + const events = fs.readFileSync(item.record, 'utf8').trim().split('\n').map((line) => JSON.parse(line)); + expect(events[0].cwd).toBe(fs.realpathSync(item.cwd)); + expect(events[0].config).not.toBe(path.join(os.homedir(), '.claude')); + const inputs = events.filter((event) => event.type === 'input'); + expect(inputs.map((event) => event.data).join('')).toBe(item.expected); + expect(inputs.every((event) => event.menuComplete)).toBe(true); + expect(inputs.every((event) => event.inputReady)).toBe(true); + if (item.affirmative !== item.selected) expect(inputs.map((event) => event.data)).toEqual(['\x1b[B', '\r']); + expect(() => process.kill(events[0].pid, 0)).toThrow(); + } + } finally { + clearTimeout(killer); + child.kill('SIGKILL'); + for (const item of cases) { + if (!fs.existsSync(item.record)) continue; + const first = JSON.parse(fs.readFileSync(item.record, 'utf8').split('\n')[0]!); + try { process.kill(first.pid, 'SIGKILL'); } catch { /* already reaped */ } + } + fs.rmSync(dir, { recursive: true, force: true }); + } + }, 15_000); +}); diff --git a/test/skill-e2e-plan-ceo-mode-routing.test.ts b/test/skill-e2e-plan-ceo-mode-routing.test.ts index a27c61007..a46bb5dad 100644 --- a/test/skill-e2e-plan-ceo-mode-routing.test.ts +++ b/test/skill-e2e-plan-ceo-mode-routing.test.ts @@ -44,6 +44,7 @@ import { type ClaudePtySession, } from './helpers/claude-pty-runner'; import { findCeoModeOption } from './helpers/ceo-mode-option'; +import { createPlanCountFixture } from './helpers/plan-count-fixture'; const describeE2E = describeE2ETier('periodic'); @@ -58,6 +59,26 @@ const CASES: ModeCase[] = [ { mode: 'SCOPE EXPANSION', postureRe: /\b(expansion|10x|delight|dream|cathedral|opt[\s-]?in)\b/i }, ]; +// Both cases review the same plan, available before the slash command starts. +// The checkout supplying skills must not become the implicit review target. +const PLAN = [ + '# Plan: Add saved project views', + '', + '## Goal', + 'Team members repeatedly recreate filters on a project task list. Let each', + 'member save a named combination of filters and sort order and reopen it later.', + '', + '## Approach', + '- Add a saved_views table scoped to the project and member.', + '- Provide authenticated create, list, update, and delete endpoints.', + '- Add a view picker and a save action beside the existing task filters.', + '- Keep existing task access rules when applying a saved view.', + '', + '## Validation', + '- Test persistence, project access, and reopening a view after task changes.', + '- Measure whether members reuse saved views during a two-week pilot.', +].join('\n'); + /** * Navigate prior AskUserQuestions by picking option 1 until we hit an AskUserQuestion whose * options match one of the 4 mode names. Returns the option index @@ -144,12 +165,15 @@ describeE2E('/plan-ceo-review mode routing (gate)', () => { test( `mode "${c.mode}" routes to its distinctive posture`, async () => { - const session = await launchClaudePty({ - permissionMode: 'plan', - timeoutMs: CAPTURE_LONG_MS, - seedSkills: true, - }); + const fixture = createPlanCountFixture(PLAN); + let session: ClaudePtySession | undefined; try { + session = await launchClaudePty({ + cwd: fixture.cwd, + permissionMode: 'plan', + timeoutMs: CAPTURE_LONG_MS, + seedSkills: true, + }); await Bun.sleep(8000); const since = session.mark(); session.send('/plan-ceo-review\r'); @@ -199,7 +223,11 @@ describeE2E('/plan-ceo-review mode routing (gate)', () => { ); } } finally { - await session.close(); + try { + await session?.close(); + } finally { + fixture.cleanup(); + } } }, CAPTURE_LONG_MS, diff --git a/test/skill-llm-eval.test.ts b/test/skill-llm-eval.test.ts index 7f7a448de..6b9eb472d 100644 --- a/test/skill-llm-eval.test.ts +++ b/test/skill-llm-eval.test.ts @@ -17,6 +17,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { callJudge, judge } from './helpers/llm-judge'; import type { JudgeScore } from './helpers/llm-judge'; +import { readWorkflowJudgeInput } from './helpers/workflow-judge-input'; import { LLM_JUDGE_TOUCHFILES } from './helpers/touchfiles'; // Runs when EVALS=1 is set (requires ANTHROPIC_API_KEY in env) — the EVALS // gate lives in the shared describeIfSelected. Selection machinery is shared @@ -586,48 +587,16 @@ async function runWorkflowJudge(opts: { const defaults = { clarity: 4, completeness: 3, actionability: 4 }; const thresholds = { ...defaults, ...opts.thresholds }; - // Read the skeleton + sections UNION so carved skills (v2 plan T9) still - // expose markers that moved into sections/*.md (e.g. plan-eng's "## Review - // Sections" + "## CRITICAL RULE", plan-design's 7 passes). Without this the - // slice markers vanish from the skeleton and the judge scores empty content. - let content = fs.readFileSync(path.join(ROOT, opts.skillPath), 'utf-8'); - const secDir = path.join(ROOT, path.dirname(opts.skillPath), 'sections'); - const sectionBodies: string[] = []; - if (fs.existsSync(secDir)) { - for (const f of fs.readdirSync(secDir).sort()) { - if (f.endsWith('.md') && !f.endsWith('.md.tmpl')) { - const body = fs.readFileSync(path.join(secDir, f), 'utf-8'); - sectionBodies.push(body); - content += '\n' + body; - } - } - } - const startIdx = content.indexOf(opts.startMarker); - if (startIdx === -1) throw new Error(`Start marker not found in ${opts.skillPath}: "${opts.startMarker}"`); - - let section: string; - if (opts.endMarker) { - const endIdx = content.indexOf(opts.endMarker, startIdx); - if (endIdx === -1) throw new Error(`End marker not found in ${opts.skillPath}: "${opts.endMarker}"`); - section = content.slice(startIdx, endIdx); - } else { - section = content.slice(startIdx); - } - - // Two carve shapes exist. plan-eng/plan-design moved the MARKERS into the - // section files, so the slice above already reaches the carved content. - // document-release instead keeps its markers in the skeleton and carves the - // workflow BODY (Steps 2-9 → sections/release-body.md) AFTER the endMarker, - // so the marker slice drops it. Re-append any carved section the window - // excluded, so the judge always sees the full workflow the agent executes. - for (const body of sectionBodies) { - const head = body.trim().slice(0, 120); - if (head && !section.includes(head)) section += '\n' + body; - } + const input = readWorkflowJudgeInput({ + root: ROOT, + skillPath: opts.skillPath, + startMarker: opts.startMarker, + endMarker: opts.endMarker, + }); const scores = await callJudge(`You are evaluating the quality of ${opts.judgeContext} for an AI coding agent. -The agent reads this document to learn ${opts.judgeGoal}. It references external tools and files +The agent reads these source files to learn ${opts.judgeGoal}. They reference external tools and files that are documented separately — do NOT penalize for missing external definitions. Rate on three dimensions (1-5 scale): @@ -638,9 +607,9 @@ Rate on three dimensions (1-5 scale): Respond with ONLY valid JSON: {"clarity": N, "completeness": N, "actionability": N, "reasoning": "brief explanation"} -Here is the document to evaluate: +Here is the source-file bundle to evaluate: -${section}`); +${input.text}`); console.log(`${opts.testName} scores:`, JSON.stringify(scores, null, 2)); @@ -667,7 +636,7 @@ describeIfSelected('Ship & Release skill evals', ['ship/SKILL.md workflow', 'doc testName: 'ship/SKILL.md workflow', suite: 'Ship & Release skill evals', skillPath: 'ship/SKILL.md', - startMarker: '# Ship:', + startMarker: '## Step 0: Detect platform and base branch', endMarker: '## Important Rules', judgeContext: 'a ship/release workflow document', judgeGoal: 'how to create a PR: merge base branch, run tests, review diff, bump version, update changelog, push, and open PR', diff --git a/test/workflow-judge-input.test.ts b/test/workflow-judge-input.test.ts new file mode 100644 index 000000000..7880ae32c --- /dev/null +++ b/test/workflow-judge-input.test.ts @@ -0,0 +1,203 @@ +/** Free regression coverage for the file bundle sent to workflow judges. */ +import { afterEach, describe, expect, test } from 'bun:test'; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { readWorkflowJudgeInput } from './helpers/workflow-judge-input'; + +const ROOT = resolve(import.meta.dir, '..'); +const scratchRoots: string[] = []; + +afterEach(() => { + for (const root of scratchRoots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function fixture(files: Record): string { + const root = mkdtempSync(join(tmpdir(), 'gstack-workflow-judge-')); + scratchRoots.push(root); + for (const [path, content] of Object.entries(files)) { + mkdirSync(dirname(join(root, path)), { recursive: true }); + writeFileSync(join(root, path), content); + } + return root; +} + +function occurrences(text: string, needle: string): number { + return text.split(needle).length - 1; +} + +function sectionPaths(skill: string): string[] { + return readdirSync(join(ROOT, skill, 'sections')) + .filter(name => name.endsWith('.md')) + .sort() + .map(name => `${skill}/sections/${name}`); +} + +describe('workflow judge file bundle', () => { + test('preserves the exact entrypoint excerpt and each complete section in sorted named files', () => { + const entrypoint = 'excluded preamble\n## Begin\nRead sections/z-last.md when directed.\n## End\nexcluded epilogue'; + const sections = { + 'example/sections/z-last.md': 'Z section prefix\nZ section suffix', + 'example/sections/a-first.md': 'A section prefix\nA section suffix', + }; + const root = fixture({ + 'example/SKILL.md': entrypoint, + ...sections, + 'example/sections/ignored.md.tmpl': 'DO NOT EVALUATE TEMPLATE', + 'example/sections/manifest.json': '{"ignored": true}', + }); + const input = readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Begin', endMarker: '## End' }); + + expect(input.files).toEqual([ + { path: 'example/SKILL.md', kind: 'entrypoint', content: '## Begin\nRead sections/z-last.md when directed.\n', startLine: 2, endLine: 3 }, + { path: 'example/sections/a-first.md', kind: 'section', content: sections['example/sections/a-first.md'], startLine: 1, endLine: 2 }, + { path: 'example/sections/z-last.md', kind: 'section', content: sections['example/sections/z-last.md'], startLine: 1, endLine: 2 }, + ]); + expect(input.text).not.toContain('excluded preamble'); + expect(input.text).not.toContain('excluded epilogue'); + expect(input.text).not.toContain('DO NOT EVALUATE TEMPLATE'); + expect(input.text).not.toContain('manifest.json'); + for (const file of input.files) { + expect(occurrences(input.text, file.content)).toBe(1); + const begin = input.text.split('\n').filter(line => line.includes('BEGIN FILE') && line.includes(file.path)); + const end = input.text.split('\n').filter(line => line.includes('END FILE') && line.includes(file.path)); + expect(begin).toHaveLength(1); + expect(end).toHaveLength(1); + expect(begin[0]).toContain(`${file.startLine}-${file.endLine}`); + } + }); + + test('describes lazy file loading without presenting bundle order as execution order', () => { + const root = fixture({ + 'example/SKILL.md': '## Begin\nSTOP and read sections/step.md.\n## End', + 'example/sections/step.md': 'Run the separately loaded step.', + }); + const { text } = readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Begin', endMarker: '## End' }); + const header = text.slice(0, text.indexOf('BEGIN FILE')); + expect(header).toMatch(/bundle/i); + expect(header).toMatch(/file/i); + expect(header).toMatch(/separate|separately/i); + expect(header).toMatch(/lazy|on-demand/i); + expect(header).toMatch(/directives|instructions/i); + expect(header).toMatch(/(?:not|isn't|does not)[^.\n]*execution order/i); + expect(text).toContain('STOP and read sections/step.md.'); + }); + + test('retains section prelude and suffix exactly once when both markers are inside a section', () => { + // Generated comments made the old 120-character prefix heuristic append + // this whole file after its partial slice, duplicating every review pass. + const section = [ + '', + '', + 'section context before the selected marker', + '## Review Sections', + '### Pass 1: Information Architecture', + 'Evaluate the first pass.', + '## CRITICAL RULE', + 'section context after the selected marker', + ].join('\n'); + const root = fixture({ + 'example/SKILL.md': 'Entrypoint preamble.\nSTOP and read sections/review-sections.md.', + 'example/sections/review-sections.md': section, + }); + const input = readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Review Sections', endMarker: '## CRITICAL RULE' }); + + expect(input.files).toEqual([ + { path: 'example/sections/review-sections.md', kind: 'section', content: section, startLine: 1, endLine: 8 }, + ]); + expect(occurrences(input.text, '### Pass 1: Information Architecture')).toBe(1); + expect(occurrences(input.text, 'section context before the selected marker')).toBe(1); + expect(occurrences(input.text, 'section context after the selected marker')).toBe(1); + expect(occurrences(input.text, section)).toBe(1); + expect(input.text).not.toContain('Entrypoint preamble.'); + }); + + test('handles marker windows spanning files without losing section prefixes or suffixes', () => { + const root = fixture({ + 'example/SKILL.md': 'omitted\n## Begin\nEntrypoint tail', + 'example/sections/a.md': 'First section prefix\nFirst section body\nFirst section suffix', + 'example/sections/b.md': 'Second section prefix\n## End\nSecond section suffix', + }); + const input = readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Begin', endMarker: '## End' }); + + expect(input.files[0]).toEqual({ + path: 'example/SKILL.md', kind: 'entrypoint', content: '## Begin\nEntrypoint tail', startLine: 2, endLine: 3, + }); + expect(input.files.slice(1).map(file => file.content)).toEqual([ + 'First section prefix\nFirst section body\nFirst section suffix', + 'Second section prefix\n## End\nSecond section suffix', + ]); + expect(occurrences(input.text, 'Second section prefix')).toBe(1); + expect(occurrences(input.text, 'Second section suffix')).toBe(1); + }); + + test('keeps separate files even when their generated preludes and contents match', () => { + const body = '\nShared instruction'; + const root = fixture({ + 'example/SKILL.md': '## Begin\nRead both sections.\n## End', + 'example/sections/a.md': body, + 'example/sections/b.md': body, + }); + const input = readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Begin', endMarker: '## End' }); + expect(input.files.filter(file => file.kind === 'section').map(file => file.path)).toEqual([ + 'example/sections/a.md', 'example/sections/b.md', + ]); + expect(occurrences(input.text, body)).toBe(2); + }); + + test('supports an uncarved workflow and an open-ended slice', () => { + const root = fixture({ 'example/SKILL.md': 'omitted\n## Begin\nRetain this final line' }); + const input = readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Begin', endMarker: null }); + expect(input.files).toEqual([ + { path: 'example/SKILL.md', kind: 'entrypoint', content: '## Begin\nRetain this final line', startLine: 2, endLine: 3 }, + ]); + }); + + test('rejects missing start markers, missing end markers, and end markers preceding the start', () => { + const root = fixture({ 'example/SKILL.md': '## Earlier\n## Begin\nBody' }); + expect(() => readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Missing', endMarker: null })).toThrow(/Start marker not found/); + expect(() => readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Begin', endMarker: '## Missing' })).toThrow(/End marker not found/); + expect(() => readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Begin', endMarker: '## Earlier' })).toThrow(/End marker not found/); + }); + + test('generated ship includes base-branch initialization and every lazy section once', () => { + const skillPath = 'ship/SKILL.md'; + const source = readFileSync(join(ROOT, skillPath), 'utf8'); + // Read the paid caller's actual slice so changing its marker back to the + // title cannot silently drop initialization while this helper test passes. + const caller = readFileSync(join(ROOT, 'test/skill-llm-eval.test.ts'), 'utf8'); + const markers = caller.match(/skillPath: 'ship\/SKILL\.md',\s+startMarker: '([^']+)',\s+endMarker: '([^']+)'/); + expect(markers).not.toBeNull(); + const [, startMarker, endMarker] = markers!; + expect(startMarker).toBe('## Step 0: Detect platform and base branch'); + const input = readWorkflowJudgeInput({ root: ROOT, skillPath, startMarker, endMarker }); + const entrypoint = input.files.find(file => file.kind === 'entrypoint'); + expect(entrypoint?.content).toBe(source.slice(source.indexOf(startMarker), source.indexOf(endMarker, source.indexOf(startMarker)))); + expect(entrypoint?.content).toContain('git remote get-url origin'); + expect(entrypoint?.content).toContain('gh pr view --json baseRefName'); + expect(entrypoint?.content).toContain('Print the detected base branch name.'); + expect(occurrences(input.text, startMarker)).toBe(1); + const sections = input.files.filter(file => file.kind === 'section'); + expect(sections.map(file => file.path)).toEqual(sectionPaths('ship')); + for (const file of sections) { + const source = readFileSync(join(ROOT, file.path), 'utf8'); + expect(file.content).toBe(source); + expect(occurrences(input.text, source)).toBe(1); + expect(file.startLine).toBe(1); + } + }); + + test('generated plan-design passes retain their full section without duplicating Pass 1', () => { + const input = readWorkflowJudgeInput({ + root: ROOT, skillPath: 'plan-design-review/SKILL.md', startMarker: '## Review Sections', endMarker: '## CRITICAL RULE', + }); + expect(input.files.filter(file => file.kind === 'entrypoint')).toHaveLength(0); + expect(input.files.map(file => file.path)).toEqual(sectionPaths('plan-design-review')); + const section = input.files.find(file => file.path === 'plan-design-review/sections/review-sections.md'); + expect(section?.content).toBe(readFileSync(join(ROOT, 'plan-design-review/sections/review-sections.md'), 'utf8')); + expect(section?.content).toStartWith('