diff --git a/test/helpers/skill-fixture.ts b/test/helpers/skill-fixture.ts new file mode 100644 index 000000000..a9abff715 --- /dev/null +++ b/test/helpers/skill-fixture.ts @@ -0,0 +1,252 @@ +/** + * Skill fixture extraction — enforces the CLAUDE.md rule "E2E test fixtures: + * extract, don't copy". + * + * Full SKILL.md files are 1000-1900 lines. When `claude -p` (or `codex exec`) + * reads a file that large, context bloat causes timeouts, flaky turn limits, + * and tests that take 5-10x longer than necessary. Every E2E fixture that + * needs skill content should extract ONLY the sections the test actually + * exercises, through one of the three helpers here: + * + * - extractSkillSections(skillDir, sections) + * frontmatter + the named `##
` blocks, concatenated in the + * order given. For tests that exercise specific workflow steps. + * - extractSkillBody(skillDir) + * frontmatter + intro + everything AFTER the shared generated preamble + * ("## Preamble (run first)" .. end of "## Plan Status Footer"). + * For tests that exercise the skill's ENTIRE specific flow but never + * touch the ~780-line shared preamble. + * - extractSkillHead(skillDir, bodyLineCount) + * frontmatter + the first N body lines. For ROUTING / discovery tests, + * where the agent only reads the frontmatter (name + description) to + * decide which skill to invoke. + * + * Failure polarity: every extraction failure (missing file, missing + * frontmatter, renamed section) THROWS with the offending name — a fixture is + * never silently written empty. test/skill-fixture.test.ts pins the exported + * section lists against the real generated SKILL.md files, so a section + * rename fails the FREE suite instead of a paid E2E run. + */ + +import * as fs from 'fs'; +import * as path from 'path'; + +// ─── Section lists shared by E2E fixtures and the free pin test ──────────── +// Keep these verbatim against the H2 headings in the generated SKILL.md files. +// If gen-skill-docs renames a heading, test/skill-fixture.test.ts fails free. + +/** /review E2E (sql-injection, enum-completeness, design-lite): the core + * review workflow without the shared preamble, Review Army, or Fix-First. */ +export const REVIEW_E2E_SECTIONS = [ + 'When to invoke this skill', + 'Step 0: Detect platform and base branch', + 'Step 1: Check branch', + 'Step 2: Read the checklist', + 'Step 2.5: Check for Greptile review comments', + 'Step 3: Get the diff', + 'Step 4: Critical pass (core review)', + 'Confidence Calibration', + 'Important Rules', +]; + +/** Review Army E2E: core workflow + Scope Drift / Plan Completion Audit + * (delivery-audit test) + Step 4.5 specialist dispatch (quality score, + * JSON findings schema, MULTI-SPECIALIST consensus, Red Team). */ +export const REVIEW_ARMY_E2E_SECTIONS = [ + 'When to invoke this skill', + 'Step 0: Detect platform and base branch', + 'Step 1: Check branch', + 'Step 1.5: Scope Drift Detection', + 'Step 2: Read the checklist', + 'Step 2.5: Check for Greptile review comments', + 'Step 3: Get the diff', + 'Step 4: Critical pass (core review)', + 'Confidence Calibration', + 'Step 4.5: Review Army — Specialist Dispatch', + 'Important Rules', +]; + +/** /retro E2E (retro, retro-base-branch): the repo-scoped retro flow + * (Steps 0-14 live under Instructions/Prior Learnings/Capture Learnings) + * + the narrative report template. Global mode and Compare mode are not + * exercised by the E2E tests and are dropped. */ +export const RETRO_E2E_SECTIONS = [ + 'When to invoke this skill', + 'Step 0: Detect platform and base branch', + 'User-invocable', + 'Arguments', + 'Instructions', + 'Prior Learnings', + 'Capture Learnings', + 'Engineering Retro: [date range]', + 'Tone', + 'Important Rules', +]; + +/** codex-review-findings E2E against the Codex host variant + * (.agents/skills/gstack-review/SKILL.md). Same core workflow as + * REVIEW_E2E_SECTIONS, minus "When to invoke this skill" (the Codex host + * adapter does not emit that section). */ +export const CODEX_REVIEW_E2E_SECTIONS = [ + 'Step 0: Detect platform and base branch', + 'Step 1: Check branch', + 'Step 2: Read the checklist', + 'Step 3: Get the diff', + 'Step 4: Critical pass (core review)', + 'Confidence Calibration', + 'Important Rules', +]; + +// ─── Parsing internals ────────────────────────────────────────────────────── + +/** First/last H2 headings of the shared preamble block that gen-skill-docs + * emits into every tier >= 2 skill. extractSkillBody drops this range. */ +const SHARED_PREAMBLE_FIRST = 'Preamble (run first)'; +const SHARED_PREAMBLE_LAST = 'Plan Status Footer'; + +interface H2Section { + heading: string; + /** index of the heading line within bodyLines */ + start: number; + /** one past the last line of the section (start of next H2, or EOF) */ + end: number; +} + +/** Accept either a skill directory or a direct path to a .md file. */ +function resolveSkillMd(skillDirOrFile: string): string { + const file = skillDirOrFile.endsWith('.md') + ? skillDirOrFile + : path.join(skillDirOrFile, 'SKILL.md'); + if (!fs.existsSync(file)) { + throw new Error(`skill-fixture: no SKILL.md at ${file}`); + } + return file; +} + +function splitFrontmatter(raw: string, file: string): { frontmatter: string; bodyLines: string[] } { + const lines = raw.split('\n'); + if ((lines[0] ?? '').trim() !== '---') { + throw new Error(`skill-fixture: ${file} does not start with YAML frontmatter ('---')`); + } + let close = -1; + for (let i = 1; i < lines.length; i++) { + if (lines[i].trim() === '---') { close = i; break; } + } + if (close === -1) { + throw new Error(`skill-fixture: ${file} frontmatter never closes ('---' missing)`); + } + return { + frontmatter: lines.slice(0, close + 1).join('\n'), + bodyLines: lines.slice(close + 1), + }; +} + +/** + * Scan body lines for H2 sections, fence-aware: `## `-prefixed lines inside + * ``` / ~~~ code fences are template content (e.g. the PLAN COMPLETION AUDIT + * output format, the /context-save checkpoint template), NOT section + * boundaries. Fences close only on a matching char of >= opening length, + * per CommonMark, so 4-backtick fences embedding 3-backtick blocks work. + */ +function scanH2Sections(bodyLines: string[]): H2Section[] { + const sections: H2Section[] = []; + let fence: { ch: string; len: number } | null = null; + + for (let i = 0; i < bodyLines.length; i++) { + const line = bodyLines[i]; + const m = line.match(/^ {0,3}(`{3,}|~{3,})(.*)$/); + if (m) { + const ch = m[1][0]; + const len = m[1].length; + if (!fence) { + fence = { ch, len }; + } else if (fence.ch === ch && len >= fence.len && m[2].trim() === '') { + fence = null; + } + continue; + } + if (!fence && line.startsWith('## ')) { + sections.push({ heading: line.slice(3).trim(), start: i, end: bodyLines.length }); + } + } + for (let s = 0; s < sections.length - 1; s++) { + sections[s].end = sections[s + 1].start; + } + return sections; +} + +function loadSkill(skillDirOrFile: string): { + file: string; + frontmatter: string; + bodyLines: string[]; + sections: H2Section[]; +} { + const file = resolveSkillMd(skillDirOrFile); + const raw = fs.readFileSync(file, 'utf-8'); + const { frontmatter, bodyLines } = splitFrontmatter(raw, file); + return { file, frontmatter, bodyLines, sections: scanH2Sections(bodyLines) }; +} + +function findSection(sections: H2Section[], name: string, file: string): H2Section { + const hit = sections.find((s) => s.heading === name) + ?? sections.find((s) => s.heading.startsWith(name)); + if (!hit) { + const available = sections.map((s) => ` ## ${s.heading}`).join('\n'); + throw new Error( + `skill-fixture: section "## ${name}" not found in ${file}.\n` + + 'The section may have been renamed — update the fixture section list ' + + '(see test/helpers/skill-fixture.ts).\n' + + `Available H2 sections:\n${available}`, + ); + } + return hit; +} + +// ─── Public API ───────────────────────────────────────────────────────────── + +/** + * Read the real SKILL.md under `skillDir` (or a direct .md path), slice each + * requested `##
` block, and return frontmatter + the sections + * concatenated in the order given. Throws loudly on a missing section. + */ +export function extractSkillSections(skillDir: string, sections: string[]): string { + const { file, frontmatter, bodyLines, sections: all } = loadSkill(skillDir); + const parts: string[] = [frontmatter, '']; + for (const name of sections) { + const hit = findSection(all, name, file); + parts.push(bodyLines.slice(hit.start, hit.end).join('\n').trimEnd(), ''); + } + return parts.join('\n'); +} + +/** + * Frontmatter + intro (everything before "## Preamble (run first)") + the + * full skill-specific body (everything after the "## Plan Status Footer" + * section). Use when a test exercises the whole skill flow: this drops the + * ~780-line shared generated preamble and nothing else. + */ +export function extractSkillBody(skillDir: string): string { + const { file, frontmatter, bodyLines, sections: all } = loadSkill(skillDir); + const first = findSection(all, SHARED_PREAMBLE_FIRST, file); + const last = findSection(all, SHARED_PREAMBLE_LAST, file); + const intro = bodyLines.slice(0, first.start).join('\n').trimEnd(); + const tail = bodyLines.slice(last.end).join('\n').trimEnd(); + if (!tail) { + throw new Error( + `skill-fixture: ${file} has no content after "## ${SHARED_PREAMBLE_LAST}" — ` + + 'refusing to write a preamble-only fixture.', + ); + } + return [frontmatter, '', intro, '', tail, ''].join('\n'); +} + +/** + * Frontmatter + the first `bodyLineCount` body lines. For routing/discovery + * fixtures: skill selection reads the frontmatter name + description, so the + * body is intentionally truncated. + */ +export function extractSkillHead(skillDir: string, bodyLineCount = 30): string { + const { frontmatter, bodyLines } = loadSkill(skillDir); + const head = bodyLines.slice(0, bodyLineCount).join('\n').trimEnd(); + return `${frontmatter}\n${head}\n\n\n`; +} diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 6b211df4a..cdee0ada7 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -90,7 +90,7 @@ export const E2E_TOUCHFILES: Record = { 'plan-ceo-review-plan-mode': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-plan-mode.test.ts'], 'plan-eng-review-plan-mode': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-eng-plan-mode.test.ts'], 'plan-design-review-plan-mode': ['plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts'], - 'plan-devex-review-plan-mode': ['plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts'], + 'plan-devex-review-plan-mode': ['plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-devex-plan-mode.test.ts'], // Covers ceo (preamble misfire) + eng/design (scope-gate bypass must not // fire outside plan mode) + the named-target exception case. 4 PTY runs; // in CI these run CONCURRENT with the rest of the pty-plan-smoke suite @@ -107,7 +107,7 @@ export const E2E_TOUCHFILES: Record = { // INSIDE the existing 4 plan-X-review-plan-mode test files (covered // transitively by the entries above). Two new standalone files exist for // skills with no prior plan-mode test: - 'office-hours-auto-mode': ['office-hours/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts'], + 'office-hours-auto-mode': ['office-hours/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-office-hours-auto-mode.test.ts'], 'office-hours-phase4-fork': ['office-hours/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/question-tuning.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-office-hours-phase4.test.ts'], 'llm-judge-recommendation': ['test/helpers/llm-judge.ts', 'test/llm-judge-recommendation.test.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'codex/SKILL.md.tmpl', 'scripts/resolvers/review.ts'], // v1.21+ AUTO_DECIDE preserve eval (periodic). Verifies the Tool resolution @@ -126,18 +126,18 @@ export const E2E_TOUCHFILES: Record = { // Each one tests behavior the SDK harness can't observe (rendered TTY, // numbered-option lists, multi-phase ordering, idempotency state echo). 'auq-format-gate': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/helpers/llm-judge.ts'], - 'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts'], - 'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts'], + 'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-mode-routing.test.ts'], + 'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-with-ui.test.ts'], 'budget-regression-pty': ['test/helpers/eval-store.ts', 'test/skill-budget-regression.test.ts'], - 'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts'], - 'ship-section-loading': ['ship/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'], + 'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-ship-idempotency.test.ts'], + 'ship-section-loading': ['ship/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-ship-section-loading.test.ts'], 'plan-ceo-section-loading': ['plan-ceo-review/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'], // Data-driven behavioral guard for the 'plan'/'prompt' carves (eng, design, // devex, office-hours + future PR2 carves). One file iterating CARVE_GUARDS; // the selector sets GSTACK_CARVE_SKILL= to scope cost to the changed // skill (D-CODEX A). Touching the registry/helper or sections.ts runs all. 'carve-section-loading': ['plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'], - 'autoplan-chain-pty': ['autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts'], + 'autoplan-chain-pty': ['autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-autoplan-chain.test.ts'], 'e2e-harness-audit': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/claude-pty-runner.ts'], // Per-finding AskUserQuestion count + review-report-at-bottom assertion. @@ -166,19 +166,19 @@ export const E2E_TOUCHFILES: Record = { // (it exits on first AUQ); runPlanSkillCounting can. 'plan-eng-multi-finding-batching': ['plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts'], 'plan-ceo-split-overflow': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts'], - 'brain-privacy-gate': ['scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'bin/gstack-brain-sync', 'bin/gstack-artifacts-init', 'bin/gstack-config', 'test/helpers/agent-sdk-runner.ts'], + 'brain-privacy-gate': ['scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'bin/gstack-brain-sync', 'bin/gstack-artifacts-init', 'bin/gstack-config', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-brain-privacy-gate.test.ts'], // /setup-gbrain Path 4 (Remote MCP) — happy + bad-token end-to-end via // Agent SDK. Gate-tier (deterministic stub server, fixed inputs); fires // when the skill template, the verify helper, the artifacts-init helper, // or the detect script changes. - 'setup-gbrain-remote': ['setup-gbrain/SKILL.md.tmpl', 'bin/gstack-gbrain-mcp-verify', 'bin/gstack-artifacts-init', 'bin/gstack-gbrain-detect', 'test/helpers/agent-sdk-runner.ts'], - 'setup-gbrain-bad-token': ['setup-gbrain/SKILL.md.tmpl', 'bin/gstack-gbrain-mcp-verify', 'test/helpers/agent-sdk-runner.ts'], + 'setup-gbrain-remote': ['setup-gbrain/SKILL.md.tmpl', 'bin/gstack-gbrain-mcp-verify', 'bin/gstack-artifacts-init', 'bin/gstack-gbrain-detect', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-setup-gbrain-remote.test.ts'], + 'setup-gbrain-bad-token': ['setup-gbrain/SKILL.md.tmpl', 'bin/gstack-gbrain-mcp-verify', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-setup-gbrain-bad-token.test.ts'], // v1.34.0.0 split-engine Path 4 + Step 4.5 Yes (local PGLite for code). // Periodic-tier per codex #12 (AgentSDK harness is non-deterministic). // Fires when the setup-gbrain template, install/verify/init helpers, or // the agent-sdk-runner harness changes. - 'setup-gbrain-path4-local-pglite': ['setup-gbrain/SKILL.md.tmpl', 'bin/gstack-gbrain-mcp-verify', 'bin/gstack-gbrain-install', 'bin/gstack-gbrain-detect', 'lib/gbrain-local-status.ts', 'test/helpers/agent-sdk-runner.ts'], + 'setup-gbrain-path4-local-pglite': ['setup-gbrain/SKILL.md.tmpl', 'bin/gstack-gbrain-mcp-verify', 'bin/gstack-gbrain-install', 'bin/gstack-gbrain-detect', 'lib/gbrain-local-status.ts', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-setup-gbrain-path4-local-pglite.test.ts'], // AskUserQuestion format regression (RECOMMENDATION + Completeness: N/10) // Fires when either template OR the two preamble resolvers change. @@ -326,7 +326,7 @@ export const E2E_TOUCHFILES: Record = { 'autoplan-dual-voice': ['autoplan/**', 'codex/**', 'bin/gstack-codex-probe', 'scripts/resolvers/review.ts', 'scripts/resolvers/design.ts'], // Multi-provider benchmark adapters — live API smoke against real claude/codex/gemini CLIs - 'benchmark-providers-live': ['bin/gstack-model-benchmark', 'test/helpers/providers/**', 'test/helpers/benchmark-runner.ts', 'test/helpers/pricing.ts'], + 'benchmark-providers-live': ['bin/gstack-model-benchmark', 'test/helpers/providers/**', 'test/helpers/benchmark-runner.ts', 'test/helpers/pricing.ts', 'test/skill-e2e-benchmark-providers.test.ts'], // Browser-skills Phase 2a — /scrape + /skillify (v1.19.0.0). Gate-tier // E2E covers the D1 (provenance guard), D3 (atomic write) contracts plus @@ -382,12 +382,14 @@ export const E2E_TOUCHFILES: Record = { 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts', 'scripts/resolvers/model-overlay.ts', + 'test/skill-e2e-overlay-harness.test.ts', ], 'overlay-harness-opus-4-7-fanout-realistic': [ 'model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts', 'scripts/resolvers/model-overlay.ts', + 'test/skill-e2e-overlay-harness.test.ts', ], // /ios-qa — agent flow E2E. Daemon + stub StateServer + codegen @@ -812,6 +814,7 @@ export const GLOBAL_TOUCHFILES = [ 'test/helpers/hermetic-env.ts', // Changes every E2E child's environment 'test/helpers/eval-store.ts', // All E2E tests store results here 'test/helpers/test-selection.ts', // Selection logic itself — a bug here mis-selects every test + 'test/helpers/skill-fixture.ts', // SKILL.md fixture extraction — reshapes the skill content most E2E suites read // NOTE: this file (touchfiles-data.ts) is deliberately NOT a global // touchfile. Changes to it route through map-diff selection in // test-selection.ts: the old git version is evaluated and the maps are diff --git a/test/skill-fixture.test.ts b/test/skill-fixture.test.ts new file mode 100644 index 000000000..84bf0f861 --- /dev/null +++ b/test/skill-fixture.test.ts @@ -0,0 +1,244 @@ +/** + * Unit + pin tests for test/helpers/skill-fixture.ts (free tier, no EVALS). + * + * Two layers: + * 1. Semantics against a synthetic SKILL.md: frontmatter always included, + * sections concatenated in caller order, missing section throws with the + * section name, fenced `## ` template headings do not split sections, + * body extraction drops exactly the shared preamble block, head + * extraction truncates the body. + * 2. Pins against the REAL generated SKILL.md files: every exported section + * list extracts cleanly from the skill it targets, and the body/head + * helpers work for every skill the E2E fixtures feed through them. A + * heading rename in gen-skill-docs fails HERE (free, <1s) instead of + * mid-flight in a paid E2E run. + */ + +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { + extractSkillSections, + extractSkillBody, + extractSkillHead, + REVIEW_E2E_SECTIONS, + REVIEW_ARMY_E2E_SECTIONS, + RETRO_E2E_SECTIONS, + CODEX_REVIEW_E2E_SECTIONS, +} from './helpers/skill-fixture'; + +const ROOT = path.resolve(import.meta.dir, '..'); + +// ─── Synthetic fixture ────────────────────────────────────────────────────── + +const SYNTHETIC_SKILL = `--- +name: fixture-test +description: synthetic skill for skill-fixture unit tests +--- +Intro line before any section. + +## When to invoke this skill +Invoke text. + +## Preamble (run first) +preamble junk that fixtures must drop + +## AskUserQuestion Format +more shared-preamble junk + +## Plan Status Footer +footer junk, last shared-preamble section + +## Step 1 — Do the thing +step one body +\`\`\`markdown +## Embedded Template Heading +template content inside a fence +\`\`\` +step one continues after the fence + +## Step 2 — Other +step two body +`; + +let tmpDir: string; +let skillDir: string; + +beforeAll(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-fixture-test-')); + skillDir = path.join(tmpDir, 'fixture-test'); + fs.mkdirSync(skillDir, { recursive: true }); + fs.writeFileSync(path.join(skillDir, 'SKILL.md'), SYNTHETIC_SKILL); +}); + +afterAll(() => { + try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {} +}); + +describe('extractSkillSections (synthetic)', () => { + test('always includes frontmatter and concatenates sections in caller order', () => { + const out = extractSkillSections(skillDir, ['Step 2 — Other', 'Step 1 — Do the thing']); + expect(out.startsWith('---\nname: fixture-test')).toBe(true); + expect(out).toContain('## Step 1 — Do the thing'); + expect(out).toContain('## Step 2 — Other'); + // Caller order preserved: Step 2 requested first, so it appears first. + expect(out.indexOf('## Step 2 — Other')).toBeLessThan(out.indexOf('## Step 1 — Do the thing')); + // Unrequested sections are dropped. + expect(out).not.toContain('preamble junk'); + expect(out).not.toContain('Intro line before any section'); + }); + + test('fenced ## headings do not terminate a section', () => { + const out = extractSkillSections(skillDir, ['Step 1 — Do the thing']); + expect(out).toContain('template content inside a fence'); + expect(out).toContain('step one continues after the fence'); + expect(out).not.toContain('step two body'); + }); + + test('missing section throws with the section name and the file path', () => { + expect(() => extractSkillSections(skillDir, ['Step 99 — Renamed'])).toThrow(/Step 99 — Renamed/); + expect(() => extractSkillSections(skillDir, ['Step 99 — Renamed'])).toThrow(/SKILL\.md/); + }); + + test('a fenced heading is not findable as a section', () => { + expect(() => extractSkillSections(skillDir, ['Embedded Template Heading'])).toThrow(/not found/); + }); +}); + +describe('extractSkillBody (synthetic)', () => { + test('keeps frontmatter + intro + full body, drops the shared preamble block', () => { + const out = extractSkillBody(skillDir); + expect(out.startsWith('---\nname: fixture-test')).toBe(true); + expect(out).toContain('Intro line before any section'); + expect(out).toContain('## When to invoke this skill'); + expect(out).toContain('## Step 1 — Do the thing'); + expect(out).toContain('template content inside a fence'); + expect(out).toContain('## Step 2 — Other'); + expect(out).not.toContain('preamble junk'); + expect(out).not.toContain('shared-preamble junk'); + expect(out).not.toContain('footer junk'); + }); + + test('throws when the preamble markers are missing', () => { + const bare = path.join(tmpDir, 'bare'); + fs.mkdirSync(bare, { recursive: true }); + fs.writeFileSync(path.join(bare, 'SKILL.md'), '---\nname: bare\n---\n## Only Section\nbody\n'); + expect(() => extractSkillBody(bare)).toThrow(/Preamble \(run first\)/); + }); +}); + +describe('extractSkillHead (synthetic)', () => { + test('keeps frontmatter + first N body lines only', () => { + const out = extractSkillHead(skillDir, 3); + expect(out.startsWith('---\nname: fixture-test')).toBe(true); + expect(out).toContain('Intro line before any section'); + expect(out).toContain('## When to invoke this skill'); + expect(out).not.toContain('## Step 1 — Do the thing'); + expect(out).toContain('body truncated by test/helpers/skill-fixture.ts'); + }); +}); + +describe('error polarity', () => { + test('missing SKILL.md throws (never writes an empty fixture)', () => { + expect(() => extractSkillSections(path.join(tmpDir, 'nope'), ['x'])).toThrow(/no SKILL\.md/); + }); + + test('file without frontmatter throws', () => { + const nofm = path.join(tmpDir, 'nofm'); + fs.mkdirSync(nofm, { recursive: true }); + fs.writeFileSync(path.join(nofm, 'SKILL.md'), '# no frontmatter\n## Section\n'); + expect(() => extractSkillHead(nofm)).toThrow(/frontmatter/); + }); +}); + +// ─── Pins against the real generated SKILL.md files ───────────────────────── +// These turn "someone renamed a section in gen-skill-docs" into a FREE test +// failure instead of a paid E2E setup throw. + +describe('real-skill pins: section lists used by E2E fixtures', () => { + test('REVIEW_E2E_SECTIONS extracts from review/SKILL.md', () => { + const out = extractSkillSections(path.join(ROOT, 'review'), REVIEW_E2E_SECTIONS); + expect(out).toContain('## Step 4: Critical pass (core review)'); + expect(out).toContain('## Important Rules'); + // Drops the shared preamble and the untested workflow tail. + expect(out).not.toContain('## Telemetry (run last)'); + expect(out).not.toContain('## Step 5: Fix-First Review'); + // Meaningfully smaller than the source. + const full = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8'); + expect(out.length).toBeLessThan(full.length * 0.5); + }); + + test('REVIEW_ARMY_E2E_SECTIONS extracts from review/SKILL.md', () => { + const out = extractSkillSections(path.join(ROOT, 'review'), REVIEW_ARMY_E2E_SECTIONS); + // The army tests reference the Plan Completion Audit (inside Step 1.5) + // and the Step 4.5 merge machinery (quality score, JSON schema, consensus). + expect(out).toContain('PLAN COMPLETION AUDIT'); + expect(out).toContain('## Step 4.5: Review Army — Specialist Dispatch'); + expect(out).toContain('quality_score'); + expect(out).toContain('MULTI-SPECIALIST CONFIRMED'); + expect(out).not.toContain('## Telemetry (run last)'); + }); + + test('RETRO_E2E_SECTIONS extracts from retro/SKILL.md', () => { + const out = extractSkillSections(path.join(ROOT, 'retro'), RETRO_E2E_SECTIONS); + // Steps 0.5-14 live under Prior Learnings / Capture Learnings. + expect(out).toContain('### Step 1: Gather Raw Data'); + expect(out).toContain('### Step 14: Write the Narrative'); + expect(out).toContain('## Engineering Retro: [date range]'); + expect(out).not.toContain('## Global Retrospective Mode'); + expect(out).not.toContain('## Telemetry (run last)'); + }); + + test('CODEX_REVIEW_E2E_SECTIONS extracts from the Codex host variant when present', () => { + const codexReview = path.join(ROOT, '.agents', 'skills', 'gstack-review'); + if (!fs.existsSync(path.join(codexReview, 'SKILL.md'))) return; // gitignored artifact, absent in fresh checkouts + const out = extractSkillSections(codexReview, CODEX_REVIEW_E2E_SECTIONS); + expect(out).toContain('## Step 4: Critical pass (core review)'); + expect(out).not.toContain('## Telemetry (run last)'); + }); +}); + +describe('real-skill pins: body/head extraction used by E2E fixtures', () => { + const BODY_EXTRACTED_SKILLS = ['scrape', 'skillify', 'context-save', 'context-restore']; + + for (const skill of BODY_EXTRACTED_SKILLS) { + test(`extractSkillBody(${skill}) drops the shared preamble, keeps the flow`, () => { + const out = extractSkillBody(path.join(ROOT, skill)); + expect(out).not.toContain('## Preamble (run first)'); + expect(out).not.toContain('## Telemetry (run last)'); + const full = fs.readFileSync(path.join(ROOT, skill, 'SKILL.md'), 'utf-8'); + expect(out.length).toBeLessThan(full.length * 0.75); + expect(out.length).toBeGreaterThan(500); + }); + } + + test('body extraction keeps the sections the skillify/context E2E tests assert on', () => { + expect(extractSkillBody(path.join(ROOT, 'skillify'))).toContain('## Step 1 — Provenance guard (D1)'); + expect(extractSkillBody(path.join(ROOT, 'scrape'))).toContain('## Step 4 — Prototype phase'); + expect(extractSkillBody(path.join(ROOT, 'context-save'))).toContain('## List flow'); + expect(extractSkillBody(path.join(ROOT, 'context-restore'))).toContain('## If no saved contexts exist'); + }); + + // The union of skills installed by the routing + opus-47 discovery fixtures. + const HEAD_EXTRACTED_SKILLS = [ + '', 'qa', 'qa-only', 'ship', 'review', 'plan-ceo-review', 'plan-eng-review', + 'plan-design-review', 'design-review', 'design-consultation', 'retro', + 'document-release', 'investigate', 'office-hours', 'browse', + 'setup-browser-cookies', 'gstack-upgrade', 'humanizer', + ]; + + test('extractSkillHead works for every discovery-fixture skill', () => { + for (const skill of HEAD_EXTRACTED_SKILLS) { + const src = path.join(ROOT, skill, 'SKILL.md'); + if (!fs.existsSync(src)) continue; // mirrors the fixtures' existsSync guard + const out = extractSkillHead(src); + expect(out.startsWith('---\n')).toBe(true); + expect(out).toContain('description:'); + // Frontmatter length varies (allowed-tools + triggers); the invariant + // is "frontmatter + 30 body lines + marker", never the full body. + const fullLines = fs.readFileSync(src, 'utf-8').split('\n').length; + expect(out.split('\n').length).toBeLessThan(Math.min(150, fullLines)); + } + }); +});