mirror of
https://github.com/garrytan/gstack.git
synced 2026-08-20 21:17:19 +02:00
test: skill-fixture helper — extract SKILL.md sections, don't copy files
extractSkillSections (fence-aware H2 scanner, loud-throw on missing sections with available-heading list), extractSkillBody (drops the shared generated preamble), extractSkillHead (frontmatter + first 30 lines, for routing fixtures). Pinned section lists per consumer, and free-tier real-skill pins so a gen-skill-docs heading rename fails the FREE suite instead of a paid run. skill-fixture.ts joins GLOBAL_TOUCHFILES (fail-safe polarity: over-select). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Fable 5
Claude Fable 5
parent
97cb10714e
commit
f945c841dd
@@ -0,0 +1,252 @@
|
||||
/**
|
||||
* Skill fixture extraction — enforces the CLAUDE.md rule "E2E test fixtures:
|
||||
* extract, don't copy".
|
||||
*
|
||||
* Full SKILL.md files are 1000-1900 lines. When `claude -p` (or `codex exec`)
|
||||
* reads a file that large, context bloat causes timeouts, flaky turn limits,
|
||||
* and tests that take 5-10x longer than necessary. Every E2E fixture that
|
||||
* needs skill content should extract ONLY the sections the test actually
|
||||
* exercises, through one of the three helpers here:
|
||||
*
|
||||
* - extractSkillSections(skillDir, sections)
|
||||
* frontmatter + the named `## <section>` blocks, concatenated in the
|
||||
* order given. For tests that exercise specific workflow steps.
|
||||
* - extractSkillBody(skillDir)
|
||||
* frontmatter + intro + everything AFTER the shared generated preamble
|
||||
* ("## Preamble (run first)" .. end of "## Plan Status Footer").
|
||||
* For tests that exercise the skill's ENTIRE specific flow but never
|
||||
* touch the ~780-line shared preamble.
|
||||
* - extractSkillHead(skillDir, bodyLineCount)
|
||||
* frontmatter + the first N body lines. For ROUTING / discovery tests,
|
||||
* where the agent only reads the frontmatter (name + description) to
|
||||
* decide which skill to invoke.
|
||||
*
|
||||
* Failure polarity: every extraction failure (missing file, missing
|
||||
* frontmatter, renamed section) THROWS with the offending name — a fixture is
|
||||
* never silently written empty. test/skill-fixture.test.ts pins the exported
|
||||
* section lists against the real generated SKILL.md files, so a section
|
||||
* rename fails the FREE suite instead of a paid E2E run.
|
||||
*/
|
||||
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
// ─── Section lists shared by E2E fixtures and the free pin test ────────────
|
||||
// Keep these verbatim against the H2 headings in the generated SKILL.md files.
|
||||
// If gen-skill-docs renames a heading, test/skill-fixture.test.ts fails free.
|
||||
|
||||
/** /review E2E (sql-injection, enum-completeness, design-lite): the core
|
||||
* review workflow without the shared preamble, Review Army, or Fix-First. */
|
||||
export const REVIEW_E2E_SECTIONS = [
|
||||
'When to invoke this skill',
|
||||
'Step 0: Detect platform and base branch',
|
||||
'Step 1: Check branch',
|
||||
'Step 2: Read the checklist',
|
||||
'Step 2.5: Check for Greptile review comments',
|
||||
'Step 3: Get the diff',
|
||||
'Step 4: Critical pass (core review)',
|
||||
'Confidence Calibration',
|
||||
'Important Rules',
|
||||
];
|
||||
|
||||
/** Review Army E2E: core workflow + Scope Drift / Plan Completion Audit
|
||||
* (delivery-audit test) + Step 4.5 specialist dispatch (quality score,
|
||||
* JSON findings schema, MULTI-SPECIALIST consensus, Red Team). */
|
||||
export const REVIEW_ARMY_E2E_SECTIONS = [
|
||||
'When to invoke this skill',
|
||||
'Step 0: Detect platform and base branch',
|
||||
'Step 1: Check branch',
|
||||
'Step 1.5: Scope Drift Detection',
|
||||
'Step 2: Read the checklist',
|
||||
'Step 2.5: Check for Greptile review comments',
|
||||
'Step 3: Get the diff',
|
||||
'Step 4: Critical pass (core review)',
|
||||
'Confidence Calibration',
|
||||
'Step 4.5: Review Army — Specialist Dispatch',
|
||||
'Important Rules',
|
||||
];
|
||||
|
||||
/** /retro E2E (retro, retro-base-branch): the repo-scoped retro flow
|
||||
* (Steps 0-14 live under Instructions/Prior Learnings/Capture Learnings)
|
||||
* + the narrative report template. Global mode and Compare mode are not
|
||||
* exercised by the E2E tests and are dropped. */
|
||||
export const RETRO_E2E_SECTIONS = [
|
||||
'When to invoke this skill',
|
||||
'Step 0: Detect platform and base branch',
|
||||
'User-invocable',
|
||||
'Arguments',
|
||||
'Instructions',
|
||||
'Prior Learnings',
|
||||
'Capture Learnings',
|
||||
'Engineering Retro: [date range]',
|
||||
'Tone',
|
||||
'Important Rules',
|
||||
];
|
||||
|
||||
/** codex-review-findings E2E against the Codex host variant
|
||||
* (.agents/skills/gstack-review/SKILL.md). Same core workflow as
|
||||
* REVIEW_E2E_SECTIONS, minus "When to invoke this skill" (the Codex host
|
||||
* adapter does not emit that section). */
|
||||
export const CODEX_REVIEW_E2E_SECTIONS = [
|
||||
'Step 0: Detect platform and base branch',
|
||||
'Step 1: Check branch',
|
||||
'Step 2: Read the checklist',
|
||||
'Step 3: Get the diff',
|
||||
'Step 4: Critical pass (core review)',
|
||||
'Confidence Calibration',
|
||||
'Important Rules',
|
||||
];
|
||||
|
||||
// ─── Parsing internals ──────────────────────────────────────────────────────
|
||||
|
||||
/** First/last H2 headings of the shared preamble block that gen-skill-docs
|
||||
* emits into every tier >= 2 skill. extractSkillBody drops this range. */
|
||||
const SHARED_PREAMBLE_FIRST = 'Preamble (run first)';
|
||||
const SHARED_PREAMBLE_LAST = 'Plan Status Footer';
|
||||
|
||||
interface H2Section {
|
||||
heading: string;
|
||||
/** index of the heading line within bodyLines */
|
||||
start: number;
|
||||
/** one past the last line of the section (start of next H2, or EOF) */
|
||||
end: number;
|
||||
}
|
||||
|
||||
/** Accept either a skill directory or a direct path to a .md file. */
|
||||
function resolveSkillMd(skillDirOrFile: string): string {
|
||||
const file = skillDirOrFile.endsWith('.md')
|
||||
? skillDirOrFile
|
||||
: path.join(skillDirOrFile, 'SKILL.md');
|
||||
if (!fs.existsSync(file)) {
|
||||
throw new Error(`skill-fixture: no SKILL.md at ${file}`);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
function splitFrontmatter(raw: string, file: string): { frontmatter: string; bodyLines: string[] } {
|
||||
const lines = raw.split('\n');
|
||||
if ((lines[0] ?? '').trim() !== '---') {
|
||||
throw new Error(`skill-fixture: ${file} does not start with YAML frontmatter ('---')`);
|
||||
}
|
||||
let close = -1;
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
if (lines[i].trim() === '---') { close = i; break; }
|
||||
}
|
||||
if (close === -1) {
|
||||
throw new Error(`skill-fixture: ${file} frontmatter never closes ('---' missing)`);
|
||||
}
|
||||
return {
|
||||
frontmatter: lines.slice(0, close + 1).join('\n'),
|
||||
bodyLines: lines.slice(close + 1),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan body lines for H2 sections, fence-aware: `## `-prefixed lines inside
|
||||
* ``` / ~~~ code fences are template content (e.g. the PLAN COMPLETION AUDIT
|
||||
* output format, the /context-save checkpoint template), NOT section
|
||||
* boundaries. Fences close only on a matching char of >= opening length,
|
||||
* per CommonMark, so 4-backtick fences embedding 3-backtick blocks work.
|
||||
*/
|
||||
function scanH2Sections(bodyLines: string[]): H2Section[] {
|
||||
const sections: H2Section[] = [];
|
||||
let fence: { ch: string; len: number } | null = null;
|
||||
|
||||
for (let i = 0; i < bodyLines.length; i++) {
|
||||
const line = bodyLines[i];
|
||||
const m = line.match(/^ {0,3}(`{3,}|~{3,})(.*)$/);
|
||||
if (m) {
|
||||
const ch = m[1][0];
|
||||
const len = m[1].length;
|
||||
if (!fence) {
|
||||
fence = { ch, len };
|
||||
} else if (fence.ch === ch && len >= fence.len && m[2].trim() === '') {
|
||||
fence = null;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!fence && line.startsWith('## ')) {
|
||||
sections.push({ heading: line.slice(3).trim(), start: i, end: bodyLines.length });
|
||||
}
|
||||
}
|
||||
for (let s = 0; s < sections.length - 1; s++) {
|
||||
sections[s].end = sections[s + 1].start;
|
||||
}
|
||||
return sections;
|
||||
}
|
||||
|
||||
function loadSkill(skillDirOrFile: string): {
|
||||
file: string;
|
||||
frontmatter: string;
|
||||
bodyLines: string[];
|
||||
sections: H2Section[];
|
||||
} {
|
||||
const file = resolveSkillMd(skillDirOrFile);
|
||||
const raw = fs.readFileSync(file, 'utf-8');
|
||||
const { frontmatter, bodyLines } = splitFrontmatter(raw, file);
|
||||
return { file, frontmatter, bodyLines, sections: scanH2Sections(bodyLines) };
|
||||
}
|
||||
|
||||
function findSection(sections: H2Section[], name: string, file: string): H2Section {
|
||||
const hit = sections.find((s) => s.heading === name)
|
||||
?? sections.find((s) => s.heading.startsWith(name));
|
||||
if (!hit) {
|
||||
const available = sections.map((s) => ` ## ${s.heading}`).join('\n');
|
||||
throw new Error(
|
||||
`skill-fixture: section "## ${name}" not found in ${file}.\n`
|
||||
+ 'The section may have been renamed — update the fixture section list '
|
||||
+ '(see test/helpers/skill-fixture.ts).\n'
|
||||
+ `Available H2 sections:\n${available}`,
|
||||
);
|
||||
}
|
||||
return hit;
|
||||
}
|
||||
|
||||
// ─── Public API ─────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Read the real SKILL.md under `skillDir` (or a direct .md path), slice each
|
||||
* requested `## <section>` block, and return frontmatter + the sections
|
||||
* concatenated in the order given. Throws loudly on a missing section.
|
||||
*/
|
||||
export function extractSkillSections(skillDir: string, sections: string[]): string {
|
||||
const { file, frontmatter, bodyLines, sections: all } = loadSkill(skillDir);
|
||||
const parts: string[] = [frontmatter, ''];
|
||||
for (const name of sections) {
|
||||
const hit = findSection(all, name, file);
|
||||
parts.push(bodyLines.slice(hit.start, hit.end).join('\n').trimEnd(), '');
|
||||
}
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Frontmatter + intro (everything before "## Preamble (run first)") + the
|
||||
* full skill-specific body (everything after the "## Plan Status Footer"
|
||||
* section). Use when a test exercises the whole skill flow: this drops the
|
||||
* ~780-line shared generated preamble and nothing else.
|
||||
*/
|
||||
export function extractSkillBody(skillDir: string): string {
|
||||
const { file, frontmatter, bodyLines, sections: all } = loadSkill(skillDir);
|
||||
const first = findSection(all, SHARED_PREAMBLE_FIRST, file);
|
||||
const last = findSection(all, SHARED_PREAMBLE_LAST, file);
|
||||
const intro = bodyLines.slice(0, first.start).join('\n').trimEnd();
|
||||
const tail = bodyLines.slice(last.end).join('\n').trimEnd();
|
||||
if (!tail) {
|
||||
throw new Error(
|
||||
`skill-fixture: ${file} has no content after "## ${SHARED_PREAMBLE_LAST}" — `
|
||||
+ 'refusing to write a preamble-only fixture.',
|
||||
);
|
||||
}
|
||||
return [frontmatter, '', intro, '', tail, ''].join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Frontmatter + the first `bodyLineCount` body lines. For routing/discovery
|
||||
* fixtures: skill selection reads the frontmatter name + description, so the
|
||||
* body is intentionally truncated.
|
||||
*/
|
||||
export function extractSkillHead(skillDir: string, bodyLineCount = 30): string {
|
||||
const { frontmatter, bodyLines } = loadSkill(skillDir);
|
||||
const head = bodyLines.slice(0, bodyLineCount).join('\n').trimEnd();
|
||||
return `${frontmatter}\n${head}\n\n<!-- body truncated by test/helpers/skill-fixture.ts — routing fixture needs frontmatter only -->\n`;
|
||||
}
|
||||
@@ -90,7 +90,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'plan-ceo-review-plan-mode': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-plan-mode.test.ts'],
|
||||
'plan-eng-review-plan-mode': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-eng-plan-mode.test.ts'],
|
||||
'plan-design-review-plan-mode': ['plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts'],
|
||||
'plan-devex-review-plan-mode': ['plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts'],
|
||||
'plan-devex-review-plan-mode': ['plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-devex-plan-mode.test.ts'],
|
||||
// Covers ceo (preamble misfire) + eng/design (scope-gate bypass must not
|
||||
// fire outside plan mode) + the named-target exception case. 4 PTY runs;
|
||||
// in CI these run CONCURRENT with the rest of the pty-plan-smoke suite
|
||||
@@ -107,7 +107,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// INSIDE the existing 4 plan-X-review-plan-mode test files (covered
|
||||
// transitively by the entries above). Two new standalone files exist for
|
||||
// skills with no prior plan-mode test:
|
||||
'office-hours-auto-mode': ['office-hours/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts'],
|
||||
'office-hours-auto-mode': ['office-hours/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-office-hours-auto-mode.test.ts'],
|
||||
'office-hours-phase4-fork': ['office-hours/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/question-tuning.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-office-hours-phase4.test.ts'],
|
||||
'llm-judge-recommendation': ['test/helpers/llm-judge.ts', 'test/llm-judge-recommendation.test.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'codex/SKILL.md.tmpl', 'scripts/resolvers/review.ts'],
|
||||
// v1.21+ AUTO_DECIDE preserve eval (periodic). Verifies the Tool resolution
|
||||
@@ -126,18 +126,18 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// Each one tests behavior the SDK harness can't observe (rendered TTY,
|
||||
// numbered-option lists, multi-phase ordering, idempotency state echo).
|
||||
'auq-format-gate': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/helpers/llm-judge.ts'],
|
||||
'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts'],
|
||||
'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts'],
|
||||
'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-mode-routing.test.ts'],
|
||||
'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-with-ui.test.ts'],
|
||||
'budget-regression-pty': ['test/helpers/eval-store.ts', 'test/skill-budget-regression.test.ts'],
|
||||
'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts'],
|
||||
'ship-section-loading': ['ship/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'],
|
||||
'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-ship-idempotency.test.ts'],
|
||||
'ship-section-loading': ['ship/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-ship-section-loading.test.ts'],
|
||||
'plan-ceo-section-loading': ['plan-ceo-review/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'],
|
||||
// Data-driven behavioral guard for the 'plan'/'prompt' carves (eng, design,
|
||||
// devex, office-hours + future PR2 carves). One file iterating CARVE_GUARDS;
|
||||
// the selector sets GSTACK_CARVE_SKILL=<name> to scope cost to the changed
|
||||
// skill (D-CODEX A). Touching the registry/helper or sections.ts runs all.
|
||||
'carve-section-loading': ['plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'],
|
||||
'autoplan-chain-pty': ['autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts'],
|
||||
'autoplan-chain-pty': ['autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-autoplan-chain.test.ts'],
|
||||
'e2e-harness-audit': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/claude-pty-runner.ts'],
|
||||
|
||||
// Per-finding AskUserQuestion count + review-report-at-bottom assertion.
|
||||
@@ -166,19 +166,19 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// (it exits on first AUQ); runPlanSkillCounting can.
|
||||
'plan-eng-multi-finding-batching': ['plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts'],
|
||||
'plan-ceo-split-overflow': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts'],
|
||||
'brain-privacy-gate': ['scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'bin/gstack-brain-sync', 'bin/gstack-artifacts-init', 'bin/gstack-config', 'test/helpers/agent-sdk-runner.ts'],
|
||||
'brain-privacy-gate': ['scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'bin/gstack-brain-sync', 'bin/gstack-artifacts-init', 'bin/gstack-config', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-brain-privacy-gate.test.ts'],
|
||||
|
||||
// /setup-gbrain Path 4 (Remote MCP) — happy + bad-token end-to-end via
|
||||
// Agent SDK. Gate-tier (deterministic stub server, fixed inputs); fires
|
||||
// when the skill template, the verify helper, the artifacts-init helper,
|
||||
// or the detect script changes.
|
||||
'setup-gbrain-remote': ['setup-gbrain/SKILL.md.tmpl', 'bin/gstack-gbrain-mcp-verify', 'bin/gstack-artifacts-init', 'bin/gstack-gbrain-detect', 'test/helpers/agent-sdk-runner.ts'],
|
||||
'setup-gbrain-bad-token': ['setup-gbrain/SKILL.md.tmpl', 'bin/gstack-gbrain-mcp-verify', 'test/helpers/agent-sdk-runner.ts'],
|
||||
'setup-gbrain-remote': ['setup-gbrain/SKILL.md.tmpl', 'bin/gstack-gbrain-mcp-verify', 'bin/gstack-artifacts-init', 'bin/gstack-gbrain-detect', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-setup-gbrain-remote.test.ts'],
|
||||
'setup-gbrain-bad-token': ['setup-gbrain/SKILL.md.tmpl', 'bin/gstack-gbrain-mcp-verify', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-setup-gbrain-bad-token.test.ts'],
|
||||
// v1.34.0.0 split-engine Path 4 + Step 4.5 Yes (local PGLite for code).
|
||||
// Periodic-tier per codex #12 (AgentSDK harness is non-deterministic).
|
||||
// Fires when the setup-gbrain template, install/verify/init helpers, or
|
||||
// the agent-sdk-runner harness changes.
|
||||
'setup-gbrain-path4-local-pglite': ['setup-gbrain/SKILL.md.tmpl', 'bin/gstack-gbrain-mcp-verify', 'bin/gstack-gbrain-install', 'bin/gstack-gbrain-detect', 'lib/gbrain-local-status.ts', 'test/helpers/agent-sdk-runner.ts'],
|
||||
'setup-gbrain-path4-local-pglite': ['setup-gbrain/SKILL.md.tmpl', 'bin/gstack-gbrain-mcp-verify', 'bin/gstack-gbrain-install', 'bin/gstack-gbrain-detect', 'lib/gbrain-local-status.ts', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-setup-gbrain-path4-local-pglite.test.ts'],
|
||||
|
||||
// AskUserQuestion format regression (RECOMMENDATION + Completeness: N/10)
|
||||
// Fires when either template OR the two preamble resolvers change.
|
||||
@@ -326,7 +326,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'autoplan-dual-voice': ['autoplan/**', 'codex/**', 'bin/gstack-codex-probe', 'scripts/resolvers/review.ts', 'scripts/resolvers/design.ts'],
|
||||
|
||||
// Multi-provider benchmark adapters — live API smoke against real claude/codex/gemini CLIs
|
||||
'benchmark-providers-live': ['bin/gstack-model-benchmark', 'test/helpers/providers/**', 'test/helpers/benchmark-runner.ts', 'test/helpers/pricing.ts'],
|
||||
'benchmark-providers-live': ['bin/gstack-model-benchmark', 'test/helpers/providers/**', 'test/helpers/benchmark-runner.ts', 'test/helpers/pricing.ts', 'test/skill-e2e-benchmark-providers.test.ts'],
|
||||
|
||||
// Browser-skills Phase 2a — /scrape + /skillify (v1.19.0.0). Gate-tier
|
||||
// E2E covers the D1 (provenance guard), D3 (atomic write) contracts plus
|
||||
@@ -382,12 +382,14 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'test/fixtures/overlay-nudges.ts',
|
||||
'test/helpers/agent-sdk-runner.ts',
|
||||
'scripts/resolvers/model-overlay.ts',
|
||||
'test/skill-e2e-overlay-harness.test.ts',
|
||||
],
|
||||
'overlay-harness-opus-4-7-fanout-realistic': [
|
||||
'model-overlays/**',
|
||||
'test/fixtures/overlay-nudges.ts',
|
||||
'test/helpers/agent-sdk-runner.ts',
|
||||
'scripts/resolvers/model-overlay.ts',
|
||||
'test/skill-e2e-overlay-harness.test.ts',
|
||||
],
|
||||
|
||||
// /ios-qa — agent flow E2E. Daemon + stub StateServer + codegen
|
||||
@@ -812,6 +814,7 @@ export const GLOBAL_TOUCHFILES = [
|
||||
'test/helpers/hermetic-env.ts', // Changes every E2E child's environment
|
||||
'test/helpers/eval-store.ts', // All E2E tests store results here
|
||||
'test/helpers/test-selection.ts', // Selection logic itself — a bug here mis-selects every test
|
||||
'test/helpers/skill-fixture.ts', // SKILL.md fixture extraction — reshapes the skill content most E2E suites read
|
||||
// NOTE: this file (touchfiles-data.ts) is deliberately NOT a global
|
||||
// touchfile. Changes to it route through map-diff selection in
|
||||
// test-selection.ts: the old git version is evaluated and the maps are
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
/**
|
||||
* Unit + pin tests for test/helpers/skill-fixture.ts (free tier, no EVALS).
|
||||
*
|
||||
* Two layers:
|
||||
* 1. Semantics against a synthetic SKILL.md: frontmatter always included,
|
||||
* sections concatenated in caller order, missing section throws with the
|
||||
* section name, fenced `## ` template headings do not split sections,
|
||||
* body extraction drops exactly the shared preamble block, head
|
||||
* extraction truncates the body.
|
||||
* 2. Pins against the REAL generated SKILL.md files: every exported section
|
||||
* list extracts cleanly from the skill it targets, and the body/head
|
||||
* helpers work for every skill the E2E fixtures feed through them. A
|
||||
* heading rename in gen-skill-docs fails HERE (free, <1s) instead of
|
||||
* mid-flight in a paid E2E run.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import {
|
||||
extractSkillSections,
|
||||
extractSkillBody,
|
||||
extractSkillHead,
|
||||
REVIEW_E2E_SECTIONS,
|
||||
REVIEW_ARMY_E2E_SECTIONS,
|
||||
RETRO_E2E_SECTIONS,
|
||||
CODEX_REVIEW_E2E_SECTIONS,
|
||||
} from './helpers/skill-fixture';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
// ─── Synthetic fixture ──────────────────────────────────────────────────────
|
||||
|
||||
const SYNTHETIC_SKILL = `---
|
||||
name: fixture-test
|
||||
description: synthetic skill for skill-fixture unit tests
|
||||
---
|
||||
Intro line before any section.
|
||||
|
||||
## When to invoke this skill
|
||||
Invoke text.
|
||||
|
||||
## Preamble (run first)
|
||||
preamble junk that fixtures must drop
|
||||
|
||||
## AskUserQuestion Format
|
||||
more shared-preamble junk
|
||||
|
||||
## Plan Status Footer
|
||||
footer junk, last shared-preamble section
|
||||
|
||||
## Step 1 — Do the thing
|
||||
step one body
|
||||
\`\`\`markdown
|
||||
## Embedded Template Heading
|
||||
template content inside a fence
|
||||
\`\`\`
|
||||
step one continues after the fence
|
||||
|
||||
## Step 2 — Other
|
||||
step two body
|
||||
`;
|
||||
|
||||
let tmpDir: string;
|
||||
let skillDir: string;
|
||||
|
||||
beforeAll(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-fixture-test-'));
|
||||
skillDir = path.join(tmpDir, 'fixture-test');
|
||||
fs.mkdirSync(skillDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(skillDir, 'SKILL.md'), SYNTHETIC_SKILL);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
describe('extractSkillSections (synthetic)', () => {
|
||||
test('always includes frontmatter and concatenates sections in caller order', () => {
|
||||
const out = extractSkillSections(skillDir, ['Step 2 — Other', 'Step 1 — Do the thing']);
|
||||
expect(out.startsWith('---\nname: fixture-test')).toBe(true);
|
||||
expect(out).toContain('## Step 1 — Do the thing');
|
||||
expect(out).toContain('## Step 2 — Other');
|
||||
// Caller order preserved: Step 2 requested first, so it appears first.
|
||||
expect(out.indexOf('## Step 2 — Other')).toBeLessThan(out.indexOf('## Step 1 — Do the thing'));
|
||||
// Unrequested sections are dropped.
|
||||
expect(out).not.toContain('preamble junk');
|
||||
expect(out).not.toContain('Intro line before any section');
|
||||
});
|
||||
|
||||
test('fenced ## headings do not terminate a section', () => {
|
||||
const out = extractSkillSections(skillDir, ['Step 1 — Do the thing']);
|
||||
expect(out).toContain('template content inside a fence');
|
||||
expect(out).toContain('step one continues after the fence');
|
||||
expect(out).not.toContain('step two body');
|
||||
});
|
||||
|
||||
test('missing section throws with the section name and the file path', () => {
|
||||
expect(() => extractSkillSections(skillDir, ['Step 99 — Renamed'])).toThrow(/Step 99 — Renamed/);
|
||||
expect(() => extractSkillSections(skillDir, ['Step 99 — Renamed'])).toThrow(/SKILL\.md/);
|
||||
});
|
||||
|
||||
test('a fenced heading is not findable as a section', () => {
|
||||
expect(() => extractSkillSections(skillDir, ['Embedded Template Heading'])).toThrow(/not found/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractSkillBody (synthetic)', () => {
|
||||
test('keeps frontmatter + intro + full body, drops the shared preamble block', () => {
|
||||
const out = extractSkillBody(skillDir);
|
||||
expect(out.startsWith('---\nname: fixture-test')).toBe(true);
|
||||
expect(out).toContain('Intro line before any section');
|
||||
expect(out).toContain('## When to invoke this skill');
|
||||
expect(out).toContain('## Step 1 — Do the thing');
|
||||
expect(out).toContain('template content inside a fence');
|
||||
expect(out).toContain('## Step 2 — Other');
|
||||
expect(out).not.toContain('preamble junk');
|
||||
expect(out).not.toContain('shared-preamble junk');
|
||||
expect(out).not.toContain('footer junk');
|
||||
});
|
||||
|
||||
test('throws when the preamble markers are missing', () => {
|
||||
const bare = path.join(tmpDir, 'bare');
|
||||
fs.mkdirSync(bare, { recursive: true });
|
||||
fs.writeFileSync(path.join(bare, 'SKILL.md'), '---\nname: bare\n---\n## Only Section\nbody\n');
|
||||
expect(() => extractSkillBody(bare)).toThrow(/Preamble \(run first\)/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractSkillHead (synthetic)', () => {
|
||||
test('keeps frontmatter + first N body lines only', () => {
|
||||
const out = extractSkillHead(skillDir, 3);
|
||||
expect(out.startsWith('---\nname: fixture-test')).toBe(true);
|
||||
expect(out).toContain('Intro line before any section');
|
||||
expect(out).toContain('## When to invoke this skill');
|
||||
expect(out).not.toContain('## Step 1 — Do the thing');
|
||||
expect(out).toContain('body truncated by test/helpers/skill-fixture.ts');
|
||||
});
|
||||
});
|
||||
|
||||
describe('error polarity', () => {
|
||||
test('missing SKILL.md throws (never writes an empty fixture)', () => {
|
||||
expect(() => extractSkillSections(path.join(tmpDir, 'nope'), ['x'])).toThrow(/no SKILL\.md/);
|
||||
});
|
||||
|
||||
test('file without frontmatter throws', () => {
|
||||
const nofm = path.join(tmpDir, 'nofm');
|
||||
fs.mkdirSync(nofm, { recursive: true });
|
||||
fs.writeFileSync(path.join(nofm, 'SKILL.md'), '# no frontmatter\n## Section\n');
|
||||
expect(() => extractSkillHead(nofm)).toThrow(/frontmatter/);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Pins against the real generated SKILL.md files ─────────────────────────
|
||||
// These turn "someone renamed a section in gen-skill-docs" into a FREE test
|
||||
// failure instead of a paid E2E setup throw.
|
||||
|
||||
describe('real-skill pins: section lists used by E2E fixtures', () => {
|
||||
test('REVIEW_E2E_SECTIONS extracts from review/SKILL.md', () => {
|
||||
const out = extractSkillSections(path.join(ROOT, 'review'), REVIEW_E2E_SECTIONS);
|
||||
expect(out).toContain('## Step 4: Critical pass (core review)');
|
||||
expect(out).toContain('## Important Rules');
|
||||
// Drops the shared preamble and the untested workflow tail.
|
||||
expect(out).not.toContain('## Telemetry (run last)');
|
||||
expect(out).not.toContain('## Step 5: Fix-First Review');
|
||||
// Meaningfully smaller than the source.
|
||||
const full = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8');
|
||||
expect(out.length).toBeLessThan(full.length * 0.5);
|
||||
});
|
||||
|
||||
test('REVIEW_ARMY_E2E_SECTIONS extracts from review/SKILL.md', () => {
|
||||
const out = extractSkillSections(path.join(ROOT, 'review'), REVIEW_ARMY_E2E_SECTIONS);
|
||||
// The army tests reference the Plan Completion Audit (inside Step 1.5)
|
||||
// and the Step 4.5 merge machinery (quality score, JSON schema, consensus).
|
||||
expect(out).toContain('PLAN COMPLETION AUDIT');
|
||||
expect(out).toContain('## Step 4.5: Review Army — Specialist Dispatch');
|
||||
expect(out).toContain('quality_score');
|
||||
expect(out).toContain('MULTI-SPECIALIST CONFIRMED');
|
||||
expect(out).not.toContain('## Telemetry (run last)');
|
||||
});
|
||||
|
||||
test('RETRO_E2E_SECTIONS extracts from retro/SKILL.md', () => {
|
||||
const out = extractSkillSections(path.join(ROOT, 'retro'), RETRO_E2E_SECTIONS);
|
||||
// Steps 0.5-14 live under Prior Learnings / Capture Learnings.
|
||||
expect(out).toContain('### Step 1: Gather Raw Data');
|
||||
expect(out).toContain('### Step 14: Write the Narrative');
|
||||
expect(out).toContain('## Engineering Retro: [date range]');
|
||||
expect(out).not.toContain('## Global Retrospective Mode');
|
||||
expect(out).not.toContain('## Telemetry (run last)');
|
||||
});
|
||||
|
||||
test('CODEX_REVIEW_E2E_SECTIONS extracts from the Codex host variant when present', () => {
|
||||
const codexReview = path.join(ROOT, '.agents', 'skills', 'gstack-review');
|
||||
if (!fs.existsSync(path.join(codexReview, 'SKILL.md'))) return; // gitignored artifact, absent in fresh checkouts
|
||||
const out = extractSkillSections(codexReview, CODEX_REVIEW_E2E_SECTIONS);
|
||||
expect(out).toContain('## Step 4: Critical pass (core review)');
|
||||
expect(out).not.toContain('## Telemetry (run last)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('real-skill pins: body/head extraction used by E2E fixtures', () => {
|
||||
const BODY_EXTRACTED_SKILLS = ['scrape', 'skillify', 'context-save', 'context-restore'];
|
||||
|
||||
for (const skill of BODY_EXTRACTED_SKILLS) {
|
||||
test(`extractSkillBody(${skill}) drops the shared preamble, keeps the flow`, () => {
|
||||
const out = extractSkillBody(path.join(ROOT, skill));
|
||||
expect(out).not.toContain('## Preamble (run first)');
|
||||
expect(out).not.toContain('## Telemetry (run last)');
|
||||
const full = fs.readFileSync(path.join(ROOT, skill, 'SKILL.md'), 'utf-8');
|
||||
expect(out.length).toBeLessThan(full.length * 0.75);
|
||||
expect(out.length).toBeGreaterThan(500);
|
||||
});
|
||||
}
|
||||
|
||||
test('body extraction keeps the sections the skillify/context E2E tests assert on', () => {
|
||||
expect(extractSkillBody(path.join(ROOT, 'skillify'))).toContain('## Step 1 — Provenance guard (D1)');
|
||||
expect(extractSkillBody(path.join(ROOT, 'scrape'))).toContain('## Step 4 — Prototype phase');
|
||||
expect(extractSkillBody(path.join(ROOT, 'context-save'))).toContain('## List flow');
|
||||
expect(extractSkillBody(path.join(ROOT, 'context-restore'))).toContain('## If no saved contexts exist');
|
||||
});
|
||||
|
||||
// The union of skills installed by the routing + opus-47 discovery fixtures.
|
||||
const HEAD_EXTRACTED_SKILLS = [
|
||||
'', 'qa', 'qa-only', 'ship', 'review', 'plan-ceo-review', 'plan-eng-review',
|
||||
'plan-design-review', 'design-review', 'design-consultation', 'retro',
|
||||
'document-release', 'investigate', 'office-hours', 'browse',
|
||||
'setup-browser-cookies', 'gstack-upgrade', 'humanizer',
|
||||
];
|
||||
|
||||
test('extractSkillHead works for every discovery-fixture skill', () => {
|
||||
for (const skill of HEAD_EXTRACTED_SKILLS) {
|
||||
const src = path.join(ROOT, skill, 'SKILL.md');
|
||||
if (!fs.existsSync(src)) continue; // mirrors the fixtures' existsSync guard
|
||||
const out = extractSkillHead(src);
|
||||
expect(out.startsWith('---\n')).toBe(true);
|
||||
expect(out).toContain('description:');
|
||||
// Frontmatter length varies (allowed-tools + triggers); the invariant
|
||||
// is "frontmatter + 30 body lines + marker", never the full body.
|
||||
const fullLines = fs.readFileSync(src, 'utf-8').split('\n').length;
|
||||
expect(out.split('\n').length).toBeLessThan(Math.min(150, fullLines));
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user