From 07b225a40c1c0b58cb852e1769c1a2fe6b4ed82f Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Tue, 25 Aug 2026 17:47:32 +0000 Subject: [PATCH] =?UTF-8?q?test:=20register=20wave-3=20carves=20(qa,=20bro?= =?UTF-8?q?wse,=20retro)=20=E2=80=94=20guards,=20touchfiles,=20pins,=20bas?= =?UTF-8?q?elines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CARVE_GUARDS gains the three entries; qa's monolith invariant retires. auq-format carve-safety now keys on the skeleton+sections union shipping the AUQ block (first tier-1 carve: browse never renders it by design). Baselines: parity v1.69.1.0 at 18 sectioned skills; ratchet recaptured. Co-Authored-By: Claude Fable 5 --- test/auq-format-always-loaded.test.ts | 16 ++- test/fixtures/context-budget.json | 6 +- test/fixtures/parity-baseline-v1.69.1.0.json | 30 ++--- test/gen-skill-docs.test.ts | 42 +++++-- test/helpers/carve-guards.ts | 71 +++++++++++ test/helpers/parity-harness.ts | 15 +-- test/helpers/touchfiles-data.ts | 24 ++-- test/skill-llm-eval.test.ts | 117 +++++++++++++------ test/skill-validation.test.ts | 68 ++++++++--- 9 files changed, 283 insertions(+), 106 deletions(-) diff --git a/test/auq-format-always-loaded.test.ts b/test/auq-format-always-loaded.test.ts index e6094cd8a..a044f01ba 100644 --- a/test/auq-format-always-loaded.test.ts +++ b/test/auq-format-always-loaded.test.ts @@ -151,8 +151,20 @@ describe('AUQ format is always-loaded (token-reduction safety net)', () => { // CARVE-SAFETY: for carved skills, the format block must be in the SKELETON, // not only a section. (The per-skill loop above already reads SKILL.md, so - // this is an explicit, named guard for the exact failure mode.) - for (const { skill, skillMd, sectionsDir } of skills.filter(s => s.sectionsDir)) { + // this is an explicit, named guard for the exact failure mode.) Keyed on the + // skeleton+sections UNION shipping the block at all: tier-1 carves (browse) + // never render AUQ format by design, so they have nothing to guard — but a + // tier≥2 carve that wrongly moved the block into a section still trips here. + const shipsAuqInUnion = (s: { skillMd: string; sectionsDir: string | null }): boolean => { + let union = fs.readFileSync(s.skillMd, 'utf-8'); + if (s.sectionsDir) { + for (const f of fs.readdirSync(s.sectionsDir).filter(f => f.endsWith('.md') && !f.endsWith('.md.tmpl'))) { + union += '\n' + fs.readFileSync(path.join(s.sectionsDir, f), 'utf-8'); + } + } + return /##\s*AskUserQuestion Format/i.test(union); + }; + for (const { skill, skillMd, sectionsDir } of skills.filter(s => s.sectionsDir && shipsAuqInUnion(s))) { test(`${skill} (carved): AUQ format block lives in the skeleton, not only sections/`, () => { const body = fs.readFileSync(skillMd, 'utf-8'); expect(body).toMatch(/##\s*AskUserQuestion Format/i); diff --git a/test/fixtures/context-budget.json b/test/fixtures/context-budget.json index 6272cd0e3..d352d05bd 100644 --- a/test/fixtures/context-budget.json +++ b/test/fixtures/context-budget.json @@ -5,7 +5,7 @@ "autoplan": 15337, "benchmark": 4856, "benchmark-models": 3829, - "browse": 10212, + "browse": 7022, "browser-skills/hackernews-frontpage": 371, "canary": 9921, "careful": 919, @@ -48,9 +48,9 @@ "plan-devex-review": 16141, "plan-eng-review": 13081, "plan-tune": 14088, - "qa": 17873, + "qa": 12582, "qa-only": 12355, - "retro": 19313, + "retro": 17894, "review": 14374, "scrape": 3939, "setup-browser-cookies": 3119, diff --git a/test/fixtures/parity-baseline-v1.69.1.0.json b/test/fixtures/parity-baseline-v1.69.1.0.json index 99c575689..9dc46a420 100644 --- a/test/fixtures/parity-baseline-v1.69.1.0.json +++ b/test/fixtures/parity-baseline-v1.69.1.0.json @@ -1,10 +1,10 @@ { "tag": "v1.69.1.0", - "capturedAt": "2026-08-25T16:58:29.504Z", - "capturedFromCommit": "2877b63a", + "capturedAt": "2026-08-25T17:36:29.077Z", + "capturedFromCommit": "706091cb", "capturedFromBranch": "prompt-token-load-reduction", "totalSkills": 53, - "totalCorpusBytes": 2807143, + "totalCorpusBytes": 2810742, "estTotalCatalogTokens": 4195, "topHeaviest": [ { @@ -141,10 +141,10 @@ }, "browse": { "skill": "browse", - "skillMdBytes": 39081, - "skillMdLines": 678, - "estTokens": 9770, - "tmplBytes": 15864, + "skillMdBytes": 41115, + "skillMdLines": 537, + "estTokens": 10279, + "tmplBytes": 16895, "descriptionLen": 181, "hasGateEval": true, "hasPeriodicEval": false @@ -511,10 +511,10 @@ }, "qa": { "skill": "qa", - "skillMdBytes": 68403, - "skillMdLines": 1326, - "estTokens": 17101, - "tmplBytes": 12701, + "skillMdBytes": 70385, + "skillMdLines": 883, + "estTokens": 17596, + "tmplBytes": 12792, "descriptionLen": 218, "hasGateEval": true, "hasPeriodicEval": false @@ -531,10 +531,10 @@ }, "retro": { "skill": "retro", - "skillMdBytes": 73913, - "skillMdLines": 1426, - "estTokens": 18478, - "tmplBytes": 42589, + "skillMdBytes": 73496, + "skillMdLines": 1214, + "estTokens": 18374, + "tmplBytes": 36523, "descriptionLen": 838, "hasGateEval": true, "hasPeriodicEval": false diff --git a/test/gen-skill-docs.test.ts b/test/gen-skill-docs.test.ts index 945f8b0e3..47a94b466 100644 --- a/test/gen-skill-docs.test.ts +++ b/test/gen-skill-docs.test.ts @@ -126,8 +126,11 @@ const CLAUDE_SKIPPED = new Set(__getHostConfig('claude').generation.skipSkills ? const CLAUDE_GENERATED_SKILLS = ALL_SKILLS.filter(s => !CLAUDE_SKIPPED.has(s.dir)); describe('gen-skill-docs', () => { + // Browse carve (token-reduction Phase 4): the command reference + snapshot + // flags render into browse/sections/command-list.md now — read the + // skeleton+sections union so these pins hold across the carve. test('generated SKILL.md contains all command categories', () => { - const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); + const content = readSkillUnion('browse'); const categories = new Set(Object.values(COMMAND_DESCRIPTIONS).map(d => d.category)); for (const cat of categories) { expect(content).toContain(`### ${cat}`); @@ -135,7 +138,7 @@ describe('gen-skill-docs', () => { }); test('generated SKILL.md contains all commands', () => { - const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); + const content = readSkillUnion('browse'); for (const [cmd, meta] of Object.entries(COMMAND_DESCRIPTIONS)) { const display = meta.usage || cmd; expect(content).toContain(display); @@ -143,7 +146,7 @@ describe('gen-skill-docs', () => { }); test('command table is sorted alphabetically within categories', () => { - const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); + const content = readSkillUnion('browse'); // Extract command names from the Navigation section as a test const navSection = content.match(/### Navigation\n\|.*\n\|.*\n([\s\S]*?)(?=\n###|\n## )/); expect(navSection).not.toBeNull(); @@ -168,7 +171,7 @@ describe('gen-skill-docs', () => { }); test('snapshot flags section contains all flags', () => { - const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); + const content = readSkillUnion('browse'); for (const flag of SNAPSHOT_FLAGS) { expect(content).toContain(flag.short); expect(content).toContain(flag.description); @@ -311,10 +314,19 @@ describe('gen-skill-docs', () => { expect(rootTmpl).not.toContain('{{COMMAND_REFERENCE}}'); expect(rootTmpl).not.toContain('{{SNAPSHOT_FLAGS}}'); + // Browse carve: the reference resolvers moved into the on-demand section + // template (so gen-skill-docs keeps them fresh from browse/src); the + // skeleton points at the section instead of inlining the reference. const browseTmpl = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md.tmpl'), 'utf-8'); - expect(browseTmpl).toContain('{{COMMAND_REFERENCE}}'); - expect(browseTmpl).toContain('{{SNAPSHOT_FLAGS}}'); + expect(browseTmpl).not.toContain('{{COMMAND_REFERENCE}}'); + expect(browseTmpl).not.toContain('{{SNAPSHOT_FLAGS}}'); + expect(browseTmpl).toContain('{{SECTION:command-list}}'); expect(browseTmpl).toContain('{{PREAMBLE}}'); + + const browseSectionTmpl = fs.readFileSync( + path.join(ROOT, 'browse', 'sections', 'command-list.md.tmpl'), 'utf-8'); + expect(browseSectionTmpl).toContain('{{COMMAND_REFERENCE}}'); + expect(browseSectionTmpl).toContain('{{SNAPSHOT_FLAGS}}'); }); test('generated SKILL.md contains operational self-improvement (replaced contributor mode)', () => { @@ -526,15 +538,20 @@ describe('gen-skill-docs', () => { }); test('qa and qa-only templates use QA_METHODOLOGY placeholder', () => { - const qaTmpl = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md.tmpl'), 'utf-8'); - expect(qaTmpl).toContain('{{QA_METHODOLOGY}}'); + // qa carve: the macro moved into the section template (the skeleton + // carries the STOP-Read pointer); qa-only remains an inline monolith. + const qaSkeletonTmpl = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md.tmpl'), 'utf-8'); + expect(qaSkeletonTmpl).toContain('{{SECTION:qa-patterns}}'); + expect(qaSkeletonTmpl).not.toContain('{{QA_METHODOLOGY}}'); + const qaSectionTmpl = fs.readFileSync(path.join(ROOT, 'qa', 'sections', 'qa-patterns.md.tmpl'), 'utf-8'); + expect(qaSectionTmpl).toContain('{{QA_METHODOLOGY}}'); const qaOnlyTmpl = fs.readFileSync(path.join(ROOT, 'qa-only', 'SKILL.md.tmpl'), 'utf-8'); expect(qaOnlyTmpl).toContain('{{QA_METHODOLOGY}}'); }); test('QA_METHODOLOGY appears expanded in both qa and qa-only generated files', () => { - const qaContent = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8'); + const qaContent = readSkillUnion('qa'); // carved: methodology lives in qa/sections/qa-patterns.md const qaOnlyContent = fs.readFileSync(path.join(ROOT, 'qa-only', 'SKILL.md'), 'utf-8'); // Both should contain the health score rubric @@ -651,8 +668,9 @@ describe('GitLab support in generated skills', () => { */ describe('description quality evals', () => { // Regression: snapshot flags lost value hints (-d , -s , -o ) + // Browse carve: the flag reference renders into browse/sections/command-list.md. test('snapshot flags with values include value hints in output', () => { - const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); + const content = readSkillUnion('browse'); for (const flag of SNAPSHOT_FLAGS) { if (flag.takesValue) { expect(flag.valueHint).toBeDefined(); @@ -1199,7 +1217,9 @@ describe('Plan file discovery shared helper', () => { // --- Retro plan completion --- describe('Retro plan completion section', () => { - const retroSkill = fs.readFileSync(path.join(ROOT, 'retro', 'SKILL.md'), 'utf-8'); + // Carved: the narrative report format (incl. Plan Completion) lives in + // retro/sections/report-format.md — read the skeleton+sections union. + const retroSkill = readSkillUnion('retro'); test('retro SKILL.md contains plan completion section', () => { expect(retroSkill).toContain('### Plan Completion'); diff --git a/test/helpers/carve-guards.ts b/test/helpers/carve-guards.ts index 1b4495396..ed6ae63f9 100644 --- a/test/helpers/carve-guards.ts +++ b/test/helpers/carve-guards.ts @@ -550,6 +550,77 @@ export const CARVE_GUARDS: Record = { mustContain: ['PGLite', 'Supabase', 'claude mcp add', 'read_secret_to_env', 'pooler'], maxSizeRatio: 1.07, // measured 1.051 vs the branch monolith: index + stubs + 4 STOP pointers }, + // ── Token-reduction Phase 4 wave 3 (v1.69.x branch) ────────────────────── + qa: { + skill: 'qa', + expectedSections: ['test-bootstrap.md', 'qa-patterns.md'], + requiredReads: ['qa-patterns.md'], + scenario: + 'Walk /qa in SIMULATION — do not launch a browser, run any $B command, or execute bash; treat the working tree as clean, the tier as Quick, and the target app as http://localhost:3000 with a small feature-branch diff touching one page. Skip the test-framework bootstrap (assume CLAUDE.md documents the test command). Read each pointed section before doing its step, then produce the QA plan as the report: the mode you selected and why, the Phase 1-6 steps you would run, and a worked health-score computation from the rubric. Do NOT use AskUserQuestion.', + staticInvariants: { + mustStayInSkeleton: [ + '## Setup', + '## SETUP (run this check BEFORE any browse command)', + '## Phases 1-6: QA Baseline', + '## Phase 7: Triage', + '## Phase 8: Fix Loop', + '8e.5. Regression Test', + 'WTF-LIKELIHOOD', + '## Additional Rules (qa-specific)', + '## Output Structure', + ], + mustPrecedeStop: ['## Setup'], + mustMoveToSection: [ + '## Test Framework Bootstrap', + 'BOOTSTRAP_DECLINED', + '## Health Score Rubric', + '### Diff-aware (automatic when on a feature branch with no URL)', + 'Never refuse to use the browser', + ], + gateAfterStop: undefined, + }, + behavioral: 'prompt', + maxSkeletonBytes: 48_750, // Phase 4 wave 3; measured 48,151 + minUnionBytes: 69_500, // measured union 70,385 + mustContain: ['bug', 'browse', 'fix', 'Health Score Rubric', 'regression'], + }, + browse: { + skill: 'browse', + expectedSections: ['command-list.md'], + requiredReads: ['command-list.md'], + scenario: + 'QA a static page: before driving it, plan the full audit — enumerate which browse commands and snapshot flags you would use, including extraction/tab/dialog commands beyond the Most-Used table, reading the full command reference first. Do not launch the browser or run any $B command; produce the command plan as the report.', + staticInvariants: { + mustStayInSkeleton: ['## SETUP', '## Core QA Patterns', '## CSS Inspector', '## Most-Used Commands'], + mustPrecedeStop: ['## SETUP'], + mustMoveToSection: ['## Full Command List', '## Snapshot Flags', '### Navigation'], + gateAfterStop: undefined, + }, + behavioral: 'prompt', + maxSkeletonBytes: 27_500, // Phase 4 wave 3; measured 26,875 + minUnionBytes: 39_500, // measured union 41,115 + // 'BEGIN/END UNTRUSTED EXTERNAL' pins the untrusted-content warning; the full + // envelope phrase wraps across lines in the rendered blockquote, so the + // contiguous-substring check needs the single-line prefix form. + mustContain: ['BEGIN/END UNTRUSTED EXTERNAL', 'snapshot -i', '@e refs', 'deviceScaleFactor', 'handoff'], + }, + retro: { + skill: 'retro', + expectedSections: ['report-format.md'], + requiredReads: ['report-format.md'], + scenario: + 'Run the repo-scoped weekly retrospective for the last 7 days on this repo. There is no origin remote — proceed with the local branch per the guard disclosure rules. The gstack-retro-metrics script is not installed, so follow the degraded path (compute the metrics manually with git). Skip any AskUserQuestion calls — this is non-interactive. Produce the full narrative retrospective report.', + staticInvariants: { + mustStayInSkeleton: ['gstack-retro-metrics', '### Step 2: Compute Metrics', '### Step 13: Save Retro History'], + mustPrecedeStop: ['### Step 2: Compute Metrics'], + mustMoveToSection: ['## Engineering Retro: [date range]', '### Team Breakdown', 'Plan Completion This Period'], + gateAfterStop: undefined, + }, + behavioral: 'prompt', + maxSkeletonBytes: 69_500, // Phase 4 wave 3; measured 68,483 (script absorption -5.4KB) + minUnionBytes: 66_000, // measured union 73,496 + mustContain: ['retrospective', '45-minute gap', 'Ship of the week', 'Praise'], + }, }; /** Sorted carved-skill names. Consumers derive their lists from this — no parallel lists. */ diff --git a/test/helpers/parity-harness.ts b/test/helpers/parity-harness.ts index 90cf4b839..4dc4deb6e 100644 --- a/test/helpers/parity-harness.ts +++ b/test/helpers/parity-harness.ts @@ -206,21 +206,8 @@ export function runParityChecks(opts: { */ const MONOLITH_INVARIANTS: ParityInvariant[] = [ // cso is now carved — its invariant is generated from CARVE_GUARDS below. - // review, codex, land-and-deploy (wave 1) and autoplan (wave 2) carved in token-reduction Phase 4 + // review, codex, land-and-deploy (w1), autoplan (w2), qa (w3) carved in token-reduction Phase 4 // wave 1 (v1.69.x branch) — their invariants generate from CARVE_GUARDS too. - { - skill: 'qa', - mustContain: ['bug', 'browse', 'fix'], - mustHaveHeadings: ['## Preamble', '## When to invoke'], - // v1.2.0 activation lift: the unified first-run-guidance section (P4 scaffold + - // P3 loop tip) is added to every skill's shared preamble — intentional, ~1KB. - // Fork port wave 2: the shared coverage-audit detection block gained the - // multi-ecosystem markers (Django/JVM, script/target/test-file census — - // e3259078 port); measured 1.111x. v1.64+v1.65 merge sums both waves' - // preamble growth; measured 1.125. - maxSizeRatio: 1.13, - minBytes: 50_000, - }, { skill: 'investigate', mustContain: ['root cause', 'hypothes'], diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 0682aeed9..83c5835bb 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -139,7 +139,7 @@ export const E2E_TOUCHFILES: Record = { // devex, office-hours + future PR2 carves). One file iterating CARVE_GUARDS; // the selector sets GSTACK_CARVE_SKILL= to scope cost to the changed // skill (D-CODEX A). Touching the registry/helper or sections.ts runs all. - 'carve-section-loading': ['review/**', 'codex/**', 'land-and-deploy/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'], + 'carve-section-loading': ['qa/**', 'browse/**', 'retro/**', 'autoplan/**', 'spec/**', 'setup-gbrain/**', 'review/**', 'codex/**', 'land-and-deploy/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'], 'autoplan-chain-pty': ['autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-autoplan-chain.test.ts'], 'e2e-harness-audit': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/claude-pty-runner.ts'], @@ -229,8 +229,8 @@ export const E2E_TOUCHFILES: Record = { 'review-dashboard-via': ['ship/**', 'scripts/resolvers/review.ts', 'codex/**', 'autoplan/**', 'land-and-deploy/**', 'test/skill-e2e-review-attribution.test.ts'], // Retro - 'retro': ['retro/**', 'test/skill-e2e-retro.test.ts'], - 'retro-base-branch': ['retro/**', 'test/skill-e2e-retro.test.ts'], + 'retro': ['bin/gstack-retro-metrics', 'retro/**', 'test/skill-e2e-retro.test.ts'], + 'retro-base-branch': ['bin/gstack-retro-metrics', 'retro/**', 'test/skill-e2e-retro.test.ts'], // Global discover 'global-discover': ['bin/gstack-global-discover.ts', 'test/global-discover.test.ts'], @@ -752,16 +752,16 @@ export const E2E_TIERS: Record = { * LLM-judge test touchfiles — keyed by test description string. */ export const LLM_JUDGE_TOUCHFILES: Record = { - 'command reference table': ['SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts'], - 'snapshot flags reference': ['SKILL.md', 'SKILL.md.tmpl', 'browse/src/snapshot.ts'], - 'browse/SKILL.md reference': ['browse/SKILL.md', 'browse/SKILL.md.tmpl', 'browse/src/**'], + 'command reference table': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts'], + 'snapshot flags reference': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/snapshot.ts'], + 'browse/SKILL.md reference': ['browse/sections/**', 'browse/SKILL.md', 'browse/SKILL.md.tmpl', 'browse/src/**'], 'setup block': ['SKILL.md', 'SKILL.md.tmpl'], - 'regression vs baseline': ['SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts', 'test/fixtures/eval-baselines.json'], - 'qa/SKILL.md workflow': ['qa/SKILL.md', 'qa/SKILL.md.tmpl'], - 'qa/SKILL.md health rubric': ['qa/SKILL.md', 'qa/SKILL.md.tmpl'], - 'qa/SKILL.md anti-refusal': ['qa/SKILL.md', 'qa/SKILL.md.tmpl', 'qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl'], + 'regression vs baseline': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts', 'test/fixtures/eval-baselines.json'], + 'qa/SKILL.md workflow': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl'], + 'qa/SKILL.md health rubric': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl'], + 'qa/SKILL.md anti-refusal': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl', 'qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl'], 'cross-skill greptile consistency': ['review/SKILL.md', 'review/SKILL.md.tmpl', 'ship/SKILL.md', 'ship/SKILL.md.tmpl', 'review/greptile-triage.md', 'retro/SKILL.md', 'retro/SKILL.md.tmpl'], - 'baseline score pinning': ['SKILL.md', 'SKILL.md.tmpl', 'test/fixtures/eval-baselines.json'], + 'baseline score pinning': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'test/fixtures/eval-baselines.json'], // Ship & Release 'ship/SKILL.md workflow': ['ship/SKILL.md', 'ship/SKILL.md.tmpl'], @@ -789,7 +789,7 @@ export const LLM_JUDGE_TOUCHFILES: Record = { 'setup-deploy/SKILL.md platform setup': ['setup-deploy/SKILL.md', 'setup-deploy/SKILL.md.tmpl'], // Other skills - 'retro/SKILL.md instructions': ['retro/SKILL.md', 'retro/SKILL.md.tmpl'], + 'retro/SKILL.md instructions': ['retro/sections/**', 'retro/SKILL.md', 'retro/SKILL.md.tmpl'], 'qa-only/SKILL.md workflow': ['qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl'], 'gstack-upgrade/SKILL.md upgrade flow': ['gstack-upgrade/SKILL.md', 'gstack-upgrade/SKILL.md.tmpl'], diff --git a/test/skill-llm-eval.test.ts b/test/skill-llm-eval.test.ts index 7bdb1d057..74e58de7c 100644 --- a/test/skill-llm-eval.test.ts +++ b/test/skill-llm-eval.test.ts @@ -34,6 +34,38 @@ import { // Eval result collector const evalCollector = createEvalCollector('llm-judge'); +/** + * Browse carve (token-reduction Phase 4): the '## Snapshot Flags' and + * '## Full Command List' reference blocks moved from browse/SKILL.md into the + * generated on-demand section browse/sections/command-list.md ('## Snapshot + * Flags' first, then '## Full Command List'). '## SETUP', '## Core QA + * Patterns', and '## CSS Inspector' stay in the skeleton. Non-empty guard: + * judging an empty slice would silently pass garbage to the judge. + */ +function readBrowseCommandSection(): string { + const p = path.join(ROOT, 'browse', 'sections', 'command-list.md'); + const content = fs.readFileSync(p, 'utf-8'); + if (!content.includes('## Snapshot Flags') || !content.includes('## Full Command List')) { + throw new Error( + `${p} is missing the expected headers — regenerate with: bun run gen:skill-docs`, + ); + } + return content; +} + +/** Slice a section out of the command-list section file, guarded non-empty. */ +function sliceBrowseSection(startHeader: string, endHeader?: string): string { + const content = readBrowseCommandSection(); + const start = content.indexOf(startHeader); + if (start < 0) throw new Error(`browse/sections/command-list.md: "${startHeader}" not found`); + const end = endHeader ? content.indexOf(endHeader) : -1; + const section = end > start ? content.slice(start, end) : content.slice(start); + if (section.trim().length < 200) { + throw new Error(`browse/sections/command-list.md slice at "${startHeader}" is empty/stub — regenerate with: bun run gen:skill-docs`); + } + return section; +} + // --- Diff-based test selection (LLM_JUDGE_TOUCHFILES, not the E2E table) --- const selectedTests = computeDiffSelection(LLM_JUDGE_TOUCHFILES, 'LLM-judge'); @@ -53,10 +85,9 @@ describeIfSelected('LLM-as-judge quality evals', [ ], () => { testIfSelected('command reference table', async () => { const t0 = Date.now(); - // P2 (v1.2.0): the command reference moved from the root router to browse/SKILL.md. - const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); - const start = content.indexOf('## Full Command List'); - const section = content.slice(start); + // Browse carve: the command reference lives in the generated on-demand + // section browse/sections/command-list.md now (read via non-empty guard). + const section = sliceBrowseSection('## Full Command List'); const scores = await judge('command reference table', section); console.log('Command reference scores:', JSON.stringify(scores, null, 2)); @@ -82,11 +113,10 @@ describeIfSelected('LLM-as-judge quality evals', [ testIfSelected('snapshot flags reference', async () => { const t0 = Date.now(); - // P2 (v1.2.0): snapshot flags moved from the root router to browse/SKILL.md. - const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); - const start = content.indexOf('## Snapshot Flags'); - const end = content.indexOf('## CSS Inspector'); - const section = content.slice(start, end); + // Browse carve: snapshot flags live in browse/sections/command-list.md now, + // ordered before '## Full Command List' (the '## CSS Inspector' end boundary + // stayed in the skeleton). + const section = sliceBrowseSection('## Snapshot Flags', '## Full Command List'); const scores = await judge('snapshot flags reference', section); console.log('Snapshot flags scores:', JSON.stringify(scores, null, 2)); @@ -109,9 +139,8 @@ describeIfSelected('LLM-as-judge quality evals', [ testIfSelected('browse/SKILL.md reference', async () => { const t0 = Date.now(); - const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); - const start = content.indexOf('## Snapshot Flags'); - const section = content.slice(start); + // Browse carve: flags + commands are the whole generated section file. + const section = sliceBrowseSection('## Snapshot Flags'); const scores = await judge('browse skill reference (flags + commands)', section); console.log('Browse SKILL.md scores:', JSON.stringify(scores, null, 2)); @@ -162,10 +191,8 @@ describeIfSelected('LLM-as-judge quality evals', [ testIfSelected('regression vs baseline', async () => { const t0 = Date.now(); - // P2 (v1.2.0): the command reference moved from the root router to browse/SKILL.md. - const generated = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); - const genStart = generated.indexOf('## Full Command List'); - const genSection = generated.slice(genStart); + // Browse carve: the command reference lives in browse/sections/command-list.md. + const genSection = sliceBrowseSection('## Full Command List'); const baseline = `## Command Reference @@ -249,14 +276,38 @@ Scores are 1-5 overall quality.`, // --- Part 7: QA skill quality evals (C6) --- -describeIfSelected('QA skill quality evals', ['qa/SKILL.md workflow', 'qa/SKILL.md health rubric', 'qa/SKILL.md anti-refusal'], () => { - const qaContent = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8'); +/** + * QA carve (token-reduction Phase 4): the '## Modes', '## Workflow', + * '## Health Score Rubric', '## Framework-Specific Guidance', and + * '## Important Rules' blocks moved from qa/SKILL.md into the generated + * on-demand section qa/sections/qa-patterns.md. Monolith-tolerant: falls back + * to the skeleton when the section file doesn't exist (pre-carve checkout). + */ +function readQaPatterns(): string { + const sectionPath = path.join(ROOT, 'qa', 'sections', 'qa-patterns.md'); + return fs.existsSync(sectionPath) + ? fs.readFileSync(sectionPath, 'utf-8') + : fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8'); +} +/** Slice out of the qa-patterns section, guarded non-empty: judging an empty + * slice would silently pass garbage to the judge. */ +function sliceQaPatterns(startHeader: string, endHeader?: string): string { + const content = readQaPatterns(); + const start = content.indexOf(startHeader); + if (start < 0) throw new Error(`qa/sections/qa-patterns.md: "${startHeader}" not found — regenerate with: bun run gen:skill-docs`); + const end = endHeader ? content.indexOf(endHeader, start) : -1; + const section = end > start ? content.slice(start, end) : content.slice(start); + if (section.trim().length < 200) { + throw new Error(`qa/sections/qa-patterns.md slice at "${startHeader}" is empty/stub — regenerate with: bun run gen:skill-docs`); + } + return section; +} + +describeIfSelected('QA skill quality evals', ['qa/SKILL.md workflow', 'qa/SKILL.md health rubric', 'qa/SKILL.md anti-refusal'], () => { testIfSelected('qa/SKILL.md workflow', async () => { const t0 = Date.now(); - const start = qaContent.indexOf('## Workflow'); - const end = qaContent.indexOf('## Health Score Rubric'); - const section = qaContent.slice(start, end); + const section = sliceQaPatterns('## Workflow', '## Health Score Rubric'); const scores = await callJudge(`You are evaluating the quality of a QA testing workflow document for an AI coding agent. @@ -297,8 +348,7 @@ ${section}`); testIfSelected('qa/SKILL.md health rubric', async () => { const t0 = Date.now(); - const start = qaContent.indexOf('## Health Score Rubric'); - const section = qaContent.slice(start); + const section = sliceQaPatterns('## Health Score Rubric'); const scores = await callJudge(`You are evaluating a health score rubric that an AI agent must follow to compute a numeric QA score. @@ -338,13 +388,14 @@ ${section}`); testIfSelected('qa/SKILL.md anti-refusal', async () => { const t0 = Date.now(); - // Extract both the diff-aware mode section and Important Rules section - const diffAwareStart = qaContent.indexOf('### Diff-aware'); - const diffAwareEnd = qaContent.indexOf('### Full'); - const rulesStart = qaContent.indexOf('## Important Rules'); - const rulesEnd = qaContent.indexOf('## Framework-Specific'); - const diffAwareSection = qaContent.slice(diffAwareStart, diffAwareEnd); - const rulesSection = qaContent.slice(rulesStart, rulesEnd); + // Extract both the diff-aware mode section and Important Rules section. + // (Pre-carve this sliced '## Important Rules' → '## Framework-Specific', + // which was EMPTY — Framework-Specific precedes Important Rules — so the + // judge only ever saw excerpt 1. The section-file slice fixes that: rules + // run to the end of qa-patterns.md, so rule 12 "Never refuse to use the + // browser" now actually reaches the judge.) + const diffAwareSection = sliceQaPatterns('### Diff-aware', '### Full'); + const rulesSection = sliceQaPatterns('## Important Rules'); const result = await callJudge<{ would_browse: boolean; fallback_behavior: string; confidence: number; reasoning: string }>(`You are evaluating whether a QA testing skill document would cause an AI agent to USE THE BROWSER or REFUSE to use the browser in a specific scenario. @@ -470,10 +521,8 @@ describeIfSelected('Baseline score pinning', ['baseline score pinning'], () => { const baselines = JSON.parse(fs.readFileSync(baselinesPath, 'utf-8')); const regressions: string[] = []; - // P2 (v1.2.0): the command reference moved from the root router to browse/SKILL.md. - const skillContent = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); - const cmdStart = skillContent.indexOf('## Full Command List'); - const cmdSection = skillContent.slice(cmdStart); + // Browse carve: the command reference lives in browse/sections/command-list.md. + const cmdSection = sliceBrowseSection('## Full Command List'); const cmdScores = await judge('command reference table', cmdSection); for (const dim of ['clarity', 'completeness', 'actionability'] as const) { diff --git a/test/skill-validation.test.ts b/test/skill-validation.test.ts index 7f23ea2c4..da0887cd4 100644 --- a/test/skill-validation.test.ts +++ b/test/skill-validation.test.ts @@ -65,6 +65,25 @@ describe('SKILL.md command validation', () => { expect(result.snapshotFlagErrors).toHaveLength(0); }); + // qa carve: the Phases 1-6 methodology (with its $B command examples) moved + // into qa/sections/*.md — validate the section files too so browse-command + // coverage doesn't silently shrink with the carve. + test('all $B commands and snapshot flags in qa/sections/*.md are valid', () => { + const secDir = path.join(ROOT, 'qa', 'sections'); + if (!fs.existsSync(secDir)) return; // pre-carve checkout + const sectionMds = fs.readdirSync(secDir).filter(f => f.endsWith('.md') && !f.endsWith('.md.tmpl')); + expect(sectionMds.length).toBeGreaterThan(0); + let validTotal = 0; + for (const f of sectionMds) { + const result = validateSkill(path.join(secDir, f)); + expect({ file: f, invalid: result.invalid }).toEqual({ file: f, invalid: [] }); + expect({ file: f, snapshotFlagErrors: result.snapshotFlagErrors }).toEqual({ file: f, snapshotFlagErrors: [] }); + validTotal += result.valid.length; + } + // Non-empty guard: the carved methodology must still carry $B examples. + expect(validTotal).toBeGreaterThan(0); + }); + test('all $B commands in qa-only/SKILL.md are valid browse commands', () => { const qaOnlySkill = path.join(ROOT, 'qa-only', 'SKILL.md'); if (!fs.existsSync(qaOnlySkill)) return; @@ -416,7 +435,9 @@ describe('Cross-skill path consistency', () => { // --- Part 7: QA skill structure validation (A2) --- describe('QA skill structure validation', () => { - const qaContent = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8'); + // qa carve: modes, Phases 1-6, and the health rubric moved into + // qa/sections/qa-patterns.md — validate the skeleton+sections union. + const qaContent = readSkillUnion('qa'); test('qa/SKILL.md has all 11 phases', () => { const phases = [ @@ -1090,8 +1111,10 @@ describe('gstack-slug', () => { // --- Test Bootstrap validation --- describe('Test Bootstrap ({{TEST_BOOTSTRAP}}) integration', () => { + // qa carve: the rendered TEST_BOOTSTRAP body lives in + // qa/sections/test-bootstrap.md — read the skeleton+sections union. test('TEST_BOOTSTRAP resolver produces valid content', () => { - const qaContent = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8'); + const qaContent = readSkillUnion('qa'); expect(qaContent).toContain('Test Framework Bootstrap'); expect(qaContent).toContain('RUNTIME:ruby'); expect(qaContent).toContain('RUNTIME:node'); @@ -1101,7 +1124,7 @@ describe('Test Bootstrap ({{TEST_BOOTSTRAP}}) integration', () => { }); test('TEST_BOOTSTRAP appears in qa/SKILL.md', () => { - const content = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8'); + const content = readSkillUnion('qa'); expect(content).toContain('Test Framework Bootstrap'); expect(content).toContain('TESTING.md'); expect(content).toContain('CLAUDE.md'); @@ -1127,7 +1150,7 @@ describe('Test Bootstrap ({{TEST_BOOTSTRAP}}) integration', () => { }); test('bootstrap includes framework knowledge table', () => { - const content = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8'); + const content = readSkillUnion('qa'); expect(content).toContain('vitest'); expect(content).toContain('minitest'); expect(content).toContain('pytest'); @@ -1137,20 +1160,20 @@ describe('Test Bootstrap ({{TEST_BOOTSTRAP}}) integration', () => { }); test('bootstrap includes CI/CD pipeline generation', () => { - const content = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8'); + const content = readSkillUnion('qa'); expect(content).toContain('.github/workflows/test.yml'); expect(content).toContain('GitHub Actions'); }); test('bootstrap includes first real tests step', () => { - const content = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8'); + const content = readSkillUnion('qa'); expect(content).toContain('First real tests'); expect(content).toContain('git log --since=30.days'); expect(content).toContain('Prioritize by risk'); }); test('bootstrap includes vibe coding philosophy', () => { - const content = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8'); + const content = readSkillUnion('qa'); expect(content).toContain('vibe coding'); expect(content).toContain('100% test coverage'); }); @@ -1325,28 +1348,43 @@ describe('ship step numbering', () => { // --- Retro test health validation --- describe('Retro test health tracking', () => { - test('retro/SKILL.md has test health data gathering commands', () => { - const content = fs.readFileSync(path.join(ROOT, 'retro', 'SKILL.md'), 'utf-8'); - expect(content).toContain('# 10. Test file count'); - expect(content).toContain('# 11. Regression test commits'); - expect(content).toContain('# 12. Test files changed'); + // The inline git pipelines moved into bin/gstack-retro-metrics (retro + // token-reduction wave); the skill interprets its labeled METRIC lines. + test('gstack-retro-metrics gathers the test health data', () => { + const script = fs.readFileSync(path.join(ROOT, 'bin', 'gstack-retro-metrics'), 'utf-8'); + expect(script).toContain('TEST_FILES_TOTAL'); + expect(script).toContain('REGRESSION_TEST_COMMITS'); + expect(script).toContain('TEST_FILES_CHANGED'); + // The historical grep targets survive the script absorption. + expect(script).toContain('test(qa):'); + expect(script).toContain('test(design):'); + expect(script).toContain('git ls-files'); + }); + + test('retro skill interprets the test-health metric lines', () => { + // Template source, not the generated render: the pin must hold across the + // regen boundary (the generated file follows the template at gen time). + const content = fs.readFileSync(path.join(ROOT, 'retro', 'SKILL.md.tmpl'), 'utf-8'); + expect(content).toContain('TEST_FILES_TOTAL'); + expect(content).toContain('REGRESSION_TEST_COMMITS'); }); test('retro/SKILL.md has Test Health metrics row', () => { - const content = fs.readFileSync(path.join(ROOT, 'retro', 'SKILL.md'), 'utf-8'); + const content = readSkillUnion('retro'); expect(content).toContain('Test Health'); expect(content).toContain('regression tests'); }); test('retro/SKILL.md has Test Health narrative section', () => { - const content = fs.readFileSync(path.join(ROOT, 'retro', 'SKILL.md'), 'utf-8'); + // Carved: the narrative report format lives in sections/report-format.md. + const content = readSkillUnion('retro'); expect(content).toContain('### Test Health'); expect(content).toContain('Total test files'); expect(content).toContain('vibe coding safe'); }); test('retro JSON schema includes test_health field', () => { - const content = fs.readFileSync(path.join(ROOT, 'retro', 'SKILL.md'), 'utf-8'); + const content = readSkillUnion('retro'); expect(content).toContain('test_health'); expect(content).toContain('total_test_files'); expect(content).toContain('regression_test_commits');