mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-09 06:28:59 +02:00
test: register wave-3 carves (qa, browse, retro) — guards, touchfiles, pins, baselines
CARVE_GUARDS gains the three entries; qa's monolith invariant retires. auq-format carve-safety now keys on the skeleton+sections union shipping the AUQ block (first tier-1 carve: browse never renders it by design). Baselines: parity v1.69.1.0 at 18 sectioned skills; ratchet recaptured. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Fable 5
Claude Fable 5
parent
e0250aa128
commit
07b225a40c
@@ -151,8 +151,20 @@ describe('AUQ format is always-loaded (token-reduction safety net)', () => {
|
||||
|
||||
// CARVE-SAFETY: for carved skills, the format block must be in the SKELETON,
|
||||
// not only a section. (The per-skill loop above already reads SKILL.md, so
|
||||
// this is an explicit, named guard for the exact failure mode.)
|
||||
for (const { skill, skillMd, sectionsDir } of skills.filter(s => s.sectionsDir)) {
|
||||
// this is an explicit, named guard for the exact failure mode.) Keyed on the
|
||||
// skeleton+sections UNION shipping the block at all: tier-1 carves (browse)
|
||||
// never render AUQ format by design, so they have nothing to guard — but a
|
||||
// tier≥2 carve that wrongly moved the block into a section still trips here.
|
||||
const shipsAuqInUnion = (s: { skillMd: string; sectionsDir: string | null }): boolean => {
|
||||
let union = fs.readFileSync(s.skillMd, 'utf-8');
|
||||
if (s.sectionsDir) {
|
||||
for (const f of fs.readdirSync(s.sectionsDir).filter(f => f.endsWith('.md') && !f.endsWith('.md.tmpl'))) {
|
||||
union += '\n' + fs.readFileSync(path.join(s.sectionsDir, f), 'utf-8');
|
||||
}
|
||||
}
|
||||
return /##\s*AskUserQuestion Format/i.test(union);
|
||||
};
|
||||
for (const { skill, skillMd, sectionsDir } of skills.filter(s => s.sectionsDir && shipsAuqInUnion(s))) {
|
||||
test(`${skill} (carved): AUQ format block lives in the skeleton, not only sections/`, () => {
|
||||
const body = fs.readFileSync(skillMd, 'utf-8');
|
||||
expect(body).toMatch(/##\s*AskUserQuestion Format/i);
|
||||
|
||||
Vendored
+3
-3
@@ -5,7 +5,7 @@
|
||||
"autoplan": 15337,
|
||||
"benchmark": 4856,
|
||||
"benchmark-models": 3829,
|
||||
"browse": 10212,
|
||||
"browse": 7022,
|
||||
"browser-skills/hackernews-frontpage": 371,
|
||||
"canary": 9921,
|
||||
"careful": 919,
|
||||
@@ -48,9 +48,9 @@
|
||||
"plan-devex-review": 16141,
|
||||
"plan-eng-review": 13081,
|
||||
"plan-tune": 14088,
|
||||
"qa": 17873,
|
||||
"qa": 12582,
|
||||
"qa-only": 12355,
|
||||
"retro": 19313,
|
||||
"retro": 17894,
|
||||
"review": 14374,
|
||||
"scrape": 3939,
|
||||
"setup-browser-cookies": 3119,
|
||||
|
||||
+15
-15
@@ -1,10 +1,10 @@
|
||||
{
|
||||
"tag": "v1.69.1.0",
|
||||
"capturedAt": "2026-08-25T16:58:29.504Z",
|
||||
"capturedFromCommit": "2877b63a",
|
||||
"capturedAt": "2026-08-25T17:36:29.077Z",
|
||||
"capturedFromCommit": "706091cb",
|
||||
"capturedFromBranch": "prompt-token-load-reduction",
|
||||
"totalSkills": 53,
|
||||
"totalCorpusBytes": 2807143,
|
||||
"totalCorpusBytes": 2810742,
|
||||
"estTotalCatalogTokens": 4195,
|
||||
"topHeaviest": [
|
||||
{
|
||||
@@ -141,10 +141,10 @@
|
||||
},
|
||||
"browse": {
|
||||
"skill": "browse",
|
||||
"skillMdBytes": 39081,
|
||||
"skillMdLines": 678,
|
||||
"estTokens": 9770,
|
||||
"tmplBytes": 15864,
|
||||
"skillMdBytes": 41115,
|
||||
"skillMdLines": 537,
|
||||
"estTokens": 10279,
|
||||
"tmplBytes": 16895,
|
||||
"descriptionLen": 181,
|
||||
"hasGateEval": true,
|
||||
"hasPeriodicEval": false
|
||||
@@ -511,10 +511,10 @@
|
||||
},
|
||||
"qa": {
|
||||
"skill": "qa",
|
||||
"skillMdBytes": 68403,
|
||||
"skillMdLines": 1326,
|
||||
"estTokens": 17101,
|
||||
"tmplBytes": 12701,
|
||||
"skillMdBytes": 70385,
|
||||
"skillMdLines": 883,
|
||||
"estTokens": 17596,
|
||||
"tmplBytes": 12792,
|
||||
"descriptionLen": 218,
|
||||
"hasGateEval": true,
|
||||
"hasPeriodicEval": false
|
||||
@@ -531,10 +531,10 @@
|
||||
},
|
||||
"retro": {
|
||||
"skill": "retro",
|
||||
"skillMdBytes": 73913,
|
||||
"skillMdLines": 1426,
|
||||
"estTokens": 18478,
|
||||
"tmplBytes": 42589,
|
||||
"skillMdBytes": 73496,
|
||||
"skillMdLines": 1214,
|
||||
"estTokens": 18374,
|
||||
"tmplBytes": 36523,
|
||||
"descriptionLen": 838,
|
||||
"hasGateEval": true,
|
||||
"hasPeriodicEval": false
|
||||
|
||||
+31
-11
@@ -126,8 +126,11 @@ const CLAUDE_SKIPPED = new Set(__getHostConfig('claude').generation.skipSkills ?
|
||||
const CLAUDE_GENERATED_SKILLS = ALL_SKILLS.filter(s => !CLAUDE_SKIPPED.has(s.dir));
|
||||
|
||||
describe('gen-skill-docs', () => {
|
||||
// Browse carve (token-reduction Phase 4): the command reference + snapshot
|
||||
// flags render into browse/sections/command-list.md now — read the
|
||||
// skeleton+sections union so these pins hold across the carve.
|
||||
test('generated SKILL.md contains all command categories', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8');
|
||||
const content = readSkillUnion('browse');
|
||||
const categories = new Set(Object.values(COMMAND_DESCRIPTIONS).map(d => d.category));
|
||||
for (const cat of categories) {
|
||||
expect(content).toContain(`### ${cat}`);
|
||||
@@ -135,7 +138,7 @@ describe('gen-skill-docs', () => {
|
||||
});
|
||||
|
||||
test('generated SKILL.md contains all commands', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8');
|
||||
const content = readSkillUnion('browse');
|
||||
for (const [cmd, meta] of Object.entries(COMMAND_DESCRIPTIONS)) {
|
||||
const display = meta.usage || cmd;
|
||||
expect(content).toContain(display);
|
||||
@@ -143,7 +146,7 @@ describe('gen-skill-docs', () => {
|
||||
});
|
||||
|
||||
test('command table is sorted alphabetically within categories', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8');
|
||||
const content = readSkillUnion('browse');
|
||||
// Extract command names from the Navigation section as a test
|
||||
const navSection = content.match(/### Navigation\n\|.*\n\|.*\n([\s\S]*?)(?=\n###|\n## )/);
|
||||
expect(navSection).not.toBeNull();
|
||||
@@ -168,7 +171,7 @@ describe('gen-skill-docs', () => {
|
||||
});
|
||||
|
||||
test('snapshot flags section contains all flags', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8');
|
||||
const content = readSkillUnion('browse');
|
||||
for (const flag of SNAPSHOT_FLAGS) {
|
||||
expect(content).toContain(flag.short);
|
||||
expect(content).toContain(flag.description);
|
||||
@@ -311,10 +314,19 @@ describe('gen-skill-docs', () => {
|
||||
expect(rootTmpl).not.toContain('{{COMMAND_REFERENCE}}');
|
||||
expect(rootTmpl).not.toContain('{{SNAPSHOT_FLAGS}}');
|
||||
|
||||
// Browse carve: the reference resolvers moved into the on-demand section
|
||||
// template (so gen-skill-docs keeps them fresh from browse/src); the
|
||||
// skeleton points at the section instead of inlining the reference.
|
||||
const browseTmpl = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md.tmpl'), 'utf-8');
|
||||
expect(browseTmpl).toContain('{{COMMAND_REFERENCE}}');
|
||||
expect(browseTmpl).toContain('{{SNAPSHOT_FLAGS}}');
|
||||
expect(browseTmpl).not.toContain('{{COMMAND_REFERENCE}}');
|
||||
expect(browseTmpl).not.toContain('{{SNAPSHOT_FLAGS}}');
|
||||
expect(browseTmpl).toContain('{{SECTION:command-list}}');
|
||||
expect(browseTmpl).toContain('{{PREAMBLE}}');
|
||||
|
||||
const browseSectionTmpl = fs.readFileSync(
|
||||
path.join(ROOT, 'browse', 'sections', 'command-list.md.tmpl'), 'utf-8');
|
||||
expect(browseSectionTmpl).toContain('{{COMMAND_REFERENCE}}');
|
||||
expect(browseSectionTmpl).toContain('{{SNAPSHOT_FLAGS}}');
|
||||
});
|
||||
|
||||
test('generated SKILL.md contains operational self-improvement (replaced contributor mode)', () => {
|
||||
@@ -526,15 +538,20 @@ describe('gen-skill-docs', () => {
|
||||
});
|
||||
|
||||
test('qa and qa-only templates use QA_METHODOLOGY placeholder', () => {
|
||||
const qaTmpl = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md.tmpl'), 'utf-8');
|
||||
expect(qaTmpl).toContain('{{QA_METHODOLOGY}}');
|
||||
// qa carve: the macro moved into the section template (the skeleton
|
||||
// carries the STOP-Read pointer); qa-only remains an inline monolith.
|
||||
const qaSkeletonTmpl = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md.tmpl'), 'utf-8');
|
||||
expect(qaSkeletonTmpl).toContain('{{SECTION:qa-patterns}}');
|
||||
expect(qaSkeletonTmpl).not.toContain('{{QA_METHODOLOGY}}');
|
||||
const qaSectionTmpl = fs.readFileSync(path.join(ROOT, 'qa', 'sections', 'qa-patterns.md.tmpl'), 'utf-8');
|
||||
expect(qaSectionTmpl).toContain('{{QA_METHODOLOGY}}');
|
||||
|
||||
const qaOnlyTmpl = fs.readFileSync(path.join(ROOT, 'qa-only', 'SKILL.md.tmpl'), 'utf-8');
|
||||
expect(qaOnlyTmpl).toContain('{{QA_METHODOLOGY}}');
|
||||
});
|
||||
|
||||
test('QA_METHODOLOGY appears expanded in both qa and qa-only generated files', () => {
|
||||
const qaContent = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8');
|
||||
const qaContent = readSkillUnion('qa'); // carved: methodology lives in qa/sections/qa-patterns.md
|
||||
const qaOnlyContent = fs.readFileSync(path.join(ROOT, 'qa-only', 'SKILL.md'), 'utf-8');
|
||||
|
||||
// Both should contain the health score rubric
|
||||
@@ -651,8 +668,9 @@ describe('GitLab support in generated skills', () => {
|
||||
*/
|
||||
describe('description quality evals', () => {
|
||||
// Regression: snapshot flags lost value hints (-d <N>, -s <sel>, -o <path>)
|
||||
// Browse carve: the flag reference renders into browse/sections/command-list.md.
|
||||
test('snapshot flags with values include value hints in output', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8');
|
||||
const content = readSkillUnion('browse');
|
||||
for (const flag of SNAPSHOT_FLAGS) {
|
||||
if (flag.takesValue) {
|
||||
expect(flag.valueHint).toBeDefined();
|
||||
@@ -1199,7 +1217,9 @@ describe('Plan file discovery shared helper', () => {
|
||||
// --- Retro plan completion ---
|
||||
|
||||
describe('Retro plan completion section', () => {
|
||||
const retroSkill = fs.readFileSync(path.join(ROOT, 'retro', 'SKILL.md'), 'utf-8');
|
||||
// Carved: the narrative report format (incl. Plan Completion) lives in
|
||||
// retro/sections/report-format.md — read the skeleton+sections union.
|
||||
const retroSkill = readSkillUnion('retro');
|
||||
|
||||
test('retro SKILL.md contains plan completion section', () => {
|
||||
expect(retroSkill).toContain('### Plan Completion');
|
||||
|
||||
@@ -550,6 +550,77 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
|
||||
mustContain: ['PGLite', 'Supabase', 'claude mcp add', 'read_secret_to_env', 'pooler'],
|
||||
maxSizeRatio: 1.07, // measured 1.051 vs the branch monolith: index + stubs + 4 STOP pointers
|
||||
},
|
||||
// ── Token-reduction Phase 4 wave 3 (v1.69.x branch) ──────────────────────
|
||||
qa: {
|
||||
skill: 'qa',
|
||||
expectedSections: ['test-bootstrap.md', 'qa-patterns.md'],
|
||||
requiredReads: ['qa-patterns.md'],
|
||||
scenario:
|
||||
'Walk /qa in SIMULATION — do not launch a browser, run any $B command, or execute bash; treat the working tree as clean, the tier as Quick, and the target app as http://localhost:3000 with a small feature-branch diff touching one page. Skip the test-framework bootstrap (assume CLAUDE.md documents the test command). Read each pointed section before doing its step, then produce the QA plan as the report: the mode you selected and why, the Phase 1-6 steps you would run, and a worked health-score computation from the rubric. Do NOT use AskUserQuestion.',
|
||||
staticInvariants: {
|
||||
mustStayInSkeleton: [
|
||||
'## Setup',
|
||||
'## SETUP (run this check BEFORE any browse command)',
|
||||
'## Phases 1-6: QA Baseline',
|
||||
'## Phase 7: Triage',
|
||||
'## Phase 8: Fix Loop',
|
||||
'8e.5. Regression Test',
|
||||
'WTF-LIKELIHOOD',
|
||||
'## Additional Rules (qa-specific)',
|
||||
'## Output Structure',
|
||||
],
|
||||
mustPrecedeStop: ['## Setup'],
|
||||
mustMoveToSection: [
|
||||
'## Test Framework Bootstrap',
|
||||
'BOOTSTRAP_DECLINED',
|
||||
'## Health Score Rubric',
|
||||
'### Diff-aware (automatic when on a feature branch with no URL)',
|
||||
'Never refuse to use the browser',
|
||||
],
|
||||
gateAfterStop: undefined,
|
||||
},
|
||||
behavioral: 'prompt',
|
||||
maxSkeletonBytes: 48_750, // Phase 4 wave 3; measured 48,151
|
||||
minUnionBytes: 69_500, // measured union 70,385
|
||||
mustContain: ['bug', 'browse', 'fix', 'Health Score Rubric', 'regression'],
|
||||
},
|
||||
browse: {
|
||||
skill: 'browse',
|
||||
expectedSections: ['command-list.md'],
|
||||
requiredReads: ['command-list.md'],
|
||||
scenario:
|
||||
'QA a static page: before driving it, plan the full audit — enumerate which browse commands and snapshot flags you would use, including extraction/tab/dialog commands beyond the Most-Used table, reading the full command reference first. Do not launch the browser or run any $B command; produce the command plan as the report.',
|
||||
staticInvariants: {
|
||||
mustStayInSkeleton: ['## SETUP', '## Core QA Patterns', '## CSS Inspector', '## Most-Used Commands'],
|
||||
mustPrecedeStop: ['## SETUP'],
|
||||
mustMoveToSection: ['## Full Command List', '## Snapshot Flags', '### Navigation'],
|
||||
gateAfterStop: undefined,
|
||||
},
|
||||
behavioral: 'prompt',
|
||||
maxSkeletonBytes: 27_500, // Phase 4 wave 3; measured 26,875
|
||||
minUnionBytes: 39_500, // measured union 41,115
|
||||
// 'BEGIN/END UNTRUSTED EXTERNAL' pins the untrusted-content warning; the full
|
||||
// envelope phrase wraps across lines in the rendered blockquote, so the
|
||||
// contiguous-substring check needs the single-line prefix form.
|
||||
mustContain: ['BEGIN/END UNTRUSTED EXTERNAL', 'snapshot -i', '@e refs', 'deviceScaleFactor', 'handoff'],
|
||||
},
|
||||
retro: {
|
||||
skill: 'retro',
|
||||
expectedSections: ['report-format.md'],
|
||||
requiredReads: ['report-format.md'],
|
||||
scenario:
|
||||
'Run the repo-scoped weekly retrospective for the last 7 days on this repo. There is no origin remote — proceed with the local branch per the guard disclosure rules. The gstack-retro-metrics script is not installed, so follow the degraded path (compute the metrics manually with git). Skip any AskUserQuestion calls — this is non-interactive. Produce the full narrative retrospective report.',
|
||||
staticInvariants: {
|
||||
mustStayInSkeleton: ['gstack-retro-metrics', '### Step 2: Compute Metrics', '### Step 13: Save Retro History'],
|
||||
mustPrecedeStop: ['### Step 2: Compute Metrics'],
|
||||
mustMoveToSection: ['## Engineering Retro: [date range]', '### Team Breakdown', 'Plan Completion This Period'],
|
||||
gateAfterStop: undefined,
|
||||
},
|
||||
behavioral: 'prompt',
|
||||
maxSkeletonBytes: 69_500, // Phase 4 wave 3; measured 68,483 (script absorption -5.4KB)
|
||||
minUnionBytes: 66_000, // measured union 73,496
|
||||
mustContain: ['retrospective', '45-minute gap', 'Ship of the week', 'Praise'],
|
||||
},
|
||||
};
|
||||
|
||||
/** Sorted carved-skill names. Consumers derive their lists from this — no parallel lists. */
|
||||
|
||||
@@ -206,21 +206,8 @@ export function runParityChecks(opts: {
|
||||
*/
|
||||
const MONOLITH_INVARIANTS: ParityInvariant[] = [
|
||||
// cso is now carved — its invariant is generated from CARVE_GUARDS below.
|
||||
// review, codex, land-and-deploy (wave 1) and autoplan (wave 2) carved in token-reduction Phase 4
|
||||
// review, codex, land-and-deploy (w1), autoplan (w2), qa (w3) carved in token-reduction Phase 4
|
||||
// wave 1 (v1.69.x branch) — their invariants generate from CARVE_GUARDS too.
|
||||
{
|
||||
skill: 'qa',
|
||||
mustContain: ['bug', 'browse', 'fix'],
|
||||
mustHaveHeadings: ['## Preamble', '## When to invoke'],
|
||||
// v1.2.0 activation lift: the unified first-run-guidance section (P4 scaffold +
|
||||
// P3 loop tip) is added to every skill's shared preamble — intentional, ~1KB.
|
||||
// Fork port wave 2: the shared coverage-audit detection block gained the
|
||||
// multi-ecosystem markers (Django/JVM, script/target/test-file census —
|
||||
// e3259078 port); measured 1.111x. v1.64+v1.65 merge sums both waves'
|
||||
// preamble growth; measured 1.125.
|
||||
maxSizeRatio: 1.13,
|
||||
minBytes: 50_000,
|
||||
},
|
||||
{
|
||||
skill: 'investigate',
|
||||
mustContain: ['root cause', 'hypothes'],
|
||||
|
||||
@@ -139,7 +139,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// devex, office-hours + future PR2 carves). One file iterating CARVE_GUARDS;
|
||||
// the selector sets GSTACK_CARVE_SKILL=<name> to scope cost to the changed
|
||||
// skill (D-CODEX A). Touching the registry/helper or sections.ts runs all.
|
||||
'carve-section-loading': ['review/**', 'codex/**', 'land-and-deploy/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'],
|
||||
'carve-section-loading': ['qa/**', 'browse/**', 'retro/**', 'autoplan/**', 'spec/**', 'setup-gbrain/**', 'review/**', 'codex/**', 'land-and-deploy/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'],
|
||||
'autoplan-chain-pty': ['autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-autoplan-chain.test.ts'],
|
||||
'e2e-harness-audit': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/claude-pty-runner.ts'],
|
||||
|
||||
@@ -229,8 +229,8 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'review-dashboard-via': ['ship/**', 'scripts/resolvers/review.ts', 'codex/**', 'autoplan/**', 'land-and-deploy/**', 'test/skill-e2e-review-attribution.test.ts'],
|
||||
|
||||
// Retro
|
||||
'retro': ['retro/**', 'test/skill-e2e-retro.test.ts'],
|
||||
'retro-base-branch': ['retro/**', 'test/skill-e2e-retro.test.ts'],
|
||||
'retro': ['bin/gstack-retro-metrics', 'retro/**', 'test/skill-e2e-retro.test.ts'],
|
||||
'retro-base-branch': ['bin/gstack-retro-metrics', 'retro/**', 'test/skill-e2e-retro.test.ts'],
|
||||
|
||||
// Global discover
|
||||
'global-discover': ['bin/gstack-global-discover.ts', 'test/global-discover.test.ts'],
|
||||
@@ -752,16 +752,16 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
* LLM-judge test touchfiles — keyed by test description string.
|
||||
*/
|
||||
export const LLM_JUDGE_TOUCHFILES: Record<string, string[]> = {
|
||||
'command reference table': ['SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts'],
|
||||
'snapshot flags reference': ['SKILL.md', 'SKILL.md.tmpl', 'browse/src/snapshot.ts'],
|
||||
'browse/SKILL.md reference': ['browse/SKILL.md', 'browse/SKILL.md.tmpl', 'browse/src/**'],
|
||||
'command reference table': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts'],
|
||||
'snapshot flags reference': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/snapshot.ts'],
|
||||
'browse/SKILL.md reference': ['browse/sections/**', 'browse/SKILL.md', 'browse/SKILL.md.tmpl', 'browse/src/**'],
|
||||
'setup block': ['SKILL.md', 'SKILL.md.tmpl'],
|
||||
'regression vs baseline': ['SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts', 'test/fixtures/eval-baselines.json'],
|
||||
'qa/SKILL.md workflow': ['qa/SKILL.md', 'qa/SKILL.md.tmpl'],
|
||||
'qa/SKILL.md health rubric': ['qa/SKILL.md', 'qa/SKILL.md.tmpl'],
|
||||
'qa/SKILL.md anti-refusal': ['qa/SKILL.md', 'qa/SKILL.md.tmpl', 'qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl'],
|
||||
'regression vs baseline': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts', 'test/fixtures/eval-baselines.json'],
|
||||
'qa/SKILL.md workflow': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl'],
|
||||
'qa/SKILL.md health rubric': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl'],
|
||||
'qa/SKILL.md anti-refusal': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl', 'qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl'],
|
||||
'cross-skill greptile consistency': ['review/SKILL.md', 'review/SKILL.md.tmpl', 'ship/SKILL.md', 'ship/SKILL.md.tmpl', 'review/greptile-triage.md', 'retro/SKILL.md', 'retro/SKILL.md.tmpl'],
|
||||
'baseline score pinning': ['SKILL.md', 'SKILL.md.tmpl', 'test/fixtures/eval-baselines.json'],
|
||||
'baseline score pinning': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'test/fixtures/eval-baselines.json'],
|
||||
|
||||
// Ship & Release
|
||||
'ship/SKILL.md workflow': ['ship/SKILL.md', 'ship/SKILL.md.tmpl'],
|
||||
@@ -789,7 +789,7 @@ export const LLM_JUDGE_TOUCHFILES: Record<string, string[]> = {
|
||||
'setup-deploy/SKILL.md platform setup': ['setup-deploy/SKILL.md', 'setup-deploy/SKILL.md.tmpl'],
|
||||
|
||||
// Other skills
|
||||
'retro/SKILL.md instructions': ['retro/SKILL.md', 'retro/SKILL.md.tmpl'],
|
||||
'retro/SKILL.md instructions': ['retro/sections/**', 'retro/SKILL.md', 'retro/SKILL.md.tmpl'],
|
||||
'qa-only/SKILL.md workflow': ['qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl'],
|
||||
'gstack-upgrade/SKILL.md upgrade flow': ['gstack-upgrade/SKILL.md', 'gstack-upgrade/SKILL.md.tmpl'],
|
||||
|
||||
|
||||
+83
-34
@@ -34,6 +34,38 @@ import {
|
||||
// Eval result collector
|
||||
const evalCollector = createEvalCollector('llm-judge');
|
||||
|
||||
/**
|
||||
* Browse carve (token-reduction Phase 4): the '## Snapshot Flags' and
|
||||
* '## Full Command List' reference blocks moved from browse/SKILL.md into the
|
||||
* generated on-demand section browse/sections/command-list.md ('## Snapshot
|
||||
* Flags' first, then '## Full Command List'). '## SETUP', '## Core QA
|
||||
* Patterns', and '## CSS Inspector' stay in the skeleton. Non-empty guard:
|
||||
* judging an empty slice would silently pass garbage to the judge.
|
||||
*/
|
||||
function readBrowseCommandSection(): string {
|
||||
const p = path.join(ROOT, 'browse', 'sections', 'command-list.md');
|
||||
const content = fs.readFileSync(p, 'utf-8');
|
||||
if (!content.includes('## Snapshot Flags') || !content.includes('## Full Command List')) {
|
||||
throw new Error(
|
||||
`${p} is missing the expected headers — regenerate with: bun run gen:skill-docs`,
|
||||
);
|
||||
}
|
||||
return content;
|
||||
}
|
||||
|
||||
/** Slice a section out of the command-list section file, guarded non-empty. */
|
||||
function sliceBrowseSection(startHeader: string, endHeader?: string): string {
|
||||
const content = readBrowseCommandSection();
|
||||
const start = content.indexOf(startHeader);
|
||||
if (start < 0) throw new Error(`browse/sections/command-list.md: "${startHeader}" not found`);
|
||||
const end = endHeader ? content.indexOf(endHeader) : -1;
|
||||
const section = end > start ? content.slice(start, end) : content.slice(start);
|
||||
if (section.trim().length < 200) {
|
||||
throw new Error(`browse/sections/command-list.md slice at "${startHeader}" is empty/stub — regenerate with: bun run gen:skill-docs`);
|
||||
}
|
||||
return section;
|
||||
}
|
||||
|
||||
// --- Diff-based test selection (LLM_JUDGE_TOUCHFILES, not the E2E table) ---
|
||||
const selectedTests = computeDiffSelection(LLM_JUDGE_TOUCHFILES, 'LLM-judge');
|
||||
|
||||
@@ -53,10 +85,9 @@ describeIfSelected('LLM-as-judge quality evals', [
|
||||
], () => {
|
||||
testIfSelected('command reference table', async () => {
|
||||
const t0 = Date.now();
|
||||
// P2 (v1.2.0): the command reference moved from the root router to browse/SKILL.md.
|
||||
const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8');
|
||||
const start = content.indexOf('## Full Command List');
|
||||
const section = content.slice(start);
|
||||
// Browse carve: the command reference lives in the generated on-demand
|
||||
// section browse/sections/command-list.md now (read via non-empty guard).
|
||||
const section = sliceBrowseSection('## Full Command List');
|
||||
|
||||
const scores = await judge('command reference table', section);
|
||||
console.log('Command reference scores:', JSON.stringify(scores, null, 2));
|
||||
@@ -82,11 +113,10 @@ describeIfSelected('LLM-as-judge quality evals', [
|
||||
|
||||
testIfSelected('snapshot flags reference', async () => {
|
||||
const t0 = Date.now();
|
||||
// P2 (v1.2.0): snapshot flags moved from the root router to browse/SKILL.md.
|
||||
const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8');
|
||||
const start = content.indexOf('## Snapshot Flags');
|
||||
const end = content.indexOf('## CSS Inspector');
|
||||
const section = content.slice(start, end);
|
||||
// Browse carve: snapshot flags live in browse/sections/command-list.md now,
|
||||
// ordered before '## Full Command List' (the '## CSS Inspector' end boundary
|
||||
// stayed in the skeleton).
|
||||
const section = sliceBrowseSection('## Snapshot Flags', '## Full Command List');
|
||||
|
||||
const scores = await judge('snapshot flags reference', section);
|
||||
console.log('Snapshot flags scores:', JSON.stringify(scores, null, 2));
|
||||
@@ -109,9 +139,8 @@ describeIfSelected('LLM-as-judge quality evals', [
|
||||
|
||||
testIfSelected('browse/SKILL.md reference', async () => {
|
||||
const t0 = Date.now();
|
||||
const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8');
|
||||
const start = content.indexOf('## Snapshot Flags');
|
||||
const section = content.slice(start);
|
||||
// Browse carve: flags + commands are the whole generated section file.
|
||||
const section = sliceBrowseSection('## Snapshot Flags');
|
||||
|
||||
const scores = await judge('browse skill reference (flags + commands)', section);
|
||||
console.log('Browse SKILL.md scores:', JSON.stringify(scores, null, 2));
|
||||
@@ -162,10 +191,8 @@ describeIfSelected('LLM-as-judge quality evals', [
|
||||
|
||||
testIfSelected('regression vs baseline', async () => {
|
||||
const t0 = Date.now();
|
||||
// P2 (v1.2.0): the command reference moved from the root router to browse/SKILL.md.
|
||||
const generated = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8');
|
||||
const genStart = generated.indexOf('## Full Command List');
|
||||
const genSection = generated.slice(genStart);
|
||||
// Browse carve: the command reference lives in browse/sections/command-list.md.
|
||||
const genSection = sliceBrowseSection('## Full Command List');
|
||||
|
||||
const baseline = `## Command Reference
|
||||
|
||||
@@ -249,14 +276,38 @@ Scores are 1-5 overall quality.`,
|
||||
|
||||
// --- Part 7: QA skill quality evals (C6) ---
|
||||
|
||||
describeIfSelected('QA skill quality evals', ['qa/SKILL.md workflow', 'qa/SKILL.md health rubric', 'qa/SKILL.md anti-refusal'], () => {
|
||||
const qaContent = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8');
|
||||
/**
|
||||
* QA carve (token-reduction Phase 4): the '## Modes', '## Workflow',
|
||||
* '## Health Score Rubric', '## Framework-Specific Guidance', and
|
||||
* '## Important Rules' blocks moved from qa/SKILL.md into the generated
|
||||
* on-demand section qa/sections/qa-patterns.md. Monolith-tolerant: falls back
|
||||
* to the skeleton when the section file doesn't exist (pre-carve checkout).
|
||||
*/
|
||||
function readQaPatterns(): string {
|
||||
const sectionPath = path.join(ROOT, 'qa', 'sections', 'qa-patterns.md');
|
||||
return fs.existsSync(sectionPath)
|
||||
? fs.readFileSync(sectionPath, 'utf-8')
|
||||
: fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8');
|
||||
}
|
||||
|
||||
/** Slice out of the qa-patterns section, guarded non-empty: judging an empty
|
||||
* slice would silently pass garbage to the judge. */
|
||||
function sliceQaPatterns(startHeader: string, endHeader?: string): string {
|
||||
const content = readQaPatterns();
|
||||
const start = content.indexOf(startHeader);
|
||||
if (start < 0) throw new Error(`qa/sections/qa-patterns.md: "${startHeader}" not found — regenerate with: bun run gen:skill-docs`);
|
||||
const end = endHeader ? content.indexOf(endHeader, start) : -1;
|
||||
const section = end > start ? content.slice(start, end) : content.slice(start);
|
||||
if (section.trim().length < 200) {
|
||||
throw new Error(`qa/sections/qa-patterns.md slice at "${startHeader}" is empty/stub — regenerate with: bun run gen:skill-docs`);
|
||||
}
|
||||
return section;
|
||||
}
|
||||
|
||||
describeIfSelected('QA skill quality evals', ['qa/SKILL.md workflow', 'qa/SKILL.md health rubric', 'qa/SKILL.md anti-refusal'], () => {
|
||||
testIfSelected('qa/SKILL.md workflow', async () => {
|
||||
const t0 = Date.now();
|
||||
const start = qaContent.indexOf('## Workflow');
|
||||
const end = qaContent.indexOf('## Health Score Rubric');
|
||||
const section = qaContent.slice(start, end);
|
||||
const section = sliceQaPatterns('## Workflow', '## Health Score Rubric');
|
||||
|
||||
const scores = await callJudge<JudgeScore>(`You are evaluating the quality of a QA testing workflow document for an AI coding agent.
|
||||
|
||||
@@ -297,8 +348,7 @@ ${section}`);
|
||||
|
||||
testIfSelected('qa/SKILL.md health rubric', async () => {
|
||||
const t0 = Date.now();
|
||||
const start = qaContent.indexOf('## Health Score Rubric');
|
||||
const section = qaContent.slice(start);
|
||||
const section = sliceQaPatterns('## Health Score Rubric');
|
||||
|
||||
const scores = await callJudge<JudgeScore>(`You are evaluating a health score rubric that an AI agent must follow to compute a numeric QA score.
|
||||
|
||||
@@ -338,13 +388,14 @@ ${section}`);
|
||||
|
||||
testIfSelected('qa/SKILL.md anti-refusal', async () => {
|
||||
const t0 = Date.now();
|
||||
// Extract both the diff-aware mode section and Important Rules section
|
||||
const diffAwareStart = qaContent.indexOf('### Diff-aware');
|
||||
const diffAwareEnd = qaContent.indexOf('### Full');
|
||||
const rulesStart = qaContent.indexOf('## Important Rules');
|
||||
const rulesEnd = qaContent.indexOf('## Framework-Specific');
|
||||
const diffAwareSection = qaContent.slice(diffAwareStart, diffAwareEnd);
|
||||
const rulesSection = qaContent.slice(rulesStart, rulesEnd);
|
||||
// Extract both the diff-aware mode section and Important Rules section.
|
||||
// (Pre-carve this sliced '## Important Rules' → '## Framework-Specific',
|
||||
// which was EMPTY — Framework-Specific precedes Important Rules — so the
|
||||
// judge only ever saw excerpt 1. The section-file slice fixes that: rules
|
||||
// run to the end of qa-patterns.md, so rule 12 "Never refuse to use the
|
||||
// browser" now actually reaches the judge.)
|
||||
const diffAwareSection = sliceQaPatterns('### Diff-aware', '### Full');
|
||||
const rulesSection = sliceQaPatterns('## Important Rules');
|
||||
|
||||
const result = await callJudge<{ would_browse: boolean; fallback_behavior: string; confidence: number; reasoning: string }>(`You are evaluating whether a QA testing skill document would cause an AI agent to USE THE BROWSER or REFUSE to use the browser in a specific scenario.
|
||||
|
||||
@@ -470,10 +521,8 @@ describeIfSelected('Baseline score pinning', ['baseline score pinning'], () => {
|
||||
const baselines = JSON.parse(fs.readFileSync(baselinesPath, 'utf-8'));
|
||||
const regressions: string[] = [];
|
||||
|
||||
// P2 (v1.2.0): the command reference moved from the root router to browse/SKILL.md.
|
||||
const skillContent = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8');
|
||||
const cmdStart = skillContent.indexOf('## Full Command List');
|
||||
const cmdSection = skillContent.slice(cmdStart);
|
||||
// Browse carve: the command reference lives in browse/sections/command-list.md.
|
||||
const cmdSection = sliceBrowseSection('## Full Command List');
|
||||
const cmdScores = await judge('command reference table', cmdSection);
|
||||
|
||||
for (const dim of ['clarity', 'completeness', 'actionability'] as const) {
|
||||
|
||||
@@ -65,6 +65,25 @@ describe('SKILL.md command validation', () => {
|
||||
expect(result.snapshotFlagErrors).toHaveLength(0);
|
||||
});
|
||||
|
||||
// qa carve: the Phases 1-6 methodology (with its $B command examples) moved
|
||||
// into qa/sections/*.md — validate the section files too so browse-command
|
||||
// coverage doesn't silently shrink with the carve.
|
||||
test('all $B commands and snapshot flags in qa/sections/*.md are valid', () => {
|
||||
const secDir = path.join(ROOT, 'qa', 'sections');
|
||||
if (!fs.existsSync(secDir)) return; // pre-carve checkout
|
||||
const sectionMds = fs.readdirSync(secDir).filter(f => f.endsWith('.md') && !f.endsWith('.md.tmpl'));
|
||||
expect(sectionMds.length).toBeGreaterThan(0);
|
||||
let validTotal = 0;
|
||||
for (const f of sectionMds) {
|
||||
const result = validateSkill(path.join(secDir, f));
|
||||
expect({ file: f, invalid: result.invalid }).toEqual({ file: f, invalid: [] });
|
||||
expect({ file: f, snapshotFlagErrors: result.snapshotFlagErrors }).toEqual({ file: f, snapshotFlagErrors: [] });
|
||||
validTotal += result.valid.length;
|
||||
}
|
||||
// Non-empty guard: the carved methodology must still carry $B examples.
|
||||
expect(validTotal).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('all $B commands in qa-only/SKILL.md are valid browse commands', () => {
|
||||
const qaOnlySkill = path.join(ROOT, 'qa-only', 'SKILL.md');
|
||||
if (!fs.existsSync(qaOnlySkill)) return;
|
||||
@@ -416,7 +435,9 @@ describe('Cross-skill path consistency', () => {
|
||||
// --- Part 7: QA skill structure validation (A2) ---
|
||||
|
||||
describe('QA skill structure validation', () => {
|
||||
const qaContent = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8');
|
||||
// qa carve: modes, Phases 1-6, and the health rubric moved into
|
||||
// qa/sections/qa-patterns.md — validate the skeleton+sections union.
|
||||
const qaContent = readSkillUnion('qa');
|
||||
|
||||
test('qa/SKILL.md has all 11 phases', () => {
|
||||
const phases = [
|
||||
@@ -1090,8 +1111,10 @@ describe('gstack-slug', () => {
|
||||
// --- Test Bootstrap validation ---
|
||||
|
||||
describe('Test Bootstrap ({{TEST_BOOTSTRAP}}) integration', () => {
|
||||
// qa carve: the rendered TEST_BOOTSTRAP body lives in
|
||||
// qa/sections/test-bootstrap.md — read the skeleton+sections union.
|
||||
test('TEST_BOOTSTRAP resolver produces valid content', () => {
|
||||
const qaContent = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8');
|
||||
const qaContent = readSkillUnion('qa');
|
||||
expect(qaContent).toContain('Test Framework Bootstrap');
|
||||
expect(qaContent).toContain('RUNTIME:ruby');
|
||||
expect(qaContent).toContain('RUNTIME:node');
|
||||
@@ -1101,7 +1124,7 @@ describe('Test Bootstrap ({{TEST_BOOTSTRAP}}) integration', () => {
|
||||
});
|
||||
|
||||
test('TEST_BOOTSTRAP appears in qa/SKILL.md', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8');
|
||||
const content = readSkillUnion('qa');
|
||||
expect(content).toContain('Test Framework Bootstrap');
|
||||
expect(content).toContain('TESTING.md');
|
||||
expect(content).toContain('CLAUDE.md');
|
||||
@@ -1127,7 +1150,7 @@ describe('Test Bootstrap ({{TEST_BOOTSTRAP}}) integration', () => {
|
||||
});
|
||||
|
||||
test('bootstrap includes framework knowledge table', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8');
|
||||
const content = readSkillUnion('qa');
|
||||
expect(content).toContain('vitest');
|
||||
expect(content).toContain('minitest');
|
||||
expect(content).toContain('pytest');
|
||||
@@ -1137,20 +1160,20 @@ describe('Test Bootstrap ({{TEST_BOOTSTRAP}}) integration', () => {
|
||||
});
|
||||
|
||||
test('bootstrap includes CI/CD pipeline generation', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8');
|
||||
const content = readSkillUnion('qa');
|
||||
expect(content).toContain('.github/workflows/test.yml');
|
||||
expect(content).toContain('GitHub Actions');
|
||||
});
|
||||
|
||||
test('bootstrap includes first real tests step', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8');
|
||||
const content = readSkillUnion('qa');
|
||||
expect(content).toContain('First real tests');
|
||||
expect(content).toContain('git log --since=30.days');
|
||||
expect(content).toContain('Prioritize by risk');
|
||||
});
|
||||
|
||||
test('bootstrap includes vibe coding philosophy', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8');
|
||||
const content = readSkillUnion('qa');
|
||||
expect(content).toContain('vibe coding');
|
||||
expect(content).toContain('100% test coverage');
|
||||
});
|
||||
@@ -1325,28 +1348,43 @@ describe('ship step numbering', () => {
|
||||
// --- Retro test health validation ---
|
||||
|
||||
describe('Retro test health tracking', () => {
|
||||
test('retro/SKILL.md has test health data gathering commands', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'retro', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('# 10. Test file count');
|
||||
expect(content).toContain('# 11. Regression test commits');
|
||||
expect(content).toContain('# 12. Test files changed');
|
||||
// The inline git pipelines moved into bin/gstack-retro-metrics (retro
|
||||
// token-reduction wave); the skill interprets its labeled METRIC lines.
|
||||
test('gstack-retro-metrics gathers the test health data', () => {
|
||||
const script = fs.readFileSync(path.join(ROOT, 'bin', 'gstack-retro-metrics'), 'utf-8');
|
||||
expect(script).toContain('TEST_FILES_TOTAL');
|
||||
expect(script).toContain('REGRESSION_TEST_COMMITS');
|
||||
expect(script).toContain('TEST_FILES_CHANGED');
|
||||
// The historical grep targets survive the script absorption.
|
||||
expect(script).toContain('test(qa):');
|
||||
expect(script).toContain('test(design):');
|
||||
expect(script).toContain('git ls-files');
|
||||
});
|
||||
|
||||
test('retro skill interprets the test-health metric lines', () => {
|
||||
// Template source, not the generated render: the pin must hold across the
|
||||
// regen boundary (the generated file follows the template at gen time).
|
||||
const content = fs.readFileSync(path.join(ROOT, 'retro', 'SKILL.md.tmpl'), 'utf-8');
|
||||
expect(content).toContain('TEST_FILES_TOTAL');
|
||||
expect(content).toContain('REGRESSION_TEST_COMMITS');
|
||||
});
|
||||
|
||||
test('retro/SKILL.md has Test Health metrics row', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'retro', 'SKILL.md'), 'utf-8');
|
||||
const content = readSkillUnion('retro');
|
||||
expect(content).toContain('Test Health');
|
||||
expect(content).toContain('regression tests');
|
||||
});
|
||||
|
||||
test('retro/SKILL.md has Test Health narrative section', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'retro', 'SKILL.md'), 'utf-8');
|
||||
// Carved: the narrative report format lives in sections/report-format.md.
|
||||
const content = readSkillUnion('retro');
|
||||
expect(content).toContain('### Test Health');
|
||||
expect(content).toContain('Total test files');
|
||||
expect(content).toContain('vibe coding safe');
|
||||
});
|
||||
|
||||
test('retro JSON schema includes test_health field', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'retro', 'SKILL.md'), 'utf-8');
|
||||
const content = readSkillUnion('retro');
|
||||
expect(content).toContain('test_health');
|
||||
expect(content).toContain('total_test_files');
|
||||
expect(content).toContain('regression_test_commits');
|
||||
|
||||
Reference in New Issue
Block a user