gstack/test/skill-coverage-floor.test.ts

/**
 * Skill coverage floor — gate-tier, free, runs every PR.
 *
 * Phase 0 of the cathedral parity-eval suite: structural-compliance smoke
 * test that covers every gstack skill with file-IO assertions. The intent
 * is "every skill ships with at least one CI-blocking check" — even when
 * a skill doesn't (yet) have a behavioral E2E test, this floor catches
 * frontmatter regressions, missing generated header, empty/trivial bodies,
 * and dangling SKILL.md.tmpl-without-SKILL.md mismatches.
 *
 * Pairs with test/parity-suite.test.ts (the content-invariant suite).
 * The floor iterates every authored skill from skillCensus(), so a new
 * skill is covered without registering it anywhere.
 */

import { describe, test, expect } from 'bun:test';
import * as fs from 'fs';
import * as path from 'path';
import { skillCensus } from './helpers/skill-census';

const REPO_ROOT = path.resolve(import.meta.dir, '..');

function readSkillMd(skill: string): string | null {
  const p = path.join(REPO_ROOT, skill, 'SKILL.md');
  try {
    return fs.readFileSync(p, 'utf-8');
  } catch {
    return null;
  }
}

describe('skill-coverage-floor: every skill passes structural compliance', () => {
  const skills = skillCensus(REPO_ROOT).authoredSkills;

  // Per-skill structural compliance (file IO only, no LLM)
  for (const skill of skills) {
    describe(`skill: ${skill}`, () => {
      test('SKILL.md exists', () => {
        const content = readSkillMd(skill);
        expect(content).not.toBeNull();
      });

      test('frontmatter is well-formed and contains name + description', () => {
        const content = readSkillMd(skill)!;
        expect(content.startsWith('---\n')).toBe(true);
        const fmEnd = content.indexOf('\n---', 4);
        expect(fmEnd).toBeGreaterThan(0);
        const fm = content.slice(4, fmEnd);
        // name: ...
        expect(/^name:\s*\S/m.test(fm)).toBe(true);
        // description: ... (either inline or block form)
        expect(/^description:\s*(\S|\|)/m.test(fm)).toBe(true);
      });

      test('frontmatter description fits the catalog-trim contract', () => {
        const content = readSkillMd(skill)!;
        const fmEnd = content.indexOf('\n---', 4);
        const fm = content.slice(4, fmEnd);
        // Inline form: description: <one line>
        const inlineMatch = fm.match(/^description:\s+(.+)$/m);
        // Block form: description: |\n  multiline
        const blockMatch = fm.match(/^description:\s*\|/m);
        if (inlineMatch) {
          // Catalog-trimmed: should be ≤ 250 chars
          expect(inlineMatch[1].length).toBeLessThanOrEqual(250);
        } else if (blockMatch) {
          // Block form is acceptable for small skills (under-120-chars baseline
          // didn't trigger catalog trim). No size cap here; the parity-suite
          // and size-budget tests handle bytes.
        } else {
          throw new Error(`${skill}: description field is not in inline or block form`);
        }
      });

      test('generated header present (only edit .tmpl, not .md)', () => {
        const content = readSkillMd(skill)!;
        expect(content).toContain('AUTO-GENERATED from SKILL.md.tmpl');
      });

      test('body is non-trivial (≥ 200 bytes after frontmatter)', () => {
        const content = readSkillMd(skill)!;
        const fmEnd = content.indexOf('\n---', 4);
        const body = content.slice(fmEnd + 5).trim();
        expect(body.length).toBeGreaterThanOrEqual(200);
      });

      test('no unresolved {{TEMPLATE}} placeholders leaked into output', () => {
        const content = readSkillMd(skill)!;
        const leaks = content.match(/\{\{[A-Z_]+(?::[^}]+)?\}\}/g);
        if (leaks) {
          throw new Error(
            `${skill}: ${leaks.length} unresolved placeholder(s) in generated SKILL.md: ${leaks.slice(0, 3).join(', ')}${leaks.length > 3 ? ', ...' : ''}`,
          );
        }
      });
    });
  }
});