mirror of
https://github.com/garrytan/gstack.git
synced 2026-06-17 23:30:09 +02:00
Garry Tan
296937d466
Phase 0 deliverable — eval-first foundation. Two new test files plus the
registry:
1. test/skill-coverage-matrix.ts — single source of truth mapping each
skill to its gate-tier + periodic-tier test files. SKILL_COVERAGE
record with 51 entries; every gstack skill on disk has at least one
gate-tier entry.
2. test/skill-coverage-matrix.test.ts — CI gate. Asserts every skill on
disk has a registry entry AND that gate[] is non-empty. Catches
"skill added but eval not registered" the moment a new SKILL.md
lands.
3. test/skill-coverage-floor.test.ts — per-skill structural compliance
(FREE, file-IO only). For each of 51 skills, verifies:
- SKILL.md exists
- Frontmatter well-formed (name + description fields)
- Catalog-trim contract (inline description ≤ 250 chars, or block form)
- Generated header present (edit .tmpl, not .md)
- Body ≥ 200 bytes (non-trivial content)
- No unresolved {{TEMPLATE}} placeholders leaked
The "floor" is the minimum eval that every skill ships with. Skills that
need deeper behavioral testing get additional entries in their coverage
record (e.g., ship has skill-e2e-ship-idempotency + workflow + floor).
Future skills only need to add the floor entry and the matrix gate
unblocks them.
Codex 2nd-pass critique #1 mitigation: eval-first floor is structural
compliance (the testable part) — judgment-skill behavior gets layered
periodic-tier evals on top. We don't pretend the floor proves
correctness, only that the skill structurally compiles.
Test plan:
- bun test test/skill-coverage-matrix.test.ts: 4 pass (matrix shape + coverage)
- bun test test/skill-coverage-floor.test.ts: 309 pass (6 checks × 51 skills + 3 registry-level)
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
73 lines
2.5 KiB
TypeScript
73 lines
2.5 KiB
TypeScript
/**
|
|
* Skill coverage matrix CI gate (v1.45.0.0 T1).
|
|
*
|
|
* Asserts every skill on disk has an entry in SKILL_COVERAGE with at
|
|
* least one gate-tier test. The detailed per-skill structural checks
|
|
* live in test/skill-coverage-floor.test.ts; this file is the matrix-
|
|
* level gate that surfaces "skill added but eval not registered" cleanly.
|
|
*/
|
|
|
|
import { describe, test, expect } from 'bun:test';
|
|
import * as fs from 'fs';
|
|
import * as path from 'path';
|
|
import { SKILL_COVERAGE, type SkillCoverage } from './skill-coverage-matrix';
|
|
|
|
const REPO_ROOT = path.resolve(import.meta.dir, '..');
|
|
|
|
function discoverSkills(): string[] {
|
|
return fs.readdirSync(REPO_ROOT, { withFileTypes: true })
|
|
.filter(e => e.isDirectory() && !e.name.startsWith('.'))
|
|
.filter(e => fs.existsSync(path.join(REPO_ROOT, e.name, 'SKILL.md')))
|
|
.map(e => e.name)
|
|
.sort();
|
|
}
|
|
|
|
describe('skill coverage matrix', () => {
|
|
test('SKILL_COVERAGE is exported and non-empty', () => {
|
|
expect(typeof SKILL_COVERAGE).toBe('object');
|
|
expect(Object.keys(SKILL_COVERAGE).length).toBeGreaterThan(0);
|
|
});
|
|
|
|
test('every entry has the right shape', () => {
|
|
for (const [skill, coverage] of Object.entries(SKILL_COVERAGE)) {
|
|
expect(Array.isArray(coverage.gate)).toBe(true);
|
|
expect(Array.isArray(coverage.periodic)).toBe(true);
|
|
expect(coverage.gate.length).toBeGreaterThan(0);
|
|
for (const p of [...coverage.gate, ...coverage.periodic]) {
|
|
expect(typeof p).toBe('string');
|
|
expect(p.startsWith('test/')).toBe(true);
|
|
expect(p.endsWith('.test.ts')).toBe(true);
|
|
}
|
|
}
|
|
});
|
|
|
|
test('every skill on disk has a registry entry', () => {
|
|
const skills = discoverSkills();
|
|
const missing: string[] = [];
|
|
for (const s of skills) {
|
|
if (!SKILL_COVERAGE[s]) missing.push(s);
|
|
}
|
|
if (missing.length > 0) {
|
|
throw new Error(
|
|
`Skills on disk missing from SKILL_COVERAGE: ${missing.join(', ')}. ` +
|
|
`Add an entry to test/skill-coverage-matrix.ts with at least ` +
|
|
`'test/skill-coverage-floor.test.ts' in gate[].`,
|
|
);
|
|
}
|
|
});
|
|
|
|
test('no registry entry references a skill that does not exist on disk', () => {
|
|
const skills = new Set(discoverSkills());
|
|
const orphans: string[] = [];
|
|
for (const skill of Object.keys(SKILL_COVERAGE)) {
|
|
if (!skills.has(skill)) orphans.push(skill);
|
|
}
|
|
if (orphans.length > 0) {
|
|
throw new Error(
|
|
`Registry references skills not on disk: ${orphans.join(', ')}. ` +
|
|
`Remove from SKILL_COVERAGE or restore the skill directory.`,
|
|
);
|
|
}
|
|
});
|
|
});
|