mirror of
https://github.com/garrytan/gstack.git
synced 2026-05-02 03:35:09 +02:00
test: quality evals for generated SKILL.md descriptions
Catches the exact regressions we shipped and caught in review: - Snapshot flags must include value hints (-d <N>, -s <sel>, -o <path>) - is command must list all valid states (visible/hidden/enabled/...) - press command must list example keys (Enter, Tab, Escape) - console command must describe --errors behavior - Snapshot -i must mention @e refs, -C must mention @c refs - All descriptions must be >= 8 chars (no empty stubs) - Tips section must use → not ->
This commit is contained in:
@@ -71,3 +71,80 @@ describe('gen-skill-docs', () => {
|
||||
expect(browseTmpl).toContain('{{SNAPSHOT_FLAGS}}');
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Quality evals — catch description regressions.
|
||||
*
|
||||
* These test that generated output is *useful for an AI agent*,
|
||||
* not just structurally valid. Each test targets a specific
|
||||
* regression we actually shipped and caught in review.
|
||||
*/
|
||||
describe('description quality evals', () => {
|
||||
// Regression: snapshot flags lost value hints (-d <N>, -s <sel>, -o <path>)
|
||||
test('snapshot flags with values include value hints in output', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
|
||||
for (const flag of SNAPSHOT_FLAGS) {
|
||||
if (flag.takesValue) {
|
||||
expect(flag.valueHint).toBeDefined();
|
||||
expect(content).toContain(`${flag.short} ${flag.valueHint}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Regression: "is" lost the valid states enum
|
||||
test('is command lists valid state values', () => {
|
||||
const desc = COMMAND_DESCRIPTIONS['is'].description;
|
||||
for (const state of ['visible', 'hidden', 'enabled', 'disabled', 'checked', 'editable', 'focused']) {
|
||||
expect(desc).toContain(state);
|
||||
}
|
||||
});
|
||||
|
||||
// Regression: "press" lost common key examples
|
||||
test('press command lists example keys', () => {
|
||||
const desc = COMMAND_DESCRIPTIONS['press'].description;
|
||||
expect(desc).toContain('Enter');
|
||||
expect(desc).toContain('Tab');
|
||||
expect(desc).toContain('Escape');
|
||||
});
|
||||
|
||||
// Regression: "console" lost --errors filter note
|
||||
test('console command describes --errors behavior', () => {
|
||||
const desc = COMMAND_DESCRIPTIONS['console'].description;
|
||||
expect(desc).toContain('--errors');
|
||||
});
|
||||
|
||||
// Regression: snapshot -i lost "@e refs" context
|
||||
test('snapshot -i mentions @e refs', () => {
|
||||
const flag = SNAPSHOT_FLAGS.find(f => f.short === '-i')!;
|
||||
expect(flag.description).toContain('@e');
|
||||
});
|
||||
|
||||
// Regression: snapshot -C lost "@c refs" context
|
||||
test('snapshot -C mentions @c refs', () => {
|
||||
const flag = SNAPSHOT_FLAGS.find(f => f.short === '-C')!;
|
||||
expect(flag.description).toContain('@c');
|
||||
});
|
||||
|
||||
// Guard: every description must be at least 8 chars (catches empty or stub descriptions)
|
||||
test('all command descriptions have meaningful length', () => {
|
||||
for (const [cmd, meta] of Object.entries(COMMAND_DESCRIPTIONS)) {
|
||||
expect(meta.description.length).toBeGreaterThanOrEqual(8);
|
||||
}
|
||||
});
|
||||
|
||||
// Guard: snapshot flag descriptions must be at least 10 chars
|
||||
test('all snapshot flag descriptions have meaningful length', () => {
|
||||
for (const flag of SNAPSHOT_FLAGS) {
|
||||
expect(flag.description.length).toBeGreaterThanOrEqual(10);
|
||||
}
|
||||
});
|
||||
|
||||
// Guard: generated output uses → not ->
|
||||
test('generated SKILL.md uses unicode arrows', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
|
||||
// Check the Tips section specifically (where we regressed -> from →)
|
||||
const tipsSection = content.slice(content.indexOf('## Tips'));
|
||||
expect(tipsSection).toContain('→');
|
||||
expect(tipsSection).not.toContain('->');
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user