mirror of
https://github.com/garrytan/gstack.git
synced 2026-05-02 11:45:20 +02:00
test: learnings resolver + bin script edge case tests — 21 new tests, free
Adds gen-skill-docs coverage for LEARNINGS_SEARCH, LEARNINGS_LOG, and CONFIDENCE_CALIBRATION resolvers. Adds bin script edge cases: timestamp preservation, special characters, files array, sort order, type grouping, combined filtering, missing fields, confidence floor at 0. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
@@ -1969,3 +1969,113 @@ describe('codex commands must not use inline $(git rev-parse --show-toplevel) fo
|
||||
expect(violations).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Learnings + Confidence Resolver Tests ─────────────────────
|
||||
|
||||
describe('LEARNINGS_SEARCH resolver', () => {
|
||||
const SEARCH_SKILLS = ['review', 'ship', 'plan-eng-review', 'investigate', 'office-hours', 'plan-ceo-review'];
|
||||
|
||||
for (const skill of SEARCH_SKILLS) {
|
||||
test(`${skill} generated SKILL.md contains learnings search`, () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, skill, 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('Prior Learnings');
|
||||
expect(content).toContain('gstack-learnings-search');
|
||||
});
|
||||
}
|
||||
|
||||
test('learnings search includes cross-project config check', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('cross_project_learnings');
|
||||
expect(content).toContain('--cross-project');
|
||||
});
|
||||
|
||||
test('learnings search includes AskUserQuestion for first-time cross-project opt-in', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('Enable cross-project learnings');
|
||||
expect(content).toContain('project-scoped only');
|
||||
});
|
||||
|
||||
test('learnings search mentions prior learning applied display format', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('Prior learning applied');
|
||||
});
|
||||
});
|
||||
|
||||
describe('LEARNINGS_LOG resolver', () => {
|
||||
const LOG_SKILLS = ['review', 'retro', 'investigate'];
|
||||
|
||||
for (const skill of LOG_SKILLS) {
|
||||
test(`${skill} generated SKILL.md contains learnings log`, () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, skill, 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('Capture Learnings');
|
||||
expect(content).toContain('gstack-learnings-log');
|
||||
});
|
||||
}
|
||||
|
||||
test('learnings log documents all type values', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8');
|
||||
for (const type of ['pattern', 'pitfall', 'preference', 'architecture', 'tool']) {
|
||||
expect(content).toContain(type);
|
||||
}
|
||||
});
|
||||
|
||||
test('learnings log documents all source values', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8');
|
||||
for (const source of ['observed', 'user-stated', 'inferred', 'cross-model']) {
|
||||
expect(content).toContain(source);
|
||||
}
|
||||
});
|
||||
|
||||
test('learnings log includes files field for staleness detection', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('"files"');
|
||||
expect(content).toContain('staleness detection');
|
||||
});
|
||||
});
|
||||
|
||||
describe('CONFIDENCE_CALIBRATION resolver', () => {
|
||||
const CONFIDENCE_SKILLS = ['review', 'ship', 'plan-eng-review', 'cso'];
|
||||
|
||||
for (const skill of CONFIDENCE_SKILLS) {
|
||||
test(`${skill} generated SKILL.md contains confidence calibration`, () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, skill, 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('Confidence Calibration');
|
||||
expect(content).toContain('confidence score');
|
||||
});
|
||||
}
|
||||
|
||||
test('confidence calibration includes scoring rubric with all tiers', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('9-10');
|
||||
expect(content).toContain('7-8');
|
||||
expect(content).toContain('5-6');
|
||||
expect(content).toContain('3-4');
|
||||
expect(content).toContain('1-2');
|
||||
});
|
||||
|
||||
test('confidence calibration includes display rules', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('Show normally');
|
||||
expect(content).toContain('Suppress from main report');
|
||||
});
|
||||
|
||||
test('confidence calibration includes finding format example', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('[P1] (confidence:');
|
||||
expect(content).toContain('SQL injection');
|
||||
});
|
||||
|
||||
test('confidence calibration includes calibration learning feedback loop', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('calibration event');
|
||||
expect(content).toContain('Log the corrected pattern');
|
||||
});
|
||||
|
||||
test('skills without confidence calibration do NOT contain it', () => {
|
||||
// office-hours and retro do NOT use confidence calibration
|
||||
for (const skill of ['office-hours', 'retro']) {
|
||||
const content = fs.readFileSync(path.join(ROOT, skill, 'SKILL.md'), 'utf-8');
|
||||
expect(content).not.toContain('## Confidence Calibration');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -191,3 +191,93 @@ describe('gstack-learnings-search', () => {
|
||||
expect(output).toContain('also-valid');
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-learnings-log edge cases', () => {
|
||||
test('preserves existing timestamp when ts is present', () => {
|
||||
const input = '{"skill":"review","type":"pattern","key":"ts-preserve","insight":"test","confidence":5,"source":"observed","ts":"2025-06-15T10:00:00Z"}';
|
||||
runLog(input);
|
||||
|
||||
const f = findLearningsFile();
|
||||
expect(f).not.toBeNull();
|
||||
const parsed = JSON.parse(fs.readFileSync(f!, 'utf-8').trim());
|
||||
expect(parsed.ts).toBe('2025-06-15T10:00:00Z');
|
||||
});
|
||||
|
||||
test('handles JSON with special characters in insight', () => {
|
||||
const input = JSON.stringify({ skill: 'review', type: 'pattern', key: 'special-chars', insight: 'Use "quotes" and \\backslashes', confidence: 7, source: 'observed' });
|
||||
runLog(input);
|
||||
|
||||
const f = findLearningsFile();
|
||||
expect(f).not.toBeNull();
|
||||
const parsed = JSON.parse(fs.readFileSync(f!, 'utf-8').trim());
|
||||
expect(parsed.insight).toContain('quotes');
|
||||
expect(parsed.insight).toContain('backslashes');
|
||||
});
|
||||
|
||||
test('handles JSON with files array field', () => {
|
||||
const input = JSON.stringify({ skill: 'review', type: 'architecture', key: 'with-files', insight: 'test', confidence: 8, source: 'observed', files: ['src/auth.ts', 'src/db.ts'] });
|
||||
runLog(input);
|
||||
|
||||
const f = findLearningsFile();
|
||||
expect(f).not.toBeNull();
|
||||
const parsed = JSON.parse(fs.readFileSync(f!, 'utf-8').trim());
|
||||
expect(parsed.files).toEqual(['src/auth.ts', 'src/db.ts']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-learnings-search edge cases', () => {
|
||||
test('sorts by confidence then recency', () => {
|
||||
// Two entries: one high confidence old, one lower confidence recent
|
||||
runLog(JSON.stringify({ skill: 'review', type: 'pattern', key: 'high-conf', insight: 'high confidence entry', confidence: 9, source: 'user-stated', ts: '2026-01-01T00:00:00Z' }));
|
||||
runLog(JSON.stringify({ skill: 'review', type: 'pattern', key: 'recent', insight: 'recent entry', confidence: 5, source: 'observed', ts: '2026-03-28T00:00:00Z' }));
|
||||
|
||||
const output = runSearch();
|
||||
const highIdx = output.indexOf('high-conf');
|
||||
const recentIdx = output.indexOf('recent');
|
||||
// High confidence should appear first
|
||||
expect(highIdx).toBeLessThan(recentIdx);
|
||||
});
|
||||
|
||||
test('groups output by type', () => {
|
||||
runLog(JSON.stringify({ skill: 'review', type: 'pattern', key: 'p1', insight: 'a pattern', confidence: 7, source: 'observed' }));
|
||||
runLog(JSON.stringify({ skill: 'review', type: 'pitfall', key: 'pit1', insight: 'a pitfall', confidence: 7, source: 'observed' }));
|
||||
|
||||
const output = runSearch();
|
||||
expect(output).toContain('## Patterns');
|
||||
expect(output).toContain('## Pitfalls');
|
||||
});
|
||||
|
||||
test('combined --type and --query filtering', () => {
|
||||
runLog(JSON.stringify({ skill: 'review', type: 'pattern', key: 'auth-token', insight: 'check token expiry', confidence: 7, source: 'observed' }));
|
||||
runLog(JSON.stringify({ skill: 'review', type: 'pitfall', key: 'auth-leak', insight: 'auth token in logs', confidence: 7, source: 'observed' }));
|
||||
runLog(JSON.stringify({ skill: 'review', type: 'pattern', key: 'cache-key', insight: 'cache invalidation', confidence: 7, source: 'observed' }));
|
||||
|
||||
const output = runSearch('--type pattern --query auth');
|
||||
expect(output).toContain('auth-token');
|
||||
expect(output).not.toContain('auth-leak'); // wrong type
|
||||
expect(output).not.toContain('cache-key'); // wrong query
|
||||
});
|
||||
|
||||
test('entries with missing key or type are skipped', () => {
|
||||
runLog(JSON.stringify({ skill: 'review', type: 'pattern', key: 'valid', insight: 'valid entry', confidence: 7, source: 'observed' }));
|
||||
const f = findLearningsFile();
|
||||
expect(f).not.toBeNull();
|
||||
// Append entries missing key and type
|
||||
fs.appendFileSync(f!, JSON.stringify({ skill: 'review', type: 'pattern', insight: 'no key', confidence: 7, source: 'observed' }) + '\n');
|
||||
fs.appendFileSync(f!, JSON.stringify({ skill: 'review', key: 'no-type', insight: 'no type', confidence: 7, source: 'observed' }) + '\n');
|
||||
|
||||
const output = runSearch();
|
||||
expect(output).toContain('valid');
|
||||
expect(output).not.toContain('no key');
|
||||
expect(output).not.toContain('no-type');
|
||||
});
|
||||
|
||||
test('confidence decay floors at 0 (never negative)', () => {
|
||||
// Entry from 1 year ago with confidence 3 — decay would be 12, clamped to 0
|
||||
const ts = new Date(Date.now() - 365 * 86400000).toISOString();
|
||||
runLog(JSON.stringify({ skill: 'review', type: 'pattern', key: 'ancient', insight: 'very old', confidence: 3, source: 'observed', ts }));
|
||||
|
||||
const output = runSearch();
|
||||
expect(output).toContain('confidence: 0/10');
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user