From b9a19e7c7e7266949977750dc63c4f74c7ee04d0 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 29 Mar 2026 08:28:01 -0700 Subject: [PATCH] =?UTF-8?q?test:=20learnings=20resolver=20+=20bin=20script?= =?UTF-8?q?=20edge=20case=20tests=20=E2=80=94=2021=20new=20tests,=20free?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds gen-skill-docs coverage for LEARNINGS_SEARCH, LEARNINGS_LOG, and CONFIDENCE_CALIBRATION resolvers. Adds bin script edge cases: timestamp preservation, special characters, files array, sort order, type grouping, combined filtering, missing fields, confidence floor at 0. Co-Authored-By: Claude Opus 4.6 (1M context) --- test/gen-skill-docs.test.ts | 110 ++++++++++++++++++++++++++++++++++++ test/learnings.test.ts | 90 +++++++++++++++++++++++++++++ 2 files changed, 200 insertions(+) diff --git a/test/gen-skill-docs.test.ts b/test/gen-skill-docs.test.ts index 3bbc1869..e5de9202 100644 --- a/test/gen-skill-docs.test.ts +++ b/test/gen-skill-docs.test.ts @@ -1969,3 +1969,113 @@ describe('codex commands must not use inline $(git rev-parse --show-toplevel) fo expect(violations).toEqual([]); }); }); + +// ─── Learnings + Confidence Resolver Tests ───────────────────── + +describe('LEARNINGS_SEARCH resolver', () => { + const SEARCH_SKILLS = ['review', 'ship', 'plan-eng-review', 'investigate', 'office-hours', 'plan-ceo-review']; + + for (const skill of SEARCH_SKILLS) { + test(`${skill} generated SKILL.md contains learnings search`, () => { + const content = fs.readFileSync(path.join(ROOT, skill, 'SKILL.md'), 'utf-8'); + expect(content).toContain('Prior Learnings'); + expect(content).toContain('gstack-learnings-search'); + }); + } + + test('learnings search includes cross-project config check', () => { + const content = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8'); + expect(content).toContain('cross_project_learnings'); + expect(content).toContain('--cross-project'); + }); + + test('learnings search includes AskUserQuestion for first-time cross-project opt-in', () => { + const content = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8'); + expect(content).toContain('Enable cross-project learnings'); + expect(content).toContain('project-scoped only'); + }); + + test('learnings search mentions prior learning applied display format', () => { + const content = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8'); + expect(content).toContain('Prior learning applied'); + }); +}); + +describe('LEARNINGS_LOG resolver', () => { + const LOG_SKILLS = ['review', 'retro', 'investigate']; + + for (const skill of LOG_SKILLS) { + test(`${skill} generated SKILL.md contains learnings log`, () => { + const content = fs.readFileSync(path.join(ROOT, skill, 'SKILL.md'), 'utf-8'); + expect(content).toContain('Capture Learnings'); + expect(content).toContain('gstack-learnings-log'); + }); + } + + test('learnings log documents all type values', () => { + const content = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8'); + for (const type of ['pattern', 'pitfall', 'preference', 'architecture', 'tool']) { + expect(content).toContain(type); + } + }); + + test('learnings log documents all source values', () => { + const content = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8'); + for (const source of ['observed', 'user-stated', 'inferred', 'cross-model']) { + expect(content).toContain(source); + } + }); + + test('learnings log includes files field for staleness detection', () => { + const content = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8'); + expect(content).toContain('"files"'); + expect(content).toContain('staleness detection'); + }); +}); + +describe('CONFIDENCE_CALIBRATION resolver', () => { + const CONFIDENCE_SKILLS = ['review', 'ship', 'plan-eng-review', 'cso']; + + for (const skill of CONFIDENCE_SKILLS) { + test(`${skill} generated SKILL.md contains confidence calibration`, () => { + const content = fs.readFileSync(path.join(ROOT, skill, 'SKILL.md'), 'utf-8'); + expect(content).toContain('Confidence Calibration'); + expect(content).toContain('confidence score'); + }); + } + + test('confidence calibration includes scoring rubric with all tiers', () => { + const content = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8'); + expect(content).toContain('9-10'); + expect(content).toContain('7-8'); + expect(content).toContain('5-6'); + expect(content).toContain('3-4'); + expect(content).toContain('1-2'); + }); + + test('confidence calibration includes display rules', () => { + const content = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8'); + expect(content).toContain('Show normally'); + expect(content).toContain('Suppress from main report'); + }); + + test('confidence calibration includes finding format example', () => { + const content = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8'); + expect(content).toContain('[P1] (confidence:'); + expect(content).toContain('SQL injection'); + }); + + test('confidence calibration includes calibration learning feedback loop', () => { + const content = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8'); + expect(content).toContain('calibration event'); + expect(content).toContain('Log the corrected pattern'); + }); + + test('skills without confidence calibration do NOT contain it', () => { + // office-hours and retro do NOT use confidence calibration + for (const skill of ['office-hours', 'retro']) { + const content = fs.readFileSync(path.join(ROOT, skill, 'SKILL.md'), 'utf-8'); + expect(content).not.toContain('## Confidence Calibration'); + } + }); +}); diff --git a/test/learnings.test.ts b/test/learnings.test.ts index b3129b5f..6d72266c 100644 --- a/test/learnings.test.ts +++ b/test/learnings.test.ts @@ -191,3 +191,93 @@ describe('gstack-learnings-search', () => { expect(output).toContain('also-valid'); }); }); + +describe('gstack-learnings-log edge cases', () => { + test('preserves existing timestamp when ts is present', () => { + const input = '{"skill":"review","type":"pattern","key":"ts-preserve","insight":"test","confidence":5,"source":"observed","ts":"2025-06-15T10:00:00Z"}'; + runLog(input); + + const f = findLearningsFile(); + expect(f).not.toBeNull(); + const parsed = JSON.parse(fs.readFileSync(f!, 'utf-8').trim()); + expect(parsed.ts).toBe('2025-06-15T10:00:00Z'); + }); + + test('handles JSON with special characters in insight', () => { + const input = JSON.stringify({ skill: 'review', type: 'pattern', key: 'special-chars', insight: 'Use "quotes" and \\backslashes', confidence: 7, source: 'observed' }); + runLog(input); + + const f = findLearningsFile(); + expect(f).not.toBeNull(); + const parsed = JSON.parse(fs.readFileSync(f!, 'utf-8').trim()); + expect(parsed.insight).toContain('quotes'); + expect(parsed.insight).toContain('backslashes'); + }); + + test('handles JSON with files array field', () => { + const input = JSON.stringify({ skill: 'review', type: 'architecture', key: 'with-files', insight: 'test', confidence: 8, source: 'observed', files: ['src/auth.ts', 'src/db.ts'] }); + runLog(input); + + const f = findLearningsFile(); + expect(f).not.toBeNull(); + const parsed = JSON.parse(fs.readFileSync(f!, 'utf-8').trim()); + expect(parsed.files).toEqual(['src/auth.ts', 'src/db.ts']); + }); +}); + +describe('gstack-learnings-search edge cases', () => { + test('sorts by confidence then recency', () => { + // Two entries: one high confidence old, one lower confidence recent + runLog(JSON.stringify({ skill: 'review', type: 'pattern', key: 'high-conf', insight: 'high confidence entry', confidence: 9, source: 'user-stated', ts: '2026-01-01T00:00:00Z' })); + runLog(JSON.stringify({ skill: 'review', type: 'pattern', key: 'recent', insight: 'recent entry', confidence: 5, source: 'observed', ts: '2026-03-28T00:00:00Z' })); + + const output = runSearch(); + const highIdx = output.indexOf('high-conf'); + const recentIdx = output.indexOf('recent'); + // High confidence should appear first + expect(highIdx).toBeLessThan(recentIdx); + }); + + test('groups output by type', () => { + runLog(JSON.stringify({ skill: 'review', type: 'pattern', key: 'p1', insight: 'a pattern', confidence: 7, source: 'observed' })); + runLog(JSON.stringify({ skill: 'review', type: 'pitfall', key: 'pit1', insight: 'a pitfall', confidence: 7, source: 'observed' })); + + const output = runSearch(); + expect(output).toContain('## Patterns'); + expect(output).toContain('## Pitfalls'); + }); + + test('combined --type and --query filtering', () => { + runLog(JSON.stringify({ skill: 'review', type: 'pattern', key: 'auth-token', insight: 'check token expiry', confidence: 7, source: 'observed' })); + runLog(JSON.stringify({ skill: 'review', type: 'pitfall', key: 'auth-leak', insight: 'auth token in logs', confidence: 7, source: 'observed' })); + runLog(JSON.stringify({ skill: 'review', type: 'pattern', key: 'cache-key', insight: 'cache invalidation', confidence: 7, source: 'observed' })); + + const output = runSearch('--type pattern --query auth'); + expect(output).toContain('auth-token'); + expect(output).not.toContain('auth-leak'); // wrong type + expect(output).not.toContain('cache-key'); // wrong query + }); + + test('entries with missing key or type are skipped', () => { + runLog(JSON.stringify({ skill: 'review', type: 'pattern', key: 'valid', insight: 'valid entry', confidence: 7, source: 'observed' })); + const f = findLearningsFile(); + expect(f).not.toBeNull(); + // Append entries missing key and type + fs.appendFileSync(f!, JSON.stringify({ skill: 'review', type: 'pattern', insight: 'no key', confidence: 7, source: 'observed' }) + '\n'); + fs.appendFileSync(f!, JSON.stringify({ skill: 'review', key: 'no-type', insight: 'no type', confidence: 7, source: 'observed' }) + '\n'); + + const output = runSearch(); + expect(output).toContain('valid'); + expect(output).not.toContain('no key'); + expect(output).not.toContain('no-type'); + }); + + test('confidence decay floors at 0 (never negative)', () => { + // Entry from 1 year ago with confidence 3 — decay would be 12, clamped to 0 + const ts = new Date(Date.now() - 365 * 86400000).toISOString(); + runLog(JSON.stringify({ skill: 'review', type: 'pattern', key: 'ancient', insight: 'very old', confidence: 3, source: 'observed', ts })); + + const output = runSearch(); + expect(output).toContain('confidence: 0/10'); + }); +});