diff --git a/package.json b/package.json index abf2fa55..5edec441 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,8 @@ "eval:summary": "bun run scripts/eval-summary.ts", "eval:watch": "bun run scripts/eval-watch.ts", "eval:select": "bun run scripts/eval-select.ts", - "analytics": "bun run scripts/analytics.ts" + "analytics": "bun run scripts/analytics.ts", + "test:audit": "bun test test/audit-compliance.test.ts" }, "dependencies": { "diff": "^7.0.0", diff --git a/test/audit-compliance.test.ts b/test/audit-compliance.test.ts new file mode 100644 index 00000000..f8f7e46f --- /dev/null +++ b/test/audit-compliance.test.ts @@ -0,0 +1,88 @@ +import { describe, test, expect } from 'bun:test'; +import { readFileSync, readdirSync, existsSync } from 'fs'; +import { join } from 'path'; + +const ROOT = join(import.meta.dir, '..'); + +function getAllSkillMds(): Array<{ name: string; content: string }> { + const results: Array<{ name: string; content: string }> = []; + const rootPath = join(ROOT, 'SKILL.md'); + if (existsSync(rootPath)) { + results.push({ name: 'root', content: readFileSync(rootPath, 'utf-8') }); + } + for (const entry of readdirSync(ROOT, { withFileTypes: true })) { + if (!entry.isDirectory() || entry.name.startsWith('.') || entry.name === 'node_modules') continue; + const skillPath = join(ROOT, entry.name, 'SKILL.md'); + if (existsSync(skillPath)) { + results.push({ name: entry.name, content: readFileSync(skillPath, 'utf-8') }); + } + } + return results; +} + +describe('Audit compliance', () => { + // Fix 1: W007 — No hardcoded credentials in documentation + test('no hardcoded credential patterns in SKILL.md.tmpl', () => { + const tmpl = readFileSync(join(ROOT, 'SKILL.md.tmpl'), 'utf-8'); + expect(tmpl).not.toContain('"password123"'); + expect(tmpl).not.toContain('"test@example.com"'); + expect(tmpl).not.toContain('"test@test.com"'); + expect(tmpl).toContain('$TEST_EMAIL'); + expect(tmpl).toContain('$TEST_PASSWORD'); + }); + + // Fix 2: Conditional telemetry — binary calls wrapped with existence check + test('preamble telemetry calls are conditional on _TEL and binary existence', () => { + const preamble = readFileSync(join(ROOT, 'scripts/resolvers/preamble.ts'), 'utf-8'); + // Pending finalization must check _TEL and binary existence + expect(preamble).toContain('_TEL" != "off"'); + expect(preamble).toContain('-x '); + expect(preamble).toContain('gstack-telemetry-log'); + // End-of-skill telemetry must also be conditional + const completionIdx = preamble.indexOf('Telemetry (run last)'); + expect(completionIdx).toBeGreaterThan(-1); + const completionSection = preamble.slice(completionIdx); + expect(completionSection).toContain('_TEL" != "off"'); + }); + + // Fix 3: W012 — Bun install is version-pinned + test('bun install commands use version pinning', () => { + const browseResolver = readFileSync(join(ROOT, 'scripts/resolvers/browse.ts'), 'utf-8'); + expect(browseResolver).toContain('BUN_VERSION'); + // Should not have unpinned curl|bash (without BUN_VERSION on same line) + const lines = browseResolver.split('\n'); + for (const line of lines) { + if (line.includes('bun.sh/install') && line.includes('bash') && !line.includes('BUN_VERSION') && !line.includes('command -v')) { + throw new Error(`Unpinned bun install found: ${line.trim()}`); + } + } + }); + + // Fix 4: W011 — Untrusted content warning in command reference + test('command reference includes untrusted content warning after Navigation', () => { + const rootSkill = readFileSync(join(ROOT, 'SKILL.md'), 'utf-8'); + const navIdx = rootSkill.indexOf('### Navigation'); + const readingIdx = rootSkill.indexOf('### Reading'); + expect(navIdx).toBeGreaterThan(-1); + expect(readingIdx).toBeGreaterThan(navIdx); + const between = rootSkill.slice(navIdx, readingIdx); + expect(between.toLowerCase()).toContain('untrusted'); + }); + + // Fix 5: Data flow documentation in review.ts + test('review.ts has data flow documentation', () => { + const review = readFileSync(join(ROOT, 'scripts/resolvers/review.ts'), 'utf-8'); + expect(review).toContain('Data sent'); + expect(review).toContain('Data NOT sent'); + }); + + // Fix 2+6: All generated SKILL.md files with telemetry are conditional + test('all generated SKILL.md files with telemetry calls use conditional pattern', () => { + const skills = getAllSkillMds(); + for (const { name, content } of skills) { + if (content.includes('gstack-telemetry-log')) { + expect(content).toContain('_TEL" != "off"'); + } + } + }); +});