mirror of
https://github.com/garrytan/gstack.git
synced 2026-05-05 05:05:08 +02:00
test: add audit compliance regression tests
6 tests enforce Snyk/Socket fixes stay in place: no hardcoded creds, conditional telemetry, version-pinned bun, untrusted content warning, data flow docs, all SKILL.md telemetry conditional.
This commit is contained in:
+2
-1
@@ -31,7 +31,8 @@
|
||||
"eval:summary": "bun run scripts/eval-summary.ts",
|
||||
"eval:watch": "bun run scripts/eval-watch.ts",
|
||||
"eval:select": "bun run scripts/eval-select.ts",
|
||||
"analytics": "bun run scripts/analytics.ts"
|
||||
"analytics": "bun run scripts/analytics.ts",
|
||||
"test:audit": "bun test test/audit-compliance.test.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"diff": "^7.0.0",
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { readFileSync, readdirSync, existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
const ROOT = join(import.meta.dir, '..');
|
||||
|
||||
function getAllSkillMds(): Array<{ name: string; content: string }> {
|
||||
const results: Array<{ name: string; content: string }> = [];
|
||||
const rootPath = join(ROOT, 'SKILL.md');
|
||||
if (existsSync(rootPath)) {
|
||||
results.push({ name: 'root', content: readFileSync(rootPath, 'utf-8') });
|
||||
}
|
||||
for (const entry of readdirSync(ROOT, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory() || entry.name.startsWith('.') || entry.name === 'node_modules') continue;
|
||||
const skillPath = join(ROOT, entry.name, 'SKILL.md');
|
||||
if (existsSync(skillPath)) {
|
||||
results.push({ name: entry.name, content: readFileSync(skillPath, 'utf-8') });
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
describe('Audit compliance', () => {
|
||||
// Fix 1: W007 — No hardcoded credentials in documentation
|
||||
test('no hardcoded credential patterns in SKILL.md.tmpl', () => {
|
||||
const tmpl = readFileSync(join(ROOT, 'SKILL.md.tmpl'), 'utf-8');
|
||||
expect(tmpl).not.toContain('"password123"');
|
||||
expect(tmpl).not.toContain('"test@example.com"');
|
||||
expect(tmpl).not.toContain('"test@test.com"');
|
||||
expect(tmpl).toContain('$TEST_EMAIL');
|
||||
expect(tmpl).toContain('$TEST_PASSWORD');
|
||||
});
|
||||
|
||||
// Fix 2: Conditional telemetry — binary calls wrapped with existence check
|
||||
test('preamble telemetry calls are conditional on _TEL and binary existence', () => {
|
||||
const preamble = readFileSync(join(ROOT, 'scripts/resolvers/preamble.ts'), 'utf-8');
|
||||
// Pending finalization must check _TEL and binary existence
|
||||
expect(preamble).toContain('_TEL" != "off"');
|
||||
expect(preamble).toContain('-x ');
|
||||
expect(preamble).toContain('gstack-telemetry-log');
|
||||
// End-of-skill telemetry must also be conditional
|
||||
const completionIdx = preamble.indexOf('Telemetry (run last)');
|
||||
expect(completionIdx).toBeGreaterThan(-1);
|
||||
const completionSection = preamble.slice(completionIdx);
|
||||
expect(completionSection).toContain('_TEL" != "off"');
|
||||
});
|
||||
|
||||
// Fix 3: W012 — Bun install is version-pinned
|
||||
test('bun install commands use version pinning', () => {
|
||||
const browseResolver = readFileSync(join(ROOT, 'scripts/resolvers/browse.ts'), 'utf-8');
|
||||
expect(browseResolver).toContain('BUN_VERSION');
|
||||
// Should not have unpinned curl|bash (without BUN_VERSION on same line)
|
||||
const lines = browseResolver.split('\n');
|
||||
for (const line of lines) {
|
||||
if (line.includes('bun.sh/install') && line.includes('bash') && !line.includes('BUN_VERSION') && !line.includes('command -v')) {
|
||||
throw new Error(`Unpinned bun install found: ${line.trim()}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Fix 4: W011 — Untrusted content warning in command reference
|
||||
test('command reference includes untrusted content warning after Navigation', () => {
|
||||
const rootSkill = readFileSync(join(ROOT, 'SKILL.md'), 'utf-8');
|
||||
const navIdx = rootSkill.indexOf('### Navigation');
|
||||
const readingIdx = rootSkill.indexOf('### Reading');
|
||||
expect(navIdx).toBeGreaterThan(-1);
|
||||
expect(readingIdx).toBeGreaterThan(navIdx);
|
||||
const between = rootSkill.slice(navIdx, readingIdx);
|
||||
expect(between.toLowerCase()).toContain('untrusted');
|
||||
});
|
||||
|
||||
// Fix 5: Data flow documentation in review.ts
|
||||
test('review.ts has data flow documentation', () => {
|
||||
const review = readFileSync(join(ROOT, 'scripts/resolvers/review.ts'), 'utf-8');
|
||||
expect(review).toContain('Data sent');
|
||||
expect(review).toContain('Data NOT sent');
|
||||
});
|
||||
|
||||
// Fix 2+6: All generated SKILL.md files with telemetry are conditional
|
||||
test('all generated SKILL.md files with telemetry calls use conditional pattern', () => {
|
||||
const skills = getAllSkillMds();
|
||||
for (const { name, content } of skills) {
|
||||
if (content.includes('gstack-telemetry-log')) {
|
||||
expect(content).toContain('_TEL" != "off"');
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user