From 0933ab074a779f688e502db6d7ddf59203ebc7dd Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Fri, 24 Apr 2026 00:09:04 -0700 Subject: [PATCH] test: secret-sink harness positive controls + real-bin negative controls 11 tests. Positive controls deliberately leak a seed in every covered channel (stdout, stderr, a file under $HOME, the telemetry JSONL path, base64-encoded, first-12-char prefix) and assert the harness catches each one. Without these, a harness that silently under-reports would look identical to a harness that works. Negative controls run real setup-gbrain bins with distinctive seeds: - supabase-verify rejects a mysql:// URL and a direct-connection URL, password never appears in any captured channel - lib.sh read_secret_to_env reads piped stdin, emits only the length, seed value stays invisible - supabase-provision on an auth-failure path fails fast without leaking the PAT to any channel Covers D21 #5 leak harness + uses it to validate D3-eng, D10, D11 discipline end-to-end on the already-shipped bins. Co-Authored-By: Claude Opus 4.7 (1M context) --- test/secret-sink-harness.test.ts | 216 +++++++++++++++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 test/secret-sink-harness.test.ts diff --git a/test/secret-sink-harness.test.ts b/test/secret-sink-harness.test.ts new file mode 100644 index 00000000..6ea85787 --- /dev/null +++ b/test/secret-sink-harness.test.ts @@ -0,0 +1,216 @@ +/** + * Tests for the secret-sink test harness (D21 #5). + * + * Positive controls: deliberately leak a seed in every covered channel and + * assert the harness catches it. A harness that silently under-reports is + * worse than no harness — these tests are the quality gate. + * + * Negative controls: run real setup-gbrain bins with known secrets; no + * leaks should appear. + */ + +import { describe, test, expect } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { runWithSecretSink } from './helpers/secret-sink-harness'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const LEAK_BIN_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'leak-bins-')); + +// Build a disposable bash script that leaks in a specific way. Returns +// path to the executable. We don't bother cleaning these up per-test — +// they live under a tmpdir that's fine to linger between tests. +function makeLeakyBin(name: string, body: string): string { + const p = path.join(LEAK_BIN_DIR, name); + fs.writeFileSync(p, `#!/bin/bash\nset -euo pipefail\n${body}\n`, { mode: 0o755 }); + return p; +} + +describe('secret-sink-harness — positive controls', () => { + test('catches a seed echoed to stdout', async () => { + const bin = makeLeakyBin( + 'leak-stdout', + 'echo "config contains: $LEAK_SEED"' + ); + const seed = 'my-secret-password-12345'; + const r = await runWithSecretSink({ + bin, + args: [], + seeds: [seed], + env: { LEAK_SEED: seed }, + }); + expect(r.leaks.length).toBeGreaterThan(0); + const stdoutLeaks = r.leaks.filter((l) => l.channel === 'stdout'); + expect(stdoutLeaks.length).toBeGreaterThan(0); + expect(stdoutLeaks.some((l) => l.matchType === 'exact')).toBe(true); + }); + + test('catches a seed echoed to stderr', async () => { + const bin = makeLeakyBin( + 'leak-stderr', + 'echo "leaked: $LEAK_SEED" >&2' + ); + const seed = 'another-secret-value-67890'; + const r = await runWithSecretSink({ + bin, + args: [], + seeds: [seed], + env: { LEAK_SEED: seed }, + }); + expect(r.leaks.some((l) => l.channel === 'stderr')).toBe(true); + }); + + test('catches a seed written to a file under $HOME', async () => { + const bin = makeLeakyBin( + 'leak-file', + 'mkdir -p "$HOME/.gstack" && echo "seed: $LEAK_SEED" > "$HOME/.gstack/debug.log"' + ); + const seed = 'file-leaked-secret-value-xyz'; + const r = await runWithSecretSink({ + bin, + args: [], + seeds: [seed], + env: { LEAK_SEED: seed }, + }); + const fileLeaks = r.leaks.filter((l) => l.channel === 'file'); + expect(fileLeaks.length).toBeGreaterThan(0); + expect(fileLeaks[0].where).toBe('.gstack/debug.log'); + }); + + test('catches a seed leaked into the telemetry channel', async () => { + const bin = makeLeakyBin( + 'leak-telemetry', + 'mkdir -p "$HOME/.gstack/analytics" && ' + + 'echo "{\\"event\\":\\"x\\",\\"leaked_secret\\":\\"$LEAK_SEED\\"}" ' + + ' >> "$HOME/.gstack/analytics/skill-usage.jsonl"' + ); + const seed = 'telemetry-leaked-abc123xyz'; + const r = await runWithSecretSink({ + bin, + args: [], + seeds: [seed], + env: { LEAK_SEED: seed }, + }); + const telemetryLeaks = r.leaks.filter((l) => l.channel === 'telemetry'); + expect(telemetryLeaks.length).toBeGreaterThan(0); + expect(telemetryLeaks[0].where).toContain('analytics/'); + }); + + test('catches a seed leaked in base64-encoded form (auth header pattern)', async () => { + // printf (not echo) so no trailing newline — matches how real auth + // headers encode: base64(seed) exactly, not base64(seed + "\n"). + const bin = makeLeakyBin( + 'leak-base64', + 'printf "%s" "$LEAK_SEED" | base64' + ); + const seed = 'base64-leaked-long-enough-secret'; + const r = await runWithSecretSink({ + bin, + args: [], + seeds: [seed], + env: { LEAK_SEED: seed }, + }); + expect(r.leaks.some((l) => l.matchType === 'base64')).toBe(true); + }); + + test('catches a first-12-char prefix leak (the "I only logged a portion" pattern)', async () => { + const bin = makeLeakyBin( + 'leak-prefix', + 'prefix="${LEAK_SEED:0:12}"; echo "debug prefix: $prefix"' + ); + const seed = 'prefix-leaked-0123456789abcdef'; + const r = await runWithSecretSink({ + bin, + args: [], + seeds: [seed], + env: { LEAK_SEED: seed }, + }); + expect(r.leaks.some((l) => l.matchType === 'prefix-12')).toBe(true); + }); + + test('clean run with no leak returns an empty leaks array', async () => { + const bin = makeLeakyBin('clean', 'echo "no secret here"'); + const r = await runWithSecretSink({ + bin, + args: [], + seeds: ['never-emitted-seed-xyz-987'], + }); + expect(r.leaks).toEqual([]); + }); +}); + +describe('secret-sink-harness — real bins (negative controls)', () => { + test('supabase-verify does not leak a URL password on reject', async () => { + const bin = path.join(ROOT, 'bin', 'gstack-gbrain-supabase-verify'); + const seedPassword = 'extremely-distinctive-password-abc-xyz-987'; + // Use a URL that will be REJECTED (wrong scheme) so all error paths run + const leakyUrl = `mysql://user:${seedPassword}@host:6543/db`; + const r = await runWithSecretSink({ + bin, + args: [leakyUrl], + seeds: [seedPassword], + }); + // Status 2 — rejected as expected + expect(r.status).toBe(2); + // No leaks in any channel + expect(r.leaks).toEqual([]); + }); + + test('supabase-verify does not leak on direct-connection rejection path', async () => { + const bin = path.join(ROOT, 'bin', 'gstack-gbrain-supabase-verify'); + const seedPassword = 'another-distinctive-secret-for-direct-conn'; + const leakyUrl = `postgresql://postgres:${seedPassword}@db.abcdef.supabase.co:5432/postgres`; + const r = await runWithSecretSink({ + bin, + args: [leakyUrl], + seeds: [seedPassword], + }); + expect(r.status).toBe(3); + expect(r.leaks).toEqual([]); + }); + + test('lib.sh read_secret_to_env does not leak stdin via captured channels', async () => { + const seed = 'piped-secret-that-should-stay-invisible-zzz'; + // Wrapper script: source lib.sh, read secret, echo only its length. + const lib = path.join(ROOT, 'bin', 'gstack-gbrain-lib.sh'); + const bin = makeLeakyBin( + 'read-secret-wrapper', + `. "${lib}"\nread_secret_to_env MY_SECRET "Prompt: "\necho "len=\${#MY_SECRET}"` + ); + const r = await runWithSecretSink({ + bin, + args: [], + seeds: [seed], + stdin: seed, + }); + expect(r.status).toBe(0); + // The length is visible (43) but the value is not + expect(r.stdout).toContain(`len=${seed.length}`); + expect(r.leaks).toEqual([]); + }); + + test('supabase-provision does not leak a PAT on auth-failure path', async () => { + const bin = path.join(ROOT, 'bin', 'gstack-gbrain-supabase-provision'); + const seedPat = 'sbp_very_distinctive_pat_seed_abc_xyz_1234567890'; + // With no SUPABASE_API_BASE override, the bin tries the real API URL. + // We want to avoid real network calls — point at a bogus URL that + // immediately fails with curl. The bin should exit with an error + // WITHOUT leaking the PAT to any channel. + const r = await runWithSecretSink({ + bin, + args: ['list-orgs'], + seeds: [seedPat], + env: { + SUPABASE_ACCESS_TOKEN: seedPat, + // Nonexistent port — curl fails fast. + SUPABASE_API_BASE: 'http://127.0.0.1:1', + }, + timeoutMs: 30_000, // curl retries with backoff — give it room to exit + }); + // Expect a non-zero exit (network failure, exit 8 per the bin's + // retry-exhausted path) + expect(r.status).not.toBe(0); + expect(r.leaks).toEqual([]); + }, 60_000); +});