diff --git a/bin/gstack-brain-sync b/bin/gstack-brain-sync index 4fe78e506..99911c036 100755 --- a/bin/gstack-brain-sync +++ b/bin/gstack-brain-sync @@ -204,9 +204,12 @@ sync_active() { return 0 } -# Secret scan over a staged diff file. Pattern families are the -# pre-commit hook's; only the family name ever leaves this function, never the -# matched text. The diff is read as bytes and decoded with replacement, so +# Secret scan over a staged diff file. The first six pattern families are the +# pre-commit hook's; the four after them mirror lib/redact-patterns.ts HIGH +# rules (stripe.secret/restricted, slack.token, npm.token, +# db.url_with_password) with its placeholder exemptions, so a row the +# learnings and timeline writers accept is not held here. Only the family name +# ever leaves this function, never the matched text. The diff is read as bytes and decoded with replacement, so # undecodable bytes cannot crash the scan into a false "clean". # secret_scan gate exit 0 clean, 3 hit (prints the # family names); anything else is @@ -238,8 +241,40 @@ patterns = [ re.IGNORECASE)), ] +structural = [re.compile(p, re.I) for p in (r'^your[_-]', r'^<[^>]*>$', r'^\*+$', r'^x{6,}$')] +substring = [re.compile(p, re.I) for p in (r'example', r'^pass(word)?$', r'^changeme$', r'^redacted', + r'^placeholder', r'^dummy', r'^fake', r'test[_-]?(key|token|secret)')] +url_password_words = {'PASSWORD', 'PASS', 'PASSWD', 'YOUR_PASSWORD', 'DB_PASSWORD', 'MY_PASSWORD', + 'CHANGEME', 'CHANGE_ME', 'PLACEHOLDER', 'REDACTED', 'EXAMPLE'} +interpolated = re.compile(r'^(\$\{.+\}|\$[A-Z_][A-Z0-9_]*)$') +postgres_loopback = re.compile(r'^postgres(?:ql)?://postgres:postgres@(?:localhost|127\.0\.0\.1)(?::\d{1,5})?(?![\w.:@%-])') + +def real_token(m): + span = m.group(1) + return not any(rx.search(span) for rx in structural + substring) + +def real_db_url(m): + pw = m.group(2) + if pw == '' or interpolated.search(pw) or pw in url_password_words: + return False + if any(rx.search(pw) for rx in structural): + return False + return not postgres_loopback.search(m.group(1)) + +patterns = [(name, rx, None) for name, rx in patterns] + [ + ('stripe-live-key', re.compile(r'\b((?:sk|rk)_live_[A-Za-z0-9]{24,})\b'), real_token), + ('slack-token', re.compile(r'\b(xox[baprs]-[A-Za-z0-9-]{10,})\b'), real_token), + ('npm-token', re.compile(r'\b(npm_[A-Za-z0-9]{36})\b'), real_token), + # A diff line of a JSONL file is JSON text: the URL stops at an unescaped + # quote, so it cannot run across into the next field's "@". + ('db-url-password', + re.compile(r'\b((?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis|amqp)://[^:\s/@"]+:((?:[^@\s/"\\]|\\.)+)@[^\s/"]+)'), + real_db_url), +] + def families(text): - return [name for name, rx in patterns if rx.search(text)] + return [name for name, rx, real in patterns + if any(real is None or real(m) for m in rx.finditer(text))] def read(path): with open(path, "rb") as f: diff --git a/bin/gstack-learnings-log b/bin/gstack-learnings-log index 8acaf450a..1e9fb33e9 100755 --- a/bin/gstack-learnings-log +++ b/bin/gstack-learnings-log @@ -56,6 +56,7 @@ trap 'rm -f "$TMPERR"' EXIT set +e VALIDATED=$(printf '%s' "$INPUT" | GSTACK_LIB_DIR="$SCRIPT_DIR/../lib" bun --no-env-file --config="${GSTACK_BUN_CONFIG:-/dev/null}" -e " const { hasInjection } = await import(process.env.GSTACK_LIB_DIR + '/jsonl-store.ts'); +const { scan } = await import(process.env.GSTACK_LIB_DIR + '/redact-engine.ts'); const raw = await Bun.stdin.text(); let j; try { j = JSON.parse(raw); } catch { process.stderr.write('gstack-learnings-log: invalid JSON, skipping\n'); process.exit(1); } @@ -95,6 +96,19 @@ if (j.insight && hasInjection(j.insight)) { process.exit(1); } +// Secrets: the same redact-engine HIGH gate gstack-decision-log applies, over +// every key and string value this row stores (learnings sync across machines). +const stored = []; +(function collect(v) { + if (typeof v === 'string') stored.push(v); + else if (v && typeof v === 'object') for (const [k, x] of Object.entries(v)) { stored.push(k); collect(x); } +})(j); +const high = scan(stored.join('\n')).findings.filter((f) => f.tier === 'HIGH'); +if (high.length > 0) { + process.stderr.write('gstack-learnings-log: learning contains a HIGH-tier secret (' + [...new Set(high.map((f) => f.id))].join(', ') + '), rejected; remove the value and log the learning again, do not log secrets\n'); + process.exit(1); +} + // Inject timestamp if not present if (!j.ts) j.ts = new Date().toISOString(); diff --git a/bin/gstack-timeline-log b/bin/gstack-timeline-log index ab6eabff1..f9564b484 100755 --- a/bin/gstack-timeline-log +++ b/bin/gstack-timeline-log @@ -12,6 +12,11 @@ set -euo pipefail GSTACK_BUN_CONFIG=/dev/null; case "${OSTYPE:-}" in msys*|cygwin*) if command -v cygpath >/dev/null 2>&1; then GSTACK_BUN_CONFIG=NUL; fi ;; esac SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# Windows git-bash (#1950): Bun cannot import a POSIX /c/... path; cygpath -m +# gives C:/... for the redact-engine import below. +case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) command -v cygpath >/dev/null 2>&1 && SCRIPT_DIR="$(cygpath -m "$SCRIPT_DIR")" ;; +esac eval "$("$SCRIPT_DIR/gstack-slug" 2>/dev/null)" . "$(dirname "$0")/gstack-state-root.sh" 2>/dev/null || { echo "$0: cannot resolve the gstack state root: $(dirname "$0")/gstack-state-root.sh is missing. fix: reinstall with ./setup or /gstack-upgrade (docs/state-root.md)" >&2; exit 1; } gstack_state_root_select @@ -20,15 +25,25 @@ mkdir -p "$GSTACK_HOME/projects/$SLUG" INPUT="$1" -# Validate: input must be parseable JSON with required fields. The validator -# prints its verdict; a non-zero exit means Bun itself failed (#3415). An exit -# code cannot carry the verdict: a Bun crash on Windows exits 3. +# Validate: input must be parseable JSON with required fields and no +# redact-engine HIGH secret in any key or string value (the gate +# gstack-decision-log applies). The validator prints its verdict; a non-zero +# exit means Bun itself failed (#3415). An exit code cannot carry the verdict: +# a Bun crash on Windows exits 3. VALIDATE_RC=0 -VALIDATE_OUT=$(printf '%s' "$INPUT" | bun --no-env-file --config="${GSTACK_BUN_CONFIG:-/dev/null}" -e " +VALIDATE_OUT=$(printf '%s' "$INPUT" | GSTACK_LIB_DIR="$SCRIPT_DIR/../lib" bun --no-env-file --config="${GSTACK_BUN_CONFIG:-/dev/null}" -e " const text = await Bun.stdin.text(); let j; try { j = JSON.parse(text); } catch {} - console.log(j && j.skill && j.event ? 'valid' : 'invalid'); + if (!(j && j.skill && j.event)) { console.log('invalid'); process.exit(0); } + const { scan } = await import(process.env.GSTACK_LIB_DIR + '/redact-engine.ts'); + const stored = []; + (function collect(v) { + if (typeof v === 'string') stored.push(v); + else if (v && typeof v === 'object') for (const [k, x] of Object.entries(v)) { stored.push(k); collect(x); } + })(j); + const ids = [...new Set(scan(stored.join('\\n')).findings.filter((f) => f.tier === 'HIGH').map((f) => f.id))]; + console.log(ids.length ? 'secret ' + ids.join(', ') : 'valid'); ") || VALIDATE_RC=$? if [ "$VALIDATE_RC" -ne 0 ]; then echo "gstack-timeline-log: bun exited $VALIDATE_RC before the event was validated; nothing was logged" >&2 @@ -37,6 +52,7 @@ fi case "$VALIDATE_OUT" in valid) ;; invalid) exit 0 ;; # skip silently, non-blocking + secret\ *) echo "gstack-timeline-log: event contains a HIGH-tier secret (${VALIDATE_OUT#secret }); nothing was logged. Remove the value and log the event again; do not log secrets" >&2; exit 1 ;; *) echo "gstack-timeline-log: bun printed no verdict while validating the event; nothing was logged" >&2; exit 1 ;; esac diff --git a/changelog.d/stop-secrets-in-learnings-timeline-and-brain-sync.md b/changelog.d/stop-secrets-in-learnings-timeline-and-brain-sync.md new file mode 100644 index 000000000..f55f3c6a0 --- /dev/null +++ b/changelog.d/stop-secrets-in-learnings-timeline-and-brain-sync.md @@ -0,0 +1,4 @@ +--- +important: security +--- +- **Learnings, timeline events and brain sync no longer store or push Stripe live keys, Slack tokens, npm tokens or database URLs with a real password.** `gstack-learnings-log` and `gstack-timeline-log` now refuse any row whose keys or string values hold a HIGH-tier secret by `lib/redact-engine`, the same gate `gstack-decision-log` already applied: the writer exits 1, names the rule (for example `db.url_with_password`) but never the value, and writes nothing. Before, `gstack-learnings-log` stored `DATABASE_URL=postgres://app:@…` and exited 0, and `gstack-timeline-log` had no secret check at all. Brain sync's push-time scanner, which only knew six families, now also holds a file with a Stripe live or restricted key, a Slack token, an npm token or a database URL with a password, using redact-engine's placeholder rules so `postgres://app:${DB_PASSWORD}@…`, `PASSWORD` placeholders and the local `postgres:postgres@localhost` default still sync. Ordinary rows that mention tokens or passwords in prose still write and sync. diff --git a/test/brain-sync.test.ts b/test/brain-sync.test.ts index b6f7882c0..36b5e69b6 100644 --- a/test/brain-sync.test.ts +++ b/test/brain-sync.test.ts @@ -379,6 +379,11 @@ describe('gstack-brain-sync secret scan', () => { ['pem-block', '-----BEGIN PRIVATE KEY-----'], ['jwt', 'eyJ0eXAiOiJKV1QiLCJh.eyJzdWIiOiIxMjM0NTY3.SflKxwRJSMeKKF30oGTbU'], ['bearer-json', '"authorization":"Bearer abcdef1234567890abcdef1234567890"'], + // GST-2196: redact-engine rates these HIGH; the scanner used to push them. + ['stripe-live-key', 'sk_' + 'live_' + 'Q7d2Lm9Xr4Tb8Vn1Kp3Wz6Hc'], + ['slack-token', 'xox' + 'b-' + '2384710293-4829103847-Zr8Qm2Lx7Vn4Tb'], + ['db-url-password', 'DATABASE_URL=postgres://app:' + 'Vq8zR2' + 'mLp7@db.internal.example.com:5432/prod'], + ['npm-token', 'npm_' + 'a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6q7R8'], ]; for (const [name, content] of SECRETS) { @@ -406,6 +411,27 @@ describe('gstack-brain-sync secret scan', () => { }); } + test('prose about tokens and passwords and placeholder database urls still sync', () => { + run(['gstack-artifacts-init', '--remote', bareRemote]); + run(['gstack-config', 'set', 'artifacts_sync_mode', 'full']); + fs.mkdirSync(path.join(tmpHome, 'projects', 'p'), { recursive: true }); + const rows = [ + { insight: 'Rotate the npm token before release; the Slack bot token and the password live in the vault' }, + { insight: 'DATABASE_URL=postgres://app:PASSWORD@host/db or postgres://app:${DB_PASSWORD}@db/prod' }, + { insight: 'local dev uses postgres://postgres:postgres@localhost:5432/app' }, + { insight: 'cache is redis://localhost:6379', owner: 'ops@team.example.com' }, + { insight: 'test keys look like sk_' + 'live_' + 'EXAMPLEEXAMPLEEXAMPLEEXAMPLE' }, + ]; + fs.writeFileSync(path.join(tmpHome, 'projects/p/learnings.jsonl'), + rows.map((r) => JSON.stringify(r)).join('\n') + '\n'); + run(['gstack-brain-enqueue', 'projects/p/learnings.jsonl']); + expect(run(['gstack-brain-sync', '--once']).status).toBe(0); + const status = JSON.parse(fs.readFileSync(path.join(tmpHome, '.brain-sync-status.json'), 'utf-8')); + expect(status.held_count ?? 0).toBe(0); + const log = spawnSync('git', ['--git-dir=' + bareRemote, 'log', '--oneline'], { encoding: 'utf-8', timeout: 30_000 }); + expect(log.stdout).toMatch(/sync: 1 file/); + }); + test('--skip-file unblocks specific file', () => { run(['gstack-artifacts-init', '--remote', bareRemote]); run(['gstack-config', 'set', 'artifacts_sync_mode', 'full']); diff --git a/test/gstack-skill-start.test.ts b/test/gstack-skill-start.test.ts index 79e072629..1a8cdf227 100644 --- a/test/gstack-skill-start.test.ts +++ b/test/gstack-skill-start.test.ts @@ -458,6 +458,7 @@ describe('gstack-skill-start behavior', () => { const log = path.join(tmp, 'egress.log'); const sinks = ['gstack-telemetry-log', 'gstack-brain-sync', 'gstack-update-check']; fs.mkdirSync(shadow); fs.mkdirSync(gh); + fs.symlinkSync(path.join(ROOT, 'lib'), path.join(tmp, 'lib')); for (const name of fs.readdirSync(path.join(ROOT, 'bin'))) { if (!sinks.includes(name)) fs.symlinkSync(path.join(ROOT, 'bin', name), path.join(shadow, name)); } diff --git a/test/learnings.test.ts b/test/learnings.test.ts index 6979b8d17..200ce5a31 100644 --- a/test/learnings.test.ts +++ b/test/learnings.test.ts @@ -345,3 +345,48 @@ describe('G3: learnings-log temp file and TMPDIR', () => { expect(stderr).toContain('TMPDIR'); }); }); + +// The four secret families brain sync's scanner used to miss (GST-2196). +// Built by concatenation so this file never holds a scannable literal. +const SECRET_FAMILIES: [string, string][] = [ + ['stripe live key', 'sk_' + 'live_' + 'Q7d2Lm9Xr4Tb8Vn1Kp3Wz6Hc'], + ['slack bot token', 'xox' + 'b-' + '2384710293-4829103847-Zr8Qm2Lx7Vn4Tb'], + ['database url with a password', 'postgres://app:' + 'Vq8zR2' + 'mLp7@db.internal.example.com:5432/prod'], + ['npm token', 'npm_' + 'a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6q7R8'], +]; + +describe('gstack-learnings-log refuses HIGH-tier secrets', () => { + for (const [name, secret] of SECRET_FAMILIES) { + test(`rejects a ${name} in the insight and writes nothing`, () => { + const input = JSON.stringify({ skill: 'review', type: 'operational', key: 'env-setup', insight: `DATABASE_URL and keys: ${secret}`, confidence: 7, source: 'observed' }); + const result = runLog(input, { expectFail: true }); + expect(result.exitCode).not.toBe(0); + expect(result.stdout).toContain('HIGH-tier secret'); + expect(result.stdout).not.toContain(secret); + expect(findLearningsFile()).toBeNull(); + }); + } + + test('scans every stored field, not just the insight', () => { + const input = JSON.stringify({ skill: 'review', type: 'tool', key: 'npm-publish', insight: 'publish needs a token', confidence: 6, source: 'observed', files: [SECRET_FAMILIES[3][1]] }); + const result = runLog(input, { expectFail: true }); + expect(result.exitCode).not.toBe(0); + expect(findLearningsFile()).toBeNull(); + }); + + test('prose about tokens and passwords still writes', () => { + const insight = 'Rotate the npm token before release; the password field and the Slack bot token live in the vault, never in the repo'; + runLog(JSON.stringify({ skill: 'review', type: 'operational', key: 'secret-handling', insight, confidence: 8, source: 'observed' })); + const f = findLearningsFile(); + expect(f).not.toBeNull(); + expect(JSON.parse(fs.readFileSync(f!, 'utf-8').trim()).insight).toBe(insight); + }); + + test('placeholder database urls still write', () => { + const insight = 'Set DATABASE_URL=postgres://app:PASSWORD@host/db, or postgres://app:${DB_PASSWORD}@db/prod in CI'; + runLog(JSON.stringify({ skill: 'review', type: 'operational', key: 'db-url-shape', insight, confidence: 8, source: 'observed' })); + const f = findLearningsFile(); + expect(f).not.toBeNull(); + expect(JSON.parse(fs.readFileSync(f!, 'utf-8').trim()).insight).toBe(insight); + }); +}); diff --git a/test/skill-e2e-bws.test.ts b/test/skill-e2e-bws.test.ts index ad33e39a1..89744ab92 100644 --- a/test/skill-e2e-bws.test.ts +++ b/test/skill-e2e-bws.test.ts @@ -223,13 +223,12 @@ Report the exact output — either "READY: " or "NEEDS_SETUP".`, fs.copyFileSync(path.join(ROOT, 'bin', script), path.join(binDir, script)); fs.chmodSync(path.join(binDir, script), 0o755); } - // gstack-learnings-log imports $SCRIPT_DIR/../lib/jsonl-store.ts (shared - // injection patterns, since v1.57.5.0) — a real install always ships bin/ - // and lib/ together, so the fixture must too. Without it the bin exits 1 - // before writing anything and the test fails on every attempt. - const libDir = path.join(opDir, 'lib'); - fs.mkdirSync(libDir, { recursive: true }); - fs.copyFileSync(path.join(ROOT, 'lib', 'jsonl-store.ts'), path.join(libDir, 'jsonl-store.ts')); + // gstack-learnings-log imports $SCRIPT_DIR/../lib/jsonl-store.ts and + // lib/redact-engine.ts (which imports further lib modules) — a real install + // always ships bin/ and lib/ together, so the fixture must too. Without it + // the bin exits 1 before writing anything and the test fails on every + // attempt. A copy, not a symlink: the agent runs inside this directory. + fs.cpSync(path.join(ROOT, 'lib'), path.join(opDir, 'lib'), { recursive: true }); // gstack-learnings-log will create the project dir automatically via gstack-slug diff --git a/test/skill-e2e-context-skills.test.ts b/test/skill-e2e-context-skills.test.ts index c76697a4a..18d9c7613 100644 --- a/test/skill-e2e-context-skills.test.ts +++ b/test/skill-e2e-context-skills.test.ts @@ -72,6 +72,9 @@ function setupWorkdir(suffix: string): { workDir: string; gstackHome: string; sl fs.chmodSync(path.join(binDir, script), 0o755); } } + // gstack-timeline-log and gstack-learnings-log import ../lib/redact-engine.ts + // from beside bin/. A copy, not a symlink: the agent runs inside workDir. + fs.cpSync(path.join(ROOT, 'lib'), path.join(workDir, 'lib'), { recursive: true }); // Routing CLAUDE.md: the ## Skill routing section gstack ships. fs.writeFileSync(path.join(workDir, 'CLAUDE.md'), `# Project Instructions diff --git a/test/skill-e2e-session-intelligence.test.ts b/test/skill-e2e-session-intelligence.test.ts index f7c058265..318aa9e53 100644 --- a/test/skill-e2e-session-intelligence.test.ts +++ b/test/skill-e2e-session-intelligence.test.ts @@ -44,6 +44,9 @@ describeIfSelected('Session Intelligence E2E', [ // gstack-skill-start and gstack-context-recovery, which call further bin // scripts, so ./bin/ is the whole runtime bin directory. fs.symlinkSync(path.join(ROOT, 'bin'), path.join(workDir, 'bin'), 'dir'); + // gstack-timeline-log and gstack-learnings-log import ../lib/redact-engine.ts + // from beside ./bin/. A copy, not a symlink: the agent runs inside workDir. + fs.cpSync(path.join(ROOT, 'lib'), path.join(workDir, 'lib'), { recursive: true }); // Compute slug (same logic as gstack-slug without git remote) slug = path.basename(workDir).replace(/[^a-zA-Z0-9._-]/g, ''); diff --git a/test/skill-e2e-skillify.test.ts b/test/skill-e2e-skillify.test.ts index 66b46b5d8..d5847f970 100644 --- a/test/skill-e2e-skillify.test.ts +++ b/test/skill-e2e-skillify.test.ts @@ -109,6 +109,9 @@ function setupSkillifyWorkdir(suffix: string, installSkills: string[] = ['scrape fs.chmodSync(path.join(binDir, script), 0o755); } } + // gstack-timeline-log and gstack-learnings-log import ../lib/redact-engine.ts + // from beside bin/. A copy, not a symlink: the agent runs inside workDir. + fs.cpSync(path.join(ROOT, 'lib'), path.join(workDir, 'lib'), { recursive: true }); fs.writeFileSync(path.join(workDir, 'CLAUDE.md'), `# Project Instructions diff --git a/test/skill-end-completion-durability.test.ts b/test/skill-end-completion-durability.test.ts index 7ea172422..994e12806 100644 --- a/test/skill-end-completion-durability.test.ts +++ b/test/skill-end-completion-durability.test.ts @@ -141,6 +141,7 @@ describe('skill-end completion durability', () => { test('logger returns only after a delayed enqueue makes its spool record durable', () => { const shadow = path.join(fixture, 'bin'); fs.mkdirSync(shadow); + fs.symlinkSync(path.resolve(BIN, '../lib'), path.join(fixture, 'lib')); for (const f of fs.readdirSync(BIN)) { fs.symlinkSync(path.join(BIN, f), path.join(shadow, f)); } for (const f of ['gstack-timeline-log', 'gstack-brain-enqueue']) { fs.unlinkSync(path.join(shadow, f)); } fs.copyFileSync(path.join(BIN, 'gstack-timeline-log'), path.join(shadow, 'gstack-timeline-log')); diff --git a/test/timeline.test.ts b/test/timeline.test.ts index d08c058a6..0f7bc9085 100644 --- a/test/timeline.test.ts +++ b/test/timeline.test.ts @@ -283,3 +283,33 @@ describe('gstack-timeline-read', () => { } } }); + +// The four secret families brain sync's scanner used to miss (GST-2196). +// Built by concatenation so this file never holds a scannable literal. +const SECRET_FAMILIES: [string, string][] = [ + ['stripe live key', 'sk_' + 'live_' + 'Q7d2Lm9Xr4Tb8Vn1Kp3Wz6Hc'], + ['slack bot token', 'xox' + 'b-' + '2384710293-4829103847-Zr8Qm2Lx7Vn4Tb'], + ['database url with a password', 'postgres://app:' + 'Vq8zR2' + 'mLp7@db.internal.example.com:5432/prod'], + ['npm token', 'npm_' + 'a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6q7R8'], +]; + +describe('gstack-timeline-log refuses HIGH-tier secrets', () => { + for (const [name, secret] of SECRET_FAMILIES) { + test(`rejects a ${name} in any field and writes nothing`, () => { + const input = JSON.stringify({ skill: 'ship', event: 'completed', branch: 'main', outcome: `deploy failed: ${secret}` }); + const result = runLog(input, { expectFail: true }); + expect(result.exitCode).not.toBe(0); + expect(result.stdout).toContain('HIGH-tier secret'); + expect(result.stdout).not.toContain(secret); + expect(findTimelineFile()).toBeNull(); + }); + } + + test('ordinary events with token and password words still write', () => { + const input = '{"skill":"review","event":"completed","branch":"fix-token-refresh","outcome":"password reset flow reviewed; token rotation ok"}'; + expect(runLog(input).exitCode).toBe(0); + const f = findTimelineFile(); + expect(f).not.toBeNull(); + expect(JSON.parse(fs.readFileSync(f!, 'utf-8').trim()).outcome).toBe('password reset flow reviewed; token rotation ok'); + }); +});