Stop learnings, timeline events and brain sync from storing Stripe, Slack, npm and database secrets (GST-2196) (#4247)

* Add failing tests: four HIGH secret families pass brain sync and the learnings and timeline writers (GST-2196)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Refuse HIGH-tier secrets in learnings and timeline rows and hold four more secret families in brain sync (GST-2196)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Give the shadow-bin test fixtures the lib directory the timeline logger now imports (GST-2196)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Copy lib/ into the four agent test fixtures that run the timeline and learnings loggers (GST-2196)

The loggers now import lib/redact-engine.ts, so the bws operational-learning, session-intelligence, context-skills and skillify E2E fixtures need lib/ beside their bin/. A copy, not a symlink, because a live agent runs inside each fixture.

Review-Finding: p1/specialist-batched#1 reader=specialist-batched
Review-Finding: p1/specialist-batched#2 reader=specialist-batched
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

bin/gstack-brain-sync

@@ -204,9 +204,12 @@ sync_active() {
   return 0
 }
 
-# Secret scan over a staged diff file. Pattern families are the
-# pre-commit hook's; only the family name ever leaves this function, never the
-# matched text. The diff is read as bytes and decoded with replacement, so
+# Secret scan over a staged diff file. The first six pattern families are the
+# pre-commit hook's; the four after them mirror lib/redact-patterns.ts HIGH
+# rules (stripe.secret/restricted, slack.token, npm.token,
+# db.url_with_password) with its placeholder exemptions, so a row the
+# learnings and timeline writers accept is not held here. Only the family name
+# ever leaves this function, never the matched text. The diff is read as bytes and decoded with replacement, so
 # undecodable bytes cannot crash the scan into a false "clean".
 #   secret_scan gate <diff>                     exit 0 clean, 3 hit (prints the
 #                                               family names); anything else is
@@ -238,8 +241,40 @@ patterns = [
                 re.IGNORECASE)),
 ]
 
+structural = [re.compile(p, re.I) for p in (r'^your[_-]', r'^<[^>]*>$', r'^\*+$', r'^x{6,}$')]
+substring = [re.compile(p, re.I) for p in (r'example', r'^pass(word)?$', r'^changeme$', r'^redacted',
+                                            r'^placeholder', r'^dummy', r'^fake', r'test[_-]?(key|token|secret)')]
+url_password_words = {'PASSWORD', 'PASS', 'PASSWD', 'YOUR_PASSWORD', 'DB_PASSWORD', 'MY_PASSWORD',
+                      'CHANGEME', 'CHANGE_ME', 'PLACEHOLDER', 'REDACTED', 'EXAMPLE'}
+interpolated = re.compile(r'^(\$\{.+\}|\$[A-Z_][A-Z0-9_]*)$')
+postgres_loopback = re.compile(r'^postgres(?:ql)?://postgres:postgres@(?:localhost|127\.0\.0\.1)(?::\d{1,5})?(?![\w.:@%-])')
+
+def real_token(m):
+    span = m.group(1)
+    return not any(rx.search(span) for rx in structural + substring)
+
+def real_db_url(m):
+    pw = m.group(2)
+    if pw == '' or interpolated.search(pw) or pw in url_password_words:
+        return False
+    if any(rx.search(pw) for rx in structural):
+        return False
+    return not postgres_loopback.search(m.group(1))
+
+patterns = [(name, rx, None) for name, rx in patterns] + [
+    ('stripe-live-key', re.compile(r'\b((?:sk|rk)_live_[A-Za-z0-9]{24,})\b'), real_token),
+    ('slack-token', re.compile(r'\b(xox[baprs]-[A-Za-z0-9-]{10,})\b'), real_token),
+    ('npm-token', re.compile(r'\b(npm_[A-Za-z0-9]{36})\b'), real_token),
+    # A diff line of a JSONL file is JSON text: the URL stops at an unescaped
+    # quote, so it cannot run across into the next field's "@".
+    ('db-url-password',
+     re.compile(r'\b((?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis|amqp)://[^:\s/@"]+:((?:[^@\s/"\\]|\\.)+)@[^\s/"]+)'),
+     real_db_url),
+]
+
 def families(text):
-    return [name for name, rx in patterns if rx.search(text)]
+    return [name for name, rx, real in patterns
+            if any(real is None or real(m) for m in rx.finditer(text))]
 
 def read(path):
     with open(path, "rb") as f:

bin/gstack-learnings-log

@@ -56,6 +56,7 @@ trap 'rm -f "$TMPERR"' EXIT
 set +e
 VALIDATED=$(printf '%s' "$INPUT" | GSTACK_LIB_DIR="$SCRIPT_DIR/../lib" bun --no-env-file --config="${GSTACK_BUN_CONFIG:-/dev/null}" -e "
 const { hasInjection } = await import(process.env.GSTACK_LIB_DIR + '/jsonl-store.ts');
+const { scan } = await import(process.env.GSTACK_LIB_DIR + '/redact-engine.ts');
 const raw = await Bun.stdin.text();
 let j;
 try { j = JSON.parse(raw); } catch { process.stderr.write('gstack-learnings-log: invalid JSON, skipping\n'); process.exit(1); }
@@ -95,6 +96,19 @@ if (j.insight && hasInjection(j.insight)) {
   process.exit(1);
 }
 
+// Secrets: the same redact-engine HIGH gate gstack-decision-log applies, over
+// every key and string value this row stores (learnings sync across machines).
+const stored = [];
+(function collect(v) {
+  if (typeof v === 'string') stored.push(v);
+  else if (v && typeof v === 'object') for (const [k, x] of Object.entries(v)) { stored.push(k); collect(x); }
+})(j);
+const high = scan(stored.join('\n')).findings.filter((f) => f.tier === 'HIGH');
+if (high.length > 0) {
+  process.stderr.write('gstack-learnings-log: learning contains a HIGH-tier secret (' + [...new Set(high.map((f) => f.id))].join(', ') + '), rejected; remove the value and log the learning again, do not log secrets\n');
+  process.exit(1);
+}
+
 // Inject timestamp if not present
 if (!j.ts) j.ts = new Date().toISOString();
 

bin/gstack-timeline-log

@@ -12,6 +12,11 @@
 set -euo pipefail
 GSTACK_BUN_CONFIG=/dev/null; case "${OSTYPE:-}" in msys*|cygwin*) if command -v cygpath >/dev/null 2>&1; then GSTACK_BUN_CONFIG=NUL; fi ;; esac
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# Windows git-bash (#1950): Bun cannot import a POSIX /c/... path; cygpath -m
+# gives C:/... for the redact-engine import below.
+case "$(uname -s)" in
+  MINGW*|MSYS*|CYGWIN*) command -v cygpath >/dev/null 2>&1 && SCRIPT_DIR="$(cygpath -m "$SCRIPT_DIR")" ;;
+esac
 eval "$("$SCRIPT_DIR/gstack-slug" 2>/dev/null)"
 . "$(dirname "$0")/gstack-state-root.sh" 2>/dev/null || { echo "$0: cannot resolve the gstack state root: $(dirname "$0")/gstack-state-root.sh is missing. fix: reinstall with ./setup or /gstack-upgrade (docs/state-root.md)" >&2; exit 1; }
 gstack_state_root_select
@@ -20,15 +25,25 @@ mkdir -p "$GSTACK_HOME/projects/$SLUG"
 
 INPUT="$1"
 
-# Validate: input must be parseable JSON with required fields. The validator
-# prints its verdict; a non-zero exit means Bun itself failed (#3415). An exit
-# code cannot carry the verdict: a Bun crash on Windows exits 3.
+# Validate: input must be parseable JSON with required fields and no
+# redact-engine HIGH secret in any key or string value (the gate
+# gstack-decision-log applies). The validator prints its verdict; a non-zero
+# exit means Bun itself failed (#3415). An exit code cannot carry the verdict:
+# a Bun crash on Windows exits 3.
 VALIDATE_RC=0
-VALIDATE_OUT=$(printf '%s' "$INPUT" | bun --no-env-file --config="${GSTACK_BUN_CONFIG:-/dev/null}" -e "
+VALIDATE_OUT=$(printf '%s' "$INPUT" | GSTACK_LIB_DIR="$SCRIPT_DIR/../lib" bun --no-env-file --config="${GSTACK_BUN_CONFIG:-/dev/null}" -e "
   const text = await Bun.stdin.text();
   let j;
   try { j = JSON.parse(text); } catch {}
-  console.log(j && j.skill && j.event ? 'valid' : 'invalid');
+  if (!(j && j.skill && j.event)) { console.log('invalid'); process.exit(0); }
+  const { scan } = await import(process.env.GSTACK_LIB_DIR + '/redact-engine.ts');
+  const stored = [];
+  (function collect(v) {
+    if (typeof v === 'string') stored.push(v);
+    else if (v && typeof v === 'object') for (const [k, x] of Object.entries(v)) { stored.push(k); collect(x); }
+  })(j);
+  const ids = [...new Set(scan(stored.join('\\n')).findings.filter((f) => f.tier === 'HIGH').map((f) => f.id))];
+  console.log(ids.length ? 'secret ' + ids.join(', ') : 'valid');
 ") || VALIDATE_RC=$?
 if [ "$VALIDATE_RC" -ne 0 ]; then
   echo "gstack-timeline-log: bun exited $VALIDATE_RC before the event was validated; nothing was logged" >&2
@@ -37,6 +52,7 @@ fi
 case "$VALIDATE_OUT" in
   valid) ;;
   invalid) exit 0 ;;  # skip silently, non-blocking
+  secret\ *) echo "gstack-timeline-log: event contains a HIGH-tier secret (${VALIDATE_OUT#secret }); nothing was logged. Remove the value and log the event again; do not log secrets" >&2; exit 1 ;;
   *) echo "gstack-timeline-log: bun printed no verdict while validating the event; nothing was logged" >&2; exit 1 ;;
 esac
 

changelog.d/stop-secrets-in-learnings-timeline-and-brain-sync.md

@@ -0,0 +1,4 @@
+---
+important: security
+---
+- **Learnings, timeline events and brain sync no longer store or push Stripe live keys, Slack tokens, npm tokens or database URLs with a real password.** `gstack-learnings-log` and `gstack-timeline-log` now refuse any row whose keys or string values hold a HIGH-tier secret by `lib/redact-engine`, the same gate `gstack-decision-log` already applied: the writer exits 1, names the rule (for example `db.url_with_password`) but never the value, and writes nothing. Before, `gstack-learnings-log` stored `DATABASE_URL=postgres://app:<password>@…` and exited 0, and `gstack-timeline-log` had no secret check at all. Brain sync's push-time scanner, which only knew six families, now also holds a file with a Stripe live or restricted key, a Slack token, an npm token or a database URL with a password, using redact-engine's placeholder rules so `postgres://app:${DB_PASSWORD}@…`, `PASSWORD` placeholders and the local `postgres:postgres@localhost` default still sync. Ordinary rows that mention tokens or passwords in prose still write and sync.

test/brain-sync.test.ts

@@ -379,6 +379,11 @@ describe('gstack-brain-sync secret scan', () => {
     ['pem-block', '-----BEGIN PRIVATE KEY-----'],
     ['jwt', 'eyJ0eXAiOiJKV1QiLCJh.eyJzdWIiOiIxMjM0NTY3.SflKxwRJSMeKKF30oGTbU'],
     ['bearer-json', '"authorization":"Bearer abcdef1234567890abcdef1234567890"'],
+    // GST-2196: redact-engine rates these HIGH; the scanner used to push them.
+    ['stripe-live-key', 'sk_' + 'live_' + 'Q7d2Lm9Xr4Tb8Vn1Kp3Wz6Hc'],
+    ['slack-token', 'xox' + 'b-' + '2384710293-4829103847-Zr8Qm2Lx7Vn4Tb'],
+    ['db-url-password', 'DATABASE_URL=postgres://app:' + 'Vq8zR2' + 'mLp7@db.internal.example.com:5432/prod'],
+    ['npm-token', 'npm_' + 'a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6q7R8'],
   ];
 
   for (const [name, content] of SECRETS) {
@@ -406,6 +411,27 @@ describe('gstack-brain-sync secret scan', () => {
     });
   }
 
+  test('prose about tokens and passwords and placeholder database urls still sync', () => {
+    run(['gstack-artifacts-init', '--remote', bareRemote]);
+    run(['gstack-config', 'set', 'artifacts_sync_mode', 'full']);
+    fs.mkdirSync(path.join(tmpHome, 'projects', 'p'), { recursive: true });
+    const rows = [
+      { insight: 'Rotate the npm token before release; the Slack bot token and the password live in the vault' },
+      { insight: 'DATABASE_URL=postgres://app:PASSWORD@host/db or postgres://app:${DB_PASSWORD}@db/prod' },
+      { insight: 'local dev uses postgres://postgres:postgres@localhost:5432/app' },
+      { insight: 'cache is redis://localhost:6379', owner: 'ops@team.example.com' },
+      { insight: 'test keys look like sk_' + 'live_' + 'EXAMPLEEXAMPLEEXAMPLEEXAMPLE' },
+    ];
+    fs.writeFileSync(path.join(tmpHome, 'projects/p/learnings.jsonl'),
+      rows.map((r) => JSON.stringify(r)).join('\n') + '\n');
+    run(['gstack-brain-enqueue', 'projects/p/learnings.jsonl']);
+    expect(run(['gstack-brain-sync', '--once']).status).toBe(0);
+    const status = JSON.parse(fs.readFileSync(path.join(tmpHome, '.brain-sync-status.json'), 'utf-8'));
+    expect(status.held_count ?? 0).toBe(0);
+    const log = spawnSync('git', ['--git-dir=' + bareRemote, 'log', '--oneline'], { encoding: 'utf-8', timeout: 30_000 });
+    expect(log.stdout).toMatch(/sync: 1 file/);
+  });
+
   test('--skip-file unblocks specific file', () => {
     run(['gstack-artifacts-init', '--remote', bareRemote]);
     run(['gstack-config', 'set', 'artifacts_sync_mode', 'full']);

test/gstack-skill-start.test.ts

@@ -458,6 +458,7 @@ describe('gstack-skill-start behavior', () => {
     const log = path.join(tmp, 'egress.log');
     const sinks = ['gstack-telemetry-log', 'gstack-brain-sync', 'gstack-update-check'];
     fs.mkdirSync(shadow); fs.mkdirSync(gh);
+    fs.symlinkSync(path.join(ROOT, 'lib'), path.join(tmp, 'lib'));
     for (const name of fs.readdirSync(path.join(ROOT, 'bin'))) {
       if (!sinks.includes(name)) fs.symlinkSync(path.join(ROOT, 'bin', name), path.join(shadow, name));
     }

test/learnings.test.ts

@@ -345,3 +345,48 @@ describe('G3: learnings-log temp file and TMPDIR', () => {
     expect(stderr).toContain('TMPDIR');
   });
 });
+
+// The four secret families brain sync's scanner used to miss (GST-2196).
+// Built by concatenation so this file never holds a scannable literal.
+const SECRET_FAMILIES: [string, string][] = [
+  ['stripe live key', 'sk_' + 'live_' + 'Q7d2Lm9Xr4Tb8Vn1Kp3Wz6Hc'],
+  ['slack bot token', 'xox' + 'b-' + '2384710293-4829103847-Zr8Qm2Lx7Vn4Tb'],
+  ['database url with a password', 'postgres://app:' + 'Vq8zR2' + 'mLp7@db.internal.example.com:5432/prod'],
+  ['npm token', 'npm_' + 'a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6q7R8'],
+];
+
+describe('gstack-learnings-log refuses HIGH-tier secrets', () => {
+  for (const [name, secret] of SECRET_FAMILIES) {
+    test(`rejects a ${name} in the insight and writes nothing`, () => {
+      const input = JSON.stringify({ skill: 'review', type: 'operational', key: 'env-setup', insight: `DATABASE_URL and keys: ${secret}`, confidence: 7, source: 'observed' });
+      const result = runLog(input, { expectFail: true });
+      expect(result.exitCode).not.toBe(0);
+      expect(result.stdout).toContain('HIGH-tier secret');
+      expect(result.stdout).not.toContain(secret);
+      expect(findLearningsFile()).toBeNull();
+    });
+  }
+
+  test('scans every stored field, not just the insight', () => {
+    const input = JSON.stringify({ skill: 'review', type: 'tool', key: 'npm-publish', insight: 'publish needs a token', confidence: 6, source: 'observed', files: [SECRET_FAMILIES[3][1]] });
+    const result = runLog(input, { expectFail: true });
+    expect(result.exitCode).not.toBe(0);
+    expect(findLearningsFile()).toBeNull();
+  });
+
+  test('prose about tokens and passwords still writes', () => {
+    const insight = 'Rotate the npm token before release; the password field and the Slack bot token live in the vault, never in the repo';
+    runLog(JSON.stringify({ skill: 'review', type: 'operational', key: 'secret-handling', insight, confidence: 8, source: 'observed' }));
+    const f = findLearningsFile();
+    expect(f).not.toBeNull();
+    expect(JSON.parse(fs.readFileSync(f!, 'utf-8').trim()).insight).toBe(insight);
+  });
+
+  test('placeholder database urls still write', () => {
+    const insight = 'Set DATABASE_URL=postgres://app:PASSWORD@host/db, or postgres://app:${DB_PASSWORD}@db/prod in CI';
+    runLog(JSON.stringify({ skill: 'review', type: 'operational', key: 'db-url-shape', insight, confidence: 8, source: 'observed' }));
+    const f = findLearningsFile();
+    expect(f).not.toBeNull();
+    expect(JSON.parse(fs.readFileSync(f!, 'utf-8').trim()).insight).toBe(insight);
+  });
+});

test/skill-e2e-bws.test.ts

@@ -223,13 +223,12 @@ Report the exact output — either "READY: <path>" or "NEEDS_SETUP".`,
       fs.copyFileSync(path.join(ROOT, 'bin', script), path.join(binDir, script));
       fs.chmodSync(path.join(binDir, script), 0o755);
     }
-    // gstack-learnings-log imports $SCRIPT_DIR/../lib/jsonl-store.ts (shared
-    // injection patterns, since v1.57.5.0) — a real install always ships bin/
-    // and lib/ together, so the fixture must too. Without it the bin exits 1
-    // before writing anything and the test fails on every attempt.
-    const libDir = path.join(opDir, 'lib');
-    fs.mkdirSync(libDir, { recursive: true });
-    fs.copyFileSync(path.join(ROOT, 'lib', 'jsonl-store.ts'), path.join(libDir, 'jsonl-store.ts'));
+    // gstack-learnings-log imports $SCRIPT_DIR/../lib/jsonl-store.ts and
+    // lib/redact-engine.ts (which imports further lib modules) — a real install
+    // always ships bin/ and lib/ together, so the fixture must too. Without it
+    // the bin exits 1 before writing anything and the test fails on every
+    // attempt. A copy, not a symlink: the agent runs inside this directory.
+    fs.cpSync(path.join(ROOT, 'lib'), path.join(opDir, 'lib'), { recursive: true });
 
     // gstack-learnings-log will create the project dir automatically via gstack-slug
 

test/skill-e2e-context-skills.test.ts

@@ -72,6 +72,9 @@ function setupWorkdir(suffix: string): { workDir: string; gstackHome: string; sl
       fs.chmodSync(path.join(binDir, script), 0o755);
     }
   }
+  // gstack-timeline-log and gstack-learnings-log import ../lib/redact-engine.ts
+  // from beside bin/. A copy, not a symlink: the agent runs inside workDir.
+  fs.cpSync(path.join(ROOT, 'lib'), path.join(workDir, 'lib'), { recursive: true });
 
   // Routing CLAUDE.md: the ## Skill routing section gstack ships.
   fs.writeFileSync(path.join(workDir, 'CLAUDE.md'), `# Project Instructions

test/skill-e2e-session-intelligence.test.ts

@@ -44,6 +44,9 @@ describeIfSelected('Session Intelligence E2E', [
     // gstack-skill-start and gstack-context-recovery, which call further bin
     // scripts, so ./bin/ is the whole runtime bin directory.
     fs.symlinkSync(path.join(ROOT, 'bin'), path.join(workDir, 'bin'), 'dir');
+    // gstack-timeline-log and gstack-learnings-log import ../lib/redact-engine.ts
+    // from beside ./bin/. A copy, not a symlink: the agent runs inside workDir.
+    fs.cpSync(path.join(ROOT, 'lib'), path.join(workDir, 'lib'), { recursive: true });
 
     // Compute slug (same logic as gstack-slug without git remote)
     slug = path.basename(workDir).replace(/[^a-zA-Z0-9._-]/g, '');

test/skill-e2e-skillify.test.ts

@@ -109,6 +109,9 @@ function setupSkillifyWorkdir(suffix: string, installSkills: string[] = ['scrape
       fs.chmodSync(path.join(binDir, script), 0o755);
     }
   }
+  // gstack-timeline-log and gstack-learnings-log import ../lib/redact-engine.ts
+  // from beside bin/. A copy, not a symlink: the agent runs inside workDir.
+  fs.cpSync(path.join(ROOT, 'lib'), path.join(workDir, 'lib'), { recursive: true });
 
   fs.writeFileSync(path.join(workDir, 'CLAUDE.md'), `# Project Instructions
 

test/skill-end-completion-durability.test.ts

@@ -141,6 +141,7 @@ describe('skill-end completion durability', () => {
 
   test('logger returns only after a delayed enqueue makes its spool record durable', () => {
     const shadow = path.join(fixture, 'bin'); fs.mkdirSync(shadow);
+    fs.symlinkSync(path.resolve(BIN, '../lib'), path.join(fixture, 'lib'));
     for (const f of fs.readdirSync(BIN)) { fs.symlinkSync(path.join(BIN, f), path.join(shadow, f)); }
     for (const f of ['gstack-timeline-log', 'gstack-brain-enqueue']) { fs.unlinkSync(path.join(shadow, f)); }
     fs.copyFileSync(path.join(BIN, 'gstack-timeline-log'), path.join(shadow, 'gstack-timeline-log'));

test/timeline.test.ts

@@ -283,3 +283,33 @@ describe('gstack-timeline-read', () => {
     }
   }
 });
+
+// The four secret families brain sync's scanner used to miss (GST-2196).
+// Built by concatenation so this file never holds a scannable literal.
+const SECRET_FAMILIES: [string, string][] = [
+  ['stripe live key', 'sk_' + 'live_' + 'Q7d2Lm9Xr4Tb8Vn1Kp3Wz6Hc'],
+  ['slack bot token', 'xox' + 'b-' + '2384710293-4829103847-Zr8Qm2Lx7Vn4Tb'],
+  ['database url with a password', 'postgres://app:' + 'Vq8zR2' + 'mLp7@db.internal.example.com:5432/prod'],
+  ['npm token', 'npm_' + 'a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6q7R8'],
+];
+
+describe('gstack-timeline-log refuses HIGH-tier secrets', () => {
+  for (const [name, secret] of SECRET_FAMILIES) {
+    test(`rejects a ${name} in any field and writes nothing`, () => {
+      const input = JSON.stringify({ skill: 'ship', event: 'completed', branch: 'main', outcome: `deploy failed: ${secret}` });
+      const result = runLog(input, { expectFail: true });
+      expect(result.exitCode).not.toBe(0);
+      expect(result.stdout).toContain('HIGH-tier secret');
+      expect(result.stdout).not.toContain(secret);
+      expect(findTimelineFile()).toBeNull();
+    });
+  }
+
+  test('ordinary events with token and password words still write', () => {
+    const input = '{"skill":"review","event":"completed","branch":"fix-token-refresh","outcome":"password reset flow reviewed; token rotation ok"}';
+    expect(runLog(input).exitCode).toBe(0);
+    const f = findTimelineFile();
+    expect(f).not.toBeNull();
+    expect(JSON.parse(fs.readFileSync(f!, 'utf-8').trim()).outcome).toBe('password reset flow reviewed; token rotation ok');
+  });
+});