mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-10 18:50:18 +01:00
Brain sync catches JSON-escaped secrets, and its pre-commit hook checks every family (GST-2263) (#4264)
* Add failing tests: escaped secrets pass brain sync, unchanged lines hold edits, hooks miss new families (GST-2263) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Brain sync scans only added lines, decodes JSON strings, and its pre-commit hook uses the same scanner (GST-2263) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Use a placeholder password in the brain sync docs example so the credential scan passes Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Test that the brain repo pre-commit hook refuses a commit when its scanner is missing or fails Review-Finding: p1/specialist-batched#2 reader=testing Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Brain repo hook keeps its own copy of the secret scanner, and the scan catches auth headers with escaped keys Review-Finding: v1/adversarial-native#1 reader=adversarial-native Review-Finding: v1/outside-voice#1 reader=outside-voice Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Name the brain sync changelog fragment after its branch Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Brain sync scans the whole staged diff again, so renamed keys and secret file names are still held (GST-2263) Review-Finding: bin/gstack-brain-secret-scan.py:111:outside-voice reader=outside-voice Review-Finding: bin/gstack-brain-secret-scan.py:104:outside-voice reader=outside-voice Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
Garry Tanandcapy-ai[bot] authored and GitHub committed
1 parent
56c5aa62ea
commit
6afe390244
7 files changed
+381
-154
No files matched your search
+22
-30
@@ -417,39 +417,31 @@ git -C "$GSTACK_HOME" config merge.union.name "union concat"
|
||||
# ---- install pre-commit hook (defense-in-depth) ----
|
||||
HOOK="$GSTACK_HOME/.git/hooks/pre-commit"
|
||||
mkdir -p "$(dirname "$HOOK")"
|
||||
cp "$SCRIPT_DIR/gstack-brain-secret-scan.py" "$(dirname "$HOOK")/gstack-brain-secret-scan.py"
|
||||
cat > "$HOOK" <<'HOOK_EOF'
|
||||
#!/usr/bin/env bash
|
||||
# gstack-artifacts pre-commit hook — secret-scan defense-in-depth.
|
||||
# The primary scanner runs inside gstack-brain-sync BEFORE staging. This hook
|
||||
# catches any manual `git commit` a user might accidentally run against the
|
||||
# artifacts repo.
|
||||
# gstack-artifacts pre-commit hook: secret-scan defense-in-depth. It runs a copy of the
|
||||
# scanner gstack-brain-sync runs before staging (same families, the whole
|
||||
# staged diff, JSON string values decoded), written beside this hook by
|
||||
# gstack-artifacts-init and gstack-brain-restore, so a manual `git commit` against
|
||||
# the artifacts repo is held the same way.
|
||||
set -uo pipefail
|
||||
|
||||
python3 -c "
|
||||
import sys, re, subprocess
|
||||
try:
|
||||
out = subprocess.check_output(['git', 'diff', '--cached'], stderr=subprocess.DEVNULL).decode('utf-8', 'replace')
|
||||
except Exception:
|
||||
sys.exit(0)
|
||||
|
||||
patterns = [
|
||||
('aws-access-key', re.compile(r'AKIA[0-9A-Z]{16}')),
|
||||
('github-token', re.compile(r'\b(gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})')),
|
||||
('openai-key', re.compile(r'\bsk-[A-Za-z0-9_-]{20,}')),
|
||||
('pem-block', re.compile(r'-----BEGIN [A-Z ]{3,}-----')),
|
||||
('jwt', re.compile(r'\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b')),
|
||||
('bearer-token-json',
|
||||
re.compile(r'\"(authorization|api[_-]?key|apikey|token|secret|password)\"\s*:\s*\"(Bearer |Basic |Token )?[A-Za-z0-9_./+=-]{16,}\"',
|
||||
re.IGNORECASE)),
|
||||
]
|
||||
for name, rx in patterns:
|
||||
if rx.search(out):
|
||||
sys.stderr.write(f'gstack-artifacts pre-commit: refusing commit — {name} detected in staged diff.\n')
|
||||
sys.stderr.write('Either edit the offending file, or if intentional, run:\n')
|
||||
sys.stderr.write(' gstack-brain-sync --skip-file <path> (to permanently exclude)\n')
|
||||
sys.exit(1)
|
||||
sys.exit(0)
|
||||
"
|
||||
SCAN="$(dirname "$0")/gstack-brain-secret-scan.py"
|
||||
if [ ! -f "$SCAN" ]; then
|
||||
echo "gstack-artifacts pre-commit: refusing commit — the secret scanner $SCAN is missing; rerun gstack-artifacts-init to reinstall this hook." >&2
|
||||
exit 1
|
||||
fi
|
||||
hits=$(git diff --cached --no-renames --no-color --no-ext-diff | python3 "$SCAN" gate -)
|
||||
rc=$?
|
||||
[ "$rc" -eq 0 ] && exit 0
|
||||
if [ "$rc" -eq 3 ]; then
|
||||
echo "gstack-artifacts pre-commit: refusing commit — $hits detected in staged diff." >&2
|
||||
echo "Either edit the offending file, or if intentional, run:" >&2
|
||||
echo " gstack-brain-sync --skip-file <path> (to permanently exclude)" >&2
|
||||
else
|
||||
echo "gstack-artifacts pre-commit: refusing commit — the secret scan failed (exit $rc)." >&2
|
||||
fi
|
||||
exit 1
|
||||
HOOK_EOF
|
||||
chmod +x "$HOOK"
|
||||
|
||||
|
||||
+20
-22
@@ -202,31 +202,29 @@ git -C "$GSTACK_HOME" config merge.union.name "union concat"
|
||||
# ---- install pre-commit hook (same as init) ----
|
||||
HOOK="$GSTACK_HOME/.git/hooks/pre-commit"
|
||||
mkdir -p "$(dirname "$HOOK")"
|
||||
cp "$SCRIPT_DIR/gstack-brain-secret-scan.py" "$(dirname "$HOOK")/gstack-brain-secret-scan.py"
|
||||
cat > "$HOOK" <<'HOOK_EOF'
|
||||
#!/usr/bin/env bash
|
||||
# gstack-brain pre-commit hook: secret-scan defense-in-depth. It runs a copy of the
|
||||
# scanner gstack-brain-sync runs before staging (same families, the whole
|
||||
# staged diff, JSON string values decoded), written beside this hook by
|
||||
# gstack-artifacts-init and gstack-brain-restore, so a manual `git commit` against
|
||||
# the artifacts repo is held the same way.
|
||||
set -uo pipefail
|
||||
python3 -c "
|
||||
import sys, re, subprocess
|
||||
try:
|
||||
out = subprocess.check_output(['git', 'diff', '--cached'], stderr=subprocess.DEVNULL).decode('utf-8', 'replace')
|
||||
except Exception:
|
||||
sys.exit(0)
|
||||
patterns = [
|
||||
('aws-access-key', re.compile(r'AKIA[0-9A-Z]{16}')),
|
||||
('github-token', re.compile(r'\b(gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})')),
|
||||
('openai-key', re.compile(r'\bsk-[A-Za-z0-9_-]{20,}')),
|
||||
('pem-block', re.compile(r'-----BEGIN [A-Z ]{3,}-----')),
|
||||
('jwt', re.compile(r'\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b')),
|
||||
('bearer-token-json',
|
||||
re.compile(r'\"(authorization|api[_-]?key|apikey|token|secret|password)\"\s*:\s*\"(Bearer |Basic |Token )?[A-Za-z0-9_./+=-]{16,}\"',
|
||||
re.IGNORECASE)),
|
||||
]
|
||||
for name, rx in patterns:
|
||||
if rx.search(out):
|
||||
sys.stderr.write(f'gstack-brain pre-commit: refusing commit — {name} detected.\n')
|
||||
sys.exit(1)
|
||||
sys.exit(0)
|
||||
"
|
||||
SCAN="$(dirname "$0")/gstack-brain-secret-scan.py"
|
||||
if [ ! -f "$SCAN" ]; then
|
||||
echo "gstack-brain pre-commit: refusing commit — the secret scanner $SCAN is missing; rerun gstack-brain-restore to reinstall this hook." >&2
|
||||
exit 1
|
||||
fi
|
||||
hits=$(git diff --cached --no-renames --no-color --no-ext-diff | python3 "$SCAN" gate -)
|
||||
rc=$?
|
||||
[ "$rc" -eq 0 ] && exit 0
|
||||
if [ "$rc" -eq 3 ]; then
|
||||
echo "gstack-brain pre-commit: refusing commit — $hits detected in staged diff." >&2
|
||||
else
|
||||
echo "gstack-brain pre-commit: refusing commit — the secret scan failed (exit $rc)." >&2
|
||||
fi
|
||||
exit 1
|
||||
HOOK_EOF
|
||||
chmod +x "$HOOK"
|
||||
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env python3
|
||||
r"""Secret scan over a staged git diff, shared by gstack-brain-sync and the
|
||||
pre-commit hook that gstack-artifacts-init and gstack-brain-restore install.
|
||||
|
||||
The whole diff is scanned as written, headers and context lines included, so
|
||||
a secret in a file name or one that lines up across an unchanged line is
|
||||
held. Each added line is also scanned with its JSON string literals decoded,
|
||||
so a secret hidden behind \u escapes or backslash-escaped quotes is caught.
|
||||
|
||||
The first six pattern families are the original hook's; the four after them
|
||||
mirror lib/redact-patterns.ts HIGH rules (stripe.secret/restricted,
|
||||
slack.token, npm.token, db.url_with_password) with its placeholder
|
||||
exemptions, so a row the learnings and timeline writers accept is not held
|
||||
here. Only the family name ever leaves this script, never the matched text.
|
||||
The diff is read as bytes and decoded with replacement, so undecodable bytes
|
||||
cannot crash the scan into a false "clean".
|
||||
|
||||
gate <diff|-> exit 0 clean, 3 hit (prints the family
|
||||
names); anything else is a scanner
|
||||
failure (fail closed). "-" reads stdin.
|
||||
attribute <diff> <names> <out> one pass: splits the diff on its
|
||||
"diff --git" headers, pairs each section
|
||||
with the NUL-separated names from
|
||||
"git diff --name-only -z" (same order),
|
||||
writes "<family>\t<path>" per flagged
|
||||
path; exit 0, or 4 when sections and
|
||||
names disagree
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
patterns = [
|
||||
('aws-access-key', re.compile(r'AKIA[0-9A-Z]{16}')),
|
||||
('github-token', re.compile(r'\b(gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})')),
|
||||
('openai-key', re.compile(r'\bsk-[A-Za-z0-9_-]{20,}')),
|
||||
('pem-block', re.compile(r'-----BEGIN [A-Z ]{3,}-----')),
|
||||
('jwt', re.compile(r'\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b')),
|
||||
('bearer-token-json',
|
||||
# JSON-embedded auth headers. The optional Bearer/Basic/Token prefix
|
||||
# matters: real auth values include a literal space after the scheme
|
||||
# name, but the value charset below does not include spaces, so
|
||||
# without the optional prefix every Bearer token in a JSON blob slips
|
||||
# past the scanner.
|
||||
re.compile(r'"(authorization|api[_-]?key|apikey|token|secret|password)"\s*:\s*"(Bearer |Basic |Token )?[A-Za-z0-9_./+=-]{16,}"',
|
||||
re.IGNORECASE)),
|
||||
]
|
||||
|
||||
structural = [re.compile(p, re.I) for p in (r'^your[_-]', r'^<[^>]*>$', r'^\*+$', r'^x{6,}$')]
|
||||
substring = [re.compile(p, re.I) for p in (r'example', r'^pass(word)?$', r'^changeme$', r'^redacted',
|
||||
r'^placeholder', r'^dummy', r'^fake', r'test[_-]?(key|token|secret)')]
|
||||
url_password_words = {'PASSWORD', 'PASS', 'PASSWD', 'YOUR_PASSWORD', 'DB_PASSWORD', 'MY_PASSWORD',
|
||||
'CHANGEME', 'CHANGE_ME', 'PLACEHOLDER', 'REDACTED', 'EXAMPLE'}
|
||||
interpolated = re.compile(r'^(\$\{.+\}|\$[A-Z_][A-Z0-9_]*)$')
|
||||
postgres_loopback = re.compile(r'^postgres(?:ql)?://postgres:postgres@(?:localhost|127\.0\.0\.1)(?::\d{1,5})?(?![\w.:@%-])')
|
||||
json_string = re.compile(r'"(?:[^"\\\n]|\\.)*"')
|
||||
|
||||
|
||||
def real_token(m):
|
||||
span = m.group(1)
|
||||
return not any(rx.search(span) for rx in structural + substring)
|
||||
|
||||
|
||||
def real_db_url(m):
|
||||
pw = m.group(2)
|
||||
if pw == '' or interpolated.search(pw) or pw in url_password_words:
|
||||
return False
|
||||
if any(rx.search(pw) for rx in structural):
|
||||
return False
|
||||
return not postgres_loopback.search(m.group(1))
|
||||
|
||||
|
||||
patterns = [(name, rx, None) for name, rx in patterns] + [
|
||||
('stripe-live-key', re.compile(r'\b((?:sk|rk)_live_[A-Za-z0-9]{24,})\b'), real_token),
|
||||
('slack-token', re.compile(r'\b(xox[baprs]-[A-Za-z0-9-]{10,})\b'), real_token),
|
||||
('npm-token', re.compile(r'\b(npm_[A-Za-z0-9]{36})\b'), real_token),
|
||||
# A diff line of a JSONL file is JSON text: the URL stops at an unescaped
|
||||
# quote, so it cannot run across into the next field's "@".
|
||||
('db-url-password',
|
||||
re.compile(r'\b((?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis|amqp)://[^:\s/@"]+:((?:[^@\s/"\\]|\\.)+)@[^\s/"]+)'),
|
||||
real_db_url),
|
||||
]
|
||||
|
||||
|
||||
def decoded_strings(text, depth=3):
|
||||
"""Every escaped JSON string literal in text, decoded, and the literals
|
||||
those hold in turn (a JSON document stored as a string value)."""
|
||||
found = []
|
||||
for m in json_string.finditer(text):
|
||||
if '\\' not in m.group(0):
|
||||
continue
|
||||
try:
|
||||
value = json.loads(m.group(0))
|
||||
except ValueError:
|
||||
continue
|
||||
found.append(value)
|
||||
if depth > 1:
|
||||
found.extend(decoded_strings(value, depth - 1))
|
||||
return found
|
||||
|
||||
|
||||
def added_lines(lines):
|
||||
"""The added lines of a unified diff, without their "+" marker. Lines
|
||||
before a section's first hunk are headers ("+++ b/path" among them)."""
|
||||
out, in_hunk = [], False
|
||||
for line in lines:
|
||||
if line.startswith(b"diff --git "):
|
||||
in_hunk = False
|
||||
elif line.startswith(b"@@"):
|
||||
in_hunk = True
|
||||
elif in_hunk and line.startswith(b"+"):
|
||||
out.append(line[1:].decode("utf-8", "replace"))
|
||||
return out
|
||||
|
||||
|
||||
def decoded_in_place(text):
|
||||
"""text with every escaped JSON string literal replaced by its decoded
|
||||
value in quotes, so a key and its value are matched together even when
|
||||
either is written with escapes."""
|
||||
def decode(m):
|
||||
if '\\' not in m.group(0):
|
||||
return m.group(0)
|
||||
try:
|
||||
return '"' + json.loads(m.group(0)) + '"'
|
||||
except ValueError:
|
||||
return m.group(0)
|
||||
return json_string.sub(decode, text)
|
||||
|
||||
|
||||
def families(lines):
|
||||
added = added_lines(lines)
|
||||
raw = b"\n".join(lines).decode("utf-8", "replace")
|
||||
texts = [raw] + [decoded_in_place(line) for line in added] + [s for line in added for s in decoded_strings(line)]
|
||||
text = "\n".join(texts)
|
||||
return [name for name, rx, real in patterns
|
||||
if any(real is None or real(m) for m in rx.finditer(text))]
|
||||
|
||||
|
||||
def read(path):
|
||||
if path == "-":
|
||||
return sys.stdin.buffer.read()
|
||||
with open(path, "rb") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
mode = sys.argv[1]
|
||||
if mode == "gate":
|
||||
hits = families(read(sys.argv[2]).split(b"\n"))
|
||||
if hits:
|
||||
print(",".join(hits))
|
||||
sys.exit(3)
|
||||
sys.exit(0)
|
||||
if mode != "attribute":
|
||||
sys.exit(2)
|
||||
|
||||
diff_path, names_path, out_path = sys.argv[2:5]
|
||||
names = [n.decode("utf-8", "surrogateescape") for n in read(names_path).split(b"\0") if n]
|
||||
sections = []
|
||||
for line in read(diff_path).split(b"\n"):
|
||||
if line.startswith(b"diff --git "):
|
||||
sections.append([line])
|
||||
elif sections:
|
||||
sections[-1].append(line)
|
||||
if len(sections) != len(names):
|
||||
sys.exit(4)
|
||||
with open(out_path, "w", encoding="utf-8", errors="surrogateescape", newline="\n") as out:
|
||||
for name, body in zip(names, sections):
|
||||
if "\n" in name or "\t" in name:
|
||||
sys.exit(4)
|
||||
hits = families(body)
|
||||
if hits:
|
||||
out.write(hits[0] + "\t" + name + "\n")
|
||||
+12
-100
@@ -204,108 +204,20 @@ sync_active() {
|
||||
return 0
|
||||
}
|
||||
|
||||
# Secret scan over a staged diff file. The first six pattern families are the
|
||||
# pre-commit hook's; the four after them mirror lib/redact-patterns.ts HIGH
|
||||
# rules (stripe.secret/restricted, slack.token, npm.token,
|
||||
# db.url_with_password) with its placeholder exemptions, so a row the
|
||||
# learnings and timeline writers accept is not held here. Only the family name
|
||||
# ever leaves this function, never the matched text. The diff is read as bytes and decoded with replacement, so
|
||||
# undecodable bytes cannot crash the scan into a false "clean".
|
||||
# Secret scan over a staged diff file: bin/gstack-brain-secret-scan.py, the
|
||||
# same scanner the artifacts repo's pre-commit hook runs. It scans the whole
|
||||
# diff as written and each added line with its JSON string literals decoded,
|
||||
# and prints only family names, never the matched text.
|
||||
# secret_scan gate <diff> exit 0 clean, 3 hit (prints the
|
||||
# family names); anything else is
|
||||
# a scanner failure (fail closed)
|
||||
# secret_scan attribute <diff> <names> <out> one pass: splits the diff on its
|
||||
# `diff --git` headers, pairs each
|
||||
# section with the NUL-separated
|
||||
# names from `git diff --name-only
|
||||
# -z` (same order), writes
|
||||
# "<family>\t<path>" per flagged
|
||||
# path; exit 0, or 4 when sections
|
||||
# and names disagree
|
||||
# secret_scan attribute <diff> <names> <out> writes "<family>\t<path>" per
|
||||
# flagged path; exit 0, or 4 when
|
||||
# the diff's sections and the
|
||||
# `git diff --name-only -z` names
|
||||
# disagree
|
||||
secret_scan() {
|
||||
python3 - "$@" <<'PYEOF'
|
||||
import re, sys
|
||||
patterns = [
|
||||
('aws-access-key', re.compile(r'AKIA[0-9A-Z]{16}')),
|
||||
('github-token', re.compile(r'\b(gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})')),
|
||||
('openai-key', re.compile(r'\bsk-[A-Za-z0-9_-]{20,}')),
|
||||
('pem-block', re.compile(r'-----BEGIN [A-Z ]{3,}-----')),
|
||||
('jwt', re.compile(r'\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b')),
|
||||
('bearer-token-json',
|
||||
# JSON-embedded auth headers. The optional Bearer/Basic/Token prefix
|
||||
# matters: real auth values include a literal space after the scheme
|
||||
# name, but the value charset below does not include spaces, so
|
||||
# without the optional prefix every Bearer token in a JSON blob slips
|
||||
# past the scanner.
|
||||
re.compile(r'"(authorization|api[_-]?key|apikey|token|secret|password)"\s*:\s*"(Bearer |Basic |Token )?[A-Za-z0-9_./+=-]{16,}"',
|
||||
re.IGNORECASE)),
|
||||
]
|
||||
|
||||
structural = [re.compile(p, re.I) for p in (r'^your[_-]', r'^<[^>]*>$', r'^\*+$', r'^x{6,}$')]
|
||||
substring = [re.compile(p, re.I) for p in (r'example', r'^pass(word)?$', r'^changeme$', r'^redacted',
|
||||
r'^placeholder', r'^dummy', r'^fake', r'test[_-]?(key|token|secret)')]
|
||||
url_password_words = {'PASSWORD', 'PASS', 'PASSWD', 'YOUR_PASSWORD', 'DB_PASSWORD', 'MY_PASSWORD',
|
||||
'CHANGEME', 'CHANGE_ME', 'PLACEHOLDER', 'REDACTED', 'EXAMPLE'}
|
||||
interpolated = re.compile(r'^(\$\{.+\}|\$[A-Z_][A-Z0-9_]*)$')
|
||||
postgres_loopback = re.compile(r'^postgres(?:ql)?://postgres:postgres@(?:localhost|127\.0\.0\.1)(?::\d{1,5})?(?![\w.:@%-])')
|
||||
|
||||
def real_token(m):
|
||||
span = m.group(1)
|
||||
return not any(rx.search(span) for rx in structural + substring)
|
||||
|
||||
def real_db_url(m):
|
||||
pw = m.group(2)
|
||||
if pw == '' or interpolated.search(pw) or pw in url_password_words:
|
||||
return False
|
||||
if any(rx.search(pw) for rx in structural):
|
||||
return False
|
||||
return not postgres_loopback.search(m.group(1))
|
||||
|
||||
patterns = [(name, rx, None) for name, rx in patterns] + [
|
||||
('stripe-live-key', re.compile(r'\b((?:sk|rk)_live_[A-Za-z0-9]{24,})\b'), real_token),
|
||||
('slack-token', re.compile(r'\b(xox[baprs]-[A-Za-z0-9-]{10,})\b'), real_token),
|
||||
('npm-token', re.compile(r'\b(npm_[A-Za-z0-9]{36})\b'), real_token),
|
||||
# A diff line of a JSONL file is JSON text: the URL stops at an unescaped
|
||||
# quote, so it cannot run across into the next field's "@".
|
||||
('db-url-password',
|
||||
re.compile(r'\b((?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis|amqp)://[^:\s/@"]+:((?:[^@\s/"\\]|\\.)+)@[^\s/"]+)'),
|
||||
real_db_url),
|
||||
]
|
||||
|
||||
def families(text):
|
||||
return [name for name, rx, real in patterns
|
||||
if any(real is None or real(m) for m in rx.finditer(text))]
|
||||
|
||||
def read(path):
|
||||
with open(path, "rb") as f:
|
||||
return f.read()
|
||||
|
||||
mode = sys.argv[1]
|
||||
if mode == "gate":
|
||||
hits = families(read(sys.argv[2]).decode("utf-8", "replace"))
|
||||
if hits:
|
||||
print(",".join(hits))
|
||||
sys.exit(3)
|
||||
sys.exit(0)
|
||||
|
||||
diff_path, names_path, out_path = sys.argv[2:5]
|
||||
names = [n.decode("utf-8", "surrogateescape") for n in read(names_path).split(b"\0") if n]
|
||||
sections = []
|
||||
for line in read(diff_path).split(b"\n"):
|
||||
if line.startswith(b"diff --git "):
|
||||
sections.append([])
|
||||
elif sections:
|
||||
sections[-1].append(line)
|
||||
if len(sections) != len(names):
|
||||
sys.exit(4)
|
||||
with open(out_path, "w", encoding="utf-8", errors="surrogateescape", newline="\n") as out:
|
||||
for name, body in zip(names, sections):
|
||||
if "\n" in name or "\t" in name:
|
||||
sys.exit(4)
|
||||
hits = families(b"\n".join(body).decode("utf-8", "replace"))
|
||||
if hits:
|
||||
out.write(hits[0] + "\t" + name + "\n")
|
||||
PYEOF
|
||||
python3 "$SCRIPT_DIR/gstack-brain-secret-scan.py" "$@"
|
||||
}
|
||||
|
||||
# True (0) if the spool holds at least one record file.
|
||||
@@ -1126,7 +1038,7 @@ hold_flagged_paths() {
|
||||
WAITING=0
|
||||
: > "$scan.hits"
|
||||
: > "$scan.blobs"
|
||||
git -C "$GSTACK_HOME" diff --cached --no-renames > "$scan.diff" 2>/dev/null || rc=1
|
||||
git -C "$GSTACK_HOME" diff --cached --no-renames --no-color --no-ext-diff > "$scan.diff" 2>/dev/null || rc=1
|
||||
git -C "$GSTACK_HOME" diff --cached --no-renames -z --name-only > "$scan.names" 2>/dev/null || rc=1
|
||||
[ "$rc" -eq 0 ] && { secret_scan attribute "$scan.diff" "$scan.names" "$scan.hits" || rc=1; }
|
||||
[ "$rc" -eq 0 ] && { append_invalid_json_hits "$scan.names" "$scan.hits" || rc=1; }
|
||||
@@ -1164,7 +1076,7 @@ hold_flagged_paths() {
|
||||
{ grep -vxF -f "$scan.unstage" "$paths_file" || true; } > "$paths_file.kept"
|
||||
mv -f "$paths_file.kept" "$paths_file"
|
||||
fi
|
||||
if ! git -C "$GSTACK_HOME" diff --cached --no-renames > "$scan.diff" 2>/dev/null; then
|
||||
if ! git -C "$GSTACK_HOME" diff --cached --no-renames --no-color --no-ext-diff > "$scan.diff" 2>/dev/null; then
|
||||
block_drain "git could not read the staged diff for the re-scan"
|
||||
return 1
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
---
|
||||
important: security
|
||||
---
|
||||
- **Brain sync now catches secrets hidden behind JSON escapes, and its pre-commit hook checks every family the sync does.** A hand-written learnings or timeline row could slip a Stripe key past the push-time scan by writing it with `\u` escapes, or slip an auth header past it inside an escaped JSON string. The scan now decodes each added line's JSON strings and scans them as well as the raw diff, so both are held. The pre-commit hook that `gstack-artifacts-init` and `gstack-brain-restore` install, which catches a manual `git commit` in `~/.gstack`, checked only the original six families; it now runs the same scanner as brain sync (`bin/gstack-brain-secret-scan.py`), so it also refuses Stripe live keys, Slack and npm tokens and database URLs with a password. Rerun `gstack-artifacts-init` to refresh a hook written by an older gstack.
|
||||
+21
-1
@@ -150,6 +150,22 @@ your machine. Blocked patterns include:
|
||||
- PEM blocks (`-----BEGIN …-----`)
|
||||
- JWTs (`eyJ…`)
|
||||
- Bearer tokens in JSON (`"authorization": "…"`, `"api_key": "…"`, etc.)
|
||||
- Stripe live secret and restricted keys (`sk_live_…`, `rk_live_…`)
|
||||
- Slack tokens (`xoxb-…`, `xoxp-…` and the other `xox?-` forms)
|
||||
- npm tokens (`npm_…`)
|
||||
- Database URLs with a password (`postgres://app:<password>@host`, and the
|
||||
same for MySQL, MongoDB, Redis and AMQP)
|
||||
|
||||
The last four follow gstack's redaction rules, so placeholders such as
|
||||
`sk_live_EXAMPLE…`, `${DB_PASSWORD}` or `postgres:postgres@localhost` are
|
||||
not held.
|
||||
|
||||
The scan reads the whole staged diff, including file names and the unchanged
|
||||
lines shown around each edit. Each added line is also scanned with its JSON
|
||||
strings decoded, so a secret written with `\u` escapes or inside an escaped
|
||||
JSON string is still caught. Because unchanged lines are scanned too, a file
|
||||
that already holds a flagged line is held on every edit until you remove the
|
||||
line or skip the file.
|
||||
|
||||
If a scan hits, only the flagged file is held back (with any file coupled
|
||||
to it); everything else syncs. The held file stays queued and is re-scanned
|
||||
@@ -169,7 +185,11 @@ To remediate:
|
||||
3. Otherwise, edit the file to remove the secret; the next skill run syncs it.
|
||||
|
||||
There's a defense-in-depth hook at `~/.gstack/.git/hooks/pre-commit` that
|
||||
runs the same scan if you manually `git commit` against the repo.
|
||||
runs the same scanner (`bin/gstack-brain-secret-scan.py`) if you manually
|
||||
`git commit` against the repo. `gstack-artifacts-init` and
|
||||
`gstack-brain-restore` install it and copy the scanner next to it, so the
|
||||
hook keeps working if gstack moves. A hook written by an older gstack checks
|
||||
only the first six patterns; rerun `gstack-artifacts-init` to refresh it.
|
||||
|
||||
Separately (v1.63.0.0+), every push writes a tamper-evident receipt to the
|
||||
egress ledger (`~/.gstack/security/egress.jsonl`) *before* anything is
|
||||
|
||||
+130
-1
@@ -273,8 +273,11 @@ describe('init + sync + restore round-trip', () => {
|
||||
test('the pre-commit hook init and restore write catches a JSON auth header with a Bearer prefix', () => {
|
||||
for (const bin of ['gstack-artifacts-init', 'gstack-brain-restore']) {
|
||||
const src = fs.readFileSync(path.join(BIN, bin), 'utf-8');
|
||||
expect(src).toContain('(Bearer |Basic |Token )?[A-Za-z0-9_./+=-]{16,}');
|
||||
expect(src).toContain('cp "$SCRIPT_DIR/gstack-brain-secret-scan.py" "$(dirname "$HOOK")/gstack-brain-secret-scan.py"');
|
||||
expect(src).toContain('SCAN="$(dirname "$0")/gstack-brain-secret-scan.py"');
|
||||
}
|
||||
const scanner = fs.readFileSync(path.join(BIN, 'gstack-brain-secret-scan.py'), 'utf-8');
|
||||
expect(scanner).toContain('(Bearer |Basic |Token )?[A-Za-z0-9_./+=-]{16,}');
|
||||
run(['gstack-artifacts-init', '--remote', bareRemote]);
|
||||
fs.mkdirSync(path.join(tmpHome, 'projects', 'p'), { recursive: true });
|
||||
const token = ['Bearer ', 'abcd', 'efgh', 'ijkl', 'mnop', 'qrst'].join('');
|
||||
@@ -285,6 +288,74 @@ describe('init + sync + restore round-trip', () => {
|
||||
expect(git(['log', '--oneline']).stdout).not.toContain('manual commit');
|
||||
});
|
||||
|
||||
test('the pre-commit hook init and restore write holds a Stripe live key (brain-sync scan families)', () => {
|
||||
const stripe = JSON.stringify({ leaked: 'sk_' + 'live_' + 'Q7d2Lm9Xr4Tb8Vn1Kp3Wz6Hc' }) + '\n';
|
||||
const commitRefused = (home: string) => {
|
||||
expect(fs.readFileSync(path.join(home, '.git/hooks/gstack-brain-secret-scan.py'), 'utf-8'))
|
||||
.toBe(fs.readFileSync(path.join(BIN, 'gstack-brain-secret-scan.py'), 'utf-8'));
|
||||
fs.mkdirSync(path.join(home, 'projects', 'p'), { recursive: true });
|
||||
fs.writeFileSync(path.join(home, 'projects/p/learnings.jsonl'), stripe);
|
||||
git(['add', '-f', 'projects/p/learnings.jsonl'], home);
|
||||
const r = git(['-c', 'user.name=fixture', '-c', 'user.email=fixture@example.com', 'commit', '-q', '-m', 'manual commit'], home);
|
||||
expect(r.status).not.toBe(0);
|
||||
expect(r.stderr).toContain('stripe-live-key');
|
||||
expect(git(['log', '--oneline'], home).stdout).not.toContain('manual commit');
|
||||
};
|
||||
expect(run(['gstack-artifacts-init', '--remote', bareRemote]).status).toBe(0);
|
||||
commitRefused(tmpHome);
|
||||
const machineB = fs.mkdtempSync(path.join(os.tmpdir(), 'brain-machineB-'));
|
||||
try {
|
||||
expect(run(['gstack-brain-restore', bareRemote], { env: { GSTACK_HOME: machineB } }).status).toBe(0);
|
||||
commitRefused(machineB);
|
||||
} finally {
|
||||
fs.rmSync(machineB, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('the pre-commit hook fails closed when its scanner is missing or fails', () => {
|
||||
expect(run(['gstack-artifacts-init', '--remote', bareRemote]).status).toBe(0);
|
||||
const hook = path.join(tmpHome, '.git/hooks/pre-commit');
|
||||
const original = fs.readFileSync(hook, 'utf-8');
|
||||
expect(original).toMatch(/^SCAN=.*$/m);
|
||||
const failingScanner = path.join(tmpHome, '.git', 'failing-scan.py');
|
||||
fs.writeFileSync(failingScanner, 'import sys\nsys.stdin.read()\nsys.exit(1)\n');
|
||||
fs.mkdirSync(path.join(tmpHome, 'projects', 'p'), { recursive: true });
|
||||
fs.writeFileSync(path.join(tmpHome, 'projects/p/learnings.jsonl'), '{"insight":"nothing secret"}\n');
|
||||
git(['add', '-f', 'projects/p/learnings.jsonl']);
|
||||
for (const [scan, message] of [
|
||||
[path.join(tmpHome, 'no-such-scanner.py'), 'secret scanner'],
|
||||
[failingScanner, 'secret scan failed (exit 1)'],
|
||||
] as [string, string][]) {
|
||||
fs.writeFileSync(hook, original.replace(/^SCAN=.*$/m, `SCAN='${scan}'`));
|
||||
const r = git(['-c', 'user.name=fixture', '-c', 'user.email=fixture@example.com', 'commit', '-q', '-m', 'manual commit']);
|
||||
expect(r.status).not.toBe(0);
|
||||
expect(r.stderr).toContain(message);
|
||||
expect(git(['log', '--oneline']).stdout).not.toContain('manual commit');
|
||||
}
|
||||
});
|
||||
|
||||
test('the pre-commit hook keeps scanning after the gstack install that wrote it is deleted', () => {
|
||||
const install = fs.mkdtempSync(path.join(os.tmpdir(), 'brain-install-'));
|
||||
try {
|
||||
fs.cpSync(BIN, path.join(install, 'bin'), { recursive: true });
|
||||
expect(run([path.join(install, 'bin', 'gstack-artifacts-init'), '--remote', bareRemote]).status).toBe(0);
|
||||
} finally {
|
||||
fs.rmSync(install, { recursive: true, force: true });
|
||||
}
|
||||
const file = path.join(tmpHome, 'projects/p/learnings.jsonl');
|
||||
const commit = () => {
|
||||
git(['add', '-f', 'projects/p/learnings.jsonl']);
|
||||
return git(['-c', 'user.name=fixture', '-c', 'user.email=fixture@example.com', 'commit', '-q', '-m', 'manual commit']);
|
||||
};
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.writeFileSync(file, '{"insight":"nothing secret"}\n');
|
||||
expect(commit().status).toBe(0);
|
||||
fs.appendFileSync(file, JSON.stringify({ leaked: 'sk_' + 'live_' + 'Q7d2Lm9Xr4Tb8Vn1Kp3Wz6Hc' }) + '\n');
|
||||
const r = commit();
|
||||
expect(r.status).not.toBe(0);
|
||||
expect(r.stderr).toContain('stripe-live-key');
|
||||
});
|
||||
|
||||
test('refuses init on different remote', () => {
|
||||
run(['gstack-artifacts-init', '--remote', bareRemote]);
|
||||
const otherRemote = fs.mkdtempSync(path.join(os.tmpdir(), 'brain-other-'));
|
||||
@@ -411,6 +482,64 @@ describe('gstack-brain-sync secret scan', () => {
|
||||
});
|
||||
}
|
||||
|
||||
// GST-2263: a hand-written row can hide a secret behind JSON escapes. The
|
||||
// scan decodes JSON string values and scans them as well as the raw line.
|
||||
const ESCAPED: [string, string][] = [
|
||||
['a \\u-escaped Stripe live key', '{"note":"sk_' + 'live_' + '\\u0051' + '7d2Lm9Xr4Tb8Vn1Kp3Wz6Hc"}'],
|
||||
['an auth header inside an escaped JSON string', '{"headers":"{\\"authorization\\": \\"Bearer ' + 'abcdef1234567890' + 'abcdef1234567890\\"}"}'],
|
||||
['an auth header whose key is \\u-escaped', '{"\\u0061uthorization":"Bearer ' + 'abcdef1234567890' + 'abcdef1234567890"}'],
|
||||
['an auth header whose Bearer prefix is \\u-escaped', '{"authorization":"\\u0042earer ' + 'abcdef1234567890' + 'abcdef1234567890"}'],
|
||||
];
|
||||
for (const [name, row] of ESCAPED) {
|
||||
test(`holds ${name}`, () => {
|
||||
run(['gstack-artifacts-init', '--remote', bareRemote]);
|
||||
run(['gstack-config', 'set', 'artifacts_sync_mode', 'full']);
|
||||
fs.mkdirSync(path.join(tmpHome, 'projects', 'p'), { recursive: true });
|
||||
fs.writeFileSync(path.join(tmpHome, 'projects/p/learnings.jsonl'), row + '\n');
|
||||
run(['gstack-brain-enqueue', 'projects/p/learnings.jsonl']);
|
||||
expect(run(['gstack-brain-sync', '--once']).status).toBe(0);
|
||||
const status = JSON.parse(fs.readFileSync(path.join(tmpHome, '.brain-sync-status.json'), 'utf-8'));
|
||||
expect(status.status).toBe('held');
|
||||
expect(status.held.map((h: any) => h.path)).toEqual(['projects/p/learnings.jsonl']);
|
||||
expect(spoolText()).toContain('projects/p/learnings.jsonl');
|
||||
});
|
||||
}
|
||||
|
||||
// GST-2263 audit: the scan reads the same diff lines main reads, context and
|
||||
// headers included, so a secret that only lines up across an unchanged line
|
||||
// or sits in a file name is still held.
|
||||
test('a key renamed to authorization above an unchanged Bearer value is held', () => {
|
||||
run(['gstack-artifacts-init', '--remote', bareRemote]);
|
||||
run(['gstack-config', 'set', 'artifacts_sync_mode', 'full']);
|
||||
const rel = 'projects/p/ceo-plans/headers.md';
|
||||
fs.mkdirSync(path.join(tmpHome, 'projects/p/ceo-plans'), { recursive: true });
|
||||
const value = ' "Bearer ' + 'abcdef1234567890' + 'abcdef1234567890"\n';
|
||||
fs.writeFileSync(path.join(tmpHome, rel), '{\n "note":\n' + value + '}\n');
|
||||
run(['gstack-brain-enqueue', rel]);
|
||||
expect(run(['gstack-brain-sync', '--once']).status).toBe(0);
|
||||
const statusJson = () => JSON.parse(fs.readFileSync(path.join(tmpHome, '.brain-sync-status.json'), 'utf-8'));
|
||||
expect(statusJson().held_count ?? 0).toBe(0);
|
||||
|
||||
fs.writeFileSync(path.join(tmpHome, rel), '{\n "authorization":\n' + value + '}\n');
|
||||
run(['gstack-brain-enqueue', rel]);
|
||||
expect(run(['gstack-brain-sync', '--once']).status).toBe(0);
|
||||
expect(statusJson().status).toBe('held');
|
||||
expect(statusJson().held.map((h: any) => h.path)).toEqual([rel]);
|
||||
});
|
||||
|
||||
test('a clean file whose name is an AWS access key is held', () => {
|
||||
run(['gstack-artifacts-init', '--remote', bareRemote]);
|
||||
run(['gstack-config', 'set', 'artifacts_sync_mode', 'full']);
|
||||
const rel = 'projects/p/ceo-plans/' + 'AKIA' + 'ABCDEFGHIJKLMNOP' + '.md';
|
||||
fs.mkdirSync(path.join(tmpHome, 'projects/p/ceo-plans'), { recursive: true });
|
||||
fs.writeFileSync(path.join(tmpHome, rel), '# a plan\n');
|
||||
run(['gstack-brain-enqueue', rel]);
|
||||
expect(run(['gstack-brain-sync', '--once']).status).toBe(0);
|
||||
const status = JSON.parse(fs.readFileSync(path.join(tmpHome, '.brain-sync-status.json'), 'utf-8'));
|
||||
expect(status.status).toBe('held');
|
||||
expect(status.held.map((h: any) => h.path)).toEqual([rel]);
|
||||
});
|
||||
|
||||
test('prose about tokens and passwords and placeholder database urls still sync', () => {
|
||||
run(['gstack-artifacts-init', '--remote', bareRemote]);
|
||||
run(['gstack-config', 'set', 'artifacts_sync_mode', 'full']);
|
||||
|
||||
Reference in new issue
Block a user