From 33ceff32bbc4856a5c2dfb4a56a531e82fe70032 Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 8 Oct 2026 14:54:53 +0000 Subject: [PATCH] feat(redact-prepush): committed exact-span allowlist at .gstack-redact-allowlist The pre-push hook never fed the engine's allowlist option, so a reviewed, known-benign value (a public dev-compose URL, a documented sample key) warned or blocked every push, and the only ways out were a bypass or reshaping code until the detector stopped matching. The hook now reads the repo-root .gstack-redact-allowlist from each pushed commit (never the working tree), one exact matched span per line with # comments. Only a finding whose matched text equals an entry is suppressed: a password alone, a substring, or a different key still reports and still blocks, and engine.* unscanned findings are untouched. Every push that carries entries prints the entry count and each suppressed finding's rule and file line. Oversized files (over 64 KiB) are ignored. gstack.redact.allowEmail keeps working as before. Co-authored-by: Boyard Engels Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- bin/gstack-redact-prepush | 74 ++++++++++++++++++++++--- docs/CONTRIBUTOR_REFERENCE.md | 3 + docs/troubleshooting.md | 18 ++++++ test/redact-prepush-hook.test.ts | 94 ++++++++++++++++++++++++++++++++ 4 files changed, 181 insertions(+), 8 deletions(-) diff --git a/bin/gstack-redact-prepush b/bin/gstack-redact-prepush index 98e9b931c..31a87c97e 100755 --- a/bin/gstack-redact-prepush +++ b/bin/gstack-redact-prepush @@ -24,6 +24,10 @@ * - `pii.email` skips the pusher's own address, `gstack.redact.allowEmail` * entries, and addresses already in the destination's commit metadata * (see emailScanOptions). + * - `.gstack-redact-allowlist`, read from each pushed commit (never the + * working tree), lists exact matched spans reviewed as benign, one per + * line, `#` comments. Only a finding whose matched text equals an entry is + * suppressed; every push that carries entries lists what they suppressed. * - GSTACK_REDACT_PREPUSH=skip → log + exit 0 (escape valve). * * Installed/uninstalled via `gstack-redact install-prepush-hook` (see the @@ -549,11 +553,40 @@ function emailScanOptions(historyRevs: string[], trackingRemote: string | null): }; } +const ALLOWLIST_PATH = ".gstack-redact-allowlist"; +const ALLOWLIST_MAX_BYTES = 64 * 1024; + +/** + * Entries of the allowlist committed in `sha`. Read from the pushed commit so + * a suppression exists only as a reviewable commit. Absent, unreadable or + * oversized means no entries: the file can record a reviewed exception, never + * loosen the scan by accident. + */ +function committedAllowlist(sha: string): string[] { + const spec = `${sha}:${ALLOWLIST_PATH}`; + const size = Number(git(["cat-file", "-s", spec]).trim()); + if (!size) return []; + if (size > ALLOWLIST_MAX_BYTES) { + process.stderr.write( + `gstack-redact-prepush: ${ALLOWLIST_PATH} in ${sha.slice(0, 12)} is over ${ALLOWLIST_MAX_BYTES} bytes; ignoring it.\n`, + ); + return []; + } + return git(["cat-file", "blob", spec]).split("\n") + .map((l) => l.trim()) + .filter((l) => l && !l.startsWith("#")); +} + +/** File line (and commit) of a finding. Never the matched value. */ +function findingWhere(f: Finding, part: AddedFile): string { + const line = part.lines[f.line - 1]; + return `${part.file || "(unknown file)"}${line ? `:${line}` : ""}` + + (part.commit ? ` in commit ${part.commit.slice(0, 12)}` : ""); +} + /** One MEDIUM report line: rule id, file line (and commit), and the fix. Never the matched value. */ function mediumLine(f: Finding, part: AddedFile): string { - const line = part.lines[f.line - 1]; - const where = `${part.file || "(unknown file)"}${line ? `:${line}` : ""}` + - (part.commit ? ` in commit ${part.commit.slice(0, 12)}` : ""); + const where = findingWhere(f, part); const fix = f.id === "pii.email" ? "remove the address, or allow it: git config --add gstack.redact.allowEmail
" : "remove the value before it becomes public"; @@ -574,7 +607,8 @@ function main() { .filter(Boolean) .map((l) => l.split(/\s+/)); - const parts: AddedFile[] = []; + const parts: Array<{ part: AddedFile; allowlist: string[] }> = []; + const allowEntries = new Set(); const historyRevs = new Set(); let trackingRemote: string | null = null; @@ -610,7 +644,9 @@ function main() { ); process.exit(1); } - parts.push(...added.filter((part) => part.text.trim())); + const allowlist = committedAllowlist(localSha); + for (const entry of allowlist) allowEntries.add(entry); + for (const part of added) if (part.text.trim()) parts.push({ part, allowlist }); historyRevs.add(localSha); if (!ZERO.test(remoteSha) && objectExists(remoteSha)) historyRevs.add(remoteSha); else { @@ -628,9 +664,21 @@ function main() { // Sliced (see scanAddedLines) so a large-but-legitimate diff is actually // scanned rather than blocked unscanned on the engine's size cap. // Scanned per file so a pattern can use the file's type (#3048). - for (const part of parts) { + // An allowlisted span is found by scanning with and without the entries, + // so the report names exactly what they suppressed. + const allowed: string[] = []; + const findingKey = (f: Finding) => `${f.id}:${f.line}:${f.col}`; + for (const { part, allowlist } of parts) { const opts = { repoVisibility: "private" as const, ...emails.options, ...(part.file ? { sourcePath: part.file } : {}) }; - for (const f of scanAddedLines(part.text, opts)) { + const findings = scanAddedLines(part.text, opts); + const kept = allowlist.length + ? new Set(scanAddedLines(part.text, { ...opts, allowlist }).map(findingKey)) + : null; + for (const f of findings) { + if (kept && !kept.has(findingKey(f))) { + if (f.severity === "HIGH" || f.severity === "MEDIUM") allowed.push(` ${f.severity} ${f.id} ${findingWhere(f, part)}\n`); + continue; + } if (f.severity === "HIGH") allHigh.push(f); else if (f.severity === "MEDIUM") { mediums.push(mediumLine(f, part)); @@ -639,6 +687,14 @@ function main() { } } + if (allowEntries.size > 0) { + process.stderr.write( + `gstack-redact-prepush: ${ALLOWLIST_PATH} (${allowEntries.size} entr${allowEntries.size === 1 ? "y" : "ies"}) ` + + `suppressed ${allowed.length} finding(s) in this push${allowed.length ? ":" : "."}\n`, + ); + for (const line of allowed) process.stderr.write(line); + } + if (mediums.length > 0) { process.stderr.write( `gstack-redact-prepush: ${mediums.length} MEDIUM finding(s) in pushed diff (PII/internal). ` + @@ -673,7 +729,9 @@ function main() { process.stderr.write(` HIGH ${f.id} ${f.preview}\n`); } process.stderr.write( - "\nRotate the credential (a pushed secret is compromised) and remove it from the diff.\n", + "\nRotate the credential (a pushed secret is compromised) and remove it from the diff.\n" + + `If the value is a known-benign sample, commit its exact matched text as a line of ${ALLOWLIST_PATH}\n` + + "at the repo root instead of reshaping code to dodge the pattern; every push lists what that file suppresses.\n", ); } diff --git a/docs/CONTRIBUTOR_REFERENCE.md b/docs/CONTRIBUTOR_REFERENCE.md index 45a05a3f8..b6b89d996 100644 --- a/docs/CONTRIBUTOR_REFERENCE.md +++ b/docs/CONTRIBUTOR_REFERENCE.md @@ -134,6 +134,9 @@ determined leaker (a CHANGELOG line that does would fail a hostile screenshotter - **Config keys:** `redact_repo_visibility` (public|private|unknown, local-only override for repos gh/glab can't read), `redact_prepush_hook` (true|false). There is intentionally NO key to disable HIGH blocking. +- **Committed allowlist:** `.gstack-redact-allowlist` at the root of the pushed commit lists + exact matched spans reviewed as benign; the pre-push hook suppresses only a + finding whose span equals an entry and lists every suppression on each push. - **Audit:** the /spec semantic pass appends a content-free record (categories + body sha256, no spec text) to `~/.gstack/security/semantic-reviews.jsonl` (0600). diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index f71008f53..64c78c1bf 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1120,6 +1120,24 @@ up deployed, so it still blocks. **Fix.** Use `postgres://postgres:postgres@localhost:5432/...` in local and CI config, or read the URL from an env var. If the credential is real, rotate it. Bypass once: `GSTACK_REDACT_PREPUSH=skip git push`. +If the URL is a reviewed, public dev-only value, list it in +[`.gstack-redact-allowlist`](#redact-allowlist). + + +### `.gstack-redact-allowlist ( entries) suppressed finding(s) in this push` + +**Meaning.** The pushed commit carries `.gstack-redact-allowlist` at the repo +root. Each line (after trimming; `#` starts a comment) is one exact matched +span: the whole `postgres://USER:PASSWORD@host:port` URL for +`db.url_with_password`, the key for a key pattern, the address for `pii.email`. A finding is suppressed only when its +matched text equals an entry, so a password alone, a substring, or a different +key still reports and still blocks. The hook reads the file from each pushed +commit, never the working tree, and ignores a file over 64 KiB. Every push that +carries entries prints this line and each suppressed finding's file and line. +It works alongside `gstack.redact.allowEmail`, which stays a local setting. + +**Fix.** Nothing, if every listed suppression is the reviewed value. Remove an +entry that no longer applies; rotate a credential that is real. ### `pii.ip_public` MEDIUM on a four-part version number diff --git a/test/redact-prepush-hook.test.ts b/test/redact-prepush-hook.test.ts index 36ced9365..66727b828 100644 --- a/test/redact-prepush-hook.test.ts +++ b/test/redact-prepush-hook.test.ts @@ -191,6 +191,100 @@ describe("release version exemption (#2856)", () => { }); }); +describe("committed allowlist (.gstack-redact-allowlist)", () => { + const HOOK_ALLOWLIST = ".gstack-redact-allowlist"; + const pushOf = (base: string, head: string) => `refs/heads/main ${head} refs/heads/main ${base}\n`; + const commitAllowlist = (content: string): string => { + return commit(HOOK_ALLOWLIST, content, "allowlist"); + }; + const DEV_URL = ["postgres://", "app:devpass", "@db:5432"].join(""); + + test("an exact committed span passes and the suppression is listed", () => { + const base = git(["rev-parse", "HEAD"]); + commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key"); + const head = commitAllowlist("# docs sample key\n" + FAKE_AWS_KEY + "\n"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(0); + expect(stderr).not.toContain("BLOCKED"); + expect(stderr).toContain(".gstack-redact-allowlist (1 entry) suppressed 2 finding(s) in this push:"); + expect(stderr).toContain("HIGH aws.access_key config.txt:1"); + expect(stderr).toContain("HIGH aws.access_key .gstack-redact-allowlist:2"); + }); + + test("a different credential still blocks next to an allowlisted one", () => { + const base = git(["rev-parse", "HEAD"]); + const otherKey = ["AKIA", "FEDCBA0987654321"].join(""); + commit("config.txt", "ok " + FAKE_AWS_KEY + "\nleak " + otherKey + "\n", "one benign one real"); + const head = commitAllowlist(FAKE_AWS_KEY + "\n"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(1); + expect(stderr).toContain("BLOCKED"); + expect(stderr).toContain("suppressed 2 finding(s)"); + const blocked = stderr.slice(stderr.indexOf("BLOCKED")); + expect(blocked.match(/HIGH {2}aws\.access_key/g)).toHaveLength(1); + }); + + test("a substring of the matched span does not suppress it", () => { + const base = git(["rev-parse", "HEAD"]); + commit("compose.txt", "DATABASE_URL=" + DEV_URL + "\n", "dev url"); + const head = commitAllowlist("app:devpass\n"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(1); + expect(stderr).toContain("db.url_with_password"); + expect(stderr).toContain("suppressed 0 finding(s) in this push."); + }); + + test("the whole matched URL suppresses that URL only", () => { + const base = git(["rev-parse", "HEAD"]); + const prodUrl = ["postgres://", "app:devpass", "@prod.internal:5432"].join(""); + commit("compose.txt", "DATABASE_URL=" + DEV_URL + "\nPROD_URL=" + prodUrl + "\n", "two urls"); + const head = commitAllowlist(DEV_URL + "\n"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(1); + expect(stderr).toContain("HIGH db.url_with_password compose.txt:1"); + expect(stderr).toMatch(/BLOCKED[\s\S]*HIGH {2}db\.url_with_password/); + }); + + test("an uncommitted allowlist in the working tree suppresses nothing", () => { + const base = git(["rev-parse", "HEAD"]); + const head = commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key"); + fs.writeFileSync(path.join(repo, HOOK_ALLOWLIST), FAKE_AWS_KEY + "\n"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(1); + expect(stderr).not.toContain("suppressed"); + }); + + test("an oversized allowlist is ignored and the scan stays strict", () => { + const base = git(["rev-parse", "HEAD"]); + commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key"); + const head = commitAllowlist(FAKE_AWS_KEY + "\n" + "#".repeat(65 * 1024) + "\n"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(1); + expect(stderr).toContain("is over 65536 bytes; ignoring it"); + }); + + test("entries cover MEDIUM emails alongside gstack.redact.allowEmail", () => { + const base = git(["rev-parse", "HEAD"]); + git(["config", "--add", "gstack.redact.allowEmail", "carol@corp.io"]); + commit("notes.md", "contact bob@corp.io or carol@corp.io or dave@corp.io\n", "contacts"); + const head = commitAllowlist(" bob@corp.io \n"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(0); + expect(stderr).toContain("MEDIUM pii.email notes.md:1"); + expect(stderr).toContain("suppressed 2 finding(s)"); + expect(stderr).toContain("1 MEDIUM finding(s) in pushed diff"); + }); + + test("the block message names the allowlist without one present", () => { + const base = git(["rev-parse", "HEAD"]); + const head = commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key"); + const { code, stderr } = runHook(pushOf(base, head)); + expect(code).toBe(1); + expect(stderr).toContain("commit its exact matched text as a line of .gstack-redact-allowlist"); + expect(stderr).not.toContain("suppressed"); + }); +}); + describe("diff direction + special refs", () => { test("only NEW content is scanned (remote..local), not pre-existing", () => { // Put a secret in the FIRST commit (already on remote), then push a clean commit.