mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-20 15:13:47 +01:00
feat(redact-prepush): committed exact-span allowlist at .gstack-redact-allowlist
The pre-push hook never fed the engine's allowlist option, so a reviewed, known-benign value (a public dev-compose URL, a documented sample key) warned or blocked every push, and the only ways out were a bypass or reshaping code until the detector stopped matching. The hook now reads the repo-root .gstack-redact-allowlist from each pushed commit (never the working tree), one exact matched span per line with # comments. Only a finding whose matched text equals an entry is suppressed: a password alone, a substring, or a different key still reports and still blocks, and engine.* unscanned findings are untouched. Every push that carries entries prints the entry count and each suppressed finding's rule and file line. Oversized files (over 64 KiB) are ignored. gstack.redact.allowEmail keeps working as before. Co-authored-by: Boyard Engels <boyard.engels@gmail.com> Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
0668443f36
commit
33ceff32bb
4 files changed
+181
-8
No files matched your search
@@ -24,6 +24,10 @@
|
||||
* - `pii.email` skips the pusher's own address, `gstack.redact.allowEmail`
|
||||
* entries, and addresses already in the destination's commit metadata
|
||||
* (see emailScanOptions).
|
||||
* - `.gstack-redact-allowlist`, read from each pushed commit (never the
|
||||
* working tree), lists exact matched spans reviewed as benign, one per
|
||||
* line, `#` comments. Only a finding whose matched text equals an entry is
|
||||
* suppressed; every push that carries entries lists what they suppressed.
|
||||
* - GSTACK_REDACT_PREPUSH=skip → log + exit 0 (escape valve).
|
||||
*
|
||||
* Installed/uninstalled via `gstack-redact install-prepush-hook` (see the
|
||||
@@ -549,11 +553,40 @@ function emailScanOptions(historyRevs: string[], trackingRemote: string | null):
|
||||
};
|
||||
}
|
||||
|
||||
const ALLOWLIST_PATH = ".gstack-redact-allowlist";
|
||||
const ALLOWLIST_MAX_BYTES = 64 * 1024;
|
||||
|
||||
/**
|
||||
* Entries of the allowlist committed in `sha`. Read from the pushed commit so
|
||||
* a suppression exists only as a reviewable commit. Absent, unreadable or
|
||||
* oversized means no entries: the file can record a reviewed exception, never
|
||||
* loosen the scan by accident.
|
||||
*/
|
||||
function committedAllowlist(sha: string): string[] {
|
||||
const spec = `${sha}:${ALLOWLIST_PATH}`;
|
||||
const size = Number(git(["cat-file", "-s", spec]).trim());
|
||||
if (!size) return [];
|
||||
if (size > ALLOWLIST_MAX_BYTES) {
|
||||
process.stderr.write(
|
||||
`gstack-redact-prepush: ${ALLOWLIST_PATH} in ${sha.slice(0, 12)} is over ${ALLOWLIST_MAX_BYTES} bytes; ignoring it.\n`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
return git(["cat-file", "blob", spec]).split("\n")
|
||||
.map((l) => l.trim())
|
||||
.filter((l) => l && !l.startsWith("#"));
|
||||
}
|
||||
|
||||
/** File line (and commit) of a finding. Never the matched value. */
|
||||
function findingWhere(f: Finding, part: AddedFile): string {
|
||||
const line = part.lines[f.line - 1];
|
||||
return `${part.file || "(unknown file)"}${line ? `:${line}` : ""}` +
|
||||
(part.commit ? ` in commit ${part.commit.slice(0, 12)}` : "");
|
||||
}
|
||||
|
||||
/** One MEDIUM report line: rule id, file line (and commit), and the fix. Never the matched value. */
|
||||
function mediumLine(f: Finding, part: AddedFile): string {
|
||||
const line = part.lines[f.line - 1];
|
||||
const where = `${part.file || "(unknown file)"}${line ? `:${line}` : ""}` +
|
||||
(part.commit ? ` in commit ${part.commit.slice(0, 12)}` : "");
|
||||
const where = findingWhere(f, part);
|
||||
const fix = f.id === "pii.email"
|
||||
? "remove the address, or allow it: git config --add gstack.redact.allowEmail <address>"
|
||||
: "remove the value before it becomes public";
|
||||
@@ -574,7 +607,8 @@ function main() {
|
||||
.filter(Boolean)
|
||||
.map((l) => l.split(/\s+/));
|
||||
|
||||
const parts: AddedFile[] = [];
|
||||
const parts: Array<{ part: AddedFile; allowlist: string[] }> = [];
|
||||
const allowEntries = new Set<string>();
|
||||
const historyRevs = new Set<string>();
|
||||
let trackingRemote: string | null = null;
|
||||
|
||||
@@ -610,7 +644,9 @@ function main() {
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
parts.push(...added.filter((part) => part.text.trim()));
|
||||
const allowlist = committedAllowlist(localSha);
|
||||
for (const entry of allowlist) allowEntries.add(entry);
|
||||
for (const part of added) if (part.text.trim()) parts.push({ part, allowlist });
|
||||
historyRevs.add(localSha);
|
||||
if (!ZERO.test(remoteSha) && objectExists(remoteSha)) historyRevs.add(remoteSha);
|
||||
else {
|
||||
@@ -628,9 +664,21 @@ function main() {
|
||||
// Sliced (see scanAddedLines) so a large-but-legitimate diff is actually
|
||||
// scanned rather than blocked unscanned on the engine's size cap.
|
||||
// Scanned per file so a pattern can use the file's type (#3048).
|
||||
for (const part of parts) {
|
||||
// An allowlisted span is found by scanning with and without the entries,
|
||||
// so the report names exactly what they suppressed.
|
||||
const allowed: string[] = [];
|
||||
const findingKey = (f: Finding) => `${f.id}:${f.line}:${f.col}`;
|
||||
for (const { part, allowlist } of parts) {
|
||||
const opts = { repoVisibility: "private" as const, ...emails.options, ...(part.file ? { sourcePath: part.file } : {}) };
|
||||
for (const f of scanAddedLines(part.text, opts)) {
|
||||
const findings = scanAddedLines(part.text, opts);
|
||||
const kept = allowlist.length
|
||||
? new Set(scanAddedLines(part.text, { ...opts, allowlist }).map(findingKey))
|
||||
: null;
|
||||
for (const f of findings) {
|
||||
if (kept && !kept.has(findingKey(f))) {
|
||||
if (f.severity === "HIGH" || f.severity === "MEDIUM") allowed.push(` ${f.severity} ${f.id} ${findingWhere(f, part)}\n`);
|
||||
continue;
|
||||
}
|
||||
if (f.severity === "HIGH") allHigh.push(f);
|
||||
else if (f.severity === "MEDIUM") {
|
||||
mediums.push(mediumLine(f, part));
|
||||
@@ -639,6 +687,14 @@ function main() {
|
||||
}
|
||||
}
|
||||
|
||||
if (allowEntries.size > 0) {
|
||||
process.stderr.write(
|
||||
`gstack-redact-prepush: ${ALLOWLIST_PATH} (${allowEntries.size} entr${allowEntries.size === 1 ? "y" : "ies"}) ` +
|
||||
`suppressed ${allowed.length} finding(s) in this push${allowed.length ? ":" : "."}\n`,
|
||||
);
|
||||
for (const line of allowed) process.stderr.write(line);
|
||||
}
|
||||
|
||||
if (mediums.length > 0) {
|
||||
process.stderr.write(
|
||||
`gstack-redact-prepush: ${mediums.length} MEDIUM finding(s) in pushed diff (PII/internal). ` +
|
||||
@@ -673,7 +729,9 @@ function main() {
|
||||
process.stderr.write(` HIGH ${f.id} ${f.preview}\n`);
|
||||
}
|
||||
process.stderr.write(
|
||||
"\nRotate the credential (a pushed secret is compromised) and remove it from the diff.\n",
|
||||
"\nRotate the credential (a pushed secret is compromised) and remove it from the diff.\n" +
|
||||
`If the value is a known-benign sample, commit its exact matched text as a line of ${ALLOWLIST_PATH}\n` +
|
||||
"at the repo root instead of reshaping code to dodge the pattern; every push lists what that file suppresses.\n",
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -134,6 +134,9 @@ determined leaker (a CHANGELOG line that does would fail a hostile screenshotter
|
||||
- **Config keys:** `redact_repo_visibility` (public|private|unknown, local-only
|
||||
override for repos gh/glab can't read), `redact_prepush_hook` (true|false).
|
||||
There is intentionally NO key to disable HIGH blocking.
|
||||
- **Committed allowlist:** `.gstack-redact-allowlist` at the root of the pushed commit lists
|
||||
exact matched spans reviewed as benign; the pre-push hook suppresses only a
|
||||
finding whose span equals an entry and lists every suppression on each push.
|
||||
- **Audit:** the /spec semantic pass appends a content-free record (categories +
|
||||
body sha256, no spec text) to `~/.gstack/security/semantic-reviews.jsonl` (0600).
|
||||
|
||||
|
||||
@@ -1120,6 +1120,24 @@ up deployed, so it still blocks.
|
||||
**Fix.** Use `postgres://postgres:postgres@localhost:5432/...` in local and CI
|
||||
config, or read the URL from an env var. If the credential is real, rotate it.
|
||||
Bypass once: `GSTACK_REDACT_PREPUSH=skip git push`.
|
||||
If the URL is a reviewed, public dev-only value, list it in
|
||||
[`.gstack-redact-allowlist`](#redact-allowlist).
|
||||
|
||||
<a id="redact-allowlist"></a>
|
||||
### `.gstack-redact-allowlist (<n> entries) suppressed <m> finding(s) in this push`
|
||||
|
||||
**Meaning.** The pushed commit carries `.gstack-redact-allowlist` at the repo
|
||||
root. Each line (after trimming; `#` starts a comment) is one exact matched
|
||||
span: the whole `postgres://USER:PASSWORD@host:port` URL for
|
||||
`db.url_with_password`, the key for a key pattern, the address for `pii.email`. A finding is suppressed only when its
|
||||
matched text equals an entry, so a password alone, a substring, or a different
|
||||
key still reports and still blocks. The hook reads the file from each pushed
|
||||
commit, never the working tree, and ignores a file over 64 KiB. Every push that
|
||||
carries entries prints this line and each suppressed finding's file and line.
|
||||
It works alongside `gstack.redact.allowEmail`, which stays a local setting.
|
||||
|
||||
**Fix.** Nothing, if every listed suppression is the reviewed value. Remove an
|
||||
entry that no longer applies; rotate a credential that is real.
|
||||
|
||||
<a id="redact-version-as-ip"></a>
|
||||
### `pii.ip_public` MEDIUM on a four-part version number
|
||||
|
||||
@@ -191,6 +191,100 @@ describe("release version exemption (#2856)", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("committed allowlist (.gstack-redact-allowlist)", () => {
|
||||
const HOOK_ALLOWLIST = ".gstack-redact-allowlist";
|
||||
const pushOf = (base: string, head: string) => `refs/heads/main ${head} refs/heads/main ${base}\n`;
|
||||
const commitAllowlist = (content: string): string => {
|
||||
return commit(HOOK_ALLOWLIST, content, "allowlist");
|
||||
};
|
||||
const DEV_URL = ["postgres://", "app:devpass", "@db:5432"].join("");
|
||||
|
||||
test("an exact committed span passes and the suppression is listed", () => {
|
||||
const base = git(["rev-parse", "HEAD"]);
|
||||
commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key");
|
||||
const head = commitAllowlist("# docs sample key\n" + FAKE_AWS_KEY + "\n");
|
||||
const { code, stderr } = runHook(pushOf(base, head));
|
||||
expect(code).toBe(0);
|
||||
expect(stderr).not.toContain("BLOCKED");
|
||||
expect(stderr).toContain(".gstack-redact-allowlist (1 entry) suppressed 2 finding(s) in this push:");
|
||||
expect(stderr).toContain("HIGH aws.access_key config.txt:1");
|
||||
expect(stderr).toContain("HIGH aws.access_key .gstack-redact-allowlist:2");
|
||||
});
|
||||
|
||||
test("a different credential still blocks next to an allowlisted one", () => {
|
||||
const base = git(["rev-parse", "HEAD"]);
|
||||
const otherKey = ["AKIA", "FEDCBA0987654321"].join("");
|
||||
commit("config.txt", "ok " + FAKE_AWS_KEY + "\nleak " + otherKey + "\n", "one benign one real");
|
||||
const head = commitAllowlist(FAKE_AWS_KEY + "\n");
|
||||
const { code, stderr } = runHook(pushOf(base, head));
|
||||
expect(code).toBe(1);
|
||||
expect(stderr).toContain("BLOCKED");
|
||||
expect(stderr).toContain("suppressed 2 finding(s)");
|
||||
const blocked = stderr.slice(stderr.indexOf("BLOCKED"));
|
||||
expect(blocked.match(/HIGH {2}aws\.access_key/g)).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("a substring of the matched span does not suppress it", () => {
|
||||
const base = git(["rev-parse", "HEAD"]);
|
||||
commit("compose.txt", "DATABASE_URL=" + DEV_URL + "\n", "dev url");
|
||||
const head = commitAllowlist("app:devpass\n");
|
||||
const { code, stderr } = runHook(pushOf(base, head));
|
||||
expect(code).toBe(1);
|
||||
expect(stderr).toContain("db.url_with_password");
|
||||
expect(stderr).toContain("suppressed 0 finding(s) in this push.");
|
||||
});
|
||||
|
||||
test("the whole matched URL suppresses that URL only", () => {
|
||||
const base = git(["rev-parse", "HEAD"]);
|
||||
const prodUrl = ["postgres://", "app:devpass", "@prod.internal:5432"].join("");
|
||||
commit("compose.txt", "DATABASE_URL=" + DEV_URL + "\nPROD_URL=" + prodUrl + "\n", "two urls");
|
||||
const head = commitAllowlist(DEV_URL + "\n");
|
||||
const { code, stderr } = runHook(pushOf(base, head));
|
||||
expect(code).toBe(1);
|
||||
expect(stderr).toContain("HIGH db.url_with_password compose.txt:1");
|
||||
expect(stderr).toMatch(/BLOCKED[\s\S]*HIGH {2}db\.url_with_password/);
|
||||
});
|
||||
|
||||
test("an uncommitted allowlist in the working tree suppresses nothing", () => {
|
||||
const base = git(["rev-parse", "HEAD"]);
|
||||
const head = commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key");
|
||||
fs.writeFileSync(path.join(repo, HOOK_ALLOWLIST), FAKE_AWS_KEY + "\n");
|
||||
const { code, stderr } = runHook(pushOf(base, head));
|
||||
expect(code).toBe(1);
|
||||
expect(stderr).not.toContain("suppressed");
|
||||
});
|
||||
|
||||
test("an oversized allowlist is ignored and the scan stays strict", () => {
|
||||
const base = git(["rev-parse", "HEAD"]);
|
||||
commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key");
|
||||
const head = commitAllowlist(FAKE_AWS_KEY + "\n" + "#".repeat(65 * 1024) + "\n");
|
||||
const { code, stderr } = runHook(pushOf(base, head));
|
||||
expect(code).toBe(1);
|
||||
expect(stderr).toContain("is over 65536 bytes; ignoring it");
|
||||
});
|
||||
|
||||
test("entries cover MEDIUM emails alongside gstack.redact.allowEmail", () => {
|
||||
const base = git(["rev-parse", "HEAD"]);
|
||||
git(["config", "--add", "gstack.redact.allowEmail", "carol@corp.io"]);
|
||||
commit("notes.md", "contact bob@corp.io or carol@corp.io or dave@corp.io\n", "contacts");
|
||||
const head = commitAllowlist(" bob@corp.io \n");
|
||||
const { code, stderr } = runHook(pushOf(base, head));
|
||||
expect(code).toBe(0);
|
||||
expect(stderr).toContain("MEDIUM pii.email notes.md:1");
|
||||
expect(stderr).toContain("suppressed 2 finding(s)");
|
||||
expect(stderr).toContain("1 MEDIUM finding(s) in pushed diff");
|
||||
});
|
||||
|
||||
test("the block message names the allowlist without one present", () => {
|
||||
const base = git(["rev-parse", "HEAD"]);
|
||||
const head = commit("config.txt", "key " + FAKE_AWS_KEY + "\n", "add key");
|
||||
const { code, stderr } = runHook(pushOf(base, head));
|
||||
expect(code).toBe(1);
|
||||
expect(stderr).toContain("commit its exact matched text as a line of .gstack-redact-allowlist");
|
||||
expect(stderr).not.toContain("suppressed");
|
||||
});
|
||||
});
|
||||
|
||||
describe("diff direction + special refs", () => {
|
||||
test("only NEW content is scanned (remote..local), not pre-existing", () => {
|
||||
// Put a secret in the FIRST commit (already on remote), then push a clean commit.
|
||||
|
||||
Reference in new issue
Block a user