mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-01 06:24:41 +01:00
Merge lane wave/redaction
This commit is contained in:
commit
c05c796bc9
8 files changed
+440
-18
No files matched your search
+171
-14
@@ -253,6 +253,95 @@ export function looksLikeParcelId(span: string, match: RegExpExecArray): boolean
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Vector geometry reads as a phone number to pii.phone.e164 (#2885, #2827).
|
||||
* The pattern accepts `.` as a group separator, so a single float
|
||||
* (`viewBox="0 0 581.66796875 695.65625"`) parses as 581 · 6679 · 6875, and a
|
||||
* run of path coordinates (`37.6188 101.694`, `100 64.5326 100`) as four
|
||||
* spaced groups. One rendered /diagram SVG carried 18 of these; a Figma icon
|
||||
* carries dozens per path.
|
||||
*
|
||||
* Phone conventions put a separator between EVERY group, so a token with
|
||||
* digits, one dot and digits is a decimal number, never a dotted phone
|
||||
* (415.555.0123 has two dots in one token and stays flagged). Exempt a span
|
||||
* whose space-separated tokens include at least one decimal and none with two
|
||||
* dots. A leading `+` is the E.164 marker, so a span starting with it is a
|
||||
* phone context and never exempt.
|
||||
*/
|
||||
export function looksLikeDecimalCoordinates(span: string): boolean {
|
||||
if (span.startsWith("+")) return false;
|
||||
const tokens = span.split(" ");
|
||||
if (tokens.some((t) => t.split(".").length > 2)) return false;
|
||||
return tokens.some((t) => /^\d+\.\d+$/.test(t));
|
||||
}
|
||||
|
||||
/** Span start/end in `match.input`, derived exactly as redact-engine.ts does. */
|
||||
function spanBounds(match: RegExpExecArray): { start: number; end: number } {
|
||||
const spanStartInMatch = match[1] !== undefined ? match[0].indexOf(match[1]) : 0;
|
||||
const start = match.index + Math.max(0, spanStartInMatch);
|
||||
return { start, end: start + (match[1] ?? match[0]).length };
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a digit span is one side of a decimal number: `<digit>.` sits
|
||||
* immediately before it or `.<digit>` immediately after. pii.cc's `\b` stops
|
||||
* at the dot, so the 14-digit fraction of `492.34399999999994` in an
|
||||
* .excalidraw scene is a Luhn candidate, and a random digit run passes Luhn
|
||||
* about one time in ten (#2827). A card number in prose is never glued to a
|
||||
* decimal point; a sentence-ending period has no digit after it and stays
|
||||
* flagged.
|
||||
*/
|
||||
export function insideDecimalNumber(match: RegExpExecArray): boolean {
|
||||
const input = match.input ?? "";
|
||||
const { start, end } = spanBounds(match);
|
||||
const digit = (i: number) => i >= 0 && i < input.length && input[i] >= "0" && input[i] <= "9";
|
||||
return (input[start - 1] === "." && digit(start - 2)) || (input[end] === "." && digit(end + 1));
|
||||
}
|
||||
|
||||
/**
|
||||
* JSON keys whose unquoted integer value is a random seed, nonce or epoch
|
||||
* timestamp. Excalidraw writes three per element (`"seed":1808177121`,
|
||||
* `"versionNonce":1365644783`, `"updated":1791059590857`), and pii.phone.e164
|
||||
* reads each as a bare 10-13 digit number: 92 per /diagram scene (#2827).
|
||||
*/
|
||||
const NUMERIC_METADATA_KEY =
|
||||
/"[A-Za-z0-9_]*(?:seed|nonce|updated|created|timestamp)(?:at|_at|ms|_ms)?"[ \t]*:[ \t]*$/i;
|
||||
|
||||
/**
|
||||
* True when a digit-only span is the unquoted JSON value of a seed, nonce or
|
||||
* timestamp key. The evidence is the key, not the digits: the same number
|
||||
* under `"phone":` still reports.
|
||||
*/
|
||||
export function isNumericMetadataValue(span: string, match: RegExpExecArray): boolean {
|
||||
if (!/^\d+$/.test(span)) return false;
|
||||
const input = match.input ?? "";
|
||||
const { start } = spanBounds(match);
|
||||
const lineStart = input.lastIndexOf("\n", start - 1) + 1;
|
||||
return NUMERIC_METADATA_KEY.test(input.slice(Math.max(lineStart, start - 80), start));
|
||||
}
|
||||
|
||||
/**
|
||||
* A four-part version (MAJOR.MINOR.PATCH.BUILD: .NET assembly versions,
|
||||
* gstack's own VERSION) is byte-for-byte a dotted quad, and `1.128.1.0` is a
|
||||
* public address, so `"version": "1.128.1.0"` raised pii.ip_public on every
|
||||
* release (#2784). The value cannot decide it, so only the declaration
|
||||
* immediately before it on the same line can: a `version` key or word
|
||||
* (`"version": "`, `app_version = `, `<Version>`, `AssemblyVersion("`,
|
||||
* `version `), or a Keep a Changelog heading `## [1.2.3.4]`. The same digits
|
||||
* after `host:`, `server = ` or in prose that merely mentions a version still
|
||||
* report, and the pre-push hook's VERSION-file rule (#2856) is unchanged.
|
||||
*/
|
||||
const VERSION_DECLARATION_BEFORE =
|
||||
/(?:\b[Vv]ersion|\bVERSION|[a-z0-9_]Version|[_-][Vv]ersion|_VERSION)["'\]]?(?:[ \t]*[:=(>][ \t]*|[ \t]+)["']?$/;
|
||||
const CHANGELOG_HEADING_BEFORE = /^#{1,6}[ \t]+\[$/;
|
||||
export function isDeclaredVersion(match: RegExpExecArray): boolean {
|
||||
const input = match.input ?? "";
|
||||
const { start, end } = spanBounds(match);
|
||||
const lineStart = input.lastIndexOf("\n", start - 1) + 1;
|
||||
if (VERSION_DECLARATION_BEFORE.test(input.slice(Math.max(lineStart, start - 48), start))) return true;
|
||||
return CHANGELOG_HEADING_BEFORE.test(input.slice(lineStart, start)) && input[end] === "]";
|
||||
}
|
||||
|
||||
// ── Placeholder suppression (per-matched-span, NOT per-line) ─────────────────
|
||||
|
||||
/**
|
||||
@@ -325,6 +414,28 @@ export function isDotenvFilename(match: RegExpExecArray): boolean {
|
||||
return spanStart > 0 && input[spanStart - 1] === ".";
|
||||
}
|
||||
|
||||
/** Extensions that mark a `<name>.local.<ext>` span as a file. Closed on
|
||||
* purpose: `.md` is also Moldova's TLD, but no internal suffix sits in front
|
||||
* of a public one, so after `.local` it can only be an extension. */
|
||||
const LOCAL_CONFIG_EXTENSIONS = /^\.(?:md|json|jsonc|ya?ml|toml|ini|conf|txt)\b/i;
|
||||
|
||||
/**
|
||||
* True when an `internal.hostname` span is the stem of a per-machine config
|
||||
* FILENAME (`CLAUDE.local.md`, `settings.local.json`, `values.staging.yaml`)
|
||||
* rather than a host (#2962). The pattern's `\b` stops at the dot before the
|
||||
* extension, so `CLAUDE.local.md` reported `CLAUDE.local`. Claude Code's own
|
||||
* per-user files are named this way, so any decision or doc that mentions
|
||||
* them failed closed in the non-interactive stores.
|
||||
*
|
||||
* Exempts ONLY a span immediately followed by `.<known extension>` and a word
|
||||
* boundary. `printer.local.` at the end of a sentence still reports, and so
|
||||
* do `printer.local`, `printer.local/md` and `printer.local.mdx`.
|
||||
*/
|
||||
export function isLocalConfigFilename(match: RegExpExecArray): boolean {
|
||||
const { end } = spanBounds(match);
|
||||
return LOCAL_CONFIG_EXTENSIONS.test((match.input ?? "").slice(end, end + 8));
|
||||
}
|
||||
|
||||
/**
|
||||
* True when the matched span sits ENTIRELY inside a UUID.
|
||||
*
|
||||
@@ -410,15 +521,52 @@ function urlPasswordIsPlaceholder(span: string): boolean {
|
||||
return PLACEHOLDER_STRUCTURAL.some((re) => re.test(pw));
|
||||
}
|
||||
|
||||
/**
|
||||
* #2913: `postgres:postgres` is the official postgres image's default pair,
|
||||
* committed in compose files and CI DATABASE_URLs. It is exempt only on a
|
||||
* loopback host (`localhost` or `127.0.0.1`, optional port), never on a
|
||||
* single-label host such as `db`: a compose service name says nothing about
|
||||
* where the same URL is also deployed, and a weak default on a reachable host
|
||||
* is exactly the leak to report. Exact user and password, case-sensitive;
|
||||
* the host must end at a port, path, query, quote or the span's end.
|
||||
*/
|
||||
const POSTGRES_LOOPBACK_DEFAULT =
|
||||
/^postgres(?:ql)?:\/\/postgres:postgres@(?:localhost|127\.0\.0\.1)(?::\d{1,5})?(?![\w.:@%-])/;
|
||||
|
||||
/** A value span that is only an environment-variable read expression (#2912). */
|
||||
const ENV_READ_SPAN =
|
||||
/^(?:os\.environ\[|os\.environ\.get\(|os\.getenv\(|getenv\(|ENV\[|process\.env\.[A-Za-z_$][\w$]*[;,)]?)$/;
|
||||
function isBareEnvRead(span: string, match: RegExpExecArray): boolean {
|
||||
if (!ENV_READ_SPAN.test(span)) return false;
|
||||
const rest = match.input.slice(match.index + match[0].length).split("\n", 1)[0];
|
||||
for (const [, literal] of rest.matchAll(/["']([^\s'"]{8,})["']/g))
|
||||
if (!isPlaceholderSpan(literal) && shannonEntropy(literal) >= 3.0) return false;
|
||||
return true;
|
||||
return !carriesSecretLiteral(match.input.slice(match.index + match[0].length).split("\n", 1)[0]);
|
||||
}
|
||||
|
||||
/** True when `text` holds a quoted, non-placeholder, high-entropy literal. */
|
||||
function carriesSecretLiteral(text: string): boolean {
|
||||
for (const [, literal] of text.matchAll(/["']([^\s'"]{8,})["']/g))
|
||||
if (!isPlaceholderSpan(literal) && shannonEntropy(literal) >= 3.0) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* #2899: the value capture is any non-space run, so it swallows code:
|
||||
* `session = _FlakySession(responses=[...])`, `token = make_token(user,`,
|
||||
* `password = getpass.getpass()`. A bare credential name plus mixed-case code
|
||||
* clears the entropy gate. The span alone cannot tell `Abc123(xyz` (a real
|
||||
* password) from a call, so the exemption needs the line's evidence: the
|
||||
* value is unquoted, starts with an identifier or dotted path followed by
|
||||
* `(`, and that call closes on the line or opens a multi-line argument list.
|
||||
* A quoted value is a literal, and a call whose arguments carry a
|
||||
* high-entropy literal (`decrypt("<secret>")`) still reports.
|
||||
*/
|
||||
const CALL_SHAPED_VALUE = /^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*\(/;
|
||||
function isCallExpression(span: string, match: RegExpExecArray): boolean {
|
||||
if (!CALL_SHAPED_VALUE.test(span)) return false;
|
||||
const { start } = spanBounds(match);
|
||||
if (match.input[start - 1] === '"' || match.input[start - 1] === "'") return false;
|
||||
const call = match.input.slice(start + span.indexOf("(")).split("\n", 1)[0];
|
||||
if (!call.includes(")") && call.trim() !== "(") return false;
|
||||
return !carriesSecretLiteral(call);
|
||||
}
|
||||
|
||||
export const PATTERNS: RedactPattern[] = [
|
||||
@@ -631,8 +779,9 @@ export const PATTERNS: RedactPattern[] = [
|
||||
category: "secret",
|
||||
description: "Database URL with embedded password",
|
||||
regex: /\b((?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis|amqp):\/\/[^:\s/@]+:[^@\s/]+@[^\s/]+)/,
|
||||
// Skip when the password segment is itself a placeholder/interpolation.
|
||||
validate: (span) => !urlPasswordIsPlaceholder(span),
|
||||
// Skip when the password segment is itself a placeholder/interpolation,
|
||||
// or the URL is postgres's default pair on a loopback host.
|
||||
validate: (span) => !urlPasswordIsPlaceholder(span) && !POSTGRES_LOOPBACK_DEFAULT.test(span),
|
||||
},
|
||||
{
|
||||
id: "creds.basic_auth_url",
|
||||
@@ -693,11 +842,14 @@ export const PATTERNS: RedactPattern[] = [
|
||||
// holding one, unless the rest of its line carries a high-entropy quoted
|
||||
// literal (`os.getenv("X", "<secret>")`). A literal appended to the read
|
||||
// itself (`process.env.X||"…"`) is not an exact read and still fires.
|
||||
// #2899: a function call assigned to the name is code, not a value (see
|
||||
// isCallExpression).
|
||||
validate: (span, match) =>
|
||||
isCredentialShapedEnvName(match[0]) &&
|
||||
!isPlaceholderSpan(span) &&
|
||||
!/^\$\{?[A-Za-z_]/.test(span) &&
|
||||
!isBareEnvRead(span, match) &&
|
||||
!isCallExpression(span, match) &&
|
||||
shannonEntropy(span) >= 3.0,
|
||||
},
|
||||
{
|
||||
@@ -738,12 +890,16 @@ export const PATTERNS: RedactPattern[] = [
|
||||
autoRedactable: true,
|
||||
redactToken: "<REDACTED-PHONE>",
|
||||
// A digit-only UUID's hyphen groups read as national phone formatting, and
|
||||
// so does a county tax-map parcel ID (see looksLikeParcelId).
|
||||
// so do a county tax-map parcel ID (see looksLikeParcelId), vector
|
||||
// coordinates (looksLikeDecimalCoordinates) and seed/nonce/timestamp JSON
|
||||
// values (isNumericMetadataValue).
|
||||
validate: (span, match) =>
|
||||
!insideUuid(match) &&
|
||||
span.replace(/\D/g, "").length >= 10 &&
|
||||
!looksLikeCompactTimestamp(span) &&
|
||||
!looksLikeParcelId(span, match),
|
||||
!looksLikeParcelId(span, match) &&
|
||||
!looksLikeDecimalCoordinates(span) &&
|
||||
!isNumericMetadataValue(span, match),
|
||||
},
|
||||
{
|
||||
id: "pii.ssn",
|
||||
@@ -767,9 +923,9 @@ export const PATTERNS: RedactPattern[] = [
|
||||
regex: /\b((?:\d[ \-]?){13,19})\b/,
|
||||
autoRedactable: true,
|
||||
redactToken: "<REDACTED-CC>",
|
||||
// A 13-19 digit slice of a digit-only UUID passes Luhn often enough to
|
||||
// matter; the enclosing-UUID check runs first so it never reaches Luhn.
|
||||
validate: (span, match) => !insideUuid(match) && luhnValid(span),
|
||||
// A 13-19 digit slice of a digit-only UUID or of a decimal number passes
|
||||
// Luhn often enough to matter; both context checks run before Luhn.
|
||||
validate: (span, match) => !insideUuid(match) && !insideDecimalNumber(match) && luhnValid(span),
|
||||
},
|
||||
{
|
||||
id: "pii.ip_public",
|
||||
@@ -777,7 +933,7 @@ export const PATTERNS: RedactPattern[] = [
|
||||
category: "pii",
|
||||
description: "Public IPv4 address",
|
||||
regex: /\b(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\b/,
|
||||
validate: (span) => isPublicIPv4(span),
|
||||
validate: (span, match) => isPublicIPv4(span) && !isDeclaredVersion(match),
|
||||
},
|
||||
{
|
||||
id: "pii.wallet",
|
||||
@@ -795,8 +951,9 @@ export const PATTERNS: RedactPattern[] = [
|
||||
category: "internal",
|
||||
description: "Internal hostname (*.internal/.corp/.local/.prod/.staging)",
|
||||
regex: /\b([a-z0-9][a-z0-9\-]*\.(?:internal|corp|local|lan|prod|staging))\b/i,
|
||||
// `.env.local` and friends are filenames, not hosts. See isDotenvFilename.
|
||||
validate: (_span, match) => !isDotenvFilename(match),
|
||||
// `.env.local`, `CLAUDE.local.md` and friends are filenames, not hosts.
|
||||
// See isDotenvFilename and isLocalConfigFilename.
|
||||
validate: (_span, match) => !isDotenvFilename(match) && !isLocalConfigFilename(match),
|
||||
},
|
||||
{
|
||||
id: "internal.url_private",
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{"type":"excalidraw","version":2,"source":"gstack-diagram-render","elements":[{"id":"FBmcfSimwEXPzs4bhpTjb","type":"rectangle","x":250.9140625,"y":8,"width":200.109375,"height":60,"angle":0,"strokeColor":"#1e1e1e","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":2,"strokeStyle":"solid","roughness":1,"opacity":100,"groupIds":[],"frameId":null,"index":"a0","roundness":null,"seed":1808177121,"version":3,"versionNonce":1365644783,"isDeleted":false,"boundElements":[{"type":"text","id":"Lk54Qp2D8qFO6X1AMxkXn"},{"id":"RZLjDS-aGR6M6Iha_fygU","type":"arrow"}],"updated":1791059590857,"link":null,"locked":false},{"id":"MgBMziDzzZsUR_ubuPqT3","type":"diamond","x":252.53125,"y":118,"width":197.875,"height":197.875,"angle":0,"strokeColor":"#1e1e1e","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":2,"strokeStyle":"solid","roughness":1,"opacity":100,"groupIds":[],"frameId":null,"index":"a1","roundness":null,"seed":192418479,"version":7,"versionNonce":1049487471,"isDeleted":false,"boundElements":[{"type":"text","id":"DfnkKci8j5sTfUPTqNM_V"},{"id":"RZLjDS-aGR6M6Iha_fygU","type":"arrow"},{"id":"QXtppr3wxF8aWGavgvHCg","type":"arrow"},{"id":"QkuvMiAt5owwKfVGV76Xe","type":"arrow"},{"id":"LwAd4VAY-m3kpx0mf8IZQ","type":"arrow"},{"id":"Xolrx0c730eIssJQTIHut","type":"arrow"}],"updated":1791059590859,"link":null,"locked":false},{"id":"GLaB0IIg5dTU6wbqLh57a","type":"rectangle","x":34.6796875,"y":395.875,"width":157.84375,"height":60,"angle":0,"strokeColor":"#1e1e1e","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":2,"strokeStyle":"solid","roughness":1,"opacity":100,"groupIds":[],"frameId":null,"index":"a2","roundness":null,"seed":365622209,"version":4,"versionNonce":1018495631,"isDeleted":false,"boundElements":[{"type":"text","id":"gqq0ycD8jE4DlXZeLikWp"},{"id":"QXtppr3wxF8aWGavgvHCg","type":"arrow"},{"id":"XUcATSTOjMVv5YvlIR8XP","type":"arrow"}],"updated":1791059590859,"link":null,"locked":false},{"id":"oWWs0W96aNRHAgR5oWl8u","type":"rectangle","x":287.359375,"y":395.875,"width":107.046875,"height":60,"angle":0,"strokeColor":"#1e1e1e","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":2,"strokeStyle":"solid","roughness":1,"opacity":100,"groupIds":[],"frameId":null,"index":"a3","roundness":null,"seed":929031375,"version":4,"versionNonce":1091537231,"isDeleted":false,"boundElements":[{"type":"text","id":"Xgc-Z9fR_xJzSZi552hrg"},{"id":"QkuvMiAt5owwKfVGV76Xe","type":"arrow"},{"id":"m0batfkyDXiNEcdJd9Z-4","type":"arrow"}],"updated":1791059590859,"link":null,"locked":false},{"id":"302jWUU1dMxH1SbQS0qE1","type":"rectangle","x":459.2265625,"y":759.234375,"width":193.421875,"height":60,"angle":0,"strokeColor":"#1e1e1e","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":2,"strokeStyle":"solid","roughness":1,"opacity":100,"groupIds":[],"frameId":null,"index":"a4","roundness":null,"seed":1519965089,"version":4,"versionNonce":1686432207,"isDeleted":false,"boundElements":[{"type":"text","id":"uP6KE-R9_gMd_Z2R1Prra"},{"id":"LwAd4VAY-m3kpx0mf8IZQ","type":"arrow"},{"id":"pRAl45BZ1BxcuUmJ9HD9b","type":"arrow"}],"updated":1791059590859,"link":null,"locked":false},{"id":"dF2UjK6ZPpz78dWKN5evK","type":"diamond","x":269.703125,"y":535.875,"width":143.359375,"height":143.359375,"angle":0,"strokeColor":"#1e1e1e","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":2,"strokeStyle":"solid","roughness":1,"opacity":100,"groupIds":[],"frameId":null,"index":"a5","roundness":null,"seed":1455977199,"version":5,"versionNonce":1134690831,"isDeleted":false,"boundElements":[{"type":"text","id":"RXs9Ti_5xCHis2KHx4duA"},{"id":"m0batfkyDXiNEcdJd9Z-4","type":"arrow"},{"id":"pRAl45BZ1BxcuUmJ9HD9b","type":"arrow"},{"id":"blZCFHehmGZ-PV5jiIJ5p","type":"arrow"}],"updated":1791059590859,"link":null,"locked":false},{"id":"gnjCVoiODEbhYmxyLsgeZ","type":"rectangle","x":174.15625,"y":759.234375,"width":126.34375,"height":60,"angle":0,"strokeColor":"#1e1e1e","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":2,"strokeStyle":"solid","roughness":1,"opacity":100,"groupIds":[],"frameId":null,"index":"a6","roundness":null,"seed":778024833,"version":5,"versionNonce":785037551,"isDeleted":false,"boundElements":[{"type":"text","id":"VAHHwE9eJhL_zD4TcnCdz"},{"id":"blZCFHehmGZ-PV5jiIJ5p","type":"arrow"},{"id":"Xolrx0c730eIssJQTIHut","type":"arrow"},{"id":"Xqf-Ryhsfa4wicqw8j_Y5","type":"arrow"}],"updated":1791059590860,"link":null,"locked":false},{"id":"qYkvsRVNy2LSpOyxA5FwA","type":"rectangle","x":8,"y":577.5546875,"width":211.203125,"height":60,"angle":0,"strokeColor":"#1e1e1e","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":2,"strokeStyle":"solid","roughness":1,"opacity":100,"groupIds":[],"frameId":null,"index":"a7","roundness":null,"seed":842677519,"version":4,"versionNonce":1135798991,"isDeleted":false,"boundElements":[{"type":"text","id":"PUslv2xht6rBVOKD0qYiZ"},{"id":"XUcATSTOjMVv5YvlIR8XP","type":"arrow"},{"id":"Xqf-Ryhsfa4wicqw8j_Y5","type":"arrow"}],"updatedLine truncated
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
flowchart TD
|
||||
A[Push requested] --> B{Redaction scan}
|
||||
B -->|HIGH| C[Block push]
|
||||
B -->|MEDIUM| D[Warn]
|
||||
B -->|clean| E[Push proceeds]
|
||||
D --> F{Reviewer}
|
||||
F -->|accept| E
|
||||
F -->|fix| G[Edit diff]
|
||||
G --> B
|
||||
C --> H[Rotate credential]
|
||||
H --> G
|
||||
+1
@@ -0,0 +1 @@
|
||||
<svg id="diagram-1" width="100%" xmlns="http://www.w3.org/2000/svg" class="flowchart" style="max-width: 581.66796875px;" viewBox="0 0 581.66796875 695.65625" role="graphics-document document" aria-roledescription="flowchart-v2"><style>#diagram-1{font-family:Helvetica,"Liberation Sans",Arial,"Hiragino Kaku Gothic ProN","Noto Sans CJK JP","Microsoft YaHei","Apple Color Emoji","Segoe UI Emoji","Noto Color Emoji",sans-serif;font-size:16px;fill:#000000;}@keyframes edge-animation-frame{from{stroke-dashoffset:0;}}@keyframes dash{to{stroke-dashoffset:0;}}#diagram-1 .edge-animation-slow{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 50s linear infinite;stroke-linecap:round;}#diagram-1 .edge-animation-fast{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 20s linear infinite;stroke-linecap:round;}#diagram-1 .error-icon{fill:#552222;}#diagram-1 .error-text{fill:#552222;stroke:#552222;}#diagram-1 .edge-thickness-normal{stroke-width:1px;}#diagram-1 .edge-thickness-thick{stroke-width:3.5px;}#diagram-1 .edge-pattern-solid{stroke-dasharray:0;}#diagram-1 .edge-thickness-invisible{stroke-width:0;fill:none;}#diagram-1 .edge-pattern-dashed{stroke-dasharray:3;}#diagram-1 .edge-pattern-dotted{stroke-dasharray:2;}#diagram-1 .marker{fill:#666;stroke:#666;}#diagram-1 .marker.cross{stroke:#666;}#diagram-1 svg{font-family:Helvetica,"Liberation Sans",Arial,"Hiragino Kaku Gothic ProN","Noto Sans CJK JP","Microsoft YaHei","Apple Color Emoji","Segoe UI Emoji","Noto Color Emoji",sans-serif;font-size:16px;}#diagram-1 p{margin:0;}#diagram-1 .label{font-family:Helvetica,"Liberation Sans",Arial,"Hiragino Kaku Gothic ProN","Noto Sans CJK JP","Microsoft YaHei","Apple Color Emoji","Segoe UI Emoji","Noto Color Emoji",sans-serif;color:#000000;}#diagram-1 .cluster-label text{fill:#333;}#diagram-1 .cluster-label span{color:#333;}#diagram-1 .cluster-label span p{background-color:transparent;}#diagram-1 .label text,#diagram-1 span{fill:#000000;color:#000000;}#diagram-1 .node rect,#diagram-1 .node circle,#diagram-1 .node ellipse,#diagram-1 .node polygon,#diagram-1 .node path{fill:#eee;stroke:#999;stroke-width:1px;}#diagram-1 .rough-node .label text,#diagram-1 .node .label text,#diagram-1 .image-shape .label,#diagram-1 .icon-shape .label{text-anchor:middle;}#diagram-1 .node .katex path{fill:#000;stroke:#000;stroke-width:1px;}#diagram-1 .rough-node .label,#diagram-1 .node .label,#diagram-1 .image-shape .label,#diagram-1 .icon-shape .label{text-align:center;}#diagram-1 .node.clickable{cursor:pointer;}#diagram-1 .root .anchor path{fill:#666!important;stroke-width:0;stroke:#666;}#diagram-1 .arrowheadPath{fill:#333333;}#diagram-1 .edgePath .path{stroke:#666;stroke-width:1px;}#diagram-1 .flowchart-link{stroke:#666;fill:none;}#diagram-1 .edgeLabel{background-color:white;text-align:center;}#diagram-1 .edgeLabel p{background-color:white;}#diagram-1 .edgeLabel rect{opacity:0.5;background-color:white;fill:white;}#diagram-1 .labelBkg{background-color:rgba(255, 255, 255, 0.5);}#diagram-1 .cluster rect{fill:hsl(0, 0%, 98.9215686275%);stroke:#707070;stroke-width:1px;}#diagram-1 .cluster text{fill:#333;}#diagram-1 .cluster span{color:#333;}#diagram-1 div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:Helvetica,"Liberation Sans",Arial,"Hiragino Kaku Gothic ProN","Noto Sans CJK JP","Microsoft YaHei","Apple Color Emoji","Segoe UI Emoji","Noto Color Emoji",sans-serif;font-size:12px;background:hsl(-160, 0%, 93.3333333333%);border:1px solid #707070;border-radius:2px;pointer-events:none;z-index:100;}#diagram-1 .flowchartTitleText{text-anchor:middle;font-size:18px;fill:#000000;}#diagram-1 rect.text{fill:none;stroke-width:0;}#diagram-1 .icon-shape,#diagram-1 .image-shape{background-color:white;text-align:center;}#diagram-1 .icon-shape p,#diagram-1 .image-shape p{background-color:white;padding:2px;}#diagram-1 .icon-shape .label rect,#diagram-1 .image-shape .label rect{opacity:0.5;background-color:white;fill:white;}#diagram-1 .label-icon{display:inline-block;height:1em;overflow:visible;vertical-align:-0.125em;}#diagram-1 .node .label-icon path{fill:currentColor;stroke:revert;stroke-width:revert;}#diagram-1 .node .neo-node{stroke:#999;}#diagram-1 [data-look="neo"].node rect,#diagram-1 [data-look="neo"].cluster rect,#diagram-1 [data-look="neo"].node polygon{stroke:url(#diagram-1-gradient);filter:drop-shadow( 1px 2px 2px rgba(185,185,185,1));}#diagram-1 [data-look="neo"].swimlane.cluster rect{filter:none;}#diagram-1 [data-look="neo"].node path{stroke:url(#diagram-1-gradient);stroke-width:1px;}#diagram-1 [data-look="neo"].node .outer-path{filter:drop-shadow( 1px 2px 2px rgba(185,185,185,1));}#diagram-1 [data-look="neo"].node .neo-line path{stroke:#999;filter:none;}#diagram-1 [data-look="neo"].node circle{stroke:url(#diagram-1-gradient);filter:drop-shadow( 1px 2px 2px rgba(185,185,185,1));}#diagram-1 [data-look="neo"].node circle .state-start{fill:#000000;}#diagram-1 [data-look="neo"].icon-shape .icon{Line truncated
|
||||
|
After Width: | Height: | Size: 25 KiB |
@@ -144,6 +144,28 @@ describe('buildHermeticEnv allowlist', () => {
|
||||
]) expect(result[name]).toBe(base[name]);
|
||||
});
|
||||
|
||||
test('a trailing qualifier does not carry a credential past the prefix rule (#2949)', () => {
|
||||
const qualified = {
|
||||
GITHUB_APP_PRIVATE_KEY_BASE64: 'synthetic-pem-base64',
|
||||
GITHUB_PRIVATE_KEY_PEM: 'synthetic-pem',
|
||||
GITHUB_TOKEN_1: 'synthetic-first-token',
|
||||
GITHUB_TOKEN_GHES: 'synthetic-enterprise-token',
|
||||
GITHUB_CLIENT_SECRET_VALUE: 'synthetic-client-secret',
|
||||
EVALS_API_KEY_FALLBACK: 'synthetic-eval-key',
|
||||
};
|
||||
const result = buildHermeticEnv({ ...CONTAMINATED, ...qualified }, HERMETIC_VARS);
|
||||
for (const name of Object.keys(qualified)) expect(result[name]).toBeUndefined();
|
||||
const serialized = JSON.stringify(result);
|
||||
for (const value of Object.values(qualified)) expect(serialized.includes(value)).toBe(false);
|
||||
|
||||
// Segments, not substrings: names that merely contain a credential word
|
||||
// stay metadata, and an explicit runner admission still wins.
|
||||
const metadata = { GITHUB_PATH: '/tmp/p', GITHUB_TOKENIZER: 't', GITHUB_KEYRING: 'k', GITHUB_REF_PROTECTED: 'false', EVALS_RUN_ID: 'r' };
|
||||
const kept = buildHermeticEnv({ ...CONTAMINATED, ...metadata, ...qualified }, HERMETIC_VARS, undefined, { extraAllow: ['GITHUB_TOKEN_1'] });
|
||||
for (const [name, value] of Object.entries(metadata)) expect(kept[name]).toBe(value);
|
||||
expect(kept.GITHUB_TOKEN_1).toBe(qualified.GITHUB_TOKEN_1);
|
||||
});
|
||||
|
||||
test('explicit provider auth, runner admissions, and overrides still win', () => {
|
||||
const base = {
|
||||
...CONTAMINATED,
|
||||
|
||||
@@ -71,15 +71,27 @@ const ALLOW_EXACT = new Set([
|
||||
* VOYAGE_API_KEY) — CI doesn't have them and eval children have no business
|
||||
* using them. A test that legitimately needs one opts in via its own env
|
||||
* override; a provider runner (codex/gemini) re-admits its auth vars via
|
||||
* opts.extraAllow. Prefix matches reject credential-shaped suffixes; exact
|
||||
* opts.extraAllow. Prefix matches reject credential-shaped segments; exact
|
||||
* and explicit runner admissions still win. */
|
||||
const ALLOW_PREFIXES = ['EVALS_', 'GITHUB_'];
|
||||
const CREDENTIAL_SUFFIXES = new Set([
|
||||
const CREDENTIAL_SEGMENTS = new Set([
|
||||
'KEY', 'KEYS', 'TOKEN', 'TOKENS', 'SECRET', 'SECRETS', 'PASSWORD', 'PASSWD',
|
||||
'PASS', 'CREDENTIAL', 'CREDENTIALS', 'AUTH', 'PAT', 'DSN', 'COOKIE',
|
||||
'SESSION', 'PRIVATE',
|
||||
]);
|
||||
|
||||
/**
|
||||
* True when any underscore-separated segment of `name` is a credential word.
|
||||
* Every segment, not just the last: a trailing qualifier moves the credential
|
||||
* word off the end (`GITHUB_APP_PRIVATE_KEY_BASE64` is the PEM itself,
|
||||
* `GITHUB_TOKEN_1` is a token). Segments, not substrings: `GITHUB_PATH`
|
||||
* contains "PAT" and `GITHUB_TOKENIZER` contains "TOKEN", and both are
|
||||
* metadata.
|
||||
*/
|
||||
function isCredentialShapedName(name: string): boolean {
|
||||
return name.toUpperCase().split('_').some((segment) => CREDENTIAL_SEGMENTS.has(segment));
|
||||
}
|
||||
|
||||
export interface HermeticEnvOpts {
|
||||
/** Per-runner additional allowed names (exact match) or prefixes (entries
|
||||
* ending in '*'). Example: codex runner passes ['OPENAI_API_KEY', 'CODEX_*']. */
|
||||
@@ -126,8 +138,7 @@ export function buildHermeticEnv(
|
||||
const allowed =
|
||||
ALLOW_EXACT.has(k) ||
|
||||
extraExact.has(k) ||
|
||||
(ALLOW_PREFIXES.some((p) => k.startsWith(p)) &&
|
||||
!CREDENTIAL_SUFFIXES.has(k.slice(k.lastIndexOf('_') + 1).toUpperCase())) ||
|
||||
(ALLOW_PREFIXES.some((p) => k.startsWith(p)) && !isCredentialShapedName(k)) ||
|
||||
extraPrefixes.some((p) => k.startsWith(p));
|
||||
if (allowed) out[k] = v;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
/**
|
||||
* Redaction false positives that blocked pushes and decision logs, each paired
|
||||
* with true-positive controls that must still report at the same tier.
|
||||
*
|
||||
* Every exemption below is decided by context, so each table carries the
|
||||
* benign vector AND the same (or same-shaped) value in a sensitive context.
|
||||
* A guard that passes the benign half by gutting the pattern fails the
|
||||
* control half.
|
||||
*/
|
||||
import { describe, test, expect } from "bun:test";
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { scan, type Finding } from "../lib/redact-engine";
|
||||
|
||||
const FIXTURES = path.join(import.meta.dir, "fixtures", "redact-diagram");
|
||||
|
||||
function findings(text: string): Finding[] {
|
||||
return scan(text, { repoVisibility: "private" }).findings;
|
||||
}
|
||||
function tierOf(text: string, id: string): string | undefined {
|
||||
return findings(text).find((f) => f.id === id)?.severity;
|
||||
}
|
||||
|
||||
function table(id: string, tier: string, benign: string[], controls: string[]): void {
|
||||
for (const input of benign) {
|
||||
test(`clean: ${input.slice(0, 70)}`, () => {
|
||||
expect(findings(input).map((f) => f.id)).not.toContain(id);
|
||||
});
|
||||
}
|
||||
for (const input of controls) {
|
||||
test(`still ${tier}: ${input.slice(0, 70)}`, () => {
|
||||
expect(tierOf(input, id)).toBe(tier);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
describe("pii.phone.e164 / pii.cc: vector geometry is not PII (#2885, #2827)", () => {
|
||||
table(
|
||||
"pii.phone.e164",
|
||||
"MEDIUM",
|
||||
[
|
||||
"M 37.6188 101.694 L 100 64.5326 100",
|
||||
"M50 1C1 2 37.6188 101.694 51.6863 99.4363Z",
|
||||
'viewBox="0 0 581.66796875 695.65625"',
|
||||
'transform="translate(296.1484375, 298.796875)"',
|
||||
"fill:hsl(0, 0%, 98.9215686275%)",
|
||||
"v 12.3456 100.125 7.5",
|
||||
'{"type":"rectangle","x":250.9140625,"y":8}',
|
||||
'{"seed":1808177121,"version":3,"versionNonce":1365644783}',
|
||||
'"updated": 1791059590857,',
|
||||
],
|
||||
[
|
||||
"call me at +1 415 555 0123 tomorrow",
|
||||
"tel: +14155550123",
|
||||
"p 415.555.0123",
|
||||
"tel +1 415.555.0123",
|
||||
"ring +44 20 7946 0958",
|
||||
"phone: (415) 555-0123",
|
||||
"+37.6188 101.694",
|
||||
'{"phone":1808177121}',
|
||||
'"contact": 14155550123,',
|
||||
"seed 1808177121",
|
||||
],
|
||||
);
|
||||
|
||||
table(
|
||||
"pii.cc",
|
||||
"MEDIUM",
|
||||
['"height":492.34399999999994,', '"gap":3.266375000000039}', "0.4111111111111111"],
|
||||
["card 4111 1111 1111 1111", "card 4111111111111111.", "4111111111111111"],
|
||||
);
|
||||
|
||||
test("a rendered /diagram triplet produces no PII findings", () => {
|
||||
for (const name of ["flow.svg", "flow.excalidraw"]) {
|
||||
const text = fs.readFileSync(path.join(FIXTURES, name), "utf8");
|
||||
const pii = findings(text).filter((f) => f.id === "pii.phone.e164" || f.id === "pii.cc");
|
||||
expect({ name, pii: pii.map((f) => f.id) }).toEqual({ name, pii: [] });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("env.kv: a function call assigned to a credential name is code (#2899)", () => {
|
||||
table(
|
||||
"env.kv",
|
||||
"MEDIUM",
|
||||
[
|
||||
' session = _FlakySession(requests.ReadTimeout("Read timed out"))',
|
||||
" session = _FlakySession(responses=[1])",
|
||||
" session = requests.Session()",
|
||||
" session = find_session(client, args.session_id, now=now, state=state)",
|
||||
' token = make_token(user, scopes=["read"])',
|
||||
" password = getpass.getpass()",
|
||||
" session = build_http_session(",
|
||||
],
|
||||
[
|
||||
'session = "Xk9pQ2mLr7Tz4vBn"',
|
||||
"SESSION_SECRET=9f8a7b6c5d4e3f2a1b0c9d8e7f6a5b4c",
|
||||
'password = "Tr0ub4dor&3!xyz"',
|
||||
"PASSWORD=Tr0ub4dor&3!xyz#",
|
||||
"password = correct.horse.battery.staple",
|
||||
'api_key: "Zx9Qw8Er7Ty6Ui5Op4As3Df2"',
|
||||
"API_SECRET=Abc123xyz(9Qm",
|
||||
'password = "getPassword(x)Q9z"',
|
||||
'token = decrypt("Zx9Qw8Er7Ty6Ui5Op4As3Df2")',
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
describe("internal.hostname: per-machine config filenames are not hosts (#2962, #2740)", () => {
|
||||
table(
|
||||
"internal.hostname",
|
||||
"MEDIUM",
|
||||
[
|
||||
"Edit CLAUDE.local.md and settings.local.json",
|
||||
"no `CLAUDE.local.md`, no @import",
|
||||
"edit .claude/settings.local.json",
|
||||
"helm -f values.staging.yaml",
|
||||
"config.prod.toml is read first",
|
||||
"npm run dev -- --env-file=.env.local",
|
||||
"copy .env.local to .env.staging",
|
||||
],
|
||||
[
|
||||
"ssh build.local",
|
||||
"ping printer.local.",
|
||||
"http://printer.local/md",
|
||||
"printer.local.mdx is odd",
|
||||
"curl http://settings.local:8080/health",
|
||||
"the api.corp gateway",
|
||||
"connect to build-7.internal",
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
describe("pii.ip_public: a four-part version needs a version declaration (#2784)", () => {
|
||||
table(
|
||||
"pii.ip_public",
|
||||
"MEDIUM",
|
||||
[
|
||||
' "version": "1.128.1.0",',
|
||||
"app_version = 1.128.1.0",
|
||||
"schema-version: 1.128.1.0",
|
||||
"APP_VERSION=1.128.1.0",
|
||||
"Released version 1.128.1.0 today",
|
||||
"## [1.128.1.0] - 2026-09-02",
|
||||
"<Version>1.128.1.0</Version>",
|
||||
'[assembly: AssemblyVersion("1.0.0.0")]',
|
||||
"released v1.128.1.0 today",
|
||||
],
|
||||
[
|
||||
"1.128.1.0",
|
||||
"host: 1.128.1.0",
|
||||
"server = 1.128.1.0",
|
||||
'"upstream": "1.128.1.0"',
|
||||
"resolver = 8.8.8.8",
|
||||
"version: 1.2.3.4 via 8.8.8.8",
|
||||
"the version endpoint moved to 1.128.1.0",
|
||||
"conversion 1.128.1.0",
|
||||
"## 1.128.1.0 is our egress",
|
||||
"## [1.128.1.0 mirror]",
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
describe("db.url_with_password: postgres:postgres only on a loopback host (#2913)", () => {
|
||||
// Assembled from parts so this file's own source never carries a
|
||||
// credential-bearing URL for the repo's secret scan to block.
|
||||
const url = (userinfo: string, rest: string, scheme = "postgresql") => `${scheme}://${userinfo}@${rest}`;
|
||||
const pg = ["postgres", "postgres"].join(":");
|
||||
table(
|
||||
"db.url_with_password",
|
||||
"HIGH",
|
||||
[
|
||||
`DATABASE_URL=${url(pg, "localhost:5432/app_test")}`,
|
||||
`DATABASE_URL=${url(pg, "127.0.0.1:5432/app", "postgres")}`,
|
||||
`DIRECT_URL="${url(pg, "localhost/app")}"`,
|
||||
`url: ${url(pg, "localhost", "postgres")}`,
|
||||
url(pg, "localhost:5432?sslmode=disable"),
|
||||
],
|
||||
[
|
||||
`DATABASE_URL=${url(pg, "db:5432/app")}`,
|
||||
`DATABASE_URL=${url(pg, "postgres:5432/app")}`,
|
||||
`DATABASE_URL=${url(pg, "prod-host:5432/app")}`,
|
||||
`DATABASE_URL=${url(pg, "localhost.evil.example.net/app")}`,
|
||||
`DATABASE_URL=${url(pg, "127.0.0.1.nip.io/app")}`,
|
||||
`DATABASE_URL=${url(pg, "localhost@prod-db/app")}`,
|
||||
`DATABASE_URL=${url(["user", "s3cretPassw0rd"].join(":"), "prod-host:5432/app")}`,
|
||||
`DATABASE_URL=${url(["postgres", "hunter2x"].join(":"), "localhost:5432/app")}`,
|
||||
`DATABASE_URL=${url(["admin", "postgres"].join(":"), "localhost:5432/app")}`,
|
||||
`DATABASE_URL=${url(["postgres", "Postgres"].join(":"), "localhost:5432/app")}`,
|
||||
`DATABASE_URL=${url(pg, "localhost:3306/app", "mysql")}`,
|
||||
],
|
||||
);
|
||||
});
|
||||
@@ -118,6 +118,32 @@ describe("pre-push hook gating", () => {
|
||||
const RELEASE = ["1", "84", "1", "0"].join(".");
|
||||
const PUBLIC_IP = ["8", "8", "8", "8"].join(".");
|
||||
|
||||
describe("pushed diffs larger than the engine cap are scanned in chunks (#2915)", () => {
|
||||
const commitLarge = (tail: string): { base: string; head: string } => {
|
||||
const base = git(["rev-parse", "HEAD"]);
|
||||
fs.writeFileSync(path.join(repo, "large-a.txt"), "clean-a ".repeat(90_000));
|
||||
fs.writeFileSync(path.join(repo, "large-b.txt"), "clean-b ".repeat(90_000) + tail);
|
||||
git(["add", "large-a.txt", "large-b.txt"]);
|
||||
git(["commit", "-q", "-m", "large diff"]);
|
||||
return { base, head: git(["rev-parse", "HEAD"]) };
|
||||
};
|
||||
|
||||
test("a clean two-file diff over the cap passes without input_too_large", () => {
|
||||
const { base, head } = commitLarge("");
|
||||
const { code, stderr } = runHook(`refs/heads/main ${head} refs/heads/main ${base}\n`);
|
||||
expect(code).toBe(0);
|
||||
expect(stderr).not.toContain("engine.input_too_large");
|
||||
});
|
||||
|
||||
test("a credential in a later file's chunk still blocks", () => {
|
||||
const { base, head } = commitLarge("\n" + FAKE_AWS_KEY + "\n");
|
||||
const { code, stderr } = runHook(`refs/heads/main ${head} refs/heads/main ${base}\n`);
|
||||
expect(code).toBe(1);
|
||||
expect(stderr).toContain("aws.access_key");
|
||||
expect(stderr).not.toContain("engine.input_too_large");
|
||||
});
|
||||
});
|
||||
|
||||
describe("release version exemption (#2856)", () => {
|
||||
const mediums = (stderr: string): number => Number(/(\d+) MEDIUM finding/.exec(stderr)?.[1] ?? 0);
|
||||
const commitAll = (files: Record<string, string>, msg: string): string => {
|
||||
|
||||
Reference in new issue
Block a user