From 693b3be53c5a670cdc9c71bd758fa355685a746d Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 31 Oct 2026 19:44:24 +0000 Subject: [PATCH] fix(redact): one string-span pass for Python/Dart (block comments, raw strings, continuations); a literal fallback after a name still reports (#3630) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- changelog.d/gsta213-3630-env-kv-source.md | 3 +- lib/redact-patterns.ts | 97 +++++++++++++---------- test/redact-engine.test.ts | 11 +++ 3 files changed, 66 insertions(+), 45 deletions(-) diff --git a/changelog.d/gsta213-3630-env-kv-source.md b/changelog.d/gsta213-3630-env-kv-source.md index 087b877ea..4fcc8fbea 100644 --- a/changelog.d/gsta213-3630-env-kv-source.md +++ b/changelog.d/gsta213-3630-env-kv-source.md @@ -6,7 +6,8 @@ - a value that is not an identifier chain; - an annotation with a literal default (`api_token: AuthToken = ""`, now caught in TypeScript too); - a `password:` line inside a Dart or Python `'''`/`"""` string, where escaped quotes don't close the string and a delimiter inside a comment or a one-line string opens nothing; - - a `password: ` pair inside a one-line string, in any of these languages; + - a `password: ` pair inside a one-line Python or Dart string; + - a literal fallback after the name (`"password": settings.db_pass or ""`); - the same lines in `.env` or YAML files. Python's `=` forms (`password=request.password`) still report. diff --git a/lib/redact-patterns.ts b/lib/redact-patterns.ts index 78db77620..24e3fedc1 100644 --- a/lib/redact-patterns.ts +++ b/lib/redact-patterns.ts @@ -687,50 +687,58 @@ const CODE_SOURCE = /\.(?:[cm]?tsx?|jsx|dart|pyi?)$/i; const CODE_NAME_CHAIN = /^[A-Za-z_$][\w$]*(?:\??\.[A-Za-z_$][\w$]*)*(?:<[\w$.,\s<>[\]|]*>)?(?:\[\])*$/; /** - * #3418: a `password:` line in a Dart or Python `'''`/`"""` block is a literal - * in an embedded config, so the name exemption must not reach it. The open - * spans of every such block are computed once per input (a scan validates - * many candidates against the same text). Outside a block, a one-line string - * and a comment (`#` in Python, `//` in Dart) are skipped, so a delimiter - * inside one never opens a block; inside a block, only the delimiter that - * opened it closes it, and a backslash escapes the next character. The - * exemption needs the whole file to see an opener: pregate passes these files + * #3418: a `password:` pair inside a string is text in an embedded config, + * not a dict entry or annotation, so the name exemption must not reach it. + * In Python and Dart (no regex literals, so every quote is a string's), one + * linear pass per input (a scan validates many candidates against the same + * text) records every string's content span: a `'''`/`"""` block, or a + * `'`/`"` string that ends at its line unless a backslash continues it. + * Comments are skipped (`#` in Python; `//` and nested `/* *\/` in Dart), so a + * delimiter inside one opens nothing; a backslash escapes the next character, except in a Dart raw + * string (`r'''…'''`). The exemption needs the + * whole file to see a block's opener: pregate passes Dart and Python files * whole, and the pre-push hook, which scans added lines alone, passes them * without their path. */ -const TRIPLE_QUOTED_SOURCE = /\.(?:dart|pyi?)$/i; -let tripleSpansCache: { text: string; comment: string; spans: Array<[number, number]> } | null = null; +const STRING_SPAN_SOURCE = /\.(?:dart|pyi?)$/i; +let stringSpansCache: { text: string; lang: string; spans: Array<[number, number]> } | null = null; -function tripleQuotedSpans(text: string, comment: string): Array<[number, number]> { - if (tripleSpansCache?.text === text && tripleSpansCache.comment === comment) return tripleSpansCache.spans; +function stringSpans(text: string, lang: "py" | "dart"): Array<[number, number]> { + if (stringSpansCache?.text === text && stringSpansCache.lang === lang) return stringSpansCache.spans; const spans: Array<[number, number]> = []; - let open: string | null = null; - let from = 0; - for (let i = 0; i < text.length; ) { - if (open !== null) { - if (text[i] === "\\") { i += 2; continue; } - if (text.startsWith(open, i)) { spans.push([from, i]); open = null; i += 3; continue; } - i += 1; + const n = text.length; + for (let i = 0; i < n; ) { + const c = text[i]!; + if (lang === "py" ? c === "#" : text.startsWith("//", i)) { + const eol = text.indexOf("\n", i); + i = eol < 0 ? n : eol; continue; } - if (text.startsWith('"""', i) || text.startsWith("'''", i)) { open = text.slice(i, i + 3); from = i + 3; i += 3; continue; } - if (text.startsWith(comment, i)) { const eol = text.indexOf("\n", i); i = eol < 0 ? text.length : eol; continue; } - if (text[i] === '"' || text[i] === "'") { - const quote = text[i]; - let j = i + 1; - while (j < text.length && text[j] !== quote && text[j] !== "\n") j += text[j] === "\\" ? 2 : 1; - i = j + 1; + if (lang === "dart" && text.startsWith("/*", i)) { + let depth = 1; + for (i += 2; i < n && depth > 0; ) { + if (text.startsWith("/*", i)) { depth += 1; i += 2; } + else if (text.startsWith("*/", i)) { depth -= 1; i += 2; } + else i += 1; + } continue; } - i += 1; + if (c !== '"' && c !== "'") { i += 1; continue; } + const close = text.startsWith(c.repeat(3), i) ? c.repeat(3) : c; + const multiline = close.length === 3; + const raw = lang === "dart" && text[i - 1] === "r" && !/[\w$]/.test(text[i - 2] ?? ""); + let j = i + close.length; + while (j < n && !text.startsWith(close, j) && (multiline || text[j] !== "\n")) j += !raw && text[j] === "\\" ? 2 : 1; + spans.push([i + close.length, Math.min(j, n)]); + i = j + close.length; } - if (open !== null) spans.push([from, text.length]); - tripleSpansCache = { text, comment, spans }; + stringSpansCache = { text, lang, spans }; return spans; } -function insideTripleQuotedString(text: string, offset: number, comment: string): boolean { - const spans = tripleQuotedSpans(text, comment); +function insideString(text: string, offset: number, sourcePath: string): boolean { + if (!STRING_SPAN_SOURCE.test(sourcePath)) return false; + const spans = stringSpans(text, /\.dart$/i.test(sourcePath) ? "dart" : "py"); let lo = 0; let hi = spans.length - 1; while (lo <= hi) { @@ -742,17 +750,19 @@ function insideTripleQuotedString(text: string, offset: number, comment: string) return false; } -function insideOneLineString(text: string, offset: number, comment: string): boolean { - let open: string | null = null; - for (let i = text.lastIndexOf("\n", offset - 1) + 1; i < offset; i += 1) { - const c = text[i]!; - if (open !== null) { - if (c === "\\") i += 1; - else if (c === open) open = null; - } else if (text.startsWith(comment, i)) return false; - else if (c === '"' || c === "'" || c === "`") open = c; +function expressionTail(rest: string): string { + let depth = 0; + for (let i = 0; i < rest.length; i += 1) { + const c = rest[i]!; + if (c === '"' || c === "'" || c === "`") { + const close = rest.indexOf(c, i + 1); + if (close < 0) return rest; + i = close; + } else if ("([{".includes(c)) depth += 1; + else if (")]}".includes(c)) { if (--depth < 0) return rest.slice(0, i); } + else if ((c === "," || c === ";") && depth === 0) return rest.slice(0, i); } - return open !== null; + return rest; } function isSourceExpression(span: string, match: RegExpExecArray, sourcePath: string | undefined): boolean { @@ -769,11 +779,10 @@ function isSourceExpression(span: string, match: RegExpExecArray, sourcePath: st const head = match[0].slice(0, match[0].length - span.length).trimEnd(); const separator = head.slice(-1); if (separator !== ":" || !CODE_NAME_CHAIN.test(span.replace(/[,;)=|]+$/, ""))) return false; - if (insideOneLineString(match.input, start, /\.pyi?$/i.test(sourcePath) ? "#" : "//")) return false; const rest = (span.endsWith("=") ? "=" : "") + match.input.slice(end).split("\n", 1)[0]; - if (/^\s*=/.test(rest) && carriesSecretLiteral(rest)) return false; + if (!/[,;]$/.test(span) && carriesSecretLiteral(expressionTail(rest))) return false; } - return !(TRIPLE_QUOTED_SOURCE.test(sourcePath) && insideTripleQuotedString(match.input, start, /\.dart$/i.test(sourcePath) ? "//" : "#")); + return !insideString(match.input, start, sourcePath); } export const PATTERNS: RedactPattern[] = [ diff --git a/test/redact-engine.test.ts b/test/redact-engine.test.ts index d993d2edd..32fe0cb7f 100644 --- a/test/redact-engine.test.ts +++ b/test/redact-engine.test.ts @@ -296,6 +296,10 @@ describe("MEDIUM demoted credential-shaped patterns (TENSION-1)", () => { ["key={workout.id ?? index}", "src/SessionList.jsx"], ["key={item.uuid || item.slug}", "src/List.tsx"], ["key={row.ready && row.id}", "src/Grid.tsx"], + // A quote inside a comment opens nothing. + ["# it's a session\ndef f(\n session: HttpSession,\n): pass", "app.py"], ["/* it's a session */\n password: request.password,", "lib/a.dart"], + // A literal in the next pair on the line belongs to that pair. + [' "session": request.session, "mode": "readwrite-primary",', "app.py"], // Quotes that close before the name leave it code. [" login('it\\'s', password: request.password);", "lib/a.dart"], [' "user": "a", "password": request.password,', "app.py"], // The triple-quote guard is Dart/Python only: a stray ''' in a TS string leaves #3048 as it was. @@ -327,6 +331,13 @@ describe("MEDIUM demoted credential-shaped patterns (TENSION-1)", () => { [`# don't use """ here\nCONFIG = """\npassword: ${vIdent}\n"""`, "tests/conftest.py"], [`DELIM = '"""'\nCONFIG = """\npassword: ${vIdent}\n"""`, "tests/conftest.py"], [`// no ''' here\nconst fixture = '''\npassword: ${vIdent}\n''';`, "test/fixtures.dart"], + [`/* ''' */\nconst fixture = '''\npassword: ${vIdent}\n''';`, "test/fixtures.dart"], + [`/* outer /* ''' */ still a comment ''' */\nconst fixture = '''\npassword: ${vIdent}\n''';`, "test/fixtures.dart"], + [`const dir = r'''C:\\dir\\''';\nconst fixture = '''\npassword: ${vIdent}\n''';`, "test/fixtures.dart"], + // A literal fallback after a name is still a literal. + [` "password": settings.db_pass or "${v}",`, "app.py"], [` 'password': request.password ?? '${v}',`, "lib/a.dart"], + // A one-line string continued by a backslash spans lines. + [`CFG = "db: \\\npassword: ${vIdent} \\\n"`, "app.py"], // A key and value inside one quoted string are text, not a dict entry. [` 'password: ${vIdent} more',`, "lib/a.dart"], [` "password: ${vIdent} more",`, "app.py"], ] as const) expect({ literal, file, kv: kv(literal, file) }).toEqual({ literal, file, kv: true });