From 1524b06742f736dad6e604d7334007aae87e9cfe Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 1 Nov 2026 02:51:40 +0000 Subject: [PATCH 1/8] fix(secrets): Python/Dart string context comes from the diffed text and skips interpolations; pass --diff-base reaches the scoped Red Team (GST-738) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- bin/gstack-review-dispatch | 3 +- changelog.d/gst-738-secrets-context.md | 6 +++ lib/pregate/checks/secrets.ts | 46 +++++++++++++++------ lib/redact-patterns.ts | 45 ++++++++++++++++---- test/pregate-secrets-source-context.test.ts | 18 +++++++- test/redact-engine.test.ts | 21 ++++++++++ test/review-pass.test.ts | 13 +++++- 7 files changed, 127 insertions(+), 25 deletions(-) create mode 100644 changelog.d/gst-738-secrets-context.md diff --git a/bin/gstack-review-dispatch b/bin/gstack-review-dispatch index 827ba0c5b..2174e9979 100755 --- a/bin/gstack-review-dispatch +++ b/bin/gstack-review-dispatch @@ -64,6 +64,7 @@ const HELP = `gstack-review-dispatch — verbatim reviewer briefs and loss-free runs gstack-review-plan and writes /pass-manifest.json: stage first (audit readers marked first in --extras), specialists (with the other --extras readers), red-team, voices (Step 11). Renders first and specialists now; prints the plan, MANIFEST: and one READER: line per reader. A plan whose NEXT is not full or scoped prints PASS: none. + --diff-base sets every reader's base, a scoped pass's Red Team included (the targeted reader keeps the merge base). An existing manifest for the same --pass is reused (resumed). --extras: [{"name","prompt_file","first"?,"model"?}] show --manifest MANIFEST_JSON: stage --manifest --stage @@ -176,7 +177,7 @@ function pass(): number { const extras = extrasFile ? JSON.parse(readFile(extrasFile, 'extras')!) as Extra[] : []; const m = buildManifest({ pass: id, out, ledger: flag('--ledger') ?? path.join(out, 'review-ledger.jsonl'), base, diffBase, root, checklistRoot: flag('--checklist-root') ?? ROOT, dispatches: p.dispatches.filter(d => d.role !== 'fix-check'), next: step, planLines, learnings: readFile(flag('--learnings-file'), 'learnings') ?? null, extras, - targetedClasses: p.targeted?.classes, mergeBase, rangeBase }); + targetedClasses: p.targeted?.classes, mergeBase, rangeBase: flag('--diff-base') ? undefined : rangeBase }); fs.mkdirSync(m.out, { recursive: true, mode: 0o700 }); const t = sectionTexts(ROOT); renderStage(m, 'first', t, branchDiff); diff --git a/changelog.d/gst-738-secrets-context.md b/changelog.d/gst-738-secrets-context.md new file mode 100644 index 000000000..63deeec5c --- /dev/null +++ b/changelog.d/gst-738-secrets-context.md @@ -0,0 +1,6 @@ +**The secrets check reads Python and Dart string context from what git diffed, and a quote inside an interpolation no longer ends a string (GST-738).** env.kv's source-syntax exemption treats a `password:` line as code only when it sits outside every string. Two inputs could make the scanner place a committed string's contents outside a string. One was a working copy that no longer matched what git diffed. The other was a quote inside an f-string or Dart `${…}` expression. A scoped review pass now applies an explicit `--diff-base` to its Red Team as well. + +### Fixed +- **`lib/pregate/checks/secrets.ts`:** a changed Python or Dart file's whole text comes from a full-context `git diff`, never from the working copy. +- **`lib/redact-patterns.ts`:** a Python f-string's `{…}`, or t-string's, and a non-raw Dart string's `${…}` are skipped whole, nested strings included. A quote inside one doesn't end the outer string. An expression left open runs to the end of the file, so the exemption fails closed. +- **`bin/gstack-review-dispatch`:** `pass --diff-base` now sets the scoped Red Team's base too. diff --git a/lib/pregate/checks/secrets.ts b/lib/pregate/checks/secrets.ts index 934def71c..6adc2bfb9 100644 --- a/lib/pregate/checks/secrets.ts +++ b/lib/pregate/checks/secrets.ts @@ -49,6 +49,26 @@ export function parseAddedLineNumbers(diff: string): Map { return files; } +/** Each file's whole postimage from a diff with full context: one hunk starting at line 1, its context and added lines; a file with any other shape is left out. */ +export function parsePostimages(diff: string): Map { + const files = new Map(); + let file = ''; + let lines: string[] | null = null; + for (const line of diff.split('\n')) { + if (line.startsWith('diff --')) { file = ''; lines = null; continue; } + if (line.startsWith('@@')) { + const first: boolean = lines === null && file !== '' && /^@@ -\d+(?:,\d+)? \+1(?:,\d+)? @@/.test(line); + files.delete(file); + lines = first ? [] : null; + if (lines) files.set(file, lines); else file = ''; + continue; + } + if (lines === null) { if (line.startsWith('+++ ')) file = line.startsWith('+++ b/') ? line.slice(6) : ''; continue; } + if (line.startsWith('+') || line.startsWith(' ')) lines.push(line.slice(1)); + } + return files; +} + /** Files whose source-syntax exemption needs the whole file: a line added inside an existing `'''`/`"""` block is a literal (#3630). */ const LEXICAL_CONTEXT_SOURCE = /\.(?:dart|pyi?)$/i; const WHOLE_FILE_MAX_BYTES = 4 * 1024 * 1024; @@ -58,10 +78,11 @@ const WHOLE_FILE_MAX_BYTES = 4 * 1024 * 1024; * untracked files, grouped by file so gstack-redact can use each file's type * (#3211). A tracked Dart or Python file is passed whole with `keep`, its * added line numbers, so the scanner sees the strings an added line sits in - * and only findings on those lines count (#3630); an untracked one keeps every - * line. One too large, unreadable, not a regular file, or whose working copy - * no longer holds the diff's added lines at their numbers - * to read whole is scanned under a neutral name, without that exemption. External diff and textconv + * and only findings on those lines count (#3630). The whole text is the + * postimage git diffed (a full-context diff), never the working copy, which + * assume-unchanged can hide from git (GST-738); an untracked file keeps every + * line. One too large, or whose full diff does not parse as one hunk holding + * its added lines, is scanned under a neutral name, without that exemption. External diff and textconv * drivers are off: either can empty or rewrite the diff, and an empty diff * reads as clean. */ @@ -70,16 +91,15 @@ export function addedLines(ctx: PregateContext): Array<{ file: string; text: str const files = parseAddedLines(diff.stdout ?? ''); const numbers = parseAddedLineNumbers(diff.stdout ?? ''); const whole = new Map(); - for (const [file, keep] of numbers) { - if (!LEXICAL_CONTEXT_SOURCE.test(file)) continue; - try { - const st = fs.lstatSync(path.join(ctx.repoRoot, file)); - if (!st.isFile() || st.size > WHOLE_FILE_MAX_BYTES) continue; - const text = fs.readFileSync(path.join(ctx.repoRoot, file), 'utf8'); - const lines = text.split('\n'); + const sources = [...numbers.keys()].filter(file => LEXICAL_CONTEXT_SOURCE.test(file)); + if (sources.length) { + const full = spawnSync('git', ['diff', '--no-color', '--no-ext-diff', '--no-textconv', '--src-prefix=a/', '--dst-prefix=b/', '--unified=100000000', '--diff-filter=ACMR', ctx.mergeBase, '--', ...sources.map(file => `:(literal)${file}`)], { cwd: ctx.repoRoot, encoding: 'utf8', timeout: 60_000, maxBuffer: 256 * 1024 * 1024, env: ctx.env }); + for (const [file, post] of parsePostimages(full.status === 0 ? full.stdout ?? '' : '')) { + const keep = numbers.get(file) ?? []; const added = files.get(file) ?? []; - if (keep.length === added.length && keep.every((n, k) => lines[n - 1] === added[k])) whole.set(file, { text, keep }); - } catch { /* unreadable: its added lines are scanned alone */ } + const text = post.join('\n'); + if (text.length <= WHOLE_FILE_MAX_BYTES && keep.length === added.length && keep.every((n, k) => post[n - 1] === added[k])) whole.set(file, { text, keep }); + } } for (const t of ctx.touched) { if (!t.uncommitted || t.status === 'D') continue; diff --git a/lib/redact-patterns.ts b/lib/redact-patterns.ts index 66d837b71..0ba0a7fee 100644 --- a/lib/redact-patterns.ts +++ b/lib/redact-patterns.ts @@ -708,6 +708,41 @@ export function clearStringSpanCache(): void { stringSpansCache = null; } +/** + * The string whose quote is at `i`: where its content starts and ends, and the + * index after its closing delimiter. An interpolated expression (`{…}` in a + * Python f- or t-string, `${…}` in a non-raw Dart string) is skipped whole, + * nested strings included, so a quote inside it never ends the outer string + * (GST-738); an expression left open runs to the end of the text. + */ +function stringAt(text: string, i: number, lang: "py" | "dart"): [number, number, number] { + const n = text.length; + const c = text[i]!; + const close = text.startsWith(c.repeat(3), i) ? c.repeat(3) : c; + const multiline = close.length === 3; + let k = i; + while (k > 0 && /[A-Za-z]/.test(text[k - 1]!)) k -= 1; + const prefix = k === 0 || !/[\w$]/.test(text[k - 1]!) ? text.slice(k, i) : ""; + const raw = lang === "dart" && prefix === "r"; + const interpolated = lang === "dart" ? !raw : /^(?:[rR]?[fFtT]|[fFtT][rR])$/.test(prefix); + let j = i + close.length; + while (j < n && !text.startsWith(close, j) && (multiline || text[j] !== "\n")) { + if (!raw && text[j] === "\\") { j += 2; continue; } + if (!interpolated) { j += 1; continue; } + if (lang === "py" && text.startsWith("{{", j)) { j += 2; continue; } + if (lang === "py" ? text[j] !== "{" : !text.startsWith("${", j)) { j += 1; continue; } + j += lang === "py" ? 1 : 2; + for (let depth = 1; j < n && depth > 0; ) { + const d = text[j]!; + if (d === '"' || d === "'") { j = stringAt(text, j, lang)[2]; continue; } + if (d === "{") depth += 1; + else if (d === "}") depth -= 1; + j += 1; + } + } + return [i + close.length, Math.min(j, n), Math.min(j, n) + close.length]; +} + function stringSpans(text: string, lang: "py" | "dart"): Array<[number, number]> { if (stringSpansCache?.text === text && stringSpansCache.lang === lang) return stringSpansCache.spans; const spans: Array<[number, number]> = []; @@ -729,13 +764,9 @@ function stringSpans(text: string, lang: "py" | "dart"): Array<[number, number]> continue; } if (c !== '"' && c !== "'") { i += 1; continue; } - const close = text.startsWith(c.repeat(3), i) ? c.repeat(3) : c; - const multiline = close.length === 3; - const raw = lang === "dart" && text[i - 1] === "r" && !/[\w$]/.test(text[i - 2] ?? ""); - let j = i + close.length; - while (j < n && !text.startsWith(close, j) && (multiline || text[j] !== "\n")) j += !raw && text[j] === "\\" ? 2 : 1; - spans.push([i + close.length, Math.min(j, n)]); - i = j + close.length; + const [from, to, next] = stringAt(text, i, lang); + spans.push([from, to]); + i = next; } stringSpansCache = { text, lang, spans }; return spans; diff --git a/test/pregate-secrets-source-context.test.ts b/test/pregate-secrets-source-context.test.ts index a46e1d178..ff7b149d3 100644 --- a/test/pregate-secrets-source-context.test.ts +++ b/test/pregate-secrets-source-context.test.ts @@ -99,7 +99,7 @@ describe('pregate secrets: source context and diff integrity', () => { } finally { cleanup(fx); } }); - test('a Python file whose working copy no longer holds the diff\'s added lines is scanned by its added lines (#3630)', () => { + test('a Python file whose working copy no longer holds the diff\'s added lines is scanned as git diffed it (#3630, GST-738)', () => { const token = 'Fk2pQ9vXz4wL7mN3rT6yB1cD5eG0hJx'; const fx: FixtureRepo = makeFixtureRepo({ pregate: BASE_PREGATE, baseFiles: { 'app.py': 'x = 1\n' } }); try { @@ -107,7 +107,21 @@ describe('pregate secrets: source context and diff integrity', () => { fx.git('update-index', '--assume-unchanged', 'app.py'); fs.writeFileSync(path.join(fx.work, 'app.py'), 'x = 1\ny = 2\n'); const check = JSON.parse(runBin(fx, 'gstack-pregate', ['--base', 'main', '--only', 'secrets', '--json']).out).checks.find((c: { id: string }) => c.id === 'secrets'); - expect({ status: check.status, kv: (check.lines as string[]).some((l) => l.includes('env.kv') && l.includes('app.py, added line 1')) }).toEqual({ status: 'fail', kv: true }); + expect({ status: check.status, kv: (check.lines as string[]).some((l) => l.includes('env.kv') && l.includes('app.py, line 2')) }).toEqual({ status: 'fail', kv: true }); + } finally { cleanup(fx); } + }); + + // Value: protects=the Python/Dart exemption reads the text git diffed, never a working copy that differs from it; fails_when=a committed credential inside a """ block passes because the working copy's other lines were edited under assume-unchanged; + // why_new=GST-738: the added lines matched, so the edited delimiters around them were trusted; seam=git diff; no_claim=a file edited after the scan + test('a committed credential in a Python string still fails when the working copy rewrites the lines around it (GST-738)', () => { + const token = 'Fk2pQ9vXz4wL7mN3rT6yB1cD5eG0hJx'; + const fx: FixtureRepo = makeFixtureRepo({ pregate: BASE_PREGATE, baseFiles: { 'app.py': 'x = 1\nCONFIG = """\n"""\n' } }); + try { + commit(fx, { 'app.py': `x = 1\nCONFIG = """\npassword: ${token}\n"""\n` }); + fx.git('update-index', '--assume-unchanged', 'app.py'); + fs.writeFileSync(path.join(fx.work, 'app.py'), `x = 1\nCONFIG = 1 #\npassword: ${token}\n#\n`); + const check = JSON.parse(runBin(fx, 'gstack-pregate', ['--base', 'main', '--only', 'secrets', '--json']).out).checks.find((c: { id: string }) => c.id === 'secrets'); + expect({ status: check.status, kv: ((check.lines ?? []) as string[]).some((l) => l.includes('env.kv') && l.includes('app.py')) }).toEqual({ status: 'fail', kv: true }); } finally { cleanup(fx); } }); diff --git a/test/redact-engine.test.ts b/test/redact-engine.test.ts index b25ef1025..9a610b416 100644 --- a/test/redact-engine.test.ts +++ b/test/redact-engine.test.ts @@ -347,6 +347,27 @@ describe("MEDIUM demoted credential-shaped patterns (TENSION-1)", () => { [` 'password: ${vIdent} more',`, "lib/a.dart"], [` "password: ${vIdent} more",`, "app.py"], ] as const) expect({ literal, file, kv: kv(literal, file) }).toEqual({ literal, file, kv: true }); }); + // Value: protects=an interpolated expression inside a Python f-string or Dart string is skipped whole, so a quote inside it never ends the outer string; fails_when=a """/''' delimiter in a nested string closes the block early and a password: line inside it reads as code; + // why_new=GST-738 (outside voice); seam=none; no_claim=Python before 3.12, where the nested quote is a syntax error + test("env.kv: a quote inside an f-string or Dart interpolation does not end the string (GST-738)", () => { + const kv = (text: string, sourcePath: string) => scan(text, { repoVisibility: "private", sourcePath, processEnv: {} }).findings.some((f) => f.id === "env.kv"); + const v = "Fk2pQ9vXz4wL7mN3rT6yB1cD5eG0hJx"; + for (const [literal, file] of [ + [`CONFIG = f'''\n{"'''"}\npassword: ${v}\n'''\n`, "app.py"], + [`CONFIG = rf"""\n{x.replace('"""', "")}\npassword: ${v}\n"""\n`, "app.py"], + [`CONFIG = f'''\n{f"{'\'\'\''}"}\npassword: ${v}\n'''\n`, "app.py"], + [`const s = '''\${"'''"}\npassword: ${v}\n''';\n`, "lib/a.dart"], + [`const s = """\${m["k"] ?? '"""'}\npassword: ${v}\n""";\n`, "lib/a.dart"], + ] as const) expect({ literal, kv: kv(literal, file) }).toEqual({ literal, kv: true }); + for (const [code, file] of [ + ["msg = f'{a}:{b}'\ndef f(\n session: HttpSession,\n): pass\n", "app.py"], + ["msg = f'''{{literal brace'''\ndef f(\n session: HttpSession,\n): pass\n", "app.py"], + ["msg = f'{x:>{width}}'\ndef f(\n session: HttpSession,\n): pass\n", "app.py"], + ["if'a' in s: pass\ndef f(\n session: HttpSession,\n): pass\n", "app.py"], + ["final s = 'hi ${user.name} and $id';\n login(password: request.password);\n", "lib/a.dart"], + ["final s = r'${raw}';\n login(password: request.password);\n", "lib/a.dart"], + ] as const) expect({ code, kv: kv(code, file) }).toEqual({ code, kv: false }); + }); test("env.kv's source-syntax check stays linear on a match-dense file (#3630)", () => { for (const file of ["app.py", "src/a.ts"]) { const text = "def f(\n" + " session: HttpSession,\n".repeat(12_000) + "): pass\n"; diff --git a/test/review-pass.test.ts b/test/review-pass.test.ts index e3458effc..b54256c9c 100644 --- a/test/review-pass.test.ts +++ b/test/review-pass.test.ts @@ -492,7 +492,7 @@ describe('the Capy skill loader', () => { describe('a scoped pass after a blocker fix (GST-619)', () => { // Value: protects=the pass command accepts the scoped plan gstack-review-plan writes after a blocking fix, invoked as GEVA-2 ran it: the Red Team reviews the fix range, Step 11's voices still pending keep the whole branch, the fix-check is named outside the manifest for Step 3.5, and a resumed pass names it again; fails_when=pass refuses the scoped plan, stages the Red Team on the merge base, stages pending Step 11 voices on the range, drops the fix-check silently, or puts it in the manifest; // why_new=GEVA-82 on 1.91.379.0 hit BRIEF_RENDER_FAILED on every scoped pass and rendered briefs by hand; seam=none; no_claim=the fix-check's own brief - function scoped(flags: string[]) { + function scoped(flags: string[], explicitMergeBase = false) { const { dir, env } = repo(); const sh = (c: string) => { const r = spawnSync('bash', ['-c', c], { cwd: dir, env, encoding: 'utf8', timeout: 60_000 }); if (r.status !== 0) throw new Error(r.stderr); return r.stdout.trim(); }; const mergeBase = sh('git merge-base main HEAD'); @@ -503,7 +503,7 @@ describe('a scoped pass after a blocker fix (GST-619)', () => { fs.mkdirSync(path.join(dir, '.state'), { recursive: true }); fs.writeFileSync(path.join(dir, '.state/qa.md'), 'QA the fix.\n'); fs.writeFileSync(extras, JSON.stringify([{ name: 'qa', prompt_file: path.join(dir, '.state/qa.md') }])); - const args = ['pass', '--base', 'main', '--out', out, '--pass', 'p2', '--extras', extras, '--', '--pass', '1', '--last', 'full', '--pass1-depth', 'standard', '--fixed-blocking', ...flags, '--range-base', rangeBase.slice(0, 7)]; + const args = ['pass', '--base', 'main', '--out', out, '--pass', 'p2', '--extras', extras, ...(explicitMergeBase ? ['--diff-base', mergeBase] : []), '--', '--pass', '1', '--last', 'full', '--pass1-depth', 'standard', '--fixed-blocking', ...flags, '--range-base', rangeBase.slice(0, 7)]; const p = bin(dir, env, args); expect(p.status, p.stdout + p.stderr).toBe(0); expect(p.stdout).toMatch(/^NEXT: scoped /m); @@ -544,6 +544,15 @@ describe('a scoped pass after a blocker fix (GST-619)', () => { expect(s.m.diff_base).toBe(s.rangeBase); expect(s.m.readers.map(r => [r.reader, r.diff_base ?? null])).toEqual([['red-team', s.rangeBase], ['adversarial-native', null], ['audit/qa', null]]); }, 120_000); + + // Value: protects=an explicit --diff-base reaches every reader, the Red Team included; fails_when=a scoped pass asked to read the whole branch still stages its Red Team on the fix range; why_new=GST-738 (outside voice); seam=none; no_claim=none + test('an explicit --diff-base widens the scoped Red Team too (GST-738)', () => { + const s = scoped([], true); + expect(s.m.diff_base).toBe(s.mergeBase); + const m = settle(s, 'specialists') && settle(s, 'red-team'); + const redTeam = fs.readFileSync(m.readers.find(r => r.reader === 'red-team')!.file!, 'utf8'); + expect({ diffBase: m.readers.find(r => r.reader === 'red-team')!.diff_base ?? null, reads: redTeam.includes(s.mergeBase) }).toEqual({ diffBase: null, reads: true }); + }, 120_000); }); describe('readers on a shared machine review the checkout the pass ships (GST-482)', () => { From 5e65b17bdac97e1346f55b8f716acd47fc9b3921 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 1 Nov 2026 02:57:03 +0000 Subject: [PATCH 2/8] fix(redact): comments inside an interpolation hold no delimiter; a Dart block comment is text (GST-738) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- changelog.d/gst-738-secrets-context.md | 2 +- lib/redact-patterns.ts | 33 +++++++++++++++++++------- test/redact-engine.test.ts | 4 ++++ 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/changelog.d/gst-738-secrets-context.md b/changelog.d/gst-738-secrets-context.md index 63deeec5c..469490fb5 100644 --- a/changelog.d/gst-738-secrets-context.md +++ b/changelog.d/gst-738-secrets-context.md @@ -2,5 +2,5 @@ ### Fixed - **`lib/pregate/checks/secrets.ts`:** a changed Python or Dart file's whole text comes from a full-context `git diff`, never from the working copy. -- **`lib/redact-patterns.ts`:** a Python f-string's `{…}`, or t-string's, and a non-raw Dart string's `${…}` are skipped whole, nested strings included. A quote inside one doesn't end the outer string. An expression left open runs to the end of the file, so the exemption fails closed. +- **`lib/redact-patterns.ts`:** a Python f-string's `{…}`, or t-string's, and a non-raw Dart string's `${…}` are skipped whole, nested strings and comments included. A quote inside one doesn't end the outer string. A `password:` line inside a Dart `/* */` comment is now read as text, not code. An expression left open runs to the end of the file, so the exemption fails closed. - **`bin/gstack-review-dispatch`:** `pass --diff-base` now sets the scoped Red Team's base too. diff --git a/lib/redact-patterns.ts b/lib/redact-patterns.ts index 0ba0a7fee..171ffbe0e 100644 --- a/lib/redact-patterns.ts +++ b/lib/redact-patterns.ts @@ -694,7 +694,8 @@ const CODE_NAME_CHAIN = /^[A-Za-z_$][\w$]*(?:\??\.[A-Za-z_$][\w$]*)*(?:<[\w$.,\s * text) records every string's content span: a `'''`/`"""` block, or a * `'`/`"` string that ends at its line unless a backslash continues it. * Comments are skipped (`#` in Python; `//` and nested `/* *\/` in Dart), so a - * delimiter inside one opens nothing; a backslash escapes the next character, except in a Dart raw + * delimiter inside one opens nothing, and a Dart block comment's own span is + * recorded too: a `password:` line in it is text, not code (GST-738); a backslash escapes the next character, except in a Dart raw * string (`r'''…'''`). The exemption needs the * whole file to see a block's opener: pregate passes Dart and Python files * whole, and the pre-push hook, which scans added lines alone, passes them @@ -708,6 +709,22 @@ export function clearStringSpanCache(): void { stringSpansCache = null; } +/** The index after the comment at `i`: a line comment ends at its newline, a Dart block comment at its matching `*\/` (they nest). */ +function commentEnd(text: string, i: number, lang: "py" | "dart"): number { + if (lang === "py" || !text.startsWith("/*", i)) { + const eol = text.indexOf("\n", i); + return eol < 0 ? text.length : eol; + } + let depth = 1; + let j = i + 2; + while (j < text.length && depth > 0) { + if (text.startsWith("/*", j)) { depth += 1; j += 2; } + else if (text.startsWith("*/", j)) { depth -= 1; j += 2; } + else j += 1; + } + return j; +} + /** * The string whose quote is at `i`: where its content starts and ends, and the * index after its closing delimiter. An interpolated expression (`{…}` in a @@ -734,6 +751,8 @@ function stringAt(text: string, i: number, lang: "py" | "dart"): [number, number j += lang === "py" ? 1 : 2; for (let depth = 1; j < n && depth > 0; ) { const d = text[j]!; + const comment = lang === "py" ? d === "#" : text.startsWith("//", j); + if (comment || (lang === "dart" && text.startsWith("/*", j))) { j = commentEnd(text, j, lang); continue; } if (d === '"' || d === "'") { j = stringAt(text, j, lang)[2]; continue; } if (d === "{") depth += 1; else if (d === "}") depth -= 1; @@ -750,17 +769,13 @@ function stringSpans(text: string, lang: "py" | "dart"): Array<[number, number]> for (let i = 0; i < n; ) { const c = text[i]!; if (lang === "py" ? c === "#" : text.startsWith("//", i)) { - const eol = text.indexOf("\n", i); - i = eol < 0 ? n : eol; + i = commentEnd(text, i, lang); continue; } if (lang === "dart" && text.startsWith("/*", i)) { - let depth = 1; - for (i += 2; i < n && depth > 0; ) { - if (text.startsWith("/*", i)) { depth += 1; i += 2; } - else if (text.startsWith("*/", i)) { depth -= 1; i += 2; } - else i += 1; - } + const end = commentEnd(text, i, lang); + spans.push([i, end]); + i = end; continue; } if (c !== '"' && c !== "'") { i += 1; continue; } diff --git a/test/redact-engine.test.ts b/test/redact-engine.test.ts index 9a610b416..f4b8d80f1 100644 --- a/test/redact-engine.test.ts +++ b/test/redact-engine.test.ts @@ -358,6 +358,10 @@ describe("MEDIUM demoted credential-shaped patterns (TENSION-1)", () => { [`CONFIG = f'''\n{f"{'\'\'\''}"}\npassword: ${v}\n'''\n`, "app.py"], [`const s = '''\${"'''"}\npassword: ${v}\n''';\n`, "lib/a.dart"], [`const s = """\${m["k"] ?? '"""'}\npassword: ${v}\n""";\n`, "lib/a.dart"], + // A comment inside an interpolation holds no delimiter, and a Dart block comment is text. + [`CONFIG = f"""\n{1 # } """\n}\npassword: ${v}\n"""\n`, "app.py"], + [`const s = '''\${x // '''\n}\npassword: ${v}\n''';\n`, "lib/a.dart"], + [`void main() {}\n/*\npassword: ${v}\n*/\n`, "lib/a.dart"], ] as const) expect({ literal, kv: kv(literal, file) }).toEqual({ literal, kv: true }); for (const [code, file] of [ ["msg = f'{a}:{b}'\ndef f(\n session: HttpSession,\n): pass\n", "app.py"], From 51c06507875d15130f47bf68969a92fa9c2b5727 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 1 Nov 2026 03:04:57 +0000 Subject: [PATCH 3/8] fix(redact): a Python format spec is text, and # is a comment only in a triple-quoted f-string (GST-738) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- lib/redact-patterns.ts | 49 ++++++++++++++++++++++++++++++-------- test/redact-engine.test.ts | 5 ++++ 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/lib/redact-patterns.ts b/lib/redact-patterns.ts index 171ffbe0e..a82e2415c 100644 --- a/lib/redact-patterns.ts +++ b/lib/redact-patterns.ts @@ -725,6 +725,44 @@ function commentEnd(text: string, i: number, lang: "py" | "dart"): number { return j; } +/** The index after a Dart `${…}` expression starting at `j`: braces balance, and its strings and comments are skipped whole. */ +function dartExpressionEnd(text: string, j: number): number { + for (let depth = 1; j < text.length && depth > 0; ) { + const d = text[j]!; + if (text.startsWith("//", j) || text.startsWith("/*", j)) { j = commentEnd(text, j, "dart"); continue; } + if (d === '"' || d === "'") { j = stringAt(text, j, "dart")[2]; continue; } + if (d === "{") depth += 1; + else if (d === "}") depth -= 1; + j += 1; + } + return Math.min(j, text.length); +} + +/** + * The index after a Python replacement field whose expression starts at `j`. + * The expression's strings, and its `#` comments in a triple-quoted string, are skipped whole; a `:` or `!` + * outside its brackets starts the conversion and format spec, which is plain + * text apart from nested `{…}` fields (`{n:#x}`, `{n:'>10}`). + */ +function pythonFieldEnd(text: string, j: number, multiline: boolean): number { + const n = text.length; + let nest = 0; + while (j < n) { + const d = text[j]!; + if (d === "#" && multiline) { j = commentEnd(text, j, "py"); continue; } + if (d === '"' || d === "'") { j = stringAt(text, j, "py")[2]; continue; } + if (d === "}" && nest === 0) return j + 1; + if (nest === 0 && (d === ":" || (d === "!" && text[j + 1] !== "="))) { + for (j += 1; j < n && text[j] !== "}"; ) j = text[j] === "{" ? pythonFieldEnd(text, j + 1, multiline) : j + 1; + return Math.min(j + 1, n); + } + if ("([{".includes(d)) nest += 1; + else if (")]}".includes(d)) nest = Math.max(0, nest - 1); + j += 1; + } + return n; +} + /** * The string whose quote is at `i`: where its content starts and ends, and the * index after its closing delimiter. An interpolated expression (`{…}` in a @@ -748,16 +786,7 @@ function stringAt(text: string, i: number, lang: "py" | "dart"): [number, number if (!interpolated) { j += 1; continue; } if (lang === "py" && text.startsWith("{{", j)) { j += 2; continue; } if (lang === "py" ? text[j] !== "{" : !text.startsWith("${", j)) { j += 1; continue; } - j += lang === "py" ? 1 : 2; - for (let depth = 1; j < n && depth > 0; ) { - const d = text[j]!; - const comment = lang === "py" ? d === "#" : text.startsWith("//", j); - if (comment || (lang === "dart" && text.startsWith("/*", j))) { j = commentEnd(text, j, lang); continue; } - if (d === '"' || d === "'") { j = stringAt(text, j, lang)[2]; continue; } - if (d === "{") depth += 1; - else if (d === "}") depth -= 1; - j += 1; - } + j = lang === "py" ? pythonFieldEnd(text, j + 1, multiline) : dartExpressionEnd(text, j + 2); } return [i + close.length, Math.min(j, n), Math.min(j, n) + close.length]; } diff --git a/test/redact-engine.test.ts b/test/redact-engine.test.ts index f4b8d80f1..7055f6bf0 100644 --- a/test/redact-engine.test.ts +++ b/test/redact-engine.test.ts @@ -362,11 +362,16 @@ describe("MEDIUM demoted credential-shaped patterns (TENSION-1)", () => { [`CONFIG = f"""\n{1 # } """\n}\npassword: ${v}\n"""\n`, "app.py"], [`const s = '''\${x // '''\n}\npassword: ${v}\n''';\n`, "lib/a.dart"], [`void main() {}\n/*\npassword: ${v}\n*/\n`, "lib/a.dart"], + // A format spec is text: its # or quote opens nothing. + [`render({\n "id": f"{n:#x}",\n}, """\npassword: ${v}\n""")\n`, "app.py"], + [`render({\n "id": f"{n:'>10}",\n}, """\npassword: ${v}\n""")\n`, "app.py"], + [`render({\n "id": f"{n!r:{w}}",\n}, """\npassword: ${v}\n""")\n`, "app.py"], ] as const) expect({ literal, kv: kv(literal, file) }).toEqual({ literal, kv: true }); for (const [code, file] of [ ["msg = f'{a}:{b}'\ndef f(\n session: HttpSession,\n): pass\n", "app.py"], ["msg = f'''{{literal brace'''\ndef f(\n session: HttpSession,\n): pass\n", "app.py"], ["msg = f'{x:>{width}}'\ndef f(\n session: HttpSession,\n): pass\n", "app.py"], + ["msg = f\"{when:%b %d '%y} {d[':']} {a[1:2]} {a!=b}\"\nrows = {\n \"password\": session.password,\n}\n", "app.py"], ["if'a' in s: pass\ndef f(\n session: HttpSession,\n): pass\n", "app.py"], ["final s = 'hi ${user.name} and $id';\n login(password: request.password);\n", "lib/a.dart"], ["final s = r'${raw}';\n login(password: request.password);\n", "lib/a.dart"], From 59de715b8da6741f5a2808505b5d7524d68962e3 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 1 Nov 2026 03:15:41 +0000 Subject: [PATCH 4/8] fix(secrets): suppressBlankEmpty cannot shift the diffed text; f-string \{ opens a field; interpolation depth is capped (GST-738) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- lib/pregate/checks/secrets.ts | 2 +- lib/redact-patterns.ts | 29 ++++++++++++--------- test/pregate-secrets-source-context.test.ts | 10 +++++++ test/redact-engine.test.ts | 5 ++++ 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/lib/pregate/checks/secrets.ts b/lib/pregate/checks/secrets.ts index 6adc2bfb9..17572c1f4 100644 --- a/lib/pregate/checks/secrets.ts +++ b/lib/pregate/checks/secrets.ts @@ -93,7 +93,7 @@ export function addedLines(ctx: PregateContext): Array<{ file: string; text: str const whole = new Map(); const sources = [...numbers.keys()].filter(file => LEXICAL_CONTEXT_SOURCE.test(file)); if (sources.length) { - const full = spawnSync('git', ['diff', '--no-color', '--no-ext-diff', '--no-textconv', '--src-prefix=a/', '--dst-prefix=b/', '--unified=100000000', '--diff-filter=ACMR', ctx.mergeBase, '--', ...sources.map(file => `:(literal)${file}`)], { cwd: ctx.repoRoot, encoding: 'utf8', timeout: 60_000, maxBuffer: 256 * 1024 * 1024, env: ctx.env }); + const full = spawnSync('git', ['-c', 'diff.suppressBlankEmpty=false', 'diff', '--no-color', '--no-ext-diff', '--no-textconv', '--src-prefix=a/', '--dst-prefix=b/', '--unified=100000000', '--diff-filter=ACMR', ctx.mergeBase, '--', ...sources.map(file => `:(literal)${file}`)], { cwd: ctx.repoRoot, encoding: 'utf8', timeout: 60_000, maxBuffer: 256 * 1024 * 1024, env: ctx.env }); for (const [file, post] of parsePostimages(full.status === 0 ? full.stdout ?? '' : '')) { const keep = numbers.get(file) ?? []; const added = files.get(file) ?? []; diff --git a/lib/redact-patterns.ts b/lib/redact-patterns.ts index a82e2415c..7ef840d4b 100644 --- a/lib/redact-patterns.ts +++ b/lib/redact-patterns.ts @@ -725,14 +725,17 @@ function commentEnd(text: string, i: number, lang: "py" | "dart"): number { return j; } +const MAX_INTERPOLATION_DEPTH = 256; + /** The index after a Dart `${…}` expression starting at `j`: braces balance, and its strings and comments are skipped whole. */ -function dartExpressionEnd(text: string, j: number): number { - for (let depth = 1; j < text.length && depth > 0; ) { +function dartExpressionEnd(text: string, j: number, depth: number): number { + if (depth > MAX_INTERPOLATION_DEPTH) return text.length; + for (let open = 1; j < text.length && open > 0; ) { const d = text[j]!; if (text.startsWith("//", j) || text.startsWith("/*", j)) { j = commentEnd(text, j, "dart"); continue; } - if (d === '"' || d === "'") { j = stringAt(text, j, "dart")[2]; continue; } - if (d === "{") depth += 1; - else if (d === "}") depth -= 1; + if (d === '"' || d === "'") { j = stringAt(text, j, "dart", depth)[2]; continue; } + if (d === "{") open += 1; + else if (d === "}") open -= 1; j += 1; } return Math.min(j, text.length); @@ -744,16 +747,17 @@ function dartExpressionEnd(text: string, j: number): number { * outside its brackets starts the conversion and format spec, which is plain * text apart from nested `{…}` fields (`{n:#x}`, `{n:'>10}`). */ -function pythonFieldEnd(text: string, j: number, multiline: boolean): number { +function pythonFieldEnd(text: string, j: number, multiline: boolean, depth: number): number { const n = text.length; + if (depth > MAX_INTERPOLATION_DEPTH) return n; let nest = 0; while (j < n) { const d = text[j]!; if (d === "#" && multiline) { j = commentEnd(text, j, "py"); continue; } - if (d === '"' || d === "'") { j = stringAt(text, j, "py")[2]; continue; } + if (d === '"' || d === "'") { j = stringAt(text, j, "py", depth)[2]; continue; } if (d === "}" && nest === 0) return j + 1; if (nest === 0 && (d === ":" || (d === "!" && text[j + 1] !== "="))) { - for (j += 1; j < n && text[j] !== "}"; ) j = text[j] === "{" ? pythonFieldEnd(text, j + 1, multiline) : j + 1; + for (j += 1; j < n && text[j] !== "}"; ) j = text[j] === "{" ? pythonFieldEnd(text, j + 1, multiline, depth + 1) : j + 1; return Math.min(j + 1, n); } if ("([{".includes(d)) nest += 1; @@ -768,9 +772,10 @@ function pythonFieldEnd(text: string, j: number, multiline: boolean): number { * index after its closing delimiter. An interpolated expression (`{…}` in a * Python f- or t-string, `${…}` in a non-raw Dart string) is skipped whole, * nested strings included, so a quote inside it never ends the outer string - * (GST-738); an expression left open runs to the end of the text. + * (GST-738); an expression left open, or nested past MAX_INTERPOLATION_DEPTH, + * runs to the end of the text. In a Python f-string `\{` still opens a field. */ -function stringAt(text: string, i: number, lang: "py" | "dart"): [number, number, number] { +function stringAt(text: string, i: number, lang: "py" | "dart", depth = 0): [number, number, number] { const n = text.length; const c = text[i]!; const close = text.startsWith(c.repeat(3), i) ? c.repeat(3) : c; @@ -782,11 +787,11 @@ function stringAt(text: string, i: number, lang: "py" | "dart"): [number, number const interpolated = lang === "dart" ? !raw : /^(?:[rR]?[fFtT]|[fFtT][rR])$/.test(prefix); let j = i + close.length; while (j < n && !text.startsWith(close, j) && (multiline || text[j] !== "\n")) { - if (!raw && text[j] === "\\") { j += 2; continue; } + if (!raw && text[j] === "\\") { j += lang === "py" && interpolated && text[j + 1] === "{" ? 1 : 2; continue; } if (!interpolated) { j += 1; continue; } if (lang === "py" && text.startsWith("{{", j)) { j += 2; continue; } if (lang === "py" ? text[j] !== "{" : !text.startsWith("${", j)) { j += 1; continue; } - j = lang === "py" ? pythonFieldEnd(text, j + 1, multiline) : dartExpressionEnd(text, j + 2); + j = lang === "py" ? pythonFieldEnd(text, j + 1, multiline, depth + 1) : dartExpressionEnd(text, j + 2, depth + 1); } return [i + close.length, Math.min(j, n), Math.min(j, n) + close.length]; } diff --git a/test/pregate-secrets-source-context.test.ts b/test/pregate-secrets-source-context.test.ts index ff7b149d3..27777514c 100644 --- a/test/pregate-secrets-source-context.test.ts +++ b/test/pregate-secrets-source-context.test.ts @@ -125,6 +125,16 @@ describe('pregate secrets: source context and diff integrity', () => { } finally { cleanup(fx); } }); + test('diff.suppressBlankEmpty does not shift the diffed Python text (GST-738)', () => { + const fx: FixtureRepo = makeFixtureRepo({ pregate: BASE_PREGATE, baseFiles: { 'app.py': 'x = 1\n\ny = 2\n' } }); + try { + commit(fx, { 'app.py': 'x = 1\n\ny = 2\nrows = {\n "password": session.password,\n}\n' }); + fx.git('config', 'diff.suppressBlankEmpty', 'true'); + const check = JSON.parse(runBin(fx, 'gstack-pregate', ['--base', 'main', '--only', 'secrets', '--json']).out).checks.find((c: { id: string }) => c.id === 'secrets'); + expect(check.status).toBe('pass'); + } finally { cleanup(fx); } + }); + test('a one-line change to a Python file counts one added line (#3630)', () => { const fx: FixtureRepo = makeFixtureRepo({ pregate: BASE_PREGATE, baseFiles: { 'app.py': 'x = 1\n'.repeat(50) } }); try { diff --git a/test/redact-engine.test.ts b/test/redact-engine.test.ts index 7055f6bf0..74c1617f1 100644 --- a/test/redact-engine.test.ts +++ b/test/redact-engine.test.ts @@ -366,6 +366,11 @@ describe("MEDIUM demoted credential-shaped patterns (TENSION-1)", () => { [`render({\n "id": f"{n:#x}",\n}, """\npassword: ${v}\n""")\n`, "app.py"], [`render({\n "id": f"{n:'>10}",\n}, """\npassword: ${v}\n""")\n`, "app.py"], [`render({\n "id": f"{n!r:{w}}",\n}, """\npassword: ${v}\n""")\n`, "app.py"], + // In an f-string a backslash does not escape the brace after it. + [`CONFIG = f"""\\{"""\npassword: ${v}\n"""}"""\n`, "app.py"], + // Nesting past the depth cap runs to the end of the text instead of overflowing the stack. + ["const s = '" + "${'".repeat(20000) + `\npassword: ${v}\n`, "lib/a.dart"], + ["x = f'" + "{a:".repeat(100000) + `\npassword: ${v}\n`, "app.py"], ] as const) expect({ literal, kv: kv(literal, file) }).toEqual({ literal, kv: true }); for (const [code, file] of [ ["msg = f'{a}:{b}'\ndef f(\n session: HttpSession,\n): pass\n", "app.py"], From f96ff5deed824f6081049e6a8a23146d16cfa066 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 1 Nov 2026 03:24:23 +0000 Subject: [PATCH 5/8] fix(redact): interpolation code is not string text; # is a comment in every f-string field (GST-738) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- lib/redact-patterns.ts | 70 ++++++++++++++++++++++++-------------- test/redact-engine.test.ts | 5 +++ 2 files changed, 50 insertions(+), 25 deletions(-) diff --git a/lib/redact-patterns.ts b/lib/redact-patterns.ts index 7ef840d4b..1d7ef39b9 100644 --- a/lib/redact-patterns.ts +++ b/lib/redact-patterns.ts @@ -727,13 +727,19 @@ function commentEnd(text: string, i: number, lang: "py" | "dart"): number { const MAX_INTERPOLATION_DEPTH = 256; -/** The index after a Dart `${…}` expression starting at `j`: braces balance, and its strings and comments are skipped whole. */ -function dartExpressionEnd(text: string, j: number, depth: number): number { +type Spans = Array<[number, number]>; + +/** + * The index after a Dart `${…}` expression starting at `j`. Its code is not + * text; its strings and block comments add their own spans. + */ +function dartExpressionEnd(text: string, j: number, depth: number, spans: Spans): number { if (depth > MAX_INTERPOLATION_DEPTH) return text.length; for (let open = 1; j < text.length && open > 0; ) { const d = text[j]!; - if (text.startsWith("//", j) || text.startsWith("/*", j)) { j = commentEnd(text, j, "dart"); continue; } - if (d === '"' || d === "'") { j = stringAt(text, j, "dart", depth)[2]; continue; } + if (text.startsWith("//", j)) { j = commentEnd(text, j, "dart"); continue; } + if (text.startsWith("/*", j)) { const end = commentEnd(text, j, "dart"); spans.push([j, end]); j = end; continue; } + if (d === '"' || d === "'") { j = stringAt(text, j, "dart", depth, spans); continue; } if (d === "{") open += 1; else if (d === "}") open -= 1; j += 1; @@ -743,21 +749,28 @@ function dartExpressionEnd(text: string, j: number, depth: number): number { /** * The index after a Python replacement field whose expression starts at `j`. - * The expression's strings, and its `#` comments in a triple-quoted string, are skipped whole; a `:` or `!` - * outside its brackets starts the conversion and format spec, which is plain - * text apart from nested `{…}` fields (`{n:#x}`, `{n:'>10}`). + * The expression is code: its `#` comments are skipped and its strings add + * their own spans. A `:` or `!` outside its brackets starts the conversion and + * format spec, which is text apart from nested `{…}` fields (`{n:#x}`, `{n:'>10}`). */ -function pythonFieldEnd(text: string, j: number, multiline: boolean, depth: number): number { +function pythonFieldEnd(text: string, j: number, depth: number, spans: Spans): number { const n = text.length; if (depth > MAX_INTERPOLATION_DEPTH) return n; let nest = 0; while (j < n) { const d = text[j]!; - if (d === "#" && multiline) { j = commentEnd(text, j, "py"); continue; } - if (d === '"' || d === "'") { j = stringAt(text, j, "py", depth)[2]; continue; } + if (d === "#") { j = commentEnd(text, j, "py"); continue; } + if (d === '"' || d === "'") { j = stringAt(text, j, "py", depth, spans); continue; } if (d === "}" && nest === 0) return j + 1; if (nest === 0 && (d === ":" || (d === "!" && text[j + 1] !== "="))) { - for (j += 1; j < n && text[j] !== "}"; ) j = text[j] === "{" ? pythonFieldEnd(text, j + 1, multiline, depth + 1) : j + 1; + let from = j + 1; + for (j += 1; j < n && text[j] !== "}"; ) { + if (text[j] !== "{") { j += 1; continue; } + if (from < j) spans.push([from, j]); + j = pythonFieldEnd(text, j + 1, depth + 1, spans); + from = j; + } + if (from < Math.min(j, n)) spans.push([from, Math.min(j, n)]); return Math.min(j + 1, n); } if ("([{".includes(d)) nest += 1; @@ -768,14 +781,15 @@ function pythonFieldEnd(text: string, j: number, multiline: boolean, depth: numb } /** - * The string whose quote is at `i`: where its content starts and ends, and the - * index after its closing delimiter. An interpolated expression (`{…}` in a - * Python f- or t-string, `${…}` in a non-raw Dart string) is skipped whole, - * nested strings included, so a quote inside it never ends the outer string - * (GST-738); an expression left open, or nested past MAX_INTERPOLATION_DEPTH, - * runs to the end of the text. In a Python f-string `\{` still opens a field. + * Adds the text spans of the string whose quote is at `i` and returns the index + * after its closing delimiter. An interpolated expression (`{…}` in a Python + * f- or t-string, `${…}` in a non-raw Dart string) is code: it is skipped + * whole, with its nested strings as their own spans, so a quote inside it never + * ends the outer string (GST-738). An expression left open, or nested past + * MAX_INTERPOLATION_DEPTH, is text to the end of the file. In a Python f-string + * `\{` still opens a field. */ -function stringAt(text: string, i: number, lang: "py" | "dart", depth = 0): [number, number, number] { +function stringAt(text: string, i: number, lang: "py" | "dart", depth: number, spans: Spans): number { const n = text.length; const c = text[i]!; const close = text.startsWith(c.repeat(3), i) ? c.repeat(3) : c; @@ -786,19 +800,27 @@ function stringAt(text: string, i: number, lang: "py" | "dart", depth = 0): [num const raw = lang === "dart" && prefix === "r"; const interpolated = lang === "dart" ? !raw : /^(?:[rR]?[fFtT]|[fFtT][rR])$/.test(prefix); let j = i + close.length; + let from = j; while (j < n && !text.startsWith(close, j) && (multiline || text[j] !== "\n")) { if (!raw && text[j] === "\\") { j += lang === "py" && interpolated && text[j + 1] === "{" ? 1 : 2; continue; } if (!interpolated) { j += 1; continue; } if (lang === "py" && text.startsWith("{{", j)) { j += 2; continue; } if (lang === "py" ? text[j] !== "{" : !text.startsWith("${", j)) { j += 1; continue; } - j = lang === "py" ? pythonFieldEnd(text, j + 1, multiline, depth + 1) : dartExpressionEnd(text, j + 2, depth + 1); + if (from < j) spans.push([from, j]); + const field = j; + const mark = spans.length; + j = lang === "py" ? pythonFieldEnd(text, j + 1, depth + 1, spans) : dartExpressionEnd(text, j + 2, depth + 1, spans); + if (j >= n) { spans.length = mark; from = field; break; } + from = j; } - return [i + close.length, Math.min(j, n), Math.min(j, n) + close.length]; + const end = Math.min(j, n); + if (from < end) spans.push([from, end]); + return end + close.length; } -function stringSpans(text: string, lang: "py" | "dart"): Array<[number, number]> { +function stringSpans(text: string, lang: "py" | "dart"): Spans { if (stringSpansCache?.text === text && stringSpansCache.lang === lang) return stringSpansCache.spans; - const spans: Array<[number, number]> = []; + const spans: Spans = []; const n = text.length; for (let i = 0; i < n; ) { const c = text[i]!; @@ -813,9 +835,7 @@ function stringSpans(text: string, lang: "py" | "dart"): Array<[number, number]> continue; } if (c !== '"' && c !== "'") { i += 1; continue; } - const [from, to, next] = stringAt(text, i, lang); - spans.push([from, to]); - i = next; + i = stringAt(text, i, lang, 0, spans); } stringSpansCache = { text, lang, spans }; return spans; diff --git a/test/redact-engine.test.ts b/test/redact-engine.test.ts index 74c1617f1..0bdef1556 100644 --- a/test/redact-engine.test.ts +++ b/test/redact-engine.test.ts @@ -366,6 +366,8 @@ describe("MEDIUM demoted credential-shaped patterns (TENSION-1)", () => { [`render({\n "id": f"{n:#x}",\n}, """\npassword: ${v}\n""")\n`, "app.py"], [`render({\n "id": f"{n:'>10}",\n}, """\npassword: ${v}\n""")\n`, "app.py"], [`render({\n "id": f"{n!r:{w}}",\n}, """\npassword: ${v}\n""")\n`, "app.py"], + // A # comment in a single-quoted f-string's multi-line field holds no delimiter either. + [`msg = f"{\n 1 # }" """\n}"\nCONFIG = """\npassword: ${v}\n"""\n`, "app.py"], // In an f-string a backslash does not escape the brace after it. [`CONFIG = f"""\\{"""\npassword: ${v}\n"""}"""\n`, "app.py"], // Nesting past the depth cap runs to the end of the text instead of overflowing the stack. @@ -377,6 +379,9 @@ describe("MEDIUM demoted credential-shaped patterns (TENSION-1)", () => { ["msg = f'''{{literal brace'''\ndef f(\n session: HttpSession,\n): pass\n", "app.py"], ["msg = f'{x:>{width}}'\ndef f(\n session: HttpSession,\n): pass\n", "app.py"], ["msg = f\"{when:%b %d '%y} {d[':']} {a[1:2]} {a!=b}\"\nrows = {\n \"password\": session.password,\n}\n", "app.py"], + // Code inside a multi-line interpolation is code, not string text. + ["msg = f\"{\n {\n 'password': request.password,\n }\n}\"\n", "app.py"], + ["final s = \"${login(\n password: request.password,\n)}\";\n", "lib/a.dart"], ["if'a' in s: pass\ndef f(\n session: HttpSession,\n): pass\n", "app.py"], ["final s = 'hi ${user.name} and $id';\n login(password: request.password);\n", "lib/a.dart"], ["final s = r'${raw}';\n login(password: request.password);\n", "lib/a.dart"], From 4d6784baaf35834d1b9b8ad90f99aae7112c26bc Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 1 Nov 2026 03:32:37 +0000 Subject: [PATCH 6/8] docs(changelog): GST-738 fragment covers the review fixes Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- changelog.d/gst-738-secrets-context.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog.d/gst-738-secrets-context.md b/changelog.d/gst-738-secrets-context.md index 469490fb5..7085473c3 100644 --- a/changelog.d/gst-738-secrets-context.md +++ b/changelog.d/gst-738-secrets-context.md @@ -1,6 +1,6 @@ **The secrets check reads Python and Dart string context from what git diffed, and a quote inside an interpolation no longer ends a string (GST-738).** env.kv's source-syntax exemption treats a `password:` line as code only when it sits outside every string. Two inputs could make the scanner place a committed string's contents outside a string. One was a working copy that no longer matched what git diffed. The other was a quote inside an f-string or Dart `${…}` expression. A scoped review pass now applies an explicit `--diff-base` to its Red Team as well. ### Fixed -- **`lib/pregate/checks/secrets.ts`:** a changed Python or Dart file's whole text comes from a full-context `git diff`, never from the working copy. -- **`lib/redact-patterns.ts`:** a Python f-string's `{…}`, or t-string's, and a non-raw Dart string's `${…}` are skipped whole, nested strings and comments included. A quote inside one doesn't end the outer string. A `password:` line inside a Dart `/* */` comment is now read as text, not code. An expression left open runs to the end of the file, so the exemption fails closed. +- **`lib/pregate/checks/secrets.ts`:** a changed Python or Dart file's whole text comes from a full-context `git diff`, never from the working copy. A `diff.suppressBlankEmpty` setting can't shift its lines. +- **`lib/redact-patterns.ts`:** a Python f-string's `{…}`, or t-string's, and a non-raw Dart string's `${…}` are read as code. Their nested strings count as strings, and a quote or comment inside them doesn't end the outer string. A Python format spec, such as `{n:#x}`, is text, and `\{` still opens a field. A `password:` line inside a Dart `/* */` comment is now read as text, not code. An expression left open, or nested more than 256 deep, runs to the end of the file, so the exemption fails closed. - **`bin/gstack-review-dispatch`:** `pass --diff-base` now sets the scoped Red Team's base too. From a4cbca25fbca8ddec2a7ed94291b8f62c3b4e762 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 1 Nov 2026 03:34:14 +0000 Subject: [PATCH 7/8] docs: pregate and review-policy describe the GST-738 secrets context and --diff-base Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- docs/pregate.md | 11 ++++++++--- docs/review-policy.md | 3 ++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/pregate.md b/docs/pregate.md index 1cc821b6d..3bd12c0ad 100644 --- a/docs/pregate.md +++ b/docs/pregate.md @@ -182,7 +182,9 @@ Two stages with separate clocks, both printed: changed Python (`.py`, `.pyi`) or Dart file is scanned whole and only findings on its added lines count, so a line added inside an existing `"""` or `'''` string is still read as a literal; its rows give the file's - line number (`app.py, line 10`). + line number (`app.py, line 10`). The whole text is the version git diffed + (a full-context `git diff`), never the working copy; a file whose full diff + does not parse is scanned on its added lines alone, without that context. - `strays`: a new file at the repository root, a new top-level directory, or a scratch name (`zz-*`, `tmp*`, `scratch*`) anywhere. - `literals`: for every changed exported constant or named timeout/retry @@ -816,8 +818,11 @@ those, each with its scope and its reason: (`session: HttpSession,`) and a dict entry or Dart named argument that reads a name (`"password": request.password`) are code, not credentials, as they are in TypeScript (#3630). Python's keyword form (`password=request.password`) - still reports. The same line inside a `"""` docstring is text and still - reports; allowlist it by its value. The pre-push hook sees only added lines, + still reports. The same line inside a `"""` docstring, any other string or + a Dart `/* */` comment is text and still reports; allowlist it by its value. + Code inside a Python f-string or t-string `{…}` field, or a non-raw Dart + string's `${…}`, is read as code, and its nested strings and quotes do not + end the outer string (GST-738). The pre-push hook sees only added lines, so it scans Python and Dart lines without this exemption and may still warn (MEDIUM, non-blocking) on them. diff --git a/docs/review-policy.md b/docs/review-policy.md index 283671003..4086c2dd0 100644 --- a/docs/review-policy.md +++ b/docs/review-policy.md @@ -505,7 +505,8 @@ ones, and the arm lost blockers. `bin/gstack-review-dispatch` closes both ends: 6. `pass --base --out --pass -- ` runs one pass from a staged manifest (`/pass-manifest.json`): audit readers marked first, then the specialists with QA and the other audits, then the Red Team, then Step 11's - voices. Each reviewer keeps its plan entry and is rendered by the same code as + voices. `--diff-base` sets every reader's base, a scoped pass's Red Team + included; the targeted reader keeps the merge base. Each reviewer keeps its plan entry and is rendered by the same code as `render`, once its inputs exist; the Red Team's findings file is the specialists' `merged.jsonl` lines byte for byte. A brief leaves the bin in indexed chunks of at most 65,536 characters (`chunk`), so no tool output line carries a whole brief and a From 1ed7877f595dc449fd9d10fc9b82cab242129e30 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 1 Nov 2026 14:57:44 +0000 Subject: [PATCH 8/8] docs(pregate): untracked and oversized Python/Dart files in the secrets stage (GST-738 re-audit) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- docs/pregate.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/pregate.md b/docs/pregate.md index 5080c7a9d..78e6c7365 100644 --- a/docs/pregate.md +++ b/docs/pregate.md @@ -182,9 +182,10 @@ Two stages with separate clocks, both printed: changed Python (`.py`, `.pyi`) or Dart file is scanned whole and only findings on its added lines count, so a line added inside an existing `"""` or `'''` string is still read as a literal; its rows give the file's - line number (`app.py, line 10`). The whole text is the version git diffed - (a full-context `git diff`), never the working copy; a file whose full diff - does not parse is scanned on its added lines alone, without that context. + line number (`app.py, line 10`). A tracked file's whole text is the version + git diffed (a full-context `git diff`), never the working copy; an untracked + file is read whole from disk. A file over the size limit, or whose full diff + does not parse, is scanned on its added lines alone, without that context. - `strays`: a new file at the repository root, a new top-level directory, or a scratch name (`zz-*`, `tmp*`, `scratch*`) anywhere. - `literals`: for every changed exported constant or named timeout/retry