mirror of
https://github.com/garrytan/gstack.git
synced 2026-06-18 15:50:11 +02:00
feat(redact): shared redaction engine + taxonomy (pure lib, no behavior change)
Add the foundation for cross-skill PII/secret/legal redaction: - lib/redact-patterns.ts — canonical 3-tier taxonomy (HIGH genuinely-secret credentials, MEDIUM PII/legal/internal + high-FP credential-shaped, LOW surface-only). Tier-1 calibration: Stripe-publishable, Google AIza, JWT, and env-KV are MEDIUM not HIGH (context-variable / high-FP). Validators: Luhn, Shannon-entropy gate, RFC1918 exclusion, wallet sanity. Per-span placeholder suppression (not line-based). - lib/redact-engine.ts — pure scan() + applyRedactions(). Normalization pass (NFKC + zero-width strip + entity decode) with offset map back to original. Oversize input fails CLOSED. No visibility-based tier promotion (records repoVisibility for sterner wording only). Tool-attributed-fence WARN-degrade for obvious doc-examples. Safe preview masking (≤4 leading chars). - 100 unit tests: per-pattern positives, FP filters, validators, email allowlist, no-promotion semantics, tool-fence degrade, normalization, oversize-fail-closed, ReDoS pattern-lint + runtime budget, auto-redact (idempotent, right-to-left, structural-corruption guard). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
@@ -0,0 +1,479 @@
|
||||
/**
|
||||
* redact-engine — pure scanning + auto-redaction over the shared taxonomy.
|
||||
*
|
||||
* No I/O. Deterministic. The CLI shim (`bin/gstack-redact`), the pre-push hook
|
||||
* (`bin/gstack-redact-prepush`), and tests all import from here.
|
||||
*
|
||||
* Key behaviors (locked in /plan-eng-review + two Codex passes):
|
||||
* - Normalization BEFORE matching (NFKC + strip zero-width + decode a small
|
||||
* set of HTML entities) so Unicode-confusable / zero-width evasion fails.
|
||||
* Findings map back to ORIGINAL offsets via an index map.
|
||||
* - ReDoS safety: a hard input-size cap that fails CLOSED (oversize input
|
||||
* returns a single synthetic HIGH "input too large to scan safely" finding,
|
||||
* so callers block rather than skip). Patterns are linear-time (lint-tested).
|
||||
* - NO visibility-based tier mutation. `repoVisibility` is recorded on each
|
||||
* finding (drives sterner AUQ wording in the skill) but never promotes a
|
||||
* MEDIUM to HIGH. (TENSION-2-followup.)
|
||||
* - Placeholder suppression is per-matched-span.
|
||||
* - Tool-attributed fences (``` ```codex-review ``` / ``` ```greptile ```)
|
||||
* degrade credential findings to a non-blocking WARN — UNLESS the span is a
|
||||
* live-format credential the doc-example heuristic can't excuse. No nonce,
|
||||
* no trust exemption (the marker scheme was dropped as theater).
|
||||
*/
|
||||
|
||||
import {
|
||||
PATTERNS,
|
||||
PATTERNS_BY_ID,
|
||||
isPlaceholderSpan,
|
||||
type RedactPattern,
|
||||
type Tier,
|
||||
type Category,
|
||||
} from "./redact-patterns";
|
||||
|
||||
export type RepoVisibility = "public" | "private" | "unknown";
|
||||
|
||||
/** A WARN is a finding that does not block but is surfaced (tool-fence degrade). */
|
||||
export type Severity = Tier | "WARN";
|
||||
|
||||
export interface Finding {
|
||||
id: string;
|
||||
tier: Tier;
|
||||
/** Effective severity after tool-fence degrade. HIGH/MEDIUM/LOW or WARN. */
|
||||
severity: Severity;
|
||||
category: Category;
|
||||
description: string;
|
||||
/** 1-based line in the ORIGINAL (un-normalized) text. */
|
||||
line: number;
|
||||
/** 1-based column in the ORIGINAL text. */
|
||||
col: number;
|
||||
/** Safe-masked preview (never more than 4 leading chars of the secret). */
|
||||
preview: string;
|
||||
/** Whether this finding offers one-keystroke auto-redact (PII subset). */
|
||||
autoRedactable: boolean;
|
||||
/** Repo visibility at scan time — drives sterner AUQ wording, not the tier. */
|
||||
repoVisibility: RepoVisibility;
|
||||
/** True when degraded to WARN because it sat in a tool-attributed fence. */
|
||||
toolFenceDegraded?: boolean;
|
||||
}
|
||||
|
||||
export interface ScanOptions {
|
||||
repoVisibility?: RepoVisibility;
|
||||
/** Extra allowlist entries (exact strings) that suppress a matched span. */
|
||||
allowlist?: string[];
|
||||
/** The invoking user's own email (from `git config user.email`) — allowlisted. */
|
||||
selfEmail?: string;
|
||||
/**
|
||||
* Emails already public in the repo (git log authors, package.json, CODEOWNERS).
|
||||
* Suppressed for `pii.email` since they're not a new leak.
|
||||
*/
|
||||
repoPublicEmails?: string[];
|
||||
/** Hard byte cap. Oversize input fails CLOSED. Default 1 MiB. */
|
||||
maxBytes?: number;
|
||||
}
|
||||
|
||||
export interface ScanResult {
|
||||
findings: Finding[];
|
||||
counts: { HIGH: number; MEDIUM: number; LOW: number; WARN: number };
|
||||
repoVisibility: RepoVisibility;
|
||||
/** True when the input-size cap tripped (caller should BLOCK). */
|
||||
oversize: boolean;
|
||||
}
|
||||
|
||||
const DEFAULT_MAX_BYTES = 1024 * 1024; // 1 MiB
|
||||
|
||||
const EMAIL_ALLOW_DOMAINS = [/@example\.(com|org|net)$/i, /@example\.[a-z]{2,}$/i];
|
||||
const EMAIL_ALLOW_LOCALPARTS = [/^noreply@/i, /^no-reply@/i, /^donotreply@/i];
|
||||
|
||||
// ── Normalization ─────────────────────────────────────────────────────────────
|
||||
|
||||
const ZERO_WIDTH = /[]/g;
|
||||
const HTML_ENTITIES: Record<string, string> = {
|
||||
"&": "&",
|
||||
"<": "<",
|
||||
">": ">",
|
||||
""": '"',
|
||||
"'": "'",
|
||||
"'": "'",
|
||||
};
|
||||
|
||||
/**
|
||||
* Normalize text for matching while producing an index map back to the original.
|
||||
* Returns the normalized string and a function mapping a normalized offset to
|
||||
* the corresponding original offset.
|
||||
*
|
||||
* Strategy: walk the original char-by-char, applying NFKC per char, dropping
|
||||
* zero-width chars, and expanding a small fixed set of HTML entities. Each
|
||||
* emitted normalized char records the original offset it came from. This keeps
|
||||
* the map exact for the transformations we apply (which are all local).
|
||||
*/
|
||||
export function normalizeWithMap(input: string): {
|
||||
normalized: string;
|
||||
map: number[];
|
||||
} {
|
||||
const out: string[] = [];
|
||||
const map: number[] = [];
|
||||
let i = 0;
|
||||
while (i < input.length) {
|
||||
// HTML entity expansion (fixed small set; longest first).
|
||||
let matchedEntity = false;
|
||||
for (const ent in HTML_ENTITIES) {
|
||||
if (input.startsWith(ent, i)) {
|
||||
const rep = HTML_ENTITIES[ent];
|
||||
for (const ch of rep) {
|
||||
out.push(ch);
|
||||
map.push(i);
|
||||
}
|
||||
i += ent.length;
|
||||
matchedEntity = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (matchedEntity) continue;
|
||||
|
||||
const ch = input[i];
|
||||
if (ZERO_WIDTH.test(ch)) {
|
||||
ZERO_WIDTH.lastIndex = 0;
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
ZERO_WIDTH.lastIndex = 0;
|
||||
|
||||
const norm = ch.normalize("NFKC");
|
||||
for (const nch of norm) {
|
||||
out.push(nch);
|
||||
map.push(i);
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
// Sentinel so an offset == length maps to the original length.
|
||||
map.push(input.length);
|
||||
return { normalized: out.join(""), map };
|
||||
}
|
||||
|
||||
// ── Offset → line/col on the ORIGINAL text ────────────────────────────────────
|
||||
|
||||
function lineColAt(original: string, offset: number): { line: number; col: number } {
|
||||
let line = 1;
|
||||
let col = 1;
|
||||
for (let i = 0; i < offset && i < original.length; i++) {
|
||||
if (original[i] === "\n") {
|
||||
line += 1;
|
||||
col = 1;
|
||||
} else {
|
||||
col += 1;
|
||||
}
|
||||
}
|
||||
return { line, col };
|
||||
}
|
||||
|
||||
// ── Safe preview masking ──────────────────────────────────────────────────────
|
||||
|
||||
/** Show ≤4 leading chars, mask the rest. Never reconstructable. */
|
||||
export function maskPreview(span: string): string {
|
||||
const visible = span.slice(0, 4);
|
||||
const masked = span.length > 4 ? "*".repeat(Math.min(span.length - 4, 8)) : "";
|
||||
return `${visible}${masked}${span.length > 12 ? "…" : ""}`;
|
||||
}
|
||||
|
||||
// ── Tool-attributed fence detection ───────────────────────────────────────────
|
||||
|
||||
const TOOL_FENCE_INFO = /^```(codex-review|greptile|eval|codex|tool-output)\b/;
|
||||
|
||||
/**
|
||||
* Returns a sorted list of [start, end) offset ranges (in normalized text) that
|
||||
* sit inside a tool-attributed fenced code block. Credential findings inside
|
||||
* these ranges degrade to WARN (unless the doc-example heuristic says the span
|
||||
* is live-format and must still block).
|
||||
*/
|
||||
function toolFenceRanges(normalized: string): Array<[number, number]> {
|
||||
const ranges: Array<[number, number]> = [];
|
||||
const lines = normalized.split("\n");
|
||||
let offset = 0;
|
||||
let inFence = false;
|
||||
let fenceStart = 0;
|
||||
for (const ln of lines) {
|
||||
const isFenceMarker = ln.startsWith("```");
|
||||
if (isFenceMarker) {
|
||||
if (!inFence && TOOL_FENCE_INFO.test(ln)) {
|
||||
inFence = true;
|
||||
fenceStart = offset + ln.length + 1; // content starts after this line
|
||||
} else if (inFence) {
|
||||
ranges.push([fenceStart, offset]); // up to start of closing fence
|
||||
inFence = false;
|
||||
}
|
||||
}
|
||||
offset += ln.length + 1; // +1 for the \n
|
||||
}
|
||||
if (inFence) ranges.push([fenceStart, normalized.length]); // unterminated → still degrade its own body
|
||||
return ranges;
|
||||
}
|
||||
|
||||
function inRanges(offset: number, ranges: Array<[number, number]>): boolean {
|
||||
for (const [s, e] of ranges) if (offset >= s && offset < e) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Doc-example heuristic: a credential span inside a tool fence still BLOCKS if
|
||||
* it looks like a LIVE credential (not an obvious placeholder/example). We only
|
||||
* downgrade-to-WARN spans that are clearly illustrative.
|
||||
*/
|
||||
function isObviousDocExample(span: string): boolean {
|
||||
return isPlaceholderSpan(span);
|
||||
}
|
||||
|
||||
// ── Proximity check ───────────────────────────────────────────────────────────
|
||||
|
||||
function hasNear(
|
||||
normalized: string,
|
||||
matchStart: number,
|
||||
matchEnd: number,
|
||||
nearRegex: RegExp,
|
||||
window: number,
|
||||
): boolean {
|
||||
const from = Math.max(0, matchStart - window);
|
||||
const to = Math.min(normalized.length, matchEnd + window);
|
||||
const slice = normalized.slice(from, to);
|
||||
const re = new RegExp(nearRegex.source, nearRegex.flags.replace(/g/g, ""));
|
||||
return re.test(slice);
|
||||
}
|
||||
|
||||
// ── Email allowlist ───────────────────────────────────────────────────────────
|
||||
|
||||
function emailAllowed(email: string, opts: ScanOptions): boolean {
|
||||
const lower = email.toLowerCase();
|
||||
if (opts.selfEmail && lower === opts.selfEmail.toLowerCase()) return true;
|
||||
if (opts.repoPublicEmails?.some((e) => e.toLowerCase() === lower)) return true;
|
||||
if (EMAIL_ALLOW_DOMAINS.some((re) => re.test(email))) return true;
|
||||
if (EMAIL_ALLOW_LOCALPARTS.some((re) => re.test(email))) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
// ── The scan ──────────────────────────────────────────────────────────────────
|
||||
|
||||
export function scan(input: string, opts: ScanOptions = {}): ScanResult {
|
||||
const repoVisibility: RepoVisibility = opts.repoVisibility ?? "unknown";
|
||||
const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
|
||||
|
||||
// Fail CLOSED on oversize input. Check byte length BEFORE heavy work.
|
||||
const byteLen = Buffer.byteLength(input, "utf8");
|
||||
if (byteLen > maxBytes) {
|
||||
const finding: Finding = {
|
||||
id: "engine.input_too_large",
|
||||
tier: "HIGH",
|
||||
severity: "HIGH",
|
||||
category: "secret",
|
||||
description: `Input too large to scan safely (${byteLen} > ${maxBytes} bytes) — blocking fail-closed`,
|
||||
line: 1,
|
||||
col: 1,
|
||||
preview: "",
|
||||
autoRedactable: false,
|
||||
repoVisibility,
|
||||
};
|
||||
return {
|
||||
findings: [finding],
|
||||
counts: { HIGH: 1, MEDIUM: 0, LOW: 0, WARN: 0 },
|
||||
repoVisibility,
|
||||
oversize: true,
|
||||
};
|
||||
}
|
||||
|
||||
const { normalized, map } = normalizeWithMap(input);
|
||||
const fenceRanges = toolFenceRanges(normalized);
|
||||
const allow = new Set(opts.allowlist ?? []);
|
||||
|
||||
const findings: Finding[] = [];
|
||||
// Dedup by (id, original-offset) so overlapping global matches don't double-count.
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const pat of PATTERNS) {
|
||||
const re = new RegExp(pat.regex.source, withFlags(pat.regex.flags));
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(normalized)) !== null) {
|
||||
// Guard against zero-width matches looping forever.
|
||||
if (m.index === re.lastIndex) re.lastIndex++;
|
||||
|
||||
const span = m[1] ?? m[0];
|
||||
const spanStartInMatch = m[1] !== undefined ? m[0].indexOf(m[1]) : 0;
|
||||
const normOffset = m.index + Math.max(0, spanStartInMatch);
|
||||
|
||||
// Per-span placeholder suppression.
|
||||
if (isPlaceholderSpan(span)) continue;
|
||||
if (allow.has(span)) continue;
|
||||
|
||||
// Pattern-specific validators (Luhn, entropy, RFC1918, etc).
|
||||
if (pat.validate && !pat.validate(span, m)) continue;
|
||||
|
||||
// Proximity requirement.
|
||||
if (
|
||||
pat.nearRegex &&
|
||||
!hasNear(normalized, m.index, m.index + m[0].length, pat.nearRegex, pat.nearWindow ?? 100)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Email allowlist (layered on top of the pattern).
|
||||
if (pat.id === "pii.email" && emailAllowed(span, opts)) continue;
|
||||
|
||||
const origOffset = map[Math.min(normOffset, map.length - 1)] ?? 0;
|
||||
const key = `${pat.id}:${origOffset}`;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
const { line, col } = lineColAt(input, origOffset);
|
||||
|
||||
// Tool-fence degrade: only credential-category, only obvious doc examples.
|
||||
let severity: Severity = pat.tier;
|
||||
let toolFenceDegraded = false;
|
||||
if (
|
||||
pat.category === "secret" &&
|
||||
inRanges(normOffset, fenceRanges) &&
|
||||
isObviousDocExample(span)
|
||||
) {
|
||||
severity = "WARN";
|
||||
toolFenceDegraded = true;
|
||||
}
|
||||
|
||||
findings.push({
|
||||
id: pat.id,
|
||||
tier: pat.tier,
|
||||
severity,
|
||||
category: pat.category,
|
||||
description: pat.description,
|
||||
line,
|
||||
col,
|
||||
preview: maskPreview(span),
|
||||
autoRedactable: !!pat.autoRedactable,
|
||||
repoVisibility,
|
||||
...(toolFenceDegraded ? { toolFenceDegraded } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Stable order: by line, then col, then id.
|
||||
findings.sort((a, b) => a.line - b.line || a.col - b.col || a.id.localeCompare(b.id));
|
||||
|
||||
const counts = { HIGH: 0, MEDIUM: 0, LOW: 0, WARN: 0 };
|
||||
for (const f of findings) counts[f.severity] += 1;
|
||||
|
||||
return { findings, counts, repoVisibility, oversize: false };
|
||||
}
|
||||
|
||||
function withFlags(flags: string): string {
|
||||
let f = flags;
|
||||
if (!f.includes("g")) f += "g";
|
||||
if (!f.includes("m")) f += "m";
|
||||
return f;
|
||||
}
|
||||
|
||||
// ── Auto-redaction ────────────────────────────────────────────────────────────
|
||||
|
||||
export interface RedactResult {
|
||||
body: string;
|
||||
/** ASCII unified-diff preview of the substitutions. */
|
||||
diff: string;
|
||||
/** Findings that could NOT be auto-redacted (structural-corruption guard). */
|
||||
skipped: Finding[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Substitute redact tokens for the given finding ids, right-to-left so offsets
|
||||
* stay valid. Refuses to redact a span that sits inside a structural token
|
||||
* (markdown link target, JSON string value) — those fall back to `skipped` so
|
||||
* the skill drops the user to manual edit rather than silently mangling output.
|
||||
*/
|
||||
export function applyRedactions(
|
||||
input: string,
|
||||
findingIds: string[],
|
||||
opts: ScanOptions = {},
|
||||
): RedactResult {
|
||||
const ids = new Set(findingIds);
|
||||
const { findings } = scan(input, opts);
|
||||
const targets = findings
|
||||
.filter((f) => ids.has(f.id) && f.autoRedactable)
|
||||
.map((f) => ({ f, ...locateSpan(input, f) }))
|
||||
.filter((t) => t.start >= 0);
|
||||
|
||||
// Right-to-left so earlier offsets remain valid after splicing.
|
||||
targets.sort((a, b) => b.start - a.start);
|
||||
|
||||
const skipped: Finding[] = [];
|
||||
const diffLines: string[] = [];
|
||||
let body = input;
|
||||
|
||||
for (const t of targets) {
|
||||
const pat = PATTERNS_BY_ID[t.f.id];
|
||||
const token = pat?.redactToken ?? "<REDACTED>";
|
||||
if (inStructuralToken(body, t.start, t.end)) {
|
||||
skipped.push(t.f);
|
||||
continue;
|
||||
}
|
||||
const before = lineContaining(body, t.start);
|
||||
body = body.slice(0, t.start) + token + body.slice(t.end);
|
||||
const after = lineContaining(body, t.start);
|
||||
diffLines.push(`- ${before}`);
|
||||
diffLines.push(`+ ${after}`);
|
||||
}
|
||||
|
||||
return { body, diff: diffLines.reverse().join("\n"), skipped };
|
||||
}
|
||||
|
||||
function locateSpan(input: string, f: Finding): { start: number; end: number } {
|
||||
// Re-derive the offset from line/col on the original text.
|
||||
let offset = 0;
|
||||
let line = 1;
|
||||
while (line < f.line && offset < input.length) {
|
||||
if (input[offset] === "\n") line++;
|
||||
offset++;
|
||||
}
|
||||
offset += f.col - 1;
|
||||
const pat = PATTERNS_BY_ID[f.id];
|
||||
if (!pat) return { start: -1, end: -1 };
|
||||
const re = new RegExp(pat.regex.source, withFlags(pat.regex.flags));
|
||||
re.lastIndex = Math.max(0, offset - 2);
|
||||
const m = re.exec(input);
|
||||
if (!m) return { start: -1, end: -1 };
|
||||
const span = m[1] ?? m[0];
|
||||
const start = m.index + (m[1] !== undefined ? m[0].indexOf(m[1]) : 0);
|
||||
return { start, end: start + span.length };
|
||||
}
|
||||
|
||||
function inStructuralToken(body: string, start: number, end: number): boolean {
|
||||
// Markdown link target: [text](...span...). The span may sit anywhere inside
|
||||
// the parenthesized target (e.g. an email embedded in a URL). Walk backward
|
||||
// from the span: if we reach `](` before hitting `)`/whitespace, and forward
|
||||
// we reach `)` before whitespace, the span is inside a link target.
|
||||
for (let i = start - 1; i >= 0; i--) {
|
||||
const ch = body[i];
|
||||
if (ch === ")" || ch === "\n" || ch === " " || ch === "\t") break;
|
||||
if (ch === "(" && i > 0 && body[i - 1] === "]") {
|
||||
for (let j = end; j < body.length; j++) {
|
||||
const c = body[j];
|
||||
if (c === " " || c === "\t" || c === "\n") break;
|
||||
if (c === ")") return true;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
// JSON string value: "key": "...span..." — span is inside a quoted value.
|
||||
const before = body.slice(Math.max(0, start - 80), start);
|
||||
const after = body.slice(end, Math.min(body.length, end + 4));
|
||||
if (/:\s*"$/.test(before) && /^"/.test(after)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function lineContaining(body: string, offset: number): string {
|
||||
const start = body.lastIndexOf("\n", offset - 1) + 1;
|
||||
let end = body.indexOf("\n", offset);
|
||||
if (end === -1) end = body.length;
|
||||
return body.slice(start, end);
|
||||
}
|
||||
|
||||
// ── Exit-code helper for the CLI shim ─────────────────────────────────────────
|
||||
|
||||
/** 0 clean, 2 MEDIUM present (no HIGH), 3 HIGH present. WARN does not gate. */
|
||||
export function exitCodeFor(result: ScanResult): 0 | 2 | 3 {
|
||||
if (result.counts.HIGH > 0) return 3;
|
||||
if (result.counts.MEDIUM > 0) return 2;
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user