diff --git a/scripts/benchmark-anonymizer.js b/scripts/benchmark-anonymizer.js new file mode 100644 index 0000000..0ffc9e7 --- /dev/null +++ b/scripts/benchmark-anonymizer.js @@ -0,0 +1,31 @@ +// Run with node scripts/benchmark-anonymizer.js [path-to-anonymize-utils.ts]. +// Fixtures are synthetic; reported timings exclude fixture creation and I/O. +const path = require("node:path"); +const { performance } = require("node:perf_hooks"); +require("ts-node/register/transpile-only"); +const { ContentAnonimizer } = require(process.argv[2] + ? path.resolve(process.argv[2]) : "../src/core/anonymize-utils"); +const terms = ["first.last", "user@example.org", "dept.example.edu", "lab.io", "project.org", "Author X. Name", + ...Array.from({ length: 41 }, (_, i) => `Researcher${i}`)]; +const row = "123,456,78.91234,-45.123456\n"; +const cases = [ + ["2 MB CSV, 47 absent terms", () => row.repeat(70000), terms], + ["100 MB CSV, 47 absent terms", () => row.repeat(3600000), terms], + ["50 MB CSV, sparse matches", () => `first.last\n${row.repeat(1800000)}\nhttps://host/user@example.org`, terms], + ["22 MB text, 1.2 million replacements", () => "Alice 123 first.last,45.678,-23.456\n".repeat(600000), ["Alice", ...terms]], +]; +for (const [name, fixture, rules] of cases) { + const input = fixture(); + const startCompile = performance.now(); + const anonymizer = new ContentAnonimizer({ terms: rules }); + const compileMs = performance.now() - startCompile; + const started = performance.now(); + try { + const output = anonymizer.anonymize(input); + console.log(JSON.stringify({ name, inputBytes: Buffer.byteLength(input), outputBytes: Buffer.byteLength(output), + compileMs: +compileMs.toFixed(1), matchMs: +(performance.now() - started).toFixed(1) })); + } catch (error) { + console.log(JSON.stringify({ name, inputBytes: Buffer.byteLength(input), + compileMs: +compileMs.toFixed(1), matchMs: +(performance.now() - started).toFixed(1), error: error.code || error.message })); + } +} diff --git a/src/core/anonymize-utils.ts b/src/core/anonymize-utils.ts index 379f481..24e03a1 100644 --- a/src/core/anonymize-utils.ts +++ b/src/core/anonymize-utils.ts @@ -6,6 +6,7 @@ import { isBinaryFileSync } from "isbinaryfile"; import { lookup as lookupMime } from "mime-types"; import config from "../config"; +import { TermPresenceIndex } from "./term-presence"; import { parseTermSpec, termVariants, @@ -211,9 +212,13 @@ const markdownImageRegex = interface CompiledTermVariant { // RE2 for regular patterns; time-limited native fallback for JS extensions. pattern: RE2JS | RegExp; - // A boundary-free, fixed-width native search can cheaply rule out literal - // terms before RE2 scans a large file. A hit still uses the usual matcher. + // A boundary-free native search for fixed-width patterns can rule out + // absent terms. Candidate replacement retains RE2's boundary semantics. literalPrefilter?: RegExp; + asciiPattern?: RegExp; + asciiReplacement?: string; + asciiBefore?: boolean; + asciiAfter?: boolean; before: boolean; after: boolean; mask: string; @@ -269,11 +274,14 @@ function compileTerms(terms: string[] | undefined): CompiledTermVariant[] { if (!useAsRegex || hasCatastrophicBacktracking(term)) { term = term.replace(/[-[\]{}()*+?.,\\^$|#]/g, "\\$&"); } - // Only plain literals qualify: expanding their letters produces fixed - // character classes, never user-controlled backtracking. Do not apply - // this shortcut to regex syntax, including escaped literals. - const isLiteral = !/[\\^$.*+?()[\]{}|]/.test(term); + // Plain text and unquantified dots have fixed width in code points. + // Exclude all other regex syntax so the native search cannot backtrack + // exponentially. This includes dotted names, emails and domains. + const isFixedWidth = !/[\\^$*+?()[\]{}|]/.test(term); for (const variant of termVariants(term)) { + // Stripping a combining-mark-only term can produce an empty variant. + // Keep zero-width matches on RE2's progress-aware iterator. + const fixedWidth = isFixedWidth && variant.pattern.length > 0; const bounded = withWordBoundaries(variant.pattern, { sniffSource: variant.sniff, unicode: variant.unicode, @@ -286,9 +294,15 @@ function compileTerms(terms: string[] | undefined): CompiledTermVariant[] { RE2JS.CASE_INSENSITIVE ); compiled.push({ pattern, before, after, mask, + asciiPattern: fixedWidth ? new RegExp(bounded.replace(/\./g, "[^\\n]"), "giu") : undefined, + asciiReplacement: fixedWidth ? mask.replace(/\$/g, () => "$$") : undefined, + asciiBefore: !variant.unicode && bounded.startsWith("\\b"), + asciiAfter: !variant.unicode && bounded.endsWith("\\b"), // RE2 uses Unicode case folding even for the non-diacritic pass. // Omit boundaries so this search can only rule out absent terms. - literalPrefilter: isLiteral ? new RegExp(variant.pattern, "iu") : undefined }); + literalPrefilter: fixedWidth + // RE2's dot excludes only LF; JS dot also excludes CR and U+2028/9. + ? new RegExp(variant.pattern.replace(/\./g, "[^\\n]"), "iu") : undefined }); } catch { // Retain JavaScript-only syntax and large repetition counts under // the execution deadline; RE2 handles the common case without backtracking. @@ -303,13 +317,37 @@ function compileTerms(terms: string[] | undefined): CompiledTermVariant[] { return compiled; } +interface CompiledTermPlan { + terms: CompiledTermVariant[]; + presence: TermPresenceIndex | null; +} + +// Every file in a ZIP uses the same rules. Reuse immutable compiled patterns +// across transformers, with bounded retention for unrelated repositories. +const termPlans = new Map(); +function compiledTermPlan(terms: string[] | undefined): CompiledTermPlan { + const key = JSON.stringify([config.ANONYMIZATION_MASK, terms || []]); + const cached = termPlans.get(key); + if (cached) return cached; + const compiled = compileTerms(terms); + const plan = { + terms: compiled, + presence: compiled.length && compiled.length <= 256 && compiled.every(term => term.literalPrefilter) + ? new TermPresenceIndex(compiled.map(term => term.literalPrefilter!)) : null, + }; + if (key.length <= 8192 && compiled.length <= 256) { + if (termPlans.size >= 64) termPlans.delete(termPlans.keys().next().value!); + termPlans.set(key, plan); + } + return plan; +} + export class ContentAnonimizer { public wasAnonymized = false; - // Compiled once per instance and reused for every anonymize() call. - // Streamed files invoke anonymize() many times per file (one per chunk), - // so caching here avoids rebuilding regexes on every chunk. + // Per-instance metadata references the bounded cache of compiled rules. private compiledTerms: CompiledTermVariant[]; private selfLinkRegexes: RegExp[] | null = null; + private termPresence: TermPresenceIndex | null = null; constructor( readonly opt: { @@ -321,7 +359,11 @@ export class ContentAnonimizer { repoId?: string; } ) { - this.compiledTerms = compileTerms(opt.terms); + const plan = compiledTermPlan(opt.terms); + // Keep per-instance metadata independent. Regex matching is synchronous; + // native replacement resets lastIndex and RE2 creates a fresh matcher. + this.compiledTerms = plan.terms.map(term => ({ ...term })); + this.termPresence = plan.presence; if (opt.repoName && opt.branchName) { const r = escapeRegex(opt.repoName); const b = escapeRegex(opt.branchName); @@ -370,19 +412,36 @@ export class ContentAnonimizer { } private replaceTerms(content: string): string { - for (const c of this.compiledTerms) { - // remove whole url if it contains the term + if (!this.compiledTerms.length) return content; + let candidates = this.termPresence?.candidates(content); + // On ASCII input both engines have identical fixed-width matching and + // word boundaries. Native replacement avoids allocating one JS match + // object per CSV value. Non-ASCII input keeps RE2 validation below. + let asciiContent = (!candidates || candidates.size > 0) && /^\p{ASCII}*$/u.test(content); + for (let index = 0; index < this.compiledTerms.length; index++) { + if (candidates && !candidates.has(index)) continue; + const c = this.compiledTerms[index]; + // An absent term cannot occur inside a URL either. + if (c.literalPrefilter && !c.literalPrefilter.test(content)) continue; + const previous = content; content = content.replace(urlRegex, (match) => { - if (replaceTerm(match, c) !== match) { + if (replaceTerm(match, c, true) !== match) { this.wasAnonymized = true; return c.mask; } return match; }); - // remove the term in the text - const replaced = replaceTerm(content, c); + // URL replacement may already have inserted a non-ASCII custom mask. + if (content !== previous) asciiContent = asciiContent && /^\p{ASCII}*$/u.test(c.mask); + const replaced = replaceTerm(content, c, asciiContent); if (replaced !== content) this.wasAnonymized = true; content = replaced; + // Masks and changed boundaries can introduce later matches. Re-index + // only after a real change, never once per absent term. + if (content !== previous) { + asciiContent = asciiContent && /^\p{ASCII}*$/u.test(c.mask); + candidates = this.termPresence?.candidates(content, index); + } } return content; } @@ -437,38 +496,62 @@ function runWithAnonymizationDeadline(run: () => string): string { return anonymizationScript.runInNewContext({ run }, { timeout: 1000 }); } -function replaceTerm(content: string, term: CompiledTermVariant): string { - if (term.literalPrefilter && !term.literalPrefilter.test(content)) return content; +function replaceTerm(content: string, term: CompiledTermVariant, asciiContent = false): string { if (term.pattern instanceof RegExp) { return content.replace(term.pattern, () => term.mask); } - // Fixed-width literal searches run natively. Validate each candidate with - // RE2 on a small window to preserve its case folding and boundary semantics - // without scanning megabytes of notebook outputs in the JS regex engine. - const candidates = term.literalPrefilter - ? content.matchAll(new RegExp(term.literalPrefilter.source, "giu")) - : null; - const matcher = candidates ? null : term.pattern.matcher(content); + if (term.literalPrefilter && asciiContent && term.asciiPattern) { + return content.replace(term.asciiPattern, term.asciiReplacement!); + } + if (term.literalPrefilter) { + const validated = new Map(); + const candidates = new RegExp(term.literalPrefilter.source, "giu"); + const pieces: string[] = []; + let cursor = 0; + let match: RegExpExecArray | null; + while ((match = candidates.exec(content))) { + const candidate = match[0], start = match.index, end = start + candidate.length; + let valid = true; + if (term.before && /[\p{L}\p{N}_]$/u.test(content.slice(Math.max(0, start - 2), start))) valid = false; + if (term.after && /^[\p{L}\p{N}_]/u.test(content.slice(end, end + 2))) valid = false; + // On ASCII candidates the fixed-width native and RE2 searches agree. + // Check RE2's ASCII word boundaries explicitly: JS /iu\b/ also treats + // Kelvin sign and long s as word characters, while RE2 does not. + if (valid && /^\p{ASCII}*$/u.test(candidate)) { + if (term.asciiBefore && asciiWordAt(content, start - 1) === asciiWordAt(content, start)) valid = false; + if (term.asciiAfter && asciiWordAt(content, end - 1) === asciiWordAt(content, end)) valid = false; + } else if (valid) { + // Keep RE2 authoritative for Unicode case folding. Repeated values in + // datasets share small contexts; cap this cache independently of size. + const offset = Math.max(0, start - 2); + const window = content.slice(offset, end + 2); + const key = `${start - offset}:${window}`; + const cached = validated.get(key); + if (cached === undefined) { + const check = (term.pattern as RE2JS).matcher(window); + valid = check.find(start - offset) && check.start() === start - offset && check.end() === end - offset; + if (validated.size < 256) validated.set(key, valid); + } else valid = cached; + } + if (!valid) { + // A boundary-rejected candidate must not hide an overlapping match, + // e.g. the second a-a in "xa-a-a". Advance by one Unicode code point. + candidates.lastIndex = start + (content.codePointAt(start)! > 0xffff ? 2 : 1); + continue; + } + pieces.push(content.slice(cursor, start), term.mask); + cursor = end; + } + if (!pieces.length) return content; + pieces.push(content.slice(cursor)); + return pieces.join(""); + } + + const matcher = term.pattern.matcher(content); const pieces: string[] = []; let cursor = 0; - const matches = function* () { - if (candidates) { - for (const candidate of candidates) { - const start = candidate.index!; - const end = start + candidate[0].length; - const offset = Math.max(0, start - 2); - const check = (term.pattern as RE2JS).matcher(content.slice(offset, end + 2)); - if (check.find(start - offset) && check.start() === start - offset && check.end() === end - offset) { - yield { start, end }; - } - } - } else { - while (matcher!.find()) yield { start: matcher!.start(), end: matcher!.end() }; - } - }; - for (const { start, end } of matches()) { - // RE2 has no lookahead. Check the generated Unicode word boundaries - // outside the engine, without executing any user-supplied native regex. + while (matcher.find()) { + const start = matcher.start(), end = matcher.end(); if (term.before && /[\p{L}\p{N}_]$/u.test(content.slice(Math.max(0, start - 2), start))) continue; if (term.after && /^[\p{L}\p{N}_]/u.test(content.slice(end, end + 2))) continue; pieces.push(content.slice(cursor, start), term.mask); @@ -478,3 +561,8 @@ function replaceTerm(content: string, term: CompiledTermVariant): string { pieces.push(content.slice(cursor)); return pieces.join(""); } + +function asciiWordAt(content: string, index: number): boolean { + const code = content.charCodeAt(index); + return code >= 48 && code <= 57 || code >= 65 && code <= 90 || code >= 97 && code <= 122 || code === 95; +} diff --git a/src/core/term-presence.ts b/src/core/term-presence.ts new file mode 100644 index 0000000..934344d --- /dev/null +++ b/src/core/term-presence.ts @@ -0,0 +1,67 @@ +/** + * Candidate index for nonempty, capture-free, fixed-width compiler patterns. + * False positives are safe; false negatives are not. + */ +export class TermPresenceIndex { + private groups: { indices: number[]; characters: string[] | null }[] = []; + + constructor(private readonly patterns: RegExp[]) { + for (let index = 0; index < patterns.length; index++) { + const characters = firstCharacters(patterns[index].source); + const group = { indices: [index], characters }; + // Merge groups whose first characters can overlap under Unicode folding. + // Then a combined search can identify one group without hiding another + // alternative at the same position (Alice versus AliceSmith, for example). + for (let i = this.groups.length - 1; i >= 0; i--) { + const other = this.groups[i]; + if (overlap(group.characters, other.characters)) { + group.indices.push(...other.indices); + group.characters = group.characters && other.characters + ? [...new Set([...group.characters, ...other.characters])] : null; + this.groups.splice(i, 1); + } + } + this.groups.push(group); + } + } + + candidates(content: string, after = -1): Set { + const found = new Set(); + const remaining = this.groups.map(group => group.indices.filter(index => index > after)) + .filter(indices => indices.length); + let offset = 0; + while (remaining.length) { + // Fixed-width inputs contain no captures or quantifiers. Removing a + // discovered group avoids revisiting millions of identical CSV values. + const search = new RegExp(remaining.map(indices => + `(${indices.map(index => this.patterns[index].source).join("|")})`).join("|"), "giu"); + search.lastIndex = offset; + const match = search.exec(content); + if (!match) break; + const selected = match.slice(1).findIndex(value => value !== undefined); + for (const index of remaining[selected]) found.add(index); + remaining.splice(selected, 1); + // Another group's match may start inside this one. Search from the next + // code point, not from the end of the match, to retain nested candidates. + offset = match.index + (content.codePointAt(match.index)! > 0xffff ? 2 : 1); + } + return found; + } +} + +function firstCharacters(source: string): string[] | null { + if (source.startsWith("[")) { + const characters = source.slice(1, source.indexOf("]")); + // Only the compiler's expanded letter classes have an enumerable prefix. + // Dot's [^\n], escapes and any future syntax conservatively join all groups. + return /^[\p{L}\p{N}]+$/u.test(characters) ? Array.from(characters) : null; + } + if (source.startsWith("\\")) return null; + return [Array.from(source)[0]]; +} + +function overlap(left: string[] | null, right: string[] | null): boolean { + if (!left || !right) return true; + const pattern = new RegExp(`^(?:${left.map(char => char.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|")})$`, "iu"); + return right.some(char => pattern.test(char)); +} diff --git a/test/anonymizer-large-files.test.js b/test/anonymizer-large-files.test.js new file mode 100644 index 0000000..38573d7 --- /dev/null +++ b/test/anonymizer-large-files.test.js @@ -0,0 +1,28 @@ +const { expect } = require("chai"); +const { Readable } = require("node:stream"); +require("ts-node/register/transpile-only"); +const { AnonymizeTransformer, ContentAnonimizer } = require("../src/core/anonymize-utils"); +const terms = ["first.last", "user@example.org", "dept.example.edu", "lab.io", "project.org", "Author X. Name", + ...Array.from({ length: 41 }, (_, i) => `Researcher${i}`)]; + +describe("large files with many anonymization terms", function () { + this.timeout(10000); + it("handles a 50 MB numeric dataset without exhausting the one-second deadline", function () { + const input = "123,456,78.91234,-45.123456\n".repeat(1800000); + expect(new ContentAnonimizer({ terms }).anonymize(input)).to.equal(input); + }); + it("redacts dotted terms and their URLs around a large payload", async function () { + const payload = "123,456,78.91234,-45.123456\n".repeat(200000); + const input = Buffer.from(`first.last\n${payload}\nhttps://host/user@example.org/path\nfirstXlast`); + const stream = new AnonymizeTransformer({ filePath: "coords.csv", terms }); + Readable.from([input.subarray(0, 6), input.subarray(6)]).pipe(stream); + const chunks = []; + for await (const chunk of stream) chunks.push(chunk); + expect(Buffer.concat(chunks).toString()).to.equal(`XXXX-1\n${payload}\nXXXX-2\nXXXX-1`); + }); + it("handles many repeated matches while preserving the surrounding values", function () { + const input = "Alice,123,45.678,-23.456\n".repeat(100000); + expect(new ContentAnonimizer({ terms: ["Alice", ...terms] }).anonymize(input)) + .to.equal("XXXX-1,123,45.678,-23.456\n".repeat(100000)); + }); +}); diff --git a/test/term-presence.test.js b/test/term-presence.test.js new file mode 100644 index 0000000..63c7544 --- /dev/null +++ b/test/term-presence.test.js @@ -0,0 +1,75 @@ +const { expect } = require("chai"); +require("ts-node/register/transpile-only"); +const { TermPresenceIndex } = require("../src/core/term-presence"); +const { compileTerms, ContentAnonimizer } = require("../src/core/anonymize-utils"); + +function reference(terms, text) { + const anonymizer = new ContentAnonimizer({ terms }); + anonymizer.termPresence = null; + for (const term of anonymizer.compiledTerms) delete term.literalPrefilter; + return anonymizer.anonymize(text); +} + +describe("term candidate index", function () { + it("retains shared prefixes, nested matches and Unicode-folded alternatives", function () { + const patterns = [/Alice/iu, /AliceSmith/iu, /Smith/iu, /[sśŝşšș]/iu, /ſ/iu, /K/iu, /k/iu, /研究/iu]; + const index = new TermPresenceIndex(patterns); + for (const text of ["AliceSmith", "ALICESMITH K ſ 研究", "nothing", "ś", "k", "aAliceSmith"]) { + for (let after = -1; after < patterns.length; after++) { + const candidates = index.candidates(text, after); + patterns.forEach((pattern, i) => { + if (i > after && pattern.test(text)) expect(candidates.has(i), `${text}: ${i}`).to.equal(true); + if (i <= after) expect(candidates.has(i)).to.equal(false); + }); + } + } + }); + + it("keeps dot's CR and Unicode line-separator matches", function () { + for (const text of ["first\rlast", "first\u2028last", "first\u2029last", "first\nlast", "first😀last"]) { + expect(new ContentAnonimizer({ terms: ["first.last"] }).anonymize(text)).to.equal(reference(["first.last"], text)); + } + }); + + it("retains ordered cascades, URL removal, and matches created across replacement boundaries", function () { + const cases = [ + [["Alice=>Bob", "Bob=>Hidden"], "Alice Bob https://host/Alice"], + [["Alice=>", "@Bob=>Hidden"], "@AliceBob"], + [["a-a", "Alice"], "xa-a-a Alice https://host/Alice/file"], + [["first.last", "firstXlast=>other"], "firstXlast first.last"], + [["Alice=>Bob", "Bob"], "https://host/Alice"], + [["k", "s"], "K ſ ék k_ xK Kx"], + [["Alice=>Bob", "B.b=>Done"], "Alice"], + [["k=>Kk"], "https://host/k"], + [["Alice=>$&", "Bob=>$$"], "Alice Bob"], + [["\u0301"], "abc"], + [["a.*b", "Alice"], "axxb Alice"], + ]; + for (const [terms, text] of cases) { + expect(new ContentAnonimizer({ terms }).anonymize(text), JSON.stringify([terms, text])).to.equal(reference(terms, text)); + } + }); + + it("matches the RE2 path for fixed-width terms and surrounding characters", function () { + const terms = ["a-a", "a.a", "k", "s", "k.s", "Alice", "@Alice", "Σ", "研究", "😀", "..", "a.", "a..b"]; + const edges = ["", "x", "_", "é", "K", "ſ", "😀", "𐐀", "\r", "\n", "\u2028"]; + for (const term of terms) { + const input = edges.flatMap(left => edges.map(right => `${left}${term}${right} ${left}${term.replaceAll(".", "\r")}${right}`)).join(" "); + expect(new ContentAnonimizer({ terms: [term] }).anonymize(input), term).to.equal(reference([term], input)); + } + }); + + it("does not reuse stale rules after the caller edits its term list", function () { + const terms = ["Alice=>first"]; + expect(new ContentAnonimizer({ terms }).anonymize("Alice")).to.equal("first"); + terms[0] = "Alice=>second"; + expect(new ContentAnonimizer({ terms }).anonymize("Alice")).to.equal("second"); + expect(new ContentAnonimizer({ terms: ["Alice=>first"] }).anonymize("Alice")).to.equal("first"); + }); + + it("does not index quantified, capturing, or escaped user patterns", function () { + for (const term of ["a.*b", "a+b", "(Alice)", "a\\.b", "a(?=b)"]) { + expect(compileTerms([term]).every(c => !c.literalPrefilter), term).to.equal(true); + } + }); +});