diff --git a/docs/performance-plan.md b/docs/performance-plan.md index 05c45ff..d092081 100644 --- a/docs/performance-plan.md +++ b/docs/performance-plan.md @@ -67,7 +67,17 @@ Run `npm run benchmark:performance` for CPU comparisons. Add `-- --memory-only` Before deploying the readers, run `npm run migrate:performance` with the application's database environment. This adds the membership, file-generation, mapping, private-name, maintenance and cleanup-recovery indexes; it never drops indexes. It also queues legacy stream-cache cleanup once. Maintenance preserves managed content caches and ZIP source roots. Rerun it if you applied the migration before the cleanup-recovery indexes were added. The command passed twice against the disposable MongoDB fixture and retained existing indexes. Deploy the compiled worker file with the application build. Source-mode workers use ts-node and have additional startup cost. -Production rollout still needs latency percentiles under the real mixed workload, cleanup-backlog observation, and real S3 multipart/network-failure checks. The local S3 fixture exercises SDK requests and short-upload publication but does not emulate all service failure modes. No deployment or production data changes were performed. +Production rollout still needs latency percentiles under the real mixed workload, cleanup-backlog observation, and real S3 multipart/network-failure checks. The local S3 fixture exercises SDK requests and short-upload publication but does not emulate all service failure modes. PRs #866 and #872 were deployed on 8 October 2026 after clean Codex reviews. API and streamer health, static assets, public repository browsing, and a complete 699-entry ZIP download passed production checks; all 12 required indexes were verified without replacing MongoDB, Redis, or the backup container. + +### Production ZIP anonymization timeouts + +The later log check found four failed ZIP requests for one repository. Its nine terms compiled to 18 RE2 variants with no JavaScript fallback; its largest files were about 8 MB. The one-second synchronous deadline also applied inside workers, so safe whole-file matching could abort a ZIP. A synthetic 7.84 MB dataset with nine safe rules reproduced that error. Merely increasing the worker timeout was insufficient because absent regex terms still triggered repeated RE2 scans over the whole input. + +Regex terms now use a conservative mandatory-prefix absence check where possible. Alternation and unfamiliar syntax keep the full matcher; the optimization never decides replacements. Worker transformation gets ten seconds within the pool's existing fifteen-second deadline. JavaScript-only backtracking replacements keep their own one-second deadline, and synchronous callers retain the one-second whole-operation limit. Output is still published only after complete anonymization succeeds, with the existing memory, spool, cancellation, and queue limits. + +Regression coverage includes safe regex jobs, dense Unicode replacements, a large ZIP entry followed by another entry, prefix parity with the original RE2 matcher, and worker recovery after a backtracking timeout. Build and lint passed, with seven existing lint warnings; the complete suite passed 955 tests with 51 pending and zero failures using disposable MongoDB/Redis and `PERFORMANCE_STREAMER_COMPILED=1`. These include 125 performance tests. A separate 100-case prefix parity probe also passed. + +The candidate Docker image passed the 7.84 MB safe-regex ZIP fixture through real Express compression on Node 22, with plain and gzip responses and zero remaining drain listeners. Its native-regex guard rejected a backtracking job after one second without publishing partial output, and the next worker job succeeded. The affected repository's nine rules were also replayed against synthetic numeric input in isolated containers with one CPU: the deployed image hit its one-second timeout, while the candidate completed in 2.4 seconds. This replay used no original repository file contents and changed no live application containers. Token-refresh warnings, missing-branch HTTP status handling, and backup-retention cleanup remain separate follow-up issues. ## Baseline and implementation rules diff --git a/docs/performance-results.json b/docs/performance-results.json index fbe9748..d42951e 100644 --- a/docs/performance-results.json +++ b/docs/performance-results.json @@ -108,14 +108,14 @@ ], "tests": { "fullSuite": { - "passing": 950, + "passing": 955, "pending": 51, "failing": 0, - "note": "Clean committed checkout after the ZIP HTTP rollout follow-up, with disposable MongoDB/Redis and memory-backed temporary storage. Compiled streamer startup/lifecycle/idle-cleanup integration remains enabled. Two real HTTP tests cover Express compression, plain and gzip output, forced backpressure, listener cleanup and chunked upstream archives.", + "note": "Isolated worktree with disposable MongoDB/Redis, memory-backed temporary storage, and PERFORMANCE_STREAMER_COMPILED=1. New coverage verifies safe-regex prefix parity, large worker and ZIP jobs, dense Unicode replacements, one-second native-regex rejection without partial output, and worker recovery.", "validationDate": "2026-10-08" }, "performanceSuite": { - "passing": 122, + "passing": 125, "note": "Included in the latest clean full-suite run with disposable MongoDB and Redis." }, "uiSuite": { @@ -153,6 +153,31 @@ "gzip": "passed", "forcedBackpressure": "passed", "drainListenerCleanup": "passed" + }, + "zipAnonymization": { + "safeRegexFixtureBytes": 7840011, + "denseUnicodeFixtureBytes": 20800000, + "workerProcessingDeadlineMs": 10000, + "workerJobDeadlineMs": 15000, + "nativeRegexDeadlineMs": 1000, + "extraPrefixParityCases": 100, + "compiledNode22Http": { + "plain": "passed", + "gzip": "passed", + "drainListenerCleanup": "passed" + }, + "compiledNode22Guard": { + "partialOutput": false, + "workerRecovery": "passed" + }, + "productionRulesReplay": { + "scope": "Synthetic numeric input using the affected repository rules, in isolated Node 22 containers limited to one CPU. No original file contents or identifying terms were printed; live application containers were unchanged.", + "terms": 9, + "inputBytes": 7840032, + "baseline": "Script execution timed out after 1000ms", + "candidateComplete": true, + "candidateElapsedMs": 2401.216523 + } } }, "sockets": { diff --git a/src/core/anonymization.worker.ts b/src/core/anonymization.worker.ts index c0882be..d45e145 100644 --- a/src/core/anonymization.worker.ts +++ b/src/core/anonymization.worker.ts @@ -10,7 +10,11 @@ parentPort!.on("message", ({ input, output, options, maxOutput, context }) => { const original = readFileSync(input); const text = original.toString("utf8"); const anonymizer = new ContentAnonimizer(options); - const transformed = anonymizer.anonymize(text); + // Large RE2 jobs legitimately take more than the synchronous one-second + // limit. The pool also terminates this worker after 15 seconds, including + // input/output and compilation. Native backtracking patterns retain their + // own one-second deadline inside ContentAnonimizer. + const transformed = anonymizer.anonymize(text, 10_000); if (Buffer.byteLength(transformed, "utf8") > maxOutput) throw new Error("anonymized_output_too_large"); const bytes = transformed === text ? original : Buffer.from(transformed, "utf8"); if (bytes.length > maxOutput) throw new Error("anonymized_output_too_large"); diff --git a/src/core/anonymize-utils.ts b/src/core/anonymize-utils.ts index fcdc4c1..57452b6 100644 --- a/src/core/anonymize-utils.ts +++ b/src/core/anonymize-utils.ts @@ -284,6 +284,7 @@ interface CompiledTermVariant { // A boundary-free native search for fixed-width patterns can rule out // absent terms. Candidate replacement retains RE2's boundary semantics. literalPrefilter?: RegExp; + requiredPrefix?: RegExp; asciiPattern?: RegExp; asciiReplacement?: string; asciiBefore?: boolean; @@ -316,6 +317,29 @@ function hasCatastrophicBacktracking(src: string): boolean { return false; } +// A leading sequence of literal characters and simple letter classes must +// occur in every match. Stop before unfamiliar syntax, drop a quantified last +// atom, and reject alternation entirely. This is only an absence check; RE2 +// still decides every replacement and all boundaries. +function requiredPrefix(pattern: string): RegExp | undefined { + if (pattern.includes("|")) return undefined; + const atoms: string[] = []; + let offset = 0; + while (offset < pattern.length) { + const char = String.fromCodePoint(pattern.codePointAt(offset)!); + if (char === "[") { + const end = pattern.indexOf("]", offset + 1); + if (end < 0 || !/^[\p{L}\p{N}]+$/u.test(pattern.slice(offset + 1, end))) break; + atoms.push(pattern.slice(offset, end + 1)); offset = end + 1; + } else { + if (/[\\^$.*+?()[\]{}]/.test(char)) break; + atoms.push(char); offset += char.length; + } + } + if (/[?*+{]/.test(pattern.charAt(offset)) && offset < pattern.length) atoms.pop(); + return atoms.length ? new RegExp(atoms.join(""), "iu") : undefined; +} + function compileTerms(terms: string[] | undefined): CompiledTermVariant[] { if (!terms || terms.length === 0) return []; @@ -363,6 +387,7 @@ function compileTerms(terms: string[] | undefined): CompiledTermVariant[] { RE2JS.CASE_INSENSITIVE ); compiled.push({ pattern, before, after, mask, + requiredPrefix: fixedWidth ? undefined : requiredPrefix(variant.pattern), asciiPattern: fixedWidth ? new RegExp(bounded.replace(/\./g, "[^\\n]"), "giu") : undefined, asciiReplacement: fixedWidth ? mask.replace(/\$/g, () => "$$") : undefined, asciiBefore: !variant.unicode && bounded.startsWith("\\b"), @@ -492,6 +517,7 @@ export class ContentAnonimizer { const c = this.compiledTerms[index]; // An absent term cannot occur inside a URL either. if (c.literalPrefilter && !c.literalPrefilter.test(content)) continue; + if (c.requiredPrefix && !c.requiredPrefix.test(content)) continue; const previous = content; content = content.replace(urlRegex, (match) => { if (replaceTerm(match, c, true) !== match) { @@ -515,14 +541,14 @@ export class ContentAnonimizer { return content; } - anonymize(content: string): string { + anonymize(content: string, timeoutMs = 1000): string { return runWithAnonymizationDeadline(() => { content = this.removeImage(content); content = this.removeLink(content); content = this.replaceGitHubSelfLinks(content); content = this.replaceTerms(content); return content; - }); + }, timeoutMs); } } @@ -561,14 +587,18 @@ function escapeRegex(value: string): string { // V8 interrupts even a native RegExp that never returns to JavaScript. A // timeout aborts the operation instead of returning partially anonymized text. const anonymizationScript = new Script("run()"); -function runWithAnonymizationDeadline(run: () => string): string { - return anonymizationScript.runInNewContext({ run }, { timeout: 1000 }); +function runWithAnonymizationDeadline(run: () => string, timeoutMs = 1000): string { + return anonymizationScript.runInNewContext({ run }, { timeout: timeoutMs }); } function replaceTerm(content: string, term: CompiledTermVariant, asciiContent = false): string { - if (term.pattern instanceof RegExp) { - return content.replace(term.pattern, () => term.mask); + const pattern = term.pattern; + if (pattern instanceof RegExp) { + // JavaScript-only patterns can backtrack even inside an isolated worker. + // Keep their short limit independent of the whole-file processing budget. + return runWithAnonymizationDeadline(() => content.replace(pattern, () => term.mask)); } + if (term.requiredPrefix && !term.requiredPrefix.test(content)) return content; if (term.literalPrefilter && asciiContent && term.asciiPattern) { return content.replace(term.asciiPattern, term.asciiReplacement!); } @@ -616,7 +646,7 @@ function replaceTerm(content: string, term: CompiledTermVariant, asciiContent = return pieces.join(""); } - const matcher = term.pattern.matcher(content); + const matcher = pattern.matcher(content); const pieces: string[] = []; let cursor = 0; while (matcher.find()) { diff --git a/test/anonymize-literal-performance.test.js b/test/anonymize-literal-performance.test.js index aa5a752..86f4e8a 100644 --- a/test/anonymize-literal-performance.test.js +++ b/test/anonymize-literal-performance.test.js @@ -39,4 +39,31 @@ describe("literal term presence checks", function () { expect(optimized.wasAnonymized).to.equal(reference.wasAnonymized); } }); + + it("keeps required-prefix absence checks conservative for arbitrary regex syntax", function () { + const cases = [ + { terms: ["Researcher[0-9]+"], text: "absent name Researcher12 Résearcher42" }, + { terms: ["foo?bar"], text: "foobar fobar" }, + { terms: ["foo*bar"], text: "foobar fobar foobarbar" }, + { terms: ["foo{0,2}bar"], text: "fobar foobar fooobar" }, + { terms: ["foo+bar"], text: "foobar fooobar" }, + { terms: ["a?", "[ab]*", "[ab]{0,2}"], text: "x a ab" }, + { terms: ["foo[a]{0}bar", "[ab]?foo"], text: "foobar foo afoo" }, + { terms: ["😀?bar"], text: "bar 😀bar" }, + { terms: ["foo|bar", "foo(bar|baz)"], text: "foo bar foobar foobaz" }, + { terms: ["foo\\|bar", "foo\\.bar"], text: "foo|bar foo.bar" }, + { terms: ["^foo", "(?:foo)?bar", "[A-Z]+foo"], text: "foo bar XXfoo" }, + { terms: ["k[0-9]+", "s[0-9]+"], text: "k12 K12 s34 ſ34 ék12" }, + { terms: ["😀[0-9]+", "研究[0-9]+"], text: "😀12 研究34" }, + { terms: ["Alice=>Researcher12", "Researcher[0-9]+=>Hidden"], text: "Alice" }, + { terms: ["Researcher[0-9]+"], text: "https://host/Researcher12/path" }, + ]; + for (const { terms, text } of cases) { + const reference = new ContentAnonimizer({ terms }); + for (const term of reference.compiledTerms) delete term.requiredPrefix; + const optimized = new ContentAnonimizer({ terms }); + expect(optimized.anonymize(text), JSON.stringify(terms)).to.equal(reference.anonymize(text)); + expect(optimized.wasAnonymized).to.equal(reference.wasAnonymized); + } + }); }); diff --git a/test/large-notebook-download.test.js b/test/large-notebook-download.test.js index 4049c7a..e40ef67 100644 --- a/test/large-notebook-download.test.js +++ b/test/large-notebook-download.test.js @@ -160,6 +160,19 @@ describe("large notebook downloads", function () { expect(JSON.parse(entries["example.ipynb"].toString()).cells[0].source[0]).to.equal("# XXXX-1's experiment"); }); + it("completes large safe regex entries and continues to the next ZIP entry", async function () { + const payload = "123,456,78.91234,-45.123456\n".repeat(280000); + upstream = Readable.from([await zip({ "root/large.csv": `Alice\n${payload}Alice`, "root/after.txt": "Alice" })]); + const response = new PassThrough(), output = collect(response); + await streamAnonymizedZip({ ...options, anonymizerOptions: { + terms: ["Alice", ...Array.from({ length: 8 }, (_, i) => `Researcher${i}[0-9]+`)], + } }, response); + const entries = await unzip(await output); + expect(entries["large.csv"].toString()).to.equal(`XXXX-1\n${payload}XXXX-1`); + expect(entries["after.txt"].toString()).to.equal("XXXX-1"); + expect(response.writableFinished).to.equal(true); + }); + it("aborts the response when an entry cannot be anonymized", async function () { upstream = Readable.from([await zip({ "root/large.txt": "Alice".repeat(1000) })]); config.MAX_FILE_SIZE = 100; diff --git a/test/performance-worker.test.js b/test/performance-worker.test.js index 740bd31..e9350d9 100644 --- a/test/performance-worker.test.js +++ b/test/performance-worker.test.js @@ -59,6 +59,44 @@ describe("large anonymization resources", function () { const options = { filePath: `recovery-${randomUUID()}.txt`, terms: ["Alice=>Hidden"] }; expect(await transform(text, options)).to.equal(new ContentAnonimizer(options).anonymize(text)); }); + it("finishes large files containing many safe regex rules", async () => { + const payload = "123,456,78.91234,-45.123456\n".repeat(280000); + const terms = ["Alice", ...Array.from({ length: 8 }, (_, i) => `Researcher${i}[0-9]+`)]; + const output = await transform(`Alice\n${payload}Alice`, { + filePath: `safe-regex-${randomUUID()}.csv`, terms, + }); + expect(output).to.equal(`XXXX-1\n${payload}XXXX-1`); + }); + + it("uses the worker budget for dense Unicode replacements", async () => { + const input = "Álice,123,45.678,-23.456\n".repeat(800000); + expect(await transform(input, { filePath: `dense-unicode-${randomUUID()}.csv`, terms: ["Alice"] })) + .to.equal("XXXX-1,123,45.678,-23.456\n".repeat(800000)); + }); + + it("retains the short deadline for backtracking regexes on a worker and recovers", async () => { + const root = await fs.mkdtemp(join(tmpdir(), "worker-regex-deadline-")); + const input = join(root, "input"), output = join(root, "output"); + const controller = new AbortController(); + try { + const text = "a".repeat(250); + await fs.writeFile(input, text); + let failure; + const started = Date.now(); + try { + await pool.anonymizeOnWorker({ input, output, options: { terms: ["a+a+a+a+a+a+b(?=x)"] }, + maxOutput: 1000000, context: { mask: "XXXX", hostname: "localhost" } }, text.length, controller.signal); + } catch (error) { failure = error; } + expect(failure).to.be.instanceOf(Error); + expect(failure.message).to.match(/timed out after 1000ms/); + expect(Date.now() - started).to.be.lessThan(8000); + let exists = true; + try { await fs.access(output); } catch (error) { if (error.code === "ENOENT") exists = false; else throw error; } + expect(exists).to.equal(false); + expect(await transform("Alice ".repeat(50000), { filePath: `regex-recovery-${randomUUID()}.txt`, terms: ["Alice"] })) + .to.equal("XXXX-1 ".repeat(50000)); + } finally { controller.abort(); await fs.rm(root, { force: true, recursive: true }); } + }); it("cleans crash leftovers while preserving recent spools and unrelated files", async () => { const root = await fs.mkdtemp(join(tmpdir(), "anonymous-spool-test-")); try {