import { Worker } from "worker_threads"; import { existsSync } from "fs"; import { join } from "path"; const MEMORY_BUDGET = Math.max(128, Number(process.env.ANONYMIZATION_MEMORY_MB) || 512) * 1024 * 1024; const MAX_WORKERS = Math.min(4, Math.max(1, Number(process.env.ANONYMIZATION_WORKERS) || 1)); interface Job { message: { input: string; output: string; options: unknown; maxOutput: number; context: { mask: string; hostname: string } }; weight: number; signal: AbortSignal; resolve: (changed: boolean) => void; reject: (error: Error) => void; timer?: NodeJS.Timeout; abort?: () => void; } const waiting: Job[] = []; const idle: Worker[] = []; let running = 0; let bytes = 0; export function anonymizeOnWorker(message: Job["message"], size: number, signal: AbortSignal): Promise { const weight = 64 * 1024 * 1024 + size * 4; if (weight > MEMORY_BUDGET) return Promise.reject(new Error("anonymization_memory_budget_exceeded")); if (waiting.length >= 16) return Promise.reject(new Error("anonymization_queue_full")); return new Promise((resolve, reject) => { const job: Job = { message, weight, signal, resolve, reject }; job.abort = () => { const index = waiting.indexOf(job); if (index < 0) return; waiting.splice(index, 1); clearTimeout(job.timer); signal.removeEventListener("abort", job.abort!); reject(new Error("anonymization_cancelled")); }; if (signal.aborted) return reject(new Error("anonymization_cancelled")); signal.addEventListener("abort", job.abort, { once: true }); job.timer = setTimeout(job.abort, 30_000); waiting.push(job); pump(); }); } function pump() { while (waiting.length && running < MAX_WORKERS && bytes + waiting[0].weight <= MEMORY_BUDGET) { const job = waiting.shift()!; clearTimeout(job.timer); job.signal.removeEventListener("abort", job.abort!); running++; bytes += job.weight; const compiled = join(__dirname, "anonymization.worker.js"); let worker: Worker; try { worker = idle.pop() || new Worker(existsSync(compiled) ? compiled : join(__dirname, "anonymization.worker.ts"), { execArgv: existsSync(compiled) ? [] : ["-r", require.resolve("ts-node/register/transpile-only")], resourceLimits: { maxOldGenerationSizeMb: Math.ceil(MEMORY_BUDGET / 1024 / 1024) }, }); } catch (error) { running--; bytes -= job.weight; job.reject(error as Error); continue; } worker.ref(); let finished = false; const finish = (error?: Error, changed = false) => { if (finished) return; finished = true; clearTimeout(deadline); job.signal.removeEventListener("abort", cancel); worker.removeListener("message", message); worker.removeListener("error", failed); worker.removeListener("exit", exited); const release = () => { running--; bytes -= job.weight; pump(); }; // Large jobs can leave substantial unreachable buffers in V8 until GC. // Release their workers before admitting the next job. if (error || job.weight >= 96 * 1024 * 1024) { void worker.terminate().then(release, release); } else { idle.push(worker); worker.unref(); release(); } if (error) job.reject(error); else job.resolve(changed); }; const message = (result: { error?: string; changed?: boolean }) => finish(result.error ? new Error(result.error) : undefined, result.changed); const failed = (error: Error) => finish(error); const exited = () => finish(new Error("anonymization_worker_exited")); const cancel = () => finish(new Error("anonymization_cancelled")); const deadline = setTimeout(() => finish(new Error("anonymization_deadline_exceeded")), 15_000); worker.once("message", message); worker.once("error", failed); worker.once("exit", exited); job.signal.addEventListener("abort", cancel, { once: true }); if (job.signal.aborted) cancel(); else worker.postMessage(job.message); } } let spoolBytes = 0; export function reserveSpool(bytes: number) { if (spoolBytes + bytes > MEMORY_BUDGET * 2) throw new Error("anonymization_spool_budget_exceeded"); spoolBytes += bytes; } export function releaseSpool(bytes: number) { spoolBytes -= bytes; }