perf: bound request work and resource usage (#866)

* perf: bound request work and resource usage

* fix: address performance review regressions

* fix: guard content expiration and dashboard state

* fix: tolerate cache outages and retire derived paths

* fix: revalidate expiration claims and status pagination

* fix: retry interrupted repository metadata cleanup

* fix: retire obsolete content cache storage

* fix: activate tree metadata and recheck expiration claims

* fix: fence cache publication across expiration

* fix: recover staged trees and enforce cache lifecycle cleanup

* fix: prune expired transforms during database outages

* fix: finish conference removals and reload expiration candidates

scripts/performance-benchmark.js

@@ -0,0 +1,79 @@
+const { performance, monitorEventLoopDelay } = require("node:perf_hooks");
+const { strict: assert } = require("node:assert");
+const { Readable, Writable } = require("node:stream");
+const { pipeline } = require("node:stream/promises");
+const { setTimeout: delay } = require("node:timers/promises");
+const { setInterval, clearInterval } = require("node:timers");
+require("ts-node/register/transpile-only");
+const { compileTerms, anonymizePath, anonymizePathCompiled, AnonymizeTransformer } = require("../src/core/anonymize-utils");
+const Repository = require("../src/core/Repository").default;
+const { closeCacheRedis } = require("../src/core/cache-coordination");
+const median = numbers => [...numbers].sort((a, b) => a - b)[Math.floor(numbers.length / 2)];
+function originalSearch(files, q) {
+  const folders = new Set();
+  for (const file of files) {
+    let accumulated = "";
+    for (const segment of file.path.split("/")) {
+      accumulated = accumulated ? `${accumulated}/${segment}` : segment;
+      if (segment.toLowerCase().includes(q)) folders.add(accumulated);
+    }
+  }
+  return files.filter(file => {
+    if (file.name.toLowerCase().includes(q)) return true;
+    const fullPath = `${file.path}/${file.name}`;
+    let found = false;
+    folders.forEach(folder => { if (fullPath.startsWith(folder + "/") || fullPath === folder) found = true; });
+    return found;
+  }).slice(0, 500);
+}
+async function benchmark() {
+  const results = [];
+  if (!process.argv.includes("--memory-only")) {
+    for (const count of [1000, 5000, 10000, 100000]) {
+      const files = Array.from({ length: count }, (_, i) => ({ name: "data.txt", path: `folder-hit-${i}`, size: 10 }));
+      const repo = { anonymizedFiles: async () => files };
+      const old = [], current = [];
+      for (let repeat = 0; repeat < 3; repeat++) {
+        const started = performance.now();
+        const actual = await Repository.prototype.searchFiles.call(repo, "hit");
+        current.push(performance.now() - started);
+        if (count <= 10000) {
+          const baseline = performance.now();
+          const expected = originalSearch(files, "hit");
+          old.push(performance.now() - baseline); assert.deepEqual(actual, expected);
+        }
+      }
+      results.push({ case: "search", files: count, medianMs: median(current), baselineMs: old.length ? median(old) : null,
+        speedup: old.length ? median(old) / median(current) : null });
+    }
+    const terms = ["secret=>hidden", "Researcher", "University"];
+    const paths = Array.from({ length: 10000 }, (_, i) => `src/secret/file${i}.ts`), old = [], current = [];
+    for (let repeat = 0; repeat < 3; repeat++) {
+      let started = performance.now(); const expected = paths.map(p => anonymizePath(p, terms)); old.push(performance.now() - started);
+      started = performance.now(); const compiled = compileTerms(terms); const actual = paths.map(p => anonymizePathCompiled(p, compiled)); current.push(performance.now() - started);
+      assert.deepEqual(actual, expected);
+    }
+    results.push({ case: "compile-once paths", files: paths.length, medianMs: median(current), baselineMs: median(old), speedup: median(old) / median(current) });
+  }
+  if (process.argv.includes("--memory") || process.argv.includes("--memory-only")) {
+    const size = Number(process.env.PERF_FILE_MIB) || 48, count = Number(process.env.PERF_CONCURRENCY) || 4;
+    const before = process.memoryUsage().rss;
+    let peak = before, output = 0;
+    const timer = setInterval(() => { peak = Math.max(peak, process.memoryUsage().rss); }, 20);
+    const histogram = monitorEventLoopDelay({ resolution: 10 }); histogram.enable();
+    await delay(30);
+    const started = performance.now();
+    await Promise.all(Array.from({ length: count }, (_, i) => pipeline(
+      Readable.from((async function* () { for (let chunk = 0; chunk < size * 16; chunk++) yield Buffer.alloc(65536, 65); })()),
+      new AnonymizeTransformer({ filePath: `memory-${i}.txt`, terms: ["Alice"], cacheGeneration: String(Date.now()) }),
+      new Writable({ write(chunk, _encoding, callback) { output += chunk.length; callback(); } })
+    )));
+    histogram.disable(); clearInterval(timer);
+    results.push({ case: "large text", concurrent: count, fileMiB: size, elapsedMs: performance.now() - started,
+      outputMiB: output / 1048576, peakRssMiB: peak / 1048576, rssIncreaseMiB: (peak - before) / 1048576,
+      eventLoopP95Ms: histogram.percentile(95) / 1e6, eventLoopMaxMs: histogram.max / 1e6 });
+  }
+  await closeCacheRedis();
+  console.log(JSON.stringify({ node: process.version, results }, null, 2));
+}
+benchmark().catch(error => { console.error(error); process.exitCode = 1; });

scripts/performance-browser.js

@@ -0,0 +1,60 @@
+// Local fixture server for resource checks in the collaborative browser.
+const express = require("express");
+const fs = require("node:fs");
+const path = require("node:path");
+const app = express();
+const root = path.resolve("public");
+const files = Array.from({ length: 20 }, (_, i) => ({ name: `file${i}.js`, path: "", sha: "1", size: 30 }));
+files.push({ name: "long.pdf", path: "", sha: "1", size: 10000 });
+function pdf(count) {
+  const objects = ["<< /Type /Catalog /Pages 2 0 R >>", "", "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"];
+  const kids = [];
+  for (let i = 0; i < count; i++) {
+    const page = objects.length + 1, content = page + 1;
+    kids.push(`${page} 0 R`);
+    objects.push(`<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> /Contents ${content} 0 R >>`);
+    const text = `BT /F1 24 Tf 72 720 Td (Performance fixture page ${i + 1}) Tj ET`;
+    objects.push(`<< /Length ${text.length} >>\nstream\n${text}\nendstream`);
+  }
+  objects[1] = `<< /Type /Pages /Kids [${kids.join(" ")}] /Count ${count} >>`;
+  let output = "%PDF-1.4\n", offsets = [0];
+  objects.forEach((object, i) => { offsets.push(Buffer.byteLength(output)); output += `${i + 1} 0 obj\n${object}\nendobj\n`; });
+  const xref = Buffer.byteLength(output);
+  output += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`;
+  output += offsets.slice(1).map(offset => String(offset).padStart(10, "0") + " 00000 n \n").join("");
+  output += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`;
+  return Buffer.from(output);
+}
+const document = pdf(100);
+app.use(express.json());
+app.get("/api/user", (_req, res) => res.json({ username: "perf-user" }));
+app.get("/api/user/quota", (_req, res) => res.json({ storage: { used: 0, total: 1000000 }, file: { used: 0, total: 0 }, repository: { used: 150, total: 200 } }));
+app.get("/api/user/dashboard", (req, res) => {
+  let rows = Array.from({ length: 150 }, (_, i) => ({ repoId: `project-${String(i).padStart(3, "0")}`, _type: "repo", status: i % 10 ? "ready" : "error", source: { fullName: `owner/project-${i}` }, pageView: i, options: { terms: [], expirationMode: "never" } }));
+  const total = rows.length;
+  if (req.query.q) rows = rows.filter(row => row.repoId.includes(req.query.q));
+  if (req.query.type && !["all", "repo"].includes(req.query.type)) rows = [];
+  if (req.query.statuses) rows = rows.filter(row => req.query.statuses.split(",").includes(row.status));
+  if (req.query.attention === "true") rows = rows.filter(row => row.status === "error");
+  const offset = Number(req.query.cursor) || 0;
+  res.json({ items: rows.slice(offset, offset + 50), cursor: offset + 50 < rows.length ? String(offset + 50) : null, total, filtered: rows.length, attention: 15 });
+});
+app.get("/api/repo/perf/options", (_req, res) => res.json({ status: "ready", terms: [], image: true, pdf: true, notebook: true, lastUpdateDate: 0 }));
+app.get("/api/repo/perf/files/", (_req, res) => res.json(files));
+app.get("/api/repo/perf/files/counts", (_req, res) => res.json({ "": files.length }));
+app.get("/api/repo/perf/file/long.pdf", (_req, res) => res.type("pdf").send(document));
+app.get("/api/repo/perf/file/:filename", (req, res) => res.type("text/plain").send(`// ${req.params.filename}\nconst value = 1;`));
+app.use("/api", (_req, res) => res.json({}));
+app.use((req, _res, next) => { req.url = req.url.replace(/\.[a-f0-9]{10}\.min\./, ".min."); next(); });
+app.use(express.static(root, { index: false }));
+const html = fs.readFileSync(path.join(root, "index.html"), "utf8")
+  .replace("__CORE_JS__", "core.min.js").replace("__VENDOR_JS__", "vendor.min.js")
+  .replace("__MERMAID_JS__", "mermaid.min.js").replace("__ALL_CSS__", "all.min.css")
+  .replace('<script src="/script/core.min.js">', `<script>
+    window.__hashListeners = new Set();
+    const add = window.addEventListener.bind(window), remove = window.removeEventListener.bind(window);
+    window.addEventListener = (type, callback, options) => { if (type === "hashchange") window.__hashListeners.add(callback); return add(type, callback, options); };
+    window.removeEventListener = (type, callback, options) => { if (type === "hashchange") window.__hashListeners.delete(callback); return remove(type, callback, options); };
+  </script><script src="/script/core.min.js">`);
+app.get(/.*/, (_req, res) => res.type("html").send(html));
+app.listen(4175, "0.0.0.0", () => console.log("Performance fixture: http://localhost:4175"));