perf: bound request work and resource usage (#866)

* perf: bound request work and resource usage

* fix: address performance review regressions

* fix: guard content expiration and dashboard state

* fix: tolerate cache outages and retire derived paths

* fix: revalidate expiration claims and status pagination

* fix: retry interrupted repository metadata cleanup

* fix: retire obsolete content cache storage

* fix: activate tree metadata and recheck expiration claims

* fix: fence cache publication across expiration

* fix: recover staged trees and enforce cache lifecycle cleanup

* fix: prune expired transforms during database outages

* fix: finish conference removals and reload expiration candidates

test/fixtures/performance-cache-worker.js

@@ -0,0 +1,23 @@
+const process = require("node:process");
+const { Readable } = require("node:stream");
+const { setTimeout: delay } = require("node:timers/promises");
+require("ts-node/register/transpile-only");
+const GitHubStream = require("../../src/core/source/GitHubStream").default;
+const source = new GitHubStream({ repoId: "replicas", organization: "owner", repoName: "repo", commit: "commit",
+  cacheGeneration: process.env.TEST_GENERATION || "generation1", getToken: () => "token" });
+source.downloadWithFallback = async () => {
+  process.send({ producer: true });
+  return Readable.from((async function* () {
+    yield Buffer.from("complete "); await delay(200); yield Buffer.from("content");
+  })());
+};
+process.once("message", async () => {
+  try {
+    const contents = await Promise.all(Array.from({ length: 5 }, async () => {
+      const input = await source.getFileContentCache("file.txt", "replicas", () => ({ sha: "blob", size: 16 }));
+      let text = ""; for await (const chunk of input) text += chunk; return text;
+    }));
+    process.send({ contents }); process.exit(0);
+  } catch (error) { process.send({ error: error.message }); process.exit(1); }
+});
+process.send({ ready: true });

test/fixtures/performance-streamer.js

@@ -0,0 +1,42 @@
+const { Readable } = require("node:stream");
+const path = require("node:path");
+const compiled = process.env.PERFORMANCE_STREAMER_COMPILED === "1";
+if (!compiled) require("ts-node/register/transpile-only");
+const base = path.join(__dirname, compiled ? "../../build" : "../../src");
+const GitHubStream = require(path.join(base, "core/source/GitHubStream")).default;
+const FileSystem = require(path.join(base, "core/storage/FileSystem")).default;
+const maintenance = require(path.join(base, "core/temporary-storage"));
+const startMaintenance = maintenance.startTemporaryStorageMaintenance;
+// Exercise the real startup/timer wiring without waiting a production minute.
+maintenance.startTemporaryStorageMaintenance = () => startMaintenance(25);
+GitHubStream.prototype.downloadWithFallback = async function () {
+  process.send({ download: true });
+  return Readable.from([Buffer.alloc(300000, "x")]);
+};
+const write = FileSystem.prototype.write;
+let release, releaseConnection;
+if (process.env.PERFORMANCE_HOLD_CONNECT === "1") {
+  const database = require(path.join(base, "server/database")), connect = database.connect;
+  database.connect = async (...args) => {
+    const held = new Promise(resolve => { releaseConnection = resolve; });
+    process.send({ connectionHeld: true }); await held;
+    return connect(...args);
+  };
+}
+FileSystem.prototype.write = async function (...args) {
+  if (args[1].endsWith("/hold.txt")) {
+    const held = new Promise(resolve => { release = resolve; });
+    process.send({ held: true }); await held;
+  }
+  return write.apply(this, args);
+};
+if (process.env.CREDENTIAL_KEYS !== "invalid") process.on("message", message => {
+  if (message === "release") release?.();
+  if (message === "connect") releaseConnection?.();
+  if (message === "inspect") {
+    const options = require("mongoose").connection.getClient().options;
+    process.send({ connected: require(path.join(base, "server/database")).isConnected,
+      maxPoolSize: options.maxPoolSize, minPoolSize: options.minPoolSize });
+  }
+});
+require(path.join(base, "streamer/index"));

test/fixtures/s3-server.js

@@ -0,0 +1,62 @@
+const http = require("node:http");
+const { URL } = require("node:url");
+const objects = new Map();
+module.exports = async function s3Fixture() {
+  objects.clear();
+  const requests = { lists: 0, deletions: [] };
+  const server = http.createServer((req, res) => {
+    const url = new URL(req.url, "http://localhost");
+    const key = decodeURIComponent(url.pathname).split("/").slice(2).join("/");
+    if (req.method === "POST" && url.searchParams.has("delete")) {
+      const chunks = [];
+      req.on("data", chunk => chunks.push(chunk));
+      req.on("end", () => {
+        const body = Buffer.concat(chunks).toString();
+        const keys = [...body.matchAll(/<Key>(.*?)<\/Key>/g)].map(match => match[1].replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&amp;/g, "&"));
+        requests.deletions.push(keys);
+        keys.forEach(name => objects.delete(name));
+        res.setHeader("Content-Type", "application/xml");
+        res.end('<DeleteResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>');
+      });
+      return;
+    }
+    if (req.method === "PUT") {
+      const chunks = [];
+      req.on("data", chunk => chunks.push(chunk));
+      req.on("end", () => {
+        let body = Buffer.concat(chunks);
+        if (req.headers["content-encoding"]?.includes("aws-chunked")) {
+          const decoded = []; let offset = 0;
+          while (offset < body.length) {
+            const end = body.indexOf("\r\n", offset); if (end < 0) break;
+            const size = parseInt(body.toString("ascii", offset, end).split(";")[0], 16);
+            if (!size) break;
+            decoded.push(body.subarray(end + 2, end + 2 + size)); offset = end + 2 + size + 2;
+          }
+          body = Buffer.concat(decoded);
+        }
+        if (key) objects.set(key, body);
+        res.setHeader("ETag", '"fixture-etag"'); res.end();
+      });
+      return;
+    }
+    if (url.searchParams.has("list-type")) {
+      const prefix = url.searchParams.get("prefix") || "";
+      requests.lists++;
+      const escaped = value => value.replace(/&/g, "&amp;").replace(/</g, "&lt;");
+      const remaining = [...objects].filter(([name]) => name.startsWith(prefix) && name > (url.searchParams.get("continuation-token") || "")).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0);
+      const page = remaining.slice(0, Number(url.searchParams.get("max-keys") || 1000));
+      const truncated = remaining.length > page.length;
+      const contents = page.map(([name, body]) => `<Contents><Key>${escaped(name)}</Key><Size>${body.length}</Size></Contents>`).join("");
+      res.setHeader("Content-Type", "application/xml");
+      res.end(`<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><IsTruncated>${truncated}</IsTruncated>${truncated ? `<NextContinuationToken>${escaped(page.at(-1)[0])}</NextContinuationToken>` : ""}${contents}</ListBucketResult>`); return;
+    }
+    if (!objects.has(key)) { res.statusCode = 404; res.end("<Error><Code>NoSuchKey</Code></Error>"); return; }
+    const body = objects.get(key);
+    res.setHeader("Content-Length", body.length); res.setHeader("Content-Type", "text/plain");
+    res.setHeader("Last-Modified", new Date(0).toUTCString());
+    if (req.method === "HEAD") res.end(); else res.end(body);
+  });
+  await new Promise(resolve => server.listen(0, "127.0.0.1", resolve));
+  return { endpoint: `http://127.0.0.1:${server.address().port}`, requests, seed: (key, data) => objects.set(key, Buffer.from(data)), close: () => new Promise(resolve => server.close(resolve)) };
+};