perf: bound request work and resource usage (#866)

* perf: bound request work and resource usage

* fix: address performance review regressions

* fix: guard content expiration and dashboard state

* fix: tolerate cache outages and retire derived paths

* fix: revalidate expiration claims and status pagination

* fix: retry interrupted repository metadata cleanup

* fix: retire obsolete content cache storage

* fix: activate tree metadata and recheck expiration claims

* fix: fence cache publication across expiration

* fix: recover staged trees and enforce cache lifecycle cleanup

* fix: prune expired transforms during database outages

* fix: finish conference removals and reload expiration candidates

src/server/routes/admin.ts

@@ -1,3 +1,4 @@
+import { AsyncCache } from "../../core/async-cache";
 import * as os from "os";
 import { execSync } from "child_process";
 import { Queue, JobType } from "bullmq";
@@ -342,18 +343,11 @@ router.get("/queues/metrics", dashboardCache, async (req, res) => {
   }
 });
 
-const queuesCache = new Map<string, { data: unknown; ts: number }>();
-const QUEUES_CACHE_TTL = 10_000;
+const queuesCache = new AsyncCache<{ queues: unknown; selectedQueue: string; jobs: Record<string, unknown>[] }>(10_000, 3);
 
 router.get("/queues", dashboardCache, async (req, res) => {
   const search = req.query.search ? String(req.query.search).toLowerCase() : "";
   const queueName = req.query.queue ? String(req.query.queue) : "";
-  const cacheKey = `${queueName}|${search}`;
-  const cached = queuesCache.get(cacheKey);
-  if (cached && Date.now() - cached.ts < QUEUES_CACHE_TTL) {
-    return res.json(cached.data);
-  }
-
   const allQueues: { key: string; label: string; queue: Queue }[] = [
     { key: "download", label: "Download", queue: downloadQueue },
     { key: "remove", label: "Remove", queue: removeQueue },
@@ -365,6 +359,7 @@ router.get("/queues", dashboardCache, async (req, res) => {
     : allQueues[0];
   const targetQueue = target ? target.queue : downloadQueue;
 
+  const data = await queuesCache.get(target?.key || "download", async () => {
   const [statsResults, ...jobsByState] = await Promise.all([
     Promise.all(
       allQueues.map(async (q) => ({
@@ -385,6 +380,19 @@ router.get("/queues", dashboardCache, async (req, res) => {
     }),
   ]);
 
+  const allJobs = (jobsByState as Record<string, unknown>[][]).flat();
+
+  const stateOrder: Record<string, number> = {
+    active: 0, waiting: 1, delayed: 2, failed: 3, completed: 4,
+  };
+  allJobs.sort((a, b) => (stateOrder[a._state as string] ?? 9) - (stateOrder[b._state as string] ?? 9));
+
+  return {
+    queues: statsResults,
+    selectedQueue: target?.key || "download",
+    jobs: allJobs,
+  };
+  });
   const matches = (job: { id?: string | undefined; name?: string }) => {
     if (!search) return true;
     return (
@@ -392,20 +400,7 @@ router.get("/queues", dashboardCache, async (req, res) => {
       (job.name || "").toLowerCase().includes(search)
     );
   };
-  const allJobs = (jobsByState as Record<string, unknown>[][]).flat().filter(matches);
-
-  const stateOrder: Record<string, number> = {
-    active: 0, waiting: 1, delayed: 2, failed: 3, completed: 4,
-  };
-  allJobs.sort((a, b) => (stateOrder[a._state as string] ?? 9) - (stateOrder[b._state as string] ?? 9));
-
-  const data = {
-    queues: statsResults,
-    selectedQueue: target?.key || "download",
-    jobs: allJobs,
-  };
-  queuesCache.set(cacheKey, { data, ts: Date.now() });
-  res.json(data);
+  res.json({ ...data, jobs: data.jobs.filter(matches) });
 });
 
 // Errors captured by the logger sink. Server-paginated to avoid pulling

src/server/routes/dashboard-names.ts

@@ -0,0 +1,48 @@
+import { isConnected } from "../database";
+import { AsyncCache } from "../../core/async-cache";
+import DashboardName from "../../core/model/dashboard-name";
+import Repo from "../../core/model/anonymizedRepositories/anonymizedRepositories.model";
+import PR from "../../core/model/anonymizedPullRequests/anonymizedPullRequests.model";
+import Gist from "../../core/model/anonymizedGists/anonymizedGists.model";
+import User from "../../core/User";
+import { projectNameKey } from "./project-names";
+const backfills = new AsyncCache<void>(5 * 60_000, 64);
+export function invalidateDashboardNames() { backfills.clear(); }
+/** Backfill legacy private names using ID-only cursors, never artifact bodies. */
+export function ensureDashboardNames(user: User) {
+  if (!isConnected) return Promise.resolve();
+  return backfills.get(user.id, async () => {
+    // The user map is authoritative, including after an interrupted dual write.
+    const existing = DashboardName.find({ owner: user.model._id }).lean().cursor();
+    try {
+      let batch: Parameters<typeof DashboardName.bulkWrite>[0] = [];
+      for await (const row of existing) {
+        const name = user.model.projectNames?.get(projectNameKey(row.type, row.artifactId));
+        batch.push(name ? { updateOne: { filter: { _id: row._id }, update: { $set: { name } } } }
+          : { deleteOne: { filter: { _id: row._id } } });
+        if (batch.length === 250) { await DashboardName.bulkWrite(batch); batch = []; }
+      }
+      if (batch.length) await DashboardName.bulkWrite(batch);
+    } finally { await existing.close(); }
+    if (!user.model.projectNames?.size) return;
+    const sources = [
+      { type: "repo", cursor: Repo.find(user.repositoryMembership()).select("repoId").lean().cursor(), field: "repoId" },
+      { type: "pr", cursor: PR.find({ owner: user.id }).select("pullRequestId").lean().cursor(), field: "pullRequestId" },
+      { type: "gist", cursor: Gist.find({ owner: user.id }).select("gistId").lean().cursor(), field: "gistId" },
+    ];
+    for (const { type, cursor, field } of sources) {
+      try {
+        let batch: Parameters<typeof DashboardName.bulkWrite>[0] = [];
+        for await (const row of cursor) {
+          const id = String((row as unknown as Record<string, unknown>)[field] || "");
+          const name = user.model.projectNames!.get(projectNameKey(type, id));
+          if (!name) continue;
+          batch.push({ updateOne: { filter: { owner: user.model._id, type, artifactId: id },
+            update: { $set: { name } }, upsert: true } });
+          if (batch.length === 250) { await DashboardName.bulkWrite(batch); batch = []; }
+        }
+        if (batch.length) await DashboardName.bulkWrite(batch);
+      } finally { await cursor.close(); }
+    }
+  });
+}

src/server/routes/dashboard-summary.ts

@@ -0,0 +1,102 @@
+import DashboardName from "../../core/model/dashboard-name";
+import { ensureDashboardNames } from "./dashboard-names";
+import { PipelineStage, Types } from "mongoose";
+import User from "../../core/User";
+import Repo from "../../core/model/anonymizedRepositories/anonymizedRepositories.model";
+import PullRequest from "../../core/model/anonymizedPullRequests/anonymizedPullRequests.model";
+import Gist from "../../core/model/anonymizedGists/anonymizedGists.model";
+import AnonymousError from "../../core/AnonymousError";
+
+const fields: Record<string, string> = { _name: "_anonymousId", _label: "_label", status: "status",
+  anonymizeDate: "anonymizeDate", lastView: "lastView", pageView: "pageView", "options.expirationDate": "options.expirationDate" };
+export function dashboardPipeline(user: User, query: Record<string, unknown>, now = new Date()): PipelineStage[] {
+  const sort = String(query.sort || "-anonymizeDate");
+  const field = sort.replace(/^-/, "");
+  if (!fields[field]) throw new AnonymousError("invalid_sort", { httpStatus: 400 });
+  const direction = sort.startsWith("-") ? -1 : 1;
+  const limit = Math.min(100, Math.max(1, Math.floor(Number(query.limit)) || 50));
+  const owner = new Types.ObjectId(user.id);
+
+  function project(type: string, id: string, source: unknown, label: unknown, extra: Record<string, unknown>) {
+    return [
+      { $project: { _id: 1, status: 1, statusMessage: 1, anonymizeDate: 1, lastView: 1, pageView: { $ifNull: ["$pageView", 0] },
+        options: 1, conference: 1, ...extra, _type: { $literal: type }, _anonymousId: { $ifNull: [`$${id}`, ""] },
+        _source: source, _fallbackLabel: label,
+
+      } },
+    ];
+  }
+  const repoProject = project("repo", "repoId", "$source.repositoryName", "$source.repositoryName", {
+    repoId: 1, source: { fullName: "$source.repositoryName", commit: "$source.commit" }, coauthors: 1, size: 1,
+    role: { $cond: [{ $eq: ["$owner", owner] }, "owner", "coauthor"] },
+  });
+  const prProject = project("pr", "pullRequestId", { $concat: [{ $ifNull: ["$source.repositoryFullName", ""] }, "#", { $toString: "$source.pullRequestId" }] },
+    { $concat: [{ $ifNull: ["$source.repositoryFullName", ""] }, "#", { $toString: "$source.pullRequestId" }] },
+    { pullRequestId: 1, source: { repositoryFullName: "$source.repositoryFullName", pullRequestId: "$source.pullRequestId" } });
+  const gistProject = project("gist", "gistId", "$source.gistId", { $concat: ["Gist ", { $ifNull: ["$gistId", "$source.gistId"] }] },
+    { gistId: 1, source: { gistId: "$source.gistId" }, isPublic: "$gist.isPublic" });
+  const pipeline: PipelineStage[] = [
+    { $match: { $or: user.repositoryMembership().$or.map(clause => clause.owner ? { owner } : clause) } }, ...repoProject as PipelineStage[],
+    { $unionWith: { coll: PullRequest.collection.name, pipeline: [{ $match: { owner } }, ...prProject] } },
+    { $unionWith: { coll: Gist.collection.name, pipeline: [{ $match: { owner } }, ...gistProject] } },
+    { $lookup: { from: DashboardName.collection.name, let: { type: "$_type", id: "$_anonymousId" },
+      pipeline: [{ $match: { owner, $expr: { $and: [{ $eq: ["$type", "$$type"] }, { $eq: ["$artifactId", "$$id"] }] } } },
+        { $project: { name: 1 } }], as: "_privateName" } },
+    { $set: { projectName: { $ifNull: [{ $arrayElemAt: ["$_privateName.name", 0] }, ""] } } },
+    { $unset: "_privateName" },
+    { $set: { status: { $cond: [{ $and: [{ $eq: ["$status", "ready"] }, { $ne: ["$options.expirationMode", "never"] },
+      { $ne: [{ $ifNull: ["$options.expirationDate", null] }, null] }, { $lte: ["$options.expirationDate", now] }] }, "expired", "$status"] },
+      _label: { $cond: [{ $ne: ["$projectName", ""] }, "$projectName", { $ifNull: ["$_fallbackLabel", "$_anonymousId"] }] } } },
+    { $set: { _statusKey: { $switch: { branches: [
+      { case: { $in: ["$status", ["ready", "error"]] }, then: "$status" },
+      { case: { $in: ["$status", ["expired", "expiring"]] }, then: "expired" },
+      { case: { $in: ["$status", ["removed", "removing"]] }, then: "removed" },
+    ], default: "progress" } } } },
+    { $set: { _attention: { $or: [{ $eq: ["$status", "error"] }, { $eq: ["$_anonymousId", ""] },
+      { $and: [{ $eq: ["$_statusKey", "progress"] }, { $ne: [{ $ifNull: ["$anonymizeDate", "$lastView"] }, null] },
+        { $lt: [{ $ifNull: ["$anonymizeDate", "$lastView"] }, new Date(now.getTime() - 2 * 3600_000)] }] }] },
+      _sort: { $ifNull: [`$${fields[field]}`, ["pageView", "anonymizeDate", "lastView", "options.expirationDate"].includes(field)
+        ? (["pageView"].includes(field) ? 0 : new Date(0)) : ""] } } },
+  ];
+  if (["_name", "_label", "status"].includes(field)) pipeline.push({ $set: { _sort: { $toLower: "$_sort" } } });
+  const filter: Record<string, unknown> = {};
+  if (["repo", "pr", "gist"].includes(String(query.type))) filter._type = query.type;
+  if (query.attention === "true") filter._attention = true;
+  const statuses = String(query.statuses || "ready,progress,error,expired").split(",").filter(s => ["ready", "progress", "error", "expired", "removed"].includes(s));
+  filter._statusKey = { $in: statuses };
+  const search = String(query.q || "").trim().slice(0, 256);
+  if (search) {
+    const pattern = search.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    filter.$or = ["_anonymousId", "_label", "_source", "conference"].map(key => ({ [key]: { $regex: pattern, $options: "i" } }));
+  }
+  const page: PipelineStage.FacetPipelineStage[] = [{ $match: filter }];
+  if (query.cursor) {
+    try {
+      if (String(query.cursor).length > 2048) throw new Error();
+      const cursor = JSON.parse(Buffer.from(String(query.cursor), "base64url").toString());
+      if (cursor.sort !== sort || typeof cursor.type !== "string" || !Types.ObjectId.isValid(cursor.id)) throw new Error();
+      const value = ["anonymizeDate", "lastView", "options.expirationDate"].includes(field) ? new Date(cursor.value) : cursor.value;
+      if ((value instanceof Date && isNaN(value.getTime())) || !(typeof value === "string" || typeof value === "number" || value instanceof Date)) throw new Error();
+      page.push({ $match: { $or: [ { _sort: { [direction === 1 ? "$gt" : "$lt"]: value } },
+        { _sort: value, _type: { $gt: cursor.type } }, { _sort: value, _type: cursor.type, _id: { $gt: new Types.ObjectId(cursor.id) } },
+      ] } });
+    } catch { throw new AnonymousError("invalid_cursor", { httpStatus: 400 }); }
+  }
+  page.push({ $sort: { _sort: direction, _type: 1, _id: 1 } }, { $limit: limit + 1 });
+  pipeline.push({ $facet: { totals: [{ $group: { _id: null, total: { $sum: 1 }, attention: { $sum: { $cond: ["$_attention", 1, 0] } } } }],
+    filtered: [{ $match: filter }, { $count: "count" }], items: page } });
+  return pipeline;
+}
+export async function dashboardSummary(user: User, query: Record<string, unknown>) {
+  await ensureDashboardNames(user);
+  const [result] = await Repo.aggregate(dashboardPipeline(user, query));
+  const limit = Math.min(100, Math.max(1, Math.floor(Number(query.limit)) || 50));
+  const hasMore = result.items.length > limit;
+  const items = result.items.slice(0, limit);
+  const last = items[items.length - 1];
+  const cursor = hasMore ? Buffer.from(JSON.stringify({ sort: String(query.sort || "-anonymizeDate"),
+    value: last._sort, type: last._type, id: last._id })).toString("base64url") : null;
+  for (const item of items) { delete item._sort; delete item._id; }
+  return { items, cursor, total: result.totals[0]?.total || 0, attention: result.totals[0]?.attention || 0,
+    filtered: result.filtered[0]?.count || 0 };
+}

src/server/routes/project-names.ts

@@ -1,3 +1,6 @@
+import { isConnected } from "../database";
+import DashboardName from "../../core/model/dashboard-name";
+import { invalidateDashboardNames } from "./dashboard-names";
 import { createHash } from "crypto";
 import AnonymousError from "../../core/AnonymousError";
 import User from "../../core/User";
@@ -31,7 +34,11 @@ export async function saveProjectName(user: User, input: unknown): Promise<strin
   const path = `projectNames.${key}`;
   await UserModel.updateOne({ _id: user.model._id }, name
     ? { $set: { [path]: name } } : { $unset: { [path]: "" } });
+  invalidateDashboardNames();
   if (name) user.model.set(path, name);
   else user.model.projectNames?.delete(key);
+  if (isConnected && name) await DashboardName.updateOne({ owner: user.model._id, type, artifactId: body.id },
+    { $set: { name } }, { upsert: true });
+  else if (isConnected) await DashboardName.deleteOne({ owner: user.model._id, type, artifactId: body.id });
   return name;
 }

src/server/routes/repository-private.ts

@@ -192,10 +192,11 @@ router.post(
         repo.model.anonymizeDate = new Date();
         updates.anonymizeDate = repo.model.anonymizeDate;
       }
-      await AnonymizedRepositoryModel.updateOne(
-        { _id: repo.model._id },
+      const saved = await AnonymizedRepositoryModel.updateOne(
+        { _id: repo.model._id, status: repo.status, statusDate: repo.model.statusDate },
         { $set: updates }
       ).exec();
+      if (saved && saved.matchedCount === 0) throw new AnonymousError("invalid_status", { httpStatus: 409 });
 
       if (reactivating) {
         // Expiration removes the cached files. Rebuild the saved commit
@@ -481,6 +482,10 @@ router.post(
 
       const repoUpdate = req.body;
 
+      if (repo.status === RepositoryStatus.EXPIRING || repo.status === RepositoryStatus.REMOVING) {
+        throw appError("invalid_status", 409);
+      }
+
       validateNewRepo(repoUpdate);
 
       // Only the source repository/commit/branch backs the cached FileModel —
@@ -602,7 +607,8 @@ router.post(
       }
       repo.model.conference = repoUpdate.conference;
       const saved = await AnonymizedRepositoryModel.updateOne(
-        { _id: repo.model._id, "githubAccess.revision": previousAccessRevision || { $exists: false } },
+        { _id: repo.model._id, status: repo.status, statusDate: repo.model.statusDate,
+          "githubAccess.revision": previousAccessRevision || { $exists: false } },
         {
           $set: {
             settingsSavedAt: new Date(),

src/server/routes/repository-public.ts

@@ -202,7 +202,7 @@ router.get(
     try {
       const repoId = repo.repoId;
       const results = await FileModel.aggregate([
-        { $match: { repoId, size: { $ne: null } } },
+        { $match: { repoId, treeGeneration: repo.model.treeGeneration || { $exists: false }, size: { $ne: null } } },
         { $project: { _id: 0, path: 1 } },
         { $group: { _id: "$path", count: { $sum: 1 } } },
       ]).exec();
@@ -256,45 +256,8 @@ router.get(
       if (!query || query.length < 2) {
         return res.json([]);
       }
-      const allFiles = await repo.anonymizedFiles({
-        includeSha: false,
-        recursive: true,
-      });
-      const q = query.toLowerCase();
+      res.json(await repo.searchFiles(query));
 
-      // Collect folder paths whose name segment matches the query
-      const matchingFolders = new Set<string>();
-      for (const f of allFiles) {
-        const segments = (f.path || "").split("/").filter(Boolean);
-        let accumulated = "";
-        for (const seg of segments) {
-          accumulated = accumulated ? `${accumulated}/${seg}` : seg;
-          if (seg.toLowerCase().includes(q)) {
-            matchingFolders.add(accumulated);
-          }
-        }
-      }
-
-      const matches = allFiles.filter((f) => {
-        // File name matches
-        if (f.name?.toLowerCase().includes(q)) return true;
-        // File is inside a matching folder
-        const fullPath = f.path ? `${f.path}/${f.name}` : f.name;
-        let found = false;
-        matchingFolders.forEach((folder) => {
-          if (fullPath?.startsWith(folder + "/") || fullPath === folder) found = true;
-        })
-        if (found) return true;
-        return false;
-      });
-
-      res.json(
-        matches.slice(0, 500).map((f) => ({
-          name: f.name,
-          path: f.path,
-          size: f.size,
-        }))
-      );
     } catch (error) {
       handleError(error, res, req);
     }
@@ -412,6 +375,7 @@ router.get(
         hasSubmodules:
           (await FileModel.exists({
             repoId: repo.repoId,
+            treeGeneration: repo.model.treeGeneration || { $exists: false },
             name: ".gitmodules",
             path: "",
           })) != null,

src/server/routes/user.ts

@@ -1,3 +1,5 @@
+import DashboardName from "../../core/model/dashboard-name";
+import { dashboardSummary } from "./dashboard-summary";
 import { revokeGrant } from "./github-app";
 import CredentialModel from "../../core/model/credentials/credentials.model";
 import * as express from "express";
@@ -48,20 +50,22 @@ router.get("/", async (req, res) => {
   }
 });
 
+router.get("/dashboard", async (req, res) => {
+  try { res.json(await dashboardSummary(await getUser(req), req.query)); }
+  catch (error) { handleError(error, res, req); }
+});
+
 router.get("/quota", async (req, res) => {
   try {
     const user = await getUser(req);
-    const repositories = (await user.getRepositories()).filter(
-      (r) => r.owner.id === user.model.id
-    );
-    const ready = repositories.filter((r) => r.status == "ready");
+    const ready = await user.quotaRepositories();
 
     let totalStorage = 0;
     let totalFiles = 0;
     const uncachedIds: string[] = [];
     for (const r of ready) {
-      const cached = r.model.size;
-      if (cached && cached.file) {
+      const cached = r.size;
+      if (cached && (cached.file > 0 || r.sizeComputedAt) && cached.storage != null) {
         totalStorage += cached.storage;
         totalFiles += cached.file;
       } else {
@@ -72,7 +76,7 @@ router.get("/quota", async (req, res) => {
     if (uncachedIds.length) {
       const uncachedSet = new Set(uncachedIds);
       const agg = await FileModel.aggregate([
-        { $match: { repoId: { $in: uncachedIds } } },
+        { $match: { $or: ready.filter(r => uncachedSet.has(r.repoId)).map(r => ({ repoId: r.repoId, treeGeneration: r.treeGeneration || { $exists: false } })) } },
         {
           $group: {
             _id: "$repoId",
@@ -90,7 +94,8 @@ router.get("/quota", async (req, res) => {
         const size = byId.get(r.repoId) || { storage: 0, file: 0 };
         totalStorage += size.storage;
         totalFiles += size.file;
-        r.model.size = size;
+        r.size = size;
+        r.sizeComputedAt = new Date();
       }
       if (isConnected) {
         await Promise.all(
@@ -98,8 +103,9 @@ router.get("/quota", async (req, res) => {
             .filter((r) => uncachedSet.has(r.repoId))
             .map((r) =>
               AnonymizedRepositoryModel.updateOne(
-                { _id: r.model._id },
-                { $set: { size: r.model.size } }
+                { _id: r._id, status: "ready", treeGeneration: r.treeGeneration || { $exists: false },
+                  fileMetadataRevision: r.fileMetadataRevision || { $exists: false } },
+                { $set: { size: r.size, sizeComputedAt: r.sizeComputedAt } }
               ).exec()
             )
         );
@@ -207,6 +213,7 @@ router.delete("/", async (req, res) => {
     }
 
     await CredentialModel.deleteMany({ ownerId: user.model._id });
+    if (isConnected) await DashboardName.deleteMany({ owner: user.model._id });
 
     await UserModel.updateOne(
       { _id: user.model._id },