perf: count folder files from a covering index

src/core/folder-counts.ts

@@ -21,7 +21,7 @@ export async function folderCounts(repo: Repository, requestedPath: unknown): Pr
   }
   const prefix = originalPath ? originalPath + "/" : "";
   const escapedPrefix = prefix.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-  const prefixLength = Array.from(prefix).length;
+  const childDepth = originalPath ? originalPath.split("/").length : 0;
   const rows = await FileModel.aggregate<{ _id: string; count: number }>([
     { $match: {
       repoId: repo.repoId,
@@ -29,12 +29,11 @@ export async function folderCounts(repo: Repository, requestedPath: unknown): Pr
       size: { $ne: null },
       ...(originalPath ? { $or: [{ path: originalPath }, { path: { $regex: "^" + escapedPrefix } }] } : {}),
     } },
-    { $project: { child: { $cond: [
+    // Group directly by the immediate child using only indexed fields.
+    { $group: { _id: originalPath ? { $cond: [
       { $eq: ["$path", { $literal: originalPath }] }, "",
-      { $arrayElemAt: [{ $split: [{ $substrCP: ["$path", prefixLength,
-        { $subtract: [{ $strLenCP: "$path" }, prefixLength] }] }, "/"] }, 0] },
-    ] } } },
-    { $group: { _id: "$child", count: { $sum: 1 } } },
+      { $arrayElemAt: [{ $split: ["$path", "/"] }, childDepth] },
+    ] } : { $arrayElemAt: [{ $split: ["$path", "/"] }, 0] }, count: { $sum: 1 } } },
     { $limit: FILE_LISTING_LIMIT + 1 },
   ]).option({ maxTimeMS: 2000, allowDiskUse: true }).exec();
   if (rows.length > FILE_LISTING_LIMIT) throw new AnonymousError("file_listing_too_large", { httpStatus: 413 });

src/core/model/files/files.schema.ts

@@ -25,6 +25,8 @@ FileSchema.index({ metadataPending: 1, repoId: 1, treeGeneration: 1 },
 FileSchema.index({ treeStaged: 1, repoId: 1, treeGeneration: 1 },
   { partialFilterExpression: { treeStaged: true } });
 FileSchema.index({ repoId: 1, treeGeneration: 1, path: 1, name: 1 });
+// Cover generation-scoped count aggregations without fetching every file.
+FileSchema.index({ repoId: 1, treeGeneration: 1, path: 1, size: 1 }, { name: "file_counts_covering" });
 FileSchema.index(
   { repoId: 1, size: 1, path: 1 },
   { name: "repoId_1_size_1_path_1" }

src/scripts/create-performance-indexes.ts

@@ -14,6 +14,7 @@ async function main() {
   // Additive only. Never use syncIndexes here, which can drop deployed indexes.
   await Repository.collection.createIndex({ "coauthors.githubId": 1 });
   await File.collection.createIndex({ repoId: 1, treeGeneration: 1, path: 1, name: 1 });
+  await File.collection.createIndex({ repoId: 1, treeGeneration: 1, path: 1, size: 1 }, { name: "file_counts_covering" });
   await File.collection.createIndex({ metadataPending: 1, repoId: 1, treeGeneration: 1 },
     { partialFilterExpression: { metadataPending: true } });
   await Repository.collection.createIndex({ retiredTreeGenerations: 1 }, { sparse: true });