perf: cut MongoDB scans in repository workflows (#780)

* Add repository name index

* Optimize repository file counts

* Optimize MongoDB query workloads

src/core/model/anonymizedGists/anonymizedGists.schema.ts

@@ -14,7 +14,7 @@ const AnonymizedGistSchema = new Schema({
   anonymizeDate: Date,
   lastView: Date,
   pageView: Number,
-  owner: Schema.Types.ObjectId,
+  owner: { type: Schema.Types.ObjectId, index: true },
   conference: String,
   source: {
     gistId: String,

src/core/model/anonymizedPullRequests/anonymizedPullRequests.schema.ts

@@ -14,7 +14,7 @@ const AnonymizedPullRequestSchema = new Schema({
   anonymizeDate: Date,
   lastView: Date,
   pageView: Number,
-  owner: Schema.Types.ObjectId,
+  owner: { type: Schema.Types.ObjectId, index: true },
   conference: String,
   source: {
     pullRequestId: Number,

src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts

@@ -77,4 +77,15 @@ const AnonymizedRepositorySchema = new Schema({
   },
 });
 
+AnonymizedRepositorySchema.index({ "source.repositoryName": 1 });
+AnonymizedRepositorySchema.index({ status: 1, statusDate: 1 });
+AnonymizedRepositorySchema.index({ lastView: 1 });
+AnonymizedRepositorySchema.index({ anonymizeDate: 1 });
+AnonymizedRepositorySchema.index({ status: 1, isReseted: 1, lastView: 1 });
+AnonymizedRepositorySchema.index({
+  status: 1,
+  isReseted: 1,
+  "options.expirationDate": 1,
+});
+
 export default AnonymizedRepositorySchema;

src/core/model/conference/conferences.schema.ts

@@ -55,4 +55,7 @@ const RepositorySchema = new Schema({
   },
 });
 
+RepositorySchema.index({ owners: 1 });
+RepositorySchema.index({ status: 1, endDate: 1 });
+
 export default RepositorySchema;

src/core/model/files/files.schema.ts

@@ -13,6 +13,14 @@ const FileSchema = new Schema({
 });
 
 FileSchema.index({ path: 1, repoId: 1 });
+FileSchema.index(
+  { repoId: 1, size: 1, path: 1 },
+  { name: "repoId_1_size_1_path_1" }
+);
+FileSchema.index(
+  { repoId: 1, path: 1, name: 1 },
+  { name: "repoId_1_path_1_name_1" }
+);
 
 FileSchema.methods.toString = function () {
   return `${this.path}/${this.name}`;

src/core/model/users/users.schema.ts

@@ -58,4 +58,6 @@ const UserSchema = new Schema({
   },
 });
 
+UserSchema.index({ dateOfEntry: 1 });
+
 export default UserSchema;

src/queue/index.ts

@@ -33,7 +33,9 @@ async function markErrorIfInFlight(repoId: string, message: string) {
           statusMessage: message || "preparation_failed",
         },
       }
-    ).exec();
+    )
+      .collation({ locale: "en", strength: 2 })
+      .exec();
   } catch (e) {
     logger.error("markErrorIfInFlight failed", {
       ...serializeError(e),

src/server/dailyStatsSnapshot.ts

@@ -17,23 +17,37 @@ export interface HomeStatsHistoryRow extends HomeStats {
 }
 
 export async function computeStats(): Promise<HomeStats> {
-  const [nbRepositories, nbUsersAgg, nbPageViews, nbPullRequests] =
+  const [nbRepositories, usageTotals, nbPullRequests] =
     await Promise.all([
       AnonymizedRepositoryModel.estimatedDocumentCount(),
       AnonymizedRepositoryModel.collection
-        .aggregate([{ $group: { _id: "$owner" } }, { $count: "n" }])
+        .aggregate([
-        .toArray(),
+          {
-      AnonymizedRepositoryModel.collection
+            $group: {
-        .aggregate([{ $group: { _id: null, total: { $sum: "$pageView" } } }])
+              _id: "$owner",
+              pageViews: { $sum: "$pageView" },
+            },
+          },
+          {
+            $group: {
+              _id: null,
+              nbUsers: { $sum: 1 },
+              nbPageViews: { $sum: "$pageViews" },
+            },
+          },
+        ])
         .toArray(),
       AnonymizedPullRequestModel.estimatedDocumentCount(),
     ]);
 
+  const usage = usageTotals[0] as
+    | { nbUsers?: number; nbPageViews?: number }
+    | undefined;
+
   return {
     nbRepositories,
-    nbUsers: (nbUsersAgg[0] as { n?: number } | undefined)?.n || 0,
+    nbUsers: usage?.nbUsers || 0,
-    nbPageViews:
+    nbPageViews: usage?.nbPageViews || 0,
-      (nbPageViews[0] as { total?: number } | undefined)?.total || 0,
     nbPullRequests,
   };
 }

src/server/routes/admin.ts

@@ -650,7 +650,6 @@ router.get("/overview", async (req, res) => {
 
     const [
       statusBreakdown,
-      totalSize,
       recentErrors,
       totalUsers,
       totalConferences,
@@ -665,9 +664,6 @@ router.get("/overview", async (req, res) => {
       AnonymizedRepositoryModel.aggregate([
         { $group: { _id: "$status", count: { $sum: 1 }, storage: { $sum: "$size.storage" } } },
       ]),
-      AnonymizedRepositoryModel.aggregate([
-        { $group: { _id: null, total: { $sum: "$size.storage" } } },
-      ]),
       AnonymizedRepositoryModel.countDocuments({
         status: "error",
         statusDate: { $gte: now24h },
@@ -774,7 +770,10 @@ router.get("/overview", async (req, res) => {
       repos: {
         total: totalRepos,
         statusBreakdown,
-        totalStorage: totalSize[0]?.total || 0,
+        totalStorage: statusBreakdown.reduce(
+          (total, row) => total + (row.storage || 0),
+          0
+        ),
         recentErrors24h: recentErrors,
         activeRepos24h,
         newRepos24h,
@@ -802,14 +801,11 @@ router.get("/overview", async (req, res) => {
 // Global stats endpoint: counts by status, total disk, recent failures
 router.get("/stats", async (req, res) => {
   try {
-    const [statusBreakdown, totalSize, recentErrors, totalUsers, totalConferences] =
+    const [statusBreakdown, recentErrors, totalUsers, totalConferences] =
       await Promise.all([
         AnonymizedRepositoryModel.aggregate([
           { $group: { _id: "$status", count: { $sum: 1 }, storage: { $sum: "$size.storage" } } },
         ]),
-        AnonymizedRepositoryModel.aggregate([
-          { $group: { _id: null, total: { $sum: "$size.storage" } } },
-        ]),
         AnonymizedRepositoryModel.countDocuments({
           status: "error",
           statusDate: { $gte: new Date(Date.now() - 1000 * 60 * 60 * 24) },
@@ -819,7 +815,10 @@ router.get("/stats", async (req, res) => {
       ]);
     res.json({
       statusBreakdown,
-      totalStorage: totalSize[0]?.total || 0,
+      totalStorage: statusBreakdown.reduce(
+        (total, row) => total + (row.storage || 0),
+        0
+      ),
       recentErrors24h: recentErrors,
       totalUsers,
       totalConferences,
@@ -921,7 +920,7 @@ router.get("/repos", async (req, res) => {
     );
   }
 
-  const [total, results, statusCounts, sizeAgg] = await Promise.all([
+  const [total, results, statusCounts] = await Promise.all([
     AnonymizedRepositoryModel.find(filter).countDocuments(),
     AnonymizedRepositoryModel.find(filter)
       .skip(skipIndex)
@@ -932,10 +931,6 @@ router.get("/repos", async (req, res) => {
       { $match: filter },
       { $group: { _id: "$status", count: { $sum: 1 }, storage: { $sum: "$size.storage" } } },
     ]),
-    AnonymizedRepositoryModel.aggregate([
-      { $match: filter },
-      { $group: { _id: null, total: { $sum: "$size.storage" } } },
-    ]),
   ]);
   res.json({
     query: filter,
@@ -944,7 +939,10 @@ router.get("/repos", async (req, res) => {
     sort,
     results,
     statusCounts,
-    totalSize: sizeAgg[0]?.total || 0,
+    totalSize: statusCounts.reduce(
+      (total, row) => total + (row.storage || 0),
+      0
+    ),
   });
 });
 

src/server/routes/repository-private.ts

@@ -129,7 +129,7 @@ router.post("/claim", async (req: express.Request, res: express.Response) => {
     await AnonymizedRepositoryModel.updateOne(
       { repoId: repoConfig.repoId },
       { $set: { owner: user.model.id } }
-    );
+    ).collation({ locale: "en", strength: 2 });
     return res.send("Ok");
   } catch (error) {
     handleError(error, res, req);

src/server/routes/repository-public.ts

@@ -196,7 +196,7 @@ router.get(
       const repoId = repo.repoId;
       const results = await FileModel.aggregate([
         { $match: { repoId, size: { $ne: null } } },
-        { $project: { path: 1 } },
+        { $project: { _id: 0, path: 1 } },
         { $group: { _id: "$path", count: { $sum: 1 } } },
       ]).exec();
 

src/server/schedule.ts

@@ -4,6 +4,7 @@ import AnonymizedRepositoryModel from "../core/model/anonymizedRepositories/anon
 import ConferenceModel from "../core/model/conference/conferences.model";
 import Repository from "../core/Repository";
 import { createLogger, serializeError } from "../core/logger";
+import { RepositoryStatus } from "../core/types";
 import { computeAndStoreDailyStats } from "./dailyStatsSnapshot";
 
 const logger = createLogger("schedule");
@@ -11,18 +12,18 @@ const logger = createLogger("schedule");
 export function conferenceStatusCheck() {
   // check every 6 hours the status of the conferences
   schedule.scheduleJob("0 */6 * * *", async () => {
-    (await ConferenceModel.find({ status: { $eq: "ready" } })).forEach(
+    const cursor = ConferenceModel.find({
-      async (data) => {
+      status: "ready",
-        const conference = new Conference(data);
+      endDate: { $lte: new Date() },
-        if (conference.isExpired() && conference.status == "ready") {
+    }).cursor();
-          try {
+    for await (const data of cursor) {
-            await conference.expire();
+      const conference = new Conference(data);
-          } catch (error) {
+      try {
-            logger.error("conference expire failed", serializeError(error));
+        await conference.expire();
-          }
+      } catch (error) {
-        }
+        logger.error("conference expire failed", serializeError(error));
       }
-    );
+    }
   });
 }
 
@@ -30,29 +31,53 @@ export function repositoryStatusCheck() {
   // check every 6 hours the status of the repositories
   schedule.scheduleJob("0 */6 * * *", async () => {
     logger.info("checking repository status and unused repositories");
-    (
+    const now = new Date();
-      await AnonymizedRepositoryModel.find({
+    const fourMonthAgo = new Date(now);
-        status: { $eq: "ready" },
+    fourMonthAgo.setMonth(fourMonthAgo.getMonth() - 4);
-        isReseted: { $eq: false },
+    const cursor = AnonymizedRepositoryModel.find({
-      })
+      status: RepositoryStatus.READY,
-    ).forEach(async (data) => {
+      isReseted: false,
-      const repo = new Repository(data);
+      $or: [
-      try {
+        {
-        await repo.check();
+          "options.expirationMode": { $in: ["redirect", "remove"] },
-      } catch {
+          "options.expirationDate": { $lte: now },
-        logger.info("repository expired", { repoId: repo.repoId });
+        },
-      }
+        { lastView: { $lt: fourMonthAgo } },
-      const fourMonthAgo = new Date();
+      ],
-      fourMonthAgo.setMonth(fourMonthAgo.getMonth() - 4);
+    }).cursor();
+    const batch: Promise<void>[] = [];
+    for await (const data of cursor) {
+      batch.push(
+        (async () => {
+          const repo = new Repository(data);
+          try {
+            await repo.check();
+          } catch {
+            logger.info("repository expired", { repoId: repo.repoId });
+          }
 
-      if (repo.model.lastView < fourMonthAgo) {
+          if (repo.model.lastView < fourMonthAgo) {
-        repo.removeCache().then(() => {
+            try {
-          logger.info("removed cache for unused repository", {
+              await repo.removeCache();
-            repoId: repo.repoId,
+            } catch (error) {
-          });
+              logger.error("repository cache removal failed", {
-        });
+                ...serializeError(error),
+                repoId: repo.repoId,
+              });
+              return;
+            }
+            logger.info("removed cache for unused repository", {
+              repoId: repo.repoId,
+            });
+          }
+        })()
+      );
+      if (batch.length >= 10) {
+        await Promise.all(batch);
+        batch.length = 0;
       }
-    });
+    }
+    await Promise.all(batch);
   });
 }