From 7aa9557905fb568e170047d0659bedac451ab894 Mon Sep 17 00:00:00 2001 From: Thomas Durieux <5577568+tdurieux@users.noreply.github.com> Date: Thu, 20 Aug 2026 00:29:26 -1000 Subject: [PATCH] perf: cut MongoDB scans in repository workflows (#780) * Add repository name index * Optimize repository file counts * Optimize MongoDB query workloads --- .../anonymizedGists/anonymizedGists.schema.ts | 2 +- .../anonymizedPullRequests.schema.ts | 2 +- .../anonymizedRepositories.schema.ts | 11 +++ .../model/conference/conferences.schema.ts | 3 + src/core/model/files/files.schema.ts | 8 ++ src/core/model/users/users.schema.ts | 2 + src/queue/index.ts | 4 +- src/server/dailyStatsSnapshot.ts | 30 +++++-- src/server/routes/admin.ts | 30 +++---- src/server/routes/repository-private.ts | 2 +- src/server/routes/repository-public.ts | 2 +- src/server/schedule.ts | 89 ++++++++++++------- 12 files changed, 124 insertions(+), 61 deletions(-) diff --git a/src/core/model/anonymizedGists/anonymizedGists.schema.ts b/src/core/model/anonymizedGists/anonymizedGists.schema.ts index b5b212d..edbde52 100644 --- a/src/core/model/anonymizedGists/anonymizedGists.schema.ts +++ b/src/core/model/anonymizedGists/anonymizedGists.schema.ts @@ -14,7 +14,7 @@ const AnonymizedGistSchema = new Schema({ anonymizeDate: Date, lastView: Date, pageView: Number, - owner: Schema.Types.ObjectId, + owner: { type: Schema.Types.ObjectId, index: true }, conference: String, source: { gistId: String, diff --git a/src/core/model/anonymizedPullRequests/anonymizedPullRequests.schema.ts b/src/core/model/anonymizedPullRequests/anonymizedPullRequests.schema.ts index 290a2e1..20cf546 100644 --- a/src/core/model/anonymizedPullRequests/anonymizedPullRequests.schema.ts +++ b/src/core/model/anonymizedPullRequests/anonymizedPullRequests.schema.ts @@ -14,7 +14,7 @@ const AnonymizedPullRequestSchema = new Schema({ anonymizeDate: Date, lastView: Date, pageView: Number, - owner: Schema.Types.ObjectId, + owner: { type: Schema.Types.ObjectId, index: true }, conference: String, source: { pullRequestId: Number, diff --git a/src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts b/src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts index 8213a58..6383b18 100644 --- a/src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts +++ b/src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts @@ -77,4 +77,15 @@ const AnonymizedRepositorySchema = new Schema({ }, }); +AnonymizedRepositorySchema.index({ "source.repositoryName": 1 }); +AnonymizedRepositorySchema.index({ status: 1, statusDate: 1 }); +AnonymizedRepositorySchema.index({ lastView: 1 }); +AnonymizedRepositorySchema.index({ anonymizeDate: 1 }); +AnonymizedRepositorySchema.index({ status: 1, isReseted: 1, lastView: 1 }); +AnonymizedRepositorySchema.index({ + status: 1, + isReseted: 1, + "options.expirationDate": 1, +}); + export default AnonymizedRepositorySchema; diff --git a/src/core/model/conference/conferences.schema.ts b/src/core/model/conference/conferences.schema.ts index ad98c0e..a0bad4c 100644 --- a/src/core/model/conference/conferences.schema.ts +++ b/src/core/model/conference/conferences.schema.ts @@ -55,4 +55,7 @@ const RepositorySchema = new Schema({ }, }); +RepositorySchema.index({ owners: 1 }); +RepositorySchema.index({ status: 1, endDate: 1 }); + export default RepositorySchema; diff --git a/src/core/model/files/files.schema.ts b/src/core/model/files/files.schema.ts index a9f630c..90f01bc 100644 --- a/src/core/model/files/files.schema.ts +++ b/src/core/model/files/files.schema.ts @@ -13,6 +13,14 @@ const FileSchema = new Schema({ }); FileSchema.index({ path: 1, repoId: 1 }); +FileSchema.index( + { repoId: 1, size: 1, path: 1 }, + { name: "repoId_1_size_1_path_1" } +); +FileSchema.index( + { repoId: 1, path: 1, name: 1 }, + { name: "repoId_1_path_1_name_1" } +); FileSchema.methods.toString = function () { return `${this.path}/${this.name}`; diff --git a/src/core/model/users/users.schema.ts b/src/core/model/users/users.schema.ts index f632fb6..59b5889 100644 --- a/src/core/model/users/users.schema.ts +++ b/src/core/model/users/users.schema.ts @@ -58,4 +58,6 @@ const UserSchema = new Schema({ }, }); +UserSchema.index({ dateOfEntry: 1 }); + export default UserSchema; diff --git a/src/queue/index.ts b/src/queue/index.ts index 40bf1f2..a5ae75d 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -33,7 +33,9 @@ async function markErrorIfInFlight(repoId: string, message: string) { statusMessage: message || "preparation_failed", }, } - ).exec(); + ) + .collation({ locale: "en", strength: 2 }) + .exec(); } catch (e) { logger.error("markErrorIfInFlight failed", { ...serializeError(e), diff --git a/src/server/dailyStatsSnapshot.ts b/src/server/dailyStatsSnapshot.ts index 675a3d2..eb4cee6 100644 --- a/src/server/dailyStatsSnapshot.ts +++ b/src/server/dailyStatsSnapshot.ts @@ -17,23 +17,37 @@ export interface HomeStatsHistoryRow extends HomeStats { } export async function computeStats(): Promise { - const [nbRepositories, nbUsersAgg, nbPageViews, nbPullRequests] = + const [nbRepositories, usageTotals, nbPullRequests] = await Promise.all([ AnonymizedRepositoryModel.estimatedDocumentCount(), AnonymizedRepositoryModel.collection - .aggregate([{ $group: { _id: "$owner" } }, { $count: "n" }]) - .toArray(), - AnonymizedRepositoryModel.collection - .aggregate([{ $group: { _id: null, total: { $sum: "$pageView" } } }]) + .aggregate([ + { + $group: { + _id: "$owner", + pageViews: { $sum: "$pageView" }, + }, + }, + { + $group: { + _id: null, + nbUsers: { $sum: 1 }, + nbPageViews: { $sum: "$pageViews" }, + }, + }, + ]) .toArray(), AnonymizedPullRequestModel.estimatedDocumentCount(), ]); + const usage = usageTotals[0] as + | { nbUsers?: number; nbPageViews?: number } + | undefined; + return { nbRepositories, - nbUsers: (nbUsersAgg[0] as { n?: number } | undefined)?.n || 0, - nbPageViews: - (nbPageViews[0] as { total?: number } | undefined)?.total || 0, + nbUsers: usage?.nbUsers || 0, + nbPageViews: usage?.nbPageViews || 0, nbPullRequests, }; } diff --git a/src/server/routes/admin.ts b/src/server/routes/admin.ts index 15b404a..eede8be 100644 --- a/src/server/routes/admin.ts +++ b/src/server/routes/admin.ts @@ -650,7 +650,6 @@ router.get("/overview", async (req, res) => { const [ statusBreakdown, - totalSize, recentErrors, totalUsers, totalConferences, @@ -665,9 +664,6 @@ router.get("/overview", async (req, res) => { AnonymizedRepositoryModel.aggregate([ { $group: { _id: "$status", count: { $sum: 1 }, storage: { $sum: "$size.storage" } } }, ]), - AnonymizedRepositoryModel.aggregate([ - { $group: { _id: null, total: { $sum: "$size.storage" } } }, - ]), AnonymizedRepositoryModel.countDocuments({ status: "error", statusDate: { $gte: now24h }, @@ -774,7 +770,10 @@ router.get("/overview", async (req, res) => { repos: { total: totalRepos, statusBreakdown, - totalStorage: totalSize[0]?.total || 0, + totalStorage: statusBreakdown.reduce( + (total, row) => total + (row.storage || 0), + 0 + ), recentErrors24h: recentErrors, activeRepos24h, newRepos24h, @@ -802,14 +801,11 @@ router.get("/overview", async (req, res) => { // Global stats endpoint: counts by status, total disk, recent failures router.get("/stats", async (req, res) => { try { - const [statusBreakdown, totalSize, recentErrors, totalUsers, totalConferences] = + const [statusBreakdown, recentErrors, totalUsers, totalConferences] = await Promise.all([ AnonymizedRepositoryModel.aggregate([ { $group: { _id: "$status", count: { $sum: 1 }, storage: { $sum: "$size.storage" } } }, ]), - AnonymizedRepositoryModel.aggregate([ - { $group: { _id: null, total: { $sum: "$size.storage" } } }, - ]), AnonymizedRepositoryModel.countDocuments({ status: "error", statusDate: { $gte: new Date(Date.now() - 1000 * 60 * 60 * 24) }, @@ -819,7 +815,10 @@ router.get("/stats", async (req, res) => { ]); res.json({ statusBreakdown, - totalStorage: totalSize[0]?.total || 0, + totalStorage: statusBreakdown.reduce( + (total, row) => total + (row.storage || 0), + 0 + ), recentErrors24h: recentErrors, totalUsers, totalConferences, @@ -921,7 +920,7 @@ router.get("/repos", async (req, res) => { ); } - const [total, results, statusCounts, sizeAgg] = await Promise.all([ + const [total, results, statusCounts] = await Promise.all([ AnonymizedRepositoryModel.find(filter).countDocuments(), AnonymizedRepositoryModel.find(filter) .skip(skipIndex) @@ -932,10 +931,6 @@ router.get("/repos", async (req, res) => { { $match: filter }, { $group: { _id: "$status", count: { $sum: 1 }, storage: { $sum: "$size.storage" } } }, ]), - AnonymizedRepositoryModel.aggregate([ - { $match: filter }, - { $group: { _id: null, total: { $sum: "$size.storage" } } }, - ]), ]); res.json({ query: filter, @@ -944,7 +939,10 @@ router.get("/repos", async (req, res) => { sort, results, statusCounts, - totalSize: sizeAgg[0]?.total || 0, + totalSize: statusCounts.reduce( + (total, row) => total + (row.storage || 0), + 0 + ), }); }); diff --git a/src/server/routes/repository-private.ts b/src/server/routes/repository-private.ts index 4878c34..aef0d50 100644 --- a/src/server/routes/repository-private.ts +++ b/src/server/routes/repository-private.ts @@ -129,7 +129,7 @@ router.post("/claim", async (req: express.Request, res: express.Response) => { await AnonymizedRepositoryModel.updateOne( { repoId: repoConfig.repoId }, { $set: { owner: user.model.id } } - ); + ).collation({ locale: "en", strength: 2 }); return res.send("Ok"); } catch (error) { handleError(error, res, req); diff --git a/src/server/routes/repository-public.ts b/src/server/routes/repository-public.ts index 546df02..5092696 100644 --- a/src/server/routes/repository-public.ts +++ b/src/server/routes/repository-public.ts @@ -196,7 +196,7 @@ router.get( const repoId = repo.repoId; const results = await FileModel.aggregate([ { $match: { repoId, size: { $ne: null } } }, - { $project: { path: 1 } }, + { $project: { _id: 0, path: 1 } }, { $group: { _id: "$path", count: { $sum: 1 } } }, ]).exec(); diff --git a/src/server/schedule.ts b/src/server/schedule.ts index 14bd465..c612c14 100644 --- a/src/server/schedule.ts +++ b/src/server/schedule.ts @@ -4,6 +4,7 @@ import AnonymizedRepositoryModel from "../core/model/anonymizedRepositories/anon import ConferenceModel from "../core/model/conference/conferences.model"; import Repository from "../core/Repository"; import { createLogger, serializeError } from "../core/logger"; +import { RepositoryStatus } from "../core/types"; import { computeAndStoreDailyStats } from "./dailyStatsSnapshot"; const logger = createLogger("schedule"); @@ -11,18 +12,18 @@ const logger = createLogger("schedule"); export function conferenceStatusCheck() { // check every 6 hours the status of the conferences schedule.scheduleJob("0 */6 * * *", async () => { - (await ConferenceModel.find({ status: { $eq: "ready" } })).forEach( - async (data) => { - const conference = new Conference(data); - if (conference.isExpired() && conference.status == "ready") { - try { - await conference.expire(); - } catch (error) { - logger.error("conference expire failed", serializeError(error)); - } - } + const cursor = ConferenceModel.find({ + status: "ready", + endDate: { $lte: new Date() }, + }).cursor(); + for await (const data of cursor) { + const conference = new Conference(data); + try { + await conference.expire(); + } catch (error) { + logger.error("conference expire failed", serializeError(error)); } - ); + } }); } @@ -30,29 +31,53 @@ export function repositoryStatusCheck() { // check every 6 hours the status of the repositories schedule.scheduleJob("0 */6 * * *", async () => { logger.info("checking repository status and unused repositories"); - ( - await AnonymizedRepositoryModel.find({ - status: { $eq: "ready" }, - isReseted: { $eq: false }, - }) - ).forEach(async (data) => { - const repo = new Repository(data); - try { - await repo.check(); - } catch { - logger.info("repository expired", { repoId: repo.repoId }); - } - const fourMonthAgo = new Date(); - fourMonthAgo.setMonth(fourMonthAgo.getMonth() - 4); + const now = new Date(); + const fourMonthAgo = new Date(now); + fourMonthAgo.setMonth(fourMonthAgo.getMonth() - 4); + const cursor = AnonymizedRepositoryModel.find({ + status: RepositoryStatus.READY, + isReseted: false, + $or: [ + { + "options.expirationMode": { $in: ["redirect", "remove"] }, + "options.expirationDate": { $lte: now }, + }, + { lastView: { $lt: fourMonthAgo } }, + ], + }).cursor(); + const batch: Promise[] = []; + for await (const data of cursor) { + batch.push( + (async () => { + const repo = new Repository(data); + try { + await repo.check(); + } catch { + logger.info("repository expired", { repoId: repo.repoId }); + } - if (repo.model.lastView < fourMonthAgo) { - repo.removeCache().then(() => { - logger.info("removed cache for unused repository", { - repoId: repo.repoId, - }); - }); + if (repo.model.lastView < fourMonthAgo) { + try { + await repo.removeCache(); + } catch (error) { + logger.error("repository cache removal failed", { + ...serializeError(error), + repoId: repo.repoId, + }); + return; + } + logger.info("removed cache for unused repository", { + repoId: repo.repoId, + }); + } + })() + ); + if (batch.length >= 10) { + await Promise.all(batch); + batch.length = 0; } - }); + } + await Promise.all(batch); }); }