perf: cut MongoDB scans in repository workflows (#780)

* Add repository name index

* Optimize repository file counts

* Optimize MongoDB query workloads
This commit is contained in:
Thomas Durieux
2026-08-20 12:29:26 +02:00
committed by GitHub
parent a30457993d
commit 7aa9557905
12 changed files with 124 additions and 61 deletions
+14 -16
View File
@@ -650,7 +650,6 @@ router.get("/overview", async (req, res) => {
const [
statusBreakdown,
totalSize,
recentErrors,
totalUsers,
totalConferences,
@@ -665,9 +664,6 @@ router.get("/overview", async (req, res) => {
AnonymizedRepositoryModel.aggregate([
{ $group: { _id: "$status", count: { $sum: 1 }, storage: { $sum: "$size.storage" } } },
]),
AnonymizedRepositoryModel.aggregate([
{ $group: { _id: null, total: { $sum: "$size.storage" } } },
]),
AnonymizedRepositoryModel.countDocuments({
status: "error",
statusDate: { $gte: now24h },
@@ -774,7 +770,10 @@ router.get("/overview", async (req, res) => {
repos: {
total: totalRepos,
statusBreakdown,
totalStorage: totalSize[0]?.total || 0,
totalStorage: statusBreakdown.reduce(
(total, row) => total + (row.storage || 0),
0
),
recentErrors24h: recentErrors,
activeRepos24h,
newRepos24h,
@@ -802,14 +801,11 @@ router.get("/overview", async (req, res) => {
// Global stats endpoint: counts by status, total disk, recent failures
router.get("/stats", async (req, res) => {
try {
const [statusBreakdown, totalSize, recentErrors, totalUsers, totalConferences] =
const [statusBreakdown, recentErrors, totalUsers, totalConferences] =
await Promise.all([
AnonymizedRepositoryModel.aggregate([
{ $group: { _id: "$status", count: { $sum: 1 }, storage: { $sum: "$size.storage" } } },
]),
AnonymizedRepositoryModel.aggregate([
{ $group: { _id: null, total: { $sum: "$size.storage" } } },
]),
AnonymizedRepositoryModel.countDocuments({
status: "error",
statusDate: { $gte: new Date(Date.now() - 1000 * 60 * 60 * 24) },
@@ -819,7 +815,10 @@ router.get("/stats", async (req, res) => {
]);
res.json({
statusBreakdown,
totalStorage: totalSize[0]?.total || 0,
totalStorage: statusBreakdown.reduce(
(total, row) => total + (row.storage || 0),
0
),
recentErrors24h: recentErrors,
totalUsers,
totalConferences,
@@ -921,7 +920,7 @@ router.get("/repos", async (req, res) => {
);
}
const [total, results, statusCounts, sizeAgg] = await Promise.all([
const [total, results, statusCounts] = await Promise.all([
AnonymizedRepositoryModel.find(filter).countDocuments(),
AnonymizedRepositoryModel.find(filter)
.skip(skipIndex)
@@ -932,10 +931,6 @@ router.get("/repos", async (req, res) => {
{ $match: filter },
{ $group: { _id: "$status", count: { $sum: 1 }, storage: { $sum: "$size.storage" } } },
]),
AnonymizedRepositoryModel.aggregate([
{ $match: filter },
{ $group: { _id: null, total: { $sum: "$size.storage" } } },
]),
]);
res.json({
query: filter,
@@ -944,7 +939,10 @@ router.get("/repos", async (req, res) => {
sort,
results,
statusCounts,
totalSize: sizeAgg[0]?.total || 0,
totalSize: statusCounts.reduce(
(total, row) => total + (row.storage || 0),
0
),
});
});
+1 -1
View File
@@ -129,7 +129,7 @@ router.post("/claim", async (req: express.Request, res: express.Response) => {
await AnonymizedRepositoryModel.updateOne(
{ repoId: repoConfig.repoId },
{ $set: { owner: user.model.id } }
);
).collation({ locale: "en", strength: 2 });
return res.send("Ok");
} catch (error) {
handleError(error, res, req);
+1 -1
View File
@@ -196,7 +196,7 @@ router.get(
const repoId = repo.repoId;
const results = await FileModel.aggregate([
{ $match: { repoId, size: { $ne: null } } },
{ $project: { path: 1 } },
{ $project: { _id: 0, path: 1 } },
{ $group: { _id: "$path", count: { $sum: 1 } } },
]).exec();