CalvinBackup/anonymous_github
1
0
Fork 0
mirror of https://github.com/tdurieux/anonymous_github.git synced 2026-09-02 00:50:39 +02:00
Code Issues Packages Projects Releases Wiki Activity

perf: cut MongoDB scans in repository workflows (#780)

Browse Source 
* Add repository name index

* Optimize repository file counts

* Optimize MongoDB query workloads
This commit is contained in:
Thomas Durieux
2026-08-20 12:29:26 +02:00
committed by GitHub
parent a30457993d
commit 7aa9557905
12 changed files with 124 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/model/anonymizedGists/anonymizedGists.schema.ts
+1 -1
View File
@@ -14,7 +14,7 @@ const AnonymizedGistSchema = new Schema({
anonymizeDate: Date,
lastView: Date,
pageView: Number,
owner: Schema.Types.ObjectId,
owner: { type: Schema.Types.ObjectId, index: true },
conference: String,
source: {
gistId: String,
src/core/model/anonymizedPullRequests/anonymizedPullRequests.schema.ts
+1 -1
View File
@@ -14,7 +14,7 @@ const AnonymizedPullRequestSchema = new Schema({
anonymizeDate: Date,
lastView: Date,
pageView: Number,
owner: Schema.Types.ObjectId,
owner: { type: Schema.Types.ObjectId, index: true },
conference: String,
source: {
pullRequestId: Number,
src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts
+11
View File
@@ -77,4 +77,15 @@ const AnonymizedRepositorySchema = new Schema({
},
});
AnonymizedRepositorySchema.index({ "source.repositoryName": 1 });
AnonymizedRepositorySchema.index({ status: 1, statusDate: 1 });
AnonymizedRepositorySchema.index({ lastView: 1 });
AnonymizedRepositorySchema.index({ anonymizeDate: 1 });
AnonymizedRepositorySchema.index({ status: 1, isReseted: 1, lastView: 1 });
AnonymizedRepositorySchema.index({
status: 1,
isReseted: 1,
"options.expirationDate": 1,
});
export default AnonymizedRepositorySchema;
src/core/model/conference/conferences.schema.ts
+3
View File
@@ -55,4 +55,7 @@ const RepositorySchema = new Schema({
},
});
RepositorySchema.index({ owners: 1 });
RepositorySchema.index({ status: 1, endDate: 1 });
export default RepositorySchema;
src/core/model/files/files.schema.ts
+8
View File
@@ -13,6 +13,14 @@ const FileSchema = new Schema({
});
FileSchema.index({ path: 1, repoId: 1 });
FileSchema.index(
{ repoId: 1, size: 1, path: 1 },
{ name: "repoId_1_size_1_path_1" }
);
FileSchema.index(
{ repoId: 1, path: 1, name: 1 },
{ name: "repoId_1_path_1_name_1" }
);
FileSchema.methods.toString = function () {
return `${this.path}/${this.name}`;
src/core/model/users/users.schema.ts
+2
View File
@@ -58,4 +58,6 @@ const UserSchema = new Schema({
},
});
UserSchema.index({ dateOfEntry: 1 });
export default UserSchema;
src/queue/index.ts
+3 -1
View File
@@ -33,7 +33,9 @@ async function markErrorIfInFlight(repoId: string, message: string) {
statusMessage: message || "preparation_failed",
},
}
).exec();
)
.collation({ locale: "en", strength: 2 })
.exec();
} catch (e) {
logger.error("markErrorIfInFlight failed", {
...serializeError(e),
src/server/dailyStatsSnapshot.ts
+22 -8
View File
@@ -17,23 +17,37 @@ export interface HomeStatsHistoryRow extends HomeStats {
}
export async function computeStats(): Promise<HomeStats> {
const [nbRepositories, nbUsersAgg, nbPageViews, nbPullRequests] =
const [nbRepositories, usageTotals, nbPullRequests] =
await Promise.all([
AnonymizedRepositoryModel.estimatedDocumentCount(),
AnonymizedRepositoryModel.collection
.aggregate([{ $group: { _id: "$owner" } }, { $count: "n" }])
.toArray(),
AnonymizedRepositoryModel.collection
.aggregate([{ $group: { _id: null, total: { $sum: "$pageView" } } }])
.aggregate([
{
$group: {
_id: "$owner",
pageViews: { $sum: "$pageView" },
},
},
{
$group: {
_id: null,
nbUsers: { $sum: 1 },
nbPageViews: { $sum: "$pageViews" },
},
},
])
.toArray(),
AnonymizedPullRequestModel.estimatedDocumentCount(),
]);
const usage = usageTotals[0] as
| { nbUsers?: number; nbPageViews?: number }
| undefined;
return {
nbRepositories,
nbUsers: (nbUsersAgg[0] as { n?: number } | undefined)?.n || 0,
nbPageViews:
(nbPageViews[0] as { total?: number } | undefined)?.total || 0,
nbUsers: usage?.nbUsers || 0,
nbPageViews: usage?.nbPageViews || 0,
nbPullRequests,
};
}
src/server/routes/admin.ts
+14 -16
View File
@@ -650,7 +650,6 @@ router.get("/overview", async (req, res) => {
const [
statusBreakdown,
totalSize,
recentErrors,
totalUsers,
totalConferences,
@@ -665,9 +664,6 @@ router.get("/overview", async (req, res) => {
AnonymizedRepositoryModel.aggregate([
{ $group: { _id: "$status", count: { $sum: 1 }, storage: { $sum: "$size.storage" } } },
]),
AnonymizedRepositoryModel.aggregate([
{ $group: { _id: null, total: { $sum: "$size.storage" } } },
]),
AnonymizedRepositoryModel.countDocuments({
status: "error",
statusDate: { $gte: now24h },
@@ -774,7 +770,10 @@ router.get("/overview", async (req, res) => {
repos: {
total: totalRepos,
statusBreakdown,
totalStorage: totalSize[0]?.total || 0,
totalStorage: statusBreakdown.reduce(
(total, row) => total + (row.storage || 0),
0
),
recentErrors24h: recentErrors,
activeRepos24h,
newRepos24h,
@@ -802,14 +801,11 @@ router.get("/overview", async (req, res) => {
// Global stats endpoint: counts by status, total disk, recent failures
router.get("/stats", async (req, res) => {
try {
const [statusBreakdown, totalSize, recentErrors, totalUsers, totalConferences] =
const [statusBreakdown, recentErrors, totalUsers, totalConferences] =
await Promise.all([
AnonymizedRepositoryModel.aggregate([
{ $group: { _id: "$status", count: { $sum: 1 }, storage: { $sum: "$size.storage" } } },
]),
AnonymizedRepositoryModel.aggregate([
{ $group: { _id: null, total: { $sum: "$size.storage" } } },
]),
AnonymizedRepositoryModel.countDocuments({
status: "error",
statusDate: { $gte: new Date(Date.now() - 1000 * 60 * 60 * 24) },
@@ -819,7 +815,10 @@ router.get("/stats", async (req, res) => {
]);
res.json({
statusBreakdown,
totalStorage: totalSize[0]?.total || 0,
totalStorage: statusBreakdown.reduce(
(total, row) => total + (row.storage || 0),
0
),
recentErrors24h: recentErrors,
totalUsers,
totalConferences,
@@ -921,7 +920,7 @@ router.get("/repos", async (req, res) => {
);
}
const [total, results, statusCounts, sizeAgg] = await Promise.all([
const [total, results, statusCounts] = await Promise.all([
AnonymizedRepositoryModel.find(filter).countDocuments(),
AnonymizedRepositoryModel.find(filter)
.skip(skipIndex)
@@ -932,10 +931,6 @@ router.get("/repos", async (req, res) => {
{ $match: filter },
{ $group: { _id: "$status", count: { $sum: 1 }, storage: { $sum: "$size.storage" } } },
]),
AnonymizedRepositoryModel.aggregate([
{ $match: filter },
{ $group: { _id: null, total: { $sum: "$size.storage" } } },
]),
]);
res.json({
query: filter,
@@ -944,7 +939,10 @@ router.get("/repos", async (req, res) => {
sort,
results,
statusCounts,
totalSize: sizeAgg[0]?.total || 0,
totalSize: statusCounts.reduce(
(total, row) => total + (row.storage || 0),
0
),
});
});
src/server/routes/repository-private.ts
+1 -1
View File
@@ -129,7 +129,7 @@ router.post("/claim", async (req: express.Request, res: express.Response) => {
await AnonymizedRepositoryModel.updateOne(
{ repoId: repoConfig.repoId },
{ $set: { owner: user.model.id } }
);
).collation({ locale: "en", strength: 2 });
return res.send("Ok");
} catch (error) {
handleError(error, res, req);
src/server/routes/repository-public.ts
+1 -1
View File
@@ -196,7 +196,7 @@ router.get(
const repoId = repo.repoId;
const results = await FileModel.aggregate([
{ $match: { repoId, size: { $ne: null } } },
{ $project: { path: 1 } },
{ $project: { _id: 0, path: 1 } },
{ $group: { _id: "$path", count: { $sum: 1 } } },
]).exec();
src/server/schedule.ts
+57 -32
View File
@@ -4,6 +4,7 @@ import AnonymizedRepositoryModel from "../core/model/anonymizedRepositories/anon
import ConferenceModel from "../core/model/conference/conferences.model";
import Repository from "../core/Repository";
import { createLogger, serializeError } from "../core/logger";
import { RepositoryStatus } from "../core/types";
import { computeAndStoreDailyStats } from "./dailyStatsSnapshot";
const logger = createLogger("schedule");
@@ -11,18 +12,18 @@ const logger = createLogger("schedule");
export function conferenceStatusCheck() {
// check every 6 hours the status of the conferences
schedule.scheduleJob("0 */6 * * *", async () => {
(await ConferenceModel.find({ status: { $eq: "ready" } })).forEach(
async (data) => {
const conference = new Conference(data);
if (conference.isExpired() && conference.status == "ready") {
try {
await conference.expire();
} catch (error) {
logger.error("conference expire failed", serializeError(error));
}
}
const cursor = ConferenceModel.find({
status: "ready",
endDate: { $lte: new Date() },
}).cursor();
for await (const data of cursor) {
const conference = new Conference(data);
try {
await conference.expire();
} catch (error) {
logger.error("conference expire failed", serializeError(error));
}
);
}
});
}
@@ -30,29 +31,53 @@ export function repositoryStatusCheck() {
// check every 6 hours the status of the repositories
schedule.scheduleJob("0 */6 * * *", async () => {
logger.info("checking repository status and unused repositories");
(
await AnonymizedRepositoryModel.find({
status: { $eq: "ready" },
isReseted: { $eq: false },
})
).forEach(async (data) => {
const repo = new Repository(data);
try {
await repo.check();
} catch {
logger.info("repository expired", { repoId: repo.repoId });
}
const fourMonthAgo = new Date();
fourMonthAgo.setMonth(fourMonthAgo.getMonth() - 4);
const now = new Date();
const fourMonthAgo = new Date(now);
fourMonthAgo.setMonth(fourMonthAgo.getMonth() - 4);
const cursor = AnonymizedRepositoryModel.find({
status: RepositoryStatus.READY,
isReseted: false,
$or: [
{
"options.expirationMode": { $in: ["redirect", "remove"] },
"options.expirationDate": { $lte: now },
},
{ lastView: { $lt: fourMonthAgo } },
],
}).cursor();
const batch: Promise<void>[] = [];
for await (const data of cursor) {
batch.push(
(async () => {
const repo = new Repository(data);
try {
await repo.check();
} catch {
logger.info("repository expired", { repoId: repo.repoId });
}
if (repo.model.lastView < fourMonthAgo) {
repo.removeCache().then(() => {
logger.info("removed cache for unused repository", {
repoId: repo.repoId,
});
});
if (repo.model.lastView < fourMonthAgo) {
try {
await repo.removeCache();
} catch (error) {
logger.error("repository cache removal failed", {
...serializeError(error),
repoId: repo.repoId,
});
return;
}
logger.info("removed cache for unused repository", {
repoId: repo.repoId,
});
}
})()
);
if (batch.length >= 10) {
await Promise.all(batch);
batch.length = 0;
}
});
}
await Promise.all(batch);
});
}