fix: isolate website limits and scope explorer counts

This commit is contained in:
Thomas Durieux committed 2026-11-03 15:03:23 +00:00
1 parent 77e22a87e8
commit d4b9fd85e7
13 files changed
+360 -32

No files matched your search

+3
View File
@@ -98,6 +98,8 @@ interface Config {
*/
TRUST_PROXY: string;
RATE_LIMIT: number;
/** Website requests per visitor per 15 minutes, independent of the API. */
WEBSITE_RATE_LIMIT: number;
}
const config: Config = {
BILLING_ENABLED: false,
@@ -171,6 +173,7 @@ const config: Config = {
// entries. Set a plain number to restore the legacy hop-count behavior.
TRUST_PROXY: "loopback,uniquelocal,cloudflare",
RATE_LIMIT: 350,
WEBSITE_RATE_LIMIT: 2000,
APP_HOSTNAME: "anonymous.4open.science",
DB_USERNAME: "admin",
DB_PASSWORD: "password",
+52
View File
@@ -0,0 +1,52 @@
import Repository from "./Repository";
import AnonymizedFile from "./AnonymizedFile";
import AnonymousError from "./AnonymousError";
import FileModel from "./model/files/files.model";
import { FILE_LISTING_LIMIT } from "./file-tree";
import { anonymizePathCompiled, compileTerms } from "./anonymize-utils";
/** Only return the current folder and its immediate children, never a whole tree. */
export async function folderCounts(repo: Repository, requestedPath: unknown): Promise<Record<string, number>> {
if (typeof requestedPath !== "string" || requestedPath.length > 4096 ||
requestedPath.split("/").some(part => part === "." || part === "..") ||
(requestedPath !== "" && (requestedPath.startsWith("/") || requestedPath.endsWith("/")))) {
throw new AnonymousError("invalid_path", { httpStatus: 400 });
}
let originalPath = "";
if (requestedPath) {
// Use the same collision resolution as browsing that anonymized directory.
const directory = await new AnonymizedFile({ repository: repo, anonymizedPath: requestedPath }).getFileInfo();
if (directory.size != null) throw new AnonymousError("folder_not_supported", { httpStatus: 404 });
originalPath = [directory.path, directory.name].filter(Boolean).join("/");
}
const prefix = originalPath ? originalPath + "/" : "";
const escapedPrefix = prefix.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const prefixLength = Array.from(prefix).length;
const rows = await FileModel.aggregate<{ _id: string; count: number }>([
{ $match: {
repoId: repo.repoId,
treeGeneration: repo.model.treeGeneration || { $exists: false },
size: { $ne: null },
...(originalPath ? { $or: [{ path: originalPath }, { path: { $regex: "^" + escapedPrefix } }] } : {}),
} },
{ $project: { child: { $cond: [
{ $eq: ["$path", originalPath] }, "",
{ $arrayElemAt: [{ $split: [{ $substrCP: ["$path", prefixLength,
{ $subtract: [{ $strLenCP: "$path" }, prefixLength] }] }, "/"] }, 0] },
] } } },
{ $group: { _id: "$child", count: { $sum: 1 } } },
{ $limit: FILE_LISTING_LIMIT + 1 },
]).option({ maxTimeMS: 2000, allowDiskUse: true }).exec();
if (rows.length > FILE_LISTING_LIMIT) throw new AnonymousError("file_listing_too_large", { httpStatus: 413 });
const counts: Record<string, number> = Object.create(null);
const compiled = compileTerms(repo.options.terms || []);
let total = 0;
for (const row of rows) {
total += row.count;
if (!row._id) continue;
const anonymousPath = anonymizePathCompiled(prefix + row._id, compiled);
counts[anonymousPath] = (counts[anonymousPath] || 0) + row.count;
}
counts[requestedPath] = total;
return counts;
}
+6 -1
View File
@@ -43,6 +43,7 @@ import { getUser } from "./routes/route-utils";
import config from "../config";
import { resolveTrustProxy } from "./trustProxy";
import { requestRateLimitKey } from "./rate-limit-key";
import { createWebsiteRateLimiter } from "./website-rate-limit";
import { createLogger, serializeError } from "../core/logger";
import { serverOptions } from "./server-options";
import { monitorRequests } from "../core/request-monitoring";
@@ -238,6 +239,10 @@ export default async function start() {
maxDelayMs: 5000,
keyGenerator: apiRateKey,
});
const websiteRate = createWebsiteRateLimiter(new RedisStore({
prefix: "website-rate:",
sendCommand: (...args: string[]) => redisClient.sendCommand(args),
}));
app.use("/github", rate, speedLimiter, githubAppRouter);
app.use("/github", rate, speedLimiter, connectionRouter);
@@ -288,7 +293,7 @@ export default async function start() {
});
// web view
app.use("/w/", rate, webViewSpeedLimiter, router.webview);
app.use("/w/", websiteRate, webViewSpeedLimiter, router.webview);
// Hashed assets (e.g. core.a1b2c3d4e5.min.js) — immutable, cache for 1 year.
// Strip the hash from the filename and serve the underlying file.
+3
View File
@@ -1,4 +1,5 @@
import { FILE_LISTING_LIMIT } from "../../core/file-tree";
import { folderCounts as scopedFolderCounts } from "../../core/folder-counts";
import { repositoryFailureStatus } from "../../core/repository-failure-status";
import { anonymizePath } from "../../core/anonymize-utils";
import * as express from "express";
@@ -214,6 +215,8 @@ router.get(
const repo = await getRepo(req, res);
if (!repo) return;
try {
if (req.query?.path !== undefined) return res.json(await scopedFolderCounts(repo, req.query.path));
// Retain the guarded legacy response for clients requesting all counts.
const repoId = repo.repoId;
const results = await FileModel.aggregate([
{ $match: { repoId, treeGeneration: repo.model.treeGeneration || { $exists: false }, size: { $ne: null } } },
+20
View File
@@ -0,0 +1,20 @@
import rateLimit, { Store } from "express-rate-limit";
import config from "../config";
import { requestRateLimitKey } from "./rate-limit-key";
import { sendRequestRateLimit } from "./request-rate-limit";
/** A page's images and scripts must not consume its visitor's API budget. */
export function createWebsiteRateLimiter(store?: Store) {
if (!Number.isSafeInteger(config.WEBSITE_RATE_LIMIT) || config.WEBSITE_RATE_LIMIT <= 0) {
throw new Error("WEBSITE_RATE_LIMIT must be a positive integer");
}
return rateLimit({
windowMs: 15 * 60_000,
limit: config.WEBSITE_RATE_LIMIT,
keyGenerator: requestRateLimitKey,
standardHeaders: "draft-6",
legacyHeaders: false,
handler: sendRequestRateLimit,
...(store ? { store } : {}),
});
}