mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-20 11:22:35 +01:00
fix: isolate website limits and scope explorer counts
This commit is contained in:
1 parent
77e22a87e8
commit
d4b9fd85e7
13 files changed
+360
-32
No files matched your search
@@ -98,6 +98,8 @@ interface Config {
|
||||
*/
|
||||
TRUST_PROXY: string;
|
||||
RATE_LIMIT: number;
|
||||
/** Website requests per visitor per 15 minutes, independent of the API. */
|
||||
WEBSITE_RATE_LIMIT: number;
|
||||
}
|
||||
const config: Config = {
|
||||
BILLING_ENABLED: false,
|
||||
@@ -171,6 +173,7 @@ const config: Config = {
|
||||
// entries. Set a plain number to restore the legacy hop-count behavior.
|
||||
TRUST_PROXY: "loopback,uniquelocal,cloudflare",
|
||||
RATE_LIMIT: 350,
|
||||
WEBSITE_RATE_LIMIT: 2000,
|
||||
APP_HOSTNAME: "anonymous.4open.science",
|
||||
DB_USERNAME: "admin",
|
||||
DB_PASSWORD: "password",
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import Repository from "./Repository";
|
||||
import AnonymizedFile from "./AnonymizedFile";
|
||||
import AnonymousError from "./AnonymousError";
|
||||
import FileModel from "./model/files/files.model";
|
||||
import { FILE_LISTING_LIMIT } from "./file-tree";
|
||||
import { anonymizePathCompiled, compileTerms } from "./anonymize-utils";
|
||||
|
||||
/** Only return the current folder and its immediate children, never a whole tree. */
|
||||
export async function folderCounts(repo: Repository, requestedPath: unknown): Promise<Record<string, number>> {
|
||||
if (typeof requestedPath !== "string" || requestedPath.length > 4096 ||
|
||||
requestedPath.split("/").some(part => part === "." || part === "..") ||
|
||||
(requestedPath !== "" && (requestedPath.startsWith("/") || requestedPath.endsWith("/")))) {
|
||||
throw new AnonymousError("invalid_path", { httpStatus: 400 });
|
||||
}
|
||||
let originalPath = "";
|
||||
if (requestedPath) {
|
||||
// Use the same collision resolution as browsing that anonymized directory.
|
||||
const directory = await new AnonymizedFile({ repository: repo, anonymizedPath: requestedPath }).getFileInfo();
|
||||
if (directory.size != null) throw new AnonymousError("folder_not_supported", { httpStatus: 404 });
|
||||
originalPath = [directory.path, directory.name].filter(Boolean).join("/");
|
||||
}
|
||||
const prefix = originalPath ? originalPath + "/" : "";
|
||||
const escapedPrefix = prefix.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const prefixLength = Array.from(prefix).length;
|
||||
const rows = await FileModel.aggregate<{ _id: string; count: number }>([
|
||||
{ $match: {
|
||||
repoId: repo.repoId,
|
||||
treeGeneration: repo.model.treeGeneration || { $exists: false },
|
||||
size: { $ne: null },
|
||||
...(originalPath ? { $or: [{ path: originalPath }, { path: { $regex: "^" + escapedPrefix } }] } : {}),
|
||||
} },
|
||||
{ $project: { child: { $cond: [
|
||||
{ $eq: ["$path", originalPath] }, "",
|
||||
{ $arrayElemAt: [{ $split: [{ $substrCP: ["$path", prefixLength,
|
||||
{ $subtract: [{ $strLenCP: "$path" }, prefixLength] }] }, "/"] }, 0] },
|
||||
] } } },
|
||||
{ $group: { _id: "$child", count: { $sum: 1 } } },
|
||||
{ $limit: FILE_LISTING_LIMIT + 1 },
|
||||
]).option({ maxTimeMS: 2000, allowDiskUse: true }).exec();
|
||||
if (rows.length > FILE_LISTING_LIMIT) throw new AnonymousError("file_listing_too_large", { httpStatus: 413 });
|
||||
const counts: Record<string, number> = Object.create(null);
|
||||
const compiled = compileTerms(repo.options.terms || []);
|
||||
let total = 0;
|
||||
for (const row of rows) {
|
||||
total += row.count;
|
||||
if (!row._id) continue;
|
||||
const anonymousPath = anonymizePathCompiled(prefix + row._id, compiled);
|
||||
counts[anonymousPath] = (counts[anonymousPath] || 0) + row.count;
|
||||
}
|
||||
counts[requestedPath] = total;
|
||||
return counts;
|
||||
}
|
||||
+6
-1
@@ -43,6 +43,7 @@ import { getUser } from "./routes/route-utils";
|
||||
import config from "../config";
|
||||
import { resolveTrustProxy } from "./trustProxy";
|
||||
import { requestRateLimitKey } from "./rate-limit-key";
|
||||
import { createWebsiteRateLimiter } from "./website-rate-limit";
|
||||
import { createLogger, serializeError } from "../core/logger";
|
||||
import { serverOptions } from "./server-options";
|
||||
import { monitorRequests } from "../core/request-monitoring";
|
||||
@@ -238,6 +239,10 @@ export default async function start() {
|
||||
maxDelayMs: 5000,
|
||||
keyGenerator: apiRateKey,
|
||||
});
|
||||
const websiteRate = createWebsiteRateLimiter(new RedisStore({
|
||||
prefix: "website-rate:",
|
||||
sendCommand: (...args: string[]) => redisClient.sendCommand(args),
|
||||
}));
|
||||
|
||||
app.use("/github", rate, speedLimiter, githubAppRouter);
|
||||
app.use("/github", rate, speedLimiter, connectionRouter);
|
||||
@@ -288,7 +293,7 @@ export default async function start() {
|
||||
});
|
||||
|
||||
// web view
|
||||
app.use("/w/", rate, webViewSpeedLimiter, router.webview);
|
||||
app.use("/w/", websiteRate, webViewSpeedLimiter, router.webview);
|
||||
|
||||
// Hashed assets (e.g. core.a1b2c3d4e5.min.js) — immutable, cache for 1 year.
|
||||
// Strip the hash from the filename and serve the underlying file.
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { FILE_LISTING_LIMIT } from "../../core/file-tree";
|
||||
import { folderCounts as scopedFolderCounts } from "../../core/folder-counts";
|
||||
import { repositoryFailureStatus } from "../../core/repository-failure-status";
|
||||
import { anonymizePath } from "../../core/anonymize-utils";
|
||||
import * as express from "express";
|
||||
@@ -214,6 +215,8 @@ router.get(
|
||||
const repo = await getRepo(req, res);
|
||||
if (!repo) return;
|
||||
try {
|
||||
if (req.query?.path !== undefined) return res.json(await scopedFolderCounts(repo, req.query.path));
|
||||
// Retain the guarded legacy response for clients requesting all counts.
|
||||
const repoId = repo.repoId;
|
||||
const results = await FileModel.aggregate([
|
||||
{ $match: { repoId, treeGeneration: repo.model.treeGeneration || { $exists: false }, size: { $ne: null } } },
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import rateLimit, { Store } from "express-rate-limit";
|
||||
import config from "../config";
|
||||
import { requestRateLimitKey } from "./rate-limit-key";
|
||||
import { sendRequestRateLimit } from "./request-rate-limit";
|
||||
|
||||
/** A page's images and scripts must not consume its visitor's API budget. */
|
||||
export function createWebsiteRateLimiter(store?: Store) {
|
||||
if (!Number.isSafeInteger(config.WEBSITE_RATE_LIMIT) || config.WEBSITE_RATE_LIMIT <= 0) {
|
||||
throw new Error("WEBSITE_RATE_LIMIT must be a positive integer");
|
||||
}
|
||||
return rateLimit({
|
||||
windowMs: 15 * 60_000,
|
||||
limit: config.WEBSITE_RATE_LIMIT,
|
||||
keyGenerator: requestRateLimitKey,
|
||||
standardHeaders: "draft-6",
|
||||
legacyHeaders: false,
|
||||
handler: sendRequestRateLimit,
|
||||
...(store ? { store } : {}),
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user