Merge commit 'refs/security-review/pr846' into HEAD

# Conflicts:
#	public/script/main.js

src/server/index.ts

@@ -12,6 +12,7 @@ import * as express from "express";
 import * as compression from "compression";
 import * as passport from "passport";
 import { connect } from "./database";
+import { startTemporaryStorageMaintenance } from "../core/temporary-storage";
 import { initSession, router as connectionRouter } from "./routes/connection";
 import { bearerTokenAuth } from "./routes/token-auth";
 import router from "./routes";
@@ -27,16 +28,18 @@ import {
   recoverStuckRemoving,
 } from "../queue";
 import {
-  computeStats,
+  getCurrentStats,
+  getStatsHistory,
   ensureTodaySnapshot,
-  HomeStatsHistoryRow,
-  mergeCurrentStatsIntoHistory,
 } from "./dailyStatsSnapshot";
-import DailyStatsModel from "../core/model/dailyStats/dailyStats.model";
 import { getUser } from "./routes/route-utils";
 import config from "../config";
-import { resolveTrustProxy, isCloudflareIP } from "./trustProxy";
+import { resolveTrustProxy } from "./trustProxy";
+import { requestRateLimitKey } from "./rate-limit-key";
 import { createLogger, serializeError } from "../core/logger";
+import { monitorRequests } from "../core/request-monitoring";
+import { startPerformanceMonitoring } from "../core/performance-monitoring";
+import { getAnonymizationPoolStats } from "../core/anonymization-pool";
 
 import { createReviewCapabilities } from "./service/review-capabilities";
 
@@ -106,6 +109,8 @@ function indexResponse(req: express.Request, res: express.Response) {
 
 export default async function start() {
   const app = express();
+  app.use(monitorRequests("api"));
+  startPerformanceMonitoring("api", getAnonymizationPoolStats);
   app.set("query parser", "extended");
   app.use("/service", createReviewCapabilities(process.env.REVIEW_SERVICE_KEYS));
   app.use("/github/app/webhook", githubAppWebhook);
@@ -159,37 +164,6 @@ export default async function start() {
 
   await redisClient.connect();
 
-  function keyGenerator(
-    request: express.Request,
-    _response: express.Response
-  ): string {
-    // Use request.ip, which Express resolves from X-Forwarded-For honouring
-    // the configured "trust proxy" setting. Do NOT key off the
-    // cf-connecting-ip header unconditionally: when the server isn't actually
-    // behind Cloudflare a client can set that header to an arbitrary value
-    // per request and trivially bypass the rate limiter (CWE-290).
-    let ip = request.ip;
-    if (!ip && request.socket.remoteAddress) {
-      logger.warn("request.ip is missing");
-      ip = request.socket.remoteAddress;
-    }
-    // If resolution stopped at a Cloudflare edge address (X-Forwarded-For no
-    // longer contains the visitor, e.g. after a Cloudflare-side change), fall
-    // back to cf-connecting-ip. This is safe: request.ip can only be a
-    // Cloudflare address when every hop up to it is trusted, i.e. the request
-    // genuinely traversed Cloudflare. A direct client forging the header is
-    // keyed by its own untrusted address instead, so the CWE-290 bypass above
-    // does not apply.
-    if (ip && isCloudflareIP(ip)) {
-      const cfConnectingIP = request.headers["cf-connecting-ip"];
-      if (typeof cfConnectingIP === "string" && cfConnectingIP) {
-        ip = cfConnectingIP.trim();
-      }
-    }
-    // remove port number from IPv4 addresses
-    return (ip || "").replace(/:\d+[^:]*$/, "");
-  }
-
   const rate = rateLimit({
     store: new RedisStore({
       sendCommand: (...args: string[]) => redisClient.sendCommand(args),
@@ -204,7 +178,7 @@ export default async function start() {
       }
       return false;
     },
-    max: async (request: express.Request, _response: express.Response) => {
+    limit: async (request: express.Request, _response: express.Response) => {
       try {
         const user = await getUser(request);
         if (user) return config.RATE_LIMIT;
@@ -214,8 +188,8 @@ export default async function start() {
       // if not logged in, limit to half the rate
       return config.RATE_LIMIT / 2;
     },
-    keyGenerator,
-    standardHeaders: true,
+    keyGenerator: requestRateLimitKey,
+    standardHeaders: "draft-6",
     legacyHeaders: false,
     message: (_request: express.Request, _response: express.Response) => {
       return `You can only make ${config.RATE_LIMIT} requests every 15min. Please try again later.`;
@@ -226,28 +200,14 @@ export default async function start() {
     delayAfter: 50,
     delayMs: () => 150,
     maxDelayMs: 5000,
-    keyGenerator,
+    keyGenerator: requestRateLimitKey,
   });
   const webViewSpeedLimiter = slowDown({
     windowMs: 15 * 60 * 1000, // 15 minutes
     delayAfter: 200,
     delayMs: () => 150,
     maxDelayMs: 5000,
-    keyGenerator,
-  });
-
-  app.use(function (req, res, next) {
-    const start = Date.now();
-    res.on("finish", function () {
-      const time = Date.now() - start;
-      logger.info("request", {
-        method: req.method,
-        status: res.statusCode,
-        url: join(req.baseUrl || "", req.url || ""),
-        ms: time,
-      });
-    });
-    next();
+    keyGenerator: requestRateLimitKey,
   });
 
   app.use("/github", rate, speedLimiter, githubAppRouter);
@@ -285,53 +245,14 @@ export default async function start() {
     res.sendStatus(404);
   });
 
-  let stat: Record<string, unknown> = {};
-  let history: HomeStatsHistoryRow[] | null = null;
-  let historyKey: number | null = null;
-
-  setInterval(() => {
-    stat = {};
-    history = null;
-    historyKey = null;
-  }, 1000 * 60 * 60);
-
   apiRouter.get("/healthcheck", async (_, res) => {
     res.json({ status: "ok" });
   });
   apiRouter.get("/stat", async (_, res) => {
-    if (stat.nbRepositories) {
-      res.json(stat);
-      return;
-    }
-    stat = { ...(await computeStats()) };
-    res.json(stat);
+    res.json(await getCurrentStats());
   });
-
   apiRouter.get("/stat/history", async (req, res) => {
-    const days = Math.min(
-      Math.max(parseInt(req.query.days as string) || 30, 1),
-      365
-    );
-    if (history && historyKey === days) {
-      res.json(history);
-      return;
-    }
-    const since = new Date();
-    since.setUTCDate(since.getUTCDate() - days + 1);
-    since.setUTCHours(0, 0, 0, 0);
-    const docs = await DailyStatsModel.find({ date: { $gte: since } })
-      .sort({ date: 1 })
-      .lean();
-    const rows = docs.map((d) => ({
-      date: d.date,
-      nbRepositories: d.nbRepositories,
-      nbUsers: d.nbUsers,
-      nbPageViews: d.nbPageViews,
-      nbPullRequests: d.nbPullRequests,
-    }));
-    history = mergeCurrentStatsIntoHistory(rows, await computeStats());
-    historyKey = days;
-    res.json(history);
+    res.json(await getStatsHistory(parseInt(String(req.query.days)) || 30));
   });
 
   // web view
@@ -386,6 +307,7 @@ export default async function start() {
   dailyStatsSnapshot();
 
   await connect();
+  await startTemporaryStorageMaintenance();
   app.listen(config.PORT);
   logger.info("server started", { port: config.PORT });
   ensureTodaySnapshot().catch((err) =>