perf: avoid needless downloads and track request latency (#876)

* perf: avoid needless downloads and track request latency

* fix: correct request timing and monitoring revalidation

src/server/routes/admin.ts

@@ -1,6 +1,7 @@
 import { AsyncCache } from "../../core/async-cache";
 import * as os from "os";
-import { execSync } from "child_process";
+import { execFile } from "child_process";
+import { promisify } from "util";
 import { Queue, JobType } from "bullmq";
 import * as express from "express";
 import AnonymousError from "../../core/AnonymousError";
@@ -30,8 +31,11 @@ import {
 } from "../../core/logger";
 import { createClient, RedisClientType } from "redis";
 import config from "../../config";
+import { performanceReport } from "../../core/performance-monitoring";
 
 const logger = createLogger("admin");
+const execFileAsync = promisify(execFile);
+const diskUsageCache = new AsyncCache<string>(30000, 1);
 
 let errorLogClient: RedisClientType | null = null;
 
@@ -78,6 +82,12 @@ router.use(
 
 router.use("/tokens", adminTokensRouter);
 
+router.get("/performance", async (req, res) => {
+  res.set("Cache-Control", "no-store");
+  try { res.json(await performanceReport(Number(req.query.minutes))); }
+  catch { res.json({ available: false, instances: [], routes: [], stages: [] }); }
+});
+
 function dashboardCache(
   _req: express.Request,
   res: express.Response,
@@ -627,7 +637,10 @@ router.get("/overview", async (req, res) => {
     // Disk usage via df (root partition)
     let diskTotal = 0, diskUsed = 0, diskFree = 0, diskPercent = 0, diskMount = "/";
     try {
-      const dfOut = execSync("df -k / 2>/dev/null", { timeout: 3000 }).toString();
+      const dfOut = await diskUsageCache.get("root", async () => {
+        const result = await execFileAsync("df", ["-k", "/"], { timeout: 3000, maxBuffer: 16384 });
+        return result.stdout;
+      });
       const lines = dfOut.trim().split("\n");
       if (lines.length >= 2) {
         const cols = lines[1].split(/\s+/);

src/server/routes/file.ts

@@ -3,6 +3,9 @@ import AnonymizedFile from "../../core/AnonymizedFile";
 import AnonymousError from "../../core/AnonymousError";
 import { getRepo, handleError } from "./route-utils";
 import { fileETag } from "./file-etag";
+import config from "../../config";
+import { lookup } from "mime-types";
+import { isTextFile } from "../../core/anonymize-utils";
 
 export const router = express.Router();
 
@@ -142,9 +145,17 @@ router.get(
       // anonymization term list left old anonymizations cached under the
       // same URL.
       res.header("Cache-Control", "private, no-cache, must-revalidate");
-      if (req.headers["if-none-match"] === etag) {
+      if (req.fresh) {
         return res.status(304).end();
       }
+      if (req.method === "HEAD") {
+        const size = await f.size();
+        if (size != null && size > config.MAX_FILE_SIZE) throw new AnonymousError("file_too_big", { httpStatus: 413 });
+        const mime = lookup(originalPath);
+        if (mime && !originalPath.endsWith(".ts")) res.type(mime);
+        else if (isTextFile(originalPath)) res.type("text/plain");
+        res.end(); return;
+      }
       await f.send(res);
       await repo.countView();
     } catch (error) {

src/server/routes/repository-public.ts

@@ -14,6 +14,7 @@ import FileModel from "../../core/model/files/files.model";
 import { createLogger, serializeError } from "../../core/logger";
 import { githubTokenForStreamer } from "../../core/github-token-context";
 import gh = require("parse-github-url");
+import { requestHeaders, startStage } from "../../core/request-monitoring";
 
 const logger = createLogger("repository-public");
 
@@ -52,16 +53,20 @@ router.get(
         });
       }
 
+      res.attachment(`${repo.repoId}.zip`);
+      if (req.method === "HEAD") { res.end(); return; }
       await repo.countView();
 
       if (config.STREAMER_ENTRYPOINT) {
         // use the streamer service
         const token = await repo.getToken();
         const anonymizer = repo.generateAnonymizeTransformer("");
+        const headersDone = startStage("streamer_headers");
         res.attachment(`${repo.repoId}.zip`);
         const reqStream = got
           .stream(join(config.STREAMER_ENTRYPOINT, "api/download"), {
             method: "POST",
+            headers: requestHeaders(),
             json: {
               token: await githubTokenForStreamer(token, repo.model.source.repositoryName),
               repoFullName: repo.model.source.repositoryName,
@@ -76,6 +81,7 @@ router.get(
             },
           })
           .on("error", (err: Error & { response?: { statusCode?: number; body?: unknown } }) => {
+            headersDone(true);
             const upstreamStatus = err?.response?.statusCode;
             let upstreamBody: string | undefined;
             let errCode = "zip_not_available";
@@ -130,6 +136,8 @@ router.get(
             // instead of waiting forever for the ZIP's central directory.
             if (streaming && !res.writableEnded) res.destroy();
           });
+        reqStream.once("response", () => headersDone());
+        reqStream.once("close", () => headersDone());
         reqStream.pipe(res);
         res.on("close", () => {
           reqStream.destroy();

src/server/routes/route-utils.ts

@@ -8,6 +8,7 @@ import { HTTPError } from "got";
 import { RepositoryStatus } from "../../core/types";
 import { createLogger, serializeError } from "../../core/logger";
 import { isDisabledAccount } from "./auth-utils";
+import { measureStage } from "../../core/request-monitoring";
 
 const logger = createLogger("route");
 
@@ -81,7 +82,8 @@ export async function getRepo(
     if (typeof req.params.repoId !== "string") {
       throw new AnonymousError("invalid_path", { httpStatus: 400 });
     }
-    const repo = await db.getRepository(req.params.repoId);
+    const repoId = req.params.repoId;
+    const repo = await measureStage("repository", () => db.getRepository(repoId));
     if (opt.nocheck !== true) {
       // redirect if the repository is expired
       if (

src/server/routes/webview.ts

@@ -5,8 +5,11 @@ import AnonymizedFile from "../../core/AnonymizedFile";
 import AnonymousError from "../../core/AnonymousError";
 import * as marked from "marked";
 import * as sanitizeHtml from "sanitize-html";
-import { streamToString } from "../../core/anonymize-utils";
+import { streamToString, isTextFile } from "../../core/anonymize-utils";
 import { IFile } from "../../core/model/files/files.types";
+import { fileETag } from "./file-etag";
+import { lookup } from "mime-types";
+import config from "../../config";
 
 function escapeHtml(str: string): string {
   return str
@@ -170,6 +173,19 @@ async function webView(
         object: f,
       });
     }
+    res.set("Cache-Control", "private, no-cache, must-revalidate");
+    const pageETag = fileETag(JSON.stringify(["web-v1", repo.model.source.commit, repo.model.treeGeneration, repo.model.contentCacheRevision,
+      await f.sha(), repo.model.source.repositoryName, repo.model.source.branch, config.APP_HOSTNAME, config.ANONYMIZATION_MASK]),
+      requestPath, repo.options);
+    res.set("ETag", pageETag);
+    if (req.fresh) { res.status(304).end(); return; }
+    if (req.method === "HEAD") {
+      const size = await f.size();
+      if (size != null && size > config.MAX_FILE_SIZE) throw new AnonymousError("file_too_big", { httpStatus: 413 });
+      const mime = lookup(requestPath);
+      res.type(f.extension() === "md" ? "text/html" : mime && !requestPath.endsWith(".ts") ? mime : isTextFile(requestPath) ? "text/plain" : "application/octet-stream");
+      res.end(); return;
+    }
     if (f.extension() == "md") {
       try {
         const content = await streamToString(await f.anonymizedContent());
@@ -177,6 +193,8 @@ async function webView(
         const html = `<!DOCTYPE html><html><head><title>Content</title></head><link rel="stylesheet" href="/css/all.min.css" /><body><div class="container p-3 file-content markdown-body">${body}</div></body></html>`;
         res.contentType("text/html").send(html);
       } catch {
+        // Raw Markdown must re-enter rendering when a transient failure clears.
+        res.set("ETag", fileETag(`web-raw:${pageETag}`, requestPath, repo.options));
         await f.send(res);
       }
     } else {