perf: bound request work and resource usage (#866)

* perf: bound request work and resource usage

* fix: address performance review regressions

* fix: guard content expiration and dashboard state

* fix: tolerate cache outages and retire derived paths

* fix: revalidate expiration claims and status pagination

* fix: retry interrupted repository metadata cleanup

* fix: retire obsolete content cache storage

* fix: activate tree metadata and recheck expiration claims

* fix: fence cache publication across expiration

* fix: recover staged trees and enforce cache lifecycle cleanup

* fix: prune expired transforms during database outages

* fix: finish conference removals and reload expiration candidates

src/core/storage/FileSystem.ts

@@ -1,7 +1,8 @@
+import { appendArchiveEntry } from "../archive-entry";
 import config from "../../config";
 import * as fs from "fs";
 import { Extract } from "unzip-stream";
-import { join, basename, dirname } from "path";
+import { join, dirname } from "path";
 import { Response } from "express";
 import { Readable, pipeline, Transform } from "stream";
 import * as archiver from "archiver";
@@ -144,6 +145,24 @@ export default class FileSystem extends StorageBase {
     });
   }
 
+  async removeLegacyContent(repoId: string): Promise<void> {
+    let directory: fs.Dir;
+    try { directory = await fs.promises.opendir(join(config.FOLDER, this.repoPath(repoId))); }
+    catch (error) {
+      if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
+      throw error;
+    }
+    for await (const entry of directory) {
+      if (entry.name !== "__content" || !entry.isDirectory()) { await this.rm(repoId, entry.name); continue; }
+      const content = await fs.promises.opendir(join(config.FOLDER, this.repoPath(repoId), "__content"));
+      for await (const generation of content) {
+        if (generation.name !== "__retired" && !/^[a-f0-9]{64}$/.test(generation.name)) {
+          await this.rm(repoId, join("__content", generation.name));
+        }
+      }
+    }
+  }
+
   /** @override */
   async mk(repoId: string, dir: string = ""): Promise<void> {
     this.assertSafePath(dir);
@@ -243,26 +262,28 @@ export default class FileSystem extends StorageBase {
   ) {
     this.assertSafePath(dir);
     const archive = archiver(opt?.format || "zip", {});
-    const fullPath = join(config.FOLDER, this.repoPath(repoId), dir);
-
-    await this.listFiles(repoId, dir, {
-      onEntry: async (file) => {
-        let rs: Readable = await this.read(repoId, file.path);
+    let active: Readable | undefined;
+    let stopped = false;
+    archive.once("close", () => { stopped = true; active?.destroy(); });
+    void (async () => {
+      for (const file of await this.listFiles(repoId, dir)) {
+        if (stopped || file.size == null) continue;
+        const filename = join(file.path, file.name);
+        const source = await this.read(repoId, filename);
+        if (stopped) { source.destroy(); return; }
+        active = source;
+        let output = source;
         if (opt?.fileTransformer) {
-          const src = rs;
-          const transformer = opt.fileTransformer(file.path);
-          src.on("error", (err) => transformer.destroy(err));
-          rs = src.pipe(transformer);
+          const transformer = opt.fileTransformer(filename);
+          source.on("error", error => transformer.destroy(error));
+          transformer.once("close", () => source.destroy());
+          output = source.pipe(transformer);
+          active = output;
         }
-        const f = file.path.replace(fullPath, "");
-        archive.append(rs, {
-          name: basename(f),
-          prefix: dirname(f),
-        });
-      },
-    }).then(() => {
-      archive.finalize();
-    });
+        await appendArchiveEntry(archive, output, { name: filename.slice(dir ? dir.length + 1 : 0) });
+      }
+      if (!stopped) await archive.finalize();
+    })().catch(error => archive.destroy(error));
     return archive;
   }
 }