improve performance

src/database/database.ts

@@ -17,6 +17,7 @@ export async function connect() {
   await mongoose.connect(MONGO_URL + "production", {
     authSource: "admin",
     appName: "Anonymous GitHub Server",
+    compressors: "zstd",
   } as ConnectOptions);
   isConnected = true;
 

src/source/GitHubBase.ts

@@ -7,6 +7,7 @@ import { Readable } from "stream";
 import UserModel from "../database/users/users.model";
 import AnonymousError from "../AnonymousError";
 import { Octokit } from "@octokit/rest";
+import { trace } from "@opentelemetry/api";
 
 export default abstract class GitHubBase {
   type: "GitHubDownload" | "GitHubStream" | "Zip";
@@ -14,6 +15,7 @@ export default abstract class GitHubBase {
   branch: Branch;
   accessToken: string | undefined;
   repository: Repository;
+  validToken: boolean = false;
 
   constructor(
     data: {
@@ -66,21 +68,34 @@ export default abstract class GitHubBase {
   }
 
   async getToken() {
-    const user = await UserModel.findById(this.repository.owner.id);
+    const span = trace.getTracer("ano-file").startSpan("GHBase.getToken");
-    if (user && user.accessTokens.github) {
+    span.setAttribute("repoId", this.repository.repoId);
-      const check = await GitHubBase.checkToken(user.accessTokens.github);
+    try {
-      if (check) {
+      if (this.validToken) {
-        this.accessToken = user.accessTokens.github;
+        return this.accessToken as string;
-        return this.accessToken;
       }
-    }
+      const user = await UserModel.findById(this.repository.owner.id, {
-    if (this.accessToken) {
+        accessTokens: 1,
-      if (await GitHubBase.checkToken(this.accessToken)) {
+      });
-        return this.accessToken;
+      if (user?.accessTokens.github) {
+        const check = await GitHubBase.checkToken(user.accessTokens.github);
+        if (check) {
+          this.accessToken = user.accessTokens.github;
+          this.validToken = true;
+          return this.accessToken;
+        }
       }
+      if (this.accessToken) {
+        if (await GitHubBase.checkToken(this.accessToken)) {
+          this.validToken = true;
+          return this.accessToken;
+        }
+      }
+      this.accessToken = config.GITHUB_TOKEN;
+      return this.accessToken;
+    } finally {
+      span.end();
     }
-    this.accessToken = config.GITHUB_TOKEN;
-    return this.accessToken;
   }
 
   get url() {

src/source/GitHubStream.ts

@@ -169,16 +169,23 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
   }
 
   private async getGHTree(sha: string, opt = { recursive: true }) {
-    const octokit = new Octokit({
+    const span = trace.getTracer("ano-file").startSpan("GHStream.getGHTree");
-      auth: await this.getToken(),
+    span.setAttribute("repoId", this.repository.repoId);
-    });
+    span.setAttribute("sha", sha);
-    const ghRes = await octokit.git.getTree({
+    try {
-      owner: this.githubRepository.owner,
+      const octokit = new Octokit({
-      repo: this.githubRepository.repo,
+        auth: await this.getToken(),
-      tree_sha: sha,
+      });
-      recursive: opt.recursive ? "1" : undefined,
+      const ghRes = await octokit.git.getTree({
-    });
+        owner: this.githubRepository.owner,
-    return ghRes.data;
+        repo: this.githubRepository.repo,
+        tree_sha: sha,
+        recursive: opt.recursive ? "1" : undefined,
+      });
+      return ghRes.data;
+    } finally {
+      span.end();
+    }
   }
 
   private async getTruncatedTree(
@@ -191,47 +198,57 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
     },
     depth = 0
   ) {
-    count.request++;
+    const span = trace
-    let data = null;
+      .getTracer("ano-file")
-
+      .startSpan("GHStream.getTruncatedTree");
+    span.setAttribute("repoId", this.repository.repoId);
+    span.setAttribute("sha", sha);
+    span.setAttribute("parentPath", parentPath);
     try {
-      data = await this.getGHTree(sha, { recursive: false });
+      count.request++;
-      this.tree2Tree(data.tree, truncatedTree, parentPath);
+      let data = null;
-    } catch (error) {
-      console.error(error);
-      this.repository.model.truckedFileList = true;
-      return;
-    }
 
-    count.file += data.tree.length;
-    if (data.tree.length < 100 && count.request < 200) {
-      const promises: Promise<any>[] = [];
-      for (const file of data.tree) {
-        if (file.type == "tree" && file.path && file.sha) {
-          const elementPath = path.join(parentPath, file.path);
-          promises.push(
-            this.getTruncatedTree(
-              file.sha,
-              truncatedTree,
-              elementPath,
-              count,
-              depth + 1
-            )
-          );
-        }
-      }
-      await Promise.all(promises);
-    } else {
       try {
-        const data = await this.getGHTree(sha, { recursive: true });
+        data = await this.getGHTree(sha, { recursive: false });
         this.tree2Tree(data.tree, truncatedTree, parentPath);
-        if (data.truncated) {
-          this.repository.model.truckedFileList = true;
-        }
       } catch (error) {
         console.error(error);
         this.repository.model.truckedFileList = true;
+        return;
       }
+
+      count.file += data.tree.length;
+      if (data.tree.length < 100 && count.request < 200) {
+        const promises: Promise<any>[] = [];
+        for (const file of data.tree) {
+          if (file.type == "tree" && file.path && file.sha) {
+            const elementPath = path.join(parentPath, file.path);
+            promises.push(
+              this.getTruncatedTree(
+                file.sha,
+                truncatedTree,
+                elementPath,
+                count,
+                depth + 1
+              )
+            );
+          }
+        }
+        await Promise.all(promises);
+      } else {
+        try {
+          const data = await this.getGHTree(sha, { recursive: true });
+          this.tree2Tree(data.tree, truncatedTree, parentPath);
+          if (data.truncated) {
+            this.repository.model.truckedFileList = true;
+          }
+        } catch (error) {
+          console.error(error);
+          this.repository.model.truckedFileList = true;
+        }
+      }
+    } finally {
+      span.end();
     }
   }
 
@@ -247,42 +264,49 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
     partialTree: Tree = {},
     parentPath: string = ""
   ) {
-    for (let elem of tree) {
+    const span = trace.getTracer("ano-file").startSpan("GHStream.tree2Tree");
-      let current = partialTree;
+    span.setAttribute("repoId", this.repository.repoId);
+    span.setAttribute("parentPath", parentPath);
+    try {
+      for (let elem of tree) {
+        let current = partialTree;
 
-      if (!elem.path) continue;
+        if (!elem.path) continue;
 
-      const paths = path.join(parentPath, elem.path).split("/");
+        const paths = path.join(parentPath, elem.path).split("/");
 
-      // if elem is a folder iterate on all folders if it is a file stop before the filename
+        // if elem is a folder iterate on all folders if it is a file stop before the filename
-      const end = elem.type == "tree" ? paths.length : paths.length - 1;
+        const end = elem.type == "tree" ? paths.length : paths.length - 1;
-      for (let i = 0; i < end; i++) {
+        for (let i = 0; i < end; i++) {
-        let p = paths[i];
+          let p = paths[i];
-        if (p[0] == "$") {
+          if (p[0] == "$") {
-          p = "\\" + p;
+            p = "\\" + p;
+          }
+          if (!current[p]) {
+            current[p] = {};
+          }
+          current = current[p] as Tree;
         }
-        if (!current[p]) {
+
-          current[p] = {};
+        // if elem is a file add the file size in the file list
+        if (elem.type == "blob") {
+          if (Object.keys(current).length > config.MAX_FILE_FOLDER) {
+            this.repository.model.truckedFileList = true;
+            continue;
+          }
+          let p = paths[end];
+          if (p[0] == "$") {
+            p = "\\" + p;
+          }
+          current[p] = {
+            size: elem.size || 0, // size in bit
+            sha: elem.sha || "",
+          };
         }
-        current = current[p] as Tree;
-      }
-
-      // if elem is a file add the file size in the file list
-      if (elem.type == "blob") {
-        if (Object.keys(current).length > config.MAX_FILE_FOLDER) {
-          this.repository.model.truckedFileList = true;
-          continue;
-        }
-        let p = paths[end];
-        if (p[0] == "$") {
-          p = "\\" + p;
-        }
-        current[p] = {
-          size: elem.size || 0, // size in bit
-          sha: elem.sha || "",
-        };
       }
+      return partialTree;
+    } finally {
+      span.end();
     }
-    return partialTree;
   }
 }

src/storage/FileSystem.ts

@@ -74,46 +74,46 @@ export default class FileSystem implements StorageBase {
     file?: AnonymizedFile,
     source?: SourceBase
   ): Promise<void> {
-    return trace
+    const span = trace.getTracer("ano-file").startSpan("fs.write");
-      .getTracer("ano-file")
+    span.setAttribute("path", p);
-      .startActiveSpan("fs.write", async (span) => {
+    try {
-        span.setAttribute("path", p);
+      await this.mk(dirname(p));
-        try {
+      return await fs.promises.writeFile(join(config.FOLDER, p), data, "utf-8");
-          await this.mk(dirname(p));
+    } finally {
-          return await fs.promises.writeFile(
+      span.end();
-            join(config.FOLDER, p),
+    }
-            data,
-            "utf-8"
-          );
-        } finally {
-          span.end();
-        }
-      });
   }
 
   /** @override */
   async rm(dir: string): Promise<void> {
     const span = trace.getTracer("ano-file").startSpan("fs.rm");
     span.setAttribute("path", dir);
-    await fs.promises.rm(join(config.FOLDER, dir), {
+    try {
-      force: true,
+      await fs.promises.rm(join(config.FOLDER, dir), {
-      recursive: true,
+        force: true,
-    });
+        recursive: true,
-    span.end();
+      });
+    } finally {
+      span.end();
+    }
   }
 
   /** @override */
   async mk(dir: string): Promise<void> {
-    return trace
+    const span = trace.getTracer("ano-file").startSpan("fs.mk");
-      .getTracer("ano-file")
+    span.setAttribute("path", dir);
-      .startActiveSpan("fs.mk", async (span) => {
+    try {
-        span.setAttribute("path", dir);
+      await fs.promises.mkdir(join(config.FOLDER, dir), {
-        if ((await this.exists(dir)) === FILE_TYPE.NOT_FOUND)
+        recursive: true,
-          await fs.promises.mkdir(join(config.FOLDER, dir), {
-            recursive: true,
-          });
-        span.end();
       });
+    } catch (err: any) {
+      if (err.code !== "EEXIST") {
+        span.recordException(err);
+        throw err;
+      }
+    } finally {
+      span.end();
+    }
   }
 
   /** @override */