improve performance

src/database/database.ts

@@ -17,6 +17,7 @@ export async function connect() {
   await mongoose.connect(MONGO_URL + "production", {
     authSource: "admin",
     appName: "Anonymous GitHub Server",
+    compressors: "zstd",
   } as ConnectOptions);
   isConnected = true;
 

src/source/GitHubBase.ts

@@ -7,6 +7,7 @@ import { Readable } from "stream";
 import UserModel from "../database/users/users.model";
 import AnonymousError from "../AnonymousError";
 import { Octokit } from "@octokit/rest";
+import { trace } from "@opentelemetry/api";
 
 export default abstract class GitHubBase {
   type: "GitHubDownload" | "GitHubStream" | "Zip";
@@ -14,6 +15,7 @@ export default abstract class GitHubBase {
   branch: Branch;
   accessToken: string | undefined;
   repository: Repository;
+  validToken: boolean = false;
 
   constructor(
     data: {
@@ -66,21 +68,34 @@ export default abstract class GitHubBase {
   }
 
   async getToken() {
-    const user = await UserModel.findById(this.repository.owner.id);
+    const span = trace.getTracer("ano-file").startSpan("GHBase.getToken");
-    if (user && user.accessTokens.github) {
+    span.setAttribute("repoId", this.repository.repoId);
+    try {
+      if (this.validToken) {
+        return this.accessToken as string;
+      }
+      const user = await UserModel.findById(this.repository.owner.id, {
+        accessTokens: 1,
+      });
+      if (user?.accessTokens.github) {
         const check = await GitHubBase.checkToken(user.accessTokens.github);
         if (check) {
           this.accessToken = user.accessTokens.github;
+          this.validToken = true;
           return this.accessToken;
         }
       }
       if (this.accessToken) {
         if (await GitHubBase.checkToken(this.accessToken)) {
+          this.validToken = true;
           return this.accessToken;
         }
       }
       this.accessToken = config.GITHUB_TOKEN;
       return this.accessToken;
+    } finally {
+      span.end();
+    }
   }
 
   get url() {

src/source/GitHubStream.ts

@@ -169,6 +169,10 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
   }
 
   private async getGHTree(sha: string, opt = { recursive: true }) {
+    const span = trace.getTracer("ano-file").startSpan("GHStream.getGHTree");
+    span.setAttribute("repoId", this.repository.repoId);
+    span.setAttribute("sha", sha);
+    try {
       const octokit = new Octokit({
         auth: await this.getToken(),
       });
@@ -179,6 +183,9 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
         recursive: opt.recursive ? "1" : undefined,
       });
       return ghRes.data;
+    } finally {
+      span.end();
+    }
   }
 
   private async getTruncatedTree(
@@ -191,6 +198,13 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
     },
     depth = 0
   ) {
+    const span = trace
+      .getTracer("ano-file")
+      .startSpan("GHStream.getTruncatedTree");
+    span.setAttribute("repoId", this.repository.repoId);
+    span.setAttribute("sha", sha);
+    span.setAttribute("parentPath", parentPath);
+    try {
       count.request++;
       let data = null;
 
@@ -233,6 +247,9 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
           this.repository.model.truckedFileList = true;
         }
       }
+    } finally {
+      span.end();
+    }
   }
 
   private tree2Tree(
@@ -247,6 +264,10 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
     partialTree: Tree = {},
     parentPath: string = ""
   ) {
+    const span = trace.getTracer("ano-file").startSpan("GHStream.tree2Tree");
+    span.setAttribute("repoId", this.repository.repoId);
+    span.setAttribute("parentPath", parentPath);
+    try {
       for (let elem of tree) {
         let current = partialTree;
 
@@ -284,5 +305,8 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
         }
       }
       return partialTree;
+    } finally {
+      span.end();
+    }
   }
 }

src/storage/FileSystem.ts

@@ -74,46 +74,46 @@ export default class FileSystem implements StorageBase {
     file?: AnonymizedFile,
     source?: SourceBase
   ): Promise<void> {
-    return trace
+    const span = trace.getTracer("ano-file").startSpan("fs.write");
-      .getTracer("ano-file")
-      .startActiveSpan("fs.write", async (span) => {
     span.setAttribute("path", p);
     try {
       await this.mk(dirname(p));
-          return await fs.promises.writeFile(
+      return await fs.promises.writeFile(join(config.FOLDER, p), data, "utf-8");
-            join(config.FOLDER, p),
-            data,
-            "utf-8"
-          );
     } finally {
       span.end();
     }
-      });
   }
 
   /** @override */
   async rm(dir: string): Promise<void> {
     const span = trace.getTracer("ano-file").startSpan("fs.rm");
     span.setAttribute("path", dir);
+    try {
       await fs.promises.rm(join(config.FOLDER, dir), {
         force: true,
         recursive: true,
       });
+    } finally {
       span.end();
     }
+  }
 
   /** @override */
   async mk(dir: string): Promise<void> {
-    return trace
+    const span = trace.getTracer("ano-file").startSpan("fs.mk");
-      .getTracer("ano-file")
-      .startActiveSpan("fs.mk", async (span) => {
     span.setAttribute("path", dir);
-        if ((await this.exists(dir)) === FILE_TYPE.NOT_FOUND)
+    try {
       await fs.promises.mkdir(join(config.FOLDER, dir), {
         recursive: true,
       });
+    } catch (err: any) {
+      if (err.code !== "EEXIST") {
+        span.recordException(err);
+        throw err;
+      }
+    } finally {
       span.end();
-      });
+    }
   }
 
   /** @override */