fix: improve get tree in big repository by limiting the number of files

config.ts

@@ -7,6 +7,7 @@ interface Config {
   CLIENT_SECRET: string;
   GITHUB_TOKEN: string;
   DEFAULT_QUOTA: number;
+  MAX_FILE_FOLDER: number;
   MAX_FILE_SIZE: number;
   MAX_REPO_SIZE: number;
   AUTO_DOWNLOAD_REPO_SIZE: number;
@@ -38,6 +39,7 @@ const config: Config = {
   CLIENT_SECRET: "CLIENT_SECRET",
   GITHUB_TOKEN: "",
   DEFAULT_QUOTA: 2 * 1024 * 1024 * 1024 * 8,
+  MAX_FILE_FOLDER: 1000,
   MAX_FILE_SIZE: 100 * 1024 * 1024, // in b, 10MB
   MAX_REPO_SIZE: 60000, // in kb, 60MB
   AUTO_DOWNLOAD_REPO_SIZE: 150, // in kb, 150kb

src/database/anonymizedRepositories/anonymizedRepositories.schema.ts

@@ -25,6 +25,10 @@ const AnonymizedRepositorySchema = new Schema({
     repositoryName: String,
     accessToken: String,
   },
+  truckedFileList: {
+    type: Boolean,
+    default: false,
+  },
   originalFiles: Schema.Types.Mixed,
   options: {
     terms: [String],

src/database/anonymizedRepositories/anonymizedRepositories.types.ts

@@ -16,6 +16,7 @@ export interface IAnonymizedRepository {
     accessToken?: string;
   };
   owner: string;
+  truckedFileList: boolean;
   originalFiles: Tree;
   conference: string;
   options: {

src/queue.ts

@@ -52,7 +52,7 @@ export function startWorker() {
     path.resolve("dist/src/processes/downloadRepository.js"),
     // downloadRepository,
     {
-      concurrency: 2,
+      concurrency: 3,
       connection: {
         host: config.REDIS_HOSTNAME,
         port: config.REDIS_PORT,

src/source/GitHubStream.ts

@@ -8,6 +8,7 @@ import * as path from "path";
 
 import * as stream from "stream";
 import AnonymousError from "../AnonymousError";
+import config from "../../config";
 
 export default class GitHubStream extends GitHubBase implements SourceBase {
   constructor(
@@ -83,21 +84,18 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
   private async getTree(
     sha: string,
     truncatedTree: Tree = {},
-    parentPath: string = ""
+    parentPath: string = "",
+    count = {
+      file: 0,
+      request: 0,
+    }
   ) {
-    const octokit = new Octokit({
+    this.repository.model.truckedFileList = false;
-      auth: await this.getToken(),
-    });
-
-    let ghRes;
 
+    let ghRes: Awaited<ReturnType<typeof this.getGHTree>>;
     try {
-      ghRes = await octokit.git.getTree({
+      count.request++;
-        owner: this.githubRepository.owner,
+      ghRes = await this.getGHTree(sha, { recursive: true });
-        repo: this.githubRepository.repo,
-        tree_sha: sha,
-        recursive: "1",
-      });
     } catch (error) {
       if (error.status == 409) {
         // empty tree
@@ -106,6 +104,9 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
         // cannot be empty otherwise it would try to download it again
         return { __: {} };
       } else {
+        console.log(
+          `[ERROR] getTree ${this.repository.repoId}@${sha}: ${error.message}`
+        );
         await this.repository.resetSate("error", "repo_not_accessible");
         throw new AnonymousError("repo_not_accessible", {
           httpStatus: error.status,
@@ -118,56 +119,67 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
         });
       }
     }
-
+    const tree = this.tree2Tree(ghRes.tree, truncatedTree, parentPath);
-    const tree = this.tree2Tree(ghRes.data.tree, truncatedTree, parentPath);
+    count.file += ghRes.tree.length;
-    if (ghRes.data.truncated) {
+    if (ghRes.truncated) {
-      await this.getTruncatedTree(sha, tree, parentPath);
+      await this.getTruncatedTree(sha, tree, parentPath, count);
     }
     if (this.repository.status != "ready")
       await this.repository.updateStatus("ready");
     return tree;
   }
 
-  private async getTruncatedTree(
+  private async getGHTree(sha: string, opt = { recursive: true }) {
-    sha: string,
-    truncatedTree: Tree = {},
-    parentPath: string = ""
-  ) {
     const octokit = new Octokit({
       auth: await this.getToken(),
     });
-    try {
+    const ghRes = await octokit.git.getTree({
-      const ghRes = await octokit.git.getTree({
+      owner: this.githubRepository.owner,
-        owner: this.githubRepository.owner,
+      repo: this.githubRepository.repo,
-        repo: this.githubRepository.repo,
+      tree_sha: sha,
-        tree_sha: sha,
+      recursive: opt.recursive ? "1" : undefined,
-      });
+    });
-      const tree = ghRes.data.tree;
+    return ghRes.data;
+  }
 
-      for (let elem of tree) {
+  private async getTruncatedTree(
-        if (!elem.path) continue;
+    sha: string,
-        if (elem.type == "tree") {
+    truncatedTree: Tree = {},
-          const elementPath = path.join(parentPath, elem.path);
+    parentPath: string = "",
-          const paths = elementPath.split("/");
+    count = {
+      file: 0,
+      request: 0,
+    },
+    depth = 0
+  ) {
+    count.request++;
+    const data = await this.getGHTree(sha, { recursive: false });
+    this.tree2Tree(data.tree, truncatedTree, parentPath);
 
-          let current = truncatedTree;
+    count.file += data.tree.length;
-          for (let i = 0; i < paths.length; i++) {
+    if (data.tree.length < 100 && count.request < 200) {
-            let p = paths[i];
+      const promises: Promise<any>[] = [];
-            if (!current[p]) {
+      for (const file of data.tree) {
-              if (elem.sha)
+        const elementPath = path.join(parentPath, file.path);
-                await this.getTree(elem.sha, truncatedTree, elementPath);
+        if (file.type == "tree") {
-              break;
+          promises.push(
-            }
+            this.getTruncatedTree(
-            current = current[p] as Tree;
+              file.sha,
-          }
+              truncatedTree,
+              elementPath,
+              count,
+              depth + 1
+            )
+          );
         }
       }
-      this.tree2Tree(ghRes.data.tree, truncatedTree, parentPath);
+      await Promise.all(promises);
-      return truncatedTree;
+    } else {
-    } catch (error) {
+      const data = await this.getGHTree(sha, { recursive: true });
-      if (error.status == 409) {
+      this.tree2Tree(data.tree, truncatedTree, parentPath);
+      if (data.truncated) {
+        this.repository.model.truckedFileList = true;
       }
-      return truncatedTree;
     }
   }
 
@@ -205,6 +217,10 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
 
       // if elem is a file add the file size in the file list
       if (elem.type == "blob") {
+        if (Object.keys(current).length > config.MAX_FILE_FOLDER) {
+          this.repository.model.truckedFileList = true;
+          continue;
+        }
         let p = paths[end];
         if (p[0] == "$") {
           p = "\\" + p;