feat: flatten file tree for better performance

src/core/storage/FileSystem.ts

@@ -1,4 +1,3 @@
-import { Tree } from "../types";
 import config from "../../config";
 import * as fs from "fs";
 import { Extract } from "unzip-stream";
@@ -10,6 +9,8 @@ import { promisify } from "util";
 import { lookup } from "mime-types";
 import { trace } from "@opentelemetry/api";
 import StorageBase, { FILE_TYPE } from "./Storage";
+import FileModel from "../model/files/files.model";
+import { IFile } from "../model/files/files.types";
 
 export default class FileSystem extends StorageBase {
   type = "FileSystem";
@@ -138,23 +139,25 @@ export default class FileSystem extends StorageBase {
     opt: {
       onEntry?: (file: { path: string; size: number }) => void;
     } = {}
-  ): Promise<Tree> {
+  ): Promise<IFile[]> {
     return trace
       .getTracer("ano-file")
       .startActiveSpan("fs.listFiles", async (span) => {
         span.setAttribute("path", dir);
         const fullPath = join(config.FOLDER, this.repoPath(repoId), dir);
         let files = await fs.promises.readdir(fullPath);
-        const output: Tree = {};
+        const output2: IFile[] = [];
         for (let file of files) {
           let filePath = join(fullPath, file);
           try {
             const stats = await fs.promises.stat(filePath);
-            if (file[0] == "$") {
-              file = "\\" + file;
-            }
             if (stats.isDirectory()) {
-              output[file] = await this.listFiles(repoId, join(dir, file), opt);
+              output2.push(
+                new FileModel({ name: file, path: dir, repoID: repoId })
+              );
+              output2.push(
+                ...(await this.listFiles(repoId, join(dir, file), opt))
+              );
             } else if (stats.isFile()) {
               if (opt.onEntry) {
                 opt.onEntry({
@@ -162,14 +165,22 @@ export default class FileSystem extends StorageBase {
                   size: stats.size,
                 });
               }
-              output[file] = { size: stats.size, sha: stats.ino.toString() };
+              output2.push(
+                new FileModel({
+                  name: file,
+                  path: dir,
+                  repoID: repoId,
+                  size: stats.size,
+                  sha: stats.ino.toString(),
+                })
+              );
             }
           } catch (error) {
             span.recordException(error as Error);
           }
         }
         span.end();
-        return output;
+        return output2;
       });
   }
 

src/core/storage/S3.ts

@@ -14,9 +14,10 @@ import { lookup } from "mime-types";
 import * as archiver from "archiver";
 import { trace } from "@opentelemetry/api";
 import { dirname, basename, join } from "path";
-import { Tree, TreeFile } from "../types";
 import AnonymousError from "../AnonymousError";
 import StorageBase, { FILE_TYPE } from "./Storage";
+import { IFile } from "../model/files/files.types";
+import FileModel from "../model/files/files.model";
 
 export default class S3Storage extends StorageBase {
   type = "AWS";
@@ -245,13 +246,13 @@ export default class S3Storage extends StorageBase {
   }
 
   /** @override */
-  async listFiles(repoId: string, dir: string = ""): Promise<Tree> {
+  async listFiles(repoId: string, dir: string = ""): Promise<IFile[]> {
     const span = trace.getTracer("ano-file").startSpan("s3.listFiles");
     span.setAttribute("path", dir);
     try {
       if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
       if (dir && dir[dir.length - 1] != "/") dir = dir + "/";
-      const out: Tree = {};
+      const out: IFile[] = [];
       let req: ListObjectsV2CommandOutput;
       let nextContinuationToken: string | undefined;
       do {
@@ -267,22 +268,15 @@ export default class S3Storage extends StorageBase {
         for (const f of req.Contents) {
           if (!f.Key) continue;
           f.Key = f.Key.replace(join(this.repoPath(repoId), dir), "");
-          const paths = f.Key.split("/");
-          let current: Tree = out;
-          for (let i = 0; i < paths.length - 1; i++) {
-            let p = paths[i];
-            if (!p) continue;
-            if (!(current[p] as Tree)) {
-              current[p] = {} as Tree;
-            }
-            current = current[p] as Tree;
-          }
-
-          if (f.ETag) {
-            const fileInfo: TreeFile = { size: f.Size || 0, sha: f.ETag };
-            const fileName = paths[paths.length - 1];
-            if (fileName) current[fileName] = fileInfo;
-          }
+          out.push(
+            new FileModel({
+              name: basename(f.Key),
+              path: dirname(f.Key),
+              repoID: repoId,
+              size: f.Size,
+              sha: f.ETag,
+            })
+          );
         }
       } while (req && req.Contents && req.IsTruncated);
       return out;

src/core/storage/Storage.ts

@@ -3,9 +3,9 @@ import { Transform, Readable } from "stream";
 import * as archiver from "archiver";
 import { Response } from "express";
 
-import { Tree } from "../types";
 import S3Storage from "./S3";
 import FileSystem from "./FileSystem";
+import { IFile } from "../model/files/files.types";
 
 export type Storage = S3Storage | FileSystem;
 
@@ -62,7 +62,7 @@ export default abstract class StorageBase {
    * List the files from dir
    * @param dir
    */
-  abstract listFiles(repoId: string, dir: string): Promise<Tree>;
+  abstract listFiles(repoId: string, dir: string): Promise<IFile[]>;
 
   /**
    * Extract the content of tar to dir