fix: fix webview & improve download progress

src/core/source/GitHubDownload.ts

@@ -42,25 +42,16 @@ export default class GitHubDownload extends GitHubBase {
         });
       }
       await storage.mk(this.data.repoId);
-      let downloadProgress: { transferred: number } | undefined = undefined;
-      let progressTimeout;
-      let inDownload = true;
-
-      async function updateProgress() {
-        if (inDownload) {
-          if (progress) {
-            progress(downloadProgress?.transferred?.toString() || "");
-          }
-          progressTimeout = setTimeout(updateProgress, 1500);
-        }
-      }
-      updateProgress();
-
       try {
         const downloadStream = got.stream(response.url);
-        downloadStream.addListener("downloadProgress", async (p) => {
-          downloadProgress = p;
-        });
+        downloadStream.addListener(
+          "downloadProgress",
+          (p: { transferred?: number }) => {
+            if (progress && p.transferred) {
+              progress("Repository download: " + humanFileSize(p.transferred));
+            }
+          }
+        );
         await storage.extractZip(
           this.data.repoId,
           "",
@@ -74,9 +65,6 @@ export default class GitHubDownload extends GitHubBase {
           cause: error as Error,
           object: this.data,
         });
-      } finally {
-        inDownload = false;
-        clearTimeout(progressTimeout);
       }
     } finally {
       span.end();
@@ -116,6 +104,40 @@ export default class GitHubDownload extends GitHubBase {
     if ((await storage.exists(this.data.repoId)) === FILE_TYPE.NOT_FOUND) {
       await this.download(progress);
     }
-    return storage.listFiles(this.data.repoId);
+    let nbFiles = 0;
+    return storage.listFiles(this.data.repoId, "", {
+      onEntry: () => {
+        if (progress) {
+          nbFiles++;
+          progress("List file: " + nbFiles);
+        }
+      },
+    });
   }
 }
+
+function humanFileSize(bytes: number, si = false, dp = 1) {
+  const thresh = si ? 1000 : 1024;
+
+  bytes = bytes / 8;
+
+  if (Math.abs(bytes) < thresh) {
+    return bytes + "B";
+  }
+
+  const units = si
+    ? ["kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]
+    : ["KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"];
+  let u = -1;
+  const r = 10 ** dp;
+
+  do {
+    bytes /= thresh;
+    ++u;
+  } while (
+    Math.round(Math.abs(bytes) * r) / r >= thresh &&
+    u < units.length - 1
+  );
+
+  return bytes.toFixed(dp) + "" + units[u];
+}

src/core/source/GitHubStream.ts

@@ -19,7 +19,7 @@ export default class GitHubStream extends GitHubBase {
     super(data);
   }
 
-  downloadFile(token: string, sha: string) {  
+  downloadFile(token: string, sha: string) {
     const span = trace.getTracer("ano-file").startSpan("GHStream.downloadFile");
     span.setAttribute("sha", sha);
     const oct = octokit(token);
@@ -128,11 +128,11 @@ export default class GitHubStream extends GitHubBase {
     }
   }
 
-  async getFiles() {
+  async getFiles(progress?: (status: string) => void) {
     const span = trace.getTracer("ano-file").startSpan("GHStream.getFiles");
     span.setAttribute("repoId", this.data.repoId);
     try {
-      return this.getTree(this.data.commit);
+      return this.getTree(this.data.commit, progress);
     } finally {
       span.end();
     }
@@ -140,6 +140,7 @@ export default class GitHubStream extends GitHubBase {
 
   private async getTree(
     sha: string,
+    progress?: (status: string) => void,
     truncatedTree: Tree = {},
     parentPath: string = "",
     count = {
@@ -155,7 +156,6 @@ export default class GitHubStream extends GitHubBase {
       count.request++;
       ghRes = await this.getGHTree(sha, { recursive: true });
     } catch (error) {
-      console.error(error);
       span.recordException(error as Error);
       if ((error as any).status == 409) {
         // cannot be empty otherwise it would try to download it again
@@ -176,8 +176,11 @@ export default class GitHubStream extends GitHubBase {
     }
     const tree = this.tree2Tree(ghRes.tree, truncatedTree, parentPath);
     count.file += ghRes.tree.length;
+    if (progress) {
+      progress("List file: " + count.file);
+    }
     if (ghRes.truncated) {
-      await this.getTruncatedTree(sha, tree, parentPath, count);
+      await this.getTruncatedTree(sha, progress, tree, parentPath, count);
     }
     span.end();
     return tree;
@@ -202,6 +205,7 @@ export default class GitHubStream extends GitHubBase {
 
   private async getTruncatedTree(
     sha: string,
+    progress?: (status: string) => void,
     truncatedTree: Tree = {},
     parentPath: string = "",
     count = {
@@ -230,6 +234,9 @@ export default class GitHubStream extends GitHubBase {
       }
 
       count.file += data.tree.length;
+      if (progress) {
+        progress("List file: " + count.file);
+      }
       if (data.tree.length < 100 && count.request < 200) {
         const promises: Promise<any>[] = [];
         for (const file of data.tree) {
@@ -238,6 +245,7 @@ export default class GitHubStream extends GitHubBase {
             promises.push(
               this.getTruncatedTree(
                 file.sha,
+                progress,
                 truncatedTree,
                 elementPath,
                 count,

src/core/source/Zip.ts

@@ -12,8 +12,16 @@ export default class Zip implements SourceBase {
     this.url = data.url;
   }
 
-  async getFiles() {
-    return storage.listFiles(this.repoId);
+  async getFiles(progress?: (status: string) => void) {
+    let nbFiles = 0;
+    return storage.listFiles(this.repoId, "", {
+      onEntry: () => {
+        if (progress) {
+          nbFiles++;
+          progress("List file: " + nbFiles);
+        }
+      },
+    });
   }
 
   async getFileContent(file: AnonymizedFile): Promise<stream.Readable> {