perf: improve the perf of Anonymous GitHub

package-lock.json

@@ -1,11 +1,11 @@
 {
-  "name": "anonymous_github",
+  "name": "@tdurieux/anonymous_github",
   "version": "2.1.1",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
-      "name": "anonymous_github",
+      "name": "@tdurieux/anonymous_github",
       "version": "2.1.1",
       "license": "GPL-3.0",
       "dependencies": {
@@ -40,6 +40,9 @@
         "unzip-stream": "^0.3.1",
         "xml-flow": "^1.0.4"
       },
+      "bin": {
+        "anonymous_github": "build/cli.js"
+      },
       "devDependencies": {
         "@types/archiver": "^5.3.1",
         "@types/compression": "^1.7.1",
@@ -61,6 +64,7 @@
         "@types/xml-flow": "^1.0.1",
         "chai": "^4.3.6",
         "mocha": "^10.1.0",
+        "nodemon": "^2.0.20",
         "ts-node": "^10.9.1",
         "typescript": "^4.8.4"
       },
@@ -712,6 +716,14 @@
         "node": ">=6.0"
       }
     },
+    "node_modules/@pm2/io/node_modules/async": {
+      "version": "2.6.4",
+      "resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz",
+      "integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==",
+      "dependencies": {
+        "lodash": "^4.17.14"
+      }
+    },
     "node_modules/@redis/bloom": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.0.2.tgz",
@@ -1192,6 +1204,12 @@
         "@types/node": "*"
       }
     },
+    "node_modules/abbrev": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
+      "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==",
+      "dev": true
+    },
     "node_modules/accepts": {
       "version": "1.3.8",
       "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
@@ -1342,11 +1360,6 @@
         "safe-buffer": "~5.1.0"
       }
     },
-    "node_modules/archiver/node_modules/async": {
-      "version": "3.2.4",
-      "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz",
-      "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ=="
-    },
     "node_modules/arg": {
       "version": "4.1.3",
       "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz",
@@ -1368,12 +1381,9 @@
       }
     },
     "node_modules/async": {
-      "version": "2.6.4",
+      "version": "3.2.4",
-      "resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz",
+      "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz",
-      "integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==",
+      "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ=="
-      "dependencies": {
-        "lodash": "^4.17.14"
-      }
     },
     "node_modules/async-listener": {
       "version": "0.6.10",
@@ -3157,6 +3167,12 @@
       "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
       "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg=="
     },
+    "node_modules/ignore-by-default": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz",
+      "integrity": "sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==",
+      "dev": true
+    },
     "node_modules/inflight": {
       "version": "1.0.6",
       "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
@@ -4232,6 +4248,88 @@
         "node": "*"
       }
     },
+    "node_modules/nodemon": {
+      "version": "2.0.20",
+      "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.20.tgz",
+      "integrity": "sha512-Km2mWHKKY5GzRg6i1j5OxOHQtuvVsgskLfigG25yTtbyfRGn/GNvIbRyOf1PSCKJ2aT/58TiuUsuOU5UToVViw==",
+      "dev": true,
+      "dependencies": {
+        "chokidar": "^3.5.2",
+        "debug": "^3.2.7",
+        "ignore-by-default": "^1.0.1",
+        "minimatch": "^3.1.2",
+        "pstree.remy": "^1.1.8",
+        "semver": "^5.7.1",
+        "simple-update-notifier": "^1.0.7",
+        "supports-color": "^5.5.0",
+        "touch": "^3.1.0",
+        "undefsafe": "^2.0.5"
+      },
+      "bin": {
+        "nodemon": "bin/nodemon.js"
+      },
+      "engines": {
+        "node": ">=8.10.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/nodemon"
+      }
+    },
+    "node_modules/nodemon/node_modules/debug": {
+      "version": "3.2.7",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
+      "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
+      "dev": true,
+      "dependencies": {
+        "ms": "^2.1.1"
+      }
+    },
+    "node_modules/nodemon/node_modules/has-flag": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+      "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
+      "dev": true,
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/nodemon/node_modules/semver": {
+      "version": "5.7.1",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
+      "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+      "dev": true,
+      "bin": {
+        "semver": "bin/semver"
+      }
+    },
+    "node_modules/nodemon/node_modules/supports-color": {
+      "version": "5.5.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
+      "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
+      "dev": true,
+      "dependencies": {
+        "has-flag": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/nopt": {
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/nopt/-/nopt-1.0.10.tgz",
+      "integrity": "sha512-NWmpvLSqUrgrAC9HCuxEvb+PSloHpqVu+FqcO4eeF2h5qYRhA7ev6KvelyQAKtegUbC6RypJnlEOhd8vloNKYg==",
+      "dev": true,
+      "dependencies": {
+        "abbrev": "1"
+      },
+      "bin": {
+        "nopt": "bin/nopt.js"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
     "node_modules/normalize-path": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
@@ -4557,6 +4655,12 @@
         "node": ">= 0.10"
       }
     },
+    "node_modules/pstree.remy": {
+      "version": "1.1.8",
+      "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz",
+      "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==",
+      "dev": true
+    },
     "node_modules/pump": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
@@ -4998,6 +5102,27 @@
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
       "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="
     },
+    "node_modules/simple-update-notifier": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-1.1.0.tgz",
+      "integrity": "sha512-VpsrsJSUcJEseSbMHkrsrAVSdvVS5I96Qo1QAQ4FxQ9wXFcB+pjj7FB7/us9+GcgfW4ziHtYMc1J0PLczb55mg==",
+      "dev": true,
+      "dependencies": {
+        "semver": "~7.0.0"
+      },
+      "engines": {
+        "node": ">=8.10.0"
+      }
+    },
+    "node_modules/simple-update-notifier/node_modules/semver": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.0.0.tgz",
+      "integrity": "sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==",
+      "dev": true,
+      "bin": {
+        "semver": "bin/semver.js"
+      }
+    },
     "node_modules/smart-buffer": {
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz",
@@ -5260,6 +5385,18 @@
         "node": ">=0.6"
       }
     },
+    "node_modules/touch": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.0.tgz",
+      "integrity": "sha512-WBx8Uy5TLtOSRtIq+M03/sKDrXCLHxwDcquSP2c43Le03/9serjQBIztjRz6FkJez9D/hleyAXTBGLwwZUw9lA==",
+      "dev": true,
+      "dependencies": {
+        "nopt": "~1.0.10"
+      },
+      "bin": {
+        "nodetouch": "bin/nodetouch.js"
+      }
+    },
     "node_modules/tr46": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz",
@@ -5427,6 +5564,12 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/undefsafe": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz",
+      "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==",
+      "dev": true
+    },
     "node_modules/universal-user-agent": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz",
@@ -6236,6 +6379,16 @@
         "shimmer": "^1.2.0",
         "signal-exit": "^3.0.3",
         "tslib": "1.9.3"
+      },
+      "dependencies": {
+        "async": {
+          "version": "2.6.4",
+          "resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz",
+          "integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==",
+          "requires": {
+            "lodash": "^4.17.14"
+          }
+        }
       }
     },
     "@redis/bloom": {
@@ -6697,6 +6850,12 @@
         "@types/node": "*"
       }
     },
+    "abbrev": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
+      "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==",
+      "dev": true
+    },
     "accepts": {
       "version": "1.3.8",
       "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
@@ -6767,13 +6926,6 @@
         "readdir-glob": "^1.0.0",
         "tar-stream": "^2.2.0",
         "zip-stream": "^4.1.0"
-      },
-      "dependencies": {
-        "async": {
-          "version": "3.2.4",
-          "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz",
-          "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ=="
-        }
       }
     },
     "archiver-utils": {
@@ -6835,12 +6987,9 @@
       "dev": true
     },
     "async": {
-      "version": "2.6.4",
+      "version": "3.2.4",
-      "resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz",
+      "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz",
-      "integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==",
+      "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ=="
-      "requires": {
-        "lodash": "^4.17.14"
-      }
     },
     "async-listener": {
       "version": "0.6.10",
@@ -8139,6 +8288,12 @@
       "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
       "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg=="
     },
+    "ignore-by-default": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz",
+      "integrity": "sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==",
+      "dev": true
+    },
     "inflight": {
       "version": "1.0.6",
       "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
@@ -8936,6 +9091,65 @@
         }
       }
     },
+    "nodemon": {
+      "version": "2.0.20",
+      "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.20.tgz",
+      "integrity": "sha512-Km2mWHKKY5GzRg6i1j5OxOHQtuvVsgskLfigG25yTtbyfRGn/GNvIbRyOf1PSCKJ2aT/58TiuUsuOU5UToVViw==",
+      "dev": true,
+      "requires": {
+        "chokidar": "^3.5.2",
+        "debug": "^3.2.7",
+        "ignore-by-default": "^1.0.1",
+        "minimatch": "^3.1.2",
+        "pstree.remy": "^1.1.8",
+        "semver": "^5.7.1",
+        "simple-update-notifier": "^1.0.7",
+        "supports-color": "^5.5.0",
+        "touch": "^3.1.0",
+        "undefsafe": "^2.0.5"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "3.2.7",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
+          "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
+          "dev": true,
+          "requires": {
+            "ms": "^2.1.1"
+          }
+        },
+        "has-flag": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+          "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
+          "dev": true
+        },
+        "semver": {
+          "version": "5.7.1",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
+          "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "5.5.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
+          "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^3.0.0"
+          }
+        }
+      }
+    },
+    "nopt": {
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/nopt/-/nopt-1.0.10.tgz",
+      "integrity": "sha512-NWmpvLSqUrgrAC9HCuxEvb+PSloHpqVu+FqcO4eeF2h5qYRhA7ev6KvelyQAKtegUbC6RypJnlEOhd8vloNKYg==",
+      "dev": true,
+      "requires": {
+        "abbrev": "1"
+      }
+    },
     "normalize-path": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
@@ -9157,6 +9371,12 @@
         "ipaddr.js": "1.9.1"
       }
     },
+    "pstree.remy": {
+      "version": "1.1.8",
+      "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz",
+      "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==",
+      "dev": true
+    },
     "pump": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
@@ -9523,6 +9743,23 @@
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
       "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="
     },
+    "simple-update-notifier": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-1.1.0.tgz",
+      "integrity": "sha512-VpsrsJSUcJEseSbMHkrsrAVSdvVS5I96Qo1QAQ4FxQ9wXFcB+pjj7FB7/us9+GcgfW4ziHtYMc1J0PLczb55mg==",
+      "dev": true,
+      "requires": {
+        "semver": "~7.0.0"
+      },
+      "dependencies": {
+        "semver": {
+          "version": "7.0.0",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-7.0.0.tgz",
+          "integrity": "sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==",
+          "dev": true
+        }
+      }
+    },
     "smart-buffer": {
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz",
@@ -9716,6 +9953,15 @@
       "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
       "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="
     },
+    "touch": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.0.tgz",
+      "integrity": "sha512-WBx8Uy5TLtOSRtIq+M03/sKDrXCLHxwDcquSP2c43Le03/9serjQBIztjRz6FkJez9D/hleyAXTBGLwwZUw9lA==",
+      "dev": true,
+      "requires": {
+        "nopt": "~1.0.10"
+      }
+    },
     "tr46": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz",
@@ -9825,6 +10071,12 @@
         "which-boxed-primitive": "^1.0.2"
       }
     },
+    "undefsafe": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz",
+      "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==",
+      "dev": true
+    },
     "universal-user-agent": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz",

package.json

@@ -82,6 +82,7 @@
     "@types/xml-flow": "^1.0.1",
     "chai": "^4.3.6",
     "mocha": "^10.1.0",
+    "nodemon": "^2.0.20",
     "ts-node": "^10.9.1",
     "typescript": "^4.8.4"
   },

public/script/app.js

@@ -508,7 +508,6 @@ angular
         $http.get("/api/user").then(
           (res) => {
             if (res) $scope.user = res.data;
-            getQuota();
           },
           () => {
             $scope.user = null;
@@ -528,22 +527,6 @@ angular
         );
       }
       getOptions();
-      function getQuota() {
-        $http.get("/api/user/quota").then((res) => {
-          $scope.quota = res.data;
-          $scope.quota.storage.percent = $scope.quota.storage.total
-            ? ($scope.quota.storage.used * 100) / $scope.quota.storage.total
-            : 100;
-          $scope.quota.file.percent = $scope.quota.file.total
-            ? ($scope.quota.file.used * 100) / $scope.quota.file.total
-            : 100;
-          $scope.quota.repository.percent = $scope.quota.repository.total
-            ? ($scope.quota.repository.used * 100) /
-              $scope.quota.repository.total
-            : 100;
-        }, console.error);
-      }
-      getQuota();
 
       function getMessage() {
         $http.get("/api/message").then(
@@ -703,6 +686,23 @@ angular
       };
       $scope.orderBy = "-anonymizeDate";
 
+      function getQuota() {
+        $http.get("/api/user/quota").then((res) => {
+          $scope.quota = res.data;
+          $scope.quota.storage.percent = $scope.quota.storage.total
+            ? ($scope.quota.storage.used * 100) / $scope.quota.storage.total
+            : 100;
+          $scope.quota.file.percent = $scope.quota.file.total
+            ? ($scope.quota.file.used * 100) / $scope.quota.file.total
+            : 100;
+          $scope.quota.repository.percent = $scope.quota.repository.total
+            ? ($scope.quota.repository.used * 100) /
+              $scope.quota.repository.total
+            : 100;
+        }, console.error);
+      }
+      getQuota();
+
       function getRepositories() {
         $http.get("/api/user/anonymized_repositories").then(
           (res) => {

src/AnonymizedFile.ts

@@ -1,34 +1,14 @@
 import { join, basename } from "path";
 import { Response } from "express";
-import { Readable, pipeline } from "stream";
+import { Readable } from "stream";
-import { promisify } from "util";
 import Repository from "./Repository";
-import { Tree, TreeElement, TreeFile } from "./types";
+import { TreeElement, TreeFile } from "./types";
 import storage from "./storage";
 import config from "../config";
 import { anonymizePath, anonymizeStream } from "./anonymize-utils";
 import AnonymousError from "./AnonymousError";
 import { handleError } from "./routes/route-utils";
 
-function tree2sha(
-  tree: any,
-  output: { [key: string]: string } = {},
-  parent: string = ""
-): { [key: string]: string } {
-  for (let i in tree) {
-    const sha = tree[i].sha as string;
-    const size = tree[i].size as number;
-    if (sha != null && size != null) {
-      output[sha] = join(parent, i);
-    } else if (tree[i].child) {
-      tree2sha(tree[i].child as Tree, output, join(parent, i));
-    } else {
-      tree2sha(tree[i] as Tree, output, join(parent, i));
-    }
-  }
-  return output;
-}
-
 /**
  * Represent a file in a anonymized repository
  */
@@ -70,27 +50,16 @@ export default class AnonymizedFile {
       });
 
     const paths = this.anonymizedPath.trim().split("/");
-
+    let currentOriginal = (await this.repository.files({
-    let currentAnonymized: TreeElement = await this.repository.anonymizedFiles({
+      force: false,
-      includeSha: true,
+    })) as TreeElement;
-    });
-    let currentOriginal: TreeElement = await this.repository.files();
     let currentOriginalPath = "";
-    let isAmbiguous = false;
     for (let i = 0; i < paths.length; i++) {
       const fileName = paths[i];
       if (fileName == "") {
         continue;
       }
-      if (!currentAnonymized[fileName]) {
+      if (!currentOriginal[fileName]) {
-        throw new AnonymousError("file_not_found", {
-          object: this,
-          httpStatus: 404,
-        });
-      }
-      currentAnonymized = currentAnonymized[fileName];
-
-      if (!isAmbiguous && !currentOriginal[fileName]) {
         // anonymize all the file in the folder and check if there is one that match the current filename
         const options = [];
         for (let originalFileName in currentOriginal) {
@@ -106,40 +75,55 @@ export default class AnonymizedFile {
         if (options.length == 1) {
           currentOriginalPath = join(currentOriginalPath, options[0]);
           currentOriginal = currentOriginal[options[0]];
+        } else if (options.length == 0) {
+          throw new AnonymousError("file_not_found", {
+            object: this,
+            httpStatus: 404,
+          });
         } else {
-          isAmbiguous = true;
+          const nextName = paths[i + 1];
+          if (!nextName) {
+            // if there is no next name we can't find the file and we return the first option
+            currentOriginalPath = join(currentOriginalPath, options[0]);
+            currentOriginal = currentOriginal[options[0]];
+          }
+          let found = false;
+          for (const option of options) {
+            const optionTree = currentOriginal[option];
+            if (optionTree.child) {
+              const optionTreeChild = optionTree.child;
+              if (optionTreeChild[nextName]) {
+                currentOriginalPath = join(currentOriginalPath, option);
+                currentOriginal = optionTreeChild;
+                found = true;
+                break;
+              }
+            }
+          }
+          if (!found) {
+            // if we didn't find the next name we return the first option
+            currentOriginalPath = join(currentOriginalPath, options[0]);
+            currentOriginal = currentOriginal[options[0]];
+          }
         }
-      } else if (!isAmbiguous) {
+      } else {
         currentOriginalPath = join(currentOriginalPath, fileName);
         currentOriginal = currentOriginal[fileName];
       }
     }
 
     if (
-      currentAnonymized.sha === undefined ||
+      currentOriginal.sha === undefined ||
-      currentAnonymized.size === undefined
+      currentOriginal.size === undefined
     ) {
       throw new AnonymousError("folder_not_supported", { object: this });
     }
 
-    const file: TreeFile = currentAnonymized as TreeFile;
+    const file = currentOriginal as TreeFile;
     this.fileSize = file.size;
     this._sha = file.sha;
 
-    if (isAmbiguous) {
+    this._originalPath = currentOriginalPath;
-      // it should never happen
-      const shaTree = tree2sha(currentOriginal);
-      if (!currentAnonymized.sha || !shaTree[file.sha]) {
-        throw new AnonymousError("file_not_found", {
-          object: this,
-          httpStatus: 404,
-        });
-      }
-
-      this._originalPath = join(currentOriginalPath, shaTree[file.sha]);
-    } else {
-      this._originalPath = currentOriginalPath;
-    }
     return this._originalPath;
   }
   extension() {
@@ -193,8 +177,7 @@ export default class AnonymizedFile {
   }
 
   async anonymizedContent() {
-    const rs = await this.content();
+    return (await this.content()).pipe(anonymizeStream(this));
-    return rs.pipe(anonymizeStream(this));
   }
 
   get originalCachePath() {
@@ -218,14 +201,24 @@ export default class AnonymizedFile {
   }
 
   async send(res: Response): Promise<void> {
-    const pipe = promisify(pipeline);
+    if (this.extension()) {
-    try {
+      res.contentType(this.extension());
-      if (this.extension()) {
-        res.contentType(this.extension());
-      }
-      await pipe(await this.anonymizedContent(), res);
-    } catch (error) {
-      handleError(error, res);
     }
+    if (this.fileSize) {
+      res.set("Content-Length", this.fileSize.toString());
+    }
+    return new Promise(async (resolve, reject) => {
+      try {
+        (await this.anonymizedContent())
+          .pipe(res)
+          .on("close", () => resolve())
+          .on("error", (error) => {
+            reject(error);
+            handleError(error, res);
+          });
+      } catch (error) {
+        handleError(error, res);
+      }
+    });
   }
 }

src/PullRequest.ts

@@ -135,10 +135,10 @@ export default class PullRequest {
    * @returns void
    */
   async anonymize() {
-    if (this.status == "ready") return;
+    if (this.status === RepositoryStatus.READY) return;
-    await this.updateStatus("preparing");
+    await this.updateStatus(RepositoryStatus.PREPARING);
     await this.updateIfNeeded({ force: true });
-    return this.updateStatus("ready");
+    return this.updateStatus(RepositoryStatus.READY);
   }
 
   /**
@@ -166,18 +166,18 @@ export default class PullRequest {
    * Expire the pullRequest
    */
   async expire() {
-    await this.updateStatus("expiring");
+    await this.updateStatus(RepositoryStatus.EXPIRING);
     await this.resetSate();
-    await this.updateStatus("expired");
+    await this.updateStatus(RepositoryStatus.EXPIRED);
   }
 
   /**
    * Remove the pullRequest
    */
   async remove() {
-    await this.updateStatus("removing");
+    await this.updateStatus(RepositoryStatus.REMOVING);
     await this.resetSate();
-    await this.updateStatus("removed");
+    await this.updateStatus(RepositoryStatus.REMOVED);
   }
 
   /**

src/Repository.ts

@@ -17,6 +17,29 @@ import AnonymousError from "./AnonymousError";
 import { downloadQueue } from "./queue";
 import { isConnected } from "./database/database";
 import AnonymizedFile from "./AnonymizedFile";
+import AnonymizedRepositoryModel from "./database/anonymizedRepositories/anonymizedRepositories.model";
+
+function anonymizeTreeRecursive(
+  tree: TreeElement,
+  terms: string[],
+  opt: {
+    /** Include the file sha in the response */
+    includeSha: boolean;
+  } = {
+    includeSha: false,
+  }
+): TreeElement {
+  if (typeof tree.size !== "object" && tree.sha !== undefined) {
+    if (opt?.includeSha) return tree as TreeFile;
+    return { size: tree.size } as TreeFile;
+  }
+  const output: Tree = {};
+  Object.getOwnPropertyNames(tree).forEach((file) => {
+    const anonymizedPath = anonymizePath(file, terms);
+    output[anonymizedPath] = anonymizeTreeRecursive(tree[file], terms, opt);
+  });
+  return output;
+}
 
 export default class Repository {
   private _model: IAnonymizedRepositoryDocument;
@@ -61,21 +84,7 @@ export default class Repository {
     }
   ): Promise<Tree> {
     const terms = this._model.options.terms || [];
-
+    return anonymizeTreeRecursive(await this.files(opt), terms, opt) as Tree;
-    function anonymizeTreeRecursive(tree: TreeElement): TreeElement {
-      if (Number.isInteger(tree.size) && tree.sha !== undefined) {
-        if (opt?.includeSha) return tree as TreeFile;
-        return { size: tree.size } as TreeFile;
-      }
-      const output: Tree = {};
-      for (const file in tree) {
-        const anonymizedPath = anonymizePath(file, terms);
-        output[anonymizedPath] = anonymizeTreeRecursive(tree[file]);
-      }
-      return output;
-    }
-
-    return anonymizeTreeRecursive(await this.files(opt)) as Tree;
   }
 
   /**
@@ -85,9 +94,15 @@ export default class Repository {
    * @returns The file tree
    */
   async files(opt: { force?: boolean } = { force: false }): Promise<Tree> {
+    if (!this._model.originalFiles && !opt.force) {
+      const res = await AnonymizedRepositoryModel.findById(this._model._id, {
+        originalFiles: 1,
+      });
+      this.model.originalFiles = res.originalFiles;
+    }
     if (
       this._model.originalFiles &&
-      Object.keys(this._model.originalFiles).length !== 0 &&
+      this._model.size.file !== 0 &&
       !opt.force
     ) {
       return this._model.originalFiles;
@@ -185,7 +200,7 @@ export default class Repository {
           console.error(
             `${branch.name} for ${this.source.githubRepository.fullName} is not found`
           );
-          await this.updateStatus("error", "branch_not_found");
+          await this.updateStatus(RepositoryStatus.ERROR, "branch_not_found");
           await this.resetSate();
           throw new AnonymousError("branch_not_found", {
             object: this,
@@ -193,7 +208,7 @@ export default class Repository {
         }
         this._model.anonymizeDate = new Date();
         console.log(`${this._model.repoId} will be updated to ${newCommit}`);
-        await this.resetSate("preparing");
+        await this.resetSate(RepositoryStatus.PREPARING);
         await downloadQueue.add(this.repoId, this, {
           jobId: this.repoId,
           attempts: 3,
@@ -207,19 +222,19 @@ export default class Repository {
    * @returns void
    */
   async anonymize() {
-    if (this.status == "ready") return;
+    if (this.status === RepositoryStatus.READY) return;
-    await this.updateStatus("preparing");
+    await this.updateStatus(RepositoryStatus.PREPARING);
     await this.files();
-    return this.updateStatus("ready");
+    return this.updateStatus(RepositoryStatus.READY);
   }
 
   /**
    * Update the last view and view count
    */
   async countView() {
-    if (!isConnected) return this.model;
     this._model.lastView = new Date();
     this._model.pageView = (this._model.pageView || 0) + 1;
+    if (!isConnected) return this.model;
     return this._model.save();
   }
 
@@ -241,18 +256,18 @@ export default class Repository {
    * Expire the repository
    */
   async expire() {
-    await this.updateStatus("expiring");
+    await this.updateStatus(RepositoryStatus.EXPIRING);
     await this.resetSate();
-    await this.updateStatus("expired");
+    await this.updateStatus(RepositoryStatus.EXPIRED);
   }
 
   /**
    * Remove the repository
    */
   async remove() {
-    await this.updateStatus("removing");
+    await this.updateStatus(RepositoryStatus.REMOVING);
     await this.resetSate();
-    await this.updateStatus("removed");
+    await this.updateStatus(RepositoryStatus.REMOVED);
   }
 
   /**

src/anonymize-utils.ts

@@ -1,5 +1,4 @@
 import config from "../config";
-import Repository from "./Repository";
 import GitHubBase from "./source/GitHubBase";
 import { isText } from "istextorbinary";
 import { basename } from "path";
@@ -60,9 +59,9 @@ export function anonymizeStream(file: AnonymizedFile) {
 
   ts._flush = function _flush(cb) {
     if (chunks.length) {
-      let data: any = Buffer.concat(chunks, len);
+      let data = Buffer.concat(chunks, len);
       if (isText(file.anonymizedPath, data)) {
-        data = anonymizeContent(data.toString(), file.repository);
+        data = Buffer.from(anonymizeContent(data.toString(), file.repository));
       }
 
       this.push(data);

src/database/anonymizedRepositories/anonymizedRepositories.schema.ts

@@ -15,7 +15,10 @@ const AnonymizedRepositorySchema = new Schema({
   lastView: Date,
   pageView: Number,
   accessToken: String,
-  owner: Schema.Types.ObjectId,
+  owner: {
+    type: Schema.Types.ObjectId,
+    index: true,
+  },
   conference: String,
   source: {
     type: { type: String },

src/database/database.ts

@@ -15,6 +15,8 @@ export let isConnected = false;
 export async function connect() {
   await mongoose.connect(MONGO_URL + "production", {
     authSource: "admin",
+    appName: "Anonymous GitHub Server",
+    compressors: "zlib",
   } as ConnectOptions);
   isConnected = true;
 

src/processes/downloadRepository.ts

@@ -2,25 +2,33 @@ import { SandboxedJob } from "bullmq";
 import { config } from "dotenv";
 config();
 import Repository from "../Repository";
+import { getRepository as getRepositoryImport } from "../database/database";
+import { RepositoryStatus } from "../types";
 
 export default async function (job: SandboxedJob<Repository, void>) {
-  const { connect, getRepository } = require("../database/database");
+  const {
-  console.log(`${job.data.repoId} is going to be downloaded`);
+    connect,
+    getRepository,
+  }: {
+    connect: () => Promise<void>;
+    getRepository: typeof getRepositoryImport;
+  } = require("../database/database");
+  console.log(`[QUEUE] ${job.data.repoId} is going to be downloaded`);
   try {
     await connect();
     const repo = await getRepository(job.data.repoId);
     job.updateProgress({ status: "get_repo" });
-    await repo.resetSate("preparing");
+    await repo.resetSate(RepositoryStatus.PREPARING, "");
     job.updateProgress({ status: "resetSate" });
     try {
       await repo.anonymize();
     } catch (error) {
-      await repo.updateStatus("error", error.message);
+      await repo.updateStatus(RepositoryStatus.ERROR, error.message);
       throw error;
     }
   } catch (error) {
     console.error(error);
   } finally {
-    console.log(`${job.data.repoId} is downloaded`);
+    console.log(`[QUEUE] ${job.data.repoId} is downloaded`);
   }
 }

src/processes/removeCache.ts

@@ -1,8 +1,15 @@
 import { SandboxedJob } from "bullmq";
 import Repository from "../Repository";
+import { getRepository as getRepositoryImport } from "../database/database";
 
 export default async function (job: SandboxedJob<Repository, void>) {
-  const { connect, getRepository } = require("../database/database");
+  const {
+    connect,
+    getRepository,
+  }: {
+    connect: () => Promise<void>;
+    getRepository: typeof getRepositoryImport;
+  } = require("../database/database");
   try {
     await connect();
     console.log(

src/processes/removeRepository.ts

@@ -1,21 +1,30 @@
 import { SandboxedJob } from "bullmq";
 import Repository from "../Repository";
+import { getRepository as getRepositoryImport } from "../database/database";
+import { RepositoryStatus } from "../types";
 
 export default async function (job: SandboxedJob<Repository, void>) {
-  const { connect, getRepository } = require("../database/database");
+  const {
+    connect,
+    getRepository,
+  }: {
+    connect: () => Promise<void>;
+    getRepository: typeof getRepositoryImport;
+  } = require("../database/database");
   try {
     await connect();
-    console.log(`${job.data.repoId} is going to be removed`);
+    console.log(`[QUEUE] ${job.data.repoId} is going to be removed`);
     const repo = await getRepository(job.data.repoId);
+    await repo.updateStatus(RepositoryStatus.REMOVING, "");
     try {
       await repo.remove();
     } catch (error) {
-      await repo.updateStatus("error", error.message);
+      await repo.updateStatus(RepositoryStatus.ERROR, error.message);
       throw error;
     }
   } catch (error) {
     console.error(error);
   } finally {
-    console.log(`${job.data.repoId} is removed`);
+    console.log(`[QUEUE] ${job.data.repoId} is removed`);
   }
 }

src/routes/file.ts

@@ -14,12 +14,13 @@ router.get(
     }
     anonymizedPath = anonymizedPath;
 
-    const repo = await getRepo(req, res);
+    const repo = await getRepo(req, res, {
+      nocheck: false,
+      includeFiles: false,
+    });
     if (!repo) return;
 
     try {
-      await repo.countView();
-
       const f = new AnonymizedFile({
         repository: repo,
         anonymizedPath,
@@ -35,7 +36,7 @@ router.get(
       );
       // cache the file for 5min
       res.header("Cache-Control", "max-age=300");
-      await f.send(res);
+      await Promise.all([repo.countView(), f.send(res)]);
     } catch (error) {
       return handleError(error, res, req);
     }

src/routes/repository-private.ts

@@ -16,6 +16,7 @@ import AnonymousError from "../AnonymousError";
 import { downloadQueue, removeQueue } from "../queue";
 import RepositoryModel from "../database/repositories/repositories.model";
 import User from "../User";
+import { RepositoryStatus } from "../types";
 
 const router = express.Router();
 
@@ -150,7 +151,7 @@ router.delete(
         });
       const user = await getUser(req);
       isOwnerOrAdmin([repo.owner.id], user);
-      await repo.updateStatus("removing");
+      await repo.updateStatus(RepositoryStatus.REMOVING);
       await removeQueue.add(repo.repoId, repo, { jobId: repo.repoId });
       return res.json({ status: repo.status });
     } catch (error) {
@@ -406,7 +407,7 @@ router.post(
         }
       }
       repo.model.conference = repoUpdate.conference;
-      await repo.updateStatus("preparing");
+      await repo.updateStatus(RepositoryStatus.PREPARING);
       res.json({ status: repo.status });
       await downloadQueue.add(repo.repoId, repo, { jobId: repo.repoId });
     } catch (error) {

src/routes/repository-public.ts

@@ -6,6 +6,7 @@ import config from "../../config";
 import { getRepo, handleError } from "./route-utils";
 import AnonymousError from "../AnonymousError";
 import { downloadQueue } from "../queue";
+import { RepositoryStatus } from "../types";
 
 const router = express.Router();
 
@@ -61,7 +62,7 @@ router.get(
   "/:repoId/files",
   async (req: express.Request, res: express.Response) => {
     res.header("Cache-Control", "no-cache");
-    const repo = await getRepo(req, res);
+    const repo = await getRepo(req, res, { includeFiles: true });
     if (!repo) return;
     try {
       res.json(await repo.anonymizedFiles({ includeSha: false }));
@@ -76,7 +77,10 @@ router.get(
   async (req: express.Request, res: express.Response) => {
     try {
       res.header("Cache-Control", "no-cache");
-      const repo = await getRepo(req, res, { nocheck: true, includeFiles: false });
+      const repo = await getRepo(req, res, {
+        nocheck: true,
+        includeFiles: false,
+      });
       if (!repo) return;
       let redirectURL = null;
       if (
@@ -105,7 +109,7 @@ router.get(
             repo.model.statusDate < fiveMinuteAgo
             // && repo.status != "preparing"
           ) {
-            await repo.updateStatus("preparing");
+            await repo.updateStatus(RepositoryStatus.PREPARING);
             await downloadQueue.add(repo.repoId, repo, {
               jobId: repo.repoId,
               attempts: 3,

src/routes/route-utils.ts

@@ -39,7 +39,7 @@ export async function getRepo(
   res: express.Response,
   opt: { nocheck?: boolean; includeFiles?: boolean } = {
     nocheck: false,
-    includeFiles: true,
+    includeFiles: false,
   }
 ) {
   try {
@@ -108,7 +108,7 @@ export function handleError(
   if (error.httpStatus) {
     status = error.httpStatus;
   } else if (message && message.indexOf("not_found") > -1) {
-    status = 400;
+    status = 404;
   } else if (message && message.indexOf("not_connected") > -1) {
     status = 401;
   }
@@ -121,17 +121,14 @@ export function handleError(
 export async function getUser(req: express.Request) {
   const user = (req.user as any).user;
   if (!user) {
-    req.logout((error) => console.error(error));
+    req.logout((error) => {
+      if (error) {
+        console.error(`[ERROR] Error while logging out: ${error}`);
+      }
+    });
     throw new AnonymousError("not_connected", {
       httpStatus: 401,
     });
   }
-  const model = await UserModel.findById(user._id);
+  return new User(new UserModel(user));
-  if (!model) {
-    req.logout((error) => console.error(error));
-    throw new AnonymousError("not_connected", {
-      httpStatus: 401,
-    });
-  }
-  return new User(model);
 }

src/routes/user.ts

@@ -10,7 +10,11 @@ router.use(ensureAuthenticated);
 
 router.get("/logout", async (req: express.Request, res: express.Response) => {
   try {
-    req.logout((error) => console.error(error));
+    req.logout((error) => {
+      if (error) {
+        console.error(`[ERROR] Logout error: ${error}`);
+      }
+    });
     res.redirect("/");
   } catch (error) {
     handleError(error, res, req);

src/source/GitHubDownload.ts

@@ -5,7 +5,7 @@ import Repository from "../Repository";
 
 import GitHubBase from "./GitHubBase";
 import AnonymizedFile from "../AnonymizedFile";
-import { SourceBase } from "../types";
+import { RepositoryStatus, SourceBase } from "../types";
 import got from "got";
 import { Readable } from "stream";
 import { OctokitResponse } from "@octokit/types";
@@ -60,7 +60,10 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
         try {
           response = await this._getZipUrl(config.GITHUB_TOKEN);
         } catch (error) {
-          await this.repository.resetSate("error", "repo_not_accessible");
+          await this.repository.resetSate(
+            RepositoryStatus.ERROR,
+            "repo_not_accessible"
+          );
           throw new AnonymousError("repo_not_accessible", {
             httpStatus: 404,
             cause: error,
@@ -68,7 +71,10 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
           });
         }
       } else {
-        await this.repository.resetSate("error", "repo_not_accessible");
+        await this.repository.resetSate(
+          RepositoryStatus.ERROR,
+          "repo_not_accessible"
+        );
         throw new AnonymousError("repo_not_accessible", {
           httpStatus: 404,
           object: this.repository,
@@ -76,7 +82,7 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
         });
       }
     }
-    await this.repository.updateStatus("download");
+    await this.repository.updateStatus(RepositoryStatus.DOWNLOAD);
     const originalPath = this.repository.originalCachePath;
     await storage.mk(originalPath);
     let progress = null;
@@ -102,7 +108,10 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
       downloadStream.addListener("downloadProgress", (p) => (progress = p));
       await storage.extractZip(originalPath, downloadStream, null, this);
     } catch (error) {
-      await this.repository.updateStatus("error", "unable_to_download");
+      await this.repository.updateStatus(
+        RepositoryStatus.ERROR,
+        "unable_to_download"
+      );
       throw new AnonymousError("unable_to_download", {
         httpStatus: 500,
         cause: error,
@@ -113,7 +122,7 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
       clearTimeout(progressTimeout);
     }
 
-    await this.repository.updateStatus("ready");
+    await this.repository.updateStatus(RepositoryStatus.READY);
   }
 
   async getFileContent(file: AnonymizedFile): Promise<Readable> {

src/source/GitHubStream.ts

@@ -3,7 +3,7 @@ import AnonymizedFile from "../AnonymizedFile";
 import Repository from "../Repository";
 import GitHubBase from "./GitHubBase";
 import storage from "../storage";
-import { SourceBase, Tree } from "../types";
+import { RepositoryStatus, SourceBase, Tree } from "../types";
 import * as path from "path";
 
 import * as stream from "stream";
@@ -26,11 +26,6 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
   }
 
   async getFileContent(file: AnonymizedFile): Promise<stream.Readable> {
-    if (!file.sha)
-      throw new AnonymousError("file_sha_not_provided", {
-        httpStatus: 400,
-        object: file,
-      });
     const octokit = new Octokit({
       auth: await this.getToken(),
     });
@@ -57,12 +52,12 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
       } else {
         content = Buffer.from("");
       }
-      if (this.repository.status != "ready")
+      if (this.repository.status !== RepositoryStatus.READY)
-        await this.repository.updateStatus("ready");
+        await this.repository.updateStatus(RepositoryStatus.READY);
       await storage.write(file.originalCachePath, content, file, this);
       return stream.Readable.from(content);
     } catch (error) {
-      if (error.status == 404) {
+      if (error.status === 404 || error.httpStatus === 404) {
         throw new AnonymousError("file_not_found", {
           httpStatus: error.status,
           cause: error,
@@ -99,15 +94,18 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
     } catch (error) {
       if (error.status == 409) {
         // empty tree
-        if (this.repository.status != "ready")
+        if (this.repository.status != RepositoryStatus.READY)
-          await this.repository.updateStatus("ready");
+          await this.repository.updateStatus(RepositoryStatus.READY);
         // cannot be empty otherwise it would try to download it again
         return { __: {} };
       } else {
         console.log(
           `[ERROR] getTree ${this.repository.repoId}@${sha}: ${error.message}`
         );
-        await this.repository.resetSate("error", "repo_not_accessible");
+        await this.repository.resetSate(
+          RepositoryStatus.ERROR,
+          "repo_not_accessible"
+        );
         throw new AnonymousError("repo_not_accessible", {
           httpStatus: error.status,
           cause: error,
@@ -124,8 +122,8 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
     if (ghRes.truncated) {
       await this.getTruncatedTree(sha, tree, parentPath, count);
     }
-    if (this.repository.status != "ready")
+    if (this.repository.status !== RepositoryStatus.READY)
-      await this.repository.updateStatus("ready");
+      await this.repository.updateStatus(RepositoryStatus.READY);
     return tree;
   }
 

src/storage/S3.ts

@@ -29,7 +29,7 @@ export default class S3Storage implements StorageBase {
       secretAccessKey: config.S3_CLIENT_SECRET,
       httpOptions: {
         timeout: 1000 * 60 * 60 * 2, // 2 hour
-      }
+      },
     });
   }
 
@@ -106,10 +106,7 @@ export default class S3Storage implements StorageBase {
           res.set("Content-Length", headers["content-length"]);
           res.set("Content-Type", headers["content-type"]);
         }
-        pipeline(
+        (response.httpResponse.createUnbufferedStream() as Readable).pipe(res);
-          response.httpResponse.createUnbufferedStream() as Readable,
-          res
-        );
       });
 
     s.send();
@@ -139,7 +136,7 @@ export default class S3Storage implements StorageBase {
       ContentType: lookup(path).toString(),
     };
     if (source) {
-      params.Tagging = `source=${source.type}`
+      params.Tagging = `source=${source.type}`;
     }
     await this.client.putObject(params).promise();
     return;

src/types.ts

@@ -6,6 +6,7 @@ import FileSystem from "./storage/FileSystem";
 import AnonymizedFile from "./AnonymizedFile";
 import * as stream from "stream";
 import * as archiver from "archiver";
+import { Response } from "express";
 
 export interface SourceBase {
   readonly type: string;
@@ -43,6 +44,8 @@ export interface StorageBase {
    */
   exists(path: string): Promise<boolean>;
 
+  send(p: string, res: Response): void;
+
   /**
    * Read the content of a file
    * @param path the path to the file
@@ -56,7 +59,12 @@ export interface StorageBase {
    * @param file the file
    * @param source the source of the file
    */
-  write(path: string, data: Buffer, file?: AnonymizedFile, source?: SourceBase): Promise<void>;
+  write(
+    path: string,
+    data: Buffer,
+    file?: AnonymizedFile,
+    source?: SourceBase
+  ): Promise<void>;
 
   /**
    * List the files from dir
@@ -71,7 +79,12 @@ export interface StorageBase {
    * @param file the file
    * @param source the source of the file
    */
-  extractZip(dir: string, tar: stream.Readable, file?: AnonymizedFile, source?: SourceBase): Promise<void>;
+  extractZip(
+    dir: string,
+    tar: stream.Readable,
+    file?: AnonymizedFile,
+    source?: SourceBase
+  ): Promise<void>;
 
   /**
    * Remove the path
@@ -113,16 +126,17 @@ export interface Branch {
   readme?: string;
 }
 
-export type RepositoryStatus =
+export enum RepositoryStatus {
-  | "queue"
+  QUEUE = "queue",
-  | "preparing"
+  PREPARING = "preparing",
-  | "download"
+  DOWNLOAD = "download",
-  | "ready"
+  READY = "ready",
-  | "expired"
+  EXPIRED = "expired",
-  | "expiring"
+  EXPIRING = "expiring",
-  | "removed"
+  REMOVED = "removed",
-  | "removing"
+  REMOVING = "removing",
-  | "error";
+  ERROR = "error",
+}
 
 export type ConferenceStatus = "ready" | "expired" | "removed";