diff --git a/package-lock.json b/package-lock.json index ab0fd17..a3716dc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,11 +1,11 @@ { - "name": "anonymous_github", + "name": "@tdurieux/anonymous_github", "version": "2.1.1", "lockfileVersion": 2, "requires": true, "packages": { "": { - "name": "anonymous_github", + "name": "@tdurieux/anonymous_github", "version": "2.1.1", "license": "GPL-3.0", "dependencies": { @@ -40,6 +40,9 @@ "unzip-stream": "^0.3.1", "xml-flow": "^1.0.4" }, + "bin": { + "anonymous_github": "build/cli.js" + }, "devDependencies": { "@types/archiver": "^5.3.1", "@types/compression": "^1.7.1", @@ -61,6 +64,7 @@ "@types/xml-flow": "^1.0.1", "chai": "^4.3.6", "mocha": "^10.1.0", + "nodemon": "^2.0.20", "ts-node": "^10.9.1", "typescript": "^4.8.4" }, @@ -712,6 +716,14 @@ "node": ">=6.0" } }, + "node_modules/@pm2/io/node_modules/async": { + "version": "2.6.4", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz", + "integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==", + "dependencies": { + "lodash": "^4.17.14" + } + }, "node_modules/@redis/bloom": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.0.2.tgz", @@ -1192,6 +1204,12 @@ "@types/node": "*" } }, + "node_modules/abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "dev": true + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -1342,11 +1360,6 @@ "safe-buffer": "~5.1.0" } }, - "node_modules/archiver/node_modules/async": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", - "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" - }, "node_modules/arg": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", @@ -1368,12 +1381,9 @@ } }, "node_modules/async": { - "version": "2.6.4", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz", - "integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==", - "dependencies": { - "lodash": "^4.17.14" - } + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", + "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" }, "node_modules/async-listener": { "version": "0.6.10", @@ -3157,6 +3167,12 @@ "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" }, + "node_modules/ignore-by-default": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", + "integrity": "sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==", + "dev": true + }, "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -4232,6 +4248,88 @@ "node": "*" } }, + "node_modules/nodemon": { + "version": "2.0.20", + "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.20.tgz", + "integrity": "sha512-Km2mWHKKY5GzRg6i1j5OxOHQtuvVsgskLfigG25yTtbyfRGn/GNvIbRyOf1PSCKJ2aT/58TiuUsuOU5UToVViw==", + "dev": true, + "dependencies": { + "chokidar": "^3.5.2", + "debug": "^3.2.7", + "ignore-by-default": "^1.0.1", + "minimatch": "^3.1.2", + "pstree.remy": "^1.1.8", + "semver": "^5.7.1", + "simple-update-notifier": "^1.0.7", + "supports-color": "^5.5.0", + "touch": "^3.1.0", + "undefsafe": "^2.0.5" + }, + "bin": { + "nodemon": "bin/nodemon.js" + }, + "engines": { + "node": ">=8.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/nodemon" + } + }, + "node_modules/nodemon/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/nodemon/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/nodemon/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/nodemon/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/nopt": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-1.0.10.tgz", + "integrity": "sha512-NWmpvLSqUrgrAC9HCuxEvb+PSloHpqVu+FqcO4eeF2h5qYRhA7ev6KvelyQAKtegUbC6RypJnlEOhd8vloNKYg==", + "dev": true, + "dependencies": { + "abbrev": "1" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": "*" + } + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -4557,6 +4655,12 @@ "node": ">= 0.10" } }, + "node_modules/pstree.remy": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", + "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==", + "dev": true + }, "node_modules/pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", @@ -4998,6 +5102,27 @@ "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" }, + "node_modules/simple-update-notifier": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-1.1.0.tgz", + "integrity": "sha512-VpsrsJSUcJEseSbMHkrsrAVSdvVS5I96Qo1QAQ4FxQ9wXFcB+pjj7FB7/us9+GcgfW4ziHtYMc1J0PLczb55mg==", + "dev": true, + "dependencies": { + "semver": "~7.0.0" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/simple-update-notifier/node_modules/semver": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.0.0.tgz", + "integrity": "sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, "node_modules/smart-buffer": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", @@ -5260,6 +5385,18 @@ "node": ">=0.6" } }, + "node_modules/touch": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.0.tgz", + "integrity": "sha512-WBx8Uy5TLtOSRtIq+M03/sKDrXCLHxwDcquSP2c43Le03/9serjQBIztjRz6FkJez9D/hleyAXTBGLwwZUw9lA==", + "dev": true, + "dependencies": { + "nopt": "~1.0.10" + }, + "bin": { + "nodetouch": "bin/nodetouch.js" + } + }, "node_modules/tr46": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", @@ -5427,6 +5564,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/undefsafe": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz", + "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", + "dev": true + }, "node_modules/universal-user-agent": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", @@ -6236,6 +6379,16 @@ "shimmer": "^1.2.0", "signal-exit": "^3.0.3", "tslib": "1.9.3" + }, + "dependencies": { + "async": { + "version": "2.6.4", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz", + "integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==", + "requires": { + "lodash": "^4.17.14" + } + } } }, "@redis/bloom": { @@ -6697,6 +6850,12 @@ "@types/node": "*" } }, + "abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "dev": true + }, "accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -6767,13 +6926,6 @@ "readdir-glob": "^1.0.0", "tar-stream": "^2.2.0", "zip-stream": "^4.1.0" - }, - "dependencies": { - "async": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", - "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" - } } }, "archiver-utils": { @@ -6835,12 +6987,9 @@ "dev": true }, "async": { - "version": "2.6.4", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz", - "integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==", - "requires": { - "lodash": "^4.17.14" - } + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", + "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" }, "async-listener": { "version": "0.6.10", @@ -8139,6 +8288,12 @@ "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" }, + "ignore-by-default": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", + "integrity": "sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==", + "dev": true + }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -8936,6 +9091,65 @@ } } }, + "nodemon": { + "version": "2.0.20", + "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.20.tgz", + "integrity": "sha512-Km2mWHKKY5GzRg6i1j5OxOHQtuvVsgskLfigG25yTtbyfRGn/GNvIbRyOf1PSCKJ2aT/58TiuUsuOU5UToVViw==", + "dev": true, + "requires": { + "chokidar": "^3.5.2", + "debug": "^3.2.7", + "ignore-by-default": "^1.0.1", + "minimatch": "^3.1.2", + "pstree.remy": "^1.1.8", + "semver": "^5.7.1", + "simple-update-notifier": "^1.0.7", + "supports-color": "^5.5.0", + "touch": "^3.1.0", + "undefsafe": "^2.0.5" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true + }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + } + } + }, + "nopt": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-1.0.10.tgz", + "integrity": "sha512-NWmpvLSqUrgrAC9HCuxEvb+PSloHpqVu+FqcO4eeF2h5qYRhA7ev6KvelyQAKtegUbC6RypJnlEOhd8vloNKYg==", + "dev": true, + "requires": { + "abbrev": "1" + } + }, "normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -9157,6 +9371,12 @@ "ipaddr.js": "1.9.1" } }, + "pstree.remy": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", + "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==", + "dev": true + }, "pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", @@ -9523,6 +9743,23 @@ "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" }, + "simple-update-notifier": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-1.1.0.tgz", + "integrity": "sha512-VpsrsJSUcJEseSbMHkrsrAVSdvVS5I96Qo1QAQ4FxQ9wXFcB+pjj7FB7/us9+GcgfW4ziHtYMc1J0PLczb55mg==", + "dev": true, + "requires": { + "semver": "~7.0.0" + }, + "dependencies": { + "semver": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.0.0.tgz", + "integrity": "sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==", + "dev": true + } + } + }, "smart-buffer": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", @@ -9716,6 +9953,15 @@ "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" }, + "touch": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.0.tgz", + "integrity": "sha512-WBx8Uy5TLtOSRtIq+M03/sKDrXCLHxwDcquSP2c43Le03/9serjQBIztjRz6FkJez9D/hleyAXTBGLwwZUw9lA==", + "dev": true, + "requires": { + "nopt": "~1.0.10" + } + }, "tr46": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", @@ -9825,6 +10071,12 @@ "which-boxed-primitive": "^1.0.2" } }, + "undefsafe": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz", + "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", + "dev": true + }, "universal-user-agent": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", diff --git a/package.json b/package.json index bc1a2ec..83a388a 100644 --- a/package.json +++ b/package.json @@ -82,6 +82,7 @@ "@types/xml-flow": "^1.0.1", "chai": "^4.3.6", "mocha": "^10.1.0", + "nodemon": "^2.0.20", "ts-node": "^10.9.1", "typescript": "^4.8.4" }, diff --git a/public/script/app.js b/public/script/app.js index a652dd1..5e3ac45 100644 --- a/public/script/app.js +++ b/public/script/app.js @@ -508,7 +508,6 @@ angular $http.get("/api/user").then( (res) => { if (res) $scope.user = res.data; - getQuota(); }, () => { $scope.user = null; @@ -528,22 +527,6 @@ angular ); } getOptions(); - function getQuota() { - $http.get("/api/user/quota").then((res) => { - $scope.quota = res.data; - $scope.quota.storage.percent = $scope.quota.storage.total - ? ($scope.quota.storage.used * 100) / $scope.quota.storage.total - : 100; - $scope.quota.file.percent = $scope.quota.file.total - ? ($scope.quota.file.used * 100) / $scope.quota.file.total - : 100; - $scope.quota.repository.percent = $scope.quota.repository.total - ? ($scope.quota.repository.used * 100) / - $scope.quota.repository.total - : 100; - }, console.error); - } - getQuota(); function getMessage() { $http.get("/api/message").then( @@ -703,6 +686,23 @@ angular }; $scope.orderBy = "-anonymizeDate"; + function getQuota() { + $http.get("/api/user/quota").then((res) => { + $scope.quota = res.data; + $scope.quota.storage.percent = $scope.quota.storage.total + ? ($scope.quota.storage.used * 100) / $scope.quota.storage.total + : 100; + $scope.quota.file.percent = $scope.quota.file.total + ? ($scope.quota.file.used * 100) / $scope.quota.file.total + : 100; + $scope.quota.repository.percent = $scope.quota.repository.total + ? ($scope.quota.repository.used * 100) / + $scope.quota.repository.total + : 100; + }, console.error); + } + getQuota(); + function getRepositories() { $http.get("/api/user/anonymized_repositories").then( (res) => { diff --git a/src/AnonymizedFile.ts b/src/AnonymizedFile.ts index 70d0b47..cb7aaac 100644 --- a/src/AnonymizedFile.ts +++ b/src/AnonymizedFile.ts @@ -1,34 +1,14 @@ import { join, basename } from "path"; import { Response } from "express"; -import { Readable, pipeline } from "stream"; -import { promisify } from "util"; +import { Readable } from "stream"; import Repository from "./Repository"; -import { Tree, TreeElement, TreeFile } from "./types"; +import { TreeElement, TreeFile } from "./types"; import storage from "./storage"; import config from "../config"; import { anonymizePath, anonymizeStream } from "./anonymize-utils"; import AnonymousError from "./AnonymousError"; import { handleError } from "./routes/route-utils"; -function tree2sha( - tree: any, - output: { [key: string]: string } = {}, - parent: string = "" -): { [key: string]: string } { - for (let i in tree) { - const sha = tree[i].sha as string; - const size = tree[i].size as number; - if (sha != null && size != null) { - output[sha] = join(parent, i); - } else if (tree[i].child) { - tree2sha(tree[i].child as Tree, output, join(parent, i)); - } else { - tree2sha(tree[i] as Tree, output, join(parent, i)); - } - } - return output; -} - /** * Represent a file in a anonymized repository */ @@ -70,27 +50,16 @@ export default class AnonymizedFile { }); const paths = this.anonymizedPath.trim().split("/"); - - let currentAnonymized: TreeElement = await this.repository.anonymizedFiles({ - includeSha: true, - }); - let currentOriginal: TreeElement = await this.repository.files(); + let currentOriginal = (await this.repository.files({ + force: false, + })) as TreeElement; let currentOriginalPath = ""; - let isAmbiguous = false; for (let i = 0; i < paths.length; i++) { const fileName = paths[i]; if (fileName == "") { continue; } - if (!currentAnonymized[fileName]) { - throw new AnonymousError("file_not_found", { - object: this, - httpStatus: 404, - }); - } - currentAnonymized = currentAnonymized[fileName]; - - if (!isAmbiguous && !currentOriginal[fileName]) { + if (!currentOriginal[fileName]) { // anonymize all the file in the folder and check if there is one that match the current filename const options = []; for (let originalFileName in currentOriginal) { @@ -106,40 +75,55 @@ export default class AnonymizedFile { if (options.length == 1) { currentOriginalPath = join(currentOriginalPath, options[0]); currentOriginal = currentOriginal[options[0]]; + } else if (options.length == 0) { + throw new AnonymousError("file_not_found", { + object: this, + httpStatus: 404, + }); } else { - isAmbiguous = true; + const nextName = paths[i + 1]; + if (!nextName) { + // if there is no next name we can't find the file and we return the first option + currentOriginalPath = join(currentOriginalPath, options[0]); + currentOriginal = currentOriginal[options[0]]; + } + let found = false; + for (const option of options) { + const optionTree = currentOriginal[option]; + if (optionTree.child) { + const optionTreeChild = optionTree.child; + if (optionTreeChild[nextName]) { + currentOriginalPath = join(currentOriginalPath, option); + currentOriginal = optionTreeChild; + found = true; + break; + } + } + } + if (!found) { + // if we didn't find the next name we return the first option + currentOriginalPath = join(currentOriginalPath, options[0]); + currentOriginal = currentOriginal[options[0]]; + } } - } else if (!isAmbiguous) { + } else { currentOriginalPath = join(currentOriginalPath, fileName); currentOriginal = currentOriginal[fileName]; } } if ( - currentAnonymized.sha === undefined || - currentAnonymized.size === undefined + currentOriginal.sha === undefined || + currentOriginal.size === undefined ) { throw new AnonymousError("folder_not_supported", { object: this }); } - const file: TreeFile = currentAnonymized as TreeFile; + const file = currentOriginal as TreeFile; this.fileSize = file.size; this._sha = file.sha; - if (isAmbiguous) { - // it should never happen - const shaTree = tree2sha(currentOriginal); - if (!currentAnonymized.sha || !shaTree[file.sha]) { - throw new AnonymousError("file_not_found", { - object: this, - httpStatus: 404, - }); - } - - this._originalPath = join(currentOriginalPath, shaTree[file.sha]); - } else { - this._originalPath = currentOriginalPath; - } + this._originalPath = currentOriginalPath; return this._originalPath; } extension() { @@ -193,8 +177,7 @@ export default class AnonymizedFile { } async anonymizedContent() { - const rs = await this.content(); - return rs.pipe(anonymizeStream(this)); + return (await this.content()).pipe(anonymizeStream(this)); } get originalCachePath() { @@ -218,14 +201,24 @@ export default class AnonymizedFile { } async send(res: Response): Promise { - const pipe = promisify(pipeline); - try { - if (this.extension()) { - res.contentType(this.extension()); - } - await pipe(await this.anonymizedContent(), res); - } catch (error) { - handleError(error, res); + if (this.extension()) { + res.contentType(this.extension()); } + if (this.fileSize) { + res.set("Content-Length", this.fileSize.toString()); + } + return new Promise(async (resolve, reject) => { + try { + (await this.anonymizedContent()) + .pipe(res) + .on("close", () => resolve()) + .on("error", (error) => { + reject(error); + handleError(error, res); + }); + } catch (error) { + handleError(error, res); + } + }); } } diff --git a/src/PullRequest.ts b/src/PullRequest.ts index 280c452..dc59383 100644 --- a/src/PullRequest.ts +++ b/src/PullRequest.ts @@ -135,10 +135,10 @@ export default class PullRequest { * @returns void */ async anonymize() { - if (this.status == "ready") return; - await this.updateStatus("preparing"); + if (this.status === RepositoryStatus.READY) return; + await this.updateStatus(RepositoryStatus.PREPARING); await this.updateIfNeeded({ force: true }); - return this.updateStatus("ready"); + return this.updateStatus(RepositoryStatus.READY); } /** @@ -166,18 +166,18 @@ export default class PullRequest { * Expire the pullRequest */ async expire() { - await this.updateStatus("expiring"); + await this.updateStatus(RepositoryStatus.EXPIRING); await this.resetSate(); - await this.updateStatus("expired"); + await this.updateStatus(RepositoryStatus.EXPIRED); } /** * Remove the pullRequest */ async remove() { - await this.updateStatus("removing"); + await this.updateStatus(RepositoryStatus.REMOVING); await this.resetSate(); - await this.updateStatus("removed"); + await this.updateStatus(RepositoryStatus.REMOVED); } /** diff --git a/src/Repository.ts b/src/Repository.ts index cc7f85d..7efc953 100644 --- a/src/Repository.ts +++ b/src/Repository.ts @@ -17,6 +17,29 @@ import AnonymousError from "./AnonymousError"; import { downloadQueue } from "./queue"; import { isConnected } from "./database/database"; import AnonymizedFile from "./AnonymizedFile"; +import AnonymizedRepositoryModel from "./database/anonymizedRepositories/anonymizedRepositories.model"; + +function anonymizeTreeRecursive( + tree: TreeElement, + terms: string[], + opt: { + /** Include the file sha in the response */ + includeSha: boolean; + } = { + includeSha: false, + } +): TreeElement { + if (typeof tree.size !== "object" && tree.sha !== undefined) { + if (opt?.includeSha) return tree as TreeFile; + return { size: tree.size } as TreeFile; + } + const output: Tree = {}; + Object.getOwnPropertyNames(tree).forEach((file) => { + const anonymizedPath = anonymizePath(file, terms); + output[anonymizedPath] = anonymizeTreeRecursive(tree[file], terms, opt); + }); + return output; +} export default class Repository { private _model: IAnonymizedRepositoryDocument; @@ -61,21 +84,7 @@ export default class Repository { } ): Promise { const terms = this._model.options.terms || []; - - function anonymizeTreeRecursive(tree: TreeElement): TreeElement { - if (Number.isInteger(tree.size) && tree.sha !== undefined) { - if (opt?.includeSha) return tree as TreeFile; - return { size: tree.size } as TreeFile; - } - const output: Tree = {}; - for (const file in tree) { - const anonymizedPath = anonymizePath(file, terms); - output[anonymizedPath] = anonymizeTreeRecursive(tree[file]); - } - return output; - } - - return anonymizeTreeRecursive(await this.files(opt)) as Tree; + return anonymizeTreeRecursive(await this.files(opt), terms, opt) as Tree; } /** @@ -85,9 +94,15 @@ export default class Repository { * @returns The file tree */ async files(opt: { force?: boolean } = { force: false }): Promise { + if (!this._model.originalFiles && !opt.force) { + const res = await AnonymizedRepositoryModel.findById(this._model._id, { + originalFiles: 1, + }); + this.model.originalFiles = res.originalFiles; + } if ( this._model.originalFiles && - Object.keys(this._model.originalFiles).length !== 0 && + this._model.size.file !== 0 && !opt.force ) { return this._model.originalFiles; @@ -185,7 +200,7 @@ export default class Repository { console.error( `${branch.name} for ${this.source.githubRepository.fullName} is not found` ); - await this.updateStatus("error", "branch_not_found"); + await this.updateStatus(RepositoryStatus.ERROR, "branch_not_found"); await this.resetSate(); throw new AnonymousError("branch_not_found", { object: this, @@ -193,7 +208,7 @@ export default class Repository { } this._model.anonymizeDate = new Date(); console.log(`${this._model.repoId} will be updated to ${newCommit}`); - await this.resetSate("preparing"); + await this.resetSate(RepositoryStatus.PREPARING); await downloadQueue.add(this.repoId, this, { jobId: this.repoId, attempts: 3, @@ -207,19 +222,19 @@ export default class Repository { * @returns void */ async anonymize() { - if (this.status == "ready") return; - await this.updateStatus("preparing"); + if (this.status === RepositoryStatus.READY) return; + await this.updateStatus(RepositoryStatus.PREPARING); await this.files(); - return this.updateStatus("ready"); + return this.updateStatus(RepositoryStatus.READY); } /** * Update the last view and view count */ async countView() { - if (!isConnected) return this.model; this._model.lastView = new Date(); this._model.pageView = (this._model.pageView || 0) + 1; + if (!isConnected) return this.model; return this._model.save(); } @@ -241,18 +256,18 @@ export default class Repository { * Expire the repository */ async expire() { - await this.updateStatus("expiring"); + await this.updateStatus(RepositoryStatus.EXPIRING); await this.resetSate(); - await this.updateStatus("expired"); + await this.updateStatus(RepositoryStatus.EXPIRED); } /** * Remove the repository */ async remove() { - await this.updateStatus("removing"); + await this.updateStatus(RepositoryStatus.REMOVING); await this.resetSate(); - await this.updateStatus("removed"); + await this.updateStatus(RepositoryStatus.REMOVED); } /** diff --git a/src/anonymize-utils.ts b/src/anonymize-utils.ts index 78a44b1..944caac 100644 --- a/src/anonymize-utils.ts +++ b/src/anonymize-utils.ts @@ -1,5 +1,4 @@ import config from "../config"; -import Repository from "./Repository"; import GitHubBase from "./source/GitHubBase"; import { isText } from "istextorbinary"; import { basename } from "path"; @@ -60,9 +59,9 @@ export function anonymizeStream(file: AnonymizedFile) { ts._flush = function _flush(cb) { if (chunks.length) { - let data: any = Buffer.concat(chunks, len); + let data = Buffer.concat(chunks, len); if (isText(file.anonymizedPath, data)) { - data = anonymizeContent(data.toString(), file.repository); + data = Buffer.from(anonymizeContent(data.toString(), file.repository)); } this.push(data); diff --git a/src/database/anonymizedRepositories/anonymizedRepositories.schema.ts b/src/database/anonymizedRepositories/anonymizedRepositories.schema.ts index 43f8f23..d09ad46 100644 --- a/src/database/anonymizedRepositories/anonymizedRepositories.schema.ts +++ b/src/database/anonymizedRepositories/anonymizedRepositories.schema.ts @@ -15,7 +15,10 @@ const AnonymizedRepositorySchema = new Schema({ lastView: Date, pageView: Number, accessToken: String, - owner: Schema.Types.ObjectId, + owner: { + type: Schema.Types.ObjectId, + index: true, + }, conference: String, source: { type: { type: String }, diff --git a/src/database/database.ts b/src/database/database.ts index 2edc001..8dcd2df 100644 --- a/src/database/database.ts +++ b/src/database/database.ts @@ -15,6 +15,8 @@ export let isConnected = false; export async function connect() { await mongoose.connect(MONGO_URL + "production", { authSource: "admin", + appName: "Anonymous GitHub Server", + compressors: "zlib", } as ConnectOptions); isConnected = true; diff --git a/src/processes/downloadRepository.ts b/src/processes/downloadRepository.ts index 4b3363a..20572b2 100644 --- a/src/processes/downloadRepository.ts +++ b/src/processes/downloadRepository.ts @@ -2,25 +2,33 @@ import { SandboxedJob } from "bullmq"; import { config } from "dotenv"; config(); import Repository from "../Repository"; +import { getRepository as getRepositoryImport } from "../database/database"; +import { RepositoryStatus } from "../types"; export default async function (job: SandboxedJob) { - const { connect, getRepository } = require("../database/database"); - console.log(`${job.data.repoId} is going to be downloaded`); + const { + connect, + getRepository, + }: { + connect: () => Promise; + getRepository: typeof getRepositoryImport; + } = require("../database/database"); + console.log(`[QUEUE] ${job.data.repoId} is going to be downloaded`); try { await connect(); const repo = await getRepository(job.data.repoId); job.updateProgress({ status: "get_repo" }); - await repo.resetSate("preparing"); + await repo.resetSate(RepositoryStatus.PREPARING, ""); job.updateProgress({ status: "resetSate" }); try { await repo.anonymize(); } catch (error) { - await repo.updateStatus("error", error.message); + await repo.updateStatus(RepositoryStatus.ERROR, error.message); throw error; } } catch (error) { console.error(error); } finally { - console.log(`${job.data.repoId} is downloaded`); + console.log(`[QUEUE] ${job.data.repoId} is downloaded`); } } diff --git a/src/processes/removeCache.ts b/src/processes/removeCache.ts index d0c65d6..3c7531a 100644 --- a/src/processes/removeCache.ts +++ b/src/processes/removeCache.ts @@ -1,8 +1,15 @@ import { SandboxedJob } from "bullmq"; import Repository from "../Repository"; +import { getRepository as getRepositoryImport } from "../database/database"; export default async function (job: SandboxedJob) { - const { connect, getRepository } = require("../database/database"); + const { + connect, + getRepository, + }: { + connect: () => Promise; + getRepository: typeof getRepositoryImport; + } = require("../database/database"); try { await connect(); console.log( diff --git a/src/processes/removeRepository.ts b/src/processes/removeRepository.ts index 030878a..14d4f1c 100644 --- a/src/processes/removeRepository.ts +++ b/src/processes/removeRepository.ts @@ -1,21 +1,30 @@ import { SandboxedJob } from "bullmq"; import Repository from "../Repository"; +import { getRepository as getRepositoryImport } from "../database/database"; +import { RepositoryStatus } from "../types"; export default async function (job: SandboxedJob) { - const { connect, getRepository } = require("../database/database"); + const { + connect, + getRepository, + }: { + connect: () => Promise; + getRepository: typeof getRepositoryImport; + } = require("../database/database"); try { await connect(); - console.log(`${job.data.repoId} is going to be removed`); + console.log(`[QUEUE] ${job.data.repoId} is going to be removed`); const repo = await getRepository(job.data.repoId); + await repo.updateStatus(RepositoryStatus.REMOVING, ""); try { await repo.remove(); } catch (error) { - await repo.updateStatus("error", error.message); + await repo.updateStatus(RepositoryStatus.ERROR, error.message); throw error; } } catch (error) { console.error(error); } finally { - console.log(`${job.data.repoId} is removed`); + console.log(`[QUEUE] ${job.data.repoId} is removed`); } } diff --git a/src/routes/file.ts b/src/routes/file.ts index 92d68fd..97287bd 100644 --- a/src/routes/file.ts +++ b/src/routes/file.ts @@ -14,12 +14,13 @@ router.get( } anonymizedPath = anonymizedPath; - const repo = await getRepo(req, res); + const repo = await getRepo(req, res, { + nocheck: false, + includeFiles: false, + }); if (!repo) return; try { - await repo.countView(); - const f = new AnonymizedFile({ repository: repo, anonymizedPath, @@ -35,7 +36,7 @@ router.get( ); // cache the file for 5min res.header("Cache-Control", "max-age=300"); - await f.send(res); + await Promise.all([repo.countView(), f.send(res)]); } catch (error) { return handleError(error, res, req); } diff --git a/src/routes/repository-private.ts b/src/routes/repository-private.ts index 6403960..e26cfd5 100644 --- a/src/routes/repository-private.ts +++ b/src/routes/repository-private.ts @@ -16,6 +16,7 @@ import AnonymousError from "../AnonymousError"; import { downloadQueue, removeQueue } from "../queue"; import RepositoryModel from "../database/repositories/repositories.model"; import User from "../User"; +import { RepositoryStatus } from "../types"; const router = express.Router(); @@ -150,7 +151,7 @@ router.delete( }); const user = await getUser(req); isOwnerOrAdmin([repo.owner.id], user); - await repo.updateStatus("removing"); + await repo.updateStatus(RepositoryStatus.REMOVING); await removeQueue.add(repo.repoId, repo, { jobId: repo.repoId }); return res.json({ status: repo.status }); } catch (error) { @@ -406,7 +407,7 @@ router.post( } } repo.model.conference = repoUpdate.conference; - await repo.updateStatus("preparing"); + await repo.updateStatus(RepositoryStatus.PREPARING); res.json({ status: repo.status }); await downloadQueue.add(repo.repoId, repo, { jobId: repo.repoId }); } catch (error) { diff --git a/src/routes/repository-public.ts b/src/routes/repository-public.ts index 253b375..a266421 100644 --- a/src/routes/repository-public.ts +++ b/src/routes/repository-public.ts @@ -6,6 +6,7 @@ import config from "../../config"; import { getRepo, handleError } from "./route-utils"; import AnonymousError from "../AnonymousError"; import { downloadQueue } from "../queue"; +import { RepositoryStatus } from "../types"; const router = express.Router(); @@ -61,7 +62,7 @@ router.get( "/:repoId/files", async (req: express.Request, res: express.Response) => { res.header("Cache-Control", "no-cache"); - const repo = await getRepo(req, res); + const repo = await getRepo(req, res, { includeFiles: true }); if (!repo) return; try { res.json(await repo.anonymizedFiles({ includeSha: false })); @@ -76,7 +77,10 @@ router.get( async (req: express.Request, res: express.Response) => { try { res.header("Cache-Control", "no-cache"); - const repo = await getRepo(req, res, { nocheck: true, includeFiles: false }); + const repo = await getRepo(req, res, { + nocheck: true, + includeFiles: false, + }); if (!repo) return; let redirectURL = null; if ( @@ -105,7 +109,7 @@ router.get( repo.model.statusDate < fiveMinuteAgo // && repo.status != "preparing" ) { - await repo.updateStatus("preparing"); + await repo.updateStatus(RepositoryStatus.PREPARING); await downloadQueue.add(repo.repoId, repo, { jobId: repo.repoId, attempts: 3, diff --git a/src/routes/route-utils.ts b/src/routes/route-utils.ts index 5213b62..08155a2 100644 --- a/src/routes/route-utils.ts +++ b/src/routes/route-utils.ts @@ -39,7 +39,7 @@ export async function getRepo( res: express.Response, opt: { nocheck?: boolean; includeFiles?: boolean } = { nocheck: false, - includeFiles: true, + includeFiles: false, } ) { try { @@ -108,7 +108,7 @@ export function handleError( if (error.httpStatus) { status = error.httpStatus; } else if (message && message.indexOf("not_found") > -1) { - status = 400; + status = 404; } else if (message && message.indexOf("not_connected") > -1) { status = 401; } @@ -121,17 +121,14 @@ export function handleError( export async function getUser(req: express.Request) { const user = (req.user as any).user; if (!user) { - req.logout((error) => console.error(error)); + req.logout((error) => { + if (error) { + console.error(`[ERROR] Error while logging out: ${error}`); + } + }); throw new AnonymousError("not_connected", { httpStatus: 401, }); } - const model = await UserModel.findById(user._id); - if (!model) { - req.logout((error) => console.error(error)); - throw new AnonymousError("not_connected", { - httpStatus: 401, - }); - } - return new User(model); + return new User(new UserModel(user)); } diff --git a/src/routes/user.ts b/src/routes/user.ts index e2ee748..387baef 100644 --- a/src/routes/user.ts +++ b/src/routes/user.ts @@ -10,7 +10,11 @@ router.use(ensureAuthenticated); router.get("/logout", async (req: express.Request, res: express.Response) => { try { - req.logout((error) => console.error(error)); + req.logout((error) => { + if (error) { + console.error(`[ERROR] Logout error: ${error}`); + } + }); res.redirect("/"); } catch (error) { handleError(error, res, req); diff --git a/src/source/GitHubDownload.ts b/src/source/GitHubDownload.ts index 9902f74..5f7d609 100644 --- a/src/source/GitHubDownload.ts +++ b/src/source/GitHubDownload.ts @@ -5,7 +5,7 @@ import Repository from "../Repository"; import GitHubBase from "./GitHubBase"; import AnonymizedFile from "../AnonymizedFile"; -import { SourceBase } from "../types"; +import { RepositoryStatus, SourceBase } from "../types"; import got from "got"; import { Readable } from "stream"; import { OctokitResponse } from "@octokit/types"; @@ -60,7 +60,10 @@ export default class GitHubDownload extends GitHubBase implements SourceBase { try { response = await this._getZipUrl(config.GITHUB_TOKEN); } catch (error) { - await this.repository.resetSate("error", "repo_not_accessible"); + await this.repository.resetSate( + RepositoryStatus.ERROR, + "repo_not_accessible" + ); throw new AnonymousError("repo_not_accessible", { httpStatus: 404, cause: error, @@ -68,7 +71,10 @@ export default class GitHubDownload extends GitHubBase implements SourceBase { }); } } else { - await this.repository.resetSate("error", "repo_not_accessible"); + await this.repository.resetSate( + RepositoryStatus.ERROR, + "repo_not_accessible" + ); throw new AnonymousError("repo_not_accessible", { httpStatus: 404, object: this.repository, @@ -76,7 +82,7 @@ export default class GitHubDownload extends GitHubBase implements SourceBase { }); } } - await this.repository.updateStatus("download"); + await this.repository.updateStatus(RepositoryStatus.DOWNLOAD); const originalPath = this.repository.originalCachePath; await storage.mk(originalPath); let progress = null; @@ -102,7 +108,10 @@ export default class GitHubDownload extends GitHubBase implements SourceBase { downloadStream.addListener("downloadProgress", (p) => (progress = p)); await storage.extractZip(originalPath, downloadStream, null, this); } catch (error) { - await this.repository.updateStatus("error", "unable_to_download"); + await this.repository.updateStatus( + RepositoryStatus.ERROR, + "unable_to_download" + ); throw new AnonymousError("unable_to_download", { httpStatus: 500, cause: error, @@ -113,7 +122,7 @@ export default class GitHubDownload extends GitHubBase implements SourceBase { clearTimeout(progressTimeout); } - await this.repository.updateStatus("ready"); + await this.repository.updateStatus(RepositoryStatus.READY); } async getFileContent(file: AnonymizedFile): Promise { diff --git a/src/source/GitHubStream.ts b/src/source/GitHubStream.ts index d079f08..80a5bcb 100644 --- a/src/source/GitHubStream.ts +++ b/src/source/GitHubStream.ts @@ -3,7 +3,7 @@ import AnonymizedFile from "../AnonymizedFile"; import Repository from "../Repository"; import GitHubBase from "./GitHubBase"; import storage from "../storage"; -import { SourceBase, Tree } from "../types"; +import { RepositoryStatus, SourceBase, Tree } from "../types"; import * as path from "path"; import * as stream from "stream"; @@ -26,11 +26,6 @@ export default class GitHubStream extends GitHubBase implements SourceBase { } async getFileContent(file: AnonymizedFile): Promise { - if (!file.sha) - throw new AnonymousError("file_sha_not_provided", { - httpStatus: 400, - object: file, - }); const octokit = new Octokit({ auth: await this.getToken(), }); @@ -57,12 +52,12 @@ export default class GitHubStream extends GitHubBase implements SourceBase { } else { content = Buffer.from(""); } - if (this.repository.status != "ready") - await this.repository.updateStatus("ready"); + if (this.repository.status !== RepositoryStatus.READY) + await this.repository.updateStatus(RepositoryStatus.READY); await storage.write(file.originalCachePath, content, file, this); return stream.Readable.from(content); } catch (error) { - if (error.status == 404) { + if (error.status === 404 || error.httpStatus === 404) { throw new AnonymousError("file_not_found", { httpStatus: error.status, cause: error, @@ -99,15 +94,18 @@ export default class GitHubStream extends GitHubBase implements SourceBase { } catch (error) { if (error.status == 409) { // empty tree - if (this.repository.status != "ready") - await this.repository.updateStatus("ready"); + if (this.repository.status != RepositoryStatus.READY) + await this.repository.updateStatus(RepositoryStatus.READY); // cannot be empty otherwise it would try to download it again return { __: {} }; } else { console.log( `[ERROR] getTree ${this.repository.repoId}@${sha}: ${error.message}` ); - await this.repository.resetSate("error", "repo_not_accessible"); + await this.repository.resetSate( + RepositoryStatus.ERROR, + "repo_not_accessible" + ); throw new AnonymousError("repo_not_accessible", { httpStatus: error.status, cause: error, @@ -124,8 +122,8 @@ export default class GitHubStream extends GitHubBase implements SourceBase { if (ghRes.truncated) { await this.getTruncatedTree(sha, tree, parentPath, count); } - if (this.repository.status != "ready") - await this.repository.updateStatus("ready"); + if (this.repository.status !== RepositoryStatus.READY) + await this.repository.updateStatus(RepositoryStatus.READY); return tree; } diff --git a/src/storage/S3.ts b/src/storage/S3.ts index cc050d5..5326d95 100644 --- a/src/storage/S3.ts +++ b/src/storage/S3.ts @@ -29,7 +29,7 @@ export default class S3Storage implements StorageBase { secretAccessKey: config.S3_CLIENT_SECRET, httpOptions: { timeout: 1000 * 60 * 60 * 2, // 2 hour - } + }, }); } @@ -106,10 +106,7 @@ export default class S3Storage implements StorageBase { res.set("Content-Length", headers["content-length"]); res.set("Content-Type", headers["content-type"]); } - pipeline( - response.httpResponse.createUnbufferedStream() as Readable, - res - ); + (response.httpResponse.createUnbufferedStream() as Readable).pipe(res); }); s.send(); @@ -139,7 +136,7 @@ export default class S3Storage implements StorageBase { ContentType: lookup(path).toString(), }; if (source) { - params.Tagging = `source=${source.type}` + params.Tagging = `source=${source.type}`; } await this.client.putObject(params).promise(); return; diff --git a/src/types.ts b/src/types.ts index a866bc4..fac039b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -6,6 +6,7 @@ import FileSystem from "./storage/FileSystem"; import AnonymizedFile from "./AnonymizedFile"; import * as stream from "stream"; import * as archiver from "archiver"; +import { Response } from "express"; export interface SourceBase { readonly type: string; @@ -43,6 +44,8 @@ export interface StorageBase { */ exists(path: string): Promise; + send(p: string, res: Response): void; + /** * Read the content of a file * @param path the path to the file @@ -56,7 +59,12 @@ export interface StorageBase { * @param file the file * @param source the source of the file */ - write(path: string, data: Buffer, file?: AnonymizedFile, source?: SourceBase): Promise; + write( + path: string, + data: Buffer, + file?: AnonymizedFile, + source?: SourceBase + ): Promise; /** * List the files from dir @@ -71,7 +79,12 @@ export interface StorageBase { * @param file the file * @param source the source of the file */ - extractZip(dir: string, tar: stream.Readable, file?: AnonymizedFile, source?: SourceBase): Promise; + extractZip( + dir: string, + tar: stream.Readable, + file?: AnonymizedFile, + source?: SourceBase + ): Promise; /** * Remove the path @@ -113,16 +126,17 @@ export interface Branch { readme?: string; } -export type RepositoryStatus = - | "queue" - | "preparing" - | "download" - | "ready" - | "expired" - | "expiring" - | "removed" - | "removing" - | "error"; +export enum RepositoryStatus { + QUEUE = "queue", + PREPARING = "preparing", + DOWNLOAD = "download", + READY = "ready", + EXPIRED = "expired", + EXPIRING = "expiring", + REMOVED = "removed", + REMOVING = "removing", + ERROR = "error", +} export type ConferenceStatus = "ready" | "expired" | "removed";