diff --git a/package-lock.json b/package-lock.json index d628ffd..093d841 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,18 +12,17 @@ "@octokit/oauth-app": "^3.3.2", "@octokit/rest": "^18.5.3", "@pm2/io": "^5.0.0", - "archive-stream-to-s3": "^1.1.3", "archiver": "^5.3.0", "aws-sdk": "^2.958.0", "bull": "^3.29.2", "compression": "^1.7.4", "connect-redis": "^6.0.0", + "decompress-stream-to-s3": "^1.2.0", "dotenv": "^10.0.0", "express": "^4.17.1", "express-rate-limit": "^5.2.6", "express-session": "^1.17.1", "got": "^11.8.2", - "gunzip-maybe": "^1.4.2", "istextorbinary": "^6.0.0", "mime-types": "^2.1.30", "mongoose": "^5.13.5", @@ -33,9 +32,9 @@ "passport-github2": "^0.1.12", "rate-limit-redis": "^2.1.0", "redis": "^3.1.2", - "tar-fs": "^2.1.1", "textextensions": "^5.12.0", "ts-custom-error": "^3.2.0", + "unzip-stream": "^0.3.1", "xml-flow": "^1.0.4" }, "devDependencies": { @@ -54,6 +53,7 @@ "@types/rate-limit-redis": "^1.7.2", "@types/redis": "^2.8.31", "@types/tar-fs": "^2.0.1", + "@types/unzip-stream": "^0.3.1", "@types/xml-flow": "^1.0.1", "chai": "^4.3.4", "mocha": "^9.0.3", @@ -752,6 +752,15 @@ "integrity": "sha512-Y0K95ThC3esLEYD6ZuqNek29lNX2EM1qxV8y2FTLUB0ff5wWrk7az+mLrnNFUnaXcgKye22+sFBRXOgpPILZNg==", "dev": true }, + "node_modules/@types/unzip-stream": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@types/unzip-stream/-/unzip-stream-0.3.1.tgz", + "integrity": "sha512-RlE3qaqvu4XaMwxkG/zR1gIunCbqXvNrmZ4BCG7OiQ8QUactFUPxm0TTrOCRJZQfPW3T6XBH7PcHQiiqkdcijw==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/xml-flow": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@types/xml-flow/-/xml-flow-1.0.1.tgz", @@ -911,21 +920,6 @@ "node": ">= 8" } }, - "node_modules/archive-stream-to-s3": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/archive-stream-to-s3/-/archive-stream-to-s3-1.1.3.tgz", - "integrity": "sha512-XTUK2fLCAFPrCu9Y0X0u1vLrM5uh/cRbmeBNwUz/X4KeoImcgCexYmOoHAswoGivbm4Eh99eBKOpbIsBa06ZTg==", - "dependencies": { - "aws-sdk": "^2.270.1", - "debug": "^3.1.0", - "gunzip-maybe": "^1.4.1", - "mime-types": "^2.1.18", - "tar-stream": "^1.6.1" - }, - "engines": { - "node": ">=8.10.0" - } - }, "node_modules/archiver": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/archiver/-/archiver-5.3.0.tgz", @@ -985,54 +979,6 @@ "safe-buffer": "~5.1.0" } }, - "node_modules/archiver/node_modules/bl": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", - "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "dependencies": { - "buffer": "^5.5.0", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, - "node_modules/archiver/node_modules/buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, - "node_modules/archiver/node_modules/tar-stream": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", - "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", - "dependencies": { - "bl": "^4.0.3", - "end-of-stream": "^1.4.1", - "fs-constants": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/arg": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", @@ -1139,6 +1085,18 @@ "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.2.tgz", "integrity": "sha512-3pZEU3NT5BFUo/AD5ERPWOgQOCZITni6iavr5AUw5AUwQjMlI0kzu5btnyD39AF0gUEsDPwJT+oY1ORBJijPjQ==" }, + "node_modules/binary": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", + "integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=", + "dependencies": { + "buffers": "~0.1.1", + "chainsaw": "~0.1.0" + }, + "engines": { + "node": "*" + } + }, "node_modules/binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", @@ -1341,20 +1299,6 @@ "isarray": "^1.0.0" } }, - "node_modules/buffer-alloc": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", - "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", - "dependencies": { - "buffer-alloc-unsafe": "^1.1.0", - "buffer-fill": "^1.0.0" - } - }, - "node_modules/buffer-alloc-unsafe": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz", - "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==" - }, "node_modules/buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", @@ -1363,16 +1307,19 @@ "node": "*" } }, - "node_modules/buffer-fill": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", - "integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw=" - }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, + "node_modules/buffers": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", + "integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s=", + "engines": { + "node": ">=0.2.0" + } + }, "node_modules/bull": { "version": "3.29.2", "resolved": "https://registry.npmjs.org/bull/-/bull-3.29.2.tgz", @@ -1498,6 +1445,17 @@ "node": ">=4" } }, + "node_modules/chainsaw": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", + "integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=", + "dependencies": { + "traverse": ">=0.3.0 <0.4" + }, + "engines": { + "node": "*" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -1556,11 +1514,6 @@ "fsevents": "~2.3.2" } }, - "node_modules/chownr": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" - }, "node_modules/ci-info": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz", @@ -1833,6 +1786,7 @@ "version": "3.2.7", "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, "dependencies": { "ms": "^2.1.1" } @@ -1882,6 +1836,43 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/decompress-stream-to-s3": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decompress-stream-to-s3/-/decompress-stream-to-s3-1.2.0.tgz", + "integrity": "sha512-jYL8AA+G3BDXhEV9+b4AKUqOQXfFI8Pv8Pfh6tj7RQvSecm++M+fa6b5t6E10WO4siFY46FlkJbLrebeZvJB/A==", + "dependencies": { + "aws-sdk": "^2.958.0", + "debug": "^4.3.2", + "gunzip-maybe": "^1.4.1", + "mime-types": "^2.1.18", + "tar-stream": "^2.2.0", + "unzip-stream": "^0.3.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/decompress-stream-to-s3/node_modules/debug": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decompress-stream-to-s3/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, "node_modules/deep-eql": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz", @@ -3491,13 +3482,18 @@ "node_modules/minimist": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", - "dev": true + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" }, - "node_modules/mkdirp-classic": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", - "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + "node_modules/mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dependencies": { + "minimist": "^1.2.5" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } }, "node_modules/mocha": { "version": "9.0.3", @@ -3726,7 +3722,8 @@ "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true }, "node_modules/nanoid": { "version": "3.1.23", @@ -4994,18 +4991,22 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, - "node_modules/tar-fs": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", - "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", "dependencies": { - "chownr": "^1.1.1", - "mkdirp-classic": "^0.5.2", - "pump": "^3.0.0", - "tar-stream": "^2.1.4" + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" } }, - "node_modules/tar-fs/node_modules/bl": { + "node_modules/tar-stream/node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", @@ -5015,7 +5016,7 @@ "readable-stream": "^3.4.0" } }, - "node_modules/tar-fs/node_modules/buffer": { + "node_modules/tar-stream/node_modules/buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", @@ -5038,69 +5039,6 @@ "ieee754": "^1.1.13" } }, - "node_modules/tar-fs/node_modules/tar-stream": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", - "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", - "dependencies": { - "bl": "^4.0.3", - "end-of-stream": "^1.4.1", - "fs-constants": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/tar-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz", - "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==", - "dependencies": { - "bl": "^1.0.0", - "buffer-alloc": "^1.2.0", - "end-of-stream": "^1.0.0", - "fs-constants": "^1.0.0", - "readable-stream": "^2.3.0", - "to-buffer": "^1.1.1", - "xtend": "^4.0.0" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/tar-stream/node_modules/bl": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.3.tgz", - "integrity": "sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==", - "dependencies": { - "readable-stream": "^2.3.5", - "safe-buffer": "^5.1.1" - } - }, - "node_modules/tar-stream/node_modules/readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/tar-stream/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, "node_modules/term-size": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/term-size/-/term-size-2.2.1.tgz", @@ -5155,11 +5093,6 @@ "safe-buffer": "~5.1.0" } }, - "node_modules/to-buffer": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", - "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==" - }, "node_modules/to-readable-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-1.0.0.tgz", @@ -5201,6 +5134,14 @@ "nodetouch": "bin/nodetouch.js" } }, + "node_modules/traverse": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", + "integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk=", + "engines": { + "node": "*" + } + }, "node_modules/ts-custom-error": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/ts-custom-error/-/ts-custom-error-3.2.0.tgz", @@ -5398,6 +5339,15 @@ "node": ">= 0.8" } }, + "node_modules/unzip-stream": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/unzip-stream/-/unzip-stream-0.3.1.tgz", + "integrity": "sha512-RzaGXLNt+CW+T41h1zl6pGz3EaeVhYlK+rdAap+7DxW5kqsqePO8kRtWPaCiVqdhZc86EctSPVYNix30YOMzmw==", + "dependencies": { + "binary": "^0.3.0", + "mkdirp": "^0.5.1" + } + }, "node_modules/update-notifier": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-4.1.3.tgz", @@ -6446,6 +6396,15 @@ "integrity": "sha512-Y0K95ThC3esLEYD6ZuqNek29lNX2EM1qxV8y2FTLUB0ff5wWrk7az+mLrnNFUnaXcgKye22+sFBRXOgpPILZNg==", "dev": true }, + "@types/unzip-stream": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@types/unzip-stream/-/unzip-stream-0.3.1.tgz", + "integrity": "sha512-RlE3qaqvu4XaMwxkG/zR1gIunCbqXvNrmZ4BCG7OiQ8QUactFUPxm0TTrOCRJZQfPW3T6XBH7PcHQiiqkdcijw==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/xml-flow": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@types/xml-flow/-/xml-flow-1.0.1.tgz", @@ -6568,18 +6527,6 @@ "picomatch": "^2.0.4" } }, - "archive-stream-to-s3": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/archive-stream-to-s3/-/archive-stream-to-s3-1.1.3.tgz", - "integrity": "sha512-XTUK2fLCAFPrCu9Y0X0u1vLrM5uh/cRbmeBNwUz/X4KeoImcgCexYmOoHAswoGivbm4Eh99eBKOpbIsBa06ZTg==", - "requires": { - "aws-sdk": "^2.270.1", - "debug": "^3.1.0", - "gunzip-maybe": "^1.4.1", - "mime-types": "^2.1.18", - "tar-stream": "^1.6.1" - } - }, "archiver": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/archiver/-/archiver-5.3.0.tgz", @@ -6592,39 +6539,6 @@ "readdir-glob": "^1.0.0", "tar-stream": "^2.2.0", "zip-stream": "^4.1.0" - }, - "dependencies": { - "bl": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", - "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "requires": { - "buffer": "^5.5.0", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, - "buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "requires": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, - "tar-stream": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", - "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", - "requires": { - "bl": "^4.0.3", - "end-of-stream": "^1.4.1", - "fs-constants": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1" - } - } } }, "archiver-utils": { @@ -6747,6 +6661,15 @@ "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.2.tgz", "integrity": "sha512-3pZEU3NT5BFUo/AD5ERPWOgQOCZITni6iavr5AUw5AUwQjMlI0kzu5btnyD39AF0gUEsDPwJT+oY1ORBJijPjQ==" }, + "binary": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", + "integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=", + "requires": { + "buffers": "~0.1.1", + "chainsaw": "~0.1.0" + } + }, "binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", @@ -6922,35 +6845,21 @@ "isarray": "^1.0.0" } }, - "buffer-alloc": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", - "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", - "requires": { - "buffer-alloc-unsafe": "^1.1.0", - "buffer-fill": "^1.0.0" - } - }, - "buffer-alloc-unsafe": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz", - "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==" - }, "buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" }, - "buffer-fill": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", - "integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw=" - }, "buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, + "buffers": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", + "integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s=" + }, "bull": { "version": "3.29.2", "resolved": "https://registry.npmjs.org/bull/-/bull-3.29.2.tgz", @@ -7045,6 +6954,14 @@ "type-detect": "^4.0.5" } }, + "chainsaw": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", + "integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=", + "requires": { + "traverse": ">=0.3.0 <0.4" + } + }, "chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -7088,11 +7005,6 @@ "readdirp": "~3.6.0" } }, - "chownr": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" - }, "ci-info": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz", @@ -7310,6 +7222,7 @@ "version": "3.2.7", "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, "requires": { "ms": "^2.1.1" } @@ -7340,6 +7253,34 @@ } } }, + "decompress-stream-to-s3": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decompress-stream-to-s3/-/decompress-stream-to-s3-1.2.0.tgz", + "integrity": "sha512-jYL8AA+G3BDXhEV9+b4AKUqOQXfFI8Pv8Pfh6tj7RQvSecm++M+fa6b5t6E10WO4siFY46FlkJbLrebeZvJB/A==", + "requires": { + "aws-sdk": "^2.958.0", + "debug": "^4.3.2", + "gunzip-maybe": "^1.4.1", + "mime-types": "^2.1.18", + "tar-stream": "^2.2.0", + "unzip-stream": "^0.3.1" + }, + "dependencies": { + "debug": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "deep-eql": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz", @@ -8528,13 +8469,15 @@ "minimist": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", - "dev": true + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" }, - "mkdirp-classic": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", - "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "requires": { + "minimist": "^1.2.5" + } }, "mocha": { "version": "9.0.3", @@ -8692,7 +8635,8 @@ "ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true }, "nanoid": { "version": "3.1.23", @@ -9657,15 +9601,16 @@ "has-flag": "^4.0.0" } }, - "tar-fs": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", - "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", "requires": { - "chownr": "^1.1.1", - "mkdirp-classic": "^0.5.2", - "pump": "^3.0.0", - "tar-stream": "^2.1.4" + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" }, "dependencies": { "bl": { @@ -9686,65 +9631,6 @@ "base64-js": "^1.3.1", "ieee754": "^1.1.13" } - }, - "tar-stream": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", - "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", - "requires": { - "bl": "^4.0.3", - "end-of-stream": "^1.4.1", - "fs-constants": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1" - } - } - } - }, - "tar-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz", - "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==", - "requires": { - "bl": "^1.0.0", - "buffer-alloc": "^1.2.0", - "end-of-stream": "^1.0.0", - "fs-constants": "^1.0.0", - "readable-stream": "^2.3.0", - "to-buffer": "^1.1.1", - "xtend": "^4.0.0" - }, - "dependencies": { - "bl": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.3.tgz", - "integrity": "sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==", - "requires": { - "readable-stream": "^2.3.5", - "safe-buffer": "^5.1.1" - } - }, - "readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } } } }, @@ -9792,11 +9678,6 @@ } } }, - "to-buffer": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", - "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==" - }, "to-readable-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-1.0.0.tgz", @@ -9826,6 +9707,11 @@ "nopt": "~1.0.10" } }, + "traverse": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", + "integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk=" + }, "ts-custom-error": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/ts-custom-error/-/ts-custom-error-3.2.0.tgz", @@ -9969,6 +9855,15 @@ "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" }, + "unzip-stream": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/unzip-stream/-/unzip-stream-0.3.1.tgz", + "integrity": "sha512-RzaGXLNt+CW+T41h1zl6pGz3EaeVhYlK+rdAap+7DxW5kqsqePO8kRtWPaCiVqdhZc86EctSPVYNix30YOMzmw==", + "requires": { + "binary": "^0.3.0", + "mkdirp": "^0.5.1" + } + }, "update-notifier": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-4.1.3.tgz", diff --git a/package.json b/package.json index 346e44b..2d3f978 100644 --- a/package.json +++ b/package.json @@ -27,18 +27,17 @@ "@octokit/oauth-app": "^3.3.2", "@octokit/rest": "^18.5.3", "@pm2/io": "^5.0.0", - "archive-stream-to-s3": "^1.1.3", "archiver": "^5.3.0", "aws-sdk": "^2.958.0", "bull": "^3.29.2", "compression": "^1.7.4", "connect-redis": "^6.0.0", + "decompress-stream-to-s3": "^1.2.0", "dotenv": "^10.0.0", "express": "^4.17.1", "express-rate-limit": "^5.2.6", "express-session": "^1.17.1", "got": "^11.8.2", - "gunzip-maybe": "^1.4.2", "istextorbinary": "^6.0.0", "mime-types": "^2.1.30", "mongoose": "^5.13.5", @@ -48,9 +47,9 @@ "passport-github2": "^0.1.12", "rate-limit-redis": "^2.1.0", "redis": "^3.1.2", - "tar-fs": "^2.1.1", "textextensions": "^5.12.0", "ts-custom-error": "^3.2.0", + "unzip-stream": "^0.3.1", "xml-flow": "^1.0.4" }, "devDependencies": { @@ -69,6 +68,7 @@ "@types/rate-limit-redis": "^1.7.2", "@types/redis": "^2.8.31", "@types/tar-fs": "^2.0.1", + "@types/unzip-stream": "^0.3.1", "@types/xml-flow": "^1.0.1", "chai": "^4.3.4", "mocha": "^9.0.3", diff --git a/src/source/GitHubDownload.ts b/src/source/GitHubDownload.ts index 4e458f4..ee8374f 100644 --- a/src/source/GitHubDownload.ts +++ b/src/source/GitHubDownload.ts @@ -30,7 +30,7 @@ export default class GitHubDownload extends GitHubBase implements SourceBase { auth?: string ): Promise> { const octokit = new Octokit({ auth }); - return octokit.rest.repos.downloadTarballArchive({ + return octokit.rest.repos.downloadZipballArchive({ owner: this.githubRepository.owner, repo: this.githubRepository.repo, ref: this.branch?.commit || "HEAD", @@ -97,7 +97,7 @@ export default class GitHubDownload extends GitHubBase implements SourceBase { try { const downloadStream = got.stream(response.url); downloadStream.addListener("downloadProgress", (p) => (progress = p)); - await storage.extractTar(originalPath, downloadStream); + await storage.extractZip(originalPath, downloadStream); } catch (error) { await this.repository.updateStatus("error", "unable_to_download"); throw new AnonymousError("unable_to_download", { diff --git a/src/storage/FileSystem.ts b/src/storage/FileSystem.ts index 24a008f..cc60118 100644 --- a/src/storage/FileSystem.ts +++ b/src/storage/FileSystem.ts @@ -2,11 +2,10 @@ import { StorageBase, Tree } from "../types"; import config from "../../config"; import * as fs from "fs"; -import * as tar from "tar-fs"; +import * as unzip from "unzip-stream"; import * as path from "path"; import * as express from "express"; import * as stream from "stream"; -import * as gunzip from "gunzip-maybe"; import * as archiver from "archiver"; import { promisify } from "util"; @@ -93,15 +92,17 @@ export default class FileSystem implements StorageBase { } /** @override */ - async extractTar(p: string, data: stream.Readable): Promise { + async extractZip(p: string, data: stream.Readable): Promise { const pipeline = promisify(stream.pipeline); return pipeline( data, - gunzip(), - tar.extract(path.join(config.FOLDER, p), { - map: (header) => { - header.name = header.name.substr(header.name.indexOf("/") + 1); - return header; + unzip.Extract({ + path: path.join(path.join(config.FOLDER, p)), + decodeString: (buf) => { + const name = buf.toString(); + const newName = name.substr(name.indexOf("/") + 1); + if (newName == "") return "/dev/null"; + return newName; }, }) ); diff --git a/src/storage/S3.ts b/src/storage/S3.ts index a1c6eb8..a3e5cfb 100644 --- a/src/storage/S3.ts +++ b/src/storage/S3.ts @@ -2,18 +2,14 @@ import { StorageBase, Tree, TreeFile } from "../types"; import { S3 } from "aws-sdk"; import config from "../../config"; import * as stream from "stream"; -import { ArchiveStreamToS3 } from "archive-stream-to-s3"; +import ArchiveStreamToS3 from "decompress-stream-to-s3"; import * as express from "express"; import * as mime from "mime-types"; import * as flow from "xml-flow"; import * as archiver from "archiver"; import * as path from "path"; -import * as gunzip from "gunzip-maybe"; import AnonymousError from "../AnonymousError"; -const originalArchiveStreamToS3Entry: Function = (ArchiveStreamToS3 as any) - .prototype.onEntry; - export default class S3Storage implements StorageBase { type = "AWS"; @@ -172,22 +168,21 @@ export default class S3Storage implements StorageBase { } /** @override */ - async extractTar(p: string, data: stream.Readable): Promise { + async extractZip(p: string, data: stream.Readable): Promise { let toS3: ArchiveStreamToS3; - (ArchiveStreamToS3 as any).prototype.onEntry = function ( - header: any, - stream: any, - next: any - ) { - header.name = header.name.substr(header.name.indexOf("/") + 1); - originalArchiveStreamToS3Entry.call(toS3, header, stream, next); - }; - return new Promise((resolve, reject) => { - toS3 = new ArchiveStreamToS3(config.S3_BUCKET, p, this.client); + toS3 = new ArchiveStreamToS3({ + bucket: config.S3_BUCKET, + prefix: p, + s3: this.client, + type: "zip", + onEntry: (header) => { + header.name = header.name.substr(header.name.indexOf("/") + 1); + }, + }); stream - .pipeline(data, gunzip(), toS3, () => {}) + .pipeline(data, toS3, () => {}) .on("finish", resolve) .on("error", reject); }); diff --git a/src/types.ts b/src/types.ts index 671f210..69758b6 100644 --- a/src/types.ts +++ b/src/types.ts @@ -67,7 +67,7 @@ export interface StorageBase { * @param dir * @param tar */ - extractTar(dir: string, tar: stream.Readable): Promise; + extractZip(dir: string, tar: stream.Readable): Promise; /** * Remove the path