feat: download zip instead of tar

This commit is contained in:
tdurieux
2021-09-12 00:06:17 +02:00
parent bafa0b325b
commit 486ad634ca
6 changed files with 245 additions and 354 deletions

541
package-lock.json generated
View File

@@ -12,18 +12,17 @@
"@octokit/oauth-app": "^3.3.2",
"@octokit/rest": "^18.5.3",
"@pm2/io": "^5.0.0",
"archive-stream-to-s3": "^1.1.3",
"archiver": "^5.3.0",
"aws-sdk": "^2.958.0",
"bull": "^3.29.2",
"compression": "^1.7.4",
"connect-redis": "^6.0.0",
"decompress-stream-to-s3": "^1.2.0",
"dotenv": "^10.0.0",
"express": "^4.17.1",
"express-rate-limit": "^5.2.6",
"express-session": "^1.17.1",
"got": "^11.8.2",
"gunzip-maybe": "^1.4.2",
"istextorbinary": "^6.0.0",
"mime-types": "^2.1.30",
"mongoose": "^5.13.5",
@@ -33,9 +32,9 @@
"passport-github2": "^0.1.12",
"rate-limit-redis": "^2.1.0",
"redis": "^3.1.2",
"tar-fs": "^2.1.1",
"textextensions": "^5.12.0",
"ts-custom-error": "^3.2.0",
"unzip-stream": "^0.3.1",
"xml-flow": "^1.0.4"
},
"devDependencies": {
@@ -54,6 +53,7 @@
"@types/rate-limit-redis": "^1.7.2",
"@types/redis": "^2.8.31",
"@types/tar-fs": "^2.0.1",
"@types/unzip-stream": "^0.3.1",
"@types/xml-flow": "^1.0.1",
"chai": "^4.3.4",
"mocha": "^9.0.3",
@@ -752,6 +752,15 @@
"integrity": "sha512-Y0K95ThC3esLEYD6ZuqNek29lNX2EM1qxV8y2FTLUB0ff5wWrk7az+mLrnNFUnaXcgKye22+sFBRXOgpPILZNg==",
"dev": true
},
"node_modules/@types/unzip-stream": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/@types/unzip-stream/-/unzip-stream-0.3.1.tgz",
"integrity": "sha512-RlE3qaqvu4XaMwxkG/zR1gIunCbqXvNrmZ4BCG7OiQ8QUactFUPxm0TTrOCRJZQfPW3T6XBH7PcHQiiqkdcijw==",
"dev": true,
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/xml-flow": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@types/xml-flow/-/xml-flow-1.0.1.tgz",
@@ -911,21 +920,6 @@
"node": ">= 8"
}
},
"node_modules/archive-stream-to-s3": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/archive-stream-to-s3/-/archive-stream-to-s3-1.1.3.tgz",
"integrity": "sha512-XTUK2fLCAFPrCu9Y0X0u1vLrM5uh/cRbmeBNwUz/X4KeoImcgCexYmOoHAswoGivbm4Eh99eBKOpbIsBa06ZTg==",
"dependencies": {
"aws-sdk": "^2.270.1",
"debug": "^3.1.0",
"gunzip-maybe": "^1.4.1",
"mime-types": "^2.1.18",
"tar-stream": "^1.6.1"
},
"engines": {
"node": ">=8.10.0"
}
},
"node_modules/archiver": {
"version": "5.3.0",
"resolved": "https://registry.npmjs.org/archiver/-/archiver-5.3.0.tgz",
@@ -985,54 +979,6 @@
"safe-buffer": "~5.1.0"
}
},
"node_modules/archiver/node_modules/bl": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
"integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
"dependencies": {
"buffer": "^5.5.0",
"inherits": "^2.0.4",
"readable-stream": "^3.4.0"
}
},
"node_modules/archiver/node_modules/buffer": {
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"dependencies": {
"base64-js": "^1.3.1",
"ieee754": "^1.1.13"
}
},
"node_modules/archiver/node_modules/tar-stream": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
"integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
"dependencies": {
"bl": "^4.0.3",
"end-of-stream": "^1.4.1",
"fs-constants": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^3.1.1"
},
"engines": {
"node": ">=6"
}
},
"node_modules/arg": {
"version": "4.1.3",
"resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz",
@@ -1139,6 +1085,18 @@
"resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.2.tgz",
"integrity": "sha512-3pZEU3NT5BFUo/AD5ERPWOgQOCZITni6iavr5AUw5AUwQjMlI0kzu5btnyD39AF0gUEsDPwJT+oY1ORBJijPjQ=="
},
"node_modules/binary": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz",
"integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=",
"dependencies": {
"buffers": "~0.1.1",
"chainsaw": "~0.1.0"
},
"engines": {
"node": "*"
}
},
"node_modules/binary-extensions": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
@@ -1341,20 +1299,6 @@
"isarray": "^1.0.0"
}
},
"node_modules/buffer-alloc": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz",
"integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==",
"dependencies": {
"buffer-alloc-unsafe": "^1.1.0",
"buffer-fill": "^1.0.0"
}
},
"node_modules/buffer-alloc-unsafe": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz",
"integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg=="
},
"node_modules/buffer-crc32": {
"version": "0.2.13",
"resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
@@ -1363,16 +1307,19 @@
"node": "*"
}
},
"node_modules/buffer-fill": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz",
"integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw="
},
"node_modules/buffer-from": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
"integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="
},
"node_modules/buffers": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz",
"integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s=",
"engines": {
"node": ">=0.2.0"
}
},
"node_modules/bull": {
"version": "3.29.2",
"resolved": "https://registry.npmjs.org/bull/-/bull-3.29.2.tgz",
@@ -1498,6 +1445,17 @@
"node": ">=4"
}
},
"node_modules/chainsaw": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz",
"integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=",
"dependencies": {
"traverse": ">=0.3.0 <0.4"
},
"engines": {
"node": "*"
}
},
"node_modules/chalk": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
@@ -1556,11 +1514,6 @@
"fsevents": "~2.3.2"
}
},
"node_modules/chownr": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
},
"node_modules/ci-info": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
@@ -1833,6 +1786,7 @@
"version": "3.2.7",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
"integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
"dev": true,
"dependencies": {
"ms": "^2.1.1"
}
@@ -1882,6 +1836,43 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/decompress-stream-to-s3": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/decompress-stream-to-s3/-/decompress-stream-to-s3-1.2.0.tgz",
"integrity": "sha512-jYL8AA+G3BDXhEV9+b4AKUqOQXfFI8Pv8Pfh6tj7RQvSecm++M+fa6b5t6E10WO4siFY46FlkJbLrebeZvJB/A==",
"dependencies": {
"aws-sdk": "^2.958.0",
"debug": "^4.3.2",
"gunzip-maybe": "^1.4.1",
"mime-types": "^2.1.18",
"tar-stream": "^2.2.0",
"unzip-stream": "^0.3.1"
},
"engines": {
"node": ">=8.10.0"
}
},
"node_modules/decompress-stream-to-s3/node_modules/debug": {
"version": "4.3.2",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz",
"integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==",
"dependencies": {
"ms": "2.1.2"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/decompress-stream-to-s3/node_modules/ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"node_modules/deep-eql": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz",
@@ -3491,13 +3482,18 @@
"node_modules/minimist": {
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
"integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==",
"dev": true
"integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw=="
},
"node_modules/mkdirp-classic": {
"version": "0.5.3",
"resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
"integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
"node_modules/mkdirp": {
"version": "0.5.5",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
"integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
"dependencies": {
"minimist": "^1.2.5"
},
"bin": {
"mkdirp": "bin/cmd.js"
}
},
"node_modules/mocha": {
"version": "9.0.3",
@@ -3726,7 +3722,8 @@
"node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"dev": true
},
"node_modules/nanoid": {
"version": "3.1.23",
@@ -4994,18 +4991,22 @@
"url": "https://github.com/chalk/supports-color?sponsor=1"
}
},
"node_modules/tar-fs": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz",
"integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==",
"node_modules/tar-stream": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
"integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
"dependencies": {
"chownr": "^1.1.1",
"mkdirp-classic": "^0.5.2",
"pump": "^3.0.0",
"tar-stream": "^2.1.4"
"bl": "^4.0.3",
"end-of-stream": "^1.4.1",
"fs-constants": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^3.1.1"
},
"engines": {
"node": ">=6"
}
},
"node_modules/tar-fs/node_modules/bl": {
"node_modules/tar-stream/node_modules/bl": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
"integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
@@ -5015,7 +5016,7 @@
"readable-stream": "^3.4.0"
}
},
"node_modules/tar-fs/node_modules/buffer": {
"node_modules/tar-stream/node_modules/buffer": {
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
@@ -5038,69 +5039,6 @@
"ieee754": "^1.1.13"
}
},
"node_modules/tar-fs/node_modules/tar-stream": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
"integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
"dependencies": {
"bl": "^4.0.3",
"end-of-stream": "^1.4.1",
"fs-constants": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^3.1.1"
},
"engines": {
"node": ">=6"
}
},
"node_modules/tar-stream": {
"version": "1.6.2",
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz",
"integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==",
"dependencies": {
"bl": "^1.0.0",
"buffer-alloc": "^1.2.0",
"end-of-stream": "^1.0.0",
"fs-constants": "^1.0.0",
"readable-stream": "^2.3.0",
"to-buffer": "^1.1.1",
"xtend": "^4.0.0"
},
"engines": {
"node": ">= 0.8.0"
}
},
"node_modules/tar-stream/node_modules/bl": {
"version": "1.2.3",
"resolved": "https://registry.npmjs.org/bl/-/bl-1.2.3.tgz",
"integrity": "sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==",
"dependencies": {
"readable-stream": "^2.3.5",
"safe-buffer": "^5.1.1"
}
},
"node_modules/tar-stream/node_modules/readable-stream": {
"version": "2.3.7",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"dependencies": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"node_modules/tar-stream/node_modules/string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"dependencies": {
"safe-buffer": "~5.1.0"
}
},
"node_modules/term-size": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/term-size/-/term-size-2.2.1.tgz",
@@ -5155,11 +5093,6 @@
"safe-buffer": "~5.1.0"
}
},
"node_modules/to-buffer": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz",
"integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg=="
},
"node_modules/to-readable-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-1.0.0.tgz",
@@ -5201,6 +5134,14 @@
"nodetouch": "bin/nodetouch.js"
}
},
"node_modules/traverse": {
"version": "0.3.9",
"resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz",
"integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk=",
"engines": {
"node": "*"
}
},
"node_modules/ts-custom-error": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/ts-custom-error/-/ts-custom-error-3.2.0.tgz",
@@ -5398,6 +5339,15 @@
"node": ">= 0.8"
}
},
"node_modules/unzip-stream": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/unzip-stream/-/unzip-stream-0.3.1.tgz",
"integrity": "sha512-RzaGXLNt+CW+T41h1zl6pGz3EaeVhYlK+rdAap+7DxW5kqsqePO8kRtWPaCiVqdhZc86EctSPVYNix30YOMzmw==",
"dependencies": {
"binary": "^0.3.0",
"mkdirp": "^0.5.1"
}
},
"node_modules/update-notifier": {
"version": "4.1.3",
"resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-4.1.3.tgz",
@@ -6446,6 +6396,15 @@
"integrity": "sha512-Y0K95ThC3esLEYD6ZuqNek29lNX2EM1qxV8y2FTLUB0ff5wWrk7az+mLrnNFUnaXcgKye22+sFBRXOgpPILZNg==",
"dev": true
},
"@types/unzip-stream": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/@types/unzip-stream/-/unzip-stream-0.3.1.tgz",
"integrity": "sha512-RlE3qaqvu4XaMwxkG/zR1gIunCbqXvNrmZ4BCG7OiQ8QUactFUPxm0TTrOCRJZQfPW3T6XBH7PcHQiiqkdcijw==",
"dev": true,
"requires": {
"@types/node": "*"
}
},
"@types/xml-flow": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@types/xml-flow/-/xml-flow-1.0.1.tgz",
@@ -6568,18 +6527,6 @@
"picomatch": "^2.0.4"
}
},
"archive-stream-to-s3": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/archive-stream-to-s3/-/archive-stream-to-s3-1.1.3.tgz",
"integrity": "sha512-XTUK2fLCAFPrCu9Y0X0u1vLrM5uh/cRbmeBNwUz/X4KeoImcgCexYmOoHAswoGivbm4Eh99eBKOpbIsBa06ZTg==",
"requires": {
"aws-sdk": "^2.270.1",
"debug": "^3.1.0",
"gunzip-maybe": "^1.4.1",
"mime-types": "^2.1.18",
"tar-stream": "^1.6.1"
}
},
"archiver": {
"version": "5.3.0",
"resolved": "https://registry.npmjs.org/archiver/-/archiver-5.3.0.tgz",
@@ -6592,39 +6539,6 @@
"readdir-glob": "^1.0.0",
"tar-stream": "^2.2.0",
"zip-stream": "^4.1.0"
},
"dependencies": {
"bl": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
"integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
"requires": {
"buffer": "^5.5.0",
"inherits": "^2.0.4",
"readable-stream": "^3.4.0"
}
},
"buffer": {
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
"requires": {
"base64-js": "^1.3.1",
"ieee754": "^1.1.13"
}
},
"tar-stream": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
"integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
"requires": {
"bl": "^4.0.3",
"end-of-stream": "^1.4.1",
"fs-constants": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^3.1.1"
}
}
}
},
"archiver-utils": {
@@ -6747,6 +6661,15 @@
"resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.2.tgz",
"integrity": "sha512-3pZEU3NT5BFUo/AD5ERPWOgQOCZITni6iavr5AUw5AUwQjMlI0kzu5btnyD39AF0gUEsDPwJT+oY1ORBJijPjQ=="
},
"binary": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz",
"integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=",
"requires": {
"buffers": "~0.1.1",
"chainsaw": "~0.1.0"
}
},
"binary-extensions": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
@@ -6922,35 +6845,21 @@
"isarray": "^1.0.0"
}
},
"buffer-alloc": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz",
"integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==",
"requires": {
"buffer-alloc-unsafe": "^1.1.0",
"buffer-fill": "^1.0.0"
}
},
"buffer-alloc-unsafe": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz",
"integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg=="
},
"buffer-crc32": {
"version": "0.2.13",
"resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
"integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI="
},
"buffer-fill": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz",
"integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw="
},
"buffer-from": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
"integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="
},
"buffers": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz",
"integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s="
},
"bull": {
"version": "3.29.2",
"resolved": "https://registry.npmjs.org/bull/-/bull-3.29.2.tgz",
@@ -7045,6 +6954,14 @@
"type-detect": "^4.0.5"
}
},
"chainsaw": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz",
"integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=",
"requires": {
"traverse": ">=0.3.0 <0.4"
}
},
"chalk": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
@@ -7088,11 +7005,6 @@
"readdirp": "~3.6.0"
}
},
"chownr": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
},
"ci-info": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
@@ -7310,6 +7222,7 @@
"version": "3.2.7",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
"integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
"dev": true,
"requires": {
"ms": "^2.1.1"
}
@@ -7340,6 +7253,34 @@
}
}
},
"decompress-stream-to-s3": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/decompress-stream-to-s3/-/decompress-stream-to-s3-1.2.0.tgz",
"integrity": "sha512-jYL8AA+G3BDXhEV9+b4AKUqOQXfFI8Pv8Pfh6tj7RQvSecm++M+fa6b5t6E10WO4siFY46FlkJbLrebeZvJB/A==",
"requires": {
"aws-sdk": "^2.958.0",
"debug": "^4.3.2",
"gunzip-maybe": "^1.4.1",
"mime-types": "^2.1.18",
"tar-stream": "^2.2.0",
"unzip-stream": "^0.3.1"
},
"dependencies": {
"debug": {
"version": "4.3.2",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz",
"integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==",
"requires": {
"ms": "2.1.2"
}
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
}
}
},
"deep-eql": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz",
@@ -8528,13 +8469,15 @@
"minimist": {
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
"integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==",
"dev": true
"integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw=="
},
"mkdirp-classic": {
"version": "0.5.3",
"resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
"integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
"mkdirp": {
"version": "0.5.5",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
"integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
"requires": {
"minimist": "^1.2.5"
}
},
"mocha": {
"version": "9.0.3",
@@ -8692,7 +8635,8 @@
"ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"dev": true
},
"nanoid": {
"version": "3.1.23",
@@ -9657,15 +9601,16 @@
"has-flag": "^4.0.0"
}
},
"tar-fs": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz",
"integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==",
"tar-stream": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
"integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
"requires": {
"chownr": "^1.1.1",
"mkdirp-classic": "^0.5.2",
"pump": "^3.0.0",
"tar-stream": "^2.1.4"
"bl": "^4.0.3",
"end-of-stream": "^1.4.1",
"fs-constants": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^3.1.1"
},
"dependencies": {
"bl": {
@@ -9686,65 +9631,6 @@
"base64-js": "^1.3.1",
"ieee754": "^1.1.13"
}
},
"tar-stream": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
"integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
"requires": {
"bl": "^4.0.3",
"end-of-stream": "^1.4.1",
"fs-constants": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^3.1.1"
}
}
}
},
"tar-stream": {
"version": "1.6.2",
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz",
"integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==",
"requires": {
"bl": "^1.0.0",
"buffer-alloc": "^1.2.0",
"end-of-stream": "^1.0.0",
"fs-constants": "^1.0.0",
"readable-stream": "^2.3.0",
"to-buffer": "^1.1.1",
"xtend": "^4.0.0"
},
"dependencies": {
"bl": {
"version": "1.2.3",
"resolved": "https://registry.npmjs.org/bl/-/bl-1.2.3.tgz",
"integrity": "sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==",
"requires": {
"readable-stream": "^2.3.5",
"safe-buffer": "^5.1.1"
}
},
"readable-stream": {
"version": "2.3.7",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"requires": {
"safe-buffer": "~5.1.0"
}
}
}
},
@@ -9792,11 +9678,6 @@
}
}
},
"to-buffer": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz",
"integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg=="
},
"to-readable-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-1.0.0.tgz",
@@ -9826,6 +9707,11 @@
"nopt": "~1.0.10"
}
},
"traverse": {
"version": "0.3.9",
"resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz",
"integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk="
},
"ts-custom-error": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/ts-custom-error/-/ts-custom-error-3.2.0.tgz",
@@ -9969,6 +9855,15 @@
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
"integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw="
},
"unzip-stream": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/unzip-stream/-/unzip-stream-0.3.1.tgz",
"integrity": "sha512-RzaGXLNt+CW+T41h1zl6pGz3EaeVhYlK+rdAap+7DxW5kqsqePO8kRtWPaCiVqdhZc86EctSPVYNix30YOMzmw==",
"requires": {
"binary": "^0.3.0",
"mkdirp": "^0.5.1"
}
},
"update-notifier": {
"version": "4.1.3",
"resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-4.1.3.tgz",

View File

@@ -27,18 +27,17 @@
"@octokit/oauth-app": "^3.3.2",
"@octokit/rest": "^18.5.3",
"@pm2/io": "^5.0.0",
"archive-stream-to-s3": "^1.1.3",
"archiver": "^5.3.0",
"aws-sdk": "^2.958.0",
"bull": "^3.29.2",
"compression": "^1.7.4",
"connect-redis": "^6.0.0",
"decompress-stream-to-s3": "^1.2.0",
"dotenv": "^10.0.0",
"express": "^4.17.1",
"express-rate-limit": "^5.2.6",
"express-session": "^1.17.1",
"got": "^11.8.2",
"gunzip-maybe": "^1.4.2",
"istextorbinary": "^6.0.0",
"mime-types": "^2.1.30",
"mongoose": "^5.13.5",
@@ -48,9 +47,9 @@
"passport-github2": "^0.1.12",
"rate-limit-redis": "^2.1.0",
"redis": "^3.1.2",
"tar-fs": "^2.1.1",
"textextensions": "^5.12.0",
"ts-custom-error": "^3.2.0",
"unzip-stream": "^0.3.1",
"xml-flow": "^1.0.4"
},
"devDependencies": {
@@ -69,6 +68,7 @@
"@types/rate-limit-redis": "^1.7.2",
"@types/redis": "^2.8.31",
"@types/tar-fs": "^2.0.1",
"@types/unzip-stream": "^0.3.1",
"@types/xml-flow": "^1.0.1",
"chai": "^4.3.4",
"mocha": "^9.0.3",

View File

@@ -30,7 +30,7 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
auth?: string
): Promise<OctokitResponse<unknown, 302>> {
const octokit = new Octokit({ auth });
return octokit.rest.repos.downloadTarballArchive({
return octokit.rest.repos.downloadZipballArchive({
owner: this.githubRepository.owner,
repo: this.githubRepository.repo,
ref: this.branch?.commit || "HEAD",
@@ -97,7 +97,7 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
try {
const downloadStream = got.stream(response.url);
downloadStream.addListener("downloadProgress", (p) => (progress = p));
await storage.extractTar(originalPath, downloadStream);
await storage.extractZip(originalPath, downloadStream);
} catch (error) {
await this.repository.updateStatus("error", "unable_to_download");
throw new AnonymousError("unable_to_download", {

View File

@@ -2,11 +2,10 @@ import { StorageBase, Tree } from "../types";
import config from "../../config";
import * as fs from "fs";
import * as tar from "tar-fs";
import * as unzip from "unzip-stream";
import * as path from "path";
import * as express from "express";
import * as stream from "stream";
import * as gunzip from "gunzip-maybe";
import * as archiver from "archiver";
import { promisify } from "util";
@@ -93,15 +92,17 @@ export default class FileSystem implements StorageBase {
}
/** @override */
async extractTar(p: string, data: stream.Readable): Promise<void> {
async extractZip(p: string, data: stream.Readable): Promise<void> {
const pipeline = promisify(stream.pipeline);
return pipeline(
data,
gunzip(),
tar.extract(path.join(config.FOLDER, p), {
map: (header) => {
header.name = header.name.substr(header.name.indexOf("/") + 1);
return header;
unzip.Extract({
path: path.join(path.join(config.FOLDER, p)),
decodeString: (buf) => {
const name = buf.toString();
const newName = name.substr(name.indexOf("/") + 1);
if (newName == "") return "/dev/null";
return newName;
},
})
);

View File

@@ -2,18 +2,14 @@ import { StorageBase, Tree, TreeFile } from "../types";
import { S3 } from "aws-sdk";
import config from "../../config";
import * as stream from "stream";
import { ArchiveStreamToS3 } from "archive-stream-to-s3";
import ArchiveStreamToS3 from "decompress-stream-to-s3";
import * as express from "express";
import * as mime from "mime-types";
import * as flow from "xml-flow";
import * as archiver from "archiver";
import * as path from "path";
import * as gunzip from "gunzip-maybe";
import AnonymousError from "../AnonymousError";
const originalArchiveStreamToS3Entry: Function = (ArchiveStreamToS3 as any)
.prototype.onEntry;
export default class S3Storage implements StorageBase {
type = "AWS";
@@ -172,22 +168,21 @@ export default class S3Storage implements StorageBase {
}
/** @override */
async extractTar(p: string, data: stream.Readable): Promise<void> {
async extractZip(p: string, data: stream.Readable): Promise<void> {
let toS3: ArchiveStreamToS3;
(ArchiveStreamToS3 as any).prototype.onEntry = function (
header: any,
stream: any,
next: any
) {
header.name = header.name.substr(header.name.indexOf("/") + 1);
originalArchiveStreamToS3Entry.call(toS3, header, stream, next);
};
return new Promise((resolve, reject) => {
toS3 = new ArchiveStreamToS3(config.S3_BUCKET, p, this.client);
toS3 = new ArchiveStreamToS3({
bucket: config.S3_BUCKET,
prefix: p,
s3: this.client,
type: "zip",
onEntry: (header) => {
header.name = header.name.substr(header.name.indexOf("/") + 1);
},
});
stream
.pipeline(data, gunzip(), toS3, () => {})
.pipeline(data, toS3, () => {})
.on("finish", resolve)
.on("error", reject);
});

View File

@@ -67,7 +67,7 @@ export interface StorageBase {
* @param dir
* @param tar
*/
extractTar(dir: string, tar: stream.Readable): Promise<void>;
extractZip(dir: string, tar: stream.Readable): Promise<void>;
/**
* Remove the path