diff --git a/package-lock.json b/package-lock.json index 09ae4ec..97e6cda 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "@pm2/io": "^5.0.0", "archiver": "^5.3.1", "aws-sdk": "^2.1180.0", - "bull": "^3.29.2", + "bullmq": "^1.86.7", "compression": "^1.7.4", "connect-redis": "^6.1.3", "decompress-stream-to-s3": "^1.2.1", @@ -39,7 +39,6 @@ }, "devDependencies": { "@types/archiver": "^5.3.1", - "@types/bull": "^3.15.4", "@types/compression": "^1.7.1", "@types/connect-redis": "^0.0.18", "@types/express": "^4.17.13", @@ -100,6 +99,78 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-2.0.2.tgz", + "integrity": "sha512-FMX5i7a+ojIguHpWbzh5MCsCouJkwf4z4ejdUY/fsgB9Vkdak4ZnoIEskOyOUMMB4lctiZFGszFQJXUeFL8tRg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-2.0.2.tgz", + "integrity": "sha512-DznYtF3lHuZDSRaIOYeif4JgO0NtO2Xf8DsngAugMx/bUdTFbg86jDTmkVJBNmV+cxszz6OjGvinnS8AbJ342g==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-2.0.2.tgz", + "integrity": "sha512-Gy9+c3Wj+rUlD3YvCZTi92gs+cRX7ZQogtwq0IhRenloTTlsbpezNgk6OCkt59V4ATEWSic9rbU92H/l7XsRvA==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-2.0.2.tgz", + "integrity": "sha512-b0jMEo566YdM2K+BurSed7bswjo3a6bcdw5ETqoIfSuxKuRLPfAiOjVbZyZBgx3J/TAM/QrvEQ/VN89A0ZAxSg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-2.0.2.tgz", + "integrity": "sha512-zrBHaePwcv4cQXxzYgNj0+A8I1uVN97E7/3LmkRocYZ+rMwUsnPpp4RuTAHSRoKlTQV3nSdCQW4Qdt4MXw/iHw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-2.0.2.tgz", + "integrity": "sha512-fpnI00dt+yO1cKx9qBXelKhPBdEgvc8ZPav1+0r09j0woYQU2N79w/jcGawSY5UGlgQ3vjaJsFHnGbGvvqdLzg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@octokit/auth-oauth-app": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-app/-/auth-oauth-app-4.3.0.tgz", @@ -508,15 +579,6 @@ "resolved": "https://registry.npmjs.org/@types/btoa-lite/-/btoa-lite-1.0.0.tgz", "integrity": "sha512-wJsiX1tosQ+J5+bY5LrSahHxr2wT+uME5UDwdN1kg4frt40euqA+wzECkmq4t5QbveHiJepfdThgQrPw6KiSlg==" }, - "node_modules/@types/bull": { - "version": "3.15.4", - "resolved": "https://registry.npmjs.org/@types/bull/-/bull-3.15.4.tgz", - "integrity": "sha512-vsw4kqsI/SLocfcgTUY2b8POTBUw9AwmTaehvSVnEAbcgJHsPheFMP8WKDFDk2I+aNTLRQX5iP21dPwz3CGLVA==", - "dev": true, - "dependencies": { - "@types/ioredis": "*" - } - }, "node_modules/@types/cacheable-request": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.2.tgz", @@ -1302,30 +1364,56 @@ "node": ">=0.2.0" } }, - "node_modules/bull": { - "version": "3.29.2", - "resolved": "https://registry.npmjs.org/bull/-/bull-3.29.2.tgz", - "integrity": "sha512-zWHyza/ElwVvJUqIEDJdUhGKd1V9EHjituUL7sJAmJoxS9Z7QMhYcMOWcgbUlWPgtiKN1g9ZlOtFAoq7C4/SQw==", + "node_modules/bullmq": { + "version": "1.86.7", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-1.86.7.tgz", + "integrity": "sha512-4fsIcoMmdRmPvll/dV205wfKJlx7jfn1zX1AzO2t56juxiVrbXv5MtYLYzFgx8FW4CnyWzhxXMqJSdTmdxxJ+g==", "dependencies": { - "cron-parser": "^2.13.0", - "debuglog": "^1.0.0", + "cron-parser": "^4.2.1", "get-port": "^5.1.1", - "ioredis": "^4.27.0", + "glob": "^7.2.0", + "ioredis": "^4.28.5", "lodash": "^4.17.21", - "p-timeout": "^3.2.0", - "promise.prototype.finally": "^3.1.2", - "semver": "^7.3.2", - "util.promisify": "^1.0.1", - "uuid": "^8.3.0" - }, - "engines": { - "node": ">=10" + "msgpackr": "^1.4.6", + "semver": "^7.3.7", + "tslib": "^1.14.1", + "uuid": "^8.3.2" } }, - "node_modules/bull/node_modules/semver": { - "version": "7.3.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", - "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "node_modules/bullmq/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/bullmq/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/bullmq/node_modules/semver": { + "version": "7.3.7", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", "dependencies": { "lru-cache": "^6.0.0" }, @@ -1336,7 +1424,12 @@ "node": ">=10" } }, - "node_modules/bull/node_modules/uuid": { + "node_modules/bullmq/node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + }, + "node_modules/bullmq/node_modules/uuid": { "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", @@ -1699,15 +1792,22 @@ "dev": true }, "node_modules/cron-parser": { - "version": "2.18.0", - "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-2.18.0.tgz", - "integrity": "sha512-s4odpheTyydAbTBQepsqd2rNWGa2iV3cyo8g7zbI2QQYGLVsfbhmwukayS1XHppe02Oy1fg7mg6xoaraVJeEcg==", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.5.0.tgz", + "integrity": "sha512-QHdeh3zLWz6YvYTFKpRb860rJlip16pEinbxXT1i2NZB/nOxBjd2RbSv54sn5UrAj9WykiSLYWWDgo8azQK0HA==", "dependencies": { - "is-nan": "^1.3.0", - "moment-timezone": "^0.5.31" + "luxon": "^2.4.0" }, "engines": { - "node": ">=0.8" + "node": ">=12.0.0" + } + }, + "node_modules/cron-parser/node_modules/luxon": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-2.5.0.tgz", + "integrity": "sha512-IDkEPB80Rb6gCAU+FEib0t4FeJ4uVOuX1CQ9GsvU3O+JAGIgu0J7sf1OarXKaKDygTZIoJyU6YdZzTFRu+YR0A==", + "engines": { + "node": ">=12" } }, "node_modules/debug": { @@ -1726,14 +1826,6 @@ } } }, - "node_modules/debuglog": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz", - "integrity": "sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI=", - "engines": { - "node": "*" - } - }, "node_modules/decamelize": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-4.0.0.tgz", @@ -2686,9 +2778,9 @@ } }, "node_modules/ioredis": { - "version": "4.27.9", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.27.9.tgz", - "integrity": "sha512-hAwrx9F+OQ0uIvaJefuS3UTqW+ByOLyLIV+j0EH8ClNVxvFyH9Vmb08hCL4yje6mDYT5zMquShhypkd50RRzkg==", + "version": "4.28.5", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz", + "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==", "dependencies": { "cluster-key-slot": "^1.1.0", "debug": "^4.3.1", @@ -3160,7 +3252,7 @@ "node_modules/lodash.isarguments": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", - "integrity": "sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo=" + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" }, "node_modules/lodash.isplainobject": { "version": "4.0.6", @@ -3404,25 +3496,6 @@ "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz", "integrity": "sha1-EUyUlnPiqKNenTV4hSeqN7Z52is=" }, - "node_modules/moment": { - "version": "2.29.3", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.3.tgz", - "integrity": "sha512-c6YRvhEo//6T2Jz/vVtYzqBzwvPT95JBQ+smCytzf7c50oMZRsR/a4w88aD34I+/QVSfnoAnSBFPJHItlOMJVw==", - "engines": { - "node": "*" - } - }, - "node_modules/moment-timezone": { - "version": "0.5.33", - "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.33.tgz", - "integrity": "sha512-PTc2vcT8K9J5/9rDEPe5czSIKgLoGsH8UNpA4qZTVw0Vd/Uz19geE9abbIOQKaAQFcnQ3v5YEXrbSc5BpshH+w==", - "dependencies": { - "moment": ">= 2.9.0" - }, - "engines": { - "node": "*" - } - }, "node_modules/mongodb": { "version": "3.6.11", "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.11.tgz", @@ -3557,6 +3630,32 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/msgpackr": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.6.1.tgz", + "integrity": "sha512-Je+xBEfdjtvA4bKaOv8iRhjC8qX2oJwpYH4f7JrG4uMVJVmnmkAT4pjKdbztKprGj3iwjcxPzb5umVZ02Qq3tA==", + "optionalDependencies": { + "msgpackr-extract": "^2.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-2.0.2.tgz", + "integrity": "sha512-coskCeJG2KDny23zWeu+6tNy7BLnAiOGgiwzlgdm4oeSsTpqEJJPguHIuKZcCdB7tzhZbXNYSg6jZAXkZErkJA==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.0.2" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "2.0.2", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "2.0.2", + "@msgpackr-extract/msgpackr-extract-linux-arm": "2.0.2", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "2.0.2", + "@msgpackr-extract/msgpackr-extract-linux-x64": "2.0.2", + "@msgpackr-extract/msgpackr-extract-win32-x64": "2.0.2" + } + }, "node_modules/nanoid": { "version": "3.1.23", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.23.tgz", @@ -3596,6 +3695,17 @@ } } }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.2.tgz", + "integrity": "sha512-PiN4NWmlQPqvbEFcH/omQsswWQbe5Z9YK/zdB23irp5j2XibaA2IrGvpSWmVVG4qMZdmPdwPctSy4a86rOMn6g==", + "optional": true, + "bin": { + "node-gyp-build-optional": "optional.js", + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-test": "build-test.js" + } + }, "node_modules/node-schedule": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/node-schedule/-/node-schedule-2.1.0.tgz", @@ -3678,22 +3788,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/object.getownpropertydescriptors": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.2.tgz", - "integrity": "sha512-WtxeKSzfBjlzL+F9b7M7hewDzMwy+C8NRssHd1YrNlzHzIDrXcXiNOMrezdAEM4UXixgV+vvnyBeN7Rygl2ttQ==", - "dependencies": { - "call-bind": "^1.0.2", - "define-properties": "^1.1.3", - "es-abstract": "^1.18.0-next.2" - }, - "engines": { - "node": ">= 0.8" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -3737,14 +3831,6 @@ "node": ">=8" } }, - "node_modules/p-finally": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", - "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=", - "engines": { - "node": ">=4" - } - }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -3783,17 +3869,6 @@ "node": ">=6" } }, - "node_modules/p-timeout": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", - "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", - "dependencies": { - "p-finally": "^1.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/pako": { "version": "0.2.9", "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz", @@ -3952,22 +4027,6 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, - "node_modules/promise.prototype.finally": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/promise.prototype.finally/-/promise.prototype.finally-3.1.2.tgz", - "integrity": "sha512-A2HuJWl2opDH0EafgdjwEw7HysI8ff/n4lW4QEVBCUXFk9QeGecBWv0Deph0UmLe3tTNYegz8MOjsVuE6SMoJA==", - "dependencies": { - "define-properties": "^1.1.3", - "es-abstract": "^1.17.0-next.0", - "function-bind": "^1.1.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -4159,7 +4218,7 @@ "node_modules/redis-errors": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", - "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", "engines": { "node": ">=4" } @@ -4167,7 +4226,7 @@ "node_modules/redis-parser": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", - "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", "dependencies": { "redis-errors": "^1.0.0" }, @@ -4816,21 +4875,6 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, - "node_modules/util.promisify": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.1.1.tgz", - "integrity": "sha512-/s3UsZUrIfa6xDhr7zZhnE9SLQ5RIXyYfiVnMMyMDzOc8WhWN4Nbh36H842OyurKbCDAesZOJaVyvmSl6fhGQw==", - "dependencies": { - "call-bind": "^1.0.0", - "define-properties": "^1.1.3", - "for-each": "^0.3.3", - "has-symbols": "^1.0.1", - "object.getownpropertydescriptors": "^2.1.1" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", @@ -5178,6 +5222,42 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-2.0.2.tgz", + "integrity": "sha512-FMX5i7a+ojIguHpWbzh5MCsCouJkwf4z4ejdUY/fsgB9Vkdak4ZnoIEskOyOUMMB4lctiZFGszFQJXUeFL8tRg==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-2.0.2.tgz", + "integrity": "sha512-DznYtF3lHuZDSRaIOYeif4JgO0NtO2Xf8DsngAugMx/bUdTFbg86jDTmkVJBNmV+cxszz6OjGvinnS8AbJ342g==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-2.0.2.tgz", + "integrity": "sha512-Gy9+c3Wj+rUlD3YvCZTi92gs+cRX7ZQogtwq0IhRenloTTlsbpezNgk6OCkt59V4ATEWSic9rbU92H/l7XsRvA==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-2.0.2.tgz", + "integrity": "sha512-b0jMEo566YdM2K+BurSed7bswjo3a6bcdw5ETqoIfSuxKuRLPfAiOjVbZyZBgx3J/TAM/QrvEQ/VN89A0ZAxSg==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-2.0.2.tgz", + "integrity": "sha512-zrBHaePwcv4cQXxzYgNj0+A8I1uVN97E7/3LmkRocYZ+rMwUsnPpp4RuTAHSRoKlTQV3nSdCQW4Qdt4MXw/iHw==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-2.0.2.tgz", + "integrity": "sha512-fpnI00dt+yO1cKx9qBXelKhPBdEgvc8ZPav1+0r09j0woYQU2N79w/jcGawSY5UGlgQ3vjaJsFHnGbGvvqdLzg==", + "optional": true + }, "@octokit/auth-oauth-app": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-app/-/auth-oauth-app-4.3.0.tgz", @@ -5542,15 +5622,6 @@ "resolved": "https://registry.npmjs.org/@types/btoa-lite/-/btoa-lite-1.0.0.tgz", "integrity": "sha512-wJsiX1tosQ+J5+bY5LrSahHxr2wT+uME5UDwdN1kg4frt40euqA+wzECkmq4t5QbveHiJepfdThgQrPw6KiSlg==" }, - "@types/bull": { - "version": "3.15.4", - "resolved": "https://registry.npmjs.org/@types/bull/-/bull-3.15.4.tgz", - "integrity": "sha512-vsw4kqsI/SLocfcgTUY2b8POTBUw9AwmTaehvSVnEAbcgJHsPheFMP8WKDFDk2I+aNTLRQX5iP21dPwz3CGLVA==", - "dev": true, - "requires": { - "@types/ioredis": "*" - } - }, "@types/cacheable-request": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.2.tgz", @@ -6247,31 +6318,56 @@ "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", "integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s=" }, - "bull": { - "version": "3.29.2", - "resolved": "https://registry.npmjs.org/bull/-/bull-3.29.2.tgz", - "integrity": "sha512-zWHyza/ElwVvJUqIEDJdUhGKd1V9EHjituUL7sJAmJoxS9Z7QMhYcMOWcgbUlWPgtiKN1g9ZlOtFAoq7C4/SQw==", + "bullmq": { + "version": "1.86.7", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-1.86.7.tgz", + "integrity": "sha512-4fsIcoMmdRmPvll/dV205wfKJlx7jfn1zX1AzO2t56juxiVrbXv5MtYLYzFgx8FW4CnyWzhxXMqJSdTmdxxJ+g==", "requires": { - "cron-parser": "^2.13.0", - "debuglog": "^1.0.0", + "cron-parser": "^4.2.1", "get-port": "^5.1.1", - "ioredis": "^4.27.0", + "glob": "^7.2.0", + "ioredis": "^4.28.5", "lodash": "^4.17.21", - "p-timeout": "^3.2.0", - "promise.prototype.finally": "^3.1.2", - "semver": "^7.3.2", - "util.promisify": "^1.0.1", - "uuid": "^8.3.0" + "msgpackr": "^1.4.6", + "semver": "^7.3.7", + "tslib": "^1.14.1", + "uuid": "^8.3.2" }, "dependencies": { + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "requires": { + "brace-expansion": "^1.1.7" + } + }, "semver": { - "version": "7.3.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", - "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "version": "7.3.7", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", "requires": { "lru-cache": "^6.0.0" } }, + "tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + }, "uuid": { "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", @@ -6552,12 +6648,18 @@ "dev": true }, "cron-parser": { - "version": "2.18.0", - "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-2.18.0.tgz", - "integrity": "sha512-s4odpheTyydAbTBQepsqd2rNWGa2iV3cyo8g7zbI2QQYGLVsfbhmwukayS1XHppe02Oy1fg7mg6xoaraVJeEcg==", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.5.0.tgz", + "integrity": "sha512-QHdeh3zLWz6YvYTFKpRb860rJlip16pEinbxXT1i2NZB/nOxBjd2RbSv54sn5UrAj9WykiSLYWWDgo8azQK0HA==", "requires": { - "is-nan": "^1.3.0", - "moment-timezone": "^0.5.31" + "luxon": "^2.4.0" + }, + "dependencies": { + "luxon": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-2.5.0.tgz", + "integrity": "sha512-IDkEPB80Rb6gCAU+FEib0t4FeJ4uVOuX1CQ9GsvU3O+JAGIgu0J7sf1OarXKaKDygTZIoJyU6YdZzTFRu+YR0A==" + } } }, "debug": { @@ -6568,11 +6670,6 @@ "ms": "2.1.2" } }, - "debuglog": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz", - "integrity": "sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI=" - }, "decamelize": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-4.0.0.tgz", @@ -7264,9 +7361,9 @@ } }, "ioredis": { - "version": "4.27.9", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.27.9.tgz", - "integrity": "sha512-hAwrx9F+OQ0uIvaJefuS3UTqW+ByOLyLIV+j0EH8ClNVxvFyH9Vmb08hCL4yje6mDYT5zMquShhypkd50RRzkg==", + "version": "4.28.5", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz", + "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==", "requires": { "cluster-key-slot": "^1.1.0", "debug": "^4.3.1", @@ -7595,7 +7692,7 @@ "lodash.isarguments": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", - "integrity": "sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo=" + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" }, "lodash.isplainobject": { "version": "4.0.6", @@ -7779,19 +7876,6 @@ "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz", "integrity": "sha1-EUyUlnPiqKNenTV4hSeqN7Z52is=" }, - "moment": { - "version": "2.29.3", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.3.tgz", - "integrity": "sha512-c6YRvhEo//6T2Jz/vVtYzqBzwvPT95JBQ+smCytzf7c50oMZRsR/a4w88aD34I+/QVSfnoAnSBFPJHItlOMJVw==" - }, - "moment-timezone": { - "version": "0.5.33", - "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.33.tgz", - "integrity": "sha512-PTc2vcT8K9J5/9rDEPe5czSIKgLoGsH8UNpA4qZTVw0Vd/Uz19geE9abbIOQKaAQFcnQ3v5YEXrbSc5BpshH+w==", - "requires": { - "moment": ">= 2.9.0" - } - }, "mongodb": { "version": "3.6.11", "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.11.tgz", @@ -7876,6 +7960,29 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "msgpackr": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.6.1.tgz", + "integrity": "sha512-Je+xBEfdjtvA4bKaOv8iRhjC8qX2oJwpYH4f7JrG4uMVJVmnmkAT4pjKdbztKprGj3iwjcxPzb5umVZ02Qq3tA==", + "requires": { + "msgpackr-extract": "^2.0.2" + } + }, + "msgpackr-extract": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-2.0.2.tgz", + "integrity": "sha512-coskCeJG2KDny23zWeu+6tNy7BLnAiOGgiwzlgdm4oeSsTpqEJJPguHIuKZcCdB7tzhZbXNYSg6jZAXkZErkJA==", + "optional": true, + "requires": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "2.0.2", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "2.0.2", + "@msgpackr-extract/msgpackr-extract-linux-arm": "2.0.2", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "2.0.2", + "@msgpackr-extract/msgpackr-extract-linux-x64": "2.0.2", + "@msgpackr-extract/msgpackr-extract-win32-x64": "2.0.2", + "node-gyp-build-optional-packages": "5.0.2" + } + }, "nanoid": { "version": "3.1.23", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.23.tgz", @@ -7895,6 +8002,12 @@ "whatwg-url": "^5.0.0" } }, + "node-gyp-build-optional-packages": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.2.tgz", + "integrity": "sha512-PiN4NWmlQPqvbEFcH/omQsswWQbe5Z9YK/zdB23irp5j2XibaA2IrGvpSWmVVG4qMZdmPdwPctSy4a86rOMn6g==", + "optional": true + }, "node-schedule": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/node-schedule/-/node-schedule-2.1.0.tgz", @@ -7952,16 +8065,6 @@ "object-keys": "^1.1.1" } }, - "object.getownpropertydescriptors": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.2.tgz", - "integrity": "sha512-WtxeKSzfBjlzL+F9b7M7hewDzMwy+C8NRssHd1YrNlzHzIDrXcXiNOMrezdAEM4UXixgV+vvnyBeN7Rygl2ttQ==", - "requires": { - "call-bind": "^1.0.2", - "define-properties": "^1.1.3", - "es-abstract": "^1.18.0-next.2" - } - }, "on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -7993,11 +8096,6 @@ "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.1.1.tgz", "integrity": "sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg==" }, - "p-finally": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", - "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=" - }, "p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -8021,14 +8119,6 @@ "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==" }, - "p-timeout": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", - "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", - "requires": { - "p-finally": "^1.0.0" - } - }, "pako": { "version": "0.2.9", "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz", @@ -8137,16 +8227,6 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, - "promise.prototype.finally": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/promise.prototype.finally/-/promise.prototype.finally-3.1.2.tgz", - "integrity": "sha512-A2HuJWl2opDH0EafgdjwEw7HysI8ff/n4lW4QEVBCUXFk9QeGecBWv0Deph0UmLe3tTNYegz8MOjsVuE6SMoJA==", - "requires": { - "define-properties": "^1.1.3", - "es-abstract": "^1.17.0-next.0", - "function-bind": "^1.1.1" - } - }, "proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -8300,12 +8380,12 @@ "redis-errors": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", - "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=" + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==" }, "redis-parser": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", - "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", "requires": { "redis-errors": "^1.0.0" } @@ -8813,18 +8893,6 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, - "util.promisify": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.1.1.tgz", - "integrity": "sha512-/s3UsZUrIfa6xDhr7zZhnE9SLQ5RIXyYfiVnMMyMDzOc8WhWN4Nbh36H842OyurKbCDAesZOJaVyvmSl6fhGQw==", - "requires": { - "call-bind": "^1.0.0", - "define-properties": "^1.1.3", - "for-each": "^0.3.3", - "has-symbols": "^1.0.1", - "object.getownpropertydescriptors": "^2.1.1" - } - }, "utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", diff --git a/package.json b/package.json index 817517e..556ece8 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "@pm2/io": "^5.0.0", "archiver": "^5.3.1", "aws-sdk": "^2.1180.0", - "bull": "^3.29.2", + "bullmq": "^1.86.7", "compression": "^1.7.4", "connect-redis": "^6.1.3", "decompress-stream-to-s3": "^1.2.1", @@ -54,7 +54,6 @@ }, "devDependencies": { "@types/archiver": "^5.3.1", - "@types/bull": "^3.15.4", "@types/compression": "^1.7.1", "@types/connect-redis": "^0.0.18", "@types/express": "^4.17.13", diff --git a/src/AnonymizedFile.ts b/src/AnonymizedFile.ts index 454a148..96a4a38 100644 --- a/src/AnonymizedFile.ts +++ b/src/AnonymizedFile.ts @@ -1,6 +1,6 @@ -import * as path from "path"; -import * as express from "express"; -import * as stream from "stream"; +import { join, basename } from "path"; +import { Response } from "express"; +import { Readable, pipeline } from "stream"; import { promisify } from "util"; import Repository from "./Repository"; import { Tree, TreeElement, TreeFile } from "./types"; @@ -19,11 +19,11 @@ function tree2sha( const sha = tree[i].sha as string; const size = tree[i].size as number; if (sha != null && size != null) { - output[sha] = path.join(parent, i); + output[sha] = join(parent, i); } else if (tree[i].child) { - tree2sha(tree[i].child as Tree, output, path.join(parent, i)); + tree2sha(tree[i].child as Tree, output, join(parent, i)); } else { - tree2sha(tree[i] as Tree, output, path.join(parent, i)); + tree2sha(tree[i] as Tree, output, join(parent, i)); } } return output; @@ -98,13 +98,13 @@ export default class AnonymizedFile { // if only one option we found the original filename if (options.length == 1) { - currentOriginalPath = path.join(currentOriginalPath, options[0]); + currentOriginalPath = join(currentOriginalPath, options[0]); currentOriginal = currentOriginal[options[0]]; } else { isAmbiguous = true; } } else if (!isAmbiguous) { - currentOriginalPath = path.join(currentOriginalPath, fileName); + currentOriginalPath = join(currentOriginalPath, fileName); currentOriginal = currentOriginal[fileName]; } } @@ -130,7 +130,7 @@ export default class AnonymizedFile { }); } - this._originalPath = path.join(currentOriginalPath, shaTree[file.sha]); + this._originalPath = join(currentOriginalPath, shaTree[file.sha]); } else { this._originalPath = currentOriginalPath; } @@ -139,7 +139,7 @@ export default class AnonymizedFile { } async isFileSupported() { - const filename = path.basename(await this.originalPath()); + const filename = basename(await this.originalPath()); const extensions = filename.split(".").reverse(); const extension = extensions[0].toLowerCase(); if (!this.repository.options.pdf && extension == "pdf") { @@ -158,7 +158,7 @@ export default class AnonymizedFile { return true; } - async content(): Promise { + async content(): Promise { if (this.fileSize && this.fileSize > config.MAX_FILE_SIZE) { throw new AnonymousError("file_too_big", { object: this, @@ -184,13 +184,13 @@ export default class AnonymizedFile { object: this, httpStatus: 400, }); - return path.join(this.repository.originalCachePath, this._originalPath); + return join(this.repository.originalCachePath, this._originalPath); } - async send(res: express.Response): Promise { - const pipeline = promisify(stream.pipeline); + async send(res: Response): Promise { + const pipe = promisify(pipeline); try { - await pipeline(await this.anonymizedContent(), res); + await pipe(await this.anonymizedContent(), res); } catch (error) { handleError(error, res); } diff --git a/src/Repository.ts b/src/Repository.ts index fd12437..9df78b7 100644 --- a/src/Repository.ts +++ b/src/Repository.ts @@ -1,7 +1,7 @@ -import * as path from "path"; +import { join } from "path"; import storage from "./storage"; import { RepositoryStatus, Source, Tree, TreeElement, TreeFile } from "./types"; -import * as stream from "stream"; +import { Readable } from "stream"; import User from "./User"; import GitHubStream from "./source/GitHubStream"; import GitHubDownload from "./source/GitHubDownload"; @@ -136,7 +136,7 @@ export default class Repository { * * @returns A stream of anonymized repository compressed */ - zip(): stream.Readable { + zip(): Readable { return storage.archive(this.originalCachePath, { format: "zip", fileTransformer: (filename) => @@ -185,11 +185,13 @@ export default class Repository { this._model.anonymizeDate = new Date(); console.log(`${this._model.repoId} will be updated to ${newCommit}`); await this.resetSate("preparing"); - await downloadQueue.add(this, { jobId: this.repoId, attempts: 3 }); + await downloadQueue.add(this.repoId, this, { + jobId: this.repoId, + attempts: 3, + }); } } } - /** * Download the require state for the repository to work * @@ -327,7 +329,7 @@ export default class Repository { } get originalCachePath() { - return path.join(this._model.repoId, "original") + "/"; + return join(this._model.repoId, "original") + "/"; } get status() { diff --git a/src/anonymize-utils.ts b/src/anonymize-utils.ts index ec55309..40d4137 100644 --- a/src/anonymize-utils.ts +++ b/src/anonymize-utils.ts @@ -2,15 +2,14 @@ import config from "../config"; import Repository from "./Repository"; import GitHubBase from "./source/GitHubBase"; import { isText } from "istextorbinary"; -import * as path from "path"; - -import * as stream from "stream"; +import { basename } from "path"; +import { Transform } from "stream"; const urlRegex = /?/g; -export function isTextFile(filePath, content) { - const filename = path.basename(filePath); +export function isTextFile(filePath: string, content: Buffer) { + const filename = basename(filePath); const extensions = filename.split(".").reverse(); const extension = extensions[0].toLowerCase(); if (config.additionalExtensions.includes(extension)) { @@ -23,7 +22,7 @@ export function isTextFile(filePath, content) { } export function anonymizeStream(filename: string, repository: Repository) { - const ts = new stream.Transform(); + const ts = new Transform(); var chunks = [], len = 0, pos = 0; diff --git a/src/processes/downloadRepository.ts b/src/processes/downloadRepository.ts index 18dda83..4b3363a 100644 --- a/src/processes/downloadRepository.ts +++ b/src/processes/downloadRepository.ts @@ -1,14 +1,17 @@ -import AnonymousError from "../AnonymousError"; -import { connect, getRepository } from "../database/database"; +import { SandboxedJob } from "bullmq"; +import { config } from "dotenv"; +config(); +import Repository from "../Repository"; -export default async function process(job) { +export default async function (job: SandboxedJob) { + const { connect, getRepository } = require("../database/database"); console.log(`${job.data.repoId} is going to be downloaded`); try { await connect(); const repo = await getRepository(job.data.repoId); - job.progress("get_repo"); - await repo.resetSate(); - job.progress("resetSate"); + job.updateProgress({ status: "get_repo" }); + await repo.resetSate("preparing"); + job.updateProgress({ status: "resetSate" }); try { await repo.anonymize(); } catch (error) { @@ -16,15 +19,7 @@ export default async function process(job) { throw error; } } catch (error) { - if (error instanceof AnonymousError) { - console.error( - "[ERROR]", - error.toString(), - error.stack.split("\n")[1].trim() - ); - } else { - console.error(error); - } + console.error(error); } finally { console.log(`${job.data.repoId} is downloaded`); } diff --git a/src/processes/removeRepository.ts b/src/processes/removeRepository.ts index 6cc0fd5..030878a 100644 --- a/src/processes/removeRepository.ts +++ b/src/processes/removeRepository.ts @@ -1,7 +1,8 @@ -import AnonymousError from "../AnonymousError"; -import { connect, getRepository } from "../database/database"; +import { SandboxedJob } from "bullmq"; +import Repository from "../Repository"; -export default async function process(job) { +export default async function (job: SandboxedJob) { + const { connect, getRepository } = require("../database/database"); try { await connect(); console.log(`${job.data.repoId} is going to be removed`); @@ -13,15 +14,7 @@ export default async function process(job) { throw error; } } catch (error) { - if (error instanceof AnonymousError) { - console.error( - "[ERROR]", - error.toString(), - error.stack.split("\n")[1].trim() - ); - } else { - console.error(error); - } + console.error(error); } finally { console.log(`${job.data.repoId} is removed`); } diff --git a/src/queue.ts b/src/queue.ts index 8a8fc99..8547ffb 100644 --- a/src/queue.ts +++ b/src/queue.ts @@ -1,27 +1,80 @@ -import * as Queue from "bull"; +import { Queue, Worker } from "bullmq"; import config from "../config"; import Repository from "./Repository"; import * as path from "path"; -export const removeQueue = new Queue("repository removal", { - redis: { - host: config.REDIS_HOSTNAME, - port: config.REDIS_PORT, - }, -}); -removeQueue.on("completed", async (job) => { - await job.remove(); -}); -export const downloadQueue = new Queue("repository download", { - redis: { - host: config.REDIS_HOSTNAME, - port: config.REDIS_PORT, - }, -}); -downloadQueue.on("completed", async (job) => { - await job.remove(); -}); +export let removeQueue: Queue; +export let downloadQueue: Queue; -removeQueue.process(5, path.resolve("src/processes/removeRepository.ts")); +// avoid to load the queue outside the main server +export function startWorker() { + removeQueue = new Queue("repository removal", { + connection: { + host: config.REDIS_HOSTNAME, + port: config.REDIS_PORT, + }, + defaultJobOptions: { + removeOnComplete: true, + }, + }); + downloadQueue = new Queue("repository download", { + connection: { + host: config.REDIS_HOSTNAME, + port: config.REDIS_PORT, + }, + defaultJobOptions: { + removeOnComplete: true, + }, + }); + const removeWorker = new Worker( + removeQueue.name, + path.resolve("dist/src/processes/removeRepository.js"), + //removeRepository, + { + concurrency: 5, + connection: { + host: config.REDIS_HOSTNAME, + port: config.REDIS_PORT, + }, + autorun: true, -downloadQueue.process(2, path.resolve("src/processes/downloadRepository.ts")); + } + ); + removeWorker.on("error", async (error) => { + console.log(error); + }); + removeWorker.on("completed", async (job) => { + await job.remove(); + }); + + const downloadWorker = new Worker( + downloadQueue.name, + path.resolve("dist/src/processes/downloadRepository.js"), + // downloadRepository, + { + concurrency: 2, + connection: { + host: config.REDIS_HOSTNAME, + port: config.REDIS_PORT, + }, + autorun: true + } + ); + if (!downloadWorker.isRunning) downloadWorker.run(); + + downloadWorker.on("active", async (job) => { + console.log("active", job.data.repoId); + }); + downloadWorker.on("completed", async (job) => { + console.log("completed", job.data.repoId); + }); + downloadWorker.on("failed", async (job) => { + console.log("failed", job.data.repoId); + }); + downloadWorker.on("closing", async (error) => { + console.log("closing", error); + }); + downloadWorker.on("error", async (error) => { + console.log(error); + }); +} diff --git a/src/routes/admin.ts b/src/routes/admin.ts index 985a936..98b9d76 100644 --- a/src/routes/admin.ts +++ b/src/routes/admin.ts @@ -1,9 +1,10 @@ -import { Queue } from "bull"; +import { Queue } from "bullmq"; import * as express from "express"; import AnonymizedRepositoryModel from "../database/anonymizedRepositories/anonymizedRepositories.model"; import ConferenceModel from "../database/conference/conferences.model"; import UserModel from "../database/users/users.model"; import { downloadQueue, removeQueue } from "../queue"; +import Repository from "../Repository"; import { ensureAuthenticated } from "./connection"; import { handleError, getUser, isOwnerOrAdmin } from "./route-utils"; @@ -29,7 +30,7 @@ router.use( ); router.post("/queue/:name/:repo_id", async (req, res) => { - let queue: Queue; + let queue: Queue; if (req.params.name == "download") { queue = downloadQueue; } else if (req.params.name == "remove") { @@ -41,7 +42,7 @@ router.post("/queue/:name/:repo_id", async (req, res) => { if (!job) { return res.status(404).json({ error: "job_not_found" }); } - job.retry(); + await job.retry(); res.send("ok"); }); diff --git a/src/routes/repository-private.ts b/src/routes/repository-private.ts index 9082cef..83ed336 100644 --- a/src/routes/repository-private.ts +++ b/src/routes/repository-private.ts @@ -123,7 +123,7 @@ router.delete( const user = await getUser(req); isOwnerOrAdmin([repo.owner.id], user); await repo.updateStatus("removing"); - await removeQueue.add(repo, { jobId: repo.repoId }); + await removeQueue.add(repo.repoId, repo, { jobId: repo.repoId }); return res.json({ status: repo.status }); } catch (error) { handleError(error, res); @@ -362,7 +362,7 @@ router.post( repo.model.conference = repoUpdate.conference; await repo.updateStatus("preparing"); res.json({ status: repo.status }); - await downloadQueue.add(repo, { jobId: repo.repoId }); + await downloadQueue.add(repo.repoId, repo, { jobId: repo.repoId }); } catch (error) { return handleError(error, res); } @@ -432,7 +432,7 @@ router.post("/", async (req: express.Request, res: express.Response) => { } res.send({ status: repo.status }); - downloadQueue.add(new Repository(repo), { + downloadQueue.add(repo.repoId, new Repository(repo), { jobId: repo.repoId, attempts: 3, }); diff --git a/src/routes/repository-public.ts b/src/routes/repository-public.ts index 21b652f..d4a2833 100644 --- a/src/routes/repository-public.ts +++ b/src/routes/repository-public.ts @@ -91,7 +91,10 @@ router.get( // && repo.status != "preparing" ) { await repo.updateStatus("preparing"); - await downloadQueue.add(repo, { jobId: repo.repoId, attempts: 3 }); + await downloadQueue.add(repo.repoId, repo, { + jobId: repo.repoId, + attempts: 3, + }); } if (repo.status == "error") { throw new AnonymousError( diff --git a/src/routes/route-utils.ts b/src/routes/route-utils.ts index 2165539..d10d629 100644 --- a/src/routes/route-utils.ts +++ b/src/routes/route-utils.ts @@ -44,6 +44,7 @@ export function isOwnerOrAdmin(authorizedUsers: string[], user: User) { function printError(error: any) { io.notifyError(error, error.value); if (error instanceof AnonymousError) { + console.log(error); console.error( "[ERROR]", error.toString(), @@ -78,14 +79,14 @@ export function handleError(error: any, res: express.Response) { export async function getUser(req: express.Request) { const user = (req.user as any).user; if (!user) { - req.logout(); + req.logout((error) => console.error(error)); throw new AnonymousError("not_connected", { httpStatus: 401, }); } const model = await UserModel.findById(user._id); if (!model) { - req.logout(); + req.logout((error) => console.error(error)); throw new AnonymousError("not_connected", { httpStatus: 401, }); diff --git a/src/routes/user.ts b/src/routes/user.ts index 252bc7b..98afb9d 100644 --- a/src/routes/user.ts +++ b/src/routes/user.ts @@ -10,7 +10,7 @@ router.use(ensureAuthenticated); router.get("/logout", async (req: express.Request, res: express.Response) => { try { - req.logout(); + req.logout((error) => console.error(error)); res.redirect("/"); } catch (error) { handleError(error, res); diff --git a/src/server.ts b/src/server.ts index a67cefb..97fe392 100644 --- a/src/server.ts +++ b/src/server.ts @@ -13,6 +13,7 @@ import * as connection from "./routes/connection"; import router from "./routes"; import AnonymizedRepositoryModel from "./database/anonymizedRepositories/anonymizedRepositories.model"; import { conferenceStatusCheck, repositoryStatusCheck } from "./schedule"; +import { startWorker } from "./queue"; function indexResponse(req: express.Request, res: express.Response) { if ( @@ -44,13 +45,15 @@ export default async function start() { app.use(passport.initialize()); app.use(passport.session()); + startWorker(); + const redisClient = createClient({ socket: { host: config.REDIS_HOSTNAME, port: config.REDIS_PORT, }, }); - redisClient.on('error', (err) => console.log('Redis Client Error', err)); + redisClient.on("error", (err) => console.log("Redis Client Error", err)); await redisClient.connect(); @@ -59,7 +62,7 @@ export default async function start() { sendCommand: (...args: string[]) => redisClient.sendCommand(args), }), windowMs: 15 * 60 * 1000, // 15 minutes - max: 200, // limit each IP + max: 1000, // limit each IP standardHeaders: true, legacyHeaders: false, // delayMs: 0, // disable delaying - full speed until the max limit is reached diff --git a/src/source/GitHubBase.ts b/src/source/GitHubBase.ts index 219de78..433c754 100644 --- a/src/source/GitHubBase.ts +++ b/src/source/GitHubBase.ts @@ -2,9 +2,8 @@ import AnonymizedFile from "../AnonymizedFile"; import { Branch, Tree } from "../types"; import { GitHubRepository } from "./GitHubRepository"; import config from "../../config"; -import { OAuthApp } from "@octokit/oauth-app"; import Repository from "../Repository"; -import * as stream from "stream"; +import { Readable } from "stream"; import UserModel from "../database/users/users.model"; import AnonymousError from "../AnonymousError"; @@ -37,7 +36,7 @@ export default abstract class GitHubBase { this.branch = { commit: data.commit, name: data.branch }; } - async getFileContent(file: AnonymizedFile): Promise { + async getFileContent(file: AnonymizedFile): Promise { throw new AnonymousError("method_not_implemented", { httpStatus: 501, object: this, diff --git a/src/source/GitHubDownload.ts b/src/source/GitHubDownload.ts index ee8374f..7462f63 100644 --- a/src/source/GitHubDownload.ts +++ b/src/source/GitHubDownload.ts @@ -7,7 +7,7 @@ import GitHubBase from "./GitHubBase"; import AnonymizedFile from "../AnonymizedFile"; import { SourceBase } from "../types"; import got from "got"; -import * as stream from "stream"; +import { Readable } from "stream"; import { OctokitResponse } from "@octokit/types"; import AnonymousError from "../AnonymousError"; @@ -113,7 +113,7 @@ export default class GitHubDownload extends GitHubBase implements SourceBase { await this.repository.updateStatus("ready"); } - async getFileContent(file: AnonymizedFile): Promise { + async getFileContent(file: AnonymizedFile): Promise { await this.download(); // update the file list await this.repository.files({ force: true }); diff --git a/src/source/Zip.ts b/src/source/Zip.ts index 63d2f83..0c9a526 100644 --- a/src/source/Zip.ts +++ b/src/source/Zip.ts @@ -1,4 +1,3 @@ -import * as path from "path"; import AnonymizedFile from "../AnonymizedFile"; import Repository from "../Repository"; import storage from "../storage"; diff --git a/src/storage/FileSystem.ts b/src/storage/FileSystem.ts index cc60118..4a205ec 100644 --- a/src/storage/FileSystem.ts +++ b/src/storage/FileSystem.ts @@ -2,10 +2,10 @@ import { StorageBase, Tree } from "../types"; import config from "../../config"; import * as fs from "fs"; -import * as unzip from "unzip-stream"; -import * as path from "path"; -import * as express from "express"; -import * as stream from "stream"; +import { Extract } from "unzip-stream"; +import { join, basename, dirname } from "path"; +import { Response } from "express"; +import { Readable, pipeline } from "stream"; import * as archiver from "archiver"; import { promisify } from "util"; @@ -16,32 +16,32 @@ export default class FileSystem implements StorageBase { /** @override */ async exists(p: string): Promise { - return fs.existsSync(path.join(config.FOLDER, p)); + return fs.existsSync(join(config.FOLDER, p)); } /** @override */ - send(p: string, res: express.Response) { - res.sendFile(path.join(config.FOLDER, p), { dotfiles: "allow" }); + send(p: string, res: Response) { + res.sendFile(join(config.FOLDER, p), { dotfiles: "allow" }); } /** @override */ - read(p: string): stream.Readable { - return fs.createReadStream(path.join(config.FOLDER, p)); + read(p: string): Readable { + return fs.createReadStream(join(config.FOLDER, p)); } /** @override */ async write(p: string, data: Buffer): Promise { - if (!(await this.exists(path.dirname(p)))) { - await fs.promises.mkdir(path.dirname(path.join(config.FOLDER, p)), { + if (!(await this.exists(dirname(p)))) { + await fs.promises.mkdir(dirname(join(config.FOLDER, p)), { recursive: true, }); } - return fs.promises.writeFile(path.join(config.FOLDER, p), data); + return fs.promises.writeFile(join(config.FOLDER, p), data); } /** @override */ async rm(dir: string): Promise { - await fs.promises.rm(path.join(config.FOLDER, dir), { + await fs.promises.rm(join(config.FOLDER, dir), { force: true, recursive: true, }); @@ -50,7 +50,7 @@ export default class FileSystem implements StorageBase { /** @override */ async mk(dir: string): Promise { if (!(await this.exists(dir))) - fs.promises.mkdir(path.join(config.FOLDER, dir), { recursive: true }); + fs.promises.mkdir(join(config.FOLDER, dir), { recursive: true }); } /** @override */ @@ -64,12 +64,12 @@ export default class FileSystem implements StorageBase { if (opt.root == null) { opt.root = config.FOLDER; } - let files = await fs.promises.readdir(path.join(opt.root, dir)); + let files = await fs.promises.readdir(join(opt.root, dir)); const output: Tree = {}; for (let file of files) { - let filePath = path.join(dir, file); + let filePath = join(dir, file); try { - const stats = await fs.promises.stat(path.join(opt.root, filePath)); + const stats = await fs.promises.stat(join(opt.root, filePath)); if (file[0] == "$") { file = "\\" + file; } @@ -92,12 +92,12 @@ export default class FileSystem implements StorageBase { } /** @override */ - async extractZip(p: string, data: stream.Readable): Promise { - const pipeline = promisify(stream.pipeline); - return pipeline( + async extractZip(p: string, data: Readable): Promise { + const pipe = promisify(pipeline); + return pipe( data, - unzip.Extract({ - path: path.join(path.join(config.FOLDER, p)), + Extract({ + path: join(join(config.FOLDER, p)), decodeString: (buf) => { const name = buf.toString(); const newName = name.substr(name.indexOf("/") + 1); @@ -127,8 +127,8 @@ export default class FileSystem implements StorageBase { } const f = file.path.replace(dir, ""); archive.append(rs, { - name: path.basename(f), - prefix: path.dirname(f), + name: basename(f), + prefix: dirname(f), }); }, }).then(() => { diff --git a/src/storage/S3.ts b/src/storage/S3.ts index a3e5cfb..5f3880f 100644 --- a/src/storage/S3.ts +++ b/src/storage/S3.ts @@ -1,13 +1,13 @@ import { StorageBase, Tree, TreeFile } from "../types"; import { S3 } from "aws-sdk"; import config from "../../config"; -import * as stream from "stream"; +import { pipeline, Readable } from "stream"; import ArchiveStreamToS3 from "decompress-stream-to-s3"; -import * as express from "express"; -import * as mime from "mime-types"; +import { Response } from "express"; +import { lookup } from "mime-types"; import * as flow from "xml-flow"; import * as archiver from "archiver"; -import * as path from "path"; +import { dirname, basename } from "path"; import AnonymousError from "../AnonymousError"; export default class S3Storage implements StorageBase { @@ -83,7 +83,7 @@ export default class S3Storage implements StorageBase { } /** @override */ - send(p: string, res: express.Response) { + send(p: string, res: Response) { const s = this.client .getObject({ Bucket: config.S3_BUCKET, @@ -102,8 +102,8 @@ export default class S3Storage implements StorageBase { res.set("Content-Length", headers["content-length"]); res.set("Content-Type", headers["content-type"]); } - stream.pipeline( - response.httpResponse.createUnbufferedStream() as stream.Readable, + pipeline( + response.httpResponse.createUnbufferedStream() as Readable, res ); }); @@ -112,7 +112,7 @@ export default class S3Storage implements StorageBase { } /** @override */ - read(path: string): stream.Readable { + read(path: string): Readable { return this.client .getObject({ Bucket: config.S3_BUCKET, @@ -128,7 +128,7 @@ export default class S3Storage implements StorageBase { Bucket: config.S3_BUCKET, Key: path, Body: data, - ContentType: mime.lookup(path).toString(), + ContentType: lookup(path).toString(), }) .promise(); return; @@ -168,7 +168,7 @@ export default class S3Storage implements StorageBase { } /** @override */ - async extractZip(p: string, data: stream.Readable): Promise { + async extractZip(p: string, data: Readable): Promise { let toS3: ArchiveStreamToS3; return new Promise((resolve, reject) => { @@ -181,8 +181,7 @@ export default class S3Storage implements StorageBase { header.name = header.name.substr(header.name.indexOf("/") + 1); }, }); - stream - .pipeline(data, toS3, () => {}) + pipeline(data, toS3, () => {}) .on("finish", resolve) .on("error", reject); }); @@ -210,14 +209,14 @@ export default class S3Storage implements StorageBase { xmlStream.on("tag:contents", function (file) { let rs = that.read(file.key); file.key = file.key.replace(dir, ""); - const filename = path.basename(file.key); + const filename = basename(file.key); if (filename == "") return; if (opt?.fileTransformer) { rs = rs.pipe(opt.fileTransformer(filename)); } archive.append(rs, { name: filename, - prefix: path.dirname(file.key), + prefix: dirname(file.key), }); }); xmlStream.on("end", () => {