From 9838891567c3e997f8f827f1a0eff5964748ca8f Mon Sep 17 00:00:00 2001 From: tdurieux Date: Wed, 8 Sep 2021 15:00:15 +0200 Subject: [PATCH] feat: handle repository download and remove using a job queue --- package-lock.json | 966 +++++++++++++++++++++++++++++++ package.json | 2 + public/script/app.js | 43 +- src/Repository.ts | 77 ++- src/queue.ts | 50 ++ src/routes/repository-private.ts | 29 +- src/routes/repository-public.ts | 4 +- src/types.ts | 2 + 8 files changed, 1121 insertions(+), 52 deletions(-) create mode 100644 src/queue.ts diff --git a/package-lock.json b/package-lock.json index a499dcf..d628ffd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "archive-stream-to-s3": "^1.1.3", "archiver": "^5.3.0", "aws-sdk": "^2.958.0", + "bull": "^3.29.2", "compression": "^1.7.4", "connect-redis": "^6.0.0", "dotenv": "^10.0.0", @@ -39,6 +40,7 @@ }, "devDependencies": { "@types/archiver": "^5.1.1", + "@types/bull": "^3.15.4", "@types/compression": "^1.7.1", "@types/connect-redis": "^0.0.17", "@types/express": "^4.17.13", @@ -460,6 +462,15 @@ "resolved": "https://registry.npmjs.org/@types/btoa-lite/-/btoa-lite-1.0.0.tgz", "integrity": "sha512-wJsiX1tosQ+J5+bY5LrSahHxr2wT+uME5UDwdN1kg4frt40euqA+wzECkmq4t5QbveHiJepfdThgQrPw6KiSlg==" }, + "node_modules/@types/bull": { + "version": "3.15.4", + "resolved": "https://registry.npmjs.org/@types/bull/-/bull-3.15.4.tgz", + "integrity": "sha512-vsw4kqsI/SLocfcgTUY2b8POTBUw9AwmTaehvSVnEAbcgJHsPheFMP8WKDFDk2I+aNTLRQX5iP21dPwz3CGLVA==", + "dev": true, + "dependencies": { + "@types/ioredis": "*" + } + }, "node_modules/@types/cacheable-request": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.2.tgz", @@ -1362,6 +1373,60 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, + "node_modules/bull": { + "version": "3.29.2", + "resolved": "https://registry.npmjs.org/bull/-/bull-3.29.2.tgz", + "integrity": "sha512-zWHyza/ElwVvJUqIEDJdUhGKd1V9EHjituUL7sJAmJoxS9Z7QMhYcMOWcgbUlWPgtiKN1g9ZlOtFAoq7C4/SQw==", + "dependencies": { + "cron-parser": "^2.13.0", + "debuglog": "^1.0.0", + "get-port": "^5.1.1", + "ioredis": "^4.27.0", + "lodash": "^4.17.21", + "p-timeout": "^3.2.0", + "promise.prototype.finally": "^3.1.2", + "semver": "^7.3.2", + "util.promisify": "^1.0.1", + "uuid": "^8.3.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/bull/node_modules/cron-parser": { + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-2.18.0.tgz", + "integrity": "sha512-s4odpheTyydAbTBQepsqd2rNWGa2iV3cyo8g7zbI2QQYGLVsfbhmwukayS1XHppe02Oy1fg7mg6xoaraVJeEcg==", + "dependencies": { + "is-nan": "^1.3.0", + "moment-timezone": "^0.5.31" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/bull/node_modules/semver": { + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", + "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/bull/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/bytes": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", @@ -1541,6 +1606,14 @@ "mimic-response": "^1.0.0" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", + "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -1764,6 +1837,14 @@ "ms": "^2.1.1" } }, + "node_modules/debuglog": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz", + "integrity": "sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI=", + "engines": { + "node": "*" + } + }, "node_modules/decamelize": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-4.0.0.tgz", @@ -1987,6 +2068,53 @@ "once": "^1.4.0" } }, + "node_modules/es-abstract": { + "version": "1.18.6", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.6.tgz", + "integrity": "sha512-kAeIT4cku5eNLNuUKhlmtuk1/TRZvQoYccn6TO0cSVdf1kzB0T7+dYuVK9MWM7l+/53W2Q8M7N2c6MQvhXFcUQ==", + "dependencies": { + "call-bind": "^1.0.2", + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "get-intrinsic": "^1.1.1", + "get-symbol-description": "^1.0.0", + "has": "^1.0.3", + "has-symbols": "^1.0.2", + "internal-slot": "^1.0.3", + "is-callable": "^1.2.4", + "is-negative-zero": "^2.0.1", + "is-regex": "^1.1.4", + "is-string": "^1.0.7", + "object-inspect": "^1.11.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.2", + "string.prototype.trimend": "^1.0.4", + "string.prototype.trimstart": "^1.0.4", + "unbox-primitive": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "dependencies": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/escalade": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", @@ -2242,6 +2370,14 @@ "flat": "cli.js" } }, + "node_modules/for-each": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", + "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", + "dependencies": { + "is-callable": "^1.1.3" + } + }, "node_modules/form-data": { "version": "2.5.1", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", @@ -2351,6 +2487,17 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-port": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", + "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/get-stream": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", @@ -2365,6 +2512,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/get-symbol-description": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz", + "integrity": "sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==", + "dependencies": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/glob": { "version": "7.1.7", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz", @@ -2476,6 +2638,14 @@ "node": ">= 0.4.0" } }, + "node_modules/has-bigints": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.1.tgz", + "integrity": "sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -2496,6 +2666,20 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/has-tostringtag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", + "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", + "dependencies": { + "has-symbols": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/has-yarn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/has-yarn/-/has-yarn-2.1.0.tgz", @@ -2611,6 +2795,65 @@ "integrity": "sha512-iKpRpXP+CrP2jyrxvg1kMUpXDyRUFDWurxbnVT1vQPx+Wz9uCYsMIqYuSBLV+PAaZG/d7kRLKRFc9oDMsH+mFQ==", "dev": true }, + "node_modules/internal-slot": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.3.tgz", + "integrity": "sha512-O0DB1JC/sPyZl7cIo78n5dR7eUSwwpYPiXRhTzNxZVAMUuB8vlnRFyLxdrVToks6XPLVnFfbzaVd5WLjhgg+vA==", + "dependencies": { + "get-intrinsic": "^1.1.0", + "has": "^1.0.3", + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ioredis": { + "version": "4.27.9", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.27.9.tgz", + "integrity": "sha512-hAwrx9F+OQ0uIvaJefuS3UTqW+ByOLyLIV+j0EH8ClNVxvFyH9Vmb08hCL4yje6mDYT5zMquShhypkd50RRzkg==", + "dependencies": { + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.1", + "denque": "^1.1.0", + "lodash.defaults": "^4.2.0", + "lodash.flatten": "^4.4.0", + "lodash.isarguments": "^3.1.0", + "p-map": "^2.1.0", + "redis-commands": "1.7.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/ioredis/node_modules/debug": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/ioredis/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -2619,6 +2862,17 @@ "node": ">= 0.10" } }, + "node_modules/is-bigint": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", + "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", + "dependencies": { + "has-bigints": "^1.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-binary-path": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", @@ -2631,6 +2885,32 @@ "node": ">=8" } }, + "node_modules/is-boolean-object": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", + "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", + "dependencies": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-callable": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.4.tgz", + "integrity": "sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-ci": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz", @@ -2654,6 +2934,20 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-date-object": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz", + "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==", + "dependencies": { + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-deflate": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-deflate/-/is-deflate-1.0.0.tgz", @@ -2728,6 +3022,17 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-negative-zero": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.1.tgz", + "integrity": "sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-npm": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-4.0.0.tgz", @@ -2746,6 +3051,20 @@ "node": ">=0.12.0" } }, + "node_modules/is-number-object": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.6.tgz", + "integrity": "sha512-bEVOqiRcvo3zO1+G2lVMy+gkkEm9Yh7cDMRusKKu5ZJKPUYSJwICTKZrNKHA2EbSP0Tu0+6B/emsYNHZyn6K8g==", + "dependencies": { + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-obj": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", @@ -2781,6 +3100,49 @@ "node": ">=0.10.0" } }, + "node_modules/is-regex": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", + "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==", + "dependencies": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-string": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz", + "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==", + "dependencies": { + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-symbol": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz", + "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==", + "dependencies": { + "has-symbols": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", @@ -2949,6 +3311,11 @@ "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo=" + }, "node_modules/lodash.isplainobject": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", @@ -2996,6 +3363,17 @@ "node": ">=8" } }, + "node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/luxon": { "version": "1.28.0", "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz", @@ -3193,6 +3571,25 @@ "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz", "integrity": "sha1-EUyUlnPiqKNenTV4hSeqN7Z52is=" }, + "node_modules/moment": { + "version": "2.29.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", + "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==", + "engines": { + "node": "*" + } + }, + "node_modules/moment-timezone": { + "version": "0.5.33", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.33.tgz", + "integrity": "sha512-PTc2vcT8K9J5/9rDEPe5czSIKgLoGsH8UNpA4qZTVw0Vd/Uz19geE9abbIOQKaAQFcnQ3v5YEXrbSc5BpshH+w==", + "dependencies": { + "moment": ">= 2.9.0" + }, + "engines": { + "node": "*" + } + }, "node_modules/mongodb": { "version": "3.6.11", "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.11.tgz", @@ -3461,6 +3858,14 @@ "resolved": "https://registry.npmjs.org/oauth/-/oauth-0.9.15.tgz", "integrity": "sha1-vR/vr2hslrdUda7VGWQS/2DPucE=" }, + "node_modules/object-inspect": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.11.0.tgz", + "integrity": "sha512-jp7ikS6Sd3GxQfZJPyH3cjcbJF6GZPClgdV+EFygjFLQ5FmW/dRUnTd9PQ9k0JhoNDabWFbpF1yCdSWCC6gexg==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/object-keys": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", @@ -3469,6 +3874,39 @@ "node": ">= 0.4" } }, + "node_modules/object.assign": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.2.tgz", + "integrity": "sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==", + "dependencies": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3", + "has-symbols": "^1.0.1", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/object.getownpropertydescriptors": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.2.tgz", + "integrity": "sha512-WtxeKSzfBjlzL+F9b7M7hewDzMwy+C8NRssHd1YrNlzHzIDrXcXiNOMrezdAEM4UXixgV+vvnyBeN7Rygl2ttQ==", + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.18.0-next.2" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", @@ -3512,6 +3950,14 @@ "node": ">=8" } }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=", + "engines": { + "node": ">=4" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -3542,6 +3988,25 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-map": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", + "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==", + "engines": { + "node": ">=6" + } + }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/package-json": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/package-json/-/package-json-6.5.0.tgz", @@ -3879,6 +4344,22 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, + "node_modules/promise.prototype.finally": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/promise.prototype.finally/-/promise.prototype.finally-3.1.2.tgz", + "integrity": "sha512-A2HuJWl2opDH0EafgdjwEw7HysI8ff/n4lW4QEVBCUXFk9QeGecBWv0Deph0UmLe3tTNYegz8MOjsVuE6SMoJA==", + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.0", + "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -4349,6 +4830,19 @@ "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==" }, + "node_modules/side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "dependencies": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/sift": { "version": "13.5.2", "resolved": "https://registry.npmjs.org/sift/-/sift-13.5.2.tgz", @@ -4378,6 +4872,11 @@ "memory-pager": "^1.0.2" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "node_modules/statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", @@ -4432,6 +4931,30 @@ "node": ">=8" } }, + "node_modules/string.prototype.trimend": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.4.tgz", + "integrity": "sha512-y9xCjw1P23Awk8EvTpcyL2NIr1j7wJ39f+k6lvRnSMz+mz9CGz9NYPelDk42kOz6+ql8xjfK8oYzy3jAP5QU5A==", + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/string.prototype.trimstart": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.4.tgz", + "integrity": "sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==", + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/strip-ansi": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", @@ -4812,6 +5335,20 @@ "resolved": "https://registry.npmjs.org/uid2/-/uid2-0.0.3.tgz", "integrity": "sha1-SDEm4Rd03y9xuLY53NeZw3YWK4I=" }, + "node_modules/unbox-primitive": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.1.tgz", + "integrity": "sha512-tZU/3NqK3dA5gpE1KtyiJUrEB0lxnGkMFHptJ7q6ewdZ8s12QrODwNbhIJStmJkd1QDXa1NRA8aF2A1zk/Ypyw==", + "dependencies": { + "function-bind": "^1.1.1", + "has-bigints": "^1.0.1", + "has-symbols": "^1.0.2", + "which-boxed-primitive": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/undefsafe": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.3.tgz", @@ -4939,6 +5476,21 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, + "node_modules/util.promisify": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.1.1.tgz", + "integrity": "sha512-/s3UsZUrIfa6xDhr7zZhnE9SLQ5RIXyYfiVnMMyMDzOc8WhWN4Nbh36H842OyurKbCDAesZOJaVyvmSl6fhGQw==", + "dependencies": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3", + "for-each": "^0.3.3", + "has-symbols": "^1.0.1", + "object.getownpropertydescriptors": "^2.1.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", @@ -4979,6 +5531,21 @@ "node": ">= 8" } }, + "node_modules/which-boxed-primitive": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", + "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==", + "dependencies": { + "is-bigint": "^1.0.1", + "is-boolean-object": "^1.1.0", + "is-number-object": "^1.0.4", + "is-string": "^1.0.5", + "is-symbol": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/wide-align": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", @@ -5139,6 +5706,11 @@ "node": ">=10" } }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, "node_modules/yargs": { "version": "16.2.0", "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", @@ -5584,6 +6156,15 @@ "resolved": "https://registry.npmjs.org/@types/btoa-lite/-/btoa-lite-1.0.0.tgz", "integrity": "sha512-wJsiX1tosQ+J5+bY5LrSahHxr2wT+uME5UDwdN1kg4frt40euqA+wzECkmq4t5QbveHiJepfdThgQrPw6KiSlg==" }, + "@types/bull": { + "version": "3.15.4", + "resolved": "https://registry.npmjs.org/@types/bull/-/bull-3.15.4.tgz", + "integrity": "sha512-vsw4kqsI/SLocfcgTUY2b8POTBUw9AwmTaehvSVnEAbcgJHsPheFMP8WKDFDk2I+aNTLRQX5iP21dPwz3CGLVA==", + "dev": true, + "requires": { + "@types/ioredis": "*" + } + }, "@types/cacheable-request": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.2.tgz", @@ -6370,6 +6951,47 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, + "bull": { + "version": "3.29.2", + "resolved": "https://registry.npmjs.org/bull/-/bull-3.29.2.tgz", + "integrity": "sha512-zWHyza/ElwVvJUqIEDJdUhGKd1V9EHjituUL7sJAmJoxS9Z7QMhYcMOWcgbUlWPgtiKN1g9ZlOtFAoq7C4/SQw==", + "requires": { + "cron-parser": "^2.13.0", + "debuglog": "^1.0.0", + "get-port": "^5.1.1", + "ioredis": "^4.27.0", + "lodash": "^4.17.21", + "p-timeout": "^3.2.0", + "promise.prototype.finally": "^3.1.2", + "semver": "^7.3.2", + "util.promisify": "^1.0.1", + "uuid": "^8.3.0" + }, + "dependencies": { + "cron-parser": { + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-2.18.0.tgz", + "integrity": "sha512-s4odpheTyydAbTBQepsqd2rNWGa2iV3cyo8g7zbI2QQYGLVsfbhmwukayS1XHppe02Oy1fg7mg6xoaraVJeEcg==", + "requires": { + "is-nan": "^1.3.0", + "moment-timezone": "^0.5.31" + } + }, + "semver": { + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", + "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "requires": { + "lru-cache": "^6.0.0" + } + }, + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" + } + } + }, "bytes": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", @@ -6507,6 +7129,11 @@ "mimic-response": "^1.0.0" } }, + "cluster-key-slot": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", + "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==" + }, "color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -6687,6 +7314,11 @@ "ms": "^2.1.1" } }, + "debuglog": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz", + "integrity": "sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI=" + }, "decamelize": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-4.0.0.tgz", @@ -6863,6 +7495,41 @@ "once": "^1.4.0" } }, + "es-abstract": { + "version": "1.18.6", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.6.tgz", + "integrity": "sha512-kAeIT4cku5eNLNuUKhlmtuk1/TRZvQoYccn6TO0cSVdf1kzB0T7+dYuVK9MWM7l+/53W2Q8M7N2c6MQvhXFcUQ==", + "requires": { + "call-bind": "^1.0.2", + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "get-intrinsic": "^1.1.1", + "get-symbol-description": "^1.0.0", + "has": "^1.0.3", + "has-symbols": "^1.0.2", + "internal-slot": "^1.0.3", + "is-callable": "^1.2.4", + "is-negative-zero": "^2.0.1", + "is-regex": "^1.1.4", + "is-string": "^1.0.7", + "object-inspect": "^1.11.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.2", + "string.prototype.trimend": "^1.0.4", + "string.prototype.trimstart": "^1.0.4", + "unbox-primitive": "^1.0.1" + } + }, + "es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "requires": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + } + }, "escalade": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", @@ -7062,6 +7729,14 @@ "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", "dev": true }, + "for-each": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", + "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", + "requires": { + "is-callable": "^1.1.3" + } + }, "form-data": { "version": "2.5.1", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", @@ -7132,6 +7807,11 @@ "has-symbols": "^1.0.1" } }, + "get-port": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", + "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==" + }, "get-stream": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", @@ -7140,6 +7820,15 @@ "pump": "^3.0.0" } }, + "get-symbol-description": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz", + "integrity": "sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==", + "requires": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.1.1" + } + }, "glob": { "version": "7.1.7", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz", @@ -7221,6 +7910,11 @@ "function-bind": "^1.1.1" } }, + "has-bigints": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.1.tgz", + "integrity": "sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==" + }, "has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -7232,6 +7926,14 @@ "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==" }, + "has-tostringtag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", + "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", + "requires": { + "has-symbols": "^1.0.2" + } + }, "has-yarn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/has-yarn/-/has-yarn-2.1.0.tgz", @@ -7328,11 +8030,62 @@ "integrity": "sha512-iKpRpXP+CrP2jyrxvg1kMUpXDyRUFDWurxbnVT1vQPx+Wz9uCYsMIqYuSBLV+PAaZG/d7kRLKRFc9oDMsH+mFQ==", "dev": true }, + "internal-slot": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.3.tgz", + "integrity": "sha512-O0DB1JC/sPyZl7cIo78n5dR7eUSwwpYPiXRhTzNxZVAMUuB8vlnRFyLxdrVToks6XPLVnFfbzaVd5WLjhgg+vA==", + "requires": { + "get-intrinsic": "^1.1.0", + "has": "^1.0.3", + "side-channel": "^1.0.4" + } + }, + "ioredis": { + "version": "4.27.9", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.27.9.tgz", + "integrity": "sha512-hAwrx9F+OQ0uIvaJefuS3UTqW+ByOLyLIV+j0EH8ClNVxvFyH9Vmb08hCL4yje6mDYT5zMquShhypkd50RRzkg==", + "requires": { + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.1", + "denque": "^1.1.0", + "lodash.defaults": "^4.2.0", + "lodash.flatten": "^4.4.0", + "lodash.isarguments": "^3.1.0", + "p-map": "^2.1.0", + "redis-commands": "1.7.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "dependencies": { + "debug": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" }, + "is-bigint": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", + "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", + "requires": { + "has-bigints": "^1.0.1" + } + }, "is-binary-path": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", @@ -7342,6 +8095,20 @@ "binary-extensions": "^2.0.0" } }, + "is-boolean-object": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", + "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", + "requires": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + } + }, + "is-callable": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.4.tgz", + "integrity": "sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w==" + }, "is-ci": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz", @@ -7359,6 +8126,14 @@ "has": "^1.0.3" } }, + "is-date-object": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz", + "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==", + "requires": { + "has-tostringtag": "^1.0.0" + } + }, "is-deflate": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-deflate/-/is-deflate-1.0.0.tgz", @@ -7409,6 +8184,11 @@ "define-properties": "^1.1.3" } }, + "is-negative-zero": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.1.tgz", + "integrity": "sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==" + }, "is-npm": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-4.0.0.tgz", @@ -7421,6 +8201,14 @@ "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", "dev": true }, + "is-number-object": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.6.tgz", + "integrity": "sha512-bEVOqiRcvo3zO1+G2lVMy+gkkEm9Yh7cDMRusKKu5ZJKPUYSJwICTKZrNKHA2EbSP0Tu0+6B/emsYNHZyn6K8g==", + "requires": { + "has-tostringtag": "^1.0.0" + } + }, "is-obj": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", @@ -7444,6 +8232,31 @@ "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==" }, + "is-regex": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", + "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==", + "requires": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + } + }, + "is-string": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz", + "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==", + "requires": { + "has-tostringtag": "^1.0.0" + } + }, + "is-symbol": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz", + "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==", + "requires": { + "has-symbols": "^1.0.2" + } + }, "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", @@ -7584,6 +8397,11 @@ "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" }, + "lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo=" + }, "lodash.isplainobject": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", @@ -7619,6 +8437,14 @@ "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz", "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==" }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "requires": { + "yallist": "^4.0.0" + } + }, "luxon": { "version": "1.28.0", "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz", @@ -7767,6 +8593,19 @@ "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz", "integrity": "sha1-EUyUlnPiqKNenTV4hSeqN7Z52is=" }, + "moment": { + "version": "2.29.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", + "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==" + }, + "moment-timezone": { + "version": "0.5.33", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.33.tgz", + "integrity": "sha512-PTc2vcT8K9J5/9rDEPe5czSIKgLoGsH8UNpA4qZTVw0Vd/Uz19geE9abbIOQKaAQFcnQ3v5YEXrbSc5BpshH+w==", + "requires": { + "moment": ">= 2.9.0" + } + }, "mongodb": { "version": "3.6.11", "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.11.tgz", @@ -7940,11 +8779,37 @@ "resolved": "https://registry.npmjs.org/oauth/-/oauth-0.9.15.tgz", "integrity": "sha1-vR/vr2hslrdUda7VGWQS/2DPucE=" }, + "object-inspect": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.11.0.tgz", + "integrity": "sha512-jp7ikS6Sd3GxQfZJPyH3cjcbJF6GZPClgdV+EFygjFLQ5FmW/dRUnTd9PQ9k0JhoNDabWFbpF1yCdSWCC6gexg==" + }, "object-keys": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" }, + "object.assign": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.2.tgz", + "integrity": "sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==", + "requires": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3", + "has-symbols": "^1.0.1", + "object-keys": "^1.1.1" + } + }, + "object.getownpropertydescriptors": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.2.tgz", + "integrity": "sha512-WtxeKSzfBjlzL+F9b7M7hewDzMwy+C8NRssHd1YrNlzHzIDrXcXiNOMrezdAEM4UXixgV+vvnyBeN7Rygl2ttQ==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.18.0-next.2" + } + }, "on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", @@ -7976,6 +8841,11 @@ "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.1.1.tgz", "integrity": "sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg==" }, + "p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=" + }, "p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -7994,6 +8864,19 @@ "p-limit": "^3.0.2" } }, + "p-map": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", + "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==" + }, + "p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "requires": { + "p-finally": "^1.0.0" + } + }, "package-json": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/package-json/-/package-json-6.5.0.tgz", @@ -8252,6 +9135,16 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, + "promise.prototype.finally": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/promise.prototype.finally/-/promise.prototype.finally-3.1.2.tgz", + "integrity": "sha512-A2HuJWl2opDH0EafgdjwEw7HysI8ff/n4lW4QEVBCUXFk9QeGecBWv0Deph0UmLe3tTNYegz8MOjsVuE6SMoJA==", + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.0", + "function-bind": "^1.1.1" + } + }, "proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -8642,6 +9535,16 @@ "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==" }, + "side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "requires": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + } + }, "sift": { "version": "13.5.2", "resolved": "https://registry.npmjs.org/sift/-/sift-13.5.2.tgz", @@ -8671,6 +9574,11 @@ "memory-pager": "^1.0.2" } }, + "standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", @@ -8707,6 +9615,24 @@ "strip-ansi": "^6.0.0" } }, + "string.prototype.trimend": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.4.tgz", + "integrity": "sha512-y9xCjw1P23Awk8EvTpcyL2NIr1j7wJ39f+k6lvRnSMz+mz9CGz9NYPelDk42kOz6+ql8xjfK8oYzy3jAP5QU5A==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" + } + }, + "string.prototype.trimstart": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.4.tgz", + "integrity": "sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==", + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" + } + }, "strip-ansi": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", @@ -8987,6 +9913,17 @@ "resolved": "https://registry.npmjs.org/uid2/-/uid2-0.0.3.tgz", "integrity": "sha1-SDEm4Rd03y9xuLY53NeZw3YWK4I=" }, + "unbox-primitive": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.1.tgz", + "integrity": "sha512-tZU/3NqK3dA5gpE1KtyiJUrEB0lxnGkMFHptJ7q6ewdZ8s12QrODwNbhIJStmJkd1QDXa1NRA8aF2A1zk/Ypyw==", + "requires": { + "function-bind": "^1.1.1", + "has-bigints": "^1.0.1", + "has-symbols": "^1.0.2", + "which-boxed-primitive": "^1.0.2" + } + }, "undefsafe": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.3.tgz", @@ -9097,6 +10034,18 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, + "util.promisify": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.1.1.tgz", + "integrity": "sha512-/s3UsZUrIfa6xDhr7zZhnE9SLQ5RIXyYfiVnMMyMDzOc8WhWN4Nbh36H842OyurKbCDAesZOJaVyvmSl6fhGQw==", + "requires": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3", + "for-each": "^0.3.3", + "has-symbols": "^1.0.1", + "object.getownpropertydescriptors": "^2.1.1" + } + }, "utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", @@ -9121,6 +10070,18 @@ "isexe": "^2.0.0" } }, + "which-boxed-primitive": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", + "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==", + "requires": { + "is-bigint": "^1.0.1", + "is-boolean-object": "^1.1.0", + "is-number-object": "^1.0.4", + "is-string": "^1.0.5", + "is-symbol": "^1.0.3" + } + }, "wide-align": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", @@ -9252,6 +10213,11 @@ "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", "dev": true }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, "yargs": { "version": "16.2.0", "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", diff --git a/package.json b/package.json index ad0ddda..346e44b 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "archive-stream-to-s3": "^1.1.3", "archiver": "^5.3.0", "aws-sdk": "^2.958.0", + "bull": "^3.29.2", "compression": "^1.7.4", "connect-redis": "^6.0.0", "dotenv": "^10.0.0", @@ -54,6 +55,7 @@ }, "devDependencies": { "@types/archiver": "^5.1.1", + "@types/bull": "^3.15.4", "@types/compression": "^1.7.1", "@types/connect-redis": "^0.0.17", "@types/express": "^4.17.13", diff --git a/public/script/app.js b/public/script/app.js index 6b9231d..4e23d0b 100644 --- a/public/script/app.js +++ b/public/script/app.js @@ -634,11 +634,22 @@ angular body: `The repository ${repo.repoId} is going to be removed.`, }; $scope.toasts.push(toast); - $http.delete(`/api/repo/${repo.repoId}`).then(() => { - toast.title = `${repo.repoId} is removed.`; - toast.body = `The repository ${repo.repoId} is removed.`; - getRepositories(); - }); + $http.delete(`/api/repo/${repo.repoId}`).then( + () => { + setTimeout(() => { + toast.title = `${repo.repoId} is removed.`; + toast.body = `The repository ${repo.repoId} is removed.`; + getRepositories(); + $scope.$apply(); + }, 5000); + }, + (error) => { + toast.title = `Error during the removal of ${repo.repoId}.`; + toast.body = error.body; + + getRepositories(); + } + ); } }; @@ -646,16 +657,26 @@ angular const toast = { title: `Refreshing ${repo.repoId}...`, date: new Date(), - body: `The repository ${repo.repoId} is going to be removed.`, + body: `The repository ${repo.repoId} is going to be refreshed.`, }; $scope.toasts.push(toast); - $http.post(`/api/repo/${repo.repoId}/refresh`).then(() => { - toast.title = `${repo.repoId} is refreshed.`; - toast.body = `The repository ${repo.repoId} is refreshed.`; + $http.post(`/api/repo/${repo.repoId}/refresh`).then( + () => { + setTimeout(() => { + toast.title = `${repo.repoId} is refreshed.`; + toast.body = `The repository ${repo.repoId} is refreshed.`; + getRepositories(); + $scope.$apply(); + }, 5000); + }, + (error) => { + toast.title = `Error during the refresh of ${repo.repoId}.`; + toast.body = error.body; - getRepositories(); - }); + getRepositories(); + } + ); }; $scope.repoFiler = (repo) => { diff --git a/src/Repository.ts b/src/Repository.ts index cb9bc3e..d2a758a 100644 --- a/src/Repository.ts +++ b/src/Repository.ts @@ -14,6 +14,7 @@ import GitHubBase from "./source/GitHubBase"; import Conference from "./Conference"; import ConferenceModel from "./database/conference/conferences.model"; import AnonymousError from "./AnonymousError"; +import { downloadQueue } from "./queue"; export default class Repository { private _model: IAnonymizedRepositoryDocument; @@ -103,16 +104,21 @@ export default class Repository { this.expire(); } } - if (this.status == "expired") { - throw new AnonymousError("repository_expired", this); - } - if (this.status == "removed") { + if ( + this.status == "expired" || + this.status == "expiring" || + this.status == "removing" || + this.status == "removed" + ) { throw new AnonymousError("repository_expired", this); } const fiveMinuteAgo = new Date(); fiveMinuteAgo.setMinutes(fiveMinuteAgo.getMinutes() - 5); - if (this.status == "download" && this._model.statusDate > fiveMinuteAgo) { + if ( + this.status == "preparing" || + (this.status == "download" && this._model.statusDate > fiveMinuteAgo) + ) { throw new AnonymousError("repository_not_ready", this); } } @@ -136,17 +142,31 @@ export default class Repository { * @returns void */ async updateIfNeeded(): Promise { + if ( + this.status == "expired" || + this.status == "expiring" || + this.status == "removing" || + this.status == "removed" + ) { + throw new AnonymousError("repository_expired", this); + } + + const fiveMinuteAgo = new Date(); + fiveMinuteAgo.setMinutes(fiveMinuteAgo.getMinutes() - 5); + if (this.status != "ready") { + if ( + this._model.statusDate < fiveMinuteAgo && + this.status != "preparing" + ) { + await this.updateStatus("preparing"); + await downloadQueue.add(this, { jobId: this.repoId }); + } + throw new AnonymousError("repository_not_ready", this); + } + const yesterday = new Date(); yesterday.setDate(yesterday.getDate() - 1); - if (this._model.options.update && this._model.lastView < yesterday) { - - const fiveMinuteAgo = new Date(); - fiveMinuteAgo.setMinutes(fiveMinuteAgo.getMinutes() - 5); - if (this.status == "download" && this._model.statusDate > fiveMinuteAgo) { - throw new AnonymousError("repository_not_ready", this); - } - // Only GitHubBase can be update for the moment if (this.source instanceof GitHubBase) { const branches = await this.source.githubRepository.branches({ @@ -154,30 +174,25 @@ export default class Repository { accessToken: await this.source.getToken(), }); const branch = this.source.branch; - if ( - branch.commit == - branches.filter((f) => f.name == branch.name)[0]?.commit - ) { + const newCommit = branches.filter((f) => f.name == branch.name)[0] + ?.commit; + if (branch.commit == newCommit) { console.log(`${this._model.repoId} is up to date`); return; } - this._model.source.commit = branches.filter( - (f) => f.name == branch.name - )[0]?.commit; - branch.commit = this._model.source.commit; + this._model.source.commit = newCommit; + branch.commit = newCommit; - if (!this._model.source.commit) { + if (!newCommit) { console.error( `${branch.name} for ${this.source.githubRepository.fullName} is not found` ); throw new AnonymousError("branch_not_found", this); } this._model.anonymizeDate = new Date(); - console.log( - `${this._model.repoId} will be updated to ${this._model.source.commit}` - ); + console.log(`${this._model.repoId} will be updated to ${newCommit}`); await this.resetSate("preparing"); - await this.anonymize(); + await downloadQueue.add(this, { jobId: this.repoId }); } } } @@ -219,14 +234,18 @@ export default class Repository { * Expire the repository */ async expire() { - return this.resetSate("expired"); + await this.updateStatus("expiring"); + await this.resetSate(); + await this.updateStatus("expired"); } /** * Remove the repository */ async remove() { - return this.resetSate("removed"); + await this.updateStatus("removing"); + await this.resetSate(); + await this.updateStatus("removed"); } /** @@ -234,8 +253,10 @@ export default class Repository { */ async resetSate(status?: RepositoryStatus) { if (status) this._model.status = status; + // remove attribute this._model.size = { storage: 0, file: 0 }; this._model.originalFiles = null; + // remove cache return Promise.all([ this._model.save(), storage.rm(this._model.repoId + "/"), diff --git a/src/queue.ts b/src/queue.ts new file mode 100644 index 0000000..02e4097 --- /dev/null +++ b/src/queue.ts @@ -0,0 +1,50 @@ +import * as Queue from "bull"; +import config from "../config"; +import { getRepository } from "./database/database"; +import Repository from "./Repository"; + +export const removeQueue = new Queue("repository removal", { + redis: { + host: config.REDIS_HOSTNAME, + port: config.REDIS_PORT, + }, +}); +removeQueue.on("completed", async (job) => { + await job.remove(); +}); +export const downloadQueue = new Queue("repository download", { + redis: { + host: config.REDIS_HOSTNAME, + port: config.REDIS_PORT, + }, +}); +downloadQueue.on("completed", async (job) => { + await job.remove(); +}); + +removeQueue.process(5, async (job) => { + console.log(`${job.data.repoId} is going to be removed`); + try { + const repo = await getRepository(job.data.repoId); + await repo.remove(); + } catch (error) { + console.log("error", error); + } finally { + console.log(`${job.data.repoId} is removed`); + } +}); + +downloadQueue.process(2, async (job) => { + console.log(`${job.data.repoId} is going to be downloaded`); + try { + const repo = await getRepository(job.data.repoId); + job.progress("get_repo"); + await repo.remove(); + job.progress("get_remove"); + await repo.anonymize(); + } catch (error) { + console.log("error", error); + } finally { + console.log(`${job.data.repoId} is downloaded`); + } +}); diff --git a/src/routes/repository-private.ts b/src/routes/repository-private.ts index e01e342..5d15f5e 100644 --- a/src/routes/repository-private.ts +++ b/src/routes/repository-private.ts @@ -12,6 +12,7 @@ import { IAnonymizedRepositoryDocument } from "../database/anonymizedRepositorie import Repository from "../Repository"; import ConferenceModel from "../database/conference/conferences.model"; import AnonymousError from "../AnonymousError"; +import { downloadQueue, removeQueue } from "../queue"; const router = express.Router(); @@ -65,12 +66,15 @@ router.post( const repo = await getRepo(req, res, { nocheck: true }); if (!repo) return; + if (repo.status == "preparing" || repo.status == "removing") return; + const user = await getUser(req); if (repo.owner.id != user.id) { return res.status(401).json({ error: "not_authorized" }); } - await repo.anonymize(); - res.end("ok"); + await repo.updateStatus("preparing"); + await downloadQueue.add(repo, {jobId: repo.repoId}); + res.json({ status: repo.status }); } catch (error) { handleError(error, res); } @@ -83,14 +87,16 @@ router.delete( async (req: express.Request, res: express.Response) => { const repo = await getRepo(req, res, { nocheck: true }); if (!repo) return; + // if (repo.status == "removing") return res.json({ status: repo.status }); try { + if (repo.status == "removed") throw new AnonymousError("is_removed"); const user = await getUser(req); if (repo.owner.id != user.id) { return res.status(401).json({ error: "not_authorized" }); } - await repo.remove(); - console.log(`${req.params.repoId} is removed`); - return res.json("ok"); + await repo.updateStatus("removing"); + await removeQueue.add(repo, {jobId: repo.repoId}); + return res.json({ status: repo.status }); } catch (error) { handleError(error, res); } @@ -305,8 +311,8 @@ router.post( } repo.model.conference = repoUpdate.conference; await repo.updateStatus("preparing"); - res.send("ok"); - new Repository(repo.model).anonymize(); + res.json({ status: repo.status }); + await downloadQueue.add(repo, {jobId: repo.repoId}); } catch (error) { return handleError(error, res); } @@ -369,11 +375,14 @@ router.post("/", async (req: express.Request, res: express.Response) => { } } - res.send("ok"); - new Repository(repo).anonymize(); + res.send({ status: repo.status }); + downloadQueue.add(new Repository(repo), {jobId: repo.repoId}); } catch (error) { if (error.message?.indexOf(" duplicate key") > -1) { - return handleError(new AnonymousError("repoId_already_used", repoUpdate.repoId), res); + return handleError( + new AnonymousError("repoId_already_used", repoUpdate.repoId), + res + ); } return handleError(error, res); } diff --git a/src/routes/repository-public.ts b/src/routes/repository-public.ts index be65bf4..46149e4 100644 --- a/src/routes/repository-public.ts +++ b/src/routes/repository-public.ts @@ -3,7 +3,6 @@ import * as express from "express"; import * as stream from "stream"; import config from "../../config"; - import { getRepo, handleError } from "./route-utils"; const router = express.Router(); @@ -23,7 +22,7 @@ router.get( // cache the file for 6 hours res.header("Cache-Control", "max-age=21600000"); - await pipeline(repo.zip(), res) + await pipeline(repo.zip(), res); } catch (error) { handleError(error, res); } @@ -60,7 +59,6 @@ router.get( ) { redirectURL = repo.source.url; } else { - repo.check(); await repo.updateIfNeeded(); } diff --git a/src/types.ts b/src/types.ts index 10bebdb..671f210 100644 --- a/src/types.ts +++ b/src/types.ts @@ -115,7 +115,9 @@ export type RepositoryStatus = | "download" | "ready" | "expired" + | "expiring" | "removed" + | "removing" | "error"; export type ConferenceStatus = "ready" | "expired" | "removed";