CalvinBackup/anonymous_github
1
0
Fork 0
mirror of https://github.com/tdurieux/anonymous_github.git synced 2026-02-12 18:32:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: replace bull by bullmq

Browse Source
This commit is contained in:
tdurieux
2022-07-22 13:06:22 +02:00
parent fa8a513d93
commit ac52999841
19 changed files with 481 additions and 367 deletions
Download Patch File Download Diff File Expand all files Collapse all files

536
package-lock.json generated
View File

@@ -14,7 +14,7 @@
"@pm2/io": "^5.0.0",
"archiver": "^5.3.1",
"aws-sdk": "^2.1180.0",
"bull": "^3.29.2",
"bullmq": "^1.86.7",
"compression": "^1.7.4",
"connect-redis": "^6.1.3",
"decompress-stream-to-s3": "^1.2.1",
@@ -39,7 +39,6 @@
},
"devDependencies": {
"@types/archiver": "^5.3.1",
"@types/bull": "^3.15.4",
"@types/compression": "^1.7.1",
"@types/connect-redis": "^0.0.18",
"@types/express": "^4.17.13",
@@ -100,6 +99,78 @@
"@jridgewell/sourcemap-codec": "^1.4.10"
}
},
"node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-2.0.2.tgz",
"integrity": "sha512-FMX5i7a+ojIguHpWbzh5MCsCouJkwf4z4ejdUY/fsgB9Vkdak4ZnoIEskOyOUMMB4lctiZFGszFQJXUeFL8tRg==",
"cpu": [
"arm64"
],
"optional": true,
"os": [
"darwin"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-2.0.2.tgz",
"integrity": "sha512-DznYtF3lHuZDSRaIOYeif4JgO0NtO2Xf8DsngAugMx/bUdTFbg86jDTmkVJBNmV+cxszz6OjGvinnS8AbJ342g==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"darwin"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-2.0.2.tgz",
"integrity": "sha512-Gy9+c3Wj+rUlD3YvCZTi92gs+cRX7ZQogtwq0IhRenloTTlsbpezNgk6OCkt59V4ATEWSic9rbU92H/l7XsRvA==",
"cpu": [
"arm"
],
"optional": true,
"os": [
"linux"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-2.0.2.tgz",
"integrity": "sha512-b0jMEo566YdM2K+BurSed7bswjo3a6bcdw5ETqoIfSuxKuRLPfAiOjVbZyZBgx3J/TAM/QrvEQ/VN89A0ZAxSg==",
"cpu": [
"arm64"
],
"optional": true,
"os": [
"linux"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-2.0.2.tgz",
"integrity": "sha512-zrBHaePwcv4cQXxzYgNj0+A8I1uVN97E7/3LmkRocYZ+rMwUsnPpp4RuTAHSRoKlTQV3nSdCQW4Qdt4MXw/iHw==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"linux"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-2.0.2.tgz",
"integrity": "sha512-fpnI00dt+yO1cKx9qBXelKhPBdEgvc8ZPav1+0r09j0woYQU2N79w/jcGawSY5UGlgQ3vjaJsFHnGbGvvqdLzg==",
"cpu": [
"x64"
],
"optional": true,
"os": [
"win32"
]
},
"node_modules/@octokit/auth-oauth-app": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/@octokit/auth-oauth-app/-/auth-oauth-app-4.3.0.tgz",
@@ -508,15 +579,6 @@
"resolved": "https://registry.npmjs.org/@types/btoa-lite/-/btoa-lite-1.0.0.tgz",
"integrity": "sha512-wJsiX1tosQ+J5+bY5LrSahHxr2wT+uME5UDwdN1kg4frt40euqA+wzECkmq4t5QbveHiJepfdThgQrPw6KiSlg=="
},
"node_modules/@types/bull": {
"version": "3.15.4",
"resolved": "https://registry.npmjs.org/@types/bull/-/bull-3.15.4.tgz",
"integrity": "sha512-vsw4kqsI/SLocfcgTUY2b8POTBUw9AwmTaehvSVnEAbcgJHsPheFMP8WKDFDk2I+aNTLRQX5iP21dPwz3CGLVA==",
"dev": true,
"dependencies": {
"@types/ioredis": "*"
}
},
"node_modules/@types/cacheable-request": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.2.tgz",
@@ -1302,30 +1364,56 @@
"node": ">=0.2.0"
}
},
"node_modules/bull": {
"version": "3.29.2",
"resolved": "https://registry.npmjs.org/bull/-/bull-3.29.2.tgz",
"integrity": "sha512-zWHyza/ElwVvJUqIEDJdUhGKd1V9EHjituUL7sJAmJoxS9Z7QMhYcMOWcgbUlWPgtiKN1g9ZlOtFAoq7C4/SQw==",
"node_modules/bullmq": {
"version": "1.86.7",
"resolved": "https://registry.npmjs.org/bullmq/-/bullmq-1.86.7.tgz",
"integrity": "sha512-4fsIcoMmdRmPvll/dV205wfKJlx7jfn1zX1AzO2t56juxiVrbXv5MtYLYzFgx8FW4CnyWzhxXMqJSdTmdxxJ+g==",
"dependencies": {
"cron-parser": "^2.13.0",
"debuglog": "^1.0.0",
"cron-parser": "^4.2.1",
"get-port": "^5.1.1",
"ioredis": "^4.27.0",
"glob": "^7.2.0",
"ioredis": "^4.28.5",
"lodash": "^4.17.21",
"p-timeout": "^3.2.0",
"promise.prototype.finally": "^3.1.2",
"semver": "^7.3.2",
"util.promisify": "^1.0.1",
"uuid": "^8.3.0"
},
"engines": {
"node": ">=10"
"msgpackr": "^1.4.6",
"semver": "^7.3.7",
"tslib": "^1.14.1",
"uuid": "^8.3.2"
}
},
"node_modules/bull/node_modules/semver": {
"version": "7.3.5",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
"integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==",
"node_modules/bullmq/node_modules/glob": {
"version": "7.2.3",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
"integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
"dependencies": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^3.1.1",
"once": "^1.3.0",
"path-is-absolute": "^1.0.0"
},
"engines": {
"node": "*"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/bullmq/node_modules/minimatch": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
"dependencies": {
"brace-expansion": "^1.1.7"
},
"engines": {
"node": "*"
}
},
"node_modules/bullmq/node_modules/semver": {
"version": "7.3.7",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.7.tgz",
"integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==",
"dependencies": {
"lru-cache": "^6.0.0"
},
@@ -1336,7 +1424,12 @@
"node": ">=10"
}
},
"node_modules/bull/node_modules/uuid": {
"node_modules/bullmq/node_modules/tslib": {
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
"integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="
},
"node_modules/bullmq/node_modules/uuid": {
"version": "8.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
"integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
@@ -1699,15 +1792,22 @@
"dev": true
},
"node_modules/cron-parser": {
"version": "2.18.0",
"resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-2.18.0.tgz",
"integrity": "sha512-s4odpheTyydAbTBQepsqd2rNWGa2iV3cyo8g7zbI2QQYGLVsfbhmwukayS1XHppe02Oy1fg7mg6xoaraVJeEcg==",
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.5.0.tgz",
"integrity": "sha512-QHdeh3zLWz6YvYTFKpRb860rJlip16pEinbxXT1i2NZB/nOxBjd2RbSv54sn5UrAj9WykiSLYWWDgo8azQK0HA==",
"dependencies": {
"is-nan": "^1.3.0",
"moment-timezone": "^0.5.31"
"luxon": "^2.4.0"
},
"engines": {
"node": ">=0.8"
"node": ">=12.0.0"
}
},
"node_modules/cron-parser/node_modules/luxon": {
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-2.5.0.tgz",
"integrity": "sha512-IDkEPB80Rb6gCAU+FEib0t4FeJ4uVOuX1CQ9GsvU3O+JAGIgu0J7sf1OarXKaKDygTZIoJyU6YdZzTFRu+YR0A==",
"engines": {
"node": ">=12"
}
},
"node_modules/debug": {
@@ -1726,14 +1826,6 @@
}
}
},
"node_modules/debuglog": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz",
"integrity": "sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI=",
"engines": {
"node": "*"
}
},
"node_modules/decamelize": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/decamelize/-/decamelize-4.0.0.tgz",
@@ -2686,9 +2778,9 @@
}
},
"node_modules/ioredis": {
"version": "4.27.9",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.27.9.tgz",
"integrity": "sha512-hAwrx9F+OQ0uIvaJefuS3UTqW+ByOLyLIV+j0EH8ClNVxvFyH9Vmb08hCL4yje6mDYT5zMquShhypkd50RRzkg==",
"version": "4.28.5",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz",
"integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==",
"dependencies": {
"cluster-key-slot": "^1.1.0",
"debug": "^4.3.1",
@@ -3160,7 +3252,7 @@
"node_modules/lodash.isarguments": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
"integrity": "sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo="
"integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg=="
},
"node_modules/lodash.isplainobject": {
"version": "4.0.6",
@@ -3404,25 +3496,6 @@
"resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz",
"integrity": "sha1-EUyUlnPiqKNenTV4hSeqN7Z52is="
},
"node_modules/moment": {
"version": "2.29.3",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.29.3.tgz",
"integrity": "sha512-c6YRvhEo//6T2Jz/vVtYzqBzwvPT95JBQ+smCytzf7c50oMZRsR/a4w88aD34I+/QVSfnoAnSBFPJHItlOMJVw==",
"engines": {
"node": "*"
}
},
"node_modules/moment-timezone": {
"version": "0.5.33",
"resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.33.tgz",
"integrity": "sha512-PTc2vcT8K9J5/9rDEPe5czSIKgLoGsH8UNpA4qZTVw0Vd/Uz19geE9abbIOQKaAQFcnQ3v5YEXrbSc5BpshH+w==",
"dependencies": {
"moment": ">= 2.9.0"
},
"engines": {
"node": "*"
}
},
"node_modules/mongodb": {
"version": "3.6.11",
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.11.tgz",
@@ -3557,6 +3630,32 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"node_modules/msgpackr": {
"version": "1.6.1",
"resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.6.1.tgz",
"integrity": "sha512-Je+xBEfdjtvA4bKaOv8iRhjC8qX2oJwpYH4f7JrG4uMVJVmnmkAT4pjKdbztKprGj3iwjcxPzb5umVZ02Qq3tA==",
"optionalDependencies": {
"msgpackr-extract": "^2.0.2"
}
},
"node_modules/msgpackr-extract": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-2.0.2.tgz",
"integrity": "sha512-coskCeJG2KDny23zWeu+6tNy7BLnAiOGgiwzlgdm4oeSsTpqEJJPguHIuKZcCdB7tzhZbXNYSg6jZAXkZErkJA==",
"hasInstallScript": true,
"optional": true,
"dependencies": {
"node-gyp-build-optional-packages": "5.0.2"
},
"optionalDependencies": {
"@msgpackr-extract/msgpackr-extract-darwin-arm64": "2.0.2",
"@msgpackr-extract/msgpackr-extract-darwin-x64": "2.0.2",
"@msgpackr-extract/msgpackr-extract-linux-arm": "2.0.2",
"@msgpackr-extract/msgpackr-extract-linux-arm64": "2.0.2",
"@msgpackr-extract/msgpackr-extract-linux-x64": "2.0.2",
"@msgpackr-extract/msgpackr-extract-win32-x64": "2.0.2"
}
},
"node_modules/nanoid": {
"version": "3.1.23",
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.23.tgz",
@@ -3596,6 +3695,17 @@
}
}
},
"node_modules/node-gyp-build-optional-packages": {
"version": "5.0.2",
"resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.2.tgz",
"integrity": "sha512-PiN4NWmlQPqvbEFcH/omQsswWQbe5Z9YK/zdB23irp5j2XibaA2IrGvpSWmVVG4qMZdmPdwPctSy4a86rOMn6g==",
"optional": true,
"bin": {
"node-gyp-build-optional": "optional.js",
"node-gyp-build-optional-packages": "bin.js",
"node-gyp-build-test": "build-test.js"
}
},
"node_modules/node-schedule": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/node-schedule/-/node-schedule-2.1.0.tgz",
@@ -3678,22 +3788,6 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/object.getownpropertydescriptors": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.2.tgz",
"integrity": "sha512-WtxeKSzfBjlzL+F9b7M7hewDzMwy+C8NRssHd1YrNlzHzIDrXcXiNOMrezdAEM4UXixgV+vvnyBeN7Rygl2ttQ==",
"dependencies": {
"call-bind": "^1.0.2",
"define-properties": "^1.1.3",
"es-abstract": "^1.18.0-next.2"
},
"engines": {
"node": ">= 0.8"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/on-finished": {
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
@@ -3737,14 +3831,6 @@
"node": ">=8"
}
},
"node_modules/p-finally": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
"integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=",
"engines": {
"node": ">=4"
}
},
"node_modules/p-limit": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
@@ -3783,17 +3869,6 @@
"node": ">=6"
}
},
"node_modules/p-timeout": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz",
"integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==",
"dependencies": {
"p-finally": "^1.0.0"
},
"engines": {
"node": ">=8"
}
},
"node_modules/pako": {
"version": "0.2.9",
"resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz",
@@ -3952,22 +4027,6 @@
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"node_modules/promise.prototype.finally": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/promise.prototype.finally/-/promise.prototype.finally-3.1.2.tgz",
"integrity": "sha512-A2HuJWl2opDH0EafgdjwEw7HysI8ff/n4lW4QEVBCUXFk9QeGecBWv0Deph0UmLe3tTNYegz8MOjsVuE6SMoJA==",
"dependencies": {
"define-properties": "^1.1.3",
"es-abstract": "^1.17.0-next.0",
"function-bind": "^1.1.1"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/proxy-addr": {
"version": "2.0.7",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
@@ -4159,7 +4218,7 @@
"node_modules/redis-errors": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
"integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=",
"integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==",
"engines": {
"node": ">=4"
}
@@ -4167,7 +4226,7 @@
"node_modules/redis-parser": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
"integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=",
"integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==",
"dependencies": {
"redis-errors": "^1.0.0"
},
@@ -4816,21 +4875,6 @@
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
},
"node_modules/util.promisify": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.1.1.tgz",
"integrity": "sha512-/s3UsZUrIfa6xDhr7zZhnE9SLQ5RIXyYfiVnMMyMDzOc8WhWN4Nbh36H842OyurKbCDAesZOJaVyvmSl6fhGQw==",
"dependencies": {
"call-bind": "^1.0.0",
"define-properties": "^1.1.3",
"for-each": "^0.3.3",
"has-symbols": "^1.0.1",
"object.getownpropertydescriptors": "^2.1.1"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/utils-merge": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
@@ -5178,6 +5222,42 @@
"@jridgewell/sourcemap-codec": "^1.4.10"
}
},
"@msgpackr-extract/msgpackr-extract-darwin-arm64": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-2.0.2.tgz",
"integrity": "sha512-FMX5i7a+ojIguHpWbzh5MCsCouJkwf4z4ejdUY/fsgB9Vkdak4ZnoIEskOyOUMMB4lctiZFGszFQJXUeFL8tRg==",
"optional": true
},
"@msgpackr-extract/msgpackr-extract-darwin-x64": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-2.0.2.tgz",
"integrity": "sha512-DznYtF3lHuZDSRaIOYeif4JgO0NtO2Xf8DsngAugMx/bUdTFbg86jDTmkVJBNmV+cxszz6OjGvinnS8AbJ342g==",
"optional": true
},
"@msgpackr-extract/msgpackr-extract-linux-arm": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-2.0.2.tgz",
"integrity": "sha512-Gy9+c3Wj+rUlD3YvCZTi92gs+cRX7ZQogtwq0IhRenloTTlsbpezNgk6OCkt59V4ATEWSic9rbU92H/l7XsRvA==",
"optional": true
},
"@msgpackr-extract/msgpackr-extract-linux-arm64": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-2.0.2.tgz",
"integrity": "sha512-b0jMEo566YdM2K+BurSed7bswjo3a6bcdw5ETqoIfSuxKuRLPfAiOjVbZyZBgx3J/TAM/QrvEQ/VN89A0ZAxSg==",
"optional": true
},
"@msgpackr-extract/msgpackr-extract-linux-x64": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-2.0.2.tgz",
"integrity": "sha512-zrBHaePwcv4cQXxzYgNj0+A8I1uVN97E7/3LmkRocYZ+rMwUsnPpp4RuTAHSRoKlTQV3nSdCQW4Qdt4MXw/iHw==",
"optional": true
},
"@msgpackr-extract/msgpackr-extract-win32-x64": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-2.0.2.tgz",
"integrity": "sha512-fpnI00dt+yO1cKx9qBXelKhPBdEgvc8ZPav1+0r09j0woYQU2N79w/jcGawSY5UGlgQ3vjaJsFHnGbGvvqdLzg==",
"optional": true
},
"@octokit/auth-oauth-app": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/@octokit/auth-oauth-app/-/auth-oauth-app-4.3.0.tgz",
@@ -5542,15 +5622,6 @@
"resolved": "https://registry.npmjs.org/@types/btoa-lite/-/btoa-lite-1.0.0.tgz",
"integrity": "sha512-wJsiX1tosQ+J5+bY5LrSahHxr2wT+uME5UDwdN1kg4frt40euqA+wzECkmq4t5QbveHiJepfdThgQrPw6KiSlg=="
},
"@types/bull": {
"version": "3.15.4",
"resolved": "https://registry.npmjs.org/@types/bull/-/bull-3.15.4.tgz",
"integrity": "sha512-vsw4kqsI/SLocfcgTUY2b8POTBUw9AwmTaehvSVnEAbcgJHsPheFMP8WKDFDk2I+aNTLRQX5iP21dPwz3CGLVA==",
"dev": true,
"requires": {
"@types/ioredis": "*"
}
},
"@types/cacheable-request": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.2.tgz",
@@ -6247,31 +6318,56 @@
"resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz",
"integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s="
},
"bull": {
"version": "3.29.2",
"resolved": "https://registry.npmjs.org/bull/-/bull-3.29.2.tgz",
"integrity": "sha512-zWHyza/ElwVvJUqIEDJdUhGKd1V9EHjituUL7sJAmJoxS9Z7QMhYcMOWcgbUlWPgtiKN1g9ZlOtFAoq7C4/SQw==",
"bullmq": {
"version": "1.86.7",
"resolved": "https://registry.npmjs.org/bullmq/-/bullmq-1.86.7.tgz",
"integrity": "sha512-4fsIcoMmdRmPvll/dV205wfKJlx7jfn1zX1AzO2t56juxiVrbXv5MtYLYzFgx8FW4CnyWzhxXMqJSdTmdxxJ+g==",
"requires": {
"cron-parser": "^2.13.0",
"debuglog": "^1.0.0",
"cron-parser": "^4.2.1",
"get-port": "^5.1.1",
"ioredis": "^4.27.0",
"glob": "^7.2.0",
"ioredis": "^4.28.5",
"lodash": "^4.17.21",
"p-timeout": "^3.2.0",
"promise.prototype.finally": "^3.1.2",
"semver": "^7.3.2",
"util.promisify": "^1.0.1",
"uuid": "^8.3.0"
"msgpackr": "^1.4.6",
"semver": "^7.3.7",
"tslib": "^1.14.1",
"uuid": "^8.3.2"
},
"dependencies": {
"glob": {
"version": "7.2.3",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
"integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
"requires": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^3.1.1",
"once": "^1.3.0",
"path-is-absolute": "^1.0.0"
}
},
"minimatch": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
"requires": {
"brace-expansion": "^1.1.7"
}
},
"semver": {
"version": "7.3.5",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
"integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==",
"version": "7.3.7",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.7.tgz",
"integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==",
"requires": {
"lru-cache": "^6.0.0"
}
},
"tslib": {
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
"integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="
},
"uuid": {
"version": "8.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
@@ -6552,12 +6648,18 @@
"dev": true
},
"cron-parser": {
"version": "2.18.0",
"resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-2.18.0.tgz",
"integrity": "sha512-s4odpheTyydAbTBQepsqd2rNWGa2iV3cyo8g7zbI2QQYGLVsfbhmwukayS1XHppe02Oy1fg7mg6xoaraVJeEcg==",
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.5.0.tgz",
"integrity": "sha512-QHdeh3zLWz6YvYTFKpRb860rJlip16pEinbxXT1i2NZB/nOxBjd2RbSv54sn5UrAj9WykiSLYWWDgo8azQK0HA==",
"requires": {
"is-nan": "^1.3.0",
"moment-timezone": "^0.5.31"
"luxon": "^2.4.0"
},
"dependencies": {
"luxon": {
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-2.5.0.tgz",
"integrity": "sha512-IDkEPB80Rb6gCAU+FEib0t4FeJ4uVOuX1CQ9GsvU3O+JAGIgu0J7sf1OarXKaKDygTZIoJyU6YdZzTFRu+YR0A=="
}
}
},
"debug": {
@@ -6568,11 +6670,6 @@
"ms": "2.1.2"
}
},
"debuglog": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz",
"integrity": "sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI="
},
"decamelize": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/decamelize/-/decamelize-4.0.0.tgz",
@@ -7264,9 +7361,9 @@
}
},
"ioredis": {
"version": "4.27.9",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.27.9.tgz",
"integrity": "sha512-hAwrx9F+OQ0uIvaJefuS3UTqW+ByOLyLIV+j0EH8ClNVxvFyH9Vmb08hCL4yje6mDYT5zMquShhypkd50RRzkg==",
"version": "4.28.5",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz",
"integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==",
"requires": {
"cluster-key-slot": "^1.1.0",
"debug": "^4.3.1",
@@ -7595,7 +7692,7 @@
"lodash.isarguments": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
"integrity": "sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo="
"integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg=="
},
"lodash.isplainobject": {
"version": "4.0.6",
@@ -7779,19 +7876,6 @@
"resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz",
"integrity": "sha1-EUyUlnPiqKNenTV4hSeqN7Z52is="
},
"moment": {
"version": "2.29.3",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.29.3.tgz",
"integrity": "sha512-c6YRvhEo//6T2Jz/vVtYzqBzwvPT95JBQ+smCytzf7c50oMZRsR/a4w88aD34I+/QVSfnoAnSBFPJHItlOMJVw=="
},
"moment-timezone": {
"version": "0.5.33",
"resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.33.tgz",
"integrity": "sha512-PTc2vcT8K9J5/9rDEPe5czSIKgLoGsH8UNpA4qZTVw0Vd/Uz19geE9abbIOQKaAQFcnQ3v5YEXrbSc5BpshH+w==",
"requires": {
"moment": ">= 2.9.0"
}
},
"mongodb": {
"version": "3.6.11",
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.11.tgz",
@@ -7876,6 +7960,29 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"msgpackr": {
"version": "1.6.1",
"resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.6.1.tgz",
"integrity": "sha512-Je+xBEfdjtvA4bKaOv8iRhjC8qX2oJwpYH4f7JrG4uMVJVmnmkAT4pjKdbztKprGj3iwjcxPzb5umVZ02Qq3tA==",
"requires": {
"msgpackr-extract": "^2.0.2"
}
},
"msgpackr-extract": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-2.0.2.tgz",
"integrity": "sha512-coskCeJG2KDny23zWeu+6tNy7BLnAiOGgiwzlgdm4oeSsTpqEJJPguHIuKZcCdB7tzhZbXNYSg6jZAXkZErkJA==",
"optional": true,
"requires": {
"@msgpackr-extract/msgpackr-extract-darwin-arm64": "2.0.2",
"@msgpackr-extract/msgpackr-extract-darwin-x64": "2.0.2",
"@msgpackr-extract/msgpackr-extract-linux-arm": "2.0.2",
"@msgpackr-extract/msgpackr-extract-linux-arm64": "2.0.2",
"@msgpackr-extract/msgpackr-extract-linux-x64": "2.0.2",
"@msgpackr-extract/msgpackr-extract-win32-x64": "2.0.2",
"node-gyp-build-optional-packages": "5.0.2"
}
},
"nanoid": {
"version": "3.1.23",
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.23.tgz",
@@ -7895,6 +8002,12 @@
"whatwg-url": "^5.0.0"
}
},
"node-gyp-build-optional-packages": {
"version": "5.0.2",
"resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.2.tgz",
"integrity": "sha512-PiN4NWmlQPqvbEFcH/omQsswWQbe5Z9YK/zdB23irp5j2XibaA2IrGvpSWmVVG4qMZdmPdwPctSy4a86rOMn6g==",
"optional": true
},
"node-schedule": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/node-schedule/-/node-schedule-2.1.0.tgz",
@@ -7952,16 +8065,6 @@
"object-keys": "^1.1.1"
}
},
"object.getownpropertydescriptors": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.2.tgz",
"integrity": "sha512-WtxeKSzfBjlzL+F9b7M7hewDzMwy+C8NRssHd1YrNlzHzIDrXcXiNOMrezdAEM4UXixgV+vvnyBeN7Rygl2ttQ==",
"requires": {
"call-bind": "^1.0.2",
"define-properties": "^1.1.3",
"es-abstract": "^1.18.0-next.2"
}
},
"on-finished": {
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
@@ -7993,11 +8096,6 @@
"resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.1.1.tgz",
"integrity": "sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg=="
},
"p-finally": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
"integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4="
},
"p-limit": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
@@ -8021,14 +8119,6 @@
"resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz",
"integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw=="
},
"p-timeout": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz",
"integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==",
"requires": {
"p-finally": "^1.0.0"
}
},
"pako": {
"version": "0.2.9",
"resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz",
@@ -8137,16 +8227,6 @@
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"promise.prototype.finally": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/promise.prototype.finally/-/promise.prototype.finally-3.1.2.tgz",
"integrity": "sha512-A2HuJWl2opDH0EafgdjwEw7HysI8ff/n4lW4QEVBCUXFk9QeGecBWv0Deph0UmLe3tTNYegz8MOjsVuE6SMoJA==",
"requires": {
"define-properties": "^1.1.3",
"es-abstract": "^1.17.0-next.0",
"function-bind": "^1.1.1"
}
},
"proxy-addr": {
"version": "2.0.7",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
@@ -8300,12 +8380,12 @@
"redis-errors": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
"integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60="
"integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w=="
},
"redis-parser": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
"integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=",
"integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==",
"requires": {
"redis-errors": "^1.0.0"
}
@@ -8813,18 +8893,6 @@
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
},
"util.promisify": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.1.1.tgz",
"integrity": "sha512-/s3UsZUrIfa6xDhr7zZhnE9SLQ5RIXyYfiVnMMyMDzOc8WhWN4Nbh36H842OyurKbCDAesZOJaVyvmSl6fhGQw==",
"requires": {
"call-bind": "^1.0.0",
"define-properties": "^1.1.3",
"for-each": "^0.3.3",
"has-symbols": "^1.0.1",
"object.getownpropertydescriptors": "^2.1.1"
}
},
"utils-merge": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",

3
package.json
View File

@@ -29,7 +29,7 @@
"@pm2/io": "^5.0.0",
"archiver": "^5.3.1",
"aws-sdk": "^2.1180.0",
"bull": "^3.29.2",
"bullmq": "^1.86.7",
"compression": "^1.7.4",
"connect-redis": "^6.1.3",
"decompress-stream-to-s3": "^1.2.1",
@@ -54,7 +54,6 @@
},
"devDependencies": {
"@types/archiver": "^5.3.1",
"@types/bull": "^3.15.4",
"@types/compression": "^1.7.1",
"@types/connect-redis": "^0.0.18",
"@types/express": "^4.17.13",

30
src/AnonymizedFile.ts
View File

@@ -1,6 +1,6 @@
import * as path from "path";
import * as express from "express";
import * as stream from "stream";
import { join, basename } from "path";
import { Response } from "express";
import { Readable, pipeline } from "stream";
import { promisify } from "util";
import Repository from "./Repository";
import { Tree, TreeElement, TreeFile } from "./types";
@@ -19,11 +19,11 @@ function tree2sha(
const sha = tree[i].sha as string;
const size = tree[i].size as number;
if (sha != null && size != null) {
output[sha] = path.join(parent, i);
output[sha] = join(parent, i);
} else if (tree[i].child) {
tree2sha(tree[i].child as Tree, output, path.join(parent, i));
tree2sha(tree[i].child as Tree, output, join(parent, i));
} else {
tree2sha(tree[i] as Tree, output, path.join(parent, i));
tree2sha(tree[i] as Tree, output, join(parent, i));
}
}
return output;
@@ -98,13 +98,13 @@ export default class AnonymizedFile {
// if only one option we found the original filename
if (options.length == 1) {
currentOriginalPath = path.join(currentOriginalPath, options[0]);
currentOriginalPath = join(currentOriginalPath, options[0]);
currentOriginal = currentOriginal[options[0]];
} else {
isAmbiguous = true;
}
} else if (!isAmbiguous) {
currentOriginalPath = path.join(currentOriginalPath, fileName);
currentOriginalPath = join(currentOriginalPath, fileName);
currentOriginal = currentOriginal[fileName];
}
}
@@ -130,7 +130,7 @@ export default class AnonymizedFile {
});
}
this._originalPath = path.join(currentOriginalPath, shaTree[file.sha]);
this._originalPath = join(currentOriginalPath, shaTree[file.sha]);
} else {
this._originalPath = currentOriginalPath;
}
@@ -139,7 +139,7 @@ export default class AnonymizedFile {
}
async isFileSupported() {
const filename = path.basename(await this.originalPath());
const filename = basename(await this.originalPath());
const extensions = filename.split(".").reverse();
const extension = extensions[0].toLowerCase();
if (!this.repository.options.pdf && extension == "pdf") {
@@ -158,7 +158,7 @@ export default class AnonymizedFile {
return true;
}
async content(): Promise<stream.Readable> {
async content(): Promise<Readable> {
if (this.fileSize && this.fileSize > config.MAX_FILE_SIZE) {
throw new AnonymousError("file_too_big", {
object: this,
@@ -184,13 +184,13 @@ export default class AnonymizedFile {
object: this,
httpStatus: 400,
});
return path.join(this.repository.originalCachePath, this._originalPath);
return join(this.repository.originalCachePath, this._originalPath);
}
async send(res: express.Response): Promise<void> {
const pipeline = promisify(stream.pipeline);
async send(res: Response): Promise<void> {
const pipe = promisify(pipeline);
try {
await pipeline(await this.anonymizedContent(), res);
await pipe(await this.anonymizedContent(), res);
} catch (error) {
handleError(error, res);
}

14
src/Repository.ts
View File

@@ -1,7 +1,7 @@
import * as path from "path";
import { join } from "path";
import storage from "./storage";
import { RepositoryStatus, Source, Tree, TreeElement, TreeFile } from "./types";
import * as stream from "stream";
import { Readable } from "stream";
import User from "./User";
import GitHubStream from "./source/GitHubStream";
import GitHubDownload from "./source/GitHubDownload";
@@ -136,7 +136,7 @@ export default class Repository {
*
* @returns A stream of anonymized repository compressed
*/
zip(): stream.Readable {
zip(): Readable {
return storage.archive(this.originalCachePath, {
format: "zip",
fileTransformer: (filename) =>
@@ -185,11 +185,13 @@ export default class Repository {
this._model.anonymizeDate = new Date();
console.log(`${this._model.repoId} will be updated to ${newCommit}`);
await this.resetSate("preparing");
await downloadQueue.add(this, { jobId: this.repoId, attempts: 3 });
await downloadQueue.add(this.repoId, this, {
jobId: this.repoId,
attempts: 3,
});
}
}
}
/**
* Download the require state for the repository to work
*
@@ -327,7 +329,7 @@ export default class Repository {
}
get originalCachePath() {
return path.join(this._model.repoId, "original") + "/";
return join(this._model.repoId, "original") + "/";
}
get status() {

11
src/anonymize-utils.ts
View File

@@ -2,15 +2,14 @@ import config from "../config";
import Repository from "./Repository";
import GitHubBase from "./source/GitHubBase";
import { isText } from "istextorbinary";
import * as path from "path";
import * as stream from "stream";
import { basename } from "path";
import { Transform } from "stream";
const urlRegex =
/<?\b((https?|ftp|file):\/\/)[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]\b\/?>?/g;
export function isTextFile(filePath, content) {
const filename = path.basename(filePath);
export function isTextFile(filePath: string, content: Buffer) {
const filename = basename(filePath);
const extensions = filename.split(".").reverse();
const extension = extensions[0].toLowerCase();
if (config.additionalExtensions.includes(extension)) {
@@ -23,7 +22,7 @@ export function isTextFile(filePath, content) {
}
export function anonymizeStream(filename: string, repository: Repository) {
const ts = new stream.Transform();
const ts = new Transform();
var chunks = [],
len = 0,
pos = 0;

25
src/processes/downloadRepository.ts
View File

@@ -1,14 +1,17 @@
import AnonymousError from "../AnonymousError";
import { connect, getRepository } from "../database/database";
import { SandboxedJob } from "bullmq";
import { config } from "dotenv";
config();
import Repository from "../Repository";
export default async function process(job) {
export default async function (job: SandboxedJob<Repository, void>) {
const { connect, getRepository } = require("../database/database");
console.log(`${job.data.repoId} is going to be downloaded`);
try {
await connect();
const repo = await getRepository(job.data.repoId);
job.progress("get_repo");
await repo.resetSate();
job.progress("resetSate");
job.updateProgress({ status: "get_repo" });
await repo.resetSate("preparing");
job.updateProgress({ status: "resetSate" });
try {
await repo.anonymize();
} catch (error) {
@@ -16,15 +19,7 @@ export default async function process(job) {
throw error;
}
} catch (error) {
if (error instanceof AnonymousError) {
console.error(
"[ERROR]",
error.toString(),
error.stack.split("\n")[1].trim()
);
} else {
console.error(error);
}
console.error(error);
} finally {
console.log(`${job.data.repoId} is downloaded`);
}

17
src/processes/removeRepository.ts
View File

@@ -1,7 +1,8 @@
import AnonymousError from "../AnonymousError";
import { connect, getRepository } from "../database/database";
import { SandboxedJob } from "bullmq";
import Repository from "../Repository";
export default async function process(job) {
export default async function (job: SandboxedJob<Repository, void>) {
const { connect, getRepository } = require("../database/database");
try {
await connect();
console.log(`${job.data.repoId} is going to be removed`);
@@ -13,15 +14,7 @@ export default async function process(job) {
throw error;
}
} catch (error) {
if (error instanceof AnonymousError) {
console.error(
"[ERROR]",
error.toString(),
error.stack.split("\n")[1].trim()
);
} else {
console.error(error);
}
console.error(error);
} finally {
console.log(`${job.data.repoId} is removed`);
}

95
src/queue.ts
View File

@@ -1,27 +1,80 @@
import * as Queue from "bull";
import { Queue, Worker } from "bullmq";
import config from "../config";
import Repository from "./Repository";
import * as path from "path";
export const removeQueue = new Queue<Repository>("repository removal", {
redis: {
host: config.REDIS_HOSTNAME,
port: config.REDIS_PORT,
},
});
removeQueue.on("completed", async (job) => {
await job.remove();
});
export const downloadQueue = new Queue<Repository>("repository download", {
redis: {
host: config.REDIS_HOSTNAME,
port: config.REDIS_PORT,
},
});
downloadQueue.on("completed", async (job) => {
await job.remove();
});
export let removeQueue: Queue<Repository>;
export let downloadQueue: Queue<Repository>;
removeQueue.process(5, path.resolve("src/processes/removeRepository.ts"));
// avoid to load the queue outside the main server
export function startWorker() {
removeQueue = new Queue<Repository>("repository removal", {
connection: {
host: config.REDIS_HOSTNAME,
port: config.REDIS_PORT,
},
defaultJobOptions: {
removeOnComplete: true,
},
});
downloadQueue = new Queue<Repository>("repository download", {
connection: {
host: config.REDIS_HOSTNAME,
port: config.REDIS_PORT,
},
defaultJobOptions: {
removeOnComplete: true,
},
});
const removeWorker = new Worker<Repository>(
removeQueue.name,
path.resolve("dist/src/processes/removeRepository.js"),
//removeRepository,
{
concurrency: 5,
connection: {
host: config.REDIS_HOSTNAME,
port: config.REDIS_PORT,
},
autorun: true,
downloadQueue.process(2, path.resolve("src/processes/downloadRepository.ts"));
}
);
removeWorker.on("error", async (error) => {
console.log(error);
});
removeWorker.on("completed", async (job) => {
await job.remove();
});
const downloadWorker = new Worker<Repository>(
downloadQueue.name,
path.resolve("dist/src/processes/downloadRepository.js"),
// downloadRepository,
{
concurrency: 2,
connection: {
host: config.REDIS_HOSTNAME,
port: config.REDIS_PORT,
},
autorun: true
}
);
if (!downloadWorker.isRunning) downloadWorker.run();
downloadWorker.on("active", async (job) => {
console.log("active", job.data.repoId);
});
downloadWorker.on("completed", async (job) => {
console.log("completed", job.data.repoId);
});
downloadWorker.on("failed", async (job) => {
console.log("failed", job.data.repoId);
});
downloadWorker.on("closing", async (error) => {
console.log("closing", error);
});
downloadWorker.on("error", async (error) => {
console.log(error);
});
}

7
src/routes/admin.ts
View File

@@ -1,9 +1,10 @@
import { Queue } from "bull";
import { Queue } from "bullmq";
import * as express from "express";
import AnonymizedRepositoryModel from "../database/anonymizedRepositories/anonymizedRepositories.model";
import ConferenceModel from "../database/conference/conferences.model";
import UserModel from "../database/users/users.model";
import { downloadQueue, removeQueue } from "../queue";
import Repository from "../Repository";
import { ensureAuthenticated } from "./connection";
import { handleError, getUser, isOwnerOrAdmin } from "./route-utils";
@@ -29,7 +30,7 @@ router.use(
);
router.post("/queue/:name/:repo_id", async (req, res) => {
let queue: Queue;
let queue: Queue<Repository, void>;
if (req.params.name == "download") {
queue = downloadQueue;
} else if (req.params.name == "remove") {
@@ -41,7 +42,7 @@ router.post("/queue/:name/:repo_id", async (req, res) => {
if (!job) {
return res.status(404).json({ error: "job_not_found" });
}
job.retry();
await job.retry();
res.send("ok");
});

6
src/routes/repository-private.ts
View File

@@ -123,7 +123,7 @@ router.delete(
const user = await getUser(req);
isOwnerOrAdmin([repo.owner.id], user);
await repo.updateStatus("removing");
await removeQueue.add(repo, { jobId: repo.repoId });
await removeQueue.add(repo.repoId, repo, { jobId: repo.repoId });
return res.json({ status: repo.status });
} catch (error) {
handleError(error, res);
@@ -362,7 +362,7 @@ router.post(
repo.model.conference = repoUpdate.conference;
await repo.updateStatus("preparing");
res.json({ status: repo.status });
await downloadQueue.add(repo, { jobId: repo.repoId });
await downloadQueue.add(repo.repoId, repo, { jobId: repo.repoId });
} catch (error) {
return handleError(error, res);
}
@@ -432,7 +432,7 @@ router.post("/", async (req: express.Request, res: express.Response) => {
}
res.send({ status: repo.status });
downloadQueue.add(new Repository(repo), {
downloadQueue.add(repo.repoId, new Repository(repo), {
jobId: repo.repoId,
attempts: 3,
});

5
src/routes/repository-public.ts
View File

@@ -91,7 +91,10 @@ router.get(
// && repo.status != "preparing"
) {
await repo.updateStatus("preparing");
await downloadQueue.add(repo, { jobId: repo.repoId, attempts: 3 });
await downloadQueue.add(repo.repoId, repo, {
jobId: repo.repoId,
attempts: 3,
});
}
if (repo.status == "error") {
throw new AnonymousError(

5
src/routes/route-utils.ts
View File

@@ -44,6 +44,7 @@ export function isOwnerOrAdmin(authorizedUsers: string[], user: User) {
function printError(error: any) {
io.notifyError(error, error.value);
if (error instanceof AnonymousError) {
console.log(error);
console.error(
"[ERROR]",
error.toString(),
@@ -78,14 +79,14 @@ export function handleError(error: any, res: express.Response) {
export async function getUser(req: express.Request) {
const user = (req.user as any).user;
if (!user) {
req.logout();
req.logout((error) => console.error(error));
throw new AnonymousError("not_connected", {
httpStatus: 401,
});
}
const model = await UserModel.findById(user._id);
if (!model) {
req.logout();
req.logout((error) => console.error(error));
throw new AnonymousError("not_connected", {
httpStatus: 401,
});

2
src/routes/user.ts
View File

@@ -10,7 +10,7 @@ router.use(ensureAuthenticated);
router.get("/logout", async (req: express.Request, res: express.Response) => {
try {
req.logout();
req.logout((error) => console.error(error));
res.redirect("/");
} catch (error) {
handleError(error, res);

7
src/server.ts
View File

@@ -13,6 +13,7 @@ import * as connection from "./routes/connection";
import router from "./routes";
import AnonymizedRepositoryModel from "./database/anonymizedRepositories/anonymizedRepositories.model";
import { conferenceStatusCheck, repositoryStatusCheck } from "./schedule";
import { startWorker } from "./queue";
function indexResponse(req: express.Request, res: express.Response) {
if (
@@ -44,13 +45,15 @@ export default async function start() {
app.use(passport.initialize());
app.use(passport.session());
startWorker();
const redisClient = createClient({
socket: {
host: config.REDIS_HOSTNAME,
port: config.REDIS_PORT,
},
});
redisClient.on('error', (err) => console.log('Redis Client Error', err));
redisClient.on("error", (err) => console.log("Redis Client Error", err));
await redisClient.connect();
@@ -59,7 +62,7 @@ export default async function start() {
sendCommand: (...args: string[]) => redisClient.sendCommand(args),
}),
windowMs: 15 * 60 * 1000, // 15 minutes
max: 200, // limit each IP
max: 1000, // limit each IP
standardHeaders: true,
legacyHeaders: false,
// delayMs: 0, // disable delaying - full speed until the max limit is reached

5
src/source/GitHubBase.ts
View File

@@ -2,9 +2,8 @@ import AnonymizedFile from "../AnonymizedFile";
import { Branch, Tree } from "../types";
import { GitHubRepository } from "./GitHubRepository";
import config from "../../config";
import { OAuthApp } from "@octokit/oauth-app";
import Repository from "../Repository";
import * as stream from "stream";
import { Readable } from "stream";
import UserModel from "../database/users/users.model";
import AnonymousError from "../AnonymousError";
@@ -37,7 +36,7 @@ export default abstract class GitHubBase {
this.branch = { commit: data.commit, name: data.branch };
}
async getFileContent(file: AnonymizedFile): Promise<stream.Readable> {
async getFileContent(file: AnonymizedFile): Promise<Readable> {
throw new AnonymousError("method_not_implemented", {
httpStatus: 501,
object: this,

4
src/source/GitHubDownload.ts
View File

@@ -7,7 +7,7 @@ import GitHubBase from "./GitHubBase";
import AnonymizedFile from "../AnonymizedFile";
import { SourceBase } from "../types";
import got from "got";
import * as stream from "stream";
import { Readable } from "stream";
import { OctokitResponse } from "@octokit/types";
import AnonymousError from "../AnonymousError";
@@ -113,7 +113,7 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
await this.repository.updateStatus("ready");
}
async getFileContent(file: AnonymizedFile): Promise<stream.Readable> {
async getFileContent(file: AnonymizedFile): Promise<Readable> {
await this.download();
// update the file list
await this.repository.files({ force: true });

1
src/source/Zip.ts
View File

@@ -1,4 +1,3 @@
import * as path from "path";
import AnonymizedFile from "../AnonymizedFile";
import Repository from "../Repository";
import storage from "../storage";

48
src/storage/FileSystem.ts
View File

@@ -2,10 +2,10 @@ import { StorageBase, Tree } from "../types";
import config from "../../config";
import * as fs from "fs";
import * as unzip from "unzip-stream";
import * as path from "path";
import * as express from "express";
import * as stream from "stream";
import { Extract } from "unzip-stream";
import { join, basename, dirname } from "path";
import { Response } from "express";
import { Readable, pipeline } from "stream";
import * as archiver from "archiver";
import { promisify } from "util";
@@ -16,32 +16,32 @@ export default class FileSystem implements StorageBase {
/** @override */
async exists(p: string): Promise<boolean> {
return fs.existsSync(path.join(config.FOLDER, p));
return fs.existsSync(join(config.FOLDER, p));
}
/** @override */
send(p: string, res: express.Response) {
res.sendFile(path.join(config.FOLDER, p), { dotfiles: "allow" });
send(p: string, res: Response) {
res.sendFile(join(config.FOLDER, p), { dotfiles: "allow" });
}
/** @override */
read(p: string): stream.Readable {
return fs.createReadStream(path.join(config.FOLDER, p));
read(p: string): Readable {
return fs.createReadStream(join(config.FOLDER, p));
}
/** @override */
async write(p: string, data: Buffer): Promise<void> {
if (!(await this.exists(path.dirname(p)))) {
await fs.promises.mkdir(path.dirname(path.join(config.FOLDER, p)), {
if (!(await this.exists(dirname(p)))) {
await fs.promises.mkdir(dirname(join(config.FOLDER, p)), {
recursive: true,
});
}
return fs.promises.writeFile(path.join(config.FOLDER, p), data);
return fs.promises.writeFile(join(config.FOLDER, p), data);
}
/** @override */
async rm(dir: string): Promise<void> {
await fs.promises.rm(path.join(config.FOLDER, dir), {
await fs.promises.rm(join(config.FOLDER, dir), {
force: true,
recursive: true,
});
@@ -50,7 +50,7 @@ export default class FileSystem implements StorageBase {
/** @override */
async mk(dir: string): Promise<void> {
if (!(await this.exists(dir)))
fs.promises.mkdir(path.join(config.FOLDER, dir), { recursive: true });
fs.promises.mkdir(join(config.FOLDER, dir), { recursive: true });
}
/** @override */
@@ -64,12 +64,12 @@ export default class FileSystem implements StorageBase {
if (opt.root == null) {
opt.root = config.FOLDER;
}
let files = await fs.promises.readdir(path.join(opt.root, dir));
let files = await fs.promises.readdir(join(opt.root, dir));
const output: Tree = {};
for (let file of files) {
let filePath = path.join(dir, file);
let filePath = join(dir, file);
try {
const stats = await fs.promises.stat(path.join(opt.root, filePath));
const stats = await fs.promises.stat(join(opt.root, filePath));
if (file[0] == "$") {
file = "\\" + file;
}
@@ -92,12 +92,12 @@ export default class FileSystem implements StorageBase {
}
/** @override */
async extractZip(p: string, data: stream.Readable): Promise<void> {
const pipeline = promisify(stream.pipeline);
return pipeline(
async extractZip(p: string, data: Readable): Promise<void> {
const pipe = promisify(pipeline);
return pipe(
data,
unzip.Extract({
path: path.join(path.join(config.FOLDER, p)),
Extract({
path: join(join(config.FOLDER, p)),
decodeString: (buf) => {
const name = buf.toString();
const newName = name.substr(name.indexOf("/") + 1);
@@ -127,8 +127,8 @@ export default class FileSystem implements StorageBase {
}
const f = file.path.replace(dir, "");
archive.append(rs, {
name: path.basename(f),
prefix: path.dirname(f),
name: basename(f),
prefix: dirname(f),
});
},
}).then(() => {

27
src/storage/S3.ts
View File

@@ -1,13 +1,13 @@
import { StorageBase, Tree, TreeFile } from "../types";
import { S3 } from "aws-sdk";
import config from "../../config";
import * as stream from "stream";
import { pipeline, Readable } from "stream";
import ArchiveStreamToS3 from "decompress-stream-to-s3";
import * as express from "express";
import * as mime from "mime-types";
import { Response } from "express";
import { lookup } from "mime-types";
import * as flow from "xml-flow";
import * as archiver from "archiver";
import * as path from "path";
import { dirname, basename } from "path";
import AnonymousError from "../AnonymousError";
export default class S3Storage implements StorageBase {
@@ -83,7 +83,7 @@ export default class S3Storage implements StorageBase {
}
/** @override */
send(p: string, res: express.Response) {
send(p: string, res: Response) {
const s = this.client
.getObject({
Bucket: config.S3_BUCKET,
@@ -102,8 +102,8 @@ export default class S3Storage implements StorageBase {
res.set("Content-Length", headers["content-length"]);
res.set("Content-Type", headers["content-type"]);
}
stream.pipeline(
response.httpResponse.createUnbufferedStream() as stream.Readable,
pipeline(
response.httpResponse.createUnbufferedStream() as Readable,
res
);
});
@@ -112,7 +112,7 @@ export default class S3Storage implements StorageBase {
}
/** @override */
read(path: string): stream.Readable {
read(path: string): Readable {
return this.client
.getObject({
Bucket: config.S3_BUCKET,
@@ -128,7 +128,7 @@ export default class S3Storage implements StorageBase {
Bucket: config.S3_BUCKET,
Key: path,
Body: data,
ContentType: mime.lookup(path).toString(),
ContentType: lookup(path).toString(),
})
.promise();
return;
@@ -168,7 +168,7 @@ export default class S3Storage implements StorageBase {
}
/** @override */
async extractZip(p: string, data: stream.Readable): Promise<void> {
async extractZip(p: string, data: Readable): Promise<void> {
let toS3: ArchiveStreamToS3;
return new Promise((resolve, reject) => {
@@ -181,8 +181,7 @@ export default class S3Storage implements StorageBase {
header.name = header.name.substr(header.name.indexOf("/") + 1);
},
});
stream
.pipeline(data, toS3, () => {})
pipeline(data, toS3, () => {})
.on("finish", resolve)
.on("error", reject);
});
@@ -210,14 +209,14 @@ export default class S3Storage implements StorageBase {
xmlStream.on("tag:contents", function (file) {
let rs = that.read(file.key);
file.key = file.key.replace(dir, "");
const filename = path.basename(file.key);
const filename = basename(file.key);
if (filename == "") return;
if (opt?.fileTransformer) {
rs = rs.pipe(opt.fileTransformer(filename));
}
archive.append(rs, {
name: filename,
prefix: path.dirname(file.key),
prefix: dirname(file.key),
});
});
xmlStream.on("end", () => {