CalvinBackup/anonymous_github
1
0
Fork 0
mirror of https://github.com/tdurieux/anonymous_github.git synced 2026-07-09 15:17:49 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix: improve S3 reliability

Browse Source
This commit is contained in:
tdurieux
2023-08-28 12:09:58 +02:00
parent 92347fbcfb
commit 2f5d7a1089
2 changed files with 8 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/source/GitHubDownload.ts
+3 -1
View File
@@ -106,7 +106,9 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
try { try {
const downloadStream = got.stream(response.url); const downloadStream = got.stream(response.url);
downloadStream.addListener("downloadProgress", (p) => (progress = p)); downloadStream.addListener("downloadProgress", async (p) => {
progress = p;
});
await storage.extractZip(originalPath, downloadStream, undefined, this); await storage.extractZip(originalPath, downloadStream, undefined, this);
} catch (error) { } catch (error) {
await this.repository.updateStatus( await this.repository.updateStatus(
src/storage/S3.ts
+5 -7
View File
@@ -71,10 +71,10 @@ export default class S3Storage implements StorageBase {
/** @override */ /** @override */
async rm(dir: string): Promise<void> { async rm(dir: string): Promise<void> {
if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set"); if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
const data = await this.client().listObjectsV2({ const data = await this.client(200000).listObjectsV2({
Bucket: config.S3_BUCKET, Bucket: config.S3_BUCKET,
Prefix: dir, Prefix: dir,
MaxKeys: 1000, MaxKeys: 100,
}); });
const params = { const params = {
@@ -92,7 +92,7 @@ export default class S3Storage implements StorageBase {
// nothing to remove // nothing to remove
return; return;
} }
await this.client().deleteObjects(params); await this.client(200000).deleteObjects(params);
if (data.IsTruncated) { if (data.IsTruncated) {
await this.rm(dir); await this.rm(dir);
@@ -249,11 +249,9 @@ export default class S3Storage implements StorageBase {
}; };
} }
}, },
maxParallel: 10,
}); });
pipeline(data, toS3, (err) => { pipeline(data, toS3, () => {})
if (err) reject(err);
else resolve();
})
.on("finish", resolve) .on("finish", resolve)
.on("error", reject); .on("error", reject);
}); });