CalvinBackup/anonymous_github
1
0
Fork 0
mirror of https://github.com/tdurieux/anonymous_github.git synced 2026-02-13 02:42:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: improve S3 reliability

Browse Source
This commit is contained in:
tdurieux
2023-08-28 12:09:58 +02:00
parent 92347fbcfb
commit 2f5d7a1089
2 changed files with 8 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/source/GitHubDownload.ts
View File

@@ -106,7 +106,9 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
try {
const downloadStream = got.stream(response.url);
downloadStream.addListener("downloadProgress", (p) => (progress = p));
downloadStream.addListener("downloadProgress", async (p) => {
progress = p;
});
await storage.extractZip(originalPath, downloadStream, undefined, this);
} catch (error) {
await this.repository.updateStatus(

12
src/storage/S3.ts
View File

@@ -71,10 +71,10 @@ export default class S3Storage implements StorageBase {
/** @override */
async rm(dir: string): Promise<void> {
if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
const data = await this.client().listObjectsV2({
const data = await this.client(200000).listObjectsV2({
Bucket: config.S3_BUCKET,
Prefix: dir,
MaxKeys: 1000,
MaxKeys: 100,
});
const params = {
@@ -92,7 +92,7 @@ export default class S3Storage implements StorageBase {
// nothing to remove
return;
}
await this.client().deleteObjects(params);
await this.client(200000).deleteObjects(params);
if (data.IsTruncated) {
await this.rm(dir);
@@ -249,11 +249,9 @@ export default class S3Storage implements StorageBase {
};
}
},
maxParallel: 10,
});
pipeline(data, toS3, (err) => {
if (err) reject(err);
else resolve();
})
pipeline(data, toS3, () => {})
.on("finish", resolve)
.on("error", reject);
});