CalvinBackup/anonymous_github
1
0
Fork 0
mirror of https://github.com/tdurieux/anonymous_github.git synced 2026-02-13 10:52:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: improve the perf of Anonymous GitHub

Browse Source
This commit is contained in:
tdurieux
2023-02-08 15:34:50 +01:00
parent 2e36b72a7f
commit 5c72f54db5
21 changed files with 529 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
src/source/GitHubDownload.ts
View File

@@ -5,7 +5,7 @@ import Repository from "../Repository";
import GitHubBase from "./GitHubBase";
import AnonymizedFile from "../AnonymizedFile";
import { SourceBase } from "../types";
import { RepositoryStatus, SourceBase } from "../types";
import got from "got";
import { Readable } from "stream";
import { OctokitResponse } from "@octokit/types";
@@ -60,7 +60,10 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
try {
response = await this._getZipUrl(config.GITHUB_TOKEN);
} catch (error) {
await this.repository.resetSate("error", "repo_not_accessible");
await this.repository.resetSate(
RepositoryStatus.ERROR,
"repo_not_accessible"
);
throw new AnonymousError("repo_not_accessible", {
httpStatus: 404,
cause: error,
@@ -68,7 +71,10 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
});
}
} else {
await this.repository.resetSate("error", "repo_not_accessible");
await this.repository.resetSate(
RepositoryStatus.ERROR,
"repo_not_accessible"
);
throw new AnonymousError("repo_not_accessible", {
httpStatus: 404,
object: this.repository,
@@ -76,7 +82,7 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
});
}
}
await this.repository.updateStatus("download");
await this.repository.updateStatus(RepositoryStatus.DOWNLOAD);
const originalPath = this.repository.originalCachePath;
await storage.mk(originalPath);
let progress = null;
@@ -102,7 +108,10 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
downloadStream.addListener("downloadProgress", (p) => (progress = p));
await storage.extractZip(originalPath, downloadStream, null, this);
} catch (error) {
await this.repository.updateStatus("error", "unable_to_download");
await this.repository.updateStatus(
RepositoryStatus.ERROR,
"unable_to_download"
);
throw new AnonymousError("unable_to_download", {
httpStatus: 500,
cause: error,
@@ -113,7 +122,7 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
clearTimeout(progressTimeout);
}
await this.repository.updateStatus("ready");
await this.repository.updateStatus(RepositoryStatus.READY);
}
async getFileContent(file: AnonymizedFile): Promise<Readable> {

26
src/source/GitHubStream.ts
View File

@@ -3,7 +3,7 @@ import AnonymizedFile from "../AnonymizedFile";
import Repository from "../Repository";
import GitHubBase from "./GitHubBase";
import storage from "../storage";
import { SourceBase, Tree } from "../types";
import { RepositoryStatus, SourceBase, Tree } from "../types";
import * as path from "path";
import * as stream from "stream";
@@ -26,11 +26,6 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
}
async getFileContent(file: AnonymizedFile): Promise<stream.Readable> {
if (!file.sha)
throw new AnonymousError("file_sha_not_provided", {
httpStatus: 400,
object: file,
});
const octokit = new Octokit({
auth: await this.getToken(),
});
@@ -57,12 +52,12 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
} else {
content = Buffer.from("");
}
if (this.repository.status != "ready")
await this.repository.updateStatus("ready");
if (this.repository.status !== RepositoryStatus.READY)
await this.repository.updateStatus(RepositoryStatus.READY);
await storage.write(file.originalCachePath, content, file, this);
return stream.Readable.from(content);
} catch (error) {
if (error.status == 404) {
if (error.status === 404 || error.httpStatus === 404) {
throw new AnonymousError("file_not_found", {
httpStatus: error.status,
cause: error,
@@ -99,15 +94,18 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
} catch (error) {
if (error.status == 409) {
// empty tree
if (this.repository.status != "ready")
await this.repository.updateStatus("ready");
if (this.repository.status != RepositoryStatus.READY)
await this.repository.updateStatus(RepositoryStatus.READY);
// cannot be empty otherwise it would try to download it again
return { __: {} };
} else {
console.log(
`[ERROR] getTree ${this.repository.repoId}@${sha}: ${error.message}`
);
await this.repository.resetSate("error", "repo_not_accessible");
await this.repository.resetSate(
RepositoryStatus.ERROR,
"repo_not_accessible"
);
throw new AnonymousError("repo_not_accessible", {
httpStatus: error.status,
cause: error,
@@ -124,8 +122,8 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
if (ghRes.truncated) {
await this.getTruncatedTree(sha, tree, parentPath, count);
}
if (this.repository.status != "ready")
await this.repository.updateStatus("ready");
if (this.repository.status !== RepositoryStatus.READY)
await this.repository.updateStatus(RepositoryStatus.READY);
return tree;
}