diff --git a/src/database/database.ts b/src/database/database.ts index 1ac63d9..bf1caae 100644 --- a/src/database/database.ts +++ b/src/database/database.ts @@ -17,6 +17,7 @@ export async function connect() { await mongoose.connect(MONGO_URL + "production", { authSource: "admin", appName: "Anonymous GitHub Server", + compressors: "zstd", } as ConnectOptions); isConnected = true; diff --git a/src/source/GitHubBase.ts b/src/source/GitHubBase.ts index 023ae4d..3a3ec9a 100644 --- a/src/source/GitHubBase.ts +++ b/src/source/GitHubBase.ts @@ -7,6 +7,7 @@ import { Readable } from "stream"; import UserModel from "../database/users/users.model"; import AnonymousError from "../AnonymousError"; import { Octokit } from "@octokit/rest"; +import { trace } from "@opentelemetry/api"; export default abstract class GitHubBase { type: "GitHubDownload" | "GitHubStream" | "Zip"; @@ -14,6 +15,7 @@ export default abstract class GitHubBase { branch: Branch; accessToken: string | undefined; repository: Repository; + validToken: boolean = false; constructor( data: { @@ -66,21 +68,34 @@ export default abstract class GitHubBase { } async getToken() { - const user = await UserModel.findById(this.repository.owner.id); - if (user && user.accessTokens.github) { - const check = await GitHubBase.checkToken(user.accessTokens.github); - if (check) { - this.accessToken = user.accessTokens.github; - return this.accessToken; + const span = trace.getTracer("ano-file").startSpan("GHBase.getToken"); + span.setAttribute("repoId", this.repository.repoId); + try { + if (this.validToken) { + return this.accessToken as string; } - } - if (this.accessToken) { - if (await GitHubBase.checkToken(this.accessToken)) { - return this.accessToken; + const user = await UserModel.findById(this.repository.owner.id, { + accessTokens: 1, + }); + if (user?.accessTokens.github) { + const check = await GitHubBase.checkToken(user.accessTokens.github); + if (check) { + this.accessToken = user.accessTokens.github; + this.validToken = true; + return this.accessToken; + } } + if (this.accessToken) { + if (await GitHubBase.checkToken(this.accessToken)) { + this.validToken = true; + return this.accessToken; + } + } + this.accessToken = config.GITHUB_TOKEN; + return this.accessToken; + } finally { + span.end(); } - this.accessToken = config.GITHUB_TOKEN; - return this.accessToken; } get url() { diff --git a/src/source/GitHubStream.ts b/src/source/GitHubStream.ts index 9aea75b..30b31ba 100644 --- a/src/source/GitHubStream.ts +++ b/src/source/GitHubStream.ts @@ -169,16 +169,23 @@ export default class GitHubStream extends GitHubBase implements SourceBase { } private async getGHTree(sha: string, opt = { recursive: true }) { - const octokit = new Octokit({ - auth: await this.getToken(), - }); - const ghRes = await octokit.git.getTree({ - owner: this.githubRepository.owner, - repo: this.githubRepository.repo, - tree_sha: sha, - recursive: opt.recursive ? "1" : undefined, - }); - return ghRes.data; + const span = trace.getTracer("ano-file").startSpan("GHStream.getGHTree"); + span.setAttribute("repoId", this.repository.repoId); + span.setAttribute("sha", sha); + try { + const octokit = new Octokit({ + auth: await this.getToken(), + }); + const ghRes = await octokit.git.getTree({ + owner: this.githubRepository.owner, + repo: this.githubRepository.repo, + tree_sha: sha, + recursive: opt.recursive ? "1" : undefined, + }); + return ghRes.data; + } finally { + span.end(); + } } private async getTruncatedTree( @@ -191,47 +198,57 @@ export default class GitHubStream extends GitHubBase implements SourceBase { }, depth = 0 ) { - count.request++; - let data = null; - + const span = trace + .getTracer("ano-file") + .startSpan("GHStream.getTruncatedTree"); + span.setAttribute("repoId", this.repository.repoId); + span.setAttribute("sha", sha); + span.setAttribute("parentPath", parentPath); try { - data = await this.getGHTree(sha, { recursive: false }); - this.tree2Tree(data.tree, truncatedTree, parentPath); - } catch (error) { - console.error(error); - this.repository.model.truckedFileList = true; - return; - } + count.request++; + let data = null; - count.file += data.tree.length; - if (data.tree.length < 100 && count.request < 200) { - const promises: Promise[] = []; - for (const file of data.tree) { - if (file.type == "tree" && file.path && file.sha) { - const elementPath = path.join(parentPath, file.path); - promises.push( - this.getTruncatedTree( - file.sha, - truncatedTree, - elementPath, - count, - depth + 1 - ) - ); - } - } - await Promise.all(promises); - } else { try { - const data = await this.getGHTree(sha, { recursive: true }); + data = await this.getGHTree(sha, { recursive: false }); this.tree2Tree(data.tree, truncatedTree, parentPath); - if (data.truncated) { - this.repository.model.truckedFileList = true; - } } catch (error) { console.error(error); this.repository.model.truckedFileList = true; + return; } + + count.file += data.tree.length; + if (data.tree.length < 100 && count.request < 200) { + const promises: Promise[] = []; + for (const file of data.tree) { + if (file.type == "tree" && file.path && file.sha) { + const elementPath = path.join(parentPath, file.path); + promises.push( + this.getTruncatedTree( + file.sha, + truncatedTree, + elementPath, + count, + depth + 1 + ) + ); + } + } + await Promise.all(promises); + } else { + try { + const data = await this.getGHTree(sha, { recursive: true }); + this.tree2Tree(data.tree, truncatedTree, parentPath); + if (data.truncated) { + this.repository.model.truckedFileList = true; + } + } catch (error) { + console.error(error); + this.repository.model.truckedFileList = true; + } + } + } finally { + span.end(); } } @@ -247,42 +264,49 @@ export default class GitHubStream extends GitHubBase implements SourceBase { partialTree: Tree = {}, parentPath: string = "" ) { - for (let elem of tree) { - let current = partialTree; + const span = trace.getTracer("ano-file").startSpan("GHStream.tree2Tree"); + span.setAttribute("repoId", this.repository.repoId); + span.setAttribute("parentPath", parentPath); + try { + for (let elem of tree) { + let current = partialTree; - if (!elem.path) continue; + if (!elem.path) continue; - const paths = path.join(parentPath, elem.path).split("/"); + const paths = path.join(parentPath, elem.path).split("/"); - // if elem is a folder iterate on all folders if it is a file stop before the filename - const end = elem.type == "tree" ? paths.length : paths.length - 1; - for (let i = 0; i < end; i++) { - let p = paths[i]; - if (p[0] == "$") { - p = "\\" + p; + // if elem is a folder iterate on all folders if it is a file stop before the filename + const end = elem.type == "tree" ? paths.length : paths.length - 1; + for (let i = 0; i < end; i++) { + let p = paths[i]; + if (p[0] == "$") { + p = "\\" + p; + } + if (!current[p]) { + current[p] = {}; + } + current = current[p] as Tree; } - if (!current[p]) { - current[p] = {}; + + // if elem is a file add the file size in the file list + if (elem.type == "blob") { + if (Object.keys(current).length > config.MAX_FILE_FOLDER) { + this.repository.model.truckedFileList = true; + continue; + } + let p = paths[end]; + if (p[0] == "$") { + p = "\\" + p; + } + current[p] = { + size: elem.size || 0, // size in bit + sha: elem.sha || "", + }; } - current = current[p] as Tree; - } - - // if elem is a file add the file size in the file list - if (elem.type == "blob") { - if (Object.keys(current).length > config.MAX_FILE_FOLDER) { - this.repository.model.truckedFileList = true; - continue; - } - let p = paths[end]; - if (p[0] == "$") { - p = "\\" + p; - } - current[p] = { - size: elem.size || 0, // size in bit - sha: elem.sha || "", - }; } + return partialTree; + } finally { + span.end(); } - return partialTree; } } diff --git a/src/storage/FileSystem.ts b/src/storage/FileSystem.ts index 6aa035b..3b1283f 100644 --- a/src/storage/FileSystem.ts +++ b/src/storage/FileSystem.ts @@ -74,46 +74,46 @@ export default class FileSystem implements StorageBase { file?: AnonymizedFile, source?: SourceBase ): Promise { - return trace - .getTracer("ano-file") - .startActiveSpan("fs.write", async (span) => { - span.setAttribute("path", p); - try { - await this.mk(dirname(p)); - return await fs.promises.writeFile( - join(config.FOLDER, p), - data, - "utf-8" - ); - } finally { - span.end(); - } - }); + const span = trace.getTracer("ano-file").startSpan("fs.write"); + span.setAttribute("path", p); + try { + await this.mk(dirname(p)); + return await fs.promises.writeFile(join(config.FOLDER, p), data, "utf-8"); + } finally { + span.end(); + } } /** @override */ async rm(dir: string): Promise { const span = trace.getTracer("ano-file").startSpan("fs.rm"); span.setAttribute("path", dir); - await fs.promises.rm(join(config.FOLDER, dir), { - force: true, - recursive: true, - }); - span.end(); + try { + await fs.promises.rm(join(config.FOLDER, dir), { + force: true, + recursive: true, + }); + } finally { + span.end(); + } } /** @override */ async mk(dir: string): Promise { - return trace - .getTracer("ano-file") - .startActiveSpan("fs.mk", async (span) => { - span.setAttribute("path", dir); - if ((await this.exists(dir)) === FILE_TYPE.NOT_FOUND) - await fs.promises.mkdir(join(config.FOLDER, dir), { - recursive: true, - }); - span.end(); + const span = trace.getTracer("ano-file").startSpan("fs.mk"); + span.setAttribute("path", dir); + try { + await fs.promises.mkdir(join(config.FOLDER, dir), { + recursive: true, }); + } catch (err: any) { + if (err.code !== "EEXIST") { + span.recordException(err); + throw err; + } + } finally { + span.end(); + } } /** @override */