diff --git a/src/AnonymizedFile.ts b/src/AnonymizedFile.ts index 28d2579..450f9e6 100644 --- a/src/AnonymizedFile.ts +++ b/src/AnonymizedFile.ts @@ -15,6 +15,7 @@ import AnonymousError from "./AnonymousError"; import { handleError } from "./routes/route-utils"; import { lookup } from "mime-types"; import { FILE_TYPE } from "./storage/Storage"; +import GitHubBase from "./source/GitHubBase"; /** * Represent a file in a anonymized repository @@ -224,9 +225,13 @@ export default class AnonymizedFile { .startActiveSpan("anonymizedContent", async (span) => { span.setAttribute("anonymizedPath", this.anonymizedPath); const content = await this.content(); - return content.pipe(new AnonymizeTransformer(this)).on("close", () => { - span.end(); - }); + return content + .pipe( + this.repository.generateAnonymizeTransformer(this.anonymizedPath) + ) + .on("close", () => { + span.end(); + }); }); } @@ -259,7 +264,9 @@ export default class AnonymizedFile { res.contentType("text/plain"); } res.header("Accept-Ranges", "none"); - const anonymizer = new AnonymizeTransformer(this); + const anonymizer = this.repository.generateAnonymizeTransformer( + this.anonymizedPath + ); anonymizer.once("transform", (data) => { if (!mime && data.isText) { res.contentType("text/plain"); diff --git a/src/Repository.ts b/src/Repository.ts index 9f03f07..87b8fa9 100644 --- a/src/Repository.ts +++ b/src/Repository.ts @@ -15,7 +15,6 @@ import ConferenceModel from "./database/conference/conferences.model"; import AnonymousError from "./AnonymousError"; import { downloadQueue } from "./queue"; import { isConnected } from "./database/database"; -import AnonymizedFile from "./AnonymizedFile"; import AnonymizedRepositoryModel from "./database/anonymizedRepositories/anonymizedRepositories.model"; import { getRepositoryFromGitHub } from "./source/GitHubRepository"; import config from "../config"; @@ -71,6 +70,15 @@ export default class Repository { }); } this.owner = new User(new UserModel({ _id: data.owner })); + if (this.source instanceof GitHubBase) { + const originalToken = this._model.source.accessToken; + this.source.getToken(this.owner.id).then((token) => { + if (originalToken != token) { + this._model.source.accessToken = token; + this._model.save(); + } + }); + } this.owner.model.isNew = false; } @@ -175,12 +183,19 @@ export default class Repository { return storage.archive(this.repoId, "", { format: "zip", fileTransformer: (filename: string) => - new AnonymizeTransformer( - new AnonymizedFile({ - repository: this, - anonymizedPath: filename, - }) - ), + this.generateAnonymizeTransformer(filename), + }); + } + + generateAnonymizeTransformer(filePath: string) { + return new AnonymizeTransformer({ + filePath: filePath, + terms: this.options.terms, + image: this.options.image, + link: this.options.link, + repoId: this.repoId, + repoName: (this.source as GitHubBase).githubRepository?.fullName, + branchName: (this.source as GitHubBase).branch?.name || "main", }); } @@ -202,7 +217,7 @@ export default class Repository { ) { // Only GitHubBase can be update for the moment if (this.source instanceof GitHubBase) { - const token = await this.source.getToken(); + const token = await this.source.getToken(this.owner.id); const branches = await this.source.githubRepository.branches({ force: true, accessToken: token, @@ -226,10 +241,13 @@ export default class Repository { accessToken: token, } ); - if (commitInfo.commit.author?.date) { - this._model.source.commitDate = new Date( - commitInfo.commit.author?.date - ); + if ( + commitInfo.commit?.author?.date || + commitInfo.commit?.committer?.date + ) { + const d = (commitInfo.commit?.author?.date || + commitInfo.commit.committer?.date) as string; + this._model.source.commitDate = new Date(d); } branch.commit = newCommit; @@ -252,7 +270,7 @@ export default class Repository { if (this.source.type == "GitHubDownload") { const repository = await getRepositoryFromGitHub({ - accessToken: await this.source.getToken(), + accessToken: await this.source.getToken(this.owner.id), owner: this.source.githubRepository.owner, repo: this.source.githubRepository.repo, }); diff --git a/src/anonymize-utils.ts b/src/anonymize-utils.ts index 575c79d..0345330 100644 --- a/src/anonymize-utils.ts +++ b/src/anonymize-utils.ts @@ -36,32 +36,26 @@ export class AnonymizeTransformer extends Transform { public wasAnonimized = false; public isText: boolean | null = null; - constructor(private readonly file: AnonymizedFile) { + constructor( + private readonly opt: { + filePath: string; + } & ConstructorParameters[1] + ) { super(); - this.isText = isTextFile(this.file.anonymizedPath); + this.isText = isTextFile(this.opt.filePath); } _transform(chunk: Buffer, encoding: string, callback: () => void) { trace .getTracer("ano-file") .startActiveSpan("AnonymizeTransformer.transform", async (span) => { - span.setAttribute("path", this.file.anonymizedPath); + span.setAttribute("path", this.opt.filePath); if (this.isText === null) { - this.isText = isTextFile(this.file.anonymizedPath, chunk); + this.isText = isTextFile(this.opt.filePath, chunk); } if (this.isText) { - const anonimizer = new ContentAnonimizer(chunk.toString(), { - repoId: this.file.repository.repoId, - image: this.file.repository.options.image, - link: this.file.repository.options.link, - terms: this.file.repository.options.terms, - repoName: (this.file.repository.source as GitHubBase) - .githubRepository?.fullName, - branchName: - (this.file.repository.source as GitHubBase).branch?.name || - "main", - }); + const anonimizer = new ContentAnonimizer(chunk.toString(), this.opt); anonimizer.anonymize(); if (anonimizer.wasAnonymized) { this.wasAnonimized = true; diff --git a/src/source/GitHubDownload.ts b/src/source/GitHubDownload.ts index bf5992b..c58a773 100644 --- a/src/source/GitHubDownload.ts +++ b/src/source/GitHubDownload.ts @@ -37,13 +37,13 @@ export default class GitHubDownload extends GitHubBase implements SourceBase { }); } - async download(progress?: (status: string) => void) { + async download(token: string, progress?: (status: string) => void) { const span = trace.getTracer("ano-file").startSpan("GHDownload.download"); span.setAttribute("repoId", this.githubRepository.fullName || ""); try { let response: OctokitResponse; try { - response = await this._getZipUrl(await this.getToken()); + response = await this._getZipUrl(token); } catch (error) { span.recordException(error as Error); throw new AnonymousError("repo_not_accessible", { @@ -117,7 +117,10 @@ export default class GitHubDownload extends GitHubBase implements SourceBase { await file.originalPath(); // the cache is not ready, we need to download the repository - await this.download(progress); + await this.download( + await this.getToken(file.repository.owner.id), + progress + ); return storage.read(this.repoId, file.filePath); } finally { span.end(); @@ -126,7 +129,7 @@ export default class GitHubDownload extends GitHubBase implements SourceBase { async getFiles() { if ((await storage.exists(this.repoId)) === FILE_TYPE.NOT_FOUND) { - await this.download(); + await this.download(await this.getToken()); } return storage.listFiles(this.repoId); } diff --git a/src/source/GitHubStream.ts b/src/source/GitHubStream.ts index 6859605..b44c420 100644 --- a/src/source/GitHubStream.ts +++ b/src/source/GitHubStream.ts @@ -129,13 +129,14 @@ export default class GitHubStream extends GitHubBase implements SourceBase { span.setAttribute("repoName", this.githubRepository.fullName || ""); try { let commit = this.branch?.commit; - return this.getTree(commit); + return this.getTree(await this.getToken(), commit); } finally { span.end(); } } private async getTree( + token: string, sha: string, truncatedTree: Tree = {}, parentPath: string = "", @@ -151,7 +152,7 @@ export default class GitHubStream extends GitHubBase implements SourceBase { let ghRes: Awaited>; try { count.request++; - ghRes = await this.getGHTree(sha, { recursive: true }); + ghRes = await this.getGHTree(token, sha, { recursive: true }); } catch (error) { console.error(error); span.recordException(error as Error); @@ -177,17 +178,21 @@ export default class GitHubStream extends GitHubBase implements SourceBase { const tree = this.tree2Tree(ghRes.tree, truncatedTree, parentPath); count.file += ghRes.tree.length; if (ghRes.truncated) { - await this.getTruncatedTree(sha, tree, parentPath, count); + await this.getTruncatedTree(token, sha, tree, parentPath, count); } span.end(); return tree; } - private async getGHTree(sha: string, opt = { recursive: true }) { + private async getGHTree( + token: string, + sha: string, + opt = { recursive: true } + ) { const span = trace.getTracer("ano-file").startSpan("GHStream.getGHTree"); span.setAttribute("sha", sha); try { - const octokit = GitHubBase.octokit(await this.getToken()); + const octokit = GitHubBase.octokit(token); const ghRes = await octokit.git.getTree({ owner: this.githubRepository.owner, repo: this.githubRepository.repo, @@ -201,6 +206,7 @@ export default class GitHubStream extends GitHubBase implements SourceBase { } private async getTruncatedTree( + token: string, sha: string, truncatedTree: Tree = {}, parentPath: string = "", @@ -220,7 +226,7 @@ export default class GitHubStream extends GitHubBase implements SourceBase { let data = null; try { - data = await this.getGHTree(sha, { recursive: false }); + data = await this.getGHTree(token, sha, { recursive: false }); this.tree2Tree(data.tree, truncatedTree, parentPath); } catch (error) { span.recordException(error as Error); @@ -235,6 +241,7 @@ export default class GitHubStream extends GitHubBase implements SourceBase { const elementPath = path.join(parentPath, file.path); promises.push( this.getTruncatedTree( + token, file.sha, truncatedTree, elementPath, @@ -247,7 +254,7 @@ export default class GitHubStream extends GitHubBase implements SourceBase { await Promise.all(promises); } else { try { - const data = await this.getGHTree(sha, { recursive: true }); + const data = await this.getGHTree(token, sha, { recursive: true }); this.tree2Tree(data.tree, truncatedTree, parentPath); if (data.truncated) { // TODO: TRUNCATED