CalvinBackup/anonymous_github
1
0
Fork 0
mirror of https://github.com/tdurieux/anonymous_github.git synced 2026-02-12 18:32:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: fix github authentification

Browse Source
This commit is contained in:
tdurieux
2024-04-02 09:10:14 +01:00
parent 8a9d2d8395
commit ea96c31e9d
5 changed files with 72 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/AnonymizedFile.ts
View File

@@ -15,6 +15,7 @@ import AnonymousError from "./AnonymousError";
import { handleError } from "./routes/route-utils";
import { lookup } from "mime-types";
import { FILE_TYPE } from "./storage/Storage";
import GitHubBase from "./source/GitHubBase";
/**
* Represent a file in a anonymized repository
@@ -224,9 +225,13 @@ export default class AnonymizedFile {
.startActiveSpan("anonymizedContent", async (span) => {
span.setAttribute("anonymizedPath", this.anonymizedPath);
const content = await this.content();
return content.pipe(new AnonymizeTransformer(this)).on("close", () => {
span.end();
});
return content
.pipe(
this.repository.generateAnonymizeTransformer(this.anonymizedPath)
)
.on("close", () => {
span.end();
});
});
}
@@ -259,7 +264,9 @@ export default class AnonymizedFile {
res.contentType("text/plain");
}
res.header("Accept-Ranges", "none");
const anonymizer = new AnonymizeTransformer(this);
const anonymizer = this.repository.generateAnonymizeTransformer(
this.anonymizedPath
);
anonymizer.once("transform", (data) => {
if (!mime && data.isText) {
res.contentType("text/plain");

44
src/Repository.ts
View File

@@ -15,7 +15,6 @@ import ConferenceModel from "./database/conference/conferences.model";
import AnonymousError from "./AnonymousError";
import { downloadQueue } from "./queue";
import { isConnected } from "./database/database";
import AnonymizedFile from "./AnonymizedFile";
import AnonymizedRepositoryModel from "./database/anonymizedRepositories/anonymizedRepositories.model";
import { getRepositoryFromGitHub } from "./source/GitHubRepository";
import config from "../config";
@@ -71,6 +70,15 @@ export default class Repository {
});
}
this.owner = new User(new UserModel({ _id: data.owner }));
if (this.source instanceof GitHubBase) {
const originalToken = this._model.source.accessToken;
this.source.getToken(this.owner.id).then((token) => {
if (originalToken != token) {
this._model.source.accessToken = token;
this._model.save();
}
});
}
this.owner.model.isNew = false;
}
@@ -175,12 +183,19 @@ export default class Repository {
return storage.archive(this.repoId, "", {
format: "zip",
fileTransformer: (filename: string) =>
new AnonymizeTransformer(
new AnonymizedFile({
repository: this,
anonymizedPath: filename,
})
),
this.generateAnonymizeTransformer(filename),
});
}
generateAnonymizeTransformer(filePath: string) {
return new AnonymizeTransformer({
filePath: filePath,
terms: this.options.terms,
image: this.options.image,
link: this.options.link,
repoId: this.repoId,
repoName: (this.source as GitHubBase).githubRepository?.fullName,
branchName: (this.source as GitHubBase).branch?.name || "main",
});
}
@@ -202,7 +217,7 @@ export default class Repository {
) {
// Only GitHubBase can be update for the moment
if (this.source instanceof GitHubBase) {
const token = await this.source.getToken();
const token = await this.source.getToken(this.owner.id);
const branches = await this.source.githubRepository.branches({
force: true,
accessToken: token,
@@ -226,10 +241,13 @@ export default class Repository {
accessToken: token,
}
);
if (commitInfo.commit.author?.date) {
this._model.source.commitDate = new Date(
commitInfo.commit.author?.date
);
if (
commitInfo.commit?.author?.date ||
commitInfo.commit?.committer?.date
) {
const d = (commitInfo.commit?.author?.date ||
commitInfo.commit.committer?.date) as string;
this._model.source.commitDate = new Date(d);
}
branch.commit = newCommit;
@@ -252,7 +270,7 @@ export default class Repository {
if (this.source.type == "GitHubDownload") {
const repository = await getRepositoryFromGitHub({
accessToken: await this.source.getToken(),
accessToken: await this.source.getToken(this.owner.id),
owner: this.source.githubRepository.owner,
repo: this.source.githubRepository.repo,
});

24
src/anonymize-utils.ts
View File

@@ -36,32 +36,26 @@ export class AnonymizeTransformer extends Transform {
public wasAnonimized = false;
public isText: boolean | null = null;
constructor(private readonly file: AnonymizedFile) {
constructor(
private readonly opt: {
filePath: string;
} & ConstructorParameters<typeof ContentAnonimizer>[1]
) {
super();
this.isText = isTextFile(this.file.anonymizedPath);
this.isText = isTextFile(this.opt.filePath);
}
_transform(chunk: Buffer, encoding: string, callback: () => void) {
trace
.getTracer("ano-file")
.startActiveSpan("AnonymizeTransformer.transform", async (span) => {
span.setAttribute("path", this.file.anonymizedPath);
span.setAttribute("path", this.opt.filePath);
if (this.isText === null) {
this.isText = isTextFile(this.file.anonymizedPath, chunk);
this.isText = isTextFile(this.opt.filePath, chunk);
}
if (this.isText) {
const anonimizer = new ContentAnonimizer(chunk.toString(), {
repoId: this.file.repository.repoId,
image: this.file.repository.options.image,
link: this.file.repository.options.link,
terms: this.file.repository.options.terms,
repoName: (this.file.repository.source as GitHubBase)
.githubRepository?.fullName,
branchName:
(this.file.repository.source as GitHubBase).branch?.name ||
"main",
});
const anonimizer = new ContentAnonimizer(chunk.toString(), this.opt);
anonimizer.anonymize();
if (anonimizer.wasAnonymized) {
this.wasAnonimized = true;

11
src/source/GitHubDownload.ts
View File

@@ -37,13 +37,13 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
});
}
async download(progress?: (status: string) => void) {
async download(token: string, progress?: (status: string) => void) {
const span = trace.getTracer("ano-file").startSpan("GHDownload.download");
span.setAttribute("repoId", this.githubRepository.fullName || "");
try {
let response: OctokitResponse<unknown, number>;
try {
response = await this._getZipUrl(await this.getToken());
response = await this._getZipUrl(token);
} catch (error) {
span.recordException(error as Error);
throw new AnonymousError("repo_not_accessible", {
@@ -117,7 +117,10 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
await file.originalPath();
// the cache is not ready, we need to download the repository
await this.download(progress);
await this.download(
await this.getToken(file.repository.owner.id),
progress
);
return storage.read(this.repoId, file.filePath);
} finally {
span.end();
@@ -126,7 +129,7 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
async getFiles() {
if ((await storage.exists(this.repoId)) === FILE_TYPE.NOT_FOUND) {
await this.download();
await this.download(await this.getToken());
}
return storage.listFiles(this.repoId);
}

21
src/source/GitHubStream.ts
View File

@@ -129,13 +129,14 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
span.setAttribute("repoName", this.githubRepository.fullName || "");
try {
let commit = this.branch?.commit;
return this.getTree(commit);
return this.getTree(await this.getToken(), commit);
} finally {
span.end();
}
}
private async getTree(
token: string,
sha: string,
truncatedTree: Tree = {},
parentPath: string = "",
@@ -151,7 +152,7 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
let ghRes: Awaited<ReturnType<typeof this.getGHTree>>;
try {
count.request++;
ghRes = await this.getGHTree(sha, { recursive: true });
ghRes = await this.getGHTree(token, sha, { recursive: true });
} catch (error) {
console.error(error);
span.recordException(error as Error);
@@ -177,17 +178,21 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
const tree = this.tree2Tree(ghRes.tree, truncatedTree, parentPath);
count.file += ghRes.tree.length;
if (ghRes.truncated) {
await this.getTruncatedTree(sha, tree, parentPath, count);
await this.getTruncatedTree(token, sha, tree, parentPath, count);
}
span.end();
return tree;
}
private async getGHTree(sha: string, opt = { recursive: true }) {
private async getGHTree(
token: string,
sha: string,
opt = { recursive: true }
) {
const span = trace.getTracer("ano-file").startSpan("GHStream.getGHTree");
span.setAttribute("sha", sha);
try {
const octokit = GitHubBase.octokit(await this.getToken());
const octokit = GitHubBase.octokit(token);
const ghRes = await octokit.git.getTree({
owner: this.githubRepository.owner,
repo: this.githubRepository.repo,
@@ -201,6 +206,7 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
}
private async getTruncatedTree(
token: string,
sha: string,
truncatedTree: Tree = {},
parentPath: string = "",
@@ -220,7 +226,7 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
let data = null;
try {
data = await this.getGHTree(sha, { recursive: false });
data = await this.getGHTree(token, sha, { recursive: false });
this.tree2Tree(data.tree, truncatedTree, parentPath);
} catch (error) {
span.recordException(error as Error);
@@ -235,6 +241,7 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
const elementPath = path.join(parentPath, file.path);
promises.push(
this.getTruncatedTree(
token,
file.sha,
truncatedTree,
elementPath,
@@ -247,7 +254,7 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
await Promise.all(promises);
} else {
try {
const data = await this.getGHTree(sha, { recursive: true });
const data = await this.getGHTree(token, sha, { recursive: true });
this.tree2Tree(data.tree, truncatedTree, parentPath);
if (data.truncated) {
// TODO: TRUNCATED