mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-09-12 21:58:57 +02:00
Remove OpenTelemetry tracing infrastructure (#662)
This commit is contained in:
Thomas Durieux
@@ -4,7 +4,6 @@ import { OctokitResponse } from "@octokit/types";
|
||||
|
||||
import storage from "../storage";
|
||||
import GitHubBase, { GitHubBaseData } from "./GitHubBase";
|
||||
import { trace } from "@opentelemetry/api";
|
||||
import { FILE_TYPE } from "../storage/Storage";
|
||||
import { octokit } from "../GitHubUtils";
|
||||
import AnonymousError from "../AnonymousError";
|
||||
@@ -27,47 +26,39 @@ export default class GitHubDownload extends GitHubBase {
|
||||
}
|
||||
|
||||
async download(progress?: (status: string) => void) {
|
||||
const span = trace.getTracer("ano-file").startSpan("GHDownload.download");
|
||||
span.setAttribute("repoId", this.data.repoId);
|
||||
let response: OctokitResponse<unknown, number>;
|
||||
try {
|
||||
let response: OctokitResponse<unknown, number>;
|
||||
try {
|
||||
response = await this.getZipUrl();
|
||||
} catch (error) {
|
||||
span.recordException(error as Error);
|
||||
throw new AnonymousError("repo_not_found", {
|
||||
httpStatus: (error as any).status || 404,
|
||||
object: this.data,
|
||||
cause: error as Error,
|
||||
});
|
||||
}
|
||||
await storage.mk(this.data.repoId);
|
||||
try {
|
||||
const downloadStream = got.stream(response.url);
|
||||
downloadStream.addListener(
|
||||
"downloadProgress",
|
||||
(p: { transferred?: number }) => {
|
||||
if (progress && p.transferred) {
|
||||
progress("Repository download: " + humanFileSize(p.transferred));
|
||||
}
|
||||
response = await this.getZipUrl();
|
||||
} catch (error) {
|
||||
throw new AnonymousError("repo_not_found", {
|
||||
httpStatus: (error as any).status || 404,
|
||||
object: this.data,
|
||||
cause: error as Error,
|
||||
});
|
||||
}
|
||||
await storage.mk(this.data.repoId);
|
||||
try {
|
||||
const downloadStream = got.stream(response.url);
|
||||
downloadStream.addListener(
|
||||
"downloadProgress",
|
||||
(p: { transferred?: number }) => {
|
||||
if (progress && p.transferred) {
|
||||
progress("Repository download: " + humanFileSize(p.transferred));
|
||||
}
|
||||
);
|
||||
await storage.extractZip(
|
||||
this.data.repoId,
|
||||
"",
|
||||
downloadStream,
|
||||
this.type
|
||||
);
|
||||
} catch (error) {
|
||||
span.recordException(error as Error);
|
||||
throw new AnonymousError("unable_to_download", {
|
||||
httpStatus: 500,
|
||||
cause: error as Error,
|
||||
object: this.data,
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
);
|
||||
await storage.extractZip(
|
||||
this.data.repoId,
|
||||
"",
|
||||
downloadStream,
|
||||
this.type
|
||||
);
|
||||
} catch (error) {
|
||||
throw new AnonymousError("unable_to_download", {
|
||||
httpStatus: 500,
|
||||
cause: error as Error,
|
||||
object: this.data,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,29 +66,21 @@ export default class GitHubDownload extends GitHubBase {
|
||||
file: AnonymizedFile,
|
||||
progress?: (status: string) => void
|
||||
): Promise<Readable> {
|
||||
const span = trace
|
||||
.getTracer("ano-file")
|
||||
.startSpan("GHDownload.getFileContent");
|
||||
span.setAttribute("repoId", this.data.repoId);
|
||||
try {
|
||||
const exists = await storage.exists(this.data.repoId, file.filePath);
|
||||
if (exists === FILE_TYPE.FILE) {
|
||||
return storage.read(this.data.repoId, file.filePath);
|
||||
} else if (exists === FILE_TYPE.FOLDER) {
|
||||
throw new AnonymousError("folder_not_supported", {
|
||||
httpStatus: 400,
|
||||
object: file,
|
||||
});
|
||||
}
|
||||
// will throw an error if the file is not in the repository
|
||||
await file.originalPath();
|
||||
|
||||
// the cache is not ready, we need to download the repository
|
||||
await this.download(progress);
|
||||
const exists = await storage.exists(this.data.repoId, file.filePath);
|
||||
if (exists === FILE_TYPE.FILE) {
|
||||
return storage.read(this.data.repoId, file.filePath);
|
||||
} finally {
|
||||
span.end();
|
||||
} else if (exists === FILE_TYPE.FOLDER) {
|
||||
throw new AnonymousError("folder_not_supported", {
|
||||
httpStatus: 400,
|
||||
object: file,
|
||||
});
|
||||
}
|
||||
// will throw an error if the file is not in the repository
|
||||
await file.originalPath();
|
||||
|
||||
// the cache is not ready, we need to download the repository
|
||||
await this.download(progress);
|
||||
return storage.read(this.data.repoId, file.filePath);
|
||||
}
|
||||
|
||||
async getFiles(progress?: (status: string) => void) {
|
||||
|
||||
+153
-189
@@ -1,7 +1,6 @@
|
||||
import { Branch } from "../types";
|
||||
import * as gh from "parse-github-url";
|
||||
import { RestEndpointMethodTypes } from "@octokit/rest";
|
||||
import { trace } from "@opentelemetry/api";
|
||||
|
||||
import AnonymousError from "../AnonymousError";
|
||||
import { isConnected } from "../../server/database";
|
||||
@@ -55,81 +54,64 @@ export class GitHubRepository {
|
||||
accessToken: string;
|
||||
}
|
||||
) {
|
||||
const span = trace
|
||||
.getTracer("ano-file")
|
||||
.startSpan("GHRepository.getCommitInfo");
|
||||
span.setAttribute("owner", this.owner);
|
||||
span.setAttribute("repo", this.repo);
|
||||
try {
|
||||
const oct = octokit(opt.accessToken);
|
||||
const commit = await oct.repos.getCommit({
|
||||
owner: this.owner,
|
||||
repo: this.repo,
|
||||
ref: sha,
|
||||
});
|
||||
return commit.data;
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
const oct = octokit(opt.accessToken);
|
||||
const commit = await oct.repos.getCommit({
|
||||
owner: this.owner,
|
||||
repo: this.repo,
|
||||
ref: sha,
|
||||
});
|
||||
return commit.data;
|
||||
}
|
||||
|
||||
async branches(opt: {
|
||||
accessToken: string;
|
||||
force?: boolean;
|
||||
}): Promise<Branch[]> {
|
||||
const span = trace.getTracer("ano-file").startSpan("GHRepository.branches");
|
||||
span.setAttribute("owner", this.owner);
|
||||
span.setAttribute("repo", this.repo);
|
||||
try {
|
||||
if (
|
||||
!this._data.branches ||
|
||||
this._data.branches.length == 0 ||
|
||||
opt?.force === true
|
||||
) {
|
||||
// get the list of repo from github
|
||||
const oct = octokit(opt.accessToken);
|
||||
try {
|
||||
const branches = (
|
||||
await oct.paginate("GET /repos/{owner}/{repo}/branches", {
|
||||
owner: this.owner,
|
||||
repo: this.repo,
|
||||
per_page: 100,
|
||||
})
|
||||
).map((b) => {
|
||||
return {
|
||||
name: b.name,
|
||||
commit: b.commit.sha,
|
||||
readme: this._data.branches?.filter(
|
||||
(f: Branch) => f.name == b.name
|
||||
)[0]?.readme,
|
||||
} as Branch;
|
||||
});
|
||||
this._data.branches = branches;
|
||||
if (isConnected) {
|
||||
await RepositoryModel.updateOne(
|
||||
{ externalId: this.id },
|
||||
{ $set: { branches } }
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
span.recordException(error as Error);
|
||||
throw new AnonymousError("repo_not_found", {
|
||||
httpStatus: (error as any).status,
|
||||
cause: error as Error,
|
||||
object: this,
|
||||
});
|
||||
if (
|
||||
!this._data.branches ||
|
||||
this._data.branches.length == 0 ||
|
||||
opt?.force === true
|
||||
) {
|
||||
// get the list of repo from github
|
||||
const oct = octokit(opt.accessToken);
|
||||
try {
|
||||
const branches = (
|
||||
await oct.paginate("GET /repos/{owner}/{repo}/branches", {
|
||||
owner: this.owner,
|
||||
repo: this.repo,
|
||||
per_page: 100,
|
||||
})
|
||||
).map((b) => {
|
||||
return {
|
||||
name: b.name,
|
||||
commit: b.commit.sha,
|
||||
readme: this._data.branches?.filter(
|
||||
(f: Branch) => f.name == b.name
|
||||
)[0]?.readme,
|
||||
} as Branch;
|
||||
});
|
||||
this._data.branches = branches;
|
||||
if (isConnected) {
|
||||
await RepositoryModel.updateOne(
|
||||
{ externalId: this.id },
|
||||
{ $set: { branches } }
|
||||
);
|
||||
}
|
||||
} else if (isConnected) {
|
||||
const q = await RepositoryModel.findOne({ externalId: this.id }).select(
|
||||
"branches"
|
||||
);
|
||||
this._data.branches = q?.branches;
|
||||
} catch (error) {
|
||||
throw new AnonymousError("repo_not_found", {
|
||||
httpStatus: (error as any).status,
|
||||
cause: error as Error,
|
||||
object: this,
|
||||
});
|
||||
}
|
||||
|
||||
return this._data.branches || [];
|
||||
} finally {
|
||||
span.end();
|
||||
} else if (isConnected) {
|
||||
const q = await RepositoryModel.findOne({ externalId: this.id }).select(
|
||||
"branches"
|
||||
);
|
||||
this._data.branches = q?.branches;
|
||||
}
|
||||
|
||||
return this._data.branches || [];
|
||||
}
|
||||
|
||||
async readme(opt: {
|
||||
@@ -137,60 +119,52 @@ export class GitHubRepository {
|
||||
force?: boolean;
|
||||
accessToken: string;
|
||||
}): Promise<string | undefined> {
|
||||
const span = trace.getTracer("ano-file").startSpan("GHRepository.readme");
|
||||
span.setAttribute("owner", this.owner);
|
||||
span.setAttribute("repo", this.repo);
|
||||
try {
|
||||
if (!opt.branch) opt.branch = this._data.defaultBranch || "master";
|
||||
if (!opt.branch) opt.branch = this._data.defaultBranch || "master";
|
||||
|
||||
const model = await RepositoryModel.findOne({
|
||||
externalId: this.id,
|
||||
}).select("branches");
|
||||
const model = await RepositoryModel.findOne({
|
||||
externalId: this.id,
|
||||
}).select("branches");
|
||||
|
||||
if (!model) {
|
||||
throw new AnonymousError("repo_not_found", { httpStatus: 404 });
|
||||
}
|
||||
if (!model) {
|
||||
throw new AnonymousError("repo_not_found", { httpStatus: 404 });
|
||||
}
|
||||
|
||||
this._data.branches = await this.branches(opt);
|
||||
model.branches = this._data.branches;
|
||||
this._data.branches = await this.branches(opt);
|
||||
model.branches = this._data.branches;
|
||||
|
||||
const selected = model.branches.filter((f) => f.name == opt.branch)[0];
|
||||
if (selected && (!selected.readme || opt?.force === true)) {
|
||||
// get the list of repo from github
|
||||
const oct = octokit(opt.accessToken);
|
||||
try {
|
||||
const ghRes = await oct.repos.getReadme({
|
||||
owner: this.owner,
|
||||
repo: this.repo,
|
||||
ref: selected?.commit,
|
||||
});
|
||||
const readme = Buffer.from(
|
||||
ghRes.data.content,
|
||||
ghRes.data.encoding as BufferEncoding
|
||||
).toString("utf-8");
|
||||
selected.readme = readme;
|
||||
await model.save();
|
||||
} catch (error) {
|
||||
span.recordException(error as Error);
|
||||
throw new AnonymousError("readme_not_available", {
|
||||
httpStatus: 404,
|
||||
cause: error as Error,
|
||||
object: this,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!selected) {
|
||||
const selected = model.branches.filter((f) => f.name == opt.branch)[0];
|
||||
if (selected && (!selected.readme || opt?.force === true)) {
|
||||
// get the list of repo from github
|
||||
const oct = octokit(opt.accessToken);
|
||||
try {
|
||||
const ghRes = await oct.repos.getReadme({
|
||||
owner: this.owner,
|
||||
repo: this.repo,
|
||||
ref: selected?.commit,
|
||||
});
|
||||
const readme = Buffer.from(
|
||||
ghRes.data.content,
|
||||
ghRes.data.encoding as BufferEncoding
|
||||
).toString("utf-8");
|
||||
selected.readme = readme;
|
||||
await model.save();
|
||||
} catch (error) {
|
||||
throw new AnonymousError("readme_not_available", {
|
||||
httpStatus: 404,
|
||||
cause: error as Error,
|
||||
object: this,
|
||||
});
|
||||
}
|
||||
|
||||
return selected.readme;
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
|
||||
if (!selected) {
|
||||
throw new AnonymousError("readme_not_available", {
|
||||
httpStatus: 404,
|
||||
object: this,
|
||||
});
|
||||
}
|
||||
|
||||
return selected.readme;
|
||||
}
|
||||
|
||||
public get owner(): string {
|
||||
@@ -235,60 +209,44 @@ export async function getRepositoryFromGitHub(opt: {
|
||||
accessToken: string;
|
||||
force?: boolean;
|
||||
}) {
|
||||
const span = trace
|
||||
.getTracer("ano-file")
|
||||
.startSpan("GHRepository.getRepositoryFromGitHub");
|
||||
span.setAttribute("owner", opt.owner);
|
||||
span.setAttribute("repo", opt.repo);
|
||||
try {
|
||||
if (opt.repo.indexOf(".git") > -1) {
|
||||
opt.repo = opt.repo.replace(".git", "");
|
||||
}
|
||||
let dbModel = null;
|
||||
if (opt.repositoryID) {
|
||||
dbModel = isConnected
|
||||
? await RepositoryModel.findById(opt.repositoryID)
|
||||
: null;
|
||||
opt.owner = dbModel?.name?.split("/")[0] || opt.owner;
|
||||
opt.repo = dbModel?.name?.split("/")[1] || opt.repo;
|
||||
} else {
|
||||
dbModel = isConnected
|
||||
? await RepositoryModel.findOne({
|
||||
name: opt.owner + "/" + opt.repo,
|
||||
})
|
||||
: null;
|
||||
}
|
||||
if (dbModel && !opt.force) {
|
||||
return new GitHubRepository(dbModel);
|
||||
}
|
||||
const oct = octokit(opt.accessToken);
|
||||
let r: RestEndpointMethodTypes["repos"]["get"]["response"]["data"];
|
||||
try {
|
||||
r = (
|
||||
await oct.repos.get({
|
||||
owner: opt.owner,
|
||||
repo: opt.repo,
|
||||
if (opt.repo.indexOf(".git") > -1) {
|
||||
opt.repo = opt.repo.replace(".git", "");
|
||||
}
|
||||
let dbModel = null;
|
||||
if (opt.repositoryID) {
|
||||
dbModel = isConnected
|
||||
? await RepositoryModel.findById(opt.repositoryID)
|
||||
: null;
|
||||
opt.owner = dbModel?.name?.split("/")[0] || opt.owner;
|
||||
opt.repo = dbModel?.name?.split("/")[1] || opt.repo;
|
||||
} else {
|
||||
dbModel = isConnected
|
||||
? await RepositoryModel.findOne({
|
||||
name: opt.owner + "/" + opt.repo,
|
||||
})
|
||||
).data;
|
||||
} catch (error) {
|
||||
span.recordException(error as Error);
|
||||
if (
|
||||
error instanceof Error &&
|
||||
error.message.includes(
|
||||
"organization has enabled OAuth App access restrictions"
|
||||
)
|
||||
) {
|
||||
throw new AnonymousError("repo_access_limited", {
|
||||
httpStatus: 403,
|
||||
object: {
|
||||
owner: opt.owner,
|
||||
repo: opt.repo,
|
||||
},
|
||||
cause: error as Error,
|
||||
});
|
||||
}
|
||||
throw new AnonymousError("repo_not_found", {
|
||||
httpStatus: (error as any).status,
|
||||
: null;
|
||||
}
|
||||
if (dbModel && !opt.force) {
|
||||
return new GitHubRepository(dbModel);
|
||||
}
|
||||
const oct = octokit(opt.accessToken);
|
||||
let r: RestEndpointMethodTypes["repos"]["get"]["response"]["data"];
|
||||
try {
|
||||
r = (
|
||||
await oct.repos.get({
|
||||
owner: opt.owner,
|
||||
repo: opt.repo,
|
||||
})
|
||||
).data;
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof Error &&
|
||||
error.message.includes(
|
||||
"organization has enabled OAuth App access restrictions"
|
||||
)
|
||||
) {
|
||||
throw new AnonymousError("repo_access_limited", {
|
||||
httpStatus: 403,
|
||||
object: {
|
||||
owner: opt.owner,
|
||||
repo: opt.repo,
|
||||
@@ -296,32 +254,38 @@ export async function getRepositoryFromGitHub(opt: {
|
||||
cause: error as Error,
|
||||
});
|
||||
}
|
||||
if (!r)
|
||||
throw new AnonymousError("repo_not_found", {
|
||||
httpStatus: 404,
|
||||
object: {
|
||||
owner: opt.owner,
|
||||
repo: opt.repo,
|
||||
},
|
||||
});
|
||||
const model = dbModel || new RepositoryModel({ externalId: "gh_" + r.id });
|
||||
model.name = r.full_name;
|
||||
model.url = r.html_url;
|
||||
model.size = r.size;
|
||||
model.defaultBranch = r.default_branch;
|
||||
model.hasPage = r.has_pages;
|
||||
if (model.hasPage) {
|
||||
const ghPageRes = await oct.repos.getPages({
|
||||
throw new AnonymousError("repo_not_found", {
|
||||
httpStatus: (error as any).status,
|
||||
object: {
|
||||
owner: opt.owner,
|
||||
repo: opt.repo,
|
||||
});
|
||||
model.pageSource = ghPageRes.data.source;
|
||||
}
|
||||
if (isConnected) {
|
||||
await model.save();
|
||||
}
|
||||
return new GitHubRepository(model);
|
||||
} finally {
|
||||
span.end();
|
||||
},
|
||||
cause: error as Error,
|
||||
});
|
||||
}
|
||||
if (!r)
|
||||
throw new AnonymousError("repo_not_found", {
|
||||
httpStatus: 404,
|
||||
object: {
|
||||
owner: opt.owner,
|
||||
repo: opt.repo,
|
||||
},
|
||||
});
|
||||
const model = dbModel || new RepositoryModel({ externalId: "gh_" + r.id });
|
||||
model.name = r.full_name;
|
||||
model.url = r.html_url;
|
||||
model.size = r.size;
|
||||
model.defaultBranch = r.default_branch;
|
||||
model.hasPage = r.has_pages;
|
||||
if (model.hasPage) {
|
||||
const ghPageRes = await oct.repos.getPages({
|
||||
owner: opt.owner,
|
||||
repo: opt.repo,
|
||||
});
|
||||
model.pageSource = ghPageRes.data.source;
|
||||
}
|
||||
if (isConnected) {
|
||||
await model.save();
|
||||
}
|
||||
return new GitHubRepository(model);
|
||||
}
|
||||
|
||||
+90
-142
@@ -7,7 +7,6 @@ import { basename, dirname } from "path";
|
||||
|
||||
import * as stream from "stream";
|
||||
import AnonymousError from "../AnonymousError";
|
||||
import { trace } from "@opentelemetry/api";
|
||||
import { FILE_TYPE } from "../storage/Storage";
|
||||
import { octokit } from "../GitHubUtils";
|
||||
import FileModel from "../model/files/files.model";
|
||||
@@ -21,8 +20,6 @@ export default class GitHubStream extends GitHubBase {
|
||||
}
|
||||
|
||||
downloadFile(token: string, sha: string) {
|
||||
const span = trace.getTracer("ano-file").startSpan("GHStream.downloadFile");
|
||||
span.setAttribute("sha", sha);
|
||||
const oct = octokit(token);
|
||||
try {
|
||||
const { url } = oct.rest.git.getBlob.endpoint({
|
||||
@@ -40,14 +37,11 @@ export default class GitHubStream extends GitHubBase {
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
// span.recordException(error as Error);
|
||||
throw new AnonymousError("repo_not_accessible", {
|
||||
httpStatus: 404,
|
||||
object: this.data,
|
||||
cause: error as Error,
|
||||
});
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,12 +50,6 @@ export default class GitHubStream extends GitHubBase {
|
||||
repoId: string,
|
||||
fileSha: () => Promise<string> | string
|
||||
) {
|
||||
const span = trace
|
||||
.getTracer("ano-file")
|
||||
.startSpan("GHStream.getFileContent");
|
||||
span.setAttribute("repoId", repoId);
|
||||
span.setAttribute("file", filePath);
|
||||
|
||||
const fileInfo = await storage.exists(repoId, filePath);
|
||||
if (fileInfo == FILE_TYPE.FILE) {
|
||||
return storage.read(repoId, filePath);
|
||||
@@ -76,10 +64,6 @@ export default class GitHubStream extends GitHubBase {
|
||||
await fileSha()
|
||||
);
|
||||
|
||||
content.on("close", () => {
|
||||
span.end();
|
||||
});
|
||||
|
||||
// duplicate the stream to write it to the storage
|
||||
const stream1 = content.pipe(new stream.PassThrough());
|
||||
const stream2 = content.pipe(new stream.PassThrough());
|
||||
@@ -99,45 +83,30 @@ export default class GitHubStream extends GitHubBase {
|
||||
}
|
||||
|
||||
async getFileContent(file: AnonymizedFile): Promise<stream.Readable> {
|
||||
const span = trace
|
||||
.getTracer("ano-file")
|
||||
.startSpan("GHStream.getFileContent");
|
||||
span.setAttribute("repoId", file.repository.repoId);
|
||||
span.setAttribute("file", file.anonymizedPath);
|
||||
try {
|
||||
try {
|
||||
void file.filePath;
|
||||
} catch (_) {
|
||||
// compute the original path if ambiguous
|
||||
await file.originalPath();
|
||||
}
|
||||
return this.getFileContentCache(
|
||||
file.filePath,
|
||||
file.repository.repoId,
|
||||
async () => {
|
||||
const fileSha = await file.sha();
|
||||
if (!fileSha) {
|
||||
throw new AnonymousError("file_not_accessible", {
|
||||
httpStatus: 404,
|
||||
object: file,
|
||||
});
|
||||
}
|
||||
return fileSha;
|
||||
}
|
||||
);
|
||||
} finally {
|
||||
span.end();
|
||||
void file.filePath;
|
||||
} catch (_) {
|
||||
// compute the original path if ambiguous
|
||||
await file.originalPath();
|
||||
}
|
||||
return this.getFileContentCache(
|
||||
file.filePath,
|
||||
file.repository.repoId,
|
||||
async () => {
|
||||
const fileSha = await file.sha();
|
||||
if (!fileSha) {
|
||||
throw new AnonymousError("file_not_accessible", {
|
||||
httpStatus: 404,
|
||||
object: file,
|
||||
});
|
||||
}
|
||||
return fileSha;
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async getFiles(progress?: (status: string) => void) {
|
||||
const span = trace.getTracer("ano-file").startSpan("GHStream.getFiles");
|
||||
span.setAttribute("repoId", this.data.repoId);
|
||||
try {
|
||||
return this.getTruncatedTree(this.data.commit, progress);
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
return this.getTruncatedTree(this.data.commit, progress);
|
||||
}
|
||||
|
||||
private async getGHTree(
|
||||
@@ -145,25 +114,19 @@ export default class GitHubStream extends GitHubBase {
|
||||
count = { request: 0, file: 0 },
|
||||
opt = { recursive: true, callback: () => {} }
|
||||
) {
|
||||
const span = trace.getTracer("ano-file").startSpan("GHStream.getGHTree");
|
||||
span.setAttribute("sha", sha);
|
||||
try {
|
||||
const oct = octokit(await this.data.getToken());
|
||||
const ghRes = await oct.git.getTree({
|
||||
owner: this.data.organization,
|
||||
repo: this.data.repoName,
|
||||
tree_sha: sha,
|
||||
recursive: opt.recursive === true ? "1" : undefined,
|
||||
});
|
||||
count.request++;
|
||||
count.file += ghRes.data.tree.length;
|
||||
if (opt.callback) {
|
||||
opt.callback();
|
||||
}
|
||||
return ghRes.data;
|
||||
} finally {
|
||||
span.end();
|
||||
const oct = octokit(await this.data.getToken());
|
||||
const ghRes = await oct.git.getTree({
|
||||
owner: this.data.organization,
|
||||
repo: this.data.repoName,
|
||||
tree_sha: sha,
|
||||
recursive: opt.recursive === true ? "1" : undefined,
|
||||
});
|
||||
count.request++;
|
||||
count.file += ghRes.data.tree.length;
|
||||
if (opt.callback) {
|
||||
opt.callback();
|
||||
}
|
||||
return ghRes.data;
|
||||
}
|
||||
|
||||
private async getTruncatedTree(
|
||||
@@ -175,65 +138,56 @@ export default class GitHubStream extends GitHubBase {
|
||||
request: 0,
|
||||
file: 0,
|
||||
};
|
||||
const span = trace
|
||||
.getTracer("ano-file")
|
||||
.startSpan("GHStream.getTruncatedTree");
|
||||
span.setAttribute("sha", sha);
|
||||
span.setAttribute("parentPath", parentPath);
|
||||
const output: IFile[] = [];
|
||||
let data = null;
|
||||
try {
|
||||
let data = null;
|
||||
try {
|
||||
data = await this.getGHTree(sha, count, {
|
||||
recursive: false,
|
||||
callback: () => {
|
||||
if (progress) {
|
||||
progress("List file: " + count.file);
|
||||
}
|
||||
},
|
||||
});
|
||||
output.push(...this.tree2Tree(data.tree, parentPath));
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
if ((error as any).status == 409 || (error as any).status == 404) {
|
||||
// empty repo
|
||||
data = { tree: [] };
|
||||
} else {
|
||||
throw new AnonymousError("repo_not_found", {
|
||||
httpStatus: (error as any).status || 404,
|
||||
object: this.data,
|
||||
cause: error as Error,
|
||||
});
|
||||
}
|
||||
}
|
||||
const promises: ReturnType<GitHubStream["getGHTree"]>[] = [];
|
||||
const parentPaths: string[] = [];
|
||||
for (const file of data.tree) {
|
||||
if (file.type == "tree" && file.path && file.sha) {
|
||||
const elementPath = path.join(parentPath, file.path);
|
||||
parentPaths.push(elementPath);
|
||||
promises.push(
|
||||
this.getGHTree(file.sha, count, {
|
||||
recursive: true,
|
||||
callback: () => {
|
||||
if (progress) {
|
||||
progress("List file: " + count.file);
|
||||
}
|
||||
},
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
(await Promise.all(promises)).forEach((data, i) => {
|
||||
if (data.truncated) {
|
||||
// TODO: the tree is truncated
|
||||
}
|
||||
output.push(...this.tree2Tree(data.tree, parentPaths[i]));
|
||||
data = await this.getGHTree(sha, count, {
|
||||
recursive: false,
|
||||
callback: () => {
|
||||
if (progress) {
|
||||
progress("List file: " + count.file);
|
||||
}
|
||||
},
|
||||
});
|
||||
return output;
|
||||
} finally {
|
||||
span.end();
|
||||
output.push(...this.tree2Tree(data.tree, parentPath));
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
if ((error as any).status == 409 || (error as any).status == 404) {
|
||||
// empty repo
|
||||
data = { tree: [] };
|
||||
} else {
|
||||
throw new AnonymousError("repo_not_found", {
|
||||
httpStatus: (error as any).status || 404,
|
||||
object: this.data,
|
||||
cause: error as Error,
|
||||
});
|
||||
}
|
||||
}
|
||||
const promises: ReturnType<GitHubStream["getGHTree"]>[] = [];
|
||||
const parentPaths: string[] = [];
|
||||
for (const file of data.tree) {
|
||||
if (file.type == "tree" && file.path && file.sha) {
|
||||
const elementPath = path.join(parentPath, file.path);
|
||||
parentPaths.push(elementPath);
|
||||
promises.push(
|
||||
this.getGHTree(file.sha, count, {
|
||||
recursive: true,
|
||||
callback: () => {
|
||||
if (progress) {
|
||||
progress("List file: " + count.file);
|
||||
}
|
||||
},
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
(await Promise.all(promises)).forEach((data, i) => {
|
||||
if (data.truncated) {
|
||||
// TODO: the tree is truncated
|
||||
}
|
||||
output.push(...this.tree2Tree(data.tree, parentPaths[i]));
|
||||
});
|
||||
return output;
|
||||
}
|
||||
|
||||
private tree2Tree(
|
||||
@@ -247,25 +201,19 @@ export default class GitHubStream extends GitHubBase {
|
||||
}[],
|
||||
parentPath: string = ""
|
||||
) {
|
||||
const span = trace.getTracer("ano-file").startSpan("GHStream.tree2Tree");
|
||||
span.setAttribute("parentPath", parentPath);
|
||||
try {
|
||||
return tree.map((elem) => {
|
||||
const fullPath = path.join(parentPath, elem.path || "");
|
||||
let pathFile = dirname(fullPath);
|
||||
if (pathFile === ".") {
|
||||
pathFile = "";
|
||||
}
|
||||
return new FileModel({
|
||||
name: basename(fullPath),
|
||||
path: pathFile,
|
||||
repoId: this.data.repoId,
|
||||
size: elem.size,
|
||||
sha: elem.sha,
|
||||
});
|
||||
return tree.map((elem) => {
|
||||
const fullPath = path.join(parentPath, elem.path || "");
|
||||
let pathFile = dirname(fullPath);
|
||||
if (pathFile === ".") {
|
||||
pathFile = "";
|
||||
}
|
||||
return new FileModel({
|
||||
name: basename(fullPath),
|
||||
path: pathFile,
|
||||
repoId: this.data.repoId,
|
||||
size: elem.size,
|
||||
sha: elem.sha,
|
||||
});
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user