CalvinBackup/anonymous_github
1
0
Fork 0
mirror of https://github.com/tdurieux/anonymous_github.git synced 2026-02-13 02:42:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: adds opentelemetry support

Browse Source
This commit is contained in:
tdurieux
2024-03-27 11:17:56 +00:00
parent 803720e2ea
commit 0caf786c9c
24 changed files with 4522 additions and 1187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

193
src/Repository.ts
View File

@@ -27,6 +27,7 @@ import AnonymizedFile from "./AnonymizedFile";
import AnonymizedRepositoryModel from "./database/anonymizedRepositories/anonymizedRepositories.model";
import { getRepositoryFromGitHub } from "./source/GitHubRepository";
import config from "../config";
import { trace } from "@opentelemetry/api";
function anonymizeTreeRecursive(
tree: TreeElement,
@@ -98,6 +99,7 @@ export default class Repository {
}
): Promise<Tree> {
const terms = this._model.options.terms || [];
if (terms.length === 0) return this.files(opt);
return anonymizeTreeRecursive(await this.files(opt), terms, opt) as Tree;
}
@@ -108,25 +110,31 @@ export default class Repository {
* @returns The file tree
*/
async files(opt: { force?: boolean } = { force: false }): Promise<Tree> {
if (!this._model.originalFiles && !opt.force) {
const res = await AnonymizedRepositoryModel.findById(this._model._id, {
originalFiles: 1,
});
if (!res) throw new AnonymousError("repository_not_found");
this.model.originalFiles = res.originalFiles;
const span = trace.getTracer("ano-file").startSpan("Repository.files");
span.setAttribute("repoId", this.repoId);
try {
if (!this._model.originalFiles && !opt.force) {
const res = await AnonymizedRepositoryModel.findById(this._model._id, {
originalFiles: 1,
});
if (!res) throw new AnonymousError("repository_not_found");
this.model.originalFiles = res.originalFiles;
}
if (
this._model.originalFiles &&
Object.getOwnPropertyNames(this._model.originalFiles).length !== 0 &&
!opt.force
) {
return this._model.originalFiles;
}
const files = await this.source.getFiles();
this._model.originalFiles = files;
this._model.size = { storage: 0, file: 0 };
await this.computeSize();
return files;
} finally {
span.end();
}
if (
this._model.originalFiles &&
Object.getOwnPropertyNames(this._model.originalFiles).length !== 0 &&
!opt.force
) {
return this._model.originalFiles;
}
const files = await this.source.getFiles();
this._model.originalFiles = files;
this._model.size = { storage: 0, file: 0 };
await this.computeSize();
return files;
}
/**
@@ -191,6 +199,10 @@ export default class Repository {
* @returns void
*/
async updateIfNeeded(opt?: { force: boolean }): Promise<void> {
const span = trace
.getTracer("ano-file")
.startSpan("Repository.updateIfNeeded");
span.setAttribute("repoId", this.repoId);
const yesterday = new Date();
yesterday.setDate(yesterday.getDate() - 1);
if (
@@ -212,6 +224,8 @@ export default class Repository {
this.status == RepositoryStatus.READY
) {
console.log(`[UPDATE] ${this._model.repoId} is up to date`);
span.setAttribute("status", "up_to_date");
span.end();
return;
}
this._model.source.commit = newCommit;
@@ -234,6 +248,8 @@ export default class Repository {
);
await this.updateStatus(RepositoryStatus.ERROR, "branch_not_found");
await this.resetSate();
span.setAttribute("status", "branch_not_found");
span.end();
throw new AnonymousError("branch_not_found", {
object: this,
});
@@ -267,6 +283,7 @@ export default class Repository {
});
}
}
span.end();
}
/**
* Download the require state for the repository to work
@@ -274,20 +291,32 @@ export default class Repository {
* @returns void
*/
async anonymize() {
if (this.status === RepositoryStatus.READY) return;
const span = trace.getTracer("ano-file").startSpan("Repository.anonymize");
span.setAttribute("repoId", this.repoId);
if (this.status === RepositoryStatus.READY) {
span.end();
return;
}
await this.updateStatus(RepositoryStatus.PREPARING);
await this.files();
return this.updateStatus(RepositoryStatus.READY);
await this.updateStatus(RepositoryStatus.READY);
span.end();
}
/**
* Update the last view and view count
*/
async countView() {
this._model.lastView = new Date();
this._model.pageView = (this._model.pageView || 0) + 1;
if (!isConnected) return this.model;
return this._model.save();
const span = trace.getTracer("ano-file").startSpan("Repository.countView");
span.setAttribute("repoId", this.repoId);
try {
this._model.lastView = new Date();
this._model.pageView = (this._model.pageView || 0) + 1;
if (!isConnected) return this.model;
return this._model.save();
} finally {
span.end();
}
}
/**
@@ -296,36 +325,54 @@ export default class Repository {
* @param errorMessage a potential error message to display
*/
async updateStatus(status: RepositoryStatus, statusMessage?: string) {
if (!status) return this.model;
this._model.status = status;
this._model.statusDate = new Date();
this._model.statusMessage = statusMessage;
if (!isConnected) return this.model;
return this._model.save();
const span = trace
.getTracer("ano-file")
.startSpan("Repository.updateStatus");
span.setAttribute("repoId", this.repoId);
span.setAttribute("status", status);
span.setAttribute("statusMessage", statusMessage || "");
try {
if (!status) return this.model;
this._model.status = status;
this._model.statusDate = new Date();
this._model.statusMessage = statusMessage;
if (!isConnected) return this.model;
return this._model.save();
} finally {
span.end();
}
}
/**
* Expire the repository
*/
async expire() {
const span = trace.getTracer("ano-file").startSpan("Repository.expire");
span.setAttribute("repoId", this.repoId);
await this.updateStatus(RepositoryStatus.EXPIRING);
await this.resetSate();
await this.updateStatus(RepositoryStatus.EXPIRED);
span.end();
}
/**
* Remove the repository
*/
async remove() {
const span = trace.getTracer("ano-file").startSpan("Repository.remove");
span.setAttribute("repoId", this.repoId);
await this.updateStatus(RepositoryStatus.REMOVING);
await this.resetSate();
await this.updateStatus(RepositoryStatus.REMOVED);
span.end();
}
/**
* Reset/delete the state of the repository
*/
async resetSate(status?: RepositoryStatus, statusMessage?: string) {
const span = trace.getTracer("ano-file").startSpan("Repository.resetState");
span.setAttribute("repoId", this.repoId);
// remove attribute
this._model.size = { storage: 0, file: 0 };
this._model.originalFiles = undefined;
@@ -335,6 +382,7 @@ export default class Repository {
// remove cache
await this.removeCache();
console.log(`[RESET] ${this._model.repoId} has been reset`);
span.end();
}
/**
@@ -342,12 +390,20 @@ export default class Repository {
* @returns
*/
async removeCache() {
this.model.isReseted = true;
await this.model.save();
if (
(await storage.exists(this._model.repoId + "/")) !== FILE_TYPE.NOT_FOUND
) {
return storage.rm(this._model.repoId + "/");
const span = trace
.getTracer("ano-file")
.startSpan("Repository.removeCache");
span.setAttribute("repoId", this.repoId);
try {
this.model.isReseted = true;
await this.model.save();
if (
(await storage.exists(this._model.repoId + "/")) !== FILE_TYPE.NOT_FOUND
) {
return storage.rm(this._model.repoId + "/");
}
} finally {
span.end();
}
}
@@ -366,28 +422,37 @@ export default class Repository {
*/
file: number;
}> {
if (this.status !== RepositoryStatus.READY) return { storage: 0, file: 0 };
if (this._model.size.file) return this._model.size;
function recursiveCount(files: Tree): { storage: number; file: number } {
const out = { storage: 0, file: 0 };
for (const name in files) {
const file = files[name];
if (file.size && parseInt(file.size.toString()) == file.size) {
out.storage += file.size as number;
out.file++;
} else if (typeof file == "object") {
const r = recursiveCount(file as Tree);
out.storage += r.storage;
out.file += r.file;
const span = trace
.getTracer("ano-file")
.startSpan("Repository.removeCache");
span.setAttribute("repoId", this.repoId);
try {
if (this.status !== RepositoryStatus.READY)
return { storage: 0, file: 0 };
if (this._model.size.file) return this._model.size;
function recursiveCount(files: Tree): { storage: number; file: number } {
const out = { storage: 0, file: 0 };
for (const name in files) {
const file = files[name];
if (file.size && parseInt(file.size.toString()) == file.size) {
out.storage += file.size as number;
out.file++;
} else if (typeof file == "object") {
const r = recursiveCount(file as Tree);
out.storage += r.storage;
out.file += r.file;
}
}
return out;
}
return out;
}
const files = await this.files();
this._model.size = recursiveCount(files);
await this._model.save();
return this._model.size;
const files = await this.files();
this._model.size = recursiveCount(files);
await this._model.save();
return this._model.size;
} finally {
span.end();
}
}
/**
@@ -396,14 +461,20 @@ export default class Repository {
* @returns conference of the repository
*/
async conference(): Promise<Conference | null> {
if (!this._model.conference) {
const span = trace.getTracer("ano-file").startSpan("Repository.conference");
span.setAttribute("repoId", this.repoId);
try {
if (!this._model.conference) {
return null;
}
const conference = await ConferenceModel.findOne({
conferenceID: this._model.conference,
});
if (conference) return new Conference(conference);
return null;
} finally {
span.end();
}
const conference = await ConferenceModel.findOne({
conferenceID: this._model.conference,
});
if (conference) return new Conference(conference);
return null;
}
/***** Getters ********/