CalvinBackup/anonymous_github
1
0
Fork 0
mirror of https://github.com/tdurieux/anonymous_github.git synced 2026-07-08 14:47:59 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: adds opentelemetry support

Browse Source
This commit is contained in:
tdurieux
2024-03-27 11:17:56 +00:00
parent 803720e2ea
commit 0caf786c9c
24 changed files with 4522 additions and 1187 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockerfile
+2 -3
View File
@@ -11,7 +11,6 @@ COPY package.json .
COPY package-lock.json . COPY package-lock.json .
COPY tsconfig.json . COPY tsconfig.json .
COPY ecosystem.config.js .
COPY healthcheck.js . COPY healthcheck.js .
COPY src ./src COPY src ./src
@@ -20,6 +19,6 @@ COPY index.ts .
COPY config.ts . COPY config.ts .
RUN npm install && npm run build && npm cache clean --force RUN npm install && npm run build && npm cache clean --force
COPY opentelemetry.js .
CMD [ "node", "--require", "./opentelemetry.js", "./build/index.js"]
CMD [ "pm2-runtime", "ecosystem.config.js"]
docker-compose.yml
+29
View File
@@ -7,6 +7,8 @@ services:
image: tdurieux/anonymous_github:v2 image: tdurieux/anonymous_github:v2
env_file: env_file:
- ./.env - ./.env
volumes:
- ./repositories:/app/build/repositories/
environment: environment:
- REDIS_HOSTNAME=redis - REDIS_HOSTNAME=redis
- DB_HOSTNAME=mongodb - DB_HOSTNAME=mongodb
@@ -23,6 +25,7 @@ services:
links: links:
- mongodb - mongodb
- redis - redis
- opentelemetry
redis: redis:
image: "redis:alpine" image: "redis:alpine"
@@ -44,6 +47,8 @@ services:
MONGO_INITDB_ROOT_PASSWORD: $DB_PASSWORD MONGO_INITDB_ROOT_PASSWORD: $DB_PASSWORD
volumes: volumes:
- mongodb_data_container:/data/db - mongodb_data_container:/data/db
ports:
- 127.0.0.1:27017:27017
command: --quiet command: --quiet
healthcheck: healthcheck:
test: test:
@@ -55,6 +60,30 @@ services:
timeout: 10s timeout: 10s
retries: 5 retries: 5
opentelemetry:
image: otel/opentelemetry-collector
restart: always
command: ["--config=/etc/otel-collector-config.yaml"]
volumes:
- ./opentelemetry-collector.yml:/etc/otel-collector-config.yaml
depends_on:
- jaeger
- prometheus
jaeger:
image: jaegertracing/all-in-one:latest
restart: always
ports:
- 127.0.0.1:9411:9411
prometheus:
image: prom/prometheus:latest
restart: always
volumes:
- ./prometheus.yaml:/etc/prometheus/prometheus.yml
ports:
- 127.0.0.1:9090:9090
mongodb-backup: mongodb-backup:
image: tiredofit/db-backup image: tiredofit/db-backup
links: links:
ecosystem.config.js
-20
View File
@@ -1,20 +0,0 @@
module.exports = {
apps: [
{
name: "AnonymousGitHub",
script: "build/index.js",
exec_mode: "fork",
watch: false,
ignore_watch: [
"node_modules",
"repositories",
"repo",
"public",
".git",
"db_backups",
"build",
],
interpreter: "node",
},
],
};
import.js
-153
View File
@@ -1,153 +0,0 @@
const fs = require("fs").promises;
const ofs = require("fs");
const path = require("path");
const gh = require("parse-github-url");
const { Octokit } = require("@octokit/rest");
const config = require("./config");
const db = require("./utils/database");
const repoUtils = require("./utils/repository");
const fileUtils = require("./utils/file");
const githubUtils = require("./utils/github");
// const ROOT = "./repositories";
const ROOT = "./repo";
(async () => {
await db.connect();
const repositories = await fs.readdir(ROOT);
let index = 0;
for (let repo of repositories) {
// for (let repo of ["14bfc5c6-b794-487e-a58a-c54103a93c7b"]) {
console.log("Import ", index++, "/", repositories.length, " ", repo);
try {
const conf = await repoUtils.getConfig(repo);
if (conf) {
continue;
}
// const repoPath = path.join("./repositories", repo);
const repoPath = path.join(ROOT, repo);
const configPath = path.join(repoPath, "config.json");
if (!ofs.existsSync(configPath)) {
continue;
}
const repoConfig = JSON.parse(await fs.readFile(configPath));
const r = gh(repoConfig.repository);
if (r == null) {
console.log(`${repoConfig.repository} is not a valid github url.`);
continue;
}
const fullName = `${r.owner}/${r.name}`;
// const octokit = new Octokit({ auth: config.GITHUB_TOKEN });
// try {
// await octokit.apps.checkToken({
// client_id: config.CLIENT_ID,
// access_token: repoConfig.token,
// });
// } catch (error) {
// delete repoConfig.token;
// continue
// }
let token = repoConfig.token;
if (!token) {
token = config.GITHUB_TOKEN;
}
const branches = await repoUtils.getRepoBranches({
fullName,
token,
});
const details = await repoUtils.getRepoDetails({
fullName,
token,
});
let branch = details.default_branch;
if (r.branch && branches[r.branch]) {
branch = r.branch;
}
if (!branches[branch]) {
console.log(branch, details.default_branch, branches);
}
let commit = branches[branch].commit.sha;
const anonymizeDate = new Date();
let mode = "stream";
// if (details.size < 1024) {
// mode = "download";
// }
let expirationDate = null;
if (repoConfig.expiration_date) {
expirationDate = new Date(repoConfig.expiration_date["$date"]);
}
const expirationMode = repoConfig.expiration
? repoConfig.expiration
: "never";
const repoConfiguration = {
repoId: repo,
fullName,
// owner: "tdurieux",
owner: r.owner,
terms: repoConfig.terms,
repository: repoConfig.repository,
token: repoConfig.token,
branch,
commit,
anonymizeDate,
options: {
image: false,
mode,
expirationMode,
expirationDate,
update: true,
page: details.has_pages,
pdf: false,
notebook: true,
loc: false,
link: true,
},
};
await db.get("anonymized_repositories").updateOne(
{
repoId: repo,
},
{
$set: repoConfiguration,
},
{ upsert: true }
);
if (ofs.existsSync(repoUtils.getOriginalPath(repo))) {
await fs.rm(repoUtils.getOriginalPath(repo), {
recursive: true,
force: true,
});
}
if (ofs.existsSync(repoUtils.getAnonymizedPath(repo))) {
await fs.rm(repoUtils.getAnonymizedPath(repo), {
recursive: true,
force: true,
});
}
// await githubUtils.downloadRepoAndAnonymize(repoConfiguration);
// await fileUtils.getFileList({ repoConfig: repoConfiguration });
await repoUtils.updateStatus(repoConfiguration, "ready");
console.log(
expirationDate,
expirationDate != null && expirationDate < new Date(),
expirationDate < new Date()
);
if (
expirationMode != "never" &&
expirationDate != null &&
expirationDate < new Date()
) {
await repoUtils.updateStatus(repoConfiguration, "expired");
}
} catch (error) {
console.error(error);
}
}
await db.close();
})();
migrateDB.ts
-217
View File
@@ -1,217 +0,0 @@
require("dotenv").config();
import * as mongoose from "mongoose";
import config from "./config";
import * as database from "./src/database/database";
import RepositoryModel from "./src/database/repositories/repositories.model";
import AnonymizedRepositoryModel from "./src/database/anonymizedRepositories/anonymizedRepositories.model";
import UserModel from "./src/database/users/users.model";
const MONGO_URL = `mongodb://${config.DB_USERNAME}:${config.DB_PASSWORD}@${config.DB_HOSTNAME}:27017/`;
async function connect(db) {
const t = new mongoose.Mongoose();
t.set("useNewUrlParser", true);
t.set("useFindAndModify", true);
t.set("useUnifiedTopology", true);
const database = t.connection;
await t.connect(MONGO_URL + db, {
authSource: "admin",
useCreateIndex: true,
useFindAndModify: true,
});
return database;
}
(async () => {
await database.connect();
const oldDB = await connect("anonymous_github");
console.log("Import Users");
let index = 0;
const userQuery = oldDB.collection("users").find();
const totalUser = await userQuery.count();
while (await userQuery.hasNext()) {
const r = await userQuery.next();
index++;
console.log(`Import User [${index}/${totalUser}]: ${r.username}`);
const newRepos = [];
const allRepoIds = [];
if (r.repositories) {
const finds = await RepositoryModel.find({
externalId: {
$in: r.repositories.map((repo) => "gh_" + repo.id),
},
}).select("externalId");
finds.forEach((f) => allRepoIds.push(f.id));
const repoIds = new Set<string>();
const toInsert = r.repositories.filter((f) => {
if (repoIds.has(f.id)) return false;
repoIds.add(f.id);
const externalId = "gh_" + f.id;
return finds.filter((f) => f.externalId == externalId).length == 0;
});
for (const repo of toInsert) {
newRepos.push(
new RepositoryModel({
externalId: "gh_" + repo.id,
name: repo.full_name,
url: repo.html_url,
size: repo.size,
defaultBranch: repo.default_branch,
})
);
}
if (newRepos.length > 0) {
await RepositoryModel.insertMany(newRepos);
}
newRepos.forEach((f) => allRepoIds.push(f.id));
}
const user = new UserModel({
accessTokens: {
github: r.accessToken,
},
externalIDs: {
github: r.profile.id,
},
username: r.username,
emails: r.profile.emails?.map((email) => {
return { email: email.value, default: false };
}),
photo: r.profile.photos[0]?.value,
repositories: allRepoIds,
default: {
terms: r.default?.terms,
options: r.default?.options,
},
});
if (user.emails?.length) user.emails[0].default = true;
await user.save();
}
console.log("Import Repositories");
const repoQuery = oldDB.collection("repositories").find();
const totalRepository = await repoQuery.count();
index = 0;
while (await repoQuery.hasNext()) {
const r = await repoQuery.next();
if (!r.id) continue;
index++;
console.log(
`Import Repository [${index}/${totalRepository}]: ${r.fullName}`
);
let find = await RepositoryModel.findOne({
externalId: "gh_" + r.id,
});
if (find == null) {
find = new RepositoryModel({
externalId: "gh_" + r.id,
name: r.fullName,
url: r.html_url,
size: r.size,
defaultBranch: r.default_branch,
});
}
if (r.branches) {
const branches = [...Object.values(r.branches)].map((b: any) => {
const o: any = { name: b.name, commit: b.commit.sha };
if (b.name == find.defaultBranch) {
o.readme = r.readme;
}
return o;
});
find.branches = branches;
}
await find.save();
}
console.log("Import Anonymized Repositories");
const anoQuery = oldDB.collection("anonymized_repositories").find();
const totalAno = await anoQuery.count();
index = 0;
while (await anoQuery.hasNext()) {
const r = await anoQuery.next();
index++;
console.log(
`Import Anonymized Repository [${index}/${totalAno}]: ${r.repoId}`
);
let repo = await RepositoryModel.findOne({ name: r.fullName });
if (repo == null) {
const tmp = await oldDB
.collection("repositories")
.findOne({ fullName: r.fullName });
if (tmp) {
repo = await RepositoryModel.findOne({ externalId: "gh_" + tmp.id });
} else {
console.error(`Repository ${r.fullName} is not found (renamed)`);
}
}
let size = { storage: 0, file: 0 };
function recursiveCount(files) {
const out = { storage: 0, file: 0 };
for (const name in files) {
const file = files[name];
if (file.size && file.sha && parseInt(file.size) == file.size) {
out.storage += file.size as number;
out.file++;
} else if (typeof file == "object") {
const r = recursiveCount(file);
out.storage += r.storage;
out.file += r.file;
}
}
return out;
}
if (r.originalFiles) {
size = recursiveCount(r.originalFiles);
}
const owner = await UserModel.findOne({ username: r.owner }).select("_id");
await new AnonymizedRepositoryModel({
repoId: r.repoId,
status: r.status,
anonymizeDate: r.anonymizeDate,
lastView: r.lastView,
pageView: r.pageView,
owner: owner?.id,
size,
source: {
accessToken: r.token,
type: r.options.mode == "download" ? "GitHubDownload" : "GitHubStream",
branch: r.branch,
commit: r.commit,
repositoryId: repo?.id,
repositoryName: r.fullName,
},
options: {
terms: r.terms,
expirationMode: r.options.expirationMode,
expirationDate: r.options.expirationDate
? new Date(r.options.expirationDate)
: null,
update: r.options.update,
image: r.options.image,
pdf: r.options.pdf,
notebook: r.options.notebook,
loc: r.options.loc,
link: r.options.link,
page: r.options.page,
pageSource: r.options.pageSource,
},
}).save();
}
console.log("Import finished!");
setTimeout(() => process.exit(), 5000);
})();
opentelemetry-collector.yml
+40
View File
@@ -0,0 +1,40 @@
receivers:
otlp:
protocols:
grpc:
exporters:
prometheus:
endpoint: "0.0.0.0:8889"
const_labels:
label1: value1
debug:
otlp:
endpoint: jaeger:4317
tls:
insecure: true
processors:
batch:
extensions:
health_check:
pprof:
endpoint: :1888
zpages:
endpoint: :55679
service:
extensions: [health_check, pprof, zpages]
pipelines:
traces:
receivers: [otlp]
exporters: [debug, otlp]
metrics:
receivers: [otlp]
exporters: [debug, prometheus]
logs:
receivers: [otlp]
exporters: [debug]
opentelemetry.js
+29
View File
@@ -0,0 +1,29 @@
const opentelemetry = require("@opentelemetry/sdk-node");
const {
getNodeAutoInstrumentations,
} = require("@opentelemetry/auto-instrumentations-node");
const {
OTLPTraceExporter,
} = require("@opentelemetry/exporter-trace-otlp-grpc");
const {
OTLPMetricExporter,
} = require("@opentelemetry/exporter-metrics-otlp-grpc");
const { PeriodicExportingMetricReader } = require("@opentelemetry/sdk-metrics");
const { diag, DiagConsoleLogger, DiagLogLevel } = require("@opentelemetry/api");
// diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO);
const sdk = new opentelemetry.NodeSDK({
serviceName: "Anonymous-GitHub",
logRecordProcessor: getNodeAutoInstrumentations().logRecordProcessor,
traceExporter: new OTLPTraceExporter({
url: "http://opentelemetry:4317/v1/traces",
}),
metricReader: new PeriodicExportingMetricReader({
exporter: new OTLPMetricExporter({
url: "http://opentelemetry:4317/v1/metrics",
}),
}),
instrumentations: [getNodeAutoInstrumentations()],
});
sdk.start();
package-lock.json
Generated
+3311 -55
View File
File diff suppressed because it is too large Load Diff
package.json
+9
View File
@@ -35,6 +35,15 @@
"@octokit/oauth-app": "^6.0.0", "@octokit/oauth-app": "^6.0.0",
"@octokit/plugin-paginate-rest": "^8.0.0", "@octokit/plugin-paginate-rest": "^8.0.0",
"@octokit/rest": "^20.0.1", "@octokit/rest": "^20.0.1",
"@opentelemetry/api": "^1.8.0",
"@opentelemetry/auto-instrumentations-node": "^0.43.0",
"@opentelemetry/exporter-metrics-otlp-grpc": "^0.49.1",
"@opentelemetry/exporter-metrics-otlp-proto": "^0.49.1",
"@opentelemetry/exporter-trace-otlp-grpc": "^0.49.1",
"@opentelemetry/exporter-trace-otlp-proto": "^0.49.1",
"@opentelemetry/sdk-metrics": "^1.22.0",
"@opentelemetry/sdk-node": "^0.49.1",
"@opentelemetry/sdk-trace-node": "^1.22.0",
"@pm2/io": "^5.0.0", "@pm2/io": "^5.0.0",
"archiver": "^5.3.1", "archiver": "^5.3.1",
"bullmq": "^2.3.2", "bullmq": "^2.3.2",
prometheus.yaml
+6
View File
@@ -0,0 +1,6 @@
scrape_configs:
- job_name: 'otel-collector'
scrape_interval: 10s
static_configs:
- targets: ['opentelemetry:8889']
- targets: ['opentelemetry:8888']
src/AnonymizedFile.ts
+54 -7
View File
@@ -1,6 +1,7 @@
import { join, basename } from "path"; import { join, basename } from "path";
import { Response } from "express"; import { Response } from "express";
import { Readable } from "stream"; import { Readable } from "stream";
import { trace } from "@opentelemetry/api";
import Repository from "./Repository"; import Repository from "./Repository";
import { FILE_TYPE, Tree, TreeElement, TreeFile } from "./types"; import { FILE_TYPE, Tree, TreeElement, TreeFile } from "./types";
import storage from "./storage"; import storage from "./storage";
@@ -13,6 +14,7 @@ import {
import AnonymousError from "./AnonymousError"; import AnonymousError from "./AnonymousError";
import { handleError } from "./routes/route-utils"; import { handleError } from "./routes/route-utils";
import { lookup } from "mime-types"; import { lookup } from "mime-types";
import AnonymizedRepositoryModel from "./database/anonymizedRepositories/anonymizedRepositories.model";
/** /**
* Represent a file in a anonymized repository * Represent a file in a anonymized repository
@@ -36,9 +38,16 @@ export default class AnonymizedFile {
} }
async sha() { async sha() {
return trace.getTracer("ano-file").startActiveSpan("sha", async (span) => {
try {
span.setAttribute("anonymizedPath", this.anonymizedPath);
if (this._sha) return this._sha.replace(/"/g, ""); if (this._sha) return this._sha.replace(/"/g, "");
await this.originalPath(); await this.originalPath();
return this._sha?.replace(/"/g, ""); return this._sha?.replace(/"/g, "");
} finally {
span.end();
}
});
} }
/** /**
@@ -47,17 +56,24 @@ export default class AnonymizedFile {
* @returns the origin relative path of the file * @returns the origin relative path of the file
*/ */
async originalPath(): Promise<string> { async originalPath(): Promise<string> {
return trace
.getTracer("ano-file")
.startActiveSpan("originalPath", async (span) => {
try {
span.setAttribute("anonymizedPath", this.anonymizedPath);
if (this._originalPath) return this._originalPath; if (this._originalPath) return this._originalPath;
if (!this.anonymizedPath) if (!this.anonymizedPath) {
throw new AnonymousError("path_not_specified", { throw new AnonymousError("path_not_specified", {
object: this, object: this,
httpStatus: 400, httpStatus: 400,
}); });
}
const paths = this.anonymizedPath.trim().split("/");
let currentOriginal = (await this.repository.files({ let currentOriginal = (await this.repository.files({
force: false, force: false,
})) as TreeElement; })) as TreeElement;
const paths = this.anonymizedPath.trim().split("/");
let currentOriginalPath = ""; let currentOriginalPath = "";
for (let i = 0; i < paths.length; i++) { for (let i = 0; i < paths.length; i++) {
const fileName = paths[i]; const fileName = paths[i];
@@ -69,8 +85,10 @@ export default class AnonymizedFile {
const options = []; const options = [];
for (let originalFileName in currentOriginal) { for (let originalFileName in currentOriginal) {
if ( if (
anonymizePath(originalFileName, this.repository.options.terms) == anonymizePath(
fileName originalFileName,
this.repository.options.terms
) == fileName
) { ) {
options.push(originalFileName); options.push(originalFileName);
} }
@@ -130,6 +148,10 @@ export default class AnonymizedFile {
this._originalPath = currentOriginalPath; this._originalPath = currentOriginalPath;
return this._originalPath; return this._originalPath;
} finally {
span.end();
}
});
} }
extension() { extension() {
const filename = basename(this.anonymizedPath); const filename = basename(this.anonymizedPath);
@@ -154,6 +176,7 @@ export default class AnonymizedFile {
"heic", "heic",
].includes(extension); ].includes(extension);
} }
isFileSupported() { isFileSupported() {
const extension = this.extension(); const extension = this.extension();
if (!this.repository.options.pdf && extension == "pdf") { if (!this.repository.options.pdf && extension == "pdf") {
@@ -166,9 +189,15 @@ export default class AnonymizedFile {
} }
async content(): Promise<Readable> { async content(): Promise<Readable> {
return trace
.getTracer("ano-file")
.startActiveSpan("content", async (span) => {
try {
span.setAttribute("anonymizedPath", this.anonymizedPath);
if (this.anonymizedPath.includes(config.ANONYMIZATION_MASK)) { if (this.anonymizedPath.includes(config.ANONYMIZATION_MASK)) {
await this.originalPath(); await this.originalPath();
} }
span.addEvent("originalPath", { originalPath: this._originalPath });
if (this.fileSize && this.fileSize > config.MAX_FILE_SIZE) { if (this.fileSize && this.fileSize > config.MAX_FILE_SIZE) {
throw new AnonymousError("file_too_big", { throw new AnonymousError("file_too_big", {
object: this, object: this,
@@ -176,6 +205,7 @@ export default class AnonymizedFile {
}); });
} }
const exist = await storage.exists(this.originalCachePath); const exist = await storage.exists(this.originalCachePath);
span.addEvent("file_exist", { exist });
if (exist == FILE_TYPE.FILE) { if (exist == FILE_TYPE.FILE) {
return storage.read(this.originalCachePath); return storage.read(this.originalCachePath);
} else if (exist == FILE_TYPE.FOLDER) { } else if (exist == FILE_TYPE.FOLDER) {
@@ -185,10 +215,22 @@ export default class AnonymizedFile {
}); });
} }
return await this.repository.source?.getFileContent(this); return await this.repository.source?.getFileContent(this);
} finally {
span.end();
}
});
} }
async anonymizedContent() { async anonymizedContent() {
return (await this.content()).pipe(new AnonymizeTransformer(this)); return trace
.getTracer("ano-file")
.startActiveSpan("anonymizedContent", async (span) => {
span.setAttribute("anonymizedPath", this.anonymizedPath);
const content = await this.content();
return content.pipe(new AnonymizeTransformer(this)).on("close", () => {
span.end();
});
});
} }
get originalCachePath() { get originalCachePath() {
@@ -212,7 +254,9 @@ export default class AnonymizedFile {
} }
async send(res: Response): Promise<void> { async send(res: Response): Promise<void> {
return new Promise(async (resolve, reject) => { return trace.getTracer("ano-file").startActiveSpan("send", async (span) => {
span.setAttribute("anonymizedPath", this.anonymizedPath);
return new Promise<void>(async (resolve, reject) => {
try { try {
const content = await this.content(); const content = await this.content();
const mime = lookup(this.anonymizedPath); const mime = lookup(this.anonymizedPath);
@@ -229,7 +273,6 @@ export default class AnonymizedFile {
// unable to get file size // unable to get file size
console.error(error); console.error(error);
} }
const anonymizer = new AnonymizeTransformer(this); const anonymizer = new AnonymizeTransformer(this);
anonymizer.once("transform", (data) => { anonymizer.once("transform", (data) => {
@@ -249,12 +292,15 @@ export default class AnonymizedFile {
if (!content.closed && !content.destroyed) { if (!content.closed && !content.destroyed) {
content.destroy(); content.destroy();
} }
span.end();
resolve(); resolve();
}) })
.on("error", (error) => { .on("error", (error) => {
if (!content.closed && !content.destroyed) { if (!content.closed && !content.destroyed) {
content.destroy(); content.destroy();
} }
span.recordException(error);
span.end();
reject(error); reject(error);
handleError(error, res); handleError(error, res);
}); });
@@ -262,5 +308,6 @@ export default class AnonymizedFile {
handleError(error, res); handleError(error, res);
} }
}); });
});
} }
} }
src/Repository.ts
+74 -3
View File
@@ -27,6 +27,7 @@ import AnonymizedFile from "./AnonymizedFile";
import AnonymizedRepositoryModel from "./database/anonymizedRepositories/anonymizedRepositories.model"; import AnonymizedRepositoryModel from "./database/anonymizedRepositories/anonymizedRepositories.model";
import { getRepositoryFromGitHub } from "./source/GitHubRepository"; import { getRepositoryFromGitHub } from "./source/GitHubRepository";
import config from "../config"; import config from "../config";
import { trace } from "@opentelemetry/api";
function anonymizeTreeRecursive( function anonymizeTreeRecursive(
tree: TreeElement, tree: TreeElement,
@@ -98,6 +99,7 @@ export default class Repository {
} }
): Promise<Tree> { ): Promise<Tree> {
const terms = this._model.options.terms || []; const terms = this._model.options.terms || [];
if (terms.length === 0) return this.files(opt);
return anonymizeTreeRecursive(await this.files(opt), terms, opt) as Tree; return anonymizeTreeRecursive(await this.files(opt), terms, opt) as Tree;
} }
@@ -108,6 +110,9 @@ export default class Repository {
* @returns The file tree * @returns The file tree
*/ */
async files(opt: { force?: boolean } = { force: false }): Promise<Tree> { async files(opt: { force?: boolean } = { force: false }): Promise<Tree> {
const span = trace.getTracer("ano-file").startSpan("Repository.files");
span.setAttribute("repoId", this.repoId);
try {
if (!this._model.originalFiles && !opt.force) { if (!this._model.originalFiles && !opt.force) {
const res = await AnonymizedRepositoryModel.findById(this._model._id, { const res = await AnonymizedRepositoryModel.findById(this._model._id, {
originalFiles: 1, originalFiles: 1,
@@ -127,6 +132,9 @@ export default class Repository {
this._model.size = { storage: 0, file: 0 }; this._model.size = { storage: 0, file: 0 };
await this.computeSize(); await this.computeSize();
return files; return files;
} finally {
span.end();
}
} }
/** /**
@@ -191,6 +199,10 @@ export default class Repository {
* @returns void * @returns void
*/ */
async updateIfNeeded(opt?: { force: boolean }): Promise<void> { async updateIfNeeded(opt?: { force: boolean }): Promise<void> {
const span = trace
.getTracer("ano-file")
.startSpan("Repository.updateIfNeeded");
span.setAttribute("repoId", this.repoId);
const yesterday = new Date(); const yesterday = new Date();
yesterday.setDate(yesterday.getDate() - 1); yesterday.setDate(yesterday.getDate() - 1);
if ( if (
@@ -212,6 +224,8 @@ export default class Repository {
this.status == RepositoryStatus.READY this.status == RepositoryStatus.READY
) { ) {
console.log(`[UPDATE] ${this._model.repoId} is up to date`); console.log(`[UPDATE] ${this._model.repoId} is up to date`);
span.setAttribute("status", "up_to_date");
span.end();
return; return;
} }
this._model.source.commit = newCommit; this._model.source.commit = newCommit;
@@ -234,6 +248,8 @@ export default class Repository {
); );
await this.updateStatus(RepositoryStatus.ERROR, "branch_not_found"); await this.updateStatus(RepositoryStatus.ERROR, "branch_not_found");
await this.resetSate(); await this.resetSate();
span.setAttribute("status", "branch_not_found");
span.end();
throw new AnonymousError("branch_not_found", { throw new AnonymousError("branch_not_found", {
object: this, object: this,
}); });
@@ -267,6 +283,7 @@ export default class Repository {
}); });
} }
} }
span.end();
} }
/** /**
* Download the require state for the repository to work * Download the require state for the repository to work
@@ -274,20 +291,32 @@ export default class Repository {
* @returns void * @returns void
*/ */
async anonymize() { async anonymize() {
if (this.status === RepositoryStatus.READY) return; const span = trace.getTracer("ano-file").startSpan("Repository.anonymize");
span.setAttribute("repoId", this.repoId);
if (this.status === RepositoryStatus.READY) {
span.end();
return;
}
await this.updateStatus(RepositoryStatus.PREPARING); await this.updateStatus(RepositoryStatus.PREPARING);
await this.files(); await this.files();
return this.updateStatus(RepositoryStatus.READY); await this.updateStatus(RepositoryStatus.READY);
span.end();
} }
/** /**
* Update the last view and view count * Update the last view and view count
*/ */
async countView() { async countView() {
const span = trace.getTracer("ano-file").startSpan("Repository.countView");
span.setAttribute("repoId", this.repoId);
try {
this._model.lastView = new Date(); this._model.lastView = new Date();
this._model.pageView = (this._model.pageView || 0) + 1; this._model.pageView = (this._model.pageView || 0) + 1;
if (!isConnected) return this.model; if (!isConnected) return this.model;
return this._model.save(); return this._model.save();
} finally {
span.end();
}
} }
/** /**
@@ -296,36 +325,54 @@ export default class Repository {
* @param errorMessage a potential error message to display * @param errorMessage a potential error message to display
*/ */
async updateStatus(status: RepositoryStatus, statusMessage?: string) { async updateStatus(status: RepositoryStatus, statusMessage?: string) {
const span = trace
.getTracer("ano-file")
.startSpan("Repository.updateStatus");
span.setAttribute("repoId", this.repoId);
span.setAttribute("status", status);
span.setAttribute("statusMessage", statusMessage || "");
try {
if (!status) return this.model; if (!status) return this.model;
this._model.status = status; this._model.status = status;
this._model.statusDate = new Date(); this._model.statusDate = new Date();
this._model.statusMessage = statusMessage; this._model.statusMessage = statusMessage;
if (!isConnected) return this.model; if (!isConnected) return this.model;
return this._model.save(); return this._model.save();
} finally {
span.end();
}
} }
/** /**
* Expire the repository * Expire the repository
*/ */
async expire() { async expire() {
const span = trace.getTracer("ano-file").startSpan("Repository.expire");
span.setAttribute("repoId", this.repoId);
await this.updateStatus(RepositoryStatus.EXPIRING); await this.updateStatus(RepositoryStatus.EXPIRING);
await this.resetSate(); await this.resetSate();
await this.updateStatus(RepositoryStatus.EXPIRED); await this.updateStatus(RepositoryStatus.EXPIRED);
span.end();
} }
/** /**
* Remove the repository * Remove the repository
*/ */
async remove() { async remove() {
const span = trace.getTracer("ano-file").startSpan("Repository.remove");
span.setAttribute("repoId", this.repoId);
await this.updateStatus(RepositoryStatus.REMOVING); await this.updateStatus(RepositoryStatus.REMOVING);
await this.resetSate(); await this.resetSate();
await this.updateStatus(RepositoryStatus.REMOVED); await this.updateStatus(RepositoryStatus.REMOVED);
span.end();
} }
/** /**
* Reset/delete the state of the repository * Reset/delete the state of the repository
*/ */
async resetSate(status?: RepositoryStatus, statusMessage?: string) { async resetSate(status?: RepositoryStatus, statusMessage?: string) {
const span = trace.getTracer("ano-file").startSpan("Repository.resetState");
span.setAttribute("repoId", this.repoId);
// remove attribute // remove attribute
this._model.size = { storage: 0, file: 0 }; this._model.size = { storage: 0, file: 0 };
this._model.originalFiles = undefined; this._model.originalFiles = undefined;
@@ -335,6 +382,7 @@ export default class Repository {
// remove cache // remove cache
await this.removeCache(); await this.removeCache();
console.log(`[RESET] ${this._model.repoId} has been reset`); console.log(`[RESET] ${this._model.repoId} has been reset`);
span.end();
} }
/** /**
@@ -342,6 +390,11 @@ export default class Repository {
* @returns * @returns
*/ */
async removeCache() { async removeCache() {
const span = trace
.getTracer("ano-file")
.startSpan("Repository.removeCache");
span.setAttribute("repoId", this.repoId);
try {
this.model.isReseted = true; this.model.isReseted = true;
await this.model.save(); await this.model.save();
if ( if (
@@ -349,6 +402,9 @@ export default class Repository {
) { ) {
return storage.rm(this._model.repoId + "/"); return storage.rm(this._model.repoId + "/");
} }
} finally {
span.end();
}
} }
/** /**
@@ -366,7 +422,13 @@ export default class Repository {
*/ */
file: number; file: number;
}> { }> {
if (this.status !== RepositoryStatus.READY) return { storage: 0, file: 0 }; const span = trace
.getTracer("ano-file")
.startSpan("Repository.removeCache");
span.setAttribute("repoId", this.repoId);
try {
if (this.status !== RepositoryStatus.READY)
return { storage: 0, file: 0 };
if (this._model.size.file) return this._model.size; if (this._model.size.file) return this._model.size;
function recursiveCount(files: Tree): { storage: number; file: number } { function recursiveCount(files: Tree): { storage: number; file: number } {
const out = { storage: 0, file: 0 }; const out = { storage: 0, file: 0 };
@@ -388,6 +450,9 @@ export default class Repository {
this._model.size = recursiveCount(files); this._model.size = recursiveCount(files);
await this._model.save(); await this._model.save();
return this._model.size; return this._model.size;
} finally {
span.end();
}
} }
/** /**
@@ -396,6 +461,9 @@ export default class Repository {
* @returns conference of the repository * @returns conference of the repository
*/ */
async conference(): Promise<Conference | null> { async conference(): Promise<Conference | null> {
const span = trace.getTracer("ano-file").startSpan("Repository.conference");
span.setAttribute("repoId", this.repoId);
try {
if (!this._model.conference) { if (!this._model.conference) {
return null; return null;
} }
@@ -404,6 +472,9 @@ export default class Repository {
}); });
if (conference) return new Conference(conference); if (conference) return new Conference(conference);
return null; return null;
} finally {
span.end();
}
} }
/***** Getters ********/ /***** Getters ********/
src/User.ts
+15 -1
View File
@@ -6,6 +6,7 @@ import Repository from "./Repository";
import { GitHubRepository } from "./source/GitHubRepository"; import { GitHubRepository } from "./source/GitHubRepository";
import PullRequest from "./PullRequest"; import PullRequest from "./PullRequest";
import AnonymizedPullRequestModel from "./database/anonymizedPullRequests/anonymizedPullRequests.model"; import AnonymizedPullRequestModel from "./database/anonymizedPullRequests/anonymizedPullRequests.model";
import { trace } from "@opentelemetry/api";
/** /**
* Model for a user * Model for a user
@@ -55,6 +56,10 @@ export default class User {
*/ */
force: boolean; force: boolean;
}): Promise<GitHubRepository[]> { }): Promise<GitHubRepository[]> {
const span = trace
.getTracer("ano-file")
.startSpan("User.getGitHubRepositories");
span.setAttribute("username", this.username);
if ( if (
!this._model.repositories || !this._model.repositories ||
this._model.repositories.length == 0 || this._model.repositories.length == 0 ||
@@ -105,11 +110,14 @@ export default class User {
// have the model // have the model
await this._model.save(); await this._model.save();
span.end();
return repositories.map((r) => new GitHubRepository(r)); return repositories.map((r) => new GitHubRepository(r));
} else { } else {
return ( const out = (
await RepositoryModel.find({ _id: { $in: this._model.repositories } }) await RepositoryModel.find({ _id: { $in: this._model.repositories } })
).map((i) => new GitHubRepository(i)); ).map((i) => new GitHubRepository(i));
span.end();
return out;
} }
} }
@@ -118,6 +126,8 @@ export default class User {
* @returns the list of anonymized repositories * @returns the list of anonymized repositories
*/ */
async getRepositories() { async getRepositories() {
const span = trace.getTracer("ano-file").startSpan("User.getRepositories");
span.setAttribute("username", this.username);
const repositories = ( const repositories = (
await AnonymizedRepositoryModel.find( await AnonymizedRepositoryModel.find(
{ {
@@ -141,6 +151,7 @@ export default class User {
} }
} }
await Promise.all(promises); await Promise.all(promises);
span.end();
return repositories; return repositories;
} }
/** /**
@@ -148,6 +159,8 @@ export default class User {
* @returns the list of anonymized repositories * @returns the list of anonymized repositories
*/ */
async getPullRequests() { async getPullRequests() {
const span = trace.getTracer("ano-file").startSpan("User.getPullRequests");
span.setAttribute("username", this.username);
const pullRequests = ( const pullRequests = (
await AnonymizedPullRequestModel.find({ await AnonymizedPullRequestModel.find({
owner: this.id, owner: this.id,
@@ -166,6 +179,7 @@ export default class User {
} }
} }
await Promise.all(promises); await Promise.all(promises);
span.end();
return pullRequests; return pullRequests;
} }
src/anonymize-utils.ts
+36 -8
View File
@@ -5,6 +5,7 @@ import { basename } from "path";
import { Transform } from "stream"; import { Transform } from "stream";
import { Readable } from "stream"; import { Readable } from "stream";
import AnonymizedFile from "./AnonymizedFile"; import AnonymizedFile from "./AnonymizedFile";
import { trace } from "@opentelemetry/api";
const urlRegex = const urlRegex =
/<?\b((https?|ftp|file):\/\/)[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]\b\/?>?/g; /<?\b((https?|ftp|file):\/\/)[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]\b\/?>?/g;
@@ -33,26 +34,33 @@ export function isTextFile(filePath: string, content?: Buffer) {
export class AnonymizeTransformer extends Transform { export class AnonymizeTransformer extends Transform {
public wasAnonimized = false; public wasAnonimized = false;
public isText = false; public isText: boolean | null = null;
constructor(private readonly file: AnonymizedFile) { constructor(private readonly file: AnonymizedFile) {
super(); super();
this.isText = isTextFile(this.file.anonymizedPath);
} }
_transform(chunk: Buffer, encoding: string, callback: () => void) { _transform(chunk: Buffer, encoding: string, callback: () => void) {
const isText = isTextFile(this.file.anonymizedPath, chunk); trace
.getTracer("ano-file")
.startActiveSpan("AnonymizeTransformer.transform", async (span) => {
span.setAttribute("path", this.file.anonymizedPath);
if (this.isText === null) {
this.isText = isTextFile(this.file.anonymizedPath, chunk);
}
if (isText) { if (this.isText) {
this.isText = true;
const anonimizer = new ContentAnonimizer(chunk.toString(), { const anonimizer = new ContentAnonimizer(chunk.toString(), {
repoId: this.file.repository.repoId, repoId: this.file.repository.repoId,
image: this.file.repository.options.image, image: this.file.repository.options.image,
link: this.file.repository.options.link, link: this.file.repository.options.link,
terms: this.file.repository.options.terms, terms: this.file.repository.options.terms,
repoName: (this.file.repository.source as GitHubBase).githubRepository repoName: (this.file.repository.source as GitHubBase)
?.fullName, .githubRepository?.fullName,
branchName: branchName:
(this.file.repository.source as GitHubBase).branch?.name || "main", (this.file.repository.source as GitHubBase).branch?.name ||
"main",
}); });
anonimizer.anonymize(); anonimizer.anonymize();
if (anonimizer.wasAnonymized) { if (anonimizer.wasAnonymized) {
@@ -62,13 +70,15 @@ export class AnonymizeTransformer extends Transform {
} }
this.emit("transform", { this.emit("transform", {
isText, isText: this.isText,
wasAnonimized: this.wasAnonimized, wasAnonimized: this.wasAnonimized,
chunk, chunk,
}); });
this.push(chunk); this.push(chunk);
span.end();
callback(); callback();
});
} }
} }
@@ -192,11 +202,22 @@ export class ContentAnonimizer {
} }
anonymize() { anonymize() {
const span = trace
.getTracer("ano-file")
.startSpan("ContentAnonimizer.anonymize");
try {
this.removeImage(); this.removeImage();
span.addEvent("removeImage");
this.removeLink(); this.removeLink();
span.addEvent("removeLink");
this.replaceGitHubSelfLinks(); this.replaceGitHubSelfLinks();
span.addEvent("replaceGitHubSelfLinks");
this.replaceTerms(); this.replaceTerms();
span.addEvent("replaceTerms");
return this.content; return this.content;
} finally {
span.end();
}
} }
} }
@@ -221,6 +242,10 @@ export function anonymizeContent(
} }
export function anonymizePath(path: string, terms: string[]) { export function anonymizePath(path: string, terms: string[]) {
return trace
.getTracer("ano-file")
.startActiveSpan("utils.anonymizePath", (span) => {
span.setAttribute("path", path);
for (let i = 0; i < terms.length; i++) { for (let i = 0; i < terms.length; i++) {
let term = terms[i]; let term = terms[i];
if (term.trim() == "") { if (term.trim() == "") {
@@ -237,5 +262,8 @@ export function anonymizePath(path: string, terms: string[]) {
config.ANONYMIZATION_MASK + "-" + (i + 1) config.ANONYMIZATION_MASK + "-" + (i + 1)
); );
} }
span.setAttribute("return", path);
span.end();
return path; return path;
});
} }
src/database/database.ts
-1
View File
@@ -17,7 +17,6 @@ export async function connect() {
await mongoose.connect(MONGO_URL + "production", { await mongoose.connect(MONGO_URL + "production", {
authSource: "admin", authSource: "admin",
appName: "Anonymous GitHub Server", appName: "Anonymous GitHub Server",
compressors: "zlib",
} as ConnectOptions); } as ConnectOptions);
isConnected = true; isConnected = true;
src/processes/downloadRepository.ts
+8 -1
View File
@@ -4,6 +4,7 @@ config();
import Repository from "../Repository"; import Repository from "../Repository";
import { getRepository as getRepositoryImport } from "../database/database"; import { getRepository as getRepositoryImport } from "../database/database";
import { RepositoryStatus } from "../types"; import { RepositoryStatus } from "../types";
import { Exception, trace } from "@opentelemetry/api";
export default async function (job: SandboxedJob<Repository, void>) { export default async function (job: SandboxedJob<Repository, void>) {
const { const {
@@ -13,6 +14,8 @@ export default async function (job: SandboxedJob<Repository, void>) {
connect: () => Promise<void>; connect: () => Promise<void>;
getRepository: typeof getRepositoryImport; getRepository: typeof getRepositoryImport;
} = require("../database/database"); } = require("../database/database");
const span = trace.getTracer("ano-file").startSpan("proc.downloadRepository");
span.setAttribute("repoId", job.data.repoId);
console.log(`[QUEUE] ${job.data.repoId} is going to be downloaded`); console.log(`[QUEUE] ${job.data.repoId} is going to be downloaded`);
try { try {
await connect(); await connect();
@@ -27,14 +30,18 @@ export default async function (job: SandboxedJob<Repository, void>) {
} catch (error) { } catch (error) {
job.updateProgress({ status: "error" }); job.updateProgress({ status: "error" });
if (error instanceof Error) { if (error instanceof Error) {
span.recordException(error as Exception);
await repo.updateStatus(RepositoryStatus.ERROR, error.message); await repo.updateStatus(RepositoryStatus.ERROR, error.message);
} else if (typeof error === "string") { } else if (typeof error === "string") {
await repo.updateStatus(RepositoryStatus.ERROR, error); await repo.updateStatus(RepositoryStatus.ERROR, error);
span.recordException(error);
} }
throw error; throw error;
} }
} catch (error) { } catch (error) {
console.error(error); span.recordException(error as Exception);
console.log(`[QUEUE] ${job.data.repoId} is finished with an error`); console.log(`[QUEUE] ${job.data.repoId} is finished with an error`);
} finally {
span.end();
} }
} }
src/processes/removeCache.ts
+6 -1
View File
@@ -1,6 +1,7 @@
import { SandboxedJob } from "bullmq"; import { SandboxedJob } from "bullmq";
import Repository from "../Repository"; import Repository from "../Repository";
import { getRepository as getRepositoryImport } from "../database/database"; import { getRepository as getRepositoryImport } from "../database/database";
import { Exception, trace } from "@opentelemetry/api";
export default async function (job: SandboxedJob<Repository, void>) { export default async function (job: SandboxedJob<Repository, void>) {
const { const {
@@ -10,6 +11,8 @@ export default async function (job: SandboxedJob<Repository, void>) {
connect: () => Promise<void>; connect: () => Promise<void>;
getRepository: typeof getRepositoryImport; getRepository: typeof getRepositoryImport;
} = require("../database/database"); } = require("../database/database");
const span = trace.getTracer("ano-file").startSpan("proc.removeCache");
span.setAttribute("repoId", job.data.repoId);
try { try {
await connect(); await connect();
console.log( console.log(
@@ -19,11 +22,13 @@ export default async function (job: SandboxedJob<Repository, void>) {
try { try {
await repo.removeCache(); await repo.removeCache();
} catch (error) { } catch (error) {
span.recordException(error as Exception);
throw error; throw error;
} }
} catch (error) { } catch (error) {
console.error(error); span.recordException(error as Exception);
} finally { } finally {
console.log(`[QUEUE] Cache of ${job.data.repoId} is removed.`); console.log(`[QUEUE] Cache of ${job.data.repoId} is removed.`);
span.end();
} }
} }
src/processes/removeRepository.ts
+7 -1
View File
@@ -2,6 +2,8 @@ import { SandboxedJob } from "bullmq";
import Repository from "../Repository"; import Repository from "../Repository";
import { getRepository as getRepositoryImport } from "../database/database"; import { getRepository as getRepositoryImport } from "../database/database";
import { RepositoryStatus } from "../types"; import { RepositoryStatus } from "../types";
import { trace } from "@opentelemetry/api";
import { Span } from "@opentelemetry/sdk-trace-node";
export default async function (job: SandboxedJob<Repository, void>) { export default async function (job: SandboxedJob<Repository, void>) {
const { const {
@@ -11,6 +13,8 @@ export default async function (job: SandboxedJob<Repository, void>) {
connect: () => Promise<void>; connect: () => Promise<void>;
getRepository: typeof getRepositoryImport; getRepository: typeof getRepositoryImport;
} = require("../database/database"); } = require("../database/database");
const span = trace.getTracer("ano-file").startSpan("proc.removeRepository");
span.setAttribute("repoId", job.data.repoId);
try { try {
await connect(); await connect();
console.log(`[QUEUE] ${job.data.repoId} is going to be removed`); console.log(`[QUEUE] ${job.data.repoId} is going to be removed`);
@@ -24,11 +28,13 @@ export default async function (job: SandboxedJob<Repository, void>) {
} else if (typeof error === "string") { } else if (typeof error === "string") {
await repo.updateStatus(RepositoryStatus.ERROR, error); await repo.updateStatus(RepositoryStatus.ERROR, error);
} }
span.recordException(error as Error);
throw error; throw error;
} }
} catch (error) { } catch (error) {
console.error(error); span.recordException(error as Error);
} finally { } finally {
console.log(`[QUEUE] ${job.data.repoId} is removed`); console.log(`[QUEUE] ${job.data.repoId} is removed`);
span.end();
} }
} }
src/source/GitHubDownload.ts
+16 -2
View File
@@ -10,7 +10,7 @@ import GitHubBase from "./GitHubBase";
import AnonymizedFile from "../AnonymizedFile"; import AnonymizedFile from "../AnonymizedFile";
import { FILE_TYPE, RepositoryStatus, SourceBase } from "../types"; import { FILE_TYPE, RepositoryStatus, SourceBase } from "../types";
import AnonymousError from "../AnonymousError"; import AnonymousError from "../AnonymousError";
import { tryCatch } from "bullmq"; import { trace } from "@opentelemetry/api";
export default class GitHubDownload extends GitHubBase implements SourceBase { export default class GitHubDownload extends GitHubBase implements SourceBase {
constructor( constructor(
@@ -40,6 +40,9 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
} }
async download(token?: string) { async download(token?: string) {
const span = trace.getTracer("ano-file").startSpan("GHDownload.download");
span.setAttribute("repoId", this.repository.repoId);
try {
const fiveMinuteAgo = new Date(); const fiveMinuteAgo = new Date();
fiveMinuteAgo.setMinutes(fiveMinuteAgo.getMinutes() - 5); fiveMinuteAgo.setMinutes(fiveMinuteAgo.getMinutes() - 5);
if ( if (
@@ -57,6 +60,7 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
} }
response = await this._getZipUrl(token); response = await this._getZipUrl(token);
} catch (error) { } catch (error) {
span.recordException(error as Error);
if ((error as any).status == 401 && config.GITHUB_TOKEN) { if ((error as any).status == 401 && config.GITHUB_TOKEN) {
try { try {
response = await this._getZipUrl(config.GITHUB_TOKEN); response = await this._getZipUrl(config.GITHUB_TOKEN);
@@ -111,6 +115,7 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
}); });
await storage.extractZip(originalPath, downloadStream, undefined, this); await storage.extractZip(originalPath, downloadStream, undefined, this);
} catch (error) { } catch (error) {
span.recordException(error as Error);
await this.repository.updateStatus( await this.repository.updateStatus(
RepositoryStatus.ERROR, RepositoryStatus.ERROR,
"unable_to_download" "unable_to_download"
@@ -129,11 +134,17 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
try { try {
await this.repository.updateStatus(RepositoryStatus.READY); await this.repository.updateStatus(RepositoryStatus.READY);
} catch (error) { } catch (error) {
console.error(error); span.recordException(error as Error);
}
} finally {
span.end();
} }
} }
async getFileContent(file: AnonymizedFile): Promise<Readable> { async getFileContent(file: AnonymizedFile): Promise<Readable> {
const span = trace.getTracer("ano-file").startSpan("GHDownload.getFileContent");
span.setAttribute("repoId", this.repository.repoId);
try {
const exists = await storage.exists(file.originalCachePath); const exists = await storage.exists(file.originalCachePath);
if (exists === FILE_TYPE.FILE) { if (exists === FILE_TYPE.FILE) {
return storage.read(file.originalCachePath); return storage.read(file.originalCachePath);
@@ -149,6 +160,9 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
// the cache is not ready, we need to download the repository // the cache is not ready, we need to download the repository
await this.download(); await this.download();
return storage.read(file.originalCachePath); return storage.read(file.originalCachePath);
} finally {
span.end();
}
} }
async getFiles() { async getFiles() {
src/source/GitHubRepository.ts
+39 -1
View File
@@ -5,6 +5,7 @@ import { Octokit, RestEndpointMethodTypes } from "@octokit/rest";
import RepositoryModel from "../database/repositories/repositories.model"; import RepositoryModel from "../database/repositories/repositories.model";
import AnonymousError from "../AnonymousError"; import AnonymousError from "../AnonymousError";
import { isConnected } from "../database/database"; import { isConnected } from "../database/database";
import { trace } from "@opentelemetry/api";
export class GitHubRepository { export class GitHubRepository {
private _data: Partial<{ private _data: Partial<{
@@ -50,6 +51,12 @@ export class GitHubRepository {
accessToken?: string; accessToken?: string;
} }
) { ) {
const span = trace
.getTracer("ano-file")
.startSpan("GHRepository.getCommitInfo");
span.setAttribute("owner", this.owner);
span.setAttribute("repo", this.repo);
try {
const octokit = new Octokit({ auth: opt.accessToken }); const octokit = new Octokit({ auth: opt.accessToken });
const commit = await octokit.repos.getCommit({ const commit = await octokit.repos.getCommit({
owner: this.owner, owner: this.owner,
@@ -57,12 +64,19 @@ export class GitHubRepository {
ref: sha, ref: sha,
}); });
return commit.data; return commit.data;
} finally {
span.end();
}
} }
async branches(opt: { async branches(opt: {
accessToken?: string; accessToken?: string;
force?: boolean; force?: boolean;
}): Promise<Branch[]> { }): Promise<Branch[]> {
const span = trace.getTracer("ano-file").startSpan("GHRepository.branches");
span.setAttribute("owner", this.owner);
span.setAttribute("repo", this.repo);
try {
if ( if (
!this._data.branches || !this._data.branches ||
this._data.branches.length == 0 || this._data.branches.length == 0 ||
@@ -94,6 +108,7 @@ export class GitHubRepository {
); );
} }
} catch (error) { } catch (error) {
span.recordException(error as Error);
throw new AnonymousError("repo_not_found", { throw new AnonymousError("repo_not_found", {
httpStatus: (error as any).status, httpStatus: (error as any).status,
cause: error as Error, cause: error as Error,
@@ -108,6 +123,9 @@ export class GitHubRepository {
} }
return this._data.branches || []; return this._data.branches || [];
} finally {
span.end();
}
} }
async readme(opt: { async readme(opt: {
@@ -115,6 +133,10 @@ export class GitHubRepository {
force?: boolean; force?: boolean;
accessToken?: string; accessToken?: string;
}): Promise<string | undefined> { }): Promise<string | undefined> {
const span = trace.getTracer("ano-file").startSpan("GHRepository.readme");
span.setAttribute("owner", this.owner);
span.setAttribute("repo", this.repo);
try {
if (!opt.branch) opt.branch = this._data.defaultBranch || "master"; if (!opt.branch) opt.branch = this._data.defaultBranch || "master";
const model = await RepositoryModel.findOne({ const model = await RepositoryModel.findOne({
@@ -145,6 +167,7 @@ export class GitHubRepository {
selected.readme = readme; selected.readme = readme;
await model.save(); await model.save();
} catch (error) { } catch (error) {
span.recordException(error as Error);
throw new AnonymousError("readme_not_available", { throw new AnonymousError("readme_not_available", {
httpStatus: 404, httpStatus: 404,
cause: error as Error, cause: error as Error,
@@ -161,6 +184,9 @@ export class GitHubRepository {
} }
return selected.readme; return selected.readme;
} finally {
span.end();
}
} }
public get owner(): string { public get owner(): string {
@@ -203,6 +229,12 @@ export async function getRepositoryFromGitHub(opt: {
repo: string; repo: string;
accessToken: string; accessToken: string;
}) { }) {
const span = trace
.getTracer("ano-file")
.startSpan("GHRepository.getRepositoryFromGitHub");
span.setAttribute("owner", opt.owner);
span.setAttribute("repo", opt.repo);
try {
if (opt.repo.indexOf(".git") > -1) { if (opt.repo.indexOf(".git") > -1) {
opt.repo = opt.repo.replace(".git", ""); opt.repo = opt.repo.replace(".git", "");
} }
@@ -216,6 +248,7 @@ export async function getRepositoryFromGitHub(opt: {
}) })
).data; ).data;
} catch (error) { } catch (error) {
span.recordException(error as Error);
throw new AnonymousError("repo_not_found", { throw new AnonymousError("repo_not_found", {
httpStatus: (error as any).status, httpStatus: (error as any).status,
object: { object: {
@@ -235,7 +268,9 @@ export async function getRepositoryFromGitHub(opt: {
}); });
let model = new RepositoryModel({ externalId: "gh_" + r.id }); let model = new RepositoryModel({ externalId: "gh_" + r.id });
if (isConnected) { if (isConnected) {
const dbModel = await RepositoryModel.findOne({ externalId: "gh_" + r.id }); const dbModel = await RepositoryModel.findOne({
externalId: "gh_" + r.id,
});
if (dbModel) { if (dbModel) {
model = dbModel; model = dbModel;
} }
@@ -256,4 +291,7 @@ export async function getRepositoryFromGitHub(opt: {
await model.save(); await model.save();
} }
return new GitHubRepository(model); return new GitHubRepository(model);
} finally {
span.end();
}
} }
src/source/GitHubStream.ts
+28 -5
View File
@@ -9,6 +9,7 @@ import * as path from "path";
import * as stream from "stream"; import * as stream from "stream";
import AnonymousError from "../AnonymousError"; import AnonymousError from "../AnonymousError";
import config from "../../config"; import config from "../../config";
import { trace } from "@opentelemetry/api";
export default class GitHubStream extends GitHubBase implements SourceBase { export default class GitHubStream extends GitHubBase implements SourceBase {
constructor( constructor(
@@ -26,6 +27,10 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
} }
async getFileContent(file: AnonymizedFile): Promise<stream.Readable> { async getFileContent(file: AnonymizedFile): Promise<stream.Readable> {
return trace
.getTracer("ano-file")
.startActiveSpan("GHStream.getFileContent", async (span) => {
span.setAttribute("path", file.anonymizedPath);
const octokit = new Octokit({ const octokit = new Octokit({
auth: await this.getToken(), auth: await this.getToken(),
}); });
@@ -66,7 +71,10 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
await this.repository.updateStatus(RepositoryStatus.READY); await this.repository.updateStatus(RepositoryStatus.READY);
return stream.Readable.from(content); return stream.Readable.from(content);
} catch (error) { } catch (error) {
if ((error as any).status === 404 || (error as any).httpStatus === 404) { if (
(error as any).status === 404 ||
(error as any).httpStatus === 404
) {
throw new AnonymousError("file_not_found", { throw new AnonymousError("file_not_found", {
httpStatus: (error as any).status || (error as any).httpStatus, httpStatus: (error as any).status || (error as any).httpStatus,
cause: error as Error, cause: error as Error,
@@ -78,15 +86,24 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
cause: error as Error, cause: error as Error,
object: file, object: file,
}); });
} finally {
span.end();
} }
});
} }
async getFiles() { async getFiles() {
const span = trace.getTracer("ano-file").startSpan("GHStream.getFiles");
span.setAttribute("repoId", this.repository.repoId);
try {
let commit = this.branch?.commit; let commit = this.branch?.commit;
if (!commit && this.repository.model.source.commit) { if (!commit && this.repository.model.source.commit) {
commit = this.repository.model.source.commit; commit = this.repository.model.source.commit;
} }
return this.getTree(commit); return this.getTree(commit);
} finally {
span.end();
}
} }
private async getTree( private async getTree(
@@ -98,6 +115,9 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
request: 0, request: 0,
} }
) { ) {
const span = trace.getTracer("ano-file").startSpan("GHStream.getTree");
span.setAttribute("repoId", this.repository.repoId);
span.setAttribute("sha", sha);
this.repository.model.truckedFileList = false; this.repository.model.truckedFileList = false;
let ghRes: Awaited<ReturnType<typeof this.getGHTree>>; let ghRes: Awaited<ReturnType<typeof this.getGHTree>>;
@@ -105,11 +125,13 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
count.request++; count.request++;
ghRes = await this.getGHTree(sha, { recursive: true }); ghRes = await this.getGHTree(sha, { recursive: true });
} catch (error) { } catch (error) {
span.recordException(error as Error);
if ((error as any).status == 409) { if ((error as any).status == 409) {
// empty tree // empty tree
if (this.repository.status != RepositoryStatus.READY) if (this.repository.status != RepositoryStatus.READY)
await this.repository.updateStatus(RepositoryStatus.READY); await this.repository.updateStatus(RepositoryStatus.READY);
// cannot be empty otherwise it would try to download it again // cannot be empty otherwise it would try to download it again
span.end();
return { __: {} }; return { __: {} };
} else { } else {
console.log( console.log(
@@ -121,7 +143,7 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
RepositoryStatus.ERROR, RepositoryStatus.ERROR,
"repo_not_accessible" "repo_not_accessible"
); );
throw new AnonymousError("repo_not_accessible", { const err = new AnonymousError("repo_not_accessible", {
httpStatus: (error as any).status, httpStatus: (error as any).status,
cause: error as Error, cause: error as Error,
object: { object: {
@@ -130,6 +152,9 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
tree_sha: sha, tree_sha: sha,
}, },
}); });
span.recordException(err);
span.end();
throw err;
} }
} }
const tree = this.tree2Tree(ghRes.tree, truncatedTree, parentPath); const tree = this.tree2Tree(ghRes.tree, truncatedTree, parentPath);
@@ -139,6 +164,7 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
} }
if (this.repository.status !== RepositoryStatus.READY) if (this.repository.status !== RepositoryStatus.READY)
await this.repository.updateStatus(RepositoryStatus.READY); await this.repository.updateStatus(RepositoryStatus.READY);
span.end();
return tree; return tree;
} }
@@ -165,9 +191,6 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
}, },
depth = 0 depth = 0
) { ) {
console.log(
`sha ${sha}, countFiles: ${count.file} countRequest: ${count.request}, parentPath: "${parentPath}"`
);
count.request++; count.request++;
let data = null; let data = null;
src/storage/FileSystem.ts
+59 -11
View File
@@ -10,6 +10,7 @@ import * as archiver from "archiver";
import { promisify } from "util"; import { promisify } from "util";
import AnonymizedFile from "../AnonymizedFile"; import AnonymizedFile from "../AnonymizedFile";
import { lookup } from "mime-types"; import { lookup } from "mime-types";
import { trace } from "@opentelemetry/api";
export default class FileSystem implements StorageBase { export default class FileSystem implements StorageBase {
type = "FileSystem"; type = "FileSystem";
@@ -18,6 +19,11 @@ export default class FileSystem implements StorageBase {
/** @override */ /** @override */
async exists(p: string): Promise<FILE_TYPE> { async exists(p: string): Promise<FILE_TYPE> {
return trace
.getTracer("ano-file")
.startActiveSpan("fs.exists", async (span) => {
span.setAttribute("path", p);
span.setAttribute("full-path", join(config.FOLDER, p));
try { try {
const stat = await fs.promises.stat(join(config.FOLDER, p)); const stat = await fs.promises.stat(join(config.FOLDER, p));
if (stat.isDirectory()) return FILE_TYPE.FOLDER; if (stat.isDirectory()) return FILE_TYPE.FOLDER;
@@ -25,12 +31,24 @@ export default class FileSystem implements StorageBase {
} catch (_) { } catch (_) {
// ignore file not found or not downloaded // ignore file not found or not downloaded
} }
span.end();
return FILE_TYPE.NOT_FOUND; return FILE_TYPE.NOT_FOUND;
});
} }
/** @override */ /** @override */
async send(p: string, res: Response) { async send(p: string, res: Response) {
res.sendFile(join(config.FOLDER, p), { dotfiles: "allow" }); return trace
.getTracer("ano-file")
.startActiveSpan("fs.send", async (span) => {
span.setAttribute("path", p);
res.sendFile(join(config.FOLDER, p), { dotfiles: "allow" }, (err) => {
if (err) {
span.recordException(err);
}
span.end();
});
});
} }
/** @override */ /** @override */
@@ -56,22 +74,46 @@ export default class FileSystem implements StorageBase {
file?: AnonymizedFile, file?: AnonymizedFile,
source?: SourceBase source?: SourceBase
): Promise<void> { ): Promise<void> {
return trace
.getTracer("ano-file")
.startActiveSpan("fs.write", async (span) => {
span.setAttribute("path", p);
try {
await this.mk(dirname(p)); await this.mk(dirname(p));
return fs.promises.writeFile(join(config.FOLDER, p), data); return await fs.promises.writeFile(
join(config.FOLDER, p),
data,
"utf-8"
);
} finally {
span.end();
} }
/** @override */
async rm(dir: string): Promise<void> {
await fs.promises.rm(join(config.FOLDER, dir), {
force: true,
recursive: true,
}); });
} }
/** @override */
async rm(dir: string): Promise<void> {
const span = trace.getTracer("ano-file").startSpan("fs.rm");
span.setAttribute("path", dir);
await fs.promises.rm(join(config.FOLDER, dir), {
force: true,
recursive: true,
});
span.end();
}
/** @override */ /** @override */
async mk(dir: string): Promise<void> { async mk(dir: string): Promise<void> {
return trace
.getTracer("ano-file")
.startActiveSpan("fs.mk", async (span) => {
span.setAttribute("path", dir);
if ((await this.exists(dir)) === FILE_TYPE.NOT_FOUND) if ((await this.exists(dir)) === FILE_TYPE.NOT_FOUND)
fs.promises.mkdir(join(config.FOLDER, dir), { recursive: true }); await fs.promises.mkdir(join(config.FOLDER, dir), {
recursive: true,
});
span.end();
});
} }
/** @override */ /** @override */
@@ -82,6 +124,10 @@ export default class FileSystem implements StorageBase {
onEntry?: (file: { path: string; size: number }) => void; onEntry?: (file: { path: string; size: number }) => void;
} = {} } = {}
): Promise<Tree> { ): Promise<Tree> {
return trace
.getTracer("ano-file")
.startActiveSpan("fs.listFiles", async (span) => {
span.setAttribute("path", dir);
if (opt.root == null) { if (opt.root == null) {
opt.root = config.FOLDER; opt.root = config.FOLDER;
} }
@@ -106,10 +152,12 @@ export default class FileSystem implements StorageBase {
output[file] = { size: stats.size, sha: stats.ino.toString() }; output[file] = { size: stats.size, sha: stats.ino.toString() };
} }
} catch (error) { } catch (error) {
console.error(error); span.recordException(error as Error);
} }
} }
span.end();
return output; return output;
});
} }
/** @override */ /** @override */
@@ -144,7 +192,7 @@ export default class FileSystem implements StorageBase {
) { ) {
const archive = archiver(opt?.format || "zip", {}); const archive = archiver(opt?.format || "zip", {});
this.listFiles(dir, { await this.listFiles(dir, {
onEntry: async (file) => { onEntry: async (file) => {
let rs = await this.read(file.path); let rs = await this.read(file.path);
if (opt?.fileTransformer) { if (opt?.fileTransformer) {
src/storage/S3.ts
+58 -2
View File
@@ -15,6 +15,7 @@ import * as archiver from "archiver";
import { dirname, basename } from "path"; import { dirname, basename } from "path";
import AnonymousError from "../AnonymousError"; import AnonymousError from "../AnonymousError";
import AnonymizedFile from "../AnonymizedFile"; import AnonymizedFile from "../AnonymizedFile";
import { trace } from "@opentelemetry/api";
export default class S3Storage implements StorageBase { export default class S3Storage implements StorageBase {
type = "AWS"; type = "AWS";
@@ -45,6 +46,9 @@ export default class S3Storage implements StorageBase {
/** @override */ /** @override */
async exists(path: string): Promise<FILE_TYPE> { async exists(path: string): Promise<FILE_TYPE> {
const span = trace.getTracer("ano-file").startSpan("s3.exists");
span.setAttribute("path", path);
try {
if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set"); if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
try { try {
// if we can get the file info, it is a file // if we can get the file info, it is a file
@@ -61,6 +65,9 @@ export default class S3Storage implements StorageBase {
? FILE_TYPE.FOLDER ? FILE_TYPE.FOLDER
: FILE_TYPE.NOT_FOUND; : FILE_TYPE.NOT_FOUND;
} }
} finally {
span.end();
}
} }
/** @override */ /** @override */
@@ -70,6 +77,9 @@ export default class S3Storage implements StorageBase {
/** @override */ /** @override */
async rm(dir: string): Promise<void> { async rm(dir: string): Promise<void> {
const span = trace.getTracer("ano-file").startSpan("s3.rm");
span.setAttribute("path", dir);
try {
if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set"); if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
const data = await this.client(200000).listObjectsV2({ const data = await this.client(200000).listObjectsV2({
Bucket: config.S3_BUCKET, Bucket: config.S3_BUCKET,
@@ -97,10 +107,16 @@ export default class S3Storage implements StorageBase {
if (data.IsTruncated) { if (data.IsTruncated) {
await this.rm(dir); await this.rm(dir);
} }
} finally {
span.end();
}
} }
/** @override */ /** @override */
async send(p: string, res: Response) { async send(p: string, res: Response) {
const span = trace.getTracer("ano-file").startSpan("s3.send");
span.setAttribute("path", p);
try {
if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set"); if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
try { try {
const command = new GetObjectCommand({ const command = new GetObjectCommand({
@@ -121,15 +137,22 @@ export default class S3Storage implements StorageBase {
res.end(); res.end();
} }
} catch (error) { } catch (error) {
span.recordException(error as Error);
try { try {
res.status(500); res.status(500);
} catch (err) { } catch (err) {
console.error(`[ERROR] S3 send ${p}`, err); console.error(`[ERROR] S3 send ${p}`, err);
} }
} }
} finally {
span.end();
}
} }
async fileInfo(path: string) { async fileInfo(path: string) {
const span = trace.getTracer("ano-file").startSpan("s3.fileInfo");
span.setAttribute("path", path);
try {
if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set"); if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
const info = await this.client(3000).headObject({ const info = await this.client(3000).headObject({
Bucket: config.S3_BUCKET, Bucket: config.S3_BUCKET,
@@ -142,10 +165,16 @@ export default class S3Storage implements StorageBase {
? info.ContentType ? info.ContentType
: (lookup(path) as string), : (lookup(path) as string),
}; };
} finally {
span.end();
}
} }
/** @override */ /** @override */
async read(path: string): Promise<Readable> { async read(path: string): Promise<Readable> {
const span = trace.getTracer("ano-file").startSpan("s3.rreadm");
span.setAttribute("path", path);
try {
if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set"); if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
const command = new GetObjectCommand({ const command = new GetObjectCommand({
Bucket: config.S3_BUCKET, Bucket: config.S3_BUCKET,
@@ -159,6 +188,9 @@ export default class S3Storage implements StorageBase {
}); });
} }
return res as Readable; return res as Readable;
} finally {
span.end();
}
} }
/** @override */ /** @override */
@@ -168,6 +200,9 @@ export default class S3Storage implements StorageBase {
file?: AnonymizedFile, file?: AnonymizedFile,
source?: SourceBase source?: SourceBase
): Promise<void> { ): Promise<void> {
const span = trace.getTracer("ano-file").startSpan("s3.rm");
span.setAttribute("path", path);
try {
if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set"); if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
const params: PutObjectCommandInput = { const params: PutObjectCommandInput = {
Bucket: config.S3_BUCKET, Bucket: config.S3_BUCKET,
@@ -181,10 +216,16 @@ export default class S3Storage implements StorageBase {
// 30s timeout // 30s timeout
await this.client(30000).putObject(params); await this.client(30000).putObject(params);
return; return;
} finally {
span.end();
}
} }
/** @override */ /** @override */
async listFiles(dir: string): Promise<Tree> { async listFiles(dir: string): Promise<Tree> {
const span = trace.getTracer("ano-file").startSpan("s3.listFiles");
span.setAttribute("path", dir);
try {
if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set"); if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
if (dir && dir[dir.length - 1] != "/") dir = dir + "/"; if (dir && dir[dir.length - 1] != "/") dir = dir + "/";
const out: Tree = {}; const out: Tree = {};
@@ -222,6 +263,9 @@ export default class S3Storage implements StorageBase {
} }
} while (req && req.Contents && req.IsTruncated); } while (req && req.Contents && req.IsTruncated);
return out; return out;
} finally {
span.end();
}
} }
/** @override */ /** @override */
@@ -232,7 +276,8 @@ export default class S3Storage implements StorageBase {
source?: SourceBase source?: SourceBase
): Promise<void> { ): Promise<void> {
let toS3: ArchiveStreamToS3; let toS3: ArchiveStreamToS3;
const span = trace.getTracer("ano-file").startSpan("s3.extractZip");
span.setAttribute("path", p);
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
if (!config.S3_BUCKET) return reject("S3_BUCKET not set"); if (!config.S3_BUCKET) return reject("S3_BUCKET not set");
toS3 = new ArchiveStreamToS3({ toS3 = new ArchiveStreamToS3({
@@ -253,11 +298,16 @@ export default class S3Storage implements StorageBase {
}); });
pipeline(data, toS3, (err) => { pipeline(data, toS3, (err) => {
if (err) { if (err) {
span.recordException(err as Error);
return reject(err); return reject(err);
} }
span.end();
resolve();
})
.on("finish", () => {
span.end();
resolve(); resolve();
}) })
.on("finish", resolve)
.on("error", reject); .on("error", reject);
}); });
} }
@@ -270,6 +320,9 @@ export default class S3Storage implements StorageBase {
fileTransformer?: (p: string) => Transform; fileTransformer?: (p: string) => Transform;
} }
) { ) {
const span = trace.getTracer("ano-file").startSpan("s3.archive");
span.setAttribute("path", dir);
try {
if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set"); if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
const archive = archiver(opt?.format || "zip", {}); const archive = archiver(opt?.format || "zip", {});
if (dir && dir[dir.length - 1] != "/") dir = dir + "/"; if (dir && dir[dir.length - 1] != "/") dir = dir + "/";
@@ -304,5 +357,8 @@ export default class S3Storage implements StorageBase {
} while (req && req.Contents?.length && req.IsTruncated); } while (req && req.Contents?.length && req.IsTruncated);
archive.finalize(); archive.finalize();
return archive; return archive;
} finally {
span.end();
}
} }
} }
tsconfig.json
+2 -1
View File
@@ -14,7 +14,8 @@
"sourceMap": false, "sourceMap": false,
"skipLibCheck": true, "skipLibCheck": true,
"strict": true, "strict": true,
"esModuleInterop": false "esModuleInterop": false,
"incremental": true
}, },
"include": ["src/**/*.ts", "index.ts", "cli.ts"], "include": ["src/**/*.ts", "index.ts", "cli.ts"],
"exclude": ["node_modules", ".vscode"] "exclude": ["node_modules", ".vscode"]