Remove OpenTelemetry tracing infrastructure (#662)

Dockerfile

@@ -16,6 +16,5 @@ COPY public ./public
 COPY src ./src
 
 RUN npm install && npm run build && npm cache clean --force
-COPY opentelemetry.js .
 
-CMD [ "node", "--require", "./opentelemetry.js", "./build/server/index.js"]
+CMD [ "node", "./build/server/index.js"]

docker-compose.yml

@@ -37,7 +37,7 @@ services:
       mode: replicated
       replicas: 4
       endpoint_mode: dnsrr
-    entrypoint: ["node", "--require", "./opentelemetry.js", "./build/streamer/index.js"]
+    entrypoint: ["node", "./build/streamer/index.js"]
     env_file:
       - ./.env
     volumes:
@@ -89,30 +89,6 @@ services:
       timeout: 10s
       retries: 5
 
-  opentelemetry:
-    image: otel/opentelemetry-collector
-    restart: always
-    command: ["--config=/etc/otel-collector-config.yaml"]
-    volumes:
-      - ./opentelemetry-collector.yml:/etc/otel-collector-config.yaml
-    depends_on:
-      - jaeger
-      - prometheus
-
-  jaeger:
-    image: jaegertracing/all-in-one:latest
-    restart: always
-    ports:
-      - 127.0.0.1:16686:16686
-
-  prometheus:
-    image: prom/prometheus:latest
-    restart: always
-    volumes:
-      - ./prometheus.yaml:/etc/prometheus/prometheus.yml
-    ports:
-      - 127.0.0.1:9090:9090
-
   mongodb-backup:
     image: tiredofit/db-backup
     links:

opentelemetry-collector.yml

@@ -1,40 +0,0 @@
-receivers:
-  otlp:
-    protocols:
-      grpc:
-
-exporters:
-  prometheus:
-    endpoint: "0.0.0.0:8889"
-    const_labels:
-      label1: value1
-
-  debug:
-
-  otlp:
-    endpoint: jaeger:4317
-    tls:
-      insecure: true
-
-processors:
-  batch:
-
-extensions:
-  health_check:
-  pprof:
-    endpoint: :1888
-  zpages:
-    endpoint: :55679
-
-service:
-  extensions: [health_check, pprof, zpages]
-  pipelines:
-    traces:
-      receivers: [otlp]
-      exporters: [debug, otlp]
-    metrics:
-      receivers: [otlp]
-      exporters: [debug, prometheus]
-    logs:
-      receivers: [otlp]
-      exporters: [debug]

opentelemetry.js

@@ -1,29 +0,0 @@
-const opentelemetry = require("@opentelemetry/sdk-node");
-const {
-  getNodeAutoInstrumentations,
-} = require("@opentelemetry/auto-instrumentations-node");
-const {
-  OTLPTraceExporter,
-} = require("@opentelemetry/exporter-trace-otlp-grpc");
-const {
-  OTLPMetricExporter,
-} = require("@opentelemetry/exporter-metrics-otlp-grpc");
-const { PeriodicExportingMetricReader } = require("@opentelemetry/sdk-metrics");
-const { diag, DiagConsoleLogger, DiagLogLevel } = require("@opentelemetry/api");
-
-// diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO);
-
-const sdk = new opentelemetry.NodeSDK({
-  serviceName: process.env.SERVICE_NAME || "Anonymous-GitHub",
-  logRecordProcessor: getNodeAutoInstrumentations().logRecordProcessor,
-  traceExporter: new OTLPTraceExporter({
-    url: "http://opentelemetry:4317/v1/traces",
-  }),
-  metricReader: new PeriodicExportingMetricReader({
-    exporter: new OTLPMetricExporter({
-      url: "http://opentelemetry:4317/v1/metrics",
-    }),
-  }),
-  instrumentations: [getNodeAutoInstrumentations()],
-});
-sdk.start();

package.json

@@ -35,15 +35,6 @@
     "@aws-sdk/lib-storage": "^3.540.0",
     "@mongodb-js/zstd": "^1.2.0",
     "@octokit/rest": "^20.0.2",
-    "@opentelemetry/api": "^1.8.0",
-    "@opentelemetry/auto-instrumentations-node": "^0.43.0",
-    "@opentelemetry/exporter-metrics-otlp-grpc": "^0.49.1",
-    "@opentelemetry/exporter-metrics-otlp-proto": "^0.49.1",
-    "@opentelemetry/exporter-trace-otlp-grpc": "^0.49.1",
-    "@opentelemetry/exporter-trace-otlp-proto": "^0.49.1",
-    "@opentelemetry/sdk-metrics": "^1.22.0",
-    "@opentelemetry/sdk-node": "^0.49.1",
-    "@opentelemetry/sdk-trace-node": "^1.22.0",
     "@smithy/node-http-handler": "^2.5.0",
     "archiver": "^5.3.2",
     "bullmq": "^2.4.0",

prometheus.yaml

@@ -1,6 +0,0 @@
-scrape_configs:
-  - job_name: 'otel-collector'
-    scrape_interval: 10s
-    static_configs:
-      - targets: ['opentelemetry:8889']
-      - targets: ['opentelemetry:8888']

src/core/AnonymizedFile.ts

@@ -1,7 +1,6 @@
 import { join, basename, dirname } from "path";
 import { Response } from "express";
 import { Readable } from "stream";
-import { trace } from "@opentelemetry/api";
 import { lookup } from "mime-types";
 import got from "got";
 
@@ -35,26 +34,12 @@ export default class AnonymizedFile {
   }
 
   async sha() {
-    return trace
-      .getTracer("ano-file")
-      .startActiveSpan("AnnoFile.sha", async (span) => {
-        try {
-          span.setAttribute("anonymizedPath", this.anonymizedPath);
     if (this._file) return this._file.sha?.replace(/"/g, "");
     this._file = await this.getFileInfo();
     return this._file.sha?.replace(/"/g, "");
-        } finally {
-          span.end();
-        }
-      });
   }
 
   async getFileInfo(): Promise<IFile> {
-    const span = trace.getTracer("ano-file").startSpan("AnnoFile.getFileInfo");
-    span.setAttribute("repoId", this.repository.repoId);
-    span.setAttribute("file", this.anonymizedPath);
-
-    try {
     if (this._file) return this._file;
     let fileDir = dirname(this.anonymizedPath);
     if (fileDir == ".") fileDir = "";
@@ -119,12 +104,6 @@ export default class AnonymizedFile {
       object: this,
       httpStatus: 404,
     });
-    } catch (error) {
-      span.recordException(error as Error);
-      throw error;
-    } finally {
-      span.end();
-    }
   }
 
   /**
@@ -133,11 +112,6 @@ export default class AnonymizedFile {
    * @returns the origin relative path of the file
    */
   async originalPath(): Promise<string> {
-    const span = trace.getTracer("ano-file").startSpan("AnnoFile.originalPath");
-    span.setAttribute("repoId", this.repository.repoId);
-    span.setAttribute("file", this.anonymizedPath);
-    try {
-      span.setAttribute("anonymizedPath", this.anonymizedPath);
     if (this.anonymizedPath == null) {
       throw new AnonymousError("path_not_specified", {
         object: this,
@@ -148,9 +122,6 @@ export default class AnonymizedFile {
       this._file = await this.getFileInfo();
     }
     return join(this._file.path, this._file.name);
-    } finally {
-      span.end();
-    }
   }
   extension() {
     const filename = basename(this._file?.name || this.anonymizedPath);
@@ -188,14 +159,9 @@ export default class AnonymizedFile {
   }
 
   async content(): Promise<Readable> {
-    return trace
-      .getTracer("ano-file")
-      .startActiveSpan("content", async (span) => {
-        try {
     if (this.anonymizedPath.includes(config.ANONYMIZATION_MASK)) {
       await this.originalPath();
     }
-          span.addEvent("filePath", { originalPath: this.filePath });
     if (this._file?.size && this._file?.size > config.MAX_FILE_SIZE) {
       throw new AnonymousError("file_too_big", {
         object: this,
@@ -211,36 +177,21 @@ export default class AnonymizedFile {
       await this.repository.updateStatus(RepositoryStatus.READY);
     }
     return content;
-        } finally {
-          span.end();
-        }
-      });
   }
 
   async anonymizedContent() {
-    const span = trace.getTracer("ano-file").startSpan("Repository.conference");
-    span.setAttribute("anonymizedPath", this.anonymizedPath);
     const anonymizer = this.repository.generateAnonymizeTransformer(
       this.anonymizedPath
     );
     if (!config.STREAMER_ENTRYPOINT) {
       // collect the content locally
       const content = await this.content();
-      return content.pipe(anonymizer).on("close", () => {
+      return content.pipe(anonymizer);
-        span.end();
-      });
     }
 
-    // const cacheableLookup = new CacheableLookup();
-    // const hostName = new URL(config.STREAMER_ENTRYPOINT).hostname;
-    // const ipHost = await cacheableLookup.lookupAsync(hostName);
-
     // use the streamer service
     return got.stream(join(config.STREAMER_ENTRYPOINT, "api"), {
       method: "POST",
-      // lookup: cacheableLookup.lookup,
-      // host: ipHost.address,
-      // dnsCache: cacheableLookup,
       json: {
         token: await this.repository.getToken(),
         repoFullName: this.repository.model.source.repositoryName,
@@ -268,19 +219,10 @@ export default class AnonymizedFile {
     return join(this._file.path, this._file.name);
   }
 
-  // cacheableLookup = new CacheableLookup({
-  //   maxTtl: 60,
-  // });
-
   async send(res: Response): Promise<void> {
     const anonymizer = this.repository.generateAnonymizeTransformer(
       this.anonymizedPath
     );
-    return trace
-      .getTracer("ano-file")
-      .startActiveSpan("AnonymizedFile.send", async (span) => {
-        span.setAttribute("repoId", this.repository.repoId);
-        span.setAttribute("anonymizedPath", this.anonymizedPath);
     // eslint-disable-next-line no-async-promise-executor
     return new Promise<void>(async (resolve, reject) => {
       try {
@@ -305,7 +247,6 @@ export default class AnonymizedFile {
               },
             })
             .on("error", (err) => {
-                  span.recordException(err);
               handleError(
                 new AnonymousError("file_not_found", {
                   object: this,
@@ -316,13 +257,10 @@ export default class AnonymizedFile {
             });
           resStream.pipe(res);
           res.on("close", () => {
-                span.end();
             resolve();
           });
           res.on("error", (err) => {
             reject(err);
-                span.recordException(err);
-                span.end();
           });
           return;
         }
@@ -348,10 +286,7 @@ export default class AnonymizedFile {
           if (!content.closed && !content.destroyed) {
             content.destroy();
           }
-              span.recordException(error);
-              span.end();
           reject(error);
-              // handleError(error, res);
         }
         content
           .on("error", handleStreamError)
@@ -362,13 +297,11 @@ export default class AnonymizedFile {
             if (!content.closed && !content.destroyed) {
               content.destroy();
             }
-                span.end();
             resolve();
           });
       } catch (error) {
         handleError(error, res);
       }
     });
-      });
   }
 }

src/core/GitHubUtils.ts

@@ -1,4 +1,3 @@
-import { trace } from "@opentelemetry/api";
 import { Octokit } from "@octokit/rest";
 
 import Repository from "./Repository";
@@ -26,10 +25,7 @@ export async function checkToken(token: string) {
 }
 
 export async function getToken(repository: Repository) {
-  const span = trace.getTracer("ano-file").startSpan("GHUtils.getToken");
-  span.setAttribute("repoId", repository.repoId);
   console.log("getToken", repository.repoId);
-  try {
   // if (repository.model.source.accessToken) {
   //   // only check the token if the repo has been visited less than 10 minutes ago
   //   if (
@@ -99,7 +95,4 @@ export async function getToken(repository: Repository) {
     }
   }
   return config.GITHUB_TOKEN;
-  } finally {
-    span.end();
-  }
 }

src/core/Repository.ts

@@ -20,7 +20,6 @@ import {
   getRepositoryFromGitHub,
   GitHubRepository,
 } from "./source/GitHubRepository";
-import { trace } from "@opentelemetry/api";
 import { getToken } from "./GitHubUtils";
 import { FILE_TYPE } from "./storage/Storage";
 import config from "../config";
@@ -152,9 +151,6 @@ export default class Repository {
       force: false,
     }
   ): Promise<IFile[]> {
-    const span = trace.getTracer("ano-file").startSpan("Repository.files");
-    span.setAttribute("repoId", this.repoId);
-    try {
     const hasFile = await FileModel.exists({ repoId: this.repoId }).exec();
     if (!hasFile || opt.force) {
       await FileModel.deleteMany({ repoId: this.repoId }).exec();
@@ -187,9 +183,6 @@ export default class Repository {
       query.path = pathQuery;
     }
     return await FileModel.find(query).exec();
-    } finally {
-      span.end();
-    }
   }
 
   /**
@@ -276,11 +269,6 @@ export default class Repository {
    * @returns void
    */
   async updateIfNeeded(opt?: { force: boolean }): Promise<void> {
-    const span = trace
-      .getTracer("ano-file")
-      .startSpan("Repository.updateIfNeeded");
-    span.setAttribute("repoId", this.repoId);
-
     if (
       this._model.options.expirationMode !== "never" &&
       this.status != RepositoryStatus.EXPIRED &&
@@ -344,8 +332,6 @@ export default class Repository {
           this.status == RepositoryStatus.READY
         ) {
           console.log(`[UPDATE] ${this._model.repoId} is up to date`);
-          span.setAttribute("status", "up_to_date");
-          span.end();
           return;
         }
         this._model.source.commit = newCommit;
@@ -368,8 +354,6 @@ export default class Repository {
           );
           await this.updateStatus(RepositoryStatus.ERROR, "branch_not_found");
           await this.resetSate();
-          span.setAttribute("status", "branch_not_found");
-          span.end();
           throw new AnonymousError("branch_not_found", {
             object: this,
           });
@@ -386,7 +370,6 @@ export default class Repository {
         });
       }
     }
-    span.end();
   }
   /**
    * Download the require state for the repository to work
@@ -394,10 +377,7 @@ export default class Repository {
    * @returns void
    */
   async anonymize(progress?: (status: string) => void) {
-    const span = trace.getTracer("ano-file").startSpan("Repository.anonymize");
-    span.setAttribute("repoId", this.repoId);
     if (this.status === RepositoryStatus.READY) {
-      span.end();
       return;
     }
     this.model.increment();
@@ -418,23 +398,16 @@ export default class Repository {
     }
     await this.updateStatus(RepositoryStatus.READY);
     await this.computeSize();
-    span.end();
   }
 
   /**
    * Update the last view and view count
    */
   async countView() {
-    const span = trace.getTracer("ano-file").startSpan("Repository.countView");
-    span.setAttribute("repoId", this.repoId);
-    try {
     this._model.lastView = new Date();
     this._model.pageView = (this._model.pageView || 0) + 1;
     if (!isConnected) return this.model;
     await this._model.save();
-    } finally {
-      span.end();
-    }
   }
 
   /**
@@ -443,54 +416,36 @@ export default class Repository {
    * @param errorMessage a potential error message to display
    */
   async updateStatus(status: RepositoryStatus, statusMessage?: string) {
-    const span = trace
-      .getTracer("ano-file")
-      .startSpan("Repository.updateStatus");
-    span.setAttribute("repoId", this.repoId);
-    span.setAttribute("status", status);
-    span.setAttribute("statusMessage", statusMessage || "");
-    try {
     if (!status) return this.model;
     this._model.status = status;
     this._model.statusDate = new Date();
     this._model.statusMessage = statusMessage;
     if (!isConnected) return this.model;
     await this._model.save();
-    } finally {
-      span.end();
-    }
   }
 
   /**
    * Expire the repository
    */
   async expire() {
-    const span = trace.getTracer("ano-file").startSpan("Repository.expire");
-    span.setAttribute("repoId", this.repoId);
     await this.updateStatus(RepositoryStatus.EXPIRING);
     await this.resetSate();
     await this.updateStatus(RepositoryStatus.EXPIRED);
-    span.end();
   }
 
   /**
    * Remove the repository
    */
   async remove() {
-    const span = trace.getTracer("ano-file").startSpan("Repository.remove");
-    span.setAttribute("repoId", this.repoId);
     await this.updateStatus(RepositoryStatus.REMOVING);
     await this.resetSate();
     await this.updateStatus(RepositoryStatus.REMOVED);
-    span.end();
   }
 
   /**
    * Reset/delete the state of the repository
    */
   async resetSate(status?: RepositoryStatus, statusMessage?: string) {
-    const span = trace.getTracer("ano-file").startSpan("Repository.resetState");
-    span.setAttribute("repoId", this.repoId);
     // remove attribute
     this._model.size = { storage: 0, file: 0 };
     if (status) {
@@ -502,7 +457,6 @@ export default class Repository {
       this.removeCache(),
     ]);
     console.log(`[RESET] ${this._model.repoId} has been reset`);
-    span.end();
   }
 
   /**
@@ -510,11 +464,6 @@ export default class Repository {
    * @returns
    */
   async removeCache() {
-    const span = trace
-      .getTracer("ano-file")
-      .startSpan("Repository.removeCache");
-    span.setAttribute("repoId", this.repoId);
-    try {
     await storage.rm(this.repoId);
     this.model.isReseted = true;
     if (isConnected) {
@@ -524,9 +473,6 @@ export default class Repository {
         console.error("[ERROR] removeCache save", error);
       }
     }
-    } finally {
-      span.end();
-    }
   }
 
   /**
@@ -544,11 +490,6 @@ export default class Repository {
      */
     file: number;
   }> {
-    const span = trace
-      .getTracer("ano-file")
-      .startSpan("Repository.computeSize");
-    span.setAttribute("repoId", this.repoId);
-    try {
     if (this.status !== RepositoryStatus.READY)
       return { storage: 0, file: 0 };
     if (this._model.size.file) return this._model.size;
@@ -574,9 +515,6 @@ export default class Repository {
       await this._model.save();
     }
     return this._model.size;
-    } finally {
-      span.end();
-    }
   }
 
   /**
@@ -585,9 +523,6 @@ export default class Repository {
    * @returns conference of the repository
    */
   async conference(): Promise<Conference | null> {
-    const span = trace.getTracer("ano-file").startSpan("Repository.conference");
-    span.setAttribute("repoId", this.repoId);
-    try {
     if (!this._model.conference) {
       return null;
     }
@@ -596,9 +531,6 @@ export default class Repository {
     });
     if (conference) return new Conference(conference);
     return null;
-    } finally {
-      span.end();
-    }
   }
 
   /***** Getters ********/

src/core/User.ts

@@ -1,5 +1,3 @@
-import { trace } from "@opentelemetry/api";
-
 import AnonymizedRepositoryModel from "./model/anonymizedRepositories/anonymizedRepositories.model";
 import RepositoryModel from "./model/repositories/repositories.model";
 import { IUserDocument } from "./model/users/users.types";
@@ -57,10 +55,6 @@ export default class User {
      */
     force: boolean;
   }): Promise<GitHubRepository[]> {
-    const span = trace
-      .getTracer("ano-file")
-      .startSpan("User.getGitHubRepositories");
-    span.setAttribute("username", this.username);
     if (
       !this._model.repositories ||
       this._model.repositories.length == 0 ||
@@ -111,13 +105,11 @@ export default class User {
 
       // have the model
       await this._model.save();
-      span.end();
       return repositories.map((r) => new GitHubRepository(r));
     } else {
       const out = (
         await RepositoryModel.find({ _id: { $in: this._model.repositories } })
       ).map((i) => new GitHubRepository(i));
-      span.end();
       return out;
     }
   }
@@ -127,8 +119,6 @@ export default class User {
    * @returns the list of anonymized repositories
    */
   async getRepositories() {
-    const span = trace.getTracer("ano-file").startSpan("User.getRepositories");
-    span.setAttribute("username", this.username);
     const repositories = (
       await AnonymizedRepositoryModel.find({
         owner: this.id,
@@ -147,7 +137,6 @@ export default class User {
       }
     }
     await Promise.all(promises);
-    span.end();
     return repositories;
   }
   /**
@@ -155,8 +144,6 @@ export default class User {
    * @returns the list of anonymized repositories
    */
   async getPullRequests() {
-    const span = trace.getTracer("ano-file").startSpan("User.getPullRequests");
-    span.setAttribute("username", this.username);
     const pullRequests = (
       await AnonymizedPullRequestModel.find({
         owner: this.id,
@@ -175,7 +162,6 @@ export default class User {
       }
     }
     await Promise.all(promises);
-    span.end();
     return pullRequests;
   }
 

src/core/anonymize-utils.ts

@@ -1,7 +1,6 @@
 import { basename } from "path";
 import { Transform, Readable } from "stream";
 import { isText } from "istextorbinary";
-import { trace } from "@opentelemetry/api";
 
 import config from "../config";
 
@@ -49,10 +48,6 @@ export class AnonymizeTransformer extends Transform {
   }
 
   _transform(chunk: Buffer, encoding: string, callback: () => void) {
-    trace
-      .getTracer("ano-file")
-      .startActiveSpan("AnonymizeTransformer.transform", async (span) => {
-        span.setAttribute("path", this.opt.filePath);
     if (this.isText === null) {
       this.isText = isTextFile(this.opt.filePath, chunk);
     }
@@ -70,9 +65,7 @@ export class AnonymizeTransformer extends Transform {
     });
 
     this.push(chunk);
-        span.end();
     callback();
-      });
   }
 }
 
@@ -179,30 +172,15 @@ export class ContentAnonimizer {
   }
 
   anonymize(content: string) {
-    const span = trace
-      .getTracer("ano-file")
-      .startSpan("ContentAnonimizer.anonymize");
-    try {
     content = this.removeImage(content);
-      span.addEvent("removeImage");
     content = this.removeLink(content);
-      span.addEvent("removeLink");
     content = this.replaceGitHubSelfLinks(content);
-      span.addEvent("replaceGitHubSelfLinks");
     content = this.replaceTerms(content);
-      span.addEvent("replaceTerms");
     return content;
-    } finally {
-      span.end();
-    }
   }
 }
 
 export function anonymizePath(path: string, terms: string[]) {
-  return trace
-    .getTracer("ano-file")
-    .startActiveSpan("utils.anonymizePath", (span) => {
-      span.setAttribute("path", path);
   for (let i = 0; i < terms.length; i++) {
     let term = terms[i];
     if (term.trim() == "") {
@@ -219,8 +197,5 @@ export function anonymizePath(path: string, terms: string[]) {
       config.ANONYMIZATION_MASK + "-" + (i + 1)
     );
   }
-      span.setAttribute("return", path);
-      span.end();
   return path;
-    });
 }

src/core/source/GitHubDownload.ts

@@ -4,7 +4,6 @@ import { OctokitResponse } from "@octokit/types";
 
 import storage from "../storage";
 import GitHubBase, { GitHubBaseData } from "./GitHubBase";
-import { trace } from "@opentelemetry/api";
 import { FILE_TYPE } from "../storage/Storage";
 import { octokit } from "../GitHubUtils";
 import AnonymousError from "../AnonymousError";
@@ -27,14 +26,10 @@ export default class GitHubDownload extends GitHubBase {
   }
 
   async download(progress?: (status: string) => void) {
-    const span = trace.getTracer("ano-file").startSpan("GHDownload.download");
-    span.setAttribute("repoId", this.data.repoId);
-    try {
     let response: OctokitResponse<unknown, number>;
     try {
       response = await this.getZipUrl();
     } catch (error) {
-        span.recordException(error as Error);
       throw new AnonymousError("repo_not_found", {
         httpStatus: (error as any).status || 404,
         object: this.data,
@@ -59,27 +54,18 @@ export default class GitHubDownload extends GitHubBase {
         this.type
       );
     } catch (error) {
-        span.recordException(error as Error);
       throw new AnonymousError("unable_to_download", {
         httpStatus: 500,
         cause: error as Error,
         object: this.data,
       });
     }
-    } finally {
-      span.end();
-    }
   }
 
   async getFileContent(
     file: AnonymizedFile,
     progress?: (status: string) => void
   ): Promise<Readable> {
-    const span = trace
-      .getTracer("ano-file")
-      .startSpan("GHDownload.getFileContent");
-    span.setAttribute("repoId", this.data.repoId);
-    try {
     const exists = await storage.exists(this.data.repoId, file.filePath);
     if (exists === FILE_TYPE.FILE) {
       return storage.read(this.data.repoId, file.filePath);
@@ -95,9 +81,6 @@ export default class GitHubDownload extends GitHubBase {
     // the cache is not ready, we need to download the repository
     await this.download(progress);
     return storage.read(this.data.repoId, file.filePath);
-    } finally {
-      span.end();
-    }
   }
 
   async getFiles(progress?: (status: string) => void) {

src/core/source/GitHubRepository.ts

@@ -1,7 +1,6 @@
 import { Branch } from "../types";
 import * as gh from "parse-github-url";
 import { RestEndpointMethodTypes } from "@octokit/rest";
-import { trace } from "@opentelemetry/api";
 
 import AnonymousError from "../AnonymousError";
 import { isConnected } from "../../server/database";
@@ -55,12 +54,6 @@ export class GitHubRepository {
       accessToken: string;
     }
   ) {
-    const span = trace
-      .getTracer("ano-file")
-      .startSpan("GHRepository.getCommitInfo");
-    span.setAttribute("owner", this.owner);
-    span.setAttribute("repo", this.repo);
-    try {
     const oct = octokit(opt.accessToken);
     const commit = await oct.repos.getCommit({
       owner: this.owner,
@@ -68,19 +61,12 @@ export class GitHubRepository {
       ref: sha,
     });
     return commit.data;
-    } finally {
-      span.end();
-    }
   }
 
   async branches(opt: {
     accessToken: string;
     force?: boolean;
   }): Promise<Branch[]> {
-    const span = trace.getTracer("ano-file").startSpan("GHRepository.branches");
-    span.setAttribute("owner", this.owner);
-    span.setAttribute("repo", this.repo);
-    try {
     if (
       !this._data.branches ||
       this._data.branches.length == 0 ||
@@ -112,7 +98,6 @@ export class GitHubRepository {
           );
         }
       } catch (error) {
-          span.recordException(error as Error);
         throw new AnonymousError("repo_not_found", {
           httpStatus: (error as any).status,
           cause: error as Error,
@@ -127,9 +112,6 @@ export class GitHubRepository {
     }
 
     return this._data.branches || [];
-    } finally {
-      span.end();
-    }
   }
 
   async readme(opt: {
@@ -137,10 +119,6 @@ export class GitHubRepository {
     force?: boolean;
     accessToken: string;
   }): Promise<string | undefined> {
-    const span = trace.getTracer("ano-file").startSpan("GHRepository.readme");
-    span.setAttribute("owner", this.owner);
-    span.setAttribute("repo", this.repo);
-    try {
     if (!opt.branch) opt.branch = this._data.defaultBranch || "master";
 
     const model = await RepositoryModel.findOne({
@@ -171,7 +149,6 @@ export class GitHubRepository {
         selected.readme = readme;
         await model.save();
       } catch (error) {
-          span.recordException(error as Error);
         throw new AnonymousError("readme_not_available", {
           httpStatus: 404,
           cause: error as Error,
@@ -188,9 +165,6 @@ export class GitHubRepository {
     }
 
     return selected.readme;
-    } finally {
-      span.end();
-    }
   }
 
   public get owner(): string {
@@ -235,12 +209,6 @@ export async function getRepositoryFromGitHub(opt: {
   accessToken: string;
   force?: boolean;
 }) {
-  const span = trace
-    .getTracer("ano-file")
-    .startSpan("GHRepository.getRepositoryFromGitHub");
-  span.setAttribute("owner", opt.owner);
-  span.setAttribute("repo", opt.repo);
-  try {
   if (opt.repo.indexOf(".git") > -1) {
     opt.repo = opt.repo.replace(".git", "");
   }
@@ -271,7 +239,6 @@ export async function getRepositoryFromGitHub(opt: {
       })
     ).data;
   } catch (error) {
-      span.recordException(error as Error);
     if (
       error instanceof Error &&
       error.message.includes(
@@ -321,7 +288,4 @@ export async function getRepositoryFromGitHub(opt: {
     await model.save();
   }
   return new GitHubRepository(model);
-  } finally {
-    span.end();
-  }
 }

src/core/source/GitHubStream.ts

@@ -7,7 +7,6 @@ import { basename, dirname } from "path";
 
 import * as stream from "stream";
 import AnonymousError from "../AnonymousError";
-import { trace } from "@opentelemetry/api";
 import { FILE_TYPE } from "../storage/Storage";
 import { octokit } from "../GitHubUtils";
 import FileModel from "../model/files/files.model";
@@ -21,8 +20,6 @@ export default class GitHubStream extends GitHubBase {
   }
 
   downloadFile(token: string, sha: string) {
-    const span = trace.getTracer("ano-file").startSpan("GHStream.downloadFile");
-    span.setAttribute("sha", sha);
     const oct = octokit(token);
     try {
       const { url } = oct.rest.git.getBlob.endpoint({
@@ -40,14 +37,11 @@ export default class GitHubStream extends GitHubBase {
       });
     } catch (error) {
       console.error(error);
-      // span.recordException(error as Error);
       throw new AnonymousError("repo_not_accessible", {
         httpStatus: 404,
         object: this.data,
         cause: error as Error,
       });
-    } finally {
-      span.end();
     }
   }
 
@@ -56,12 +50,6 @@ export default class GitHubStream extends GitHubBase {
     repoId: string,
     fileSha: () => Promise<string> | string
   ) {
-    const span = trace
-      .getTracer("ano-file")
-      .startSpan("GHStream.getFileContent");
-    span.setAttribute("repoId", repoId);
-    span.setAttribute("file", filePath);
-
     const fileInfo = await storage.exists(repoId, filePath);
     if (fileInfo == FILE_TYPE.FILE) {
       return storage.read(repoId, filePath);
@@ -76,10 +64,6 @@ export default class GitHubStream extends GitHubBase {
       await fileSha()
     );
 
-    content.on("close", () => {
-      span.end();
-    });
-
     // duplicate the stream to write it to the storage
     const stream1 = content.pipe(new stream.PassThrough());
     const stream2 = content.pipe(new stream.PassThrough());
@@ -99,12 +83,6 @@ export default class GitHubStream extends GitHubBase {
   }
 
   async getFileContent(file: AnonymizedFile): Promise<stream.Readable> {
-    const span = trace
-      .getTracer("ano-file")
-      .startSpan("GHStream.getFileContent");
-    span.setAttribute("repoId", file.repository.repoId);
-    span.setAttribute("file", file.anonymizedPath);
-    try {
     try {
       void file.filePath;
     } catch (_) {
@@ -125,19 +103,10 @@ export default class GitHubStream extends GitHubBase {
         return fileSha;
       }
     );
-    } finally {
-      span.end();
-    }
   }
 
   async getFiles(progress?: (status: string) => void) {
-    const span = trace.getTracer("ano-file").startSpan("GHStream.getFiles");
-    span.setAttribute("repoId", this.data.repoId);
-    try {
     return this.getTruncatedTree(this.data.commit, progress);
-    } finally {
-      span.end();
-    }
   }
 
   private async getGHTree(
@@ -145,9 +114,6 @@ export default class GitHubStream extends GitHubBase {
     count = { request: 0, file: 0 },
     opt = { recursive: true, callback: () => {} }
   ) {
-    const span = trace.getTracer("ano-file").startSpan("GHStream.getGHTree");
-    span.setAttribute("sha", sha);
-    try {
     const oct = octokit(await this.data.getToken());
     const ghRes = await oct.git.getTree({
       owner: this.data.organization,
@@ -161,9 +127,6 @@ export default class GitHubStream extends GitHubBase {
       opt.callback();
     }
     return ghRes.data;
-    } finally {
-      span.end();
-    }
   }
 
   private async getTruncatedTree(
@@ -175,13 +138,7 @@ export default class GitHubStream extends GitHubBase {
       request: 0,
       file: 0,
     };
-    const span = trace
-      .getTracer("ano-file")
-      .startSpan("GHStream.getTruncatedTree");
-    span.setAttribute("sha", sha);
-    span.setAttribute("parentPath", parentPath);
     const output: IFile[] = [];
-    try {
     let data = null;
     try {
       data = await this.getGHTree(sha, count, {
@@ -231,9 +188,6 @@ export default class GitHubStream extends GitHubBase {
       output.push(...this.tree2Tree(data.tree, parentPaths[i]));
     });
     return output;
-    } finally {
-      span.end();
-    }
   }
 
   private tree2Tree(
@@ -247,9 +201,6 @@ export default class GitHubStream extends GitHubBase {
     }[],
     parentPath: string = ""
   ) {
-    const span = trace.getTracer("ano-file").startSpan("GHStream.tree2Tree");
-    span.setAttribute("parentPath", parentPath);
-    try {
     return tree.map((elem) => {
       const fullPath = path.join(parentPath, elem.path || "");
       let pathFile = dirname(fullPath);
@@ -264,8 +215,5 @@ export default class GitHubStream extends GitHubBase {
         sha: elem.sha,
       });
     });
-    } finally {
-      span.end();
-    }
   }
 }

src/core/storage/FileSystem.ts

@@ -7,7 +7,6 @@ import { Readable, pipeline, Transform } from "stream";
 import * as archiver from "archiver";
 import { promisify } from "util";
 import { lookup } from "mime-types";
-import { trace } from "@opentelemetry/api";
 import StorageBase, { FILE_TYPE } from "./Storage";
 import FileModel from "../model/files/files.model";
 import { IFile } from "../model/files/files.types";
@@ -22,11 +21,6 @@ export default class FileSystem extends StorageBase {
   /** @override */
   async exists(repoId: string, p: string = ""): Promise<FILE_TYPE> {
     const fullPath = join(config.FOLDER, this.repoPath(repoId), p);
-    return trace
-      .getTracer("ano-file")
-      .startActiveSpan("fs.exists", async (span) => {
-        span.setAttribute("path", p);
-        span.setAttribute("full-path", fullPath);
     try {
       const stat = await fs.promises.stat(fullPath);
       if (stat.isDirectory()) return FILE_TYPE.FOLDER;
@@ -34,25 +28,13 @@ export default class FileSystem extends StorageBase {
     } catch (_) {
       // ignore file not found or not downloaded
     }
-        span.end();
     return FILE_TYPE.NOT_FOUND;
-      });
   }
 
   /** @override */
   async send(repoId: string, p: string, res: Response) {
     const fullPath = join(config.FOLDER, this.repoPath(repoId), p);
-    return trace
+    res.sendFile(fullPath, { dotfiles: "allow" });
-      .getTracer("ano-file")
-      .startActiveSpan("fs.send", async (span) => {
-        span.setAttribute("path", fullPath);
-        res.sendFile(fullPath, { dotfiles: "allow" }, (err) => {
-          if (err) {
-            span.recordException(err);
-          }
-          span.end();
-        });
-      });
   }
 
   /** @override */
@@ -79,9 +61,7 @@ export default class FileSystem extends StorageBase {
     p: string,
     data: string | Readable
   ): Promise<void> {
-    const span = trace.getTracer("ano-file").startSpan("fs.write");
     const fullPath = join(config.FOLDER, this.repoPath(repoId), p);
-    span.setAttribute("path", fullPath);
     try {
       await this.mk(repoId, dirname(p));
       if (data instanceof Readable) {
@@ -91,32 +71,21 @@ export default class FileSystem extends StorageBase {
       }
       return await fs.promises.writeFile(fullPath, data, "utf-8");
     } catch (err: any) {
-      span.recordException(err);
       // throw err;
-    } finally {
-      span.end();
     }
   }
 
   /** @override */
   async rm(repoId: string, dir: string = ""): Promise<void> {
-    const span = trace.getTracer("ano-file").startSpan("fs.rm");
     const fullPath = join(config.FOLDER, this.repoPath(repoId), dir);
-    span.setAttribute("path", fullPath);
-    try {
     await fs.promises.rm(fullPath, {
       force: true,
       recursive: true,
     });
-    } finally {
-      span.end();
-    }
   }
 
   /** @override */
   async mk(repoId: string, dir: string = ""): Promise<void> {
-    const span = trace.getTracer("ano-file").startSpan("fs.mk");
-    span.setAttribute("path", dir);
     const fullPath = join(config.FOLDER, this.repoPath(repoId), dir);
     try {
       await fs.promises.mkdir(fullPath, {
@@ -124,11 +93,8 @@ export default class FileSystem extends StorageBase {
       });
     } catch (err: any) {
       if (err.code !== "EEXIST") {
-        span.recordException(err);
         throw err;
       }
-    } finally {
-      span.end();
     }
   }
 
@@ -140,10 +106,6 @@ export default class FileSystem extends StorageBase {
       onEntry?: (file: { path: string; size: number }) => void;
     } = {}
   ): Promise<IFile[]> {
-    return trace
-      .getTracer("ano-file")
-      .startActiveSpan("fs.listFiles", async (span) => {
-        span.setAttribute("path", dir);
     const fullPath = join(config.FOLDER, this.repoPath(repoId), dir);
     const files = await fs.promises.readdir(fullPath);
     const output2: IFile[] = [];
@@ -174,12 +136,10 @@ export default class FileSystem extends StorageBase {
           );
         }
       } catch (error) {
-            span.recordException(error as Error);
+        // ignore stat errors for individual files
       }
     }
-        span.end();
     return output2;
-      });
   }
 
   /** @override */

src/core/storage/S3.ts

@@ -12,7 +12,6 @@ import ArchiveStreamToS3 from "decompress-stream-to-s3";
 import { Response } from "express";
 import { lookup } from "mime-types";
 import * as archiver from "archiver";
-import { trace } from "@opentelemetry/api";
 import { dirname, basename, join } from "path";
 import AnonymousError from "../AnonymousError";
 import StorageBase, { FILE_TYPE } from "./Storage";
@@ -51,9 +50,6 @@ export default class S3Storage extends StorageBase {
 
   /** @override */
   async exists(repoId: string, path: string = ""): Promise<FILE_TYPE> {
-    const span = trace.getTracer("ano-file").startSpan("s3.exists");
-    span.setAttribute("path", path);
-    try {
     if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
     try {
       // if we can get the file info, it is a file
@@ -70,9 +66,6 @@ export default class S3Storage extends StorageBase {
         ? FILE_TYPE.FOLDER
         : FILE_TYPE.NOT_FOUND;
     }
-    } finally {
-      span.end();
-    }
   }
 
   /** @override */
@@ -82,10 +75,6 @@ export default class S3Storage extends StorageBase {
 
   /** @override */
   async rm(repoId: string, dir: string = ""): Promise<void> {
-    const span = trace.getTracer("ano-file").startSpan("s3.rm");
-    span.setAttribute("repoId", repoId);
-    span.setAttribute("path", dir);
-    try {
     if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
     const data = await this.client(200000).listObjectsV2({
       Bucket: config.S3_BUCKET,
@@ -113,17 +102,10 @@ export default class S3Storage extends StorageBase {
     if (data.IsTruncated) {
       await this.rm(repoId, dir);
     }
-    } finally {
-      span.end();
-    }
   }
 
   /** @override */
   async send(repoId: string, path: string, res: Response) {
-    const span = trace.getTracer("ano-file").startSpan("s3.send");
-    span.setAttribute("repoId", repoId);
-    span.setAttribute("path", path);
-    try {
     if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
     try {
       const command = new GetObjectCommand({
@@ -144,23 +126,15 @@ export default class S3Storage extends StorageBase {
         res.end();
       }
     } catch (error) {
-        span.recordException(error as Error);
       try {
         res.status(500);
       } catch (err) {
         console.error(`[ERROR] S3 send ${path}`, err);
       }
     }
-    } finally {
-      span.end();
-    }
   }
 
   async fileInfo(repoId: string, path: string) {
-    const span = trace.getTracer("ano-file").startSpan("s3.fileInfo");
-    span.setAttribute("repoId", repoId);
-    span.setAttribute("path", path);
-    try {
     if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
     const info = await this.client(3000).headObject({
       Bucket: config.S3_BUCKET,
@@ -173,17 +147,10 @@ export default class S3Storage extends StorageBase {
         ? info.ContentType
         : (lookup(path) as string),
     };
-    } finally {
-      span.end();
-    }
   }
 
   /** @override */
   async read(repoId: string, path: string): Promise<Readable> {
-    const span = trace.getTracer("ano-file").startSpan("s3.rreadm");
-    span.setAttribute("repoId", repoId);
-    span.setAttribute("path", path);
-    try {
     if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
     const command = new GetObjectCommand({
       Bucket: config.S3_BUCKET,
@@ -197,9 +164,6 @@ export default class S3Storage extends StorageBase {
       });
     }
     return res as Readable;
-    } finally {
-      span.end();
-    }
   }
 
   /** @override */
@@ -209,16 +173,11 @@ export default class S3Storage extends StorageBase {
     data: string | Readable,
     source?: string
   ): Promise<void> {
-    const span = trace.getTracer("ano-file").startSpan("s3.rm");
-    span.setAttribute("repoId", repoId);
-    span.setAttribute("path", path);
-    try {
     if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
 
     if (data instanceof Readable) {
       data.on("error", (err) => {
         console.error(`[ERROR] S3 write ${path}`, err);
-          span.recordException(err as Error);
         this.rm(repoId, path);
       });
     }
@@ -240,16 +199,10 @@ export default class S3Storage extends StorageBase {
     });
 
     await parallelUploads3.done();
-    } finally {
-      span.end();
-    }
   }
 
   /** @override */
   async listFiles(repoId: string, dir: string = ""): Promise<IFile[]> {
-    const span = trace.getTracer("ano-file").startSpan("s3.listFiles");
-    span.setAttribute("path", dir);
-    try {
     if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
     if (dir && dir[dir.length - 1] != "/") dir = dir + "/";
     const out: IFile[] = [];
@@ -280,9 +233,6 @@ export default class S3Storage extends StorageBase {
       }
     } while (req && req.Contents && req.IsTruncated);
     return out;
-    } finally {
-      span.end();
-    }
   }
 
   /** @override */
@@ -293,8 +243,6 @@ export default class S3Storage extends StorageBase {
     source?: string
   ): Promise<void> {
     let toS3: ArchiveStreamToS3;
-    const span = trace.getTracer("ano-file").startSpan("s3.extractZip");
-    span.setAttribute("path", path);
     return new Promise((resolve, reject) => {
       if (!config.S3_BUCKET) return reject("S3_BUCKET not set");
       toS3 = new ArchiveStreamToS3({
@@ -315,14 +263,11 @@ export default class S3Storage extends StorageBase {
       });
       pipeline(data, toS3, (err) => {
         if (err) {
-          span.recordException(err as Error);
           return reject(err);
         }
-        span.end();
         resolve();
       })
         .on("finish", () => {
-          span.end();
           resolve();
         })
         .on("error", reject);
@@ -338,10 +283,6 @@ export default class S3Storage extends StorageBase {
       fileTransformer?: (p: string) => Transform;
     }
   ) {
-    const span = trace.getTracer("ano-file").startSpan("s3.archive");
-    span.setAttribute("repoId", repoId);
-    span.setAttribute("path", dir);
-    try {
     if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
     const archive = archiver(opt?.format || "zip", {});
     if (dir && dir[dir.length - 1] != "/") dir = dir + "/";
@@ -378,8 +319,5 @@ export default class S3Storage extends StorageBase {
     } while (req && req.Contents?.length && req.IsTruncated);
     archive.finalize();
     return archive;
-    } finally {
-      span.end();
-    }
   }
 }

src/queue/processes/downloadRepository.ts

@@ -1,4 +1,3 @@
-import { Exception, trace } from "@opentelemetry/api";
 import { SandboxedJob } from "bullmq";
 import { config } from "dotenv";
 config();
@@ -14,8 +13,6 @@ export default async function (job: SandboxedJob<Repository, void>) {
     connect: () => Promise<void>;
     getRepository: typeof getRepositoryImport;
   } = require("../../server/database");
-  const span = trace.getTracer("ano-file").startSpan("proc.downloadRepository");
-  span.setAttribute("repoId", job.data.repoId);
   console.log(`[QUEUE] ${job.data.repoId} is going to be downloaded`);
   let statusInterval: any = null;
   await connect();
@@ -58,17 +55,14 @@ export default async function (job: SandboxedJob<Repository, void>) {
     } catch (error) {
       updateProgress({ status: "error" });
       if (error instanceof Error) {
-        span.recordException(error as Exception);
         await repo.updateStatus(RepositoryStatus.ERROR, error.message);
       } else if (typeof error === "string") {
         await repo.updateStatus(RepositoryStatus.ERROR, error);
-        span.recordException(error);
       }
       throw error;
     }
   } catch (error: any) {
     clearInterval(statusInterval);
-    span.recordException(error as Exception);
     console.log(`[QUEUE] ${job.data.repoId} is finished with an error`, error);
     setTimeout(async () => {
       // delay to avoid double saving
@@ -78,6 +72,5 @@ export default async function (job: SandboxedJob<Repository, void>) {
     }, 400);
   } finally {
     clearInterval(statusInterval);
-    span.end();
   }
 }

src/queue/processes/removeCache.ts

@@ -1,4 +1,3 @@
-import { Exception, trace } from "@opentelemetry/api";
 import { SandboxedJob } from "bullmq";
 import Repository from "../../core/Repository";
 import { getRepository as getRepositoryImport } from "../../server/database";
@@ -11,24 +10,16 @@ export default async function (job: SandboxedJob<Repository, void>) {
     connect: () => Promise<void>;
     getRepository: typeof getRepositoryImport;
   } = require("../../server/database");
-  const span = trace.getTracer("ano-file").startSpan("proc.removeCache");
-  span.setAttribute("repoId", job.data.repoId);
   try {
     await connect();
     console.log(
       `[QUEUE] Cache of ${job.data.repoId} is going to be removed...`
     );
     const repo = await getRepository(job.data.repoId);
-    try {
     await repo.removeCache();
   } catch (error) {
-      span.recordException(error as Exception);
+    // error already handled
-      throw error;
-    }
-  } catch (error) {
-    span.recordException(error as Exception);
   } finally {
     console.log(`[QUEUE] Cache of ${job.data.repoId} is removed.`);
-    span.end();
   }
 }

src/queue/processes/removeRepository.ts

@@ -1,4 +1,3 @@
-import { trace } from "@opentelemetry/api";
 import { SandboxedJob } from "bullmq";
 import Repository from "../../core/Repository";
 import { getRepository as getRepositoryImport } from "../../server/database";
@@ -12,8 +11,6 @@ export default async function (job: SandboxedJob<Repository, void>) {
     connect: () => Promise<void>;
     getRepository: typeof getRepositoryImport;
   } = require("../../server/database");
-  const span = trace.getTracer("ano-file").startSpan("proc.removeRepository");
-  span.setAttribute("repoId", job.data.repoId);
   try {
     await connect();
     console.log(`[QUEUE] ${job.data.repoId} is going to be removed`);
@@ -27,13 +24,11 @@ export default async function (job: SandboxedJob<Repository, void>) {
       } else if (typeof error === "string") {
         await repo.updateStatus(RepositoryStatus.ERROR, error);
       }
-      span.recordException(error as Error);
       throw error;
     }
   } catch (error) {
-    span.recordException(error as Error);
+    // error already handled
   } finally {
     console.log(`[QUEUE] ${job.data.repoId} is removed`);
-    span.end();
   }
 }