improve performance

index.ts

@@ -1,7 +1,16 @@
+const cluster = require('node:cluster');
 import { config } from "dotenv";
 config();
 
 import server from "./src/server";
 
-// start the server
+if (cluster.isPrimary) {
-server();
+  console.log(`Master process ${process.pid} is running`);
+
+  for (let i = 0; i < 8; i++) {
+    cluster.fork();
+  }
+} else {
+  // start the server
+  server();
+}

src/PullRequest.ts

@@ -7,8 +7,8 @@ import ConferenceModel from "./database/conference/conferences.model";
 import AnonymousError from "./AnonymousError";
 import { IAnonymizedPullRequestDocument } from "./database/anonymizedPullRequests/anonymizedPullRequests.types";
 import config from "../config";
-import { Octokit } from "@octokit/rest";
 import got from "got";
+import GitHubBase from "./source/GitHubBase";
 
 export default class PullRequest {
   private _model: IAnonymizedPullRequestDocument;
@@ -52,8 +52,7 @@ export default class PullRequest {
       "[INFO] Downloading pull request",
       this._model.source.pullRequestId
     );
-    const auth = await this.getToken();
+    const octokit = GitHubBase.octokit(await this.getToken());
-    const octokit = new Octokit({ auth: auth });
 
     const [owner, repo] = this._model.source.repositoryFullName.split("/");
     const pull_number = this._model.source.pullRequestId;

src/User.ts

@@ -1,4 +1,3 @@
-import { Octokit } from "@octokit/rest";
 import AnonymizedRepositoryModel from "./database/anonymizedRepositories/anonymizedRepositories.model";
 import RepositoryModel from "./database/repositories/repositories.model";
 import { IUserDocument } from "./database/users/users.types";
@@ -7,6 +6,7 @@ import { GitHubRepository } from "./source/GitHubRepository";
 import PullRequest from "./PullRequest";
 import AnonymizedPullRequestModel from "./database/anonymizedPullRequests/anonymizedPullRequests.model";
 import { trace } from "@opentelemetry/api";
+import GitHubBase from "./source/GitHubBase";
 
 /**
  * Model for a user
@@ -66,7 +66,7 @@ export default class User {
       opt?.force === true
     ) {
       // get the list of repo from github
-      const octokit = new Octokit({ auth: this.accessToken });
+      const octokit = GitHubBase.octokit(this.accessToken);
       const repositories = (
         await octokit.paginate("GET /user/repos", {
           visibility: "all",

src/source/GitHubBase.ts

@@ -1,13 +1,14 @@
+import { Octokit } from "@octokit/rest";
+import { trace } from "@opentelemetry/api";
+import { Readable } from "stream";
+
 import AnonymizedFile from "../AnonymizedFile";
 import { Branch, Tree } from "../types";
 import { GitHubRepository } from "./GitHubRepository";
 import config from "../../config";
 import Repository from "../Repository";
-import { Readable } from "stream";
 import UserModel from "../database/users/users.model";
 import AnonymousError from "../AnonymousError";
-import { Octokit } from "@octokit/rest";
-import { trace } from "@opentelemetry/api";
 
 export default abstract class GitHubBase {
   type: "GitHubDownload" | "GitHubStream" | "Zip";
@@ -57,8 +58,17 @@ export default abstract class GitHubBase {
     });
   }
 
+  static octokit(token: string) {
+    return new Octokit({
+      auth: token,
+      request: {
+        fetch: fetch,
+      },
+    });
+  }
+
   static async checkToken(token: string) {
-    const octokit = new Octokit({ auth: token });
+    const octokit = GitHubBase.octokit(token);
     try {
       await octokit.users.getAuthenticated();
       return true;

src/source/GitHubDownload.ts

@@ -1,4 +1,3 @@
-import { Octokit } from "@octokit/rest";
 import got from "got";
 import { Readable } from "stream";
 import { OctokitResponse } from "@octokit/types";
@@ -30,7 +29,7 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
   private async _getZipUrl(
     auth?: string
   ): Promise<OctokitResponse<unknown, 302>> {
-    const octokit = new Octokit({ auth });
+    const octokit = GitHubBase.octokit(auth as string);
     return octokit.rest.repos.downloadZipballArchive({
       owner: this.githubRepository.owner,
       repo: this.githubRepository.repo,
@@ -142,7 +141,9 @@ export default class GitHubDownload extends GitHubBase implements SourceBase {
   }
 
   async getFileContent(file: AnonymizedFile): Promise<Readable> {
-    const span = trace.getTracer("ano-file").startSpan("GHDownload.getFileContent");
+    const span = trace
+      .getTracer("ano-file")
+      .startSpan("GHDownload.getFileContent");
     span.setAttribute("repoId", this.repository.repoId);
     try {
       const exists = await storage.exists(file.originalCachePath);

src/source/GitHubRepository.ts

@@ -1,11 +1,12 @@
 import { Branch } from "../types";
 import * as gh from "parse-github-url";
 import { IRepositoryDocument } from "../database/repositories/repositories.types";
-import { Octokit, RestEndpointMethodTypes } from "@octokit/rest";
+import { RestEndpointMethodTypes } from "@octokit/rest";
 import RepositoryModel from "../database/repositories/repositories.model";
 import AnonymousError from "../AnonymousError";
 import { isConnected } from "../database/database";
 import { trace } from "@opentelemetry/api";
+import GitHubBase from "./GitHubBase";
 
 export class GitHubRepository {
   private _data: Partial<{
@@ -57,7 +58,7 @@ export class GitHubRepository {
     span.setAttribute("owner", this.owner);
     span.setAttribute("repo", this.repo);
     try {
-      const octokit = new Octokit({ auth: opt.accessToken });
+      const octokit = GitHubBase.octokit(opt.accessToken as string);
       const commit = await octokit.repos.getCommit({
         owner: this.owner,
         repo: this.repo,
@@ -83,7 +84,7 @@ export class GitHubRepository {
         opt?.force === true
       ) {
         // get the list of repo from github
-        const octokit = new Octokit({ auth: opt.accessToken });
+        const octokit = GitHubBase.octokit(opt.accessToken as string);
         try {
           const branches = (
             await octokit.paginate("GET /repos/{owner}/{repo}/branches", {
@@ -153,7 +154,7 @@ export class GitHubRepository {
       const selected = model.branches.filter((f) => f.name == opt.branch)[0];
       if (selected && (!selected.readme || opt?.force === true)) {
         // get the list of repo from github
-        const octokit = new Octokit({ auth: opt.accessToken });
+        const octokit = GitHubBase.octokit(opt.accessToken as string);
         try {
           const ghRes = await octokit.repos.getReadme({
             owner: this.owner,
@@ -238,7 +239,7 @@ export async function getRepositoryFromGitHub(opt: {
     if (opt.repo.indexOf(".git") > -1) {
       opt.repo = opt.repo.replace(".git", "");
     }
-    const octokit = new Octokit({ auth: opt.accessToken });
+    const octokit = GitHubBase.octokit(opt.accessToken as string);
     let r: RestEndpointMethodTypes["repos"]["get"]["response"]["data"];
     try {
       r = (

src/source/GitHubStream.ts

@@ -1,4 +1,3 @@
-import { Octokit } from "@octokit/rest";
 import AnonymizedFile from "../AnonymizedFile";
 import Repository from "../Repository";
 import GitHubBase from "./GitHubBase";
@@ -31,9 +30,7 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
       .getTracer("ano-file")
       .startActiveSpan("GHStream.getFileContent", async (span) => {
         span.setAttribute("path", file.anonymizedPath);
-        const octokit = new Octokit({
+        const octokit = GitHubBase.octokit(await this.getToken());
-          auth: await this.getToken(),
-        });
 
         const file_sha = await file.sha();
         if (!file_sha) {
@@ -173,9 +170,7 @@ export default class GitHubStream extends GitHubBase implements SourceBase {
     span.setAttribute("repoId", this.repository.repoId);
     span.setAttribute("sha", sha);
     try {
-      const octokit = new Octokit({
+      const octokit = GitHubBase.octokit(await this.getToken());
-        auth: await this.getToken(),
-      });
       const ghRes = await octokit.git.getTree({
         owner: this.githubRepository.owner,
         repo: this.githubRepository.repo,