From d3924698f6159124e2647f6e745a91ea45dca131 Mon Sep 17 00:00:00 2001 From: tdurieux Date: Sun, 31 Mar 2024 15:12:46 +0100 Subject: [PATCH] improve performance --- index.ts | 13 +++++++++++-- src/PullRequest.ts | 5 ++--- src/User.ts | 4 ++-- src/source/GitHubBase.ts | 18 ++++++++++++++---- src/source/GitHubDownload.ts | 7 ++++--- src/source/GitHubRepository.ts | 11 ++++++----- src/source/GitHubStream.ts | 9 ++------- 7 files changed, 41 insertions(+), 26 deletions(-) diff --git a/index.ts b/index.ts index fbd7cd1..83da5be 100644 --- a/index.ts +++ b/index.ts @@ -1,7 +1,16 @@ +const cluster = require('node:cluster'); import { config } from "dotenv"; config(); import server from "./src/server"; -// start the server -server(); +if (cluster.isPrimary) { + console.log(`Master process ${process.pid} is running`); + + for (let i = 0; i < 8; i++) { + cluster.fork(); + } +} else { + // start the server + server(); +} diff --git a/src/PullRequest.ts b/src/PullRequest.ts index ed9ed6b..9b6f0b8 100644 --- a/src/PullRequest.ts +++ b/src/PullRequest.ts @@ -7,8 +7,8 @@ import ConferenceModel from "./database/conference/conferences.model"; import AnonymousError from "./AnonymousError"; import { IAnonymizedPullRequestDocument } from "./database/anonymizedPullRequests/anonymizedPullRequests.types"; import config from "../config"; -import { Octokit } from "@octokit/rest"; import got from "got"; +import GitHubBase from "./source/GitHubBase"; export default class PullRequest { private _model: IAnonymizedPullRequestDocument; @@ -52,8 +52,7 @@ export default class PullRequest { "[INFO] Downloading pull request", this._model.source.pullRequestId ); - const auth = await this.getToken(); - const octokit = new Octokit({ auth: auth }); + const octokit = GitHubBase.octokit(await this.getToken()); const [owner, repo] = this._model.source.repositoryFullName.split("/"); const pull_number = this._model.source.pullRequestId; diff --git a/src/User.ts b/src/User.ts index eec440a..f0337cc 100644 --- a/src/User.ts +++ b/src/User.ts @@ -1,4 +1,3 @@ -import { Octokit } from "@octokit/rest"; import AnonymizedRepositoryModel from "./database/anonymizedRepositories/anonymizedRepositories.model"; import RepositoryModel from "./database/repositories/repositories.model"; import { IUserDocument } from "./database/users/users.types"; @@ -7,6 +6,7 @@ import { GitHubRepository } from "./source/GitHubRepository"; import PullRequest from "./PullRequest"; import AnonymizedPullRequestModel from "./database/anonymizedPullRequests/anonymizedPullRequests.model"; import { trace } from "@opentelemetry/api"; +import GitHubBase from "./source/GitHubBase"; /** * Model for a user @@ -66,7 +66,7 @@ export default class User { opt?.force === true ) { // get the list of repo from github - const octokit = new Octokit({ auth: this.accessToken }); + const octokit = GitHubBase.octokit(this.accessToken); const repositories = ( await octokit.paginate("GET /user/repos", { visibility: "all", diff --git a/src/source/GitHubBase.ts b/src/source/GitHubBase.ts index 3a3ec9a..26ff14e 100644 --- a/src/source/GitHubBase.ts +++ b/src/source/GitHubBase.ts @@ -1,13 +1,14 @@ +import { Octokit } from "@octokit/rest"; +import { trace } from "@opentelemetry/api"; +import { Readable } from "stream"; + import AnonymizedFile from "../AnonymizedFile"; import { Branch, Tree } from "../types"; import { GitHubRepository } from "./GitHubRepository"; import config from "../../config"; import Repository from "../Repository"; -import { Readable } from "stream"; import UserModel from "../database/users/users.model"; import AnonymousError from "../AnonymousError"; -import { Octokit } from "@octokit/rest"; -import { trace } from "@opentelemetry/api"; export default abstract class GitHubBase { type: "GitHubDownload" | "GitHubStream" | "Zip"; @@ -57,8 +58,17 @@ export default abstract class GitHubBase { }); } + static octokit(token: string) { + return new Octokit({ + auth: token, + request: { + fetch: fetch, + }, + }); + } + static async checkToken(token: string) { - const octokit = new Octokit({ auth: token }); + const octokit = GitHubBase.octokit(token); try { await octokit.users.getAuthenticated(); return true; diff --git a/src/source/GitHubDownload.ts b/src/source/GitHubDownload.ts index 0d06827..95f11e5 100644 --- a/src/source/GitHubDownload.ts +++ b/src/source/GitHubDownload.ts @@ -1,4 +1,3 @@ -import { Octokit } from "@octokit/rest"; import got from "got"; import { Readable } from "stream"; import { OctokitResponse } from "@octokit/types"; @@ -30,7 +29,7 @@ export default class GitHubDownload extends GitHubBase implements SourceBase { private async _getZipUrl( auth?: string ): Promise> { - const octokit = new Octokit({ auth }); + const octokit = GitHubBase.octokit(auth as string); return octokit.rest.repos.downloadZipballArchive({ owner: this.githubRepository.owner, repo: this.githubRepository.repo, @@ -142,7 +141,9 @@ export default class GitHubDownload extends GitHubBase implements SourceBase { } async getFileContent(file: AnonymizedFile): Promise { - const span = trace.getTracer("ano-file").startSpan("GHDownload.getFileContent"); + const span = trace + .getTracer("ano-file") + .startSpan("GHDownload.getFileContent"); span.setAttribute("repoId", this.repository.repoId); try { const exists = await storage.exists(file.originalCachePath); diff --git a/src/source/GitHubRepository.ts b/src/source/GitHubRepository.ts index 7c613fa..33ac84f 100644 --- a/src/source/GitHubRepository.ts +++ b/src/source/GitHubRepository.ts @@ -1,11 +1,12 @@ import { Branch } from "../types"; import * as gh from "parse-github-url"; import { IRepositoryDocument } from "../database/repositories/repositories.types"; -import { Octokit, RestEndpointMethodTypes } from "@octokit/rest"; +import { RestEndpointMethodTypes } from "@octokit/rest"; import RepositoryModel from "../database/repositories/repositories.model"; import AnonymousError from "../AnonymousError"; import { isConnected } from "../database/database"; import { trace } from "@opentelemetry/api"; +import GitHubBase from "./GitHubBase"; export class GitHubRepository { private _data: Partial<{ @@ -57,7 +58,7 @@ export class GitHubRepository { span.setAttribute("owner", this.owner); span.setAttribute("repo", this.repo); try { - const octokit = new Octokit({ auth: opt.accessToken }); + const octokit = GitHubBase.octokit(opt.accessToken as string); const commit = await octokit.repos.getCommit({ owner: this.owner, repo: this.repo, @@ -83,7 +84,7 @@ export class GitHubRepository { opt?.force === true ) { // get the list of repo from github - const octokit = new Octokit({ auth: opt.accessToken }); + const octokit = GitHubBase.octokit(opt.accessToken as string); try { const branches = ( await octokit.paginate("GET /repos/{owner}/{repo}/branches", { @@ -153,7 +154,7 @@ export class GitHubRepository { const selected = model.branches.filter((f) => f.name == opt.branch)[0]; if (selected && (!selected.readme || opt?.force === true)) { // get the list of repo from github - const octokit = new Octokit({ auth: opt.accessToken }); + const octokit = GitHubBase.octokit(opt.accessToken as string); try { const ghRes = await octokit.repos.getReadme({ owner: this.owner, @@ -238,7 +239,7 @@ export async function getRepositoryFromGitHub(opt: { if (opt.repo.indexOf(".git") > -1) { opt.repo = opt.repo.replace(".git", ""); } - const octokit = new Octokit({ auth: opt.accessToken }); + const octokit = GitHubBase.octokit(opt.accessToken as string); let r: RestEndpointMethodTypes["repos"]["get"]["response"]["data"]; try { r = ( diff --git a/src/source/GitHubStream.ts b/src/source/GitHubStream.ts index 30b31ba..57e46a5 100644 --- a/src/source/GitHubStream.ts +++ b/src/source/GitHubStream.ts @@ -1,4 +1,3 @@ -import { Octokit } from "@octokit/rest"; import AnonymizedFile from "../AnonymizedFile"; import Repository from "../Repository"; import GitHubBase from "./GitHubBase"; @@ -31,9 +30,7 @@ export default class GitHubStream extends GitHubBase implements SourceBase { .getTracer("ano-file") .startActiveSpan("GHStream.getFileContent", async (span) => { span.setAttribute("path", file.anonymizedPath); - const octokit = new Octokit({ - auth: await this.getToken(), - }); + const octokit = GitHubBase.octokit(await this.getToken()); const file_sha = await file.sha(); if (!file_sha) { @@ -173,9 +170,7 @@ export default class GitHubStream extends GitHubBase implements SourceBase { span.setAttribute("repoId", this.repository.repoId); span.setAttribute("sha", sha); try { - const octokit = new Octokit({ - auth: await this.getToken(), - }); + const octokit = GitHubBase.octokit(await this.getToken()); const ghRes = await octokit.git.getTree({ owner: this.githubRepository.owner, repo: this.githubRepository.repo,