feat: introduce streamers that handle the stream and anonymization from github

src/core/source/GitHubBase.ts

@@ -0,0 +1,27 @@
+import { Readable } from "stream";
+
+import AnonymizedFile from "../AnonymizedFile";
+import { Tree } from "../types";
+import { SourceBase } from "./Source";
+
+export interface GitHubBaseData {
+  getToken: () => string | Promise<string>;
+  repoId: string;
+  organization: string;
+  repoName: string;
+  commit: string;
+}
+
+export default abstract class GitHubBase implements SourceBase {
+  abstract type: "GitHubDownload" | "GitHubStream" | "Zip";
+  accessToken: string | undefined;
+
+  constructor(readonly data: GitHubBaseData) {}
+
+  abstract getFileContent(
+    file: AnonymizedFile,
+    progress?: (status: string) => void
+  ): Promise<Readable>;
+
+  abstract getFiles(progress?: (status: string) => void): Promise<Tree>;
+}

src/core/source/GitHubDownload.ts

@@ -0,0 +1,121 @@
+import got from "got";
+import { Readable } from "stream";
+import { OctokitResponse } from "@octokit/types";
+
+import storage from "../storage";
+import GitHubBase, { GitHubBaseData } from "./GitHubBase";
+import { trace } from "@opentelemetry/api";
+import { FILE_TYPE } from "../storage/Storage";
+import { octokit } from "../GitHubUtils";
+import AnonymousError from "../AnonymousError";
+import AnonymizedFile from "../AnonymizedFile";
+
+export default class GitHubDownload extends GitHubBase {
+  type: "GitHubDownload" | "GitHubStream" | "Zip" = "GitHubDownload";
+  constructor(data: GitHubBaseData) {
+    super(data);
+  }
+
+  private async _getZipUrl(): Promise<OctokitResponse<unknown, 302>> {
+    const oct = octokit(await this.data.getToken());
+    return oct.rest.repos.downloadZipballArchive({
+      owner: this.data.organization,
+      repo: this.data.repoName,
+      ref: this.data.commit || "HEAD",
+      method: "HEAD",
+    });
+  }
+
+  async download(progress?: (status: string) => void) {
+    const span = trace.getTracer("ano-file").startSpan("GHDownload.download");
+    span.setAttribute("repoId", this.data.repoId);
+    try {
+      let response: OctokitResponse<unknown, number>;
+      try {
+        response = await this._getZipUrl();
+      } catch (error) {
+        span.recordException(error as Error);
+        throw new AnonymousError("repo_not_accessible", {
+          httpStatus: 404,
+          object: this.data,
+          cause: error as Error,
+        });
+      }
+      await storage.mk(this.data.repoId);
+      let downloadProgress: { transferred: number } | undefined = undefined;
+      let progressTimeout;
+      let inDownload = true;
+
+      async function updateProgress() {
+        if (inDownload) {
+          if (progress) {
+            progress(downloadProgress?.transferred?.toString() || "");
+          }
+          progressTimeout = setTimeout(updateProgress, 1500);
+        }
+      }
+      updateProgress();
+
+      try {
+        const downloadStream = got.stream(response.url);
+        downloadStream.addListener("downloadProgress", async (p) => {
+          downloadProgress = p;
+        });
+        await storage.extractZip(
+          this.data.repoId,
+          "",
+          downloadStream,
+          this.type
+        );
+      } catch (error) {
+        span.recordException(error as Error);
+        throw new AnonymousError("unable_to_download", {
+          httpStatus: 500,
+          cause: error as Error,
+          object: this.data,
+        });
+      } finally {
+        inDownload = false;
+        clearTimeout(progressTimeout);
+      }
+    } finally {
+      span.end();
+    }
+  }
+
+  async getFileContent(
+    file: AnonymizedFile,
+    progress?: (status: string) => void
+  ): Promise<Readable> {
+    const span = trace
+      .getTracer("ano-file")
+      .startSpan("GHDownload.getFileContent");
+    span.setAttribute("repoId", file.repository.repoId);
+    try {
+      const exists = await storage.exists(file.filePath);
+      if (exists === FILE_TYPE.FILE) {
+        return storage.read(this.data.repoId, file.filePath);
+      } else if (exists === FILE_TYPE.FOLDER) {
+        throw new AnonymousError("folder_not_supported", {
+          httpStatus: 400,
+          object: file,
+        });
+      }
+      // will throw an error if the file is not in the repository
+      await file.originalPath();
+
+      // the cache is not ready, we need to download the repository
+      await this.download(progress);
+      return storage.read(this.data.repoId, file.filePath);
+    } finally {
+      span.end();
+    }
+  }
+
+  async getFiles(progress?: (status: string) => void) {
+    if ((await storage.exists(this.data.repoId)) === FILE_TYPE.NOT_FOUND) {
+      await this.download(progress);
+    }
+    return storage.listFiles(this.data.repoId);
+  }
+}

src/core/source/GitHubRepository.ts

@@ -0,0 +1,299 @@
+import { Branch } from "../types";
+import * as gh from "parse-github-url";
+import { RestEndpointMethodTypes } from "@octokit/rest";
+import { trace } from "@opentelemetry/api";
+
+import AnonymousError from "../AnonymousError";
+import { isConnected } from "../../server/database";
+import { octokit } from "../GitHubUtils";
+import { IRepositoryDocument } from "../model/repositories/repositories.types";
+import RepositoryModel from "../model/repositories/repositories.model";
+
+export class GitHubRepository {
+  private _data: Partial<{
+    [P in keyof IRepositoryDocument]: IRepositoryDocument[P];
+  }>;
+  constructor(
+    data: Partial<{ [P in keyof IRepositoryDocument]: IRepositoryDocument[P] }>
+  ) {
+    this._data = data;
+  }
+
+  toJSON() {
+    return {
+      repo: this.repo,
+      owner: this.owner,
+      hasPage: this._data.hasPage,
+      pageSource: this._data.pageSource,
+      fullName: this.fullName,
+      defaultBranch: this._data.defaultBranch,
+      size: this.size,
+    };
+  }
+
+  get model() {
+    return this._data;
+  }
+
+  public get fullName(): string | undefined {
+    return this._data.name;
+  }
+
+  public get id(): string | undefined {
+    return this._data.externalId;
+  }
+
+  public get size(): number | undefined {
+    return this._data.size;
+  }
+
+  async getCommitInfo(
+    sha: string,
+    opt: {
+      accessToken: string;
+    }
+  ) {
+    const span = trace
+      .getTracer("ano-file")
+      .startSpan("GHRepository.getCommitInfo");
+    span.setAttribute("owner", this.owner);
+    span.setAttribute("repo", this.repo);
+    try {
+      const oct = octokit(opt.accessToken);
+      const commit = await oct.repos.getCommit({
+        owner: this.owner,
+        repo: this.repo,
+        ref: sha,
+      });
+      return commit.data;
+    } finally {
+      span.end();
+    }
+  }
+
+  async branches(opt: {
+    accessToken: string;
+    force?: boolean;
+  }): Promise<Branch[]> {
+    const span = trace.getTracer("ano-file").startSpan("GHRepository.branches");
+    span.setAttribute("owner", this.owner);
+    span.setAttribute("repo", this.repo);
+    try {
+      if (
+        !this._data.branches ||
+        this._data.branches.length == 0 ||
+        opt?.force === true
+      ) {
+        // get the list of repo from github
+        const oct = octokit(opt.accessToken);
+        try {
+          const branches = (
+            await oct.paginate("GET /repos/{owner}/{repo}/branches", {
+              owner: this.owner,
+              repo: this.repo,
+              per_page: 100,
+            })
+          ).map((b) => {
+            return {
+              name: b.name,
+              commit: b.commit.sha,
+              readme: this._data.branches?.filter(
+                (f: Branch) => f.name == b.name
+              )[0]?.readme,
+            } as Branch;
+          });
+          this._data.branches = branches;
+          if (isConnected) {
+            await RepositoryModel.updateOne(
+              { externalId: this.id },
+              { $set: { branches } }
+            );
+          }
+        } catch (error) {
+          span.recordException(error as Error);
+          throw new AnonymousError("repo_not_found", {
+            httpStatus: (error as any).status,
+            cause: error as Error,
+            object: this,
+          });
+        }
+      } else if (isConnected) {
+        const q = await RepositoryModel.findOne({ externalId: this.id }).select(
+          "branches"
+        );
+        this._data.branches = q?.branches;
+      }
+
+      return this._data.branches || [];
+    } finally {
+      span.end();
+    }
+  }
+
+  async readme(opt: {
+    branch?: string;
+    force?: boolean;
+    accessToken: string;
+  }): Promise<string | undefined> {
+    const span = trace.getTracer("ano-file").startSpan("GHRepository.readme");
+    span.setAttribute("owner", this.owner);
+    span.setAttribute("repo", this.repo);
+    try {
+      if (!opt.branch) opt.branch = this._data.defaultBranch || "master";
+
+      const model = await RepositoryModel.findOne({
+        externalId: this.id,
+      }).select("branches");
+
+      if (!model) {
+        throw new AnonymousError("repo_not_found", { httpStatus: 404 });
+      }
+
+      this._data.branches = await this.branches(opt);
+      model.branches = this._data.branches;
+
+      const selected = model.branches.filter((f) => f.name == opt.branch)[0];
+      if (selected && (!selected.readme || opt?.force === true)) {
+        // get the list of repo from github
+        const oct = octokit(opt.accessToken);
+        try {
+          const ghRes = await oct.repos.getReadme({
+            owner: this.owner,
+            repo: this.repo,
+            ref: selected?.commit,
+          });
+          const readme = Buffer.from(
+            ghRes.data.content,
+            ghRes.data.encoding as BufferEncoding
+          ).toString("utf-8");
+          selected.readme = readme;
+          await model.save();
+        } catch (error) {
+          span.recordException(error as Error);
+          throw new AnonymousError("readme_not_available", {
+            httpStatus: 404,
+            cause: error as Error,
+            object: this,
+          });
+        }
+      }
+
+      if (!selected) {
+        throw new AnonymousError("readme_not_available", {
+          httpStatus: 404,
+          object: this,
+        });
+      }
+
+      return selected.readme;
+    } finally {
+      span.end();
+    }
+  }
+
+  public get owner(): string {
+    if (!this.fullName) {
+      throw new AnonymousError("invalid_repo", {
+        httpStatus: 400,
+        object: this,
+      });
+    }
+    const repo = gh(this.fullName);
+    if (!repo) {
+      throw new AnonymousError("invalid_repo", {
+        httpStatus: 400,
+        object: this,
+      });
+    }
+    return repo.owner || this.fullName;
+  }
+
+  public get repo(): string {
+    if (!this.fullName) {
+      throw new AnonymousError("invalid_repo", {
+        httpStatus: 400,
+        object: this,
+      });
+    }
+    const repo = gh(this.fullName);
+    if (!repo) {
+      throw new AnonymousError("invalid_repo", {
+        httpStatus: 400,
+        object: this,
+      });
+    }
+    return repo.name || this.fullName;
+  }
+}
+
+export async function getRepositoryFromGitHub(opt: {
+  owner: string;
+  repo: string;
+  accessToken: string;
+}) {
+  const span = trace
+    .getTracer("ano-file")
+    .startSpan("GHRepository.getRepositoryFromGitHub");
+  span.setAttribute("owner", opt.owner);
+  span.setAttribute("repo", opt.repo);
+  try {
+    if (opt.repo.indexOf(".git") > -1) {
+      opt.repo = opt.repo.replace(".git", "");
+    }
+    const oct = octokit(opt.accessToken);
+    let r: RestEndpointMethodTypes["repos"]["get"]["response"]["data"];
+    try {
+      r = (
+        await oct.repos.get({
+          owner: opt.owner,
+          repo: opt.repo,
+        })
+      ).data;
+    } catch (error) {
+      span.recordException(error as Error);
+      throw new AnonymousError("repo_not_found", {
+        httpStatus: (error as any).status,
+        object: {
+          owner: opt.owner,
+          repo: opt.repo,
+        },
+        cause: error as Error,
+      });
+    }
+    if (!r)
+      throw new AnonymousError("repo_not_found", {
+        httpStatus: 404,
+        object: {
+          owner: opt.owner,
+          repo: opt.repo,
+        },
+      });
+    let model = new RepositoryModel({ externalId: "gh_" + r.id });
+    if (isConnected) {
+      const dbModel = await RepositoryModel.findOne({
+        externalId: "gh_" + r.id,
+      });
+      if (dbModel) {
+        model = dbModel;
+      }
+    }
+    model.name = r.full_name;
+    model.url = r.html_url;
+    model.size = r.size;
+    model.defaultBranch = r.default_branch;
+    model.hasPage = r.has_pages;
+    if (model.hasPage) {
+      const ghPageRes = await oct.repos.getPages({
+        owner: opt.owner,
+        repo: opt.repo,
+      });
+      model.pageSource = ghPageRes.data.source;
+    }
+    if (isConnected) {
+      await model.save();
+    }
+    return new GitHubRepository(model);
+  } finally {
+    span.end();
+  }
+}

src/core/source/GitHubStream.ts

@@ -0,0 +1,301 @@
+import AnonymizedFile from "../AnonymizedFile";
+import GitHubBase, { GitHubBaseData } from "./GitHubBase";
+import storage from "../storage";
+import { Tree } from "../types";
+import * as path from "path";
+import got from "got";
+
+import * as stream from "stream";
+import AnonymousError from "../AnonymousError";
+import config from "../../config";
+import { trace } from "@opentelemetry/api";
+import { FILE_TYPE } from "../storage/Storage";
+import { octokit } from "../GitHubUtils";
+
+export default class GitHubStream extends GitHubBase {
+  type: "GitHubDownload" | "GitHubStream" | "Zip" = "GitHubStream";
+
+  constructor(data: GitHubBaseData) {
+    super(data);
+  }
+
+  downloadFile(token: string, sha: string) {
+    const span = trace.getTracer("ano-file").startSpan("GHStream.downloadFile");
+    span.setAttribute("sha", sha);
+    const oct = octokit(token);
+    try {
+      const { url } = oct.rest.git.getBlob.endpoint({
+        owner: this.data.organization,
+        repo: this.data.repoName,
+        file_sha: sha,
+      });
+      return got.stream(url, {
+        headers: {
+          "X-GitHub-Api-Version": "2022-11-28",
+          accept: "application/vnd.github.raw+json",
+          authorization: `token ${token}`,
+        },
+      });
+    } catch (error) {
+      console.error(error);
+      // span.recordException(error as Error);
+      throw new AnonymousError("repo_not_accessible", {
+        httpStatus: 404,
+        object: this.data,
+        cause: error as Error,
+      });
+    } finally {
+      span.end();
+    }
+  }
+
+  async getFileContent(file: AnonymizedFile): Promise<stream.Readable> {
+    const span = trace
+      .getTracer("ano-file")
+      .startSpan("GHStream.getFileContent");
+    span.setAttribute("repoId", file.repository.repoId);
+    span.setAttribute("file", file.anonymizedPath);
+    try {
+      try {
+        file.filePath;
+      } catch (_) {
+        // compute the original path if ambiguous
+        await file.originalPath();
+      }
+      const fileInfo = await storage.exists(
+        file.repository.repoId,
+        file.filePath
+      );
+      if (fileInfo == FILE_TYPE.FILE) {
+        return storage.read(file.repository.repoId, file.filePath);
+      } else if (fileInfo == FILE_TYPE.FOLDER) {
+        throw new AnonymousError("folder_not_supported", {
+          httpStatus: 400,
+          object: file,
+        });
+      }
+      span.setAttribute("path", file.filePath);
+      const file_sha = await file.sha();
+      if (!file_sha) {
+        throw new AnonymousError("file_not_accessible", {
+          httpStatus: 404,
+          object: file,
+        });
+      }
+      const content = this.downloadFile(await this.data.getToken(), file_sha);
+
+      // duplicate the stream to write it to the storage
+      const stream1 = content.pipe(new stream.PassThrough());
+      const stream2 = content.pipe(new stream.PassThrough());
+
+      content.on("error", (error) => {
+        error = new AnonymousError("file_not_found", {
+          httpStatus: (error as any).status || (error as any).httpStatus,
+          cause: error as Error,
+          object: file,
+        });
+        stream1.emit("error", error);
+        stream2.emit("error", error);
+      });
+
+      storage.write(file.repository.repoId, file.filePath, stream1, this.type);
+      return stream2;
+    } finally {
+      span.end();
+    }
+  }
+
+  async getFiles() {
+    const span = trace.getTracer("ano-file").startSpan("GHStream.getFiles");
+    span.setAttribute("repoId", this.data.repoId);
+    try {
+      return this.getTree(this.data.commit);
+    } finally {
+      span.end();
+    }
+  }
+
+  private async getTree(
+    sha: string,
+    truncatedTree: Tree = {},
+    parentPath: string = "",
+    count = {
+      file: 0,
+      request: 0,
+    }
+  ) {
+    const span = trace.getTracer("ano-file").startSpan("GHStream.getTree");
+    span.setAttribute("sha", sha);
+
+    let ghRes: Awaited<ReturnType<typeof this.getGHTree>>;
+    try {
+      count.request++;
+      ghRes = await this.getGHTree(sha, { recursive: true });
+    } catch (error) {
+      console.error(error);
+      span.recordException(error as Error);
+      if ((error as any).status == 409) {
+        // cannot be empty otherwise it would try to download it again
+        span.end();
+        return { __: {} };
+      } else {
+        const err = new AnonymousError("repo_not_accessible", {
+          httpStatus: (error as any).status,
+          cause: error as Error,
+          object: {
+            tree_sha: sha,
+          },
+        });
+        span.recordException(err);
+        span.end();
+        throw err;
+      }
+    }
+    const tree = this.tree2Tree(ghRes.tree, truncatedTree, parentPath);
+    count.file += ghRes.tree.length;
+    if (ghRes.truncated) {
+      await this.getTruncatedTree(sha, tree, parentPath, count);
+    }
+    span.end();
+    return tree;
+  }
+
+  private async getGHTree(sha: string, opt = { recursive: true }) {
+    const span = trace.getTracer("ano-file").startSpan("GHStream.getGHTree");
+    span.setAttribute("sha", sha);
+    try {
+      const oct = octokit(await this.data.getToken());
+      const ghRes = await oct.git.getTree({
+        owner: this.data.organization,
+        repo: this.data.repoName,
+        tree_sha: sha,
+        recursive: opt.recursive ? "1" : undefined,
+      });
+      return ghRes.data;
+    } finally {
+      span.end();
+    }
+  }
+
+  private async getTruncatedTree(
+    sha: string,
+    truncatedTree: Tree = {},
+    parentPath: string = "",
+    count = {
+      file: 0,
+      request: 0,
+    },
+    depth = 0
+  ) {
+    const span = trace
+      .getTracer("ano-file")
+      .startSpan("GHStream.getTruncatedTree");
+    span.setAttribute("sha", sha);
+    span.setAttribute("parentPath", parentPath);
+    try {
+      count.request++;
+      let data = null;
+
+      try {
+        data = await this.getGHTree(sha, {
+          recursive: false,
+        });
+        this.tree2Tree(data.tree, truncatedTree, parentPath);
+      } catch (error) {
+        span.recordException(error as Error);
+        return;
+      }
+
+      count.file += data.tree.length;
+      if (data.tree.length < 100 && count.request < 200) {
+        const promises: Promise<any>[] = [];
+        for (const file of data.tree) {
+          if (file.type == "tree" && file.path && file.sha) {
+            const elementPath = path.join(parentPath, file.path);
+            promises.push(
+              this.getTruncatedTree(
+                file.sha,
+                truncatedTree,
+                elementPath,
+                count,
+                depth + 1
+              )
+            );
+          }
+        }
+        await Promise.all(promises);
+      } else {
+        try {
+          const data = await this.getGHTree(sha, {
+            recursive: true,
+          });
+          this.tree2Tree(data.tree, truncatedTree, parentPath);
+          if (data.truncated) {
+            // TODO: TRUNCATED
+          }
+        } catch (error) {
+          span.recordException(error as Error);
+        }
+      }
+    } finally {
+      span.end();
+    }
+  }
+
+  private tree2Tree(
+    tree: {
+      path?: string;
+      mode?: string;
+      type?: string;
+      sha?: string;
+      size?: number;
+      url?: string;
+    }[],
+    partialTree: Tree = {},
+    parentPath: string = ""
+  ) {
+    const span = trace.getTracer("ano-file").startSpan("GHStream.tree2Tree");
+    span.setAttribute("parentPath", parentPath);
+    try {
+      for (let elem of tree) {
+        let current = partialTree;
+
+        if (!elem.path) continue;
+
+        const paths = path.join(parentPath, elem.path).split("/");
+
+        // if elem is a folder iterate on all folders if it is a file stop before the filename
+        const end = elem.type == "tree" ? paths.length : paths.length - 1;
+        for (let i = 0; i < end; i++) {
+          let p = paths[i];
+          if (p[0] == "$") {
+            p = "\\" + p;
+          }
+          if (!current[p]) {
+            current[p] = {};
+          }
+          current = current[p] as Tree;
+        }
+
+        // if elem is a file add the file size in the file list
+        if (elem.type == "blob") {
+          if (Object.keys(current).length > config.MAX_FILE_FOLDER) {
+            // TODO: TRUNCATED
+            continue;
+          }
+          let p = paths[end];
+          if (p[0] == "$") {
+            p = "\\" + p;
+          }
+          current[p] = {
+            size: elem.size || 0, // size in bit
+            sha: elem.sha || "",
+          };
+        }
+      }
+      return partialTree;
+    } finally {
+      span.end();
+    }
+  }
+}

src/core/source/Source.ts

@@ -0,0 +1,24 @@
+import { Readable } from "stream";
+
+import AnonymizedFile from "../AnonymizedFile";
+import { Tree } from "../types";
+import GitHubDownload from "./GitHubDownload";
+import GitHubStream from "./GitHubStream";
+import Zip from "./Zip";
+
+export type Source = GitHubDownload | GitHubStream | Zip;
+
+export interface SourceBase {
+  readonly type: string;
+
+  /**
+   * Retrieve the fie content
+   * @param file the file of the content to retrieve
+   */
+  getFileContent(file: AnonymizedFile): Promise<Readable>;
+
+  /**
+   * Get all the files from a specific source
+   */
+  getFiles(progress?: (status: string) => void): Promise<Tree>;
+}

src/core/source/Zip.ts

@@ -0,0 +1,28 @@
+import * as stream from "stream";
+
+import AnonymizedFile from "../AnonymizedFile";
+import storage from "../storage";
+import { SourceBase } from "./Source";
+
+export default class Zip implements SourceBase {
+  type = "Zip";
+  url?: string;
+
+  constructor(data: any, readonly repoId: string) {
+    this.url = data.url;
+  }
+
+  async getFiles() {
+    return storage.listFiles(this.repoId);
+  }
+
+  async getFileContent(file: AnonymizedFile): Promise<stream.Readable> {
+    return storage.read(file.repository.repoId, file.filePath);
+  }
+
+  toJSON(): any {
+    return {
+      type: this.type,
+    };
+  }
+}