migrate JavaScript to TypeScript

src/source/GitHubBase.ts

@@ -0,0 +1,83 @@
+import AnonymizedFile from "../AnonymizedFile";
+import { Branch, Tree } from "../types";
+import { GitHubRepository } from "./GitHubRepository";
+import config from "../../config";
+import { OAuthApp } from "@octokit/oauth-app";
+import Repository from "../Repository";
+import * as stream from "stream";
+import UserModel from "../database/users/users.model";
+
+export default abstract class GitHubBase {
+  type: "GitHubDownload" | "GitHubStream" | "Zip";
+  githubRepository: GitHubRepository;
+  branch: Branch;
+  accessToken: string;
+  repository: Repository;
+
+  constructor(
+    data: {
+      type: "GitHubDownload" | "GitHubStream" | "Zip";
+      branch?: string;
+      commit?: string;
+      repositoryId?: string;
+      repositoryName?: string;
+      accessToken?: string;
+    },
+    repository: Repository
+  ) {
+    this.type = data.type;
+    this.accessToken = data.accessToken;
+    this.githubRepository = new GitHubRepository({
+      name: data.repositoryName,
+      externalId: data.repositoryId,
+      branches: [{ commit: data.commit, name: data.branch }],
+    });
+    this.repository = repository;
+    this.branch = { commit: data.commit, name: data.branch };
+  }
+
+  async getFileContent(file: AnonymizedFile): Promise<stream.Readable> {
+    throw new Error("Method not implemented.");
+  }
+  getFiles(): Promise<Tree> {
+    throw new Error("Method not implemented.");
+  }
+
+  async getToken(owner?: string) {
+    if (owner) {
+      const user = await UserModel.findOne({ username: owner });
+      if (user && user.accessToken) {
+        return user.accessToken as string;
+      }
+    }
+    if (this.accessToken) {
+      try {
+        const app = new OAuthApp({
+          clientType: "github-app",
+          clientId: config.CLIENT_ID,
+          clientSecret: config.CLIENT_SECRET,
+        });
+        await app.checkToken({
+          token: this.accessToken,
+        });
+        return this.accessToken;
+      } catch (error) {
+        // console.debug("Token is invalid.", error);
+        this.accessToken = config.GITHUB_TOKEN;
+      }
+    }
+    return config.GITHUB_TOKEN;
+  }
+
+  get url() {
+    return "https://github.com/" + this.githubRepository.fullName;
+  }
+
+  toJSON(): any {
+    return {
+      type: this.type,
+      fullName: this.githubRepository.fullName?.toString(),
+      branch: this.branch,
+    };
+  }
+}

src/source/GitHubDownload.ts

@@ -0,0 +1,75 @@
+import { Octokit } from "@octokit/rest";
+import * as path from "path";
+import config from "../../config";
+import storage from "../storage";
+import Repository from "../Repository";
+
+import GitHubBase from "./GitHubBase";
+import AnonymizedFile from "../AnonymizedFile";
+import { SourceBase } from "../types";
+import * as got from "got";
+import * as stream from "stream";
+import { OctokitResponse } from "@octokit/types";
+
+export default class GitHubDownload extends GitHubBase implements SourceBase {
+  constructor(
+    data: {
+      type: "GitHubDownload" | "GitHubStream" | "Zip";
+      branch?: string;
+      commit?: string;
+      repositoryId?: string;
+      repositoryName?: string;
+      accessToken?: string;
+    },
+    repository: Repository
+  ) {
+    super(data, repository);
+  }
+
+  private async _getZipUrl(
+    auth?: string
+  ): Promise<OctokitResponse<unknown, 302>> {
+    const octokit = new Octokit({ auth });
+    return octokit.rest.repos.downloadTarballArchive({
+      owner: this.githubRepository.owner,
+      repo: this.githubRepository.repo,
+      ref: this.branch?.commit || "HEAD",
+      method: "HEAD",
+    });
+  }
+
+  async download() {
+    let response: OctokitResponse<unknown, number>;
+    try {
+      response = await this._getZipUrl(await this.getToken());
+    } catch (error) {
+      if (error.status == 401 && config.GITHUB_TOKEN) {
+        try {
+          response = await this._getZipUrl(config.GITHUB_TOKEN);
+        } catch (error) {
+          throw new Error("repo_not_accessible");
+        }
+      } else {
+        throw new Error("repo_not_accessible");
+      }
+    }
+    const originalPath = this.repository.originalCachePath;
+    await storage.mk(originalPath);
+    await storage.extractTar(originalPath, got.stream(response.url));
+  }
+
+  async getFileContent(file: AnonymizedFile): Promise<stream.Readable> {
+    await this.download();
+    // update the file list
+    await this.repository.files({ force: true });
+    return storage.read(file.originalCachePath);
+  }
+
+  async getFiles() {
+    const folder = this.repository.originalCachePath;
+    if (!(await storage.exists(folder))) {
+      await this.download();
+    }
+    return storage.listFiles(folder);
+  }
+}

src/source/GitHubRepository.ts

@@ -0,0 +1,171 @@
+import { Branch } from "../types";
+import * as gh from "parse-github-url";
+import { IRepositoryDocument } from "../database/repositories/repositories.types";
+import { Octokit } from "@octokit/rest";
+import RepositoryModel from "../database/repositories/repositories.model";
+
+export class GitHubRepository {
+  private _data: Partial<
+    { [P in keyof IRepositoryDocument]: IRepositoryDocument[P] }
+  >;
+  constructor(
+    data: Partial<{ [P in keyof IRepositoryDocument]: IRepositoryDocument[P] }>
+  ) {
+    this._data = data;
+  }
+
+  toJSON() {
+    return {
+      repo: this.repo,
+      owner: this.owner,
+      hasPage: this._data.hasPage,
+      pageSource: this._data.pageSource,
+      fullName: this.fullName,
+      defaultBranch: this._data.defaultBranch,
+      size: this.size,
+    };
+  }
+
+  get model() {
+    return this._data;
+  }
+
+  public get fullName(): string {
+    return this._data.name;
+  }
+
+  public get id(): string {
+    return this._data.externalId;
+  }
+
+  public get size(): number {
+    return this._data.size;
+  }
+
+  async branches(opt: {
+    accessToken?: string;
+    force?: boolean;
+  }): Promise<Branch[]> {
+    if (
+      !this._data.branches ||
+      this._data.branches.length == 0 ||
+      opt?.force === true
+    ) {
+      // get the list of repo from github
+      const octokit = new Octokit({ auth: opt.accessToken });
+      const branches = (
+        await octokit.paginate(octokit.repos.listBranches, {
+          owner: this.owner,
+          repo: this.repo,
+          per_page: 100,
+        })
+      ).map((b) => {
+        return {
+          name: b.name,
+          commit: b.commit.sha,
+          readme: this._data.branches?.filter(
+            (f: Branch) => f.name == b.name
+          )[0]?.readme,
+        } as Branch;
+      });
+      this._data.branches = branches;
+
+      await RepositoryModel.updateOne(
+        { externalId: this.id },
+        { $set: { branches } }
+      );
+    } else {
+      this._data.branches = (
+        await RepositoryModel.findOne({ externalId: this.id }).select(
+          "branches"
+        )
+      ).branches;
+    }
+
+    return this._data.branches;
+  }
+
+  async readme(opt: {
+    branch?: string;
+    force?: boolean;
+    accessToken?: string;
+  }): Promise<string> {
+    if (!opt.branch) opt.branch = this._data.defaultBranch || "master";
+
+    const model = await RepositoryModel.findOne({
+      externalId: this.id,
+    }).select("branches");
+
+    this._data.branches = await this.branches(opt);
+    model.branches = this._data.branches;
+    
+    const selected = model.branches.filter((f) => f.name == opt.branch)[0];
+    if (!selected?.readme || opt?.force === true) {
+      // get the list of repo from github
+      const octokit = new Octokit({ auth: opt.accessToken });
+      const ghRes = await octokit.repos.getReadme({
+        owner: this.owner,
+        repo: this.repo,
+        ref: selected?.commit,
+      });
+      const readme = Buffer.from(
+        ghRes.data.content,
+        ghRes.data.encoding as BufferEncoding
+      ).toString("utf-8");
+      selected.readme = readme;
+
+      await model.save();
+    }
+
+    return selected.readme;
+  }
+
+  public get owner(): string {
+    const repo = gh(this.fullName);
+    if (!repo) {
+      throw "invalid_repo";
+    }
+    return repo.owner || this.fullName;
+  }
+
+  public get repo(): string {
+    const repo = gh(this.fullName);
+    if (!repo) {
+      throw "invalid_repo";
+    }
+    return repo.name || this.fullName;
+  }
+}
+
+export async function getRepositoryFromGitHub(opt: {
+  owner: string;
+  repo: string;
+  accessToken: string;
+}) {
+  const octokit = new Octokit({ auth: opt.accessToken });
+  const r = (
+    await octokit.repos.get({
+      owner: opt.owner,
+      repo: opt.repo,
+    })
+  ).data;
+  if (!r) throw new Error("repo_not_found");
+  let model = await RepositoryModel.findOne({ externalId: "gh_" + r.id });
+  if (!model) {
+    model = new RepositoryModel({ externalId: "gh_" + r.id });
+  }
+  model.name = r.full_name;
+  model.url = r.html_url;
+  model.size = r.size;
+  model.defaultBranch = r.default_branch;
+  model.hasPage = r.has_pages;
+  if (model.hasPage) {
+    const ghPageRes = await octokit.repos.getPages({
+      owner: opt.owner,
+      repo: opt.repo,
+    });
+    model.pageSource = ghPageRes.data.source;
+  }
+  await model.save();
+  return new GitHubRepository(model);
+}

src/source/GitHubStream.ts

@@ -0,0 +1,171 @@
+import { Octokit } from "@octokit/rest";
+import AnonymizedFile from "../AnonymizedFile";
+import Repository from "../Repository";
+import GitHubBase from "./GitHubBase";
+import storage from "../storage";
+import { SourceBase, Tree } from "../types";
+import * as path from "path";
+
+import * as stream from "stream";
+
+export default class GitHubStream extends GitHubBase implements SourceBase {
+  constructor(
+    data: {
+      type: "GitHubDownload" | "GitHubStream" | "Zip";
+      branch?: string;
+      commit?: string;
+      repositoryId?: string;
+      repositoryName?: string;
+      accessToken?: string;
+    },
+    repository: Repository
+  ) {
+    super(data, repository);
+  }
+
+  async getFileContent(file: AnonymizedFile): Promise<stream.Readable> {
+    if (!file.sha) throw new Error("file_sha_not_provided");
+    const octokit = new Octokit({
+      auth: await this.getToken(),
+    });
+
+    try {
+      const ghRes = await octokit.rest.git.getBlob({
+        owner: this.githubRepository.owner,
+        repo: this.githubRepository.repo,
+        file_sha: file.sha,
+      });
+      if (!ghRes.data.content && ghRes.data.size != 0) {
+        throw new Error("file_not_accessible");
+      }
+      // empty file
+      let content: Buffer;
+      if (ghRes.data.content) {
+        content = Buffer.from(
+          ghRes.data.content,
+          ghRes.data.encoding as BufferEncoding
+        );
+      } else {
+        content = Buffer.from("");
+      }
+      await storage.write(file.originalCachePath, content);
+      return stream.Readable.from(content.toString());
+    } catch (error) {
+      if (error.status == 403) {
+        throw new Error("file_too_big");
+      }
+      console.error(error);
+    }
+    throw new Error("file_not_accessible");
+  }
+
+  async getFiles() {
+    return this.getTree(this.branch.commit);
+  }
+
+  private async getTree(
+    sha: string,
+    truncatedTree: Tree = {},
+    parentPath: string = ""
+  ) {
+    const octokit = new Octokit({
+      auth: await this.getToken(),
+    });
+    const ghRes = await octokit.git.getTree({
+      owner: this.githubRepository.owner,
+      repo: this.githubRepository.repo,
+      tree_sha: sha,
+      recursive: "1",
+    });
+
+    const tree = this.tree2Tree(ghRes.data.tree, truncatedTree, parentPath);
+    if (ghRes.data.truncated) {
+      await this.getTruncatedTree(sha, tree, parentPath);
+    }
+    return tree;
+  }
+
+  private async getTruncatedTree(
+    sha: string,
+    truncatedTree: Tree = {},
+    parentPath: string = ""
+  ) {
+    const octokit = new Octokit({
+      auth: await this.getToken(),
+    });
+    const ghRes = await octokit.git.getTree({
+      owner: this.githubRepository.owner,
+      repo: this.githubRepository.repo,
+      tree_sha: sha,
+    });
+    const tree = ghRes.data.tree;
+
+    for (let elem of tree) {
+      if (!elem.path) continue;
+      if (elem.type == "tree") {
+        const elementPath = path.join(parentPath, elem.path);
+        const paths = elementPath.split("/");
+
+        let current = truncatedTree;
+        for (let i = 0; i < paths.length; i++) {
+          let p = paths[i];
+          if (!current[p]) {
+            if (elem.sha)
+              await this.getTree(elem.sha, truncatedTree, elementPath);
+            break;
+          }
+          current = current[p] as Tree;
+        }
+      }
+    }
+    this.tree2Tree(ghRes.data.tree, truncatedTree, parentPath);
+    return truncatedTree;
+  }
+
+  private tree2Tree(
+    tree: {
+      path?: string;
+      mode?: string;
+      type?: string;
+      sha?: string;
+      size?: number;
+      url?: string;
+    }[],
+    partialTree: Tree = {},
+    parentPath: string = ""
+  ) {
+    for (let elem of tree) {
+      let current = partialTree;
+
+      if (!elem.path) continue;
+
+      const paths = path.join(parentPath, elem.path).split("/");
+
+      // if elem is a folder iterate on all folders if it is a file stop before the filename
+      const end = elem.type == "tree" ? paths.length : paths.length - 1;
+      for (let i = 0; i < end; i++) {
+        let p = paths[i];
+        if (p[0] == "$") {
+          p = "\\" + p;
+        }
+        if (!current[p]) {
+          current[p] = {};
+        }
+        current = current[p] as Tree;
+      }
+
+      // if elem is a file add the file size in the file list
+      if (elem.type == "blob") {
+        let p = paths[end];
+        if (p[0] == "$") {
+          p = "\\" + p;
+        }
+        current[p] = {
+          size: elem.size || 0, // size in bit
+          sha: elem.sha || "",
+        };
+      }
+    }
+    return partialTree;
+  }
+}

src/source/Zip.ts

@@ -0,0 +1,31 @@
+import * as path from "path";
+import AnonymizedFile from "../AnonymizedFile";
+import Repository from "../Repository";
+import storage from "../storage";
+import { SourceBase } from "../types";
+import * as stream from "stream";
+
+export default class Zip implements SourceBase {
+  type = "Zip";
+  repository: Repository;
+  url?: string;
+
+  constructor(data: any, repository: Repository) {
+    this.repository = repository;
+    this.url = data.url;
+  }
+
+  async getFiles() {
+    return storage.listFiles(this.repository.originalCachePath);
+  }
+
+  async getFileContent(file: AnonymizedFile): Promise<stream.Readable> {
+    return storage.read(file.originalCachePath);
+  }
+
+  toJSON(): any {
+    return {
+      type: this.type,
+    };
+  }
+}