From da194d9d713b575a60ee10eebf6ffa0c7a22040d Mon Sep 17 00:00:00 2001 From: tdurieux Date: Wed, 15 Feb 2023 19:23:45 +0100 Subject: [PATCH] fix(#174) improve cli interface to anonimize repositories --- cli.ts | 55 ++++++++++++++++++++++++---------- src/source/GitHubRepository.ts | 26 ++++++++++------ src/storage/FileSystem.ts | 4 +-- 3 files changed, 58 insertions(+), 27 deletions(-) diff --git a/cli.ts b/cli.ts index 6449e0f..eecfce8 100644 --- a/cli.ts +++ b/cli.ts @@ -2,10 +2,10 @@ import { config as dot } from "dotenv"; dot(); +process.env.STORAGE = "filesystem"; import { writeFile } from "fs/promises"; import { join } from "path"; -import { tmpdir } from "os"; import * as gh from "parse-github-url"; import * as inquirer from "inquirer"; @@ -15,6 +15,7 @@ import config from "./config"; import GitHubDownload from "./src/source/GitHubDownload"; import Repository from "./src/Repository"; import AnonymizedRepositoryModel from "./src/database/anonymizedRepositories/anonymizedRepositories.model"; +import { getRepositoryFromGitHub } from "./src/source/GitHubRepository"; function generateRandomFileName(size: number) { const characters = @@ -45,18 +46,47 @@ async function main() { name: "terms", message: `Terms to remove from your repository (separated with comma).`, }, + { + type: "string", + name: "output", + message: `The output folder where to save the zipped repository.`, + default: process.cwd(), + }, ]); - const ghURL = gh(inq.repo) || { owner: "", name: "", branch: "", commit: "" }; + const ghURL = gh(inq.repo) || { + owner: undefined, + name: undefined, + branch: undefined, + commit: undefined, + }; + + if (!ghURL.owner || !ghURL.name) { + throw new Error("Invalid GitHub URL"); + } + + const ghRepo = await getRepositoryFromGitHub({ + accessToken: inq.token, + owner: ghURL.owner, + repo: ghURL.name, + }); + const branches = await ghRepo.branches({ + accessToken: inq.token, + force: true, + }); + const branchToFind = inq.repo.includes(ghURL.branch) + ? ghURL.branch + : ghRepo.model.defaultBranch || "master"; + const branch = branches.find((b) => b.name === branchToFind); const repository = new Repository( new AnonymizedRepositoryModel({ - repoId: "test", + repoId: `${ghURL.name}-${branch?.name}`, source: { type: "GitHubDownload", accessToken: inq.token, - branch: ghURL.branch || "master", - commit: ghURL.branch || "HEAD", + branch: branchToFind, + commit: branch?.commit || "HEAD", repositoryName: `${ghURL.owner}/${ghURL.name}`, }, options: { @@ -72,18 +102,11 @@ async function main() { }) ); - const source = new GitHubDownload( - { - type: "GitHubDownload", - accessToken: inq.token, - repositoryName: inq.repo, - }, - repository + console.info( + `[INFO] Downloading repository: ${repository.model.source.repositoryName} from branch ${repository.model.source.branch} and commit ${repository.model.source.commit}...` ); - - console.info("[INFO] Downloading repository..."); - await source.download(inq.token); - const outputFileName = join(tmpdir(), generateRandomFileName(8) + ".zip"); + await (repository.source as GitHubDownload).download(inq.token); + const outputFileName = join(inq.output, generateRandomFileName(8) + ".zip"); console.info("[INFO] Anonymizing repository and creation zip file..."); await writeFile(outputFileName, repository.zip()); console.log(`Anonymized repository saved at ${outputFileName}`); diff --git a/src/source/GitHubRepository.ts b/src/source/GitHubRepository.ts index 85626fc..3cb0273 100644 --- a/src/source/GitHubRepository.ts +++ b/src/source/GitHubRepository.ts @@ -4,6 +4,7 @@ import { IRepositoryDocument } from "../database/repositories/repositories.types import { Octokit, RestEndpointMethodTypes } from "@octokit/rest"; import RepositoryModel from "../database/repositories/repositories.model"; import AnonymousError from "../AnonymousError"; +import { database, isConnected } from "../database/database"; export class GitHubRepository { private _data: Partial<{ @@ -71,11 +72,13 @@ export class GitHubRepository { }); this._data.branches = branches; - await RepositoryModel.updateOne( - { externalId: this.id }, - { $set: { branches } } - ); - } else { + if (isConnected) { + await RepositoryModel.updateOne( + { externalId: this.id }, + { $set: { branches } } + ); + } + } else if (isConnected) { const q = await RepositoryModel.findOne({ externalId: this.id }).select( "branches" ); @@ -208,9 +211,12 @@ export async function getRepositoryFromGitHub(opt: { repo: opt.repo, }, }); - let model = await RepositoryModel.findOne({ externalId: "gh_" + r.id }); - if (!model) { - model = new RepositoryModel({ externalId: "gh_" + r.id }); + let model = new RepositoryModel({ externalId: "gh_" + r.id }); + if (isConnected) { + const dbModel = await RepositoryModel.findOne({ externalId: "gh_" + r.id }); + if (dbModel) { + model = dbModel; + } } model.name = r.full_name; model.url = r.html_url; @@ -224,6 +230,8 @@ export async function getRepositoryFromGitHub(opt: { }); model.pageSource = ghPageRes.data.source; } - await model.save(); + if (isConnected) { + await model.save(); + } return new GitHubRepository(model); } diff --git a/src/storage/FileSystem.ts b/src/storage/FileSystem.ts index f4f0d58..2c37ac2 100644 --- a/src/storage/FileSystem.ts +++ b/src/storage/FileSystem.ts @@ -108,7 +108,7 @@ export default class FileSystem implements StorageBase { return pipe( data, Extract({ - path: join(join(config.FOLDER, p)), + path: join(config.FOLDER, p), decodeString: (buf) => { const name = buf.toString(); const newName = name.substr(name.indexOf("/") + 1); @@ -124,7 +124,7 @@ export default class FileSystem implements StorageBase { dir: string, opt?: { format?: "zip" | "tar"; - fileTransformer? (path: string): Transform; + fileTransformer?: (path: string) => Transform; } ) { const archive = archiver(opt?.format || "zip", {});