anonymous_github/import.js

const fs = require("fs").promises;
const ofs = require("fs");
const path = require("path");
const gh = require("parse-github-url");
const { Octokit } = require("@octokit/rest");

const config = require("./config");
const db = require("./utils/database");
const repoUtils = require("./utils/repository");
const fileUtils = require("./utils/file");
const githubUtils = require("./utils/github");

// const ROOT = "./repositories";
const ROOT = "./repo";
(async () => {
  await db.connect();
  const repositories = await fs.readdir(ROOT);
  let index = 0;
  for (let repo of repositories) {
    // for (let repo of ["14bfc5c6-b794-487e-a58a-c54103a93c7b"]) {
    console.log("Import ", index++, "/", repositories.length, " ", repo);
    try {
      const conf = await repoUtils.getConfig(repo);
      if (conf) {
        continue;
      }
      // const repoPath = path.join("./repositories", repo);
      const repoPath = path.join(ROOT, repo);
      const configPath = path.join(repoPath, "config.json");
      if (!ofs.existsSync(configPath)) {
        continue;
      }
      const repoConfig = JSON.parse(await fs.readFile(configPath));
      const r = gh(repoConfig.repository);
      if (r == null) {
        console.log(`${repoConfig.repository} is not a valid github url.`);
        continue;
      }
      const fullName = `${r.owner}/${r.name}`;

      // const octokit = new Octokit({ auth: config.GITHUB_TOKEN });
      // try {
      //   await octokit.apps.checkToken({
      //     client_id: config.CLIENT_ID,
      //     access_token: repoConfig.token,
      //   });
      // } catch (error) {
      //   delete repoConfig.token;
      //   continue
      // }
      let token = repoConfig.token;

      if (!token) {
        token = config.GITHUB_TOKEN;
      }

      const branches = await repoUtils.getRepoBranches({
        fullName,
        token,
      });
      const details = await repoUtils.getRepoDetails({
        fullName,
        token,
      });
      let branch = details.default_branch;
      if (r.branch && branches[r.branch]) {
        branch = r.branch;
      }
      if (!branches[branch]) {
        console.log(branch, details.default_branch, branches);
      }
      let commit = branches[branch].commit.sha;
      const anonymizeDate = new Date();

      let mode = "stream";
      // if (details.size < 1024) {
      //   mode = "download";
      // }

      let expirationDate = null;
      if (repoConfig.expiration_date) {
        expirationDate = new Date(repoConfig.expiration_date["$date"]);
      }
      const expirationMode = repoConfig.expiration
        ? repoConfig.expiration
        : "never";

      const repoConfiguration = {
        repoId: repo,
        fullName,
        // owner: "tdurieux",
        owner: r.owner,
        terms: repoConfig.terms,
        repository: repoConfig.repository,
        token: repoConfig.token,
        branch,
        commit,
        anonymizeDate,
        options: {
          image: false,
          mode,
          expirationMode,
          expirationDate,
          update: true,
          page: details.has_pages,
          pdf: false,
          notebook: true,
          loc: false,
          link: true,
        },
      };
      await db.get("anonymized_repositories").updateOne(
        {
          repoId: repo,
        },
        {
          $set: repoConfiguration,
        },
        { upsert: true }
      );
      if (ofs.existsSync(repoUtils.getOriginalPath(repo))) {
        await fs.rm(repoUtils.getOriginalPath(repo), {
          recursive: true,
          force: true,
        });
      }
      if (ofs.existsSync(repoUtils.getAnonymizedPath(repo))) {
        await fs.rm(repoUtils.getAnonymizedPath(repo), {
          recursive: true,
          force: true,
        });
      }
      // await githubUtils.downloadRepoAndAnonymize(repoConfiguration);
      // await fileUtils.getFileList({ repoConfig: repoConfiguration });
      await repoUtils.updateStatus(repoConfiguration, "ready");
      console.log(
        expirationDate,
        expirationDate != null && expirationDate < new Date(),
        expirationDate < new Date()
      );
      if (
        expirationMode != "never" &&
        expirationDate != null &&
        expirationDate < new Date()
      ) {
        await repoUtils.updateStatus(repoConfiguration, "expired");
      }
    } catch (error) {
      console.error(error);
    }
  }
  await db.close();
})();