v2 wip

import.js

@@ -0,0 +1,153 @@
+const fs = require("fs").promises;
+const ofs = require("fs");
+const path = require("path");
+const gh = require("parse-github-url");
+const { Octokit } = require("@octokit/rest");
+
+const config = require("./config");
+const db = require("./utils/database");
+const repoUtils = require("./utils/repository");
+const fileUtils = require("./utils/file");
+const githubUtils = require("./utils/github");
+
+// const ROOT = "./repositories";
+const ROOT = "./repo";
+(async () => {
+  await db.connect();
+  const repositories = await fs.readdir(ROOT);
+  let index = 0;
+  for (let repo of repositories) {
+    // for (let repo of ["14bfc5c6-b794-487e-a58a-c54103a93c7b"]) {
+    console.log("Import ", index++, "/", repositories.length, " ", repo);
+    try {
+      const conf = await repoUtils.getConfig(repo);
+      if (conf) {
+        continue;
+      }
+      // const repoPath = path.join("./repositories", repo);
+      const repoPath = path.join(ROOT, repo);
+      const configPath = path.join(repoPath, "config.json");
+      if (!ofs.existsSync(configPath)) {
+        continue;
+      }
+      const repoConfig = JSON.parse(await fs.readFile(configPath));
+      const r = gh(repoConfig.repository);
+      if (r == null) {
+        console.log(`${repoConfig.repository} is not a valid github url.`);
+        continue;
+      }
+      const fullName = `${r.owner}/${r.name}`;
+
+      // const octokit = new Octokit({ auth: config.GITHUB_TOKEN });
+      // try {
+      //   await octokit.apps.checkToken({
+      //     client_id: config.CLIENT_ID,
+      //     access_token: repoConfig.token,
+      //   });
+      // } catch (error) {
+      //   delete repoConfig.token;
+      //   continue
+      // }
+      let token = repoConfig.token;
+
+      if (!token) {
+        token = config.GITHUB_TOKEN;
+      }
+
+      const branches = await repoUtils.getRepoBranches({
+        fullName,
+        token,
+      });
+      const details = await repoUtils.getRepoDetails({
+        fullName,
+        token,
+      });
+      let branch = details.default_branch;
+      if (r.branch && branches[r.branch]) {
+        branch = r.branch;
+      }
+      if (!branches[branch]) {
+        console.log(branch, details.default_branch, branches);
+      }
+      let commit = branches[branch].commit.sha;
+      const anonymizeDate = new Date();
+
+      let mode = "stream";
+      // if (details.size < 1024) {
+      //   mode = "download";
+      // }
+
+      let expirationDate = null;
+      if (repoConfig.expiration_date) {
+        expirationDate = new Date(repoConfig.expiration_date["$date"]);
+      }
+      const expirationMode = repoConfig.expiration
+        ? repoConfig.expiration
+        : "never";
+
+      const repoConfiguration = {
+        repoId: repo,
+        fullName,
+        // owner: "tdurieux",
+        owner: r.owner,
+        terms: repoConfig.terms,
+        repository: repoConfig.repository,
+        token: repoConfig.token,
+        branch,
+        commit,
+        anonymizeDate,
+        options: {
+          image: false,
+          mode,
+          expirationMode,
+          expirationDate,
+          update: true,
+          page: details.has_pages,
+          pdf: false,
+          notebook: true,
+          loc: false,
+          link: true,
+        },
+      };
+      await db.get("anonymized_repositories").updateOne(
+        {
+          repoId: repo,
+        },
+        {
+          $set: repoConfiguration,
+        },
+        { upsert: true }
+      );
+      if (ofs.existsSync(repoUtils.getOriginalPath(repo))) {
+        await fs.rm(repoUtils.getOriginalPath(repo), {
+          recursive: true,
+          force: true,
+        });
+      }
+      if (ofs.existsSync(repoUtils.getAnonymizedPath(repo))) {
+        await fs.rm(repoUtils.getAnonymizedPath(repo), {
+          recursive: true,
+          force: true,
+        });
+      }
+      // await githubUtils.downloadRepoAndAnonymize(repoConfiguration);
+      // await fileUtils.getFileList({ repoConfig: repoConfiguration });
+      await repoUtils.updateStatus(repoConfiguration, "ready");
+      console.log(
+        expirationDate,
+        expirationDate != null && expirationDate < new Date(),
+        expirationDate < new Date()
+      );
+      if (
+        expirationMode != "never" &&
+        expirationDate != null &&
+        expirationDate < new Date()
+      ) {
+        await repoUtils.updateStatus(repoConfiguration, "expired");
+      }
+    } catch (error) {
+      console.error(error);
+    }
+  }
+  await db.close();
+})();