v2 wip

routes/connection.js

@@ -0,0 +1,102 @@
+const redis = require("redis");
+
+const passport = require("passport");
+const session = require("express-session");
+const redisStore = require("connect-redis")(session);
+const GitHubStrategy = require("passport-github2").Strategy;
+
+const express = require("express");
+
+const router = express.Router();
+
+const db = require("../utils/database");
+const config = require("../config");
+
+function ensureAuthenticated(req, res, next) {
+  if (req.isAuthenticated()) {
+    return next();
+  }
+  res.status(401).json({ error: "not_connected" });
+}
+
+passport.serializeUser(function(user, done) {
+  delete user.profile._json;
+  done(null, user);
+});
+
+passport.deserializeUser(function(obj, done) {
+  done(null, obj);
+});
+
+passport.use(
+  new GitHubStrategy(
+    {
+      clientID: config.CLIENT_ID,
+      clientSecret: config.CLIENT_SECRET,
+      callbackURL: config.AUTH_CALLBACK,
+    },
+    async (accessToken, refreshToken, profile, done) => {
+      try {
+        await db
+          .get()
+          .collection("users")
+          .updateOne(
+            { username: profile.username },
+            {
+              $set: {
+                username: profile.username,
+                profile,
+                accessToken,
+                refreshToken,
+              },
+            },
+            { upsert: true }
+          );
+      } catch (error) {
+        console.error(error);
+      } finally {
+        done(null, {
+          username: profile.username,
+          accessToken,
+          refreshToken,
+          profile,
+        });
+      }
+    }
+  )
+);
+
+const rediscli = redis.createClient({
+  host: "redis",
+  ttl: 260,
+});
+
+const appSession = session({
+  secret: "keyboard cat",
+  store: new redisStore({
+    client: rediscli,
+  }),
+  saveUninitialized: false,
+  resave: false,
+});
+
+router.get(
+  "/login",
+  passport.authenticate("github", { scope: ["repo"] }), // Note the scope here
+  function(req, res) {
+    res.redirect("/");
+  }
+);
+
+router.get(
+  "/auth",
+  passport.authenticate("github", { failureRedirect: "/" }),
+  function(req, res) {
+    res.redirect("/");
+  }
+);
+
+module.exports.ensureAuthenticated = ensureAuthenticated;
+module.exports.passport = passport;
+module.exports.session = appSession;
+module.exports.router = router;

routes/file.js

@@ -0,0 +1,161 @@
+const express = require("express");
+const path = require("path");
+
+const db = require("../utils/database");
+const fileUtils = require("../utils/file");
+const repoUtils = require("../utils/repository");
+const githubUtils = require("../utils/github");
+
+const router = express.Router();
+
+async function anonymizeRepository(options) {
+  let repoConfig = options.repoConfig;
+  if (!repoConfig) {
+    repoConfig = await repoUtils.getConfig(options.repoId);
+  }
+
+  if (repoConfig == null) {
+    throw "repo_not_found";
+  }
+
+  if (repoConfig.options.expirationMode != "never") {
+    if (repoConfig.options.expirationDate <= new Date()) {
+      console.log("The repository is expired");
+      await repoUtils.updateStatus(repoConfig, "expired");
+      await repoUtils.removeRepository(repoConfig);
+      return;
+    }
+  }
+
+  const lastView = repoConfig.lastView;
+
+  const yesterday = new Date();
+  yesterday.setDate(yesterday.getDate() - 1);
+
+  if (repoConfig.options.update && lastView < yesterday) {
+    console.log("check for updates in the repository.");
+    try {
+    } catch (error) {
+      console.error("Error while updating the repository.");
+      console.error(error);
+    }
+    await repoUtils.updateAnonimizedRepository(repoConfig);
+  }
+  await githubUtils.downloadRepoAndAnonymize(repoConfig);
+}
+
+router.get("/:repoId/files", async (req, res) => {
+  const repoConfig = await repoUtils.getConfig(req.params.repoId);
+  if (repoConfig == null) {
+    return res.status(500).json({ error: "repo_not_found" });
+  }
+
+  if (repoConfig.status != "ready") {
+    return res.status(500).json({ error: "repo_not_ready" });
+  }
+
+  try {
+    const files = await fileUtils.getFileList({ repoConfig });
+    return res.json(files);
+  } catch (error) {
+    return res.status(500).json({ error });
+  }
+});
+
+router.get("/:repoId/stats", async (req, res) => {
+  const repoConfig = await repoUtils.getConfig(req.params.repoId);
+
+  if (repoConfig == null) {
+    return res.status(500).json({ error: "repo_not_found" });
+  }
+  if (repoConfig.status != "ready") {
+    return res.status(500).json({ error: "repo_not_ready" });
+  }
+
+  if (repoConfig.options.mode == "stream") {
+    return res.status(500).json({ error: "stream_not_supported" });
+  }
+
+  try {
+    const stats = await fileUtils.getStats({ repoConfig });
+    return res.json(stats.languages);
+  } catch (error) {
+    console.log(error);
+    return res.status(500).json({ error });
+  }
+});
+
+router.get("/:repoId/options", async (req, res) => {
+  const repoConfig = await repoUtils.getConfig(req.params.repoId);
+  if (repoConfig == null) {
+    return res.status(500).json({ error: "repo_not_found" });
+  }
+  try {
+    try {
+      await anonymizeRepository({ repoConfig });
+    } catch (error) {
+      console.log("Error during the anonymization of the repository");
+      console.log(error);
+    }
+    if (repoConfig.status == "removed") {
+      throw "repository_expired";
+    }
+    if (repoConfig.status == "expired") {
+      if (repoConfig.options.expirationMode == "redirect") {
+        repoConfig.options.url = "https://github.com/" + repoConfig.fullName;
+      } else {
+        throw "repository_expired";
+      }
+    } else if (repoConfig.status != "ready") {
+      throw "repository_not_ready";
+    }
+
+    return res.json(repoConfig.options);
+  } catch (error) {
+    console.log(error);
+    return res.status(500).json({ error });
+  }
+});
+
+router.get("/:repoId/file/:path*", async (req, res) => {
+  const repoConfig = await repoUtils.getConfig(req.params.repoId);
+
+  if (repoConfig == null) {
+    return res.status(500).json({ error: "repo_not_found" });
+  }
+  if (repoConfig.status != "ready") {
+    return res.status(500).json({ error: "repo_not_ready" });
+  }
+
+  let requestPath = req.params.path;
+  if (req.params[0]) {
+    requestPath += req.params[0];
+  }
+
+  try {
+    const isValid = await fileUtils.isFilePathValid({
+      repoConfig,
+      path: requestPath,
+    });
+    if (isValid) {
+      await db
+        .get("anonymized_repositories")
+        .updateOne(
+          { repoId: repoConfig.repoId },
+          { $set: { lastView: new Date() }, $inc: { pageView: 1 } }
+        );
+      const ppath = path.join(
+        repoUtils.getAnonymizedPath(repoConfig.repoId),
+        requestPath
+      );
+      return res.sendFile(ppath, { dotfiles: "allow" });
+    } else {
+      return res.status(404).send({ error: "file_not_found" });
+    }
+  } catch (error) {
+    console.error(error);
+    return res.status(500).send({ error });
+  }
+});
+
+module.exports = router;

routes/repositoy.js

@@ -0,0 +1,334 @@
+const ofs = require("fs");
+const fs = require("fs").promises;
+
+const express = require("express");
+const gh = require("parse-github-url");
+const arrayEquals = require("array-equal");
+
+const connection = require("./connection");
+const githubUtils = require("../utils/github");
+const db = require("../utils/database");
+const repoUtils = require("../utils/repository");
+const config = require("../config");
+
+const router = express.Router();
+
+// user needs to be connected for all user API
+router.use(connection.ensureAuthenticated);
+
+router.get("/:repoId/", async (req, res) => {
+  try {
+    const repository = await repoUtils.getAnonymizedRepoDetails(
+      req.params.repoId,
+      req.user
+    );
+    if (repository) {
+      return res.json(repository);
+    }
+    res.status(404).send("repo_not_found");
+  } catch (error) {
+    res.status(500).send(error);
+  }
+});
+
+// update a repository
+router.post("/:repoId/", async (req, res) => {
+  const repoUpdate = req.body;
+
+  let repoConfig = await repoUtils.getConfig(req.params.repoId);
+  if (repoConfig == null) {
+    return res.status(500).json({ error: "repo_not_found" });
+  }
+  if (repoConfig.owner != req.user.username) {
+    return res.status(401).json({ error: "not_authorized" });
+  }
+  if (!repoUpdate.branch) {
+    return res.status(500).json({ error: "branch_not_specified" });
+  }
+  if (!repoUpdate.options) {
+    return res.status(500).json({ error: "options_not_provided" });
+  }
+  if (!Array.isArray(repoUpdate.terms)) {
+    return res.status(500).send({ error: "invalid_terms_format" });
+  }
+
+  const details = await repoUtils.getRepoDetails({
+    repoConfig,
+    force: true,
+    token: req.user.accessToken,
+  });
+  if (repoConfig.options.mode == "download") {
+    // details.size is in kilobytes
+    if (details.size > config.MAX_REPO_SIZE) {
+      return res.status(500).send({ error: "invalid_mode" });
+    }
+  }
+
+  try {
+    if (repoUpdate.commit != repoConfig.commit) {
+      repoUpdate.anonymizeDate = new Date();
+      await repoUtils.removeRepository(repoConfig);
+    }
+    if (
+      !arrayEquals(repoUpdate.terms, repoConfig.terms) ||
+      repoUpdate.options.link != repoConfig.options.link ||
+      repoUpdate.options.image != repoConfig.options.image
+    ) {
+      repoUpdate.anonymizeDate = new Date();
+      if (ofs.existsSync(repoUtils.getAnonymizedPath(repoConfig.repoId))) {
+        await fs.rm(repoUtils.getAnonymizedPath(repoConfig.repoId), {
+          recursive: true,
+          force: true,
+        });
+      }
+    }
+    const data = {
+      terms: repoUpdate.terms,
+      branch: repoUpdate.branch,
+      commit: repoUpdate.commit,
+      options: {
+        expirationMode: repoUpdate.options.expirationMode,
+        expirationDate: repoUpdate.options.expirationDate,
+        update: repoUpdate.options.update,
+        image: repoUpdate.options.image,
+        pdf: repoUpdate.options.pdf,
+        notebook: repoUpdate.options.notebook,
+        loc: repoUpdate.options.loc,
+        link: repoUpdate.options.link,
+        mode: repoUpdate.options.mode,
+        page: repoUpdate.options.page,
+      },
+    };
+    if (repoConfig.options.page) {
+      data.options.pageSource = details.pageSource;
+    }
+    await db.get("anonymized_repositories").updateOne(
+      {
+        repoId: repoConfig.repoId,
+      },
+      {
+        $set: data,
+      }
+    );
+
+    repoConfig = await repoUtils.getConfig(repoUpdate.repoId);
+
+    await githubUtils.downloadRepoAndAnonymize(repoConfig);
+    await repoUtils.updateStatus(repoConfig, "ready");
+
+    return res.send("ok");
+  } catch (error) {
+    console.error(error);
+    return res.status(500).json({ error });
+  }
+});
+
+// refresh a repository
+router.post("/:repoId/refresh", async (req, res) => {
+  const repoConfig = await repoUtils.getConfig(req.params.repoId);
+  if (repoConfig == null) {
+    return res.status(500).json({ error: "repo_not_found" });
+  }
+  if (repoConfig.owner != req.user.username) {
+    return res.status(401).json({ error: "not_authorized" });
+  }
+  try {
+    await repoUtils.updateAnonimizedRepository(repoConfig);
+    return res.send("ok");
+  } catch (error) {
+    return res.status(500).json({ error });
+  }
+});
+
+// delete a repository
+router.delete("/:repoId/", async (req, res) => {
+  const repoConfig = await repoUtils.getConfig(req.params.repoId);
+  if (repoConfig == null) {
+    return res.status(500).json({ error: "repo_not_found" });
+  }
+  if (repoConfig.owner != req.user.username) {
+    return res.status(401).json({ error: "not_authorized" });
+  }
+  try {
+    await repoUtils.updateStatus(repoConfig, "removed");
+    await repoUtils.removeRepository(repoConfig);
+    return res.send("ok");
+  } catch (error) {
+    return res.status(500).json({ error });
+  }
+});
+
+// claim a repository
+router.post("/claim", async (req, res) => {
+  if (!req.body.repoId) {
+    return res.status(500).json({ error: "repoId_not_defined" });
+  }
+  if (!req.body.repoUrl) {
+    return res.status(500).json({ error: "repoUrl_not_defined" });
+  }
+
+  const repoConfig = await repoUtils.getConfig(req.body.repoId);
+  if (repoConfig == null) {
+    return res.status(500).json({ error: "repo_not_found" });
+  }
+
+  const repo = gh(req.body.repoUrl);
+  if (repoConfig.fullName != repo.repository) {
+    return res.status(500).json({ error: "repo_not_found" });
+  }
+
+  console.log(`${req.user.username} claims ${repoConfig.fullName}.`);
+  try {
+    await db
+      .get("anonymized_repositories")
+      .updateOne(
+        { repoId: repoConfig.repoId },
+        { $set: { owner: req.user.username } }
+      );
+    return res.send("Ok");
+  } catch (error) {
+    return res.status(500).json({ error });
+  }
+});
+
+router.get("/:owner/:repo/", async (req, res) => {
+  try {
+    const repository = await repoUtils.getRepoDetails({
+      owner: req.params.owner,
+      repo: req.params.repo,
+      token: req.user.accessToken,
+      force: req.query.force === "1",
+    });
+    if (repository) {
+      return res.json(repository);
+    }
+    res.status(404).send("repo_not_found");
+  } catch (error) {
+    res.status(500).send(error);
+  }
+});
+
+router.get("/:owner/:repo/branches", async (req, res) => {
+  try {
+    const repository = await repoUtils.getRepoBranches({
+      owner: req.params.owner,
+      repo: req.params.repo,
+      token: req.user.accessToken,
+      force: req.query.force === "1",
+    });
+    if (repository) {
+      return res.json(repository);
+    }
+    res.status(404).send("repo_not_found");
+  } catch (error) {
+    res.status(500).send(error);
+  }
+});
+
+router.get("/:owner/:repo/readme", async (req, res) => {
+  try {
+    const readme = await repoUtils.getRepoReadme({
+      owner: req.params.owner,
+      repo: req.params.repo,
+      token: req.user.accessToken,
+      force: req.query.force === "1",
+    });
+    if (readme) {
+      return res.send(readme);
+    }
+    res.status(404).send("repo_not_found");
+  } catch (error) {
+    res.status(500).send(error);
+  }
+});
+
+router.post("/", async (req, res) => {
+  const repoConfig = req.body;
+
+  const repository = await repoUtils.getConfig(repoConfig.repoId);
+  const cacheExist = ofs.existsSync(
+    repoUtils.getOriginalPath(repoConfig.repoId)
+  );
+  if (repository && cacheExist) {
+    return res.status(500).send({ error: "repoId_already_used" });
+  }
+  var validCharacters = /^[0-9a-zA-Z\-\_]+$/;
+  if (
+    !repoConfig.repoId.match(validCharacters) ||
+    repoConfig.repoId.length < 3
+  ) {
+    return res.status(500).send({ error: "invalid_repoId" });
+  }
+  if (!repoConfig.branch) {
+    return res.status(500).json({ error: "branch_not_specified" });
+  }
+  if (!repoConfig.options) {
+    return res.status(500).json({ error: "options_not_provided" });
+  }
+  if (!Array.isArray(repoConfig.terms)) {
+    return res.status(500).send({ error: "invalid_terms_format" });
+  }
+
+  await repoUtils.getRepoBranches({ repoConfig, token: req.user.accessToken });
+  const details = await repoUtils.getRepoDetails({
+    repoConfig,
+    token: req.user.accessToken,
+  });
+  if (repoConfig.options.mode == "download") {
+    // details.size is in kilobytes
+    if (details.size > config.MAX_REPO_SIZE) {
+      return res.status(500).send({ error: "invalid_mode" });
+    }
+  }
+
+  const data = {
+    repoId: repoConfig.repoId,
+    fullName: repoConfig.fullName,
+    status: "preparing",
+    terms: repoConfig.terms,
+    owner: req.user.profile.username,
+    token: req.user.accessToken,
+    branch: repoConfig.branch,
+    commit: repoConfig.commit
+      ? repoConfig.commit
+      : details.branches[repoConfig.branch].commit.sha,
+    anonymizeDate: new Date(),
+    options: {
+      expirationMode: repoConfig.options.expirationMode,
+      expirationDate: repoConfig.options.expirationDate,
+      update: repoConfig.options.update,
+      image: repoConfig.options.image,
+      pdf: repoConfig.options.pdf,
+      notebook: repoConfig.options.notebook,
+      loc: repoConfig.options.loc,
+      link: repoConfig.options.link,
+      mode: repoConfig.options.mode,
+      page: repoConfig.options.page,
+    },
+  };
+  if (repoConfig.options.page) {
+    data.options.pageSource = details.pageSource;
+  }
+  await db.get("anonymized_repositories").updateOne(
+    {
+      repoId: data.repoId,
+    },
+    {
+      $set: data,
+    },
+    { upsert: true }
+  );
+  try {
+    await githubUtils.downloadRepoAndAnonymize(data);
+    await repoUtils.updateStatus(repoConfig, "ready");
+  } catch (error) {
+    console.error(error);
+    await repoUtils.updateStatus(repoConfig, "error");
+    return res
+      .status(500)
+      .json({ error: "unable_to_access", message: error.message });
+  }
+  res.send("ok");
+});
+
+module.exports = router;

routes/user.js

@@ -0,0 +1,96 @@
+const express = require("express");
+const { Octokit } = require("@octokit/rest");
+
+const connection = require("./connection");
+const db = require("../utils/database");
+const repoUtils = require("../utils/repository");
+
+const router = express.Router();
+
+// user needs to be connected for all user API
+router.use(connection.ensureAuthenticated);
+
+router.get("/logout", async (req, res) => {
+  req.logout();
+  res.redirect("/");
+});
+
+router.get("/", async (req, res) => {
+  const photo = req.user.profile.photos.length
+    ? req.user.profile.photos[0].value
+    : null;
+  res.json({ username: req.user.profile.username, photo });
+});
+
+router.get("/anonymized_repositories", async (req, res) => {
+  const repos = await db
+    .get("anonymized_repositories")
+    .find(
+      {
+        owner: req.user.username,
+      },
+      { projection: { token: 0, files: 0, originalFiles: 0 } }
+    )
+    .toArray();
+  for (let repo of repos) {
+    if (repo.options.expirationDate) {
+      repo.options.expirationDate = new Date(repo.options.expirationDate);
+    }
+    if (
+      repo.options.expirationMode != "never" &&
+      repo.options.expirationDate != null &&
+      repo.options.expirationDate < new Date()
+    ) {
+      console.log(
+        repo.options.expirationDate,
+        repo.options.expirationDate < new Date()
+      );
+      await repoUtils.updateStatus({ repoId: repo.repoId }, "expired");
+      repo.status = "expired";
+    } else {
+      await repoUtils.updateStatus({ repoId: repo.repoId }, "ready");
+      repo.status = "ready";
+    }
+  }
+  res.json(repos);
+});
+
+router.get("/all_repositories", async (req, res) => {
+  const user = await db
+    .get()
+    .collection("users")
+    .findOne(
+      { username: req.user.username },
+      { projection: { repositories: 1 } }
+    );
+  if (!user) {
+    res.status(401).send("User not found");
+  }
+  if (user.repositories && req.query.force !== "1") {
+    return res.json(user.repositories);
+  } else {
+    const octokit = new Octokit({ auth: req.user.accessToken });
+    const repositories = await octokit.paginate(
+      octokit.repos.listForAuthenticatedUser,
+      {
+        visibility: "all",
+        sort: "pushed",
+        per_page: 100,
+      }
+    );
+    try {
+      await db
+        .get()
+        .collection("users")
+        .updateOne(
+          { username: req.user.profile.username },
+          { $set: { repositories } }
+        );
+      res.json(repositories);
+    } catch (error) {
+      res.status(500).send(error);
+    }
+  }
+});
+
+module.exports = router;

routes/webview.js

@@ -0,0 +1,70 @@
+const express = require("express");
+const path = require("path");
+
+const fileUtils = require("../utils/file");
+const repoUtils = require("../utils/repository");
+
+const router = express.Router();
+
+async function webView(req, res) {
+  try {
+    const repoId = req.params.repoId;
+    const repoConfig = await repoUtils.getConfig(repoId);
+
+    if (!repoConfig.options.page) {
+      throw "page_not_activated";
+    }
+    if (!repoConfig.options.pageSource) {
+      throw "page_not_activated";
+    }
+
+    if (repoConfig.options.pageSource.branch != repoConfig.branch) {
+      throw "page_not_supported_on_different_branch";
+    }
+
+    let requestPath = req.path.substring(
+      req.path.indexOf(repoId) + repoId.length
+    );
+
+    if (requestPath[requestPath.length - 1] == "/") {
+      requestPath = path.join(requestPath, "index.html");
+    }
+    // TODO: handle website that are not in the docs folder (master, docs, gh-pages)
+    requestPath = path.join(repoConfig.options.pageSource.path, requestPath);
+
+    if (await fileUtils.isFilePathValid({ repoConfig, path: requestPath })) {
+      const ppath = path.join(
+        repoUtils.getAnonymizedPath(repoConfig.repoId),
+        requestPath
+      );
+      return res.sendFile(ppath, { dotfiles: "allow" }, (err) => {
+        if (err) {
+          if (err.path) {
+            const newPath = path.join(
+              req.path,
+              err.path.replace(
+                path.join(
+                  repoUtils.getAnonymizedPath(repoConfig.repoId),
+                  "docs"
+                ),
+                ""
+              )
+            );
+            if (newPath != req.path) {
+              return res.redirect(newPath);
+            }
+          }
+        }
+        console.log(err);
+      });
+    }
+  } catch (error) {
+    console.error(error);
+    return res.status(500).send({ error });
+  }
+  return res.status(404).send("File_not_found");
+}
+
+router.get("/:repoId/*", webView);
+
+module.exports = router;