feat: handle repository download and remove using a job queue

src/routes/repository-private.ts

@@ -12,6 +12,7 @@ import { IAnonymizedRepositoryDocument } from "../database/anonymizedRepositorie
 import Repository from "../Repository";
 import ConferenceModel from "../database/conference/conferences.model";
 import AnonymousError from "../AnonymousError";
+import { downloadQueue, removeQueue } from "../queue";
 
 const router = express.Router();
 
@@ -65,12 +66,15 @@ router.post(
       const repo = await getRepo(req, res, { nocheck: true });
       if (!repo) return;
 
+      if (repo.status == "preparing" || repo.status == "removing") return;
+
       const user = await getUser(req);
       if (repo.owner.id != user.id) {
         return res.status(401).json({ error: "not_authorized" });
       }
-      await repo.anonymize();
-      res.end("ok");
+      await repo.updateStatus("preparing");
+      await downloadQueue.add(repo, {jobId: repo.repoId});
+      res.json({ status: repo.status });
     } catch (error) {
       handleError(error, res);
     }
@@ -83,14 +87,16 @@ router.delete(
   async (req: express.Request, res: express.Response) => {
     const repo = await getRepo(req, res, { nocheck: true });
     if (!repo) return;
+    // if (repo.status == "removing") return res.json({ status: repo.status });
     try {
+      if (repo.status == "removed") throw new AnonymousError("is_removed");
       const user = await getUser(req);
       if (repo.owner.id != user.id) {
         return res.status(401).json({ error: "not_authorized" });
       }
-      await repo.remove();
-      console.log(`${req.params.repoId} is removed`);
-      return res.json("ok");
+      await repo.updateStatus("removing");
+      await removeQueue.add(repo, {jobId: repo.repoId});
+      return res.json({ status: repo.status });
     } catch (error) {
       handleError(error, res);
     }
@@ -305,8 +311,8 @@ router.post(
       }
       repo.model.conference = repoUpdate.conference;
       await repo.updateStatus("preparing");
-      res.send("ok");
-      new Repository(repo.model).anonymize();
+      res.json({ status: repo.status });
+      await downloadQueue.add(repo, {jobId: repo.repoId});
     } catch (error) {
       return handleError(error, res);
     }
@@ -369,11 +375,14 @@ router.post("/", async (req: express.Request, res: express.Response) => {
       }
     }
 
-    res.send("ok");
-    new Repository(repo).anonymize();
+    res.send({ status: repo.status });
+    downloadQueue.add(new Repository(repo), {jobId: repo.repoId});
   } catch (error) {
     if (error.message?.indexOf(" duplicate key") > -1) {
-      return handleError(new AnonymousError("repoId_already_used", repoUpdate.repoId), res);
+      return handleError(
+        new AnonymousError("repoId_already_used", repoUpdate.repoId),
+        res
+      );
     }
     return handleError(error, res);
   }

src/routes/repository-public.ts

@@ -3,7 +3,6 @@ import * as express from "express";
 import * as stream from "stream";
 import config from "../../config";
 
-
 import { getRepo, handleError } from "./route-utils";
 
 const router = express.Router();
@@ -23,7 +22,7 @@ router.get(
 
       // cache the file for 6 hours
       res.header("Cache-Control", "max-age=21600000");
-      await pipeline(repo.zip(), res)
+      await pipeline(repo.zip(), res);
     } catch (error) {
       handleError(error, res);
     }
@@ -60,7 +59,6 @@ router.get(
       ) {
         redirectURL = repo.source.url;
       } else {
-        repo.check();
         await repo.updateIfNeeded();
       }