feat: admin to remove repo cache

public/partials/admin/queues.htm

@@ -148,5 +148,79 @@
         There is no job to display.
       </li>
     </ul>
+    <h1>Remove Cache</h1>
+    <ul class="p-0 m-0 w-100">
+      <li
+        class="col-12 d-flex px-0 py-3 border-bottom color-border-secondary"
+        ng-repeat="job in removeCaches as filteredRemoveCache"
+      >
+        <div class="w-100">
+          <div class="">
+            <h3>
+              <a target="__blank" ng-href="/r/{{job.id}}" ng-bind="job.id"></a>
+              <span class="badge" ng-bind="job.progress.status | title"></span>
+            </h3>
+          </div>
+          <div class="color-text-secondary mb-1">
+            <span ng-if="job.timestamp">
+              Created on:
+              <span ng-bind="job.timestamp | humanTime"></span>
+            </span>
+            <span ng-if="job.finishedOn">
+              Finished on:
+              <span ng-bind="job.finishedOn | humanTime"></span>
+            </span>
+            <span ng-if="job.processedOn">
+              Processed on:
+              <span ng-bind="job.processedOn | humanTime"></span>
+            </span>
+          </div>
+          <div>
+            <pre
+              ng-repeat="stack in job.stacktrace track by $index"
+            ><code ng-bind="stack"></code></pre>
+          </div>
+        </div>
+        <div class="d-flex">
+          <div class="dropdown">
+            <button
+              class="btn black_border dropdown-toggle btn-sm"
+              type="button"
+              id="dropdownMenuButton"
+              data-toggle="dropdown"
+              aria-haspopup="true"
+              aria-expanded="false"
+            >
+              Actions
+            </button>
+            <div class="dropdown-menu" aria-labelledby="dropdownMenuButton">
+              <a
+                class="dropdown-item"
+                href="#"
+                ng-click="removeJob('remove', job)"
+              >
+                <i class="fas fa-trash-alt"></i> Remove
+              </a>
+              <a
+                class="dropdown-item"
+                href="#"
+                ng-click="retryJob('remove', job)"
+              >
+                <i class="fas fa-sync"></i> Retry
+              </a>
+              <a class="dropdown-item" href="/anonymize/{{job.id}}">
+                <i class="far fa-edit" aria-hidden="true"></i> Edit
+              </a>
+            </div>
+          </div>
+        </div>
+      </li>
+      <li
+        class="col-12 d-flex px-0 py-3 border-bottom color-border-secondary"
+        ng-if="filteredRemoveCache.length == 0"
+      >
+        There is no job to display.
+      </li>
+    </ul>
   </div>
 </div>

public/partials/admin/repositories.htm

@@ -313,6 +313,9 @@
               Actions
             </button>
             <div class="dropdown-menu" aria-labelledby="dropdownMenuButton">
+              <a class="dropdown-item" href="#" ng-click="removeCache(repo)">
+                <i class="fas fa-trash-alt"></i> Remove Cache
+              </a>
               <a class="dropdown-item" href="/anonymize/{{repo.repoId}}">
                 <i class="far fa-edit" aria-hidden="true"></i> Edit
               </a>

public/script/admin.js

@@ -29,6 +29,16 @@ angular
         preparing: true,
       };
 
+      $scope.removeCache = (repo) => {
+        $http.delete("/api/admin/repos/" + repo.repoId).then(
+          (res) => {
+            $scope.$apply();
+          },
+          (err) => {
+            console.error(err);
+          }
+        );
+      };
       function getRepositories() {
         $http.get("/api/admin/repos", { params: $scope.query }).then(
           (res) => {
@@ -138,7 +148,7 @@ angular
 
         return false;
       };
-      
+
       function getUserRepositories(username) {
         $http.get("/api/admin/users/" + username + "/repos", {}).then(
           (res) => {
@@ -247,6 +257,7 @@ angular
           (res) => {
             $scope.downloadJobs = res.data.downloadQueue;
             $scope.removeJobs = res.data.removeQueue;
+            $scope.removeCaches = res.data.cacheQueue;
           },
           (err) => {
             console.error(err);

src/processes/removeCache.ts

@@ -0,0 +1,22 @@
+import { SandboxedJob } from "bullmq";
+import Repository from "../Repository";
+
+export default async function (job: SandboxedJob<Repository, void>) {
+  const { connect, getRepository } = require("../database/database");
+  try {
+    await connect();
+    console.log(
+      `[QUEUE] Cache of ${job.data.repoId} is going to be removed...`
+    );
+    const repo = await getRepository(job.data.repoId);
+    try {
+      await repo.removeCache();
+    } catch (error) {
+      throw error;
+    }
+  } catch (error) {
+    console.error(error);
+  } finally {
+    console.log(`[QUEUE] Cache of ${job.data.repoId} is removed.`);
+  }
+}

src/queue.ts

@@ -3,11 +3,21 @@ import config from "../config";
 import Repository from "./Repository";
 import * as path from "path";
 
+export let cacheQueue: Queue<Repository>;
 export let removeQueue: Queue<Repository>;
 export let downloadQueue: Queue<Repository>;
 
 // avoid to load the queue outside the main server
 export function startWorker() {
+  cacheQueue = new Queue<Repository>("cache removal", {
+    connection: {
+      host: config.REDIS_HOSTNAME,
+      port: config.REDIS_PORT,
+    },
+    defaultJobOptions: {
+      removeOnComplete: true,
+    },
+  });
   removeQueue = new Queue<Repository>("repository removal", {
     connection: {
       host: config.REDIS_HOSTNAME,
@@ -26,10 +36,27 @@ export function startWorker() {
       removeOnComplete: true,
     },
   });
-  const removeWorker = new Worker<Repository>(
+  const cacheWorker = new Worker<Repository>(
-    removeQueue.name,
+    cacheQueue.name,
-    path.resolve("build/src/processes/removeRepository.js"),
+    path.resolve("build/src/processes/removeCache.js"),
-    //removeRepository,
+    {
+      concurrency: 5,
+      connection: {
+        host: config.REDIS_HOSTNAME,
+        port: config.REDIS_PORT,
+      },
+      autorun: true,
+    }
+  );
+  cacheWorker.on("error", async (error) => {
+    console.log(error);
+  });
+  cacheWorker.on("completed", async (job) => {
+    await job.remove();
+  });
+  const removeWorker = new Worker<Repository>(
+    removeQueue.name,
+    path.resolve("build/src/processes/removeRepository.js"),
     {
       concurrency: 5,
       connection: {
@@ -37,7 +64,6 @@ export function startWorker() {
         port: config.REDIS_PORT,
       },
       autorun: true,
-
     }
   );
   removeWorker.on("error", async (error) => {
@@ -50,14 +76,13 @@ export function startWorker() {
   const downloadWorker = new Worker<Repository>(
     downloadQueue.name,
     path.resolve("build/src/processes/downloadRepository.js"),
-    // downloadRepository,
     {
       concurrency: 3,
       connection: {
         host: config.REDIS_HOSTNAME,
         port: config.REDIS_PORT,
       },
-      autorun: true
+      autorun: true,
     }
   );
   if (!downloadWorker.isRunning) downloadWorker.run();

src/routes/admin.ts

@@ -4,11 +4,11 @@ import AnonymousError from "../AnonymousError";
 import AnonymizedRepositoryModel from "../database/anonymizedRepositories/anonymizedRepositories.model";
 import ConferenceModel from "../database/conference/conferences.model";
 import UserModel from "../database/users/users.model";
-import { downloadQueue, removeQueue } from "../queue";
+import { cacheQueue, downloadQueue, removeQueue } from "../queue";
 import Repository from "../Repository";
 import User from "../User";
 import { ensureAuthenticated } from "./connection";
-import { handleError, getUser, isOwnerOrAdmin } from "./route-utils";
+import { handleError, getUser, isOwnerOrAdmin, getRepo } from "./route-utils";
 
 const router = express.Router();
 
@@ -91,10 +91,12 @@ router.get("/queues", async (req, res) => {
       "failed",
       "delayed",
     ]),
+    cacheQueue.getJobs(["waiting", "active", "completed", "failed", "delayed"]),
   ]);
   res.json({
     downloadQueue: out[0],
     removeQueue: out[1],
+    cacheQueue: out[2],
   });
 });
 
@@ -151,6 +153,21 @@ router.get("/repos", async (req, res) => {
   });
 });
 
+// delete a repository
+router.delete(
+  "/repos/:repoId/",
+  async (req: express.Request, res: express.Response) => {
+    const repo = await getRepo(req, res, { nocheck: true });
+    if (!repo) return;
+    try {
+      await cacheQueue.add(repo.repoId, repo, { jobId: repo.repoId });
+      return res.json({ status: repo.status });
+    } catch (error) {
+      handleError(error, res, req);
+    }
+  }
+);
+
 router.get("/users", async (req, res) => {
   const page = parseInt(req.query.page as string) || 1;
   const limit = parseInt(req.query.limit as string) || 10;