From 73f7582fd2b9ad33b32c80ef4652666afbab3408 Mon Sep 17 00:00:00 2001 From: tdurieux Date: Tue, 7 Feb 2023 13:27:06 +0100 Subject: [PATCH] feat: admin to remove repo cache --- public/partials/admin/queues.htm | 74 ++++++++++++++++++++++++++ public/partials/admin/repositories.htm | 3 ++ public/script/admin.js | 13 ++++- src/processes/removeCache.ts | 22 ++++++++ src/queue.ts | 39 +++++++++++--- src/routes/admin.ts | 21 +++++++- 6 files changed, 162 insertions(+), 10 deletions(-) create mode 100644 src/processes/removeCache.ts diff --git a/public/partials/admin/queues.htm b/public/partials/admin/queues.htm index db7dee2..a2af6b8 100644 --- a/public/partials/admin/queues.htm +++ b/public/partials/admin/queues.htm @@ -148,5 +148,79 @@ There is no job to display. +

Remove Cache

+ diff --git a/public/partials/admin/repositories.htm b/public/partials/admin/repositories.htm index 8874a6f..ba7885f 100644 --- a/public/partials/admin/repositories.htm +++ b/public/partials/admin/repositories.htm @@ -313,6 +313,9 @@ Actions
+ + Remove Cache + Edit diff --git a/public/script/admin.js b/public/script/admin.js index 4805ab7..8317945 100644 --- a/public/script/admin.js +++ b/public/script/admin.js @@ -29,6 +29,16 @@ angular preparing: true, }; + $scope.removeCache = (repo) => { + $http.delete("/api/admin/repos/" + repo.repoId).then( + (res) => { + $scope.$apply(); + }, + (err) => { + console.error(err); + } + ); + }; function getRepositories() { $http.get("/api/admin/repos", { params: $scope.query }).then( (res) => { @@ -138,7 +148,7 @@ angular return false; }; - + function getUserRepositories(username) { $http.get("/api/admin/users/" + username + "/repos", {}).then( (res) => { @@ -247,6 +257,7 @@ angular (res) => { $scope.downloadJobs = res.data.downloadQueue; $scope.removeJobs = res.data.removeQueue; + $scope.removeCaches = res.data.cacheQueue; }, (err) => { console.error(err); diff --git a/src/processes/removeCache.ts b/src/processes/removeCache.ts new file mode 100644 index 0000000..d0c65d6 --- /dev/null +++ b/src/processes/removeCache.ts @@ -0,0 +1,22 @@ +import { SandboxedJob } from "bullmq"; +import Repository from "../Repository"; + +export default async function (job: SandboxedJob) { + const { connect, getRepository } = require("../database/database"); + try { + await connect(); + console.log( + `[QUEUE] Cache of ${job.data.repoId} is going to be removed...` + ); + const repo = await getRepository(job.data.repoId); + try { + await repo.removeCache(); + } catch (error) { + throw error; + } + } catch (error) { + console.error(error); + } finally { + console.log(`[QUEUE] Cache of ${job.data.repoId} is removed.`); + } +} diff --git a/src/queue.ts b/src/queue.ts index f038b7f..bfa3240 100644 --- a/src/queue.ts +++ b/src/queue.ts @@ -3,11 +3,21 @@ import config from "../config"; import Repository from "./Repository"; import * as path from "path"; +export let cacheQueue: Queue; export let removeQueue: Queue; export let downloadQueue: Queue; // avoid to load the queue outside the main server export function startWorker() { + cacheQueue = new Queue("cache removal", { + connection: { + host: config.REDIS_HOSTNAME, + port: config.REDIS_PORT, + }, + defaultJobOptions: { + removeOnComplete: true, + }, + }); removeQueue = new Queue("repository removal", { connection: { host: config.REDIS_HOSTNAME, @@ -26,10 +36,27 @@ export function startWorker() { removeOnComplete: true, }, }); - const removeWorker = new Worker( - removeQueue.name, - path.resolve("build/src/processes/removeRepository.js"), - //removeRepository, + const cacheWorker = new Worker( + cacheQueue.name, + path.resolve("build/src/processes/removeCache.js"), + { + concurrency: 5, + connection: { + host: config.REDIS_HOSTNAME, + port: config.REDIS_PORT, + }, + autorun: true, + } + ); + cacheWorker.on("error", async (error) => { + console.log(error); + }); + cacheWorker.on("completed", async (job) => { + await job.remove(); + }); + const removeWorker = new Worker( + removeQueue.name, + path.resolve("build/src/processes/removeRepository.js"), { concurrency: 5, connection: { @@ -37,7 +64,6 @@ export function startWorker() { port: config.REDIS_PORT, }, autorun: true, - } ); removeWorker.on("error", async (error) => { @@ -50,14 +76,13 @@ export function startWorker() { const downloadWorker = new Worker( downloadQueue.name, path.resolve("build/src/processes/downloadRepository.js"), - // downloadRepository, { concurrency: 3, connection: { host: config.REDIS_HOSTNAME, port: config.REDIS_PORT, }, - autorun: true + autorun: true, } ); if (!downloadWorker.isRunning) downloadWorker.run(); diff --git a/src/routes/admin.ts b/src/routes/admin.ts index da6568c..d4d737e 100644 --- a/src/routes/admin.ts +++ b/src/routes/admin.ts @@ -4,11 +4,11 @@ import AnonymousError from "../AnonymousError"; import AnonymizedRepositoryModel from "../database/anonymizedRepositories/anonymizedRepositories.model"; import ConferenceModel from "../database/conference/conferences.model"; import UserModel from "../database/users/users.model"; -import { downloadQueue, removeQueue } from "../queue"; +import { cacheQueue, downloadQueue, removeQueue } from "../queue"; import Repository from "../Repository"; import User from "../User"; import { ensureAuthenticated } from "./connection"; -import { handleError, getUser, isOwnerOrAdmin } from "./route-utils"; +import { handleError, getUser, isOwnerOrAdmin, getRepo } from "./route-utils"; const router = express.Router(); @@ -91,10 +91,12 @@ router.get("/queues", async (req, res) => { "failed", "delayed", ]), + cacheQueue.getJobs(["waiting", "active", "completed", "failed", "delayed"]), ]); res.json({ downloadQueue: out[0], removeQueue: out[1], + cacheQueue: out[2], }); }); @@ -151,6 +153,21 @@ router.get("/repos", async (req, res) => { }); }); +// delete a repository +router.delete( + "/repos/:repoId/", + async (req: express.Request, res: express.Response) => { + const repo = await getRepo(req, res, { nocheck: true }); + if (!repo) return; + try { + await cacheQueue.add(repo.repoId, repo, { jobId: repo.repoId }); + return res.json({ status: repo.status }); + } catch (error) { + handleError(error, res, req); + } + } +); + router.get("/users", async (req, res) => { const page = parseInt(req.query.page as string) || 1; const limit = parseInt(req.query.limit as string) || 10;