Fix repository cleanup after removal and expiration (#782)

src/server/index.ts

@@ -17,9 +17,14 @@ import router from "./routes";
 import {
   conferenceStatusCheck,
   repositoryStatusCheck,
+  runRepositoryStatusCheck,
   dailyStatsSnapshot,
 } from "./schedule";
-import { startWorker, recoverStuckPreparing } from "../queue";
+import {
+  startWorker,
+  recoverStuckPreparing,
+  recoverStuckRemoving,
+} from "../queue";
 import {
   computeStats,
   ensureTodaySnapshot,
@@ -363,6 +368,12 @@ export default async function start() {
   recoverStuckPreparing().catch((err) =>
     logger.error("recoverStuckPreparing failed", serializeError(err))
   );
+  recoverStuckRemoving().catch((err) =>
+    logger.error("recoverStuckRemoving failed", serializeError(err))
+  );
+  runRepositoryStatusCheck().catch((err) =>
+    logger.error("initial repository status check failed", serializeError(err))
+  );
 }
 
 start();

src/server/routes/repository-private.ts

@@ -17,7 +17,7 @@ import { IAnonymizedRepositoryDocument } from "../../core/model/anonymizedReposi
 import UserModel from "../../core/model/users/users.model";
 import ConferenceModel from "../../core/model/conference/conferences.model";
 import AnonymousError from "../../core/AnonymousError";
-import { downloadQueue, removeQueue } from "../../queue";
+import { addRemovalJob, downloadQueue } from "../../queue";
 import RepositoryModel from "../../core/model/repositories/repositories.model";
 import User from "../../core/User";
 import { RepositoryStatus } from "../../core/types";
@@ -241,7 +241,14 @@ router.delete(
       const user = await getUser(req);
       isOwnerOrAdmin([repo.owner.id], user);
       await repo.updateStatus(RepositoryStatus.REMOVING);
-      await removeQueue.add(repo.repoId, { repoId: repo.repoId }, { jobId: `repo-${repo.repoId}` });
+      try {
+        await addRemovalJob(repo.repoId);
+      } catch (error) {
+        const message =
+          error instanceof Error ? error.message : "removal_enqueue_failed";
+        await repo.updateStatus(RepositoryStatus.ERROR, message);
+        throw error;
+      }
       return res.json({ status: repo.status });
     } catch (error) {
       handleError(error, res, req);

src/server/schedule.ts

@@ -30,55 +30,120 @@ export function conferenceStatusCheck() {
 export function repositoryStatusCheck() {
   // check every 6 hours the status of the repositories
   schedule.scheduleJob("0 */6 * * *", async () => {
-    logger.info("checking repository status and unused repositories");
-    const now = new Date();
-    const fourMonthAgo = new Date(now);
-    fourMonthAgo.setMonth(fourMonthAgo.getMonth() - 4);
-    const cursor = AnonymizedRepositoryModel.find({
-      status: RepositoryStatus.READY,
-      isReseted: false,
-      $or: [
-        {
-          "options.expirationMode": { $in: ["redirect", "remove"] },
-          "options.expirationDate": { $lte: now },
-        },
-        { lastView: { $lt: fourMonthAgo } },
-      ],
-    }).cursor();
-    const batch: Promise<void>[] = [];
-    for await (const data of cursor) {
-      batch.push(
-        (async () => {
-          const repo = new Repository(data);
-          try {
-            await repo.check();
-          } catch {
-            logger.info("repository expired", { repoId: repo.repoId });
-          }
+    await runRepositoryStatusCheck();
+  });
+}
 
-          if (repo.model.lastView < fourMonthAgo) {
-            try {
-              await repo.removeCache();
-            } catch (error) {
-              logger.error("repository cache removal failed", {
-                ...serializeError(error),
-                repoId: repo.repoId,
-              });
-              return;
-            }
-            logger.info("removed cache for unused repository", {
+export function repositoryMaintenanceQuery(now: Date) {
+  const fourMonthAgo = new Date(now);
+  fourMonthAgo.setMonth(fourMonthAgo.getMonth() - 4);
+  return {
+    status: RepositoryStatus.READY,
+    $or: [
+      {
+        "options.expirationMode": { $in: ["redirect", "remove"] },
+        "options.expirationDate": { $lte: now },
+      },
+      {
+        isReseted: { $ne: true },
+        lastView: { $lt: fourMonthAgo },
+      },
+    ],
+  };
+}
+
+export async function runRepositoryStatusCheck(now = new Date()) {
+  logger.info("checking repository status and unused repositories");
+  const fourMonthAgo = new Date(now);
+  fourMonthAgo.setMonth(fourMonthAgo.getMonth() - 4);
+  const batch: Promise<void>[] = [];
+  const flushBatch = async () => {
+    await Promise.all(batch);
+    batch.length = 0;
+  };
+
+  const cursor = AnonymizedRepositoryModel.find(
+    repositoryMaintenanceQuery(now)
+  ).cursor();
+  for await (const data of cursor) {
+    batch.push(
+      (async () => {
+        const repo = new Repository(data);
+        const shouldExpire =
+          repo.options.expirationMode !== "never" &&
+          repo.options.expirationDate != null &&
+          repo.options.expirationDate <= now;
+        if (shouldExpire) {
+          try {
+            await repo.expire();
+            logger.info("repository expired", { repoId: repo.repoId });
+          } catch (error) {
+            logger.error("repository expiration failed", {
+              ...serializeError(error),
               repoId: repo.repoId,
             });
           }
-        })()
-      );
-      if (batch.length >= 10) {
-        await Promise.all(batch);
-        batch.length = 0;
-      }
+          return;
+        }
+
+        if (
+          repo.model.isReseted !== true &&
+          repo.model.lastView < fourMonthAgo
+        ) {
+          try {
+            await repo.removeCache();
+            logger.info("removed cache for unused repository", {
+              repoId: repo.repoId,
+            });
+          } catch (error) {
+            logger.error("repository cache removal failed", {
+              ...serializeError(error),
+              repoId: repo.repoId,
+            });
+          }
+        }
+      })()
+    );
+    if (batch.length >= 10) {
+      await flushBatch();
     }
-    await Promise.all(batch);
-  });
+  }
+  await flushBatch();
+
+  // Repair terminal records left with data by an older or interrupted
+  // expiration. This makes the cleanup idempotent across deployments.
+  const dirtyTerminalCursor = AnonymizedRepositoryModel.find({
+    $or: [
+      { status: RepositoryStatus.EXPIRING },
+      {
+        status: RepositoryStatus.EXPIRED,
+        isReseted: { $ne: true },
+      },
+    ],
+  }).cursor();
+  for await (const data of dirtyTerminalCursor) {
+    batch.push(
+      (async () => {
+        const repo = new Repository(data);
+        try {
+          await repo.resetSate();
+          await repo.updateStatus(RepositoryStatus.EXPIRED);
+          logger.info("recovered expired repository cleanup", {
+            repoId: repo.repoId,
+          });
+        } catch (error) {
+          logger.error("expired repository cleanup failed", {
+            ...serializeError(error),
+            repoId: repo.repoId,
+          });
+        }
+      })()
+    );
+    if (batch.length >= 10) {
+      await flushBatch();
+    }
+  }
+  await flushBatch();
 }
 
 export function dailyStatsSnapshot() {