Fix repository cleanup after removal and expiration (#782)

src/server/schedule.ts

@@ -30,55 +30,120 @@ export function conferenceStatusCheck() {
 export function repositoryStatusCheck() {
   // check every 6 hours the status of the repositories
   schedule.scheduleJob("0 */6 * * *", async () => {
-    logger.info("checking repository status and unused repositories");
-    const now = new Date();
-    const fourMonthAgo = new Date(now);
-    fourMonthAgo.setMonth(fourMonthAgo.getMonth() - 4);
-    const cursor = AnonymizedRepositoryModel.find({
-      status: RepositoryStatus.READY,
-      isReseted: false,
-      $or: [
-        {
-          "options.expirationMode": { $in: ["redirect", "remove"] },
-          "options.expirationDate": { $lte: now },
-        },
-        { lastView: { $lt: fourMonthAgo } },
-      ],
-    }).cursor();
-    const batch: Promise<void>[] = [];
-    for await (const data of cursor) {
-      batch.push(
-        (async () => {
-          const repo = new Repository(data);
-          try {
-            await repo.check();
-          } catch {
-            logger.info("repository expired", { repoId: repo.repoId });
-          }
+    await runRepositoryStatusCheck();
+  });
+}
 
-          if (repo.model.lastView < fourMonthAgo) {
-            try {
-              await repo.removeCache();
-            } catch (error) {
-              logger.error("repository cache removal failed", {
-                ...serializeError(error),
-                repoId: repo.repoId,
-              });
-              return;
-            }
-            logger.info("removed cache for unused repository", {
+export function repositoryMaintenanceQuery(now: Date) {
+  const fourMonthAgo = new Date(now);
+  fourMonthAgo.setMonth(fourMonthAgo.getMonth() - 4);
+  return {
+    status: RepositoryStatus.READY,
+    $or: [
+      {
+        "options.expirationMode": { $in: ["redirect", "remove"] },
+        "options.expirationDate": { $lte: now },
+      },
+      {
+        isReseted: { $ne: true },
+        lastView: { $lt: fourMonthAgo },
+      },
+    ],
+  };
+}
+
+export async function runRepositoryStatusCheck(now = new Date()) {
+  logger.info("checking repository status and unused repositories");
+  const fourMonthAgo = new Date(now);
+  fourMonthAgo.setMonth(fourMonthAgo.getMonth() - 4);
+  const batch: Promise<void>[] = [];
+  const flushBatch = async () => {
+    await Promise.all(batch);
+    batch.length = 0;
+  };
+
+  const cursor = AnonymizedRepositoryModel.find(
+    repositoryMaintenanceQuery(now)
+  ).cursor();
+  for await (const data of cursor) {
+    batch.push(
+      (async () => {
+        const repo = new Repository(data);
+        const shouldExpire =
+          repo.options.expirationMode !== "never" &&
+          repo.options.expirationDate != null &&
+          repo.options.expirationDate <= now;
+        if (shouldExpire) {
+          try {
+            await repo.expire();
+            logger.info("repository expired", { repoId: repo.repoId });
+          } catch (error) {
+            logger.error("repository expiration failed", {
+              ...serializeError(error),
               repoId: repo.repoId,
             });
           }
-        })()
-      );
-      if (batch.length >= 10) {
-        await Promise.all(batch);
-        batch.length = 0;
-      }
+          return;
+        }
+
+        if (
+          repo.model.isReseted !== true &&
+          repo.model.lastView < fourMonthAgo
+        ) {
+          try {
+            await repo.removeCache();
+            logger.info("removed cache for unused repository", {
+              repoId: repo.repoId,
+            });
+          } catch (error) {
+            logger.error("repository cache removal failed", {
+              ...serializeError(error),
+              repoId: repo.repoId,
+            });
+          }
+        }
+      })()
+    );
+    if (batch.length >= 10) {
+      await flushBatch();
     }
-    await Promise.all(batch);
-  });
+  }
+  await flushBatch();
+
+  // Repair terminal records left with data by an older or interrupted
+  // expiration. This makes the cleanup idempotent across deployments.
+  const dirtyTerminalCursor = AnonymizedRepositoryModel.find({
+    $or: [
+      { status: RepositoryStatus.EXPIRING },
+      {
+        status: RepositoryStatus.EXPIRED,
+        isReseted: { $ne: true },
+      },
+    ],
+  }).cursor();
+  for await (const data of dirtyTerminalCursor) {
+    batch.push(
+      (async () => {
+        const repo = new Repository(data);
+        try {
+          await repo.resetSate();
+          await repo.updateStatus(RepositoryStatus.EXPIRED);
+          logger.info("recovered expired repository cleanup", {
+            repoId: repo.repoId,
+          });
+        } catch (error) {
+          logger.error("expired repository cleanup failed", {
+            ...serializeError(error),
+            repoId: repo.repoId,
+          });
+        }
+      })()
+    );
+    if (batch.length >= 10) {
+      await flushBatch();
+    }
+  }
+  await flushBatch();
 }
 
 export function dailyStatsSnapshot() {