improve queue

src/queue/index.ts

@@ -4,6 +4,7 @@ import AnonymizedRepositoryModel from "../core/model/anonymizedRepositories/anon
 import { RepositoryStatus } from "../core/types";
 import * as path from "path";
 import { createLogger, serializeError } from "../core/logger";
+import { recordMetric } from "./queueMetrics";
 
 const logger = createLogger("queue");
 
@@ -119,12 +120,15 @@ export function startWorker() {
       concurrency: 5,
       connection,
       autorun: true,
-      metrics: { maxDataPoints: 120 },
     }
   );
   cacheWorker.on("completed", async (job) => {
+    recordMetric("cache", "completed", (job.finishedOn || Date.now()) - (job.processedOn || job.timestamp));
     await job.remove();
   });
+  cacheWorker.on("failed", async (job) => {
+    if (job) recordMetric("cache", "failed", Date.now() - (job.processedOn || job.timestamp));
+  });
   const removeWorker = new Worker<RepoJobData>(
     removeQueue.name,
     path.resolve("build/queue/processes/removeRepository.js"),
@@ -132,12 +136,15 @@ export function startWorker() {
       concurrency: 5,
       connection,
       autorun: true,
-      metrics: { maxDataPoints: 120 },
     }
   );
   removeWorker.on("completed", async (job) => {
+    recordMetric("remove", "completed", (job.finishedOn || Date.now()) - (job.processedOn || job.timestamp));
     await job.remove();
   });
+  removeWorker.on("failed", async (job) => {
+    if (job) recordMetric("remove", "failed", Date.now() - (job.processedOn || job.timestamp));
+  });
 
   const downloadWorker = new Worker<RepoJobData>(
     downloadQueue.name,
@@ -146,7 +153,6 @@ export function startWorker() {
       concurrency: 3,
       connection,
       autorun: true,
-      metrics: { maxDataPoints: 120 },
     }
   );
   if (!downloadWorker.isRunning()) downloadWorker.run();
@@ -156,8 +162,10 @@ export function startWorker() {
   });
   downloadWorker.on("completed", async (job) => {
     logger.info("download completed", { repoId: job.data.repoId });
+    recordMetric("download", "completed", (job.finishedOn || Date.now()) - (job.processedOn || job.timestamp));
   });
   downloadWorker.on("failed", async (job, err) => {
+    if (job) recordMetric("download", "failed", Date.now() - (job.processedOn || job.timestamp));
     const repoId = job?.data?.repoId;
     logger.error("download failed", {
       ...serializeError(err),

src/queue/processes/downloadRepository.ts

@@ -5,6 +5,8 @@ import { getRepository as getRepositoryImport } from "../../server/database";
 import { RepositoryStatus } from "../../core/types";
 import { RepoJobData } from "../index";
 import { createLogger, serializeError } from "../../core/logger";
+import { RateLimitDelayError, getRedisGateResetAt, setRedisGateFromWorker } from "../../core/GitHubUtils";
+import { DelayedError } from "bullmq";
 
 const logger = createLogger("queue:download");
 
@@ -20,6 +22,24 @@ export default async function (job: SandboxedJob<RepoJobData, void>) {
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   let statusInterval: any = null;
   await connect();
+
+  const gateResetAt = await getRedisGateResetAt();
+  if (gateResetAt > 0) {
+    const delaySec = Math.ceil((gateResetAt - Date.now()) / 1000);
+    logger.info("rate limit gate active, delaying job before work", {
+      repoId: job.data.repoId,
+      delaySec,
+      resetAt: new Date(gateResetAt).toISOString(),
+    });
+    const repo = await getRepository(job.data.repoId);
+    await repo.updateStatus(
+      RepositoryStatus.QUEUE,
+      `rate_limited:${gateResetAt}`
+    );
+    await job.moveToDelayed(gateResetAt);
+    throw new DelayedError();
+  }
+
   const repo = await getRepository(job.data.repoId);
   let tickPromise: Promise<void> | null = null;
   try {
@@ -68,6 +88,30 @@ export default async function (job: SandboxedJob<RepoJobData, void>) {
     } catch (error) {
       clearInterval(statusInterval);
       if (tickPromise) await tickPromise;
+
+      // Rate-limited: delay the job and free the worker slot
+      const isRateDelay = error instanceof RateLimitDelayError;
+      const isRateError = !isRateDelay && error instanceof Error &&
+        (error.message === "github_rate_limit_exceeded" || error.message.includes("rate limit"));
+      if (isRateDelay || isRateError) {
+        const resetAt = isRateDelay
+          ? (error as RateLimitDelayError).resetAt
+          : Date.now() + 60_000; // fallback: retry in 1 min
+        const delaySec = Math.ceil(Math.max(0, resetAt - Date.now()) / 1000);
+        logger.info("rate-limited, delaying job", {
+          repoId: job.data.repoId,
+          delaySec,
+          resetAt: new Date(resetAt).toISOString(),
+        });
+        await setRedisGateFromWorker(resetAt);
+        await repo.updateStatus(
+          RepositoryStatus.QUEUE,
+          `rate_limited:${resetAt}`
+        );
+        await job.moveToDelayed(resetAt);
+        throw new DelayedError();
+      }
+
       updateProgress({ status: "error" });
       if (error instanceof Error) {
         await repo.updateStatus(RepositoryStatus.ERROR, error.message);
@@ -83,6 +127,9 @@ export default async function (job: SandboxedJob<RepoJobData, void>) {
         await tickPromise;
       } catch { /* ignored */ }
     }
+    if (error instanceof DelayedError || (error instanceof Error && error.name === "DelayedError")) {
+      throw error;
+    }
     logger.error("finished with error", {
       ...serializeError(error),
       repoId: job.data.repoId,

src/queue/queueMetrics.ts

@@ -0,0 +1,110 @@
+import { createClient, RedisClientType } from "redis";
+import config from "../config";
+
+const KEY_PREFIX = "qmetrics";
+const TTL_SECONDS = 7 * 24 * 3600 + 3600;
+
+let client: RedisClientType | null = null;
+let disabled = false;
+
+function getClient(): RedisClientType | null {
+  if (disabled) return null;
+  if (client) return client;
+  try {
+    client = createClient({
+      socket: {
+        host: config.REDIS_HOSTNAME,
+        port: config.REDIS_PORT,
+        reconnectStrategy: () => false as any,
+      },
+    }) as RedisClientType;
+    client.on("error", () => {
+      disabled = true;
+      client?.disconnect().catch(() => {});
+      client = null;
+    });
+    client.connect().catch(() => {
+      disabled = true;
+      client = null;
+    });
+    return client;
+  } catch {
+    disabled = true;
+    return null;
+  }
+}
+
+function minuteTs(now?: number): number {
+  return Math.floor((now || Date.now()) / 60000) * 60000;
+}
+
+function key(queue: string, ts: number): string {
+  return `${KEY_PREFIX}:${queue}:${ts}`;
+}
+
+export async function recordMetric(
+  queue: string,
+  type: "completed" | "failed",
+  durationMs: number
+): Promise<void> {
+  const c = getClient();
+  if (!c || !c.isOpen) return;
+  const k = key(queue, minuteTs());
+  const field = type === "completed" ? "c" : "f";
+  try {
+    const pipe = c.multi();
+    pipe.hIncrBy(k, field, 1);
+    pipe.hIncrBy(k, "ms", Math.round(Math.max(0, durationMs)));
+    pipe.expire(k, TTL_SECONDS);
+    await pipe.exec();
+  } catch {
+    // non-critical, don't crash workers
+  }
+}
+
+export interface MetricPoint {
+  ts: number;
+  completed: number;
+  failed: number;
+  avgMs: number;
+}
+
+export async function queryMetrics(
+  queue: string,
+  rangeMinutes: number
+): Promise<MetricPoint[]> {
+  const c = getClient();
+  if (!c || !c.isOpen) return [];
+
+  const now = minuteTs();
+  const start = now - (rangeMinutes - 1) * 60000;
+
+  const keys: string[] = [];
+  const timestamps: number[] = [];
+  for (let t = start; t <= now; t += 60000) {
+    keys.push(key(queue, t));
+    timestamps.push(t);
+  }
+
+  try {
+    const pipe = c.multi();
+    for (const k of keys) pipe.hGetAll(k);
+    const results = await pipe.exec();
+
+    return timestamps.map((ts, i) => {
+      const h = (results[i] as unknown as Record<string, string>) || {};
+      const completed = parseInt(h.c || "0", 10) || 0;
+      const failed = parseInt(h.f || "0", 10) || 0;
+      const totalMs = parseInt(h.ms || "0", 10) || 0;
+      const total = completed + failed;
+      return {
+        ts,
+        completed,
+        failed,
+        avgMs: total > 0 ? Math.round(totalMs / total) : 0,
+      };
+    });
+  } catch {
+    return [];
+  }
+}