CalvinBackup/anonymous_github
1
0
Fork 0
mirror of https://github.com/tdurieux/anonymous_github.git synced 2026-05-17 07:13:36 +02:00
Code Issues Packages Projects Releases Wiki Activity

error logging improvement, regex fix

Browse Source
This commit is contained in:
tdurieux
2026-05-06 11:09:17 +03:00
parent e34f45522f
commit c2d43164d0
39 changed files with 747 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/queue/index.ts
+18 -10
View File
@@ -3,6 +3,9 @@ import config from "../config";
import AnonymizedRepositoryModel from "../core/model/anonymizedRepositories/anonymizedRepositories.model";
import { RepositoryStatus } from "../core/types";
import * as path from "path";
import { createLogger, serializeError } from "../core/logger";
const logger = createLogger("queue");
// Minimal payload for queue jobs. Workers re-fetch the Repository from the
// database via getRepository(repoId), so passing the full Mongoose-backed
@@ -31,7 +34,10 @@ async function markErrorIfInFlight(repoId: string, message: string) {
}
).exec();
} catch (e) {
console.log("[QUEUE] markErrorIfInFlight error", repoId, e);
logger.error("markErrorIfInFlight failed", {
repoId,
err: serializeError(e),
});
}
}
@@ -58,13 +64,16 @@ export async function recoverStuckPreparing() {
}
}
await markErrorIfInFlight(doc.repoId, "preparation_interrupted");
console.log("[QUEUE] recovered stuck repo", doc.repoId);
logger.info("recovered stuck repo", { repoId: doc.repoId });
} catch (e) {
console.log("[QUEUE] recover error for", doc.repoId, e);
logger.warn("recover failed", {
repoId: doc.repoId,
err: serializeError(e),
});
}
}
} catch (e) {
console.log("[QUEUE] recoverStuckPreparing failed", e);
logger.error("recoverStuckPreparing failed", serializeError(e));
}
}
@@ -140,18 +149,17 @@ export function startWorker() {
if (!downloadWorker.isRunning()) downloadWorker.run();
downloadWorker.on("active", async (job) => {
console.log("[QUEUE] download repository start", job.data.repoId);
logger.info("download start", { repoId: job.data.repoId });
});
downloadWorker.on("completed", async (job) => {
console.log("[QUEUE] download repository completed", job.data.repoId);
logger.info("download completed", { repoId: job.data.repoId });
});
downloadWorker.on("failed", async (job, err) => {
const repoId = job?.data?.repoId;
console.log(
"[QUEUE] download repository failed",
logger.error("download failed", {
repoId,
err?.message || err
);
err: serializeError(err),
});
if (!repoId) return;
if (job && typeof job.attemptsMade === "number" && job.opts?.attempts) {
if (job.attemptsMade < job.opts.attempts) return;
src/queue/processes/downloadRepository.ts
+17 -10
View File
@@ -4,6 +4,9 @@ config();
import { getRepository as getRepositoryImport } from "../../server/database";
import { RepositoryStatus } from "../../core/types";
import { RepoJobData } from "../index";
import { createLogger, serializeError } from "../../core/logger";
const logger = createLogger("queue:download");
export default async function (job: SandboxedJob<RepoJobData, void>) {
const {
@@ -13,7 +16,7 @@ export default async function (job: SandboxedJob<RepoJobData, void>) {
connect: () => Promise<void>;
getRepository: typeof getRepositoryImport;
} = require("../../server/database");
console.log(`[QUEUE] ${job.data.repoId} is going to be downloaded`);
logger.info("queued for download", { repoId: job.data.repoId });
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let statusInterval: any = null;
await connect();
@@ -37,9 +40,10 @@ export default async function (job: SandboxedJob<RepoJobData, void>) {
repo.status &&
repo.model.statusMessage !== progress?.status
) {
console.log(
`[QUEUE] Progress: ${job.data.repoId} ${progress.status}`
);
logger.debug("progress", {
repoId: job.data.repoId,
status: progress.status,
});
await repo.updateStatus(repo.status, progress?.status || "");
}
} catch {
@@ -60,7 +64,7 @@ export default async function (job: SandboxedJob<RepoJobData, void>) {
clearInterval(statusInterval);
if (tickPromise) await tickPromise;
await repo.updateStatus(RepositoryStatus.READY, "");
console.log(`[QUEUE] ${job.data.repoId} is downloaded`);
logger.info("downloaded", { repoId: job.data.repoId });
} catch (error) {
clearInterval(statusInterval);
if (tickPromise) await tickPromise;
@@ -79,17 +83,20 @@ export default async function (job: SandboxedJob<RepoJobData, void>) {
await tickPromise;
} catch { /* ignored */ }
}
console.log(`[QUEUE] ${job.data.repoId} is finished with an error`, error);
logger.error("finished with error", {
repoId: job.data.repoId,
err: serializeError(error),
});
try {
await repo.updateStatus(
RepositoryStatus.ERROR,
error instanceof Error ? error.message : String(error)
);
} catch (persistError) {
console.log(
`[QUEUE] failed to persist ERROR status for ${job.data.repoId}`,
persistError
);
logger.error("failed to persist ERROR status", {
repoId: job.data.repoId,
err: serializeError(persistError),
});
}
throw error;
} finally {
src/queue/processes/removeCache.ts
+5 -4
View File
@@ -1,6 +1,9 @@
import { SandboxedJob } from "bullmq";
import { getRepository as getRepositoryImport } from "../../server/database";
import { RepoJobData } from "../index";
import { createLogger } from "../../core/logger";
const logger = createLogger("queue:cache");
export default async function (job: SandboxedJob<RepoJobData, void>) {
const {
@@ -12,14 +15,12 @@ export default async function (job: SandboxedJob<RepoJobData, void>) {
} = require("../../server/database");
try {
await connect();
console.log(
`[QUEUE] Cache of ${job.data.repoId} is going to be removed...`
);
logger.info("removing cache", { repoId: job.data.repoId });
const repo = await getRepository(job.data.repoId);
await repo.removeCache();
} catch {
// error already handled
} finally {
console.log(`[QUEUE] Cache of ${job.data.repoId} is removed.`);
logger.info("cache removed", { repoId: job.data.repoId });
}
}
src/queue/processes/removeRepository.ts
+5 -2
View File
@@ -2,6 +2,9 @@ import { SandboxedJob } from "bullmq";
import { getRepository as getRepositoryImport } from "../../server/database";
import { RepositoryStatus } from "../../core/types";
import { RepoJobData } from "../index";
import { createLogger } from "../../core/logger";
const logger = createLogger("queue:remove");
export default async function (job: SandboxedJob<RepoJobData, void>) {
const {
@@ -13,7 +16,7 @@ export default async function (job: SandboxedJob<RepoJobData, void>) {
} = require("../../server/database");
try {
await connect();
console.log(`[QUEUE] ${job.data.repoId} is going to be removed`);
logger.info("removing repository", { repoId: job.data.repoId });
const repo = await getRepository(job.data.repoId);
await repo.updateStatus(RepositoryStatus.REMOVING, "");
try {
@@ -29,6 +32,6 @@ export default async function (job: SandboxedJob<RepoJobData, void>) {
} catch {
// error already handled
} finally {
console.log(`[QUEUE] ${job.data.repoId} is removed`);
logger.info("repository removed", { repoId: job.data.repoId });
}
}