mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-05-16 14:59:07 +02:00
error logging improvement, regex fix
This commit is contained in:
tdurieux
+5
-2
@@ -8,6 +8,9 @@ import { IAnonymizedGistDocument } from "./model/anonymizedGists/anonymizedGists
|
||||
import config from "../config";
|
||||
import { octokit } from "./GitHubUtils";
|
||||
import { ContentAnonimizer } from "./anonymize-utils";
|
||||
import { createLogger } from "./logger";
|
||||
|
||||
const logger = createLogger("gist");
|
||||
|
||||
type GistPayload = {
|
||||
description: string;
|
||||
@@ -59,14 +62,14 @@ export default class Gist {
|
||||
try {
|
||||
return this._model.source.accessToken;
|
||||
} catch {
|
||||
console.debug("[ERROR] Token is invalid", this._model.source.gistId);
|
||||
logger.warn("invalid token", { gistId: this._model.source.gistId });
|
||||
}
|
||||
}
|
||||
return config.GITHUB_TOKEN;
|
||||
}
|
||||
|
||||
async download() {
|
||||
console.debug("[INFO] Downloading gist", this._model.source.gistId);
|
||||
logger.info("downloading gist", { gistId: this._model.source.gistId });
|
||||
const oct = octokit(await this.getToken());
|
||||
|
||||
const gist_id = this._model.source.gistId;
|
||||
|
||||
@@ -3,6 +3,9 @@ import { Octokit } from "@octokit/rest";
|
||||
import Repository from "./Repository";
|
||||
import UserModel from "./model/users/users.model";
|
||||
import config from "../config";
|
||||
import { createLogger } from "./logger";
|
||||
|
||||
const logger = createLogger("github");
|
||||
|
||||
export function octokit(token: string) {
|
||||
return new Octokit({
|
||||
@@ -24,7 +27,7 @@ export async function checkToken(token: string) {
|
||||
}
|
||||
|
||||
export async function getToken(repository: Repository) {
|
||||
console.log("getToken", repository.repoId);
|
||||
logger.debug("getToken", { repoId: repository.repoId });
|
||||
// if (repository.model.source.accessToken) {
|
||||
// // only check the token if the repo has been visited less than 10 minutes ago
|
||||
// if (
|
||||
@@ -101,9 +104,10 @@ export async function getToken(repository: Repository) {
|
||||
return refreshed;
|
||||
}
|
||||
}
|
||||
console.warn(
|
||||
`[getToken] refresh failed for ${repository.owner.model.username} (status ${res.status}); falling back`
|
||||
);
|
||||
logger.warn("token refresh failed; falling back", {
|
||||
username: repository.owner.model.username,
|
||||
status: res.status,
|
||||
});
|
||||
// fall through to the checkToken path / config.GITHUB_TOKEN
|
||||
}
|
||||
const check = await checkToken(ownerAccessToken);
|
||||
|
||||
+15
-16
@@ -9,6 +9,9 @@ import config from "../config";
|
||||
import got, { HTTPError } from "got";
|
||||
import { octokit } from "./GitHubUtils";
|
||||
import { ContentAnonimizer } from "./anonymize-utils";
|
||||
import { createLogger } from "./logger";
|
||||
|
||||
const logger = createLogger("pull-request");
|
||||
|
||||
export default class PullRequest {
|
||||
private _model: IAnonymizedPullRequestDocument;
|
||||
@@ -38,20 +41,18 @@ export default class PullRequest {
|
||||
try {
|
||||
return this._model.source.accessToken;
|
||||
} catch {
|
||||
console.debug(
|
||||
"[ERROR] Token is invalid",
|
||||
this._model.source.pullRequestId
|
||||
);
|
||||
logger.warn("invalid token", {
|
||||
pullRequestId: this._model.source.pullRequestId,
|
||||
});
|
||||
}
|
||||
}
|
||||
return config.GITHUB_TOKEN;
|
||||
}
|
||||
|
||||
async download() {
|
||||
console.debug(
|
||||
"[INFO] Downloading pull request",
|
||||
this._model.source.pullRequestId
|
||||
);
|
||||
logger.info("downloading pull request", {
|
||||
pullRequestId: this._model.source.pullRequestId,
|
||||
});
|
||||
const oct = octokit(await this.getToken());
|
||||
|
||||
const [owner, repo] = this._model.source.repositoryFullName.split("/");
|
||||
@@ -89,10 +90,9 @@ export default class PullRequest {
|
||||
user?: { login?: string } | null;
|
||||
}> => {
|
||||
if ((err as { status?: number }).status === 404) {
|
||||
console.warn(
|
||||
"[WARN] Failed to fetch PR comments (404), continuing without them",
|
||||
`${owner}/${repo}#${pull_number}`
|
||||
);
|
||||
logger.warn("PR comments 404, continuing without them", {
|
||||
pr: `${owner}/${repo}#${pull_number}`,
|
||||
});
|
||||
return [];
|
||||
}
|
||||
throw err;
|
||||
@@ -102,10 +102,9 @@ export default class PullRequest {
|
||||
`https://github.com/${owner}/${repo}/pull/${pull_number}.diff`
|
||||
).catch((err) => {
|
||||
if (err instanceof HTTPError && err.response.statusCode === 404) {
|
||||
console.warn(
|
||||
"[WARN] Failed to fetch PR diff (404), continuing without it",
|
||||
`${owner}/${repo}#${pull_number}`
|
||||
);
|
||||
logger.warn("PR diff 404, continuing without it", {
|
||||
pr: `${owner}/${repo}#${pull_number}`,
|
||||
});
|
||||
return { body: "" };
|
||||
}
|
||||
throw err;
|
||||
|
||||
+14
-9
@@ -23,6 +23,9 @@ import { getToken } from "./GitHubUtils";
|
||||
import config from "../config";
|
||||
import FileModel from "./model/files/files.model";
|
||||
import AnonymizedRepositoryModel from "./model/anonymizedRepositories/anonymizedRepositories.model";
|
||||
import { createLogger, serializeError } from "./logger";
|
||||
|
||||
const logger = createLogger("repository");
|
||||
import { IFile } from "./model/files/files.types";
|
||||
import AnonymizedFile from "./AnonymizedFile";
|
||||
import { FilterQuery } from "mongoose";
|
||||
@@ -316,9 +319,10 @@ export default class Repository {
|
||||
?.commit;
|
||||
|
||||
if (!newCommit) {
|
||||
console.error(
|
||||
`${branchName} for ${this.model.source.repositoryName} is not found`
|
||||
);
|
||||
logger.error("branch not found", {
|
||||
branch: branchName,
|
||||
repo: this.model.source.repositoryName,
|
||||
});
|
||||
await this.updateStatus(RepositoryStatus.ERROR, "branch_not_found");
|
||||
await this.resetSate();
|
||||
throw new AnonymousError("branch_not_found", {
|
||||
@@ -330,7 +334,7 @@ export default class Repository {
|
||||
this.model.source.commit == newCommit &&
|
||||
this.status == RepositoryStatus.READY
|
||||
) {
|
||||
console.log(`[UPDATE] ${this._model.repoId} is up to date`);
|
||||
logger.info("up to date", { repoId: this._model.repoId });
|
||||
return;
|
||||
}
|
||||
this._model.source.commit = newCommit;
|
||||
@@ -347,9 +351,10 @@ export default class Repository {
|
||||
}
|
||||
this.model.source.commit = newCommit;
|
||||
this._model.anonymizeDate = new Date();
|
||||
console.log(
|
||||
`[UPDATE] ${this._model.repoId} will be updated to ${newCommit}`
|
||||
);
|
||||
logger.info("update queued", {
|
||||
repoId: this._model.repoId,
|
||||
commit: newCommit,
|
||||
});
|
||||
|
||||
await this.resetSate(RepositoryStatus.PREPARING);
|
||||
await downloadQueue.add(this.repoId, { repoId: this.repoId }, {
|
||||
@@ -450,7 +455,7 @@ export default class Repository {
|
||||
FileModel.deleteMany({ repoId: this.repoId }).exec(),
|
||||
this.removeCache(),
|
||||
]);
|
||||
console.log(`[RESET] ${this._model.repoId} has been reset`);
|
||||
logger.info("reset", { repoId: this._model.repoId });
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -464,7 +469,7 @@ export default class Repository {
|
||||
try {
|
||||
await this.model.save();
|
||||
} catch (error) {
|
||||
console.error("[ERROR] removeCache save", error);
|
||||
logger.error("removeCache save failed", serializeError(error));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
import { createClient, RedisClientType } from "redis";
|
||||
import config from "../config";
|
||||
|
||||
export const ERROR_LOG_KEY = "admin:errors";
|
||||
export const ERROR_LOG_MAX = 1000;
|
||||
|
||||
export type Logger = {
|
||||
debug: (...args: unknown[]) => void;
|
||||
info: (...args: unknown[]) => void;
|
||||
warn: (...args: unknown[]) => void;
|
||||
error: (...args: unknown[]) => void;
|
||||
};
|
||||
|
||||
type Level = "debug" | "info" | "warn" | "error";
|
||||
|
||||
const LEVEL_ORDER: Record<Level, number> = {
|
||||
debug: 10,
|
||||
info: 20,
|
||||
warn: 30,
|
||||
error: 40,
|
||||
};
|
||||
|
||||
function resolveThreshold(): number {
|
||||
const raw = (process.env.LOG_LEVEL || "").toLowerCase() as Level;
|
||||
if (raw in LEVEL_ORDER) return LEVEL_ORDER[raw];
|
||||
return process.env.NODE_ENV === "production"
|
||||
? LEVEL_ORDER.info
|
||||
: LEVEL_ORDER.debug;
|
||||
}
|
||||
|
||||
const threshold = resolveThreshold();
|
||||
|
||||
function formatArg(a: unknown): string {
|
||||
if (typeof a === "string") return a;
|
||||
if (a instanceof Error) return JSON.stringify(serializeError(a));
|
||||
try {
|
||||
return JSON.stringify(a);
|
||||
} catch {
|
||||
return String(a);
|
||||
}
|
||||
}
|
||||
|
||||
let redisClient: RedisClientType | null = null;
|
||||
let redisDisabled = false;
|
||||
|
||||
function getRedis(): RedisClientType | null {
|
||||
if (redisDisabled) return null;
|
||||
if (redisClient) return redisClient;
|
||||
try {
|
||||
redisClient = createClient({
|
||||
socket: {
|
||||
host: config.REDIS_HOSTNAME,
|
||||
port: config.REDIS_PORT,
|
||||
// Give up on first failure — we don't want the redis client's
|
||||
// reconnect timer keeping the event loop alive (breaks unit tests
|
||||
// that just import the logger), and we don't want logger.error to
|
||||
// recursively retrigger if redis is down.
|
||||
reconnectStrategy: false,
|
||||
},
|
||||
}) as RedisClientType;
|
||||
redisClient.on("error", () => {
|
||||
if (!redisDisabled) {
|
||||
redisDisabled = true;
|
||||
try {
|
||||
redisClient?.disconnect();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
});
|
||||
redisClient.connect().catch(() => {
|
||||
redisDisabled = true;
|
||||
});
|
||||
return redisClient;
|
||||
} catch {
|
||||
redisDisabled = true;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function persistError(entry: {
|
||||
ts: string;
|
||||
module: string;
|
||||
message: string;
|
||||
raw: unknown[];
|
||||
}) {
|
||||
const client = getRedis();
|
||||
if (!client || !client.isOpen) return;
|
||||
const payload = JSON.stringify(entry);
|
||||
client
|
||||
.multi()
|
||||
.lPush(ERROR_LOG_KEY, payload)
|
||||
.lTrim(ERROR_LOG_KEY, 0, ERROR_LOG_MAX - 1)
|
||||
.exec()
|
||||
.catch(() => undefined);
|
||||
}
|
||||
|
||||
function emit(level: Level, module: string, args: unknown[]) {
|
||||
if (LEVEL_ORDER[level] < threshold) return;
|
||||
const ts = new Date().toISOString();
|
||||
const formatted = args.map(formatArg);
|
||||
const line = `${ts} ${level.toUpperCase()} [${module}] ${formatted.join(" ")}`;
|
||||
const sink =
|
||||
level === "error"
|
||||
? console.error
|
||||
: level === "warn"
|
||||
? console.warn
|
||||
: level === "debug"
|
||||
? console.debug
|
||||
: console.log;
|
||||
sink(line);
|
||||
if (level === "error") {
|
||||
persistError({
|
||||
ts,
|
||||
module,
|
||||
message: typeof args[0] === "string" ? args[0] : "",
|
||||
raw: args.map((a) => {
|
||||
if (a instanceof Error) return serializeError(a);
|
||||
return a;
|
||||
}),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function createLogger(module: string): Logger {
|
||||
return {
|
||||
debug: (...args) => emit("debug", module, args),
|
||||
info: (...args) => emit("info", module, args),
|
||||
warn: (...args) => emit("warn", module, args),
|
||||
error: (...args) => emit("error", module, args),
|
||||
};
|
||||
}
|
||||
|
||||
type ErrorLike = {
|
||||
name?: string;
|
||||
message?: string;
|
||||
stack?: string;
|
||||
status?: number;
|
||||
httpStatus?: number;
|
||||
code?: string | number;
|
||||
cause?: unknown;
|
||||
request?: { url?: string; method?: string };
|
||||
response?: { url?: string; status?: number };
|
||||
};
|
||||
|
||||
export function serializeError(err: unknown): Record<string, unknown> {
|
||||
if (err == null) return { value: err };
|
||||
if (typeof err !== "object") return { value: String(err) };
|
||||
|
||||
const e = err as ErrorLike;
|
||||
const out: Record<string, unknown> = {};
|
||||
if (e.name) out.name = e.name;
|
||||
if (e.message) out.message = e.message;
|
||||
|
||||
// Octokit RequestError / HTTP-shaped errors: surface status + url + method,
|
||||
// skip the giant headers/response body dump.
|
||||
if (typeof e.status === "number") out.status = e.status;
|
||||
if (e.request?.url) out.url = e.request.url;
|
||||
if (e.request?.method) out.method = e.request.method;
|
||||
if (!e.request && e.response?.url) out.url = e.response.url;
|
||||
|
||||
// AnonymousError carries an httpStatus and an inner cause.
|
||||
if (typeof e.httpStatus === "number") out.httpStatus = e.httpStatus;
|
||||
if (e.code !== undefined && e.code !== e.message) out.code = e.code;
|
||||
if (e.cause) out.cause = serializeError(e.cause);
|
||||
|
||||
// Only include the stack when there's nothing else useful — avoids dumping
|
||||
// a stack for handled HTTP errors but keeps debuggability for plain Errors.
|
||||
if (!out.status && !out.httpStatus && e.stack) out.stack = e.stack;
|
||||
|
||||
return out;
|
||||
}
|
||||
@@ -14,6 +14,9 @@ import { FILE_TYPE } from "../storage/Storage";
|
||||
import { octokit } from "../GitHubUtils";
|
||||
import FileModel from "../model/files/files.model";
|
||||
import { IFile } from "../model/files/files.types";
|
||||
import { createLogger, serializeError } from "../logger";
|
||||
|
||||
const logger = createLogger("gh-stream");
|
||||
|
||||
export default class GitHubStream extends GitHubBase {
|
||||
type: "GitHubDownload" | "GitHubStream" | "Zip" = "GitHubStream";
|
||||
@@ -36,7 +39,7 @@ export default class GitHubStream extends GitHubBase {
|
||||
repo: this.data.repoName,
|
||||
file_sha: sha,
|
||||
});
|
||||
console.log("[GHStream] Downloading file", url);
|
||||
logger.debug("downloading file", { url });
|
||||
return got.stream(url, {
|
||||
headers: {
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
@@ -45,7 +48,7 @@ export default class GitHubStream extends GitHubBase {
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
logger.error("downloadFile failed", serializeError(error));
|
||||
throw new AnonymousError("repo_not_accessible", {
|
||||
httpStatus: 404,
|
||||
object: this.data,
|
||||
@@ -60,7 +63,7 @@ export default class GitHubStream extends GitHubBase {
|
||||
// as the fallback for LFS files (#95).
|
||||
private downloadFileViaRaw(token: string, filePath: string) {
|
||||
const url = `https://github.com/${this.data.organization}/${this.data.repoName}/raw/${this.data.commit}/${filePath}`;
|
||||
console.log("[GHStream] Downloading via raw URL (LFS)", url);
|
||||
logger.debug("downloading via raw URL (LFS)", { url });
|
||||
return got.stream(url, {
|
||||
headers: { authorization: `token ${token}` },
|
||||
followRedirect: true,
|
||||
@@ -267,9 +270,9 @@ export default class GitHubStream extends GitHubBase {
|
||||
}
|
||||
output.push(...this.tree2Tree(data.tree, parentPath));
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
const status = (error as { status?: number }).status;
|
||||
if (status === 409) {
|
||||
logger.debug("getTree empty repo", serializeError(error));
|
||||
throw new AnonymousError("repo_empty", {
|
||||
httpStatus: 409,
|
||||
object: this.data,
|
||||
@@ -277,6 +280,7 @@ export default class GitHubStream extends GitHubBase {
|
||||
});
|
||||
}
|
||||
if (status === 404) {
|
||||
logger.debug("getTree miss", serializeError(error));
|
||||
const code = await classifyGitHubMissError(error, this.data);
|
||||
throw new AnonymousError(code, {
|
||||
httpStatus: 404,
|
||||
@@ -284,6 +288,7 @@ export default class GitHubStream extends GitHubBase {
|
||||
cause: error as Error,
|
||||
});
|
||||
}
|
||||
logger.warn("getTree failed", serializeError(error));
|
||||
throw new AnonymousError("repo_not_found", {
|
||||
httpStatus: status || 500,
|
||||
object: this.data,
|
||||
|
||||
@@ -10,6 +10,9 @@ import { lookup } from "mime-types";
|
||||
import StorageBase, { FILE_TYPE } from "./Storage";
|
||||
import FileModel from "../model/files/files.model";
|
||||
import { IFile } from "../model/files/files.types";
|
||||
import { createLogger, serializeError } from "../logger";
|
||||
|
||||
const logger = createLogger("fs");
|
||||
|
||||
export default class FileSystem extends StorageBase {
|
||||
type = "FileSystem";
|
||||
@@ -97,7 +100,7 @@ export default class FileSystem extends StorageBase {
|
||||
}
|
||||
await fs.promises.rename(tmpPath, fullPath);
|
||||
} catch (err) {
|
||||
console.error("[ERROR] FileSystem.write failed:", err);
|
||||
logger.error("write failed", serializeError(err));
|
||||
await fs.promises.rm(tmpPath, { force: true }).catch(() => undefined);
|
||||
throw err;
|
||||
}
|
||||
|
||||
@@ -17,6 +17,9 @@ import AnonymousError from "../AnonymousError";
|
||||
import StorageBase, { FILE_TYPE } from "./Storage";
|
||||
import { IFile } from "../model/files/files.types";
|
||||
import FileModel from "../model/files/files.model";
|
||||
import { createLogger, serializeError } from "../logger";
|
||||
|
||||
const logger = createLogger("s3");
|
||||
|
||||
export default class S3Storage extends StorageBase {
|
||||
type = "AWS";
|
||||
@@ -129,7 +132,7 @@ export default class S3Storage extends StorageBase {
|
||||
try {
|
||||
res.status(500).json({ error: "file_not_found" });
|
||||
} catch (err) {
|
||||
console.error(`[ERROR] S3 send ${path}`, err);
|
||||
logger.error("send failed", { path, err: serializeError(err) });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,6 +82,7 @@ const UNICODE_ESCAPE_CHARS = new Set(
|
||||
export function diacriticInsensitive(escapedTerm: string): string {
|
||||
let out = "";
|
||||
let i = 0;
|
||||
let inClass = false;
|
||||
while (i < escapedTerm.length) {
|
||||
const c = escapedTerm[i];
|
||||
if (c === "\\" && i + 1 < escapedTerm.length) {
|
||||
@@ -96,6 +97,26 @@ export function diacriticInsensitive(escapedTerm: string): string {
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
// Inside a character class, leave letters alone — expanding them into
|
||||
// bracketed alternatives would produce nested `[...]` which is a syntax
|
||||
// error. The user's regex is responsible for its own char-class content.
|
||||
if (c === "[" && !inClass) {
|
||||
inClass = true;
|
||||
out += c;
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (c === "]" && inClass) {
|
||||
inClass = false;
|
||||
out += c;
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (inClass) {
|
||||
out += c;
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
const lower = c.toLowerCase();
|
||||
out += DIACRITIC_CLASSES[lower] || c;
|
||||
i += 1;
|
||||
|
||||
@@ -8,6 +8,9 @@ import {
|
||||
anonymizePathCompiled,
|
||||
compileTerms,
|
||||
} from "./anonymize-utils";
|
||||
import { createLogger, serializeError } from "./logger";
|
||||
|
||||
const logger = createLogger("zip-stream");
|
||||
|
||||
export interface StreamAnonymizedZipOptions {
|
||||
repoId: string;
|
||||
@@ -82,7 +85,7 @@ export async function streamAnonymizedZip(
|
||||
const downloadStream = got.stream(response.url);
|
||||
|
||||
res.on("error", (error) => {
|
||||
console.error(error);
|
||||
logger.error("response stream error", serializeError(error));
|
||||
downloadStream.destroy();
|
||||
});
|
||||
res.on("close", () => {
|
||||
@@ -101,7 +104,7 @@ export async function streamAnonymizedZip(
|
||||
// bug as #694.
|
||||
let upstreamSucceeded = false;
|
||||
const fail = (error: Error) => {
|
||||
console.error(error);
|
||||
logger.error("upstream zipball failed", serializeError(error));
|
||||
archive.abort();
|
||||
const destroyable = res as unknown as {
|
||||
destroy?: (err?: Error) => void;
|
||||
@@ -140,7 +143,7 @@ export async function streamAnonymizedZip(
|
||||
archive.append(st, { name: fileName });
|
||||
} catch (error) {
|
||||
entry.autodrain();
|
||||
console.error(error);
|
||||
logger.error("entry transform failed", serializeError(error));
|
||||
}
|
||||
} else {
|
||||
entry.autodrain();
|
||||
@@ -157,7 +160,7 @@ export async function streamAnonymizedZip(
|
||||
});
|
||||
|
||||
archive.pipe(res).on("error", (error) => {
|
||||
console.error(error);
|
||||
logger.error("archive pipe error", serializeError(error));
|
||||
if (!upstreamSucceeded) {
|
||||
// archive errored while we were still depending on upstream bytes:
|
||||
// treat as failure rather than truncating.
|
||||
|
||||
+18
-10
@@ -3,6 +3,9 @@ import config from "../config";
|
||||
import AnonymizedRepositoryModel from "../core/model/anonymizedRepositories/anonymizedRepositories.model";
|
||||
import { RepositoryStatus } from "../core/types";
|
||||
import * as path from "path";
|
||||
import { createLogger, serializeError } from "../core/logger";
|
||||
|
||||
const logger = createLogger("queue");
|
||||
|
||||
// Minimal payload for queue jobs. Workers re-fetch the Repository from the
|
||||
// database via getRepository(repoId), so passing the full Mongoose-backed
|
||||
@@ -31,7 +34,10 @@ async function markErrorIfInFlight(repoId: string, message: string) {
|
||||
}
|
||||
).exec();
|
||||
} catch (e) {
|
||||
console.log("[QUEUE] markErrorIfInFlight error", repoId, e);
|
||||
logger.error("markErrorIfInFlight failed", {
|
||||
repoId,
|
||||
err: serializeError(e),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,13 +64,16 @@ export async function recoverStuckPreparing() {
|
||||
}
|
||||
}
|
||||
await markErrorIfInFlight(doc.repoId, "preparation_interrupted");
|
||||
console.log("[QUEUE] recovered stuck repo", doc.repoId);
|
||||
logger.info("recovered stuck repo", { repoId: doc.repoId });
|
||||
} catch (e) {
|
||||
console.log("[QUEUE] recover error for", doc.repoId, e);
|
||||
logger.warn("recover failed", {
|
||||
repoId: doc.repoId,
|
||||
err: serializeError(e),
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.log("[QUEUE] recoverStuckPreparing failed", e);
|
||||
logger.error("recoverStuckPreparing failed", serializeError(e));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -140,18 +149,17 @@ export function startWorker() {
|
||||
if (!downloadWorker.isRunning()) downloadWorker.run();
|
||||
|
||||
downloadWorker.on("active", async (job) => {
|
||||
console.log("[QUEUE] download repository start", job.data.repoId);
|
||||
logger.info("download start", { repoId: job.data.repoId });
|
||||
});
|
||||
downloadWorker.on("completed", async (job) => {
|
||||
console.log("[QUEUE] download repository completed", job.data.repoId);
|
||||
logger.info("download completed", { repoId: job.data.repoId });
|
||||
});
|
||||
downloadWorker.on("failed", async (job, err) => {
|
||||
const repoId = job?.data?.repoId;
|
||||
console.log(
|
||||
"[QUEUE] download repository failed",
|
||||
logger.error("download failed", {
|
||||
repoId,
|
||||
err?.message || err
|
||||
);
|
||||
err: serializeError(err),
|
||||
});
|
||||
if (!repoId) return;
|
||||
if (job && typeof job.attemptsMade === "number" && job.opts?.attempts) {
|
||||
if (job.attemptsMade < job.opts.attempts) return;
|
||||
|
||||
@@ -4,6 +4,9 @@ config();
|
||||
import { getRepository as getRepositoryImport } from "../../server/database";
|
||||
import { RepositoryStatus } from "../../core/types";
|
||||
import { RepoJobData } from "../index";
|
||||
import { createLogger, serializeError } from "../../core/logger";
|
||||
|
||||
const logger = createLogger("queue:download");
|
||||
|
||||
export default async function (job: SandboxedJob<RepoJobData, void>) {
|
||||
const {
|
||||
@@ -13,7 +16,7 @@ export default async function (job: SandboxedJob<RepoJobData, void>) {
|
||||
connect: () => Promise<void>;
|
||||
getRepository: typeof getRepositoryImport;
|
||||
} = require("../../server/database");
|
||||
console.log(`[QUEUE] ${job.data.repoId} is going to be downloaded`);
|
||||
logger.info("queued for download", { repoId: job.data.repoId });
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
let statusInterval: any = null;
|
||||
await connect();
|
||||
@@ -37,9 +40,10 @@ export default async function (job: SandboxedJob<RepoJobData, void>) {
|
||||
repo.status &&
|
||||
repo.model.statusMessage !== progress?.status
|
||||
) {
|
||||
console.log(
|
||||
`[QUEUE] Progress: ${job.data.repoId} ${progress.status}`
|
||||
);
|
||||
logger.debug("progress", {
|
||||
repoId: job.data.repoId,
|
||||
status: progress.status,
|
||||
});
|
||||
await repo.updateStatus(repo.status, progress?.status || "");
|
||||
}
|
||||
} catch {
|
||||
@@ -60,7 +64,7 @@ export default async function (job: SandboxedJob<RepoJobData, void>) {
|
||||
clearInterval(statusInterval);
|
||||
if (tickPromise) await tickPromise;
|
||||
await repo.updateStatus(RepositoryStatus.READY, "");
|
||||
console.log(`[QUEUE] ${job.data.repoId} is downloaded`);
|
||||
logger.info("downloaded", { repoId: job.data.repoId });
|
||||
} catch (error) {
|
||||
clearInterval(statusInterval);
|
||||
if (tickPromise) await tickPromise;
|
||||
@@ -79,17 +83,20 @@ export default async function (job: SandboxedJob<RepoJobData, void>) {
|
||||
await tickPromise;
|
||||
} catch { /* ignored */ }
|
||||
}
|
||||
console.log(`[QUEUE] ${job.data.repoId} is finished with an error`, error);
|
||||
logger.error("finished with error", {
|
||||
repoId: job.data.repoId,
|
||||
err: serializeError(error),
|
||||
});
|
||||
try {
|
||||
await repo.updateStatus(
|
||||
RepositoryStatus.ERROR,
|
||||
error instanceof Error ? error.message : String(error)
|
||||
);
|
||||
} catch (persistError) {
|
||||
console.log(
|
||||
`[QUEUE] failed to persist ERROR status for ${job.data.repoId}`,
|
||||
persistError
|
||||
);
|
||||
logger.error("failed to persist ERROR status", {
|
||||
repoId: job.data.repoId,
|
||||
err: serializeError(persistError),
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import { SandboxedJob } from "bullmq";
|
||||
import { getRepository as getRepositoryImport } from "../../server/database";
|
||||
import { RepoJobData } from "../index";
|
||||
import { createLogger } from "../../core/logger";
|
||||
|
||||
const logger = createLogger("queue:cache");
|
||||
|
||||
export default async function (job: SandboxedJob<RepoJobData, void>) {
|
||||
const {
|
||||
@@ -12,14 +15,12 @@ export default async function (job: SandboxedJob<RepoJobData, void>) {
|
||||
} = require("../../server/database");
|
||||
try {
|
||||
await connect();
|
||||
console.log(
|
||||
`[QUEUE] Cache of ${job.data.repoId} is going to be removed...`
|
||||
);
|
||||
logger.info("removing cache", { repoId: job.data.repoId });
|
||||
const repo = await getRepository(job.data.repoId);
|
||||
await repo.removeCache();
|
||||
} catch {
|
||||
// error already handled
|
||||
} finally {
|
||||
console.log(`[QUEUE] Cache of ${job.data.repoId} is removed.`);
|
||||
logger.info("cache removed", { repoId: job.data.repoId });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,9 @@ import { SandboxedJob } from "bullmq";
|
||||
import { getRepository as getRepositoryImport } from "../../server/database";
|
||||
import { RepositoryStatus } from "../../core/types";
|
||||
import { RepoJobData } from "../index";
|
||||
import { createLogger } from "../../core/logger";
|
||||
|
||||
const logger = createLogger("queue:remove");
|
||||
|
||||
export default async function (job: SandboxedJob<RepoJobData, void>) {
|
||||
const {
|
||||
@@ -13,7 +16,7 @@ export default async function (job: SandboxedJob<RepoJobData, void>) {
|
||||
} = require("../../server/database");
|
||||
try {
|
||||
await connect();
|
||||
console.log(`[QUEUE] ${job.data.repoId} is going to be removed`);
|
||||
logger.info("removing repository", { repoId: job.data.repoId });
|
||||
const repo = await getRepository(job.data.repoId);
|
||||
await repo.updateStatus(RepositoryStatus.REMOVING, "");
|
||||
try {
|
||||
@@ -29,6 +32,6 @@ export default async function (job: SandboxedJob<RepoJobData, void>) {
|
||||
} catch {
|
||||
// error already handled
|
||||
} finally {
|
||||
console.log(`[QUEUE] ${job.data.repoId} is removed`);
|
||||
logger.info("repository removed", { repoId: job.data.repoId });
|
||||
}
|
||||
}
|
||||
|
||||
+14
-9
@@ -20,6 +20,9 @@ import { startWorker, recoverStuckPreparing } from "../queue";
|
||||
import AnonymizedPullRequestModel from "../core/model/anonymizedPullRequests/anonymizedPullRequests.model";
|
||||
import { getUser } from "./routes/route-utils";
|
||||
import config from "../config";
|
||||
import { createLogger, serializeError } from "../core/logger";
|
||||
|
||||
const logger = createLogger("server");
|
||||
|
||||
function indexResponse(req: express.Request, res: express.Response) {
|
||||
if (
|
||||
@@ -67,7 +70,9 @@ export default async function start() {
|
||||
port: config.REDIS_PORT,
|
||||
},
|
||||
});
|
||||
redisClient.on("error", (err) => console.log("Redis Client Error", err));
|
||||
redisClient.on("error", (err) =>
|
||||
logger.error("redis client error", serializeError(err))
|
||||
);
|
||||
|
||||
await redisClient.connect();
|
||||
|
||||
@@ -79,7 +84,7 @@ export default async function start() {
|
||||
return request.headers["cf-connecting-ip"] as string;
|
||||
}
|
||||
if (!request.ip && request.socket.remoteAddress) {
|
||||
console.error("Warning: request.ip is missing!");
|
||||
logger.warn("request.ip is missing");
|
||||
return request.socket.remoteAddress;
|
||||
}
|
||||
// remove port number from IPv4 addresses
|
||||
@@ -136,12 +141,12 @@ export default async function start() {
|
||||
const start = Date.now();
|
||||
res.on("finish", function () {
|
||||
const time = Date.now() - start;
|
||||
console.log(
|
||||
`${req.method} ${res.statusCode} ${join(
|
||||
req.baseUrl || "",
|
||||
req.url || ""
|
||||
)} ${time}ms`
|
||||
);
|
||||
logger.info("request", {
|
||||
method: req.method,
|
||||
status: res.statusCode,
|
||||
url: join(req.baseUrl || "", req.url || ""),
|
||||
ms: time,
|
||||
});
|
||||
});
|
||||
next();
|
||||
});
|
||||
@@ -252,7 +257,7 @@ export default async function start() {
|
||||
await connect();
|
||||
await recoverStuckPreparing();
|
||||
app.listen(config.PORT);
|
||||
console.log("Database connected and Server started on port: " + config.PORT);
|
||||
logger.info("server started", { port: config.PORT });
|
||||
}
|
||||
|
||||
start();
|
||||
|
||||
@@ -10,6 +10,30 @@ import { ensureAuthenticated } from "./connection";
|
||||
import { handleError, getUser, isOwnerOrAdmin, getRepo } from "./route-utils";
|
||||
import adminTokensRouter from "./admin-tokens";
|
||||
import { octokit, getToken } from "../../core/GitHubUtils";
|
||||
import { createLogger, serializeError, ERROR_LOG_KEY, ERROR_LOG_MAX } from "../../core/logger";
|
||||
import { createClient, RedisClientType } from "redis";
|
||||
import config from "../../config";
|
||||
|
||||
const logger = createLogger("admin");
|
||||
|
||||
let errorLogClient: RedisClientType | null = null;
|
||||
async function getErrorLogClient(): Promise<RedisClientType | null> {
|
||||
if (errorLogClient && errorLogClient.isOpen) return errorLogClient;
|
||||
try {
|
||||
errorLogClient = createClient({
|
||||
socket: {
|
||||
host: config.REDIS_HOSTNAME,
|
||||
port: config.REDIS_PORT,
|
||||
},
|
||||
}) as RedisClientType;
|
||||
errorLogClient.on("error", () => undefined);
|
||||
await errorLogClient.connect();
|
||||
return errorLogClient;
|
||||
} catch (err) {
|
||||
logger.error("error log redis connect failed", serializeError(err));
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
@@ -203,6 +227,39 @@ router.get("/queues", async (req, res) => {
|
||||
});
|
||||
});
|
||||
|
||||
// Errors captured by the logger sink (last ERROR_LOG_MAX entries).
|
||||
router.get("/errors", async (req, res) => {
|
||||
try {
|
||||
const client = await getErrorLogClient();
|
||||
if (!client) {
|
||||
return res.json({ entries: [], max: ERROR_LOG_MAX, available: false });
|
||||
}
|
||||
const raw = await client.lRange(ERROR_LOG_KEY, 0, ERROR_LOG_MAX - 1);
|
||||
const entries = raw.map((s) => {
|
||||
try {
|
||||
return JSON.parse(s);
|
||||
} catch {
|
||||
return { ts: null, module: null, message: s, raw: [] };
|
||||
}
|
||||
});
|
||||
res.json({ entries, max: ERROR_LOG_MAX, available: true });
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
});
|
||||
|
||||
router.delete("/errors", async (req, res) => {
|
||||
try {
|
||||
const client = await getErrorLogClient();
|
||||
if (!client) return res.json({ ok: true, cleared: 0 });
|
||||
const len = await client.lLen(ERROR_LOG_KEY);
|
||||
await client.del(ERROR_LOG_KEY);
|
||||
res.json({ ok: true, cleared: len });
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
}
|
||||
});
|
||||
|
||||
// Global stats endpoint: counts by status, total disk, recent failures
|
||||
router.get("/stats", async (req, res) => {
|
||||
try {
|
||||
@@ -538,7 +595,9 @@ router.get(
|
||||
localField: "repositories",
|
||||
});
|
||||
if (!model) {
|
||||
req.logout((error) => console.error(error));
|
||||
req.logout((error) =>
|
||||
logger.error("logout failed", serializeError(error))
|
||||
);
|
||||
throw new AnonymousError("user_not_found", {
|
||||
httpStatus: 404,
|
||||
});
|
||||
@@ -556,7 +615,9 @@ router.get(
|
||||
try {
|
||||
const model = await UserModel.findOne({ username: req.params.username });
|
||||
if (!model) {
|
||||
req.logout((error) => console.error(error));
|
||||
req.logout((error) =>
|
||||
logger.error("logout failed", serializeError(error))
|
||||
);
|
||||
throw new AnonymousError("user_not_found", {
|
||||
httpStatus: 404,
|
||||
});
|
||||
|
||||
@@ -12,6 +12,9 @@ import { IUserDocument } from "../../core/model/users/users.types";
|
||||
import AnonymousError from "../../core/AnonymousError";
|
||||
import AnonymizedPullRequestModel from "../../core/model/anonymizedPullRequests/anonymizedPullRequests.model";
|
||||
import { hashToken } from "./token-auth";
|
||||
import { createLogger, serializeError } from "../../core/logger";
|
||||
|
||||
const logger = createLogger("auth");
|
||||
|
||||
export function ensureAuthenticated(
|
||||
req: express.Request,
|
||||
@@ -97,7 +100,7 @@ const verify = async (
|
||||
user,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
logger.error("verify failed", serializeError(error));
|
||||
done(
|
||||
new AnonymousError("unable_to_connect_user", {
|
||||
httpStatus: 500,
|
||||
@@ -135,7 +138,9 @@ export function initSession() {
|
||||
host: config.REDIS_HOSTNAME,
|
||||
},
|
||||
});
|
||||
redisClient.on("error", (err) => console.log("Redis Client Error", err));
|
||||
redisClient.on("error", (err) =>
|
||||
logger.error("redis client error", serializeError(err))
|
||||
);
|
||||
redisClient.connect();
|
||||
const redisStore = new RedisStore({
|
||||
client: redisClient,
|
||||
@@ -200,7 +205,7 @@ router.all(
|
||||
};
|
||||
req.login(synthUser, (err) => {
|
||||
if (err) {
|
||||
console.error("[login-token] req.login failed", err);
|
||||
logger.error("login-token req.login failed", serializeError(err));
|
||||
return res.status(500).json({ error: "login_failed" });
|
||||
}
|
||||
UserModel.updateOne(
|
||||
@@ -211,7 +216,7 @@ router.all(
|
||||
return res.json({ ok: true, username: model.username });
|
||||
});
|
||||
} catch (err) {
|
||||
console.error("[login-token] error", err);
|
||||
logger.error("login-token failed", serializeError(err));
|
||||
res.status(500).json({ error: "server_error" });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import * as express from "express";
|
||||
|
||||
import { getGist, handleError } from "./route-utils";
|
||||
import { getGist, getUser, handleError } from "./route-utils";
|
||||
import AnonymousError from "../../core/AnonymousError";
|
||||
import User from "../../core/User";
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
@@ -12,13 +13,22 @@ router.get(
|
||||
res.header("Cache-Control", "no-cache");
|
||||
const gist = await getGist(req, res, { nocheck: true });
|
||||
if (!gist) return;
|
||||
|
||||
let user: User | undefined = undefined;
|
||||
try {
|
||||
user = await getUser(req);
|
||||
} catch { /* not logged in */ }
|
||||
const canEdit =
|
||||
!!user && (user.isAdmin || user.id == gist.model.owner);
|
||||
|
||||
let redirectURL = null;
|
||||
if (
|
||||
!canEdit &&
|
||||
gist.status == "expired" &&
|
||||
gist.options.expirationMode == "redirect"
|
||||
) {
|
||||
redirectURL = `https://gist.github.com/${gist.source.gistId}`;
|
||||
} else {
|
||||
} else if (!canEdit) {
|
||||
if (
|
||||
gist.status == "expired" ||
|
||||
gist.status == "expiring" ||
|
||||
@@ -60,6 +70,8 @@ router.get(
|
||||
res.json({
|
||||
url: redirectURL,
|
||||
lastUpdateDate: gist.model.statusDate,
|
||||
isAdmin: user?.isAdmin === true,
|
||||
isOwner: user?.id == gist.model.owner,
|
||||
});
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import * as express from "express";
|
||||
|
||||
import { getPullRequest, handleError } from "./route-utils";
|
||||
import { getPullRequest, getUser, handleError } from "./route-utils";
|
||||
import AnonymousError from "../../core/AnonymousError";
|
||||
import User from "../../core/User";
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
@@ -12,10 +13,18 @@ router.get(
|
||||
res.header("Cache-Control", "no-cache");
|
||||
const pr = await getPullRequest(req, res, { nocheck: true });
|
||||
if (!pr) return;
|
||||
|
||||
let user: User | undefined = undefined;
|
||||
try {
|
||||
user = await getUser(req);
|
||||
} catch { /* not logged in */ }
|
||||
const canEdit =
|
||||
!!user && (user.isAdmin || user.id == pr.model.owner);
|
||||
|
||||
let redirectURL = null;
|
||||
if (pr.status == "expired" && pr.options.expirationMode == "redirect") {
|
||||
if (!canEdit && pr.status == "expired" && pr.options.expirationMode == "redirect") {
|
||||
redirectURL = `https://github.com/${pr.source.repositoryFullName}/pull/${pr.source.pullRequestId}`;
|
||||
} else {
|
||||
} else if (!canEdit) {
|
||||
if (
|
||||
pr.status == "expired" ||
|
||||
pr.status == "expiring" ||
|
||||
@@ -60,6 +69,8 @@ router.get(
|
||||
res.json({
|
||||
url: redirectURL,
|
||||
lastUpdateDate: pr.model.statusDate,
|
||||
isAdmin: user?.isAdmin === true,
|
||||
isOwner: user?.id == pr.model.owner,
|
||||
});
|
||||
} catch (error) {
|
||||
handleError(error, res, req);
|
||||
|
||||
@@ -22,6 +22,9 @@ import User from "../../core/User";
|
||||
import { RepositoryStatus } from "../../core/types";
|
||||
import { IUserDocument } from "../../core/model/users/users.types";
|
||||
import { checkToken, octokit } from "../../core/GitHubUtils";
|
||||
import { createLogger, serializeError } from "../../core/logger";
|
||||
|
||||
const logger = createLogger("route:repo");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
@@ -55,7 +58,7 @@ async function getTokenForAdmin(user: User, req: express.Request) {
|
||||
return existingRepo.source.accessToken;
|
||||
}
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
logger.warn("getToken lookup failed", serializeError(error));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -116,7 +119,10 @@ router.post("/claim", async (req: express.Request, res: express.Response) => {
|
||||
});
|
||||
}
|
||||
|
||||
console.log(`${user.username} claims ${r.repository}.`);
|
||||
logger.info("repo claimed", {
|
||||
user: user.username,
|
||||
repo: r.repository,
|
||||
});
|
||||
repoConfig.owner = user;
|
||||
|
||||
await AnonymizedRepositoryModel.updateOne(
|
||||
|
||||
@@ -3,7 +3,7 @@ import config from "../../config";
|
||||
import got from "got";
|
||||
import { join } from "path";
|
||||
|
||||
import { getRepo, getUser, handleError } from "./route-utils";
|
||||
import { getRepo, getUser, handleError, isCoauthor } from "./route-utils";
|
||||
import AnonymousError from "../../core/AnonymousError";
|
||||
import { downloadQueue } from "../../queue";
|
||||
import { RepositoryStatus } from "../../core/types";
|
||||
@@ -150,14 +150,26 @@ router.get(
|
||||
nocheck: true,
|
||||
});
|
||||
if (!repo) return;
|
||||
|
||||
let user: User | undefined = undefined;
|
||||
try {
|
||||
user = await getUser(req);
|
||||
} catch { /* not logged in */ }
|
||||
const canEdit =
|
||||
!!user &&
|
||||
(user.isAdmin ||
|
||||
user.id == repo.model.owner ||
|
||||
isCoauthor(repo, user));
|
||||
|
||||
let redirectURL = null;
|
||||
if (
|
||||
!canEdit &&
|
||||
repo.status == RepositoryStatus.EXPIRED &&
|
||||
repo.options.expirationMode == "redirect" &&
|
||||
repo.model.source.repositoryName
|
||||
) {
|
||||
redirectURL = `https://github.com/${repo.model.source.repositoryName}`;
|
||||
} else {
|
||||
} else if (!canEdit) {
|
||||
if (
|
||||
repo.status == RepositoryStatus.EXPIRED ||
|
||||
repo.status == RepositoryStatus.EXPIRING ||
|
||||
@@ -207,11 +219,6 @@ router.get(
|
||||
if (!!config.ENABLE_DOWNLOAD && !!config.STREAMER_ENTRYPOINT) {
|
||||
download = true;
|
||||
}
|
||||
|
||||
let user: User | undefined = undefined;
|
||||
try {
|
||||
user = await getUser(req);
|
||||
} catch { /* not logged in */ }
|
||||
res.json({
|
||||
url: redirectURL,
|
||||
download: download || user?.isAdmin === true,
|
||||
|
||||
@@ -6,6 +6,9 @@ import User from "../../core/User";
|
||||
import Repository from "../../core/Repository";
|
||||
import { HTTPError } from "got";
|
||||
import { RepositoryStatus } from "../../core/types";
|
||||
import { createLogger, serializeError } from "../../core/logger";
|
||||
|
||||
const logger = createLogger("route");
|
||||
|
||||
export async function getGist(
|
||||
req: express.Request,
|
||||
@@ -114,24 +117,18 @@ export function isOwnerCoauthorOrAdmin(repo: Repository, user: User) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
function printError(error: any, req?: express.Request) {
|
||||
if (error instanceof AnonymousError) {
|
||||
let message = `[ERROR] ${error.toString()} ${error.stack
|
||||
?.split("\n")[1]
|
||||
.trim()}`;
|
||||
if (req) {
|
||||
message += ` ${req.originalUrl}`;
|
||||
// ignore common error
|
||||
if (req.originalUrl === "/api/repo/undefined/options") return;
|
||||
}
|
||||
console.error(message);
|
||||
if (req?.originalUrl === "/api/repo/undefined/options") return;
|
||||
logger.error("anonymous error", {
|
||||
...serializeError(error),
|
||||
url: req?.originalUrl,
|
||||
});
|
||||
} else if (error instanceof HTTPError) {
|
||||
const message = `[ERROR] HTTP.${
|
||||
error.code
|
||||
} ${error.message.toString()} ${error.stack?.split("\n")[1].trim()}`;
|
||||
console.error(message);
|
||||
} else if (error instanceof Error) {
|
||||
console.error(error);
|
||||
logger.error("http error", {
|
||||
code: error.code,
|
||||
message: error.message,
|
||||
});
|
||||
} else {
|
||||
console.error(error);
|
||||
logger.error("unhandled error", serializeError(error));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,7 +169,7 @@ export async function getUser(req: express.Request) {
|
||||
function notConnected(): never {
|
||||
req.logout((error) => {
|
||||
if (error) {
|
||||
console.error(`[ERROR] Error while logging out: ${error}`);
|
||||
logger.error("logout failed", serializeError(error));
|
||||
}
|
||||
});
|
||||
throw new AnonymousError("not_connected", {
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import * as express from "express";
|
||||
import * as crypto from "crypto";
|
||||
import UserModel from "../../core/model/users/users.model";
|
||||
import { createLogger, serializeError } from "../../core/logger";
|
||||
|
||||
const logger = createLogger("token-auth");
|
||||
|
||||
export function hashToken(token: string): string {
|
||||
return crypto.createHash("sha256").update(token).digest("hex");
|
||||
@@ -38,9 +41,11 @@ export async function bearerTokenAuth(
|
||||
UserModel.updateOne(
|
||||
{ _id: model._id, "apiTokens.tokenHash": tokenHash },
|
||||
{ $set: { "apiTokens.$.lastUsedAt": new Date() } }
|
||||
).catch((err) => console.error("[token-auth] lastUsedAt update failed", err));
|
||||
).catch((err) =>
|
||||
logger.error("lastUsedAt update failed", serializeError(err))
|
||||
);
|
||||
} catch (err) {
|
||||
console.error("[token-auth] lookup failed", err);
|
||||
logger.error("lookup failed", serializeError(err));
|
||||
}
|
||||
return next();
|
||||
}
|
||||
|
||||
@@ -7,6 +7,9 @@ import User from "../../core/User";
|
||||
import FileModel from "../../core/model/files/files.model";
|
||||
import { isConnected } from "../database";
|
||||
import { octokit } from "../../core/GitHubUtils";
|
||||
import { createLogger, serializeError } from "../../core/logger";
|
||||
|
||||
const logger = createLogger("user");
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
@@ -17,7 +20,7 @@ router.get("/logout", async (req: express.Request, res: express.Response) => {
|
||||
try {
|
||||
req.logout((error) => {
|
||||
if (error) {
|
||||
console.error(`[ERROR] Logout error: ${error}`);
|
||||
logger.error("logout failed", serializeError(error));
|
||||
}
|
||||
});
|
||||
res.redirect("/");
|
||||
|
||||
@@ -3,6 +3,9 @@ import Conference from "../core/Conference";
|
||||
import AnonymizedRepositoryModel from "../core/model/anonymizedRepositories/anonymizedRepositories.model";
|
||||
import ConferenceModel from "../core/model/conference/conferences.model";
|
||||
import Repository from "../core/Repository";
|
||||
import { createLogger, serializeError } from "../core/logger";
|
||||
|
||||
const logger = createLogger("schedule");
|
||||
|
||||
export function conferenceStatusCheck() {
|
||||
// check every 6 hours the status of the conferences
|
||||
@@ -14,7 +17,7 @@ export function conferenceStatusCheck() {
|
||||
try {
|
||||
await conference.expire();
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
logger.error("conference expire failed", serializeError(error));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -25,7 +28,7 @@ export function conferenceStatusCheck() {
|
||||
export function repositoryStatusCheck() {
|
||||
// check every 6 hours the status of the repositories
|
||||
schedule.scheduleJob("0 */6 * * *", async () => {
|
||||
console.log("[schedule] Check repository status and unused repositories");
|
||||
logger.info("checking repository status and unused repositories");
|
||||
(
|
||||
await AnonymizedRepositoryModel.find({
|
||||
status: { $eq: "ready" },
|
||||
@@ -36,16 +39,16 @@ export function repositoryStatusCheck() {
|
||||
try {
|
||||
repo.check();
|
||||
} catch {
|
||||
console.log(`Repository ${repo.repoId} is expired`);
|
||||
logger.info("repository expired", { repoId: repo.repoId });
|
||||
}
|
||||
const fourMonthAgo = new Date();
|
||||
fourMonthAgo.setMonth(fourMonthAgo.getMonth() - 4);
|
||||
|
||||
if (repo.model.lastView < fourMonthAgo) {
|
||||
repo.removeCache().then(() => {
|
||||
console.log(
|
||||
`Repository ${repo.repoId} not visited for 4 months remove the cached files`
|
||||
);
|
||||
logger.info("removed cache for unused repository", {
|
||||
repoId: repo.repoId,
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
@@ -8,6 +8,9 @@ import config from "../config";
|
||||
import router from "./route";
|
||||
import { handleError } from "../server/routes/route-utils";
|
||||
import AnonymousError from "../core/AnonymousError";
|
||||
import { createLogger } from "../core/logger";
|
||||
|
||||
const logger = createLogger("streamer");
|
||||
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
@@ -31,5 +34,5 @@ app.all("*", (req, res) => {
|
||||
);
|
||||
});
|
||||
app.listen(config.PORT, () => {
|
||||
console.log(`Server started on http://streamer:${config.PORT}`);
|
||||
logger.info("streamer started", { port: config.PORT });
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user