error logging improvement, regex fix

src/server/index.ts

@@ -20,6 +20,9 @@ import { startWorker, recoverStuckPreparing } from "../queue";
 import AnonymizedPullRequestModel from "../core/model/anonymizedPullRequests/anonymizedPullRequests.model";
 import { getUser } from "./routes/route-utils";
 import config from "../config";
+import { createLogger, serializeError } from "../core/logger";
+
+const logger = createLogger("server");
 
 function indexResponse(req: express.Request, res: express.Response) {
   if (
@@ -67,7 +70,9 @@ export default async function start() {
       port: config.REDIS_PORT,
     },
   });
-  redisClient.on("error", (err) => console.log("Redis Client Error", err));
+  redisClient.on("error", (err) =>
+    logger.error("redis client error", serializeError(err))
+  );
 
   await redisClient.connect();
 
@@ -79,7 +84,7 @@ export default async function start() {
       return request.headers["cf-connecting-ip"] as string;
     }
     if (!request.ip && request.socket.remoteAddress) {
-      console.error("Warning: request.ip is missing!");
+      logger.warn("request.ip is missing");
       return request.socket.remoteAddress;
     }
     // remove port number from IPv4 addresses
@@ -136,12 +141,12 @@ export default async function start() {
     const start = Date.now();
     res.on("finish", function () {
       const time = Date.now() - start;
-      console.log(
-        `${req.method} ${res.statusCode} ${join(
-          req.baseUrl || "",
-          req.url || ""
-        )} ${time}ms`
-      );
+      logger.info("request", {
+        method: req.method,
+        status: res.statusCode,
+        url: join(req.baseUrl || "", req.url || ""),
+        ms: time,
+      });
     });
     next();
   });
@@ -252,7 +257,7 @@ export default async function start() {
   await connect();
   await recoverStuckPreparing();
   app.listen(config.PORT);
-  console.log("Database connected and Server started on port: " + config.PORT);
+  logger.info("server started", { port: config.PORT });
 }
 
 start();

src/server/routes/admin.ts

@@ -10,6 +10,30 @@ import { ensureAuthenticated } from "./connection";
 import { handleError, getUser, isOwnerOrAdmin, getRepo } from "./route-utils";
 import adminTokensRouter from "./admin-tokens";
 import { octokit, getToken } from "../../core/GitHubUtils";
+import { createLogger, serializeError, ERROR_LOG_KEY, ERROR_LOG_MAX } from "../../core/logger";
+import { createClient, RedisClientType } from "redis";
+import config from "../../config";
+
+const logger = createLogger("admin");
+
+let errorLogClient: RedisClientType | null = null;
+async function getErrorLogClient(): Promise<RedisClientType | null> {
+  if (errorLogClient && errorLogClient.isOpen) return errorLogClient;
+  try {
+    errorLogClient = createClient({
+      socket: {
+        host: config.REDIS_HOSTNAME,
+        port: config.REDIS_PORT,
+      },
+    }) as RedisClientType;
+    errorLogClient.on("error", () => undefined);
+    await errorLogClient.connect();
+    return errorLogClient;
+  } catch (err) {
+    logger.error("error log redis connect failed", serializeError(err));
+    return null;
+  }
+}
 
 const router = express.Router();
 
@@ -203,6 +227,39 @@ router.get("/queues", async (req, res) => {
   });
 });
 
+// Errors captured by the logger sink (last ERROR_LOG_MAX entries).
+router.get("/errors", async (req, res) => {
+  try {
+    const client = await getErrorLogClient();
+    if (!client) {
+      return res.json({ entries: [], max: ERROR_LOG_MAX, available: false });
+    }
+    const raw = await client.lRange(ERROR_LOG_KEY, 0, ERROR_LOG_MAX - 1);
+    const entries = raw.map((s) => {
+      try {
+        return JSON.parse(s);
+      } catch {
+        return { ts: null, module: null, message: s, raw: [] };
+      }
+    });
+    res.json({ entries, max: ERROR_LOG_MAX, available: true });
+  } catch (error) {
+    handleError(error, res, req);
+  }
+});
+
+router.delete("/errors", async (req, res) => {
+  try {
+    const client = await getErrorLogClient();
+    if (!client) return res.json({ ok: true, cleared: 0 });
+    const len = await client.lLen(ERROR_LOG_KEY);
+    await client.del(ERROR_LOG_KEY);
+    res.json({ ok: true, cleared: len });
+  } catch (error) {
+    handleError(error, res, req);
+  }
+});
+
 // Global stats endpoint: counts by status, total disk, recent failures
 router.get("/stats", async (req, res) => {
   try {
@@ -538,7 +595,9 @@ router.get(
         localField: "repositories",
       });
       if (!model) {
-        req.logout((error) => console.error(error));
+        req.logout((error) =>
+          logger.error("logout failed", serializeError(error))
+        );
         throw new AnonymousError("user_not_found", {
           httpStatus: 404,
         });
@@ -556,7 +615,9 @@ router.get(
     try {
       const model = await UserModel.findOne({ username: req.params.username });
       if (!model) {
-        req.logout((error) => console.error(error));
+        req.logout((error) =>
+          logger.error("logout failed", serializeError(error))
+        );
         throw new AnonymousError("user_not_found", {
           httpStatus: 404,
         });

src/server/routes/connection.ts

@@ -12,6 +12,9 @@ import { IUserDocument } from "../../core/model/users/users.types";
 import AnonymousError from "../../core/AnonymousError";
 import AnonymizedPullRequestModel from "../../core/model/anonymizedPullRequests/anonymizedPullRequests.model";
 import { hashToken } from "./token-auth";
+import { createLogger, serializeError } from "../../core/logger";
+
+const logger = createLogger("auth");
 
 export function ensureAuthenticated(
   req: express.Request,
@@ -97,7 +100,7 @@ const verify = async (
       user,
     });
   } catch (error) {
-    console.error(error);
+    logger.error("verify failed", serializeError(error));
     done(
       new AnonymousError("unable_to_connect_user", {
         httpStatus: 500,
@@ -135,7 +138,9 @@ export function initSession() {
       host: config.REDIS_HOSTNAME,
     },
   });
-  redisClient.on("error", (err) => console.log("Redis Client Error", err));
+  redisClient.on("error", (err) =>
+    logger.error("redis client error", serializeError(err))
+  );
   redisClient.connect();
   const redisStore = new RedisStore({
     client: redisClient,
@@ -200,7 +205,7 @@ router.all(
       };
       req.login(synthUser, (err) => {
         if (err) {
-          console.error("[login-token] req.login failed", err);
+          logger.error("login-token req.login failed", serializeError(err));
           return res.status(500).json({ error: "login_failed" });
         }
         UserModel.updateOne(
@@ -211,7 +216,7 @@ router.all(
         return res.json({ ok: true, username: model.username });
       });
     } catch (err) {
-      console.error("[login-token] error", err);
+      logger.error("login-token failed", serializeError(err));
       res.status(500).json({ error: "server_error" });
     }
   }

src/server/routes/gist-public.ts

@@ -1,7 +1,8 @@
 import * as express from "express";
 
-import { getGist, handleError } from "./route-utils";
+import { getGist, getUser, handleError } from "./route-utils";
 import AnonymousError from "../../core/AnonymousError";
+import User from "../../core/User";
 
 const router = express.Router();
 
@@ -12,13 +13,22 @@ router.get(
       res.header("Cache-Control", "no-cache");
       const gist = await getGist(req, res, { nocheck: true });
       if (!gist) return;
+
+      let user: User | undefined = undefined;
+      try {
+        user = await getUser(req);
+      } catch { /* not logged in */ }
+      const canEdit =
+        !!user && (user.isAdmin || user.id == gist.model.owner);
+
       let redirectURL = null;
       if (
+        !canEdit &&
         gist.status == "expired" &&
         gist.options.expirationMode == "redirect"
       ) {
         redirectURL = `https://gist.github.com/${gist.source.gistId}`;
-      } else {
+      } else if (!canEdit) {
         if (
           gist.status == "expired" ||
           gist.status == "expiring" ||
@@ -60,6 +70,8 @@ router.get(
       res.json({
         url: redirectURL,
         lastUpdateDate: gist.model.statusDate,
+        isAdmin: user?.isAdmin === true,
+        isOwner: user?.id == gist.model.owner,
       });
     } catch (error) {
       handleError(error, res, req);

src/server/routes/pullRequest-public.ts

@@ -1,7 +1,8 @@
 import * as express from "express";
 
-import { getPullRequest, handleError } from "./route-utils";
+import { getPullRequest, getUser, handleError } from "./route-utils";
 import AnonymousError from "../../core/AnonymousError";
+import User from "../../core/User";
 
 const router = express.Router();
 
@@ -12,10 +13,18 @@ router.get(
       res.header("Cache-Control", "no-cache");
       const pr = await getPullRequest(req, res, { nocheck: true });
       if (!pr) return;
+
+      let user: User | undefined = undefined;
+      try {
+        user = await getUser(req);
+      } catch { /* not logged in */ }
+      const canEdit =
+        !!user && (user.isAdmin || user.id == pr.model.owner);
+
       let redirectURL = null;
-      if (pr.status == "expired" && pr.options.expirationMode == "redirect") {
+      if (!canEdit && pr.status == "expired" && pr.options.expirationMode == "redirect") {
         redirectURL = `https://github.com/${pr.source.repositoryFullName}/pull/${pr.source.pullRequestId}`;
-      } else {
+      } else if (!canEdit) {
         if (
           pr.status == "expired" ||
           pr.status == "expiring" ||
@@ -60,6 +69,8 @@ router.get(
       res.json({
         url: redirectURL,
         lastUpdateDate: pr.model.statusDate,
+        isAdmin: user?.isAdmin === true,
+        isOwner: user?.id == pr.model.owner,
       });
     } catch (error) {
       handleError(error, res, req);

src/server/routes/repository-private.ts

@@ -22,6 +22,9 @@ import User from "../../core/User";
 import { RepositoryStatus } from "../../core/types";
 import { IUserDocument } from "../../core/model/users/users.types";
 import { checkToken, octokit } from "../../core/GitHubUtils";
+import { createLogger, serializeError } from "../../core/logger";
+
+const logger = createLogger("route:repo");
 
 const router = express.Router();
 
@@ -55,7 +58,7 @@ async function getTokenForAdmin(user: User, req: express.Request) {
         return existingRepo.source.accessToken;
       }
     } catch (error) {
-      console.log(error);
+      logger.warn("getToken lookup failed", serializeError(error));
     }
   }
 }
@@ -116,7 +119,10 @@ router.post("/claim", async (req: express.Request, res: express.Response) => {
       });
     }
 
-    console.log(`${user.username} claims ${r.repository}.`);
+    logger.info("repo claimed", {
+      user: user.username,
+      repo: r.repository,
+    });
     repoConfig.owner = user;
 
     await AnonymizedRepositoryModel.updateOne(

src/server/routes/repository-public.ts

@@ -3,7 +3,7 @@ import config from "../../config";
 import got from "got";
 import { join } from "path";
 
-import { getRepo, getUser, handleError } from "./route-utils";
+import { getRepo, getUser, handleError, isCoauthor } from "./route-utils";
 import AnonymousError from "../../core/AnonymousError";
 import { downloadQueue } from "../../queue";
 import { RepositoryStatus } from "../../core/types";
@@ -150,14 +150,26 @@ router.get(
         nocheck: true,
       });
       if (!repo) return;
+
+      let user: User | undefined = undefined;
+      try {
+        user = await getUser(req);
+      } catch { /* not logged in */ }
+      const canEdit =
+        !!user &&
+        (user.isAdmin ||
+          user.id == repo.model.owner ||
+          isCoauthor(repo, user));
+
       let redirectURL = null;
       if (
+        !canEdit &&
         repo.status == RepositoryStatus.EXPIRED &&
         repo.options.expirationMode == "redirect" &&
         repo.model.source.repositoryName
       ) {
         redirectURL = `https://github.com/${repo.model.source.repositoryName}`;
-      } else {
+      } else if (!canEdit) {
         if (
           repo.status == RepositoryStatus.EXPIRED ||
           repo.status == RepositoryStatus.EXPIRING ||
@@ -207,11 +219,6 @@ router.get(
       if (!!config.ENABLE_DOWNLOAD && !!config.STREAMER_ENTRYPOINT) {
         download = true;
       }
-
-      let user: User | undefined = undefined;
-      try {
-        user = await getUser(req);
-      } catch { /* not logged in */ }
       res.json({
         url: redirectURL,
         download: download || user?.isAdmin === true,

src/server/routes/route-utils.ts

@@ -6,6 +6,9 @@ import User from "../../core/User";
 import Repository from "../../core/Repository";
 import { HTTPError } from "got";
 import { RepositoryStatus } from "../../core/types";
+import { createLogger, serializeError } from "../../core/logger";
+
+const logger = createLogger("route");
 
 export async function getGist(
   req: express.Request,
@@ -114,24 +117,18 @@ export function isOwnerCoauthorOrAdmin(repo: Repository, user: User) {
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 function printError(error: any, req?: express.Request) {
   if (error instanceof AnonymousError) {
-    let message = `[ERROR] ${error.toString()} ${error.stack
-      ?.split("\n")[1]
-      .trim()}`;
-    if (req) {
-      message += ` ${req.originalUrl}`;
-      // ignore common error
-      if (req.originalUrl === "/api/repo/undefined/options") return;
-    }
-    console.error(message);
+    if (req?.originalUrl === "/api/repo/undefined/options") return;
+    logger.error("anonymous error", {
+      ...serializeError(error),
+      url: req?.originalUrl,
+    });
   } else if (error instanceof HTTPError) {
-    const message = `[ERROR] HTTP.${
-      error.code
-    } ${error.message.toString()}  ${error.stack?.split("\n")[1].trim()}`;
-    console.error(message);
-  } else if (error instanceof Error) {
-    console.error(error);
+    logger.error("http error", {
+      code: error.code,
+      message: error.message,
+    });
   } else {
-    console.error(error);
+    logger.error("unhandled error", serializeError(error));
   }
 }
 
@@ -172,7 +169,7 @@ export async function getUser(req: express.Request) {
   function notConnected(): never {
     req.logout((error) => {
       if (error) {
-        console.error(`[ERROR] Error while logging out: ${error}`);
+        logger.error("logout failed", serializeError(error));
       }
     });
     throw new AnonymousError("not_connected", {

src/server/routes/token-auth.ts

@@ -1,6 +1,9 @@
 import * as express from "express";
 import * as crypto from "crypto";
 import UserModel from "../../core/model/users/users.model";
+import { createLogger, serializeError } from "../../core/logger";
+
+const logger = createLogger("token-auth");
 
 export function hashToken(token: string): string {
   return crypto.createHash("sha256").update(token).digest("hex");
@@ -38,9 +41,11 @@ export async function bearerTokenAuth(
     UserModel.updateOne(
       { _id: model._id, "apiTokens.tokenHash": tokenHash },
       { $set: { "apiTokens.$.lastUsedAt": new Date() } }
-    ).catch((err) => console.error("[token-auth] lastUsedAt update failed", err));
+    ).catch((err) =>
+      logger.error("lastUsedAt update failed", serializeError(err))
+    );
   } catch (err) {
-    console.error("[token-auth] lookup failed", err);
+    logger.error("lookup failed", serializeError(err));
   }
   return next();
 }

src/server/routes/user.ts

@@ -7,6 +7,9 @@ import User from "../../core/User";
 import FileModel from "../../core/model/files/files.model";
 import { isConnected } from "../database";
 import { octokit } from "../../core/GitHubUtils";
+import { createLogger, serializeError } from "../../core/logger";
+
+const logger = createLogger("user");
 
 const router = express.Router();
 
@@ -17,7 +20,7 @@ router.get("/logout", async (req: express.Request, res: express.Response) => {
   try {
     req.logout((error) => {
       if (error) {
-        console.error(`[ERROR] Logout error: ${error}`);
+        logger.error("logout failed", serializeError(error));
       }
     });
     res.redirect("/");

src/server/schedule.ts

@@ -3,6 +3,9 @@ import Conference from "../core/Conference";
 import AnonymizedRepositoryModel from "../core/model/anonymizedRepositories/anonymizedRepositories.model";
 import ConferenceModel from "../core/model/conference/conferences.model";
 import Repository from "../core/Repository";
+import { createLogger, serializeError } from "../core/logger";
+
+const logger = createLogger("schedule");
 
 export function conferenceStatusCheck() {
   // check every 6 hours the status of the conferences
@@ -14,7 +17,7 @@ export function conferenceStatusCheck() {
           try {
             await conference.expire();
           } catch (error) {
-            console.error(error);
+            logger.error("conference expire failed", serializeError(error));
           }
         }
       }
@@ -25,7 +28,7 @@ export function conferenceStatusCheck() {
 export function repositoryStatusCheck() {
   // check every 6 hours the status of the repositories
   schedule.scheduleJob("0 */6 * * *", async () => {
-    console.log("[schedule] Check repository status and unused repositories");
+    logger.info("checking repository status and unused repositories");
     (
       await AnonymizedRepositoryModel.find({
         status: { $eq: "ready" },
@@ -36,16 +39,16 @@ export function repositoryStatusCheck() {
       try {
         repo.check();
       } catch {
-        console.log(`Repository ${repo.repoId} is expired`);
+        logger.info("repository expired", { repoId: repo.repoId });
       }
       const fourMonthAgo = new Date();
       fourMonthAgo.setMonth(fourMonthAgo.getMonth() - 4);
 
       if (repo.model.lastView < fourMonthAgo) {
         repo.removeCache().then(() => {
-          console.log(
-            `Repository ${repo.repoId} not visited for 4 months remove the cached files`
-          );
+          logger.info("removed cache for unused repository", {
+            repoId: repo.repoId,
+          });
         });
       }
     });