fix: improve token management

docker-compose.yml

@@ -37,13 +37,14 @@ services:
       mode: replicated
       replicas: 4
       endpoint_mode: dnsrr
-    entrypoint: ["node", "./build/streamer/index.js"]
+    entrypoint: ["node", "--require", "./opentelemetry.js", "./build/streamer/index.js"]
     env_file:
       - ./.env
     volumes:
       - ./repositories:/app/repositories/
     environment:
       - PORT=5000
+      - SERVICE_NAME=Streamer
     healthcheck:
       test:
         - CMD

opentelemetry.js

@@ -14,7 +14,7 @@ const { diag, DiagConsoleLogger, DiagLogLevel } = require("@opentelemetry/api");
 // diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO);
 
 const sdk = new opentelemetry.NodeSDK({
-  serviceName: "Anonymous-GitHub",
+  serviceName: process.env.SERVICE_NAME || "Anonymous-GitHub",
   logRecordProcessor: getNodeAutoInstrumentations().logRecordProcessor,
   traceExporter: new OTLPTraceExporter({
     url: "http://opentelemetry:4317/v1/traces",

src/core/GitHubUtils.ts

@@ -29,35 +29,68 @@ export async function getToken(repository: Repository) {
   span.setAttribute("repoId", repository.repoId);
   try {
     // only check the token if the repo has been visited more than one day ago
-    if (
-      repository.model.source.accessToken &&
-      repository.model.lastView > new Date(Date.now() - 1000 * 60 * 60 * 24)
-    ) {
-      return repository.model.source.accessToken;
-    }
+    // if (
+    //   repository.model.source.accessToken &&
+    //   repository.model.lastView > new Date(Date.now() - 1000 * 60 * 60 * 24)
+    // ) {
+    //   return repository.model.source.accessToken;
+    // }
     if (repository.model.source.accessToken) {
       if (await checkToken(repository.model.source.accessToken)) {
         return repository.model.source.accessToken;
       }
     }
     if (!repository.owner.model.accessTokens?.github) {
-      const accessTokens = (
-        await UserModel.findById(repository.owner.id, {
-          accessTokens: 1,
-        })
-      )?.accessTokens;
-      if (accessTokens) {
-        repository.owner.model.accessTokens = accessTokens;
+      const query = await UserModel.findById(repository.owner.id, {
+        accessTokens: 1,
+        accessTokenDates: 1,
+      });
+      if (query?.accessTokens) {
+        repository.owner.model.accessTokens = query.accessTokens;
+        repository.owner.model.accessTokenDates = query.accessTokenDates;
       }
     }
-    if (repository.owner.model.accessTokens?.github) {
-      const check = await checkToken(
-        repository.owner.model.accessTokens?.github
-      );
+    const ownerAccessToken = repository.owner.model.accessTokens?.github;
+    if (ownerAccessToken) {
+      const tokenAge = repository.owner.model.accessTokenDates?.github;
+      if (!tokenAge || tokenAge < new Date(Date.now() - 1000 * 60 * 60 * 24)) {
+        const url = `https://api.github.com/applications/${config.CLIENT_ID}/token`;
+        const headers = {
+          Accept: "application/vnd.github+json",
+          "X-GitHub-Api-Version": "2022-11-28",
+        };
+
+        const res = await fetch(url, {
+          method: "PATCH",
+          body: JSON.stringify({
+            access_token: ownerAccessToken,
+          }),
+          credentials: "include",
+          headers: {
+            ...headers,
+            Authorization:
+              "Basic " +
+              Buffer.from(
+                config.CLIENT_ID + ":" + config.CLIENT_SECRET
+              ).toString("base64"),
+          },
+        });
+        const resBody = (await res.json()) as { token: string };
+        repository.owner.model.accessTokens.github = resBody.token;
+        if (!repository.owner.model.accessTokenDates) {
+          repository.owner.model.accessTokenDates = {
+            github: new Date(),
+          };
+        } else {
+          repository.owner.model.accessTokenDates.github = new Date();
+        }
+        await repository.owner.model.save();
+        return resBody.token;
+      }
+      const check = await checkToken(ownerAccessToken);
       if (check) {
-        repository.model.source.accessToken =
-          repository.owner.model.accessTokens?.github;
-        return repository.owner.model.accessTokens?.github;
+        repository.model.source.accessToken = ownerAccessToken;
+        return ownerAccessToken;
       }
     }
     return config.GITHUB_TOKEN;

src/core/Repository.ts

@@ -518,7 +518,11 @@ export default class Repository {
       await storage.rm(this.repoId);
       this.model.isReseted = true;
       if (isConnected) {
-        await this.model.save();
+        try {
+          await this.model.save();
+        } catch (error) {
+          console.error("[ERROR] removeCache save", error);
+        }
       }
     } finally {
       span.end();
@@ -542,7 +546,7 @@ export default class Repository {
   }> {
     const span = trace
       .getTracer("ano-file")
-      .startSpan("Repository.removeCache");
+      .startSpan("Repository.computeSize");
     span.setAttribute("repoId", this.repoId);
     try {
       if (this.status !== RepositoryStatus.READY)

src/core/model/users/users.schema.ts

@@ -4,6 +4,9 @@ const UserSchema = new Schema({
   accessTokens: {
     github: { type: String },
   },
+  accessTokenDates: {
+    github: { type: Date },
+  },
   externalIDs: {
     github: { type: String, index: true },
   },

src/core/model/users/users.types.ts

@@ -4,6 +4,9 @@ export interface IUser {
   accessTokens: {
     github: string;
   };
+  accessTokenDates?: {
+    github: Date;
+  };
   externalIDs: {
     github: string;
   };

src/server/index.ts

@@ -130,14 +130,6 @@ export default async function start() {
     keyGenerator,
   });
 
-  app.use(
-    express.static(join("public"), {
-      etag: true,
-      lastModified: true,
-      maxAge: 3600, // 1h
-    })
-  );
-
   app.use(function (req, res, next) {
     const start = Date.now();
     res.on("finish", function () {
@@ -216,6 +208,14 @@ export default async function start() {
   // web view
   app.use("/w/", rate, webViewSpeedLimiter, router.webview);
 
+  app.use(
+    express.static(join("public"), {
+      etag: true,
+      lastModified: true,
+      maxAge: 3600, // 1h
+    })
+  );
+
   app
     .get("/", indexResponse)
     .get("/404", indexResponse)

src/server/routes/connection.ts

@@ -50,6 +50,13 @@ const verify = async (
       });
       if (user.emails?.length) user.emails[0].default = true;
     }
+    if (!user.accessTokenDates) {
+      user.accessTokenDates = {
+        github: new Date(),
+      };
+    } else {
+      user.accessTokenDates.github = new Date();
+    }
     await user.save();
   } catch (error) {
     console.error(error);

src/streamer/route.ts

@@ -48,6 +48,13 @@ router.post(
 
       const archive = archiver("zip", {});
       downloadStream
+        .on("error", (error) => {
+          console.error(error);
+          archive.finalize();
+        })
+        .on("close", () => {
+          archive.finalize();
+        })
         .pipe(Parse())
         .on("entry", (entry) => {
           if (entry.type === "File") {