From 532c094388b9a8d9d18ef87a93b4c35c51d7ef5d Mon Sep 17 00:00:00 2001 From: tdurieux Date: Tue, 18 Jun 2024 12:00:45 +0200 Subject: [PATCH] fix: improve token management --- docker-compose.yml | 3 +- opentelemetry.js | 2 +- src/core/GitHubUtils.ts | 73 ++++++++++++++++++++-------- src/core/Repository.ts | 8 ++- src/core/model/users/users.schema.ts | 3 ++ src/core/model/users/users.types.ts | 3 ++ src/server/index.ts | 16 +++--- src/server/routes/connection.ts | 7 +++ src/streamer/route.ts | 7 +++ 9 files changed, 90 insertions(+), 32 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7d6323d..7f2f70d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,13 +37,14 @@ services: mode: replicated replicas: 4 endpoint_mode: dnsrr - entrypoint: ["node", "./build/streamer/index.js"] + entrypoint: ["node", "--require", "./opentelemetry.js", "./build/streamer/index.js"] env_file: - ./.env volumes: - ./repositories:/app/repositories/ environment: - PORT=5000 + - SERVICE_NAME=Streamer healthcheck: test: - CMD diff --git a/opentelemetry.js b/opentelemetry.js index a121474..70a94e8 100644 --- a/opentelemetry.js +++ b/opentelemetry.js @@ -14,7 +14,7 @@ const { diag, DiagConsoleLogger, DiagLogLevel } = require("@opentelemetry/api"); // diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO); const sdk = new opentelemetry.NodeSDK({ - serviceName: "Anonymous-GitHub", + serviceName: process.env.SERVICE_NAME || "Anonymous-GitHub", logRecordProcessor: getNodeAutoInstrumentations().logRecordProcessor, traceExporter: new OTLPTraceExporter({ url: "http://opentelemetry:4317/v1/traces", diff --git a/src/core/GitHubUtils.ts b/src/core/GitHubUtils.ts index e85b8d2..841505e 100644 --- a/src/core/GitHubUtils.ts +++ b/src/core/GitHubUtils.ts @@ -29,35 +29,68 @@ export async function getToken(repository: Repository) { span.setAttribute("repoId", repository.repoId); try { // only check the token if the repo has been visited more than one day ago - if ( - repository.model.source.accessToken && - repository.model.lastView > new Date(Date.now() - 1000 * 60 * 60 * 24) - ) { - return repository.model.source.accessToken; - } + // if ( + // repository.model.source.accessToken && + // repository.model.lastView > new Date(Date.now() - 1000 * 60 * 60 * 24) + // ) { + // return repository.model.source.accessToken; + // } if (repository.model.source.accessToken) { if (await checkToken(repository.model.source.accessToken)) { return repository.model.source.accessToken; } } if (!repository.owner.model.accessTokens?.github) { - const accessTokens = ( - await UserModel.findById(repository.owner.id, { - accessTokens: 1, - }) - )?.accessTokens; - if (accessTokens) { - repository.owner.model.accessTokens = accessTokens; + const query = await UserModel.findById(repository.owner.id, { + accessTokens: 1, + accessTokenDates: 1, + }); + if (query?.accessTokens) { + repository.owner.model.accessTokens = query.accessTokens; + repository.owner.model.accessTokenDates = query.accessTokenDates; } } - if (repository.owner.model.accessTokens?.github) { - const check = await checkToken( - repository.owner.model.accessTokens?.github - ); + const ownerAccessToken = repository.owner.model.accessTokens?.github; + if (ownerAccessToken) { + const tokenAge = repository.owner.model.accessTokenDates?.github; + if (!tokenAge || tokenAge < new Date(Date.now() - 1000 * 60 * 60 * 24)) { + const url = `https://api.github.com/applications/${config.CLIENT_ID}/token`; + const headers = { + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }; + + const res = await fetch(url, { + method: "PATCH", + body: JSON.stringify({ + access_token: ownerAccessToken, + }), + credentials: "include", + headers: { + ...headers, + Authorization: + "Basic " + + Buffer.from( + config.CLIENT_ID + ":" + config.CLIENT_SECRET + ).toString("base64"), + }, + }); + const resBody = (await res.json()) as { token: string }; + repository.owner.model.accessTokens.github = resBody.token; + if (!repository.owner.model.accessTokenDates) { + repository.owner.model.accessTokenDates = { + github: new Date(), + }; + } else { + repository.owner.model.accessTokenDates.github = new Date(); + } + await repository.owner.model.save(); + return resBody.token; + } + const check = await checkToken(ownerAccessToken); if (check) { - repository.model.source.accessToken = - repository.owner.model.accessTokens?.github; - return repository.owner.model.accessTokens?.github; + repository.model.source.accessToken = ownerAccessToken; + return ownerAccessToken; } } return config.GITHUB_TOKEN; diff --git a/src/core/Repository.ts b/src/core/Repository.ts index d37ce1b..5ecfcc3 100644 --- a/src/core/Repository.ts +++ b/src/core/Repository.ts @@ -518,7 +518,11 @@ export default class Repository { await storage.rm(this.repoId); this.model.isReseted = true; if (isConnected) { - await this.model.save(); + try { + await this.model.save(); + } catch (error) { + console.error("[ERROR] removeCache save", error); + } } } finally { span.end(); @@ -542,7 +546,7 @@ export default class Repository { }> { const span = trace .getTracer("ano-file") - .startSpan("Repository.removeCache"); + .startSpan("Repository.computeSize"); span.setAttribute("repoId", this.repoId); try { if (this.status !== RepositoryStatus.READY) diff --git a/src/core/model/users/users.schema.ts b/src/core/model/users/users.schema.ts index 731d92f..c430bc1 100644 --- a/src/core/model/users/users.schema.ts +++ b/src/core/model/users/users.schema.ts @@ -4,6 +4,9 @@ const UserSchema = new Schema({ accessTokens: { github: { type: String }, }, + accessTokenDates: { + github: { type: Date }, + }, externalIDs: { github: { type: String, index: true }, }, diff --git a/src/core/model/users/users.types.ts b/src/core/model/users/users.types.ts index a9ab5e5..89e2aa4 100644 --- a/src/core/model/users/users.types.ts +++ b/src/core/model/users/users.types.ts @@ -4,6 +4,9 @@ export interface IUser { accessTokens: { github: string; }; + accessTokenDates?: { + github: Date; + }; externalIDs: { github: string; }; diff --git a/src/server/index.ts b/src/server/index.ts index 83e5971..012bea0 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -130,14 +130,6 @@ export default async function start() { keyGenerator, }); - app.use( - express.static(join("public"), { - etag: true, - lastModified: true, - maxAge: 3600, // 1h - }) - ); - app.use(function (req, res, next) { const start = Date.now(); res.on("finish", function () { @@ -216,6 +208,14 @@ export default async function start() { // web view app.use("/w/", rate, webViewSpeedLimiter, router.webview); + app.use( + express.static(join("public"), { + etag: true, + lastModified: true, + maxAge: 3600, // 1h + }) + ); + app .get("/", indexResponse) .get("/404", indexResponse) diff --git a/src/server/routes/connection.ts b/src/server/routes/connection.ts index 45f4401..c0891e8 100644 --- a/src/server/routes/connection.ts +++ b/src/server/routes/connection.ts @@ -50,6 +50,13 @@ const verify = async ( }); if (user.emails?.length) user.emails[0].default = true; } + if (!user.accessTokenDates) { + user.accessTokenDates = { + github: new Date(), + }; + } else { + user.accessTokenDates.github = new Date(); + } await user.save(); } catch (error) { console.error(error); diff --git a/src/streamer/route.ts b/src/streamer/route.ts index fb527b5..6376dfb 100644 --- a/src/streamer/route.ts +++ b/src/streamer/route.ts @@ -48,6 +48,13 @@ router.post( const archive = archiver("zip", {}); downloadStream + .on("error", (error) => { + console.error(error); + archive.finalize(); + }) + .on("close", () => { + archive.finalize(); + }) .pipe(Parse()) .on("entry", (entry) => { if (entry.type === "File") {