From cde9214579775f52b67c9e1c6f65d23421521d27 Mon Sep 17 00:00:00 2001 From: tdurieux Date: Tue, 24 Aug 2021 09:35:48 +0200 Subject: [PATCH] fix configuration issue during anonymization --- config.ts | 2 +- public/partials/anonymize.htm | 2 +- public/script/app.js | 42 ++++++++++++++++------ src/routes/index.ts | 2 ++ src/routes/option.ts | 14 ++++++++ src/routes/repository-private.ts | 62 +++++++++++++++++--------------- src/routes/repository-public.ts | 3 +- src/server.ts | 1 + 8 files changed, 87 insertions(+), 41 deletions(-) create mode 100644 src/routes/option.ts diff --git a/config.ts b/config.ts index 6c545d9..5ee5563 100644 --- a/config.ts +++ b/config.ts @@ -35,7 +35,7 @@ const config: Config = { GITHUB_TOKEN: "", DEFAULT_QUOTA: 2 * 1024 * 1024 * 1024 * 8, MAX_FILE_SIZE: 10 * 1024 * 1024, // in b - MAX_REPO_SIZE: 8 * 1024, // in kb + MAX_REPO_SIZE: 500 * 8 * 1024, // in kb ENABLE_DOWNLOAD: false, AUTH_CALLBACK: "http://localhost:5000/github/auth", ANONYMIZATION_MASK: "XXXX", diff --git a/public/partials/anonymize.htm b/public/partials/anonymize.htm index 8e2a4e2..c1205e5 100644 --- a/public/partials/anonymize.htm +++ b/public/partials/anonymize.htm @@ -341,7 +341,7 @@ How the repository will be anonymized. Stream mode will request the content on the flight. This is the only - option for repositories bigger than 10mb. Download will + option for repositories bigger than {{site_options.MAX_REPO_SIZE * 1024| humanFileSize}}. Download will download the repository the repository on the anonymous.4open.science server, it is faster and offer more features. { + if (res) $scope.site_options = res.data; + }, + () => { + $scope.site_options = null; + } + ); + } + getOptions(); + function getMessage() { $http.get("/api/message").then( (res) => { @@ -435,7 +448,6 @@ angular notebook: true, loc: true, link: true, - mode: "GitHubDownload", }; function getDefault() { @@ -859,7 +871,7 @@ angular resetValidity(); const res = await $http.get(`/api/repo/${o.owner}/${o.repo}/`); $scope.details = res.data; - if ($scope.details.size > 500 * 1024 * 8) { + if ($scope.details.size > $scope.site_options.MAX_REPO_SIZE) { $scope.options.mode = "GitHubStream"; $scope.anonymize.mode.$$element[0].disabled = true; } @@ -902,41 +914,51 @@ angular ); } if (!$scope.options.link) { - content = content.replace(urlRegex, "XXXX"); + content = content.replace( + urlRegex, + $scope.site_options.ANONYMIZATION_MASK + ); } + const host = document.location.protocol + "//" + document.location.host; + content = content.replace( new RegExp( `\\b${$scope.repoUrl}/blob/${$scope.source.branch}\\b`, "gi" ), - `https://anonymous.4open.science/r/${$scope.repoId}` + `${host}/r/${$scope.repoId}` ); content = content.replace( new RegExp( `\\b${$scope.repoUrl}/tree/${$scope.source.branch}\\b`, "gi" ), - `https://anonymous.4open.science/r/${$scope.repoId}` + `${host}/r/${$scope.repoId}` ); content = content.replace( new RegExp(`\\b${$scope.repoUrl}`, "gi"), - `https://anonymous.4open.science/r/${$scope.repoId}` + `${host}/r/${$scope.repoId}` ); - - for (let term of $scope.terms.split("\n")) { + const terms = $scope.terms.split("\n"); + for (let i = 0; i < terms.length; i++) { + const term = terms[i]; if (term.trim() == "") { continue; } // remove whole url if it contains the term content = content.replace(urlRegex, (match) => { - if (new RegExp(`\\b${term}\\b`, "gi").test(match)) return "XXXX"; + if (new RegExp(`\\b${term}\\b`, "gi").test(match)) + return $scope.site_options.ANONYMIZATION_MASK + "-" + (i + 1); return match; }); // remove the term in the text - content = content.replace(new RegExp(`\\b${term}\\b`, "gi"), "XXXX"); + content = content.replace( + new RegExp(`\\b${term}\\b`, "gi"), + $scope.site_options.ANONYMIZATION_MASK + "-" + (i + 1) + ); } $scope.anonymize_readme = content; diff --git a/src/routes/index.ts b/src/routes/index.ts index f8b553a..220b90c 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -3,6 +3,7 @@ import repositoryPublic from "./repository-public"; import file from "./file"; import webview from "./webview"; import user from "./user"; +import option from "./option"; export default { repositoryPrivate, @@ -10,4 +11,5 @@ export default { file, webview, user, + option, }; diff --git a/src/routes/option.ts b/src/routes/option.ts new file mode 100644 index 0000000..06c04e2 --- /dev/null +++ b/src/routes/option.ts @@ -0,0 +1,14 @@ +import * as express from "express"; +import config from "../../config"; +export const router = express.Router(); + +router.get("/", async (req: express.Request, res: express.Response) => { + res.json({ + ENABLE_DOWNLOAD: config.ENABLE_DOWNLOAD, + MAX_FILE_SIZE: config.MAX_FILE_SIZE, + MAX_REPO_SIZE: config.MAX_REPO_SIZE, + ANONYMIZATION_MASK: config.ANONYMIZATION_MASK, + }); +}); + +export default router; diff --git a/src/routes/repository-private.ts b/src/routes/repository-private.ts index f56932a..a71974f 100644 --- a/src/routes/repository-private.ts +++ b/src/routes/repository-private.ts @@ -159,6 +159,22 @@ router.get( } ); +// get repository information +router.get("/:repoId/", async (req: express.Request, res: express.Response) => { + try { + const repo = await getRepo(req, res, { nocheck: true }); + if (!repo) throw new Error("repo_not_found"); + + const user = await getUser(req); + if (user.username != repo.model.owner) { + return res.status(401).send({ error: "not_authorized" }); + } + res.json((await db.getRepository(req.params.repoId)).toJSON()); + } catch (error) { + handleError(error, res); + } +}); + function validateNewRepo(repoUpdate) { const validCharacters = /^[0-9a-zA-Z\-\_]+$/; if ( @@ -184,7 +200,19 @@ function validateNewRepo(repoUpdate) { } } -function updateRepoModel(model: IAnonymizedRepositoryDocument, repoUpdate) { +function updateRepoModel( + model: IAnonymizedRepositoryDocument, + repoUpdate: any +) { + if (repoUpdate.source.type) { + model.source.type = repoUpdate.source.type; + if ( + model.source.type != "GitHubStream" && + model.source.type != "GitHubDownload" + ) { + model.source.type = "GitHubStream"; + } + } model.source.commit = repoUpdate.source.commit; model.source.branch = repoUpdate.source.branch; model.conference = repoUpdate.conference; @@ -204,22 +232,6 @@ function updateRepoModel(model: IAnonymizedRepositoryDocument, repoUpdate) { }; } -// get repository information -router.get("/:repoId/", async (req: express.Request, res: express.Response) => { - try { - const repo = await getRepo(req, res, { nocheck: true }); - if (!repo) throw new Error("repo_not_found"); - - const user = await getUser(req); - if (user.username != repo.model.owner) { - return res.status(401).send({ error: "not_authorized" }); - } - res.json((await db.getRepository(req.params.repoId)).toJSON()); - } catch (error) { - handleError(error, res); - } -}); - // update a repository router.post( "/:repoId/", @@ -273,15 +285,11 @@ router.post("/", async (req: express.Request, res: express.Response) => { repo.repoId = repoUpdate.repoId; repo.anonymizeDate = new Date(); repo.owner = user.username; - repo.source = { - type: - repoUpdate.options.mode == "download" - ? "GitHubDownload" - : "GitHubStream", - accessToken: user.accessToken, - repositoryId: repository.model.id, - repositoryName: repoUpdate.fullName, - }; + + updateRepoModel(repo, repoUpdate); + repo.source.accessToken = user.accessToken; + repo.source.repositoryId = repository.model.id; + repo.source.repositoryName = repoUpdate.fullName; if (repo.source.type == "GitHubDownload") { // details.size is in kilobytes @@ -290,8 +298,6 @@ router.post("/", async (req: express.Request, res: express.Response) => { } } - updateRepoModel(repo, repoUpdate); - await repo.save(); res.send("ok"); new Repository(repo).anonymize(); diff --git a/src/routes/repository-public.ts b/src/routes/repository-public.ts index 17f5d5f..9d59dcf 100644 --- a/src/routes/repository-public.ts +++ b/src/routes/repository-public.ts @@ -64,7 +64,8 @@ router.get( res.json({ url: redirectURL, - download: !!config.ENABLE_DOWNLOAD, + download: + !!config.ENABLE_DOWNLOAD && repo.source.type == "GitHubDownload", }); } catch (error) { handleError(error, res); diff --git a/src/server.ts b/src/server.ts index 761824b..e293a9f 100644 --- a/src/server.ts +++ b/src/server.ts @@ -57,6 +57,7 @@ export default async function start() { app.use("/github", rate, connection.router); // api routes + app.use("/api/options", rate, router.option); app.use("/api/user", rate, router.user); app.use("/api/repo", rate, router.repositoryPublic); app.use("/api/repo", rate, router.file);