chore(deps): bump tmp and inquirer

Removes [tmp](https://github.com/raszi/node-tmp). It's no longer used after updating ancestor dependency [inquirer](https://github.com/SBoudrias/Inquirer.js). These dependencies need to be updated together.


Removes `tmp`

Updates `inquirer` from 8.2.6 to 8.2.7
- [Release notes](https://github.com/SBoudrias/Inquirer.js/releases)
- [Commits](https://github.com/SBoudrias/Inquirer.js/compare/inquirer@8.2.6...inquirer@8.2.7)

---
updated-dependencies:
- dependency-name: tmp
  dependency-version: 
  dependency-type: indirect
- dependency-name: inquirer
  dependency-version: 8.2.7
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>

Dockerfile

@@ -1,25 +1,21 @@
-FROM node:18-slim
+FROM node:21-slim
 
 ENV PORT 5000
 EXPOSE $PORT
 
 WORKDIR /app
 
-RUN npm install pm2 -g && pm2 install typescript && npm cache clean --force;
-
 COPY package.json .
 COPY package-lock.json .
 
+COPY gulpfile.js .
 COPY tsconfig.json .
-COPY ecosystem.config.js .
 COPY healthcheck.js .
 
-COPY src ./src
 COPY public ./public
-COPY index.ts .
-COPY config.ts .
+COPY src ./src
 
 RUN npm install && npm run build && npm cache clean --force
+COPY opentelemetry.js .
 
-
-CMD [ "pm2-runtime", "ecosystem.config.js"]
+CMD [ "node", "--require", "./opentelemetry.js", "./build/server/index.js"]

README.md

@@ -73,7 +73,7 @@ In double-anonymous peer-review, the boundary of anonymization is the paper plus
 
 ## How does it work?
 
-Anonymous Github either download the complete repository and anonymize the content of the file or proxy the request to GitHub. In both case, the original and anonymized versions of the file are cached on the server.
+Anonymous Github either downloads the complete repository and anonymizes the content of the file or proxies the request to GitHub. In both cases, the original and anonymized versions of the file are cached on the server.
 
 ## Related tools
 

docker-compose.yml

@@ -1,17 +1,21 @@
-version: "3"
+version: "3.8"
 
 services:
   anonymous_github:
     build: .
     restart: always
     image: tdurieux/anonymous_github:v2
+    ports:
+      - $EXPOSED_PORT:5000
     env_file:
       - ./.env
-    environment: 
+    volumes:
+      - ./repositories:/app/repositories/
+    environment:
+      - PORT=5000
       - REDIS_HOSTNAME=redis
       - DB_HOSTNAME=mongodb
-    ports:
-      - $PORT:$PORT
+      - STREAMER_ENTRYPOINT=http://streamer:5000/
     healthcheck:
       test:
         - CMD
@@ -20,13 +24,41 @@ services:
       interval: 10s
       timeout: 10s
       retries: 5
-    links:
+    depends_on:
       - mongodb
       - redis
+      - streamer
+
+  streamer:
+    build: .
+    restart: always
+    image: tdurieux/anonymous_github:v2
+    deploy:
+      mode: replicated
+      replicas: 4
+      endpoint_mode: dnsrr
+    entrypoint: ["node", "--require", "./opentelemetry.js", "./build/streamer/index.js"]
+    env_file:
+      - ./.env
+    volumes:
+      - ./repositories:/app/repositories/
+    environment:
+      - PORT=5000
+      - SERVICE_NAME=Streamer
+    healthcheck:
+      test:
+        - CMD
+        - node
+        - healthcheck.js
+      interval: 10s
+      timeout: 10s
+      retries: 5
 
   redis:
     image: "redis:alpine"
     restart: always
+    ports:
+      - 127.0.0.1:6379:6379
     healthcheck:
       test:
         - CMD
@@ -35,7 +67,7 @@ services:
       interval: 10s
       timeout: 10s
       retries: 5
-  
+
   mongodb:
     image: mongo:latest
     restart: on-failure
@@ -44,28 +76,56 @@ services:
       MONGO_INITDB_ROOT_PASSWORD: $DB_PASSWORD
     volumes:
       - mongodb_data_container:/data/db
+    ports:
+      - 127.0.0.1:27017:27017
     command: --quiet
     healthcheck:
       test:
         - CMD
-        - mongo
+        - mongosh
         - --eval
         - "db.adminCommand('ping')"
       interval: 10s
       timeout: 10s
       retries: 5
 
+  opentelemetry:
+    image: otel/opentelemetry-collector
+    restart: always
+    command: ["--config=/etc/otel-collector-config.yaml"]
+    volumes:
+      - ./opentelemetry-collector.yml:/etc/otel-collector-config.yaml
+    depends_on:
+      - jaeger
+      - prometheus
+
+  jaeger:
+    image: jaegertracing/all-in-one:latest
+    restart: always
+    ports:
+      - 127.0.0.1:16686:16686
+
+  prometheus:
+    image: prom/prometheus:latest
+    restart: always
+    volumes:
+      - ./prometheus.yaml:/etc/prometheus/prometheus.yml
+    ports:
+      - 127.0.0.1:9090:9090
+
   mongodb-backup:
     image: tiredofit/db-backup
     links:
       - mongodb
+    env_file:
+      - ./.env
     volumes:
       - ./db_backups:/backup
     environment:
       - DB_TYPE=mongo
       - DB_HOST=mongodb
-      - DB_DUMP_FREQ=60
-      - DB_CLEANUP_TIME=240
+      - DB_DUMP_FREQ=120
+      - DB_CLEANUP_TIME=500
       - COMPRESSION=XZ
       - DB_USER=$DB_USERNAME
       - DB_PASS=$DB_PASSWORD

ecosystem.config.js

@@ -1,20 +0,0 @@
-module.exports = {
-  apps: [
-    {
-      name: "AnonymousGitHub",
-      script: "build/index.js",
-      exec_mode: "fork",
-      watch: false,
-      ignore_watch: [
-        "node_modules",
-        "repositories",
-        "repo",
-        "public",
-        ".git",
-        "db_backups",
-        "build",
-      ],
-      interpreter: "node",
-    },
-  ],
-};

gulpfile.js

@@ -0,0 +1,59 @@
+const { src, dest } = require("gulp");
+const uglify = require("gulp-uglify");
+const concat = require("gulp-concat");
+var order = require("gulp-order");
+const cleanCss = require("gulp-clean-css");
+
+function defaultTask(cb) {
+  const jsFiles = [
+    "public/script/external/angular.min.js",
+    "public/script/external/angular-translate.min.js",
+    "public/script/external/angular-translate-loader-static-files.min.js",
+    "public/script/external/angular-sanitize.min.js",
+    "public/script/external/angular-route.min.js",
+    "public/script/external/pdf.compat.js",
+    "public/script/external/pdf.js",
+    "public/script/external/github-emojis.js",
+    "public/script/external/marked-emoji.js",
+    "public/script/external/marked.min.js",
+    "public/script/external/purify.min.js",
+    "public/script/external/ansi_up.min.js",
+    "public/script/external/prism.min.js",
+    "public/script/external/katex.min.js",
+    "public/script/external/katex-auto-render.min.js",
+    "public/script/external/marked-katex-extension.umd.min.js",
+    "public/script/external/notebook.min.js",
+    "public/script/external/org.js",
+    "public/script/external/jquery-3.4.1.min.js",
+    "public/script/external/popper.min.js",
+    "public/script/external/bootstrap.min.js",
+    "public/script/external/ace/ace.js",
+    "public/script/external/ui-ace.min.js",
+    "public/script/utils.js",
+    "public/script/ng-pdfviewer.min.js",
+    "public/script/app.js",
+    "public/script/admin.js",
+  ];
+  const cssFiles = [
+    "public/css/bootstrap.min.css",
+    "public/css/font-awesome.min.css",
+    "public/css/notebook.css",
+    "public/css/katex.min.css",
+    "public/css/github-markdown.min.css",
+    "public/css/style.css",
+  ];
+  src(jsFiles)
+    .pipe(order(jsFiles, { base: "./" }))
+    .pipe(concat("bundle.min.js"))
+    .pipe(uglify())
+    .pipe(dest("public/script"))
+    .on("end", cb);
+
+  src(cssFiles)
+    .pipe(order(cssFiles, { base: "./" }))
+    .pipe(concat("all.min.css"))
+    .pipe(cleanCss())
+    .pipe(dest("public/css"));
+}
+
+exports.default = defaultTask;

healthcheck.js

@@ -1,22 +1,26 @@
+require("dotenv").config();
 const http = require("http");
-const config = require("./config");
+const config = require("./build/config");
 
 const options = {
-  host: "localhost",
-  port: config.PORT,
+  path: "/healthcheck",
+  method: "GET",
+  host: "127.0.0.1",
+  port: config.default.PORT,
   timeout: 2000,
 };
-
 const request = http.request(options, (res) => {
-  if (res.statusCode == 200) {
+  if (res.statusCode == 200 || res.statusCode == 404) {
     process.exit(0);
   } else {
+    const reqURL = `${res.req.protocol}://${res.req.host}:${options.port}${res.req.path}`;
+    console.log(reqURL, res.statusCode);
     process.exit(1);
   }
 });
 
 request.on("error", (err) => {
-  console.log("ERROR");
+  console.log("ERROR", err);
   process.exit(1);
 });
 

import.js

@@ -1,153 +0,0 @@
-const fs = require("fs").promises;
-const ofs = require("fs");
-const path = require("path");
-const gh = require("parse-github-url");
-const { Octokit } = require("@octokit/rest");
-
-const config = require("./config");
-const db = require("./utils/database");
-const repoUtils = require("./utils/repository");
-const fileUtils = require("./utils/file");
-const githubUtils = require("./utils/github");
-
-// const ROOT = "./repositories";
-const ROOT = "./repo";
-(async () => {
-  await db.connect();
-  const repositories = await fs.readdir(ROOT);
-  let index = 0;
-  for (let repo of repositories) {
-    // for (let repo of ["14bfc5c6-b794-487e-a58a-c54103a93c7b"]) {
-    console.log("Import ", index++, "/", repositories.length, " ", repo);
-    try {
-      const conf = await repoUtils.getConfig(repo);
-      if (conf) {
-        continue;
-      }
-      // const repoPath = path.join("./repositories", repo);
-      const repoPath = path.join(ROOT, repo);
-      const configPath = path.join(repoPath, "config.json");
-      if (!ofs.existsSync(configPath)) {
-        continue;
-      }
-      const repoConfig = JSON.parse(await fs.readFile(configPath));
-      const r = gh(repoConfig.repository);
-      if (r == null) {
-        console.log(`${repoConfig.repository} is not a valid github url.`);
-        continue;
-      }
-      const fullName = `${r.owner}/${r.name}`;
-
-      // const octokit = new Octokit({ auth: config.GITHUB_TOKEN });
-      // try {
-      //   await octokit.apps.checkToken({
-      //     client_id: config.CLIENT_ID,
-      //     access_token: repoConfig.token,
-      //   });
-      // } catch (error) {
-      //   delete repoConfig.token;
-      //   continue
-      // }
-      let token = repoConfig.token;
-
-      if (!token) {
-        token = config.GITHUB_TOKEN;
-      }
-
-      const branches = await repoUtils.getRepoBranches({
-        fullName,
-        token,
-      });
-      const details = await repoUtils.getRepoDetails({
-        fullName,
-        token,
-      });
-      let branch = details.default_branch;
-      if (r.branch && branches[r.branch]) {
-        branch = r.branch;
-      }
-      if (!branches[branch]) {
-        console.log(branch, details.default_branch, branches);
-      }
-      let commit = branches[branch].commit.sha;
-      const anonymizeDate = new Date();
-
-      let mode = "stream";
-      // if (details.size < 1024) {
-      //   mode = "download";
-      // }
-
-      let expirationDate = null;
-      if (repoConfig.expiration_date) {
-        expirationDate = new Date(repoConfig.expiration_date["$date"]);
-      }
-      const expirationMode = repoConfig.expiration
-        ? repoConfig.expiration
-        : "never";
-
-      const repoConfiguration = {
-        repoId: repo,
-        fullName,
-        // owner: "tdurieux",
-        owner: r.owner,
-        terms: repoConfig.terms,
-        repository: repoConfig.repository,
-        token: repoConfig.token,
-        branch,
-        commit,
-        anonymizeDate,
-        options: {
-          image: false,
-          mode,
-          expirationMode,
-          expirationDate,
-          update: true,
-          page: details.has_pages,
-          pdf: false,
-          notebook: true,
-          loc: false,
-          link: true,
-        },
-      };
-      await db.get("anonymized_repositories").updateOne(
-        {
-          repoId: repo,
-        },
-        {
-          $set: repoConfiguration,
-        },
-        { upsert: true }
-      );
-      if (ofs.existsSync(repoUtils.getOriginalPath(repo))) {
-        await fs.rm(repoUtils.getOriginalPath(repo), {
-          recursive: true,
-          force: true,
-        });
-      }
-      if (ofs.existsSync(repoUtils.getAnonymizedPath(repo))) {
-        await fs.rm(repoUtils.getAnonymizedPath(repo), {
-          recursive: true,
-          force: true,
-        });
-      }
-      // await githubUtils.downloadRepoAndAnonymize(repoConfiguration);
-      // await fileUtils.getFileList({ repoConfig: repoConfiguration });
-      await repoUtils.updateStatus(repoConfiguration, "ready");
-      console.log(
-        expirationDate,
-        expirationDate != null && expirationDate < new Date(),
-        expirationDate < new Date()
-      );
-      if (
-        expirationMode != "never" &&
-        expirationDate != null &&
-        expirationDate < new Date()
-      ) {
-        await repoUtils.updateStatus(repoConfiguration, "expired");
-      }
-    } catch (error) {
-      console.error(error);
-    }
-  }
-  await db.close();
-})();

index.ts

@@ -1,7 +0,0 @@
-import { config } from "dotenv";
-config();
-
-import server from "./src/server";
-
-// start the server
-server();

migrateDB.ts

@@ -1,217 +0,0 @@
-require("dotenv").config();
-
-import * as mongoose from "mongoose";
-import config from "./config";
-import * as database from "./src/database/database";
-import RepositoryModel from "./src/database/repositories/repositories.model";
-import AnonymizedRepositoryModel from "./src/database/anonymizedRepositories/anonymizedRepositories.model";
-import UserModel from "./src/database/users/users.model";
-
-const MONGO_URL = `mongodb://${config.DB_USERNAME}:${config.DB_PASSWORD}@${config.DB_HOSTNAME}:27017/`;
-
-async function connect(db) {
-  const t = new mongoose.Mongoose();
-  t.set("useNewUrlParser", true);
-  t.set("useFindAndModify", true);
-  t.set("useUnifiedTopology", true);
-
-  const database = t.connection;
-
-  await t.connect(MONGO_URL + db, {
-    authSource: "admin",
-    useCreateIndex: true,
-    useFindAndModify: true,
-  });
-
-  return database;
-}
-
-(async () => {
-  await database.connect();
-  const oldDB = await connect("anonymous_github");
-
-  console.log("Import Users");
-  let index = 0;
-  const userQuery = oldDB.collection("users").find();
-  const totalUser = await userQuery.count();
-
-  while (await userQuery.hasNext()) {
-    const r = await userQuery.next();
-    index++;
-    console.log(`Import User [${index}/${totalUser}]: ${r.username}`);
-
-    const newRepos = [];
-    const allRepoIds = [];
-    if (r.repositories) {
-      const finds = await RepositoryModel.find({
-        externalId: {
-          $in: r.repositories.map((repo) => "gh_" + repo.id),
-        },
-      }).select("externalId");
-      finds.forEach((f) => allRepoIds.push(f.id));
-
-      const repoIds = new Set<string>();
-      const toInsert = r.repositories.filter((f) => {
-        if (repoIds.has(f.id)) return false;
-        repoIds.add(f.id);
-        const externalId = "gh_" + f.id;
-        return finds.filter((f) => f.externalId == externalId).length == 0;
-      });
-
-      for (const repo of toInsert) {
-        newRepos.push(
-          new RepositoryModel({
-            externalId: "gh_" + repo.id,
-            name: repo.full_name,
-            url: repo.html_url,
-            size: repo.size,
-            defaultBranch: repo.default_branch,
-          })
-        );
-      }
-      if (newRepos.length > 0) {
-        await RepositoryModel.insertMany(newRepos);
-      }
-      newRepos.forEach((f) => allRepoIds.push(f.id));
-    }
-    const user = new UserModel({
-      accessTokens: {
-        github: r.accessToken,
-      },
-      externalIDs: {
-        github: r.profile.id,
-      },
-      username: r.username,
-      emails: r.profile.emails?.map((email) => {
-        return { email: email.value, default: false };
-      }),
-      photo: r.profile.photos[0]?.value,
-      repositories: allRepoIds,
-      default: {
-        terms: r.default?.terms,
-        options: r.default?.options,
-      },
-    });
-    if (user.emails?.length) user.emails[0].default = true;
-
-    await user.save();
-  }
-
-  console.log("Import Repositories");
-  const repoQuery = oldDB.collection("repositories").find();
-  const totalRepository = await repoQuery.count();
-  index = 0;
-  while (await repoQuery.hasNext()) {
-    const r = await repoQuery.next();
-    if (!r.id) continue;
-    index++;
-    console.log(
-      `Import Repository [${index}/${totalRepository}]: ${r.fullName}`
-    );
-
-    let find = await RepositoryModel.findOne({
-      externalId: "gh_" + r.id,
-    });
-
-    if (find == null) {
-      find = new RepositoryModel({
-        externalId: "gh_" + r.id,
-        name: r.fullName,
-        url: r.html_url,
-        size: r.size,
-        defaultBranch: r.default_branch,
-      });
-    }
-    if (r.branches) {
-      const branches = [...Object.values(r.branches)].map((b: any) => {
-        const o: any = { name: b.name, commit: b.commit.sha };
-        if (b.name == find.defaultBranch) {
-          o.readme = r.readme;
-        }
-        return o;
-      });
-      find.branches = branches;
-    }
-    await find.save();
-  }
-
-  console.log("Import Anonymized Repositories");
-  const anoQuery = oldDB.collection("anonymized_repositories").find();
-  const totalAno = await anoQuery.count();
-  index = 0;
-  while (await anoQuery.hasNext()) {
-    const r = await anoQuery.next();
-
-    index++;
-    console.log(
-      `Import Anonymized Repository [${index}/${totalAno}]: ${r.repoId}`
-    );
-
-    let repo = await RepositoryModel.findOne({ name: r.fullName });
-    if (repo == null) {
-      const tmp = await oldDB
-        .collection("repositories")
-        .findOne({ fullName: r.fullName });
-      if (tmp) {
-        repo = await RepositoryModel.findOne({ externalId: "gh_" + tmp.id });
-      } else {
-        console.error(`Repository ${r.fullName} is not found (renamed)`);
-      }
-    }
-    let size = { storage: 0, file: 0 };
-    function recursiveCount(files) {
-      const out = { storage: 0, file: 0 };
-      for (const name in files) {
-        const file = files[name];
-        if (file.size && file.sha && parseInt(file.size) == file.size) {
-          out.storage += file.size as number;
-          out.file++;
-        } else if (typeof file == "object") {
-          const r = recursiveCount(file);
-          out.storage += r.storage;
-          out.file += r.file;
-        }
-      }
-      return out;
-    }
-
-    if (r.originalFiles) {
-      size = recursiveCount(r.originalFiles);
-    }
-    const owner = await UserModel.findOne({ username: r.owner }).select("_id");
-    await new AnonymizedRepositoryModel({
-      repoId: r.repoId,
-      status: r.status,
-      anonymizeDate: r.anonymizeDate,
-      lastView: r.lastView,
-      pageView: r.pageView,
-      owner: owner?.id,
-      size,
-      source: {
-        accessToken: r.token,
-        type: r.options.mode == "download" ? "GitHubDownload" : "GitHubStream",
-        branch: r.branch,
-        commit: r.commit,
-        repositoryId: repo?.id,
-        repositoryName: r.fullName,
-      },
-      options: {
-        terms: r.terms,
-        expirationMode: r.options.expirationMode,
-        expirationDate: r.options.expirationDate
-          ? new Date(r.options.expirationDate)
-          : null,
-        update: r.options.update,
-        image: r.options.image,
-        pdf: r.options.pdf,
-        notebook: r.options.notebook,
-        loc: r.options.loc,
-        link: r.options.link,
-        page: r.options.page,
-        pageSource: r.options.pageSource,
-      },
-    }).save();
-  }
-  console.log("Import finished!");
-  setTimeout(() => process.exit(), 5000);
-})();

opentelemetry-collector.yml

@@ -0,0 +1,40 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+
+exporters:
+  prometheus:
+    endpoint: "0.0.0.0:8889"
+    const_labels:
+      label1: value1
+
+  debug:
+
+  otlp:
+    endpoint: jaeger:4317
+    tls:
+      insecure: true
+
+processors:
+  batch:
+
+extensions:
+  health_check:
+  pprof:
+    endpoint: :1888
+  zpages:
+    endpoint: :55679
+
+service:
+  extensions: [health_check, pprof, zpages]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [debug, otlp]
+    metrics:
+      receivers: [otlp]
+      exporters: [debug, prometheus]
+    logs:
+      receivers: [otlp]
+      exporters: [debug]

opentelemetry.js

@@ -0,0 +1,29 @@
+const opentelemetry = require("@opentelemetry/sdk-node");
+const {
+  getNodeAutoInstrumentations,
+} = require("@opentelemetry/auto-instrumentations-node");
+const {
+  OTLPTraceExporter,
+} = require("@opentelemetry/exporter-trace-otlp-grpc");
+const {
+  OTLPMetricExporter,
+} = require("@opentelemetry/exporter-metrics-otlp-grpc");
+const { PeriodicExportingMetricReader } = require("@opentelemetry/sdk-metrics");
+const { diag, DiagConsoleLogger, DiagLogLevel } = require("@opentelemetry/api");
+
+// diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO);
+
+const sdk = new opentelemetry.NodeSDK({
+  serviceName: process.env.SERVICE_NAME || "Anonymous-GitHub",
+  logRecordProcessor: getNodeAutoInstrumentations().logRecordProcessor,
+  traceExporter: new OTLPTraceExporter({
+    url: "http://opentelemetry:4317/v1/traces",
+  }),
+  metricReader: new PeriodicExportingMetricReader({
+    exporter: new OTLPMetricExporter({
+      url: "http://opentelemetry:4317/v1/metrics",
+    }),
+  }),
+  instrumentations: [getNodeAutoInstrumentations()],
+});
+sdk.start();

package.json

@@ -3,14 +3,14 @@
   "version": "2.2.0",
   "description": "Anonymise Github repositories for double-anonymous reviews",
   "bin": {
-    "anonymous_github": "build/cli.js"
+    "anonymous_github": "build/cli/index.js"
   },
   "scripts": {
     "test": "mocha --reporter spec",
-    "start": "node --inspect=5858 -r ts-node/register ./index.ts",
-    "dev": "nodemon --transpile-only index.ts",
-    "migrateDB": "ts-node --transpile-only migrateDB.ts",
-    "build": "rm -rf build && tsc"
+    "start": "node --inspect=5858 -r ts-node/register ./src/server/index.ts",
+    "dev": "nodemon --transpile-only ./src/server/index.ts",
+    "build": "rm -rf build && tsc && gulp",
+    "knip": "knip"
   },
   "repository": {
     "type": "git",
@@ -30,63 +30,72 @@
     "build"
   ],
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.374.0",
-    "@aws-sdk/node-http-handler": "^3.374.0",
-    "@octokit/oauth-app": "^6.0.0",
-    "@octokit/plugin-paginate-rest": "^8.0.0",
-    "@octokit/rest": "^20.0.1",
-    "@pm2/io": "^5.0.0",
-    "archiver": "^5.3.1",
-    "bullmq": "^2.3.2",
+    "@aws-sdk/client-s3": "^3.540.0",
+    "@aws-sdk/lib-storage": "^3.540.0",
+    "@mongodb-js/zstd": "^1.2.0",
+    "@octokit/rest": "^20.0.2",
+    "@opentelemetry/api": "^1.8.0",
+    "@opentelemetry/auto-instrumentations-node": "^0.43.0",
+    "@opentelemetry/exporter-metrics-otlp-grpc": "^0.49.1",
+    "@opentelemetry/exporter-metrics-otlp-proto": "^0.49.1",
+    "@opentelemetry/exporter-trace-otlp-grpc": "^0.49.1",
+    "@opentelemetry/exporter-trace-otlp-proto": "^0.49.1",
+    "@opentelemetry/sdk-metrics": "^1.22.0",
+    "@opentelemetry/sdk-node": "^0.49.1",
+    "@opentelemetry/sdk-trace-node": "^1.22.0",
+    "@smithy/node-http-handler": "^2.5.0",
+    "archiver": "^5.3.2",
+    "bullmq": "^2.4.0",
+    "cacheable-lookup": "^5.0.3",
     "compression": "^1.7.4",
     "connect-redis": "^7.0.1",
+    "crypto-js": "^4.2.0",
     "decompress-stream-to-s3": "^2.1.1",
-    "dotenv": "^16.0.3",
-    "express": "^4.18.2",
-    "express-rate-limit": "^6.8.0",
-    "express-session": "^1.17.3",
-    "express-slow-down": "^1.6.0",
-    "got": "^11.8.5",
-    "inquirer": "^8.2.5",
-    "istextorbinary": "^6.0.0",
+    "dotenv": "^16.4.5",
+    "express": "^4.19.2",
+    "express-rate-limit": "^6.11.2",
+    "express-session": "^1.18.0",
+    "express-slow-down": "^2.0.1",
+    "got": "^11.8.6",
+    "inquirer": "^8.2.6",
+    "istextorbinary": "^9.5.0",
     "marked": "^5.1.2",
     "mime-types": "^2.1.35",
-    "mongoose": "^7.4.1",
+    "mongoose": "^7.6.10",
     "node-schedule": "^2.1.1",
     "parse-github-url": "^1.0.2",
     "passport": "^0.6.0",
     "passport-github2": "^0.1.12",
-    "rate-limit-redis": "^3.0.2",
-    "redis": "^4.6.7",
-    "textextensions": "^5.16.0",
+    "rate-limit-redis": "^4.2.0",
+    "redis": "^4.6.13",
     "ts-custom-error": "^3.3.1",
-    "unzip-stream": "^0.3.1",
-    "xml-flow": "^1.0.4"
+    "unzip-stream": "^0.3.1"
   },
   "devDependencies": {
-    "@types/archiver": "^5.3.1",
-    "@types/compression": "^1.7.1",
-    "@types/connect-redis": "^0.0.20",
-    "@types/express": "^4.17.14",
-    "@types/express-rate-limit": "^6.0.0",
-    "@types/express-session": "^1.17.5",
-    "@types/express-slow-down": "^1.3.2",
+    "@types/archiver": "^5.3.4",
+    "@types/compression": "^1.7.5",
+    "@types/crypto-js": "^4.2.2",
+    "@types/express": "^4.17.21",
+    "@types/express-session": "^1.18.0",
     "@types/got": "^9.6.12",
-    "@types/inquirer": "^8.0.0",
-    "@types/marked": "^5.0.1",
-    "@types/mime-types": "^2.1.0",
-    "@types/node-schedule": "^2.1.0",
-    "@types/parse-github-url": "^1.0.0",
-    "@types/passport": "^1.0.11",
-    "@types/passport-github2": "^1.2.5",
-    "@types/tar-fs": "^2.0.1",
-    "@types/unzip-stream": "^0.3.1",
-    "@types/xml-flow": "^1.0.1",
-    "chai": "^4.3.6",
-    "mocha": "^10.1.0",
-    "nodemon": "^3.0.1",
-    "ts-node": "^10.9.1",
-    "typescript": "^4.8.4"
+    "@types/inquirer": "^8.2.10",
+    "@types/marked": "^5.0.2",
+    "@types/mime-types": "^2.1.4",
+    "@types/node-schedule": "^2.1.6",
+    "@types/parse-github-url": "^1.0.3",
+    "@types/passport": "^1.0.16",
+    "@types/passport-github2": "^1.2.9",
+    "@types/unzip-stream": "^0.3.4",
+    "gulp": "^5.0.0",
+    "gulp-clean-css": "^4.3.0",
+    "gulp-concat": "^2.6.1",
+    "gulp-order": "^1.2.0",
+    "gulp-uglify": "^3.0.2",
+    "knip": "^5.1.0",
+    "mocha": "^10.4.0",
+    "nodemon": "^3.1.0",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.4.3"
   },
   "nodemonConfig": {
     "ignore": [

prometheus.yaml

@@ -0,0 +1,6 @@
+scrape_configs:
+  - job_name: 'otel-collector'
+    scrape_interval: 10s
+    static_configs:
+      - targets: ['opentelemetry:8889']
+      - targets: ['opentelemetry:8888']

public/css/prism-okaidia.css

@@ -0,0 +1,123 @@
+/**
+ * okaidia theme for JavaScript, CSS and HTML
+ * Loosely based on Monokai textmate theme by http://www.monokai.nl/
+ * @author ocodia
+ */
+
+code[class*="language-"],
+pre[class*="language-"] {
+	color: #f8f8f2;
+	background: none;
+	text-shadow: 0 1px rgba(0, 0, 0, 0.3);
+	font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace;
+	font-size: 1em;
+	text-align: left;
+	white-space: pre;
+	word-spacing: normal;
+	word-break: normal;
+	word-wrap: normal;
+	line-height: 1.5;
+
+	-moz-tab-size: 4;
+	-o-tab-size: 4;
+	tab-size: 4;
+
+	-webkit-hyphens: none;
+	-moz-hyphens: none;
+	-ms-hyphens: none;
+	hyphens: none;
+}
+
+/* Code blocks */
+pre[class*="language-"] {
+	padding: 1em;
+	margin: .5em 0;
+	overflow: auto;
+	border-radius: 0.3em;
+}
+
+:not(pre) > code[class*="language-"],
+pre[class*="language-"] {
+	background: #272822;
+}
+
+/* Inline code */
+:not(pre) > code[class*="language-"] {
+	padding: .1em;
+	border-radius: .3em;
+	white-space: normal;
+}
+
+.token.comment,
+.token.prolog,
+.token.doctype,
+.token.cdata {
+	color: #8292a2;
+}
+
+.token.punctuation {
+	color: #f8f8f2;
+}
+
+.token.namespace {
+	opacity: .7;
+}
+
+.token.property,
+.token.tag,
+.token.constant,
+.token.symbol,
+.token.deleted {
+	color: #f92672;
+}
+
+.token.boolean,
+.token.number {
+	color: #ae81ff;
+}
+
+.token.selector,
+.token.attr-name,
+.token.string,
+.token.char,
+.token.builtin,
+.token.inserted {
+	color: #a6e22e;
+}
+
+.token.operator,
+.token.entity,
+.token.url,
+.language-css .token.string,
+.style .token.string,
+.token.variable {
+	color: #f8f8f2;
+}
+
+.token.atrule,
+.token.attr-value,
+.token.function,
+.token.class-name {
+	color: #e6db74;
+}
+
+.token.keyword {
+	color: #66d9ef;
+}
+
+.token.regex,
+.token.important {
+	color: #fd971f;
+}
+
+.token.important,
+.token.bold {
+	font-weight: bold;
+}
+.token.italic {
+	font-style: italic;
+}
+
+.token.entity {
+	cursor: help;
+}

public/css/style.css

@@ -434,6 +434,11 @@ notebook {
   padding-left: 100px;
 }
 
+.nb-output th,
+.nb-output td {
+  border: 1px solid var(--border-color) !important;
+}
+
 .floatingchat-container-wrap {
   left: inherit !important;
   bottom: inherit !important;
@@ -585,6 +590,7 @@ pre,
 code {
   font-family: "Fira Code", "Courier New", Courier, monospace;
   line-height: 1.1;
+  color: var(--color)
 }
 
 .diff-lines,

public/favicon/site.webmanifest

@@ -3,12 +3,12 @@
     "short_name": "Anonymous Github",
     "icons": [
         {
-            "src": "/android-chrome-192x192.png",
+            "src": "/favicon/android-chrome-192x192.png",
             "sizes": "192x192",
             "type": "image/png"
         },
         {
-            "src": "/android-chrome-256x256.png",
+            "src": "/favicon/android-chrome-256x256.png",
             "sizes": "256x256",
             "type": "image/png"
         }

public/i18n/locale-en.json

@@ -3,6 +3,7 @@
     "unknown_error": "Unknown error, contact the admin.",
     "unreachable": "Anonymous GitHub is unreachable, contact the admin.",
     "request_error": "Unable to download the file, check your connection or contact the admin.",
+    "repo_access_limited": "Access to repository limited by org.",
     "repo_not_found": "The repository is not found.",
     "repo_not_accessible": "Anonymous GitHub is unable to or is forbidden to access the repository.",
     "repository_expired": "The repository is expired",

public/index.html

@@ -33,55 +33,7 @@
     <meta name="msapplication-config" content="/favicon/browserconfig.xml" />
     <meta name="theme-color" content="#ffffff" />
 
-    <!-- CSS -->
-    <link rel="stylesheet" href="/css/bootstrap.min.css" />
-    <link rel="stylesheet" href="/css/font-awesome.min.css" />
-
-    <link rel="stylesheet" href="/css/color-schema.css" />
-    <link rel="stylesheet" href="/css/style.css" />
-    <link rel="stylesheet" href="/css/notebook.css" />
-    <link rel="stylesheet" href="/css/prism.css" />
-    <link rel="stylesheet" href="/css/katex.min.css" />
-    <link rel="stylesheet" href="/css/github-markdown.min.css" />
-
-    <!-- JS -->
-    <script src="/script/external/angular.min.js"></script>
-    <script src="/script/external/angular-translate.min.js"></script>
-    <script src="/script/external/angular-translate-loader-static-files.min.js"></script>
-    <script src="/script/external/angular-sanitize.min.js"></script>
-    <script src="/script/external/angular-route.min.js"></script>
-
-    <script src="/script/external/jquery-3.4.1.min.js"></script>
-    <script src="/script/external/popper.min.js"></script>
-    <script src="/script/external/bootstrap.min.js"></script>
-
-    <!-- PDF -->
-    <script src="/script/external/pdf.compat.js"></script>
-    <script src="/script/external/pdf.js"></script>
-
-    <!-- Code -->
-    <script src="/script/external/ace/ace.js"></script>
-    <script src="/script/external/ui-ace.min.js"></script>
-    <script src="/script/langColors.js"></script>
-
-    <!-- Notebook -->
-    <script src="/script/external/github-emojis.js"></script>
-    <script src="/script/external/marked-emoji.js"></script>
-    <script src="/script/external/marked.min.js"></script>
-    <script src="/script/external/purify.min.js"></script>
-    <script src="/script/external/ansi_up.min.js"></script>
-    <script src="/script/external/prism.min.js"></script>
-    <script src="/script/external/katex.min.js"></script>
-    <script src="/script/external/katex-auto-render.min.js"></script>
-    <script src="/script/external/notebook.min.js"></script>
-
-    <script src="/script/external/org.js"></script>
-
-    <!-- Anonymous GitHub scripts -->
-    <script src="/script/utils.js"></script>
-    <script src="/script/ng-pdfviewer.min.js"></script>
-    <script src="/script/admin.js"></script>
-    <script src="/script/app.js"></script>
+    <link rel="stylesheet" href="/css/all.min.css" />
   </head>
   <body keypress-events class="d-flex flex-column">
     <ng-include src="'partials/header.htm'"></ng-include>
@@ -116,6 +68,11 @@
       </div>
     </div>
 
+    <script src="/script/bundle.min.js"></script>
+    <script>
+      ace.config.set("basePath", "/script/external/ace/");
+      PDFJS.workerSrc = "/script/external/pdf.worker.js";
+    </script>
     <script src="https://storage.ko-fi.com/cdn/scripts/overlay-widget.js"></script>
     <script>
       kofiWidgetOverlay.draw("tdurieux", {

public/partials/admin/queues.htm

@@ -197,14 +197,14 @@
               <a
                 class="dropdown-item"
                 href="#"
-                ng-click="removeJob('remove', job)"
+                ng-click="removeJob('cache', job)"
               >
                 <i class="fas fa-trash-alt"></i> Remove
               </a>
               <a
                 class="dropdown-item"
                 href="#"
-                ng-click="retryJob('remove', job)"
+                ng-click="retryJob('cache', job)"
               >
                 <i class="fas fa-sync"></i> Retry
               </a>

public/partials/explorer.htm

@@ -1,6 +1,6 @@
 <div class="container-fluid h-100">
   <div class="row h-100">
-    <div class="leftCol shadow p-1 overflow-auto" ng-show="files">
+    <div class="leftCol shadow p-1 overflow-auto" ng-show="files.length">
       <tree class="files" file="files"></tree>
       <div class="bottom column">
         <div
@@ -22,13 +22,21 @@
         </ol>
         <div class="">
           <a
+            ng-if="options.isAdmin || options.isOwner"
+            ng-href="/anonymize/{{repoId}}"
+            class="btn btn-outline-primary btn-sm"
+            >Edit</a
+          >
+          <a
+            ng-show="content != null"
             ng-href="{{url}}"
             target="__self"
             class="btn btn-outline-primary btn-sm"
             >View raw</a
           >
           <a
-            ng-href="{{url}}?download=true"
+            ng-show="content != null"
+            ng-href="{{url}}&download=true"
             target="__self"
             class="btn btn-outline-primary btn-sm"
             >Download file</a
@@ -40,6 +48,13 @@
             class="btn btn-outline-primary btn-sm"
             >Download Repository</a
           >
+          <a
+            ng-if="options.hasWebsite"
+            ng-href="/w/{{repoId}}/"
+            target="__self"
+            class="btn btn-outline-primary btn-sm"
+            >Website</a
+          >
         </div>
       </div>
       <div class="align-items-stretch h-100 w-100 overflow-auto">

public/partials/faq.htm

@@ -13,42 +13,6 @@
             role="tablist"
             aria-multiselectable="true"
           >
-            <div class="panel panel-default mb-4">
-              <div class="panel-heading p-3" role="tab" id="heading0">
-                <h3 class="panel-title">
-                  <a
-                    class="collapsed"
-                    role="button"
-                    title=""
-                    data-toggle="collapse"
-                    data-parent="#faq"
-                    href="#download"
-                    aria-expanded="true"
-                    aria-controls="download"
-                  >
-                    Can I download the repository?
-                  </a>
-                </h3>
-              </div>
-              <div
-                id="download"
-                class="panel-collapse collapse"
-                role="tabpanel"
-                aria-labelledby="heading0"
-              >
-                <div class="panel-body p-3">
-                  <p>
-                    It is currently not possible to download an anonymized
-                    repository neither to clone it.
-                    It is technically possible to implement however it 
-                    would require additional processing power and storage. 
-                    I am currently not able to cover the cost of this feature.
-                    If you want to see this feature on Anonymous GitHub, please consider doing a donation.
-                  </p>
-                </div>
-              </div>
-            </div>
-
             <div class="panel panel-default mb-4">
               <div class="panel-heading p-3" role="tab" id="heading6">
                 <h3 class="panel-title">
@@ -75,11 +39,9 @@
                 <div class="panel-body p-3">
                   <p>
                     <ul>
-                      <li>
-                        Anonymous GitHub does not allow to download the repository.
-                      </li>
                       <li>
                         Anonymous GitHub only anonymizes textual files.
+                        It does not support the use of a static site generator, such as Jekyll, with GitHub Pages (although Markdown files are converted to HTML without any special formatting).
                       </li>
                       <li>
                         Anonymous GitHub does not support files that are larger than 8Mo.
@@ -119,12 +81,7 @@
                 <div class="panel-body p-3">
                   <p>
                     Anonymous Github is able to display pure textual files, such as text or source code. It can also render images, PDFs, and notbooks.
-                    However, only textual based files are anonymized. Anonymous Github considers the following file format as textual:
-                  </p>
-                  <p>
-                    <ul>
-                      <li ng-repeat="format in supportedFileTypes" ng-bind="format"></li>
-                    </ul>
+                    However, only textual based files are anonymized. Anonymous Github analyzes the content of the file to detect if it is textual or not.
                   </p>
                 </div>
               </div>

public/partials/header.htm

@@ -137,7 +137,7 @@
           aria-expanded="false"
         >
           <img
-            src="{{user.photo}}"
+            ng-src="{{user.photo}}"
             ng-if="user.photo"
             width="30"
             height="30"

public/partials/home.htm

@@ -96,7 +96,8 @@
             The reviewers can explore your repository with ease, the source code
             is highlighted, PDFs, images, Notebook are rendered. The goal is to
             make is as easy as possible for the reviewer to explore and review
-            the repository. GitHub pages are also supported.
+            the repository. <a href="https://pages.github.com">GitHub Pages</a>
+            is also supported, but not static site generators, such as Jekyll.
           </p>
         </div>
         <div class="col-md-5">

public/partials/pageView.htm

@@ -6,7 +6,10 @@
 <div class="h-100 overflow-auto" ng-if="type == 'pdf'">
   <pdfviewer class="h-100 overflow-auto" src="{{url}}" id="viewer"></pdfviewer>
 </div>
+<div ng-if="type == 'audio'"><audio controls="controls"><source src="{{url}}" /></audio></div>
 <div ng-if="type == 'IPython'"><notebook file="url"></notebook></div>
 <div ng-if="type == 'error'" class="file-error container d-flex h-100"><h1 class="display-1 m-auto" translate="ERRORS.{{content}}">Error</h1></div></div>
 <div ng-if="type == 'loading' && !error" class="file-error container d-flex h-100"><h1 class="display-1 m-auto">Loading...</h1></div></div>
-<div ng-if="content == null" class="file-error container d-flex h-100"><h1 class="display-1 m-auto">Empty file!</h1></div>
+<div ng-if="type == 'empty'" class="file-error container d-flex h-100"><h1 class="display-1 m-auto">Empty repository!</h1></div>
+<div ng-if="content == null && type != 'empty'" class="file-error container d-flex h-100"><h1 class="display-1 m-auto">Empty file!</h1></div>
+<div ng-if="type == 'binary'" class="file-error container d-flex h-100"><h1 class="display-1 m-auto">Unsupported binary file. You can download the file: <a target="_blank" ng-href="{{url}}&download=true">here</a>.</h1></div>

public/partials/status.htm

@@ -21,10 +21,9 @@
       >
         <span>
           {{repo.status | title}}
-          <span
-            ng-if="repo.status == 'download' && repo.statusMessage"
-            ng-bind="repo.statusMessage | humanFileSize"
-          ></span>
+          <span ng-if="repo.statusMessage"
+            >: {{repo.statusMessage | title}}</span
+          >
         </span>
       </div>
     </div>
@@ -56,10 +55,16 @@
     </p>
   </section>
 
-  <hr />
-
   <section class="py-4">
-    <h2>Support the project</h2>
+    <h2 class="md-1">Support Anonymous GitHub</h2>
+
+    <iframe
+      id="kofiframe"
+      src="https://ko-fi.com/tdurieux/?hidefeed=true&widget=true&embed=true&preview=true"
+      style="border: none; width: 100%"
+      height="650"
+      title="tdurieux"
+    ></iframe>
 
     <div class="row text-center">
       <div class="col-lg-4">

public/script/external/katex-auto-render.min.js

@@ -1 +1 @@
-!function(e,t){"object"==typeof exports&&"object"==typeof module?module.exports=t(require("katex")):"function"==typeof define&&define.amd?define(["katex"],t):"object"==typeof exports?exports.renderMathInElement=t(require("katex")):e.renderMathInElement=t(e.katex)}("undefined"!=typeof self?self:this,function(e){return function(e){var t={};function r(n){if(t[n])return t[n].exports;var o=t[n]={i:n,l:!1,exports:{}};return e[n].call(o.exports,o,o.exports,r),o.l=!0,o.exports}return r.m=e,r.c=t,r.d=function(e,t,n){r.o(e,t)||Object.defineProperty(e,t,{enumerable:!0,get:n})},r.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},r.t=function(e,t){if(1&t&&(e=r(e)),8&t)return e;if(4&t&&"object"==typeof e&&e&&e.__esModule)return e;var n=Object.create(null);if(r.r(n),Object.defineProperty(n,"default",{enumerable:!0,value:e}),2&t&&"string"!=typeof e)for(var o in e)r.d(n,o,function(t){return e[t]}.bind(null,o));return n},r.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return r.d(t,"a",t),t},r.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},r.p="",r(r.s=1)}([function(t,r){t.exports=e},function(e,t,r){"use strict";r.r(t);var n=r(0),o=r.n(n),a=function(e,t,r){for(var n=r,o=0,a=e.length;n<t.length;){var i=t[n];if(o<=0&&t.slice(n,n+a)===e)return n;"\\"===i?n++:"{"===i?o++:"}"===i&&o--,n++}return-1},i=function(e,t,r,n){for(var o=[],i=0;i<e.length;i++)if("text"===e[i].type){var l=e[i].data,d=!0,s=0,f=void 0;for(-1!==(f=l.indexOf(t))&&(s=f,o.push({type:"text",data:l.slice(0,s)}),d=!1);;){if(d){if(-1===(f=l.indexOf(t,s)))break;o.push({type:"text",data:l.slice(s,f)}),s=f}else{if(-1===(f=a(r,l,s+t.length)))break;o.push({type:"math",data:l.slice(s+t.length,f),rawData:l.slice(s,f+r.length),display:n}),s=f+r.length}d=!d}o.push({type:"text",data:l.slice(s)})}else o.push(e[i]);return o},l=function(e,t){var r=function(e,t){for(var r=[{type:"text",data:e}],n=0;n<t.length;n++){var o=t[n];r=i(r,o.left,o.right,o.display||!1)}return r}(e,t.delimiters);if(1===r.length&&"text"===r[0].type)return null;for(var n=document.createDocumentFragment(),a=0;a<r.length;a++)if("text"===r[a].type)n.appendChild(document.createTextNode(r[a].data));else{var l=document.createElement("span"),d=r[a].data;t.displayMode=r[a].display;try{t.preProcess&&(d=t.preProcess(d)),o.a.render(d,l,t)}catch(e){if(!(e instanceof o.a.ParseError))throw e;t.errorCallback("KaTeX auto-render: Failed to parse `"+r[a].data+"` with ",e),n.appendChild(document.createTextNode(r[a].rawData));continue}n.appendChild(l)}return n};t.default=function(e,t){if(!e)throw new Error("No element provided to render");var r={};for(var n in t)t.hasOwnProperty(n)&&(r[n]=t[n]);r.delimiters=r.delimiters||[{left:"$$",right:"$$",display:!0},{left:"\\(",right:"\\)",display:!1},{left:"\\[",right:"\\]",display:!0}],r.ignoredTags=r.ignoredTags||["script","noscript","style","textarea","pre","code","option"],r.ignoredClasses=r.ignoredClasses||[],r.errorCallback=r.errorCallback||console.error,r.macros=r.macros||{},function e(t,r){for(var n=0;n<t.childNodes.length;n++){var o=t.childNodes[n];if(3===o.nodeType){var a=l(o.textContent,r);a&&(n+=a.childNodes.length-1,t.replaceChild(a,o))}else 1===o.nodeType&&function(){var t=" "+o.className+" ";-1===r.ignoredTags.indexOf(o.nodeName.toLowerCase())&&r.ignoredClasses.every(function(e){return-1===t.indexOf(" "+e+" ")})&&e(o,r)}()}}(e,r)}}]).default});
+!function(e,t){"object"==typeof exports&&"object"==typeof module?module.exports=t(require("katex")):"function"==typeof define&&define.amd?define(["katex"],t):"object"==typeof exports?exports.renderMathInElement=t(require("katex")):e.renderMathInElement=t(e.katex)}("undefined"!=typeof self?self:this,(function(e){return function(){"use strict";var t={771:function(t){t.exports=e}},n={};function r(e){var o=n[e];if(void 0!==o)return o.exports;var i=n[e]={exports:{}};return t[e](i,i.exports,r),i.exports}r.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return r.d(t,{a:t}),t},r.d=function(e,t){for(var n in t)r.o(t,n)&&!r.o(e,n)&&Object.defineProperty(e,n,{enumerable:!0,get:t[n]})},r.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)};var o={};return function(){r.d(o,{default:function(){return d}});var e=r(771),t=r.n(e);const n=function(e,t,n){let r=n,o=0;const i=e.length;for(;r<t.length;){const n=t[r];if(o<=0&&t.slice(r,r+i)===e)return r;"\\"===n?r++:"{"===n?o++:"}"===n&&o--,r++}return-1},i=/^\\begin{/;var a=function(e,t){let r;const o=[],a=new RegExp("("+t.map((e=>e.left.replace(/[-/\\^$*+?.()|[\]{}]/g,"\\$&"))).join("|")+")");for(;r=e.search(a),-1!==r;){r>0&&(o.push({type:"text",data:e.slice(0,r)}),e=e.slice(r));const a=t.findIndex((t=>e.startsWith(t.left)));if(r=n(t[a].right,e,t[a].left.length),-1===r)break;const l=e.slice(0,r+t[a].right.length),s=i.test(l)?l:e.slice(t[a].left.length,r);o.push({type:"math",data:s,rawData:l,display:t[a].display}),e=e.slice(r+t[a].right.length)}return""!==e&&o.push({type:"text",data:e}),o};const l=function(e,n){const r=a(e,n.delimiters);if(1===r.length&&"text"===r[0].type)return null;const o=document.createDocumentFragment();for(let e=0;e<r.length;e++)if("text"===r[e].type)o.appendChild(document.createTextNode(r[e].data));else{const i=document.createElement("span");let a=r[e].data;n.displayMode=r[e].display;try{n.preProcess&&(a=n.preProcess(a)),t().render(a,i,n)}catch(i){if(!(i instanceof t().ParseError))throw i;n.errorCallback("KaTeX auto-render: Failed to parse `"+r[e].data+"` with ",i),o.appendChild(document.createTextNode(r[e].rawData));continue}o.appendChild(i)}return o},s=function(e,t){for(let n=0;n<e.childNodes.length;n++){const r=e.childNodes[n];if(3===r.nodeType){let o=r.textContent,i=r.nextSibling,a=0;for(;i&&i.nodeType===Node.TEXT_NODE;)o+=i.textContent,i=i.nextSibling,a++;const s=l(o,t);if(s){for(let e=0;e<a;e++)r.nextSibling.remove();n+=s.childNodes.length-1,e.replaceChild(s,r)}else n+=a}else if(1===r.nodeType){const e=" "+r.className+" ";-1===t.ignoredTags.indexOf(r.nodeName.toLowerCase())&&t.ignoredClasses.every((t=>-1===e.indexOf(" "+t+" ")))&&s(r,t)}}};var d=function(e,t){if(!e)throw new Error("No element provided to render");const n={};for(const e in t)t.hasOwnProperty(e)&&(n[e]=t[e]);n.delimiters=n.delimiters||[{left:"$$",right:"$$",display:!0},{left:"\\(",right:"\\)",display:!1},{left:"\\begin{equation}",right:"\\end{equation}",display:!0},{left:"\\begin{align}",right:"\\end{align}",display:!0},{left:"\\begin{alignat}",right:"\\end{alignat}",display:!0},{left:"\\begin{gather}",right:"\\end{gather}",display:!0},{left:"\\begin{CD}",right:"\\end{CD}",display:!0},{left:"\\[",right:"\\]",display:!0}],n.ignoredTags=n.ignoredTags||["script","noscript","style","textarea","pre","code","option"],n.ignoredClasses=n.ignoredClasses||[],n.errorCallback=n.errorCallback||console.error,n.macros=n.macros||{},s(e,n)}}(),o=o.default}()}));

public/script/external/marked-katex-extension.umd.min.js

@@ -0,0 +1,8 @@
+/**
+ * Minified by jsDelivr using Terser v5.19.2.
+ * Original file: /npm/marked-katex-extension@5.0.1/lib/index.umd.js
+ *
+ * Do NOT use SRI with dynamically generated files! More information: https://www.jsdelivr.com/using-sri-with-dynamic-files
+ */
+!function(e,t){"object"==typeof exports&&"undefined"!=typeof module?module.exports=t(require("katex")):"function"==typeof define&&define.amd?define(["katex"],t):(e="undefined"!=typeof globalThis?globalThis:e||self).markedKatex=t(e.katex)}(this,(function(e){"use strict";const t=/^(\${1,2})(?!\$)((?:\\.|[^\\\n])*?(?:\\.|[^\\\n\$]))\1(?=[\s?!\.,:？！。，：]|$)/,n=/^(\${1,2})\n((?:\\[^]|[^\\])+?)\n\1(?:\n|$)/;function r(t,n){return r=>e.renderToString(r.text,{...t,displayMode:r.displayMode})+(n?"\n":"")}function i(e,n){return{name:"inlineKatex",level:"inline",start(e){let n,r=e;for(;r;){if(n=r.indexOf("$"),-1===n)return;if(0===n||" "===r.charAt(n-1)){if(r.substring(n).match(t))return n}r=r.substring(n+1).replace(/^\$+/,"")}},tokenizer(e,n){const r=e.match(t);if(r)return{type:"inlineKatex",raw:r[0],text:r[2].trim(),displayMode:2===r[1].length}},renderer:n}}function o(e,t){return{name:"blockKatex",level:"block",tokenizer(e,t){const r=e.match(n);if(r)return{type:"blockKatex",raw:r[0],text:r[2].trim(),displayMode:2===r[1].length}},renderer:t}}return function(e={}){return{extensions:[i(e,r(e,!1)),o(e,r(e,!0))]}}}));
+//# sourceMappingURL=/sm/25cb250346ed1262c6185eedae00cfd6f44bba21eb2c1bf521e994aa1d145555.map

public/script/utils.js

@@ -1,4 +1,33 @@
-function urlRel2abs(url) {
+function humanFileSize(bytes, si = false, dp = 1) {
+  const thresh = si ? 1000 : 1024;
+
+  bytes = bytes / 8;
+
+  if (Math.abs(bytes) < thresh) {
+    return bytes + "B";
+  }
+
+  const units = si
+    ? ["kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]
+    : ["KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"];
+  let u = -1;
+  const r = 10 ** dp;
+
+  do {
+    bytes /= thresh;
+    ++u;
+  } while (
+    Math.round(Math.abs(bytes) * r) / r >= thresh &&
+    u < units.length - 1
+  );
+
+  return bytes.toFixed(dp) + "" + units[u];
+}
+
+function urlRel2abs(
+  url,
+  baseUrl = location.href.match(/^(.+)\/?(?:#.+)?$/)[0] + "/"
+) {
   /* Only accept commonly trusted protocols:
    * Only data-image URLs are accepted, Exotic flavours (escaped slash,
    * html-entitied characters) are not supported to keep the function fast */
@@ -7,17 +36,14 @@ function urlRel2abs(url) {
   ) {
     return url; //Url is already absolute
   }
-  var base_url = location.href.match(/^(.+)\/?(?:#.+)?$/)[0] + "/";
 
   if (url.substring(0, 2) == "//") return location.protocol + url;
-  else if (url.charAt(0) == "/")
-    return location.protocol + "//" + location.host + url;
+  else if (url.charAt(0) == "/") return baseUrl + url;
   else if (url.substring(0, 2) == "./") url = "." + url;
   else if (/^\s*$/.test(url)) return "";
   //Empty = Return nothing
-  else url = "../" + url;
 
-  url = base_url + url;
+  url = baseUrl + url;
 
   while (/\/\.\.\//.test((url = url.replace(/[^\/]+\/+\.\.\//g, ""))));
   /* Escape certain characters to prevent XSS */
@@ -35,13 +61,15 @@ function urlRel2abs(url) {
 const charactersAttributes = "[^-a-z0-9:._]";
 const allTagCharacters = "(?:[^>\"']*(?:\"[^\"]*\"|'[^']*'))*?[^>]*";
 
-function by(match, group1, group2, group3) {
-  /* Note that this function can also be used to remove links:
-   * return group1 + "javascript://" + group3; */
-  return group1 + urlRel2abs(group2) + group3;
+function by(baseUrl) {
+  return (match, group1, group2, group3) => {
+    /* Note that this function can also be used to remove links:
+     * return group1 + "javascript://" + group3; */
+    return group1 + urlRel2abs(group2, baseUrl) + group3;
+  };
 }
 
-function cr(html, selector, attribute) {
+function cr(html, selector, attribute, baseUrl) {
   if (typeof selector == "string") selector = new RegExp(selector, "gi");
   attribute = charactersAttributes + attribute;
   const marker = "\\s*=\\s*";
@@ -53,12 +81,15 @@ function cr(html, selector, attribute) {
     "gi"
   );
   html = html.replace(selector, function (match) {
-    return match.replace(re1, by).replace(re2, by).replace(re3, by);
+    return match
+      .replace(re1, by(baseUrl))
+      .replace(re2, by(baseUrl))
+      .replace(re3, by(baseUrl));
   });
   return html;
 }
 
-function contentAbs2Relative(content) {
+function contentAbs2Relative(content, baseUrl) {
   if (!content) return content;
   content = cr(
     content,
@@ -68,7 +99,8 @@ function contentAbs2Relative(content) {
       "href\\s*=" +
       allTagCharacters +
       ">",
-    "href"
+    "href",
+    baseUrl
   );
   content = cr(
     content,
@@ -78,7 +110,8 @@ function contentAbs2Relative(content) {
       "src\\s*=" +
       allTagCharacters +
       ">",
-    "src"
+    "src",
+    baseUrl
   );
   return content;
 }
@@ -110,48 +143,15 @@ function parseGithubUrl(url) {
   }
 }
 
-marked.use(
-  markedEmoji({
-    emojis: githubEmojis,
-    unicode: false,
-  })
-);
-
-function renderMD(md, baseUrl) {
-  md = contentAbs2Relative(md);
+function renderMD(md, baseUrlValue) {
+  marked.use(
+    markedEmoji({
+      emojis: githubEmojis,
+      unicode: false,
+    })
+  );
+  md = contentAbs2Relative(md, baseUrlValue);
   const renderer = new marked.Renderer();
-  // katex
-  function mathsExpression(expr) {
-    if (expr.match(/^\$\$[\s\S]*\$\$$/)) {
-      expr = expr.substr(2, expr.length - 4);
-      return katex.renderToString(expr, { displayMode: true });
-    } else if (expr.match(/^\$[\s\S]*\$$/)) {
-      expr = expr.substr(1, expr.length - 2);
-      return katex.renderToString(expr, { isplayMode: false });
-    }
-  }
-
-  const rendererCode = renderer.code;
-  renderer.code = function (code, lang, escaped) {
-    if (!lang) {
-      const math = mathsExpression(code);
-      if (math) {
-        return math;
-      }
-    }
-    // call default renderer
-    return rendererCode.call(this, code, lang, escaped);
-  };
-
-  const rendererCodespan = renderer.codespan;
-  renderer.codespan = function (text) {
-    const math = mathsExpression(text);
-    if (math) {
-      return math;
-    }
-
-    return rendererCodespan.call(this, text);
-  };
 
   const rendererLink = renderer.link;
   renderer.link = function (href, title, text) {
@@ -161,5 +161,26 @@ function renderMD(md, baseUrl) {
     }
     return rendererLink.call(this, href, title, text);
   };
-  return marked.parse(md, { baseUrl, renderer });
+
+  marked.setOptions({
+    renderer: renderer,
+    pedantic: false,
+    gfm: true,
+    breaks: false,
+    sanitize: false,
+    smartLists: true,
+    smartypants: false,
+    xhtml: false,
+    headerIds: false,
+    katex: katex,
+  });
+  if (baseUrlValue) {
+    marked.use(baseUrl(baseUrlValue));
+  }
+  marked.use(
+    markedKatex({
+      throwOnError: false,
+    })
+  );
+  return marked.parse(md, { renderer });
 }

src/AnonymizedFile.ts

@@ -1,266 +0,0 @@
-import { join, basename } from "path";
-import { Response } from "express";
-import { Readable } from "stream";
-import Repository from "./Repository";
-import { FILE_TYPE, Tree, TreeElement, TreeFile } from "./types";
-import storage from "./storage";
-import config from "../config";
-import {
-  anonymizePath,
-  AnonymizeTransformer,
-  isTextFile,
-} from "./anonymize-utils";
-import AnonymousError from "./AnonymousError";
-import { handleError } from "./routes/route-utils";
-import { lookup } from "mime-types";
-
-/**
- * Represent a file in a anonymized repository
- */
-export default class AnonymizedFile {
-  private _originalPath: string | undefined;
-  private fileSize?: number;
-
-  repository: Repository;
-  anonymizedPath: string;
-  _sha?: string;
-
-  constructor(data: { repository: Repository; anonymizedPath: string }) {
-    this.repository = data.repository;
-    if (!this.repository.options.terms)
-      throw new AnonymousError("terms_not_specified", {
-        object: this,
-        httpStatus: 400,
-      });
-    this.anonymizedPath = data.anonymizedPath;
-  }
-
-  async sha() {
-    if (this._sha) return this._sha.replace(/"/g, "");
-    await this.originalPath();
-    return this._sha?.replace(/"/g, "");
-  }
-
-  /**
-   * De-anonymize the path
-   *
-   * @returns the origin relative path of the file
-   */
-  async originalPath(): Promise<string> {
-    if (this._originalPath) return this._originalPath;
-    if (!this.anonymizedPath)
-      throw new AnonymousError("path_not_specified", {
-        object: this,
-        httpStatus: 400,
-      });
-
-    const paths = this.anonymizedPath.trim().split("/");
-    let currentOriginal = (await this.repository.files({
-      force: false,
-    })) as TreeElement;
-    let currentOriginalPath = "";
-    for (let i = 0; i < paths.length; i++) {
-      const fileName = paths[i];
-      if (fileName == "") {
-        continue;
-      }
-      if (!(currentOriginal as Tree)[fileName]) {
-        // anonymize all the file in the folder and check if there is one that match the current filename
-        const options = [];
-        for (let originalFileName in currentOriginal) {
-          if (
-            anonymizePath(originalFileName, this.repository.options.terms) ==
-            fileName
-          ) {
-            options.push(originalFileName);
-          }
-        }
-
-        // if only one option we found the original filename
-        if (options.length == 1) {
-          currentOriginalPath = join(currentOriginalPath, options[0]);
-          currentOriginal = (currentOriginal as Tree)[options[0]];
-        } else if (options.length == 0) {
-          throw new AnonymousError("file_not_found", {
-            object: this,
-            httpStatus: 404,
-          });
-        } else {
-          const nextName = paths[i + 1];
-          if (!nextName) {
-            // if there is no next name we can't find the file and we return the first option
-            currentOriginalPath = join(currentOriginalPath, options[0]);
-            currentOriginal = (currentOriginal as Tree)[options[0]];
-          }
-          let found = false;
-          for (const option of options) {
-            const optionTree = (currentOriginal as Tree)[option];
-            if ((optionTree as Tree).child) {
-              const optionTreeChild = (optionTree as Tree).child;
-              if ((optionTreeChild as Tree)[nextName]) {
-                currentOriginalPath = join(currentOriginalPath, option);
-                currentOriginal = optionTreeChild;
-                found = true;
-                break;
-              }
-            }
-          }
-          if (!found) {
-            // if we didn't find the next name we return the first option
-            currentOriginalPath = join(currentOriginalPath, options[0]);
-            currentOriginal = (currentOriginal as Tree)[options[0]];
-          }
-        }
-      } else {
-        currentOriginalPath = join(currentOriginalPath, fileName);
-        currentOriginal = (currentOriginal as Tree)[fileName];
-      }
-    }
-
-    if (
-      currentOriginal.sha === undefined ||
-      currentOriginal.size === undefined
-    ) {
-      throw new AnonymousError("folder_not_supported", { object: this });
-    }
-
-    const file = currentOriginal as TreeFile;
-    this.fileSize = file.size;
-    this._sha = file.sha;
-
-    this._originalPath = currentOriginalPath;
-    return this._originalPath;
-  }
-  extension() {
-    const filename = basename(this.anonymizedPath);
-    const extensions = filename.split(".").reverse();
-    return extensions[0].toLowerCase();
-  }
-  isImage() {
-    const extension = this.extension();
-    return [
-      "png",
-      "jpg",
-      "jpeg",
-      "gif",
-      "svg",
-      "ico",
-      "bmp",
-      "tiff",
-      "tif",
-      "webp",
-      "avif",
-      "heif",
-      "heic",
-    ].includes(extension);
-  }
-  isFileSupported() {
-    const extension = this.extension();
-    if (!this.repository.options.pdf && extension == "pdf") {
-      return false;
-    }
-    if (!this.repository.options.image && this.isImage()) {
-      return false;
-    }
-    return true;
-  }
-
-  async content(): Promise<Readable> {
-    if (this.anonymizedPath.includes(config.ANONYMIZATION_MASK)) {
-      await this.originalPath();
-    }
-    if (this.fileSize && this.fileSize > config.MAX_FILE_SIZE) {
-      throw new AnonymousError("file_too_big", {
-        object: this,
-        httpStatus: 403,
-      });
-    }
-    const exist = await storage.exists(this.originalCachePath);
-    if (exist == FILE_TYPE.FILE) {
-      return storage.read(this.originalCachePath);
-    } else if (exist == FILE_TYPE.FOLDER) {
-      throw new AnonymousError("folder_not_supported", {
-        object: this,
-        httpStatus: 400,
-      });
-    }
-    return await this.repository.source?.getFileContent(this);
-  }
-
-  async anonymizedContent() {
-    return (await this.content()).pipe(new AnonymizeTransformer(this));
-  }
-
-  get originalCachePath() {
-    if (!this.originalPath)
-      throw new AnonymousError("path_not_defined", {
-        object: this,
-        httpStatus: 400,
-      });
-    if (!this._originalPath) {
-      if (this.anonymizedPath.includes(config.ANONYMIZATION_MASK)) {
-        throw new AnonymousError("path_not_defined", {
-          object: this,
-          httpStatus: 400,
-        });
-      } else {
-        return join(this.repository.originalCachePath, this.anonymizedPath);
-      }
-    }
-
-    return join(this.repository.originalCachePath, this._originalPath);
-  }
-
-  async send(res: Response): Promise<void> {
-    return new Promise(async (resolve, reject) => {
-      try {
-        const content = await this.content();
-        const mime = lookup(this.anonymizedPath);
-        if (mime && this.extension() != "ts") {
-          res.contentType(mime);
-        } else if (isTextFile(this.anonymizedPath)) {
-          res.contentType("text/plain");
-        }
-        res.header("Accept-Ranges", "none");
-        let fileInfo: Awaited<ReturnType<typeof storage.fileInfo>>;
-        try {
-          fileInfo = await storage.fileInfo(this.originalCachePath);
-        } catch (error) {
-          // unable to get file size
-          console.error(error);
-        }
-
-        const anonymizer = new AnonymizeTransformer(this);
-
-        anonymizer.once("transform", (data) => {
-          if (data.isText && !mime) {
-            res.contentType("text/plain");
-          }
-          if (fileInfo?.size && !data.wasAnonimized) {
-            // the text files may be anonymized and therefore the size may be different
-            res.header("Content-Length", fileInfo.size.toString());
-          }
-        });
-
-        content
-          .pipe(anonymizer)
-          .pipe(res)
-          .on("close", () => {
-            if (!content.closed && !content.destroyed) {
-              content.destroy();
-            }
-            resolve();
-          })
-          .on("error", (error) => {
-            if (!content.closed && !content.destroyed) {
-              content.destroy();
-            }
-            reject(error);
-            handleError(error, res);
-          });
-      } catch (error) {
-        handleError(error, res);
-      }
-    });
-  }
-}

src/Repository.ts

@@ -1,449 +0,0 @@
-import { join } from "path";
-import storage from "./storage";
-import {
-  FILE_TYPE,
-  RepositoryStatus,
-  Source,
-  Tree,
-  TreeElement,
-  TreeFile,
-} from "./types";
-import { Readable } from "stream";
-import User from "./User";
-import GitHubStream from "./source/GitHubStream";
-import GitHubDownload from "./source/GitHubDownload";
-import Zip from "./source/Zip";
-import { anonymizePath } from "./anonymize-utils";
-import UserModel from "./database/users/users.model";
-import { IAnonymizedRepositoryDocument } from "./database/anonymizedRepositories/anonymizedRepositories.types";
-import { AnonymizeTransformer } from "./anonymize-utils";
-import GitHubBase from "./source/GitHubBase";
-import Conference from "./Conference";
-import ConferenceModel from "./database/conference/conferences.model";
-import AnonymousError from "./AnonymousError";
-import { downloadQueue, removeQueue } from "./queue";
-import { isConnected } from "./database/database";
-import AnonymizedFile from "./AnonymizedFile";
-import AnonymizedRepositoryModel from "./database/anonymizedRepositories/anonymizedRepositories.model";
-import { getRepositoryFromGitHub } from "./source/GitHubRepository";
-import config from "../config";
-
-function anonymizeTreeRecursive(
-  tree: TreeElement,
-  terms: string[],
-  opt: {
-    /** Include the file sha in the response */
-    includeSha: boolean;
-  } = {
-    includeSha: false,
-  }
-): TreeElement {
-  if (typeof tree.size !== "object" && tree.sha !== undefined) {
-    if (opt?.includeSha) return tree as TreeFile;
-    return { size: tree.size } as TreeFile;
-  }
-  const output: Tree = {};
-  Object.getOwnPropertyNames(tree).forEach((file) => {
-    const anonymizedPath = anonymizePath(file, terms);
-    output[anonymizedPath] = anonymizeTreeRecursive(
-      (tree as Tree)[file],
-      terms,
-      opt
-    );
-  });
-  return output;
-}
-
-export default class Repository {
-  private _model: IAnonymizedRepositoryDocument;
-  source: Source;
-  owner: User;
-
-  constructor(data: IAnonymizedRepositoryDocument) {
-    this._model = data;
-    switch (data.source.type) {
-      case "GitHubDownload":
-        this.source = new GitHubDownload(data.source, this);
-        break;
-      case "GitHubStream":
-        this.source = new GitHubStream(data.source, this);
-        break;
-      case "Zip":
-        this.source = new Zip(data.source, this);
-        break;
-      default:
-        throw new AnonymousError("unsupported_source", {
-          object: data.source.type,
-          httpStatus: 400,
-        });
-    }
-    this.owner = new User(new UserModel({ _id: data.owner }));
-    this.owner.model.isNew = false;
-  }
-
-  /**
-   * Get the anonymized file tree
-   * @param opt force to get an updated list of files
-   * @returns The anonymized file tree
-   */
-  async anonymizedFiles(
-    opt: {
-      /** Force to refresh the file tree */
-      force?: boolean;
-      /** Include the file sha in the response */
-      includeSha: boolean;
-    } = {
-      force: false,
-      includeSha: false,
-    }
-  ): Promise<Tree> {
-    const terms = this._model.options.terms || [];
-    return anonymizeTreeRecursive(await this.files(opt), terms, opt) as Tree;
-  }
-
-  /**
-   * Get the file tree
-   *
-   * @param opt force to get an updated list of files
-   * @returns The file tree
-   */
-  async files(opt: { force?: boolean } = { force: false }): Promise<Tree> {
-    if (!this._model.originalFiles && !opt.force) {
-      const res = await AnonymizedRepositoryModel.findById(this._model._id, {
-        originalFiles: 1,
-      });
-      if (!res) throw new AnonymousError("repository_not_found");
-      this.model.originalFiles = res.originalFiles;
-    }
-    if (
-      this._model.originalFiles &&
-      Object.getOwnPropertyNames(this._model.originalFiles).length !== 0 &&
-      !opt.force
-    ) {
-      return this._model.originalFiles;
-    }
-    const files = await this.source.getFiles();
-    this._model.originalFiles = files;
-    this._model.size = { storage: 0, file: 0 };
-    await this.computeSize();
-    return files;
-  }
-
-  /**
-   * Check the status of the repository
-   */
-  check() {
-    if (
-      this._model.options.expirationMode !== "never" &&
-      this.status == "ready" &&
-      this._model.options.expirationDate
-    ) {
-      if (this._model.options.expirationDate <= new Date()) {
-        this.expire();
-      }
-    }
-    if (
-      this.status == "expired" ||
-      this.status == "expiring" ||
-      this.status == "removing" ||
-      this.status == "removed"
-    ) {
-      throw new AnonymousError("repository_expired", {
-        object: this,
-        httpStatus: 410,
-      });
-    }
-    const fiveMinuteAgo = new Date();
-    fiveMinuteAgo.setMinutes(fiveMinuteAgo.getMinutes() - 5);
-
-    if (
-      this.status == "preparing" ||
-      (this.status == "download" && this._model.statusDate > fiveMinuteAgo)
-    ) {
-      throw new AnonymousError("repository_not_ready", {
-        object: this,
-      });
-    }
-  }
-
-  /**
-   * Compress and anonymize the repository
-   *
-   * @returns A stream of anonymized repository compressed
-   */
-  zip(): Promise<Readable> {
-    return storage.archive(this.originalCachePath, {
-      format: "zip",
-      fileTransformer: (filename: string) =>
-        new AnonymizeTransformer(
-          new AnonymizedFile({
-            repository: this,
-            anonymizedPath: filename,
-          })
-        ),
-    });
-  }
-
-  /**
-   * Update the repository if a new commit exists
-   *
-   * @returns void
-   */
-  async updateIfNeeded(opt?: { force: boolean }): Promise<void> {
-    const yesterday = new Date();
-    yesterday.setDate(yesterday.getDate() - 1);
-    if (
-      opt?.force ||
-      (this._model.options.update && this._model.lastView < yesterday)
-    ) {
-      // Only GitHubBase can be update for the moment
-      if (this.source instanceof GitHubBase) {
-        const token = await this.source.getToken();
-        const branches = await this.source.githubRepository.branches({
-          force: true,
-          accessToken: token,
-        });
-        const branch = this.source.branch;
-        const newCommit = branches.filter((f) => f.name == branch.name)[0]
-          ?.commit;
-        if (branch.commit == newCommit && this.status == "ready") {
-          console.log(`[UPDATE] ${this._model.repoId} is up to date`);
-          return;
-        }
-        this._model.source.commit = newCommit;
-        const commitInfo = await this.source.githubRepository.getCommitInfo(
-          newCommit,
-          {
-            accessToken: token,
-          }
-        );
-        if (commitInfo.commit.author?.date) {
-          this._model.source.commitDate = new Date(
-            commitInfo.commit.author?.date
-          );
-        }
-        branch.commit = newCommit;
-
-        if (!newCommit) {
-          console.error(
-            `${branch.name} for ${this.source.githubRepository.fullName} is not found`
-          );
-          await this.updateStatus(RepositoryStatus.ERROR, "branch_not_found");
-          await this.resetSate();
-          throw new AnonymousError("branch_not_found", {
-            object: this,
-          });
-        }
-        this._model.anonymizeDate = new Date();
-        console.log(
-          `[UPDATE] ${this._model.repoId} will be updated to ${newCommit}`
-        );
-
-        if (this.source.type == "GitHubDownload") {
-          const repository = await getRepositoryFromGitHub({
-            accessToken: await this.source.getToken(),
-            owner: this.source.githubRepository.owner,
-            repo: this.source.githubRepository.repo,
-          });
-          if (
-            repository.size === undefined ||
-            repository.size > config.MAX_REPO_SIZE
-          ) {
-            console.log(
-              `[UPDATE] ${this._model.repoId} will be streamed instead of downloaded`
-            );
-            this._model.source.type = "GitHubStream";
-          }
-        }
-
-        await this.resetSate(RepositoryStatus.PREPARING);
-        await downloadQueue.add(this.repoId, this, {
-          jobId: this.repoId,
-          attempts: 3,
-        });
-      }
-    }
-  }
-  /**
-   * Download the require state for the repository to work
-   *
-   * @returns void
-   */
-  async anonymize() {
-    if (this.status === RepositoryStatus.READY) return;
-    await this.updateStatus(RepositoryStatus.PREPARING);
-    await this.files();
-    return this.updateStatus(RepositoryStatus.READY);
-  }
-
-  /**
-   * Update the last view and view count
-   */
-  async countView() {
-    this._model.lastView = new Date();
-    this._model.pageView = (this._model.pageView || 0) + 1;
-    if (!isConnected) return this.model;
-    return this._model.save();
-  }
-
-  /**
-   * Update the status of the repository
-   * @param status the new status
-   * @param errorMessage a potential error message to display
-   */
-  async updateStatus(status: RepositoryStatus, statusMessage?: string) {
-    if (!status) return this.model;
-    this._model.status = status;
-    this._model.statusDate = new Date();
-    this._model.statusMessage = statusMessage;
-    if (!isConnected) return this.model;
-    return this._model.save();
-  }
-
-  /**
-   * Expire the repository
-   */
-  async expire() {
-    await this.updateStatus(RepositoryStatus.EXPIRING);
-    await this.resetSate();
-    await this.updateStatus(RepositoryStatus.EXPIRED);
-  }
-
-  /**
-   * Remove the repository
-   */
-  async remove() {
-    await this.updateStatus(RepositoryStatus.REMOVING);
-    await this.resetSate();
-    await this.updateStatus(RepositoryStatus.REMOVED);
-  }
-
-  /**
-   * Reset/delete the state of the repository
-   */
-  async resetSate(status?: RepositoryStatus, statusMessage?: string) {
-    // remove attribute
-    this._model.size = { storage: 0, file: 0 };
-    this._model.originalFiles = undefined;
-    if (status) {
-      await this.updateStatus(status, statusMessage);
-    }
-    // remove cache
-    await this.removeCache();
-    console.log(`[RESET] ${this._model.repoId} has been reset`);
-  }
-
-  /**
-   * Remove the cached files
-   * @returns
-   */
-  async removeCache() {
-    this.model.isReseted = true;
-    await this.model.save();
-    if (
-      (await storage.exists(this._model.repoId + "/")) !== FILE_TYPE.NOT_FOUND
-    ) {
-      return storage.rm(this._model.repoId + "/");
-    }
-  }
-
-  /**
-   * Compute the size of the repository in term of storage and number of files.
-   *
-   * @returns The size of the repository in bite
-   */
-  async computeSize(): Promise<{
-    /**
-     * Size of the repository in bit
-     */
-    storage: number;
-    /**
-     * The number of files
-     */
-    file: number;
-  }> {
-    if (this.status !== RepositoryStatus.READY) return { storage: 0, file: 0 };
-    if (this._model.size.file) return this._model.size;
-    function recursiveCount(files: Tree): { storage: number; file: number } {
-      const out = { storage: 0, file: 0 };
-      for (const name in files) {
-        const file = files[name];
-        if (file.size && parseInt(file.size.toString()) == file.size) {
-          out.storage += file.size as number;
-          out.file++;
-        } else if (typeof file == "object") {
-          const r = recursiveCount(file as Tree);
-          out.storage += r.storage;
-          out.file += r.file;
-        }
-      }
-      return out;
-    }
-
-    const files = await this.files();
-    this._model.size = recursiveCount(files);
-    await this._model.save();
-    return this._model.size;
-  }
-
-  /**
-   * Returns the conference of the repository
-   *
-   * @returns conference of the repository
-   */
-  async conference(): Promise<Conference | null> {
-    if (!this._model.conference) {
-      return null;
-    }
-    const conference = await ConferenceModel.findOne({
-      conferenceID: this._model.conference,
-    });
-    if (conference) return new Conference(conference);
-    return null;
-  }
-
-  /***** Getters ********/
-
-  get repoId() {
-    return this._model.repoId;
-  }
-
-  get options() {
-    return this._model.options;
-  }
-
-  get model() {
-    return this._model;
-  }
-
-  get originalCachePath() {
-    return (
-      join(this._model.repoId, "original") +
-      (process.platform === "win32" ? "\\" : "/")
-    );
-  }
-
-  get status() {
-    return this._model.status;
-  }
-
-  get size() {
-    if (this.status != "ready") return { storage: 0, file: 0 };
-    return this._model.size;
-  }
-
-  toJSON() {
-    return {
-      repoId: this._model.repoId,
-      options: this._model.options,
-      conference: this._model.conference,
-      anonymizeDate: this._model.anonymizeDate,
-      status: this.status,
-      statusMessage: this._model.statusMessage,
-      source: this.source.toJSON(),
-      lastView: this._model.lastView,
-      pageView: this._model.pageView,
-      size: this.size,
-    };
-  }
-}

src/cli/index.ts

@@ -10,12 +10,12 @@ import { join } from "path";
 import * as gh from "parse-github-url";
 import * as inquirer from "inquirer";
 
-import server from "./src/server";
-import config from "./config";
-import GitHubDownload from "./src/source/GitHubDownload";
-import Repository from "./src/Repository";
-import AnonymizedRepositoryModel from "./src/database/anonymizedRepositories/anonymizedRepositories.model";
-import { getRepositoryFromGitHub } from "./src/source/GitHubRepository";
+import server from "../server";
+import config from "../config";
+import GitHubDownload from "../core/source/GitHubDownload";
+import Repository from "../core/Repository";
+import AnonymizedRepositoryModel from "../core/model/anonymizedRepositories/anonymizedRepositories.model";
+import { getRepositoryFromGitHub } from "../core/source/GitHubRepository";
 
 function generateRandomFileName(size: number) {
   const characters =
@@ -69,6 +69,7 @@ async function main() {
     accessToken: inq.token,
     owner: ghURL.owner,
     repo: ghURL.name,
+    force: true,
   });
   const branches = await ghRepo.branches({
     accessToken: inq.token,

src/config.ts

@@ -18,6 +18,7 @@ interface Config {
    * Allow to download repository and files
    */
   ENABLE_DOWNLOAD: boolean;
+  STREAMER_ENTRYPOINT: string | null;
   ANONYMIZATION_MASK: string;
   PORT: number;
   APP_HOSTNAME: string;
@@ -26,11 +27,11 @@ interface Config {
   DB_HOSTNAME: string;
   FOLDER: string;
   additionalExtensions: string[];
-  S3_BUCKET?: string;
-  S3_CLIENT_ID?: string;
-  S3_CLIENT_SECRET?: string;
-  S3_ENDPOINT?: string;
-  S3_REGION?: string;
+  S3_BUCKET: string | null;
+  S3_CLIENT_ID: string | null;
+  S3_CLIENT_SECRET: string | null;
+  S3_ENDPOINT: string | null;
+  S3_REGION: string | null;
   STORAGE: "filesystem" | "s3";
   TRUST_PROXY: number;
   RATE_LIMIT: number;
@@ -58,7 +59,7 @@ const config: Config = {
   DB_HOSTNAME: "mongodb",
   REDIS_HOSTNAME: "redis",
   REDIS_PORT: 6379,
-  FOLDER: resolve(__dirname, "repositories"),
+  FOLDER: resolve(__dirname, "..", "repositories"),
   additionalExtensions: [
     "license",
     "dockerfile",
@@ -70,11 +71,12 @@ const config: Config = {
     "in",
   ],
   STORAGE: "filesystem",
-  S3_BUCKET: process.env.S3_BUCKET,
-  S3_CLIENT_ID: process.env.S3_CLIENT_ID,
-  S3_CLIENT_SECRET: process.env.S3_CLIENT_SECRET,
-  S3_ENDPOINT: process.env.S3_ENDPOINT,
-  S3_REGION: process.env.S3_REGION,
+  STREAMER_ENTRYPOINT: null,
+  S3_BUCKET: null,
+  S3_CLIENT_ID: null,
+  S3_CLIENT_SECRET: null,
+  S3_ENDPOINT: null,
+  S3_REGION: null,
 };
 
 for (let conf in process.env) {