feat: introduce streamers that handle the stream and anonymization from github

src/streamer/index.ts

@@ -0,0 +1,31 @@
+import { config as dotenv } from "dotenv";
+dotenv();
+
+import * as express from "express";
+import * as compression from "compression";
+
+import config from "../config";
+import router from "./route";
+import { handleError } from "../server/routes/route-utils";
+import AnonymousError from "../core/AnonymousError";
+
+const app = express();
+app.use(express.json());
+
+app.use(compression());
+
+app.use("/api", router);
+
+app.all("*", (req, res) => {
+  handleError(
+    new AnonymousError("file_not_found", {
+      httpStatus: 404,
+      object: req.originalUrl,
+    }),
+    res,
+    req
+  );
+});
+app.listen(config.PORT, () => {
+  console.log(`Server started on http://streamer:${config.PORT}`);
+});

src/streamer/route.ts

@@ -0,0 +1,63 @@
+import * as express from "express";
+import GitHubStream from "../core/source/GitHubStream";
+import { AnonymizeTransformer, isTextFile } from "../core/anonymize-utils";
+import { handleError } from "../server/routes/route-utils";
+import { contentType } from "mime-types";
+
+export const router = express.Router();
+
+router.post("/", async (req: express.Request, res: express.Response) => {
+  req.body = req.body || {};
+  const token: string = req.body.token;
+  const repoFullName = req.body.repoFullName.split("/");
+  const repoId = req.body.repoId;
+  const branch = req.body.branch;
+  const fileSha = req.body.sha;
+  const commit = req.body.commit;
+  const filePath = req.body.filePath;
+  const anonymizerOptions = req.body.anonymizerOptions;
+  const anonymizer = new AnonymizeTransformer(anonymizerOptions);
+
+  const source = new GitHubStream({
+    repoId,
+    organization: repoFullName[0],
+    repoName: repoFullName[1],
+    commit: commit,
+    getToken: () => token,
+  });
+  const content = source.downloadFile(token, fileSha);
+  try {
+    const mime = contentType(filePath);
+    if (mime && !filePath.endsWith(".ts")) {
+      res.contentType(mime);
+    } else if (isTextFile(filePath)) {
+      res.contentType("text/plain");
+    }
+    res.header("Accept-Ranges", "none");
+    anonymizer.once("transform", (data) => {
+      if (!mime && data.isText) {
+        res.contentType("text/plain");
+      }
+    });
+    function handleStreamError(error: Error) {
+      if (!content.closed && !content.destroyed) {
+        content.destroy();
+      }
+      handleError(error, res);
+    }
+    content
+      .on("error", handleStreamError)
+      .pipe(anonymizer)
+      .pipe(res)
+      .on("error", handleStreamError)
+      .on("close", () => {
+        if (!content.closed && !content.destroyed) {
+          content.destroy();
+        }
+      });
+  } catch (error) {
+    handleError(error, res);
+  }
+});
+
+export default router;