fix(#199): stop content download when request is canceled and always define contentLength

src/AnonymizedFile.ts

@@ -5,9 +5,10 @@ import Repository from "./Repository";
 import { Tree, TreeElement, TreeFile } from "./types";
 import storage from "./storage";
 import config from "../config";
-import { anonymizePath, anonymizeStream } from "./anonymize-utils";
+import { anonymizePath, AnonymizeTransformer } from "./anonymize-utils";
 import AnonymousError from "./AnonymousError";
 import { handleError } from "./routes/route-utils";
+import { tryCatch } from "bullmq";
 
 /**
  * Represent a file in a anonymized repository
@@ -177,7 +178,7 @@ export default class AnonymizedFile {
   }
 
   async anonymizedContent() {
-    return (await this.content()).pipe(anonymizeStream(this));
+    return (await this.content()).pipe(new AnonymizeTransformer(this));
   }
 
   get originalCachePath() {
@@ -204,15 +205,32 @@ export default class AnonymizedFile {
     if (this.extension()) {
       res.contentType(this.extension());
     }
-    if (this.fileSize) {
+    res.header("Accept-Ranges", "none");
-      res.set("Content-Length", this.fileSize.toString());
-    }
     return new Promise(async (resolve, reject) => {
       try {
-        (await this.anonymizedContent())
+        const content = await this.content();
+        try {
+          const fileInfo = await storage.fileInfo(this.originalCachePath);
+          if (fileInfo.size) {
+            res.header("Content-Length", fileInfo.size.toString());
+          }
+        } catch (error) {
+          // unable to get file size
+          console.error(error);
+        }
+        content
+          .pipe(new AnonymizeTransformer(this))
           .pipe(res)
-          .on("close", () => resolve())
+          .on("close", () => {
+            if (!content.closed && !content.destroyed) {
+              content.destroy();
+            }
+            resolve();
+          })
           .on("error", (error) => {
+            if (!content.closed && !content.destroyed) {
+              content.destroy();
+            }
             reject(error);
             handleError(error, res);
           });

src/Repository.ts

@@ -9,7 +9,7 @@ import Zip from "./source/Zip";
 import { anonymizePath } from "./anonymize-utils";
 import UserModel from "./database/users/users.model";
 import { IAnonymizedRepositoryDocument } from "./database/anonymizedRepositories/anonymizedRepositories.types";
-import { anonymizeStream } from "./anonymize-utils";
+import { AnonymizeTransformer } from "./anonymize-utils";
 import GitHubBase from "./source/GitHubBase";
 import Conference from "./Conference";
 import ConferenceModel from "./database/conference/conferences.model";
@@ -168,7 +168,7 @@ export default class Repository {
     return storage.archive(this.originalCachePath, {
       format: "zip",
       fileTransformer: (filename: string) =>
-        anonymizeStream(
+        new AnonymizeTransformer(
           new AnonymizedFile({
             repository: this,
             anonymizedPath: filename,

src/anonymize-utils.ts

@@ -31,45 +31,20 @@ export function isTextFile(filePath: string, content: Buffer) {
   return isText(filename, content);
 }
 
-export function anonymizeStream(file: AnonymizedFile) {
+export class AnonymizeTransformer extends Transform {
-  const ts = new Transform();
+  constructor(private readonly file: AnonymizedFile) {
-  var chunks: Buffer[] = [],
+    super();
-    len = 0,
+  }
-    pos = 0;
 
-  ts._transform = function _transform(chunk, enc, cb) {
+  _transform(chunk: Buffer, encoding: string, callback: () => void) {
-    chunks.push(chunk);
+    if (isTextFile(this.file.anonymizedPath, chunk)) {
-    len += chunk.length;
+      chunk = Buffer.from(
-
+        anonymizeContent(chunk.toString(), this.file.repository)
-    if (pos === 1) {
+      );
-      let data: any = Buffer.concat(chunks, len);
-      if (isTextFile(file.anonymizedPath, data)) {
-        data = anonymizeContent(data.toString(), file.repository);
-      }
-
-      chunks = [];
-      len = 0;
-
-      this.push(data);
     }
-
+    this.push(chunk);
-    pos = 1 ^ pos;
+    callback();
-    cb(null);
+  }
-  };
-
-  ts._flush = function _flush(cb) {
-    if (chunks.length) {
-      let data = Buffer.concat(chunks, len);
-      if (isText(file.anonymizedPath, data)) {
-        data = Buffer.from(anonymizeContent(data.toString(), file.repository));
-      }
-
-      this.push(data);
-    }
-
-    cb(null);
-  };
-  return ts;
 }
 
 interface Anonymizationptions {

src/storage/FileSystem.ts

@@ -9,6 +9,7 @@ import { Readable, pipeline, Transform } from "stream";
 import * as archiver from "archiver";
 import { promisify } from "util";
 import AnonymizedFile from "../AnonymizedFile";
+import { lookup } from "mime-types";
 
 export default class FileSystem implements StorageBase {
   type = "FileSystem";
@@ -30,6 +31,17 @@ export default class FileSystem implements StorageBase {
     return fs.createReadStream(join(config.FOLDER, p));
   }
 
+  async fileInfo(path: string) {
+    const info = await fs.promises.stat(join(config.FOLDER, path));
+    return {
+      size: info.size,
+      lastModified: info.mtime,
+      contentType: info.isDirectory()
+        ? "application/x-directory"
+        : lookup(join(config.FOLDER, path)) as string,
+    };
+  }
+
   /** @override */
   async write(
     p: string,

src/storage/S3.ts

@@ -122,6 +122,23 @@ export default class S3Storage implements StorageBase {
     s.send();
   }
 
+  async fileInfo(path: string) {
+    if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");
+    const info = await this.client(3000)
+      .headObject({
+        Bucket: config.S3_BUCKET,
+        Key: path,
+      })
+      .promise();
+    return {
+      size: info.ContentLength,
+      lastModified: info.LastModified,
+      contentType: info.ContentType
+        ? info.ContentType
+        : (lookup(path) as string),
+    };
+  }
+
   /** @override */
   read(path: string): Readable {
     if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set");

src/types.ts

@@ -52,6 +52,12 @@ export interface StorageBase {
    */
   read(path: string): Readable;
 
+  fileInfo(path: string): Promise<{
+    size: number | undefined;
+    lastModified: Date | undefined;
+    contentType: string;
+  }>;
+
   /**
    * Write data to a file
    * @param path the path to the file