From 13e5e35d4608e8fe9d63eb76476dbfa97380a145 Mon Sep 17 00:00:00 2001 From: tdurieux Date: Thu, 20 Apr 2023 23:20:08 +0200 Subject: [PATCH] fix(#199): stop content download when request is canceled and always define contentLength --- src/AnonymizedFile.ts | 32 +++++++++++++++++++------ src/Repository.ts | 4 ++-- src/anonymize-utils.ts | 49 ++++++++++----------------------------- src/storage/FileSystem.ts | 12 ++++++++++ src/storage/S3.ts | 17 ++++++++++++++ src/types.ts | 6 +++++ 6 files changed, 74 insertions(+), 46 deletions(-) diff --git a/src/AnonymizedFile.ts b/src/AnonymizedFile.ts index 697b51e..8bfe4fa 100644 --- a/src/AnonymizedFile.ts +++ b/src/AnonymizedFile.ts @@ -5,9 +5,10 @@ import Repository from "./Repository"; import { Tree, TreeElement, TreeFile } from "./types"; import storage from "./storage"; import config from "../config"; -import { anonymizePath, anonymizeStream } from "./anonymize-utils"; +import { anonymizePath, AnonymizeTransformer } from "./anonymize-utils"; import AnonymousError from "./AnonymousError"; import { handleError } from "./routes/route-utils"; +import { tryCatch } from "bullmq"; /** * Represent a file in a anonymized repository @@ -177,7 +178,7 @@ export default class AnonymizedFile { } async anonymizedContent() { - return (await this.content()).pipe(anonymizeStream(this)); + return (await this.content()).pipe(new AnonymizeTransformer(this)); } get originalCachePath() { @@ -204,15 +205,32 @@ export default class AnonymizedFile { if (this.extension()) { res.contentType(this.extension()); } - if (this.fileSize) { - res.set("Content-Length", this.fileSize.toString()); - } + res.header("Accept-Ranges", "none"); return new Promise(async (resolve, reject) => { try { - (await this.anonymizedContent()) + const content = await this.content(); + try { + const fileInfo = await storage.fileInfo(this.originalCachePath); + if (fileInfo.size) { + res.header("Content-Length", fileInfo.size.toString()); + } + } catch (error) { + // unable to get file size + console.error(error); + } + content + .pipe(new AnonymizeTransformer(this)) .pipe(res) - .on("close", () => resolve()) + .on("close", () => { + if (!content.closed && !content.destroyed) { + content.destroy(); + } + resolve(); + }) .on("error", (error) => { + if (!content.closed && !content.destroyed) { + content.destroy(); + } reject(error); handleError(error, res); }); diff --git a/src/Repository.ts b/src/Repository.ts index b5cf972..46c688c 100644 --- a/src/Repository.ts +++ b/src/Repository.ts @@ -9,7 +9,7 @@ import Zip from "./source/Zip"; import { anonymizePath } from "./anonymize-utils"; import UserModel from "./database/users/users.model"; import { IAnonymizedRepositoryDocument } from "./database/anonymizedRepositories/anonymizedRepositories.types"; -import { anonymizeStream } from "./anonymize-utils"; +import { AnonymizeTransformer } from "./anonymize-utils"; import GitHubBase from "./source/GitHubBase"; import Conference from "./Conference"; import ConferenceModel from "./database/conference/conferences.model"; @@ -168,7 +168,7 @@ export default class Repository { return storage.archive(this.originalCachePath, { format: "zip", fileTransformer: (filename: string) => - anonymizeStream( + new AnonymizeTransformer( new AnonymizedFile({ repository: this, anonymizedPath: filename, diff --git a/src/anonymize-utils.ts b/src/anonymize-utils.ts index 98cddeb..cc3db45 100644 --- a/src/anonymize-utils.ts +++ b/src/anonymize-utils.ts @@ -31,45 +31,20 @@ export function isTextFile(filePath: string, content: Buffer) { return isText(filename, content); } -export function anonymizeStream(file: AnonymizedFile) { - const ts = new Transform(); - var chunks: Buffer[] = [], - len = 0, - pos = 0; +export class AnonymizeTransformer extends Transform { + constructor(private readonly file: AnonymizedFile) { + super(); + } - ts._transform = function _transform(chunk, enc, cb) { - chunks.push(chunk); - len += chunk.length; - - if (pos === 1) { - let data: any = Buffer.concat(chunks, len); - if (isTextFile(file.anonymizedPath, data)) { - data = anonymizeContent(data.toString(), file.repository); - } - - chunks = []; - len = 0; - - this.push(data); + _transform(chunk: Buffer, encoding: string, callback: () => void) { + if (isTextFile(this.file.anonymizedPath, chunk)) { + chunk = Buffer.from( + anonymizeContent(chunk.toString(), this.file.repository) + ); } - - pos = 1 ^ pos; - cb(null); - }; - - ts._flush = function _flush(cb) { - if (chunks.length) { - let data = Buffer.concat(chunks, len); - if (isText(file.anonymizedPath, data)) { - data = Buffer.from(anonymizeContent(data.toString(), file.repository)); - } - - this.push(data); - } - - cb(null); - }; - return ts; + this.push(chunk); + callback(); + } } interface Anonymizationptions { diff --git a/src/storage/FileSystem.ts b/src/storage/FileSystem.ts index 2c37ac2..856ab98 100644 --- a/src/storage/FileSystem.ts +++ b/src/storage/FileSystem.ts @@ -9,6 +9,7 @@ import { Readable, pipeline, Transform } from "stream"; import * as archiver from "archiver"; import { promisify } from "util"; import AnonymizedFile from "../AnonymizedFile"; +import { lookup } from "mime-types"; export default class FileSystem implements StorageBase { type = "FileSystem"; @@ -30,6 +31,17 @@ export default class FileSystem implements StorageBase { return fs.createReadStream(join(config.FOLDER, p)); } + async fileInfo(path: string) { + const info = await fs.promises.stat(join(config.FOLDER, path)); + return { + size: info.size, + lastModified: info.mtime, + contentType: info.isDirectory() + ? "application/x-directory" + : lookup(join(config.FOLDER, path)) as string, + }; + } + /** @override */ async write( p: string, diff --git a/src/storage/S3.ts b/src/storage/S3.ts index c4ba275..164c485 100644 --- a/src/storage/S3.ts +++ b/src/storage/S3.ts @@ -122,6 +122,23 @@ export default class S3Storage implements StorageBase { s.send(); } + async fileInfo(path: string) { + if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set"); + const info = await this.client(3000) + .headObject({ + Bucket: config.S3_BUCKET, + Key: path, + }) + .promise(); + return { + size: info.ContentLength, + lastModified: info.LastModified, + contentType: info.ContentType + ? info.ContentType + : (lookup(path) as string), + }; + } + /** @override */ read(path: string): Readable { if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set"); diff --git a/src/types.ts b/src/types.ts index fff94f3..f5f2842 100644 --- a/src/types.ts +++ b/src/types.ts @@ -52,6 +52,12 @@ export interface StorageBase { */ read(path: string): Readable; + fileInfo(path: string): Promise<{ + size: number | undefined; + lastModified: Date | undefined; + contentType: string; + }>; + /** * Write data to a file * @param path the path to the file