From 027f14ffbc5017f29af54047f502e569e513db5d Mon Sep 17 00:00:00 2001 From: tdurieux Date: Mon, 8 May 2023 15:16:56 +0200 Subject: [PATCH] fix: improve error message for folders --- src/AnonymizedFile.ts | 12 +++++++++--- src/Repository.ts | 13 +++++++++++-- src/source/GitHubDownload.ts | 12 +++++++++--- src/storage/FileSystem.ts | 21 ++++++++++++--------- src/storage/S3.ts | 16 ++++++++-------- src/types.ts | 8 +++++++- 6 files changed, 56 insertions(+), 26 deletions(-) diff --git a/src/AnonymizedFile.ts b/src/AnonymizedFile.ts index 34cd11c..7dcae15 100644 --- a/src/AnonymizedFile.ts +++ b/src/AnonymizedFile.ts @@ -2,7 +2,7 @@ import { join, basename } from "path"; import { Response } from "express"; import { Readable } from "stream"; import Repository from "./Repository"; -import { Tree, TreeElement, TreeFile } from "./types"; +import { FILE_TYPE, Tree, TreeElement, TreeFile } from "./types"; import storage from "./storage"; import config from "../config"; import { @@ -175,8 +175,14 @@ export default class AnonymizedFile { httpStatus: 403, }); } - if (await storage.exists(this.originalCachePath)) { + const exist = await storage.exists(this.originalCachePath); + if (exist == FILE_TYPE.FILE) { return storage.read(this.originalCachePath); + } else if (exist == FILE_TYPE.FOLDER) { + throw new AnonymousError("folder_not_supported", { + object: this, + httpStatus: 400, + }); } return await this.repository.source?.getFileContent(this); } @@ -223,7 +229,7 @@ export default class AnonymizedFile { // unable to get file size console.error(error); } - + const anonymizer = new AnonymizeTransformer(this); anonymizer.once("transform", (data) => { diff --git a/src/Repository.ts b/src/Repository.ts index 64cd4cb..f97d0e9 100644 --- a/src/Repository.ts +++ b/src/Repository.ts @@ -1,6 +1,13 @@ import { join } from "path"; import storage from "./storage"; -import { RepositoryStatus, Source, Tree, TreeElement, TreeFile } from "./types"; +import { + FILE_TYPE, + RepositoryStatus, + Source, + Tree, + TreeElement, + TreeFile, +} from "./types"; import { Readable } from "stream"; import User from "./User"; import GitHubStream from "./source/GitHubStream"; @@ -333,7 +340,9 @@ export default class Repository { async removeCache() { this.model.isReseted = true; await this.model.save(); - if (await storage.exists(this._model.repoId + "/")) { + if ( + (await storage.exists(this._model.repoId + "/")) !== FILE_TYPE.NOT_FOUND + ) { return storage.rm(this._model.repoId + "/"); } } diff --git a/src/source/GitHubDownload.ts b/src/source/GitHubDownload.ts index 58866a2..5830d44 100644 --- a/src/source/GitHubDownload.ts +++ b/src/source/GitHubDownload.ts @@ -8,7 +8,7 @@ import storage from "../storage"; import Repository from "../Repository"; import GitHubBase from "./GitHubBase"; import AnonymizedFile from "../AnonymizedFile"; -import { RepositoryStatus, SourceBase } from "../types"; +import { FILE_TYPE, RepositoryStatus, SourceBase } from "../types"; import AnonymousError from "../AnonymousError"; import { tryCatch } from "bullmq"; @@ -132,8 +132,14 @@ export default class GitHubDownload extends GitHubBase implements SourceBase { } async getFileContent(file: AnonymizedFile): Promise { - if (await storage.exists(file.originalCachePath)) { + const exists = await storage.exists(file.originalCachePath); + if (exists === FILE_TYPE.FILE) { return storage.read(file.originalCachePath); + } else if (exists === FILE_TYPE.FOLDER) { + throw new AnonymousError("folder_not_supported", { + httpStatus: 400, + object: file, + }); } // will throw an error if the file is not in the repository await file.originalPath(); @@ -145,7 +151,7 @@ export default class GitHubDownload extends GitHubBase implements SourceBase { async getFiles() { const folder = this.repository.originalCachePath; - if (!(await storage.exists(folder))) { + if ((await storage.exists(folder)) === FILE_TYPE.NOT_FOUND) { await this.download(); } return storage.listFiles(folder); diff --git a/src/storage/FileSystem.ts b/src/storage/FileSystem.ts index 9255c41..ab40489 100644 --- a/src/storage/FileSystem.ts +++ b/src/storage/FileSystem.ts @@ -1,4 +1,4 @@ -import { SourceBase, StorageBase, Tree } from "../types"; +import { FILE_TYPE, SourceBase, StorageBase, Tree } from "../types"; import config from "../../config"; import * as fs from "fs"; @@ -17,8 +17,15 @@ export default class FileSystem implements StorageBase { constructor() {} /** @override */ - async exists(p: string): Promise { - return fs.existsSync(join(config.FOLDER, p)); + async exists(p: string): Promise { + try { + const stat = await fs.promises.stat(join(config.FOLDER, p)); + if (stat.isDirectory()) return FILE_TYPE.FOLDER; + if (stat.isFile()) return FILE_TYPE.FILE; + } catch (_) { + // ignore file not found or not downloaded + } + return FILE_TYPE.NOT_FOUND; } /** @override */ @@ -49,11 +56,7 @@ export default class FileSystem implements StorageBase { file?: AnonymizedFile, source?: SourceBase ): Promise { - if (!(await this.exists(dirname(p)))) { - await fs.promises.mkdir(dirname(join(config.FOLDER, p)), { - recursive: true, - }); - } + await this.mk(dirname(p)); return fs.promises.writeFile(join(config.FOLDER, p), data); } @@ -67,7 +70,7 @@ export default class FileSystem implements StorageBase { /** @override */ async mk(dir: string): Promise { - if (!(await this.exists(dir))) + if ((await this.exists(dir)) === FILE_TYPE.NOT_FOUND) fs.promises.mkdir(join(config.FOLDER, dir), { recursive: true }); } diff --git a/src/storage/S3.ts b/src/storage/S3.ts index 4b9abfa..f09bb1d 100644 --- a/src/storage/S3.ts +++ b/src/storage/S3.ts @@ -1,4 +1,4 @@ -import { SourceBase, StorageBase, Tree, TreeFile } from "../types"; +import { FILE_TYPE, SourceBase, StorageBase, Tree, TreeFile } from "../types"; import { GetObjectCommand, ListObjectsV2CommandOutput, @@ -44,14 +44,12 @@ export default class S3Storage implements StorageBase { } /** @override */ - async exists(path: string): Promise { + async exists(path: string): Promise { if (!config.S3_BUCKET) throw new Error("S3_BUCKET not set"); try { - await this.client().headObject({ - Bucket: config.S3_BUCKET, - Key: path, - }); - return true; + // if we can get the file info, it is a file + await this.fileInfo(path); + return FILE_TYPE.FILE; } catch (err) { // check if it is a directory const data = await this.client().listObjectsV2({ @@ -59,7 +57,9 @@ export default class S3Storage implements StorageBase { Prefix: path, MaxKeys: 1, }); - return (data.Contents?.length || 0) > 0; + return (data.Contents?.length || 0) > 0 + ? FILE_TYPE.FOLDER + : FILE_TYPE.NOT_FOUND; } } diff --git a/src/types.ts b/src/types.ts index 04a6e9e..48c6df6 100644 --- a/src/types.ts +++ b/src/types.ts @@ -32,6 +32,12 @@ export interface SourceBase { export type Source = GitHubDownload | GitHubStream | Zip; +export enum FILE_TYPE { + FILE = "file", + FOLDER = "folder", + NOT_FOUND = "not_found", +} + export interface StorageBase { /** * The type of storage @@ -42,7 +48,7 @@ export interface StorageBase { * check if the path exists * @param path the path to check */ - exists(path: string): Promise; + exists(path: string): Promise; send(p: string, res: Response): Promise;