import { appendArchiveEntry, zipEntryOptions } from "../archive-entry"; import config from "../../config"; import * as fs from "fs"; import { Extract } from "unzip-stream"; import { join, dirname } from "path"; import { Response } from "express"; import { Readable, pipeline, Transform } from "stream"; import * as archiver from "archiver"; import { promisify } from "util"; import { lookup } from "mime-types"; import StorageBase, { FILE_TYPE } from "./Storage"; import FileModel from "../model/files/files.model"; import { IFile } from "../model/files/files.types"; import { createLogger, serializeError } from "../logger"; import AnonymousError from "../AnonymousError"; const logger = createLogger("fs"); export default class FileSystem extends StorageBase { type = "FileSystem"; constructor() { super(); } /** @override */ async exists(repoId: string, p: string = ""): Promise { this.assertSafePath(p); const fullPath = join(config.FOLDER, this.repoPath(repoId), p); try { const stat = await fs.promises.stat(fullPath); if (stat.isDirectory()) return FILE_TYPE.FOLDER; if (stat.isFile()) return FILE_TYPE.FILE; } catch { // ignore file not found or not downloaded } return FILE_TYPE.NOT_FOUND; } /** @override */ async send(repoId: string, p: string, res: Response) { this.assertSafePath(p); const fullPath = join(config.FOLDER, this.repoPath(repoId), p); res.sendFile(fullPath, { dotfiles: "allow" }); } /** @override */ async read(repoId: string, p: string): Promise { this.assertSafePath(p); const fullPath = join(config.FOLDER, this.repoPath(repoId), p); return fs.createReadStream(fullPath); } async fileInfo(repoId: string, path: string) { this.assertSafePath(path); const fullPath = join(config.FOLDER, this.repoPath(repoId), path); const info = await fs.promises.stat(fullPath); return { size: info.size, lastModified: info.mtime, contentType: info.isDirectory() ? "application/x-directory" : (lookup(fullPath) as string), }; } /** @override */ async write( repoId: string, p: string, data: string | Readable, _source?: string, expectedSize?: number ): Promise { this.assertSafePath(p); const fullPath = join(config.FOLDER, this.repoPath(repoId), p); // Atomic write: stream into a sibling .tmp and only rename into place // when the source stream finishes successfully. If the source errors // mid-flight (transient GitHub 5xx, socket reset, etc.), we drop the // tmp and leave any pre-existing cached file untouched. Without this, // a partial fetch would commit a 0-byte or truncated cache entry that // future reads would happily serve as the file's content. await this.mk(repoId, dirname(p)); const tmpPath = `${fullPath}.tmp.${process.pid}.${Date.now()}.${Math.random() .toString(36) .slice(2, 8)}`; let sourceFailed = false; try { if (typeof data === "string") { await fs.promises.writeFile(tmpPath, data); } else { await new Promise((resolve, reject) => { const ws = fs.createWriteStream(tmpPath); let settled = false; const finish = (err?: Error) => { if (settled) return; settled = true; if (err) { ws.destroy(); reject(err); } else { resolve(); } }; data.on("error", (error) => { if (!settled) sourceFailed = true; finish(error); }); ws.on("error", finish); ws.on("finish", () => finish()); data.pipe(ws); }); } // Size guard: stat the tmp before renaming. A short read produces a // truncated tmp file that we MUST NOT promote — accepting it would // shadow the real content on every subsequent request. Allow >= // expectedSize because Git LFS files resolve to content larger than // the pointer's recorded size. if (typeof expectedSize === "number" && expectedSize > 0) { const stat = await fs.promises.stat(tmpPath); if (stat.size < expectedSize) { await fs.promises.rm(tmpPath, { force: true }).catch(() => undefined); throw new AnonymousError("storage_write_size_mismatch", { httpStatus: 502, object: { path: p, expected: expectedSize, actual: stat.size }, }); } } await fs.promises.rename(tmpPath, fullPath); } catch (err) { if (sourceFailed) logger.warn("source stream failed; cache write discarded", serializeError(err)); else logger.error("write failed", serializeError(err)); await fs.promises.rm(tmpPath, { force: true }).catch(() => undefined); throw err; } } /** @override */ async rm(repoId: string, dir: string = ""): Promise { this.assertSafePath(dir); const fullPath = join(config.FOLDER, this.repoPath(repoId), dir); await fs.promises.rm(fullPath, { force: true, recursive: true, }); } async removeLegacyContent(repoId: string): Promise { let directory: fs.Dir; try { directory = await fs.promises.opendir(join(config.FOLDER, this.repoPath(repoId))); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return; throw error; } for await (const entry of directory) { if (entry.name !== "__content" || !entry.isDirectory()) { await this.rm(repoId, entry.name); continue; } const content = await fs.promises.opendir(join(config.FOLDER, this.repoPath(repoId), "__content")); for await (const generation of content) { if (generation.name !== "__retired" && !/^[a-f0-9]{64}$/.test(generation.name)) { await this.rm(repoId, join("__content", generation.name)); } } } } /** @override */ async mk(repoId: string, dir: string = ""): Promise { this.assertSafePath(dir); const fullPath = join(config.FOLDER, this.repoPath(repoId), dir); try { await fs.promises.mkdir(fullPath, { recursive: true, }); } catch (err: unknown) { if (err instanceof Error && (err as NodeJS.ErrnoException).code !== "EEXIST") { throw err; } } } /** @override */ async listFiles( repoId: string, dir: string = "", opt: { onEntry?: (file: { path: string; size: number }) => void; } = {} ): Promise { this.assertSafePath(dir); const fullPath = join(config.FOLDER, this.repoPath(repoId), dir); const files = await fs.promises.readdir(fullPath); const output2: IFile[] = []; for (const file of files) { const filePath = join(fullPath, file); try { const stats = await fs.promises.stat(filePath); if (stats.isDirectory()) { output2.push(new FileModel({ name: file, path: dir, repoId })); output2.push( ...(await this.listFiles(repoId, join(dir, file), opt)) ); } else if (stats.isFile()) { if (opt.onEntry) { opt.onEntry({ path: join(dir, file), size: stats.size, }); } // Don't synthesise a sha here. The previous value (stats.ino) // wasn't a content hash — just an inode number — and any code // that compared it to an upstream Git blob sha would silently // disagree. Leave it undefined so callers either look up the // real sha from FileModel/GitHub or skip sha-keyed paths. output2.push( new FileModel({ name: file, path: dir, repoId: repoId, size: stats.size, }) ); } } catch { // ignore stat errors for individual files } } return output2; } /** @override */ async extractZip(repoId: string, p: string, data: Readable): Promise { this.assertSafePath(p); const pipe = promisify(pipeline); const fullPath = join(config.FOLDER, this.repoPath(repoId), p); const extractor = Extract({ path: fullPath, decodeString: (buf) => { const name = buf.toString(); // Strip the top-level directory GitHub wraps every entry in, then // drop any "../" / absolute segments so a malicious entry name // cannot escape the extraction root (zip-slip, CWE-24). const stripped = name.substr(name.indexOf("/") + 1); const newName = this.sanitizeZipEntryName(stripped); if (newName == "") { return "___IGNORE___"; } return newName; }, }); await pipe(data, extractor); await this.rm(repoId, join(p, "___IGNORE___")); } /** @override */ async archive( repoId: string, dir: string, opt?: { format?: "zip" | "tar"; fileTransformer?: (path: string) => Transform; } ) { this.assertSafePath(dir); const archive = archiver(opt?.format || "zip", {}); let active: Readable | undefined; let stopped = false; archive.once("close", () => { stopped = true; active?.destroy(); }); void (async () => { for (const file of await this.listFiles(repoId, dir)) { if (stopped || file.size == null) continue; const filename = join(file.path, file.name); const source = await this.read(repoId, filename); if (stopped) { source.destroy(); return; } active = source; let output = source; if (opt?.fileTransformer) { const transformer = opt.fileTransformer(filename); source.on("error", error => transformer.destroy(error)); transformer.once("close", () => source.destroy()); output = source.pipe(transformer); active = output; } await appendArchiveEntry(archive, output, zipEntryOptions(filename.slice(dir ? dir.length + 1 : 0), filename)); } if (!stopped) await archive.finalize(); })().catch(error => archive.destroy(error)); return archive; } }