anonymous_github/src/core/storage/FileSystem.ts

import { appendArchiveEntry, zipEntryOptions } from "../archive-entry";
import config from "../../config";
import * as fs from "fs";
import { Extract } from "unzip-stream";
import { join, dirname } from "path";
import { Response } from "express";
import { Readable, pipeline, Transform } from "stream";
import * as archiver from "archiver";
import { promisify } from "util";
import { lookup } from "mime-types";
import StorageBase, { FILE_TYPE } from "./Storage";
import FileModel from "../model/files/files.model";
import { IFile } from "../model/files/files.types";
import { createLogger, serializeError } from "../logger";
import AnonymousError from "../AnonymousError";

const logger = createLogger("fs");

export default class FileSystem extends StorageBase {
  type = "FileSystem";

  constructor() {
    super();
  }

  /** @override */
  async exists(repoId: string, p: string = ""): Promise<FILE_TYPE> {
    this.assertSafePath(p);
    const fullPath = join(config.FOLDER, this.repoPath(repoId), p);
    try {
      const stat = await fs.promises.stat(fullPath);
      if (stat.isDirectory()) return FILE_TYPE.FOLDER;
      if (stat.isFile()) return FILE_TYPE.FILE;
    } catch {
      // ignore file not found or not downloaded
    }
    return FILE_TYPE.NOT_FOUND;
  }

  /** @override */
  async send(repoId: string, p: string, res: Response) {
    this.assertSafePath(p);
    const fullPath = join(config.FOLDER, this.repoPath(repoId), p);
    res.sendFile(fullPath, { dotfiles: "allow" });
  }

  /** @override */
  async read(repoId: string, p: string): Promise<Readable> {
    this.assertSafePath(p);
    const fullPath = join(config.FOLDER, this.repoPath(repoId), p);
    return fs.createReadStream(fullPath);
  }

  async fileInfo(repoId: string, path: string) {
    this.assertSafePath(path);
    const fullPath = join(config.FOLDER, this.repoPath(repoId), path);
    const info = await fs.promises.stat(fullPath);
    return {
      size: info.size,
      lastModified: info.mtime,
      contentType: info.isDirectory()
        ? "application/x-directory"
        : (lookup(fullPath) as string),
    };
  }

  /** @override */
  async write(
    repoId: string,
    p: string,
    data: string | Readable,
    _source?: string,
    expectedSize?: number
  ): Promise<void> {
    this.assertSafePath(p);
    const fullPath = join(config.FOLDER, this.repoPath(repoId), p);
    // Atomic write: stream into a sibling .tmp and only rename into place
    // when the source stream finishes successfully. If the source errors
    // mid-flight (transient GitHub 5xx, socket reset, etc.), we drop the
    // tmp and leave any pre-existing cached file untouched. Without this,
    // a partial fetch would commit a 0-byte or truncated cache entry that
    // future reads would happily serve as the file's content.
    await this.mk(repoId, dirname(p));
    const tmpPath = `${fullPath}.tmp.${process.pid}.${Date.now()}.${Math.random()
      .toString(36)
      .slice(2, 8)}`;
    let sourceFailed = false;
    try {
      if (typeof data === "string") {
        await fs.promises.writeFile(tmpPath, data);
      } else {
        await new Promise<void>((resolve, reject) => {
          const ws = fs.createWriteStream(tmpPath);
          let settled = false;
          const finish = (err?: Error) => {
            if (settled) return;
            settled = true;
            if (err) {
              ws.destroy();
              reject(err);
            } else {
              resolve();
            }
          };
          data.on("error", (error) => {
            if (!settled) sourceFailed = true;
            finish(error);
          });
          ws.on("error", finish);
          ws.on("finish", () => finish());
          data.pipe(ws);
        });
      }
      // Size guard: stat the tmp before renaming. A short read produces a
      // truncated tmp file that we MUST NOT promote — accepting it would
      // shadow the real content on every subsequent request. Allow >=
      // expectedSize because Git LFS files resolve to content larger than
      // the pointer's recorded size.
      if (typeof expectedSize === "number" && expectedSize > 0) {
        const stat = await fs.promises.stat(tmpPath);
        if (stat.size < expectedSize) {
          await fs.promises.rm(tmpPath, { force: true }).catch(() => undefined);
          throw new AnonymousError("storage_write_size_mismatch", {
            httpStatus: 502,
            object: { path: p, expected: expectedSize, actual: stat.size },
          });
        }
      }
      await fs.promises.rename(tmpPath, fullPath);
    } catch (err) {
      if (sourceFailed) logger.warn("source stream failed; cache write discarded", serializeError(err));
      else logger.error("write failed", serializeError(err));
      await fs.promises.rm(tmpPath, { force: true }).catch(() => undefined);
      throw err;
    }
  }

  /** @override */
  async rm(repoId: string, dir: string = ""): Promise<void> {
    this.assertSafePath(dir);
    const fullPath = join(config.FOLDER, this.repoPath(repoId), dir);
    await fs.promises.rm(fullPath, {
      force: true,
      recursive: true,
    });
  }

  async removeLegacyContent(repoId: string): Promise<void> {
    let directory: fs.Dir;
    try { directory = await fs.promises.opendir(join(config.FOLDER, this.repoPath(repoId))); }
    catch (error) {
      if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
      throw error;
    }
    for await (const entry of directory) {
      if (entry.name !== "__content" || !entry.isDirectory()) { await this.rm(repoId, entry.name); continue; }
      const content = await fs.promises.opendir(join(config.FOLDER, this.repoPath(repoId), "__content"));
      for await (const generation of content) {
        if (generation.name !== "__retired" && !/^[a-f0-9]{64}$/.test(generation.name)) {
          await this.rm(repoId, join("__content", generation.name));
        }
      }
    }
  }

  /** @override */
  async mk(repoId: string, dir: string = ""): Promise<void> {
    this.assertSafePath(dir);
    const fullPath = join(config.FOLDER, this.repoPath(repoId), dir);
    try {
      await fs.promises.mkdir(fullPath, {
        recursive: true,
      });
    } catch (err: unknown) {
      if (err instanceof Error && (err as NodeJS.ErrnoException).code !== "EEXIST") {
        throw err;
      }
    }
  }

  /** @override */
  async listFiles(
    repoId: string,
    dir: string = "",
    opt: {
      onEntry?: (file: { path: string; size: number }) => void;
    } = {}
  ): Promise<IFile[]> {
    this.assertSafePath(dir);
    const fullPath = join(config.FOLDER, this.repoPath(repoId), dir);
    const files = await fs.promises.readdir(fullPath);
    const output2: IFile[] = [];
    for (const file of files) {
      const filePath = join(fullPath, file);
      try {
        const stats = await fs.promises.stat(filePath);
        if (stats.isDirectory()) {
          output2.push(new FileModel({ name: file, path: dir, repoId }));
          output2.push(
            ...(await this.listFiles(repoId, join(dir, file), opt))
          );
        } else if (stats.isFile()) {
          if (opt.onEntry) {
            opt.onEntry({
              path: join(dir, file),
              size: stats.size,
            });
          }
          // Don't synthesise a sha here. The previous value (stats.ino)
          // wasn't a content hash — just an inode number — and any code
          // that compared it to an upstream Git blob sha would silently
          // disagree. Leave it undefined so callers either look up the
          // real sha from FileModel/GitHub or skip sha-keyed paths.
          output2.push(
            new FileModel({
              name: file,
              path: dir,
              repoId: repoId,
              size: stats.size,
            })
          );
        }
      } catch {
        // ignore stat errors for individual files
      }
    }
    return output2;
  }

  /** @override */
  async extractZip(repoId: string, p: string, data: Readable): Promise<void> {
    this.assertSafePath(p);
    const pipe = promisify(pipeline);
    const fullPath = join(config.FOLDER, this.repoPath(repoId), p);
    const extractor = Extract({
      path: fullPath,
      decodeString: (buf) => {
        const name = buf.toString();
        // Strip the top-level directory GitHub wraps every entry in, then
        // drop any "../" / absolute segments so a malicious entry name
        // cannot escape the extraction root (zip-slip, CWE-24).
        const stripped = name.substr(name.indexOf("/") + 1);
        const newName = this.sanitizeZipEntryName(stripped);
        if (newName == "") {
          return "___IGNORE___";
        }
        return newName;
      },
    });
    await pipe(data, extractor);
    await this.rm(repoId, join(p, "___IGNORE___"));
  }

  /** @override */
  async archive(
    repoId: string,
    dir: string,
    opt?: {
      format?: "zip" | "tar";
      fileTransformer?: (path: string) => Transform;
    }
  ) {
    this.assertSafePath(dir);
    const archive = archiver(opt?.format || "zip", {});
    let active: Readable | undefined;
    let stopped = false;
    archive.once("close", () => { stopped = true; active?.destroy(); });
    void (async () => {
      for (const file of await this.listFiles(repoId, dir)) {
        if (stopped || file.size == null) continue;
        const filename = join(file.path, file.name);
        const source = await this.read(repoId, filename);
        if (stopped) { source.destroy(); return; }
        active = source;
        let output = source;
        if (opt?.fileTransformer) {
          const transformer = opt.fileTransformer(filename);
          source.on("error", error => transformer.destroy(error));
          transformer.once("close", () => source.destroy());
          output = source.pipe(transformer);
          active = output;
        }
        await appendArchiveEntry(archive, output, zipEntryOptions(filename.slice(dir ? dir.length + 1 : 0), filename));
      }
      if (!stopped) await archive.finalize();
    })().catch(error => archive.destroy(error));
    return archive;
  }
}