import { recordUpstreamFailure, recordResponseFailure } from "./request-evidence"; import got from "got"; import config from "../config"; import { MAX_SUBMODULE_ENTRIES, SubmoduleMount, safeRepositoryPath, validateMounts } from "./source/github-submodules"; import { Readable, Transform, Writable } from "stream"; import { Parse } from "unzip-stream"; import { boundedZip } from "./bounded-zip"; import GitHubDownload from "./source/GitHubDownload"; import { classifyGitHubMissError } from "./source/GitHubBase"; import AnonymousError from "./AnonymousError"; import { AnonymizeTransformer, anonymizePathCompiled, compileTerms, } from "./anonymize-utils"; import { createLogger, serializeError } from "./logger"; import { startStage } from "./request-monitoring"; import { zipEntryOptions } from "./archive-entry"; const logger = createLogger("zip-stream"); export interface StreamAnonymizedZipOptions { repoId: string; submodules?: SubmoduleMount[]; organization: string; repoName: string; commit: string; getToken: () => string | Promise; anonymizerOptions: ConstructorParameters[0]; /** * Per-repo content gates. Matches Repository.options — `image: true` * includes images, `pdf: true` includes PDFs. The single-file `/file/...` * endpoint enforces these via AnonymizedFile.isFileSupported; without * the same gate here, the ZIP shipped a superset of what the per-file * API exposes, which is privacy-relevant when a maintainer toggles * image=false to suppress identifying screenshots. */ contentOptions?: { image?: boolean; pdf?: boolean; }; } const IMAGE_EXTENSIONS = new Set([ "png", "jpg", "jpeg", "gif", "svg", "ico", "bmp", "tiff", "tif", "webp", "avif", "heif", "heic", ]); function isEntryAllowed( filename: string, contentOptions?: { image?: boolean; pdf?: boolean } ): boolean { if (!contentOptions) return true; const ext = filename.split(".").pop()?.toLowerCase() ?? ""; if (contentOptions.pdf === false && ext === "pdf") return false; if (contentOptions.image === false && IMAGE_EXTENSIONS.has(ext)) return false; return true; } /** * Stream the GitHub source zip for a repository, anonymize each entry on the * fly, and pipe the resulting archive into the provided writable response. * * Large text entries spool privately before worker anonymization. The parser * admits the next entry only after the archive output drains to the response. */ export async function streamAnonymizedZip( opt: StreamAnonymizedZipOptions, res: NodeJS.WritableStream & { on(event: string, listener: (...args: unknown[]) => void): unknown; } ): Promise { const submodules = validateMounts(opt.submodules); let response; try { const token = await opt.getToken(); if (!token) { // The API already checked public access. Codeload serves public archives // directly, avoiding the streamer's shared unauthenticated REST quota. response = { url: `https://codeload.github.com/${encodeURIComponent(opt.organization)}/${encodeURIComponent(opt.repoName)}/zip/${encodeURIComponent(opt.commit || "HEAD")}` }; } else { const source = new GitHubDownload({ repoId: opt.repoId, organization: opt.organization, repoName: opt.repoName, commit: opt.commit, getToken: () => token, }); response = await source.getZipUrl(); } } catch (error) { const code = await classifyGitHubMissError(error, { organization: opt.organization, repoName: opt.repoName, repoId: opt.repoId, commit: opt.commit, getToken: opt.getToken, }); const status = (error as { status?: number }).status; throw new AnonymousError(code, { httpStatus: status && status >= 500 ? 502 : status || 502, cause: error as Error, object: { repoId: opt.repoId, fullName: `${opt.organization}/${opt.repoName}`, }, }); } let downloadStream: Readable; const archive = boundedZip(); let parser: Transform; let childEntries = 0; const activeStreams = new Set(); const compiledTerms = compileTerms(opt.anonymizerOptions.terms || []); let stopped = false; let entryWaiting = false; let cleanupDrain: (() => void) | undefined; const resumeParser = () => { if (!stopped && entryWaiting && archive.readableLength === 0 && output.writableLength === 0) { entryWaiting = false; parser.resume(); } }; // Middleware can omit write callbacks. Honor the writable backpressure // contract instead so queued output stays bounded. const output = new Writable({ write(chunk, encoding, callback) { const written = () => { cleanupDrain?.(); cleanupDrain = undefined; callback(); setImmediate(resumeParser); }; if (res.write(chunk, encoding)) written(); else { // Compression forwards drain subscriptions to its underlying stream. // Remove the listener from the emitter returned by on(), rather than // accumulating wrappers on that stream through response.once(). const emitter = res.on("drain", written) as NodeJS.EventEmitter; cleanupDrain = () => emitter.removeListener("drain", written); } }, final(callback) { res.end(); callback(); }, }); output.on("error", error => fail(error)); archive.on("data", () => { if (entryWaiting) setImmediate(resumeParser); }); const stop = () => { if (stopped) return; stopped = true; cleanupDrain?.(); cleanupDrain = undefined; downloadStream?.destroy(); parser?.destroy(); for (const stream of activeStreams) stream.destroy(); activeStreams.clear(); output.destroy(); archive.abort(); }; const fail = (error: Error) => { if (stopped) return; recordResponseFailure(res, "unknown", "local_abort"); logger.error("zip stream failed", serializeError(error)); stop(); const destroyable = res as unknown as { destroy?: (err?: Error) => void; end?: () => void; }; if (typeof destroyable.destroy === "function") { destroyable.destroy(error); } else if (typeof destroyable.end === "function") { destroyable.end(); } }; res.on("error", (error) => fail(error as Error)); res.on("close", stop); archive.on("error", fail); archive.pipe(output); const startArchive = (url: string, prefix: string, next: number) => { if (stopped) return; const upstreamDone = startStage("upstream"); const input = got.stream(url); downloadStream = input; input.once("end", () => upstreamDone()); input.once("error", () => upstreamDone(true)); input.once("close", () => upstreamDone(!input.readableEnded)); parser = Parse() as Transform; downloadStream .on("error", error => { recordUpstreamFailure(res, error); fail(error); }) .pipe(parser) .on("entry", (entry: Readable & { type: string; path: string; autodrain: () => void }) => { if (stopped) { entry.destroy(); return; } entry.on("error", fail); activeStreams.add(entry); entry.once("close", () => activeStreams.delete(entry)); if (entry.type === "File") { try { const relativePath = entry.path.substring(entry.path.indexOf("/") + 1); if (!safeRepositoryPath(relativePath)) throw new AnonymousError("invalid_file_path", { httpStatus: 502 }); if (relativePath.split("/").pop() === ".gitmodules") { entry.autodrain(); return; } if (prefix && ++childEntries > MAX_SUBMODULE_ENTRIES) throw new AnonymousError("file_too_big", { httpStatus: 413 }); const mountedPath = prefix ? `${prefix}/${relativePath}` : relativePath; const fileName = anonymizePathCompiled(mountedPath, compiledTerms); if (!isEntryAllowed(entry.path, opt.contentOptions)) { entry.autodrain(); return; } // Pass filePath via the constructor — AnonymizeTransformer reads it // there to decide whether the entry is text (and therefore should be // anonymized) vs binary (passthrough). Assigning afterwards leaves // isText=false for every file, so the zip ships unanonymized. parser.pause(); const onArchived = () => { entryWaiting = true; setImmediate(resumeParser); }; archive.once("entry", onArchived); const anonymizer = new AnonymizeTransformer({ ...opt.anonymizerOptions, filePath: mountedPath, }); anonymizer.on("error", fail); activeStreams.add(anonymizer); anonymizer.once("close", () => activeStreams.delete(anonymizer)); let source: Readable = entry; if (prefix) { let bytes = 0; const limited = new Transform({ transform(chunk, encoding, callback) { bytes += chunk.length; callback(bytes > config.MAX_FILE_SIZE ? new AnonymousError("file_too_big", { httpStatus: 413 }) : null, chunk); } }); limited.on("error", fail); activeStreams.add(limited); limited.once("close", () => activeStreams.delete(limited)); source = entry.pipe(limited); } const st = source.pipe(anonymizer); archive.append(st, zipEntryOptions(fileName, mountedPath)); } catch (error) { fail(error as Error); } } else { entry.autodrain(); } }) .on("error", fail) .on("end", () => { if (stopped) return; const mount = submodules[next]; if (mount) { // Public child archives never receive parent credentials. startArchive(`https://codeload.github.com/${encodeURIComponent(mount.organization)}/${encodeURIComponent(mount.repoName)}/zip/${mount.commit}`, mount.path, next + 1); } else void archive.finalize().catch(fail); }); }; startArchive(response.url, "", 0); }