Files
anonymous_github/src/core/zipStream.ts
T

271 lines
9.9 KiB
TypeScript

import { recordUpstreamFailure, recordResponseFailure } from "./request-evidence";
import got from "got";
import config from "../config";
import { MAX_SUBMODULE_ENTRIES, SubmoduleMount, safeRepositoryPath, validateMounts } from "./source/github-submodules";
import { Readable, Transform, Writable } from "stream";
import { Parse } from "unzip-stream";
import { boundedZip } from "./bounded-zip";
import GitHubDownload from "./source/GitHubDownload";
import { classifyGitHubMissError } from "./source/GitHubBase";
import AnonymousError from "./AnonymousError";
import {
AnonymizeTransformer,
anonymizePathCompiled,
compileTerms,
} from "./anonymize-utils";
import { createLogger, serializeError } from "./logger";
import { startStage } from "./request-monitoring";
import { zipEntryOptions } from "./archive-entry";
const logger = createLogger("zip-stream");
export interface StreamAnonymizedZipOptions {
repoId: string;
submodules?: SubmoduleMount[];
organization: string;
repoName: string;
commit: string;
getToken: () => string | Promise<string>;
anonymizerOptions: ConstructorParameters<typeof AnonymizeTransformer>[0];
/**
* Per-repo content gates. Matches Repository.options — `image: true`
* includes images, `pdf: true` includes PDFs. The single-file `/file/...`
* endpoint enforces these via AnonymizedFile.isFileSupported; without
* the same gate here, the ZIP shipped a superset of what the per-file
* API exposes, which is privacy-relevant when a maintainer toggles
* image=false to suppress identifying screenshots.
*/
contentOptions?: {
image?: boolean;
pdf?: boolean;
};
}
const IMAGE_EXTENSIONS = new Set([
"png",
"jpg",
"jpeg",
"gif",
"svg",
"ico",
"bmp",
"tiff",
"tif",
"webp",
"avif",
"heif",
"heic",
]);
function isEntryAllowed(
filename: string,
contentOptions?: { image?: boolean; pdf?: boolean }
): boolean {
if (!contentOptions) return true;
const ext = filename.split(".").pop()?.toLowerCase() ?? "";
if (contentOptions.pdf === false && ext === "pdf") return false;
if (contentOptions.image === false && IMAGE_EXTENSIONS.has(ext)) return false;
return true;
}
/**
* Stream the GitHub source zip for a repository, anonymize each entry on the
* fly, and pipe the resulting archive into the provided writable response.
*
* Large text entries spool privately before worker anonymization. The parser
* admits the next entry only after the archive output drains to the response.
*/
export async function streamAnonymizedZip(
opt: StreamAnonymizedZipOptions,
res: NodeJS.WritableStream & {
on(event: string, listener: (...args: unknown[]) => void): unknown;
}
): Promise<void> {
const submodules = validateMounts(opt.submodules);
let response;
try {
const token = await opt.getToken();
if (!token) {
// The API already checked public access. Codeload serves public archives
// directly, avoiding the streamer's shared unauthenticated REST quota.
response = { url: `https://codeload.github.com/${encodeURIComponent(opt.organization)}/${encodeURIComponent(opt.repoName)}/zip/${encodeURIComponent(opt.commit || "HEAD")}` };
} else {
const source = new GitHubDownload({
repoId: opt.repoId,
organization: opt.organization,
repoName: opt.repoName,
commit: opt.commit,
getToken: () => token,
});
response = await source.getZipUrl();
}
} catch (error) {
const code = await classifyGitHubMissError(error, {
organization: opt.organization,
repoName: opt.repoName,
repoId: opt.repoId,
commit: opt.commit,
getToken: opt.getToken,
});
const status = (error as { status?: number }).status;
throw new AnonymousError(code, {
httpStatus: status && status >= 500 ? 502 : status || 502,
cause: error as Error,
object: {
repoId: opt.repoId,
fullName: `${opt.organization}/${opt.repoName}`,
},
});
}
let downloadStream: Readable;
const archive = boundedZip();
let parser: Transform;
let childEntries = 0;
const activeStreams = new Set<Readable>();
const compiledTerms = compileTerms(opt.anonymizerOptions.terms || []);
let stopped = false;
let entryWaiting = false;
let cleanupDrain: (() => void) | undefined;
const resumeParser = () => {
if (!stopped && entryWaiting && archive.readableLength === 0 && output.writableLength === 0) {
entryWaiting = false;
parser.resume();
}
};
// Middleware can omit write callbacks. Honor the writable backpressure
// contract instead so queued output stays bounded.
const output = new Writable({
write(chunk, encoding, callback) {
const written = () => {
cleanupDrain?.();
cleanupDrain = undefined;
callback();
setImmediate(resumeParser);
};
if (res.write(chunk, encoding)) written();
else {
// Compression forwards drain subscriptions to its underlying stream.
// Remove the listener from the emitter returned by on(), rather than
// accumulating wrappers on that stream through response.once().
const emitter = res.on("drain", written) as NodeJS.EventEmitter;
cleanupDrain = () => emitter.removeListener("drain", written);
}
},
final(callback) { res.end(); callback(); },
});
output.on("error", error => fail(error));
archive.on("data", () => { if (entryWaiting) setImmediate(resumeParser); });
const stop = () => {
if (stopped) return;
stopped = true;
cleanupDrain?.();
cleanupDrain = undefined;
downloadStream?.destroy();
parser?.destroy();
for (const stream of activeStreams) stream.destroy();
activeStreams.clear();
output.destroy();
archive.abort();
};
const fail = (error: Error) => {
if (stopped) return;
recordResponseFailure(res, "unknown", "local_abort");
logger.error("zip stream failed", serializeError(error));
stop();
const destroyable = res as unknown as {
destroy?: (err?: Error) => void;
end?: () => void;
};
if (typeof destroyable.destroy === "function") {
destroyable.destroy(error);
} else if (typeof destroyable.end === "function") {
destroyable.end();
}
};
res.on("error", (error) => fail(error as Error));
res.on("close", stop);
archive.on("error", fail);
archive.pipe(output);
const startArchive = (url: string, prefix: string, next: number) => {
if (stopped) return;
const upstreamDone = startStage("upstream");
const input = got.stream(url);
downloadStream = input;
input.once("end", () => upstreamDone());
input.once("error", () => upstreamDone(true));
input.once("close", () => upstreamDone(!input.readableEnded));
parser = Parse() as Transform;
downloadStream
.on("error", error => { recordUpstreamFailure(res, error); fail(error); })
.pipe(parser)
.on("entry", (entry: Readable & { type: string; path: string; autodrain: () => void }) => {
if (stopped) {
entry.destroy();
return;
}
entry.on("error", fail);
activeStreams.add(entry);
entry.once("close", () => activeStreams.delete(entry));
if (entry.type === "File") {
try {
const relativePath = entry.path.substring(entry.path.indexOf("/") + 1);
if (!safeRepositoryPath(relativePath)) throw new AnonymousError("invalid_file_path", { httpStatus: 502 });
if (relativePath.split("/").pop() === ".gitmodules") { entry.autodrain(); return; }
if (prefix && ++childEntries > MAX_SUBMODULE_ENTRIES) throw new AnonymousError("file_too_big", { httpStatus: 413 });
const mountedPath = prefix ? `${prefix}/${relativePath}` : relativePath;
const fileName = anonymizePathCompiled(mountedPath, compiledTerms);
if (!isEntryAllowed(entry.path, opt.contentOptions)) {
entry.autodrain();
return;
}
// Pass filePath via the constructor — AnonymizeTransformer reads it
// there to decide whether the entry is text (and therefore should be
// anonymized) vs binary (passthrough). Assigning afterwards leaves
// isText=false for every file, so the zip ships unanonymized.
parser.pause();
const onArchived = () => { entryWaiting = true; setImmediate(resumeParser); };
archive.once("entry", onArchived);
const anonymizer = new AnonymizeTransformer({
...opt.anonymizerOptions,
filePath: mountedPath,
});
anonymizer.on("error", fail);
activeStreams.add(anonymizer);
anonymizer.once("close", () => activeStreams.delete(anonymizer));
let source: Readable = entry;
if (prefix) {
let bytes = 0;
const limited = new Transform({ transform(chunk, encoding, callback) {
bytes += chunk.length;
callback(bytes > config.MAX_FILE_SIZE ? new AnonymousError("file_too_big", { httpStatus: 413 }) : null, chunk);
} });
limited.on("error", fail);
activeStreams.add(limited);
limited.once("close", () => activeStreams.delete(limited));
source = entry.pipe(limited);
}
const st = source.pipe(anonymizer);
archive.append(st, zipEntryOptions(fileName, mountedPath));
} catch (error) {
fail(error as Error);
}
} else {
entry.autodrain();
}
})
.on("error", fail)
.on("end", () => {
if (stopped) return;
const mount = submodules[next];
if (mount) {
// Public child archives never receive parent credentials.
startArchive(`https://codeload.github.com/${encodeURIComponent(mount.organization)}/${encodeURIComponent(mount.repoName)}/zip/${mount.commit}`, mount.path, next + 1);
} else void archive.finalize().catch(fail);
});
};
startArchive(response.url, "", 0);
}