mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-18 01:54:23 +01:00
271 lines
9.9 KiB
TypeScript
271 lines
9.9 KiB
TypeScript
import { recordUpstreamFailure, recordResponseFailure } from "./request-evidence";
|
|
import got from "got";
|
|
import config from "../config";
|
|
import { MAX_SUBMODULE_ENTRIES, SubmoduleMount, safeRepositoryPath, validateMounts } from "./source/github-submodules";
|
|
import { Readable, Transform, Writable } from "stream";
|
|
import { Parse } from "unzip-stream";
|
|
import { boundedZip } from "./bounded-zip";
|
|
|
|
import GitHubDownload from "./source/GitHubDownload";
|
|
import { classifyGitHubMissError } from "./source/GitHubBase";
|
|
import AnonymousError from "./AnonymousError";
|
|
import {
|
|
AnonymizeTransformer,
|
|
anonymizePathCompiled,
|
|
compileTerms,
|
|
} from "./anonymize-utils";
|
|
import { createLogger, serializeError } from "./logger";
|
|
import { startStage } from "./request-monitoring";
|
|
import { zipEntryOptions } from "./archive-entry";
|
|
|
|
const logger = createLogger("zip-stream");
|
|
|
|
export interface StreamAnonymizedZipOptions {
|
|
repoId: string;
|
|
submodules?: SubmoduleMount[];
|
|
organization: string;
|
|
repoName: string;
|
|
commit: string;
|
|
getToken: () => string | Promise<string>;
|
|
anonymizerOptions: ConstructorParameters<typeof AnonymizeTransformer>[0];
|
|
/**
|
|
* Per-repo content gates. Matches Repository.options — `image: true`
|
|
* includes images, `pdf: true` includes PDFs. The single-file `/file/...`
|
|
* endpoint enforces these via AnonymizedFile.isFileSupported; without
|
|
* the same gate here, the ZIP shipped a superset of what the per-file
|
|
* API exposes, which is privacy-relevant when a maintainer toggles
|
|
* image=false to suppress identifying screenshots.
|
|
*/
|
|
contentOptions?: {
|
|
image?: boolean;
|
|
pdf?: boolean;
|
|
};
|
|
}
|
|
|
|
const IMAGE_EXTENSIONS = new Set([
|
|
"png",
|
|
"jpg",
|
|
"jpeg",
|
|
"gif",
|
|
"svg",
|
|
"ico",
|
|
"bmp",
|
|
"tiff",
|
|
"tif",
|
|
"webp",
|
|
"avif",
|
|
"heif",
|
|
"heic",
|
|
]);
|
|
|
|
function isEntryAllowed(
|
|
filename: string,
|
|
contentOptions?: { image?: boolean; pdf?: boolean }
|
|
): boolean {
|
|
if (!contentOptions) return true;
|
|
const ext = filename.split(".").pop()?.toLowerCase() ?? "";
|
|
if (contentOptions.pdf === false && ext === "pdf") return false;
|
|
if (contentOptions.image === false && IMAGE_EXTENSIONS.has(ext)) return false;
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Stream the GitHub source zip for a repository, anonymize each entry on the
|
|
* fly, and pipe the resulting archive into the provided writable response.
|
|
*
|
|
* Large text entries spool privately before worker anonymization. The parser
|
|
* admits the next entry only after the archive output drains to the response.
|
|
*/
|
|
export async function streamAnonymizedZip(
|
|
opt: StreamAnonymizedZipOptions,
|
|
res: NodeJS.WritableStream & {
|
|
on(event: string, listener: (...args: unknown[]) => void): unknown;
|
|
}
|
|
): Promise<void> {
|
|
const submodules = validateMounts(opt.submodules);
|
|
let response;
|
|
try {
|
|
const token = await opt.getToken();
|
|
if (!token) {
|
|
// The API already checked public access. Codeload serves public archives
|
|
// directly, avoiding the streamer's shared unauthenticated REST quota.
|
|
response = { url: `https://codeload.github.com/${encodeURIComponent(opt.organization)}/${encodeURIComponent(opt.repoName)}/zip/${encodeURIComponent(opt.commit || "HEAD")}` };
|
|
} else {
|
|
const source = new GitHubDownload({
|
|
repoId: opt.repoId,
|
|
organization: opt.organization,
|
|
repoName: opt.repoName,
|
|
commit: opt.commit,
|
|
getToken: () => token,
|
|
});
|
|
response = await source.getZipUrl();
|
|
}
|
|
} catch (error) {
|
|
const code = await classifyGitHubMissError(error, {
|
|
organization: opt.organization,
|
|
repoName: opt.repoName,
|
|
repoId: opt.repoId,
|
|
commit: opt.commit,
|
|
getToken: opt.getToken,
|
|
});
|
|
const status = (error as { status?: number }).status;
|
|
throw new AnonymousError(code, {
|
|
httpStatus: status && status >= 500 ? 502 : status || 502,
|
|
cause: error as Error,
|
|
object: {
|
|
repoId: opt.repoId,
|
|
fullName: `${opt.organization}/${opt.repoName}`,
|
|
},
|
|
});
|
|
}
|
|
let downloadStream: Readable;
|
|
const archive = boundedZip();
|
|
let parser: Transform;
|
|
let childEntries = 0;
|
|
const activeStreams = new Set<Readable>();
|
|
const compiledTerms = compileTerms(opt.anonymizerOptions.terms || []);
|
|
let stopped = false;
|
|
let entryWaiting = false;
|
|
let cleanupDrain: (() => void) | undefined;
|
|
const resumeParser = () => {
|
|
if (!stopped && entryWaiting && archive.readableLength === 0 && output.writableLength === 0) {
|
|
entryWaiting = false;
|
|
parser.resume();
|
|
}
|
|
};
|
|
// Middleware can omit write callbacks. Honor the writable backpressure
|
|
// contract instead so queued output stays bounded.
|
|
const output = new Writable({
|
|
write(chunk, encoding, callback) {
|
|
const written = () => {
|
|
cleanupDrain?.();
|
|
cleanupDrain = undefined;
|
|
callback();
|
|
setImmediate(resumeParser);
|
|
};
|
|
if (res.write(chunk, encoding)) written();
|
|
else {
|
|
// Compression forwards drain subscriptions to its underlying stream.
|
|
// Remove the listener from the emitter returned by on(), rather than
|
|
// accumulating wrappers on that stream through response.once().
|
|
const emitter = res.on("drain", written) as NodeJS.EventEmitter;
|
|
cleanupDrain = () => emitter.removeListener("drain", written);
|
|
}
|
|
},
|
|
final(callback) { res.end(); callback(); },
|
|
});
|
|
output.on("error", error => fail(error));
|
|
archive.on("data", () => { if (entryWaiting) setImmediate(resumeParser); });
|
|
const stop = () => {
|
|
if (stopped) return;
|
|
stopped = true;
|
|
cleanupDrain?.();
|
|
cleanupDrain = undefined;
|
|
downloadStream?.destroy();
|
|
parser?.destroy();
|
|
for (const stream of activeStreams) stream.destroy();
|
|
activeStreams.clear();
|
|
output.destroy();
|
|
archive.abort();
|
|
};
|
|
const fail = (error: Error) => {
|
|
if (stopped) return;
|
|
recordResponseFailure(res, "unknown", "local_abort");
|
|
logger.error("zip stream failed", serializeError(error));
|
|
stop();
|
|
const destroyable = res as unknown as {
|
|
destroy?: (err?: Error) => void;
|
|
end?: () => void;
|
|
};
|
|
if (typeof destroyable.destroy === "function") {
|
|
destroyable.destroy(error);
|
|
} else if (typeof destroyable.end === "function") {
|
|
destroyable.end();
|
|
}
|
|
};
|
|
|
|
res.on("error", (error) => fail(error as Error));
|
|
res.on("close", stop);
|
|
archive.on("error", fail);
|
|
archive.pipe(output);
|
|
|
|
const startArchive = (url: string, prefix: string, next: number) => {
|
|
if (stopped) return;
|
|
const upstreamDone = startStage("upstream");
|
|
const input = got.stream(url);
|
|
downloadStream = input;
|
|
input.once("end", () => upstreamDone());
|
|
input.once("error", () => upstreamDone(true));
|
|
input.once("close", () => upstreamDone(!input.readableEnded));
|
|
parser = Parse() as Transform;
|
|
downloadStream
|
|
.on("error", error => { recordUpstreamFailure(res, error); fail(error); })
|
|
.pipe(parser)
|
|
.on("entry", (entry: Readable & { type: string; path: string; autodrain: () => void }) => {
|
|
if (stopped) {
|
|
entry.destroy();
|
|
return;
|
|
}
|
|
entry.on("error", fail);
|
|
activeStreams.add(entry);
|
|
entry.once("close", () => activeStreams.delete(entry));
|
|
if (entry.type === "File") {
|
|
try {
|
|
const relativePath = entry.path.substring(entry.path.indexOf("/") + 1);
|
|
if (!safeRepositoryPath(relativePath)) throw new AnonymousError("invalid_file_path", { httpStatus: 502 });
|
|
if (relativePath.split("/").pop() === ".gitmodules") { entry.autodrain(); return; }
|
|
if (prefix && ++childEntries > MAX_SUBMODULE_ENTRIES) throw new AnonymousError("file_too_big", { httpStatus: 413 });
|
|
const mountedPath = prefix ? `${prefix}/${relativePath}` : relativePath;
|
|
const fileName = anonymizePathCompiled(mountedPath, compiledTerms);
|
|
if (!isEntryAllowed(entry.path, opt.contentOptions)) {
|
|
entry.autodrain();
|
|
return;
|
|
}
|
|
// Pass filePath via the constructor — AnonymizeTransformer reads it
|
|
// there to decide whether the entry is text (and therefore should be
|
|
// anonymized) vs binary (passthrough). Assigning afterwards leaves
|
|
// isText=false for every file, so the zip ships unanonymized.
|
|
parser.pause();
|
|
const onArchived = () => { entryWaiting = true; setImmediate(resumeParser); };
|
|
archive.once("entry", onArchived);
|
|
const anonymizer = new AnonymizeTransformer({
|
|
...opt.anonymizerOptions,
|
|
filePath: mountedPath,
|
|
});
|
|
anonymizer.on("error", fail);
|
|
activeStreams.add(anonymizer);
|
|
anonymizer.once("close", () => activeStreams.delete(anonymizer));
|
|
let source: Readable = entry;
|
|
if (prefix) {
|
|
let bytes = 0;
|
|
const limited = new Transform({ transform(chunk, encoding, callback) {
|
|
bytes += chunk.length;
|
|
callback(bytes > config.MAX_FILE_SIZE ? new AnonymousError("file_too_big", { httpStatus: 413 }) : null, chunk);
|
|
} });
|
|
limited.on("error", fail);
|
|
activeStreams.add(limited);
|
|
limited.once("close", () => activeStreams.delete(limited));
|
|
source = entry.pipe(limited);
|
|
}
|
|
const st = source.pipe(anonymizer);
|
|
archive.append(st, zipEntryOptions(fileName, mountedPath));
|
|
} catch (error) {
|
|
fail(error as Error);
|
|
}
|
|
} else {
|
|
entry.autodrain();
|
|
}
|
|
})
|
|
.on("error", fail)
|
|
.on("end", () => {
|
|
if (stopped) return;
|
|
const mount = submodules[next];
|
|
if (mount) {
|
|
// Public child archives never receive parent credentials.
|
|
startArchive(`https://codeload.github.com/${encodeURIComponent(mount.organization)}/${encodeURIComponent(mount.repoName)}/zip/${mount.commit}`, mount.path, next + 1);
|
|
} else void archive.finalize().catch(fail);
|
|
});
|
|
};
|
|
startArchive(response.url, "", 0);
|
|
}
|