From d0b217a448695445850bbbf88ee13ecdc526c5be Mon Sep 17 00:00:00 2001 From: Thomas Durieux <5577568+tdurieux@users.noreply.github.com> Date: Mon, 2 Nov 2026 09:00:02 +0100 Subject: [PATCH] feat: include public GitHub submodules in shared repositories (#914) * feat: include public GitHub submodules in shared repositories * fix: preserve submodule access across token and document contexts --- README.md | 14 ++ public/i18n/locale-en.json | 3 +- src/core/AnonymizedFile.ts | 21 +- src/core/GitHubUtils.ts | 2 +- src/core/Repository.ts | 19 +- src/core/model/anonymized-path.ts | 2 + src/core/model/files/files.schema.ts | 4 + src/core/model/files/files.types.ts | 10 + src/core/source/GitHubStream.ts | 44 +++- src/core/source/github-submodules.ts | 99 ++++++++ src/core/zipStream.ts | 143 +++++++----- src/server/routes/repository-public.ts | 13 +- src/streamer/route.ts | 3 +- test/github-submodules.test.js | 308 +++++++++++++++++++++++++ test/owner-notifications.test.js | 4 +- test/performance-integration.test.js | 13 ++ test/zip-proxy-errors.test.js | 2 +- 17 files changed, 617 insertions(+), 87 deletions(-) create mode 100644 src/core/source/github-submodules.ts create mode 100644 test/github-submodules.test.js diff --git a/README.md b/README.md index 0aeece3..0f9937b 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,20 @@ A free public instance is available at **[anonymous.4open.science](https://anony Just open **[anonymous.4open.science](https://anonymous.4open.science/)**, paste your repository URL, add the terms to hide, and share the generated link. +### Git submodules + +Direct public GitHub submodules are included in the file browser and ZIP downloads +at the commit recorded by the parent repository. Their paths and contents use the +parent's anonymization settings. `.gitmodules` files are omitted from shared GitHub +content because they contain source repository URLs. + +The service expands up to 20 direct submodules and 10,000 combined child tree entries. +Private, non-GitHub, nested, missing, oversized, and truncated submodules remain empty +folders with a warning. Child files are fetched anonymously on demand; child tree +lookups use GitHub's unauthenticated API quota. ZIP downloads fetch public child archives. +Existing cached repositories with `.gitmodules` rebuild their file trees on the next +listing or ZIP download. No manual database migration is required. + ### CLI Anonymize a repository locally and produce an anonymized zip: diff --git a/public/i18n/locale-en.json b/public/i18n/locale-en.json index dee6ad3..c0c1113 100644 --- a/public/i18n/locale-en.json +++ b/public/i18n/locale-en.json @@ -72,6 +72,7 @@ "invalid_request": "The request is missing required fields or is malformed.", "invalid_project_name": "Use a project name of up to 100 characters without line breaks or control characters.", "no_file_selected": "Please select a file.", + "submodule_not_available": "This public submodule file is no longer available at its recorded commit.", "file_not_found": "The requested file is not found.", "file_not_accessible": "The requested file is not accessible.", "file_not_supported": "The file type is not supported. Anonymous GitHub cannot handle it.", @@ -164,6 +165,6 @@ "page_branch_mismatch": "GitHub Pages on this repository is served from the '{{pageBranch}}' branch, but you selected '{{selectedBranch}}'. Switch the branch above to '{{pageBranch}}' to anonymize the Pages site.", "folder_truncated": "This folder has more than 10,000 entries; only a partial listing is shown.", "repo_truncated": "Some folders in this repository have too many files to be fully listed. Affected folders are marked with a warning icon.", - "submodules_not_included": "This repository uses git submodules. Submodule contents are not included in the anonymized repository and appear as empty folders." + "submodules_not_included": "Some submodule contents could not be included. Only direct public GitHub submodules within the size limits are supported. Private, external, nested, or unavailable submodules appear as empty folders." } } diff --git a/src/core/AnonymizedFile.ts b/src/core/AnonymizedFile.ts index 858a4bb..020d521 100644 --- a/src/core/AnonymizedFile.ts +++ b/src/core/AnonymizedFile.ts @@ -153,8 +153,18 @@ export default class AnonymizedFile { return this._file.size; } + private visibleFile(file: IFile): IFile { + if (this.repository.model?.source?.type !== "Zip" && file.name === ".gitmodules") { + throw new AnonymousError("file_not_found", { httpStatus: 404 }); + } + return file; + } + async getFileInfo(): Promise { - if (this._file) return this._file; + if (this.repository.model?.source?.type !== "Zip" && basename(this.anonymizedPath) === ".gitmodules") { + throw new AnonymousError("file_not_found", { httpStatus: 404 }); + } + if (this._file) return this.visibleFile(this._file); let fileDir = dirname(this.anonymizedPath); if (fileDir == ".") fileDir = ""; if (fileDir.endsWith("/")) fileDir = fileDir.slice(0, -1); @@ -180,7 +190,7 @@ export default class AnonymizedFile { if (exact) { if (exact.metadataPending) await this.repository.completeRecoveredFileMetadata(exact); this._file = exact; - return exact; + return this.visibleFile(exact); } const terms = this.repository.options.terms || []; @@ -213,7 +223,7 @@ export default class AnonymizedFile { const compiled = compileTerms(terms); if (typeof this.repository.findAnonymizedPath === "function") { const indexed = await this.repository.findAnonymizedPath(this.anonymizedPath); - if (indexed) { this._file = indexed; return indexed; } + if (indexed) { this._file = indexed; return this.visibleFile(indexed); } if (indexed === null) { if (anonymizePathCompiled(this.anonymizedPath, compiled) === this.anonymizedPath) { const recovered = await this.recoverTruncatedFile(fileDir); @@ -237,7 +247,7 @@ export default class AnonymizedFile { ) { if (candidate.metadataPending) await this.repository.completeRecoveredFileMetadata(candidate); this._file = candidate; - return candidate; + return this.visibleFile(candidate); } } @@ -312,6 +322,7 @@ export default class AnonymizedFile { if (!this._file) { this._file = await this.getFileInfo(); } + this.visibleFile(this._file); return join(this._file.path, this._file.name); } extension() { @@ -398,6 +409,7 @@ export default class AnonymizedFile { branch: this.repository.model.source.branch, repoId: this.repository.repoId, filePath: this.filePath, + upstream: (await this.getFileInfo()).upstream, sha: await this.sha(), size: await this.size(), anonymizerOptions: anonymizer.opt, @@ -446,6 +458,7 @@ export default class AnonymizedFile { json: { sha, size, + upstream: (await this.getFileInfo()).upstream, token: await githubTokenForStreamer(token, this.repository.model.source.repositoryName), repoFullName: this.repository.model.source.repositoryName, commit: this.repository.model.source.commit, diff --git a/src/core/GitHubUtils.ts b/src/core/GitHubUtils.ts index e0f562e..f644bd4 100644 --- a/src/core/GitHubUtils.ts +++ b/src/core/GitHubUtils.ts @@ -244,7 +244,7 @@ export function octokit(token: string) { const prefix = `/repos/${context.publicRepository.split("/").map(encodeURIComponent).join("/")}`.toLowerCase(); const path = url.pathname.toLowerCase(); const suffix = path.slice(prefix.length); - const readable = /^(?:\/?|\/branches(?:\/[^/]+)?|\/commits(?:\/[^/]+)?|\/readme|\/pages|\/zipball\/[^/]+|\/git\/(?:trees|blobs)\/[^/]+|\/pulls\/\d+|\/issues\/\d+\/comments)$/.test(suffix); + const readable = /^(?:\/?|\/branches(?:\/[^/]+)?|\/commits(?:\/[^/]+)?|\/contents(?:\/[^/]+)*|\/readme|\/pages|\/zipball\/[^/]+|\/git\/(?:trees|blobs)\/[^/]+|\/pulls\/\d+|\/issues\/\d+\/comments)$/.test(suffix); if (!readable || (options.method !== "GET" && !(options.method === "HEAD" && suffix.startsWith("/zipball/"))) || url.origin !== "https://api.github.com" || (path !== prefix && !path.startsWith(prefix + "/"))) { throw new AnonymousError("github_app_access_required", { httpStatus: 403 }); diff --git a/src/core/Repository.ts b/src/core/Repository.ts index a2efb18..2511c97 100644 --- a/src/core/Repository.ts +++ b/src/core/Repository.ts @@ -8,6 +8,7 @@ import { RepositoryStatus } from "./types"; import { Readable } from "stream"; import * as sha1 from "crypto-js/sha1"; import User from "./User"; +import { SubmoduleMount } from "./source/github-submodules"; import GitHubStream from "./source/GitHubStream"; import Zip from "./source/Zip"; import { @@ -43,14 +44,14 @@ async function buildPathIndex(repoId: string, key: string, terms: string[], gene const task = (async () => { const compiled = compileTerms(terms); const cursor = FileModel.find({ repoId, treeGeneration: generation || { $exists: false } }) - .select("name path sha size").sort({ path: 1, name: 1 }).lean().cursor(); + .select("name path sha size upstream").sort({ path: 1, name: 1 }).lean().cursor(); let batch: Parameters[0] = []; try { for await (const file of cursor) { const name = anonymizePathCompiled(file.name, compiled); const directory = anonymizePathCompiled(file.path, compiled); batch.push({ updateOne: { filter: { repoId, key, originalId: file._id }, update: { $set: { - repoId, key, originalId: file._id, name: file.name, path: file.path, sha: file.sha, size: file.size, + repoId, key, originalId: file._id, name: file.name, path: file.path, sha: file.sha, size: file.size, upstream: file.upstream, anonymousName: name, anonymousDirectory: directory, createdAt: new Date(), searchName: name.toLowerCase(), searchDirectory: directory.toLowerCase(), anonymousPath: anonymizePathCompiled(file.path ? `${file.path}/${file.name}` : file.name, compiled), @@ -202,6 +203,17 @@ export default class Repository { return await FileModel.find(query).exec(); } + async submoduleMounts(): Promise { + if (this.model.source.type === "Zip") return []; + await this.ensureFileTree(); + const mounts = await FileModel.find({ repoId: this.repoId, + treeGeneration: this.model.treeGeneration || { $exists: false }, submodule: "included", + }).select("name path upstream").lean().exec(); + return mounts.filter(file => file.upstream).map(file => ({ + ...file.upstream!, path: [file.path, file.name].filter(Boolean).join("/"), + })); + } + private async ensureFileTree(opt: { force?: boolean; progress?: (status: string) => void } = {}) { this.assertNotArchived(); let hasFile = this.hasEmptyTree() || await FileModel.exists({ repoId: this.repoId, treeGeneration: this.model.treeGeneration || { $exists: false } }).exec(); @@ -219,6 +231,9 @@ export default class Repository { hasFile = null; } } + if (hasFile && this.source instanceof GitHubStream && await FileModel.exists({ + repoId: this.repoId, treeGeneration: this.model.treeGeneration || { $exists: false }, name: ".gitmodules", + }).exec()) hasFile = null; if (!hasFile || opt.force) { const previousGeneration = this.model.treeGeneration; const previousPrefix = contentGenerationPrefix(`${previousGeneration || "legacy"}:${this.model.anonymizeDate?.toISOString() || ""}`); diff --git a/src/core/model/anonymized-path.ts b/src/core/model/anonymized-path.ts index d7ee41e..363f59a 100644 --- a/src/core/model/anonymized-path.ts +++ b/src/core/model/anonymized-path.ts @@ -1,7 +1,9 @@ +import { UpstreamFileSchema } from "./files/files.schema"; import { IFile } from "./files/files.types"; import { model, Schema } from "mongoose"; const schema = new Schema({ repoId: { type: String, required: true }, key: { type: String, required: true }, originalId: Schema.Types.ObjectId, name: { type: String, required: true }, path: { type: String, default: "", required: true }, sha: String, size: Number, + upstream: { type: UpstreamFileSchema, default: undefined }, anonymousName: String, anonymousDirectory: String, anonymousPath: String, searchName: String, searchDirectory: String, createdAt: { type: Date, default: Date.now } }); diff --git a/src/core/model/files/files.schema.ts b/src/core/model/files/files.schema.ts index 7838320..bb97e56 100644 --- a/src/core/model/files/files.schema.ts +++ b/src/core/model/files/files.schema.ts @@ -1,5 +1,7 @@ import { Schema } from "mongoose"; +export const UpstreamFileSchema = new Schema({ organization: String, repoName: String, commit: String, path: String }, { _id: false }); + const FileSchema = new Schema({ name: { type: String, index: true }, path: { type: String, index: true }, @@ -7,6 +9,8 @@ const FileSchema = new Schema({ treeGeneration: String, treeStaged: Boolean, metadataPending: Boolean, + upstream: { type: UpstreamFileSchema, default: undefined }, + submodule: { type: String, enum: ["included", "unsupported", "unavailable", "limit", "nested"] }, sha: { type: String, }, diff --git a/src/core/model/files/files.types.ts b/src/core/model/files/files.types.ts index f0fd8be..e6db347 100644 --- a/src/core/model/files/files.types.ts +++ b/src/core/model/files/files.types.ts @@ -1,6 +1,16 @@ import { Document, Model } from "mongoose"; +export interface UpstreamFile { + organization: string; + repoName: string; + commit: string; + path: string; +} + export interface IFile { + /** Internal source metadata, never returned by the public tree API. */ + upstream?: UpstreamFile; + submodule?: "included" | "unsupported" | "unavailable" | "limit" | "nested"; name: string; path: string; repoId: string; diff --git a/src/core/source/GitHubStream.ts b/src/core/source/GitHubStream.ts index 564b7aa..122e8c4 100644 --- a/src/core/source/GitHubStream.ts +++ b/src/core/source/GitHubStream.ts @@ -18,7 +18,8 @@ import * as stream from "stream"; import AnonymousError from "../AnonymousError"; import { octokit, waitForTokenGate, RateLimitDelayError } from "../GitHubUtils"; import FileModel from "../model/files/files.model"; -import { IFile } from "../model/files/files.types"; +import { expandSubmodules, validUpstream } from "./github-submodules"; +import { IFile, UpstreamFile } from "../model/files/files.types"; import { createLogger, serializeError } from "../logger"; import { measureStage, recordCacheHit } from "../request-monitoring"; import config from "../../config"; @@ -322,16 +323,27 @@ export default class GitHubStream extends GitHubBase { filePath: string, repoId: string, fileMeta: () => - | Promise<{ sha: string; size?: number }> - | { sha: string; size?: number } + | Promise<{ sha: string; size?: number; upstream?: UpstreamFile }> + | { sha: string; size?: number; upstream?: UpstreamFile } | Promise | string ) { const meta = await fileMeta(); - const expected: { sha: string; size?: number } = + const expected: { sha: string; size?: number; upstream?: UpstreamFile } = typeof meta === "string" ? { sha: meta } : meta; + if (expected.upstream && !validUpstream(expected.upstream)) throw new AnonymousError("invalid_request", { httpStatus: 400 }); + // Hydrated Mongoose subdocuments expose source fields through getters, + // so spreading them would retain the parent's repository coordinates. + const origin = expected.upstream ? new GitHubStream({ + ...this.data, + organization: expected.upstream.organization, + repoName: expected.upstream.repoName, + commit: expected.upstream.commit, + getToken: () => "", + }) : this; + const sourcePath = expected.upstream?.path || filePath; const generation = createHash("sha256").update(JSON.stringify([ - this.data.commit, expected.sha, this.data.cacheGeneration, this.data.cacheRevision, + this.data.commit, expected.sha, this.data.cacheGeneration, this.data.cacheRevision, ...(expected.upstream ? [expected.upstream] : []), ])).digest("hex"); const cachePath = this.data.cacheGeneration ? `${contentGenerationPrefix(this.data.cacheGeneration)}/${generation}/${filePath}` : filePath; const assertActive = async () => { @@ -383,8 +395,9 @@ export default class GitHubStream extends GitHubBase { if (cacheFills.size >= 64) throw new AnonymousError("cache_busy", { httpStatus: 503 }); filling = coordinatedFill(key, cached, async () => { await assertActive(); - const token = await this.data.getToken(); - const sourceMiss = cachedContentMiss(this.data, token); + const parentToken = await this.data.getToken(); + const token = expected.upstream ? "" : parentToken; + const sourceMiss = cachedContentMiss(this.data, parentToken); if (sourceMiss && sourceMiss !== "file_not_found") throw new AnonymousError(sourceMiss, { httpStatus: 404 }); const missKey = createHash("sha256").update(JSON.stringify([ key, this.data.organization, this.data.repoName, token, @@ -393,7 +406,7 @@ export default class GitHubStream extends GitHubBase { if (missed) throw new AnonymousError(missed, { httpStatus: 404 }); try { const fetchContent = () => measureStage("upstream", async () => { - const content = await this.downloadWithFallback(token, expected.sha, filePath) + const content = await origin.downloadWithFallback(token, expected.sha, sourcePath) .catch(error => { throw downloadError(error, filePath); }); let count = 0; const limited = new stream.Transform({ transform(chunk, encoding, callback) { @@ -413,9 +426,14 @@ export default class GitHubStream extends GitHubBase { finally { clearTimeout(deadline); limited.destroy(); content.destroy(); } return true; }); - return await this.confirmOwnerAccess(repoId, fetchContent); + return expected.upstream ? await fetchContent() : await this.confirmOwnerAccess(repoId, fetchContent); } catch (error) { - if (error instanceof AnonymousError && error.message === "file_not_found" && error.httpStatus === 404) { + if (expected.upstream && error instanceof AnonymousError && + ["file_not_found", "file_not_accessible"].includes(error.message)) { + // A removed public dependency is not a lost parent credential. + throw new AnonymousError("submodule_not_available", { httpStatus: 404, cause: error }); + } + if (!expected.upstream && error instanceof AnonymousError && error.message === "file_not_found" && error.httpStatus === 404) { const code = await classifyContentMiss(this.data, token); missingContent.set(missKey, code, 30_000); if (code !== "file_not_found") throw new AnonymousError(code, { httpStatus: 404, cause: error }); @@ -448,14 +466,14 @@ export default class GitHubStream extends GitHubBase { object: file, }); } - return { sha: fileSha, size: await file.size() }; + return { sha: fileSha, size: await file.size(), upstream: (await file.getFileInfo()).upstream }; } ); } async getFiles(progress?: (status: string) => void) { this._truncatedFolders = []; - return this.getTruncatedTree(this.data.commit, progress); + return expandSubmodules(await this.getTruncatedTree(this.data.commit, progress), this.data); } /** @@ -485,6 +503,7 @@ export default class GitHubStream extends GitHubBase { } private async fetchFileMetadata(filePath: string): Promise { + if (basename(filePath) === ".gitmodules") return null; const token = await this.data.getToken(); const oct = octokit(token); try { @@ -665,6 +684,7 @@ export default class GitHubStream extends GitHubBase { path: pathFile, repoId: this.data.repoId, size: elem.size, + ...(elem.type === "commit" ? { submodule: "unsupported" } : {}), sha: elem.sha, }); }); diff --git a/src/core/source/github-submodules.ts b/src/core/source/github-submodules.ts new file mode 100644 index 0000000..16477da --- /dev/null +++ b/src/core/source/github-submodules.ts @@ -0,0 +1,99 @@ +import { octokit, waitForTokenGate } from "../GitHubUtils"; +import AnonymousError from "../AnonymousError"; +import { IFile, UpstreamFile } from "../model/files/files.types"; +import { GitHubBaseData } from "./GitHubBase"; + +export const MAX_SUBMODULES = 20; +export const MAX_SUBMODULE_ENTRIES = 10000; +export interface SubmoduleMount extends Omit { path: string; } + +export function safeRepositoryPath(value: unknown): value is string { + return typeof value === "string" && value.length > 0 && + !value.includes("\\") && !value.includes("\0") && + value.split("/").every(part => part !== "" && part !== "." && part !== ".."); +} + +export function validUpstream(value: unknown): value is UpstreamFile { + if (!value || typeof value !== "object") return false; + const v = value as UpstreamFile; + return typeof v.organization === "string" && /^[a-zA-Z0-9-]+$/.test(v.organization) && + typeof v.repoName === "string" && /^[a-zA-Z0-9_.-]+$/.test(v.repoName) && ![".", ".."].includes(v.repoName) && + typeof v.commit === "string" && /^[a-f0-9]{40}$/.test(v.commit) && safeRepositoryPath(v.path); +} + +export function submoduleRepository(url: unknown, parent: GitHubBaseData): { organization: string; repoName: string } | null { + if (typeof url !== "string") return null; + const normalized = url.replace(/^git@github\.com:/, "https://github.com/").replace(/^ssh:\/\/git@github\.com\//, "https://github.com/").replace(/^git:\/\/github\.com\//, "https://github.com/"); + try { + const parsed = new URL(normalized, url.startsWith("./") || url.startsWith("../") + ? `https://github.com/${parent.organization}/${parent.repoName}/` : undefined); + if (parsed.protocol !== "https:" || parsed.hostname !== "github.com" || parsed.port || parsed.username || parsed.password || parsed.search || parsed.hash) return null; + const parts = parsed.pathname.replace(/\.git\/?$/, "").replace(/\/$/, "").split("/"); + if (parts.length !== 3) return null; + const candidate = { organization: parts[1], repoName: parts[2], commit: "a".repeat(40), path: "file" }; + return validUpstream(candidate) ? { organization: candidate.organization, repoName: candidate.repoName } : null; + } catch { return null; } +} + +/** Resolve only direct public GitHub dependencies. Never send the parent token to a child. */ +export async function expandSubmodules(files: IFile[], data: GitHubBaseData): Promise { + const output = files.filter(file => file.name !== ".gitmodules"); + const modules = output.filter(file => file.submodule); + let remaining = MAX_SUBMODULE_ENTRIES; + for (const [index, mount] of modules.entries()) { + if (index >= MAX_SUBMODULES) { mount.submodule = "limit"; continue; } + const mountPath = [mount.path, mount.name].filter(Boolean).join("/"); + if (!safeRepositoryPath(mountPath) || !mount.sha || !/^[a-f0-9]{40}$/.test(mount.sha)) continue; + const token = await data.getToken(); + await waitForTokenGate(token); + try { + const response = await octokit(token).repos.getContent({ + owner: data.organization, repo: data.repoName, path: mountPath, ref: data.commit, + }); + const metadata = response.data as { submodule_git_url?: string; sha?: string }; + const repository = submoduleRepository(metadata.submodule_git_url, data); + if (!repository || metadata.sha !== mount.sha) continue; + // An unauthenticated request proves public access, even for an owner whose + // parent token can also read private dependencies. No credentials cross repos. + await waitForTokenGate(""); + const child = await octokit("").git.getTree({ + owner: repository.organization, repo: repository.repoName, tree_sha: mount.sha, recursive: "1", + }); + if (child.data.truncated || child.data.tree.length > remaining) { mount.submodule = "limit"; continue; } + const entries = child.data.tree; + if (entries.some(entry => !safeRepositoryPath(entry.path))) { mount.submodule = "unsupported"; continue; } + remaining -= entries.length; + mount.submodule = "included"; + mount.upstream = { ...repository, commit: mount.sha, path: "" }; + for (const entry of entries) { + const relative = entry.path!; + if (relative.split("/").pop() === ".gitmodules") continue; + const fullPath = `${mountPath}/${relative}`; + const slash = fullPath.lastIndexOf("/"); + output.push({ + repoId: data.repoId, name: fullPath.slice(slash + 1), path: fullPath.slice(0, slash), + sha: entry.sha, size: entry.size, + ...(entry.type === "commit" ? { submodule: "nested" as const } : {}), + ...(entry.type === "blob" ? { upstream: { ...repository, commit: mount.sha, path: relative } } : {}), + }); + } + } catch (error) { + if ((error as { status?: number }).status === 404) { mount.submodule = "unavailable"; continue; } + // Do not persist a partial tree on rate limits, outages or credential errors. + throw error; + } + } + return output; +} + +export function validateMounts(value: unknown): SubmoduleMount[] { + if (value === undefined) return []; + if (!Array.isArray(value) || value.length > MAX_SUBMODULES || value.some(mount => !validUpstream(mount))) { + throw new AnonymousError("invalid_request", { httpStatus: 400 }); + } + const paths = value.map(mount => mount.path as string); + if (paths.some((path, i) => paths.some((other, j) => i !== j && (path === other || path.startsWith(other + "/"))))) { + throw new AnonymousError("invalid_request", { httpStatus: 400 }); + } + return value; +} diff --git a/src/core/zipStream.ts b/src/core/zipStream.ts index 9aa55a7..b088d05 100644 --- a/src/core/zipStream.ts +++ b/src/core/zipStream.ts @@ -1,4 +1,6 @@ import got from "got"; +import config from "../config"; +import { MAX_SUBMODULE_ENTRIES, SubmoduleMount, safeRepositoryPath, validateMounts } from "./source/github-submodules"; import { Readable, Transform, Writable } from "stream"; import { Parse } from "unzip-stream"; import archiver = require("archiver"); @@ -19,6 +21,7 @@ const logger = createLogger("zip-stream"); export interface StreamAnonymizedZipOptions { repoId: string; + submodules?: SubmoduleMount[]; organization: string; repoName: string; commit: string; @@ -78,6 +81,7 @@ export async function streamAnonymizedZip( on(event: string, listener: (...args: unknown[]) => void): unknown; } ): Promise { + const submodules = validateMounts(opt.submodules); let response; try { const token = await opt.getToken(); @@ -113,14 +117,10 @@ export async function streamAnonymizedZip( }, }); } - const upstreamDone = startStage("upstream"); - const downloadStream = got.stream(response.url); - downloadStream.once("end", () => upstreamDone()); - downloadStream.once("error", () => upstreamDone(true)); - downloadStream.once("close", () => upstreamDone(!downloadStream.readableEnded)); - + let downloadStream: Readable; const archive = archiver("zip", {}); - const parser = Parse() as Transform; + let parser: Transform; + let childEntries = 0; const activeStreams = new Set(); const compiledTerms = compileTerms(opt.anonymizerOptions.terms || []); let stopped = false; @@ -160,8 +160,8 @@ export async function streamAnonymizedZip( stopped = true; cleanupDrain?.(); cleanupDrain = undefined; - downloadStream.destroy(); - parser.destroy(); + downloadStream?.destroy(); + parser?.destroy(); for (const stream of activeStreams) stream.destroy(); activeStreams.clear(); output.destroy(); @@ -187,53 +187,82 @@ export async function streamAnonymizedZip( archive.on("error", fail); archive.pipe(output); - downloadStream - .on("error", fail) - .pipe(parser) - .on("entry", (entry: Readable & { type: string; path: string; autodrain: () => void }) => { - if (stopped) { - entry.destroy(); - return; - } - entry.on("error", fail); - activeStreams.add(entry); - entry.once("close", () => activeStreams.delete(entry)); - if (entry.type === "File") { - try { - const fileName = anonymizePathCompiled( - entry.path.substring(entry.path.indexOf("/") + 1), - compiledTerms - ); - if (!isEntryAllowed(entry.path, opt.contentOptions)) { - entry.autodrain(); - return; - } - // Pass filePath via the constructor — AnonymizeTransformer reads it - // there to decide whether the entry is text (and therefore should be - // anonymized) vs binary (passthrough). Assigning afterwards leaves - // isText=false for every file, so the zip ships unanonymized. - parser.pause(); - const onArchived = () => { entryWaiting = true; setImmediate(resumeParser); }; - archive.once("entry", onArchived); - const anonymizer = new AnonymizeTransformer({ - ...opt.anonymizerOptions, - filePath: entry.path, - }); - anonymizer.on("error", fail); - activeStreams.add(anonymizer); - anonymizer.once("close", () => activeStreams.delete(anonymizer)); - const st = entry.pipe(anonymizer); - archive.append(st, zipEntryOptions(fileName, entry.path)); - } catch (error) { - fail(error as Error); + const startArchive = (url: string, prefix: string, next: number) => { + if (stopped) return; + const upstreamDone = startStage("upstream"); + const input = got.stream(url); + downloadStream = input; + input.once("end", () => upstreamDone()); + input.once("error", () => upstreamDone(true)); + input.once("close", () => upstreamDone(!input.readableEnded)); + parser = Parse() as Transform; + downloadStream + .on("error", fail) + .pipe(parser) + .on("entry", (entry: Readable & { type: string; path: string; autodrain: () => void }) => { + if (stopped) { + entry.destroy(); + return; } - } else { - entry.autodrain(); - } - }) - .on("error", fail) - .on("end", () => { - if (stopped) return; - void archive.finalize().catch(fail); - }); + entry.on("error", fail); + activeStreams.add(entry); + entry.once("close", () => activeStreams.delete(entry)); + if (entry.type === "File") { + try { + const relativePath = entry.path.substring(entry.path.indexOf("/") + 1); + if (!safeRepositoryPath(relativePath)) throw new AnonymousError("invalid_file_path", { httpStatus: 502 }); + if (relativePath.split("/").pop() === ".gitmodules") { entry.autodrain(); return; } + if (prefix && ++childEntries > MAX_SUBMODULE_ENTRIES) throw new AnonymousError("file_too_big", { httpStatus: 413 }); + const mountedPath = prefix ? `${prefix}/${relativePath}` : relativePath; + const fileName = anonymizePathCompiled(mountedPath, compiledTerms); + if (!isEntryAllowed(entry.path, opt.contentOptions)) { + entry.autodrain(); + return; + } + // Pass filePath via the constructor — AnonymizeTransformer reads it + // there to decide whether the entry is text (and therefore should be + // anonymized) vs binary (passthrough). Assigning afterwards leaves + // isText=false for every file, so the zip ships unanonymized. + parser.pause(); + const onArchived = () => { entryWaiting = true; setImmediate(resumeParser); }; + archive.once("entry", onArchived); + const anonymizer = new AnonymizeTransformer({ + ...opt.anonymizerOptions, + filePath: mountedPath, + }); + anonymizer.on("error", fail); + activeStreams.add(anonymizer); + anonymizer.once("close", () => activeStreams.delete(anonymizer)); + let source: Readable = entry; + if (prefix) { + let bytes = 0; + const limited = new Transform({ transform(chunk, encoding, callback) { + bytes += chunk.length; + callback(bytes > config.MAX_FILE_SIZE ? new AnonymousError("file_too_big", { httpStatus: 413 }) : null, chunk); + } }); + limited.on("error", fail); + activeStreams.add(limited); + limited.once("close", () => activeStreams.delete(limited)); + source = entry.pipe(limited); + } + const st = source.pipe(anonymizer); + archive.append(st, zipEntryOptions(fileName, mountedPath)); + } catch (error) { + fail(error as Error); + } + } else { + entry.autodrain(); + } + }) + .on("error", fail) + .on("end", () => { + if (stopped) return; + const mount = submodules[next]; + if (mount) { + // Public child archives never receive parent credentials. + startArchive(`https://codeload.github.com/${encodeURIComponent(mount.organization)}/${encodeURIComponent(mount.repoName)}/zip/${mount.commit}`, mount.path, next + 1); + } else void archive.finalize().catch(fail); + }); + }; + startArchive(response.url, "", 0); } diff --git a/src/server/routes/repository-public.ts b/src/server/routes/repository-public.ts index ef1c8ad..e6388f1 100644 --- a/src/server/routes/repository-public.ts +++ b/src/server/routes/repository-public.ts @@ -57,6 +57,7 @@ router.get( res.attachment(`${repo.repoId}.zip`); if (req.method === "HEAD") { res.end(); return; } await repo.countView(); + const submodules = await repo.submoduleMounts(); if (config.STREAMER_ENTRYPOINT) { // use the streamer service @@ -75,6 +76,7 @@ router.get( branch: repo.model.source.branch, repoId: repo.repoId, anonymizerOptions: anonymizer.opt, + submodules, contentOptions: { image: repo.options.image, pdf: repo.options.pdf, @@ -169,6 +171,7 @@ router.get( commit: repo.model.source.commit || "HEAD", getToken: () => repo.getToken(), anonymizerOptions: anonymizer.opt, + submodules, contentOptions: { image: repo.options.image, pdf: repo.options.pdf, @@ -383,15 +386,15 @@ router.get( truncatedFolders: (repo.model.truncatedFolders || []).map((path) => anonymizePath(path, repo.options.terms || []) ), - // Submodule contents are not included in GitHub archives/trees, so - // they end up as empty folders in the anonymized repository. Surface - // a warning in the explorer when the repository uses submodules (#737). + // Legacy trees retain .gitmodules until their next tree refresh. hasSubmodules: (await FileModel.exists({ repoId: repo.repoId, treeGeneration: repo.model.treeGeneration || { $exists: false }, - name: ".gitmodules", - path: "", + $or: [ + { submodule: { $exists: true, $ne: "included" } }, + { name: ".gitmodules", path: "" }, + ], })) != null, }); } catch (error) { diff --git a/src/streamer/route.ts b/src/streamer/route.ts index e96abf3..447c759 100644 --- a/src/streamer/route.ts +++ b/src/streamer/route.ts @@ -34,6 +34,7 @@ router.post( getToken: () => token, anonymizerOptions: { ...anonymizerOptions, repoId }, contentOptions, + submodules: req.body.submodules, }, res ); @@ -89,7 +90,7 @@ router.post("/", async (req: express.Request, res: express.Response) => { const content = await source.getFileContentCache( filePath, repoId, - () => ({ sha: fileSha, size: fileSize }) + () => ({ sha: fileSha, size: fileSize, upstream: req.body.upstream }) ); if (res.destroyed || res.writableEnded) { content.destroy(); anonymizer.destroy(); return; } const mime = lookup(filePath); diff --git a/test/github-submodules.test.js b/test/github-submodules.test.js new file mode 100644 index 0000000..d011474 --- /dev/null +++ b/test/github-submodules.test.js @@ -0,0 +1,308 @@ +const { expect } = require("chai"); +const { Readable, PassThrough } = require("node:stream"); +const { once } = require("node:events"); +const { setImmediate } = require("node:timers"); +const archiver = require("archiver"); +const got = require("got"); +require("ts-node/register/transpile-only"); +const utils = require("../src/core/GitHubUtils"); +const { expandSubmodules, submoduleRepository, validateMounts, MAX_SUBMODULES, MAX_SUBMODULE_ENTRIES } = require("../src/core/source/github-submodules"); +const GitHubStream = require("../src/core/source/GitHubStream").default; +const FileModel = require("../src/core/model/files/files.model").default; +const storage = require("../src/core/storage").default; +const { streamAnonymizedZip } = require("../src/core/zipStream"); +const config = require("../src/config").default; +const sha = "a".repeat(40), blob = "b".repeat(40); +const parent = { repoId: "submodule-test", organization: "parent", repoName: "repo", commit: "c".repeat(40), getToken: () => "parent-private-token" }; +const upstream = { organization: "child", repoName: "public", commit: sha, path: "src/Alice.txt" }; +const mount = () => new FileModel({ repoId: parent.repoId, name: "dependency", path: "vendor", sha, submodule: "unsupported" }); + +async function zip(entries) { + const archive = archiver("zip"); const chunks = []; + archive.on("data", chunk => chunks.push(chunk)); const done = once(archive, "end"); + for (const [name, content] of Object.entries(entries)) archive.append(content, { name }); + await archive.finalize(); await done; return Buffer.concat(chunks); +} + +async function readZip(options) { + const response = new PassThrough(); const parser = require("unzip-stream").Parse(); + const entries = [], files = {}; + parser.on("entry", entry => { + const chunks = []; entry.on("data", chunk => chunks.push(chunk)); + entries.push(once(entry, "end").then(() => { files[entry.path] = Buffer.concat(chunks).toString(); })); + }); + response.pipe(parser); const done = once(parser, "finish"); + await streamAnonymizedZip(options, response); await done; await Promise.all(entries); + return files; +} + +const zipOptions = { ...parent, getToken: () => "", anonymizerOptions: { terms: ["Alice"], image: true, link: true }, + submodules: [{ organization: "child", repoName: "public", commit: sha, path: "vendor/dependency" }] }; + +describe("public GitHub submodules", function () { + this.timeout(10000); + const restore = []; + function stub(object, key, value) { const previous = object[key]; object[key] = value; restore.push(() => { object[key] = previous; }); } + afterEach(() => restore.splice(0).reverse().forEach(fn => fn())); + beforeEach(() => stub(utils, "waitForTokenGate", async () => {})); + + function api(tree = [{ path: "src/Alice.txt", type: "blob", sha: blob, size: 5 }], options = {}) { + const calls = []; + stub(utils, "octokit", token => ({ + repos: { getContent: async args => { + calls.push({ method: "contents", token, ...args }); + return { data: { submodule_git_url: options.url || "https://github.com/child/public.git", sha } }; + } }, + git: { getTree: async args => { + calls.push({ method: "tree", token, ...args }); + if (options.error) throw options.error; + return { data: { tree, truncated: options.truncated || false } }; + } }, + })); + return calls; + } + + it("mounts a pinned child tree without forwarding parent credentials or exposing .gitmodules", async () => { + const calls = api([ + { path: ".gitmodules", type: "blob", sha: blob, size: 100 }, + { path: "src", type: "tree", sha: blob }, + { path: "src/Alice.txt", type: "blob", sha: blob, size: 5 }, + { path: "nested", type: "commit", sha }, + ]); + const files = await expandSubmodules([mount(), { name: ".gitmodules", path: "" }], parent); + expect(files.map(file => file.name)).to.deep.equal(["dependency", "src", "Alice.txt", "nested"]); + expect(files[0].submodule).to.equal("included"); + expect(files[2]).to.include({ path: "vendor/dependency/src", sha: blob, size: 5 }); + expect(files[2].upstream).to.deep.equal(upstream); + expect(files[3].submodule).to.equal("nested"); + expect(calls).to.deep.equal([ + { method: "contents", token: "parent-private-token", owner: "parent", repo: "repo", path: "vendor/dependency", ref: parent.commit }, + { method: "tree", token: "", owner: "child", repo: "public", tree_sha: sha, recursive: "1" }, + ]); + expect(new FileModel(files[2]).toObject().upstream).to.deep.equal(upstream); + }); + + it("discovers submodules through scoped public-read contexts without authorizing child or write requests", async () => { + const { registerGitHubToken } = require("../src/core/github-token-context"); + const token = "public-read:submodule-discovery"; + registerGitHubToken(token, { publicRepository: "parent/repo", quotaKey: "submodule-test", + renew: async () => "parent-user-token" }); + const calls = []; + stub(globalThis, "fetch", async (url, options) => { + calls.push({ url: String(url), authorization: new globalThis.Headers(options.headers).get("authorization") }); + const result = String(url).includes("/contents/") + ? { submodule_git_url: "https://github.com/child/public.git", sha } + : { tree: [{ path: "src/Alice.txt", type: "blob", sha: blob, size: 5 }], truncated: false }; + return new globalThis.Response(JSON.stringify(result), { headers: { "content-type": "application/json" } }); + }); + const files = await expandSubmodules([mount()], { ...parent, getToken: () => token }); + expect(files[0].submodule).to.equal("included"); + expect(calls).to.deep.equal([ + { url: `https://api.github.com/repos/parent/repo/contents/vendor%2Fdependency?ref=${parent.commit}`, authorization: "token parent-user-token" }, + { url: `https://api.github.com/repos/child/public/git/trees/${sha}?recursive=1`, authorization: null }, + ]); + for (const request of [ + "GET /repos/child/public/contents/file.txt", "GET /repos/parent/repo-other/contents/file.txt", + "PUT /repos/parent/repo/contents/file.txt", "DELETE /repos/parent/repo/contents/file.txt", + ]) { + let caught; try { await utils.octokit(token).request(request); } catch (error) { caught = error; } + expect(caught?.message).to.equal("github_app_access_required"); + } + expect(calls).to.have.length(2); + }); + + it("recognizes gitlinks from the parent tree and leaves ordinary repositories at one request", async () => { + const source = new GitHubStream(parent); let calls = 0; + source.getGHTree = async () => { calls++; return { tree: [{ path: "README.md", type: "blob", sha: blob, size: 5 }] }; }; + expect((await source.getFiles()).map(file => file.name)).to.deep.equal(["README.md"]); expect(calls).to.equal(1); + api(); + source.getGHTree = async () => ({ tree: [{ path: "vendor/dependency", type: "commit", mode: "160000", sha }] }); + expect((await source.getFiles())[0].submodule).to.equal("included"); + }); + + it("supports relative and SSH GitHub URLs and rejects other hosts and unsafe paths", () => { + for (const url of ["../public.git", "git@github.com:parent/public.git", "ssh://git@github.com/parent/public.git", "https://github.com/parent/public.git"]) { + expect(submoduleRepository(url, parent)).to.deep.equal({ organization: "parent", repoName: "public" }); + } + for (const url of ["https://evil.test/child/repo", "file:///tmp/repo", "https://github.com@evil.test/a/b", "https://github.com/a/b?token=secret", "https://github.com/a%2fb/c"]) expect(submoduleRepository(url, parent)).to.equal(null); + for (const path of ["../escape", "/absolute", "a/../b", "a\\b", "a//b", "a\0b"]) expect(() => validateMounts([{ ...upstream, path }])).to.throw("invalid_request"); + expect(() => validateMounts([{ ...upstream, path: "lib" }, { ...upstream, path: "lib/nested" }])).to.throw("invalid_request"); + }); + + it("leaves private, missing and external children unsupported without blocking parent files", async () => { + api([], { error: Object.assign(new Error("not found"), { status: 404 }) }); + const result = await expandSubmodules([mount(), { name: "README.md", path: "", size: 3 }], parent); + expect(result[0].submodule).to.equal("unavailable"); expect(result[1].name).to.equal("README.md"); + const calls = api([], { url: "https://gitlab.com/child/repo.git" }); + expect((await expandSubmodules([mount()], parent))[0].submodule).to.equal("unsupported"); + expect(calls).to.have.length(1); + }); + + it("does not persist transient failures as permanently missing dependencies", async () => { + const error = Object.assign(new Error("rate limited"), { status: 403 }); api([], { error }); + let caught; try { await expandSubmodules([mount()], parent); } catch (err) { caught = err; } + expect(caught).to.equal(error); + }); + + it("bounds module count and rejects oversized or truncated child trees", async () => { + const calls = api(); + const mounts = Array.from({ length: MAX_SUBMODULES + 1 }, (_, i) => ({ ...mount().toObject(), name: `dependency${i}` })); + const result = await expandSubmodules(mounts, parent); + expect(result[MAX_SUBMODULES].submodule).to.equal("limit"); expect(calls).to.have.length(MAX_SUBMODULES * 2); + api(Array.from({ length: MAX_SUBMODULE_ENTRIES + 1 }, (_, i) => ({ path: `file${i}`, type: "blob", sha: blob, size: 1 }))); + expect((await expandSubmodules([mount()], parent))[0].submodule).to.equal("limit"); + api([], { truncated: true }); expect((await expandSubmodules([mount()], parent))[0].submodule).to.equal("limit"); + }); + + for (const hydrated of [false, true]) it(`fetches child content from ${hydrated ? "hydrated" : "plain"} metadata and caches under its parent mount`, async () => { + const fileOrigin = hydrated ? new FileModel({ upstream }).upstream : upstream; + const source = new GitHubStream({ ...parent, cacheGeneration: "tree:date" }); const cache = new Map(), requests = []; + stub(storage, "fileInfo", async (_id, path) => { if (!cache.has(path)) throw Object.assign(Error("missing"), { code: "ENOENT" }); return { size: cache.get(path).length }; }); + stub(storage, "write", async (id, path, input) => { expect(id).to.equal(parent.repoId); const chunks = []; for await (const chunk of input) chunks.push(Buffer.from(chunk)); cache.set(path, Buffer.concat(chunks)); }); + stub(storage, "read", async (_id, path) => Readable.from([cache.get(path)])); + stub(got, "stream", (url, options) => { requests.push({ url, options }); return Readable.from([Buffer.from("Alice")]); }); + for (let i = 0; i < 2; i++) { + const stream = await source.getFileContentCache("vendor/dependency/src/Alice.txt", parent.repoId, () => ({ sha: blob, size: 5, upstream: fileOrigin })); + for await (const chunk of stream) expect(chunk.toString()).to.equal("Alice"); + } + expect(requests).to.have.length(1); + expect(requests[0].url).to.equal(`https://github.com/child/public/raw/${sha}/src/Alice.txt`); + expect(requests[0].options.headers).to.deep.equal({}); + expect([...cache.keys()][0]).to.match(/\/vendor\/dependency\/src\/Alice\.txt$/); + const changed = { ...upstream, commit: "d".repeat(40) }; + const stream = await source.getFileContentCache("vendor/dependency/src/Alice.txt", parent.repoId, () => ({ sha: blob, size: 5, upstream: changed })); + stream.resume(); expect(requests).to.have.length(2); + }); + + it("does not report an unavailable public child as lost parent access", async () => { + const { isAccessFailure } = require("../src/core/owner-notifications"); + const source = new GitHubStream(parent); + stub(storage, "fileInfo", async () => { throw Object.assign(Error("missing"), { code: "ENOENT" }); }); + stub(storage, "write", async (_id, _path, input) => { for await (const _chunk of input) { /* consume */ } }); + stub(got, "stream", () => Readable.from((async function* () { + yield* []; + throw Object.assign(Error("not found"), { response: { statusCode: 404 } }); + })())); + let error; + try { await source.getFileContentCache("vendor/lib/file.txt", parent.repoId, () => ({ sha: blob, size: 5, upstream })); } + catch (caught) { error = caught; } + expect(error?.message).to.equal("submodule_not_available"); + expect(isAccessFailure(error)).to.equal(false); + }); + + it("includes pinned child archives with anonymized names and content, omitting manifests and disabled images", async () => { + const main = await zip({ "parent/README.md": "Alice parent", "parent/.gitmodules": "https://github.com/private/identity" }); + const child = await zip({ "child/src/Alice.txt": "Alice child", "child/.gitmodules": "identity", "child/photo.png": "image" }); + const calls = []; + stub(got, "stream", (url, options) => { calls.push({ url, options }); return Readable.from([calls.length === 1 ? main : child]); }); + expect(await readZip({ ...zipOptions, contentOptions: { image: false } })).to.deep.equal({ + "README.md": "XXXX-1 parent", "vendor/dependency/src/XXXX-1.txt": "XXXX-1 child", + }); + expect(calls.map(call => call.url)).to.deep.equal([ + `https://codeload.github.com/parent/repo/zip/${parent.commit}`, `https://codeload.github.com/child/public/zip/${sha}`, + ]); + expect(calls.every(call => call.options === undefined)).to.equal(true); + }); + + it("aborts the ZIP if a child exceeds the per-file limit", async () => { + const main = await zip({ "parent/README.md": "ok" }), child = await zip({ "child/large.txt": "too much data" }); + let calls = 0; stub(got, "stream", () => Readable.from([++calls === 1 ? main : child])); stub(config, "MAX_FILE_SIZE", 4); + const response = new PassThrough(); response.resume(); const errors = []; + response.on("error", error => errors.push(error)); const done = new Promise(resolve => response.once("close", resolve)); + await streamAnonymizedZip(zipOptions, response); await done; + expect(errors.map(error => error.message)).to.deep.equal(["file_too_big"]); + }); +}); + +describe("submodule transport and public metadata", function () { + this.timeout(10000); + it("does not expose upstream repository coordinates in the public tree", async () => { + const Repository = require("../src/core/Repository").default; + const files = await Repository.prototype.anonymizedFiles.call({ + _model: { options: { terms: ["Alice"] } }, + files: async () => [{ name: "Alice.txt", path: "vendor/dependency/src", sha: blob, size: 5, upstream, submodule: "included" }], + }, { includeSha: false }); + expect(files[0]).to.have.all.keys("name", "path", "sha", "size"); + expect(files[0].name).to.equal("XXXX-1.txt"); + expect(JSON.stringify(files)).not.to.include("public"); + }); + + for (const mode of ["send", "anonymizedContent"]) { + it(`preserves child origin through ${mode} and the real streamer route`, async () => { + const express = require("express"); + const File = require("../src/core/AnonymizedFile").default; + const { AnonymizeTransformer } = require("../src/core/anonymize-utils"); + const router = require("../src/streamer/route").router; + const previous = { endpoint: config.STREAMER_ENTRYPOINT, stream: got.stream, + info: storage.fileInfo, write: storage.write, read: storage.read }; + let server, payload; const cache = new Map(), requests = []; + try { + storage.fileInfo = async (_id, path) => { if (!cache.has(path)) throw Object.assign(Error("missing"), { code: "ENOENT" }); return { size: cache.get(path).length }; }; + storage.write = async (_id, path, stream) => { const chunks = []; for await (const chunk of stream) chunks.push(Buffer.from(chunk)); cache.set(path, Buffer.concat(chunks)); }; + storage.read = async (_id, path) => Readable.from([cache.get(path)]); + got.stream = (url, options) => { + if (!String(url).startsWith("https://github.com/")) return previous.stream(url, options); + requests.push({ url, options }); return Readable.from([Buffer.from("Alice")]); + }; + const app = express(); app.use(express.json()); + app.use((req, _res, next) => { payload = req.body; next(); }); app.use("/api", router); + server = await new Promise(resolve => { const listening = app.listen(0, "127.0.0.1", () => resolve(listening)); }); + config.STREAMER_ENTRYPOINT = `http://127.0.0.1:${server.address().port}/`; + const file = new File({ anonymizedPath: "vendor/dependency/src/XXXX-1.txt", repository: { + repoId: parent.repoId, options: { terms: ["Alice"] }, + model: { source: { repositoryName: "parent/repo", commit: parent.commit } }, + getToken: parent.getToken, owner: { id: "owner" }, + generateAnonymizeTransformer: filePath => new AnonymizeTransformer({ terms: ["Alice"], filePath, image: true, link: true }), + } }); + file._file = { name: "Alice.txt", path: "vendor/dependency/src", sha: blob, size: 5, upstream }; + let content = ""; + if (mode === "send") { + const response = new PassThrough(); response.setHeader = () => {}; + response.on("data", chunk => { content += chunk; }); + await file.send(response); + } else { + for await (const chunk of await file.anonymizedContent()) content += chunk; + } + expect(content).to.equal("XXXX-1"); + expect(payload.upstream).to.deep.equal(upstream); + expect(payload.filePath).to.equal("vendor/dependency/src/Alice.txt"); + expect(requests).to.have.length(1); + expect(requests[0].url).to.equal(`https://github.com/child/public/raw/${sha}/src/Alice.txt`); + expect(requests[0].options.headers).to.deep.equal({}); + } finally { + config.STREAMER_ENTRYPOINT = previous.endpoint; got.stream = previous.stream; + storage.fileInfo = previous.info; storage.write = previous.write; storage.read = previous.read; + if (server) { server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); } + } + }); + } + + it("cancels the current child archive when the reader disconnects", async () => { + const main = await zip({ "parent/README.md": "ok" }); const child = new PassThrough(); + const previous = got.stream; let calls = 0, started; + const childStarted = new Promise(resolve => { started = resolve; }); + const response = new PassThrough(); response.resume(); + try { + got.stream = () => { if (++calls === 1) return Readable.from([main]); started(); return child; }; + await streamAnonymizedZip({ ...zipOptions, submodules: [...zipOptions.submodules, + { ...zipOptions.submodules[0], path: "other/dependency" }] }, response); + await childStarted; response.destroy(); + await new Promise(resolve => setImmediate(resolve)); + expect(child.destroyed).to.equal(true); expect(calls).to.equal(2); + } finally { got.stream = previous; child.destroy(); response.destroy(); } + }); +}); + +describe("legacy submodule manifests", function () { + it("rejects a cached .gitmodules file even when its public name was anonymized", async () => { + const File = require("../src/core/AnonymizedFile").default; + const file = new File({ anonymizedPath: ".XXXX-1", repository: { + options: { terms: ["gitmodules"] }, model: { source: { type: "GitHubStream" } }, + } }); + file._file = { name: ".gitmodules", path: "", sha: blob, size: 5 }; + for (const read of [() => file.getFileInfo(), () => file.originalPath()]) { + let error; try { await read(); } catch (caught) { error = caught; } + expect(error?.message).to.equal("file_not_found"); + } + }); +}); diff --git a/test/owner-notifications.test.js b/test/owner-notifications.test.js index 0343bee..49756cc 100644 --- a/test/owner-notifications.test.js +++ b/test/owner-notifications.test.js @@ -500,9 +500,7 @@ describe("streamed file access alerts", () => { stub(config, "STREAMER_ENTRYPOINT", "http://streamer/"); const stream = new PassThrough(); stub(got, "stream", () => stream); - file.originalPath = async () => "README.md"; - file.sha = async () => "sha"; - file.size = async () => 10; + file._file = { name: "README.md", path: "", sha: "sha", size: 10 }; const returned = await file.anonymizedContent(); expect(returned).to.equal(stream); const error = Object.assign(new Error("streamer failure"), { response: { statusCode: status, body: JSON.stringify({ error: status === 503 ? "upstream_error" : "file_not_accessible" }) } }); diff --git a/test/performance-integration.test.js b/test/performance-integration.test.js index c0bc500..8fa69d8 100644 --- a/test/performance-integration.test.js +++ b/test/performance-integration.test.js @@ -40,6 +40,19 @@ const uri = process.env.PERFORMANCE_MONGO_URI; }); beforeEach(async () => { await Promise.all([Repo, PR, Gist, File, Path, Name, ConferenceModel].map(model => model.deleteMany({}))); }); + it("preserves submodule origin through the persisted anonymized path index", async () => { + const row = await Repo.create({ repoId: "submodule-index", owner: user.model._id, status: "ready", treeGeneration: "generation", + source: { repositoryName: "parent/repo", commit: "a".repeat(40) }, options: { terms: ["Alice"] } }); + const upstream = { organization: "child", repoName: "public", commit: "b".repeat(40), path: "Alice.txt" }; + await File.create({ repoId: row.repoId, treeGeneration: "generation", name: "Alice.txt", path: "vendor/lib", sha: "c".repeat(40), size: 5, upstream }); + const repository = new Repository(row); + const file = new AnonymizedFile({ repository, anonymizedPath: "vendor/lib/XXXX-1.txt" }); + expect((await file.getFileInfo()).upstream).to.deep.equal(upstream); + const publicTree = await repository.anonymizedFiles({ includeSha: false }); + expect(publicTree[0]).not.to.have.property("upstream"); + expect(JSON.stringify(publicTree)).not.to.include("child"); + }); + it("uses membership indexes for owner, stable GitHub ID, and legacy username", async () => { const other = new mongoose.Types.ObjectId(); await Repo.insertMany(Array.from({ length: 2000 }, (_, i) => ({ repoId: `filler-${i}`, owner: other, coauthors: [{ username: `other-${i}`, githubId: String(i + 50000) }] }))); diff --git a/test/zip-proxy-errors.test.js b/test/zip-proxy-errors.test.js index 626e081..fb7ee80 100644 --- a/test/zip-proxy-errors.test.js +++ b/test/zip-proxy-errors.test.js @@ -25,7 +25,7 @@ describe("ZIP proxy failures", function () { utils.getUser = async () => { throw new Error("anonymous"); }; utils.getRepo = async () => ({ repoId: "test", options: {}, model: { source: { repositoryName: "owner/repo", commit: "abc" } }, - countView: async () => {}, getToken: async () => "", + countView: async () => {}, submoduleMounts: async () => [], getToken: async () => "", generateAnonymizeTransformer: () => ({ opt: {} }), }); upstream = new PassThrough();