feat: include public GitHub submodules in shared repositories (#914)

* feat: include public GitHub submodules in shared repositories

* fix: preserve submodule access across token and document contexts

README.md

@@ -33,6 +33,20 @@ A free public instance is available at **[anonymous.4open.science](https://anony
 
 Just open **[anonymous.4open.science](https://anonymous.4open.science/)**, paste your repository URL, add the terms to hide, and share the generated link.
 
+### Git submodules
+
+Direct public GitHub submodules are included in the file browser and ZIP downloads
+at the commit recorded by the parent repository. Their paths and contents use the
+parent's anonymization settings. `.gitmodules` files are omitted from shared GitHub
+content because they contain source repository URLs.
+
+The service expands up to 20 direct submodules and 10,000 combined child tree entries.
+Private, non-GitHub, nested, missing, oversized, and truncated submodules remain empty
+folders with a warning. Child files are fetched anonymously on demand; child tree
+lookups use GitHub's unauthenticated API quota. ZIP downloads fetch public child archives.
+Existing cached repositories with `.gitmodules` rebuild their file trees on the next
+listing or ZIP download. No manual database migration is required.
+
 ### CLI
 
 Anonymize a repository locally and produce an anonymized zip:

public/i18n/locale-en.json

@@ -72,6 +72,7 @@
     "invalid_request": "The request is missing required fields or is malformed.",
     "invalid_project_name": "Use a project name of up to 100 characters without line breaks or control characters.",
     "no_file_selected": "Please select a file.",
+    "submodule_not_available": "This public submodule file is no longer available at its recorded commit.",
     "file_not_found": "The requested file is not found.",
     "file_not_accessible": "The requested file is not accessible.",
     "file_not_supported": "The file type is not supported. Anonymous GitHub cannot handle it.",
@@ -164,6 +165,6 @@
     "page_branch_mismatch": "GitHub Pages on this repository is served from the '{{pageBranch}}' branch, but you selected '{{selectedBranch}}'. Switch the branch above to '{{pageBranch}}' to anonymize the Pages site.",
     "folder_truncated": "This folder has more than 10,000 entries; only a partial listing is shown.",
     "repo_truncated": "Some folders in this repository have too many files to be fully listed. Affected folders are marked with a warning icon.",
-    "submodules_not_included": "This repository uses git submodules. Submodule contents are not included in the anonymized repository and appear as empty folders."
+    "submodules_not_included": "Some submodule contents could not be included. Only direct public GitHub submodules within the size limits are supported. Private, external, nested, or unavailable submodules appear as empty folders."
   }
 }

src/core/AnonymizedFile.ts

@@ -153,8 +153,18 @@ export default class AnonymizedFile {
     return this._file.size;
   }
 
+  private visibleFile(file: IFile): IFile {
+    if (this.repository.model?.source?.type !== "Zip" && file.name === ".gitmodules") {
+      throw new AnonymousError("file_not_found", { httpStatus: 404 });
+    }
+    return file;
+  }
+
   async getFileInfo(): Promise<IFile> {
-    if (this._file) return this._file;
+    if (this.repository.model?.source?.type !== "Zip" && basename(this.anonymizedPath) === ".gitmodules") {
+      throw new AnonymousError("file_not_found", { httpStatus: 404 });
+    }
+    if (this._file) return this.visibleFile(this._file);
     let fileDir = dirname(this.anonymizedPath);
     if (fileDir == ".") fileDir = "";
     if (fileDir.endsWith("/")) fileDir = fileDir.slice(0, -1);
@@ -180,7 +190,7 @@ export default class AnonymizedFile {
     if (exact) {
       if (exact.metadataPending) await this.repository.completeRecoveredFileMetadata(exact);
       this._file = exact;
-      return exact;
+      return this.visibleFile(exact);
     }
 
     const terms = this.repository.options.terms || [];
@@ -213,7 +223,7 @@ export default class AnonymizedFile {
     const compiled = compileTerms(terms);
     if (typeof this.repository.findAnonymizedPath === "function") {
       const indexed = await this.repository.findAnonymizedPath(this.anonymizedPath);
-      if (indexed) { this._file = indexed; return indexed; }
+      if (indexed) { this._file = indexed; return this.visibleFile(indexed); }
       if (indexed === null) {
         if (anonymizePathCompiled(this.anonymizedPath, compiled) === this.anonymizedPath) {
           const recovered = await this.recoverTruncatedFile(fileDir);
@@ -237,7 +247,7 @@ export default class AnonymizedFile {
       ) {
         if (candidate.metadataPending) await this.repository.completeRecoveredFileMetadata(candidate);
         this._file = candidate;
-        return candidate;
+        return this.visibleFile(candidate);
       }
     }
 
@@ -312,6 +322,7 @@ export default class AnonymizedFile {
     if (!this._file) {
       this._file = await this.getFileInfo();
     }
+    this.visibleFile(this._file);
     return join(this._file.path, this._file.name);
   }
   extension() {
@@ -398,6 +409,7 @@ export default class AnonymizedFile {
         branch: this.repository.model.source.branch,
         repoId: this.repository.repoId,
         filePath: this.filePath,
+        upstream: (await this.getFileInfo()).upstream,
         sha: await this.sha(),
         size: await this.size(),
         anonymizerOptions: anonymizer.opt,
@@ -446,6 +458,7 @@ export default class AnonymizedFile {
               json: {
                 sha,
                 size,
+                upstream: (await this.getFileInfo()).upstream,
                 token: await githubTokenForStreamer(token, this.repository.model.source.repositoryName),
                 repoFullName: this.repository.model.source.repositoryName,
                 commit: this.repository.model.source.commit,

src/core/GitHubUtils.ts

@@ -244,7 +244,7 @@ export function octokit(token: string) {
         const prefix = `/repos/${context.publicRepository.split("/").map(encodeURIComponent).join("/")}`.toLowerCase();
         const path = url.pathname.toLowerCase();
         const suffix = path.slice(prefix.length);
-        const readable = /^(?:\/?|\/branches(?:\/[^/]+)?|\/commits(?:\/[^/]+)?|\/readme|\/pages|\/zipball\/[^/]+|\/git\/(?:trees|blobs)\/[^/]+|\/pulls\/\d+|\/issues\/\d+\/comments)$/.test(suffix);
+        const readable = /^(?:\/?|\/branches(?:\/[^/]+)?|\/commits(?:\/[^/]+)?|\/contents(?:\/[^/]+)*|\/readme|\/pages|\/zipball\/[^/]+|\/git\/(?:trees|blobs)\/[^/]+|\/pulls\/\d+|\/issues\/\d+\/comments)$/.test(suffix);
         if (!readable || (options.method !== "GET" && !(options.method === "HEAD" && suffix.startsWith("/zipball/"))) || url.origin !== "https://api.github.com" ||
             (path !== prefix && !path.startsWith(prefix + "/"))) {
           throw new AnonymousError("github_app_access_required", { httpStatus: 403 });

src/core/Repository.ts

@@ -8,6 +8,7 @@ import { RepositoryStatus } from "./types";
 import { Readable } from "stream";
 import * as sha1 from "crypto-js/sha1";
 import User from "./User";
+import { SubmoduleMount } from "./source/github-submodules";
 import GitHubStream from "./source/GitHubStream";
 import Zip from "./source/Zip";
 import {
@@ -43,14 +44,14 @@ async function buildPathIndex(repoId: string, key: string, terms: string[], gene
   const task = (async () => {
     const compiled = compileTerms(terms);
     const cursor = FileModel.find({ repoId, treeGeneration: generation || { $exists: false } })
-      .select("name path sha size").sort({ path: 1, name: 1 }).lean().cursor();
+      .select("name path sha size upstream").sort({ path: 1, name: 1 }).lean().cursor();
     let batch: Parameters<typeof AnonymizedPathModel.bulkWrite>[0] = [];
     try {
       for await (const file of cursor) {
         const name = anonymizePathCompiled(file.name, compiled);
         const directory = anonymizePathCompiled(file.path, compiled);
         batch.push({ updateOne: { filter: { repoId, key, originalId: file._id }, update: { $set: {
-          repoId, key, originalId: file._id, name: file.name, path: file.path, sha: file.sha, size: file.size,
+          repoId, key, originalId: file._id, name: file.name, path: file.path, sha: file.sha, size: file.size, upstream: file.upstream,
           anonymousName: name, anonymousDirectory: directory, createdAt: new Date(),
           searchName: name.toLowerCase(), searchDirectory: directory.toLowerCase(),
           anonymousPath: anonymizePathCompiled(file.path ? `${file.path}/${file.name}` : file.name, compiled),
@@ -202,6 +203,17 @@ export default class Repository {
     return await FileModel.find(query).exec();
   }
 
+  async submoduleMounts(): Promise<SubmoduleMount[]> {
+    if (this.model.source.type === "Zip") return [];
+    await this.ensureFileTree();
+    const mounts = await FileModel.find({ repoId: this.repoId,
+      treeGeneration: this.model.treeGeneration || { $exists: false }, submodule: "included",
+    }).select("name path upstream").lean().exec();
+    return mounts.filter(file => file.upstream).map(file => ({
+      ...file.upstream!, path: [file.path, file.name].filter(Boolean).join("/"),
+    }));
+  }
+
   private async ensureFileTree(opt: { force?: boolean; progress?: (status: string) => void } = {}) {
     this.assertNotArchived();
     let hasFile = this.hasEmptyTree() || await FileModel.exists({ repoId: this.repoId, treeGeneration: this.model.treeGeneration || { $exists: false } }).exec();
@@ -219,6 +231,9 @@ export default class Repository {
         hasFile = null;
       }
     }
+    if (hasFile && this.source instanceof GitHubStream && await FileModel.exists({
+      repoId: this.repoId, treeGeneration: this.model.treeGeneration || { $exists: false }, name: ".gitmodules",
+    }).exec()) hasFile = null;
     if (!hasFile || opt.force) {
       const previousGeneration = this.model.treeGeneration;
       const previousPrefix = contentGenerationPrefix(`${previousGeneration || "legacy"}:${this.model.anonymizeDate?.toISOString() || ""}`);

src/core/model/anonymized-path.ts

@@ -1,7 +1,9 @@
+import { UpstreamFileSchema } from "./files/files.schema";
 import { IFile } from "./files/files.types";
 import { model, Schema } from "mongoose";
 const schema = new Schema({ repoId: { type: String, required: true }, key: { type: String, required: true }, originalId: Schema.Types.ObjectId,
   name: { type: String, required: true }, path: { type: String, default: "", required: true }, sha: String, size: Number,
+  upstream: { type: UpstreamFileSchema, default: undefined },
   anonymousName: String, anonymousDirectory: String, anonymousPath: String,
   searchName: String, searchDirectory: String,
   createdAt: { type: Date, default: Date.now } });

src/core/model/files/files.schema.ts

@@ -1,5 +1,7 @@
 import { Schema } from "mongoose";
 
+export const UpstreamFileSchema = new Schema({ organization: String, repoName: String, commit: String, path: String }, { _id: false });
+
 const FileSchema = new Schema({
   name: { type: String, index: true },
   path: { type: String, index: true },
@@ -7,6 +9,8 @@ const FileSchema = new Schema({
   treeGeneration: String,
   treeStaged: Boolean,
   metadataPending: Boolean,
+  upstream: { type: UpstreamFileSchema, default: undefined },
+  submodule: { type: String, enum: ["included", "unsupported", "unavailable", "limit", "nested"] },
   sha: {
     type: String,
   },

src/core/model/files/files.types.ts

@@ -1,6 +1,16 @@
 import { Document, Model } from "mongoose";
 
+export interface UpstreamFile {
+  organization: string;
+  repoName: string;
+  commit: string;
+  path: string;
+}
+
 export interface IFile {
+  /** Internal source metadata, never returned by the public tree API. */
+  upstream?: UpstreamFile;
+  submodule?: "included" | "unsupported" | "unavailable" | "limit" | "nested";
   name: string;
   path: string;
   repoId: string;

src/core/source/GitHubStream.ts

@@ -18,7 +18,8 @@ import * as stream from "stream";
 import AnonymousError from "../AnonymousError";
 import { octokit, waitForTokenGate, RateLimitDelayError } from "../GitHubUtils";
 import FileModel from "../model/files/files.model";
-import { IFile } from "../model/files/files.types";
+import { expandSubmodules, validUpstream } from "./github-submodules";
+import { IFile, UpstreamFile } from "../model/files/files.types";
 import { createLogger, serializeError } from "../logger";
 import { measureStage, recordCacheHit } from "../request-monitoring";
 import config from "../../config";
@@ -322,16 +323,27 @@ export default class GitHubStream extends GitHubBase {
     filePath: string,
     repoId: string,
     fileMeta: () =>
-      | Promise<{ sha: string; size?: number }>
+      | Promise<{ sha: string; size?: number; upstream?: UpstreamFile }>
-      | { sha: string; size?: number }
+      | { sha: string; size?: number; upstream?: UpstreamFile }
       | Promise<string>
       | string
   ) {
     const meta = await fileMeta();
-    const expected: { sha: string; size?: number } =
+    const expected: { sha: string; size?: number; upstream?: UpstreamFile } =
       typeof meta === "string" ? { sha: meta } : meta;
+    if (expected.upstream && !validUpstream(expected.upstream)) throw new AnonymousError("invalid_request", { httpStatus: 400 });
+    // Hydrated Mongoose subdocuments expose source fields through getters,
+    // so spreading them would retain the parent's repository coordinates.
+    const origin = expected.upstream ? new GitHubStream({
+      ...this.data,
+      organization: expected.upstream.organization,
+      repoName: expected.upstream.repoName,
+      commit: expected.upstream.commit,
+      getToken: () => "",
+    }) : this;
+    const sourcePath = expected.upstream?.path || filePath;
     const generation = createHash("sha256").update(JSON.stringify([
-      this.data.commit, expected.sha, this.data.cacheGeneration, this.data.cacheRevision,
+      this.data.commit, expected.sha, this.data.cacheGeneration, this.data.cacheRevision, ...(expected.upstream ? [expected.upstream] : []),
     ])).digest("hex");
     const cachePath = this.data.cacheGeneration ? `${contentGenerationPrefix(this.data.cacheGeneration)}/${generation}/${filePath}` : filePath;
     const assertActive = async () => {
@@ -383,8 +395,9 @@ export default class GitHubStream extends GitHubBase {
       if (cacheFills.size >= 64) throw new AnonymousError("cache_busy", { httpStatus: 503 });
       filling = coordinatedFill(key, cached, async () => {
         await assertActive();
-        const token = await this.data.getToken();
+        const parentToken = await this.data.getToken();
-        const sourceMiss = cachedContentMiss(this.data, token);
+        const token = expected.upstream ? "" : parentToken;
+        const sourceMiss = cachedContentMiss(this.data, parentToken);
         if (sourceMiss && sourceMiss !== "file_not_found") throw new AnonymousError(sourceMiss, { httpStatus: 404 });
         const missKey = createHash("sha256").update(JSON.stringify([
           key, this.data.organization, this.data.repoName, token,
@@ -393,7 +406,7 @@ export default class GitHubStream extends GitHubBase {
         if (missed) throw new AnonymousError(missed, { httpStatus: 404 });
         try {
           const fetchContent = () => measureStage("upstream", async () => {
-            const content = await this.downloadWithFallback(token, expected.sha, filePath)
+            const content = await origin.downloadWithFallback(token, expected.sha, sourcePath)
               .catch(error => { throw downloadError(error, filePath); });
             let count = 0;
             const limited = new stream.Transform({ transform(chunk, encoding, callback) {
@@ -413,9 +426,14 @@ export default class GitHubStream extends GitHubBase {
             finally { clearTimeout(deadline); limited.destroy(); content.destroy(); }
             return true;
           });
-          return await this.confirmOwnerAccess(repoId, fetchContent);
+          return expected.upstream ? await fetchContent() : await this.confirmOwnerAccess(repoId, fetchContent);
         } catch (error) {
-          if (error instanceof AnonymousError && error.message === "file_not_found" && error.httpStatus === 404) {
+          if (expected.upstream && error instanceof AnonymousError &&
+              ["file_not_found", "file_not_accessible"].includes(error.message)) {
+            // A removed public dependency is not a lost parent credential.
+            throw new AnonymousError("submodule_not_available", { httpStatus: 404, cause: error });
+          }
+          if (!expected.upstream && error instanceof AnonymousError && error.message === "file_not_found" && error.httpStatus === 404) {
             const code = await classifyContentMiss(this.data, token);
             missingContent.set(missKey, code, 30_000);
             if (code !== "file_not_found") throw new AnonymousError(code, { httpStatus: 404, cause: error });
@@ -448,14 +466,14 @@ export default class GitHubStream extends GitHubBase {
             object: file,
           });
         }
-        return { sha: fileSha, size: await file.size() };
+        return { sha: fileSha, size: await file.size(), upstream: (await file.getFileInfo()).upstream };
       }
     );
   }
 
   async getFiles(progress?: (status: string) => void) {
     this._truncatedFolders = [];
-    return this.getTruncatedTree(this.data.commit, progress);
+    return expandSubmodules(await this.getTruncatedTree(this.data.commit, progress), this.data);
   }
 
   /**
@@ -485,6 +503,7 @@ export default class GitHubStream extends GitHubBase {
   }
 
   private async fetchFileMetadata(filePath: string): Promise<IFile | null> {
+    if (basename(filePath) === ".gitmodules") return null;
     const token = await this.data.getToken();
     const oct = octokit(token);
     try {
@@ -665,6 +684,7 @@ export default class GitHubStream extends GitHubBase {
         path: pathFile,
         repoId: this.data.repoId,
         size: elem.size,
+        ...(elem.type === "commit" ? { submodule: "unsupported" } : {}),
         sha: elem.sha,
       });
     });

src/core/source/github-submodules.ts

@@ -0,0 +1,99 @@
+import { octokit, waitForTokenGate } from "../GitHubUtils";
+import AnonymousError from "../AnonymousError";
+import { IFile, UpstreamFile } from "../model/files/files.types";
+import { GitHubBaseData } from "./GitHubBase";
+
+export const MAX_SUBMODULES = 20;
+export const MAX_SUBMODULE_ENTRIES = 10000;
+export interface SubmoduleMount extends Omit<UpstreamFile, "path"> { path: string; }
+
+export function safeRepositoryPath(value: unknown): value is string {
+  return typeof value === "string" && value.length > 0 &&
+    !value.includes("\\") && !value.includes("\0") &&
+    value.split("/").every(part => part !== "" && part !== "." && part !== "..");
+}
+
+export function validUpstream(value: unknown): value is UpstreamFile {
+  if (!value || typeof value !== "object") return false;
+  const v = value as UpstreamFile;
+  return typeof v.organization === "string" && /^[a-zA-Z0-9-]+$/.test(v.organization) &&
+    typeof v.repoName === "string" && /^[a-zA-Z0-9_.-]+$/.test(v.repoName) && ![".", ".."].includes(v.repoName) &&
+    typeof v.commit === "string" && /^[a-f0-9]{40}$/.test(v.commit) && safeRepositoryPath(v.path);
+}
+
+export function submoduleRepository(url: unknown, parent: GitHubBaseData): { organization: string; repoName: string } | null {
+  if (typeof url !== "string") return null;
+  const normalized = url.replace(/^git@github\.com:/, "https://github.com/").replace(/^ssh:\/\/git@github\.com\//, "https://github.com/").replace(/^git:\/\/github\.com\//, "https://github.com/");
+  try {
+    const parsed = new URL(normalized, url.startsWith("./") || url.startsWith("../")
+      ? `https://github.com/${parent.organization}/${parent.repoName}/` : undefined);
+    if (parsed.protocol !== "https:" || parsed.hostname !== "github.com" || parsed.port || parsed.username || parsed.password || parsed.search || parsed.hash) return null;
+    const parts = parsed.pathname.replace(/\.git\/?$/, "").replace(/\/$/, "").split("/");
+    if (parts.length !== 3) return null;
+    const candidate = { organization: parts[1], repoName: parts[2], commit: "a".repeat(40), path: "file" };
+    return validUpstream(candidate) ? { organization: candidate.organization, repoName: candidate.repoName } : null;
+  } catch { return null; }
+}
+
+/** Resolve only direct public GitHub dependencies. Never send the parent token to a child. */
+export async function expandSubmodules(files: IFile[], data: GitHubBaseData): Promise<IFile[]> {
+  const output = files.filter(file => file.name !== ".gitmodules");
+  const modules = output.filter(file => file.submodule);
+  let remaining = MAX_SUBMODULE_ENTRIES;
+  for (const [index, mount] of modules.entries()) {
+    if (index >= MAX_SUBMODULES) { mount.submodule = "limit"; continue; }
+    const mountPath = [mount.path, mount.name].filter(Boolean).join("/");
+    if (!safeRepositoryPath(mountPath) || !mount.sha || !/^[a-f0-9]{40}$/.test(mount.sha)) continue;
+    const token = await data.getToken();
+    await waitForTokenGate(token);
+    try {
+      const response = await octokit(token).repos.getContent({
+        owner: data.organization, repo: data.repoName, path: mountPath, ref: data.commit,
+      });
+      const metadata = response.data as { submodule_git_url?: string; sha?: string };
+      const repository = submoduleRepository(metadata.submodule_git_url, data);
+      if (!repository || metadata.sha !== mount.sha) continue;
+      // An unauthenticated request proves public access, even for an owner whose
+      // parent token can also read private dependencies. No credentials cross repos.
+      await waitForTokenGate("");
+      const child = await octokit("").git.getTree({
+        owner: repository.organization, repo: repository.repoName, tree_sha: mount.sha, recursive: "1",
+      });
+      if (child.data.truncated || child.data.tree.length > remaining) { mount.submodule = "limit"; continue; }
+      const entries = child.data.tree;
+      if (entries.some(entry => !safeRepositoryPath(entry.path))) { mount.submodule = "unsupported"; continue; }
+      remaining -= entries.length;
+      mount.submodule = "included";
+      mount.upstream = { ...repository, commit: mount.sha, path: "" };
+      for (const entry of entries) {
+        const relative = entry.path!;
+        if (relative.split("/").pop() === ".gitmodules") continue;
+        const fullPath = `${mountPath}/${relative}`;
+        const slash = fullPath.lastIndexOf("/");
+        output.push({
+          repoId: data.repoId, name: fullPath.slice(slash + 1), path: fullPath.slice(0, slash),
+          sha: entry.sha, size: entry.size,
+          ...(entry.type === "commit" ? { submodule: "nested" as const } : {}),
+          ...(entry.type === "blob" ? { upstream: { ...repository, commit: mount.sha, path: relative } } : {}),
+        });
+      }
+    } catch (error) {
+      if ((error as { status?: number }).status === 404) { mount.submodule = "unavailable"; continue; }
+      // Do not persist a partial tree on rate limits, outages or credential errors.
+      throw error;
+    }
+  }
+  return output;
+}
+
+export function validateMounts(value: unknown): SubmoduleMount[] {
+  if (value === undefined) return [];
+  if (!Array.isArray(value) || value.length > MAX_SUBMODULES || value.some(mount => !validUpstream(mount))) {
+    throw new AnonymousError("invalid_request", { httpStatus: 400 });
+  }
+  const paths = value.map(mount => mount.path as string);
+  if (paths.some((path, i) => paths.some((other, j) => i !== j && (path === other || path.startsWith(other + "/"))))) {
+    throw new AnonymousError("invalid_request", { httpStatus: 400 });
+  }
+  return value;
+}

src/core/zipStream.ts

@@ -1,4 +1,6 @@
 import got from "got";
+import config from "../config";
+import { MAX_SUBMODULE_ENTRIES, SubmoduleMount, safeRepositoryPath, validateMounts } from "./source/github-submodules";
 import { Readable, Transform, Writable } from "stream";
 import { Parse } from "unzip-stream";
 import archiver = require("archiver");
@@ -19,6 +21,7 @@ const logger = createLogger("zip-stream");
 
 export interface StreamAnonymizedZipOptions {
   repoId: string;
+  submodules?: SubmoduleMount[];
   organization: string;
   repoName: string;
   commit: string;
@@ -78,6 +81,7 @@ export async function streamAnonymizedZip(
     on(event: string, listener: (...args: unknown[]) => void): unknown;
   }
 ): Promise<void> {
+  const submodules = validateMounts(opt.submodules);
   let response;
   try {
     const token = await opt.getToken();
@@ -113,14 +117,10 @@ export async function streamAnonymizedZip(
       },
     });
   }
-  const upstreamDone = startStage("upstream");
+  let downloadStream: Readable;
-  const downloadStream = got.stream(response.url);
-  downloadStream.once("end", () => upstreamDone());
-  downloadStream.once("error", () => upstreamDone(true));
-  downloadStream.once("close", () => upstreamDone(!downloadStream.readableEnded));
-
   const archive = archiver("zip", {});
-  const parser = Parse() as Transform;
+  let parser: Transform;
+  let childEntries = 0;
   const activeStreams = new Set<Readable>();
   const compiledTerms = compileTerms(opt.anonymizerOptions.terms || []);
   let stopped = false;
@@ -160,8 +160,8 @@ export async function streamAnonymizedZip(
     stopped = true;
     cleanupDrain?.();
     cleanupDrain = undefined;
-    downloadStream.destroy();
+    downloadStream?.destroy();
-    parser.destroy();
+    parser?.destroy();
     for (const stream of activeStreams) stream.destroy();
     activeStreams.clear();
     output.destroy();
@@ -187,53 +187,82 @@ export async function streamAnonymizedZip(
   archive.on("error", fail);
   archive.pipe(output);
 
-  downloadStream
+  const startArchive = (url: string, prefix: string, next: number) => {
-    .on("error", fail)
+    if (stopped) return;
-    .pipe(parser)
+    const upstreamDone = startStage("upstream");
-    .on("entry", (entry: Readable & { type: string; path: string; autodrain: () => void }) => {
+    const input = got.stream(url);
-      if (stopped) {
+    downloadStream = input;
-        entry.destroy();
+    input.once("end", () => upstreamDone());
-        return;
+    input.once("error", () => upstreamDone(true));
-      }
+    input.once("close", () => upstreamDone(!input.readableEnded));
-      entry.on("error", fail);
+    parser = Parse() as Transform;
-      activeStreams.add(entry);
+    downloadStream
-      entry.once("close", () => activeStreams.delete(entry));
+      .on("error", fail)
-      if (entry.type === "File") {
+      .pipe(parser)
-        try {
+      .on("entry", (entry: Readable & { type: string; path: string; autodrain: () => void }) => {
-          const fileName = anonymizePathCompiled(
+        if (stopped) {
-            entry.path.substring(entry.path.indexOf("/") + 1),
+          entry.destroy();
-            compiledTerms
+          return;
-          );
-          if (!isEntryAllowed(entry.path, opt.contentOptions)) {
-            entry.autodrain();
-            return;
-          }
-          // Pass filePath via the constructor — AnonymizeTransformer reads it
-          // there to decide whether the entry is text (and therefore should be
-          // anonymized) vs binary (passthrough). Assigning afterwards leaves
-          // isText=false for every file, so the zip ships unanonymized.
-          parser.pause();
-          const onArchived = () => { entryWaiting = true; setImmediate(resumeParser); };
-          archive.once("entry", onArchived);
-          const anonymizer = new AnonymizeTransformer({
-            ...opt.anonymizerOptions,
-            filePath: entry.path,
-          });
-          anonymizer.on("error", fail);
-          activeStreams.add(anonymizer);
-          anonymizer.once("close", () => activeStreams.delete(anonymizer));
-          const st = entry.pipe(anonymizer);
-          archive.append(st, zipEntryOptions(fileName, entry.path));
-        } catch (error) {
-          fail(error as Error);
         }
-      } else {
+        entry.on("error", fail);
-        entry.autodrain();
+        activeStreams.add(entry);
-      }
+        entry.once("close", () => activeStreams.delete(entry));
-    })
+        if (entry.type === "File") {
-    .on("error", fail)
+          try {
-    .on("end", () => {
+            const relativePath = entry.path.substring(entry.path.indexOf("/") + 1);
-      if (stopped) return;
+            if (!safeRepositoryPath(relativePath)) throw new AnonymousError("invalid_file_path", { httpStatus: 502 });
-      void archive.finalize().catch(fail);
+            if (relativePath.split("/").pop() === ".gitmodules") { entry.autodrain(); return; }
-    });
+            if (prefix && ++childEntries > MAX_SUBMODULE_ENTRIES) throw new AnonymousError("file_too_big", { httpStatus: 413 });
+            const mountedPath = prefix ? `${prefix}/${relativePath}` : relativePath;
+            const fileName = anonymizePathCompiled(mountedPath, compiledTerms);
+            if (!isEntryAllowed(entry.path, opt.contentOptions)) {
+              entry.autodrain();
+              return;
+            }
+            // Pass filePath via the constructor — AnonymizeTransformer reads it
+            // there to decide whether the entry is text (and therefore should be
+            // anonymized) vs binary (passthrough). Assigning afterwards leaves
+            // isText=false for every file, so the zip ships unanonymized.
+            parser.pause();
+            const onArchived = () => { entryWaiting = true; setImmediate(resumeParser); };
+            archive.once("entry", onArchived);
+            const anonymizer = new AnonymizeTransformer({
+              ...opt.anonymizerOptions,
+              filePath: mountedPath,
+            });
+            anonymizer.on("error", fail);
+            activeStreams.add(anonymizer);
+            anonymizer.once("close", () => activeStreams.delete(anonymizer));
+            let source: Readable = entry;
+            if (prefix) {
+              let bytes = 0;
+              const limited = new Transform({ transform(chunk, encoding, callback) {
+                bytes += chunk.length;
+                callback(bytes > config.MAX_FILE_SIZE ? new AnonymousError("file_too_big", { httpStatus: 413 }) : null, chunk);
+              } });
+              limited.on("error", fail);
+              activeStreams.add(limited);
+              limited.once("close", () => activeStreams.delete(limited));
+              source = entry.pipe(limited);
+            }
+            const st = source.pipe(anonymizer);
+            archive.append(st, zipEntryOptions(fileName, mountedPath));
+          } catch (error) {
+            fail(error as Error);
+          }
+        } else {
+          entry.autodrain();
+        }
+      })
+      .on("error", fail)
+      .on("end", () => {
+        if (stopped) return;
+        const mount = submodules[next];
+        if (mount) {
+          // Public child archives never receive parent credentials.
+          startArchive(`https://codeload.github.com/${encodeURIComponent(mount.organization)}/${encodeURIComponent(mount.repoName)}/zip/${mount.commit}`, mount.path, next + 1);
+        } else void archive.finalize().catch(fail);
+      });
+  };
+  startArchive(response.url, "", 0);
 }

src/server/routes/repository-public.ts

@@ -57,6 +57,7 @@ router.get(
       res.attachment(`${repo.repoId}.zip`);
       if (req.method === "HEAD") { res.end(); return; }
       await repo.countView();
+      const submodules = await repo.submoduleMounts();
 
       if (config.STREAMER_ENTRYPOINT) {
         // use the streamer service
@@ -75,6 +76,7 @@ router.get(
               branch: repo.model.source.branch,
               repoId: repo.repoId,
               anonymizerOptions: anonymizer.opt,
+              submodules,
               contentOptions: {
                 image: repo.options.image,
                 pdf: repo.options.pdf,
@@ -169,6 +171,7 @@ router.get(
           commit: repo.model.source.commit || "HEAD",
           getToken: () => repo.getToken(),
           anonymizerOptions: anonymizer.opt,
+          submodules,
           contentOptions: {
             image: repo.options.image,
             pdf: repo.options.pdf,
@@ -383,15 +386,15 @@ router.get(
         truncatedFolders: (repo.model.truncatedFolders || []).map((path) =>
           anonymizePath(path, repo.options.terms || [])
         ),
-        // Submodule contents are not included in GitHub archives/trees, so
+        // Legacy trees retain .gitmodules until their next tree refresh.
-        // they end up as empty folders in the anonymized repository. Surface
-        // a warning in the explorer when the repository uses submodules (#737).
         hasSubmodules:
           (await FileModel.exists({
             repoId: repo.repoId,
             treeGeneration: repo.model.treeGeneration || { $exists: false },
-            name: ".gitmodules",
+            $or: [
-            path: "",
+              { submodule: { $exists: true, $ne: "included" } },
+              { name: ".gitmodules", path: "" },
+            ],
           })) != null,
       });
     } catch (error) {

src/streamer/route.ts

@@ -34,6 +34,7 @@ router.post(
           getToken: () => token,
           anonymizerOptions: { ...anonymizerOptions, repoId },
           contentOptions,
+          submodules: req.body.submodules,
         },
         res
       );
@@ -89,7 +90,7 @@ router.post("/", async (req: express.Request, res: express.Response) => {
     const content = await source.getFileContentCache(
       filePath,
       repoId,
-      () => ({ sha: fileSha, size: fileSize })
+      () => ({ sha: fileSha, size: fileSize, upstream: req.body.upstream })
     );
     if (res.destroyed || res.writableEnded) { content.destroy(); anonymizer.destroy(); return; }
     const mime = lookup(filePath);

test/github-submodules.test.js

@@ -0,0 +1,308 @@
+const { expect } = require("chai");
+const { Readable, PassThrough } = require("node:stream");
+const { once } = require("node:events");
+const { setImmediate } = require("node:timers");
+const archiver = require("archiver");
+const got = require("got");
+require("ts-node/register/transpile-only");
+const utils = require("../src/core/GitHubUtils");
+const { expandSubmodules, submoduleRepository, validateMounts, MAX_SUBMODULES, MAX_SUBMODULE_ENTRIES } = require("../src/core/source/github-submodules");
+const GitHubStream = require("../src/core/source/GitHubStream").default;
+const FileModel = require("../src/core/model/files/files.model").default;
+const storage = require("../src/core/storage").default;
+const { streamAnonymizedZip } = require("../src/core/zipStream");
+const config = require("../src/config").default;
+const sha = "a".repeat(40), blob = "b".repeat(40);
+const parent = { repoId: "submodule-test", organization: "parent", repoName: "repo", commit: "c".repeat(40), getToken: () => "parent-private-token" };
+const upstream = { organization: "child", repoName: "public", commit: sha, path: "src/Alice.txt" };
+const mount = () => new FileModel({ repoId: parent.repoId, name: "dependency", path: "vendor", sha, submodule: "unsupported" });
+
+async function zip(entries) {
+  const archive = archiver("zip"); const chunks = [];
+  archive.on("data", chunk => chunks.push(chunk)); const done = once(archive, "end");
+  for (const [name, content] of Object.entries(entries)) archive.append(content, { name });
+  await archive.finalize(); await done; return Buffer.concat(chunks);
+}
+
+async function readZip(options) {
+  const response = new PassThrough(); const parser = require("unzip-stream").Parse();
+  const entries = [], files = {};
+  parser.on("entry", entry => {
+    const chunks = []; entry.on("data", chunk => chunks.push(chunk));
+    entries.push(once(entry, "end").then(() => { files[entry.path] = Buffer.concat(chunks).toString(); }));
+  });
+  response.pipe(parser); const done = once(parser, "finish");
+  await streamAnonymizedZip(options, response); await done; await Promise.all(entries);
+  return files;
+}
+
+const zipOptions = { ...parent, getToken: () => "", anonymizerOptions: { terms: ["Alice"], image: true, link: true },
+  submodules: [{ organization: "child", repoName: "public", commit: sha, path: "vendor/dependency" }] };
+
+describe("public GitHub submodules", function () {
+  this.timeout(10000);
+  const restore = [];
+  function stub(object, key, value) { const previous = object[key]; object[key] = value; restore.push(() => { object[key] = previous; }); }
+  afterEach(() => restore.splice(0).reverse().forEach(fn => fn()));
+  beforeEach(() => stub(utils, "waitForTokenGate", async () => {}));
+
+  function api(tree = [{ path: "src/Alice.txt", type: "blob", sha: blob, size: 5 }], options = {}) {
+    const calls = [];
+    stub(utils, "octokit", token => ({
+      repos: { getContent: async args => {
+        calls.push({ method: "contents", token, ...args });
+        return { data: { submodule_git_url: options.url || "https://github.com/child/public.git", sha } };
+      } },
+      git: { getTree: async args => {
+        calls.push({ method: "tree", token, ...args });
+        if (options.error) throw options.error;
+        return { data: { tree, truncated: options.truncated || false } };
+      } },
+    }));
+    return calls;
+  }
+
+  it("mounts a pinned child tree without forwarding parent credentials or exposing .gitmodules", async () => {
+    const calls = api([
+      { path: ".gitmodules", type: "blob", sha: blob, size: 100 },
+      { path: "src", type: "tree", sha: blob },
+      { path: "src/Alice.txt", type: "blob", sha: blob, size: 5 },
+      { path: "nested", type: "commit", sha },
+    ]);
+    const files = await expandSubmodules([mount(), { name: ".gitmodules", path: "" }], parent);
+    expect(files.map(file => file.name)).to.deep.equal(["dependency", "src", "Alice.txt", "nested"]);
+    expect(files[0].submodule).to.equal("included");
+    expect(files[2]).to.include({ path: "vendor/dependency/src", sha: blob, size: 5 });
+    expect(files[2].upstream).to.deep.equal(upstream);
+    expect(files[3].submodule).to.equal("nested");
+    expect(calls).to.deep.equal([
+      { method: "contents", token: "parent-private-token", owner: "parent", repo: "repo", path: "vendor/dependency", ref: parent.commit },
+      { method: "tree", token: "", owner: "child", repo: "public", tree_sha: sha, recursive: "1" },
+    ]);
+    expect(new FileModel(files[2]).toObject().upstream).to.deep.equal(upstream);
+  });
+
+  it("discovers submodules through scoped public-read contexts without authorizing child or write requests", async () => {
+    const { registerGitHubToken } = require("../src/core/github-token-context");
+    const token = "public-read:submodule-discovery";
+    registerGitHubToken(token, { publicRepository: "parent/repo", quotaKey: "submodule-test",
+      renew: async () => "parent-user-token" });
+    const calls = [];
+    stub(globalThis, "fetch", async (url, options) => {
+      calls.push({ url: String(url), authorization: new globalThis.Headers(options.headers).get("authorization") });
+      const result = String(url).includes("/contents/")
+        ? { submodule_git_url: "https://github.com/child/public.git", sha }
+        : { tree: [{ path: "src/Alice.txt", type: "blob", sha: blob, size: 5 }], truncated: false };
+      return new globalThis.Response(JSON.stringify(result), { headers: { "content-type": "application/json" } });
+    });
+    const files = await expandSubmodules([mount()], { ...parent, getToken: () => token });
+    expect(files[0].submodule).to.equal("included");
+    expect(calls).to.deep.equal([
+      { url: `https://api.github.com/repos/parent/repo/contents/vendor%2Fdependency?ref=${parent.commit}`, authorization: "token parent-user-token" },
+      { url: `https://api.github.com/repos/child/public/git/trees/${sha}?recursive=1`, authorization: null },
+    ]);
+    for (const request of [
+      "GET /repos/child/public/contents/file.txt", "GET /repos/parent/repo-other/contents/file.txt",
+      "PUT /repos/parent/repo/contents/file.txt", "DELETE /repos/parent/repo/contents/file.txt",
+    ]) {
+      let caught; try { await utils.octokit(token).request(request); } catch (error) { caught = error; }
+      expect(caught?.message).to.equal("github_app_access_required");
+    }
+    expect(calls).to.have.length(2);
+  });
+
+  it("recognizes gitlinks from the parent tree and leaves ordinary repositories at one request", async () => {
+    const source = new GitHubStream(parent); let calls = 0;
+    source.getGHTree = async () => { calls++; return { tree: [{ path: "README.md", type: "blob", sha: blob, size: 5 }] }; };
+    expect((await source.getFiles()).map(file => file.name)).to.deep.equal(["README.md"]); expect(calls).to.equal(1);
+    api();
+    source.getGHTree = async () => ({ tree: [{ path: "vendor/dependency", type: "commit", mode: "160000", sha }] });
+    expect((await source.getFiles())[0].submodule).to.equal("included");
+  });
+
+  it("supports relative and SSH GitHub URLs and rejects other hosts and unsafe paths", () => {
+    for (const url of ["../public.git", "git@github.com:parent/public.git", "ssh://git@github.com/parent/public.git", "https://github.com/parent/public.git"]) {
+      expect(submoduleRepository(url, parent)).to.deep.equal({ organization: "parent", repoName: "public" });
+    }
+    for (const url of ["https://evil.test/child/repo", "file:///tmp/repo", "https://github.com@evil.test/a/b", "https://github.com/a/b?token=secret", "https://github.com/a%2fb/c"]) expect(submoduleRepository(url, parent)).to.equal(null);
+    for (const path of ["../escape", "/absolute", "a/../b", "a\\b", "a//b", "a\0b"]) expect(() => validateMounts([{ ...upstream, path }])).to.throw("invalid_request");
+    expect(() => validateMounts([{ ...upstream, path: "lib" }, { ...upstream, path: "lib/nested" }])).to.throw("invalid_request");
+  });
+
+  it("leaves private, missing and external children unsupported without blocking parent files", async () => {
+    api([], { error: Object.assign(new Error("not found"), { status: 404 }) });
+    const result = await expandSubmodules([mount(), { name: "README.md", path: "", size: 3 }], parent);
+    expect(result[0].submodule).to.equal("unavailable"); expect(result[1].name).to.equal("README.md");
+    const calls = api([], { url: "https://gitlab.com/child/repo.git" });
+    expect((await expandSubmodules([mount()], parent))[0].submodule).to.equal("unsupported");
+    expect(calls).to.have.length(1);
+  });
+
+  it("does not persist transient failures as permanently missing dependencies", async () => {
+    const error = Object.assign(new Error("rate limited"), { status: 403 }); api([], { error });
+    let caught; try { await expandSubmodules([mount()], parent); } catch (err) { caught = err; }
+    expect(caught).to.equal(error);
+  });
+
+  it("bounds module count and rejects oversized or truncated child trees", async () => {
+    const calls = api();
+    const mounts = Array.from({ length: MAX_SUBMODULES + 1 }, (_, i) => ({ ...mount().toObject(), name: `dependency${i}` }));
+    const result = await expandSubmodules(mounts, parent);
+    expect(result[MAX_SUBMODULES].submodule).to.equal("limit"); expect(calls).to.have.length(MAX_SUBMODULES * 2);
+    api(Array.from({ length: MAX_SUBMODULE_ENTRIES + 1 }, (_, i) => ({ path: `file${i}`, type: "blob", sha: blob, size: 1 })));
+    expect((await expandSubmodules([mount()], parent))[0].submodule).to.equal("limit");
+    api([], { truncated: true }); expect((await expandSubmodules([mount()], parent))[0].submodule).to.equal("limit");
+  });
+
+  for (const hydrated of [false, true]) it(`fetches child content from ${hydrated ? "hydrated" : "plain"} metadata and caches under its parent mount`, async () => {
+    const fileOrigin = hydrated ? new FileModel({ upstream }).upstream : upstream;
+    const source = new GitHubStream({ ...parent, cacheGeneration: "tree:date" }); const cache = new Map(), requests = [];
+    stub(storage, "fileInfo", async (_id, path) => { if (!cache.has(path)) throw Object.assign(Error("missing"), { code: "ENOENT" }); return { size: cache.get(path).length }; });
+    stub(storage, "write", async (id, path, input) => { expect(id).to.equal(parent.repoId); const chunks = []; for await (const chunk of input) chunks.push(Buffer.from(chunk)); cache.set(path, Buffer.concat(chunks)); });
+    stub(storage, "read", async (_id, path) => Readable.from([cache.get(path)]));
+    stub(got, "stream", (url, options) => { requests.push({ url, options }); return Readable.from([Buffer.from("Alice")]); });
+    for (let i = 0; i < 2; i++) {
+      const stream = await source.getFileContentCache("vendor/dependency/src/Alice.txt", parent.repoId, () => ({ sha: blob, size: 5, upstream: fileOrigin }));
+      for await (const chunk of stream) expect(chunk.toString()).to.equal("Alice");
+    }
+    expect(requests).to.have.length(1);
+    expect(requests[0].url).to.equal(`https://github.com/child/public/raw/${sha}/src/Alice.txt`);
+    expect(requests[0].options.headers).to.deep.equal({});
+    expect([...cache.keys()][0]).to.match(/\/vendor\/dependency\/src\/Alice\.txt$/);
+    const changed = { ...upstream, commit: "d".repeat(40) };
+    const stream = await source.getFileContentCache("vendor/dependency/src/Alice.txt", parent.repoId, () => ({ sha: blob, size: 5, upstream: changed }));
+    stream.resume(); expect(requests).to.have.length(2);
+  });
+
+  it("does not report an unavailable public child as lost parent access", async () => {
+    const { isAccessFailure } = require("../src/core/owner-notifications");
+    const source = new GitHubStream(parent);
+    stub(storage, "fileInfo", async () => { throw Object.assign(Error("missing"), { code: "ENOENT" }); });
+    stub(storage, "write", async (_id, _path, input) => { for await (const _chunk of input) { /* consume */ } });
+    stub(got, "stream", () => Readable.from((async function* () {
+      yield* [];
+      throw Object.assign(Error("not found"), { response: { statusCode: 404 } });
+    })()));
+    let error;
+    try { await source.getFileContentCache("vendor/lib/file.txt", parent.repoId, () => ({ sha: blob, size: 5, upstream })); }
+    catch (caught) { error = caught; }
+    expect(error?.message).to.equal("submodule_not_available");
+    expect(isAccessFailure(error)).to.equal(false);
+  });
+
+  it("includes pinned child archives with anonymized names and content, omitting manifests and disabled images", async () => {
+    const main = await zip({ "parent/README.md": "Alice parent", "parent/.gitmodules": "https://github.com/private/identity" });
+    const child = await zip({ "child/src/Alice.txt": "Alice child", "child/.gitmodules": "identity", "child/photo.png": "image" });
+    const calls = [];
+    stub(got, "stream", (url, options) => { calls.push({ url, options }); return Readable.from([calls.length === 1 ? main : child]); });
+    expect(await readZip({ ...zipOptions, contentOptions: { image: false } })).to.deep.equal({
+      "README.md": "XXXX-1 parent", "vendor/dependency/src/XXXX-1.txt": "XXXX-1 child",
+    });
+    expect(calls.map(call => call.url)).to.deep.equal([
+      `https://codeload.github.com/parent/repo/zip/${parent.commit}`, `https://codeload.github.com/child/public/zip/${sha}`,
+    ]);
+    expect(calls.every(call => call.options === undefined)).to.equal(true);
+  });
+
+  it("aborts the ZIP if a child exceeds the per-file limit", async () => {
+    const main = await zip({ "parent/README.md": "ok" }), child = await zip({ "child/large.txt": "too much data" });
+    let calls = 0; stub(got, "stream", () => Readable.from([++calls === 1 ? main : child])); stub(config, "MAX_FILE_SIZE", 4);
+    const response = new PassThrough(); response.resume(); const errors = [];
+    response.on("error", error => errors.push(error)); const done = new Promise(resolve => response.once("close", resolve));
+    await streamAnonymizedZip(zipOptions, response); await done;
+    expect(errors.map(error => error.message)).to.deep.equal(["file_too_big"]);
+  });
+});
+
+describe("submodule transport and public metadata", function () {
+  this.timeout(10000);
+  it("does not expose upstream repository coordinates in the public tree", async () => {
+    const Repository = require("../src/core/Repository").default;
+    const files = await Repository.prototype.anonymizedFiles.call({
+      _model: { options: { terms: ["Alice"] } },
+      files: async () => [{ name: "Alice.txt", path: "vendor/dependency/src", sha: blob, size: 5, upstream, submodule: "included" }],
+    }, { includeSha: false });
+    expect(files[0]).to.have.all.keys("name", "path", "sha", "size");
+    expect(files[0].name).to.equal("XXXX-1.txt");
+    expect(JSON.stringify(files)).not.to.include("public");
+  });
+
+  for (const mode of ["send", "anonymizedContent"]) {
+    it(`preserves child origin through ${mode} and the real streamer route`, async () => {
+      const express = require("express");
+      const File = require("../src/core/AnonymizedFile").default;
+      const { AnonymizeTransformer } = require("../src/core/anonymize-utils");
+      const router = require("../src/streamer/route").router;
+      const previous = { endpoint: config.STREAMER_ENTRYPOINT, stream: got.stream,
+        info: storage.fileInfo, write: storage.write, read: storage.read };
+      let server, payload; const cache = new Map(), requests = [];
+      try {
+        storage.fileInfo = async (_id, path) => { if (!cache.has(path)) throw Object.assign(Error("missing"), { code: "ENOENT" }); return { size: cache.get(path).length }; };
+        storage.write = async (_id, path, stream) => { const chunks = []; for await (const chunk of stream) chunks.push(Buffer.from(chunk)); cache.set(path, Buffer.concat(chunks)); };
+        storage.read = async (_id, path) => Readable.from([cache.get(path)]);
+        got.stream = (url, options) => {
+          if (!String(url).startsWith("https://github.com/")) return previous.stream(url, options);
+          requests.push({ url, options }); return Readable.from([Buffer.from("Alice")]);
+        };
+        const app = express(); app.use(express.json());
+        app.use((req, _res, next) => { payload = req.body; next(); }); app.use("/api", router);
+        server = await new Promise(resolve => { const listening = app.listen(0, "127.0.0.1", () => resolve(listening)); });
+        config.STREAMER_ENTRYPOINT = `http://127.0.0.1:${server.address().port}/`;
+        const file = new File({ anonymizedPath: "vendor/dependency/src/XXXX-1.txt", repository: {
+          repoId: parent.repoId, options: { terms: ["Alice"] },
+          model: { source: { repositoryName: "parent/repo", commit: parent.commit } },
+          getToken: parent.getToken, owner: { id: "owner" },
+          generateAnonymizeTransformer: filePath => new AnonymizeTransformer({ terms: ["Alice"], filePath, image: true, link: true }),
+        } });
+        file._file = { name: "Alice.txt", path: "vendor/dependency/src", sha: blob, size: 5, upstream };
+        let content = "";
+        if (mode === "send") {
+          const response = new PassThrough(); response.setHeader = () => {};
+          response.on("data", chunk => { content += chunk; });
+          await file.send(response);
+        } else {
+          for await (const chunk of await file.anonymizedContent()) content += chunk;
+        }
+        expect(content).to.equal("XXXX-1");
+        expect(payload.upstream).to.deep.equal(upstream);
+        expect(payload.filePath).to.equal("vendor/dependency/src/Alice.txt");
+        expect(requests).to.have.length(1);
+        expect(requests[0].url).to.equal(`https://github.com/child/public/raw/${sha}/src/Alice.txt`);
+        expect(requests[0].options.headers).to.deep.equal({});
+      } finally {
+        config.STREAMER_ENTRYPOINT = previous.endpoint; got.stream = previous.stream;
+        storage.fileInfo = previous.info; storage.write = previous.write; storage.read = previous.read;
+        if (server) { server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); }
+      }
+    });
+  }
+
+  it("cancels the current child archive when the reader disconnects", async () => {
+    const main = await zip({ "parent/README.md": "ok" }); const child = new PassThrough();
+    const previous = got.stream; let calls = 0, started;
+    const childStarted = new Promise(resolve => { started = resolve; });
+    const response = new PassThrough(); response.resume();
+    try {
+      got.stream = () => { if (++calls === 1) return Readable.from([main]); started(); return child; };
+      await streamAnonymizedZip({ ...zipOptions, submodules: [...zipOptions.submodules,
+        { ...zipOptions.submodules[0], path: "other/dependency" }] }, response);
+      await childStarted; response.destroy();
+      await new Promise(resolve => setImmediate(resolve));
+      expect(child.destroyed).to.equal(true); expect(calls).to.equal(2);
+    } finally { got.stream = previous; child.destroy(); response.destroy(); }
+  });
+});
+
+describe("legacy submodule manifests", function () {
+  it("rejects a cached .gitmodules file even when its public name was anonymized", async () => {
+    const File = require("../src/core/AnonymizedFile").default;
+    const file = new File({ anonymizedPath: ".XXXX-1", repository: {
+      options: { terms: ["gitmodules"] }, model: { source: { type: "GitHubStream" } },
+    } });
+    file._file = { name: ".gitmodules", path: "", sha: blob, size: 5 };
+    for (const read of [() => file.getFileInfo(), () => file.originalPath()]) {
+      let error; try { await read(); } catch (caught) { error = caught; }
+      expect(error?.message).to.equal("file_not_found");
+    }
+  });
+});

test/owner-notifications.test.js

@@ -500,9 +500,7 @@ describe("streamed file access alerts", () => {
     stub(config, "STREAMER_ENTRYPOINT", "http://streamer/");
     const stream = new PassThrough();
     stub(got, "stream", () => stream);
-    file.originalPath = async () => "README.md";
+    file._file = { name: "README.md", path: "", sha: "sha", size: 10 };
-    file.sha = async () => "sha";
-    file.size = async () => 10;
     const returned = await file.anonymizedContent();
     expect(returned).to.equal(stream);
     const error = Object.assign(new Error("streamer failure"), { response: { statusCode: status, body: JSON.stringify({ error: status === 503 ? "upstream_error" : "file_not_accessible" }) } });

test/performance-integration.test.js

@@ -40,6 +40,19 @@ const uri = process.env.PERFORMANCE_MONGO_URI;
   });
   beforeEach(async () => { await Promise.all([Repo, PR, Gist, File, Path, Name, ConferenceModel].map(model => model.deleteMany({}))); });
 
+  it("preserves submodule origin through the persisted anonymized path index", async () => {
+    const row = await Repo.create({ repoId: "submodule-index", owner: user.model._id, status: "ready", treeGeneration: "generation",
+      source: { repositoryName: "parent/repo", commit: "a".repeat(40) }, options: { terms: ["Alice"] } });
+    const upstream = { organization: "child", repoName: "public", commit: "b".repeat(40), path: "Alice.txt" };
+    await File.create({ repoId: row.repoId, treeGeneration: "generation", name: "Alice.txt", path: "vendor/lib", sha: "c".repeat(40), size: 5, upstream });
+    const repository = new Repository(row);
+    const file = new AnonymizedFile({ repository, anonymizedPath: "vendor/lib/XXXX-1.txt" });
+    expect((await file.getFileInfo()).upstream).to.deep.equal(upstream);
+    const publicTree = await repository.anonymizedFiles({ includeSha: false });
+    expect(publicTree[0]).not.to.have.property("upstream");
+    expect(JSON.stringify(publicTree)).not.to.include("child");
+  });
+
   it("uses membership indexes for owner, stable GitHub ID, and legacy username", async () => {
     const other = new mongoose.Types.ObjectId();
     await Repo.insertMany(Array.from({ length: 2000 }, (_, i) => ({ repoId: `filler-${i}`, owner: other, coauthors: [{ username: `other-${i}`, githubId: String(i + 50000) }] })));

test/zip-proxy-errors.test.js

@@ -25,7 +25,7 @@ describe("ZIP proxy failures", function () {
     utils.getUser = async () => { throw new Error("anonymous"); };
     utils.getRepo = async () => ({ repoId: "test", options: {},
       model: { source: { repositoryName: "owner/repo", commit: "abc" } },
-      countView: async () => {}, getToken: async () => "",
+      countView: async () => {}, submoduleMounts: async () => [], getToken: async () => "",
       generateAnonymizeTransformer: () => ({ opt: {} }),
     });
     upstream = new PassThrough();