fix: handle production source failures and reduce ZIP work

src/core/archive-entry.ts

@@ -1,5 +1,17 @@
-import { Archiver, EntryData } from "archiver";
+import { Archiver, EntryData, ZipEntryData } from "archiver";
+import { extname } from "path";
 import { Readable } from "stream";
+
+const compressedExtensions = new Set([
+  ".png", ".jpg", ".jpeg", ".gif", ".webp", ".avif", ".heif", ".heic",
+  ".mp4", ".m4v", ".mov", ".webm", ".mp3", ".m4a", ".aac", ".ogg", ".flac",
+  ".zip", ".gz", ".tgz", ".bz2", ".xz", ".7z", ".rar", ".zst", ".woff", ".woff2",
+]);
+
+/** Store already compressed assets; every entry still passes through anonymization. */
+export function zipEntryOptions(name: string, originalName = name): ZipEntryData {
+  return { name, store: compressedExtensions.has(extname(originalName).toLowerCase()) };
+}
 /** Allow only one active entry; compression and the response provide backpressure. */
 export function appendArchiveEntry(archive: Archiver, source: Readable, options: EntryData): Promise<void> {
   return new Promise((resolve, reject) => {

src/core/logger.ts

@@ -320,7 +320,10 @@ export function serializeError(err: unknown): Record<string, unknown> {
 
   // AnonymousError carries an httpStatus and an inner cause.
   if (typeof e.httpStatus === "number") out.httpStatus = e.httpStatus;
-  if (e.code !== undefined && e.code !== e.message) out.code = e.code;
+  // Octokit's deprecated `code` getter emits a stack on every read. Only
+  // inspect data properties so logging never invokes that getter.
+  const code = Object.getOwnPropertyDescriptor(e, "code")?.value;
+  if (code !== undefined && code !== e.message) out.code = code;
   if (typeof e.url === "function") {
     try {
       const u = e.url();

src/core/repository-failure-status.ts

@@ -6,6 +6,7 @@ export function repositoryFailureStatus(code?: string): number {
     case "token_expired": case "not_connected": return 401;
     case "github_rate_limit_exceeded": return 429;
     case "github_unavailable": return 502;
+    case "repo_empty": return 409;
     default: return 500;
   }
 }

src/core/request-monitoring.ts

@@ -148,8 +148,12 @@ export function monitorRequests(service: Service, metrics = requestMetrics,
       const slow = ms > (transfer ? 10000 : 1000);
       metrics.observe("request", route, method, ms, { bytes, aborted, error: res.statusCode >= 500, slow });
       if (firstByteMs != null) metrics.observe("first_byte", route, method, firstByteMs);
+      const responseMs = firstByteMs == null ? null : Math.max(0, ms - firstByteMs);
       const detail = { requestId: id, service, method, route, status: res.statusCode,
-        outcome: aborted ? "interrupted" : "completed", ms: Math.round(ms), firstByteMs: firstByteMs == null ? null : Math.round(firstByteMs), bytes, stages: trace.stages };
+        outcome: aborted ? "interrupted" : "completed", ms: Math.round(ms), firstByteMs: firstByteMs == null ? null : Math.round(firstByteMs),
+        responseMs: responseMs == null ? null : Math.round(responseMs),
+        bytesPerSecond: responseMs != null && responseMs >= 1 && bytes > 0 ? Math.round(bytes * 1000 / responseMs) : null,
+        bytes, stages: trace.stages };
       if (slow || aborted) log.warn(aborted ? "request interrupted" : "slow request", detail);
       else if (!route.includes("healthcheck")) log.info("request", detail);
     };

src/core/source/GitHubStream.ts

@@ -15,7 +15,7 @@ import { basename, dirname } from "path";
 
 import * as stream from "stream";
 import AnonymousError from "../AnonymousError";
-import { octokit, waitForTokenGate } from "../GitHubUtils";
+import { octokit, waitForTokenGate, RateLimitDelayError } from "../GitHubUtils";
 import FileModel from "../model/files/files.model";
 import { IFile } from "../model/files/files.types";
 import { createLogger, serializeError } from "../logger";
@@ -24,13 +24,20 @@ import config from "../../config";
 import { isConnected } from "../../server/database";
 import AnonymizedRepositoryModel from "../model/anonymizedRepositories/anonymizedRepositories.model";
 import { RepositoryStatus } from "../types";
+import { cachedContentMiss, classifyContentMiss } from "./github-content-miss";
 
 
 const logger = createLogger("gh-stream");
 
 const GH_API_CONCURRENCY = 6;
 const cacheFills = new Map<string, Promise<void>>();
-const missingContent = new ExpiringMap<boolean>(2048);
+const missingContent = new ExpiringMap<string>(2048);
+
+/** Stop upstream before Readable.from waits for the pending iterator to finish. */
+function stopSourceOnDestroy(output: stream.Readable, stop: () => void): void {
+  const destroy = output._destroy;
+  output._destroy = (error, callback) => { stop(); destroy.call(output, error, callback); };
+}
 
 async function pMap<T, R>(
   items: T[],
@@ -68,6 +75,7 @@ export function githubRawFileUrl(
 
 function downloadError(error: Error, filePath: string): AnonymousError {
   if (error instanceof AnonymousError) return error;
+  if (error instanceof RateLimitDelayError) return new AnonymousError("github_rate_limit_exceeded", { httpStatus: 429, cause: error });
   const response = error as Error & { response?: { statusCode?: number }; status?: number; httpStatus?: number };
   const status = response.response?.statusCode ?? response.status ?? response.httpStatus;
   const code = status === 401 || status === 403 ? "file_not_accessible"
@@ -115,10 +123,9 @@ export default class GitHubStream extends GitHubBase {
     }
   }
 
-  // GitHub's web raw URL auto-resolves Git LFS pointers via redirect to
-  // media.githubusercontent.com, with the auth header carried through. The
-  // blob endpoint above returns the raw pointer text instead, so we use this
-  // as the fallback for LFS files (#95).
+  // The web URL resolves public LFS pointers. Private web URLs can return
+  // 404 even when the REST API can read the file, so use the scoped contents
+  // API to obtain a short-lived download URL in that case.
   private downloadFileViaRaw(token: string, filePath: string) {
     const url = githubRawFileUrl(
       this.data.organization,
@@ -127,11 +134,73 @@ export default class GitHubStream extends GitHubBase {
       filePath
     );
     logger.debug("downloading via raw URL (LFS)", { url });
-    return got.stream(url, {
-      hooks: { beforeRequest: [async () => { await githubTokenContext(token)?.renew(); }] },
-      headers: !token || githubTokenContext(token)?.publicRepository ? {} : { authorization: `token ${token}` },
-      followRedirect: true,
-    });
+    const data = this.data;
+    const cancellation = new AbortController();
+    let active: stream.Readable | undefined;
+    const output = stream.Readable.from((async function* () {
+      try {
+        active = got.stream(url, {
+          hooks: { beforeRequest: [async () => { await githubTokenContext(token)?.renew(); }] },
+          headers: !token || githubTokenContext(token)?.publicRepository ? {} : { authorization: `token ${token}` },
+          followRedirect: true,
+        });
+        for await (const chunk of active) yield chunk;
+        return;
+      } catch (error) {
+        if (!token || githubTokenContext(token)?.publicRepository ||
+            (error as { response?: { statusCode?: number } }).response?.statusCode !== 404) throw error;
+        await waitForTokenGate(token);
+        const response = await octokit(token).repos.getContent({
+          owner: data.organization, repo: data.repoName,
+          path: filePath, ref: data.commit || "HEAD", mediaType: { format: "object" },
+          request: { signal: cancellation.signal, timeout: 30_000 },
+        }).catch(apiError => {
+          if (apiError instanceof AnonymousError) throw apiError;
+          const upstream = apiError as { status?: number; response?: { data?: { message?: string } } };
+          if (upstream.status === 422) {
+            const missingCommit = /^No commit found for SHA\b/.test(upstream.response?.data?.message || "");
+            throw new AnonymousError(missingCommit ? "commit_not_found" : "upstream_error", {
+              httpStatus: missingCommit ? 404 : 502, cause: apiError as Error,
+            });
+          }
+          throw apiError;
+        });
+        if (cancellation.signal.aborted) return;
+        const metadata = response.data as { type?: string; download_url?: string };
+        if (metadata.type !== "file" || !metadata.download_url) throw error;
+        const assertDownloadUrl = (value: URL) => {
+          if (value.protocol !== "https:" || value.username || value.password || value.port ||
+              !["raw.githubusercontent.com", "media.githubusercontent.com"].includes(value.hostname)) {
+            throw new AnonymousError("upstream_error", { httpStatus: 502 });
+          }
+        };
+        assertDownloadUrl(new URL(metadata.download_url));
+        // The API URL supplies its own authorization. Never forward the
+        // owner's token to the download host or a redirect destination.
+        active = got.stream(metadata.download_url, {
+          hooks: { beforeRedirect: [options => { assertDownloadUrl(options.url); }] },
+        });
+        let probe = Buffer.alloc(0);
+        let checked = false;
+        for await (const chunk of active) {
+          if (checked) { yield chunk; continue; }
+          probe = Buffer.concat([probe, Buffer.from(chunk)]);
+          if (probe.length < 150) continue;
+          if (probe.toString("utf8", 0, 39) === "version https://git-lfs.github.com/spec/") {
+            throw new AnonymousError("upstream_error", { httpStatus: 502 });
+          }
+          checked = true; yield probe;
+        }
+        if (!checked && probe.length) {
+          if (probe.toString("utf8").startsWith("version https://git-lfs.github.com/spec/")) {
+            throw new AnonymousError("upstream_error", { httpStatus: 502 });
+          }
+          yield probe;
+        }
+      }
+    })(), { objectMode: false });
+    stopSourceOnDestroy(output, () => { cancellation.abort(); active?.destroy(); });
+    return output;
   }
 
   private deferredRawDownload(token: string, filePath: string): stream.Readable {
@@ -141,7 +210,7 @@ export default class GitHubStream extends GitHubBase {
       active = download();
       for await (const chunk of active) yield chunk;
     })(), { objectMode: false });
-    output.once("close", () => active?.destroy());
+    stopSourceOnDestroy(output, () => active?.destroy());
     return output;
   }
 
@@ -244,7 +313,7 @@ export default class GitHubStream extends GitHubBase {
     }
     const resolved = stream.Readable.from(resolve(), { objectMode: false });
     // Keep the generator lazy until the consumer has installed its handlers.
-    resolved.once("close", () => active.destroy());
+    stopSourceOnDestroy(resolved, () => active.destroy());
     return resolved;
   }
 
@@ -314,10 +383,13 @@ export default class GitHubStream extends GitHubBase {
       filling = coordinatedFill(key, cached, async () => {
         await assertActive();
         const token = await this.data.getToken();
+        const sourceMiss = cachedContentMiss(this.data, token);
+        if (sourceMiss && sourceMiss !== "file_not_found") throw new AnonymousError(sourceMiss, { httpStatus: 404 });
         const missKey = createHash("sha256").update(JSON.stringify([
           key, this.data.organization, this.data.repoName, token,
         ])).digest("hex");
-        if (missingContent.get(missKey)) throw new AnonymousError("file_not_found", { httpStatus: 404 });
+        const missed = missingContent.get(missKey);
+        if (missed) throw new AnonymousError(missed, { httpStatus: 404 });
         try {
           return await measureStage("upstream", async () => {
             const content = await this.downloadWithFallback(token, expected.sha, filePath)
@@ -342,7 +414,9 @@ export default class GitHubStream extends GitHubBase {
           });
         } catch (error) {
           if (error instanceof AnonymousError && error.message === "file_not_found" && error.httpStatus === 404) {
-            missingContent.set(missKey, true, 30_000);
+            const code = await classifyContentMiss(this.data, token);
+            missingContent.set(missKey, code, 30_000);
+            if (code !== "file_not_found") throw new AnonymousError(code, { httpStatus: 404, cause: error });
           }
           throw error;
         }
@@ -430,19 +504,37 @@ export default class GitHubStream extends GitHubBase {
     count = { request: 0, file: 0 },
     opt = { recursive: true, callback: () => {} }
   ) {
-    await waitForTokenGate(token);
-    const ghRes = await oct.git.getTree({
-      owner: this.data.organization,
-      repo: this.data.repoName,
-      tree_sha: sha,
-      recursive: opt.recursive === true ? "1" : undefined,
-    });
-    count.request++;
-    count.file += ghRes.data.tree.length;
-    if (opt.callback) {
-      opt.callback();
+    try {
+      await waitForTokenGate(token);
+      const ghRes = await oct.git.getTree({
+        owner: this.data.organization,
+        repo: this.data.repoName,
+        tree_sha: sha,
+        recursive: opt.recursive === true ? "1" : undefined,
+      });
+      count.request++;
+      count.file += ghRes.data.tree.length;
+      if (opt.callback) opt.callback();
+      return ghRes.data;
+    } catch (error) {
+      // Keep quota, credential and lifecycle errors intact. A failed tree
+      // request does not imply that the repository has disappeared.
+      if (error instanceof AnonymousError || error instanceof RateLimitDelayError) throw error;
+      const upstream = error as { status?: number; response?: { data?: { message?: string } } };
+      const status = upstream.status;
+      const invalidObject = status === 422 && /^Invalid object requested\. SHA must identify a commit or a tree\b/.test(upstream.response?.data?.message || "");
+      const code = status === 409 ? "repo_empty"
+        : status === 404 ? await classifyGitHubMissError(error, this.data)
+        : invalidObject ? "commit_not_found"
+        : status === 401 ? "token_expired"
+        : status === 403 ? "repo_not_accessible" : "github_unavailable";
+      throw new AnonymousError(code, {
+        httpStatus: status === 409 ? 409 : status === 404 || invalidObject ? 404
+          : status === 401 || status === 403 ? status : 502,
+        object: this.data,
+        cause: error as Error,
+      });
     }
-    return ghRes.data;
   }
 
   private async getTruncatedTree(
@@ -457,55 +549,17 @@ export default class GitHubStream extends GitHubBase {
       file: 0,
     };
     const output: IFile[] = [];
-    let data;
-    try {
-      data = await this.getGHTree(oct, token, sha, count, {
-        recursive: true,
-        callback: () => {
-          if (progress) {
-            progress("List file: " + count.file);
-          }
-        },
-      });
-      if (!data.truncated) return this.tree2Tree(data.tree, parentPath);
-      data = await this.getGHTree(oct, token, sha, count, {
-        recursive: false,
-        callback: () => {
-          if (progress) {
-            progress("List file: " + count.file);
-          }
-        },
-      });
-      if (data.truncated) {
-        this._truncatedFolders.push(parentPath);
-      }
-      output.push(...this.tree2Tree(data.tree, parentPath));
-    } catch (error) {
-      const status = (error as { status?: number }).status;
-      if (status === 409) {
-        logger.debug("getTree empty repo", serializeError(error));
-        throw new AnonymousError("repo_empty", {
-          httpStatus: 409,
-          object: this.data,
-          cause: error as Error,
-        });
-      }
-      if (status === 404) {
-        logger.debug("getTree miss", serializeError(error));
-        const code = await classifyGitHubMissError(error, this.data);
-        throw new AnonymousError(code, {
-          httpStatus: 404,
-          object: this.data,
-          cause: error as Error,
-        });
-      }
-      logger.warn("getTree failed", serializeError(error));
-      throw new AnonymousError("repo_not_found", {
-        httpStatus: status || 404,
-        object: this.data,
-        cause: error as Error,
-      });
-    }
+    let data = await this.getGHTree(oct, token, sha, count, {
+      recursive: true,
+      callback: () => { progress?.("List file: " + count.file); },
+    });
+    if (!data.truncated) return this.tree2Tree(data.tree, parentPath);
+    data = await this.getGHTree(oct, token, sha, count, {
+      recursive: false,
+      callback: () => { progress?.("List file: " + count.file); },
+    });
+    if (data.truncated) this._truncatedFolders.push(parentPath);
+    output.push(...this.tree2Tree(data.tree, parentPath));
     const subtrees: { sha: string; parentPath: string }[] = [];
     for (const file of data.tree) {
       if (file.type == "tree" && file.path && file.sha) {

src/core/source/github-content-miss.ts

@@ -0,0 +1,65 @@
+import { createHash } from "crypto";
+import AnonymousError from "../AnonymousError";
+import { ExpiringMap } from "../expiring-map";
+import { octokit, waitForTokenGate, RateLimitDelayError } from "../GitHubUtils";
+import { GitHubBaseData } from "./GitHubBase";
+
+type Miss = "file_not_found" | "repo_not_found" | "commit_not_found";
+const results = new ExpiringMap<Miss>(2048);
+const pending = new Map<string, Promise<Miss>>();
+
+function missKey(data: GitHubBaseData, token: string): string {
+  return createHash("sha256").update(JSON.stringify([
+    data.repoId, data.organization, data.repoName, data.commit, data.cacheGeneration, data.cacheRevision, token,
+  ])).digest("hex");
+}
+
+export function cachedContentMiss(data: GitHubBaseData, token: string): Miss | undefined {
+  return results.get(missKey(data, token));
+}
+
+/** A raw 404 can mean a missing file, lost repository access, or a removed revision. */
+export async function classifyContentMiss(data: GitHubBaseData, token: string): Promise<Miss> {
+  const key = missKey(data, token);
+  const cached = results.get(key);
+  if (cached) return cached;
+  const active = pending.get(key);
+  if (active) return active;
+  if (pending.size >= 64) throw new AnonymousError("cache_busy", { httpStatus: 503 });
+  const probe = (async (): Promise<Miss> => {
+    const oct = octokit(token);
+    const options = { owner: data.organization, repo: data.repoName, request: { timeout: 15_000 } };
+    try {
+      await waitForTokenGate(token);
+      await oct.repos.get(options);
+    } catch (error) {
+      if ((error as { status?: number }).status === 404) return "repo_not_found";
+      throw probeError(error);
+    }
+    try {
+      await waitForTokenGate(token);
+      // Only check that the revision resolves; do not download a potentially
+      // large commit diff just to classify a missing file.
+      await oct.repos.getCommit({ ...options, ref: data.commit || "HEAD", mediaType: { format: "sha" } });
+    } catch (error) {
+      const upstream = error as { status?: number; response?: { data?: { message?: string } } };
+      if (upstream.status === 404 || upstream.status === 409 || (upstream.status === 422 &&
+          /^No commit found for SHA\b/.test(upstream.response?.data?.message || ""))) return "commit_not_found";
+      throw probeError(error);
+    }
+    return "file_not_found";
+  })().then(result => { results.set(key, result, 30_000); return result; });
+  pending.set(key, probe);
+  try { return await probe; }
+  finally { if (pending.get(key) === probe) pending.delete(key); }
+}
+
+function probeError(error: unknown): AnonymousError {
+  if (error instanceof AnonymousError) return error;
+  if (error instanceof RateLimitDelayError) return new AnonymousError("github_rate_limit_exceeded", { httpStatus: 429, cause: error });
+  const status = (error as { status?: number }).status;
+  return new AnonymousError(status === 401 ? "token_expired" : status === 403 ? "repo_not_accessible" : "github_unavailable", {
+    httpStatus: status === 401 || status === 403 ? status : 502,
+    cause: error as Error,
+  });
+}

src/core/storage/FileSystem.ts

@@ -1,4 +1,4 @@
-import { appendArchiveEntry } from "../archive-entry";
+import { appendArchiveEntry, zipEntryOptions } from "../archive-entry";
 import config from "../../config";
 import * as fs from "fs";
 import { Extract } from "unzip-stream";
@@ -280,7 +280,7 @@ export default class FileSystem extends StorageBase {
           output = source.pipe(transformer);
           active = output;
         }
-        await appendArchiveEntry(archive, output, { name: filename.slice(dir ? dir.length + 1 : 0) });
+        await appendArchiveEntry(archive, output, zipEntryOptions(filename.slice(dir ? dir.length + 1 : 0), filename));
       }
       if (!stopped) await archive.finalize();
     })().catch(error => archive.destroy(error));

src/core/storage/S3.ts

@@ -1,4 +1,4 @@
-import { appendArchiveEntry } from "../archive-entry";
+import { appendArchiveEntry, zipEntryOptions } from "../archive-entry";
 import {
   GetObjectCommand,
   ListObjectsV2CommandOutput,
@@ -398,7 +398,7 @@ export default class S3Storage extends StorageBase {
             output = source.pipe(transformer);
             active = output;
           }
-          await appendArchiveEntry(archive, output, { name: filename.slice(dir.length) });
+          await appendArchiveEntry(archive, output, zipEntryOptions(filename.slice(dir.length), filename));
         }
         if (listing.IsTruncated && !next) throw new Error("Invalid S3 continuation token");
       } while (next && !stopped);

src/core/zipStream.ts

@@ -13,6 +13,7 @@ import {
 } from "./anonymize-utils";
 import { createLogger, serializeError } from "./logger";
 import { startStage } from "./request-monitoring";
+import { zipEntryOptions } from "./archive-entry";
 
 const logger = createLogger("zip-stream");
 
@@ -222,7 +223,7 @@ export async function streamAnonymizedZip(
           activeStreams.add(anonymizer);
           anonymizer.once("close", () => activeStreams.delete(anonymizer));
           const st = entry.pipe(anonymizer);
-          archive.append(st, { name: fileName });
+          archive.append(st, zipEntryOptions(fileName, entry.path));
         } catch (error) {
           fail(error as Error);
         }

src/server/index.ts

@@ -39,6 +39,7 @@ import config from "../config";
 import { resolveTrustProxy } from "./trustProxy";
 import { requestRateLimitKey } from "./rate-limit-key";
 import { createLogger, serializeError } from "../core/logger";
+import { serverOptions } from "./server-options";
 import { monitorRequests } from "../core/request-monitoring";
 import { startPerformanceMonitoring } from "../core/performance-monitoring";
 import { getAnonymizationPoolStats } from "../core/anonymization-pool";
@@ -113,6 +114,7 @@ function indexResponse(req: express.Request, res: express.Response) {
 export default async function start() {
   const app = express();
   app.use(monitorRequests("api"));
+  app.use(serverOptions);
   startPerformanceMonitoring("api", getAnonymizationPoolStats);
   app.set("query parser", "extended");
   app.use("/service", createReviewCapabilities(process.env.REVIEW_SERVICE_KEYS));

src/server/server-options.ts

@@ -0,0 +1,9 @@
+import { RequestHandler } from "express";
+
+/** The asterisk request target asks about the server, not a session or route. */
+export const serverOptions: RequestHandler = (req, res, next) => {
+  if (req.originalUrl !== "*") return next();
+  if (req.method !== "OPTIONS") { res.status(400).json({ error: "invalid_request" }); return; }
+  res.setHeader("Allow", "GET, HEAD, POST, PUT, PATCH, DELETE, OPTIONS");
+  res.status(204).end();
+};