fix: contain upstream failures and back off repeated retries

test/credential-migration.test.js

@@ -179,13 +179,15 @@ describeMongo("credential access (MongoDB)", function () {
     await setCredential(owner.id, "old-token");
     await Credential.updateOne({ ownerId: owner._id }, { $set: { updatedAt: new Date(0) } });
     const originalFetch = global.fetch;
+    const oauthConfig = [config.GITHUB_OAUTH_ENABLED, config.CLIENT_ID, config.CLIENT_SECRET];
+    config.GITHUB_OAUTH_ENABLED = true; config.CLIENT_ID = "fixture-client"; config.CLIENT_SECRET = "fixture-secret";
     global.fetch = async (url, options) => {
       expect(url).to.include("api.github.com/applications/");
       expect(JSON.parse(options.body).access_token).to.equal("old-token");
       return { ok: true, json: async () => ({ token: "refreshed-token" }) };
     };
     try { expect(await repo.getToken()).to.equal("refreshed-token"); }
-    finally { global.fetch = originalFetch; }
+    finally { global.fetch = originalFetch; [config.GITHUB_OAUTH_ENABLED, config.CLIENT_ID, config.CLIENT_SECRET] = oauthConfig; }
     expect(await getCredentialToken(owner.id)).to.equal("refreshed-token");
     expect((await RepoModel.collection.findOne({ _id: repo.model._id })).source.accessToken).to.equal(undefined);
   });

test/fixtures/github-stream-failure.js

@@ -0,0 +1,44 @@
+const assert = require("node:assert/strict");
+const { Readable } = require("node:stream");
+const { setImmediate } = require("node:timers");
+require("ts-node/register/transpile-only");
+const GitHubStream = require("../../src/core/source/GitHubStream").default;
+const failure = Object.assign(new Error("upstream failed"), process.argv[2] === "reset" ? { code: "ECONNRESET" } : { response: { statusCode: 401 } });
+const source = new GitHubStream({ repoId: "fixture", organization: "owner", repoName: "repo", commit: "abc", getToken: () => "token" });
+async function main() {
+  if (process.argv[2] === "fatal") {
+    require("../../src/core/process-monitoring").installFatalErrorLogging("streamer");
+    await new Promise(() => setImmediate(() => { throw new Error("fixture fatal"); }));
+  }
+  if (process.argv[2] === "fallback" || process.argv[2] === "public-raw") {
+    source.downloadFile = () => {
+      const input = new Readable({ read() {} });
+      process.nextTick(() => input.destroy(Object.assign(new Error("blob unavailable"), { response: { statusCode: 404 } })));
+      return input;
+    };
+    source.downloadFileViaRaw = () => {
+      const input = new Readable({ read() {} });
+      process.nextTick(() => input.destroy(failure));
+      return input;
+    };
+    const content = await source.downloadWithFallback(process.argv[2] === "public-raw" ? "" : "token", "blob", "file");
+    await new Promise(resolve => setImmediate(resolve));
+    await assert.rejects(async () => { for await (const chunk of content) void chunk; }, error => error === failure);
+  } else if (process.argv[2] === "lfs") {
+    const pointer = "version https://git-lfs.github.com/spec/v1\noid sha256:" + "a".repeat(64) + "\nsize 10\n";
+    source.downloadFileViaRaw = () => new Readable({ read() { this.destroy(failure); } });
+    const content = source.resolveLfsPointer(Readable.from([pointer]), "token", "file");
+    // A producer must not start emitting errors while its consumer is still awaiting.
+    await new Promise(resolve => setImmediate(resolve));
+    await assert.rejects(async () => { for await (const chunk of content) void chunk; }, error => error === failure);
+  } else {
+    source.downloadFile = () => {
+      const content = new Readable({ read() {} });
+      process.nextTick(() => content.destroy(failure));
+      return content;
+    };
+    await assert.rejects(source.downloadWithFallback("token", "blob", "file"), error => error === failure);
+    await new Promise(resolve => setImmediate(resolve));
+  }
+}
+main().then(() => { process.stdout.write("failure contained\n"); process.exit(0); }, error => { console.error(error); process.exit(1); });

test/fixtures/performance-streamer.js

@@ -9,7 +9,18 @@ const maintenance = require(path.join(base, "core/temporary-storage"));
 const startMaintenance = maintenance.startTemporaryStorageMaintenance;
 // Exercise the real startup/timer wiring without waiting a production minute.
 maintenance.startTemporaryStorageMaintenance = () => startMaintenance(25);
-GitHubStream.prototype.downloadWithFallback = async function () {
+const downloadWithFallback = GitHubStream.prototype.downloadWithFallback;
+GitHubStream.prototype.downloadWithFallback = async function (token, sha, filePath) {
+  if (filePath.startsWith("fail-")) {
+    this.downloadFile = () => {
+      const input = new Readable({ read() {} });
+      const failure = Object.assign(new Error("fixture upstream failure"), filePath === "fail-reset.txt"
+        ? { code: "ECONNRESET" } : { response: { statusCode: 401 } });
+      process.nextTick(() => input.destroy(failure));
+      return input;
+    };
+    return downloadWithFallback.call(this, token, sha, filePath);
+  }
   process.send({ download: true });
   return Readable.from([Buffer.alloc(300000, "x")]);
 };

test/performance-regressions.test.js

@@ -121,22 +121,22 @@ describe("performance regressions", function () {
     expect(files).to.have.length(20); expect(calls).to.deep.equal([{ sha: "head", recursive: true }]);
   });
 
-  for (const [upstream, code, status] of [
+  for (const [caseId, [upstream, code, status]] of [
     [Object.assign(new Error("forbidden"), { response: { statusCode: 403 } }), "file_not_accessible", 403],
     [Object.assign(new Error("missing"), { response: { statusCode: 404 } }), "file_not_found", 404],
     [Object.assign(new Error("large"), { status: 422 }), "file_too_big", 422],
     [Object.assign(new Error("missing"), { httpStatus: 404 }), "file_not_found", 404],
     [Object.assign(new Error("reset"), { code: "ECONNRESET" }), "upstream_error", 502],
     [new (require("../src/core/AnonymousError").default)("file_not_accessible", { httpStatus: 403 }), "file_not_accessible", 403],
-  ]) {
+  ].entries()) {
     it(`preserves ${code}/${status} for failed downloads (${upstream.message})`, async () => {
-      const source = new GitHubStream({ repoId: "failed-download", organization: "owner", repoName: "repo", getToken: () => "token" });
+      const source = new GitHubStream({ repoId: `failed-download-${caseId}`, organization: "owner", repoName: "repo", getToken: () => "token" });
       const input = new PassThrough();
       stub(storage, "fileInfo", async () => { throw Error("cache miss"); });
       let committed = false;
       stub(storage, "write", async (_repo, _path, stream) => { for await (const chunk of stream) void chunk; committed = true; });
       source.downloadWithFallback = async () => { setImmediate(() => input.destroy(upstream)); return input; };
-      try { await source.getFileContentCache("file.txt", "failed-download", () => ({ sha: "blob", size: 1 })); throw Error("expected failure"); }
+      try { await source.getFileContentCache("file.txt", source.data.repoId, () => ({ sha: "blob", size: 1 })); throw Error("expected failure"); }
       catch (error) {
         expect(error.message).to.equal(code); expect(error.httpStatus).to.equal(status);
         if (upstream instanceof require("../src/core/AnonymousError").default) expect(error).to.equal(upstream);

test/performance-streamer-integration.test.js

@@ -75,6 +75,22 @@ const uri = process.env.PERFORMANCE_MONGO_URI;
       repoId: row.repoId, repoFullName: "owner/repo", commit: "abc", ...(legacy ? {} : { cacheGeneration: options.cacheGeneration }), sha: "blob", size: 300000, filePath, anonymizerOptions: options }) });
   }
 
+  for (const [filePath, expectedStatus] of [["fail-401.txt", 401], ["fail-reset.txt", 502]]) {
+    it(`contains ${filePath} errors in the actual streamer and serves another request`, async () => {
+      const runtime = await launch();
+      try {
+        await runtime.ready;
+        const row = await createRow(filePath);
+        const failure = await request(runtime, row, filePath);
+        expect(failure.status).to.equal(expectedStatus); await failure.text();
+        expect((await fetch(runtime.url + "/healthcheck")).status).to.equal(200);
+        const recovered = await request(runtime, row, "working.txt");
+        expect(recovered.status).to.equal(200); expect((await recovered.text()).length).to.equal(300000);
+        expect(runtime.child.exitCode).to.equal(null);
+      } finally { await stop(runtime); }
+    });
+  }
+
   it("connects before serving and rejects a fill published after expiration in the actual streamer entrypoint", async () => {
     const runtime = await launch(false, true); let pending;
     try {

test/production-failure-fixes.test.js

@@ -0,0 +1,133 @@
+const { expect } = require("chai");
+const { spawnSync } = require("node:child_process");
+const path = require("node:path");
+require("ts-node/register/transpile-only");
+const config = require("../src/config").default;
+const credentials = require("../src/core/credentials");
+const { refreshLegacyToken } = require("../src/core/legacy-token-refresh");
+const { ExpiringMap } = require("../src/core/expiring-map");
+const { sendRequestRateLimit } = require("../src/server/request-rate-limit");
+const GitHubStream = require("../src/core/source/GitHubStream").default;
+const storage = require("../src/core/storage").default;
+const { logSummary, observeLine, containerSummary } = require("../scripts/check-production-logs");
+
+describe("production failure fixes", function () {
+  this.timeout(20000);
+  const restores = [];
+  function stub(object, key, value) { const old = object[key]; object[key] = value; restores.push(() => { object[key] = old; }); }
+  afterEach(() => restores.splice(0).reverse().forEach(restore => restore()));
+  for (const mode of ["401", "reset", "lfs", "fallback", "public-raw"]) {
+    it(`contains ${mode} upstream stream errors in a process with no recovery handler`, function () {
+      const result = spawnSync(process.execPath, [path.join(__dirname, "fixtures/github-stream-failure.js"), mode], {
+        timeout: 15000, encoding: "utf8", env: { ...process.env, NODE_ENV: "test", GITHUB_APP_ENABLED: "false", REDIS_HOSTNAME: "127.0.0.1", REDIS_PORT: "1" },
+      });
+      expect(result.status, result.stderr).to.equal(0);
+      expect(result.stdout).to.include("failure contained");
+    });
+  }
+  it("records a fatal event without keeping a broken process running", function () {
+    const result = spawnSync(process.execPath, [path.join(__dirname, "fixtures/github-stream-failure.js"), "fatal"], {
+      timeout: 15000, encoding: "utf8", env: { ...process.env, NODE_ENV: "test", GITHUB_APP_ENABLED: "false", REDIS_PORT: "1" },
+    });
+    expect(result.status).to.equal(1);
+    expect(result.stderr + result.stdout).to.include("process_fatal_error");
+  });
+  function oauth() {
+    stub(config, "GITHUB_OAUTH_ENABLED", true); stub(config, "CLIENT_ID", "test-client"); stub(config, "CLIENT_SECRET", "test-secret");
+  }
+  it("coalesces token refreshes and backs off on permanent 404 responses without replacing credentials", async () => {
+    oauth(); let calls = 0, replacements = 0;
+    stub(globalThis, "fetch", async () => { calls++; return { ok: false, status: 404, body: { cancel: async () => {} } }; });
+    stub(credentials, "replaceCredential", async () => { replacements++; return true; });
+    const first = await Promise.all(Array.from({ length: 8 }, () => refreshLegacyToken("owner-failure", "gho_first")));
+    expect(first).to.deep.equal(Array(8).fill(null));
+    expect(await refreshLegacyToken("owner-failure", "gho_first")).to.equal(null);
+    expect(calls).to.equal(1); expect(replacements).to.equal(0);
+    await refreshLegacyToken("owner-failure", "gho_reconnected"); expect(calls).to.equal(2);
+    config.CLIENT_SECRET = "rotated-secret";
+    await refreshLegacyToken("owner-failure", "gho_first"); expect(calls).to.equal(3);
+  });
+  it("skips PAT and GitHub App tokens and disabled OAuth", async () => {
+    oauth(); stub(globalThis, "fetch", () => { throw Error("unexpected refresh"); });
+    for (const token of ["ghp_token", "ghs_token", "ghu_token", "ghr_token", "github_pat_token"]) expect(await refreshLegacyToken("owner", token)).to.equal(null);
+    config.GITHUB_OAUTH_ENABLED = false; expect(await refreshLegacyToken("owner", "gho_token")).to.equal(null);
+  });
+  it("keeps a newer login when the refresh compare-and-swap loses", async () => {
+    oauth(); stub(globalThis, "fetch", async () => ({ ok: true, status: 200, json: async () => ({ token: "refreshed" }) }));
+    stub(credentials, "replaceCredential", async () => false);
+    stub(credentials, "getCredentialToken", async () => "new-login");
+    expect(await refreshLegacyToken("owner-race", "gho_old")).to.equal("new-login");
+  });
+  it("backs off on network failures while leaving the current token intact", async () => {
+    oauth(); let calls = 0;
+    stub(globalThis, "fetch", async () => { calls++; throw Error("network failure"); });
+    expect(await refreshLegacyToken("owner-network", "gho_old")).to.equal(null);
+    expect(await refreshLegacyToken("owner-network", "gho_old")).to.equal(null);
+    expect(calls).to.equal(1);
+  });
+  it("bounds failure caches and expires entries so repaired content can be retried", () => {
+    let now = 0; const cache = new ExpiringMap(2, () => now);
+    cache.set("a", true, 30); cache.set("b", true, 30); cache.set("c", true, 30);
+    expect(cache.get("a")).to.equal(undefined); expect(cache.get("b")).to.equal(true);
+    now = 30; expect(cache.get("b")).to.equal(undefined); expect(cache.get("c")).to.equal(undefined);
+  });
+  it("caches confirmed file misses by credential and commit, without caching permission failures", async () => {
+    let token = "credential-a", attempts = 0, now = Date.now();
+    stub(Date, "now", () => now);
+    stub(storage, "fileInfo", async () => { throw Object.assign(Error("missing"), { code: "ENOENT" }); });
+    stub(storage, "write", async (_repo, _path, input) => { for await (const chunk of input) void chunk; });
+    const source = new GitHubStream({ repoId: "missing-fixture", organization: "owner", repoName: "repo", commit: "first", getToken: () => token });
+    source.downloadWithFallback = async () => { attempts++; throw Object.assign(Error("missing"), { response: { statusCode: 404 } }); };
+    const read = () => source.getFileContentCache("missing.js", "missing-fixture", () => ({ sha: "sha" }));
+    for (let i = 0; i < 3; i++) { try { await read(); } catch (error) { expect(error.httpStatus || error.response?.statusCode).to.equal(404); } }
+    expect(attempts).to.equal(1);
+    token = "credential-b"; await read().catch(() => {}); expect(attempts).to.equal(2);
+    source.data.commit = "second"; await read().catch(() => {}); expect(attempts).to.equal(3);
+    now += 30001; await read().catch(() => {}); expect(attempts).to.equal(4);
+    source.downloadWithFallback = async () => { attempts++; throw Object.assign(Error("forbidden"), { response: { statusCode: 403 } }); };
+    now += 30001; await read().catch(() => {}); await read().catch(() => {}); expect(attempts).to.equal(6);
+  });
+  it("does not cache a storage 404 as a missing upstream file", async () => {
+    let attempts = 0;
+    stub(storage, "fileInfo", async () => { throw Object.assign(Error("missing"), { code: "ENOENT" }); });
+    stub(storage, "write", async (_repo, _path, input) => {
+      for await (const chunk of input) void chunk;
+      throw Object.assign(Error("storage unavailable"), { httpStatus: 404 });
+    });
+    const source = new GitHubStream({ repoId: "storage-failure-fixture", organization: "owner", repoName: "repo", commit: "first", getToken: () => "token" });
+    source.downloadWithFallback = async () => { attempts++; return require("node:stream").Readable.from(["content"]); };
+    for (let i = 0; i < 2; i++) {
+      try { await source.getFileContentCache("file.js", source.data.repoId, () => ({ sha: "sha" })); throw Error("expected storage failure"); }
+      catch (error) { expect(error.message).to.equal("storage unavailable"); }
+    }
+    expect(attempts).to.equal(2);
+  });
+  it("returns a client error from the real options route for an inaccessible source", async () => {
+    const utils = require("../src/server/routes/route-utils");
+    const repo = { assertNotArchived() {}, status: "error", options: {},
+      model: { statusDate: new Date(), statusMessage: "repo_not_found", source: {} } };
+    stub(utils, "getRepo", async () => repo); stub(utils, "getUser", async () => { throw Error("anonymous"); });
+    const router = require("../src/server/routes/repository-public").default;
+    const handler = router.stack.find(layer => layer.route?.path === "/:repoId/options").route.stack[0].handle;
+    const res = { header() {}, status(value) { this.statusCode = value; return this; }, json(value) { this.body = value; } };
+    await handler({ params: { repoId: "inaccessible" } }, res);
+    expect(res.statusCode).to.equal(404); expect(res.body).to.deep.equal({ error: "repo_not_found" });
+    repo.model.statusMessage = "unexpected_bug";
+    await handler({ params: { repoId: "inaccessible" } }, res); expect(res.statusCode).to.equal(500);
+  });
+  it("returns a structured 429 and the actual store retry time", () => {
+    const headers = {}, res = { setHeader: (key, value) => { headers[key] = value; }, status(value) { this.statusCode = value; return this; }, json(value) { this.body = value; } };
+    const resetAt = Date.now() + 10000;
+    sendRequestRateLimit({ rateLimit: { resetTime: new Date(resetAt) } }, res);
+    expect(res.statusCode).to.equal(429); expect(res.body).to.deep.equal({ error: "rate_limited", resetAt });
+    expect(Number(headers["Retry-After"])).to.be.within(9, 10);
+  });
+  it("flags native unhandled stream errors and restarts without reporting private log fields", () => {
+    const summary = logSummary();
+    observeLine(summary, "throw er; // Unhandled 'error' event");
+    observeLine(summary, '2026-10-09T00:00:00Z ERROR [process] process fatal error {"code":"process_fatal_error","message":"private data"}');
+    observeLine(summary, '2026-10-09T00:00:00Z INFO [requests] request {"status":502,"outcome":"completed","ms":100,"url":"private"}');
+    expect(summary.fatalErrors).to.equal(2); expect(summary.completed5xx).to.equal(1); expect(JSON.stringify(summary)).not.to.include("private");
+    expect(containerSummary({ Name: "/streamer", RestartCount: 3, State: { Running: true, Health: { Status: "healthy" }, OOMKilled: false, StartedAt: "now" } }).restartCount).to.equal(3);
+  });
+});

test/request-monitoring.test.js

@@ -149,7 +149,7 @@ describe("bounded request monitoring", function () {
         try {
           const input = await source.getFileContentCache("file.txt", source.data.repoId, () => ({ sha: "blob", size: 7 }));
           input.destroy(); res.send("ok");
-        } catch (error) { res.status(502).send(error === upstreamError ? "original error" : "unexpected error"); }
+        } catch (error) { res.status(502).send(error.cause === upstreamError ? "original error" : "unexpected error"); }
       });
       try {
         await listen(); const response = await fetch(server, "/");

test/route-utils-handleError.test.js

@@ -38,6 +38,24 @@ after(function () {
 });
 
 describe("route-utils.handleError", function () {
+  for (const status of [401, 403, 406, 429, 502]) {
+    it(`preserves Octokit status ${status} without exposing upstream messages`, function () {
+      const res = makeRes();
+      handleError(Object.assign(new Error("private upstream details"), { status }), res);
+      expect(res.statusCode).to.equal(status);
+      expect(res.body).to.deep.equal({ error: "internal_error" });
+    });
+  }
+  it("ignores non-error HTTP statuses and rejects invalid status values", function () {
+    for (const status of [200, "406", 999]) {
+      const res = makeRes(); handleError(Object.assign(new Error("unknown"), { status }), res);
+      expect(res.statusCode).to.equal(500);
+    }
+  });
+  it("preserves got response statusCode", function () {
+    const res = makeRes(); handleError(Object.assign(new Error("upstream"), { response: { statusCode: 403 } }), res);
+    expect(res.statusCode).to.equal(403);
+  });
   it("uses error.httpStatus when present", function () {
     const res = makeRes();
     const err = new AnonymousError("boom", { httpStatus: 418 });