fix: verify reconnects through supported GitHub routes and protect active jobs

test/anonymizer-large-files.test.js

@@ -0,0 +1,28 @@
+const { expect } = require("chai");
+const { Readable } = require("node:stream");
+require("ts-node/register/transpile-only");
+const { AnonymizeTransformer, ContentAnonimizer } = require("../src/core/anonymize-utils");
+const terms = ["first.last", "user@example.org", "dept.example.edu", "lab.io", "project.org", "Author X. Name",
+  ...Array.from({ length: 41 }, (_, i) => `Researcher${i}`)];
+
+describe("large files with many anonymization terms", function () {
+  this.timeout(10000);
+  it("handles a 50 MB numeric dataset without exhausting the one-second deadline", function () {
+    const input = "123,456,78.91234,-45.123456\n".repeat(1800000);
+    expect(new ContentAnonimizer({ terms }).anonymize(input)).to.equal(input);
+  });
+  it("redacts dotted terms and their URLs around a large payload", async function () {
+    const payload = "123,456,78.91234,-45.123456\n".repeat(200000);
+    const input = Buffer.from(`first.last\n${payload}\nhttps://host/user@example.org/path\nfirstXlast`);
+    const stream = new AnonymizeTransformer({ filePath: "coords.csv", terms });
+    Readable.from([input.subarray(0, 6), input.subarray(6)]).pipe(stream);
+    const chunks = [];
+    for await (const chunk of stream) chunks.push(chunk);
+    expect(Buffer.concat(chunks).toString()).to.equal(`XXXX-1\n${payload}\nXXXX-2\nXXXX-1`);
+  });
+  it("handles many repeated matches while preserving the surrounding values", function () {
+    const input = "Alice,123,45.678,-23.456\n".repeat(100000);
+    expect(new ContentAnonimizer({ terms: ["Alice", ...terms] }).anonymize(input))
+      .to.equal("XXXX-1,123,45.678,-23.456\n".repeat(100000));
+  });
+});

test/app-reinstall-recovery.test.js

@@ -55,11 +55,18 @@ for (const type of ["repository", "pull-request"]) {
       expect(writes[0].update).to.deep.equal({ $set: { githubAccess: selected.binding } });
       expect(writes[0].filter.githubAccess).to.deep.equal(model.githubAccess);
       expect(writes[0].filter.source).to.deep.equal(model.source);
-      expect(checks).to.deep.equal([{ url: type === "repository" ? "/repositories/42/commits/saved-sha" : "/repositories/42/pulls/7", token: "fresh-token" }]);
+      expect(checks).to.deep.equal([{ url: type === "repository" ? "/repos/owner/private/commits/saved-sha" : "/repos/owner/private/pulls/7", token: "fresh-token" }]);
     });
     it("rejects a replacement repository at the same name", async function () {
       selected.binding.repositoryId = 99; await fails("connection_changed"); expect(checks).to.have.length(0);
     });
+    for (const status of ["preparing", "queue", "download", "removing", "expiring"]) {
+      it("rejects reconnect during " + status, async function () {
+        model.status = status;
+        await fails("repository_busy");
+        expect(checks).to.have.length(0);
+      });
+    }
     it("requires access to the existing snapshot before saving", async function () {
       stub(app, "githubRequest", async () => { throw new Error("github_app_access_required"); });
       await fails("github_app_access_required");

test/asset-build.test.js

@@ -66,6 +66,23 @@ describe("asset build", function () {
     }
   });
 
+  it("ships the upstream PDF.js bytes without a second minification pass", function () {
+    const upstream = fs.readFileSync(path.join(path.dirname(gulpfile), groups.pdfFiles[0]));
+    fs.writeFileSync(path.join(directory, groups.pdfFiles[0]), upstream);
+    const result = build();
+    expect(result.status, result.stderr).to.equal(0);
+    const bundled = fs.readFileSync(path.join(directory, "public/script/pdf.min.js"));
+    expect(bundled.equals(upstream)).to.equal(true);
+  });
+
+  it("rejects invalid PDF.js without publishing a manifest", function () {
+    fs.writeFileSync(path.join(directory, groups.pdfFiles[0]), "function {");
+    const result = build();
+    expect(result.status).not.to.equal(0);
+    expect(result.stderr).to.include("SyntaxError");
+    expect(fs.existsSync(path.join(directory, "public/asset-manifest.json"))).to.equal(false);
+  });
+
   it("fails on missing input without publishing a manifest", function () {
     fs.unlinkSync(path.join(directory, groups.coreJsFiles[0]));
     const result = build();
@@ -75,7 +92,7 @@ describe("asset build", function () {
   });
 
   it("fails on invalid JavaScript without publishing a manifest", function () {
-    fs.writeFileSync(path.join(directory, groups.pdfFiles[0]), "function {");
+    fs.writeFileSync(path.join(directory, groups.markdownFiles[0]), "function {");
     const result = build();
     expect(result.status).not.to.equal(0);
     expect(result.stderr).to.include("uglify");

test/github-app.test.js

@@ -335,7 +335,7 @@ describeMongo("GitHub App credential and repository integration", function () {
   it("does not require scoped-token minting for an uninstalled public repository", async () => {
     await app.saveAppGrant(owner.id, data());
     mock(url => url.includes("/user/installations") ? { installations: [] } :
-      { id: 7, private: false, visibility: "public" }, () => ({ status: 403 }));
+      { id: 7, full_name: "other/public", private: false, visibility: "public" }, () => ({ status: 403 }));
     const selected = await app.selectRepositoryAccess(owner.id, "other/public", "github-app");
     expect(selected.token).to.match(/^public-read:/);
     expect(calls.some(call => call.url.endsWith("/token/scoped"))).to.equal(false);
@@ -496,6 +496,29 @@ describeMongo("GitHub App credential and repository integration", function () {
     expect(await Credentials.countDocuments()).to.equal(0);
     expect(await Users.countDocuments()).to.equal(1);
   });
+  it("reconnects an uninstalled public repository with a real user token", async () => {
+    const Repos = require("../src/core/model/anonymizedRepositories/anonymizedRepositories.model").default;
+    await app.saveAppGrant(owner.id, data());
+    const resource = await Repos.create({ repoId: "public-reconnect", owner: owner.id, status: "ready",
+      source: { type: "GitHubStream", repositoryName: "other/public", commit: "abc", branch: "main" },
+      githubAccess: { kind: "github-app", repositoryId: 42, publicRead: true, revision: "old" } });
+    mock((url, options) => {
+      if (url.includes("/user/installations?")) return { installations: [] };
+      if (url.endsWith("/repositories/42") || url.endsWith("/repos/other/public")) return {
+        id: 42, full_name: "other/public", private: false, visibility: "public",
+      };
+      if (url.endsWith("/repos/other/public/commits/abc")) {
+        expect(new globalThis.Headers(options.headers).get("authorization")).to.equal("Bearer ghu_access1");
+        return { sha: "abc" };
+      }
+      throw new Error("Unexpected request " + url);
+    });
+    const body = { type: "repository", id: resource.repoId, connection: "github-app" };
+    expect((await request("/github/connections/migrate", { ...body, preview: true })).status).to.equal(200);
+    expect((await request("/github/connections/migrate", body)).status).to.equal(200);
+    expect((await Repos.findById(resource.id)).githubAccess.publicRead).to.equal(true);
+  });
+
   it("recovers a renamed private pull request through its stable repository ID", async () => {
     const PRs = require("../src/core/model/anonymizedPullRequests/anonymizedPullRequests.model").default;
     await app.saveAppGrant(owner.id, data());
@@ -510,7 +533,7 @@ describeMongo("GitHub App credential and repository integration", function () {
       if (url.endsWith("/repositories/42")) return { id: 42 };
       if (url.endsWith("/app/installations/4")) return { app_id: 123, permissions: { contents: "read", metadata: "read" } };
       if (url.endsWith("/access_tokens")) return { token: "ghs_reinstalled", expires_at: new Date(Date.now() + 3600000).toISOString() };
-      if (url.endsWith("/repositories/42/pulls/7")) return { number: 7 };
+      if (url.endsWith("/repos/owner/new-name/pulls/7")) return { number: 7 };
       throw new Error("Unexpected request " + url);
     });
     const body = { type: "pull-request", id: resource.pullRequestId, connection: "github-app" };

test/term-presence.test.js

@@ -0,0 +1,75 @@
+const { expect } = require("chai");
+require("ts-node/register/transpile-only");
+const { TermPresenceIndex } = require("../src/core/term-presence");
+const { compileTerms, ContentAnonimizer } = require("../src/core/anonymize-utils");
+
+function reference(terms, text) {
+  const anonymizer = new ContentAnonimizer({ terms });
+  anonymizer.termPresence = null;
+  for (const term of anonymizer.compiledTerms) delete term.literalPrefilter;
+  return anonymizer.anonymize(text);
+}
+
+describe("term candidate index", function () {
+  it("retains shared prefixes, nested matches and Unicode-folded alternatives", function () {
+    const patterns = [/Alice/iu, /AliceSmith/iu, /Smith/iu, /[sśŝşšș]/iu, /ſ/iu, /K/iu, /k/iu, /研究/iu];
+    const index = new TermPresenceIndex(patterns);
+    for (const text of ["AliceSmith", "ALICESMITH K ſ 研究", "nothing", "ś", "k", "aAliceSmith"]) {
+      for (let after = -1; after < patterns.length; after++) {
+        const candidates = index.candidates(text, after);
+        patterns.forEach((pattern, i) => {
+          if (i > after && pattern.test(text)) expect(candidates.has(i), `${text}: ${i}`).to.equal(true);
+          if (i <= after) expect(candidates.has(i)).to.equal(false);
+        });
+      }
+    }
+  });
+
+  it("keeps dot's CR and Unicode line-separator matches", function () {
+    for (const text of ["first\rlast", "first\u2028last", "first\u2029last", "first\nlast", "first😀last"]) {
+      expect(new ContentAnonimizer({ terms: ["first.last"] }).anonymize(text)).to.equal(reference(["first.last"], text));
+    }
+  });
+
+  it("retains ordered cascades, URL removal, and matches created across replacement boundaries", function () {
+    const cases = [
+      [["Alice=>Bob", "Bob=>Hidden"], "Alice Bob https://host/Alice"],
+      [["Alice=>", "@Bob=>Hidden"], "@AliceBob"],
+      [["a-a", "Alice"], "xa-a-a Alice https://host/Alice/file"],
+      [["first.last", "firstXlast=>other"], "firstXlast first.last"],
+      [["Alice=>Bob", "Bob"], "https://host/Alice"],
+      [["k", "s"], "K ſ ék k_ xK Kx"],
+      [["Alice=>Bob", "B.b=>Done"], "Alice"],
+      [["k=>Kk"], "https://host/k"],
+      [["Alice=>$&", "Bob=>$$"], "Alice Bob"],
+      [["\u0301"], "abc"],
+      [["a.*b", "Alice"], "axxb Alice"],
+    ];
+    for (const [terms, text] of cases) {
+      expect(new ContentAnonimizer({ terms }).anonymize(text), JSON.stringify([terms, text])).to.equal(reference(terms, text));
+    }
+  });
+
+  it("matches the RE2 path for fixed-width terms and surrounding characters", function () {
+    const terms = ["a-a", "a.a", "k", "s", "k.s", "Alice", "@Alice", "Σ", "研究", "😀", "..", "a.", "a..b"];
+    const edges = ["", "x", "_", "é", "K", "ſ", "😀", "𐐀", "\r", "\n", "\u2028"];
+    for (const term of terms) {
+      const input = edges.flatMap(left => edges.map(right => `${left}${term}${right} ${left}${term.replaceAll(".", "\r")}${right}`)).join(" ");
+      expect(new ContentAnonimizer({ terms: [term] }).anonymize(input), term).to.equal(reference([term], input));
+    }
+  });
+
+  it("does not reuse stale rules after the caller edits its term list", function () {
+    const terms = ["Alice=>first"];
+    expect(new ContentAnonimizer({ terms }).anonymize("Alice")).to.equal("first");
+    terms[0] = "Alice=>second";
+    expect(new ContentAnonimizer({ terms }).anonymize("Alice")).to.equal("second");
+    expect(new ContentAnonimizer({ terms: ["Alice=>first"] }).anonymize("Alice")).to.equal("first");
+  });
+
+  it("does not index quantified, capturing, or escaped user patterns", function () {
+    for (const term of ["a.*b", "a+b", "(Alice)", "a\\.b", "a(?=b)"]) {
+      expect(compileTerms([term]).every(c => !c.literalPrefilter), term).to.equal(true);
+    }
+  });
+});

test/vue-ui.test.js

@@ -155,10 +155,11 @@ describe("Vue 3 UI", function () {
     expect(ui.errors).to.deep.equal([]);
   });
 
-  it("hides connection changes on archived resources", async function () {
+  for (const status of ["archived", "preparing", "queue", "download", "removing", "expiring"]) {
+  it(`hides connection changes on ${status} resources`, async function () {
     ui = await browser("/connections", {
       "/github/connections": { appEnabled: true, appConnected: true, oauthConnected: true,
-        resources: [{ type: "repository", id: "archived", connection: "github-app", status: "archived", eligible: true }] },
+        resources: [{ type: "repository", id: "archived", connection: "github-app", status, eligible: true }] },
     });
     const resource = ui.window.document.querySelector(".connection-resource");
     expect(resource).not.to.equal(null);
@@ -166,6 +167,8 @@ describe("Vue 3 UI", function () {
     expect(ui.errors).to.deep.equal([]);
   });
 
+  }
+
   for (const connection of ["oauth", "github-app"]) {
   it(`previews and saves ${connection} access with CSRF protection`, async function () {
     const action = connection === "github-app" ? "Reconnect read-only access" : "Switch to read-only access";

test/zip-proxy-errors.test.js

@@ -0,0 +1,94 @@
+const { expect } = require("chai");
+const express = require("express");
+const compression = require("compression");
+const { gunzipSync } = require("node:zlib");
+const http = require("node:http");
+const { PassThrough } = require("node:stream");
+const got = require("got");
+require("ts-node/register/transpile-only");
+const config = require("../src/config").default;
+const utils = require("../src/server/routes/route-utils");
+const router = require("../src/server/routes/repository-public").default;
+
+describe("ZIP proxy failures", function () {
+  let restore, upstream, server;
+  beforeEach(async function () {
+    const previous = { download: config.ENABLE_DOWNLOAD, endpoint: config.STREAMER_ENTRYPOINT,
+      getRepo: utils.getRepo, getUser: utils.getUser, stream: got.stream };
+    restore = () => {
+      config.ENABLE_DOWNLOAD = previous.download;
+      config.STREAMER_ENTRYPOINT = previous.endpoint;
+      utils.getRepo = previous.getRepo; utils.getUser = previous.getUser; got.stream = previous.stream;
+    };
+    config.ENABLE_DOWNLOAD = true;
+    config.STREAMER_ENTRYPOINT = "http://streamer.test/";
+    utils.getUser = async () => { throw new Error("anonymous"); };
+    utils.getRepo = async () => ({ repoId: "test", options: {},
+      model: { source: { repositoryName: "owner/repo", commit: "abc" } },
+      countView: async () => {}, getToken: async () => "",
+      generateAnonymizeTransformer: () => ({ opt: {} }),
+    });
+    upstream = new PassThrough();
+    got.stream = () => upstream;
+    const app = express(); app.use(compression({ filter: () => true })); app.use(router);
+    server = await new Promise(resolve => {
+      const listening = app.listen(0, "127.0.0.1", () => resolve(listening));
+    });
+  });
+  afterEach(async function () {
+    upstream.destroy(); server.closeAllConnections();
+    await new Promise(resolve => server.close(resolve)); restore();
+  });
+  function request(onResponse, headers = {}) {
+    return http.get(`http://127.0.0.1:${server.address().port}/test/zip`, { headers }, onResponse);
+  }
+  it("aborts a partial ZIP when the upstream fails after headers", async function () {
+    const result = new Promise((resolve, reject) => {
+      const client = request(response => {
+        expect(response.statusCode).to.equal(200);
+        response.once("data", () => upstream.destroy(new Error("upstream disconnected")));
+        response.on("aborted", resolve);
+        response.on("end", () => reject(new Error("truncated ZIP reported as complete")));
+        response.on("error", () => {});
+      });
+      client.on("error", reject);
+    });
+    upstream.write(Buffer.from("PK partial archive"));
+    await result;
+  });
+  it("finishes compressed JSON errors before any ZIP bytes are sent", async function () {
+    const error = "unavailable".repeat(300);
+    const result = new Promise((resolve, reject) => {
+      request(response => {
+        const chunks = [];
+        response.on("data", chunk => chunks.push(chunk));
+        response.on("error", reject);
+        response.on("end", () => resolve({
+          status: response.statusCode, encoding: response.headers["content-encoding"],
+          body: JSON.parse(gunzipSync(Buffer.concat(chunks)).toString()),
+        }));
+      }, { "Accept-Encoding": "gzip" }).on("error", reject);
+    });
+    upstream.once("newListener", event => {
+      if (event === "error") globalThis.queueMicrotask(() => upstream.destroy(Object.assign(new Error("unavailable"), {
+        response: { statusCode: 503, body: JSON.stringify({ error }) },
+      })));
+    });
+    expect(await result).to.deep.equal({ status: 502, encoding: "gzip", body: { error } });
+  });
+  it("keeps JSON errors before any ZIP bytes are sent", async function () {
+    const result = new Promise((resolve, reject) => {
+      request(response => {
+        let body = ""; response.on("data", chunk => { body += chunk; });
+        response.on("end", () => resolve({ status: response.statusCode, body: JSON.parse(body) }));
+      }).on("error", reject);
+    });
+    // Fail only after the route has attached its upstream listener.
+    upstream.once("newListener", event => {
+      if (event === "error") globalThis.queueMicrotask(() => upstream.destroy(Object.assign(new Error("unavailable"), {
+        response: { statusCode: 503, body: '{"error":"zip_not_available"}' },
+      })));
+    });
+    expect(await result).to.deep.equal({ status: 502, body: { error: "zip_not_available" } });
+  });
+});