feat: let authors leave their bibliography unredacted (#941) (#943)

* feat: let owners leave chosen files unredacted (#941)

Redacting self-citations in a bibliography reveals which papers are the
authors' own. Owners can now pick files whose content is served as
written; file names are still redacted. Patterns follow .gitignore rules.

The form gets a file picker fed by GitHub's tree API, and the terms box
grows with its content and only warns about regex symbols that can stop a
term from matching.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: honour !patterns, keep picked braces literal, preview unredacted README

- Patterns now apply in order and the last match wins, as in .gitignore,
  so "*.bib" then "!private.bib" excludes private.bib instead of
  preserving every other file.
- Picked paths escape glob characters, braces included, with a
  backslash; "[{]" did not survive brace expansion.
- The README preview drops terms when the README is an unredacted file,
  and refreshes when the list changes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: keep rule order when deduplicating unredacted-file patterns

Rules apply in order and the last match wins, so dedupe keeps the last
copy of a repeated pattern instead of the first. The picker also keeps a
plain path in its place when it follows another rule, rather than moving
it to the front as a pick.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: apply unredacted-file rules to folders, as in .gitignore

A rule that matches a folder now covers every file inside it, so "*.bib"
then "!private" keeps private/refs.bib redacted. A trailing "/" limits a
rule to folders.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: folder unchecks win over broad rules; accept short commit hashes

- Unchecking a folder that a broader rule such as "*.bib" still covers
  appends a "!/folder/" exclusion, so its files really go back to being
  redacted.
- The tree endpoint accepts the 6+ character hashes the commit field
  allows and resolves them to a full SHA before reading the tree.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: picker counts folder rules exactly as the server does

Rules ending in "/" stay patterns instead of being read as picked files,
and the checkboxes now run the saved rule list (picks, then patterns)
through the server's matcher, so the dialog cannot show files as
redacted that the server leaves unredacted.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: insert escaped terms literally in "Match as written"

String.replace expanded "$&", "$'" and "$1" in the term, so the fix
could corrupt the line. Splice by index instead.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

test/frontend-regressions.test.js

@@ -5,6 +5,10 @@ const path = require("path");
 const { setImmediate } = require("timers");
 const { URLSearchParams } = require("node:url");
 const source = fs.readFileSync(path.join(__dirname, "../public/script/app.js"), "utf8");
+require("ts-node/register/transpile-only");
+const minimatch = require("minimatch");
+const { compilePreserveMatcher, dedupePreservePatterns } = require("../src/core/preserve-files");
+const { termHints, escapeTermLine } = require("../src/core/term-hints");
 
 function harness(date) {
   const defs = {}, routes = {}, timers = new Map();
@@ -18,6 +22,8 @@ function harness(date) {
     $: () => ({ on() {}, tooltip() {} }),
     setTimeout: fn => { timers.set(++timerId, fn); return timerId; },
     clearTimeout: id => timers.delete(id), setInterval: () => 0, clearInterval() {},
+    markRaw: value => value, globEscape: minimatch.escape, globUnescape: minimatch.unescape,
+    compilePreserveMatcher, dedupePreservePatterns, termHints, escapeTermLine,
   };
   context.createTimers = () => ({ timeout: Object.assign(context.setTimeout, { cancel: context.clearTimeout }), interval: Object.assign(context.setInterval, { cancel: context.clearInterval }) });
   context.createListeners = () => (target, name, callback) => target.addEventListener(name, callback);
@@ -133,6 +139,100 @@ describe("frontend production regressions", function () {
       expect(h.routes[route].preserveExplorer).to.equal(false);
     }
   });
+  it("saves picked files as anchored, escaped paths and folders as patterns (#941)", async function () {
+    const h = harness(); const timeout = () => 0; timeout.cancel = () => {};
+    h.context.document.getElementById = () => null;
+    h.defs.anonymizeController(h.scope, h.http, {}, {}, {}, () => {}, timeout);
+    h.scope.options.preserveFiles = ["*.bib"];
+    h.scope.source = { commit: "a".repeat(40) };
+    h.scope.sourceUrl = "https://github.com/owner/repo";
+    h.scope.openPreserveFiles();
+    const request = h.requests.at(-1);
+    expect(request.url).to.equal("/api/repo/owner/repo/tree");
+    expect(request.body.params.commit).to.equal("a".repeat(40));
+    request.resolve({ data: { files: ["configs/a.yaml", "configs/b.yaml", "paper/[draft].tex", "paper/refs.bib"], truncated: false } });
+    await h.flush();
+    expect(h.scope.preserveCount()).to.equal(1);
+    h.scope.togglePreserveFile("paper/[draft].tex");
+    h.scope.togglePreserveFolder("configs");
+    expect(h.scope.preserveCount()).to.equal(4);
+    h.scope.savePreserveFiles();
+    expect(Array.from(h.scope.options.preserveFiles)).to.deep.equal(["/paper/\\[draft\\].tex", "*.bib", "/configs/**"]);
+    expect(h.scope.preserveSummary()).to.equal("paper/[draft].tex, *.bib and 1 more");
+  });
+  it("lets a later !pattern exclude a picked file, and keeps braces literal (#941)", async function () {
+    const h = harness(); const timeout = () => 0; timeout.cancel = () => {};
+    h.context.document.getElementById = () => null;
+    h.defs.anonymizeController(h.scope, h.http, {}, {}, {}, () => {}, timeout);
+    h.scope.options.preserveFiles = ["/paper/\\{a,b\\}.bib", "!a.bib"];
+    h.scope.source = { commit: "a".repeat(40) };
+    h.scope.sourceUrl = "https://github.com/owner/repo";
+    h.scope.openPreserveFiles();
+    expect(h.scope.preserve.picked["paper/{a,b}.bib"]).to.equal(true);
+    h.requests.at(-1).resolve({ data: { files: ["a.bib", "paper/a.bib", "paper/{a,b}.bib"], truncated: false } });
+    await h.flush();
+    expect(h.scope.preserveCount()).to.equal(1);
+    h.scope.togglePreserveFile("a.bib");
+    expect(h.scope.preserveCount()).to.equal(1);
+  });
+  it("keeps folder rules as patterns and counts them like the server (#941)", async function () {
+    const h = harness(); const timeout = () => 0; timeout.cancel = () => {};
+    h.context.document.getElementById = () => null;
+    h.defs.anonymizeController(h.scope, h.http, {}, {}, {}, () => {}, timeout);
+    h.scope.options.preserveFiles = ["/paper/", "/refs.bib"];
+    h.scope.source = { commit: "abc123" };
+    h.scope.sourceUrl = "https://github.com/owner/repo";
+    h.scope.openPreserveFiles();
+    expect(Object.keys(h.scope.preserve.picked)).to.deep.equal([]);
+    expect(h.scope.preserve.patterns).to.equal("/paper/\n/refs.bib");
+    h.requests.at(-1).resolve({ data: { files: ["paper/a.tex", "paper/b/c.tex", "refs.bib", "x.tex"], truncated: false } });
+    await h.flush();
+    expect(h.scope.preserveCount()).to.equal(3);
+    const paper = h.scope.preserveRows().find((row) => row.path === "paper");
+    expect(paper.state).to.equal("all");
+  });
+  it("unchecking a folder overrides broader rules (#941)", async function () {
+    const h = harness(); const timeout = () => 0; timeout.cancel = () => {};
+    h.context.document.getElementById = () => null;
+    h.defs.anonymizeController(h.scope, h.http, {}, {}, {}, () => {}, timeout);
+    h.scope.options.preserveFiles = ["*.bib"];
+    h.scope.source = { commit: "abc123" };
+    h.scope.sourceUrl = "https://github.com/owner/repo";
+    h.scope.openPreserveFiles();
+    h.requests.at(-1).resolve({ data: { files: ["paper/refs.bib", "refs.bib"], truncated: false } });
+    await h.flush();
+    expect(h.scope.preserveCount()).to.equal(2);
+    h.scope.togglePreserveFolder("paper");
+    expect(h.scope.preserveCount()).to.equal(1);
+    h.scope.togglePreserveFolder("paper");
+    expect(h.scope.preserveCount()).to.equal(2);
+    h.scope.togglePreserveFolder("paper");
+    h.scope.savePreserveFiles();
+    expect(Array.from(h.scope.options.preserveFiles)).to.deep.equal(["*.bib", "!/paper/"]);
+  });
+  it("keeps the order of saved rules when the picker opens and saves (#941)", function () {
+    const h = harness(); const timeout = () => 0; timeout.cancel = () => {};
+    h.context.document.getElementById = () => null;
+    h.defs.anonymizeController(h.scope, h.http, {}, {}, {}, () => {}, timeout);
+    h.scope.options.preserveFiles = ["/a.bib", "*.bib", "!/refs.bib", "/refs.bib"];
+    h.scope.source = { commit: "a".repeat(40) };
+    h.scope.sourceUrl = "https://github.com/owner/repo";
+    h.scope.openPreserveFiles();
+    expect(Object.keys(h.scope.preserve.picked)).to.deep.equal(["a.bib"]);
+    expect(h.scope.preserve.patterns).to.equal("*.bib\n!/refs.bib\n/refs.bib");
+    h.scope.savePreserveFiles();
+    expect(Array.from(h.scope.options.preserveFiles)).to.deep.equal(["/a.bib", "*.bib", "!/refs.bib", "/refs.bib"]);
+  });
+  it("shows the README unredacted in the preview when it is an unredacted file (#941)", function () {
+    const h = harness(); const timeout = () => 0; timeout.cancel = () => {};
+    h.defs.anonymizeController(h.scope, h.http, {}, {}, {}, () => {}, timeout);
+    h.scope.detectedType = "repo";
+    h.scope.options.preserveFiles = ["*.bib"];
+    expect(h.scope.readmePreserved()).to.equal(false);
+    h.scope.options.preserveFiles = ["/README.md"];
+    expect(h.scope.readmePreserved()).to.equal(true);
+    expect(typeof h.watches["options.preserveFiles"]).to.equal("function");
+  });
   it("includes PR comment authors and bodies in the preview batch", async function () {
     const h = harness(); const pending = new Map(); let id = 0;
     const timeout = fn => { pending.set(++id, fn); return id; }; timeout.cancel = id => pending.delete(id);

test/maintenance-api.test.js

@@ -114,6 +114,14 @@ describe("maintenance settings patch", () => {
       "options.terms": [], "options.update": false,
     });
   });
+  it("normalizes preserveFiles patterns (#941)", () => {
+    expect(maintenanceSettings({ preserveFiles: [" refs.bib ", ""] })).to.deep.equal({
+      "options.preserveFiles": ["refs.bib"],
+    });
+    for (const body of [{ preserveFiles: "refs.bib" }, { preserveFiles: null }, { preserveFiles: ["../x"] }]) {
+      expect(() => maintenanceSettings(body)).to.throw("invalid_settings");
+    }
+  });
   it("rejects source changes, arbitrary option paths, invalid types and oversized terms", () => {
     for (const body of [null, [], {}, { source: {} }, { "options.update": true },
       { image: "false" }, { terms: [1] }, { terms: [""] }, { terms: ["a".repeat(1001)] },

test/preserve-files.test.js

@@ -0,0 +1,133 @@
+const { expect } = require("chai");
+require("ts-node/register/transpile-only");
+const {
+  isPreservedPath,
+  normalizePreserveFiles,
+} = require("../src/core/preserve-files");
+const { AnonymizeTransformer } = require("../src/core/anonymize-utils");
+const { decisionKey } = require("../src/core/cached-response");
+
+function run(options, input) {
+  const transformer = new AnonymizeTransformer(options);
+  const chunks = [];
+  return new Promise((resolve, reject) => {
+    transformer.on("data", (chunk) => chunks.push(Buffer.from(chunk)));
+    transformer.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")));
+    transformer.on("error", reject);
+    transformer.end(Buffer.from(input));
+  });
+}
+
+describe("preserveFiles (#941)", function () {
+  describe("isPreservedPath", function () {
+    it("matches a bare name or glob in any folder", function () {
+      expect(isPreservedPath("refs.bib", ["refs.bib"])).to.equal(true);
+      expect(isPreservedPath("paper/refs.bib", ["refs.bib"])).to.equal(true);
+      expect(isPreservedPath("a/b/other.bib", ["*.bib"])).to.equal(true);
+      expect(isPreservedPath("paper/refs.tex", ["*.bib"])).to.equal(false);
+    });
+
+    it("anchors patterns containing a slash at the repository root", function () {
+      expect(isPreservedPath("paper/refs.bib", ["paper/refs.bib"])).to.equal(true);
+      expect(isPreservedPath("x/paper/refs.bib", ["paper/refs.bib"])).to.equal(false);
+      expect(isPreservedPath("paper/sub/a.bib", ["paper/**"])).to.equal(true);
+    });
+
+    it("anchors a leading slash at the root, like .gitignore", function () {
+      expect(isPreservedPath("refs.bib", ["/refs.bib"])).to.equal(true);
+      expect(isPreservedPath("paper/refs.bib", ["/refs.bib"])).to.equal(false);
+      expect(isPreservedPath("paper/refs.bib", ["/paper/refs.bib"])).to.equal(true);
+    });
+
+    it("matches picked paths escaped with backslashes, braces included", function () {
+      expect(isPreservedPath("paper/[draft].bib", ["/paper/\\[draft\\].bib"])).to.equal(true);
+      expect(isPreservedPath("paper/d.bib", ["/paper/\\[draft\\].bib"])).to.equal(false);
+      expect(isPreservedPath("paper/{draft,final}.bib", ["/paper/\\{draft,final\\}.bib"])).to.equal(true);
+      expect(isPreservedPath("paper/draft.bib", ["/paper/\\{draft,final\\}.bib"])).to.equal(false);
+    });
+
+    it("applies folder rules to everything inside, like .gitignore", function () {
+      expect(isPreservedPath("private/refs.bib", ["*.bib", "!private"])).to.equal(false);
+      expect(isPreservedPath("paper/private/refs.bib", ["*.bib", "!private/"])).to.equal(false);
+      expect(isPreservedPath("paper/refs.bib", ["*.bib", "!private"])).to.equal(true);
+      expect(isPreservedPath("paper/a/b.tex", ["paper"])).to.equal(true);
+      expect(isPreservedPath("x/paper/b.tex", ["/paper"])).to.equal(false);
+      expect(isPreservedPath("private.bib", ["*.bib", "!private/"])).to.equal(true);
+      expect(isPreservedPath("private/x.bib", ["/private/x.bib", "!/private/"])).to.equal(false);
+    });
+
+    it("applies !patterns as exclusions, last match wins", function () {
+      const patterns = ["*.bib", "!private.bib"];
+      expect(isPreservedPath("paper/refs.bib", patterns)).to.equal(true);
+      expect(isPreservedPath("paper/private.bib", patterns)).to.equal(false);
+      expect(isPreservedPath("README.md", patterns)).to.equal(false);
+      expect(isPreservedPath("paper/private.bib", ["!private.bib", "*.bib"])).to.equal(true);
+      expect(isPreservedPath("a/x.bib", ["a/**", "!a/*.bib"])).to.equal(false);
+      expect(isPreservedPath("#notes.txt", ["#notes.txt"])).to.equal(true);
+    });
+
+    it("matches dotfiles and ignores empty input", function () {
+      expect(isPreservedPath(".zenodo.json", ["*.json"])).to.equal(true);
+      expect(isPreservedPath("refs.bib", [])).to.equal(false);
+      expect(isPreservedPath("", ["*"])).to.equal(false);
+      expect(isPreservedPath("refs.bib", undefined)).to.equal(false);
+    });
+  });
+
+  describe("normalizePreserveFiles", function () {
+    it("trims, drops blanks and duplicates", function () {
+      expect(normalizePreserveFiles([" refs.bib ", "", "refs.bib", "*.bib"]))
+        .to.deep.equal(["refs.bib", "*.bib"]);
+    });
+
+    it("keeps the last copy of a repeated rule so the result does not change", function () {
+      const raw = ["!a.bib", "*.bib", "!a.bib"];
+      const normalized = normalizePreserveFiles(raw);
+      expect(normalized).to.deep.equal(["*.bib", "!a.bib"]);
+      expect(isPreservedPath("a.bib", normalized)).to.equal(isPreservedPath("a.bib", raw));
+      expect(isPreservedPath("a.bib", normalized)).to.equal(false);
+      expect(normalizePreserveFiles(undefined)).to.deep.equal([]);
+      expect(normalizePreserveFiles(["!private/", "paper/"])).to.deep.equal(["!private/", "paper/"]);
+      expect(normalizePreserveFiles(["/refs.bib", "!private.bib", "/a\\{b\\}"])).to.deep.equal(["/refs.bib", "!private.bib", "/a\\{b\\}"]);
+    });
+
+    it("rejects invalid input", function () {
+      expect(normalizePreserveFiles("refs.bib")).to.equal(null);
+      expect(normalizePreserveFiles([1])).to.equal(null);
+      expect(normalizePreserveFiles(["//x"])).to.equal(null);
+      expect(normalizePreserveFiles(["/"])).to.equal(null);
+      expect(normalizePreserveFiles(["../x"])).to.equal(null);
+      expect(normalizePreserveFiles(["!"])).to.equal(null);
+      expect(normalizePreserveFiles(["!/"])).to.equal(null);
+      expect(normalizePreserveFiles(["//"])).to.equal(null);
+      expect(normalizePreserveFiles(["!../x"])).to.equal(null);
+      expect(normalizePreserveFiles(["x".repeat(513)])).to.equal(null);
+      expect(normalizePreserveFiles(Array.from({ length: 201 }, (_, i) => `f${i}`))).to.equal(null);
+    });
+  });
+
+  describe("AnonymizeTransformer", function () {
+    const bib = "@article{smith2024, author = {Jane Smith}}";
+
+    it("leaves matching files unredacted", async function () {
+      const out = await run({ filePath: "paper/refs.bib", terms: ["Smith"], preserveFiles: ["*.bib"] }, bib);
+      expect(out).to.equal(bib);
+    });
+
+    it("still redacts files that do not match", async function () {
+      const out = await run({ filePath: "paper/main.tex", terms: ["Smith"], preserveFiles: ["*.bib"] }, "by Jane Smith");
+      expect(out).to.equal("by Jane XXXX-1");
+    });
+
+    it("keeps the full options for re-evaluation downstream", function () {
+      const t = new AnonymizeTransformer({ filePath: "refs.bib", terms: ["Smith"], preserveFiles: ["refs.bib"] });
+      expect(t.opt.terms).to.deep.equal(["Smith"]);
+    });
+  });
+
+  it("changes cache keys when the pattern list changes", function () {
+    const base = { filePath: "refs.bib", terms: ["Smith"] };
+    expect(decisionKey("d", { ...base, preserveFiles: [] }))
+      .to.not.equal(decisionKey("d", { ...base, preserveFiles: ["refs.bib"] }));
+  });
+});

test/source-file-list.test.js

@@ -0,0 +1,57 @@
+const { expect } = require("chai");
+require("ts-node/register/transpile-only");
+const GitHubUtils = require("../src/core/GitHubUtils");
+const { sourceFileList, SOURCE_FILE_LIST_LIMIT } = require("../src/core/source-file-list");
+
+describe("sourceFileList (#941)", function () {
+  const original = GitHubUtils.octokit;
+  let response;
+  let request;
+  beforeEach(() => {
+    GitHubUtils.octokit = () => ({
+      repos: { getCommit: async ({ ref }) => ({ data: { sha: ref.padEnd(40, "0") } }) },
+      git: { getTree: async (params) => {
+        request = params;
+        if (response instanceof Error) throw response;
+        return { data: response };
+      } },
+    });
+  });
+  afterEach(() => { GitHubUtils.octokit = original; });
+
+  const opt = { owner: "o", repo: "r", commit: "a".repeat(40), accessToken: "t" };
+
+  it("lists text files of the commit, sorted, without folders, submodules or binaries", async function () {
+    response = { truncated: false, tree: [
+      { type: "tree", path: "paper" },
+      { type: "blob", path: "paper/refs.bib" },
+      { type: "blob", path: "README.md" },
+      { type: "blob", path: "fig.png" },
+      { type: "blob", path: ".gitmodules" },
+      { type: "commit", path: "vendor/lib" },
+    ] };
+    expect(await sourceFileList(opt)).to.deep.equal({ files: ["README.md", "paper/refs.bib"], truncated: false });
+    expect(request).to.include({ owner: "o", repo: "r", tree_sha: "a".repeat(40), recursive: "true" });
+  });
+
+  it("resolves an abbreviated commit hash before reading the tree", async function () {
+    response = { truncated: false, tree: [] };
+    await sourceFileList({ ...opt, commit: "abc123" });
+    expect(request.tree_sha).to.equal("abc123".padEnd(40, "0"));
+  });
+
+  it("reports GitHub truncation and caps the list", async function () {
+    response = { truncated: true, tree: [{ type: "blob", path: "a.txt" }] };
+    expect((await sourceFileList(opt)).truncated).to.equal(true);
+    response = { truncated: false, tree: Array.from({ length: SOURCE_FILE_LIST_LIMIT + 1 }, (_, i) => ({ type: "blob", path: `f${i}.txt` })) };
+    const capped = await sourceFileList(opt);
+    expect(capped.files).to.have.length(SOURCE_FILE_LIST_LIMIT);
+    expect(capped.truncated).to.equal(true);
+  });
+
+  it("maps a missing repository or commit to repo_not_found", async function () {
+    response = Object.assign(new Error("Not Found"), { status: 404 });
+    try { await sourceFileList(opt); expect.fail("should throw"); }
+    catch (error) { expect(error.message).to.equal("repo_not_found"); }
+  });
+});

test/term-hints.test.js

@@ -0,0 +1,31 @@
+const { expect } = require("chai");
+require("ts-node/register/transpile-only");
+const { termHints, escapeTermLine } = require("../src/core/term-hints");
+
+describe("term hints", function () {
+  it("ignores dots, hyphens and emails", function () {
+    expect(termHints("mchen-research\nhartwell.edu\nmaya.chen@cs.example.edu\nA, B #1")).to.deep.equal([]);
+  });
+
+  it("flags symbols that change what a term matches", function () {
+    expect(termHints("Maya Chen\nSmith (MIT)\nC++")).to.deep.equal([
+      { line: 1, term: "Smith (MIT)", symbols: ["(", ")"] },
+      { line: 2, term: "C++", symbols: ["+"] },
+    ]);
+  });
+
+  it("skips escaped symbols and the replacement side", function () {
+    expect(termHints("Smith \\(MIT\\)\nAnon=>A (B)")).to.deep.equal([]);
+  });
+
+  it("escapes one line and keeps its replacement", function () {
+    expect(escapeTermLine("a\nSmith (MIT).x=>Anon (1)\nb", 1)).to.equal("a\nSmith \\(MIT\\)\\.x=>Anon (1)\nb");
+    expect(escapeTermLine("C\\+\\+ +", 0)).to.equal("C\\+\\+ \\+");
+  });
+
+  it("inserts terms containing $ sequences literally", function () {
+    expect(escapeTermLine("a$&b (x)", 0)).to.equal("a\\$&b \\(x\\)");
+    expect(escapeTermLine("  $'(y)=>z", 0)).to.equal("  \\$'\\(y\\)=>z");
+    expect(escapeTermLine("$1+", 0)).to.equal("\\$1\\+");
+  });
+});