feat: let authors leave their bibliography unredacted (#941) (#943)

* feat: let owners leave chosen files unredacted (#941)

Redacting self-citations in a bibliography reveals which papers are the
authors' own. Owners can now pick files whose content is served as
written; file names are still redacted. Patterns follow .gitignore rules.

The form gets a file picker fed by GitHub's tree API, and the terms box
grows with its content and only warns about regex symbols that can stop a
term from matching.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: honour !patterns, keep picked braces literal, preview unredacted README

- Patterns now apply in order and the last match wins, as in .gitignore,
  so "*.bib" then "!private.bib" excludes private.bib instead of
  preserving every other file.
- Picked paths escape glob characters, braces included, with a
  backslash; "[{]" did not survive brace expansion.
- The README preview drops terms when the README is an unredacted file,
  and refreshes when the list changes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: keep rule order when deduplicating unredacted-file patterns

Rules apply in order and the last match wins, so dedupe keeps the last
copy of a repeated pattern instead of the first. The picker also keeps a
plain path in its place when it follows another rule, rather than moving
it to the front as a pick.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: apply unredacted-file rules to folders, as in .gitignore

A rule that matches a folder now covers every file inside it, so "*.bib"
then "!private" keeps private/refs.bib redacted. A trailing "/" limits a
rule to folders.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: folder unchecks win over broad rules; accept short commit hashes

- Unchecking a folder that a broader rule such as "*.bib" still covers
  appends a "!/folder/" exclusion, so its files really go back to being
  redacted.
- The tree endpoint accepts the 6+ character hashes the commit field
  allows and resolves them to a full SHA before reading the tree.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: picker counts folder rules exactly as the server does

Rules ending in "/" stay patterns instead of being read as picked files,
and the checkboxes now run the saved rule list (picks, then patterns)
through the server's matcher, so the dialog cannot show files as
redacted that the server leaves unredacted.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: insert escaped terms literally in "Match as written"

String.replace expanded "$&", "$'" and "$1" in the term, so the fix
could corrupt the line. Splice by index instead.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

src/core/Repository.ts

@@ -610,6 +610,7 @@ export default class Repository {
       filePath: filePath,
       cacheGeneration: `${this.model.treeGeneration || "legacy"}:${this.model.anonymizeDate?.toISOString() || ""}`,
       terms: this.options.terms,
+      preserveFiles: this.options.preserveFiles || [],
       cacheRevision: this.model.contentCacheRevision,
       image: this.options.image,
       link: this.options.link,

src/core/anonymize-utils.ts

@@ -16,6 +16,7 @@ import { lookup as lookupMime } from "mime-types";
 
 import config from "../config";
 import { TermPresenceIndex } from "./term-presence";
+import { isPreservedPath } from "./preserve-files";
 import {
   parseTermSpec,
   termVariants,
@@ -154,6 +155,11 @@ export class AnonymizeTransformer extends Transform {
   private spoolOperation?: Promise<void>;
   private readonly memoryThreshold = 256 * 1024;
   private readonly bufferWholeStream: boolean;
+  // Options actually applied to the content. Identical to `opt` except that
+  // files matching `preserveFiles` drop their terms (#941). `opt` keeps the
+  // full configuration because it is forwarded to the streamer and zip
+  // builder, which re-evaluate it per file.
+  private readonly contentOpt: ConstructorParameters<typeof ContentAnonimizer>[0];
 
   constructor(
     readonly opt: {
@@ -161,9 +167,11 @@ export class AnonymizeTransformer extends Transform {
       cacheGeneration?: string;
       cacheRevision?: string;
       decisionManaged?: boolean;
+      preserveFiles?: string[];
     } & ConstructorParameters<typeof ContentAnonimizer>[0]
   ) {
     super();
+    this.contentOpt = isPreservedPath(opt.filePath, opt.preserveFiles) ? { ...opt, terms: [] } : opt;
     // Tri-state: name-based check returns true (known text), false (known
     // binary), or null (name inconclusive). For null we defer to a content
     // sniff on the first chunk in _transform — known binary extensions
@@ -171,14 +179,14 @@ export class AnonymizeTransformer extends Transform {
     // reach the sniff path (#493).
     this.nameVerdict = classifyByName(this.opt.filePath);
     if (this.nameVerdict !== null) this.isText = this.nameVerdict;
-    this.anonimizer = new ContentAnonimizer(this.opt);
+    this.anonimizer = new ContentAnonimizer(this.contentOpt);
 
     // Matching a prefix in isolation can split even a fixed-length name or
     // turn the split into a false word boundary. Preserve the complete text
     // whenever a rewrite is enabled, with MAX_FILE_SIZE as a hard limit.
     this.bufferWholeStream =
       opt.link === false || opt.image === false ||
-      (opt.terms || []).length > 0 || !!(opt.repoName && opt.branchName);
+      (this.contentOpt.terms || []).length > 0 || !!(opt.repoName && opt.branchName);
   }
 
   get wasAnonimized() {
@@ -247,8 +255,8 @@ export class AnonymizeTransformer extends Transform {
       const output = join(this.spool, "output");
       const started = performance.now();
       const produce = () => anonymizeOnWorker({ input: join(this.spool!, "input"), output,
-        options: this.opt, maxOutput: config.MAX_FILE_SIZE * 4, context: { mask: config.ANONYMIZATION_MASK, hostname: config.APP_HOSTNAME } }, this.wholeBytes, this.cancelled.signal);
-      void (this.opt.decisionManaged ? produce() : transformedFile(this.digest.digest("hex"), this.opt, output, produce))
+        options: this.contentOpt, maxOutput: config.MAX_FILE_SIZE * 4, context: { mask: config.ANONYMIZATION_MASK, hostname: config.APP_HOSTNAME } }, this.wholeBytes, this.cancelled.signal);
+      void (this.opt.decisionManaged ? produce() : transformedFile(this.digest.digest("hex"), this.contentOpt, output, produce))
         .then(changed => {
           if (this.destroyed) return;
           this.anonimizer.wasAnonymized = changed;

src/core/model/anonymizedRepositories/anonymizedRepositories.schema.ts

@@ -76,6 +76,7 @@ const AnonymizedRepositorySchema = new Schema({
   },
   options: {
     terms: [String],
+    preserveFiles: { type: [String], default: undefined },
     expirationMode: { type: String },
     expirationDate: Date,
     update: Boolean,

src/core/model/anonymizedRepositories/anonymizedRepositories.types.ts

@@ -49,6 +49,8 @@ export interface IAnonymizedRepository {
   conference: string;
   options: {
     terms: string[];
+    /** Glob patterns of files whose content skips term redaction (#941). */
+    preserveFiles?: string[];
     expirationMode: "never" | "redirect" | "remove";
     expirationDate?: Date;
     update: boolean;

src/core/preserve-files.ts

@@ -0,0 +1,89 @@
+import { Minimatch } from "minimatch";
+
+/**
+ * Patterns for files whose content is served without term redaction (#941),
+ * e.g. a bibliography where redacted self-citations would reveal the authors.
+ * Paths are still anonymized; only file content is left untouched.
+ *
+ * Same rules as .gitignore: a pattern without "/" matches the file name in any
+ * folder (`*.bib`); a pattern containing "/" matches from the repository root
+ * (`paper/refs.bib`, `/refs.bib`, `paper/**`); a pattern that matches a
+ * folder covers everything inside it, and a trailing "/" matches folders only
+ * (`private/`); `!pattern` excludes again, and the last matching pattern wins. Picked files are stored as root-anchored
+ * paths with glob characters escaped by a backslash (`/paper/\{a,b\}.bib`).
+ */
+export const MAX_PRESERVE_PATTERNS = 200;
+export const MAX_PRESERVE_PATTERN_LENGTH = 512;
+
+function parsePattern(pattern: string) {
+  const negated = pattern.startsWith("!");
+  const rest = negated ? pattern.slice(1) : pattern;
+  const anchored = rest.startsWith("/");
+  const withSlash = anchored ? rest.slice(1) : rest;
+  const directoryOnly = withSlash.endsWith("/");
+  const body = directoryOnly ? withSlash.slice(0, -1) : withSlash;
+  return { negated, anchored, directoryOnly, body };
+}
+
+/**
+ * Drops repeated patterns but keeps the last copy: rules apply in order and
+ * the last match wins, so an earlier copy can never decide the result.
+ */
+export function dedupePreservePatterns(patterns: string[]): string[] {
+  return patterns.filter((pattern, index) => patterns.lastIndexOf(pattern) === index);
+}
+
+/** Returns the normalized patterns, or null when the input is invalid. */
+export function normalizePreserveFiles(value: unknown): string[] | null {
+  if (value === undefined || value === null) return [];
+  if (!Array.isArray(value)) return null;
+  const patterns: string[] = [];
+  for (const raw of value) {
+    if (typeof raw !== "string") return null;
+    const pattern = raw.trim();
+    if (!pattern) continue;
+    if (pattern.length > MAX_PRESERVE_PATTERN_LENGTH) return null;
+    const { body } = parsePattern(pattern);
+    if (!body || body.split("/").some(segment => segment === ".." || segment === "")) return null;
+    patterns.push(pattern);
+  }
+  const unique = dedupePreservePatterns(patterns);
+  return unique.length > MAX_PRESERVE_PATTERNS ? null : unique;
+}
+
+/**
+ * Compiles the patterns once. The returned function applies them in order to
+ * `initial` (whether the file is already preserved), so a later `!pattern`
+ * can exclude what an earlier pattern included.
+ */
+export function compilePreserveMatcher(patterns: string[] | undefined): (filePath: string, initial?: boolean) => boolean {
+  const rules = (patterns || [])
+    .filter((pattern): pattern is string => typeof pattern === "string" && pattern.length > 0)
+    .map(pattern => {
+      const { negated, anchored, directoryOnly, body } = parsePattern(pattern);
+      return {
+        negated,
+        directoryOnly,
+        matcher: new Minimatch(body, { dot: true, nonegate: true, nocomment: true, matchBase: !anchored && !body.includes("/") }),
+      };
+    });
+  return (filePath: string, initial = false) => {
+    const path = (filePath || "").replace(/^\/+/, "");
+    if (!path) return false;
+    // A rule applies to a file when it matches the file or one of its folders.
+    const parts = path.split("/");
+    const folders = parts.slice(0, -1).map((_, i) => parts.slice(0, i + 1).join("/"));
+    let preserved = initial;
+    for (const rule of rules) {
+      if (rule.negated ? !preserved : preserved) continue;
+      const candidates = rule.directoryOnly ? folders : [...folders, path];
+      if (candidates.some(candidate => rule.matcher.match(candidate))) preserved = !rule.negated;
+    }
+    return preserved;
+  };
+}
+
+export function isPreservedPath(filePath: string | undefined, patterns: string[] | undefined): boolean {
+  if (!filePath || !patterns || patterns.length === 0) return false;
+  return compilePreserveMatcher(patterns)(filePath);
+}

src/core/source-file-list.ts

@@ -0,0 +1,53 @@
+import { octokit } from "./GitHubUtils";
+import AnonymousError from "./AnonymousError";
+import { classifyByName } from "./anonymize-utils";
+
+/** Enough for the picker; larger repositories fall back to patterns. */
+export const SOURCE_FILE_LIST_LIMIT = 20000;
+
+export interface SourceFileList {
+  files: string[];
+  truncated: boolean;
+}
+
+/**
+ * Text files of a commit, read from GitHub's recursive tree API. Used by the
+ * "Leave files unredacted" picker (#941) before the repository is downloaded.
+ * Known binary files are dropped: their content is never redacted anyway.
+ */
+export async function sourceFileList(opt: {
+  owner: string;
+  repo: string;
+  commit: string;
+  accessToken: string;
+}): Promise<SourceFileList> {
+  const oct = octokit(opt.accessToken);
+  let data;
+  try {
+    // The commit field accepts abbreviated hashes; the tree API needs a full one.
+    let commit = opt.commit;
+    if (!/^[0-9a-f]{40}$/i.test(commit)) {
+      commit = (await oct.repos.getCommit({ owner: opt.owner, repo: opt.repo, ref: commit })).data.sha;
+    }
+    ({ data } = await oct.git.getTree({
+      owner: opt.owner,
+      repo: opt.repo,
+      tree_sha: commit,
+      recursive: "true",
+    }));
+  } catch (error) {
+    const status = (error as { status?: number }).status;
+    if (status === 404 || status === 409 || status === 422) {
+      throw new AnonymousError("repo_not_found", { httpStatus: 404, cause: error as Error });
+    }
+    throw error;
+  }
+  const files: string[] = [];
+  for (const entry of data.tree) {
+    if (entry.type !== "blob" || !entry.path) continue;
+    if (entry.path === ".gitmodules" || classifyByName(entry.path) === false) continue;
+    if (files.length === SOURCE_FILE_LIST_LIMIT) return { files: files.sort(), truncated: true };
+    files.push(entry.path);
+  }
+  return { files: files.sort(), truncated: !!data.truncated };
+}

src/core/term-hints.ts

@@ -0,0 +1,48 @@
+import { parseTermSpec } from "./term-matching";
+
+// Symbols that change what a term matches. "." is left out on purpose: it
+// matches any character, which can only redact more (hartwell.edu also hides
+// hartwell-edu), never less, so warning about every email and domain was
+// noise. "-", "," and "#" are literal outside a character class.
+const STRUCTURAL = new Set(["*", "+", "?", "(", ")", "[", "]", "{", "}", "|", "^", "$"]);
+
+export interface TermHint {
+  /** Zero-based line in the terms text. */
+  line: number;
+  term: string;
+  symbols: string[];
+}
+
+/** Terms whose regex symbols probably were meant literally, e.g. "Smith (MIT)" or "C++". */
+export function termHints(text: string | undefined): TermHint[] {
+  const hints: TermHint[] = [];
+  String(text || "").split("\n").forEach((line, index) => {
+    const { term } = parseTermSpec(line.trim());
+    if (!term) return;
+    const symbols: string[] = [];
+    for (let i = 0; i < term.length; i++) {
+      if (term[i] === "\\") { i++; continue; }
+      if (STRUCTURAL.has(term[i]) && !symbols.includes(term[i])) symbols.push(term[i]);
+    }
+    if (symbols.length) hints.push({ line: index, term, symbols });
+  });
+  return hints;
+}
+
+/** Rewrites one line so its term matches the text as written; keeps any "=>" replacement. */
+export function escapeTermLine(text: string, line: number): string {
+  const lines = String(text || "").split("\n");
+  const raw = lines[line];
+  if (raw === undefined) return text;
+  const { term } = parseTermSpec(raw.trim());
+  if (!term) return text;
+  let escaped = "";
+  for (let i = 0; i < term.length; i++) {
+    if (term[i] === "\\") { escaped += term.slice(i, i + 2); i++; continue; }
+    escaped += STRUCTURAL.has(term[i]) || term[i] === "." ? "\\" + term[i] : term[i];
+  }
+  // Splice by index: String.replace would expand "$&", "$'" and "$1" in the term.
+  const start = raw.indexOf(term);
+  lines[line] = raw.slice(0, start) + escaped + raw.slice(start + term.length);
+  return lines.join("\n");
+}

src/server/routes/maintenance.ts

@@ -1,6 +1,7 @@
 import * as express from "express";
 import { getRepo, getUser, handleError, isOwnerCoauthorOrAdmin } from "./route-utils";
 import AnonymousError from "../../core/AnonymousError";
+import { normalizePreserveFiles } from "../../core/preserve-files";
 import RepoModel from "../../core/model/anonymizedRepositories/anonymizedRepositories.model";
 
 const router = express.Router();
@@ -17,6 +18,11 @@ export function maintenanceSettings(body: unknown): Record<string, unknown> {
     if (key === "terms") {
       if (!Array.isArray(value) || value.length > 1000 ||
         !value.every(t => typeof t === "string" && t.length > 0 && t.length <= 1000)) throw invalid();
+    } else if (key === "preserveFiles") {
+      const patterns = normalizePreserveFiles(value);
+      if (!Array.isArray(value) || patterns === null) throw invalid();
+      updates["options.preserveFiles"] = patterns;
+      continue;
     } else if (!toggles.includes(key) || typeof value !== "boolean") throw invalid();
     updates[`options.${key}`] = value;
   }

src/server/routes/repository-private.ts

@@ -30,6 +30,8 @@ import RepositoryModel from "../../core/model/repositories/repositories.model";
 import { RepositoryStatus } from "../../core/types";
 import { octokit, getRedisGateResetAt, getToken } from "../../core/GitHubUtils";
 import { createLogger } from "../../core/logger";
+import { normalizePreserveFiles } from "../../core/preserve-files";
+import { sourceFileList } from "../../core/source-file-list";
 
 const logger = createLogger("route:repo");
 
@@ -350,6 +352,27 @@ router.get(
   }
 );
 
+// Text files of a source commit, for the "Leave files unredacted" picker (#941).
+router.get(
+  "/:owner/:repo/tree",
+  async (req, res) => {
+    try {
+      const commit = typeof req.query.commit === "string" ? req.query.commit : "";
+      if (!/^[0-9a-f]{6,40}$/i.test(commit)) throw appError("invalid_commit_format", 400);
+      const token = await previewToken(req);
+      res.set("Cache-Control", "private, no-store");
+      res.json(await sourceFileList({
+        owner: req.params.owner,
+        repo: req.params.repo,
+        commit,
+        accessToken: token,
+      }));
+    } catch (error) {
+      handleError(error, res, req);
+    }
+  }
+);
+
 // get repository information
 router.get("/:repoId/", async (req, res) => {
   try {
@@ -428,6 +451,14 @@ function validateNewRepo(repoUpdate: any): void {
       httpStatus: 400,
     });
   }
+  const preserveFiles = normalizePreserveFiles(repoUpdate.options.preserveFiles);
+  if (preserveFiles === null) {
+    throw new AnonymousError("invalid_preserve_files_format", {
+      object: repoUpdate,
+      httpStatus: 400,
+    });
+  }
+  repoUpdate.options.preserveFiles = preserveFiles;
   if (!/^[a-fA-F0-9]+$/.test(repoUpdate.source.commit)) {
     throw new AnonymousError("invalid_commit_format", {
       object: repoUpdate,
@@ -446,6 +477,7 @@ function updateRepoModel(
   model.source.branch = repoUpdate.source.branch;
   model.options = {
     terms: repoUpdate.terms,
+    preserveFiles: normalizePreserveFiles(repoUpdate.options.preserveFiles) || [],
     expirationMode: repoUpdate.options.expirationMode,
     expirationDate: repoUpdate.options.expirationDate
       ? new Date(repoUpdate.options.expirationDate)