fix: fill gaps in GitHub Markdown rendering (#912)

* fix: fill gaps in GitHub Markdown rendering

* fix: harden Markdown dependencies and address review findings
This commit is contained in:
Thomas Durieux authored and GitHub committed 2026-11-02 07:56:32 +01:00
1 parent 075c3a4622
commit 6cf5445972
31 files changed
+7445 -1895

No files matched your search

+145
View File
@@ -0,0 +1,145 @@
import { geoPath, geoMercator, geoCentroid, geoArea } from "d3-geo";
import { feature } from "topojson-client";
import type { Topology, GeometryObject } from "topojson-specification";
import type { GeoPermissibleObjects } from "d3-geo";
import type { GeoJSON } from "geojson";
const MAX_SOURCE_LENGTH = 1_000_000;
const MAX_POINTS = 60_000;
function validateMap(input: unknown, topology: boolean): void {
let nodes = 0;
const inspect = (value: unknown, depth = 0): void => {
if (++nodes > 300_000 || depth > 32) throw new Error("Map complexity limit exceeded");
if (value && typeof value === "object") for (const child of Object.values(value)) inspect(child, depth + 1);
};
inspect(input);
const array = (value: unknown): unknown[] => {
if (!Array.isArray(value)) throw new Error("Expected a map array");
return value;
};
const object = (value: unknown): Record<string, unknown> => {
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Expected a map object");
return value as Record<string, unknown>;
};
let points = 0;
const position = (value: unknown): void => {
const numbers = array(value);
if (numbers.length < 2 || numbers.some(n => typeof n !== "number" || !Number.isFinite(n) || Math.abs(n) > 1e100)) throw new Error("Invalid map position");
if (++points > MAX_POINTS) throw new Error("Map point limit exceeded");
};
const positions = (value: unknown, depth: number): void => {
if (depth === 0) position(value);
else for (const child of array(value)) positions(child, depth - 1);
};
const root = object(input);
let arcLengths: number[] = [];
if (topology) {
if (root.type !== "Topology") throw new Error("Expected Topology");
arcLengths = array(root.arcs).map(arc => {
const coordinates = array(arc);
if (!coordinates.length) throw new Error("Empty topology arc");
coordinates.forEach(position);
return coordinates.length;
});
if (root.transform != null) {
const transform = object(root.transform);
position(transform.scale); position(transform.translate);
}
}
let expandedPoints = 0;
const arcs = (value: unknown, depth: number): void => {
for (const child of array(value)) {
if (depth > 0) arcs(child, depth - 1);
else {
if (typeof child !== "number" || !Number.isInteger(child) || child < -2147483648 || child > 2147483647) throw new Error("Invalid arc index");
const length = arcLengths[child < 0 ? ~child : child];
if (length === undefined) throw new Error("Missing topology arc");
expandedPoints += Math.max(4, length);
if (expandedPoints > MAX_POINTS) throw new Error("Topology expansion limit exceeded");
}
}
};
const geometry = (value: unknown): void => {
const item = object(value);
switch (item.type) {
case "FeatureCollection": array(item.features).forEach(geometry); break;
case "Feature": if (item.geometry != null) geometry(item.geometry); break;
case "GeometryCollection": array(item.geometries).forEach(geometry); break;
case "Point": position(item.coordinates); break;
case "MultiPoint": positions(item.coordinates, 1); break;
case "LineString": if (topology) arcs(item.arcs, 0); else positions(item.coordinates, 1); break;
case "MultiLineString":
case "Polygon": if (topology) arcs(item.arcs, 1); else positions(item.coordinates, 2); break;
case "MultiPolygon": if (topology) arcs(item.arcs, 2); else positions(item.coordinates, 3); break;
default: throw new Error("Unsupported map geometry");
}
};
if (topology) Object.values(object(root.objects)).forEach(geometry);
else geometry(root);
}
// D3's spherical polygons use the opposite winding from RFC 7946 GeoJSON.
function orientPolygons(value: GeoJSON): void {
if (value.type === "FeatureCollection") value.features.forEach(orientPolygons);
else if (value.type === "Feature" && value.geometry) orientPolygons(value.geometry);
else if (value.type === "GeometryCollection") value.geometries.forEach(orientPolygons);
else if (value.type === "Polygon" || value.type === "MultiPolygon") {
const polygons = value.type === "Polygon" ? [value.coordinates] : value.coordinates;
for (const polygon of polygons) polygon.forEach((ring, index) => {
const large = geoArea({ type: "Polygon", coordinates: [ring] }) > 2 * Math.PI;
if (large === (index === 0)) ring.reverse();
});
}
}
type Vertex = [number, number, number];
export function stlPaths(vertices: Vertex[], angle = 0.6): string {
const points = vertices.map(([x, y, z]) => {
const u = x * Math.cos(angle) - z * Math.sin(angle);
const depth = x * Math.sin(angle) + z * Math.cos(angle);
return [u, -y * Math.cos(0.45) + depth * Math.sin(0.45), depth];
});
const bounds = points.reduce((b, p) => [Math.min(b[0], p[0]), Math.min(b[1], p[1]), Math.max(b[2], p[0]), Math.max(b[3], p[1])], [Infinity, Infinity, -Infinity, -Infinity]);
const scale = 260 / (Math.max(bounds[2] - bounds[0], bounds[3] - bounds[1]) || 1);
const triangles: { depth: number; path: string }[] = [];
for (let i = 0; i < points.length; i += 3) {
const triangle = points.slice(i, i + 3);
const coordinates = triangle.map(p => `${(320 + (p[0] - (bounds[0] + bounds[2]) / 2) * scale).toFixed(3)},${(160 + (p[1] - (bounds[1] + bounds[3]) / 2) * scale).toFixed(3)}`);
triangles.push({ depth: triangle.reduce((sum, p) => sum + p[2], 0), path: `<path d="M${coordinates.join("L")}Z"/>` });
}
return triangles.sort((a, b) => a.depth - b.depth).map(t => t.path).join("");
}
export function renderDiagram(source: string, language: string): string {
if (source.length > MAX_SOURCE_LENGTH) throw new Error("Diagram source limit exceeded");
let paths: string;
let data = "";
if (language === "stl") {
if (!/^\s*solid\b/.test(source) || !/\bendsolid\b/.test(source)) throw new Error("Invalid ASCII STL");
const vertices: Vertex[] = [...source.matchAll(/\bvertex\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)/g)].map(match => match.slice(1).map(Number) as Vertex);
if (!vertices.length || vertices.length % 3 || vertices.length > 60000 || vertices.some(v => v.some(n => !Number.isFinite(n) || Math.abs(n) > 1e100))) throw new Error("Invalid STL vertices");
paths = stlPaths(vertices);
data = ` data-vertices="${JSON.stringify(vertices)}"`;
} else {
const input = JSON.parse(source);
validateMap(input, language === "topojson");
const geometry = language === "topojson"
? { type: "FeatureCollection", features: Object.values((input as Topology).objects).flatMap(object => {
const converted = feature(input as Topology, object as GeometryObject);
return converted.type === "FeatureCollection" ? converted.features : [converted];
}) }
: input;
if (language === "topojson") validateMap(geometry, false);
orientPolygons(geometry);
const projection = geoMercator().fitExtent([[20, 20], [620, 300]], geometry as GeoPermissibleObjects);
// A lone point has zero extent; center it at a useful finite scale.
if (!Number.isFinite(projection.scale())) projection.scale(100).center(geoCentroid(geometry as GeoPermissibleObjects)).translate([320, 160]);
const path = geoPath(projection).pointRadius(5)(geometry as GeoPermissibleObjects);
if (!path || /NaN|Infinity/.test(path)) throw new Error("Invalid map geometry");
paths = `<path d="${path}"/>`;
}
const label = language === "stl" ? "3D model" : "Map";
return `<figure class="markdown-diagram" data-diagram="${language}"${data}><svg viewBox="0 0 640 320" role="img" aria-label="${label}"><g>${paths}</g></svg><figcaption>${label}</figcaption><div class="diagram-controls"><button type="button" data-diagram-action="in" aria-label="Zoom in">+</button><button type="button" data-diagram-action="out" aria-label="Zoom out">−</button>${language === "stl" ? '<button type="button" data-diagram-action="rotate">Rotate</button>' : ''}<button type="button" data-diagram-action="reset">Reset</button></div></figure>`;
}
+173
View File
@@ -0,0 +1,173 @@
import { Marked, Tokens } from "marked";
const markedFootnote: typeof import("marked-footnote").default = require("marked-footnote");
const markedAlert: typeof import("marked-alert").default = require("marked-alert");
const markedKatex: typeof import("marked-katex-extension").default = require("marked-katex-extension");
import { gfmHeadingId } from "marked-gfm-heading-id";
import * as katex from "katex";
import sanitizeHtml = require("sanitize-html");
const hljs: typeof import("highlight.js/lib/common").default = require("highlight.js/lib/common");
import { renderDiagram } from "./markdown-diagrams";
const emojis: Record<string, string> = require("../public/script/external/github-emojis");
export const escapeHtml = (value: string): string => value.replace(/[&<>"']/g, c => ({
"&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;",
}[c]!));
function resolveUrl(url: string, base?: string): string {
if (!base || !url || /^(?:#|\/|[a-z][a-z\d+.-]*:)/i.test(url)) return url;
try {
const resolved = new URL(url, new URL(base, "https://markdown.invalid"));
return /^https?:/i.test(base) ? resolved.href : resolved.pathname + resolved.search + resolved.hash;
} catch { return url; }
}
/** The browser and project-page route use this same parser and sanitizer. */
export function renderMarkdown(source: string, base?: string): string {
// Only renderer-generated math/SVG bypasses HTML sanitization. An unpredictable,
// per-document marker prevents raw HTML from impersonating generated output.
const nonce = Array.from(globalThis.crypto.getRandomValues(new Uint32Array(4)), n => n.toString(16)).join("-");
const generated: string[] = [];
const trusted = (html: string) => `<span data-render-token="${nonce}-${generated.push(html) - 1}"></span>`;
const math = (text: string, displayMode: boolean) => trusted(katex.renderToString(text, {
displayMode, throwOnError: false, trust: false, maxExpand: 1000, maxSize: 20,
}));
const parser = new Marked({ gfm: true, breaks: false, async: false });
const footnotes = markedFootnote({ prefixId: `footnote-${nonce}-` });
for (const extension of footnotes.extensions || []) {
if ("renderer" in extension && extension.renderer) {
const render = extension.renderer;
extension.renderer = function (token) {
const html = render.call(this, token);
return typeof html === "string" ? html.replaceAll(`data-footnote-${nonce}-`, "data-footnote-") : html;
};
}
}
parser.use(footnotes, gfmHeadingId());
const alerts = markedAlert();
const topLevelQuotes = new WeakSet();
const alertWalk = alerts.walkTokens!;
alerts.walkTokens = function (token) {
if (topLevelQuotes.has(token)) return alertWalk.call(this, token);
};
alerts.hooks = { processAllTokens(tokens) {
for (const token of tokens) if (token.type === "blockquote") topLevelQuotes.add(token);
return tokens;
} };
// Preserve the extension's fixed icons without admitting arbitrary source SVG.
for (const extension of alerts.extensions || []) {
if ("renderer" in extension && extension.renderer) {
const render = extension.renderer;
extension.renderer = function (token) {
const icon = token.meta.icon;
token.meta.icon = trusted(icon);
return render.call(this, token);
};
}
}
parser.use(alerts);
const mathExtension = markedKatex({ throwOnError: false, nonStandard: false });
for (const extension of mathExtension.extensions || []) {
if ("renderer" in extension) extension.renderer = token => math(token.text, token.displayMode);
}
parser.use(mathExtension, {
extensions: [{
name: "protocolEmail", level: "inline",
start: src => {
const match = /(?:^|[\s/])(mailto:|xmpp:)/.exec(src);
return match ? match.index + match[0].length - match[1].length : undefined;
},
tokenizer(src) {
if (this.lexer.state.inLink) return;
const match = /^(mailto:|xmpp:)([\w.+-]+@[\w.-]+\.[\w-]+)(\/[^\s<>]*)?/.exec(src);
if (!match) return;
const text = (match[1] + match[2] + (match[1] === "xmpp:" ? match[3] || "" : "")).replace(/[.,!?]+$/, "");
return { type: "protocolEmail", raw: text, text };
},
renderer: token => `<a href="${escapeHtml(token.text)}">${escapeHtml(token.text)}</a>`,
}, {
name: "githubMath", level: "inline",
start: src => src.indexOf("$`"),
tokenizer(src) {
const match = /^\$`([^\n]*?)`\$/.exec(src);
if (match) return { type: "githubMath", raw: match[0], text: match[1] };
},
renderer: token => math(token.text, false),
}, {
name: "emoji", level: "inline", start: src => src.search(/:[\w+-]+:/),
tokenizer(src) {
const match = /^:([\w+-]+):/.exec(src);
if (match && Object.prototype.hasOwnProperty.call(emojis, match[1])) return { type: "emoji", raw: match[0], name: match[1] };
},
renderer: token => `<img class="emoji" alt=":${token.name}:" width="20" height="20" src="${emojis[token.name]}">`,
}],
tokenizer: {
url(src) {
const match = /^(?:https?:\/\/|www\.)[^\s<>]+/.exec(src);
if (!match) return false;
const host = match[0].replace(/^https?:\/\//, "").split(/[/?#]/)[0].split(".");
if (host.slice(-2).some(part => part.includes("_"))) return { type: "link", raw: match[0], text: match[0], href: "", invalidAutolink: true, tokens: [] };
const unicodeHost = /^https?:\/\//.test(match[0]) && match[0].slice(match[0].indexOf("://") + 3).charCodeAt(0) > 127;
if (!match[0].startsWith("www._") && !unicodeHost) return false;
const text = match[0].replace(/[?!.,:*_'"~]+$/, "");
return { type: "link", raw: text, href: encodeURI(text.startsWith("www.") ? `http://${text}` : text), text, tokens: [{ type: "text", raw: text, text: escapeHtml(text) }] };
},
},
renderer: {
link(token: Tokens.Link & { invalidAutolink?: boolean }) {
if (token.invalidAutolink) return escapeHtml(token.text);
if (/\.(mp4|mov)(?:[?#]|$)/i.test(token.href)) return `<video controls src="${escapeHtml(token.href)}" title="${escapeHtml(token.title || token.text)}">${this.parser.parseInline(token.tokens)}</video>`;
return false;
},
code(token: Tokens.Code) {
const language = (token.lang || "").split(/\s+/)[0].toLowerCase();
if (language === "math") return math(token.text, true);
if (language === "mermaid") return `<pre class="mermaid">${escapeHtml(token.text)}</pre>`;
if (["geojson", "topojson", "stl"].includes(language)) {
try { return trusted(renderDiagram(token.text, language)); }
catch { /* Invalid diagrams retain their readable source. */ }
}
if (language && hljs.getLanguage(language)) {
return `<pre><code class="language-${escapeHtml(language)} hljs">${hljs.highlight(token.text, { language, ignoreIllegals: true }).value}</code></pre>\n`;
}
return false;
},
},
});
const raw = parser.parse(source) as string;
const clean = sanitizeMarkdownHtml(raw, base);
return clean.replace(new RegExp(`<span data-render-token="${nonce}-(\\d+)"></span>`, "g"), (_match, index) => generated[Number(index)]);
}
export function sanitizeMarkdownHtml(raw: string, base?: string): string {
return sanitizeHtml(raw, {
allowedTags: sanitizeHtml.defaults.allowedTags.concat([
"img", "picture", "source", "video", "input", "details", "summary", "del", "ins", "sup", "sub", "kbd", "samp", "var", "ruby", "rt", "rp",
]),
allowedAttributes: {
"*": ["id", "class", "dir", "lang", "title", "aria-label", "aria-labelledby", "aria-describedby", "aria-hidden", "align"],
a: ["href", "name", "rel", "data-footnote-ref", "data-footnote-backref"],
img: ["src", "srcset", "alt", "width", "height", "loading", "align"],
source: ["src", "srcset", "media", "type"],
video: ["src", "controls", "width", "height", "poster"],
input: ["type", "checked", "disabled"],
details: ["open"], ol: ["start", "reversed"], li: ["value"],
td: ["align", "colspan", "rowspan"], th: ["align", "colspan", "rowspan", "scope"],
span: ["data-render-token"], section: ["data-footnotes"],
},
allowedSchemes: ["http", "https", "mailto", "xmpp"],
transformTags: {
input: (_tag, attrs) => ({ tagName: "input", attribs: { type: "checkbox", disabled: "", ...(Object.prototype.hasOwnProperty.call(attrs, "checked") ? { checked: "" } : {}) } }),
"*": (tagName, attribs) => {
if (tagName === "img" && /#gh-(dark|light)-mode-only$/.test(attribs.src || "")) {
attribs.class = `${attribs.class || ""} ${attribs.src.endsWith("#gh-dark-mode-only") ? "gh-dark-mode-only" : "gh-light-mode-only"}`.trim();
}
for (const attr of ["href", "src", "poster"]) if (attribs[attr]) attribs[attr] = resolveUrl(attribs[attr], base);
if (attribs.srcset) attribs.srcset = attribs.srcset.split(",").map(candidate => {
const [url, ...descriptor] = candidate.trim().split(/\s+/);
return [resolveUrl(url, base), ...descriptor].join(" ");
}).join(", ");
return { tagName, attribs };
},
},
});
}
+7
View File
@@ -289,6 +289,13 @@ export default async function start() {
app.use(
express.static(join("public"), {
etag: true,
setHeaders(res, filePath) {
// Sandboxed Markdown pages have an opaque origin; public math fonts
// must permit anonymous cross-origin requests without relaxing CSP.
if (/[/\\]css[/\\]fonts[/\\]KaTeX_[^/\\]+\.(?:woff2?|ttf)$/.test(filePath)) {
res.setHeader("Access-Control-Allow-Origin", "*");
}
},
lastModified: true,
maxAge: 86400000, // 1 day (fonts, images, partials)
index: false, // don't serve index.html for "/" — indexResponse handles it
+5 -33
View File
@@ -3,8 +3,7 @@ import { getRepo, handleError } from "./route-utils";
import * as path from "path";
import AnonymizedFile from "../../core/AnonymizedFile";
import AnonymousError from "../../core/AnonymousError";
import * as marked from "marked";
import * as sanitizeHtml from "sanitize-html";
import { renderMarkdown } from "../../markdown";
import { streamToString, isTextFile } from "../../core/anonymize-utils";
import { IFile } from "../../core/model/files/files.types";
import { fileETag } from "./file-etag";
@@ -20,33 +19,6 @@ function escapeHtml(str: string): string {
.replace(/'/g, "&#039;");
}
const sanitizeOptions: sanitizeHtml.IOptions = {
allowedTags: sanitizeHtml.defaults.allowedTags.concat([
"img",
"video",
"input",
"details",
"summary",
"del",
"ins",
"sup",
"sub",
]),
allowedAttributes: {
...sanitizeHtml.defaults.allowedAttributes,
img: ["src", "srcset", "alt", "title", "width", "height", "loading"],
video: ["src", "controls", "title"],
input: ["type", "checked", "disabled"],
code: ["class"],
span: ["class"],
div: ["class"],
pre: ["class"],
td: ["align"],
th: ["align"],
},
allowedSchemes: ["http", "https", "mailto"],
};
const router = express.Router();
const indexPriority = [
@@ -161,7 +133,7 @@ async function webView(
}">${escapeHtml(c.name) + (c.size == null ? "/" : "")}</a>`
)
.join("")}</div></div>`;
const html = `<!DOCTYPE html><html><head><title>Content</title></head><link rel="stylesheet" href="/css/all.min.css" /><body>${body}</body></html>`;
const html = `<!DOCTYPE html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>Content</title></head><link rel="stylesheet" href="/css/all.min.css" /><body>${body}</body></html>`;
return res.contentType("text/html").send(html);
}
}
@@ -174,7 +146,7 @@ async function webView(
});
}
res.set("Cache-Control", "private, no-cache, must-revalidate");
const pageETag = fileETag(JSON.stringify(["web-v1", repo.model.source.commit, repo.model.treeGeneration, repo.model.contentCacheRevision,
const pageETag = fileETag(JSON.stringify(["web-v2", repo.model.source.commit, repo.model.treeGeneration, repo.model.contentCacheRevision,
await f.sha(), repo.model.source.repositoryName, repo.model.source.branch, config.APP_HOSTNAME, config.ANONYMIZATION_MASK]),
requestPath, repo.options);
res.set("ETag", pageETag);
@@ -189,8 +161,8 @@ async function webView(
if (f.extension() == "md") {
try {
const content = await streamToString(await f.anonymizedContent());
const body = sanitizeHtml(marked.marked.parse(content, { async: false }), sanitizeOptions);
const html = `<!DOCTYPE html><html><head><title>Content</title></head><link rel="stylesheet" href="/css/all.min.css" /><body><div class="container p-3 file-content markdown-body">${body}</div></body></html>`;
const body = renderMarkdown(content);
const html = `<!DOCTYPE html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>Content</title></head><link rel="stylesheet" href="/css/all.min.css" /><body><div class="container p-3 file-content markdown-body">${body}</div><script defer src="/script/markdown.min.js"></script></body></html>`;
res.contentType("text/html").send(html);
} catch {
// Raw Markdown must re-enter rendering when a transient failure clears.