perf(scripts): single-parse website build + CI caching (#144)

Optimize the lesson -> GitHub Pages conversion in build_website.py: - BS4 single parse: thread one BeautifulSoup tree through heading-id normalisation, TOC extraction, and link rewriting instead of reparsing the rendered HTML 3x per page. Build drops 2.97s -> 1.77s (~40% faster) on the ~222-page site; html.parser reparsing was the dominant cost. - Nav O(n^2) -> O(n): build the sidebar section grouping once (build_nav_skeleton) and resolve per-page relative URLs in localize_nav, removing ~49k redundant relative_link/grouping calls. - Read each .md once: cache source text on PageInfo.content and reuse it across collect_pages, render_pages, and copy_assets (was 2-3 reads/page). - CI: enable uv cache and cache scripts/.vendor-cache so deploys skip cold dep resolution and vendor (Tailwind/Mermaid/fonts) downloads. Public string APIs (derive_page_title, render_markdown, extract_toc, rewrite_links, normalise_heading_ids) preserved as thin wrappers over the new soup-based cores. Full suite green (92 tests), ruff/mypy clean. Output is semantically identical to the prior build; the only diff is raw hand-written <img/> badges serializing as <img></img> (valid HTML5, renders identically; nothing validates generated site HTML).
2026-07-07 11:17:48 +02:00 · 2026-06-17 12:14:59 +02:00
parent 6963f37e5d
commit 8143e37d1b
2 changed files with 133 additions and 49 deletions
@@ -151,6 +151,7 @@ class PageInfo:
    title: str
    section: str  # the chapter group name (e.g. "Slash Commands")
    is_section_index: bool = False
+    content: str | None = None  # cached source text (read once during collection)


@dataclass
@@ -239,11 +240,17 @@ def is_excluded_top_level_markdown(name: str) -> bool:
    )


-def derive_page_title(md_path: Path, default: str) -> str:
-    """Pick a page title from the first H1, falling back to filename / default."""
+def read_source(md_path: Path) -> str | None:
+    """Read a markdown source file once, returning None on read failure."""
    try:
-        content = md_path.read_text(encoding="utf-8")
+        return md_path.read_text(encoding="utf-8")
    except (OSError, UnicodeDecodeError):
+        return None
+
+
+def title_from_content(content: str | None, default: str) -> str:
+    """Pick a page title from the first H1, falling back to the default."""
+    if content is None:
        return default
    match = re.search(r"^#\s+(.+?)\s*$", content, flags=re.MULTILINE)
    if match:
@@ -251,6 +258,11 @@ def derive_page_title(md_path: Path, default: str) -> str:
    return default


+def derive_page_title(md_path: Path, default: str) -> str:
+    """Pick a page title from a file's first H1, falling back to the default."""
+    return title_from_content(read_source(md_path), default)
+
+
 def source_to_site_url(rel_source: str) -> str:
    """Map `01-slash-commands/README.md` → `01-slash-commands/index.html`."""
    if rel_source == "README.md":
@@ -298,7 +310,8 @@ def collect_pages(config: WebsiteConfig, logger: logging.Logger) -> BuildState:
            if is_excluded_top_level_markdown(item) or item in seen:
                continue
            seen.add(item)
-            page_title = derive_page_title(item_path, display_name)
+            content = read_source(item_path)
+            page_title = title_from_content(content, display_name)
            url = _disambiguate_url(source_to_site_url(item), used_urls, item)
            used_urls.add(url.lower())
            state.pages.append(
@@ -309,6 +322,7 @@ def collect_pages(config: WebsiteConfig, logger: logging.Logger) -> BuildState:
                    title=page_title,
                    section=display_name,
                    is_section_index=True,
+                    content=content,
                )
            )
        elif item_path.is_dir():
@@ -319,7 +333,10 @@ def collect_pages(config: WebsiteConfig, logger: logging.Logger) -> BuildState:
                    continue
                seen.add(rel)
                is_index = md.name == "README.md" and md.parent == item_path
-                title = derive_page_title(md, display_name if is_index else md.stem)
+                content = read_source(md)
+                title = title_from_content(
+                    content, display_name if is_index else md.stem
+                )
                url = _disambiguate_url(source_to_site_url(rel), used_urls, rel)
                used_urls.add(url.lower())
                state.pages.append(
@@ -330,6 +347,7 @@ def collect_pages(config: WebsiteConfig, logger: logging.Logger) -> BuildState:
                        title=title,
                        section=display_name,
                        is_section_index=is_index,
+                        content=content,
                    )
                )
        else:
@@ -341,7 +359,8 @@ def collect_pages(config: WebsiteConfig, logger: logging.Logger) -> BuildState:
            continue
        seen.add(rel)
        title_default = md.stem.replace("-", " ").replace("_", " ").title()
-        title = derive_page_title(md, title_default)
+        content = read_source(md)
+        title = title_from_content(content, title_default)
        url = _disambiguate_url(source_to_site_url(rel), used_urls, rel)
        used_urls.add(url.lower())
        state.pages.append(
@@ -352,6 +371,7 @@ def collect_pages(config: WebsiteConfig, logger: logging.Logger) -> BuildState:
                title=title,
                section="Additional Docs",
                is_section_index=False,
+                content=content,
            )
        )

@@ -474,16 +494,14 @@ def _rewrite_asset_ref(
    element[attr] = relative_link(page.output_url, target)  # type: ignore[index]


-def rewrite_links(
-    html_content: str,
+def rewrite_links_in_soup(
+    soup: BeautifulSoup,
    page: PageInfo,
    state: BuildState,
    config: WebsiteConfig,
    logger: logging.Logger,
-) -> str:
-    """Rewrite anchor/image hrefs so links resolve correctly on the site."""
-    soup = BeautifulSoup(html_content, "html.parser")
-
+) -> None:
+    """Rewrite anchor/image hrefs of `soup` in place to resolve on the site."""
    for a in soup.find_all("a"):
        _rewrite_anchor(a, page, state, config, logger)

@@ -495,6 +513,17 @@ def rewrite_links(
    for source in soup.find_all("source"):
        _rewrite_asset_ref(source, "srcset", source.get("srcset", ""), page, config)

+
+def rewrite_links(
+    html_content: str,
+    page: PageInfo,
+    state: BuildState,
+    config: WebsiteConfig,
+    logger: logging.Logger,
+) -> str:
+    """String wrapper around :func:`rewrite_links_in_soup`."""
+    soup = BeautifulSoup(html_content, "html.parser")
+    rewrite_links_in_soup(soup, page, state, config, logger)
    return str(soup)


@@ -521,13 +550,12 @@ def replace_mermaid_blocks(md_content: str) -> str:
 # =============================================================================


-def normalise_heading_ids(html_content: str) -> str:
-    """Force GitHub-style anchor ids on every heading.
+def normalise_heading_ids_in_soup(soup: BeautifulSoup) -> None:
+    """Force GitHub-style anchor ids on every heading of `soup`, in place.

    `python-markdown`'s `toc` extension generates its own slug; we re-write them
    so they match `check_cross_references.heading_to_anchor`.
    """
-    soup = BeautifulSoup(html_content, "html.parser")
    used: dict[str, int] = {}
    for level in ("h1", "h2", "h3", "h4", "h5", "h6"):
        for h in soup.find_all(level):
@@ -539,12 +567,17 @@ def normalise_heading_ids(html_content: str) -> str:
            final = anchor if count == 0 else f"{anchor}-{count}"
            used[anchor] = count + 1
            h["id"] = final
+
+
+def normalise_heading_ids(html_content: str) -> str:
+    """String wrapper around :func:`normalise_heading_ids_in_soup`."""
+    soup = BeautifulSoup(html_content, "html.parser")
+    normalise_heading_ids_in_soup(soup)
    return str(soup)


-def extract_toc(html_content: str) -> list[dict[str, str]]:
-    """Pull H2/H3 headings into a flat list for in-page navigation."""
-    soup = BeautifulSoup(html_content, "html.parser")
+def extract_toc_from_soup(soup: BeautifulSoup) -> list[dict[str, str]]:
+    """Pull H2/H3 headings of `soup` into a flat list for in-page navigation."""
    toc: list[dict[str, str]] = []
    for h in soup.find_all(["h2", "h3"]):
        anchor = h.get("id")
@@ -560,15 +593,31 @@ def extract_toc(html_content: str) -> list[dict[str, str]]:
    return toc


-def render_markdown(md_content: str) -> str:
-    """Convert markdown to HTML using the same extensions as the EPUB build."""
+def extract_toc(html_content: str) -> list[dict[str, str]]:
+    """String wrapper around :func:`extract_toc_from_soup`."""
+    return extract_toc_from_soup(BeautifulSoup(html_content, "html.parser"))
+
+
+def render_markdown_to_soup(md_content: str) -> BeautifulSoup:
+    """Render markdown to a parsed soup with GitHub-style heading ids applied.
+
+    Parsing the rendered HTML once here lets the caller run TOC extraction and
+    link rewriting against the same tree instead of reparsing per stage.
+    """
    md_content = replace_mermaid_blocks(md_content)
    html_content = markdown.markdown(
        md_content,
        extensions=["tables", "fenced_code", "codehilite", "toc"],
        extension_configs={"codehilite": {"guess_lang": False}},
    )
-    return normalise_heading_ids(html_content)
+    soup = BeautifulSoup(html_content, "html.parser")
+    normalise_heading_ids_in_soup(soup)
+    return soup
+
+
+def render_markdown(md_content: str) -> str:
+    """Convert markdown to HTML using the same extensions as the EPUB build."""
+    return str(render_markdown_to_soup(md_content))


 # =============================================================================
@@ -596,9 +645,8 @@ def copy_assets(
    copied: set[Path] = set()

    for page in state.pages:
-        try:
-            content = page.source.read_text(encoding="utf-8")
-        except (OSError, UnicodeDecodeError):
+        content = page.content if page.content is not None else read_source(page.source)
+        if content is None:
            continue
        for match in re.finditer(r"!\[[^\]]*\]\(([^)]+)\)", content):
            src = match.group(1).split(" ", 1)[0]
@@ -641,11 +689,13 @@ def copy_assets(
 # =============================================================================


-def build_navigation(state: BuildState, current_url: str) -> list[dict[str, object]]:
-    """Group pages into the sidebar navigation tree, rooted at `current_url`.
+def build_nav_skeleton(state: BuildState) -> list[dict[str, object]]:
+    """Group pages into sidebar sections once, keyed by section name.

-    Each item's `url` is a relative URL from `current_url` so the same nav
-    structure works from any page depth (top-level vs nested chapter folders).
+    The grouping (sections + ordered page list) is identical for every page,
+    so it is computed a single time and reused. `localize_nav` then turns the
+    absolute page URLs into per-page relative URLs — an O(n) pass instead of
+    re-grouping all pages for every page (which was O(n²)).
    """
    sections: list[dict[str, object]] = []
    section_map: dict[str, dict[str, object]] = {}
@@ -653,27 +703,42 @@ def build_navigation(state: BuildState, current_url: str) -> list[dict[str, obje
    for page in state.pages:
        section = section_map.get(page.section)
        if section is None:
-            section = {
-                "name": page.section,
-                "items": [],
-            }
+            section = {"name": page.section, "items": []}
            section_map[page.section] = section
            sections.append(section)

        items = section["items"]
        assert isinstance(items, list)
-        items.append(
-            {
-                "title": page.title,
-                "url": relative_link(current_url, page.output_url),
-                "is_current": page.output_url == current_url,
-                "is_index": page.is_section_index,
-            }
-        )
+        items.append(page)

    return sections


+def localize_nav(
+    skeleton: list[dict[str, object]], current_url: str
+) -> list[dict[str, object]]:
+    """Resolve the shared nav skeleton into relative URLs for `current_url`."""
+    localized: list[dict[str, object]] = []
+    for section in skeleton:
+        pages = section["items"]
+        assert isinstance(pages, list)
+        localized.append(
+            {
+                "name": section["name"],
+                "items": [
+                    {
+                        "title": page.title,
+                        "url": relative_link(current_url, page.output_url),
+                        "is_current": page.output_url == current_url,
+                        "is_index": page.is_section_index,
+                    }
+                    for page in pages
+                ],
+            }
+        )
+    return localized
+
+
 def render_pages(
    config: WebsiteConfig,
    state: BuildState,
@@ -684,16 +749,25 @@ def render_pages(
    template = env.get_template("page.html.j2")
    total = len(state.pages)

-    for idx, page in enumerate(state.pages):
-        nav = build_navigation(state, page.output_url)
-        try:
-            md_content = page.source.read_text(encoding="utf-8")
-        except UnicodeDecodeError as e:
-            raise RuntimeError(f"Failed to read {page.source}: {e}") from e
+    nav_skeleton = build_nav_skeleton(state)

-        html_content = render_markdown(md_content)
-        html_content = rewrite_links(html_content, page, state, config, logger)
-        toc = extract_toc(html_content)
+    for idx, page in enumerate(state.pages):
+        nav = localize_nav(nav_skeleton, page.output_url)
+        if page.content is not None:
+            md_content = page.content
+        else:
+            try:
+                md_content = page.source.read_text(encoding="utf-8")
+            except UnicodeDecodeError as e:
+                raise RuntimeError(f"Failed to read {page.source}: {e}") from e
+
+        # Parse the rendered markdown once, then run heading-id normalisation,
+        # TOC extraction, and link rewriting against the same tree before a
+        # single serialization — avoids reparsing the HTML three times per page.
+        soup = render_markdown_to_soup(md_content)
+        toc = extract_toc_from_soup(soup)
+        rewrite_links_in_soup(soup, page, state, config, logger)
+        html_content = str(soup)

        prev_page = state.pages[idx - 1] if idx > 0 else None
        next_page = state.pages[idx + 1] if idx < total - 1 else None