From 8143e37d1b265e6225b9e28be1a795ee6f00e836 Mon Sep 17 00:00:00 2001 From: Luong NGUYEN Date: Wed, 17 Jun 2026 12:14:59 +0200 Subject: [PATCH] perf(scripts): single-parse website build + CI caching (#144) Optimize the lesson -> GitHub Pages conversion in build_website.py: - BS4 single parse: thread one BeautifulSoup tree through heading-id normalisation, TOC extraction, and link rewriting instead of reparsing the rendered HTML 3x per page. Build drops 2.97s -> 1.77s (~40% faster) on the ~222-page site; html.parser reparsing was the dominant cost. - Nav O(n^2) -> O(n): build the sidebar section grouping once (build_nav_skeleton) and resolve per-page relative URLs in localize_nav, removing ~49k redundant relative_link/grouping calls. - Read each .md once: cache source text on PageInfo.content and reuse it across collect_pages, render_pages, and copy_assets (was 2-3 reads/page). - CI: enable uv cache and cache scripts/.vendor-cache so deploys skip cold dep resolution and vendor (Tailwind/Mermaid/fonts) downloads. Public string APIs (derive_page_title, render_markdown, extract_toc, rewrite_links, normalise_heading_ids) preserved as thin wrappers over the new soup-based cores. Full suite green (92 tests), ruff/mypy clean. Output is semantically identical to the prior build; the only diff is raw hand-written badges serializing as (valid HTML5, renders identically; nothing validates generated site HTML). --- .github/workflows/pages.yml | 10 +++ scripts/build_website.py | 172 ++++++++++++++++++++++++++---------- 2 files changed, 133 insertions(+), 49 deletions(-) diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 2aeed56..855f356 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -30,6 +30,16 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + + - name: Cache vendor assets + uses: actions/cache@v4 + with: + path: scripts/.vendor-cache + key: vendor-${{ hashFiles('scripts/vendor_assets.py') }} + restore-keys: | + vendor- - name: Build website run: uv run scripts/build_website.py --verbose diff --git a/scripts/build_website.py b/scripts/build_website.py index 83abe3a..0b716fe 100644 --- a/scripts/build_website.py +++ b/scripts/build_website.py @@ -151,6 +151,7 @@ class PageInfo: title: str section: str # the chapter group name (e.g. "Slash Commands") is_section_index: bool = False + content: str | None = None # cached source text (read once during collection) @dataclass @@ -239,11 +240,17 @@ def is_excluded_top_level_markdown(name: str) -> bool: ) -def derive_page_title(md_path: Path, default: str) -> str: - """Pick a page title from the first H1, falling back to filename / default.""" +def read_source(md_path: Path) -> str | None: + """Read a markdown source file once, returning None on read failure.""" try: - content = md_path.read_text(encoding="utf-8") + return md_path.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): + return None + + +def title_from_content(content: str | None, default: str) -> str: + """Pick a page title from the first H1, falling back to the default.""" + if content is None: return default match = re.search(r"^#\s+(.+?)\s*$", content, flags=re.MULTILINE) if match: @@ -251,6 +258,11 @@ def derive_page_title(md_path: Path, default: str) -> str: return default +def derive_page_title(md_path: Path, default: str) -> str: + """Pick a page title from a file's first H1, falling back to the default.""" + return title_from_content(read_source(md_path), default) + + def source_to_site_url(rel_source: str) -> str: """Map `01-slash-commands/README.md` ā†’ `01-slash-commands/index.html`.""" if rel_source == "README.md": @@ -298,7 +310,8 @@ def collect_pages(config: WebsiteConfig, logger: logging.Logger) -> BuildState: if is_excluded_top_level_markdown(item) or item in seen: continue seen.add(item) - page_title = derive_page_title(item_path, display_name) + content = read_source(item_path) + page_title = title_from_content(content, display_name) url = _disambiguate_url(source_to_site_url(item), used_urls, item) used_urls.add(url.lower()) state.pages.append( @@ -309,6 +322,7 @@ def collect_pages(config: WebsiteConfig, logger: logging.Logger) -> BuildState: title=page_title, section=display_name, is_section_index=True, + content=content, ) ) elif item_path.is_dir(): @@ -319,7 +333,10 @@ def collect_pages(config: WebsiteConfig, logger: logging.Logger) -> BuildState: continue seen.add(rel) is_index = md.name == "README.md" and md.parent == item_path - title = derive_page_title(md, display_name if is_index else md.stem) + content = read_source(md) + title = title_from_content( + content, display_name if is_index else md.stem + ) url = _disambiguate_url(source_to_site_url(rel), used_urls, rel) used_urls.add(url.lower()) state.pages.append( @@ -330,6 +347,7 @@ def collect_pages(config: WebsiteConfig, logger: logging.Logger) -> BuildState: title=title, section=display_name, is_section_index=is_index, + content=content, ) ) else: @@ -341,7 +359,8 @@ def collect_pages(config: WebsiteConfig, logger: logging.Logger) -> BuildState: continue seen.add(rel) title_default = md.stem.replace("-", " ").replace("_", " ").title() - title = derive_page_title(md, title_default) + content = read_source(md) + title = title_from_content(content, title_default) url = _disambiguate_url(source_to_site_url(rel), used_urls, rel) used_urls.add(url.lower()) state.pages.append( @@ -352,6 +371,7 @@ def collect_pages(config: WebsiteConfig, logger: logging.Logger) -> BuildState: title=title, section="Additional Docs", is_section_index=False, + content=content, ) ) @@ -474,16 +494,14 @@ def _rewrite_asset_ref( element[attr] = relative_link(page.output_url, target) # type: ignore[index] -def rewrite_links( - html_content: str, +def rewrite_links_in_soup( + soup: BeautifulSoup, page: PageInfo, state: BuildState, config: WebsiteConfig, logger: logging.Logger, -) -> str: - """Rewrite anchor/image hrefs so links resolve correctly on the site.""" - soup = BeautifulSoup(html_content, "html.parser") - +) -> None: + """Rewrite anchor/image hrefs of `soup` in place to resolve on the site.""" for a in soup.find_all("a"): _rewrite_anchor(a, page, state, config, logger) @@ -495,6 +513,17 @@ def rewrite_links( for source in soup.find_all("source"): _rewrite_asset_ref(source, "srcset", source.get("srcset", ""), page, config) + +def rewrite_links( + html_content: str, + page: PageInfo, + state: BuildState, + config: WebsiteConfig, + logger: logging.Logger, +) -> str: + """String wrapper around :func:`rewrite_links_in_soup`.""" + soup = BeautifulSoup(html_content, "html.parser") + rewrite_links_in_soup(soup, page, state, config, logger) return str(soup) @@ -521,13 +550,12 @@ def replace_mermaid_blocks(md_content: str) -> str: # ============================================================================= -def normalise_heading_ids(html_content: str) -> str: - """Force GitHub-style anchor ids on every heading. +def normalise_heading_ids_in_soup(soup: BeautifulSoup) -> None: + """Force GitHub-style anchor ids on every heading of `soup`, in place. `python-markdown`'s `toc` extension generates its own slug; we re-write them so they match `check_cross_references.heading_to_anchor`. """ - soup = BeautifulSoup(html_content, "html.parser") used: dict[str, int] = {} for level in ("h1", "h2", "h3", "h4", "h5", "h6"): for h in soup.find_all(level): @@ -539,12 +567,17 @@ def normalise_heading_ids(html_content: str) -> str: final = anchor if count == 0 else f"{anchor}-{count}" used[anchor] = count + 1 h["id"] = final + + +def normalise_heading_ids(html_content: str) -> str: + """String wrapper around :func:`normalise_heading_ids_in_soup`.""" + soup = BeautifulSoup(html_content, "html.parser") + normalise_heading_ids_in_soup(soup) return str(soup) -def extract_toc(html_content: str) -> list[dict[str, str]]: - """Pull H2/H3 headings into a flat list for in-page navigation.""" - soup = BeautifulSoup(html_content, "html.parser") +def extract_toc_from_soup(soup: BeautifulSoup) -> list[dict[str, str]]: + """Pull H2/H3 headings of `soup` into a flat list for in-page navigation.""" toc: list[dict[str, str]] = [] for h in soup.find_all(["h2", "h3"]): anchor = h.get("id") @@ -560,15 +593,31 @@ def extract_toc(html_content: str) -> list[dict[str, str]]: return toc -def render_markdown(md_content: str) -> str: - """Convert markdown to HTML using the same extensions as the EPUB build.""" +def extract_toc(html_content: str) -> list[dict[str, str]]: + """String wrapper around :func:`extract_toc_from_soup`.""" + return extract_toc_from_soup(BeautifulSoup(html_content, "html.parser")) + + +def render_markdown_to_soup(md_content: str) -> BeautifulSoup: + """Render markdown to a parsed soup with GitHub-style heading ids applied. + + Parsing the rendered HTML once here lets the caller run TOC extraction and + link rewriting against the same tree instead of reparsing per stage. + """ md_content = replace_mermaid_blocks(md_content) html_content = markdown.markdown( md_content, extensions=["tables", "fenced_code", "codehilite", "toc"], extension_configs={"codehilite": {"guess_lang": False}}, ) - return normalise_heading_ids(html_content) + soup = BeautifulSoup(html_content, "html.parser") + normalise_heading_ids_in_soup(soup) + return soup + + +def render_markdown(md_content: str) -> str: + """Convert markdown to HTML using the same extensions as the EPUB build.""" + return str(render_markdown_to_soup(md_content)) # ============================================================================= @@ -596,9 +645,8 @@ def copy_assets( copied: set[Path] = set() for page in state.pages: - try: - content = page.source.read_text(encoding="utf-8") - except (OSError, UnicodeDecodeError): + content = page.content if page.content is not None else read_source(page.source) + if content is None: continue for match in re.finditer(r"!\[[^\]]*\]\(([^)]+)\)", content): src = match.group(1).split(" ", 1)[0] @@ -641,11 +689,13 @@ def copy_assets( # ============================================================================= -def build_navigation(state: BuildState, current_url: str) -> list[dict[str, object]]: - """Group pages into the sidebar navigation tree, rooted at `current_url`. +def build_nav_skeleton(state: BuildState) -> list[dict[str, object]]: + """Group pages into sidebar sections once, keyed by section name. - Each item's `url` is a relative URL from `current_url` so the same nav - structure works from any page depth (top-level vs nested chapter folders). + The grouping (sections + ordered page list) is identical for every page, + so it is computed a single time and reused. `localize_nav` then turns the + absolute page URLs into per-page relative URLs ā€” an O(n) pass instead of + re-grouping all pages for every page (which was O(nĀ²)). """ sections: list[dict[str, object]] = [] section_map: dict[str, dict[str, object]] = {} @@ -653,27 +703,42 @@ def build_navigation(state: BuildState, current_url: str) -> list[dict[str, obje for page in state.pages: section = section_map.get(page.section) if section is None: - section = { - "name": page.section, - "items": [], - } + section = {"name": page.section, "items": []} section_map[page.section] = section sections.append(section) items = section["items"] assert isinstance(items, list) - items.append( - { - "title": page.title, - "url": relative_link(current_url, page.output_url), - "is_current": page.output_url == current_url, - "is_index": page.is_section_index, - } - ) + items.append(page) return sections +def localize_nav( + skeleton: list[dict[str, object]], current_url: str +) -> list[dict[str, object]]: + """Resolve the shared nav skeleton into relative URLs for `current_url`.""" + localized: list[dict[str, object]] = [] + for section in skeleton: + pages = section["items"] + assert isinstance(pages, list) + localized.append( + { + "name": section["name"], + "items": [ + { + "title": page.title, + "url": relative_link(current_url, page.output_url), + "is_current": page.output_url == current_url, + "is_index": page.is_section_index, + } + for page in pages + ], + } + ) + return localized + + def render_pages( config: WebsiteConfig, state: BuildState, @@ -684,16 +749,25 @@ def render_pages( template = env.get_template("page.html.j2") total = len(state.pages) - for idx, page in enumerate(state.pages): - nav = build_navigation(state, page.output_url) - try: - md_content = page.source.read_text(encoding="utf-8") - except UnicodeDecodeError as e: - raise RuntimeError(f"Failed to read {page.source}: {e}") from e + nav_skeleton = build_nav_skeleton(state) - html_content = render_markdown(md_content) - html_content = rewrite_links(html_content, page, state, config, logger) - toc = extract_toc(html_content) + for idx, page in enumerate(state.pages): + nav = localize_nav(nav_skeleton, page.output_url) + if page.content is not None: + md_content = page.content + else: + try: + md_content = page.source.read_text(encoding="utf-8") + except UnicodeDecodeError as e: + raise RuntimeError(f"Failed to read {page.source}: {e}") from e + + # Parse the rendered markdown once, then run heading-id normalisation, + # TOC extraction, and link rewriting against the same tree before a + # single serialization ā€” avoids reparsing the HTML three times per page. + soup = render_markdown_to_soup(md_content) + toc = extract_toc_from_soup(soup) + rewrite_links_in_soup(soup, page, state, config, logger) + html_content = str(soup) prev_page = state.pages[idx - 1] if idx > 0 else None next_page = state.pages[idx + 1] if idx < total - 1 else None