From a4795f19384f596a2bff0ce8935da58b4c662781 Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 22 Jul 2026 15:30:47 -0500 Subject: [PATCH 1/4] Convert Kotlin website Writerside content to JSON, add it to database, generate and add templates to display it to database, optimize and insert Kotlin website media. Script to sync current kotlin-stdlib documentation against a newly-generated documentation set (for now, used to do pruning for ADFA-4737 https://appdevforall.atlassian.net/browse/ADFA-4737) --- .../ProcessKotlinWebsiteJSON/README.md | 153 +++++ .../ProcessKotlinWebsiteJSON/build_nav.py | 216 ++++++ .../find_missing_assets.py | 142 ++++ .../insert_optimized_media.py | 446 +++++++++++++ .../ProcessKotlinWebsiteJSON/md_to_json.py | 619 ++++++++++++++++++ .../optimize_media.py | 570 ++++++++++++++++ .../ProcessKotlinWebsiteJSON/populate_db.py | 616 +++++++++++++++++ .../sync_kdoc_json_to_db.py | 203 ++++++ 8 files changed, 2965 insertions(+) create mode 100644 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md create mode 100644 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/build_nav.py create mode 100644 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/find_missing_assets.py create mode 100644 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/insert_optimized_media.py create mode 100644 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/md_to_json.py create mode 100644 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/optimize_media.py create mode 100644 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py create mode 100755 scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md new file mode 100644 index 000000000..b1c7f560b --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md @@ -0,0 +1,153 @@ +# Process Kotlin Website JSON + +Scripts for converting a `kotlin-web-site/docs` checkout (JetBrains +Writerside-flavored Markdown) into the JSON block schema this project's +templating engine renders, and for loading that JSON, its navigation tree, +and its media straight into a `documentation.db`-schema SQLite database. + +## Scripts + +| Script | Purpose | +|---|---| +| [`md_to_json.py`](md_to_json.py) | Converts every `topics/**/*.md` page into one JSON file (see schema below). Writes `theme.json` and copies `images/` into the output directory. | +| [`build_nav.py`](build_nav.py) | Builds `nav.json`/`nav.html` sidebar navigation from `kr.tree`, resolving each `` against `md_to_json.py`'s output. | +| [`find_missing_assets.py`](find_missing_assets.py) | QA pass: reports cross-page links, images, and `` targets in the source tree that don't resolve to anything. Reuses `md_to_json.py`'s own resolution logic, so it flags exactly what would end up broken on the rendered site. | +| [`populate_db.py`](populate_db.py) | The database path: converts the docs tree the same way `md_to_json.py` does, builds nav the same way `build_nav.py` does, and inserts pages + nav + images + CSS/JS directly into `documentation.db` (replacing everything under `k/html/` and `assets/`). Supports pruning whole `kr.tree` subtrees via `--blacklisted-element-titles`. | +| [`optimize_media.py`](optimize_media.py) | Standalone media optimizer: downscales/recompresses a directory of images (pngquant, Pillow, Scour/cairosvg for SVG) into a mirrored output directory. | +| [`insert_optimized_media.py`](insert_optimized_media.py) | Runs `optimize_media.py`'s pipeline over a directory of raw media, then replaces the corresponding `k/html/images/*` rows in an existing database, rewriting any page that referenced a renamed file and deleting anything left unreferenced. | + +## Requirements + +- Python 3.10+ +- `pip install markdown-it-py Pillow scour brotli` +- `cairosvg` (only needed if an optimized SVG exceeds `--svg-rasterize-threshold`): `pip install cairosvg` +- `pngquant` on `PATH` (e.g. `apt install pngquant`) — required by `optimize_media.py`/`insert_optimized_media.py`, and by `populate_db.py` for the images it inserts directly from the Writerside export. + +`populate_db.py` also expects, relative to its own location (not copied into +this directory — see note below): + +- `templates/page.peb`, `templates/nav.peb` — Pebble templates upserted into the `Templates` table. +- `assets/docs.css`, `assets/tabs.js`, `assets/sidebar.js` — static assets inserted at `assets/`. + +> **Note:** per ADFA-4737, only the `.py` files from the original working +> directory were copied here. `templates/` and `assets/` (and any `.config` +> files you use) must be placed alongside these scripts before running +> `populate_db.py`. + +## Inputs you need before starting + +- A checkout of `kotlin-web-site/docs` (the `` argument below) — contains `topics/`, `images/`, `v.list`, and `kr.tree`. +- A config JSON with theming colors, e.g.: + ```json + {"broken-ext-link-color": "#cc0000", "menu-no-link-color": "#999999"} + ``` +- Writerside's own image export zip (e.g. `webHelpImages.zip`, found next to `kr.tree`) if you're using `populate_db.py`. + +## Workflow: generate JSON + nav for a static/templated preview + +Use this to produce standalone JSON pages and nav data (not the database) +for local inspection or a different renderer. + +```bash +# 1. Convert every topic .md into JSON, one file per page +python3 md_to_json.py config.json + +# 2. Build the sidebar nav from kr.tree against that JSON output +python3 build_nav.py + +# 3. (optional) Check for broken links/images/includes in the source tree +python3 find_missing_assets.py missing-assets-report.md +``` + +`` ends up containing: +- `topics/**/*.json` — one page per source `.md` file (schema below) +- `theme.json` — the two theming colors, carried from `config.json` +- `images/` — copied straight from `/images/` +- `nav.json` / `nav.html` — sidebar tree and a pre-rendered static copy + +### Page JSON schema + +```json +{ + "id": "enum-classes", + "sourceFile": "topics/enum-classes.md", + "title": "Enum classes", + "blocks": [ { "type": "heading", "level": 2, "id": "...", "html": "..." }, "..." ] +} +``` + +Block types: `heading`, `paragraph`, `code`, `blockquote`, `list`, `table`, +`image`, `hr`, `tabs`, `note`/`tip`/`warning`, `html` (raw passthrough). See +the module docstring in [`md_to_json.py`](md_to_json.py) for full shapes and +known limitations (nested tabs, `` resolution, variable +substitution). + +## Workflow: generate + insert directly into the documentation database + +This is the path that actually populates `documentation.db`. It performs +the same conversion as `md_to_json.py`/`build_nav.py` internally — you don't +run those scripts first. + +```bash +python3 populate_db.py config.json [db-path] +``` + +- `db-path` defaults to `documentation.db` in the current directory, and must already exist with the expected schema (`Languages`, `ContentTypes`, `Templates` tables populated). +- A timestamped backup (`.backup-`) is written before any changes, via SQLite's `VACUUM INTO`. +- Everything under `k/html/` and `assets/` is deleted and re-inserted in a single transaction (rolled back on error), then the database is `VACUUM`ed. + +### Pruning documentation you don't want (ADFA-4737) + +To leave a whole `kr.tree` subtree out of the database entirely — nav +entry, converted pages, and all — pass `--blacklisted-element-titles` with +the full `toc-title` path from a top-level element down to the one you want +to drop. Levels are joined with `\/` (backslash-slash), not a bare `/`, +since a bare `/` commonly appears inside a real title. The example below is +illustrative only — open `/kr.tree` and copy the actual +`toc-title` chain for whatever section you're dropping (e.g. Kotlin/Wasm): + +```bash +python3 populate_db.py config.json documentation.db \ + --blacklisted-element-titles \ + "\/" +``` + +Any other page's in-content link to a pruned topic renders as a styled +"broken" link (via `broken-ext-link-color`) rather than a dead link with no +indication anything changed. Run with `--blacklisted-element-titles` first +against a scratch copy of the database and check the warnings on stderr for +any path that didn't match — that usually means the toc-title or ancestor +chain was copied wrong. + +## Workflow: optimizing and inserting media + +Two options, depending on whether the database already has pages loaded: + +**Standalone optimization only** (no database involved): + +```bash +python3 optimize_media.py [--max-width 500] [--webp] [...] +``` + +**Optimize and update an existing database's images in place:** + +```bash +python3 insert_optimized_media.py [work-dir] [options] +``` + +This re-runs `optimize_media.py`'s pipeline, backs up the database first, +replaces each `k/html/images/` row with the optimized bytes, rewrites +any page/nav reference to a file that got renamed during optimization (e.g. +`--webp` conversion or SVG rasterization), and deletes any image no page +references anymore. Both scripts share the same tuning flags +(`--max-width`, `--jpeg-quality`, `--webp`, `--webp-quality`, +`--pngquant-speed`, `--svg-precision`, `--svg-rasterize-threshold`, +`--verbose`, `--log-file`), settable via `--config ` instead of the +command line — see either script's module docstring for the full option +reference. + +## Recommended order for a full refresh + +1. `find_missing_assets.py` against the new `` — fix anything broken in the source before converting it. +2. `populate_db.py`, with `--blacklisted-element-titles` for anything you don't want documented (e.g. Kotlin/Wasm per ADFA-4737). +3. `insert_optimized_media.py` against the raw media directory, if you want optimized (resized/compressed) images rather than Writerside's own export as-is. diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/build_nav.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/build_nav.py new file mode 100644 index 000000000..ce3084d75 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/build_nav.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +""" +Builds sidebar navigation data/HTML from a JetBrains Writerside .tree file +(e.g. kotlin-web-site/docs/kr.tree), for use with templates/nav.peb. + +Usage: + python3 build_nav.py [--tree-file kr.tree] + + is the checkout of kotlin-web-site/docs (contains kr.tree). + is the output of md_to_json.py, used to resolve each + to the page's "id" (its path + relative to docs-root, e.g. "topics/ksp/ksp-overview") and, + when a has no toc-title, its rendered "title". + receives nav.json (the tree, for feeding into nav.peb) and + nav.html (a pre-rendered static copy of what nav.peb outputs). + +Writerside lets a both link to its own page (topic="...") and +contain nested children (sub-pages) at once, and topic +references are bare filenames resolved by uniqueness across the whole +topics/ subtree, not by path - this script relies on that same uniqueness. + +A topic="foo.md" that doesn't resolve to any converted page is treated as +manually deleted (rather than e.g. a typo) and dropped from the nav: if the +toc-element has no children either, build_node() returns None for it and it's +filtered out of its parent's children entirely; if it still has children +(sub-topics that do still exist), the element is kept as a link-less category +header for them instead of losing those children too. This only applies to +topic="*.md" references - kr.tree itself is never modified, so a purged +topic's (and any of its still-live children) stays in the tree +forever; this is what re-derives "not actually present" from the docs-json +output on every run instead. + +If /theme.json (written by md_to_json.py from its own config +argument) has a "menu-no-link-color", each node that doesn't actually lead to +a generated page - a pure category header with no topic="...", a +topic="foo.topic" that isn't a converted .md page (e.g. home.topic, +api-references.topic), or a topic="foo.md" whose page was deleted but which +still has live children - gets a "noLinkColor" of that value baked directly +into its entry in nav.json (null otherwise), which nav.peb turns into an +inline style. A toc-element using kr.tree's other, unrelated href="https://..." +attribute (e.g. "Test library (kotlin.test)" under API reference) is not +treated any differently here - it has no topic="...", so it's already a +no-link category header like any other; this script does not read that +external href at all, so no link is added for it. +""" +import argparse +import json +import re +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +HUMANIZE_RE = re.compile(r"[-_]+") + + +def humanize(stem: str) -> str: + return HUMANIZE_RE.sub(" ", stem).strip().title() + + +def load_page_index(docs_json_dir: Path) -> tuple[dict, dict]: + """Returns (stem -> id, id -> title) built from every generated page JSON.""" + stem_to_id = {} + id_to_title = {} + for json_path in docs_json_dir.rglob("*.json"): + page = json.loads(json_path.read_text(encoding="utf-8")) + page_id = page.get("id") + if not page_id: + continue + stem = Path(page_id).name + stem_to_id[stem] = page_id + id_to_title[page_id] = page.get("title") + return stem_to_id, id_to_title + + +def build_node(el: ET.Element, stem_to_id: dict, id_to_title: dict, warnings: list, no_link_color: str, + id_prefix: str = "") -> dict | None: + """Returns None when this element's own topic="*.md" no longer resolves + to a converted page (deleted) and it has no children left worth keeping - + callers must filter these out of whatever list they collect build_node() + results into (both build_node() itself, for its own children, and the + top-level toc-element loop in main()).""" + topic = el.get("topic") + toc_title = el.get("toc-title") + hidden = el.get("hidden") == "true" + + children = [ + build_node(c, stem_to_id, id_to_title, warnings, no_link_color, id_prefix) + for c in el.findall("toc-element") + ] + children = [c for c in children if c is not None] + + page_id = None + title = toc_title + no_link = True + deleted = False + if topic: + stem = Path(topic).stem + page_id = stem_to_id.get(stem) + if page_id is None: + if topic.endswith(".md"): + # Was a real page at some point, isn't anymore - the .md was + # manually deleted. Drop this node; if children still resolve + # to real pages, keep it around as a no-link category header + # for them rather than losing those children too. + deleted = True + if children: + warnings.append(f"topic={topic!r} has no converted page (deleted); " + f"keeping as a header for its {len(children)} remaining child page(s)") + else: + warnings.append(f"topic={topic!r} has no converted page (deleted); dropping from nav") + return None + else: + # Not a converted .md page (e.g. home.topic, api-references.topic): + # still rendered as a link (for consistency, and it's still the + # best URL guess) but it doesn't lead anywhere real, so it's + # colored the same as a no-topic-at-all category header. Prefixed + # the same way as a resolved stem_to_id hit would be (e.g. + # populate_db.py passes id_prefix="k/html/" since stem_to_id + # there already maps every real page to "k/html/"), so + # this fallback id follows the same URL convention as everything + # else on the page rather than silently reverting to a bare one. + page_id = f"{id_prefix}{stem}" + warnings.append(f"no converted page for topic={topic!r}; using id={page_id!r}") + else: + no_link = False + if not title: + title = (id_to_title.get(page_id) if page_id else None) or humanize(stem) + elif not title: + title = "Untitled" + + return { + "title": title, + "id": None if deleted else page_id, + "hidden": hidden, + "noLinkColor": no_link_color if (no_link or deleted) else None, + "children": children, + } + + +def render_node(node: dict, indent: int = 0) -> str: + # Kept byte-identical to templates/nav.peb's renderNavNode macro, since + # RenderDocs.java re-renders nav.peb over nav.json for the live site and + # this function only produces a standalone preview copy of the same HTML. + classes = "nav-item" + (" nav-hidden" if node["hidden"] else "") + has_children = bool(node["children"]) + style = f' style="color: {node["noLinkColor"]};"' if node["noLinkColor"] else "" + if node["id"]: + label = f'{node["title"]}' + else: + label = f'{node["title"]}' + toggle = ( + f'' + if has_children else '' + ) + parts = [f'
  • ', '
    ', toggle, label, "
    "] + if has_children: + parts.append('
      ') + for child in node["children"]: + parts.append(render_node(child)) + parts.append("
    ") + parts.append("
    • ") + return "".join(parts) + + +def render_html(tree: list) -> str: + body = "".join(render_node(node) for node in tree) + return ( + '" + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("docs_root", type=Path, help="Path to kotlin-web-site/docs") + parser.add_argument("docs_json_dir", type=Path, help="Path to md_to_json.py output directory") + parser.add_argument("output_dir", type=Path, help="Directory to write nav.json and nav.html into") + parser.add_argument("--tree-file", default="kr.tree", help="Filename of the Writerside tree file under docs_root") + args = parser.parse_args() + + tree_path = args.docs_root / args.tree_file + if not tree_path.is_file(): + print(f"error: {tree_path} does not exist", file=sys.stderr) + sys.exit(1) + + stem_to_id, id_to_title = load_page_index(args.docs_json_dir) + + theme_path = args.docs_json_dir / "theme.json" + no_link_color = None + if theme_path.is_file(): + no_link_color = json.loads(theme_path.read_text(encoding="utf-8")).get("menu-no-link-color") + else: + print(f"warning: {theme_path} not found (run md_to_json.py first); " + f"no-link menu items won't be colored", file=sys.stderr) + + root = ET.parse(tree_path).getroot() + warnings = [] + nav_tree = [build_node(el, stem_to_id, id_to_title, warnings, no_link_color) for el in root.findall("toc-element")] + nav_tree = [node for node in nav_tree if node is not None] + + for w in warnings: + print(f"warning: {w}", file=sys.stderr) + + args.output_dir.mkdir(parents=True, exist_ok=True) + (args.output_dir / "nav.json").write_text( + json.dumps(nav_tree, separators=(",", ":"), ensure_ascii=False), encoding="utf-8" + ) + (args.output_dir / "nav.html").write_text(render_html(nav_tree), encoding="utf-8") + + print(f"Wrote nav.json and nav.html ({len(warnings)} warning(s)) into {args.output_dir}") + + +if __name__ == "__main__": + main() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/find_missing_assets.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/find_missing_assets.py new file mode 100644 index 000000000..653982113 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/find_missing_assets.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +""" +Documents every reference in the docs source (kotlin-web-site/docs) that +points at an asset - another topic page, an image, or an target - +which doesn't actually exist, so broken source content can be found and +fixed without having to read every generated page. + +Usage: + python3 find_missing_assets.py [report-path] [--topics-subdir topics] [--images-subdir images] + +Reuses md_to_json.py's own link/image resolution (build_topic_index, +build_image_index, Converter) instead of re-parsing links with regexes, so +this reports exactly what would end up broken in the rendered site - e.g. a +"foo.md" written inside a fenced code sample (showing readers what Writerside +markup looks like) is correctly ignored, since it's never tokenized as a +real link in the first place. + + targets are a separate check: those +aren't links/images so md_to_json.py's Converter never resolves them, but a +missing include is still a broken asset reference worth surfacing. This is +checked with a small standalone regex scan (fenced code blocks stripped +first) against every filename that exists anywhere under /, +regardless of extension (include targets can be ".md" or ".topic"). +""" +import argparse +import re +import sys +from collections import defaultdict +from pathlib import Path + +from md_to_json import Converter, build_image_index, build_topic_index, load_variables, make_markdown_it + +INCLUDE_RE = re.compile(r']*\bfrom\s*=\s*"([^"]+)"') +FENCE_RE = re.compile(r"```.*?```", re.S) + + +def find_include_warnings(topics_dir: Path) -> list: + all_filenames = {p.name for p in topics_dir.rglob("*") if p.is_file()} + warnings = [] + for md_path in sorted(topics_dir.rglob("*.md")): + text_no_fences = FENCE_RE.sub("", md_path.read_text(encoding="utf-8")) + source_rel = str(md_path.relative_to(topics_dir.parent)) + for target in INCLUDE_RE.findall(text_no_fences): + if target not in all_filenames: + warnings.append({"kind": "include", "source": source_rel, "reference": target}) + return warnings + + +def group_by_reference(warnings: list) -> dict: + grouped = defaultdict(set) + for w in warnings: + grouped[w["reference"]].add(w["source"]) + return {ref: sorted(sources) for ref, sources in sorted(grouped.items())} + + +def render_section(title: str, grouped: dict) -> str: + lines = [f"## {title} ({len(grouped)})", ""] + if not grouped: + lines.append("None.") + for ref, sources in grouped.items(): + lines.append(f"- `{ref}`") + for src in sources: + lines.append(f" - {src}") + lines.append("") + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("docs_root", type=Path, help="Path to kotlin-web-site/docs") + parser.add_argument("report_path", type=Path, nargs="?", default=Path("missing-assets-report.md"), + help="Where to write the Markdown report (default: ./missing-assets-report.md)") + parser.add_argument("--topics-subdir", default="topics", help="Subdirectory of docs_root holding .md files") + parser.add_argument("--images-subdir", default="images", help="Subdirectory of docs_root holding image files") + args = parser.parse_args() + + docs_root: Path = args.docs_root + topics_dir = docs_root / args.topics_subdir + images_dir = docs_root / args.images_subdir + if not topics_dir.is_dir(): + print(f"error: {topics_dir} is not a directory", file=sys.stderr) + sys.exit(1) + + variables = load_variables(docs_root) + topic_index = build_topic_index(topics_dir) + image_index, image_collisions = build_image_index(images_dir) + md = make_markdown_it() + converter = Converter(md, variables, topic_index, image_index) + + md_files = sorted(topics_dir.rglob("*.md")) + for md_path in md_files: + rel = md_path.relative_to(topics_dir) + page_id = str(rel.with_suffix("")) + source_rel = str(Path(args.topics_subdir) / rel) + try: + converter.convert_file(md_path, page_id, source_rel) + except Exception as exc: # noqa: BLE001 - surface which file broke, keep auditing the rest + print(f"error scanning {md_path}: {exc}", file=sys.stderr) + + link_warnings = [w for w in converter.warnings if w["kind"] == "link"] + image_warnings = [w for w in converter.warnings if w["kind"] == "image"] + include_warnings = find_include_warnings(topics_dir) + + links = group_by_reference(link_warnings) + images = group_by_reference(image_warnings) + includes = group_by_reference(include_warnings) + collisions = {name: rels for name, rels in image_collisions} + + report = [ + "# Missing Assets Report", + "", + f"Scanned {len(md_files)} Markdown files under `{topics_dir}`.", + "", + "## Summary", + "", + f"- {len(links)} unresolved cross-page link target(s)", + f"- {len(images)} missing image(s)", + f"- {len(collisions)} ambiguous image filename(s) (exist in more than one place under images/)", + f"- {len(includes)} unresolved `` target(s)", + "", + render_section("Unresolved cross-page links", links), + render_section("Missing images", images), + render_section("Unresolved targets", includes), + f"## Ambiguous image filenames ({len(collisions)})", + "", + ] + if not collisions: + report.append("None.") + for name, rels in collisions.items(): + report.append(f"- `{name}`") + report.append(f" - used: images/{rels[0]}") + for rel in rels[1:]: + report.append(f" - ignored: images/{rel}") + report.append("") + + args.report_path.write_text("\n".join(report), encoding="utf-8") + print(f"Wrote {args.report_path} " + f"({len(links)} links, {len(images)} images, {len(collisions)} ambiguous, {len(includes)} includes)") + + +if __name__ == "__main__": + main() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/insert_optimized_media.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/insert_optimized_media.py new file mode 100644 index 000000000..c0624d312 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/insert_optimized_media.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python3 +""" +insert_optimized_media.py + +Runs optimize_media.py's image optimizer over a directory of raw media, +then updates an existing documentation.db-schema database (as +populate_db.py produces) with the optimized results, fixing up every page +that referenced a file under its old name. + +What this does, inside a single transaction (rolled back on any error): + 1. Backs up first, same as populate_db.py (VACUUM INTO a + timestamped sibling file). + 2. Optimizes every file under into a staging directory + (--work-dir, or a temporary one removed afterwards) via + optimize_media.py's own pipeline - see its own module docstring for + what "optimized" means (resize, pngquant, Scour, optional WEBP + conversion / SVG rasterization). Aborts before touching the database + if any file fails to optimize. + 3. Replaces every "k/html/images/" Content row with the optimized + bytes, deleting the old row (and any leftover chunked fragments) first + - Content.path is UNIQUE, so a stale row has to go before its + replacement can be inserted. Images are addressed by bare filename + only, matching populate_db.py's own flat "k/html/images/*" convention: + subdirectories are flattened to their basename, and a + basename collision across two different subdirectories is a warning + (keeping the first, sorted, skipping the rest), not an error. + 4. Wherever optimization renamed a file (webp conversion, or an oversized + SVG rasterized to PNG/WEBP), rewrites every "/k/html/images/" + reference still pointing at the old name, in every k/html/*.html page + and the nav row, to the new name - so a page doesn't end up linking to + a filename that no longer exists. + 5. Deletes every remaining "k/html/images/" row (base row and any + chunked fragments) that, after the rename rewriting above, no + k/html/*.html page or the nav row references even once - not just ones + touched by this run's rename_map, but every currently-stored image, + so media that fell out of use in an earlier run (e.g. a topic's .md + was deleted, or an reference was removed by hand) gets cleaned + up too, not just this run's renames. + 6. VACUUMs the database afterwards (outside the transaction - SQLite + refuses to VACUUM inside one), same as populate_db.py. + +Usage: + python3 insert_optimized_media.py [work_dir] [options] + python3 insert_optimized_media.py --config myjob.config + + are optimize_media.py's own tuning flags (--max-width, +--jpeg-quality, --webp, --webp-quality, --pngquant-speed, --svg-precision, +--svg-rasterize-threshold, --verbose, --log-file, --config) - see +optimize_media.py's own docstring for what each one does. media-dir/db-path/ +work-dir can also be set via --config (as "input-dir"/"db-path"/ +"output-dir"), the same as optimize_media.py's own options. + +Note: --webp requires this database's ContentTypes table to already have an +"image/webp" row (checked up front, before any optimization work starts) - +this project's documentation.db doesn't ship with one. +""" +import argparse +import re +import shutil +import sqlite3 +import sys +import tempfile +from pathlib import Path + +import brotli + +from optimize_media import ( + BUILTIN_DEFAULTS, Logger, OPTION_SPECS, add_optimize_arguments, find_pngquant, optimize_directory, + resolve_config, +) +from populate_db import ( + CHUNK_SIZE, EXTENSION_TO_CONTENT_TYPE, IMAGES_DB_PATH_PREFIX, IMAGES_URL_PREFIX, LANGUAGE, PAGE_CONTENT_TYPE, + backup_database, get_content_type, get_id, insert_chunked_content, +) + +WEBP_CONTENT_TYPE = "image/webp" +# populate_db.py's own EXTENSION_TO_CONTENT_TYPE has no ".webp" entry - its +# image source (a Writerside export) never produces one, but +# optimize_media.py's --webp does, so it's added here rather than touching +# that shared dict. +IMAGE_EXTENSION_TO_CONTENT_TYPE = {**EXTENSION_TO_CONTENT_TYPE, ".webp": WEBP_CONTENT_TYPE} + +# This script's own options, layered on top of optimize_media.py's (db-path +# has no equivalent there) - passed to resolve_config/load_config_file so +# --config can set any of them, the same mechanism optimize_media.py uses +# for its own options. +OWN_OPTION_SPECS = {**OPTION_SPECS, "db-path": ("db_path", Path)} + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("input_dir", type=Path, nargs="?", default=None, metavar="media_dir", + help="Directory of raw media to optimize, recursively (or set input-dir in --config)") + parser.add_argument("db_path", type=Path, nargs="?", default=None, + help="SQLite database to update, e.g. documentation.db (or set db-path in --config)") + parser.add_argument("output_dir", type=Path, nargs="?", default=None, metavar="work_dir", + help="Staging directory for optimized files; default: a temporary directory removed " + "afterwards (or set output-dir in --config)") + add_optimize_arguments(parser) + return parser + + +def delete_content(conn, path: str) -> None: + """Deletes a Content row and any chunked continuation fragments for it + (see insert_chunked_content/CHUNK_SIZE) - safe to call even if nothing + exists yet at that path. Content.path is UNIQUE, so this has to run + before any re-insert at the same path.""" + conn.execute("DELETE FROM Content WHERE path = ? OR path LIKE ?", (path, f"{path}-%")) + + +def insert_optimized_file(conn, data: bytes, name: str, db_path: str, language_id: int, content_type_cache: dict, + chunked_log: list) -> bool: + """Inserts one already-optimized file's bytes as-is. Unlike + populate_db.py's own insert_file, this does not run pngquant itself - + optimize_media.py already did, and running it again here would just + re-quantize an already-quantized image for no benefit. Returns False + (skipping the file, with a warning) for an extension with no known + content type.""" + content_type_value = IMAGE_EXTENSION_TO_CONTENT_TYPE.get(Path(name).suffix.lower()) + if content_type_value is None: + print(f"warning: no known content type for {name!r}; skipping", file=sys.stderr) + return False + if content_type_value not in content_type_cache: + content_type_cache[content_type_value] = get_content_type(conn, content_type_value) + content_type_id, compress = content_type_cache[content_type_value] + + if compress: + data = brotli.compress(data) + delete_content(conn, db_path) + insert_chunked_content(conn, db_path, language_id, content_type_id, 0, data, chunked_log) + return True + + +def build_rename_map(manifest: dict, logger: Logger) -> dict: + """Flattens optimize_directory's {relative_src: relative_dst} manifest + to {old_basename: new_basename}, matching k/html/images/*'s bare-filename + addressing. Warns (keeping the first) if two different renames collide + on the same old basename - e.g. two identically-named files in + different subdirectories of media_dir.""" + rename_map = {} + for old_rel, new_rel in sorted(manifest.items()): + old_name = Path(old_rel).name + new_name = Path(new_rel).name + if old_name == new_name: + continue + if old_name in rename_map and rename_map[old_name] != new_name: + logger.error( + f"warning: {old_rel!r} and an earlier file both renamed from {old_name!r}, to different names " + f"({rename_map[old_name]!r} vs {new_name!r}); keeping the first" + ) + continue + rename_map[old_name] = new_name + return rename_map + + +def reassemble_content(conn, path: str, first_content: bytes) -> bytes: + """Reassembles a possibly-chunked row's full bytes - mirrors + WebServer.kt's own reassembly protocol (see CHUNK_SIZE's docstring in + populate_db.py): a row is fragmented purely when its content is exactly + CHUNK_SIZE bytes, in which case "-1", "-2", ... are + concatenated until a missing or shorter-than-CHUNK_SIZE row is hit.""" + if len(first_content) < CHUNK_SIZE: + return first_content + parts = [first_content] + n = 1 + while True: + row = conn.execute("SELECT content FROM Content WHERE path = ?", (f"{path}-{n}",)).fetchone() + if row is None: + break + parts.append(row[0]) + if len(row[0]) < CHUNK_SIZE: + break + n += 1 + return b"".join(parts) + + +def rewrite_pages(conn, rename_map: dict, language_id: int, page_content_type_id: int, logger: Logger, + chunked_log: list) -> int: + """Rewrites every k/html/*.html page (and the nav row) that references a + renamed image, replacing "/k/html/images/" with + "/k/html/images/" wherever it appears. Operates directly on + each row's decompressed JSON text rather than parsing it: every image + reference is a literal IMAGES_URL_PREFIX+filename substring, baked in at + conversion time by md_to_json.py's Converter (resolve_image_src), so a + plain text substitution finds it correctly regardless of which block + type it ends up nested inside - no need to understand that nested block + schema here. The match is anchored on the escaped quote (\\") that + always immediately follows a rewritten src="..." attribute in the + stored JSON (see resolve_image_src/rewrite_urls - image references are + only ever embedded as HTML attributes, never as a bare JSON field on + their own), so a renamed file's name can't accidentally match as a + prefix of some other, unrelated, longer filename. Returns the number of + rows changed. + + ".html" is the exact literal suffix populate_db.py gives every base + page/nav row; fragment continuation rows are named "-" (the + "-N" appended after the ".html" already in path), so the path filter + below naturally excludes them without needing to detect chunking up + front.""" + if not rename_map: + return 0 + + rows = conn.execute( + "SELECT path, content, templateId FROM Content WHERE path LIKE 'k/html/%.html' AND contentTypeID = ? " + "AND templateId != 0", + (page_content_type_id,), + ).fetchall() + + changed = 0 + for path, first_content, template_id in rows: + full = reassemble_content(conn, path, first_content) + text = brotli.decompress(full).decode("utf-8") + new_text = text + hits = 0 + for old_name, new_name in rename_map.items(): + old_ref = f'{IMAGES_URL_PREFIX}{old_name}\\"' + new_ref = f'{IMAGES_URL_PREFIX}{new_name}\\"' + hits += new_text.count(old_ref) + new_text = new_text.replace(old_ref, new_ref) + if new_text == text: + continue + blob = brotli.compress(new_text.encode("utf-8")) + delete_content(conn, path) + insert_chunked_content(conn, path, language_id, page_content_type_id, template_id, blob, chunked_log) + changed += 1 + logger.info(f"[URL FIX] {path}: updated {hits} image reference(s)") + return changed + + +# Matches a rewritten image src's filename, anchored the same way +# rewrite_pages' own known-rename substitutions are: resolve_image_src/ +# rewrite_urls only ever embed an image reference as an HTML src="..." +# attribute, which - JSON-encoded - always has the escaped quote (\") right +# after it, so this can't accidentally swallow past the end of the filename. +IMAGE_REF_RE = re.compile(re.escape(IMAGES_URL_PREFIX) + r'([^\\"]+)\\"') + + +def collect_referenced_media(conn, page_content_type_id: int) -> set: + """Bare filenames (e.g. "mascot.png") referenced by at least one + src="/k/html/images/" anywhere across current k/html/*.html page + content and the nav row - the same row selection/reassembly + rewrite_pages uses, just extracting every image reference found instead + of only substituting the ones in a known rename_map.""" + rows = conn.execute( + "SELECT path, content FROM Content WHERE path LIKE 'k/html/%.html' AND contentTypeID = ? AND templateId != 0", + (page_content_type_id,), + ).fetchall() + referenced = set() + for path, first_content in rows: + full = reassemble_content(conn, path, first_content) + text = brotli.decompress(full).decode("utf-8") + referenced.update(IMAGE_REF_RE.findall(text)) + return referenced + + +def list_stored_media(conn) -> dict: + """Bare filename -> Content.path (e.g. "mascot.png" -> "k/html/images/ + mascot.png") for every image currently stored under IMAGES_DB_PATH_PREFIX, + collapsing chunked continuation fragments ("-1", "-2", ...) + back into their base row, since deleting the base via delete_content + already takes its fragments with it (see CHUNK_SIZE's docstring in + populate_db.py for that fragmentation convention). A path is treated as + a fragment when stripping a trailing "-" yields another path + that's also present - the same convention this whole pipeline already + relies on elsewhere, ambiguous only for a base filename that itself + looks like "-", which no real optimized + media filename does.""" + paths = {row[0] for row in conn.execute( + "SELECT path FROM Content WHERE path LIKE ?", (f"{IMAGES_DB_PATH_PREFIX}%",) + )} + + def is_fragment(path: str) -> bool: + prefix, sep, suffix = path.rpartition("-") + return sep == "-" and suffix.isdigit() and prefix in paths + + return {path[len(IMAGES_DB_PATH_PREFIX):]: path for path in paths if not is_fragment(path)} + + +def delete_unreferenced_media(conn, page_content_type_id: int, logger: Logger) -> int: + """Deletes every currently-stored k/html/images/ row (base row and + any chunked fragments) that no page or the nav row references even once. + Must run after insertion and rename-rewriting, so it sees the final, + up-to-date state of both stored media and in-content references - a file + renamed this run is only "unreferenced" under its stale old name, which + rewrite_pages will have already fixed up by the time this runs. Returns + the number of images removed.""" + stored = list_stored_media(conn) + referenced = collect_referenced_media(conn, page_content_type_id) + removed = 0 + for name, path in sorted(stored.items()): + if name in referenced: + continue + delete_content(conn, path) + removed += 1 + logger.info(f"[UNUSED] removed {path} (not referenced by any page)") + return removed + + +def main() -> None: + parser = build_parser() + args = parser.parse_args() + + try: + cfg = resolve_config(args, OWN_OPTION_SPECS, BUILTIN_DEFAULTS) + except RuntimeError as exc: + parser.error(str(exc)) + return + + if cfg["input_dir"] is None or cfg["db_path"] is None: + parser.error("media_dir and db_path must be given either as positional arguments or in --config") + + log_file_handle = open(cfg["log_file"], "w", encoding="utf-8") if cfg["log_file"] else None + logger = Logger(log_file_handle) + work_dir_is_temp = cfg["output_dir"] is None + work_dir = cfg["output_dir"] or Path(tempfile.mkdtemp(prefix="insert_optimized_media_")) + + try: + if not cfg["input_dir"].is_dir(): + logger.error(f"error: {cfg['input_dir']} is not a directory") + sys.exit(1) + if not cfg["db_path"].is_file(): + logger.error(f"error: {cfg['db_path']} does not exist") + sys.exit(1) + + if cfg["verbose"]: + logger.info("Config parameters:") + for key, (dest, _converter) in OWN_OPTION_SPECS.items(): + logger.info(f" {key} = {cfg.get(dest)}") + logger.info(f" work-dir = {work_dir}{' (temporary)' if work_dir_is_temp else ''}") + if args.config: + logger.info(f" (loaded from {args.config})") + + try: + pngquant_path = find_pngquant() + except RuntimeError as exc: + logger.error(f"error: {exc}") + sys.exit(1) + + # Fail fast on a schema this database doesn't support - before + # spending time optimizing every file - rather than discovering it + # partway through the (rolled-back, but still wasted) DB transaction. + preflight_conn = sqlite3.connect(cfg["db_path"]) + try: + get_id(preflight_conn, "Languages", LANGUAGE) + get_id(preflight_conn, "ContentTypes", PAGE_CONTENT_TYPE) + if cfg["webp"]: + get_content_type(preflight_conn, WEBP_CONTENT_TYPE) + except RuntimeError as exc: + logger.error(f"error: {exc}") + sys.exit(1) + finally: + preflight_conn.close() + + work_dir.mkdir(parents=True, exist_ok=True) + stats = {"raster": 0, "svg": 0, "svg_rasterized": 0, "copied": 0, "errors": 0, "original_bytes": 0, + "optimized_bytes": 0} + logger.info(f"Optimizing media from {cfg['input_dir']} into {work_dir}...") + manifest = optimize_directory(cfg["input_dir"], work_dir, cfg=cfg, pngquant_path=pngquant_path, + logger=logger, stats=stats) + if stats["errors"]: + logger.error( + f"error: {stats['errors']} file(s) failed to optimize; aborting before touching the database" + ) + sys.exit(1) + rename_map = build_rename_map(manifest, logger) + + logger.info(f"Backing up {cfg['db_path']}...") + backup_path = backup_database(cfg["db_path"]) + logger.info(f"Backup written to {backup_path}") + + conn = sqlite3.connect(cfg["db_path"]) + try: + conn.execute("BEGIN") + language_id = get_id(conn, "Languages", LANGUAGE) + page_content_type_id = get_id(conn, "ContentTypes", PAGE_CONTENT_TYPE) + + content_type_cache = {} + chunked_log = [] + inserted = 0 + seen_names = {} + for out_path in sorted(work_dir.rglob("*")): + if out_path.is_dir(): + continue + name = out_path.name + if name in seen_names: + logger.error( + f"warning: {out_path} has the same filename as {seen_names[name]}; keeping the first, " + "skipping this one" + ) + continue + seen_names[name] = out_path + db_path = f"{IMAGES_DB_PATH_PREFIX}{name}" + if insert_optimized_file(conn, out_path.read_bytes(), name, db_path, language_id, content_type_cache, + chunked_log): + inserted += 1 + if cfg["verbose"]: + logger.info(f"[OK] {out_path} -> {db_path}") + + # A renamed file's old basename no longer appears anywhere under + # work_dir (that's what makes it a rename), so the loop above + # never visits its old db_path to replace it - it'd otherwise + # linger forever as an orphaned, no-longer-referenced row. + removed = 0 + for old_name in rename_map: + old_db_path = f"{IMAGES_DB_PATH_PREFIX}{old_name}" + delete_content(conn, old_db_path) + removed += 1 + if cfg["verbose"]: + logger.info(f"[REMOVED] {old_db_path} (renamed to {IMAGES_DB_PATH_PREFIX}{rename_map[old_name]})") + + changed_pages = rewrite_pages(conn, rename_map, language_id, page_content_type_id, logger, chunked_log) + + unreferenced_removed = delete_unreferenced_media(conn, page_content_type_id, logger) + + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() + + logger.info("Vacuuming database to reclaim freed space...") + vacuum_conn = sqlite3.connect(cfg["db_path"]) + try: + vacuum_conn.execute("VACUUM") + finally: + vacuum_conn.close() + + logger.info( + f"Done: inserted/updated {inserted} image(s) in {cfg['db_path']}, {removed} stale renamed-away row(s) " + f"removed, {changed_pages} page(s)/nav row(s) updated to match {len(rename_map)} renamed file(s), " + f"{unreferenced_removed} unreferenced image(s) deleted." + ) + if chunked_log: + logger.info(f"Chunked {len(chunked_log)} file(s) over {CHUNK_SIZE:,} bytes:") + for path, total_size, chunk_count in chunked_log: + logger.info(f" {path}: {total_size:,} bytes -> {chunk_count} chunks") + finally: + if log_file_handle is not None: + log_file_handle.close() + if work_dir_is_temp: + shutil.rmtree(work_dir, ignore_errors=True) + + +if __name__ == "__main__": + main() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/md_to_json.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/md_to_json.py new file mode 100644 index 000000000..352ac15f7 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/md_to_json.py @@ -0,0 +1,619 @@ +#!/usr/bin/env python3 +""" +Converts JetBrains Writerside-flavored Markdown (as used by kotlin-web-site/docs) +into a simple JSON block schema suitable for a templating engine. + +Usage: + python3 md_to_json.py [--topics-subdir topics] + + is the checkout of kotlin-web-site/docs (contains v.list, topics/, ...). +One JSON file is written per input .md file, mirroring its relative path under +. + + is a JSON file with: + {"broken-ext-link-color": "#cc0000", "menu-no-link-color": "#999999"} +"broken-ext-link-color" colors tags in the rendered content that are +either off-site (any http(s)/mailto: link) or a same-tree ".md" reference +that doesn't resolve to a real page. "menu-no-link-color" isn't used here - +it's carried through to /theme.json for build_nav.py (which +builds the sidebar from kr.tree, a separate input this script doesn't read) +to pick up. + +Output schema (one object per page): +{ + "id": "enum-classes", + "sourceFile": "topics/enum-classes.md", + "title": "Enum classes", + "blocks": [ , ... ] +} + +Block shapes: + {"type": "heading", "level": 2, "id": "anonymous-classes", "html": "..."} + {"type": "paragraph", "html": "..."} + {"type": "code", "lang": "kotlin", "code": "...", "attrs": {"kotlin-runnable": "true"}} + {"type": "blockquote", "attrs": {"style": "note"}, "blocks": [...]} + {"type": "list", "ordered": false, "items": [{"blocks": [...]}]} + {"type": "table", "headers": ["a", "b"], "rows": [["1", "2"]]} + {"type": "image", "src": "...", "alt": "..."} + {"type": "hr"} + {"type": "tabs", "attrs": {"group": "build-system"}, + "tabs": [{"title": "Gradle", "attrs": {"group-key": "gradle"}, "blocks": [...]}]} + {"type": "html", "html": ""} + +Cross-page links (`[text](other-page.md#anchor)`) and images (`![alt](foo.png)`) +use Writerside's bare-filename convention - the referenced file is looked up +by name anywhere under / (links) or images/ (images), the same +way kr.tree's topic="..." references are resolved. Rendered HTML rewrites +these to root-relative URLs that only resolve once this JSON has been passed +through templates/page.peb and rendered by RenderDocs: links become +"/.html#anchor", and images become "/images/". +The images/ directory itself is copied to /images/ so the +resolved paths have something to point at. + +Known limitations (fine for a first pass, worth revisiting before production use): + - / grouping and attribute-line merging (the "{...}" line after a + fence/blockquote) only happen at the top level of a page and inside list + items/blockquotes/table cells one level deep; deeply nested tabs-in-tabs + are not handled. + - // admonitions are recognized as block-level tags. The + inline, single-line form seen inside HTML tables (e.g. roadmap.md) is passed + through as raw "html" blocks instead of being unpacked, since those pages + are basically hand-written HTML tables rather than prose. + - elements are passed through as raw "html" blocks; resolving them + to the referenced snippet is not implemented. + - %variables% (defined in v.list) are substituted textually in rendered HTML + and code, using simple %name% -> value replacement. + - A handful of images/ filenames collide across subdirectories (leftover + duplicates in the source tree); resolution keeps the first match in sorted + order and prints a warning rather than guessing which one is "correct". +""" +import argparse +import json +import re +import shutil +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +from markdown_it import MarkdownIt +from markdown_it.token import Token + +TITLE_RE = re.compile(r"^\[//\]:\s*#\s*\(title:\s*(.*?)\)\s*$", re.MULTILINE) +ATTR_LINE_RE = re.compile(r"^\{(.*)\}$") +ATTR_PAIR_RE = re.compile(r'([\w-]+)=(?:"([^"]*)"|(\S+))') +TAG_RE = re.compile(r"^<(/?)(tabs|tab|note|tip|warning)([^>]*)/?>$", re.I) +VAR_RE = re.compile(r"%([\w.-]+)%") +MD_LINK_RE = re.compile(r"^([\w.-]+)\.md(#.*)?$") +EXTERNAL_HREF_RE = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+.-]*:)?//|^mailto:", re.I) +LINK_TAG_RE = re.compile(r']*\bhref="([^"]*)"[^>]*>') + +CONTAINER_TAGS = {"tabs", "tab", "note", "tip", "warning"} + + +def build_topic_index(topics_dir: Path) -> dict: + """Bare filename stem (e.g. "enum-classes") -> page id (e.g. "kotlin-tour/enum-classes").""" + index = {} + for md_path in sorted(topics_dir.rglob("*.md")): + page_id = str(md_path.relative_to(topics_dir).with_suffix("")).replace("\\", "/") + index.setdefault(md_path.stem, page_id) + return index + + +def build_image_index(images_dir: Path): + """Bare filename (e.g. "mascot-main.png") -> path relative to images_dir. + + Returns (index, collisions), where collisions is a list of + (filename, [candidate relative paths]) for filenames that exist in more + than one place under images_dir - index keeps the first (sorted) one.""" + index = {} + candidates = {} + if not images_dir.is_dir(): + return index, [] + for img_path in sorted(images_dir.rglob("*")): + if not img_path.is_file(): + continue + rel = img_path.relative_to(images_dir).as_posix() + candidates.setdefault(img_path.name, []).append(rel) + index.setdefault(img_path.name, rel) + collisions = [(name, rels) for name, rels in sorted(candidates.items()) if len(rels) > 1] + for name, rels in collisions: + print(f"warning: ambiguous image filename {name!r}: " + f"using images/{rels[0]}, ignoring {', '.join('images/' + r for r in rels[1:])}", file=sys.stderr) + return index, collisions + + +def load_variables(docs_root: Path) -> dict: + v_list = docs_root / "v.list" + if not v_list.exists(): + return {} + tree = ET.parse(v_list) + return {el.get("name"): el.get("value") for el in tree.getroot().findall("var")} + + +def substitute_vars(text: str, variables: dict) -> str: + if not text: + return text + return VAR_RE.sub(lambda m: variables.get(m.group(1), m.group(0)), text) + + +def parse_attrs(attr_str: str) -> dict: + attrs = {} + for name, quoted, bare in ATTR_PAIR_RE.findall(attr_str or ""): + attrs[name] = quoted if quoted != "" or '="' in (attr_str or "") else bare + return attrs + + +def extract_title(raw_text: str): + m = TITLE_RE.search(raw_text) + if not m: + return None, raw_text + title = m.group(1).strip() + remaining = raw_text[: m.start()] + raw_text[m.end():] + return title, remaining + + +def slugify(text: str) -> str: + slug = re.sub(r"[^\w\s-]", "", text.lower()).strip() + return re.sub(r"[\s_]+", "-", slug) + + +class Node: + """Generic open/close tree built from markdown-it's flat token stream.""" + + __slots__ = ("token", "children") + + def __init__(self, token: Token): + self.token = token + self.children = [] + + +def build_tree(tokens) -> list: + root = [] + stack = [root] + for tok in tokens: + if tok.nesting == 1: + node = Node(tok) + stack[-1].append(node) + stack.append(node.children) + elif tok.nesting == -1: + stack.pop() + else: + stack[-1].append(Node(tok)) + return root + + +class Converter: + def __init__(self, md: MarkdownIt, variables: dict, topic_index: dict = None, image_index: dict = None, + broken_ext_link_color: str = None, image_url_prefix: str = "/images/"): + self.md = md + self.variables = variables + self.topic_index = topic_index or {} + self.image_index = image_index or {} + self.broken_ext_link_color = broken_ext_link_color + # Overridable so a different deployment target (e.g. populate_db.py's + # database-backed site, which serves images from "/k/html/images/" + # rather than a bare "/images/") can retarget every image src without + # a separate rewrite pass - resolve_image_src just uses this prefix + # directly. + self.image_url_prefix = image_url_prefix + self.current_source = None + # Populated as a side effect of resolve_href/resolve_image_src failing + # to resolve a reference; find_missing_assets.py reuses this same + # resolution logic (rather than re-parsing links with regexes) by + # running convert_file over every page and reading this list back. + self.warnings = [] + + def resolve_href(self, href: str): + """"other-page.md#anchor" -> "/.html#anchor", or None to leave href untouched.""" + m = MD_LINK_RE.match(href or "") + if not m: + return None + stem, anchor = m.groups() + page_id = self.topic_index.get(stem) + if page_id is None: + print(f"warning: link to unknown topic {href!r}", file=sys.stderr) + self.warnings.append({"kind": "link", "source": self.current_source, "reference": href}) + return None + return f"/{page_id}.html{anchor or ''}" + + def resolve_image_src(self, src: str): + """"foo.png" -> "", or None to leave src untouched.""" + if not src or "://" in src or src.startswith("/") or "/" in src: + return None + rel = self.image_index.get(src) + if rel is None: + print(f"warning: image not found: {src!r}", file=sys.stderr) + self.warnings.append({"kind": "image", "source": self.current_source, "reference": src}) + return None + return f"{self.image_url_prefix}{rel}" + + def rewrite_urls(self, html: str) -> str: + """Rewrites every href="...md" / src="foo.png" attribute found in a + blob of rendered/raw HTML. Applied to markdown-rendered HTML *and* to + Writerside's raw / passthrough HTML (which markdown-it never + tokenizes as links/images at all, so token-level rewriting alone + would miss it); resolve_href/resolve_image_src already leave anything + that isn't a bare same-tree ".md"/image reference untouched, so this + is safe to run unconditionally on any HTML string.""" + if not html: + return html + + def href_repl(m): + new_href = self.resolve_href(m.group(1)) + return f'href="{new_href}"' if new_href is not None else m.group(0) + + def src_repl(m): + new_src = self.resolve_image_src(m.group(1)) + return f'src="{new_src}"' if new_src is not None else m.group(0) + + html = re.sub(r'href="([^"]*)"', href_repl, html) + html = re.sub(r'src="([^"]*)"', src_repl, html) + return self.style_broken_and_external_links(html) + + def classify_href(self, href: str): + """Classifies an *already rewritten* href: 'external' for off-site + (or mailto:) links, 'broken' for a same-tree ".md" reference that's + still literally "foo.md" because resolve_href couldn't find it, or + None for anything that resolved fine (now "/id.html...") or is a + same-page "#anchor" link.""" + if not href or href.startswith("#"): + return None + if EXTERNAL_HREF_RE.match(href): + return "external" + if MD_LINK_RE.match(href): + return "broken" + return None + + def style_broken_and_external_links(self, html: str) -> str: + """Colors broken and off-site tags with --broken-ext-link-color + (from the config passed on the command line) so readers can tell at a + glance which links leave this site or don't go anywhere at all.""" + if not self.broken_ext_link_color: + return html + + def repl(m): + tag = m.group(0) + if self.classify_href(m.group(1)) is None: + return tag + return tag[:-1] + f' style="color: {self.broken_ext_link_color};">' + + return LINK_TAG_RE.sub(repl, html) + + def fold_image_attrs(self, tokens) -> list: + """Folds a `{...}` attribute-text token immediately following an + image (e.g. `![alt](x.png){width="500"}` - CommonMark has no syntax + for this, so it otherwise tokenizes as a literal "{width=..}" text + run right after the image) into that image's own HTML attributes + instead of leaving it as visible text.""" + result = [] + for tok in tokens: + if tok.type == "text" and result and result[-1].type == "image": + m = ATTR_LINE_RE.match(tok.content.strip()) + if m: + result[-1].attrs.update(parse_attrs(m.group(1))) + continue + if tok.children: + tok.children = self.fold_image_attrs(tok.children) + result.append(tok) + return result + + def render_inline(self, inline_token: Token) -> str: + children = self.fold_image_attrs(inline_token.children) + return self.rewrite_urls(self.md.renderer.render(children, self.md.options, {})) + + def convert_nodes(self, nodes: list) -> list: + blocks = [] + for n in nodes: + result = self.convert_node(n) + if result is None: + continue + if isinstance(result, list): + blocks.extend(result) + else: + blocks.append(result) + blocks = self.merge_attr_lines(blocks) + blocks = self.group_containers(blocks) + return blocks + + def convert_node(self, node: Node): + t = node.token + ttype = t.type + + if ttype == "paragraph_open": + inline = node.children[0].token + # "_raw" carries the unescaped markdown source so merge_attr_lines + # can detect/parse a trailing "{...}" attribute line; it is + # stripped out of the final block before output. + return {"type": "paragraph", "html": self.render_inline(inline), "_raw": inline.content} + + if ttype == "heading_open": + inline = node.children[0].token + level = int(t.tag[1:]) + text = inline.content + return { + "type": "heading", + "level": level, + "id": slugify(text), + "html": self.render_inline(inline), + } + + if ttype == "fence": + return { + "type": "code", + "lang": (t.info or "").strip() or None, + "code": t.content, + "attrs": {}, + } + + if ttype == "blockquote_open": + return { + "type": "blockquote", + "attrs": {}, + "blocks": self.convert_nodes(node.children), + } + + if ttype in ("bullet_list_open", "ordered_list_open"): + items = [] + for item_node in node.children: + if item_node.token.type == "list_item_open": + items.append({"blocks": self.convert_nodes(item_node.children)}) + return {"type": "list", "ordered": ttype == "ordered_list_open", "items": items} + + if ttype == "table_open": + headers = [] + rows = [] + for section in node.children: + if section.token.type == "thead_open": + for tr in section.children: + row = [self.render_inline(td.children[0].token) for td in tr.children] + headers = row + elif section.token.type == "tbody_open": + for tr in section.children: + row = [self.render_inline(td.children[0].token) for td in tr.children] + rows.append(row) + return {"type": "table", "headers": headers, "rows": rows} + + if ttype == "hr": + return {"type": "hr"} + + if ttype == "html_block": + # CommonMark merges consecutive non-blank-line-separated HTML + # lines into a single html_block token, so a lone and + # the that immediately follows it commonly land in the + # same token. Split back into lines so each tag can be matched + # (and grouped into a container) independently. + result = [] + raw_run = [] + + def flush_raw(): + if raw_run: + result.append({"type": "html", "html": self.rewrite_urls("\n".join(raw_run))}) + raw_run.clear() + + for line in t.content.splitlines(): + m = TAG_RE.match(line.strip()) + if m: + flush_raw() + closing, tag, attrstr = m.groups() + result.append({ + "type": "tag_marker", + "closing": bool(closing), + "tag": tag.lower(), + "attrs": parse_attrs(attrstr), + }) + elif line.strip(): + raw_run.append(line) + flush_raw() + return result + + if ttype == "inline": + # top-level bare inline content (e.g. an attribute line with no + # surrounding paragraph); treat like a paragraph. + return {"type": "paragraph", "html": self.render_inline(t), "_raw": t.content} + + # Fallback: anything not explicitly handled (images are inline-only, + # so plain "image" blocks don't occur at block level; captured via + # paragraph HTML instead). + return {"type": "html", "html": self.rewrite_urls(str(t.content or ""))} + + @staticmethod + def merge_attr_lines(blocks: list) -> list: + """Fold a standalone `{key="value"}` paragraph into the preceding block's attrs.""" + merged = [] + for b in blocks: + if b["type"] == "paragraph": + raw = b.pop("_raw", "").strip() + m = ATTR_LINE_RE.match(raw) + if m and merged: + merged[-1].setdefault("attrs", {}).update(parse_attrs(m.group(1))) + continue + merged.append(b) + return merged + + @staticmethod + def group_containers(blocks: list) -> list: + """Turn //// tag_marker pairs into nested blocks.""" + root: dict = {"blocks": []} + stack = [root] + for b in blocks: + if b["type"] == "tag_marker": + if not b["closing"]: + node = {"type": b["tag"], "attrs": b["attrs"], "blocks": []} + stack[-1]["blocks"].append(node) + stack.append(node) + elif len(stack) > 1 and stack[-1]["type"] == b["tag"]: + stack.pop() + continue + stack[-1]["blocks"].append(b) + + return Converter._finalize_list(root["blocks"]) + + @staticmethod + def _finalize_list(blocks: list) -> list: + """Maps _finalize_container over a list, splicing in any block it + unwraps back into a plain list instead of a "tabs" block (see below) + in place, rather than nesting a list-within-a-list.""" + result = [] + for b in blocks: + out = Converter._finalize_container(b) + if isinstance(out, list): + result.extend(out) + else: + result.append(out) + return result + + @staticmethod + def _finalize_container(b: dict): + if b.get("type") == "tabs" and "blocks" in b: + children = Converter._finalize_list(b["blocks"]) + tab_children = [c for c in children if c.get("type") == "tab"] + if tab_children: + # Well-formed ....... + return { + "type": "tabs", + "attrs": b["attrs"], + "tabs": [ + {"title": c["attrs"].get("title"), "attrs": c["attrs"], "blocks": c["blocks"]} + for c in tab_children + ], + } + if len(children) >= 2 and all(c.get("type") == "code" for c in children): + # Bare wrapping only fenced code blocks with no + # tags at all (seen in some compatibility guide pages, e.g. + # a Kotlin and a Groovy fence back to back) - Writerside + # authors apparently rely on adjacency here instead of + # writing explicitly, so treat each code block's own + # language as its tab instead of rendering a tabs shell with + # no tabs in it (which silently dropped every one of these + # code blocks - see the "type": "tabs" but no "tabs" key + # schema mismatch this used to produce). + return { + "type": "tabs", + "attrs": b["attrs"], + "tabs": [ + { + "title": (c["lang"] or "").capitalize() or f"Tab {i + 1}", + "attrs": {"group-key": (c["lang"] or "").lower() or str(i + 1)}, + "blocks": [c], + } + for i, c in enumerate(children) + ], + } + # Bare wrapper with no children and no recognizable + # tab structure to synthesize - there's nothing tab-like left to + # preserve, so drop the wrapper and splice its content in place + # instead of emitting a "tabs" block with no "tabs" key (which + # templates/page.peb can't render - it silently produces an + # empty
    and drops the content entirely). + return children + if "blocks" in b: + b["blocks"] = Converter._finalize_list(b["blocks"]) + return b + + def convert_file(self, path: Path, page_id: str, source_rel: str) -> dict: + self.current_source = source_rel + raw = path.read_text(encoding="utf-8") + # Substitute %variables% in the raw source, before markdown-it ever + # sees it. Doing this post-render instead would (a) miss the title, + # which is extracted straight from the raw source, and (b) run into + # markdown-it percent-encoding "%" inside link URLs, which turns + # "%kotlinEapVersion%" into "%25kotlinEapVersion%25" before we'd get + # a chance to match it. + raw = substitute_vars(raw, self.variables) + title, body = extract_title(raw) + tokens = self.md.parse(body) + tree = build_tree(tokens) + blocks = self.convert_nodes(tree) + return { + "id": page_id, + "sourceFile": source_rel, + "title": title, + "blocks": blocks, + } + + +def make_markdown_it() -> MarkdownIt: + md = MarkdownIt("commonmark") + md.enable("table") + return md + + +CONFIG_KEYS = ("broken-ext-link-color", "menu-no-link-color") + + +def load_config(config_path: Path) -> dict: + """Loads the {"broken-ext-link-color": ..., "menu-no-link-color": ...} + theming config. Missing keys just disable that particular styling (a + warning is printed) rather than being a hard error, since neither is + required for the JSON conversion itself to be correct.""" + config = json.loads(config_path.read_text(encoding="utf-8")) + for key in CONFIG_KEYS: + if key not in config: + print(f"warning: config {config_path} is missing {key!r}; that styling will be skipped", file=sys.stderr) + return config + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("docs_root", type=Path, help="Path to kotlin-web-site/docs") + parser.add_argument("output_dir", type=Path, help="Directory to write JSON files into") + parser.add_argument("config", type=Path, + help='Path to a JSON config with "broken-ext-link-color" and "menu-no-link-color"') + parser.add_argument("--topics-subdir", default="topics", help="Subdirectory of docs_root holding .md files") + parser.add_argument("--images-subdir", default="images", help="Subdirectory of docs_root holding image files") + args = parser.parse_args() + + docs_root: Path = args.docs_root + topics_dir = docs_root / args.topics_subdir + images_dir = docs_root / args.images_subdir + if not topics_dir.is_dir(): + print(f"error: {topics_dir} is not a directory", file=sys.stderr) + sys.exit(1) + if not args.config.is_file(): + print(f"error: {args.config} does not exist", file=sys.stderr) + sys.exit(1) + + config = load_config(args.config) + variables = load_variables(docs_root) + topic_index = build_topic_index(topics_dir) + image_index, _image_collisions = build_image_index(images_dir) + md = make_markdown_it() + converter = Converter(md, variables, topic_index, image_index, + broken_ext_link_color=config.get("broken-ext-link-color")) + + md_files = sorted(topics_dir.rglob("*.md")) + args.output_dir.mkdir(parents=True, exist_ok=True) + + # menu-no-link-color applies to the sidebar, which build_nav.py builds + # separately from nav.json/kr.tree - it reads this back from the output + # directory it's already given, rather than needing its own copy of the + # config on its own command line. + (args.output_dir / "theme.json").write_text( + json.dumps({key: config.get(key) for key in CONFIG_KEYS}, indent=2), encoding="utf-8" + ) + + if images_dir.is_dir(): + shutil.copytree(images_dir, args.output_dir / "images", dirs_exist_ok=True) + else: + print(f"warning: {images_dir} not found; image references will 404", file=sys.stderr) + + count = 0 + for md_path in md_files: + rel = md_path.relative_to(topics_dir) + page_id = str(rel.with_suffix("")) + source_rel = str(Path(args.topics_subdir) / rel) + try: + page = converter.convert_file(md_path, page_id, source_rel) + except Exception as exc: # noqa: BLE001 - surface which file broke + print(f"error converting {md_path}: {exc}", file=sys.stderr) + continue + out_path = args.output_dir / args.topics_subdir / rel.with_suffix(".json") + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(page, separators=(",", ":"), ensure_ascii=False), encoding="utf-8") + count += 1 + + print(f"Converted {count}/{len(md_files)} files into {args.output_dir}") + + +if __name__ == "__main__": + main() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/optimize_media.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/optimize_media.py new file mode 100644 index 000000000..df1a538f8 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/optimize_media.py @@ -0,0 +1,570 @@ +#!/usr/bin/env python3 +""" +optimize_media.py + +Recursively mirrors an input directory into an output directory, optimizing +image files along the way and copying everything else unchanged. + + - Raster images (png, jpg/jpeg, gif, bmp, tif/tiff, webp) are downscaled to + a maximum width (default 500px, preserving aspect ratio, never upscaled), + then optimized: PNGs through pngquant (at a configurable --pngquant-speed + trade-off - see https://pngquant.org/), other formats through Pillow's + own encoder (quality/optimize flags). With --webp, the resized image is + saved as WEBP instead of its original format. Animated GIFs get every + frame resized the same way (preserving frame count/duration/loop count) + rather than being copied through unchanged; --webp is not applied to + them, since animated WEBP re-encoding isn't implemented here. Any other + animated format (e.g. an animated WEBP given as input) is still copied + through unchanged, since per-frame resizing isn't implemented for it. + - SVGs (.svg) are aggressively optimized with the Scour library: metadata, + comments and editor cruft stripped, ids shortened, whitespace collapsed, + and every number rounded to --svg-precision decimal places. If the + optimized SVG still exceeds --svg-rasterize-threshold bytes, it's + rasterized (via cairosvg) and run through the same raster pipeline above + instead of being kept as a vector. If rasterizing fails for any reason + (e.g. cairosvg isn't installed), that's logged as a warning and the + optimized (still oversized) SVG is written instead - every input file + always ends up with something at its mirrored output path. + - Every other file is copied through unchanged. + +Usage: + python3 optimize_media.py [options] + python3 optimize_media.py --config myjob.config + +All options may instead be set in a .config file (one `key = value` per +line, '#' for comments) passed via --config; see OPTION_SPECS below for the +recognized keys, which are the same as the long-form CLI flags. Values +explicitly given on the command line always take precedence over the config +file. + +Every media file processed logs a line with its original and optimized +locations, regardless of --verbose (which adds byte sizes to that line and +prints all resolved config parameters up front). --log-file redirects all +log output (those per-file lines, warnings, and errors) to that file +instead of stdout/stderr. + +Requires the "pngquant" binary on PATH, the Pillow and scour Python packages +(pip install Pillow scour), and - only if any SVG actually needs rasterizing +- the cairosvg package (pip install cairosvg). +""" +import argparse +import io +import shutil +import subprocess +import sys +from pathlib import Path + +from PIL import Image + +try: + RESAMPLE = Image.Resampling.LANCZOS +except AttributeError: # Pillow < 9.1 + RESAMPLE = Image.LANCZOS + +RASTER_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tif", ".tiff", ".webp"} +SVG_EXTENSION = ".svg" + + +class Logger: + """Routes both info-level and error-level messages to a single + destination: the --log-file path if one was given (so a redirected run + still has one coherent, chronological log to inspect afterwards), or + stdout/stderr otherwise (so a normal terminal run keeps its usual + split - errors are still visible even if stdout is piped elsewhere).""" + + def __init__(self, file_handle): + self._fh = file_handle + + def info(self, msg: str) -> None: + print(msg, file=self._fh if self._fh is not None else sys.stdout, flush=self._fh is not None) + + def error(self, msg: str) -> None: + print(msg, file=self._fh if self._fh is not None else sys.stderr, flush=self._fh is not None) + + +def parse_bool(value: str) -> bool: + v = value.strip().lower() + if v in ("1", "true", "yes", "on", "y", "t"): + return True + if v in ("0", "false", "no", "off", "n", "f"): + return False + raise ValueError(f"not a boolean: {value!r}") + + +# Maps a config-file key (and, with dashes, a --long-form CLI flag) to the +# argparse dest it corresponds to and the converter used to parse its value +# out of the config file's plain-text "key = value" form. Order here is also +# the order config parameters get listed in when --verbose is on. +OPTION_SPECS = { + "input-dir": ("input_dir", Path), + "output-dir": ("output_dir", Path), + "max-width": ("max_width", int), + "jpeg-quality": ("jpeg_quality", int), + "webp": ("webp", parse_bool), + "webp-quality": ("webp_quality", int), + "pngquant-speed": ("pngquant_speed", int), + "svg-precision": ("svg_precision", int), + "svg-rasterize-threshold": ("svg_rasterize_threshold", int), + "verbose": ("verbose", parse_bool), + "log-file": ("log_file", Path), +} + +# Used for any option left unset by both the CLI and (if given) --config. +BUILTIN_DEFAULTS = { + "max_width": 500, + "jpeg_quality": 82, + "webp": False, + "webp_quality": 80, + "pngquant_speed": 4, # pngquant's own default; 1 = slow/best, 11 = fast/rough + "svg_precision": 4, + "svg_rasterize_threshold": 300 * 1024, # 300KB + "verbose": False, +} + + +def load_config_file(path: Path, option_specs: dict = OPTION_SPECS) -> dict: + """Parses a simple `key = value` (or `key: value`) config file, one + option per line; blank lines and lines starting with '#' or ';' are + ignored. A bare key with no value means true (for boolean options). + Keys match `option_specs` (case-insensitive, dashes or underscores) - + defaulting to this module's own OPTION_SPECS, but overridable so a + caller layering its own options on top (e.g. insert_optimized_media.py + adding "db-path") can reuse this same parser for its extended key set. + Returns {dest: converted_value}.""" + if not path.is_file(): + raise RuntimeError(f"config file not found: {path}") + + overrides = {} + for lineno, raw_line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + line = raw_line.strip() + if not line or line.startswith("#") or line.startswith(";"): + continue + if "=" in line: + key, _, value = line.partition("=") + elif ":" in line: + key, _, value = line.partition(":") + else: + key, value = line, "true" + key = key.strip().lower().replace("_", "-") + value = value.strip() + + spec = option_specs.get(key) + if spec is None: + raise RuntimeError(f"{path}:{lineno}: unknown config option {key!r}") + dest, converter = spec + try: + overrides[dest] = converter(value) + except ValueError as exc: + raise RuntimeError(f"{path}:{lineno}: invalid value for {key!r}: {exc}") from exc + return overrides + + +def find_pngquant() -> str: + path = shutil.which("pngquant") + if path is None: + raise RuntimeError("pngquant not found on PATH; install it (e.g. `apt install pngquant`) and retry") + return path + + +def quantize_png_bytes(data: bytes, pngquant_path: str, speed: int, name: str, logger: Logger) -> bytes: + """Runs pngquant on raw PNG bytes (stdin -> stdout, no temp files) at the + given --speed trade-off (1 = slow/best quality, 11 = fast/rough - see + https://pngquant.org/), returning the compressed bytes. Falls back to + the original bytes if pngquant declines (e.g. exit 99: result would fall + below --quality's floor) or otherwise fails - a slightly larger PNG + beats a missing one.""" + result = subprocess.run( + [pngquant_path, "--quality", "65-95", "--speed", str(speed), "--strip", "--force", "--output", "-", "-"], + input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, + ) + if result.returncode != 0 or not result.stdout: + logger.error( + f"warning: pngquant declined to compress {name!r} " + f"(exit {result.returncode}: {result.stderr.decode(errors='replace').strip()}); keeping original" + ) + return data + return result.stdout + + +def resize_if_needed(img: Image.Image, max_width: int) -> Image.Image: + if img.width <= max_width: + return img + new_height = max(1, round(img.height * (max_width / img.width))) + return img.resize((max_width, new_height), RESAMPLE) + + +def normalize_mode(img: Image.Image) -> Image.Image: + """Flattens palette/CMYK modes to something every downstream encoder + (JPEG, WEBP, PNG) can handle directly, preserving alpha where present.""" + if img.mode == "P": + return img.convert("RGBA") if img.info.get("transparency") is not None else img.convert("RGB") + if img.mode == "CMYK": + return img.convert("RGB") + return img + + +def encode_raster(img: Image.Image, dst: Path, *, suffix: str, max_width: int, jpeg_quality: int, webp: bool, + webp_quality: int, pngquant_path: str, pngquant_speed: int, logger: Logger) -> Path: + """Resizes an already-loaded image and writes it under dst (whose suffix + may be swapped to .webp), choosing the encoder by `suffix` (the source + file's extension, or ".png" for a freshly rasterized SVG). Returns the + path actually written. Shared by optimize_raster and optimize_svg's + rasterize-on-oversize fallback so both go through identical resize/ + encode logic.""" + img = normalize_mode(img) + + if suffix == ".png" and not webp: + # pngquant runs against the full-resolution pixels here, before any + # downscaling - its palette selection and dithering have the whole + # original image's color detail to work from, rather than the + # coarser, already-blended pixels a resize would leave it with. The + # quantized result is then decoded back and resized down below, same + # as any other image. + buf = io.BytesIO() + img.save(buf, "PNG", optimize=True) + quantized = quantize_png_bytes(buf.getvalue(), pngquant_path, pngquant_speed, str(dst), logger) + img = Image.open(io.BytesIO(quantized)) + img.load() + img = normalize_mode(img) # pngquant's output PNG is palette ("P") mode + + img = resize_if_needed(img, max_width) + + if webp: + dst = dst.with_suffix(".webp") + img.save(dst, "WEBP", quality=webp_quality, method=6) + return dst + + if suffix == ".png": + # Resizing the first quantize pass's palette image back down blended + # it back into full RGB(A) - re-quantize at the final size to + # restore a compact palette PNG, now informed by both the full- + # resolution pass above and the actual delivered dimensions. + buf = io.BytesIO() + img.save(buf, "PNG", optimize=True) + dst.write_bytes(quantize_png_bytes(buf.getvalue(), pngquant_path, pngquant_speed, str(dst), logger)) + elif suffix in (".jpg", ".jpeg"): + if img.mode != "RGB": + img = img.convert("RGB") + img.save(dst, "JPEG", quality=jpeg_quality, optimize=True, progressive=True) + else: + img.save(dst, optimize=True) + return dst + + +def resize_animated_gif(img: Image.Image, dst: Path, max_width: int) -> Path: + """Resizes every frame of an animated GIF down to max_width, preserving + frame count, each frame's own duration, and the loop count - a naive + single-frame resize (or the old copy-through-unchanged behavior) would + otherwise silently drop the animation entirely. seek()+convert("RGBA") + composites each frame the way Pillow's GIF plugin normally displays it + (accounting for the previous frame's disposal method), so every frame + saved below is a complete, standalone image rather than a partial + update relying on its predecessor - hence disposal=2 (restore to + background) on save, rather than trying to preserve each original + frame's own disposal method. --webp is intentionally not honored here: + animated WEBP re-encoding is a separate feature this doesn't attempt.""" + n_frames = getattr(img, "n_frames", 1) + loop = img.info.get("loop", 0) + frames = [] + durations = [] + for i in range(n_frames): + img.seek(i) + frames.append(resize_if_needed(img.convert("RGBA"), max_width)) + durations.append(img.info.get("duration", 100)) + frames[0].save( + dst, save_all=True, append_images=frames[1:], duration=durations, loop=loop, disposal=2, optimize=True, + ) + return dst + + +def optimize_raster(src: Path, dst: Path, **encode_kwargs) -> Path: + with Image.open(src) as img: + if getattr(img, "is_animated", False): + if src.suffix.lower() == ".gif": + return resize_animated_gif(img, dst, encode_kwargs["max_width"]) + # Animated non-GIF (e.g. webp): per-frame resizing/re-encoding is + # out of scope here - copy through unchanged rather than + # flattening it to a single frame and silently breaking the + # animation. + shutil.copy2(src, dst) + return dst + return encode_raster(img, dst, suffix=src.suffix.lower(), **encode_kwargs) + + +def rasterize_svg(svg_text: str, max_width: int) -> Image.Image: + """Renders SVG markup to a raster image at exactly `max_width` pixels + wide (cairosvg computes the proportional height from the SVG's own + viewBox/aspect ratio), for SVGs too large to keep as vector output.""" + try: + import cairosvg + except ImportError as exc: + raise RuntimeError( + "cairosvg is required to rasterize oversized SVGs; install it with `pip install cairosvg`" + ) from exc + png_bytes = cairosvg.svg2png(bytestring=svg_text.encode("utf-8"), output_width=max_width) + img = Image.open(io.BytesIO(png_bytes)) + img.load() + return img + + +def optimize_svg(src: Path, dst: Path, *, precision: int, rasterize_threshold: int, max_width: int, + jpeg_quality: int, webp: bool, webp_quality: int, pngquant_path: str, pngquant_speed: int, + logger: Logger) -> tuple: + """Optimizes one SVG with Scour, rounding numbers to `precision` decimal + places. If the optimized markup is still over `rasterize_threshold` + bytes, rasterizes it and runs it through the raster pipeline instead of + writing it as a (still large) vector. Returns (final_path, was_rasterized).""" + from scour import scour + + options = scour.generateDefaultOptions() + # Aggressive settings, roughly equivalent to: + # scour --enable-viewboxing --enable-id-stripping --enable-comment-stripping + # --shorten-ids --indent=none --strip-xml-prolog --set-precision= + options.remove_metadata = True + options.remove_descriptive_elements = True + options.remove_titles = True + options.remove_descriptions = True + options.strip_comments = True + options.strip_ids = True + options.shorten_ids = True + options.keep_editor_data = False + options.strip_xml_prolog = True + options.enable_viewboxing = True + options.simple_colors = True + options.style_to_xml = True + options.group_collapse = True + options.group_create = True + options.indent_type = "none" + options.newlines = False + options.digits = precision + + in_string = src.read_text(encoding="utf-8") + out_string = scour.scourString(in_string, options) + out_bytes = out_string.encode("utf-8") + + if len(out_bytes) > rasterize_threshold: + try: + img = rasterize_svg(out_string, max_width) + final = encode_raster( + img, dst.with_suffix(".png"), suffix=".png", max_width=max_width, jpeg_quality=jpeg_quality, + webp=webp, webp_quality=webp_quality, pngquant_path=pngquant_path, pngquant_speed=pngquant_speed, + logger=logger, + ) + logger.info( + f"note: rasterized {src} -> {final} (optimized SVG was {len(out_bytes):,} bytes, " + f"over the {rasterize_threshold:,} byte threshold)" + ) + return final, True + except Exception as exc: # noqa: BLE001 - fall through to writing the vector below instead + logger.error(f"warning: failed to rasterize {src} ({exc}); keeping optimized SVG instead") + + dst.write_bytes(out_bytes) + return dst, False + + +def process_file(src: Path, dst: Path, *, cfg: dict, pngquant_path: str, stats: dict, logger: Logger) -> Path: + """Optimizes (or copies through) one file. Returns the path actually + written on success (which may differ from `dst` - webp conversion or + SVG rasterization changes the extension), or None on error (already + logged; `stats["errors"]` is incremented so callers can tell without + inspecting the return value).""" + dst.parent.mkdir(parents=True, exist_ok=True) + suffix = src.suffix.lower() + original_size = src.stat().st_size + + try: + if suffix == SVG_EXTENSION: + dst_final, rasterized = optimize_svg( + src, dst, precision=cfg["svg_precision"], rasterize_threshold=cfg["svg_rasterize_threshold"], + max_width=cfg["max_width"], jpeg_quality=cfg["jpeg_quality"], webp=cfg["webp"], + webp_quality=cfg["webp_quality"], pngquant_path=pngquant_path, pngquant_speed=cfg["pngquant_speed"], + logger=logger, + ) + if rasterized: + stats["svg_rasterized"] += 1 + kind = "svg, rasterized" + else: + stats["svg"] += 1 + kind = "svg" + elif suffix in RASTER_EXTENSIONS: + dst_final = optimize_raster( + src, dst, max_width=cfg["max_width"], jpeg_quality=cfg["jpeg_quality"], webp=cfg["webp"], + webp_quality=cfg["webp_quality"], pngquant_path=pngquant_path, pngquant_speed=cfg["pngquant_speed"], + logger=logger, + ) + stats["raster"] += 1 + kind = "raster" + else: + shutil.copy2(src, dst) + dst_final = dst + stats["copied"] += 1 + kind = "copied" + except Exception as exc: # noqa: BLE001 - keep processing the rest of the tree + stats["errors"] += 1 + logger.error(f"error: failed to process {src}: {exc}") + return None + + optimized_size = dst_final.stat().st_size + stats["original_bytes"] += original_size + stats["optimized_bytes"] += optimized_size + if kind != "copied": + # Always shown (not just under --verbose) - a per-file record of + # where the optimized copy of each media file actually landed, + # since that's not otherwise derivable once optimization has + # renamed a file (webp conversion, SVG rasterization). + message = f"Optimized {src} -> {dst_final}" + if cfg["verbose"]: + saved = original_size - optimized_size + pct = (saved / original_size * 100) if original_size else 0.0 + message = ( + f"[OK][{kind}] {src} -> {dst_final}: {original_size:,} -> {optimized_size:,} bytes " + f"(saved {saved:,} bytes, {pct:.1f}%)" + ) + logger.info(message) + return dst_final + + +def optimize_directory(input_dir: Path, output_dir: Path, *, cfg: dict, pngquant_path: str, logger: Logger, + stats: dict) -> dict: + """Walks input_dir recursively, optimizing every file into the mirrored + location under output_dir (see process_file). Returns + {relative_src_path: relative_dst_path} for every file whose output path + ended up different from its input path (webp conversion, or an SVG + rasterized to PNG/WEBP) - callers that also maintain references to these + files elsewhere (e.g. insert_optimized_media.py, fixing up image URLs + stored in a database) use this to know what changed.""" + renamed = {} + for src in sorted(input_dir.rglob("*")): + if src.is_dir(): + continue + rel = src.relative_to(input_dir) + dst = output_dir / rel + dst_final = process_file(src, dst, cfg=cfg, pngquant_path=pngquant_path, stats=stats, logger=logger) + if dst_final is None: + continue + rel_final = dst_final.relative_to(output_dir) + if rel_final != rel: + renamed[str(rel)] = str(rel_final) + return renamed + + +def add_optimize_arguments(parser: argparse.ArgumentParser) -> None: + """Adds every --tuning-flag (everything except the input/output + positionals) to `parser` - split out from build_parser() so a caller + with its own positional arguments (e.g. insert_optimized_media.py, which + also needs a database path) can still get these for free instead of + redeclaring them.""" + parser.add_argument("--config", type=Path, default=None, + help="Path to a .config file providing any of the options below") + parser.add_argument("--max-width", type=int, default=None, + help=f"Max width in pixels for raster images (default: {BUILTIN_DEFAULTS['max_width']})") + parser.add_argument("--jpeg-quality", type=int, default=None, + help=f"JPEG output quality, 0-95 (default: {BUILTIN_DEFAULTS['jpeg_quality']})") + parser.add_argument("--webp", action="store_true", default=None, + help="Convert optimized raster images (and rasterized SVGs) to WEBP") + parser.add_argument("--webp-quality", type=int, default=None, + help=f"WEBP output quality, 0-100 (default: {BUILTIN_DEFAULTS['webp_quality']})") + parser.add_argument("--pngquant-speed", type=int, default=None, + help="pngquant speed/quality trade-off, 1 (slow/best) - 11 (fast/rough); " + f"see https://pngquant.org/ (default: {BUILTIN_DEFAULTS['pngquant_speed']})") + parser.add_argument("--svg-precision", type=int, default=None, + help="Decimal places to round SVG numbers to via Scour " + f"(default: {BUILTIN_DEFAULTS['svg_precision']})") + parser.add_argument("--svg-rasterize-threshold", type=int, default=None, + help="Rasterize an optimized SVG if it's still over this many bytes " + f"(default: {BUILTIN_DEFAULTS['svg_rasterize_threshold']:,})") + parser.add_argument("--verbose", action="store_true", default=None, + help="Add original/optimized byte sizes to the per-file log line every media file " + "already gets, plus all resolved config parameters up front") + parser.add_argument("--log-file", type=Path, default=None, + help="Write all log output here instead of stdout/stderr") + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("input_dir", type=Path, nargs="?", default=None, + help="Directory to read files from, recursively (or set input-dir in --config)") + parser.add_argument("output_dir", type=Path, nargs="?", default=None, + help="Directory to mirror optimized output into (or set output-dir in --config)") + add_optimize_arguments(parser) + return parser + + +def resolve_config(args: argparse.Namespace, option_specs: dict = OPTION_SPECS, + builtin_defaults: dict = BUILTIN_DEFAULTS) -> dict: + """Merges CLI args over --config file values over builtin_defaults (in + that precedence order) into one dict keyed by dest name. option_specs/ + builtin_defaults default to this module's own, but are overridable for a + caller (e.g. insert_optimized_media.py) extending them with its own + extra options (like "db-path").""" + file_overrides = load_config_file(args.config, option_specs) if args.config else {} + + cfg = {} + for dest, _converter in option_specs.values(): + cli_value = getattr(args, dest, None) + if cli_value is not None: + cfg[dest] = cli_value + elif dest in file_overrides: + cfg[dest] = file_overrides[dest] + elif dest in builtin_defaults: + cfg[dest] = builtin_defaults[dest] + else: + cfg[dest] = None # input_dir/output_dir: no built-in default + return cfg + + +def main() -> None: + parser = build_parser() + args = parser.parse_args() + + try: + cfg = resolve_config(args) + except RuntimeError as exc: + parser.error(str(exc)) + return + + if cfg["input_dir"] is None or cfg["output_dir"] is None: + parser.error("input_dir and output_dir must be given either as positional arguments or in --config") + + log_file_handle = open(cfg["log_file"], "w", encoding="utf-8") if cfg["log_file"] else None + logger = Logger(log_file_handle) + try: + if not cfg["input_dir"].is_dir(): + logger.error(f"error: {cfg['input_dir']} is not a directory") + sys.exit(1) + + if cfg["verbose"]: + logger.info("Config parameters:") + for key, (dest, _converter) in OPTION_SPECS.items(): + logger.info(f" {key} = {cfg[dest]}") + if args.config: + logger.info(f" (loaded from {args.config})") + + try: + pngquant_path = find_pngquant() + except RuntimeError as exc: + logger.error(f"error: {exc}") + sys.exit(1) + + stats = {"raster": 0, "svg": 0, "svg_rasterized": 0, "copied": 0, "errors": 0, "original_bytes": 0, + "optimized_bytes": 0} + optimize_directory(cfg["input_dir"], cfg["output_dir"], cfg=cfg, pngquant_path=pngquant_path, logger=logger, + stats=stats) + + saved = stats["original_bytes"] - stats["optimized_bytes"] + pct = (saved / stats["original_bytes"] * 100) if stats["original_bytes"] else 0.0 + logger.info( + f"Done: {stats['raster']} raster image(s) optimized, {stats['svg']} SVG(s) optimized, " + f"{stats['svg_rasterized']} SVG(s) rasterized, {stats['copied']} other file(s) copied, " + f"{stats['errors']} error(s). Total size: {stats['original_bytes']:,} -> {stats['optimized_bytes']:,} " + f"bytes (saved {saved:,} bytes, {pct:.1f}%)." + ) + if stats["errors"]: + sys.exit(1) + finally: + if log_file_handle is not None: + log_file_handle.close() + + +if __name__ == "__main__": + main() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py new file mode 100644 index 000000000..c338b9dc0 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py @@ -0,0 +1,616 @@ +#!/usr/bin/env python3 +""" +Populates a documentation.db-schema SQLite database with the same content +templates/page.peb and this project's md_to_json.py conversion pipeline +produce for the static site, replacing what's currently at k/html/*. + +Usage: + python3 populate_db.py [db-path] + [--tree-file kr.tree] [--topics-subdir topics] + [--blacklisted-element-titles "Ancestor\\/.../Element Title" ...] + +--blacklisted-element-titles names element(s) +to drop from kr.tree entirely before anything else below reads it: the +element and its whole subtree get no nav entry, none of their .md sub-topics +get converted or inserted, and any *other*, non-blacklisted page's in-content +link to one of those .md files renders broken/styled (same as any other +unresolved link - see broken-ext-link-color) rather than pointing somewhere +that no longer exists. + +Each value is the *full* toc-title path from a top-level down +to the one being blacklisted, since toc-title alone is not unique across +kr.tree (e.g. plenty of "Overview"s). Levels are joined by the two-character +sequence "\\/" (backslash then slash) rather than a bare "/", because a bare +"/" routinely appears *within* a single real toc-title (e.g. "Swift/ +Objective-C and C interop") and this way that overwhelmingly common case +needs no escaping at all - only the rare level separator does. So to +blacklist the "Swift/Objective-C and C interop" element nested under the +top-level "Interoperability" element, pass +"Interoperability\\/Swift/Objective-C and C interop": split on "\\/" that's +["Interoperability", "Swift/Objective-C and C interop"], matching kr.tree's +actual nesting - the inner "/" is left untouched since it wasn't preceded by +a backslash. + + defaults to "documentation.db". A safety backup (via SQLite's +"VACUUM INTO", which is safe even against a live/WAL-mode database) is +written next to it before any changes: ".backup-". + + is Writerside's own official image output for this doc set +(e.g. "webHelpImages.zip", found next to kr.tree) - a flat archive with no +subdirectories, one entry per image, already exactly as Writerside itself +would serve them. Rather than re-deriving image content/sizing ourselves +from the raw source tree (which is a plain, uncompressed truecolor export - +several times larger than what a real Writerside build actually ships, +since it applies its own image optimization we have no easy way to +replicate faithfully), this script just copies that zip's entries in +directly, so k/html/images/ ends up byte-for-byte what Writerside +itself produces. + +What this does, inside a single transaction (rolled back on any error): + 1. Deletes every Content row with path LIKE 'k/html/%' or 'assets/%' - the + former includes the existing *.html doc pages AND everything else + parked there (images, the old Writerside JS bundle under + k/html/frontend/, none of which this script replaces); the latter is + wherever a previous run of this script put images/CSS/JS, all of + which get freshly re-inserted below. + 2. Upserts templates/page.peb and templates/nav.peb into Templates. + page.peb's