Build documentation for LLMs (#2917)

Generates an `LLM.txt` that combines the README, reference index, and articles index, and generates a `.md` file for each `.html` file (using pandoc with a few xml2 tweaks for nicer rendering).

---------

Co-authored-by: Hadley Wickham <[email protected]>

Author: maelle

NAMESPACE

@@ -116,6 +116,7 @@ export(build_articles_index)
 export(build_favicons)
 export(build_home)
 export(build_home_index)
+export(build_llm_docs)
 export(build_news)
 export(build_redirects)
 export(build_reference)

NEWS.md

@@ -1,5 +1,6 @@
 # pkgdown (development version)
 
+* New `build_llm_docs()` generates a `LLMs.txt` at the root directory of your site, and provides a `.md` version of every page. You can disable by adding `llm-docs: false` to your `_pkgdown.yaml` (#2914, @maelle)
 * Links generated with `\code{\link{foo}()}` now have the `()` moved into the `<a>` in the generated output (@maelle).
 * Plots in dark mode are now transformed with a CSS filter to improve their
   visibility (thanks to @gadenbuie).

R/build-llm-dl.R

@@ -0,0 +1,66 @@
+simplify_dls <- function(html) {
+  dls <- xml2::xml_find_all(html, ".//dl")
+  for (dl in dls) {
+    simplify_dl(dl)
+  }
+  invisible()
+}
+
+simplify_dl <- function(dl) {
+  children <- xml2::xml_children(dl)
+
+  names <- xml2::xml_name(children)
+  if (!is_simple_dl(names)) {
+    cli::cli_warn("Skipping this <dl>: not a simple term-definition list")
+    return()
+  }
+
+  groups <- split(children, (seq_along(children) - 1) %/% 2)
+
+  bullets <- lapply(groups, create_li_from_group)
+  ul <- xml2::read_xml("<ul></ul>")
+  xml_insert(ul, bullets)
+
+  xml2::xml_replace(dl, ul)
+}
+
+# Must have an even number of children that alternate between dt and dd
+is_simple_dl <- function(names) {
+  if (length(names) %% 2 != 0) {
+    return(FALSE)
+  }
+  odd <- names[seq_along(names) %% 2 == 1]
+  even <- names[seq_along(names) %% 2 == 0]
+
+  all(odd == "dt") && all(even == "dd")
+}
+
+create_li_from_group <- function(group) {
+  dt <- group[[1]]
+  dd <- group[[2]]
+
+  if (has_children(dd)) {
+    # params case
+    para <- xml2::read_xml("<p></p>")
+    xml_insert(para, xml2::xml_contents(dt))
+    xml2::xml_add_child(para, xml_text_node(": "))
+
+    bullet <- xml2::read_xml("<li></li>")
+    xml2::xml_add_child(bullet, para)
+  } else {
+    # reference index
+    bullet <- xml2::read_xml("<li></li>")
+    xml_insert(bullet, xml2::xml_contents(dt))
+    xml2::xml_add_child(bullet, xml_text_node(": "))
+  }
+  xml_insert(bullet, xml2::xml_contents(dd))
+
+  bullet
+}
+
+has_children <- function(x) length(xml2::xml_children(x)) > 0
+
+xml_text_node <- function(x) {
+  span <- xml2::read_xml(paste0("<span>", x, "</span>"))
+  xml2::xml_find_first(span, ".//text()")
+}

R/build-llm.R

@@ -0,0 +1,199 @@
+#' Build docs for LLMs
+#'
+#' @description
+#' `build_llm_docs()` creates an `LLMs.txt` at the root of your site
+#' that contains the contents of your `README.md`, your reference index,
+#' and your articles index. It also creates a `.md` file for every existing
+#' `.html` file in your site. Together, this gives an LLM an overview of your
+#' package and the ability to find out more by following links.
+#'
+#' If you don't want these files generated for your site, you can opt-out by
+#' adding the following to your `pkgdown.yml`:
+#'
+#' ```yaml
+#' llm-docs: false
+#' ```
+#'
+#' @family site components
+#' @inheritParams build_site
+#' @export
+build_llm_docs <- function(pkg = ".") {
+  pkg <- as_pkgdown(pkg)
+  if (isFALSE(pkg$meta$`llm-docs`)) {
+    return(invisible())
+  }
+
+  cli::cli_rule("Building docs for llms")
+
+  paths <- get_site_paths(pkg)
+  purrr::walk(paths, \(path) {
+    src_path <- path(pkg[["dst_path"]], path)
+    dst_path <- path_ext_set(src_path, "md")
+    convert_md(src_path, dst_path, full_url(pkg, path))
+  })
+
+  index <- c(
+    read_lines(path(pkg$dst_path, "index.md")),
+    "",
+    read_file_if_exists(path(pkg$dst_path, "reference", "index.md")),
+    "",
+    read_file_if_exists(path(pkg$dst_path, "articles", "index.md"))
+  )
+  write_lines(index, path(pkg$dst_path, "llms.txt"))
+
+  invisible()
+}
+
+full_url <- function(pkg, path) {
+  if (is.null(pkg$meta$url)) {
+    return()
+  }
+
+  url <- paste0(pkg$meta$url, "/")
+  if (pkg$development$in_dev) {
+    url <- paste0(url, pkg$prefix)
+  }
+
+  xml2::url_absolute(paste0(path_dir(path), "/"), url)
+}
+
+convert_md <- function(src_path, dst_path, url = NULL) {
+  html <- xml2::read_html(src_path)
+  main_html <- xml2::xml_find_first(html, ".//main")
+  if (length(main_html) == 0) {
+    return()
+  }
+
+  simplify_page_header(main_html)
+  simplify_anchors(main_html)
+  simplify_code(main_html)
+  simplify_popovers_to_footnotes(main_html)
+  simplify_lifecycle_badges(main_html)
+  simplify_dls(main_html)
+  create_absolute_links(main_html, url)
+
+  path <- file_temp()
+  xml2::write_html(main_html, path, format = FALSE)
+  on.exit(file_delete(path), add = TRUE)
+
+  rmarkdown::pandoc_convert(
+    input = path,
+    output = dst_path,
+    from = "html",
+    to = "gfm+definition_lists-raw_html",
+  )
+}
+
+# Helpers ---------------------------------------------------------------------
+
+# simplify page header (which includes logo + source link)
+simplify_page_header <- function(html) {
+  title <- xml2::xml_find_first(html, ".//h1")
+  # website for a package without README/index.md
+  if (length(title) > 0) {
+    xml2::xml_remove(xml2::xml_find_first(html, ".//div[@class='page-header']"))
+    xml2::xml_add_child(html, title, .where = 0)
+  }
+  invisible()
+}
+
+# drop internal anchors
+simplify_anchors <- function(html) {
+  xml2::xml_remove(xml2::xml_find_all(html, ".//a[@class='anchor']"))
+  invisible()
+}
+
+# strip extraneoous classes
+simplify_code <- function(html) {
+  extract_lang <- function(class) {
+    trimws(gsub("sourceCode|downlit", "", class))
+  }
+  code <- xml2::xml_find_all(html, ".//pre[contains(@class, 'sourceCode')]")
+
+  purrr::walk(code, \(x) {
+    xml2::xml_attr(x, "class") <- extract_lang(xml2::xml_attr(x, "class"))
+  })
+  invisible()
+}
+
+simplify_popovers_to_footnotes <- function(main_html) {
+  popover_refs <- xml2::xml_find_all(main_html, ".//a[@class='footnote-ref']")
+  if (length(popover_refs) == 0) {
+    return()
+  }
+
+  # Create footnotes section
+  footnotes_section <- xml2::xml_find_first(
+    main_html,
+    ".//section[@class='footnotes']"
+  )
+  if (length(footnotes_section) == 0) {
+    footnotes_section <- xml2::xml_add_child(
+      main_html,
+      "section",
+      id = "footnotes",
+      class = "footnotes footnotes-end-of-document",
+      role = "doc-endnotes"
+    )
+    xml2::xml_add_child(footnotes_section, "hr")
+    footnotes_ol <- xml2::xml_add_child(footnotes_section, "ol")
+  } else {
+    footnotes_ol <- xml2::xml_find_first(footnotes_section, ".//ol")
+  }
+
+  purrr::iwalk(popover_refs, function(ref, i) {
+    text_content <- xml2::xml_attr(ref, "data-bs-content")
+    fn_id <- paste0("fn", i)
+    fnref_id <- paste0("fnref", i)
+    xml2::xml_attrs(ref) <- list(
+      href = paste0("#", fn_id),
+      id = fnref_id,
+      role = "doc-noteref",
+      class = "footnote-ref"
+    )
+
+    fn_li <- xml2::xml_add_child(footnotes_ol, "li", id = fn_id)
+    parsed_content <- xml2::read_html(text_content) |>
+      xml2::xml_find_first(".//body") |>
+      xml2::xml_children()
+    purrr::walk(parsed_content, \(x) xml2::xml_add_child(fn_li, x))
+  })
+}
+
+simplify_lifecycle_badges <- function(html) {
+  # on reference index
+  badges <- xml2::xml_find_all(html, "//span[contains(@class, 'lifecycle')]")
+  xml2::xml_replace(badges, "strong", paste0("[", xml2::xml_text(badges), "]"))
+
+  # on individual pages
+  badges <- xml2::xml_find_all(
+    html,
+    "//a[.//img[starts-with(@src, 'figures/lifecycle-')]]"
+  )
+  imgs <- xml2::xml_find_first(badges, ".//img")
+  xml2::xml_replace(badges, "strong", tolower(xml2::xml_attr(imgs, "alt")))
+
+  invisible()
+}
+
+create_absolute_links <- function(main_html, url = NULL) {
+  a <- xml2::xml_find_all(main_html, ".//a")
+  xml2::xml_attr(a, "class") <- NULL
+
+  href <- xml2::xml_attr(a, "href")
+  is_internal <- !startsWith(href, "https") & !startsWith(href, "#")
+  if (!is.null(url)) {
+    href[is_internal] <- xml2::url_absolute(href[is_internal], url)
+  }
+  href[is_internal] <- sub("html$", "md", href[is_internal])
+
+  xml2::xml_attr(a[is_internal], "href") <- href[is_internal]
+
+  invisible()
+}
+
+read_file_if_exists <- function(path) {
+  if (file_exists(path)) {
+    read_lines(path)
+  }
+}

R/build.R

@@ -10,6 +10,7 @@
 #' * [build_tutorials()]
 #' * [build_news()]
 #' * [build_redirects()]
+#' * [build_llm_docs()]
 #'
 #' See the documentation for the each function to learn how to control
 #' that aspect of the site. This page documents options that affect the
@@ -467,6 +468,9 @@ build_site_local <- function(
   build_tutorials(pkg, override = override, preview = FALSE)
   build_news(pkg, override = override, preview = FALSE)
   build_sitemap(pkg)
+  if (pkg$bs_version > 3) {
+    build_llm_docs(pkg)
+  }
   build_redirects(pkg, override = override)
   if (pkg$bs_version == 3) {
     build_docsearch_json(pkg)

R/tweak-reference.R

@@ -84,14 +84,16 @@ tweak_highlight_other <- function(div) {
 
 xml_replace_contents <- function(node, new) {
   xml2::xml_remove(xml2::xml_contents(node))
-
   contents <- xml2::xml_contents(new)
-  for (child in contents) {
+  xml_insert(node, contents)
+}
+
+xml_insert <- function(node, new) {
+  for (child in new) {
     xml2::xml_add_child(node, child)
   }
 }
 
-
 tweak_extra_logo <- function(html) {
   img <- xml2::xml_find_all(
     html,

inst/BS5/templates/content-reference-index.html

@@ -10,14 +10,14 @@ <h1>{{{pagetitle}}}</h1>
       {{#subtitle}}<h3>{{{.}}}</h3>{{/subtitle}}
       {{#desc}}<div class="section-desc">{{{desc}}}</div>{{/desc}}
 
-      {{#topics}}<dl>
+      <dl>{{#topics}}
         <dt>
           {{#has_icons}}{{#icon}}<a class="icon" href="{{path}}"><img src="icons/{{{.}}}" alt=""/></a>{{/icon}}{{/has_icons}}
           {{#aliases}}<code><a href="{{path}}">{{{.}}}</a></code> {{/aliases}}
           {{#lifecycle}}<span class="badge lifecycle lifecycle-{{.}}">{{.}}</span>{{/lifecycle}}
         </dt>
         <dd>{{{title}}}</dd>
-      </dl>{{/topics}}
+      {{/topics}}</dl>
     </div>{{/rows}}
   </main>