parallel article building via n_cores

DESCRIPTION

@@ -36,7 +36,7 @@ Imports:
     lifecycle,
     openssl,
     nanonext (>= 1.8.0),
-    purrr (>= 1.0.0),
+    purrr (>= 1.1.0),
     ragg (>= 1.4.0),
     rlang (>= 1.1.4),
     rmarkdown (>= 2.27),
@@ -46,6 +46,7 @@ Imports:
     xml2 (>= 1.3.1),
     yaml (>= 2.3.9)
 Suggests:
+    carrier,
     covr,
     diffviewer,
     evaluate (>= 0.24.0),
@@ -56,6 +57,8 @@ Suggests:
     knitr (>= 1.50),
     magick,
     methods,
+    mirai,
+    parallel,
     pkgload (>= 1.0.2),
     quarto,
     rsconnect,

NEWS.md

@@ -1,5 +1,7 @@
 # pkgdown (development version)
 
+* `build_articles()`, `build_site()`, and `build_site_github_pages()` gain an `n_cores` argument to build articles in parallel via `purrr::in_parallel()`. The default (`n_cores = 1L`) preserves the traditional serial build; values greater than 1 require the mirai package, and `Inf` autodetects via `parallel::detectCores()`.
+
 * When previewing a site, it is now served via a local http server. This enables dynamic features such as search to work correctly (@shikokuchuo, #2975).
 
 * do not autolink code that is in a link (href) in Rd files (#2972)

R/build-articles.R

@@ -191,20 +191,28 @@
 #'   make article output reproducible. An integer scalar or `NULL` for no seed.
 #' @param preview If `TRUE`, or `is.na(preview) && interactive()`, will preview
 #'   freshly generated section in browser.
+#' @param n_cores Number of workers to use when building articles in
+#'   parallel. A positive integer (fractional values are rounded up), or
+#'   `Inf` to use `parallel::detectCores()`. Defaults to `1L`, which keeps
+#'   the traditional serial build and does not require the \pkg{mirai}
+#'   package. Values greater than 1 require \pkg{mirai} and use
+#'   [purrr::in_parallel()].
 #' @export
 #' @order 1
 build_articles <- function(
   pkg = ".",
   quiet = TRUE,
   lazy = TRUE,
   seed = 1014L,
+  n_cores = 1L,
   override = list(),
   preview = FALSE
 ) {
   pkg <- section_init(pkg, "articles", override = override)
   check_bool(quiet)
   check_bool(lazy)
   check_number_whole(seed, allow_null = TRUE)
+  n_cores <- check_n_cores(n_cores)
 
   if (nrow(pkg$vignettes) == 0L) {
     return(invisible())
@@ -213,19 +221,68 @@ build_articles <- function(
   cli::cli_rule("Building articles")
 
   build_articles_index(pkg)
-  unwrap_purrr_error(purrr::walk(
-    pkg$vignettes$name[pkg$vignettes$type == "rmd"],
-    build_article,
-    pkg = pkg,
-    lazy = lazy,
-    seed = seed,
-    quiet = quiet
-  ))
-  build_quarto_articles(pkg, quiet = quiet)
+  rmd_names <- pkg$vignettes$name[pkg$vignettes$type == "rmd"]
+  if (n_cores == 1L) {
+    unwrap_purrr_error(purrr::walk(
+      rmd_names,
+      build_article,
+      pkg = pkg,
+      lazy = lazy,
+      seed = seed,
+      quiet = quiet
+    ))
+  } else {
+    rlang::check_installed(c("mirai", "carrier"))
+    mirai::daemons(n_cores)
+    withr::defer(mirai::daemons(0))
+    unwrap_purrr_error(purrr::walk(
+      rmd_names,
+      purrr::in_parallel(
+        function(name) {
+          .libPaths(libs)
+          pkgdown::build_article(
+            name,
+            pkg = pkg,
+            lazy = lazy,
+            seed = seed,
+            quiet = quiet
+          )
+        },
+        pkg = pkg,
+        lazy = lazy,
+        seed = seed,
+        quiet = quiet,
+        libs = .libPaths()
+      )
+    ))
+  }
+  build_quarto_articles(pkg, quiet = quiet, n_cores = n_cores)
 
   preview_site(pkg, "articles", preview = preview)
 }
 
+check_n_cores <- function(
+  n_cores,
+  arg = rlang::caller_arg(n_cores),
+  call = rlang::caller_env()
+) {
+  if (
+    !is.numeric(n_cores) ||
+      length(n_cores) != 1L ||
+      is.na(n_cores) ||
+      n_cores < 1
+  ) {
+    cli::cli_abort(
+      "{.arg {arg}} must be a positive integer or {.code Inf}.",
+      call = call
+    )
+  }
+  if (is.infinite(n_cores)) {
+    return(as.integer(parallel::detectCores()))
+  }
+  as.integer(ceiling(n_cores))
+}
+
 # Articles index ----------------------------------------------------------
 
 #' @export

R/build-github.R

@@ -22,7 +22,8 @@ build_site_github_pages <- function(
   dest_dir = "docs",
   clean = TRUE,
   install = FALSE,
-  new_process = FALSE
+  new_process = FALSE,
+  n_cores = 1L
 ) {
   pkg <- as_pkgdown(pkg, override = list(destination = dest_dir))
 
@@ -36,6 +37,7 @@ build_site_github_pages <- function(
     preview = FALSE,
     install = install,
     new_process = new_process,
+    n_cores = n_cores,
     ...
   )
   build_github_pages(pkg)

R/build-quarto-articles.R

@@ -1,5 +1,11 @@
-build_quarto_articles <- function(pkg = ".", article = NULL, quiet = TRUE) {
+build_quarto_articles <- function(
+  pkg = ".",
+  article = NULL,
+  quiet = TRUE,
+  n_cores = 1L
+) {
   pkg <- as_pkgdown(pkg)
+  n_cores <- check_n_cores(n_cores)
 
   qmds <- pkg$vignettes[pkg$vignettes$type == "qmd", ]
   if (!is.null(article)) {
@@ -57,24 +63,41 @@ build_quarto_articles <- function(pkg = ".", article = NULL, quiet = TRUE) {
   }
 
   # Read generated data from quarto template and render into pkgdown template
-  unwrap_purrr_error(purrr::walk2(
-    qmds$file_in,
-    qmds$file_out,
-    function(input_file, output_file) {
-      built_path <- path(output_dir, path_rel(output_file, "articles"))
-      if (!file_exists(built_path)) {
-        cli::cli_abort("No built file found for {.file {input_file}}")
-      }
-      if (path_ext(output_file) == "html") {
-        data <- data_quarto_article(pkg, built_path, input_file)
-        render_page(pkg, "quarto", data, output_file, quiet = TRUE)
-
-        update_html(path(pkg$dst_path, output_file), tweak_quarto_html)
-      } else {
-        file_copy(built_path, path(pkg$dst_path, output_file), overwrite = TRUE)
-      }
-    }
-  ))
+  if (n_cores == 1L) {
+    unwrap_purrr_error(purrr::walk2(
+      qmds$file_in,
+      qmds$file_out,
+      quarto_article_postprocess,
+      pkg = pkg,
+      output_dir = output_dir
+    ))
+  } else {
+    rlang::check_installed(c("mirai", "carrier"))
+    mirai::daemons(n_cores)
+    withr::defer(mirai::daemons(0))
+    unwrap_purrr_error(purrr::walk2(
+      qmds$file_in,
+      qmds$file_out,
+      purrr::in_parallel(
+        function(input_file, output_file) {
+          .libPaths(libs)
+          postprocess <- utils::getFromNamespace(
+            "quarto_article_postprocess",
+            "pkgdown"
+          )
+          postprocess(
+            input_file,
+            output_file,
+            pkg = pkg,
+            output_dir = output_dir
+          )
+        },
+        pkg = pkg,
+        output_dir = output_dir,
+        libs = .libPaths()
+      )
+    ))
+  }
 
   # Report on which files have changed
   new_digest <- purrr::map_chr(path(pkg$dst_path, qmds$file_out), file_digest)
@@ -99,6 +122,21 @@ build_quarto_articles <- function(pkg = ".", article = NULL, quiet = TRUE) {
   invisible()
 }
 
+quarto_article_postprocess <- function(input_file, output_file, pkg, output_dir) {
+  built_path <- path(output_dir, path_rel(output_file, "articles"))
+  if (!file_exists(built_path)) {
+    cli::cli_abort("No built file found for {.file {input_file}}")
+  }
+  if (path_ext(output_file) == "html") {
+    data <- data_quarto_article(pkg, built_path, input_file)
+    render_page(pkg, "quarto", data, output_file, quiet = TRUE)
+
+    update_html(path(pkg$dst_path, output_file), tweak_quarto_html)
+  } else {
+    file_copy(built_path, path(pkg$dst_path, output_file), overwrite = TRUE)
+  }
+}
+
 quarto_render <- function(pkg, path, quiet = TRUE, frame = caller_env()) {
   # Override default quarto format
   metadata_path <- withr::local_tempfile(

R/build.R

@@ -317,6 +317,7 @@ build_site <- function(
   run_dont_run = FALSE,
   seed = 1014L,
   lazy = FALSE,
+  n_cores = 1L,
   override = list(),
   preview = NA,
   devel = FALSE,
@@ -354,6 +355,7 @@ build_site <- function(
       run_dont_run = run_dont_run,
       seed = seed,
       lazy = lazy,
+      n_cores = n_cores,
       override = override,
       preview = preview,
       devel = devel,
@@ -366,6 +368,7 @@ build_site <- function(
       run_dont_run = run_dont_run,
       seed = seed,
       lazy = lazy,
+      n_cores = n_cores,
       override = override,
       preview = preview,
       devel = devel,
@@ -380,6 +383,7 @@ build_site_external <- function(
   run_dont_run = FALSE,
   seed = 1014L,
   lazy = FALSE,
+  n_cores = 1L,
   override = list(),
   preview = NA,
   devel = TRUE,
@@ -392,6 +396,7 @@ build_site_external <- function(
     run_dont_run = run_dont_run,
     seed = seed,
     lazy = lazy,
+    n_cores = n_cores,
     override = override,
     install = FALSE,
     preview = FALSE,
@@ -428,6 +433,7 @@ build_site_local <- function(
   run_dont_run = FALSE,
   seed = 1014L,
   lazy = FALSE,
+  n_cores = 1L,
   override = list(),
   preview = NA,
   devel = TRUE,
@@ -461,6 +467,7 @@ build_site_local <- function(
   build_articles(
     pkg,
     lazy = lazy,
+    n_cores = n_cores,
     override = override,
     quiet = quiet,
     preview = FALSE