From 4ddd5bc46487c6adeb1716cd129e9578be204b31 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 21 May 2025 14:58:01 +0200 Subject: [PATCH 1/8] update fileInformationCache based on `keep-ipynb` --- src/command/render/render-contexts.ts | 2 +- src/command/render/render-files.ts | 6 +++++- src/execute/jupyter/jupyter.ts | 24 ++++++++++++++++-------- src/execute/types.ts | 6 +++++- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/command/render/render-contexts.ts b/src/command/render/render-contexts.ts index 8b020f27180..90cb7cc4572 100644 --- a/src/command/render/render-contexts.ts +++ b/src/command/render/render-contexts.ts @@ -298,7 +298,7 @@ export async function renderContexts( // if this isn't for execute then cleanup context if (!forExecute && engine.executeTargetSkipped) { - engine.executeTargetSkipped(target, formats[formatKey].format); + engine.executeTargetSkipped(target, formats[formatKey].format, project); } } return contexts; diff --git a/src/command/render/render-files.ts b/src/command/render/render-files.ts index 0e8ffec77fd..c2bdd8dc276 100644 --- a/src/command/render/render-files.ts +++ b/src/command/render/render-files.ts @@ -200,7 +200,11 @@ export async function renderExecute( // notify engine that we skipped execute if (context.engine.executeTargetSkipped) { - context.engine.executeTargetSkipped(context.target, context.format); + context.engine.executeTargetSkipped( + context.target, + context.format, + context.project, + ); } // return results diff --git a/src/execute/jupyter/jupyter.ts b/src/execute/jupyter/jupyter.ts index 5578bb11306..0dd36b65bfe 100644 --- a/src/execute/jupyter/jupyter.ts +++ b/src/execute/jupyter/jupyter.ts @@ -17,7 +17,7 @@ import { readYamlFromMarkdown } from "../../core/yaml.ts"; import { isInteractiveSession } from "../../core/platform.ts"; import { partitionMarkdown } from "../../core/pandoc/pandoc-partition.ts"; -import { dirAndStem, normalizePath, removeIfExists } from "../../core/path.ts"; +import { dirAndStem, normalizePath } from "../../core/path.ts"; import { runningInCI } from "../../core/ci-info.ts"; import { @@ -109,7 +109,10 @@ import { import { jupyterCapabilities } from "../../core/jupyter/capabilities.ts"; import { runExternalPreviewServer } from "../../preview/preview-server.ts"; import { onCleanup } from "../../core/cleanup.ts"; -import { projectOutputDir } from "../../project/project-shared.ts"; +import { + ensureFileInformationCache, + projectOutputDir, +} from "../../project/project-shared.ts"; import { assert } from "testing/asserts"; export const jupyterEngine: ExecutionEngine = { @@ -436,7 +439,7 @@ export const jupyterEngine: ExecutionEngine = { // if it's a transient notebook then remove it // (unless keep-ipynb was specified) - cleanupNotebook(options.target, options.format); + cleanupNotebook(options.target, options.format, options.project); // Create markdown from the result const outputs = result.cellOutputs.map((output) => output.markdown); @@ -713,12 +716,17 @@ async function disableDaemonForNotebook(target: ExecutionTarget) { return false; } -function cleanupNotebook(target: ExecutionTarget, format: Format) { - // remove transient notebook if appropriate +function cleanupNotebook( + target: ExecutionTarget, + format: Format, + project: ProjectContext, +) { + // Make notebook non-transient when keep-ipynb is set const data = target.data as JupyterTargetData; - if (data.transient) { - if (!format.execute[kKeepIpynb]) { - removeIfExists(target.input); + const cached = ensureFileInformationCache(project, target.source); + if (data.transient && format.execute[kKeepIpynb]) { + if (cached.target && cached.target.data) { + (cached.target.data as JupyterTargetData).transient = false; } } } diff --git a/src/execute/types.ts b/src/execute/types.ts index 1f57a02c15e..7975212bd8a 100644 --- a/src/execute/types.ts +++ b/src/execute/types.ts @@ -49,7 +49,11 @@ export interface ExecutionEngine { format: Format, ) => Format; execute: (options: ExecuteOptions) => Promise; - executeTargetSkipped?: (target: ExecutionTarget, format: Format) => void; + executeTargetSkipped?: ( + target: ExecutionTarget, + format: Format, + project: ProjectContext, + ) => void; dependencies: (options: DependenciesOptions) => Promise; postprocess: (options: PostProcessOptions) => Promise; canFreeze: boolean; From f1c53151f9b8ced6dd003792ea727e8c09ba4a2a Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 21 May 2025 15:06:14 +0200 Subject: [PATCH 2/8] clean import --- src/command/render/render-contexts.ts | 3 --- src/core/jupyter/jupyter-embed.ts | 9 ++++++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/command/render/render-contexts.ts b/src/command/render/render-contexts.ts index 90cb7cc4572..9fd44c04cf4 100644 --- a/src/command/render/render-contexts.ts +++ b/src/command/render/render-contexts.ts @@ -63,7 +63,6 @@ import { ExecutionEngine, ExecutionTarget } from "../../execute/types.ts"; import { deleteProjectMetadata, directoryMetadataForInputFile, - projectTypeIsWebsite, toInputRelativePaths, } from "../../project/project-shared.ts"; import { @@ -71,8 +70,6 @@ import { kProjectType, ProjectContext, } from "../../project/types.ts"; -import { isHtmlDashboardOutput, isHtmlOutput } from "../../config/format.ts"; -import { formatHasBootstrap } from "../../format/html/format-html-info.ts"; import { warnOnce } from "../../core/log.ts"; import { dirAndStem } from "../../core/path.ts"; import { fileExecutionEngineAndTarget } from "../../execute/engine.ts"; diff --git a/src/core/jupyter/jupyter-embed.ts b/src/core/jupyter/jupyter-embed.ts index 128dd49c717..fc6d3402c11 100644 --- a/src/core/jupyter/jupyter-embed.ts +++ b/src/core/jupyter/jupyter-embed.ts @@ -43,7 +43,13 @@ import { JupyterCellOutput, } from "../jupyter/types.ts"; -import { dirname, extname, join, basename, isAbsolute } from "../../deno_ral/path.ts"; +import { + basename, + dirname, + extname, + isAbsolute, + join, +} from "../../deno_ral/path.ts"; import { languages } from "../handlers/base.ts"; import { extractJupyterWidgetDependencies, @@ -596,6 +602,7 @@ async function getCachedNotebookInfo( quiet: flags.quiet, previewServer: context.options.previewServer, handledLanguages: languages(), + project: context.project, }; const [dir, stem] = dirAndStem(nbAddress.path); From 91e1424815ea79e09553b154a420a20338b761d0 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 21 May 2025 15:06:39 +0200 Subject: [PATCH 3/8] Always pass ProjectContext in ExecuteOptions --- src/execute/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/execute/types.ts b/src/execute/types.ts index 7975212bd8a..489b378cc80 100644 --- a/src/execute/types.ts +++ b/src/execute/types.ts @@ -93,7 +93,7 @@ export interface ExecuteOptions { quiet?: boolean; previewServer?: boolean; handledLanguages: string[]; // list of languages handled by cell language handlers, after the execution engine - project?: ProjectContext; + project: ProjectContext; } // result of execution From 37d28247a6ff052bd21251c3ccf0edc387f8abbe Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 21 May 2025 16:00:07 +0200 Subject: [PATCH 4/8] ensure fileInformationCache is not cloned by safeCloneDeep This allows to always get the global fileInformationCache from project context to get / set information in it from different contexts. --- src/project/project-context.ts | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/project/project-context.ts b/src/project/project-context.ts index 7d3ec560344..550fe3188fb 100644 --- a/src/project/project-context.ts +++ b/src/project/project-context.ts @@ -19,6 +19,7 @@ import * as ld from "../core/lodash.ts"; import { ProjectType } from "./types/types.ts"; import { Format, Metadata, PandocFlags } from "../config/types.ts"; import { + FileInformation, kProjectLibDir, kProjectOutputDir, kProjectPostRender, @@ -106,6 +107,16 @@ import { createTempContext } from "../core/temp.ts"; import { onCleanup } from "../core/cleanup.ts"; import { once } from "../core/once.ts"; +import { Cloneable } from "../core/safe-clone-deep.ts"; + +// Create a class that extends Map and implements Cloneable +class FileInformationCacheMap extends Map + implements Cloneable> { + clone(): Map { + // Return the same instance (reference) instead of creating a clone + return this; + } +} export async function projectContext( path: string, @@ -272,7 +283,7 @@ export async function projectContext( dir: join(dir, ".quarto"), prefix: "quarto-session-temp", }); - const fileInformationCache = new Map(); + const fileInformationCache = new FileInformationCacheMap(); const result: ProjectContext = { resolveBrand: async (fileName?: string) => projectResolveBrand(result, fileName), @@ -368,7 +379,7 @@ export async function projectContext( dir: join(dir, ".quarto"), prefix: "quarto-session-temp", }); - const fileInformationCache = new Map(); + const fileInformationCache = new FileInformationCacheMap(); const result: ProjectContext = { resolveBrand: async (fileName?: string) => projectResolveBrand(result, fileName), @@ -443,7 +454,7 @@ export async function projectContext( dir: join(originalDir, ".quarto"), prefix: "quarto-session-temp", }); - const fileInformationCache = new Map(); + const fileInformationCache = new FileInformationCacheMap(); const context: ProjectContext = { resolveBrand: async (fileName?: string) => projectResolveBrand(context, fileName), From 032a5b82495c4dc19540b09ced920b260e9a7738 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 21 May 2025 16:22:56 +0200 Subject: [PATCH 5/8] tests - add for project context and single file context --- .../2025/05/21/keep_ipynb_project/.gitignore | 1 + .../2025/05/21/keep_ipynb_project/12780.qmd | 15 +++++++++++++++ .../2025/05/21/keep_ipynb_project/_quarto.yml | 2 ++ .../2025/05/21/keep_ipynb_single-file/12780.qmd | 15 +++++++++++++++ 4 files changed, 33 insertions(+) create mode 100644 tests/docs/smoke-all/2025/05/21/keep_ipynb_project/.gitignore create mode 100644 tests/docs/smoke-all/2025/05/21/keep_ipynb_project/12780.qmd create mode 100644 tests/docs/smoke-all/2025/05/21/keep_ipynb_project/_quarto.yml create mode 100644 tests/docs/smoke-all/2025/05/21/keep_ipynb_single-file/12780.qmd diff --git a/tests/docs/smoke-all/2025/05/21/keep_ipynb_project/.gitignore b/tests/docs/smoke-all/2025/05/21/keep_ipynb_project/.gitignore new file mode 100644 index 00000000000..075b2542afb --- /dev/null +++ b/tests/docs/smoke-all/2025/05/21/keep_ipynb_project/.gitignore @@ -0,0 +1 @@ +/.quarto/ diff --git a/tests/docs/smoke-all/2025/05/21/keep_ipynb_project/12780.qmd b/tests/docs/smoke-all/2025/05/21/keep_ipynb_project/12780.qmd new file mode 100644 index 00000000000..fb12e2ad0e7 --- /dev/null +++ b/tests/docs/smoke-all/2025/05/21/keep_ipynb_project/12780.qmd @@ -0,0 +1,15 @@ +--- +format: html +keep-ipynb: true +_quarto: + tests: + html: + fileExists: + outputPath: 12780.quarto_ipynb + postRenderCleanup: + - ${input_stem}.quarto_ipynb +--- + +```{python} +1 + 1 +``` \ No newline at end of file diff --git a/tests/docs/smoke-all/2025/05/21/keep_ipynb_project/_quarto.yml b/tests/docs/smoke-all/2025/05/21/keep_ipynb_project/_quarto.yml new file mode 100644 index 00000000000..b8bae5830fa --- /dev/null +++ b/tests/docs/smoke-all/2025/05/21/keep_ipynb_project/_quarto.yml @@ -0,0 +1,2 @@ +project: + type: default diff --git a/tests/docs/smoke-all/2025/05/21/keep_ipynb_single-file/12780.qmd b/tests/docs/smoke-all/2025/05/21/keep_ipynb_single-file/12780.qmd new file mode 100644 index 00000000000..fb12e2ad0e7 --- /dev/null +++ b/tests/docs/smoke-all/2025/05/21/keep_ipynb_single-file/12780.qmd @@ -0,0 +1,15 @@ +--- +format: html +keep-ipynb: true +_quarto: + tests: + html: + fileExists: + outputPath: 12780.quarto_ipynb + postRenderCleanup: + - ${input_stem}.quarto_ipynb +--- + +```{python} +1 + 1 +``` \ No newline at end of file From 2e4b4cf199aa4054ece40bf39308fef63e62bce1 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 21 May 2025 16:36:05 +0200 Subject: [PATCH 6/8] single-file render also need to use the specific Map for fileInformationCache --- src/project/project-context.ts | 11 +---------- src/project/project-shared.ts | 10 ++++++++++ src/project/types/single-file/single-file.ts | 3 ++- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/project/project-context.ts b/src/project/project-context.ts index 550fe3188fb..2486d6031ce 100644 --- a/src/project/project-context.ts +++ b/src/project/project-context.ts @@ -71,6 +71,7 @@ import { projectResourceFiles } from "./project-resources.ts"; import { cleanupFileInformationCache, + FileInformationCacheMap, ignoreFieldsForProjectType, normalizeFormatYaml, projectConfigFile, @@ -107,16 +108,6 @@ import { createTempContext } from "../core/temp.ts"; import { onCleanup } from "../core/cleanup.ts"; import { once } from "../core/once.ts"; -import { Cloneable } from "../core/safe-clone-deep.ts"; - -// Create a class that extends Map and implements Cloneable -class FileInformationCacheMap extends Map - implements Cloneable> { - clone(): Map { - // Return the same instance (reference) instead of creating a clone - return this; - } -} export async function projectContext( path: string, diff --git a/src/project/project-shared.ts b/src/project/project-shared.ts index c454f4612e2..234908ecc34 100644 --- a/src/project/project-shared.ts +++ b/src/project/project-shared.ts @@ -50,6 +50,7 @@ import { refSchema } from "../core/lib/yaml-schema/common.ts"; import { Zod } from "../resources/types/zod/schema-types.ts"; import { Brand } from "../core/brand/brand.ts"; import { assert } from "testing/asserts"; +import { Cloneable } from "../core/safe-clone-deep.ts"; export function projectExcludeDirs(context: ProjectContext): string[] { const outputDir = projectOutputDir(context); @@ -633,6 +634,15 @@ export async function projectResolveBrand( } } +// Create a class that extends Map and implements Cloneable +export class FileInformationCacheMap extends Map + implements Cloneable> { + clone(): Map { + // Return the same instance (reference) instead of creating a clone + return this; + } +} + export function cleanupFileInformationCache(project: ProjectContext) { project.fileInformationCache.forEach((entry) => { if (entry?.target?.data) { diff --git a/src/project/types/single-file/single-file.ts b/src/project/types/single-file/single-file.ts index a38b5ec227d..cd5b6c0e287 100644 --- a/src/project/types/single-file/single-file.ts +++ b/src/project/types/single-file/single-file.ts @@ -21,6 +21,7 @@ import { MappedString } from "../../../core/mapped-text.ts"; import { fileExecutionEngineAndTarget } from "../../../execute/engine.ts"; import { cleanupFileInformationCache, + FileInformationCacheMap, projectFileMetadata, projectResolveBrand, projectResolveFullMarkdownForFile, @@ -49,7 +50,7 @@ export async function singleFileProjectContext( notebookContext, environment: () => environmentMemoizer(result), renderFormats, - fileInformationCache: new Map(), + fileInformationCache: new FileInformationCacheMap(), fileExecutionEngineAndTarget: ( file: string, ) => { From 422a6709cbcec6e108f071c5812181a44ae9b6c3 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Fri, 23 May 2025 11:36:56 +0200 Subject: [PATCH 7/8] remove unused import --- src/project/project-context.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/project/project-context.ts b/src/project/project-context.ts index 2486d6031ce..b661c56b01c 100644 --- a/src/project/project-context.ts +++ b/src/project/project-context.ts @@ -19,7 +19,6 @@ import * as ld from "../core/lodash.ts"; import { ProjectType } from "./types/types.ts"; import { Format, Metadata, PandocFlags } from "../config/types.ts"; import { - FileInformation, kProjectLibDir, kProjectOutputDir, kProjectPostRender, From f553f905db830e164b43971ef1380c0f6cd30ade Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Fri, 23 May 2025 11:38:25 +0200 Subject: [PATCH 8/8] Add to changelog --- news/changelog-1.8.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/news/changelog-1.8.md b/news/changelog-1.8.md index e8bc3b88496..98cf3c092b2 100644 --- a/news/changelog-1.8.md +++ b/news/changelog-1.8.md @@ -5,6 +5,7 @@ All changes included in 1.8: - ([#6607](https://github.com/quarto-dev/quarto-cli/issues/6607)): Add missing beamer template update for beamer theme options: `colorthemeoptions`, `fontthemeoptions`, `innerthemeoptions` and `outerthemeoptions`. - ([#12625](https://github.com/quarto-dev/quarto-cli/pull/12625)): Fire resize event on window when light/dark toggle is clicked, to tell widgets to resize. - ([#12657](https://github.com/quarto-dev/quarto-cli/pull/12657)): Load Giscus in generated script tag, to avoid wrong-theming in Chrome. +- ([#12780](https://github.com/quarto-dev/quarto-cli/issues/12780)): `keep-ipynb: true` now works again correctly and intermediate `.quarto_ipynb` is not removed. ## Formats @@ -63,4 +64,4 @@ All changes included in 1.8: ## Other fixes and improvements - ([#11321](https://github.com/quarto-dev/quarto-cli/issues/11321)): Follow [recommendation from LaTeX project](https://latex-project.org/news/latex2e-news/ltnews40.pdf) and use `lualatex` instead of `xelatex` as the default PDF engine. -- ([#12782](https://github.com/quarto-dev/quarto-cli/pull/12782)): fix bug on `safeRemoveDirSync`'s detection of safe directory boundaries. \ No newline at end of file +- ([#12782](https://github.com/quarto-dev/quarto-cli/pull/12782)): fix bug on `safeRemoveDirSync`'s detection of safe directory boundaries.