GitHub Issue #1130: Metrics to track R & Python package usage

api/src/org/labkey/api/reports/ExternalScriptEngine.java

@@ -23,6 +23,7 @@
 import org.labkey.api.miniprofiler.MiniProfiler;
 import org.labkey.api.pipeline.PipelineJobService;
 import org.labkey.api.reader.Readers;
+import org.labkey.api.reports.report.ScriptPackageUsageTracker;
 import org.labkey.api.reports.report.r.ParamReplacementSvc;
 import org.labkey.api.util.ExceptionUtil;
 import org.labkey.api.util.LabKeyProcessBuilder;
@@ -83,6 +84,8 @@ public class ExternalScriptEngine extends AbstractScriptEngine implements LabKey
     public static final String DEFAULT_WORKING_DIRECTORY = "ExternalScript";
     private static final Pattern scriptCmdPattern = Pattern.compile("'([^']+)'|\\\"([^\\\"]+)\\\"|(^[^\\s]+)|(\\s[^\\s^'^\\\"]+)");
 
+    private static final int MAX_PACKAGES_PER_RUN = 250;
+
     private FileLike _workingDirectory;
 
     protected ExternalScriptEngineDefinition _def;
@@ -113,15 +116,123 @@ public boolean isBinary(FileLike file)
     public Object eval(String script, ScriptContext context) throws ScriptException
     {
         List<String> extensions = getFactory().getExtensions();
+        if (extensions.isEmpty())
+            throw new ScriptException("There are no file name extensions registered for this ScriptEngine : " + getFactory().getLanguageName());
+
+        FileLike scriptFile = prepareScriptFile(appendPackageCaptureEpilog(script, context), context, extensions);
+        Object result = eval(scriptFile, context);
+
+        // Only reached when the script succeeded; a failed run reports no package usage
+        recordPackageUsage(context);
+
+        return result;
+    }
+
+    /**
+     * Prepare the on-disk script file that will be executed. The default writes the script as-is; subclasses (e.g. the
+     * R engine's knitr handling) may wrap it in a different driver script.
+     */
+    protected FileLike prepareScriptFile(String script, ScriptContext context, List<String> extensions)
+    {
+        return writeScriptFile(script, context, extensions);
+    }
 
-        if (!extensions.isEmpty())
+    /**
+     * GitHub Issue #1130
+     * Script appended to the end of the user script (running in the same process) that captures the loaded
+     * packages/modules, writing them one per line to a sidecar file in the working directory for
+     * {@link #recordPackageUsage} to read back. The default returns null (no capture); language-specific engines
+     * (e.g. R, Python) override this.
+     */
+    protected @Nullable String getPackageCaptureEpilog(ScriptContext context)
+    {
+        return null;
+    }
+
+    /**
+     * GitHub Issue #1130
+     * Append this engine's package capture epilog, if it has one, to the end of the given script.
+     * Never throws: package tracking must not affect script execution.
+     */
+    protected String appendPackageCaptureEpilog(String script, ScriptContext context)
+    {
+        try
         {
-            // write out the script file to disk using the first extension as the default
-            FileLike scriptFile = writeScriptFile(script, context, extensions);
-            return eval(scriptFile, context);
+            String epilog = getPackageCaptureEpilog(context);
+            if (epilog != null)
+                return script + "\n" + epilog;
+        }
+        catch (Exception e)
+        {
+            LOG.warn("Failed to build the script package capture epilog", e);
+        }
+
+        return script;
+    }
+
+    /**
+     * GitHub Issue #1130
+     * Called after a script has run successfully (eval returned without throwing), to record the packages it loaded.
+     * The default does nothing; language-specific engines override this, typically delegating to
+     * {@link #readPackageSidecar}.
+     */
+    protected void recordPackageUsage(ScriptContext context)
+    {
+    }
+
+    /**
+     * GitHub Issue #1130
+     * Read a sidecar file of package names (one per line) from the working directory and record each under the given
+     * language in {@link ScriptPackageUsageTracker}, up to {@link #MAX_PACKAGES_PER_RUN} per run. Never throws. A
+     * missing file means the capture code never ran (or was skipped) - nothing to do.
+     * The file is deleted after reading.
+     */
+    protected void readPackageSidecar(ScriptContext context, String fileName, String language)
+    {
+        FileLike packagesFile;
+        try
+        {
+            packagesFile = getWorkingDir(context).resolveChild(fileName);
+        }
+        catch (Exception e)
+        {
+            LOG.warn("Failed to locate " + language + " package usage sidecar", e);
+            return;
+        }
+
+        if (!packagesFile.exists())
+            return;
+
+        try (BufferedReader reader = Readers.getReader(packagesFile.openInputStream()))
+        {
+            String packageName;
+            int recorded = 0;
+            while ((packageName = reader.readLine()) != null)
+            {
+                if (recorded >= MAX_PACKAGES_PER_RUN)
+                {
+                    LOG.warn("Recorded the first {} {} packages for this script run and ignored the rest", MAX_PACKAGES_PER_RUN, language);
+                    break;
+                }
+                ScriptPackageUsageTracker.record(language, packageName);
+                recorded++;
+            }
+        }
+        catch (Exception e)
+        {
+            LOG.warn("Failed to record " + language + " package usage", e);
+        }
+        finally
+        {
+            try
+            {
+                packagesFile.delete();
+            }
+            catch (Exception e)
+            {
+                LOG.warn("Failed to delete " + language + " package usage sidecar", e);
+            }
         }
-        else
-            throw new ScriptException("There are no file name extensions registered for this ScriptEngine : " + getFactory().getLanguageName());
     }
 
     protected Object eval(FileLike scriptFile, ScriptContext context) throws ScriptException

api/src/org/labkey/api/reports/report/ScriptPackageUsageTracker.java

@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2026 LabKey Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.labkey.api.reports.report;
+
+import org.apache.logging.log4j.Logger;
+import org.labkey.api.reports.ExternalScriptEngine;
+import org.labkey.api.usageMetrics.SimpleMetricsService;
+import org.labkey.api.util.logging.LogHelper;
+
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Tracks which packages/modules are loaded by scripts run on this server (R reports, assay transform scripts, Python
+ * scripts, and anything else that runs through {@link ExternalScriptEngine}). Populated by a language-specific epilog
+ * appended to each script that writes the loaded packages to a sidecar file, which the engine reads back after the
+ * script has run successfully. Usage is tracked per language (e.g. "r", "python").
+ * <p>
+ * Note that this only sees scripts that ran to completion: a script that fails, or that exits early via q() or
+ * sys.exit(), never reaches the epilog and so reports nothing. Counts are a lower bound.
+ *
+ * Each load is recorded via {@link SimpleMetricsService}, which persists a cumulative per-package load count across
+ * restarts and reports it to mothership under "simpleMetricCounts". The package name is the metric name and the feature
+ * area is "&lt;language&gt;PackageUsage".
+ */
+public class ScriptPackageUsageTracker
+{
+    private static final Logger LOG = LogHelper.getLogger(ScriptPackageUsageTracker.class, "Tracks R & Python package usage by server-side scripts");
+
+    private static final String MODULE_NAME = "API";
+    private static final String FEATURE_AREA_SUFFIX = "PackageUsage";
+    private static final int MAX_METRIC_NAME_LENGTH = 255;
+
+    /**
+     * Packages that ship with a given language's runtime and are always present, so aren't interesting as "library
+     * usage". R's base packages are filtered here; Python needs no entry because the capture epilog only reports names
+     * that map to an installed distribution, which the standard library never does.
+     */
+    private static final Map<String, Set<String>> BASE_PACKAGES = Map.of(
+        "r", Set.of("base", "compiler", "datasets", "graphics", "grDevices", "grid", "methods", "parallel", "splines", "stats", "stats4", "tcltk", "tools", "utils")
+    );
+
+    private ScriptPackageUsageTracker()
+    {
+    }
+
+    private static boolean isBasePackage(String language, String packageName)
+    {
+        return BASE_PACKAGES.getOrDefault(language, Set.of()).contains(packageName);
+    }
+
+    /**
+     * Record that the given package was loaded by a script run in the given language (e.g. "r", "python"). Safe to call
+     * repeatedly; base packages and blank names are ignored.
+     */
+    public static void record(String language, String packageName)
+    {
+        if (packageName == null || packageName.isBlank() || isBasePackage(language, packageName))
+            return;
+
+        try
+        {
+            SimpleMetricsService.get().increment(MODULE_NAME, language + FEATURE_AREA_SUFFIX, truncateMetricName(packageName));
+        }
+        catch (Exception e)
+        {
+            LOG.warn("Failed to record {} package usage for '{}'", language, packageName, e);
+        }
+    }
+
+    /**
+     * The package name is used as the metric name, and the names come from whatever the script actually loaded rather
+     * than from a fixed list, so cap the length at what the DB column holds.
+     */
+    private static String truncateMetricName(String packageName)
+    {
+        return packageName.length() <= MAX_METRIC_NAME_LENGTH ? packageName : packageName.substring(0, MAX_METRIC_NAME_LENGTH);
+    }
+}

api/src/org/labkey/api/reports/report/python/PythonScriptEngine.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2026 LabKey Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.labkey.api.reports.report.python;
+
+import org.jetbrains.annotations.Nullable;
+import org.labkey.api.reports.ExternalScriptEngine;
+import org.labkey.api.reports.ExternalScriptEngineDefinition;
+
+import javax.script.ScriptContext;
+
+/**
+ * Script engine for locally-executed Python scripts (Python assay transform scripts configured as an
+ * external ".py" engine). Behaves like the base {@link ExternalScriptEngine} except that it appends a capture epilog to
+ * track which Python packages each script loads; see
+ * {@link org.labkey.api.reports.report.ScriptPackageUsageTracker}.
+ */
+public class PythonScriptEngine extends ExternalScriptEngine
+{
+    private static final String PACKAGES_FILE = "labkeyPythonPackages.txt";
+
+    // Python appended to a user script to capture the packages it loaded.
+    // try/except means a capture failure can never break the script run.
+    private static final String PACKAGE_CAPTURE_EPILOG = """
+            # --- LabKey Python package usage capture ---
+            try:
+                import importlib.metadata as _lk_md, os as _lk_os, sys as _lk_sys
+                _lk_dists = _lk_md.packages_distributions()
+                _lk_tops = {m.split('.')[0] for m in list(_lk_sys.modules)} - set(_lk_sys.stdlib_module_names)
+                _lk_names = sorted({d for t in _lk_tops if t and not t.startswith('_') for d in _lk_dists.get(t, ())})
+                with open(_lk_os.path.join(_lk_os.getcwd(), '%s'), 'w') as _lk_f:
+                    _lk_f.write('\\n'.join(_lk_names))
+            except Exception:
+                pass
+            """.formatted(PACKAGES_FILE);
+
+    public PythonScriptEngine(ExternalScriptEngineDefinition def)
+    {
+        super(def);
+    }
+
+    @Override
+    protected @Nullable String getPackageCaptureEpilog(ScriptContext context)
+    {
+        return PACKAGE_CAPTURE_EPILOG;
+    }
+
+    @Override
+    protected void recordPackageUsage(ScriptContext context)
+    {
+        readPackageSidecar(context, PACKAGES_FILE, "python");
+    }
+}

api/src/org/labkey/api/reports/report/r/RScriptEngine.java

@@ -16,15 +16,16 @@
 package org.labkey.api.reports.report.r;
 
 import org.apache.commons.lang3.StringUtils;
+import org.jetbrains.annotations.Nullable;
 import org.labkey.api.data.JdbcType;
 import org.labkey.api.reports.ExternalScriptEngine;
 import org.labkey.api.reports.ExternalScriptEngineDefinition;
+import org.labkey.api.reports.report.ScriptPackageUsageTracker;
 import org.labkey.vfs.FileLike;
 
 import javax.script.Bindings;
 import javax.script.ScriptContext;
 import javax.script.ScriptEngineFactory;
-import javax.script.ScriptException;
 import java.util.Arrays;
 import java.util.List;
 
@@ -35,6 +36,7 @@
 */
 public class RScriptEngine extends ExternalScriptEngine
 {
+    protected static final String PACKAGES_FILE = "labkeyRPackages.txt";
     public static final String KNITR_FORMAT = "r.script.engine.knitrFormat";
     public static final String KNITR_OUTPUT = "r.script.engine.knitrOutput";
     public static final String PANDOC_USE_DEFAULT_OUTPUT_FORMAT = "r.script.engine.pandocUseDefaultOutputFormat";
@@ -59,6 +61,7 @@ public ScriptEngineFactory getFactory()
         return new RScriptEngineFactory(_def);
     }
 
+    @Override
     protected FileLike prepareScriptFile(String script, ScriptContext context, List<String> extensions)
     {
         FileLike scriptFile;
@@ -86,18 +89,43 @@ protected FileLike prepareScriptFile(String script, ScriptContext context, List<
         return scriptFile;
     }
 
+    /**
+     * R appended to the end of the user script (running in the same R session) that writes the set of loaded packages,
+     * one per line, to a sidecar file for {@link #recordPackageUsage} to read back. Notes: tryCatch means a capture
+     * failure can never break the report or transform run.
+     */
     @Override
-    public Object eval(String script, ScriptContext context) throws ScriptException
+    protected @Nullable String getPackageCaptureEpilog(ScriptContext context)
     {
-        List<String> extensions = getFactory().getExtensions();
+        // For knitr the user's script is written as the knitr input (.rhtml/.rmd), where appended bare R would render
+        // as text rather than run. We instead record the generated wrapper's known libraries in recordPackageUsage().
+        if (getKnitrFormat(context) != RReportDescriptor.KnitrFormat.None)
+            return null;
+
+        return """
+                # --- LabKey R package usage capture ---
+                tryCatch(writeLines(sort(loadedNamespaces()), file.path(getwd(), "%s")), error = function(e) invisible(NULL))
+                """.formatted(PACKAGES_FILE);
+    }
 
-        if (!extensions.isEmpty())
+    @Override
+    protected void recordPackageUsage(ScriptContext context)
+    {
+        // Non-knitr R runs report their loaded packages via the capture epilog (see getPackageCaptureEpilog).
+        if (getKnitrFormat(context) == RReportDescriptor.KnitrFormat.None)
         {
-            FileLike scriptFile = prepareScriptFile(script, context, extensions);
-            return eval(scriptFile, context);
+            readPackageSidecar(context, PACKAGES_FILE, "r");
+            return;
         }
-        else
-            throw new ScriptException("There are no file name extensions registered for this ScriptEngine : " + getFactory().getLanguageName());
+
+        // For knitr, the generated wrapper (createKnitrScript) always loads knitr and, for the markdown+pandoc path,
+        // rmarkdown; record those as R package usage so they show up alongside other packages.
+        // NOTE: this only captures the wrapper's libraries, not packages loaded inside the report's own R chunks (e.g.
+        // ggplot2 used within an .rmd). Fully capturing those would require injecting a loadedNamespaces() write into
+        // createKnitrScript after the knit()/render() call.
+        ScriptPackageUsageTracker.record("r", "knitr");
+        if (getKnitrFormat(context) == RReportDescriptor.KnitrFormat.Markdown && isPandocEnabled())
+            ScriptPackageUsageTracker.record("r", "rmarkdown");
     }
 
     private boolean isPandocEnabled()