Make Pipeline Overview responsive on large in-progress builds

src/main/java/io/jenkins/plugins/pipelinegraphview/livestate/LiveGraphLifecycle.java

@@ -0,0 +1,87 @@
+package io.jenkins.plugins.pipelinegraphview.livestate;
+
+import edu.umd.cs.findbugs.annotations.NonNull;
+import hudson.Extension;
+import io.jenkins.plugins.pipelinegraphview.utils.PipelineGraph;
+import io.jenkins.plugins.pipelinegraphview.utils.PipelineGraphApi;
+import io.jenkins.plugins.pipelinegraphview.utils.PipelineGraphViewCache;
+import io.jenkins.plugins.pipelinegraphview.utils.PipelineStepApi;
+import io.jenkins.plugins.pipelinegraphview.utils.PipelineStepList;
+import org.jenkinsci.plugins.workflow.flow.FlowExecution;
+import org.jenkinsci.plugins.workflow.flow.FlowExecutionListener;
+import org.jenkinsci.plugins.workflow.job.WorkflowRun;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Creates and readies {@link LiveGraphState} entries at execution start / resume, and
+ * hands the final graph off to the on-disk cache at completion so the first post-build
+ * read doesn't need a fresh scanner sweep.
+ */
+@Extension
+public class LiveGraphLifecycle extends FlowExecutionListener {
+
+    private static final Logger logger = LoggerFactory.getLogger(LiveGraphLifecycle.class);
+
+    @Override
+    public void onRunning(@NonNull FlowExecution execution) {
+        try {
+            // Fresh execution — no prior nodes to catch up.
+            LiveGraphState state = LiveGraphRegistry.get().getOrCreate(execution);
+            if (state != null) {
+                state.markReady();
+            }
+        } catch (Throwable t) {
+            logger.warn("onRunning failed", t);
+        }
+    }
+
+    @Override
+    public void onResumed(@NonNull FlowExecution execution) {
+        try {
+            // Resumed after a Jenkins restart: the execution's persisted graph already holds
+            // nodes we never saw live. Running here (not on the CPS VM) makes a scanner walk
+            // safe.
+            LiveGraphState state = LiveGraphRegistry.get().getOrCreate(execution);
+            if (state != null) {
+                LiveGraphPopulator.catchUp(execution, state);
+                state.markReady();
+            }
+        } catch (Throwable t) {
+            logger.warn("onResumed failed", t);
+        }
+    }
+
+    @Override
+    public void onCompleted(@NonNull FlowExecution execution) {
+        try {
+            WorkflowRun run = workflowRunFor(execution);
+            if (run != null) {
+                PipelineGraph graph = new PipelineGraphApi(run).computeTree();
+                PipelineStepList allSteps = new PipelineStepApi(run).computeAllSteps();
+                // WorkflowRun.isBuilding() can still be true here even though FlowExecution
+                // is complete; rebuild with runIsComplete=true so the persisted copy matches
+                // reality. PipelineGraph.complete already reflects FlowExecution.isComplete().
+                PipelineStepList finalSteps = new PipelineStepList(allSteps.steps, true);
+                PipelineGraphViewCache.get().seed(run, graph, finalSteps);
+            }
+        } catch (Throwable t) {
+            logger.warn("seeding disk cache on completion failed", t);
+        } finally {
+            try {
+                LiveGraphRegistry.get().remove(execution);
+            } catch (Throwable t) {
+                logger.warn("state eviction on completion failed", t);
+            }
+        }
+    }
+
+    private static WorkflowRun workflowRunFor(FlowExecution execution) {
+        try {
+            Object exec = execution.getOwner().getExecutable();
+            return exec instanceof WorkflowRun r ? r : null;
+        } catch (Exception e) {
+            return null;
+        }
+    }
+}

src/main/java/io/jenkins/plugins/pipelinegraphview/livestate/LiveGraphPopulator.java

@@ -0,0 +1,64 @@
+package io.jenkins.plugins.pipelinegraphview.livestate;
+
+import hudson.Extension;
+import org.jenkinsci.plugins.workflow.flow.FlowExecution;
+import org.jenkinsci.plugins.workflow.flow.GraphListener;
+import org.jenkinsci.plugins.workflow.graph.FlowNode;
+import org.jenkinsci.plugins.workflow.graphanalysis.DepthFirstScanner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Extension that captures every new {@link FlowNode} across every running execution and
+ * feeds it to the corresponding {@link LiveGraphState}.
+ *
+ * <p>We use {@link GraphListener.Synchronous} because callers expect that once a node is a
+ * head, the next API read reflects it — async delivery would create a lag window where the
+ * snapshot is behind the execution. The listener therefore does the minimum under the
+ * monitor and never scans: any catch-up belongs in {@link LiveGraphLifecycle}, on a Jenkins
+ * event thread.
+ */
+@Extension
+public class LiveGraphPopulator implements GraphListener.Synchronous {
+
+    private static final Logger logger = LoggerFactory.getLogger(LiveGraphPopulator.class);
+
+    @Override
+    public void onNewHead(FlowNode node) {
+        LiveGraphState state = null;
+        try {
+            FlowExecution execution = node.getExecution();
+            state = LiveGraphRegistry.get().getOrCreate(execution);
+            if (state == null) {
+                return; // feature disabled or execution not a WorkflowRun
+            }
+            state.addNode(node);
+        } catch (Throwable t) {
+            // A thrown exception here propagates into the CPS VM and can abort the build.
+            // Poison the state so subsequent reads fall back to the scanner; log the failure
+            // but never rethrow.
+            logger.warn("live state failed; falling back to scanner", t);
+            if (state != null) {
+                state.poison();
+            }
+        }
+    }
+
+    /**
+     * Backfills the live state from the execution's persisted graph. Called only from
+     * {@link LiveGraphLifecycle#onResumed}, which runs on a Jenkins event thread — never
+     * from a {@link GraphListener.Synchronous} path on the CPS VM.
+     */
+    static void catchUp(FlowExecution execution, LiveGraphState state) {
+        try {
+            DepthFirstScanner scanner = new DepthFirstScanner();
+            scanner.setup(execution.getCurrentHeads());
+            for (FlowNode existing : scanner) {
+                state.addNode(existing);
+            }
+        } catch (Throwable t) {
+            logger.warn("catch-up failed; poisoning state", t);
+            state.poison();
+        }
+    }
+}

src/main/java/io/jenkins/plugins/pipelinegraphview/livestate/LiveGraphRegistry.java

@@ -0,0 +1,132 @@
+package io.jenkins.plugins.pipelinegraphview.livestate;
+
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import io.jenkins.plugins.pipelinegraphview.utils.PipelineGraph;
+import io.jenkins.plugins.pipelinegraphview.utils.PipelineStepList;
+import java.time.Duration;
+import jenkins.util.SystemProperties;
+import org.jenkinsci.plugins.workflow.flow.FlowExecution;
+import org.jenkinsci.plugins.workflow.job.WorkflowRun;
+
+/**
+ * Singleton holding one {@link LiveGraphState} per in-progress run.
+ * Entries are created on demand by the listener / lifecycle code, removed on completion,
+ * and otherwise bounded by a Caffeine LRU so abandoned entries (deleted runs, listener
+ * bugs) don't leak.
+ */
+public final class LiveGraphRegistry {
+
+    // IMPORTANT: keep CACHE_MAX_SIZE declared BEFORE INSTANCE. Static fields initialise in
+    // source order, and the instance's Caffeine builder reads CACHE_MAX_SIZE — if INSTANCE
+    // were declared first, CACHE_MAX_SIZE would still be 0 during construction and Caffeine
+    // would evict every entry immediately (maximumSize(0) means "no entries allowed").
+    private static final int CACHE_MAX_SIZE =
+            SystemProperties.getInteger(LiveGraphRegistry.class.getName() + ".size", 512);
+
+    private static final LiveGraphRegistry INSTANCE = new LiveGraphRegistry();
+
+    public static LiveGraphRegistry get() {
+        return INSTANCE;
+    }
+
+    private final Cache<String, LiveGraphState> states = Caffeine.newBuilder()
+            .maximumSize(CACHE_MAX_SIZE)
+            .expireAfterAccess(Duration.ofMinutes(30))
+            .build();
+
+    LiveGraphRegistry() {}
+
+    /**
+     * Escape hatch. Setting this system property to {@code false} makes
+     * {@link #snapshot(WorkflowRun)} always return {@code null}, forcing callers to use the
+     * scanner fallback. Useful if a regression lands in the live-state path.
+     */
+    private static boolean disabled() {
+        return !SystemProperties.getBoolean(LiveGraphRegistry.class.getName() + ".enabled", true);
+    }
+
+    LiveGraphState getOrCreate(FlowExecution execution) {
+        if (disabled()) {
+            return null;
+        }
+        String key = keyFor(execution);
+        if (key == null) {
+            return null;
+        }
+        return states.get(key, k -> new LiveGraphState());
+    }
+
+    /**
+     * Returns a snapshot of the live state for this run, or {@code null} if none exists
+     * (feature disabled, state never populated, state poisoned). Callers must treat
+     * {@code null} as "fall back to the scanner path."
+     */
+    public LiveGraphSnapshot snapshot(WorkflowRun run) {
+        if (disabled()) {
+            return null;
+        }
+        LiveGraphState state = states.getIfPresent(run.getExternalizableId());
+        return state == null ? null : state.snapshot();
+    }
+
+    void remove(FlowExecution execution) {
+        String key = keyFor(execution);
+        if (key != null) {
+            states.invalidate(key);
+        }
+    }
+
+    /**
+     * Returns a previously-cached {@link PipelineGraph} for this run if it was computed at
+     * or after {@code minVersion}, otherwise {@code null}. Use {@link LiveGraphSnapshot#version()}
+     * as the argument — cache entries older than the caller's snapshot are rejected.
+     */
+    public PipelineGraph cachedGraph(WorkflowRun run, long minVersion) {
+        if (disabled()) {
+            return null;
+        }
+        LiveGraphState state = states.getIfPresent(run.getExternalizableId());
+        return state == null ? null : state.cachedGraph(minVersion);
+    }
+
+    public void cacheGraph(WorkflowRun run, long version, PipelineGraph graph) {
+        if (disabled()) {
+            return;
+        }
+        LiveGraphState state = states.getIfPresent(run.getExternalizableId());
+        if (state != null) {
+            state.cacheGraph(version, graph);
+        }
+    }
+
+    public PipelineStepList cachedAllSteps(WorkflowRun run, long minVersion) {
+        if (disabled()) {
+            return null;
+        }
+        LiveGraphState state = states.getIfPresent(run.getExternalizableId());
+        return state == null ? null : state.cachedAllSteps(minVersion);
+    }
+
+    public void cacheAllSteps(WorkflowRun run, long version, PipelineStepList steps) {
+        if (disabled()) {
+            return;
+        }
+        LiveGraphState state = states.getIfPresent(run.getExternalizableId());
+        if (state != null) {
+            state.cacheAllSteps(version, steps);
+        }
+    }
+
+    private static String keyFor(FlowExecution execution) {
+        try {
+            Object exec = execution.getOwner().getExecutable();
+            if (exec instanceof WorkflowRun run) {
+                return run.getExternalizableId();
+            }
+            return null;
+        } catch (Exception e) {
+            return null;
+        }
+    }
+}

src/main/java/io/jenkins/plugins/pipelinegraphview/livestate/LiveGraphSnapshot.java

@@ -0,0 +1,11 @@
+package io.jenkins.plugins.pipelinegraphview.livestate;
+
+import java.util.List;
+import org.jenkinsci.plugins.workflow.graph.FlowNode;
+
+/**
+ * Immutable projection of a {@link LiveGraphState} at a point in time. {@code version} is
+ * a monotonic counter that bumps on every new flow node, so callers can use it as a cache
+ * key for computed DTOs.
+ */
+public record LiveGraphSnapshot(List<FlowNode> nodes, List<FlowNode> workspaceNodes, long version) {}