Skip to content

Commit a457d5d

Browse files
timjatimja
authored
Save computed graph to disk if build is complete (#1224)
1 parent aeacce1 commit a457d5d

6 files changed

Lines changed: 285 additions & 8 deletions

File tree

‎pom.xml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,10 @@
5555
</dependencyManagement>
5656

5757
<dependencies>
58+
<dependency>
59+
<groupId>io.jenkins.plugins</groupId>
60+
<artifactId>caffeine-api</artifactId>
61+
</dependency>
5862
<dependency>
5963
<groupId>io.jenkins.plugins</groupId>
6064
<artifactId>ionicons-api</artifactId>

‎src/main/java/io/jenkins/plugins/pipelinegraphview/utils/PipelineGraphApi.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,10 @@ private static String getStageNode(FlowNodeWrapper flowNodeWrapper) {
157157
}
158158

159159
public PipelineGraph createTree() {
160+
return PipelineGraphViewCache.get().getGraph(run, this::computeTree);
161+
}
162+
163+
PipelineGraph computeTree() {
160164
return createTree(CachedPipelineNodeGraphAdaptor.instance.getFor(run));
161165
}
162166
}

‎src/main/java/io/jenkins/plugins/pipelinegraphview/utils/PipelineGraphViewCache.java‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
package io.jenkins.plugins.pipelinegraphview.utils;
2+
3+
import com.github.benmanes.caffeine.cache.Cache;
4+
import com.github.benmanes.caffeine.cache.Caffeine;
5+
import hudson.XmlFile;
6+
import hudson.util.XStream2;
7+
import io.jenkins.plugins.pipelinegraphview.analysis.TimingInfo;
8+
import java.io.File;
9+
import java.io.IOException;
10+
import java.util.function.Supplier;
11+
import org.jenkinsci.plugins.workflow.job.WorkflowRun;
12+
import org.slf4j.Logger;
13+
import org.slf4j.LoggerFactory;
14+
15+
/**
16+
* Disk-backed cache for the computed pipeline graph and step list of completed runs.
17+
* For in-progress runs the cache is transparent (every call recomputes). Once a run is
18+
* no longer building, results are persisted under the run's directory, so later
19+
* requests — including after a Jenkins restart — are served without recomputation.
20+
*/
21+
public class PipelineGraphViewCache {
22+
23+
/**
24+
* Hand-bump when any persisted DTO shape changes in a non-backwards-compatible way.
25+
* Older files are ignored and recomputed on the next read.
26+
*/
27+
static final int SCHEMA_VERSION = 1;
28+
29+
private static final String CACHE_FILE_NAME = "pipeline-graph-view-cache.xml";
30+
private static final Logger logger = LoggerFactory.getLogger(PipelineGraphViewCache.class);
31+
private static final PipelineGraphViewCache INSTANCE = new PipelineGraphViewCache();
32+
33+
private final Cache<String, CachedPayload> memCache =
34+
Caffeine.newBuilder().maximumSize(256).build();
35+
36+
public static PipelineGraphViewCache get() {
37+
return INSTANCE;
38+
}
39+
40+
PipelineGraphViewCache() {}
41+
42+
public PipelineGraph getGraph(WorkflowRun run, Supplier<PipelineGraph> compute) {
43+
if (run.isBuilding()) {
44+
return compute.get();
45+
}
46+
CachedPayload payload = load(run);
47+
synchronized (payload) {
48+
if (payload.graph == null) {
49+
payload.graph = compute.get();
50+
payload.schemaVersion = SCHEMA_VERSION;
51+
write(run, payload);
52+
}
53+
return payload.graph;
54+
}
55+
}
56+
57+
public PipelineStepList getAllSteps(WorkflowRun run, Supplier<PipelineStepList> compute) {
58+
if (run.isBuilding()) {
59+
return compute.get();
60+
}
61+
CachedPayload payload = load(run);
62+
synchronized (payload) {
63+
if (payload.allSteps == null) {
64+
payload.allSteps = compute.get();
65+
payload.schemaVersion = SCHEMA_VERSION;
66+
write(run, payload);
67+
}
68+
return payload.allSteps;
69+
}
70+
}
71+
72+
private CachedPayload load(WorkflowRun run) {
73+
return memCache.get(run.getExternalizableId(), k -> readFromDisk(run));
74+
}
75+
76+
private CachedPayload readFromDisk(WorkflowRun run) {
77+
XmlFile file = cacheFile(run);
78+
if (!file.exists()) {
79+
return new CachedPayload();
80+
}
81+
try {
82+
Object read = file.read();
83+
if (read instanceof CachedPayload loaded && loaded.schemaVersion == SCHEMA_VERSION) {
84+
return loaded;
85+
}
86+
logger.debug("Discarding pipeline graph cache for {}: schema version mismatch", run.getExternalizableId());
87+
} catch (IOException e) {
88+
logger.warn("Failed to read pipeline graph cache for {}", run.getExternalizableId(), e);
89+
}
90+
return new CachedPayload();
91+
}
92+
93+
private void write(WorkflowRun run, CachedPayload payload) {
94+
try {
95+
cacheFile(run).write(payload);
96+
} catch (IOException e) {
97+
logger.warn("Failed to write pipeline graph cache for {}", run.getExternalizableId(), e);
98+
}
99+
}
100+
101+
private XmlFile cacheFile(WorkflowRun run) {
102+
return new XmlFile(XSTREAM, new File(run.getRootDir(), CACHE_FILE_NAME));
103+
}
104+
105+
/** Test hook: drop in-memory entries so the next call goes through the disk read path. */
106+
void invalidateMemory() {
107+
memCache.invalidateAll();
108+
}
109+
110+
static class CachedPayload {
111+
int schemaVersion;
112+
PipelineGraph graph;
113+
PipelineStepList allSteps;
114+
}
115+
116+
private static final XStream2 XSTREAM = new XStream2();
117+
118+
static {
119+
XSTREAM.alias("pipeline-graph-view-cache", CachedPayload.class);
120+
XSTREAM.alias("pipeline-graph", PipelineGraph.class);
121+
XSTREAM.alias("pipeline-stage", PipelineStage.class);
122+
XSTREAM.alias("pipeline-step", PipelineStep.class);
123+
XSTREAM.alias("pipeline-step-list", PipelineStepList.class);
124+
XSTREAM.alias("pipeline-input-step", PipelineInputStep.class);
125+
XSTREAM.alias("timing-info", TimingInfo.class);
126+
XSTREAM.alias("pipeline-state", PipelineState.class);
127+
}
128+
}

‎src/main/java/io/jenkins/plugins/pipelinegraphview/utils/PipelineStepApi.java‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -105,13 +105,20 @@ private PipelineStepList getAllSteps(PipelineStepBuilderApi builder, boolean run
105105
}
106106

107107
public PipelineStepList getSteps(String stageId) {
108-
// Look up the completed state before computing steps.
109-
boolean runIsComplete = !run.isBuilding();
110-
return getSteps(stageId, CachedPipelineNodeGraphAdaptor.instance.getFor(run), runIsComplete);
108+
PipelineStepList all = getAllSteps();
109+
List<PipelineStep> filtered =
110+
all.steps.stream().filter(s -> stageId.equals(s.stageId)).collect(Collectors.toList());
111+
PipelineStepList result = new PipelineStepList(filtered, all.runIsComplete);
112+
result.sort();
113+
return result;
111114
}
112115

113116
/* Returns a PipelineStepList, sorted by stageId and Id. */
114117
public PipelineStepList getAllSteps() {
118+
return PipelineGraphViewCache.get().getAllSteps(run, this::computeAllSteps);
119+
}
120+
121+
PipelineStepList computeAllSteps() {
115122
// Look up the completed state before computing steps.
116123
boolean runIsComplete = !run.isBuilding();
117124
return getAllSteps(CachedPipelineNodeGraphAdaptor.instance.getFor(run), runIsComplete);

‎src/test/java/io/jenkins/plugins/pipelinegraphview/PipelineGraphViewTest.java‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,10 @@
1515
import org.jenkinsci.plugins.workflow.test.steps.SemaphoreStep;
1616
import org.junit.jupiter.api.Test;
1717
import org.jvnet.hudson.test.Issue;
18-
import org.slf4j.Logger;
19-
import org.slf4j.LoggerFactory;
2018

2119
@WithJenkinsConfiguredWithCode
2220
@UsePlaywright(PlaywrightConfig.class)
2321
class PipelineGraphViewTest {
24-
private static final Logger log = LoggerFactory.getLogger(PipelineGraphViewTest.class);
25-
2622
// Code generation can be generated against local using to give an idea of what commands to use
2723
// mvn exec:java -e -D exec.mainClass="com.microsoft.playwright.CLI" -Dexec.classpathScope=test -Dexec.args="codegen
2824
// http://localhost:8080/jenkins
@@ -168,7 +164,7 @@ void searchOffScreen(Page p, JenkinsConfiguredWithCodeRule j) throws Exception {
168164
@Test
169165
@ConfiguredWithCode("configure-appearance.yml")
170166
void errorWithMessage(Page p, JenkinsConfiguredWithCodeRule j) throws Exception {
171-
String name = "gh1169";
167+
String name = "gh1169_errorWithMessage";
172168
WorkflowRun run = TestUtils.createAndRunJob(j, name, "gh1169_errorWithMessage.jenkinsfile", Result.FAILURE);
173169

174170
// Note that the locator used in stageHasSteps accumulates the error step's message text content into the found

‎src/test/java/io/jenkins/plugins/pipelinegraphview/utils/PipelineGraphViewCacheTest.java‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
package io.jenkins.plugins.pipelinegraphview.utils;
2+
3+
import static org.awaitility.Awaitility.await;
4+
import static org.hamcrest.MatcherAssert.assertThat;
5+
import static org.hamcrest.Matchers.equalTo;
6+
import static org.hamcrest.Matchers.greaterThan;
7+
import static org.hamcrest.Matchers.is;
8+
import static org.hamcrest.Matchers.not;
9+
import static org.hamcrest.Matchers.nullValue;
10+
import static org.jenkinsci.plugins.workflow.test.steps.SemaphoreStep.*;
11+
12+
import hudson.XmlFile;
13+
import hudson.model.Result;
14+
import hudson.util.XStream2;
15+
import io.jenkins.plugins.pipelinegraphview.utils.PipelineGraphViewCache.CachedPayload;
16+
import java.io.File;
17+
import java.nio.file.Files;
18+
import java.time.Duration;
19+
import java.util.concurrent.atomic.AtomicInteger;
20+
import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition;
21+
import org.jenkinsci.plugins.workflow.job.WorkflowJob;
22+
import org.jenkinsci.plugins.workflow.job.WorkflowRun;
23+
import org.junit.jupiter.api.BeforeEach;
24+
import org.junit.jupiter.api.Test;
25+
import org.jvnet.hudson.test.JenkinsRule;
26+
import org.jvnet.hudson.test.junit.jupiter.WithJenkins;
27+
28+
@WithJenkins
29+
class PipelineGraphViewCacheTest {
30+
31+
private static final String CACHE_FILE_NAME = "pipeline-graph-view-cache.xml";
32+
33+
private JenkinsRule j;
34+
private PipelineGraphViewCache cache;
35+
36+
@BeforeEach
37+
void setUp(JenkinsRule j) {
38+
this.j = j;
39+
// Use a fresh per-test instance so state doesn't leak between cases.
40+
this.cache = new PipelineGraphViewCache();
41+
}
42+
43+
@Test
44+
void coldCache_computesAndWritesFile() throws Exception {
45+
WorkflowRun run = TestUtils.createAndRunJob(j, "cold", "smokeTest.jenkinsfile", Result.FAILURE);
46+
File cacheFile = new File(run.getRootDir(), CACHE_FILE_NAME);
47+
assertThat("no cache file before first call", cacheFile.exists(), is(false));
48+
49+
AtomicInteger computes = new AtomicInteger();
50+
PipelineGraph graph = cache.getGraph(run, () -> {
51+
computes.incrementAndGet();
52+
return new PipelineGraphApi(run).computeTree();
53+
});
54+
55+
assertThat(graph, is(not(nullValue())));
56+
assertThat(graph.stages.isEmpty(), is(false));
57+
assertThat("supplier ran exactly once", computes.get(), equalTo(1));
58+
assertThat("cache file exists after first call", cacheFile.exists(), is(true));
59+
assertThat("cache file is non-empty", Files.size(cacheFile.toPath()), greaterThan(0L));
60+
}
61+
62+
@Test
63+
void warmCache_returnsPayloadWithoutComputing() throws Exception {
64+
WorkflowRun run = TestUtils.createAndRunJob(j, "warm", "smokeTest.jenkinsfile", Result.FAILURE);
65+
cache.getGraph(run, () -> new PipelineGraphApi(run).computeTree());
66+
67+
// Drop in-memory cache to force re-read from disk on next call.
68+
cache.invalidateMemory();
69+
70+
AtomicInteger computes = new AtomicInteger();
71+
PipelineGraph again = cache.getGraph(run, () -> {
72+
computes.incrementAndGet();
73+
return new PipelineGraphApi(run).computeTree();
74+
});
75+
76+
assertThat(again, is(not(nullValue())));
77+
assertThat("supplier did not run — served from disk", computes.get(), equalTo(0));
78+
assertThat(again.stages.isEmpty(), is(false));
79+
}
80+
81+
@Test
82+
void inProgressRun_doesNotPersist() throws Exception {
83+
WorkflowRun run = startLongRunningJob();
84+
try {
85+
AtomicInteger computes = new AtomicInteger();
86+
cache.getGraph(run, () -> {
87+
computes.incrementAndGet();
88+
return new PipelineGraphApi(run).computeTree();
89+
});
90+
cache.getGraph(run, () -> {
91+
computes.incrementAndGet();
92+
return new PipelineGraphApi(run).computeTree();
93+
});
94+
95+
File cacheFile = new File(run.getRootDir(), CACHE_FILE_NAME);
96+
assertThat("no cache file while run is in-progress", cacheFile.exists(), is(false));
97+
assertThat("both calls recomputed", computes.get(), equalTo(2));
98+
} finally {
99+
run.getExecutor().interrupt();
100+
j.waitForCompletion(run);
101+
}
102+
}
103+
104+
@Test
105+
void schemaVersionMismatch_isIgnoredAndRecomputed() throws Exception {
106+
WorkflowRun run = TestUtils.createAndRunJob(j, "schema", "smokeTest.jenkinsfile", Result.FAILURE);
107+
File cacheFile = new File(run.getRootDir(), CACHE_FILE_NAME);
108+
109+
// Write a stale-version payload directly to disk.
110+
CachedPayload stale = new CachedPayload();
111+
stale.schemaVersion = Integer.MIN_VALUE;
112+
XStream2 xstream = new XStream2();
113+
xstream.alias("pipeline-graph-view-cache", CachedPayload.class);
114+
new XmlFile(xstream, cacheFile).write(stale);
115+
assertThat(cacheFile.exists(), is(true));
116+
117+
AtomicInteger computes = new AtomicInteger();
118+
PipelineGraph graph = cache.getGraph(run, () -> {
119+
computes.incrementAndGet();
120+
return new PipelineGraphApi(run).computeTree();
121+
});
122+
123+
assertThat("stale file ignored; supplier ran", computes.get(), equalTo(1));
124+
assertThat(graph.stages.isEmpty(), is(false));
125+
}
126+
127+
private WorkflowRun startLongRunningJob() throws Exception {
128+
String jenkinsfile = "node { echo 'hi'; semaphore 'wait' }";
129+
var job = j.createProject(WorkflowJob.class, "running");
130+
job.setDefinition(new CpsFlowDefinition(jenkinsfile, true));
131+
var future = job.scheduleBuild2(0);
132+
WorkflowRun run = future.waitForStart();
133+
// Wait until the semaphore step is actually blocking so the run is reliably "building".
134+
await().atMost(Duration.ofSeconds(30)).until(run::isBuilding);
135+
waitForStart("wait/1", run);
136+
return run;
137+
}
138+
}

0 commit comments

Comments
 (0)