Skip to content

Commit 475bcfd

Browse files
authored
Add timeout for resource allocation in lock step and freestyle jobs (#1010)
Adds timeoutForAllocateResource and timeoutUnit parameters to the lock() pipeline step and lockTimeout/lockTimeoutUnit to freestyle job resource configuration. When a timeout is set and the resource is not acquired within the specified duration, the pipeline build fails with a LockWaitTimeoutException and the freestyle queue item is cancelled. Implementation uses a single scheduled task targeting the earliest deadline across all queued entries, with a periodic safety-net fallback. Fixes #866 Fixes #849 Fixes #30
1 parent cb968d8 commit 475bcfd

20 files changed

Lines changed: 929 additions & 16 deletions
Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
## Lock with allocation timeout
2+
3+
By default, the `lock` step waits indefinitely until the requested resource becomes available.
4+
With `timeoutForAllocateResource` you can set a maximum wait time — if the resource is not
5+
acquired within that period, the build fails immediately instead of blocking the queue forever.
6+
7+
This is useful when:
8+
- You prefer a fast failure over an indefinitely blocked pipeline
9+
- You want to detect resource starvation early
10+
- Your CI/CD has SLAs that cap how long a job may wait
11+
12+
### Pipeline (scripted)
13+
14+
```groovy
15+
node {
16+
// Wait up to 5 minutes for the resource, then fail
17+
lock(resource: 'my-printer', timeoutForAllocateResource: 5, timeoutUnit: 'MINUTES') {
18+
echo "Printer locked, printing ..."
19+
}
20+
}
21+
```
22+
23+
### Pipeline (declarative)
24+
25+
```groovy
26+
pipeline {
27+
agent any
28+
stages {
29+
stage('Deploy') {
30+
options {
31+
lock(resource: 'staging-env', timeoutForAllocateResource: 10, timeoutUnit: 'MINUTES')
32+
}
33+
steps {
34+
echo "Deploying to staging ..."
35+
}
36+
}
37+
}
38+
}
39+
```
40+
41+
### Label-based locking with timeout
42+
43+
```groovy
44+
pipeline {
45+
agent any
46+
stages {
47+
stage('Test') {
48+
steps {
49+
lock(label: 'phone', quantity: 1, variable: 'PHONE',
50+
timeoutForAllocateResource: 2, timeoutUnit: 'MINUTES') {
51+
echo "Running tests on ${env.PHONE}"
52+
}
53+
}
54+
}
55+
}
56+
}
57+
```
58+
59+
### Freestyle jobs
60+
61+
In a freestyle job configuration, go to **This build requires lockable resources** and set:
62+
- **Lock wait timeout**: the maximum time to wait (e.g. `5`)
63+
- **Timeout unit**: `SECONDS`, `MINUTES`, or `HOURS`
64+
65+
If the resource is not available within the configured timeout, the build is automatically
66+
removed from the Jenkins queue.
67+
68+
### Timeout values
69+
70+
| `timeoutUnit` | Description |
71+
|---------------|-------------|
72+
| `SECONDS` | Timeout in seconds |
73+
| `MINUTES` | Timeout in minutes (default) |
74+
| `HOURS` | Timeout in hours |
75+
76+
Setting `timeoutForAllocateResource: 0` (the default) disables the timeout — the build
77+
waits indefinitely, which preserves the original behaviour.

src/doc/examples/readme.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,4 @@ If you have a question, please open a [GitHub issue](https://github.com/jenkinsc
1313
- [Scripted vs declarative pipeline](scripted-vs-declarative-pipeline.md)
1414
- [Timeout inside lock](timeout-inside-lock.md)
1515
- [Dynamic resource pool expansion](dynamic-resource-pool-expansion.md)
16+
- [Lock with allocation timeout](lock-with-timeout.md)

src/main/java/org/jenkins/plugins/lockableresources/LockStep.java

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,20 @@ public class LockStep extends Step implements Serializable {
7171
@SuppressFBWarnings(value = "PA_PUBLIC_PRIMITIVE_ATTRIBUTE", justification = "Preserve API compatibility.")
7272
public int priority = 0;
7373

74+
/**
75+
* Timeout in the specified {@link #timeoutUnit} for waiting to acquire the resource.
76+
* 0 means no timeout (wait indefinitely). When the timeout expires, the step fails
77+
* with an exception instead of waiting forever.
78+
*/
79+
@SuppressFBWarnings(value = "PA_PUBLIC_PRIMITIVE_ATTRIBUTE", justification = "Preserve API compatibility.")
80+
public long timeoutForAllocateResource = 0;
81+
82+
/**
83+
* Time unit for {@link #timeoutForAllocateResource}. Defaults to MINUTES.
84+
*/
85+
@SuppressFBWarnings(value = "PA_PUBLIC_PRIMITIVE_ATTRIBUTE", justification = "Preserve API compatibility.")
86+
public String timeoutUnit = "MINUTES";
87+
7488
// it should be LockStep() - without params. But keeping this for backward compatibility
7589
// so `lock('resource1')` still works and `lock(label: 'label1', quantity: 3)` works too (resource
7690
// is not required)
@@ -154,6 +168,24 @@ public void setExtra(@CheckForNull List<LockStepResource> extra) {
154168
this.extra = extra;
155169
}
156170

171+
@DataBoundSetter
172+
public void setTimeoutForAllocateResource(long timeoutForAllocateResource) {
173+
this.timeoutForAllocateResource = Math.max(0, timeoutForAllocateResource);
174+
}
175+
176+
@DataBoundSetter
177+
public void setTimeoutUnit(String timeoutUnit) {
178+
if (timeoutUnit != null && !timeoutUnit.trim().isEmpty()) {
179+
// Validate it is a valid TimeUnit name
180+
try {
181+
java.util.concurrent.TimeUnit.valueOf(timeoutUnit.toUpperCase(Locale.ENGLISH));
182+
} catch (IllegalArgumentException e) {
183+
throw new IllegalArgumentException("Invalid timeoutUnit: " + timeoutUnit);
184+
}
185+
this.timeoutUnit = timeoutUnit.toUpperCase(Locale.ENGLISH);
186+
}
187+
}
188+
157189
@Extension
158190
public static final class DescriptorImpl extends StepDescriptor {
159191

@@ -229,6 +261,20 @@ public static FormValidation doCheckResourceSelectStrategy(
229261
return FormValidation.ok();
230262
}
231263

264+
@RequirePOST
265+
public ListBoxModel doFillTimeoutUnitItems(@AncestorInPath Item item) {
266+
if (item != null) {
267+
item.checkPermission(Item.CONFIGURE);
268+
} else {
269+
Jenkins.get().checkPermission(Jenkins.ADMINISTER);
270+
}
271+
ListBoxModel items = new ListBoxModel();
272+
items.add("Seconds", "SECONDS");
273+
items.add("Minutes", "MINUTES");
274+
items.add("Hours", "HOURS");
275+
return items;
276+
}
277+
232278
@Override
233279
public Set<Class<?>> getRequiredContext() {
234280
return Collections.singleton(TaskListener.class);

src/main/java/org/jenkins/plugins/lockableresources/LockStepExecution.java

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -147,8 +147,12 @@ private void onLockFailed(PrintStream logger, List<LockableResourcesStruct> reso
147147
getContext().onSuccess(null);
148148
} else {
149149
this.printBlockCause(logger, resourceHolderList);
150-
LockableResourcesManager.printLogs(
151-
"[" + step + "] is not free, waiting for execution ...", Level.FINE, LOGGER, logger);
150+
String waitMsg = "[" + step + "] is not free, waiting for execution ...";
151+
if (step.timeoutForAllocateResource > 0) {
152+
waitMsg += " (timeout: " + step.timeoutForAllocateResource + " "
153+
+ step.timeoutUnit.toLowerCase(java.util.Locale.ENGLISH) + ")";
154+
}
155+
LockableResourcesManager.printLogs(waitMsg, Level.FINE, LOGGER, logger);
152156
LockableResourcesManager lrm = LockableResourcesManager.get();
153157
lrm.queueContext(
154158
getContext(),
@@ -157,7 +161,9 @@ private void onLockFailed(PrintStream logger, List<LockableResourcesStruct> reso
157161
step.variable,
158162
step.inversePrecedence,
159163
step.priority,
160-
step.reason);
164+
step.reason,
165+
step.timeoutForAllocateResource,
166+
step.timeoutUnit);
161167
}
162168
}
163169

src/main/java/org/jenkins/plugins/lockableresources/LockableResourcesManager.java

Lines changed: 122 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,12 @@ public class LockableResourcesManager extends GlobalConfiguration {
9595
private transient volatile AtomicBoolean savePending;
9696
private transient volatile ScheduledExecutorService saveExecutor;
9797

98+
/** Single scheduled timeout task. Guarded by {@link #syncResources}. */
99+
private transient java.util.concurrent.ScheduledFuture<?> nextTimeoutTask;
100+
101+
/** Deadline (epoch ms) the current {@link #nextTimeoutTask} targets. 0 = none. */
102+
private transient long nextTimeoutDeadline;
103+
98104
@DataBoundSetter
99105
public void setAllowEmptyOrNullValues(boolean allowEmptyOrNullValues) {
100106
this.allowEmptyOrNullValues = allowEmptyOrNullValues;
@@ -870,8 +876,9 @@ private QueuedContextStruct getNextQueuedContext() {
870876

871877
LOGGER.fine("current queue size: " + this.queuedContexts.size());
872878
LOGGER.finest("current queue: " + this.queuedContexts);
873-
List<QueuedContextStruct> orphan = new ArrayList<>();
879+
List<QueuedContextStruct> toRemove = new ArrayList<>();
874880
QueuedContextStruct nextEntry = null;
881+
long earliestDeadline = Long.MAX_VALUE;
875882

876883
// the first one added lock is the oldest one, and this wins
877884

@@ -880,18 +887,43 @@ private QueuedContextStruct getNextQueuedContext() {
880887
// check queue list first
881888
if (!entry.isValid()) {
882889
LOGGER.fine("well be removed: " + idx + " " + entry);
883-
orphan.add(entry);
890+
toRemove.add(entry);
891+
continue;
892+
}
893+
894+
// check if the entry has timed out waiting for resources
895+
if (entry.isTimedOut()) {
896+
LOGGER.info("Queue entry timed out waiting for resources: " + entry);
897+
toRemove.add(entry);
898+
PrintStream logger = entry.getLogger();
899+
String msg = "[" + entry.getResourceDescription()
900+
+ "] timed out waiting for resource allocation after "
901+
+ entry.getTimeoutForAllocateResource() + " "
902+
+ entry.getTimeoutUnit().toLowerCase(java.util.Locale.ENGLISH);
903+
printLogs(msg, logger, Level.WARNING);
904+
entry.getContext()
905+
.onFailure(new org.jenkins.plugins.lockableresources.queue.LockWaitTimeoutException(msg));
884906
continue;
885907
}
908+
909+
// track the earliest deadline among remaining entries for rescheduling
910+
long deadline = entry.getTimeoutDeadlineMillis();
911+
if (deadline > 0 && deadline < earliestDeadline) {
912+
earliestDeadline = deadline;
913+
}
914+
886915
LOGGER.finest("oldest win - index: " + idx + " " + entry);
887916

888917
nextEntry = getNextQueuedContextEntry(entry);
889918
}
890919

891-
if (!orphan.isEmpty()) {
892-
this.queuedContexts.removeAll(orphan);
920+
if (!toRemove.isEmpty()) {
921+
this.queuedContexts.removeAll(toRemove);
893922
}
894923

924+
// reschedule for the next earliest deadline
925+
scheduleTimeoutAt(earliestDeadline);
926+
895927
return nextEntry;
896928
}
897929

@@ -1458,12 +1490,22 @@ public void queueContext(
14581490
String variableName,
14591491
boolean inversePrecedence,
14601492
int priority) {
1461-
queueContext(context, requiredResources, resourceDescription, variableName, inversePrecedence, priority, null);
1493+
queueContext(
1494+
context,
1495+
requiredResources,
1496+
resourceDescription,
1497+
variableName,
1498+
inversePrecedence,
1499+
priority,
1500+
null,
1501+
0,
1502+
"MINUTES");
14621503
}
14631504

1505+
// ---------------------------------------------------------------------------
14641506
/*
14651507
* Adds the given context and the required resources to the queue if
1466-
* this context is not yet queued.
1508+
* this context is not yet queued, with reason and timeout for resource allocation.
14671509
*/
14681510
@Restricted(NoExternalUse.class)
14691511
public void queueContext(
@@ -1473,7 +1515,9 @@ public void queueContext(
14731515
String variableName,
14741516
boolean inversePrecedence,
14751517
int priority,
1476-
String reason) {
1518+
String reason,
1519+
long timeoutForAllocateResource,
1520+
String timeoutUnit) {
14771521
synchronized (syncResources) {
14781522
for (QueuedContextStruct entry : this.queuedContexts) {
14791523
if (entry.getContext() == context) {
@@ -1484,7 +1528,14 @@ public void queueContext(
14841528

14851529
int queueIndex = 0;
14861530
QueuedContextStruct newQueueItem = new QueuedContextStruct(
1487-
context, requiredResources, resourceDescription, variableName, priority, reason);
1531+
context,
1532+
requiredResources,
1533+
resourceDescription,
1534+
variableName,
1535+
priority,
1536+
reason,
1537+
timeoutForAllocateResource,
1538+
timeoutUnit);
14881539

14891540
if (!inversePrecedence || priority != 0) {
14901541
queueIndex = this.queuedContexts.size() - 1;
@@ -1506,6 +1557,13 @@ public void queueContext(
15061557
Level.FINE);
15071558

15081559
save();
1560+
1561+
// If this entry has a timeout and its deadline is earlier than the
1562+
// currently scheduled one, (re)schedule so it fires on time.
1563+
long deadline = newQueueItem.getTimeoutDeadlineMillis();
1564+
if (deadline > 0 && (nextTimeoutDeadline == 0 || deadline < nextTimeoutDeadline)) {
1565+
scheduleTimeoutAt(deadline);
1566+
}
15091567
}
15101568
}
15111569

@@ -1562,7 +1620,7 @@ public void refreshQueue() {
15621620
// Invalidate cached candidates so waiting jobs re-evaluate with current labels
15631621
cachedCandidates.invalidateAll();
15641622

1565-
// Process waiting pipeline jobs
1623+
// Process waiting pipeline jobs (also handles timeouts)
15661624
synchronized (syncResources) {
15671625
while (proceedNextContext()) {
15681626
// process as many contexts as possible
@@ -1573,6 +1631,61 @@ public void refreshQueue() {
15731631
scheduleQueueMaintenance();
15741632
}
15751633

1634+
// ---------------------------------------------------------------------------
1635+
/**
1636+
* Checks for timed-out entries in the pipeline lock queue and fails them.
1637+
* Called by {@link org.jenkins.plugins.lockableresources.queue.LockWaitTimeoutPeriodicWork}
1638+
* as a safety net.
1639+
*/
1640+
@Restricted(NoExternalUse.class)
1641+
public void checkTimeouts() {
1642+
synchronized (syncResources) {
1643+
// proceedNextContext → getNextQueuedContext handles timeouts + rescheduling
1644+
while (proceedNextContext()) {
1645+
// process as many contexts as possible
1646+
}
1647+
}
1648+
}
1649+
1650+
// ---------------------------------------------------------------------------
1651+
/**
1652+
* Schedules (or reschedules) the single timeout task to fire at the given
1653+
* deadline. If {@code deadline} is {@link Long#MAX_VALUE} the current task
1654+
* is cancelled and nothing new is scheduled.
1655+
* Must be called while holding {@link #syncResources}.
1656+
*/
1657+
private void scheduleTimeoutAt(long deadline) {
1658+
// Cancel the current task — we will either replace it or clear it
1659+
if (nextTimeoutTask != null) {
1660+
nextTimeoutTask.cancel(false);
1661+
nextTimeoutTask = null;
1662+
nextTimeoutDeadline = 0;
1663+
}
1664+
1665+
if (deadline == Long.MAX_VALUE || deadline <= 0) {
1666+
return;
1667+
}
1668+
1669+
nextTimeoutDeadline = deadline;
1670+
// Small buffer so the deadline has definitely passed when we check
1671+
long delayMs = Math.max(0, deadline - System.currentTimeMillis()) + 500L;
1672+
LOGGER.fine("Scheduling timeout check in " + delayMs + "ms");
1673+
nextTimeoutTask = jenkins.util.Timer.get()
1674+
.schedule(
1675+
() -> {
1676+
LOGGER.fine("Scheduled timeout check fired");
1677+
synchronized (syncResources) {
1678+
nextTimeoutDeadline = 0;
1679+
nextTimeoutTask = null;
1680+
while (proceedNextContext()) {
1681+
// process as many contexts as possible
1682+
}
1683+
}
1684+
},
1685+
delayMs,
1686+
java.util.concurrent.TimeUnit.MILLISECONDS);
1687+
}
1688+
15761689
// ---------------------------------------------------------------------------
15771690
private AtomicBoolean getSavePending() {
15781691
AtomicBoolean sp = savePending;

0 commit comments

Comments
 (0)