Skip to content

Commit

Permalink
Experimental change to prevent offlining of slave
Browse files Browse the repository at this point in the history
Check to see if any items in the queue (including blocked items) ask for this node explicitly.
This is the case for individual Ivy/Maven Module builds which must build on the same node as their parent.
If there are any then don't offline this node.
  • Loading branch information
Timothy Bingaman authored and perrinmorrow committed Mar 14, 2021
1 parent 6bfebb0 commit c5876d6
Showing 1 changed file with 26 additions and 2 deletions.
28 changes: 26 additions & 2 deletions src/main/java/hudson/plugins/ec2/EC2RetentionStrategy.java
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import hudson.model.ExecutorListener;
import hudson.model.Queue;
import hudson.plugins.ec2.util.MinimumInstanceChecker;
import hudson.model.Label;
import hudson.slaves.RetentionStrategy;
import jenkins.model.Jenkins;

Expand Down Expand Up @@ -197,7 +198,8 @@ private long internalCheck(EC2Computer computer) {
// TODO: really think about the right strategy here, see
// JENKINS-23792

if (idleMilliseconds > TimeUnit.MINUTES.toMillis(idleTerminationMinutes)) {
if (idleMilliseconds > TimeUnit.MINUTES.toMillis(idleTerminationMinutes) &&
!itemsInQueueForThisSlave(computer)){

LOGGER.info("Idle timeout of " + computer.getName() + " after "
+ TimeUnit.MILLISECONDS.toMinutes(idleMilliseconds) +
Expand All @@ -216,7 +218,7 @@ private long internalCheck(EC2Computer computer) {
// if we have less "free" (aka already paid for) time left than
// our idle time, stop/terminate the instance
// See JENKINS-23821
if (freeSecondsLeft <= TimeUnit.MINUTES.toSeconds(Math.abs(idleTerminationMinutes))) {
if (freeSecondsLeft <= TimeUnit.MINUTES.toSeconds(Math.abs(idleTerminationMinutes)) && !itemsInQueueForThisSlave(computer)) {
LOGGER.info("Idle timeout of " + computer.getName() + " after "
+ TimeUnit.MILLISECONDS.toMinutes(idleMilliseconds) + " idle minutes, with "
+ TimeUnit.SECONDS.toMinutes(freeSecondsLeft)
Expand All @@ -231,6 +233,28 @@ private long internalCheck(EC2Computer computer) {
return 1;
}

/*
* Checks if there are any items in the queue that are waiting for this node explicitly.
* This prevents a node from being taken offline while there are Ivy/Maven Modules waiting to build.
* Need to check entire queue as some modules may be blocked by upstream dependencies.
* Accessing the queue in this way can block other threads, so only perform this check just prior
* to timing out the slave.
*/
private boolean itemsInQueueForThisSlave(EC2Computer c) {
final Label selfLabel = c.getNode().getSelfLabel();
Queue.Item[] items = Jenkins.getInstance().getQueue().getItems();
for (int i = 0; i < items.length; i++) {
Queue.Item item = items[i];
final Label assignedLabel = item.getAssignedLabel();
if (assignedLabel == selfLabel) {
LOGGER.fine("Preventing idle timeout of " + c.getName()
+ " as there is at least one item in the queue explicitly waiting for this slave");
return true;
}
}
return false;
}

/**
* Called when a new {@link EC2Computer} object is introduced (such as when Hudson started, or when
* a new agent is added.)
Expand Down

0 comments on commit c5876d6

Please sign in to comment.