From c5876d694e52d8c5a8da2ae6c3912021f7951068 Mon Sep 17 00:00:00 2001 From: Timothy Bingaman Date: Tue, 5 Jul 2016 13:22:06 -0400 Subject: [PATCH] Experimental change to prevent offlining of slave Check to see if any items in the queue (including blocked items) ask for this node explicitly. This is the case for individual Ivy/Maven Module builds which must build on the same node as their parent. If there are any then don't offline this node. --- .../plugins/ec2/EC2RetentionStrategy.java | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/main/java/hudson/plugins/ec2/EC2RetentionStrategy.java b/src/main/java/hudson/plugins/ec2/EC2RetentionStrategy.java index cb25d70ad..1db85ca7e 100644 --- a/src/main/java/hudson/plugins/ec2/EC2RetentionStrategy.java +++ b/src/main/java/hudson/plugins/ec2/EC2RetentionStrategy.java @@ -32,6 +32,7 @@ import hudson.model.ExecutorListener; import hudson.model.Queue; import hudson.plugins.ec2.util.MinimumInstanceChecker; +import hudson.model.Label; import hudson.slaves.RetentionStrategy; import jenkins.model.Jenkins; @@ -197,7 +198,8 @@ private long internalCheck(EC2Computer computer) { // TODO: really think about the right strategy here, see // JENKINS-23792 - if (idleMilliseconds > TimeUnit.MINUTES.toMillis(idleTerminationMinutes)) { + if (idleMilliseconds > TimeUnit.MINUTES.toMillis(idleTerminationMinutes) && + !itemsInQueueForThisSlave(computer)){ LOGGER.info("Idle timeout of " + computer.getName() + " after " + TimeUnit.MILLISECONDS.toMinutes(idleMilliseconds) + @@ -216,7 +218,7 @@ private long internalCheck(EC2Computer computer) { // if we have less "free" (aka already paid for) time left than // our idle time, stop/terminate the instance // See JENKINS-23821 - if (freeSecondsLeft <= TimeUnit.MINUTES.toSeconds(Math.abs(idleTerminationMinutes))) { + if (freeSecondsLeft <= TimeUnit.MINUTES.toSeconds(Math.abs(idleTerminationMinutes)) && !itemsInQueueForThisSlave(computer)) { LOGGER.info("Idle timeout of " + computer.getName() + " after " + TimeUnit.MILLISECONDS.toMinutes(idleMilliseconds) + " idle minutes, with " + TimeUnit.SECONDS.toMinutes(freeSecondsLeft) @@ -231,6 +233,28 @@ private long internalCheck(EC2Computer computer) { return 1; } + /* + * Checks if there are any items in the queue that are waiting for this node explicitly. + * This prevents a node from being taken offline while there are Ivy/Maven Modules waiting to build. + * Need to check entire queue as some modules may be blocked by upstream dependencies. + * Accessing the queue in this way can block other threads, so only perform this check just prior + * to timing out the slave. + */ + private boolean itemsInQueueForThisSlave(EC2Computer c) { + final Label selfLabel = c.getNode().getSelfLabel(); + Queue.Item[] items = Jenkins.getInstance().getQueue().getItems(); + for (int i = 0; i < items.length; i++) { + Queue.Item item = items[i]; + final Label assignedLabel = item.getAssignedLabel(); + if (assignedLabel == selfLabel) { + LOGGER.fine("Preventing idle timeout of " + c.getName() + + " as there is at least one item in the queue explicitly waiting for this slave"); + return true; + } + } + return false; + } + /** * Called when a new {@link EC2Computer} object is introduced (such as when Hudson started, or when * a new agent is added.)