Label based configuration (jenkinsci#160)

Label based configuration

README.md

@@ -19,6 +19,7 @@ automatically scaling the capacity with the load.
   * [Scaling](#scaling)
   * [Groovy](#groovy)
   * [Preconfigure Slave](#preconfigure-slave)
+  * [Label Based Configuration (beta)](docs/LABEL-BASED-CONFIGURATION.md)
 * [Development](#development)
 
 # Overview
@@ -36,6 +37,7 @@ the fleet within the specified price range. For more information, see
 - Allow no delay scale up strategy, enable ```No Delay Provision Strategy``` in configuration
 - Add tags to EC2 instances used by plugin, for easy search, tag format ```ec2-fleet-plugin:cloud-name=<MyCloud>```
 - Allow custom EC2 API endpoint
+- Auto Fleet creation based on Job label [see](docs/LABEL-BASED-CONFIGURATION.md)
 
 # Change Log
 

docs/LABEL-BASED-CONFIGURATION.md

@@ -0,0 +1,94 @@
+[Back to README](../README.md)
+
+# Label Based Configuration
+
+* [Overview](#overview)
+* [How it works](#how-it-works)
+* [Supported Parameters](#supported-parameters)
+* [Configuration](#configuration)
+
+# Overview
+
+Feature in *beta* mode. Please report all problem [here](https://github.com/jenkinsci/ec2-fleet-plugin/issues/new)
+
+This feature auto manages EC2 Spot Fleet or ASG based Fleets for Jenkins based on 
+label attached to Jenkins Jobs.
+
+With this feature user of EC2 Fleet Plugin doesn't need to have pre-created AWS resources 
+to start configuration and run Jobs. Plugin required just AWS Credentials 
+with permissions to be able create resources.
+
+# How It Works
+
+- Plugin detects all labeled Jobs where Label starts from Name configured in plugin configuration ```Cloud Name```
+- Plugin parses Label to get Fleet configuration
+- Plugin creates dedicated fleet for each unique Label
+  - Plugin uses [CloudFormation Stacks](https://aws.amazon.com/cloudformation/) to provision Fleet and all required resources
+- When Label is not used by any Job Plugin deletes Stack and release resources  
+
+Label format
+```
+<CloudName>_parameter1=value1,parameter2=value2
+```
+
+# Supported Parameters
+
+*Note* Parameter name is case insensitive
+
+| Parameter | Value Example | Value |      
+| --- | ---| ---- |
+| imageId | ```ami-0080e4c5bc078760e``` | *Required* AMI ID https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html |
+| max | ```12``` | Fleet Max Size, positive value or zero. If not specified plugin configuration Max will be used |
+| min | ```1``` | Fleet Min Size, positive value or zero. If not specified plugin configuration Min will be used |
+| instanceType | ```c4.large``` | EC2 Instance Type https://aws.amazon.com/ec2/instance-types/. If not specified ```m4.large``` will be used |
+| spotPrice | ```0.4``` | Max Spot Price, if not specified EC2 Spot Fleet API will use default price. |
+
+### Examples
+
+Minimum configuration just Image ID
+```
+<FleetName>_imageId=ami-0080e4c5bc078760e
+```
+
+# Configuration
+
+1. Create AWS User
+1. Add Inline User Permissions
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": [{
+        "Effect": "Allow",
+        "Action": [
+            "cloudformation:*",
+            "ec2:*",
+            "autoscaling:*",
+            "iam:ListRoles",
+            "iam:PassRole",
+            "iam:ListInstanceProfiles",
+            "iam:CreateRole",
+            "iam:AttachRolePolicy",
+            "iam:GetRole"
+        ],
+        "Resource": "*"
+    }]
+}
+```
+1. Goto ```Manage Jenkins > Configure Jenkins```
+1. Add Cloud ```Amazon EC2 Fleet label based```
+1. Specify ```AWS Credentials```
+1. Specify ```SSH Credentials```
+   - Jenkins need to be able to connect to EC2 Instances to run Jobs
+1. Set ```Region```
+1. Provide base configuration
+   - Note ```Cloud Name```
+1. Goto to Jenkins Job which you want to run on this Fleet
+  1. Goto Job ```Configuration```
+  1. Enable ```Restrict where this project can be run```
+  1. Set Label value to ```<Cloud Name>_parameterName=paremeterValue,p2=v2``` 
+  1. Click ```Save```
+
+In some short time plugin will detect Job and will create required resources to be able 
+run it in future.  
+
+That's all, you can repeat this for other Jobs.

src/main/java/com/amazon/jenkins/ec2fleet/AbstractEC2FleetCloud.java

@@ -0,0 +1,21 @@
+package com.amazon.jenkins.ec2fleet;
+
+import hudson.slaves.Cloud;
+
+public abstract class AbstractEC2FleetCloud extends Cloud {
+
+    protected AbstractEC2FleetCloud(String name) {
+        super(name);
+    }
+
+    public abstract boolean isDisableTaskResubmit();
+
+    public abstract int getIdleMinutes();
+
+    public abstract boolean isAlwaysReconnect();
+
+    public abstract boolean scheduleToTerminate(String instanceId);
+
+    public abstract String getOldId();
+
+}

src/main/java/com/amazon/jenkins/ec2fleet/CloudFormationApi.java

@@ -0,0 +1,133 @@
+package com.amazon.jenkins.ec2fleet;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.regions.Region;
+import com.amazonaws.regions.RegionUtils;
+import com.amazonaws.services.cloudformation.AmazonCloudFormation;
+import com.amazonaws.services.cloudformation.AmazonCloudFormationClient;
+import com.amazonaws.services.cloudformation.model.Capability;
+import com.amazonaws.services.cloudformation.model.CreateStackRequest;
+import com.amazonaws.services.cloudformation.model.DeleteStackRequest;
+import com.amazonaws.services.cloudformation.model.DescribeStacksRequest;
+import com.amazonaws.services.cloudformation.model.DescribeStacksResult;
+import com.amazonaws.services.cloudformation.model.Parameter;
+import com.amazonaws.services.cloudformation.model.Stack;
+import com.amazonaws.services.cloudformation.model.StackStatus;
+import com.amazonaws.services.cloudformation.model.Tag;
+import com.cloudbees.jenkins.plugins.awscredentials.AWSCredentialsHelper;
+import com.cloudbees.jenkins.plugins.awscredentials.AmazonWebServicesCredentials;
+import jenkins.model.Jenkins;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+
+import javax.annotation.Nullable;
+import java.util.HashMap;
+import java.util.Map;
+
+public class CloudFormationApi {
+
+    public AmazonCloudFormation connect(final String awsCredentialsId, final String regionName, final String endpoint) {
+        final ClientConfiguration clientConfiguration = AWSUtils.getClientConfiguration();
+        final AmazonWebServicesCredentials credentials = AWSCredentialsHelper.getCredentials(awsCredentialsId, Jenkins.getInstance());
+        final AmazonCloudFormation client =
+                credentials != null ?
+                        new AmazonCloudFormationClient(credentials, clientConfiguration) :
+                        new AmazonCloudFormationClient(clientConfiguration);
+
+        final String effectiveEndpoint = getEndpoint(regionName, endpoint);
+        if (effectiveEndpoint != null) client.setEndpoint(effectiveEndpoint);
+        return client;
+    }
+
+    // todo do we want to merge with EC2Api#getEndpoint
+    @Nullable
+    private String getEndpoint(@Nullable final String regionName, @Nullable final String endpoint) {
+        if (StringUtils.isNotEmpty(endpoint)) {
+            return endpoint;
+        } else if (StringUtils.isNotEmpty(regionName)) {
+            final Region region = RegionUtils.getRegion(regionName);
+            if (region != null && region.isServiceSupported(endpoint)) {
+                return region.getServiceEndpoint(endpoint);
+            } else {
+                final String domain = regionName.startsWith("cn-") ? "amazonaws.com.cn" : "amazonaws.com";
+                return "https://cloudformation." + regionName + "." + domain;
+            }
+        } else {
+            return null;
+        }
+    }
+
+    public void delete(final AmazonCloudFormation client, final String stackId) {
+        client.deleteStack(new DeleteStackRequest().withStackName(stackId));
+    }
+
+    public void create(
+            final AmazonCloudFormation client, final String fleetName, final String keyName, final String parametersString) {
+        final EC2FleetLabelParameters parameters = new EC2FleetLabelParameters(parametersString);
+
+        try {
+            final String type = parameters.getOrDefault("type", "ec2-spot-fleet");
+            final String imageId = parameters.get("imageId"); //"ami-0080e4c5bc078760e";
+            final int maxSize = parameters.getIntOrDefault("maxSize", 10);
+            final int minSize = parameters.getIntOrDefault("minSize", 0);
+            final String instanceType = parameters.getOrDefault("instanceType", "m4.large");
+            final String spotPrice = parameters.get("spotPrice"); // "0.04"
+
+            final String template = "/com/amazon/jenkins/ec2fleet/" + (type.equals("asg") ? "auto-scaling-group.yml" : "ec2-spot-fleet.yml");
+            client.createStack(
+                    new CreateStackRequest()
+                            .withStackName(fleetName + "-" + System.currentTimeMillis())
+                            .withTags(
+                                    new Tag().withKey("ec2-fleet-plugin")
+                                            .withValue(parametersString)
+                            )
+                            .withTemplateBody(IOUtils.toString(CloudFormationApi.class.getResourceAsStream(template)))
+                            // to allow some of templates create iam
+                            .withCapabilities(Capability.CAPABILITY_IAM)
+                            .withParameters(
+                                    new Parameter().withParameterKey("ImageId").withParameterValue(imageId),
+                                    new Parameter().withParameterKey("InstanceType").withParameterValue(instanceType),
+                                    new Parameter().withParameterKey("MaxSize").withParameterValue(Integer.toString(maxSize)),
+                                    new Parameter().withParameterKey("MinSize").withParameterValue(Integer.toString(minSize)),
+                                    new Parameter().withParameterKey("SpotPrice").withParameterValue(spotPrice),
+                                    new Parameter().withParameterKey("KeyName").withParameterValue(keyName)
+                            ));
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static class StackInfo {
+        public final String stackId;
+        public final String fleetId;
+        public final StackStatus stackStatus;
+
+        public StackInfo(String stackId, String fleetId, StackStatus stackStatus) {
+            this.stackId = stackId;
+            this.fleetId = fleetId;
+            this.stackStatus = stackStatus;
+        }
+    }
+
+    public Map<String, StackInfo> describe(
+            final AmazonCloudFormation client, final String fleetName) {
+        Map<String, StackInfo> r = new HashMap<>();
+
+        String nextToken = null;
+        do {
+            DescribeStacksResult describeStacksResult = client.describeStacks(
+                    new DescribeStacksRequest().withNextToken(nextToken));
+            for (Stack stack : describeStacksResult.getStacks()) {
+                if (stack.getStackName().startsWith(fleetName)) {
+                    final String fleetId = stack.getOutputs().isEmpty() ? null : stack.getOutputs().get(0).getOutputValue();
+                    r.put(stack.getTags().get(0).getValue(), new StackInfo(
+                            stack.getStackId(), fleetId, StackStatus.valueOf(stack.getStackStatus())));
+                }
+            }
+            nextToken = describeStacksResult.getNextToken();
+        } while (nextToken != null);
+
+        return r;
+    }
+
+}

src/main/java/com/amazon/jenkins/ec2fleet/EC2Api.java

@@ -23,6 +23,7 @@
 
 import javax.annotation.Nullable;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -125,7 +126,7 @@ private static void describeInstancesBatch(
      * @param ec2 ec2 client
      * @param instanceIds set of instance ids
      */
-    public void terminateInstances(final AmazonEC2 ec2, final Set<String> instanceIds) {
+    public void terminateInstances(final AmazonEC2 ec2, final Collection<String> instanceIds) {
         final List<String> temp = new ArrayList<>(instanceIds);
         while (temp.size() > 0) {
             // terminateInstances is idempotent so it can be called until it's successful

src/main/java/com/amazon/jenkins/ec2fleet/EC2FleetAutoResubmitComputerLauncher.java

@@ -78,7 +78,7 @@ public void afterDisconnect(final SlaveComputer computer, final TaskListener lis
         if (computer == null) return;
 
         // in some multi-thread edge cases cloud could be null for some time, just be ok with that
-        final EC2FleetCloud cloud = ((EC2FleetNodeComputer) computer).getCloud();
+        final AbstractEC2FleetCloud cloud = ((EC2FleetNodeComputer) computer).getCloud();
         if (cloud == null) {
             LOGGER.warning("Edge case cloud is null for computer " + computer.getDisplayName()
                     + " should be autofixed in a few minutes, if no please create issue for plugin");

src/main/java/com/amazon/jenkins/ec2fleet/EC2FleetCloud.java

@@ -52,7 +52,7 @@
  * @see CloudNanny
  */
 @SuppressWarnings({"unused", "WeakerAccess"})
-public class EC2FleetCloud extends Cloud {
+public class EC2FleetCloud extends AbstractEC2FleetCloud {
 
     public static final String EC2_INSTANCE_TAG_NAMESPACE = "ec2-fleet-plugin";
     public static final String EC2_INSTANCE_CLOUD_NAME_TAG = EC2_INSTANCE_TAG_NAMESPACE + ":cloud-name";
@@ -692,6 +692,7 @@ private void addNewSlave(final AmazonEC2 ec2, final Instance instance, FleetStat
         // jenkins automatically remove old node with same name if any
         jenkins.addNode(node);
 
+        // todo use plannedNodesCache in thread-safe way
         final SettableFuture<Node> future;
         if (plannedNodesCache.isEmpty()) {
             future = SettableFuture.create();

src/main/java/com/amazon/jenkins/ec2fleet/EC2FleetCloudAware.java

@@ -16,8 +16,8 @@
  */
 public interface EC2FleetCloudAware {
 
-    EC2FleetCloud getCloud();
+    AbstractEC2FleetCloud getCloud();
 
-    void setCloud(@Nonnull EC2FleetCloud cloud);
+    void setCloud(@Nonnull AbstractEC2FleetCloud cloud);
 
 }

src/main/java/com/amazon/jenkins/ec2fleet/EC2FleetCloudAwareUtils.java

@@ -15,7 +15,7 @@ public class EC2FleetCloudAwareUtils {
 
     private static final Logger LOGGER = Logger.getLogger(EC2FleetCloudAwareUtils.class.getName());
 
-    public static void reassign(final @Nonnull String oldId, @Nonnull final EC2FleetCloud cloud) {
+    public static void reassign(final @Nonnull String oldId, @Nonnull final AbstractEC2FleetCloud cloud) {
         for (final Computer computer : Jenkins.getActiveInstance().getComputers()) {
             checkAndReassign(oldId, cloud, computer);
         }
@@ -27,10 +27,10 @@ public static void reassign(final @Nonnull String oldId, @Nonnull final EC2Fleet
         LOGGER.info("Finish to reassign resources from old cloud with id " + oldId + " to " + cloud.getDisplayName());
     }
 
-    private static void checkAndReassign(final String oldId, final EC2FleetCloud cloud, final Object object) {
+    private static void checkAndReassign(final String oldId, final AbstractEC2FleetCloud cloud, final Object object) {
         if (object instanceof EC2FleetCloudAware) {
             final EC2FleetCloudAware cloudAware = (EC2FleetCloudAware) object;
-            final EC2FleetCloud oldCloud = cloudAware.getCloud();
+            final AbstractEC2FleetCloud oldCloud = cloudAware.getCloud();
             if (oldCloud != null && oldId.equals(oldCloud.getOldId())) {
                 ((EC2FleetCloudAware) object).setCloud(cloud);
                 LOGGER.info("Reassign " + object + " from " + oldCloud.getDisplayName() + " to " + cloud.getDisplayName());