sam.yml

---

AWSTemplateFormatVersion: 2010-09-09
Description: Step Functions for Spoptimize
Transform: AWS::Serverless-2016-10-31

Parameters:
  StackBasename:
    Description: Base name of Spoptimize resources
    Type: String
    Default: spoptimize
  RolePath:
    Description: Path of IAM resources
    Type: String
    Default: /
  SnsTopicNameOverride:
    Description: Name of existing SNS topic that publishes autoscaling launch notifications
    Type: String
    Default: default
  DebugLambdas:
    Description: Enable debug logging of lambda functions
    Type: String
    Default: "false"
    AllowedValues: ["false", "true"]
  MaximumIterationCount:
    Description: Maximum number of iterations
    Type: Number
    Default: 48
  AlarmTopicName:
    Description: Name of SNS topic for CloudWatch Alarms
    Type: String
    Default: ""
  IamTemplateUrl:
    Description: Base URL of IAM template in S3 (replace token with URL to deploy all resources)
    Type: String
    Default: ''  # URL_TOKEN

Metadata:
  AWS::CloudFormation::Interface:
    ParameterGroups:
      -
        Label:
          default: Basic Configuration
        Parameters:
          - StackBasename
          - AlarmTopicName
          - DebugLambdas
      -
        Label:
          default: Advanced Configuration
        Parameters:
          - SnsTopicNameOverride
          - RolePath
          - MaximumIterationCount
          - IamTemplateUrl
    ParameterLabels:
      StackBaseName:
        default: Name prefix for spoptimize resources
      AlarmTopicName:
        default: Topic Name for Alarms
      SnsTopicNameOverride:
        default: Override for Launch Notification Topic
      DebugLambdas:
        default: Debug Lambdas?
      RolePath:
        default: Path override for IAM resources
      MaximumIterationCount:
        default: Max iterations after failed spot requests
      IamTemplateUrl:
        default: Humans probably shouldn't change this

Conditions:
  NoAlarmNotifications: !Equals [!Ref AlarmTopicName, '']
  DefaultSnsTopic: !Equals [!Ref SnsTopicNameOverride, 'default']
  CreateIamStack: !Not [!Equals [!Ref IamTemplateUrl, '']]
  CreateLaunchTopic: !And [
    !Equals [!Ref SnsTopicNameOverride, 'default'],
    !Not [!Equals [!Ref IamTemplateUrl, '']]
  ]

Globals:
  Function:
    Runtime: python2.7
    Timeout: 30
    Handler: handler.handler
    MemorySize: 1024
    Environment:
      Variables:
        SPOPTIMIZE_DEBUG: !Ref DebugLambdas
        SPOPTIMIZE_LOCK_TABLE: !Ref LockTable
        SPOPTIMIZE_SFN_ARN: !Ref SpotRequestor

Resources:

  LaunchTopic:
    Type: AWS::SNS::Topic
    Condition: CreateLaunchTopic
    Properties:
      DisplayName: !Sub "autoscaling launch notifications - ${AWS::StackName}"
      TopicName: !Sub "${StackBasename}-init"

  Iam:
    Type: AWS::CloudFormation::Stack
    Condition: CreateIamStack
    Properties:
      TemplateURL: !Ref IamTemplateUrl
      Parameters:
        StackBasename: !Ref StackBasename
        RolePath: !Ref RolePath

  LockTable:
    Type: AWS::DynamoDB::Table
    Properties:
      TableName: !Sub "${StackBasename}-autoscaling-group-locks"
      AttributeDefinitions:
        - AttributeName: group_name
          AttributeType: S
      KeySchema:
        - AttributeName: group_name
          KeyType: HASH
      ProvisionedThroughput:
        ReadCapacityUnits: 5
        WriteCapacityUnits: 5
      TimeToLiveSpecification:
        AttributeName: ttl
        Enabled: true

  StartStateMachineFn:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: !Sub "${StackBasename}-start-state-machine"
      Description: Processes autoscaling launch notifications via SNS and starts Spoptimize Step Functions
      Role: !If [
        CreateIamStack,
        !GetAtt [Iam, Outputs.LambdaRoleArn],
        !Sub "arn:aws:iam::${AWS::AccountId}:role${RolePath}${StackBasename}-iam-global-lambda-role"
      ]
      CodeUri: ./target/lambda-pkg.zip
      Environment:
        Variables:
          SPOPTIMIZE_ACTION: 'start-state-machine'
      Events:
        AsgLaunchNotifications:
          Type: SNS
          Properties:
            Topic: !If [
              DefaultSnsTopic,
              !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:${StackBasename}-init",
              !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:${SnsTopicNameOverride}"
            ]

  SpotWarningFn:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: !Sub "${StackBasename}-spot-warning"
      Description: Processes EC2 spot instance warnings and terminates via autoscaling API
      Role: !If [
        CreateIamStack,
        !GetAtt [Iam, Outputs.LambdaRoleArn],
        !Sub "arn:aws:iam::${AWS::AccountId}:role${RolePath}${StackBasename}-iam-global-lambda-role"
      ]
      CodeUri: ./target/lambda-pkg.zip
      Handler: handler.spot_warning_handler
      Events:
        SpotWarningForSpoptimize:
          Type: CloudWatchEvent
          Properties:
            Pattern:
              source: ["aws.ec2"]
              detail-type: ["EC2 Spot Instance Interruption Warning"]

  TestNewAsgInstanceFn:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: !Sub "${StackBasename}-ondemand-instance-healthy"
      Description: Checks health and status of launched autoscaling instance
      Role: !If [
        CreateIamStack,
        !GetAtt [Iam, Outputs.LambdaRoleArn],
        !Sub "arn:aws:iam::${AWS::AccountId}:role${RolePath}${StackBasename}-iam-global-lambda-role"
      ]
      CodeUri: ./target/lambda-pkg.zip
      Environment:
        Variables:
          SPOPTIMIZE_ACTION: 'ondemand-instance-healthy'

  IncrementCountFn:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: !Sub "${StackBasename}-increment-count"
      Description: Increment iteration counter
      Role: !If [
        CreateIamStack,
        !GetAtt [Iam, Outputs.LambdaRoleArn],
        !Sub "arn:aws:iam::${AWS::AccountId}:role${RolePath}${StackBasename}-iam-global-lambda-role"
      ]
      CodeUri: ./target/lambda-pkg.zip
      Environment:
        Variables:
          SPOPTIMIZE_ACTION: 'increment-count'

  RequestSpotInstanceFn:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: !Sub "${StackBasename}-request-spot"
      Description: Requests a spot instance to replace launched autoscaling instance
      Role: !If [
        CreateIamStack,
        !GetAtt [Iam, Outputs.LambdaRoleArn],
        !Sub "arn:aws:iam::${AWS::AccountId}:role${RolePath}${StackBasename}-iam-global-lambda-role"
      ]
      CodeUri: ./target/lambda-pkg.zip
      Environment:
        Variables:
          SPOPTIMIZE_ACTION: 'request-spot'

  CheckSpotRequestFn:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: !Sub "${StackBasename}-check-spot"
      Description: Checks the status of spot instance request
      Role: !If [
        CreateIamStack,
        !GetAtt [Iam, Outputs.LambdaRoleArn],
        !Sub "arn:aws:iam::${AWS::AccountId}:role${RolePath}${StackBasename}-iam-global-lambda-role"
      ]
      CodeUri: ./target/lambda-pkg.zip
      Environment:
        Variables:
          SPOPTIMIZE_ACTION: 'check-spot'

  AutoScalingGroupDisappearedFn:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: !Sub "${StackBasename}-term-spot-instance"
      Description: Terminates spot instance (if online) after autoscaling group disappears
      Role: !If [
        CreateIamStack,
        !GetAtt [Iam, Outputs.LambdaRoleArn],
        !Sub "arn:aws:iam::${AWS::AccountId}:role${RolePath}${StackBasename}-iam-global-lambda-role"
      ]
      CodeUri: ./target/lambda-pkg.zip
      Environment:
        Variables:
          SPOPTIMIZE_ACTION: 'term-spot-instance'

  AcquireAutoScalingGroupLock:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: !Sub "${StackBasename}-acquire-lock"
      Description: Acquires lock for autoscaling group
      Role: !If [
        CreateIamStack,
        !GetAtt [Iam, Outputs.LambdaRoleArn],
        !Sub "arn:aws:iam::${AWS::AccountId}:role${RolePath}${StackBasename}-iam-global-lambda-role"
      ]
      CodeUri: ./target/lambda-pkg.zip
      Environment:
        Variables:
          SPOPTIMIZE_ACTION: 'acquire-lock'

  ReleaseAutoScalingGroupLock:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: !Sub "${StackBasename}-release-lock"
      Description: Releases lock for autoscaling group
      Role: !If [
        CreateIamStack,
        !GetAtt [Iam, Outputs.LambdaRoleArn],
        !Sub "arn:aws:iam::${AWS::AccountId}:role${RolePath}${StackBasename}-iam-global-lambda-role"
      ]
      CodeUri: ./target/lambda-pkg.zip
      Environment:
        Variables:
          SPOPTIMIZE_ACTION: 'release-lock'

  AttachSpot:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: !Sub "${StackBasename}-attach-spot"
      Description: Attaches spot instance to autoscaling group
      Role: !If [
        CreateIamStack,
        !GetAtt [Iam, Outputs.LambdaRoleArn],
        !Sub "arn:aws:iam::${AWS::AccountId}:role${RolePath}${StackBasename}-iam-global-lambda-role"
      ]
      CodeUri: ./target/lambda-pkg.zip
      Environment:
        Variables:
          SPOPTIMIZE_ACTION: 'attach-spot'

  TestAttachedInstance:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: !Sub "${StackBasename}-spot-instance-healthy"
      Description: Checks health and status of attached spot instance
      Role: !If [
        CreateIamStack,
        !GetAtt [Iam, Outputs.LambdaRoleArn],
        !Sub "arn:aws:iam::${AWS::AccountId}:role${RolePath}${StackBasename}-iam-global-lambda-role"
      ]
      CodeUri: ./target/lambda-pkg.zip
      Environment:
        Variables:
          SPOPTIMIZE_ACTION: 'spot-instance-healthy'

  SpotRequestor:
    Type: AWS::StepFunctions::StateMachine
    Properties:
      StateMachineName: !Sub "${StackBasename}-spot-requestor"
      RoleArn: !If [
        CreateIamStack,
        !GetAtt [Iam, Outputs.StateMachineRoleArn],
        !Sub "arn:aws:iam::${AWS::AccountId}:role${RolePath}${StackBasename}-iam-global-state-machine"
      ]
      DefinitionString:
        Fn::Sub: |-
          {
            "Comment": "Spoptimize State Machine",
            "StartAt": "Wait for New ASG Instance",
            "States": {
              "Wait for New ASG Instance": {
                "Type": "Wait",
                "SecondsPath": "$.init_sleep_interval",
                "Next": "Test New ASG Instance"
              },
              "Test New ASG Instance": {
                "Type": "Task",
                "Resource": "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${StackBasename}-ondemand-instance-healthy",
                "Next": "OD Instance Healthy?",
                "ResultPath": "$.ondemand_instance_status",
                "Retry": [{
                  "ErrorEquals": [ "InstancePending" ],
                  "IntervalSeconds": 30,
                  "MaxAttempts": 30,
                  "BackoffRate": 1
                },{
                  "ErrorEquals": [ "States.ALL" ],
                  "IntervalSeconds": 5,
                  "MaxAttempts": 5,
                  "BackoffRate": 2.5
                }]
              },
              "OD Instance Healthy?": {
                "Type": "Choice",
                "Choices": [{
                  "Variable": "$.ondemand_instance_status",
                  "StringEquals": "Healthy",
                  "Next": "Request Spot Instance"
                },{
                  "Variable": "$.ondemand_instance_status",
                  "StringEquals": "Terminated",
                  "Next": "Instance Protected/Detached/Terminated or ASG Disappeared"
                },{
                  "Variable": "$.ondemand_instance_status",
                  "StringEquals": "Protected",
                  "Next": "Instance Protected/Detached/Terminated or ASG Disappeared"
                },{
                  "Variable": "$.ondemand_instance_status",
                  "StringEquals": "AutoScaling Group Disappeared",
                  "Next": "Instance Protected/Detached/Terminated or ASG Disappeared"
                }],
                "Default": "Unrecoverable OD Instance Failure"
              },
              "Unrecoverable OD Instance Failure": {
                "Type": "Fail"
              },
              "Instance Protected/Detached/Terminated or ASG Disappeared": {
                "Type": "Succeed"
              },
              "Request Spot Instance": {
                "Type": "Task",
                "Resource": "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${StackBasename}-request-spot",
                "ResultPath": "$.spot_request",
                "Next": "Wait For Spot Request",
                "Retry": [{
                  "ErrorEquals": [ "States.ALL" ],
                  "IntervalSeconds": 5,
                  "MaxAttempts": 5,
                  "BackoffRate": 2.5
                }]
              },
              "Wait For Spot Request": {
                "Type": "Wait",
                "SecondsPath": "$.spot_req_sleep_interval",
                "Next": "Check Spot Request"
              },
              "Check Spot Request": {
                "Type": "Task",
                "Resource": "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${StackBasename}-check-spot",
                "Next": "Spot Request Status?",
                "ResultPath": "$.spot_request_result",
                "Retry": [{
                  "ErrorEquals": [ "States.ALL" ],
                  "IntervalSeconds": 5,
                  "MaxAttempts": 5,
                  "BackoffRate": 2.5
                }]
              },
              "Spot Request Status?": {
                "Type": "Choice",
                "Choices": [{
                  "Variable": "$.spot_request_result",
                  "StringEquals": "Pending",
                  "Next": "Wait For Spot Request"
                },{
                  "Variable": "$.spot_request_result",
                  "StringEquals": "Failure",
                  "Next": "Increment Failure Count"
                }],
                "Default": "Acquire AutoScaling Group Lock"
              },
              "Increment Failure Count": {
                "Type": "Task",
                "Resource": "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${StackBasename}-increment-count",
                "Next": "Check Iteration Count?",
                "ResultPath": "$.iteration_count",
                "Retry": [{
                  "ErrorEquals": [ "States.ALL" ],
                  "IntervalSeconds": 1,
                  "MaxAttempts": 5
                }]
              },
              "Check Iteration Count?": {
                "Type": "Choice",
                "Choices": [{
                  "Variable": "$.iteration_count",
                  "NumericLessThanEquals": ${MaximumIterationCount},
                  "Next": "Sleep after Failed Spot Request"
                }],
                "Default": "Execution Exhaustion"
              },
              "Execution Exhaustion": {
                "Type": "Fail"
              },
              "Sleep after Failed Spot Request": {
                "Type": "Wait",
                "SecondsPath": "$.spot_failure_sleep_interval",
                "Next": "Test New ASG Instance"
              },
              "Release Lock Before Spot Term": {
                "Type": "Task",
                "Resource": "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${StackBasename}-release-lock",
                "ResultPath": "$.asg_lock",
                "Next": "Terminate Spot",
                "Retry": [{
                  "ErrorEquals": [ "States.ALL" ],
                  "IntervalSeconds": 5,
                  "MaxAttempts": 5,
                  "BackoffRate": 2.5
                }]
              },
              "Terminate Spot": {
                "Type": "Task",
                "Resource": "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${StackBasename}-term-spot-instance",
                "End": true,
                "Retry": [{
                  "ErrorEquals": [ "States.ALL" ],
                  "IntervalSeconds": 5,
                  "MaxAttempts": 5,
                  "BackoffRate": 2.5
                }]
              },
              "Acquire AutoScaling Group Lock": {
                "Type": "Task",
                "Resource": "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${StackBasename}-acquire-lock",
                "ResultPath": "$.asg_lock",
                "Next": "Attach Spot Instance",
                "Retry": [{
                  "ErrorEquals": [ "GroupLocked" ],
                  "IntervalSeconds": 5,
                  "MaxAttempts": 20,
                  "BackoffRate": 1.5
                },{
                  "ErrorEquals": [ "States.ALL" ],
                  "IntervalSeconds": 5,
                  "MaxAttempts": 5,
                  "BackoffRate": 2.5
                }]
              },
              "Attach Spot Instance": {
                "Type": "Task",
                "Resource": "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${StackBasename}-attach-spot",
                "ResultPath": "$.spot_attach_result",
                "Next": "Check Attachment?",
                "Retry": [{
                  "ErrorEquals": [ "States.ALL" ],
                  "IntervalSeconds": 5,
                  "MaxAttempts": 5,
                  "BackoffRate": 2.5
                }]
              },
              "Check Attachment?": {
                "Type": "Choice",
                "Choices": [{
                  "Variable": "$.spot_attach_result",
                  "StringEquals": "AutoScaling Group Disappeared",
                  "Next": "Release Lock Before Spot Term"
                },{
                  "Variable": "$.spot_attach_result",
                  "StringEquals": "OD Instance Disappeared Or Protected",
                  "Next": "Release Lock Before Spot Term"
                },{
                  "Variable": "$.spot_attach_result",
                  "StringEquals": "AutoScaling group not sized correctly",
                  "Next": "Release Lock Before Spot Term"
                },{
                  "Variable": "$.spot_attach_result",
                  "StringEquals": "Spot Instance Disappeared",
                  "Next": "Release Lock Before Increment"
                },{
                  "Variable": "$.spot_attach_result",
                  "StringEquals": "Instance missing",
                  "Next": "Release Lock Before Increment"
                },{
                  "Variable": "$.spot_attach_result",
                  "StringEquals": "Invalid instance",
                  "Next": "Release Lock Before Increment"
                }],
                "Default": "Wait for Attachment"
              },
              "Release Lock Before Increment": {
                "Type": "Task",
                "Resource": "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${StackBasename}-release-lock",
                "ResultPath": "$.asg_lock",
                "Next": "Increment Failure Count",
                "Retry": [{
                  "ErrorEquals": [ "States.ALL" ],
                  "IntervalSeconds": 5,
                  "MaxAttempts": 5,
                  "BackoffRate": 2.5
                }]
              },
              "Wait for Attachment": {
                "Type": "Wait",
                "SecondsPath": "$.spot_attach_sleep_interval",
                "Next": "Test Attached Instance"
              },
              "Test Attached Instance": {
                "Type": "Task",
                "Resource": "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${StackBasename}-spot-instance-healthy",
                "Next": "Spot Instance Healthy?",
                "ResultPath": "$.spot_instance_status",
                "Retry": [{
                  "ErrorEquals": [ "InstancePending" ],
                  "IntervalSeconds": 30,
                  "MaxAttempts": 30,
                  "BackoffRate": 1
                },{
                  "ErrorEquals": [ "States.ALL" ],
                  "IntervalSeconds": 5,
                  "MaxAttempts": 5,
                  "BackoffRate": 2.5
                }]
              },
              "Spot Instance Healthy?": {
                "Type": "Choice",
                "Choices": [{
                  "Variable": "$.spot_instance_status",
                  "StringEquals": "Healthy",
                  "Next": "Release Lock after Success"
                },{
                  "Variable": "$.spot_instance_status",
                  "StringEquals": "AutoScaling Group Disappeared",
                  "Next": "Release Lock Before Spot Term"
                }],
                "Default": "Release Lock After Spot Failure"
              },
              "Release Lock After Spot Failure": {
                "Type": "Task",
                "Resource": "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${StackBasename}-release-lock",
                "ResultPath": "$.asg_lock",
                "Next": "Unrecoverable Spot Instance Failure",
                "Retry": [{
                  "ErrorEquals": [ "States.ALL" ],
                  "IntervalSeconds": 5,
                  "MaxAttempts": 5,
                  "BackoffRate": 2.5
                }]
              },
              "Unrecoverable Spot Instance Failure": {
                "Type": "Fail"
              },
              "Release Lock after Success": {
                "Type": "Task",
                "Resource": "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${StackBasename}-release-lock",
                "ResultPath": "$.asg_lock",
                "End": true,
                "Retry": [{
                  "ErrorEquals": [ "States.ALL" ],
                  "IntervalSeconds": 5,
                  "MaxAttempts": 5,
                  "BackoffRate": 2.5
                }]
              }
            }
          }

  SpotRequestorFailedAlrm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      AlarmName: !Sub "${AWS::StackName}-failed-executions"
      AlarmDescription: "Failed executions of Spoptimize spot-requestor step functions"
      Namespace: AWS/States
      MetricName: ExecutionsFailed
      Dimensions:
        - Name: StateMachineArn
          Value: !Ref SpotRequestor
      Period: 300
      EvaluationPeriods: 1
      Statistic: Sum
      ComparisonOperator: GreaterThanThreshold
      Threshold: 0
      TreatMissingData: notBreaching
      ActionsEnabled: !If [NoAlarmNotifications, false, true]
      AlarmActions:
        Fn::If:
          - NoAlarmNotifications
          - !Ref AWS::NoValue
          - [!Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:${AlarmTopicName}"]

  SpotRequestorTimedOutAlrm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      AlarmName: !Sub "${AWS::StackName}-timed-out-executions"
      AlarmDescription: "Timed Out executions of Spoptimize spot-requestor step functions"
      Namespace: AWS/States
      MetricName: ExecutionsTimedOut
      Dimensions:
        - Name: StateMachineArn
          Value: !Ref SpotRequestor
      Period: 300
      EvaluationPeriods: 1
      Statistic: Sum
      ComparisonOperator: GreaterThanThreshold
      Threshold: 0
      TreatMissingData: notBreaching
      ActionsEnabled: !If [NoAlarmNotifications, false, true]
      AlarmActions:
        Fn::If:
          - NoAlarmNotifications
          - !Ref AWS::NoValue
          - [!Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:${AlarmTopicName}"]

Outputs:
  SpotRequestorArn:
    Description: Arn of Spot Requestor State Machine
    Value: !Ref SpotRequestor
  SpotRequestorName:
    Description: Name of Spot Requestor State Machine
    Value: !GetAtt SpotRequestor.Name