Auto Scaling (#72)

docs/CONFIG-VARS.md

@@ -188,12 +188,13 @@ Custom Policy:
 | <div style="width:50px">Name</div> | <div style="width:150px">Description</div> | <div style="width:50px">Type</div> | <div style="width:75px">Default</div> | <div style="width:150px">Notes</div> |
 | :--- | :--- | :--- | :--- | :--- |
 | create_static_kubeconfig | Allows the user to create a provider / service account based kube config file | bool | false | A value of `false` will default to using the cloud providers mechanism for generating the kubeconfig file. A value of `true` will create a static kubeconfig which utilizes a `Service Account` and `Cluster Role Binding` to provide credentials. |
-| kubernetes_version | The EKS cluster K8S version | string | "1.18" | |
+| kubernetes_version | The EKS cluster K8S version | string | "1.19" | |
 | create_jump_vm | Create bastion host | bool | true| |
 | create_jump_public_ip | Add public ip to jump VM | bool | true | |
 | jump_vm_admin | OS Admin User for the Jump VM | string | "jumpuser" | |
 | jump_rwx_filestore_path | File store mount point on Jump server | string | "/viya-share" | This location cannot include "/mnt" as it's root location. This disk is ephemeral on Ubuntu which is the operating system being used for the Jump/NFS servers. |
 | tags | Map of common tags to be placed on all AWS resources created by this script | map | { project_name = "viya" } | |
+| autoscaling_enabled | Enable Cluster Autoscaling | bool | true | |
 
 ## Nodepools
 

files/policies/devops-iac-eks-policy.json

@@ -139,6 +139,7 @@
                 "iam:UntagRole",
                 "iam:TagPolicy",
                 "iam:TagInstanceProfile",
+                "iam:TagOpenIDConnectProvider",
                 "iam:UpdateAssumeRolePolicy",
                 "iam:UpdateAccessKey",
                 "resource-groups:*",

main.tf

@@ -354,14 +354,15 @@ module "eks" {
   subnets                               = module.vpc.private_subnets
   vpc_id                                = module.vpc.vpc_id
   tags                                  = var.tags
+  enable_irsa                           = var.autoscaling_enabled
 
   manage_worker_iam_resources           = var.workers_iam_role_name == null ? true : false
   workers_role_name                     = var.workers_iam_role_name
   manage_cluster_iam_resources          = var.cluster_iam_role_name == null ? true : false
   cluster_iam_role_name                 = var.cluster_iam_role_name
 
   workers_group_defaults = {
-    # tags = var.tags
+    tags = [ { key = "k8s.io/cluster-autoscaler/${local.cluster_name}", value = "owned", propagate_at_launch = true }, { key = "k8s.io/cluster-autoscaler/enabled", value = "true", propagate_at_launch = true} ]
     additional_security_group_ids        = [local.security_group_id]
     metadata_http_tokens                 = "required"
     metadata_http_put_response_hop_limit = 1
@@ -374,6 +375,16 @@ module "eks" {
   worker_groups = local.worker_groups
 }
 
+module "autoscaling" {
+  source       = "./modules/aws_autoscaling"
+  count        = var.autoscaling_enabled ? 1 : 0
+
+  prefix       = var.prefix
+  cluster_name = local.cluster_name
+  tags         = var.tags
+  oidc_url     = module.eks.cluster_oidc_issuer_url
+}
+
 module "kubeconfig" {
   source                   = "./modules/kubeconfig"
   prefix                   = var.prefix

modules/aws_autoscaling/main.tf

@@ -0,0 +1,65 @@
+data "aws_iam_policy_document" "worker_autoscaling" {
+  statement {
+    sid    = "eksWorkerAutoscalingAll"
+    effect = "Allow"
+
+    actions = [
+      "autoscaling:DescribeAutoScalingGroups",
+      "autoscaling:DescribeAutoScalingInstances",
+      "autoscaling:DescribeLaunchConfigurations",
+      "autoscaling:DescribeTags",
+      "ec2:DescribeLaunchTemplateVersions",
+    ]
+
+    resources = ["*"]
+  }
+
+  statement {
+    sid    = "eksWorkerAutoscalingOwn"
+    effect = "Allow"
+
+    actions = [
+      "autoscaling:SetDesiredCapacity",
+      "autoscaling:TerminateInstanceInAutoScalingGroup",
+      "autoscaling:UpdateAutoScalingGroup",
+    ]
+
+    resources = ["*"]
+
+    condition {
+      test     = "StringEquals"
+      variable = "autoscaling:ResourceTag/kubernetes.io/cluster/${var.cluster_name}"
+      values   = ["owned"]
+    }
+
+    condition {
+      test     = "StringEquals"
+      variable = "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/enabled"
+      values   = ["true"]
+    }
+  }
+}
+
+resource "aws_iam_policy" "worker_autoscaling" {
+  name_prefix = "${var.prefix}-eks-worker-autoscaling"
+  description = "EKS worker node autoscaling policy for cluster ${var.cluster_name}"
+  policy      = data.aws_iam_policy_document.worker_autoscaling.json
+  tags        = var.tags
+}
+
+module "iam_assumable_role_with_oidc" {
+  source  = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc"
+  version = "4.1.0"
+
+  create_role = true
+  role_name = "${var.prefix}-cluster-autoscaler"
+  provider_url = replace(var.oidc_url, "https://", "")
+  role_policy_arns = [aws_iam_policy.worker_autoscaling.arn]
+  oidc_fully_qualified_subjects = ["system:serviceaccount:kube-system:cluster-autoscaler"]
+
+  tags = {
+    Role = "${var.prefix}-cluster-autoscaler"
+  }
+
+}
+

modules/aws_autoscaling/output.tf

@@ -0,0 +1,3 @@
+output "autoscaler_account" {
+    value = module.iam_assumable_role_with_oidc.iam_role_arn
+}

modules/aws_autoscaling/variables.tf

@@ -0,0 +1,22 @@
+variable "prefix" {
+  description = "A prefix used for all AWS Cloud resources created by this script"
+  type        = string
+  default     = null
+}
+
+variable "cluster_name" {
+  description = "Name of EKS cluster"
+  type        = string
+  default     = null
+}
+
+variable "tags" {
+  description = "Tags used for autoscaling"
+  default     = null
+}
+
+variable "oidc_url" {
+  description = "OIDC URL of EKS cluster"
+  type        = string
+  default     = null
+}

outputs.tf

@@ -121,3 +121,7 @@ output "cr_endpoint" {
 output "cluster_node_pool_mode" {
   value = var.cluster_node_pool_mode
 }
+
+output "autoscaler_account" {
+    value = var.autoscaling_enabled ? module.autoscaling.0.autoscaler_account : null
+}

variables.tf

@@ -493,4 +493,11 @@ variable "cluster_node_pool_mode" {
   description = "Flag for predefined cluster node configurations - Values : default, minimal"
   type        = string
   default     = "default"
+
+}
+
+variable "autoscaling_enabled" {
+    description = "Enable autoscaling for your AWS cluster."
+    type        = bool
+    default     = true
 }