diff --git a/README.md b/README.md
index fd786d5..88a671d 100644
--- a/README.md
+++ b/README.md
@@ -16,21 +16,22 @@ For more details see: https://github.com/GlueOps/terraform-module-cloud-aws-kube
```hcl
module "captain" {
iam_role_to_assume = "arn:aws:iam::1234567890:role/glueops-captain"
- source = "git::https://github.com/GlueOps/terraform-module-cloud-aws-kubernetes-cluster.git"
+ source = "git::https://github.com/GlueOps/terraform-module-cloud-aws-kubernetes-cluster.git?ref=feat/multiple-node-pools"
eks_version = "1.26"
- csi_driver_version = "v1.17.0-eksbuild.1"
- vpc_cidr_block = "10.65.0.0/16"
+ csi_driver_version = "v1.18.0-eksbuild.1"
+ vpc_cidr_block = "10.65.0.0/26"
region = "us-west-2"
availability_zones = ["us-west-2a", "us-west-2b"]
node_pools = [
- {
- "ami_image_id" : "amazon-eks-node-1.26-v20230411",
- "instance_type" : "t3a.large",
- "name" : "clusterwide-node-pool-1",
- "node_count" : 3,
- "spot" : false,
- "disk_size_gb" : 20
- }
+# {
+# "ami_image_id" : "amazon-eks-node-1.26-v20230501",
+# "instance_type" : "t3a.large",
+# "name" : "clusterwide-node-pool-1",
+# "node_count" : 3,
+# "spot" : false,
+# "disk_size_gb" : 20,
+# "max_pods" : 110
+# }
]
}
```
@@ -52,7 +53,7 @@ module "captain" {
| Name | Source | Version |
|------|--------|---------|
| [kubernetes](#module\_kubernetes) | cloudposse/eks-cluster/aws | 2.6.0 |
-| [node\_pool](#module\_node\_pool) | cloudposse/eks-node-group/aws | 2.9.0 |
+| [node\_pool](#module\_node\_pool) | cloudposse/eks-node-group/aws | 2.9.1 |
| [subnets](#module\_subnets) | cloudposse/dynamic-subnets/aws | 2.0.4 |
| [vpc](#module\_vpc) | cloudposse/vpc/aws | 2.0.0 |
| [vpc\_peering\_accepter\_with\_routes](#module\_vpc\_peering\_accepter\_with\_routes) | ./modules/vpc_peering_accepter_with_routes | n/a |
@@ -79,7 +80,7 @@ module "captain" {
| [csi\_driver\_version](#input\_csi\_driver\_version) | You should grab the appropriate version number from: https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/CHANGELOG.md | `string` | `"v1.17.0-eksbuild.1"` | no |
| [eks\_version](#input\_eks\_version) | The version of EKS to deploy | `string` | `"1.26"` | no |
| [iam\_role\_to\_assume](#input\_iam\_role\_to\_assume) | The full ARN of the IAM role to assume | `string` | n/a | yes |
-| [node\_pools](#input\_node\_pools) | node pool configurations:
- name (string): Name of the node pool. MUST BE UNIQUE! Recommended to use YYYYMMDD in the name
- node\_count (number): number of nodes to create in the node pool.
- instance\_type (string): Instance type to use for the nodes. ref: https://instances.vantage.sh/
- ami\_image\_id (string): AMI to use for EKS worker nodes. ref: https://github.com/awslabs/amazon-eks-ami/releases
- spot (bool): Enable spot instances for the nodes. DO NOT ENABLE IN PROD!
- disk\_size\_gb (number): Disk size in GB for the nodes. |
list(object({|
name = string
node_count = number
instance_type = string
ami_image_id = string
spot = bool
disk_size_gb = number
}))
[| no | +| [node\_pools](#input\_node\_pools) | node pool configurations:
{
"ami_image_id": "amazon-eks-node-1.24-v20230406",
"disk_size_gb": 20,
"instance_type": "t3a.large",
"name": "default-pool",
"node_count": 1,
"spot": false
}
]
list(object({|
name = string
node_count = number
instance_type = string
ami_image_id = string
spot = bool
disk_size_gb = number
max_pods = number
}))
[| no | | [peering\_configs](#input\_peering\_configs) | A list of maps containing VPC peering configuration details |
{
"ami_image_id": "amazon-eks-node-1.24-v20230406",
"disk_size_gb": 20,
"instance_type": "t3a.large",
"max_pods": 110,
"name": "default-pool",
"node_count": 1,
"spot": false
}
]
list(object({| `[]` | no | | [region](#input\_region) | The AWS region to deploy into | `string` | n/a | yes | | [vpc\_cidr\_block](#input\_vpc\_cidr\_block) | The CIDR block for the VPC | `string` | `"10.65.0.0/16"` | no | diff --git a/docs/.header.md b/docs/.header.md index 634170c..4356d06 100644 --- a/docs/.header.md +++ b/docs/.header.md @@ -17,19 +17,20 @@ module "captain" { iam_role_to_assume = "arn:aws:iam::1234567890:role/glueops-captain" source = "git::https://github.com/GlueOps/terraform-module-cloud-aws-kubernetes-cluster.git?ref=feat/multiple-node-pools" eks_version = "1.26" - csi_driver_version = "v1.17.0-eksbuild.1" - vpc_cidr_block = "10.65.0.0/16" + csi_driver_version = "v1.18.0-eksbuild.1" + vpc_cidr_block = "10.65.0.0/26" region = "us-west-2" availability_zones = ["us-west-2a", "us-west-2b"] node_pools = [ - { - "ami_image_id" : "amazon-eks-node-1.26-v20230411", - "instance_type" : "t3a.large", - "name" : "clusterwide-node-pool-1", - "node_count" : 3, - "spot" : false, - "disk_size_gb" : 20 - } +# { +# "ami_image_id" : "amazon-eks-node-1.26-v20230501", +# "instance_type" : "t3a.large", +# "name" : "clusterwide-node-pool-1", +# "node_count" : 3, +# "spot" : false, +# "disk_size_gb" : 20, +# "max_pods" : 110 +# } ] } ``` diff --git a/main.tf b/main.tf index b8f205b..0e141e3 100644 --- a/main.tf +++ b/main.tf @@ -29,7 +29,7 @@ module "node_pool" { for_each = { for np in var.node_pools : np.name => np } source = "cloudposse/eks-node-group/aws" # Cloud Posse recommends pinning every module to a specific version - version = "2.9.0" + version = "2.9.1" instance_types = [each.value.instance_type] subnet_ids = module.subnets.public_subnet_ids @@ -52,6 +52,9 @@ module "node_pool" { "volume_type" : "gp2" } ] + kubelet_additional_options = [ + "--max-pods=${each.value.max_pods}" + ] associated_security_group_ids = [aws_security_group.captain.id] } @@ -100,6 +103,7 @@ resource "aws_eks_addon" "ebs_csi" { resolve_conflicts = "OVERWRITE" service_account_role_arn = aws_iam_role.eks_addon_ebs_csi_role.arn depends_on = [aws_iam_role_policy_attachment.ebs_csi, module.node_pool] + count = length(var.node_pools) > 0 ? 1 : 0 } diff --git a/network.tf b/network.tf index ae51287..af86977 100644 --- a/network.tf +++ b/network.tf @@ -20,6 +20,7 @@ module "subnets" { private_subnets_enabled = false public_subnets_enabled = true availability_zones = var.availability_zones + max_subnet_count = length(var.availability_zones) } resource "aws_security_group" "captain" { diff --git a/peering.tf b/peering.tf index 9713f35..03004b4 100644 --- a/peering.tf +++ b/peering.tf @@ -2,7 +2,7 @@ module "vpc_peering_accepter_with_routes" { - source = "./modules/vpc_peering_accepter_with_routes" - route_table_ids = concat(module.subnets.private_route_table_ids, module.subnets.public_route_table_ids) - peering_configs = var.peering_configs + source = "./modules/vpc_peering_accepter_with_routes" + route_table_ids = concat(module.subnets.private_route_table_ids, module.subnets.public_route_table_ids) + peering_configs = var.peering_configs } diff --git a/tests/main.tf b/tests/main.tf index 160b4e3..5e42b77 100644 --- a/tests/main.tf +++ b/tests/main.tf @@ -2,18 +2,19 @@ module "captain" { iam_role_to_assume = "arn:aws:iam::761182885829:role/glueops-captain" source = "../" eks_version = "1.26" - csi_driver_version = "v1.17.0-eksbuild.1" - vpc_cidr_block = "10.65.0.0/16" + csi_driver_version = "v1.18.0-eksbuild.1" + vpc_cidr_block = "10.65.0.0/26" region = "us-west-2" availability_zones = ["us-west-2a", "us-west-2b"] node_pools = [ - { - "ami_image_id" : "amazon-eks-node-1.26-v20230406", - "instance_type" : "t3a.medium", - "name" : "clusterwide-node-pool-1", - "node_count" : 1, - "spot" : false, - "disk_size_gb" : 20 - } + # { + # "ami_image_id" : "amazon-eks-node-1.26-v20230411", + # "instance_type" : "t3a.small", + # "name" : "clusterwide-node-pool-1", + # "node_count" : 2, + # "spot" : false, + # "disk_size_gb" : 20, + # "max_pods" : 1000 + # } ] -} \ No newline at end of file +} diff --git a/tests/run.sh b/tests/run.sh index 046c7d4..972deb1 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -1,5 +1,7 @@ # #!/usr/bin/env bash +set -e + ./destroy-aws.sh echo "Terraform Init" @@ -8,12 +10,20 @@ echo "Terraform Plan" terraform plan echo "Terraform Apply" terraform apply -auto-approve -terraform apply -auto-approve echo "Authenticate with Kubernetes" aws eks update-kubeconfig --region us-west-2 --name captain-cluster --role-arn arn:aws:iam::761182885829:role/glueops-captain +echo "Delete AWS CNI" +kubectl delete daemonset -n kube-system aws-node +echo "Install Calico CNI" +helm repo add projectcalico https://docs.tigera.io/calico/charts +helm repo update +helm install calico projectcalico/tigera-operator --version v3.25.1 --namespace tigera-operator -f values.yaml --create-namespace +echo "Deploy node pool" +sed -i 's/#//g' main.tf +terraform apply -auto-approve echo "Get nodes and pods from kubernetes" kubectl get nodes -kubectl get pods --all-namespaces +kubectl get pods -A -o=wide echo "Start Test Suite" ./k8s-test.sh echo "Test Suite Complete" diff --git a/tests/values.yaml b/tests/values.yaml new file mode 100644 index 0000000..0c66ce7 --- /dev/null +++ b/tests/values.yaml @@ -0,0 +1,9 @@ +installation: + kubernetesProvider: EKS + cni: + type: Calico + calicoNetwork: + bgp: Disabled + ipPools: + - cidr: 172.16.0.0/16 + encapsulation: VXLAN \ No newline at end of file diff --git a/variables.tf b/variables.tf index 73cd6dd..9c51ed9 100644 --- a/variables.tf +++ b/variables.tf @@ -36,6 +36,7 @@ variable "node_pools" { ami_image_id = string spot = bool disk_size_gb = number + max_pods = number })) default = [{ name = "default-pool" @@ -44,6 +45,7 @@ variable "node_pools" { ami_image_id = "amazon-eks-node-1.24-v20230406" spot = false disk_size_gb = 20 + max_pods = 110 }] description = <<-DESC node pool configurations: @@ -53,6 +55,7 @@ variable "node_pools" { - ami_image_id (string): AMI to use for EKS worker nodes. ref: https://github.com/awslabs/amazon-eks-ami/releases - spot (bool): Enable spot instances for the nodes. DO NOT ENABLE IN PROD! - disk_size_gb (number): Disk size in GB for the nodes. + - max_pods (number): max pods that can be scheduled per node. DESC }
vpc_peering_connection_id = string
destination_cidr_block = string
}))