From 34ab8fe44ef324241d2e20eb164195f10ec8c9fc Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Wed, 6 Sep 2023 14:26:56 +0200 Subject: [PATCH 01/15] feat(bootstrap): TF changes for AWS CAPI migration Signed-off-by: David van der Spek --- .../aws-bootstrap-cluster-api/deps.yaml | 11 - .../aws-bootstrap-cluster-api/main.tf | 7 - .../terraform.tfvars | 2 - .../aws-bootstrap-cluster-api/variables.tf | 10 - .../aws-bootstrap/aws-lb-controller.tf | 2 +- bootstrap/terraform/aws-bootstrap/capa-sa.tf | 369 ++++++++++++++++++ .../terraform/aws-bootstrap/certmanager.tf | 2 +- bootstrap/terraform/aws-bootstrap/deps.yaml | 1 + bootstrap/terraform/aws-bootstrap/existing.tf | 2 +- .../aws-bootstrap/irsa-autoscaler.tf | 4 +- .../aws-bootstrap/irsa-externaldns.tf | 2 +- bootstrap/terraform/aws-bootstrap/irsa.tf | 2 +- bootstrap/terraform/aws-bootstrap/locals.tf | 16 +- bootstrap/terraform/aws-bootstrap/main.tf | 16 +- bootstrap/terraform/aws-bootstrap/output.tf | 20 +- .../terraform/aws-bootstrap/terraform.tfvars | 3 + .../terraform/aws-bootstrap/variables.tf | 10 + 17 files changed, 423 insertions(+), 56 deletions(-) delete mode 100644 bootstrap/terraform/aws-bootstrap-cluster-api/deps.yaml delete mode 100644 bootstrap/terraform/aws-bootstrap-cluster-api/main.tf delete mode 100644 bootstrap/terraform/aws-bootstrap-cluster-api/terraform.tfvars delete mode 100644 bootstrap/terraform/aws-bootstrap-cluster-api/variables.tf create mode 100644 bootstrap/terraform/aws-bootstrap/capa-sa.tf diff --git a/bootstrap/terraform/aws-bootstrap-cluster-api/deps.yaml b/bootstrap/terraform/aws-bootstrap-cluster-api/deps.yaml deleted file mode 100644 index 23b38d48e..000000000 --- a/bootstrap/terraform/aws-bootstrap-cluster-api/deps.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: plural.sh/v1alpha1 -kind: Dependencies -metadata: - description: Creates an EKS cluster and prepares it for bootstrapping - version: 0.1.1 -spec: - breaking: true - dependencies: [] - providers: - - aws - diff --git a/bootstrap/terraform/aws-bootstrap-cluster-api/main.tf b/bootstrap/terraform/aws-bootstrap-cluster-api/main.tf deleted file mode 100644 index 8a588bcdd..000000000 --- a/bootstrap/terraform/aws-bootstrap-cluster-api/main.tf +++ /dev/null @@ -1,7 +0,0 @@ -provider "aws" { - region = var.aws_region -} - -data "aws_eks_cluster" "cluster" { - name = var.cluster_name -} \ No newline at end of file diff --git a/bootstrap/terraform/aws-bootstrap-cluster-api/terraform.tfvars b/bootstrap/terraform/aws-bootstrap-cluster-api/terraform.tfvars deleted file mode 100644 index 95fdccd3c..000000000 --- a/bootstrap/terraform/aws-bootstrap-cluster-api/terraform.tfvars +++ /dev/null @@ -1,2 +0,0 @@ -aws_region = {{ .Region | quote }} -cluster_name = {{ .Cluster | quote }} \ No newline at end of file diff --git a/bootstrap/terraform/aws-bootstrap-cluster-api/variables.tf b/bootstrap/terraform/aws-bootstrap-cluster-api/variables.tf deleted file mode 100644 index 76efb1ca2..000000000 --- a/bootstrap/terraform/aws-bootstrap-cluster-api/variables.tf +++ /dev/null @@ -1,10 +0,0 @@ -variable "aws_region" { - type = string - default = "us-east-2" - description = "The region you wish to deploy to" -} - -variable "cluster_name" { - type = string - default = "plural" -} \ No newline at end of file diff --git a/bootstrap/terraform/aws-bootstrap/aws-lb-controller.tf b/bootstrap/terraform/aws-bootstrap/aws-lb-controller.tf index a45301848..6ae3eb6cd 100644 --- a/bootstrap/terraform/aws-bootstrap/aws-lb-controller.tf +++ b/bootstrap/terraform/aws-bootstrap/aws-lb-controller.tf @@ -258,4 +258,4 @@ resource "aws_iam_policy" "alb" { ] } POLICY -} \ No newline at end of file +} diff --git a/bootstrap/terraform/aws-bootstrap/capa-sa.tf b/bootstrap/terraform/aws-bootstrap/capa-sa.tf new file mode 100644 index 000000000..96cc73726 --- /dev/null +++ b/bootstrap/terraform/aws-bootstrap/capa-sa.tf @@ -0,0 +1,369 @@ +module "asummable_role_capa" { + # count = var.enable_cluster_capa ? 1 : 0 + + source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" + version = "3.14.0" + create_role = true + role_name = "${var.cluster_name}-capa-controller" + provider_url = replace(local.cluster_oidc_issuer_url, "https://", "") + role_policy_arns = [aws_iam_policy.capa_controller.arn, aws_iam_policy.capa_controller_eks.arn] + oidc_fully_qualified_subjects = ["system:serviceaccount:${var.namespace}:${var.capa_serviceaccount}", "system:serviceaccount:${var.namespace}:${var.capi_serviceaccount}"] +} + +resource "aws_iam_policy" "capa_controller" { + # count = var.enable_cluster_capa ? 1 : 0 + + name_prefix = "cluster-capa" + description = "EKS cluster api provider aws policy for cluster ${var.cluster_name}" + policy = data.aws_iam_policy_document.capa_controller.json +} + +resource "aws_iam_policy" "capa_controller_eks" { + # count = var.enable_cluster_capa ? 1 : 0 + + name_prefix = "cluster-capa" + description = "EKS cluster api provider aws policy for cluster ${var.cluster_name}" + policy = data.aws_iam_policy_document.capa_controller_eks.json +} + +data "aws_iam_policy_document" "capa_controller" { + statement { + sid = "" + effect = "Allow" + resources = ["*"] + + actions = [ + "ec2:AttachNetworkInterface", + "ec2:DetachNetworkInterface", + "ec2:AllocateAddress", + "ec2:AssignIpv6Addresses", + "ec2:AssignPrivateIpAddresses", + "ec2:UnassignPrivateIpAddresses", + "ec2:AssociateRouteTable", + "ec2:AttachInternetGateway", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateInternetGateway", + "ec2:CreateEgressOnlyInternetGateway", + "ec2:CreateNatGateway", + "ec2:CreateNetworkInterface", + "ec2:CreateRoute", + "ec2:CreateRouteTable", + "ec2:CreateSecurityGroup", + "ec2:CreateSubnet", + "ec2:CreateTags", + "ec2:CreateVpc", + "ec2:ModifyVpcAttribute", + "ec2:DeleteInternetGateway", + "ec2:DeleteEgressOnlyInternetGateway", + "ec2:DeleteNatGateway", + "ec2:DeleteRouteTable", + "ec2:ReplaceRoute", + "ec2:DeleteSecurityGroup", + "ec2:DeleteSubnet", + "ec2:DeleteTags", + "ec2:DeleteVpc", + "ec2:DescribeAccountAttributes", + "ec2:DescribeAddresses", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInternetGateways", + "ec2:DescribeEgressOnlyInternetGateways", + "ec2:DescribeInstanceTypes", + "ec2:DescribeImages", + "ec2:DescribeNatGateways", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeNetworkInterfaceAttribute", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs", + "ec2:DescribeVpcAttribute", + "ec2:DescribeVolumes", + "ec2:DescribeTags", + "ec2:DetachInternetGateway", + "ec2:DisassociateRouteTable", + "ec2:DisassociateAddress", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyNetworkInterfaceAttribute", + "ec2:ModifySubnetAttribute", + "ec2:ReleaseAddress", + "ec2:RevokeSecurityGroupIngress", + "ec2:RunInstances", + "ec2:TerminateInstances", + "tag:GetResources", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:DescribeTags", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:RemoveTags", + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeInstanceRefreshes", + "ec2:CreateLaunchTemplate", + "ec2:CreateLaunchTemplateVersion", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DeleteLaunchTemplate", + "ec2:DeleteLaunchTemplateVersions", + "ec2:DescribeKeyPairs", + "ec2:ModifyInstanceMetadataOptions", + ] + } + + statement { + sid = "" + effect = "Allow" + resources = ["arn:*:autoscaling:*:*:autoScalingGroup:*:autoScalingGroupName/*"] + + actions = [ + "autoscaling:CreateAutoScalingGroup", + "autoscaling:UpdateAutoScalingGroup", + "autoscaling:CreateOrUpdateTags", + "autoscaling:StartInstanceRefresh", + "autoscaling:DeleteAutoScalingGroup", + "autoscaling:DeleteTags", + ] + } + + statement { + sid = "" + effect = "Allow" + resources = ["arn:*:iam::*:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling"] + actions = ["iam:CreateServiceLinkedRole"] + + condition { + test = "StringLike" + variable = "iam:AWSServiceName" + values = ["autoscaling.amazonaws.com"] + } + } + + statement { + sid = "" + effect = "Allow" + resources = ["arn:*:iam::*:role/aws-service-role/elasticloadbalancing.amazonaws.com/AWSServiceRoleForElasticLoadBalancing"] + actions = ["iam:CreateServiceLinkedRole"] + + condition { + test = "StringLike" + variable = "iam:AWSServiceName" + values = ["elasticloadbalancing.amazonaws.com"] + } + } + + statement { + sid = "" + effect = "Allow" + resources = ["arn:*:iam::*:role/aws-service-role/spot.amazonaws.com/AWSServiceRoleForEC2Spot"] + actions = ["iam:CreateServiceLinkedRole"] + + condition { + test = "StringLike" + variable = "iam:AWSServiceName" + values = ["spot.amazonaws.com"] + } + } + + statement { + sid = "" + effect = "Allow" + resources = ["arn:*:iam::*:role/*.cluster-api-provider-aws.sigs.k8s.io"] + actions = ["iam:PassRole"] + } + + statement { + sid = "" + effect = "Allow" + resources = ["arn:*:secretsmanager:*:*:secret:aws.cluster.x-k8s.io/*"] + + actions = [ + "secretsmanager:CreateSecret", + "secretsmanager:DeleteSecret", + "secretsmanager:TagResource", + ] + } +} + +data "aws_iam_policy_document" "capa_controller_eks" { + + statement { + sid = "" + effect = "Allow" + resources = ["arn:*:ssm:*:*:parameter/aws/service/eks/optimized-ami/*"] + actions = ["ssm:GetParameter"] + } + + statement { + sid = "" + effect = "Allow" + resources = ["arn:*:iam::*:role/aws-service-role/eks.amazonaws.com/AWSServiceRoleForAmazonEKS"] + actions = ["iam:CreateServiceLinkedRole"] + + condition { + test = "StringLike" + variable = "iam:AWSServiceName" + values = ["eks.amazonaws.com"] + } + } + + statement { + sid = "" + effect = "Allow" + resources = ["arn:*:iam::*:role/aws-service-role/eks-nodegroup.amazonaws.com/AWSServiceRoleForAmazonEKSNodegroup"] + actions = ["iam:CreateServiceLinkedRole"] + + condition { + test = "StringLike" + variable = "iam:AWSServiceName" + values = ["eks-nodegroup.amazonaws.com"] + } + } + + statement { + sid = "" + effect = "Allow" + resources = ["arn:aws:iam::*:role/aws-service-role/eks-fargate-pods.amazonaws.com/AWSServiceRoleForAmazonEKSForFargate"] + actions = ["iam:CreateServiceLinkedRole"] + + condition { + test = "StringLike" + variable = "iam:AWSServiceName" + values = ["eks-fargate.amazonaws.com"] + } + } + + statement { + sid = "" + effect = "Allow" + resources = ["*"] + + actions = [ + "iam:ListOpenIDConnectProviders", + "iam:GetOpenIDConnectProvider", + "iam:CreateOpenIDConnectProvider", + "iam:AddClientIDToOpenIDConnectProvider", + "iam:UpdateOpenIDConnectProviderThumbprint", + "iam:DeleteOpenIDConnectProvider", + "iam:TagOpenIDConnectProvider", + ] + } + + statement { + sid = "" + effect = "Allow" + resources = ["arn:*:iam::*:role/*"] + + actions = [ + "iam:GetRole", + "iam:ListAttachedRolePolicies", + "iam:DetachRolePolicy", + "iam:DeleteRole", + "iam:CreateRole", + "iam:TagRole", + "iam:AttachRolePolicy", + ] + } + + statement { + sid = "" + effect = "Allow" + resources = [ + "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", + "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", + "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", + "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", + ] + actions = ["iam:GetPolicy"] + } + + statement { + sid = "" + effect = "Allow" + + resources = [ + "arn:*:eks:*:*:cluster/*", + "arn:*:eks:*:*:nodegroup/*/*/*", + ] + + actions = [ + "eks:DescribeCluster", + "eks:ListClusters", + "eks:CreateCluster", + "eks:TagResource", + "eks:UpdateClusterVersion", + "eks:DeleteCluster", + "eks:UpdateClusterConfig", + "eks:UntagResource", + "eks:UpdateNodegroupVersion", + "eks:DescribeNodegroup", + "eks:DeleteNodegroup", + "eks:UpdateNodegroupConfig", + "eks:CreateNodegroup", + "eks:AssociateEncryptionConfig", + "eks:ListIdentityProviderConfigs", + "eks:AssociateIdentityProviderConfig", + "eks:DescribeIdentityProviderConfig", + "eks:DisassociateIdentityProviderConfig", + ] + } + + statement { + sid = "" + effect = "Allow" + resources = ["*"] + + actions = [ + "ec2:AssociateVpcCidrBlock", + "ec2:DisassociateVpcCidrBlock", + "eks:ListAddons", + "eks:CreateAddon", + "eks:DescribeAddonVersions", + "eks:DescribeAddon", + "eks:DeleteAddon", + "eks:UpdateAddon", + "eks:TagResource", + "eks:DescribeFargateProfile", + "eks:CreateFargateProfile", + "eks:DeleteFargateProfile", + ] + } + + statement { + sid = "" + effect = "Allow" + resources = ["*"] + actions = ["iam:PassRole"] + + condition { + test = "StringEquals" + variable = "iam:PassedToService" + values = ["eks.amazonaws.com"] + } + } + + statement { + sid = "" + effect = "Allow" + resources = ["*"] + + actions = [ + "kms:CreateGrant", + "kms:DescribeKey", + ] + + condition { + test = "ForAnyValue:StringLike" + variable = "kms:ResourceAliases" + values = ["alias/cluster-api-provider-aws-*"] + } + } +} diff --git a/bootstrap/terraform/aws-bootstrap/certmanager.tf b/bootstrap/terraform/aws-bootstrap/certmanager.tf index 0d5657279..da9ffc540 100644 --- a/bootstrap/terraform/aws-bootstrap/certmanager.tf +++ b/bootstrap/terraform/aws-bootstrap/certmanager.tf @@ -36,4 +36,4 @@ resource "aws_iam_policy" "certmanager" { ] } POLICY -} \ No newline at end of file +} diff --git a/bootstrap/terraform/aws-bootstrap/deps.yaml b/bootstrap/terraform/aws-bootstrap/deps.yaml index 7cce61471..a9c03297d 100644 --- a/bootstrap/terraform/aws-bootstrap/deps.yaml +++ b/bootstrap/terraform/aws-bootstrap/deps.yaml @@ -9,6 +9,7 @@ spec: providers: - aws outputs: + capa_iam_role_arn: capa_iam_role_arn endpoint: cluster_endpoint cluster_private_subnets: cluster_private_subnets cluster_worker_private_subnets: cluster_worker_private_subnets diff --git a/bootstrap/terraform/aws-bootstrap/existing.tf b/bootstrap/terraform/aws-bootstrap/existing.tf index eef978d70..5971e0999 100644 --- a/bootstrap/terraform/aws-bootstrap/existing.tf +++ b/bootstrap/terraform/aws-bootstrap/existing.tf @@ -17,7 +17,7 @@ data "aws_subnet" "private_subnets" { count = length(local.private_subnet_ids) id = local.private_subnet_ids[count.index] } - + data "aws_subnet" "public_subnets" { count = length(local.public_subnet_ids) id = local.public_subnet_ids[count.index] diff --git a/bootstrap/terraform/aws-bootstrap/irsa-autoscaler.tf b/bootstrap/terraform/aws-bootstrap/irsa-autoscaler.tf index 5a534f962..5cc29df3e 100644 --- a/bootstrap/terraform/aws-bootstrap/irsa-autoscaler.tf +++ b/bootstrap/terraform/aws-bootstrap/irsa-autoscaler.tf @@ -50,7 +50,7 @@ data "aws_iam_policy_document" "cluster_autoscaler" { condition { test = "StringEquals" - variable = "autoscaling:ResourceTag/kubernetes.io/cluster/${module.cluster.cluster_id}" + variable = "autoscaling:ResourceTag/kubernetes.io/cluster/${local.cluster_id}" values = ["owned"] } @@ -60,4 +60,4 @@ data "aws_iam_policy_document" "cluster_autoscaler" { values = ["true"] } } -} \ No newline at end of file +} diff --git a/bootstrap/terraform/aws-bootstrap/irsa-externaldns.tf b/bootstrap/terraform/aws-bootstrap/irsa-externaldns.tf index 527d9b9bd..0051d5d81 100644 --- a/bootstrap/terraform/aws-bootstrap/irsa-externaldns.tf +++ b/bootstrap/terraform/aws-bootstrap/irsa-externaldns.tf @@ -37,4 +37,4 @@ data "aws_iam_policy_document" "externaldns" { resources = ["*"] } -} \ No newline at end of file +} diff --git a/bootstrap/terraform/aws-bootstrap/irsa.tf b/bootstrap/terraform/aws-bootstrap/irsa.tf index b5c447a1c..33e5d4c9c 100644 --- a/bootstrap/terraform/aws-bootstrap/irsa.tf +++ b/bootstrap/terraform/aws-bootstrap/irsa.tf @@ -1,5 +1,5 @@ resource "aws_iam_openid_connect_provider" "oidc_provider" { - count = var.enable_irsa && !var.create_cluster ? 1 : 0 + count = var.enable_irsa ? 0 : 1 client_id_list = [local.sts_principal] thumbprint_list = [var.eks_oidc_root_ca_thumbprint] url = local.cluster_oidc_issuer_url diff --git a/bootstrap/terraform/aws-bootstrap/locals.tf b/bootstrap/terraform/aws-bootstrap/locals.tf index 9bcb0a2b5..7634fb7f9 100644 --- a/bootstrap/terraform/aws-bootstrap/locals.tf +++ b/bootstrap/terraform/aws-bootstrap/locals.tf @@ -1,12 +1,12 @@ locals { sts_principal = "sts.${data.aws_partition.current.dns_suffix}" create_vpc = var.create_cluster && var.create_vpc ? true : false - private_subnet_ids = var.create_cluster ? module.vpc.private_subnets_ids : var.private_subnet_ids - public_subnet_ids = var.create_cluster ? module.vpc.public_subnets_ids : var.public_subnet_ids - worker_private_subnet_ids = var.create_cluster ? module.vpc.worker_private_subnets_ids : var.worker_private_subnet_ids - vpc_id = var.create_cluster ? module.vpc.vpc_id : data.aws_eks_cluster.cluster[0].vpc_config[0].vpc_id - cluster_id = var.create_cluster ? module.cluster.cluster_id : data.aws_eks_cluster.cluster[0].id - cluster_config = try(var.create_cluster ? module.cluster.config_map_aws_auth : tomap(false), {}) - cluster_oidc_issuer_url = var.create_cluster ? module.cluster.cluster_oidc_issuer_url : data.aws_eks_cluster.cluster[0].identity[0].oidc.0.issuer - cluster_endpoint = var.create_cluster ? module.cluster.cluster_endpoint : data.aws_eks_cluster.cluster[0].endpoint + private_subnet_ids = var.create_cluster ? module.vpc[0].private_subnets_ids : var.private_subnet_ids + public_subnet_ids = var.create_cluster ? module.vpc[0].public_subnets_ids : var.public_subnet_ids + worker_private_subnet_ids = var.create_cluster ? module.vpc[0].worker_private_subnets_ids : var.worker_private_subnet_ids + vpc_id = var.create_cluster ? module.vpc[0].vpc_id : data.aws_eks_cluster.cluster[0].vpc_config[0].vpc_id + cluster_id = var.create_cluster ? module.cluster[0].cluster_id : data.aws_eks_cluster.cluster[0].id + cluster_config = try(var.create_cluster ? module.cluster[0].config_map_aws_auth : tomap(false), {}) + cluster_oidc_issuer_url = var.create_cluster ? module.cluster[0].cluster_oidc_issuer_url : data.aws_eks_cluster.cluster[0].identity[0].oidc.0.issuer + cluster_endpoint = var.create_cluster ? module.cluster[0].cluster_endpoint : data.aws_eks_cluster.cluster[0].endpoint } diff --git a/bootstrap/terraform/aws-bootstrap/main.tf b/bootstrap/terraform/aws-bootstrap/main.tf index 0075bd2e4..ec6329fd8 100644 --- a/bootstrap/terraform/aws-bootstrap/main.tf +++ b/bootstrap/terraform/aws-bootstrap/main.tf @@ -3,6 +3,8 @@ data "aws_availability_zones" "available" {} data "aws_caller_identity" "current" {} module "vpc" { + count = var.create_cluster ? 1:0 + source = "github.com/pluralsh/terraform-aws-vpc?ref=worker_subnet" name = var.vpc_name cidr = var.vpc_cidr @@ -36,6 +38,8 @@ module "vpc" { } module "cluster" { + count = var.create_cluster ? 1:0 + source = "github.com/pluralsh/terraform-aws-eks?ref=output-service-cidr" cluster_name = var.cluster_name cluster_version = var.kubernetes_version @@ -59,15 +63,17 @@ module "cluster" { } module "single_az_node_groups" { + count = var.create_cluster ? 1:0 + source = "github.com/pluralsh/module-library//terraform/eks-node-groups/single-az-node-groups?ref=20e64863ffc5e361045db8e6b81b9d244a55809e" cluster_name = var.cluster_name - default_iam_role_arn = module.cluster.worker_iam_role_arn + default_iam_role_arn = module.cluster[0].worker_iam_role_arn tags = {} node_groups_defaults = var.node_groups_defaults node_groups = try(var.create_cluster ? var.single_az_node_groups : tomap(false), {}) set_desired_size = false - private_subnets = var.create_cluster ? module.vpc.worker_private_subnets : [] + private_subnets = var.create_cluster ? module.vpc[0].worker_private_subnets : [] ng_depends_on = [ local.cluster_config @@ -75,9 +81,11 @@ module "single_az_node_groups" { } module "multi_az_node_groups" { + count = var.create_cluster ? 1:0 + source = "github.com/pluralsh/module-library//terraform/eks-node-groups/multi-az-node-groups?ref=20e64863ffc5e361045db8e6b81b9d244a55809e" cluster_name = var.cluster_name - default_iam_role_arn = module.cluster.worker_iam_role_arn + default_iam_role_arn = one(module.cluster[*].worker_iam_role_arn) tags = {} node_groups_defaults = var.node_groups_defaults @@ -136,6 +144,8 @@ resource "aws_eks_addon" "kube_proxy" { } resource "kubernetes_namespace" "bootstrap" { + count = var.create_cluster ? 1:0 + metadata { name = "bootstrap" labels = { diff --git a/bootstrap/terraform/aws-bootstrap/output.tf b/bootstrap/terraform/aws-bootstrap/output.tf index d10f42048..147841aad 100644 --- a/bootstrap/terraform/aws-bootstrap/output.tf +++ b/bootstrap/terraform/aws-bootstrap/output.tf @@ -1,4 +1,3 @@ - output "cluster_name" { value = local.cluster_id } @@ -36,26 +35,31 @@ output "cluster_public_subnet_ids" { } output "worker_role_arn" { - value = module.cluster.worker_iam_role_arn + value = var.create_cluster ? module.cluster[0].worker_iam_role_arn : "" } output "node_groups" { - value = [for d in merge(module.single_az_node_groups.node_groups, module.multi_az_node_groups.node_groups): d] + value = try(var.create_cluster ?[for d in merge(module.single_az_node_groups[0].node_groups, module.multi_az_node_groups[0].node_groups): d]: tomap(false), {}) } output "vpc" { - value = try(var.create_cluster ? module.vpc : tomap(false), data.aws_vpc.vpc[0]) + value = try(var.create_cluster ? module.vpc[0] : tomap(false), data.aws_vpc.vpc[0]) } output "vpc_cidr" { - value = var.create_cluster ? module.vpc.vpc_cidr_block : data.aws_vpc.vpc[0].cidr_block + value = var.create_cluster ? module.vpc[0].vpc_cidr_block : data.aws_vpc.vpc[0].cidr_block } output "cluster" { - value = try(var.create_cluster ? module.cluster : tomap(false), data.aws_eks_cluster.cluster[0]) + value = try(var.create_cluster ? module.cluster[0] : tomap(false), data.aws_eks_cluster.cluster[0]) } output "cluster_service_ipv4_cidr" { - value = var.create_cluster ? module.cluster.cluster_service_ipv4_cidr : data.aws_eks_cluster.cluster[0].kubernetes_network_config[0].service_ipv4_cidr -} \ No newline at end of file + value = var.create_cluster ? module.cluster[0].cluster_service_ipv4_cidr : data.aws_eks_cluster.cluster[0].kubernetes_network_config[0].service_ipv4_cidr +} + +output "capa_iam_role_arn" { + description = "ARN of IAM role that allows access to the Harbor S3 buckets." + value = module.asummable_role_capa.this_iam_role_arn +} diff --git a/bootstrap/terraform/aws-bootstrap/terraform.tfvars b/bootstrap/terraform/aws-bootstrap/terraform.tfvars index d827a452f..037036bdf 100644 --- a/bootstrap/terraform/aws-bootstrap/terraform.tfvars +++ b/bootstrap/terraform/aws-bootstrap/terraform.tfvars @@ -1,5 +1,8 @@ vpc_name = {{ .Values.vpc_name | quote }} cluster_name = {{ .Cluster | quote }} +{{- if eq .ClusterAPI true }} +create_cluster = false +{{- end }} map_roles = [ { diff --git a/bootstrap/terraform/aws-bootstrap/variables.tf b/bootstrap/terraform/aws-bootstrap/variables.tf index 1f6fa1319..c9529e214 100644 --- a/bootstrap/terraform/aws-bootstrap/variables.tf +++ b/bootstrap/terraform/aws-bootstrap/variables.tf @@ -368,3 +368,13 @@ variable "aws_region" { default = "us-east-2" description = "The region you wish to deploy to" } + +variable "capa_serviceaccount" { + type = string + default = "bootstrap-cluster-api-provider-aws" +} + +variable "capi_serviceaccount" { + type = string + default = "bootstrap-cluster-api-core" +} From c78784166851888046babf0eb1f345a7926b56c7 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Wed, 6 Sep 2023 14:27:00 +0200 Subject: [PATCH 02/15] slightly cleanup locals Signed-off-by: David van der Spek --- bootstrap/terraform/aws-bootstrap/locals.tf | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/bootstrap/terraform/aws-bootstrap/locals.tf b/bootstrap/terraform/aws-bootstrap/locals.tf index 7634fb7f9..3a67e77ae 100644 --- a/bootstrap/terraform/aws-bootstrap/locals.tf +++ b/bootstrap/terraform/aws-bootstrap/locals.tf @@ -1,12 +1,12 @@ locals { sts_principal = "sts.${data.aws_partition.current.dns_suffix}" create_vpc = var.create_cluster && var.create_vpc ? true : false - private_subnet_ids = var.create_cluster ? module.vpc[0].private_subnets_ids : var.private_subnet_ids - public_subnet_ids = var.create_cluster ? module.vpc[0].public_subnets_ids : var.public_subnet_ids - worker_private_subnet_ids = var.create_cluster ? module.vpc[0].worker_private_subnets_ids : var.worker_private_subnet_ids - vpc_id = var.create_cluster ? module.vpc[0].vpc_id : data.aws_eks_cluster.cluster[0].vpc_config[0].vpc_id - cluster_id = var.create_cluster ? module.cluster[0].cluster_id : data.aws_eks_cluster.cluster[0].id - cluster_config = try(var.create_cluster ? module.cluster[0].config_map_aws_auth : tomap(false), {}) - cluster_oidc_issuer_url = var.create_cluster ? module.cluster[0].cluster_oidc_issuer_url : data.aws_eks_cluster.cluster[0].identity[0].oidc.0.issuer - cluster_endpoint = var.create_cluster ? module.cluster[0].cluster_endpoint : data.aws_eks_cluster.cluster[0].endpoint + private_subnet_ids = var.create_cluster ? one(module.vpc[*].private_subnets_ids) : var.private_subnet_ids + public_subnet_ids = var.create_cluster ? one(module.vpc[*].public_subnets_ids) : var.public_subnet_ids + worker_private_subnet_ids = var.create_cluster ? one(module.vpc[*].worker_private_subnets_ids) : var.worker_private_subnet_ids + vpc_id = var.create_cluster ? one(module.vpc[*].vpc_id) : data.aws_eks_cluster.cluster[0].vpc_config[0].vpc_id + cluster_id = var.create_cluster ? one(module.cluster[*].cluster_id) : one(data.aws_eks_cluster.cluster[*].id) + cluster_config = try(var.create_cluster ? one(module.cluster[*].config_map_aws_auth) : tomap(false), {}) + cluster_oidc_issuer_url = var.create_cluster ? one(module.cluster[*].cluster_oidc_issuer_url) : one(data.aws_eks_cluster.cluster[*].identity[0].oidc.0.issuer) + cluster_endpoint = var.create_cluster ? one(module.cluster[*].cluster_endpoint) : one(data.aws_eks_cluster.cluster[*].endpoint) } From 83daf6a31c04ca21da4fe1d564f761c0e7539339 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Thu, 7 Sep 2023 11:22:02 +0200 Subject: [PATCH 03/15] more initial cleanup Signed-off-by: David van der Spek --- bootstrap/terraform/aws-bootstrap/existing.tf | 33 +++++++++++-------- bootstrap/terraform/aws-bootstrap/irsa.tf | 2 +- bootstrap/terraform/aws-bootstrap/locals.tf | 25 ++++++++------ bootstrap/terraform/aws-bootstrap/main.tf | 23 ++++++------- bootstrap/terraform/aws-bootstrap/output.tf | 16 ++++----- .../terraform/aws-bootstrap/terraform.tfvars | 1 + 6 files changed, 57 insertions(+), 43 deletions(-) diff --git a/bootstrap/terraform/aws-bootstrap/existing.tf b/bootstrap/terraform/aws-bootstrap/existing.tf index 5971e0999..5c173035b 100644 --- a/bootstrap/terraform/aws-bootstrap/existing.tf +++ b/bootstrap/terraform/aws-bootstrap/existing.tf @@ -1,24 +1,31 @@ data "aws_eks_cluster" "cluster" { - count = var.create_cluster ? 0 : 1 - name = var.cluster_name + count = var.create_cluster ? 0 : 1 + name = var.cluster_name } data "aws_vpc" "vpc" { - count = var.create_cluster ? 0 : 1 - id = local.vpc_id + count = var.create_cluster ? 0 : 1 + id = local.vpc_id } -data "aws_subnet" "worker_private_subnets" { - count = length(local.worker_private_subnet_ids) - id = local.worker_private_subnet_ids[count.index] +data "aws_eks_node_groups" "cluster" { + count = var.create_cluster ? 0 : 1 + cluster_name = var.cluster_name +} + +data "aws_eks_node_group" "cluster" { + count = var.create_cluster ? 0 : length(one(data.aws_eks_node_groups.cluster[*].names)) + cluster_name = var.cluster_name + node_group_name = tolist(one(data.aws_eks_node_groups.cluster[*].names))[count.index] } -data "aws_subnet" "private_subnets" { - count = length(local.private_subnet_ids) - id = local.private_subnet_ids[count.index] +data "aws_subnet" "cluster_subnets" { + count = var.create_cluster ? 0 : length(one(data.aws_eks_cluster.test_cluster[*].vpc_config[0].subnet_ids)) + id = tolist(one(data.aws_eks_cluster.test_cluster[*].vpc_config[0].subnet_ids))[count.index] } -data "aws_subnet" "public_subnets" { - count = length(local.public_subnet_ids) - id = local.public_subnet_ids[count.index] + +data "aws_subnet" "worker_private_subnets" { + count = length(local.worker_private_subnet_ids) + id = local.worker_private_subnet_ids[count.index] } diff --git a/bootstrap/terraform/aws-bootstrap/irsa.tf b/bootstrap/terraform/aws-bootstrap/irsa.tf index 33e5d4c9c..b5c447a1c 100644 --- a/bootstrap/terraform/aws-bootstrap/irsa.tf +++ b/bootstrap/terraform/aws-bootstrap/irsa.tf @@ -1,5 +1,5 @@ resource "aws_iam_openid_connect_provider" "oidc_provider" { - count = var.enable_irsa ? 0 : 1 + count = var.enable_irsa && !var.create_cluster ? 1 : 0 client_id_list = [local.sts_principal] thumbprint_list = [var.eks_oidc_root_ca_thumbprint] url = local.cluster_oidc_issuer_url diff --git a/bootstrap/terraform/aws-bootstrap/locals.tf b/bootstrap/terraform/aws-bootstrap/locals.tf index 3a67e77ae..1c60a8ac4 100644 --- a/bootstrap/terraform/aws-bootstrap/locals.tf +++ b/bootstrap/terraform/aws-bootstrap/locals.tf @@ -1,12 +1,17 @@ locals { - sts_principal = "sts.${data.aws_partition.current.dns_suffix}" - create_vpc = var.create_cluster && var.create_vpc ? true : false - private_subnet_ids = var.create_cluster ? one(module.vpc[*].private_subnets_ids) : var.private_subnet_ids - public_subnet_ids = var.create_cluster ? one(module.vpc[*].public_subnets_ids) : var.public_subnet_ids - worker_private_subnet_ids = var.create_cluster ? one(module.vpc[*].worker_private_subnets_ids) : var.worker_private_subnet_ids - vpc_id = var.create_cluster ? one(module.vpc[*].vpc_id) : data.aws_eks_cluster.cluster[0].vpc_config[0].vpc_id - cluster_id = var.create_cluster ? one(module.cluster[*].cluster_id) : one(data.aws_eks_cluster.cluster[*].id) - cluster_config = try(var.create_cluster ? one(module.cluster[*].config_map_aws_auth) : tomap(false), {}) - cluster_oidc_issuer_url = var.create_cluster ? one(module.cluster[*].cluster_oidc_issuer_url) : one(data.aws_eks_cluster.cluster[*].identity[0].oidc.0.issuer) - cluster_endpoint = var.create_cluster ? one(module.cluster[*].cluster_endpoint) : one(data.aws_eks_cluster.cluster[*].endpoint) + sts_principal = "sts.${data.aws_partition.current.dns_suffix}" + create_vpc = var.create_cluster && var.create_vpc ? true : false + vpc_id = var.create_cluster ? one(module.vpc[*].vpc_id) : one(data.aws_eks_cluster.cluster[*].vpc_config[0].vpc_id) + cluster_id = var.create_cluster ? one(module.cluster[*].cluster_id) : one(data.aws_eks_cluster.cluster[*].id) + cluster_config = var.create_cluster ? one(module.cluster[*].config_map_aws_auth) : {} + cluster_oidc_issuer_url = var.create_cluster ? one(module.cluster[*].cluster_oidc_issuer_url) : one(data.aws_eks_cluster.cluster[*].identity[0].oidc.0.issuer) + cluster_endpoint = var.create_cluster ? one(module.cluster[*].cluster_endpoint) : one(data.aws_eks_cluster.cluster[*].endpoint) +} + +locals { + private_subnet_ids = var.create_cluster ? one(module.vpc[*].private_subnets_ids) : [for index, subnet in data.aws_subnet.cluster_subnets : subnet.id if contains(keys(subnet.tags), "kubernetes.io/role/internal-elb")] + private_subnet = var.create_cluster ? one(module.vpc[*].private_subnets_ids) : [for index, subnet in data.aws_subnet.cluster_subnets : subnet if contains(keys(subnet.tags), "kubernetes.io/role/internal-elb")] + public_subnet_ids = var.create_cluster ? one(module.vpc[*].public_subnets_ids) : [for index, subnet in data.aws_subnet.cluster_subnets : subnet.id if contains(keys(subnet.tags), "kubernetes.io/role/elb")] + public_subnet = var.create_cluster ? one(module.vpc[*].public_subnets_ids) : [for index, subnet in data.aws_subnet.cluster_subnets : subnet if contains(keys(subnet.tags), "kubernetes.io/role/elb")] + worker_private_subnet_ids = var.create_cluster ? one(module.vpc[*].worker_private_subnets_ids) : distinct(flatten([for index, group in data.aws_eks_node_group.cluster : group.subnet_ids ])) } diff --git a/bootstrap/terraform/aws-bootstrap/main.tf b/bootstrap/terraform/aws-bootstrap/main.tf index ec6329fd8..b17e2cb5a 100644 --- a/bootstrap/terraform/aws-bootstrap/main.tf +++ b/bootstrap/terraform/aws-bootstrap/main.tf @@ -3,7 +3,7 @@ data "aws_availability_zones" "available" {} data "aws_caller_identity" "current" {} module "vpc" { - count = var.create_cluster ? 1:0 + count = local.create_vpc ? 1 : 0 source = "github.com/pluralsh/terraform-aws-vpc?ref=worker_subnet" name = var.vpc_name @@ -14,7 +14,6 @@ module "vpc" { worker_private_subnets = var.worker_private_subnets enable_dns_hostnames = true enable_ipv6 = true - create_vpc = local.create_vpc database_subnets = var.database_subnets @@ -38,7 +37,7 @@ module "vpc" { } module "cluster" { - count = var.create_cluster ? 1:0 + count = var.create_cluster ? 1 : 0 source = "github.com/pluralsh/terraform-aws-eks?ref=output-service-cidr" cluster_name = var.cluster_name @@ -49,7 +48,6 @@ module "cluster" { vpc_id = local.vpc_id enable_irsa = true write_kubeconfig = false - create_eks = var.create_cluster cluster_enabled_log_types = var.cluster_enabled_log_types cluster_log_retention_in_days = var.cluster_log_retention_in_days cluster_log_kms_key_id = var.cluster_log_kms_key_id @@ -63,17 +61,17 @@ module "cluster" { } module "single_az_node_groups" { - count = var.create_cluster ? 1:0 + count = var.create_cluster ? 1 : 0 source = "github.com/pluralsh/module-library//terraform/eks-node-groups/single-az-node-groups?ref=20e64863ffc5e361045db8e6b81b9d244a55809e" cluster_name = var.cluster_name - default_iam_role_arn = module.cluster[0].worker_iam_role_arn + default_iam_role_arn = one(module.cluster[*].worker_iam_role_arn) tags = {} node_groups_defaults = var.node_groups_defaults - node_groups = try(var.create_cluster ? var.single_az_node_groups : tomap(false), {}) + node_groups = var.single_az_node_groups set_desired_size = false - private_subnets = var.create_cluster ? module.vpc[0].worker_private_subnets : [] + private_subnets = one(module.vpc[*].worker_private_subnets) ng_depends_on = [ local.cluster_config @@ -81,7 +79,7 @@ module "single_az_node_groups" { } module "multi_az_node_groups" { - count = var.create_cluster ? 1:0 + count = var.create_cluster ? 1 : 0 source = "github.com/pluralsh/module-library//terraform/eks-node-groups/multi-az-node-groups?ref=20e64863ffc5e361045db8e6b81b9d244a55809e" cluster_name = var.cluster_name @@ -89,7 +87,7 @@ module "multi_az_node_groups" { tags = {} node_groups_defaults = var.node_groups_defaults - node_groups = try(var.create_cluster ? var.multi_az_node_groups : tomap(false), {}) + node_groups = var.multi_az_node_groups set_desired_size = false private_subnet_ids = local.worker_private_subnet_ids @@ -100,6 +98,7 @@ module "multi_az_node_groups" { resource "aws_eks_addon" "vpc_cni" { count = var.create_cluster ? 1 : 0 + cluster_name = local.cluster_id addon_name = "vpc-cni" addon_version = var.vpc_cni_addon_version @@ -115,6 +114,7 @@ resource "aws_eks_addon" "vpc_cni" { resource "aws_eks_addon" "core_dns" { count = var.create_cluster ? 1 : 0 + cluster_name = local.cluster_id addon_name = "coredns" addon_version = var.core_dns_addon_version @@ -130,6 +130,7 @@ resource "aws_eks_addon" "core_dns" { resource "aws_eks_addon" "kube_proxy" { count = var.create_cluster ? 1 : 0 + cluster_name = local.cluster_id addon_name = "kube-proxy" addon_version = var.kube_proxy_addon_version @@ -144,7 +145,7 @@ resource "aws_eks_addon" "kube_proxy" { } resource "kubernetes_namespace" "bootstrap" { - count = var.create_cluster ? 1:0 + count = var.create_cluster ? 1 : 0 metadata { name = "bootstrap" diff --git a/bootstrap/terraform/aws-bootstrap/output.tf b/bootstrap/terraform/aws-bootstrap/output.tf index 147841aad..e9f369f80 100644 --- a/bootstrap/terraform/aws-bootstrap/output.tf +++ b/bootstrap/terraform/aws-bootstrap/output.tf @@ -11,7 +11,7 @@ output "cluster_oidc_issuer_url" { } output "cluster_private_subnets" { - value = data.aws_subnet.private_subnets + value = local.private_subnet } output "cluster_worker_private_subnets" { @@ -19,7 +19,7 @@ output "cluster_worker_private_subnets" { } output "cluster_public_subnets" { - value = data.aws_subnet.public_subnets + value = local.public_subnet } output "cluster_private_subnet_ids" { @@ -35,28 +35,28 @@ output "cluster_public_subnet_ids" { } output "worker_role_arn" { - value = var.create_cluster ? module.cluster[0].worker_iam_role_arn : "" + value = var.create_cluster ? one(module.cluster[*].worker_iam_role_arn) : "" } output "node_groups" { - value = try(var.create_cluster ?[for d in merge(module.single_az_node_groups[0].node_groups, module.multi_az_node_groups[0].node_groups): d]: tomap(false), {}) + value = var.create_cluster ? [for d in merge(one(module.single_az_node_groups[*].node_groups), one(module.multi_az_node_groups[*].node_groups)): d] : {} } output "vpc" { - value = try(var.create_cluster ? module.vpc[0] : tomap(false), data.aws_vpc.vpc[0]) + value = var.create_cluster ? one(module.vpc[*]) : one(data.aws_vpc.vpc[*]) } output "vpc_cidr" { - value = var.create_cluster ? module.vpc[0].vpc_cidr_block : data.aws_vpc.vpc[0].cidr_block + value = var.create_cluster ? one(module.vpc[*].vpc_cidr_block) : one(data.aws_vpc.vpc[*].cidr_block) } output "cluster" { - value = try(var.create_cluster ? module.cluster[0] : tomap(false), data.aws_eks_cluster.cluster[0]) + value = var.create_cluster ? one(module.cluster[*]) : (data.aws_eks_cluster.cluster[*]) } output "cluster_service_ipv4_cidr" { - value = var.create_cluster ? module.cluster[0].cluster_service_ipv4_cidr : data.aws_eks_cluster.cluster[0].kubernetes_network_config[0].service_ipv4_cidr + value = var.create_cluster ? one(module.cluster[*].cluster_service_ipv4_cidr) : one(data.aws_eks_cluster.cluster[*].kubernetes_network_config[0].service_ipv4_cidr) } output "capa_iam_role_arn" { diff --git a/bootstrap/terraform/aws-bootstrap/terraform.tfvars b/bootstrap/terraform/aws-bootstrap/terraform.tfvars index 037036bdf..437a4acb8 100644 --- a/bootstrap/terraform/aws-bootstrap/terraform.tfvars +++ b/bootstrap/terraform/aws-bootstrap/terraform.tfvars @@ -2,6 +2,7 @@ vpc_name = {{ .Values.vpc_name | quote }} cluster_name = {{ .Cluster | quote }} {{- if eq .ClusterAPI true }} create_cluster = false +enable_irsa = false {{- end }} map_roles = [ From 28e347a69f24406bc88f78e1330803c96f826411 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Thu, 7 Sep 2023 12:08:28 +0200 Subject: [PATCH 04/15] first possibly valid setup Signed-off-by: David van der Spek --- bootstrap/terraform/aws-bootstrap/existing.tf | 4 +- bootstrap/terraform/aws-bootstrap/locals.tf | 2 +- bootstrap/terraform/aws-bootstrap/moved.tf | 39 +++++++++++++++++++ bootstrap/terraform/aws-bootstrap/output.tf | 8 ++-- 4 files changed, 46 insertions(+), 7 deletions(-) create mode 100644 bootstrap/terraform/aws-bootstrap/moved.tf diff --git a/bootstrap/terraform/aws-bootstrap/existing.tf b/bootstrap/terraform/aws-bootstrap/existing.tf index 5c173035b..2297d7824 100644 --- a/bootstrap/terraform/aws-bootstrap/existing.tf +++ b/bootstrap/terraform/aws-bootstrap/existing.tf @@ -20,8 +20,8 @@ data "aws_eks_node_group" "cluster" { } data "aws_subnet" "cluster_subnets" { - count = var.create_cluster ? 0 : length(one(data.aws_eks_cluster.test_cluster[*].vpc_config[0].subnet_ids)) - id = tolist(one(data.aws_eks_cluster.test_cluster[*].vpc_config[0].subnet_ids))[count.index] + count = var.create_cluster ? 0 : length(one(data.aws_eks_cluster.cluster[*].vpc_config[0].subnet_ids)) + id = tolist(one(data.aws_eks_cluster.cluster[*].vpc_config[0].subnet_ids))[count.index] } diff --git a/bootstrap/terraform/aws-bootstrap/locals.tf b/bootstrap/terraform/aws-bootstrap/locals.tf index 1c60a8ac4..9393ee06a 100644 --- a/bootstrap/terraform/aws-bootstrap/locals.tf +++ b/bootstrap/terraform/aws-bootstrap/locals.tf @@ -3,7 +3,7 @@ locals { create_vpc = var.create_cluster && var.create_vpc ? true : false vpc_id = var.create_cluster ? one(module.vpc[*].vpc_id) : one(data.aws_eks_cluster.cluster[*].vpc_config[0].vpc_id) cluster_id = var.create_cluster ? one(module.cluster[*].cluster_id) : one(data.aws_eks_cluster.cluster[*].id) - cluster_config = var.create_cluster ? one(module.cluster[*].config_map_aws_auth) : {} + cluster_config = try(var.create_cluster ? one(module.cluster[*].config_map_aws_auth) : tomap(false), {}) cluster_oidc_issuer_url = var.create_cluster ? one(module.cluster[*].cluster_oidc_issuer_url) : one(data.aws_eks_cluster.cluster[*].identity[0].oidc.0.issuer) cluster_endpoint = var.create_cluster ? one(module.cluster[*].cluster_endpoint) : one(data.aws_eks_cluster.cluster[*].endpoint) } diff --git a/bootstrap/terraform/aws-bootstrap/moved.tf b/bootstrap/terraform/aws-bootstrap/moved.tf new file mode 100644 index 000000000..c2ed34899 --- /dev/null +++ b/bootstrap/terraform/aws-bootstrap/moved.tf @@ -0,0 +1,39 @@ +moved { + from = module.cluster + to = module.cluster[0] +} + +moved { + from = module.vpc + to = module.vpc[0] +} + +moved { + from = module.single_az_node_groups + to = module.single_az_node_groups[0] +} + +moved { + from = module.multi_az_node_groups + to = module.multi_az_node_groups[0] +} + +moved { + from = aws_eks_addon.vpc_cni + to = aws_eks_addon.vpc_cni[0] +} + +moved { + from = aws_eks_addon.core_dns + to = aws_eks_addon.core_dns[0] +} + +moved { + from = aws_eks_addon.kube_proxy + to = aws_eks_addon.kube_proxy[0] +} + +moved { + from = kubernetes_namespace.bootstrap + to = kubernetes_namespace.bootstrap[0] +} diff --git a/bootstrap/terraform/aws-bootstrap/output.tf b/bootstrap/terraform/aws-bootstrap/output.tf index e9f369f80..9c3b87769 100644 --- a/bootstrap/terraform/aws-bootstrap/output.tf +++ b/bootstrap/terraform/aws-bootstrap/output.tf @@ -35,15 +35,15 @@ output "cluster_public_subnet_ids" { } output "worker_role_arn" { - value = var.create_cluster ? one(module.cluster[*].worker_iam_role_arn) : "" + value = var.create_cluster ? one(module.cluster[*].worker_iam_role_arn) : data.aws_eks_node_group.cluster[0].node_role_arn } output "node_groups" { - value = var.create_cluster ? [for d in merge(one(module.single_az_node_groups[*].node_groups), one(module.multi_az_node_groups[*].node_groups)): d] : {} + value = try(var.create_cluster ? [for d in merge(one(module.single_az_node_groups[*].node_groups), one(module.multi_az_node_groups[*].node_groups)): d] : tomap(false), {}) } output "vpc" { - value = var.create_cluster ? one(module.vpc[*]) : one(data.aws_vpc.vpc[*]) + value = try(var.create_cluster ? one(module.vpc[*]) : tomap(false), one(data.aws_vpc.vpc[*])) } output "vpc_cidr" { @@ -52,7 +52,7 @@ output "vpc_cidr" { output "cluster" { - value = var.create_cluster ? one(module.cluster[*]) : (data.aws_eks_cluster.cluster[*]) + value = try(var.create_cluster ? one(module.cluster[*]) : tomap(false), one(data.aws_eks_cluster.cluster[*])) } output "cluster_service_ipv4_cidr" { From 77d0e6eb4dab9480b3cde8f38619bc52e84da1b3 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Thu, 7 Sep 2023 12:43:46 +0200 Subject: [PATCH 05/15] fix iam role Signed-off-by: David van der Spek --- bootstrap/terraform/aws-bootstrap/capa-sa.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/bootstrap/terraform/aws-bootstrap/capa-sa.tf b/bootstrap/terraform/aws-bootstrap/capa-sa.tf index 96cc73726..59c19fd70 100644 --- a/bootstrap/terraform/aws-bootstrap/capa-sa.tf +++ b/bootstrap/terraform/aws-bootstrap/capa-sa.tf @@ -269,6 +269,7 @@ data "aws_iam_policy_document" "capa_controller_eks" { "iam:DeleteRole", "iam:CreateRole", "iam:TagRole", + "iam:UntagRole", "iam:AttachRolePolicy", ] } From 03dab860590749b932a1e39ee564b97cbfe2adc8 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Thu, 7 Sep 2023 13:17:28 +0200 Subject: [PATCH 06/15] fix some output changes Signed-off-by: David van der Spek --- bootstrap/terraform/aws-bootstrap/existing.tf | 9 +++++++++ bootstrap/terraform/aws-bootstrap/locals.tf | 2 -- bootstrap/terraform/aws-bootstrap/output.tf | 4 ++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/bootstrap/terraform/aws-bootstrap/existing.tf b/bootstrap/terraform/aws-bootstrap/existing.tf index 2297d7824..f6d58002e 100644 --- a/bootstrap/terraform/aws-bootstrap/existing.tf +++ b/bootstrap/terraform/aws-bootstrap/existing.tf @@ -24,6 +24,15 @@ data "aws_subnet" "cluster_subnets" { id = tolist(one(data.aws_eks_cluster.cluster[*].vpc_config[0].subnet_ids))[count.index] } +data "aws_subnet" "private_subnets" { + count = length(local.private_subnet_ids) + id = local.private_subnet_ids[count.index] +} + +data "aws_subnet" "public_subnets" { + count = length(local.public_subnet_ids) + id = local.public_subnet_ids[count.index] +} data "aws_subnet" "worker_private_subnets" { count = length(local.worker_private_subnet_ids) diff --git a/bootstrap/terraform/aws-bootstrap/locals.tf b/bootstrap/terraform/aws-bootstrap/locals.tf index 9393ee06a..daa0f4df1 100644 --- a/bootstrap/terraform/aws-bootstrap/locals.tf +++ b/bootstrap/terraform/aws-bootstrap/locals.tf @@ -10,8 +10,6 @@ locals { locals { private_subnet_ids = var.create_cluster ? one(module.vpc[*].private_subnets_ids) : [for index, subnet in data.aws_subnet.cluster_subnets : subnet.id if contains(keys(subnet.tags), "kubernetes.io/role/internal-elb")] - private_subnet = var.create_cluster ? one(module.vpc[*].private_subnets_ids) : [for index, subnet in data.aws_subnet.cluster_subnets : subnet if contains(keys(subnet.tags), "kubernetes.io/role/internal-elb")] public_subnet_ids = var.create_cluster ? one(module.vpc[*].public_subnets_ids) : [for index, subnet in data.aws_subnet.cluster_subnets : subnet.id if contains(keys(subnet.tags), "kubernetes.io/role/elb")] - public_subnet = var.create_cluster ? one(module.vpc[*].public_subnets_ids) : [for index, subnet in data.aws_subnet.cluster_subnets : subnet if contains(keys(subnet.tags), "kubernetes.io/role/elb")] worker_private_subnet_ids = var.create_cluster ? one(module.vpc[*].worker_private_subnets_ids) : distinct(flatten([for index, group in data.aws_eks_node_group.cluster : group.subnet_ids ])) } diff --git a/bootstrap/terraform/aws-bootstrap/output.tf b/bootstrap/terraform/aws-bootstrap/output.tf index 9c3b87769..f544cf3a7 100644 --- a/bootstrap/terraform/aws-bootstrap/output.tf +++ b/bootstrap/terraform/aws-bootstrap/output.tf @@ -11,7 +11,7 @@ output "cluster_oidc_issuer_url" { } output "cluster_private_subnets" { - value = local.private_subnet + value = data.aws_subnet.private_subnets } output "cluster_worker_private_subnets" { @@ -19,7 +19,7 @@ output "cluster_worker_private_subnets" { } output "cluster_public_subnets" { - value = local.public_subnet + value = data.aws_subnet.public_subnets } output "cluster_private_subnet_ids" { From 9c22612a12fbae3ced83be92d035603a4abb49b6 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Thu, 7 Sep 2023 13:49:31 +0200 Subject: [PATCH 07/15] no cycle error Signed-off-by: David van der Spek --- bootstrap/terraform/aws-bootstrap/locals.tf | 4 ++-- bootstrap/terraform/aws-bootstrap/main.tf | 3 ++- bootstrap/terraform/aws-bootstrap/moved.tf | 8 ++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/bootstrap/terraform/aws-bootstrap/locals.tf b/bootstrap/terraform/aws-bootstrap/locals.tf index daa0f4df1..ab3e57013 100644 --- a/bootstrap/terraform/aws-bootstrap/locals.tf +++ b/bootstrap/terraform/aws-bootstrap/locals.tf @@ -2,8 +2,8 @@ locals { sts_principal = "sts.${data.aws_partition.current.dns_suffix}" create_vpc = var.create_cluster && var.create_vpc ? true : false vpc_id = var.create_cluster ? one(module.vpc[*].vpc_id) : one(data.aws_eks_cluster.cluster[*].vpc_config[0].vpc_id) - cluster_id = var.create_cluster ? one(module.cluster[*].cluster_id) : one(data.aws_eks_cluster.cluster[*].id) - cluster_config = try(var.create_cluster ? one(module.cluster[*].config_map_aws_auth) : tomap(false), {}) + cluster_id = var.create_cluster ? module.cluster.cluster_id : one(data.aws_eks_cluster.cluster[*].id) + cluster_config = try(var.create_cluster ? module.cluster.config_map_aws_auth : tomap(false), {}) cluster_oidc_issuer_url = var.create_cluster ? one(module.cluster[*].cluster_oidc_issuer_url) : one(data.aws_eks_cluster.cluster[*].identity[0].oidc.0.issuer) cluster_endpoint = var.create_cluster ? one(module.cluster[*].cluster_endpoint) : one(data.aws_eks_cluster.cluster[*].endpoint) } diff --git a/bootstrap/terraform/aws-bootstrap/main.tf b/bootstrap/terraform/aws-bootstrap/main.tf index b17e2cb5a..b24bb3f71 100644 --- a/bootstrap/terraform/aws-bootstrap/main.tf +++ b/bootstrap/terraform/aws-bootstrap/main.tf @@ -37,7 +37,7 @@ module "vpc" { } module "cluster" { - count = var.create_cluster ? 1 : 0 + # count = var.create_cluster ? 1 : 0 source = "github.com/pluralsh/terraform-aws-eks?ref=output-service-cidr" cluster_name = var.cluster_name @@ -48,6 +48,7 @@ module "cluster" { vpc_id = local.vpc_id enable_irsa = true write_kubeconfig = false + create_eks = var.create_cluster cluster_enabled_log_types = var.cluster_enabled_log_types cluster_log_retention_in_days = var.cluster_log_retention_in_days cluster_log_kms_key_id = var.cluster_log_kms_key_id diff --git a/bootstrap/terraform/aws-bootstrap/moved.tf b/bootstrap/terraform/aws-bootstrap/moved.tf index c2ed34899..eaa6c2016 100644 --- a/bootstrap/terraform/aws-bootstrap/moved.tf +++ b/bootstrap/terraform/aws-bootstrap/moved.tf @@ -1,7 +1,7 @@ -moved { - from = module.cluster - to = module.cluster[0] -} +# moved { +# from = module.cluster +# to = module.cluster[0] +# } moved { from = module.vpc From 4b35b3d9bb57ce34f9cf0ff161eeae193a43550c Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Thu, 7 Sep 2023 14:17:38 +0200 Subject: [PATCH 08/15] cleanup comments Signed-off-by: David van der Spek --- bootstrap/terraform/aws-bootstrap/main.tf | 2 -- bootstrap/terraform/aws-bootstrap/moved.tf | 5 ----- 2 files changed, 7 deletions(-) diff --git a/bootstrap/terraform/aws-bootstrap/main.tf b/bootstrap/terraform/aws-bootstrap/main.tf index b24bb3f71..6bf1c9799 100644 --- a/bootstrap/terraform/aws-bootstrap/main.tf +++ b/bootstrap/terraform/aws-bootstrap/main.tf @@ -37,8 +37,6 @@ module "vpc" { } module "cluster" { - # count = var.create_cluster ? 1 : 0 - source = "github.com/pluralsh/terraform-aws-eks?ref=output-service-cidr" cluster_name = var.cluster_name cluster_version = var.kubernetes_version diff --git a/bootstrap/terraform/aws-bootstrap/moved.tf b/bootstrap/terraform/aws-bootstrap/moved.tf index eaa6c2016..78171bc24 100644 --- a/bootstrap/terraform/aws-bootstrap/moved.tf +++ b/bootstrap/terraform/aws-bootstrap/moved.tf @@ -1,8 +1,3 @@ -# moved { -# from = module.cluster -# to = module.cluster[0] -# } - moved { from = module.vpc to = module.vpc[0] From 3e074b17c7b5989eb0421f4e2a235b90f75b1367 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Thu, 7 Sep 2023 14:28:10 +0200 Subject: [PATCH 09/15] update capi recipe for testing Signed-off-by: David van der Spek --- bootstrap/plural/recipes/aws-cluster-api-simple-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/plural/recipes/aws-cluster-api-simple-test.yaml b/bootstrap/plural/recipes/aws-cluster-api-simple-test.yaml index 2cc0b9596..141e8a1e2 100644 --- a/bootstrap/plural/recipes/aws-cluster-api-simple-test.yaml +++ b/bootstrap/plural/recipes/aws-cluster-api-simple-test.yaml @@ -16,7 +16,7 @@ sections: message: must begin with a lowercase letter, and can only contain lowercase letters, numbers or hyphens after items: - type: TERRAFORM - name: aws-bootstrap-cluster-api + name: aws-bootstrap - type: HELM name: bootstrap - type: HELM From eb6cf9431af39175dd01c28cfa4d4be3231bd959 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Fri, 8 Sep 2023 13:08:19 +0200 Subject: [PATCH 10/15] add missing role Signed-off-by: David van der Spek --- bootstrap/terraform/aws-bootstrap/capa-sa.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/bootstrap/terraform/aws-bootstrap/capa-sa.tf b/bootstrap/terraform/aws-bootstrap/capa-sa.tf index 59c19fd70..35cf8273b 100644 --- a/bootstrap/terraform/aws-bootstrap/capa-sa.tf +++ b/bootstrap/terraform/aws-bootstrap/capa-sa.tf @@ -96,6 +96,7 @@ data "aws_iam_policy_document" "capa_controller" { "elasticloadbalancing:CreateLoadBalancer", "elasticloadbalancing:ConfigureHealthCheck", "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:CreateTargetGroup", "elasticloadbalancing:DeleteTargetGroup", "elasticloadbalancing:DescribeLoadBalancers", "elasticloadbalancing:DescribeLoadBalancerAttributes", From 9fd3b9417c94075228121a8805dc47da3b6d9235 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Fri, 8 Sep 2023 13:17:34 +0200 Subject: [PATCH 11/15] fix node group output for CAPI clusters Signed-off-by: David van der Spek --- bootstrap/terraform/aws-bootstrap/output.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/terraform/aws-bootstrap/output.tf b/bootstrap/terraform/aws-bootstrap/output.tf index f544cf3a7..5beab5771 100644 --- a/bootstrap/terraform/aws-bootstrap/output.tf +++ b/bootstrap/terraform/aws-bootstrap/output.tf @@ -39,7 +39,7 @@ output "worker_role_arn" { } output "node_groups" { - value = try(var.create_cluster ? [for d in merge(one(module.single_az_node_groups[*].node_groups), one(module.multi_az_node_groups[*].node_groups)): d] : tomap(false), {}) + value = try(var.create_cluster ? [for d in merge(one(module.single_az_node_groups[*].node_groups), one(module.multi_az_node_groups[*].node_groups)): d] : tomap(false), one(data.aws_eks_node_group.cluster[*])) } output "vpc" { From 64a85df8482a3a13580899acc94881e2dbd2e188 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Fri, 8 Sep 2023 13:24:05 +0200 Subject: [PATCH 12/15] fix node output if capi Signed-off-by: David van der Spek --- bootstrap/terraform/aws-bootstrap/output.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/terraform/aws-bootstrap/output.tf b/bootstrap/terraform/aws-bootstrap/output.tf index 5beab5771..c040ef5d7 100644 --- a/bootstrap/terraform/aws-bootstrap/output.tf +++ b/bootstrap/terraform/aws-bootstrap/output.tf @@ -39,7 +39,7 @@ output "worker_role_arn" { } output "node_groups" { - value = try(var.create_cluster ? [for d in merge(one(module.single_az_node_groups[*].node_groups), one(module.multi_az_node_groups[*].node_groups)): d] : tomap(false), one(data.aws_eks_node_group.cluster[*])) + value = var.create_cluster ? [for d in merge(one(module.single_az_node_groups[*].node_groups), one(module.multi_az_node_groups[*].node_groups)): d] : one(data.aws_eks_node_group.cluster[*]) } output "vpc" { From 40ac29d1c15755217ada7040de74dbeb17969e8e Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Fri, 8 Sep 2023 13:28:04 +0200 Subject: [PATCH 13/15] node group output + role fix Signed-off-by: David van der Spek --- bootstrap/terraform/aws-bootstrap/capa-sa.tf | 1 + bootstrap/terraform/aws-bootstrap/output.tf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bootstrap/terraform/aws-bootstrap/capa-sa.tf b/bootstrap/terraform/aws-bootstrap/capa-sa.tf index 35cf8273b..bfc919d36 100644 --- a/bootstrap/terraform/aws-bootstrap/capa-sa.tf +++ b/bootstrap/terraform/aws-bootstrap/capa-sa.tf @@ -101,6 +101,7 @@ data "aws_iam_policy_document" "capa_controller" { "elasticloadbalancing:DescribeLoadBalancers", "elasticloadbalancing:DescribeLoadBalancerAttributes", "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeListeners", "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", "elasticloadbalancing:DescribeTags", "elasticloadbalancing:ModifyLoadBalancerAttributes", diff --git a/bootstrap/terraform/aws-bootstrap/output.tf b/bootstrap/terraform/aws-bootstrap/output.tf index c040ef5d7..9c949f8d3 100644 --- a/bootstrap/terraform/aws-bootstrap/output.tf +++ b/bootstrap/terraform/aws-bootstrap/output.tf @@ -39,7 +39,7 @@ output "worker_role_arn" { } output "node_groups" { - value = var.create_cluster ? [for d in merge(one(module.single_az_node_groups[*].node_groups), one(module.multi_az_node_groups[*].node_groups)): d] : one(data.aws_eks_node_group.cluster[*]) + value = var.create_cluster ? [for d in merge(one(module.single_az_node_groups[*].node_groups), one(module.multi_az_node_groups[*].node_groups)): d] : data.aws_eks_node_group.cluster[*] } output "vpc" { From 383b6e3d3f193d0935d226f49e15b6cd9347cb90 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Fri, 8 Sep 2023 14:06:14 +0200 Subject: [PATCH 14/15] more iam policy fixes Signed-off-by: David van der Spek --- bootstrap/terraform/aws-bootstrap/capa-sa.tf | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/bootstrap/terraform/aws-bootstrap/capa-sa.tf b/bootstrap/terraform/aws-bootstrap/capa-sa.tf index bfc919d36..4c4863637 100644 --- a/bootstrap/terraform/aws-bootstrap/capa-sa.tf +++ b/bootstrap/terraform/aws-bootstrap/capa-sa.tf @@ -93,21 +93,39 @@ data "aws_iam_policy_document" "capa_controller" { "ec2:TerminateInstances", "tag:GetResources", "elasticloadbalancing:AddTags", + "elasticloadbalancing:AttachLoadBalancerToSubnets", "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateLoadBalancerListeners", "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:CreateLoadBalancerPolicy", "elasticloadbalancing:DeleteLoadBalancer", "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:DescribeTargetHealth", "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteLoadBalancerListeners", "elasticloadbalancing:DescribeLoadBalancers", "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeTargetGroupAttributes", "elasticloadbalancing:DescribeTargetGroups", "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", "elasticloadbalancing:DescribeTags", "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:ModifyTargetGroupAttributes", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:ModifyListener", "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:DescribeLoadBalancerPolicies", "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener", "elasticloadbalancing:RemoveTags", + "elasticloadbalancing:SetSubnets", "autoscaling:DescribeAutoScalingGroups", "autoscaling:DescribeInstanceRefreshes", "ec2:CreateLaunchTemplate", From a75f932cadcb9b745c53df5330d8f7970bb5fe6b Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Tue, 12 Sep 2023 10:29:53 +0200 Subject: [PATCH 15/15] bump version Signed-off-by: David van der Spek --- bootstrap/terraform/aws-bootstrap/deps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/terraform/aws-bootstrap/deps.yaml b/bootstrap/terraform/aws-bootstrap/deps.yaml index a9c03297d..a7fc845cc 100644 --- a/bootstrap/terraform/aws-bootstrap/deps.yaml +++ b/bootstrap/terraform/aws-bootstrap/deps.yaml @@ -2,7 +2,7 @@ apiVersion: plural.sh/v1alpha1 kind: Dependencies metadata: description: Creates an EKS cluster and prepares it for bootstrapping - version: 0.1.53 + version: 0.1.54 spec: breaking: false dependencies: []