From ed6b13df779fd26df71fed94315da769e24d8509 Mon Sep 17 00:00:00 2001 From: David van der Spek <28541758+DavidSpek@users.noreply.github.com> Date: Tue, 12 Sep 2023 10:31:39 +0200 Subject: [PATCH] feat(bootstrap): TF changes for AWS CAPI migration (#837) * feat(bootstrap): TF changes for AWS CAPI migration Signed-off-by: David van der Spek * slightly cleanup locals Signed-off-by: David van der Spek * more initial cleanup Signed-off-by: David van der Spek * first possibly valid setup Signed-off-by: David van der Spek * fix iam role Signed-off-by: David van der Spek * fix some output changes Signed-off-by: David van der Spek * no cycle error Signed-off-by: David van der Spek * cleanup comments Signed-off-by: David van der Spek * update capi recipe for testing Signed-off-by: David van der Spek * add missing role Signed-off-by: David van der Spek * fix node group output for CAPI clusters Signed-off-by: David van der Spek * fix node output if capi Signed-off-by: David van der Spek * node group output + role fix Signed-off-by: David van der Spek * more iam policy fixes Signed-off-by: David van der Spek * bump version Signed-off-by: David van der Spek --------- Signed-off-by: David van der Spek --- .../recipes/aws-cluster-api-simple-test.yaml | 2 +- .../aws-bootstrap-cluster-api/deps.yaml | 11 - .../aws-bootstrap-cluster-api/main.tf | 7 - .../terraform.tfvars | 2 - .../aws-bootstrap-cluster-api/variables.tf | 10 - .../aws-bootstrap/aws-lb-controller.tf | 2 +- bootstrap/terraform/aws-bootstrap/capa-sa.tf | 390 ++++++++++++++++++ .../terraform/aws-bootstrap/certmanager.tf | 2 +- bootstrap/terraform/aws-bootstrap/deps.yaml | 3 +- bootstrap/terraform/aws-bootstrap/existing.tf | 40 +- .../aws-bootstrap/irsa-autoscaler.tf | 4 +- .../aws-bootstrap/irsa-externaldns.tf | 2 +- bootstrap/terraform/aws-bootstrap/locals.tf | 23 +- bootstrap/terraform/aws-bootstrap/main.tf | 22 +- bootstrap/terraform/aws-bootstrap/moved.tf | 34 ++ bootstrap/terraform/aws-bootstrap/output.tf | 20 +- .../terraform/aws-bootstrap/terraform.tfvars | 4 + .../terraform/aws-bootstrap/variables.tf | 10 + 18 files changed, 515 insertions(+), 73 deletions(-) delete mode 100644 bootstrap/terraform/aws-bootstrap-cluster-api/deps.yaml delete mode 100644 bootstrap/terraform/aws-bootstrap-cluster-api/main.tf delete mode 100644 bootstrap/terraform/aws-bootstrap-cluster-api/terraform.tfvars delete mode 100644 bootstrap/terraform/aws-bootstrap-cluster-api/variables.tf create mode 100644 bootstrap/terraform/aws-bootstrap/capa-sa.tf create mode 100644 bootstrap/terraform/aws-bootstrap/moved.tf diff --git a/bootstrap/plural/recipes/aws-cluster-api-simple-test.yaml b/bootstrap/plural/recipes/aws-cluster-api-simple-test.yaml index 2cc0b9596..141e8a1e2 100644 --- a/bootstrap/plural/recipes/aws-cluster-api-simple-test.yaml +++ b/bootstrap/plural/recipes/aws-cluster-api-simple-test.yaml @@ -16,7 +16,7 @@ sections: message: must begin with a lowercase letter, and can only contain lowercase letters, numbers or hyphens after items: - type: TERRAFORM - name: aws-bootstrap-cluster-api + name: aws-bootstrap - type: HELM name: bootstrap - type: HELM diff --git a/bootstrap/terraform/aws-bootstrap-cluster-api/deps.yaml b/bootstrap/terraform/aws-bootstrap-cluster-api/deps.yaml deleted file mode 100644 index 23b38d48e..000000000 --- a/bootstrap/terraform/aws-bootstrap-cluster-api/deps.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: plural.sh/v1alpha1 -kind: Dependencies -metadata: - description: Creates an EKS cluster and prepares it for bootstrapping - version: 0.1.1 -spec: - breaking: true - dependencies: [] - providers: - - aws - diff --git a/bootstrap/terraform/aws-bootstrap-cluster-api/main.tf b/bootstrap/terraform/aws-bootstrap-cluster-api/main.tf deleted file mode 100644 index 8a588bcdd..000000000 --- a/bootstrap/terraform/aws-bootstrap-cluster-api/main.tf +++ /dev/null @@ -1,7 +0,0 @@ -provider "aws" { - region = var.aws_region -} - -data "aws_eks_cluster" "cluster" { - name = var.cluster_name -} \ No newline at end of file diff --git a/bootstrap/terraform/aws-bootstrap-cluster-api/terraform.tfvars b/bootstrap/terraform/aws-bootstrap-cluster-api/terraform.tfvars deleted file mode 100644 index 95fdccd3c..000000000 --- a/bootstrap/terraform/aws-bootstrap-cluster-api/terraform.tfvars +++ /dev/null @@ -1,2 +0,0 @@ -aws_region = {{ .Region | quote }} -cluster_name = {{ .Cluster | quote }} \ No newline at end of file diff --git a/bootstrap/terraform/aws-bootstrap-cluster-api/variables.tf b/bootstrap/terraform/aws-bootstrap-cluster-api/variables.tf deleted file mode 100644 index 76efb1ca2..000000000 --- a/bootstrap/terraform/aws-bootstrap-cluster-api/variables.tf +++ /dev/null @@ -1,10 +0,0 @@ -variable "aws_region" { - type = string - default = "us-east-2" - description = "The region you wish to deploy to" -} - -variable "cluster_name" { - type = string - default = "plural" -} \ No newline at end of file diff --git a/bootstrap/terraform/aws-bootstrap/aws-lb-controller.tf b/bootstrap/terraform/aws-bootstrap/aws-lb-controller.tf index a45301848..6ae3eb6cd 100644 --- a/bootstrap/terraform/aws-bootstrap/aws-lb-controller.tf +++ b/bootstrap/terraform/aws-bootstrap/aws-lb-controller.tf @@ -258,4 +258,4 @@ resource "aws_iam_policy" "alb" { ] } POLICY -} \ No newline at end of file +} diff --git a/bootstrap/terraform/aws-bootstrap/capa-sa.tf b/bootstrap/terraform/aws-bootstrap/capa-sa.tf new file mode 100644 index 000000000..4c4863637 --- /dev/null +++ b/bootstrap/terraform/aws-bootstrap/capa-sa.tf @@ -0,0 +1,390 @@ +module "asummable_role_capa" { + # count = var.enable_cluster_capa ? 1 : 0 + + source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" + version = "3.14.0" + create_role = true + role_name = "${var.cluster_name}-capa-controller" + provider_url = replace(local.cluster_oidc_issuer_url, "https://", "") + role_policy_arns = [aws_iam_policy.capa_controller.arn, aws_iam_policy.capa_controller_eks.arn] + oidc_fully_qualified_subjects = ["system:serviceaccount:${var.namespace}:${var.capa_serviceaccount}", "system:serviceaccount:${var.namespace}:${var.capi_serviceaccount}"] +} + +resource "aws_iam_policy" "capa_controller" { + # count = var.enable_cluster_capa ? 1 : 0 + + name_prefix = "cluster-capa" + description = "EKS cluster api provider aws policy for cluster ${var.cluster_name}" + policy = data.aws_iam_policy_document.capa_controller.json +} + +resource "aws_iam_policy" "capa_controller_eks" { + # count = var.enable_cluster_capa ? 1 : 0 + + name_prefix = "cluster-capa" + description = "EKS cluster api provider aws policy for cluster ${var.cluster_name}" + policy = data.aws_iam_policy_document.capa_controller_eks.json +} + +data "aws_iam_policy_document" "capa_controller" { + statement { + sid = "" + effect = "Allow" + resources = ["*"] + + actions = [ + "ec2:AttachNetworkInterface", + "ec2:DetachNetworkInterface", + "ec2:AllocateAddress", + "ec2:AssignIpv6Addresses", + "ec2:AssignPrivateIpAddresses", + "ec2:UnassignPrivateIpAddresses", + "ec2:AssociateRouteTable", + "ec2:AttachInternetGateway", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateInternetGateway", + "ec2:CreateEgressOnlyInternetGateway", + "ec2:CreateNatGateway", + "ec2:CreateNetworkInterface", + "ec2:CreateRoute", + "ec2:CreateRouteTable", + "ec2:CreateSecurityGroup", + "ec2:CreateSubnet", + "ec2:CreateTags", + "ec2:CreateVpc", + "ec2:ModifyVpcAttribute", + "ec2:DeleteInternetGateway", + "ec2:DeleteEgressOnlyInternetGateway", + "ec2:DeleteNatGateway", + "ec2:DeleteRouteTable", + "ec2:ReplaceRoute", + "ec2:DeleteSecurityGroup", + "ec2:DeleteSubnet", + "ec2:DeleteTags", + "ec2:DeleteVpc", + "ec2:DescribeAccountAttributes", + "ec2:DescribeAddresses", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInternetGateways", + "ec2:DescribeEgressOnlyInternetGateways", + "ec2:DescribeInstanceTypes", + "ec2:DescribeImages", + "ec2:DescribeNatGateways", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeNetworkInterfaceAttribute", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs", + "ec2:DescribeVpcAttribute", + "ec2:DescribeVolumes", + "ec2:DescribeTags", + "ec2:DetachInternetGateway", + "ec2:DisassociateRouteTable", + "ec2:DisassociateAddress", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyNetworkInterfaceAttribute", + "ec2:ModifySubnetAttribute", + "ec2:ReleaseAddress", + "ec2:RevokeSecurityGroupIngress", + "ec2:RunInstances", + "ec2:TerminateInstances", + "tag:GetResources", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeTargetGroupAttributes", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:DescribeTags", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:ModifyTargetGroupAttributes", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener", + "elasticloadbalancing:RemoveTags", + "elasticloadbalancing:SetSubnets", + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeInstanceRefreshes", + "ec2:CreateLaunchTemplate", + "ec2:CreateLaunchTemplateVersion", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DeleteLaunchTemplate", + "ec2:DeleteLaunchTemplateVersions", + "ec2:DescribeKeyPairs", + "ec2:ModifyInstanceMetadataOptions", + ] + } + + statement { + sid = "" + effect = "Allow" + resources = ["arn:*:autoscaling:*:*:autoScalingGroup:*:autoScalingGroupName/*"] + + actions = [ + "autoscaling:CreateAutoScalingGroup", + "autoscaling:UpdateAutoScalingGroup", + "autoscaling:CreateOrUpdateTags", + "autoscaling:StartInstanceRefresh", + "autoscaling:DeleteAutoScalingGroup", + "autoscaling:DeleteTags", + ] + } + + statement { + sid = "" + effect = "Allow" + resources = ["arn:*:iam::*:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling"] + actions = ["iam:CreateServiceLinkedRole"] + + condition { + test = "StringLike" + variable = "iam:AWSServiceName" + values = ["autoscaling.amazonaws.com"] + } + } + + statement { + sid = "" + effect = "Allow" + resources = ["arn:*:iam::*:role/aws-service-role/elasticloadbalancing.amazonaws.com/AWSServiceRoleForElasticLoadBalancing"] + actions = ["iam:CreateServiceLinkedRole"] + + condition { + test = "StringLike" + variable = "iam:AWSServiceName" + values = ["elasticloadbalancing.amazonaws.com"] + } + } + + statement { + sid = "" + effect = "Allow" + resources = ["arn:*:iam::*:role/aws-service-role/spot.amazonaws.com/AWSServiceRoleForEC2Spot"] + actions = ["iam:CreateServiceLinkedRole"] + + condition { + test = "StringLike" + variable = "iam:AWSServiceName" + values = ["spot.amazonaws.com"] + } + } + + statement { + sid = "" + effect = "Allow" + resources = ["arn:*:iam::*:role/*.cluster-api-provider-aws.sigs.k8s.io"] + actions = ["iam:PassRole"] + } + + statement { + sid = "" + effect = "Allow" + resources = ["arn:*:secretsmanager:*:*:secret:aws.cluster.x-k8s.io/*"] + + actions = [ + "secretsmanager:CreateSecret", + "secretsmanager:DeleteSecret", + "secretsmanager:TagResource", + ] + } +} + +data "aws_iam_policy_document" "capa_controller_eks" { + + statement { + sid = "" + effect = "Allow" + resources = ["arn:*:ssm:*:*:parameter/aws/service/eks/optimized-ami/*"] + actions = ["ssm:GetParameter"] + } + + statement { + sid = "" + effect = "Allow" + resources = ["arn:*:iam::*:role/aws-service-role/eks.amazonaws.com/AWSServiceRoleForAmazonEKS"] + actions = ["iam:CreateServiceLinkedRole"] + + condition { + test = "StringLike" + variable = "iam:AWSServiceName" + values = ["eks.amazonaws.com"] + } + } + + statement { + sid = "" + effect = "Allow" + resources = ["arn:*:iam::*:role/aws-service-role/eks-nodegroup.amazonaws.com/AWSServiceRoleForAmazonEKSNodegroup"] + actions = ["iam:CreateServiceLinkedRole"] + + condition { + test = "StringLike" + variable = "iam:AWSServiceName" + values = ["eks-nodegroup.amazonaws.com"] + } + } + + statement { + sid = "" + effect = "Allow" + resources = ["arn:aws:iam::*:role/aws-service-role/eks-fargate-pods.amazonaws.com/AWSServiceRoleForAmazonEKSForFargate"] + actions = ["iam:CreateServiceLinkedRole"] + + condition { + test = "StringLike" + variable = "iam:AWSServiceName" + values = ["eks-fargate.amazonaws.com"] + } + } + + statement { + sid = "" + effect = "Allow" + resources = ["*"] + + actions = [ + "iam:ListOpenIDConnectProviders", + "iam:GetOpenIDConnectProvider", + "iam:CreateOpenIDConnectProvider", + "iam:AddClientIDToOpenIDConnectProvider", + "iam:UpdateOpenIDConnectProviderThumbprint", + "iam:DeleteOpenIDConnectProvider", + "iam:TagOpenIDConnectProvider", + ] + } + + statement { + sid = "" + effect = "Allow" + resources = ["arn:*:iam::*:role/*"] + + actions = [ + "iam:GetRole", + "iam:ListAttachedRolePolicies", + "iam:DetachRolePolicy", + "iam:DeleteRole", + "iam:CreateRole", + "iam:TagRole", + "iam:UntagRole", + "iam:AttachRolePolicy", + ] + } + + statement { + sid = "" + effect = "Allow" + resources = [ + "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", + "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", + "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", + "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", + ] + actions = ["iam:GetPolicy"] + } + + statement { + sid = "" + effect = "Allow" + + resources = [ + "arn:*:eks:*:*:cluster/*", + "arn:*:eks:*:*:nodegroup/*/*/*", + ] + + actions = [ + "eks:DescribeCluster", + "eks:ListClusters", + "eks:CreateCluster", + "eks:TagResource", + "eks:UpdateClusterVersion", + "eks:DeleteCluster", + "eks:UpdateClusterConfig", + "eks:UntagResource", + "eks:UpdateNodegroupVersion", + "eks:DescribeNodegroup", + "eks:DeleteNodegroup", + "eks:UpdateNodegroupConfig", + "eks:CreateNodegroup", + "eks:AssociateEncryptionConfig", + "eks:ListIdentityProviderConfigs", + "eks:AssociateIdentityProviderConfig", + "eks:DescribeIdentityProviderConfig", + "eks:DisassociateIdentityProviderConfig", + ] + } + + statement { + sid = "" + effect = "Allow" + resources = ["*"] + + actions = [ + "ec2:AssociateVpcCidrBlock", + "ec2:DisassociateVpcCidrBlock", + "eks:ListAddons", + "eks:CreateAddon", + "eks:DescribeAddonVersions", + "eks:DescribeAddon", + "eks:DeleteAddon", + "eks:UpdateAddon", + "eks:TagResource", + "eks:DescribeFargateProfile", + "eks:CreateFargateProfile", + "eks:DeleteFargateProfile", + ] + } + + statement { + sid = "" + effect = "Allow" + resources = ["*"] + actions = ["iam:PassRole"] + + condition { + test = "StringEquals" + variable = "iam:PassedToService" + values = ["eks.amazonaws.com"] + } + } + + statement { + sid = "" + effect = "Allow" + resources = ["*"] + + actions = [ + "kms:CreateGrant", + "kms:DescribeKey", + ] + + condition { + test = "ForAnyValue:StringLike" + variable = "kms:ResourceAliases" + values = ["alias/cluster-api-provider-aws-*"] + } + } +} diff --git a/bootstrap/terraform/aws-bootstrap/certmanager.tf b/bootstrap/terraform/aws-bootstrap/certmanager.tf index 0d5657279..da9ffc540 100644 --- a/bootstrap/terraform/aws-bootstrap/certmanager.tf +++ b/bootstrap/terraform/aws-bootstrap/certmanager.tf @@ -36,4 +36,4 @@ resource "aws_iam_policy" "certmanager" { ] } POLICY -} \ No newline at end of file +} diff --git a/bootstrap/terraform/aws-bootstrap/deps.yaml b/bootstrap/terraform/aws-bootstrap/deps.yaml index 7cce61471..a7fc845cc 100644 --- a/bootstrap/terraform/aws-bootstrap/deps.yaml +++ b/bootstrap/terraform/aws-bootstrap/deps.yaml @@ -2,13 +2,14 @@ apiVersion: plural.sh/v1alpha1 kind: Dependencies metadata: description: Creates an EKS cluster and prepares it for bootstrapping - version: 0.1.53 + version: 0.1.54 spec: breaking: false dependencies: [] providers: - aws outputs: + capa_iam_role_arn: capa_iam_role_arn endpoint: cluster_endpoint cluster_private_subnets: cluster_private_subnets cluster_worker_private_subnets: cluster_worker_private_subnets diff --git a/bootstrap/terraform/aws-bootstrap/existing.tf b/bootstrap/terraform/aws-bootstrap/existing.tf index eef978d70..f6d58002e 100644 --- a/bootstrap/terraform/aws-bootstrap/existing.tf +++ b/bootstrap/terraform/aws-bootstrap/existing.tf @@ -1,24 +1,40 @@ data "aws_eks_cluster" "cluster" { - count = var.create_cluster ? 0 : 1 - name = var.cluster_name + count = var.create_cluster ? 0 : 1 + name = var.cluster_name } data "aws_vpc" "vpc" { - count = var.create_cluster ? 0 : 1 - id = local.vpc_id + count = var.create_cluster ? 0 : 1 + id = local.vpc_id } -data "aws_subnet" "worker_private_subnets" { - count = length(local.worker_private_subnet_ids) - id = local.worker_private_subnet_ids[count.index] +data "aws_eks_node_groups" "cluster" { + count = var.create_cluster ? 0 : 1 + cluster_name = var.cluster_name +} + +data "aws_eks_node_group" "cluster" { + count = var.create_cluster ? 0 : length(one(data.aws_eks_node_groups.cluster[*].names)) + cluster_name = var.cluster_name + node_group_name = tolist(one(data.aws_eks_node_groups.cluster[*].names))[count.index] +} + +data "aws_subnet" "cluster_subnets" { + count = var.create_cluster ? 0 : length(one(data.aws_eks_cluster.cluster[*].vpc_config[0].subnet_ids)) + id = tolist(one(data.aws_eks_cluster.cluster[*].vpc_config[0].subnet_ids))[count.index] } data "aws_subnet" "private_subnets" { - count = length(local.private_subnet_ids) - id = local.private_subnet_ids[count.index] + count = length(local.private_subnet_ids) + id = local.private_subnet_ids[count.index] } - + data "aws_subnet" "public_subnets" { - count = length(local.public_subnet_ids) - id = local.public_subnet_ids[count.index] + count = length(local.public_subnet_ids) + id = local.public_subnet_ids[count.index] +} + +data "aws_subnet" "worker_private_subnets" { + count = length(local.worker_private_subnet_ids) + id = local.worker_private_subnet_ids[count.index] } diff --git a/bootstrap/terraform/aws-bootstrap/irsa-autoscaler.tf b/bootstrap/terraform/aws-bootstrap/irsa-autoscaler.tf index 5a534f962..5cc29df3e 100644 --- a/bootstrap/terraform/aws-bootstrap/irsa-autoscaler.tf +++ b/bootstrap/terraform/aws-bootstrap/irsa-autoscaler.tf @@ -50,7 +50,7 @@ data "aws_iam_policy_document" "cluster_autoscaler" { condition { test = "StringEquals" - variable = "autoscaling:ResourceTag/kubernetes.io/cluster/${module.cluster.cluster_id}" + variable = "autoscaling:ResourceTag/kubernetes.io/cluster/${local.cluster_id}" values = ["owned"] } @@ -60,4 +60,4 @@ data "aws_iam_policy_document" "cluster_autoscaler" { values = ["true"] } } -} \ No newline at end of file +} diff --git a/bootstrap/terraform/aws-bootstrap/irsa-externaldns.tf b/bootstrap/terraform/aws-bootstrap/irsa-externaldns.tf index 527d9b9bd..0051d5d81 100644 --- a/bootstrap/terraform/aws-bootstrap/irsa-externaldns.tf +++ b/bootstrap/terraform/aws-bootstrap/irsa-externaldns.tf @@ -37,4 +37,4 @@ data "aws_iam_policy_document" "externaldns" { resources = ["*"] } -} \ No newline at end of file +} diff --git a/bootstrap/terraform/aws-bootstrap/locals.tf b/bootstrap/terraform/aws-bootstrap/locals.tf index 9bcb0a2b5..ab3e57013 100644 --- a/bootstrap/terraform/aws-bootstrap/locals.tf +++ b/bootstrap/terraform/aws-bootstrap/locals.tf @@ -1,12 +1,15 @@ locals { - sts_principal = "sts.${data.aws_partition.current.dns_suffix}" - create_vpc = var.create_cluster && var.create_vpc ? true : false - private_subnet_ids = var.create_cluster ? module.vpc.private_subnets_ids : var.private_subnet_ids - public_subnet_ids = var.create_cluster ? module.vpc.public_subnets_ids : var.public_subnet_ids - worker_private_subnet_ids = var.create_cluster ? module.vpc.worker_private_subnets_ids : var.worker_private_subnet_ids - vpc_id = var.create_cluster ? module.vpc.vpc_id : data.aws_eks_cluster.cluster[0].vpc_config[0].vpc_id - cluster_id = var.create_cluster ? module.cluster.cluster_id : data.aws_eks_cluster.cluster[0].id - cluster_config = try(var.create_cluster ? module.cluster.config_map_aws_auth : tomap(false), {}) - cluster_oidc_issuer_url = var.create_cluster ? module.cluster.cluster_oidc_issuer_url : data.aws_eks_cluster.cluster[0].identity[0].oidc.0.issuer - cluster_endpoint = var.create_cluster ? module.cluster.cluster_endpoint : data.aws_eks_cluster.cluster[0].endpoint + sts_principal = "sts.${data.aws_partition.current.dns_suffix}" + create_vpc = var.create_cluster && var.create_vpc ? true : false + vpc_id = var.create_cluster ? one(module.vpc[*].vpc_id) : one(data.aws_eks_cluster.cluster[*].vpc_config[0].vpc_id) + cluster_id = var.create_cluster ? module.cluster.cluster_id : one(data.aws_eks_cluster.cluster[*].id) + cluster_config = try(var.create_cluster ? module.cluster.config_map_aws_auth : tomap(false), {}) + cluster_oidc_issuer_url = var.create_cluster ? one(module.cluster[*].cluster_oidc_issuer_url) : one(data.aws_eks_cluster.cluster[*].identity[0].oidc.0.issuer) + cluster_endpoint = var.create_cluster ? one(module.cluster[*].cluster_endpoint) : one(data.aws_eks_cluster.cluster[*].endpoint) +} + +locals { + private_subnet_ids = var.create_cluster ? one(module.vpc[*].private_subnets_ids) : [for index, subnet in data.aws_subnet.cluster_subnets : subnet.id if contains(keys(subnet.tags), "kubernetes.io/role/internal-elb")] + public_subnet_ids = var.create_cluster ? one(module.vpc[*].public_subnets_ids) : [for index, subnet in data.aws_subnet.cluster_subnets : subnet.id if contains(keys(subnet.tags), "kubernetes.io/role/elb")] + worker_private_subnet_ids = var.create_cluster ? one(module.vpc[*].worker_private_subnets_ids) : distinct(flatten([for index, group in data.aws_eks_node_group.cluster : group.subnet_ids ])) } diff --git a/bootstrap/terraform/aws-bootstrap/main.tf b/bootstrap/terraform/aws-bootstrap/main.tf index 0075bd2e4..6bf1c9799 100644 --- a/bootstrap/terraform/aws-bootstrap/main.tf +++ b/bootstrap/terraform/aws-bootstrap/main.tf @@ -3,6 +3,8 @@ data "aws_availability_zones" "available" {} data "aws_caller_identity" "current" {} module "vpc" { + count = local.create_vpc ? 1 : 0 + source = "github.com/pluralsh/terraform-aws-vpc?ref=worker_subnet" name = var.vpc_name cidr = var.vpc_cidr @@ -12,7 +14,6 @@ module "vpc" { worker_private_subnets = var.worker_private_subnets enable_dns_hostnames = true enable_ipv6 = true - create_vpc = local.create_vpc database_subnets = var.database_subnets @@ -59,15 +60,17 @@ module "cluster" { } module "single_az_node_groups" { + count = var.create_cluster ? 1 : 0 + source = "github.com/pluralsh/module-library//terraform/eks-node-groups/single-az-node-groups?ref=20e64863ffc5e361045db8e6b81b9d244a55809e" cluster_name = var.cluster_name - default_iam_role_arn = module.cluster.worker_iam_role_arn + default_iam_role_arn = one(module.cluster[*].worker_iam_role_arn) tags = {} node_groups_defaults = var.node_groups_defaults - node_groups = try(var.create_cluster ? var.single_az_node_groups : tomap(false), {}) + node_groups = var.single_az_node_groups set_desired_size = false - private_subnets = var.create_cluster ? module.vpc.worker_private_subnets : [] + private_subnets = one(module.vpc[*].worker_private_subnets) ng_depends_on = [ local.cluster_config @@ -75,13 +78,15 @@ module "single_az_node_groups" { } module "multi_az_node_groups" { + count = var.create_cluster ? 1 : 0 + source = "github.com/pluralsh/module-library//terraform/eks-node-groups/multi-az-node-groups?ref=20e64863ffc5e361045db8e6b81b9d244a55809e" cluster_name = var.cluster_name - default_iam_role_arn = module.cluster.worker_iam_role_arn + default_iam_role_arn = one(module.cluster[*].worker_iam_role_arn) tags = {} node_groups_defaults = var.node_groups_defaults - node_groups = try(var.create_cluster ? var.multi_az_node_groups : tomap(false), {}) + node_groups = var.multi_az_node_groups set_desired_size = false private_subnet_ids = local.worker_private_subnet_ids @@ -92,6 +97,7 @@ module "multi_az_node_groups" { resource "aws_eks_addon" "vpc_cni" { count = var.create_cluster ? 1 : 0 + cluster_name = local.cluster_id addon_name = "vpc-cni" addon_version = var.vpc_cni_addon_version @@ -107,6 +113,7 @@ resource "aws_eks_addon" "vpc_cni" { resource "aws_eks_addon" "core_dns" { count = var.create_cluster ? 1 : 0 + cluster_name = local.cluster_id addon_name = "coredns" addon_version = var.core_dns_addon_version @@ -122,6 +129,7 @@ resource "aws_eks_addon" "core_dns" { resource "aws_eks_addon" "kube_proxy" { count = var.create_cluster ? 1 : 0 + cluster_name = local.cluster_id addon_name = "kube-proxy" addon_version = var.kube_proxy_addon_version @@ -136,6 +144,8 @@ resource "aws_eks_addon" "kube_proxy" { } resource "kubernetes_namespace" "bootstrap" { + count = var.create_cluster ? 1 : 0 + metadata { name = "bootstrap" labels = { diff --git a/bootstrap/terraform/aws-bootstrap/moved.tf b/bootstrap/terraform/aws-bootstrap/moved.tf new file mode 100644 index 000000000..78171bc24 --- /dev/null +++ b/bootstrap/terraform/aws-bootstrap/moved.tf @@ -0,0 +1,34 @@ +moved { + from = module.vpc + to = module.vpc[0] +} + +moved { + from = module.single_az_node_groups + to = module.single_az_node_groups[0] +} + +moved { + from = module.multi_az_node_groups + to = module.multi_az_node_groups[0] +} + +moved { + from = aws_eks_addon.vpc_cni + to = aws_eks_addon.vpc_cni[0] +} + +moved { + from = aws_eks_addon.core_dns + to = aws_eks_addon.core_dns[0] +} + +moved { + from = aws_eks_addon.kube_proxy + to = aws_eks_addon.kube_proxy[0] +} + +moved { + from = kubernetes_namespace.bootstrap + to = kubernetes_namespace.bootstrap[0] +} diff --git a/bootstrap/terraform/aws-bootstrap/output.tf b/bootstrap/terraform/aws-bootstrap/output.tf index d10f42048..9c949f8d3 100644 --- a/bootstrap/terraform/aws-bootstrap/output.tf +++ b/bootstrap/terraform/aws-bootstrap/output.tf @@ -1,4 +1,3 @@ - output "cluster_name" { value = local.cluster_id } @@ -36,26 +35,31 @@ output "cluster_public_subnet_ids" { } output "worker_role_arn" { - value = module.cluster.worker_iam_role_arn + value = var.create_cluster ? one(module.cluster[*].worker_iam_role_arn) : data.aws_eks_node_group.cluster[0].node_role_arn } output "node_groups" { - value = [for d in merge(module.single_az_node_groups.node_groups, module.multi_az_node_groups.node_groups): d] + value = var.create_cluster ? [for d in merge(one(module.single_az_node_groups[*].node_groups), one(module.multi_az_node_groups[*].node_groups)): d] : data.aws_eks_node_group.cluster[*] } output "vpc" { - value = try(var.create_cluster ? module.vpc : tomap(false), data.aws_vpc.vpc[0]) + value = try(var.create_cluster ? one(module.vpc[*]) : tomap(false), one(data.aws_vpc.vpc[*])) } output "vpc_cidr" { - value = var.create_cluster ? module.vpc.vpc_cidr_block : data.aws_vpc.vpc[0].cidr_block + value = var.create_cluster ? one(module.vpc[*].vpc_cidr_block) : one(data.aws_vpc.vpc[*].cidr_block) } output "cluster" { - value = try(var.create_cluster ? module.cluster : tomap(false), data.aws_eks_cluster.cluster[0]) + value = try(var.create_cluster ? one(module.cluster[*]) : tomap(false), one(data.aws_eks_cluster.cluster[*])) } output "cluster_service_ipv4_cidr" { - value = var.create_cluster ? module.cluster.cluster_service_ipv4_cidr : data.aws_eks_cluster.cluster[0].kubernetes_network_config[0].service_ipv4_cidr -} \ No newline at end of file + value = var.create_cluster ? one(module.cluster[*].cluster_service_ipv4_cidr) : one(data.aws_eks_cluster.cluster[*].kubernetes_network_config[0].service_ipv4_cidr) +} + +output "capa_iam_role_arn" { + description = "ARN of IAM role that allows access to the Harbor S3 buckets." + value = module.asummable_role_capa.this_iam_role_arn +} diff --git a/bootstrap/terraform/aws-bootstrap/terraform.tfvars b/bootstrap/terraform/aws-bootstrap/terraform.tfvars index d827a452f..437a4acb8 100644 --- a/bootstrap/terraform/aws-bootstrap/terraform.tfvars +++ b/bootstrap/terraform/aws-bootstrap/terraform.tfvars @@ -1,5 +1,9 @@ vpc_name = {{ .Values.vpc_name | quote }} cluster_name = {{ .Cluster | quote }} +{{- if eq .ClusterAPI true }} +create_cluster = false +enable_irsa = false +{{- end }} map_roles = [ { diff --git a/bootstrap/terraform/aws-bootstrap/variables.tf b/bootstrap/terraform/aws-bootstrap/variables.tf index 1f6fa1319..c9529e214 100644 --- a/bootstrap/terraform/aws-bootstrap/variables.tf +++ b/bootstrap/terraform/aws-bootstrap/variables.tf @@ -368,3 +368,13 @@ variable "aws_region" { default = "us-east-2" description = "The region you wish to deploy to" } + +variable "capa_serviceaccount" { + type = string + default = "bootstrap-cluster-api-provider-aws" +} + +variable "capi_serviceaccount" { + type = string + default = "bootstrap-cluster-api-core" +}