From b6e2492d3aa5ecf5bda82188e512dd824104de43 Mon Sep 17 00:00:00 2001 From: Mike VanDenburgh Date: Wed, 26 Jun 2024 18:33:41 -0400 Subject: [PATCH] Migrate to AWS Load Balancer Controller Resource: https://aws.amazon.com/blogs/containers/exposing-kubernetes-applications-part-3-nginx-ingress-controller/ --- .../aws-load-balancer-controller/release.yaml | 40 +++ k8s/production/sealed-secrets/release.yaml | 2 +- terraform/modules/spack_aws_k8s/eks.tf | 293 ++++++++++++++++++ 3 files changed, 334 insertions(+), 1 deletion(-) create mode 100644 k8s/production/aws-load-balancer-controller/release.yaml diff --git a/k8s/production/aws-load-balancer-controller/release.yaml b/k8s/production/aws-load-balancer-controller/release.yaml new file mode 100644 index 000000000..6b4856152 --- /dev/null +++ b/k8s/production/aws-load-balancer-controller/release.yaml @@ -0,0 +1,40 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: aws-load-balancer-controller + namespace: kube-system +spec: + interval: 10m + url: https://aws.github.io/eks-charts + +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: aws-load-balancer-controller + namespace: kube-system +spec: + interval: 10m + chart: + spec: + chart: aws-load-balancer-controller + version: 1.9.0 # aws-load-balancer-controller@v2.9.0 + sourceRef: + kind: HelmRepository + name: aws-load-balancer-controller + install: + crds: CreateReplace + upgrade: + crds: CreateReplace + valuesFrom: + # See terraform/modules/spack/eks.tf + - kind: ConfigMap + name: aws-lb-controller-config + valuesKey: values.yaml + values: + tolerations: + # The LB Controller should be scheduled on the initial managed nodegroup pods + # so that it is not dependent on Karpenter to be scheduled. + - key: "CriticalAddonsOnly" + operator: "Exists" diff --git a/k8s/production/sealed-secrets/release.yaml b/k8s/production/sealed-secrets/release.yaml index 6f607b9d7..de5d7c526 100644 --- a/k8s/production/sealed-secrets/release.yaml +++ b/k8s/production/sealed-secrets/release.yaml @@ -28,5 +28,5 @@ spec: spack.io/node-pool: base tolerations: - key: CriticalAddonsOnly - operator: Equal + operator: Exists effect: NoSchedule diff --git a/terraform/modules/spack_aws_k8s/eks.tf b/terraform/modules/spack_aws_k8s/eks.tf index 9a1785709..2f852e32d 100644 --- a/terraform/modules/spack_aws_k8s/eks.tf +++ b/terraform/modules/spack_aws_k8s/eks.tf @@ -296,3 +296,296 @@ resource "kubectl_manifest" "cluster_name_config_map" { cluster-name: ${module.eks.cluster_name} YAML } + + +resource "aws_iam_role" "load_balancer_controller" { + name = "AmazonEKSLoadBalancerControllerRole-${var.deployment_name}-${var.deployment_stage}" + assume_role_policy = jsonencode({ + "Version" : "2012-10-17", + "Statement" : [ + { + "Effect" : "Allow", + "Principal" : { + "Federated" : module.eks.oidc_provider_arn, + }, + "Action" : "sts:AssumeRoleWithWebIdentity", + "Condition" : { + "StringEquals" : { + "${module.eks.oidc_provider}:aud" : "sts.amazonaws.com", + "${module.eks.oidc_provider}:sub" : "system:serviceaccount:kube-system:aws-load-balancer-controller" + } + } + } + ] + }) +} + +resource "aws_iam_role_policy" "load_balancer_controller" { + name = "AmazonEKSLoadBalancerControllerPolicy-${var.deployment_name}-${var.deployment_stage}" + role = aws_iam_role.load_balancer_controller.id + # Copied from https://docs.aws.amazon.com/eks/latest/userguide/lbc-manifest.html#lbc-iam + policy = jsonencode({ + "Version" : "2012-10-17", + "Statement" : [ + { + "Effect" : "Allow", + "Action" : [ + "iam:CreateServiceLinkedRole" + ], + "Resource" : "*", + "Condition" : { + "StringEquals" : { + "iam:AWSServiceName" : "elasticloadbalancing.amazonaws.com" + } + } + }, + { + "Effect" : "Allow", + "Action" : [ + "ec2:DescribeAccountAttributes", + "ec2:DescribeAddresses", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeInternetGateways", + "ec2:DescribeVpcs", + "ec2:DescribeVpcPeeringConnections", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeInstances", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeTags", + "ec2:GetCoipPoolUsage", + "ec2:DescribeCoipPools", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeListenerCertificates", + "elasticloadbalancing:DescribeSSLPolicies", + "elasticloadbalancing:DescribeRules", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetGroupAttributes", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:DescribeTags", + "elasticloadbalancing:DescribeTrustStores", + "elasticloadbalancing:DescribeListenerAttributes" + ], + "Resource" : "*" + }, + { + "Effect" : "Allow", + "Action" : [ + "cognito-idp:DescribeUserPoolClient", + "acm:ListCertificates", + "acm:DescribeCertificate", + "iam:ListServerCertificates", + "iam:GetServerCertificate", + "waf-regional:GetWebACL", + "waf-regional:GetWebACLForResource", + "waf-regional:AssociateWebACL", + "waf-regional:DisassociateWebACL", + "wafv2:GetWebACL", + "wafv2:GetWebACLForResource", + "wafv2:AssociateWebACL", + "wafv2:DisassociateWebACL", + "shield:GetSubscriptionState", + "shield:DescribeProtection", + "shield:CreateProtection", + "shield:DeleteProtection" + ], + "Resource" : "*" + }, + { + "Effect" : "Allow", + "Action" : [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:RevokeSecurityGroupIngress" + ], + "Resource" : "*" + }, + { + "Effect" : "Allow", + "Action" : [ + "ec2:CreateSecurityGroup" + ], + "Resource" : "*" + }, + { + "Effect" : "Allow", + "Action" : [ + "ec2:CreateTags" + ], + "Resource" : "arn:aws:ec2:*:*:security-group/*", + "Condition" : { + "StringEquals" : { + "ec2:CreateAction" : "CreateSecurityGroup" + }, + "Null" : { + "aws:RequestTag/elbv2.k8s.aws/cluster" : "false" + } + } + }, + { + "Effect" : "Allow", + "Action" : [ + "ec2:CreateTags", + "ec2:DeleteTags" + ], + "Resource" : "arn:aws:ec2:*:*:security-group/*", + "Condition" : { + "Null" : { + "aws:RequestTag/elbv2.k8s.aws/cluster" : "true", + "aws:ResourceTag/elbv2.k8s.aws/cluster" : "false" + } + } + }, + { + "Effect" : "Allow", + "Action" : [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:RevokeSecurityGroupIngress", + "ec2:DeleteSecurityGroup" + ], + "Resource" : "*", + "Condition" : { + "Null" : { + "aws:ResourceTag/elbv2.k8s.aws/cluster" : "false" + } + } + }, + { + "Effect" : "Allow", + "Action" : [ + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateTargetGroup" + ], + "Resource" : "*", + "Condition" : { + "Null" : { + "aws:RequestTag/elbv2.k8s.aws/cluster" : "false" + } + } + }, + { + "Effect" : "Allow", + "Action" : [ + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:CreateRule", + "elasticloadbalancing:DeleteRule" + ], + "Resource" : "*" + }, + { + "Effect" : "Allow", + "Action" : [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:RemoveTags" + ], + "Resource" : [ + "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*" + ], + "Condition" : { + "Null" : { + "aws:RequestTag/elbv2.k8s.aws/cluster" : "true", + "aws:ResourceTag/elbv2.k8s.aws/cluster" : "false" + } + } + }, + { + "Effect" : "Allow", + "Action" : [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:RemoveTags" + ], + "Resource" : [ + "arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*" + ] + }, + { + "Effect" : "Allow", + "Action" : [ + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:SetIpAddressType", + "elasticloadbalancing:SetSecurityGroups", + "elasticloadbalancing:SetSubnets", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:ModifyTargetGroupAttributes", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:ModifyListenerAttributes" + ], + "Resource" : "*", + "Condition" : { + "Null" : { + "aws:ResourceTag/elbv2.k8s.aws/cluster" : "false" + } + } + }, + { + "Effect" : "Allow", + "Action" : [ + "elasticloadbalancing:AddTags" + ], + "Resource" : [ + "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*" + ], + "Condition" : { + "StringEquals" : { + "elasticloadbalancing:CreateAction" : [ + "CreateTargetGroup", + "CreateLoadBalancer" + ] + }, + "Null" : { + "aws:RequestTag/elbv2.k8s.aws/cluster" : "false" + } + } + }, + { + "Effect" : "Allow", + "Action" : [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets" + ], + "Resource" : "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*" + }, + { + "Effect" : "Allow", + "Action" : [ + "elasticloadbalancing:SetWebAcl", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:AddListenerCertificates", + "elasticloadbalancing:RemoveListenerCertificates", + "elasticloadbalancing:ModifyRule" + ], + "Resource" : "*" + } + ] + }) +} + +resource "kubectl_manifest" "load_balancer_controller" { + yaml_body = <<-YAML + apiVersion: v1 + kind: ConfigMap + metadata: + name: aws-lb-controller-config + namespace: kube-system + data: + values.yaml: | + clusterName: ${module.eks.cluster_name} + serviceAccount: + create: true + name: aws-load-balancer-controller + annotations: + eks.amazonaws.com/role-arn: ${aws_iam_role.load_balancer_controller.arn} + region: ${data.aws_region.current.name} + vpcId: ${module.vpc.vpc_id} + YAML +}