From 400b55b0e0d80907da9f7ad333ad13176b743751 Mon Sep 17 00:00:00 2001 From: Smaine Kahlouch Date: Sun, 29 Oct 2023 10:08:52 +0100 Subject: [PATCH] refactor(gapi): private gateway --- clusters/mycluster-0/flux-config.yaml | 3 - clusters/mycluster-0/infrastructure.yaml | 12 -- clusters/mycluster-0/observability.yaml | 8 +- clusters/mycluster-0/security.yaml | 3 - flux-config/base/weave-gitops/httproute.yaml | 2 +- .../base/crossplane/controller/release.yaml | 2 +- .../crossplane/providers/provider-iam.yaml | 2 +- .../crossplane/providers/provider-s3.yaml | 2 +- infrastructure/base/gapi/kustomization.yaml | 3 +- ...way.yaml => platform-private-gateway.yaml} | 4 +- .../base/gapi/platform-public-gateway.yaml | 26 ++++ .../grafana-dashboards/kubernetes_logs.json | 34 ++++- .../kube-prometheus-stack/helmrelease.yaml | 2 +- .../base/kube-prometheus-stack/httproute.yaml | 2 +- observability/base/loki/helmrelease.yaml | 5 + .../base/vector-agent/helmrelease.yaml | 3 +- security/mycluster-0/kustomization.yaml | 3 +- ...=> platform-private-gw-clusterpolicy.yaml} | 8 +- .../platform-public-gw-clusterpolicy.yaml | 29 +++++ taskfile.yaml | 1 + terraform/eks/data.tf | 3 + terraform/eks/flux.tf | 18 +-- terraform/eks/helm.tf | 4 +- .../eks/helm_values/aws-ebs-csi-driver.yaml | 4 + terraform/eks/iam.tf | 119 +++++++++++++++++- terraform/eks/kubernetes.tf | 3 + terraform/eks/main.tf | 2 +- terraform/eks/providers.tf | 3 +- terraform/eks/variables.tf | 16 ++- terraform/eks/versions.tf | 4 + 30 files changed, 262 insertions(+), 68 deletions(-) rename infrastructure/base/gapi/{platform-gateway.yaml => platform-private-gateway.yaml} (90%) create mode 100644 infrastructure/base/gapi/platform-public-gateway.yaml rename security/mycluster-0/{platform-gw-clusterpolicy.yaml => platform-private-gw-clusterpolicy.yaml} (76%) create mode 100644 security/mycluster-0/platform-public-gw-clusterpolicy.yaml diff --git a/clusters/mycluster-0/flux-config.yaml b/clusters/mycluster-0/flux-config.yaml index 2a96fdda..50e5a377 100644 --- a/clusters/mycluster-0/flux-config.yaml +++ b/clusters/mycluster-0/flux-config.yaml @@ -15,11 +15,8 @@ spec: substitute: domain_name: "cloud.ogenki.io" substituteFrom: - - kind: ConfigMap - name: eks-mycluster-0-vars - kind: Secret name: eks-mycluster-0-vars - optional: true dependsOn: - name: crds healthChecks: diff --git a/clusters/mycluster-0/infrastructure.yaml b/clusters/mycluster-0/infrastructure.yaml index 831f0421..491cbc46 100644 --- a/clusters/mycluster-0/infrastructure.yaml +++ b/clusters/mycluster-0/infrastructure.yaml @@ -13,11 +13,8 @@ spec: name: flux-system postBuild: substituteFrom: - - kind: ConfigMap - name: eks-mycluster-0-vars - kind: Secret name: eks-mycluster-0-vars - optional: true healthChecks: - apiVersion: helm.toolkit.fluxcd.io/v1beta1 kind: HelmRelease @@ -40,11 +37,8 @@ spec: name: flux-system postBuild: substituteFrom: - - kind: ConfigMap - name: eks-mycluster-0-vars - kind: Secret name: eks-mycluster-0-vars - optional: true dependsOn: - name: crossplane-controller --- @@ -62,11 +56,8 @@ spec: name: flux-system postBuild: substituteFrom: - - kind: ConfigMap - name: eks-mycluster-0-vars - kind: Secret name: eks-mycluster-0-vars - optional: true dependsOn: - name: crossplane-providers --- @@ -84,11 +75,8 @@ spec: substitute: domain_name: "cloud.ogenki.io" substituteFrom: - - kind: ConfigMap - name: eks-mycluster-0-vars - kind: Secret name: eks-mycluster-0-vars - optional: true sourceRef: kind: GitRepository name: flux-system diff --git a/clusters/mycluster-0/observability.yaml b/clusters/mycluster-0/observability.yaml index b4389303..ea15fe53 100644 --- a/clusters/mycluster-0/observability.yaml +++ b/clusters/mycluster-0/observability.yaml @@ -15,13 +15,11 @@ spec: substitute: domain_name: "cloud.ogenki.io" substituteFrom: - - kind: ConfigMap - name: eks-mycluster-0-vars - kind: Secret name: eks-mycluster-0-vars - optional: true dependsOn: - name: crds + - name: security healthChecks: - apiVersion: helm.toolkit.fluxcd.io/v1beta1 kind: HelmRelease @@ -31,3 +29,7 @@ spec: kind: HelmRelease name: loki namespace: observability + - apiVersion: helm.toolkit.fluxcd.io/v1beta1 + kind: HelmRelease + name: vector-agent + namespace: observability diff --git a/clusters/mycluster-0/security.yaml b/clusters/mycluster-0/security.yaml index 297389ac..29a2e52c 100644 --- a/clusters/mycluster-0/security.yaml +++ b/clusters/mycluster-0/security.yaml @@ -14,11 +14,8 @@ spec: substitute: domain_name: "cloud.ogenki.io" substituteFrom: - - kind: ConfigMap - name: eks-mycluster-0-vars - kind: Secret name: eks-mycluster-0-vars - optional: true dependsOn: - name: crds healthChecks: diff --git a/flux-config/base/weave-gitops/httproute.yaml b/flux-config/base/weave-gitops/httproute.yaml index 2ea63baf..1029a805 100644 --- a/flux-config/base/weave-gitops/httproute.yaml +++ b/flux-config/base/weave-gitops/httproute.yaml @@ -4,7 +4,7 @@ metadata: name: weave-gitops spec: parentRefs: - - name: platform + - name: platform-private namespace: infrastructure hostnames: - "gitops-${cluster_name}.priv.${domain_name}" diff --git a/infrastructure/base/crossplane/controller/release.yaml b/infrastructure/base/crossplane/controller/release.yaml index bd3a9153..8c372a83 100644 --- a/infrastructure/base/crossplane/controller/release.yaml +++ b/infrastructure/base/crossplane/controller/release.yaml @@ -11,7 +11,7 @@ spec: kind: HelmRepository name: crossplane namespace: crossplane-system - version: "1.13.2" + version: "1.14.0" interval: 10m0s install: remediation: diff --git a/infrastructure/base/crossplane/providers/provider-iam.yaml b/infrastructure/base/crossplane/providers/provider-iam.yaml index 5b3bae1e..ad03dadf 100644 --- a/infrastructure/base/crossplane/providers/provider-iam.yaml +++ b/infrastructure/base/crossplane/providers/provider-iam.yaml @@ -3,6 +3,6 @@ kind: Provider metadata: name: provider-aws-iam spec: - package: xpkg.upbound.io/upbound/provider-aws-iam:v0.42.0 + package: xpkg.upbound.io/upbound/provider-aws-iam:v0.43.1 controllerConfigRef: name: aws-config diff --git a/infrastructure/base/crossplane/providers/provider-s3.yaml b/infrastructure/base/crossplane/providers/provider-s3.yaml index 585782a5..b710bfd3 100644 --- a/infrastructure/base/crossplane/providers/provider-s3.yaml +++ b/infrastructure/base/crossplane/providers/provider-s3.yaml @@ -3,6 +3,6 @@ kind: Provider metadata: name: provider-aws-s3 spec: - package: xpkg.upbound.io/upbound/provider-aws-s3:v0.42.0 + package: xpkg.upbound.io/upbound/provider-aws-s3:v0.43.1 controllerConfigRef: name: aws-config diff --git a/infrastructure/base/gapi/kustomization.yaml b/infrastructure/base/gapi/kustomization.yaml index 2fcbde52..9ddc36d8 100644 --- a/infrastructure/base/gapi/kustomization.yaml +++ b/infrastructure/base/gapi/kustomization.yaml @@ -3,4 +3,5 @@ kind: Kustomization namespace: infrastructure resources: - - platform-gateway.yaml + - platform-private-gateway.yaml + - platform-public-gateway.yaml diff --git a/infrastructure/base/gapi/platform-gateway.yaml b/infrastructure/base/gapi/platform-private-gateway.yaml similarity index 90% rename from infrastructure/base/gapi/platform-gateway.yaml rename to infrastructure/base/gapi/platform-private-gateway.yaml index 14d6164b..7498752d 100644 --- a/infrastructure/base/gapi/platform-gateway.yaml +++ b/infrastructure/base/gapi/platform-private-gateway.yaml @@ -1,7 +1,7 @@ apiVersion: gateway.networking.k8s.io/v1beta1 kind: Gateway metadata: - name: platform + name: platform-private annotations: cert-manager.io/cluster-issuer: private-ca spec: @@ -24,4 +24,4 @@ spec: tls: mode: Terminate certificateRefs: - - name: platform-tls + - name: platform-private-tls diff --git a/infrastructure/base/gapi/platform-public-gateway.yaml b/infrastructure/base/gapi/platform-public-gateway.yaml new file mode 100644 index 00000000..4b1ad1d9 --- /dev/null +++ b/infrastructure/base/gapi/platform-public-gateway.yaml @@ -0,0 +1,26 @@ +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: Gateway +metadata: + name: platform-public + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod +spec: + gatewayClassName: cilium + listeners: + - name: http + hostname: "*.${domain_name}" + port: 443 + protocol: HTTPS + allowedRoutes: + namespaces: + from: Selector + selector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: + - observability + tls: + mode: Terminate + certificateRefs: + - name: platform-public-tls diff --git a/observability/base/kube-prometheus-stack/grafana-dashboards/kubernetes_logs.json b/observability/base/kube-prometheus-stack/grafana-dashboards/kubernetes_logs.json index 22008383..e0091503 100644 --- a/observability/base/kube-prometheus-stack/grafana-dashboards/kubernetes_logs.json +++ b/observability/base/kube-prometheus-stack/grafana-dashboards/kubernetes_logs.json @@ -137,7 +137,7 @@ }, "targets": [ { - "expr": "sum(count_over_time({namespace=~\"$namespace\", node_name=~\"$node_name\", pod_owner=~\"$pod_owner\", pod_name=~\"$pod_name\", container_name=~\"$container_name\"} |= \"$query\" [$__interval]))", + "expr": "sum(count_over_time({namespace=~\"$namespace\", node_name=~\"$node_name\", pod_owner=~\"$pod_owner\", pod_name=~\"$pod_name\", container_name=~\"$container_name\", stream=\"$stream\"} |= \"$query\" [$__interval]))", "instant": false, "legendFormat": "Log count", "range": true, @@ -168,7 +168,7 @@ }, "targets": [ { - "expr": "{namespace=~\"$namespace\", node_name=~\"$node_name\", pod_owner=~\"$pod_owner\", pod_name=~\"$pod_name\", container_name=~\"$container_name\"} |= \"$query\"", + "expr": "{namespace=~\"$namespace\", node_name=~\"$node_name\", pod_owner=~\"$pod_owner\", pod_name=~\"$pod_name\", container_name=~\"$container_name\", stream=~\"$stream\"} |= \"$query\"", "refId": "A" } ], @@ -330,6 +330,36 @@ "skipUrlSync": false, "sort": 1, "type": "query" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "${DS_LOKI}", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "stream", + "multi": false, + "name": "stream", + "options": [], + "query": { + "label": "stream", + "stream": "", + "type": 1 + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" } ] }, diff --git a/observability/base/kube-prometheus-stack/helmrelease.yaml b/observability/base/kube-prometheus-stack/helmrelease.yaml index cb370be2..b878c8dd 100644 --- a/observability/base/kube-prometheus-stack/helmrelease.yaml +++ b/observability/base/kube-prometheus-stack/helmrelease.yaml @@ -11,7 +11,7 @@ spec: kind: HelmRepository name: prometheus-community namespace: flux-system - version: "51.0.3" + version: "52.1.0" interval: 10m0s timeout: 30m install: diff --git a/observability/base/kube-prometheus-stack/httproute.yaml b/observability/base/kube-prometheus-stack/httproute.yaml index 3bb2904d..9eb74b49 100644 --- a/observability/base/kube-prometheus-stack/httproute.yaml +++ b/observability/base/kube-prometheus-stack/httproute.yaml @@ -4,7 +4,7 @@ metadata: name: grafana spec: parentRefs: - - name: platform + - name: platform-private namespace: infrastructure hostnames: - "grafana-${cluster_name}.priv.${domain_name}" diff --git a/observability/base/loki/helmrelease.yaml b/observability/base/loki/helmrelease.yaml index 88f5f107..a4e03a64 100644 --- a/observability/base/loki/helmrelease.yaml +++ b/observability/base/loki/helmrelease.yaml @@ -39,3 +39,8 @@ spec: s3ForcePathStyle: true insecure: false sse_encryption: false + # We already use Vector for logs collection + monitoring: + selfMonitoring: + grafanaAgent: + installOperator: false diff --git a/observability/base/vector-agent/helmrelease.yaml b/observability/base/vector-agent/helmrelease.yaml index ee735e2e..a8622137 100644 --- a/observability/base/vector-agent/helmrelease.yaml +++ b/observability/base/vector-agent/helmrelease.yaml @@ -18,7 +18,6 @@ spec: remediation: retries: 3 values: - # fullnameOverride -- Override the full name of resources. fullnameOverride: "vector-agent" role: "Agent" customConfig: @@ -63,6 +62,8 @@ spec: {{ print "{{ kubernetes.pod_node_name }}" }} container_name: |- {{ print "{{ kubernetes.container_name }}" }} + stream: |- + {{ print "{{ stream }}" }} cluster_name: "${cluster_name}" # Configure a PodMonitor for Vector, requires the PodMonitor CRD to be installed. diff --git a/security/mycluster-0/kustomization.yaml b/security/mycluster-0/kustomization.yaml index b4ca7ec6..5d19c861 100644 --- a/security/mycluster-0/kustomization.yaml +++ b/security/mycluster-0/kustomization.yaml @@ -5,4 +5,5 @@ resources: - ../base/kyverno - ../base/cert-manager - external-secrets - - platform-gw-clusterpolicy.yaml + - platform-public-gw-clusterpolicy.yaml + - platform-private-gw-clusterpolicy.yaml diff --git a/security/mycluster-0/platform-gw-clusterpolicy.yaml b/security/mycluster-0/platform-private-gw-clusterpolicy.yaml similarity index 76% rename from security/mycluster-0/platform-gw-clusterpolicy.yaml rename to security/mycluster-0/platform-private-gw-clusterpolicy.yaml index 8923962d..9f910a26 100644 --- a/security/mycluster-0/platform-gw-clusterpolicy.yaml +++ b/security/mycluster-0/platform-private-gw-clusterpolicy.yaml @@ -1,12 +1,12 @@ apiVersion: kyverno.io/v1 kind: ClusterPolicy metadata: - name: mutate-cilium-platform-gateway + name: mutate-cilium-platform-private-gateway annotations: - policies.kyverno.io/title: Mutate shared gateway + policies.kyverno.io/title: Mutate private shared gateway policies.kyverno.io/severity: medium policies.kyverno.io/description: >- - We need to mutate the shared gateway service in order to configure the AWS LB. + We need to mutate the private shared gateway service in order to configure the AWS LB. spec: rules: - name: mutate-svc-annotations @@ -17,7 +17,7 @@ spec: - Service namespaces: - infrastructure - name: cilium-gateway-platform + name: cilium-gateway-platform-private mutate: patchStrategicMerge: metadata: diff --git a/security/mycluster-0/platform-public-gw-clusterpolicy.yaml b/security/mycluster-0/platform-public-gw-clusterpolicy.yaml new file mode 100644 index 00000000..3f4ebfad --- /dev/null +++ b/security/mycluster-0/platform-public-gw-clusterpolicy.yaml @@ -0,0 +1,29 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: mutate-cilium-platform-public-gateway + annotations: + policies.kyverno.io/title: Mutate Vector Cloudwatch Gateway + policies.kyverno.io/severity: medium + policies.kyverno.io/description: >- + We need to mutate the public shared gateway service in order to configure the AWS LB. +spec: + rules: + - name: mutate-svc-annotations + match: + any: + - resources: + kinds: + - Service + namespaces: + - infrastructure + name: cilium-gateway-platform-public + mutate: + patchStrategicMerge: + metadata: + annotations: + external-dns.alpha.kubernetes.io/hostname: vector-cloudwatch-${cluster_name}.${domain_name} + service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp + spec: + loadBalancerClass: service.k8s.aws/nlb diff --git a/taskfile.yaml b/taskfile.yaml index a7e2c69e..cd764893 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -18,6 +18,7 @@ tasks: pre-commit: deps: [clean] cmds: + - defer: { task: clean } - | USERID=$(id -u) GROUPID=$(id -g) diff --git a/terraform/eks/data.tf b/terraform/eks/data.tf index d614eb05..c2b1a926 100644 --- a/terraform/eks/data.tf +++ b/terraform/eks/data.tf @@ -1,5 +1,8 @@ data "aws_caller_identity" "this" {} +# tflint-ignore: terraform_unused_declarations +data "aws_ecr_authorization_token" "token" {} + data "aws_vpc" "selected" { filter { name = "tag:project" diff --git a/terraform/eks/flux.tf b/terraform/eks/flux.tf index 8f6e5c11..cfcce417 100644 --- a/terraform/eks/flux.tf +++ b/terraform/eks/flux.tf @@ -19,7 +19,8 @@ resource "flux_bootstrap_git" "this" { ] } -resource "kubernetes_config_map" "flux_clusters_vars" { +# Write secret items in order to use them as variables with flux's variables substitions +resource "kubernetes_secret" "flux_clusters_vars" { metadata { name = "eks-${var.cluster_name}-vars" namespace = "flux-system" @@ -37,18 +38,3 @@ resource "kubernetes_config_map" "flux_clusters_vars" { } depends_on = [flux_bootstrap_git.this] } - -# Write Github secrets in order to use them as variables with flux's variables substitions -resource "kubernetes_secret" "flux_github_creds" { - metadata { - name = "flux-github-creds" - namespace = "flux-system" - } - - data = { - github_owner = var.github_owner - github_token = var.github_token - github_repository = var.github_repository - } - depends_on = [flux_bootstrap_git.this] -} diff --git a/terraform/eks/helm.tf b/terraform/eks/helm.tf index f828cc16..d1be9f36 100644 --- a/terraform/eks/helm.tf +++ b/terraform/eks/helm.tf @@ -49,6 +49,8 @@ resource "helm_release" "aws_ebs_csi_driver" { values = [ file("${path.module}/helm_values/aws-ebs-csi-driver.yaml") ] + + depends_on = [helm_release.cilium] } resource "helm_release" "karpenter" { @@ -58,7 +60,7 @@ resource "helm_release" "karpenter" { name = "karpenter" repository = "oci://public.ecr.aws/karpenter" chart = "karpenter" - version = "v0.31.1" + version = var.karpenter_version values = [ templatefile( diff --git a/terraform/eks/helm_values/aws-ebs-csi-driver.yaml b/terraform/eks/helm_values/aws-ebs-csi-driver.yaml index 9bede3fb..567cc838 100644 --- a/terraform/eks/helm_values/aws-ebs-csi-driver.yaml +++ b/terraform/eks/helm_values/aws-ebs-csi-driver.yaml @@ -14,3 +14,7 @@ node: limits: cpu: 200m memory: 200Mi + serviceMonitor: + forceEnable: true + labels: + prometheus-instance: main diff --git a/terraform/eks/iam.tf b/terraform/eks/iam.tf index 4e48049b..12431954 100644 --- a/terraform/eks/iam.tf +++ b/terraform/eks/iam.tf @@ -28,8 +28,11 @@ module "irsa_crossplane" { assume_role_condition_test = "StringLike" role_policy_arns = { - irsa = aws_iam_policy.crossplane_irsa.arn, - s3 = aws_iam_policy.crossplane_s3.arn + cloudwatch = aws_iam_policy.crossplane_cloudwatch.arn, + kinesis = aws_iam_policy.crossplane_kinesis.arn, + firehose = aws_iam_policy.crossplane_firehose.arn, + irsa = aws_iam_policy.crossplane_irsa.arn, + s3 = aws_iam_policy.crossplane_s3.arn } oidc_providers = { @@ -42,7 +45,7 @@ module "irsa_crossplane" { #tfsec:ignore:aws-iam-no-policy-wildcards resource "aws_iam_policy" "crossplane_irsa" { - name = "crossplane_irsa_policy_${var.cluster_name}" + name = "crossplane_irsa_${var.cluster_name}" path = "/" description = "Policy for creating IRSA on EKS" @@ -57,6 +60,7 @@ resource "aws_iam_policy" "crossplane_irsa" { "iam:TagRole", "iam:CreateRole", "iam:CreatePolicy", + "iam:PutRolePolicy", "iam:DeletePolicy", "iam:DeleteRole", "iam:DetachRolePolicy", @@ -84,7 +88,7 @@ EOF #tfsec:ignore:aws-iam-no-policy-wildcards resource "aws_iam_policy" "crossplane_s3" { - name = "crossplane_s3_policy_${var.cluster_name}" + name = "crossplane_s3_${var.cluster_name}" path = "/" description = "Policy for managing S3 Buckets on EKS" @@ -97,7 +101,9 @@ resource "aws_iam_policy" "crossplane_s3" { "Action": "s3:*", "Resource": [ "arn:aws:s3:::${var.region}-ogenki-loki", - "arn:aws:s3:::${var.region}-ogenki-loki/*" + "arn:aws:s3:::${var.region}-ogenki-loki/*", + "arn:aws:s3:::${var.region}-ogenki-vector-stream", + "arn:aws:s3:::${var.region}-ogenki-vector-stream/*" ] }, { @@ -109,10 +115,111 @@ resource "aws_iam_policy" "crossplane_s3" { ], "Resource": [ "arn:aws:s3:::${var.region}-ogenki-loki", - "arn:aws:s3:::${var.region}-ogenki-loki/*" + "arn:aws:s3:::${var.region}-ogenki-loki/*", + "arn:aws:s3:::${var.region}-ogenki-vector-stream", + "arn:aws:s3:::${var.region}-ogenki-vector-stream/*" ] } ] } EOF } + +#tfsec:ignore:aws-iam-no-policy-wildcards +resource "aws_iam_policy" "crossplane_cloudwatch" { + name = "crossplane_cloudwatch_${var.cluster_name}" + path = "/" + description = "Policy for managing Log groups and streams on EKS" + + policy = <