From 973067e5b5beab7cc1a3eb08d1537272601b1934 Mon Sep 17 00:00:00 2001 From: facchettos Date: Thu, 25 Jan 2024 14:57:22 +0100 Subject: [PATCH 1/6] removed unused values since the merge into a single container --- charts/eks/values.yaml | 40 ---------------------------- charts/k8s/values.yaml | 59 +----------------------------------------- 2 files changed, 1 insertion(+), 98 deletions(-) diff --git a/charts/eks/values.yaml b/charts/eks/values.yaml index 8eb06438c..5d4da3165 100644 --- a/charts/eks/values.yaml +++ b/charts/eks/values.yaml @@ -220,52 +220,12 @@ etcd: controller: image: public.ecr.aws/eks-distro/kubernetes/kube-controller-manager:v1.28.2-eks-1-28-6 imagePullPolicy: "" - # The amount of replicas to run the deployment with - replicas: 1 - # NodeSelector used - nodeSelector: {} - # Affinity to apply - affinity: {} - # Tolerations to apply - tolerations: [] - # Extra Labels - labels: {} - # Extra Annotations - annotations: {} - podAnnotations: {} - podLabels: {} - resources: - requests: - cpu: 15m - priorityClassName: "" - securityContext: {} # Kubernetes API Server settings api: image: public.ecr.aws/eks-distro/kubernetes/kube-apiserver:v1.28.2-eks-1-28-6 imagePullPolicy: "" extraArgs: [] - # The amount of replicas to run the deployment with - replicas: 1 - # NodeSelector used to schedule the syncer - nodeSelector: {} - # Affinity to apply to the syncer deployment - affinity: {} - # Tolerations to apply to the syncer deployment - tolerations: [] - # Extra Labels for the syncer deployment - labels: {} - # Extra Annotations for the syncer deployment - annotations: {} - podAnnotations: {} - podLabels: {} - resources: - requests: - cpu: 40m - memory: 300Mi - priorityClassName: "" - securityContext: {} - serviceAnnotations: {} # Core DNS settings coredns: diff --git a/charts/k8s/values.yaml b/charts/k8s/values.yaml index d943e3f48..536ba9175 100644 --- a/charts/k8s/values.yaml +++ b/charts/k8s/values.yaml @@ -224,47 +224,11 @@ etcd: controller: image: registry.k8s.io/kube-controller-manager:v1.29.0 imagePullPolicy: "" - # The amount of replicas to run the deployment with - replicas: 1 - # NodeSelector used - nodeSelector: {} - # Affinity to apply - affinity: {} - # Tolerations to apply - tolerations: [] - # Extra Labels - labels: {} - # Extra Annotations - annotations: {} - podAnnotations: {} - podLabels: {} - resources: - requests: - cpu: 15m - priorityClassName: "" - securityContext: {} + # Kubernetes Scheduler settings. Only enabled if sync.nodes.enableScheduler is true scheduler: image: registry.k8s.io/kube-scheduler:v1.29.0 imagePullPolicy: "" - # The amount of replicas to run the deployment with - replicas: 1 - # NodeSelector used - nodeSelector: {} - # Affinity to apply - affinity: {} - # Tolerations to apply - tolerations: [] - # Extra Labels - labels: {} - # Extra Annotations - annotations: {} - podAnnotations: {} - podLabels: {} - resources: - requests: - cpu: 10m - priorityClassName: "" disabled: true # Kubernetes API Server settings @@ -272,27 +236,6 @@ api: image: registry.k8s.io/kube-apiserver:v1.29.0 imagePullPolicy: "" extraArgs: [] - # The amount of replicas to run the deployment with - replicas: 1 - # NodeSelector used to schedule the syncer - nodeSelector: {} - # Affinity to apply to the syncer deployment - affinity: {} - # Tolerations to apply to the syncer deployment - tolerations: [] - # Extra Labels for the syncer deployment - labels: {} - # Extra Annotations for the syncer deployment - annotations: {} - podAnnotations: {} - podLabels: {} - resources: - requests: - cpu: 40m - memory: 300Mi - priorityClassName: "" - securityContext: {} - serviceAnnotations: {} # Service account that should be used by the vcluster serviceAccount: From 40eefc972fe16bab7dabc3b951711e32816e2cc9 Mon Sep 17 00:00:00 2001 From: facchettos Date: Thu, 25 Jan 2024 15:34:42 +0100 Subject: [PATCH 2/6] removed enableHA --- charts/eks/templates/rbac/role.yaml | 2 +- charts/eks/templates/syncer.yaml | 6 +++--- charts/eks/values.yaml | 4 ---- charts/k8s/templates/etcd-statefulset.yaml | 2 +- charts/k8s/templates/rbac/role.yaml | 2 +- charts/k8s/templates/syncer.yaml | 8 ++++---- charts/k8s/values.yaml | 4 ---- 7 files changed, 10 insertions(+), 18 deletions(-) diff --git a/charts/eks/templates/rbac/role.yaml b/charts/eks/templates/rbac/role.yaml index eb837e640..da4362387 100644 --- a/charts/eks/templates/rbac/role.yaml +++ b/charts/eks/templates/rbac/role.yaml @@ -50,7 +50,7 @@ rules: resources: ["endpoints"] verbs: ["create", "delete", "patch", "update"] {{- end }} - {{- if or .Values.enableHA .Values.rbac.role.extended }} + {{- if or (gt (int .Values.syncer.replicas) 1) .Values.rbac.role.extended }} - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] diff --git a/charts/eks/templates/syncer.yaml b/charts/eks/templates/syncer.yaml index b9412d970..e573ece24 100644 --- a/charts/eks/templates/syncer.yaml +++ b/charts/eks/templates/syncer.yaml @@ -49,7 +49,7 @@ spec: {{- if .Values.syncer.affinity }} affinity: {{ toYaml .Values.syncer.affinity | indent 8 }} - {{- else if .Values.enableHA }} + {{- else if (gt (int .Values.syncer.replicas) 1) }} affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -228,7 +228,7 @@ spec: {{- if .Values.syncer.kubeConfigContextName }} - --kube-config-context-name={{ .Values.syncer.kubeConfigContextName }} {{- end }} - {{- if .Values.enableHA }} + {{- if (gt (int .Values.syncer.replicas) 1) }} - --leader-elect=true {{- else }} - --leader-elect=false @@ -393,7 +393,7 @@ spec: {{- else }} - '--service-cluster-ip-range=$(SERVICE_CIDR)' {{- end }} - {{- if .Values.enableHA }} + {{- if (gt (int .Values.syncer.replicas) 1) }} - '--leader-elect=true' {{- else }} - '--leader-elect=false' diff --git a/charts/eks/values.yaml b/charts/eks/values.yaml index 5d4da3165..4156b327d 100644 --- a/charts/eks/values.yaml +++ b/charts/eks/values.yaml @@ -16,10 +16,6 @@ monitoring: serviceMonitor: enabled: false -# If the control plane is deployed in high availability mode -# Make sure to scale up the syncer.replicas, etcd.replicas, api.replicas & controller.replicas -enableHA: false - # Plugins that should get loaded. Usually you want to apply those via 'vcluster create ... -f https://.../plugin.yaml' plugin: {} # Manually configure a plugin called test diff --git a/charts/k8s/templates/etcd-statefulset.yaml b/charts/k8s/templates/etcd-statefulset.yaml index ba4ce1700..d13fbfb45 100644 --- a/charts/k8s/templates/etcd-statefulset.yaml +++ b/charts/k8s/templates/etcd-statefulset.yaml @@ -65,7 +65,7 @@ spec: {{- if .Values.etcd.affinity }} affinity: {{ toYaml .Values.etcd.affinity | indent 8 }} - {{- else if .Values.enableHA }} + {{- else }} affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: diff --git a/charts/k8s/templates/rbac/role.yaml b/charts/k8s/templates/rbac/role.yaml index eb837e640..da4362387 100644 --- a/charts/k8s/templates/rbac/role.yaml +++ b/charts/k8s/templates/rbac/role.yaml @@ -50,7 +50,7 @@ rules: resources: ["endpoints"] verbs: ["create", "delete", "patch", "update"] {{- end }} - {{- if or .Values.enableHA .Values.rbac.role.extended }} + {{- if or (gt (int .Values.syncer.replicas) 1) .Values.rbac.role.extended }} - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] diff --git a/charts/k8s/templates/syncer.yaml b/charts/k8s/templates/syncer.yaml index 6161043b0..a19c18316 100644 --- a/charts/k8s/templates/syncer.yaml +++ b/charts/k8s/templates/syncer.yaml @@ -49,7 +49,7 @@ spec: {{- if .Values.syncer.affinity }} affinity: {{ toYaml .Values.syncer.affinity | indent 8 }} - {{- else if .Values.enableHA }} + {{- else if (gt (int .Values.syncer.replicas) 1) }} affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -245,7 +245,7 @@ spec: {{- if .Values.syncer.kubeConfigContextName }} - --kube-config-context-name={{ .Values.syncer.kubeConfigContextName }} {{- end }} - {{- if .Values.enableHA }} + {{- if (gt (int .Values.syncer.replicas) 1)}} - --leader-elect=true {{- else }} - --leader-elect=false @@ -414,7 +414,7 @@ spec: {{- else }} - '--service-cluster-ip-range=$(SERVICE_CIDR)' {{- end }} - {{- if .Values.enableHA }} + {{- if (gt (int .Values.syncer.replicas) 1) }} - '--leader-elect=true' {{- else }} - '--leader-elect=false' @@ -439,7 +439,7 @@ spec: - '--authorization-kubeconfig=/pki/scheduler.conf' - '--bind-address=127.0.0.1' - '--kubeconfig=/pki/scheduler.conf' - {{- if .Values.enableHA }} + {{- if (gt (int .Values.syncer.replicas) 1) }} - '--leader-elect=true' {{- else }} - '--leader-elect=false' diff --git a/charts/k8s/values.yaml b/charts/k8s/values.yaml index 536ba9175..e4dbd93f4 100644 --- a/charts/k8s/values.yaml +++ b/charts/k8s/values.yaml @@ -16,10 +16,6 @@ monitoring: serviceMonitor: enabled: false -# If the control plane is deployed in high availability mode -# Make sure to scale up the syncer.replicas, etcd.replicas, api.replicas & controller.replicas -enableHA: false - # Plugins that should get loaded. Usually you want to apply those via 'vcluster create ... -f https://.../plugin.yaml' plugin: {} # Manually configure a plugin called test From cda7e3d8e47afe865984bb98352182f7768b90b9 Mon Sep 17 00:00:00 2001 From: facchettos Date: Thu, 25 Jan 2024 15:48:29 +0100 Subject: [PATCH 3/6] removed enableHA from k0s --- charts/k0s/templates/rbac/role.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/k0s/templates/rbac/role.yaml b/charts/k0s/templates/rbac/role.yaml index eb837e640..68f2fccde 100644 --- a/charts/k0s/templates/rbac/role.yaml +++ b/charts/k0s/templates/rbac/role.yaml @@ -50,7 +50,7 @@ rules: resources: ["endpoints"] verbs: ["create", "delete", "patch", "update"] {{- end }} - {{- if or .Values.enableHA .Values.rbac.role.extended }} + {{- if or (gt (int .Values.replicas) 1) (gt (int .Values.syncer.replicas) 1) .Values.rbac.role.extended }} - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] From a928775aaf182319d3bfe13102f36029c9db64c8 Mon Sep 17 00:00:00 2001 From: facchettos Date: Tue, 30 Jan 2024 15:43:37 +0100 Subject: [PATCH 4/6] fix --- charts/k0s/templates/rbac/role.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/k0s/templates/rbac/role.yaml b/charts/k0s/templates/rbac/role.yaml index 68f2fccde..f0280e31c 100644 --- a/charts/k0s/templates/rbac/role.yaml +++ b/charts/k0s/templates/rbac/role.yaml @@ -50,7 +50,7 @@ rules: resources: ["endpoints"] verbs: ["create", "delete", "patch", "update"] {{- end }} - {{- if or (gt (int .Values.replicas) 1) (gt (int .Values.syncer.replicas) 1) .Values.rbac.role.extended }} + {{- if or ( gt (int (include "vcluster.replicas" . ) ) 1) .Values.rbac.role.extended }} - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] From 9ac324b3b62ca3800f7d02573723962f7941ff84 Mon Sep 17 00:00:00 2001 From: facchettos Date: Tue, 30 Jan 2024 15:50:18 +0100 Subject: [PATCH 5/6] removed enableHA --- test/values_ha.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/values_ha.yaml b/test/values_ha.yaml index ced46787b..b0b6e35af 100644 --- a/test/values_ha.yaml +++ b/test/values_ha.yaml @@ -1,12 +1,6 @@ # this is for k3s pro replicas: 3 - -# this is for k8s -# -# Enable HA mode -enableHA: true - # Scale up syncer replicas syncer: replicas: 3 From eb1f33c2403f91516a32117ed0f2e891faf346ef Mon Sep 17 00:00:00 2001 From: facchettos Date: Tue, 30 Jan 2024 15:53:33 +0100 Subject: [PATCH 6/6] removed references to enableHA --- .../deploying-vclusters/high-availability.mdx | 21 +++++-------------- .../pages/fragments/high-availability-k3s.mdx | 7 ++----- .../pages/fragments/high-availability-k8s.mdx | 6 ------ load-test/ha-k8s.yaml | 4 ---- load-test/vcluster-k8s-3.yml | 3 --- 5 files changed, 7 insertions(+), 34 deletions(-) diff --git a/docs/pages/deploying-vclusters/high-availability.mdx b/docs/pages/deploying-vclusters/high-availability.mdx index 87cb29b63..fb46cae98 100644 --- a/docs/pages/deploying-vclusters/high-availability.mdx +++ b/docs/pages/deploying-vclusters/high-availability.mdx @@ -26,8 +26,8 @@ A distro in vcluster is the Kubernetes distribution that runs inside the virtual - k8s (a Kubernetes with etcd) The vcluster HA feature is only supported by the k3s and k8s distros. You need to choose one of those to take advantage of HA. -For this tutorial, we will use the k8s distro. -If you're interested in enabling HA in rootless mode, or using the k3s distro, see some examples at the bottom of this page. +For this tutorial, we will use the k8s distro. +If you're interested in enabling HA in rootless mode, or using the k3s distro, see some examples at the bottom of this page. # 3. Create a values.yaml file The values.yaml file is used to specify configuration options for the virtual cluster. In the case of the HA feature, we will specify the number of replicas we want to run for each vCluster component. @@ -35,9 +35,6 @@ The values.yaml file is used to specify configuration options for the virtual cl Create the file called values.yaml on the computer that the vcluster client is installed on with these contents: ``` -# Enable HA mode -enableHA: true - # Scale up syncer replicas syncer: replicas: 3 @@ -68,7 +65,7 @@ NAME STATUS ROLES AGE VERSION minikube Ready control-plane 2m5s v1.26.3 minikube-m02 Ready 105s v1.26.3 minikube-m03 Ready 93s v1.26.3 -minikube-m04 Ready 83s v1.26.3 +minikube-m04 Ready 83s v1.26.3 ``` # 4. Create the HA virtual cluster @@ -174,11 +171,9 @@ In order to run vCluster with k3s as Kubernetes distribution in high availabilit First create a `values.yaml` in the following form and make sure to change the connection string in `K3S_DATASTORE_ENDPOINT`: ``` -# Enable HA mode -enableHA: true - # Scale up k3s replicas -replicas: 2 +syncer: + replicas: 2 # Set external datastore endpoint vcluster: @@ -232,9 +227,6 @@ Check the [GitHub repository](https://github.com/loft-sh/vcluster/tree/main/char In order to run vCluster in high availability mode, create a `values.yaml` in the following form: ``` -# Enable HA mode -enableHA: true - # Scale up syncer replicas syncer: replicas: 3 @@ -298,9 +290,6 @@ You can find more about rootless mode [here](../security/rootless-mode.mdx). Below is HA configuration for running rootless vCluster with vanilla Kubernetes distribution. ``` -# Enable HA mode -enableHA: true - # Scale up syncer replicas syncer: replicas: 3 diff --git a/docs/pages/fragments/high-availability-k3s.mdx b/docs/pages/fragments/high-availability-k3s.mdx index 126e11c20..e9f3d706b 100644 --- a/docs/pages/fragments/high-availability-k3s.mdx +++ b/docs/pages/fragments/high-availability-k3s.mdx @@ -8,11 +8,8 @@ In order to run vCluster with k3s as Kubernetes distribution in high availabilit First create a `values.yaml` in the following form and make sure to change the connection string in `K3S_DATASTORE_ENDPOINT`: ``` -# Enable HA mode -enableHA: true - -# Scale up k3s replicas -replicas: 2 +syncer: + replicas: 3 # Set external datastore endpoint vcluster: diff --git a/docs/pages/fragments/high-availability-k8s.mdx b/docs/pages/fragments/high-availability-k8s.mdx index 62af0c8c6..73857dd6e 100644 --- a/docs/pages/fragments/high-availability-k8s.mdx +++ b/docs/pages/fragments/high-availability-k8s.mdx @@ -3,9 +3,6 @@ In order to run vCluster in high availability mode, create a `values.yaml` in the following form: ``` -# Enable HA mode -enableHA: true - # Scale up syncer replicas syncer: replicas: 3 @@ -69,9 +66,6 @@ You can find more about rootless mode [here](../security/rootless-mode.mdx). Below is HA configuration for running rootless vCluster with vanilla Kubernetes distribution. ``` -# Enable HA mode -enableHA: true - # Scale up syncer replicas syncer: replicas: 3 diff --git a/load-test/ha-k8s.yaml b/load-test/ha-k8s.yaml index a304cef44..1bd7cff2f 100644 --- a/load-test/ha-k8s.yaml +++ b/load-test/ha-k8s.yaml @@ -1,7 +1,3 @@ - -# Enable HA mode -enableHA: true - # Scale up syncer replicas syncer: replicas: 3 diff --git a/load-test/vcluster-k8s-3.yml b/load-test/vcluster-k8s-3.yml index 0c83463fe..ae7a02088 100644 --- a/load-test/vcluster-k8s-3.yml +++ b/load-test/vcluster-k8s-3.yml @@ -14,9 +14,6 @@ sync: # will sync all nodes instead of only the ones where some pods are running. syncAllNodes: true -# Enable HA mode -enableHA: true - # Scale up syncer replicas syncer: replicas: 3