Merge pull request #1620 from FabianKramm/main

feat: allow external etcd for k3s & k0s

.github/workflows/release.yaml

@@ -30,7 +30,7 @@ jobs:
       - name: Setup Cosgin
         uses: sigstore/cosign-installer@main
         with:
-          cosign-release: "v2.0.2"
+          cosign-release: "v2.2.3"
       - name: Setup Syft
         uses: anchore/sbom-action/[email protected]
       - name: Set up QEMU

devspace_start.sh

@@ -5,9 +5,6 @@ COLOR_CYAN="\033[0;36m"
 COLOR_RESET="\033[0m"
 
 RUN_CMD="go run -mod vendor cmd/vcluster/main.go start"
-RUN_CMD_K8S="echo \"Run syncer with k8s flags\" && go run -mod vendor cmd/vcluster/main.go start --request-header-ca-cert=/pki/ca.crt --client-ca-cert=/pki/ca.crt --server-ca-cert=/pki/ca.crt --server-ca-key=/pki/ca.key --kube-config=/pki/admin.conf"
-RUN_CMD_K0S="echo \"Run syncer with k0s flags\" && go run -mod vendor cmd/vcluster/main.go start --request-header-ca-cert=/data/k0s/pki/ca.crt --client-ca-cert=/data/k0s/pki/ca.crt --server-ca-cert=/data/k0s/pki/ca.crt --server-ca-key=/data/k0s/pki/ca.key --kube-config=/data/k0s/pki/admin.conf"
-RUN_CMD_EKS="echo \"Run syncer with eks flags\" && go run -mod vendor cmd/vcluster/main.go start  --request-header-ca-cert=/pki/ca.crt --client-ca-cert=/pki/ca.crt --server-ca-cert=/pki/ca.crt --server-ca-key=/pki/ca.key --kube-config=/pki/admin.conf"
 DEBUG_CMD="dlv debug ./cmd/vcluster/main.go --listen=0.0.0.0:2345 --api-version=2 --output /tmp/__debug_bin --headless --build-flags=\"-mod=vendor\" -- start"
 
 echo -e "${COLOR_CYAN}
@@ -34,16 +31,13 @@ If you wish to run vcluster in the debug mode with delve, run:
   ${COLOR_CYAN}Note:${COLOR_RESET} vcluster won't start until you connect with the debugger.
   ${COLOR_CYAN}Note:${COLOR_RESET} vcluster will be stopped once you detach your debugger session.
 
-${COLOR_CYAN}TIP:${COLOR_RESET} hit an up arrow on your keyboard to find the commands mentioned above :) 
+${COLOR_CYAN}TIP:${COLOR_RESET} hit an up arrow on your keyboard to find the commands mentioned above :)
 "
 # add useful commands to the history for convenience
 export HISTFILE=/tmp/.bash_history
-history -s $RUN_CMD_EKS
-history -s $RUN_CMD_K0S
-history -s $RUN_CMD_K8S
 history -s $DEBUG_CMD
 history -s $RUN_CMD
 history -a
 
 # hide "I have no name!" from the bash prompt when running as non root
-bash --init-file <(echo "export PS1=\"\\H:\\W\\$ \"")
+bash --init-file <(echo "export PS1=\"\\H:\\W\\$ \"")

pkg/certs/ensure.go

@@ -37,19 +37,22 @@ func EnsureCerts(
 	secretName := vClusterName + "-certs"
 	secret, err := currentNamespaceClient.CoreV1().Secrets(currentNamespace).Get(ctx, secretName, metav1.GetOptions{})
 	if err == nil {
+		// download certs from secret
 		err = downloadCertsFromSecret(secret, certificateDir)
 		if err != nil {
 			return err
 		}
+
+		// update kube config
 		shouldUpdate, err := updateKubeconfigInSecret(secret)
 		if err != nil {
 			return err
 		} else if !shouldUpdate {
 			return nil
 		}
 
-		klog.Info("removing outdated certs")
 		// delete the certs and recreate them
+		klog.Info("removing outdated certs")
 		cfg, err := createConfig(serviceCIDR, vClusterName, certificateDir, clusterDomain, etcdSans)
 		if err != nil {
 			return err
@@ -83,6 +86,7 @@ func EnsureCerts(
 		if err != nil {
 			return err
 		}
+
 		return downloadCertsFromSecret(secret, certificateDir)
 	}
 

pkg/config/config.go

@@ -60,11 +60,11 @@ func (v VirtualClusterConfig) VirtualClusterKubeConfig() config.VirtualClusterKu
 		}
 	case config.EKSDistro, config.K8SDistro:
 		distroConfig = config.VirtualClusterKubeConfig{
-			KubeConfig:          "/pki/admin.conf",
-			ServerCAKey:         "/pki/ca.key",
-			ServerCACert:        "/pki/ca.crt",
-			ClientCACert:        "/pki/ca.crt",
-			RequestHeaderCACert: "/pki/front-proxy-ca.crt",
+			KubeConfig:          "/data/pki/admin.conf",
+			ServerCAKey:         "/data/pki/ca.key",
+			ServerCACert:        "/data/pki/ca.crt",
+			ClientCACert:        "/data/pki/ca.crt",
+			RequestHeaderCACert: "/data/pki/front-proxy-ca.crt",
 		}
 	}
 

pkg/etcd/util.go

@@ -56,12 +56,14 @@ func WaitForEtcdClient(parentCtx context.Context, certificates *Certificates, en
 	waitErr := wait.PollUntilContextTimeout(parentCtx, time.Second, waitForClientTimeout, true, func(ctx context.Context) (bool, error) {
 		etcdClient, err = GetEtcdClient(parentCtx, certificates, endpoints...)
 		if err == nil {
+			defer func() {
+				_ = etcdClient.Close()
+			}()
+
 			_, err = etcdClient.MemberList(ctx)
 			if err == nil {
 				return true, nil
 			}
-
-			_ = etcdClient.Close()
 		}
 
 		klog.Infof("Couldn't connect to embedded etcd (will retry in a second): %v", err)

pkg/k0s/k0s.go

@@ -13,6 +13,7 @@ import (
 
 	vclusterconfig "github.com/loft-sh/vcluster/config"
 	"github.com/loft-sh/vcluster/pkg/config"
+	"github.com/loft-sh/vcluster/pkg/etcd"
 	"github.com/loft-sh/vcluster/pkg/util/commandwriter"
 	"k8s.io/klog/v2"
 )
@@ -61,6 +62,15 @@ spec:
         etcdPrefix: "/registry"
         clientCertFile: /data/k0s/pki/apiserver-etcd-client.crt
         clientKeyFile: /data/k0s/pki/apiserver-etcd-client.key
+  {{- else if .Values.controlPlane.backingStore.externalEtcd.enabled }}
+  storage:
+    etcd:
+      externalCluster:
+        endpoints: ["{{ .Release.Name }}-etcd:2379"]
+        caFile: /data/k0s/pki/etcd/ca.crt
+        etcdPrefix: "/registry"
+        clientCertFile: /data/k0s/pki/apiserver-etcd-client.crt
+        clientKeyFile: /data/k0s/pki/apiserver-etcd-client.key
   {{- end }}`
 
 func StartK0S(ctx context.Context, cancel context.CancelFunc, vConfig *config.VirtualClusterConfig) error {
@@ -74,6 +84,18 @@ func StartK0S(ctx context.Context, cancel context.CancelFunc, vConfig *config.Vi
 		_ = os.RemoveAll(filepath.Join(runDir, entry.Name()))
 	}
 
+	// wait until etcd is up and running
+	if vConfig.ControlPlane.BackingStore.ExternalEtcd.Enabled {
+		_, err := etcd.WaitForEtcdClient(ctx, &etcd.Certificates{
+			CaCert:     "/data/k0s/pki/etcd/ca.crt",
+			ServerCert: "/data/k0s/pki/apiserver-etcd-client.crt",
+			ServerKey:  "/data/k0s/pki/apiserver-etcd-client.key",
+		}, "https://"+vConfig.Name+"-etcd:2379")
+		if err != nil {
+			return err
+		}
+	}
+
 	// build args
 	args := []string{}
 	if len(vConfig.ControlPlane.Distro.K0S.Command) > 0 {

pkg/k3s/k3s.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/loft-sh/vcluster/pkg/config"
+	"github.com/loft-sh/vcluster/pkg/etcd"
 	"github.com/loft-sh/vcluster/pkg/util/commandwriter"
 	"github.com/loft-sh/vcluster/pkg/util/random"
 	corev1 "k8s.io/api/core/v1"
@@ -48,7 +49,22 @@ func StartK3S(ctx context.Context, vConfig *config.VirtualClusterConfig, service
 			args = append(args, "--kube-controller-manager-arg=controllers=*,-nodeipam,-nodelifecycle,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle,-ttl")
 			args = append(args, "--kube-apiserver-arg=endpoint-reconciler-type=none")
 		}
-		if vConfig.ControlPlane.BackingStore.EmbeddedEtcd.Enabled {
+		if vConfig.ControlPlane.BackingStore.ExternalEtcd.Enabled {
+			// wait until etcd is up and running
+			_, err := etcd.WaitForEtcdClient(ctx, &etcd.Certificates{
+				CaCert:     "/data/pki/etcd/ca.crt",
+				ServerCert: "/data/pki/apiserver-etcd-client.crt",
+				ServerKey:  "/data/pki/apiserver-etcd-client.key",
+			}, "https://"+vConfig.Name+"-etcd:2379")
+			if err != nil {
+				return err
+			}
+
+			args = append(args, "--datastore-endpoint=https://"+vConfig.Name+"-etcd:2379")
+			args = append(args, "--datastore-cafile=/data/pki/etcd/ca.crt")
+			args = append(args, "--datastore-certfile=/data/pki/apiserver-etcd-client.crt")
+			args = append(args, "--datastore-keyfile=/data/pki/apiserver-etcd-client.key")
+		} else if vConfig.ControlPlane.BackingStore.EmbeddedEtcd.Enabled {
 			args = append(args, "--datastore-endpoint=https://localhost:2379")
 			args = append(args, "--datastore-cafile=/data/pki/etcd/ca.crt")
 			args = append(args, "--datastore-certfile=/data/pki/apiserver-etcd-client.crt")

pkg/k8s/k8s.go

@@ -45,27 +45,27 @@ func StartK8S(
 				args = append(args, "--authorization-mode=RBAC")
 				args = append(args, "--client-ca-file="+vConfig.VirtualClusterKubeConfig().ClientCACert)
 				args = append(args, "--enable-bootstrap-token-auth=true")
-				args = append(args, "--etcd-cafile=/pki/etcd/ca.crt")
-				args = append(args, "--etcd-certfile=/pki/apiserver-etcd-client.crt")
-				args = append(args, "--etcd-keyfile=/pki/apiserver-etcd-client.key")
+				args = append(args, "--etcd-cafile=/data/pki/etcd/ca.crt")
+				args = append(args, "--etcd-certfile=/data/pki/apiserver-etcd-client.crt")
+				args = append(args, "--etcd-keyfile=/data/pki/apiserver-etcd-client.key")
 				if vConfig.ControlPlane.BackingStore.EmbeddedEtcd.Enabled {
 					args = append(args, "--etcd-servers=https://127.0.0.1:2379")
 				} else {
 					args = append(args, "--etcd-servers=https://"+vConfig.Name+"-etcd:2379")
 				}
-				args = append(args, "--proxy-client-cert-file=/pki/front-proxy-client.crt")
-				args = append(args, "--proxy-client-key-file=/pki/front-proxy-client.key")
+				args = append(args, "--proxy-client-cert-file=/data/pki/front-proxy-client.crt")
+				args = append(args, "--proxy-client-key-file=/data/pki/front-proxy-client.key")
 				args = append(args, "--requestheader-allowed-names=front-proxy-client")
-				args = append(args, "--requestheader-client-ca-file=/pki/front-proxy-ca.crt")
+				args = append(args, "--requestheader-client-ca-file=/data/pki/front-proxy-ca.crt")
 				args = append(args, "--requestheader-extra-headers-prefix=X-Remote-Extra-")
 				args = append(args, "--requestheader-group-headers=X-Remote-Group")
 				args = append(args, "--requestheader-username-headers=X-Remote-User")
 				args = append(args, "--secure-port=6443")
 				args = append(args, "--service-account-issuer=https://kubernetes.default.svc.cluster.local")
-				args = append(args, "--service-account-key-file=/pki/sa.pub")
-				args = append(args, "--service-account-signing-key-file=/pki/sa.key")
-				args = append(args, "--tls-cert-file=/pki/apiserver.crt")
-				args = append(args, "--tls-private-key-file=/pki/apiserver.key")
+				args = append(args, "--service-account-key-file=/data/pki/sa.pub")
+				args = append(args, "--service-account-signing-key-file=/data/pki/sa.key")
+				args = append(args, "--tls-cert-file=/data/pki/apiserver.crt")
+				args = append(args, "--tls-private-key-file=/data/pki/apiserver.key")
 				args = append(args, "--watch-cache=false")
 				args = append(args, "--endpoint-reconciler-type=none")
 			}
@@ -106,21 +106,21 @@ func StartK8S(
 			} else {
 				args = append(args, "/binaries/kube-controller-manager")
 				args = append(args, serviceCIDRArg)
-				args = append(args, "--authentication-kubeconfig=/pki/controller-manager.conf")
-				args = append(args, "--authorization-kubeconfig=/pki/controller-manager.conf")
+				args = append(args, "--authentication-kubeconfig=/data/pki/controller-manager.conf")
+				args = append(args, "--authorization-kubeconfig=/data/pki/controller-manager.conf")
 				args = append(args, "--bind-address=127.0.0.1")
-				args = append(args, "--client-ca-file=/pki/ca.crt")
+				args = append(args, "--client-ca-file=/data/pki/ca.crt")
 				args = append(args, "--cluster-name=kubernetes")
-				args = append(args, "--cluster-signing-cert-file=/pki/ca.crt")
-				args = append(args, "--cluster-signing-key-file=/pki/ca.key")
+				args = append(args, "--cluster-signing-cert-file=/data/pki/ca.crt")
+				args = append(args, "--cluster-signing-key-file=/data/pki/ca.key")
 				args = append(args, "--horizontal-pod-autoscaler-sync-period=60s")
-				args = append(args, "--kubeconfig=/pki/controller-manager.conf")
+				args = append(args, "--kubeconfig=/data/pki/controller-manager.conf")
 				args = append(args, "--node-monitor-grace-period=180s")
 				args = append(args, "--node-monitor-period=30s")
 				args = append(args, "--pvclaimbinder-sync-period=60s")
-				args = append(args, "--requestheader-client-ca-file=/pki/front-proxy-ca.crt")
-				args = append(args, "--root-ca-file=/pki/ca.crt")
-				args = append(args, "--service-account-private-key-file=/pki/sa.key")
+				args = append(args, "--requestheader-client-ca-file=/data/pki/front-proxy-ca.crt")
+				args = append(args, "--root-ca-file=/data/pki/ca.crt")
+				args = append(args, "--service-account-private-key-file=/data/pki/sa.key")
 				args = append(args, "--use-service-account-credentials=true")
 				if vConfig.ControlPlane.StatefulSet.HighAvailability.Replicas > 1 {
 					args = append(args, "--leader-elect=true")
@@ -151,10 +151,10 @@ func StartK8S(
 				args = append(args, scheduler.Command...)
 			} else {
 				args = append(args, "/binaries/kube-scheduler")
-				args = append(args, "--authentication-kubeconfig=/pki/scheduler.conf")
-				args = append(args, "--authorization-kubeconfig=/pki/scheduler.conf")
+				args = append(args, "--authentication-kubeconfig=/data/pki/scheduler.conf")
+				args = append(args, "--authorization-kubeconfig=/data/pki/scheduler.conf")
 				args = append(args, "--bind-address=127.0.0.1")
-				args = append(args, "--kubeconfig=/pki/scheduler.conf")
+				args = append(args, "--kubeconfig=/data/pki/scheduler.conf")
 				if vConfig.ControlPlane.StatefulSet.HighAvailability.Replicas > 1 {
 					args = append(args, "--leader-elect=true")
 				} else {

pkg/setup/initialize.go

@@ -100,7 +100,7 @@ func initialize(
 
 		// create certificates if they are not there yet
 		certificatesDir := "/data/k0s/pki"
-		err = GenerateCertsWithEtcdSans(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.Networking.Advanced.ClusterDomain)
+		err = GenerateCerts(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.Networking.Advanced.ClusterDomain)
 		if err != nil {
 			return err
 		}
@@ -144,15 +144,15 @@ func initialize(
 			return err
 		}
 
+		// generate etcd certificates
+		certificatesDir := "/data/pki"
+		err = GenerateCerts(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.Networking.Advanced.ClusterDomain)
+		if err != nil {
+			return err
+		}
+
 		// should start embedded etcd?
 		if options.ControlPlane.BackingStore.EmbeddedEtcd.Enabled {
-			// generate certificates
-			certificatesDir := "/data/pki"
-			err := GenerateCertsWithEtcdSans(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.Networking.Advanced.ClusterDomain)
-			if err != nil {
-				return err
-			}
-
 			// we need to run this with the parent ctx as otherwise this context
 			// will be cancelled by the wait loop in Initialize
 			err = pro.StartEmbeddedEtcd(
@@ -180,8 +180,8 @@ func initialize(
 	case vclusterconfig.K8SDistro, vclusterconfig.EKSDistro:
 		// try to generate k8s certificates
 		certificatesDir := filepath.Dir(options.VirtualClusterKubeConfig().ServerCACert)
-		if certificatesDir == "/pki" {
-			err := GenerateK8sCerts(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.Networking.Advanced.ClusterDomain)
+		if certificatesDir == "/data/pki" {
+			err := GenerateCerts(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.Networking.Advanced.ClusterDomain)
 			if err != nil {
 				return err
 			}
@@ -233,9 +233,9 @@ func initialize(
 		}()
 	case vclusterconfig.Unknown:
 		certificatesDir := filepath.Dir(options.VirtualClusterKubeConfig().ServerCACert)
-		if certificatesDir == "/pki" {
+		if certificatesDir == "/data/pki" {
 			// generate k8s certificates
-			err := GenerateK8sCerts(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.Networking.Advanced.ClusterDomain)
+			err := GenerateCerts(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.Networking.Advanced.ClusterDomain)
 			if err != nil {
 				return err
 			}
@@ -245,35 +245,25 @@ func initialize(
 	return nil
 }
 
-func GenerateCertsWithEtcdSans(ctx context.Context, currentNamespaceClient kubernetes.Interface, vClusterName, currentNamespace, serviceCIDR, certificatesDir, clusterDomain string) error {
-	// generate etcd server and peer sans
-	etcdSans := []string{
-		"localhost",
-		"*." + vClusterName + "-headless",
-		"*." + vClusterName + "-headless" + "." + currentNamespace,
-		"*." + vClusterName + "-headless" + "." + currentNamespace + ".svc",
-		"*." + vClusterName + "-headless" + "." + currentNamespace + ".svc." + clusterDomain,
-	}
-
-	// generate certificates
-	err := certs.EnsureCerts(ctx, serviceCIDR, currentNamespace, currentNamespaceClient, vClusterName, certificatesDir, clusterDomain, etcdSans)
-	if err != nil {
-		return fmt.Errorf("ensure certs: %w", err)
-	}
-
-	return nil
-}
-
-func GenerateK8sCerts(ctx context.Context, currentNamespaceClient kubernetes.Interface, vClusterName, currentNamespace, serviceCIDR, certificatesDir, clusterDomain string) error {
+func GenerateCerts(ctx context.Context, currentNamespaceClient kubernetes.Interface, vClusterName, currentNamespace, serviceCIDR, certificatesDir, clusterDomain string) error {
 	// generate etcd server and peer sans
 	etcdService := vClusterName + "-etcd"
 	etcdSans := []string{
 		"localhost",
 		etcdService,
 		etcdService + "." + currentNamespace,
 		etcdService + "." + currentNamespace + ".svc",
-		"*." + etcdService + "-headless",
-		"*." + etcdService + "-headless" + "." + currentNamespace,
+	}
+
+	// add wildcard
+	for _, service := range []string{vClusterName, etcdService} {
+		etcdSans = append(
+			etcdSans,
+			"*."+service+"-headless",
+			"*."+service+"-headless"+"."+currentNamespace,
+			"*."+service+"-headless"+"."+currentNamespace+".svc",
+			"*."+service+"-headless"+"."+currentNamespace+".svc."+clusterDomain,
+		)
 	}
 
 	//expect up to 20 etcd members, number could be lower since more
@@ -282,6 +272,7 @@ func GenerateK8sCerts(ctx context.Context, currentNamespaceClient kubernetes.Int
 		// this is for embedded etcd
 		hostname := vClusterName + "-" + strconv.Itoa(i)
 		etcdSans = append(etcdSans, hostname, hostname+"."+vClusterName+"-headless", hostname+"."+vClusterName+"-headless"+"."+currentNamespace)
+
 		// this is for external etcd
 		etcdHostname := etcdService + "-" + strconv.Itoa(i)
 		etcdSans = append(etcdSans, etcdHostname, etcdHostname+"."+etcdService+"-headless", etcdHostname+"."+etcdService+"-headless"+"."+currentNamespace)