From 501c8da65135b8401e88c2f140b5997af61583aa Mon Sep 17 00:00:00 2001 From: Fabian Kramm Date: Mon, 25 Mar 2024 12:48:16 +0100 Subject: [PATCH] feat: allow external etcd for k3s & k0s --- .github/workflows/release.yaml | 2 +- devspace_start.sh | 10 ++---- pkg/certs/ensure.go | 6 +++- pkg/config/config.go | 10 +++--- pkg/etcd/util.go | 6 ++-- pkg/k0s/k0s.go | 22 +++++++++++++ pkg/k3s/k3s.go | 18 ++++++++++- pkg/k8s/k8s.go | 44 ++++++++++++------------- pkg/setup/initialize.go | 59 ++++++++++++++-------------------- 9 files changed, 103 insertions(+), 74 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a01c01305..caf490001 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -30,7 +30,7 @@ jobs: - name: Setup Cosgin uses: sigstore/cosign-installer@main with: - cosign-release: "v2.0.2" + cosign-release: "v2.2.3" - name: Setup Syft uses: anchore/sbom-action/download-syft@v0.15.1 - name: Set up QEMU diff --git a/devspace_start.sh b/devspace_start.sh index 673364659..3b6c56d02 100755 --- a/devspace_start.sh +++ b/devspace_start.sh @@ -5,9 +5,6 @@ COLOR_CYAN="\033[0;36m" COLOR_RESET="\033[0m" RUN_CMD="go run -mod vendor cmd/vcluster/main.go start" -RUN_CMD_K8S="echo \"Run syncer with k8s flags\" && go run -mod vendor cmd/vcluster/main.go start --request-header-ca-cert=/pki/ca.crt --client-ca-cert=/pki/ca.crt --server-ca-cert=/pki/ca.crt --server-ca-key=/pki/ca.key --kube-config=/pki/admin.conf" -RUN_CMD_K0S="echo \"Run syncer with k0s flags\" && go run -mod vendor cmd/vcluster/main.go start --request-header-ca-cert=/data/k0s/pki/ca.crt --client-ca-cert=/data/k0s/pki/ca.crt --server-ca-cert=/data/k0s/pki/ca.crt --server-ca-key=/data/k0s/pki/ca.key --kube-config=/data/k0s/pki/admin.conf" -RUN_CMD_EKS="echo \"Run syncer with eks flags\" && go run -mod vendor cmd/vcluster/main.go start --request-header-ca-cert=/pki/ca.crt --client-ca-cert=/pki/ca.crt --server-ca-cert=/pki/ca.crt --server-ca-key=/pki/ca.key --kube-config=/pki/admin.conf" DEBUG_CMD="dlv debug ./cmd/vcluster/main.go --listen=0.0.0.0:2345 --api-version=2 --output /tmp/__debug_bin --headless --build-flags=\"-mod=vendor\" -- start" echo -e "${COLOR_CYAN} @@ -34,16 +31,13 @@ If you wish to run vcluster in the debug mode with delve, run: ${COLOR_CYAN}Note:${COLOR_RESET} vcluster won't start until you connect with the debugger. ${COLOR_CYAN}Note:${COLOR_RESET} vcluster will be stopped once you detach your debugger session. -${COLOR_CYAN}TIP:${COLOR_RESET} hit an up arrow on your keyboard to find the commands mentioned above :) +${COLOR_CYAN}TIP:${COLOR_RESET} hit an up arrow on your keyboard to find the commands mentioned above :) " # add useful commands to the history for convenience export HISTFILE=/tmp/.bash_history -history -s $RUN_CMD_EKS -history -s $RUN_CMD_K0S -history -s $RUN_CMD_K8S history -s $DEBUG_CMD history -s $RUN_CMD history -a # hide "I have no name!" from the bash prompt when running as non root -bash --init-file <(echo "export PS1=\"\\H:\\W\\$ \"") \ No newline at end of file +bash --init-file <(echo "export PS1=\"\\H:\\W\\$ \"") diff --git a/pkg/certs/ensure.go b/pkg/certs/ensure.go index 9ffcdff04..39d3c2b3d 100644 --- a/pkg/certs/ensure.go +++ b/pkg/certs/ensure.go @@ -37,10 +37,13 @@ func EnsureCerts( secretName := vClusterName + "-certs" secret, err := currentNamespaceClient.CoreV1().Secrets(currentNamespace).Get(ctx, secretName, metav1.GetOptions{}) if err == nil { + // download certs from secret err = downloadCertsFromSecret(secret, certificateDir) if err != nil { return err } + + // update kube config shouldUpdate, err := updateKubeconfigInSecret(secret) if err != nil { return err @@ -48,8 +51,8 @@ func EnsureCerts( return nil } - klog.Info("removing outdated certs") // delete the certs and recreate them + klog.Info("removing outdated certs") cfg, err := createConfig(serviceCIDR, vClusterName, certificateDir, clusterDomain, etcdSans) if err != nil { return err @@ -83,6 +86,7 @@ func EnsureCerts( if err != nil { return err } + return downloadCertsFromSecret(secret, certificateDir) } diff --git a/pkg/config/config.go b/pkg/config/config.go index 2a1a319b5..177929aeb 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -60,11 +60,11 @@ func (v VirtualClusterConfig) VirtualClusterKubeConfig() config.VirtualClusterKu } case config.EKSDistro, config.K8SDistro: distroConfig = config.VirtualClusterKubeConfig{ - KubeConfig: "/pki/admin.conf", - ServerCAKey: "/pki/ca.key", - ServerCACert: "/pki/ca.crt", - ClientCACert: "/pki/ca.crt", - RequestHeaderCACert: "/pki/front-proxy-ca.crt", + KubeConfig: "/data/pki/admin.conf", + ServerCAKey: "/data/pki/ca.key", + ServerCACert: "/data/pki/ca.crt", + ClientCACert: "/data/pki/ca.crt", + RequestHeaderCACert: "/data/pki/front-proxy-ca.crt", } } diff --git a/pkg/etcd/util.go b/pkg/etcd/util.go index e5e078e5f..91e6cf7a8 100644 --- a/pkg/etcd/util.go +++ b/pkg/etcd/util.go @@ -56,12 +56,14 @@ func WaitForEtcdClient(parentCtx context.Context, certificates *Certificates, en waitErr := wait.PollUntilContextTimeout(parentCtx, time.Second, waitForClientTimeout, true, func(ctx context.Context) (bool, error) { etcdClient, err = GetEtcdClient(parentCtx, certificates, endpoints...) if err == nil { + defer func() { + _ = etcdClient.Close() + }() + _, err = etcdClient.MemberList(ctx) if err == nil { return true, nil } - - _ = etcdClient.Close() } klog.Infof("Couldn't connect to embedded etcd (will retry in a second): %v", err) diff --git a/pkg/k0s/k0s.go b/pkg/k0s/k0s.go index abb8d0782..81df8665c 100644 --- a/pkg/k0s/k0s.go +++ b/pkg/k0s/k0s.go @@ -13,6 +13,7 @@ import ( vclusterconfig "github.com/loft-sh/vcluster/config" "github.com/loft-sh/vcluster/pkg/config" + "github.com/loft-sh/vcluster/pkg/etcd" "github.com/loft-sh/vcluster/pkg/util/commandwriter" "k8s.io/klog/v2" ) @@ -61,6 +62,15 @@ spec: etcdPrefix: "/registry" clientCertFile: /data/k0s/pki/apiserver-etcd-client.crt clientKeyFile: /data/k0s/pki/apiserver-etcd-client.key + {{- else if .Values.controlPlane.backingStore.externalEtcd.enabled }} + storage: + etcd: + externalCluster: + endpoints: ["{{ .Release.Name }}-etcd:2379"] + caFile: /data/k0s/pki/etcd/ca.crt + etcdPrefix: "/registry" + clientCertFile: /data/k0s/pki/apiserver-etcd-client.crt + clientKeyFile: /data/k0s/pki/apiserver-etcd-client.key {{- end }}` func StartK0S(ctx context.Context, cancel context.CancelFunc, vConfig *config.VirtualClusterConfig) error { @@ -74,6 +84,18 @@ func StartK0S(ctx context.Context, cancel context.CancelFunc, vConfig *config.Vi _ = os.RemoveAll(filepath.Join(runDir, entry.Name())) } + // wait until etcd is up and running + if vConfig.ControlPlane.BackingStore.ExternalEtcd.Enabled { + _, err := etcd.WaitForEtcdClient(ctx, &etcd.Certificates{ + CaCert: "/data/k0s/pki/etcd/ca.crt", + ServerCert: "/data/k0s/pki/apiserver-etcd-client.crt", + ServerKey: "/data/k0s/pki/apiserver-etcd-client.key", + }, "https://"+vConfig.Name+"-etcd:2379") + if err != nil { + return err + } + } + // build args args := []string{} if len(vConfig.ControlPlane.Distro.K0S.Command) > 0 { diff --git a/pkg/k3s/k3s.go b/pkg/k3s/k3s.go index 31e3f6f8d..93debf148 100644 --- a/pkg/k3s/k3s.go +++ b/pkg/k3s/k3s.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/loft-sh/vcluster/pkg/config" + "github.com/loft-sh/vcluster/pkg/etcd" "github.com/loft-sh/vcluster/pkg/util/commandwriter" "github.com/loft-sh/vcluster/pkg/util/random" corev1 "k8s.io/api/core/v1" @@ -48,7 +49,22 @@ func StartK3S(ctx context.Context, vConfig *config.VirtualClusterConfig, service args = append(args, "--kube-controller-manager-arg=controllers=*,-nodeipam,-nodelifecycle,-persistentvolume-binder,-attachdetach,-persistentvolume-expander,-cloud-node-lifecycle,-ttl") args = append(args, "--kube-apiserver-arg=endpoint-reconciler-type=none") } - if vConfig.ControlPlane.BackingStore.EmbeddedEtcd.Enabled { + if vConfig.ControlPlane.BackingStore.ExternalEtcd.Enabled { + // wait until etcd is up and running + _, err := etcd.WaitForEtcdClient(ctx, &etcd.Certificates{ + CaCert: "/data/pki/etcd/ca.crt", + ServerCert: "/data/pki/apiserver-etcd-client.crt", + ServerKey: "/data/pki/apiserver-etcd-client.key", + }, "https://"+vConfig.Name+"-etcd:2379") + if err != nil { + return err + } + + args = append(args, "--datastore-endpoint=https://"+vConfig.Name+"-etcd:2379") + args = append(args, "--datastore-cafile=/data/pki/etcd/ca.crt") + args = append(args, "--datastore-certfile=/data/pki/apiserver-etcd-client.crt") + args = append(args, "--datastore-keyfile=/data/pki/apiserver-etcd-client.key") + } else if vConfig.ControlPlane.BackingStore.EmbeddedEtcd.Enabled { args = append(args, "--datastore-endpoint=https://localhost:2379") args = append(args, "--datastore-cafile=/data/pki/etcd/ca.crt") args = append(args, "--datastore-certfile=/data/pki/apiserver-etcd-client.crt") diff --git a/pkg/k8s/k8s.go b/pkg/k8s/k8s.go index caa71771f..3e93d468e 100644 --- a/pkg/k8s/k8s.go +++ b/pkg/k8s/k8s.go @@ -45,27 +45,27 @@ func StartK8S( args = append(args, "--authorization-mode=RBAC") args = append(args, "--client-ca-file="+vConfig.VirtualClusterKubeConfig().ClientCACert) args = append(args, "--enable-bootstrap-token-auth=true") - args = append(args, "--etcd-cafile=/pki/etcd/ca.crt") - args = append(args, "--etcd-certfile=/pki/apiserver-etcd-client.crt") - args = append(args, "--etcd-keyfile=/pki/apiserver-etcd-client.key") + args = append(args, "--etcd-cafile=/data/pki/etcd/ca.crt") + args = append(args, "--etcd-certfile=/data/pki/apiserver-etcd-client.crt") + args = append(args, "--etcd-keyfile=/data/pki/apiserver-etcd-client.key") if vConfig.ControlPlane.BackingStore.EmbeddedEtcd.Enabled { args = append(args, "--etcd-servers=https://127.0.0.1:2379") } else { args = append(args, "--etcd-servers=https://"+vConfig.Name+"-etcd:2379") } - args = append(args, "--proxy-client-cert-file=/pki/front-proxy-client.crt") - args = append(args, "--proxy-client-key-file=/pki/front-proxy-client.key") + args = append(args, "--proxy-client-cert-file=/data/pki/front-proxy-client.crt") + args = append(args, "--proxy-client-key-file=/data/pki/front-proxy-client.key") args = append(args, "--requestheader-allowed-names=front-proxy-client") - args = append(args, "--requestheader-client-ca-file=/pki/front-proxy-ca.crt") + args = append(args, "--requestheader-client-ca-file=/data/pki/front-proxy-ca.crt") args = append(args, "--requestheader-extra-headers-prefix=X-Remote-Extra-") args = append(args, "--requestheader-group-headers=X-Remote-Group") args = append(args, "--requestheader-username-headers=X-Remote-User") args = append(args, "--secure-port=6443") args = append(args, "--service-account-issuer=https://kubernetes.default.svc.cluster.local") - args = append(args, "--service-account-key-file=/pki/sa.pub") - args = append(args, "--service-account-signing-key-file=/pki/sa.key") - args = append(args, "--tls-cert-file=/pki/apiserver.crt") - args = append(args, "--tls-private-key-file=/pki/apiserver.key") + args = append(args, "--service-account-key-file=/data/pki/sa.pub") + args = append(args, "--service-account-signing-key-file=/data/pki/sa.key") + args = append(args, "--tls-cert-file=/data/pki/apiserver.crt") + args = append(args, "--tls-private-key-file=/data/pki/apiserver.key") args = append(args, "--watch-cache=false") args = append(args, "--endpoint-reconciler-type=none") } @@ -106,21 +106,21 @@ func StartK8S( } else { args = append(args, "/binaries/kube-controller-manager") args = append(args, serviceCIDRArg) - args = append(args, "--authentication-kubeconfig=/pki/controller-manager.conf") - args = append(args, "--authorization-kubeconfig=/pki/controller-manager.conf") + args = append(args, "--authentication-kubeconfig=/data/pki/controller-manager.conf") + args = append(args, "--authorization-kubeconfig=/data/pki/controller-manager.conf") args = append(args, "--bind-address=127.0.0.1") - args = append(args, "--client-ca-file=/pki/ca.crt") + args = append(args, "--client-ca-file=/data/pki/ca.crt") args = append(args, "--cluster-name=kubernetes") - args = append(args, "--cluster-signing-cert-file=/pki/ca.crt") - args = append(args, "--cluster-signing-key-file=/pki/ca.key") + args = append(args, "--cluster-signing-cert-file=/data/pki/ca.crt") + args = append(args, "--cluster-signing-key-file=/data/pki/ca.key") args = append(args, "--horizontal-pod-autoscaler-sync-period=60s") - args = append(args, "--kubeconfig=/pki/controller-manager.conf") + args = append(args, "--kubeconfig=/data/pki/controller-manager.conf") args = append(args, "--node-monitor-grace-period=180s") args = append(args, "--node-monitor-period=30s") args = append(args, "--pvclaimbinder-sync-period=60s") - args = append(args, "--requestheader-client-ca-file=/pki/front-proxy-ca.crt") - args = append(args, "--root-ca-file=/pki/ca.crt") - args = append(args, "--service-account-private-key-file=/pki/sa.key") + args = append(args, "--requestheader-client-ca-file=/data/pki/front-proxy-ca.crt") + args = append(args, "--root-ca-file=/data/pki/ca.crt") + args = append(args, "--service-account-private-key-file=/data/pki/sa.key") args = append(args, "--use-service-account-credentials=true") if vConfig.ControlPlane.StatefulSet.HighAvailability.Replicas > 1 { args = append(args, "--leader-elect=true") @@ -151,10 +151,10 @@ func StartK8S( args = append(args, scheduler.Command...) } else { args = append(args, "/binaries/kube-scheduler") - args = append(args, "--authentication-kubeconfig=/pki/scheduler.conf") - args = append(args, "--authorization-kubeconfig=/pki/scheduler.conf") + args = append(args, "--authentication-kubeconfig=/data/pki/scheduler.conf") + args = append(args, "--authorization-kubeconfig=/data/pki/scheduler.conf") args = append(args, "--bind-address=127.0.0.1") - args = append(args, "--kubeconfig=/pki/scheduler.conf") + args = append(args, "--kubeconfig=/data/pki/scheduler.conf") if vConfig.ControlPlane.StatefulSet.HighAvailability.Replicas > 1 { args = append(args, "--leader-elect=true") } else { diff --git a/pkg/setup/initialize.go b/pkg/setup/initialize.go index b8774a9d8..735867e33 100644 --- a/pkg/setup/initialize.go +++ b/pkg/setup/initialize.go @@ -100,7 +100,7 @@ func initialize( // create certificates if they are not there yet certificatesDir := "/data/k0s/pki" - err = GenerateCertsWithEtcdSans(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.Networking.Advanced.ClusterDomain) + err = GenerateCerts(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.Networking.Advanced.ClusterDomain) if err != nil { return err } @@ -144,15 +144,15 @@ func initialize( return err } + // generate etcd certificates + certificatesDir := "/data/pki" + err = GenerateCerts(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.Networking.Advanced.ClusterDomain) + if err != nil { + return err + } + // should start embedded etcd? if options.ControlPlane.BackingStore.EmbeddedEtcd.Enabled { - // generate certificates - certificatesDir := "/data/pki" - err := GenerateCertsWithEtcdSans(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.Networking.Advanced.ClusterDomain) - if err != nil { - return err - } - // we need to run this with the parent ctx as otherwise this context // will be cancelled by the wait loop in Initialize err = pro.StartEmbeddedEtcd( @@ -180,8 +180,8 @@ func initialize( case vclusterconfig.K8SDistro, vclusterconfig.EKSDistro: // try to generate k8s certificates certificatesDir := filepath.Dir(options.VirtualClusterKubeConfig().ServerCACert) - if certificatesDir == "/pki" { - err := GenerateK8sCerts(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.Networking.Advanced.ClusterDomain) + if certificatesDir == "/data/pki" { + err := GenerateCerts(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.Networking.Advanced.ClusterDomain) if err != nil { return err } @@ -233,9 +233,9 @@ func initialize( }() case vclusterconfig.Unknown: certificatesDir := filepath.Dir(options.VirtualClusterKubeConfig().ServerCACert) - if certificatesDir == "/pki" { + if certificatesDir == "/data/pki" { // generate k8s certificates - err := GenerateK8sCerts(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.Networking.Advanced.ClusterDomain) + err := GenerateCerts(ctx, currentNamespaceClient, vClusterName, currentNamespace, serviceCIDR, certificatesDir, options.Networking.Advanced.ClusterDomain) if err != nil { return err } @@ -245,26 +245,7 @@ func initialize( return nil } -func GenerateCertsWithEtcdSans(ctx context.Context, currentNamespaceClient kubernetes.Interface, vClusterName, currentNamespace, serviceCIDR, certificatesDir, clusterDomain string) error { - // generate etcd server and peer sans - etcdSans := []string{ - "localhost", - "*." + vClusterName + "-headless", - "*." + vClusterName + "-headless" + "." + currentNamespace, - "*." + vClusterName + "-headless" + "." + currentNamespace + ".svc", - "*." + vClusterName + "-headless" + "." + currentNamespace + ".svc." + clusterDomain, - } - - // generate certificates - err := certs.EnsureCerts(ctx, serviceCIDR, currentNamespace, currentNamespaceClient, vClusterName, certificatesDir, clusterDomain, etcdSans) - if err != nil { - return fmt.Errorf("ensure certs: %w", err) - } - - return nil -} - -func GenerateK8sCerts(ctx context.Context, currentNamespaceClient kubernetes.Interface, vClusterName, currentNamespace, serviceCIDR, certificatesDir, clusterDomain string) error { +func GenerateCerts(ctx context.Context, currentNamespaceClient kubernetes.Interface, vClusterName, currentNamespace, serviceCIDR, certificatesDir, clusterDomain string) error { // generate etcd server and peer sans etcdService := vClusterName + "-etcd" etcdSans := []string{ @@ -272,8 +253,17 @@ func GenerateK8sCerts(ctx context.Context, currentNamespaceClient kubernetes.Int etcdService, etcdService + "." + currentNamespace, etcdService + "." + currentNamespace + ".svc", - "*." + etcdService + "-headless", - "*." + etcdService + "-headless" + "." + currentNamespace, + } + + // add wildcard + for _, service := range []string{vClusterName, etcdService} { + etcdSans = append( + etcdSans, + "*."+service+"-headless", + "*."+service+"-headless"+"."+currentNamespace, + "*."+service+"-headless"+"."+currentNamespace+".svc", + "*."+service+"-headless"+"."+currentNamespace+".svc."+clusterDomain, + ) } //expect up to 20 etcd members, number could be lower since more @@ -282,6 +272,7 @@ func GenerateK8sCerts(ctx context.Context, currentNamespaceClient kubernetes.Int // this is for embedded etcd hostname := vClusterName + "-" + strconv.Itoa(i) etcdSans = append(etcdSans, hostname, hostname+"."+vClusterName+"-headless", hostname+"."+vClusterName+"-headless"+"."+currentNamespace) + // this is for external etcd etcdHostname := etcdService + "-" + strconv.Itoa(i) etcdSans = append(etcdSans, etcdHostname, etcdHostname+"."+etcdService+"-headless", etcdHostname+"."+etcdService+"-headless"+"."+currentNamespace)