Merge pull request #1518 from facchettos/k0s-embedded-etcd

added support for embedded etcd

charts/k0s/templates/secret.yaml

@@ -51,5 +51,15 @@ stringData:
           node-monitor-grace-period: 1h
           node-monitor-period: 1h
           {{- end }}
+      {{- if .Values.embeddedEtcd.enabled }}
+      storage:
+        etcd:
+          externalCluster:
+            endpoints: ["127.0.0.1:2379"]
+            caFile: /data/pki/k0s/etcd/ca.crt
+            etcdPrefix: "/registry"
+            clientCertFile: /data/k0s/pki/apiserver-etcd-client.crt
+            clientKeyFile: /data/k0s/pki/apiserver-etcd-client.key
+      {{- end }}
   {{- end }}
   {{- end }}

charts/k0s/templates/statefulset-service.yaml

@@ -15,13 +15,24 @@ metadata:
 {{ toYaml $annotations | indent 4 }}
   {{- end }}
 spec:
+  publishNotReadyAddresses: true
   ports:
     - name: https
       port: 443
       targetPort: 8443
       protocol: TCP
+    {{- if .Values.embeddedEtcd.enabled }}
+    - name: etcd
+      port: 2379
+      targetPort: 2379
+      protocol: TCP
+    - name: peer
+      port: 2380
+      targetPort: 2380
+      protocol: TCP
+    {{- end }}
   clusterIP: None
   selector:
     app: vcluster
     release: "{{ .Release.Name }}"
-{{- end }}
+{{- end }}

charts/k0s/templates/syncer.yaml

@@ -160,6 +160,15 @@ spec:
           - --server-ca-cert=/data/k0s/pki/ca.crt
           - --server-ca-key=/data/k0s/pki/ca.key
           - --kube-config=/data/k0s/pki/admin.conf
+          {{- if and .Values.embeddedEtcd.enabled .Values.pro }}
+          - --etcd-embedded
+          - --etcd-replicas={{ .Values.syncer.replicas }}
+          {{- end }}
+          {{- if (gt (int .Values.syncer.replicas ) 1) }}
+          - --leader-elect=true
+          {{- else }}
+          - --leader-elect=false
+          {{- end }}
           {{- include "vcluster.legacyPlugins.args" . | indent 10 }}
           {{- include "vcluster.serviceMapping.fromHost" . | indent 10 }}
           {{- include "vcluster.serviceMapping.fromVirtual" . | indent 10 }}
@@ -256,6 +265,10 @@ spec:
         {{- include "vcluster.plugins.config" . | indent 10 }}
           - name: VCLUSTER_DISTRO
             value: k0s
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
         {{- if eq ( ( include "vcluster.replicas" . ) | toString | atoi) 1 }}
           - name: VCLUSTER_NODE_NAME
             valueFrom:

charts/k0s/values.yaml

@@ -15,6 +15,11 @@ pro: false
 # secrets within the cluster.
 proLicenseSecret: ""
 
+# Embedded etcd settings
+embeddedEtcd:
+  # If embedded etcd should be enabled, this is a PRO only feature
+  enabled: false
+
 # If true, will deploy vcluster in headless mode, which means no deployment
 # or statefulset is created.
 headless: false

cmd/vclusterctl/cmd/get/service_cidr.go

@@ -37,7 +37,7 @@ vcluster get service-cidr
 10.96.0.0/12
 #######################################################
 	`,
-		RunE: func(cobraCmd *cobra.Command, args []string) error {
+		RunE: func(cobraCmd *cobra.Command, _ []string) error {
 			return cmd.Run(cobraCmd)
 		}}
 

cmd/vclusterctl/cmd/telemetry/disable.go

@@ -29,7 +29,7 @@ docs: https://www.vcluster.com/docs/advanced-topics/telemetry
 
 #######################################################
 	`,
-		RunE: func(cobraCmd *cobra.Command, args []string) error {
+		RunE: func(cobraCmd *cobra.Command, _ []string) error {
 			return cmd.Run(cobraCmd)
 		}}
 

cmd/vclusterctl/cmd/telemetry/enable.go

@@ -29,7 +29,7 @@ docs: https://www.vcluster.com/docs/advanced-topics/telemetry
 
 #######################################################
 	`,
-		RunE: func(cobraCmd *cobra.Command, args []string) error {
+		RunE: func(cobraCmd *cobra.Command, _ []string) error {
 			return cmd.Run(cobraCmd)
 		}}
 

pkg/certs/constants.go

@@ -169,3 +169,20 @@ var certMap = map[string]string{
 	EtcdServerCertName: strings.ReplaceAll(EtcdServerCertName, "/", "-"),
 	EtcdServerKeyName:  strings.ReplaceAll(EtcdServerKeyName, "/", "-"),
 }
+
+var K0sFiles = map[string]bool{
+	"admin.crt":         true,
+	"admin.key":         true,
+	"ccm.conf":          true,
+	"ccm.crt":           true,
+	"ccm.key":           true,
+	"konnectivity.key":  true,
+	"k0s-api.crt":       true,
+	"scheduler.crt":     true,
+	"k0s-api.key":       true,
+	"scheduler.key":     true,
+	"konnectivity.conf": true,
+	"server.crt":        true,
+	"konnectivity.crt":  true,
+	"server.key":        true,
+}

pkg/certs/ensure.go

@@ -2,10 +2,14 @@ package certs
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"io/fs"
 	"os"
 	"path/filepath"
+	"slices"
 
+	"golang.org/x/exp/maps"
 	corev1 "k8s.io/api/core/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -32,32 +36,14 @@ func EnsureCerts(
 		return downloadCertsFromSecret(secret, certificateDir)
 	}
 
-	// init config
-	cfg, err := SetInitDynamicDefaults()
-	if err != nil {
-		return err
-	}
-
-	cfg.ClusterName = "kubernetes"
-	cfg.NodeRegistration.Name = vClusterName
-	cfg.Etcd.Local = &LocalEtcd{
-		ServerCertSANs: etcdSans,
-		PeerCertSANs:   etcdSans,
-	}
-	cfg.Networking.ServiceSubnet = serviceCIDR
-	cfg.Networking.DNSDomain = clusterDomain
-	cfg.ControlPlaneEndpoint = "127.0.0.1:6443"
-	cfg.CertificatesDir = certificateDir
-	cfg.LocalAPIEndpoint.AdvertiseAddress = "0.0.0.0"
-	cfg.LocalAPIEndpoint.BindPort = 443
-	err = CreatePKIAssets(cfg)
-	if err != nil {
-		return fmt.Errorf("create pki assets: %w", err)
-	}
-
-	err = CreateJoinControlPlaneKubeConfigFiles(cfg.CertificatesDir, cfg)
-	if err != nil {
-		return fmt.Errorf("create kube configs: %w", err)
+	// we check if the files are already there
+	_, err = os.Stat(filepath.Join(certificateDir, CAKeyName))
+	if errors.Is(err, fs.ErrNotExist) {
+		// try to generate the certificates
+		err = generateCertificates(serviceCIDR, vClusterName, certificateDir, clusterDomain, etcdSans)
+		if err != nil {
+			return err
+		}
 	}
 
 	// build secret
@@ -77,6 +63,15 @@ func EnsureCerts(
 		secret.Data[toName] = data
 	}
 
+	// find extra files in the folder and add them to the secret
+	extraFiles, err := extraFiles(certificateDir)
+	if err != nil {
+		return fmt.Errorf("read extra file: %w", err)
+	}
+	for k, v := range extraFiles {
+		secret.Data[k] = v
+	}
+
 	// finally create the secret
 	secret, err = currentNamespaceClient.CoreV1().Secrets(currentNamespace).Create(ctx, secret, metav1.CreateOptions{})
 	if err != nil {
@@ -96,26 +91,111 @@ func EnsureCerts(
 	return downloadCertsFromSecret(secret, certificateDir)
 }
 
+func generateCertificates(
+	serviceCIDR string,
+	vClusterName string,
+	certificateDir string,
+	clusterDomain string,
+	etcdSans []string,
+) error {
+	// init config
+	cfg, err := SetInitDynamicDefaults()
+	if err != nil {
+		return err
+	}
+
+	cfg.ClusterName = "kubernetes"
+	cfg.NodeRegistration.Name = vClusterName
+	cfg.Etcd.Local = &LocalEtcd{
+		ServerCertSANs: etcdSans,
+		PeerCertSANs:   etcdSans,
+	}
+	cfg.Networking.ServiceSubnet = serviceCIDR
+	cfg.Networking.DNSDomain = clusterDomain
+	cfg.ControlPlaneEndpoint = "127.0.0.1:6443"
+	cfg.CertificatesDir = certificateDir
+	cfg.LocalAPIEndpoint.AdvertiseAddress = "0.0.0.0"
+	cfg.LocalAPIEndpoint.BindPort = 443
+
+	// only create the files if the files are not there yet
+	err = CreatePKIAssets(cfg)
+	if err != nil {
+		return fmt.Errorf("create pki assets: %w", err)
+	}
+
+	err = CreateJoinControlPlaneKubeConfigFiles(cfg.CertificatesDir, cfg)
+	if err != nil {
+		return fmt.Errorf("create kube configs: %w", err)
+	}
+
+	return nil
+}
+
+// downloadCertsFromSecret writes to the filesystem the content of each field in the secret
+// if the field has an equivalent inside the certmap, we write with the corresponding name
+// otherwise the file has the same name than the field
 func downloadCertsFromSecret(
 	secret *corev1.Secret,
 	certificateDir string,
 ) error {
-	for toName, fromName := range certMap {
-		if len(secret.Data[fromName]) == 0 {
-			return fmt.Errorf("secret is missing %s", fromName)
+	certMapValues := maps.Values(certMap)
+	for secretEntry, fileBytes := range secret.Data {
+		name := secretEntry
+		if slices.Contains(certMapValues, secretEntry) {
+			// we need to replace with the actual name
+			for key, sEntry := range certMap {
+				// guarranteed to evaluate to true at least once because of slices.contains
+				if sEntry == secretEntry {
+					if len(fileBytes) == 0 {
+						return fmt.Errorf("secret is missing %s", secretEntry)
+					}
+					name = key
+					break
+				}
+			}
 		}
 
-		name := filepath.Join(certificateDir, toName)
+		name = filepath.Join(certificateDir, name)
 		err := os.MkdirAll(filepath.Dir(name), 0777)
 		if err != nil {
 			return fmt.Errorf("create directory %s", filepath.Dir(name))
 		}
 
-		err = os.WriteFile(name, secret.Data[fromName], 0666)
+		err = os.WriteFile(name, fileBytes, 0666)
 		if err != nil {
-			return fmt.Errorf("write %s: %w", fromName, err)
+			return fmt.Errorf("write %s: %w", name, err)
 		}
 	}
 
 	return nil
 }
+
+func extraFiles(certificateDir string) (map[string][]byte, error) {
+	files := make(map[string][]byte)
+	entries, err := os.ReadDir(certificateDir)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, v := range entries {
+		if v.IsDir() {
+			// ignore subdirectories for now
+			// etcd files should be picked up by the map
+			continue
+		}
+
+		// if it's not in the cert map, add to the map
+		name := v.Name()
+		_, ok := certMap[name]
+		if !ok {
+			b, err := os.ReadFile(filepath.Join(certificateDir, name))
+			if err != nil {
+				return nil, err
+			}
+
+			files[name] = b
+		}
+	}
+
+	return files, err
+}

pkg/controllers/coredns/nodehosts.go

@@ -100,7 +100,7 @@ func (r *NodeHostsReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	funcs := predicate.NewPredicateFuncs(p)
 
 	// use modified handler to avoid triggering reconcile for each Node
-	eventHandler := handler.EnqueueRequestsFromMapFunc(func(_ context.Context, o client.Object) []reconcile.Request {
+	eventHandler := handler.EnqueueRequestsFromMapFunc(func(_ context.Context, _ client.Object) []reconcile.Request {
 		return []reconcile.Request{{
 			NamespacedName: types.NamespacedName{Namespace: Namespace, Name: ConfigMapName},
 		}}

pkg/controllers/resources/priorityclasses/syncer.go

@@ -59,7 +59,7 @@ func (s *priorityClassSyncer) Sync(ctx *synccontext.SyncContext, pObj client.Obj
 }
 
 func NewPriorityClassTranslator() translate.PhysicalNameTranslator {
-	return func(vName string, vObj client.Object) string {
+	return func(vName string, _ client.Object) string {
 		return translatePriorityClassName(vName)
 	}
 }

pkg/k0s/k0s.go

@@ -13,7 +13,9 @@ import (
 	"k8s.io/klog/v2"
 )
 
-const VClusterCommandEnv = "VCLUSTER_COMMAND"
+const (
+	VClusterCommandEnv = "VCLUSTER_COMMAND"
+)
 
 type k0sCommand struct {
 	Command []string `json:"command,omitempty"`
@@ -22,12 +24,17 @@ type k0sCommand struct {
 
 const runDir = "/run/k0s"
 
-func StartK0S(ctx context.Context) error {
+func StartK0S(ctx context.Context, cancel context.CancelFunc) error {
+	// this is not really useful but go isn't happy if we don't cancel the context
+	// everywhere
+	defer cancel()
+
 	// make sure we delete the contents of /run/k0s
 	dirEntries, _ := os.ReadDir(runDir)
 	for _, entry := range dirEntries {
 		_ = os.RemoveAll(filepath.Join(runDir, entry.Name()))
 	}
+
 	// create command
 	command := &k0sCommand{}
 	err := yaml.Unmarshal([]byte(os.Getenv(VClusterCommandEnv)), command)

pkg/server/filters/k3s_connect.go

@@ -41,7 +41,7 @@ func WithK3sConnect(h http.Handler) http.Handler {
 		HandshakeTimeout: 45 * time.Second,
 		TLSClientConfig:  tlsCfg,
 	}
-	proxy.Backend = func(r *http.Request) *url.URL {
+	proxy.Backend = func(_ *http.Request) *url.URL {
 		u := *serverURL
 		u.Path = K3sConnectPath
 		return &u