diff --git a/charts/k0s/templates/secret.yaml b/charts/k0s/templates/secret.yaml index fc5ec659a3..51af302987 100644 --- a/charts/k0s/templates/secret.yaml +++ b/charts/k0s/templates/secret.yaml @@ -51,5 +51,15 @@ stringData: node-monitor-grace-period: 1h node-monitor-period: 1h {{- end }} + {{- if .Values.embeddedEtcd.enabled }} + storage: + etcd: + externalCluster: + endpoints: ["127.0.0.1:2379"] + caFile: /data/pki/k0s/etcd/ca.crt + etcdPrefix: "/registry" + clientCertFile: /data/k0s/pki/apiserver-etcd-client.crt + clientKeyFile: /data/k0s/pki/apiserver-etcd-client.key + {{- end }} {{- end }} {{- end }} diff --git a/charts/k0s/templates/statefulset-service.yaml b/charts/k0s/templates/statefulset-service.yaml index bcfd6c7406..7523bf36e7 100644 --- a/charts/k0s/templates/statefulset-service.yaml +++ b/charts/k0s/templates/statefulset-service.yaml @@ -15,13 +15,24 @@ metadata: {{ toYaml $annotations | indent 4 }} {{- end }} spec: + publishNotReadyAddresses: true ports: - name: https port: 443 targetPort: 8443 protocol: TCP + {{- if .Values.embeddedEtcd.enabled }} + - name: etcd + port: 2379 + targetPort: 2379 + protocol: TCP + - name: peer + port: 2380 + targetPort: 2380 + protocol: TCP + {{- end }} clusterIP: None selector: app: vcluster release: "{{ .Release.Name }}" -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/k0s/templates/syncer.yaml b/charts/k0s/templates/syncer.yaml index 4de43fb8d1..24e0c0c46c 100644 --- a/charts/k0s/templates/syncer.yaml +++ b/charts/k0s/templates/syncer.yaml @@ -160,6 +160,15 @@ spec: - --server-ca-cert=/data/k0s/pki/ca.crt - --server-ca-key=/data/k0s/pki/ca.key - --kube-config=/data/k0s/pki/admin.conf + {{- if and .Values.embeddedEtcd.enabled .Values.pro }} + - --etcd-embedded + - --etcd-replicas={{ .Values.syncer.replicas }} + {{- end }} + {{- if (gt (int .Values.syncer.replicas ) 1) }} + - --leader-elect=true + {{- else }} + - --leader-elect=false + {{- end }} {{- include "vcluster.legacyPlugins.args" . | indent 10 }} {{- include "vcluster.serviceMapping.fromHost" . | indent 10 }} {{- include "vcluster.serviceMapping.fromVirtual" . | indent 10 }} @@ -256,6 +265,10 @@ spec: {{- include "vcluster.plugins.config" . | indent 10 }} - name: VCLUSTER_DISTRO value: k0s + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name {{- if eq ( ( include "vcluster.replicas" . ) | toString | atoi) 1 }} - name: VCLUSTER_NODE_NAME valueFrom: diff --git a/charts/k0s/values.yaml b/charts/k0s/values.yaml index 84d7a6ffda..04f4ae4f93 100644 --- a/charts/k0s/values.yaml +++ b/charts/k0s/values.yaml @@ -15,6 +15,11 @@ pro: false # secrets within the cluster. proLicenseSecret: "" +# Embedded etcd settings +embeddedEtcd: + # If embedded etcd should be enabled, this is a PRO only feature + enabled: false + # If true, will deploy vcluster in headless mode, which means no deployment # or statefulset is created. headless: false diff --git a/cmd/vclusterctl/cmd/get/service_cidr.go b/cmd/vclusterctl/cmd/get/service_cidr.go index 48973cb666..881b3dd0dc 100644 --- a/cmd/vclusterctl/cmd/get/service_cidr.go +++ b/cmd/vclusterctl/cmd/get/service_cidr.go @@ -37,7 +37,7 @@ vcluster get service-cidr 10.96.0.0/12 ####################################################### `, - RunE: func(cobraCmd *cobra.Command, args []string) error { + RunE: func(cobraCmd *cobra.Command, _ []string) error { return cmd.Run(cobraCmd) }} diff --git a/cmd/vclusterctl/cmd/telemetry/disable.go b/cmd/vclusterctl/cmd/telemetry/disable.go index bc076bee7e..8bb79fdf38 100644 --- a/cmd/vclusterctl/cmd/telemetry/disable.go +++ b/cmd/vclusterctl/cmd/telemetry/disable.go @@ -29,7 +29,7 @@ docs: https://www.vcluster.com/docs/advanced-topics/telemetry ####################################################### `, - RunE: func(cobraCmd *cobra.Command, args []string) error { + RunE: func(cobraCmd *cobra.Command, _ []string) error { return cmd.Run(cobraCmd) }} diff --git a/cmd/vclusterctl/cmd/telemetry/enable.go b/cmd/vclusterctl/cmd/telemetry/enable.go index aae938df9a..316760d50a 100644 --- a/cmd/vclusterctl/cmd/telemetry/enable.go +++ b/cmd/vclusterctl/cmd/telemetry/enable.go @@ -29,7 +29,7 @@ docs: https://www.vcluster.com/docs/advanced-topics/telemetry ####################################################### `, - RunE: func(cobraCmd *cobra.Command, args []string) error { + RunE: func(cobraCmd *cobra.Command, _ []string) error { return cmd.Run(cobraCmd) }} diff --git a/pkg/certs/constants.go b/pkg/certs/constants.go index 798fe58def..e500545779 100644 --- a/pkg/certs/constants.go +++ b/pkg/certs/constants.go @@ -169,3 +169,20 @@ var certMap = map[string]string{ EtcdServerCertName: strings.ReplaceAll(EtcdServerCertName, "/", "-"), EtcdServerKeyName: strings.ReplaceAll(EtcdServerKeyName, "/", "-"), } + +var K0sFiles = map[string]bool{ + "admin.crt": true, + "admin.key": true, + "ccm.conf": true, + "ccm.crt": true, + "ccm.key": true, + "konnectivity.key": true, + "k0s-api.crt": true, + "scheduler.crt": true, + "k0s-api.key": true, + "scheduler.key": true, + "konnectivity.conf": true, + "server.crt": true, + "konnectivity.crt": true, + "server.key": true, +} diff --git a/pkg/certs/ensure.go b/pkg/certs/ensure.go index 4adaa8eeff..2843090528 100644 --- a/pkg/certs/ensure.go +++ b/pkg/certs/ensure.go @@ -2,10 +2,14 @@ package certs import ( "context" + "errors" "fmt" + "io/fs" "os" "path/filepath" + "slices" + "golang.org/x/exp/maps" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,32 +36,14 @@ func EnsureCerts( return downloadCertsFromSecret(secret, certificateDir) } - // init config - cfg, err := SetInitDynamicDefaults() - if err != nil { - return err - } - - cfg.ClusterName = "kubernetes" - cfg.NodeRegistration.Name = vClusterName - cfg.Etcd.Local = &LocalEtcd{ - ServerCertSANs: etcdSans, - PeerCertSANs: etcdSans, - } - cfg.Networking.ServiceSubnet = serviceCIDR - cfg.Networking.DNSDomain = clusterDomain - cfg.ControlPlaneEndpoint = "127.0.0.1:6443" - cfg.CertificatesDir = certificateDir - cfg.LocalAPIEndpoint.AdvertiseAddress = "0.0.0.0" - cfg.LocalAPIEndpoint.BindPort = 443 - err = CreatePKIAssets(cfg) - if err != nil { - return fmt.Errorf("create pki assets: %w", err) - } - - err = CreateJoinControlPlaneKubeConfigFiles(cfg.CertificatesDir, cfg) - if err != nil { - return fmt.Errorf("create kube configs: %w", err) + // we check if the files are already there + _, err = os.Stat(filepath.Join(certificateDir, CAKeyName)) + if errors.Is(err, fs.ErrNotExist) { + // try to generate the certificates + err = generateCertificates(serviceCIDR, vClusterName, certificateDir, clusterDomain, etcdSans) + if err != nil { + return err + } } // build secret @@ -77,6 +63,15 @@ func EnsureCerts( secret.Data[toName] = data } + // find extra files in the folder and add them to the secret + extraFiles, err := extraFiles(certificateDir) + if err != nil { + return fmt.Errorf("read extra file: %w", err) + } + for k, v := range extraFiles { + secret.Data[k] = v + } + // finally create the secret secret, err = currentNamespaceClient.CoreV1().Secrets(currentNamespace).Create(ctx, secret, metav1.CreateOptions{}) if err != nil { @@ -96,26 +91,111 @@ func EnsureCerts( return downloadCertsFromSecret(secret, certificateDir) } +func generateCertificates( + serviceCIDR string, + vClusterName string, + certificateDir string, + clusterDomain string, + etcdSans []string, +) error { + // init config + cfg, err := SetInitDynamicDefaults() + if err != nil { + return err + } + + cfg.ClusterName = "kubernetes" + cfg.NodeRegistration.Name = vClusterName + cfg.Etcd.Local = &LocalEtcd{ + ServerCertSANs: etcdSans, + PeerCertSANs: etcdSans, + } + cfg.Networking.ServiceSubnet = serviceCIDR + cfg.Networking.DNSDomain = clusterDomain + cfg.ControlPlaneEndpoint = "127.0.0.1:6443" + cfg.CertificatesDir = certificateDir + cfg.LocalAPIEndpoint.AdvertiseAddress = "0.0.0.0" + cfg.LocalAPIEndpoint.BindPort = 443 + + // only create the files if the files are not there yet + err = CreatePKIAssets(cfg) + if err != nil { + return fmt.Errorf("create pki assets: %w", err) + } + + err = CreateJoinControlPlaneKubeConfigFiles(cfg.CertificatesDir, cfg) + if err != nil { + return fmt.Errorf("create kube configs: %w", err) + } + + return nil +} + +// downloadCertsFromSecret writes to the filesystem the content of each field in the secret +// if the field has an equivalent inside the certmap, we write with the corresponding name +// otherwise the file has the same name than the field func downloadCertsFromSecret( secret *corev1.Secret, certificateDir string, ) error { - for toName, fromName := range certMap { - if len(secret.Data[fromName]) == 0 { - return fmt.Errorf("secret is missing %s", fromName) + certMapValues := maps.Values(certMap) + for secretEntry, fileBytes := range secret.Data { + name := secretEntry + if slices.Contains(certMapValues, secretEntry) { + // we need to replace with the actual name + for key, sEntry := range certMap { + // guarranteed to evaluate to true at least once because of slices.contains + if sEntry == secretEntry { + if len(fileBytes) == 0 { + return fmt.Errorf("secret is missing %s", secretEntry) + } + name = key + break + } + } } - name := filepath.Join(certificateDir, toName) + name = filepath.Join(certificateDir, name) err := os.MkdirAll(filepath.Dir(name), 0777) if err != nil { return fmt.Errorf("create directory %s", filepath.Dir(name)) } - err = os.WriteFile(name, secret.Data[fromName], 0666) + err = os.WriteFile(name, fileBytes, 0666) if err != nil { - return fmt.Errorf("write %s: %w", fromName, err) + return fmt.Errorf("write %s: %w", name, err) } } return nil } + +func extraFiles(certificateDir string) (map[string][]byte, error) { + files := make(map[string][]byte) + entries, err := os.ReadDir(certificateDir) + if err != nil { + return nil, err + } + + for _, v := range entries { + if v.IsDir() { + // ignore subdirectories for now + // etcd files should be picked up by the map + continue + } + + // if it's not in the cert map, add to the map + name := v.Name() + _, ok := certMap[name] + if !ok { + b, err := os.ReadFile(filepath.Join(certificateDir, name)) + if err != nil { + return nil, err + } + + files[name] = b + } + } + + return files, err +} diff --git a/pkg/controllers/coredns/nodehosts.go b/pkg/controllers/coredns/nodehosts.go index fbc20680eb..0b786b27e8 100644 --- a/pkg/controllers/coredns/nodehosts.go +++ b/pkg/controllers/coredns/nodehosts.go @@ -100,7 +100,7 @@ func (r *NodeHostsReconciler) SetupWithManager(mgr ctrl.Manager) error { funcs := predicate.NewPredicateFuncs(p) // use modified handler to avoid triggering reconcile for each Node - eventHandler := handler.EnqueueRequestsFromMapFunc(func(_ context.Context, o client.Object) []reconcile.Request { + eventHandler := handler.EnqueueRequestsFromMapFunc(func(_ context.Context, _ client.Object) []reconcile.Request { return []reconcile.Request{{ NamespacedName: types.NamespacedName{Namespace: Namespace, Name: ConfigMapName}, }} diff --git a/pkg/controllers/resources/priorityclasses/syncer.go b/pkg/controllers/resources/priorityclasses/syncer.go index bcb73ecafd..107c9c230c 100644 --- a/pkg/controllers/resources/priorityclasses/syncer.go +++ b/pkg/controllers/resources/priorityclasses/syncer.go @@ -59,7 +59,7 @@ func (s *priorityClassSyncer) Sync(ctx *synccontext.SyncContext, pObj client.Obj } func NewPriorityClassTranslator() translate.PhysicalNameTranslator { - return func(vName string, vObj client.Object) string { + return func(vName string, _ client.Object) string { return translatePriorityClassName(vName) } } diff --git a/pkg/k0s/k0s.go b/pkg/k0s/k0s.go index 0633715fb2..3847c0640b 100644 --- a/pkg/k0s/k0s.go +++ b/pkg/k0s/k0s.go @@ -13,7 +13,9 @@ import ( "k8s.io/klog/v2" ) -const VClusterCommandEnv = "VCLUSTER_COMMAND" +const ( + VClusterCommandEnv = "VCLUSTER_COMMAND" +) type k0sCommand struct { Command []string `json:"command,omitempty"` @@ -22,12 +24,17 @@ type k0sCommand struct { const runDir = "/run/k0s" -func StartK0S(ctx context.Context) error { +func StartK0S(ctx context.Context, cancel context.CancelFunc) error { + // this is not really useful but go isn't happy if we don't cancel the context + // everywhere + defer cancel() + // make sure we delete the contents of /run/k0s dirEntries, _ := os.ReadDir(runDir) for _, entry := range dirEntries { _ = os.RemoveAll(filepath.Join(runDir, entry.Name())) } + // create command command := &k0sCommand{} err := yaml.Unmarshal([]byte(os.Getenv(VClusterCommandEnv)), command) diff --git a/pkg/server/filters/k3s_connect.go b/pkg/server/filters/k3s_connect.go index 927b47eb60..33c489e247 100644 --- a/pkg/server/filters/k3s_connect.go +++ b/pkg/server/filters/k3s_connect.go @@ -41,7 +41,7 @@ func WithK3sConnect(h http.Handler) http.Handler { HandshakeTimeout: 45 * time.Second, TLSClientConfig: tlsCfg, } - proxy.Backend = func(r *http.Request) *url.URL { + proxy.Backend = func(_ *http.Request) *url.URL { u := *serverURL u.Path = K3sConnectPath return &u diff --git a/pkg/setup/initialize.go b/pkg/setup/initialize.go index 26524937be..14eaf1ee45 100644 --- a/pkg/setup/initialize.go +++ b/pkg/setup/initialize.go @@ -3,7 +3,10 @@ package setup import ( "context" "fmt" + "io/fs" + "os" "path/filepath" + "reflect" "strconv" "time" @@ -16,6 +19,9 @@ import ( "github.com/loft-sh/vcluster/pkg/specialservices" "github.com/loft-sh/vcluster/pkg/telemetry" "github.com/loft-sh/vcluster/pkg/util/servicecidr" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" @@ -86,20 +92,34 @@ func initialize( switch distro { case constants.K0SDistro: // ensure service cidr - _, err := servicecidr.EnsureServiceCIDRInK0sSecret(ctx, workspaceNamespaceClient, currentNamespaceClient, workspaceNamespace, currentNamespace, vClusterName) + serviceCIDR, err := servicecidr.EnsureServiceCIDRInK0sSecret(ctx, workspaceNamespaceClient, currentNamespaceClient, workspaceNamespace, currentNamespace, vClusterName) + if err != nil { + return err + } + + // create certificates if they are not there yet + err = certs.EnsureCerts(ctx, serviceCIDR, currentNamespace, currentNamespaceClient, vClusterName, "/data/k0s/pki", "", nil) if err != nil { return err } // start k0s + parentCtxWithCancel, cancel := context.WithCancel(parentCtx) go func() { // we need to run this with the parent ctx as otherwise this context will be cancelled by the wait // loop in Initialize - err := k0s.StartK0S(parentCtx) + err := k0s.StartK0S(parentCtxWithCancel, cancel) if err != nil { klog.Fatalf("Error running k0s: %v", err) } }() + + // try to update the certs secret with the k0s certificates + err = UpdateSecretWithK0sCerts(ctx, currentNamespaceClient, currentNamespace, vClusterName) + if err != nil { + cancel() + return err + } case constants.K3SDistro: // its k3s, let's create the token secret k3sToken, err := k3s.EnsureK3SToken(ctx, currentNamespaceClient, currentNamespace, vClusterName) @@ -146,7 +166,6 @@ func initialize( } } - klog.Info("finished running initialize") return nil } @@ -181,3 +200,102 @@ func GenerateK8sCerts(ctx context.Context, currentNamespaceClient kubernetes.Int return nil } + +func UpdateSecretWithK0sCerts( + ctx context.Context, + currentNamespaceClient kubernetes.Interface, + currentNamespace, vClusterName string, +) error { + // wait for k0s to generate the secrets for us + files, err := waitForK0sFiles(ctx, "/data/k0s/pki") + if err != nil { + return err + } + + // retrieve cert secret + secret, err := currentNamespaceClient.CoreV1().Secrets(currentNamespace).Get(ctx, vClusterName+"-certs", metav1.GetOptions{}) + if err != nil { + return err + } else if secret.Data == nil { + return fmt.Errorf("error while trying to update the secret, data was empty, will try to fetch it again") + } + + // check if the secret contains the k0s files now, which would mean somebody was faster than we were + if secretContainsK0sCerts(secret) { + if secretIsUpToDate(secret, files) { + return nil + } + + return fmt.Errorf("error while trying to update the secret, it was already updated, will try to fetch it again") + } + + // update the secret to include the k0s certs + for fileName, content := range files { + secret.Data[fileName] = content + } + + // if any error we will retry from the poll loop + _, err = currentNamespaceClient.CoreV1().Secrets(currentNamespace).Update(ctx, secret, metav1.UpdateOptions{}) + return err +} + +func waitForK0sFiles(ctx context.Context, certDir string) (map[string][]byte, error) { + for { + filesFound := 0 + for file := range certs.K0sFiles { + _, err := os.ReadFile(filepath.Join(certDir, file)) + if errors.Is(err, fs.ErrNotExist) { + break + } else if err != nil { + return nil, err + } + + filesFound++ + } + if filesFound == len(certs.K0sFiles) { + break + } + + select { + case <-ctx.Done(): + return nil, context.DeadlineExceeded + case <-time.After(time.Second): + } + } + return readK0sFiles(certDir) +} + +func readK0sFiles(certDir string) (map[string][]byte, error) { + files := make(map[string][]byte) + for file := range certs.K0sFiles { + b, err := os.ReadFile(filepath.Join(certDir, file)) + if err != nil { + return nil, err + } + files[file] = b + } + + return files, nil +} + +func secretContainsK0sCerts(secret *corev1.Secret) bool { + if secret.Data == nil { + return false + } + for k := range secret.Data { + if certs.K0sFiles[k] { + return true + } + } + return false +} + +func secretIsUpToDate(secret *corev1.Secret, files map[string][]byte) bool { + for fileName, content := range files { + if !reflect.DeepEqual(secret.Data[fileName], content) { + return false + } + } + + return true +} diff --git a/pkg/util/crds.go b/pkg/util/crds.go index 4fb2d7fff6..927edb34de 100644 --- a/pkg/util/crds.go +++ b/pkg/util/crds.go @@ -37,7 +37,7 @@ func EnsureCRD(ctx context.Context, config *rest.Config, manifest []byte, groupV } var lastErr error - err = wait.ExponentialBackoffWithContext(ctx, wait.Backoff{Duration: time.Second, Factor: 1.5, Cap: time.Minute, Steps: math.MaxInt32}, func(ctx context.Context) (bool, error) { + err = wait.ExponentialBackoffWithContext(ctx, wait.Backoff{Duration: time.Second, Factor: 1.5, Cap: time.Minute, Steps: math.MaxInt32}, func(_ context.Context) (bool, error) { var found bool found, lastErr = KindExists(config, groupVersionKind) return found, nil diff --git a/pkg/util/translate/multi_namespace.go b/pkg/util/translate/multi_namespace.go index d173dbc0c8..b2a8ae64f5 100644 --- a/pkg/util/translate/multi_namespace.go +++ b/pkg/util/translate/multi_namespace.go @@ -182,7 +182,7 @@ func (s *multiNamespace) LegacyGetTargetNamespace() (string, error) { } func (s *multiNamespace) ApplyMetadata(vObj client.Object, syncedLabels []string, excludedAnnotations ...string) client.Object { - pObj, err := s.SetupMetadataWithName(vObj, func(vName string, vObj client.Object) string { + pObj, err := s.SetupMetadataWithName(vObj, func(_ string, vObj client.Object) string { return s.objectPhysicalName(vObj) }) if err != nil { diff --git a/pkg/util/translate/single_namespace.go b/pkg/util/translate/single_namespace.go index aa81221d19..17a8fc2d02 100644 --- a/pkg/util/translate/single_namespace.go +++ b/pkg/util/translate/single_namespace.go @@ -171,7 +171,7 @@ func (s *singleNamespace) LegacyGetTargetNamespace() (string, error) { } func (s *singleNamespace) ApplyMetadata(vObj client.Object, syncedLabels []string, excludedAnnotations ...string) client.Object { - pObj, err := s.SetupMetadataWithName(vObj, func(vName string, vObj client.Object) string { + pObj, err := s.SetupMetadataWithName(vObj, func(_ string, vObj client.Object) string { return s.objectPhysicalName(vObj) }) if err != nil {