diff --git a/Dockerfile b/Dockerfile index 1e88ea03..1f21362f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.20 as builder +FROM golang:1.22 as builder WORKDIR /vcluster-hpm-dev ARG TARGETOS diff --git a/cmd/hostpaths/hostpaths.go b/cmd/hostpaths/hostpaths.go index a81debd3..70226615 100644 --- a/cmd/hostpaths/hostpaths.go +++ b/cmd/hostpaths/hostpaths.go @@ -6,9 +6,11 @@ import ( "io/fs" "os" "path/filepath" + "slices" "strings" "time" + "github.com/loft-sh/vcluster/pkg/controllers/resources/namespaces" podtranslate "github.com/loft-sh/vcluster/pkg/controllers/resources/pods/translate" "github.com/loft-sh/vcluster/pkg/util/clienthelper" @@ -18,13 +20,12 @@ import ( "github.com/loft-sh/vcluster/pkg/util/translate" "github.com/pkg/errors" "github.com/spf13/cobra" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -168,13 +169,17 @@ func Start(ctx context.Context, options *context2.VirtualClusterOptions, init bo return err } - localManager, err := ctrl.NewManager(inClusterConfig, ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: "0", - LeaderElection: false, - Namespace: options.TargetNamespace, - NewClient: pluginhookclient.NewPhysicalPluginClientFactory(blockingcacheclient.NewCacheClient), - }) + kubeClient, err := kubernetes.NewForConfig(inClusterConfig) + if err != nil { + return errors.Wrap(err, "create kube client") + } + + err = findVclusterModeAndSetDefaultTranslation(ctx, kubeClient, options) + if err != nil { + return errors.Wrap(err, "find vcluster mode") + } + + localManager, err := ctrl.NewManager(inClusterConfig, localManagerCtrlOptions(options)) if err != nil { return err } @@ -193,11 +198,6 @@ func Start(ctx context.Context, options *context2.VirtualClusterOptions, init bo startManagers(ctx, localManager, virtualClusterManager) - err = findVclusterModeAndSetDefaultTranslation(ctx, localManager, options) - if err != nil { - return err - } - if init { klog.Info("is init container mode") defer ctx.Done() @@ -216,28 +216,14 @@ func Start(ctx context.Context, options *context2.VirtualClusterOptions, init bo return nil } -func getVclusterObject(ctx context.Context, localManager manager.Manager, vclusterName, vclusterNamespace string, object client.Object) error { - err := localManager.GetClient().Get(ctx, types.NamespacedName{ - Name: vclusterName, - Namespace: vclusterNamespace, - }, object) - if err != nil { - return err - } - - return nil -} - -func getSyncerPodSpec(ctx context.Context, localManager manager.Manager, vclusterName, vclusterNamespace string) (*corev1.PodSpec, error) { +func getSyncerPodSpec(ctx context.Context, kubeClient kubernetes.Interface, vclusterName, vclusterNamespace string) (*corev1.PodSpec, error) { // try looking for the stateful set first - vclusterSts := &appsv1.StatefulSet{} - err := getVclusterObject(ctx, localManager, vclusterName, vclusterNamespace, vclusterSts) + vclusterSts, err := kubeClient.AppsV1().StatefulSets(vclusterNamespace).Get(ctx, vclusterName, metav1.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { // try looking for deployment - in case of eks/k8s - vclusterDeploy := &appsv1.Deployment{} - err := getVclusterObject(ctx, localManager, vclusterName, vclusterNamespace, vclusterDeploy) + vclusterDeploy, err := kubeClient.AppsV1().Deployments(vclusterNamespace).Get(ctx, vclusterName, metav1.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { klog.Errorf("could not find vcluster either in statefulset or deployment: %v", err) @@ -257,8 +243,23 @@ func getSyncerPodSpec(ctx context.Context, localManager manager.Manager, vcluste return &vclusterSts.Spec.Template.Spec, nil } -func findVclusterModeAndSetDefaultTranslation(ctx context.Context, localManager manager.Manager, options *context2.VirtualClusterOptions) error { - vclusterPodSpec, err := getSyncerPodSpec(ctx, localManager, options.Name, options.TargetNamespace) +func localManagerCtrlOptions(options *context2.VirtualClusterOptions) manager.Options { + controllerOptions := ctrl.Options{ + Scheme: scheme, + MetricsBindAddress: "0", + LeaderElection: false, + NewClient: pluginhookclient.NewPhysicalPluginClientFactory(blockingcacheclient.NewCacheClient), + } + + if !options.MultiNamespaceMode { + controllerOptions.Cache.Namespaces = []string{options.TargetNamespace} + } + + return controllerOptions +} + +func findVclusterModeAndSetDefaultTranslation(ctx context.Context, kubeClient kubernetes.Interface, options *context2.VirtualClusterOptions) error { + vclusterPodSpec, err := getSyncerPodSpec(ctx, kubeClient, options.Name, options.TargetNamespace) if err != nil { return err } @@ -268,6 +269,7 @@ func findVclusterModeAndSetDefaultTranslation(ctx context.Context, localManager // iterate over command args for _, arg := range container.Args { if strings.Contains(arg, MultiNamespaceMode) { + options.MultiNamespaceMode = true translate.Default = translate.NewMultiNamespaceTranslator(options.TargetNamespace) return nil } @@ -339,17 +341,28 @@ func mapHostPaths(ctx context.Context, pManager, vManager manager.Manager) { wait.Forever(func() { podList := &corev1.PodList{} - err := pManager.GetClient().List(ctx, podList, &client.ListOptions{ - Namespace: options.TargetNamespace, + podListOptions := &client.ListOptions{ FieldSelector: fields.SelectorFromSet(fields.Set{ NodeIndexName: os.Getenv(HostpathMapperSelfNodeNameEnvVar), }), - }) + } + + if !options.MultiNamespaceMode { + podListOptions.Namespace = options.TargetNamespace + } + + err := pManager.GetClient().List(ctx, podList, podListOptions) if err != nil { klog.Errorf("unable to list pods: %v", err) return } + err = filterManagedPods(ctx, options, podList, pManager) + if err != nil { + klog.Errorf("unable to filter pods: %v", err) + return + } + podMappings := make(PhysicalPodMap) fillUpPodMapping(ctx, podList, podMappings) @@ -420,6 +433,32 @@ func mapHostPaths(ctx context.Context, pManager, vManager manager.Manager) { }, time.Second*5) } +func filterManagedPods(ctx context.Context, options *context2.VirtualClusterOptions, podList *corev1.PodList, pManager manager.Manager) error { + if options.MultiNamespaceMode { + // Limit Pods + nsList := &corev1.NamespaceList{} + err := pManager.GetClient().List(ctx, nsList, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(labels.Set{ + namespaces.VclusterNamespaceAnnotation: options.TargetNamespace, + }), + }) + if err != nil { + return errors.Wrap(err, "unable to list namespaces") + } + + managedNamespaces := make([]string, 0, len(nsList.Items)) + for _, ns := range nsList.Items { + managedNamespaces = append(managedNamespaces, ns.Name) + } + + podList.Items = slices.DeleteFunc[[]corev1.Pod, corev1.Pod](podList.Items, func(pod corev1.Pod) bool { + return !slices.Contains[[]string, string](managedNamespaces, pod.Namespace) + }) + } + + return nil +} + func cleanupOldContainerPaths(ctx context.Context, existingVPodsWithNS map[string]bool) error { options := ctx.Value(optionsKey).(*context2.VirtualClusterOptions) diff --git a/vendor/github.com/loft-sh/vcluster/pkg/controllers/resources/namespaces/syncer.go b/vendor/github.com/loft-sh/vcluster/pkg/controllers/resources/namespaces/syncer.go new file mode 100644 index 00000000..1412a4ad --- /dev/null +++ b/vendor/github.com/loft-sh/vcluster/pkg/controllers/resources/namespaces/syncer.go @@ -0,0 +1,125 @@ +package namespaces + +import ( + "fmt" + "strings" + + "github.com/loft-sh/vcluster/pkg/constants" + "github.com/loft-sh/vcluster/pkg/controllers/syncer" + synccontext "github.com/loft-sh/vcluster/pkg/controllers/syncer/context" + "github.com/loft-sh/vcluster/pkg/controllers/syncer/translator" + "github.com/loft-sh/vcluster/pkg/util/translate" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/validation" + "k8s.io/apimachinery/pkg/util/validation/field" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// Unsafe annotations based on the docs here: +// https://kubernetes.io/docs/reference/labels-annotations-taints/ +var excludedAnnotations = []string{ + "scheduler.alpha.kubernetes.io/node-selector", + "scheduler.alpha.kubernetes.io/defaultTolerations", +} + +const ( + VclusterNameAnnotation = "vcluster.loft.sh/vcluster-name" + VclusterNamespaceAnnotation = "vcluster.loft.sh/vcluster-namespace" +) + +func New(ctx *synccontext.RegisterContext) (syncer.Object, error) { + namespaceLabels, err := parseNamespaceLabels(ctx.Options.NamespaceLabels) + if err != nil { + return nil, fmt.Errorf("invalid value of the namespace-labels flag: %v", err) + } + + namespaceLabels[VclusterNameAnnotation] = ctx.Options.Name + namespaceLabels[VclusterNamespaceAnnotation] = ctx.CurrentNamespace + + return &namespaceSyncer{ + Translator: translator.NewClusterTranslator(ctx, "namespace", &corev1.Namespace{}, NamespaceNameTranslator, excludedAnnotations...), + workloadServiceAccountName: ctx.Options.ServiceAccount, + namespaceLabels: namespaceLabels, + }, nil +} + +type namespaceSyncer struct { + translator.Translator + workloadServiceAccountName string + namespaceLabels map[string]string +} + +var _ syncer.IndicesRegisterer = &namespaceSyncer{} + +func (s *namespaceSyncer) RegisterIndices(ctx *synccontext.RegisterContext) error { + return ctx.VirtualManager.GetFieldIndexer().IndexField(ctx.Context, &corev1.Namespace{}, constants.IndexByPhysicalName, func(rawObj client.Object) []string { + return []string{NamespaceNameTranslator(rawObj.GetName(), rawObj)} + }) +} + +var _ syncer.Syncer = &namespaceSyncer{} + +func (s *namespaceSyncer) SyncDown(ctx *synccontext.SyncContext, vObj client.Object) (ctrl.Result, error) { + newNamespace := s.translate(ctx.Context, vObj.(*corev1.Namespace)) + ctx.Log.Infof("create physical namespace %s", newNamespace.Name) + err := ctx.PhysicalClient.Create(ctx.Context, newNamespace) + if err != nil { + ctx.Log.Infof("error syncing %s to physical cluster: %v", vObj.GetName(), err) + return ctrl.Result{}, err + } + + return ctrl.Result{}, s.EnsureWorkloadServiceAccount(ctx, newNamespace.Name) +} + +func (s *namespaceSyncer) Sync(ctx *synccontext.SyncContext, pObj client.Object, vObj client.Object) (ctrl.Result, error) { + updated := s.translateUpdate(ctx.Context, pObj.(*corev1.Namespace), vObj.(*corev1.Namespace)) + if updated != nil { + ctx.Log.Infof("updating physical namespace %s, because virtual namespace has changed", updated.Name) + translator.PrintChanges(pObj, updated, ctx.Log) + err := ctx.PhysicalClient.Update(ctx.Context, updated) + if err != nil { + return ctrl.Result{}, err + } + } + + return ctrl.Result{}, s.EnsureWorkloadServiceAccount(ctx, pObj.GetName()) +} + +func (s *namespaceSyncer) EnsureWorkloadServiceAccount(ctx *synccontext.SyncContext, pNamespace string) error { + if s.workloadServiceAccountName == "" { + return nil + } + + svc := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: pNamespace, + Name: s.workloadServiceAccountName, + }, + } + _, err := controllerutil.CreateOrPatch(ctx.Context, ctx.PhysicalClient, svc, func() error { return nil }) + return err +} + +func NamespaceNameTranslator(vName string, _ client.Object) string { + return translate.Default.PhysicalNamespace(vName) +} + +func parseNamespaceLabels(labels []string) (map[string]string, error) { + out := map[string]string{} + for _, v := range labels { + parts := strings.SplitN(v, "=", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("incorrect format, expected: key=value got: %s", v) + } + out[parts[0]] = parts[1] + } + errs := validation.ValidateLabels(out, field.NewPath("namespace-labels")) + if len(errs) != 0 { + return nil, fmt.Errorf("invalid labels: %v", errs) + } + + return out, nil +} diff --git a/vendor/github.com/loft-sh/vcluster/pkg/controllers/resources/namespaces/translate.go b/vendor/github.com/loft-sh/vcluster/pkg/controllers/resources/namespaces/translate.go new file mode 100644 index 00000000..4ab8c56c --- /dev/null +++ b/vendor/github.com/loft-sh/vcluster/pkg/controllers/resources/namespaces/translate.go @@ -0,0 +1,41 @@ +package namespaces + +import ( + "context" + + "github.com/loft-sh/vcluster/pkg/controllers/syncer/translator" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func (s *namespaceSyncer) translate(ctx context.Context, vObj client.Object) *corev1.Namespace { + newNamespace := s.TranslateMetadata(ctx, vObj).(*corev1.Namespace) + + // add user defined namespace labels + for k, v := range s.namespaceLabels { + newNamespace.Labels[k] = v + } + + return newNamespace +} + +func (s *namespaceSyncer) translateUpdate(ctx context.Context, pObj, vObj *corev1.Namespace) *corev1.Namespace { + var updated *corev1.Namespace + + _, updatedAnnotations, updatedLabels := s.TranslateMetadataUpdate(ctx, vObj, pObj) + // add user defined namespace labels + for k, v := range s.namespaceLabels { + updatedLabels[k] = v + } + // set the kubernetes.io/metadata.name label + updatedLabels[corev1.LabelMetadataName] = pObj.Name + // check if any labels or annotations changed + if !equality.Semantic.DeepEqual(updatedAnnotations, pObj.GetAnnotations()) || !equality.Semantic.DeepEqual(updatedLabels, pObj.GetLabels()) { + updated = translator.NewIfNil(updated, pObj) + updated.Annotations = updatedAnnotations + updated.Labels = updatedLabels + } + + return updated +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 6f6047a1..c519d007 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -156,6 +156,7 @@ github.com/liggitt/tabwriter github.com/loft-sh/vcluster/cmd/vcluster/context github.com/loft-sh/vcluster/pkg/constants github.com/loft-sh/vcluster/pkg/controllers/resources/configmaps +github.com/loft-sh/vcluster/pkg/controllers/resources/namespaces github.com/loft-sh/vcluster/pkg/controllers/resources/pods/translate github.com/loft-sh/vcluster/pkg/controllers/resources/priorityclasses github.com/loft-sh/vcluster/pkg/controllers/syncer