diff --git a/cmd/vcluster/cmd/start.go b/cmd/vcluster/cmd/start.go index 36137e62fb..644d493d51 100644 --- a/cmd/vcluster/cmd/start.go +++ b/cmd/vcluster/cmd/start.go @@ -6,9 +6,9 @@ import ( "runtime/debug" "github.com/loft-sh/vcluster/pkg/leaderelection" + "github.com/loft-sh/vcluster/pkg/options" "github.com/loft-sh/vcluster/pkg/scheme" "github.com/loft-sh/vcluster/pkg/setup" - "github.com/loft-sh/vcluster/pkg/setup/options" "github.com/loft-sh/vcluster/pkg/telemetry" "github.com/loft-sh/vcluster/pkg/util/blockingcacheclient" "github.com/loft-sh/vcluster/pkg/util/clienthelper" diff --git a/docs/pages/advanced-topics/plugins-development.mdx b/docs/pages/advanced-topics/plugins-development.mdx index 6e0b2f327e..a108739a1f 100644 --- a/docs/pages/advanced-topics/plugins-development.mdx +++ b/docs/pages/advanced-topics/plugins-development.mdx @@ -3,8 +3,11 @@ title: "Development tutorial" sidebar_label: "Development tutorial" --- -In this tutorial, we will implement a ConfigMap syncer. Vcluster syncs ConfigMaps out of the box, but only those that are used by one of the pods created in vCluster. Here we will have a step-by-step look at a plugin implementation that will synchronize all ConfigMaps using the [vcluster plugin SDK](https://github.com/loft-sh/vcluster-sdk). +In this tutorial we will implement a Car syncer. Here we will have a step-by-step look at a plugin implementation that will synchronize all custom car objects using the [vCluster plugin SDK](https://github.com/loft-sh/vcluster-sdk). +:::info +You can find other examples in the [vCluster SDK Repository](https://github.com/loft-sh/vcluster-sdk/tree/main/examples) +::: ### Prerequisites @@ -22,136 +25,136 @@ Check out the vCluster plugin example via: git clone https://github.com/loft-sh/vcluster-plugin-example.git ``` -You'll see a bunch of files already created, but let's take a look at the `main.go` file: +You'll see a bunch of files already created, but lets take a look at the `main.go` file: ``` package main import ( + "github.com/loft-sh/vcluster-plugin-example/syncers" "github.com/loft-sh/vcluster-sdk/plugin" - "github.com/loft-sh/vcluster-sync-all-configmaps/syncers" ) func main() { - ctx := plugin.MustInit("sync-all-configmaps-plugin") - plugin.MustRegister(syncers.NewConfigMapSyncer(ctx)) + ctx := plugin.MustInit() + plugin.MustRegister(syncers.NewCarSyncer(ctx)) plugin.MustStart() } ``` Let's break down what is happening in the `main()` function above. -`ctx := plugin.MustInit("sync-all-configmaps-plugin")` - SDK will contact the vCluster backend server and retrieve it's configuration. The returned struct of type [`RegisterContext`](https://pkg.go.dev/github.com/loft-sh/vcluster-sdk/syncer/context#RegisterContext) contains information about vCluster flags, namespace, vCluster client config, controller manager objects, etc. +`ctx := plugin.MustInit()` - SDK will init the plugin and retrieve configuration from the vCluster syncer. The returned struct contains information about vCluster flags, namespace, vCluster client config, controller manager objects, etc. -`plugin.MustRegister(syncers.NewConfigMapSyncer(ctx))` - we will implement the `NewConfigMapSyncer` function below, but for now, all we need to know is that it should return a struct that implements [`Base`](https://pkg.go.dev/github.com/loft-sh/vcluster-sdk/syncer#Base) interface, which is accepted by the [`MustRegister`](https://pkg.go.dev/github.com/loft-sh/vcluster-sdk/plugin#MustRegister) function. We should call [`MustRegister`](https://pkg.go.dev/github.com/loft-sh/vcluster-sdk/plugin#MustRegister) function for each syncer that we wish to be managed by the plugins controller manager. - -`plugin.MustStart()` - this blocking function will wait until the vCluster pod where this plugin container is running becomes the leader. Next, it will call the `Init()` and `RegisterIndices()` functions on the syncers that implement the [`Initializer`](https://pkg.go.dev/github.com/loft-sh/vcluster-sdk/syncer#Initializer) and [`IndicesRegisterer`](https://pkg.go.dev/github.com/loft-sh/vcluster-sdk/syncer#IndicesRegisterer) respectively. Afterwards, the SDK will start its controller managers and call the `RegisterSyncer` or `RegisterFakeSyncer` function on the syncers that implement [`FakeSyncer`](https://pkg.go.dev/github.com/loft-sh/vcluster-sdk/syncer#FakeSyncer) and [`Syncer`](https://pkg.go.dev/github.com/loft-sh/vcluster-sdk/syncer#Syncer) interfaces. Additionally, after configuring the default controller for the syncers, the `ModifyController` function is called for the syncers that implement [`ControllerModifier`](https://pkg.go.dev/github.com/loft-sh/vcluster-sdk/syncer#ControllerModifier) interface, which gives a plugin developer a chance to interact with the controller builder object. All these interfaces act like hooks into different points of the SDK to allow you to customize the controller that will call your syncer based on the changes to the watched resources. +`plugin.MustRegister(syncers.NewCarSyncer(ctx))` - we will implement the `NewCarSyncer` function below, but for now, all we need to know is that it should return a struct that implements an interface which is accepted by the `MustRegister` function. +`plugin.MustStart()` - this blocking function will wait until the vCluster pod where this plugin is running becomes the leader. ### Implementing a syncer for a namespaced resource -In this chapter, we take a look at the `sync-all-configmaps.go` file that can be found in the `syncer` directory. +In this chapter, we take a look at the `car.go` file that can be found in the `syncer` directory. ``` package syncers import ( - "github.com/loft-sh/vcluster-sdk/syncer" - syncercontext "github.com/loft-sh/vcluster-sdk/syncer/context" - "github.com/loft-sh/vcluster-sdk/syncer/translator" - corev1 "k8s.io/api/core/v1" + "context" + "os" + + examplev1 "github.com/loft-sh/vcluster-plugin-example/apis/v1" + synccontext "github.com/loft-sh/vcluster/pkg/controllers/syncer/context" + "github.com/loft-sh/vcluster/pkg/controllers/syncer/translator" + "github.com/loft-sh/vcluster/pkg/scheme" + synctypes "github.com/loft-sh/vcluster/pkg/types" + "github.com/loft-sh/vcluster/pkg/util" + "github.com/loft-sh/vcluster/pkg/util/translate" "k8s.io/apimachinery/pkg/api/equality" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) -func NewConfigMapSyncer(ctx *syncercontext.RegisterContext) syncer.Syncer { - return &configMapSyncer{ - NamespacedTranslator: translator.NewNamespacedTranslator(ctx, "configmap", &corev1.ConfigMap{}), +func init() { + // Make sure our scheme is registered + _ = examplev1.AddToScheme(scheme.Scheme) +} + +func NewCarSyncer(ctx *synccontext.RegisterContext) synctypes.Base { + return &carSyncer{ + NamespacedTranslator: translator.NewNamespacedTranslator(ctx, "car", &examplev1.Car{}), } } -type configMapSyncer struct { +type carSyncer struct { translator.NamespacedTranslator } ``` -After an import block, we see the `NewConfigMapSyncer` function, which is being called from the `main.go`. It returns a new instance of the `configMapSyncer` struct, which is defined by a single nested anonymous struct of type [`NamespacedTranslator`](https://pkg.go.dev/github.com/loft-sh/vcluster-sdk/syncer/translator#NamespacedTranslator). The [`NamespacedTranslator`](https://pkg.go.dev/github.com/loft-sh/vcluster-sdk/syncer/translator#NamespacedTranslator) implements many functions of the [`Syncer`](https://pkg.go.dev/github.com/loft-sh/vcluster-sdk/syncer#Syncer) interface for us, and we will implement the remaining ones - `SyncDown` and `Sync`. - -:::info -You can get more familiar with the interfaces mentioned above by reading the SDK source files on GitHub - [vcluster-sdk/syncer/types.go](https://github.com/loft-sh/vcluster-sdk/blob/main/syncer/types.go) and [vcluster-sdk/syncer/translator/translator.go](https://github.com/loft-sh/vcluster-sdk/blob/main/syncer/translator/translator.go), or by using pkg.go.dev website - [Syncer](https://pkg.go.dev/github.com/loft-sh/vcluster-sdk/syncer#Syncer) and [NamespacedTranslator](https://pkg.go.dev/github.com/loft-sh/vcluster-sdk/syncer/translator#NamespacedTranslator). -::: +After an import block, we see the `NewCarSyncer` function, which is being called from the `main.go`. It returns a new instance of the `carSyncer` struct, which is defined by a single nested anonymous struct of type `NamespacedTranslator`. The `NamespacedTranslator` implements many functions of the [`Syncer`](https://pkg.go.dev/github.com/loft-sh/vcluster-sdk/syncer#Syncer) interface for us, and we will implement the remaining ones - `SyncDown` and `Sync`. - -The `SyncDown` function mentioned above is called by the vCluster SDK when a given resource, e.g. a ConfigMap, is created in the vCluster, but it doesn't exist in the host cluster yet. To create a ConfigMap in the host cluster we will call the `SyncToHostCreate` function with the output of the `translate` function as a third parameter. This demonstrates a typical pattern used in the vCluster syncer implementations. +The `SyncDown` function mentioned above is called by the vCluster SDK when a given resource, e.g. a Car, is created in the vCluster, but it doesn't exist in the host cluster yet. To create a ConfigMap in the host cluster we will call the `SyncToHostCreate` function with the output of the `translate` function as third parameter. This demonstrates a typical pattern used in the vCluster syncer implementations. ``` -func (s *configMapSyncer) SyncToHost(ctx *syncercontext.syncercontext, vObj client.Object) (ctrl.Result, error) { - return s.SyncToHostCreate(ctx, vObj, s.translate(vObj.(*corev1.ConfigMap))) +func (s *carSyncer) SyncToHost(ctx *synccontext.SyncContext, vObj client.Object) (ctrl.Result, error) { + return s.SyncToHostCreate(ctx, vObj, s.TranslateMetadata(ctx.Context, vObj).(*examplev1.Car)) } -func (s *configMapSyncer) translate(vObj client.Object) *corev1.ConfigMap { - return s.TranslateMetadata(vObj).(*corev1.ConfigMap) +func (s *carSyncer) Sync(ctx *synccontext.SyncContext, pObj client.Object, vObj client.Object) (ctrl.Result, error) { + return s.SyncToHostUpdate(ctx, vObj, s.translateUpdate(ctx.Context, pObj.(*examplev1.Car), vObj.(*examplev1.Car))) } ``` -The `TranslateMetadata` function used above produces a ConfigMap object that will be created in the host cluster. It is a deep copy of the ConfigMap from vCluster, but with certain metadata modifications - the name and labels are transformed, some vCluster labels and annotations are added, and many metadata fields are stripped (uid, resourceVersion, etc.). +The `TranslateMetadata` function used above produces a Car object that will be created in the host cluster. It is a deep copy of the Car from vCluster, but with certain metadata modifications - the name and labels are transformed, some vCluster labels and annotations are added, many metadata fields are stripped (uid, resourceVersion, etc.). -Next, we need to implement code that will handle the updates of the ConfigMap. When a ConfigMap in vCluster or host cluster is updated, the vCluster SDK will call the `Sync` function of the syncer. Current ConfigMap resources from the host cluster and from vCluster are passed as the second and third parameters respectively. In the implementation below, you can see another pattern used by the vCluster syncers. The `translateUpdate` function will return nil when no change to the ConfigMap in the host cluster is needed, and the `SyncToHostUpdate` function will not do an unnecessary update API call in such case. +Next, we need to implement code that will handle the updates of the Car. When a CAr in vCluster or host cluster is updated, the vCluster SDK will call the `Sync` function of the syncer. Current Car resource from the host cluster and from vCluster are passed as the second and third parameters respectively. In the implementation below, you can see another pattern used by the vCluster syncers. The `translateUpdate` function will return nil when no change to the Car in the host cluster is needed, and the `SyncToHostUpdate` function will not do an unnecessary update API call in such case. ``` -func (s *configMapSyncer) Sync(ctx *syncercontext.SyncContext, pObj client.Object, vObj client.Object) (ctrl.Result, error) { - return s.SyncToHostUpdate(ctx, vObj, s.translateUpdate(pObj.(*corev1.ConfigMap), vObj.(*corev1.ConfigMap))) +func (s *carSyncer) Sync(ctx *synccontext.SyncContext, pObj client.Object, vObj client.Object) (ctrl.Result, error) { + return s.SyncToHostUpdate(ctx, vObj, s.translateUpdate(ctx.Context, pObj.(*examplev1.Car), vObj.(*examplev1.Car))) } -func (s *configMapSyncer) translateUpdate(pObj, vObj *corev1.ConfigMap) *corev1.ConfigMap { - var updated *corev1.ConfigMap +func (s *carSyncer) translateUpdate(ctx context.Context, pObj, vObj *examplev1.Car) *examplev1.Car { + var updated *examplev1.Car - changed, updatedAnnotations, updatedLabels := s.TranslateMetadataUpdate(vObj, pObj) + // check annotations & labels + changed, updatedAnnotations, updatedLabels := s.TranslateMetadataUpdate(ctx, vObj, pObj) if changed { updated = translator.NewIfNil(updated, pObj) updated.Labels = updatedLabels updated.Annotations = updatedAnnotations } - // check if the data has changed - if !equality.Semantic.DeepEqual(vObj.Data, pObj.Data) { + // check spec + if !equality.Semantic.DeepEqual(vObj.Spec, pObj.Spec) { updated = translator.NewIfNil(updated, pObj) - updated.Data = vObj.Data + updated.Spec = vObj.Spec } - // check if the binary data has changed - if !equality.Semantic.DeepEqual(vObj.BinaryData, pObj.BinaryData) { - updated = translator.NewIfNil(updated, pObj) - updated.BinaryData = vObj.BinaryData - } return updated } ``` -As you might have noticed, the changes to the Immutable field of the ConfigMap are not being checked and propagated to the updated ConfigMap. That is done just for the simplification of the code in this tutorial. In real world use cases, there will likely be many scenarios and edge cases that you will need to handle differently than just with a simple comparison and assignment. For example, you will need to look out for label selectors that are interpreted in the host cluster, e.g. pod selectors in the NetworkPolicy resources are interpreted by the host cluster network plugin. Such selectors must be translated when synced down to the host resources. Several functions for the common use cases are [built into the SDK in the `syncer/translator` package](https://pkg.go.dev/github.com/loft-sh/vcluster-sdk/syncer/translator#pkg-functions), including the `TranslateLabelSelector` function. - -Also, notice that this example lacks the updates to the ConfigMap resource in vCluster. Here we propagate the changes only down to the ConfigMap in the host cluster, but there are resources or use cases where a syncer would update the synced resource in vCluster. For example, this might be an update of the status subresource or synchronization of any other field that some controller sets on the host side, e.g., finalizers. Implementation of such updates needs to be considered on a case-by-case basis. -For some use cases, you may need to sync the resources in the opposite direction, from the host cluster up into the vCluster, or even in both directions. If that is what your plugin needs to do, you will implement the [`UpSyncer`](https://pkg.go.dev/github.com/loft-sh/vcluster-sdk/syncer#UpSyncer) interface defined by the SDK. +Here we propagate the changes only down to the Car in the host cluster, but there are resources or use cases where a syncer would update the synced resource in vCluster. For example, this might be an update of the status subresource or synchronization of any other field that some controller sets on the host side, e.g., finalizers. Implementation of such updates needs to be considered on case-by-case basis. +For some use cases, you may need to sync the resources in the opposite direction, from the host cluster up into the vCluster, or even in both directions. If that is what your plugin needs to do, you will implement the `UpSyncer` interface defined by the SDK. ### Adding a hook for changing a resource on the fly -Hooks are a great feature to adjust the current syncing behaviour of vCluster without the need to override an already existing syncer in vCluster completely. They allow you to change outgoing objects of vCluster similar to an mutating admission controller in Kubernetes. The requirement for a hook to work correctly is that vCluster itself would sync the resource, so hooks only work for the core resources that are synced by vCluster such as pods, services, secrets etc. +Hooks are a great feature to adjust current syncing behaviour of vCluster without the need to override an already existing syncer in vCluster completely. They allow you to change outgoing objects of vCluster similar to an mutating admission controller in Kubernetes. Requirement for an hook to work correctly is that vCluster itself would sync the resource, so hooks only work for the core resources that are synced by vCluster such as pods, services, secrets etc. To add a hook to your plugin, you simply need to create a new struct that implements the `ClientHook` interface: ``` -package myhook +package hooks import ( "context" "fmt" - "github.com/loft-sh/vcluster-sdk/hook" + + "github.com/loft-sh/vcluster-sdk/plugin" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) -func NewPodHook() hook.ClientHook { +func NewPodHook() plugin.ClientHook { return &podHook{} } @@ -204,7 +207,7 @@ type MutateGetPhysical interface { By implementing one or more of the above interfaces you will receive events from vCluster that allows you to mutate an outgoing or incoming object to vCluster. For example, to add an hook that adds a custom label to a pod, you can add the following code: ``` -var _ hook.MutateCreatePhysical = &podHook{} +var _ plugin.MutateCreatePhysical = &podHook{} func (p *podHook) MutateCreatePhysical(ctx context.Context, obj client.Object) (client.Object, error) { pod, ok := obj.(*corev1.Pod) @@ -219,7 +222,7 @@ func (p *podHook) MutateCreatePhysical(ctx context.Context, obj client.Object) ( return pod, nil } -var _ hook.MutateUpdatePhysical = &podHook{} +var _ plugin.MutateUpdatePhysical = &podHook{} func (p *podHook) MutateUpdatePhysical(ctx context.Context, obj client.Object) (client.Object, error) { pod, ok := obj.(*corev1.Pod) @@ -241,28 +244,41 @@ This can be useful if you don't want vCluster to change something you have mutat ### Build and push your plugin Now you can run docker commands to build your container image and push it to the registry. -``` -docker build -t your_org/vcluster-sync-all-configmaps . && docker push your_org/vcluster-sync-all-configmaps -``` +`docker build -t your_org/vcluster-plugin-example . && docker push your_org/vcluster-plugin-example` ### Add plugin.yaml The last step before installing your plugin is creating a yaml file with your plugin metadata. This file follows the format of the Helm values files. It will be merged with other values files when a vCluster is installed or upgraded. For the plugin we just implemented and built it would look like this: ``` +# Plugin Definition below. This is essentially a valid helm values file that will be merged +# with the other vcluster values during vcluster create or helm install. plugin: - sync-all-configmaps-plugin: - image: your_org/vcluster-sync-all-configmaps -syncer: - extraArgs: - - "--sync=-configmaps" + vcluster-plugin-example: + version: v2 + image: ghcr.io/loft-sh/vcluster-plugin-example:v1 + rbac: + role: + extraRules: + - apiGroups: ["example.loft.sh"] + resources: ["cars"] + verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] + clusterRole: + extraRules: + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "list", "watch"] + +# Make sure the cluster role is enabled or otherwise the plugin won't be able to watch custom +# resource definitions. +rbac: + clusterRole: + create: true ``` -The first three lines contain a minimal definition of a vCluster plugin - a container name based on the key (second line) and container image (third line). The last three lines then contain extra values that the plugin will apply to the vCluster chart. These are needed for this particular plugin and are not mandatory otherwise. Our plugin would be syncing some ConfigMaps that would also be synced by the built-in "configmaps" syncer of the vCluster, and to avoid conflicting updates we will disable the built-in syncer by passing an additional command-line argument to the syncer container. - ### Deploy the plugin -You can deploy your plugin to a vCluster using the same commands as [described on the overview page](./plugins-overview.mdx#loading-and-installing-plugins-to-vcluster), for example, with the vCluster CLI. +You can deploy your plugin to a vCluster using the same commands as [described on the overview page](./overview.mdx#loading-and-installing-plugins-to-vcluster), for example, with the vCluster CLI. ``` vcluster create my-vcluster -n my-vcluster -f plugin.yaml ``` @@ -271,10 +287,6 @@ vcluster create my-vcluster -n my-vcluster -f plugin.yaml When developing your plugin we recommend using the [devspace](https://devspace.sh/) CLI tool for running your local plugin source code directly in Kubernetes. The appropriate configuration is already present in the `devspace.yaml` and you can start developing by running the following command: -:::info -If you want to develop within a remote Kubernetes cluster (as opposed to docker-desktop or minikube), make sure to exchange `PLUGIN_IMAGE` in the `devspace.yaml` with a valid registry path you can push to. -::: - After successfully setting up the tools, start the development environment with: ``` devspace dev -n vcluster @@ -282,16 +294,7 @@ devspace dev -n vcluster After a while a terminal should show up with additional instructions. Enter the following command to start the plugin: ``` -go run -mod vendor main.go -``` - -The output should look something like this: -``` -I0124 11:20:14.702799 4185 logr.go:249] plugin: Try creating context... -I0124 11:20:14.730044 4185 logr.go:249] plugin: Waiting for vcluster to become leader... -I0124 11:20:14.731097 4185 logr.go:249] plugin: Starting syncers... -[...] -I0124 11:20:15.957331 4185 logr.go:249] plugin: Successfully started plugin. +go build -mod vendor -o plugin main.go && /vcluster/syncer start ``` You can now change a file locally in your IDE and then restart the command in the terminal to apply the changes to the plugin. diff --git a/docs/pages/advanced-topics/plugins-overview.mdx b/docs/pages/advanced-topics/plugins-overview.mdx index 2bbde94eb6..bebb155440 100644 --- a/docs/pages/advanced-topics/plugins-overview.mdx +++ b/docs/pages/advanced-topics/plugins-overview.mdx @@ -22,10 +22,10 @@ In order to better understand how vCluster plugins work, it is recommended to re ## Architecture -Each plugin will run as a sidecar container inside the vCluster pod. +Each plugin will run as a binary inside the vCluster syncer container. It is by default copied over from an init container. This is done to allow easier communication between vCluster and the plugins as well as provide certain capabilities such as high-availability out of the box. -The plugin itself will contact the vCluster pod during startup to obtain the access credentials to the virtual and host cluster. -The plugin controllers are started with these credentials, similar to how vCluster itself starts its resource syncers. +The plugin itself uses [go-plugin](https://github.com/hashicorp/go-plugin), which is used by many popular open-source projects to extend their functionality. The vCluster syncer container will call the plugin binary during startup and pass all relevant information to the plugin. +The plugin controllers are then started with these credentials, similar to how vCluster itself starts its resource syncers. ### Plugin Controllers @@ -45,9 +45,9 @@ For this use case you can label resources vCluster should ignore either on the p ### Plugin Hooks -Plugin hooks are a great feature to adjust the current syncing behaviour of vCluster without the need to override an already existing syncer in vCluster completely. +Plugin hooks are a great feature to adjust current syncing behaviour of vCluster without the need to override an already existing syncer in vCluster completely. They allow you to change outgoing objects of vCluster similar to an mutating admission controller in Kubernetes. -The requirement for a hook to work correctly is that vCluster itself would sync the resource, so hooks only work for the core resources that are synced by vCluster such as pods, services, secrets etc. +Requirement for an hook to work correctly is that vCluster itself would sync the resource, so hooks only work for the core resources that are synced by vCluster such as pods, services, secrets etc. If a plugin registers a hook to a specific resource, vCluster will forward all requests that match the plugin's defined hooks to the plugin and the plugin can then adjust or even deny the request completely. This opens up a wide variety of adjustment possibilities for plugins, where you for example only want to add a custom label or annotation. @@ -59,7 +59,7 @@ If you want to start developing your own vCluster plugins, it is recommended tha ::: vCluster provides an [SDK](https://github.com/loft-sh/vcluster-sdk) for writing plugin controllers that abstracts a lot of the syncer complexity away from the user, but still gives you access to the underlying structures if you need it. -Internally, the vCluster SDK uses the popular [controller-runtime](https://github.com/kubernetes-sigs/controller-runtime) project, which is used by vCluster itself to create the controllers. +Internally, the vCluster SDK uses the popular [controller-runtime](https://github.com/kubernetes-sigs/controller-runtime) project, that is used by vCluster itself to create the controllers. The vCluster SDK lets you write custom plugin controllers with just a few lines of code. Since the plugin SDK interfaces are mostly compatible with the vCluster syncers, you can also take a look at how those are implemented in [the vCluster itself](https://github.com/loft-sh/vcluster/tree/main/pkg/controllers/resources), which work in most cases the same way as if those would be implemented in a plugin. @@ -75,6 +75,8 @@ If you develop a plugin of your own, we recommend creating a `plugin.yaml` (the # with the other vCluster values during vCluster create or helm install. plugin: myPlugin: + # Version v2 is required to use the new plugin api + version: v2 image: plugin-image # Other optional sidecar values below # command: ... @@ -91,7 +93,7 @@ plugin: # ... ``` -The `plugin.yaml` is a valid helm values file used to define the plugin's sidecar configuration and additional RBAC rules needed to function properly. If you want to distribute that plugin to others, it's also possible to install a plugin through a URL: +The `plugin.yaml` is a valid helm values file used to define the plugin's init container configuration and additional RBAC rules needed to function properly. If you want to distribute that plugin for others, it's also possible to install a plugin through an URL: ``` # Install a plugin with a local plugin.yaml diff --git a/docs/pages/plugins/overview.mdx b/docs/pages/plugins/overview.mdx deleted file mode 100644 index bebb155440..0000000000 --- a/docs/pages/plugins/overview.mdx +++ /dev/null @@ -1,115 +0,0 @@ ---- -title: Overview -sidebar_label: Plugins Overview ---- - -Plugins are a feature to extend the capabilities of vCluster. They allow you to add custom functionality, such as: - -1. Syncing specific resources from or to the virtual clusters, including cluster scoped resources like cluster roles -2. Syncing custom resources from or to the virtual cluster -3. Deploying resources on virtual cluster startup, such as CRDs, applications, etc. -4. Manage resources and applications inside the host or virtual cluster -5. Enforcing certain restrictions on synced resources or extending the existing syncers of vCluster -6. Any other operator use case that could benefit from having access to the virtual cluster and the host cluster simultaneously. - -A plugin in its purest form is a [Kubernetes operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) that will have access to both the virtual cluster and the host cluster simultaneously. -This is the main difference between a vCluster plugin and a regular Kubernetes operator that you would just install inside the vCluster itself. -Given this dual access, the plugin is able to translate resources between both clusters, which is the basic building block of [how vCluster works](../what-are-virtual-clusters.mdx). - -:::tip Recommended Reads -In order to better understand how vCluster plugins work, it is recommended to read about Kubernetes [operators](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) as well as [controllers](https://kubernetes.io/docs/concepts/architecture/controller/). -::: - -## Architecture - -Each plugin will run as a binary inside the vCluster syncer container. It is by default copied over from an init container. -This is done to allow easier communication between vCluster and the plugins as well as provide certain capabilities such as high-availability out of the box. -The plugin itself uses [go-plugin](https://github.com/hashicorp/go-plugin), which is used by many popular open-source projects to extend their functionality. The vCluster syncer container will call the plugin binary during startup and pass all relevant information to the plugin. -The plugin controllers are then started with these credentials, similar to how vCluster itself starts its resource syncers. - -### Plugin Controllers - -Resource syncing is the heart of vCluster which enables the virtual cluster to behave like an actual Kubernetes cluster. -A [Kubernetes controller](https://kubernetes.io/docs/concepts/architecture/controller/) that is responsible for resource syncing in vCluster is called a syncer. -This controller reacts on changes to objects within the virtual cluster and on changes to objects within the host cluster. -The syncer tries to map each virtual object to a physical object in the host cluster and then compares those. -After it discovers a change, the syncer ensures that the virtual cluster object and the physical cluster object are aligned in the desired state, -and if not, the syncer changes either one of those objects to reflect the desired state. - -Each plugin can define several of those resource syncers that would work exactly like the built-in syncers of vCluster. -However, you'll not need to sync every Kubernetes resource to the host cluster, as some can stay purely virtual. -Only resources that influence the workloads need to be synced, for example, pods, services, and endpoints, while others such as deployments, replicasets, namespaces etc. are only relevant to the Kubernetes control plane and hence are not needed in the host cluster. - -There are sometimes also cases where you want to manage specific core resources yourself without interfering with what vCluster is syncing, for example special secrets or configmaps that were created from the host cluster or a different resource inside the host cluster. -For this use case you can label resources vCluster should ignore either on the physical or virtual side with a label `vcluster.loft.sh/controlled-by` and a custom value of your choosing. This will tell vCluster to ignore the resource in its syncers. - -### Plugin Hooks - -Plugin hooks are a great feature to adjust current syncing behaviour of vCluster without the need to override an already existing syncer in vCluster completely. -They allow you to change outgoing objects of vCluster similar to an mutating admission controller in Kubernetes. -Requirement for an hook to work correctly is that vCluster itself would sync the resource, so hooks only work for the core resources that are synced by vCluster such as pods, services, secrets etc. - -If a plugin registers a hook to a specific resource, vCluster will forward all requests that match the plugin's defined hooks to the plugin and the plugin can then adjust or even deny the request completely. -This opens up a wide variety of adjustment possibilities for plugins, where you for example only want to add a custom label or annotation. - -### Plugin SDK - -:::tip Recommended Reads -If you want to start developing your own vCluster plugins, it is recommended that you read about [controller-runtime](https://github.com/kubernetes-sigs/controller-runtime) as well as [kube builder](https://book.kubebuilder.io/introduction.html) that uses the controller runtime internally. -::: - -vCluster provides an [SDK](https://github.com/loft-sh/vcluster-sdk) for writing plugin controllers that abstracts a lot of the syncer complexity away from the user, but still gives you access to the underlying structures if you need it. -Internally, the vCluster SDK uses the popular [controller-runtime](https://github.com/kubernetes-sigs/controller-runtime) project, that is used by vCluster itself to create the controllers. -The vCluster SDK lets you write custom plugin controllers with just a few lines of code. - -Since the plugin SDK interfaces are mostly compatible with the vCluster syncers, you can also take a look at how those are implemented in [the vCluster itself](https://github.com/loft-sh/vcluster/tree/main/pkg/controllers/resources), which work in most cases the same way as if those would be implemented in a plugin. -It would be even possible to reimplement all vCluster syncers in a separate plugin. - -## Loading and Installing Plugins to vCluster - -Since the most common distribution method of vCluster is helm, plugins are also configured via helm values. -If you develop a plugin of your own, we recommend creating a `plugin.yaml` (the name has no special functionality, you could also name it `my-plugin.yaml` or `extra-values.yaml`) in the following format: - -``` -# Plugin Definition below. This is essentially a valid helm values file that will be merged -# with the other vCluster values during vCluster create or helm install. -plugin: - myPlugin: - # Version v2 is required to use the new plugin api - version: v2 - image: plugin-image - # Other optional sidecar values below - # command: ... - # env: ... - # Configure Extra RBAC Rules like this - #rbac: - # role: - # extraRules: - # - apiGroups: ["example.loft.sh"] - # ... - # clusterRole: - # extraRules: - # - apiGroups: ["apiextensions.k8s.io"] - # ... -``` - -The `plugin.yaml` is a valid helm values file used to define the plugin's init container configuration and additional RBAC rules needed to function properly. If you want to distribute that plugin for others, it's also possible to install a plugin through an URL: - -``` -# Install a plugin with a local plugin.yaml -vcluster create my-vcluster -n my-vcluster -f plugin.yaml -f other-values.yaml - -# Install a plugin with a remote URL -vcluster create my-vcluster -n my-vcluster -f https://github.com/my-org/my-plugin/plugin.yaml -f other-values.yaml - -# Install a plugin with helm with a remote URL -helm install my-vcluster vcluster -n my-vcluster --repo https://charts.loft.sh -f https://github.com/my-org/my-plugin/plugin.yaml -f other-values.yaml -``` - -:::info Examples -You can take a look at the [vcluster-sdk repo](https://github.com/loft-sh/vcluster-sdk/tree/main/examples) for some working examples. -::: - -:::warning Don't install untrusted plugins -A plugin runs with the same permissions as vCluster itself does in the Kubernetes cluster and can also define additional permissions through its `plugin.yaml`, so make sure you only install plugins you trust. -::: diff --git a/docs/pages/plugins/tutorial.mdx b/docs/pages/plugins/tutorial.mdx deleted file mode 100644 index a108739a1f..0000000000 --- a/docs/pages/plugins/tutorial.mdx +++ /dev/null @@ -1,315 +0,0 @@ ---- -title: "Development tutorial" -sidebar_label: "Development tutorial" ---- - -In this tutorial we will implement a Car syncer. Here we will have a step-by-step look at a plugin implementation that will synchronize all custom car objects using the [vCluster plugin SDK](https://github.com/loft-sh/vcluster-sdk). - -:::info -You can find other examples in the [vCluster SDK Repository](https://github.com/loft-sh/vcluster-sdk/tree/main/examples) -::: - -### Prerequisites - -Before starting to develop, make sure you have installed the following tools on your computer: -- [docker](https://docs.docker.com/) -- [kubectl](https://kubernetes.io/docs/tasks/tools/) with a valid kube context configured -- [helm](https://helm.sh/docs/intro/install/), which is used to deploy vCluster and the plugin -- [vcluster CLI](https://www.vcluster.com/docs/getting-started/setup) v0.9.1 or higher -- [Go](https://go.dev/dl/) programming language build tools - -## Implementation - -Check out the vCluster plugin example via: -``` -git clone https://github.com/loft-sh/vcluster-plugin-example.git -``` - -You'll see a bunch of files already created, but lets take a look at the `main.go` file: -``` -package main - -import ( - "github.com/loft-sh/vcluster-plugin-example/syncers" - "github.com/loft-sh/vcluster-sdk/plugin" -) - -func main() { - ctx := plugin.MustInit() - plugin.MustRegister(syncers.NewCarSyncer(ctx)) - plugin.MustStart() -} -``` - -Let's break down what is happening in the `main()` function above. - -`ctx := plugin.MustInit()` - SDK will init the plugin and retrieve configuration from the vCluster syncer. The returned struct contains information about vCluster flags, namespace, vCluster client config, controller manager objects, etc. - -`plugin.MustRegister(syncers.NewCarSyncer(ctx))` - we will implement the `NewCarSyncer` function below, but for now, all we need to know is that it should return a struct that implements an interface which is accepted by the `MustRegister` function. - -`plugin.MustStart()` - this blocking function will wait until the vCluster pod where this plugin is running becomes the leader. - -### Implementing a syncer for a namespaced resource - -In this chapter, we take a look at the `car.go` file that can be found in the `syncer` directory. - -``` -package syncers - -import ( - "context" - "os" - - examplev1 "github.com/loft-sh/vcluster-plugin-example/apis/v1" - synccontext "github.com/loft-sh/vcluster/pkg/controllers/syncer/context" - "github.com/loft-sh/vcluster/pkg/controllers/syncer/translator" - "github.com/loft-sh/vcluster/pkg/scheme" - synctypes "github.com/loft-sh/vcluster/pkg/types" - "github.com/loft-sh/vcluster/pkg/util" - "github.com/loft-sh/vcluster/pkg/util/translate" - "k8s.io/apimachinery/pkg/api/equality" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -func init() { - // Make sure our scheme is registered - _ = examplev1.AddToScheme(scheme.Scheme) -} - -func NewCarSyncer(ctx *synccontext.RegisterContext) synctypes.Base { - return &carSyncer{ - NamespacedTranslator: translator.NewNamespacedTranslator(ctx, "car", &examplev1.Car{}), - } -} - -type carSyncer struct { - translator.NamespacedTranslator -} -``` - -After an import block, we see the `NewCarSyncer` function, which is being called from the `main.go`. It returns a new instance of the `carSyncer` struct, which is defined by a single nested anonymous struct of type `NamespacedTranslator`. The `NamespacedTranslator` implements many functions of the [`Syncer`](https://pkg.go.dev/github.com/loft-sh/vcluster-sdk/syncer#Syncer) interface for us, and we will implement the remaining ones - `SyncDown` and `Sync`. - -The `SyncDown` function mentioned above is called by the vCluster SDK when a given resource, e.g. a Car, is created in the vCluster, but it doesn't exist in the host cluster yet. To create a ConfigMap in the host cluster we will call the `SyncToHostCreate` function with the output of the `translate` function as third parameter. This demonstrates a typical pattern used in the vCluster syncer implementations. - -``` -func (s *carSyncer) SyncToHost(ctx *synccontext.SyncContext, vObj client.Object) (ctrl.Result, error) { - return s.SyncToHostCreate(ctx, vObj, s.TranslateMetadata(ctx.Context, vObj).(*examplev1.Car)) -} - -func (s *carSyncer) Sync(ctx *synccontext.SyncContext, pObj client.Object, vObj client.Object) (ctrl.Result, error) { - return s.SyncToHostUpdate(ctx, vObj, s.translateUpdate(ctx.Context, pObj.(*examplev1.Car), vObj.(*examplev1.Car))) -} -``` -The `TranslateMetadata` function used above produces a Car object that will be created in the host cluster. It is a deep copy of the Car from vCluster, but with certain metadata modifications - the name and labels are transformed, some vCluster labels and annotations are added, many metadata fields are stripped (uid, resourceVersion, etc.). - - -Next, we need to implement code that will handle the updates of the Car. When a CAr in vCluster or host cluster is updated, the vCluster SDK will call the `Sync` function of the syncer. Current Car resource from the host cluster and from vCluster are passed as the second and third parameters respectively. In the implementation below, you can see another pattern used by the vCluster syncers. The `translateUpdate` function will return nil when no change to the Car in the host cluster is needed, and the `SyncToHostUpdate` function will not do an unnecessary update API call in such case. - -``` - -func (s *carSyncer) Sync(ctx *synccontext.SyncContext, pObj client.Object, vObj client.Object) (ctrl.Result, error) { - return s.SyncToHostUpdate(ctx, vObj, s.translateUpdate(ctx.Context, pObj.(*examplev1.Car), vObj.(*examplev1.Car))) -} - -func (s *carSyncer) translateUpdate(ctx context.Context, pObj, vObj *examplev1.Car) *examplev1.Car { - var updated *examplev1.Car - - // check annotations & labels - changed, updatedAnnotations, updatedLabels := s.TranslateMetadataUpdate(ctx, vObj, pObj) - if changed { - updated = translator.NewIfNil(updated, pObj) - updated.Labels = updatedLabels - updated.Annotations = updatedAnnotations - } - - // check spec - if !equality.Semantic.DeepEqual(vObj.Spec, pObj.Spec) { - updated = translator.NewIfNil(updated, pObj) - updated.Spec = vObj.Spec - } - - return updated -} -``` - -Here we propagate the changes only down to the Car in the host cluster, but there are resources or use cases where a syncer would update the synced resource in vCluster. For example, this might be an update of the status subresource or synchronization of any other field that some controller sets on the host side, e.g., finalizers. Implementation of such updates needs to be considered on case-by-case basis. -For some use cases, you may need to sync the resources in the opposite direction, from the host cluster up into the vCluster, or even in both directions. If that is what your plugin needs to do, you will implement the `UpSyncer` interface defined by the SDK. - -### Adding a hook for changing a resource on the fly - -Hooks are a great feature to adjust current syncing behaviour of vCluster without the need to override an already existing syncer in vCluster completely. They allow you to change outgoing objects of vCluster similar to an mutating admission controller in Kubernetes. Requirement for an hook to work correctly is that vCluster itself would sync the resource, so hooks only work for the core resources that are synced by vCluster such as pods, services, secrets etc. - -To add a hook to your plugin, you simply need to create a new struct that implements the `ClientHook` interface: - -``` -package hooks - -import ( - "context" - "fmt" - - "github.com/loft-sh/vcluster-sdk/plugin" - corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -func NewPodHook() plugin.ClientHook { - return &podHook{} -} - -type podHook struct{} - -func (p *podHook) Name() string { - return "pod-hook" -} - -func (p *podHook) Resource() client.Object { - return &corev1.Pod{} -} -``` - -The `Name()` function defines the name of the hook which is used for logging purposes. The `Resource()` function returns the object you want to mutate. Besides those functions you can now define what actions you want to hook into inside vCluster's syncer: -``` -type MutateCreateVirtual interface { - MutateCreateVirtual(ctx context.Context, obj client.Object) (client.Object, error) -} - -type MutateUpdateVirtual interface { - MutateUpdateVirtual(ctx context.Context, obj client.Object) (client.Object, error) -} - -type MutateDeleteVirtual interface { - MutateDeleteVirtual(ctx context.Context, obj client.Object) (client.Object, error) -} - -type MutateGetVirtual interface { - MutateGetVirtual(ctx context.Context, obj client.Object) (client.Object, error) -} - -type MutateCreatePhysical interface { - MutateCreatePhysical(ctx context.Context, obj client.Object) (client.Object, error) -} - -type MutateUpdatePhysical interface { - MutateUpdatePhysical(ctx context.Context, obj client.Object) (client.Object, error) -} - -type MutateDeletePhysical interface { - MutateDeletePhysical(ctx context.Context, obj client.Object) (client.Object, error) -} - -type MutateGetPhysical interface { - MutateGetPhysical(ctx context.Context, obj client.Object) (client.Object, error) -} -``` - -By implementing one or more of the above interfaces you will receive events from vCluster that allows you to mutate an outgoing or incoming object to vCluster. -For example, to add an hook that adds a custom label to a pod, you can add the following code: -``` -var _ plugin.MutateCreatePhysical = &podHook{} - -func (p *podHook) MutateCreatePhysical(ctx context.Context, obj client.Object) (client.Object, error) { - pod, ok := obj.(*corev1.Pod) - if !ok { - return nil, fmt.Errorf("object %v is not a pod", obj) - } - - if pod.Labels == nil { - pod.Labels = map[string]string{} - } - pod.Labels["created-by-plugin"] = "pod-hook" - return pod, nil -} - -var _ plugin.MutateUpdatePhysical = &podHook{} - -func (p *podHook) MutateUpdatePhysical(ctx context.Context, obj client.Object) (client.Object, error) { - pod, ok := obj.(*corev1.Pod) - if !ok { - return nil, fmt.Errorf("object %v is not a pod", obj) - } - - if pod.Labels == nil { - pod.Labels = map[string]string{} - } - pod.Labels["created-by-plugin"] = "pod-hook" - return pod, nil -} -``` - -Incoming objects into vCluster can be modified through the `MutateGetPhysical` or `MutateGetVirtual` which allows you to change how vCluster is retrieving objects from either the virtual or physical cluster. -This can be useful if you don't want vCluster to change something you have mutated back for example. - -### Build and push your plugin - -Now you can run docker commands to build your container image and push it to the registry. -`docker build -t your_org/vcluster-plugin-example . && docker push your_org/vcluster-plugin-example` - -### Add plugin.yaml - -The last step before installing your plugin is creating a yaml file with your plugin metadata. This file follows the format of the Helm values files. It will be merged with other values files when a vCluster is installed or upgraded. For the plugin we just implemented and built it would look like this: - -``` -# Plugin Definition below. This is essentially a valid helm values file that will be merged -# with the other vcluster values during vcluster create or helm install. -plugin: - vcluster-plugin-example: - version: v2 - image: ghcr.io/loft-sh/vcluster-plugin-example:v1 - rbac: - role: - extraRules: - - apiGroups: ["example.loft.sh"] - resources: ["cars"] - verbs: ["create", "delete", "patch", "update", "get", "list", "watch"] - clusterRole: - extraRules: - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["get", "list", "watch"] - -# Make sure the cluster role is enabled or otherwise the plugin won't be able to watch custom -# resource definitions. -rbac: - clusterRole: - create: true -``` - -### Deploy the plugin - -You can deploy your plugin to a vCluster using the same commands as [described on the overview page](./overview.mdx#loading-and-installing-plugins-to-vcluster), for example, with the vCluster CLI. -``` -vcluster create my-vcluster -n my-vcluster -f plugin.yaml -``` - -### Fast Plugin Development with DevSpace - -When developing your plugin we recommend using the [devspace](https://devspace.sh/) CLI tool for running your local plugin source code directly in Kubernetes. The appropriate configuration is already present in the `devspace.yaml` and you can start developing by running the following command: - -After successfully setting up the tools, start the development environment with: -``` -devspace dev -n vcluster -``` - -After a while a terminal should show up with additional instructions. Enter the following command to start the plugin: -``` -go build -mod vendor -o plugin main.go && /vcluster/syncer start -``` - -You can now change a file locally in your IDE and then restart the command in the terminal to apply the changes to the plugin. - -DevSpace will create a development vCluster which will execute your plugin. Any changes made within the vCluster created by DevSpace will execute against your plugin. -``` -vcluster list - - NAME NAMESPACE STATUS CONNECTED CREATED AGE - vcluster vcluster Running True 2022-09-06 20:33:20 +1000 AEST 2h26m8s -``` - -After you are done developing or you want to recreate the environment, delete the development environment with: -``` -devspace purge -n vcluster -``` - - diff --git a/docs/sidebars.js b/docs/sidebars.js index 3178dbdf7d..303b5ad19e 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -122,12 +122,6 @@ module.exports = { value: `Generic Resource Patches
p
l
`, defaultStyle: true, }, - { - type: "category", - label: "Plugins", - collapsed: true, - items: ["plugins/overview", "plugins/tutorial"], - }, ], }, { diff --git a/hack/assets/main.go b/hack/assets/main.go index 135f0fd6b7..dccd51a424 100644 --- a/hack/assets/main.go +++ b/hack/assets/main.go @@ -7,7 +7,7 @@ import ( "github.com/loft-sh/vcluster/pkg/constants" "github.com/loft-sh/vcluster/pkg/coredns" - "github.com/loft-sh/vcluster/pkg/setup/options" + "github.com/loft-sh/vcluster/pkg/options" "github.com/loft-sh/vcluster-values/values" ) diff --git a/pkg/constants/controllers.go b/pkg/constants/controllers.go new file mode 100644 index 0000000000..9e4c205364 --- /dev/null +++ b/pkg/constants/controllers.go @@ -0,0 +1,50 @@ +package constants + +import "k8s.io/apimachinery/pkg/util/sets" + +var ExistingControllers = sets.New( + "services", + "configmaps", + "secrets", + "endpoints", + "pods", + "events", + "fake-nodes", + "fake-persistentvolumes", + "persistentvolumeclaims", + "ingresses", + "ingressclasses", + "nodes", + "persistentvolumes", + "storageclasses", + "hoststorageclasses", + "priorityclasses", + "networkpolicies", + "volumesnapshots", + "poddisruptionbudgets", + "serviceaccounts", + "csinodes", + "csidrivers", + "csistoragecapacities", + "namespaces", +) + +var DefaultEnabledControllers = sets.New( + // helm charts need to be updated when changing this! + // values.yaml and template/_helpers.tpl reference these + "services", + "configmaps", + "secrets", + "endpoints", + "pods", + "events", + "persistentvolumeclaims", + "fake-nodes", + "fake-persistentvolumes", +) + +var SchedulerRequiredControllers = sets.New( + "csinodes", + "csidrivers", + "csistoragecapacities", +) diff --git a/pkg/controllers/generic/export_syncer.go b/pkg/controllers/generic/export_syncer.go index cb167730fa..cf7bc5b40e 100644 --- a/pkg/controllers/generic/export_syncer.go +++ b/pkg/controllers/generic/export_syncer.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/loft-sh/vcluster/pkg/setup/options" + "github.com/loft-sh/vcluster/pkg/options" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" diff --git a/pkg/controllers/generic/import_syncer.go b/pkg/controllers/generic/import_syncer.go index e20de63734..6f365889ed 100644 --- a/pkg/controllers/generic/import_syncer.go +++ b/pkg/controllers/generic/import_syncer.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/loft-sh/vcluster/pkg/setup/options" + "github.com/loft-sh/vcluster/pkg/options" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" diff --git a/pkg/controllers/k8sdefaultendpoint/k8sdefaultendpoint.go b/pkg/controllers/k8sdefaultendpoint/k8sdefaultendpoint.go index 4c8f0df092..23df1281c5 100644 --- a/pkg/controllers/k8sdefaultendpoint/k8sdefaultendpoint.go +++ b/pkg/controllers/k8sdefaultendpoint/k8sdefaultendpoint.go @@ -6,7 +6,7 @@ import ( "time" "github.com/loft-sh/vcluster/pkg/constants" - "github.com/loft-sh/vcluster/pkg/setup/options" + "github.com/loft-sh/vcluster/pkg/options" "sigs.k8s.io/controller-runtime/pkg/controller" "github.com/loft-sh/vcluster/pkg/specialservices" diff --git a/pkg/controllers/k8sdefaultendpoint/register.go b/pkg/controllers/k8sdefaultendpoint/register.go index 559ff4992f..d9e1bb2248 100644 --- a/pkg/controllers/k8sdefaultendpoint/register.go +++ b/pkg/controllers/k8sdefaultendpoint/register.go @@ -1,7 +1,7 @@ package k8sdefaultendpoint import ( - "github.com/loft-sh/vcluster/pkg/setup/options" + "github.com/loft-sh/vcluster/pkg/options" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/discovery" "k8s.io/klog/v2" diff --git a/pkg/controllers/register.go b/pkg/controllers/register.go index 446c6e087d..c183cddfae 100644 --- a/pkg/controllers/register.go +++ b/pkg/controllers/register.go @@ -9,7 +9,7 @@ import ( "time" "github.com/loft-sh/vcluster/cmd/vclusterctl/cmd" - "github.com/loft-sh/vcluster/pkg/setup/options" + "github.com/loft-sh/vcluster/pkg/options" "github.com/loft-sh/vcluster/pkg/util/kubeconfig" "github.com/loft-sh/vcluster/pkg/util/translate" "k8s.io/apimachinery/pkg/util/wait" diff --git a/pkg/controllers/syncer/context/context.go b/pkg/controllers/syncer/context/context.go index 0fa6e0e981..58749c70a3 100644 --- a/pkg/controllers/syncer/context/context.go +++ b/pkg/controllers/syncer/context/context.go @@ -3,7 +3,7 @@ package context import ( "context" - "github.com/loft-sh/vcluster/pkg/setup/options" + "github.com/loft-sh/vcluster/pkg/options" "github.com/loft-sh/vcluster/pkg/util/loghelper" "k8s.io/apimachinery/pkg/util/sets" ctrl "sigs.k8s.io/controller-runtime" diff --git a/pkg/controllers/syncer/testing/context.go b/pkg/controllers/syncer/testing/context.go index 3373033c7d..5c626ec1fa 100644 --- a/pkg/controllers/syncer/testing/context.go +++ b/pkg/controllers/syncer/testing/context.go @@ -4,7 +4,8 @@ import ( "context" "testing" - "github.com/loft-sh/vcluster/pkg/setup/options" + "github.com/loft-sh/vcluster/pkg/constants" + "github.com/loft-sh/vcluster/pkg/options" "github.com/loft-sh/vcluster/pkg/util/translate" "github.com/loft-sh/vcluster/pkg/util/log" @@ -53,7 +54,7 @@ func NewFakeRegisterContext(pClient *testingutil.FakeIndexClient, vClient *testi ServiceName: DefaultTestVclusterServiceName, TargetNamespace: DefaultTestTargetNamespace, }, - Controllers: options.ExistingControllers.Clone(), + Controllers: constants.ExistingControllers.Clone(), CurrentNamespace: DefaultTestCurrentNamespace, CurrentNamespaceClient: pClient, VirtualManager: newFakeManager(vClient), diff --git a/pkg/leaderelection/leaderelection.go b/pkg/leaderelection/leaderelection.go index d44f2f7eae..2024f132c8 100644 --- a/pkg/leaderelection/leaderelection.go +++ b/pkg/leaderelection/leaderelection.go @@ -6,7 +6,7 @@ import ( "os" "time" - "github.com/loft-sh/vcluster/pkg/setup/options" + "github.com/loft-sh/vcluster/pkg/options" "github.com/loft-sh/vcluster/pkg/telemetry" "github.com/loft-sh/vcluster/pkg/util/translate" "github.com/pkg/errors" diff --git a/pkg/metricsapiservice/register.go b/pkg/metricsapiservice/register.go index 43746faf4e..25e3ba9ab8 100644 --- a/pkg/metricsapiservice/register.go +++ b/pkg/metricsapiservice/register.go @@ -5,7 +5,7 @@ import ( "math" "time" - "github.com/loft-sh/vcluster/pkg/setup/options" + "github.com/loft-sh/vcluster/pkg/options" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/setup/options/controller_context.go b/pkg/options/controller_context.go similarity index 100% rename from pkg/setup/options/controller_context.go rename to pkg/options/controller_context.go diff --git a/pkg/setup/options/flags.go b/pkg/options/flags.go similarity index 100% rename from pkg/setup/options/flags.go rename to pkg/options/flags.go diff --git a/pkg/setup/options/options.go b/pkg/options/options.go similarity index 97% rename from pkg/setup/options/options.go rename to pkg/options/options.go index dbe0fb3c09..84f8a6b9d3 100644 --- a/pkg/setup/options/options.go +++ b/pkg/options/options.go @@ -7,6 +7,10 @@ const ( // VirtualClusterOptions holds the cmd flags type VirtualClusterOptions struct { + // PRO Options + ProOptions VirtualClusterProOptions `json:",inline"` + + // OSS Options below Controllers []string `json:"controllers,omitempty"` ServerCaCert string `json:"serverCaCert,omitempty"` diff --git a/pkg/options/pro_options.go b/pkg/options/pro_options.go new file mode 100644 index 0000000000..b1ae4a68fa --- /dev/null +++ b/pkg/options/pro_options.go @@ -0,0 +1,17 @@ +package options + +type VirtualClusterProOptions struct { + RemoteKubeConfig string `json:"remoteKubeConfig,omitempty"` + RemoteNamespace string `json:"remoteNamespace,omitempty"` + RemoteServiceName string `json:"remoteServiceName,omitempty"` + EnforceValidatingHooks []string `json:"enforceValidatingHooks"` + EnforceMutatingHooks []string `json:"enforceMutatingHooks"` + EtcdReplicas int `json:"etcdReplicas,omitempty"` + IntegratedCoredns bool `json:"integratedCoreDNS,omitempty"` + UseCoreDNSPlugin bool `json:"useCoreDNSPlugin,omitempty"` + EtcdEmbedded bool `json:"etcdEmbedded,omitempty"` + MigrateFrom string `json:"migrateFrom,omitempty"` + + NoopSyncer bool `json:"noopSyncer,omitempty"` + SyncKubernetesService bool `json:"synck8sService,omitempty"` +} diff --git a/pkg/plugin/plugin.go b/pkg/plugin/plugin.go index 34de609ed6..b47cd720f1 100644 --- a/pkg/plugin/plugin.go +++ b/pkg/plugin/plugin.go @@ -4,10 +4,10 @@ import ( "context" "fmt" + "github.com/loft-sh/vcluster/pkg/options" plugintypes "github.com/loft-sh/vcluster/pkg/plugin/types" pluginv1 "github.com/loft-sh/vcluster/pkg/plugin/v1" pluginv2 "github.com/loft-sh/vcluster/pkg/plugin/v2" - "github.com/loft-sh/vcluster/pkg/setup/options" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" @@ -79,3 +79,7 @@ func (m *manager) HasClientHooksForType(versionKindType plugintypes.VersionKindT func (m *manager) HasPlugins() bool { return m.legacyManager.HasPlugins() || m.pluginManager.HasPlugins() } + +func (m *manager) SetProFeatures(proFeatures map[string]bool) { + m.pluginManager.ProFeatures = proFeatures +} diff --git a/pkg/plugin/types/types.go b/pkg/plugin/types/types.go index eb638e8fb1..a40f45a2c2 100644 --- a/pkg/plugin/types/types.go +++ b/pkg/plugin/types/types.go @@ -3,7 +3,7 @@ package types import ( "context" - "github.com/loft-sh/vcluster/pkg/setup/options" + "github.com/loft-sh/vcluster/pkg/options" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" @@ -35,6 +35,9 @@ type Manager interface { // HasPlugins returns if there are any plugins to start HasPlugins() bool + + // SetProFeatures is used by vCluster.Pro to signal what pro features are enabled + SetProFeatures(proFeatures map[string]bool) } type VersionKindType struct { diff --git a/pkg/plugin/v1/plugin.go b/pkg/plugin/v1/plugin.go index 8377ab063f..b3467be03d 100644 --- a/pkg/plugin/v1/plugin.go +++ b/pkg/plugin/v1/plugin.go @@ -8,8 +8,8 @@ import ( "sync" "time" + "github.com/loft-sh/vcluster/pkg/options" plugintypes "github.com/loft-sh/vcluster/pkg/plugin/types" - "github.com/loft-sh/vcluster/pkg/setup/options" "github.com/loft-sh/vcluster/pkg/util/kubeconfig" "github.com/loft-sh/vcluster/pkg/util/loghelper" "github.com/loft-sh/vcluster/pkg/util/random" diff --git a/pkg/plugin/v2/config.go b/pkg/plugin/v2/config.go index f5430b4a71..0522b7bb78 100644 --- a/pkg/plugin/v2/config.go +++ b/pkg/plugin/v2/config.go @@ -4,11 +4,18 @@ import "encoding/json" // InitConfig is the config the syncer sends to the plugin type InitConfig struct { - PhysicalClusterConfig []byte `json:"physicalClusterConfig,omitempty"` - SyncerConfig []byte `json:"syncerConfig,omitempty"` - CurrentNamespace string `json:"currentNamespace,omitempty"` - Options []byte `json:"options,omitempty"` - WorkingDir string `json:"workingDir,omitempty"` + Pro InitConfigPro `json:"pro,omitempty"` + PhysicalClusterConfig []byte `json:"physicalClusterConfig,omitempty"` + SyncerConfig []byte `json:"syncerConfig,omitempty"` + CurrentNamespace string `json:"currentNamespace,omitempty"` + Options []byte `json:"options,omitempty"` + WorkingDir string `json:"workingDir,omitempty"` +} + +// InitConfigPro is used to signal the plugin if vCluster.Pro is enabled and what features are allowed +type InitConfigPro struct { + Enabled bool `json:"enabled,omitempty"` + Features map[string]bool `json:"features,omitempty"` } // PluginConfig is the config the plugin sends back to the syncer diff --git a/pkg/plugin/v2/plugin.go b/pkg/plugin/v2/plugin.go index a0b566e241..34d56a4007 100644 --- a/pkg/plugin/v2/plugin.go +++ b/pkg/plugin/v2/plugin.go @@ -13,9 +13,9 @@ import ( "github.com/ghodss/yaml" "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-plugin" + "github.com/loft-sh/vcluster/pkg/options" plugintypes "github.com/loft-sh/vcluster/pkg/plugin/types" "github.com/loft-sh/vcluster/pkg/plugin/v2/pluginv2" - "github.com/loft-sh/vcluster/pkg/setup/options" "github.com/loft-sh/vcluster/pkg/util/kubeconfig" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" @@ -49,6 +49,9 @@ type Manager struct { // ClientHooks that were loaded ClientHooks map[plugintypes.VersionKindType][]*vClusterPlugin + + // ProFeatures are pro features to hand-over to the plugin + ProFeatures map[string]bool } type vClusterPlugin struct { @@ -267,6 +270,10 @@ func (m *Manager) buildInitRequest( // marshal init config initConfig, err := json.Marshal(&InitConfig{ + Pro: InitConfigPro{ + Enabled: m.ProFeatures != nil, + Features: m.ProFeatures, + }, PhysicalClusterConfig: phyisicalConfigBytes, SyncerConfig: syncerConfigBytes, CurrentNamespace: currentNamespace, diff --git a/pkg/plugin/v2/types.go b/pkg/plugin/v2/types.go index 207e534f99..527d8bed01 100644 --- a/pkg/plugin/v2/types.go +++ b/pkg/plugin/v2/types.go @@ -3,8 +3,8 @@ package v2 import ( "context" + "github.com/loft-sh/vcluster/pkg/options" plugintypes "github.com/loft-sh/vcluster/pkg/plugin/types" - "github.com/loft-sh/vcluster/pkg/setup/options" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" diff --git a/pkg/server/cert/syncer.go b/pkg/server/cert/syncer.go index 02636ba872..8c6d97f9af 100644 --- a/pkg/server/cert/syncer.go +++ b/pkg/server/cert/syncer.go @@ -11,7 +11,7 @@ import ( "github.com/loft-sh/vcluster/pkg/constants" "github.com/loft-sh/vcluster/pkg/controllers/resources/nodes/nodeservice" - "github.com/loft-sh/vcluster/pkg/setup/options" + "github.com/loft-sh/vcluster/pkg/options" "github.com/loft-sh/vcluster/pkg/util/translate" corev1 "k8s.io/api/core/v1" diff --git a/pkg/server/server.go b/pkg/server/server.go index 2ac88f1356..36b0cadb22 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -17,11 +17,11 @@ import ( "github.com/loft-sh/vcluster/pkg/constants" "github.com/loft-sh/vcluster/pkg/controllers/resources/nodes" "github.com/loft-sh/vcluster/pkg/controllers/resources/nodes/nodeservice" + "github.com/loft-sh/vcluster/pkg/options" "github.com/loft-sh/vcluster/pkg/server/cert" "github.com/loft-sh/vcluster/pkg/server/filters" "github.com/loft-sh/vcluster/pkg/server/handler" servertypes "github.com/loft-sh/vcluster/pkg/server/types" - "github.com/loft-sh/vcluster/pkg/setup/options" "github.com/loft-sh/vcluster/pkg/util/blockingcacheclient" "github.com/loft-sh/vcluster/pkg/util/pluginhookclient" "github.com/loft-sh/vcluster/pkg/util/serverhelper" diff --git a/pkg/setup/controller_context.go b/pkg/setup/controller_context.go index 429858be96..a4da774d5d 100644 --- a/pkg/setup/controller_context.go +++ b/pkg/setup/controller_context.go @@ -7,8 +7,8 @@ import ( "time" "github.com/loft-sh/vcluster/pkg/controllers/resources/nodes" + "github.com/loft-sh/vcluster/pkg/options" "github.com/loft-sh/vcluster/pkg/plugin" - "github.com/loft-sh/vcluster/pkg/setup/options" "github.com/loft-sh/vcluster/pkg/telemetry" "github.com/loft-sh/vcluster/pkg/util/blockingcacheclient" "github.com/loft-sh/vcluster/pkg/util/toleration" @@ -334,7 +334,7 @@ func InitControllerContext( } // parse enabled controllers - controllers, err := options.ParseControllers(vClusterOptions) + controllers, err := ParseControllers(vClusterOptions) if err != nil { return nil, err } @@ -344,7 +344,7 @@ func InitControllerContext( return nil, err } - controllers, err = options.DisableMissingAPIs(localDiscoveryClient, controllers) + controllers, err = DisableMissingAPIs(localDiscoveryClient, controllers) if err != nil { return nil, err } diff --git a/pkg/setup/controllers.go b/pkg/setup/controllers.go index 4bb9684893..59659168ca 100644 --- a/pkg/setup/controllers.go +++ b/pkg/setup/controllers.go @@ -11,8 +11,8 @@ import ( synccontext "github.com/loft-sh/vcluster/pkg/controllers/syncer/context" "github.com/loft-sh/vcluster/pkg/coredns" "github.com/loft-sh/vcluster/pkg/metricsapiservice" + "github.com/loft-sh/vcluster/pkg/options" "github.com/loft-sh/vcluster/pkg/plugin" - "github.com/loft-sh/vcluster/pkg/setup/options" "github.com/loft-sh/vcluster/pkg/specialservices" syncertypes "github.com/loft-sh/vcluster/pkg/types" "github.com/loft-sh/vcluster/pkg/util/kubeconfig" diff --git a/pkg/setup/options/enable_controllers.go b/pkg/setup/enable_controllers.go similarity index 77% rename from pkg/setup/options/enable_controllers.go rename to pkg/setup/enable_controllers.go index e944cb8920..d8b4fc598d 100644 --- a/pkg/setup/options/enable_controllers.go +++ b/pkg/setup/enable_controllers.go @@ -1,62 +1,17 @@ -package options +package setup import ( "fmt" "strings" + "github.com/loft-sh/vcluster/pkg/constants" + "github.com/loft-sh/vcluster/pkg/options" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/discovery" "k8s.io/klog/v2" ) -var ExistingControllers = sets.New( - "services", - "configmaps", - "secrets", - "endpoints", - "pods", - "events", - "fake-nodes", - "fake-persistentvolumes", - "persistentvolumeclaims", - "ingresses", - "ingressclasses", - "nodes", - "persistentvolumes", - "storageclasses", - "hoststorageclasses", - "priorityclasses", - "networkpolicies", - "volumesnapshots", - "poddisruptionbudgets", - "serviceaccounts", - "csinodes", - "csidrivers", - "csistoragecapacities", - "namespaces", -) - -var DefaultEnabledControllers = sets.New( - // helm charts need to be updated when changing this! - // values.yaml and template/_helpers.tpl reference these - "services", - "configmaps", - "secrets", - "endpoints", - "pods", - "events", - "persistentvolumeclaims", - "fake-nodes", - "fake-persistentvolumes", -) - -var schedulerRequiredControllers = sets.New( - "csinodes", - "csidrivers", - "csistoragecapacities", -) - const ( storageV1GroupVersion = "storage.k8s.io/v1" ) @@ -64,11 +19,11 @@ const ( // map from groupversion to list of resources in that groupversion // the syncers will be disabled unless that resource is advertised in that groupversion var possibleMissing = map[string][]string{ - storageV1GroupVersion: schedulerRequiredControllers.UnsortedList(), + storageV1GroupVersion: constants.SchedulerRequiredControllers.UnsortedList(), } -func ParseControllers(options *VirtualClusterOptions) (sets.Set[string], error) { - enabledControllers := DefaultEnabledControllers.Clone() +func ParseControllers(options *options.VirtualClusterOptions) (sets.Set[string], error) { + enabledControllers := constants.DefaultEnabledControllers.Clone() disabledControllers := sets.New[string]() // migrate deprecated flags @@ -101,7 +56,7 @@ func ParseControllers(options *VirtualClusterOptions) (sets.Set[string], error) enabledControllers.Insert(controller) } - if !ExistingControllers.Has(controller) { + if !constants.ExistingControllers.Has(controller) { return nil, fmt.Errorf("unrecognized controller %s, available controllers: %s", controller, availableControllers()) } } @@ -119,9 +74,9 @@ func ParseControllers(options *VirtualClusterOptions) (sets.Set[string], error) // enable additional controllers required for scheduling with storage if options.EnableScheduler && enabledControllers.Has("persistentvolumeclaims") { - klog.Infof("persistentvolumeclaim syncing and scheduler enabled, enabling required controllers: %q", schedulerRequiredControllers) - enabledControllers = enabledControllers.Union(schedulerRequiredControllers) - requiredButDisabled := disabledControllers.Intersection(schedulerRequiredControllers) + klog.Infof("persistentvolumeclaim syncing and scheduler enabled, enabling required controllers: %q", constants.SchedulerRequiredControllers) + enabledControllers = enabledControllers.Union(constants.SchedulerRequiredControllers) + requiredButDisabled := disabledControllers.Intersection(constants.SchedulerRequiredControllers) if requiredButDisabled.Len() > 0 { klog.Warningf("persistentvolumeclaim syncing and scheduler enabled, but required syncers explicitly disabled: %q. This may result in incorrect pod scheduling.", sets.List(requiredButDisabled)) } @@ -153,7 +108,7 @@ func ParseControllers(options *VirtualClusterOptions) (sets.Set[string], error) } func availableControllers() string { - return strings.Join(sets.List(ExistingControllers), ", ") + return strings.Join(sets.List(constants.ExistingControllers), ", ") } // DisableMissingAPIs checks if the apis are enabled, if any are missing, disable the syncer and print a log diff --git a/pkg/setup/options/enable_controllers_test.go b/pkg/setup/enable_controllers_test.go similarity index 82% rename from pkg/setup/options/enable_controllers_test.go rename to pkg/setup/enable_controllers_test.go index 6cec8adb2e..5a76f6812d 100644 --- a/pkg/setup/options/enable_controllers_test.go +++ b/pkg/setup/enable_controllers_test.go @@ -1,8 +1,10 @@ -package options +package setup import ( "testing" + "github.com/loft-sh/vcluster/pkg/constants" + "github.com/loft-sh/vcluster/pkg/options" "github.com/spf13/pflag" "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" @@ -15,7 +17,7 @@ import ( func TestEnableControllers(t *testing.T) { testTable := []struct { desc string - optsModifier func(*VirtualClusterOptions) + optsModifier func(*options.VirtualClusterOptions) expectEnabled []string expectDisabled []string expectError bool @@ -24,14 +26,14 @@ func TestEnableControllers(t *testing.T) { }{ { desc: "default case", - optsModifier: func(v *VirtualClusterOptions) {}, - expectEnabled: sets.List(DefaultEnabledControllers), - expectDisabled: sets.List(ExistingControllers.Difference(DefaultEnabledControllers)), + optsModifier: func(v *options.VirtualClusterOptions) {}, + expectEnabled: sets.List(constants.DefaultEnabledControllers), + expectDisabled: sets.List(constants.ExistingControllers.Difference(constants.DefaultEnabledControllers)), expectError: false, }, { desc: "scheduler with pvc enabled, nodes not enabled", - optsModifier: func(v *VirtualClusterOptions) { + optsModifier: func(v *options.VirtualClusterOptions) { v.Controllers = []string{"persistentvolumeclaims"} v.EnableScheduler = true }, @@ -39,46 +41,46 @@ func TestEnableControllers(t *testing.T) { }, { desc: "scheduler with pvc enabled, storageclasses not enabled", - optsModifier: func(v *VirtualClusterOptions) { + optsModifier: func(v *options.VirtualClusterOptions) { v.Controllers = []string{"persistentvolumeclaims", "nodes"} v.EnableScheduler = true }, - expectEnabled: append([]string{"hoststorageclasses"}, sets.List(schedulerRequiredControllers)...), + expectEnabled: append([]string{"hoststorageclasses"}, sets.List(constants.SchedulerRequiredControllers)...), expectDisabled: []string{"storageclasses"}, expectError: false, }, { desc: "scheduler with pvc enabled, storageclasses enabled", - optsModifier: func(v *VirtualClusterOptions) { + optsModifier: func(v *options.VirtualClusterOptions) { v.Controllers = []string{"persistentvolumeclaims", "nodes", "storageclasses"} v.EnableScheduler = true }, - expectEnabled: append([]string{"storageclasses"}, sets.List(schedulerRequiredControllers)...), + expectEnabled: append([]string{"storageclasses"}, sets.List(constants.SchedulerRequiredControllers)...), expectDisabled: []string{"hoststorageclasses"}, expectError: false, }, { desc: "scheduler with pvc enabled, hoststorageclasses enabled", - optsModifier: func(v *VirtualClusterOptions) { + optsModifier: func(v *options.VirtualClusterOptions) { v.Controllers = []string{"persistentvolumeclaims", "nodes"} v.EnableScheduler = true }, - expectEnabled: append([]string{"hoststorageclasses"}, sets.List(schedulerRequiredControllers)...), + expectEnabled: append([]string{"hoststorageclasses"}, sets.List(constants.SchedulerRequiredControllers)...), expectDisabled: []string{"storageclasses"}, expectError: false, }, { desc: "scheduler disabled, storageclasses not enabled", - optsModifier: func(v *VirtualClusterOptions) { + optsModifier: func(v *options.VirtualClusterOptions) { v.Controllers = []string{"persistentvolumeclaims"} }, expectEnabled: []string{}, - expectDisabled: append([]string{"storageclasses", "hoststorageclasses"}, sets.List(schedulerRequiredControllers)...), + expectDisabled: append([]string{"storageclasses", "hoststorageclasses"}, sets.List(constants.SchedulerRequiredControllers)...), expectError: false, }, { desc: "storageclasses and hoststorageclasses enabled", - optsModifier: func(v *VirtualClusterOptions) { + optsModifier: func(v *options.VirtualClusterOptions) { v.Controllers = []string{"storageclasses", "hoststorageclasses"} }, expectEnabled: []string{}, @@ -87,7 +89,7 @@ func TestEnableControllers(t *testing.T) { }, { desc: "syncAllNodes true, nodes not enabled", - optsModifier: func(v *VirtualClusterOptions) { + optsModifier: func(v *options.VirtualClusterOptions) { v.Controllers = []string{} v.SyncAllNodes = true }, @@ -98,7 +100,7 @@ func TestEnableControllers(t *testing.T) { }, { desc: "syncAllNodes true, nodes enabled", - optsModifier: func(v *VirtualClusterOptions) { + optsModifier: func(v *options.VirtualClusterOptions) { v.Controllers = []string{"nodes"} v.SyncAllNodes = true }, @@ -112,8 +114,8 @@ func TestEnableControllers(t *testing.T) { if tc.pause { t.Log("you can put a breakpoint here") } - var opts VirtualClusterOptions - AddFlags(pflag.NewFlagSet("test", pflag.PanicOnError), &opts) + var opts options.VirtualClusterOptions + options.AddFlags(pflag.NewFlagSet("test", pflag.PanicOnError), &opts) t.Logf("test case: %q", tc.desc) if tc.optsModifier != nil { tc.optsModifier(&opts) @@ -171,7 +173,7 @@ func TestDisableMissingAPIs(t *testing.T) { { name: "K8s 1.21 or lower", apis: map[string][]metav1.APIResource{}, - expectedNotFound: schedulerRequiredControllers, + expectedNotFound: constants.SchedulerRequiredControllers, expectedFound: sets.New[string](), }, { @@ -202,7 +204,7 @@ func TestDisableMissingAPIs(t *testing.T) { fakeDisoveryClient := &fakeDiscovery.FakeDiscovery{Fake: &clientTesting.Fake{Resources: resourceLists}} // run function - actualControllers, err := DisableMissingAPIs(fakeDisoveryClient, ExistingControllers.Clone()) + actualControllers, err := DisableMissingAPIs(fakeDisoveryClient, constants.ExistingControllers.Clone()) assert.NilError(t, err) // unexpectedly not disabled diff --git a/pkg/setup/initialize.go b/pkg/setup/initialize.go index b68dd691f0..26524937be 100644 --- a/pkg/setup/initialize.go +++ b/pkg/setup/initialize.go @@ -12,7 +12,7 @@ import ( "github.com/loft-sh/vcluster/pkg/k0s" "github.com/loft-sh/vcluster/pkg/k3s" "github.com/loft-sh/vcluster/pkg/k8s" - "github.com/loft-sh/vcluster/pkg/setup/options" + "github.com/loft-sh/vcluster/pkg/options" "github.com/loft-sh/vcluster/pkg/specialservices" "github.com/loft-sh/vcluster/pkg/telemetry" "github.com/loft-sh/vcluster/pkg/util/servicecidr" diff --git a/pkg/setup/proxy.go b/pkg/setup/proxy.go index 6d8f49f8f0..42007f2aed 100644 --- a/pkg/setup/proxy.go +++ b/pkg/setup/proxy.go @@ -1,8 +1,8 @@ package setup import ( + "github.com/loft-sh/vcluster/pkg/options" "github.com/loft-sh/vcluster/pkg/server" - "github.com/loft-sh/vcluster/pkg/setup/options" "k8s.io/klog/v2" ) diff --git a/pkg/telemetry/collect.go b/pkg/telemetry/collect.go index 436d1ebd5d..701602037e 100644 --- a/pkg/telemetry/collect.go +++ b/pkg/telemetry/collect.go @@ -10,7 +10,7 @@ import ( "github.com/loft-sh/analytics-client/client" managementv1 "github.com/loft-sh/api/v3/pkg/apis/management/v1" "github.com/loft-sh/log" - "github.com/loft-sh/vcluster/pkg/setup/options" + "github.com/loft-sh/vcluster/pkg/options" "github.com/loft-sh/vcluster/pkg/upgrade" "github.com/loft-sh/vcluster/pkg/util/clihelper" corev1 "k8s.io/api/core/v1" diff --git a/pkg/telemetry/helpers.go b/pkg/telemetry/helpers.go index ebc1e6b9bc..3555665b70 100644 --- a/pkg/telemetry/helpers.go +++ b/pkg/telemetry/helpers.go @@ -10,7 +10,7 @@ import ( managementv1 "github.com/loft-sh/api/v3/pkg/apis/management/v1" "github.com/loft-sh/log" "github.com/loft-sh/vcluster/pkg/helm" - "github.com/loft-sh/vcluster/pkg/setup/options" + "github.com/loft-sh/vcluster/pkg/options" "github.com/loft-sh/vcluster/pkg/util/cliconfig" "github.com/loft-sh/vcluster/pkg/util/translate" homedir "github.com/mitchellh/go-homedir" diff --git a/pkg/telemetry/noop.go b/pkg/telemetry/noop.go index e749fd926a..84382b5bea 100644 --- a/pkg/telemetry/noop.go +++ b/pkg/telemetry/noop.go @@ -4,7 +4,7 @@ import ( "context" managementv1 "github.com/loft-sh/api/v3/pkg/apis/management/v1" - "github.com/loft-sh/vcluster/pkg/setup/options" + "github.com/loft-sh/vcluster/pkg/options" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ) diff --git a/pkg/util/context/converter.go b/pkg/util/context/converter.go index 507908dc77..9f6922361c 100644 --- a/pkg/util/context/converter.go +++ b/pkg/util/context/converter.go @@ -2,7 +2,7 @@ package util import ( synccontext "github.com/loft-sh/vcluster/pkg/controllers/syncer/context" - "github.com/loft-sh/vcluster/pkg/setup/options" + "github.com/loft-sh/vcluster/pkg/options" ) func ToRegisterContext(ctx *options.ControllerContext) *synccontext.RegisterContext {